1 /**
2  * ae.sys.persistence.keyvalue
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Vladimir Panteleev <ae@cy.md>
12  */
13 
14 module ae.sys.persistence.keyvalue;
15 
16 import std.exception;
17 import std.traits;
18 
19 import ae.sys.persistence.core;
20 import ae.sys.sqlite3;
21 import ae.utils.array : nonNull;
22 import ae.utils.json;
23 
24 // ****************************************************************************
25 
26 /// Persistent indexed key-value store, backed by an SQLite database.
27 /// Non-string keys/values are JSON-encoded.
28 struct KeyValueStore(K, V)
29 {
30 	KeyValueDatabase* db; ///
31 	string tableName; ///
32 
33 	/// Constructor with `KeyValueDatabase` and `tableName`.
34 	/// Allows using the same database file for multiple key/value tables.
35 	this(KeyValueDatabase* db, string tableName = "values")
36 	{
37 		this.db = db;
38 		this.tableName = tableName;
39 	}
40 
41 	/// Constructor with file name.
42 	/// Creates a new `KeyValueDatabase` for private use.
43 	this(string fn)
44 	{
45 		auto db = new KeyValueDatabase(fn);
46 		this(db);
47 	}
48 
49 	/// Implements common D associative array operations.
50 	V opIndex()(auto ref const K k)
51 	{
52 		checkInitialized();
53 		foreach (SqlType!V v; sqlGet.iterate(toSqlType(k)))
54 			return fromSqlType!V(v);
55 		throw new Exception("Value not in KeyValueStore");
56 	}
57 
58 	V get()(auto ref const K k, auto ref V defaultValue)
59 	{
60 		checkInitialized();
61 		foreach (SqlType!V v; sqlGet.iterate(toSqlType(k)))
62 			return fromSqlType!V(v);
63 		return defaultValue;
64 	} /// ditto
65 
66 	V require()(auto ref const K k, lazy V defaultValue)
67 	{
68 		checkInitialized();
69 		foreach (SqlType!V v; sqlGet.iterate(toSqlType(k)))
70 			return fromSqlType!V(v);
71 		auto v = defaultValue();
72 		sqlSet.exec(toSqlType(k), toSqlType(v));
73 		return v;
74 	} /// ditto
75 	deprecated alias getOrAdd = require;
76 
77 	bool opBinaryRight(string op)(auto ref const K k)
78 	if (op == "in")
79 	{
80 		checkInitialized();
81 		foreach (int count; sqlExists.iterate(toSqlType(k)))
82 			return count > 0;
83 		assert(false);
84 	} /// ditto
85 
86 	auto ref const(V) opIndexAssign()(auto ref const V v, auto ref const K k)
87 	{
88 		checkInitialized();
89 		sqlSet.exec(toSqlType(k), toSqlType(v));
90 		return v;
91 	} /// ditto
92 
93 	void remove()(auto ref const K k)
94 	{
95 		checkInitialized();
96 		sqlDelete.exec(toSqlType(k));
97 	} /// ditto
98 
99 	@property int length()
100 	{
101 		checkInitialized();
102 		foreach (int count; sqlLength.iterate())
103 			return count;
104 		assert(false);
105 	} /// ditto
106 
107 	@property K[] keys()
108 	{
109 		checkInitialized();
110 		K[] result;
111 		foreach (SqlType!K key; sqlListKeys.iterate())
112 			result ~= fromSqlType!K(key);
113 		return result;
114 	}
115 
116 	int opApply(int delegate(K key, V value) dg)
117 	{
118 		checkInitialized();
119 		foreach (SqlType!K key, SqlType!V value; sqlListPairs.iterate())
120 		{
121 			auto res = dg(fromSqlType!K(key), fromSqlType!V(value));
122 			if (res)
123 				return res;
124 		}
125 		return 0;
126 	}
127 
128 private:
129 	static SqlType!T toSqlType(T)(auto ref T v)
130 	{
131 		alias S = SqlType!T;
132 		static if (is(T : long)) // long
133 			return v;
134 		else
135 		static if (is(T : const(char)[])) // string
136 			return cast(S) v.nonNull;
137 		else
138 		static if (is(T U : U[]) && !hasIndirections!U) // void[]
139 			return v.nonNull;
140 		else
141 			return toJson(v);
142 	}
143 
144 	static T fromSqlType(T)(SqlType!T v)
145 	{
146 		static if (is(T : long)) // long
147 			return cast(T) v;
148 		else
149 		static if (is(T : const(char)[])) // string
150 			return cast(T) v;
151 		else
152 		static if (is(T U : U[]) && !hasIndirections!U) // void[]
153 			static if (is(T V : V[N], size_t N))
154 			{
155 				assert(v.length == N * V.sizeof, "Static array length mismatch");
156 				return cast(T) v[0 .. N * V.sizeof];
157 			}
158 			else
159 				return cast(T) v;
160 		else
161 			return jsonParse!T(cast(string) v);
162 	}
163 
164 	template SqlType(T)
165 	{
166 		static if (is(T : long))
167 			alias SqlType = long;
168 		else
169 		static if (is(T : const(char)[]))
170 			alias SqlType = string;
171 		else
172 		static if (is(T U : U[]) && !hasIndirections!U)
173 			alias SqlType = const(void)[];
174 		else
175 			alias SqlType = string; // JSON-encoded
176 	}
177 
178 	static assert(is(SqlType!int == long));
179 	static assert(is(SqlType!string == string));
180 
181 	template sqlTypeName(T)
182 	{
183 		alias S = SqlType!T;
184 		static if (is(S == long))
185 			enum sqlTypeName = "INTEGER";
186 		else
187 		static if (is(S == string))
188 			enum sqlTypeName = "TEXT";
189 		else
190 		static if (is(S == void[]))
191 			enum sqlTypeName = "BLOB";
192 		else
193 			enum sqlTypeName = "TEXT"; // JSON
194 	}
195 
196 	bool initialized;
197 
198 	SQLite.PreparedStatement sqlGet, sqlSet, sqlDelete, sqlExists, sqlLength, sqlListKeys, sqlListPairs;
199 
200 	void checkInitialized()
201 	{
202 		if (!initialized)
203 		{
204 			assert(db, "KeyValueStore database not set");
205 			db.exec("CREATE TABLE IF NOT EXISTS [" ~ tableName ~ "] ([key] " ~ sqlTypeName!K ~ " PRIMARY KEY, [value] " ~ sqlTypeName!V ~ ")");
206 			db.exec("PRAGMA SYNCHRONOUS=OFF");
207 			sqlGet = db.prepare("SELECT [value] FROM [" ~ tableName ~ "] WHERE [key]=?");
208 			sqlSet = db.prepare("INSERT OR REPLACE INTO [" ~ tableName ~ "] VALUES (?, ?)");
209 			sqlDelete = db.prepare("DELETE FROM [" ~ tableName ~ "] WHERE [key]=?");
210 			sqlExists = db.prepare("SELECT COUNT(*) FROM [" ~ tableName ~ "] WHERE [key]=? LIMIT 1");
211 			sqlLength = db.prepare("SELECT COUNT(*) FROM [" ~ tableName ~ "]");
212 			sqlListKeys = db.prepare("SELECT [key] FROM [" ~ tableName ~ "]");
213 			sqlListPairs = db.prepare("SELECT [key], [value] FROM [" ~ tableName ~ "]");
214 			initialized = true;
215 		}
216 	}
217 }
218 
219 /// A `KeyValueDatabase` holds one or more key/value tables (`KeyValueStore`).
220 struct KeyValueDatabase
221 {
222 	string fileName; /// Database file name.
223 
224 	SQLite sqlite; /// SQLite database instance. Initialized automatically.
225 
226 	@property SQLite _getSQLite()
227 	{
228 		if (sqlite is null)
229 		{
230 			enforce(fileName, "KeyValueDatabase filename not set");
231 			sqlite = new SQLite(fileName);
232 		}
233 		return sqlite;
234 	}
235 
236 	alias _getSQLite this;
237 }
238 
239 unittest
240 {
241 	import std.file;
242 
243 	string fn = tempDir ~ "/ae-sys-persistence-keyvalue-test.s3db";
244 	if (fn.exists) fn.remove();
245 	scope(success) fn.remove();
246 
247 	auto store = KeyValueStore!(string, string)(fn);
248 
249 	assert(store.length == 0);
250 	assert("key" !in store);
251 	assert(store.get("key", null) is null);
252 	assert(store.keys.length == 0);
253 
254 	store["key"] = "value";
255 
256 	assert(store.length == 1);
257 	assert("key" in store);
258 	assert(store["key"] == "value");
259 	assert(store.get("key", null) == "value");
260 	assert(store.keys == ["key"]);
261 
262 	store["key"] = "value2";
263 
264 	assert(store.length == 1);
265 	assert("key" in store);
266 	assert(store.get("key", null) == "value2");
267 	assert(store.keys == ["key"]);
268 
269 	store["key2"] = "value3";
270 
271 	assert(store.length == 2);
272 	assert("key" in store);
273 	assert("key2" in store);
274 	assert(store.get("key", null) == "value2");
275 	assert(store.get("key2", null) == "value3");
276 	assert(store.keys == ["key", "key2"]);
277 
278 	store.remove("key");
279 
280 	assert(store.length == 1);
281 	assert("key" !in store);
282 	assert("key2" in store);
283 	assert(store.get("key", null) is null);
284 	assert(store.keys == ["key2"]);
285 }
286 
287 unittest
288 {
289 	if (false)
290 	{
291 		KeyValueStore!(string, ubyte[20]) kv;
292 		ubyte[20] s = kv[""];
293 	}
294 }
295 
296 unittest
297 {
298 	if (false)
299 	{
300 		KeyValueStore!(string, float[20]) kv;
301 		float[20] s = kv[""];
302 	}
303 }
304 
305 unittest
306 {
307 	if (false)
308 	{
309 		struct K {}
310 		KeyValueStore!(K, K) kv;
311 		assert(K.init !in kv);
312 		immutable K ik;
313 		assert(ik !in kv);
314 	}
315 }
316 
317 unittest
318 {
319 	import std.file;
320 
321 	string fn = tempDir ~ "/ae-sys-persistence-keyvalue-test.s3db";
322 	if (fn.exists) fn.remove();
323 	scope(success) fn.remove();
324 
325 	KeyValueStore!(float[], float[]) kv;
326 	kv = typeof(kv)(fn);
327 	assert(null !in kv);
328 	kv[null] = null;
329 	assert(null in kv);
330 	assert(kv[null] == null);
331 }