1 /**
2  * ae.sys.persistence.keyvalue
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Vladimir Panteleev <ae@cy.md>
12  */
13 
14 module ae.sys.persistence.keyvalue;
15 
16 import std.exception;
17 import std.traits;
18 
19 import ae.sys.persistence.core;
20 import ae.sys.sqlite3;
21 import ae.utils.array : nonNull;
22 import ae.utils.json;
23 
24 // ****************************************************************************
25 
26 /// Persistent indexed key-value store, backed by an SQLite database.
27 /// Non-string keys/values are JSON-encoded.
28 struct KeyValueStore(K, V)
29 {
30 	KeyValueDatabase* db; ///
31 	string tableName; ///
32 
33 	/// Constructor with `KeyValueDatabase` and `tableName`.
34 	/// Allows using the same database file for multiple key/value tables.
35 	this(KeyValueDatabase* db, string tableName = "values")
36 	{
37 		this.db = db;
38 		this.tableName = tableName;
39 	}
40 
41 	/// Constructor with file name.
42 	/// Creates a new `KeyValueDatabase` for private use.
43 	this(string fn)
44 	{
45 		auto db = new KeyValueDatabase(fn);
46 		this(db);
47 	}
48 
49 	/// Implements common D associative array operations.
50 	V opIndex()(auto ref const K k)
51 	{
52 		checkInitialized();
53 		foreach (SqlType!V v; sqlGet.iterate(toSqlType(k)))
54 			return fromSqlType!V(v);
55 		throw new Exception("Value not in KeyValueStore");
56 	}
57 
58 	V get()(auto ref const K k, auto ref V defaultValue)
59 	{
60 		checkInitialized();
61 		foreach (SqlType!V v; sqlGet.iterate(toSqlType(k)))
62 			return fromSqlType!V(v);
63 		return defaultValue;
64 	} /// ditto
65 
66 	V getOrAdd()(auto ref const K k, lazy V defaultValue)
67 	{
68 		checkInitialized();
69 		foreach (SqlType!V v; sqlGet.iterate(toSqlType(k)))
70 			return fromSqlType!V(v);
71 		auto v = defaultValue();
72 		sqlSet.exec(toSqlType(k), toSqlType(v));
73 		return v;
74 	} /// ditto
75 
76 	bool opBinaryRight(string op)(auto ref const K k)
77 	if (op == "in")
78 	{
79 		checkInitialized();
80 		foreach (int count; sqlExists.iterate(toSqlType(k)))
81 			return count > 0;
82 		assert(false);
83 	} /// ditto
84 
85 	auto ref const(V) opIndexAssign()(auto ref const V v, auto ref const K k)
86 	{
87 		checkInitialized();
88 		sqlSet.exec(toSqlType(k), toSqlType(v));
89 		return v;
90 	} /// ditto
91 
92 	void remove()(auto ref const K k)
93 	{
94 		checkInitialized();
95 		sqlDelete.exec(toSqlType(k));
96 	} /// ditto
97 
98 	@property int length()
99 	{
100 		checkInitialized();
101 		foreach (int count; sqlLength.iterate())
102 			return count;
103 		assert(false);
104 	} /// ditto
105 
106 	@property K[] keys()
107 	{
108 		checkInitialized();
109 		K[] result;
110 		foreach (SqlType!K key; sqlListKeys.iterate())
111 			result ~= fromSqlType!K(key);
112 		return result;
113 	}
114 
115 	int opApply(int delegate(K key, V value) dg)
116 	{
117 		checkInitialized();
118 		foreach (SqlType!K key, SqlType!V value; sqlListPairs.iterate())
119 		{
120 			auto res = dg(fromSqlType!K(key), fromSqlType!V(value));
121 			if (res)
122 				return res;
123 		}
124 		return 0;
125 	}
126 
127 private:
128 	static SqlType!T toSqlType(T)(auto ref T v)
129 	{
130 		alias S = SqlType!T;
131 		static if (is(T : long)) // long
132 			return v;
133 		else
134 		static if (is(T : const(char)[])) // string
135 			return cast(S) v.nonNull;
136 		else
137 		static if (is(T U : U[]) && !hasIndirections!U) // void[]
138 			return v.nonNull;
139 		else
140 			return toJson(v);
141 	}
142 
143 	static T fromSqlType(T)(SqlType!T v)
144 	{
145 		static if (is(T : long)) // long
146 			return cast(T) v;
147 		else
148 		static if (is(T : const(char)[])) // string
149 			return cast(T) v;
150 		else
151 		static if (is(T U : U[]) && !hasIndirections!U) // void[]
152 			static if (is(T V : V[N], size_t N))
153 			{
154 				assert(v.length == N * V.sizeof, "Static array length mismatch");
155 				return cast(T) v[0 .. N * V.sizeof];
156 			}
157 			else
158 				return cast(T) v;
159 		else
160 			return jsonParse!T(cast(string) v);
161 	}
162 
163 	template SqlType(T)
164 	{
165 		static if (is(T : long))
166 			alias SqlType = long;
167 		else
168 		static if (is(T : const(char)[]))
169 			alias SqlType = string;
170 		else
171 		static if (is(T U : U[]) && !hasIndirections!U)
172 			alias SqlType = const(void)[];
173 		else
174 			alias SqlType = string; // JSON-encoded
175 	}
176 
177 	static assert(is(SqlType!int == long));
178 	static assert(is(SqlType!string == string));
179 
180 	template sqlTypeName(T)
181 	{
182 		alias S = SqlType!T;
183 		static if (is(S == long))
184 			enum sqlTypeName = "INTEGER";
185 		else
186 		static if (is(S == string))
187 			enum sqlTypeName = "TEXT";
188 		else
189 		static if (is(S == void[]))
190 			enum sqlTypeName = "BLOB";
191 		else
192 			enum sqlTypeName = "TEXT"; // JSON
193 	}
194 
195 	bool initialized;
196 
197 	SQLite.PreparedStatement sqlGet, sqlSet, sqlDelete, sqlExists, sqlLength, sqlListKeys, sqlListPairs;
198 
199 	void checkInitialized()
200 	{
201 		if (!initialized)
202 		{
203 			assert(db, "KeyValueStore database not set");
204 			db.exec("CREATE TABLE IF NOT EXISTS [" ~ tableName ~ "] ([key] " ~ sqlTypeName!K ~ " PRIMARY KEY, [value] " ~ sqlTypeName!V ~ ")");
205 			db.exec("PRAGMA SYNCHRONOUS=OFF");
206 			sqlGet = db.prepare("SELECT [value] FROM [" ~ tableName ~ "] WHERE [key]=?");
207 			sqlSet = db.prepare("INSERT OR REPLACE INTO [" ~ tableName ~ "] VALUES (?, ?)");
208 			sqlDelete = db.prepare("DELETE FROM [" ~ tableName ~ "] WHERE [key]=?");
209 			sqlExists = db.prepare("SELECT COUNT(*) FROM [" ~ tableName ~ "] WHERE [key]=? LIMIT 1");
210 			sqlLength = db.prepare("SELECT COUNT(*) FROM [" ~ tableName ~ "]");
211 			sqlListKeys = db.prepare("SELECT [key] FROM [" ~ tableName ~ "]");
212 			sqlListPairs = db.prepare("SELECT [key], [value] FROM [" ~ tableName ~ "]");
213 			initialized = true;
214 		}
215 	}
216 }
217 
218 /// A `KeyValueDatabase` holds one or more key/value tables (`KeyValueStore`).
219 struct KeyValueDatabase
220 {
221 	string fileName; /// Database file name.
222 
223 	SQLite sqlite; /// SQLite database instance. Initialized automatically.
224 
225 	@property SQLite _getSQLite()
226 	{
227 		if (sqlite is null)
228 		{
229 			enforce(fileName, "KeyValueDatabase filename not set");
230 			sqlite = new SQLite(fileName);
231 		}
232 		return sqlite;
233 	}
234 
235 	alias _getSQLite this;
236 }
237 
238 unittest
239 {
240 	import std.file;
241 
242 	string fn = tempDir ~ "/ae-sys-persistence-keyvalue-test.s3db";
243 	if (fn.exists) fn.remove();
244 	scope(success) fn.remove();
245 
246 	auto store = KeyValueStore!(string, string)(fn);
247 
248 	assert(store.length == 0);
249 	assert("key" !in store);
250 	assert(store.get("key", null) is null);
251 	assert(store.keys.length == 0);
252 
253 	store["key"] = "value";
254 
255 	assert(store.length == 1);
256 	assert("key" in store);
257 	assert(store["key"] == "value");
258 	assert(store.get("key", null) == "value");
259 	assert(store.keys == ["key"]);
260 
261 	store["key"] = "value2";
262 
263 	assert(store.length == 1);
264 	assert("key" in store);
265 	assert(store.get("key", null) == "value2");
266 	assert(store.keys == ["key"]);
267 
268 	store["key2"] = "value3";
269 
270 	assert(store.length == 2);
271 	assert("key" in store);
272 	assert("key2" in store);
273 	assert(store.get("key", null) == "value2");
274 	assert(store.get("key2", null) == "value3");
275 	assert(store.keys == ["key", "key2"]);
276 
277 	store.remove("key");
278 
279 	assert(store.length == 1);
280 	assert("key" !in store);
281 	assert("key2" in store);
282 	assert(store.get("key", null) is null);
283 	assert(store.keys == ["key2"]);
284 }
285 
286 unittest
287 {
288 	if (false)
289 	{
290 		KeyValueStore!(string, ubyte[20]) kv;
291 		ubyte[20] s = kv[""];
292 	}
293 }
294 
295 unittest
296 {
297 	if (false)
298 	{
299 		KeyValueStore!(string, float[20]) kv;
300 		float[20] s = kv[""];
301 	}
302 }
303 
304 unittest
305 {
306 	if (false)
307 	{
308 		struct K {}
309 		KeyValueStore!(K, K) kv;
310 		assert(K.init !in kv);
311 		immutable K ik;
312 		assert(ik !in kv);
313 	}
314 }
315 
316 unittest
317 {
318 	import std.file;
319 
320 	string fn = tempDir ~ "/ae-sys-persistence-keyvalue-test.s3db";
321 	if (fn.exists) fn.remove();
322 	scope(success) fn.remove();
323 
324 	KeyValueStore!(float[], float[]) kv;
325 	kv = typeof(kv)(fn);
326 	assert(null !in kv);
327 	kv[null] = null;
328 	assert(null in kv);
329 	assert(kv[null] == null);
330 }