1 /** 2 * ae.sys.persistence.keyvalue 3 * 4 * License: 5 * This Source Code Form is subject to the terms of 6 * the Mozilla Public License, v. 2.0. If a copy of 7 * the MPL was not distributed with this file, You 8 * can obtain one at http://mozilla.org/MPL/2.0/. 9 * 10 * Authors: 11 * Vladimir Panteleev <ae@cy.md> 12 */ 13 14 module ae.sys.persistence.keyvalue; 15 16 import std.exception; 17 import std.traits; 18 19 import ae.sys.persistence.core; 20 import ae.sys.sqlite3; 21 import ae.utils.array : nonNull; 22 import ae.utils.json; 23 24 // **************************************************************************** 25 26 /// Persistent indexed key-value store, backed by an SQLite database. 27 /// Non-string keys/values are JSON-encoded. 28 struct KeyValueStore(K, V) 29 { 30 KeyValueDatabase* db; /// 31 string tableName; /// 32 33 /// Constructor with `KeyValueDatabase` and `tableName`. 34 /// Allows using the same database file for multiple key/value tables. 35 this(KeyValueDatabase* db, string tableName = "values") 36 { 37 this.db = db; 38 this.tableName = tableName; 39 } 40 41 /// Constructor with file name. 42 /// Creates a new `KeyValueDatabase` for private use. 43 this(string fn) 44 { 45 auto db = new KeyValueDatabase(fn); 46 this(db); 47 } 48 49 /// Implements common D associative array operations. 50 V opIndex()(auto ref const K k) 51 { 52 checkInitialized(); 53 foreach (SqlType!V v; sqlGet.iterate(toSqlType(k))) 54 return fromSqlType!V(v); 55 throw new Exception("Value not in KeyValueStore"); 56 } 57 58 V get()(auto ref const K k, auto ref V defaultValue) 59 { 60 checkInitialized(); 61 foreach (SqlType!V v; sqlGet.iterate(toSqlType(k))) 62 return fromSqlType!V(v); 63 return defaultValue; 64 } /// ditto 65 66 V getOrAdd()(auto ref const K k, lazy V defaultValue) 67 { 68 checkInitialized(); 69 foreach (SqlType!V v; sqlGet.iterate(toSqlType(k))) 70 return fromSqlType!V(v); 71 auto v = defaultValue(); 72 sqlSet.exec(toSqlType(k), toSqlType(v)); 73 return v; 74 } /// ditto 75 76 bool opBinaryRight(string op)(auto ref const K k) 77 if (op == "in") 78 { 79 checkInitialized(); 80 foreach (int count; sqlExists.iterate(toSqlType(k))) 81 return count > 0; 82 assert(false); 83 } /// ditto 84 85 auto ref const(V) opIndexAssign()(auto ref const V v, auto ref const K k) 86 { 87 checkInitialized(); 88 sqlSet.exec(toSqlType(k), toSqlType(v)); 89 return v; 90 } /// ditto 91 92 void remove()(auto ref const K k) 93 { 94 checkInitialized(); 95 sqlDelete.exec(toSqlType(k)); 96 } /// ditto 97 98 @property int length() 99 { 100 checkInitialized(); 101 foreach (int count; sqlLength.iterate()) 102 return count; 103 assert(false); 104 } /// ditto 105 106 @property K[] keys() 107 { 108 checkInitialized(); 109 K[] result; 110 foreach (SqlType!K key; sqlListKeys.iterate()) 111 result ~= fromSqlType!K(key); 112 return result; 113 } 114 115 int opApply(int delegate(K key, V value) dg) 116 { 117 checkInitialized(); 118 foreach (SqlType!K key, SqlType!V value; sqlListPairs.iterate()) 119 { 120 auto res = dg(fromSqlType!K(key), fromSqlType!V(value)); 121 if (res) 122 return res; 123 } 124 return 0; 125 } 126 127 private: 128 static SqlType!T toSqlType(T)(auto ref T v) 129 { 130 alias S = SqlType!T; 131 static if (is(T : long)) // long 132 return v; 133 else 134 static if (is(T : const(char)[])) // string 135 return cast(S) v.nonNull; 136 else 137 static if (is(T U : U[]) && !hasIndirections!U) // void[] 138 return v.nonNull; 139 else 140 return toJson(v); 141 } 142 143 static T fromSqlType(T)(SqlType!T v) 144 { 145 static if (is(T : long)) // long 146 return cast(T) v; 147 else 148 static if (is(T : const(char)[])) // string 149 return cast(T) v; 150 else 151 static if (is(T U : U[]) && !hasIndirections!U) // void[] 152 static if (is(T V : V[N], size_t N)) 153 { 154 assert(v.length == N * V.sizeof, "Static array length mismatch"); 155 return cast(T) v[0 .. N * V.sizeof]; 156 } 157 else 158 return cast(T) v; 159 else 160 return jsonParse!T(cast(string) v); 161 } 162 163 template SqlType(T) 164 { 165 static if (is(T : long)) 166 alias SqlType = long; 167 else 168 static if (is(T : const(char)[])) 169 alias SqlType = string; 170 else 171 static if (is(T U : U[]) && !hasIndirections!U) 172 alias SqlType = const(void)[]; 173 else 174 alias SqlType = string; // JSON-encoded 175 } 176 177 static assert(is(SqlType!int == long)); 178 static assert(is(SqlType!string == string)); 179 180 template sqlTypeName(T) 181 { 182 alias S = SqlType!T; 183 static if (is(S == long)) 184 enum sqlTypeName = "INTEGER"; 185 else 186 static if (is(S == string)) 187 enum sqlTypeName = "TEXT"; 188 else 189 static if (is(S == void[])) 190 enum sqlTypeName = "BLOB"; 191 else 192 enum sqlTypeName = "TEXT"; // JSON 193 } 194 195 bool initialized; 196 197 SQLite.PreparedStatement sqlGet, sqlSet, sqlDelete, sqlExists, sqlLength, sqlListKeys, sqlListPairs; 198 199 void checkInitialized() 200 { 201 if (!initialized) 202 { 203 assert(db, "KeyValueStore database not set"); 204 db.exec("CREATE TABLE IF NOT EXISTS [" ~ tableName ~ "] ([key] " ~ sqlTypeName!K ~ " PRIMARY KEY, [value] " ~ sqlTypeName!V ~ ")"); 205 db.exec("PRAGMA SYNCHRONOUS=OFF"); 206 sqlGet = db.prepare("SELECT [value] FROM [" ~ tableName ~ "] WHERE [key]=?"); 207 sqlSet = db.prepare("INSERT OR REPLACE INTO [" ~ tableName ~ "] VALUES (?, ?)"); 208 sqlDelete = db.prepare("DELETE FROM [" ~ tableName ~ "] WHERE [key]=?"); 209 sqlExists = db.prepare("SELECT COUNT(*) FROM [" ~ tableName ~ "] WHERE [key]=? LIMIT 1"); 210 sqlLength = db.prepare("SELECT COUNT(*) FROM [" ~ tableName ~ "]"); 211 sqlListKeys = db.prepare("SELECT [key] FROM [" ~ tableName ~ "]"); 212 sqlListPairs = db.prepare("SELECT [key], [value] FROM [" ~ tableName ~ "]"); 213 initialized = true; 214 } 215 } 216 } 217 218 /// A `KeyValueDatabase` holds one or more key/value tables (`KeyValueStore`). 219 struct KeyValueDatabase 220 { 221 string fileName; /// Database file name. 222 223 SQLite sqlite; /// SQLite database instance. Initialized automatically. 224 225 @property SQLite _getSQLite() 226 { 227 if (sqlite is null) 228 { 229 enforce(fileName, "KeyValueDatabase filename not set"); 230 sqlite = new SQLite(fileName); 231 } 232 return sqlite; 233 } 234 235 alias _getSQLite this; 236 } 237 238 unittest 239 { 240 import std.file; 241 242 string fn = tempDir ~ "/ae-sys-persistence-keyvalue-test.s3db"; 243 if (fn.exists) fn.remove(); 244 scope(success) fn.remove(); 245 246 auto store = KeyValueStore!(string, string)(fn); 247 248 assert(store.length == 0); 249 assert("key" !in store); 250 assert(store.get("key", null) is null); 251 assert(store.keys.length == 0); 252 253 store["key"] = "value"; 254 255 assert(store.length == 1); 256 assert("key" in store); 257 assert(store["key"] == "value"); 258 assert(store.get("key", null) == "value"); 259 assert(store.keys == ["key"]); 260 261 store["key"] = "value2"; 262 263 assert(store.length == 1); 264 assert("key" in store); 265 assert(store.get("key", null) == "value2"); 266 assert(store.keys == ["key"]); 267 268 store["key2"] = "value3"; 269 270 assert(store.length == 2); 271 assert("key" in store); 272 assert("key2" in store); 273 assert(store.get("key", null) == "value2"); 274 assert(store.get("key2", null) == "value3"); 275 assert(store.keys == ["key", "key2"]); 276 277 store.remove("key"); 278 279 assert(store.length == 1); 280 assert("key" !in store); 281 assert("key2" in store); 282 assert(store.get("key", null) is null); 283 assert(store.keys == ["key2"]); 284 } 285 286 unittest 287 { 288 if (false) 289 { 290 KeyValueStore!(string, ubyte[20]) kv; 291 ubyte[20] s = kv[""]; 292 } 293 } 294 295 unittest 296 { 297 if (false) 298 { 299 KeyValueStore!(string, float[20]) kv; 300 float[20] s = kv[""]; 301 } 302 } 303 304 unittest 305 { 306 if (false) 307 { 308 struct K {} 309 KeyValueStore!(K, K) kv; 310 assert(K.init !in kv); 311 immutable K ik; 312 assert(ik !in kv); 313 } 314 } 315 316 unittest 317 { 318 import std.file; 319 320 string fn = tempDir ~ "/ae-sys-persistence-keyvalue-test.s3db"; 321 if (fn.exists) fn.remove(); 322 scope(success) fn.remove(); 323 324 KeyValueStore!(float[], float[]) kv; 325 kv = typeof(kv)(fn); 326 assert(null !in kv); 327 kv[null] = null; 328 assert(null in kv); 329 assert(kv[null] == null); 330 }