1 /** 2 * ae.sys.persistence.keyvalue 3 * 4 * License: 5 * This Source Code Form is subject to the terms of 6 * the Mozilla Public License, v. 2.0. If a copy of 7 * the MPL was not distributed with this file, You 8 * can obtain one at http://mozilla.org/MPL/2.0/. 9 * 10 * Authors: 11 * Vladimir Panteleev <ae@cy.md> 12 */ 13 14 module ae.sys.persistence.keyvalue; 15 16 import std.exception; 17 import std.traits; 18 19 import ae.sys.persistence.core; 20 import ae.sys.sqlite3; 21 import ae.utils.array : nonNull; 22 import ae.utils.json; 23 24 // **************************************************************************** 25 26 /// Persistent indexed key-value store, backed by an SQLite database. 27 /// Non-string keys/values are JSON-encoded. 28 struct KeyValueStore(K, V) 29 { 30 KeyValueDatabase* db; /// 31 string tableName; /// 32 33 /// Constructor with `KeyValueDatabase` and `tableName`. 34 /// Allows using the same database file for multiple key/value tables. 35 this(KeyValueDatabase* db, string tableName = "values") 36 { 37 this.db = db; 38 this.tableName = tableName; 39 } 40 41 /// Constructor with file name. 42 /// Creates a new `KeyValueDatabase` for private use. 43 this(string fn) 44 { 45 auto db = new KeyValueDatabase(fn); 46 this(db); 47 } 48 49 /// Implements common D associative array operations. 50 V opIndex()(auto ref const K k) 51 { 52 checkInitialized(); 53 foreach (SqlType!V v; sqlGet.iterate(toSqlType(k))) 54 return fromSqlType!V(v); 55 throw new Exception("Value not in KeyValueStore"); 56 } 57 58 V get()(auto ref const K k, auto ref V defaultValue) 59 { 60 checkInitialized(); 61 foreach (SqlType!V v; sqlGet.iterate(toSqlType(k))) 62 return fromSqlType!V(v); 63 return defaultValue; 64 } /// ditto 65 66 V require()(auto ref const K k, lazy V defaultValue) 67 { 68 checkInitialized(); 69 foreach (SqlType!V v; sqlGet.iterate(toSqlType(k))) 70 return fromSqlType!V(v); 71 auto v = defaultValue(); 72 sqlSet.exec(toSqlType(k), toSqlType(v)); 73 return v; 74 } /// ditto 75 deprecated alias getOrAdd = require; 76 77 bool opBinaryRight(string op)(auto ref const K k) 78 if (op == "in") 79 { 80 checkInitialized(); 81 foreach (int count; sqlExists.iterate(toSqlType(k))) 82 return count > 0; 83 assert(false); 84 } /// ditto 85 86 auto ref const(V) opIndexAssign()(auto ref const V v, auto ref const K k) 87 { 88 checkInitialized(); 89 sqlSet.exec(toSqlType(k), toSqlType(v)); 90 return v; 91 } /// ditto 92 93 void remove()(auto ref const K k) 94 { 95 checkInitialized(); 96 sqlDelete.exec(toSqlType(k)); 97 } /// ditto 98 99 @property int length() 100 { 101 checkInitialized(); 102 foreach (int count; sqlLength.iterate()) 103 return count; 104 assert(false); 105 } /// ditto 106 107 @property K[] keys() 108 { 109 checkInitialized(); 110 K[] result; 111 foreach (SqlType!K key; sqlListKeys.iterate()) 112 result ~= fromSqlType!K(key); 113 return result; 114 } 115 116 int opApply(int delegate(K key, V value) dg) 117 { 118 checkInitialized(); 119 foreach (SqlType!K key, SqlType!V value; sqlListPairs.iterate()) 120 { 121 auto res = dg(fromSqlType!K(key), fromSqlType!V(value)); 122 if (res) 123 return res; 124 } 125 return 0; 126 } 127 128 private: 129 static SqlType!T toSqlType(T)(auto ref T v) 130 { 131 alias S = SqlType!T; 132 static if (is(T : long)) // long 133 return v; 134 else 135 static if (is(T : const(char)[])) // string 136 return cast(S) v.nonNull; 137 else 138 static if (is(T U : U[]) && !hasIndirections!U) // void[] 139 return v.nonNull; 140 else 141 return toJson(v); 142 } 143 144 static T fromSqlType(T)(SqlType!T v) 145 { 146 static if (is(T : long)) // long 147 return cast(T) v; 148 else 149 static if (is(T : const(char)[])) // string 150 return cast(T) v; 151 else 152 static if (is(T U : U[]) && !hasIndirections!U) // void[] 153 static if (is(T V : V[N], size_t N)) 154 { 155 assert(v.length == N * V.sizeof, "Static array length mismatch"); 156 return cast(T) v[0 .. N * V.sizeof]; 157 } 158 else 159 return cast(T) v; 160 else 161 return jsonParse!T(cast(string) v); 162 } 163 164 template SqlType(T) 165 { 166 static if (is(T : long)) 167 alias SqlType = long; 168 else 169 static if (is(T : const(char)[])) 170 alias SqlType = string; 171 else 172 static if (is(T U : U[]) && !hasIndirections!U) 173 alias SqlType = const(void)[]; 174 else 175 alias SqlType = string; // JSON-encoded 176 } 177 178 static assert(is(SqlType!int == long)); 179 static assert(is(SqlType!string == string)); 180 181 template sqlTypeName(T) 182 { 183 alias S = SqlType!T; 184 static if (is(S == long)) 185 enum sqlTypeName = "INTEGER"; 186 else 187 static if (is(S == string)) 188 enum sqlTypeName = "TEXT"; 189 else 190 static if (is(S == void[])) 191 enum sqlTypeName = "BLOB"; 192 else 193 enum sqlTypeName = "TEXT"; // JSON 194 } 195 196 bool initialized; 197 198 SQLite.PreparedStatement sqlGet, sqlSet, sqlDelete, sqlExists, sqlLength, sqlListKeys, sqlListPairs; 199 200 void checkInitialized() 201 { 202 if (!initialized) 203 { 204 assert(db, "KeyValueStore database not set"); 205 db.exec("CREATE TABLE IF NOT EXISTS [" ~ tableName ~ "] ([key] " ~ sqlTypeName!K ~ " PRIMARY KEY, [value] " ~ sqlTypeName!V ~ ")"); 206 db.exec("PRAGMA SYNCHRONOUS=OFF"); 207 sqlGet = db.prepare("SELECT [value] FROM [" ~ tableName ~ "] WHERE [key]=?"); 208 sqlSet = db.prepare("INSERT OR REPLACE INTO [" ~ tableName ~ "] VALUES (?, ?)"); 209 sqlDelete = db.prepare("DELETE FROM [" ~ tableName ~ "] WHERE [key]=?"); 210 sqlExists = db.prepare("SELECT COUNT(*) FROM [" ~ tableName ~ "] WHERE [key]=? LIMIT 1"); 211 sqlLength = db.prepare("SELECT COUNT(*) FROM [" ~ tableName ~ "]"); 212 sqlListKeys = db.prepare("SELECT [key] FROM [" ~ tableName ~ "]"); 213 sqlListPairs = db.prepare("SELECT [key], [value] FROM [" ~ tableName ~ "]"); 214 initialized = true; 215 } 216 } 217 } 218 219 /// A `KeyValueDatabase` holds one or more key/value tables (`KeyValueStore`). 220 struct KeyValueDatabase 221 { 222 string fileName; /// Database file name. 223 224 SQLite sqlite; /// SQLite database instance. Initialized automatically. 225 226 @property SQLite _getSQLite() 227 { 228 if (sqlite is null) 229 { 230 enforce(fileName, "KeyValueDatabase filename not set"); 231 sqlite = new SQLite(fileName); 232 } 233 return sqlite; 234 } 235 236 alias _getSQLite this; 237 } 238 239 unittest 240 { 241 import std.file; 242 243 string fn = tempDir ~ "/ae-sys-persistence-keyvalue-test.s3db"; 244 if (fn.exists) fn.remove(); 245 scope(success) fn.remove(); 246 247 auto store = KeyValueStore!(string, string)(fn); 248 249 assert(store.length == 0); 250 assert("key" !in store); 251 assert(store.get("key", null) is null); 252 assert(store.keys.length == 0); 253 254 store["key"] = "value"; 255 256 assert(store.length == 1); 257 assert("key" in store); 258 assert(store["key"] == "value"); 259 assert(store.get("key", null) == "value"); 260 assert(store.keys == ["key"]); 261 262 store["key"] = "value2"; 263 264 assert(store.length == 1); 265 assert("key" in store); 266 assert(store.get("key", null) == "value2"); 267 assert(store.keys == ["key"]); 268 269 store["key2"] = "value3"; 270 271 assert(store.length == 2); 272 assert("key" in store); 273 assert("key2" in store); 274 assert(store.get("key", null) == "value2"); 275 assert(store.get("key2", null) == "value3"); 276 assert(store.keys == ["key", "key2"]); 277 278 store.remove("key"); 279 280 assert(store.length == 1); 281 assert("key" !in store); 282 assert("key2" in store); 283 assert(store.get("key", null) is null); 284 assert(store.keys == ["key2"]); 285 } 286 287 unittest 288 { 289 if (false) 290 { 291 KeyValueStore!(string, ubyte[20]) kv; 292 ubyte[20] s = kv[""]; 293 } 294 } 295 296 unittest 297 { 298 if (false) 299 { 300 KeyValueStore!(string, float[20]) kv; 301 float[20] s = kv[""]; 302 } 303 } 304 305 unittest 306 { 307 if (false) 308 { 309 struct K {} 310 KeyValueStore!(K, K) kv; 311 assert(K.init !in kv); 312 immutable K ik; 313 assert(ik !in kv); 314 } 315 } 316 317 unittest 318 { 319 import std.file; 320 321 string fn = tempDir ~ "/ae-sys-persistence-keyvalue-test.s3db"; 322 if (fn.exists) fn.remove(); 323 scope(success) fn.remove(); 324 325 KeyValueStore!(float[], float[]) kv; 326 kv = typeof(kv)(fn); 327 assert(null !in kv); 328 kv[null] = null; 329 assert(null in kv); 330 assert(kv[null] == null); 331 }