1 /** 2 * Data digest (hashes etc.) 3 * 4 * License: 5 * This Source Code Form is subject to the terms of 6 * the Mozilla Public License, v. 2.0. If a copy of 7 * the MPL was not distributed with this file, You 8 * can obtain one at http://mozilla.org/MPL/2.0/. 9 * 10 * Authors: 11 * Vladimir Panteleev <ae@cy.md> 12 */ 13 14 module ae.utils.digest; 15 16 import std.traits; 17 18 import ae.sys.cmd; 19 import ae.utils.text; 20 import ae.utils.array; 21 22 deprecated ("Use std.digest.hmac") 23 ubyte[] hmac_sha1(string message, ubyte[] privateKey) 24 { 25 return 26 arrayFromHex(sha1sum(vector!("^")(padRight(privateKey, 64, cast(ubyte)0x00), repeatOne(cast(ubyte)0x5c, 64)) ~ 27 arrayFromHex(sha1sum(vector!("^")(padRight(privateKey, 64, cast(ubyte)0x00), repeatOne(cast(ubyte)0x36, 64)) ~ 28 cast(ubyte[])message)))); 29 } 30 31 // the standard Phobos crc32 function relies on inlining for usable performance 32 /// Inlinable CRC32 implementation. 33 uint[256] crc32Table = [ 34 0x00000000,0x77073096,0xee0e612c,0x990951ba,0x076dc419,0x706af48f,0xe963a535,0x9e6495a3,0x0edb8832,0x79dcb8a4,0xe0d5e91e,0x97d2d988,0x09b64c2b,0x7eb17cbd,0xe7b82d07,0x90bf1d91, 35 0x1db71064,0x6ab020f2,0xf3b97148,0x84be41de,0x1adad47d,0x6ddde4eb,0xf4d4b551,0x83d385c7,0x136c9856,0x646ba8c0,0xfd62f97a,0x8a65c9ec,0x14015c4f,0x63066cd9,0xfa0f3d63,0x8d080df5, 36 0x3b6e20c8,0x4c69105e,0xd56041e4,0xa2677172,0x3c03e4d1,0x4b04d447,0xd20d85fd,0xa50ab56b,0x35b5a8fa,0x42b2986c,0xdbbbc9d6,0xacbcf940,0x32d86ce3,0x45df5c75,0xdcd60dcf,0xabd13d59, 37 0x26d930ac,0x51de003a,0xc8d75180,0xbfd06116,0x21b4f4b5,0x56b3c423,0xcfba9599,0xb8bda50f,0x2802b89e,0x5f058808,0xc60cd9b2,0xb10be924,0x2f6f7c87,0x58684c11,0xc1611dab,0xb6662d3d, 38 0x76dc4190,0x01db7106,0x98d220bc,0xefd5102a,0x71b18589,0x06b6b51f,0x9fbfe4a5,0xe8b8d433,0x7807c9a2,0x0f00f934,0x9609a88e,0xe10e9818,0x7f6a0dbb,0x086d3d2d,0x91646c97,0xe6635c01, 39 0x6b6b51f4,0x1c6c6162,0x856530d8,0xf262004e,0x6c0695ed,0x1b01a57b,0x8208f4c1,0xf50fc457,0x65b0d9c6,0x12b7e950,0x8bbeb8ea,0xfcb9887c,0x62dd1ddf,0x15da2d49,0x8cd37cf3,0xfbd44c65, 40 0x4db26158,0x3ab551ce,0xa3bc0074,0xd4bb30e2,0x4adfa541,0x3dd895d7,0xa4d1c46d,0xd3d6f4fb,0x4369e96a,0x346ed9fc,0xad678846,0xda60b8d0,0x44042d73,0x33031de5,0xaa0a4c5f,0xdd0d7cc9, 41 0x5005713c,0x270241aa,0xbe0b1010,0xc90c2086,0x5768b525,0x206f85b3,0xb966d409,0xce61e49f,0x5edef90e,0x29d9c998,0xb0d09822,0xc7d7a8b4,0x59b33d17,0x2eb40d81,0xb7bd5c3b,0xc0ba6cad, 42 0xedb88320,0x9abfb3b6,0x03b6e20c,0x74b1d29a,0xead54739,0x9dd277af,0x04db2615,0x73dc1683,0xe3630b12,0x94643b84,0x0d6d6a3e,0x7a6a5aa8,0xe40ecf0b,0x9309ff9d,0x0a00ae27,0x7d079eb1, 43 0xf00f9344,0x8708a3d2,0x1e01f268,0x6906c2fe,0xf762575d,0x806567cb,0x196c3671,0x6e6b06e7,0xfed41b76,0x89d32be0,0x10da7a5a,0x67dd4acc,0xf9b9df6f,0x8ebeeff9,0x17b7be43,0x60b08ed5, 44 0xd6d6a3e8,0xa1d1937e,0x38d8c2c4,0x4fdff252,0xd1bb67f1,0xa6bc5767,0x3fb506dd,0x48b2364b,0xd80d2bda,0xaf0a1b4c,0x36034af6,0x41047a60,0xdf60efc3,0xa867df55,0x316e8eef,0x4669be79, 45 0xcb61b38c,0xbc66831a,0x256fd2a0,0x5268e236,0xcc0c7795,0xbb0b4703,0x220216b9,0x5505262f,0xc5ba3bbe,0xb2bd0b28,0x2bb45a92,0x5cb36a04,0xc2d7ffa7,0xb5d0cf31,0x2cd99e8b,0x5bdeae1d, 46 0x9b64c2b0,0xec63f226,0x756aa39c,0x026d930a,0x9c0906a9,0xeb0e363f,0x72076785,0x05005713,0x95bf4a82,0xe2b87a14,0x7bb12bae,0x0cb61b38,0x92d28e9b,0xe5d5be0d,0x7cdcefb7,0x0bdbdf21, 47 0x86d3d2d4,0xf1d4e242,0x68ddb3f8,0x1fda836e,0x81be16cd,0xf6b9265b,0x6fb077e1,0x18b74777,0x88085ae6,0xff0f6a70,0x66063bca,0x11010b5c,0x8f659eff,0xf862ae69,0x616bffd3,0x166ccf45, 48 0xa00ae278,0xd70dd2ee,0x4e048354,0x3903b3c2,0xa7672661,0xd06016f7,0x4969474d,0x3e6e77db,0xaed16a4a,0xd9d65adc,0x40df0b66,0x37d83bf0,0xa9bcae53,0xdebb9ec5,0x47b2cf7f,0x30b5ffe9, 49 0xbdbdf21c,0xcabac28a,0x53b39330,0x24b4a3a6,0xbad03605,0xcdd70693,0x54de5729,0x23d967bf,0xb3667a2e,0xc4614ab8,0x5d681b02,0x2a6f2b94,0xb40bbe37,0xc30c8ea1,0x5a05df1b,0x2d02ef8d, 50 ]; 51 52 /// ditto 53 uint fastCRC(in void[] data, uint start = cast(uint)-1) 54 { 55 uint crc = start; 56 foreach (val; cast(ubyte[])data) 57 crc = crc32Table[cast(ubyte) crc ^ val] ^ (crc >> 8); 58 return crc; 59 } 60 61 62 // Struct (for streaming) 63 /// D port of MurmurHash2A. 64 /// TODO: reimplement via std.digest 65 struct MurmurHash2A 66 { 67 private static string mmix(string h, string k) { return "{ "~k~" *= m; "~k~" ^= "~k~" >> r; "~k~" *= m; "~h~" *= m; "~h~" ^= "~k~"; }"; } 68 69 public: 70 /// 71 void Begin ( uint seed = 0 ) 72 { 73 m_hash = seed; 74 m_tail = 0; 75 m_count = 0; 76 m_size = 0; 77 } 78 79 /// 80 void Add ( const(void) * vdata, sizediff_t len ) 81 { 82 ubyte * data = cast(ubyte*)vdata; 83 m_size += len; 84 85 MixTail(data,len); 86 87 while(len >= 4) 88 { 89 uint k = *cast(uint*)data; 90 91 mixin(mmix("m_hash","k")); 92 93 data += 4; 94 len -= 4; 95 } 96 97 MixTail(data,len); 98 } 99 100 /// 101 uint End ( ) 102 { 103 mixin(mmix("m_hash","m_tail")); 104 mixin(mmix("m_hash","m_size")); 105 106 m_hash ^= m_hash >> 13; 107 m_hash *= m; 108 m_hash ^= m_hash >> 15; 109 110 return m_hash; 111 } 112 113 /// `Add` overloads for some common D types. 114 void Add(ref ubyte v) { Add(&v, v.sizeof); } /// ditto 115 void Add(ref int v) { Add(&v, v.sizeof); } /// ditto 116 void Add(ref uint v) { Add(&v, v.sizeof); } /// ditto 117 void Add(string s) { Add(s.ptr, cast(uint)s.length); } /// ditto 118 void Add(ubyte[] s) { Add(s.ptr, cast(uint)s.length); } /// ditto 119 120 private: 121 122 static const uint m = 0x5bd1e995; 123 static const int r = 24; 124 125 void MixTail ( ref ubyte * data, ref sizediff_t len ) 126 { 127 while( len && ((len<4) || m_count) ) 128 { 129 m_tail |= (*data++) << (m_count * 8); 130 131 m_count++; 132 len--; 133 134 if(m_count == 4) 135 { 136 mixin(mmix("m_hash","m_tail")); 137 m_tail = 0; 138 m_count = 0; 139 } 140 } 141 } 142 143 uint m_hash; 144 uint m_tail; 145 uint m_count; 146 uint m_size; 147 } 148 149 /// ditto 150 uint murmurHash2(in void[] data, uint seed=0) 151 { 152 enum { m = 0x5bd1e995, r = 24 } 153 uint len = cast(uint)data.length; 154 uint h = seed ^ len; 155 ubyte* p = cast(ubyte*)data.ptr; 156 157 while (len >= 4) 158 { 159 uint k = *cast(uint*)p; 160 161 k *= m; 162 k ^= k >> r; 163 k *= m; 164 165 h *= m; 166 h ^= k; 167 168 p += 4; 169 len -= 4; 170 } 171 172 switch(len) 173 { 174 case 3: h ^= p[2] << 16; goto case; 175 case 2: h ^= p[1] << 8; goto case; 176 case 1: h ^= p[0]; 177 /* */ h *= m; goto case; 178 case 0: break; 179 default: assert(0); 180 } 181 182 // Do a few final mixes of the hash to ensure the last few 183 // bytes are well-incorporated. 184 185 h ^= h >> 13; 186 h *= m; 187 h ^= h >> 15; 188 189 return h; 190 } 191 192 deprecated import ae.utils.digest_murmurhash3; 193 194 deprecated ("Use std.digest.murmurhash") 195 { 196 uint murmurHash3_32(in void[] data, uint seed=0) 197 { 198 uint result; 199 MurmurHash3_x86_32(data.ptr, cast(uint)data.length, seed, &result); 200 return result; 201 } 202 203 alias uint[4] MH3Digest128; 204 205 MH3Digest128 murmurHash3_x86_128(in void[] data, uint seed=0) 206 { 207 MH3Digest128 result; 208 MurmurHash3_x86_128(data.ptr, cast(uint)data.length, seed, &result); 209 return result; 210 } 211 212 MH3Digest128 murmurHash3_x64_128(in void[] data, uint seed=0) 213 { 214 MH3Digest128 result; 215 MurmurHash3_x64_128(data.ptr, cast(uint)data.length, seed, &result); 216 return result; 217 } 218 219 /// Select version optimized for target platform. 220 /// WARNING: Output depends on platform. 221 version(D_LP64) 222 alias murmurHash3_x64_128 murmurHash3_128; 223 else 224 alias murmurHash3_x86_128 murmurHash3_128; 225 226 string digestToStringMH3(MH3Digest128 digest) 227 { 228 import std..string; 229 return format("%08X%08X%08X%08X", digest[0], digest[1], digest[2], digest[3]); 230 } 231 232 unittest 233 { 234 assert(murmurHash3_32("The quick brown fox jumps over the lazy dog") == 0x2e4ff723); 235 assert(murmurHash3_32("The quick brown fox jumps over the lazy cog") == 0xf08200fc); 236 237 assert(murmurHash3_x86_128("The quick brown fox jumps over the lazy dog") == [ 0x2f1583c3 , 0xecee2c67 , 0x5d7bf66c , 0xe5e91d2c ]); 238 assert(murmurHash3_x86_128("The quick brown fox jumps over the lazy cog") == [ 0x0ed64388 , 0x3e9ae779 , 0x97034593 , 0x49b3f32f ]); 239 240 assert(murmurHash3_x64_128("The quick brown fox jumps over the lazy dog") == [ 0xbc071b6c , 0xe34bbc7b , 0xc49a9347 , 0x7a433ca9 ]); 241 assert(murmurHash3_x64_128("The quick brown fox jumps over the lazy cog") == [ 0xff85269a , 0x658ca970 , 0xa68e5c3e , 0x43fee3ea ]); 242 243 assert(digestToStringMH3(murmurHash3_x86_128("The quick brown fox jumps over the lazy dog")) == "2F1583C3ECEE2C675D7BF66CE5E91D2C"); 244 } 245 } 246 247 // ************************************************************************ 248 249 deprecated public import std.digest.md; 250 251 /// Get digest string of given data. 252 /// Short-hand for std.digest.md5Of (and similar) and toHexString. 253 /// Similar to the old std.md5.getDigestString. 254 template getDigestString(Digest) 255 { 256 string getDigestString(T)(const(T)[][] data...) 257 if (!hasIndirections!T) 258 { 259 return getDigestStringImpl(arrOfArrCastInPlace!(const(ubyte))(data)); 260 } 261 262 // A dirty hack to fix the array lengths right on the stack, to avoid an allocation. 263 T[][] arrOfArrCastInPlace(T, S)(S[][] arrs) 264 { 265 static struct Array 266 { 267 size_t length; 268 void* ptr; 269 } 270 auto rawArrs = cast(Array[])arrs; 271 foreach (ref ra; rawArrs) 272 ra.length = ra.length * S.sizeof / T.sizeof; 273 return cast(T[][])rawArrs; 274 } 275 276 string getDigestStringImpl(in ubyte[][] data) 277 { 278 Digest digest; 279 digest.start(); 280 foreach (datum; data) 281 digest.put(cast(const(ubyte)[])datum); 282 auto result = digest.finish(); 283 // https://issues.dlang.org/show_bug.cgi?id=9279 284 auto str = result.toHexString(); 285 return str[].idup; 286 } 287 } 288 289 /// 290 unittest 291 { 292 assert(getDigestString!MD5("abc") == "900150983CD24FB0D6963F7D28E17F72"); 293 } 294 295 // ************************************************************************ 296 297 deprecated("Use std.digest.hmac") 298 { 299 /// HMAC digest with a hash algorithm. 300 /// Params: 301 /// Algorithm = std.digest-compatible hash type 302 /// blockSize = Algorithm block size, in bytes 303 /// key = Secret key bytes 304 /// message = Message data 305 template HMAC(alias Algorithm, size_t blockSize) 306 { 307 alias Digest = typeof(Algorithm.init.finish()); 308 309 Digest HMAC(in ubyte[] key, in ubyte[] message) 310 { 311 ubyte[blockSize] keyBlock = 0; 312 if (key.length > blockSize) 313 keyBlock[0..Digest.length] = digest!Algorithm(key)[]; 314 else 315 keyBlock[0..key.length] = key[]; 316 317 ubyte[blockSize] oKeyPad = 0x5C; oKeyPad[] ^= keyBlock[]; 318 ubyte[blockSize] iKeyPad = 0x36; iKeyPad[] ^= keyBlock[]; 319 320 Algorithm oHash; 321 oHash.start(); 322 oHash.put(oKeyPad[]); 323 324 Algorithm iHash; 325 iHash.start(); 326 iHash.put(iKeyPad[]); 327 iHash.put(message); 328 auto iDigest = iHash.finish(); 329 330 oHash.put(iDigest[]); 331 auto oDigest = oHash.finish(); 332 333 return oDigest; 334 } 335 } 336 337 auto HMAC_MD5 ()(in ubyte[] key, in ubyte[] message) { import std.digest.md ; return HMAC!(MD5 , 64)(key, message); } 338 auto HMAC_SHA1 ()(in ubyte[] key, in ubyte[] message) { import std.digest.sha; return HMAC!(SHA1 , 64)(key, message); } 339 auto HMAC_SHA256 ()(in ubyte[] key, in ubyte[] message) { import std.digest.sha; return HMAC!(SHA256, 64)(key, message); } 340 341 unittest 342 { 343 import std..string : representation; 344 import std.conv : hexString; 345 346 assert(HMAC_MD5([], []) == hexString!"74e6f7298a9c2d168935f58c001bad88".representation); 347 assert(HMAC_SHA1([], []) == hexString!"fbdb1d1b18aa6c08324b7d64b71fb76370690e1d".representation); 348 assert(HMAC_SHA256([], []) == hexString!"b613679a0814d9ec772f95d778c35fc5ff1697c493715653c6c712144292c5ad".representation); 349 350 auto message = "The quick brown fox jumps over the lazy dog".representation; 351 auto key = "key".representation; 352 assert(HMAC_MD5 (key, message) == hexString!"80070713463e7749b90c2dc24911e275".representation); 353 assert(HMAC_SHA1 (key, message) == hexString!"de7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9".representation); 354 assert(HMAC_SHA256(key, message) == hexString!"f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8".representation); 355 } 356 }