1 /**
2  * Data digest (hashes etc.)
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Vladimir Panteleev <ae@cy.md>
12  */
13 
14 module ae.utils.digest;
15 
16 import std.traits;
17 
18 import ae.sys.cmd;
19 import ae.utils.text;
20 import ae.utils.array;
21 
22 deprecated ("Use std.digest.hmac")
23 ubyte[] hmac_sha1(string message, ubyte[] privateKey)
24 {
25 	return
26 		arrayFromHex(sha1sum(vector!("^")(padRight(privateKey, 64, cast(ubyte)0x00), repeatOne(cast(ubyte)0x5c, 64)) ~
27 		arrayFromHex(sha1sum(vector!("^")(padRight(privateKey, 64, cast(ubyte)0x00), repeatOne(cast(ubyte)0x36, 64)) ~
28 		cast(ubyte[])message))));
29 }
30 
31 // the standard Phobos crc32 function relies on inlining for usable performance
32 /// Inlinable CRC32 implementation.
33 uint[256] crc32Table = [
34 	0x00000000,0x77073096,0xee0e612c,0x990951ba,0x076dc419,0x706af48f,0xe963a535,0x9e6495a3,0x0edb8832,0x79dcb8a4,0xe0d5e91e,0x97d2d988,0x09b64c2b,0x7eb17cbd,0xe7b82d07,0x90bf1d91,
35 	0x1db71064,0x6ab020f2,0xf3b97148,0x84be41de,0x1adad47d,0x6ddde4eb,0xf4d4b551,0x83d385c7,0x136c9856,0x646ba8c0,0xfd62f97a,0x8a65c9ec,0x14015c4f,0x63066cd9,0xfa0f3d63,0x8d080df5,
36 	0x3b6e20c8,0x4c69105e,0xd56041e4,0xa2677172,0x3c03e4d1,0x4b04d447,0xd20d85fd,0xa50ab56b,0x35b5a8fa,0x42b2986c,0xdbbbc9d6,0xacbcf940,0x32d86ce3,0x45df5c75,0xdcd60dcf,0xabd13d59,
37 	0x26d930ac,0x51de003a,0xc8d75180,0xbfd06116,0x21b4f4b5,0x56b3c423,0xcfba9599,0xb8bda50f,0x2802b89e,0x5f058808,0xc60cd9b2,0xb10be924,0x2f6f7c87,0x58684c11,0xc1611dab,0xb6662d3d,
38 	0x76dc4190,0x01db7106,0x98d220bc,0xefd5102a,0x71b18589,0x06b6b51f,0x9fbfe4a5,0xe8b8d433,0x7807c9a2,0x0f00f934,0x9609a88e,0xe10e9818,0x7f6a0dbb,0x086d3d2d,0x91646c97,0xe6635c01,
39 	0x6b6b51f4,0x1c6c6162,0x856530d8,0xf262004e,0x6c0695ed,0x1b01a57b,0x8208f4c1,0xf50fc457,0x65b0d9c6,0x12b7e950,0x8bbeb8ea,0xfcb9887c,0x62dd1ddf,0x15da2d49,0x8cd37cf3,0xfbd44c65,
40 	0x4db26158,0x3ab551ce,0xa3bc0074,0xd4bb30e2,0x4adfa541,0x3dd895d7,0xa4d1c46d,0xd3d6f4fb,0x4369e96a,0x346ed9fc,0xad678846,0xda60b8d0,0x44042d73,0x33031de5,0xaa0a4c5f,0xdd0d7cc9,
41 	0x5005713c,0x270241aa,0xbe0b1010,0xc90c2086,0x5768b525,0x206f85b3,0xb966d409,0xce61e49f,0x5edef90e,0x29d9c998,0xb0d09822,0xc7d7a8b4,0x59b33d17,0x2eb40d81,0xb7bd5c3b,0xc0ba6cad,
42 	0xedb88320,0x9abfb3b6,0x03b6e20c,0x74b1d29a,0xead54739,0x9dd277af,0x04db2615,0x73dc1683,0xe3630b12,0x94643b84,0x0d6d6a3e,0x7a6a5aa8,0xe40ecf0b,0x9309ff9d,0x0a00ae27,0x7d079eb1,
43 	0xf00f9344,0x8708a3d2,0x1e01f268,0x6906c2fe,0xf762575d,0x806567cb,0x196c3671,0x6e6b06e7,0xfed41b76,0x89d32be0,0x10da7a5a,0x67dd4acc,0xf9b9df6f,0x8ebeeff9,0x17b7be43,0x60b08ed5,
44 	0xd6d6a3e8,0xa1d1937e,0x38d8c2c4,0x4fdff252,0xd1bb67f1,0xa6bc5767,0x3fb506dd,0x48b2364b,0xd80d2bda,0xaf0a1b4c,0x36034af6,0x41047a60,0xdf60efc3,0xa867df55,0x316e8eef,0x4669be79,
45 	0xcb61b38c,0xbc66831a,0x256fd2a0,0x5268e236,0xcc0c7795,0xbb0b4703,0x220216b9,0x5505262f,0xc5ba3bbe,0xb2bd0b28,0x2bb45a92,0x5cb36a04,0xc2d7ffa7,0xb5d0cf31,0x2cd99e8b,0x5bdeae1d,
46 	0x9b64c2b0,0xec63f226,0x756aa39c,0x026d930a,0x9c0906a9,0xeb0e363f,0x72076785,0x05005713,0x95bf4a82,0xe2b87a14,0x7bb12bae,0x0cb61b38,0x92d28e9b,0xe5d5be0d,0x7cdcefb7,0x0bdbdf21,
47 	0x86d3d2d4,0xf1d4e242,0x68ddb3f8,0x1fda836e,0x81be16cd,0xf6b9265b,0x6fb077e1,0x18b74777,0x88085ae6,0xff0f6a70,0x66063bca,0x11010b5c,0x8f659eff,0xf862ae69,0x616bffd3,0x166ccf45,
48 	0xa00ae278,0xd70dd2ee,0x4e048354,0x3903b3c2,0xa7672661,0xd06016f7,0x4969474d,0x3e6e77db,0xaed16a4a,0xd9d65adc,0x40df0b66,0x37d83bf0,0xa9bcae53,0xdebb9ec5,0x47b2cf7f,0x30b5ffe9,
49 	0xbdbdf21c,0xcabac28a,0x53b39330,0x24b4a3a6,0xbad03605,0xcdd70693,0x54de5729,0x23d967bf,0xb3667a2e,0xc4614ab8,0x5d681b02,0x2a6f2b94,0xb40bbe37,0xc30c8ea1,0x5a05df1b,0x2d02ef8d,
50 ];
51 
52 /// ditto
53 uint fastCRC(const(void)[] data, uint start = cast(uint)-1)
54 {
55 	uint crc = start;
56 	foreach (val; cast(ubyte[])data)
57 		crc = crc32Table[cast(ubyte) crc ^ val] ^ (crc >> 8);
58 	return crc;
59 }
60 
61 
62 // Struct (for streaming)
63 /// D port of MurmurHash2A.
64 /// TODO: reimplement via std.digest
65 struct MurmurHash2A
66 {
67 	private static string mmix(string h, string k) { return "{ "~k~" *= m; "~k~" ^= "~k~" >> r; "~k~" *= m; "~h~" *= m; "~h~" ^= "~k~"; }"; }
68 
69 public:
70 	///
71 	void Begin ( uint seed = 0 )
72 	{
73 		m_hash  = seed;
74 		m_tail  = 0;
75 		m_count = 0;
76 		m_size  = 0;
77 	}
78 
79 	///
80 	void Add ( const(void) * vdata, sizediff_t len )
81 	{
82 		ubyte * data = cast(ubyte*)vdata;
83 		m_size += len;
84 
85 		MixTail(data,len);
86 
87 		while(len >= 4)
88 		{
89 			uint k = *cast(uint*)data;
90 
91 			mixin(mmix("m_hash","k"));
92 
93 			data += 4;
94 			len -= 4;
95 		}
96 
97 		MixTail(data,len);
98 	}
99 
100 	///
101 	uint End ( )
102 	{
103 		mixin(mmix("m_hash","m_tail"));
104 		mixin(mmix("m_hash","m_size"));
105 
106 		m_hash ^= m_hash >> 13;
107 		m_hash *= m;
108 		m_hash ^= m_hash >> 15;
109 
110 		return m_hash;
111 	}
112 
113 	/// `Add` overloads for some common D types.
114 	void Add(ref ubyte v) { Add(&v, v.sizeof); } /// ditto
115 	void Add(ref int v) { Add(&v, v.sizeof); } /// ditto
116 	void Add(ref uint v) { Add(&v, v.sizeof); } /// ditto
117 	void Add(string s) { Add(s.ptr, cast(uint)s.length); } /// ditto
118 	void Add(ubyte[] s) { Add(s.ptr, cast(uint)s.length); } /// ditto
119 
120 private:
121 
122 	static const uint m = 0x5bd1e995;
123 	static const int r = 24;
124 
125 	void MixTail ( ref ubyte * data, ref sizediff_t len )
126 	{
127 		while( len && ((len<4) || m_count) )
128 		{
129 			m_tail |= (*data++) << (m_count * 8);
130 
131 			m_count++;
132 			len--;
133 
134 			if(m_count == 4)
135 			{
136 				mixin(mmix("m_hash","m_tail"));
137 				m_tail = 0;
138 				m_count = 0;
139 			}
140 		}
141 	}
142 
143 	uint m_hash;
144 	uint m_tail;
145 	uint m_count;
146 	uint m_size;
147 }
148 
149 /// ditto
150 uint murmurHash2(const(void)[] data, uint seed=0)
151 {
152 	enum { m = 0x5bd1e995, r = 24 }
153 	uint len = cast(uint)data.length;
154 	uint h = seed ^ len;
155 	ubyte* p = cast(ubyte*)data.ptr;
156 
157 	while (len >= 4)
158 	{
159 		uint k = *cast(uint*)p;
160 
161 		k *= m;
162 		k ^= k >> r;
163 		k *= m;
164 
165 		h *= m;
166 		h ^= k;
167 
168 		p += 4;
169 		len -= 4;
170 	}
171 
172 	switch(len)
173 	{
174 		case 3: h ^= p[2] << 16; goto case;
175 		case 2: h ^= p[1] << 8;  goto case;
176 		case 1: h ^= p[0];
177 		/*   */ h *= m;          goto case;
178 		case 0: break;
179 		default: assert(0);
180 	}
181 
182 	// Do a few final mixes of the hash to ensure the last few
183 	// bytes are well-incorporated.
184 
185 	h ^= h >> 13;
186 	h *= m;
187 	h ^= h >> 15;
188 
189 	return h;
190 }
191 
192 deprecated import ae.utils.digest_murmurhash3;
193 
194 deprecated ("Use std.digest.murmurhash")
195 {
196 	uint murmurHash3_32(const(void)[] data, uint seed=0)
197 	{
198 		uint result;
199 		MurmurHash3_x86_32(data.ptr, cast(uint)data.length, seed, &result);
200 		return result;
201 	}
202 
203 	alias uint[4] MH3Digest128;
204 
205 	MH3Digest128 murmurHash3_x86_128(const(void)[] data, uint seed=0)
206 	{
207 		MH3Digest128 result;
208 		MurmurHash3_x86_128(data.ptr, cast(uint)data.length, seed, &result);
209 		return result;
210 	}
211 
212 	MH3Digest128 murmurHash3_x64_128(const(void)[] data, uint seed=0)
213 	{
214 		MH3Digest128 result;
215 		MurmurHash3_x64_128(data.ptr, cast(uint)data.length, seed, &result);
216 		return result;
217 	}
218 
219 	/// Select version optimized for target platform.
220 	/// WARNING: Output depends on platform.
221 	version(D_LP64)
222 		alias murmurHash3_x64_128 murmurHash3_128;
223 	else
224 		alias murmurHash3_x86_128 murmurHash3_128;
225 
226 	string digestToStringMH3(MH3Digest128 digest)
227 	{
228 		import std.string;
229 		return format("%08X%08X%08X%08X", digest[0], digest[1], digest[2], digest[3]);
230 	}
231 
232 	unittest
233 	{
234 		assert(murmurHash3_32("The quick brown fox jumps over the lazy dog") == 0x2e4ff723);
235 		assert(murmurHash3_32("The quick brown fox jumps over the lazy cog") == 0xf08200fc);
236 
237 		assert(murmurHash3_x86_128("The quick brown fox jumps over the lazy dog") == [ 0x2f1583c3 , 0xecee2c67 , 0x5d7bf66c , 0xe5e91d2c ]);
238 		assert(murmurHash3_x86_128("The quick brown fox jumps over the lazy cog") == [ 0x0ed64388 , 0x3e9ae779 , 0x97034593 , 0x49b3f32f ]);
239 
240 		assert(murmurHash3_x64_128("The quick brown fox jumps over the lazy dog") == [ 0xbc071b6c , 0xe34bbc7b , 0xc49a9347 , 0x7a433ca9 ]);
241 		assert(murmurHash3_x64_128("The quick brown fox jumps over the lazy cog") == [ 0xff85269a , 0x658ca970 , 0xa68e5c3e , 0x43fee3ea ]);
242 
243 		assert(digestToStringMH3(murmurHash3_x86_128("The quick brown fox jumps over the lazy dog")) == "2F1583C3ECEE2C675D7BF66CE5E91D2C");
244 	}
245 }
246 
247 // ************************************************************************
248 
249 deprecated("legacy transitive import - please `import std.digest.md;`.")
250 public import std.digest.md;
251 
252 /// Get digest string of given data.
253 /// Short-hand for std.digest.md5Of (and similar) and toHexString.
254 /// Similar to the old std.md5.getDigestString.
255 template getDigestString(Digest)
256 {
257 	string getDigestString(T)(const(T)[][] data...)
258 		if (!hasIndirections!T)
259 	{
260 		return getDigestStringImpl(arrOfArrCastInPlace!(const(ubyte))(data));
261 	}
262 
263 	// A dirty hack to fix the array lengths right on the stack, to avoid an allocation.
264 	T[][] arrOfArrCastInPlace(T, S)(S[][] arrs)
265 	{
266 		static struct Array
267 		{
268 			size_t length;
269 			void* ptr;
270 		}
271 		auto rawArrs = cast(Array[])arrs;
272 		foreach (ref ra; rawArrs)
273 			ra.length = ra.length * S.sizeof / T.sizeof;
274 		return cast(T[][])rawArrs;
275 	}
276 
277 	string getDigestStringImpl(in ubyte[][] data)
278 	{
279 		Digest digest;
280 		digest.start();
281 		foreach (datum; data)
282 			digest.put(cast(const(ubyte)[])datum);
283 		auto result = digest.finish();
284 		// https://issues.dlang.org/show_bug.cgi?id=9279
285 		auto str = result.toHexString();
286 		return str[].idup;
287 	}
288 }
289 
290 ///
291 unittest
292 {
293 	assert(getDigestString!MD5("abc") == "900150983CD24FB0D6963F7D28E17F72");
294 }
295 
296 // ************************************************************************
297 
298 deprecated("Use std.digest.hmac")
299 {
300 	/// HMAC digest with a hash algorithm.
301 	/// Params:
302 	///   Algorithm = std.digest-compatible hash type
303 	///   blockSize = Algorithm block size, in bytes
304 	///   key       = Secret key bytes
305 	///   message   = Message data
306 	template HMAC(alias Algorithm, size_t blockSize)
307 	{
308 		alias Digest = typeof(Algorithm.init.finish());
309 
310 		Digest HMAC(in ubyte[] key, in ubyte[] message)
311 		{
312 			ubyte[blockSize] keyBlock = 0;
313 			if (key.length > blockSize)
314 				keyBlock[0..Digest.length] = digest!Algorithm(key)[];
315 			else
316 				keyBlock[0..key.length] = key[];
317 
318 			ubyte[blockSize] oKeyPad = 0x5C; oKeyPad[] ^= keyBlock[];
319 			ubyte[blockSize] iKeyPad = 0x36; iKeyPad[] ^= keyBlock[];
320 
321 			Algorithm oHash;
322 			oHash.start();
323 			oHash.put(oKeyPad[]);
324 
325 			Algorithm iHash;
326 			iHash.start();
327 			iHash.put(iKeyPad[]);
328 			iHash.put(message);
329 			auto iDigest = iHash.finish();
330 
331 			oHash.put(iDigest[]);
332 			auto oDigest = oHash.finish();
333 
334 			return oDigest;
335 		}
336 	}
337 
338 	auto HMAC_MD5    ()(in ubyte[] key, in ubyte[] message) { import std.digest.md ; return HMAC!(MD5   , 64)(key, message); }
339 	auto HMAC_SHA1   ()(in ubyte[] key, in ubyte[] message) { import std.digest.sha; return HMAC!(SHA1  , 64)(key, message); }
340 	auto HMAC_SHA256 ()(in ubyte[] key, in ubyte[] message) { import std.digest.sha; return HMAC!(SHA256, 64)(key, message); }
341 
342 	unittest
343 	{
344 		import std.string : representation;
345 		import std.conv : hexString;
346 
347 		assert(HMAC_MD5([], []) == hexString!"74e6f7298a9c2d168935f58c001bad88".representation);
348 		assert(HMAC_SHA1([], []) == hexString!"fbdb1d1b18aa6c08324b7d64b71fb76370690e1d".representation);
349 		assert(HMAC_SHA256([], []) == hexString!"b613679a0814d9ec772f95d778c35fc5ff1697c493715653c6c712144292c5ad".representation);
350 
351 		auto message = "The quick brown fox jumps over the lazy dog".representation;
352 		auto key = "key".representation;
353 		assert(HMAC_MD5   (key, message) == hexString!"80070713463e7749b90c2dc24911e275".representation);
354 		assert(HMAC_SHA1  (key, message) == hexString!"de7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9".representation);
355 		assert(HMAC_SHA256(key, message) == hexString!"f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8".representation);
356 	}
357 }