1 /**
2  * JSON encoding.
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Vladimir Panteleev <vladimir@thecybershadow.net>
12  */
13 
14 module ae.utils.json;
15 
16 import std.exception;
17 import std.string;
18 import std.traits;
19 import std.typecons;
20 
21 import ae.utils.exception;
22 import ae.utils.meta;
23 import ae.utils.textout;
24 
25 // ************************************************************************
26 
27 struct CustomJsonWriter(WRITER)
28 {
29 	/// You can set this to something to e.g. write to another buffer.
30 	WRITER output;
31 
32 	void putString(in char[] s)
33 	{
34 		// TODO: escape Unicode characters?
35 		// TODO: Handle U+2028 and U+2029 ( http://timelessrepo.com/json-isnt-a-javascript-subset )
36 
37 		output.put('"');
38 		auto start = s.ptr, p = start, end = start+s.length;
39 
40 		while (p < end)
41 		{
42 			auto c = *p++;
43 			if (Escapes.escaped[c])
44 				output.put(start[0..p-start-1], Escapes.chars[c]),
45 				start = p;
46 		}
47 
48 		output.put(start[0..p-start], '"');
49 	}
50 
51 	void put(T)(T v)
52 	{
53 		static if (is(T == enum))
54 			put(to!string(v));
55 		else
56 		static if (is(T : const(char)[]))
57 			putString(v);
58 		else
59 		static if (is(T == bool))
60 			return output.put(v ? "true" : "false");
61 		else
62 		static if (is(T : long))
63 			return .put(output, v);
64 		else
65 		static if (is(T : real))
66 			return output.put(fpToString!T(v)); // TODO: don't allocate
67 		else
68 		static if (is(T U : U[]))
69 		{
70 			output.put('[');
71 			if (v.length)
72 			{
73 				put(v[0]);
74 				foreach (i; v[1..$])
75 				{
76 					output.put(',');
77 					put(i);
78 				}
79 			}
80 			output.put(']');
81 		}
82 		else
83 		static if (isTuple!T)
84 		{
85 			// TODO: serialize as object if tuple has names
86 			enum N = v.expand.length;
87 			static if (N == 0)
88 				return;
89 			else
90 			static if (N == 1)
91 				put(v.expand[0]);
92 			else
93 			{
94 				output.put('[');
95 				foreach (n; RangeTuple!N)
96 				{
97 					static if (n)
98 						output.put(',');
99 					put(v.expand[n]);
100 				}
101 				output.put(']');
102 			}
103 		}
104 		else
105 		static if (is(T==struct))
106 		{
107 			output.put('{');
108 			bool first = true;
109 			foreach (i, field; v.tupleof)
110 			{
111 				static if (!doSkipSerialize!(T, v.tupleof[i].stringof[2..$]))
112 				{
113 					if (!first)
114 						output.put(',');
115 					else
116 						first = false;
117 					put(getJsonName!(T, v.tupleof[i].stringof[2..$]));
118 					output.put(':');
119 					put(field);
120 				}
121 			}
122 			output.put('}');
123 		}
124 		else
125 		static if (isAssociativeArray!T)
126 		{
127 			output.put('{');
128 			bool first = true;
129 			foreach (key, value; v)
130 			{
131 				if (!first)
132 					output.put(',');
133 				else
134 					first = false;
135 				put(key);
136 				output.put(':');
137 				put(value);
138 			}
139 			output.put('}');
140 		}
141 		else
142 		static if (is(typeof(*v)))
143 		{
144 			if (v)
145 				put(*v);
146 			else
147 				output.put("null");
148 		}
149 		else
150 			static assert(0, "Can't serialize " ~ T.stringof ~ " to JSON");
151 	}
152 }
153 
154 alias CustomJsonWriter!StringBuilder JsonWriter;
155 
156 private struct Escapes
157 {
158 	static __gshared string[256] chars;
159 	static __gshared bool[256] escaped;
160 
161 	shared static this()
162 	{
163 		import std.string;
164 
165 		escaped[] = true;
166 		foreach (c; 0..256)
167 			if (c=='\\')
168 				chars[c] = `\\`;
169 			else
170 			if (c=='\"')
171 				chars[c] = `\"`;
172 			else
173 			if (c=='\b')
174 				chars[c] = `\b`;
175 			else
176 			if (c=='\f')
177 				chars[c] = `\f`;
178 			else
179 			if (c=='\n')
180 				chars[c] = `\n`;
181 			else
182 			if (c=='\r')
183 				chars[c] = `\r`;
184 			else
185 			if (c=='\t')
186 				chars[c] = `\t`;
187 			else
188 			if (c<'\x20' || c == '\x7F' || c=='<' || c=='>' || c=='&')
189 				chars[c] = format(`\u%04x`, c);
190 			else
191 				chars[c] = [cast(char)c],
192 				escaped[c] = false;
193 	}
194 }
195 
196 // ************************************************************************
197 
198 string toJson(T)(T v)
199 {
200 	JsonWriter writer;
201 	writer.put(v);
202 	return writer.output.get();
203 }
204 
205 unittest
206 {
207 	struct X { int a; string b; }
208 	X x = {17, "aoeu"};
209 	assert(toJson(x) == `{"a":17,"b":"aoeu"}`);
210 	int[] arr = [1,5,7];
211 	assert(toJson(arr) == `[1,5,7]`);
212 	assert(toJson(true) == `true`);
213 
214 	assert(toJson(tuple()) == ``);
215 	assert(toJson(tuple(42)) == `42`);
216 	assert(toJson(tuple(42, "banana")) == `[42,"banana"]`);
217 }
218 
219 // ************************************************************************
220 
221 import std.ascii;
222 import std.utf;
223 import std.conv;
224 
225 import ae.utils.text;
226 
227 private struct JsonParser(C)
228 {
229 	C[] s;
230 	size_t p;
231 
232 	char next()
233 	{
234 		enforce(p < s.length);
235 		return s[p++];
236 	}
237 
238 	string readN(uint n)
239 	{
240 		string r;
241 		for (int i=0; i<n; i++)
242 			r ~= next();
243 		return r;
244 	}
245 
246 	char peek()
247 	{
248 		enforce(p < s.length);
249 		return s[p];
250 	}
251 
252 	@property bool eof() { return p == s.length; }
253 
254 	void skipWhitespace()
255 	{
256 		while (isWhite(peek()))
257 			p++;
258 	}
259 
260 	void expect(char c)
261 	{
262 		auto n = next();
263 		enforce(n==c, "Expected " ~ c ~ ", got " ~ n);
264 	}
265 
266 	T read(T)()
267 	{
268 		static if (is(T X == Nullable!X))
269 			return readNullable!X();
270 		else
271 		static if (is(T==enum))
272 			return readEnum!(T)();
273 		else
274 		static if (is(T==string))
275 			return readString();
276 		else
277 		static if (is(T==bool))
278 			return readBool();
279 		else
280 		static if (is(T : real))
281 			return readNumber!(T)();
282 		else
283 		static if (isDynamicArray!T)
284 			return readArray!(typeof(T.init[0]))();
285 		else
286 		static if (isStaticArray!T)
287 		{
288 			T result = readArray!(typeof(T.init[0]))()[];
289 			return result;
290 		}
291 		else
292 		static if (isTuple!T)
293 			return readTuple!T();
294 		else
295 		static if (is(T==struct))
296 			return readObject!(T)();
297 		else
298 		static if (is(typeof(T.init.keys)) && is(typeof(T.init.values)) && is(typeof(T.init.keys[0])==string))
299 			return readAA!(T)();
300 		else
301 		static if (is(T U : U*))
302 			return readPointer!T();
303 		else
304 			static assert(0, "Can't decode " ~ T.stringof ~ " from JSON");
305 	}
306 
307 	auto readTuple(T)()
308 	{
309 		// TODO: serialize as object if tuple has names
310 		enum N = T.expand.length;
311 		static if (N == 0)
312 			return T();
313 		else
314 		static if (N == 1)
315 			return T(read!(typeof(T.expand[0])));
316 		else
317 		{
318 			T v;
319 			expect('[');
320 			foreach (n, ref f; v.expand)
321 			{
322 				static if (n)
323 					expect(',');
324 				f = read!(typeof(f));
325 			}
326 			expect(']');
327 			return v;
328 		}
329 	}
330 
331 	auto readNullable(T)()
332 	{
333 		if (peek() == 'n')
334 		{
335 			next();
336 			expect('u');
337 			expect('l');
338 			expect('l');
339 			return Nullable!T();
340 		}
341 		else
342 			return Nullable!T(read!T);
343 	}
344 
345 	string readString()
346 	{
347 		skipWhitespace();
348 		auto c = peek();
349 		if (c == '"')
350 		{
351 			next(); // '"'
352 			string result;
353 			while (true)
354 			{
355 				c = next();
356 				if (c=='"')
357 					break;
358 				else
359 				if (c=='\\')
360 					switch (next())
361 					{
362 						case '"':  result ~= '"'; break;
363 						case '/':  result ~= '/'; break;
364 						case '\\': result ~= '\\'; break;
365 						case 'b':  result ~= '\b'; break;
366 						case 'f':  result ~= '\f'; break;
367 						case 'n':  result ~= '\n'; break;
368 						case 'r':  result ~= '\r'; break;
369 						case 't':  result ~= '\t'; break;
370 						case 'u':
371 						{
372 							wstring buf;
373 							goto Unicode_start;
374 
375 							while (s[p..$].startsWith(`\u`))
376 							{
377 								p+=2;
378 							Unicode_start:
379 								buf ~= cast(wchar)fromHex!ushort(readN(4));
380 							}
381 							result ~= toUTF8(buf);
382 							break;
383 						}
384 						default: enforce(false, "Unknown escape");
385 					}
386 				else
387 					result ~= c;
388 			}
389 			return result;
390 		}
391 		else
392 		if (isDigit(c) || c=='-') // For languages that don't distinguish numeric strings from numbers
393 		{
394 			static immutable bool[256] numeric =
395 			[
396 				'0':true,
397 				'1':true,
398 				'2':true,
399 				'3':true,
400 				'4':true,
401 				'5':true,
402 				'6':true,
403 				'7':true,
404 				'8':true,
405 				'9':true,
406 				'.':true,
407 				'-':true,
408 				'+':true,
409 				'e':true,
410 				'E':true,
411 			];
412 
413 			string s;
414 			while (c=peek(), numeric[c])
415 				s ~= c, p++;
416 			return s;
417 		}
418 		else
419 		{
420 			foreach (n; "null")
421 				expect(n);
422 			return null;
423 		}
424 	}
425 
426 	bool readBool()
427 	{
428 		skipWhitespace();
429 		if (peek()=='t')
430 		{
431 			enforce(readN(4) == "true", "Bad boolean");
432 			return true;
433 		}
434 		else
435 		if (peek()=='f')
436 		{
437 			enforce(readN(5) == "false", "Bad boolean");
438 			return false;
439 		}
440 		else
441 		{
442 			ubyte i = readNumber!ubyte();
443 			enforce(i < 2);
444 			return !!i;
445 		}
446 	}
447 
448 	T readNumber(T)()
449 	{
450 		skipWhitespace();
451 		T v;
452 		string s;
453 		char c = peek();
454 		if (c == '"')
455 			s = readString();
456 		else
457 			while (c=='+' || c=='-' || (c>='0' && c<='9') || c=='e' || c=='E' || c=='.')
458 			{
459 				s ~= c, p++;
460 				if (eof) break;
461 				c=peek();
462 			}
463 		static if (is(T : real))
464 			return to!T(s);
465 		else
466 			static assert(0, "Don't know how to parse numerical type " ~ T.stringof);
467 	}
468 
469 	T[] readArray(T)()
470 	{
471 		skipWhitespace();
472 		expect('[');
473 		skipWhitespace();
474 		T[] result;
475 		if (peek()==']')
476 		{
477 			p++;
478 			return result;
479 		}
480 		while(true)
481 		{
482 			result ~= read!(T)();
483 			skipWhitespace();
484 			if (peek()==']')
485 			{
486 				p++;
487 				return result;
488 			}
489 			else
490 				expect(',');
491 		}
492 	}
493 
494 	T readObject(T)()
495 	{
496 		skipWhitespace();
497 		expect('{');
498 		skipWhitespace();
499 		T v;
500 		if (peek()=='}')
501 		{
502 			p++;
503 			return v;
504 		}
505 
506 		while (true)
507 		{
508 			string jsonField = readString();
509 			mixin(exceptionContext(q{"Error with field " ~ jsonField}));
510 			skipWhitespace();
511 			expect(':');
512 
513 			bool found;
514 			foreach (i, field; v.tupleof)
515 			{
516 				enum name = getJsonName!(T, v.tupleof[i].stringof[2..$]);
517 				if (name == jsonField)
518 				{
519 					v.tupleof[i] = read!(typeof(v.tupleof[i]))();
520 					found = true;
521 					break;
522 				}
523 			}
524 			enforce(found, "Unknown field " ~ jsonField);
525 
526 			skipWhitespace();
527 			if (peek()=='}')
528 			{
529 				p++;
530 				return v;
531 			}
532 			else
533 				expect(',');
534 		}
535 	}
536 
537 	T readAA(T)()
538 	{
539 		skipWhitespace();
540 		expect('{');
541 		skipWhitespace();
542 		T v;
543 		if (peek()=='}')
544 		{
545 			p++;
546 			return v;
547 		}
548 
549 		while (true)
550 		{
551 			string jsonField = readString();
552 			skipWhitespace();
553 			expect(':');
554 
555 			v[jsonField] = read!(typeof(v.values[0]))();
556 
557 			skipWhitespace();
558 			if (peek()=='}')
559 			{
560 				p++;
561 				return v;
562 			}
563 			else
564 				expect(',');
565 		}
566 	}
567 
568 	T readEnum(T)()
569 	{
570 		return to!T(readString());
571 	}
572 
573 	T readPointer(T)()
574 	{
575 		skipWhitespace();
576 		if (peek()=='n')
577 		{
578 			enforce(readN(4) == "null", "Null expected");
579 			return null;
580 		}
581 		alias typeof(*T.init) S;
582 		T v = new S;
583 		*v = read!S();
584 		return v;
585 	}
586 }
587 
588 T jsonParse(T, C)(C[] s)
589 {
590 	auto parser = JsonParser!C(s);
591 	mixin(exceptionContext(q{format("Error at position %d", parser.p)}));
592 	return parser.read!T();
593 }
594 
595 unittest
596 {
597 	struct S { int i; S[] arr; S* p0, p1; }
598 	S s = S(42, [S(1), S(2)], null, new S(15));
599 	auto s2 = jsonParse!S(toJson(s));
600 	//assert(s == s2); // Issue 3789
601 	assert(s.i == s2.i && s.arr == s2.arr && s.p0 is s2.p0 && *s.p1 == *s2.p1);
602 	jsonParse!S(toJson(s).dup);
603 
604 	assert(jsonParse!(Tuple!())(``) == tuple());
605 	assert(jsonParse!(Tuple!int)(`42`) == tuple(42));
606 	assert(jsonParse!(Tuple!(int, string))(`[42, "banana"]`) == tuple(42, "banana"));
607 }
608 
609 // ************************************************************************
610 
611 // TODO: migrate to UDAs
612 
613 /**
614  * A template that designates fields which should not be serialized to Json.
615  *
616  * Example:
617  * ---
618  * struct Point { int x, y, z; mixin NonSerialized!(x, z); }
619  * assert(jsonParse!Point(toJson(Point(1, 2, 3))) == Point(0, 2, 0));
620  * ---
621  */
622 template NonSerialized(fields...)
623 {
624 	mixin(NonSerializedFields(stringofArray!fields()));
625 }
626 
627 private string NonSerializedFields(string[] fields)
628 {
629 	string result;
630 	foreach (field; fields)
631 		result ~= "enum bool " ~ field ~ "_nonSerialized = 1;";
632 	return result;
633 }
634 
635 private template doSkipSerialize(T, string member)
636 {
637 	enum bool doSkipSerialize = __traits(hasMember, T, member ~ "_nonSerialized");
638 }
639 
640 unittest
641 {
642 	struct Point { int x, y, z; mixin NonSerialized!(x, z); }
643 	assert(jsonParse!Point(toJson(Point(1, 2, 3))) == Point(0, 2, 0));
644 }
645 
646 unittest
647 {
648 	enum En { one, two }
649 	struct S { int i1, i2; S[] arr1, arr2; string[string] dic; En en; mixin NonSerialized!(i2, arr2); }
650 	S s = S(42, 5, [S(1), S(2)], [S(3), S(4)], ["apple":"fruit", "pizza":"vegetable"], En.two);
651 	auto s2 = jsonParse!S(toJson(s));
652 	assert(s.i1 == s2.i1 && s2.i2 is int.init && s.arr1 == s2.arr1 && s2.arr2 is null && s.dic == s2.dic && s.en == En.two);
653 }
654 
655 unittest
656 {
657 	alias B = Nullable!bool;
658 	B b;
659 
660 	b = jsonParse!B("true");
661 	assert(!b.isNull);
662 	assert(b == true);
663 
664 	b = jsonParse!B("false");
665 	assert(!b.isNull);
666 	assert(b == false);
667 
668 	b = jsonParse!B("null");
669 	assert(b.isNull);
670 }
671 
672 // ************************************************************************
673 
674 /// User-defined attribute - specify name for JSON object field.
675 /// Useful when a JSON object may contain fields, the name of which are not valid D identifiers.
676 struct JSONName { string name; }
677 
678 private template getJsonName(S, string FIELD)
679 {
680 	static if (hasAttribute!(JSONName, __traits(getMember, S, FIELD)))
681 		enum getJsonName = getAttribute!(JSONName, __traits(getMember, S, FIELD)).name;
682 	else
683 		enum getJsonName = FIELD;
684 }