1 /**
2  * JSON encoding.
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Vladimir Panteleev <vladimir@thecybershadow.net>
12  */
13 
14 module ae.utils.json;
15 
16 import std.exception;
17 import std.string;
18 import std.traits;
19 import std.typecons;
20 
21 import ae.utils.exception;
22 import ae.utils.meta;
23 import ae.utils.textout;
24 
25 // ************************************************************************
26 
27 struct CustomJsonWriter(WRITER)
28 {
29 	/// You can set this to something to e.g. write to another buffer.
30 	WRITER output;
31 
32 	void putString(in char[] s)
33 	{
34 		// TODO: escape Unicode characters?
35 		// TODO: Handle U+2028 and U+2029 ( http://timelessrepo.com/json-isnt-a-javascript-subset )
36 
37 		output.put('"');
38 		auto start = s.ptr, p = start, end = start+s.length;
39 
40 		while (p < end)
41 		{
42 			auto c = *p++;
43 			if (Escapes.escaped[c])
44 				output.put(start[0..p-start-1], Escapes.chars[c]),
45 				start = p;
46 		}
47 
48 		output.put(start[0..p-start], '"');
49 	}
50 
51 	void put(T)(T v)
52 	{
53 		static if (is(T == enum))
54 			put(to!string(v));
55 		else
56 		static if (is(T : const(char)[]))
57 			putString(v);
58 		else
59 		static if (is(T == bool))
60 			return output.put(v ? "true" : "false");
61 		else
62 		static if (is(T : long))
63 			return .put(output, v);
64 		else
65 		static if (is(T : real))
66 			return output.put(fpToString!T(v)); // TODO: don't allocate
67 		else
68 		static if (is(T U : U[]))
69 		{
70 			output.put('[');
71 			if (v.length)
72 			{
73 				put(v[0]);
74 				foreach (i; v[1..$])
75 				{
76 					output.put(',');
77 					put(i);
78 				}
79 			}
80 			output.put(']');
81 		}
82 		else
83 		static if (isTuple!T)
84 		{
85 			// TODO: serialize as object if tuple has names
86 			enum N = v.expand.length;
87 			static if (N == 0)
88 				return;
89 			else
90 			static if (N == 1)
91 				put(v.expand[0]);
92 			else
93 			{
94 				output.put('[');
95 				foreach (n; RangeTuple!N)
96 				{
97 					static if (n)
98 						output.put(',');
99 					put(v.expand[n]);
100 				}
101 				output.put(']');
102 			}
103 		}
104 		else
105 		static if (is(T==struct))
106 		{
107 			output.put('{');
108 			bool first = true;
109 			foreach (i, field; v.tupleof)
110 			{
111 				static if (!doSkipSerialize!(T, v.tupleof[i].stringof[2..$]))
112 				{
113 					static if (hasAttribute!(JSONOptional, v.tupleof[i]))
114 						if (v.tupleof[i] == T.init.tupleof[i])
115 							continue;
116 					if (!first)
117 						output.put(',');
118 					else
119 						first = false;
120 					put(getJsonName!(T, v.tupleof[i].stringof[2..$]));
121 					output.put(':');
122 					put(field);
123 				}
124 			}
125 			output.put('}');
126 		}
127 		else
128 		static if (isAssociativeArray!T)
129 		{
130 			output.put('{');
131 			bool first = true;
132 			foreach (key, value; v)
133 			{
134 				if (!first)
135 					output.put(',');
136 				else
137 					first = false;
138 				put(key);
139 				output.put(':');
140 				put(value);
141 			}
142 			output.put('}');
143 		}
144 		else
145 		static if (is(typeof(*v)))
146 		{
147 			if (v)
148 				put(*v);
149 			else
150 				output.put("null");
151 		}
152 		else
153 			static assert(0, "Can't serialize " ~ T.stringof ~ " to JSON");
154 	}
155 }
156 
157 alias CustomJsonWriter!StringBuilder JsonWriter;
158 
159 private struct Escapes
160 {
161 	static __gshared string[256] chars;
162 	static __gshared bool[256] escaped;
163 
164 	shared static this()
165 	{
166 		import std.string;
167 
168 		escaped[] = true;
169 		foreach (c; 0..256)
170 			if (c=='\\')
171 				chars[c] = `\\`;
172 			else
173 			if (c=='\"')
174 				chars[c] = `\"`;
175 			else
176 			if (c=='\b')
177 				chars[c] = `\b`;
178 			else
179 			if (c=='\f')
180 				chars[c] = `\f`;
181 			else
182 			if (c=='\n')
183 				chars[c] = `\n`;
184 			else
185 			if (c=='\r')
186 				chars[c] = `\r`;
187 			else
188 			if (c=='\t')
189 				chars[c] = `\t`;
190 			else
191 			if (c<'\x20' || c == '\x7F' || c=='<' || c=='>' || c=='&')
192 				chars[c] = format(`\u%04x`, c);
193 			else
194 				chars[c] = [cast(char)c],
195 				escaped[c] = false;
196 	}
197 }
198 
199 // ************************************************************************
200 
201 string toJson(T)(T v)
202 {
203 	JsonWriter writer;
204 	writer.put(v);
205 	return writer.output.get();
206 }
207 
208 unittest
209 {
210 	struct X { int a; string b; }
211 	X x = {17, "aoeu"};
212 	assert(toJson(x) == `{"a":17,"b":"aoeu"}`);
213 	int[] arr = [1,5,7];
214 	assert(toJson(arr) == `[1,5,7]`);
215 	assert(toJson(true) == `true`);
216 
217 	assert(toJson(tuple()) == ``);
218 	assert(toJson(tuple(42)) == `42`);
219 	assert(toJson(tuple(42, "banana")) == `[42,"banana"]`);
220 }
221 
222 // ************************************************************************
223 
224 import std.ascii;
225 import std.utf;
226 import std.conv;
227 
228 import ae.utils.text;
229 
230 private struct JsonParser(C)
231 {
232 	C[] s;
233 	size_t p;
234 
235 	char next()
236 	{
237 		enforce(p < s.length);
238 		return s[p++];
239 	}
240 
241 	string readN(uint n)
242 	{
243 		string r;
244 		for (int i=0; i<n; i++)
245 			r ~= next();
246 		return r;
247 	}
248 
249 	char peek()
250 	{
251 		enforce(p < s.length);
252 		return s[p];
253 	}
254 
255 	@property bool eof() { return p == s.length; }
256 
257 	void skipWhitespace()
258 	{
259 		while (isWhite(peek()))
260 			p++;
261 	}
262 
263 	void expect(char c)
264 	{
265 		auto n = next();
266 		enforce(n==c, "Expected " ~ c ~ ", got " ~ n);
267 	}
268 
269 	T read(T)()
270 	{
271 		static if (is(T X == Nullable!X))
272 			return readNullable!X();
273 		else
274 		static if (is(T==enum))
275 			return readEnum!(T)();
276 		else
277 		static if (is(T==string))
278 			return readString();
279 		else
280 		static if (is(T==bool))
281 			return readBool();
282 		else
283 		static if (is(T : real))
284 			return readNumber!(T)();
285 		else
286 		static if (isDynamicArray!T)
287 			return readArray!(typeof(T.init[0]))();
288 		else
289 		static if (isStaticArray!T)
290 		{
291 			T result = readArray!(typeof(T.init[0]))()[];
292 			return result;
293 		}
294 		else
295 		static if (isTuple!T)
296 			return readTuple!T();
297 		else
298 		static if (is(T==struct))
299 			return readObject!(T)();
300 		else
301 		static if (is(typeof(T.init.keys)) && is(typeof(T.init.values)) && is(typeof(T.init.keys[0])==string))
302 			return readAA!(T)();
303 		else
304 		static if (is(T U : U*))
305 			return readPointer!T();
306 		else
307 			static assert(0, "Can't decode " ~ T.stringof ~ " from JSON");
308 	}
309 
310 	auto readTuple(T)()
311 	{
312 		// TODO: serialize as object if tuple has names
313 		enum N = T.expand.length;
314 		static if (N == 0)
315 			return T();
316 		else
317 		static if (N == 1)
318 			return T(read!(typeof(T.expand[0])));
319 		else
320 		{
321 			T v;
322 			expect('[');
323 			foreach (n, ref f; v.expand)
324 			{
325 				static if (n)
326 					expect(',');
327 				f = read!(typeof(f));
328 			}
329 			expect(']');
330 			return v;
331 		}
332 	}
333 
334 	auto readNullable(T)()
335 	{
336 		if (peek() == 'n')
337 		{
338 			next();
339 			expect('u');
340 			expect('l');
341 			expect('l');
342 			return Nullable!T();
343 		}
344 		else
345 			return Nullable!T(read!T);
346 	}
347 
348 	C[] readSimpleString() /// i.e. without escapes
349 	{
350 		skipWhitespace();
351 		expect('"');
352 		auto start = p;
353 		while (true)
354 		{
355 			auto c = next();
356 			if (c=='"')
357 				break;
358 			else
359 			if (c=='\\')
360 				throw new Exception("Unexpected escaped character");
361 		}
362 		return s[start..p-1];
363 	}
364 
365 	string readString()
366 	{
367 		skipWhitespace();
368 		auto c = peek();
369 		if (c == '"')
370 		{
371 			next(); // '"'
372 			string result;
373 			auto start = p;
374 			while (true)
375 			{
376 				c = next();
377 				if (c=='"')
378 					break;
379 				else
380 				if (c=='\\')
381 				{
382 					result ~= s[start..p-1];
383 					switch (next())
384 					{
385 						case '"':  result ~= '"'; break;
386 						case '/':  result ~= '/'; break;
387 						case '\\': result ~= '\\'; break;
388 						case 'b':  result ~= '\b'; break;
389 						case 'f':  result ~= '\f'; break;
390 						case 'n':  result ~= '\n'; break;
391 						case 'r':  result ~= '\r'; break;
392 						case 't':  result ~= '\t'; break;
393 						case 'u':
394 						{
395 							wstring buf;
396 							goto Unicode_start;
397 
398 							while (s[p..$].startsWith(`\u`))
399 							{
400 								p+=2;
401 							Unicode_start:
402 								buf ~= cast(wchar)fromHex!ushort(readN(4));
403 							}
404 							result ~= toUTF8(buf);
405 							break;
406 						}
407 						default: enforce(false, "Unknown escape");
408 					}
409 					start = p;
410 				}
411 			}
412 			result ~= s[start..p-1];
413 			return result;
414 		}
415 		else
416 		if (isDigit(c) || c=='-') // For languages that don't distinguish numeric strings from numbers
417 		{
418 			static immutable bool[256] numeric =
419 			[
420 				'0':true,
421 				'1':true,
422 				'2':true,
423 				'3':true,
424 				'4':true,
425 				'5':true,
426 				'6':true,
427 				'7':true,
428 				'8':true,
429 				'9':true,
430 				'.':true,
431 				'-':true,
432 				'+':true,
433 				'e':true,
434 				'E':true,
435 			];
436 
437 			auto start = p;
438 			while (numeric[c = peek()])
439 				p++;
440 			return s[start..p].idup;
441 		}
442 		else
443 		{
444 			foreach (n; "null")
445 				expect(n);
446 			return null;
447 		}
448 	}
449 
450 	bool readBool()
451 	{
452 		skipWhitespace();
453 		if (peek()=='t')
454 		{
455 			enforce(readN(4) == "true", "Bad boolean");
456 			return true;
457 		}
458 		else
459 		if (peek()=='f')
460 		{
461 			enforce(readN(5) == "false", "Bad boolean");
462 			return false;
463 		}
464 		else
465 		{
466 			ubyte i = readNumber!ubyte();
467 			enforce(i < 2);
468 			return !!i;
469 		}
470 	}
471 
472 	T readNumber(T)()
473 	{
474 		skipWhitespace();
475 		T v;
476 		const(char)[] n;
477 		auto start = p;
478 		char c = peek();
479 		if (c == '"')
480 			n = readSimpleString();
481 		else
482 		{
483 			while (c=='+' || c=='-' || (c>='0' && c<='9') || c=='e' || c=='E' || c=='.')
484 			{
485 				p++;
486 				if (eof) break;
487 				c=peek();
488 			}
489 			n = s[start..p];
490 		}
491 		static if (is(T : real))
492 			return to!T(n);
493 		else
494 			static assert(0, "Don't know how to parse numerical type " ~ T.stringof);
495 	}
496 
497 	T[] readArray(T)()
498 	{
499 		skipWhitespace();
500 		expect('[');
501 		skipWhitespace();
502 		T[] result;
503 		if (peek()==']')
504 		{
505 			p++;
506 			return result;
507 		}
508 		while(true)
509 		{
510 			result ~= read!(T)();
511 			skipWhitespace();
512 			if (peek()==']')
513 			{
514 				p++;
515 				return result;
516 			}
517 			else
518 				expect(',');
519 		}
520 	}
521 
522 	T readObject(T)()
523 	{
524 		skipWhitespace();
525 		expect('{');
526 		skipWhitespace();
527 		T v;
528 		if (peek()=='}')
529 		{
530 			p++;
531 			return v;
532 		}
533 
534 		while (true)
535 		{
536 			auto jsonField = readSimpleString();
537 			mixin(exceptionContext(q{"Error with field " ~ to!string(jsonField)}));
538 			skipWhitespace();
539 			expect(':');
540 
541 			bool found;
542 			foreach (i, field; v.tupleof)
543 			{
544 				enum name = getJsonName!(T, v.tupleof[i].stringof[2..$]);
545 				if (name == jsonField)
546 				{
547 					v.tupleof[i] = read!(typeof(v.tupleof[i]))();
548 					found = true;
549 					break;
550 				}
551 			}
552 			enforce(found, "Unknown field " ~ jsonField);
553 
554 			skipWhitespace();
555 			if (peek()=='}')
556 			{
557 				p++;
558 				return v;
559 			}
560 			else
561 				expect(',');
562 		}
563 	}
564 
565 	T readAA(T)()
566 	{
567 		skipWhitespace();
568 		expect('{');
569 		skipWhitespace();
570 		T v;
571 		if (peek()=='}')
572 		{
573 			p++;
574 			return v;
575 		}
576 
577 		while (true)
578 		{
579 			string jsonField = readString();
580 			skipWhitespace();
581 			expect(':');
582 
583 			v[jsonField] = read!(typeof(v.values[0]))();
584 
585 			skipWhitespace();
586 			if (peek()=='}')
587 			{
588 				p++;
589 				return v;
590 			}
591 			else
592 				expect(',');
593 		}
594 	}
595 
596 	T readEnum(T)()
597 	{
598 		return to!T(readSimpleString());
599 	}
600 
601 	T readPointer(T)()
602 	{
603 		skipWhitespace();
604 		if (peek()=='n')
605 		{
606 			enforce(readN(4) == "null", "Null expected");
607 			return null;
608 		}
609 		alias typeof(*T.init) S;
610 		T v = new S;
611 		*v = read!S();
612 		return v;
613 	}
614 }
615 
616 T jsonParse(T, C)(C[] s)
617 {
618 	auto parser = JsonParser!C(s);
619 	mixin(exceptionContext(q{format("Error at position %d", parser.p)}));
620 	return parser.read!T();
621 }
622 
623 unittest
624 {
625 	struct S { int i; S[] arr; S* p0, p1; }
626 	S s = S(42, [S(1), S(2)], null, new S(15));
627 	auto s2 = jsonParse!S(toJson(s));
628 	//assert(s == s2); // Issue 3789
629 	assert(s.i == s2.i && s.arr == s2.arr && s.p0 is s2.p0 && *s.p1 == *s2.p1);
630 	jsonParse!S(toJson(s).dup);
631 
632 	assert(jsonParse!(Tuple!())(``) == tuple());
633 	assert(jsonParse!(Tuple!int)(`42`) == tuple(42));
634 	assert(jsonParse!(Tuple!(int, string))(`[42, "banana"]`) == tuple(42, "banana"));
635 }
636 
637 // ************************************************************************
638 
639 // TODO: migrate to UDAs
640 
641 /**
642  * A template that designates fields which should not be serialized to Json.
643  *
644  * Example:
645  * ---
646  * struct Point { int x, y, z; mixin NonSerialized!(x, z); }
647  * assert(jsonParse!Point(toJson(Point(1, 2, 3))) == Point(0, 2, 0));
648  * ---
649  */
650 template NonSerialized(fields...)
651 {
652 	import ae.utils.meta : stringofArray;
653 	mixin(NonSerializedFields(stringofArray!fields()));
654 }
655 
656 string NonSerializedFields(string[] fields)
657 {
658 	string result;
659 	foreach (field; fields)
660 		result ~= "enum bool " ~ field ~ "_nonSerialized = 1;";
661 	return result;
662 }
663 
664 private template doSkipSerialize(T, string member)
665 {
666 	enum bool doSkipSerialize = __traits(hasMember, T, member ~ "_nonSerialized");
667 }
668 
669 unittest
670 {
671 	struct Point { int x, y, z; mixin NonSerialized!(x, z); }
672 	assert(jsonParse!Point(toJson(Point(1, 2, 3))) == Point(0, 2, 0));
673 }
674 
675 unittest
676 {
677 	enum En { one, two }
678 	assert(En.one.toJson() == `"one"`);
679 	struct S { int i1, i2; S[] arr1, arr2; string[string] dic; En en; mixin NonSerialized!(i2, arr2); }
680 	S s = S(42, 5, [S(1), S(2)], [S(3), S(4)], ["apple":"fruit", "pizza":"vegetable"], En.two);
681 	auto s2 = jsonParse!S(toJson(s));
682 	assert(s.i1 == s2.i1 && s2.i2 is int.init && s.arr1 == s2.arr1 && s2.arr2 is null && s.dic == s2.dic && s.en == En.two);
683 }
684 
685 unittest
686 {
687 	alias B = Nullable!bool;
688 	B b;
689 
690 	b = jsonParse!B("true");
691 	assert(!b.isNull);
692 	assert(b == true);
693 
694 	b = jsonParse!B("false");
695 	assert(!b.isNull);
696 	assert(b == false);
697 
698 	b = jsonParse!B("null");
699 	assert(b.isNull);
700 }
701 
702 // ************************************************************************
703 
704 /// User-defined attribute - specify name for JSON object field.
705 /// Useful when a JSON object may contain fields, the name of which are not valid D identifiers.
706 struct JSONName { string name; }
707 
708 private template getJsonName(S, string FIELD)
709 {
710 	static if (hasAttribute!(JSONName, __traits(getMember, S, FIELD)))
711 		enum getJsonName = getAttribute!(JSONName, __traits(getMember, S, FIELD)).name;
712 	else
713 		enum getJsonName = FIELD;
714 }
715 
716 // ************************************************************************
717 
718 /// User-defined attribute - only serialize this field if its value is different from its .init value.
719 struct JSONOptional {}
720 
721 unittest
722 {
723 	static struct S { @JSONOptional bool a=true, b=false; }
724 	assert(S().toJson == `{}`, S().toJson);
725 	assert(S(false, true).toJson == `{"a":false,"b":true}`);
726 }