1 /**
2  * JSON encoding.
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Vladimir Panteleev <vladimir@thecybershadow.net>
12  */
13 
14 module ae.utils.json;
15 
16 import std.exception;
17 import std.string;
18 import std.traits;
19 import std.typecons;
20 
21 import ae.utils.exception;
22 import ae.utils.meta;
23 import ae.utils.textout;
24 
25 // ************************************************************************
26 
27 struct CustomJsonWriter(WRITER)
28 {
29 	/// You can set this to something to e.g. write to another buffer.
30 	WRITER output;
31 
32 	void putString(in char[] s)
33 	{
34 		// TODO: escape Unicode characters?
35 		// TODO: Handle U+2028 and U+2029 ( http://timelessrepo.com/json-isnt-a-javascript-subset )
36 
37 		output.put('"');
38 		auto start = s.ptr, p = start, end = start+s.length;
39 
40 		while (p < end)
41 		{
42 			auto c = *p++;
43 			if (Escapes.escaped[c])
44 				output.put(start[0..p-start-1], Escapes.chars[c]),
45 				start = p;
46 		}
47 
48 		output.put(start[0..p-start], '"');
49 	}
50 
51 	void put(T)(T v)
52 	{
53 		static if (is(T == enum))
54 			put(to!string(v));
55 		else
56 		static if (is(T : const(char)[]))
57 			putString(v);
58 		else
59 		static if (is(T == bool))
60 			return output.put(v ? "true" : "false");
61 		else
62 		static if (is(T : long))
63 			return .put(output, v);
64 		else
65 		static if (is(T : real))
66 			return output.put(fpToString!T(v)); // TODO: don't allocate
67 		else
68 		static if (is(T U : U[]))
69 		{
70 			output.put('[');
71 			if (v.length)
72 			{
73 				put(v[0]);
74 				foreach (i; v[1..$])
75 				{
76 					output.put(',');
77 					put(i);
78 				}
79 			}
80 			output.put(']');
81 		}
82 		else
83 		static if (isTuple!T)
84 		{
85 			// TODO: serialize as object if tuple has names
86 			enum N = v.expand.length;
87 			static if (N == 0)
88 				return;
89 			else
90 			static if (N == 1)
91 				put(v.expand[0]);
92 			else
93 			{
94 				output.put('[');
95 				foreach (n; RangeTuple!N)
96 				{
97 					static if (n)
98 						output.put(',');
99 					put(v.expand[n]);
100 				}
101 				output.put(']');
102 			}
103 		}
104 		else
105 		static if (is(T==struct))
106 		{
107 			output.put('{');
108 			bool first = true;
109 			foreach (i, field; v.tupleof)
110 			{
111 				static if (!doSkipSerialize!(T, v.tupleof[i].stringof[2..$]))
112 				{
113 					static if (hasAttribute!(JSONOptional, v.tupleof[i]))
114 						if (v.tupleof[i] == T.init.tupleof[i])
115 							continue;
116 					if (!first)
117 						output.put(',');
118 					else
119 						first = false;
120 					put(getJsonName!(T, v.tupleof[i].stringof[2..$]));
121 					output.put(':');
122 					put(field);
123 				}
124 			}
125 			output.put('}');
126 		}
127 		else
128 		static if (isAssociativeArray!T)
129 		{
130 			output.put('{');
131 			bool first = true;
132 			foreach (key, value; v)
133 			{
134 				if (!first)
135 					output.put(',');
136 				else
137 					first = false;
138 				put(key);
139 				output.put(':');
140 				put(value);
141 			}
142 			output.put('}');
143 		}
144 		else
145 		static if (is(typeof(*v)))
146 		{
147 			if (v)
148 				put(*v);
149 			else
150 				output.put("null");
151 		}
152 		else
153 			static assert(0, "Can't serialize " ~ T.stringof ~ " to JSON");
154 	}
155 }
156 
157 alias CustomJsonWriter!StringBuilder JsonWriter;
158 
159 private struct Escapes
160 {
161 	static __gshared string[256] chars;
162 	static __gshared bool[256] escaped;
163 
164 	shared static this()
165 	{
166 		import std.string;
167 
168 		escaped[] = true;
169 		foreach (c; 0..256)
170 			if (c=='\\')
171 				chars[c] = `\\`;
172 			else
173 			if (c=='\"')
174 				chars[c] = `\"`;
175 			else
176 			if (c=='\b')
177 				chars[c] = `\b`;
178 			else
179 			if (c=='\f')
180 				chars[c] = `\f`;
181 			else
182 			if (c=='\n')
183 				chars[c] = `\n`;
184 			else
185 			if (c=='\r')
186 				chars[c] = `\r`;
187 			else
188 			if (c=='\t')
189 				chars[c] = `\t`;
190 			else
191 			if (c<'\x20' || c == '\x7F' || c=='<' || c=='>' || c=='&')
192 				chars[c] = format(`\u%04x`, c);
193 			else
194 				chars[c] = [cast(char)c],
195 				escaped[c] = false;
196 	}
197 }
198 
199 // ************************************************************************
200 
201 string toJson(T)(T v)
202 {
203 	JsonWriter writer;
204 	writer.put(v);
205 	return writer.output.get();
206 }
207 
208 unittest
209 {
210 	struct X { int a; string b; }
211 	X x = {17, "aoeu"};
212 	assert(toJson(x) == `{"a":17,"b":"aoeu"}`);
213 	int[] arr = [1,5,7];
214 	assert(toJson(arr) == `[1,5,7]`);
215 	assert(toJson(true) == `true`);
216 
217 	assert(toJson(tuple()) == ``);
218 	assert(toJson(tuple(42)) == `42`);
219 	assert(toJson(tuple(42, "banana")) == `[42,"banana"]`);
220 }
221 
222 // ************************************************************************
223 
224 import std.ascii;
225 import std.utf;
226 import std.conv;
227 
228 import ae.utils.text;
229 
230 private struct JsonParser(C)
231 {
232 	C[] s;
233 	size_t p;
234 
235 	char next()
236 	{
237 		enforce(p < s.length);
238 		return s[p++];
239 	}
240 
241 	string readN(uint n)
242 	{
243 		string r;
244 		for (int i=0; i<n; i++)
245 			r ~= next();
246 		return r;
247 	}
248 
249 	char peek()
250 	{
251 		enforce(p < s.length);
252 		return s[p];
253 	}
254 
255 	@property bool eof() { return p == s.length; }
256 
257 	void skipWhitespace()
258 	{
259 		while (isWhite(peek()))
260 			p++;
261 	}
262 
263 	void expect(char c)
264 	{
265 		auto n = next();
266 		enforce(n==c, "Expected " ~ c ~ ", got " ~ n);
267 	}
268 
269 	T read(T)()
270 	{
271 		static if (is(T X == Nullable!X))
272 			return readNullable!X();
273 		else
274 		static if (is(T==enum))
275 			return readEnum!(T)();
276 		else
277 		static if (is(T==string))
278 			return readString();
279 		else
280 		static if (is(T==bool))
281 			return readBool();
282 		else
283 		static if (is(T : real))
284 			return readNumber!(T)();
285 		else
286 		static if (isDynamicArray!T)
287 			return readArray!(typeof(T.init[0]))();
288 		else
289 		static if (isStaticArray!T)
290 		{
291 			T result = readArray!(typeof(T.init[0]))()[];
292 			return result;
293 		}
294 		else
295 		static if (isTuple!T)
296 			return readTuple!T();
297 		else
298 		static if (is(T==struct))
299 			return readObject!(T)();
300 		else
301 		static if (is(typeof(T.init.keys)) && is(typeof(T.init.values)) && is(typeof(T.init.keys[0])==string))
302 			return readAA!(T)();
303 		else
304 		static if (is(T U : U*))
305 			return readPointer!T();
306 		else
307 			static assert(0, "Can't decode " ~ T.stringof ~ " from JSON");
308 	}
309 
310 	auto readTuple(T)()
311 	{
312 		// TODO: serialize as object if tuple has names
313 		enum N = T.expand.length;
314 		static if (N == 0)
315 			return T();
316 		else
317 		static if (N == 1)
318 			return T(read!(typeof(T.expand[0])));
319 		else
320 		{
321 			T v;
322 			expect('[');
323 			foreach (n, ref f; v.expand)
324 			{
325 				static if (n)
326 					expect(',');
327 				f = read!(typeof(f));
328 			}
329 			expect(']');
330 			return v;
331 		}
332 	}
333 
334 	auto readNullable(T)()
335 	{
336 		if (peek() == 'n')
337 		{
338 			next();
339 			expect('u');
340 			expect('l');
341 			expect('l');
342 			return Nullable!T();
343 		}
344 		else
345 			return Nullable!T(read!T);
346 	}
347 
348 	string readString()
349 	{
350 		skipWhitespace();
351 		auto c = peek();
352 		if (c == '"')
353 		{
354 			next(); // '"'
355 			string result;
356 			while (true)
357 			{
358 				c = next();
359 				if (c=='"')
360 					break;
361 				else
362 				if (c=='\\')
363 					switch (next())
364 					{
365 						case '"':  result ~= '"'; break;
366 						case '/':  result ~= '/'; break;
367 						case '\\': result ~= '\\'; break;
368 						case 'b':  result ~= '\b'; break;
369 						case 'f':  result ~= '\f'; break;
370 						case 'n':  result ~= '\n'; break;
371 						case 'r':  result ~= '\r'; break;
372 						case 't':  result ~= '\t'; break;
373 						case 'u':
374 						{
375 							wstring buf;
376 							goto Unicode_start;
377 
378 							while (s[p..$].startsWith(`\u`))
379 							{
380 								p+=2;
381 							Unicode_start:
382 								buf ~= cast(wchar)fromHex!ushort(readN(4));
383 							}
384 							result ~= toUTF8(buf);
385 							break;
386 						}
387 						default: enforce(false, "Unknown escape");
388 					}
389 				else
390 					result ~= c;
391 			}
392 			return result;
393 		}
394 		else
395 		if (isDigit(c) || c=='-') // For languages that don't distinguish numeric strings from numbers
396 		{
397 			static immutable bool[256] numeric =
398 			[
399 				'0':true,
400 				'1':true,
401 				'2':true,
402 				'3':true,
403 				'4':true,
404 				'5':true,
405 				'6':true,
406 				'7':true,
407 				'8':true,
408 				'9':true,
409 				'.':true,
410 				'-':true,
411 				'+':true,
412 				'e':true,
413 				'E':true,
414 			];
415 
416 			string s;
417 			while (c=peek(), numeric[c])
418 				s ~= c, p++;
419 			return s;
420 		}
421 		else
422 		{
423 			foreach (n; "null")
424 				expect(n);
425 			return null;
426 		}
427 	}
428 
429 	bool readBool()
430 	{
431 		skipWhitespace();
432 		if (peek()=='t')
433 		{
434 			enforce(readN(4) == "true", "Bad boolean");
435 			return true;
436 		}
437 		else
438 		if (peek()=='f')
439 		{
440 			enforce(readN(5) == "false", "Bad boolean");
441 			return false;
442 		}
443 		else
444 		{
445 			ubyte i = readNumber!ubyte();
446 			enforce(i < 2);
447 			return !!i;
448 		}
449 	}
450 
451 	T readNumber(T)()
452 	{
453 		skipWhitespace();
454 		T v;
455 		string s;
456 		char c = peek();
457 		if (c == '"')
458 			s = readString();
459 		else
460 			while (c=='+' || c=='-' || (c>='0' && c<='9') || c=='e' || c=='E' || c=='.')
461 			{
462 				s ~= c, p++;
463 				if (eof) break;
464 				c=peek();
465 			}
466 		static if (is(T : real))
467 			return to!T(s);
468 		else
469 			static assert(0, "Don't know how to parse numerical type " ~ T.stringof);
470 	}
471 
472 	T[] readArray(T)()
473 	{
474 		skipWhitespace();
475 		expect('[');
476 		skipWhitespace();
477 		T[] result;
478 		if (peek()==']')
479 		{
480 			p++;
481 			return result;
482 		}
483 		while(true)
484 		{
485 			result ~= read!(T)();
486 			skipWhitespace();
487 			if (peek()==']')
488 			{
489 				p++;
490 				return result;
491 			}
492 			else
493 				expect(',');
494 		}
495 	}
496 
497 	T readObject(T)()
498 	{
499 		skipWhitespace();
500 		expect('{');
501 		skipWhitespace();
502 		T v;
503 		if (peek()=='}')
504 		{
505 			p++;
506 			return v;
507 		}
508 
509 		while (true)
510 		{
511 			string jsonField = readString();
512 			mixin(exceptionContext(q{"Error with field " ~ jsonField}));
513 			skipWhitespace();
514 			expect(':');
515 
516 			bool found;
517 			foreach (i, field; v.tupleof)
518 			{
519 				enum name = getJsonName!(T, v.tupleof[i].stringof[2..$]);
520 				if (name == jsonField)
521 				{
522 					v.tupleof[i] = read!(typeof(v.tupleof[i]))();
523 					found = true;
524 					break;
525 				}
526 			}
527 			enforce(found, "Unknown field " ~ jsonField);
528 
529 			skipWhitespace();
530 			if (peek()=='}')
531 			{
532 				p++;
533 				return v;
534 			}
535 			else
536 				expect(',');
537 		}
538 	}
539 
540 	T readAA(T)()
541 	{
542 		skipWhitespace();
543 		expect('{');
544 		skipWhitespace();
545 		T v;
546 		if (peek()=='}')
547 		{
548 			p++;
549 			return v;
550 		}
551 
552 		while (true)
553 		{
554 			string jsonField = readString();
555 			skipWhitespace();
556 			expect(':');
557 
558 			v[jsonField] = read!(typeof(v.values[0]))();
559 
560 			skipWhitespace();
561 			if (peek()=='}')
562 			{
563 				p++;
564 				return v;
565 			}
566 			else
567 				expect(',');
568 		}
569 	}
570 
571 	T readEnum(T)()
572 	{
573 		return to!T(readString());
574 	}
575 
576 	T readPointer(T)()
577 	{
578 		skipWhitespace();
579 		if (peek()=='n')
580 		{
581 			enforce(readN(4) == "null", "Null expected");
582 			return null;
583 		}
584 		alias typeof(*T.init) S;
585 		T v = new S;
586 		*v = read!S();
587 		return v;
588 	}
589 }
590 
591 T jsonParse(T, C)(C[] s)
592 {
593 	auto parser = JsonParser!C(s);
594 	mixin(exceptionContext(q{format("Error at position %d", parser.p)}));
595 	return parser.read!T();
596 }
597 
598 unittest
599 {
600 	struct S { int i; S[] arr; S* p0, p1; }
601 	S s = S(42, [S(1), S(2)], null, new S(15));
602 	auto s2 = jsonParse!S(toJson(s));
603 	//assert(s == s2); // Issue 3789
604 	assert(s.i == s2.i && s.arr == s2.arr && s.p0 is s2.p0 && *s.p1 == *s2.p1);
605 	jsonParse!S(toJson(s).dup);
606 
607 	assert(jsonParse!(Tuple!())(``) == tuple());
608 	assert(jsonParse!(Tuple!int)(`42`) == tuple(42));
609 	assert(jsonParse!(Tuple!(int, string))(`[42, "banana"]`) == tuple(42, "banana"));
610 }
611 
612 // ************************************************************************
613 
614 // TODO: migrate to UDAs
615 
616 /**
617  * A template that designates fields which should not be serialized to Json.
618  *
619  * Example:
620  * ---
621  * struct Point { int x, y, z; mixin NonSerialized!(x, z); }
622  * assert(jsonParse!Point(toJson(Point(1, 2, 3))) == Point(0, 2, 0));
623  * ---
624  */
625 template NonSerialized(fields...)
626 {
627 	import ae.utils.meta : stringofArray;
628 	mixin(NonSerializedFields(stringofArray!fields()));
629 }
630 
631 string NonSerializedFields(string[] fields)
632 {
633 	string result;
634 	foreach (field; fields)
635 		result ~= "enum bool " ~ field ~ "_nonSerialized = 1;";
636 	return result;
637 }
638 
639 private template doSkipSerialize(T, string member)
640 {
641 	enum bool doSkipSerialize = __traits(hasMember, T, member ~ "_nonSerialized");
642 }
643 
644 unittest
645 {
646 	struct Point { int x, y, z; mixin NonSerialized!(x, z); }
647 	assert(jsonParse!Point(toJson(Point(1, 2, 3))) == Point(0, 2, 0));
648 }
649 
650 unittest
651 {
652 	enum En { one, two }
653 	struct S { int i1, i2; S[] arr1, arr2; string[string] dic; En en; mixin NonSerialized!(i2, arr2); }
654 	S s = S(42, 5, [S(1), S(2)], [S(3), S(4)], ["apple":"fruit", "pizza":"vegetable"], En.two);
655 	auto s2 = jsonParse!S(toJson(s));
656 	assert(s.i1 == s2.i1 && s2.i2 is int.init && s.arr1 == s2.arr1 && s2.arr2 is null && s.dic == s2.dic && s.en == En.two);
657 }
658 
659 unittest
660 {
661 	alias B = Nullable!bool;
662 	B b;
663 
664 	b = jsonParse!B("true");
665 	assert(!b.isNull);
666 	assert(b == true);
667 
668 	b = jsonParse!B("false");
669 	assert(!b.isNull);
670 	assert(b == false);
671 
672 	b = jsonParse!B("null");
673 	assert(b.isNull);
674 }
675 
676 // ************************************************************************
677 
678 /// User-defined attribute - specify name for JSON object field.
679 /// Useful when a JSON object may contain fields, the name of which are not valid D identifiers.
680 struct JSONName { string name; }
681 
682 private template getJsonName(S, string FIELD)
683 {
684 	static if (hasAttribute!(JSONName, __traits(getMember, S, FIELD)))
685 		enum getJsonName = getAttribute!(JSONName, __traits(getMember, S, FIELD)).name;
686 	else
687 		enum getJsonName = FIELD;
688 }
689 
690 // ************************************************************************
691 
692 /// User-defined attribute - only serialize this field if its value is different from its .init value.
693 struct JSONOptional {}
694 
695 unittest
696 {
697 	static struct S { @JSONOptional bool a=true, b=false; }
698 	assert(S().toJson == `{}`, S().toJson);
699 	assert(S(false, true).toJson == `{"a":false,"b":true}`);
700 }