1 /**
2  * Concepts shared between HTTP clients and servers.
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Stéphan Kochen <stephan@kochen.nl>
12  *   Vladimir Panteleev <ae@cy.md>
13  *   Simon Arlott
14  */
15 
16 module ae.net.http.common;
17 
18 import core.time;
19 
20 import std.algorithm;
21 import std.array;
22 import std.base64;
23 import std.string;
24 import std.conv;
25 import std.ascii;
26 import std.exception;
27 import std.datetime;
28 import std.typecons : tuple;
29 
30 import ae.net.ietf.headers;
31 import ae.sys.data;
32 import ae.sys.dataset;
33 import ae.utils.array : amap, afilter, auniq, asort, asBytes, as;
34 import ae.utils.text;
35 import ae.utils.time;
36 
37 version (ae_with_zlib) // Explicit override
38 	enum haveZlib = true;
39 else
40 version (ae_without_zlib) // Explicit override
41 	enum haveZlib = false;
42 else
43 version (Have_ae) // Building with Dub
44 {
45 	version (Have_ae_zlib) // Using ae:zlib
46 		enum haveZlib = true;
47 	else
48 		enum haveZlib = false;
49 }
50 else // Not building with Dub
51 	enum haveZlib = true; // Pull in zlib by default
52 
53 static if (haveZlib)
54 {
55 	import zlib = ae.utils.zlib;
56 	import gzip = ae.utils.gzip;
57 }
58 
59 /// Base HTTP message class
60 private abstract class HttpMessage
61 {
62 public:
63 	string protocol = "http";
64 	string protocolVersion = "1.0";
65 	Headers headers;
66 	DataVec data;
67 	SysTime creationTime;
68 
69 	this()
70 	{
71 		creationTime = Clock.currTime();
72 	}
73 
74 	@property Duration age()
75 	{
76 		return Clock.currTime() - creationTime;
77 	}
78 
79 	/// For `dup`.
80 	protected void copyTo(typeof(this) other)
81 	{
82 		other.protocol = protocol;
83 		other.protocolVersion = protocolVersion;
84 		other.headers = headers.dup;
85 		other.data = data.dup;
86 		other.creationTime = creationTime;
87 	}
88 }
89 
90 // TODO: Separate this from an URL type
91 
92 /// HTTP request class
93 class HttpRequest : HttpMessage
94 {
95 public:
96 	/// HTTP method, e.g., "GET".
97 	string method = "GET";
98 
99 	/// If this request is going through a HTTP proxy server, this
100 	/// should be set to its address.
101 	string proxy;
102 
103 	this()
104 	{
105 	} ///
106 
107 	this(string url)
108 	{
109 		this.resource = url;
110 	} ///
111 
112 	/// For `dup`.
113 	protected void copyTo(typeof(this) other)
114 	{
115 		super.copyTo(other);
116 		other.method = method;
117 		other.proxy = proxy;
118 		other._resource = _resource;
119 		other._port = _port;
120 	}
121 	alias copyTo = typeof(super).copyTo;
122 
123 	final typeof(this) dup()
124 	{
125 		auto result = new typeof(this);
126 		copyTo(result);
127 		return result;
128 	} ///
129 
130 	/// Resource part of URL (everything after the hostname)
131 	@property string resource() const pure nothrow @nogc
132 	{
133 		return _resource;
134 	}
135 
136 	/// Set the resource part of the URL, or the entire URL.
137 	/// Setting the resource to a full URL will fill in the Host header, as well.
138 	@property void resource(string value) pure
139 	{
140 		_resource = value;
141 
142 		// applies to both Client/Server as some clients put a full URL in the GET line instead of using a "Host" header
143 		string protocol;
144 		if (_resource.asciiStartsWith("http://"))
145 			protocol = "http";
146 		else
147 		if (_resource.asciiStartsWith("https://"))
148 			protocol = "https";
149 
150 		if (protocol)
151 		{
152 			this.protocol = protocol;
153 
154 			value = value[protocol.length+3..$];
155 			auto pathstart = value.indexOf('/');
156 			string _host;
157 			if (pathstart == -1)
158 			{
159 				_host = value;
160 				_resource = "/";
161 			}
162 			else
163 			{
164 				_host = value[0..pathstart];
165 				_resource = value[pathstart..$];
166 			}
167 
168 			auto authEnd = _host.indexOf('@');
169 			if (authEnd != -1)
170 			{
171 				// Assume HTTP Basic auth
172 				import ae.utils.array : asBytes;
173 				headers["Authorization"] = "Basic " ~ Base64.encode(_host[0 .. authEnd].asBytes).assumeUnique;
174 				_host = _host[authEnd + 1 .. $];
175 			}
176 
177 			auto portstart = _host.indexOf(':');
178 			if (portstart != -1)
179 			{
180 				port = to!ushort(_host[portstart+1..$]);
181 				_host = _host[0..portstart];
182 			}
183 			host = _host;
184 		}
185 	}
186 
187 	/// The hostname, without the port number
188 	@property string host() const pure nothrow @nogc
189 	{
190 		string _host = headers.get("Host", null);
191 		// auto colon = _host.lastIndexOf(':'); // https://issues.dlang.org/show_bug.cgi?id=24008
192 		sizediff_t colon = -1; foreach_reverse (i, c; _host) if (c == ':') { colon = i; break; }
193 		return colon < 0 ? _host : _host[0 .. colon];
194 	}
195 
196 	/// Sets the hostname (and the `"Host"` header).
197 	/// Must not include a port number.
198 	/// Does not change the previously-set port number.
199 	@property void host(string _host) pure
200 	{
201 		auto _port = this.port;
202 		headers["Host"] = _port==protocolDefaultPort ? _host : _host ~ ":" ~ text(_port);
203 	}
204 
205 	/// Retrieves the default port number for the currently set `protocol`.
206 	@property ushort protocolDefaultPort() const pure
207 	{
208 		switch (protocol)
209 		{
210 			case "http":
211 				return 80;
212 			case "https":
213 				return 443;
214 			default:
215 				throw new Exception("Unknown protocol: " ~ protocol);
216 		}
217 	}
218 
219 	/// Port number, from `"Host"` header.
220 	/// Defaults to `protocolDefaultPort`.
221 	@property ushort port() const pure
222 	{
223 		if ("Host" in headers)
224 		{
225 			string _host = headers["Host"];
226 			auto colon = _host.lastIndexOf(":");
227 			return colon<0 ? protocolDefaultPort : to!ushort(_host[colon+1..$]);
228 		}
229 		else
230 			return _port ? _port : protocolDefaultPort;
231 	}
232 
233 	/// Sets the port number.
234 	/// If it is equal to `protocolDefaultPort`, then it is not
235 	/// included in the `"Host"` header.
236 	@property void port(ushort _port) pure
237 	{
238 		if ("Host" in headers)
239 		{
240 			if (_port == protocolDefaultPort)
241 				headers["Host"] = this.host;
242 			else
243 				headers["Host"] = this.host ~ ":" ~ text(_port);
244 		}
245 		else
246 			this._port = _port;
247 	}
248 
249 	/// Path part of request (until the `'?'`).
250 	@property string path() const pure nothrow @nogc
251 	{
252 		auto p = resource.indexOf('?');
253 		if (p >= 0)
254 			return resource[0..p];
255 		else
256 			return resource;
257 	}
258 
259 	/// Query string part of request (atfer the `'?'`).
260 	@property string queryString() const pure nothrow @nogc
261 	{
262 		auto p = resource.indexOf('?');
263 		if (p >= 0)
264 			return resource[p+1..$];
265 		else
266 			return null;
267 	}
268 
269 	/// ditto
270 	@property void queryString(string value) pure
271 	{
272 		auto p = resource.indexOf('?');
273 		if (p >= 0)
274 			resource = resource[0..p];
275 		if (value)
276 			resource = resource ~ '?' ~ value;
277 	}
278 
279 	/// The query string parameters.
280 	@property UrlParameters urlParameters()
281 	{
282 		return decodeUrlParameters(queryString);
283 	}
284 
285 	/// ditto
286 	@property void urlParameters(UrlParameters parameters)
287 	{
288 		queryString = encodeUrlParameters(parameters);
289 	}
290 
291 	/// URL without resource (protocol, host and port).
292 	@property string root()
293 	{
294 		return protocol ~ "://" ~ host ~ (port==protocolDefaultPort ? null : ":" ~ to!string(port));
295 	}
296 
297 	/// Full URL.
298 	@property string url()
299 	{
300 		return root ~ resource;
301 	}
302 
303 	/// Full URL without query parameters or fragment.
304 	@property string baseURL()
305 	{
306 		return root ~ resource.findSplit("?")[0];
307 	}
308 
309 	/// The hostname part of the proxy address, if any.
310 	@property string proxyHost()
311 	{
312 		auto portstart = proxy.indexOf(':');
313 		if (portstart != -1)
314 			return proxy[0..portstart];
315 		return proxy;
316 	}
317 
318 	/// The port number of the proxy address if it specified, otherwise `80`.
319 	@property ushort proxyPort()
320 	{
321 		auto portstart = proxy.indexOf(':');
322 		if (portstart != -1)
323 			return to!ushort(proxy[portstart+1..$]);
324 		return 80;
325 	}
326 
327 	/// Parse the first line in a HTTP request ("METHOD /resource HTTP/1.x").
328 	void parseRequestLine(string reqLine)
329 	{
330 		enforce(reqLine.length > 10, "Request line too short");
331 		auto methodEnd = reqLine.indexOf(' ');
332 		enforce(methodEnd > 0, "Malformed request line");
333 		method = reqLine[0 .. methodEnd];
334 		reqLine = reqLine[methodEnd + 1 .. reqLine.length];
335 
336 		auto resourceEnd = reqLine.lastIndexOf(' ');
337 		enforce(resourceEnd > 0, "Malformed request line");
338 		resource = reqLine[0 .. resourceEnd];
339 
340 		string protocol = reqLine[resourceEnd+1..$];
341 		enforce(protocol.startsWith("HTTP/"));
342 		protocolVersion = protocol[5..$];
343 	}
344 
345 	/// Decodes submitted form data, and returns an AA of values.
346 	UrlParameters decodePostData()
347 	{
348 		auto contentType = headers.get("Content-Type", "").decodeTokenHeader;
349 
350 		switch (contentType.value)
351 		{
352 			case "application/x-www-form-urlencoded":
353 				return decodeUrlParameters(data.joinToGC().as!string);
354 			case "multipart/form-data":
355 				return decodeMultipart(data.joinData(), contentType.properties.get("boundary", null))
356 					.map!((ref part) => tuple(
357 						part.headers.get("Content-Disposition", null).decodeTokenHeader.properties.get("name", null),
358 						part.data.asDataOf!char.toGC().assumeUnique,
359 					))
360 					.array // https://issues.dlang.org/show_bug.cgi?id=24050
361 					.UrlParameters;
362 			case "":
363 				throw new Exception("No Content-Type");
364 			default:
365 				throw new Exception("Unknown Content-Type: " ~ contentType.value);
366 		}
367 	}
368 
369 	/// Get list of hosts as specified in headers (e.g. X-Forwarded-For).
370 	/// First item in returned array is the node furthest away.
371 	/// Duplicates are removed.
372 	/// Specify socket remote address in remoteHost to add it to the list.
373 	deprecated("Insecure, use HttpServer.remoteIPHeader")
374 	string[] remoteHosts(string remoteHost = null)
375 	{
376 		return
377 			(headers.get("X-Forwarded-For", null).split(",").amap!(std..string.strip)() ~
378 			 headers.get("X-Forwarded-Host", null) ~
379 			 remoteHost)
380 			.afilter!`a && a != "unknown"`()
381 			.auniq();
382 	}
383 
384 	debug(ae_unittest) deprecated unittest
385 	{
386 		auto req = new HttpRequest();
387 		assert(req.remoteHosts() == []);
388 		assert(req.remoteHosts("3.3.3.3") == ["3.3.3.3"]);
389 
390 		req.headers["X-Forwarded-For"] = "1.1.1.1, 2.2.2.2";
391 		req.headers["X-Forwarded-Host"] = "2.2.2.2";
392 		assert(req.remoteHosts("3.3.3.3") == ["1.1.1.1", "2.2.2.2", "3.3.3.3"]);
393 	}
394 
395 	/// Basic cookie parsing
396 	string[string] getCookies()
397 	{
398 		string[string] cookies;
399 		foreach (segment; headers.get("Cookie", null).split(";"))
400 		{
401 			segment = segment.strip();
402 			auto p = segment.indexOf('=');
403 			if (p > 0)
404 				cookies[segment[0..p]] = segment[p+1..$];
405 		}
406 		return cookies;
407 	}
408 
409 private:
410 	string _resource;
411 	ushort _port = 0; // used only when no "Host" in headers; otherwise, taken from there
412 }
413 
414 /// HTTP response status codes
415 enum HttpStatusCode : ushort
416 {
417 	None                         =   0,  ///
418 
419 	Continue                     = 100,  ///
420 	SwitchingProtocols           = 101,  ///
421 
422 	OK                           = 200,  ///
423 	Created                      = 201,  ///
424 	Accepted                     = 202,  ///
425 	NonAuthoritativeInformation  = 203,  ///
426 	NoContent                    = 204,  ///
427 	ResetContent                 = 205,  ///
428 	PartialContent               = 206,  ///
429 
430 	MultipleChoices              = 300,  ///
431 	MovedPermanently             = 301,  ///
432 	Found                        = 302,  ///
433 	SeeOther                     = 303,  ///
434 	NotModified                  = 304,  ///
435 	UseProxy                     = 305,  ///
436 	//(Unused)                   = 306,  ///
437 	TemporaryRedirect            = 307,  ///
438 
439 	BadRequest                   = 400,  ///
440 	Unauthorized                 = 401,  ///
441 	PaymentRequired              = 402,  ///
442 	Forbidden                    = 403,  ///
443 	NotFound                     = 404,  ///
444 	MethodNotAllowed             = 405,  ///
445 	NotAcceptable                = 406,  ///
446 	ProxyAuthenticationRequired  = 407,  ///
447 	RequestTimeout               = 408,  ///
448 	Conflict                     = 409,  ///
449 	Gone                         = 410,  ///
450 	LengthRequired               = 411,  ///
451 	PreconditionFailed           = 412,  ///
452 	RequestEntityTooLarge        = 413,  ///
453 	RequestUriTooLong            = 414,  ///
454 	UnsupportedMediaType         = 415,  ///
455 	RequestedRangeNotSatisfiable = 416,  ///
456 	ExpectationFailed            = 417,  ///
457 	MisdirectedRequest           = 421,  ///
458 	UnprocessableContent         = 422,  ///
459 	Locked                       = 423,  ///
460 	FailedDependency             = 424,  ///
461 	TooEarly                     = 425,  ///
462 	UpgradeRequired              = 426,  ///
463 	PreconditionRequired         = 428,  ///
464 	TooManyRequests              = 429,  ///
465 	RequestHeaderFieldsTooLarge  = 431,  ///
466 	UnavailableForLegalReasons   = 451,  ///
467 
468 	InternalServerError          = 500,  ///
469 	NotImplemented               = 501,  ///
470 	BadGateway                   = 502,  ///
471 	ServiceUnavailable           = 503,  ///
472 	GatewayTimeout               = 504,  ///
473 	HttpVersionNotSupported      = 505,  ///
474 }
475 
476 /// HTTP reply class
477 class HttpResponse : HttpMessage
478 {
479 public:
480 	HttpStatusCode status; /// HTTP status code
481 	string statusMessage; /// HTTP status message, if one was supplied
482 
483 	/// What Zlib compression level to use when compressing the reply.
484 	/// Set to a negative value to disable compression.
485 	int compressionLevel = 1;
486 
487 	/// Returns the message corresponding to the given `HttpStatusCode`,
488 	/// or `null` if the code is unknown.
489 	static string getStatusMessage(HttpStatusCode code)
490 	{
491 		switch(code)
492 		{
493 			case 100: return "Continue";
494 			case 101: return "Switching Protocols";
495 
496 			case 200: return "OK";
497 			case 201: return "Created";
498 			case 202: return "Accepted";
499 			case 203: return "Non-Authoritative Information";
500 			case 204: return "No Content";
501 			case 205: return "Reset Content";
502 			case 206: return "Partial Content";
503 			case 300: return "Multiple Choices";
504 			case 301: return "Moved Permanently";
505 			case 302: return "Found";
506 			case 303: return "See Other";
507 			case 304: return "Not Modified";
508 			case 305: return "Use Proxy";
509 			case 306: return "(Unused)";
510 			case 307: return "Temporary Redirect";
511 
512 			case 400: return "Bad Request";
513 			case 401: return "Unauthorized";
514 			case 402: return "Payment Required";
515 			case 403: return "Forbidden";
516 			case 404: return "Not Found";
517 			case 405: return "Method Not Allowed";
518 			case 406: return "Not Acceptable";
519 			case 407: return "Proxy Authentication Required";
520 			case 408: return "Request Timeout";
521 			case 409: return "Conflict";
522 			case 410: return "Gone";
523 			case 411: return "Length Required";
524 			case 412: return "Precondition Failed";
525 			case 413: return "Request Entity Too Large";
526 			case 414: return "Request-URI Too Long";
527 			case 415: return "Unsupported Media Type";
528 			case 416: return "Requested Range Not Satisfiable";
529 			case 417: return "Expectation Failed";
530 
531 			case 500: return "Internal Server Error";
532 			case 501: return "Not Implemented";
533 			case 502: return "Bad Gateway";
534 			case 503: return "Service Unavailable";
535 			case 504: return "Gateway Timeout";
536 			case 505: return "HTTP Version Not Supported";
537 			default: return null;
538 		}
539 	}
540 
541 	/// Set the response status code and message
542 	void setStatus(HttpStatusCode code)
543 	{
544 		status = code;
545 		statusMessage = getStatusMessage(code);
546 	}
547 
548 	/// Initializes this `HttpResponse` with the given `statusLine`.
549 	final void parseStatusLine(string statusLine)
550 	{
551 		auto versionEnd = statusLine.indexOf(' ');
552 		if (versionEnd == -1)
553 			throw new Exception("Malformed status line");
554 		protocolVersion = statusLine[0..versionEnd];
555 		statusLine = statusLine[versionEnd+1..statusLine.length];
556 
557 		auto statusEnd = statusLine.indexOf(' ');
558 		string statusCode;
559 		if (statusEnd >= 0)
560 		{
561 			statusCode = statusLine[0 .. statusEnd];
562 			statusMessage = statusLine[statusEnd+1..statusLine.length];
563 		}
564 		else
565 		{
566 			statusCode = statusLine;
567 			statusMessage = null;
568 		}
569 		status = cast(HttpStatusCode)to!ushort(statusCode);
570 	}
571 
572 	/// If the data is compressed, return the decompressed data
573 	// this is not a property on purpose - to avoid using it multiple times as it will unpack the data on every access
574 	// TODO: there is no reason for above limitation
575 	Data getContent()
576 	{
577 		if ("Content-Encoding" in headers && headers["Content-Encoding"]=="deflate")
578 		{
579 			static if (haveZlib)
580 				return zlib.uncompress(data[]).joinData();
581 			else
582 				throw new Exception("Built without zlib - can't decompress \"Content-Encoding: deflate\" content");
583 		}
584 		if ("Content-Encoding" in headers && headers["Content-Encoding"]=="gzip")
585 		{
586 			static if (haveZlib)
587 				return gzip.uncompress(data[]).joinData();
588 			else
589 				throw new Exception("Built without zlib - can't decompress \"Content-Encoding: gzip\" content");
590 		}
591 		return data.joinData();
592 	}
593 
594 	static if (haveZlib)
595 	protected void compressWithDeflate()
596 	{
597 		assert(compressionLevel >= 0);
598 		data = zlib.compress(data[], zlib.ZlibOptions(compressionLevel));
599 	}
600 
601 	static if (haveZlib)
602 	protected void compressWithGzip()
603 	{
604 		assert(compressionLevel >= 0);
605 		data = gzip.compress(data[], zlib.ZlibOptions(compressionLevel));
606 	}
607 
608 	/// Called by the server to compress content, if possible/appropriate
609 	final package void optimizeData(ref const Headers requestHeaders)
610 	{
611 		if (compressionLevel < 0)
612 			return;
613 		auto acceptEncoding = requestHeaders.get("Accept-Encoding", null);
614 		if (acceptEncoding && "Content-Encoding" !in headers && "Content-Length" !in headers)
615 		{
616 			auto contentType = headers.get("Content-Type", null);
617 			if (contentType.startsWith("text/")
618 			 || contentType == "application/json"
619 			 || contentType == "image/vnd.microsoft.icon"
620 			 || contentType == "image/svg+xml")
621 			{
622 				auto supported = parseItemList(acceptEncoding) ~ ["*"];
623 				foreach (method; supported)
624 					switch (method)
625 					{
626 						static if (haveZlib)
627 						{
628 							case "deflate":
629 								headers["Content-Encoding"] = method;
630 								headers.add("Vary", "Accept-Encoding");
631 								compressWithDeflate();
632 								return;
633 							case "gzip":
634 								headers["Content-Encoding"] = method;
635 								headers.add("Vary", "Accept-Encoding");
636 								compressWithGzip();
637 								return;
638 						}
639 						case "*":
640 							if("Content-Encoding" in headers)
641 								headers.remove("Content-Encoding");
642 							return;
643 						default:
644 							break;
645 					}
646 				assert(0);
647 			}
648 		}
649 	}
650 
651 	/// Called by the server to apply range request.
652 	final package void sliceData(ref const Headers requestHeaders)
653 	{
654 		if (status == HttpStatusCode.OK &&
655 			"Content-Range" !in headers &&
656 			"Accept-Ranges" !in headers &&
657 			"Content-Length" !in headers)
658 		{
659 			if ("If-Modified-Since" in requestHeaders &&
660 				"Last-Modified" in headers &&
661 				headers["Last-Modified"].parseTime!(TimeFormats.HTTP) <= requestHeaders["If-Modified-Since"].parseTime!(TimeFormats.HTTP))
662 			{
663 				setStatus(HttpStatusCode.NotModified);
664 				data = null;
665 				return;
666 			}
667 
668 			headers["Accept-Ranges"] = "bytes";
669 			auto prange = "Range" in requestHeaders;
670 			if (prange && (*prange).startsWith("bytes="))
671 			{
672 				auto ranges = (*prange)[6..$].split(",")[0].split("-").map!(s => s.length ? s.to!size_t : size_t.max)().array();
673 				enforce(ranges.length == 2, "Bad range request");
674 				ranges[1]++;
675 				auto datum = this.data.bytes;
676 				auto datumLength = datum.length;
677 				if (ranges[1] == size_t.min) // was not specified (size_t.max overflowed into 0)
678 					ranges[1] = datumLength;
679 				if (ranges[0] >= datumLength || ranges[0] >= ranges[1] || ranges[1] > datumLength)
680 				{
681 					//writeError(HttpStatusCode.RequestedRangeNotSatisfiable);
682 					setStatus(HttpStatusCode.RequestedRangeNotSatisfiable);
683 					data = DataVec(Data(statusMessage.asBytes));
684 					return;
685 				}
686 				else
687 				{
688 					setStatus(HttpStatusCode.PartialContent);
689 					this.data = datum[ranges[0]..ranges[1]];
690 					headers["Content-Range"] = "bytes %d-%d/%d".format(ranges[0], ranges[0] + this.data.bytes.length - 1, datumLength);
691 				}
692 			}
693 		}
694 	}
695 
696 	protected void copyTo(typeof(this) other)
697 	{
698 		other.status = status;
699 		other.statusMessage = statusMessage;
700 		other.compressionLevel = compressionLevel;
701 	}
702 	alias copyTo = typeof(super).copyTo;
703 
704 	final typeof(this) dup()
705 	{
706 		auto result = new typeof(this);
707 		copyTo(result);
708 		return result;
709 	} ///
710 }
711 
712 /// Sets headers to request clients to not cache a response.
713 void disableCache(ref Headers headers)
714 {
715 	headers["Expires"] = "Mon, 26 Jul 1997 05:00:00 GMT";  // disable IE caching
716 	//headers["Last-Modified"] = "" . gmdate( "D, d M Y H:i:s" ) . " GMT";
717 	headers["Cache-Control"] = "no-cache, must-revalidate";
718 	headers["Pragma"] = "no-cache";
719 }
720 
721 /// Sets headers to request clients to cache a response indefinitely.
722 void cacheForever(ref Headers headers)
723 {
724 	headers["Expires"] = httpTime(Clock.currTime().add!"years"(1));
725 	headers["Cache-Control"] = "public, max-age=31536000";
726 }
727 
728 /// Formats a timestamp in the format used by HTTP (RFC 2822).
729 string httpTime(SysTime time)
730 {
731 	time.timezone = UTC();
732 	return time.formatTime!(TimeFormats.HTTP)();
733 }
734 
735 import std.algorithm : sort;
736 
737 /// Parses a list in the format of "a, b, c;q=0.5, d" and returns
738 /// an array of items sorted by "q" (["a", "b", "d", "c"])
739 string[] parseItemList(string s)
740 {
741 	static struct Item
742 	{
743 		float q = 1.0;
744 		string str;
745 
746 		this(string s)
747 		{
748 			auto params = s.split(";");
749 			if (!params.length) return;
750 			str = params[0];
751 			foreach (param; params[1..$])
752 				if (param.startsWith("q="))
753 					q = to!float(param[2..$]);
754 		}
755 	}
756 
757 	return s
758 		.split(",")
759 		.amap!(a => Item(strip(a)))()
760 		.asort!`a.q > b.q`()
761 		.amap!`a.str`();
762 }
763 
764 debug(ae_unittest) unittest
765 {
766 	assert(parseItemList("a, b, c;q=0.5, d") == ["a", "b", "d", "c"]);
767 }
768 
769 // TODO: optimize / move to HtmlWriter
770 deprecated("Use ae.utils.xml.entities")
771 string httpEscape(string str)
772 {
773 	string result;
774 	foreach(c;str)
775 		switch(c)
776 		{
777 			case '<':
778 				result ~= "&lt;";
779 				break;
780 			case '>':
781 				result ~= "&gt;";
782 				break;
783 			case '&':
784 				result ~= "&amp;";
785 				break;
786 			case '\xDF':  // the beta-like symbol
787 				result ~= "&szlig;";
788 				break;
789 			default:
790 				result ~= [c];
791 		}
792 	return result;
793 }
794 
795 public import ae.net.ietf.url : UrlParameters, encodeUrlParameter, encodeUrlParameters, decodeUrlParameter, decodeUrlParameters;
796 
797 /// Represents a part from a multipart/* message.
798 struct MultipartPart
799 {
800 	/// The part's individual headers.
801 	Headers headers;
802 
803 	/// The part's contents.
804 	Data data;
805 }
806 
807 /// Encode a multipart body with the given parts and boundary.
808 Data encodeMultipart(MultipartPart[] parts, string boundary)
809 {
810 	TData!char data;
811 	foreach (ref part; parts)
812 	{
813 		data ~= "--" ~ boundary ~ "\r\n";
814 		foreach (name, value; part.headers)
815 			data ~= name ~ ": " ~ value ~ "\r\n";
816 		data ~= "\r\n";
817 		assert(part.data.asDataOf!char.indexOf(boundary) < 0);
818 		data ~= part.data.asDataOf!char;
819 		data ~= "\r\n";
820 	}
821 	data ~= "--" ~ boundary ~ "--\r\n";
822 	return data.asDataOf!ubyte;
823 }
824 
825 /// Decode a multipart body using the given boundary.
826 MultipartPart[] decodeMultipart(Data data, string boundary)
827 {
828 	MultipartPart[] result;
829 	data.asDataOf!char.enter((scope s) {
830 		auto term = "\r\n--" ~ boundary ~ "--\r\n";
831 		enforce(s.endsWith(term), "Bad multipart terminator");
832 		s = s[0..$-term.length];
833 		auto delim = "--" ~ boundary ~ "\r\n";
834 		enforce(s.skipOver(delim), "Bad multipart start");
835 		delim = "\r\n" ~ delim;
836 		auto parts = s.split(delim);
837 		foreach (part; parts)
838 		{
839 			auto segs = part.findSplit("\r\n\r\n");
840 			enforce(segs[1], "Can't find headers in multipart part");
841 			MultipartPart p;
842 			foreach (line; segs[0].split("\r\n"))
843 			{
844 				auto hparts = line.findSplit(":");
845 				p.headers[hparts[0].strip.idup] = hparts[2].strip.idup;
846 			}
847 			p.data = Data(segs[2].asBytes);
848 			result ~= p;
849 		}
850 	});
851 	return result;
852 }
853 
854 debug(ae_unittest) unittest
855 {
856 	auto parts = [
857 		MultipartPart(Headers(["Foo" : "bar"]), Data.init),
858 		MultipartPart(Headers(["Baz" : "quux", "Frob" : "xyzzy"]), Data("Content goes here\xFF".asBytes)),
859 	];
860 	auto boundary = "abcde";
861 	auto parts2 = parts.encodeMultipart(boundary).decodeMultipart(boundary);
862 	assert(parts2.length == parts.length);
863 	foreach (p; 0..parts.length)
864 	{
865 		assert(parts[p].headers == parts2[p].headers);
866 		assert(parts[p].data.unsafeContents == parts2[p].data.unsafeContents);
867 	}
868 }
869 
870 private bool asciiStartsWith(string s, string prefix) pure nothrow @nogc
871 {
872 	if (s.length < prefix.length)
873 		return false;
874 	import std.ascii;
875 	foreach (i, c; prefix)
876 		if (toLower(c) != toLower(s[i]))
877 			return false;
878 	return true;
879 }