1 /**
2  * Concepts shared between HTTP clients and servers.
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Stéphan Kochen <stephan@kochen.nl>
12  *   Vladimir Panteleev <ae@cy.md>
13  *   Simon Arlott
14  */
15 
16 module ae.net.http.common;
17 
18 import core.time;
19 
20 import std.algorithm;
21 import std.array;
22 import std.base64;
23 import std.string;
24 import std.conv;
25 import std.ascii;
26 import std.exception;
27 import std.datetime;
28 import std.typecons : tuple;
29 
30 import ae.net.ietf.headers;
31 import ae.sys.data;
32 import ae.sys.dataset;
33 import ae.utils.array : amap, afilter, auniq, asort, asBytes, as;
34 import ae.utils.text;
35 import ae.utils.time;
36 
37 version (ae_with_zlib) // Explicit override
38 	enum haveZlib = true;
39 else
40 version (ae_without_zlib) // Explicit override
41 	enum haveZlib = false;
42 else
43 version (Have_ae) // Building with Dub
44 {
45 	version (Have_ae_zlib) // Using ae:zlib
46 		enum haveZlib = true;
47 	else
48 		enum haveZlib = false;
49 }
50 else // Not building with Dub
51 	enum haveZlib = true; // Pull in zlib by default
52 
53 static if (haveZlib)
54 {
55 	import zlib = ae.utils.zlib;
56 	import gzip = ae.utils.gzip;
57 }
58 
59 /// Base HTTP message class
60 private abstract class HttpMessage
61 {
62 public:
63 	string protocol = "http";
64 	string protocolVersion = "1.0";
65 	Headers headers;
66 	DataVec data;
67 	SysTime creationTime;
68 
69 	this()
70 	{
71 		creationTime = Clock.currTime();
72 	}
73 
74 	@property Duration age()
75 	{
76 		return Clock.currTime() - creationTime;
77 	}
78 
79 	/// For `dup`.
80 	protected void copyTo(typeof(this) other)
81 	{
82 		other.protocol = protocol;
83 		other.protocolVersion = protocolVersion;
84 		other.headers = headers.dup;
85 		other.data = data.dup;
86 		other.creationTime = creationTime;
87 	}
88 }
89 
90 // TODO: Separate this from an URL type
91 
92 /// HTTP request class
93 class HttpRequest : HttpMessage
94 {
95 public:
96 	/// HTTP method, e.g., "GET".
97 	string method = "GET";
98 
99 	/// If this request is going through a HTTP proxy server, this
100 	/// should be set to its address.
101 	string proxy;
102 
103 	this()
104 	{
105 	} ///
106 
107 	this(string url)
108 	{
109 		this.resource = url;
110 	} ///
111 
112 	/// For `dup`.
113 	protected void copyTo(typeof(this) other)
114 	{
115 		super.copyTo(other);
116 		other.method = method;
117 		other.proxy = proxy;
118 		other._resource = _resource;
119 		other._port = _port;
120 	}
121 	alias copyTo = typeof(super).copyTo;
122 
123 	final typeof(this) dup()
124 	{
125 		auto result = new typeof(this);
126 		copyTo(result);
127 		return result;
128 	} ///
129 
130 	/// Resource part of URL (everything after the hostname)
131 	@property string resource() const pure nothrow @nogc
132 	{
133 		return _resource;
134 	}
135 
136 	/// Set the resource part of the URL, or the entire URL.
137 	/// Setting the resource to a full URL will fill in the Host header, as well.
138 	@property void resource(string value) pure
139 	{
140 		_resource = value;
141 
142 		// applies to both Client/Server as some clients put a full URL in the GET line instead of using a "Host" header
143 		string protocol;
144 		if (_resource.asciiStartsWith("http://"))
145 			protocol = "http";
146 		else
147 		if (_resource.asciiStartsWith("https://"))
148 			protocol = "https";
149 
150 		if (protocol)
151 		{
152 			this.protocol = protocol;
153 
154 			value = value[protocol.length+3..$];
155 			auto pathstart = value.indexOf('/');
156 			string _host;
157 			if (pathstart == -1)
158 			{
159 				_host = value;
160 				_resource = "/";
161 			}
162 			else
163 			{
164 				_host = value[0..pathstart];
165 				_resource = value[pathstart..$];
166 			}
167 
168 			auto authEnd = _host.indexOf('@');
169 			if (authEnd != -1)
170 			{
171 				// Assume HTTP Basic auth
172 				import ae.utils.array : asBytes;
173 				headers["Authorization"] = "Basic " ~ Base64.encode(_host[0 .. authEnd].asBytes).assumeUnique;
174 				_host = _host[authEnd + 1 .. $];
175 			}
176 
177 			auto portstart = _host.indexOf(':');
178 			if (portstart != -1)
179 			{
180 				port = to!ushort(_host[portstart+1..$]);
181 				_host = _host[0..portstart];
182 			}
183 			host = _host;
184 		}
185 	}
186 
187 	/// The hostname, without the port number
188 	@property string host() const pure nothrow @nogc
189 	{
190 		string _host = headers.get("Host", null);
191 		// auto colon = _host.lastIndexOf(':'); // https://issues.dlang.org/show_bug.cgi?id=24008
192 		sizediff_t colon = -1; foreach_reverse (i, c; _host) if (c == ':') { colon = i; break; }
193 		return colon < 0 ? _host : _host[0 .. colon];
194 	}
195 
196 	/// Sets the hostname (and the `"Host"` header).
197 	/// Must not include a port number.
198 	/// Does not change the previously-set port number.
199 	@property void host(string _host) pure
200 	{
201 		auto _port = this.port;
202 		headers["Host"] = _port==protocolDefaultPort ? _host : _host ~ ":" ~ text(_port);
203 	}
204 
205 	/// Retrieves the default port number for the currently set `protocol`.
206 	@property ushort protocolDefaultPort() const pure
207 	{
208 		switch (protocol)
209 		{
210 			case "http":
211 				return 80;
212 			case "https":
213 				return 443;
214 			default:
215 				throw new Exception("Unknown protocol: " ~ protocol);
216 		}
217 	}
218 
219 	/// Port number, from `"Host"` header.
220 	/// Defaults to `protocolDefaultPort`.
221 	@property ushort port() const pure
222 	{
223 		if ("Host" in headers)
224 		{
225 			string _host = headers["Host"];
226 			auto colon = _host.lastIndexOf(":");
227 			return colon<0 ? protocolDefaultPort : to!ushort(_host[colon+1..$]);
228 		}
229 		else
230 			return _port ? _port : protocolDefaultPort;
231 	}
232 
233 	/// Sets the port number.
234 	/// If it is equal to `protocolDefaultPort`, then it is not
235 	/// included in the `"Host"` header.
236 	@property void port(ushort _port) pure
237 	{
238 		if ("Host" in headers)
239 		{
240 			if (_port == protocolDefaultPort)
241 				headers["Host"] = this.host;
242 			else
243 				headers["Host"] = this.host ~ ":" ~ text(_port);
244 		}
245 		else
246 			this._port = _port;
247 	}
248 
249 	/// Path part of request (until the `'?'`).
250 	@property string path() const pure nothrow @nogc
251 	{
252 		auto p = resource.indexOf('?');
253 		if (p >= 0)
254 			return resource[0..p];
255 		else
256 			return resource;
257 	}
258 
259 	/// Query string part of request (atfer the `'?'`).
260 	@property string queryString() const pure nothrow @nogc
261 	{
262 		auto p = resource.indexOf('?');
263 		if (p >= 0)
264 			return resource[p+1..$];
265 		else
266 			return null;
267 	}
268 
269 	/// ditto
270 	@property void queryString(string value) pure
271 	{
272 		auto p = resource.indexOf('?');
273 		if (p >= 0)
274 			resource = resource[0..p];
275 		if (value)
276 			resource = resource ~ '?' ~ value;
277 	}
278 
279 	/// The query string parameters.
280 	@property UrlParameters urlParameters()
281 	{
282 		return decodeUrlParameters(queryString);
283 	}
284 
285 	/// ditto
286 	@property void urlParameters(UrlParameters parameters)
287 	{
288 		queryString = encodeUrlParameters(parameters);
289 	}
290 
291 	/// URL without resource (protocol, host and port).
292 	@property string root()
293 	{
294 		return protocol ~ "://" ~ host ~ (port==protocolDefaultPort ? null : ":" ~ to!string(port));
295 	}
296 
297 	/// Full URL.
298 	@property string url()
299 	{
300 		return root ~ resource;
301 	}
302 
303 	/// Full URL without query parameters or fragment.
304 	@property string baseURL()
305 	{
306 		return root ~ resource.findSplit("?")[0];
307 	}
308 
309 	/// The hostname part of the proxy address, if any.
310 	@property string proxyHost()
311 	{
312 		auto portstart = proxy.indexOf(':');
313 		if (portstart != -1)
314 			return proxy[0..portstart];
315 		return proxy;
316 	}
317 
318 	/// The port number of the proxy address if it specified, otherwise `80`.
319 	@property ushort proxyPort()
320 	{
321 		auto portstart = proxy.indexOf(':');
322 		if (portstart != -1)
323 			return to!ushort(proxy[portstart+1..$]);
324 		return 80;
325 	}
326 
327 	/// Parse the first line in a HTTP request ("METHOD /resource HTTP/1.x").
328 	void parseRequestLine(string reqLine)
329 	{
330 		enforce(reqLine.length > 10, "Request line too short");
331 		auto methodEnd = reqLine.indexOf(' ');
332 		enforce(methodEnd > 0, "Malformed request line");
333 		method = reqLine[0 .. methodEnd];
334 		reqLine = reqLine[methodEnd + 1 .. reqLine.length];
335 
336 		auto resourceEnd = reqLine.lastIndexOf(' ');
337 		enforce(resourceEnd > 0, "Malformed request line");
338 		resource = reqLine[0 .. resourceEnd];
339 
340 		string protocol = reqLine[resourceEnd+1..$];
341 		enforce(protocol.startsWith("HTTP/"));
342 		protocolVersion = protocol[5..$];
343 	}
344 
345 	/// Decodes submitted form data, and returns an AA of values.
346 	UrlParameters decodePostData()
347 	{
348 		auto contentType = headers.get("Content-Type", "").decodeTokenHeader;
349 
350 		switch (contentType.value)
351 		{
352 			case "application/x-www-form-urlencoded":
353 				return decodeUrlParameters(data.joinToGC().as!string);
354 			case "multipart/form-data":
355 				return decodeMultipart(data.joinData(), contentType.properties.get("boundary", null))
356 					.map!(part => tuple(
357 						part.headers.get("Content-Disposition", null).decodeTokenHeader.properties.get("name", null),
358 						part.data.asDataOf!char.toGC().assumeUnique,
359 					))
360 					.UrlParameters;
361 			case "":
362 				throw new Exception("No Content-Type");
363 			default:
364 				throw new Exception("Unknown Content-Type: " ~ contentType.value);
365 		}
366 	}
367 
368 	/// Get list of hosts as specified in headers (e.g. X-Forwarded-For).
369 	/// First item in returned array is the node furthest away.
370 	/// Duplicates are removed.
371 	/// Specify socket remote address in remoteHost to add it to the list.
372 	deprecated("Insecure, use HttpServer.remoteIPHeader")
373 	string[] remoteHosts(string remoteHost = null)
374 	{
375 		return
376 			(headers.get("X-Forwarded-For", null).split(",").amap!(std..string.strip)() ~
377 			 headers.get("X-Forwarded-Host", null) ~
378 			 remoteHost)
379 			.afilter!`a && a != "unknown"`()
380 			.auniq();
381 	}
382 
383 	deprecated unittest
384 	{
385 		auto req = new HttpRequest();
386 		assert(req.remoteHosts() == []);
387 		assert(req.remoteHosts("3.3.3.3") == ["3.3.3.3"]);
388 
389 		req.headers["X-Forwarded-For"] = "1.1.1.1, 2.2.2.2";
390 		req.headers["X-Forwarded-Host"] = "2.2.2.2";
391 		assert(req.remoteHosts("3.3.3.3") == ["1.1.1.1", "2.2.2.2", "3.3.3.3"]);
392 	}
393 
394 	/// Basic cookie parsing
395 	string[string] getCookies()
396 	{
397 		string[string] cookies;
398 		foreach (segment; headers.get("Cookie", null).split(";"))
399 		{
400 			segment = segment.strip();
401 			auto p = segment.indexOf('=');
402 			if (p > 0)
403 				cookies[segment[0..p]] = segment[p+1..$];
404 		}
405 		return cookies;
406 	}
407 
408 private:
409 	string _resource;
410 	ushort _port = 0; // used only when no "Host" in headers; otherwise, taken from there
411 }
412 
413 /// HTTP response status codes
414 enum HttpStatusCode : ushort
415 {
416 	None                         =   0,  ///
417 
418 	Continue                     = 100,  ///
419 	SwitchingProtocols           = 101,  ///
420 
421 	OK                           = 200,  ///
422 	Created                      = 201,  ///
423 	Accepted                     = 202,  ///
424 	NonAuthoritativeInformation  = 203,  ///
425 	NoContent                    = 204,  ///
426 	ResetContent                 = 205,  ///
427 	PartialContent               = 206,  ///
428 
429 	MultipleChoices              = 300,  ///
430 	MovedPermanently             = 301,  ///
431 	Found                        = 302,  ///
432 	SeeOther                     = 303,  ///
433 	NotModified                  = 304,  ///
434 	UseProxy                     = 305,  ///
435 	//(Unused)                   = 306,  ///
436 	TemporaryRedirect            = 307,  ///
437 
438 	BadRequest                   = 400,  ///
439 	Unauthorized                 = 401,  ///
440 	PaymentRequired              = 402,  ///
441 	Forbidden                    = 403,  ///
442 	NotFound                     = 404,  ///
443 	MethodNotAllowed             = 405,  ///
444 	NotAcceptable                = 406,  ///
445 	ProxyAuthenticationRequired  = 407,  ///
446 	RequestTimeout               = 408,  ///
447 	Conflict                     = 409,  ///
448 	Gone                         = 410,  ///
449 	LengthRequired               = 411,  ///
450 	PreconditionFailed           = 412,  ///
451 	RequestEntityTooLarge        = 413,  ///
452 	RequestUriTooLong            = 414,  ///
453 	UnsupportedMediaType         = 415,  ///
454 	RequestedRangeNotSatisfiable = 416,  ///
455 	ExpectationFailed            = 417,  ///
456 	MisdirectedRequest           = 421,  ///
457 	UnprocessableContent         = 422,  ///
458 	Locked                       = 423,  ///
459 	FailedDependency             = 424,  ///
460 	TooEarly                     = 425,  ///
461 	UpgradeRequired              = 426,  ///
462 	PreconditionRequired         = 428,  ///
463 	TooManyRequests              = 429,  ///
464 	RequestHeaderFieldsTooLarge  = 431,  ///
465 	UnavailableForLegalReasons   = 451,  ///
466 
467 	InternalServerError          = 500,  ///
468 	NotImplemented               = 501,  ///
469 	BadGateway                   = 502,  ///
470 	ServiceUnavailable           = 503,  ///
471 	GatewayTimeout               = 504,  ///
472 	HttpVersionNotSupported      = 505,  ///
473 }
474 
475 /// HTTP reply class
476 class HttpResponse : HttpMessage
477 {
478 public:
479 	HttpStatusCode status; /// HTTP status code
480 	string statusMessage; /// HTTP status message, if one was supplied
481 
482 	/// What Zlib compression level to use when compressing the reply.
483 	/// Set to a negative value to disable compression.
484 	int compressionLevel = 1;
485 
486 	/// Returns the message corresponding to the given `HttpStatusCode`,
487 	/// or `null` if the code is unknown.
488 	static string getStatusMessage(HttpStatusCode code)
489 	{
490 		switch(code)
491 		{
492 			case 100: return "Continue";
493 			case 101: return "Switching Protocols";
494 
495 			case 200: return "OK";
496 			case 201: return "Created";
497 			case 202: return "Accepted";
498 			case 203: return "Non-Authoritative Information";
499 			case 204: return "No Content";
500 			case 205: return "Reset Content";
501 			case 206: return "Partial Content";
502 			case 300: return "Multiple Choices";
503 			case 301: return "Moved Permanently";
504 			case 302: return "Found";
505 			case 303: return "See Other";
506 			case 304: return "Not Modified";
507 			case 305: return "Use Proxy";
508 			case 306: return "(Unused)";
509 			case 307: return "Temporary Redirect";
510 
511 			case 400: return "Bad Request";
512 			case 401: return "Unauthorized";
513 			case 402: return "Payment Required";
514 			case 403: return "Forbidden";
515 			case 404: return "Not Found";
516 			case 405: return "Method Not Allowed";
517 			case 406: return "Not Acceptable";
518 			case 407: return "Proxy Authentication Required";
519 			case 408: return "Request Timeout";
520 			case 409: return "Conflict";
521 			case 410: return "Gone";
522 			case 411: return "Length Required";
523 			case 412: return "Precondition Failed";
524 			case 413: return "Request Entity Too Large";
525 			case 414: return "Request-URI Too Long";
526 			case 415: return "Unsupported Media Type";
527 			case 416: return "Requested Range Not Satisfiable";
528 			case 417: return "Expectation Failed";
529 
530 			case 500: return "Internal Server Error";
531 			case 501: return "Not Implemented";
532 			case 502: return "Bad Gateway";
533 			case 503: return "Service Unavailable";
534 			case 504: return "Gateway Timeout";
535 			case 505: return "HTTP Version Not Supported";
536 			default: return null;
537 		}
538 	}
539 
540 	/// Set the response status code and message
541 	void setStatus(HttpStatusCode code)
542 	{
543 		status = code;
544 		statusMessage = getStatusMessage(code);
545 	}
546 
547 	/// Initializes this `HttpResponse` with the given `statusLine`.
548 	final void parseStatusLine(string statusLine)
549 	{
550 		auto versionEnd = statusLine.indexOf(' ');
551 		if (versionEnd == -1)
552 			throw new Exception("Malformed status line");
553 		protocolVersion = statusLine[0..versionEnd];
554 		statusLine = statusLine[versionEnd+1..statusLine.length];
555 
556 		auto statusEnd = statusLine.indexOf(' ');
557 		string statusCode;
558 		if (statusEnd >= 0)
559 		{
560 			statusCode = statusLine[0 .. statusEnd];
561 			statusMessage = statusLine[statusEnd+1..statusLine.length];
562 		}
563 		else
564 		{
565 			statusCode = statusLine;
566 			statusMessage = null;
567 		}
568 		status = cast(HttpStatusCode)to!ushort(statusCode);
569 	}
570 
571 	/// If the data is compressed, return the decompressed data
572 	// this is not a property on purpose - to avoid using it multiple times as it will unpack the data on every access
573 	// TODO: there is no reason for above limitation
574 	Data getContent()
575 	{
576 		if ("Content-Encoding" in headers && headers["Content-Encoding"]=="deflate")
577 		{
578 			static if (haveZlib)
579 				return zlib.uncompress(data[]).joinData();
580 			else
581 				throw new Exception("Built without zlib - can't decompress \"Content-Encoding: deflate\" content");
582 		}
583 		if ("Content-Encoding" in headers && headers["Content-Encoding"]=="gzip")
584 		{
585 			static if (haveZlib)
586 				return gzip.uncompress(data[]).joinData();
587 			else
588 				throw new Exception("Built without zlib - can't decompress \"Content-Encoding: gzip\" content");
589 		}
590 		return data.joinData();
591 	}
592 
593 	static if (haveZlib)
594 	protected void compressWithDeflate()
595 	{
596 		assert(compressionLevel >= 0);
597 		data = zlib.compress(data[], zlib.ZlibOptions(compressionLevel));
598 	}
599 
600 	static if (haveZlib)
601 	protected void compressWithGzip()
602 	{
603 		assert(compressionLevel >= 0);
604 		data = gzip.compress(data[], zlib.ZlibOptions(compressionLevel));
605 	}
606 
607 	/// Called by the server to compress content, if possible/appropriate
608 	final package void optimizeData(ref const Headers requestHeaders)
609 	{
610 		if (compressionLevel < 0)
611 			return;
612 		auto acceptEncoding = requestHeaders.get("Accept-Encoding", null);
613 		if (acceptEncoding && "Content-Encoding" !in headers && "Content-Length" !in headers)
614 		{
615 			auto contentType = headers.get("Content-Type", null);
616 			if (contentType.startsWith("text/")
617 			 || contentType == "application/json"
618 			 || contentType == "image/vnd.microsoft.icon"
619 			 || contentType == "image/svg+xml")
620 			{
621 				auto supported = parseItemList(acceptEncoding) ~ ["*"];
622 				foreach (method; supported)
623 					switch (method)
624 					{
625 						static if (haveZlib)
626 						{
627 							case "deflate":
628 								headers["Content-Encoding"] = method;
629 								headers.add("Vary", "Accept-Encoding");
630 								compressWithDeflate();
631 								return;
632 							case "gzip":
633 								headers["Content-Encoding"] = method;
634 								headers.add("Vary", "Accept-Encoding");
635 								compressWithGzip();
636 								return;
637 						}
638 						case "*":
639 							if("Content-Encoding" in headers)
640 								headers.remove("Content-Encoding");
641 							return;
642 						default:
643 							break;
644 					}
645 				assert(0);
646 			}
647 		}
648 	}
649 
650 	/// Called by the server to apply range request.
651 	final package void sliceData(ref const Headers requestHeaders)
652 	{
653 		if (status == HttpStatusCode.OK &&
654 			"Content-Range" !in headers &&
655 			"Accept-Ranges" !in headers &&
656 			"Content-Length" !in headers)
657 		{
658 			if ("If-Modified-Since" in requestHeaders &&
659 				"Last-Modified" in headers &&
660 				headers["Last-Modified"].parseTime!(TimeFormats.HTTP) <= requestHeaders["If-Modified-Since"].parseTime!(TimeFormats.HTTP))
661 			{
662 				setStatus(HttpStatusCode.NotModified);
663 				data = null;
664 				return;
665 			}
666 
667 			headers["Accept-Ranges"] = "bytes";
668 			auto prange = "Range" in requestHeaders;
669 			if (prange && (*prange).startsWith("bytes="))
670 			{
671 				auto ranges = (*prange)[6..$].split(",")[0].split("-").map!(s => s.length ? s.to!size_t : size_t.max)().array();
672 				enforce(ranges.length == 2, "Bad range request");
673 				ranges[1]++;
674 				auto datum = this.data.bytes;
675 				auto datumLength = datum.length;
676 				if (ranges[1] == size_t.min) // was not specified (size_t.max overflowed into 0)
677 					ranges[1] = datumLength;
678 				if (ranges[0] >= datumLength || ranges[0] >= ranges[1] || ranges[1] > datumLength)
679 				{
680 					//writeError(HttpStatusCode.RequestedRangeNotSatisfiable);
681 					setStatus(HttpStatusCode.RequestedRangeNotSatisfiable);
682 					data = DataVec(Data(statusMessage.asBytes));
683 					return;
684 				}
685 				else
686 				{
687 					setStatus(HttpStatusCode.PartialContent);
688 					this.data = datum[ranges[0]..ranges[1]];
689 					headers["Content-Range"] = "bytes %d-%d/%d".format(ranges[0], ranges[0] + this.data.bytes.length - 1, datumLength);
690 				}
691 			}
692 		}
693 	}
694 
695 	protected void copyTo(typeof(this) other)
696 	{
697 		other.status = status;
698 		other.statusMessage = statusMessage;
699 		other.compressionLevel = compressionLevel;
700 	}
701 	alias copyTo = typeof(super).copyTo;
702 
703 	final typeof(this) dup()
704 	{
705 		auto result = new typeof(this);
706 		copyTo(result);
707 		return result;
708 	} ///
709 }
710 
711 /// Sets headers to request clients to not cache a response.
712 void disableCache(ref Headers headers)
713 {
714 	headers["Expires"] = "Mon, 26 Jul 1997 05:00:00 GMT";  // disable IE caching
715 	//headers["Last-Modified"] = "" . gmdate( "D, d M Y H:i:s" ) . " GMT";
716 	headers["Cache-Control"] = "no-cache, must-revalidate";
717 	headers["Pragma"] = "no-cache";
718 }
719 
720 /// Sets headers to request clients to cache a response indefinitely.
721 void cacheForever(ref Headers headers)
722 {
723 	headers["Expires"] = httpTime(Clock.currTime().add!"years"(1));
724 	headers["Cache-Control"] = "public, max-age=31536000";
725 }
726 
727 /// Formats a timestamp in the format used by HTTP (RFC 2822).
728 string httpTime(SysTime time)
729 {
730 	time.timezone = UTC();
731 	return time.formatTime!(TimeFormats.HTTP)();
732 }
733 
734 import std.algorithm : sort;
735 
736 /// Parses a list in the format of "a, b, c;q=0.5, d" and returns
737 /// an array of items sorted by "q" (["a", "b", "d", "c"])
738 string[] parseItemList(string s)
739 {
740 	static struct Item
741 	{
742 		float q = 1.0;
743 		string str;
744 
745 		this(string s)
746 		{
747 			auto params = s.split(";");
748 			if (!params.length) return;
749 			str = params[0];
750 			foreach (param; params[1..$])
751 				if (param.startsWith("q="))
752 					q = to!float(param[2..$]);
753 		}
754 	}
755 
756 	return s
757 		.split(",")
758 		.amap!(a => Item(strip(a)))()
759 		.asort!`a.q > b.q`()
760 		.amap!`a.str`();
761 }
762 
763 unittest
764 {
765 	assert(parseItemList("a, b, c;q=0.5, d") == ["a", "b", "d", "c"]);
766 }
767 
768 // TODO: optimize / move to HtmlWriter
769 deprecated("Use ae.utils.xml.entities")
770 string httpEscape(string str)
771 {
772 	string result;
773 	foreach(c;str)
774 		switch(c)
775 		{
776 			case '<':
777 				result ~= "&lt;";
778 				break;
779 			case '>':
780 				result ~= "&gt;";
781 				break;
782 			case '&':
783 				result ~= "&amp;";
784 				break;
785 			case '\xDF':  // the beta-like symbol
786 				result ~= "&szlig;";
787 				break;
788 			default:
789 				result ~= [c];
790 		}
791 	return result;
792 }
793 
794 public import ae.net.ietf.url : UrlParameters, encodeUrlParameter, encodeUrlParameters, decodeUrlParameter, decodeUrlParameters;
795 
796 /// Represents a part from a multipart/* message.
797 struct MultipartPart
798 {
799 	/// The part's individual headers.
800 	Headers headers;
801 
802 	/// The part's contents.
803 	Data data;
804 }
805 
806 /// Encode a multipart body with the given parts and boundary.
807 Data encodeMultipart(MultipartPart[] parts, string boundary)
808 {
809 	TData!char data;
810 	foreach (ref part; parts)
811 	{
812 		data ~= "--" ~ boundary ~ "\r\n";
813 		foreach (name, value; part.headers)
814 			data ~= name ~ ": " ~ value ~ "\r\n";
815 		data ~= "\r\n";
816 		assert(part.data.asDataOf!char.indexOf(boundary) < 0);
817 		data ~= part.data.asDataOf!char;
818 		data ~= "\r\n";
819 	}
820 	data ~= "--" ~ boundary ~ "--\r\n";
821 	return data.asDataOf!ubyte;
822 }
823 
824 /// Decode a multipart body using the given boundary.
825 MultipartPart[] decodeMultipart(Data data, string boundary)
826 {
827 	MultipartPart[] result;
828 	data.asDataOf!char.enter((scope s) {
829 		auto term = "\r\n--" ~ boundary ~ "--\r\n";
830 		enforce(s.endsWith(term), "Bad multipart terminator");
831 		s = s[0..$-term.length];
832 		auto delim = "--" ~ boundary ~ "\r\n";
833 		enforce(s.skipOver(delim), "Bad multipart start");
834 		delim = "\r\n" ~ delim;
835 		auto parts = s.split(delim);
836 		foreach (part; parts)
837 		{
838 			auto segs = part.findSplit("\r\n\r\n");
839 			enforce(segs[1], "Can't find headers in multipart part");
840 			MultipartPart p;
841 			foreach (line; segs[0].split("\r\n"))
842 			{
843 				auto hparts = line.findSplit(":");
844 				p.headers[hparts[0].strip.idup] = hparts[2].strip.idup;
845 			}
846 			p.data = Data(segs[2].asBytes);
847 			result ~= p;
848 		}
849 	});
850 	return result;
851 }
852 
853 unittest
854 {
855 	auto parts = [
856 		MultipartPart(Headers(["Foo" : "bar"]), Data.init),
857 		MultipartPart(Headers(["Baz" : "quux", "Frob" : "xyzzy"]), Data("Content goes here\xFF".asBytes)),
858 	];
859 	auto boundary = "abcde";
860 	auto parts2 = parts.encodeMultipart(boundary).decodeMultipart(boundary);
861 	assert(parts2.length == parts.length);
862 	foreach (p; 0..parts.length)
863 	{
864 		assert(parts[p].headers == parts2[p].headers);
865 		assert(parts[p].data.unsafeContents == parts2[p].data.unsafeContents);
866 	}
867 }
868 
869 private bool asciiStartsWith(string s, string prefix) pure nothrow @nogc
870 {
871 	if (s.length < prefix.length)
872 		return false;
873 	import std.ascii;
874 	foreach (i, c; prefix)
875 		if (toLower(c) != toLower(s[i]))
876 			return false;
877 	return true;
878 }