1 /**
2  * A simple HTTP server.
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Stéphan Kochen <stephan@kochen.nl>
12  *   Vladimir Panteleev <vladimir@thecybershadow.net>
13  *   Simon Arlott
14  */
15 
16 module ae.net.http.server;
17 
18 import std.conv;
19 import std.datetime;
20 import std.exception;
21 import std.range;
22 import std.socket;
23 import std.string;
24 import std.uri;
25 
26 import ae.net.asockets;
27 import ae.net.ietf.headerparse;
28 import ae.net.ietf.headers;
29 import ae.net.ssl;
30 import ae.sys.data;
31 import ae.sys.log;
32 import ae.utils.container.listnode;
33 import ae.utils.exception;
34 import ae.utils.text;
35 import ae.utils.textout;
36 
37 public import ae.net.http.common;
38 
39 debug(HTTP) import std.stdio : stderr;
40 
41 class BaseHttpServerConnection
42 {
43 public:
44 	TimeoutAdapter timer;
45 	IConnection conn;
46 
47 	HttpRequest currentRequest;
48 	bool persistent;
49 
50 	bool connected = true;
51 	Logger log;
52 
53 	void delegate(HttpRequest request) handleRequest;
54 
55 protected:
56 	Data[] inBuffer;
57 	sizediff_t expect;
58 	size_t responseSize;
59 	bool requestProcessing; // user code is asynchronously processing current request
60 	Duration timeout = HttpServer.defaultTimeout;
61 	bool timeoutActive;
62 	string banner;
63 
64 	this(IConnection c)
65 	{
66 		debug (HTTP) debugLog("New connection from %s", remoteAddressStr);
67 
68 		timer = new TimeoutAdapter(c);
69 		timer.setIdleTimeout(timeout);
70 		c = timer;
71 
72 		this.conn = c;
73 		conn.handleReadData = &onNewRequest;
74 		conn.handleDisconnect = &onDisconnect;
75 
76 		timeoutActive = true;
77 	}
78 
79 	debug (HTTP)
80 	final void debugLog(Args...)(Args args)
81 	{
82 		stderr.writef("[%s %s] ", Clock.currTime(), cast(void*)this);
83 		stderr.writefln(args);
84 	}
85 
86 	final void onNewRequest(Data data)
87 	{
88 		try
89 		{
90 			inBuffer ~= data;
91 			debug (HTTP) debugLog("Receiving start of request (%d new bytes, %d total)", data.length, inBuffer.bytes.length);
92 
93 			string reqLine;
94 			Headers headers;
95 
96 			if (!parseHeaders(inBuffer, reqLine, headers))
97 			{
98 				debug (HTTP) debugLog("Headers not yet received. Data in buffer:\n%s---", cast(string)inBuffer.joinToHeap());
99 				return;
100 			}
101 
102 			debug (HTTP)
103 			{
104 				debugLog("Headers received:");
105 				debugLog("> %s", reqLine);
106 				foreach (name, value; headers)
107 					debugLog("> %s: %s", name, value);
108 			}
109 
110 			currentRequest = new HttpRequest;
111 			currentRequest.parseRequestLine(reqLine);
112 			currentRequest.headers = headers;
113 
114 			auto connection = toLower(currentRequest.headers.get("Connection", null));
115 			switch (currentRequest.protocolVersion)
116 			{
117 				case "1.0":
118 					persistent = connection == "keep-alive";
119 					break;
120 				default: // 1.1+
121 					persistent = connection != "close";
122 					break;
123 			}
124 			debug (HTTP) debugLog("This %s connection %s persistent", currentRequest.protocolVersion, persistent ? "IS" : "is NOT");
125 
126 			expect = 0;
127 			if ("Content-Length" in currentRequest.headers)
128 				expect = to!size_t(currentRequest.headers["Content-Length"]);
129 
130 			if (expect > 0)
131 			{
132 				if (expect > inBuffer.bytes.length)
133 					conn.handleReadData = &onContinuation;
134 				else
135 					processRequest(inBuffer.popFront(expect));
136 			}
137 			else
138 				processRequest(null);
139 		}
140 		catch (CaughtException e)
141 		{
142 			debug (HTTP) debugLog("Exception onNewRequest: %s", e);
143 			HttpResponse response;
144 			debug
145 			{
146 				response = new HttpResponse();
147 				response.status = HttpStatusCode.InternalServerError;
148 				response.statusMessage = HttpResponse.getStatusMessage(HttpStatusCode.InternalServerError);
149 				response.headers["Content-Type"] = "text/plain";
150 				response.data = [Data(e.toString())];
151 			}
152 			sendResponse(response);
153 		}
154 	}
155 
156 	void onDisconnect(string reason, DisconnectType type)
157 	{
158 		debug (HTTP) debugLog("Disconnect: %s", reason);
159 		connected = false;
160 	}
161 
162 	final void onContinuation(Data data)
163 	{
164 		debug (HTTP) debugLog("Receiving continuation of request: \n%s---", cast(string)data.contents);
165 		inBuffer ~= data;
166 
167 		if (!requestProcessing && inBuffer.bytes.length >= expect)
168 		{
169 			debug (HTTP) debugLog("%s/%s", inBuffer.bytes.length, expect);
170 			processRequest(inBuffer.popFront(expect));
171 		}
172 	}
173 
174 	final void processRequest(Data[] data)
175 	{
176 		debug (HTTP) debugLog("processRequest (%d bytes)", data.bytes.length);
177 		currentRequest.data = data;
178 		timeoutActive = false;
179 		timer.cancelIdleTimeout();
180 		if (handleRequest)
181 		{
182 			// Log unhandled exceptions, but don't mess up the stack trace
183 			//scope(failure) logRequest(currentRequest, null);
184 
185 			// sendResponse may be called immediately, or later
186 			requestProcessing = true;
187 			handleRequest(currentRequest);
188 		}
189 	}
190 
191 	final void logRequest(HttpRequest request, HttpResponse response)
192 	{
193 		debug // avoid linewrap in terminal during development
194 			enum DEBUG = true;
195 		else
196 			enum DEBUG = false;
197 
198 		if (log) log(([
199 			"", // align IP to tab
200 			request ? request.remoteHosts(remoteAddressStr)[0] : remoteAddressStr,
201 			response ? text(cast(ushort)response.status) : "-",
202 			request ? format("%9.2f ms", request.age.total!"usecs" / 1000f) : "-",
203 			request ? request.method : "-",
204 			request ? formatLocalAddress(request) ~ request.resource : "-",
205 			response ? response.headers.get("Content-Type", "-") : "-",
206 		] ~ (DEBUG ? [] : [
207 			request ? request.headers.get("Referer", "-") : "-",
208 			request ? request.headers.get("User-Agent", "-") : "-",
209 		])).join("\t"));
210 	}
211 
212 	abstract string formatLocalAddress(HttpRequest r);
213 	abstract @property string remoteAddressStr();
214 
215 	final @property bool idle()
216 	{
217 		if (requestProcessing)
218 			return false;
219 		foreach (datum; inBuffer)
220 			if (datum.length)
221 				return false;
222 		return true;
223 	}
224 
225 public:
226 	final void sendHeaders(Headers headers, HttpStatusCode status, string statusMessage = null)
227 	{
228 		assert(status, "Unset status code");
229 
230 		if (!statusMessage)
231 			statusMessage = HttpResponse.getStatusMessage(status);
232 
233 		StringBuilder respMessage;
234 		auto protocolVersion = currentRequest ? currentRequest.protocolVersion : "1.0";
235 		respMessage.put("HTTP/", protocolVersion, " ");
236 
237 		if (banner && "X-Powered-By" !in headers)
238 			headers["X-Powered-By"] = banner;
239 
240 		if ("Date" !in headers)
241 			headers["Date"] = httpTime(Clock.currTime());
242 
243 		if ("Connection" !in headers)
244 		{
245 			if (persistent && protocolVersion=="1.0")
246 				headers["Connection"] = "Keep-Alive";
247 			else
248 			if (!persistent && protocolVersion=="1.1")
249 				headers["Connection"] = "close";
250 		}
251 
252 		respMessage.put("%d %s\r\n".format(status, statusMessage));
253 		foreach (string header, string value; headers)
254 			respMessage.put(header, ": ", value, "\r\n");
255 
256 		debug (HTTP) debugLog("Response headers:\n> %s", respMessage.get().chomp().replace("\r\n", "\n> "));
257 
258 		respMessage.put("\r\n");
259 		conn.send(Data(respMessage.get()));
260 	}
261 
262 	final void sendHeaders(HttpResponse response)
263 	{
264 		sendHeaders(response.headers, response.status, response.statusMessage);
265 	}
266 
267 	final void sendResponse(HttpResponse response)
268 	{
269 		requestProcessing = false;
270 		if (!response)
271 		{
272 			debug (HTTP) debugLog("sendResponse(null) - generating dummy response");
273 			response = new HttpResponse();
274 			response.status = HttpStatusCode.InternalServerError;
275 			response.statusMessage = HttpResponse.getStatusMessage(HttpStatusCode.InternalServerError);
276 			response.data = [Data("Internal Server Error")];
277 		}
278 
279 		if (currentRequest)
280 		{
281 			response.optimizeData(currentRequest.headers);
282 			response.sliceData(currentRequest.headers);
283 		}
284 
285 		if ("Content-Length" !in response.headers)
286 			response.headers["Content-Length"] = text(response.data.bytes.length);
287 
288 		sendHeaders(response);
289 
290 		bool isHead = currentRequest ? currentRequest.method == "HEAD" : false;
291 		if (response && response.data.length && !isHead)
292 			sendData(response.data);
293 
294 		responseSize = response ? response.data.bytes.length : 0;
295 		debug (HTTP) debugLog("Sent response (%d bytes data)", responseSize);
296 
297 		closeResponse();
298 
299 		logRequest(currentRequest, response);
300 	}
301 
302 	final void sendData(Data[] data)
303 	{
304 		conn.send(data);
305 	}
306 
307 	/// Accept more requests on the same connection?
308 	bool acceptMore() { return true; }
309 
310 	final void closeResponse()
311 	{
312 		if (persistent && acceptMore)
313 		{
314 			// reset for next request
315 			debug (HTTP) debugLog("  Waiting for next request.");
316 			conn.handleReadData = &onNewRequest;
317 			if (!timeoutActive)
318 			{
319 				// Give the client time to download large requests.
320 				// Assume a minimal speed of 1kb/s.
321 				timer.setIdleTimeout(timeout + (responseSize / 1024).seconds);
322 				timeoutActive = true;
323 			}
324 			if (inBuffer.bytes.length) // a second request has been pipelined
325 			{
326 				debug (HTTP) debugLog("A second request has been pipelined: %d datums, %d bytes", inBuffer.length, inBuffer.bytes.length);
327 				onNewRequest(Data());
328 			}
329 		}
330 		else
331 		{
332 			string reason = persistent ? "Server has been shut down" : "Non-persistent connection";
333 			debug (HTTP) debugLog("  Closing connection (%s).", reason);
334 			conn.disconnect(reason);
335 		}
336 	}
337 }
338 
339 class HttpServer
340 {
341 	enum defaultTimeout = 30.seconds;
342 public:
343 	this(Duration timeout = defaultTimeout)
344 	{
345 		assert(timeout > Duration.zero);
346 		this.timeout = timeout;
347 
348 		conn = new TcpServer();
349 		conn.handleClose = &onClose;
350 		conn.handleAccept = &onAccept;
351 	}
352 
353 	ushort listen(ushort port, string addr = null)
354 	{
355 		port = conn.listen(port, addr);
356 		if (log)
357 			foreach (address; conn.localAddresses)
358 				log("Listening on " ~ formatAddress(protocol, address) ~ " [" ~ to!string(address.addressFamily) ~ "]");
359 		return port;
360 	}
361 
362 	void listen(AddressInfo[] addresses)
363 	{
364 		conn.listen(addresses);
365 		if (log)
366 			foreach (address; conn.localAddresses)
367 				log("Listening on " ~ formatAddress(protocol, address) ~ " [" ~ to!string(address.addressFamily) ~ "]");
368 	}
369 
370 	void close()
371 	{
372 		debug(HTTP) stderr.writeln("Shutting down");
373 		if (log) log("Shutting down.");
374 		conn.close();
375 
376 		debug(HTTP) stderr.writefln("There still are %d active connections", connections.iterator.walkLength);
377 
378 		// Close idle connections
379 		foreach (connection; connections.iterator.array)
380 			if (connection.idle && connection.conn.state == ConnectionState.connected)
381 				connection.conn.disconnect("HTTP server shutting down");
382 	}
383 
384 	Logger log;
385 
386 	/// Single-ended doubly-linked list of active connections
387 	SEDListContainer!HttpServerConnection connections;
388 
389 	/// Callback for when the socket was closed.
390 	void delegate() handleClose;
391 	/// Callback for an incoming request.
392 	void delegate(HttpRequest request, HttpServerConnection conn) handleRequest;
393 
394 	string banner = "ae.net.http.server (+https://github.com/CyberShadow/ae)";
395 
396 protected:
397 	TcpServer conn;
398 	Duration timeout;
399 
400 	void onClose()
401 	{
402 		if (handleClose)
403 			handleClose();
404 	}
405 
406 	IConnection createConnection(TcpConnection tcp)
407 	{
408 		return tcp;
409 	}
410 
411 	@property string protocol() { return "http"; }
412 
413 	void onAccept(TcpConnection incoming)
414 	{
415 		try
416 			new HttpServerConnection(this, incoming, createConnection(incoming), protocol);
417 		catch (Exception e)
418 		{
419 			if (log)
420 				log("Error accepting connection: " ~ e.msg);
421 			if (incoming.state == ConnectionState.connected)
422 				incoming.disconnect();
423 		}
424 	}
425 }
426 
427 /// HTTPS server. Set SSL parameters on ctx after instantiation.
428 /// Example:
429 /// ---
430 ///	auto s = new HttpsServer();
431 ///	s.ctx.enableDH(4096);
432 ///	s.ctx.enableECDH();
433 ///	s.ctx.setCertificate("server.crt");
434 ///	s.ctx.setPrivateKey("server.key");
435 /// ---
436 class HttpsServer : HttpServer
437 {
438 	SSLContext ctx;
439 
440 	this()
441 	{
442 		ctx = ssl.createContext(SSLContext.Kind.server);
443 	}
444 
445 protected:
446 	override @property string protocol() { return "https"; }
447 
448 	override IConnection createConnection(TcpConnection tcp)
449 	{
450 		return ssl.createAdapter(ctx, tcp);
451 	}
452 }
453 
454 final class HttpServerConnection : BaseHttpServerConnection
455 {
456 	TcpConnection tcp;
457 	HttpServer server;
458 	Address localAddress, remoteAddress;
459 
460 	mixin DListLink;
461 
462 protected:
463 	string protocol;
464 
465 	this(HttpServer server, TcpConnection tcp, IConnection c, string protocol = "http")
466 	{
467 		this.server = server;
468 		this.tcp = tcp;
469 		this.log = server.log;
470 		this.protocol = protocol;
471 		this.banner = server.banner;
472 		this.timeout = server.timeout;
473 		this.handleRequest = (HttpRequest r) => server.handleRequest(r, this);
474 		this.localAddress = tcp.localAddress;
475 		this.remoteAddress = tcp.remoteAddress;
476 
477 		super(c);
478 
479 		server.connections.pushFront(this);
480 	}
481 
482 	override void onDisconnect(string reason, DisconnectType type)
483 	{
484 		super.onDisconnect(reason, type);
485 		server.connections.remove(this);
486 	}
487 
488 	override bool acceptMore() { return server.conn.isListening; }
489 	override string formatLocalAddress(HttpRequest r) { return formatAddress(protocol, localAddress, r.host, r.port); }
490 	override @property string remoteAddressStr() { return remoteAddress.toAddrString(); }
491 }
492 
493 version (Posix)
494 final class FileHttpServerConnection : BaseHttpServerConnection
495 {
496 	this(File input = stdin, File output = stdout, string protocol = "stdin")
497 	{
498 		this.protocol = protocol;
499 
500 		auto c = new Duplex(
501 			new FileConnection(input.fileno),
502 			new FileConnection(output.fileno),
503 		);
504 
505 		super(c);
506 	}
507 
508 protected:
509 	import std.stdio : File, stdin, stdout;
510 
511 	string protocol;
512 
513 	override string formatLocalAddress(HttpRequest r) { return protocol ~ "://"; }
514 	override @property string remoteAddressStr() { return "-"; }
515 }
516 
517 string formatAddress(string protocol, Address address, string vhost = null, ushort logPort = 0)
518 {
519 	string addr = address.toAddrString();
520 	string port =
521 		address.addressFamily == AddressFamily.UNIX ? null :
522 		logPort ? text(logPort) :
523 		address.toPortString();
524 	return protocol ~ "://" ~
525 		(vhost ? vhost : addr == "0.0.0.0" || addr == "::" ? "*" : addr.contains(":") ? "[" ~ addr ~ "]" : addr) ~
526 		(port is null || port == "80" ? "" : ":" ~ port);
527 }
528 
529 version (unittest) import ae.net.http.client;
530 version (unittest) import ae.net.http.responseex;
531 unittest
532 {
533 	int[] replies;
534 	int closeAfter;
535 
536 	// Sum "a" from GET and "b" from POST
537 	auto s = new HttpServer;
538 	s.handleRequest = (HttpRequest request, HttpServerConnection conn) {
539 		auto get  = request.urlParameters;
540 		auto post = request.decodePostData();
541 		auto response = new HttpResponseEx;
542 		auto result = to!int(get["a"]) + to!int(post["b"]);
543 		replies ~= result;
544 		conn.sendResponse(response.serveJson(result));
545 		if (--closeAfter == 0)
546 			s.close();
547 	};
548 
549 	// Test server, client, parameter encoding
550 	replies = null;
551 	closeAfter = 1;
552 	auto port = s.listen(0, "127.0.0.1");
553 	httpPost("http://127.0.0.1:" ~ to!string(port) ~ "/?" ~ encodeUrlParameters(["a":"2"]), UrlParameters(["b":"3"]), (string s) { assert(s=="5"); }, null);
554 	socketManager.loop();
555 
556 	// Test pipelining, protocol errors
557 	replies = null;
558 	closeAfter = 2;
559 	port = s.listen(0, "127.0.0.1");
560 	TcpConnection c = new TcpConnection;
561 	c.handleConnect = {
562 		c.send(Data(
563 "GET /?a=123456 HTTP/1.1
564 Content-length: 8
565 Content-type: application/x-www-form-urlencoded
566 
567 b=654321" ~
568 "GET /derp HTTP/1.1
569 Content-length: potato
570 
571 " ~
572 "GET /?a=1234567 HTTP/1.1
573 Content-length: 9
574 Content-type: application/x-www-form-urlencoded
575 
576 b=7654321"));
577 		c.disconnect();
578 	};
579 	c.connect("127.0.0.1", port);
580 
581 	socketManager.loop();
582 
583 	assert(replies == [777777, 8888888]);
584 
585 	// Test bad headers
586 	s.handleRequest = (HttpRequest request, HttpServerConnection conn) {
587 		auto response = new HttpResponseEx;
588 		conn.sendResponse(response.serveText("OK"));
589 		if (--closeAfter == 0)
590 			s.close();
591 	};
592 	closeAfter = 1;
593 
594 	port = s.listen(0, "127.0.0.1");
595 	c = new TcpConnection;
596 	c.handleConnect = {
597 		c.send(Data("\n\n\n\n\n"));
598 		c.disconnect();
599 
600 		// Now send a valid request to end the loop
601 		c = new TcpConnection;
602 		c.handleConnect = {
603 			c.send(Data("GET / HTTP/1.0\n\n"));
604 			c.disconnect();
605 		};
606 		c.connect("127.0.0.1", port);
607 	};
608 	c.connect("127.0.0.1", port);
609 
610 	socketManager.loop();
611 
612 /+
613 	void testFile(string fn)
614 	{
615 		std.file.write(fn, "42");
616 		s.handleRequest = (HttpRequest request, HttpServerConnection conn) {
617 			auto response = new HttpResponseEx;
618 			conn.sendResponse(response.serveFile(request.resource[1..$], ""));
619 			if (--closeAfter == 0)
620 				s.close();
621 		};
622 		port = s.listen(0, "127.0.0.1");
623 		closeAfter = 1;
624 		httpGet("http://127.0.0.1:" ~ to!string(port) ~ "/" ~ fn, (string s) { assert(s=="42"); }, null);
625 		socketManager.loop();
626 		std.file.remove(fn);
627 	}
628 
629 	testFile("http-test.bin");
630 	testFile("http-test.txt");
631 +/
632 }