1 /**
2  * A simple HTTP client.
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Stéphan Kochen <stephan@kochen.nl>
12  *   Vladimir Panteleev <vladimir@thecybershadow.net>
13  *   Vincent Povirk <madewokherd@gmail.com>
14  *   Simon Arlott
15  */
16 
17 module ae.net.http.client;
18 
19 import std.string;
20 import std.conv;
21 import std.datetime;
22 import std.uri;
23 import std.utf;
24 
25 import ae.net.asockets;
26 import ae.net.ietf.headers;
27 import ae.net.ietf.headerparse;
28 import ae.net.ietf.url;
29 import ae.net.ssl;
30 import ae.utils.array : toArray;
31 import ae.utils.exception : CaughtException;
32 import ae.sys.data;
33 debug(HTTP) import std.stdio : stderr;
34 
35 public import ae.net.http.common;
36 
37 class HttpClient
38 {
39 private:
40 	Connector connector;  // Bottom-level transport factory.
41 	TimeoutAdapter timer; // Timeout adapter.
42 	IConnection conn;     // Top-level abstract connection. Reused for new connections.
43 
44 	Data[] inBuffer;
45 
46 protected:
47 	HttpRequest currentRequest;
48 
49 	HttpResponse currentResponse;
50 	size_t expect;
51 
52 	void onConnect()
53 	{
54 		sendRequest(currentRequest);
55 	}
56 
57 	void sendRequest(HttpRequest request)
58 	{
59 		if ("User-Agent" !in request.headers && agent)
60 			request.headers["User-Agent"] = agent;
61 		if (!compat)
62 		{
63 			if ("Accept-Encoding" !in request.headers)
64 				request.headers["Accept-Encoding"] = "gzip, deflate, *;q=0";
65 			if (request.data)
66 				request.headers["Content-Length"] = to!string(request.data.bytes.length);
67 		}
68 		else
69 		{
70 			if ("Pragma" !in request.headers)
71 				request.headers["Pragma"] = "No-Cache";
72 		}
73 		if ("Connection" !in request.headers)
74 			request.headers["Connection"] = keepAlive ? "keep-alive" : "close";
75 
76 		sendRawRequest(request);
77 	}
78 
79 	void sendRawRequest(HttpRequest request)
80 	{
81 		string reqMessage = request.method ~ " ";
82 		if (request.proxy !is null) {
83 			reqMessage ~= "http://" ~ request.host;
84 			if (compat || request.port != 80)
85 				reqMessage ~= format(":%d", request.port);
86 		}
87 		reqMessage ~= request.resource ~ " HTTP/1.0\r\n";
88 
89 		foreach (string header, string value; request.headers)
90 			if (value !is null)
91 				reqMessage ~= header ~ ": " ~ value ~ "\r\n";
92 
93 		reqMessage ~= "\r\n";
94 		debug(HTTP)
95 		{
96 			stderr.writefln("Sending request:");
97 			foreach (line; reqMessage.split("\r\n"))
98 				stderr.writeln("> ", line);
99 			if (request.data)
100 				stderr.writefln("} (%d bytes data follow)", request.data.bytes.length);
101 		}
102 
103 		conn.send(Data(reqMessage));
104 		conn.send(request.data);
105 	}
106 
107 	void onNewResponse(Data data)
108 	{
109 		try
110 		{
111 			inBuffer ~= data;
112 			if (timer)
113 				timer.markNonIdle();
114 
115 			string statusLine;
116 			Headers headers;
117 
118 			debug(HTTP) auto oldData = inBuffer.dup;
119 
120 			if (!parseHeaders(inBuffer, statusLine, headers))
121 				return;
122 
123 			debug(HTTP)
124 			{
125 				stderr.writefln("Got response:");
126 				auto reqMessage = cast(string)oldData.bytes[0..oldData.bytes.length-inBuffer.bytes.length].joinToHeap();
127 				foreach (line; reqMessage.split("\r\n"))
128 					stderr.writeln("< ", line);
129 			}
130 
131 			currentResponse = new HttpResponse;
132 			currentResponse.parseStatusLine(statusLine);
133 			currentResponse.headers = headers;
134 
135 			onHeadersReceived();
136 		}
137 		catch (CaughtException e)
138 		{
139 			if (conn.state == ConnectionState.connected)
140 				conn.disconnect(e.msg.length ? e.msg : e.classinfo.name, DisconnectType.error);
141 			else
142 				throw new Exception("Unhandled exception after connection was closed: " ~ e.msg, e);
143 		}
144 	}
145 
146 	void onHeadersReceived()
147 	{
148 		expect = size_t.max;
149 		if ("Content-Length" in currentResponse.headers)
150 			expect = to!size_t(strip(currentResponse.headers["Content-Length"]));
151 
152 		if (inBuffer.bytes.length < expect)
153 		{
154 			onData(inBuffer);
155 			conn.handleReadData = &onContinuation;
156 		}
157 		else
158 		{
159 			onData(inBuffer.bytes[0 .. expect]); // TODO: pipelining
160 			onDone();
161 		}
162 
163 		inBuffer.destroy();
164 	}
165 
166 	void onData(Data[] data)
167 	{
168 		currentResponse.data ~= data;
169 	}
170 
171 	void onContinuation(Data data)
172 	{
173 		onData(data.toArray);
174 		if (timer)
175 			timer.markNonIdle();
176 
177 		auto received = currentResponse.data.bytes.length;
178 		if (expect!=size_t.max && received >= expect)
179 		{
180 			inBuffer = currentResponse.data.bytes[expect..received];
181 			currentResponse.data = currentResponse.data.bytes[0..expect];
182 			onDone();
183 		}
184 	}
185 
186 	void onDone()
187 	{
188 		if (keepAlive)
189 			processResponse();
190 		else
191 			conn.disconnect("All data read");
192 	}
193 
194 	void processResponse(string reason = "All data read")
195 	{
196 		auto response = currentResponse;
197 
198 		currentRequest = null;
199 		currentResponse = null;
200 		expect = -1;
201 		conn.handleReadData = null;
202 
203 		if (handleResponse)
204 			handleResponse(response, reason);
205 	}
206 
207 	void onDisconnect(string reason, DisconnectType type)
208 	{
209 		if (type == DisconnectType.error)
210 			currentResponse = null;
211 
212 		if (currentRequest)
213 			processResponse(reason);
214 	}
215 
216 	IConnection adaptConnection(IConnection conn)
217 	{
218 		return conn;
219 	}
220 
221 public:
222 	string agent = "ae.net.http.client (+https://github.com/CyberShadow/ae)";
223 	bool compat = false;
224 	bool keepAlive = false;
225 	string[] cookies;
226 
227 public:
228 	this(Duration timeout = 30.seconds, Connector connector = new TcpConnector)
229 	{
230 		assert(timeout >= Duration.zero);
231 
232 		this.connector = connector;
233 		IConnection c = connector.getConnection();
234 
235 		c = adaptConnection(c);
236 
237 		if (timeout > Duration.zero)
238 		{
239 			timer = new TimeoutAdapter(c);
240 			timer.setIdleTimeout(timeout);
241 			c = timer;
242 		}
243 
244 		conn = c;
245 		conn.handleConnect = &onConnect;
246 		conn.handleDisconnect = &onDisconnect;
247 	}
248 
249 	void request(HttpRequest request)
250 	{
251 		//debug writefln("New HTTP request: %s", request.url);
252 		currentRequest = request;
253 		currentResponse = null;
254 		conn.handleReadData = &onNewResponse;
255 		expect = 0;
256 
257 		if (conn.state != ConnectionState.disconnected)
258 		{
259 			assert(conn.state == ConnectionState.connected, "Attempting a HTTP request on a %s connection".format(conn.state));
260 			assert(keepAlive, "Attempting a second HTTP request on a connected non-keepalive connection");
261 			sendRequest(request);
262 		}
263 		else
264 		{
265 			if (request.proxy !is null)
266 				connector.connect(request.proxyHost, request.proxyPort);
267 			else
268 				connector.connect(request.host, request.port);
269 		}
270 	}
271 
272 	bool connected()
273 	{
274 		if (currentRequest !is null)
275 			return true;
276 		if (keepAlive && conn.state == ConnectionState.connected)
277 			return true;
278 		return false;
279 	}
280 
281 	void disconnect(string reason = IConnection.defaultDisconnectReason)
282 	{
283 		conn.disconnect(reason);
284 	}
285 
286 public:
287 	// Provide the following callbacks
288 	void delegate(HttpResponse response, string disconnectReason) handleResponse;
289 }
290 
291 class HttpsClient : HttpClient
292 {
293 	SSLContext ctx;
294 	SSLAdapter adapter;
295 
296 	this(Duration timeout = 30.seconds)
297 	{
298 		ctx = ssl.createContext(SSLContext.Kind.client);
299 		super(timeout);
300 	}
301 
302 	override IConnection adaptConnection(IConnection conn)
303 	{
304 		adapter = ssl.createAdapter(ctx, conn);
305 		return adapter;
306 	}
307 
308 	override void request(HttpRequest request)
309 	{
310 		super.request(request);
311 		if (conn.state == ConnectionState.connecting)
312 			adapter.setHostName(request.host);
313 	}
314 }
315 
316 // Experimental for now
317 class Connector
318 {
319 	abstract IConnection getConnection();
320 	abstract void connect(string host, ushort port);
321 }
322 
323 // ditto
324 class TcpConnector : Connector
325 {
326 	protected TcpConnection conn;
327 
328 	this()
329 	{
330 		conn = new TcpConnection();
331 	}
332 
333 	override IConnection getConnection()
334 	{
335 		return conn;
336 	}
337 
338 	override void connect(string host, ushort port)
339 	{
340 		conn.connect(host, port);
341 	}
342 }
343 
344 // ditto
345 version(Posix)
346 class UnixConnector : TcpConnector
347 {
348 	string path;
349 
350 	this(string path)
351 	{
352 		this.path = path;
353 	}
354 
355 	override void connect(string host, ushort port)
356 	{
357 		import std.socket;
358 		auto addr = new UnixAddress(path);
359 		conn.connect([AddressInfo(AddressFamily.UNIX, SocketType.STREAM, cast(ProtocolType)0, addr, path)]);
360 	}
361 }
362 
363 /// Asynchronous HTTP request
364 void httpRequest(HttpRequest request, void delegate(HttpResponse response, string disconnectReason) responseHandler)
365 {
366 	HttpClient client;
367 	if (request.protocol == "https")
368 		client = new HttpsClient;
369 	else
370 		client = new HttpClient;
371 
372 	client.handleResponse = responseHandler;
373 	client.request(request);
374 }
375 
376 /// ditto
377 void httpRequest(HttpRequest request, void delegate(Data) resultHandler, void delegate(string) errorHandler, int redirectCount = 0)
378 {
379 	void responseHandler(HttpResponse response, string disconnectReason)
380 	{
381 		if (!response)
382 			if (errorHandler)
383 				errorHandler(disconnectReason);
384 			else
385 				throw new Exception(disconnectReason);
386 		else
387 		if (response.status >= 300 && response.status < 400 && "Location" in response.headers)
388 		{
389 			if (redirectCount == 15)
390 				throw new Exception("HTTP redirect loop: " ~ request.url);
391 			request.resource = applyRelativeURL(request.url, response.headers["Location"]);
392 			if (response.status == HttpStatusCode.SeeOther)
393 			{
394 				request.method = "GET";
395 				request.data = null;
396 			}
397 			httpRequest(request, resultHandler, errorHandler, redirectCount+1);
398 		}
399 		else
400 			if (errorHandler)
401 				try
402 					resultHandler(response.getContent());
403 				catch (Exception e)
404 					errorHandler(e.msg);
405 			else
406 				resultHandler(response.getContent());
407 	}
408 
409 	httpRequest(request, &responseHandler);
410 }
411 
412 /// ditto
413 void httpGet(string url, void delegate(HttpResponse response, string disconnectReason) responseHandler)
414 {
415 	httpRequest(new HttpRequest(url), responseHandler);
416 }
417 
418 /// ditto
419 void httpGet(string url, void delegate(Data) resultHandler, void delegate(string) errorHandler)
420 {
421 	httpRequest(new HttpRequest(url), resultHandler, errorHandler);
422 }
423 
424 /// ditto
425 void httpGet(string url, void delegate(string) resultHandler, void delegate(string) errorHandler)
426 {
427 	httpGet(url,
428 		(Data data)
429 		{
430 			auto result = (cast(char[])data.contents).idup;
431 			std.utf.validate(result);
432 			resultHandler(result);
433 		},
434 		errorHandler);
435 }
436 
437 /// ditto
438 void httpPost(string url, Data[] postData, string contentType, void delegate(Data) resultHandler, void delegate(string) errorHandler)
439 {
440 	auto request = new HttpRequest;
441 	request.resource = url;
442 	request.method = "POST";
443 	if (contentType)
444 		request.headers["Content-Type"] = contentType;
445 	request.data = postData;
446 	httpRequest(request, resultHandler, errorHandler);
447 }
448 
449 /// ditto
450 void httpPost(string url, Data[] postData, string contentType, void delegate(string) resultHandler, void delegate(string) errorHandler)
451 {
452 	httpPost(url, postData, contentType,
453 		(Data data)
454 		{
455 			auto result = (cast(char[])data.contents).idup;
456 			std.utf.validate(result);
457 			resultHandler(result);
458 		},
459 		errorHandler);
460 }
461 
462 /// ditto
463 void httpPost(string url, UrlParameters vars, void delegate(string) resultHandler, void delegate(string) errorHandler)
464 {
465 	return httpPost(url, [Data(encodeUrlParameters(vars))], "application/x-www-form-urlencoded", resultHandler, errorHandler);
466 }
467 
468 // http://d.puremagic.com/issues/show_bug.cgi?id=7016
469 version (unittest)
470 {
471 	static import ae.net.http.server;
472 	static import ae.net.http.responseex;
473 }
474 
475 unittest
476 {
477 	import ae.net.http.server;
478 	import ae.net.http.responseex;
479 
480 	void test(bool keepAlive)
481 	{
482 		auto s = new HttpServer;
483 		s.handleRequest = (HttpRequest request, HttpServerConnection conn) {
484 			auto response = new HttpResponseEx;
485 			conn.sendResponse(response.serveText("Hello!"));
486 		};
487 		auto port = s.listen(0, "127.0.0.1");
488 
489 		auto c = new HttpClient;
490 		c.keepAlive = keepAlive;
491 		auto r = new HttpRequest("http://127.0.0.1:" ~ to!string(port));
492 		int count;
493 		c.handleResponse =
494 			(HttpResponse response, string disconnectReason)
495 			{
496 				assert(response, "HTTP server error");
497 				assert(cast(string)response.getContent.toHeap == "Hello!");
498 				if (++count == 5)
499 				{
500 					s.close();
501 					if (c.connected)
502 						c.disconnect();
503 				}
504 				else
505 					c.request(r);
506 			};
507 		c.request(r);
508 
509 		socketManager.loop();
510 
511 		assert(count == 5);
512 	}
513 
514 	test(false);
515 	test(true);
516 }