1 /**
2  * Synchronous helper to access the GitHub API
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Vladimir Panteleev <ae@cy.md>
12  */
13 
14 module ae.net.github.rest;
15 
16 package(ae):
17 
18 import std.algorithm.searching;
19 import std.conv;
20 import std.string;
21 import std.utf;
22 
23 import ae.net.http.common;
24 import ae.net.ietf.headers;
25 import ae.net.ietf.url;
26 import ae.sys.data;
27 import ae.sys.log;
28 import ae.sys.net;
29 import ae.utils.json;
30 import ae.utils.meta;
31 
32 struct GitHub
33 {
34 	string token;
35 
36 	void delegate(string) log;
37 
38 	struct CacheEntry
39 	{
40 		string[string] headers;
41 		string data;
42 	}
43 	interface ICache
44 	{
45 		string get(string key);
46 		void put(string key, string value);
47 	}
48 	static class NoCache : ICache
49 	{
50 		string get(string key) { return null; }
51 		void put(string key, string value) {}
52 	}
53 	ICache cache = new NoCache;
54 
55 	struct Result
56 	{
57 		string[string] headers;
58 		string data;
59 	}
60 
61 	Result query(string url)
62 	{
63 		auto request = new HttpRequest;
64 		request.resource = url;
65 		if (token)
66 			request.headers["Authorization"] = "token " ~ token;
67 
68 		auto cacheKey = url;
69 
70 		CacheEntry cacheEntry;
71 		auto cacheEntryStr = cache.get(cacheKey);
72 		if (cacheEntryStr)
73 		{
74 			cacheEntry = cacheEntryStr.jsonParse!CacheEntry();
75 			auto cacheHeaders = Headers(cacheEntry.headers);
76 
77 			if (auto p = "ETag" in cacheHeaders)
78 				request.headers["If-None-Match"] = *p;
79 			if (auto p = "Last-Modified" in cacheHeaders)
80 				request.headers["If-Modified-Since"] = *p;
81 		}
82 
83 		if (log) log("Getting URL " ~ url);
84 
85 		auto response = net.httpRequest(request);
86 		while (true) // Redirect loop
87 		{
88 			if (response.status == HttpStatusCode.NotModified)
89 			{
90 				if (log) log(" > Cache hit");
91 				return Result(cacheEntry.headers, cacheEntry.data);
92 			}
93 			else
94 			if (response.status == HttpStatusCode.OK)
95 			{
96 				if (log) log(" > Cache miss; ratelimit: %s/%s".format(
97 					response.headers.get("X-Ratelimit-Remaining", "?"),
98 					response.headers.get("X-Ratelimit-Limit", "?"),
99 				));
100 				scope(failure) if (log) log(response.headers.text);
101 				auto headers = response.headers.to!(string[string]);
102 				auto data = (cast(char[])response.getContent().contents).idup;
103 				cacheEntry.headers = headers;
104 				cacheEntry.data = data;
105 				cache.put(cacheKey, toJson(cacheEntry));
106 				return Result(headers, data);
107 			}
108 			else
109 			if (response.status >= 300 && response.status < 400 && "Location" in response.headers)
110 			{
111 				auto location = response.headers["Location"];
112 				if (log) log(" > Redirect: " ~ location);
113 				request.resource = applyRelativeURL(request.url, location);
114 				if (response.status == HttpStatusCode.SeeOther)
115 				{
116 					request.method = "GET";
117 					request.data = null;
118 				}
119 				response = net.httpRequest(request);
120 			}
121 			else
122 				throw new Exception("Error with URL " ~ url ~ ": " ~ text(response.status));
123 		}
124 	}
125 
126 	static import std.json;
127 
128 	std.json.JSONValue[] pagedQuery(string url)
129 	{
130 		import std.json : JSONValue, parseJSON;
131 
132 		JSONValue[] result;
133 		while (true)
134 		{
135 			auto page = query(url);
136 			result ~= page.data.parseJSON().array;
137 			auto links = page.headers.get("Link", null).I!parseLinks();
138 			if ("next" in links)
139 				url = links["next"];
140 			else
141 				break;
142 		}
143 		return result;
144 	}
145 
146 	/// Parse a "Link" header.
147 	private static string[string] parseLinks(string s)
148 	{
149 		string[string] result;
150 		auto items = s.split(", "); // Hacky but should never occur inside an URL or "rel" value
151 		foreach (item; items)
152 		{
153 			auto parts = item.split("; "); // ditto
154 			string url; string[string] args;
155 			foreach (part; parts)
156 			{
157 				if (part.startsWith("<") && part.endsWith(">"))
158 					url = part[1..$-1];
159 				else
160 				{
161 					auto ps = part.findSplit("=");
162 					auto key = ps[0];
163 					auto value = ps[2];
164 					if (value.startsWith('"') && value.endsWith('"'))
165 						value = value[1..$-1];
166 					args[key] = value;
167 				}
168 			}
169 			result[args.get("rel", null)] = url;
170 		}
171 		return result;
172 	}
173 
174 	unittest
175 	{
176 		auto header = `<https://api.github.com/repositories/1257070/pulls?per_page=100&page=2>; rel="next", ` ~
177 			`<https://api.github.com/repositories/1257070/pulls?per_page=100&page=3>; rel="last"`;
178 		assert(parseLinks(header) == [
179 			"next" : "https://api.github.com/repositories/1257070/pulls?per_page=100&page=2",
180 			"last" : "https://api.github.com/repositories/1257070/pulls?per_page=100&page=3",
181 		]);
182 	}
183 
184 	string post(string url, Data jsonData)
185 	{
186 		auto request = new HttpRequest;
187 		request.resource = url;
188 		request.method = "POST";
189 		if (token)
190 			request.headers["Authorization"] = "token " ~ token;
191 		request.headers["Content-Type"] = "application/json";
192 		request.data = DataVec(jsonData);
193 
194 		auto response = net.httpRequest(request);
195 		string result = cast(string)response.data.joinToHeap;
196 		validate(result);
197 		return result;
198 	}
199 }