1 /**
2  * Synchronous helper to access the GitHub API
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Vladimir Panteleev <ae@cy.md>
12  */
13 
14 module ae.net.github.rest;
15 
16 package(ae):
17 
18 import std.algorithm.searching;
19 import std.conv;
20 import std.string;
21 import std.utf;
22 
23 import ae.net.http.common;
24 import ae.net.ietf.headers;
25 import ae.net.ietf.url;
26 import ae.sys.data;
27 import ae.sys.dataset : DataVec, joinToGC;
28 import ae.sys.log;
29 import ae.sys.net;
30 import ae.utils.array : as, fromBytes;
31 import ae.utils.json;
32 import ae.utils.meta;
33 
34 struct GitHub
35 {
36 	string token;
37 
38 	void delegate(string) log;
39 
40 	bool offline; /// Use cached objects without re-validating them
41 
42 	struct CacheEntry
43 	{
44 		string[string] headers;
45 		string data;
46 	}
47 	interface ICache
48 	{
49 		string get(string key);
50 		void put(string key, string value);
51 	}
52 	static class NoCache : ICache
53 	{
54 		string get(string key) { return null; }
55 		void put(string key, string value) {}
56 	}
57 	ICache cache = new NoCache;
58 
59 	struct Result
60 	{
61 		string[string] headers;
62 		string data;
63 	}
64 
65 	Result query(string url)
66 	{
67 		auto request = new HttpRequest;
68 		request.resource = url;
69 		if (token)
70 			request.headers["Authorization"] = "token " ~ token;
71 
72 		auto cacheKey = url;
73 
74 		CacheEntry cacheEntry;
75 		auto cacheEntryStr = cache.get(cacheKey);
76 		if (cacheEntryStr)
77 		{
78 			cacheEntry = cacheEntryStr.jsonParse!CacheEntry();
79 
80 			if (offline)
81 			{
82 				if (log) log(" > Cache hit (offline mode)");
83 				return Result(cacheEntry.headers, cacheEntry.data);
84 			}
85 
86 			auto cacheHeaders = Headers(cacheEntry.headers);
87 
88 			if (auto p = "ETag" in cacheHeaders)
89 				request.headers["If-None-Match"] = *p;
90 			if (auto p = "Last-Modified" in cacheHeaders)
91 				request.headers["If-Modified-Since"] = *p;
92 		}
93 
94 		if (log) log("Getting URL " ~ url);
95 
96 		auto response = net.httpRequest(request);
97 		while (true) // Redirect loop
98 		{
99 			if (response.status == HttpStatusCode.NotModified)
100 			{
101 				if (log) log(" > Cache hit");
102 				return Result(cacheEntry.headers, cacheEntry.data);
103 			}
104 			else
105 			if (response.status == HttpStatusCode.OK)
106 			{
107 				if (log) log(" > Cache miss; ratelimit: %s/%s".format(
108 					response.headers.get("X-Ratelimit-Remaining", "?"),
109 					response.headers.get("X-Ratelimit-Limit", "?"),
110 				));
111 				scope(failure) if (log) log(response.headers.text);
112 				auto headers = response.headers.to!(string[string]);
113 				auto data = response.getContent().toGC().as!string;
114 				cacheEntry.headers = headers;
115 				cacheEntry.data = data;
116 				cache.put(cacheKey, toJson(cacheEntry));
117 				return Result(headers, data);
118 			}
119 			else
120 			if (response.status >= 300 && response.status < 400 && "Location" in response.headers)
121 			{
122 				auto location = response.headers["Location"];
123 				if (log) log(" > Redirect: " ~ location);
124 				request.resource = applyRelativeURL(request.url, location);
125 				if (response.status == HttpStatusCode.SeeOther)
126 				{
127 					request.method = "GET";
128 					request.data = null;
129 				}
130 				response = net.httpRequest(request);
131 			}
132 			else
133 				throw new Exception("Error with URL " ~ url ~ ": " ~ text(response.status));
134 		}
135 	}
136 
137 	static import std.json;
138 
139 	std.json.JSONValue[] pagedQuery(string url)
140 	{
141 		import std.json : JSONValue, parseJSON;
142 
143 		JSONValue[] result;
144 		while (true)
145 		{
146 			auto page = query(url);
147 			result ~= page.data.parseJSON().array;
148 			auto links = page.headers.get("Link", null).I!parseLinks();
149 			if ("next" in links)
150 				url = links["next"];
151 			else
152 				break;
153 		}
154 		return result;
155 	}
156 
157 	/// Parse a "Link" header.
158 	private static string[string] parseLinks(string s)
159 	{
160 		string[string] result;
161 		auto items = s.split(", "); // Hacky but should never occur inside an URL or "rel" value
162 		foreach (item; items)
163 		{
164 			auto parts = item.split("; "); // ditto
165 			string url; string[string] args;
166 			foreach (part; parts)
167 			{
168 				if (part.startsWith("<") && part.endsWith(">"))
169 					url = part[1..$-1];
170 				else
171 				{
172 					auto ps = part.findSplit("=");
173 					auto key = ps[0];
174 					auto value = ps[2];
175 					if (value.startsWith('"') && value.endsWith('"'))
176 						value = value[1..$-1];
177 					args[key] = value;
178 				}
179 			}
180 			result[args.get("rel", null)] = url;
181 		}
182 		return result;
183 	}
184 
185 	unittest
186 	{
187 		auto header = `<https://api.github.com/repositories/1257070/pulls?per_page=100&page=2>; rel="next", ` ~
188 			`<https://api.github.com/repositories/1257070/pulls?per_page=100&page=3>; rel="last"`;
189 		assert(parseLinks(header) == [
190 			"next" : "https://api.github.com/repositories/1257070/pulls?per_page=100&page=2",
191 			"last" : "https://api.github.com/repositories/1257070/pulls?per_page=100&page=3",
192 		]);
193 	}
194 
195 	string post(string url, Data jsonData)
196 	{
197 		auto request = new HttpRequest;
198 		request.resource = url;
199 		request.method = "POST";
200 		if (token)
201 			request.headers["Authorization"] = "token " ~ token;
202 		request.headers["Content-Type"] = "application/json";
203 		request.data = DataVec(jsonData);
204 
205 		auto response = net.httpRequest(request);
206 		string result = response.data.joinToGC.as!string;
207 		validate(result);
208 		return result;
209 	}
210 }