1 /**
2  * Synchronous helper to access the GitHub API
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Vladimir Panteleev <ae@cy.md>
12  */
13 
14 module ae.net.github.rest;
15 
16 package(ae):
17 
18 import std.algorithm.searching;
19 import std.conv;
20 import std.string;
21 import std.utf;
22 
23 import ae.net.http.common;
24 import ae.net.ietf.headers;
25 import ae.net.ietf.url;
26 import ae.sys.data;
27 import ae.sys.log;
28 import ae.sys.net;
29 import ae.utils.json;
30 import ae.utils.meta;
31 
32 struct GitHub
33 {
34 	string token;
35 
36 	void delegate(string) log;
37 
38 	bool offline; /// Use cached objects without re-validating them
39 
40 	struct CacheEntry
41 	{
42 		string[string] headers;
43 		string data;
44 	}
45 	interface ICache
46 	{
47 		string get(string key);
48 		void put(string key, string value);
49 	}
50 	static class NoCache : ICache
51 	{
52 		string get(string key) { return null; }
53 		void put(string key, string value) {}
54 	}
55 	ICache cache = new NoCache;
56 
57 	struct Result
58 	{
59 		string[string] headers;
60 		string data;
61 	}
62 
63 	Result query(string url)
64 	{
65 		auto request = new HttpRequest;
66 		request.resource = url;
67 		if (token)
68 			request.headers["Authorization"] = "token " ~ token;
69 
70 		auto cacheKey = url;
71 
72 		CacheEntry cacheEntry;
73 		auto cacheEntryStr = cache.get(cacheKey);
74 		if (cacheEntryStr)
75 		{
76 			cacheEntry = cacheEntryStr.jsonParse!CacheEntry();
77 
78 			if (offline)
79 			{
80 				if (log) log(" > Cache hit (offline mode)");
81 				return Result(cacheEntry.headers, cacheEntry.data);
82 			}
83 
84 			auto cacheHeaders = Headers(cacheEntry.headers);
85 
86 			if (auto p = "ETag" in cacheHeaders)
87 				request.headers["If-None-Match"] = *p;
88 			if (auto p = "Last-Modified" in cacheHeaders)
89 				request.headers["If-Modified-Since"] = *p;
90 		}
91 
92 		if (log) log("Getting URL " ~ url);
93 
94 		auto response = net.httpRequest(request);
95 		while (true) // Redirect loop
96 		{
97 			if (response.status == HttpStatusCode.NotModified)
98 			{
99 				if (log) log(" > Cache hit");
100 				return Result(cacheEntry.headers, cacheEntry.data);
101 			}
102 			else
103 			if (response.status == HttpStatusCode.OK)
104 			{
105 				if (log) log(" > Cache miss; ratelimit: %s/%s".format(
106 					response.headers.get("X-Ratelimit-Remaining", "?"),
107 					response.headers.get("X-Ratelimit-Limit", "?"),
108 				));
109 				scope(failure) if (log) log(response.headers.text);
110 				auto headers = response.headers.to!(string[string]);
111 				auto data = (cast(char[])response.getContent().contents).idup;
112 				cacheEntry.headers = headers;
113 				cacheEntry.data = data;
114 				cache.put(cacheKey, toJson(cacheEntry));
115 				return Result(headers, data);
116 			}
117 			else
118 			if (response.status >= 300 && response.status < 400 && "Location" in response.headers)
119 			{
120 				auto location = response.headers["Location"];
121 				if (log) log(" > Redirect: " ~ location);
122 				request.resource = applyRelativeURL(request.url, location);
123 				if (response.status == HttpStatusCode.SeeOther)
124 				{
125 					request.method = "GET";
126 					request.data = null;
127 				}
128 				response = net.httpRequest(request);
129 			}
130 			else
131 				throw new Exception("Error with URL " ~ url ~ ": " ~ text(response.status));
132 		}
133 	}
134 
135 	static import std.json;
136 
137 	std.json.JSONValue[] pagedQuery(string url)
138 	{
139 		import std.json : JSONValue, parseJSON;
140 
141 		JSONValue[] result;
142 		while (true)
143 		{
144 			auto page = query(url);
145 			result ~= page.data.parseJSON().array;
146 			auto links = page.headers.get("Link", null).I!parseLinks();
147 			if ("next" in links)
148 				url = links["next"];
149 			else
150 				break;
151 		}
152 		return result;
153 	}
154 
155 	/// Parse a "Link" header.
156 	private static string[string] parseLinks(string s)
157 	{
158 		string[string] result;
159 		auto items = s.split(", "); // Hacky but should never occur inside an URL or "rel" value
160 		foreach (item; items)
161 		{
162 			auto parts = item.split("; "); // ditto
163 			string url; string[string] args;
164 			foreach (part; parts)
165 			{
166 				if (part.startsWith("<") && part.endsWith(">"))
167 					url = part[1..$-1];
168 				else
169 				{
170 					auto ps = part.findSplit("=");
171 					auto key = ps[0];
172 					auto value = ps[2];
173 					if (value.startsWith('"') && value.endsWith('"'))
174 						value = value[1..$-1];
175 					args[key] = value;
176 				}
177 			}
178 			result[args.get("rel", null)] = url;
179 		}
180 		return result;
181 	}
182 
183 	unittest
184 	{
185 		auto header = `<https://api.github.com/repositories/1257070/pulls?per_page=100&page=2>; rel="next", ` ~
186 			`<https://api.github.com/repositories/1257070/pulls?per_page=100&page=3>; rel="last"`;
187 		assert(parseLinks(header) == [
188 			"next" : "https://api.github.com/repositories/1257070/pulls?per_page=100&page=2",
189 			"last" : "https://api.github.com/repositories/1257070/pulls?per_page=100&page=3",
190 		]);
191 	}
192 
193 	string post(string url, Data jsonData)
194 	{
195 		auto request = new HttpRequest;
196 		request.resource = url;
197 		request.method = "POST";
198 		if (token)
199 			request.headers["Authorization"] = "token " ~ token;
200 		request.headers["Content-Type"] = "application/json";
201 		request.data = DataVec(jsonData);
202 
203 		auto response = net.httpRequest(request);
204 		string result = cast(string)response.data.joinToHeap;
205 		validate(result);
206 		return result;
207 	}
208 }