1 /** 2 * Synchronous helper to access the GitHub API 3 * 4 * License: 5 * This Source Code Form is subject to the terms of 6 * the Mozilla Public License, v. 2.0. If a copy of 7 * the MPL was not distributed with this file, You 8 * can obtain one at http://mozilla.org/MPL/2.0/. 9 * 10 * Authors: 11 * Vladimir Panteleev <ae@cy.md> 12 */ 13 14 module ae.net.github.rest; 15 16 package(ae): 17 18 import std.algorithm.searching; 19 import std.conv; 20 import std.string; 21 import std.utf; 22 23 import ae.net.http.common; 24 import ae.net.ietf.headers; 25 import ae.net.ietf.url; 26 import ae.sys.data; 27 import ae.sys.log; 28 import ae.sys.net; 29 import ae.utils.json; 30 import ae.utils.meta; 31 32 struct GitHub 33 { 34 string token; 35 36 void delegate(string) log; 37 38 bool offline; /// Use cached objects without re-validating them 39 40 struct CacheEntry 41 { 42 string[string] headers; 43 string data; 44 } 45 interface ICache 46 { 47 string get(string key); 48 void put(string key, string value); 49 } 50 static class NoCache : ICache 51 { 52 string get(string key) { return null; } 53 void put(string key, string value) {} 54 } 55 ICache cache = new NoCache; 56 57 struct Result 58 { 59 string[string] headers; 60 string data; 61 } 62 63 Result query(string url) 64 { 65 auto request = new HttpRequest; 66 request.resource = url; 67 if (token) 68 request.headers["Authorization"] = "token " ~ token; 69 70 auto cacheKey = url; 71 72 CacheEntry cacheEntry; 73 auto cacheEntryStr = cache.get(cacheKey); 74 if (cacheEntryStr) 75 { 76 cacheEntry = cacheEntryStr.jsonParse!CacheEntry(); 77 78 if (offline) 79 { 80 if (log) log(" > Cache hit (offline mode)"); 81 return Result(cacheEntry.headers, cacheEntry.data); 82 } 83 84 auto cacheHeaders = Headers(cacheEntry.headers); 85 86 if (auto p = "ETag" in cacheHeaders) 87 request.headers["If-None-Match"] = *p; 88 if (auto p = "Last-Modified" in cacheHeaders) 89 request.headers["If-Modified-Since"] = *p; 90 } 91 92 if (log) log("Getting URL " ~ url); 93 94 auto response = net.httpRequest(request); 95 while (true) // Redirect loop 96 { 97 if (response.status == HttpStatusCode.NotModified) 98 { 99 if (log) log(" > Cache hit"); 100 return Result(cacheEntry.headers, cacheEntry.data); 101 } 102 else 103 if (response.status == HttpStatusCode.OK) 104 { 105 if (log) log(" > Cache miss; ratelimit: %s/%s".format( 106 response.headers.get("X-Ratelimit-Remaining", "?"), 107 response.headers.get("X-Ratelimit-Limit", "?"), 108 )); 109 scope(failure) if (log) log(response.headers.text); 110 auto headers = response.headers.to!(string[string]); 111 auto data = (cast(char[])response.getContent().contents).idup; 112 cacheEntry.headers = headers; 113 cacheEntry.data = data; 114 cache.put(cacheKey, toJson(cacheEntry)); 115 return Result(headers, data); 116 } 117 else 118 if (response.status >= 300 && response.status < 400 && "Location" in response.headers) 119 { 120 auto location = response.headers["Location"]; 121 if (log) log(" > Redirect: " ~ location); 122 request.resource = applyRelativeURL(request.url, location); 123 if (response.status == HttpStatusCode.SeeOther) 124 { 125 request.method = "GET"; 126 request.data = null; 127 } 128 response = net.httpRequest(request); 129 } 130 else 131 throw new Exception("Error with URL " ~ url ~ ": " ~ text(response.status)); 132 } 133 } 134 135 static import std.json; 136 137 std.json.JSONValue[] pagedQuery(string url) 138 { 139 import std.json : JSONValue, parseJSON; 140 141 JSONValue[] result; 142 while (true) 143 { 144 auto page = query(url); 145 result ~= page.data.parseJSON().array; 146 auto links = page.headers.get("Link", null).I!parseLinks(); 147 if ("next" in links) 148 url = links["next"]; 149 else 150 break; 151 } 152 return result; 153 } 154 155 /// Parse a "Link" header. 156 private static string[string] parseLinks(string s) 157 { 158 string[string] result; 159 auto items = s.split(", "); // Hacky but should never occur inside an URL or "rel" value 160 foreach (item; items) 161 { 162 auto parts = item.split("; "); // ditto 163 string url; string[string] args; 164 foreach (part; parts) 165 { 166 if (part.startsWith("<") && part.endsWith(">")) 167 url = part[1..$-1]; 168 else 169 { 170 auto ps = part.findSplit("="); 171 auto key = ps[0]; 172 auto value = ps[2]; 173 if (value.startsWith('"') && value.endsWith('"')) 174 value = value[1..$-1]; 175 args[key] = value; 176 } 177 } 178 result[args.get("rel", null)] = url; 179 } 180 return result; 181 } 182 183 unittest 184 { 185 auto header = `<https://api.github.com/repositories/1257070/pulls?per_page=100&page=2>; rel="next", ` ~ 186 `<https://api.github.com/repositories/1257070/pulls?per_page=100&page=3>; rel="last"`; 187 assert(parseLinks(header) == [ 188 "next" : "https://api.github.com/repositories/1257070/pulls?per_page=100&page=2", 189 "last" : "https://api.github.com/repositories/1257070/pulls?per_page=100&page=3", 190 ]); 191 } 192 193 string post(string url, Data jsonData) 194 { 195 auto request = new HttpRequest; 196 request.resource = url; 197 request.method = "POST"; 198 if (token) 199 request.headers["Authorization"] = "token " ~ token; 200 request.headers["Content-Type"] = "application/json"; 201 request.data = DataVec(jsonData); 202 203 auto response = net.httpRequest(request); 204 string result = cast(string)response.data.joinToHeap; 205 validate(result); 206 return result; 207 } 208 }