1 /** 2 * Synchronous helper to access the GitHub API 3 * 4 * License: 5 * This Source Code Form is subject to the terms of 6 * the Mozilla Public License, v. 2.0. If a copy of 7 * the MPL was not distributed with this file, You 8 * can obtain one at http://mozilla.org/MPL/2.0/. 9 * 10 * Authors: 11 * Vladimir Panteleev <ae@cy.md> 12 */ 13 14 module ae.net.github.rest; 15 16 package(ae): 17 18 import std.algorithm.searching; 19 import std.conv; 20 import std.string; 21 import std.utf; 22 23 import ae.net.http.common; 24 import ae.net.ietf.headers; 25 import ae.net.ietf.url; 26 import ae.sys.data; 27 import ae.sys.dataset : DataVec, joinToGC; 28 import ae.sys.log; 29 import ae.sys.net; 30 import ae.utils.array : as, fromBytes; 31 import ae.utils.json; 32 import ae.utils.meta; 33 34 struct GitHub 35 { 36 string token; 37 38 void delegate(string) log; 39 40 bool offline; /// Use cached objects without re-validating them 41 42 struct CacheEntry 43 { 44 string[string] headers; 45 string data; 46 } 47 interface ICache 48 { 49 string get(string key); 50 void put(string key, string value); 51 } 52 static class NoCache : ICache 53 { 54 string get(string key) { return null; } 55 void put(string key, string value) {} 56 } 57 ICache cache = new NoCache; 58 59 struct Result 60 { 61 string[string] headers; 62 string data; 63 } 64 65 Result query(string url) 66 { 67 auto request = new HttpRequest; 68 request.resource = url; 69 if (token) 70 request.headers["Authorization"] = "token " ~ token; 71 72 auto cacheKey = url; 73 74 CacheEntry cacheEntry; 75 auto cacheEntryStr = cache.get(cacheKey); 76 if (cacheEntryStr) 77 { 78 cacheEntry = cacheEntryStr.jsonParse!CacheEntry(); 79 80 if (offline) 81 { 82 if (log) log(" > Cache hit (offline mode)"); 83 return Result(cacheEntry.headers, cacheEntry.data); 84 } 85 86 auto cacheHeaders = Headers(cacheEntry.headers); 87 88 if (auto p = "ETag" in cacheHeaders) 89 request.headers["If-None-Match"] = *p; 90 if (auto p = "Last-Modified" in cacheHeaders) 91 request.headers["If-Modified-Since"] = *p; 92 } 93 94 if (log) log("Getting URL " ~ url); 95 96 auto response = net.httpRequest(request); 97 while (true) // Redirect loop 98 { 99 if (response.status == HttpStatusCode.NotModified) 100 { 101 if (log) log(" > Cache hit"); 102 return Result(cacheEntry.headers, cacheEntry.data); 103 } 104 else 105 if (response.status == HttpStatusCode.OK) 106 { 107 if (log) log(" > Cache miss; ratelimit: %s/%s".format( 108 response.headers.get("X-Ratelimit-Remaining", "?"), 109 response.headers.get("X-Ratelimit-Limit", "?"), 110 )); 111 scope(failure) if (log) log(response.headers.text); 112 auto headers = response.headers.to!(string[string]); 113 auto data = response.getContent().toGC().as!string; 114 cacheEntry.headers = headers; 115 cacheEntry.data = data; 116 cache.put(cacheKey, toJson(cacheEntry)); 117 return Result(headers, data); 118 } 119 else 120 if (response.status >= 300 && response.status < 400 && "Location" in response.headers) 121 { 122 auto location = response.headers["Location"]; 123 if (log) log(" > Redirect: " ~ location); 124 request.resource = applyRelativeURL(request.url, location); 125 if (response.status == HttpStatusCode.SeeOther) 126 { 127 request.method = "GET"; 128 request.data = null; 129 } 130 response = net.httpRequest(request); 131 } 132 else 133 throw new Exception("Error with URL " ~ url ~ ": " ~ text(response.status)); 134 } 135 } 136 137 static import std.json; 138 139 std.json.JSONValue[] pagedQuery(string url) 140 { 141 import std.json : JSONValue, parseJSON; 142 143 JSONValue[] result; 144 while (true) 145 { 146 auto page = query(url); 147 result ~= page.data.parseJSON().array; 148 auto links = page.headers.get("Link", null).I!parseLinks(); 149 if ("next" in links) 150 url = links["next"]; 151 else 152 break; 153 } 154 return result; 155 } 156 157 /// Parse a "Link" header. 158 private static string[string] parseLinks(string s) 159 { 160 string[string] result; 161 auto items = s.split(", "); // Hacky but should never occur inside an URL or "rel" value 162 foreach (item; items) 163 { 164 auto parts = item.split("; "); // ditto 165 string url; string[string] args; 166 foreach (part; parts) 167 { 168 if (part.startsWith("<") && part.endsWith(">")) 169 url = part[1..$-1]; 170 else 171 { 172 auto ps = part.findSplit("="); 173 auto key = ps[0]; 174 auto value = ps[2]; 175 if (value.startsWith('"') && value.endsWith('"')) 176 value = value[1..$-1]; 177 args[key] = value; 178 } 179 } 180 result[args.get("rel", null)] = url; 181 } 182 return result; 183 } 184 185 unittest 186 { 187 auto header = `<https://api.github.com/repositories/1257070/pulls?per_page=100&page=2>; rel="next", ` ~ 188 `<https://api.github.com/repositories/1257070/pulls?per_page=100&page=3>; rel="last"`; 189 assert(parseLinks(header) == [ 190 "next" : "https://api.github.com/repositories/1257070/pulls?per_page=100&page=2", 191 "last" : "https://api.github.com/repositories/1257070/pulls?per_page=100&page=3", 192 ]); 193 } 194 195 string post(string url, Data jsonData) 196 { 197 auto request = new HttpRequest; 198 request.resource = url; 199 request.method = "POST"; 200 if (token) 201 request.headers["Authorization"] = "token " ~ token; 202 request.headers["Content-Type"] = "application/json"; 203 request.data = DataVec(jsonData); 204 205 auto response = net.httpRequest(request); 206 string result = response.data.joinToGC.as!string; 207 validate(result); 208 return result; 209 } 210 }