1 /** 2 * Synchronous helper to access the GitHub API 3 * 4 * License: 5 * This Source Code Form is subject to the terms of 6 * the Mozilla Public License, v. 2.0. If a copy of 7 * the MPL was not distributed with this file, You 8 * can obtain one at http://mozilla.org/MPL/2.0/. 9 * 10 * Authors: 11 * Vladimir Panteleev <ae@cy.md> 12 */ 13 14 module ae.net.github.rest; 15 16 package(ae): 17 18 import std.algorithm.searching; 19 import std.conv; 20 import std.string; 21 import std.utf; 22 23 import ae.net.http.common; 24 import ae.net.ietf.headers; 25 import ae.net.ietf.url; 26 import ae.sys.data; 27 import ae.sys.log; 28 import ae.sys.net; 29 import ae.utils.json; 30 import ae.utils.meta; 31 32 struct GitHub 33 { 34 string token; 35 36 void delegate(string) log; 37 38 struct CacheEntry 39 { 40 string[string] headers; 41 string data; 42 } 43 interface ICache 44 { 45 string get(string key); 46 void put(string key, string value); 47 } 48 static class NoCache : ICache 49 { 50 string get(string key) { return null; } 51 void put(string key, string value) {} 52 } 53 ICache cache = new NoCache; 54 55 struct Result 56 { 57 string[string] headers; 58 string data; 59 } 60 61 Result query(string url) 62 { 63 auto request = new HttpRequest; 64 request.resource = url; 65 if (token) 66 request.headers["Authorization"] = "token " ~ token; 67 68 auto cacheKey = url; 69 70 CacheEntry cacheEntry; 71 auto cacheEntryStr = cache.get(cacheKey); 72 if (cacheEntryStr) 73 { 74 cacheEntry = cacheEntryStr.jsonParse!CacheEntry(); 75 auto cacheHeaders = Headers(cacheEntry.headers); 76 77 if (auto p = "ETag" in cacheHeaders) 78 request.headers["If-None-Match"] = *p; 79 if (auto p = "Last-Modified" in cacheHeaders) 80 request.headers["If-Modified-Since"] = *p; 81 } 82 83 if (log) log("Getting URL " ~ url); 84 85 auto response = net.httpRequest(request); 86 while (true) // Redirect loop 87 { 88 if (response.status == HttpStatusCode.NotModified) 89 { 90 if (log) log(" > Cache hit"); 91 return Result(cacheEntry.headers, cacheEntry.data); 92 } 93 else 94 if (response.status == HttpStatusCode.OK) 95 { 96 if (log) log(" > Cache miss; ratelimit: %s/%s".format( 97 response.headers.get("X-Ratelimit-Remaining", "?"), 98 response.headers.get("X-Ratelimit-Limit", "?"), 99 )); 100 scope(failure) if (log) log(response.headers.text); 101 auto headers = response.headers.to!(string[string]); 102 auto data = (cast(char[])response.getContent().contents).idup; 103 cacheEntry.headers = headers; 104 cacheEntry.data = data; 105 cache.put(cacheKey, toJson(cacheEntry)); 106 return Result(headers, data); 107 } 108 else 109 if (response.status >= 300 && response.status < 400 && "Location" in response.headers) 110 { 111 auto location = response.headers["Location"]; 112 if (log) log(" > Redirect: " ~ location); 113 request.resource = applyRelativeURL(request.url, location); 114 if (response.status == HttpStatusCode.SeeOther) 115 { 116 request.method = "GET"; 117 request.data = null; 118 } 119 response = net.httpRequest(request); 120 } 121 else 122 throw new Exception("Error with URL " ~ url ~ ": " ~ text(response.status)); 123 } 124 } 125 126 static import std.json; 127 128 std.json.JSONValue[] pagedQuery(string url) 129 { 130 import std.json : JSONValue, parseJSON; 131 132 JSONValue[] result; 133 while (true) 134 { 135 auto page = query(url); 136 result ~= page.data.parseJSON().array; 137 auto links = page.headers.get("Link", null).I!parseLinks(); 138 if ("next" in links) 139 url = links["next"]; 140 else 141 break; 142 } 143 return result; 144 } 145 146 /// Parse a "Link" header. 147 private static string[string] parseLinks(string s) 148 { 149 string[string] result; 150 auto items = s.split(", "); // Hacky but should never occur inside an URL or "rel" value 151 foreach (item; items) 152 { 153 auto parts = item.split("; "); // ditto 154 string url; string[string] args; 155 foreach (part; parts) 156 { 157 if (part.startsWith("<") && part.endsWith(">")) 158 url = part[1..$-1]; 159 else 160 { 161 auto ps = part.findSplit("="); 162 auto key = ps[0]; 163 auto value = ps[2]; 164 if (value.startsWith('"') && value.endsWith('"')) 165 value = value[1..$-1]; 166 args[key] = value; 167 } 168 } 169 result[args.get("rel", null)] = url; 170 } 171 return result; 172 } 173 174 unittest 175 { 176 auto header = `<https://api.github.com/repositories/1257070/pulls?per_page=100&page=2>; rel="next", ` ~ 177 `<https://api.github.com/repositories/1257070/pulls?per_page=100&page=3>; rel="last"`; 178 assert(parseLinks(header) == [ 179 "next" : "https://api.github.com/repositories/1257070/pulls?per_page=100&page=2", 180 "last" : "https://api.github.com/repositories/1257070/pulls?per_page=100&page=3", 181 ]); 182 } 183 184 string post(string url, Data jsonData) 185 { 186 auto request = new HttpRequest; 187 request.resource = url; 188 request.method = "POST"; 189 if (token) 190 request.headers["Authorization"] = "token " ~ token; 191 request.headers["Content-Type"] = "application/json"; 192 request.data = DataVec(jsonData); 193 194 auto response = net.httpRequest(request); 195 string result = cast(string)response.data.joinToHeap; 196 validate(result); 197 return result; 198 } 199 }