1 /** 2 * Code to manage a D component repository. 3 * 4 * License: 5 * This Source Code Form is subject to the terms of 6 * the Mozilla Public License, v. 2.0. If a copy of 7 * the MPL was not distributed with this file, You 8 * can obtain one at http://mozilla.org/MPL/2.0/. 9 * 10 * Authors: 11 * Vladimir Panteleev <vladimir@thecybershadow.net> 12 */ 13 14 module ae.sys.d.repo; 15 16 import std.algorithm; 17 import std.conv : text; 18 import std.exception; 19 import std.file; 20 import std.process : environment; 21 import std.range; 22 import std.regex; 23 import std.string; 24 import std.path; 25 26 import ae.sys.git; 27 import ae.utils.exception; 28 import ae.utils.json; 29 import ae.utils.regex; 30 import ae.utils.time : StdTime; 31 32 /// Base class for a managed repository. 33 class ManagedRepository 34 { 35 /// Git repository we manage. 36 public Repository git; 37 38 /// Should we fetch the latest stuff? 39 public bool offline; 40 41 /// Ensure we have a repository. 42 public void needRepo() 43 { 44 assert(git.path, "No repository"); 45 } 46 47 public @property string name() { needRepo(); return git.path.baseName; } 48 49 // Head 50 51 /// Ensure the repository's HEAD is as indicated. 52 public void needHead(string hash) 53 { 54 needClean(); 55 if (getHead() == hash) 56 return; 57 58 try 59 performCheckout(hash); 60 catch (Exception e) 61 { 62 log("Error checking out %s: %s".format(hash, e)); 63 64 // Might be a GC-ed merge. Try to recreate the merge 65 auto hit = mergeCache.find!(entry => entry.result == hash)(); 66 enforce(!hit.empty, "Unknown hash %s".format(hash)); 67 performMerge(hit.front.base, hit.front.branch, hit.front.revert, hit.front.mainline); 68 enforce(getHead() == hash, "Unexpected merge result: expected %s, got %s".format(hash, getHead())); 69 } 70 } 71 72 private string currentHead = null; 73 74 /// Returns the SHA1 of the given named ref. 75 public string getRef(string name) 76 { 77 return git.query("rev-parse", name); 78 } 79 80 /// Return the commit the repository HEAD is pointing at. 81 /// Cache the result. 82 public string getHead() 83 { 84 if (!currentHead) 85 currentHead = getRef("HEAD"); 86 87 return currentHead; 88 } 89 90 protected void performCheckout(string hash) 91 { 92 needClean(); 93 94 log("Checking out %s commit %s...".format(name, hash)); 95 96 if (offline) 97 git.run("checkout", hash); 98 else 99 { 100 try 101 git.run("checkout", hash); 102 catch (Exception e) 103 { 104 log("Checkout failed, updating and retrying..."); 105 update(); 106 git.run("checkout", hash); 107 } 108 } 109 110 saveState(); 111 currentHead = hash; 112 } 113 114 /// Update the remote. 115 public void update() 116 { 117 if (!offline) 118 { 119 needRepo(); 120 log("Updating " ~ name ~ "..."); 121 git.run("-c", "fetch.recurseSubmodules=false", "remote", "update", "--prune"); 122 git.run("-c", "fetch.recurseSubmodules=false", "fetch", "--tags"); 123 } 124 } 125 126 // Clean 127 128 bool clean = false; 129 130 /// Ensure the repository's working copy is clean. 131 public void needClean() 132 { 133 if (clean) 134 return; 135 performCleanup(); 136 clean = true; 137 } 138 139 private void performCleanup() 140 { 141 checkState(); 142 clearState(); 143 144 log("Cleaning repository %s...".format(name)); 145 needRepo(); 146 try 147 { 148 git.run("reset", "--hard"); 149 git.run("clean", "--force", "-x", "-d", "--quiet"); 150 } 151 catch (Exception e) 152 throw new RepositoryCleanException(e.msg, e); 153 saveState(); 154 } 155 156 // Merge cache 157 158 private static struct MergeInfo 159 { 160 string base, branch; 161 bool revert = false; 162 int mainline = 0; 163 string result; 164 } 165 private alias MergeCache = MergeInfo[]; 166 private MergeCache mergeCacheData; 167 private bool haveMergeCache; 168 169 private @property ref MergeCache mergeCache() 170 { 171 if (!haveMergeCache) 172 { 173 if (mergeCachePath.exists) 174 mergeCacheData = mergeCachePath.readText().jsonParse!MergeCache; 175 haveMergeCache = true; 176 } 177 178 return mergeCacheData; 179 } 180 181 private void saveMergeCache() 182 { 183 std.file.write(mergeCachePath(), toJson(mergeCache)); 184 } 185 186 private @property string mergeCachePath() 187 { 188 needRepo(); 189 return buildPath(git.gitDir, "ae-sys-d-mergecache.json"); 190 } 191 192 // Merge 193 194 private void setupGitEnv() 195 { 196 string[string] mergeEnv; 197 foreach (person; ["AUTHOR", "COMMITTER"]) 198 { 199 mergeEnv["GIT_%s_DATE".format(person)] = "Thu, 01 Jan 1970 00:00:00 +0000"; 200 mergeEnv["GIT_%s_NAME".format(person)] = "ae.sys.d"; 201 mergeEnv["GIT_%s_EMAIL".format(person)] = "ae.sys.d\x40thecybershadow.net"; 202 } 203 foreach (k, v; mergeEnv) 204 environment[k] = v; 205 // TODO: restore environment 206 } 207 208 /// Returns the hash of the merge between the base and branch commits. 209 /// Performs the merge if necessary. Caches the result. 210 public string getMerge(string base, string branch) 211 { 212 return getMergeImpl(base, branch, false, 0); 213 } 214 215 /// Returns the resulting hash when reverting the branch from the base commit. 216 /// Performs the revert if necessary. Caches the result. 217 /// mainline is the 1-based mainline index (as per `man git-revert`), 218 /// or 0 if commit is not a merge commit. 219 public string getRevert(string base, string branch, int mainline) 220 { 221 return getMergeImpl(base, branch, true, mainline); 222 } 223 224 private string getMergeImpl(string base, string branch, bool revert, int mainline) 225 { 226 auto hit = mergeCache.find!(entry => 227 entry.base == base && 228 entry.branch == branch && 229 entry.revert == revert && 230 entry.mainline == mainline)(); 231 if (!hit.empty) 232 return hit.front.result; 233 234 performMerge(base, branch, revert, mainline); 235 236 auto head = getHead(); 237 mergeCache ~= MergeInfo(base, branch, revert, mainline, head); 238 saveMergeCache(); 239 return head; 240 } 241 242 private static const string mergeCommitMessage = "ae.sys.d merge"; 243 private static const string revertCommitMessage = "ae.sys.d revert"; 244 245 // Performs a merge or revert. 246 private void performMerge(string base, string branch, bool revert, int mainline) 247 { 248 needHead(base); 249 currentHead = null; 250 251 log("%s %s into %s.".format(revert ? "Reverting" : "Merging", branch, base)); 252 253 scope (failure) 254 { 255 if (!revert) 256 { 257 log("Aborting merge..."); 258 git.run("merge", "--abort"); 259 } 260 else 261 { 262 log("Aborting revert..."); 263 git.run("revert", "--abort"); 264 } 265 clean = false; 266 } 267 268 void doMerge() 269 { 270 setupGitEnv(); 271 if (!revert) 272 git.run("merge", "--no-ff", "-m", mergeCommitMessage, branch); 273 else 274 { 275 string[] args = ["revert", "--no-edit"]; 276 if (mainline) 277 args ~= ["--mainline", text(mainline)]; 278 args ~= [branch]; 279 git.run(args); 280 } 281 } 282 283 if (git.path.baseName() == "dmd") 284 { 285 try 286 doMerge(); 287 catch (Exception) 288 { 289 log("Merge failed. Attempting conflict resolution..."); 290 git.run("checkout", "--theirs", "test"); 291 git.run("add", "test"); 292 if (!revert) 293 git.run("-c", "rerere.enabled=false", "commit", "-m", mergeCommitMessage); 294 else 295 git.run("revert", "--continue"); 296 } 297 } 298 else 299 doMerge(); 300 301 saveState(); 302 log("Merge successful."); 303 } 304 305 /// Finds and returns the merge parents of the given merge commit. 306 /// Queries the git repository if necessary. Caches the result. 307 public MergeInfo getMergeInfo(string merge) 308 { 309 auto hit = mergeCache.find!(entry => entry.result == merge && !entry.revert)(); 310 if (!hit.empty) 311 return hit.front; 312 313 auto parents = git.query(["log", "--pretty=%P", "-n", "1", merge]).split(); 314 enforce(parents.length > 1, "Not a merge: " ~ merge); 315 enforce(parents.length == 2, "Too many parents: " ~ merge); 316 317 auto info = MergeInfo(parents[0], parents[1], false, 0, merge); 318 mergeCache ~= info; 319 return info; 320 } 321 322 /// Follows the string of merges starting from the given 323 /// head commit, up till the merge with the given branch. 324 /// Then, reapplies all merges in order, 325 /// except for that with the given branch. 326 public string getUnMerge(string head, string branch) 327 { 328 // This could be optimized using an interactive rebase 329 330 auto info = getMergeInfo(head); 331 if (info.branch == branch) 332 return info.base; 333 334 return getMerge(getUnMerge(info.base, branch), info.branch); 335 } 336 337 // Branches, forks and customization 338 339 /// Return SHA1 of the given remote ref. 340 /// Fetches the remote first, unless offline mode is on. 341 string getRemoteRef(string remote, string remoteRef, string localRef) 342 { 343 needRepo(); 344 if (!offline) 345 { 346 log("Fetching from %s (%s -> %s) ...".format(remote, remoteRef, localRef)); 347 git.run("fetch", remote, "+%s:%s".format(remoteRef, localRef)); 348 } 349 return getRef(localRef); 350 } 351 352 /// Return SHA1 of the given pull request #. 353 /// Fetches the pull request first, unless offline mode is on. 354 string getPull(int pull) 355 { 356 return getRemoteRef( 357 "origin", 358 "refs/pull/%d/head".format(pull), 359 "refs/digger/pull/%d".format(pull), 360 ); 361 } 362 363 /// Return SHA1 of the given GitHub fork. 364 /// Fetches the fork first, unless offline mode is on. 365 /// (This is a thin wrapper around getRemoteBranch.) 366 string getFork(string user, string branch) 367 { 368 enforce(user .match(re!`^\w[\w\-]*$`), "Bad remote name"); 369 enforce(branch.match(re!`^\w[\w\-\.]*$`), "Bad branch name"); 370 371 return getRemoteRef( 372 "https://github.com/%s/%s".format(user, name), 373 "refs/heads/%s".format(branch), 374 "refs/digger/fork/%s/%s".format(user, branch), 375 ); 376 } 377 378 /// Find the child of a commit, and, if the commit was a merge, 379 /// the mainline index of said commit for the child. 380 void getChild(string branch, string commit, out string child, out int mainline) 381 { 382 log("Querying history for commit children..."); 383 auto history = git.getHistory(); 384 385 bool[Hash] seen; 386 void visit(Commit* commit) 387 { 388 if (commit.hash !in seen) 389 { 390 seen[commit.hash] = true; 391 foreach (parent; commit.parents) 392 visit(parent); 393 } 394 } 395 auto branchHash = branch.toCommitHash(); 396 auto pBranchCommit = branchHash in history.commits; 397 enforce(pBranchCommit, "Can't find commit in history"); 398 visit(*pBranchCommit); 399 400 auto commitHash = commit.toCommitHash(); 401 auto pCommit = commitHash in history.commits; 402 enforce(pCommit, "Can't find commit in history"); 403 auto children = (*pCommit).children; 404 enforce(children.length, "Commit has no children"); 405 children = children.filter!(child => child.hash in seen).array(); 406 enforce(children.length, "Commit has no children under specified branch"); 407 enforce(children.length == 1, "Commit has more than one child"); 408 auto childCommit = children[0]; 409 child = childCommit.hash.toString(); 410 411 if (childCommit.parents.length == 1) 412 mainline = 0; 413 else 414 { 415 enforce(childCommit.parents.length == 2, "Can't get mainline of multiple-branch merges"); 416 if (childCommit.parents[0] is *pCommit) 417 mainline = 2; 418 else 419 mainline = 1; 420 421 auto mergeInfo = MergeInfo( 422 childCommit.parents[0].hash.toString(), 423 childCommit.parents[1].hash.toString(), 424 true, mainline, commit); 425 if (!mergeCache.canFind(mergeInfo)) 426 { 427 mergeCache ~= mergeInfo; 428 saveMergeCache(); 429 } 430 } 431 } 432 433 // State saving and checking 434 435 struct FileState 436 { 437 ulong size; 438 StdTime modificationTime; 439 } 440 441 FileState getFileState(string file) 442 { 443 auto path = git.path.buildPath(file); 444 auto de = DirEntry(path); 445 return FileState(de.size, de.timeLastModified.stdTime); 446 } 447 448 alias RepositoryState = FileState[string]; 449 450 /// Return the working tree "state". 451 /// This returns a file list, along with size and modification time. 452 RepositoryState getState() 453 { 454 needRepo(); 455 auto files = git.query(["ls-files"]).splitLines(); 456 RepositoryState state; 457 foreach (file; files) 458 state[file] = getFileState(file); 459 return state; 460 } 461 462 private @property string workTreeStatePath() 463 { 464 needRepo(); 465 return buildPath(git.gitDir, "ae-sys-d-worktree.json"); 466 } 467 468 /// Save the state of the working tree for versioned files 469 /// to a .json file, which can later be verified with checkState. 470 /// This should be called after any git command which mutates the git state. 471 void saveState() 472 { 473 std.file.write(workTreeStatePath, getState().toJson()); 474 } 475 476 /// Save the state of just one file. 477 /// This should be called after automatic edits to repository files during a build. 478 /// The file parameter should be relative to the directory root, and use forward slashes. 479 void saveFileState(string file) 480 { 481 if (!workTreeStatePath.exists) 482 return; 483 auto state = workTreeStatePath.readText.jsonParse!RepositoryState(); 484 state[file] = getFileState(file); 485 std.file.write(workTreeStatePath, state.toJson()); 486 } 487 488 /// Verify that the state of the working tree matches the one 489 /// when saveState was last called. Throw an exception otherwise. 490 /// This and clearState should be called before any git command 491 /// which destroys working directory changes. 492 void checkState() 493 { 494 if (!workTreeStatePath.exists) 495 return; 496 auto savedState = workTreeStatePath.readText.jsonParse!RepositoryState(); 497 auto currentState = getState(); 498 try 499 { 500 foreach (file, fileState; currentState) 501 { 502 enforce(file in savedState, "New file: " ~ file); 503 enforce(savedState[file] == fileState, "File modified: " ~ file); 504 } 505 } 506 catch (Exception e) 507 throw new Exception(e.msg ~ "\n" ~ "Save / commit your changes, then delete " ~ workTreeStatePath); 508 } 509 510 /// Delete the saved working tree state, if any. 511 void clearState() 512 { 513 if (workTreeStatePath.exists) 514 workTreeStatePath.remove(); 515 } 516 517 // Misc 518 519 /// Reset internal state. 520 protected void reset() 521 { 522 currentHead = null; 523 clean = false; 524 haveMergeCache = false; 525 mergeCacheData = null; 526 } 527 528 /// Override to add logging. 529 protected abstract void log(string line); 530 } 531 532 /// Used to communicate that a "reset --hard" failed. 533 /// Generally this indicates git repository corruption. 534 mixin DeclareException!q{RepositoryCleanException};