1 /**
2  * Code to manage a D component repository.
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Vladimir Panteleev <vladimir@thecybershadow.net>
12  */
13 
14 module ae.sys.d.repo;
15 
16 import std.algorithm;
17 import std.conv : text;
18 import std.exception;
19 import std.file;
20 import std.process : environment;
21 import std.range;
22 import std.regex;
23 import std.string;
24 import std.path;
25 
26 import ae.sys.git;
27 import ae.utils.exception;
28 import ae.utils.json;
29 import ae.utils.regex;
30 import ae.utils.time : StdTime;
31 
32 /// Base class for a managed repository.
33 class ManagedRepository
34 {
35 	/// Git repository we manage.
36 	public Repository git;
37 
38 	/// Should we fetch the latest stuff?
39 	public bool offline;
40 
41 	/// Ensure we have a repository.
42 	public void needRepo()
43 	{
44 		assert(git.path, "No repository");
45 	}
46 
47 	public @property string name() { needRepo(); return git.path.baseName; }
48 
49 	// Head
50 
51 	/// Ensure the repository's HEAD is as indicated.
52 	public void needHead(string hash)
53 	{
54 		needClean();
55 		if (getHead() == hash)
56 			return;
57 
58 		try
59 			performCheckout(hash);
60 		catch (Exception e)
61 		{
62 			log("Error checking out %s: %s".format(hash, e));
63 
64 			// Might be a GC-ed merge. Try to recreate the merge
65 			auto hit = mergeCache.find!(entry => entry.result == hash)();
66 			enforce(!hit.empty, "Unknown hash %s".format(hash));
67 			performMerge(hit.front.base, hit.front.branch, hit.front.revert, hit.front.mainline);
68 			enforce(getHead() == hash, "Unexpected merge result: expected %s, got %s".format(hash, getHead()));
69 		}
70 	}
71 
72 	private string currentHead = null;
73 
74 	/// Returns the SHA1 of the given named ref.
75 	public string getRef(string name)
76 	{
77 		return git.query("rev-parse", name);
78 	}
79 
80 	/// Return the commit the repository HEAD is pointing at.
81 	/// Cache the result.
82 	public string getHead()
83 	{
84 		if (!currentHead)
85 			currentHead = getRef("HEAD");
86 
87 		return currentHead;
88 	}
89 
90 	protected void performCheckout(string hash)
91 	{
92 		needClean();
93 
94 		log("Checking out %s commit %s...".format(name, hash));
95 
96 		if (offline)
97 			git.run("checkout", hash);
98 		else
99 		{
100 			try
101 				git.run("checkout", hash);
102 			catch (Exception e)
103 			{
104 				log("Checkout failed, updating and retrying...");
105 				update();
106 				git.run("checkout", hash);
107 			}
108 		}
109 
110 		saveState();
111 		currentHead = hash;
112 	}
113 
114 	/// Update the remote.
115 	public void update()
116 	{
117 		if (!offline)
118 		{
119 			needRepo();
120 			log("Updating " ~ name ~ "...");
121 			git.run("-c", "fetch.recurseSubmodules=false", "remote", "update", "--prune");
122 			git.run("-c", "fetch.recurseSubmodules=false", "fetch", "--tags");
123 		}
124 	}
125 
126 	// Clean
127 
128 	bool clean = false;
129 
130 	/// Ensure the repository's working copy is clean.
131 	public void needClean()
132 	{
133 		if (clean)
134 			return;
135 		performCleanup();
136 		clean = true;
137 	}
138 
139 	private void performCleanup()
140 	{
141 		checkState();
142 		clearState();
143 
144 		log("Cleaning repository %s...".format(name));
145 		needRepo();
146 		try
147 		{
148 			git.run("reset", "--hard");
149 			git.run("clean", "--force", "-x", "-d", "--quiet");
150 		}
151 		catch (Exception e)
152 			throw new RepositoryCleanException(e.msg, e);
153 		saveState();
154 	}
155 
156 	// Merge cache
157 
158 	private static struct MergeInfo
159 	{
160 		string base, branch;
161 		bool revert = false;
162 		int mainline = 0;
163 		string result;
164 	}
165 	private alias MergeCache = MergeInfo[];
166 	private MergeCache mergeCacheData;
167 	private bool haveMergeCache;
168 
169 	private @property ref MergeCache mergeCache()
170 	{
171 		if (!haveMergeCache)
172 		{
173 			if (mergeCachePath.exists)
174 				mergeCacheData = mergeCachePath.readText().jsonParse!MergeCache;
175 			haveMergeCache = true;
176 		}
177 
178 		return mergeCacheData;
179 	}
180 
181 	private void saveMergeCache()
182 	{
183 		std.file.write(mergeCachePath(), toJson(mergeCache));
184 	}
185 
186 	private @property string mergeCachePath()
187 	{
188 		needRepo();
189 		return buildPath(git.gitDir, "ae-sys-d-mergecache.json");
190 	}
191 
192 	// Merge
193 
194 	private void setupGitEnv()
195 	{
196 		string[string] mergeEnv;
197 		foreach (person; ["AUTHOR", "COMMITTER"])
198 		{
199 			mergeEnv["GIT_%s_DATE".format(person)] = "Thu, 01 Jan 1970 00:00:00 +0000";
200 			mergeEnv["GIT_%s_NAME".format(person)] = "ae.sys.d";
201 			mergeEnv["GIT_%s_EMAIL".format(person)] = "ae.sys.d\x40thecybershadow.net";
202 		}
203 		foreach (k, v; mergeEnv)
204 			environment[k] = v;
205 		// TODO: restore environment
206 	}
207 
208 	/// Returns the hash of the merge between the base and branch commits.
209 	/// Performs the merge if necessary. Caches the result.
210 	public string getMerge(string base, string branch)
211 	{
212 		return getMergeImpl(base, branch, false, 0);
213 	}
214 
215 	/// Returns the resulting hash when reverting the branch from the base commit.
216 	/// Performs the revert if necessary. Caches the result.
217 	/// mainline is the 1-based mainline index (as per `man git-revert`),
218 	/// or 0 if commit is not a merge commit.
219 	public string getRevert(string base, string branch, int mainline)
220 	{
221 		return getMergeImpl(base, branch, true, mainline);
222 	}
223 
224 	private string getMergeImpl(string base, string branch, bool revert, int mainline)
225 	{
226 		auto hit = mergeCache.find!(entry =>
227 			entry.base == base &&
228 			entry.branch == branch &&
229 			entry.revert == revert &&
230 			entry.mainline == mainline)();
231 		if (!hit.empty)
232 			return hit.front.result;
233 
234 		performMerge(base, branch, revert, mainline);
235 
236 		auto head = getHead();
237 		mergeCache ~= MergeInfo(base, branch, revert, mainline, head);
238 		saveMergeCache();
239 		return head;
240 	}
241 
242 	private static const string mergeCommitMessage = "ae.sys.d merge";
243 	private static const string revertCommitMessage = "ae.sys.d revert";
244 
245 	// Performs a merge or revert.
246 	private void performMerge(string base, string branch, bool revert, int mainline)
247 	{
248 		needHead(base);
249 		currentHead = null;
250 
251 		log("%s %s into %s.".format(revert ? "Reverting" : "Merging", branch, base));
252 
253 		scope (failure)
254 		{
255 			if (!revert)
256 			{
257 				log("Aborting merge...");
258 				git.run("merge", "--abort");
259 			}
260 			else
261 			{
262 				log("Aborting revert...");
263 				git.run("revert", "--abort");
264 			}
265 			clean = false;
266 		}
267 
268 		void doMerge()
269 		{
270 			setupGitEnv();
271 			if (!revert)
272 				git.run("merge", "--no-ff", "-m", mergeCommitMessage, branch);
273 			else
274 			{
275 				string[] args = ["revert", "--no-edit"];
276 				if (mainline)
277 					args ~= ["--mainline", text(mainline)];
278 				args ~= [branch];
279 				git.run(args);
280 			}
281 		}
282 
283 		if (git.path.baseName() == "dmd")
284 		{
285 			try
286 				doMerge();
287 			catch (Exception)
288 			{
289 				log("Merge failed. Attempting conflict resolution...");
290 				git.run("checkout", "--theirs", "test");
291 				git.run("add", "test");
292 				if (!revert)
293 					git.run("-c", "rerere.enabled=false", "commit", "-m", mergeCommitMessage);
294 				else
295 					git.run("revert", "--continue");
296 			}
297 		}
298 		else
299 			doMerge();
300 
301 		saveState();
302 		log("Merge successful.");
303 	}
304 
305 	/// Finds and returns the merge parents of the given merge commit.
306 	/// Queries the git repository if necessary. Caches the result.
307 	public MergeInfo getMergeInfo(string merge)
308 	{
309 		auto hit = mergeCache.find!(entry => entry.result == merge && !entry.revert)();
310 		if (!hit.empty)
311 			return hit.front;
312 
313 		auto parents = git.query(["log", "--pretty=%P", "-n", "1", merge]).split();
314 		enforce(parents.length > 1, "Not a merge: " ~ merge);
315 		enforce(parents.length == 2, "Too many parents: " ~ merge);
316 
317 		auto info = MergeInfo(parents[0], parents[1], false, 0, merge);
318 		mergeCache ~= info;
319 		return info;
320 	}
321 
322 	/// Follows the string of merges starting from the given
323 	/// head commit, up till the merge with the given branch.
324 	/// Then, reapplies all merges in order,
325 	/// except for that with the given branch.
326 	public string getUnMerge(string head, string branch)
327 	{
328 		// This could be optimized using an interactive rebase
329 
330 		auto info = getMergeInfo(head);
331 		if (info.branch == branch)
332 			return info.base;
333 
334 		return getMerge(getUnMerge(info.base, branch), info.branch);
335 	}
336 
337 	// Branches, forks and customization
338 
339 	/// Return SHA1 of the given remote ref.
340 	/// Fetches the remote first, unless offline mode is on.
341 	string getRemoteRef(string remote, string remoteRef, string localRef)
342 	{
343 		needRepo();
344 		if (!offline)
345 		{
346 			log("Fetching from %s (%s -> %s) ...".format(remote, remoteRef, localRef));
347 			git.run("fetch", remote, "+%s:%s".format(remoteRef, localRef));
348 		}
349 		return getRef(localRef);
350 	}
351 
352 	/// Return SHA1 of the given pull request #.
353 	/// Fetches the pull request first, unless offline mode is on.
354 	string getPull(int pull)
355 	{
356 		return getRemoteRef(
357 			"origin",
358 			"refs/pull/%d/head".format(pull),
359 			"refs/digger/pull/%d".format(pull),
360 		);
361 	}
362 
363 	/// Return SHA1 of the given GitHub fork.
364 	/// Fetches the fork first, unless offline mode is on.
365 	/// (This is a thin wrapper around getRemoteBranch.)
366 	string getFork(string user, string branch)
367 	{
368 		enforce(user  .match(re!`^\w[\w\-]*$`), "Bad remote name");
369 		enforce(branch.match(re!`^\w[\w\-\.]*$`), "Bad branch name");
370 
371 		return getRemoteRef(
372 			"https://github.com/%s/%s".format(user, name),
373 			"refs/heads/%s".format(branch),
374 			"refs/digger/fork/%s/%s".format(user, branch),
375 		);
376 	}
377 
378 	/// Find the child of a commit, and, if the commit was a merge,
379 	/// the mainline index of said commit for the child.
380 	void getChild(string branch, string commit, out string child, out int mainline)
381 	{
382 		log("Querying history for commit children...");
383 		auto history = git.getHistory();
384 
385 		bool[Hash] seen;
386 		void visit(Commit* commit)
387 		{
388 			if (commit.hash !in seen)
389 			{
390 				seen[commit.hash] = true;
391 				foreach (parent; commit.parents)
392 					visit(parent);
393 			}
394 		}
395 		auto branchHash = branch.toCommitHash();
396 		auto pBranchCommit = branchHash in history.commits;
397 		enforce(pBranchCommit, "Can't find commit in history");
398 		visit(*pBranchCommit);
399 
400 		auto commitHash = commit.toCommitHash();
401 		auto pCommit = commitHash in history.commits;
402 		enforce(pCommit, "Can't find commit in history");
403 		auto children = (*pCommit).children;
404 		enforce(children.length, "Commit has no children");
405 		children = children.filter!(child => child.hash in seen).array();
406 		enforce(children.length, "Commit has no children under specified branch");
407 		enforce(children.length == 1, "Commit has more than one child");
408 		auto childCommit = children[0];
409 		child = childCommit.hash.toString();
410 
411 		if (childCommit.parents.length == 1)
412 			mainline = 0;
413 		else
414 		{
415 			enforce(childCommit.parents.length == 2, "Can't get mainline of multiple-branch merges");
416 			if (childCommit.parents[0] is *pCommit)
417 				mainline = 2;
418 			else
419 				mainline = 1;
420 
421 			auto mergeInfo = MergeInfo(
422 				childCommit.parents[0].hash.toString(),
423 				childCommit.parents[1].hash.toString(),
424 				true, mainline, commit);
425 			if (!mergeCache.canFind(mergeInfo))
426 			{
427 				mergeCache ~= mergeInfo;
428 				saveMergeCache();
429 			}
430 		}
431 	}
432 
433 	// State saving and checking
434 
435 	struct FileState
436 	{
437 		ulong size;
438 		StdTime modificationTime;
439 	}
440 
441 	FileState getFileState(string file)
442 	{
443 		auto path = git.path.buildPath(file);
444 		auto de = DirEntry(path);
445 		return FileState(de.size, de.timeLastModified.stdTime);
446 	}
447 
448 	alias RepositoryState = FileState[string];
449 
450 	/// Return the working tree "state".
451 	/// This returns a file list, along with size and modification time.
452 	RepositoryState getState()
453 	{
454 		needRepo();
455 		auto files = git.query(["ls-files"]).splitLines();
456 		RepositoryState state;
457 		foreach (file; files)
458 			state[file] = getFileState(file);
459 		return state;
460 	}
461 
462 	private @property string workTreeStatePath()
463 	{
464 		needRepo();
465 		return buildPath(git.gitDir, "ae-sys-d-worktree.json");
466 	}
467 
468 	/// Save the state of the working tree for versioned files
469 	/// to a .json file, which can later be verified with checkState.
470 	/// This should be called after any git command which mutates the git state.
471 	void saveState()
472 	{
473 		std.file.write(workTreeStatePath, getState().toJson());
474 	}
475 
476 	/// Save the state of just one file.
477 	/// This should be called after automatic edits to repository files during a build.
478 	/// The file parameter should be relative to the directory root, and use forward slashes.
479 	void saveFileState(string file)
480 	{
481 		if (!workTreeStatePath.exists)
482 			return;
483 		auto state = workTreeStatePath.readText.jsonParse!RepositoryState();
484 		state[file] = getFileState(file);
485 		std.file.write(workTreeStatePath, state.toJson());
486 	}
487 
488 	/// Verify that the state of the working tree matches the one
489 	/// when saveState was last called. Throw an exception otherwise.
490 	/// This and clearState should be called before any git command
491 	/// which destroys working directory changes.
492 	void checkState()
493 	{
494 		if (!workTreeStatePath.exists)
495 			return;
496 		auto savedState = workTreeStatePath.readText.jsonParse!RepositoryState();
497 		auto currentState = getState();
498 		try
499 		{
500 			foreach (file, fileState; currentState)
501 			{
502 				enforce(file in savedState, "New file: " ~ file);
503 				enforce(savedState[file] == fileState, "File modified: " ~ file);
504 			}
505 		}
506 		catch (Exception e)
507 			throw new Exception(e.msg ~ "\n" ~ "Save / commit your changes, then delete " ~ workTreeStatePath);
508 	}
509 
510 	/// Delete the saved working tree state, if any.
511 	void clearState()
512 	{
513 		if (workTreeStatePath.exists)
514 			workTreeStatePath.remove();
515 	}
516 
517 	// Misc
518 
519 	/// Reset internal state.
520 	protected void reset()
521 	{
522 		currentHead = null;
523 		clean = false;
524 		haveMergeCache = false;
525 		mergeCacheData = null;
526 	}
527 
528 	/// Override to add logging.
529 	protected abstract void log(string line);
530 }
531 
532 /// Used to communicate that a "reset --hard" failed.
533 /// Generally this indicates git repository corruption.
534 mixin DeclareException!q{RepositoryCleanException};