1 /**
2  * Code to manage a D checkout and its dependencies.
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Vladimir Panteleev <vladimir@thecybershadow.net>
12  */
13 
14 module ae.sys.d.manager;
15 
16 import std.algorithm;
17 import std.array;
18 import std.conv;
19 import std.datetime;
20 import std.exception;
21 import std.file;
22 import std.path;
23 import std.process : spawnProcess, wait, escapeShellCommand;
24 import std.range;
25 import std.regex;
26 import std.string;
27 import std.typecons;
28 
29 import ae.sys.d.cache;
30 import ae.sys.d.repo;
31 import ae.sys.file;
32 import ae.sys.git;
33 import ae.utils.aa;
34 import ae.utils.array;
35 import ae.utils.digest;
36 import ae.utils.json;
37 import ae.utils.regex;
38 
39 alias ensureDirExists = ae.sys.file.ensureDirExists;
40 
41 version (Windows)
42 {
43 	import ae.sys.install.dmc;
44 	import ae.sys.install.msys;
45 	import ae.sys.install.vs;
46 
47 	extern(Windows) void SetErrorMode(int);
48 }
49 
50 import ae.sys.install.dmd;
51 import ae.sys.install.git;
52 import ae.sys.install.kindlegen;
53 
54 static import std.process;
55 
56 /// Class which manages a D checkout and its dependencies.
57 class DManager : ICacheHost
58 {
59 	// **************************** Configuration ****************************
60 
61 	struct Config /// DManager configuration.
62 	{
63 		struct Build /// Build configuration
64 		{
65 			struct Components
66 			{
67 				bool[string] enable;
68 
69 				string[] getEnabledComponentNames()
70 				{
71 					foreach (componentName; enable.byKey)
72 						enforce(allComponents.canFind(componentName), "Unknown component: " ~ componentName);
73 					return allComponents
74 						.filter!(componentName =>
75 							enable.get(componentName, defaultComponents.canFind(componentName)))
76 						.array
77 						.dup;
78 				}
79 
80 				Component.CommonConfig common;
81 				DMD.Config dmd;
82 				Website.Config website;
83 			}
84 			Components components;
85 
86 			/// Additional environment variables.
87 			/// Supports %VAR% expansion - see applyEnv.
88 			string[string] environment;
89 		}
90 		Build build; /// ditto
91 
92 		/// Machine-local configuration
93 		/// These settings should not affect the build output.
94 		struct Local
95 		{
96 			/// URL of D git repository hosting D components.
97 			/// Defaults to (and must have the layout of) D.git:
98 			/// https://github.com/CyberShadow/D-dot-git
99 			string repoUrl = "https://bitbucket.org/cybershadow/d.git";
100 
101 			/// Location for the checkout, temporary files, etc.
102 			string workDir;
103 
104 			/// If present, passed to GNU make via -j parameter.
105 			/// Can also be "auto" or "unlimited".
106 			string makeJobs;
107 
108 			/// Don't get latest updates from GitHub.
109 			bool offline;
110 
111 			/// How to cache built files.
112 			string cache;
113 
114 			/// Maximum execution time, in seconds, of any single
115 			/// command.
116 			int timeout;
117 		}
118 		Local local; /// ditto
119 	}
120 	Config config; /// ditto
121 
122 	// Behavior options that generally depend on the host program.
123 
124 	/// Automatically re-clone the repository in case
125 	/// "git reset --hard" fails.
126 	bool autoClean;
127 
128 	/// Whether to verify working tree state
129 	/// to make sure we don't clobber user changes
130 	bool verifyWorkTree;
131 
132 	/// Whether we should cache failed builds.
133 	bool cacheFailures = true;
134 
135 	/// Current build environment.
136 	struct Environment
137 	{
138 		struct Deps /// Configuration for software dependencies
139 		{
140 			string dmcDir;   /// Where dmc.zip is unpacked.
141 			string vsDir;    /// Where Visual Studio is installed
142 			string sdkDir;   /// Where the Windows SDK is installed
143 			string hostDC;   /// Host D compiler (for DDMD bootstrapping)
144 		}
145 		Deps deps; /// ditto
146 
147 		/// Calculated local environment, incl. dependencies
148 		string[string] vars;
149 	}
150 
151 	/// Get a specific subdirectory of the work directory.
152 	@property string subDir(string name)() { return buildPath(config.local.workDir, name); }
153 
154 	alias repoDir    = subDir!"repo";        /// The git repository location.
155 	alias buildDir   = subDir!"build";       /// The build directory.
156 	alias dlDir      = subDir!"dl";          /// The directory for downloaded software.
157 	alias tmpDir     = subDir!"tmp";         /// Directory for $TMPDIR etc.
158 	alias homeDir    = subDir!"home";        /// Directory for $HOME.
159 	alias binDir     = subDir!"bin" ;        /// For wrapper scripts.
160 
161 	/// This number increases with each incompatible change to cached data.
162 	enum cacheVersion = 3;
163 
164 	string cacheEngineDir(string engineName)
165 	{
166 		// Keep compatibility with old cache paths
167 		string engineDirName =
168 			engineName.isOneOf("directory", "true") ? "cache"      :
169 			engineName.isOneOf("", "none", "false") ? "temp-cache" :
170 			"cache-" ~ engineName;
171 		return buildPath(
172 			config.local.workDir,
173 			engineDirName,
174 			"v%d".format(cacheVersion),
175 		);
176 	}
177 
178 	version (Windows)
179 	{
180 		enum string binExt = ".exe";
181 		enum configFileName = "sc.ini";
182 	}
183 	else
184 	{
185 		enum string binExt = "";
186 		enum configFileName = "dmd.conf";
187 	}
188 
189 	static bool needConfSwitch() { return exists(std.process.environment.get("HOME", null).buildPath(configFileName)); }
190 
191 	// **************************** Repositories *****************************
192 
193 	class DManagerRepository : ManagedRepository
194 	{
195 		this()
196 		{
197 			this.offline = config.local.offline;
198 			this.verify = this.outer.verifyWorkTree;
199 		}
200 
201 		override void log(string s) { return this.outer.log(s); }
202 	}
203 
204 	class MetaRepository : DManagerRepository
205 	{
206 		override void needRepo()
207 		{
208 			needGit();
209 
210 			if (!repoDir.exists)
211 			{
212 				log("Cloning initial repository...");
213 				atomic!performClone(config.local.repoUrl, repoDir);
214 			}
215 
216 			if (!git.path)
217 				git = Repository(repoDir);
218 		}
219 
220 		static void performClone(string url, string target)
221 		{
222 			import ae.sys.cmd;
223 			run(["git", "clone", url, target]);
224 		}
225 
226 		override void performCheckout(string hash)
227 		{
228 			super.performCheckout(hash);
229 			submodules = null;
230 		}
231 
232 		string[string][string] submoduleCache;
233 
234 		string[string] getSubmoduleCommits(string head)
235 		{
236 			auto pcacheEntry = head in submoduleCache;
237 			if (pcacheEntry)
238 				return (*pcacheEntry).dup;
239 
240 			string[string] result;
241 			needRepo();
242 			foreach (line; git.query("ls-tree", head).splitLines())
243 			{
244 				auto parts = line.split();
245 				if (parts.length == 4 && parts[1] == "commit")
246 					result[parts[3]] = parts[2];
247 			}
248 			assert(result.length, "No submodules found");
249 			submoduleCache[head] = result;
250 			return result.dup;
251 		}
252 
253 		/// Get the submodule state for all commits in the history.
254 		/// Returns: result[commitHash][submoduleName] == submoduleCommitHash
255 		string[string][string] getSubmoduleHistory(string[] refs)
256 		{
257 			auto marksFile = buildPath(config.local.workDir, "temp", "marks.txt");
258 			ensurePathExists(marksFile);
259 			scope(exit) if (marksFile.exists) marksFile.remove();
260 			log("Running fast-export...");
261 			auto fastExportData = git.query([
262 				"fast-export",
263 				"--full-tree",
264 				"--no-data",
265 				"--export-marks=" ~ marksFile.absolutePath,
266 				] ~ refs
267 			);
268 
269 			log("Parsing fast-export marks...");
270 
271 			auto markLines = marksFile.readText.strip.splitLines;
272 			auto marks = new string[markLines.length];
273 			foreach (line; markLines)
274 			{
275 				auto parts = line.split(' ');
276 				auto markIndex = parts[0][1..$].to!int-1;
277 				marks[markIndex] = parts[1];
278 			}
279 
280 			log("Parsing fast-export data...");
281 
282 			string[string][string] result;
283 			foreach (i, commitData; fastExportData.split("deleteall\n")[1..$])
284 				result[marks[i]] = commitData
285 					.matchAll(re!(`^M 160000 ([0-9a-f]{40}) (\S+)$`, "m"))
286 					.map!(m => tuple(m.captures[2], m.captures[1]))
287 					.assocArray
288 				;
289 			return result;
290 		}
291 	}
292 
293 	class SubmoduleRepository : DManagerRepository
294 	{
295 		string dir;
296 
297 		override void needRepo()
298 		{
299 			getMetaRepo().needRepo();
300 
301 			if (!git.path)
302 				git = Repository(dir);
303 		}
304 
305 		override void needHead(string hash)
306 		{
307 			if (!autoClean)
308 				super.needHead(hash);
309 			else
310 			try
311 				super.needHead(hash);
312 			catch (RepositoryCleanException e)
313 			{
314 				log("Error during repository cleanup.");
315 
316 				log("Nuking %s...".format(dir));
317 				rmdirRecurse(dir);
318 
319 				auto name = baseName(dir);
320 				auto gitDir = buildPath(dirName(dir), ".git", "modules", name);
321 				log("Nuking %s...".format(gitDir));
322 				rmdirRecurse(gitDir);
323 
324 				log("Updating submodule...");
325 				getMetaRepo().git.run(["submodule", "update", name]);
326 
327 				reset();
328 
329 				log("Trying again...");
330 				super.needHead(hash);
331 			}
332 		}
333 	}
334 
335 	/// The meta-repository, which contains the sub-project submodules.
336 	private MetaRepository metaRepo;
337 
338 	MetaRepository getMetaRepo() /// ditto
339 	{
340 		if (!metaRepo)
341 			metaRepo = new MetaRepository;
342 		return metaRepo;
343 	}
344 
345 	/// Sub-project repositories.
346 	private SubmoduleRepository[string] submodules;
347 
348 	ManagedRepository getSubmodule(string name) /// ditto
349 	{
350 		assert(name, "This component is not associated with a submodule");
351 		if (name !in submodules)
352 		{
353 			getMetaRepo().needRepo();
354 			enforce(name in getMetaRepo().getSubmoduleCommits(getMetaRepo().getRef("origin/master")),
355 				"Unknown submodule: " ~ name);
356 
357 			auto path = buildPath(metaRepo.git.path, name);
358 			auto gitPath = buildPath(path, ".git");
359 
360 			if (!gitPath.exists)
361 			{
362 				log("Initializing and updating submodule %s...".format(name));
363 				getMetaRepo().git.run(["submodule", "update", "--init", name]);
364 			}
365 
366 			submodules[name] = new SubmoduleRepository();
367 			submodules[name].dir = path;
368 		}
369 
370 		return submodules[name];
371 	}
372 
373 	// ***************************** Components ******************************
374 
375 	/// Base class for a D component.
376 	class Component
377 	{
378 		/// Name of this component, as registered in DManager.components AA.
379 		string name;
380 
381 		/// Corresponding subproject repository name.
382 		@property abstract string submoduleName();
383 		@property ManagedRepository submodule() { return getSubmodule(submoduleName); }
384 
385 		/// Configuration applicable to multiple (not all) components.
386 		// Note: don't serialize this structure whole!
387 		// Only serialize used fields.
388 		struct CommonConfig
389 		{
390 			version (Windows)
391 				enum defaultModel = "32";
392 			else
393 			version (D_LP64)
394 				enum defaultModel = "64";
395 			else
396 				enum defaultModel = "32";
397 
398 			/// Target comma-separated models ("32", "64", and on Windows, "32mscoff").
399 			/// Controls the models of the built Phobos and Druntime libraries.
400 			string model = defaultModel;
401 
402 			@property string[] models() { return model.split(","); }
403 			@property void models(string[] value) { this.model = value.join(","); }
404 
405 			string[] makeArgs; /// Additional make parameters,
406 			                   /// e.g. "HOST_CC=g++48"
407 		}
408 
409 		/// A string description of this component's configuration.
410 		abstract @property string configString();
411 
412 		/// Commit in the component's repo from which to build this component.
413 		@property string commit() { return incrementalBuild ? "incremental" : getComponentCommit(name); }
414 
415 		/// The components the source code of which this component depends on.
416 		/// Used for calculating the cache key.
417 		@property abstract string[] sourceDependencies();
418 
419 		/// The components the state and configuration of which this component depends on.
420 		/// Used for calculating the cache key.
421 		@property abstract string[] dependencies();
422 
423 		/// This metadata is saved to a .json file,
424 		/// and is also used to calculate the cache key.
425 		struct Metadata
426 		{
427 			int cacheVersion;
428 			string name;
429 			string commit;
430 			string configString;
431 			string[] sourceDepCommits;
432 			Metadata[] dependencyMetadata;
433 		}
434 
435 		Metadata getMetadata() /// ditto
436 		{
437 			return Metadata(
438 				cacheVersion,
439 				name,
440 				commit,
441 				configString,
442 				sourceDependencies.map!(
443 					dependency => getComponent(dependency).commit
444 				).array(),
445 				dependencies.map!(
446 					dependency => getComponent(dependency).getMetadata()
447 				).array(),
448 			);
449 		}
450 
451 		void saveMetaData(string target)
452 		{
453 			std.file.write(buildPath(target, "digger-metadata.json"), getMetadata().toJson());
454 			// Use a separate file to avoid double-encoding JSON
455 			std.file.write(buildPath(target, "digger-config.json"), configString);
456 		}
457 
458 		/// Calculates the cache key, which should be unique and immutable
459 		/// for the same source, build parameters, and build algorithm.
460 		string getBuildID()
461 		{
462 			auto configBlob = getMetadata().toJson() ~ configString;
463 			return "%s-%s-%s".format(
464 				name,
465 				commit,
466 				configBlob.getDigestString!MD5().toLower(),
467 			);
468 		}
469 
470 		@property string sourceDir() { submodule.needRepo(); return submodule.git.path; }
471 
472 		/// Directory to which built files are copied to.
473 		/// This will then be atomically added to the cache.
474 		protected string stageDir;
475 
476 		/// Prepare the source checkout for this component.
477 		/// Usually needed by other components.
478 		void needSource()
479 		{
480 			tempError++; scope(success) tempError--;
481 
482 			if (incrementalBuild)
483 				return;
484 			if (!submoduleName)
485 				return;
486 			foreach (component; getSubmoduleComponents(submoduleName))
487 				component.haveBuild = false;
488 
489 			submodule.needHead(commit);
490 			submodule.clean = false;
491 		}
492 
493 		private bool haveBuild;
494 
495 		/// Build the component in-place, as needed,
496 		/// without moving the built files anywhere.
497 		void needBuild()
498 		{
499 			if (haveBuild) return;
500 			scope(success) haveBuild = true;
501 
502 			log("needBuild: " ~ getBuildID());
503 
504 			needSource();
505 
506 			prepareEnv();
507 
508 			log("Building " ~ getBuildID());
509 			performBuild();
510 			log(getBuildID() ~ " built OK!");
511 		}
512 
513 		/// Set up / clean the build environment.
514 		private void prepareEnv()
515 		{
516 			// Nuke any additional directories cloned by makefiles
517 			if (!incrementalBuild)
518 			{
519 				getMetaRepo().git.run(["clean", "-ffdx"]);
520 
521 				foreach (dir; [tmpDir, homeDir])
522 				{
523 					if (dir.exists && !dir.dirEntries(SpanMode.shallow).empty)
524 						log("Clearing %s ...".format(dir));
525 					dir.recreateEmptyDirectory();
526 				}
527 			}
528 
529 			// Set up compiler wrappers.
530 			recreateEmptyDirectory(binDir);
531 			version (linux)
532 			{
533 				foreach (cc; ["cc", "gcc", "c++", "g++"])
534 				{
535 					auto fileName = binDir.buildPath(cc);
536 					write(fileName, q"EOF
537 #!/bin/sh
538 set -eu
539 
540 tool=$(basename "$0")
541 next=/usr/bin/$tool
542 flagfile=$TMP/nopie-flag-$tool
543 
544 if [ ! -e "$flagfile" ]
545 then
546 	echo 'Testing for -no-pie...' 1>&2
547 	testfile=$TMP/test-$$.c
548 	echo 'int main(){return 0;}' > $testfile
549 	if $next -no-pie -c -o$testfile.o $testfile
550 	then
551 		printf "%s" "-no-pie" > "$flagfile".$$.tmp
552 		mv "$flagfile".$$.tmp "$flagfile"
553 	else
554 		touch "$flagfile"
555 	fi
556 	rm -f "$testfile" "$testfile.o"
557 fi
558 
559 exec "$next" $(cat "$flagfile") "$@"
560 EOF");
561 					setAttributes(fileName, octal!755);
562 				}
563 			}
564 		}
565 
566 		private bool haveInstalled;
567 
568 		/// Build and "install" the component to buildDir as necessary.
569 		void needInstalled()
570 		{
571 			if (haveInstalled) return;
572 			scope(success) haveInstalled = true;
573 
574 			auto buildID = getBuildID();
575 			log("needInstalled: " ~ buildID);
576 
577 			needCacheEngine();
578 			if (cacheEngine.haveEntry(buildID))
579 			{
580 				log("Cache hit!");
581 				if (cacheEngine.listFiles(buildID).canFind(unbuildableMarker))
582 					throw new Exception(buildID ~ " was cached as unbuildable");
583 			}
584 			else
585 			{
586 				log("Cache miss.");
587 
588 				auto tempDir = buildPath(config.local.workDir, "temp");
589 				if (tempDir.exists)
590 					tempDir.removeRecurse();
591 				stageDir = buildPath(tempDir, buildID);
592 				stageDir.mkdirRecurse();
593 
594 				bool failed = false;
595 				tempError = 0;
596 
597 				// Save the results to cache, failed or not
598 				void saveToCache()
599 				{
600 					// Use a separate function to work around
601 					// "cannot put scope(success) statement inside scope(exit)"
602 
603 					int currentTempError = tempError;
604 
605 					// Treat cache errors an environmental errors
606 					// (for when needInstalled is invoked to build a dependency)
607 					tempError++; scope(success) tempError--;
608 
609 					// tempDir might be removed by a dependency's build failure.
610 					if (!tempDir.exists)
611 						log("Not caching %s dependency build failure.".format(name));
612 					else
613 					// Don't cache failed build results due to temporary/environment problems
614 					if (failed && currentTempError > 0)
615 					{
616 						log("Not caching %s build failure due to temporary/environment error.".format(name));
617 						rmdirRecurse(tempDir);
618 					}
619 					else
620 					// Don't cache failed build results during delve
621 					if (failed && !cacheFailures)
622 					{
623 						log("Not caching failed %s build.".format(name));
624 						rmdirRecurse(tempDir);
625 					}
626 					else
627 					if (cacheEngine.haveEntry(buildID))
628 					{
629 						// Can happen due to force==true
630 						log("Already in cache.");
631 						rmdirRecurse(tempDir);
632 					}
633 					else
634 					{
635 						log("Saving to cache.");
636 						saveMetaData(stageDir);
637 						cacheEngine.add(buildID, stageDir);
638 						rmdirRecurse(tempDir);
639 					}
640 				}
641 
642 				scope (exit)
643 					saveToCache();
644 
645 				// An incomplete build is useless, nuke the directory
646 				// and create a new one just for the "unbuildable" marker.
647 				scope (failure)
648 				{
649 					failed = true;
650 					if (stageDir.exists)
651 					{
652 						rmdirRecurse(stageDir);
653 						mkdir(stageDir);
654 						buildPath(stageDir, unbuildableMarker).touch();
655 					}
656 				}
657 
658 				needBuild();
659 
660 				performStage();
661 			}
662 
663 			install();
664 		}
665 
666 		/// Build the component in-place, without moving the built files anywhere.
667 		void performBuild() {}
668 
669 		/// Place resulting files to stageDir
670 		void performStage() {}
671 
672 		/// Update the environment post-install, to allow
673 		/// building components that depend on this one.
674 		void updateEnv(ref Environment env) {}
675 
676 		/// Copy build results from cacheDir to buildDir
677 		void install()
678 		{
679 			log("Installing " ~ getBuildID());
680 			needCacheEngine().extract(getBuildID(), buildDir, de => !de.baseName.startsWith("digger-"));
681 		}
682 
683 		/// Prepare the dependencies then run the component's tests.
684 		void test()
685 		{
686 			log("Testing " ~ getBuildID());
687 
688 			needSource();
689 
690 			submodule.clean = false;
691 			performTest();
692 			log(getBuildID() ~ " tests OK!");
693 		}
694 
695 		/// Run the component's tests.
696 		void performTest() {}
697 
698 	protected final:
699 		// Utility declarations for component implementations
700 
701 		string modelSuffix(string model) { return model == "32" ? "" : model; }
702 		version (Windows)
703 		{
704 			enum string makeFileName = "win32.mak";
705 			string makeFileNameModel(string model)
706 			{
707 				if (model == "32mscoff")
708 					model = "64";
709 				return "win"~model~".mak";
710 			}
711 			enum string binExt = ".exe";
712 		}
713 		else
714 		{
715 			enum string makeFileName = "posix.mak";
716 			string makeFileNameModel(string model) { return "posix.mak"; }
717 			enum string binExt = "";
718 		}
719 
720 		version (Windows)
721 			enum platform = "windows";
722 		else
723 		version (linux)
724 			enum platform = "linux";
725 		else
726 		version (OSX)
727 			enum platform = "osx";
728 		else
729 		version (FreeBSD)
730 			enum platform = "freebsd";
731 		else
732 			static assert(false);
733 
734 		/// Returns the command for the make utility.
735 		string[] getMake(in ref Environment env)
736 		{
737 			return [env.vars.get("MAKE", "make")];
738 		}
739 
740 		/// Returns the path to the built dmd executable.
741 		@property string dmd() { return buildPath(buildDir, "bin", "dmd" ~ binExt).absolutePath(); }
742 
743 		/// Escape a path for d_do_test's very "special" criteria.
744 		/// Spaces must be escaped, but there must be no double-quote at the end.
745 		private static string dDoTestEscape(string str)
746 		{
747 			return str.replaceAll(re!`\\([^\\ ]*? [^\\]*)(?=\\)`, `\"$1"`);
748 		}
749 
750 		unittest
751 		{
752 			assert(dDoTestEscape(`C:\Foo boo bar\baz quuz\derp.exe`) == `C:\"Foo boo bar"\"baz quuz"\derp.exe`);
753 		}
754 
755 		string[] getPlatformMakeVars(in ref Environment env, string model)
756 		{
757 			string[] args;
758 
759 			args ~= "MODEL=" ~ model;
760 
761 			version (Windows)
762 				if (model != "32")
763 				{
764 					args ~= "VCDIR="  ~ env.deps.vsDir.buildPath("VC").absolutePath();
765 					args ~= "SDKDIR=" ~ env.deps.sdkDir.absolutePath();
766 					args ~= "CC=" ~ '"' ~ env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "cl.exe").absolutePath() ~ '"';
767 					args ~= "LD=" ~ '"' ~ env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "link.exe").absolutePath() ~ '"';
768 					args ~= "AR=" ~ '"' ~ env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "lib.exe").absolutePath() ~ '"';
769 				}
770 
771 			return args;
772 		}
773 
774 		@property string[] gnuMakeArgs()
775 		{
776 			string[] args;
777 			if (config.local.makeJobs)
778 			{
779 				if (config.local.makeJobs == "auto")
780 				{
781 					import std.parallelism, std.conv;
782 					args ~= "-j" ~ text(totalCPUs);
783 				}
784 				else
785 				if (config.local.makeJobs == "unlimited")
786 					args ~= "-j";
787 				else
788 					args ~= "-j" ~ config.local.makeJobs;
789 			}
790 			return args;
791 		}
792 
793 		@property string[] dMakeArgs()
794 		{
795 			version (Windows)
796 				return null; // On Windows, DigitalMars make is used for all makefiles except the dmd test suite
797 			else
798 				return gnuMakeArgs;
799 		}
800 
801 		/// Older versions did not use the posix.mak/win32.mak convention.
802 		static string findMakeFile(string dir, string fn)
803 		{
804 			version (OSX)
805 				if (!dir.buildPath(fn).exists && dir.buildPath("osx.mak").exists)
806 					return "osx.mak";
807 			version (Posix)
808 				if (!dir.buildPath(fn).exists && dir.buildPath("linux.mak").exists)
809 					return "linux.mak";
810 			return fn;
811 		}
812 
813 		void needCC(ref Environment env, string model, string dmcVer = null)
814 		{
815 			version (Windows)
816 			{
817 				needDMC(env, dmcVer); // We need DMC even for 64-bit builds (for DM make)
818 				if (model != "32")
819 					needVC(env, model);
820 			}
821 		}
822 
823 		void run(const(string)[] args, in string[string] newEnv, string dir)
824 		{
825 			// Apply user environment
826 			auto env = applyEnv(newEnv, config.build.environment);
827 
828 			// Temporarily apply PATH from newEnv to our process,
829 			// so process creation lookup can use it.
830 			string oldPath = std.process.environment["PATH"];
831 			scope (exit) std.process.environment["PATH"] = oldPath;
832 			std.process.environment["PATH"] = env["PATH"];
833 
834 			// Apply timeout setting
835 			if (config.local.timeout)
836 				args = ["timeout", config.local.timeout.text] ~ args;
837 
838 			foreach (name, value; env)
839 				log("Environment: " ~ name ~ "=" ~ value);
840 			log("Working directory: " ~ dir);
841 			log("Running: " ~ escapeShellCommand(args));
842 
843 			auto status = spawnProcess(args, env, std.process.Config.newEnv, dir).wait();
844 			enforce(status == 0, "Command %s failed with status %d".format(args, status));
845 		}
846 	}
847 
848 	/// The dmd executable
849 	final class DMD : Component
850 	{
851 		@property override string submoduleName  () { return "dmd"; }
852 		@property override string[] sourceDependencies() { return []; }
853 		@property override string[] dependencies() { return []; }
854 
855 		struct Config
856 		{
857 			/// Whether to build a debug DMD.
858 			/// Debug builds are faster to build,
859 			/// but run slower.
860 			@JSONOptional bool debugDMD = false;
861 
862 			/// Whether to build a release DMD.
863 			/// Mutually exclusive with debugDMD.
864 			@JSONOptional bool releaseDMD = false;
865 
866 			/// Model for building DMD itself (on Windows).
867 			/// Can be used to build a 64-bit DMD, to avoid 4GB limit.
868 			@JSONOptional string dmdModel = CommonConfig.defaultModel;
869 
870 			/// How to build DMD versions written in D.
871 			/// We can either download a pre-built binary DMD
872 			/// package, or build an  earlier version from source
873 			/// (e.g. starting with the last C++-only version.)
874 			struct Bootstrap
875 			{
876 				/// Whether to download a pre-built D version,
877 				/// or build one from source. If set, then build
878 				/// from source according to the value of ver,
879 				@JSONOptional bool fromSource = false;
880 
881 				/// Version specification.
882 				/// When building from source, syntax can be defined
883 				/// by outer application (see parseSpec method);
884 				/// When the bootstrapping compiler is not built from source,
885 				/// it is understood as a version number, such as "v2.070.2",
886 				/// which also doubles as a tag name.
887 				/// By default (when set to null), an appropriate version
888 				/// is selected automatically.
889 				@JSONOptional string ver = null;
890 
891 				/// Build configuration for the compiler used for bootstrapping.
892 				/// If not set, then use the default build configuration.
893 				/// Used when fromSource is set.
894 				@JSONOptional DManager.Config.Build* build;
895 			}
896 			@JSONOptional Bootstrap bootstrap; /// ditto
897 
898 			/// Use Visual C++ to build DMD instead of DMC.
899 			/// Currently, this is a hack, as msbuild will consult the system
900 			/// registry and use the system-wide installation of Visual Studio.
901 			/// Only relevant for older versions, as newer versions are written in D.
902 			@JSONOptional bool useVC;
903 		}
904 
905 		@property override string configString()
906 		{
907 			static struct FullConfig
908 			{
909 				Config config;
910 				string[] makeArgs;
911 
912 				// Include the common models as well as the DMD model (from config).
913 				// Necessary to ensure the correct sc.ini is generated on Windows
914 				// (we don't want to pull in MSVC unless either DMD or Phobos are
915 				// built as 64-bit, but also we can't reuse a DMD build with 32-bit
916 				// DMD and Phobos for a 64-bit Phobos build because it won't have
917 				// the VC vars set up in its sc.ini).
918 				// Possibly refactor the compiler configuration to a separate
919 				// component in the future to avoid the inefficiency of rebuilding
920 				// DMD just to generate a different sc.ini.
921 				@JSONOptional string commonModel = Component.CommonConfig.defaultModel;
922 			}
923 
924 			return FullConfig(
925 				config.build.components.dmd,
926 				config.build.components.common.makeArgs,
927 				config.build.components.common.model,
928 			).toJson();
929 		}
930 
931 		@property string vsConfiguration() { return config.build.components.dmd.debugDMD ? "Debug" : "Release"; }
932 		@property string vsPlatform     () { return config.build.components.dmd.dmdModel == "64" ? "x64" : "Win32"; }
933 
934 		override void performBuild()
935 		{
936 			// We need an older DMC for older DMD versions
937 			string dmcVer = null;
938 			auto idgen = buildPath(sourceDir, "src", "idgen.c");
939 			if (idgen.exists && idgen.readText().indexOf(`{ "alignof" },`) >= 0)
940 				dmcVer = "850";
941 
942 			auto env = baseEnvironment;
943 			needCC(env, config.build.components.dmd.dmdModel, dmcVer); // Need VC too for VSINSTALLDIR
944 
945 			if (buildPath(sourceDir, "src", "idgen.d").exists ||
946 			    buildPath(sourceDir, "src", "ddmd", "idgen.d").exists ||
947 			    buildPath(sourceDir, "src", "ddmd", "mars.d").exists ||
948 			    buildPath(sourceDir, "src", "dmd", "mars.d").exists)
949 			{
950 				// Required for bootstrapping.
951 				needDMD(env);
952 				// Go back to our commit.
953 				needSource();
954 				submodule.clean = false;
955 			}
956 
957 			auto srcDir = buildPath(sourceDir, "src");
958 
959 			if (config.build.components.dmd.useVC) // Mostly obsolete, see useVC ddoc
960 			{
961 				version (Windows)
962 				{
963 					needVC(env, config.build.components.dmd.dmdModel);
964 
965 					env.vars["PATH"] = env.vars["PATH"] ~ pathSeparator ~ env.deps.hostDC.dirName;
966 
967 					auto solutionFile = `dmd_msc_vs10.sln`;
968 					if (!exists(srcDir.buildPath(solutionFile)))
969 						solutionFile = `vcbuild\dmd.sln`;
970 					if (!exists(srcDir.buildPath(solutionFile)))
971 						throw new Exception("Can't find Visual Studio solution file");
972 
973 					return run(["msbuild", "/p:Configuration=" ~ vsConfiguration, "/p:Platform=" ~ vsPlatform, solutionFile], env.vars, srcDir);
974 				}
975 				else
976 					throw new Exception("Can only use Visual Studio on Windows");
977 			}
978 
979 			version (Windows)
980 				auto scRoot = env.deps.dmcDir.absolutePath();
981 
982 			string dmdMakeFileName = findMakeFile(srcDir, makeFileName);
983 			string dmdMakeFullName = srcDir.buildPath(dmdMakeFileName);
984 
985 			string modelFlag = config.build.components.dmd.dmdModel;
986 			if (dmdMakeFullName.readText().canFind("MODEL=-m32"))
987 				modelFlag = "-m" ~ modelFlag;
988 
989 			version (Windows)
990 			{
991 				// A make argument is insufficient,
992 				// because of recursive make invocations
993 				auto m = dmdMakeFullName.readText();
994 				m = m
995 					.replace(`CC=\dm\bin\dmc`, `CC=dmc`)
996 					.replace(`SCROOT=$D\dm`, `SCROOT=` ~ scRoot)
997 				;
998 				dmdMakeFullName.write(m);
999 			}
1000 			else
1001 			{
1002 				auto m = dmdMakeFullName.readText();
1003 				m = m
1004 					// Fix hard-coded reference to gcc as linker
1005 					.replace(`gcc -m32 -lstdc++`, `g++ -m32 -lstdc++`)
1006 					.replace(`gcc $(MODEL) -lstdc++`, `g++ $(MODEL) -lstdc++`)
1007 					// Fix compilation of older versions of go.c with GCC 6
1008 					.replace(`-Wno-deprecated`, `-Wno-deprecated -Wno-narrowing`)
1009 				;
1010 				// Fix pthread linker error
1011 				version (linux)
1012 					m = m.replace(`-lpthread`, `-pthread`);
1013 				dmdMakeFullName.write(m);
1014 			}
1015 
1016 			submodule.saveFileState("src/" ~ dmdMakeFileName);
1017 
1018 			// Fix compilation error of older DMDs with glibc >= 2.25
1019 			version (linux)
1020 			{{
1021 				auto fn = srcDir.buildPath("root", "port.c");
1022 				if (fn.exists)
1023 				{
1024 					fn.write(fn.readText
1025 						.replace(`#include <bits/mathdef.h>`, `#include <complex.h>`)
1026 					);
1027 					submodule.saveFileState(fn.relativePath(sourceDir));
1028 				}
1029 			}}
1030 
1031 			// Fix alignment issue in older DMDs with GCC >= 7
1032 			// See https://issues.dlang.org/show_bug.cgi?id=17726
1033 			version (Posix)
1034 			{
1035 				foreach (fn; [srcDir.buildPath("tk", "mem.c"), srcDir.buildPath("ddmd", "tk", "mem.c")])
1036 					if (fn.exists)
1037 					{
1038 						fn.write(fn.readText.replace(
1039 								// `#if defined(__llvm__) && (defined(__GNUC__) || defined(__clang__))`,
1040 								// `#if defined(__GNUC__) || defined(__clang__)`,
1041 								`numbytes = (numbytes + 3) & ~3;`,
1042 								`numbytes = (numbytes + 0xF) & ~0xF;`
1043 						));
1044 						submodule.saveFileState(fn.relativePath(sourceDir));
1045 					}
1046 			}
1047 
1048 			string[] extraArgs, targets;
1049 			version (Posix)
1050 			{
1051 				if (config.build.components.dmd.debugDMD)
1052 					extraArgs ~= "DEBUG=1";
1053 				if (config.build.components.dmd.releaseDMD)
1054 					extraArgs ~= "ENABLE_RELEASE=1";
1055 			}
1056 			else
1057 			{
1058 				if (config.build.components.dmd.debugDMD)
1059 					targets ~= [];
1060 				else
1061 				if (config.build.components.dmd.releaseDMD && dmdMakeFullName.readText().canFind("reldmd"))
1062 					targets ~= ["reldmd"];
1063 				else
1064 					targets ~= ["dmd"];
1065 			}
1066 
1067 			version (Windows)
1068 			{
1069 				if (config.build.components.dmd.dmdModel != CommonConfig.defaultModel)
1070 				{
1071 					dmdMakeFileName = "win64.mak";
1072 					dmdMakeFullName = srcDir.buildPath(dmdMakeFileName);
1073 					enforce(dmdMakeFullName.exists, "dmdModel not supported for this DMD version");
1074 					extraArgs ~= "DMODEL=-m" ~ config.build.components.dmd.dmdModel;
1075 					if (config.build.components.dmd.dmdModel == "32mscoff")
1076 					{
1077 						auto objFiles = dmdMakeFullName.readText().splitLines().filter!(line => line.startsWith("OBJ_MSVC="));
1078 						enforce(!objFiles.empty, "Can't find OBJ_MSVC in win64.mak");
1079 						extraArgs ~= "OBJ_MSVC=" ~ objFiles.front.findSplit("=")[2].split().filter!(obj => obj != "ldfpu.obj").join(" ");
1080 					}
1081 				}
1082 			}
1083 
1084 			// Avoid HOST_DC reading ~/dmd.conf
1085 			string hostDC = env.deps.hostDC;
1086 			version (Posix)
1087 			if (hostDC && needConfSwitch())
1088 			{
1089 				auto dcProxy = buildPath(config.local.workDir, "host-dc-proxy.sh");
1090 				std.file.write(dcProxy, escapeShellCommand(["exec", hostDC, "-conf=" ~ buildPath(dirName(hostDC), configFileName)]) ~ ` "$@"`);
1091 				setAttributes(dcProxy, octal!755);
1092 				hostDC = dcProxy;
1093 			}
1094 
1095 			run(getMake(env) ~ [
1096 					"-f", dmdMakeFileName,
1097 					"MODEL=" ~ modelFlag,
1098 					"HOST_DC=" ~ hostDC,
1099 				] ~ config.build.components.common.makeArgs ~ dMakeArgs ~ extraArgs ~ targets,
1100 				env.vars, srcDir
1101 			);
1102 		}
1103 
1104 		override void performStage()
1105 		{
1106 			if (config.build.components.dmd.useVC)
1107 			{
1108 				foreach (ext; [".exe", ".pdb"])
1109 					cp(
1110 						buildPath(sourceDir, "src", "vcbuild", vsPlatform, vsConfiguration, "dmd_msc" ~ ext),
1111 						buildPath(stageDir , "bin", "dmd" ~ ext),
1112 					);
1113 			}
1114 			else
1115 			{
1116 				string dmdPath = buildPath(sourceDir, "generated", platform, "release", config.build.components.dmd.dmdModel, "dmd" ~ binExt);
1117 				if (!dmdPath.exists)
1118 					dmdPath = buildPath(sourceDir, "src", "dmd" ~ binExt); // legacy
1119 				enforce(dmdPath.exists && dmdPath.isFile, "Can't find built DMD executable");
1120 
1121 				cp(
1122 					dmdPath,
1123 					buildPath(stageDir , "bin", "dmd" ~ binExt),
1124 				);
1125 			}
1126 
1127 			version (Windows)
1128 			{
1129 				auto env = baseEnvironment;
1130 				needCC(env, config.build.components.dmd.dmdModel);
1131 				foreach (model; config.build.components.common.models)
1132 					needCC(env, model);
1133 
1134 				auto ini = q"EOS
1135 [Environment]
1136 LIB="%@P%\..\lib"
1137 DFLAGS="-I%@P%\..\import"
1138 DMC=__DMC__
1139 LINKCMD=%DMC%\link.exe
1140 EOS"
1141 				.replace("__DMC__", env.deps.dmcDir.buildPath(`bin`).absolutePath())
1142 			;
1143 
1144 				if (env.deps.vsDir && env.deps.sdkDir)
1145 				{
1146 					ini ~= q"EOS
1147 
1148 [Environment64]
1149 LIB="%@P%\..\lib"
1150 DFLAGS=%DFLAGS% -L/OPT:NOICF
1151 VSINSTALLDIR=__VS__\
1152 VCINSTALLDIR=%VSINSTALLDIR%VC\
1153 PATH=%PATH%;%VCINSTALLDIR%\bin\__MODELDIR__;%VCINSTALLDIR%\bin
1154 WindowsSdkDir=__SDK__
1155 LINKCMD=%VCINSTALLDIR%\bin\__MODELDIR__\link.exe
1156 LIB=%LIB%;"%VCINSTALLDIR%\lib\amd64"
1157 LIB=%LIB%;"%WindowsSdkDir%\Lib\x64"
1158 
1159 [Environment32mscoff]
1160 LIB="%@P%\..\lib"
1161 DFLAGS=%DFLAGS% -L/OPT:NOICF
1162 VSINSTALLDIR=__VS__\
1163 VCINSTALLDIR=%VSINSTALLDIR%VC\
1164 PATH=%PATH%;%VCINSTALLDIR%\bin
1165 WindowsSdkDir=__SDK__
1166 LINKCMD=%VCINSTALLDIR%\bin\link.exe
1167 LIB=%LIB%;"%VCINSTALLDIR%\lib"
1168 LIB=%LIB%;"%WindowsSdkDir%\Lib"
1169 EOS"
1170 						.replace("__VS__"      , env.deps.vsDir .absolutePath())
1171 						.replace("__SDK__"     , env.deps.sdkDir.absolutePath())
1172 						.replace("__MODELDIR__", msvcModelDir("64"))
1173 					;
1174 				}
1175 
1176 				buildPath(stageDir, "bin", configFileName).write(ini);
1177 			}
1178 			else version (OSX)
1179 			{
1180 				auto ini = q"EOS
1181 [Environment]
1182 DFLAGS="-I%@P%/../import" "-L-L%@P%/../lib"
1183 EOS";
1184 				buildPath(stageDir, "bin", configFileName).write(ini);
1185 			}
1186 			else
1187 			{
1188 				auto ini = q"EOS
1189 [Environment]
1190 DFLAGS="-I%@P%/../import" "-L-L%@P%/../lib" -L--export-dynamic
1191 EOS";
1192 				buildPath(stageDir, "bin", configFileName).write(ini);
1193 			}
1194 		}
1195 
1196 		override void updateEnv(ref Environment env)
1197 		{
1198 			// Add the DMD we built for Phobos/Druntime/Tools
1199 			env.vars["PATH"] = buildPath(buildDir, "bin").absolutePath() ~ pathSeparator ~ env.vars["PATH"];
1200 		}
1201 
1202 		override void performTest()
1203 		{
1204 			foreach (dep; ["dmd", "druntime", "phobos"])
1205 				getComponent(dep).needBuild();
1206 
1207 			foreach (model; config.build.components.common.models)
1208 			{
1209 				auto env = baseEnvironment;
1210 				version (Windows)
1211 				{
1212 					// In this order so it uses the MSYS make
1213 					needCC(env, model);
1214 					needMSYS(env);
1215 
1216 					disableCrashDialog();
1217 				}
1218 
1219 				auto makeArgs = getMake(env) ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env, model) ~ gnuMakeArgs;
1220 				version (Windows)
1221 				{
1222 					makeArgs ~= ["OS=win" ~ model[0..2], "SHELL=bash"];
1223 					if (model == "32")
1224 					{
1225 						auto extrasDir = needExtras();
1226 						// The autotester seems to pass this via environment. Why does that work there???
1227 						makeArgs ~= "LIB=" ~ extrasDir.buildPath("localextras-windows", "dmd2", "windows", "lib") ~ `;..\..\phobos`;
1228 					}
1229 					else
1230 					{
1231 						// Fix path for d_do_test and its special escaping (default is the system VS2010 install)
1232 						// We can't use the same syntax in getPlatformMakeVars because win64.mak uses "CC=\$(CC32)"\""
1233 						auto cl = env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "cl.exe");
1234 						foreach (ref arg; makeArgs)
1235 							if (arg.startsWith("CC="))
1236 								arg = "CC=" ~ dDoTestEscape(cl);
1237 					}
1238 				}
1239 
1240 				version (test)
1241 				{
1242 					// Only try a few tests during CI runs, to check for
1243 					// platform integration and correct invocation.
1244 					// For this purpose, the C++ ABI tests will do nicely.
1245 					makeArgs ~= [
1246 					//	"test_results/runnable/cppa.d.out", // https://github.com/dlang/dmd/pull/5686
1247 						"test_results/runnable/cpp_abi_tests.d.out",
1248 						"test_results/runnable/cabi1.d.out",
1249 					];
1250 				}
1251 
1252 				run(makeArgs, env.vars, sourceDir.buildPath("test"));
1253 			}
1254 		}
1255 	}
1256 
1257 	/// Phobos import files.
1258 	/// In older versions of D, Druntime depended on Phobos modules.
1259 	final class PhobosIncludes : Component
1260 	{
1261 		@property override string submoduleName() { return "phobos"; }
1262 		@property override string[] sourceDependencies() { return []; }
1263 		@property override string[] dependencies() { return []; }
1264 		@property override string configString() { return null; }
1265 
1266 		override void performStage()
1267 		{
1268 			foreach (f; ["std", "etc", "crc32.d"])
1269 				if (buildPath(sourceDir, f).exists)
1270 					cp(
1271 						buildPath(sourceDir, f),
1272 						buildPath(stageDir , "import", f),
1273 					);
1274 		}
1275 	}
1276 
1277 	/// Druntime. Installs only import files, but builds the library too.
1278 	final class Druntime : Component
1279 	{
1280 		@property override string submoduleName    () { return "druntime"; }
1281 		@property override string[] sourceDependencies() { return ["phobos", "phobos-includes"]; }
1282 		@property override string[] dependencies() { return ["dmd"]; }
1283 
1284 		@property override string configString()
1285 		{
1286 			static struct FullConfig
1287 			{
1288 				string model;
1289 				string[] makeArgs;
1290 			}
1291 
1292 			return FullConfig(
1293 				config.build.components.common.model,
1294 				config.build.components.common.makeArgs,
1295 			).toJson();
1296 		}
1297 
1298 		override void performBuild()
1299 		{
1300 			getComponent("phobos").needSource();
1301 			getComponent("dmd").needSource();
1302 			getComponent("dmd").needInstalled();
1303 			getComponent("phobos-includes").needInstalled();
1304 
1305 			foreach (model; config.build.components.common.models)
1306 			{
1307 				auto env = baseEnvironment;
1308 				needCC(env, model);
1309 
1310 				mkdirRecurse(sourceDir.buildPath("import"));
1311 				mkdirRecurse(sourceDir.buildPath("lib"));
1312 
1313 				setTimes(sourceDir.buildPath("src", "rt", "minit.obj"), Clock.currTime(), Clock.currTime()); // Don't rebuild
1314 				submodule.saveFileState("src/rt/minit.obj");
1315 
1316 				run(getMake(env) ~ ["-f", makeFileNameModel(model), "import", "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env, model) ~ dMakeArgs, env.vars, sourceDir);
1317 				run(getMake(env) ~ ["-f", makeFileNameModel(model)          , "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env, model) ~ dMakeArgs, env.vars, sourceDir);
1318 			}
1319 		}
1320 
1321 		override void performStage()
1322 		{
1323 			cp(
1324 				buildPath(sourceDir, "import"),
1325 				buildPath(stageDir , "import"),
1326 			);
1327 		}
1328 
1329 		override void performTest()
1330 		{
1331 			getComponent("druntime").needBuild();
1332 			getComponent("dmd").needInstalled();
1333 
1334 			foreach (model; config.build.components.common.models)
1335 			{
1336 				auto env = baseEnvironment;
1337 				needCC(env, model);
1338 				run(getMake(env) ~ ["-f", makeFileNameModel(model), "unittest", "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env, model) ~ dMakeArgs, env.vars, sourceDir);
1339 			}
1340 		}
1341 	}
1342 
1343 	/// Phobos library and imports.
1344 	final class Phobos : Component
1345 	{
1346 		@property override string submoduleName    () { return "phobos"; }
1347 		@property override string[] sourceDependencies() { return []; }
1348 		@property override string[] dependencies() { return ["druntime", "dmd"]; }
1349 
1350 		@property override string configString()
1351 		{
1352 			static struct FullConfig
1353 			{
1354 				string model;
1355 				string[] makeArgs;
1356 			}
1357 
1358 			return FullConfig(
1359 				config.build.components.common.model,
1360 				config.build.components.common.makeArgs,
1361 			).toJson();
1362 		}
1363 
1364 		string[] targets;
1365 
1366 		override void performBuild()
1367 		{
1368 			getComponent("dmd").needSource();
1369 			getComponent("dmd").needInstalled();
1370 			getComponent("druntime").needBuild();
1371 
1372 			targets = null;
1373 
1374 			foreach (model; config.build.components.common.models)
1375 			{
1376 				// Clean up old object files with mismatching model.
1377 				// Necessary for a consecutive 32/64 build.
1378 				version (Windows)
1379 				{
1380 					foreach (de; dirEntries(sourceDir.buildPath("etc", "c", "zlib"), "*.obj", SpanMode.shallow))
1381 					{
1382 						auto data = cast(ubyte[])read(de.name);
1383 
1384 						string fileModel;
1385 						if (data.length < 4)
1386 							fileModel = "invalid";
1387 						else
1388 						if (data[0] == 0x80)
1389 							fileModel = "32"; // OMF
1390 						else
1391 						if (data[0] == 0x01 && data[0] == 0x4C)
1392 							fileModel = "32mscoff"; // COFF - IMAGE_FILE_MACHINE_I386
1393 						else
1394 						if (data[0] == 0x86 && data[0] == 0x64)
1395 							fileModel = "64"; // COFF - IMAGE_FILE_MACHINE_AMD64
1396 						else
1397 							fileModel = "unknown";
1398 
1399 						if (fileModel != model)
1400 						{
1401 							log("Cleaning up object file '%s' with mismatching model (file is %s, building %s)".format(de.name, fileModel, model));
1402 							remove(de.name);
1403 						}
1404 					}
1405 				}
1406 
1407 				auto env = baseEnvironment;
1408 				needCC(env, model);
1409 
1410 				string phobosMakeFileName = findMakeFile(sourceDir, makeFileNameModel(model));
1411 				string phobosMakeFullName = sourceDir.buildPath(phobosMakeFileName);
1412 
1413 				auto makeArgs = getMake(env) ~ ["-f", phobosMakeFileName, "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env, model) ~ dMakeArgs;
1414 
1415 				version (Windows)
1416 				{
1417 					auto lib = "phobos%s.lib".format(modelSuffix(model));
1418 					run(makeArgs ~ lib, env.vars, sourceDir);
1419 					enforce(sourceDir.buildPath(lib).exists);
1420 					targets ~= ["phobos%s.lib".format(modelSuffix(model))];
1421 				}
1422 				else
1423 				{
1424 					if (phobosMakeFullName.readText().canFind("DRUNTIME = $(DRUNTIME_PATH)/lib/libdruntime-$(OS)$(MODEL).a") &&
1425 						getComponent("druntime").sourceDir.buildPath("lib").dirEntries(SpanMode.shallow).walkLength == 0 &&
1426 						exists(getComponent("druntime").sourceDir.buildPath("generated")))
1427 					{
1428 						auto dir = getComponent("druntime").sourceDir.buildPath("generated");
1429 						auto aFile  = dir.dirEntries("libdruntime.a", SpanMode.depth);
1430 						if (!aFile .empty) makeArgs ~= ["DRUNTIME="   ~ aFile .front];
1431 						auto soFile = dir.dirEntries("libdruntime.so.a", SpanMode.depth);
1432 						if (!soFile.empty) makeArgs ~= ["DRUNTIMESO=" ~ soFile.front];
1433 					}
1434 					run(makeArgs, env.vars, sourceDir);
1435 					targets ~= sourceDir
1436 						.buildPath("generated")
1437 						.dirEntries(SpanMode.depth)
1438 						.filter!(de => de.name.endsWith(".a") || de.name.endsWith(".so"))
1439 						.map!(de => de.name.relativePath(sourceDir))
1440 						.array()
1441 					;
1442 				}
1443 			}
1444 		}
1445 
1446 		override void performStage()
1447 		{
1448 			assert(targets.length, "Druntime stage without build");
1449 			foreach (lib; targets)
1450 				cp(
1451 					buildPath(sourceDir, lib),
1452 					buildPath(stageDir , "lib", lib.baseName()),
1453 				);
1454 		}
1455 
1456 		override void performTest()
1457 		{
1458 			getComponent("druntime").needBuild();
1459 			getComponent("phobos").needBuild();
1460 			getComponent("dmd").needInstalled();
1461 
1462 			foreach (model; config.build.components.common.models)
1463 			{
1464 				auto env = baseEnvironment;
1465 				needCC(env, model);
1466 				version (Windows)
1467 				{
1468 					getComponent("curl").needInstalled();
1469 					getComponent("curl").updateEnv(env);
1470 
1471 					// Patch out std.datetime unittest to work around Digger test
1472 					// suite failure on AppVeyor due to Windows time zone changes
1473 					auto stdDateTime = buildPath(sourceDir, "std", "datetime.d");
1474 					if (stdDateTime.exists && !stdDateTime.readText().canFind("Altai Standard Time"))
1475 					{
1476 						auto m = stdDateTime.readText();
1477 						m = m
1478 							.replace(`assert(tzName !is null, format("TZName which is missing: %s", winName));`, ``)
1479 							.replace(`assert(tzDatabaseNameToWindowsTZName(tzName) !is null, format("TZName which failed: %s", tzName));`, `{}`)
1480 							.replace(`assert(windowsTZNameToTZDatabaseName(tzName) !is null, format("TZName which failed: %s", tzName));`, `{}`)
1481 						;
1482 						stdDateTime.write(m);
1483 						submodule.saveFileState("std/datetime.d");
1484 					}
1485 
1486 					if (model == "32")
1487 						getComponent("extras").needInstalled();
1488 				}
1489 				run(getMake(env) ~ ["-f", makeFileNameModel(model), "unittest", "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env, model) ~ dMakeArgs, env.vars, sourceDir);
1490 			}
1491 		}
1492 	}
1493 
1494 	/// The rdmd build tool by itself.
1495 	/// It predates the tools package.
1496 	final class RDMD : Component
1497 	{
1498 		@property override string submoduleName() { return "tools"; }
1499 		@property override string[] sourceDependencies() { return []; }
1500 		@property override string[] dependencies() { return ["dmd", "druntime", "phobos"]; }
1501 
1502 		@property string model() { return config.build.components.common.models.get(0); }
1503 
1504 		@property override string configString()
1505 		{
1506 			static struct FullConfig
1507 			{
1508 				string model;
1509 			}
1510 
1511 			return FullConfig(
1512 				this.model,
1513 			).toJson();
1514 		}
1515 
1516 		override void performBuild()
1517 		{
1518 			foreach (dep; ["dmd", "druntime", "phobos", "phobos-includes"])
1519 				getComponent(dep).needInstalled();
1520 
1521 			auto env = baseEnvironment;
1522 			needCC(env, this.model);
1523 
1524 			// Just build rdmd
1525 			bool needModel; // Need -mXX switch?
1526 
1527 			if (sourceDir.buildPath("posix.mak").exists)
1528 				needModel = true; // Known to be needed for recent versions
1529 
1530 			string[] args;
1531 			if (needConfSwitch())
1532 				args ~= ["-conf=" ~ buildPath(buildDir , "bin", configFileName)];
1533 			args ~= ["rdmd"];
1534 
1535 			if (!needModel)
1536 				try
1537 					run([dmd] ~ args, env.vars, sourceDir);
1538 				catch (Exception e)
1539 					needModel = true;
1540 
1541 			if (needModel)
1542 				run([dmd, "-m" ~ this.model] ~ args, env.vars, sourceDir);
1543 		}
1544 
1545 		override void performStage()
1546 		{
1547 			cp(
1548 				buildPath(sourceDir, "rdmd" ~ binExt),
1549 				buildPath(stageDir , "bin", "rdmd" ~ binExt),
1550 			);
1551 		}
1552 
1553 		override void performTest()
1554 		{
1555 			version (Windows)
1556 				if (this.model != "32")
1557 				{
1558 					// Can't test rdmd on non-32-bit Windows until compiler model matches Phobos model.
1559 					// rdmd_test does not use -m when building rdmd, thus linking will fail
1560 					// (because of model mismatch with the phobos we built).
1561 					log("Can't test rdmd with model " ~ this.model ~ ", skipping");
1562 					return;
1563 				}
1564 
1565 			foreach (dep; ["dmd", "druntime", "phobos", "phobos-includes"])
1566 				getComponent(dep).needInstalled();
1567 
1568 			auto env = baseEnvironment;
1569 			getComponent("dmd").updateEnv(env);
1570 			run(["dmd", "-run", "rdmd_test.d"], env.vars, sourceDir);
1571 		}
1572 	}
1573 
1574 	/// Tools package with all its components, including rdmd.
1575 	final class Tools : Component
1576 	{
1577 		@property override string submoduleName() { return "tools"; }
1578 		@property override string[] sourceDependencies() { return []; }
1579 		@property override string[] dependencies() { return ["dmd", "druntime", "phobos"]; }
1580 
1581 		@property string model() { return config.build.components.common.models.get(0); }
1582 
1583 		@property override string configString()
1584 		{
1585 			static struct FullConfig
1586 			{
1587 				string model;
1588 				string[] makeArgs;
1589 			}
1590 
1591 			return FullConfig(
1592 				this.model,
1593 				config.build.components.common.makeArgs,
1594 			).toJson();
1595 		}
1596 
1597 		override void performBuild()
1598 		{
1599 			getComponent("dmd").needSource();
1600 			foreach (dep; ["dmd", "druntime", "phobos"])
1601 				getComponent(dep).needInstalled();
1602 
1603 			auto env = baseEnvironment;
1604 			needCC(env, this.model);
1605 
1606 			run(getMake(env) ~ ["-f", makeFileName, "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env, this.model) ~ dMakeArgs, env.vars, sourceDir);
1607 		}
1608 
1609 		override void performStage()
1610 		{
1611 			foreach (os; buildPath(sourceDir, "generated").dirEntries(SpanMode.shallow))
1612 			{
1613 				auto dir = os.buildPath(this.model);
1614 				cp(dir, buildPath(stageDir , "bin"));
1615 			}
1616 		}
1617 	}
1618 
1619 	/// Website (dlang.org). Only buildable on POSIX.
1620 	final class Website : Component
1621 	{
1622 		@property override string submoduleName() { return "dlang.org"; }
1623 		@property override string[] sourceDependencies() { return ["druntime", "phobos"]; }
1624 		@property override string[] dependencies() { return ["dmd", "druntime", "phobos", "rdmd"]; }
1625 
1626 		struct Config
1627 		{
1628 			/// Do not include timestamps, line numbers, or other
1629 			/// volatile dynamic content in generated .ddoc files.
1630 			/// Improves cache efficiency and allows meaningful diffs.
1631 			bool diffable = false;
1632 
1633 			deprecated alias noDateTime = diffable;
1634 		}
1635 
1636 		@property override string configString()
1637 		{
1638 			static struct FullConfig
1639 			{
1640 				Config config;
1641 			}
1642 
1643 			return FullConfig(
1644 				config.build.components.website,
1645 			).toJson();
1646 		}
1647 
1648 		/// Get the latest version of DMD at the time.
1649 		/// Needed for the makefile's "LATEST" parameter.
1650 		string getLatest()
1651 		{
1652 			auto dmd = getComponent("dmd").submodule;
1653 			dmd.needRepo();
1654 
1655 			auto t = dmd.git.query(["log", "--pretty=format:%ct"]).splitLines.map!(to!int).filter!(n => n > 0).front;
1656 
1657 			foreach (line; dmd.git.query(["log", "--decorate=full", "--tags", "--pretty=format:%ct%d"]).splitLines())
1658 				if (line.length > 10 && line[0..10].to!int < t)
1659 					if (line[10..$].startsWith(" (") && line.endsWith(")"))
1660 					{
1661 						foreach (r; line[12..$-1].split(", "))
1662 							if (r.skipOver("tag: refs/tags/"))
1663 								if (r.match(re!`^v2\.\d\d\d(\.\d)?$`))
1664 									return r[1..$];
1665 					}
1666 			throw new Exception("Can't find any DMD version tags at this point!");
1667 		}
1668 
1669 		private enum Target { build, test }
1670 
1671 		private void make(Target target)
1672 		{
1673 			auto env = baseEnvironment;
1674 
1675 			version (Windows)
1676 				throw new Exception("The dlang.org website is only buildable on POSIX platforms.");
1677 			else
1678 			{
1679 				getComponent("dmd").updateEnv(env);
1680 
1681 				needKindleGen(env);
1682 
1683 				foreach (dep; dependencies)
1684 					getComponent(dep).submodule.clean = false;
1685 
1686 				auto makeFullName = sourceDir.buildPath(makeFileName);
1687 				makeFullName
1688 					.readText()
1689 					// https://github.com/D-Programming-Language/dlang.org/pull/1011
1690 					.replace(": modlist.d", ": modlist.d $(DMD)")
1691 					// https://github.com/D-Programming-Language/dlang.org/pull/1017
1692 					.replace("dpl-docs: ${DUB} ${STABLE_DMD}\n\tDFLAGS=", "dpl-docs: ${DUB} ${STABLE_DMD}\n\t${DUB} upgrade --missing-only --root=${DPL_DOCS_PATH}\n\tDFLAGS=")
1693 					.toFile(makeFullName)
1694 				;
1695 				submodule.saveFileState(makeFileName);
1696 
1697 				// Retroactive OpenSSL 1.1.0 fix
1698 				// See https://github.com/dlang/dlang.org/pull/1654
1699 				auto dubJson = sourceDir.buildPath("dpl-docs/dub.json");
1700 				dubJson
1701 					.readText()
1702 					.replace(`"versions": ["VibeCustomMain"]`, `"versions": ["VibeCustomMain", "VibeNoSSL"]`)
1703 					.toFile(dubJson);
1704 				submodule.saveFileState("dpl-docs/dub.json");
1705 				scope(exit) submodule.saveFileState("dpl-docs/dub.selections.json");
1706 
1707 				string latest = null;
1708 				if (!sourceDir.buildPath("VERSION").exists)
1709 				{
1710 					latest = getLatest();
1711 					log("LATEST=" ~ latest);
1712 				}
1713 				else
1714 					log("VERSION file found, not passing LATEST parameter");
1715 
1716 				string[] diffable = null;
1717 
1718 				string[] targets =
1719 					[
1720 						config.build.components.website.diffable
1721 						? ["all", "verbatim", "pdf"] ~ (
1722 							makeFullName.readText.indexOf("diffable-intermediaries") >= 0
1723 							? ["diffable-intermediaries"]
1724 							: ["dlangspec.html"])
1725 						: ["all", "verbatim", "pdf", "kindle"],
1726 						["test"]
1727 					][target];
1728 
1729 				if (config.build.components.website.diffable)
1730 				{
1731 					if (makeFullName.readText.indexOf("DIFFABLE") >= 0)
1732 						diffable = ["DIFFABLE=1"];
1733 					else
1734 						diffable = ["NODATETIME=nodatetime.ddoc"];
1735 
1736 					env.vars["SOURCE_DATE_EPOCH"] = "0";
1737 				}
1738 
1739 				auto args =
1740 					getMake(env) ~
1741 					[ "-f", makeFileName ] ~
1742 					diffable ~
1743 					(latest ? ["LATEST=" ~ latest] : []) ~
1744 					targets ~
1745 					gnuMakeArgs;
1746 				run(args, env.vars, sourceDir);
1747 			}
1748 		}
1749 
1750 		override void performBuild()
1751 		{
1752 			foreach (dep; ["dmd", "druntime", "phobos"])
1753 			{
1754 				auto c = getComponent(dep);
1755 				c.needInstalled();
1756 
1757 				// Need DMD source because https://github.com/dlang/phobos/pull/4613#issuecomment-266462596
1758 				// Need Druntime/Phobos source because we are building its documentation from there.
1759 				c.needSource();
1760 			}
1761 			getComponent("tools").needSource(); // for changed.d
1762 
1763 			make(Target.build);
1764 		}
1765 
1766 		override void performTest()
1767 		{
1768 			make(Target.test);
1769 		}
1770 
1771 		override void performStage()
1772 		{
1773 			foreach (item; ["web", "dlangspec.tex", "dlangspec.html"])
1774 			{
1775 				auto src = buildPath(sourceDir, item);
1776 				auto dst = buildPath(stageDir , item);
1777 				if (src.exists)
1778 					cp(src, dst);
1779 			}
1780 		}
1781 	}
1782 
1783 	/// Extras not built from source (DigitalMars and third-party tools and libraries)
1784 	final class Extras : Component
1785 	{
1786 		@property override string submoduleName() { return null; }
1787 		@property override string[] sourceDependencies() { return []; }
1788 		@property override string[] dependencies() { return []; }
1789 		@property override string configString() { return null; }
1790 
1791 		override void performBuild()
1792 		{
1793 			needExtras();
1794 		}
1795 
1796 		override void performStage()
1797 		{
1798 			auto extrasDir = needExtras();
1799 
1800 			void copyDir(string source, string target)
1801 			{
1802 				source = buildPath(extrasDir, "localextras-" ~ platform, "dmd2", platform, source);
1803 				target = buildPath(stageDir, target);
1804 				if (source.exists)
1805 					cp(source, target);
1806 			}
1807 
1808 			copyDir("bin", "bin");
1809 			foreach (model; config.build.components.common.models)
1810 				copyDir("bin" ~ model, "bin");
1811 			copyDir("lib", "lib");
1812 
1813 			version (Windows)
1814 				foreach (model; config.build.components.common.models)
1815 					if (model == "32")
1816 					{
1817 						// The version of snn.lib bundled with DMC will be newer.
1818 						Environment env;
1819 						needDMC(env);
1820 						cp(buildPath(env.deps.dmcDir, "lib", "snn.lib"), buildPath(stageDir, "lib", "snn.lib"));
1821 					}
1822 		}
1823 	}
1824 
1825 	/// libcurl DLL and import library for Windows.
1826 	final class Curl : Component
1827 	{
1828 		@property override string submoduleName() { return null; }
1829 		@property override string[] sourceDependencies() { return []; }
1830 		@property override string[] dependencies() { return []; }
1831 		@property override string configString() { return null; }
1832 
1833 		override void performBuild()
1834 		{
1835 			version (Windows)
1836 				needCurl();
1837 			else
1838 				log("Not on Windows, skipping libcurl download");
1839 		}
1840 
1841 		override void performStage()
1842 		{
1843 			version (Windows)
1844 			{
1845 				auto curlDir = needCurl();
1846 
1847 				void copyDir(string source, string target)
1848 				{
1849 					source = buildPath(curlDir, "dmd2", "windows", source);
1850 					target = buildPath(stageDir, target);
1851 					if (source.exists)
1852 						cp(source, target);
1853 				}
1854 
1855 				foreach (model; config.build.components.common.models)
1856 				{
1857 					auto suffix = model == "64" ? "64" : "";
1858 					copyDir("bin" ~ suffix, "bin");
1859 					copyDir("lib" ~ suffix, "lib");
1860 				}
1861 			}
1862 			else
1863 				log("Not on Windows, skipping libcurl install");
1864 		}
1865 
1866 		override void updateEnv(ref Environment env)
1867 		{
1868 			env.vars["PATH"] = buildPath(buildDir, "bin").absolutePath() ~ pathSeparator ~ env.vars["PATH"];
1869 		}
1870 	}
1871 
1872 	private int tempError;
1873 
1874 	private Component[string] components;
1875 
1876 	Component getComponent(string name)
1877 	{
1878 		if (name !in components)
1879 		{
1880 			Component c;
1881 
1882 			switch (name)
1883 			{
1884 				case "dmd":
1885 					c = new DMD();
1886 					break;
1887 				case "phobos-includes":
1888 					c = new PhobosIncludes();
1889 					break;
1890 				case "druntime":
1891 					c = new Druntime();
1892 					break;
1893 				case "phobos":
1894 					c = new Phobos();
1895 					break;
1896 				case "rdmd":
1897 					c = new RDMD();
1898 					break;
1899 				case "tools":
1900 					c = new Tools();
1901 					break;
1902 				case "website":
1903 					c = new Website();
1904 					break;
1905 				case "extras":
1906 					c = new Extras();
1907 					break;
1908 				case "curl":
1909 					c = new Curl();
1910 					break;
1911 				default:
1912 					throw new Exception("Unknown component: " ~ name);
1913 			}
1914 
1915 			c.name = name;
1916 			return components[name] = c;
1917 		}
1918 
1919 		return components[name];
1920 	}
1921 
1922 	Component[] getSubmoduleComponents(string submoduleName)
1923 	{
1924 		return components
1925 			.byValue
1926 			.filter!(component => component.submoduleName == submoduleName)
1927 			.array();
1928 	}
1929 
1930 	// **************************** Customization ****************************
1931 
1932 	/// Fetch latest D history.
1933 	void update()
1934 	{
1935 		getMetaRepo().update();
1936 	}
1937 
1938 	struct SubmoduleState
1939 	{
1940 		string[string] submoduleCommits;
1941 	}
1942 
1943 	/// Begin customization, starting at the specified commit.
1944 	SubmoduleState begin(string commit)
1945 	{
1946 		log("Starting at meta repository commit " ~ commit);
1947 		return SubmoduleState(getMetaRepo().getSubmoduleCommits(commit));
1948 	}
1949 
1950 	/// Applies a merge onto the given SubmoduleState.
1951 	void merge(ref SubmoduleState submoduleState, string submoduleName, string branch)
1952 	{
1953 		log("Merging %s commit %s".format(submoduleName, branch));
1954 		enforce(submoduleName in submoduleState.submoduleCommits, "Unknown submodule: " ~ submoduleName);
1955 		auto submodule = getSubmodule(submoduleName);
1956 		auto head = submoduleState.submoduleCommits[submoduleName];
1957 		auto result = submodule.getMerge(head, branch);
1958 		submoduleState.submoduleCommits[submoduleName] = result;
1959 	}
1960 
1961 	/// Removes a merge from the given SubmoduleState.
1962 	void unmerge(ref SubmoduleState submoduleState, string submoduleName, string branch)
1963 	{
1964 		log("Unmerging %s commit %s".format(submoduleName, branch));
1965 		enforce(submoduleName in submoduleState.submoduleCommits, "Unknown submodule: " ~ submoduleName);
1966 		auto submodule = getSubmodule(submoduleName);
1967 		auto head = submoduleState.submoduleCommits[submoduleName];
1968 		auto result = submodule.getUnMerge(head, branch);
1969 		submoduleState.submoduleCommits[submoduleName] = result;
1970 	}
1971 
1972 	/// Reverts a commit from the given SubmoduleState.
1973 	/// parent is the 1-based mainline index (as per `man git-revert`),
1974 	/// or 0 if commit is not a merge commit.
1975 	void revert(ref SubmoduleState submoduleState, string submoduleName, string commit, int parent)
1976 	{
1977 		log("Reverting %s commit %s".format(submoduleName, commit));
1978 		enforce(submoduleName in submoduleState.submoduleCommits, "Unknown submodule: " ~ submoduleName);
1979 		auto submodule = getSubmodule(submoduleName);
1980 		auto head = submoduleState.submoduleCommits[submoduleName];
1981 		auto result = submodule.getRevert(head, commit, parent);
1982 		submoduleState.submoduleCommits[submoduleName] = result;
1983 	}
1984 
1985 	/// Returns the commit hash for the given pull request #.
1986 	/// The result can then be used with addMerge/removeMerge.
1987 	string getPull(string submoduleName, int pullNumber)
1988 	{
1989 		return getSubmodule(submoduleName).getPull(pullNumber);
1990 	}
1991 
1992 	/// Returns the commit hash for the given GitHub fork.
1993 	/// The result can then be used with addMerge/removeMerge.
1994 	string getFork(string submoduleName, string user, string branch)
1995 	{
1996 		return getSubmodule(submoduleName).getFork(user, branch);
1997 	}
1998 
1999 	/// Find the child of a commit (starting with the current submodule state),
2000 	/// and, if the commit was a merge, the mainline index of said commit for the child.
2001 	void getChild(ref SubmoduleState submoduleState, string submoduleName, string commit, out string child, out int mainline)
2002 	{
2003 		enforce(submoduleName in submoduleState.submoduleCommits, "Unknown submodule: " ~ submoduleName);
2004 		auto head = submoduleState.submoduleCommits[submoduleName];
2005 		return getSubmodule(submoduleName).getChild(head, commit, child, mainline);
2006 	}
2007 
2008 	// ****************************** Building *******************************
2009 
2010 	private SubmoduleState submoduleState;
2011 	private bool incrementalBuild;
2012 
2013 	@property string cacheEngineName()
2014 	{
2015 		if (incrementalBuild)
2016 			return "none";
2017 		else
2018 			return config.local.cache;
2019 	}
2020 
2021 	private string getComponentCommit(string componentName)
2022 	{
2023 		auto submoduleName = getComponent(componentName).submoduleName;
2024 		auto commit = submoduleState.submoduleCommits.get(submoduleName, null);
2025 		enforce(commit, "Unknown commit to build for component %s (submodule %s)"
2026 			.format(componentName, submoduleName));
2027 		return commit;
2028 	}
2029 
2030 	static const string[] defaultComponents = ["dmd", "druntime", "phobos-includes", "phobos", "rdmd"];
2031 	static const string[] additionalComponents = ["tools", "website", "extras", "curl"];
2032 	static const string[] allComponents = defaultComponents ~ additionalComponents;
2033 
2034 	/// Build the specified components according to the specified configuration.
2035 	void build(SubmoduleState submoduleState, bool incremental = false)
2036 	{
2037 		auto componentNames = config.build.components.getEnabledComponentNames();
2038 		log("Building components %-(%s, %)".format(componentNames));
2039 
2040 		this.components = null;
2041 		this.submoduleState = submoduleState;
2042 		this.incrementalBuild = incremental;
2043 
2044 		if (buildDir.exists)
2045 			buildDir.removeRecurse();
2046 		enforce(!buildDir.exists);
2047 
2048 		scope(exit) if (cacheEngine) cacheEngine.finalize();
2049 
2050 		foreach (componentName; componentNames)
2051 			getComponent(componentName).needInstalled();
2052 	}
2053 
2054 	/// Shortcut for begin + build
2055 	void buildRev(string rev)
2056 	{
2057 		auto submoduleState = begin(rev);
2058 		build(submoduleState);
2059 	}
2060 
2061 	/// Simply check out the source code for the given submodules.
2062 	void checkout(SubmoduleState submoduleState)
2063 	{
2064 		auto componentNames = config.build.components.getEnabledComponentNames();
2065 		log("Checking out components %-(%s, %)".format(componentNames));
2066 
2067 		this.components = null;
2068 		this.submoduleState = submoduleState;
2069 		this.incrementalBuild = false;
2070 
2071 		foreach (componentName; componentNames)
2072 			getComponent(componentName).needSource();
2073 	}
2074 
2075 	/// Rerun build without cleaning up any files.
2076 	void rebuild()
2077 	{
2078 		build(SubmoduleState(null), true);
2079 	}
2080 
2081 	/// Run all tests for the current checkout (like rebuild).
2082 	void test()
2083 	{
2084 		auto componentNames = config.build.components.getEnabledComponentNames();
2085 		log("Testing components %-(%s, %)".format(componentNames));
2086 
2087 		this.components = null;
2088 		this.submoduleState = SubmoduleState(null);
2089 		this.incrementalBuild = true;
2090 
2091 		foreach (componentName; componentNames)
2092 			getComponent(componentName).test();
2093 	}
2094 
2095 	bool isCached(SubmoduleState submoduleState)
2096 	{
2097 		this.components = null;
2098 		this.submoduleState = submoduleState;
2099 
2100 		needCacheEngine();
2101 		foreach (componentName; config.build.components.getEnabledComponentNames())
2102 			if (!cacheEngine.haveEntry(getComponent(componentName).getBuildID()))
2103 				return false;
2104 		return true;
2105 	}
2106 
2107 	/// Returns the isCached state for all commits in the history of the given ref.
2108 	bool[string] getCacheState(string[string][string] history)
2109 	{
2110 		log("Enumerating cache entries...");
2111 		auto cacheEntries = needCacheEngine().getEntries().toSet();
2112 
2113 		this.components = null;
2114 		auto componentNames = config.build.components.getEnabledComponentNames();
2115 		auto components = componentNames.map!(componentName => getComponent(componentName)).array;
2116 		auto requiredSubmodules = components
2117 			.map!(component => chain(component.name.only, component.sourceDependencies, component.dependencies))
2118 			.joiner
2119 			.map!(componentName => getComponent(componentName).submoduleName)
2120 			.array.sort().uniq().array
2121 		;
2122 
2123 		log("Collating cache state...");
2124 		bool[string] result;
2125 		foreach (commit, submoduleCommits; history)
2126 		{
2127 			this.submoduleState.submoduleCommits = submoduleCommits;
2128 
2129 			result[commit] =
2130 				requiredSubmodules.all!(submoduleName => submoduleName in submoduleCommits) &&
2131 				componentNames.all!(componentName =>
2132 					getComponent(componentName).I!(component =>
2133 						component.getBuildID() in cacheEntries
2134 					)
2135 				);
2136 		}
2137 		return result;
2138 	}
2139 
2140 	/// ditto
2141 	bool[string] getCacheState(string[] refs)
2142 	{
2143 		auto history = getMetaRepo().getSubmoduleHistory(refs);
2144 		return getCacheState(history);
2145 	}
2146 
2147 	// **************************** Dependencies *****************************
2148 
2149 	private void needInstaller()
2150 	{
2151 		Installer.logger = &log;
2152 		Installer.installationDirectory = dlDir;
2153 	}
2154 
2155 	/// Pull in a built DMD as configured.
2156 	/// Note that this function invalidates the current repository state.
2157 	void needDMD(ref Environment env)
2158 	{
2159 		tempError++; scope(success) tempError--;
2160 
2161 		auto dmdVer = config.build.components.dmd.bootstrap.ver;
2162 		if (!dmdVer)
2163 		{
2164 			dmdVer = "v2.067.1";
2165 			version (Windows)
2166 				if (config.build.components.dmd.dmdModel != Component.CommonConfig.defaultModel)
2167 					dmdVer = "v2.070.2"; // dmd/src/builtin.d needs core.stdc.math.fabsl. 2.068.2 generates a dmd which crashes on building Phobos
2168 		}
2169 
2170 		if (config.build.components.dmd.bootstrap.fromSource)
2171 		{
2172 			log("Bootstrapping DMD " ~ dmdVer);
2173 
2174 			auto bootstrapBuildConfig = config.build.components.dmd.bootstrap.build;
2175 
2176 			// Back up and clear component state
2177 			enum backupTemplate = q{
2178 				auto VARBackup = this.VAR;
2179 				this.VAR = typeof(VAR).init;
2180 				scope(exit) this.VAR = VARBackup;
2181 			};
2182 			mixin(backupTemplate.replace(q{VAR}, q{components}));
2183 			mixin(backupTemplate.replace(q{VAR}, q{config}));
2184 			mixin(backupTemplate.replace(q{VAR}, q{submoduleState}));
2185 
2186 			config.local = configBackup.local;
2187 			if (bootstrapBuildConfig)
2188 				config.build = *bootstrapBuildConfig;
2189 
2190 			// Disable building rdmd in the bootstrap compiler by default
2191 			if ("rdmd" !in config.build.components.enable)
2192 				config.build.components.enable["rdmd"] = false;
2193 
2194 			build(parseSpec(dmdVer));
2195 
2196 			log("Built bootstrap DMD " ~ dmdVer ~ " successfully.");
2197 
2198 			auto bootstrapDir = buildPath(config.local.workDir, "bootstrap");
2199 			if (bootstrapDir.exists)
2200 				bootstrapDir.removeRecurse();
2201 			ensurePathExists(bootstrapDir);
2202 			rename(buildDir, bootstrapDir);
2203 
2204 			env.deps.hostDC = buildPath(bootstrapDir, "bin", "dmd" ~ binExt);
2205 		}
2206 		else
2207 		{
2208 			import std.ascii;
2209 			log("Preparing DMD " ~ dmdVer);
2210 			enforce(dmdVer.startsWith("v"), "Invalid DMD version spec for binary bootstrap. Did you forget to " ~
2211 				((dmdVer.length && dmdVer[0].isDigit && dmdVer.contains('.')) ? "add a leading 'v'" : "enable fromSource") ~ "?");
2212 			needInstaller();
2213 			auto dmdInstaller = new DMDInstaller(dmdVer[1..$]);
2214 			dmdInstaller.requireLocal(false);
2215 			env.deps.hostDC = dmdInstaller.exePath("dmd").absolutePath();
2216 		}
2217 
2218 		log("hostDC=" ~ env.deps.hostDC);
2219 	}
2220 
2221 	void needKindleGen(ref Environment env)
2222 	{
2223 		needInstaller();
2224 		kindleGenInstaller.requireLocal(false);
2225 		env.vars["PATH"] = kindleGenInstaller.directory ~ pathSeparator ~ env.vars["PATH"];
2226 	}
2227 
2228 	version (Windows)
2229 	void needMSYS(ref Environment env)
2230 	{
2231 		needInstaller();
2232 		MSYS.msysCORE.requireLocal(false);
2233 		MSYS.libintl.requireLocal(false);
2234 		MSYS.libiconv.requireLocal(false);
2235 		MSYS.libtermcap.requireLocal(false);
2236 		MSYS.libregex.requireLocal(false);
2237 		MSYS.coreutils.requireLocal(false);
2238 		MSYS.bash.requireLocal(false);
2239 		MSYS.make.requireLocal(false);
2240 		MSYS.grep.requireLocal(false);
2241 		MSYS.sed.requireLocal(false);
2242 		MSYS.diffutils.requireLocal(false);
2243 		env.vars["PATH"] = MSYS.bash.directory.buildPath("bin") ~ pathSeparator ~ env.vars["PATH"];
2244 	}
2245 
2246 	/// Get DMD unbuildable extras
2247 	/// (proprietary DigitalMars utilities, 32-bit import libraries)
2248 	string needExtras()
2249 	{
2250 		import ae.utils.meta : I, singleton;
2251 
2252 		static class DExtrasInstaller : Installer
2253 		{
2254 			@property override string name() { return "dmd-localextras"; }
2255 			string url = "http://semitwist.com/download/app/dmd-localextras.7z";
2256 
2257 			override void installImpl(string target)
2258 			{
2259 				url
2260 					.I!save()
2261 					.I!unpackTo(target);
2262 			}
2263 
2264 			static this()
2265 			{
2266 				urlDigests["http://semitwist.com/download/app/dmd-localextras.7z"] = "ef367c2d25d4f19f45ade56ab6991c726b07d3d9";
2267 			}
2268 		}
2269 
2270 		alias extrasInstaller = singleton!DExtrasInstaller;
2271 
2272 		needInstaller();
2273 		extrasInstaller.requireLocal(false);
2274 		return extrasInstaller.directory;
2275 	}
2276 
2277 	/// Get libcurl for Windows (DLL and import libraries)
2278 	version (Windows)
2279 	string needCurl()
2280 	{
2281 		import ae.utils.meta : I, singleton;
2282 
2283 		static class DCurlInstaller : Installer
2284 		{
2285 			@property override string name() { return "libcurl-" ~ curlVersion; }
2286 			string curlVersion = "7.47.1";
2287 			@property string url() { return "http://downloads.dlang.org/other/libcurl-" ~ curlVersion ~ "-WinSSL-zlib-x86-x64.zip"; }
2288 
2289 			override void installImpl(string target)
2290 			{
2291 				url
2292 					.I!save()
2293 					.I!unpackTo(target);
2294 			}
2295 
2296 			static this()
2297 			{
2298 				urlDigests["http://downloads.dlang.org/other/libcurl-7.47.1-WinSSL-zlib-x86-x64.zip"] = "4b8a7bb237efab25a96588093ae51994c821e097";
2299 			}
2300 		}
2301 
2302 		alias curlInstaller = singleton!DCurlInstaller;
2303 
2304 		needInstaller();
2305 		curlInstaller.requireLocal(false);
2306 		return curlInstaller.directory;
2307 	}
2308 
2309 	version (Windows)
2310 	void needDMC(ref Environment env, string ver = null)
2311 	{
2312 		tempError++; scope(success) tempError--;
2313 
2314 		needInstaller();
2315 
2316 		auto dmc = ver ? new LegacyDMCInstaller(ver) : dmcInstaller;
2317 		if (!dmc.installedLocally)
2318 			log("Preparing DigitalMars C++ " ~ ver);
2319 		dmc.requireLocal(false);
2320 		env.deps.dmcDir = dmc.directory;
2321 
2322 		auto binPath = buildPath(env.deps.dmcDir, `bin`).absolutePath();
2323 		log("DMC=" ~ binPath);
2324 		env.vars["DMC"] = binPath;
2325 		env.vars["PATH"] = binPath ~ pathSeparator ~ env.vars.get("PATH", null);
2326 	}
2327 
2328 	version (Windows)
2329 	auto getVSInstaller()
2330 	{
2331 		needInstaller();
2332 		return vs2013community;
2333 	}
2334 
2335 	version (Windows)
2336 	static string msvcModelStr(string model, string str32, string str64)
2337 	{
2338 		switch (model)
2339 		{
2340 			case "32":
2341 				throw new Exception("Shouldn't need VC for 32-bit builds");
2342 			case "64":
2343 				return str64;
2344 			case "32mscoff":
2345 				return str32;
2346 			default:
2347 				throw new Exception("Unknown model: " ~ model);
2348 		}
2349 	}
2350 
2351 	version (Windows)
2352 	static string msvcModelDir(string model, string dir64 = "x86_amd64")
2353 	{
2354 		return msvcModelStr(model, null, dir64);
2355 	}
2356 
2357 	version (Windows)
2358 	void needVC(ref Environment env, string model)
2359 	{
2360 		tempError++; scope(success) tempError--;
2361 
2362 		auto vs = getVSInstaller();
2363 
2364 		// At minimum, we want the C compiler (cl.exe) and linker (link.exe).
2365 		vs["vc_compilercore86"].requireLocal(false); // Contains both x86 and x86_amd64 cl.exe
2366 		vs["vc_compilercore86res"].requireLocal(false); // Contains clui.dll needed by cl.exe
2367 
2368 		// Include files. Needed when using VS to build either DMD or Druntime.
2369 		vs["vc_librarycore86"].requireLocal(false); // Contains include files, e.g. errno.h needed by Druntime
2370 
2371 		// C runtime. Needed for all programs built with VC.
2372 		vs[msvcModelStr(model, "vc_libraryDesktop_x86", "vc_libraryDesktop_x64")].requireLocal(false); // libcmt.lib
2373 
2374 		// XP-compatible import libraries.
2375 		vs["win_xpsupport"].requireLocal(false); // shell32.lib
2376 
2377 		// MSBuild, for the useVC option
2378 		if (config.build.components.dmd.useVC)
2379 			vs["Msi_BuildTools_MSBuild_x86"].requireLocal(false); // msbuild.exe
2380 
2381 		env.deps.vsDir  = vs.directory.buildPath("Program Files (x86)", "Microsoft Visual Studio 12.0").absolutePath();
2382 		env.deps.sdkDir = vs.directory.buildPath("Program Files", "Microsoft SDKs", "Windows", "v7.1A").absolutePath();
2383 
2384 		env.vars["PATH"] ~= pathSeparator ~ vs.modelBinPaths(msvcModelDir(model)).map!(path => vs.directory.buildPath(path).absolutePath()).join(pathSeparator);
2385 		env.vars["VCINSTALLDIR"] = env.deps.vsDir.buildPath("VC") ~ dirSeparator;
2386 		env.vars["INCLUDE"] = env.deps.vsDir.buildPath("VC", "include") ~ ";" ~ env.deps.sdkDir.buildPath("Include");
2387 		env.vars["LIB"] = env.deps.vsDir.buildPath("VC", "lib", msvcModelDir(model, "amd64")) ~ ";" ~ env.deps.sdkDir.buildPath("Lib", msvcModelDir(model, "x64"));
2388 		env.vars["WindowsSdkDir"] = env.deps.sdkDir ~ dirSeparator;
2389 		env.vars["Platform"] = "x64";
2390 		env.vars["LINKCMD64"] = env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "link.exe"); // Used by dmd
2391 		env.vars["MSVC_CC"] = env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "cl.exe"); // For the msvc-dmc wrapper
2392 		env.vars["MSVC_AR"] = env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "lib.exe"); // For the msvc-lib wrapper
2393 		env.vars["CL"] = "-D_USING_V110_SDK71_"; // Work around __userHeader macro redifinition VS bug
2394 	}
2395 
2396 	private void needGit()
2397 	{
2398 		tempError++; scope(success) tempError--;
2399 
2400 		needInstaller();
2401 		gitInstaller.require();
2402 	}
2403 
2404 	/// Disable the "<program> has stopped working"
2405 	/// standard Windows dialog.
2406 	version (Windows)
2407 	static void disableCrashDialog()
2408 	{
2409 		enum : uint { SEM_FAILCRITICALERRORS = 1, SEM_NOGPFAULTERRORBOX = 2 }
2410 		SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX);
2411 	}
2412 
2413 	/// Create a build environment base.
2414 	protected @property Environment baseEnvironment()
2415 	{
2416 		Environment env;
2417 
2418 		// Build a new environment from scratch, to avoid tainting the build with the current environment.
2419 		string[] newPaths;
2420 
2421 		version (Windows)
2422 		{
2423 			import std.utf;
2424 			import ae.sys.windows.imports;
2425 			mixin(importWin32!q{winbase});
2426 			mixin(importWin32!q{winnt});
2427 
2428 			TCHAR[1024] buf;
2429 			// Needed for DLLs
2430 			auto winDir = buf[0..GetWindowsDirectory(buf.ptr, buf.length)].toUTF8();
2431 			auto sysDir = buf[0..GetSystemDirectory (buf.ptr, buf.length)].toUTF8();
2432 			newPaths ~= [sysDir, winDir];
2433 		}
2434 		else
2435 		{
2436 			// Needed for coreutils, make, gcc, git etc.
2437 			newPaths = ["/bin", "/usr/bin"];
2438 
2439 			version (linux)
2440 			{
2441 				// GCC wrappers
2442 				ensureDirExists(binDir);
2443 				newPaths = binDir ~ newPaths;
2444 			}
2445 		}
2446 
2447 		env.vars["PATH"] = newPaths.join(pathSeparator);
2448 
2449 		ensureDirExists(tmpDir);
2450 		env.vars["TMPDIR"] = env.vars["TEMP"] = env.vars["TMP"] = tmpDir;
2451 
2452 		version (Windows)
2453 		{
2454 			env.vars["SystemDrive"] = winDir.driveName;
2455 			env.vars["SystemRoot"] = winDir;
2456 		}
2457 
2458 		ensureDirExists(homeDir);
2459 		env.vars["HOME"] = homeDir;
2460 
2461 		return env;
2462 	}
2463 
2464 	/// Apply user modifications onto an environment.
2465 	/// Supports Windows-style %VAR% expansions.
2466 	static string[string] applyEnv(in string[string] target, in string[string] source)
2467 	{
2468 		// The source of variable expansions is variables in the target environment,
2469 		// if they exist, and the host environment otherwise, so e.g.
2470 		// `PATH=C:\...;%PATH%` and `MAKE=%MAKE%` work as expected.
2471 		auto oldEnv = std.process.environment.toAA();
2472 		foreach (name, value; target)
2473 			oldEnv[name] = value;
2474 
2475 		string[string] result;
2476 		foreach (name, value; target)
2477 			result[name] = value;
2478 		foreach (name, value; source)
2479 		{
2480 			string newValue = value;
2481 			foreach (oldName, oldValue; oldEnv)
2482 				newValue = newValue.replace("%" ~ oldName ~ "%", oldValue);
2483 			result[name] = oldEnv[name] = newValue;
2484 		}
2485 		return result;
2486 	}
2487 
2488 	// ******************************** Cache ********************************
2489 
2490 	enum unbuildableMarker = "unbuildable";
2491 
2492 	DCache cacheEngine;
2493 
2494 	DCache needCacheEngine()
2495 	{
2496 		if (!cacheEngine)
2497 		{
2498 			if (cacheEngineName == "git")
2499 				needGit();
2500 			cacheEngine = createCache(cacheEngineName, cacheEngineDir(cacheEngineName), this);
2501 		}
2502 		return cacheEngine;
2503 	}
2504 
2505 	void cp(string src, string dst)
2506 	{
2507 		needCacheEngine().cp(src, dst);
2508 	}
2509 
2510 	private string[] getComponentKeyOrder(string componentName)
2511 	{
2512 		auto submodule = getComponent(componentName).submodule;
2513 		submodule.needRepo();
2514 		return submodule
2515 			.git.query("log", "--pretty=format:%H", "--all", "--topo-order")
2516 			.splitLines()
2517 			.map!(commit => componentName ~ "-" ~ commit ~ "-")
2518 			.array
2519 		;
2520 	}
2521 
2522 	string componentNameFromKey(string key)
2523 	{
2524 		auto parts = key.split("-");
2525 		return parts[0..$-2].join("-");
2526 	}
2527 
2528 	string[][] getKeyOrder(string key)
2529 	{
2530 		if (key !is null)
2531 			return [getComponentKeyOrder(componentNameFromKey(key))];
2532 		else
2533 			return allComponents.map!(componentName => getComponentKeyOrder(componentName)).array;
2534 	}
2535 
2536 	/// Optimize entire cache.
2537 	void optimizeCache()
2538 	{
2539 		needCacheEngine().optimize();
2540 	}
2541 
2542 	bool shouldPurge(string key)
2543 	{
2544 		auto files = cacheEngine.listFiles(key);
2545 		if (files.canFind(unbuildableMarker))
2546 			return true;
2547 
2548 		if (componentNameFromKey(key) == "druntime")
2549 		{
2550 			if (!files.canFind("import/core/memory.d")
2551 			 && !files.canFind("import/core/memory.di"))
2552 				return true;
2553 		}
2554 
2555 		return false;
2556 	}
2557 
2558 	/// Delete cached "unbuildable" build results.
2559 	void purgeUnbuildable()
2560 	{
2561 		needCacheEngine()
2562 			.getEntries
2563 			.filter!(key => shouldPurge(key))
2564 			.each!((key)
2565 			{
2566 				log("Deleting: " ~ key);
2567 				cacheEngine.remove(key);
2568 			})
2569 		;
2570 	}
2571 
2572 	/// Move cached files from one cache engine to another.
2573 	void migrateCache(string sourceEngineName, string targetEngineName)
2574 	{
2575 		auto sourceEngine = createCache(sourceEngineName, cacheEngineDir(sourceEngineName), this);
2576 		auto targetEngine = createCache(targetEngineName, cacheEngineDir(targetEngineName), this);
2577 		auto tempDir = buildPath(config.local.workDir, "temp");
2578 		if (tempDir.exists)
2579 			tempDir.removeRecurse();
2580 		log("Enumerating source entries...");
2581 		auto sourceEntries = sourceEngine.getEntries();
2582 		log("Enumerating target entries...");
2583 		auto targetEntries = targetEngine.getEntries().sort();
2584 		foreach (key; sourceEntries)
2585 			if (!targetEntries.canFind(key))
2586 			{
2587 				log(key);
2588 				sourceEngine.extract(key, tempDir, fn => true);
2589 				targetEngine.add(key, tempDir);
2590 				if (tempDir.exists)
2591 					tempDir.removeRecurse();
2592 			}
2593 		targetEngine.optimize();
2594 	}
2595 
2596 	// **************************** Miscellaneous ****************************
2597 
2598 	struct LogEntry
2599 	{
2600 		string hash;
2601 		string[] message;
2602 		SysTime time;
2603 	}
2604 
2605 	/// Gets the D merge log (newest first).
2606 	LogEntry[] getLog(string refName = "refs/remotes/origin/master")
2607 	{
2608 		getMetaRepo().needRepo();
2609 		auto history = getMetaRepo().git.getHistory();
2610 		LogEntry[] logs;
2611 		auto master = history.commits[history.refs[refName]];
2612 		for (auto c = master; c; c = c.parents.length ? c.parents[0] : null)
2613 		{
2614 			auto time = SysTime(c.time.unixTimeToStdTime);
2615 			logs ~= LogEntry(c.hash.toString(), c.message, time);
2616 		}
2617 		return logs;
2618 	}
2619 
2620 	// ***************************** Integration *****************************
2621 
2622 	/// Override to add logging.
2623 	void log(string line)
2624 	{
2625 	}
2626 
2627 	/// Bootstrap description resolution.
2628 	/// See DMD.Config.Bootstrap.spec.
2629 	/// This is essentially a hack to allow the entire
2630 	/// Config structure to be parsed from an .ini file.
2631 	SubmoduleState parseSpec(string spec)
2632 	{
2633 		getMetaRepo().needRepo();
2634 		auto rev = getMetaRepo().getRef("refs/tags/" ~ spec);
2635 		log("Resolved " ~ spec ~ " to " ~ rev);
2636 		return begin(rev);
2637 	}
2638 
2639 	/// Override this method with one which returns a command,
2640 	/// which will invoke the unmergeRebaseEdit function below,
2641 	/// passing to it any additional parameters.
2642 	/// Note: Currently unused. Was previously used
2643 	/// for unmerging things using interactive rebase.
2644 	abstract string getCallbackCommand();
2645 
2646 	void callback(string[] args) { assert(false); }
2647 }