1 /**
2  * Code to manage a D checkout and its dependencies.
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Vladimir Panteleev <vladimir@thecybershadow.net>
12  */
13 
14 module ae.sys.d.manager;
15 
16 import std.algorithm;
17 import std.array;
18 import std.conv;
19 import std.datetime;
20 import std.exception;
21 import std.file;
22 import std.path;
23 import std.process : spawnProcess, wait, escapeShellCommand;
24 import std.range;
25 import std.regex;
26 import std.string;
27 import std.typecons;
28 
29 import ae.sys.d.cache;
30 import ae.sys.d.repo;
31 import ae.sys.file;
32 import ae.sys.git;
33 import ae.utils.aa;
34 import ae.utils.array;
35 import ae.utils.digest;
36 import ae.utils.json;
37 import ae.utils.regex;
38 
39 alias ensureDirExists = ae.sys.file.ensureDirExists;
40 
41 version (Windows)
42 {
43 	import ae.sys.install.dmc;
44 	import ae.sys.install.msys;
45 	import ae.sys.install.vs;
46 
47 	extern(Windows) void SetErrorMode(int);
48 }
49 
50 import ae.sys.install.dmd;
51 import ae.sys.install.git;
52 import ae.sys.install.kindlegen;
53 
54 static import std.process;
55 
56 /// Class which manages a D checkout and its dependencies.
57 class DManager : ICacheHost
58 {
59 	// **************************** Configuration ****************************
60 
61 	struct Config /// DManager configuration.
62 	{
63 		struct Build /// Build configuration
64 		{
65 			struct Components
66 			{
67 				bool[string] enable;
68 
69 				string[] getEnabledComponentNames()
70 				{
71 					foreach (componentName; enable.byKey)
72 						enforce(allComponents.canFind(componentName), "Unknown component: " ~ componentName);
73 					return allComponents
74 						.filter!(componentName =>
75 							enable.get(componentName, defaultComponents.canFind(componentName)))
76 						.array
77 						.dup;
78 				}
79 
80 				Component.CommonConfig common;
81 				DMD.Config dmd;
82 				Website.Config website;
83 			}
84 			Components components;
85 
86 			/// Additional environment variables.
87 			/// Supports %VAR% expansion - see applyEnv.
88 			string[string] environment;
89 		}
90 		Build build; /// ditto
91 
92 		/// Machine-local configuration
93 		/// These settings should not affect the build output.
94 		struct Local
95 		{
96 			/// URL of D git repository hosting D components.
97 			/// Defaults to (and must have the layout of) D.git:
98 			/// https://github.com/CyberShadow/D-dot-git
99 			string repoUrl = "https://bitbucket.org/cybershadow/d.git";
100 
101 			/// Location for the checkout, temporary files, etc.
102 			string workDir;
103 
104 			/// If present, passed to GNU make via -j parameter.
105 			/// Can also be "auto" or "unlimited".
106 			string makeJobs;
107 
108 			/// Don't get latest updates from GitHub.
109 			bool offline;
110 
111 			/// How to cache built files.
112 			string cache;
113 		}
114 		Local local; /// ditto
115 	}
116 	Config config; /// ditto
117 
118 	// Behavior options that generally depend on the host program.
119 
120 	/// Automatically re-clone the repository in case
121 	/// "git reset --hard" fails.
122 	bool autoClean;
123 
124 	/// Whether to verify working tree state
125 	/// to make sure we don't clobber user changes
126 	bool verifyWorkTree;
127 
128 	/// Whether we should cache failed builds.
129 	bool cacheFailures = true;
130 
131 	/// Current build environment.
132 	struct Environment
133 	{
134 		struct Deps /// Configuration for software dependencies
135 		{
136 			string dmcDir;   /// Where dmc.zip is unpacked.
137 			string vsDir;    /// Where Visual Studio is installed
138 			string sdkDir;   /// Where the Windows SDK is installed
139 			string hostDC;   /// Host D compiler (for DDMD bootstrapping)
140 		}
141 		Deps deps; /// ditto
142 
143 		/// Calculated local environment, incl. dependencies
144 		string[string] vars;
145 	}
146 
147 	/// Get a specific subdirectory of the work directory.
148 	@property string subDir(string name)() { return buildPath(config.local.workDir, name); }
149 
150 	alias repoDir    = subDir!"repo";        /// The git repository location.
151 	alias buildDir   = subDir!"build";       /// The build directory.
152 	alias dlDir      = subDir!"dl";          /// The directory for downloaded software.
153 
154 	/// This number increases with each incompatible change to cached data.
155 	enum cacheVersion = 3;
156 
157 	string cacheEngineDir(string engineName)
158 	{
159 		// Keep compatibility with old cache paths
160 		string engineDirName =
161 			engineName.isOneOf("directory", "true") ? "cache"      :
162 			engineName.isOneOf("", "none", "false") ? "temp-cache" :
163 			"cache-" ~ engineName;
164 		return buildPath(
165 			config.local.workDir,
166 			engineDirName,
167 			"v%d".format(cacheVersion),
168 		);
169 	}
170 
171 	version (Windows)
172 	{
173 		enum string binExt = ".exe";
174 		enum configFileName = "sc.ini";
175 	}
176 	else
177 	{
178 		enum string binExt = "";
179 		enum configFileName = "dmd.conf";
180 	}
181 
182 	static bool needConfSwitch() { return exists(std.process.environment.get("HOME", null).buildPath(configFileName)); }
183 
184 	// **************************** Repositories *****************************
185 
186 	class DManagerRepository : ManagedRepository
187 	{
188 		this()
189 		{
190 			this.offline = config.local.offline;
191 			this.verify = this.outer.verifyWorkTree;
192 		}
193 
194 		override void log(string s) { return this.outer.log(s); }
195 	}
196 
197 	class MetaRepository : DManagerRepository
198 	{
199 		override void needRepo()
200 		{
201 			needGit();
202 
203 			if (!repoDir.exists)
204 			{
205 				log("Cloning initial repository...");
206 				atomic!performClone(config.local.repoUrl, repoDir);
207 			}
208 
209 			if (!git.path)
210 				git = Repository(repoDir);
211 		}
212 
213 		static void performClone(string url, string target)
214 		{
215 			import ae.sys.cmd;
216 			run(["git", "clone", url, target]);
217 		}
218 
219 		override void performCheckout(string hash)
220 		{
221 			super.performCheckout(hash);
222 			submodules = null;
223 		}
224 
225 		string[string][string] submoduleCache;
226 
227 		string[string] getSubmoduleCommits(string head)
228 		{
229 			auto pcacheEntry = head in submoduleCache;
230 			if (pcacheEntry)
231 				return (*pcacheEntry).dup;
232 
233 			string[string] result;
234 			needRepo();
235 			foreach (line; git.query("ls-tree", head).splitLines())
236 			{
237 				auto parts = line.split();
238 				if (parts.length == 4 && parts[1] == "commit")
239 					result[parts[3]] = parts[2];
240 			}
241 			assert(result.length, "No submodules found");
242 			submoduleCache[head] = result;
243 			return result.dup;
244 		}
245 
246 		/// Get the submodule state for all commits in the history.
247 		/// Returns: result[commitHash][submoduleName] == submoduleCommitHash
248 		string[string][string] getSubmoduleHistory(string[] refs)
249 		{
250 			auto marksFile = buildPath(config.local.workDir, "temp", "marks.txt");
251 			ensurePathExists(marksFile);
252 			scope(exit) if (marksFile.exists) marksFile.remove();
253 			log("Running fast-export...");
254 			auto fastExportData = git.query([
255 				"fast-export",
256 				"--full-tree",
257 				"--no-data",
258 				"--export-marks=" ~ marksFile.absolutePath,
259 				] ~ refs
260 			);
261 
262 			log("Parsing fast-export marks...");
263 
264 			auto markLines = marksFile.readText.strip.splitLines;
265 			auto marks = new string[markLines.length];
266 			foreach (line; markLines)
267 			{
268 				auto parts = line.split(' ');
269 				auto markIndex = parts[0][1..$].to!int-1;
270 				marks[markIndex] = parts[1];
271 			}
272 
273 			log("Parsing fast-export data...");
274 
275 			string[string][string] result;
276 			foreach (i, commitData; fastExportData.split("deleteall\n")[1..$])
277 				result[marks[i]] = commitData
278 					.matchAll(re!(`^M 160000 ([0-9a-f]{40}) (\S+)$`, "m"))
279 					.map!(m => tuple(m.captures[2], m.captures[1]))
280 					.assocArray
281 				;
282 			return result;
283 		}
284 	}
285 
286 	class SubmoduleRepository : DManagerRepository
287 	{
288 		string dir;
289 
290 		override void needRepo()
291 		{
292 			getMetaRepo().needRepo();
293 
294 			if (!git.path)
295 				git = Repository(dir);
296 		}
297 
298 		override void needHead(string hash)
299 		{
300 			if (!autoClean)
301 				super.needHead(hash);
302 			else
303 			try
304 				super.needHead(hash);
305 			catch (RepositoryCleanException e)
306 			{
307 				log("Error during repository cleanup.");
308 
309 				log("Nuking %s...".format(dir));
310 				rmdirRecurse(dir);
311 
312 				auto name = baseName(dir);
313 				auto gitDir = buildPath(dirName(dir), ".git", "modules", name);
314 				log("Nuking %s...".format(gitDir));
315 				rmdirRecurse(gitDir);
316 
317 				log("Updating submodule...");
318 				getMetaRepo().git.run(["submodule", "update", name]);
319 
320 				reset();
321 
322 				log("Trying again...");
323 				super.needHead(hash);
324 			}
325 		}
326 	}
327 
328 	/// The meta-repository, which contains the sub-project submodules.
329 	private MetaRepository metaRepo;
330 
331 	MetaRepository getMetaRepo() /// ditto
332 	{
333 		if (!metaRepo)
334 			metaRepo = new MetaRepository;
335 		return metaRepo;
336 	}
337 
338 	/// Sub-project repositories.
339 	private SubmoduleRepository[string] submodules;
340 
341 	ManagedRepository getSubmodule(string name) /// ditto
342 	{
343 		assert(name, "This component is not associated with a submodule");
344 		if (name !in submodules)
345 		{
346 			getMetaRepo().needRepo();
347 			enforce(name in getMetaRepo().getSubmoduleCommits(getMetaRepo().getRef("origin/master")),
348 				"Unknown submodule: " ~ name);
349 
350 			auto path = buildPath(metaRepo.git.path, name);
351 			auto gitPath = buildPath(path, ".git");
352 
353 			if (!gitPath.exists)
354 			{
355 				log("Initializing and updating submodule %s...".format(name));
356 				getMetaRepo().git.run(["submodule", "update", "--init", name]);
357 			}
358 
359 			submodules[name] = new SubmoduleRepository();
360 			submodules[name].dir = path;
361 		}
362 
363 		return submodules[name];
364 	}
365 
366 	// ***************************** Components ******************************
367 
368 	/// Base class for a D component.
369 	class Component
370 	{
371 		/// Name of this component, as registered in DManager.components AA.
372 		string name;
373 
374 		/// Corresponding subproject repository name.
375 		@property abstract string submoduleName();
376 		@property ManagedRepository submodule() { return getSubmodule(submoduleName); }
377 
378 		/// Configuration applicable to multiple (not all) components.
379 		/// Note: don't serialize this structure whole!
380 		/// Only serialize used fields.
381 		struct CommonConfig
382 		{
383 			version (Windows)
384 				enum defaultModel = "32";
385 			else
386 			version (D_LP64)
387 				enum defaultModel = "64";
388 			else
389 				enum defaultModel = "32";
390 
391 			string model = defaultModel; /// Target model ("32" or "64").
392 
393 			string[] makeArgs; /// Additional make parameters,
394 			                   /// e.g. "HOST_CC=g++48"
395 		}
396 
397 		/// A string description of this component's configuration.
398 		abstract @property string configString();
399 
400 		/// Commit in the component's repo from which to build this component.
401 		@property string commit() { return incrementalBuild ? "incremental" : getComponentCommit(name); }
402 
403 		/// The components the source code of which this component depends on.
404 		/// Used for calculating the cache key.
405 		@property abstract string[] sourceDependencies();
406 
407 		/// The components the state and configuration of which this component depends on.
408 		/// Used for calculating the cache key.
409 		@property abstract string[] dependencies();
410 
411 		/// This metadata is saved to a .json file,
412 		/// and is also used to calculate the cache key.
413 		struct Metadata
414 		{
415 			int cacheVersion;
416 			string name;
417 			string commit;
418 			string configString;
419 			string[] sourceDepCommits;
420 			Metadata[] dependencyMetadata;
421 		}
422 
423 		Metadata getMetadata() /// ditto
424 		{
425 			return Metadata(
426 				cacheVersion,
427 				name,
428 				commit,
429 				configString,
430 				sourceDependencies.map!(
431 					dependency => getComponent(dependency).commit
432 				).array(),
433 				dependencies.map!(
434 					dependency => getComponent(dependency).getMetadata()
435 				).array(),
436 			);
437 		}
438 
439 		void saveMetaData(string target)
440 		{
441 			std.file.write(buildPath(target, "digger-metadata.json"), getMetadata().toJson());
442 			// Use a separate file to avoid double-encoding JSON
443 			std.file.write(buildPath(target, "digger-config.json"), configString);
444 		}
445 
446 		/// Calculates the cache key, which should be unique and immutable
447 		/// for the same source, build parameters, and build algorithm.
448 		string getBuildID()
449 		{
450 			auto configBlob = getMetadata().toJson() ~ configString;
451 			return "%s-%s-%s".format(
452 				name,
453 				commit,
454 				configBlob.getDigestString!MD5().toLower(),
455 			);
456 		}
457 
458 		@property string sourceDir() { submodule.needRepo(); return submodule.git.path; }
459 
460 		/// Directory to which built files are copied to.
461 		/// This will then be atomically added to the cache.
462 		protected string stageDir;
463 
464 		/// Prepare the source checkout for this component.
465 		/// Usually needed by other components.
466 		void needSource()
467 		{
468 			tempError++; scope(success) tempError--;
469 
470 			if (incrementalBuild)
471 				return;
472 			if (!submoduleName)
473 				return;
474 			foreach (component; getSubmoduleComponents(submoduleName))
475 				component.haveBuild = false;
476 
477 			submodule.needHead(commit);
478 		}
479 
480 		private bool haveBuild;
481 
482 		/// Build the component in-place, as needed,
483 		/// without moving the built files anywhere.
484 		void needBuild()
485 		{
486 			if (haveBuild) return;
487 			scope(success) haveBuild = true;
488 
489 			log("needBuild: " ~ getBuildID());
490 
491 			needSource();
492 
493 			log("Building " ~ getBuildID());
494 			if (submoduleName)
495 				submodule.clean = false;
496 			performBuild();
497 			log(getBuildID() ~ " built OK!");
498 		}
499 
500 		private bool haveInstalled;
501 
502 		/// Build and "install" the component to buildDir as necessary.
503 		void needInstalled()
504 		{
505 			if (haveInstalled) return;
506 			scope(success) haveInstalled = true;
507 
508 			auto buildID = getBuildID();
509 			log("needInstalled: " ~ buildID);
510 
511 			needCacheEngine();
512 			if (cacheEngine.haveEntry(buildID))
513 			{
514 				log("Cache hit!");
515 				if (cacheEngine.listFiles(buildID).canFind(unbuildableMarker))
516 					throw new Exception(buildID ~ " was cached as unbuildable");
517 			}
518 			else
519 			{
520 				log("Cache miss.");
521 
522 				auto tempDir = buildPath(config.local.workDir, "temp");
523 				if (tempDir.exists)
524 					tempDir.removeRecurse();
525 				stageDir = buildPath(tempDir, buildID);
526 				stageDir.mkdirRecurse();
527 
528 				bool failed = false;
529 				tempError = 0;
530 
531 				// Save the results to cache, failed or not
532 				void saveToCache()
533 				{
534 					// Use a separate function to work around
535 					// "cannot put scope(success) statement inside scope(exit)"
536 
537 					tempError++; scope(success) tempError--;
538 
539 					// tempDir might be removed by a dependency's build failure.
540 					if (!tempDir.exists)
541 						log("Not caching %s dependency build failure.".format(name));
542 					else
543 					// Don't cache failed build results due to temporary/environment problems
544 					if (failed && tempError > 0)
545 					{
546 						log("Not caching %s build failure due to temporary/environment error.".format(name));
547 						rmdirRecurse(tempDir);
548 					}
549 					else
550 					// Don't cache failed build results during delve
551 					if (failed && !cacheFailures)
552 					{
553 						log("Not caching failed %s build.".format(name));
554 						rmdirRecurse(tempDir);
555 					}
556 					else
557 					if (cacheEngine.haveEntry(buildID))
558 					{
559 						// Can happen due to force==true
560 						log("Already in cache.");
561 						rmdirRecurse(tempDir);
562 					}
563 					else
564 					{
565 						log("Saving to cache.");
566 						saveMetaData(stageDir);
567 						cacheEngine.add(buildID, stageDir);
568 						rmdirRecurse(tempDir);
569 					}
570 				}
571 
572 				scope (exit)
573 					saveToCache();
574 
575 				// An incomplete build is useless, nuke the directory
576 				// and create a new one just for the "unbuildable" marker.
577 				scope (failure)
578 				{
579 					failed = true;
580 					if (stageDir.exists)
581 					{
582 						rmdirRecurse(stageDir);
583 						mkdir(stageDir);
584 						buildPath(stageDir, unbuildableMarker).touch();
585 					}
586 				}
587 
588 				needBuild();
589 
590 				performStage();
591 			}
592 
593 			install();
594 		}
595 
596 		/// Build the component in-place, without moving the built files anywhere.
597 		void performBuild() {}
598 
599 		/// Place resulting files to stageDir
600 		void performStage() {}
601 
602 		/// Update the environment post-install, to allow
603 		/// building components that depend on this one.
604 		void updateEnv(ref Environment env) {}
605 
606 		/// Copy build results from cacheDir to buildDir
607 		void install()
608 		{
609 			log("Installing " ~ getBuildID());
610 			needCacheEngine().extract(getBuildID(), buildDir, de => !de.baseName.startsWith("digger-"));
611 		}
612 
613 		/// Prepare the dependencies then run the component's tests.
614 		void test()
615 		{
616 			log("Testing " ~ getBuildID());
617 
618 			needSource();
619 
620 			submodule.clean = false;
621 			performTest();
622 			log(getBuildID() ~ " tests OK!");
623 		}
624 
625 		/// Run the component's tests.
626 		void performTest() {}
627 
628 	protected final:
629 		// Utility declarations for component implementations
630 
631 		@property string modelSuffix() { return config.build.components.common.model == "32" ? "" : config.build.components.common.model; }
632 		version (Windows)
633 		{
634 			enum string makeFileName = "win32.mak";
635 			@property string makeFileNameModel()
636 			{
637 				string model = config.build.components.common.model;
638 				if (model == "32mscoff")
639 					model = "64";
640 				return "win"~model~".mak";
641 			}
642 			enum string binExt = ".exe";
643 		}
644 		else
645 		{
646 			enum string makeFileName = "posix.mak";
647 			enum string makeFileNameModel = "posix.mak";
648 			enum string binExt = "";
649 		}
650 
651 		/// Returns the command for the make utility.
652 		string[] getMake(in ref Environment env)
653 		{
654 			return [env.vars.get("MAKE", "make")];
655 		}
656 
657 		/// Returns the path to the built dmd executable.
658 		@property string dmd() { return buildPath(buildDir, "bin", "dmd" ~ binExt).absolutePath(); }
659 
660 		/// Escape a path for d_do_test's very "special" criteria.
661 		/// Spaces must be escaped, but there must be no double-quote at the end.
662 		private static string dDoTestEscape(string str)
663 		{
664 			return str.replaceAll(re!`\\([^\\ ]*? [^\\]*)(?=\\)`, `\"$1"`);
665 		}
666 
667 		unittest
668 		{
669 			assert(dDoTestEscape(`C:\Foo boo bar\baz quuz\derp.exe`) == `C:\"Foo boo bar"\"baz quuz"\derp.exe`);
670 		}
671 
672 		string[] getPlatformMakeVars(in ref Environment env)
673 		{
674 			string model = config.build.components.common.model;
675 			string[] args;
676 
677 			args ~= "MODEL=" ~ model;
678 
679 			version (Windows)
680 				if (model != "32")
681 				{
682 					args ~= "VCDIR="  ~ env.deps.vsDir.buildPath("VC").absolutePath();
683 					args ~= "SDKDIR=" ~ env.deps.sdkDir.absolutePath();
684 					args ~= "CC=" ~ '"' ~ env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "cl.exe").absolutePath() ~ '"';
685 					args ~= "LD=" ~ '"' ~ env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "link.exe").absolutePath() ~ '"';
686 					args ~= "AR=" ~ '"' ~ env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "lib.exe").absolutePath() ~ '"';
687 				}
688 
689 			return args;
690 		}
691 
692 		@property string[] gnuMakeArgs()
693 		{
694 			string[] args;
695 			if (config.local.makeJobs)
696 			{
697 				if (config.local.makeJobs == "auto")
698 				{
699 					import std.parallelism, std.conv;
700 					args ~= "-j" ~ text(totalCPUs);
701 				}
702 				else
703 				if (config.local.makeJobs == "unlimited")
704 					args ~= "-j";
705 				else
706 					args ~= "-j" ~ config.local.makeJobs;
707 			}
708 			return args;
709 		}
710 
711 		@property string[] dMakeArgs()
712 		{
713 			version (Windows)
714 				return null; // On Windows, DigitalMars make is used for all makefiles except the dmd test suite
715 			else
716 				return gnuMakeArgs;
717 		}
718 
719 		/// Older versions did not use the posix.mak/win32.mak convention.
720 		static string findMakeFile(string dir, string fn)
721 		{
722 			version (OSX)
723 				if (!dir.buildPath(fn).exists && dir.buildPath("osx.mak").exists)
724 					return "osx.mak";
725 			version (Posix)
726 				if (!dir.buildPath(fn).exists && dir.buildPath("linux.mak").exists)
727 					return "linux.mak";
728 			return fn;
729 		}
730 
731 		void needCC(ref Environment env, string model, string dmcVer = null)
732 		{
733 			version (Windows)
734 			{
735 				needDMC(env, dmcVer); // We need DMC even for 64-bit builds (for DM make)
736 				if (model != "32")
737 					needVC(env, model);
738 			}
739 		}
740 
741 		void run(in string[] args, in string[string] newEnv, string dir)
742 		{
743 			log("Running: " ~ escapeShellCommand(args));
744 
745 			// Apply user environment
746 			auto env = applyEnv(newEnv, config.build.environment);
747 
748 			// Temporarily apply PATH from newEnv to our process,
749 			// so process creation lookup can use it.
750 			string oldPath = std.process.environment["PATH"];
751 			scope (exit) std.process.environment["PATH"] = oldPath;
752 			std.process.environment["PATH"] = env["PATH"];
753 			log("PATH=" ~ env["PATH"]);
754 
755 			auto status = spawnProcess(args, env, std.process.Config.newEnv, dir).wait();
756 			enforce(status == 0, "Command %s failed with status %d".format(args, status));
757 		}
758 	}
759 
760 	/// The dmd executable
761 	final class DMD : Component
762 	{
763 		@property override string submoduleName  () { return "dmd"; }
764 		@property override string[] sourceDependencies() { return []; }
765 		@property override string[] dependencies() { return []; }
766 
767 		struct Config
768 		{
769 			/// Whether to build a debug DMD.
770 			/// Debug builds are faster to build,
771 			/// but run slower.
772 			@JSONOptional bool debugDMD = false;
773 
774 			/// Whether to build a release DMD.
775 			/// Mutually exclusive with debugDMD.
776 			@JSONOptional bool releaseDMD = false;
777 
778 			/// Model for building DMD itself (on Windows).
779 			/// Can be used to build a 64-bit DMD, to avoid 4GB limit.
780 			/// Currently only implemented on Windows, for DMD 2.072 or later.
781 			@JSONOptional string dmdModel = CommonConfig.defaultModel;
782 
783 			/// How to build DMD versions written in D.
784 			/// We can either download a pre-built binary DMD
785 			/// package, or build an  earlier version from source
786 			/// (e.g. starting with the last C++-only version.)
787 			struct Bootstrap
788 			{
789 				/// Whether to download a pre-built D version,
790 				/// or build one from source. If set, then build
791 				/// from source according to the value of ver,
792 				@JSONOptional bool fromSource = false;
793 
794 				/// Version specification.
795 				/// When building from source, syntax can be defined
796 				/// by outer application (see parseSpec method);
797 				/// By default, it is understood as a version number,
798 				/// such as "v2.070.2", which also doubles as a tag name.
799 				@JSONOptional string ver = null;
800 
801 				/// Build configuration for the compiler used for bootstrapping.
802 				/// If not set, then use the same build configuration as for the
803 				/// target build. Used when fromSource is set.
804 				@JSONOptional DManager.Config.Build* build;
805 			}
806 			@JSONOptional Bootstrap bootstrap; /// ditto
807 
808 			/// Use Visual C++ to build DMD instead of DMC.
809 			/// Currently, this is a hack, as msbuild will consult the system
810 			/// registry and use the system-wide installation of Visual Studio.
811 			/// Only relevant for older versions, as newer versions are written in D.
812 			@JSONOptional bool useVC;
813 		}
814 
815 		@property override string configString()
816 		{
817 			static struct FullConfig
818 			{
819 				Config config;
820 				string[] makeArgs;
821 			}
822 
823 			return FullConfig(
824 				config.build.components.dmd,
825 				config.build.components.common.makeArgs,
826 			).toJson();
827 		}
828 
829 		@property string vsConfiguration() { return config.build.components.dmd.debugDMD ? "Debug" : "Release"; }
830 		@property string vsPlatform     () { return config.build.components.common.model == "64" ? "x64" : "Win32"; }
831 
832 		override void performBuild()
833 		{
834 			// We need an older DMC for older DMD versions
835 			string dmcVer = null;
836 			auto idgen = buildPath(sourceDir, "src", "idgen.c");
837 			if (idgen.exists && idgen.readText().indexOf(`{ "alignof" },`) >= 0)
838 				dmcVer = "850";
839 
840 			auto env = baseEnvironment;
841 			needCC(env, config.build.components.dmd.dmdModel, dmcVer); // Need VC too for VSINSTALLDIR
842 
843 			if (buildPath(sourceDir, "src", "idgen.d").exists)
844 			{
845 				// Required for bootstrapping.
846 				needDMD(env);
847 				// Go back to our commit.
848 				needSource();
849 				submodule.clean = false;
850 			}
851 
852 			auto srcDir = buildPath(sourceDir, "src");
853 
854 			if (config.build.components.dmd.useVC) // Mostly obsolete, see useVC ddoc
855 			{
856 				version (Windows)
857 				{
858 					needVC(env, config.build.components.dmd.dmdModel);
859 
860 					env.vars["PATH"] = env.vars["PATH"] ~ pathSeparator ~ env.deps.hostDC.dirName;
861 
862 					auto solutionFile = `dmd_msc_vs10.sln`;
863 					if (!exists(srcDir.buildPath(solutionFile)))
864 						solutionFile = `vcbuild\dmd.sln`;
865 					if (!exists(srcDir.buildPath(solutionFile)))
866 						throw new Exception("Can't find Visual Studio solution file");
867 
868 					return run(["msbuild", "/p:Configuration=" ~ vsConfiguration, "/p:Platform=" ~ vsPlatform, solutionFile], env.vars, srcDir);
869 				}
870 				else
871 					throw new Exception("Can only use Visual Studio on Windows");
872 			}
873 
874 			version (Windows)
875 				auto scRoot = env.deps.dmcDir.absolutePath();
876 
877 			string dmdMakeFileName = findMakeFile(srcDir, makeFileName);
878 			string dmdMakeFullName = srcDir.buildPath(dmdMakeFileName);
879 
880 			string modelFlag = config.build.components.common.model;
881 			if (dmdMakeFullName.readText().canFind("MODEL=-m32"))
882 				modelFlag = "-m" ~ modelFlag;
883 
884 			version (Windows)
885 			{
886 				// A make argument is insufficient,
887 				// because of recursive make invocations
888 				auto m = dmdMakeFullName.readText();
889 				m = m
890 					.replace(`CC=\dm\bin\dmc`, `CC=dmc`)
891 					.replace(`SCROOT=$D\dm`, `SCROOT=` ~ scRoot)
892 				;
893 				dmdMakeFullName.write(m);
894 			}
895 			else
896 			{
897 				auto m = dmdMakeFullName.readText();
898 				m = m
899 					// Fix hard-coded reference to gcc as linker
900 					.replace(`gcc -m32 -lstdc++`, `g++ -m32 -lstdc++`)
901 					.replace(`gcc $(MODEL) -lstdc++`, `g++ $(MODEL) -lstdc++`)
902 					// Fix compilation of older versions of go.c with GCC 6
903 					.replace(`-Wno-deprecated`, `-Wno-deprecated -Wno-narrowing`)
904 				;
905 				// Fix pthread linker error
906 				version (linux)
907 					m = m.replace(`-lpthread`, `-pthread`);
908 				dmdMakeFullName.write(m);
909 			}
910 			submodule.saveFileState("src/" ~ dmdMakeFileName);
911 
912 			string[] extraArgs, targets;
913 			version (Posix)
914 			{
915 				if (config.build.components.dmd.debugDMD)
916 					extraArgs ~= "DEBUG=1";
917 				if (config.build.components.dmd.releaseDMD)
918 					extraArgs ~= "ENABLE_RELEASE=1";
919 			}
920 			else
921 			{
922 				if (config.build.components.dmd.debugDMD)
923 					targets ~= [];
924 				else
925 				if (config.build.components.dmd.releaseDMD && dmdMakeFullName.readText().canFind("reldmd"))
926 					targets ~= ["reldmd"];
927 				else
928 					targets ~= ["dmd"];
929 			}
930 
931 			if (config.build.components.dmd.dmdModel != CommonConfig.defaultModel)
932 			{
933 				version (Windows)
934 				{
935 					dmdMakeFileName = "win64.mak";
936 					dmdMakeFullName = srcDir.buildPath(dmdMakeFileName);
937 					enforce(dmdMakeFullName.exists, "dmdModel not supported for this DMD version");
938 					extraArgs ~= "DMODEL=-m" ~ config.build.components.dmd.dmdModel;
939 					if (config.build.components.dmd.dmdModel == "32mscoff")
940 					{
941 						auto objFiles = dmdMakeFullName.readText().splitLines().filter!(line => line.startsWith("OBJ_MSVC="));
942 						enforce(!objFiles.empty, "Can't find OBJ_MSVC in win64.mak");
943 						extraArgs ~= "OBJ_MSVC=" ~ objFiles.front.findSplit("=")[2].split().filter!(obj => obj != "ldfpu.obj").join(" ");
944 					}
945 				}
946 				else
947 					throw new Exception("dmdModel is only supported on Windows");
948 			}
949 
950 			// Avoid HOST_DC reading ~/dmd.conf
951 			string hostDC = env.deps.hostDC;
952 			version (Posix)
953 			if (hostDC && needConfSwitch())
954 			{
955 				auto dcProxy = buildPath(config.local.workDir, "host-dc-proxy.sh");
956 				std.file.write(dcProxy, escapeShellCommand(["exec", hostDC, "-conf=" ~ buildPath(dirName(hostDC), configFileName)]) ~ ` "$@"`);
957 				setAttributes(dcProxy, octal!755);
958 				hostDC = dcProxy;
959 			}
960 
961 			run(getMake(env) ~ [
962 					"-f", dmdMakeFileName,
963 					"MODEL=" ~ modelFlag,
964 					"HOST_DC=" ~ hostDC,
965 				] ~ config.build.components.common.makeArgs ~ dMakeArgs ~ extraArgs ~ targets,
966 				env.vars, srcDir
967 			);
968 		}
969 
970 		override void performStage()
971 		{
972 			if (config.build.components.dmd.useVC)
973 			{
974 				foreach (ext; [".exe", ".pdb"])
975 					cp(
976 						buildPath(sourceDir, "src", "vcbuild", vsPlatform, vsConfiguration, "dmd_msc" ~ ext),
977 						buildPath(stageDir , "bin", "dmd" ~ ext),
978 					);
979 			}
980 			else
981 			{
982 				cp(
983 					buildPath(sourceDir, "src", "dmd" ~ binExt),
984 					buildPath(stageDir , "bin", "dmd" ~ binExt),
985 				);
986 			}
987 
988 			version (Windows)
989 			{
990 				auto ini = q"EOS
991 [Environment]
992 LIB="%@P%\..\lib"
993 DFLAGS="-I%@P%\..\import"
994 DMC=__DMC__
995 LINKCMD=%DMC%\link.exe
996 
997 [Environment64]
998 LIB="%@P%\..\lib"
999 DFLAGS=%DFLAGS% -L/OPT:NOICF
1000 VSINSTALLDIR=__VS__\
1001 VCINSTALLDIR=%VSINSTALLDIR%VC\
1002 PATH=%PATH%;%VCINSTALLDIR%\bin\amd64
1003 WindowsSdkDir=__SDK__
1004 LINKCMD=%VCINSTALLDIR%\bin\amd64\link.exe
1005 LIB=%LIB%;"%VCINSTALLDIR%\lib\amd64"
1006 LIB=%LIB%;"%WindowsSdkDir%\Lib\x64"
1007 
1008 [Environment32mscoff]
1009 LIB="%@P%\..\lib"
1010 DFLAGS=%DFLAGS% -L/OPT:NOICF
1011 VSINSTALLDIR=__VS__\
1012 VCINSTALLDIR=%VSINSTALLDIR%VC\
1013 PATH=%PATH%;%VCINSTALLDIR%\bin
1014 WindowsSdkDir=__SDK__
1015 LINKCMD=%VCINSTALLDIR%\bin\link.exe
1016 LIB=%LIB%;"%VCINSTALLDIR%\lib"
1017 LIB=%LIB%;"%WindowsSdkDir%\Lib"
1018 EOS";
1019 
1020 				auto env = baseEnvironment;
1021 				needCC(env, config.build.components.common.model);
1022 
1023 				ini = ini.replace("__DMC__", env.deps.dmcDir.buildPath(`bin`).absolutePath());
1024 				ini = ini.replace("__VS__" , env.deps.vsDir .absolutePath());
1025 				ini = ini.replace("__SDK__", env.deps.sdkDir.absolutePath());
1026 
1027 				buildPath(stageDir, "bin", configFileName).write(ini);
1028 			}
1029 			else version (OSX)
1030 			{
1031 				auto ini = q"EOS
1032 [Environment]
1033 DFLAGS="-I%@P%/../import" "-L-L%@P%/../lib"
1034 EOS";
1035 				buildPath(stageDir, "bin", configFileName).write(ini);
1036 			}
1037 			else
1038 			{
1039 				auto ini = q"EOS
1040 [Environment]
1041 DFLAGS="-I%@P%/../import" "-L-L%@P%/../lib" -L--export-dynamic
1042 EOS";
1043 				buildPath(stageDir, "bin", configFileName).write(ini);
1044 			}
1045 		}
1046 
1047 		override void updateEnv(ref Environment env)
1048 		{
1049 			// Add the DMD we built for Phobos/Druntime/Tools
1050 			env.vars["PATH"] = buildPath(buildDir, "bin").absolutePath() ~ pathSeparator ~ env.vars["PATH"];
1051 		}
1052 
1053 		override void performTest()
1054 		{
1055 			foreach (dep; ["dmd", "druntime", "phobos"])
1056 				getComponent(dep).needBuild();
1057 
1058 			auto env = baseEnvironment;
1059 			version (Windows)
1060 			{
1061 				// In this order so it uses the MSYS make
1062 				needCC(env, config.build.components.common.model);
1063 				needMSYS(env);
1064 
1065 				disableCrashDialog();
1066 			}
1067 
1068 			auto makeArgs = getMake(env) ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env) ~ gnuMakeArgs;
1069 			version (Windows)
1070 			{
1071 				makeArgs ~= ["OS=win" ~ config.build.components.common.model[0..2], "SHELL=bash"];
1072 				if (config.build.components.common.model == "32")
1073 				{
1074 					auto extrasDir = needExtras();
1075 					// The autotester seems to pass this via environment. Why does that work there???
1076 					makeArgs ~= "LIB=" ~ extrasDir.buildPath("localextras-windows", "dmd2", "windows", "lib") ~ `;..\..\phobos`;
1077 				}
1078 				else
1079 				{
1080 					// Fix path for d_do_test and its special escaping (default is the system VS2010 install)
1081 					// We can't use the same syntax in getPlatformMakeVars because win64.mak uses "CC=\$(CC32)"\""
1082 					auto cl = env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(config.build.components.common.model), "cl.exe");
1083 					foreach (ref arg; makeArgs)
1084 						if (arg.startsWith("CC="))
1085 							arg = "CC=" ~ dDoTestEscape(cl);
1086 				}
1087 			}
1088 
1089 			version (test)
1090 			{
1091 				// Only try a few tests during CI runs, to check for
1092 				// platform integration and correct invocation.
1093 				// For this purpose, the C++ ABI tests will do nicely.
1094 				makeArgs ~= [
1095 				//	"test_results/runnable/cppa.d.out", // https://github.com/dlang/dmd/pull/5686
1096 					"test_results/runnable/cpp_abi_tests.d.out",
1097 					"test_results/runnable/cabi1.d.out",
1098 				];
1099 			}
1100 
1101 			run(makeArgs, env.vars, sourceDir.buildPath("test"));
1102 		}
1103 	}
1104 
1105 	/// Phobos import files.
1106 	/// In older versions of D, Druntime depended on Phobos modules.
1107 	final class PhobosIncludes : Component
1108 	{
1109 		@property override string submoduleName() { return "phobos"; }
1110 		@property override string[] sourceDependencies() { return []; }
1111 		@property override string[] dependencies() { return []; }
1112 		@property override string configString() { return null; }
1113 
1114 		override void performStage()
1115 		{
1116 			foreach (f; ["std", "etc", "crc32.d"])
1117 				if (buildPath(sourceDir, f).exists)
1118 					cp(
1119 						buildPath(sourceDir, f),
1120 						buildPath(stageDir , "import", f),
1121 					);
1122 		}
1123 	}
1124 
1125 	/// Druntime. Installs only import files, but builds the library too.
1126 	final class Druntime : Component
1127 	{
1128 		@property override string submoduleName    () { return "druntime"; }
1129 		@property override string[] sourceDependencies() { return ["phobos", "phobos-includes"]; }
1130 		@property override string[] dependencies() { return ["dmd"]; }
1131 
1132 		@property override string configString()
1133 		{
1134 			static struct FullConfig
1135 			{
1136 				string model;
1137 				string[] makeArgs;
1138 			}
1139 
1140 			return FullConfig(
1141 				config.build.components.common.model,
1142 				config.build.components.common.makeArgs,
1143 			).toJson();
1144 		}
1145 
1146 		override void performBuild()
1147 		{
1148 			getComponent("phobos").needSource();
1149 			getComponent("dmd").needInstalled();
1150 			getComponent("phobos-includes").needInstalled();
1151 
1152 			auto env = baseEnvironment;
1153 			needCC(env, config.build.components.common.model);
1154 
1155 			mkdirRecurse(sourceDir.buildPath("import"));
1156 			mkdirRecurse(sourceDir.buildPath("lib"));
1157 
1158 			setTimes(sourceDir.buildPath("src", "rt", "minit.obj"), Clock.currTime(), Clock.currTime()); // Don't rebuild
1159 			submodule.saveFileState("src/rt/minit.obj");
1160 
1161 			run(getMake(env) ~ ["-f", makeFileNameModel, "import", "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env) ~ dMakeArgs, env.vars, sourceDir);
1162 			run(getMake(env) ~ ["-f", makeFileNameModel          , "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env) ~ dMakeArgs, env.vars, sourceDir);
1163 		}
1164 
1165 		override void performStage()
1166 		{
1167 			cp(
1168 				buildPath(sourceDir, "import"),
1169 				buildPath(stageDir , "import"),
1170 			);
1171 		}
1172 
1173 		override void performTest()
1174 		{
1175 			getComponent("druntime").needBuild();
1176 			getComponent("dmd").needInstalled();
1177 
1178 			auto env = baseEnvironment;
1179 			needCC(env, config.build.components.common.model);
1180 			run(getMake(env) ~ ["-f", makeFileNameModel, "unittest", "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env) ~ dMakeArgs, env.vars, sourceDir);
1181 		}
1182 	}
1183 
1184 	/// Phobos library and imports.
1185 	final class Phobos : Component
1186 	{
1187 		@property override string submoduleName    () { return "phobos"; }
1188 		@property override string[] sourceDependencies() { return []; }
1189 		@property override string[] dependencies() { return ["druntime", "dmd"]; }
1190 
1191 		@property override string configString()
1192 		{
1193 			static struct FullConfig
1194 			{
1195 				string model;
1196 				string[] makeArgs;
1197 			}
1198 
1199 			return FullConfig(
1200 				config.build.components.common.model,
1201 				config.build.components.common.makeArgs,
1202 			).toJson();
1203 		}
1204 
1205 		string[] targets;
1206 
1207 		override void performBuild()
1208 		{
1209 			getComponent("dmd").needInstalled();
1210 			getComponent("druntime").needBuild();
1211 
1212 			auto env = baseEnvironment;
1213 			needCC(env, config.build.components.common.model);
1214 
1215 			string phobosMakeFileName = findMakeFile(sourceDir, makeFileNameModel);
1216 			string phobosMakeFullName = sourceDir.buildPath(phobosMakeFileName);
1217 
1218 			auto makeArgs = getMake(env) ~ ["-f", phobosMakeFileName, "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env) ~ dMakeArgs;
1219 
1220 			version (Windows)
1221 			{
1222 				auto lib = "phobos%s.lib".format(modelSuffix);
1223 				run(makeArgs ~ lib, env.vars, sourceDir);
1224 				enforce(sourceDir.buildPath(lib).exists);
1225 				targets = ["phobos%s.lib".format(modelSuffix)];
1226 			}
1227 			else
1228 			{
1229 				if (phobosMakeFullName.readText().canFind("DRUNTIME = $(DRUNTIME_PATH)/lib/libdruntime-$(OS)$(MODEL).a") &&
1230 					getComponent("druntime").sourceDir.buildPath("lib").dirEntries(SpanMode.shallow).walkLength == 0 &&
1231 					exists(getComponent("druntime").sourceDir.buildPath("generated")))
1232 				{
1233 					auto dir = getComponent("druntime").sourceDir.buildPath("generated");
1234 					auto aFile  = dir.dirEntries("libdruntime.a", SpanMode.depth);
1235 					if (!aFile .empty) makeArgs ~= ["DRUNTIME="   ~ aFile .front];
1236 					auto soFile = dir.dirEntries("libdruntime.so.a", SpanMode.depth);
1237 					if (!soFile.empty) makeArgs ~= ["DRUNTIMESO=" ~ soFile.front];
1238 				}
1239 				run(makeArgs, env.vars, sourceDir);
1240 				targets = sourceDir
1241 					.buildPath("generated")
1242 					.dirEntries(SpanMode.depth)
1243 					.filter!(de => de.name.endsWith(".a") || de.name.endsWith(".so"))
1244 					.map!(de => de.name.relativePath(sourceDir))
1245 					.array()
1246 				;
1247 			}
1248 		}
1249 
1250 		override void performStage()
1251 		{
1252 			assert(targets.length, "Druntime stage without build");
1253 			foreach (lib; targets)
1254 				cp(
1255 					buildPath(sourceDir, lib),
1256 					buildPath(stageDir , "lib", lib.baseName()),
1257 				);
1258 		}
1259 
1260 		override void performTest()
1261 		{
1262 			getComponent("druntime").needBuild();
1263 			getComponent("phobos").needBuild();
1264 			getComponent("dmd").needInstalled();
1265 
1266 			auto env = baseEnvironment;
1267 			needCC(env, config.build.components.common.model);
1268 			version (Windows)
1269 			{
1270 				getComponent("curl").needInstalled();
1271 				getComponent("curl").updateEnv(env);
1272 
1273 				// Patch out std.datetime unittest to work around Digger test
1274 				// suite failure on AppVeyor due to Windows time zone changes
1275 				auto stdDateTime = buildPath(sourceDir, "std", "datetime.d");
1276 				if (stdDateTime.exists && !stdDateTime.readText().canFind("Altai Standard Time"))
1277 				{
1278 					auto m = stdDateTime.readText();
1279 					m = m
1280 						.replace(`assert(tzName !is null, format("TZName which is missing: %s", winName));`, ``)
1281 						.replace(`assert(tzDatabaseNameToWindowsTZName(tzName) !is null, format("TZName which failed: %s", tzName));`, `{}`)
1282 						.replace(`assert(windowsTZNameToTZDatabaseName(tzName) !is null, format("TZName which failed: %s", tzName));`, `{}`)
1283 					;
1284 					stdDateTime.write(m);
1285 					submodule.saveFileState("std/datetime.d");
1286 				}
1287 
1288 				if (config.build.components.common.model == "32")
1289 					getComponent("extras").needInstalled();
1290 			}
1291 			run(getMake(env) ~ ["-f", makeFileNameModel, "unittest", "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env) ~ dMakeArgs, env.vars, sourceDir);
1292 		}
1293 	}
1294 
1295 	/// The rdmd build tool by itself.
1296 	/// It predates the tools package.
1297 	final class RDMD : Component
1298 	{
1299 		@property override string submoduleName() { return "tools"; }
1300 		@property override string[] sourceDependencies() { return []; }
1301 		@property override string[] dependencies() { return ["dmd", "druntime", "phobos"]; }
1302 
1303 		@property override string configString()
1304 		{
1305 			static struct FullConfig
1306 			{
1307 				string model;
1308 			}
1309 
1310 			return FullConfig(
1311 				config.build.components.common.model,
1312 			).toJson();
1313 		}
1314 
1315 		override void performBuild()
1316 		{
1317 			foreach (dep; ["dmd", "druntime", "phobos", "phobos-includes"])
1318 				getComponent(dep).needInstalled();
1319 
1320 			auto env = baseEnvironment;
1321 			needCC(env, config.build.components.common.model);
1322 
1323 			// Just build rdmd
1324 			bool needModel; // Need -mXX switch?
1325 
1326 			if (sourceDir.buildPath("posix.mak").exists)
1327 				needModel = true; // Known to be needed for recent versions
1328 
1329 			string[] args;
1330 			if (needConfSwitch())
1331 				args ~= ["-conf=" ~ buildPath(buildDir , "bin", configFileName)];
1332 			args ~= ["rdmd"];
1333 
1334 			if (!needModel)
1335 				try
1336 					run([dmd] ~ args, env.vars, sourceDir);
1337 				catch (Exception e)
1338 					needModel = true;
1339 
1340 			if (needModel)
1341 				run([dmd, "-m" ~ config.build.components.common.model] ~ args, env.vars, sourceDir);
1342 		}
1343 
1344 		override void performStage()
1345 		{
1346 			cp(
1347 				buildPath(sourceDir, "rdmd" ~ binExt),
1348 				buildPath(stageDir , "bin", "rdmd" ~ binExt),
1349 			);
1350 		}
1351 
1352 		override void performTest()
1353 		{
1354 			version (Windows)
1355 				if (config.build.components.common.model != "32")
1356 				{
1357 					// Can't test rdmd on non-32-bit Windows until compiler model matches Phobos model.
1358 					// rdmd_test does not use -m when building rdmd, thus linking will fail
1359 					// (because of model mismatch with the phobos we built).
1360 					log("Can't test rdmd with model " ~ config.build.components.common.model ~ ", skipping");
1361 					return;
1362 				}
1363 
1364 			foreach (dep; ["dmd", "druntime", "phobos", "phobos-includes"])
1365 				getComponent(dep).needInstalled();
1366 
1367 			auto env = baseEnvironment;
1368 			getComponent("dmd").updateEnv(env);
1369 			run(["dmd", "-run", "rdmd_test.d"], env.vars, sourceDir);
1370 		}
1371 	}
1372 
1373 	/// Tools package with all its components, including rdmd.
1374 	final class Tools : Component
1375 	{
1376 		@property override string submoduleName() { return "tools"; }
1377 		@property override string[] sourceDependencies() { return []; }
1378 		@property override string[] dependencies() { return ["dmd", "druntime", "phobos"]; }
1379 
1380 		@property override string configString()
1381 		{
1382 			static struct FullConfig
1383 			{
1384 				string model;
1385 				string[] makeArgs;
1386 			}
1387 
1388 			return FullConfig(
1389 				config.build.components.common.model,
1390 				config.build.components.common.makeArgs,
1391 			).toJson();
1392 		}
1393 
1394 		override void performBuild()
1395 		{
1396 			foreach (dep; ["dmd", "druntime", "phobos"])
1397 				getComponent(dep).needInstalled();
1398 
1399 			auto env = baseEnvironment;
1400 			needCC(env, config.build.components.common.model);
1401 
1402 			run(getMake(env) ~ ["-f", makeFileName, "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env) ~ dMakeArgs, env.vars, sourceDir);
1403 		}
1404 
1405 		override void performStage()
1406 		{
1407 			foreach (os; buildPath(sourceDir, "generated").dirEntries(SpanMode.shallow))
1408 			{
1409 				auto dir = os.buildPath(config.build.components.common.model);
1410 				cp(dir, buildPath(stageDir , "bin"));
1411 			}
1412 		}
1413 	}
1414 
1415 	/// Website (dlang.org). Only buildable on POSIX.
1416 	final class Website : Component
1417 	{
1418 		@property override string submoduleName() { return "dlang.org"; }
1419 		@property override string[] sourceDependencies() { return ["druntime", "phobos"]; }
1420 		@property override string[] dependencies() { return ["dmd", "druntime", "phobos", "rdmd"]; }
1421 
1422 		struct Config
1423 		{
1424 			/// Do not include a timestamp in generated .ddoc files.
1425 			/// Improves cache efficiency and allows meaningful diffs.
1426 			bool noDateTime = false;
1427 		}
1428 
1429 		@property override string configString()
1430 		{
1431 			static struct FullConfig
1432 			{
1433 				Config config;
1434 			}
1435 
1436 			return FullConfig(
1437 				config.build.components.website,
1438 			).toJson();
1439 		}
1440 
1441 		/// Get the latest version of DMD at the time.
1442 		/// Needed for the makefile's "LATEST" parameter.
1443 		string getLatest()
1444 		{
1445 			auto dmd = getComponent("dmd").submodule;
1446 			dmd.needRepo();
1447 
1448 			auto t = dmd.git.query(["log", "--pretty=format:%ct"]).splitLines.map!(to!int).filter!(n => n > 0).front;
1449 
1450 			foreach (line; dmd.git.query(["log", "--decorate=full", "--tags", "--pretty=format:%ct%d"]).splitLines())
1451 				if (line.length > 10 && line[0..10].to!int < t)
1452 					if (line[10..$].startsWith(" (") && line.endsWith(")"))
1453 					{
1454 						foreach (r; line[12..$-1].split(", "))
1455 							if (r.skipOver("tag: refs/tags/"))
1456 								if (r.match(re!`^v2\.\d\d\d(\.\d)?$`))
1457 									return r[1..$];
1458 					}
1459 			throw new Exception("Can't find any DMD version tags at this point!");
1460 		}
1461 
1462 		override void performBuild()
1463 		{
1464 			auto env = baseEnvironment;
1465 
1466 			version (Windows)
1467 				throw new Exception("The dlang.org website is only buildable on POSIX platforms.");
1468 			else
1469 			{
1470 				foreach (dep; ["dmd", "druntime", "phobos"])
1471 				{
1472 					auto c = getComponent(dep);
1473 					c.needInstalled();
1474 
1475 					// Need DMD source because https://github.com/dlang/phobos/pull/4613#issuecomment-266462596
1476 					// Need Druntime/Phobos source because we are building its documentation from there.
1477 					c.needSource();
1478 				}
1479 				getComponent("tools").needSource(); // for changed.d
1480 				getComponent("dmd").updateEnv(env);
1481 
1482 				needKindleGen(env);
1483 
1484 				foreach (dep; dependencies)
1485 					getComponent(dep).submodule.clean = false;
1486 
1487 				auto makeFullName = sourceDir.buildPath(makeFileName);
1488 				makeFullName
1489 					.readText()
1490 					// https://github.com/D-Programming-Language/dlang.org/pull/1011
1491 					.replace(": modlist.d", ": modlist.d $(DMD)")
1492 					// https://github.com/D-Programming-Language/dlang.org/pull/1017
1493 					.replace("dpl-docs: ${DUB} ${STABLE_DMD}\n\tDFLAGS=", "dpl-docs: ${DUB} ${STABLE_DMD}\n\t${DUB} upgrade --missing-only --root=${DPL_DOCS_PATH}\n\tDFLAGS=")
1494 					.toFile(makeFullName)
1495 				;
1496 				submodule.saveFileState(makeFileName);
1497 				scope(exit) submodule.saveFileState("dpl-docs/dub.selections.json");
1498 
1499 				string latest = null;
1500 				if (!sourceDir.buildPath("VERSION").exists)
1501 				{
1502 					latest = getLatest();
1503 					log("LATEST=" ~ latest);
1504 				}
1505 				else
1506 					log("VERSION file found, not passing LATEST parameter");
1507 
1508 				auto args =
1509 					getMake(env) ~
1510 					[ "-f", makeFileName ] ~
1511 					(config.build.components.website.noDateTime ? ["NODATETIME=nodatetime.ddoc"] : []) ~
1512 					(latest ? ["LATEST=" ~ latest] : []) ~
1513 					[ "all", "kindle", "pdf", "verbatim" ] ~
1514 					gnuMakeArgs;
1515 				run(args, env.vars, sourceDir);
1516 			}
1517 		}
1518 
1519 		override void performStage()
1520 		{
1521 			foreach (item; ["web", "dlangspec.tex", "dlangspec.html"])
1522 				cp(
1523 					buildPath(sourceDir, item),
1524 					buildPath(stageDir , item),
1525 				);
1526 		}
1527 	}
1528 
1529 	/// Extras not built from source (DigitalMars and third-party tools and libraries)
1530 	final class Extras : Component
1531 	{
1532 		@property override string submoduleName() { return null; }
1533 		@property override string[] sourceDependencies() { return []; }
1534 		@property override string[] dependencies() { return []; }
1535 		@property override string configString() { return null; }
1536 
1537 		override void performBuild()
1538 		{
1539 			needExtras();
1540 		}
1541 
1542 		override void performStage()
1543 		{
1544 			version (Windows)
1545 				enum platform = "windows";
1546 			else
1547 			version (linux)
1548 				enum platform = "linux";
1549 			else
1550 			version (OSX)
1551 				enum platform = "osx";
1552 			else
1553 			version (FreeBSD)
1554 				enum platform = "freebsd";
1555 			else
1556 				static assert(false);
1557 
1558 			auto extrasDir = needExtras();
1559 
1560 			void copyDir(string source, string target)
1561 			{
1562 				source = buildPath(extrasDir, "localextras-" ~ platform, "dmd2", platform, source);
1563 				target = buildPath(stageDir, target);
1564 				if (source.exists)
1565 					cp(source, target);
1566 			}
1567 
1568 			copyDir("bin", "bin");
1569 			copyDir("bin" ~ config.build.components.common.model, "bin");
1570 			copyDir("lib", "lib");
1571 
1572 			version (Windows)
1573 				if (config.build.components.common.model == "32")
1574 				{
1575 					// The version of snn.lib bundled with DMC will be newer.
1576 					Environment env;
1577 					needDMC(env);
1578 					cp(buildPath(env.deps.dmcDir, "lib", "snn.lib"), buildPath(stageDir, "lib", "snn.lib"));
1579 				}
1580 		}
1581 	}
1582 
1583 	/// libcurl DLL and import library for Windows.
1584 	final class Curl : Component
1585 	{
1586 		@property override string submoduleName() { return null; }
1587 		@property override string[] sourceDependencies() { return []; }
1588 		@property override string[] dependencies() { return []; }
1589 		@property override string configString() { return null; }
1590 
1591 		override void performBuild()
1592 		{
1593 			version (Windows)
1594 				needCurl();
1595 			else
1596 				log("Not on Windows, skipping libcurl download");
1597 		}
1598 
1599 		override void performStage()
1600 		{
1601 			version (Windows)
1602 			{
1603 				auto curlDir = needCurl();
1604 
1605 				void copyDir(string source, string target)
1606 				{
1607 					source = buildPath(curlDir, "dmd2", "windows", source);
1608 					target = buildPath(stageDir, target);
1609 					if (source.exists)
1610 						cp(source, target);
1611 				}
1612 
1613 				auto suffix = config.build.components.common.model == "64" ? "64" : "";
1614 				copyDir("bin" ~ suffix, "bin");
1615 				copyDir("lib" ~ suffix, "lib");
1616 			}
1617 			else
1618 				log("Not on Windows, skipping libcurl install");
1619 		}
1620 
1621 		override void updateEnv(ref Environment env)
1622 		{
1623 			env.vars["PATH"] = buildPath(buildDir, "bin").absolutePath() ~ pathSeparator ~ env.vars["PATH"];
1624 		}
1625 	}
1626 
1627 	private int tempError;
1628 
1629 	private Component[string] components;
1630 
1631 	Component getComponent(string name)
1632 	{
1633 		if (name !in components)
1634 		{
1635 			Component c;
1636 
1637 			switch (name)
1638 			{
1639 				case "dmd":
1640 					c = new DMD();
1641 					break;
1642 				case "phobos-includes":
1643 					c = new PhobosIncludes();
1644 					break;
1645 				case "druntime":
1646 					c = new Druntime();
1647 					break;
1648 				case "phobos":
1649 					c = new Phobos();
1650 					break;
1651 				case "rdmd":
1652 					c = new RDMD();
1653 					break;
1654 				case "tools":
1655 					c = new Tools();
1656 					break;
1657 				case "website":
1658 					c = new Website();
1659 					break;
1660 				case "extras":
1661 					c = new Extras();
1662 					break;
1663 				case "curl":
1664 					c = new Curl();
1665 					break;
1666 				default:
1667 					throw new Exception("Unknown component: " ~ name);
1668 			}
1669 
1670 			c.name = name;
1671 			return components[name] = c;
1672 		}
1673 
1674 		return components[name];
1675 	}
1676 
1677 	Component[] getSubmoduleComponents(string submoduleName)
1678 	{
1679 		return components
1680 			.byValue
1681 			.filter!(component => component.submoduleName == submoduleName)
1682 			.array();
1683 	}
1684 
1685 	// **************************** Customization ****************************
1686 
1687 	/// Fetch latest D history.
1688 	void update()
1689 	{
1690 		getMetaRepo().update();
1691 	}
1692 
1693 	struct SubmoduleState
1694 	{
1695 		string[string] submoduleCommits;
1696 	}
1697 
1698 	/// Begin customization, starting at the specified commit.
1699 	SubmoduleState begin(string commit)
1700 	{
1701 		log("Starting at meta repository commit " ~ commit);
1702 		return SubmoduleState(getMetaRepo().getSubmoduleCommits(commit));
1703 	}
1704 
1705 	/// Applies a merge onto the given SubmoduleState.
1706 	void merge(ref SubmoduleState submoduleState, string submoduleName, string branch)
1707 	{
1708 		log("Merging %s commit %s".format(submoduleName, branch));
1709 		enforce(submoduleName in submoduleState.submoduleCommits, "Unknown submodule: " ~ submoduleName);
1710 		auto submodule = getSubmodule(submoduleName);
1711 		auto head = submoduleState.submoduleCommits[submoduleName];
1712 		auto result = submodule.getMerge(head, branch);
1713 		submoduleState.submoduleCommits[submoduleName] = result;
1714 	}
1715 
1716 	/// Removes a merge from the given SubmoduleState.
1717 	void unmerge(ref SubmoduleState submoduleState, string submoduleName, string branch)
1718 	{
1719 		log("Unmerging %s commit %s".format(submoduleName, branch));
1720 		enforce(submoduleName in submoduleState.submoduleCommits, "Unknown submodule: " ~ submoduleName);
1721 		auto submodule = getSubmodule(submoduleName);
1722 		auto head = submoduleState.submoduleCommits[submoduleName];
1723 		auto result = submodule.getUnMerge(head, branch);
1724 		submoduleState.submoduleCommits[submoduleName] = result;
1725 	}
1726 
1727 	/// Reverts a commit from the given SubmoduleState.
1728 	/// parent is the 1-based mainline index (as per `man git-revert`),
1729 	/// or 0 if commit is not a merge commit.
1730 	void revert(ref SubmoduleState submoduleState, string submoduleName, string commit, int parent)
1731 	{
1732 		log("Reverting %s commit %s".format(submoduleName, commit));
1733 		enforce(submoduleName in submoduleState.submoduleCommits, "Unknown submodule: " ~ submoduleName);
1734 		auto submodule = getSubmodule(submoduleName);
1735 		auto head = submoduleState.submoduleCommits[submoduleName];
1736 		auto result = submodule.getRevert(head, commit, parent);
1737 		submoduleState.submoduleCommits[submoduleName] = result;
1738 	}
1739 
1740 	/// Returns the commit hash for the given pull request #.
1741 	/// The result can then be used with addMerge/removeMerge.
1742 	string getPull(string submoduleName, int pullNumber)
1743 	{
1744 		return getSubmodule(submoduleName).getPull(pullNumber);
1745 	}
1746 
1747 	/// Returns the commit hash for the given GitHub fork.
1748 	/// The result can then be used with addMerge/removeMerge.
1749 	string getFork(string submoduleName, string user, string branch)
1750 	{
1751 		return getSubmodule(submoduleName).getFork(user, branch);
1752 	}
1753 
1754 	/// Find the child of a commit (starting with the current submodule state),
1755 	/// and, if the commit was a merge, the mainline index of said commit for the child.
1756 	void getChild(ref SubmoduleState submoduleState, string submoduleName, string commit, out string child, out int mainline)
1757 	{
1758 		enforce(submoduleName in submoduleState.submoduleCommits, "Unknown submodule: " ~ submoduleName);
1759 		auto head = submoduleState.submoduleCommits[submoduleName];
1760 		return getSubmodule(submoduleName).getChild(head, commit, child, mainline);
1761 	}
1762 
1763 	// ****************************** Building *******************************
1764 
1765 	private SubmoduleState submoduleState;
1766 	private bool incrementalBuild;
1767 
1768 	@property string cacheEngineName()
1769 	{
1770 		if (incrementalBuild)
1771 			return "none";
1772 		else
1773 			return config.local.cache;
1774 	}
1775 
1776 	private string getComponentCommit(string componentName)
1777 	{
1778 		auto submoduleName = getComponent(componentName).submoduleName;
1779 		auto commit = submoduleState.submoduleCommits.get(submoduleName, null);
1780 		enforce(commit, "Unknown commit to build for component %s (submodule %s)"
1781 			.format(componentName, submoduleName));
1782 		return commit;
1783 	}
1784 
1785 	static const string[] defaultComponents = ["dmd", "druntime", "phobos-includes", "phobos", "rdmd"];
1786 	static const string[] additionalComponents = ["tools", "website", "extras", "curl"];
1787 	static const string[] allComponents = defaultComponents ~ additionalComponents;
1788 
1789 	/// Build the specified components according to the specified configuration.
1790 	void build(SubmoduleState submoduleState, bool incremental = false)
1791 	{
1792 		auto componentNames = config.build.components.getEnabledComponentNames();
1793 		log("Building components %-(%s, %)".format(componentNames));
1794 
1795 		this.components = null;
1796 		this.submoduleState = submoduleState;
1797 		this.incrementalBuild = incremental;
1798 
1799 		if (buildDir.exists)
1800 			buildDir.removeRecurse();
1801 		enforce(!buildDir.exists);
1802 
1803 		scope(exit) if (cacheEngine) cacheEngine.finalize();
1804 
1805 		foreach (componentName; componentNames)
1806 			getComponent(componentName).needInstalled();
1807 	}
1808 
1809 	/// Shortcut for begin + build
1810 	void buildRev(string rev)
1811 	{
1812 		auto submoduleState = begin(rev);
1813 		build(submoduleState);
1814 	}
1815 
1816 	/// Simply check out the source code for the given submodules.
1817 	void checkout(SubmoduleState submoduleState)
1818 	{
1819 		auto componentNames = config.build.components.getEnabledComponentNames();
1820 		log("Checking out components %-(%s, %)".format(componentNames));
1821 
1822 		this.components = null;
1823 		this.submoduleState = submoduleState;
1824 		this.incrementalBuild = false;
1825 
1826 		foreach (componentName; componentNames)
1827 			getComponent(componentName).needSource();
1828 	}
1829 
1830 	/// Rerun build without cleaning up any files.
1831 	void rebuild()
1832 	{
1833 		build(SubmoduleState(null), true);
1834 	}
1835 
1836 	/// Run all tests for the current checkout (like rebuild).
1837 	void test()
1838 	{
1839 		auto componentNames = config.build.components.getEnabledComponentNames();
1840 		log("Testing components %-(%s, %)".format(componentNames));
1841 
1842 		this.components = null;
1843 		this.submoduleState = SubmoduleState(null);
1844 		this.incrementalBuild = true;
1845 
1846 		foreach (componentName; componentNames)
1847 			getComponent(componentName).test();
1848 	}
1849 
1850 	bool isCached(SubmoduleState submoduleState)
1851 	{
1852 		this.components = null;
1853 		this.submoduleState = submoduleState;
1854 
1855 		needCacheEngine();
1856 		foreach (componentName; config.build.components.getEnabledComponentNames())
1857 			if (!cacheEngine.haveEntry(getComponent(componentName).getBuildID()))
1858 				return false;
1859 		return true;
1860 	}
1861 
1862 	/// Returns the isCached state for all commits in the history of the given ref.
1863 	bool[string] getCacheState(string[string][string] history)
1864 	{
1865 		log("Enumerating cache entries...");
1866 		auto cacheEntries = needCacheEngine().getEntries().toSet();
1867 
1868 		this.components = null;
1869 		auto componentNames = config.build.components.getEnabledComponentNames();
1870 		auto components = componentNames.map!(componentName => getComponent(componentName)).array;
1871 		auto requiredSubmodules = components
1872 			.map!(component => chain(component.name.only, component.sourceDependencies, component.dependencies))
1873 			.joiner
1874 			.map!(componentName => getComponent(componentName).submoduleName)
1875 			.array.sort().uniq().array
1876 		;
1877 
1878 		log("Collating cache state...");
1879 		bool[string] result;
1880 		foreach (commit, submoduleCommits; history)
1881 		{
1882 			this.submoduleState.submoduleCommits = submoduleCommits;
1883 
1884 			result[commit] =
1885 				requiredSubmodules.all!(submoduleName => submoduleName in submoduleCommits) &&
1886 				componentNames.all!(componentName =>
1887 					getComponent(componentName).I!(component =>
1888 						component.getBuildID() in cacheEntries
1889 					)
1890 				);
1891 		}
1892 		return result;
1893 	}
1894 
1895 	/// ditto
1896 	bool[string] getCacheState(string[] refs)
1897 	{
1898 		auto history = getMetaRepo().getSubmoduleHistory(refs);
1899 		return getCacheState(history);
1900 	}
1901 
1902 	// **************************** Dependencies *****************************
1903 
1904 	private void needInstaller()
1905 	{
1906 		Installer.logger = &log;
1907 		Installer.installationDirectory = dlDir;
1908 	}
1909 
1910 	/// Pull in a built DMD as configured.
1911 	/// Note that this function invalidates the current repository state.
1912 	void needDMD(ref Environment env)
1913 	{
1914 		tempError++; scope(success) tempError--;
1915 
1916 		auto dmdVer = config.build.components.dmd.bootstrap.ver;
1917 		if (!dmdVer)
1918 		{
1919 			dmdVer = "v2.067.1";
1920 			version (Windows)
1921 				if (config.build.components.dmd.dmdModel != Component.CommonConfig.defaultModel)
1922 					dmdVer = "v2.070.2"; // dmd/src/builtin.d needs core.stdc.math.fabsl. 2.068.2 generates a dmd which crashes on building Phobos
1923 		}
1924 
1925 		if (config.build.components.dmd.bootstrap.fromSource)
1926 		{
1927 			log("Bootstrapping DMD " ~ dmdVer);
1928 
1929 			// Back up and clear component state
1930 			enum backupTemplate = q{
1931 				auto VARBackup = this.VAR;
1932 				scope(exit) this.VAR = VARBackup;
1933 			};
1934 			mixin(backupTemplate.replace(q{VAR}, q{components}));
1935 			mixin(backupTemplate.replace(q{VAR}, q{config}));
1936 			mixin(backupTemplate.replace(q{VAR}, q{submoduleState}));
1937 
1938 			components = null;
1939 			if (config.build.components.dmd.bootstrap.build)
1940 				config.build = *config.build.components.dmd.bootstrap.build;
1941 			build(parseSpec(dmdVer));
1942 
1943 			auto bootstrapDir = buildPath(config.local.workDir, "bootstrap");
1944 			if (bootstrapDir.exists)
1945 				bootstrapDir.removeRecurse();
1946 			ensurePathExists(bootstrapDir);
1947 			rename(buildDir, bootstrapDir);
1948 
1949 			env.deps.hostDC = buildPath(bootstrapDir, "bin", "dmd" ~ binExt);
1950 		}
1951 		else
1952 		{
1953 			log("Preparing DMD " ~ dmdVer);
1954 			enforce(dmdVer.startsWith("v"), "Invalid DMD version spec for binary bootstrap. Did you forget to enable fromSource?");
1955 			needInstaller();
1956 			auto dmdInstaller = new DMDInstaller(dmdVer[1..$]);
1957 			dmdInstaller.requireLocal(false);
1958 			env.deps.hostDC = dmdInstaller.exePath("dmd").absolutePath();
1959 		}
1960 
1961 		log("hostDC=" ~ env.deps.hostDC);
1962 	}
1963 
1964 	void needKindleGen(ref Environment env)
1965 	{
1966 		needInstaller();
1967 		kindleGenInstaller.requireLocal(false);
1968 		env.vars["PATH"] = kindleGenInstaller.directory ~ pathSeparator ~ env.vars["PATH"];
1969 	}
1970 
1971 	version (Windows)
1972 	void needMSYS(ref Environment env)
1973 	{
1974 		needInstaller();
1975 		MSYS.msysCORE.requireLocal(false);
1976 		MSYS.libintl.requireLocal(false);
1977 		MSYS.libiconv.requireLocal(false);
1978 		MSYS.libtermcap.requireLocal(false);
1979 		MSYS.libregex.requireLocal(false);
1980 		MSYS.coreutils.requireLocal(false);
1981 		MSYS.bash.requireLocal(false);
1982 		MSYS.make.requireLocal(false);
1983 		MSYS.grep.requireLocal(false);
1984 		MSYS.sed.requireLocal(false);
1985 		MSYS.diffutils.requireLocal(false);
1986 		env.vars["PATH"] = MSYS.bash.directory.buildPath("bin") ~ pathSeparator ~ env.vars["PATH"];
1987 	}
1988 
1989 	/// Get DMD unbuildable extras
1990 	/// (proprietary DigitalMars utilities, 32-bit import libraries)
1991 	string needExtras()
1992 	{
1993 		import ae.utils.meta : I, singleton;
1994 
1995 		static class DExtrasInstaller : Installer
1996 		{
1997 			@property override string name() { return "dmd-localextras"; }
1998 			string url = "http://semitwist.com/download/app/dmd-localextras.7z";
1999 
2000 			override void installImpl(string target)
2001 			{
2002 				url
2003 					.I!save()
2004 					.I!unpackTo(target);
2005 			}
2006 
2007 			static this()
2008 			{
2009 				urlDigests["http://semitwist.com/download/app/dmd-localextras.7z"] = "ef367c2d25d4f19f45ade56ab6991c726b07d3d9";
2010 			}
2011 		}
2012 
2013 		alias extrasInstaller = singleton!DExtrasInstaller;
2014 
2015 		needInstaller();
2016 		extrasInstaller.requireLocal(false);
2017 		return extrasInstaller.directory;
2018 	}
2019 
2020 	/// Get libcurl for Windows (DLL and import libraries)
2021 	version (Windows)
2022 	string needCurl()
2023 	{
2024 		import ae.utils.meta : I, singleton;
2025 
2026 		static class DCurlInstaller : Installer
2027 		{
2028 			@property override string name() { return "libcurl-" ~ curlVersion; }
2029 			string curlVersion = "7.47.1";
2030 			@property string url() { return "http://downloads.dlang.org/other/libcurl-" ~ curlVersion ~ "-WinSSL-zlib-x86-x64.zip"; }
2031 
2032 			override void installImpl(string target)
2033 			{
2034 				url
2035 					.I!save()
2036 					.I!unpackTo(target);
2037 			}
2038 
2039 			static this()
2040 			{
2041 				urlDigests["http://downloads.dlang.org/other/libcurl-7.47.1-WinSSL-zlib-x86-x64.zip"] = "4b8a7bb237efab25a96588093ae51994c821e097";
2042 			}
2043 		}
2044 
2045 		alias curlInstaller = singleton!DCurlInstaller;
2046 
2047 		needInstaller();
2048 		curlInstaller.requireLocal(false);
2049 		return curlInstaller.directory;
2050 	}
2051 
2052 	version (Windows)
2053 	void needDMC(ref Environment env, string ver = null)
2054 	{
2055 		tempError++; scope(success) tempError--;
2056 
2057 		needInstaller();
2058 
2059 		auto dmc = ver ? new LegacyDMCInstaller(ver) : dmcInstaller;
2060 		if (!dmc.installedLocally)
2061 			log("Preparing DigitalMars C++ " ~ ver);
2062 		dmc.requireLocal(false);
2063 		env.deps.dmcDir = dmc.directory;
2064 
2065 		auto binPath = buildPath(env.deps.dmcDir, `bin`).absolutePath();
2066 		log("DMC=" ~ binPath);
2067 		env.vars["DMC"] = binPath;
2068 		env.vars["PATH"] = binPath ~ pathSeparator ~ env.vars.get("PATH", null);
2069 	}
2070 
2071 	version (Windows)
2072 	auto getVSInstaller()
2073 	{
2074 		needInstaller();
2075 		return vs2013community;
2076 	}
2077 
2078 	version (Windows)
2079 	static string msvcModelStr(string model, string str32, string str64)
2080 	{
2081 		switch (model)
2082 		{
2083 			case "32":
2084 				throw new Exception("Shouldn't need VC for 32-bit builds");
2085 			case "64":
2086 				return str64;
2087 			case "32mscoff":
2088 				return str32;
2089 			default:
2090 				throw new Exception("Unknown model: " ~ model);
2091 		}
2092 	}
2093 
2094 	version (Windows)
2095 	static string msvcModelDir(string model, string dir64 = "x86_amd64")
2096 	{
2097 		return msvcModelStr(model, null, dir64);
2098 	}
2099 
2100 	version (Windows)
2101 	void needVC(ref Environment env, string model)
2102 	{
2103 		tempError++; scope(success) tempError--;
2104 
2105 		auto vs = getVSInstaller();
2106 
2107 		// At minimum, we want the C compiler (cl.exe) and linker (link.exe).
2108 		vs["vc_compilercore86"].requireLocal(false); // Contains both x86 and x86_amd64 cl.exe
2109 		vs["vc_compilercore86res"].requireLocal(false); // Contains clui.dll needed by cl.exe
2110 
2111 		// Include files. Needed when using VS to build either DMD or Druntime.
2112 		vs["vc_librarycore86"].requireLocal(false); // Contains include files, e.g. errno.h needed by Druntime
2113 
2114 		// C runtime. Needed for all programs built with VC.
2115 		vs[msvcModelStr(model, "vc_libraryDesktop_x86", "vc_libraryDesktop_x64")].requireLocal(false); // libcmt.lib
2116 
2117 		// XP-compatible import libraries.
2118 		vs["win_xpsupport"].requireLocal(false); // shell32.lib
2119 
2120 		// MSBuild, for the useVC option
2121 		if (config.build.components.dmd.useVC)
2122 			vs["Msi_BuildTools_MSBuild_x86"].requireLocal(false); // msbuild.exe
2123 
2124 		// These packages were previously pulled it, but it's not clear why:
2125 		// "vcRuntimeMinimum_x86", "vcRuntimeMinimum_x64", "vc_compilerx64nat", "vc_compilerx64natres"
2126 
2127 		env.deps.vsDir  = vs.directory.buildPath("Program Files (x86)", "Microsoft Visual Studio 12.0").absolutePath();
2128 		env.deps.sdkDir = vs.directory.buildPath("Program Files", "Microsoft SDKs", "Windows", "v7.1A").absolutePath();
2129 
2130 		env.vars["PATH"] ~= pathSeparator ~ vs.modelBinPaths(msvcModelDir(model)).map!(path => vs.directory.buildPath(path).absolutePath()).join(pathSeparator);
2131 		env.vars["VCINSTALLDIR"] = env.deps.vsDir.buildPath("VC") ~ dirSeparator;
2132 		env.vars["INCLUDE"] = env.deps.vsDir.buildPath("VC", "include") ~ ";" ~ env.deps.sdkDir.buildPath("Include");
2133 		env.vars["LIB"] = env.deps.vsDir.buildPath("VC", "lib", msvcModelDir(model, "amd64")) ~ ";" ~ env.deps.sdkDir.buildPath("Lib", msvcModelDir(model, "x64"));
2134 		env.vars["WindowsSdkDir"] = env.deps.sdkDir ~ dirSeparator;
2135 		env.vars["Platform"] = "x64";
2136 		env.vars["LINKCMD64"] = env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "link.exe"); // Used by dmd
2137 		env.vars["MSVC_CC"] = env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "cl.exe"); // For the msvc-dmc wrapper
2138 		env.vars["MSVC_AR"] = env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "lib.exe"); // For the msvc-lib wrapper
2139 		env.vars["CL"] = "-D_USING_V110_SDK71_"; // Work around __userHeader macro redifinition VS bug
2140 	}
2141 
2142 	private void needGit()
2143 	{
2144 		tempError++; scope(success) tempError--;
2145 
2146 		needInstaller();
2147 		gitInstaller.require();
2148 	}
2149 
2150 	/// Disable the "<program> has stopped working"
2151 	/// standard Windows dialog.
2152 	version (Windows)
2153 	static void disableCrashDialog()
2154 	{
2155 		enum : uint { SEM_FAILCRITICALERRORS = 1, SEM_NOGPFAULTERRORBOX = 2 }
2156 		SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX);
2157 	}
2158 
2159 	/// Create a build environment base.
2160 	protected @property Environment baseEnvironment()
2161 	{
2162 		Environment env;
2163 
2164 		// Build a new environment from scratch, to avoid tainting the build with the current environment.
2165 		string[] newPaths;
2166 
2167 		version (Windows)
2168 		{
2169 			import std.utf;
2170 			import ae.sys.windows.imports;
2171 			mixin(importWin32!q{winbase});
2172 			mixin(importWin32!q{winnt});
2173 
2174 			TCHAR[1024] buf;
2175 			// Needed for DLLs
2176 			auto winDir = buf[0..GetWindowsDirectory(buf.ptr, buf.length)].toUTF8();
2177 			auto sysDir = buf[0..GetSystemDirectory (buf.ptr, buf.length)].toUTF8();
2178 			//auto tmpDir = buf[0..GetTempPath(buf.length, buf.ptr)].toUTF8()[0..$-1];
2179 			newPaths ~= [sysDir, winDir];
2180 		}
2181 		else
2182 		{
2183 			// Needed for coreutils, make, gcc, git etc.
2184 			newPaths = ["/bin", "/usr/bin"];
2185 		}
2186 
2187 		env.vars["PATH"] = newPaths.join(pathSeparator);
2188 
2189 		version (Windows)
2190 		{
2191 			auto tmpDir = buildPath(config.local.workDir, "tmp");
2192 			tmpDir.recreateEmptyDirectory();
2193 			env.vars["TEMP"] = env.vars["TMP"] = tmpDir;
2194 			env.vars["SystemDrive"] = winDir.driveName;
2195 			env.vars["SystemRoot"] = winDir;
2196 		}
2197 		else
2198 		{
2199 			auto home = buildPath(config.local.workDir, "home");
2200 			ensureDirExists(home);
2201 			env.vars["HOME"] = home;
2202 		}
2203 
2204 		return env;
2205 	}
2206 
2207 	/// Apply user modifications onto an environment.
2208 	/// Supports Windows-style %VAR% expansions.
2209 	static string[string] applyEnv(in string[string] target, in string[string] source)
2210 	{
2211 		// The source of variable expansions is variables in the target environment,
2212 		// if they exist, and the host environment otherwise, so e.g.
2213 		// `PATH=C:\...;%PATH%` and `MAKE=%MAKE%` work as expected.
2214 		auto oldEnv = std.process.environment.toAA();
2215 		foreach (name, value; target)
2216 			oldEnv[name] = value;
2217 
2218 		string[string] result;
2219 		foreach (name, value; target)
2220 			result[name] = value;
2221 		foreach (name, value; source)
2222 		{
2223 			string newValue = value;
2224 			foreach (oldName, oldValue; oldEnv)
2225 				newValue = newValue.replace("%" ~ oldName ~ "%", oldValue);
2226 			result[name] = oldEnv[name] = newValue;
2227 		}
2228 		return result;
2229 	}
2230 
2231 	// ******************************** Cache ********************************
2232 
2233 	enum unbuildableMarker = "unbuildable";
2234 
2235 	DCache cacheEngine;
2236 
2237 	DCache needCacheEngine()
2238 	{
2239 		if (!cacheEngine)
2240 		{
2241 			if (cacheEngineName == "git")
2242 				needGit();
2243 			cacheEngine = createCache(cacheEngineName, cacheEngineDir(cacheEngineName), this);
2244 		}
2245 		return cacheEngine;
2246 	}
2247 
2248 	void cp(string src, string dst)
2249 	{
2250 		needCacheEngine().cp(src, dst);
2251 	}
2252 
2253 	private string[] getComponentKeyOrder(string componentName)
2254 	{
2255 		auto submodule = getComponent(componentName).submodule;
2256 		submodule.needRepo();
2257 		return submodule
2258 			.git.query("log", "--pretty=format:%H", "--all", "--topo-order")
2259 			.splitLines()
2260 			.map!(commit => componentName ~ "-" ~ commit ~ "-")
2261 			.array
2262 		;
2263 	}
2264 
2265 	string componentNameFromKey(string key)
2266 	{
2267 		auto parts = key.split("-");
2268 		return parts[0..$-2].join("-");
2269 	}
2270 
2271 	string[][] getKeyOrder(string key)
2272 	{
2273 		if (key !is null)
2274 			return [getComponentKeyOrder(componentNameFromKey(key))];
2275 		else
2276 			return allComponents.map!(componentName => getComponentKeyOrder(componentName)).array;
2277 	}
2278 
2279 	/// Optimize entire cache.
2280 	void optimizeCache()
2281 	{
2282 		needCacheEngine().optimize();
2283 	}
2284 
2285 	bool shouldPurge(string key)
2286 	{
2287 		auto files = cacheEngine.listFiles(key);
2288 		if (files.canFind(unbuildableMarker))
2289 			return true;
2290 
2291 		if (componentNameFromKey(key) == "druntime")
2292 		{
2293 			if (!files.canFind("import/core/memory.d")
2294 			 && !files.canFind("import/core/memory.di"))
2295 				return true;
2296 		}
2297 
2298 		return false;
2299 	}
2300 
2301 	/// Delete cached "unbuildable" build results.
2302 	void purgeUnbuildable()
2303 	{
2304 		needCacheEngine()
2305 			.getEntries
2306 			.filter!(key => shouldPurge(key))
2307 			.each!((key)
2308 			{
2309 				log("Deleting: " ~ key);
2310 				cacheEngine.remove(key);
2311 			})
2312 		;
2313 	}
2314 
2315 	/// Move cached files from one cache engine to another.
2316 	void migrateCache(string sourceEngineName, string targetEngineName)
2317 	{
2318 		auto sourceEngine = createCache(sourceEngineName, cacheEngineDir(sourceEngineName), this);
2319 		auto targetEngine = createCache(targetEngineName, cacheEngineDir(targetEngineName), this);
2320 		auto tempDir = buildPath(config.local.workDir, "temp");
2321 		if (tempDir.exists)
2322 			tempDir.removeRecurse();
2323 		log("Enumerating source entries...");
2324 		auto sourceEntries = sourceEngine.getEntries();
2325 		log("Enumerating target entries...");
2326 		auto targetEntries = targetEngine.getEntries().sort();
2327 		foreach (key; sourceEntries)
2328 			if (!targetEntries.canFind(key))
2329 			{
2330 				log(key);
2331 				sourceEngine.extract(key, tempDir, fn => true);
2332 				targetEngine.add(key, tempDir);
2333 				if (tempDir.exists)
2334 					tempDir.removeRecurse();
2335 			}
2336 		targetEngine.optimize();
2337 	}
2338 
2339 	// **************************** Miscellaneous ****************************
2340 
2341 	struct LogEntry
2342 	{
2343 		string hash;
2344 		string[] message;
2345 		SysTime time;
2346 	}
2347 
2348 	/// Gets the D merge log (newest first).
2349 	LogEntry[] getLog(string refName = "refs/remotes/origin/master")
2350 	{
2351 		getMetaRepo().needRepo();
2352 		auto history = getMetaRepo().git.getHistory();
2353 		LogEntry[] logs;
2354 		auto master = history.commits[history.refs[refName]];
2355 		for (auto c = master; c; c = c.parents.length ? c.parents[0] : null)
2356 		{
2357 			auto time = SysTime(c.time.unixTimeToStdTime);
2358 			logs ~= LogEntry(c.hash.toString(), c.message, time);
2359 		}
2360 		return logs;
2361 	}
2362 
2363 	// ***************************** Integration *****************************
2364 
2365 	/// Override to add logging.
2366 	void log(string line)
2367 	{
2368 	}
2369 
2370 	/// Bootstrap description resolution.
2371 	/// See DMD.Config.Bootstrap.spec.
2372 	/// This is essentially a hack to allow the entire
2373 	/// Config structure to be parsed from an .ini file.
2374 	SubmoduleState parseSpec(string spec)
2375 	{
2376 		getMetaRepo().needRepo();
2377 		auto rev = getMetaRepo().getRef("refs/tags/" ~ spec);
2378 		log("Resolved " ~ spec ~ " to " ~ rev);
2379 		return begin(rev);
2380 	}
2381 
2382 	/// Override this method with one which returns a command,
2383 	/// which will invoke the unmergeRebaseEdit function below,
2384 	/// passing to it any additional parameters.
2385 	/// Note: Currently unused. Was previously used
2386 	/// for unmerging things using interactive rebase.
2387 	abstract string getCallbackCommand();
2388 
2389 	void callback(string[] args) { assert(false); }
2390 }