1 /** 2 * Code to manage a D checkout and its dependencies. 3 * 4 * License: 5 * This Source Code Form is subject to the terms of 6 * the Mozilla Public License, v. 2.0. If a copy of 7 * the MPL was not distributed with this file, You 8 * can obtain one at http://mozilla.org/MPL/2.0/. 9 * 10 * Authors: 11 * Vladimir Panteleev <vladimir@thecybershadow.net> 12 */ 13 14 module ae.sys.d.manager; 15 16 import std.algorithm; 17 import std.array; 18 import std.conv; 19 import std.datetime; 20 import std.exception; 21 import std.file; 22 import std.path; 23 import std.process : spawnProcess, wait, escapeShellCommand; 24 import std.range; 25 import std.regex; 26 import std.string; 27 import std.typecons; 28 29 import ae.sys.d.cache; 30 import ae.sys.d.repo; 31 import ae.sys.file; 32 import ae.sys.git; 33 import ae.utils.aa; 34 import ae.utils.array; 35 import ae.utils.digest; 36 import ae.utils.json; 37 import ae.utils.regex; 38 39 alias ensureDirExists = ae.sys.file.ensureDirExists; 40 41 version (Windows) 42 { 43 import ae.sys.install.dmc; 44 import ae.sys.install.msys; 45 import ae.sys.install.vs; 46 47 extern(Windows) void SetErrorMode(int); 48 } 49 50 import ae.sys.install.dmd; 51 import ae.sys.install.git; 52 import ae.sys.install.kindlegen; 53 54 static import std.process; 55 56 /// Class which manages a D checkout and its dependencies. 57 class DManager : ICacheHost 58 { 59 // **************************** Configuration **************************** 60 61 struct Config /// DManager configuration. 62 { 63 struct Build /// Build configuration 64 { 65 struct Components 66 { 67 bool[string] enable; 68 69 string[] getEnabledComponentNames() 70 { 71 foreach (componentName; enable.byKey) 72 enforce(allComponents.canFind(componentName), "Unknown component: " ~ componentName); 73 return allComponents 74 .filter!(componentName => 75 enable.get(componentName, defaultComponents.canFind(componentName))) 76 .array 77 .dup; 78 } 79 80 Component.CommonConfig common; 81 DMD.Config dmd; 82 Website.Config website; 83 } 84 Components components; 85 86 /// Additional environment variables. 87 /// Supports %VAR% expansion - see applyEnv. 88 string[string] environment; 89 } 90 Build build; /// ditto 91 92 /// Machine-local configuration 93 /// These settings should not affect the build output. 94 struct Local 95 { 96 /// URL of D git repository hosting D components. 97 /// Defaults to (and must have the layout of) D.git: 98 /// https://github.com/CyberShadow/D-dot-git 99 string repoUrl = "https://bitbucket.org/cybershadow/d.git"; 100 101 /// Location for the checkout, temporary files, etc. 102 string workDir; 103 104 /// If present, passed to GNU make via -j parameter. 105 /// Can also be "auto" or "unlimited". 106 string makeJobs; 107 108 /// Don't get latest updates from GitHub. 109 bool offline; 110 111 /// How to cache built files. 112 string cache; 113 114 /// Maximum execution time, in seconds, of any single 115 /// command. 116 int timeout; 117 } 118 Local local; /// ditto 119 } 120 Config config; /// ditto 121 122 // Behavior options that generally depend on the host program. 123 124 /// Automatically re-clone the repository in case 125 /// "git reset --hard" fails. 126 bool autoClean; 127 128 /// Whether to verify working tree state 129 /// to make sure we don't clobber user changes 130 bool verifyWorkTree; 131 132 /// Whether we should cache failed builds. 133 bool cacheFailures = true; 134 135 /// Current build environment. 136 struct Environment 137 { 138 struct Deps /// Configuration for software dependencies 139 { 140 string dmcDir; /// Where dmc.zip is unpacked. 141 string vsDir; /// Where Visual Studio is installed 142 string sdkDir; /// Where the Windows SDK is installed 143 string hostDC; /// Host D compiler (for DDMD bootstrapping) 144 } 145 Deps deps; /// ditto 146 147 /// Calculated local environment, incl. dependencies 148 string[string] vars; 149 } 150 151 /// Get a specific subdirectory of the work directory. 152 @property string subDir(string name)() { return buildPath(config.local.workDir, name); } 153 154 alias repoDir = subDir!"repo"; /// The git repository location. 155 alias buildDir = subDir!"build"; /// The build directory. 156 alias dlDir = subDir!"dl"; /// The directory for downloaded software. 157 alias tmpDir = subDir!"tmp"; /// Directory for $TMPDIR etc. 158 alias homeDir = subDir!"home"; /// Directory for $HOME. 159 alias binDir = subDir!"bin" ; /// For wrapper scripts. 160 161 /// This number increases with each incompatible change to cached data. 162 enum cacheVersion = 3; 163 164 string cacheEngineDir(string engineName) 165 { 166 // Keep compatibility with old cache paths 167 string engineDirName = 168 engineName.isOneOf("directory", "true") ? "cache" : 169 engineName.isOneOf("", "none", "false") ? "temp-cache" : 170 "cache-" ~ engineName; 171 return buildPath( 172 config.local.workDir, 173 engineDirName, 174 "v%d".format(cacheVersion), 175 ); 176 } 177 178 version (Windows) 179 { 180 enum string binExt = ".exe"; 181 enum configFileName = "sc.ini"; 182 } 183 else 184 { 185 enum string binExt = ""; 186 enum configFileName = "dmd.conf"; 187 } 188 189 static bool needConfSwitch() { return exists(std.process.environment.get("HOME", null).buildPath(configFileName)); } 190 191 // **************************** Repositories ***************************** 192 193 class DManagerRepository : ManagedRepository 194 { 195 this() 196 { 197 this.offline = config.local.offline; 198 this.verify = this.outer.verifyWorkTree; 199 } 200 201 override void log(string s) { return this.outer.log(s); } 202 } 203 204 class MetaRepository : DManagerRepository 205 { 206 override void needRepo() 207 { 208 needGit(); 209 210 if (!repoDir.exists) 211 { 212 log("Cloning initial repository..."); 213 atomic!performClone(config.local.repoUrl, repoDir); 214 } 215 216 if (!git.path) 217 git = Repository(repoDir); 218 } 219 220 static void performClone(string url, string target) 221 { 222 import ae.sys.cmd; 223 run(["git", "clone", url, target]); 224 } 225 226 override void performCheckout(string hash) 227 { 228 super.performCheckout(hash); 229 submodules = null; 230 } 231 232 string[string][string] submoduleCache; 233 234 string[string] getSubmoduleCommits(string head) 235 { 236 auto pcacheEntry = head in submoduleCache; 237 if (pcacheEntry) 238 return (*pcacheEntry).dup; 239 240 string[string] result; 241 needRepo(); 242 foreach (line; git.query("ls-tree", head).splitLines()) 243 { 244 auto parts = line.split(); 245 if (parts.length == 4 && parts[1] == "commit") 246 result[parts[3]] = parts[2]; 247 } 248 assert(result.length, "No submodules found"); 249 submoduleCache[head] = result; 250 return result.dup; 251 } 252 253 /// Get the submodule state for all commits in the history. 254 /// Returns: result[commitHash][submoduleName] == submoduleCommitHash 255 string[string][string] getSubmoduleHistory(string[] refs) 256 { 257 auto marksFile = buildPath(config.local.workDir, "temp", "marks.txt"); 258 ensurePathExists(marksFile); 259 scope(exit) if (marksFile.exists) marksFile.remove(); 260 log("Running fast-export..."); 261 auto fastExportData = git.query([ 262 "fast-export", 263 "--full-tree", 264 "--no-data", 265 "--export-marks=" ~ marksFile.absolutePath, 266 ] ~ refs 267 ); 268 269 log("Parsing fast-export marks..."); 270 271 auto markLines = marksFile.readText.strip.splitLines; 272 auto marks = new string[markLines.length]; 273 foreach (line; markLines) 274 { 275 auto parts = line.split(' '); 276 auto markIndex = parts[0][1..$].to!int-1; 277 marks[markIndex] = parts[1]; 278 } 279 280 log("Parsing fast-export data..."); 281 282 string[string][string] result; 283 foreach (i, commitData; fastExportData.split("deleteall\n")[1..$]) 284 result[marks[i]] = commitData 285 .matchAll(re!(`^M 160000 ([0-9a-f]{40}) (\S+)$`, "m")) 286 .map!(m => tuple(m.captures[2], m.captures[1])) 287 .assocArray 288 ; 289 return result; 290 } 291 } 292 293 class SubmoduleRepository : DManagerRepository 294 { 295 string dir; 296 297 override void needRepo() 298 { 299 getMetaRepo().needRepo(); 300 301 if (!git.path) 302 git = Repository(dir); 303 } 304 305 override void needHead(string hash) 306 { 307 if (!autoClean) 308 super.needHead(hash); 309 else 310 try 311 super.needHead(hash); 312 catch (RepositoryCleanException e) 313 { 314 log("Error during repository cleanup."); 315 316 log("Nuking %s...".format(dir)); 317 rmdirRecurse(dir); 318 319 auto name = baseName(dir); 320 auto gitDir = buildPath(dirName(dir), ".git", "modules", name); 321 log("Nuking %s...".format(gitDir)); 322 rmdirRecurse(gitDir); 323 324 log("Updating submodule..."); 325 getMetaRepo().git.run(["submodule", "update", name]); 326 327 reset(); 328 329 log("Trying again..."); 330 super.needHead(hash); 331 } 332 } 333 } 334 335 /// The meta-repository, which contains the sub-project submodules. 336 private MetaRepository metaRepo; 337 338 MetaRepository getMetaRepo() /// ditto 339 { 340 if (!metaRepo) 341 metaRepo = new MetaRepository; 342 return metaRepo; 343 } 344 345 /// Sub-project repositories. 346 private SubmoduleRepository[string] submodules; 347 348 ManagedRepository getSubmodule(string name) /// ditto 349 { 350 assert(name, "This component is not associated with a submodule"); 351 if (name !in submodules) 352 { 353 getMetaRepo().needRepo(); 354 enforce(name in getMetaRepo().getSubmoduleCommits(getMetaRepo().getRef("origin/master")), 355 "Unknown submodule: " ~ name); 356 357 auto path = buildPath(metaRepo.git.path, name); 358 auto gitPath = buildPath(path, ".git"); 359 360 if (!gitPath.exists) 361 { 362 log("Initializing and updating submodule %s...".format(name)); 363 getMetaRepo().git.run(["submodule", "update", "--init", name]); 364 } 365 366 submodules[name] = new SubmoduleRepository(); 367 submodules[name].dir = path; 368 } 369 370 return submodules[name]; 371 } 372 373 // ***************************** Components ****************************** 374 375 /// Base class for a D component. 376 class Component 377 { 378 /// Name of this component, as registered in DManager.components AA. 379 string name; 380 381 /// Corresponding subproject repository name. 382 @property abstract string submoduleName(); 383 @property ManagedRepository submodule() { return getSubmodule(submoduleName); } 384 385 /// Configuration applicable to multiple (not all) components. 386 // Note: don't serialize this structure whole! 387 // Only serialize used fields. 388 struct CommonConfig 389 { 390 version (Windows) 391 enum defaultModel = "32"; 392 else 393 version (D_LP64) 394 enum defaultModel = "64"; 395 else 396 enum defaultModel = "32"; 397 398 /// Target comma-separated models ("32", "64", and on Windows, "32mscoff"). 399 /// Controls the models of the built Phobos and Druntime libraries. 400 string model = defaultModel; 401 402 @property string[] models() { return model.split(","); } 403 @property void models(string[] value) { this.model = value.join(","); } 404 405 string[] makeArgs; /// Additional make parameters, 406 /// e.g. "HOST_CC=g++48" 407 } 408 409 /// A string description of this component's configuration. 410 abstract @property string configString(); 411 412 /// Commit in the component's repo from which to build this component. 413 @property string commit() { return incrementalBuild ? "incremental" : getComponentCommit(name); } 414 415 /// The components the source code of which this component depends on. 416 /// Used for calculating the cache key. 417 @property abstract string[] sourceDependencies(); 418 419 /// The components the state and configuration of which this component depends on. 420 /// Used for calculating the cache key. 421 @property abstract string[] dependencies(); 422 423 /// This metadata is saved to a .json file, 424 /// and is also used to calculate the cache key. 425 struct Metadata 426 { 427 int cacheVersion; 428 string name; 429 string commit; 430 string configString; 431 string[] sourceDepCommits; 432 Metadata[] dependencyMetadata; 433 } 434 435 Metadata getMetadata() /// ditto 436 { 437 return Metadata( 438 cacheVersion, 439 name, 440 commit, 441 configString, 442 sourceDependencies.map!( 443 dependency => getComponent(dependency).commit 444 ).array(), 445 dependencies.map!( 446 dependency => getComponent(dependency).getMetadata() 447 ).array(), 448 ); 449 } 450 451 void saveMetaData(string target) 452 { 453 std.file.write(buildPath(target, "digger-metadata.json"), getMetadata().toJson()); 454 // Use a separate file to avoid double-encoding JSON 455 std.file.write(buildPath(target, "digger-config.json"), configString); 456 } 457 458 /// Calculates the cache key, which should be unique and immutable 459 /// for the same source, build parameters, and build algorithm. 460 string getBuildID() 461 { 462 auto configBlob = getMetadata().toJson() ~ configString; 463 return "%s-%s-%s".format( 464 name, 465 commit, 466 configBlob.getDigestString!MD5().toLower(), 467 ); 468 } 469 470 @property string sourceDir() { submodule.needRepo(); return submodule.git.path; } 471 472 /// Directory to which built files are copied to. 473 /// This will then be atomically added to the cache. 474 protected string stageDir; 475 476 /// Prepare the source checkout for this component. 477 /// Usually needed by other components. 478 void needSource() 479 { 480 tempError++; scope(success) tempError--; 481 482 if (incrementalBuild) 483 return; 484 if (!submoduleName) 485 return; 486 foreach (component; getSubmoduleComponents(submoduleName)) 487 component.haveBuild = false; 488 489 submodule.needHead(commit); 490 submodule.clean = false; 491 } 492 493 private bool haveBuild; 494 495 /// Build the component in-place, as needed, 496 /// without moving the built files anywhere. 497 void needBuild() 498 { 499 if (haveBuild) return; 500 scope(success) haveBuild = true; 501 502 log("needBuild: " ~ getBuildID()); 503 504 needSource(); 505 506 prepareEnv(); 507 508 log("Building " ~ getBuildID()); 509 performBuild(); 510 log(getBuildID() ~ " built OK!"); 511 } 512 513 /// Set up / clean the build environment. 514 private void prepareEnv() 515 { 516 // Nuke any additional directories cloned by makefiles 517 if (!incrementalBuild) 518 { 519 getMetaRepo().git.run(["clean", "-ffdx"]); 520 521 foreach (dir; [tmpDir, homeDir]) 522 { 523 if (dir.exists && !dir.dirEntries(SpanMode.shallow).empty) 524 log("Clearing %s ...".format(dir)); 525 dir.recreateEmptyDirectory(); 526 } 527 } 528 529 // Set up compiler wrappers. 530 recreateEmptyDirectory(binDir); 531 version (linux) 532 { 533 foreach (cc; ["cc", "gcc", "c++", "g++"]) 534 { 535 auto fileName = binDir.buildPath(cc); 536 write(fileName, q"EOF 537 #!/bin/sh 538 set -eu 539 540 tool=$(basename "$0") 541 next=/usr/bin/$tool 542 flagfile=$TMP/nopie-flag-$tool 543 544 if [ ! -e "$flagfile" ] 545 then 546 echo 'Testing for -no-pie...' 1>&2 547 testfile=$TMP/test-$$.c 548 echo 'int main(){return 0;}' > $testfile 549 if $next -no-pie -c -o$testfile.o $testfile 550 then 551 printf "%s" "-no-pie" > "$flagfile".$$.tmp 552 mv "$flagfile".$$.tmp "$flagfile" 553 else 554 touch "$flagfile" 555 fi 556 rm -f "$testfile" "$testfile.o" 557 fi 558 559 exec "$next" $(cat "$flagfile") "$@" 560 EOF"); 561 setAttributes(fileName, octal!755); 562 } 563 } 564 } 565 566 private bool haveInstalled; 567 568 /// Build and "install" the component to buildDir as necessary. 569 void needInstalled() 570 { 571 if (haveInstalled) return; 572 scope(success) haveInstalled = true; 573 574 auto buildID = getBuildID(); 575 log("needInstalled: " ~ buildID); 576 577 needCacheEngine(); 578 if (cacheEngine.haveEntry(buildID)) 579 { 580 log("Cache hit!"); 581 if (cacheEngine.listFiles(buildID).canFind(unbuildableMarker)) 582 throw new Exception(buildID ~ " was cached as unbuildable"); 583 } 584 else 585 { 586 log("Cache miss."); 587 588 auto tempDir = buildPath(config.local.workDir, "temp"); 589 if (tempDir.exists) 590 tempDir.removeRecurse(); 591 stageDir = buildPath(tempDir, buildID); 592 stageDir.mkdirRecurse(); 593 594 bool failed = false; 595 tempError = 0; 596 597 // Save the results to cache, failed or not 598 void saveToCache() 599 { 600 // Use a separate function to work around 601 // "cannot put scope(success) statement inside scope(exit)" 602 603 int currentTempError = tempError; 604 605 // Treat cache errors an environmental errors 606 // (for when needInstalled is invoked to build a dependency) 607 tempError++; scope(success) tempError--; 608 609 // tempDir might be removed by a dependency's build failure. 610 if (!tempDir.exists) 611 log("Not caching %s dependency build failure.".format(name)); 612 else 613 // Don't cache failed build results due to temporary/environment problems 614 if (failed && currentTempError > 0) 615 { 616 log("Not caching %s build failure due to temporary/environment error.".format(name)); 617 rmdirRecurse(tempDir); 618 } 619 else 620 // Don't cache failed build results during delve 621 if (failed && !cacheFailures) 622 { 623 log("Not caching failed %s build.".format(name)); 624 rmdirRecurse(tempDir); 625 } 626 else 627 if (cacheEngine.haveEntry(buildID)) 628 { 629 // Can happen due to force==true 630 log("Already in cache."); 631 rmdirRecurse(tempDir); 632 } 633 else 634 { 635 log("Saving to cache."); 636 saveMetaData(stageDir); 637 cacheEngine.add(buildID, stageDir); 638 rmdirRecurse(tempDir); 639 } 640 } 641 642 scope (exit) 643 saveToCache(); 644 645 // An incomplete build is useless, nuke the directory 646 // and create a new one just for the "unbuildable" marker. 647 scope (failure) 648 { 649 failed = true; 650 if (stageDir.exists) 651 { 652 rmdirRecurse(stageDir); 653 mkdir(stageDir); 654 buildPath(stageDir, unbuildableMarker).touch(); 655 } 656 } 657 658 needBuild(); 659 660 performStage(); 661 } 662 663 install(); 664 } 665 666 /// Build the component in-place, without moving the built files anywhere. 667 void performBuild() {} 668 669 /// Place resulting files to stageDir 670 void performStage() {} 671 672 /// Update the environment post-install, to allow 673 /// building components that depend on this one. 674 void updateEnv(ref Environment env) {} 675 676 /// Copy build results from cacheDir to buildDir 677 void install() 678 { 679 log("Installing " ~ getBuildID()); 680 needCacheEngine().extract(getBuildID(), buildDir, de => !de.baseName.startsWith("digger-")); 681 } 682 683 /// Prepare the dependencies then run the component's tests. 684 void test() 685 { 686 log("Testing " ~ getBuildID()); 687 688 needSource(); 689 690 submodule.clean = false; 691 performTest(); 692 log(getBuildID() ~ " tests OK!"); 693 } 694 695 /// Run the component's tests. 696 void performTest() {} 697 698 protected final: 699 // Utility declarations for component implementations 700 701 string modelSuffix(string model) { return model == "32" ? "" : model; } 702 version (Windows) 703 { 704 enum string makeFileName = "win32.mak"; 705 string makeFileNameModel(string model) 706 { 707 if (model == "32mscoff") 708 model = "64"; 709 return "win"~model~".mak"; 710 } 711 enum string binExt = ".exe"; 712 } 713 else 714 { 715 enum string makeFileName = "posix.mak"; 716 string makeFileNameModel(string model) { return "posix.mak"; } 717 enum string binExt = ""; 718 } 719 720 version (Windows) 721 enum platform = "windows"; 722 else 723 version (linux) 724 enum platform = "linux"; 725 else 726 version (OSX) 727 enum platform = "osx"; 728 else 729 version (FreeBSD) 730 enum platform = "freebsd"; 731 else 732 static assert(false); 733 734 /// Returns the command for the make utility. 735 string[] getMake(in ref Environment env) 736 { 737 return [env.vars.get("MAKE", "make")]; 738 } 739 740 /// Returns the path to the built dmd executable. 741 @property string dmd() { return buildPath(buildDir, "bin", "dmd" ~ binExt).absolutePath(); } 742 743 /// Escape a path for d_do_test's very "special" criteria. 744 /// Spaces must be escaped, but there must be no double-quote at the end. 745 private static string dDoTestEscape(string str) 746 { 747 return str.replaceAll(re!`\\([^\\ ]*? [^\\]*)(?=\\)`, `\"$1"`); 748 } 749 750 unittest 751 { 752 assert(dDoTestEscape(`C:\Foo boo bar\baz quuz\derp.exe`) == `C:\"Foo boo bar"\"baz quuz"\derp.exe`); 753 } 754 755 string[] getPlatformMakeVars(in ref Environment env, string model) 756 { 757 string[] args; 758 759 args ~= "MODEL=" ~ model; 760 761 version (Windows) 762 if (model != "32") 763 { 764 args ~= "VCDIR=" ~ env.deps.vsDir.buildPath("VC").absolutePath(); 765 args ~= "SDKDIR=" ~ env.deps.sdkDir.absolutePath(); 766 args ~= "CC=" ~ '"' ~ env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "cl.exe").absolutePath() ~ '"'; 767 args ~= "LD=" ~ '"' ~ env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "link.exe").absolutePath() ~ '"'; 768 args ~= "AR=" ~ '"' ~ env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "lib.exe").absolutePath() ~ '"'; 769 } 770 771 return args; 772 } 773 774 @property string[] gnuMakeArgs() 775 { 776 string[] args; 777 if (config.local.makeJobs) 778 { 779 if (config.local.makeJobs == "auto") 780 { 781 import std.parallelism, std.conv; 782 args ~= "-j" ~ text(totalCPUs); 783 } 784 else 785 if (config.local.makeJobs == "unlimited") 786 args ~= "-j"; 787 else 788 args ~= "-j" ~ config.local.makeJobs; 789 } 790 return args; 791 } 792 793 @property string[] dMakeArgs() 794 { 795 version (Windows) 796 return null; // On Windows, DigitalMars make is used for all makefiles except the dmd test suite 797 else 798 return gnuMakeArgs; 799 } 800 801 /// Older versions did not use the posix.mak/win32.mak convention. 802 static string findMakeFile(string dir, string fn) 803 { 804 version (OSX) 805 if (!dir.buildPath(fn).exists && dir.buildPath("osx.mak").exists) 806 return "osx.mak"; 807 version (Posix) 808 if (!dir.buildPath(fn).exists && dir.buildPath("linux.mak").exists) 809 return "linux.mak"; 810 return fn; 811 } 812 813 void needCC(ref Environment env, string model, string dmcVer = null) 814 { 815 version (Windows) 816 { 817 needDMC(env, dmcVer); // We need DMC even for 64-bit builds (for DM make) 818 if (model != "32") 819 needVC(env, model); 820 } 821 } 822 823 void run(const(string)[] args, in string[string] newEnv, string dir) 824 { 825 // Apply user environment 826 auto env = applyEnv(newEnv, config.build.environment); 827 828 // Temporarily apply PATH from newEnv to our process, 829 // so process creation lookup can use it. 830 string oldPath = std.process.environment["PATH"]; 831 scope (exit) std.process.environment["PATH"] = oldPath; 832 std.process.environment["PATH"] = env["PATH"]; 833 834 // Apply timeout setting 835 if (config.local.timeout) 836 args = ["timeout", config.local.timeout.text] ~ args; 837 838 foreach (name, value; env) 839 log("Environment: " ~ name ~ "=" ~ value); 840 log("Working directory: " ~ dir); 841 log("Running: " ~ escapeShellCommand(args)); 842 843 auto status = spawnProcess(args, env, std.process.Config.newEnv, dir).wait(); 844 enforce(status == 0, "Command %s failed with status %d".format(args, status)); 845 } 846 } 847 848 /// The dmd executable 849 final class DMD : Component 850 { 851 @property override string submoduleName () { return "dmd"; } 852 @property override string[] sourceDependencies() { return []; } 853 @property override string[] dependencies() { return []; } 854 855 struct Config 856 { 857 /// Whether to build a debug DMD. 858 /// Debug builds are faster to build, 859 /// but run slower. 860 @JSONOptional bool debugDMD = false; 861 862 /// Whether to build a release DMD. 863 /// Mutually exclusive with debugDMD. 864 @JSONOptional bool releaseDMD = false; 865 866 /// Model for building DMD itself (on Windows). 867 /// Can be used to build a 64-bit DMD, to avoid 4GB limit. 868 @JSONOptional string dmdModel = CommonConfig.defaultModel; 869 870 /// How to build DMD versions written in D. 871 /// We can either download a pre-built binary DMD 872 /// package, or build an earlier version from source 873 /// (e.g. starting with the last C++-only version.) 874 struct Bootstrap 875 { 876 /// Whether to download a pre-built D version, 877 /// or build one from source. If set, then build 878 /// from source according to the value of ver, 879 @JSONOptional bool fromSource = false; 880 881 /// Version specification. 882 /// When building from source, syntax can be defined 883 /// by outer application (see parseSpec method); 884 /// When the bootstrapping compiler is not built from source, 885 /// it is understood as a version number, such as "v2.070.2", 886 /// which also doubles as a tag name. 887 /// By default (when set to null), an appropriate version 888 /// is selected automatically. 889 @JSONOptional string ver = null; 890 891 /// Build configuration for the compiler used for bootstrapping. 892 /// If not set, then use the default build configuration. 893 /// Used when fromSource is set. 894 @JSONOptional DManager.Config.Build* build; 895 } 896 @JSONOptional Bootstrap bootstrap; /// ditto 897 898 /// Use Visual C++ to build DMD instead of DMC. 899 /// Currently, this is a hack, as msbuild will consult the system 900 /// registry and use the system-wide installation of Visual Studio. 901 /// Only relevant for older versions, as newer versions are written in D. 902 @JSONOptional bool useVC; 903 } 904 905 @property override string configString() 906 { 907 static struct FullConfig 908 { 909 Config config; 910 string[] makeArgs; 911 912 // Include the common models as well as the DMD model (from config). 913 // Necessary to ensure the correct sc.ini is generated on Windows 914 // (we don't want to pull in MSVC unless either DMD or Phobos are 915 // built as 64-bit, but also we can't reuse a DMD build with 32-bit 916 // DMD and Phobos for a 64-bit Phobos build because it won't have 917 // the VC vars set up in its sc.ini). 918 // Possibly refactor the compiler configuration to a separate 919 // component in the future to avoid the inefficiency of rebuilding 920 // DMD just to generate a different sc.ini. 921 @JSONOptional string commonModel = Component.CommonConfig.defaultModel; 922 } 923 924 return FullConfig( 925 config.build.components.dmd, 926 config.build.components.common.makeArgs, 927 config.build.components.common.model, 928 ).toJson(); 929 } 930 931 @property string vsConfiguration() { return config.build.components.dmd.debugDMD ? "Debug" : "Release"; } 932 @property string vsPlatform () { return config.build.components.dmd.dmdModel == "64" ? "x64" : "Win32"; } 933 934 override void performBuild() 935 { 936 // We need an older DMC for older DMD versions 937 string dmcVer = null; 938 auto idgen = buildPath(sourceDir, "src", "idgen.c"); 939 if (idgen.exists && idgen.readText().indexOf(`{ "alignof" },`) >= 0) 940 dmcVer = "850"; 941 942 auto env = baseEnvironment; 943 needCC(env, config.build.components.dmd.dmdModel, dmcVer); // Need VC too for VSINSTALLDIR 944 945 if (buildPath(sourceDir, "src", "idgen.d").exists || 946 buildPath(sourceDir, "src", "ddmd", "idgen.d").exists || 947 buildPath(sourceDir, "src", "ddmd", "mars.d").exists || 948 buildPath(sourceDir, "src", "dmd", "mars.d").exists) 949 { 950 // Required for bootstrapping. 951 needDMD(env); 952 // Go back to our commit. 953 needSource(); 954 submodule.clean = false; 955 } 956 957 auto srcDir = buildPath(sourceDir, "src"); 958 959 if (config.build.components.dmd.useVC) // Mostly obsolete, see useVC ddoc 960 { 961 version (Windows) 962 { 963 needVC(env, config.build.components.dmd.dmdModel); 964 965 env.vars["PATH"] = env.vars["PATH"] ~ pathSeparator ~ env.deps.hostDC.dirName; 966 967 auto solutionFile = `dmd_msc_vs10.sln`; 968 if (!exists(srcDir.buildPath(solutionFile))) 969 solutionFile = `vcbuild\dmd.sln`; 970 if (!exists(srcDir.buildPath(solutionFile))) 971 throw new Exception("Can't find Visual Studio solution file"); 972 973 return run(["msbuild", "/p:Configuration=" ~ vsConfiguration, "/p:Platform=" ~ vsPlatform, solutionFile], env.vars, srcDir); 974 } 975 else 976 throw new Exception("Can only use Visual Studio on Windows"); 977 } 978 979 version (Windows) 980 auto scRoot = env.deps.dmcDir.absolutePath(); 981 982 string dmdMakeFileName = findMakeFile(srcDir, makeFileName); 983 string dmdMakeFullName = srcDir.buildPath(dmdMakeFileName); 984 985 string modelFlag = config.build.components.dmd.dmdModel; 986 if (dmdMakeFullName.readText().canFind("MODEL=-m32")) 987 modelFlag = "-m" ~ modelFlag; 988 989 version (Windows) 990 { 991 // A make argument is insufficient, 992 // because of recursive make invocations 993 auto m = dmdMakeFullName.readText(); 994 m = m 995 .replace(`CC=\dm\bin\dmc`, `CC=dmc`) 996 .replace(`SCROOT=$D\dm`, `SCROOT=` ~ scRoot) 997 ; 998 dmdMakeFullName.write(m); 999 } 1000 else 1001 { 1002 auto m = dmdMakeFullName.readText(); 1003 m = m 1004 // Fix hard-coded reference to gcc as linker 1005 .replace(`gcc -m32 -lstdc++`, `g++ -m32 -lstdc++`) 1006 .replace(`gcc $(MODEL) -lstdc++`, `g++ $(MODEL) -lstdc++`) 1007 // Fix compilation of older versions of go.c with GCC 6 1008 .replace(`-Wno-deprecated`, `-Wno-deprecated -Wno-narrowing`) 1009 ; 1010 // Fix pthread linker error 1011 version (linux) 1012 m = m.replace(`-lpthread`, `-pthread`); 1013 dmdMakeFullName.write(m); 1014 } 1015 1016 submodule.saveFileState("src/" ~ dmdMakeFileName); 1017 1018 // Fix compilation error of older DMDs with glibc >= 2.25 1019 version (linux) 1020 {{ 1021 auto fn = srcDir.buildPath("root", "port.c"); 1022 if (fn.exists) 1023 { 1024 fn.write(fn.readText 1025 .replace(`#include <bits/mathdef.h>`, `#include <complex.h>`) 1026 ); 1027 submodule.saveFileState(fn.relativePath(sourceDir)); 1028 } 1029 }} 1030 1031 // Fix alignment issue in older DMDs with GCC >= 7 1032 // See https://issues.dlang.org/show_bug.cgi?id=17726 1033 version (Posix) 1034 { 1035 foreach (fn; [srcDir.buildPath("tk", "mem.c"), srcDir.buildPath("ddmd", "tk", "mem.c")]) 1036 if (fn.exists) 1037 { 1038 fn.write(fn.readText.replace( 1039 // `#if defined(__llvm__) && (defined(__GNUC__) || defined(__clang__))`, 1040 // `#if defined(__GNUC__) || defined(__clang__)`, 1041 `numbytes = (numbytes + 3) & ~3;`, 1042 `numbytes = (numbytes + 0xF) & ~0xF;` 1043 )); 1044 submodule.saveFileState(fn.relativePath(sourceDir)); 1045 } 1046 } 1047 1048 string[] extraArgs, targets; 1049 version (Posix) 1050 { 1051 if (config.build.components.dmd.debugDMD) 1052 extraArgs ~= "DEBUG=1"; 1053 if (config.build.components.dmd.releaseDMD) 1054 extraArgs ~= "ENABLE_RELEASE=1"; 1055 } 1056 else 1057 { 1058 if (config.build.components.dmd.debugDMD) 1059 targets ~= []; 1060 else 1061 if (config.build.components.dmd.releaseDMD && dmdMakeFullName.readText().canFind("reldmd")) 1062 targets ~= ["reldmd"]; 1063 else 1064 targets ~= ["dmd"]; 1065 } 1066 1067 version (Windows) 1068 { 1069 if (config.build.components.dmd.dmdModel != CommonConfig.defaultModel) 1070 { 1071 dmdMakeFileName = "win64.mak"; 1072 dmdMakeFullName = srcDir.buildPath(dmdMakeFileName); 1073 enforce(dmdMakeFullName.exists, "dmdModel not supported for this DMD version"); 1074 extraArgs ~= "DMODEL=-m" ~ config.build.components.dmd.dmdModel; 1075 if (config.build.components.dmd.dmdModel == "32mscoff") 1076 { 1077 auto objFiles = dmdMakeFullName.readText().splitLines().filter!(line => line.startsWith("OBJ_MSVC=")); 1078 enforce(!objFiles.empty, "Can't find OBJ_MSVC in win64.mak"); 1079 extraArgs ~= "OBJ_MSVC=" ~ objFiles.front.findSplit("=")[2].split().filter!(obj => obj != "ldfpu.obj").join(" "); 1080 } 1081 } 1082 } 1083 1084 // Avoid HOST_DC reading ~/dmd.conf 1085 string hostDC = env.deps.hostDC; 1086 version (Posix) 1087 if (hostDC && needConfSwitch()) 1088 { 1089 auto dcProxy = buildPath(config.local.workDir, "host-dc-proxy.sh"); 1090 std.file.write(dcProxy, escapeShellCommand(["exec", hostDC, "-conf=" ~ buildPath(dirName(hostDC), configFileName)]) ~ ` "$@"`); 1091 setAttributes(dcProxy, octal!755); 1092 hostDC = dcProxy; 1093 } 1094 1095 run(getMake(env) ~ [ 1096 "-f", dmdMakeFileName, 1097 "MODEL=" ~ modelFlag, 1098 "HOST_DC=" ~ hostDC, 1099 ] ~ config.build.components.common.makeArgs ~ dMakeArgs ~ extraArgs ~ targets, 1100 env.vars, srcDir 1101 ); 1102 } 1103 1104 override void performStage() 1105 { 1106 if (config.build.components.dmd.useVC) 1107 { 1108 foreach (ext; [".exe", ".pdb"]) 1109 cp( 1110 buildPath(sourceDir, "src", "vcbuild", vsPlatform, vsConfiguration, "dmd_msc" ~ ext), 1111 buildPath(stageDir , "bin", "dmd" ~ ext), 1112 ); 1113 } 1114 else 1115 { 1116 string dmdPath = buildPath(sourceDir, "generated", platform, "release", config.build.components.dmd.dmdModel, "dmd" ~ binExt); 1117 if (!dmdPath.exists) 1118 dmdPath = buildPath(sourceDir, "src", "dmd" ~ binExt); // legacy 1119 enforce(dmdPath.exists && dmdPath.isFile, "Can't find built DMD executable"); 1120 1121 cp( 1122 dmdPath, 1123 buildPath(stageDir , "bin", "dmd" ~ binExt), 1124 ); 1125 } 1126 1127 version (Windows) 1128 { 1129 auto env = baseEnvironment; 1130 needCC(env, config.build.components.dmd.dmdModel); 1131 foreach (model; config.build.components.common.models) 1132 needCC(env, model); 1133 1134 auto ini = q"EOS 1135 [Environment] 1136 LIB="%@P%\..\lib" 1137 DFLAGS="-I%@P%\..\import" 1138 DMC=__DMC__ 1139 LINKCMD=%DMC%\link.exe 1140 EOS" 1141 .replace("__DMC__", env.deps.dmcDir.buildPath(`bin`).absolutePath()) 1142 ; 1143 1144 if (env.deps.vsDir && env.deps.sdkDir) 1145 { 1146 ini ~= q"EOS 1147 1148 [Environment64] 1149 LIB="%@P%\..\lib" 1150 DFLAGS=%DFLAGS% -L/OPT:NOICF 1151 VSINSTALLDIR=__VS__\ 1152 VCINSTALLDIR=%VSINSTALLDIR%VC\ 1153 PATH=%PATH%;%VCINSTALLDIR%\bin\__MODELDIR__;%VCINSTALLDIR%\bin 1154 WindowsSdkDir=__SDK__ 1155 LINKCMD=%VCINSTALLDIR%\bin\__MODELDIR__\link.exe 1156 LIB=%LIB%;"%VCINSTALLDIR%\lib\amd64" 1157 LIB=%LIB%;"%WindowsSdkDir%\Lib\x64" 1158 1159 [Environment32mscoff] 1160 LIB="%@P%\..\lib" 1161 DFLAGS=%DFLAGS% -L/OPT:NOICF 1162 VSINSTALLDIR=__VS__\ 1163 VCINSTALLDIR=%VSINSTALLDIR%VC\ 1164 PATH=%PATH%;%VCINSTALLDIR%\bin 1165 WindowsSdkDir=__SDK__ 1166 LINKCMD=%VCINSTALLDIR%\bin\link.exe 1167 LIB=%LIB%;"%VCINSTALLDIR%\lib" 1168 LIB=%LIB%;"%WindowsSdkDir%\Lib" 1169 EOS" 1170 .replace("__VS__" , env.deps.vsDir .absolutePath()) 1171 .replace("__SDK__" , env.deps.sdkDir.absolutePath()) 1172 .replace("__MODELDIR__", msvcModelDir("64")) 1173 ; 1174 } 1175 1176 buildPath(stageDir, "bin", configFileName).write(ini); 1177 } 1178 else version (OSX) 1179 { 1180 auto ini = q"EOS 1181 [Environment] 1182 DFLAGS="-I%@P%/../import" "-L-L%@P%/../lib" 1183 EOS"; 1184 buildPath(stageDir, "bin", configFileName).write(ini); 1185 } 1186 else 1187 { 1188 auto ini = q"EOS 1189 [Environment] 1190 DFLAGS="-I%@P%/../import" "-L-L%@P%/../lib" -L--export-dynamic 1191 EOS"; 1192 buildPath(stageDir, "bin", configFileName).write(ini); 1193 } 1194 } 1195 1196 override void updateEnv(ref Environment env) 1197 { 1198 // Add the DMD we built for Phobos/Druntime/Tools 1199 env.vars["PATH"] = buildPath(buildDir, "bin").absolutePath() ~ pathSeparator ~ env.vars["PATH"]; 1200 } 1201 1202 override void performTest() 1203 { 1204 foreach (dep; ["dmd", "druntime", "phobos"]) 1205 getComponent(dep).needBuild(); 1206 1207 foreach (model; config.build.components.common.models) 1208 { 1209 auto env = baseEnvironment; 1210 version (Windows) 1211 { 1212 // In this order so it uses the MSYS make 1213 needCC(env, model); 1214 needMSYS(env); 1215 1216 disableCrashDialog(); 1217 } 1218 1219 auto makeArgs = getMake(env) ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env, model) ~ gnuMakeArgs; 1220 version (Windows) 1221 { 1222 makeArgs ~= ["OS=win" ~ model[0..2], "SHELL=bash"]; 1223 if (model == "32") 1224 { 1225 auto extrasDir = needExtras(); 1226 // The autotester seems to pass this via environment. Why does that work there??? 1227 makeArgs ~= "LIB=" ~ extrasDir.buildPath("localextras-windows", "dmd2", "windows", "lib") ~ `;..\..\phobos`; 1228 } 1229 else 1230 { 1231 // Fix path for d_do_test and its special escaping (default is the system VS2010 install) 1232 // We can't use the same syntax in getPlatformMakeVars because win64.mak uses "CC=\$(CC32)"\"" 1233 auto cl = env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "cl.exe"); 1234 foreach (ref arg; makeArgs) 1235 if (arg.startsWith("CC=")) 1236 arg = "CC=" ~ dDoTestEscape(cl); 1237 } 1238 } 1239 1240 version (test) 1241 { 1242 // Only try a few tests during CI runs, to check for 1243 // platform integration and correct invocation. 1244 // For this purpose, the C++ ABI tests will do nicely. 1245 makeArgs ~= [ 1246 // "test_results/runnable/cppa.d.out", // https://github.com/dlang/dmd/pull/5686 1247 "test_results/runnable/cpp_abi_tests.d.out", 1248 "test_results/runnable/cabi1.d.out", 1249 ]; 1250 } 1251 1252 run(makeArgs, env.vars, sourceDir.buildPath("test")); 1253 } 1254 } 1255 } 1256 1257 /// Phobos import files. 1258 /// In older versions of D, Druntime depended on Phobos modules. 1259 final class PhobosIncludes : Component 1260 { 1261 @property override string submoduleName() { return "phobos"; } 1262 @property override string[] sourceDependencies() { return []; } 1263 @property override string[] dependencies() { return []; } 1264 @property override string configString() { return null; } 1265 1266 override void performStage() 1267 { 1268 foreach (f; ["std", "etc", "crc32.d"]) 1269 if (buildPath(sourceDir, f).exists) 1270 cp( 1271 buildPath(sourceDir, f), 1272 buildPath(stageDir , "import", f), 1273 ); 1274 } 1275 } 1276 1277 /// Druntime. Installs only import files, but builds the library too. 1278 final class Druntime : Component 1279 { 1280 @property override string submoduleName () { return "druntime"; } 1281 @property override string[] sourceDependencies() { return ["phobos", "phobos-includes"]; } 1282 @property override string[] dependencies() { return ["dmd"]; } 1283 1284 @property override string configString() 1285 { 1286 static struct FullConfig 1287 { 1288 string model; 1289 string[] makeArgs; 1290 } 1291 1292 return FullConfig( 1293 config.build.components.common.model, 1294 config.build.components.common.makeArgs, 1295 ).toJson(); 1296 } 1297 1298 override void performBuild() 1299 { 1300 getComponent("phobos").needSource(); 1301 getComponent("dmd").needSource(); 1302 getComponent("dmd").needInstalled(); 1303 getComponent("phobos-includes").needInstalled(); 1304 1305 foreach (model; config.build.components.common.models) 1306 { 1307 auto env = baseEnvironment; 1308 needCC(env, model); 1309 1310 mkdirRecurse(sourceDir.buildPath("import")); 1311 mkdirRecurse(sourceDir.buildPath("lib")); 1312 1313 setTimes(sourceDir.buildPath("src", "rt", "minit.obj"), Clock.currTime(), Clock.currTime()); // Don't rebuild 1314 submodule.saveFileState("src/rt/minit.obj"); 1315 1316 run(getMake(env) ~ ["-f", makeFileNameModel(model), "import", "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env, model) ~ dMakeArgs, env.vars, sourceDir); 1317 run(getMake(env) ~ ["-f", makeFileNameModel(model) , "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env, model) ~ dMakeArgs, env.vars, sourceDir); 1318 } 1319 } 1320 1321 override void performStage() 1322 { 1323 cp( 1324 buildPath(sourceDir, "import"), 1325 buildPath(stageDir , "import"), 1326 ); 1327 } 1328 1329 override void performTest() 1330 { 1331 getComponent("druntime").needBuild(); 1332 getComponent("dmd").needInstalled(); 1333 1334 foreach (model; config.build.components.common.models) 1335 { 1336 auto env = baseEnvironment; 1337 needCC(env, model); 1338 run(getMake(env) ~ ["-f", makeFileNameModel(model), "unittest", "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env, model) ~ dMakeArgs, env.vars, sourceDir); 1339 } 1340 } 1341 } 1342 1343 /// Phobos library and imports. 1344 final class Phobos : Component 1345 { 1346 @property override string submoduleName () { return "phobos"; } 1347 @property override string[] sourceDependencies() { return []; } 1348 @property override string[] dependencies() { return ["druntime", "dmd"]; } 1349 1350 @property override string configString() 1351 { 1352 static struct FullConfig 1353 { 1354 string model; 1355 string[] makeArgs; 1356 } 1357 1358 return FullConfig( 1359 config.build.components.common.model, 1360 config.build.components.common.makeArgs, 1361 ).toJson(); 1362 } 1363 1364 string[] targets; 1365 1366 override void performBuild() 1367 { 1368 getComponent("dmd").needSource(); 1369 getComponent("dmd").needInstalled(); 1370 getComponent("druntime").needBuild(); 1371 1372 targets = null; 1373 1374 foreach (model; config.build.components.common.models) 1375 { 1376 // Clean up old object files with mismatching model. 1377 // Necessary for a consecutive 32/64 build. 1378 version (Windows) 1379 { 1380 foreach (de; dirEntries(sourceDir.buildPath("etc", "c", "zlib"), "*.obj", SpanMode.shallow)) 1381 { 1382 auto data = cast(ubyte[])read(de.name); 1383 1384 string fileModel; 1385 if (data.length < 4) 1386 fileModel = "invalid"; 1387 else 1388 if (data[0] == 0x80) 1389 fileModel = "32"; // OMF 1390 else 1391 if (data[0] == 0x01 && data[0] == 0x4C) 1392 fileModel = "32mscoff"; // COFF - IMAGE_FILE_MACHINE_I386 1393 else 1394 if (data[0] == 0x86 && data[0] == 0x64) 1395 fileModel = "64"; // COFF - IMAGE_FILE_MACHINE_AMD64 1396 else 1397 fileModel = "unknown"; 1398 1399 if (fileModel != model) 1400 { 1401 log("Cleaning up object file '%s' with mismatching model (file is %s, building %s)".format(de.name, fileModel, model)); 1402 remove(de.name); 1403 } 1404 } 1405 } 1406 1407 auto env = baseEnvironment; 1408 needCC(env, model); 1409 1410 string phobosMakeFileName = findMakeFile(sourceDir, makeFileNameModel(model)); 1411 string phobosMakeFullName = sourceDir.buildPath(phobosMakeFileName); 1412 1413 auto makeArgs = getMake(env) ~ ["-f", phobosMakeFileName, "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env, model) ~ dMakeArgs; 1414 1415 version (Windows) 1416 { 1417 auto lib = "phobos%s.lib".format(modelSuffix(model)); 1418 run(makeArgs ~ lib, env.vars, sourceDir); 1419 enforce(sourceDir.buildPath(lib).exists); 1420 targets ~= ["phobos%s.lib".format(modelSuffix(model))]; 1421 } 1422 else 1423 { 1424 if (phobosMakeFullName.readText().canFind("DRUNTIME = $(DRUNTIME_PATH)/lib/libdruntime-$(OS)$(MODEL).a") && 1425 getComponent("druntime").sourceDir.buildPath("lib").dirEntries(SpanMode.shallow).walkLength == 0 && 1426 exists(getComponent("druntime").sourceDir.buildPath("generated"))) 1427 { 1428 auto dir = getComponent("druntime").sourceDir.buildPath("generated"); 1429 auto aFile = dir.dirEntries("libdruntime.a", SpanMode.depth); 1430 if (!aFile .empty) makeArgs ~= ["DRUNTIME=" ~ aFile .front]; 1431 auto soFile = dir.dirEntries("libdruntime.so.a", SpanMode.depth); 1432 if (!soFile.empty) makeArgs ~= ["DRUNTIMESO=" ~ soFile.front]; 1433 } 1434 run(makeArgs, env.vars, sourceDir); 1435 targets ~= sourceDir 1436 .buildPath("generated") 1437 .dirEntries(SpanMode.depth) 1438 .filter!(de => de.name.endsWith(".a") || de.name.endsWith(".so")) 1439 .map!(de => de.name.relativePath(sourceDir)) 1440 .array() 1441 ; 1442 } 1443 } 1444 } 1445 1446 override void performStage() 1447 { 1448 assert(targets.length, "Druntime stage without build"); 1449 foreach (lib; targets) 1450 cp( 1451 buildPath(sourceDir, lib), 1452 buildPath(stageDir , "lib", lib.baseName()), 1453 ); 1454 } 1455 1456 override void performTest() 1457 { 1458 getComponent("druntime").needBuild(); 1459 getComponent("phobos").needBuild(); 1460 getComponent("dmd").needInstalled(); 1461 1462 foreach (model; config.build.components.common.models) 1463 { 1464 auto env = baseEnvironment; 1465 needCC(env, model); 1466 version (Windows) 1467 { 1468 getComponent("curl").needInstalled(); 1469 getComponent("curl").updateEnv(env); 1470 1471 // Patch out std.datetime unittest to work around Digger test 1472 // suite failure on AppVeyor due to Windows time zone changes 1473 auto stdDateTime = buildPath(sourceDir, "std", "datetime.d"); 1474 if (stdDateTime.exists && !stdDateTime.readText().canFind("Altai Standard Time")) 1475 { 1476 auto m = stdDateTime.readText(); 1477 m = m 1478 .replace(`assert(tzName !is null, format("TZName which is missing: %s", winName));`, ``) 1479 .replace(`assert(tzDatabaseNameToWindowsTZName(tzName) !is null, format("TZName which failed: %s", tzName));`, `{}`) 1480 .replace(`assert(windowsTZNameToTZDatabaseName(tzName) !is null, format("TZName which failed: %s", tzName));`, `{}`) 1481 ; 1482 stdDateTime.write(m); 1483 submodule.saveFileState("std/datetime.d"); 1484 } 1485 1486 if (model == "32") 1487 getComponent("extras").needInstalled(); 1488 } 1489 run(getMake(env) ~ ["-f", makeFileNameModel(model), "unittest", "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env, model) ~ dMakeArgs, env.vars, sourceDir); 1490 } 1491 } 1492 } 1493 1494 /// The rdmd build tool by itself. 1495 /// It predates the tools package. 1496 final class RDMD : Component 1497 { 1498 @property override string submoduleName() { return "tools"; } 1499 @property override string[] sourceDependencies() { return []; } 1500 @property override string[] dependencies() { return ["dmd", "druntime", "phobos"]; } 1501 1502 @property string model() { return config.build.components.common.models.get(0); } 1503 1504 @property override string configString() 1505 { 1506 static struct FullConfig 1507 { 1508 string model; 1509 } 1510 1511 return FullConfig( 1512 this.model, 1513 ).toJson(); 1514 } 1515 1516 override void performBuild() 1517 { 1518 foreach (dep; ["dmd", "druntime", "phobos", "phobos-includes"]) 1519 getComponent(dep).needInstalled(); 1520 1521 auto env = baseEnvironment; 1522 needCC(env, this.model); 1523 1524 // Just build rdmd 1525 bool needModel; // Need -mXX switch? 1526 1527 if (sourceDir.buildPath("posix.mak").exists) 1528 needModel = true; // Known to be needed for recent versions 1529 1530 string[] args; 1531 if (needConfSwitch()) 1532 args ~= ["-conf=" ~ buildPath(buildDir , "bin", configFileName)]; 1533 args ~= ["rdmd"]; 1534 1535 if (!needModel) 1536 try 1537 run([dmd] ~ args, env.vars, sourceDir); 1538 catch (Exception e) 1539 needModel = true; 1540 1541 if (needModel) 1542 run([dmd, "-m" ~ this.model] ~ args, env.vars, sourceDir); 1543 } 1544 1545 override void performStage() 1546 { 1547 cp( 1548 buildPath(sourceDir, "rdmd" ~ binExt), 1549 buildPath(stageDir , "bin", "rdmd" ~ binExt), 1550 ); 1551 } 1552 1553 override void performTest() 1554 { 1555 version (Windows) 1556 if (this.model != "32") 1557 { 1558 // Can't test rdmd on non-32-bit Windows until compiler model matches Phobos model. 1559 // rdmd_test does not use -m when building rdmd, thus linking will fail 1560 // (because of model mismatch with the phobos we built). 1561 log("Can't test rdmd with model " ~ this.model ~ ", skipping"); 1562 return; 1563 } 1564 1565 foreach (dep; ["dmd", "druntime", "phobos", "phobos-includes"]) 1566 getComponent(dep).needInstalled(); 1567 1568 auto env = baseEnvironment; 1569 getComponent("dmd").updateEnv(env); 1570 run(["dmd", "-run", "rdmd_test.d"], env.vars, sourceDir); 1571 } 1572 } 1573 1574 /// Tools package with all its components, including rdmd. 1575 final class Tools : Component 1576 { 1577 @property override string submoduleName() { return "tools"; } 1578 @property override string[] sourceDependencies() { return []; } 1579 @property override string[] dependencies() { return ["dmd", "druntime", "phobos"]; } 1580 1581 @property string model() { return config.build.components.common.models.get(0); } 1582 1583 @property override string configString() 1584 { 1585 static struct FullConfig 1586 { 1587 string model; 1588 string[] makeArgs; 1589 } 1590 1591 return FullConfig( 1592 this.model, 1593 config.build.components.common.makeArgs, 1594 ).toJson(); 1595 } 1596 1597 override void performBuild() 1598 { 1599 getComponent("dmd").needSource(); 1600 foreach (dep; ["dmd", "druntime", "phobos"]) 1601 getComponent(dep).needInstalled(); 1602 1603 auto env = baseEnvironment; 1604 needCC(env, this.model); 1605 1606 run(getMake(env) ~ ["-f", makeFileName, "DMD=" ~ dmd] ~ config.build.components.common.makeArgs ~ getPlatformMakeVars(env, this.model) ~ dMakeArgs, env.vars, sourceDir); 1607 } 1608 1609 override void performStage() 1610 { 1611 foreach (os; buildPath(sourceDir, "generated").dirEntries(SpanMode.shallow)) 1612 { 1613 auto dir = os.buildPath(this.model); 1614 cp(dir, buildPath(stageDir , "bin")); 1615 } 1616 } 1617 } 1618 1619 /// Website (dlang.org). Only buildable on POSIX. 1620 final class Website : Component 1621 { 1622 @property override string submoduleName() { return "dlang.org"; } 1623 @property override string[] sourceDependencies() { return ["druntime", "phobos"]; } 1624 @property override string[] dependencies() { return ["dmd", "druntime", "phobos", "rdmd"]; } 1625 1626 struct Config 1627 { 1628 /// Do not include timestamps, line numbers, or other 1629 /// volatile dynamic content in generated .ddoc files. 1630 /// Improves cache efficiency and allows meaningful diffs. 1631 bool diffable = false; 1632 1633 deprecated alias noDateTime = diffable; 1634 } 1635 1636 @property override string configString() 1637 { 1638 static struct FullConfig 1639 { 1640 Config config; 1641 } 1642 1643 return FullConfig( 1644 config.build.components.website, 1645 ).toJson(); 1646 } 1647 1648 /// Get the latest version of DMD at the time. 1649 /// Needed for the makefile's "LATEST" parameter. 1650 string getLatest() 1651 { 1652 auto dmd = getComponent("dmd").submodule; 1653 dmd.needRepo(); 1654 1655 auto t = dmd.git.query(["log", "--pretty=format:%ct"]).splitLines.map!(to!int).filter!(n => n > 0).front; 1656 1657 foreach (line; dmd.git.query(["log", "--decorate=full", "--tags", "--pretty=format:%ct%d"]).splitLines()) 1658 if (line.length > 10 && line[0..10].to!int < t) 1659 if (line[10..$].startsWith(" (") && line.endsWith(")")) 1660 { 1661 foreach (r; line[12..$-1].split(", ")) 1662 if (r.skipOver("tag: refs/tags/")) 1663 if (r.match(re!`^v2\.\d\d\d(\.\d)?$`)) 1664 return r[1..$]; 1665 } 1666 throw new Exception("Can't find any DMD version tags at this point!"); 1667 } 1668 1669 private enum Target { build, test } 1670 1671 private void make(Target target) 1672 { 1673 auto env = baseEnvironment; 1674 1675 version (Windows) 1676 throw new Exception("The dlang.org website is only buildable on POSIX platforms."); 1677 else 1678 { 1679 getComponent("dmd").updateEnv(env); 1680 1681 needKindleGen(env); 1682 1683 foreach (dep; dependencies) 1684 getComponent(dep).submodule.clean = false; 1685 1686 auto makeFullName = sourceDir.buildPath(makeFileName); 1687 makeFullName 1688 .readText() 1689 // https://github.com/D-Programming-Language/dlang.org/pull/1011 1690 .replace(": modlist.d", ": modlist.d $(DMD)") 1691 // https://github.com/D-Programming-Language/dlang.org/pull/1017 1692 .replace("dpl-docs: ${DUB} ${STABLE_DMD}\n\tDFLAGS=", "dpl-docs: ${DUB} ${STABLE_DMD}\n\t${DUB} upgrade --missing-only --root=${DPL_DOCS_PATH}\n\tDFLAGS=") 1693 .toFile(makeFullName) 1694 ; 1695 submodule.saveFileState(makeFileName); 1696 1697 // Retroactive OpenSSL 1.1.0 fix 1698 // See https://github.com/dlang/dlang.org/pull/1654 1699 auto dubJson = sourceDir.buildPath("dpl-docs/dub.json"); 1700 dubJson 1701 .readText() 1702 .replace(`"versions": ["VibeCustomMain"]`, `"versions": ["VibeCustomMain", "VibeNoSSL"]`) 1703 .toFile(dubJson); 1704 submodule.saveFileState("dpl-docs/dub.json"); 1705 scope(exit) submodule.saveFileState("dpl-docs/dub.selections.json"); 1706 1707 string latest = null; 1708 if (!sourceDir.buildPath("VERSION").exists) 1709 { 1710 latest = getLatest(); 1711 log("LATEST=" ~ latest); 1712 } 1713 else 1714 log("VERSION file found, not passing LATEST parameter"); 1715 1716 string[] diffable = null; 1717 1718 string[] targets = 1719 [ 1720 config.build.components.website.diffable 1721 ? ["all", "verbatim", "pdf"] ~ ( 1722 makeFullName.readText.indexOf("diffable-intermediaries") >= 0 1723 ? ["diffable-intermediaries"] 1724 : ["dlangspec.html"]) 1725 : ["all", "verbatim", "pdf", "kindle"], 1726 ["test"] 1727 ][target]; 1728 1729 if (config.build.components.website.diffable) 1730 { 1731 if (makeFullName.readText.indexOf("DIFFABLE") >= 0) 1732 diffable = ["DIFFABLE=1"]; 1733 else 1734 diffable = ["NODATETIME=nodatetime.ddoc"]; 1735 1736 env.vars["SOURCE_DATE_EPOCH"] = "0"; 1737 } 1738 1739 auto args = 1740 getMake(env) ~ 1741 [ "-f", makeFileName ] ~ 1742 diffable ~ 1743 (latest ? ["LATEST=" ~ latest] : []) ~ 1744 targets ~ 1745 gnuMakeArgs; 1746 run(args, env.vars, sourceDir); 1747 } 1748 } 1749 1750 override void performBuild() 1751 { 1752 foreach (dep; ["dmd", "druntime", "phobos"]) 1753 { 1754 auto c = getComponent(dep); 1755 c.needInstalled(); 1756 1757 // Need DMD source because https://github.com/dlang/phobos/pull/4613#issuecomment-266462596 1758 // Need Druntime/Phobos source because we are building its documentation from there. 1759 c.needSource(); 1760 } 1761 getComponent("tools").needSource(); // for changed.d 1762 1763 make(Target.build); 1764 } 1765 1766 override void performTest() 1767 { 1768 make(Target.test); 1769 } 1770 1771 override void performStage() 1772 { 1773 foreach (item; ["web", "dlangspec.tex", "dlangspec.html"]) 1774 { 1775 auto src = buildPath(sourceDir, item); 1776 auto dst = buildPath(stageDir , item); 1777 if (src.exists) 1778 cp(src, dst); 1779 } 1780 } 1781 } 1782 1783 /// Extras not built from source (DigitalMars and third-party tools and libraries) 1784 final class Extras : Component 1785 { 1786 @property override string submoduleName() { return null; } 1787 @property override string[] sourceDependencies() { return []; } 1788 @property override string[] dependencies() { return []; } 1789 @property override string configString() { return null; } 1790 1791 override void performBuild() 1792 { 1793 needExtras(); 1794 } 1795 1796 override void performStage() 1797 { 1798 auto extrasDir = needExtras(); 1799 1800 void copyDir(string source, string target) 1801 { 1802 source = buildPath(extrasDir, "localextras-" ~ platform, "dmd2", platform, source); 1803 target = buildPath(stageDir, target); 1804 if (source.exists) 1805 cp(source, target); 1806 } 1807 1808 copyDir("bin", "bin"); 1809 foreach (model; config.build.components.common.models) 1810 copyDir("bin" ~ model, "bin"); 1811 copyDir("lib", "lib"); 1812 1813 version (Windows) 1814 foreach (model; config.build.components.common.models) 1815 if (model == "32") 1816 { 1817 // The version of snn.lib bundled with DMC will be newer. 1818 Environment env; 1819 needDMC(env); 1820 cp(buildPath(env.deps.dmcDir, "lib", "snn.lib"), buildPath(stageDir, "lib", "snn.lib")); 1821 } 1822 } 1823 } 1824 1825 /// libcurl DLL and import library for Windows. 1826 final class Curl : Component 1827 { 1828 @property override string submoduleName() { return null; } 1829 @property override string[] sourceDependencies() { return []; } 1830 @property override string[] dependencies() { return []; } 1831 @property override string configString() { return null; } 1832 1833 override void performBuild() 1834 { 1835 version (Windows) 1836 needCurl(); 1837 else 1838 log("Not on Windows, skipping libcurl download"); 1839 } 1840 1841 override void performStage() 1842 { 1843 version (Windows) 1844 { 1845 auto curlDir = needCurl(); 1846 1847 void copyDir(string source, string target) 1848 { 1849 source = buildPath(curlDir, "dmd2", "windows", source); 1850 target = buildPath(stageDir, target); 1851 if (source.exists) 1852 cp(source, target); 1853 } 1854 1855 foreach (model; config.build.components.common.models) 1856 { 1857 auto suffix = model == "64" ? "64" : ""; 1858 copyDir("bin" ~ suffix, "bin"); 1859 copyDir("lib" ~ suffix, "lib"); 1860 } 1861 } 1862 else 1863 log("Not on Windows, skipping libcurl install"); 1864 } 1865 1866 override void updateEnv(ref Environment env) 1867 { 1868 env.vars["PATH"] = buildPath(buildDir, "bin").absolutePath() ~ pathSeparator ~ env.vars["PATH"]; 1869 } 1870 } 1871 1872 private int tempError; 1873 1874 private Component[string] components; 1875 1876 Component getComponent(string name) 1877 { 1878 if (name !in components) 1879 { 1880 Component c; 1881 1882 switch (name) 1883 { 1884 case "dmd": 1885 c = new DMD(); 1886 break; 1887 case "phobos-includes": 1888 c = new PhobosIncludes(); 1889 break; 1890 case "druntime": 1891 c = new Druntime(); 1892 break; 1893 case "phobos": 1894 c = new Phobos(); 1895 break; 1896 case "rdmd": 1897 c = new RDMD(); 1898 break; 1899 case "tools": 1900 c = new Tools(); 1901 break; 1902 case "website": 1903 c = new Website(); 1904 break; 1905 case "extras": 1906 c = new Extras(); 1907 break; 1908 case "curl": 1909 c = new Curl(); 1910 break; 1911 default: 1912 throw new Exception("Unknown component: " ~ name); 1913 } 1914 1915 c.name = name; 1916 return components[name] = c; 1917 } 1918 1919 return components[name]; 1920 } 1921 1922 Component[] getSubmoduleComponents(string submoduleName) 1923 { 1924 return components 1925 .byValue 1926 .filter!(component => component.submoduleName == submoduleName) 1927 .array(); 1928 } 1929 1930 // **************************** Customization **************************** 1931 1932 /// Fetch latest D history. 1933 void update() 1934 { 1935 getMetaRepo().update(); 1936 } 1937 1938 struct SubmoduleState 1939 { 1940 string[string] submoduleCommits; 1941 } 1942 1943 /// Begin customization, starting at the specified commit. 1944 SubmoduleState begin(string commit) 1945 { 1946 log("Starting at meta repository commit " ~ commit); 1947 return SubmoduleState(getMetaRepo().getSubmoduleCommits(commit)); 1948 } 1949 1950 /// Applies a merge onto the given SubmoduleState. 1951 void merge(ref SubmoduleState submoduleState, string submoduleName, string branch) 1952 { 1953 log("Merging %s commit %s".format(submoduleName, branch)); 1954 enforce(submoduleName in submoduleState.submoduleCommits, "Unknown submodule: " ~ submoduleName); 1955 auto submodule = getSubmodule(submoduleName); 1956 auto head = submoduleState.submoduleCommits[submoduleName]; 1957 auto result = submodule.getMerge(head, branch); 1958 submoduleState.submoduleCommits[submoduleName] = result; 1959 } 1960 1961 /// Removes a merge from the given SubmoduleState. 1962 void unmerge(ref SubmoduleState submoduleState, string submoduleName, string branch) 1963 { 1964 log("Unmerging %s commit %s".format(submoduleName, branch)); 1965 enforce(submoduleName in submoduleState.submoduleCommits, "Unknown submodule: " ~ submoduleName); 1966 auto submodule = getSubmodule(submoduleName); 1967 auto head = submoduleState.submoduleCommits[submoduleName]; 1968 auto result = submodule.getUnMerge(head, branch); 1969 submoduleState.submoduleCommits[submoduleName] = result; 1970 } 1971 1972 /// Reverts a commit from the given SubmoduleState. 1973 /// parent is the 1-based mainline index (as per `man git-revert`), 1974 /// or 0 if commit is not a merge commit. 1975 void revert(ref SubmoduleState submoduleState, string submoduleName, string commit, int parent) 1976 { 1977 log("Reverting %s commit %s".format(submoduleName, commit)); 1978 enforce(submoduleName in submoduleState.submoduleCommits, "Unknown submodule: " ~ submoduleName); 1979 auto submodule = getSubmodule(submoduleName); 1980 auto head = submoduleState.submoduleCommits[submoduleName]; 1981 auto result = submodule.getRevert(head, commit, parent); 1982 submoduleState.submoduleCommits[submoduleName] = result; 1983 } 1984 1985 /// Returns the commit hash for the given pull request #. 1986 /// The result can then be used with addMerge/removeMerge. 1987 string getPull(string submoduleName, int pullNumber) 1988 { 1989 return getSubmodule(submoduleName).getPull(pullNumber); 1990 } 1991 1992 /// Returns the commit hash for the given GitHub fork. 1993 /// The result can then be used with addMerge/removeMerge. 1994 string getFork(string submoduleName, string user, string branch) 1995 { 1996 return getSubmodule(submoduleName).getFork(user, branch); 1997 } 1998 1999 /// Find the child of a commit (starting with the current submodule state), 2000 /// and, if the commit was a merge, the mainline index of said commit for the child. 2001 void getChild(ref SubmoduleState submoduleState, string submoduleName, string commit, out string child, out int mainline) 2002 { 2003 enforce(submoduleName in submoduleState.submoduleCommits, "Unknown submodule: " ~ submoduleName); 2004 auto head = submoduleState.submoduleCommits[submoduleName]; 2005 return getSubmodule(submoduleName).getChild(head, commit, child, mainline); 2006 } 2007 2008 // ****************************** Building ******************************* 2009 2010 private SubmoduleState submoduleState; 2011 private bool incrementalBuild; 2012 2013 @property string cacheEngineName() 2014 { 2015 if (incrementalBuild) 2016 return "none"; 2017 else 2018 return config.local.cache; 2019 } 2020 2021 private string getComponentCommit(string componentName) 2022 { 2023 auto submoduleName = getComponent(componentName).submoduleName; 2024 auto commit = submoduleState.submoduleCommits.get(submoduleName, null); 2025 enforce(commit, "Unknown commit to build for component %s (submodule %s)" 2026 .format(componentName, submoduleName)); 2027 return commit; 2028 } 2029 2030 static const string[] defaultComponents = ["dmd", "druntime", "phobos-includes", "phobos", "rdmd"]; 2031 static const string[] additionalComponents = ["tools", "website", "extras", "curl"]; 2032 static const string[] allComponents = defaultComponents ~ additionalComponents; 2033 2034 /// Build the specified components according to the specified configuration. 2035 void build(SubmoduleState submoduleState, bool incremental = false) 2036 { 2037 auto componentNames = config.build.components.getEnabledComponentNames(); 2038 log("Building components %-(%s, %)".format(componentNames)); 2039 2040 this.components = null; 2041 this.submoduleState = submoduleState; 2042 this.incrementalBuild = incremental; 2043 2044 if (buildDir.exists) 2045 buildDir.removeRecurse(); 2046 enforce(!buildDir.exists); 2047 2048 scope(exit) if (cacheEngine) cacheEngine.finalize(); 2049 2050 foreach (componentName; componentNames) 2051 getComponent(componentName).needInstalled(); 2052 } 2053 2054 /// Shortcut for begin + build 2055 void buildRev(string rev) 2056 { 2057 auto submoduleState = begin(rev); 2058 build(submoduleState); 2059 } 2060 2061 /// Simply check out the source code for the given submodules. 2062 void checkout(SubmoduleState submoduleState) 2063 { 2064 auto componentNames = config.build.components.getEnabledComponentNames(); 2065 log("Checking out components %-(%s, %)".format(componentNames)); 2066 2067 this.components = null; 2068 this.submoduleState = submoduleState; 2069 this.incrementalBuild = false; 2070 2071 foreach (componentName; componentNames) 2072 getComponent(componentName).needSource(); 2073 } 2074 2075 /// Rerun build without cleaning up any files. 2076 void rebuild() 2077 { 2078 build(SubmoduleState(null), true); 2079 } 2080 2081 /// Run all tests for the current checkout (like rebuild). 2082 void test() 2083 { 2084 auto componentNames = config.build.components.getEnabledComponentNames(); 2085 log("Testing components %-(%s, %)".format(componentNames)); 2086 2087 this.components = null; 2088 this.submoduleState = SubmoduleState(null); 2089 this.incrementalBuild = true; 2090 2091 foreach (componentName; componentNames) 2092 getComponent(componentName).test(); 2093 } 2094 2095 bool isCached(SubmoduleState submoduleState) 2096 { 2097 this.components = null; 2098 this.submoduleState = submoduleState; 2099 2100 needCacheEngine(); 2101 foreach (componentName; config.build.components.getEnabledComponentNames()) 2102 if (!cacheEngine.haveEntry(getComponent(componentName).getBuildID())) 2103 return false; 2104 return true; 2105 } 2106 2107 /// Returns the isCached state for all commits in the history of the given ref. 2108 bool[string] getCacheState(string[string][string] history) 2109 { 2110 log("Enumerating cache entries..."); 2111 auto cacheEntries = needCacheEngine().getEntries().toSet(); 2112 2113 this.components = null; 2114 auto componentNames = config.build.components.getEnabledComponentNames(); 2115 auto components = componentNames.map!(componentName => getComponent(componentName)).array; 2116 auto requiredSubmodules = components 2117 .map!(component => chain(component.name.only, component.sourceDependencies, component.dependencies)) 2118 .joiner 2119 .map!(componentName => getComponent(componentName).submoduleName) 2120 .array.sort().uniq().array 2121 ; 2122 2123 log("Collating cache state..."); 2124 bool[string] result; 2125 foreach (commit, submoduleCommits; history) 2126 { 2127 this.submoduleState.submoduleCommits = submoduleCommits; 2128 2129 result[commit] = 2130 requiredSubmodules.all!(submoduleName => submoduleName in submoduleCommits) && 2131 componentNames.all!(componentName => 2132 getComponent(componentName).I!(component => 2133 component.getBuildID() in cacheEntries 2134 ) 2135 ); 2136 } 2137 return result; 2138 } 2139 2140 /// ditto 2141 bool[string] getCacheState(string[] refs) 2142 { 2143 auto history = getMetaRepo().getSubmoduleHistory(refs); 2144 return getCacheState(history); 2145 } 2146 2147 // **************************** Dependencies ***************************** 2148 2149 private void needInstaller() 2150 { 2151 Installer.logger = &log; 2152 Installer.installationDirectory = dlDir; 2153 } 2154 2155 /// Pull in a built DMD as configured. 2156 /// Note that this function invalidates the current repository state. 2157 void needDMD(ref Environment env) 2158 { 2159 tempError++; scope(success) tempError--; 2160 2161 auto dmdVer = config.build.components.dmd.bootstrap.ver; 2162 if (!dmdVer) 2163 { 2164 dmdVer = "v2.067.1"; 2165 version (Windows) 2166 if (config.build.components.dmd.dmdModel != Component.CommonConfig.defaultModel) 2167 dmdVer = "v2.070.2"; // dmd/src/builtin.d needs core.stdc.math.fabsl. 2.068.2 generates a dmd which crashes on building Phobos 2168 } 2169 2170 if (config.build.components.dmd.bootstrap.fromSource) 2171 { 2172 log("Bootstrapping DMD " ~ dmdVer); 2173 2174 auto bootstrapBuildConfig = config.build.components.dmd.bootstrap.build; 2175 2176 // Back up and clear component state 2177 enum backupTemplate = q{ 2178 auto VARBackup = this.VAR; 2179 this.VAR = typeof(VAR).init; 2180 scope(exit) this.VAR = VARBackup; 2181 }; 2182 mixin(backupTemplate.replace(q{VAR}, q{components})); 2183 mixin(backupTemplate.replace(q{VAR}, q{config})); 2184 mixin(backupTemplate.replace(q{VAR}, q{submoduleState})); 2185 2186 config.local = configBackup.local; 2187 if (bootstrapBuildConfig) 2188 config.build = *bootstrapBuildConfig; 2189 2190 // Disable building rdmd in the bootstrap compiler by default 2191 if ("rdmd" !in config.build.components.enable) 2192 config.build.components.enable["rdmd"] = false; 2193 2194 build(parseSpec(dmdVer)); 2195 2196 log("Built bootstrap DMD " ~ dmdVer ~ " successfully."); 2197 2198 auto bootstrapDir = buildPath(config.local.workDir, "bootstrap"); 2199 if (bootstrapDir.exists) 2200 bootstrapDir.removeRecurse(); 2201 ensurePathExists(bootstrapDir); 2202 rename(buildDir, bootstrapDir); 2203 2204 env.deps.hostDC = buildPath(bootstrapDir, "bin", "dmd" ~ binExt); 2205 } 2206 else 2207 { 2208 import std.ascii; 2209 log("Preparing DMD " ~ dmdVer); 2210 enforce(dmdVer.startsWith("v"), "Invalid DMD version spec for binary bootstrap. Did you forget to " ~ 2211 ((dmdVer.length && dmdVer[0].isDigit && dmdVer.contains('.')) ? "add a leading 'v'" : "enable fromSource") ~ "?"); 2212 needInstaller(); 2213 auto dmdInstaller = new DMDInstaller(dmdVer[1..$]); 2214 dmdInstaller.requireLocal(false); 2215 env.deps.hostDC = dmdInstaller.exePath("dmd").absolutePath(); 2216 } 2217 2218 log("hostDC=" ~ env.deps.hostDC); 2219 } 2220 2221 void needKindleGen(ref Environment env) 2222 { 2223 needInstaller(); 2224 kindleGenInstaller.requireLocal(false); 2225 env.vars["PATH"] = kindleGenInstaller.directory ~ pathSeparator ~ env.vars["PATH"]; 2226 } 2227 2228 version (Windows) 2229 void needMSYS(ref Environment env) 2230 { 2231 needInstaller(); 2232 MSYS.msysCORE.requireLocal(false); 2233 MSYS.libintl.requireLocal(false); 2234 MSYS.libiconv.requireLocal(false); 2235 MSYS.libtermcap.requireLocal(false); 2236 MSYS.libregex.requireLocal(false); 2237 MSYS.coreutils.requireLocal(false); 2238 MSYS.bash.requireLocal(false); 2239 MSYS.make.requireLocal(false); 2240 MSYS.grep.requireLocal(false); 2241 MSYS.sed.requireLocal(false); 2242 MSYS.diffutils.requireLocal(false); 2243 env.vars["PATH"] = MSYS.bash.directory.buildPath("bin") ~ pathSeparator ~ env.vars["PATH"]; 2244 } 2245 2246 /// Get DMD unbuildable extras 2247 /// (proprietary DigitalMars utilities, 32-bit import libraries) 2248 string needExtras() 2249 { 2250 import ae.utils.meta : I, singleton; 2251 2252 static class DExtrasInstaller : Installer 2253 { 2254 @property override string name() { return "dmd-localextras"; } 2255 string url = "http://semitwist.com/download/app/dmd-localextras.7z"; 2256 2257 override void installImpl(string target) 2258 { 2259 url 2260 .I!save() 2261 .I!unpackTo(target); 2262 } 2263 2264 static this() 2265 { 2266 urlDigests["http://semitwist.com/download/app/dmd-localextras.7z"] = "ef367c2d25d4f19f45ade56ab6991c726b07d3d9"; 2267 } 2268 } 2269 2270 alias extrasInstaller = singleton!DExtrasInstaller; 2271 2272 needInstaller(); 2273 extrasInstaller.requireLocal(false); 2274 return extrasInstaller.directory; 2275 } 2276 2277 /// Get libcurl for Windows (DLL and import libraries) 2278 version (Windows) 2279 string needCurl() 2280 { 2281 import ae.utils.meta : I, singleton; 2282 2283 static class DCurlInstaller : Installer 2284 { 2285 @property override string name() { return "libcurl-" ~ curlVersion; } 2286 string curlVersion = "7.47.1"; 2287 @property string url() { return "http://downloads.dlang.org/other/libcurl-" ~ curlVersion ~ "-WinSSL-zlib-x86-x64.zip"; } 2288 2289 override void installImpl(string target) 2290 { 2291 url 2292 .I!save() 2293 .I!unpackTo(target); 2294 } 2295 2296 static this() 2297 { 2298 urlDigests["http://downloads.dlang.org/other/libcurl-7.47.1-WinSSL-zlib-x86-x64.zip"] = "4b8a7bb237efab25a96588093ae51994c821e097"; 2299 } 2300 } 2301 2302 alias curlInstaller = singleton!DCurlInstaller; 2303 2304 needInstaller(); 2305 curlInstaller.requireLocal(false); 2306 return curlInstaller.directory; 2307 } 2308 2309 version (Windows) 2310 void needDMC(ref Environment env, string ver = null) 2311 { 2312 tempError++; scope(success) tempError--; 2313 2314 needInstaller(); 2315 2316 auto dmc = ver ? new LegacyDMCInstaller(ver) : dmcInstaller; 2317 if (!dmc.installedLocally) 2318 log("Preparing DigitalMars C++ " ~ ver); 2319 dmc.requireLocal(false); 2320 env.deps.dmcDir = dmc.directory; 2321 2322 auto binPath = buildPath(env.deps.dmcDir, `bin`).absolutePath(); 2323 log("DMC=" ~ binPath); 2324 env.vars["DMC"] = binPath; 2325 env.vars["PATH"] = binPath ~ pathSeparator ~ env.vars.get("PATH", null); 2326 } 2327 2328 version (Windows) 2329 auto getVSInstaller() 2330 { 2331 needInstaller(); 2332 return vs2013community; 2333 } 2334 2335 version (Windows) 2336 static string msvcModelStr(string model, string str32, string str64) 2337 { 2338 switch (model) 2339 { 2340 case "32": 2341 throw new Exception("Shouldn't need VC for 32-bit builds"); 2342 case "64": 2343 return str64; 2344 case "32mscoff": 2345 return str32; 2346 default: 2347 throw new Exception("Unknown model: " ~ model); 2348 } 2349 } 2350 2351 version (Windows) 2352 static string msvcModelDir(string model, string dir64 = "x86_amd64") 2353 { 2354 return msvcModelStr(model, null, dir64); 2355 } 2356 2357 version (Windows) 2358 void needVC(ref Environment env, string model) 2359 { 2360 tempError++; scope(success) tempError--; 2361 2362 auto vs = getVSInstaller(); 2363 2364 // At minimum, we want the C compiler (cl.exe) and linker (link.exe). 2365 vs["vc_compilercore86"].requireLocal(false); // Contains both x86 and x86_amd64 cl.exe 2366 vs["vc_compilercore86res"].requireLocal(false); // Contains clui.dll needed by cl.exe 2367 2368 // Include files. Needed when using VS to build either DMD or Druntime. 2369 vs["vc_librarycore86"].requireLocal(false); // Contains include files, e.g. errno.h needed by Druntime 2370 2371 // C runtime. Needed for all programs built with VC. 2372 vs[msvcModelStr(model, "vc_libraryDesktop_x86", "vc_libraryDesktop_x64")].requireLocal(false); // libcmt.lib 2373 2374 // XP-compatible import libraries. 2375 vs["win_xpsupport"].requireLocal(false); // shell32.lib 2376 2377 // MSBuild, for the useVC option 2378 if (config.build.components.dmd.useVC) 2379 vs["Msi_BuildTools_MSBuild_x86"].requireLocal(false); // msbuild.exe 2380 2381 env.deps.vsDir = vs.directory.buildPath("Program Files (x86)", "Microsoft Visual Studio 12.0").absolutePath(); 2382 env.deps.sdkDir = vs.directory.buildPath("Program Files", "Microsoft SDKs", "Windows", "v7.1A").absolutePath(); 2383 2384 env.vars["PATH"] ~= pathSeparator ~ vs.modelBinPaths(msvcModelDir(model)).map!(path => vs.directory.buildPath(path).absolutePath()).join(pathSeparator); 2385 env.vars["VCINSTALLDIR"] = env.deps.vsDir.buildPath("VC") ~ dirSeparator; 2386 env.vars["INCLUDE"] = env.deps.vsDir.buildPath("VC", "include") ~ ";" ~ env.deps.sdkDir.buildPath("Include"); 2387 env.vars["LIB"] = env.deps.vsDir.buildPath("VC", "lib", msvcModelDir(model, "amd64")) ~ ";" ~ env.deps.sdkDir.buildPath("Lib", msvcModelDir(model, "x64")); 2388 env.vars["WindowsSdkDir"] = env.deps.sdkDir ~ dirSeparator; 2389 env.vars["Platform"] = "x64"; 2390 env.vars["LINKCMD64"] = env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "link.exe"); // Used by dmd 2391 env.vars["MSVC_CC"] = env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "cl.exe"); // For the msvc-dmc wrapper 2392 env.vars["MSVC_AR"] = env.deps.vsDir.buildPath("VC", "bin", msvcModelDir(model), "lib.exe"); // For the msvc-lib wrapper 2393 env.vars["CL"] = "-D_USING_V110_SDK71_"; // Work around __userHeader macro redifinition VS bug 2394 } 2395 2396 private void needGit() 2397 { 2398 tempError++; scope(success) tempError--; 2399 2400 needInstaller(); 2401 gitInstaller.require(); 2402 } 2403 2404 /// Disable the "<program> has stopped working" 2405 /// standard Windows dialog. 2406 version (Windows) 2407 static void disableCrashDialog() 2408 { 2409 enum : uint { SEM_FAILCRITICALERRORS = 1, SEM_NOGPFAULTERRORBOX = 2 } 2410 SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); 2411 } 2412 2413 /// Create a build environment base. 2414 protected @property Environment baseEnvironment() 2415 { 2416 Environment env; 2417 2418 // Build a new environment from scratch, to avoid tainting the build with the current environment. 2419 string[] newPaths; 2420 2421 version (Windows) 2422 { 2423 import std.utf; 2424 import ae.sys.windows.imports; 2425 mixin(importWin32!q{winbase}); 2426 mixin(importWin32!q{winnt}); 2427 2428 TCHAR[1024] buf; 2429 // Needed for DLLs 2430 auto winDir = buf[0..GetWindowsDirectory(buf.ptr, buf.length)].toUTF8(); 2431 auto sysDir = buf[0..GetSystemDirectory (buf.ptr, buf.length)].toUTF8(); 2432 newPaths ~= [sysDir, winDir]; 2433 } 2434 else 2435 { 2436 // Needed for coreutils, make, gcc, git etc. 2437 newPaths = ["/bin", "/usr/bin"]; 2438 2439 version (linux) 2440 { 2441 // GCC wrappers 2442 ensureDirExists(binDir); 2443 newPaths = binDir ~ newPaths; 2444 } 2445 } 2446 2447 env.vars["PATH"] = newPaths.join(pathSeparator); 2448 2449 ensureDirExists(tmpDir); 2450 env.vars["TMPDIR"] = env.vars["TEMP"] = env.vars["TMP"] = tmpDir; 2451 2452 version (Windows) 2453 { 2454 env.vars["SystemDrive"] = winDir.driveName; 2455 env.vars["SystemRoot"] = winDir; 2456 } 2457 2458 ensureDirExists(homeDir); 2459 env.vars["HOME"] = homeDir; 2460 2461 return env; 2462 } 2463 2464 /// Apply user modifications onto an environment. 2465 /// Supports Windows-style %VAR% expansions. 2466 static string[string] applyEnv(in string[string] target, in string[string] source) 2467 { 2468 // The source of variable expansions is variables in the target environment, 2469 // if they exist, and the host environment otherwise, so e.g. 2470 // `PATH=C:\...;%PATH%` and `MAKE=%MAKE%` work as expected. 2471 auto oldEnv = std.process.environment.toAA(); 2472 foreach (name, value; target) 2473 oldEnv[name] = value; 2474 2475 string[string] result; 2476 foreach (name, value; target) 2477 result[name] = value; 2478 foreach (name, value; source) 2479 { 2480 string newValue = value; 2481 foreach (oldName, oldValue; oldEnv) 2482 newValue = newValue.replace("%" ~ oldName ~ "%", oldValue); 2483 result[name] = oldEnv[name] = newValue; 2484 } 2485 return result; 2486 } 2487 2488 // ******************************** Cache ******************************** 2489 2490 enum unbuildableMarker = "unbuildable"; 2491 2492 DCache cacheEngine; 2493 2494 DCache needCacheEngine() 2495 { 2496 if (!cacheEngine) 2497 { 2498 if (cacheEngineName == "git") 2499 needGit(); 2500 cacheEngine = createCache(cacheEngineName, cacheEngineDir(cacheEngineName), this); 2501 } 2502 return cacheEngine; 2503 } 2504 2505 void cp(string src, string dst) 2506 { 2507 needCacheEngine().cp(src, dst); 2508 } 2509 2510 private string[] getComponentKeyOrder(string componentName) 2511 { 2512 auto submodule = getComponent(componentName).submodule; 2513 submodule.needRepo(); 2514 return submodule 2515 .git.query("log", "--pretty=format:%H", "--all", "--topo-order") 2516 .splitLines() 2517 .map!(commit => componentName ~ "-" ~ commit ~ "-") 2518 .array 2519 ; 2520 } 2521 2522 string componentNameFromKey(string key) 2523 { 2524 auto parts = key.split("-"); 2525 return parts[0..$-2].join("-"); 2526 } 2527 2528 string[][] getKeyOrder(string key) 2529 { 2530 if (key !is null) 2531 return [getComponentKeyOrder(componentNameFromKey(key))]; 2532 else 2533 return allComponents.map!(componentName => getComponentKeyOrder(componentName)).array; 2534 } 2535 2536 /// Optimize entire cache. 2537 void optimizeCache() 2538 { 2539 needCacheEngine().optimize(); 2540 } 2541 2542 bool shouldPurge(string key) 2543 { 2544 auto files = cacheEngine.listFiles(key); 2545 if (files.canFind(unbuildableMarker)) 2546 return true; 2547 2548 if (componentNameFromKey(key) == "druntime") 2549 { 2550 if (!files.canFind("import/core/memory.d") 2551 && !files.canFind("import/core/memory.di")) 2552 return true; 2553 } 2554 2555 return false; 2556 } 2557 2558 /// Delete cached "unbuildable" build results. 2559 void purgeUnbuildable() 2560 { 2561 needCacheEngine() 2562 .getEntries 2563 .filter!(key => shouldPurge(key)) 2564 .each!((key) 2565 { 2566 log("Deleting: " ~ key); 2567 cacheEngine.remove(key); 2568 }) 2569 ; 2570 } 2571 2572 /// Move cached files from one cache engine to another. 2573 void migrateCache(string sourceEngineName, string targetEngineName) 2574 { 2575 auto sourceEngine = createCache(sourceEngineName, cacheEngineDir(sourceEngineName), this); 2576 auto targetEngine = createCache(targetEngineName, cacheEngineDir(targetEngineName), this); 2577 auto tempDir = buildPath(config.local.workDir, "temp"); 2578 if (tempDir.exists) 2579 tempDir.removeRecurse(); 2580 log("Enumerating source entries..."); 2581 auto sourceEntries = sourceEngine.getEntries(); 2582 log("Enumerating target entries..."); 2583 auto targetEntries = targetEngine.getEntries().sort(); 2584 foreach (key; sourceEntries) 2585 if (!targetEntries.canFind(key)) 2586 { 2587 log(key); 2588 sourceEngine.extract(key, tempDir, fn => true); 2589 targetEngine.add(key, tempDir); 2590 if (tempDir.exists) 2591 tempDir.removeRecurse(); 2592 } 2593 targetEngine.optimize(); 2594 } 2595 2596 // **************************** Miscellaneous **************************** 2597 2598 struct LogEntry 2599 { 2600 string hash; 2601 string[] message; 2602 SysTime time; 2603 } 2604 2605 /// Gets the D merge log (newest first). 2606 LogEntry[] getLog(string refName = "refs/remotes/origin/master") 2607 { 2608 getMetaRepo().needRepo(); 2609 auto history = getMetaRepo().git.getHistory(); 2610 LogEntry[] logs; 2611 auto master = history.commits[history.refs[refName]]; 2612 for (auto c = master; c; c = c.parents.length ? c.parents[0] : null) 2613 { 2614 auto time = SysTime(c.time.unixTimeToStdTime); 2615 logs ~= LogEntry(c.hash.toString(), c.message, time); 2616 } 2617 return logs; 2618 } 2619 2620 // ***************************** Integration ***************************** 2621 2622 /// Override to add logging. 2623 void log(string line) 2624 { 2625 } 2626 2627 /// Bootstrap description resolution. 2628 /// See DMD.Config.Bootstrap.spec. 2629 /// This is essentially a hack to allow the entire 2630 /// Config structure to be parsed from an .ini file. 2631 SubmoduleState parseSpec(string spec) 2632 { 2633 getMetaRepo().needRepo(); 2634 auto rev = getMetaRepo().getRef("refs/tags/" ~ spec); 2635 log("Resolved " ~ spec ~ " to " ~ rev); 2636 return begin(rev); 2637 } 2638 2639 /// Override this method with one which returns a command, 2640 /// which will invoke the unmergeRebaseEdit function below, 2641 /// passing to it any additional parameters. 2642 /// Note: Currently unused. Was previously used 2643 /// for unmerging things using interactive rebase. 2644 abstract string getCallbackCommand(); 2645 2646 void callback(string[] args) { assert(false); } 2647 }