1 /**
2  * Code to manage a D checkout and its dependencies.
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Vladimir Panteleev <vladimir@thecybershadow.net>
12  */
13 
14 module ae.sys.d.manager;
15 
16 import std.algorithm;
17 import std.array;
18 import std.datetime;
19 import std.exception;
20 import std.file;
21 import std.parallelism : parallel;
22 import std.path;
23 import std.process;
24 import std.range;
25 import std.string;
26 
27 import ae.sys.cmd;
28 import ae.sys.d.builder;
29 import ae.sys.file;
30 import ae.sys.git;
31 
32 version(Windows)
33 {
34 	import ae.sys.install.dmc;
35 	import ae.sys.install.vs;
36 }
37 
38 import ae.sys.install.git;
39 
40 /// Class which manages a D checkout and its dependencies.
41 class DManager
42 {
43 	// **************************** Configuration ****************************
44 
45 	/// DManager configuration.
46 	struct Config
47 	{
48 		/// URL of D git repository hosting D components.
49 		/// Defaults to (and must have the layout of) D.git:
50 		/// https://github.com/CyberShadow/D-dot-git
51 		string repoUrl = "https://bitbucket.org/cybershadow/d.git";
52 
53 		/// Location for the checkout, temporary files, etc.
54 		string workDir;
55 
56 		/// Build configuration.
57 		DBuilder.Config.Build build;
58 	}
59 	Config config; /// ditto
60 
61 	// ******************************* Fields ********************************
62 
63 	/// Get a specific subdirectory of the work directory.
64 	@property string subDir(string name)() { return buildPath(config.workDir, name); }
65 
66 	alias repoDir    = subDir!"repo";        /// The git repository location.
67 	alias buildDir   = subDir!"build";       /// The build directory.
68 	alias dlDir      = subDir!"dl" ;         /// The directory for downloaded software.
69 
70 	version(Windows) string dmcDir, vsDir, sdkDir;
71 	string[] paths;
72 
73 	/// Environment used when building D.
74 	string[string] dEnv;
75 
76 	/// Our custom D builder.
77 	class Builder : DBuilder
78 	{
79 		override void log(string s)
80 		{
81 			this.outer.log(s);
82 		}
83 	}
84 	Builder builder; /// ditto
85 
86 	// **************************** Main methods *****************************
87 
88 	/// Initialize the repository and prerequisites.
89 	void initialize(bool update)
90 	{
91 		log("Preparing prerequisites...");
92 		prepareRepoPrerequisites();
93 
94 		log("Preparing repository...");
95 		prepareRepo(update);
96 	}
97 
98 	/// Build D.
99 	void build()
100 	{
101 		log("Preparing build prerequisites...");
102 		prepareBuildPrerequisites();
103 
104 		log("Preparing to build...");
105 		prepareEnv();
106 		prepareBuilder();
107 
108 		log("Building...");
109 		mkdir(buildDir);
110 		builder.build();
111 	}
112 
113 	void incrementalBuild()
114 	{
115 		prepareEnv();
116 		prepareBuilder();
117 		builder.build();
118 	}
119 
120 	/// Go to a specific revision.
121 	/// Assumes a clean state (call reset first).
122 	void checkout(string rev)
123 	{
124 		if (!rev)
125 			rev = "origin/master";
126 
127 		log("Checking out %s...".format(rev));
128 		repo.run("checkout", rev);
129 
130 		log("Updating submodules...");
131 		repo.run("submodule", "update");
132 	}
133 
134 	struct LogEntry
135 	{
136 		string message, hash;
137 		SysTime time;
138 	}
139 
140 	/// Gets the D merge log (newest first).
141 	LogEntry[] getLog()
142 	{
143 		auto history = repo.getHistory();
144 		LogEntry[] logs;
145 		auto master = history.commits[history.refs["refs/remotes/origin/master"]];
146 		for (auto c = master; c; c = c.parents.length ? c.parents[0] : null)
147 		{
148 			auto title = c.message.length ? c.message[0] : null;
149 			auto time = SysTime(c.time.unixTimeToStdTime);
150 			logs ~= LogEntry(title, c.hash.toString(), time);
151 		}
152 		return logs;
153 	}
154 
155 	/// Clean up (delete all built and intermediary files).
156 	void reset()
157 	{
158 		log("Cleaning up...");
159 
160 		if (buildDir.exists)
161 			buildDir.rmdirRecurse();
162 		enforce(!buildDir.exists);
163 
164 		prepareRepoPrerequisites();
165 		repo.run("submodule", "foreach", "git", "reset", "--hard");
166 		repo.run("submodule", "foreach", "git", "clean", "--force", "-x", "-d", "--quiet");
167 		repo.run("submodule", "update");
168 	}
169 
170 	// ************************** Auxiliary methods **************************
171 
172 	/// The repository.
173 	@property Repository repo() { return Repository(repoDir); }
174 
175 	/// Prepare the build environment (dEnv).
176 	void prepareEnv()
177 	{
178 		if (dEnv)
179 			return;
180 
181 		auto oldPaths = environment["PATH"].split(pathSeparator);
182 
183 		// Build a new environment from scratch, to avoid tainting the build with the current environment.
184 		string[] newPaths;
185 
186 		version(Windows)
187 		{
188 			import std.utf;
189 			import win32.winbase;
190 			import win32.winnt;
191 
192 			TCHAR[1024] buf;
193 			auto winDir = buf[0..GetWindowsDirectory(buf.ptr, buf.length)].toUTF8();
194 			auto sysDir = buf[0..GetSystemDirectory (buf.ptr, buf.length)].toUTF8();
195 			auto tmpDir = buf[0..GetTempPath(buf.length, buf.ptr)].toUTF8()[0..$-1];
196 			newPaths ~= [sysDir, winDir];
197 		}
198 		else
199 			newPaths = ["/bin", "/usr/bin"];
200 
201 		// Add component paths, if any
202 		newPaths ~= paths;
203 
204 		// Add the DMD we built
205 		newPaths ~= buildPath(buildDir, "bin").absolutePath();   // For Phobos/Druntime/Tools
206 
207 		// Add the DM tools
208 		version (Windows)
209 		{
210 			if (!dmcDir)
211 				prepareBuildPrerequisites();
212 
213 			auto dmc = buildPath(dmcDir, `bin`).absolutePath();
214 			log("DMC=" ~ dmc);
215 			dEnv["DMC"] = dmc;
216 			newPaths ~= dmc;
217 		}
218 
219 		dEnv["PATH"] = newPaths.join(pathSeparator);
220 
221 		version(Windows)
222 		{
223 			dEnv["TEMP"] = dEnv["TMP"] = tmpDir;
224 			dEnv["SystemRoot"] = winDir;
225 		}
226 	}
227 
228 	/// Create the Builder.
229 	void prepareBuilder()
230 	{
231 		builder = new Builder();
232 		builder.config.build = config.build;
233 		builder.config.local.repoDir = repoDir;
234 		builder.config.local.buildDir = buildDir;
235 		version(Windows)
236 		{
237 			builder.config.local.dmcDir = dmcDir;
238 			builder.config.local.vsDir  = vsDir ;
239 			builder.config.local.sdkDir = sdkDir;
240 		}
241 		builder.config.local.env = dEnv;
242 	}
243 
244 	/// Obtains prerequisites necessary for managing the D repository.
245 	void prepareRepoPrerequisites()
246 	{
247 		Installer.logger = &log;
248 		Installer.installationDirectory = dlDir;
249 
250 		gitInstaller.require();
251 	}
252 
253 	/// Obtains prerequisites necessary for building D with the current configuration.
254 	void prepareBuildPrerequisites()
255 	{
256 		Installer.logger = &log;
257 		Installer.installationDirectory = dlDir;
258 
259 		version(Windows)
260 		{
261 			if (config.build.model == "64")
262 			{
263 				vs2013.requirePackages(
264 					[
265 						"vcRuntimeMinimum_x86",
266 						"vc_compilercore86",
267 						"vc_compilercore86res",
268 						"vc_librarycore86",
269 						"vc_libraryDesktop_x64",
270 						"win_xpsupport",
271 					],
272 				);
273 				vs2013.requireLocal(false);
274 				vsDir  = vs2013.directory.buildPath("Program Files (x86)", "Microsoft Visual Studio 12.0").absolutePath();
275 				sdkDir = vs2013.directory.buildPath("Program Files", "Microsoft SDKs", "Windows", "v7.1A").absolutePath();
276 				paths ~= vs2013.directory.buildPath("Windows", "system32").absolutePath();
277 
278 				// D makefiles use the 64-bit (host architecture) compilers,
279 				// which the Express edition does not include.
280 				// Patch up the local VC installation instead.
281 				auto binDir = vsDir.buildPath("VC", "bin");
282 				cached!(dirLink!(), "link")(buildPath(binDir, "x86_amd64"), buildPath(binDir, "amd64"));
283 				cached!(hardLink!())(buildPath(binDir, "mspdb120.dll"), buildPath(binDir, "amd64", "mspdb120.dll"));
284 			}
285 
286 			// We need DMC even for 64-bit builds (for DM make)
287 			dmcInstaller.requireLocal(false);
288 			dmcDir = dmcInstaller.directory;
289 			log("dmcDir=" ~ dmcDir);
290 		}
291 	}
292 
293 	/// Return array of component (submodule) names.
294 	string[] listComponents()
295 	{
296 		return repo
297 			.query("ls-files")
298 			.splitLines()
299 			.filter!(r => r != ".gitmodules")
300 			.array();
301 	}
302 
303 	/// Return the Git repository of the specified component.
304 	Repository componentRepo(string component)
305 	{
306 		prepareRepoPrerequisites();
307 		return Repository(buildPath(repoDir, component));
308 	}
309 
310 	/// Prepare the checkout and initialize the repository.
311 	/// Clone if necessary, checkout master, optionally update.
312 	void prepareRepo(bool update)
313 	{
314 		if (!repoDir.exists)
315 		{
316 			log("Cloning initial repository...");
317 			scope(failure) log("Check that you have git installed and accessible from PATH.");
318 			run(["git", "clone", "--recursive", config.repoUrl, repoDir]);
319 			return;
320 		}
321 
322 		repo.run("bisect", "reset");
323 		repo.run("checkout", "--force", "master");
324 
325 		if (update)
326 		{
327 			log("Updating repositories...");
328 			auto allRepos = listComponents()
329 				.map!(r => buildPath(repoDir, r))
330 				.chain(repoDir.only)
331 				.array();
332 			foreach (r; allRepos.parallel)
333 				Repository(r).run("-c", "fetch.recurseSubmodules=false", "remote", "update");
334 		}
335 
336 		repo.run("reset", "--hard", "origin/master");
337 	}
338 
339 	/// Override to add logging.
340 	void log(string line)
341 	{
342 	}
343 
344 	void logProgress(string s)
345 	{
346 		log((" " ~ s ~ " ").center(70, '-'));
347 	}
348 }