1 /**
2  * Simple execution of shell commands,
3  * and wrappers for common utilities.
4  *
5  * License:
6  *   This Source Code Form is subject to the terms of
7  *   the Mozilla Public License, v. 2.0. If a copy of
8  *   the MPL was not distributed with this file, You
9  *   can obtain one at http://mozilla.org/MPL/2.0/.
10  *
11  * Authors:
12  *   Vladimir Panteleev <ae@cy.md>
13  */
14 
15 module ae.sys.cmd;
16 
17 import core.thread;
18 
19 import std.exception;
20 import std.process;
21 import std.stdio;
22 import std.string;
23 import std.traits;
24 
25 import ae.sys.data : TData;
26 import ae.sys.file;
27 
28 /// Returns a very unique name for a temporary file.
29 string getTempFileName(string extension)
30 {
31 	import std.random : uniform;
32 	import std.file : tempDir;
33 	import std.path : buildPath;
34 
35 	static int counter;
36 	return buildPath(tempDir(), format("run-%d-%d-%d-%d.%s",
37 		getpid(),
38 		getCurrentThreadID(),
39 		uniform!uint(),
40 		counter++,
41 		extension
42 	));
43 }
44 
45 version (linux)
46 {
47 	import core.sys.posix.sys.types : pid_t;
48 	private extern(C) pid_t gettid();
49 }
50 
51 /// Like `thisProcessID`, but for threads.
52 ulong getCurrentThreadID()
53 {
54 	version (Windows)
55 	{
56 		import core.sys.windows.windows : GetCurrentThreadId;
57 		return GetCurrentThreadId();
58 	}
59 	else
60 	version (linux)
61 	{
62 		return gettid();
63 	}
64 	else
65 	version (Posix)
66 	{
67 		import core.sys.posix.pthread : pthread_self;
68 		return cast(ulong)pthread_self();
69 	}
70 }
71 
72 // ************************************************************************
73 
74 private struct ProcessParams
75 {
76 	string shellCommand;
77 	string[] processArgs;
78 
79 	string toShellCommand() { return shellCommand ? shellCommand :  escapeShellCommand(processArgs); }
80 	// Note: a portable toProcessArgs() cannot exist because CMD.EXE does not use CommandLineToArgvW.
81 
82 	const(string[string]) environment = null;
83 	std.process.Config config = std.process.Config.none;
84 	size_t maxOutput = size_t.max;
85 	string workDir = null;
86 	File[3] files;
87 	size_t numFiles;
88 
89 	this(Params...)(Params params)
90 	{
91 		files = [stdin, stdout, stderr];
92 
93 		static foreach (i; 0 .. params.length)
94 		{{
95 			auto arg = params[i];
96 			static if (i == 0)
97 			{
98 				static if (is(typeof(arg) == string))
99 					shellCommand = arg;
100 				else
101 				static if (is(typeof(arg) == string[]))
102 					processArgs = arg;
103 				else
104 					static assert(false, "Unknown type for process invocation command line: " ~ typeof(arg).stringof);
105 				assert(arg, "Null command");
106 			}
107 			else
108 			{
109 				static if (is(typeof(arg) == string))
110 					workDir = arg;
111 				else
112 				static if (is(typeof(arg) : const(string[string])))
113 					environment = arg;
114 				else
115 				static if (is(typeof(arg) == size_t))
116 					maxOutput = arg;
117 				else
118 				static if (is(typeof(arg) == std.process.Config))
119 					config |= arg;
120 				else
121 				static if (is(typeof(arg) == File))
122 					files[numFiles++] = arg;
123 				else
124 					static assert(false, "Unknown type for process invocation parameter: " ~ typeof(arg).stringof);
125 			}
126 		}}
127 	}
128 }
129 
130 private void invoke(alias runner)(string command)
131 {
132 	//debug scope(failure) std.stdio.writeln("[CWD] ", getcwd());
133 	debug(CMD) std.stdio.stderr.writeln("invoke: ", command);
134 	auto status = runner();
135 	enforce(status == 0,
136 		"Command `%s` failed with status %d".format(command, status));
137 }
138 
139 /// std.process helper.
140 /// Run a command, and throw if it exited with a non-zero status.
141 void run(Params...)(Params params)
142 {
143 	auto parsed = ProcessParams(params);
144 	invoke!({
145 		auto pid = parsed.processArgs
146 			? spawnProcess(
147 				parsed.processArgs, parsed.files[0], parsed.files[1], parsed.files[2],
148 				parsed.environment, parsed.config, parsed.workDir
149 			)
150 			: spawnShell(
151 				parsed.shellCommand, parsed.files[0], parsed.files[1], parsed.files[2],
152 				parsed.environment, parsed.config, parsed.workDir
153 			);
154 		return pid.wait();
155 	})(parsed.toShellCommand());
156 }
157 
158 /// std.process helper.
159 /// Run a command and collect its output.
160 /// Throw if it exited with a non-zero status.
161 string query(Params...)(Params params)
162 {
163 	auto parsed = ProcessParams(params, Config.stderrPassThrough);
164 	assert(parsed.numFiles == 0, "Can't specify files with query");
165 	string output;
166 	invoke!({
167 		auto result = parsed.processArgs
168 			? execute(parsed.processArgs, parsed.environment, parsed.config, parsed.maxOutput, parsed.workDir)
169 			: executeShell(parsed.shellCommand, parsed.environment, parsed.config, parsed.maxOutput, parsed.workDir);
170 		output = result.output.stripRight();
171 		return result.status;
172 	})(parsed.toShellCommand());
173 	return output;
174 }
175 
176 /// std.process helper.
177 /// Run a command, feed it the given input, and collect its output.
178 /// Throw if it exited with non-zero status. Return output.
179 T[] pipe(T, Params...)(in T[] input, Params params)
180 if (!hasIndirections!T)
181 {
182 	auto parsed = ProcessParams(params);
183 	assert(parsed.numFiles == 0, "Can't specify files with pipe");
184 	T[] output;
185 	invoke!({
186 		auto pipes = parsed.processArgs
187 			? pipeProcess(parsed.processArgs, Redirect.stdin | Redirect.stdout,
188 				parsed.environment, parsed.config, parsed.workDir)
189 			: pipeShell(parsed.shellCommand, Redirect.stdin | Redirect.stdout,
190 				parsed.environment, parsed.config, parsed.workDir);
191 		auto f = pipes.stdin;
192 		auto writer = writeFileAsync(f, input);
193 		scope(exit) writer.join();
194 		output = cast(T[])readFile(pipes.stdout);
195 		return pipes.pid.wait();
196 	})(parsed.toShellCommand());
197 	return output;
198 }
199 
200 TData!T pipe(T, Params...)(in TData!T input, Params params)
201 if (!hasIndirections!T)
202 {
203 	import ae.sys.dataio : readFileData;
204 
205 	auto parsed = ProcessParams(params);
206 	assert(parsed.numFiles == 0, "Can't specify files with pipe");
207 	TData!T output;
208 	invoke!({
209 		auto pipes = parsed.processArgs
210 			? pipeProcess(parsed.processArgs, Redirect.stdin | Redirect.stdout,
211 				parsed.environment, parsed.config, parsed.workDir)
212 			: pipeShell(parsed.shellCommand, Redirect.stdin | Redirect.stdout,
213 				parsed.environment, parsed.config, parsed.workDir);
214 		auto f = pipes.stdin;
215 		auto writer = writeFileAsync(f, input.unsafeContents);
216 		scope(exit) writer.join();
217 		output = readFileData(pipes.stdout).asDataOf!T;
218 		return pipes.pid.wait();
219 	})(parsed.toShellCommand());
220 	return output;
221 }
222 
223 deprecated T[] pipe(T, Params...)(string[] args, in T[] input, Params params)
224 if (!hasIndirections!T)
225 {
226 	return pipe(input, args, params);
227 }
228 
229 debug(ae_unittest) unittest
230 {
231 	if (false) // Instantiation test
232 	{
233 		import ae.sys.data : Data;
234 		import ae.utils.array : asBytes;
235 
236 		run("cat");
237 		run(["cat"]);
238 		query(["cat"]);
239 		query("cat | cat");
240 		pipe("hello", "rev");
241 		pipe(Data("hello".asBytes), "rev");
242 	}
243 }
244 
245 // ************************************************************************
246 
247 /// Wrapper for the `iconv` program.
248 ubyte[] iconv(const(void)[] data, string inputEncoding, string outputEncoding)
249 {
250 	auto args = ["timeout", "30", "iconv", "-f", inputEncoding, "-t", outputEncoding];
251 	auto result = data.pipe(args);
252 	return cast(ubyte[])result;
253 }
254 
255 /// ditto
256 string iconv(const(void)[] data, string inputEncoding)
257 {
258 	import std.utf : validate;
259 	auto result = cast(string)iconv(data, inputEncoding, "UTF-8");
260 	validate(result);
261 	return result;
262 }
263 
264 version (HAVE_UNIX)
265 debug(ae_unittest) unittest
266 {
267 	assert(iconv("Hello"w, "UTF-16LE") == "Hello");
268 }
269 
270 /// Wrapper for the `sha1sum` program.
271 string sha1sum(const(void)[] data)
272 {
273 	auto output = cast(string)data.pipe(["sha1sum", "-b", "-"]);
274 	return output[0..40];
275 }
276 
277 version (HAVE_UNIX)
278 debug(ae_unittest) unittest
279 {
280 	assert(sha1sum("") == "da39a3ee5e6b4b0d3255bfef95601890afd80709");
281 	assert(sha1sum("a  b\nc\r\nd") == "667c71ffe2ac8a4fe500e3b96435417e4c5ec13b");
282 }
283 
284 // ************************************************************************
285 
286 import ae.utils.path;
287 deprecated alias NULL_FILE = nullFileName;
288 
289 // ************************************************************************
290 
291 /// Reverse of std.process.environment.toAA
292 void setEnvironment(string[string] env)
293 {
294 	foreach (k, v; env)
295 		if (k.length)
296 			environment[k] = v;
297 	foreach (k, v; environment.toAA())
298 		if (k.length && k !in env)
299 			environment.remove(k);
300 }
301 
302 /// Expand Windows-like variable placeholders (`"%VAR%"`) in the given string.
303 string expandWindowsEnvVars(alias getenv = environment.get)(string s)
304 {
305 	import std.array : appender;
306 	auto buf = appender!string();
307 
308 	size_t lastPercent = 0;
309 	bool inPercent = false;
310 
311 	foreach (i, c; s)
312 		if (c == '%')
313 		{
314 			if (inPercent)
315 				buf.put(lastPercent == i ? "%" : getenv(s[lastPercent .. i]));
316 			else
317 				buf.put(s[lastPercent .. i]);
318 			inPercent = !inPercent;
319 			lastPercent = i + 1;
320 		}
321 	enforce(!inPercent, "Unterminated environment variable name");
322 	buf.put(s[lastPercent .. $]);
323 	return buf.data;
324 }
325 
326 debug(ae_unittest) unittest
327 {
328 	std.process.environment[`FOOTEST`] = `bar`;
329 	assert("a%FOOTEST%b".expandWindowsEnvVars() == "abarb");
330 }
331 
332 // ************************************************************************
333 
334 /// Like `std.process.wait`, but with a timeout.
335 /// If the timeout is exceeded, the program is killed.
336 int waitTimeout(Pid pid, Duration time)
337 {
338 	bool ok = false;
339 	auto t = new Thread({
340 		Thread.sleep(time);
341 		if (!ok)
342 			try
343 				pid.kill();
344 			catch (Exception) {} // Ignore race condition
345 	}).start();
346 	scope(exit) t.join();
347 
348 	auto result = pid.wait();
349 	ok = true;
350 	return result;
351 }
352 
353 /// Wait for process to exit asynchronously.
354 /// Call callback when it exits.
355 /// WARNING: the callback will be invoked in another thread!
356 Thread waitAsync(Pid pid, void delegate(int) callback = null)
357 {
358 	return new Thread({
359 		auto result = pid.wait();
360 		if (callback)
361 			callback(result);
362 	}).start();
363 }