1 /**
2  * Simple execution of shell commands,
3  * and wrappers for common utilities.
4  *
5  * License:
6  *   This Source Code Form is subject to the terms of
7  *   the Mozilla Public License, v. 2.0. If a copy of
8  *   the MPL was not distributed with this file, You
9  *   can obtain one at http://mozilla.org/MPL/2.0/.
10  *
11  * Authors:
12  *   Vladimir Panteleev <ae@cy.md>
13  */
14 
15 module ae.sys.cmd;
16 
17 import core.thread;
18 
19 import std.exception;
20 import std.process;
21 import std.stdio;
22 import std.string;
23 import std.traits;
24 
25 import ae.sys.data : TData;
26 import ae.sys.file;
27 
28 /// Returns a very unique name for a temporary file.
29 string getTempFileName(string extension)
30 {
31 	import std.random : uniform;
32 	import std.file : tempDir;
33 	import std.path : buildPath;
34 
35 	static int counter;
36 	return buildPath(tempDir(), format("run-%d-%d-%d-%d.%s",
37 		getpid(),
38 		getCurrentThreadID(),
39 		uniform!uint(),
40 		counter++,
41 		extension
42 	));
43 }
44 
45 /// Like `thisProcessID`, but for threads.
46 ulong getCurrentThreadID()
47 {
48 	version (Windows)
49 	{
50 		import core.sys.windows.windows : GetCurrentThreadId;
51 		return GetCurrentThreadId();
52 	}
53 	else
54 	version (Posix)
55 	{
56 		import core.sys.posix.pthread : pthread_self;
57 		return cast(ulong)pthread_self();
58 	}
59 }
60 
61 // ************************************************************************
62 
63 private struct ProcessParams
64 {
65 	string shellCommand;
66 	string[] processArgs;
67 
68 	string toShellCommand() { return shellCommand ? shellCommand :  escapeShellCommand(processArgs); }
69 	// Note: a portable toProcessArgs() cannot exist because CMD.EXE does not use CommandLineToArgvW.
70 
71 	const(string[string]) environment = null;
72 	std.process.Config config = std.process.Config.none;
73 	size_t maxOutput = size_t.max;
74 	string workDir = null;
75 	File[3] files;
76 	size_t numFiles;
77 
78 	this(Params...)(Params params)
79 	{
80 		files = [stdin, stdout, stderr];
81 
82 		static foreach (i; 0 .. params.length)
83 		{{
84 			auto arg = params[i];
85 			static if (i == 0)
86 			{
87 				static if (is(typeof(arg) == string))
88 					shellCommand = arg;
89 				else
90 				static if (is(typeof(arg) == string[]))
91 					processArgs = arg;
92 				else
93 					static assert(false, "Unknown type for process invocation command line: " ~ typeof(arg).stringof);
94 				assert(arg, "Null command");
95 			}
96 			else
97 			{
98 				static if (is(typeof(arg) == string))
99 					workDir = arg;
100 				else
101 				static if (is(typeof(arg) : const(string[string])))
102 					environment = arg;
103 				else
104 				static if (is(typeof(arg) == size_t))
105 					maxOutput = arg;
106 				else
107 				static if (is(typeof(arg) == std.process.Config))
108 					config |= arg;
109 				else
110 				static if (is(typeof(arg) == File))
111 					files[numFiles++] = arg;
112 				else
113 					static assert(false, "Unknown type for process invocation parameter: " ~ typeof(arg).stringof);
114 			}
115 		}}
116 	}
117 }
118 
119 private void invoke(alias runner)(string command)
120 {
121 	//debug scope(failure) std.stdio.writeln("[CWD] ", getcwd());
122 	debug(CMD) std.stdio.stderr.writeln("invoke: ", command);
123 	auto status = runner();
124 	enforce(status == 0,
125 		"Command `%s` failed with status %d".format(command, status));
126 }
127 
128 /// std.process helper.
129 /// Run a command, and throw if it exited with a non-zero status.
130 void run(Params...)(Params params)
131 {
132 	auto parsed = ProcessParams(params);
133 	invoke!({
134 		auto pid = parsed.processArgs
135 			? spawnProcess(
136 				parsed.processArgs, parsed.files[0], parsed.files[1], parsed.files[2],
137 				parsed.environment, parsed.config, parsed.workDir
138 			)
139 			: spawnShell(
140 				parsed.shellCommand, parsed.files[0], parsed.files[1], parsed.files[2],
141 				parsed.environment, parsed.config, parsed.workDir
142 			);
143 		return pid.wait();
144 	})(parsed.toShellCommand());
145 }
146 
147 /// std.process helper.
148 /// Run a command and collect its output.
149 /// Throw if it exited with a non-zero status.
150 string query(Params...)(Params params)
151 {
152 	auto parsed = ProcessParams(params, Config.stderrPassThrough);
153 	assert(parsed.numFiles == 0, "Can't specify files with query");
154 	string output;
155 	invoke!({
156 		auto result = parsed.processArgs
157 			? execute(parsed.processArgs, parsed.environment, parsed.config, parsed.maxOutput, parsed.workDir)
158 			: executeShell(parsed.shellCommand, parsed.environment, parsed.config, parsed.maxOutput, parsed.workDir);
159 		output = result.output.stripRight();
160 		return result.status;
161 	})(parsed.toShellCommand());
162 	return output;
163 }
164 
165 /// std.process helper.
166 /// Run a command, feed it the given input, and collect its output.
167 /// Throw if it exited with non-zero status. Return output.
168 T[] pipe(T, Params...)(in T[] input, Params params)
169 if (!hasIndirections!T)
170 {
171 	auto parsed = ProcessParams(params);
172 	assert(parsed.numFiles == 0, "Can't specify files with pipe");
173 	T[] output;
174 	invoke!({
175 		auto pipes = parsed.processArgs
176 			? pipeProcess(parsed.processArgs, Redirect.stdin | Redirect.stdout,
177 				parsed.environment, parsed.config, parsed.workDir)
178 			: pipeShell(parsed.shellCommand, Redirect.stdin | Redirect.stdout,
179 				parsed.environment, parsed.config, parsed.workDir);
180 		auto f = pipes.stdin;
181 		auto writer = writeFileAsync(f, input);
182 		scope(exit) writer.join();
183 		output = cast(T[])readFile(pipes.stdout);
184 		return pipes.pid.wait();
185 	})(parsed.toShellCommand());
186 	return output;
187 }
188 
189 TData!T pipe(T, Params...)(in TData!T input, Params params)
190 if (!hasIndirections!T)
191 {
192 	import ae.sys.dataio : readFileData;
193 
194 	auto parsed = ProcessParams(params);
195 	assert(parsed.numFiles == 0, "Can't specify files with pipe");
196 	TData!T output;
197 	invoke!({
198 		auto pipes = parsed.processArgs
199 			? pipeProcess(parsed.processArgs, Redirect.stdin | Redirect.stdout,
200 				parsed.environment, parsed.config, parsed.workDir)
201 			: pipeShell(parsed.shellCommand, Redirect.stdin | Redirect.stdout,
202 				parsed.environment, parsed.config, parsed.workDir);
203 		auto f = pipes.stdin;
204 		auto writer = writeFileAsync(f, input.unsafeContents);
205 		scope(exit) writer.join();
206 		output = readFileData(pipes.stdout).asDataOf!T;
207 		return pipes.pid.wait();
208 	})(parsed.toShellCommand());
209 	return output;
210 }
211 
212 deprecated T[] pipe(T, Params...)(string[] args, in T[] input, Params params)
213 if (!hasIndirections!T)
214 {
215 	return pipe(input, args, params);
216 }
217 
218 debug(ae_unittest) unittest
219 {
220 	if (false) // Instantiation test
221 	{
222 		import ae.sys.data : Data;
223 		import ae.utils.array : asBytes;
224 
225 		run("cat");
226 		run(["cat"]);
227 		query(["cat"]);
228 		query("cat | cat");
229 		pipe("hello", "rev");
230 		pipe(Data("hello".asBytes), "rev");
231 	}
232 }
233 
234 // ************************************************************************
235 
236 /// Wrapper for the `iconv` program.
237 ubyte[] iconv(const(void)[] data, string inputEncoding, string outputEncoding)
238 {
239 	auto args = ["timeout", "30", "iconv", "-f", inputEncoding, "-t", outputEncoding];
240 	auto result = data.pipe(args);
241 	return cast(ubyte[])result;
242 }
243 
244 /// ditto
245 string iconv(const(void)[] data, string inputEncoding)
246 {
247 	import std.utf : validate;
248 	auto result = cast(string)iconv(data, inputEncoding, "UTF-8");
249 	validate(result);
250 	return result;
251 }
252 
253 version (HAVE_UNIX)
254 debug(ae_unittest) unittest
255 {
256 	assert(iconv("Hello"w, "UTF-16LE") == "Hello");
257 }
258 
259 /// Wrapper for the `sha1sum` program.
260 string sha1sum(const(void)[] data)
261 {
262 	auto output = cast(string)data.pipe(["sha1sum", "-b", "-"]);
263 	return output[0..40];
264 }
265 
266 version (HAVE_UNIX)
267 debug(ae_unittest) unittest
268 {
269 	assert(sha1sum("") == "da39a3ee5e6b4b0d3255bfef95601890afd80709");
270 	assert(sha1sum("a  b\nc\r\nd") == "667c71ffe2ac8a4fe500e3b96435417e4c5ec13b");
271 }
272 
273 // ************************************************************************
274 
275 import ae.utils.path;
276 deprecated alias NULL_FILE = nullFileName;
277 
278 // ************************************************************************
279 
280 /// Reverse of std.process.environment.toAA
281 void setEnvironment(string[string] env)
282 {
283 	foreach (k, v; env)
284 		if (k.length)
285 			environment[k] = v;
286 	foreach (k, v; environment.toAA())
287 		if (k.length && k !in env)
288 			environment.remove(k);
289 }
290 
291 /// Expand Windows-like variable placeholders (`"%VAR%"`) in the given string.
292 string expandWindowsEnvVars(alias getenv = environment.get)(string s)
293 {
294 	import std.array : appender;
295 	auto buf = appender!string();
296 
297 	size_t lastPercent = 0;
298 	bool inPercent = false;
299 
300 	foreach (i, c; s)
301 		if (c == '%')
302 		{
303 			if (inPercent)
304 				buf.put(lastPercent == i ? "%" : getenv(s[lastPercent .. i]));
305 			else
306 				buf.put(s[lastPercent .. i]);
307 			inPercent = !inPercent;
308 			lastPercent = i + 1;
309 		}
310 	enforce(!inPercent, "Unterminated environment variable name");
311 	buf.put(s[lastPercent .. $]);
312 	return buf.data;
313 }
314 
315 debug(ae_unittest) unittest
316 {
317 	std.process.environment[`FOOTEST`] = `bar`;
318 	assert("a%FOOTEST%b".expandWindowsEnvVars() == "abarb");
319 }
320 
321 // ************************************************************************
322 
323 /// Like `std.process.wait`, but with a timeout.
324 /// If the timeout is exceeded, the program is killed.
325 int waitTimeout(Pid pid, Duration time)
326 {
327 	bool ok = false;
328 	auto t = new Thread({
329 		Thread.sleep(time);
330 		if (!ok)
331 			try
332 				pid.kill();
333 			catch (Exception) {} // Ignore race condition
334 	}).start();
335 	scope(exit) t.join();
336 
337 	auto result = pid.wait();
338 	ok = true;
339 	return result;
340 }
341 
342 /// Wait for process to exit asynchronously.
343 /// Call callback when it exits.
344 /// WARNING: the callback will be invoked in another thread!
345 Thread waitAsync(Pid pid, void delegate(int) callback = null)
346 {
347 	return new Thread({
348 		auto result = pid.wait();
349 		if (callback)
350 			callback(result);
351 	}).start();
352 }