1 /**
2  * Simple execution of shell commands,
3  * and wrappers for common utilities.
4  *
5  * License:
6  *   This Source Code Form is subject to the terms of
7  *   the Mozilla Public License, v. 2.0. If a copy of
8  *   the MPL was not distributed with this file, You
9  *   can obtain one at http://mozilla.org/MPL/2.0/.
10  *
11  * Authors:
12  *   Vladimir Panteleev <ae@cy.md>
13  */
14 
15 module ae.sys.cmd;
16 
17 import core.thread;
18 
19 import std.exception;
20 import std.process;
21 import std.stdio;
22 import std.string;
23 import std.traits;
24 
25 import ae.sys.data : TData;
26 import ae.sys.file;
27 
28 /// Returns a very unique name for a temporary file.
29 string getTempFileName(string extension)
30 {
31 	import std.random;
32 	import std.file;
33 	import std.path : buildPath;
34 
35 	static int counter;
36 	return buildPath(tempDir(), format("run-%d-%d-%d-%d.%s",
37 		getpid(),
38 		getCurrentThreadID(),
39 		uniform!uint(),
40 		counter++,
41 		extension
42 	));
43 }
44 
45 /// Like `thisProcessID`, but for threads.
46 ulong getCurrentThreadID()
47 {
48 	version (Windows)
49 	{
50 		import core.sys.windows.windows;
51 		return GetCurrentThreadId();
52 	}
53 	else
54 	version (Posix)
55 	{
56 		import core.sys.posix.pthread;
57 		return cast(ulong)pthread_self();
58 	}
59 }
60 
61 // ************************************************************************
62 
63 private struct ProcessParams
64 {
65 	const(string[string]) environment = null;
66 	std.process.Config config = std.process.Config.none;
67 	size_t maxOutput = size_t.max;
68 	string workDir = null;
69 
70 	this(Params...)(Params params)
71 	{
72 		foreach (arg; params)
73 		{
74 			static if (is(typeof(arg) == string))
75 				workDir = arg;
76 			else
77 			static if (is(typeof(arg) : const(string[string])))
78 				environment = arg;
79 			else
80 			static if (is(typeof(arg) == size_t))
81 				maxOutput = arg;
82 			else
83 			static if (is(typeof(arg) == std.process.Config))
84 				config = arg;
85 			else
86 				static assert(false, "Unknown type for process invocation parameter: " ~ typeof(arg).stringof);
87 		}
88 	}
89 }
90 
91 private void invoke(alias runner)(string[] args)
92 {
93 	//debug scope(failure) std.stdio.writeln("[CWD] ", getcwd());
94 	debug(CMD) std.stdio.stderr.writeln("invoke: ", args);
95 	auto status = runner();
96 	enforce(status == 0,
97 		"Command `%s` failed with status %d".format(escapeShellCommand(args), status));
98 }
99 
100 /// std.process helper.
101 /// Run a command, and throw if it exited with a non-zero status.
102 void run(Params...)(string[] args, Params params)
103 {
104 	auto parsed = ProcessParams(params);
105 	invoke!({ return spawnProcess(
106 				args, stdin, stdout, stderr,
107 				parsed.environment, parsed.config, parsed.workDir
108 			).wait(); })(args);
109 }
110 
111 /// std.process helper.
112 /// Run a command and collect its output.
113 /// Throw if it exited with a non-zero status.
114 string query(Params...)(string[] args, Params params)
115 {
116 	auto parsed = ProcessParams(params);
117 	string output;
118 	invoke!({
119 		// Don't use execute due to https://issues.dlang.org/show_bug.cgi?id=17844
120 		version (none)
121 		{
122 			auto result = execute(args, parsed.environment, parsed.config, parsed.maxOutput, parsed.workDir);
123 			output = result.output.stripRight();
124 			return result.status;
125 		}
126 		else
127 		{
128 			auto pipes = pipeProcess(args, Redirect.stdout,
129 				parsed.environment, parsed.config, parsed.workDir);
130 			output = cast(string)readFile(pipes.stdout);
131 			return pipes.pid.wait();
132 		}
133 	})(args);
134 	return output;
135 }
136 
137 /// std.process helper.
138 /// Run a command, feed it the given input, and collect its output.
139 /// Throw if it exited with non-zero status. Return output.
140 T[] pipe(T, Params...)(in T[] input, string[] args, Params params)
141 if (!hasIndirections!T)
142 {
143 	auto parsed = ProcessParams(params);
144 	T[] output;
145 	invoke!({
146 		auto pipes = pipeProcess(args, Redirect.stdin | Redirect.stdout,
147 			parsed.environment, parsed.config, parsed.workDir);
148 		auto f = pipes.stdin;
149 		auto writer = writeFileAsync(f, input);
150 		scope(exit) writer.join();
151 		output = cast(T[])readFile(pipes.stdout);
152 		return pipes.pid.wait();
153 	})(args);
154 	return output;
155 }
156 
157 TData!T pipe(T, Params...)(in TData!T input, string[] args, Params params)
158 if (!hasIndirections!T)
159 {
160 	import ae.sys.dataio : readFileData;
161 
162 	auto parsed = ProcessParams(params);
163 	TData!T output;
164 	invoke!({
165 		auto pipes = pipeProcess(args, Redirect.stdin | Redirect.stdout,
166 			parsed.environment, parsed.config, parsed.workDir);
167 		auto f = pipes.stdin;
168 		auto writer = writeFileAsync(f, input.unsafeContents);
169 		scope(exit) writer.join();
170 		output = readFileData(pipes.stdout).asDataOf!T;
171 		return pipes.pid.wait();
172 	})(args);
173 	return output;
174 }
175 
176 deprecated T[] pipe(T, Params...)(string[] args, in T[] input, Params params)
177 if (!hasIndirections!T)
178 {
179 	return pipe(input, args, params);
180 }
181 
182 // ************************************************************************
183 
184 /// Wrapper for the `iconv` program.
185 ubyte[] iconv(const(void)[] data, string inputEncoding, string outputEncoding)
186 {
187 	auto args = ["timeout", "30", "iconv", "-f", inputEncoding, "-t", outputEncoding];
188 	auto result = data.pipe(args);
189 	return cast(ubyte[])result;
190 }
191 
192 /// ditto
193 string iconv(const(void)[] data, string inputEncoding)
194 {
195 	import std.utf;
196 	auto result = cast(string)iconv(data, inputEncoding, "UTF-8");
197 	validate(result);
198 	return result;
199 }
200 
201 version (HAVE_UNIX)
202 unittest
203 {
204 	assert(iconv("Hello"w, "UTF-16LE") == "Hello");
205 }
206 
207 /// Wrapper for the `sha1sum` program.
208 string sha1sum(const(void)[] data)
209 {
210 	auto output = cast(string)data.pipe(["sha1sum", "-b", "-"]);
211 	return output[0..40];
212 }
213 
214 version (HAVE_UNIX)
215 unittest
216 {
217 	assert(sha1sum("") == "da39a3ee5e6b4b0d3255bfef95601890afd80709");
218 	assert(sha1sum("a  b\nc\r\nd") == "667c71ffe2ac8a4fe500e3b96435417e4c5ec13b");
219 }
220 
221 // ************************************************************************
222 
223 import ae.utils.path;
224 deprecated alias NULL_FILE = nullFileName;
225 
226 // ************************************************************************
227 
228 /// Reverse of std.process.environment.toAA
229 void setEnvironment(string[string] env)
230 {
231 	foreach (k, v; env)
232 		if (k.length)
233 			environment[k] = v;
234 	foreach (k, v; environment.toAA())
235 		if (k.length && k !in env)
236 			environment.remove(k);
237 }
238 
239 /// Expand Windows-like variable placeholders (`"%VAR%"`) in the given string.
240 string expandWindowsEnvVars(alias getenv = environment.get)(string s)
241 {
242 	import std.array;
243 	auto buf = appender!string();
244 
245 	size_t lastPercent = 0;
246 	bool inPercent = false;
247 
248 	foreach (i, c; s)
249 		if (c == '%')
250 		{
251 			if (inPercent)
252 				buf.put(lastPercent == i ? "%" : getenv(s[lastPercent .. i]));
253 			else
254 				buf.put(s[lastPercent .. i]);
255 			inPercent = !inPercent;
256 			lastPercent = i + 1;
257 		}
258 	enforce(!inPercent, "Unterminated environment variable name");
259 	buf.put(s[lastPercent .. $]);
260 	return buf.data;
261 }
262 
263 unittest
264 {
265 	std.process.environment[`FOOTEST`] = `bar`;
266 	assert("a%FOOTEST%b".expandWindowsEnvVars() == "abarb");
267 }
268 
269 // ************************************************************************
270 
271 /// Like `std.process.wait`, but with a timeout.
272 /// If the timeout is exceeded, the program is killed.
273 int waitTimeout(Pid pid, Duration time)
274 {
275 	bool ok = false;
276 	auto t = new Thread({
277 		Thread.sleep(time);
278 		if (!ok)
279 			try
280 				pid.kill();
281 			catch (Exception) {} // Ignore race condition
282 	}).start();
283 	auto result = pid.wait();
284 	ok = true;
285 	return result;
286 }
287 
288 /// Wait for process to exit asynchronously.
289 /// Call callback when it exits.
290 /// WARNING: the callback will be invoked in another thread!
291 void waitAsync(Pid pid, void delegate(int) callback = null)
292 {
293 	auto t = new Thread({
294 		auto result = pid.wait();
295 		if (callback)
296 			callback(result);
297 	}).start();
298 }