1 /** 2 * Simple execution of shell commands, 3 * and wrappers for common utilities. 4 * 5 * License: 6 * This Source Code Form is subject to the terms of 7 * the Mozilla Public License, v. 2.0. If a copy of 8 * the MPL was not distributed with this file, You 9 * can obtain one at http://mozilla.org/MPL/2.0/. 10 * 11 * Authors: 12 * Vladimir Panteleev <ae@cy.md> 13 */ 14 15 module ae.sys.cmd; 16 17 import core.thread; 18 19 import std.exception; 20 import std.process; 21 import std.stdio; 22 import std.string; 23 import std.traits; 24 25 import ae.sys.data : TData; 26 import ae.sys.file; 27 28 /// Returns a very unique name for a temporary file. 29 string getTempFileName(string extension) 30 { 31 import std.random : uniform; 32 import std.file : tempDir; 33 import std.path : buildPath; 34 35 static int counter; 36 return buildPath(tempDir(), format("run-%d-%d-%d-%d.%s", 37 getpid(), 38 getCurrentThreadID(), 39 uniform!uint(), 40 counter++, 41 extension 42 )); 43 } 44 45 /// Like `thisProcessID`, but for threads. 46 ulong getCurrentThreadID() 47 { 48 version (Windows) 49 { 50 import core.sys.windows.windows : GetCurrentThreadId; 51 return GetCurrentThreadId(); 52 } 53 else 54 version (Posix) 55 { 56 import core.sys.posix.pthread : pthread_self; 57 return cast(ulong)pthread_self(); 58 } 59 } 60 61 // ************************************************************************ 62 63 private struct ProcessParams 64 { 65 string shellCommand; 66 string[] processArgs; 67 68 string toShellCommand() { return shellCommand ? shellCommand : escapeShellCommand(processArgs); } 69 // Note: a portable toProcessArgs() cannot exist because CMD.EXE does not use CommandLineToArgvW. 70 71 const(string[string]) environment = null; 72 std.process.Config config = std.process.Config.none; 73 size_t maxOutput = size_t.max; 74 string workDir = null; 75 File[3] files; 76 size_t numFiles; 77 78 this(Params...)(Params params) 79 { 80 files = [stdin, stdout, stderr]; 81 82 static foreach (i; 0 .. params.length) 83 {{ 84 auto arg = params[i]; 85 static if (i == 0) 86 { 87 static if (is(typeof(arg) == string)) 88 shellCommand = arg; 89 else 90 static if (is(typeof(arg) == string[])) 91 processArgs = arg; 92 else 93 static assert(false, "Unknown type for process invocation command line: " ~ typeof(arg).stringof); 94 assert(arg, "Null command"); 95 } 96 else 97 { 98 static if (is(typeof(arg) == string)) 99 workDir = arg; 100 else 101 static if (is(typeof(arg) : const(string[string]))) 102 environment = arg; 103 else 104 static if (is(typeof(arg) == size_t)) 105 maxOutput = arg; 106 else 107 static if (is(typeof(arg) == std.process.Config)) 108 config |= arg; 109 else 110 static if (is(typeof(arg) == File)) 111 files[numFiles++] = arg; 112 else 113 static assert(false, "Unknown type for process invocation parameter: " ~ typeof(arg).stringof); 114 } 115 }} 116 } 117 } 118 119 private void invoke(alias runner)(string command) 120 { 121 //debug scope(failure) std.stdio.writeln("[CWD] ", getcwd()); 122 debug(CMD) std.stdio.stderr.writeln("invoke: ", command); 123 auto status = runner(); 124 enforce(status == 0, 125 "Command `%s` failed with status %d".format(command, status)); 126 } 127 128 /// std.process helper. 129 /// Run a command, and throw if it exited with a non-zero status. 130 void run(Params...)(Params params) 131 { 132 auto parsed = ProcessParams(params); 133 invoke!({ 134 auto pid = parsed.processArgs 135 ? spawnProcess( 136 parsed.processArgs, parsed.files[0], parsed.files[1], parsed.files[2], 137 parsed.environment, parsed.config, parsed.workDir 138 ) 139 : spawnShell( 140 parsed.shellCommand, parsed.files[0], parsed.files[1], parsed.files[2], 141 parsed.environment, parsed.config, parsed.workDir 142 ); 143 return pid.wait(); 144 })(parsed.toShellCommand()); 145 } 146 147 /// std.process helper. 148 /// Run a command and collect its output. 149 /// Throw if it exited with a non-zero status. 150 string query(Params...)(Params params) 151 { 152 auto parsed = ProcessParams(params, Config.stderrPassThrough); 153 assert(parsed.numFiles == 0, "Can't specify files with query"); 154 string output; 155 invoke!({ 156 auto result = parsed.processArgs 157 ? execute(parsed.processArgs, parsed.environment, parsed.config, parsed.maxOutput, parsed.workDir) 158 : executeShell(parsed.shellCommand, parsed.environment, parsed.config, parsed.maxOutput, parsed.workDir); 159 output = result.output.stripRight(); 160 return result.status; 161 })(parsed.toShellCommand()); 162 return output; 163 } 164 165 /// std.process helper. 166 /// Run a command, feed it the given input, and collect its output. 167 /// Throw if it exited with non-zero status. Return output. 168 T[] pipe(T, Params...)(in T[] input, Params params) 169 if (!hasIndirections!T) 170 { 171 auto parsed = ProcessParams(params); 172 assert(parsed.numFiles == 0, "Can't specify files with pipe"); 173 T[] output; 174 invoke!({ 175 auto pipes = parsed.processArgs 176 ? pipeProcess(parsed.processArgs, Redirect.stdin | Redirect.stdout, 177 parsed.environment, parsed.config, parsed.workDir) 178 : pipeShell(parsed.shellCommand, Redirect.stdin | Redirect.stdout, 179 parsed.environment, parsed.config, parsed.workDir); 180 auto f = pipes.stdin; 181 auto writer = writeFileAsync(f, input); 182 scope(exit) writer.join(); 183 output = cast(T[])readFile(pipes.stdout); 184 return pipes.pid.wait(); 185 })(parsed.toShellCommand()); 186 return output; 187 } 188 189 TData!T pipe(T, Params...)(in TData!T input, Params params) 190 if (!hasIndirections!T) 191 { 192 import ae.sys.dataio : readFileData; 193 194 auto parsed = ProcessParams(params); 195 assert(parsed.numFiles == 0, "Can't specify files with pipe"); 196 TData!T output; 197 invoke!({ 198 auto pipes = parsed.processArgs 199 ? pipeProcess(parsed.processArgs, Redirect.stdin | Redirect.stdout, 200 parsed.environment, parsed.config, parsed.workDir) 201 : pipeShell(parsed.shellCommand, Redirect.stdin | Redirect.stdout, 202 parsed.environment, parsed.config, parsed.workDir); 203 auto f = pipes.stdin; 204 auto writer = writeFileAsync(f, input.unsafeContents); 205 scope(exit) writer.join(); 206 output = readFileData(pipes.stdout).asDataOf!T; 207 return pipes.pid.wait(); 208 })(parsed.toShellCommand()); 209 return output; 210 } 211 212 deprecated T[] pipe(T, Params...)(string[] args, in T[] input, Params params) 213 if (!hasIndirections!T) 214 { 215 return pipe(input, args, params); 216 } 217 218 debug(ae_unittest) unittest 219 { 220 if (false) // Instantiation test 221 { 222 import ae.sys.data : Data; 223 import ae.utils.array : asBytes; 224 225 run("cat"); 226 run(["cat"]); 227 query(["cat"]); 228 query("cat | cat"); 229 pipe("hello", "rev"); 230 pipe(Data("hello".asBytes), "rev"); 231 } 232 } 233 234 // ************************************************************************ 235 236 /// Wrapper for the `iconv` program. 237 ubyte[] iconv(const(void)[] data, string inputEncoding, string outputEncoding) 238 { 239 auto args = ["timeout", "30", "iconv", "-f", inputEncoding, "-t", outputEncoding]; 240 auto result = data.pipe(args); 241 return cast(ubyte[])result; 242 } 243 244 /// ditto 245 string iconv(const(void)[] data, string inputEncoding) 246 { 247 import std.utf : validate; 248 auto result = cast(string)iconv(data, inputEncoding, "UTF-8"); 249 validate(result); 250 return result; 251 } 252 253 version (HAVE_UNIX) 254 debug(ae_unittest) unittest 255 { 256 assert(iconv("Hello"w, "UTF-16LE") == "Hello"); 257 } 258 259 /// Wrapper for the `sha1sum` program. 260 string sha1sum(const(void)[] data) 261 { 262 auto output = cast(string)data.pipe(["sha1sum", "-b", "-"]); 263 return output[0..40]; 264 } 265 266 version (HAVE_UNIX) 267 debug(ae_unittest) unittest 268 { 269 assert(sha1sum("") == "da39a3ee5e6b4b0d3255bfef95601890afd80709"); 270 assert(sha1sum("a b\nc\r\nd") == "667c71ffe2ac8a4fe500e3b96435417e4c5ec13b"); 271 } 272 273 // ************************************************************************ 274 275 import ae.utils.path; 276 deprecated alias NULL_FILE = nullFileName; 277 278 // ************************************************************************ 279 280 /// Reverse of std.process.environment.toAA 281 void setEnvironment(string[string] env) 282 { 283 foreach (k, v; env) 284 if (k.length) 285 environment[k] = v; 286 foreach (k, v; environment.toAA()) 287 if (k.length && k !in env) 288 environment.remove(k); 289 } 290 291 /// Expand Windows-like variable placeholders (`"%VAR%"`) in the given string. 292 string expandWindowsEnvVars(alias getenv = environment.get)(string s) 293 { 294 import std.array : appender; 295 auto buf = appender!string(); 296 297 size_t lastPercent = 0; 298 bool inPercent = false; 299 300 foreach (i, c; s) 301 if (c == '%') 302 { 303 if (inPercent) 304 buf.put(lastPercent == i ? "%" : getenv(s[lastPercent .. i])); 305 else 306 buf.put(s[lastPercent .. i]); 307 inPercent = !inPercent; 308 lastPercent = i + 1; 309 } 310 enforce(!inPercent, "Unterminated environment variable name"); 311 buf.put(s[lastPercent .. $]); 312 return buf.data; 313 } 314 315 debug(ae_unittest) unittest 316 { 317 std.process.environment[`FOOTEST`] = `bar`; 318 assert("a%FOOTEST%b".expandWindowsEnvVars() == "abarb"); 319 } 320 321 // ************************************************************************ 322 323 /// Like `std.process.wait`, but with a timeout. 324 /// If the timeout is exceeded, the program is killed. 325 int waitTimeout(Pid pid, Duration time) 326 { 327 bool ok = false; 328 auto t = new Thread({ 329 Thread.sleep(time); 330 if (!ok) 331 try 332 pid.kill(); 333 catch (Exception) {} // Ignore race condition 334 }).start(); 335 scope(exit) t.join(); 336 337 auto result = pid.wait(); 338 ok = true; 339 return result; 340 } 341 342 /// Wait for process to exit asynchronously. 343 /// Call callback when it exits. 344 /// WARNING: the callback will be invoked in another thread! 345 Thread waitAsync(Pid pid, void delegate(int) callback = null) 346 { 347 return new Thread({ 348 auto result = pid.wait(); 349 if (callback) 350 callback(result); 351 }).start(); 352 }