1 /** 2 * Simple execution of shell commands, 3 * and wrappers for common utilities. 4 * 5 * License: 6 * This Source Code Form is subject to the terms of 7 * the Mozilla Public License, v. 2.0. If a copy of 8 * the MPL was not distributed with this file, You 9 * can obtain one at http://mozilla.org/MPL/2.0/. 10 * 11 * Authors: 12 * Vladimir Panteleev <ae@cy.md> 13 */ 14 15 module ae.sys.cmd; 16 17 import core.thread; 18 19 import std.exception; 20 import std.process; 21 import std.stdio; 22 import std.string; 23 import std.traits; 24 25 import ae.sys.data : TData; 26 import ae.sys.file; 27 28 /// Returns a very unique name for a temporary file. 29 string getTempFileName(string extension) 30 { 31 import std.random : uniform; 32 import std.file : tempDir; 33 import std.path : buildPath; 34 35 static int counter; 36 return buildPath(tempDir(), format("run-%d-%d-%d-%d.%s", 37 getpid(), 38 getCurrentThreadID(), 39 uniform!uint(), 40 counter++, 41 extension 42 )); 43 } 44 45 version (linux) 46 { 47 import core.sys.posix.sys.types : pid_t; 48 private extern(C) pid_t gettid(); 49 } 50 51 /// Like `thisProcessID`, but for threads. 52 ulong getCurrentThreadID() 53 { 54 version (Windows) 55 { 56 import core.sys.windows.windows : GetCurrentThreadId; 57 return GetCurrentThreadId(); 58 } 59 else 60 version (linux) 61 { 62 return gettid(); 63 } 64 else 65 version (Posix) 66 { 67 import core.sys.posix.pthread : pthread_self; 68 return cast(ulong)pthread_self(); 69 } 70 } 71 72 // ************************************************************************ 73 74 private struct ProcessParams 75 { 76 string shellCommand; 77 string[] processArgs; 78 79 string toShellCommand() { return shellCommand ? shellCommand : escapeShellCommand(processArgs); } 80 // Note: a portable toProcessArgs() cannot exist because CMD.EXE does not use CommandLineToArgvW. 81 82 const(string[string]) environment = null; 83 std.process.Config config = std.process.Config.none; 84 size_t maxOutput = size_t.max; 85 string workDir = null; 86 File[3] files; 87 size_t numFiles; 88 89 this(Params...)(Params params) 90 { 91 files = [stdin, stdout, stderr]; 92 93 static foreach (i; 0 .. params.length) 94 {{ 95 auto arg = params[i]; 96 static if (i == 0) 97 { 98 static if (is(typeof(arg) == string)) 99 shellCommand = arg; 100 else 101 static if (is(typeof(arg) == string[])) 102 processArgs = arg; 103 else 104 static assert(false, "Unknown type for process invocation command line: " ~ typeof(arg).stringof); 105 assert(arg, "Null command"); 106 } 107 else 108 { 109 static if (is(typeof(arg) == string)) 110 workDir = arg; 111 else 112 static if (is(typeof(arg) : const(string[string]))) 113 environment = arg; 114 else 115 static if (is(typeof(arg) == size_t)) 116 maxOutput = arg; 117 else 118 static if (is(typeof(arg) == std.process.Config)) 119 config |= arg; 120 else 121 static if (is(typeof(arg) == File)) 122 files[numFiles++] = arg; 123 else 124 static assert(false, "Unknown type for process invocation parameter: " ~ typeof(arg).stringof); 125 } 126 }} 127 } 128 } 129 130 private void invoke(alias runner)(string command) 131 { 132 //debug scope(failure) std.stdio.writeln("[CWD] ", getcwd()); 133 debug(CMD) std.stdio.stderr.writeln("invoke: ", command); 134 auto status = runner(); 135 enforce(status == 0, 136 "Command `%s` failed with status %d".format(command, status)); 137 } 138 139 /// std.process helper. 140 /// Run a command, and throw if it exited with a non-zero status. 141 void run(Params...)(Params params) 142 { 143 auto parsed = ProcessParams(params); 144 invoke!({ 145 auto pid = parsed.processArgs 146 ? spawnProcess( 147 parsed.processArgs, parsed.files[0], parsed.files[1], parsed.files[2], 148 parsed.environment, parsed.config, parsed.workDir 149 ) 150 : spawnShell( 151 parsed.shellCommand, parsed.files[0], parsed.files[1], parsed.files[2], 152 parsed.environment, parsed.config, parsed.workDir 153 ); 154 return pid.wait(); 155 })(parsed.toShellCommand()); 156 } 157 158 /// std.process helper. 159 /// Run a command and collect its output. 160 /// Throw if it exited with a non-zero status. 161 string query(Params...)(Params params) 162 { 163 auto parsed = ProcessParams(params, Config.stderrPassThrough); 164 assert(parsed.numFiles == 0, "Can't specify files with query"); 165 string output; 166 invoke!({ 167 auto result = parsed.processArgs 168 ? execute(parsed.processArgs, parsed.environment, parsed.config, parsed.maxOutput, parsed.workDir) 169 : executeShell(parsed.shellCommand, parsed.environment, parsed.config, parsed.maxOutput, parsed.workDir); 170 output = result.output.stripRight(); 171 return result.status; 172 })(parsed.toShellCommand()); 173 return output; 174 } 175 176 /// std.process helper. 177 /// Run a command, feed it the given input, and collect its output. 178 /// Throw if it exited with non-zero status. Return output. 179 T[] pipe(T, Params...)(in T[] input, Params params) 180 if (!hasIndirections!T) 181 { 182 auto parsed = ProcessParams(params); 183 assert(parsed.numFiles == 0, "Can't specify files with pipe"); 184 T[] output; 185 invoke!({ 186 auto pipes = parsed.processArgs 187 ? pipeProcess(parsed.processArgs, Redirect.stdin | Redirect.stdout, 188 parsed.environment, parsed.config, parsed.workDir) 189 : pipeShell(parsed.shellCommand, Redirect.stdin | Redirect.stdout, 190 parsed.environment, parsed.config, parsed.workDir); 191 auto f = pipes.stdin; 192 auto writer = writeFileAsync(f, input); 193 scope(exit) writer.join(); 194 output = cast(T[])readFile(pipes.stdout); 195 return pipes.pid.wait(); 196 })(parsed.toShellCommand()); 197 return output; 198 } 199 200 TData!T pipe(T, Params...)(in TData!T input, Params params) 201 if (!hasIndirections!T) 202 { 203 import ae.sys.dataio : readFileData; 204 205 auto parsed = ProcessParams(params); 206 assert(parsed.numFiles == 0, "Can't specify files with pipe"); 207 TData!T output; 208 invoke!({ 209 auto pipes = parsed.processArgs 210 ? pipeProcess(parsed.processArgs, Redirect.stdin | Redirect.stdout, 211 parsed.environment, parsed.config, parsed.workDir) 212 : pipeShell(parsed.shellCommand, Redirect.stdin | Redirect.stdout, 213 parsed.environment, parsed.config, parsed.workDir); 214 auto f = pipes.stdin; 215 auto writer = writeFileAsync(f, input.unsafeContents); 216 scope(exit) writer.join(); 217 output = readFileData(pipes.stdout).asDataOf!T; 218 return pipes.pid.wait(); 219 })(parsed.toShellCommand()); 220 return output; 221 } 222 223 deprecated T[] pipe(T, Params...)(string[] args, in T[] input, Params params) 224 if (!hasIndirections!T) 225 { 226 return pipe(input, args, params); 227 } 228 229 debug(ae_unittest) unittest 230 { 231 if (false) // Instantiation test 232 { 233 import ae.sys.data : Data; 234 import ae.utils.array : asBytes; 235 236 run("cat"); 237 run(["cat"]); 238 query(["cat"]); 239 query("cat | cat"); 240 pipe("hello", "rev"); 241 pipe(Data("hello".asBytes), "rev"); 242 } 243 } 244 245 // ************************************************************************ 246 247 /// Wrapper for the `iconv` program. 248 ubyte[] iconv(const(void)[] data, string inputEncoding, string outputEncoding) 249 { 250 auto args = ["timeout", "30", "iconv", "-f", inputEncoding, "-t", outputEncoding]; 251 auto result = data.pipe(args); 252 return cast(ubyte[])result; 253 } 254 255 /// ditto 256 string iconv(const(void)[] data, string inputEncoding) 257 { 258 import std.utf : validate; 259 auto result = cast(string)iconv(data, inputEncoding, "UTF-8"); 260 validate(result); 261 return result; 262 } 263 264 version (HAVE_UNIX) 265 debug(ae_unittest) unittest 266 { 267 assert(iconv("Hello"w, "UTF-16LE") == "Hello"); 268 } 269 270 /// Wrapper for the `sha1sum` program. 271 string sha1sum(const(void)[] data) 272 { 273 auto output = cast(string)data.pipe(["sha1sum", "-b", "-"]); 274 return output[0..40]; 275 } 276 277 version (HAVE_UNIX) 278 debug(ae_unittest) unittest 279 { 280 assert(sha1sum("") == "da39a3ee5e6b4b0d3255bfef95601890afd80709"); 281 assert(sha1sum("a b\nc\r\nd") == "667c71ffe2ac8a4fe500e3b96435417e4c5ec13b"); 282 } 283 284 // ************************************************************************ 285 286 import ae.utils.path; 287 deprecated alias NULL_FILE = nullFileName; 288 289 // ************************************************************************ 290 291 /// Reverse of std.process.environment.toAA 292 void setEnvironment(string[string] env) 293 { 294 foreach (k, v; env) 295 if (k.length) 296 environment[k] = v; 297 foreach (k, v; environment.toAA()) 298 if (k.length && k !in env) 299 environment.remove(k); 300 } 301 302 /// Expand Windows-like variable placeholders (`"%VAR%"`) in the given string. 303 string expandWindowsEnvVars(alias getenv = environment.get)(string s) 304 { 305 import std.array : appender; 306 auto buf = appender!string(); 307 308 size_t lastPercent = 0; 309 bool inPercent = false; 310 311 foreach (i, c; s) 312 if (c == '%') 313 { 314 if (inPercent) 315 buf.put(lastPercent == i ? "%" : getenv(s[lastPercent .. i])); 316 else 317 buf.put(s[lastPercent .. i]); 318 inPercent = !inPercent; 319 lastPercent = i + 1; 320 } 321 enforce(!inPercent, "Unterminated environment variable name"); 322 buf.put(s[lastPercent .. $]); 323 return buf.data; 324 } 325 326 debug(ae_unittest) unittest 327 { 328 std.process.environment[`FOOTEST`] = `bar`; 329 assert("a%FOOTEST%b".expandWindowsEnvVars() == "abarb"); 330 } 331 332 // ************************************************************************ 333 334 /// Like `std.process.wait`, but with a timeout. 335 /// If the timeout is exceeded, the program is killed. 336 int waitTimeout(Pid pid, Duration time) 337 { 338 bool ok = false; 339 auto t = new Thread({ 340 Thread.sleep(time); 341 if (!ok) 342 try 343 pid.kill(); 344 catch (Exception) {} // Ignore race condition 345 }).start(); 346 scope(exit) t.join(); 347 348 auto result = pid.wait(); 349 ok = true; 350 return result; 351 } 352 353 /// Wait for process to exit asynchronously. 354 /// Call callback when it exits. 355 /// WARNING: the callback will be invoked in another thread! 356 Thread waitAsync(Pid pid, void delegate(int) callback = null) 357 { 358 return new Thread({ 359 auto result = pid.wait(); 360 if (callback) 361 callback(result); 362 }).start(); 363 }