1 /** 2 * Simple execution of shell commands, 3 * and wrappers for common utilities. 4 * 5 * License: 6 * This Source Code Form is subject to the terms of 7 * the Mozilla Public License, v. 2.0. If a copy of 8 * the MPL was not distributed with this file, You 9 * can obtain one at http://mozilla.org/MPL/2.0/. 10 * 11 * Authors: 12 * Vladimir Panteleev <ae@cy.md> 13 */ 14 15 module ae.sys.cmd; 16 17 import core.thread; 18 19 import std.exception; 20 import std.process; 21 import std.stdio; 22 import std.string; 23 import std.traits; 24 25 import ae.sys.data : TData; 26 import ae.sys.file; 27 28 /// Returns a very unique name for a temporary file. 29 string getTempFileName(string extension) 30 { 31 import std.random; 32 import std.file; 33 import std.path : buildPath; 34 35 static int counter; 36 return buildPath(tempDir(), format("run-%d-%d-%d-%d.%s", 37 getpid(), 38 getCurrentThreadID(), 39 uniform!uint(), 40 counter++, 41 extension 42 )); 43 } 44 45 /// Like `thisProcessID`, but for threads. 46 ulong getCurrentThreadID() 47 { 48 version (Windows) 49 { 50 import core.sys.windows.windows; 51 return GetCurrentThreadId(); 52 } 53 else 54 version (Posix) 55 { 56 import core.sys.posix.pthread; 57 return cast(ulong)pthread_self(); 58 } 59 } 60 61 // ************************************************************************ 62 63 private struct ProcessParams 64 { 65 const(string[string]) environment = null; 66 std.process.Config config = std.process.Config.none; 67 size_t maxOutput = size_t.max; 68 string workDir = null; 69 70 this(Params...)(Params params) 71 { 72 foreach (arg; params) 73 { 74 static if (is(typeof(arg) == string)) 75 workDir = arg; 76 else 77 static if (is(typeof(arg) : const(string[string]))) 78 environment = arg; 79 else 80 static if (is(typeof(arg) == size_t)) 81 maxOutput = arg; 82 else 83 static if (is(typeof(arg) == std.process.Config)) 84 config = arg; 85 else 86 static assert(false, "Unknown type for process invocation parameter: " ~ typeof(arg).stringof); 87 } 88 } 89 } 90 91 private void invoke(alias runner)(string[] args) 92 { 93 //debug scope(failure) std.stdio.writeln("[CWD] ", getcwd()); 94 debug(CMD) std.stdio.stderr.writeln("invoke: ", args); 95 auto status = runner(); 96 enforce(status == 0, 97 "Command `%s` failed with status %d".format(escapeShellCommand(args), status)); 98 } 99 100 /// std.process helper. 101 /// Run a command, and throw if it exited with a non-zero status. 102 void run(Params...)(string[] args, Params params) 103 { 104 auto parsed = ProcessParams(params); 105 invoke!({ return spawnProcess( 106 args, stdin, stdout, stderr, 107 parsed.environment, parsed.config, parsed.workDir 108 ).wait(); })(args); 109 } 110 111 /// std.process helper. 112 /// Run a command and collect its output. 113 /// Throw if it exited with a non-zero status. 114 string query(Params...)(string[] args, Params params) 115 { 116 auto parsed = ProcessParams(params); 117 string output; 118 invoke!({ 119 // Don't use execute due to https://issues.dlang.org/show_bug.cgi?id=17844 120 version (none) 121 { 122 auto result = execute(args, parsed.environment, parsed.config, parsed.maxOutput, parsed.workDir); 123 output = result.output.stripRight(); 124 return result.status; 125 } 126 else 127 { 128 auto pipes = pipeProcess(args, Redirect.stdout, 129 parsed.environment, parsed.config, parsed.workDir); 130 output = cast(string)readFile(pipes.stdout); 131 return pipes.pid.wait(); 132 } 133 })(args); 134 return output; 135 } 136 137 /// std.process helper. 138 /// Run a command, feed it the given input, and collect its output. 139 /// Throw if it exited with non-zero status. Return output. 140 T[] pipe(T, Params...)(in T[] input, string[] args, Params params) 141 if (!hasIndirections!T) 142 { 143 auto parsed = ProcessParams(params); 144 T[] output; 145 invoke!({ 146 auto pipes = pipeProcess(args, Redirect.stdin | Redirect.stdout, 147 parsed.environment, parsed.config, parsed.workDir); 148 auto f = pipes.stdin; 149 auto writer = writeFileAsync(f, input); 150 scope(exit) writer.join(); 151 output = cast(T[])readFile(pipes.stdout); 152 return pipes.pid.wait(); 153 })(args); 154 return output; 155 } 156 157 TData!T pipe(T, Params...)(in TData!T input, string[] args, Params params) 158 if (!hasIndirections!T) 159 { 160 import ae.sys.dataio : readFileData; 161 162 auto parsed = ProcessParams(params); 163 TData!T output; 164 invoke!({ 165 auto pipes = pipeProcess(args, Redirect.stdin | Redirect.stdout, 166 parsed.environment, parsed.config, parsed.workDir); 167 auto f = pipes.stdin; 168 auto writer = writeFileAsync(f, input.unsafeContents); 169 scope(exit) writer.join(); 170 output = readFileData(pipes.stdout).asDataOf!T; 171 return pipes.pid.wait(); 172 })(args); 173 return output; 174 } 175 176 deprecated T[] pipe(T, Params...)(string[] args, in T[] input, Params params) 177 if (!hasIndirections!T) 178 { 179 return pipe(input, args, params); 180 } 181 182 // ************************************************************************ 183 184 /// Wrapper for the `iconv` program. 185 ubyte[] iconv(const(void)[] data, string inputEncoding, string outputEncoding) 186 { 187 auto args = ["timeout", "30", "iconv", "-f", inputEncoding, "-t", outputEncoding]; 188 auto result = data.pipe(args); 189 return cast(ubyte[])result; 190 } 191 192 /// ditto 193 string iconv(const(void)[] data, string inputEncoding) 194 { 195 import std.utf; 196 auto result = cast(string)iconv(data, inputEncoding, "UTF-8"); 197 validate(result); 198 return result; 199 } 200 201 version (HAVE_UNIX) 202 unittest 203 { 204 assert(iconv("Hello"w, "UTF-16LE") == "Hello"); 205 } 206 207 /// Wrapper for the `sha1sum` program. 208 string sha1sum(const(void)[] data) 209 { 210 auto output = cast(string)data.pipe(["sha1sum", "-b", "-"]); 211 return output[0..40]; 212 } 213 214 version (HAVE_UNIX) 215 unittest 216 { 217 assert(sha1sum("") == "da39a3ee5e6b4b0d3255bfef95601890afd80709"); 218 assert(sha1sum("a b\nc\r\nd") == "667c71ffe2ac8a4fe500e3b96435417e4c5ec13b"); 219 } 220 221 // ************************************************************************ 222 223 import ae.utils.path; 224 deprecated alias NULL_FILE = nullFileName; 225 226 // ************************************************************************ 227 228 /// Reverse of std.process.environment.toAA 229 void setEnvironment(string[string] env) 230 { 231 foreach (k, v; env) 232 if (k.length) 233 environment[k] = v; 234 foreach (k, v; environment.toAA()) 235 if (k.length && k !in env) 236 environment.remove(k); 237 } 238 239 /// Expand Windows-like variable placeholders (`"%VAR%"`) in the given string. 240 string expandWindowsEnvVars(alias getenv = environment.get)(string s) 241 { 242 import std.array; 243 auto buf = appender!string(); 244 245 size_t lastPercent = 0; 246 bool inPercent = false; 247 248 foreach (i, c; s) 249 if (c == '%') 250 { 251 if (inPercent) 252 buf.put(lastPercent == i ? "%" : getenv(s[lastPercent .. i])); 253 else 254 buf.put(s[lastPercent .. i]); 255 inPercent = !inPercent; 256 lastPercent = i + 1; 257 } 258 enforce(!inPercent, "Unterminated environment variable name"); 259 buf.put(s[lastPercent .. $]); 260 return buf.data; 261 } 262 263 unittest 264 { 265 std.process.environment[`FOOTEST`] = `bar`; 266 assert("a%FOOTEST%b".expandWindowsEnvVars() == "abarb"); 267 } 268 269 // ************************************************************************ 270 271 /// Like `std.process.wait`, but with a timeout. 272 /// If the timeout is exceeded, the program is killed. 273 int waitTimeout(Pid pid, Duration time) 274 { 275 bool ok = false; 276 auto t = new Thread({ 277 Thread.sleep(time); 278 if (!ok) 279 try 280 pid.kill(); 281 catch (Exception) {} // Ignore race condition 282 }).start(); 283 auto result = pid.wait(); 284 ok = true; 285 return result; 286 } 287 288 /// Wait for process to exit asynchronously. 289 /// Call callback when it exits. 290 /// WARNING: the callback will be invoked in another thread! 291 void waitAsync(Pid pid, void delegate(int) callback = null) 292 { 293 auto t = new Thread({ 294 auto result = pid.wait(); 295 if (callback) 296 callback(result); 297 }).start(); 298 }