1 /** 2 * File stuff 3 * 4 * License: 5 * This Source Code Form is subject to the terms of 6 * the Mozilla Public License, v. 2.0. If a copy of 7 * the MPL was not distributed with this file, You 8 * can obtain one at http://mozilla.org/MPL/2.0/. 9 * 10 * Authors: 11 * Vladimir Panteleev <vladimir@thecybershadow.net> 12 */ 13 14 module ae.sys.file; 15 16 import core.stdc.wchar_; 17 import core.thread; 18 19 import std.array; 20 import std.conv; 21 import std.file; 22 import std.path; 23 import std.stdio : File; 24 import std.string; 25 import std.typecons; 26 import std.utf; 27 28 alias wcscmp = core.stdc.wchar_.wcscmp; 29 alias wcslen = core.stdc.wchar_.wcslen; 30 31 // ************************************************************************ 32 33 version(Windows) 34 { 35 string[] fastListDir(bool recursive = false, bool symlinks=false)(string pathname, string pattern = null) 36 { 37 import core.sys.windows.windows; 38 39 static if (recursive) 40 enforce(!pattern, "TODO: recursive fastListDir with pattern"); 41 42 string[] result; 43 string c; 44 HANDLE h; 45 46 c = buildPath(pathname, pattern ? pattern : "*.*"); 47 WIN32_FIND_DATAW fileinfo; 48 49 h = FindFirstFileW(toUTF16z(c), &fileinfo); 50 if (h != INVALID_HANDLE_VALUE) 51 { 52 scope(exit) FindClose(h); 53 54 do 55 { 56 // Skip "." and ".." 57 if (wcscmp(fileinfo.cFileName.ptr, ".") == 0 || 58 wcscmp(fileinfo.cFileName.ptr, "..") == 0) 59 continue; 60 61 static if (!symlinks) 62 { 63 if (fileinfo.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) 64 continue; 65 } 66 67 size_t clength = wcslen(fileinfo.cFileName.ptr); 68 string name = std.utf.toUTF8(fileinfo.cFileName[0 .. clength]); 69 string path = buildPath(pathname, name); 70 71 static if (recursive) 72 { 73 if (fileinfo.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) 74 { 75 result ~= fastListDir!recursive(path); 76 continue; 77 } 78 } 79 80 result ~= path; 81 } while (FindNextFileW(h,&fileinfo) != FALSE); 82 } 83 return result; 84 } 85 } 86 else 87 version (Posix) 88 { 89 private import core.stdc.errno; 90 private import core.sys.posix.dirent; 91 private import core.stdc.string; 92 93 string[] fastListDir(bool recursive=false, bool symlinks=false)(string pathname, string pattern = null) 94 { 95 string[] result; 96 DIR* h; 97 dirent* fdata; 98 99 h = opendir(toStringz(pathname)); 100 if (h) 101 { 102 try 103 { 104 while((fdata = readdir(h)) != null) 105 { 106 // Skip "." and ".." 107 if (!core.stdc..string.strcmp(fdata.d_name.ptr, ".") || 108 !core.stdc..string.strcmp(fdata.d_name.ptr, "..")) 109 continue; 110 111 static if (!symlinks) 112 { 113 if (fdata.d_type & DT_LNK) 114 continue; 115 } 116 117 size_t len = core.stdc..string.strlen(fdata.d_name.ptr); 118 string name = fdata.d_name[0 .. len].idup; 119 if (pattern && !globMatch(name, pattern)) 120 continue; 121 string path = buildPath(pathname, name); 122 123 static if (recursive) 124 { 125 if (fdata.d_type & DT_DIR) 126 { 127 result ~= fastListDir!(recursive, symlinks)(path); 128 continue; 129 } 130 } 131 132 result ~= path; 133 } 134 } 135 finally 136 { 137 closedir(h); 138 } 139 } 140 else 141 { 142 throw new std.file.FileException(pathname, errno); 143 } 144 return result; 145 } 146 } 147 else 148 static assert(0, "TODO"); 149 150 // ************************************************************************ 151 152 string buildPath2(string[] segments...) { return segments.length ? buildPath(segments) : null; } 153 154 /// Shell-like expansion of ?, * and ** in path components 155 DirEntry[] fileList(string pattern) 156 { 157 auto components = cast(string[])array(pathSplitter(pattern)); 158 foreach (i, component; components[0..$-1]) 159 if (component.contains("?") || component.contains("*")) // TODO: escape? 160 { 161 DirEntry[] expansions; // TODO: filter range instead? 162 auto dir = buildPath2(components[0..i]); 163 if (component == "**") 164 expansions = array(dirEntries(dir, SpanMode.depth)); 165 else 166 expansions = array(dirEntries(dir, component, SpanMode.shallow)); 167 168 DirEntry[] result; 169 foreach (expansion; expansions) 170 if (expansion.isDir()) 171 result ~= fileList(buildPath(expansion.name ~ components[i+1..$])); 172 return result; 173 } 174 175 auto dir = buildPath2(components[0..$-1]); 176 if (!dir || exists(dir)) 177 return array(dirEntries(dir, components[$-1], SpanMode.shallow)); 178 else 179 return null; 180 } 181 182 /// ditto 183 DirEntry[] fileList(string pattern0, string[] patterns...) 184 { 185 DirEntry[] result; 186 foreach (pattern; [pattern0] ~ patterns) 187 result ~= fileList(pattern); 188 return result; 189 } 190 191 /// ditto 192 string[] fastFileList(string pattern) 193 { 194 auto components = cast(string[])array(pathSplitter(pattern)); 195 foreach (i, component; components[0..$-1]) 196 if (component.contains("?") || component.contains("*")) // TODO: escape? 197 { 198 string[] expansions; // TODO: filter range instead? 199 auto dir = buildPath2(components[0..i]); 200 if (component == "**") 201 expansions = fastListDir!true(dir); 202 else 203 expansions = fastListDir(dir, component); 204 205 string[] result; 206 foreach (expansion; expansions) 207 if (expansion.isDir()) 208 result ~= fastFileList(buildPath(expansion ~ components[i+1..$])); 209 return result; 210 } 211 212 auto dir = buildPath2(components[0..$-1]); 213 if (!dir || exists(dir)) 214 return fastListDir(dir, components[$-1]); 215 else 216 return null; 217 } 218 219 /// ditto 220 string[] fastFileList(string pattern0, string[] patterns...) 221 { 222 string[] result; 223 foreach (pattern; [pattern0] ~ patterns) 224 result ~= fastFileList(pattern); 225 return result; 226 } 227 228 // ************************************************************************ 229 230 import std.datetime; 231 import std.exception; 232 233 deprecated SysTime getMTime(string name) 234 { 235 return timeLastModified(name); 236 } 237 238 /// If target exists, update its modification time; 239 /// otherwise create it as an empty file. 240 void touch(in char[] target) 241 { 242 if (exists(target)) 243 { 244 auto now = Clock.currTime(); 245 setTimes(target, now, now); 246 } 247 else 248 std.file.write(target, ""); 249 } 250 251 /// Returns true if the target file doesn't exist, 252 /// or source is newer than the target. 253 bool newerThan(string source, string target) 254 { 255 if (!target.exists) 256 return true; 257 return source.timeLastModified() > target.timeLastModified(); 258 } 259 260 /// Returns true if the target file doesn't exist, 261 /// or any of the sources are newer than the target. 262 bool anyNewerThan(string[] sources, string target) 263 { 264 if (!target.exists) 265 return true; 266 auto targetTime = target.timeLastModified(); 267 return sources.any!(source => source.timeLastModified() > targetTime)(); 268 } 269 270 /// Try to rename; copy/delete if rename fails 271 void move(string src, string dst) 272 { 273 try 274 src.rename(dst); 275 catch (Exception e) 276 { 277 atomicCopy(src, dst); 278 src.remove(); 279 } 280 } 281 282 /// Make sure that the path exists (and create directories as necessary). 283 void ensurePathExists(string fn) 284 { 285 auto path = dirName(fn); 286 if (!exists(path)) 287 mkdirRecurse(path); 288 } 289 290 import ae.utils.text; 291 292 /// Forcibly remove a file or directory. 293 /// If atomic is true, the entire directory is deleted "atomically" 294 /// (it is first moved/renamed to another location). 295 /// On Windows, this will move the file/directory out of the way, 296 /// if it is in use and cannot be deleted (but can be renamed). 297 void forceDelete(bool atomic=true)(string fn, bool recursive = false) 298 { 299 import std.process : environment; 300 version(Windows) 301 { 302 import win32.winnt; 303 import win32.winbase; 304 import ae.sys.windows; 305 } 306 307 auto name = fn.baseName(); 308 fn = fn.absolutePath().longPath(); 309 310 version(Windows) 311 { 312 auto fnW = toUTF16z(fn); 313 auto attr = GetFileAttributesW(fnW); 314 wenforce(attr != INVALID_FILE_ATTRIBUTES, "GetFileAttributes"); 315 if (attr & FILE_ATTRIBUTE_READONLY) 316 SetFileAttributesW(fnW, attr & ~FILE_ATTRIBUTE_READONLY).wenforce("SetFileAttributes"); 317 } 318 319 static if (atomic) 320 { 321 // To avoid zombifying locked directories, try renaming it first. 322 // Attempting to delete a locked directory will make it inaccessible. 323 324 bool tryMoveTo(string target) 325 { 326 target = target.longPath(); 327 if (target.endsWith(`\`)) 328 target = target[0..$-1]; 329 if (target.length && !target.exists) 330 return false; 331 332 string newfn; 333 do 334 newfn = format("%s\\deleted-%s.%s.%s", target, name, thisProcessID, randomString()); 335 while (newfn.exists); 336 337 version(Windows) 338 { 339 auto newfnW = toUTF16z(newfn); 340 if (!MoveFileW(fnW, newfnW)) 341 return false; 342 } 343 else 344 { 345 try 346 rename(fn, newfn); 347 catch (FileException e) 348 return false; 349 } 350 351 fn = newfn; 352 version(Windows) fnW = newfnW; 353 return true; 354 } 355 356 void tryMove() 357 { 358 auto tmp = environment.get("TEMP"); 359 if (tmp) 360 if (tryMoveTo(tmp)) 361 return; 362 363 version(Windows) 364 string tempDir = fn[0..7]~"Temp"; 365 else 366 enum tempDir = "/tmp"; 367 368 if (tryMoveTo(tempDir)) 369 return; 370 371 if (tryMoveTo(fn.dirName())) 372 return; 373 374 throw new Exception("Unable to delete " ~ fn ~ " atomically (all rename attempts failed)"); 375 } 376 377 tryMove(); 378 } 379 380 version(Windows) 381 { 382 if (attr & FILE_ATTRIBUTE_DIRECTORY) 383 { 384 if (recursive && (attr & FILE_ATTRIBUTE_REPARSE_POINT) == 0) 385 { 386 foreach (de; fn.dirEntries(SpanMode.shallow)) 387 forceDelete!false(de.name, true); 388 } 389 // Will fail if !recursive and directory is not empty 390 RemoveDirectoryW(fnW).wenforce("RemoveDirectory"); 391 } 392 else 393 DeleteFileW(fnW).wenforce("DeleteFile"); 394 } 395 else 396 { 397 if (recursive) 398 fn.removeRecurse(); 399 else 400 if (fn.isDir) 401 fn.rmdir(); 402 else 403 fn.remove(); 404 } 405 } 406 407 /// If fn is a directory, delete it recursively. 408 /// Otherwise, delete the file fn. 409 void removeRecurse(string fn) 410 { 411 if (fn.isDir) 412 fn.rmdirRecurse(); 413 else 414 fn.remove(); 415 } 416 417 /// Create an empty directory, deleting 418 /// all its contents if it already exists. 419 void recreateEmptyDirectory()(string dir) 420 { 421 if (dir.exists) 422 dir.forceDelete(true); 423 mkdir(dir); 424 } 425 426 bool isHidden()(string fn) 427 { 428 if (baseName(fn).startsWith(".")) 429 return true; 430 version (Windows) 431 { 432 import win32.winnt; 433 if (getAttributes(fn) & FILE_ATTRIBUTE_HIDDEN) 434 return true; 435 } 436 return false; 437 } 438 439 /// Return a file's unique ID. 440 ulong getFileID()(string fn) 441 { 442 version (Windows) 443 { 444 import win32.winnt; 445 import win32.winbase; 446 447 import ae.sys.windows; 448 449 auto fnW = toUTF16z(fn); 450 auto h = CreateFileW(fnW, FILE_READ_ATTRIBUTES, 0, null, OPEN_EXISTING, 0, HANDLE.init); 451 wenforce(h!=INVALID_HANDLE_VALUE, fn); 452 scope(exit) CloseHandle(h); 453 BY_HANDLE_FILE_INFORMATION fi; 454 GetFileInformationByHandle(h, &fi).wenforce("GetFileInformationByHandle"); 455 456 ULARGE_INTEGER li; 457 li.LowPart = fi.nFileIndexLow; 458 li.HighPart = fi.nFileIndexHigh; 459 auto result = li.QuadPart; 460 enforce(result, "Null file ID"); 461 return result; 462 } 463 else 464 { 465 return DirEntry(fn).statBuf.st_ino; 466 } 467 } 468 469 unittest 470 { 471 touch("a"); 472 scope(exit) remove("a"); 473 hardLink("a", "b"); 474 scope(exit) remove("b"); 475 touch("c"); 476 scope(exit) remove("c"); 477 assert(getFileID("a") == getFileID("b")); 478 assert(getFileID("a") != getFileID("c")); 479 } 480 481 deprecated alias std.file.getSize getSize2; 482 483 /// Using UNC paths bypasses path length limitation when using Windows wide APIs. 484 string longPath(string s) 485 { 486 version (Windows) 487 { 488 if (!s.startsWith(`\\`)) 489 return `\\?\` ~ s.absolutePath().buildNormalizedPath().replace(`/`, `\`); 490 } 491 return s; 492 } 493 494 version (Windows) 495 { 496 void createReparsePoint(string reparseBufferName, string extraInitialization, string reparseTagName)(in char[] target, in char[] print, in char[] link) 497 { 498 import win32.winbase; 499 import win32.windef; 500 import win32.winioctl; 501 502 import ae.sys.windows; 503 504 enum SYMLINK_FLAG_RELATIVE = 1; 505 506 HANDLE hLink = CreateFileW(link.toUTF16z(), GENERIC_READ | GENERIC_WRITE, 0, null, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, null); 507 wenforce(hLink && hLink != INVALID_HANDLE_VALUE, "CreateFileW"); 508 scope(exit) CloseHandle(hLink); 509 510 enum pathOffset = 511 mixin(q{REPARSE_DATA_BUFFER..} ~ reparseBufferName) .offsetof + 512 mixin(q{REPARSE_DATA_BUFFER..} ~ reparseBufferName)._PathBuffer.offsetof; 513 514 auto targetW = target.toUTF16(); 515 auto printW = print .toUTF16(); 516 517 // Despite MSDN, two NUL-terminating characters are needed, one for each string. 518 519 auto pathBufferSize = targetW.length + 1 + printW.length + 1; // in chars 520 auto buf = new ubyte[pathOffset + pathBufferSize * WCHAR.sizeof]; 521 auto r = cast(REPARSE_DATA_BUFFER*)buf.ptr; 522 523 r.ReparseTag = mixin(reparseTagName); 524 r.ReparseDataLength = to!WORD(buf.length - mixin(q{r..} ~ reparseBufferName).offsetof); 525 526 auto pathBuffer = mixin(q{r..} ~ reparseBufferName).PathBuffer; 527 auto p = pathBuffer; 528 529 mixin(q{r..} ~ reparseBufferName).SubstituteNameOffset = to!WORD((p-pathBuffer) * WCHAR.sizeof); 530 mixin(q{r..} ~ reparseBufferName).SubstituteNameLength = to!WORD(targetW.length * WCHAR.sizeof); 531 p[0..targetW.length] = targetW; 532 p += targetW.length; 533 *p++ = 0; 534 535 mixin(q{r..} ~ reparseBufferName).PrintNameOffset = to!WORD((p-pathBuffer) * WCHAR.sizeof); 536 mixin(q{r..} ~ reparseBufferName).PrintNameLength = to!WORD(printW .length * WCHAR.sizeof); 537 p[0..printW.length] = printW; 538 p += printW.length; 539 *p++ = 0; 540 541 assert(p-pathBuffer == pathBufferSize); 542 543 mixin(extraInitialization); 544 545 DWORD dwRet; // Needed despite MSDN 546 DeviceIoControl(hLink, FSCTL_SET_REPARSE_POINT, buf.ptr, buf.length.to!DWORD(), null, 0, &dwRet, null).wenforce("DeviceIoControl"); 547 } 548 549 void acquirePrivilege(S)(S name) 550 { 551 import win32.winbase; 552 import win32.windef; 553 554 import ae.sys.windows; 555 556 HANDLE hToken = null; 557 wenforce(OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &hToken)); 558 scope(exit) CloseHandle(hToken); 559 560 TOKEN_PRIVILEGES tp; 561 wenforce(LookupPrivilegeValue(null, name.toUTF16z(), &tp.Privileges[0].Luid), "LookupPrivilegeValue"); 562 563 tp.PrivilegeCount = 1; 564 tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; 565 wenforce(AdjustTokenPrivileges(hToken, FALSE, &tp, cast(DWORD)TOKEN_PRIVILEGES.sizeof, null, null), "AdjustTokenPrivileges"); 566 } 567 568 /// Link a directory. 569 /// Uses symlinks on POSIX, and directory junctions on Windows. 570 void dirLink()(in char[] original, in char[] link) 571 { 572 mkdir(link); 573 scope(failure) rmdir(link); 574 575 auto target = `\??\` ~ original.idup.absolutePath(); 576 if (target[$-1] != '\\') 577 target ~= '\\'; 578 579 createReparsePoint!(q{MountPointReparseBuffer}, q{}, q{IO_REPARSE_TAG_MOUNT_POINT})(target, null, link); 580 } 581 582 void symlink()(in char[] original, in char[] link) 583 { 584 import win32.winnt; 585 586 acquirePrivilege(SE_CREATE_SYMBOLIC_LINK_NAME); 587 588 touch(link); 589 scope(failure) remove(link); 590 591 createReparsePoint!(q{SymbolicLinkReparseBuffer}, q{r.SymbolicLinkReparseBuffer.Flags = link.isAbsolute() ? 0 : SYMLINK_FLAG_RELATIVE;}, q{IO_REPARSE_TAG_SYMLINK})(original, original, link); 592 } 593 } 594 else 595 alias std.file.symlink dirLink; 596 597 version (unittest) version(Windows) static import ae.sys.windows; 598 599 unittest 600 { 601 mkdir("a"); scope(exit) rmdir("a"); 602 touch("a/f"); scope(exit) remove("a/f"); 603 dirLink("a", "b"); scope(exit) version(Windows) rmdir("b"); else remove("b"); 604 //symlink("a/f", "c"); scope(exit) remove("c"); 605 assert("b".isSymlink()); 606 //assert("c".isSymlink()); 607 assert("b/f".exists()); 608 } 609 610 version (Windows) 611 { 612 void hardLink()(string src, string dst) 613 { 614 import win32.w32api; 615 616 static assert(_WIN32_WINNT >= 0x501, "CreateHardLinkW not available for target Windows platform. Specify -version=WindowsXP"); 617 618 import win32.winnt; 619 import win32.winbase; 620 621 wenforce(CreateHardLinkW(toUTF16z(dst), toUTF16z(src), null), "CreateHardLink failed: " ~ src ~ " -> " ~ dst); 622 } 623 } 624 version (Posix) 625 { 626 void hardLink()(string src, string dst) 627 { 628 import core.sys.posix.unistd; 629 enforce(link(toUTFz!(const char*)(src), toUTFz!(const char*)(dst)) == 0, "link() failed: " ~ dst); 630 } 631 } 632 633 version (Windows) 634 { 635 /// Enumerate all hard links to the specified file. 636 string[] enumerateHardLinks()(string fn) 637 { 638 import win32.winnt; 639 import win32.winbase; 640 import ae.sys.windows; 641 642 alias extern(System) HANDLE function(LPCWSTR lpFileName, DWORD dwFlags, LPDWORD StringLength, PWCHAR LinkName) TFindFirstFileNameW; 643 alias extern(System) BOOL function(HANDLE hFindStream, LPDWORD StringLength, PWCHAR LinkName) TFindNextFileNameW; 644 645 auto kernel32 = GetModuleHandle("kernel32.dll"); 646 auto FindFirstFileNameW = cast(TFindFirstFileNameW)GetProcAddress(kernel32, "FindFirstFileNameW").wenforce("GetProcAddress(FindFirstFileNameW)"); 647 auto FindNextFileNameW = cast(TFindNextFileNameW)GetProcAddress(kernel32, "FindNextFileNameW").wenforce("GetProcAddress(FindNextFileNameW)"); 648 649 static WCHAR[0x8000] buf; 650 DWORD len = buf.length; 651 auto h = FindFirstFileNameW(toUTF16z(fn), 0, &len, buf.ptr); 652 wenforce(h != INVALID_HANDLE_VALUE, "FindFirstFileNameW"); 653 scope(exit) FindClose(h); 654 655 string[] result; 656 do 657 { 658 enforce(len > 0 && len < buf.length && buf[len-1] == 0, "Bad FindFirst/NextFileNameW result"); 659 result ~= buf[0..len-1].toUTF8(); 660 len = buf.length; 661 auto ok = FindNextFileNameW(h, &len, buf.ptr); 662 if (!ok && GetLastError() == ERROR_HANDLE_EOF) 663 break; 664 wenforce(ok, "FindNextFileNameW"); 665 } while(true); 666 return result; 667 } 668 } 669 670 version(Windows) 671 unittest 672 { 673 touch("a.test"); 674 scope(exit) remove("a.test"); 675 hardLink("a.test", "b.test"); 676 scope(exit) remove("b.test"); 677 678 auto paths = enumerateHardLinks("a.test"); 679 assert(paths.length == 2); 680 paths.sort(); 681 assert(paths[0].endsWith(`\a.test`), paths[0]); 682 assert(paths[1].endsWith(`\b.test`)); 683 } 684 685 void toFile(in void[] data, in char[] name) 686 { 687 std.file.write(name, data); 688 } 689 690 /// Uses UNC paths to open a file. 691 /// Requires https://github.com/D-Programming-Language/phobos/pull/1888 692 File openFile()(string fn, string mode = "rb") 693 { 694 File f; 695 static if (is(typeof(&f.windowsHandleOpen))) 696 { 697 import core.sys.windows.windows; 698 import ae.sys.windows.exception; 699 700 string winMode; 701 foreach (c; mode) 702 switch (c) 703 { 704 case 'r': 705 case 'w': 706 case 'a': 707 case '+': 708 winMode ~= c; 709 break; 710 case 'b': 711 case 't': 712 break; 713 default: 714 assert(false, "Unknown character in mode"); 715 } 716 DWORD access, creation; 717 bool append; 718 switch (winMode) 719 { 720 case "r" : access = GENERIC_READ ; creation = OPEN_EXISTING; break; 721 case "r+": access = GENERIC_READ | GENERIC_WRITE; creation = OPEN_EXISTING; break; 722 case "w" : access = GENERIC_WRITE; creation = OPEN_ALWAYS ; break; 723 case "w+": access = GENERIC_READ | GENERIC_WRITE; creation = OPEN_ALWAYS ; break; 724 case "a" : access = GENERIC_WRITE; creation = OPEN_ALWAYS ; append = true; break; 725 case "a+": assert(false, "Not implemented"); // requires two file pointers 726 default: assert(false, "Bad file mode: " ~ mode); 727 } 728 729 auto pathW = toUTF16z(longPath(fn)); 730 auto h = CreateFileW(pathW, access, FILE_SHARE_READ, null, creation, 0, HANDLE.init); 731 wenforce(h != INVALID_HANDLE_VALUE); 732 733 if (append) 734 h.SetFilePointer(0, null, FILE_END); 735 736 f.windowsHandleOpen(h, mode); 737 } 738 else 739 f.open(fn, mode); 740 return f; 741 } 742 743 ubyte[16] mdFile()(string fn) 744 { 745 import std.digest.md; 746 747 MD5 context; 748 context.start(); 749 750 auto f = openFile(fn, "rb"); 751 static ubyte[64 * 1024] buffer = void; 752 while (true) 753 { 754 auto readBuffer = f.rawRead(buffer); 755 if (!readBuffer.length) 756 break; 757 context.put(cast(ubyte[])readBuffer); 758 } 759 f.close(); 760 761 ubyte[16] digest = context.finish(); 762 return digest; 763 } 764 765 /// Read a File (which might be a stream) into an array 766 void[] readFile(File f) 767 { 768 ubyte[] result; 769 static ubyte[64 * 1024] buffer = void; 770 while (true) 771 { 772 auto readBuffer = f.rawRead(buffer); 773 if (!readBuffer.length) 774 break; 775 result ~= readBuffer; 776 } 777 return result; 778 } 779 780 /// Like std.file.readText for non-UTF8 781 ascii readAscii()(string fileName) 782 { 783 return cast(ascii)readFile(openFile(fileName, "rb")); 784 } 785 786 // http://d.puremagic.com/issues/show_bug.cgi?id=7016 787 version(Posix) static import ae.sys.signals; 788 789 /// Start a thread which writes data to f asynchronously. 790 Thread writeFileAsync(File f, in void[] data) 791 { 792 static class Writer : Thread 793 { 794 File target; 795 const void[] data; 796 797 this(ref File f, in void[] data) 798 { 799 this.target = f; 800 this.data = data; 801 super(&run); 802 } 803 804 void run() 805 { 806 version (Posix) 807 { 808 import ae.sys.signals; 809 collectSignal(SIGPIPE, &write); 810 } 811 else 812 write(); 813 } 814 815 void write() 816 { 817 target.rawWrite(data); 818 target.close(); 819 } 820 } 821 822 auto t = new Writer(f, data); 823 t.start(); 824 return t; 825 } 826 827 /// Write data to a file, and ensure it gets written to disk 828 /// before this function returns. 829 /// Consider using as atomic!syncWrite. 830 /// See also: syncUpdate 831 void syncWrite()(string target, in void[] data) 832 { 833 auto f = File(target, "wb"); 834 f.rawWrite(data); 835 version (Windows) 836 { 837 import win32.windows; 838 FlushFileBuffers(f.windowsHandle); 839 } 840 else 841 { 842 import core.sys.posix.unistd; 843 fsync(f.fileno); 844 } 845 f.close(); 846 } 847 848 /// Atomically save data to a file (if the file doesn't exist, 849 /// or its contents differs). 850 void syncUpdate()(string fn, in void[] data) 851 { 852 if (!fn.exists || fn.read() != data) 853 atomic!(syncWrite!())(fn, data); 854 } 855 856 version(Windows) import ae.sys.windows.exception; 857 858 struct NamedPipeImpl 859 { 860 immutable string fileName; 861 862 /// Create a named pipe, and reserve a filename. 863 this()(string name) 864 { 865 version(Windows) 866 { 867 import win32.winbase; 868 869 fileName = `\\.\pipe\` ~ name; 870 auto h = CreateNamedPipeW(fileName.toUTF16z, PIPE_ACCESS_OUTBOUND, PIPE_TYPE_BYTE, 10, 4096, 4096, 0, null).wenforce("CreateNamedPipeW"); 871 f.windowsHandleOpen(h, "wb"); 872 } 873 else 874 { 875 import core.sys.posix.sys.stat; 876 877 fileName = `/tmp/` ~ name ~ `.fifo`; 878 mkfifo(fileName.toStringz, S_IWUSR | S_IRUSR); 879 } 880 } 881 882 /// Wait for a peer to open the other end of the pipe. 883 File connect()() 884 { 885 version(Windows) 886 { 887 import win32.winbase; 888 889 ConnectNamedPipe(f.windowsHandle, null).wenforce("ConnectNamedPipe"); 890 return f; 891 } 892 else 893 { 894 return File(fileName, "w"); 895 } 896 } 897 898 ~this() 899 { 900 version(Windows) 901 { 902 // File.~this will take care of cleanup 903 } 904 else 905 fileName.remove(); 906 } 907 908 private: 909 File f; 910 } 911 alias NamedPipe = RefCounted!NamedPipeImpl; 912 913 // **************************************************************************** 914 915 /// Change the current directory to the given directory. Does nothing if dir is null. 916 /// Return a scope guard which, upon destruction, restores the previous directory. 917 /// Asserts that only one thread has changed the process's current directory at any time. 918 auto pushd(string dir) 919 { 920 import core.atomic; 921 922 static int threadCount = 0; 923 static shared int processCount = 0; 924 925 static struct Popd 926 { 927 string oldPath; 928 this(string cwd) { oldPath = cwd; } 929 ~this() { if (oldPath) pop(); } 930 @disable this(); 931 @disable this(this); 932 933 void pop() 934 { 935 assert(oldPath); 936 scope(exit) oldPath = null; 937 chdir(oldPath); 938 939 auto newThreadCount = --threadCount; 940 auto newProcessCount = atomicOp!"-="(processCount, 1); 941 assert(newThreadCount == newProcessCount); // Shouldn't happen 942 } 943 } 944 945 string cwd; 946 if (dir) 947 { 948 auto newThreadCount = ++threadCount; 949 auto newProcessCount = atomicOp!"+="(processCount, 1); 950 assert(newThreadCount == newProcessCount, "Another thread already has an active pushd"); 951 952 cwd = getcwd(); 953 chdir(dir); 954 } 955 return Popd(cwd); 956 } 957 958 // **************************************************************************** 959 960 import std.algorithm; 961 import std.process : thisProcessID; 962 import std.traits; 963 import std.typetuple; 964 import ae.utils.meta; 965 966 enum targetParameterNames = "target/to/name/dst"; 967 968 /// Wrap an operation which creates a file or directory, 969 /// so that it is created safely and, for files, atomically 970 /// (by performing the underlying operation to a temporary 971 /// location, then renaming the completed file/directory to 972 /// the actual target location). targetName specifies the name 973 /// of the parameter containing the target file/directory. 974 auto atomic(alias impl, string targetName = targetParameterNames)(staticMap!(Unqual, ParameterTypeTuple!impl) args) 975 { 976 enum targetIndex = findParameter([ParameterIdentifierTuple!impl], targetName, __traits(identifier, impl)); 977 return atomic!(impl, targetIndex)(args); 978 } 979 980 /// ditto 981 auto atomic(alias impl, size_t targetIndex)(staticMap!(Unqual, ParameterTypeTuple!impl) args) 982 { 983 // idup for https://d.puremagic.com/issues/show_bug.cgi?id=12503 984 auto target = args[targetIndex].idup; 985 auto temp = "%s.%s.temp".format(target, thisProcessID); 986 if (temp.exists) temp.removeRecurse(); 987 scope(success) rename(temp, target); 988 scope(failure) if (temp.exists) temp.removeRecurse(); 989 args[targetIndex] = temp; 990 return impl(args); 991 } 992 993 /// ditto 994 // Workaround for https://d.puremagic.com/issues/show_bug.cgi?id=12230 995 // Can't be an overload because of https://issues.dlang.org/show_bug.cgi?id=13374 996 //R atomicDg(string targetName = "target", R, Args...)(R delegate(Args) impl, staticMap!(Unqual, Args) args) 997 auto atomicDg(size_t targetIndexA = size_t.max, Impl, Args...)(Impl impl, Args args) 998 { 999 enum targetIndex = targetIndexA == size_t.max ? ParameterTypeTuple!impl.length-1 : targetIndexA; 1000 return atomic!(impl, targetIndex)(args); 1001 } 1002 1003 deprecated alias safeUpdate = atomic; 1004 1005 unittest 1006 { 1007 enum fn = "atomic.tmp"; 1008 scope(exit) if (fn.exists) fn.remove(); 1009 1010 atomic!touch(fn); 1011 assert(fn.exists); 1012 fn.remove(); 1013 1014 atomicDg(&touch, fn); 1015 assert(fn.exists); 1016 } 1017 1018 /// Wrap an operation so that it is skipped entirely 1019 /// if the target already exists. Implies atomic. 1020 auto cached(alias impl, string targetName = targetParameterNames)(ParameterTypeTuple!impl args) 1021 { 1022 enum targetIndex = findParameter([ParameterIdentifierTuple!impl], targetName, __traits(identifier, impl)); 1023 auto target = args[targetIndex]; 1024 if (!target.exists) 1025 atomic!(impl, targetIndex)(args); 1026 return target; 1027 } 1028 1029 /// ditto 1030 // Exists due to the same reasons as atomicDg 1031 auto cachedDg(size_t targetIndexA = size_t.max, Impl, Args...)(Impl impl, Args args) 1032 { 1033 enum targetIndex = targetIndexA == size_t.max ? ParameterTypeTuple!impl.length-1 : targetIndexA; 1034 auto target = args[targetIndex]; 1035 if (!target.exists) 1036 atomic!(impl, targetIndex)(args); 1037 return target; 1038 } 1039 1040 deprecated alias obtainUsing = cached; 1041 1042 /// Create a file, or replace an existing file's contents 1043 /// atomically. 1044 /// Note: Consider using atomic!syncWrite or 1045 /// atomic!syncUpdate instead. 1046 alias atomic!(std.file.write) atomicWrite; 1047 deprecated alias safeWrite = atomicWrite; 1048 1049 // Work around for https://github.com/D-Programming-Language/phobos/pull/2784#issuecomment-68117241 1050 private void copy2(string source, string target) { std.file.copy(source, target); } 1051 1052 /// Copy a file, or replace an existing file's contents 1053 /// with another file's, atomically. 1054 alias atomic!copy2 atomicCopy; 1055 1056 unittest 1057 { 1058 enum fn = "cached.tmp"; 1059 scope(exit) if (fn.exists) fn.remove(); 1060 1061 cached!touch(fn); 1062 assert(fn.exists); 1063 1064 std.file.write(fn, "test"); 1065 1066 cachedDg!0(&std.file.write, fn, "test2"); 1067 assert(fn.readText() == "test"); 1068 } 1069 1070 // **************************************************************************** 1071 1072 template withTarget(alias targetGen, alias fun) 1073 { 1074 auto withTarget(Args...)(auto ref Args args) 1075 { 1076 auto target = targetGen(args); 1077 fun(args, target); 1078 return target; 1079 } 1080 } 1081 1082 /// Two-argument buildPath with reversed arguments. 1083 /// Useful for UFCS chaining. 1084 string prependPath(string target, string path) 1085 { 1086 return buildPath(path, target); 1087 }