1 /** 2 * File stuff 3 * 4 * License: 5 * This Source Code Form is subject to the terms of 6 * the Mozilla Public License, v. 2.0. If a copy of 7 * the MPL was not distributed with this file, You 8 * can obtain one at http://mozilla.org/MPL/2.0/. 9 * 10 * Authors: 11 * Vladimir Panteleev <vladimir@thecybershadow.net> 12 */ 13 14 module ae.sys.file; 15 16 import core.stdc.wchar_; 17 import core.thread; 18 19 import std.array; 20 import std.conv; 21 import std.file; 22 import std.path; 23 import std.range.primitives; 24 import std.stdio : File; 25 import std.string; 26 import std.typecons; 27 import std.utf; 28 29 import ae.sys.cmd : getCurrentThreadID; 30 import ae.utils.path; 31 32 public import std.typecons : No, Yes; 33 34 alias wcscmp = core.stdc.wchar_.wcscmp; 35 alias wcslen = core.stdc.wchar_.wcslen; 36 37 version(Windows) import ae.sys.windows.imports; 38 39 // ************************************************************************ 40 41 version (Windows) 42 { 43 // Work around std.file overload 44 mixin(importWin32!(q{winnt}, null, q{FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT})); 45 } 46 version (Posix) 47 { 48 private import core.stdc.errno; 49 private import core.sys.posix.dirent; 50 private import core.stdc.string; 51 } 52 53 // ************************************************************************ 54 55 deprecated string[] fastListDir(bool recursive = false, bool symlinks=false)(string pathname, string pattern = null) 56 { 57 string[] result; 58 59 listDir!((e) { 60 static if (!symlinks) 61 { 62 // Note: shouldn't this just skip recursion? 63 if (e.isSymlink) 64 return; 65 } 66 67 if (pattern && !globMatch(e.baseName, pattern)) 68 return; 69 70 static if (recursive) 71 { 72 if (e.entryIsDir) 73 { 74 // Note: why exclude directories from results? 75 e.recurse(); 76 return; 77 } 78 } 79 80 result ~= e.fullName; 81 })(pathname); 82 return result; 83 } 84 85 // ************************************************************************ 86 87 version (Windows) 88 { 89 mixin(importWin32!(q{winnt}, null, q{WCHAR})); 90 mixin(importWin32!(q{winbase}, null, q{WIN32_FIND_DATAW})); 91 } 92 93 /// The OS's "native" filesystem character type (private in Phobos). 94 version (Windows) 95 alias FSChar = WCHAR; 96 else version (Posix) 97 alias FSChar = char; 98 else 99 static assert(0); 100 101 /// Reads a time field from a stat_t with full precision (private in Phobos). 102 SysTime statTimeToStdTime(string which)(ref const stat_t statbuf) 103 { 104 auto unixTime = mixin(`statbuf.st_` ~ which ~ `time`); 105 auto stdTime = unixTimeToStdTime(unixTime); 106 107 static if (is(typeof(mixin(`statbuf.st_` ~ which ~ `tim`)))) 108 stdTime += mixin(`statbuf.st_` ~ which ~ `tim.tv_nsec`) / 100; 109 else 110 static if (is(typeof(mixin(`statbuf.st_` ~ which ~ `timensec`)))) 111 stdTime += mixin(`statbuf.st_` ~ which ~ `timensec`) / 100; 112 else 113 static if (is(typeof(mixin(`statbuf.st_` ~ which ~ `time_nsec`)))) 114 stdTime += mixin(`statbuf.st_` ~ which ~ `time_nsec`) / 100; 115 else 116 static if (is(typeof(mixin(`statbuf.__st_` ~ which ~ `timensec`)))) 117 stdTime += mixin(`statbuf.__st_` ~ which ~ `timensec`) / 100; 118 119 return SysTime(stdTime); 120 } 121 122 version (OSX) 123 version = Darwin; 124 else version (iOS) 125 version = Darwin; 126 else version (TVOS) 127 version = Darwin; 128 else version (WatchOS) 129 version = Darwin; 130 131 private 132 version (Posix) 133 { 134 // TODO: upstream into Druntime 135 extern (C) 136 { 137 int dirfd(DIR *dirp) pure nothrow @nogc; 138 int openat(int fd, const char *path, int oflag, ...) nothrow @nogc; 139 140 version (Darwin) 141 { 142 pragma(mangle, "fstatat$INODE64") 143 int fstatat(int fd, const char *path, stat_t *buf, int flag) nothrow @nogc; 144 145 pragma(mangle, "fdopendir$INODE64") 146 DIR *fdopendir(int fd) nothrow @nogc; 147 } 148 else 149 { 150 int fstatat(int fd, const(char)* path, stat_t* buf, int flag) nothrow @nogc; 151 DIR *fdopendir(int fd) nothrow @nogc; 152 } 153 } 154 version (linux) 155 { 156 enum AT_SYMLINK_NOFOLLOW = 0x100; 157 enum O_DIRECTORY = 0x10000; 158 } 159 version (Darwin) 160 { 161 enum AT_SYMLINK_NOFOLLOW = 0x20; 162 enum O_DIRECTORY = 0x100000; 163 } 164 version (FreeBSD) 165 { 166 enum AT_SYMLINK_NOFOLLOW = 0x200; 167 enum O_DIRECTORY = 0x20000; 168 } 169 } 170 171 import ae.utils.range : nullTerminated; 172 173 // http://d.puremagic.com/issues/show_bug.cgi?id=7016 174 version (Windows) static import ae.sys.windows.misc; 175 176 /// Fast templated directory iterator 177 template listDir(alias handler) 178 { 179 /*non-static*/ struct Context 180 { 181 // Tether to handler alias context 182 void callHandler(Entry* e) { handler(e); } 183 184 bool timeToStop = false; 185 186 FSChar[] pathBuf; 187 } 188 189 static struct Entry 190 { 191 version (Posix) 192 { 193 dirent* ent; 194 195 stat_t[enumLength!StatTarget] statBuf; 196 enum StatResult : int 197 { 198 noInfo = 0, 199 statOK = int.max, 200 unknownError = int.min, 201 // other values are the same as errno 202 } 203 } 204 version (Windows) 205 { 206 WIN32_FIND_DATAW findData; 207 } 208 209 // Cleared (memset to 0) for every directory entry. 210 struct Data 211 { 212 FSChar[] baseNameFS; 213 string baseName; 214 string fullName; 215 size_t pathTailPos; 216 217 version (Posix) 218 { 219 StatResult[enumLength!StatTarget] statResult; 220 } 221 } 222 Data data; 223 224 // Recursion 225 226 Entry* parent; 227 Context* context; 228 229 version (Posix) 230 { 231 int dirFD; 232 233 void recurse() 234 { 235 import core.sys.posix.fcntl; 236 int flags = O_RDONLY; 237 static if (is(typeof(O_DIRECTORY))) 238 flags |= O_DIRECTORY; 239 auto fd = openat(dirFD, this.ent.d_name.ptr, flags); 240 errnoEnforce(fd >= 0, 241 "Failed to open %s as subdirectory of directory %s" 242 .format(this.baseNameFS, this.parent.fullName)); 243 auto subdir = fdopendir(fd); 244 errnoEnforce(subdir, 245 "Failed to open subdirectory %s of directory %s as directory" 246 .format(this.baseNameFS, this.parent.fullName)); 247 scan(subdir, fd, &this); 248 } 249 } 250 version (Windows) 251 { 252 void recurse() 253 { 254 needFullPath(); 255 appendString(context.pathBuf, 256 data.pathTailPos, "\\*.*\0"w); 257 scan(&this); 258 } 259 } 260 261 void stop() { context.timeToStop = true; } 262 263 // Name 264 265 const(FSChar)* baseNameFSPtr() pure nothrow @nogc // fastest 266 { 267 version (Posix) return ent.d_name.ptr; 268 version (Windows) return findData.cFileName.ptr; 269 } 270 271 // Bounded variant of std.string.fromStringz for static arrays. 272 private static T[] fromStringz(T, size_t n)(ref T[n] buf) 273 { 274 foreach (i, c; buf) 275 if (!c) 276 return buf[0 .. i]; 277 // This should only happen in case of an OS / libc bug. 278 assert(false, "File name buffer is not null-terminated"); 279 } 280 281 const(FSChar)[] baseNameFS() pure nothrow @nogc // fast 282 { 283 if (!data.baseNameFS) 284 { 285 version (Posix) data.baseNameFS = fromStringz(ent.d_name); 286 version (Windows) data.baseNameFS = fromStringz(findData.cFileName); 287 } 288 return data.baseNameFS; 289 } 290 291 string baseName() // allocates 292 { 293 if (!data.baseName) 294 data.baseName = baseNameFS.to!string; 295 return data.baseName; 296 } 297 298 private void needFullPath() nothrow @nogc 299 { 300 if (!data.pathTailPos) 301 { 302 version (Posix) 303 parent.needFullPath(); 304 version (Windows) 305 { 306 // directory separator was added during recursion 307 auto startPos = parent.data.pathTailPos + 1; 308 } 309 version (Posix) 310 { 311 immutable FSChar[] separator = "/"; 312 auto startPos = appendString(context.pathBuf, 313 parent.data.pathTailPos, separator); 314 } 315 data.pathTailPos = appendString(context.pathBuf, 316 startPos, 317 baseNameFSPtr.nullTerminated 318 ); 319 } 320 } 321 322 const(FSChar)[] fullNameFS() nothrow @nogc // fast 323 { 324 needFullPath(); 325 return context.pathBuf[0 .. data.pathTailPos]; 326 } 327 328 string fullName() // allocates 329 { 330 if (!data.fullName) 331 data.fullName = fullNameFS.to!string; 332 return data.fullName; 333 } 334 335 // Attributes 336 337 version (Posix) 338 { 339 enum StatTarget 340 { 341 dirEntry, // do not dereference (lstat) 342 linkTarget, // dereference 343 } 344 private bool tryStat(StatTarget target)() nothrow @nogc 345 { 346 if (data.statResult[target] == StatResult.noInfo) 347 { 348 // If we already did the other kind of stat, can we reuse its result? 349 if (data.statResult[1 - target] != StatResult.noInfo) 350 { 351 // Yes, if we know this isn't a link from the directory entry. 352 static if (__traits(compiles, ent.d_type)) 353 if (ent.d_type != DT_UNKNOWN && ent.d_type != DT_LNK) 354 goto reuse; 355 // Yes, if we already found out this isn't a link from an lstat call. 356 static if (target == StatTarget.linkTarget) 357 if (data.statResult[StatTarget.dirEntry] == StatResult.statOK 358 && (statBuf[StatTarget.dirEntry].st_mode & S_IFMT) != S_IFLNK) 359 goto reuse; 360 } 361 362 if (false) 363 { 364 reuse: 365 statBuf[target] = statBuf[1 - target]; 366 data.statResult[target] = data.statResult[1 - target]; 367 } 368 else 369 { 370 int flags = target == StatTarget.dirEntry ? AT_SYMLINK_NOFOLLOW : 0; 371 auto res = fstatat(dirFD, ent.d_name.ptr, &statBuf[target], flags); 372 if (res) 373 { 374 auto error = errno; 375 data.statResult[target] = cast(StatResult)error; 376 if (error == StatResult.noInfo || error == StatResult.statOK) 377 data.statResult[target] = StatResult.unknownError; // unknown error? 378 } 379 else 380 data.statResult[target] = StatResult.statOK; // no error 381 } 382 } 383 return data.statResult[target] == StatResult.statOK; 384 } 385 386 ErrnoException statError(StatTarget target)() 387 { 388 errno = data.statResult[target]; 389 return new ErrnoException("Failed to stat " ~ 390 (target == StatTarget.linkTarget ? "link target" : "directory entry") ~ 391 ": " ~ fullName); 392 } 393 394 stat_t* needStat(StatTarget target)() 395 { 396 if (!tryStat!target) 397 throw statError!target(); 398 return &statBuf[target]; 399 } 400 401 // Check if this is an object of the given type. 402 private bool deIsType(typeof(DT_REG) dType, typeof(S_IFREG) statType) 403 { 404 static if (__traits(compiles, ent.d_type)) 405 if (ent.d_type != DT_UNKNOWN) 406 return ent.d_type == dType; 407 408 return (needStat!(StatTarget.dirEntry)().st_mode & S_IFMT) == statType; 409 } 410 411 /// Returns true if this is a symlink. 412 @property bool isSymlink() 413 { 414 return deIsType(DT_LNK, S_IFLNK); 415 } 416 417 /// Returns true if this is a directory. 418 /// You probably want to use this one to decide whether to recurse. 419 @property bool entryIsDir() 420 { 421 return deIsType(DT_DIR, S_IFDIR); 422 } 423 424 // Check if this is an object of the given type, or a link pointing to one. 425 private bool ltIsType(typeof(DT_REG) dType, typeof(S_IFREG) statType) 426 { 427 static if (__traits(compiles, ent.d_type)) 428 if (ent.d_type != DT_UNKNOWN && ent.d_type != DT_LNK) 429 return ent.d_type == dType; 430 431 if (tryStat!(StatTarget.linkTarget)()) 432 return (statBuf[StatTarget.linkTarget].st_mode & S_IFMT) == statType; 433 434 if (isSymlink()) // broken symlink? 435 return false; // a broken symlink does not point at anything. 436 437 throw statError!(StatTarget.linkTarget)(); 438 } 439 440 /// Returns true if this is a file, or a link pointing to one. 441 @property bool isFile() 442 { 443 return ltIsType(DT_REG, S_IFREG); 444 } 445 446 /// Returns true if this is a directory, or a link pointing to one. 447 @property bool isDir() 448 { 449 return ltIsType(DT_DIR, S_IFDIR); 450 } 451 452 @property uint attributes() 453 { 454 return needStat!(StatTarget.linkTarget)().st_mode; 455 } 456 457 @property uint linkAttributes() 458 { 459 return needStat!(StatTarget.dirEntry)().st_mode; 460 } 461 462 // Other attributes 463 464 @property SysTime timeStatusChanged() 465 { 466 return statTimeToStdTime!"c"(*needStat!(StatTarget.linkTarget)()); 467 } 468 469 @property SysTime timeLastAccessed() 470 { 471 return statTimeToStdTime!"a"(*needStat!(StatTarget.linkTarget)()); 472 } 473 474 @property SysTime timeLastModified() 475 { 476 return statTimeToStdTime!"m"(*needStat!(StatTarget.linkTarget)()); 477 } 478 479 static if (is(typeof(&statTimeToStdTime!"birth"))) 480 @property SysTime timeCreated() 481 { 482 return statTimeToStdTime!"m"(*needStat!(StatTarget.linkTarget)()); 483 } 484 485 @property ulong size() 486 { 487 return needStat!(StatTarget.linkTarget)().st_size; 488 } 489 490 @property ulong fileID() 491 { 492 static if (__traits(compiles, ent.d_ino)) 493 return ent.d_ino; 494 else 495 return needStat!(StatTarget.linkTarget)().st_ino; 496 } 497 } 498 499 version (Windows) 500 { 501 @property bool isDir() const pure nothrow 502 { 503 return (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; 504 } 505 506 @property bool isFile() const pure nothrow 507 { 508 return !isDir; 509 } 510 511 @property bool isSymlink() const pure nothrow 512 { 513 return (findData.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0; 514 } 515 516 @property bool entryIsDir() const pure nothrow 517 { 518 return isDir && !isSymlink; 519 } 520 521 @property ulong size() const pure nothrow 522 { 523 return makeUlong(findData.nFileSizeLow, findData.nFileSizeHigh); 524 } 525 526 @property SysTime timeCreated() const 527 { 528 return FILETIMEToSysTime(&findData.ftCreationTime); 529 } 530 531 @property SysTime timeLastAccessed() const 532 { 533 return FILETIMEToSysTime(&findData.ftLastAccessTime); 534 } 535 536 @property SysTime timeLastModified() const 537 { 538 return FILETIMEToSysTime(&findData.ftLastWriteTime); 539 } 540 541 @property ulong fileID() 542 { 543 return getFileID(fullName); 544 } 545 } 546 } 547 548 version (Posix) 549 { 550 // The length of the buffer on the stack. 551 enum initialPathBufLength = 256; 552 553 static void scan(DIR* dir, int dirFD, Entry* parentEntry) 554 { 555 Entry entry = void; 556 entry.parent = parentEntry; 557 entry.context = entry.parent.context; 558 entry.dirFD = dirFD; 559 560 scope(exit) closedir(dir); 561 562 dirent* ent; 563 while ((ent = readdir(dir)) != null) 564 { 565 // Apparently happens on some OS X versions. 566 enforce(ent.d_name[0], 567 "Empty dir entry name (OS bug?)"); 568 569 // Skip "." and ".." 570 if (ent.d_name[0] == '.' && ( 571 ent.d_name[1] == 0 || 572 (ent.d_name[1] == '.' && ent.d_name[2] == 0))) 573 continue; 574 575 entry.ent = ent; 576 entry.data = Entry.Data.init; 577 entry.context.callHandler(&entry); 578 if (entry.context.timeToStop) 579 break; 580 } 581 } 582 } 583 584 enum isPath(Path) = (isForwardRange!Path || isSomeString!Path) && 585 isSomeChar!(ElementEncodingType!Path); 586 587 import core.stdc.stdlib : malloc, realloc, free; 588 589 static FSChar[] reallocPathBuf(FSChar[] buf, size_t newLength) nothrow @nogc 590 { 591 if (buf.length == initialPathBufLength) // current buffer is on stack 592 { 593 auto ptr = cast(FSChar*) malloc(newLength * FSChar.sizeof); 594 ptr[0 .. buf.length] = buf[]; 595 return ptr[0 .. newLength]; 596 } 597 else // current buffer on C heap (malloc'd above) 598 { 599 auto ptr = cast(FSChar*) realloc(buf.ptr, newLength * FSChar.sizeof); 600 return ptr[0 .. newLength]; 601 } 602 } 603 604 // Append a string to the buffer, reallocating as necessary. 605 // Returns the new length of the string in the buffer. 606 static size_t appendString(Str)(ref FSChar[] buf, size_t pos, Str str) nothrow @nogc 607 if (isPath!Str) 608 { 609 static if (ElementEncodingType!Str.sizeof == FSChar.sizeof 610 && is(typeof(str.length))) 611 { 612 // No transcoding needed and length known 613 auto remainingSpace = buf.length - pos; 614 if (str.length > remainingSpace) 615 buf = reallocPathBuf(buf, (pos + str.length) * 3 / 2); 616 buf[pos .. pos + str.length] = str[]; 617 pos += str.length; 618 } 619 else 620 { 621 // Need to transcode 622 auto p = buf.ptr + pos; 623 auto bufEnd = buf.ptr + buf.length; 624 foreach (c; byUTF!FSChar(str)) 625 { 626 if (p == bufEnd) // out of room 627 { 628 auto newBuf = reallocPathBuf(buf, buf.length * 3 / 2); 629 630 // Update pointers to point into the new buffer. 631 p = newBuf.ptr + (p - buf.ptr); 632 buf = newBuf; 633 bufEnd = buf.ptr + buf.length; 634 } 635 *p++ = c; 636 } 637 pos = p - buf.ptr; 638 } 639 return pos; 640 } 641 642 version (Windows) 643 { 644 mixin(importWin32!(q{winbase})); 645 import ae.sys.windows.misc : makeUlong; 646 647 // The length of the buffer on the stack. 648 enum initialPathBufLength = MAX_PATH; 649 650 enum FIND_FIRST_EX_LARGE_FETCH = 2; 651 enum FindExInfoBasic = cast(FINDEX_INFO_LEVELS)1; 652 653 static void scan(Entry* parentEntry) 654 { 655 Entry entry = void; 656 entry.parent = parentEntry; 657 entry.context = parentEntry.context; 658 659 HANDLE hFind = FindFirstFileExW( 660 entry.context.pathBuf.ptr, 661 FindExInfoBasic, 662 &entry.findData, 663 FINDEX_SEARCH_OPS.FindExSearchNameMatch, 664 null, 665 FIND_FIRST_EX_LARGE_FETCH, // https://blogs.msdn.microsoft.com/oldnewthing/20131024-00/?p=2843 666 ); 667 if (hFind == INVALID_HANDLE_VALUE) 668 throw new WindowsException(GetLastError(), 669 text("FindFirstFileW: ", parentEntry.fullNameFS)); 670 scope(exit) FindClose(hFind); 671 do 672 { 673 // Skip "." and ".." 674 auto fn = entry.findData.cFileName.ptr; 675 if (fn[0] == '.' && ( 676 fn[1] == 0 || 677 (fn[1] == '.' && fn[2] == 0))) 678 continue; 679 680 entry.data = Entry.Data.init; 681 entry.context.callHandler(&entry); 682 if (entry.context.timeToStop) 683 break; 684 } 685 while (FindNextFileW(hFind, &entry.findData)); 686 if (GetLastError() != ERROR_NO_MORE_FILES) 687 throw new WindowsException(GetLastError(), 688 text("FindNextFileW: ", parentEntry.fullNameFS)); 689 } 690 } 691 692 void listDir(Path)(Path dirPath) 693 if (isPath!Path) 694 { 695 import std.internal.cstring; 696 697 if (dirPath.empty) 698 return listDir("."); 699 700 Context context; 701 702 FSChar[initialPathBufLength] pathBufStore = void; 703 context.pathBuf = pathBufStore[]; 704 705 scope (exit) 706 { 707 if (context.pathBuf.length != initialPathBufLength) 708 free(context.pathBuf.ptr); 709 } 710 711 Entry rootEntry = void; 712 rootEntry.context = &context; 713 714 auto endPos = appendString(context.pathBuf, 0, dirPath); 715 rootEntry.data.pathTailPos = endPos - (endPos > 0 && context.pathBuf[endPos - 1].isDirSeparator() ? 1 : 0); 716 assert(rootEntry.data.pathTailPos > 0); 717 718 version (Posix) 719 { 720 auto dir = opendir(tempCString(dirPath)); 721 checkDir(dir, dirPath); 722 723 scan(dir, dirfd(dir), &rootEntry); 724 } 725 else 726 version (Windows) 727 { 728 const WCHAR[] tailString = endPos == 0 || context.pathBuf[endPos - 1].isDirSeparator() ? "*.*\0"w : "\\*.*\0"w; 729 appendString(context.pathBuf, endPos, tailString); 730 731 scan(&rootEntry); 732 } 733 } 734 735 // Workaround for https://github.com/ldc-developers/ldc/issues/2960 736 version (Posix) 737 private void checkDir(Path)(DIR* dir, auto ref Path dirPath) 738 { 739 errnoEnforce(dir, "Failed to open directory " ~ dirPath); 740 } 741 } 742 743 unittest 744 { 745 auto tmpDir = deleteme ~ "-dir"; 746 if (tmpDir.exists) tmpDir.removeRecurse(); 747 mkdirRecurse(tmpDir); 748 scope(exit) rmdirRecurse(tmpDir); 749 750 touch(tmpDir ~ "/a"); 751 touch(tmpDir ~ "/b"); 752 mkdir(tmpDir ~ "/c"); 753 touch(tmpDir ~ "/c/1"); 754 touch(tmpDir ~ "/c/2"); 755 756 string[] entries; 757 listDir!((e) { 758 assert(equal(e.fullNameFS, e.fullName)); 759 entries ~= e.fullName.relPath(tmpDir); 760 if (e.entryIsDir) 761 e.recurse(); 762 })(tmpDir); 763 764 assert(equal( 765 entries.sort, 766 ["a", "b", "c", "c/1", "c/2"].map!(name => name.replace("/", dirSeparator)), 767 ), text(entries)); 768 769 entries = null; 770 import std.ascii : isDigit; 771 listDir!((e) { 772 entries ~= e.fullName.relPath(tmpDir); 773 if (e.baseNameFS[0].isDigit) 774 e.stop(); 775 else 776 if (e.entryIsDir) 777 e.recurse(); 778 })(tmpDir); 779 780 assert(entries.length < 5 && entries[$-1][$-1].isDigit, text(entries)); 781 782 // Symlink test 783 (){ 784 // Wine's implementation of symlinks/junctions is incomplete 785 version (Windows) 786 if (getWineVersion()) 787 return; 788 789 dirLink("c", tmpDir ~ "/d"); 790 dirLink("x", tmpDir ~ "/e"); 791 792 string[] entries; 793 listDir!((e) { 794 entries ~= e.fullName.relPath(tmpDir); 795 if (e.entryIsDir) 796 e.recurse(); 797 })(tmpDir); 798 799 assert(equal( 800 entries.sort, 801 ["a", "b", "c", "c/1", "c/2", "d", "e"].map!(name => name.replace("/", dirSeparator)), 802 )); 803 804 // Recurse into symlinks 805 806 entries = null; 807 listDir!((e) { 808 entries ~= e.fullName.relPath(tmpDir); 809 if (e.isDir) 810 try 811 e.recurse(); 812 catch (Exception e) // broken junctions on Windows throw 813 {} 814 })(tmpDir); 815 816 assert(equal( 817 entries.sort, 818 ["a", "b", "c", "c/1", "c/2", "d", "d/1", "d/2", "e"].map!(name => name.replace("/", dirSeparator)), 819 )); 820 }(); 821 } 822 823 // ************************************************************************ 824 825 string buildPath2(string[] segments...) { return segments.length ? buildPath(segments) : null; } 826 827 /// Shell-like expansion of ?, * and ** in path components 828 DirEntry[] fileList(string pattern) 829 { 830 auto components = cast(string[])array(pathSplitter(pattern)); 831 foreach (i, component; components[0..$-1]) 832 if (component.contains("?") || component.contains("*")) // TODO: escape? 833 { 834 DirEntry[] expansions; // TODO: filter range instead? 835 auto dir = buildPath2(components[0..i]); 836 if (component == "**") 837 expansions = array(dirEntries(dir, SpanMode.depth)); 838 else 839 expansions = array(dirEntries(dir, component, SpanMode.shallow)); 840 841 DirEntry[] result; 842 foreach (expansion; expansions) 843 if (expansion.isDir()) 844 result ~= fileList(buildPath(expansion.name ~ components[i+1..$])); 845 return result; 846 } 847 848 auto dir = buildPath2(components[0..$-1]); 849 if (!dir || exists(dir)) 850 return array(dirEntries(dir, components[$-1], SpanMode.shallow)); 851 else 852 return null; 853 } 854 855 /// ditto 856 DirEntry[] fileList(string pattern0, string[] patterns...) 857 { 858 DirEntry[] result; 859 foreach (pattern; [pattern0] ~ patterns) 860 result ~= fileList(pattern); 861 return result; 862 } 863 864 /// ditto 865 deprecated string[] fastFileList(string pattern) 866 { 867 auto components = cast(string[])array(pathSplitter(pattern)); 868 foreach (i, component; components[0..$-1]) 869 if (component.contains("?") || component.contains("*")) // TODO: escape? 870 { 871 string[] expansions; // TODO: filter range instead? 872 auto dir = buildPath2(components[0..i]); 873 if (component == "**") 874 expansions = fastListDir!true(dir); 875 else 876 expansions = fastListDir(dir, component); 877 878 string[] result; 879 foreach (expansion; expansions) 880 if (expansion.isDir()) 881 result ~= fastFileList(buildPath(expansion ~ components[i+1..$])); 882 return result; 883 } 884 885 auto dir = buildPath2(components[0..$-1]); 886 if (!dir || exists(dir)) 887 return fastListDir(dir, components[$-1]); 888 else 889 return null; 890 } 891 892 /// ditto 893 deprecated string[] fastFileList(string pattern0, string[] patterns...) 894 { 895 string[] result; 896 foreach (pattern; [pattern0] ~ patterns) 897 result ~= fastFileList(pattern); 898 return result; 899 } 900 901 // ************************************************************************ 902 903 import std.datetime; 904 import std.exception; 905 906 deprecated SysTime getMTime(string name) 907 { 908 return timeLastModified(name); 909 } 910 911 /// If target exists, update its modification time; 912 /// otherwise create it as an empty file. 913 void touch(in char[] target) 914 { 915 if (exists(target)) 916 { 917 auto now = Clock.currTime(); 918 setTimes(target, now, now); 919 } 920 else 921 std.file.write(target, ""); 922 } 923 924 /// Returns true if the target file doesn't exist, 925 /// or source is newer than the target. 926 bool newerThan(string source, string target) 927 { 928 if (!target.exists) 929 return true; 930 return source.timeLastModified() > target.timeLastModified(); 931 } 932 933 /// Returns true if the target file doesn't exist, 934 /// or any of the sources are newer than the target. 935 bool anyNewerThan(string[] sources, string target) 936 { 937 if (!target.exists) 938 return true; 939 auto targetTime = target.timeLastModified(); 940 return sources.any!(source => source.timeLastModified() > targetTime)(); 941 } 942 943 version (Posix) 944 { 945 import core.sys.posix.sys.stat; 946 import core.sys.posix.unistd; 947 948 int getOwner(string fn) 949 { 950 stat_t s; 951 errnoEnforce(stat(toStringz(fn), &s) == 0, "stat: " ~ fn); 952 return s.st_uid; 953 } 954 955 int getGroup(string fn) 956 { 957 stat_t s; 958 errnoEnforce(stat(toStringz(fn), &s) == 0, "stat: " ~ fn); 959 return s.st_gid; 960 } 961 962 void setOwner(string fn, int uid, int gid) 963 { 964 errnoEnforce(chown(toStringz(fn), uid, gid) == 0, "chown: " ~ fn); 965 } 966 } 967 968 /// Try to rename; copy/delete if rename fails 969 void move(string src, string dst) 970 { 971 try 972 src.rename(dst); 973 catch (Exception e) 974 { 975 atomicCopy(src, dst); 976 src.remove(); 977 } 978 } 979 980 /// Make sure that the given directory exists 981 /// (and create parent directories as necessary). 982 void ensureDirExists(string path) 983 { 984 if (!path.exists) 985 path.mkdirRecurse(); 986 } 987 988 /// Make sure that the path to the given file name 989 /// exists (and create directories as necessary). 990 void ensurePathExists(string fn) 991 { 992 fn.dirName.ensureDirExists(); 993 } 994 995 static import core.stdc.errno; 996 version (Windows) 997 { 998 static import core.sys.windows.winerror; 999 static import std.windows.syserror; 1000 static import ae.sys.windows.exception; 1001 } 1002 1003 bool collectOSError(alias checkCError, alias checkWinError)(scope void delegate() operation) 1004 { 1005 mixin(() { 1006 string code = q{ 1007 try 1008 { 1009 operation(); 1010 return true; 1011 } 1012 catch (FileException e) 1013 { 1014 version (Windows) 1015 bool collect = checkWinError(e.errno); 1016 else 1017 bool collect = checkCError(e.errno); 1018 if (collect) 1019 return false; 1020 else 1021 throw e; 1022 } 1023 catch (ErrnoException e) 1024 { 1025 if (checkCError(e.errno)) 1026 return false; 1027 else 1028 throw e; 1029 } 1030 }; 1031 version(Windows) code ~= q{ 1032 catch (std.windows.syserror.WindowsException e) 1033 { 1034 if (checkWinError(e.code)) 1035 return false; 1036 else 1037 throw e; 1038 } 1039 catch (ae.sys.windows.exception.WindowsException e) 1040 { 1041 if (checkWinError(e.code)) 1042 return false; 1043 else 1044 throw e; 1045 } 1046 }; 1047 return code; 1048 }()); 1049 } 1050 1051 alias collectNotFoundError = collectOSError!( 1052 errno => errno == core.stdc.errno.ENOENT, 1053 (code) { version(Windows) return 1054 code == core.sys.windows.winerror.ERROR_FILE_NOT_FOUND || 1055 code == core.sys.windows.winerror.ERROR_PATH_NOT_FOUND; }, 1056 ); 1057 1058 unittest 1059 { 1060 auto fn = deleteme; 1061 if (fn.exists) fn.removeRecurse(); 1062 foreach (dg; [ 1063 { openFile(fn, "rb"); }, 1064 { mkdir(fn.buildPath("b")); }, 1065 { hardLink(fn, fn ~ "2"); }, 1066 ]) 1067 assert(!dg.collectNotFoundError); 1068 } 1069 1070 alias collectFileExistsError = collectOSError!( 1071 errno => errno == core.stdc.errno.EEXIST, 1072 (code) { version(Windows) return 1073 code == core.sys.windows.winerror.ERROR_FILE_EXISTS || 1074 code == core.sys.windows.winerror.ERROR_ALREADY_EXISTS; }, 1075 ); 1076 1077 unittest 1078 { 1079 auto fn = deleteme; 1080 foreach (dg; [ 1081 { mkdir(fn); }, 1082 { openFile(fn, "wxb"); }, 1083 { touch(fn ~ "2"); hardLink(fn ~ "2", fn); }, 1084 ]) 1085 { 1086 if (fn.exists) fn.removeRecurse(); 1087 assert( dg.collectFileExistsError); 1088 assert(!dg.collectFileExistsError); 1089 } 1090 } 1091 1092 import ae.utils.text; 1093 1094 /// Forcibly remove a file or directory. 1095 /// If atomic is true, the entire directory is deleted "atomically" 1096 /// (it is first moved/renamed to another location). 1097 /// On Windows, this will move the file/directory out of the way, 1098 /// if it is in use and cannot be deleted (but can be renamed). 1099 void forceDelete(Flag!"atomic" atomic=Yes.atomic)(string fn, Flag!"recursive" recursive = No.recursive) 1100 { 1101 import std.process : environment; 1102 version(Windows) 1103 { 1104 mixin(importWin32!q{winnt}); 1105 mixin(importWin32!q{winbase}); 1106 } 1107 1108 auto name = fn.baseName(); 1109 fn = fn.absolutePath().longPath(); 1110 1111 version(Windows) 1112 { 1113 auto fnW = toUTF16z(fn); 1114 auto attr = GetFileAttributesW(fnW); 1115 wenforce(attr != INVALID_FILE_ATTRIBUTES, "GetFileAttributes"); 1116 if (attr & FILE_ATTRIBUTE_READONLY) 1117 SetFileAttributesW(fnW, attr & ~FILE_ATTRIBUTE_READONLY).wenforce("SetFileAttributes"); 1118 } 1119 1120 static if (atomic) 1121 { 1122 // To avoid zombifying locked directories, try renaming it first. 1123 // Attempting to delete a locked directory will make it inaccessible. 1124 1125 bool tryMoveTo(string target) 1126 { 1127 target = target.longPath(); 1128 if (target.endsWith(dirSeparator)) 1129 target = target[0..$-1]; 1130 if (target.length && !target.exists) 1131 return false; 1132 1133 string newfn; 1134 do 1135 newfn = format("%s%sdeleted-%s.%s.%s", target, dirSeparator, name, thisProcessID, randomString()); 1136 while (newfn.exists); 1137 1138 version(Windows) 1139 { 1140 auto newfnW = toUTF16z(newfn); 1141 if (!MoveFileW(fnW, newfnW)) 1142 return false; 1143 } 1144 else 1145 { 1146 try 1147 rename(fn, newfn); 1148 catch (FileException e) 1149 return false; 1150 } 1151 1152 fn = newfn; 1153 version(Windows) fnW = newfnW; 1154 return true; 1155 } 1156 1157 void tryMove() 1158 { 1159 auto tmp = environment.get("TEMP"); 1160 if (tmp) 1161 if (tryMoveTo(tmp)) 1162 return; 1163 1164 version(Windows) 1165 string tempDir = fn[0..7]~"Temp"; 1166 else 1167 enum tempDir = "/tmp"; 1168 1169 if (tryMoveTo(tempDir)) 1170 return; 1171 1172 if (tryMoveTo(fn.dirName())) 1173 return; 1174 1175 throw new Exception("Unable to delete " ~ fn ~ " atomically (all rename attempts failed)"); 1176 } 1177 1178 tryMove(); 1179 } 1180 1181 version(Windows) 1182 { 1183 if (attr & FILE_ATTRIBUTE_DIRECTORY) 1184 { 1185 if (recursive && (attr & FILE_ATTRIBUTE_REPARSE_POINT) == 0) 1186 { 1187 foreach (de; fn.dirEntries(SpanMode.shallow)) 1188 forceDelete!(No.atomic)(de.name, Yes.recursive); 1189 } 1190 // Will fail if !recursive and directory is not empty 1191 RemoveDirectoryW(fnW).wenforce("RemoveDirectory"); 1192 } 1193 else 1194 DeleteFileW(fnW).wenforce("DeleteFile"); 1195 } 1196 else 1197 { 1198 if (recursive) 1199 fn.removeRecurse(); 1200 else 1201 if (fn.isDir) 1202 fn.rmdir(); 1203 else 1204 fn.remove(); 1205 } 1206 } 1207 1208 1209 deprecated void forceDelete(bool atomic)(string fn, bool recursive = false) { forceDelete!(cast(Flag!"atomic")atomic)(fn, cast(Flag!"recursive")recursive); } 1210 //deprecated void forceDelete()(string fn, bool recursive) { forceDelete!(Yes.atomic)(fn, cast(Flag!"recursive")recursive); } 1211 1212 deprecated unittest 1213 { 1214 mkdir("testdir"); touch("testdir/b"); forceDelete!(false )("testdir", true); 1215 mkdir("testdir"); touch("testdir/b"); forceDelete!(true )("testdir", true); 1216 } 1217 1218 unittest 1219 { 1220 mkdir("testdir"); touch("testdir/b"); forceDelete ("testdir", Yes.recursive); 1221 mkdir("testdir"); touch("testdir/b"); forceDelete!(No .atomic)("testdir", Yes.recursive); 1222 mkdir("testdir"); touch("testdir/b"); forceDelete!(Yes.atomic)("testdir", Yes.recursive); 1223 } 1224 1225 /// If fn is a directory, delete it recursively. 1226 /// Otherwise, delete the file or symlink fn. 1227 void removeRecurse(string fn) 1228 { 1229 auto attr = fn.getAttributes(); 1230 if (attr.attrIsSymlink) 1231 { 1232 version (Windows) 1233 if (attr.attrIsDir) 1234 fn.rmdir(); 1235 else 1236 fn.remove(); 1237 else 1238 fn.remove(); 1239 } 1240 else 1241 if (attr.attrIsDir) 1242 version (Windows) 1243 fn.forceDelete!(No.atomic)(Yes.recursive); // For read-only files 1244 else 1245 fn.rmdirRecurse(); 1246 else 1247 fn.remove(); 1248 } 1249 1250 /// Create an empty directory, deleting 1251 /// all its contents if it already exists. 1252 void recreateEmptyDirectory()(string dir) 1253 { 1254 if (dir.exists) 1255 dir.forceDelete(Yes.recursive); 1256 mkdir(dir); 1257 } 1258 1259 void copyRecurse(DirEntry src, string dst) 1260 { 1261 version (Posix) 1262 if (src.isSymlink) 1263 return symlink(dst, readLink(src)); 1264 if (src.isFile) 1265 return copy(src, dst, PreserveAttributes.yes); 1266 dst.mkdir(); 1267 foreach (de; src.dirEntries(SpanMode.shallow)) 1268 copyRecurse(de, dst.buildPath(de.baseName)); 1269 } 1270 void copyRecurse(string src, string dst) { copyRecurse(DirEntry(src), dst); } 1271 1272 bool isHidden()(string fn) 1273 { 1274 if (baseName(fn).startsWith(".")) 1275 return true; 1276 version (Windows) 1277 { 1278 mixin(importWin32!q{winnt}); 1279 if (getAttributes(fn) & FILE_ATTRIBUTE_HIDDEN) 1280 return true; 1281 } 1282 return false; 1283 } 1284 1285 /// Return a file's unique ID. 1286 ulong getFileID()(string fn) 1287 { 1288 version (Windows) 1289 { 1290 mixin(importWin32!q{winnt}); 1291 mixin(importWin32!q{winbase}); 1292 1293 auto fnW = toUTF16z(fn); 1294 auto h = CreateFileW(fnW, FILE_READ_ATTRIBUTES, 0, null, OPEN_EXISTING, 0, HANDLE.init); 1295 wenforce(h!=INVALID_HANDLE_VALUE, fn); 1296 scope(exit) CloseHandle(h); 1297 BY_HANDLE_FILE_INFORMATION fi; 1298 GetFileInformationByHandle(h, &fi).wenforce("GetFileInformationByHandle"); 1299 1300 ULARGE_INTEGER li; 1301 li.LowPart = fi.nFileIndexLow; 1302 li.HighPart = fi.nFileIndexHigh; 1303 auto result = li.QuadPart; 1304 enforce(result, "Null file ID"); 1305 return result; 1306 } 1307 else 1308 { 1309 return DirEntry(fn).statBuf.st_ino; 1310 } 1311 } 1312 1313 unittest 1314 { 1315 touch("a"); 1316 scope(exit) remove("a"); 1317 hardLink("a", "b"); 1318 scope(exit) remove("b"); 1319 touch("c"); 1320 scope(exit) remove("c"); 1321 assert(getFileID("a") == getFileID("b")); 1322 assert(getFileID("a") != getFileID("c")); 1323 } 1324 1325 deprecated alias std.file.getSize getSize2; 1326 1327 /// Using UNC paths bypasses path length limitation when using Windows wide APIs. 1328 string longPath(string s) 1329 { 1330 version (Windows) 1331 { 1332 if (!s.startsWith(`\\`)) 1333 return `\\?\` ~ s.absolutePath().buildNormalizedPath().replace(`/`, `\`); 1334 } 1335 return s; 1336 } 1337 1338 version (Windows) 1339 { 1340 static if (__traits(compiles, { mixin importWin32!q{winnt}; })) 1341 static mixin(importWin32!q{winnt}); 1342 1343 void createReparsePoint(string reparseBufferName, string extraInitialization, string reparseTagName)(in char[] target, in char[] print, in char[] link) 1344 { 1345 mixin(importWin32!q{winbase}); 1346 mixin(importWin32!q{windef}); 1347 mixin(importWin32!q{winioctl}); 1348 1349 enum SYMLINK_FLAG_RELATIVE = 1; 1350 1351 HANDLE hLink = CreateFileW(link.toUTF16z(), GENERIC_READ | GENERIC_WRITE, 0, null, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, null); 1352 wenforce(hLink && hLink != INVALID_HANDLE_VALUE, "CreateFileW"); 1353 scope(exit) CloseHandle(hLink); 1354 1355 enum pathOffset = 1356 mixin(q{REPARSE_DATA_BUFFER..} ~ reparseBufferName) .offsetof + 1357 mixin(q{REPARSE_DATA_BUFFER..} ~ reparseBufferName)._PathBuffer.offsetof; 1358 1359 auto targetW = target.toUTF16(); 1360 auto printW = print .toUTF16(); 1361 1362 // Despite MSDN, two NUL-terminating characters are needed, one for each string. 1363 1364 auto pathBufferSize = targetW.length + 1 + printW.length + 1; // in chars 1365 auto buf = new ubyte[pathOffset + pathBufferSize * WCHAR.sizeof]; 1366 auto r = cast(REPARSE_DATA_BUFFER*)buf.ptr; 1367 1368 r.ReparseTag = mixin(reparseTagName); 1369 r.ReparseDataLength = to!WORD(buf.length - mixin(q{r..} ~ reparseBufferName).offsetof); 1370 1371 auto pathBuffer = mixin(q{r..} ~ reparseBufferName).PathBuffer; 1372 auto p = pathBuffer; 1373 1374 mixin(q{r..} ~ reparseBufferName).SubstituteNameOffset = to!WORD((p-pathBuffer) * WCHAR.sizeof); 1375 mixin(q{r..} ~ reparseBufferName).SubstituteNameLength = to!WORD(targetW.length * WCHAR.sizeof); 1376 p[0..targetW.length] = targetW; 1377 p += targetW.length; 1378 *p++ = 0; 1379 1380 mixin(q{r..} ~ reparseBufferName).PrintNameOffset = to!WORD((p-pathBuffer) * WCHAR.sizeof); 1381 mixin(q{r..} ~ reparseBufferName).PrintNameLength = to!WORD(printW .length * WCHAR.sizeof); 1382 p[0..printW.length] = printW; 1383 p += printW.length; 1384 *p++ = 0; 1385 1386 assert(p-pathBuffer == pathBufferSize); 1387 1388 mixin(extraInitialization); 1389 1390 DWORD dwRet; // Needed despite MSDN 1391 DeviceIoControl(hLink, FSCTL_SET_REPARSE_POINT, buf.ptr, buf.length.to!DWORD(), null, 0, &dwRet, null).wenforce("DeviceIoControl"); 1392 } 1393 1394 void acquirePrivilege(S)(S name) 1395 { 1396 mixin(importWin32!q{winbase}); 1397 mixin(importWin32!q{windef}); 1398 1399 import ae.sys.windows; 1400 1401 HANDLE hToken = null; 1402 wenforce(OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &hToken)); 1403 scope(exit) CloseHandle(hToken); 1404 1405 TOKEN_PRIVILEGES tp; 1406 wenforce(LookupPrivilegeValue(null, name.toUTF16z(), &tp.Privileges[0].Luid), "LookupPrivilegeValue"); 1407 1408 tp.PrivilegeCount = 1; 1409 tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; 1410 wenforce(AdjustTokenPrivileges(hToken, FALSE, &tp, cast(DWORD)TOKEN_PRIVILEGES.sizeof, null, null), "AdjustTokenPrivileges"); 1411 } 1412 1413 /// Link a directory. 1414 /// Uses symlinks on POSIX, and directory junctions on Windows. 1415 void dirLink()(in char[] original, in char[] link) 1416 { 1417 mkdir(link); 1418 scope(failure) rmdir(link); 1419 1420 auto target = `\??\` ~ (cast(string)original).absolutePath((cast(string)link.dirName).absolutePath).buildNormalizedPath; 1421 if (target[$-1] != '\\') 1422 target ~= '\\'; 1423 1424 createReparsePoint!(q{MountPointReparseBuffer}, q{}, q{IO_REPARSE_TAG_MOUNT_POINT})(target, null, link); 1425 } 1426 1427 void symlink()(in char[] original, in char[] link) 1428 { 1429 mixin(importWin32!q{winnt}); 1430 1431 acquirePrivilege(SE_CREATE_SYMBOLIC_LINK_NAME); 1432 1433 touch(link); 1434 scope(failure) remove(link); 1435 1436 createReparsePoint!(q{SymbolicLinkReparseBuffer}, q{r.SymbolicLinkReparseBuffer.Flags = link.isAbsolute() ? 0 : SYMLINK_FLAG_RELATIVE;}, q{IO_REPARSE_TAG_SYMLINK})(original, original, link); 1437 } 1438 } 1439 else 1440 alias std.file.symlink dirLink; 1441 1442 version(Windows) version(unittest) static mixin(importWin32!q{winnt}); 1443 1444 unittest 1445 { 1446 // Wine's implementation of symlinks/junctions is incomplete 1447 version (Windows) 1448 if (getWineVersion()) 1449 return; 1450 1451 mkdir("a"); scope(exit) rmdir("a"[]); 1452 touch("a/f"); scope(exit) remove("a/f"); 1453 dirLink("a", "b"); scope(exit) version(Windows) rmdir("b"); else remove("b"); 1454 //symlink("a/f", "c"); scope(exit) remove("c"); 1455 assert("b".isSymlink()); 1456 //assert("c".isSymlink()); 1457 assert("b/f".exists()); 1458 } 1459 1460 version (Windows) 1461 { 1462 void hardLink()(string src, string dst) 1463 { 1464 mixin(importWin32!q{w32api}); 1465 1466 static assert(_WIN32_WINNT >= 0x501, "CreateHardLinkW not available for target Windows platform. Specify -version=WindowsXP"); 1467 1468 mixin(importWin32!q{winnt}); 1469 mixin(importWin32!q{winbase}); 1470 1471 wenforce(CreateHardLinkW(toUTF16z(dst), toUTF16z(src), null), "CreateHardLink failed: " ~ src ~ " -> " ~ dst); 1472 } 1473 1474 /// Deletes a file, which might be a read-only hard link 1475 /// (thus, deletes the read-only file/link without affecting other links to it). 1476 void deleteHardLink()(string fn) 1477 { 1478 mixin(importWin32!q{winbase}); 1479 1480 auto fnW = toUTF16z(fn); 1481 1482 DWORD attrs = GetFileAttributesW(fnW); 1483 wenforce(attrs != INVALID_FILE_ATTRIBUTES, "GetFileAttributesW failed: " ~ fn); 1484 1485 if (attrs & FILE_ATTRIBUTE_READONLY) 1486 SetFileAttributesW(fnW, attrs & ~FILE_ATTRIBUTE_READONLY) 1487 .wenforce("SetFileAttributesW failed: " ~ fn); 1488 HANDLE h = CreateFileW(fnW, GENERIC_READ|GENERIC_WRITE, 7, null, OPEN_EXISTING, 1489 FILE_FLAG_DELETE_ON_CLOSE, null); 1490 wenforce(h != INVALID_HANDLE_VALUE, "CreateFileW failed: " ~ fn); 1491 if (attrs & FILE_ATTRIBUTE_READONLY) 1492 SetFileAttributesW(fnW, attrs) 1493 .wenforce("SetFileAttributesW failed: " ~ fn); 1494 CloseHandle(h).wenforce("CloseHandle failed: " ~ fn); 1495 } 1496 } 1497 version (Posix) 1498 { 1499 void hardLink()(string src, string dst) 1500 { 1501 import core.sys.posix.unistd; 1502 errnoEnforce(link(toUTFz!(const char*)(src), toUTFz!(const char*)(dst)) == 0, "link() failed: " ~ dst); 1503 } 1504 1505 alias deleteHardLink = remove; 1506 } 1507 1508 unittest 1509 { 1510 write("a", "foo"); scope(exit) remove("a"); 1511 hardLink("a", "b"); 1512 assert("b".readText == "foo"); 1513 deleteHardLink("b"); 1514 assert(!"b".exists); 1515 } 1516 1517 version (Posix) 1518 { 1519 string realPath(string path) 1520 { 1521 // TODO: Windows version 1522 import core.sys.posix.stdlib; 1523 auto p = realpath(toUTFz!(const char*)(path), null); 1524 errnoEnforce(p, "realpath"); 1525 string result = fromStringz(p).idup; 1526 free(p); 1527 return result; 1528 } 1529 } 1530 1531 // /proc/self/mounts parsing 1532 version (linux) 1533 { 1534 struct MountInfo 1535 { 1536 string spec; /// device path 1537 string file; /// mount path 1538 string vfstype; /// file system 1539 string mntops; /// options 1540 int freq; /// dump flag 1541 int passno; /// fsck order 1542 } 1543 1544 string unescapeMountString(in char[] s) 1545 { 1546 string result; 1547 1548 size_t p = 0; 1549 for (size_t i=0; i+3<s.length;) 1550 { 1551 auto c = s[i]; 1552 if (c == '\\') 1553 { 1554 result ~= s[p..i]; 1555 result ~= to!int(s[i+1..i+4], 8); 1556 i += 4; 1557 p = i; 1558 } 1559 else 1560 i++; 1561 } 1562 result ~= s[p..$]; 1563 return result; 1564 } 1565 1566 unittest 1567 { 1568 assert(unescapeMountString(`a\040b\040c`) == "a b c"); 1569 assert(unescapeMountString(`\040`) == " "); 1570 } 1571 1572 MountInfo parseMountInfo(in char[] line) 1573 { 1574 const(char)[][6] parts; 1575 copy(line.splitter(" "), parts[]); 1576 return MountInfo( 1577 unescapeMountString(parts[0]), 1578 unescapeMountString(parts[1]), 1579 unescapeMountString(parts[2]), 1580 unescapeMountString(parts[3]), 1581 parts[4].to!int, 1582 parts[5].to!int, 1583 ); 1584 } 1585 1586 /// Returns an iterator of MountInfo structs. 1587 auto getMounts() 1588 { 1589 return File("/proc/self/mounts", "rb").byLine().map!parseMountInfo(); 1590 } 1591 1592 /// Get MountInfo with longest mount point matching path. 1593 /// Returns MountInfo.init if none match. 1594 MountInfo getPathMountInfo(string path) 1595 { 1596 path = realPath(path); 1597 size_t bestLength; MountInfo bestInfo; 1598 foreach (ref info; getMounts()) 1599 { 1600 if (path.pathStartsWith(info.file)) 1601 { 1602 if (bestLength < info.file.length) 1603 { 1604 bestLength = info.file.length; 1605 bestInfo = info; 1606 } 1607 } 1608 } 1609 return bestInfo; 1610 } 1611 1612 /// Get the name of the filesystem that the given path is mounted under. 1613 /// Returns null if none match. 1614 string getPathFilesystem(string path) 1615 { 1616 return getPathMountInfo(path).vfstype; 1617 } 1618 } 1619 1620 // **************************************************************************** 1621 1622 version (linux) 1623 { 1624 import core.sys.linux.sys.xattr; 1625 import core.stdc.errno; 1626 alias ENOATTR = ENODATA; 1627 1628 /// AA-like object for accessing a file's extended attributes. 1629 struct XAttrs(Obj, string funPrefix) 1630 { 1631 Obj obj; 1632 1633 mixin("alias getFun = " ~ funPrefix ~ "getxattr;"); 1634 mixin("alias setFun = " ~ funPrefix ~ "setxattr;"); 1635 mixin("alias removeFun = " ~ funPrefix ~ "removexattr;"); 1636 mixin("alias listFun = " ~ funPrefix ~ "listxattr;"); 1637 1638 bool supported() 1639 { 1640 auto size = getFun(obj, "user.\x01", null, 0); 1641 return size >= 0 || errno != EOPNOTSUPP; 1642 } 1643 1644 void[] opIndex(string key) 1645 { 1646 auto cKey = key.toStringz(); 1647 size_t size = 0; 1648 void[] buf; 1649 do 1650 { 1651 buf.length = size; 1652 size = getFun(obj, cKey, buf.ptr, buf.length); 1653 errnoEnforce(size >= 0, __traits(identifier, getFun)); 1654 } while (size != buf.length); 1655 return buf; 1656 } 1657 1658 bool opBinaryRight(string op)(string key) 1659 if (op == "in") 1660 { 1661 auto cKey = key.toStringz(); 1662 auto size = getFun(obj, cKey, null, 0); 1663 if (size >= 0) 1664 return true; 1665 else 1666 if (errno == ENOATTR) 1667 return false; 1668 else 1669 errnoEnforce(false, __traits(identifier, getFun)); 1670 assert(false); 1671 } 1672 1673 void opIndexAssign(in void[] value, string key) 1674 { 1675 auto ret = setFun(obj, key.toStringz(), value.ptr, value.length, 0); 1676 errnoEnforce(ret == 0, __traits(identifier, setFun)); 1677 } 1678 1679 void remove(string key) 1680 { 1681 auto ret = removeFun(obj, key.toStringz()); 1682 errnoEnforce(ret == 0, __traits(identifier, removeFun)); 1683 } 1684 1685 string[] keys() 1686 { 1687 size_t size = 0; 1688 char[] buf; 1689 do 1690 { 1691 buf.length = size; 1692 size = listFun(obj, buf.ptr, buf.length); 1693 errnoEnforce(size >= 0, __traits(identifier, listFun)); 1694 } while (size != buf.length); 1695 1696 char[][] result; 1697 size_t start; 1698 foreach (p, c; buf) 1699 if (!c) 1700 { 1701 result ~= buf[start..p]; 1702 start = p+1; 1703 } 1704 1705 return cast(string[])result; 1706 } 1707 } 1708 1709 auto xAttrs(string path) 1710 { 1711 return XAttrs!(const(char)*, "")(path.toStringz()); 1712 } 1713 1714 auto linkXAttrs(string path) 1715 { 1716 return XAttrs!(const(char)*, "l")(path.toStringz()); 1717 } 1718 1719 auto xAttrs(in ref File f) 1720 { 1721 return XAttrs!(int, "f")(f.fileno); 1722 } 1723 1724 unittest 1725 { 1726 if (!xAttrs(".").supported) 1727 { 1728 import std.stdio : stderr; 1729 stderr.writeln("ae.sys.file: xattrs not supported on current filesystem, skipping test."); 1730 return; 1731 } 1732 1733 enum fn = "test.txt"; 1734 std.file.write(fn, "test"); 1735 scope(exit) remove(fn); 1736 1737 auto attrs = xAttrs(fn); 1738 enum key = "user.foo"; 1739 assert(key !in attrs); 1740 assert(attrs.keys == []); 1741 1742 attrs[key] = "bar"; 1743 assert(key in attrs); 1744 assert(attrs[key] == "bar"); 1745 assert(attrs.keys == [key]); 1746 1747 attrs.remove(key); 1748 assert(key !in attrs); 1749 assert(attrs.keys == []); 1750 } 1751 } 1752 1753 // **************************************************************************** 1754 1755 version (Windows) 1756 { 1757 /// Enumerate all hard links to the specified file. 1758 // TODO: Return a range 1759 string[] enumerateHardLinks()(string fn) 1760 { 1761 mixin(importWin32!q{winnt}); 1762 mixin(importWin32!q{winbase}); 1763 1764 alias extern(System) HANDLE function(LPCWSTR lpFileName, DWORD dwFlags, LPDWORD StringLength, PWCHAR LinkName) TFindFirstFileNameW; 1765 alias extern(System) BOOL function(HANDLE hFindStream, LPDWORD StringLength, PWCHAR LinkName) TFindNextFileNameW; 1766 1767 auto kernel32 = GetModuleHandle("kernel32.dll"); 1768 auto FindFirstFileNameW = cast(TFindFirstFileNameW)GetProcAddress(kernel32, "FindFirstFileNameW").wenforce("GetProcAddress(FindFirstFileNameW)"); 1769 auto FindNextFileNameW = cast(TFindNextFileNameW)GetProcAddress(kernel32, "FindNextFileNameW").wenforce("GetProcAddress(FindNextFileNameW)"); 1770 1771 static WCHAR[0x8000] buf; 1772 DWORD len = buf.length; 1773 auto h = FindFirstFileNameW(toUTF16z(fn), 0, &len, buf.ptr); 1774 wenforce(h != INVALID_HANDLE_VALUE, "FindFirstFileNameW"); 1775 scope(exit) FindClose(h); 1776 1777 string[] result; 1778 do 1779 { 1780 enforce(len > 0 && len < buf.length && buf[len-1] == 0, "Bad FindFirst/NextFileNameW result"); 1781 result ~= buf[0..len-1].toUTF8(); 1782 len = buf.length; 1783 auto ok = FindNextFileNameW(h, &len, buf.ptr); 1784 if (!ok && GetLastError() == ERROR_HANDLE_EOF) 1785 break; 1786 wenforce(ok, "FindNextFileNameW"); 1787 } while(true); 1788 return result; 1789 } 1790 } 1791 1792 uint hardLinkCount(string fn) 1793 { 1794 version (Windows) 1795 { 1796 // TODO: Optimize (don't transform strings) 1797 return cast(uint)fn.enumerateHardLinks.length; 1798 } 1799 else 1800 { 1801 import core.sys.posix.sys.stat; 1802 1803 stat_t s; 1804 errnoEnforce(stat(fn.toStringz(), &s) == 0, "stat"); 1805 return s.st_nlink.to!uint; 1806 } 1807 } 1808 1809 // http://d.puremagic.com/issues/show_bug.cgi?id=7016 1810 version (unittest) 1811 version (Windows) 1812 import ae.sys.windows.misc : getWineVersion; 1813 1814 unittest 1815 { 1816 // FindFirstFileNameW not implemented in Wine 1817 version (Windows) 1818 if (getWineVersion()) 1819 return; 1820 1821 touch("a.test"); 1822 scope(exit) remove("a.test"); 1823 assert("a.test".hardLinkCount() == 1); 1824 1825 hardLink("a.test", "b.test"); 1826 scope(exit) remove("b.test"); 1827 assert("a.test".hardLinkCount() == 2); 1828 assert("b.test".hardLinkCount() == 2); 1829 1830 version(Windows) 1831 { 1832 auto paths = enumerateHardLinks("a.test"); 1833 assert(paths.length == 2); 1834 paths.sort(); 1835 assert(paths[0].endsWith(`\a.test`), paths[0]); 1836 assert(paths[1].endsWith(`\b.test`)); 1837 } 1838 } 1839 1840 static if (is(typeof({ import std.stdio : toFile; }))) 1841 { 1842 static import std.stdio; 1843 alias toFile = std.stdio.toFile; 1844 } 1845 else 1846 { 1847 void toFile(in void[] data, in char[] name) 1848 { 1849 std.file.write(name, data); 1850 } 1851 } 1852 1853 /// Same as toFile, but accepts void[] and does not conflict with the 1854 /// std.stdio function. 1855 void writeTo(in void[] data, in char[] target) 1856 { 1857 std.file.write(target, data); 1858 } 1859 1860 /// Polyfill for Windows fopen implementations with support for UNC 1861 /// paths and the 'x' subspecifier. 1862 File openFile()(string fn, string mode = "rb") 1863 { 1864 File f; 1865 static if (is(typeof(&f.windowsHandleOpen))) 1866 { 1867 import core.sys.windows.windows; 1868 import ae.sys.windows.exception; 1869 1870 string winMode, cMode; 1871 foreach (c; mode) 1872 { 1873 switch (c) 1874 { 1875 case 'r': 1876 case 'w': 1877 case 'a': 1878 case '+': 1879 case 'x': 1880 winMode ~= c; 1881 break; 1882 case 'b': 1883 case 't': 1884 break; 1885 default: 1886 assert(false, "Unknown character in mode"); 1887 } 1888 if (c != 'x') 1889 cMode ~= c; 1890 } 1891 DWORD access, creation; 1892 bool append; 1893 switch (winMode) 1894 { 1895 case "r" : access = GENERIC_READ ; creation = OPEN_EXISTING; break; 1896 case "r+" : access = GENERIC_READ | GENERIC_WRITE; creation = OPEN_EXISTING; break; 1897 case "w" : access = GENERIC_WRITE; creation = CREATE_ALWAYS; break; 1898 case "w+" : access = GENERIC_READ | GENERIC_WRITE; creation = CREATE_ALWAYS; break; 1899 case "a" : access = GENERIC_WRITE; creation = OPEN_ALWAYS ; version (CRuntime_Microsoft) append = true; break; 1900 case "a+" : access = GENERIC_READ | GENERIC_WRITE; creation = OPEN_ALWAYS ; version (CRuntime_Microsoft) assert(false, "MSVCRT can't fdopen with a+"); else break; 1901 case "wx" : access = GENERIC_WRITE; creation = CREATE_NEW ; break; 1902 case "w+x": access = GENERIC_READ | GENERIC_WRITE; creation = CREATE_NEW ; break; 1903 case "ax" : access = GENERIC_WRITE; creation = CREATE_NEW ; version (CRuntime_Microsoft) append = true; break; 1904 case "a+x": access = GENERIC_READ | GENERIC_WRITE; creation = CREATE_NEW ; version (CRuntime_Microsoft) assert(false, "MSVCRT can't fdopen with a+"); else break; 1905 default: assert(false, "Bad file mode: " ~ mode); 1906 } 1907 1908 auto pathW = toUTF16z(longPath(fn)); 1909 auto h = CreateFileW(pathW, access, FILE_SHARE_READ, null, creation, 0, HANDLE.init); 1910 wenforce(h != INVALID_HANDLE_VALUE); 1911 1912 if (append) 1913 h.SetFilePointer(0, null, FILE_END); 1914 1915 f.windowsHandleOpen(h, cMode); 1916 } 1917 else 1918 f.open(fn, mode); 1919 return f; 1920 } 1921 1922 unittest 1923 { 1924 enum Existence { any, mustExist, mustNotExist } 1925 enum Pos { none /* not readable/writable */, start, end, empty } 1926 static struct Behavior 1927 { 1928 Existence existence; 1929 bool truncating; 1930 Pos read, write; 1931 } 1932 1933 void test(string mode, in Behavior expected) 1934 { 1935 static if (isVersion!q{CRuntime_Microsoft} || isVersion!q{OSX}) 1936 if (mode == "a+" || mode == "a+x") 1937 return; 1938 1939 Behavior behavior; 1940 1941 static int counter; 1942 auto fn = text(deleteme, counter++); 1943 1944 collectException(fn.remove()); 1945 bool mustExist = !!collectException(openFile(fn, mode)); 1946 touch(fn); 1947 bool mustNotExist = !!collectException(openFile(fn, mode)); 1948 1949 if (!mustExist) 1950 if (!mustNotExist) 1951 behavior.existence = Existence.any; 1952 else 1953 behavior.existence = Existence.mustNotExist; 1954 else 1955 if (!mustNotExist) 1956 behavior.existence = Existence.mustExist; 1957 else 1958 assert(false, "Can't open file whether it exists or not"); 1959 1960 void create() 1961 { 1962 if (mustNotExist) 1963 collectException(fn.remove()); 1964 else 1965 write(fn, "foo"); 1966 } 1967 1968 create(); 1969 openFile(fn, mode); 1970 behavior.truncating = getSize(fn) == 0; 1971 1972 create(); 1973 { 1974 auto f = openFile(fn, mode); 1975 ubyte[] buf; 1976 if (collectException(f.rawRead(new ubyte[1]), buf)) 1977 { 1978 behavior.read = Pos.none; 1979 // Work around https://issues.dlang.org/show_bug.cgi?id=19751 1980 f.reopen(fn, "w"); 1981 } 1982 else 1983 if (buf.length) 1984 behavior.read = Pos.start; 1985 else 1986 if (f.size) 1987 behavior.read = Pos.end; 1988 else 1989 behavior.read = Pos.empty; 1990 } 1991 1992 create(); 1993 { 1994 string s; 1995 { 1996 auto f = openFile(fn, mode); 1997 if (collectException(f.rawWrite("b"))) 1998 { 1999 s = null; 2000 // Work around https://issues.dlang.org/show_bug.cgi?id=19751 2001 f.reopen(fn, "w"); 2002 } 2003 else 2004 { 2005 f.close(); 2006 s = fn.readText; 2007 } 2008 } 2009 2010 if (s is null) 2011 behavior.write = Pos.none; 2012 else 2013 if (s == "b") 2014 behavior.write = Pos.empty; 2015 else 2016 if (s.endsWith("b")) 2017 behavior.write = Pos.end; 2018 else 2019 if (s.startsWith("b")) 2020 behavior.write = Pos.start; 2021 else 2022 assert(false, "Can't detect write position"); 2023 } 2024 2025 2026 if (behavior != expected) 2027 { 2028 import ae.utils.array : isOneOf; 2029 version (Windows) 2030 if (getWineVersion() && mode.isOneOf("w", "a", "wx", "ax")) 2031 { 2032 // Ignore bug in Wine msvcrt implementation 2033 return; 2034 } 2035 2036 assert(false, text(mode, ": expected ", expected, ", got ", behavior)); 2037 } 2038 } 2039 2040 test("r" , Behavior(Existence.mustExist , false, Pos.start, Pos.none )); 2041 test("r+" , Behavior(Existence.mustExist , false, Pos.start, Pos.start)); 2042 test("w" , Behavior(Existence.any , true , Pos.none , Pos.empty)); 2043 test("w+" , Behavior(Existence.any , true , Pos.empty, Pos.empty)); 2044 test("a" , Behavior(Existence.any , false, Pos.none , Pos.end )); 2045 test("a+" , Behavior(Existence.any , false, Pos.start, Pos.end )); 2046 test("wx" , Behavior(Existence.mustNotExist, true , Pos.none , Pos.empty)); 2047 test("w+x", Behavior(Existence.mustNotExist, true , Pos.empty, Pos.empty)); 2048 test("ax" , Behavior(Existence.mustNotExist, true , Pos.none , Pos.empty)); 2049 test("a+x", Behavior(Existence.mustNotExist, true , Pos.empty, Pos.empty)); 2050 } 2051 2052 private version(Windows) 2053 { 2054 version (CRuntime_Microsoft) 2055 { 2056 alias chsize_size_t = long; 2057 extern(C) int _chsize_s(int fd, chsize_size_t size); 2058 alias chsize = _chsize_s; 2059 } 2060 else 2061 { 2062 import core.stdc.config : c_long; 2063 alias chsize_size_t = c_long; 2064 extern(C) int chsize(int fd, c_long size); 2065 } 2066 } 2067 2068 void truncate(File f, ulong length) 2069 { 2070 f.flush(); 2071 version (Windows) 2072 chsize(f.fileno, length.to!chsize_size_t); 2073 else 2074 ftruncate(f.fileno, length.to!off_t); 2075 } 2076 2077 unittest 2078 { 2079 write("test.txt", "abcde"); 2080 auto f = File("test.txt", "r+b"); 2081 f.write("xyz"); 2082 f.truncate(f.tell); 2083 f.close(); 2084 assert("test.txt".readText == "xyz"); 2085 } 2086 2087 auto fileDigest(Digest)(string fn) 2088 { 2089 import std.range.primitives; 2090 Digest context; 2091 context.start(); 2092 put(context, openFile(fn, "rb").byChunk(64 * 1024)); 2093 auto digest = context.finish(); 2094 return digest; 2095 } 2096 2097 template mdFile() 2098 { 2099 import std.digest.md; 2100 alias mdFile = fileDigest!MD5; 2101 } 2102 2103 version (HAVE_WIN32) 2104 unittest 2105 { 2106 import std.digest : toHexString; 2107 write("test.txt", "Hello, world!"); 2108 scope(exit) remove("test.txt"); 2109 assert(mdFile("test.txt").toHexString() == "6CD3556DEB0DA54BCA060B4C39479839"); 2110 } 2111 2112 auto fileDigestCached(Digest)(string fn) 2113 { 2114 static typeof(Digest.init.finish())[ulong] cache; 2115 auto id = getFileID(fn); 2116 auto phash = id in cache; 2117 if (phash) 2118 return *phash; 2119 return cache[id] = fileDigest!Digest(fn); 2120 } 2121 2122 template mdFileCached() 2123 { 2124 import std.digest.md; 2125 alias mdFileCached = fileDigestCached!MD5; 2126 } 2127 2128 version (HAVE_WIN32) 2129 unittest 2130 { 2131 import std.digest : toHexString; 2132 write("test.txt", "Hello, world!"); 2133 scope(exit) remove("test.txt"); 2134 assert(mdFileCached("test.txt").toHexString() == "6CD3556DEB0DA54BCA060B4C39479839"); 2135 write("test.txt", "Something else"); 2136 assert(mdFileCached("test.txt").toHexString() == "6CD3556DEB0DA54BCA060B4C39479839"); 2137 } 2138 2139 /// Read a File (which might be a stream) into an array 2140 void[] readFile(File f) 2141 { 2142 import std.range.primitives; 2143 auto result = appender!(ubyte[]); 2144 put(result, f.byChunk(64*1024)); 2145 return result.data; 2146 } 2147 2148 unittest 2149 { 2150 auto s = "0123456789".replicate(10_000); 2151 write("test.txt", s); 2152 scope(exit) remove("test.txt"); 2153 assert(readFile(File("test.txt")) == s); 2154 } 2155 2156 /// Read exactly `buf.length` bytes and return true. 2157 /// On EOF, return false. 2158 bool readExactly(ref File f, ubyte[] buf) 2159 { 2160 if (!buf.length) 2161 return true; 2162 auto read = f.rawRead(buf); 2163 if (read.length==0) return false; 2164 enforce(read.length == buf.length, "Unexpected end of file"); 2165 return true; 2166 } 2167 2168 private 2169 version (Windows) 2170 { 2171 version (CRuntime_DigitalMars) 2172 extern(C) sizediff_t read(int, void*, size_t); 2173 else 2174 { 2175 extern(C) sizediff_t _read(int, void*, size_t); 2176 alias read = _read; 2177 } 2178 } 2179 else 2180 import core.sys.posix.unistd : read; 2181 2182 /// Like `File.rawRead`, but returns as soon as any data is available. 2183 void[] readPartial(File f, void[] buf) 2184 { 2185 assert(buf.length); 2186 auto numRead = read(f.fileno, buf.ptr, buf.length); 2187 errnoEnforce(numRead >= 0); 2188 return buf[0 .. numRead]; 2189 } 2190 2191 /// Like std.file.readText for non-UTF8 2192 ascii readAscii()(string fileName) 2193 { 2194 return cast(ascii)readFile(openFile(fileName, "rb")); 2195 } 2196 2197 // http://d.puremagic.com/issues/show_bug.cgi?id=7016 2198 version(Posix) static import ae.sys.signals; 2199 2200 /// Start a thread which writes data to f asynchronously. 2201 Thread writeFileAsync(File f, in void[] data) 2202 { 2203 static class Writer : Thread 2204 { 2205 File target; 2206 const void[] data; 2207 2208 this(ref File f, in void[] data) 2209 { 2210 this.target = f; 2211 this.data = data; 2212 super(&run); 2213 } 2214 2215 void run() 2216 { 2217 version (Posix) 2218 { 2219 import ae.sys.signals; 2220 collectSignal(SIGPIPE, &write); 2221 } 2222 else 2223 write(); 2224 } 2225 2226 void write() 2227 { 2228 target.rawWrite(data); 2229 target.close(); 2230 } 2231 } 2232 2233 auto t = new Writer(f, data); 2234 t.start(); 2235 return t; 2236 } 2237 2238 /// Write data to a file, and ensure it gets written to disk 2239 /// before this function returns. 2240 /// Consider using as atomic!syncWrite. 2241 /// See also: syncUpdate 2242 void syncWrite()(string target, in void[] data) 2243 { 2244 auto f = File(target, "wb"); 2245 f.rawWrite(data); 2246 version (Windows) 2247 { 2248 mixin(importWin32!q{windows}); 2249 FlushFileBuffers(f.windowsHandle); 2250 } 2251 else 2252 { 2253 import core.sys.posix.unistd; 2254 fsync(f.fileno); 2255 } 2256 f.close(); 2257 } 2258 2259 /// Atomically save data to a file (if the file doesn't exist, 2260 /// or its contents differs). The update operation as a whole 2261 /// is not atomic, only the write is. 2262 void syncUpdate()(string fn, in void[] data) 2263 { 2264 if (!fn.exists || fn.read() != data) 2265 atomic!(syncWrite!())(fn, data); 2266 } 2267 2268 version(Windows) import ae.sys.windows.exception; 2269 2270 struct NamedPipeImpl 2271 { 2272 immutable string fileName; 2273 2274 /// Create a named pipe, and reserve a filename. 2275 this()(string name) 2276 { 2277 version(Windows) 2278 { 2279 mixin(importWin32!q{winbase}); 2280 2281 fileName = `\\.\pipe\` ~ name; 2282 auto h = CreateNamedPipeW(fileName.toUTF16z, PIPE_ACCESS_OUTBOUND, PIPE_TYPE_BYTE, 10, 4096, 4096, 0, null).wenforce("CreateNamedPipeW"); 2283 f.windowsHandleOpen(h, "wb"); 2284 } 2285 else 2286 { 2287 import core.sys.posix.sys.stat; 2288 2289 fileName = `/tmp/` ~ name ~ `.fifo`; 2290 mkfifo(fileName.toStringz, S_IWUSR | S_IRUSR); 2291 } 2292 } 2293 2294 /// Wait for a peer to open the other end of the pipe. 2295 File connect()() 2296 { 2297 version(Windows) 2298 { 2299 mixin(importWin32!q{winbase}); 2300 mixin(importWin32!q{windef}); 2301 2302 BOOL bSuccess = ConnectNamedPipe(f.windowsHandle, null); 2303 2304 // "If a client connects before the function is called, the function returns zero 2305 // and GetLastError returns ERROR_PIPE_CONNECTED. This can happen if a client 2306 // connects in the interval between the call to CreateNamedPipe and the call to 2307 // ConnectNamedPipe. In this situation, there is a good connection between client 2308 // and server, even though the function returns zero." 2309 if (!bSuccess) 2310 wenforce(GetLastError() == ERROR_PIPE_CONNECTED, "ConnectNamedPipe"); 2311 2312 return f; 2313 } 2314 else 2315 { 2316 return File(fileName, "w"); 2317 } 2318 } 2319 2320 ~this() 2321 { 2322 version(Windows) 2323 { 2324 // File.~this will take care of cleanup 2325 } 2326 else 2327 fileName.remove(); 2328 } 2329 2330 private: 2331 File f; 2332 } 2333 alias NamedPipe = RefCounted!NamedPipeImpl; 2334 2335 import ae.utils.textout : StringBuilder; 2336 2337 /// Avoid std.stdio.File.readln's memory corruption bug 2338 /// https://issues.dlang.org/show_bug.cgi?id=13856 2339 string safeReadln(File f) 2340 { 2341 StringBuilder buf; 2342 char[1] arr; 2343 while (true) 2344 { 2345 auto result = f.rawRead(arr[]); 2346 if (!result.length) 2347 break; 2348 buf.put(result); 2349 if (result[0] == '\x0A') 2350 break; 2351 } 2352 return buf.get(); 2353 } 2354 2355 // **************************************************************************** 2356 2357 /// Change the current directory to the given directory. Does nothing if dir is null. 2358 /// Return a scope guard which, upon destruction, restores the previous directory. 2359 /// Asserts that only one thread has changed the process's current directory at any time. 2360 auto pushd(string dir) 2361 { 2362 import core.atomic; 2363 2364 static int threadCount = 0; 2365 static shared int processCount = 0; 2366 2367 static struct Popd 2368 { 2369 string oldPath; 2370 this(string cwd) { oldPath = cwd; } 2371 ~this() { if (oldPath) pop(); } 2372 @disable this(); 2373 @disable this(this); 2374 2375 void pop() 2376 { 2377 assert(oldPath); 2378 scope(exit) oldPath = null; 2379 chdir(oldPath); 2380 2381 auto newThreadCount = --threadCount; 2382 auto newProcessCount = atomicOp!"-="(processCount, 1); 2383 assert(newThreadCount == newProcessCount); // Shouldn't happen 2384 } 2385 } 2386 2387 string cwd; 2388 if (dir) 2389 { 2390 auto newThreadCount = ++threadCount; 2391 auto newProcessCount = atomicOp!"+="(processCount, 1); 2392 assert(newThreadCount == newProcessCount, "Another thread already has an active pushd"); 2393 2394 cwd = getcwd(); 2395 chdir(dir); 2396 } 2397 return Popd(cwd); 2398 } 2399 2400 // **************************************************************************** 2401 2402 import std.algorithm; 2403 import std.process : thisProcessID; 2404 import std.traits; 2405 import std.typetuple; 2406 import ae.utils.meta; 2407 2408 enum targetParameterNames = "target/to/name/dst"; 2409 2410 /// Wrap an operation which creates a file or directory, 2411 /// so that it is created safely and, for files, atomically 2412 /// (by performing the underlying operation to a temporary 2413 /// location, then renaming the completed file/directory to 2414 /// the actual target location). targetName specifies the name 2415 /// of the parameter containing the target file/directory. 2416 auto atomic(alias impl, string targetName = targetParameterNames)(staticMap!(Unqual, ParameterTypeTuple!impl) args) 2417 { 2418 enum targetIndex = findParameter([ParameterIdentifierTuple!impl], targetName, __traits(identifier, impl)); 2419 return atomic!(impl, targetIndex)(args); 2420 } 2421 2422 /// ditto 2423 auto atomic(alias impl, size_t targetIndex)(staticMap!(Unqual, ParameterTypeTuple!impl) args) 2424 { 2425 // idup for https://d.puremagic.com/issues/show_bug.cgi?id=12503 2426 auto target = args[targetIndex].idup; 2427 auto temp = "%s.%s.%s.temp".format(target, thisProcessID, getCurrentThreadID); 2428 if (temp.exists) temp.removeRecurse(); 2429 scope(success) rename(temp, target); 2430 scope(failure) if (temp.exists) temp.removeRecurse(); 2431 args[targetIndex] = temp; 2432 return impl(args); 2433 } 2434 2435 /// ditto 2436 // Workaround for https://d.puremagic.com/issues/show_bug.cgi?id=12230 2437 // Can't be an overload because of https://issues.dlang.org/show_bug.cgi?id=13374 2438 //R atomicDg(string targetName = "target", R, Args...)(R delegate(Args) impl, staticMap!(Unqual, Args) args) 2439 auto atomicDg(size_t targetIndexA = size_t.max, Impl, Args...)(Impl impl, Args args) 2440 { 2441 enum targetIndex = targetIndexA == size_t.max ? ParameterTypeTuple!impl.length-1 : targetIndexA; 2442 return atomic!(impl, targetIndex)(args); 2443 } 2444 2445 deprecated alias safeUpdate = atomic; 2446 2447 unittest 2448 { 2449 enum fn = "atomic.tmp"; 2450 scope(exit) if (fn.exists) fn.remove(); 2451 2452 atomic!touch(fn); 2453 assert(fn.exists); 2454 fn.remove(); 2455 2456 atomicDg(&touch, fn); 2457 assert(fn.exists); 2458 } 2459 2460 /// Wrap an operation so that it is skipped entirely 2461 /// if the target already exists. Implies atomic. 2462 auto cached(alias impl, string targetName = targetParameterNames)(ParameterTypeTuple!impl args) 2463 { 2464 enum targetIndex = findParameter([ParameterIdentifierTuple!impl], targetName, __traits(identifier, impl)); 2465 auto target = args[targetIndex]; 2466 if (!target.exists) 2467 atomic!(impl, targetIndex)(args); 2468 return target; 2469 } 2470 2471 /// ditto 2472 // Exists due to the same reasons as atomicDg 2473 auto cachedDg(size_t targetIndexA = size_t.max, Impl, Args...)(Impl impl, Args args) 2474 { 2475 enum targetIndex = targetIndexA == size_t.max ? ParameterTypeTuple!impl.length-1 : targetIndexA; 2476 auto target = args[targetIndex]; 2477 if (!target.exists) 2478 atomic!(impl, targetIndex)(args); 2479 return target; 2480 } 2481 2482 deprecated alias obtainUsing = cached; 2483 2484 /// Create a file, or replace an existing file's contents 2485 /// atomically. 2486 /// Note: Consider using atomic!syncWrite or 2487 /// atomic!syncUpdate instead. 2488 alias atomic!writeProxy atomicWrite; 2489 deprecated alias safeWrite = atomicWrite; 2490 /*private*/ void writeProxy(string target, in void[] data) 2491 { 2492 std.file.write(target, data); 2493 } 2494 2495 // Work around for https://github.com/D-Programming-Language/phobos/pull/2784#issuecomment-68117241 2496 private void copy2(string source, string target) { std.file.copy(source, target); } 2497 2498 /// Copy a file, or replace an existing file's contents 2499 /// with another file's, atomically. 2500 alias atomic!copy2 atomicCopy; 2501 2502 unittest 2503 { 2504 enum fn = "cached.tmp"; 2505 scope(exit) if (fn.exists) fn.remove(); 2506 2507 cached!touch(fn); 2508 assert(fn.exists); 2509 2510 std.file.write(fn, "test"); 2511 2512 cachedDg!0(&writeProxy, fn, "test2"); 2513 assert(fn.readText() == "test"); 2514 } 2515 2516 // **************************************************************************** 2517 2518 template withTarget(alias targetGen, alias fun) 2519 { 2520 auto withTarget(Args...)(auto ref Args args) 2521 { 2522 auto target = targetGen(args); 2523 fun(args, target); 2524 return target; 2525 } 2526 } 2527 2528 /// Two-argument buildPath with reversed arguments. 2529 /// Useful for UFCS chaining. 2530 string prependPath(string target, string path) 2531 { 2532 return buildPath(path, target); 2533 }