1 /** 2 * File stuff 3 * 4 * License: 5 * This Source Code Form is subject to the terms of 6 * the Mozilla Public License, v. 2.0. If a copy of 7 * the MPL was not distributed with this file, You 8 * can obtain one at http://mozilla.org/MPL/2.0/. 9 * 10 * Authors: 11 * Vladimir Panteleev <ae@cy.md> 12 */ 13 14 module ae.sys.file; 15 16 import core.stdc.wchar_; 17 import core.thread; 18 19 import std.array; 20 import std.conv; 21 import std.file; 22 import std.path; 23 import std.range.primitives; 24 import std.stdio : File; 25 import std..string; 26 import std.typecons; 27 import std.utf; 28 29 import ae.sys.cmd : getCurrentThreadID; 30 import ae.utils.path; 31 32 public import std.typecons : No, Yes; 33 34 alias wcscmp = core.stdc.wchar_.wcscmp; 35 alias wcslen = core.stdc.wchar_.wcslen; 36 37 version(Windows) import ae.sys.windows.imports; 38 39 // ************************************************************************ 40 41 version (Windows) 42 { 43 // Work around std.file overload 44 mixin(importWin32!(q{winnt}, null, q{FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT})); 45 } 46 version (Posix) 47 { 48 private import core.stdc.errno; 49 private import core.sys.posix.dirent; 50 private import core.stdc..string; 51 } 52 53 // ************************************************************************ 54 55 deprecated string[] fastListDir(bool recursive = false, bool symlinks=false)(string pathname, string pattern = null) 56 { 57 string[] result; 58 59 listDir!((e) { 60 static if (!symlinks) 61 { 62 // Note: shouldn't this just skip recursion? 63 if (e.isSymlink) 64 return; 65 } 66 67 if (pattern && !globMatch(e.baseName, pattern)) 68 return; 69 70 static if (recursive) 71 { 72 if (e.entryIsDir) 73 { 74 // Note: why exclude directories from results? 75 e.recurse(); 76 return; 77 } 78 } 79 80 result ~= e.fullName; 81 })(pathname); 82 return result; 83 } 84 85 // ************************************************************************ 86 87 version (Windows) 88 { 89 mixin(importWin32!(q{winnt}, null, q{WCHAR})); 90 mixin(importWin32!(q{winbase}, null, q{WIN32_FIND_DATAW})); 91 } 92 93 /// The OS's "native" filesystem character type (private in Phobos). 94 version (Windows) 95 alias FSChar = WCHAR; 96 else version (Posix) 97 alias FSChar = char; 98 else 99 static assert(0); 100 101 /// Reads a time field from a stat_t with full precision (private in Phobos). 102 SysTime statTimeToStdTime(string which)(ref const stat_t statbuf) 103 { 104 auto unixTime = mixin(`statbuf.st_` ~ which ~ `time`); 105 auto stdTime = unixTimeToStdTime(unixTime); 106 107 static if (is(typeof(mixin(`statbuf.st_` ~ which ~ `tim`)))) 108 stdTime += mixin(`statbuf.st_` ~ which ~ `tim.tv_nsec`) / 100; 109 else 110 static if (is(typeof(mixin(`statbuf.st_` ~ which ~ `timensec`)))) 111 stdTime += mixin(`statbuf.st_` ~ which ~ `timensec`) / 100; 112 else 113 static if (is(typeof(mixin(`statbuf.st_` ~ which ~ `time_nsec`)))) 114 stdTime += mixin(`statbuf.st_` ~ which ~ `time_nsec`) / 100; 115 else 116 static if (is(typeof(mixin(`statbuf.__st_` ~ which ~ `timensec`)))) 117 stdTime += mixin(`statbuf.__st_` ~ which ~ `timensec`) / 100; 118 119 return SysTime(stdTime); 120 } 121 122 version (OSX) 123 version = Darwin; 124 else version (iOS) 125 version = Darwin; 126 else version (TVOS) 127 version = Darwin; 128 else version (WatchOS) 129 version = Darwin; 130 131 private 132 version (Posix) 133 { 134 // TODO: upstream into Druntime 135 extern (C) 136 { 137 int dirfd(DIR *dirp) pure nothrow @nogc; 138 int openat(int fd, const char *path, int oflag, ...) nothrow @nogc; 139 140 version (Darwin) 141 { 142 pragma(mangle, "fstatat$INODE64") 143 int fstatat(int fd, const char *path, stat_t *buf, int flag) nothrow @nogc; 144 145 pragma(mangle, "fdopendir$INODE64") 146 DIR *fdopendir(int fd) nothrow @nogc; 147 } 148 else 149 { 150 int fstatat(int fd, const(char)* path, stat_t* buf, int flag) nothrow @nogc; 151 DIR *fdopendir(int fd) nothrow @nogc; 152 } 153 } 154 version (linux) 155 { 156 enum AT_SYMLINK_NOFOLLOW = 0x100; 157 enum O_DIRECTORY = 0x10000; 158 } 159 version (Darwin) 160 { 161 enum AT_SYMLINK_NOFOLLOW = 0x20; 162 enum O_DIRECTORY = 0x100000; 163 } 164 version (FreeBSD) 165 { 166 enum AT_SYMLINK_NOFOLLOW = 0x200; 167 enum O_DIRECTORY = 0x20000; 168 } 169 } 170 171 import ae.utils.range : nullTerminated; 172 173 // http://d.puremagic.com/issues/show_bug.cgi?id=7016 174 version (Windows) static import ae.sys.windows.misc; 175 176 /** 177 Fast templated directory iterator 178 179 Example: 180 --- 181 string[] entries; 182 listDir!((e) { 183 entries ~= e.fullName.relPath(tmpDir); 184 if (e.entryIsDir) 185 e.recurse(); 186 })(tmpDir); 187 --- 188 */ 189 template listDir(alias handler) 190 { 191 /*non-static*/ struct Context 192 { 193 // Tether to handler alias context 194 void callHandler(Entry* e) { handler(e); } 195 196 bool timeToStop = false; 197 198 FSChar[] pathBuf; 199 } 200 201 /// A pointer to this type will be passed to the `listDir` predicate. 202 static struct Entry 203 { 204 version (Posix) 205 { 206 dirent* ent; /// POSIX `dirent`. 207 208 stat_t[enumLength!StatTarget] statBuf; 209 /// Result of `stat` call. 210 /// Other values are the same as `errno`. 211 enum StatResult : int 212 { 213 noInfo = 0, /// Not called yet. 214 statOK = int.max, /// All OK 215 unknownError = int.min, /// `errno` returned 0 or `int.max` 216 } 217 218 int dirFD; /// POSIX directory file descriptor. 219 } 220 version (Windows) 221 { 222 WIN32_FIND_DATAW findData; /// Windows `WIN32_FIND_DATAW`. 223 } 224 225 // Cached values. 226 // Cleared (memset to 0) for every directory entry. 227 struct Data 228 { 229 FSChar[] baseNameFS; 230 string baseName; 231 string fullName; 232 size_t pathTailPos; 233 234 version (Posix) 235 { 236 StatResult[enumLength!StatTarget] statResult; 237 } 238 } 239 Data data; 240 241 // Recursion 242 243 Entry* parent; /// 244 Context* context; 245 246 /// Request recursion on the current `entry`. 247 version (Posix) 248 { 249 void recurse() 250 { 251 import core.sys.posix.fcntl; 252 int flags = O_RDONLY; 253 static if (is(typeof(O_DIRECTORY))) 254 flags |= O_DIRECTORY; 255 auto fd = openat(dirFD, this.ent.d_name.ptr, flags); 256 errnoEnforce(fd >= 0, 257 "Failed to open %s as subdirectory of directory %s" 258 .format(this.baseNameFS, this.parent.fullName)); 259 auto subdir = fdopendir(fd); 260 errnoEnforce(subdir, 261 "Failed to open subdirectory %s of directory %s as directory" 262 .format(this.baseNameFS, this.parent.fullName)); 263 scan(subdir, fd, &this); 264 } 265 } 266 else 267 version (Windows) 268 { 269 void recurse() 270 { 271 needFullPath(); 272 appendString(context.pathBuf, 273 data.pathTailPos, "\\*.*\0"w); 274 scan(&this); 275 } 276 } 277 278 /// Stop iteration. 279 void stop() { context.timeToStop = true; } 280 281 // Name 282 283 /// Returns a pointer to the base file name, as a 284 /// null-terminated string, in the operating system's 285 /// character type. Fastest. 286 const(FSChar)* baseNameFSPtr() pure nothrow @nogc 287 { 288 version (Posix) return ent.d_name.ptr; 289 version (Windows) return findData.cFileName.ptr; 290 } 291 292 // Bounded variant of std.string.fromStringz for static arrays. 293 private static T[] fromStringz(T, size_t n)(ref T[n] buf) 294 { 295 foreach (i, c; buf) 296 if (!c) 297 return buf[0 .. i]; 298 // This should only happen in case of an OS / libc bug. 299 assert(false, "File name buffer is not null-terminated"); 300 } 301 302 /// Returns the base file name, as a D character array, in the 303 /// operating system's character type. Fast. 304 const(FSChar)[] baseNameFS() pure nothrow @nogc 305 { 306 if (!data.baseNameFS) 307 { 308 version (Posix) data.baseNameFS = fromStringz(ent.d_name); 309 version (Windows) data.baseNameFS = fromStringz(findData.cFileName); 310 } 311 return data.baseNameFS; 312 } 313 314 /// Returns the base file name as a D string. Allocates. 315 string baseName() // allocates 316 { 317 if (!data.baseName) 318 data.baseName = baseNameFS.to!string; 319 return data.baseName; 320 } 321 322 private void needFullPath() nothrow @nogc 323 { 324 if (!data.pathTailPos) 325 { 326 version (Posix) 327 parent.needFullPath(); 328 version (Windows) 329 { 330 // directory separator was added during recursion 331 auto startPos = parent.data.pathTailPos + 1; 332 } 333 version (Posix) 334 { 335 immutable FSChar[] separator = "/"; 336 auto startPos = appendString(context.pathBuf, 337 parent.data.pathTailPos, separator); 338 } 339 data.pathTailPos = appendString(context.pathBuf, 340 startPos, 341 baseNameFSPtr.nullTerminated 342 ); 343 } 344 } 345 346 /// Returns the full file name, as a D character array, in the 347 /// operating system's character type. Fast. 348 const(FSChar)[] fullNameFS() nothrow @nogc // fast 349 { 350 needFullPath(); 351 return context.pathBuf[0 .. data.pathTailPos]; 352 } 353 354 /// Returns the full file name as a D string. Allocates. 355 string fullName() // allocates 356 { 357 if (!data.fullName) 358 data.fullName = fullNameFS.to!string; 359 return data.fullName; 360 } 361 362 // Attributes 363 364 version (Posix) 365 { 366 /// We can stat two different things on POSIX: the directory entry itself, 367 /// or the link target (if the directory entry is a symbolic link). 368 enum StatTarget 369 { 370 dirEntry, /// do not dereference (lstat) 371 linkTarget, /// dereference 372 } 373 374 private bool tryStat(StatTarget target)() nothrow @nogc 375 { 376 if (data.statResult[target] == StatResult.noInfo) 377 { 378 // If we already did the other kind of stat, can we reuse its result? 379 if (data.statResult[1 - target] != StatResult.noInfo) 380 { 381 // Yes, if we know this isn't a link from the directory entry. 382 static if (__traits(compiles, ent.d_type)) 383 if (ent.d_type != DT_UNKNOWN && ent.d_type != DT_LNK) 384 goto reuse; 385 // Yes, if we already found out this isn't a link from an lstat call. 386 static if (target == StatTarget.linkTarget) 387 if (data.statResult[StatTarget.dirEntry] == StatResult.statOK 388 && (statBuf[StatTarget.dirEntry].st_mode & S_IFMT) != S_IFLNK) 389 goto reuse; 390 } 391 392 if (false) 393 { 394 reuse: 395 statBuf[target] = statBuf[1 - target]; 396 data.statResult[target] = data.statResult[1 - target]; 397 } 398 else 399 { 400 int flags = target == StatTarget.dirEntry ? AT_SYMLINK_NOFOLLOW : 0; 401 auto res = fstatat(dirFD, ent.d_name.ptr, &statBuf[target], flags); 402 if (res) 403 { 404 auto error = errno; 405 data.statResult[target] = cast(StatResult)error; 406 if (error == StatResult.noInfo || error == StatResult.statOK) 407 data.statResult[target] = StatResult.unknownError; // unknown error? 408 } 409 else 410 data.statResult[target] = StatResult.statOK; // no error 411 } 412 } 413 return data.statResult[target] == StatResult.statOK; 414 } 415 416 ErrnoException statError(StatTarget target)() 417 { 418 errno = data.statResult[target]; 419 return new ErrnoException("Failed to stat " ~ 420 (target == StatTarget.linkTarget ? "link target" : "directory entry") ~ 421 ": " ~ fullName); 422 } 423 424 /// Return the result of `stat` / `lstat` (depending on `target`) 425 /// for this `Entry`, performing it first if necessary. 426 stat_t* needStat(StatTarget target)() 427 { 428 if (!tryStat!target) 429 throw statError!target(); 430 return &statBuf[target]; 431 } 432 433 // Check if this is an object of the given type. 434 private bool deIsType(typeof(DT_REG) dType, typeof(S_IFREG) statType) 435 { 436 static if (__traits(compiles, ent.d_type)) 437 if (ent.d_type != DT_UNKNOWN) 438 return ent.d_type == dType; 439 440 return (needStat!(StatTarget.dirEntry)().st_mode & S_IFMT) == statType; 441 } 442 443 /// Returns true if this is a symlink. 444 @property bool isSymlink() 445 { 446 return deIsType(DT_LNK, S_IFLNK); 447 } 448 449 /// Returns true if this is a directory. 450 /// You probably want to use this one to decide whether to recurse. 451 @property bool entryIsDir() 452 { 453 return deIsType(DT_DIR, S_IFDIR); 454 } 455 456 // Check if this is an object of the given type, or a link pointing to one. 457 private bool ltIsType(typeof(DT_REG) dType, typeof(S_IFREG) statType) 458 { 459 static if (__traits(compiles, ent.d_type)) 460 if (ent.d_type != DT_UNKNOWN && ent.d_type != DT_LNK) 461 return ent.d_type == dType; 462 463 if (tryStat!(StatTarget.linkTarget)()) 464 return (statBuf[StatTarget.linkTarget].st_mode & S_IFMT) == statType; 465 466 if (isSymlink()) // broken symlink? 467 return false; // a broken symlink does not point at anything. 468 469 throw statError!(StatTarget.linkTarget)(); 470 } 471 472 /// Returns true if this is a file, or a link pointing to one. 473 @property bool isFile() 474 { 475 return ltIsType(DT_REG, S_IFREG); 476 } 477 478 /// Returns true if this is a directory, or a link pointing to one. 479 @property bool isDir() 480 { 481 return ltIsType(DT_DIR, S_IFDIR); 482 } 483 484 /// Returns the raw POSIX attributes of this directory entry. 485 @property uint attributes() 486 { 487 return needStat!(StatTarget.linkTarget)().st_mode; 488 } 489 490 /// Returns the raw POSIX attributes of this directory entry, 491 /// or the link target if this directory entry is a symlink. 492 @property uint linkAttributes() 493 { 494 return needStat!(StatTarget.dirEntry)().st_mode; 495 } 496 497 // Other attributes 498 499 /// Returns the "c" time of this directory entry, 500 /// or the link target if this directory entry is a symlink. 501 @property SysTime timeStatusChanged() 502 { 503 return statTimeToStdTime!"c"(*needStat!(StatTarget.linkTarget)()); 504 } 505 506 /// Returns the "a" time of this directory entry, 507 /// or the link target if this directory entry is a symlink. 508 @property SysTime timeLastAccessed() 509 { 510 return statTimeToStdTime!"a"(*needStat!(StatTarget.linkTarget)()); 511 } 512 513 /// Returns the "m" time of this directory entry, 514 /// or the link target if this directory entry is a symlink. 515 @property SysTime timeLastModified() 516 { 517 return statTimeToStdTime!"m"(*needStat!(StatTarget.linkTarget)()); 518 } 519 520 /// Returns the "birth" time of this directory entry, 521 /// or the link target if this directory entry is a symlink. 522 static if (is(typeof(&statTimeToStdTime!"birth"))) 523 @property SysTime timeCreated() 524 { 525 return statTimeToStdTime!"m"(*needStat!(StatTarget.linkTarget)()); 526 } 527 528 /// Returns the size in bytes of this directory entry, 529 /// or the link target if this directory entry is a symlink. 530 @property ulong size() 531 { 532 return needStat!(StatTarget.linkTarget)().st_size; 533 } 534 535 /// Returns the inode number of this directory entry, 536 /// or the link target if this directory entry is a symlink. 537 @property ulong fileID() 538 { 539 static if (__traits(compiles, ent.d_ino)) 540 return ent.d_ino; 541 else 542 return needStat!(StatTarget.linkTarget)().st_ino; 543 } 544 } 545 546 version (Windows) 547 { 548 /// Returns true if this is a directory, or a reparse point pointing to one. 549 @property bool isDir() const pure nothrow 550 { 551 return (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; 552 } 553 554 /// Returns true if this is a file, or a reparse point pointing to one. 555 @property bool isFile() const pure nothrow 556 { 557 return !isDir; 558 } 559 560 /// Returns true if this is a reparse point. 561 @property bool isSymlink() const pure nothrow 562 { 563 return (findData.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0; 564 } 565 566 /// Returns true if this is a directory. 567 /// You probably want to use this one to decide whether to recurse. 568 @property bool entryIsDir() const pure nothrow 569 { 570 return isDir && !isSymlink; 571 } 572 573 /// Returns the size in bytes of this directory entry. 574 @property ulong size() const pure nothrow 575 { 576 return makeUlong(findData.nFileSizeLow, findData.nFileSizeHigh); 577 } 578 579 /// Returns the creation time of this directory entry. 580 @property SysTime timeCreated() const 581 { 582 return FILETIMEToSysTime(&findData.ftCreationTime); 583 } 584 585 /// Returns the last access time of this directory entry. 586 @property SysTime timeLastAccessed() const 587 { 588 return FILETIMEToSysTime(&findData.ftLastAccessTime); 589 } 590 591 /// Returns the last modification time of this directory entry. 592 @property SysTime timeLastModified() const 593 { 594 return FILETIMEToSysTime(&findData.ftLastWriteTime); 595 } 596 597 /// Returns the 64-bit unique file index of this file. 598 @property ulong fileID() 599 { 600 return getFileID(fullName); 601 } 602 } 603 } 604 605 version (Posix) 606 { 607 // The length of the buffer on the stack. 608 enum initialPathBufLength = 256; 609 610 static void scan(DIR* dir, int dirFD, Entry* parentEntry) 611 { 612 Entry entry = void; 613 entry.parent = parentEntry; 614 entry.context = entry.parent.context; 615 entry.dirFD = dirFD; 616 617 scope(exit) closedir(dir); 618 619 dirent* ent; 620 while ((ent = readdir(dir)) != null) 621 { 622 // Apparently happens on some OS X versions. 623 enforce(ent.d_name[0], 624 "Empty dir entry name (OS bug?)"); 625 626 // Skip "." and ".." 627 if (ent.d_name[0] == '.' && ( 628 ent.d_name[1] == 0 || 629 (ent.d_name[1] == '.' && ent.d_name[2] == 0))) 630 continue; 631 632 entry.ent = ent; 633 entry.data = Entry.Data.init; 634 entry.context.callHandler(&entry); 635 if (entry.context.timeToStop) 636 break; 637 } 638 } 639 } 640 641 enum isPath(Path) = (isForwardRange!Path || isSomeString!Path) && 642 isSomeChar!(ElementEncodingType!Path); 643 644 import core.stdc.stdlib : malloc, realloc, free; 645 646 static FSChar[] reallocPathBuf(FSChar[] buf, size_t newLength) nothrow @nogc 647 { 648 if (buf.length == initialPathBufLength) // current buffer is on stack 649 { 650 auto ptr = cast(FSChar*) malloc(newLength * FSChar.sizeof); 651 ptr[0 .. buf.length] = buf[]; 652 return ptr[0 .. newLength]; 653 } 654 else // current buffer on C heap (malloc'd above) 655 { 656 auto ptr = cast(FSChar*) realloc(buf.ptr, newLength * FSChar.sizeof); 657 return ptr[0 .. newLength]; 658 } 659 } 660 661 // Append a string to the buffer, reallocating as necessary. 662 // Returns the new length of the string in the buffer. 663 static size_t appendString(Str)(ref FSChar[] buf, size_t pos, Str str) nothrow @nogc 664 if (isPath!Str) 665 { 666 static if (ElementEncodingType!Str.sizeof == FSChar.sizeof 667 && is(typeof(str.length))) 668 { 669 // No transcoding needed and length known 670 auto remainingSpace = buf.length - pos; 671 if (str.length > remainingSpace) 672 buf = reallocPathBuf(buf, (pos + str.length) * 3 / 2); 673 buf[pos .. pos + str.length] = str[]; 674 pos += str.length; 675 } 676 else 677 { 678 // Need to transcode 679 auto p = buf.ptr + pos; 680 auto bufEnd = buf.ptr + buf.length; 681 foreach (c; byUTF!FSChar(str)) 682 { 683 if (p == bufEnd) // out of room 684 { 685 auto newBuf = reallocPathBuf(buf, buf.length * 3 / 2); 686 687 // Update pointers to point into the new buffer. 688 p = newBuf.ptr + (p - buf.ptr); 689 buf = newBuf; 690 bufEnd = buf.ptr + buf.length; 691 } 692 *p++ = c; 693 } 694 pos = p - buf.ptr; 695 } 696 return pos; 697 } 698 699 version (Windows) 700 { 701 mixin(importWin32!(q{winbase})); 702 import ae.sys.windows.misc : makeUlong; 703 704 // The length of the buffer on the stack. 705 enum initialPathBufLength = MAX_PATH; 706 707 enum FIND_FIRST_EX_LARGE_FETCH = 2; 708 enum FindExInfoBasic = cast(FINDEX_INFO_LEVELS)1; 709 710 static void scan(Entry* parentEntry) 711 { 712 Entry entry = void; 713 entry.parent = parentEntry; 714 entry.context = parentEntry.context; 715 716 HANDLE hFind = FindFirstFileExW( 717 entry.context.pathBuf.ptr, 718 FindExInfoBasic, 719 &entry.findData, 720 FINDEX_SEARCH_OPS.FindExSearchNameMatch, 721 null, 722 FIND_FIRST_EX_LARGE_FETCH, // https://blogs.msdn.microsoft.com/oldnewthing/20131024-00/?p=2843 723 ); 724 if (hFind == INVALID_HANDLE_VALUE) 725 throw new WindowsException(GetLastError(), 726 text("FindFirstFileW: ", parentEntry.fullNameFS)); 727 scope(exit) FindClose(hFind); 728 do 729 { 730 // Skip "." and ".." 731 auto fn = entry.findData.cFileName.ptr; 732 if (fn[0] == '.' && ( 733 fn[1] == 0 || 734 (fn[1] == '.' && fn[2] == 0))) 735 continue; 736 737 entry.data = Entry.Data.init; 738 entry.context.callHandler(&entry); 739 if (entry.context.timeToStop) 740 break; 741 } 742 while (FindNextFileW(hFind, &entry.findData)); 743 if (GetLastError() != ERROR_NO_MORE_FILES) 744 throw new WindowsException(GetLastError(), 745 text("FindNextFileW: ", parentEntry.fullNameFS)); 746 } 747 } 748 749 void listDir(Path)(Path dirPath) 750 if (isPath!Path) 751 { 752 import std.internal.cstring; 753 754 if (dirPath.empty) 755 return listDir("."); 756 757 Context context; 758 759 FSChar[initialPathBufLength] pathBufStore = void; 760 context.pathBuf = pathBufStore[]; 761 762 scope (exit) 763 { 764 if (context.pathBuf.length != initialPathBufLength) 765 free(context.pathBuf.ptr); 766 } 767 768 Entry rootEntry = void; 769 rootEntry.context = &context; 770 771 auto endPos = appendString(context.pathBuf, 0, dirPath); 772 rootEntry.data.pathTailPos = endPos - (endPos > 0 && context.pathBuf[endPos - 1].isDirSeparator() ? 1 : 0); 773 assert(rootEntry.data.pathTailPos > 0); 774 775 version (Posix) 776 { 777 auto dir = opendir(tempCString(dirPath)); 778 checkDir(dir, dirPath); 779 780 scan(dir, dirfd(dir), &rootEntry); 781 } 782 else 783 version (Windows) 784 { 785 const WCHAR[] tailString = endPos == 0 || context.pathBuf[endPos - 1].isDirSeparator() ? "*.*\0"w : "\\*.*\0"w; 786 appendString(context.pathBuf, endPos, tailString); 787 788 scan(&rootEntry); 789 } 790 } 791 792 // Workaround for https://github.com/ldc-developers/ldc/issues/2960 793 version (Posix) 794 private void checkDir(Path)(DIR* dir, auto ref Path dirPath) 795 { 796 errnoEnforce(dir, "Failed to open directory " ~ dirPath); 797 } 798 } 799 800 unittest 801 { 802 auto tmpDir = deleteme ~ "-dir"; 803 if (tmpDir.exists) tmpDir.removeRecurse(); 804 mkdirRecurse(tmpDir); 805 scope(exit) rmdirRecurse(tmpDir); 806 807 touch(tmpDir ~ "/a"); 808 touch(tmpDir ~ "/b"); 809 mkdir(tmpDir ~ "/c"); 810 touch(tmpDir ~ "/c/1"); 811 touch(tmpDir ~ "/c/2"); 812 813 string[] entries; 814 listDir!((e) { 815 assert(equal(e.fullNameFS, e.fullName)); 816 entries ~= e.fullName.relPath(tmpDir); 817 if (e.entryIsDir) 818 e.recurse(); 819 })(tmpDir); 820 821 assert(equal( 822 entries.sort, 823 ["a", "b", "c", "c/1", "c/2"].map!(name => name.replace("/", dirSeparator)), 824 ), text(entries)); 825 826 entries = null; 827 import std.ascii : isDigit; 828 listDir!((e) { 829 entries ~= e.fullName.relPath(tmpDir); 830 if (e.baseNameFS[0].isDigit) 831 e.stop(); 832 else 833 if (e.entryIsDir) 834 e.recurse(); 835 })(tmpDir); 836 837 assert(entries.length < 5 && entries[$-1][$-1].isDigit, text(entries)); 838 839 // Symlink test 840 (){ 841 // Wine's implementation of symlinks/junctions is incomplete 842 version (Windows) 843 if (getWineVersion()) 844 return; 845 846 dirLink("c", tmpDir ~ "/d"); 847 dirLink("x", tmpDir ~ "/e"); 848 849 string[] entries; 850 listDir!((e) { 851 entries ~= e.fullName.relPath(tmpDir); 852 if (e.entryIsDir) 853 e.recurse(); 854 })(tmpDir); 855 856 assert(equal( 857 entries.sort, 858 ["a", "b", "c", "c/1", "c/2", "d", "e"].map!(name => name.replace("/", dirSeparator)), 859 )); 860 861 // Recurse into symlinks 862 863 entries = null; 864 listDir!((e) { 865 entries ~= e.fullName.relPath(tmpDir); 866 if (e.isDir) 867 try 868 e.recurse(); 869 catch (Exception e) // broken junctions on Windows throw 870 {} 871 })(tmpDir); 872 873 assert(equal( 874 entries.sort, 875 ["a", "b", "c", "c/1", "c/2", "d", "d/1", "d/2", "e"].map!(name => name.replace("/", dirSeparator)), 876 )); 877 }(); 878 } 879 880 // ************************************************************************ 881 882 string buildPath2(string[] segments...) { return segments.length ? buildPath(segments) : null; } 883 884 /// Shell-like expansion of ?, * and ** in path components 885 DirEntry[] fileList(string pattern) 886 { 887 auto components = cast(string[])array(pathSplitter(pattern)); 888 foreach (i, component; components[0..$-1]) 889 if (component.contains("?") || component.contains("*")) // TODO: escape? 890 { 891 DirEntry[] expansions; // TODO: filter range instead? 892 auto dir = buildPath2(components[0..i]); 893 if (component == "**") 894 expansions = array(dirEntries(dir, SpanMode.depth)); 895 else 896 expansions = array(dirEntries(dir, component, SpanMode.shallow)); 897 898 DirEntry[] result; 899 foreach (expansion; expansions) 900 if (expansion.isDir()) 901 result ~= fileList(buildPath(expansion.name ~ components[i+1..$])); 902 return result; 903 } 904 905 auto dir = buildPath2(components[0..$-1]); 906 if (!dir || exists(dir)) 907 return array(dirEntries(dir, components[$-1], SpanMode.shallow)); 908 else 909 return null; 910 } 911 912 /// ditto 913 DirEntry[] fileList(string pattern0, string[] patterns...) 914 { 915 DirEntry[] result; 916 foreach (pattern; [pattern0] ~ patterns) 917 result ~= fileList(pattern); 918 return result; 919 } 920 921 /// ditto 922 deprecated string[] fastFileList(string pattern) 923 { 924 auto components = cast(string[])array(pathSplitter(pattern)); 925 foreach (i, component; components[0..$-1]) 926 if (component.contains("?") || component.contains("*")) // TODO: escape? 927 { 928 string[] expansions; // TODO: filter range instead? 929 auto dir = buildPath2(components[0..i]); 930 if (component == "**") 931 expansions = fastListDir!true(dir); 932 else 933 expansions = fastListDir(dir, component); 934 935 string[] result; 936 foreach (expansion; expansions) 937 if (expansion.isDir()) 938 result ~= fastFileList(buildPath(expansion ~ components[i+1..$])); 939 return result; 940 } 941 942 auto dir = buildPath2(components[0..$-1]); 943 if (!dir || exists(dir)) 944 return fastListDir(dir, components[$-1]); 945 else 946 return null; 947 } 948 949 /// ditto 950 deprecated string[] fastFileList(string pattern0, string[] patterns...) 951 { 952 string[] result; 953 foreach (pattern; [pattern0] ~ patterns) 954 result ~= fastFileList(pattern); 955 return result; 956 } 957 958 // ************************************************************************ 959 960 import std.datetime; 961 import std.exception; 962 963 deprecated SysTime getMTime(string name) 964 { 965 return timeLastModified(name); 966 } 967 968 /// If target exists, update its modification time; 969 /// otherwise create it as an empty file. 970 void touch(in char[] target) 971 { 972 if (exists(target)) 973 { 974 auto now = Clock.currTime(); 975 setTimes(target, now, now); 976 } 977 else 978 std.file.write(target, ""); 979 } 980 981 /// Returns true if the target file doesn't exist, 982 /// or source is newer than the target. 983 bool newerThan(string source, string target) 984 { 985 if (!target.exists) 986 return true; 987 return source.timeLastModified() > target.timeLastModified(); 988 } 989 990 /// Returns true if the target file doesn't exist, 991 /// or any of the sources are newer than the target. 992 bool anyNewerThan(string[] sources, string target) 993 { 994 if (!target.exists) 995 return true; 996 auto targetTime = target.timeLastModified(); 997 return sources.any!(source => source.timeLastModified() > targetTime)(); 998 } 999 1000 version (Posix) 1001 { 1002 import core.sys.posix.sys.stat; 1003 import core.sys.posix.unistd; 1004 1005 /// Get the ID of the user owning this file. 1006 int getOwner(string fn) 1007 { 1008 stat_t s; 1009 errnoEnforce(stat(toStringz(fn), &s) == 0, "stat: " ~ fn); 1010 return s.st_uid; 1011 } 1012 1013 /// Get the ID of the group owning this file. 1014 int getGroup(string fn) 1015 { 1016 stat_t s; 1017 errnoEnforce(stat(toStringz(fn), &s) == 0, "stat: " ~ fn); 1018 return s.st_gid; 1019 } 1020 1021 /// Set the owner user and group of this file. 1022 void setOwner(string fn, int uid, int gid) 1023 { 1024 errnoEnforce(chown(toStringz(fn), uid, gid) == 0, "chown: " ~ fn); 1025 } 1026 } 1027 1028 /// Try to rename; copy/delete if rename fails 1029 void move(string src, string dst) 1030 { 1031 try 1032 src.rename(dst); 1033 catch (Exception e) 1034 { 1035 atomicCopy(src, dst); 1036 src.remove(); 1037 } 1038 } 1039 1040 /// Make sure that the given directory exists 1041 /// (and create parent directories as necessary). 1042 void ensureDirExists(string path) 1043 { 1044 if (!path.exists) 1045 path.mkdirRecurse(); 1046 } 1047 1048 /// Make sure that the path to the given file name 1049 /// exists (and create directories as necessary). 1050 void ensurePathExists(string fn) 1051 { 1052 fn.dirName.ensureDirExists(); 1053 } 1054 1055 static import core.stdc.errno; 1056 version (Windows) 1057 { 1058 static import core.sys.windows.winerror; 1059 static import std.windows.syserror; 1060 static import ae.sys.windows.exception; 1061 } 1062 1063 /// Catch common Phobos exception types corresponding to file operations. 1064 bool collectOSError(alias checkCError, alias checkWinError)(scope void delegate() operation) 1065 { 1066 mixin(() { 1067 string code = q{ 1068 try 1069 { 1070 operation(); 1071 return true; 1072 } 1073 catch (FileException e) 1074 { 1075 version (Windows) 1076 bool collect = checkWinError(e.errno); 1077 else 1078 bool collect = checkCError(e.errno); 1079 if (collect) 1080 return false; 1081 else 1082 throw e; 1083 } 1084 catch (ErrnoException e) 1085 { 1086 if (checkCError(e.errno)) 1087 return false; 1088 else 1089 throw e; 1090 } 1091 }; 1092 version(Windows) code ~= q{ 1093 catch (std.windows.syserror.WindowsException e) 1094 { 1095 if (checkWinError(e.code)) 1096 return false; 1097 else 1098 throw e; 1099 } 1100 catch (ae.sys.windows.exception.WindowsException e) 1101 { 1102 if (checkWinError(e.code)) 1103 return false; 1104 else 1105 throw e; 1106 } 1107 }; 1108 return code; 1109 }()); 1110 } 1111 1112 /// Collect a "file not found" error. 1113 alias collectNotFoundError = collectOSError!( 1114 errno => errno == core.stdc.errno.ENOENT, 1115 (code) { version(Windows) return 1116 code == core.sys.windows.winerror.ERROR_FILE_NOT_FOUND || 1117 code == core.sys.windows.winerror.ERROR_PATH_NOT_FOUND; }, 1118 ); 1119 1120 /// 1121 unittest 1122 { 1123 auto fn = deleteme; 1124 if (fn.exists) fn.removeRecurse(); 1125 foreach (dg; [ 1126 { openFile(fn, "rb"); }, 1127 { mkdir(fn.buildPath("b")); }, 1128 { hardLink(fn, fn ~ "2"); }, 1129 ]) 1130 assert(!dg.collectNotFoundError); 1131 } 1132 1133 /// Collect a "file already exists" error. 1134 alias collectFileExistsError = collectOSError!( 1135 errno => errno == core.stdc.errno.EEXIST, 1136 (code) { version(Windows) return 1137 code == core.sys.windows.winerror.ERROR_FILE_EXISTS || 1138 code == core.sys.windows.winerror.ERROR_ALREADY_EXISTS; }, 1139 ); 1140 1141 /// 1142 unittest 1143 { 1144 auto fn = deleteme; 1145 foreach (dg; [ 1146 { mkdir(fn); }, 1147 { openFile(fn, "wxb"); }, 1148 { touch(fn ~ "2"); hardLink(fn ~ "2", fn); }, 1149 ]) 1150 { 1151 if (fn.exists) fn.removeRecurse(); 1152 assert( dg.collectFileExistsError); 1153 assert(!dg.collectFileExistsError); 1154 } 1155 } 1156 1157 import ae.utils.text; 1158 1159 /// Forcibly remove a file or directory. 1160 /// If atomic is true, the entire directory is deleted "atomically" 1161 /// (it is first moved/renamed to another location). 1162 /// On Windows, this will move the file/directory out of the way, 1163 /// if it is in use and cannot be deleted (but can be renamed). 1164 void forceDelete(Flag!"atomic" atomic=Yes.atomic)(string fn, Flag!"recursive" recursive = No.recursive) 1165 { 1166 import std.process : environment; 1167 version(Windows) 1168 { 1169 mixin(importWin32!q{winnt}); 1170 mixin(importWin32!q{winbase}); 1171 } 1172 1173 auto name = fn.baseName(); 1174 fn = fn.absolutePath().longPath(); 1175 1176 version(Windows) 1177 { 1178 auto fnW = toUTF16z(fn); 1179 auto attr = GetFileAttributesW(fnW); 1180 wenforce(attr != INVALID_FILE_ATTRIBUTES, "GetFileAttributes"); 1181 if (attr & FILE_ATTRIBUTE_READONLY) 1182 SetFileAttributesW(fnW, attr & ~FILE_ATTRIBUTE_READONLY).wenforce("SetFileAttributes"); 1183 } 1184 1185 static if (atomic) 1186 { 1187 // To avoid zombifying locked directories, try renaming it first. 1188 // Attempting to delete a locked directory will make it inaccessible. 1189 1190 bool tryMoveTo(string target) 1191 { 1192 target = target.longPath(); 1193 if (target.endsWith(dirSeparator)) 1194 target = target[0..$-1]; 1195 if (target.length && !target.exists) 1196 return false; 1197 1198 string newfn; 1199 do 1200 newfn = format("%s%sdeleted-%s.%s.%s", target, dirSeparator, name, thisProcessID, randomString()); 1201 while (newfn.exists); 1202 1203 version(Windows) 1204 { 1205 auto newfnW = toUTF16z(newfn); 1206 if (!MoveFileW(fnW, newfnW)) 1207 return false; 1208 } 1209 else 1210 { 1211 try 1212 rename(fn, newfn); 1213 catch (FileException e) 1214 return false; 1215 } 1216 1217 fn = newfn; 1218 version(Windows) fnW = newfnW; 1219 return true; 1220 } 1221 1222 void tryMove() 1223 { 1224 auto tmp = environment.get("TEMP"); 1225 if (tmp) 1226 if (tryMoveTo(tmp)) 1227 return; 1228 1229 version(Windows) 1230 string tempDir = fn[0..7]~"Temp"; 1231 else 1232 enum tempDir = "/tmp"; 1233 1234 if (tryMoveTo(tempDir)) 1235 return; 1236 1237 if (tryMoveTo(fn.dirName())) 1238 return; 1239 1240 throw new Exception("Unable to delete " ~ fn ~ " atomically (all rename attempts failed)"); 1241 } 1242 1243 tryMove(); 1244 } 1245 1246 version(Windows) 1247 { 1248 if (attr & FILE_ATTRIBUTE_DIRECTORY) 1249 { 1250 if (recursive && (attr & FILE_ATTRIBUTE_REPARSE_POINT) == 0) 1251 { 1252 foreach (de; fn.dirEntries(SpanMode.shallow)) 1253 forceDelete!(No.atomic)(de.name, Yes.recursive); 1254 } 1255 // Will fail if !recursive and directory is not empty 1256 RemoveDirectoryW(fnW).wenforce("RemoveDirectory"); 1257 } 1258 else 1259 DeleteFileW(fnW).wenforce("DeleteFile"); 1260 } 1261 else 1262 { 1263 if (recursive) 1264 fn.removeRecurse(); 1265 else 1266 if (fn.isDir) 1267 fn.rmdir(); 1268 else 1269 fn.remove(); 1270 } 1271 } 1272 1273 1274 deprecated void forceDelete(bool atomic)(string fn, bool recursive = false) { forceDelete!(cast(Flag!"atomic")atomic)(fn, cast(Flag!"recursive")recursive); } 1275 //deprecated void forceDelete()(string fn, bool recursive) { forceDelete!(Yes.atomic)(fn, cast(Flag!"recursive")recursive); } 1276 1277 deprecated unittest 1278 { 1279 mkdir("testdir"); touch("testdir/b"); forceDelete!(false )("testdir", true); 1280 mkdir("testdir"); touch("testdir/b"); forceDelete!(true )("testdir", true); 1281 } 1282 1283 unittest 1284 { 1285 mkdir("testdir"); touch("testdir/b"); forceDelete ("testdir", Yes.recursive); 1286 mkdir("testdir"); touch("testdir/b"); forceDelete!(No .atomic)("testdir", Yes.recursive); 1287 mkdir("testdir"); touch("testdir/b"); forceDelete!(Yes.atomic)("testdir", Yes.recursive); 1288 } 1289 1290 /// If fn is a directory, delete it recursively. 1291 /// Otherwise, delete the file or symlink fn. 1292 void removeRecurse(string fn) 1293 { 1294 auto attr = fn.getAttributes(); 1295 if (attr.attrIsSymlink) 1296 { 1297 version (Windows) 1298 if (attr.attrIsDir) 1299 fn.rmdir(); 1300 else 1301 fn.remove(); 1302 else 1303 fn.remove(); 1304 } 1305 else 1306 if (attr.attrIsDir) 1307 version (Windows) 1308 fn.forceDelete!(No.atomic)(Yes.recursive); // For read-only files 1309 else 1310 fn.rmdirRecurse(); 1311 else 1312 fn.remove(); 1313 } 1314 1315 /// Create an empty directory, deleting 1316 /// all its contents if it already exists. 1317 void recreateEmptyDirectory()(string dir) 1318 { 1319 if (dir.exists) 1320 dir.forceDelete(Yes.recursive); 1321 mkdir(dir); 1322 } 1323 1324 /// Copy a directory recursively. 1325 void copyRecurse(DirEntry src, string dst) 1326 { 1327 version (Posix) 1328 if (src.isSymlink) 1329 return symlink(dst, readLink(src)); 1330 if (src.isFile) 1331 return copy(src, dst, PreserveAttributes.yes); 1332 dst.mkdir(); 1333 foreach (de; src.dirEntries(SpanMode.shallow)) 1334 copyRecurse(de, dst.buildPath(de.baseName)); 1335 } 1336 void copyRecurse(string src, string dst) { copyRecurse(DirEntry(src), dst); } /// ditto 1337 1338 /// Return true if the given file would be hidden from directory listings. 1339 /// Returns true for files starting with `'.'`, and, on Windows, hidden files. 1340 bool isHidden()(string fn) 1341 { 1342 if (baseName(fn).startsWith(".")) 1343 return true; 1344 version (Windows) 1345 { 1346 mixin(importWin32!q{winnt}); 1347 if (getAttributes(fn) & FILE_ATTRIBUTE_HIDDEN) 1348 return true; 1349 } 1350 return false; 1351 } 1352 1353 /// Return a file's unique ID. 1354 ulong getFileID()(string fn) 1355 { 1356 version (Windows) 1357 { 1358 mixin(importWin32!q{winnt}); 1359 mixin(importWin32!q{winbase}); 1360 1361 auto fnW = toUTF16z(fn); 1362 auto h = CreateFileW(fnW, FILE_READ_ATTRIBUTES, 0, null, OPEN_EXISTING, 0, HANDLE.init); 1363 wenforce(h!=INVALID_HANDLE_VALUE, fn); 1364 scope(exit) CloseHandle(h); 1365 BY_HANDLE_FILE_INFORMATION fi; 1366 GetFileInformationByHandle(h, &fi).wenforce("GetFileInformationByHandle"); 1367 1368 ULARGE_INTEGER li; 1369 li.LowPart = fi.nFileIndexLow; 1370 li.HighPart = fi.nFileIndexHigh; 1371 auto result = li.QuadPart; 1372 enforce(result, "Null file ID"); 1373 return result; 1374 } 1375 else 1376 { 1377 return DirEntry(fn).statBuf.st_ino; 1378 } 1379 } 1380 1381 unittest 1382 { 1383 auto base = deleteme; 1384 touch(base ~ "a"); 1385 scope(exit) remove(base ~ "a"); 1386 hardLink(base ~ "a", base ~ "b"); 1387 scope(exit) remove(base ~ "b"); 1388 touch(base ~ "c"); 1389 scope(exit) remove(base ~ "c"); 1390 assert(getFileID(base ~ "a") == getFileID(base ~ "b")); 1391 assert(getFileID(base ~ "a") != getFileID(base ~ "c")); 1392 } 1393 1394 deprecated alias std.file.getSize getSize2; 1395 1396 /// Using UNC paths bypasses path length limitation when using Windows wide APIs. 1397 string longPath(string s) 1398 { 1399 version (Windows) 1400 { 1401 if (!s.startsWith(`\\`)) 1402 return `\\?\` ~ s.absolutePath().buildNormalizedPath().replace(`/`, `\`); 1403 } 1404 return s; 1405 } 1406 1407 version (Windows) 1408 { 1409 static if (__traits(compiles, { mixin importWin32!q{winnt}; })) 1410 static mixin(importWin32!q{winnt}); 1411 1412 /// Common code for creating Windows reparse points. 1413 void createReparsePoint(string reparseBufferName, string extraInitialization, string reparseTagName)(in char[] target, in char[] print, in char[] link) 1414 { 1415 mixin(importWin32!q{winbase}); 1416 mixin(importWin32!q{windef}); 1417 mixin(importWin32!q{winioctl}); 1418 1419 enum SYMLINK_FLAG_RELATIVE = 1; 1420 1421 HANDLE hLink = CreateFileW(link.toUTF16z(), GENERIC_READ | GENERIC_WRITE, 0, null, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, null); 1422 wenforce(hLink && hLink != INVALID_HANDLE_VALUE, "CreateFileW"); 1423 scope(exit) CloseHandle(hLink); 1424 1425 enum pathOffset = 1426 mixin(q{REPARSE_DATA_BUFFER..} ~ reparseBufferName) .offsetof + 1427 mixin(q{REPARSE_DATA_BUFFER..} ~ reparseBufferName)._PathBuffer.offsetof; 1428 1429 auto targetW = target.toUTF16(); 1430 auto printW = print .toUTF16(); 1431 1432 // Despite MSDN, two NUL-terminating characters are needed, one for each string. 1433 1434 auto pathBufferSize = targetW.length + 1 + printW.length + 1; // in chars 1435 auto buf = new ubyte[pathOffset + pathBufferSize * WCHAR.sizeof]; 1436 auto r = cast(REPARSE_DATA_BUFFER*)buf.ptr; 1437 1438 r.ReparseTag = mixin(reparseTagName); 1439 r.ReparseDataLength = to!WORD(buf.length - mixin(q{r..} ~ reparseBufferName).offsetof); 1440 1441 auto pathBuffer = mixin(q{r..} ~ reparseBufferName).PathBuffer; 1442 auto p = pathBuffer; 1443 1444 mixin(q{r..} ~ reparseBufferName).SubstituteNameOffset = to!WORD((p-pathBuffer) * WCHAR.sizeof); 1445 mixin(q{r..} ~ reparseBufferName).SubstituteNameLength = to!WORD(targetW.length * WCHAR.sizeof); 1446 p[0..targetW.length] = targetW; 1447 p += targetW.length; 1448 *p++ = 0; 1449 1450 mixin(q{r..} ~ reparseBufferName).PrintNameOffset = to!WORD((p-pathBuffer) * WCHAR.sizeof); 1451 mixin(q{r..} ~ reparseBufferName).PrintNameLength = to!WORD(printW .length * WCHAR.sizeof); 1452 p[0..printW.length] = printW; 1453 p += printW.length; 1454 *p++ = 0; 1455 1456 assert(p-pathBuffer == pathBufferSize); 1457 1458 mixin(extraInitialization); 1459 1460 DWORD dwRet; // Needed despite MSDN 1461 DeviceIoControl(hLink, FSCTL_SET_REPARSE_POINT, buf.ptr, buf.length.to!DWORD(), null, 0, &dwRet, null).wenforce("DeviceIoControl"); 1462 } 1463 1464 /// Attempt to acquire the specified privilege. 1465 void acquirePrivilege(S)(S name) 1466 { 1467 mixin(importWin32!q{winbase}); 1468 mixin(importWin32!q{windef}); 1469 1470 import ae.sys.windows; 1471 1472 HANDLE hToken = null; 1473 wenforce(OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &hToken)); 1474 scope(exit) CloseHandle(hToken); 1475 1476 TOKEN_PRIVILEGES tp; 1477 wenforce(LookupPrivilegeValue(null, name.toUTF16z(), &tp.Privileges[0].Luid), "LookupPrivilegeValue"); 1478 1479 tp.PrivilegeCount = 1; 1480 tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; 1481 wenforce(AdjustTokenPrivileges(hToken, FALSE, &tp, cast(DWORD)TOKEN_PRIVILEGES.sizeof, null, null), "AdjustTokenPrivileges"); 1482 } 1483 1484 /// Link a directory. 1485 /// Uses symlinks on POSIX, and directory junctions on Windows. 1486 void dirLink()(in char[] original, in char[] link) 1487 { 1488 mkdir(link); 1489 scope(failure) rmdir(link); 1490 1491 auto target = `\??\` ~ (cast(string)original).absolutePath((cast(string)link.dirName).absolutePath).buildNormalizedPath; 1492 if (target[$-1] != '\\') 1493 target ~= '\\'; 1494 1495 createReparsePoint!(q{MountPointReparseBuffer}, q{}, q{IO_REPARSE_TAG_MOUNT_POINT})(target, null, link); 1496 } 1497 1498 /// Windows implementation of `std.file.symlink`. 1499 void symlink()(in char[] original, in char[] link) 1500 { 1501 mixin(importWin32!q{winnt}); 1502 1503 acquirePrivilege(SE_CREATE_SYMBOLIC_LINK_NAME); 1504 1505 touch(link); 1506 scope(failure) remove(link); 1507 1508 createReparsePoint!(q{SymbolicLinkReparseBuffer}, q{r.SymbolicLinkReparseBuffer.Flags = link.isAbsolute() ? 0 : SYMLINK_FLAG_RELATIVE;}, q{IO_REPARSE_TAG_SYMLINK})(original, original, link); 1509 } 1510 } 1511 else 1512 alias std.file.symlink dirLink; /// `std.file.symlink` is used to implement `dirLink` on POSIX. 1513 1514 version(Windows) version(unittest) static mixin(importWin32!q{winnt}); 1515 1516 unittest 1517 { 1518 // Wine's implementation of symlinks/junctions is incomplete 1519 version (Windows) 1520 if (getWineVersion()) 1521 return; 1522 1523 mkdir("a"); scope(exit) rmdir("a"[]); 1524 touch("a/f"); scope(exit) remove("a/f"); 1525 dirLink("a", "b"); scope(exit) version(Windows) rmdir("b"); else remove("b"); 1526 //symlink("a/f", "c"); scope(exit) remove("c"); 1527 assert("b".isSymlink()); 1528 //assert("c".isSymlink()); 1529 assert("b/f".exists()); 1530 } 1531 1532 version (Windows) 1533 { 1534 /// Create a hard link. 1535 void hardLink()(string src, string dst) 1536 { 1537 mixin(importWin32!q{w32api}); 1538 1539 static assert(_WIN32_WINNT >= 0x501, "CreateHardLinkW not available for target Windows platform. Specify -version=WindowsXP"); 1540 1541 mixin(importWin32!q{winnt}); 1542 mixin(importWin32!q{winbase}); 1543 1544 wenforce(CreateHardLinkW(toUTF16z(dst), toUTF16z(src), null), "CreateHardLink failed: " ~ src ~ " -> " ~ dst); 1545 } 1546 1547 /// Deletes a file, which might be a read-only hard link 1548 /// (thus, deletes the read-only file/link without affecting other links to it). 1549 void deleteHardLink()(string fn) 1550 { 1551 mixin(importWin32!q{winbase}); 1552 1553 auto fnW = toUTF16z(fn); 1554 1555 DWORD attrs = GetFileAttributesW(fnW); 1556 wenforce(attrs != INVALID_FILE_ATTRIBUTES, "GetFileAttributesW failed: " ~ fn); 1557 1558 if (attrs & FILE_ATTRIBUTE_READONLY) 1559 SetFileAttributesW(fnW, attrs & ~FILE_ATTRIBUTE_READONLY) 1560 .wenforce("SetFileAttributesW failed: " ~ fn); 1561 HANDLE h = CreateFileW(fnW, GENERIC_READ|GENERIC_WRITE, 7, null, OPEN_EXISTING, 1562 FILE_FLAG_DELETE_ON_CLOSE, null); 1563 wenforce(h != INVALID_HANDLE_VALUE, "CreateFileW failed: " ~ fn); 1564 if (attrs & FILE_ATTRIBUTE_READONLY) 1565 SetFileAttributesW(fnW, attrs) 1566 .wenforce("SetFileAttributesW failed: " ~ fn); 1567 CloseHandle(h).wenforce("CloseHandle failed: " ~ fn); 1568 } 1569 } 1570 version (Posix) 1571 { 1572 /// Create a hard link. 1573 void hardLink()(string src, string dst) 1574 { 1575 import core.sys.posix.unistd; 1576 errnoEnforce(link(toUTFz!(const char*)(src), toUTFz!(const char*)(dst)) == 0, "link() failed: " ~ dst); 1577 } 1578 1579 alias deleteHardLink = remove; /// `std.file.remove` is used to implement `deleteHardLink` on POSIX. 1580 } 1581 1582 unittest 1583 { 1584 write("a", "foo"); scope(exit) remove("a"); 1585 hardLink("a", "b"); 1586 assert("b".readText == "foo"); 1587 deleteHardLink("b"); 1588 assert(!"b".exists); 1589 } 1590 1591 version (Posix) 1592 { 1593 /// Wrapper around the C `realpath` function. 1594 string realPath(string path) 1595 { 1596 // TODO: Windows version 1597 import core.sys.posix.stdlib; 1598 auto p = realpath(toUTFz!(const char*)(path), null); 1599 errnoEnforce(p, "realpath"); 1600 string result = fromStringz(p).idup; 1601 free(p); 1602 return result; 1603 } 1604 } 1605 1606 // /proc/self/mounts parsing 1607 version (linux) 1608 { 1609 /// A parsed line from /proc/self/mounts. 1610 struct MountInfo 1611 { 1612 string spec; /// device path 1613 string file; /// mount path 1614 string vfstype; /// file system 1615 string mntops; /// options 1616 int freq; /// dump flag 1617 int passno; /// fsck order 1618 } 1619 1620 string unescapeMountString(in char[] s) 1621 { 1622 string result; 1623 1624 size_t p = 0; 1625 for (size_t i=0; i+3<s.length;) 1626 { 1627 auto c = s[i]; 1628 if (c == '\\') 1629 { 1630 result ~= s[p..i]; 1631 result ~= to!int(s[i+1..i+4], 8); 1632 i += 4; 1633 p = i; 1634 } 1635 else 1636 i++; 1637 } 1638 result ~= s[p..$]; 1639 return result; 1640 } 1641 1642 unittest 1643 { 1644 assert(unescapeMountString(`a\040b\040c`) == "a b c"); 1645 assert(unescapeMountString(`\040`) == " "); 1646 } 1647 1648 /// Parse a line from /proc/self/mounts. 1649 MountInfo parseMountInfo(in char[] line) 1650 { 1651 const(char)[][6] parts; 1652 copy(line.splitter(" "), parts[]); 1653 return MountInfo( 1654 unescapeMountString(parts[0]), 1655 unescapeMountString(parts[1]), 1656 unescapeMountString(parts[2]), 1657 unescapeMountString(parts[3]), 1658 parts[4].to!int, 1659 parts[5].to!int, 1660 ); 1661 } 1662 1663 /// Returns an iterator of MountInfo structs. 1664 auto getMounts() 1665 { 1666 return File("/proc/self/mounts", "rb").byLine().map!parseMountInfo(); 1667 } 1668 1669 /// Get MountInfo with longest mount point matching path. 1670 /// Returns MountInfo.init if none match. 1671 MountInfo getPathMountInfo(string path) 1672 { 1673 path = realPath(path); 1674 size_t bestLength; MountInfo bestInfo; 1675 foreach (ref info; getMounts()) 1676 { 1677 if (path.pathStartsWith(info.file)) 1678 { 1679 if (bestLength < info.file.length) 1680 { 1681 bestLength = info.file.length; 1682 bestInfo = info; 1683 } 1684 } 1685 } 1686 return bestInfo; 1687 } 1688 1689 /// Get the name of the filesystem that the given path is mounted under. 1690 /// Returns null if none match. 1691 string getPathFilesystem(string path) 1692 { 1693 return getPathMountInfo(path).vfstype; 1694 } 1695 } 1696 1697 // **************************************************************************** 1698 1699 version (linux) 1700 { 1701 import core.sys.linux.sys.xattr; 1702 import core.stdc.errno; 1703 alias ENOATTR = ENODATA; 1704 1705 /// AA-like object for accessing a file's extended attributes. 1706 struct XAttrs(Obj, string funPrefix) 1707 { 1708 Obj obj; 1709 1710 mixin("alias getFun = " ~ funPrefix ~ "getxattr;"); 1711 mixin("alias setFun = " ~ funPrefix ~ "setxattr;"); 1712 mixin("alias removeFun = " ~ funPrefix ~ "removexattr;"); 1713 mixin("alias listFun = " ~ funPrefix ~ "listxattr;"); 1714 1715 /// True if extended attributes are supported on this filesystem. 1716 bool supported() 1717 { 1718 auto size = getFun(obj, "user.\x01", null, 0); 1719 return size >= 0 || errno != EOPNOTSUPP; 1720 } 1721 1722 /// Read an extended attribute. 1723 void[] opIndex(string key) 1724 { 1725 auto cKey = key.toStringz(); 1726 size_t size = 0; 1727 void[] buf; 1728 do 1729 { 1730 buf.length = size; 1731 size = getFun(obj, cKey, buf.ptr, buf.length); 1732 errnoEnforce(size >= 0, __traits(identifier, getFun)); 1733 } while (size != buf.length); 1734 return buf; 1735 } 1736 1737 /// Check if an extended attribute is present. 1738 bool opBinaryRight(string op)(string key) 1739 if (op == "in") 1740 { 1741 auto cKey = key.toStringz(); 1742 auto size = getFun(obj, cKey, null, 0); 1743 if (size >= 0) 1744 return true; 1745 else 1746 if (errno == ENOATTR) 1747 return false; 1748 else 1749 errnoEnforce(false, __traits(identifier, getFun)); 1750 assert(false); 1751 } 1752 1753 /// Write an extended attribute. 1754 void opIndexAssign(in void[] value, string key) 1755 { 1756 auto ret = setFun(obj, key.toStringz(), value.ptr, value.length, 0); 1757 errnoEnforce(ret == 0, __traits(identifier, setFun)); 1758 } 1759 1760 /// Delete an extended attribute. 1761 void remove(string key) 1762 { 1763 auto ret = removeFun(obj, key.toStringz()); 1764 errnoEnforce(ret == 0, __traits(identifier, removeFun)); 1765 } 1766 1767 /// Return a list of all extended attribute names. 1768 string[] keys() 1769 { 1770 size_t size = 0; 1771 char[] buf; 1772 do 1773 { 1774 buf.length = size; 1775 size = listFun(obj, buf.ptr, buf.length); 1776 errnoEnforce(size >= 0, __traits(identifier, listFun)); 1777 } while (size != buf.length); 1778 1779 char[][] result; 1780 size_t start; 1781 foreach (p, c; buf) 1782 if (!c) 1783 { 1784 result ~= buf[start..p]; 1785 start = p+1; 1786 } 1787 1788 return cast(string[])result; 1789 } 1790 } 1791 1792 /// Return `XAttrs` for the given path, 1793 /// or the link destination if the path leads to as symbolic link. 1794 auto xAttrs(string path) 1795 { 1796 return XAttrs!(const(char)*, "")(path.toStringz()); 1797 } 1798 1799 /// Return `XAttrs` for the given path. 1800 auto linkXAttrs(string path) 1801 { 1802 return XAttrs!(const(char)*, "l")(path.toStringz()); 1803 } 1804 1805 /// Return `XAttrs` for the given open file. 1806 auto xAttrs(ref const File f) 1807 { 1808 return XAttrs!(int, "f")(f.fileno); 1809 } 1810 1811 /// 1812 unittest 1813 { 1814 if (!xAttrs(".").supported) 1815 { 1816 import std.stdio : stderr; 1817 stderr.writeln("ae.sys.file: xattrs not supported on current filesystem, skipping test."); 1818 return; 1819 } 1820 1821 enum fn = "test.txt"; 1822 std.file.write(fn, "test"); 1823 scope(exit) remove(fn); 1824 1825 auto attrs = xAttrs(fn); 1826 enum key = "user.foo"; 1827 assert(key !in attrs); 1828 assert(attrs.keys == []); 1829 1830 attrs[key] = "bar"; 1831 assert(key in attrs); 1832 assert(attrs[key] == "bar"); 1833 assert(attrs.keys == [key]); 1834 1835 attrs.remove(key); 1836 assert(key !in attrs); 1837 assert(attrs.keys == []); 1838 } 1839 } 1840 1841 // **************************************************************************** 1842 1843 version (Windows) 1844 { 1845 /// Enumerate all hard links to the specified file. 1846 // TODO: Return a range 1847 string[] enumerateHardLinks()(string fn) 1848 { 1849 mixin(importWin32!q{winnt}); 1850 mixin(importWin32!q{winbase}); 1851 1852 alias extern(System) HANDLE function(LPCWSTR lpFileName, DWORD dwFlags, LPDWORD StringLength, PWCHAR LinkName) TFindFirstFileNameW; 1853 alias extern(System) BOOL function(HANDLE hFindStream, LPDWORD StringLength, PWCHAR LinkName) TFindNextFileNameW; 1854 1855 auto kernel32 = GetModuleHandle("kernel32.dll"); 1856 auto FindFirstFileNameW = cast(TFindFirstFileNameW)GetProcAddress(kernel32, "FindFirstFileNameW").wenforce("GetProcAddress(FindFirstFileNameW)"); 1857 auto FindNextFileNameW = cast(TFindNextFileNameW)GetProcAddress(kernel32, "FindNextFileNameW").wenforce("GetProcAddress(FindNextFileNameW)"); 1858 1859 static WCHAR[0x8000] buf; 1860 DWORD len = buf.length; 1861 auto h = FindFirstFileNameW(toUTF16z(fn), 0, &len, buf.ptr); 1862 wenforce(h != INVALID_HANDLE_VALUE, "FindFirstFileNameW"); 1863 scope(exit) FindClose(h); 1864 1865 string[] result; 1866 do 1867 { 1868 enforce(len > 0 && len < buf.length && buf[len-1] == 0, "Bad FindFirst/NextFileNameW result"); 1869 result ~= buf[0..len-1].toUTF8(); 1870 len = buf.length; 1871 auto ok = FindNextFileNameW(h, &len, buf.ptr); 1872 if (!ok && GetLastError() == ERROR_HANDLE_EOF) 1873 break; 1874 wenforce(ok, "FindNextFileNameW"); 1875 } while(true); 1876 return result; 1877 } 1878 } 1879 1880 /// Obtain the hard link count for the given file. 1881 uint hardLinkCount(string fn) 1882 { 1883 version (Windows) 1884 { 1885 // TODO: Optimize (don't transform strings) 1886 return cast(uint)fn.enumerateHardLinks.length; 1887 } 1888 else 1889 { 1890 import core.sys.posix.sys.stat; 1891 1892 stat_t s; 1893 errnoEnforce(stat(fn.toStringz(), &s) == 0, "stat"); 1894 return s.st_nlink.to!uint; 1895 } 1896 } 1897 1898 // http://d.puremagic.com/issues/show_bug.cgi?id=7016 1899 version (unittest) 1900 version (Windows) 1901 import ae.sys.windows.misc : getWineVersion; 1902 1903 unittest 1904 { 1905 // FindFirstFileNameW not implemented in Wine 1906 version (Windows) 1907 if (getWineVersion()) 1908 return; 1909 1910 touch("a.test"); 1911 scope(exit) remove("a.test"); 1912 assert("a.test".hardLinkCount() == 1); 1913 1914 hardLink("a.test", "b.test"); 1915 scope(exit) remove("b.test"); 1916 assert("a.test".hardLinkCount() == 2); 1917 assert("b.test".hardLinkCount() == 2); 1918 1919 version(Windows) 1920 { 1921 auto paths = enumerateHardLinks("a.test"); 1922 assert(paths.length == 2); 1923 paths.sort(); 1924 assert(paths[0].endsWith(`\a.test`), paths[0]); 1925 assert(paths[1].endsWith(`\b.test`)); 1926 } 1927 } 1928 1929 /// Argument-reversed version of `std.file.write`, 1930 /// usable at the end of an UFCS chain. 1931 static if (is(typeof({ import std.stdio : toFile; }))) 1932 { 1933 static import std.stdio; 1934 alias toFile = std.stdio.toFile; 1935 } 1936 else 1937 { 1938 void toFile(in void[] data, in char[] name) 1939 { 1940 std.file.write(name, data); 1941 } 1942 } 1943 1944 /// Same as toFile, but accepts void[] and does not conflict with the 1945 /// std.stdio function. 1946 void writeTo(in void[] data, in char[] target) 1947 { 1948 std.file.write(target, data); 1949 } 1950 1951 /// Polyfill for Windows fopen implementations with support for UNC 1952 /// paths and the 'x' subspecifier. 1953 File openFile()(string fn, string mode = "rb") 1954 { 1955 File f; 1956 static if (is(typeof(&f.windowsHandleOpen))) 1957 { 1958 import core.sys.windows.windows; 1959 import ae.sys.windows.exception; 1960 1961 string winMode, cMode; 1962 foreach (c; mode) 1963 { 1964 switch (c) 1965 { 1966 case 'r': 1967 case 'w': 1968 case 'a': 1969 case '+': 1970 case 'x': 1971 winMode ~= c; 1972 break; 1973 case 'b': 1974 case 't': 1975 break; 1976 default: 1977 assert(false, "Unknown character in mode"); 1978 } 1979 if (c != 'x') 1980 cMode ~= c; 1981 } 1982 DWORD access, creation; 1983 bool append; 1984 switch (winMode) 1985 { 1986 case "r" : access = GENERIC_READ ; creation = OPEN_EXISTING; break; 1987 case "r+" : access = GENERIC_READ | GENERIC_WRITE; creation = OPEN_EXISTING; break; 1988 case "w" : access = GENERIC_WRITE; creation = CREATE_ALWAYS; break; 1989 case "w+" : access = GENERIC_READ | GENERIC_WRITE; creation = CREATE_ALWAYS; break; 1990 case "a" : access = GENERIC_WRITE; creation = OPEN_ALWAYS ; version (CRuntime_Microsoft) append = true; break; 1991 case "a+" : access = GENERIC_READ | GENERIC_WRITE; creation = OPEN_ALWAYS ; version (CRuntime_Microsoft) assert(false, "MSVCRT can't fdopen with a+"); else break; 1992 case "wx" : access = GENERIC_WRITE; creation = CREATE_NEW ; break; 1993 case "w+x": access = GENERIC_READ | GENERIC_WRITE; creation = CREATE_NEW ; break; 1994 case "ax" : access = GENERIC_WRITE; creation = CREATE_NEW ; version (CRuntime_Microsoft) append = true; break; 1995 case "a+x": access = GENERIC_READ | GENERIC_WRITE; creation = CREATE_NEW ; version (CRuntime_Microsoft) assert(false, "MSVCRT can't fdopen with a+"); else break; 1996 default: assert(false, "Bad file mode: " ~ mode); 1997 } 1998 1999 auto pathW = toUTF16z(longPath(fn)); 2000 auto h = CreateFileW(pathW, access, FILE_SHARE_READ, null, creation, 0, HANDLE.init); 2001 wenforce(h != INVALID_HANDLE_VALUE); 2002 2003 if (append) 2004 h.SetFilePointer(0, null, FILE_END); 2005 2006 f.windowsHandleOpen(h, cMode); 2007 } 2008 else 2009 f.open(fn, mode); 2010 return f; 2011 } 2012 2013 unittest 2014 { 2015 enum Existence { any, mustExist, mustNotExist } 2016 enum Pos { none /* not readable/writable */, start, end, empty } 2017 static struct Behavior 2018 { 2019 Existence existence; 2020 bool truncating; 2021 Pos read, write; 2022 } 2023 2024 void test(string mode, in Behavior expected) 2025 { 2026 static if (isVersion!q{CRuntime_Microsoft} || isVersion!q{OSX}) 2027 if (mode == "a+" || mode == "a+x") 2028 return; 2029 2030 Behavior behavior; 2031 2032 static int counter; 2033 auto fn = text(deleteme, counter++); 2034 2035 collectException(fn.remove()); 2036 bool mustExist = !!collectException(openFile(fn, mode)); 2037 touch(fn); 2038 bool mustNotExist = !!collectException(openFile(fn, mode)); 2039 2040 if (!mustExist) 2041 if (!mustNotExist) 2042 behavior.existence = Existence.any; 2043 else 2044 behavior.existence = Existence.mustNotExist; 2045 else 2046 if (!mustNotExist) 2047 behavior.existence = Existence.mustExist; 2048 else 2049 assert(false, "Can't open file whether it exists or not"); 2050 2051 void create() 2052 { 2053 if (mustNotExist) 2054 collectException(fn.remove()); 2055 else 2056 write(fn, "foo"); 2057 } 2058 2059 create(); 2060 openFile(fn, mode); 2061 behavior.truncating = getSize(fn) == 0; 2062 2063 create(); 2064 { 2065 auto f = openFile(fn, mode); 2066 ubyte[] buf; 2067 if (collectException(f.rawRead(new ubyte[1]), buf)) 2068 { 2069 behavior.read = Pos.none; 2070 // Work around https://issues.dlang.org/show_bug.cgi?id=19751 2071 f.reopen(fn, "w"); 2072 } 2073 else 2074 if (buf.length) 2075 behavior.read = Pos.start; 2076 else 2077 if (f.size) 2078 behavior.read = Pos.end; 2079 else 2080 behavior.read = Pos.empty; 2081 } 2082 2083 create(); 2084 { 2085 string s; 2086 { 2087 auto f = openFile(fn, mode); 2088 if (collectException(f.rawWrite("b"))) 2089 { 2090 s = null; 2091 // Work around https://issues.dlang.org/show_bug.cgi?id=19751 2092 f.reopen(fn, "w"); 2093 } 2094 else 2095 { 2096 f.close(); 2097 s = fn.readText; 2098 } 2099 } 2100 2101 if (s is null) 2102 behavior.write = Pos.none; 2103 else 2104 if (s == "b") 2105 behavior.write = Pos.empty; 2106 else 2107 if (s.endsWith("b")) 2108 behavior.write = Pos.end; 2109 else 2110 if (s.startsWith("b")) 2111 behavior.write = Pos.start; 2112 else 2113 assert(false, "Can't detect write position"); 2114 } 2115 2116 2117 if (behavior != expected) 2118 { 2119 import ae.utils.array : isOneOf; 2120 version (Windows) 2121 if (getWineVersion() && mode.isOneOf("w", "a", "wx", "ax")) 2122 { 2123 // Ignore bug in Wine msvcrt implementation 2124 return; 2125 } 2126 2127 assert(false, text(mode, ": expected ", expected, ", got ", behavior)); 2128 } 2129 } 2130 2131 test("r" , Behavior(Existence.mustExist , false, Pos.start, Pos.none )); 2132 test("r+" , Behavior(Existence.mustExist , false, Pos.start, Pos.start)); 2133 test("w" , Behavior(Existence.any , true , Pos.none , Pos.empty)); 2134 test("w+" , Behavior(Existence.any , true , Pos.empty, Pos.empty)); 2135 test("a" , Behavior(Existence.any , false, Pos.none , Pos.end )); 2136 test("a+" , Behavior(Existence.any , false, Pos.start, Pos.end )); 2137 test("wx" , Behavior(Existence.mustNotExist, true , Pos.none , Pos.empty)); 2138 test("w+x", Behavior(Existence.mustNotExist, true , Pos.empty, Pos.empty)); 2139 test("ax" , Behavior(Existence.mustNotExist, true , Pos.none , Pos.empty)); 2140 test("a+x", Behavior(Existence.mustNotExist, true , Pos.empty, Pos.empty)); 2141 } 2142 2143 private version(Windows) 2144 { 2145 version (CRuntime_Microsoft) 2146 { 2147 alias chsize_size_t = long; 2148 extern(C) int _chsize_s(int fd, chsize_size_t size); 2149 alias chsize = _chsize_s; 2150 } 2151 else 2152 { 2153 import core.stdc.config : c_long; 2154 alias chsize_size_t = c_long; 2155 extern(C) int chsize(int fd, c_long size); 2156 } 2157 } 2158 2159 /// Truncate the given file to the given size. 2160 void truncate(File f, ulong length) 2161 { 2162 f.flush(); 2163 version (Windows) 2164 chsize(f.fileno, length.to!chsize_size_t); 2165 else 2166 ftruncate(f.fileno, length.to!off_t); 2167 } 2168 2169 unittest 2170 { 2171 write("test.txt", "abcde"); 2172 auto f = File("test.txt", "r+b"); 2173 f.write("xyz"); 2174 f.truncate(f.tell); 2175 f.close(); 2176 assert("test.txt".readText == "xyz"); 2177 } 2178 2179 /// Calculate the digest of a file. 2180 auto fileDigest(Digest)(string fn) 2181 { 2182 import std.range.primitives; 2183 Digest context; 2184 context.start(); 2185 put(context, openFile(fn, "rb").byChunk(64 * 1024)); 2186 auto digest = context.finish(); 2187 return digest; 2188 } 2189 2190 /// Calculate the MD5 hash of a file. 2191 template mdFile() 2192 { 2193 import std.digest.md; 2194 alias mdFile = fileDigest!MD5; 2195 } 2196 2197 version (HAVE_WIN32) 2198 unittest 2199 { 2200 import std.digest : toHexString; 2201 write("test.txt", "Hello, world!"); 2202 scope(exit) remove("test.txt"); 2203 assert(mdFile("test.txt").toHexString() == "6CD3556DEB0DA54BCA060B4C39479839"); 2204 } 2205 2206 /// Calculate the digest of a file, and memoize it. 2207 auto fileDigestCached(Digest)(string fn) 2208 { 2209 static typeof(Digest.init.finish())[ulong] cache; 2210 auto id = getFileID(fn); 2211 auto phash = id in cache; 2212 if (phash) 2213 return *phash; 2214 return cache[id] = fileDigest!Digest(fn); 2215 } 2216 2217 /// Calculate the MD5 hash of a file, and memoize it. 2218 template mdFileCached() 2219 { 2220 import std.digest.md; 2221 alias mdFileCached = fileDigestCached!MD5; 2222 } 2223 2224 version (HAVE_WIN32) 2225 unittest 2226 { 2227 import std.digest : toHexString; 2228 write("test.txt", "Hello, world!"); 2229 scope(exit) remove("test.txt"); 2230 assert(mdFileCached("test.txt").toHexString() == "6CD3556DEB0DA54BCA060B4C39479839"); 2231 write("test.txt", "Something else"); 2232 assert(mdFileCached("test.txt").toHexString() == "6CD3556DEB0DA54BCA060B4C39479839"); 2233 } 2234 2235 /// Read a File (which might be a stream) into an array 2236 void[] readFile(File f) 2237 { 2238 import std.range.primitives; 2239 auto result = appender!(ubyte[]); 2240 put(result, f.byChunk(64*1024)); 2241 return result.data; 2242 } 2243 2244 unittest 2245 { 2246 auto s = "0123456789".replicate(10_000); 2247 write("test.txt", s); 2248 scope(exit) remove("test.txt"); 2249 assert(readFile(File("test.txt")) == s); 2250 } 2251 2252 /// Read exactly `buf.length` bytes and return true. 2253 /// On EOF, return false. 2254 bool readExactly(ref File f, ubyte[] buf) 2255 { 2256 if (!buf.length) 2257 return true; 2258 auto read = f.rawRead(buf); 2259 if (read.length==0) return false; 2260 enforce(read.length == buf.length, "Unexpected end of file"); 2261 return true; 2262 } 2263 2264 private 2265 version (Windows) 2266 { 2267 version (CRuntime_DigitalMars) 2268 extern(C) sizediff_t read(int, void*, size_t); 2269 else 2270 { 2271 extern(C) sizediff_t _read(int, void*, size_t); 2272 alias read = _read; 2273 } 2274 } 2275 else 2276 import core.sys.posix.unistd : read; 2277 2278 /// Like `File.rawRead`, but returns as soon as any data is available. 2279 void[] readPartial(File f, void[] buf) 2280 { 2281 assert(buf.length); 2282 auto numRead = read(f.fileno, buf.ptr, buf.length); 2283 errnoEnforce(numRead >= 0); 2284 return buf[0 .. numRead]; 2285 } 2286 2287 /// Like std.file.readText for non-UTF8 2288 ascii readAscii()(string fileName) 2289 { 2290 return cast(ascii)readFile(openFile(fileName, "rb")); 2291 } 2292 2293 // http://d.puremagic.com/issues/show_bug.cgi?id=7016 2294 version(Posix) static import ae.sys.signals; 2295 2296 /// Start a thread which writes data to f asynchronously. 2297 Thread writeFileAsync(File f, in void[] data) 2298 { 2299 static class Writer : Thread 2300 { 2301 File target; 2302 const void[] data; 2303 2304 this(ref File f, in void[] data) 2305 { 2306 this.target = f; 2307 this.data = data; 2308 super(&run); 2309 } 2310 2311 void run() 2312 { 2313 version (Posix) 2314 { 2315 import ae.sys.signals; 2316 collectSignal(SIGPIPE, &write); 2317 } 2318 else 2319 write(); 2320 } 2321 2322 void write() 2323 { 2324 target.rawWrite(data); 2325 target.close(); 2326 } 2327 } 2328 2329 auto t = new Writer(f, data); 2330 t.start(); 2331 return t; 2332 } 2333 2334 /// Write data to a file, and ensure it gets written to disk 2335 /// before this function returns. 2336 /// Consider using as atomic!syncWrite. 2337 /// See also: syncUpdate 2338 void syncWrite()(string target, in void[] data) 2339 { 2340 auto f = File(target, "wb"); 2341 f.rawWrite(data); 2342 version (Windows) 2343 { 2344 mixin(importWin32!q{windows}); 2345 FlushFileBuffers(f.windowsHandle); 2346 } 2347 else 2348 { 2349 import core.sys.posix.unistd; 2350 fsync(f.fileno); 2351 } 2352 f.close(); 2353 } 2354 2355 /// Atomically save data to a file (if the file doesn't exist, 2356 /// or its contents differs). The update operation as a whole 2357 /// is not atomic, only the write is. 2358 void syncUpdate()(string fn, in void[] data) 2359 { 2360 if (!fn.exists || fn.read() != data) 2361 atomic!(syncWrite!())(fn, data); 2362 } 2363 2364 version(Windows) import ae.sys.windows.exception; 2365 2366 /// Create a named pipe, and allow interacting with it using a `std.stdio.File`. 2367 struct NamedPipeImpl 2368 { 2369 immutable string fileName; /// 2370 2371 /// Create a named pipe, and reserve a filename. 2372 this()(string name) 2373 { 2374 version(Windows) 2375 { 2376 mixin(importWin32!q{winbase}); 2377 2378 fileName = `\\.\pipe\` ~ name; 2379 auto h = CreateNamedPipeW(fileName.toUTF16z, PIPE_ACCESS_OUTBOUND, PIPE_TYPE_BYTE, 10, 4096, 4096, 0, null).wenforce("CreateNamedPipeW"); 2380 f.windowsHandleOpen(h, "wb"); 2381 } 2382 else 2383 { 2384 import core.sys.posix.sys.stat; 2385 2386 fileName = `/tmp/` ~ name ~ `.fifo`; 2387 mkfifo(fileName.toStringz, S_IWUSR | S_IRUSR); 2388 } 2389 } 2390 2391 /// Wait for a peer to open the other end of the pipe. 2392 File connect()() 2393 { 2394 version(Windows) 2395 { 2396 mixin(importWin32!q{winbase}); 2397 mixin(importWin32!q{windef}); 2398 2399 BOOL bSuccess = ConnectNamedPipe(f.windowsHandle, null); 2400 2401 // "If a client connects before the function is called, the function returns zero 2402 // and GetLastError returns ERROR_PIPE_CONNECTED. This can happen if a client 2403 // connects in the interval between the call to CreateNamedPipe and the call to 2404 // ConnectNamedPipe. In this situation, there is a good connection between client 2405 // and server, even though the function returns zero." 2406 if (!bSuccess) 2407 wenforce(GetLastError() == ERROR_PIPE_CONNECTED, "ConnectNamedPipe"); 2408 2409 return f; 2410 } 2411 else 2412 { 2413 return File(fileName, "w"); 2414 } 2415 } 2416 2417 ~this() 2418 { 2419 version(Windows) 2420 { 2421 // File.~this will take care of cleanup 2422 } 2423 else 2424 fileName.remove(); 2425 } 2426 2427 private: 2428 File f; 2429 } 2430 alias NamedPipe = RefCounted!NamedPipeImpl; /// ditto 2431 2432 import ae.utils.textout : StringBuilder; 2433 2434 /// Avoid std.stdio.File.readln's memory corruption bug 2435 /// https://issues.dlang.org/show_bug.cgi?id=13856 2436 string safeReadln(File f) 2437 { 2438 StringBuilder buf; 2439 char[1] arr; 2440 while (true) 2441 { 2442 auto result = f.rawRead(arr[]); 2443 if (!result.length) 2444 break; 2445 buf.put(result); 2446 if (result[0] == '\x0A') 2447 break; 2448 } 2449 return buf.get(); 2450 } 2451 2452 // **************************************************************************** 2453 2454 /// Change the current directory to the given directory. Does nothing if dir is null. 2455 /// Return a scope guard which, upon destruction, restores the previous directory. 2456 /// Asserts that only one thread has changed the process's current directory at any time. 2457 auto pushd(string dir) 2458 { 2459 import core.atomic; 2460 2461 static int threadCount = 0; 2462 static shared int processCount = 0; 2463 2464 static struct Popd 2465 { 2466 string oldPath; 2467 this(string cwd) { oldPath = cwd; } 2468 ~this() { if (oldPath) pop(); } 2469 @disable this(); 2470 @disable this(this); 2471 2472 void pop() 2473 { 2474 assert(oldPath); 2475 scope(exit) oldPath = null; 2476 chdir(oldPath); 2477 2478 auto newThreadCount = --threadCount; 2479 auto newProcessCount = atomicOp!"-="(processCount, 1); 2480 assert(newThreadCount == newProcessCount); // Shouldn't happen 2481 } 2482 } 2483 2484 string cwd; 2485 if (dir) 2486 { 2487 auto newThreadCount = ++threadCount; 2488 auto newProcessCount = atomicOp!"+="(processCount, 1); 2489 assert(newThreadCount == newProcessCount, "Another thread already has an active pushd"); 2490 2491 cwd = getcwd(); 2492 chdir(dir); 2493 } 2494 return Popd(cwd); 2495 } 2496 2497 // **************************************************************************** 2498 2499 import std.algorithm; 2500 import std.process : thisProcessID; 2501 import std.traits; 2502 import std.typetuple; 2503 import ae.utils.meta; 2504 2505 /// Parameter names that `atomic` assumes 2506 /// indicate a destination file by default. 2507 enum targetParameterNames = "target/to/name/dst"; 2508 2509 /// Wrap an operation which creates a file or directory, 2510 /// so that it is created safely and, for files, atomically 2511 /// (by performing the underlying operation to a temporary 2512 /// location, then renaming the completed file/directory to 2513 /// the actual target location). targetName specifies the name 2514 /// of the parameter containing the target file/directory. 2515 auto atomic(alias impl, string targetName = targetParameterNames)(staticMap!(Unqual, ParameterTypeTuple!impl) args) 2516 { 2517 enum targetIndex = findParameter([ParameterIdentifierTuple!impl], targetName, __traits(identifier, impl)); 2518 return atomic!(impl, targetIndex)(args); 2519 } 2520 2521 /// ditto 2522 auto atomic(alias impl, size_t targetIndex)(staticMap!(Unqual, ParameterTypeTuple!impl) args) 2523 { 2524 // idup for https://d.puremagic.com/issues/show_bug.cgi?id=12503 2525 auto target = args[targetIndex].idup; 2526 auto temp = "%s.%s.%s.temp".format(target, thisProcessID, getCurrentThreadID); 2527 if (temp.exists) temp.removeRecurse(); 2528 scope(success) rename(temp, target); 2529 scope(failure) if (temp.exists) temp.removeRecurse(); 2530 args[targetIndex] = temp; 2531 return impl(args); 2532 } 2533 2534 /// ditto 2535 // Workaround for https://d.puremagic.com/issues/show_bug.cgi?id=12230 2536 // Can't be an overload because of https://issues.dlang.org/show_bug.cgi?id=13374 2537 //R atomicDg(string targetName = "target", R, Args...)(R delegate(Args) impl, staticMap!(Unqual, Args) args) 2538 auto atomicDg(size_t targetIndexA = size_t.max, Impl, Args...)(Impl impl, Args args) 2539 { 2540 enum targetIndex = targetIndexA == size_t.max ? ParameterTypeTuple!impl.length-1 : targetIndexA; 2541 return atomic!(impl, targetIndex)(args); 2542 } 2543 2544 deprecated alias safeUpdate = atomic; 2545 2546 unittest 2547 { 2548 enum fn = "atomic.tmp"; 2549 scope(exit) if (fn.exists) fn.remove(); 2550 2551 atomic!touch(fn); 2552 assert(fn.exists); 2553 fn.remove(); 2554 2555 atomicDg(&touch, fn); 2556 assert(fn.exists); 2557 } 2558 2559 /// Wrap an operation so that it is skipped entirely 2560 /// if the target already exists. Implies atomic. 2561 auto cached(alias impl, string targetName = targetParameterNames)(ParameterTypeTuple!impl args) 2562 { 2563 enum targetIndex = findParameter([ParameterIdentifierTuple!impl], targetName, __traits(identifier, impl)); 2564 auto target = args[targetIndex]; 2565 if (!target.exists) 2566 atomic!(impl, targetIndex)(args); 2567 return target; 2568 } 2569 2570 /// ditto 2571 // Exists due to the same reasons as atomicDg 2572 auto cachedDg(size_t targetIndexA = size_t.max, Impl, Args...)(Impl impl, Args args) 2573 { 2574 enum targetIndex = targetIndexA == size_t.max ? ParameterTypeTuple!impl.length-1 : targetIndexA; 2575 auto target = args[targetIndex]; 2576 if (!target.exists) 2577 atomic!(impl, targetIndex)(args); 2578 return target; 2579 } 2580 2581 deprecated alias obtainUsing = cached; 2582 2583 /// Create a file, or replace an existing file's contents 2584 /// atomically. 2585 /// Note: Consider using atomic!syncWrite or 2586 /// atomic!syncUpdate instead. 2587 alias atomic!writeProxy atomicWrite; 2588 deprecated alias safeWrite = atomicWrite; 2589 /*private*/ void writeProxy(string target, in void[] data) 2590 { 2591 std.file.write(target, data); 2592 } 2593 2594 // Work around for https://github.com/D-Programming-Language/phobos/pull/2784#issuecomment-68117241 2595 private void copy2(string source, string target) { std.file.copy(source, target); } 2596 2597 /// Copy a file, or replace an existing file's contents 2598 /// with another file's, atomically. 2599 alias atomic!copy2 atomicCopy; 2600 2601 unittest 2602 { 2603 enum fn = "cached.tmp"; 2604 scope(exit) if (fn.exists) fn.remove(); 2605 2606 cached!touch(fn); 2607 assert(fn.exists); 2608 2609 std.file.write(fn, "test"); 2610 2611 cachedDg!0(&writeProxy, fn, "test2"); 2612 assert(fn.readText() == "test"); 2613 } 2614 2615 // **************************************************************************** 2616 2617 /// Composes a function which generates a file name 2618 /// with a function which creates the file. 2619 /// Returns the file name. 2620 template withTarget(alias targetGen, alias fun) 2621 { 2622 auto withTarget(Args...)(auto ref Args args) 2623 { 2624 auto target = targetGen(args); 2625 fun(args, target); 2626 return target; 2627 } 2628 } 2629 2630 /// Two-argument buildPath with reversed arguments. 2631 /// Useful for UFCS chaining. 2632 string prependPath(string target, string path) 2633 { 2634 return buildPath(path, target); 2635 }