1 /** 2 * File stuff 3 * 4 * License: 5 * This Source Code Form is subject to the terms of 6 * the Mozilla Public License, v. 2.0. If a copy of 7 * the MPL was not distributed with this file, You 8 * can obtain one at http://mozilla.org/MPL/2.0/. 9 * 10 * Authors: 11 * Vladimir Panteleev <vladimir@thecybershadow.net> 12 */ 13 14 module ae.sys.file; 15 16 import core.stdc.wchar_; 17 import core.thread; 18 19 import std.array; 20 import std.conv; 21 import std.file; 22 import std.path; 23 import std.stdio : File; 24 import std.string; 25 import std.typecons; 26 import std.utf; 27 28 import ae.sys.cmd : getCurrentThreadID; 29 import ae.utils.path; 30 31 public import std.typecons : No, Yes; 32 33 alias wcscmp = core.stdc.wchar_.wcscmp; 34 alias wcslen = core.stdc.wchar_.wcslen; 35 36 version(Windows) import ae.sys.windows.imports; 37 38 // ************************************************************************ 39 40 version (Windows) 41 { 42 // Work around std.file overload 43 mixin(importWin32!(q{winnt}, null, q{FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT})); 44 } 45 46 // ************************************************************************ 47 48 version(Windows) 49 { 50 string[] fastListDir(bool recursive = false, bool symlinks=false)(string pathname, string pattern = null) 51 { 52 import core.sys.windows.windows; 53 54 static if (recursive) 55 enforce(!pattern, "TODO: recursive fastListDir with pattern"); 56 57 string[] result; 58 string c; 59 HANDLE h; 60 61 c = buildPath(pathname, pattern ? pattern : "*.*"); 62 WIN32_FIND_DATAW fileinfo; 63 64 h = FindFirstFileW(toUTF16z(c), &fileinfo); 65 if (h != INVALID_HANDLE_VALUE) 66 { 67 scope(exit) FindClose(h); 68 69 do 70 { 71 // Skip "." and ".." 72 if (wcscmp(fileinfo.cFileName.ptr, ".") == 0 || 73 wcscmp(fileinfo.cFileName.ptr, "..") == 0) 74 continue; 75 76 static if (!symlinks) 77 { 78 if (fileinfo.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) 79 continue; 80 } 81 82 size_t clength = wcslen(fileinfo.cFileName.ptr); 83 string name = std.utf.toUTF8(fileinfo.cFileName[0 .. clength]); 84 string path = buildPath(pathname, name); 85 86 static if (recursive) 87 { 88 if (fileinfo.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) 89 { 90 result ~= fastListDir!recursive(path); 91 continue; 92 } 93 } 94 95 result ~= path; 96 } while (FindNextFileW(h,&fileinfo) != FALSE); 97 } 98 return result; 99 } 100 } 101 else 102 version (Posix) 103 { 104 private import core.stdc.errno; 105 private import core.sys.posix.dirent; 106 private import core.stdc.string; 107 108 string[] fastListDir(bool recursive=false, bool symlinks=false)(string pathname, string pattern = null) 109 { 110 string[] result; 111 DIR* h; 112 dirent* fdata; 113 114 h = opendir(toStringz(pathname)); 115 if (h) 116 { 117 try 118 { 119 while((fdata = readdir(h)) != null) 120 { 121 // Skip "." and ".." 122 if (!core.stdc..string.strcmp(fdata.d_name.ptr, ".") || 123 !core.stdc..string.strcmp(fdata.d_name.ptr, "..")) 124 continue; 125 126 static if (!symlinks) 127 { 128 if (fdata.d_type == DT_LNK) 129 continue; 130 } 131 132 size_t len = core.stdc..string.strlen(fdata.d_name.ptr); 133 string name = fdata.d_name[0 .. len].idup; 134 if (pattern && !globMatch(name, pattern)) 135 continue; 136 string path = pathname ~ (pathname.length && pathname[$-1] != '/' ? "/" : "") ~ name; 137 138 static if (recursive) 139 { 140 if (fdata.d_type & DT_DIR) 141 { 142 result ~= fastListDir!(recursive, symlinks)(path); 143 continue; 144 } 145 } 146 147 result ~= path; 148 } 149 } 150 finally 151 { 152 closedir(h); 153 } 154 } 155 else 156 { 157 throw new std.file.FileException(pathname, errno); 158 } 159 return result; 160 } 161 } 162 else 163 static assert(0, "TODO"); 164 165 // ************************************************************************ 166 167 string buildPath2(string[] segments...) { return segments.length ? buildPath(segments) : null; } 168 169 /// Shell-like expansion of ?, * and ** in path components 170 DirEntry[] fileList(string pattern) 171 { 172 auto components = cast(string[])array(pathSplitter(pattern)); 173 foreach (i, component; components[0..$-1]) 174 if (component.contains("?") || component.contains("*")) // TODO: escape? 175 { 176 DirEntry[] expansions; // TODO: filter range instead? 177 auto dir = buildPath2(components[0..i]); 178 if (component == "**") 179 expansions = array(dirEntries(dir, SpanMode.depth)); 180 else 181 expansions = array(dirEntries(dir, component, SpanMode.shallow)); 182 183 DirEntry[] result; 184 foreach (expansion; expansions) 185 if (expansion.isDir()) 186 result ~= fileList(buildPath(expansion.name ~ components[i+1..$])); 187 return result; 188 } 189 190 auto dir = buildPath2(components[0..$-1]); 191 if (!dir || exists(dir)) 192 return array(dirEntries(dir, components[$-1], SpanMode.shallow)); 193 else 194 return null; 195 } 196 197 /// ditto 198 DirEntry[] fileList(string pattern0, string[] patterns...) 199 { 200 DirEntry[] result; 201 foreach (pattern; [pattern0] ~ patterns) 202 result ~= fileList(pattern); 203 return result; 204 } 205 206 /// ditto 207 string[] fastFileList(string pattern) 208 { 209 auto components = cast(string[])array(pathSplitter(pattern)); 210 foreach (i, component; components[0..$-1]) 211 if (component.contains("?") || component.contains("*")) // TODO: escape? 212 { 213 string[] expansions; // TODO: filter range instead? 214 auto dir = buildPath2(components[0..i]); 215 if (component == "**") 216 expansions = fastListDir!true(dir); 217 else 218 expansions = fastListDir(dir, component); 219 220 string[] result; 221 foreach (expansion; expansions) 222 if (expansion.isDir()) 223 result ~= fastFileList(buildPath(expansion ~ components[i+1..$])); 224 return result; 225 } 226 227 auto dir = buildPath2(components[0..$-1]); 228 if (!dir || exists(dir)) 229 return fastListDir(dir, components[$-1]); 230 else 231 return null; 232 } 233 234 /// ditto 235 string[] fastFileList(string pattern0, string[] patterns...) 236 { 237 string[] result; 238 foreach (pattern; [pattern0] ~ patterns) 239 result ~= fastFileList(pattern); 240 return result; 241 } 242 243 // ************************************************************************ 244 245 import std.datetime; 246 import std.exception; 247 248 deprecated SysTime getMTime(string name) 249 { 250 return timeLastModified(name); 251 } 252 253 /// If target exists, update its modification time; 254 /// otherwise create it as an empty file. 255 void touch(in char[] target) 256 { 257 if (exists(target)) 258 { 259 auto now = Clock.currTime(); 260 setTimes(target, now, now); 261 } 262 else 263 std.file.write(target, ""); 264 } 265 266 /// Returns true if the target file doesn't exist, 267 /// or source is newer than the target. 268 bool newerThan(string source, string target) 269 { 270 if (!target.exists) 271 return true; 272 return source.timeLastModified() > target.timeLastModified(); 273 } 274 275 /// Returns true if the target file doesn't exist, 276 /// or any of the sources are newer than the target. 277 bool anyNewerThan(string[] sources, string target) 278 { 279 if (!target.exists) 280 return true; 281 auto targetTime = target.timeLastModified(); 282 return sources.any!(source => source.timeLastModified() > targetTime)(); 283 } 284 285 version (Posix) 286 { 287 import core.sys.posix.sys.stat; 288 import core.sys.posix.unistd; 289 290 int getOwner(string fn) 291 { 292 stat_t s; 293 errnoEnforce(stat(toStringz(fn), &s) == 0, "stat: " ~ fn); 294 return s.st_uid; 295 } 296 297 int getGroup(string fn) 298 { 299 stat_t s; 300 errnoEnforce(stat(toStringz(fn), &s) == 0, "stat: " ~ fn); 301 return s.st_gid; 302 } 303 304 void setOwner(string fn, int uid, int gid) 305 { 306 errnoEnforce(chown(toStringz(fn), uid, gid) == 0, "chown: " ~ fn); 307 } 308 } 309 310 /// Try to rename; copy/delete if rename fails 311 void move(string src, string dst) 312 { 313 try 314 src.rename(dst); 315 catch (Exception e) 316 { 317 atomicCopy(src, dst); 318 src.remove(); 319 } 320 } 321 322 /// Make sure that the given directory exists 323 /// (and create parent directories as necessary). 324 void ensureDirExists(string path) 325 { 326 if (!path.exists) 327 path.mkdirRecurse(); 328 } 329 330 /// Make sure that the path to the given file name 331 /// exists (and create directories as necessary). 332 void ensurePathExists(string fn) 333 { 334 fn.dirName.ensureDirExists(); 335 } 336 337 import ae.utils.text; 338 339 /// Forcibly remove a file or directory. 340 /// If atomic is true, the entire directory is deleted "atomically" 341 /// (it is first moved/renamed to another location). 342 /// On Windows, this will move the file/directory out of the way, 343 /// if it is in use and cannot be deleted (but can be renamed). 344 void forceDelete(Flag!"atomic" atomic=Yes.atomic)(string fn, Flag!"recursive" recursive = No.recursive) 345 { 346 import std.process : environment; 347 version(Windows) 348 { 349 mixin(importWin32!q{winnt}); 350 mixin(importWin32!q{winbase}); 351 } 352 353 auto name = fn.baseName(); 354 fn = fn.absolutePath().longPath(); 355 356 version(Windows) 357 { 358 auto fnW = toUTF16z(fn); 359 auto attr = GetFileAttributesW(fnW); 360 wenforce(attr != INVALID_FILE_ATTRIBUTES, "GetFileAttributes"); 361 if (attr & FILE_ATTRIBUTE_READONLY) 362 SetFileAttributesW(fnW, attr & ~FILE_ATTRIBUTE_READONLY).wenforce("SetFileAttributes"); 363 } 364 365 static if (atomic) 366 { 367 // To avoid zombifying locked directories, try renaming it first. 368 // Attempting to delete a locked directory will make it inaccessible. 369 370 bool tryMoveTo(string target) 371 { 372 target = target.longPath(); 373 if (target.endsWith(dirSeparator)) 374 target = target[0..$-1]; 375 if (target.length && !target.exists) 376 return false; 377 378 string newfn; 379 do 380 newfn = format("%s%sdeleted-%s.%s.%s", target, dirSeparator, name, thisProcessID, randomString()); 381 while (newfn.exists); 382 383 version(Windows) 384 { 385 auto newfnW = toUTF16z(newfn); 386 if (!MoveFileW(fnW, newfnW)) 387 return false; 388 } 389 else 390 { 391 try 392 rename(fn, newfn); 393 catch (FileException e) 394 return false; 395 } 396 397 fn = newfn; 398 version(Windows) fnW = newfnW; 399 return true; 400 } 401 402 void tryMove() 403 { 404 auto tmp = environment.get("TEMP"); 405 if (tmp) 406 if (tryMoveTo(tmp)) 407 return; 408 409 version(Windows) 410 string tempDir = fn[0..7]~"Temp"; 411 else 412 enum tempDir = "/tmp"; 413 414 if (tryMoveTo(tempDir)) 415 return; 416 417 if (tryMoveTo(fn.dirName())) 418 return; 419 420 throw new Exception("Unable to delete " ~ fn ~ " atomically (all rename attempts failed)"); 421 } 422 423 tryMove(); 424 } 425 426 version(Windows) 427 { 428 if (attr & FILE_ATTRIBUTE_DIRECTORY) 429 { 430 if (recursive && (attr & FILE_ATTRIBUTE_REPARSE_POINT) == 0) 431 { 432 foreach (de; fn.dirEntries(SpanMode.shallow)) 433 forceDelete!(No.atomic)(de.name, Yes.recursive); 434 } 435 // Will fail if !recursive and directory is not empty 436 RemoveDirectoryW(fnW).wenforce("RemoveDirectory"); 437 } 438 else 439 DeleteFileW(fnW).wenforce("DeleteFile"); 440 } 441 else 442 { 443 if (recursive) 444 fn.removeRecurse(); 445 else 446 if (fn.isDir) 447 fn.rmdir(); 448 else 449 fn.remove(); 450 } 451 } 452 453 454 deprecated void forceDelete(bool atomic)(string fn, bool recursive = false) { forceDelete!(cast(Flag!"atomic")atomic)(fn, cast(Flag!"recursive")recursive); } 455 //deprecated void forceDelete()(string fn, bool recursive) { forceDelete!(Yes.atomic)(fn, cast(Flag!"recursive")recursive); } 456 457 deprecated unittest 458 { 459 mkdir("testdir"); touch("testdir/b"); forceDelete!(false )("testdir", true); 460 mkdir("testdir"); touch("testdir/b"); forceDelete!(true )("testdir", true); 461 } 462 463 unittest 464 { 465 mkdir("testdir"); touch("testdir/b"); forceDelete ("testdir", Yes.recursive); 466 mkdir("testdir"); touch("testdir/b"); forceDelete!(No .atomic)("testdir", Yes.recursive); 467 mkdir("testdir"); touch("testdir/b"); forceDelete!(Yes.atomic)("testdir", Yes.recursive); 468 } 469 470 /// If fn is a directory, delete it recursively. 471 /// Otherwise, delete the file or symlink fn. 472 void removeRecurse(string fn) 473 { 474 auto attr = fn.getAttributes(); 475 if (attr.attrIsSymlink) 476 { 477 version (Windows) 478 if (attr.attrIsDir) 479 fn.rmdir(); 480 else 481 fn.remove(); 482 else 483 fn.remove(); 484 } 485 else 486 if (attr.attrIsDir) 487 version (Windows) 488 fn.forceDelete!(No.atomic)(Yes.recursive); // For read-only files 489 else 490 fn.rmdirRecurse(); 491 else 492 fn.remove(); 493 } 494 495 /// Create an empty directory, deleting 496 /// all its contents if it already exists. 497 void recreateEmptyDirectory()(string dir) 498 { 499 if (dir.exists) 500 dir.forceDelete(Yes.recursive); 501 mkdir(dir); 502 } 503 504 bool isHidden()(string fn) 505 { 506 if (baseName(fn).startsWith(".")) 507 return true; 508 version (Windows) 509 { 510 mixin(importWin32!q{winnt}); 511 if (getAttributes(fn) & FILE_ATTRIBUTE_HIDDEN) 512 return true; 513 } 514 return false; 515 } 516 517 /// Return a file's unique ID. 518 ulong getFileID()(string fn) 519 { 520 version (Windows) 521 { 522 mixin(importWin32!q{winnt}); 523 mixin(importWin32!q{winbase}); 524 525 auto fnW = toUTF16z(fn); 526 auto h = CreateFileW(fnW, FILE_READ_ATTRIBUTES, 0, null, OPEN_EXISTING, 0, HANDLE.init); 527 wenforce(h!=INVALID_HANDLE_VALUE, fn); 528 scope(exit) CloseHandle(h); 529 BY_HANDLE_FILE_INFORMATION fi; 530 GetFileInformationByHandle(h, &fi).wenforce("GetFileInformationByHandle"); 531 532 ULARGE_INTEGER li; 533 li.LowPart = fi.nFileIndexLow; 534 li.HighPart = fi.nFileIndexHigh; 535 auto result = li.QuadPart; 536 enforce(result, "Null file ID"); 537 return result; 538 } 539 else 540 { 541 return DirEntry(fn).statBuf.st_ino; 542 } 543 } 544 545 unittest 546 { 547 touch("a"); 548 scope(exit) remove("a"); 549 hardLink("a", "b"); 550 scope(exit) remove("b"); 551 touch("c"); 552 scope(exit) remove("c"); 553 assert(getFileID("a") == getFileID("b")); 554 assert(getFileID("a") != getFileID("c")); 555 } 556 557 deprecated alias std.file.getSize getSize2; 558 559 /// Using UNC paths bypasses path length limitation when using Windows wide APIs. 560 string longPath(string s) 561 { 562 version (Windows) 563 { 564 if (!s.startsWith(`\\`)) 565 return `\\?\` ~ s.absolutePath().buildNormalizedPath().replace(`/`, `\`); 566 } 567 return s; 568 } 569 570 version (Windows) 571 { 572 static if (__traits(compiles, { mixin importWin32!q{winnt}; })) 573 static mixin(importWin32!q{winnt}); 574 575 void createReparsePoint(string reparseBufferName, string extraInitialization, string reparseTagName)(in char[] target, in char[] print, in char[] link) 576 { 577 mixin(importWin32!q{winbase}); 578 mixin(importWin32!q{windef}); 579 mixin(importWin32!q{winioctl}); 580 581 enum SYMLINK_FLAG_RELATIVE = 1; 582 583 HANDLE hLink = CreateFileW(link.toUTF16z(), GENERIC_READ | GENERIC_WRITE, 0, null, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, null); 584 wenforce(hLink && hLink != INVALID_HANDLE_VALUE, "CreateFileW"); 585 scope(exit) CloseHandle(hLink); 586 587 enum pathOffset = 588 mixin(q{REPARSE_DATA_BUFFER..} ~ reparseBufferName) .offsetof + 589 mixin(q{REPARSE_DATA_BUFFER..} ~ reparseBufferName)._PathBuffer.offsetof; 590 591 auto targetW = target.toUTF16(); 592 auto printW = print .toUTF16(); 593 594 // Despite MSDN, two NUL-terminating characters are needed, one for each string. 595 596 auto pathBufferSize = targetW.length + 1 + printW.length + 1; // in chars 597 auto buf = new ubyte[pathOffset + pathBufferSize * WCHAR.sizeof]; 598 auto r = cast(REPARSE_DATA_BUFFER*)buf.ptr; 599 600 r.ReparseTag = mixin(reparseTagName); 601 r.ReparseDataLength = to!WORD(buf.length - mixin(q{r..} ~ reparseBufferName).offsetof); 602 603 auto pathBuffer = mixin(q{r..} ~ reparseBufferName).PathBuffer; 604 auto p = pathBuffer; 605 606 mixin(q{r..} ~ reparseBufferName).SubstituteNameOffset = to!WORD((p-pathBuffer) * WCHAR.sizeof); 607 mixin(q{r..} ~ reparseBufferName).SubstituteNameLength = to!WORD(targetW.length * WCHAR.sizeof); 608 p[0..targetW.length] = targetW; 609 p += targetW.length; 610 *p++ = 0; 611 612 mixin(q{r..} ~ reparseBufferName).PrintNameOffset = to!WORD((p-pathBuffer) * WCHAR.sizeof); 613 mixin(q{r..} ~ reparseBufferName).PrintNameLength = to!WORD(printW .length * WCHAR.sizeof); 614 p[0..printW.length] = printW; 615 p += printW.length; 616 *p++ = 0; 617 618 assert(p-pathBuffer == pathBufferSize); 619 620 mixin(extraInitialization); 621 622 DWORD dwRet; // Needed despite MSDN 623 DeviceIoControl(hLink, FSCTL_SET_REPARSE_POINT, buf.ptr, buf.length.to!DWORD(), null, 0, &dwRet, null).wenforce("DeviceIoControl"); 624 } 625 626 void acquirePrivilege(S)(S name) 627 { 628 mixin(importWin32!q{winbase}); 629 mixin(importWin32!q{windef}); 630 631 import ae.sys.windows; 632 633 HANDLE hToken = null; 634 wenforce(OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &hToken)); 635 scope(exit) CloseHandle(hToken); 636 637 TOKEN_PRIVILEGES tp; 638 wenforce(LookupPrivilegeValue(null, name.toUTF16z(), &tp.Privileges[0].Luid), "LookupPrivilegeValue"); 639 640 tp.PrivilegeCount = 1; 641 tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; 642 wenforce(AdjustTokenPrivileges(hToken, FALSE, &tp, cast(DWORD)TOKEN_PRIVILEGES.sizeof, null, null), "AdjustTokenPrivileges"); 643 } 644 645 /// Link a directory. 646 /// Uses symlinks on POSIX, and directory junctions on Windows. 647 void dirLink()(in char[] original, in char[] link) 648 { 649 mkdir(link); 650 scope(failure) rmdir(link); 651 652 auto target = `\??\` ~ original.idup.absolutePath(); 653 if (target[$-1] != '\\') 654 target ~= '\\'; 655 656 createReparsePoint!(q{MountPointReparseBuffer}, q{}, q{IO_REPARSE_TAG_MOUNT_POINT})(target, null, link); 657 } 658 659 void symlink()(in char[] original, in char[] link) 660 { 661 mixin(importWin32!q{winnt}); 662 663 acquirePrivilege(SE_CREATE_SYMBOLIC_LINK_NAME); 664 665 touch(link); 666 scope(failure) remove(link); 667 668 createReparsePoint!(q{SymbolicLinkReparseBuffer}, q{r.SymbolicLinkReparseBuffer.Flags = link.isAbsolute() ? 0 : SYMLINK_FLAG_RELATIVE;}, q{IO_REPARSE_TAG_SYMLINK})(original, original, link); 669 } 670 } 671 else 672 alias std.file.symlink dirLink; 673 674 version(Windows) version(unittest) static mixin(importWin32!q{winnt}); 675 676 unittest 677 { 678 mkdir("a"); scope(exit) rmdir("a"[]); 679 touch("a/f"); scope(exit) remove("a/f"); 680 dirLink("a", "b"); scope(exit) version(Windows) rmdir("b"); else remove("b"); 681 //symlink("a/f", "c"); scope(exit) remove("c"); 682 assert("b".isSymlink()); 683 //assert("c".isSymlink()); 684 assert("b/f".exists()); 685 } 686 687 version (Windows) 688 { 689 void hardLink()(string src, string dst) 690 { 691 mixin(importWin32!q{w32api}); 692 693 static assert(_WIN32_WINNT >= 0x501, "CreateHardLinkW not available for target Windows platform. Specify -version=WindowsXP"); 694 695 mixin(importWin32!q{winnt}); 696 mixin(importWin32!q{winbase}); 697 698 wenforce(CreateHardLinkW(toUTF16z(dst), toUTF16z(src), null), "CreateHardLink failed: " ~ src ~ " -> " ~ dst); 699 } 700 } 701 version (Posix) 702 { 703 void hardLink()(string src, string dst) 704 { 705 import core.sys.posix.unistd; 706 enforce(link(toUTFz!(const char*)(src), toUTFz!(const char*)(dst)) == 0, "link() failed: " ~ dst); 707 } 708 } 709 710 version (Posix) 711 { 712 string realPath(string path) 713 { 714 // TODO: Windows version 715 import core.sys.posix.stdlib; 716 auto p = realpath(toUTFz!(const char*)(path), null); 717 errnoEnforce(p, "realpath"); 718 string result = fromStringz(p).idup; 719 free(p); 720 return result; 721 } 722 } 723 724 // /proc/self/mounts parsing 725 version (linux) 726 { 727 struct MountInfo 728 { 729 string spec; /// device path 730 string file; /// mount path 731 string vfstype; /// file system 732 string mntops; /// options 733 int freq; /// dump flag 734 int passno; /// fsck order 735 } 736 737 string unescapeMountString(in char[] s) 738 { 739 string result; 740 741 size_t p = 0; 742 for (size_t i=0; i+3<s.length;) 743 { 744 auto c = s[i]; 745 if (c == '\\') 746 { 747 result ~= s[p..i]; 748 result ~= to!int(s[i+1..i+4], 8); 749 i += 4; 750 p = i; 751 } 752 else 753 i++; 754 } 755 result ~= s[p..$]; 756 return result; 757 } 758 759 unittest 760 { 761 assert(unescapeMountString(`a\040b\040c`) == "a b c"); 762 assert(unescapeMountString(`\040`) == " "); 763 } 764 765 MountInfo parseMountInfo(in char[] line) 766 { 767 const(char)[][6] parts; 768 copy(line.splitter(" "), parts[]); 769 return MountInfo( 770 unescapeMountString(parts[0]), 771 unescapeMountString(parts[1]), 772 unescapeMountString(parts[2]), 773 unescapeMountString(parts[3]), 774 parts[4].to!int, 775 parts[5].to!int, 776 ); 777 } 778 779 /// Returns an iterator of MountInfo structs. 780 auto getMounts() 781 { 782 return File("/proc/self/mounts", "rb").byLine().map!parseMountInfo(); 783 } 784 785 /// Get MountInfo with longest mount point matching path. 786 /// Returns MountInfo.init if none match. 787 MountInfo getPathMountInfo(string path) 788 { 789 path = realPath(path); 790 size_t bestLength; MountInfo bestInfo; 791 foreach (ref info; getMounts()) 792 { 793 if (path.pathStartsWith(info.file)) 794 { 795 if (bestLength < info.file.length) 796 { 797 bestLength = info.file.length; 798 bestInfo = info; 799 } 800 } 801 } 802 return bestInfo; 803 } 804 805 /// Get the name of the filesystem that the given path is mounted under. 806 /// Returns null if none match. 807 string getPathFilesystem(string path) 808 { 809 return getPathMountInfo(path).vfstype; 810 } 811 } 812 813 // **************************************************************************** 814 815 version (linux) 816 { 817 import core.sys.linux.sys.xattr; 818 import core.stdc.errno; 819 alias ENOATTR = ENODATA; 820 821 /// AA-like object for accessing a file's extended attributes. 822 struct XAttrs(Obj, string funPrefix) 823 { 824 Obj obj; 825 826 mixin("alias getFun = " ~ funPrefix ~ "getxattr;"); 827 mixin("alias setFun = " ~ funPrefix ~ "setxattr;"); 828 mixin("alias removeFun = " ~ funPrefix ~ "removexattr;"); 829 mixin("alias listFun = " ~ funPrefix ~ "listxattr;"); 830 831 void[] opIndex(string key) 832 { 833 auto cKey = key.toStringz(); 834 auto size = getFun(obj, cKey, null, 0); 835 errnoEnforce(size >= 0); 836 auto result = new void[size]; 837 // TODO: race condition, retry 838 size = getFun(obj, cKey, result.ptr, result.length); 839 errnoEnforce(size == result.length); 840 return result; 841 } 842 843 bool opIn_r(string key) 844 { 845 auto cKey = key.toStringz(); 846 auto size = getFun(obj, cKey, null, 0); 847 if (size >= 0) 848 return true; 849 else 850 if (errno == ENOATTR) 851 return false; 852 else 853 errnoEnforce(false, "Error reading file xattrs"); 854 assert(false); 855 } 856 857 void opIndexAssign(in void[] value, string key) 858 { 859 auto ret = setFun(obj, key.toStringz(), value.ptr, value.length, 0); 860 errnoEnforce(ret == 0); 861 } 862 863 void remove(string key) 864 { 865 auto ret = removeFun(obj, key.toStringz()); 866 errnoEnforce(ret == 0); 867 } 868 869 string[] keys() 870 { 871 auto size = listFun(obj, null, 0); 872 errnoEnforce(size >= 0); 873 auto buf = new char[size]; 874 // TODO: race condition, retry 875 size = listFun(obj, buf.ptr, buf.length); 876 errnoEnforce(size == buf.length); 877 878 char[][] result; 879 size_t start; 880 foreach (p, c; buf) 881 if (!c) 882 { 883 result ~= buf[start..p]; 884 start = p+1; 885 } 886 887 return cast(string[])result; 888 } 889 } 890 891 auto xAttrs(string path) 892 { 893 return XAttrs!(const(char)*, "")(path.toStringz()); 894 } 895 896 auto linkXAttrs(string path) 897 { 898 return XAttrs!(const(char)*, "l")(path.toStringz()); 899 } 900 901 auto xAttrs(in ref File f) 902 { 903 return XAttrs!(int, "f")(f.fileno); 904 } 905 906 unittest 907 { 908 enum fn = "test.txt"; 909 std.file.write(fn, "test"); 910 scope(exit) remove(fn); 911 912 auto attrs = xAttrs(fn); 913 enum key = "user.foo"; 914 assert(key !in attrs); 915 assert(attrs.keys == []); 916 917 attrs[key] = "bar"; 918 assert(key in attrs); 919 assert(attrs[key] == "bar"); 920 assert(attrs.keys == [key]); 921 922 attrs.remove(key); 923 assert(key !in attrs); 924 assert(attrs.keys == []); 925 } 926 } 927 928 // **************************************************************************** 929 930 version (Windows) 931 { 932 /// Enumerate all hard links to the specified file. 933 // TODO: Return a range 934 string[] enumerateHardLinks()(string fn) 935 { 936 mixin(importWin32!q{winnt}); 937 mixin(importWin32!q{winbase}); 938 939 alias extern(System) HANDLE function(LPCWSTR lpFileName, DWORD dwFlags, LPDWORD StringLength, PWCHAR LinkName) TFindFirstFileNameW; 940 alias extern(System) BOOL function(HANDLE hFindStream, LPDWORD StringLength, PWCHAR LinkName) TFindNextFileNameW; 941 942 auto kernel32 = GetModuleHandle("kernel32.dll"); 943 auto FindFirstFileNameW = cast(TFindFirstFileNameW)GetProcAddress(kernel32, "FindFirstFileNameW").wenforce("GetProcAddress(FindFirstFileNameW)"); 944 auto FindNextFileNameW = cast(TFindNextFileNameW)GetProcAddress(kernel32, "FindNextFileNameW").wenforce("GetProcAddress(FindNextFileNameW)"); 945 946 static WCHAR[0x8000] buf; 947 DWORD len = buf.length; 948 auto h = FindFirstFileNameW(toUTF16z(fn), 0, &len, buf.ptr); 949 wenforce(h != INVALID_HANDLE_VALUE, "FindFirstFileNameW"); 950 scope(exit) FindClose(h); 951 952 string[] result; 953 do 954 { 955 enforce(len > 0 && len < buf.length && buf[len-1] == 0, "Bad FindFirst/NextFileNameW result"); 956 result ~= buf[0..len-1].toUTF8(); 957 len = buf.length; 958 auto ok = FindNextFileNameW(h, &len, buf.ptr); 959 if (!ok && GetLastError() == ERROR_HANDLE_EOF) 960 break; 961 wenforce(ok, "FindNextFileNameW"); 962 } while(true); 963 return result; 964 } 965 } 966 967 uint hardLinkCount(string fn) 968 { 969 version (Windows) 970 { 971 // TODO: Optimize (don't transform strings) 972 return cast(uint)fn.enumerateHardLinks.length; 973 } 974 else 975 { 976 import core.sys.posix.sys.stat; 977 978 stat_t s; 979 errnoEnforce(stat(fn.toStringz(), &s) == 0, "stat"); 980 return s.st_nlink.to!uint; 981 } 982 } 983 984 unittest 985 { 986 touch("a.test"); 987 scope(exit) remove("a.test"); 988 assert("a.test".hardLinkCount() == 1); 989 990 hardLink("a.test", "b.test"); 991 scope(exit) remove("b.test"); 992 assert("a.test".hardLinkCount() == 2); 993 assert("b.test".hardLinkCount() == 2); 994 995 version(Windows) 996 { 997 auto paths = enumerateHardLinks("a.test"); 998 assert(paths.length == 2); 999 paths.sort(); 1000 assert(paths[0].endsWith(`\a.test`), paths[0]); 1001 assert(paths[1].endsWith(`\b.test`)); 1002 } 1003 } 1004 1005 void toFile(in void[] data, in char[] name) 1006 { 1007 std.file.write(name, data); 1008 } 1009 1010 /// Uses UNC paths to open a file. 1011 /// Requires https://github.com/D-Programming-Language/phobos/pull/1888 1012 File openFile()(string fn, string mode = "rb") 1013 { 1014 File f; 1015 static if (is(typeof(&f.windowsHandleOpen))) 1016 { 1017 import core.sys.windows.windows; 1018 import ae.sys.windows.exception; 1019 1020 string winMode; 1021 foreach (c; mode) 1022 switch (c) 1023 { 1024 case 'r': 1025 case 'w': 1026 case 'a': 1027 case '+': 1028 winMode ~= c; 1029 break; 1030 case 'b': 1031 case 't': 1032 break; 1033 default: 1034 assert(false, "Unknown character in mode"); 1035 } 1036 DWORD access, creation; 1037 bool append; 1038 switch (winMode) 1039 { 1040 case "r" : access = GENERIC_READ ; creation = OPEN_EXISTING; break; 1041 case "r+": access = GENERIC_READ | GENERIC_WRITE; creation = OPEN_EXISTING; break; 1042 case "w" : access = GENERIC_WRITE; creation = OPEN_ALWAYS ; break; 1043 case "w+": access = GENERIC_READ | GENERIC_WRITE; creation = OPEN_ALWAYS ; break; 1044 case "a" : access = GENERIC_WRITE; creation = OPEN_ALWAYS ; append = true; break; 1045 case "a+": assert(false, "Not implemented"); // requires two file pointers 1046 default: assert(false, "Bad file mode: " ~ mode); 1047 } 1048 1049 auto pathW = toUTF16z(longPath(fn)); 1050 auto h = CreateFileW(pathW, access, FILE_SHARE_READ, null, creation, 0, HANDLE.init); 1051 wenforce(h != INVALID_HANDLE_VALUE); 1052 1053 if (append) 1054 h.SetFilePointer(0, null, FILE_END); 1055 1056 f.windowsHandleOpen(h, mode); 1057 } 1058 else 1059 f.open(fn, mode); 1060 return f; 1061 } 1062 1063 auto fileDigest(Digest)(string fn) 1064 { 1065 import std.range.primitives; 1066 Digest context; 1067 context.start(); 1068 put(context, openFile(fn, "rb").byChunk(64 * 1024)); 1069 auto digest = context.finish(); 1070 return digest; 1071 } 1072 1073 template mdFile() 1074 { 1075 import std.digest.md; 1076 alias mdFile = fileDigest!MD5; 1077 } 1078 1079 version (HAVE_WIN32) 1080 unittest 1081 { 1082 import std.digest.digest : toHexString; 1083 write("test.txt", "Hello, world!"); 1084 scope(exit) remove("test.txt"); 1085 assert(mdFile("test.txt").toHexString() == "6CD3556DEB0DA54BCA060B4C39479839"); 1086 } 1087 1088 auto fileDigestCached(Digest)(string fn) 1089 { 1090 static typeof(Digest.init.finish())[ulong] cache; 1091 auto id = getFileID(fn); 1092 auto phash = id in cache; 1093 if (phash) 1094 return *phash; 1095 return cache[id] = fileDigest!Digest(fn); 1096 } 1097 1098 template mdFileCached() 1099 { 1100 import std.digest.md; 1101 alias mdFileCached = fileDigestCached!MD5; 1102 } 1103 1104 version (HAVE_WIN32) 1105 unittest 1106 { 1107 import std.digest.digest : toHexString; 1108 write("test.txt", "Hello, world!"); 1109 scope(exit) remove("test.txt"); 1110 assert(mdFileCached("test.txt").toHexString() == "6CD3556DEB0DA54BCA060B4C39479839"); 1111 write("test.txt", "Something else"); 1112 assert(mdFileCached("test.txt").toHexString() == "6CD3556DEB0DA54BCA060B4C39479839"); 1113 } 1114 1115 /// Read a File (which might be a stream) into an array 1116 void[] readFile(File f) 1117 { 1118 import std.range.primitives; 1119 auto result = appender!(ubyte[]); 1120 put(result, f.byChunk(64*1024)); 1121 return result.data; 1122 } 1123 1124 unittest 1125 { 1126 auto s = "0123456789".replicate(10_000); 1127 write("test.txt", s); 1128 scope(exit) remove("test.txt"); 1129 assert(readFile(File("test.txt")) == s); 1130 } 1131 1132 /// Like std.file.readText for non-UTF8 1133 ascii readAscii()(string fileName) 1134 { 1135 return cast(ascii)readFile(openFile(fileName, "rb")); 1136 } 1137 1138 // http://d.puremagic.com/issues/show_bug.cgi?id=7016 1139 version(Posix) static import ae.sys.signals; 1140 1141 /// Start a thread which writes data to f asynchronously. 1142 Thread writeFileAsync(File f, in void[] data) 1143 { 1144 static class Writer : Thread 1145 { 1146 File target; 1147 const void[] data; 1148 1149 this(ref File f, in void[] data) 1150 { 1151 this.target = f; 1152 this.data = data; 1153 super(&run); 1154 } 1155 1156 void run() 1157 { 1158 version (Posix) 1159 { 1160 import ae.sys.signals; 1161 collectSignal(SIGPIPE, &write); 1162 } 1163 else 1164 write(); 1165 } 1166 1167 void write() 1168 { 1169 target.rawWrite(data); 1170 target.close(); 1171 } 1172 } 1173 1174 auto t = new Writer(f, data); 1175 t.start(); 1176 return t; 1177 } 1178 1179 /// Write data to a file, and ensure it gets written to disk 1180 /// before this function returns. 1181 /// Consider using as atomic!syncWrite. 1182 /// See also: syncUpdate 1183 void syncWrite()(string target, in void[] data) 1184 { 1185 auto f = File(target, "wb"); 1186 f.rawWrite(data); 1187 version (Windows) 1188 { 1189 mixin(importWin32!q{windows}); 1190 FlushFileBuffers(f.windowsHandle); 1191 } 1192 else 1193 { 1194 import core.sys.posix.unistd; 1195 fsync(f.fileno); 1196 } 1197 f.close(); 1198 } 1199 1200 /// Atomically save data to a file (if the file doesn't exist, 1201 /// or its contents differs). The update operation as a whole 1202 /// is not atomic, only the write is. 1203 void syncUpdate()(string fn, in void[] data) 1204 { 1205 if (!fn.exists || fn.read() != data) 1206 atomic!(syncWrite!())(fn, data); 1207 } 1208 1209 version(Windows) import ae.sys.windows.exception; 1210 1211 struct NamedPipeImpl 1212 { 1213 immutable string fileName; 1214 1215 /// Create a named pipe, and reserve a filename. 1216 this()(string name) 1217 { 1218 version(Windows) 1219 { 1220 mixin(importWin32!q{winbase}); 1221 1222 fileName = `\\.\pipe\` ~ name; 1223 auto h = CreateNamedPipeW(fileName.toUTF16z, PIPE_ACCESS_OUTBOUND, PIPE_TYPE_BYTE, 10, 4096, 4096, 0, null).wenforce("CreateNamedPipeW"); 1224 f.windowsHandleOpen(h, "wb"); 1225 } 1226 else 1227 { 1228 import core.sys.posix.sys.stat; 1229 1230 fileName = `/tmp/` ~ name ~ `.fifo`; 1231 mkfifo(fileName.toStringz, S_IWUSR | S_IRUSR); 1232 } 1233 } 1234 1235 /// Wait for a peer to open the other end of the pipe. 1236 File connect()() 1237 { 1238 version(Windows) 1239 { 1240 mixin(importWin32!q{winbase}); 1241 mixin(importWin32!q{windef}); 1242 1243 BOOL bSuccess = ConnectNamedPipe(f.windowsHandle, null); 1244 1245 // "If a client connects before the function is called, the function returns zero 1246 // and GetLastError returns ERROR_PIPE_CONNECTED. This can happen if a client 1247 // connects in the interval between the call to CreateNamedPipe and the call to 1248 // ConnectNamedPipe. In this situation, there is a good connection between client 1249 // and server, even though the function returns zero." 1250 if (!bSuccess) 1251 wenforce(GetLastError() == ERROR_PIPE_CONNECTED, "ConnectNamedPipe"); 1252 1253 return f; 1254 } 1255 else 1256 { 1257 return File(fileName, "w"); 1258 } 1259 } 1260 1261 ~this() 1262 { 1263 version(Windows) 1264 { 1265 // File.~this will take care of cleanup 1266 } 1267 else 1268 fileName.remove(); 1269 } 1270 1271 private: 1272 File f; 1273 } 1274 alias NamedPipe = RefCounted!NamedPipeImpl; 1275 1276 import ae.utils.textout : StringBuilder; 1277 1278 /// Avoid std.stdio.File.readln's memory corruption bug 1279 /// https://issues.dlang.org/show_bug.cgi?id=13856 1280 string safeReadln(File f) 1281 { 1282 StringBuilder buf; 1283 char[1] arr; 1284 while (true) 1285 { 1286 auto result = f.rawRead(arr[]); 1287 if (!result.length) 1288 break; 1289 buf.put(result); 1290 if (result[0] == '\x0A') 1291 break; 1292 } 1293 return buf.get(); 1294 } 1295 1296 // **************************************************************************** 1297 1298 /// Change the current directory to the given directory. Does nothing if dir is null. 1299 /// Return a scope guard which, upon destruction, restores the previous directory. 1300 /// Asserts that only one thread has changed the process's current directory at any time. 1301 auto pushd(string dir) 1302 { 1303 import core.atomic; 1304 1305 static int threadCount = 0; 1306 static shared int processCount = 0; 1307 1308 static struct Popd 1309 { 1310 string oldPath; 1311 this(string cwd) { oldPath = cwd; } 1312 ~this() { if (oldPath) pop(); } 1313 @disable this(); 1314 @disable this(this); 1315 1316 void pop() 1317 { 1318 assert(oldPath); 1319 scope(exit) oldPath = null; 1320 chdir(oldPath); 1321 1322 auto newThreadCount = --threadCount; 1323 auto newProcessCount = atomicOp!"-="(processCount, 1); 1324 assert(newThreadCount == newProcessCount); // Shouldn't happen 1325 } 1326 } 1327 1328 string cwd; 1329 if (dir) 1330 { 1331 auto newThreadCount = ++threadCount; 1332 auto newProcessCount = atomicOp!"+="(processCount, 1); 1333 assert(newThreadCount == newProcessCount, "Another thread already has an active pushd"); 1334 1335 cwd = getcwd(); 1336 chdir(dir); 1337 } 1338 return Popd(cwd); 1339 } 1340 1341 // **************************************************************************** 1342 1343 import std.algorithm; 1344 import std.process : thisProcessID; 1345 import std.traits; 1346 import std.typetuple; 1347 import ae.utils.meta; 1348 1349 enum targetParameterNames = "target/to/name/dst"; 1350 1351 /// Wrap an operation which creates a file or directory, 1352 /// so that it is created safely and, for files, atomically 1353 /// (by performing the underlying operation to a temporary 1354 /// location, then renaming the completed file/directory to 1355 /// the actual target location). targetName specifies the name 1356 /// of the parameter containing the target file/directory. 1357 auto atomic(alias impl, string targetName = targetParameterNames)(staticMap!(Unqual, ParameterTypeTuple!impl) args) 1358 { 1359 enum targetIndex = findParameter([ParameterIdentifierTuple!impl], targetName, __traits(identifier, impl)); 1360 return atomic!(impl, targetIndex)(args); 1361 } 1362 1363 /// ditto 1364 auto atomic(alias impl, size_t targetIndex)(staticMap!(Unqual, ParameterTypeTuple!impl) args) 1365 { 1366 // idup for https://d.puremagic.com/issues/show_bug.cgi?id=12503 1367 auto target = args[targetIndex].idup; 1368 auto temp = "%s.%s.%s.temp".format(target, thisProcessID, getCurrentThreadID); 1369 if (temp.exists) temp.removeRecurse(); 1370 scope(success) rename(temp, target); 1371 scope(failure) if (temp.exists) temp.removeRecurse(); 1372 args[targetIndex] = temp; 1373 return impl(args); 1374 } 1375 1376 /// ditto 1377 // Workaround for https://d.puremagic.com/issues/show_bug.cgi?id=12230 1378 // Can't be an overload because of https://issues.dlang.org/show_bug.cgi?id=13374 1379 //R atomicDg(string targetName = "target", R, Args...)(R delegate(Args) impl, staticMap!(Unqual, Args) args) 1380 auto atomicDg(size_t targetIndexA = size_t.max, Impl, Args...)(Impl impl, Args args) 1381 { 1382 enum targetIndex = targetIndexA == size_t.max ? ParameterTypeTuple!impl.length-1 : targetIndexA; 1383 return atomic!(impl, targetIndex)(args); 1384 } 1385 1386 deprecated alias safeUpdate = atomic; 1387 1388 unittest 1389 { 1390 enum fn = "atomic.tmp"; 1391 scope(exit) if (fn.exists) fn.remove(); 1392 1393 atomic!touch(fn); 1394 assert(fn.exists); 1395 fn.remove(); 1396 1397 atomicDg(&touch, fn); 1398 assert(fn.exists); 1399 } 1400 1401 /// Wrap an operation so that it is skipped entirely 1402 /// if the target already exists. Implies atomic. 1403 auto cached(alias impl, string targetName = targetParameterNames)(ParameterTypeTuple!impl args) 1404 { 1405 enum targetIndex = findParameter([ParameterIdentifierTuple!impl], targetName, __traits(identifier, impl)); 1406 auto target = args[targetIndex]; 1407 if (!target.exists) 1408 atomic!(impl, targetIndex)(args); 1409 return target; 1410 } 1411 1412 /// ditto 1413 // Exists due to the same reasons as atomicDg 1414 auto cachedDg(size_t targetIndexA = size_t.max, Impl, Args...)(Impl impl, Args args) 1415 { 1416 enum targetIndex = targetIndexA == size_t.max ? ParameterTypeTuple!impl.length-1 : targetIndexA; 1417 auto target = args[targetIndex]; 1418 if (!target.exists) 1419 atomic!(impl, targetIndex)(args); 1420 return target; 1421 } 1422 1423 deprecated alias obtainUsing = cached; 1424 1425 /// Create a file, or replace an existing file's contents 1426 /// atomically. 1427 /// Note: Consider using atomic!syncWrite or 1428 /// atomic!syncUpdate instead. 1429 alias atomic!writeProxy atomicWrite; 1430 deprecated alias safeWrite = atomicWrite; 1431 void writeProxy(string target, in void[] data) 1432 { 1433 std.file.write(target, data); 1434 } 1435 1436 // Work around for https://github.com/D-Programming-Language/phobos/pull/2784#issuecomment-68117241 1437 private void copy2(string source, string target) { std.file.copy(source, target); } 1438 1439 /// Copy a file, or replace an existing file's contents 1440 /// with another file's, atomically. 1441 alias atomic!copy2 atomicCopy; 1442 1443 unittest 1444 { 1445 enum fn = "cached.tmp"; 1446 scope(exit) if (fn.exists) fn.remove(); 1447 1448 cached!touch(fn); 1449 assert(fn.exists); 1450 1451 std.file.write(fn, "test"); 1452 1453 cachedDg!0(&writeProxy, fn, "test2"); 1454 assert(fn.readText() == "test"); 1455 } 1456 1457 // **************************************************************************** 1458 1459 template withTarget(alias targetGen, alias fun) 1460 { 1461 auto withTarget(Args...)(auto ref Args args) 1462 { 1463 auto target = targetGen(args); 1464 fun(args, target); 1465 return target; 1466 } 1467 } 1468 1469 /// Two-argument buildPath with reversed arguments. 1470 /// Useful for UFCS chaining. 1471 string prependPath(string target, string path) 1472 { 1473 return buildPath(path, target); 1474 }