[util] fix fstat for archive directories on windows

Currently TFileStat has a bunch of problems on Windows:
1. Mode computation ORs different types, but file attributes it checks are not mutually exclusive. E.g. it is possible for directory to have both `FILE_ATTRIBUTE_DIRECTORY` and `FILE_ATTRIBUTE_ARCHIVE`, but it will currently lead to invalid `_S_IFDIR | _S_IFREG` mode.
2. Any file with `FILE_ATTRIBUTE_REPARSE_POINT` flag is considered to be symlink. But there are many other types of reparse points even user-defined ones. For more info see https://docs.microsoft.com/en-us/windows/win32/fileio/reparse-point-tags

To fix all this we do the following:
1. Add `ReparseTag` field to `TSystemFStat` and fill it for reparse points in `GetStatByHandle`.
2. Refactor `GetWinFileType` out of `GetFileMode` to ensure that we always select only single `S_IFMT` value.
3. Pass reparse tag value into `GetWinFileType` and select `S_IFLNK` only for `IO_REPARSE_TAG_SYMLINK` and `IO_REPARSE_TAG_MOUNT_POINT` tags. The latter one is a bit strange, but this behaviour is aligned with current implementation of `WinReadLink`.
4. Factor `ReadReparsePoint` out of `WinReadLink`. This function uses `DeviceIoControl` to read reparse point data into structures copied from WDK.
5. Add `WinReadReparseTag` that uses `ReadReparsePoint` to extract reparse point tag. We use this approach instead of MSDN-recommended `FindFirstFile` because latter requires file path, but we have file handle at this point. AFAIK golang and python use similar approach for this.
6. Add test for archived directory case.
7. Make symlink nofollow tests run only when symlink creation is possible for current user.

Discovered in https://st.yandex-team.ru/ARC-3931

ref:4f6735817b9f85f3351a1021a56dd7eb4606bd65

4 files changed, 154 insertions, 32 deletions

diff --git a/util/system/fs_win.cpp b/util/system/fs_win.cpp
index a410ccac06..94016c9fc4 100644
--- a/util/system/fs_win.cpp
+++ b/util/system/fs_win.cpp
@@ -180,36 +180,54 @@ namespace NFsPrivate {
 
     // the end of edited part of <Ntifs.h>
 
-    TString WinReadLink(const TString& name) {
-        TFileHandle h = CreateFileWithUtf8Name(name, GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING,
-                                               FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, true);
-        TTempBuf buf;
+    // For more info see:
+    // * https://docs.microsoft.com/en-us/windows/win32/fileio/reparse-points
+    // * https://docs.microsoft.com/en-us/windows-hardware/drivers/ifs/fsctl-get-reparse-point
+    // * https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/ns-ntifs-_reparse_data_buffer
+    REPARSE_DATA_BUFFER* ReadReparsePoint(HANDLE h, TTempBuf& buf) {
         while (true) {
             DWORD bytesReturned = 0;
             BOOL res = DeviceIoControl(h, FSCTL_GET_REPARSE_POINT, nullptr, 0, buf.Data(), buf.Size(), &bytesReturned, nullptr);
             if (res) {
                 REPARSE_DATA_BUFFER* rdb = (REPARSE_DATA_BUFFER*)buf.Data();
-                if (rdb->ReparseTag == IO_REPARSE_TAG_SYMLINK) {
-                    wchar16* str = (wchar16*)&rdb->SymbolicLinkReparseBuffer.PathBuffer[rdb->SymbolicLinkReparseBuffer.SubstituteNameOffset / sizeof(wchar16)];
-                    size_t len = rdb->SymbolicLinkReparseBuffer.SubstituteNameLength / sizeof(wchar16);
-                    return WideToUTF8(str, len);
-                } else if (rdb->ReparseTag == IO_REPARSE_TAG_MOUNT_POINT) {
-                    wchar16* str = (wchar16*)&rdb->MountPointReparseBuffer.PathBuffer[rdb->MountPointReparseBuffer.SubstituteNameOffset / sizeof(wchar16)];
-                    size_t len = rdb->MountPointReparseBuffer.SubstituteNameLength / sizeof(wchar16);
-                    return WideToUTF8(str, len);
-                }
-                //this reparse point is unsupported in arcadia
-                return TString();
+                return rdb;
             } else {
                 if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
                     buf = TTempBuf(buf.Size() * 2);
                 } else {
-                    ythrow yexception() << "can't read link " << name;
+                    return nullptr;
                 }
             }
         }
     }
 
+    TString WinReadLink(const TString& name) {
+        TFileHandle h = CreateFileWithUtf8Name(name, GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING,
+                                               FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, true);
+        TTempBuf buf;
+        REPARSE_DATA_BUFFER* rdb = ReadReparsePoint(h, buf);
+        if (rdb == nullptr) {
+            ythrow TIoSystemError() << "can't read reparse point " << name;
+        }
+        if (rdb->ReparseTag == IO_REPARSE_TAG_SYMLINK) {
+            wchar16* str = (wchar16*)&rdb->SymbolicLinkReparseBuffer.PathBuffer[rdb->SymbolicLinkReparseBuffer.SubstituteNameOffset / sizeof(wchar16)];
+            size_t len = rdb->SymbolicLinkReparseBuffer.SubstituteNameLength / sizeof(wchar16);
+            return WideToUTF8(str, len);
+        } else if (rdb->ReparseTag == IO_REPARSE_TAG_MOUNT_POINT) {
+            wchar16* str = (wchar16*)&rdb->MountPointReparseBuffer.PathBuffer[rdb->MountPointReparseBuffer.SubstituteNameOffset / sizeof(wchar16)];
+            size_t len = rdb->MountPointReparseBuffer.SubstituteNameLength / sizeof(wchar16);
+            return WideToUTF8(str, len);
+        }
+        //this reparse point is unsupported in arcadia
+        return TString();
+    }
+
+    ULONG WinReadReparseTag(HANDLE h) {
+        TTempBuf buf;
+        REPARSE_DATA_BUFFER* rdb = ReadReparsePoint(h, buf);
+        return rdb ? rdb->ReparseTag : 0;
+    }
+
     // we can't use this function to get an analog of unix inode due to a lot of NTFS folders do not have this GUID
     //(it will be 'create' case really)
     /*
diff --git a/util/system/fs_win.h b/util/system/fs_win.h
index 8086129828..2dfdcb2f92 100644
--- a/util/system/fs_win.h
+++ b/util/system/fs_win.h
@@ -15,6 +15,8 @@ namespace NFsPrivate {
 
     TString WinReadLink(const TString& path);
 
+    ULONG WinReadReparseTag(HANDLE h);
+
     HANDLE CreateFileWithUtf8Name(const TStringBuf fName, ui32 accessMode, ui32 shareMode, ui32 createMode, ui32 attributes, bool inheritHandle);
 
     bool WinRemove(const TString& path);
diff --git a/util/system/fstat.cpp b/util/system/fstat.cpp
index 81e98cbc6b..c759f68f11 100644
--- a/util/system/fstat.cpp
+++ b/util/system/fstat.cpp
@@ -15,20 +15,43 @@
     #endif
     #define _S_IFLNK 0x80000000
 
-ui32 GetFileMode(DWORD fileAttributes) {
+// See https://docs.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants
+// for possible flag values
+static ui32 GetWinFileType(DWORD fileAttributes, ULONG reparseTag) {
+    // I'm not really sure, why it is done like this. MSDN tells that
+    // FILE_ATTRIBUTE_DEVICE is reserved for system use. Some more info is
+    // available at https://stackoverflow.com/questions/3419527/setting-file-attribute-device-in-visual-studio
+    // We should probably replace this with GetFileType call and check for
+    // FILE_TYPE_CHAR and FILE_TYPE_PIPE return values.
+    if (fileAttributes & FILE_ATTRIBUTE_DEVICE) {
+        return _S_IFCHR;
+    }
+
+    if (fileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) {
+        // We consider IO_REPARSE_TAG_SYMLINK and IO_REPARSE_TAG_MOUNT_POINT
+        // both to be symlinks to align with current WinReadLink behaviour.
+        if (reparseTag == IO_REPARSE_TAG_SYMLINK || reparseTag == IO_REPARSE_TAG_MOUNT_POINT) {
+            return _S_IFLNK;
+        }
+    }
+
+    if (fileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+        return _S_IFDIR;
+    }
+
+    return _S_IFREG;
+}
+
+static ui32 GetFileMode(DWORD fileAttributes, ULONG reparseTag) {
     ui32 mode = 0;
     if (fileAttributes == 0xFFFFFFFF)
         return mode;
-    if (fileAttributes & FILE_ATTRIBUTE_DEVICE)
-        mode |= _S_IFCHR;
-    if (fileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
-        mode |= _S_IFLNK; // todo: was undefined by the moment of writing this code
-    if (fileAttributes & FILE_ATTRIBUTE_DIRECTORY)
-        mode |= _S_IFDIR;
-    if (fileAttributes & (FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE))
-        mode |= _S_IFREG;
-    if ((fileAttributes & FILE_ATTRIBUTE_READONLY) == 0)
+
+    mode |= GetWinFileType(fileAttributes, reparseTag);
+
+    if ((fileAttributes & FILE_ATTRIBUTE_READONLY) == 0) {
         mode |= _S_IWRITE;
+    }
     return mode;
 }
 
@@ -36,7 +59,9 @@ ui32 GetFileMode(DWORD fileAttributes) {
     #define S_ISREG(st_mode) (st_mode & _S_IFREG)
     #define S_ISLNK(st_mode) (st_mode & _S_IFLNK)
 
-using TSystemFStat = BY_HANDLE_FILE_INFORMATION;
+struct TSystemFStat: public BY_HANDLE_FILE_INFORMATION {
+    ULONG ReparseTag = 0;
+};
 
 #else
 
@@ -65,7 +90,7 @@ static void MakeStat(TFileStat& st, const TSystemFStat& fs) {
     FileTimeToTimeval(&fs.ftLastWriteTime, &tv);
     st.MTime = tv.tv_sec;
     st.NLinks = fs.nNumberOfLinks;
-    st.Mode = GetFileMode(fs.dwFileAttributes);
+    st.Mode = GetFileMode(fs.dwFileAttributes, fs.ReparseTag);
     st.Uid = 0;
     st.Gid = 0;
     st.Size = ((ui64)fs.nFileSizeHigh << 32) | fs.nFileSizeLow;
@@ -76,7 +101,13 @@ static void MakeStat(TFileStat& st, const TSystemFStat& fs) {
 
 static bool GetStatByHandle(TSystemFStat& fs, FHANDLE f) {
 #ifdef _win_
-    return GetFileInformationByHandle(f, &fs);
+    if (!GetFileInformationByHandle(f, &fs)) {
+        return false;
+    }
+    if (fs.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) {
+        fs.ReparseTag = NFsPrivate::WinReadReparseTag(f);
+    }
+    return true;
 #else
     return !fstat(f, &fs);
 #endif
diff --git a/util/system/fstat_ut.cpp b/util/system/fstat_ut.cpp
index 160ecd936e..300f76d7e6 100644
--- a/util/system/fstat_ut.cpp
+++ b/util/system/fstat_ut.cpp
@@ -62,11 +62,53 @@ Y_UNIT_TEST_SUITE(TestFileStat) {
         UNIT_ASSERT(fs.CTime == 0);
     }
 
+#ifdef _win_
+    // Symlinks require additional privileges on windows.
+    // Skip test if we are not allowed to create one.
+    // Wine returns true from NFs::SymLink, but actually does nothing
+    #define SAFE_SYMLINK(target, link)                                                \
+        do {                                                                          \
+            auto res = NFs::SymLink(target, link);                                    \
+            if (!res) {                                                               \
+                auto err = LastSystemError();                                         \
+                Cerr << "can't create symlink: " << LastSystemErrorText(err) << Endl; \
+                UNIT_ASSERT(err == ERROR_PRIVILEGE_NOT_HELD);                         \
+                return;                                                               \
+            }                                                                         \
+            if (!NFs::Exists(link) && IsWine()) {                                     \
+                Cerr << "wine does not support symlinks" << Endl;                     \
+                return;                                                               \
+            }                                                                         \
+        } while (false)
+
+    bool IsWine() {
+        HKEY subKey = nullptr;
+        LONG result = RegOpenKeyEx(HKEY_CURRENT_USER, "Software\\Wine", 0, KEY_READ, &subKey);
+        if (result == ERROR_SUCCESS) {
+            return true;
+        }
+        result = RegOpenKeyEx(HKEY_LOCAL_MACHINE, "Software\\Wine", 0, KEY_READ, &subKey);
+        if (result == ERROR_SUCCESS) {
+            return true;
+        }
+
+        HMODULE hntdll = GetModuleHandle("ntdll.dll");
+        if (!hntdll) {
+            return false;
+        }
+
+        auto func = GetProcAddress(hntdll, "wine_get_version");
+        return func != nullptr;
+    }
+#else
+    #define SAFE_SYMLINK(target, link) UNIT_ASSERT(NFs::SymLink(target, link))
+#endif
+
     Y_UNIT_TEST(SymlinkToExistingFileTest) {
         const auto path = GetOutputPath() / "file_1";
         const auto link = GetOutputPath() / "symlink_1";
         TFile(path, EOpenModeFlag::CreateNew | EOpenModeFlag::RdWr);
-        UNIT_ASSERT(NFs::SymLink(path, link));
+        SAFE_SYMLINK(path, link);
 
         const TFileStat statNoFollow(link, false);
         UNIT_ASSERT_VALUES_EQUAL_C(false, statNoFollow.IsNull(), ToString(statNoFollow.Mode));
@@ -84,7 +126,7 @@ Y_UNIT_TEST_SUITE(TestFileStat) {
     Y_UNIT_TEST(SymlinkToNonExistingFileTest) {
         const auto path = GetOutputPath() / "file_2";
         const auto link = GetOutputPath() / "symlink_2";
-        UNIT_ASSERT(NFs::SymLink(path, link));
+        SAFE_SYMLINK(path, link);
 
         const TFileStat statNoFollow(link, false);
         UNIT_ASSERT_VALUES_EQUAL_C(true, statNoFollow.IsNull(), ToString(statNoFollow.Mode));
@@ -102,7 +144,7 @@ Y_UNIT_TEST_SUITE(TestFileStat) {
     Y_UNIT_TEST(SymlinkToFileThatCantExistTest) {
         const auto path = TFsPath("/path") / "that" / "does" / "not" / "exists";
         const auto link = GetOutputPath() / "symlink_3";
-        UNIT_ASSERT(NFs::SymLink(path, link));
+        SAFE_SYMLINK(path, link);
 
         const TFileStat statNoFollow(link, false);
         UNIT_ASSERT_VALUES_EQUAL_C(true, statNoFollow.IsNull(), ToString(statNoFollow.Mode));
@@ -154,4 +196,33 @@ Y_UNIT_TEST_SUITE(TestFileStat) {
         UNIT_ASSERT(unlink(fileName.c_str()) == 0);
     }
 
+#ifdef _win_
+    Y_UNIT_TEST(WinArchiveDirectoryTest) {
+        TFsPath dir = "archive_dir";
+        dir.MkDirs();
+
+        SetFileAttributesA(dir.c_str(), FILE_ATTRIBUTE_ARCHIVE);
+
+        const TFileStat stat(dir);
+        UNIT_ASSERT(stat.IsDir());
+        UNIT_ASSERT(!stat.IsSymlink());
+        UNIT_ASSERT(!stat.IsFile());
+        UNIT_ASSERT(!stat.IsNull());
+    }
+
+    Y_UNIT_TEST(WinArchiveFileTest) {
+        TFsPath filename = "archive_file";
+        TFile file(filename, OpenAlways | WrOnly);
+        file.Write("1", 1);
+        file.Close();
+
+        SetFileAttributesA(filename.c_str(), FILE_ATTRIBUTE_ARCHIVE);
+
+        const TFileStat stat(filename);
+        UNIT_ASSERT(!stat.IsDir());
+        UNIT_ASSERT(!stat.IsSymlink());
+        UNIT_ASSERT(stat.IsFile());
+        UNIT_ASSERT(!stat.IsNull());
+    }
+#endif
 }