diff options
author | Devtools Arcadia <arcadia-devtools@yandex-team.ru> | 2022-02-07 18:08:42 +0300 |
---|---|---|
committer | Devtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net> | 2022-02-07 18:08:42 +0300 |
commit | 1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch) | |
tree | e26c9fed0de5d9873cce7e00bc214573dc2195b7 /util/folder | |
download | ydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz |
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'util/folder')
30 files changed, 6150 insertions, 0 deletions
diff --git a/util/folder/dirent_win.c b/util/folder/dirent_win.c new file mode 100644 index 0000000000..7e6db74ce5 --- /dev/null +++ b/util/folder/dirent_win.c @@ -0,0 +1,125 @@ +#include <util/system/defaults.h> + +#ifdef _win_ + + #include <stdio.h> + #include "dirent_win.h" + + #if defined(_MSC_VER) && (_MSC_VER < 1900) +void __cdecl _dosmaperr(unsigned long); + +static void SetErrno() { + _dosmaperr(GetLastError()); +} + #else +void __cdecl __acrt_errno_map_os_error(unsigned long const oserrno); + +static void SetErrno() { + __acrt_errno_map_os_error(GetLastError()); +} + #endif + +struct DIR* opendir(const char* dirname) { + struct DIR* dir = (struct DIR*)malloc(sizeof(struct DIR)); + if (!dir) { + return NULL; + } + dir->sh = INVALID_HANDLE_VALUE; + dir->fff_templ = NULL; + dir->file_no = 0; + dir->readdir_buf = NULL; + + int len = strlen(dirname); + //Remove trailing slashes + while (len && (dirname[len - 1] == '\\' || dirname[len - 1] == '/')) { + --len; + } + int len_converted = MultiByteToWideChar(CP_UTF8, 0, dirname, len, 0, 0); + if (len_converted == 0) { + closedir(dir); + return NULL; + } + dir->fff_templ = (WCHAR*)malloc((len_converted + 5) * sizeof(WCHAR)); + if (!dir->fff_templ) { + closedir(dir); + return NULL; + } + MultiByteToWideChar(CP_UTF8, 0, dirname, len, dir->fff_templ, len_converted); + + WCHAR append[] = {'\\', '*', '.', '*', 0}; + memcpy(dir->fff_templ + len_converted, append, sizeof(append)); + dir->sh = FindFirstFileW(dir->fff_templ, &dir->wfd); + if (dir->sh == INVALID_HANDLE_VALUE) { + SetErrno(); + closedir(dir); + return NULL; + } + + return dir; +} + +int closedir(struct DIR* dir) { + if (dir->sh != INVALID_HANDLE_VALUE) + FindClose(dir->sh); + free(dir->fff_templ); + free(dir->readdir_buf); + free(dir); + return 0; +} + +int readdir_r(struct DIR* dir, struct dirent* entry, struct dirent** result) { + if (!FindNextFileW(dir->sh, &dir->wfd)) { + int err = GetLastError(); + *result = 0; + if (err == ERROR_NO_MORE_FILES) { + SetLastError(0); + return 0; + } else { + return err; + } + } + entry->d_fileno = dir->file_no++; + entry->d_reclen = sizeof(struct dirent); + if (dir->wfd.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT && + (dir->wfd.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT || dir->wfd.dwReserved0 == IO_REPARSE_TAG_SYMLINK)) + { + entry->d_type = DT_LNK; + } else if (dir->wfd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + entry->d_type = DT_DIR; + } else { + entry->d_type = DT_REG; + } + int len = lstrlenW(dir->wfd.cFileName); + int conv_len = WideCharToMultiByte(CP_UTF8, 0, dir->wfd.cFileName, len, 0, 0, 0, 0); + if (conv_len == 0) { + return -1; + } + if (conv_len > sizeof(entry->d_name) - 1) { + SetLastError(ERROR_INSUFFICIENT_BUFFER); + return ERROR_INSUFFICIENT_BUFFER; + } + entry->d_namlen = conv_len; + WideCharToMultiByte(CP_UTF8, 0, dir->wfd.cFileName, len, entry->d_name, conv_len, 0, 0); + entry->d_name[conv_len] = 0; + *result = entry; + return 0; +} + +struct dirent* readdir(struct DIR* dir) { + struct dirent* res; + if (!dir->readdir_buf) { + dir->readdir_buf = (struct dirent*)malloc(sizeof(struct dirent)); + if (dir->readdir_buf == 0) + return 0; + } + readdir_r(dir, dir->readdir_buf, &res); + return res; +} + +void rewinddir(struct DIR* dir) { + FindClose(dir->sh); + dir->sh = FindFirstFileW(dir->fff_templ, &dir->wfd); + dir->file_no = 0; +} + +#endif //_win_ diff --git a/util/folder/dirent_win.h b/util/folder/dirent_win.h new file mode 100644 index 0000000000..ac11a64c04 --- /dev/null +++ b/util/folder/dirent_win.h @@ -0,0 +1,46 @@ +#pragma once + +#include <util/system/defaults.h> + +#ifdef _win_ + + #include <util/system/winint.h> + + #ifdef __cplusplus +extern "C" { + #endif + + struct DIR { + HANDLE sh; + WIN32_FIND_DATAW wfd; + WCHAR* fff_templ; + int file_no; + struct dirent* readdir_buf; + }; + + #define MAXNAMLEN (MAX_PATH - 1) * 2 + #define MAXPATHLEN MAX_PATH + #define DT_DIR 4 + #define DT_REG 8 + #define DT_LNK 10 + #define DT_UNKNOWN 0 + + struct dirent { + ui32 d_fileno; /* file number of entry */ + ui16 d_reclen; /* length of this record */ + ui8 d_type; /* file type */ + ui8 d_namlen; /* length of string in d_name */ + char d_name[MAXNAMLEN + 1]; /* name must be no longer than this */ + }; + + struct DIR* opendir(const char* dirname); + int closedir(struct DIR* dir); + int readdir_r(struct DIR* dir, struct dirent* entry, struct dirent** result); + struct dirent* readdir(struct DIR* dir); + void rewinddir(struct DIR* dir); + + #ifdef __cplusplus +} + #endif + +#endif //_win_ diff --git a/util/folder/dirut.cpp b/util/folder/dirut.cpp new file mode 100644 index 0000000000..ffc9b09f96 --- /dev/null +++ b/util/folder/dirut.cpp @@ -0,0 +1,621 @@ +#include "dirut.h" +#include "iterator.h" +#include "filelist.h" +#include "fts.h" +#include "pathsplit.h" +#include "path.h" + +#include <util/generic/yexception.h> +#include <util/system/compiler.h> +#include <util/system/fs.h> +#include <util/system/maxlen.h> +#include <util/system/yassert.h> + +void SlashFolderLocal(TString& folder) { + if (!folder) + return; +#ifdef _win32_ + size_t pos; + while ((pos = folder.find('/')) != TString::npos) + folder.replace(pos, 1, LOCSLASH_S); +#endif + if (folder[folder.size() - 1] != LOCSLASH_C) + folder.append(LOCSLASH_S); +} + +#ifndef _win32_ + +bool correctpath(TString& folder) { + return resolvepath(folder, "/"); +} + +bool resolvepath(TString& folder, const TString& home) { + Y_ASSERT(home && home.at(0) == '/'); + if (!folder) { + return false; + } + // may be from windows + char* ptr = folder.begin(); + while ((ptr = strchr(ptr, '\\')) != nullptr) + *ptr = '/'; + + if (folder.at(0) == '~') { + if (folder.length() == 1 || folder.at(1) == '/') { + folder = GetHomeDir() + (folder.data() + 1); + } else { + char* buf = (char*)alloca(folder.length() + 1); + strcpy(buf, folder.data() + 1); + char* p = strchr(buf, '/'); + if (p) + *p++ = 0; + passwd* pw = getpwnam(buf); + if (pw) { + folder = pw->pw_dir; + folder += "/"; + if (p) + folder += p; + } else { + return false; // unknown user + } + } + } + int len = folder.length() + home.length() + 1; + char* path = (char*)alloca(len); + if (folder.at(0) != '/') { + strcpy(path, home.data()); + strcpy(strrchr(path, '/') + 1, folder.data()); // the last char must be '/' if it's a dir + } else { + strcpy(path, folder.data()); + } + len = strlen(path) + 1; + // grabbed from url.cpp + char* newpath = (char*)alloca(len + 2); + const char** pp = (const char**)alloca(len * sizeof(char*)); + int i = 0; + for (char* s = path; s;) { + pp[i++] = s; + s = strchr(s, '/'); + if (s) + *s++ = 0; + } + + for (int j = 1; j < i;) { + const char*& p = pp[j]; + if (strcmp(p, ".") == 0 || strcmp(p, "") == 0) { + if (j == i - 1) { + p = ""; + break; + } else { + memmove(pp + j, pp + j + 1, (i - j - 1) * sizeof(p)); + --i; + } + } else if (strcmp(p, "..") == 0) { + if (j == i - 1) { + if (j == 1) { + p = ""; + } else { + --i; + pp[j - 1] = ""; + } + break; + } else { + if (j == 1) { + memmove(pp + j, pp + j + 1, (i - j - 1) * sizeof(p)); + --i; + } else { + memmove(pp + j - 1, pp + j + 1, (i - j - 1) * sizeof(p)); + i -= 2; + --j; + } + } + } else + ++j; + } + + char* s = newpath; + for (int k = 0; k < i; k++) { + s = strchr(strcpy(s, pp[k]), 0); + *s++ = '/'; + } + *(--s) = 0; + folder = newpath; + return true; +} + +#else + +using dir_type = enum { + dt_empty, + dt_error, + dt_up, + dt_dir +}; + +// precondition: *ptr != '\\' || *ptr == 0 (cause dt_error) +// postcondition: *ptr != '\\' +template <typename T> +static int next_dir(T*& ptr) { + int has_blank = 0; + int has_dot = 0; + int has_letter = 0; + int has_ctrl = 0; + + while (*ptr && *ptr != '\\') { + int c = (unsigned char)*ptr++; + switch (c) { + case ' ': + ++has_blank; + break; + case '.': + ++has_dot; + break; + case '/': + case ':': + case '*': + case '?': + case '"': + case '<': + case '>': + case '|': + ++has_ctrl; + break; + default: + if (c == 127 || c < ' ') + ++has_ctrl; + else + ++has_letter; + } + } + if (*ptr) + ++ptr; + if (has_ctrl) + return dt_error; + if (has_letter) + return dt_dir; + if (has_dot && has_blank) + return dt_error; + if (has_dot == 1) + return dt_empty; + if (has_dot == 2) + return dt_up; + return dt_error; +} + +using disk_type = enum { + dk_noflags = 0, + dk_unc = 1, + dk_hasdrive = 2, + dk_fromroot = 4, + dk_error = 8 +}; + +// root slash (if any) - part of disk +template <typename T> +static int skip_disk(T*& ptr) { + int result = dk_noflags; + if (!*ptr) + return result; + if (ptr[0] == '\\' && ptr[1] == '\\') { + result |= dk_unc | dk_fromroot; + ptr += 2; + if (next_dir(ptr) != dt_dir) + return dk_error; // has no host name + if (next_dir(ptr) != dt_dir) + return dk_error; // has no share name + } else { + if (*ptr && *(ptr + 1) == ':') { + result |= dk_hasdrive; + ptr += 2; + } + if (*ptr == '\\' || *ptr == '/') { + ++ptr; + result |= dk_fromroot; + } + } + return result; +} + +int correctpath(char* cpath, const char* path) { + if (!path || !*path) { + *cpath = 0; + return 1; + } + char* ptr = (char*)path; + char* cptr = cpath; + int counter = 0; + while (*ptr) { + char c = *ptr; + if (c == '/') + c = '\\'; + if (c == '\\') + ++counter; + else + counter = 0; + if (counter <= 1) { + *cptr = c; + ++cptr; + } + ++ptr; + } + *cptr = 0; + // replace '/' by '\' + int dk = skip_disk(cpath); + + if (dk == dk_error) + return 0; + + char* ptr1 = ptr = cpath; + int level = 0; + while (*ptr) { + switch (next_dir(ptr)) { + case dt_dir: + ++level; + break; + case dt_empty: + memmove(ptr1, ptr, strlen(ptr) + 1); + ptr = ptr1; + break; + case dt_up: + --level; + if (level >= 0) { + *--ptr1 = 0; + ptr1 = strrchr(cpath, '\\'); + if (!ptr1) + ptr1 = cpath; + else + ++ptr1; + memmove(ptr1, ptr, strlen(ptr) + 1); + ptr = ptr1; + break; + } else if (level == -1 && (dk & dk_hasdrive)) { + if (*ptr && *(ptr + 1) == ':' && *(cpath - 2) == ':') { + memmove(cpath - 3, ptr, strlen(ptr) + 1); + return 1; + } + } + if (dk & dk_fromroot) + return 0; + break; + case dt_error: + default: + return 0; + } + ptr1 = ptr; + } + + if ((ptr > cpath || ptr == cpath && dk & dk_unc) && *(ptr - 1) == '\\') + *(ptr - 1) = 0; + return 1; +} + +static inline int normchar(unsigned char c) { + return (c < 'a' || c > 'z') ? c : c - 32; +} + +static inline char* strslashcat(char* a, const char* b) { + size_t len = strlen(a); + if (len && a[len - 1] != '\\') + a[len++] = '\\'; + strcpy(a + len, b); + return a; +} + +int resolvepath(char* apath, const char* rpath, const char* cpath) { + const char* redisk = rpath; + if (!rpath || !*rpath) + return 0; + int rdt = skip_disk(redisk); + if (rdt == dk_error) + return 0; + if (rdt & dk_unc || rdt & dk_hasdrive && rdt & dk_fromroot) { + return correctpath(apath, rpath); + } + + const char* cedisk = cpath; + if (!cpath || !*cpath) + return 0; + int cdt = skip_disk(cedisk); + if (cdt == dk_error) + return 0; + + char* tpath = (char*)alloca(strlen(rpath) + strlen(cpath) + 3); + + // rdt&dk_hasdrive && !rdt&dk_fromroot + if (rdt & dk_hasdrive) { + if (!(cdt & dk_fromroot)) + return 0; + if (cdt & dk_hasdrive && normchar(*rpath) != normchar(*cpath)) + return 0; + memcpy(tpath, rpath, 2); + memcpy(tpath + 2, cedisk, strlen(cedisk) + 1); + strslashcat(tpath, redisk); + + // !rdt&dk_hasdrive && rdt&dk_fromroot + } else if (rdt & dk_fromroot) { + if (!(cdt & dk_hasdrive) && !(cdt & dk_unc)) + return 0; + memcpy(tpath, cpath, cedisk - cpath); + tpath[cedisk - cpath] = 0; + strslashcat(tpath, redisk); + + // !rdt&dk_hasdrive && !rdt&dk_fromroot + } else { + if (!(cdt & dk_fromroot) || !(cdt & dk_hasdrive) && !(cdt & dk_unc)) + return 0; + strslashcat(strcpy(tpath, cpath), redisk); + } + + return correctpath(apath, tpath); +} + +bool correctpath(TString& filename) { + char* ptr = (char*)alloca(filename.size() + 2); + if (correctpath(ptr, filename.data())) { + filename = ptr; + return true; + } + return false; +} + +bool resolvepath(TString& folder, const TString& home) { + char* ptr = (char*)alloca(folder.size() + 3 + home.size()); + if (resolvepath(ptr, folder.data(), home.data())) { + folder = ptr; + return true; + } + return false; +} + +#endif // !defined _win32_ + +char GetDirectorySeparator() { + return LOCSLASH_C; +} + +const char* GetDirectorySeparatorS() { + return LOCSLASH_S; +} + +void RemoveDirWithContents(TString dirName) { + SlashFolderLocal(dirName); + + TDirIterator dir(dirName, TDirIterator::TOptions(FTS_NOSTAT)); + + for (auto it = dir.begin(); it != dir.end(); ++it) { + switch (it->fts_info) { + case FTS_F: + case FTS_DEFAULT: + case FTS_DP: + case FTS_SL: + case FTS_SLNONE: + if (!NFs::Remove(it->fts_path)) + ythrow TSystemError() << "error while removing " << it->fts_path; + break; + } + } +} + +int mkpath(char* path, int mode) { + return NFs::MakeDirectoryRecursive(path, NFs::EFilePermission(mode)) ? 0 : -1; +} + +// Implementation of realpath in FreeBSD (version 9.0 and less) and GetFullPathName in Windows +// did not require last component of the file name to exist (other implementations will fail +// if it does not). Use RealLocation if that behaviour is required. +TString RealPath(const TString& path) { + TTempBuf result; + Y_ASSERT(result.Size() > MAX_PATH); //TMP_BUF_LEN > MAX_PATH +#ifdef _win_ + if (GetFullPathName(path.data(), result.Size(), result.Data(), nullptr) == 0) +#else + if (realpath(path.data(), result.Data()) == nullptr) +#endif + ythrow TFileError() << "RealPath failed \"" << path << "\""; + return result.Data(); +} + +TString RealLocation(const TString& path) { + if (NFs::Exists(path)) + return RealPath(path); + TString dirpath = GetDirName(path); + if (NFs::Exists(dirpath)) + return RealPath(dirpath) + GetDirectorySeparatorS() + GetFileNameComponent(path.data()); + ythrow TFileError() << "RealLocation failed \"" << path << "\""; +} + +int MakeTempDir(char path[/*FILENAME_MAX*/], const char* prefix) { + int ret; + + TString sysTmp; + +#ifdef _win32_ + if (!prefix || *prefix == '/') { +#else + if (!prefix) { +#endif + sysTmp = GetSystemTempDir(); + prefix = sysTmp.data(); + } + + if ((ret = ResolvePath(prefix, nullptr, path, 1)) != 0) + return ret; + if (!TFileStat(path).IsDir()) + return ENOENT; + if ((strlcat(path, "tmpXXXXXX", FILENAME_MAX) > FILENAME_MAX - 100)) + return EINVAL; + if (!(mkdtemp(path))) + return errno ? errno : EINVAL; + strcat(path, LOCSLASH_S); + return 0; +} + +bool IsDir(const TString& path) { + return TFileStat(path).IsDir(); +} + +TString GetHomeDir() { + TString s(getenv("HOME")); + if (!s) { +#ifndef _win32_ + passwd* pw = nullptr; + s = getenv("USER"); + if (s) + pw = getpwnam(s.data()); + else + pw = getpwuid(getuid()); + if (pw) + s = pw->pw_dir; + else +#endif + { + char* cur_dir = getcwd(nullptr, 0); + s = cur_dir; + free(cur_dir); + } + } + return s; +} + +void MakeDirIfNotExist(const char* path, int mode) { + if (!NFs::MakeDirectory(path, NFs::EFilePermission(mode)) && !NFs::Exists(path)) { + ythrow TSystemError() << "failed to create directory " << path; + } +} + +void MakePathIfNotExist(const char* path, int mode) { + NFs::MakeDirectoryRecursive(path, NFs::EFilePermission(mode)); + if (!NFs::Exists(path) || !TFileStat(path).IsDir()) { + ythrow TSystemError() << "failed to create directory " << path; + } +} + +const char* GetFileNameComponent(const char* f) { + const char* p = strrchr(f, LOCSLASH_C); +#ifdef _win_ + // "/" is also valid char separator on Windows + const char* p2 = strrchr(f, '/'); + if (p2 > p) + p = p2; +#endif + + if (p) { + return p + 1; + } + + return f; +} + +TString GetSystemTempDir() { +#ifdef _win_ + char buffer[1024]; + DWORD size = GetTempPath(1024, buffer); + if (!size) { + ythrow TSystemError() << "failed to get system temporary directory"; + } + return TString(buffer, size); +#else + const char* var = "TMPDIR"; + const char* def = "/tmp"; + const char* r = getenv(var); + const char* result = r ? r : def; + return result[0] == '/' ? result : ResolveDir(result); +#endif +} + +TString ResolveDir(const char* path) { + return ResolvePath(path, true); +} + +bool SafeResolveDir(const char* path, TString& result) { + try { + result = ResolvePath(path, true); + return true; + } catch (...) { + return false; + } +} + +TString GetDirName(const TString& path) { + return TFsPath(path).Dirname(); +} + +#ifdef _win32_ + +char* realpath(const char* pathname, char resolved_path[MAXPATHLEN]) { + // partial implementation: no path existence check + return _fullpath(resolved_path, pathname, MAXPATHLEN - 1); +} + +#endif + +TString GetBaseName(const TString& path) { + return TFsPath(path).Basename(); +} + +static bool IsAbsolutePath(const char* str) { + return str && TPathSplitTraitsLocal::IsAbsolutePath(TStringBuf(str, NStringPrivate::GetStringLengthWithLimit(str, 3))); +} + +int ResolvePath(const char* rel, const char* abs, char res[/*MAXPATHLEN*/], bool isdir) { + char t[MAXPATHLEN * 2 + 3]; + size_t len; + + *res = 0; + if (!rel || !*rel) + return EINVAL; + if (!IsAbsolutePath(rel) && IsAbsolutePath(abs)) { + len = strlcpy(t, abs, sizeof(t)); + if (len >= sizeof(t) - 3) + return EINVAL; + if (t[len - 1] != LOCSLASH_C) + t[len++] = LOCSLASH_C; + len += strlcpy(t + len, rel, sizeof(t) - len); + } else { + len = strlcpy(t, rel, sizeof(t)); + } + if (len >= sizeof(t) - 3) + return EINVAL; + if (isdir && t[len - 1] != LOCSLASH_C) { + t[len++] = LOCSLASH_C; + t[len] = 0; + } + if (!realpath(t, res)) { + if (!isdir && realpath(GetDirName(t).data(), res)) { + len = strlen(res); + if (res[len - 1] != LOCSLASH_C) { + res[len++] = LOCSLASH_C; + res[len] = 0; + } + strcpy(res + len, GetBaseName(t).data()); + return 0; + } + return errno ? errno : ENOENT; + } + if (isdir) { + len = strlen(res); + if (res[len - 1] != LOCSLASH_C) { + res[len++] = LOCSLASH_C; + res[len] = 0; + } + } + return 0; +} + +TString ResolvePath(const char* rel, const char* abs, bool isdir) { + char buf[PATH_MAX]; + if (ResolvePath(rel, abs, buf, isdir)) + ythrow yexception() << "cannot resolve path: \"" << rel << "\""; + return buf; +} + +TString ResolvePath(const char* path, bool isDir) { + return ResolvePath(path, nullptr, isDir); +} + +TString StripFileComponent(const TString& fileName) { + TString dir = IsDir(fileName) ? fileName : GetDirName(fileName); + if (!dir.empty() && dir.back() != GetDirectorySeparator()) { + dir.append(GetDirectorySeparator()); + } + return dir; +} diff --git a/util/folder/dirut.h b/util/folder/dirut.h new file mode 100644 index 0000000000..2537027b12 --- /dev/null +++ b/util/folder/dirut.h @@ -0,0 +1,119 @@ +#pragma once + +#include <util/system/defaults.h> +#include <util/system/sysstat.h> +#include <util/system/fs.h> +#include <util/generic/string.h> +#include <util/generic/yexception.h> + +#include <sys/types.h> + +#include <cerrno> +#include <cstdlib> + +#ifdef _win32_ + #include <util/system/winint.h> + #include <direct.h> + #include <malloc.h> + #include <time.h> + #include <io.h> + #include "dirent_win.h" + +// these live in mktemp_system.cpp +extern "C" int mkstemps(char* path, int slen); +char* mkdtemp(char* path); + +#else + #ifdef _sun_ + #include <alloca.h> + +char* mkdtemp(char* path); + #endif + #include <unistd.h> + #include <pwd.h> + #include <dirent.h> + #ifndef DT_DIR + #include <sys/stat.h> + #endif +#endif + +bool IsDir(const TString& path); + +int mkpath(char* path, int mode = 0777); + +TString GetHomeDir(); + +void MakeDirIfNotExist(const char* path, int mode = 0777); + +inline void MakeDirIfNotExist(const TString& path, int mode = 0777) { + MakeDirIfNotExist(path.data(), mode); +} + +/// Create path making parent directories as needed +void MakePathIfNotExist(const char* path, int mode = 0777); + +void SlashFolderLocal(TString& folder); +bool correctpath(TString& filename); +bool resolvepath(TString& folder, const TString& home); + +char GetDirectorySeparator(); +const char* GetDirectorySeparatorS(); + +void RemoveDirWithContents(TString dirName); + +const char* GetFileNameComponent(const char* f); + +inline TString GetFileNameComponent(const TString& f) { + return GetFileNameComponent(f.data()); +} + +/// RealPath doesn't guarantee trailing separator to be stripped or left in place for directories. +TString RealPath(const TString& path); // throws +TString RealLocation(const TString& path); /// throws; last file name component doesn't need to exist + +TString GetSystemTempDir(); + +int MakeTempDir(char path[/*FILENAME_MAX*/], const char* prefix); + +int ResolvePath(const char* rel, const char* abs, char res[/*FILENAME_MAX*/], bool isdir = false); +TString ResolvePath(const char* rel, const char* abs, bool isdir = false); +TString ResolvePath(const char* path, bool isDir = false); + +TString ResolveDir(const char* path); + +bool SafeResolveDir(const char* path, TString& result); + +TString GetDirName(const TString& path); + +TString GetBaseName(const TString& path); + +TString StripFileComponent(const TString& fileName); + +class TExistenceChecker { +public: + TExistenceChecker(bool strict = false) + : Strict(strict) + { + } + + void SetStrict(bool strict) { + Strict = strict; + } + + bool IsStrict() const { + return Strict; + } + + const char* Check(const char* fname) const { + if (!fname || !*fname) + return nullptr; + if (Strict) { + NFs::EnsureExists(fname); + } else if (!NFs::Exists(fname)) + fname = nullptr; + return fname; + } + +private: + bool Strict; +}; diff --git a/util/folder/dirut_ut.cpp b/util/folder/dirut_ut.cpp new file mode 100644 index 0000000000..45ebfc842c --- /dev/null +++ b/util/folder/dirut_ut.cpp @@ -0,0 +1,133 @@ +#include "dirut.h" +#include "tempdir.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/string.h> +#include <util/memory/tempbuf.h> +#include <util/stream/file.h> +#include <util/system/platform.h> + +Y_UNIT_TEST_SUITE(TDirutTest) { + Y_UNIT_TEST(TestRealPath) { + UNIT_ASSERT(IsDir(RealPath("."))); + } + + Y_UNIT_TEST(TestRealLocation) { + UNIT_ASSERT(IsDir(RealLocation("."))); + + TTempDir tempDir; + TString base = RealPath(tempDir()); + UNIT_ASSERT(!base.empty()); + + if (base.back() == GetDirectorySeparator()) { + base.pop_back(); + } + + TString path; + TString pathNotNorm; + + path = base + GetDirectorySeparatorS() + "no_such_file"; + UNIT_ASSERT(NFs::Exists(GetDirName(path))); + UNIT_ASSERT(!NFs::Exists(path)); + path = RealLocation(path); + UNIT_ASSERT(NFs::Exists(GetDirName(path))); + UNIT_ASSERT(!NFs::Exists(path)); + UNIT_ASSERT_EQUAL(GetDirName(path), base); + + pathNotNorm = base + GetDirectorySeparatorS() + "some_dir" + GetDirectorySeparatorS() + ".." + GetDirectorySeparatorS() + "no_such_file"; + MakeDirIfNotExist((base + GetDirectorySeparatorS() + "some_dir").data()); + pathNotNorm = RealLocation(pathNotNorm); + UNIT_ASSERT(NFs::Exists(GetDirName(pathNotNorm))); + UNIT_ASSERT(!NFs::Exists(pathNotNorm)); + UNIT_ASSERT_EQUAL(GetDirName(pathNotNorm), base); + + UNIT_ASSERT_EQUAL(path, pathNotNorm); + + path = base + GetDirectorySeparatorS() + "file"; + { + TFixedBufferFileOutput file(path); + } + UNIT_ASSERT(NFs::Exists(GetDirName(path))); + UNIT_ASSERT(NFs::Exists(path)); + UNIT_ASSERT(NFs::Exists(path)); + path = RealLocation(path); + UNIT_ASSERT(NFs::Exists(GetDirName(path))); + UNIT_ASSERT(NFs::Exists(path)); + UNIT_ASSERT_EQUAL(GetDirName(path), base); + } + + void DoTest(const char* p, const char* base, const char* canon) { + TString path(p); + UNIT_ASSERT(resolvepath(path, base)); + UNIT_ASSERT(path == canon); + } + + Y_UNIT_TEST(TestResolvePath) { +#ifdef _win_ + DoTest("bar", "c:\\foo\\baz", "c:\\foo\\baz\\bar"); + DoTest("c:\\foo\\bar", "c:\\bar\\baz", "c:\\foo\\bar"); +#else + DoTest("bar", "/foo/baz", "/foo/bar"); + DoTest("/foo/bar", "/bar/baz", "/foo/bar"); + + #ifdef NDEBUG + DoTest("bar", "./baz", "./bar"); + #if 0 // should we support, for consistency, single-label dirs + DoTest("bar", "baz", "bar"); + #endif + #endif +#endif + } + + Y_UNIT_TEST(TestResolvePathRelative) { + TTempDir tempDir; + TTempBuf tempBuf; + TString base = RealPath(tempDir()); + if (base.back() == GetDirectorySeparator()) { + base.pop_back(); + } + + // File + TString path = base + GetDirectorySeparatorS() + "file"; + { + TFixedBufferFileOutput file(path); + } + ResolvePath("file", base.data(), tempBuf.Data(), false); + UNIT_ASSERT_EQUAL(tempBuf.Data(), path); + + // Dir + path = base + GetDirectorySeparatorS() + "dir"; + MakeDirIfNotExist(path.data()); + ResolvePath("dir", base.data(), tempBuf.Data(), true); + UNIT_ASSERT_EQUAL(tempBuf.Data(), path + GetDirectorySeparatorS()); + + // Absent file in existent dir + path = base + GetDirectorySeparatorS() + "nofile"; + ResolvePath("nofile", base.data(), tempBuf.Data(), false); + UNIT_ASSERT_EQUAL(tempBuf.Data(), path); + } + + Y_UNIT_TEST(TestGetDirName) { + UNIT_ASSERT_VALUES_EQUAL(".", GetDirName("parambambam")); + } + + Y_UNIT_TEST(TestStripFileComponent) { + static const TString tmpDir = "tmp_dir_for_tests"; + static const TString tmpSubDir = tmpDir + GetDirectorySeparatorS() + "subdir"; + static const TString tmpFile = tmpDir + GetDirectorySeparatorS() + "file"; + + // creating tmp dir and subdirs + MakeDirIfNotExist(tmpDir.data()); + MakeDirIfNotExist(tmpSubDir.data()); + { + TFixedBufferFileOutput file(tmpFile); + } + + UNIT_ASSERT_EQUAL(StripFileComponent(tmpDir), tmpDir + GetDirectorySeparatorS()); + UNIT_ASSERT_EQUAL(StripFileComponent(tmpSubDir), tmpSubDir + GetDirectorySeparatorS()); + UNIT_ASSERT_EQUAL(StripFileComponent(tmpFile), tmpDir + GetDirectorySeparatorS()); + + RemoveDirWithContents(tmpDir); + } +}; diff --git a/util/folder/filelist.cpp b/util/folder/filelist.cpp new file mode 100644 index 0000000000..b21fcdbf20 --- /dev/null +++ b/util/folder/filelist.cpp @@ -0,0 +1,40 @@ +#include "dirut.h" +#include "filelist.h" +#include "iterator.h" + +#include <util/system/defaults.h> + +void TFileEntitiesList::Fill(const TString& dirname, TStringBuf prefix, TStringBuf suffix, int depth, bool sort) { + TDirIterator::TOptions opts; + opts.SetMaxLevel(depth); + if (sort) { + opts.SetSortByName(); + } + + TDirIterator dir(dirname, opts); + Clear(); + + size_t dirNameLength = dirname.length(); + while (dirNameLength && (dirname[dirNameLength - 1] == '\\' || dirname[dirNameLength - 1] == '/')) { + --dirNameLength; + } + + for (auto file = dir.begin(); file != dir.end(); ++file) { + if (file->fts_pathlen == file->fts_namelen || file->fts_pathlen <= dirNameLength) { + continue; + } + + TStringBuf filename = file->fts_path + dirNameLength + 1; + + if (filename.empty() || !filename.StartsWith(prefix) || !filename.EndsWith(suffix)) { + continue; + } + + if (((Mask & EM_FILES) && file->fts_info == FTS_F) || ((Mask & EM_DIRS) && file->fts_info == FTS_D) || ((Mask & EM_SLINKS) && file->fts_info == FTS_SL)) { + ++FileNamesSize; + FileNames.Append(filename.data(), filename.size() + 1); + } + } + + Restart(); +} diff --git a/util/folder/filelist.h b/util/folder/filelist.h new file mode 100644 index 0000000000..3f615fa4c2 --- /dev/null +++ b/util/folder/filelist.h @@ -0,0 +1,81 @@ +#pragma once + +#include <util/generic/buffer.h> +#include <util/generic/string.h> +#include <util/generic/strbuf.h> +#include <util/generic/flags.h> + +class TFileEntitiesList { +public: + enum EMaskFlag { + EM_FILES = 1, + EM_DIRS = 2, + EM_SLINKS = 4, + + EM_FILES_DIRS = EM_FILES | EM_DIRS, + EM_FILES_SLINKS = EM_FILES | EM_SLINKS, + EM_DIRS_SLINKS = EM_DIRS | EM_SLINKS, + EM_FILES_DIRS_SLINKS = EM_FILES | EM_DIRS | EM_SLINKS + }; + Y_DECLARE_FLAGS(EMask, EMaskFlag) + + TFileEntitiesList(EMask mask) + : Mask(mask) + { + Clear(); + } + + void Clear() { + Cur = nullptr; + FileNamesSize = CurName = 0; + FileNames.Clear(); + FileNames.Append("", 1); + } + + const char* Next() { + return Cur = (CurName++ < FileNamesSize ? strchr(Cur, 0) + 1 : nullptr); + } + + size_t Size() { + return FileNamesSize; + } + + inline void Fill(const TString& dirname, bool sort = false) { + Fill(dirname, TStringBuf(), sort); + } + + inline void Fill(const TString& dirname, TStringBuf prefix, bool sort = false) { + Fill(dirname, prefix, TStringBuf(), 1, sort); + } + + void Fill(const TString& dirname, TStringBuf prefix, TStringBuf suffix, int depth, bool sort = false); + + void Restart() { + Cur = FileNames.Data(); + CurName = 0; + } + +protected: + TBuffer FileNames; + size_t FileNamesSize, CurName; + const char* Cur; + EMask Mask; +}; + +Y_DECLARE_OPERATORS_FOR_FLAGS(TFileEntitiesList::EMask) + +class TFileList: public TFileEntitiesList { +public: + TFileList() + : TFileEntitiesList(TFileEntitiesList::EM_FILES) + { + } +}; + +class TDirsList: public TFileEntitiesList { +public: + TDirsList() + : TFileEntitiesList(TFileEntitiesList::EM_DIRS) + { + } +}; diff --git a/util/folder/filelist_ut.cpp b/util/folder/filelist_ut.cpp new file mode 100644 index 0000000000..0cdcdf3d00 --- /dev/null +++ b/util/folder/filelist_ut.cpp @@ -0,0 +1,56 @@ +#include "dirut.h" +#include "filelist.h" +#include "tempdir.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/system/file.h> +#include <util/generic/string.h> + +class TFileListTest: public TTestBase { + UNIT_TEST_SUITE(TFileListTest); + UNIT_TEST(TestSimple); + UNIT_TEST(TestPrefix); + UNIT_TEST_SUITE_END(); + +public: + void TestSimple(); + void TestPrefix(); +}; + +void TFileListTest::TestSimple() { + TTempDir tempDir("nonexistingdir"); + MakeDirIfNotExist((tempDir() + LOCSLASH_S "subdir").data()); + TFile((tempDir() + LOCSLASH_S "subdir" LOCSLASH_S "file").data(), CreateAlways); + + TFileList fileList; + fileList.Fill(tempDir().data(), "", "", 1000); + TString fileName(fileList.Next()); + UNIT_ASSERT_EQUAL(fileName, "subdir" LOCSLASH_S "file"); + UNIT_ASSERT_EQUAL(fileList.Next(), nullptr); +} + +void TFileListTest::TestPrefix() { + TTempDir tempDir("nonexistingdir"); + TFile((tempDir() + LOCSLASH_S "good_file1").data(), CreateAlways); + TFile((tempDir() + LOCSLASH_S "good_file2").data(), CreateAlways); + TFile((tempDir() + LOCSLASH_S "bad_file1").data(), CreateAlways); + TFile((tempDir() + LOCSLASH_S "bad_file2").data(), CreateAlways); + + const bool SORT = true; + TFileList fileList; + { + fileList.Fill(tempDir().data(), "good_file", SORT); + UNIT_ASSERT_EQUAL(TString(fileList.Next()), "good_file1"); + UNIT_ASSERT_EQUAL(TString(fileList.Next()), "good_file2"); + UNIT_ASSERT_EQUAL(fileList.Next(), nullptr); + } + { + fileList.Fill(tempDir().data(), "bad_file", SORT); + UNIT_ASSERT_EQUAL(TString(fileList.Next()), "bad_file1"); + UNIT_ASSERT_EQUAL(TString(fileList.Next()), "bad_file2"); + UNIT_ASSERT_EQUAL(fileList.Next(), nullptr); + } +} + +UNIT_TEST_SUITE_REGISTRATION(TFileListTest); diff --git a/util/folder/fts.cpp b/util/folder/fts.cpp new file mode 100644 index 0000000000..0e6a6f86eb --- /dev/null +++ b/util/folder/fts.cpp @@ -0,0 +1,1432 @@ +/*- + * Copyright (c) 1990, 1993, 1994 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $OpenBSD: fts.c,v 1.22 1999/10/03 19:22:22 millert Exp $ + */ + +#include <util/memory/tempbuf.h> +#include <util/system/compat.h> +#include <util/system/compiler.h> +#include <util/system/defaults.h> +#include <util/system/error.h> + +#include <stdlib.h> +#ifndef _win_ + #include <inttypes.h> + #include <sys/param.h> + #include <dirent.h> + #include <errno.h> + #include <string.h> + #include <unistd.h> +#else + #include <direct.h> + #include "dirent_win.h" + #include "lstat_win.h" +#endif + +#include <sys/stat.h> + +#include <fcntl.h> + +#include "fts.h" +#include <limits.h> + +#ifndef _win_ + +static const dird invalidDirD = -1; + +dird get_cwdd() { + return open(".", O_RDONLY, 0); +} + +dird get_dird(char* path) { + return open(path, O_RDONLY, 0); +} + +int valid_dird(dird fd) { + return fd < 0; +} + +void close_dird(dird fd) { + (void)close(fd); +} + +int chdir_dird(dird fd) { + return fchdir(fd); +} + +int cmp_dird(dird fd1, dird fd2) { + return fd1 - fd2; +} + +#else // ndef _win_ + +int stat64UTF(const char* path, struct _stat64* _Stat) { + int len_converted = MultiByteToWideChar(CP_UTF8, 0, path, -1, 0, 0); + if (len_converted == 0) { + return -1; + } + WCHAR* buf = (WCHAR*)malloc(sizeof(WCHAR) * (len_converted)); + if (buf == nullptr) { + return -1; + } + MultiByteToWideChar(CP_UTF8, 0, path, -1, buf, len_converted); + + int ret = _wstat64(buf, _Stat); + free(buf); + return ret; +} + +int stat64UTF(dird path, struct _stat64* _Stat) { + return _wstat64(path, _Stat); +} + +const dird invalidDirD = nullptr; + +dird get_cwdd() { + return _wgetcwd(nullptr, 0); +} + +int valid_dird(dird fd) { + return fd == nullptr; +} + +void close_dird(dird fd) { + free(fd); +} + +int chdir_dird(dird fd) { + return _wchdir(fd); +} + +int chdir_dird(const char* path) { + int len_converted = MultiByteToWideChar(CP_UTF8, 0, path, -1, 0, 0); + if (len_converted == 0) { + return -1; + } + WCHAR* buf = (WCHAR*)malloc(sizeof(WCHAR) * (len_converted)); + if (buf == nullptr) { + return -1; + } + MultiByteToWideChar(CP_UTF8, 0, path, -1, buf, len_converted); + int ret = _wchdir(buf); + free(buf); + return ret; +} + +int cmp_dird(dird fd1, dird fd2) { + return lstrcmpW(fd1, fd2); +} + +dird get_dird(char* path) { + int len_converted = MultiByteToWideChar(CP_UTF8, 0, path, -1, 0, 0); + if (len_converted == 0) { + return nullptr; + } + WCHAR* buf = (WCHAR*)malloc(sizeof(WCHAR) * (len_converted)); + if (buf == nullptr) { + return nullptr; + } + MultiByteToWideChar(CP_UTF8, 0, path, -1, buf, len_converted); + + WCHAR* ret = _wfullpath(0, buf, 0); + + free(buf); + return ret; +} + +#endif // ndef _win_ + +#ifdef _win_ + #define S_ISDIR(st_mode) ((st_mode & _S_IFMT) == _S_IFDIR) + #define S_ISREG(st_mode) ((st_mode & _S_IFMT) == _S_IFREG) + #define S_ISLNK(st_mode) ((st_mode & _S_IFMT) == _S_IFLNK) + #define O_RDONLY _O_RDONLY +static int fts_safe_changedir(FTS*, FTSENT*, int, dird); +#endif + +#if defined(__svr4__) || defined(__linux__) || defined(__CYGWIN__) || defined(_win_) + #ifdef MAX + #undef MAX + #endif + #define MAX(a, b) ((a) > (b) ? (a) : (b)) + #undef ALIGNBYTES + #undef ALIGN + #define ALIGNBYTES (sizeof(long long) - 1) + #define ALIGN(p) (((uintptr_t)(p) + ALIGNBYTES) & ~ALIGNBYTES) + #if !defined(__linux__) && !defined(__CYGWIN__) + #define dirfd(dirp) ((dirp)->dd_fd) + #endif + #define D_NAMLEN(dirp) (strlen(dirp->d_name)) +#else + #define D_NAMLEN(dirp) (dirp->d_namlen) +#endif + +static FTSENT* fts_alloc(FTS*, const char*, int); +static FTSENT* fts_build(FTS*, int); +static void fts_lfree(FTSENT*); +static void fts_load(FTS*, FTSENT*); +static size_t fts_maxarglen(char* const*); +static void fts_padjust(FTS*); +static int fts_palloc(FTS*, size_t); +static FTSENT* fts_sort(FTS*, FTSENT*, int); +static u_short fts_stat(FTS*, FTSENT*, int); +static int fts_safe_changedir(FTS*, FTSENT*, int, const char*); + +#define ISDOT(a) (a[0] == '.' && (!a[1] || (a[1] == '.' && !a[2]))) + +#define CLR(opt) (sp->fts_options &= ~(opt)) +#define ISSET(opt) (sp->fts_options & (opt)) +#define SET(opt) (sp->fts_options |= (opt)) + +#define FCHDIR(sp, fd) (!ISSET(FTS_NOCHDIR) && chdir_dird(fd)) + +/* fts_build flags */ +#define BCHILD 1 /* yfts_children */ +#define BNAMES 2 /* yfts_children, names only */ +#define BREAD 3 /* yfts_read */ + +static u_short +yfts_type_from_info(u_short info) { + if (info == FTS_D || info == FTS_DP || info == FTS_DOT) { + return FTS_D; + } else if (info == FTS_F) { + return FTS_F; + } else if (info == FTS_SL || info == FTS_SLNONE) { + return FTS_SL; + } + return FTS_NSOK; +} + +static void* +yreallocf(void* ptr, size_t size) +{ + void* nptr; + + nptr = realloc(ptr, size); + if (!nptr && ptr) { + free(ptr); + } + return (nptr); +} + +FTS* yfts_open(char* const* argv, int options, int (*compar)(const FTSENT**, const FTSENT**)) +{ + FTS* sp; + FTSENT *p, *root; + int nitems; + FTSENT *parent, *tmp; + int len; + + errno = 0; + + Y_ASSERT(argv); + if (!*argv) { + errno = ENOENT; + return nullptr; + } + + /* Options check. */ + if (options & ~FTS_OPTIONMASK) { + errno = EINVAL; + return nullptr; + } + + /* Allocate/initialize the stream */ + if ((sp = (FTS*)malloc(sizeof(FTS))) == nullptr) { + return nullptr; + } + memset(sp, 0, sizeof(FTS)); + sp->fts_compar = compar; + sp->fts_options = options; + + /* Shush, GCC. */ + tmp = nullptr; + + /* Logical walks turn on NOCHDIR; symbolic links are too hard. */ + if (ISSET(FTS_LOGICAL)) { + SET(FTS_NOCHDIR); + } + + /* + * Start out with 1K of path space, and enough, in any case, + * to hold the user's paths. + */ + if (fts_palloc(sp, MAX(fts_maxarglen(argv), MAXPATHLEN))) { + goto mem1; + } + + /* Allocate/initialize root's parent. */ + if ((parent = fts_alloc(sp, "", 0)) == nullptr) { + goto mem2; + } + parent->fts_level = FTS_ROOTPARENTLEVEL; + + /* Allocate/initialize root(s). */ + for (root = nullptr, nitems = 0; *argv; ++argv, ++nitems) { + /* Don't allow zero-length paths. */ + + len = strlen(*argv); + +//Any subsequent windows call will expect no trailing slashes so we will remove them here +#ifdef _win_ + while (len && ((*argv)[len - 1] == '\\' || (*argv)[len - 1] == '/')) { + --len; + } +#endif + + if (len == 0) { + errno = ENOENT; + goto mem3; + } + + p = fts_alloc(sp, *argv, len); + p->fts_level = FTS_ROOTLEVEL; + p->fts_parent = parent; + p->fts_accpath = p->fts_name; + p->fts_info = fts_stat(sp, p, ISSET(FTS_COMFOLLOW)); + p->fts_type = yfts_type_from_info(p->fts_info); + + /* Command-line "." and ".." are real directories. */ + if (p->fts_info == FTS_DOT) { + p->fts_info = FTS_D; + } + + /* + * If comparison routine supplied, traverse in sorted + * order; otherwise traverse in the order specified. + */ + if (compar) { + p->fts_link = root; + root = p; + } else { + p->fts_link = nullptr; + if (root == nullptr) { + tmp = root = p; + } else { + tmp->fts_link = p; + tmp = p; + } + } + } + if (compar && nitems > 1) { + root = fts_sort(sp, root, nitems); + } + + /* + * Allocate a dummy pointer and make yfts_read think that we've just + * finished the node before the root(s); set p->fts_info to FTS_INIT + * so that everything about the "current" node is ignored. + */ + if ((sp->fts_cur = fts_alloc(sp, "", 0)) == nullptr) { + goto mem3; + } + sp->fts_cur->fts_level = FTS_ROOTLEVEL; + sp->fts_cur->fts_link = root; + sp->fts_cur->fts_info = FTS_INIT; + + /* + * If using chdir(2), grab a file descriptor pointing to dot to ensure + * that we can get back here; this could be avoided for some paths, + * but almost certainly not worth the effort. Slashes, symbolic links, + * and ".." are all fairly nasty problems. Note, if we can't get the + * descriptor we run anyway, just more slowly. + */ + + if (!ISSET(FTS_NOCHDIR) && valid_dird(sp->fts_rfd = get_cwdd())) { + SET(FTS_NOCHDIR); + } + + return (sp); + +mem3: + fts_lfree(root); + free(parent); +mem2: + free(sp->fts_path); +mem1: + free(sp); + return nullptr; +} + +static void +fts_load(FTS* sp, FTSENT* p) +{ + size_t len; + char* cp; + + /* + * Load the stream structure for the next traversal. Since we don't + * actually enter the directory until after the preorder visit, set + * the fts_accpath field specially so the chdir gets done to the right + * place and the user can access the first node. From yfts_open it's + * known that the path will fit. + */ + len = p->fts_pathlen = p->fts_namelen; + memmove((void*)sp->fts_path, (void*)p->fts_name, len + 1); + if ((cp = strrchr(p->fts_name, LOCSLASH_C)) != nullptr && (cp != p->fts_name || cp[1])) { + len = strlen(++cp); + memmove((void*)p->fts_name, (void*)cp, len + 1); + p->fts_namelen = (u_short)len; + } + p->fts_accpath = p->fts_path = sp->fts_path; + sp->fts_dev = p->fts_dev; +} + +int yfts_close(FTS* sp) +{ + FTSENT *freep, *p; + int saved_errno; + + /* + * This still works if we haven't read anything -- the dummy structure + * points to the root list, so we step through to the end of the root + * list which has a valid parent pointer. + */ + if (sp->fts_cur) { + for (p = sp->fts_cur; p->fts_level >= FTS_ROOTLEVEL;) { + freep = p; + p = p->fts_link ? p->fts_link : p->fts_parent; + free(freep); + } + free(p); + } + + /* Free up child linked list, sort array, path buffer. */ + if (sp->fts_child) { + fts_lfree(sp->fts_child); + } + if (sp->fts_array) { + free(sp->fts_array); + } + free(sp->fts_path); + + /* Return to original directory, save errno if necessary. */ + if (!ISSET(FTS_NOCHDIR)) { + saved_errno = chdir_dird(sp->fts_rfd) ? errno : 0; + close_dird(sp->fts_rfd); + + /* Set errno and return. */ + if (saved_errno != 0) { + /* Free up the stream pointer. */ + free(sp); + errno = saved_errno; + return (-1); + } + } + + /* Free up the stream pointer. */ + free(sp); + return (0); +} + +/* + * Special case of "/" at the end of the path so that slashes aren't + * appended which would cause paths to be written as "....//foo". + */ +#define NAPPEND(p) \ + (p->fts_path[p->fts_pathlen - 1] == LOCSLASH_C \ + ? p->fts_pathlen - 1 \ + : p->fts_pathlen) + +FTSENT* +yfts_read(FTS* sp) { + FTSENT *p, *tmp; + int instr; + char* t; + int saved_errno; + + ClearLastSystemError(); + + /* If finished or unrecoverable error, return NULL. */ + if (sp->fts_cur == nullptr || ISSET(FTS_STOP)) { + return nullptr; + } + + /* Set current node pointer. */ + p = sp->fts_cur; + + /* Save and zero out user instructions. */ + instr = p->fts_instr; + p->fts_instr = FTS_NOINSTR; + + /* Any type of file may be re-visited; re-stat and re-turn. */ + if (instr == FTS_AGAIN) { + p->fts_info = fts_stat(sp, p, 0); + p->fts_type = yfts_type_from_info(p->fts_info); + return (p); + } + + /* + * Following a symlink -- SLNONE test allows application to see + * SLNONE and recover. If indirecting through a symlink, have + * keep a pointer to current location. If unable to get that + * pointer, follow fails. + */ + if (instr == FTS_FOLLOW && + (p->fts_info == FTS_SL || p->fts_info == FTS_SLNONE)) { + p->fts_info = fts_stat(sp, p, 1); + p->fts_type = yfts_type_from_info(p->fts_info); + if (p->fts_info == FTS_D && !ISSET(FTS_NOCHDIR)) { + if (valid_dird(p->fts_symfd = get_cwdd())) { + p->fts_errno = errno; + p->fts_info = FTS_ERR; + } else { + p->fts_flags |= FTS_SYMFOLLOW; + } + } + return (p); + } + + /* Directory in pre-order. */ + if (p->fts_info == FTS_D) { + /* If skipped or crossed mount point, do post-order visit. */ + if (instr == FTS_SKIP || + (ISSET(FTS_XDEV) && p->fts_dev != sp->fts_dev)) { + if (p->fts_flags & FTS_SYMFOLLOW) { + close_dird(p->fts_symfd); + } + if (sp->fts_child) { + fts_lfree(sp->fts_child); + sp->fts_child = nullptr; + } + p->fts_info = FTS_DP; + return (p); + } + + /* Rebuild if only read the names and now traversing. */ + if (sp->fts_child && ISSET(FTS_NAMEONLY)) { + CLR(FTS_NAMEONLY); + fts_lfree(sp->fts_child); + sp->fts_child = nullptr; + } + + /* + * Cd to the subdirectory. + * + * If have already read and now fail to chdir, whack the list + * to make the names come out right, and set the parent errno + * so the application will eventually get an error condition. + * Set the FTS_DONTCHDIR flag so that when we logically change + * directories back to the parent we don't do a chdir. + * + * If haven't read do so. If the read fails, fts_build sets + * FTS_STOP or the fts_info field of the node. + */ + if (sp->fts_child) { + if (fts_safe_changedir(sp, p, -1, p->fts_accpath)) { + p->fts_errno = errno; + p->fts_flags |= FTS_DONTCHDIR; + for (p = sp->fts_child; p; p = p->fts_link) { + p->fts_accpath = + p->fts_parent->fts_accpath; + } + } + } else if ((sp->fts_child = fts_build(sp, BREAD)) == nullptr) { + if (ISSET(FTS_STOP)) { + return nullptr; + } + return (p); + } + p = sp->fts_child; + sp->fts_child = nullptr; + goto name; + } + + /* Move to the next node on this level. */ +next: + tmp = p; + if ((p = p->fts_link) != nullptr) { + free(tmp); + + /* + * If reached the top, return to the original directory (or + * the root of the tree), and load the paths for the next root. + */ + if (p->fts_level == FTS_ROOTLEVEL) { + if (FCHDIR(sp, sp->fts_rfd)) { + SET(FTS_STOP); + return nullptr; + } + fts_load(sp, p); + return (sp->fts_cur = p); + } + + /* + * User may have called yfts_set on the node. If skipped, + * ignore. If followed, get a file descriptor so we can + * get back if necessary. + */ + if (p->fts_instr == FTS_SKIP) { + goto next; + } + if (p->fts_instr == FTS_FOLLOW) { + p->fts_info = fts_stat(sp, p, 1); + p->fts_type = yfts_type_from_info(p->fts_info); + if (p->fts_info == FTS_D && !ISSET(FTS_NOCHDIR)) { + if (valid_dird(p->fts_symfd = + get_cwdd())) { + p->fts_errno = errno; + p->fts_info = FTS_ERR; + } else { + p->fts_flags |= FTS_SYMFOLLOW; + } + } + p->fts_instr = FTS_NOINSTR; + } + + name: + t = sp->fts_path + NAPPEND(p->fts_parent); + *t++ = LOCSLASH_C; + memmove(t, p->fts_name, (size_t)p->fts_namelen + 1); + return (sp->fts_cur = p); + } + + /* Move up to the parent node. */ + p = tmp->fts_parent; + free(tmp); + + if (p->fts_level == FTS_ROOTPARENTLEVEL) { + /* + * Done; free everything up and set errno to 0 so the user + * can distinguish between error and EOF. + */ + free(p); + errno = 0; + return (sp->fts_cur = nullptr); + } + + /* NUL terminate the pathname. */ + sp->fts_path[p->fts_pathlen] = '\0'; + + /* + * Return to the parent directory. If at a root node or came through + * a symlink, go back through the file descriptor. Otherwise, cd up + * one directory. + */ + if (p->fts_level == FTS_ROOTLEVEL) { + if (FCHDIR(sp, sp->fts_rfd)) { + SET(FTS_STOP); + return nullptr; + } + } else if (p->fts_flags & FTS_SYMFOLLOW) { + if (FCHDIR(sp, p->fts_symfd)) { + saved_errno = errno; + close_dird(p->fts_symfd); + errno = saved_errno; + SET(FTS_STOP); + return nullptr; + } + close_dird(p->fts_symfd); + } else if (!(p->fts_flags & FTS_DONTCHDIR) && + fts_safe_changedir(sp, p->fts_parent, -1, "..")) { + SET(FTS_STOP); + return nullptr; + } + p->fts_info = p->fts_errno ? FTS_ERR : FTS_DP; + return (sp->fts_cur = p); +} + +/* + * Fts_set takes the stream as an argument although it's not used in this + * implementation; it would be necessary if anyone wanted to add global + * semantics to fts using yfts_set. An error return is allowed for similar + * reasons. + */ +/* ARGSUSED */ +int yfts_set(FTS* sp, FTSENT* p, int instr) +{ + (void)sp; //Unused + if (instr && instr != FTS_AGAIN && instr != FTS_FOLLOW && + instr != FTS_NOINSTR && instr != FTS_SKIP) { + errno = EINVAL; + return (1); + } + p->fts_instr = (u_short)instr; + return (0); +} + +FTSENT* +yfts_children(FTS* sp, int instr) +{ + FTSENT* p; + dird fd; + if (instr && instr != FTS_NAMEONLY) { + errno = EINVAL; + return nullptr; + } + + /* Set current node pointer. */ + p = sp->fts_cur; + + /* + * Errno set to 0 so user can distinguish empty directory from + * an error. + */ + errno = 0; + + /* Fatal errors stop here. */ + if (ISSET(FTS_STOP)) { + return nullptr; + } + + /* Return logical hierarchy of user's arguments. */ + if (p->fts_info == FTS_INIT) { + return (p->fts_link); + } + + /* + * If not a directory being visited in pre-order, stop here. Could + * allow FTS_DNR, assuming the user has fixed the problem, but the + * same effect is available with FTS_AGAIN. + */ + if (p->fts_info != FTS_D /* && p->fts_info != FTS_DNR */) { + return nullptr; + } + + /* Free up any previous child list. */ + if (sp->fts_child) { + fts_lfree(sp->fts_child); + } + + if (instr == FTS_NAMEONLY) { + SET(FTS_NAMEONLY); + instr = BNAMES; + } else { + instr = BCHILD; + } + + /* + * If using chdir on a relative path and called BEFORE yfts_read does + * its chdir to the root of a traversal, we can lose -- we need to + * chdir into the subdirectory, and we don't know where the current + * directory is, so we can't get back so that the upcoming chdir by + * yfts_read will work. + */ + if (p->fts_level != FTS_ROOTLEVEL || p->fts_accpath[0] == LOCSLASH_C || + ISSET(FTS_NOCHDIR)) { + return (sp->fts_child = fts_build(sp, instr)); + } + + if (valid_dird(fd = get_cwdd())) { + return nullptr; + } + sp->fts_child = fts_build(sp, instr); + if (chdir_dird(fd)) { + close_dird(fd); + return nullptr; + } + close_dird(fd); + return (sp->fts_child); +} + +static inline struct dirent* yreaddir(DIR* dir, struct dirent* de) { + // TODO(yazevnul|IGNIETFERRO-1070): remove these macroses by replacing `readdir_r` with proper + // alternative + Y_PRAGMA_DIAGNOSTIC_PUSH + Y_PRAGMA_NO_DEPRECATED + if (readdir_r(dir, de, &de) == 0) { + Y_PRAGMA_DIAGNOSTIC_POP + return de; + } + + return nullptr; +} + +/* + * This is the tricky part -- do not casually change *anything* in here. The + * idea is to build the linked list of entries that are used by yfts_children + * and yfts_read. There are lots of special cases. + * + * The real slowdown in walking the tree is the stat calls. If FTS_NOSTAT is + * set and it's a physical walk (so that symbolic links can't be directories), + * we can do things quickly. First, if it's a 4.4BSD file system, the type + * of the file is in the directory entry. Otherwise, we assume that the number + * of subdirectories in a node is equal to the number of links to the parent. + * The former skips all stat calls. The latter skips stat calls in any leaf + * directories and for any files after the subdirectories in the directory have + * been found, cutting the stat calls by about 2/3. + */ +static FTSENT* +fts_build(FTS* sp, int type) +{ + struct dirent* dp; + FTSENT *p, *head; + int nitems; + FTSENT *cur, *tail; + +#ifdef _win_ + dird dirpd; + struct DIR* dirp; +#else + DIR* dirp; +#endif + + void* oldaddr; + int cderrno, descend, len, level, maxlen, nlinks, saved_errno, + nostat, doadjust; + char* cp; + + /* Set current node pointer. */ + cur = sp->fts_cur; + + /* + * Open the directory for reading. If this fails, we're done. + * If being called from yfts_read, set the fts_info field. + */ +#ifdef FTS_WHITEOUT + if (ISSET(FTS_WHITEOUT)) + oflag = DTF_NODUP | DTF_REWIND; + else + oflag = DTF_HIDEW | DTF_NODUP | DTF_REWIND; +#else + #define __opendir2(path, flag) opendir(path) +#endif + if ((dirp = __opendir2(cur->fts_accpath, oflag)) == nullptr) { + if (type == BREAD) { + cur->fts_info = FTS_DNR; + cur->fts_errno = errno; + } + return nullptr; + } + +#ifdef _win_ + dirpd = get_dird(cur->fts_accpath); +#endif + + /* + * Nlinks is the number of possible entries of type directory in the + * directory if we're cheating on stat calls, 0 if we're not doing + * any stat calls at all, -1 if we're doing stats on everything. + */ + if (type == BNAMES) { + nlinks = 0; + /* Be quiet about nostat, GCC. */ + nostat = 0; + } else if (ISSET(FTS_NOSTAT) && ISSET(FTS_PHYSICAL)) { + nlinks = cur->fts_nlink - (ISSET(FTS_SEEDOT) ? 0 : 2); + nostat = 1; + } else { + nlinks = -1; + nostat = 0; + } + + /* + * If we're going to need to stat anything or we want to descend + * and stay in the directory, chdir. If this fails we keep going, + * but set a flag so we don't chdir after the post-order visit. + * We won't be able to stat anything, but we can still return the + * names themselves. Note, that since yfts_read won't be able to + * chdir into the directory, it will have to return different path + * names than before, i.e. "a/b" instead of "b". Since the node + * has already been visited in pre-order, have to wait until the + * post-order visit to return the error. There is a special case + * here, if there was nothing to stat then it's not an error to + * not be able to stat. This is all fairly nasty. If a program + * needed sorted entries or stat information, they had better be + * checking FTS_NS on the returned nodes. + */ + cderrno = 0; + if (nlinks || type == BREAD) { +#ifndef _win_ + if (fts_safe_changedir(sp, cur, dirfd(dirp), nullptr)) { +#else + if (fts_safe_changedir(sp, cur, -1, dirpd)) { +#endif + + if (nlinks && type == BREAD) { + cur->fts_errno = errno; + } + cur->fts_flags |= FTS_DONTCHDIR; + descend = 0; + cderrno = errno; + (void)closedir(dirp); + dirp = nullptr; +#ifdef _win_ + close_dird(dirpd); + dirpd = invalidDirD; +#else + Y_UNUSED(invalidDirD); +#endif + } else { + descend = 1; + } + } else { + descend = 0; + } + + /* + * Figure out the max file name length that can be stored in the + * current path -- the inner loop allocates more path as necessary. + * We really wouldn't have to do the maxlen calculations here, we + * could do them in yfts_read before returning the path, but it's a + * lot easier here since the length is part of the dirent structure. + * + * If not changing directories set a pointer so that can just append + * each new name into the path. + */ + len = NAPPEND(cur); + if (ISSET(FTS_NOCHDIR)) { + cp = sp->fts_path + len; + *cp++ = LOCSLASH_C; + } else { + /* GCC, you're too verbose. */ + cp = nullptr; + } + ++len; + maxlen = sp->fts_pathlen - len; + + level = cur->fts_level + 1; + + /* Read the directory, attaching each entry to the `link' pointer. */ + doadjust = 0; + + //to ensure enough buffer + TTempBuf dpe; + + for (head = tail = nullptr, nitems = 0; dirp && (dp = yreaddir(dirp, (struct dirent*)dpe.Data())) != nullptr;) { + if (!ISSET(FTS_SEEDOT) && ISDOT(dp->d_name)) { + continue; + } + + if ((p = fts_alloc(sp, dp->d_name, (int)strlen(dp->d_name))) == nullptr) { + goto mem1; + } + if (strlen(dp->d_name) >= (size_t)maxlen) { /* include space for NUL */ + oldaddr = sp->fts_path; + if (fts_palloc(sp, strlen(dp->d_name) + len + 1)) { + /* + * No more memory for path or structures. Save + * errno, free up the current structure and the + * structures already allocated. + */ + mem1: + saved_errno = errno; + if (p) { + free(p); + } + fts_lfree(head); + (void)closedir(dirp); +#ifdef _win_ + close_dird(dirpd); +#endif + cur->fts_info = FTS_ERR; + SET(FTS_STOP); + errno = saved_errno; + return nullptr; + } + /* Did realloc() change the pointer? */ + if (oldaddr != sp->fts_path) { + doadjust = 1; + if (ISSET(FTS_NOCHDIR)) { + cp = sp->fts_path + len; + } + } + maxlen = sp->fts_pathlen - len; + } + + if (len + strlen(dp->d_name) >= USHRT_MAX) { + /* + * In an FTSENT, fts_pathlen is a u_short so it is + * possible to wraparound here. If we do, free up + * the current structure and the structures already + * allocated, then error out with ENAMETOOLONG. + */ + free(p); + fts_lfree(head); + (void)closedir(dirp); +#ifdef _win_ + close_dird(dirpd); +#endif + cur->fts_info = FTS_ERR; + SET(FTS_STOP); + errno = ENAMETOOLONG; + return nullptr; + } + p->fts_level = (short)level; + p->fts_parent = sp->fts_cur; + p->fts_pathlen = u_short(len + strlen(dp->d_name)); + +#ifdef FTS_WHITEOUT + if (dp->d_type == DT_WHT) + p->fts_flags |= FTS_ISW; +#endif + +#ifdef _DIRENT_HAVE_D_TYPE + if (dp->d_type == DT_DIR) { + p->fts_type = FTS_D; + } else if (dp->d_type == DT_REG) { + p->fts_type = FTS_F; + } else if (dp->d_type == DT_LNK) { + p->fts_type = FTS_SL; + } +#endif + + // coverity[dead_error_line]: false positive + if (cderrno) { + if (nlinks) { + p->fts_info = FTS_NS; + p->fts_errno = cderrno; + } else { + p->fts_info = FTS_NSOK; + } + p->fts_accpath = cur->fts_accpath; + } else if (nlinks == 0 +#ifdef DT_DIR + || (nostat && + dp->d_type != DT_DIR && dp->d_type != DT_UNKNOWN) +#endif + ) { + p->fts_accpath = + ISSET(FTS_NOCHDIR) ? p->fts_path : p->fts_name; + p->fts_info = FTS_NSOK; + } else { + /* Build a file name for fts_stat to stat. */ + if (ISSET(FTS_NOCHDIR)) { + p->fts_accpath = p->fts_path; + memmove((void*)cp, (void*)p->fts_name, (size_t)p->fts_namelen + 1); + } else { + p->fts_accpath = p->fts_name; + } + /* Stat it. */ + p->fts_info = fts_stat(sp, p, 0); + p->fts_type = yfts_type_from_info(p->fts_info); + + /* Decrement link count if applicable. */ + if (nlinks > 0 && (p->fts_info == FTS_D || + p->fts_info == FTS_DC || p->fts_info == FTS_DOT)) { + --nlinks; + } + } + + /* We walk in directory order so "ls -f" doesn't get upset. */ + p->fts_link = nullptr; + if (head == nullptr) { + head = tail = p; + } else { + tail->fts_link = p; + tail = p; + } + ++nitems; + } + if (dirp) { + (void)closedir(dirp); +#ifdef _win_ + close_dird(dirpd); +#endif + } + + /* + * If realloc() changed the address of the path, adjust the + * addresses for the rest of the tree and the dir list. + */ + if (doadjust) { + fts_padjust(sp); + } + + /* + * If not changing directories, reset the path back to original + * state. + */ + if (ISSET(FTS_NOCHDIR)) { + if (len == sp->fts_pathlen || nitems == 0) { + --cp; + } + *cp = '\0'; + } + + /* + * If descended after called from yfts_children or after called from + * yfts_read and nothing found, get back. At the root level we use + * the saved fd; if one of yfts_open()'s arguments is a relative path + * to an empty directory, we wind up here with no other way back. If + * can't get back, we're done. + */ + if (descend && (type == BCHILD || !nitems) && + (cur->fts_level == FTS_ROOTLEVEL ? FCHDIR(sp, sp->fts_rfd) : fts_safe_changedir(sp, cur->fts_parent, -1, ".."))) { + cur->fts_info = FTS_ERR; + SET(FTS_STOP); + fts_lfree(head); + return nullptr; + } + + /* If didn't find anything, return NULL. */ + if (!nitems) { + if (type == BREAD) { + cur->fts_info = FTS_DP; + } + fts_lfree(head); + return nullptr; + } + + /* Sort the entries. */ + if (sp->fts_compar && nitems > 1) { + head = fts_sort(sp, head, nitems); + } + return (head); +} + +static u_short +fts_stat(FTS* sp, FTSENT* p, int follow) +{ + dev_t dev; + ino_t ino; + stat_struct *sbp, sb; + int saved_errno; + /* If user needs stat info, stat buffer already allocated. */ + sbp = ISSET(FTS_NOSTAT) ? &sb : p->fts_statp; + +#ifdef FTS_WHITEOUT + /* check for whiteout */ + if (p->fts_flags & FTS_ISW) { + if (sbp != &sb) { + memset(sbp, '\0', sizeof(*sbp)); + sbp->st_mode = S_IFWHT; + } + return (FTS_W); + } +#endif + + /* + * If doing a logical walk, or application requested FTS_FOLLOW, do + * a stat(2). If that fails, check for a non-existent symlink. If + * fail, set the errno from the stat call. + */ + if (ISSET(FTS_LOGICAL) || follow) { + if (STAT_FUNC(p->fts_accpath, sbp)) { + saved_errno = errno; + if (!lstat(p->fts_accpath, sbp)) { + errno = 0; + return (FTS_SLNONE); + } + p->fts_errno = saved_errno; + memset(sbp, 0, sizeof(stat_struct)); + return (FTS_NS); + } + } else if (lstat(p->fts_accpath, sbp)) { + p->fts_errno = errno; + memset(sbp, 0, sizeof(stat_struct)); + return (FTS_NS); + } + + if (S_ISDIR(sbp->st_mode)) { + /* + * Set the device/inode. Used to find cycles and check for + * crossing mount points. Also remember the link count, used + * in fts_build to limit the number of stat calls. It is + * understood that these fields are only referenced if fts_info + * is set to FTS_D. + */ + dev = p->fts_dev = sbp->st_dev; + ino = p->fts_ino = sbp->st_ino; + p->fts_nlink = sbp->st_nlink; + + const char* fts_name_x = p->fts_name; + if (ISDOT(fts_name_x)) { + return (FTS_DOT); + } + + /* + * Cycle detection is done by brute force when the directory + * is first encountered. If the tree gets deep enough or the + * number of symbolic links to directories is high enough, + * something faster might be worthwhile. + */ + + //There is no way to detect symlink or mount cycles on win32 + +#ifndef _win_ + FTSENT* t; + for (t = p->fts_parent; + t->fts_level >= FTS_ROOTLEVEL; t = t->fts_parent) { + if (ino == t->fts_ino && dev == t->fts_dev) { + p->fts_cycle = t; + return (FTS_DC); + } + } +#endif /*_win_*/ + return (FTS_D); + } + if (S_ISLNK(sbp->st_mode)) { + return (FTS_SL); + } + if (S_ISREG(sbp->st_mode)) { + return (FTS_F); + } + return (FTS_DEFAULT); +} + +static FTSENT* +fts_sort(FTS* sp, FTSENT* head, int nitems) +{ + FTSENT **ap, *p; + + /* + * Construct an array of pointers to the structures and call qsort(3). + * Reassemble the array in the order returned by qsort. If unable to + * sort for memory reasons, return the directory entries in their + * current order. Allocate enough space for the current needs plus + * 40 so don't realloc one entry at a time. + */ + if (nitems > sp->fts_nitems) { + struct _ftsent** a; + + sp->fts_nitems = nitems + 40; + if ((a = (struct _ftsent**)realloc(sp->fts_array, + sp->fts_nitems * sizeof(FTSENT*))) == nullptr) { + if (sp->fts_array) { + free(sp->fts_array); + } + sp->fts_array = nullptr; + sp->fts_nitems = 0; + return (head); + } + sp->fts_array = a; + } + for (ap = sp->fts_array, p = head; p; p = p->fts_link) { + *ap++ = p; + } + qsort((void*)sp->fts_array, (size_t)nitems, sizeof(FTSENT*), (int (*)(const void*, const void*))sp->fts_compar); + for (head = *(ap = sp->fts_array); --nitems; ++ap) { + ap[0]->fts_link = ap[1]; + } + ap[0]->fts_link = nullptr; + return (head); +} + +static FTSENT* +fts_alloc(FTS* sp, const char* name, int namelen) +{ + FTSENT* p; + size_t len; + + /* + * The file name is a variable length array and no stat structure is + * necessary if the user has set the nostat bit. Allocate the FTSENT + * structure, the file name and the stat structure in one chunk, but + * be careful that the stat structure is reasonably aligned. Since the + * fts_name field is declared to be of size 1, the fts_name pointer is + * namelen + 2 before the first possible address of the stat structure. + */ + len = sizeof(FTSENT) + namelen; + if (!ISSET(FTS_NOSTAT)) { + len += sizeof(stat_struct) + ALIGNBYTES; + } + if ((p = (FTSENT*)malloc(len)) == nullptr) { + return nullptr; + } + + /* Copy the name and guarantee NUL termination. */ + memmove((void*)p->fts_name, (void*)name, (size_t)namelen); + p->fts_name[namelen] = '\0'; + + if (!ISSET(FTS_NOSTAT)) { + p->fts_statp = (stat_struct*)ALIGN(p->fts_name + namelen + 2); + } + p->fts_namelen = (u_short)namelen; + p->fts_path = sp->fts_path; + p->fts_errno = 0; + p->fts_flags = 0; + p->fts_instr = FTS_NOINSTR; + p->fts_number = 0; + p->fts_pointer = nullptr; + p->fts_type = FTS_NSOK; + return (p); +} + +static void +fts_lfree(FTSENT* head) +{ + FTSENT* p; + + /* Free a linked list of structures. */ + while ((p = head) != nullptr) { + head = head->fts_link; + free(p); + } +} + +/* + * Allow essentially unlimited paths; find, rm, ls should all work on any tree. + * Most systems will allow creation of paths much longer than MAXPATHLEN, even + * though the kernel won't resolve them. Add the size (not just what's needed) + * plus 256 bytes so don't realloc the path 2 bytes at a time. + */ +static int +fts_palloc(FTS* sp, size_t more) +{ + sp->fts_pathlen += more + 256; + sp->fts_path = (char*)yreallocf(sp->fts_path, (size_t)sp->fts_pathlen); + return (sp->fts_path == nullptr); +} + +static void +ADJUST(FTSENT* p, void* addr) +{ + if ((p)->fts_accpath >= (p)->fts_path && + (p)->fts_accpath < (p)->fts_path + (p)->fts_pathlen) { + if (p->fts_accpath != p->fts_path) { + errx(1, "fts ADJUST: accpath %p path %p", + p->fts_accpath, p->fts_path); + } + if (p->fts_level != 0) { + errx(1, "fts ADJUST: level %d not 0", p->fts_level); + } + (p)->fts_accpath = + (char*)addr + ((p)->fts_accpath - (p)->fts_path); + } + (p)->fts_path = (char*)addr; +} + +/* + * When the path is realloc'd, have to fix all of the pointers in structures + * already returned. + */ +static void +fts_padjust(FTS* sp) +{ + FTSENT* p; + char* addr = sp->fts_path; + +#define ADJUST1(p) \ + { \ + if ((p)->fts_accpath == (p)->fts_path) \ + (p)->fts_accpath = (addr); \ + (p)->fts_path = addr; \ + } + /* Adjust the current set of children. */ + for (p = sp->fts_child; p; p = p->fts_link) { + ADJUST(p, addr); + } + + /* Adjust the rest of the tree. */ + for (p = sp->fts_cur; p->fts_level >= FTS_ROOTLEVEL;) { + ADJUST(p, addr); + p = p->fts_link ? p->fts_link : p->fts_parent; + } +} + +static size_t +fts_maxarglen(char* const* argv) +{ + size_t len, max; + + for (max = 0; *argv; ++argv) { + if ((len = strlen(*argv)) > max) { + max = len; + } + } + return (max + 1); +} + +/* + * Change to dir specified by fd or p->fts_accpath without getting + * tricked by someone changing the world out from underneath us. + * Assumes p->fts_dev and p->fts_ino are filled in. + */ + +#ifndef _win_ +static int +fts_safe_changedir(FTS* sp, FTSENT* p, int fd, const char* path) +{ + int ret, oerrno, newfd; + stat_struct sb; + + newfd = fd; + if (ISSET(FTS_NOCHDIR)) { + return (0); + } + if (fd < 0 && (newfd = open(path, O_RDONLY, 0)) < 0) { + return (-1); + } + if (fstat(newfd, &sb)) { + ret = -1; + goto bail; + } + if (p->fts_dev != sb.st_dev || p->fts_ino != sb.st_ino) { + errno = ENOENT; /* disinformation */ + ret = -1; + goto bail; + } + ret = fchdir(newfd); +bail: + oerrno = errno; + if (fd < 0) { + (void)close(newfd); + } + errno = oerrno; + return (ret); +} +#else +static int +fts_safe_changedir(FTS* sp, FTSENT* p, int /*fd*/, const char* path) +{ + int ret; + stat_struct sb; + + if (ISSET(FTS_NOCHDIR)) + return (0); + if (STAT_FUNC(path, &sb)) { + ret = -1; + goto bail; + } + if (p->fts_dev != sb.st_dev) { + errno = ENOENT; /* disinformation */ + ret = -1; + goto bail; + } + ret = chdir_dird(path); +bail: + return (ret); +} + +static int +fts_safe_changedir(FTS* sp, FTSENT* p, int /*fd*/, dird path) { + int ret; + stat_struct sb; + + if (ISSET(FTS_NOCHDIR)) + return (0); + if (STAT_FUNC(path, &sb)) { + ret = -1; + goto bail; + } + if (p->fts_dev != sb.st_dev) { + errno = ENOENT; /* disinformation */ + ret = -1; + goto bail; + } + ret = chdir_dird(path); +bail: + return (ret); +} +#endif diff --git a/util/folder/fts.h b/util/folder/fts.h new file mode 100644 index 0000000000..f3c799e8c8 --- /dev/null +++ b/util/folder/fts.h @@ -0,0 +1,108 @@ +#pragma once + +#include <sys/types.h> + +#include <util/system/defaults.h> + +#ifndef _win32_ +typedef int dird; +typedef struct stat stat_struct; + #define STAT_FUNC stat +#else + #include <util/folder/dirent_win.h> +typedef WCHAR* dird; +typedef unsigned short u_short; +typedef unsigned int nlink_t; +typedef struct _stat64 stat_struct; + #define STAT_FUNC stat64UTF + //TODO: remove from global scope stat64UTF stat64UTF + #ifdef __cplusplus +int stat64UTF(const char* path, struct _stat64* _Stat); +int stat64UTF(dird path, struct _stat64* _Stat); + #endif +#endif + +typedef struct { + struct _ftsent* fts_cur; /* current node */ + struct _ftsent* fts_child; /* linked list of children */ + struct _ftsent** fts_array; /* sort array */ + dev_t fts_dev; /* starting device # */ + char* fts_path; /* path for this descent */ + dird fts_rfd; /* fd for root */ + int fts_pathlen; /* sizeof(path) */ + int fts_nitems; /* elements in the sort array */ + int(*fts_compar) /* compare function */ + (const struct _ftsent**, const struct _ftsent**); + +#define FTS_COMFOLLOW 0x001 /* follow command line symlinks */ +#define FTS_LOGICAL 0x002 /* logical walk */ +#define FTS_NOCHDIR 0x004 /* don't change directories */ +#define FTS_NOSTAT 0x008 /* don't get stat info */ +#define FTS_PHYSICAL 0x010 /* physical walk */ +#define FTS_SEEDOT 0x020 /* return dot and dot-dot */ +#define FTS_XDEV 0x040 /* don't cross devices */ +#define FTS_OPTIONMASK 0x0ff /* valid user option mask */ + +#define FTS_NAMEONLY 0x100 /* (private) child names only */ +#define FTS_STOP 0x200 /* (private) unrecoverable error */ + int fts_options; /* yfts_open options, global flags */ +} FTS; + +typedef struct _ftsent { + struct _ftsent* fts_cycle; /* cycle node */ + struct _ftsent* fts_parent; /* parent directory */ + struct _ftsent* fts_link; /* next file in directory */ + long fts_number; /* local numeric value */ + void* fts_pointer; /* local address value */ + char* fts_accpath; /* access path */ + char* fts_path; /* root path */ + int fts_errno; /* errno for this node */ + dird fts_symfd; /* fd for symlink */ + u_short fts_pathlen; /* strlen(fts_path) */ + u_short fts_namelen; /* strlen(fts_name) */ + + ino_t fts_ino; /* inode */ + dev_t fts_dev; /* device */ + nlink_t fts_nlink; /* link count */ + +#define FTS_ROOTPARENTLEVEL -1 +#define FTS_ROOTLEVEL 0 + short fts_level; /* depth (-1 to N) */ + +#define FTS_D 1 /* preorder directory */ +#define FTS_DC 2 /* directory that causes cycles */ +#define FTS_DEFAULT 3 /* none of the above */ +#define FTS_DNR 4 /* unreadable directory */ +#define FTS_DOT 5 /* dot or dot-dot */ +#define FTS_DP 6 /* postorder directory */ +#define FTS_ERR 7 /* error; errno is set */ +#define FTS_F 8 /* regular file */ +#define FTS_INIT 9 /* initialized only */ +#define FTS_NS 10 /* stat(2) failed */ +#define FTS_NSOK 11 /* no stat(2) requested */ +#define FTS_SL 12 /* symbolic link */ +#define FTS_SLNONE 13 /* symbolic link without target */ +#define FTS_W 14 /* whiteout object */ + u_short fts_info; /* user flags for FTSENT structure */ + u_short fts_type; /* type of fs node; one of FTS_D, FTS_F, FTS_SL */ + +#define FTS_DONTCHDIR 0x01 /* don't chdir .. to the parent */ +#define FTS_SYMFOLLOW 0x02 /* followed a symlink to get here */ +#define FTS_ISW 0x04 /* this is a whiteout object */ + u_short fts_flags; /* private flags for FTSENT structure */ + +#define FTS_AGAIN 1 /* read node again */ +#define FTS_FOLLOW 2 /* follow symbolic link */ +#define FTS_NOINSTR 3 /* no instructions */ +#define FTS_SKIP 4 /* discard node */ + u_short fts_instr; /* yfts_set() instructions */ + + stat_struct* fts_statp; /* stat(2) information */ + char fts_name[1]; /* file name */ +} FTSENT; + +FTSENT* yfts_children(FTS*, int); +int yfts_close(FTS*); +FTS* yfts_open(char* const*, int, int (*)(const FTSENT**, const FTSENT**)); +FTSENT* yfts_read(FTS*); +int yfts_set(FTS*, FTSENT*, int); diff --git a/util/folder/fts_ut.cpp b/util/folder/fts_ut.cpp new file mode 100644 index 0000000000..c5d59e35f4 --- /dev/null +++ b/util/folder/fts_ut.cpp @@ -0,0 +1,123 @@ +#include "fts.h" +#include "dirut.h" +#include "tempdir.h" + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/threading/future/async.h> + +#include <util/system/file.h> +#include <util/system/tempfile.h> +#include <util/generic/string.h> + +class TFtsTest: public TTestBase { + UNIT_TEST_SUITE(TFtsTest); + UNIT_TEST(TestSimple); + UNIT_TEST(TestNoLeakChangingAccessToFolder); + UNIT_TEST_SUITE_END(); + +public: + void TestSimple(); + void TestNoLeakChangingAccessToFolder(); +}; + +void MakeFile(const char* path) { + TFile(path, CreateAlways); +} + +//There potentially could be problems in listing order on different platforms +int FtsCmp(const FTSENT** ent1, const FTSENT** ent2) { + return strcmp((*ent1)->fts_accpath, (*ent2)->fts_accpath); +} + +void CheckEnt(FTSENT* ent, const char* name, int type) { + UNIT_ASSERT(ent); + UNIT_ASSERT_STRINGS_EQUAL(ent->fts_path, name); + UNIT_ASSERT_EQUAL(ent->fts_info, type); +} + +class TFileTree { +public: + TFileTree(char* const* argv, int options, int (*compar)(const FTSENT**, const FTSENT**)) { + Fts_ = yfts_open(argv, options, compar); + } + + ~TFileTree() { + yfts_close(Fts_); + } + + FTS* operator()() { + return Fts_; + } + +private: + FTS* Fts_; +}; + +void TFtsTest::TestSimple() { + const char* dotPath[2] = {"." LOCSLASH_S, nullptr}; + TFileTree currentDirTree((char* const*)dotPath, 0, FtsCmp); + UNIT_ASSERT(currentDirTree()); + TTempDir tempDir = MakeTempName(yfts_read(currentDirTree())->fts_path); + MakeDirIfNotExist(tempDir().data()); + MakeDirIfNotExist((tempDir() + LOCSLASH_S "dir1").data()); + MakeFile((tempDir() + LOCSLASH_S "dir1" LOCSLASH_S "file1").data()); + MakeFile((tempDir() + LOCSLASH_S "dir1" LOCSLASH_S "file2").data()); + MakeDirIfNotExist((tempDir() + LOCSLASH_S "dir2").data()); + MakeFile((tempDir() + LOCSLASH_S "dir2" LOCSLASH_S "file3").data()); + MakeFile((tempDir() + LOCSLASH_S "dir2" LOCSLASH_S "file4").data()); + + const char* path[2] = {tempDir().data(), nullptr}; + TFileTree fileTree((char* const*)path, 0, FtsCmp); + UNIT_ASSERT(fileTree()); + CheckEnt(yfts_read(fileTree()), tempDir().data(), FTS_D); + CheckEnt(yfts_read(fileTree()), (tempDir() + LOCSLASH_S "dir1").data(), FTS_D); + CheckEnt(yfts_read(fileTree()), (tempDir() + LOCSLASH_S "dir1" LOCSLASH_S "file1").data(), FTS_F); + CheckEnt(yfts_read(fileTree()), (tempDir() + LOCSLASH_S "dir1" LOCSLASH_S "file2").data(), FTS_F); + CheckEnt(yfts_read(fileTree()), (tempDir() + LOCSLASH_S "dir1").data(), FTS_DP); + CheckEnt(yfts_read(fileTree()), (tempDir() + LOCSLASH_S "dir2").data(), FTS_D); + CheckEnt(yfts_read(fileTree()), (tempDir() + LOCSLASH_S "dir2" LOCSLASH_S "file3").data(), FTS_F); + CheckEnt(yfts_read(fileTree()), (tempDir() + LOCSLASH_S "dir2" LOCSLASH_S "file4").data(), FTS_F); + CheckEnt(yfts_read(fileTree()), (tempDir() + LOCSLASH_S "dir2").data(), FTS_DP); + CheckEnt(yfts_read(fileTree()), (tempDir()).data(), FTS_DP); + UNIT_ASSERT_EQUAL(yfts_read(fileTree()), nullptr); +} + +class TTempDirWithLostAccess: public TTempDir { +public: + ~TTempDirWithLostAccess() { + chmod(Name().data(), 0777); + } +}; + +// https://st.yandex-team.ru/YQ-318 +// Test that detects memory leak in case of error in chdir in fts_build function. +void TFtsTest::TestNoLeakChangingAccessToFolder() { + TTempDirWithLostAccess tempDir; + TString tmpPath = tempDir(); + if (tmpPath.EndsWith(LOCSLASH_S)) { + tmpPath.resize(tmpPath.size() - 1); + } + MakeDirIfNotExist((tmpPath + LOCSLASH_S + "subdir").data()); + + const char* path[2] = {tmpPath.data(), nullptr}; + TFileTree fileTree((char* const*)path, FTS_SEEDOT, FtsCmp); + UNIT_ASSERT(fileTree()); + + CheckEnt(yfts_read(fileTree()), tmpPath.data(), FTS_D); +#ifndef _win32_ + CheckEnt(yfts_read(fileTree()), (tmpPath + LOCSLASH_S ".").data(), FTS_DOT); +#endif // _win32_ + CheckEnt(yfts_read(fileTree()), (tmpPath + LOCSLASH_S "..").data(), FTS_DOT); + CheckEnt(yfts_read(fileTree()), (tmpPath + LOCSLASH_S "subdir").data(), FTS_D); + auto pool = CreateThreadPool(2); + auto chmodFuture = NThreading::Async([name = tmpPath] { + UNIT_ASSERT_C(!chmod(name.data(), 0), "Errno: " << errno); + }, *pool); + auto childrenFuture = NThreading::Async([&] { + yfts_children(fileTree(), 0); + }, *pool); + childrenFuture.Wait(); + chmodFuture.Wait(); +} + +UNIT_TEST_SUITE_REGISTRATION(TFtsTest); diff --git a/util/folder/fwd.cpp b/util/folder/fwd.cpp new file mode 100644 index 0000000000..4214b6df83 --- /dev/null +++ b/util/folder/fwd.cpp @@ -0,0 +1 @@ +#include "fwd.h" diff --git a/util/folder/fwd.h b/util/folder/fwd.h new file mode 100644 index 0000000000..8b1869a2d9 --- /dev/null +++ b/util/folder/fwd.h @@ -0,0 +1,5 @@ +#pragma once + +class TFsPath; + +class TTempDir; diff --git a/util/folder/iterator.cpp b/util/folder/iterator.cpp new file mode 100644 index 0000000000..73703d31f9 --- /dev/null +++ b/util/folder/iterator.cpp @@ -0,0 +1,11 @@ +#include "iterator.h" + +#include <cstring> + +static int SortFTSENTByName(const FTSENT** a, const FTSENT** b) { + return strcmp((*a)->fts_name, (*b)->fts_name); +} + +TDirIterator::TOptions& TDirIterator::TOptions::SetSortByName() noexcept { + return SetSortFunctor(SortFTSENTByName); +} diff --git a/util/folder/iterator.h b/util/folder/iterator.h new file mode 100644 index 0000000000..69e025b9c4 --- /dev/null +++ b/util/folder/iterator.h @@ -0,0 +1,109 @@ +#pragma once + +#include "fts.h" + +#include <util/system/error.h> +#include <util/generic/ptr.h> +#include <util/generic/iterator.h> +#include <util/generic/yexception.h> + +/// Note this magic API traverses directory hierarchy + +class TDirIterator: public TInputRangeAdaptor<TDirIterator> { + struct TFtsDestroy { + static inline void Destroy(FTS* f) noexcept { + yfts_close(f); + } + }; + +public: + class TError: public TSystemError { + public: + inline TError(int err) + : TSystemError(err) + { + } + }; + + using TCompare = int (*)(const FTSENT**, const FTSENT**); + + struct TOptions { + inline TOptions() { + Init(FTS_PHYSICAL); + } + + inline TOptions(int opts) { + Init(opts); + } + + inline TOptions& SetMaxLevel(size_t level) noexcept { + MaxLevel = level; + + return *this; + } + + inline TOptions& SetSortFunctor(TCompare cmp) noexcept { + Cmp = cmp; + + return *this; + } + + TOptions& SetSortByName() noexcept; + + int FtsOptions; + size_t MaxLevel; + TCompare Cmp; + + private: + inline void Init(int opts) noexcept { + FtsOptions = opts | FTS_NOCHDIR; + MaxLevel = Max<size_t>(); + Cmp = nullptr; + } + }; + + inline TDirIterator(const TString& path, const TOptions& options = TOptions()) + : Options_(options) + , Path_(path) + { + Trees_[0] = Path_.begin(); + Trees_[1] = nullptr; + + ClearLastSystemError(); + FileTree_.Reset(yfts_open(Trees_, Options_.FtsOptions, Options_.Cmp)); + + const int err = LastSystemError(); + + if (err) { + ythrow TError(err) << "can not open '" << Path_ << "'"; + } + } + + inline FTSENT* Next() { + FTSENT* ret = yfts_read(FileTree_.Get()); + + if (ret) { + if ((size_t)(ret->fts_level + 1) > Options_.MaxLevel) { + yfts_set(FileTree_.Get(), ret, FTS_SKIP); + } + } else { + const int err = LastSystemError(); + + if (err) { + ythrow TError(err) << "error while iterating " << Path_; + } + } + + return ret; + } + + inline void Skip(FTSENT* ent) { + yfts_set(FileTree_.Get(), ent, FTS_SKIP); + } + +private: + TOptions Options_; + TString Path_; + char* Trees_[2]; + THolder<FTS, TFtsDestroy> FileTree_; +}; diff --git a/util/folder/iterator_ut.cpp b/util/folder/iterator_ut.cpp new file mode 100644 index 0000000000..936becd139 --- /dev/null +++ b/util/folder/iterator_ut.cpp @@ -0,0 +1,227 @@ +#include "dirut.h" +#include "iterator.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/system/fs.h> +#include <util/system/file.h> +#include <util/generic/hash.h> +#include <util/generic/algorithm.h> +#include <util/random/mersenne.h> + +static TString JoinWithNewline(const TVector<TString>& strings) { + TStringStream ss; + for (const auto& string : strings) { + ss << string << "\n"; + } + return ss.Str(); +} + +class TDirIteratorTest: public TTestBase { + UNIT_TEST_SUITE(TDirIteratorTest); + UNIT_TEST(TestIt) + UNIT_TEST(TestError) + UNIT_TEST(TestLocal) + UNIT_TEST(TestSkip) + UNIT_TEST(TestSort) + UNIT_TEST_SUITE_END(); + +private: + class TDirHier { + public: + struct TPath { + TString Path; + int Type; + }; + + inline void AddFile(const TString& path) { + Add(path, 0); + } + + inline void AddDir(const TString& path) { + Add(path, 1); + } + + inline void Add(const TString& path, int type) { + const TPath p = { + path, type}; + + Add(p); + } + + inline void Add(const TPath& path) { + switch (path.Type) { + case 0: + TFile(path.Path, CreateAlways | RdWr); + break; + + case 1: + MakeDirIfNotExist(path.Path.data()); + break; + + case 2: + ythrow yexception() << "unknown path type"; + } + + Paths_.push_back(path); + Srch_[path.Path] = path; + } + + inline int Type(const TString& path) { + THashMap<TString, TPath>::const_iterator it = Srch_.find(path); + + UNIT_ASSERT(it != Srch_.end()); + + return it->second.Type; + } + + inline bool Have(const TString& path, int type) { + return Type(path) == type; + } + + inline ~TDirHier() { + for (size_t i = 0; i < Paths_.size(); ++i) { + NFs::Remove(Paths_[Paths_.size() - i - 1].Path); + } + } + + private: + TVector<TPath> Paths_; + THashMap<TString, TPath> Srch_; + }; + + inline void TestLocal() { + TString dirname("." LOCSLASH_S); + TDirIterator d(dirname, FTS_NOCHDIR); + for (auto it = d.begin(); it != d.end(); ++it) { + } + } + + inline void TestIt() { + TDirHier hier; + + const TString dir = "tmpdir"; + const TDirHier::TPath path = {dir, 1}; + + hier.Add(path); + + for (size_t i = 0; i < 10; ++i) { + const TString dir1 = dir + LOCSLASH_C + ToString(i); + const TDirHier::TPath path1 = {dir1, 1}; + + hier.Add(path1); + + for (size_t j = 0; j < 10; ++j) { + const TString subdir2 = ToString(j); + const TString dir2 = dir1 + LOCSLASH_C + subdir2; + const TDirHier::TPath path2 = {dir2, 1}; + + hier.Add(path2); + + for (size_t k = 0; k < 3; ++k) { + const TString file = dir2 + LOCSLASH_C + "file" + ToString(k); + const TDirHier::TPath fpath = {file, 0}; + + hier.Add(fpath); + } + } + } + + TDirIterator d(dir); + + for (auto it = d.begin(); it != d.end(); ++it) { + UNIT_ASSERT(hier.Have(it->fts_path, it->fts_info != FTS_F)); + } + } + + inline void TestSkip() { + TDirHier hier; + + const TString dir = "tmpdir"; + const TDirHier::TPath path = {dir, 1}; + + hier.Add(path); + hier.AddDir(dir + LOCSLASH_C + "dir1"); + hier.AddDir(dir + LOCSLASH_C + "dir1" + LOCSLASH_C + "dir2"); + // + // Without skip + // + { + TDirIterator di(dir); + + UNIT_ASSERT(di.Next()); + UNIT_ASSERT_EQUAL(TStringBuf(di.Next()->fts_name), "dir1"); + UNIT_ASSERT_EQUAL(TStringBuf(di.Next()->fts_name), "dir2"); + UNIT_ASSERT_EQUAL(TStringBuf(di.Next()->fts_name), "dir2"); + UNIT_ASSERT_EQUAL(TStringBuf(di.Next()->fts_name), "dir1"); + UNIT_ASSERT(di.Next()); + UNIT_ASSERT_EQUAL(di.Next(), nullptr); + } + // + // With skip + // + { + TDirIterator di(dir); + + UNIT_ASSERT(di.Next()); + auto ent = di.Next(); + UNIT_ASSERT_EQUAL(TStringBuf(ent->fts_name), "dir1"); + di.Skip(ent); + UNIT_ASSERT_EQUAL(TStringBuf(di.Next()->fts_name), "dir1"); + UNIT_ASSERT(di.Next()); + UNIT_ASSERT_EQUAL(di.Next(), nullptr); + } + } + + inline void TestSort() { + TDirHier dh; + const TString dir("tmpdir"); + + //prepare fs + { + TMersenne<ui32> rnd; + const TString prefixes[] = { + "a", "b", "xxx", "111", ""}; + + dh.AddDir(dir); + + for (size_t i = 0; i < 100; ++i) { + const TString fname = dir + LOCSLASH_C + prefixes[i % Y_ARRAY_SIZE(prefixes)] + ToString(rnd.GenRand()); + + dh.AddFile(fname); + } + } + + TVector<TString> fnames; + + { + TDirIterator d(dir, TDirIterator::TOptions().SetSortByName()); + + for (auto it = d.begin(); it != d.end(); ++it) { + if (it->fts_info == FTS_F) { + fnames.push_back(it->fts_name); + } + } + } + + TVector<TString> sorted(fnames); + Sort(sorted.begin(), sorted.end()); + + UNIT_ASSERT_VALUES_EQUAL(JoinWithNewline(fnames), JoinWithNewline(sorted)); + } + + inline void TestError() { + try { + TDirIterator d("./notexistingfilename"); + + UNIT_ASSERT(false); + } catch (const TDirIterator::TError&) { + } catch (...) { + UNIT_ASSERT(false); + } + + UNIT_ASSERT(true); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TDirIteratorTest); diff --git a/util/folder/lstat_win.c b/util/folder/lstat_win.c new file mode 100644 index 0000000000..cf94cec01a --- /dev/null +++ b/util/folder/lstat_win.c @@ -0,0 +1,35 @@ +#include <util/system/defaults.h> + +#ifdef _win_ + #include <util/system/winint.h> + #include "lstat_win.h" + +int lstat(const char* fileName, stat_struct* fileStat) { + int len = strlen(fileName); + int convRes = MultiByteToWideChar(CP_UTF8, 0, fileName, len, 0, 0); + if (convRes == 0) { + return -1; + } + WCHAR* buf = malloc(sizeof(WCHAR) * (convRes + 1)); + MultiByteToWideChar(CP_UTF8, 0, fileName, len, buf, convRes); + buf[convRes] = 0; + + HANDLE findHandle; + WIN32_FIND_DATAW findBuf; + int result; + result = _wstat64(buf, fileStat); + if (result == 0) { + SetLastError(0); + findHandle = FindFirstFileW(buf, &findBuf); + if (findBuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT && + (findBuf.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT || findBuf.dwReserved0 == IO_REPARSE_TAG_SYMLINK)) + { + fileStat->st_mode = fileStat->st_mode & ~_S_IFMT | _S_IFLNK; + } + FindClose(findHandle); + } + free(buf); + return result; +} + +#endif //_win_ diff --git a/util/folder/lstat_win.h b/util/folder/lstat_win.h new file mode 100644 index 0000000000..0bf7c19055 --- /dev/null +++ b/util/folder/lstat_win.h @@ -0,0 +1,20 @@ +#pragma once + +#include <util/system/defaults.h> +#include "fts.h" + +#ifdef _win_ + #include <sys/stat.h> + + #ifdef __cplusplus +extern "C" { + #endif + + #define _S_IFLNK 0xA000 + int lstat(const char* fileName, stat_struct* fileStat); + + #ifdef __cplusplus +} + #endif + +#endif //_win_ diff --git a/util/folder/path.cpp b/util/folder/path.cpp new file mode 100644 index 0000000000..bfe0c67d68 --- /dev/null +++ b/util/folder/path.cpp @@ -0,0 +1,487 @@ +#include "dirut.h" +#include "path.h" +#include "pathsplit.h" + +#include <util/generic/yexception.h> +#include <util/string/cast.h> +#include <util/system/compiler.h> +#include <util/system/file.h> +#include <util/system/fs.h> + +struct TFsPath::TSplit: public TAtomicRefCount<TSplit>, public TPathSplit { + inline TSplit(const TStringBuf path) + : TPathSplit(path) + { + } +}; + +void TFsPath::CheckDefined() const { + if (!IsDefined()) { + ythrow TIoException() << TStringBuf("must be defined"); + } +} + +bool TFsPath::IsSubpathOf(const TFsPath& that) const { + const TSplit& split = GetSplit(); + const TSplit& rsplit = that.GetSplit(); + + if (rsplit.IsAbsolute != split.IsAbsolute) { + return false; + } + + if (rsplit.Drive != split.Drive) { + return false; + } + + if (rsplit.size() >= split.size()) { + return false; + } + + return std::equal(rsplit.begin(), rsplit.end(), split.begin()); +} + +bool TFsPath::IsNonStrictSubpathOf(const TFsPath& that) const { + const TSplit& split = GetSplit(); + const TSplit& rsplit = that.GetSplit(); + + if (rsplit.IsAbsolute != split.IsAbsolute) { + return false; + } + + if (rsplit.Drive != split.Drive) { + return false; + } + + if (rsplit.size() > split.size()) { + return false; + } + + return std::equal(rsplit.begin(), rsplit.end(), split.begin()); +} + +TFsPath TFsPath::RelativeTo(const TFsPath& root) const { + TSplit split = GetSplit(); + const TSplit& rsplit = root.GetSplit(); + + if (split.Reconstruct() == rsplit.Reconstruct()) { + return TFsPath(); + } + + if (!this->IsSubpathOf(root)) { + ythrow TIoException() << "path " << *this << " is not subpath of " << root; + } + + split.erase(split.begin(), split.begin() + rsplit.size()); + split.IsAbsolute = false; + + return TFsPath(split.Reconstruct()); +} + +TFsPath TFsPath::RelativePath(const TFsPath& root) const { + TSplit split = GetSplit(); + const TSplit& rsplit = root.GetSplit(); + size_t cnt = 0; + + while (split.size() > cnt && rsplit.size() > cnt && split[cnt] == rsplit[cnt]) { + ++cnt; + } + bool absboth = split.IsAbsolute && rsplit.IsAbsolute; + if (cnt == 0 && !absboth) { + ythrow TIoException() << "No common parts in " << *this << " and " << root; + } + TString r; + for (size_t i = 0; i < rsplit.size() - cnt; i++) { + r += i == 0 ? ".." : "/.."; + } + for (size_t i = cnt; i < split.size(); i++) { + r += (i == 0 || i == cnt && rsplit.size() - cnt == 0 ? "" : "/"); + r += split[i]; + } + return r.size() ? TFsPath(r) : TFsPath(); +} + +TFsPath TFsPath::Parent() const { + if (!IsDefined()) { + return TFsPath(); + } + + TSplit split = GetSplit(); + if (split.size()) { + split.pop_back(); + } + if (!split.size() && !split.IsAbsolute) { + return TFsPath("."); + } + return TFsPath(split.Reconstruct()); +} + +TFsPath& TFsPath::operator/=(const TFsPath& that) { + if (!IsDefined()) { + *this = that; + + } else if (that.IsDefined() && that.GetPath() != ".") { + if (!that.IsRelative()) { + ythrow TIoException() << "path should be relative: " << that.GetPath(); + } + + TSplit split = GetSplit(); + const TSplit& rsplit = that.GetSplit(); + split.insert(split.end(), rsplit.begin(), rsplit.end()); + *this = TFsPath(split.Reconstruct()); + } + return *this; +} + +TFsPath& TFsPath::Fix() { + // just normalize via reconstruction + TFsPath(GetSplit().Reconstruct()).Swap(*this); + + return *this; +} + +TString TFsPath::GetName() const { + if (!IsDefined()) { + return TString(); + } + + const TSplit& split = GetSplit(); + + if (split.size() > 0) { + if (split.back() != "..") { + return TString(split.back()); + } else { + // cannot just drop last component, because path itself may be a symlink + return RealPath().GetName(); + } + } else { + if (split.IsAbsolute) { + return split.Reconstruct(); + } else { + return Cwd().GetName(); + } + } +} + +TString TFsPath::GetExtension() const { + return TString(GetSplit().Extension()); +} + +bool TFsPath::IsAbsolute() const { + return GetSplit().IsAbsolute; +} + +bool TFsPath::IsRelative() const { + return !IsAbsolute(); +} + +void TFsPath::InitSplit() const { + Split_ = new TSplit(Path_); +} + +TFsPath::TSplit& TFsPath::GetSplit() const { + // XXX: race condition here + if (!Split_) { + InitSplit(); + } + return *Split_; +} + +static Y_FORCE_INLINE void VerifyPath(const TStringBuf path) { + Y_VERIFY(!path.Contains('\0'), "wrong format of TFsPath"); +} + +TFsPath::TFsPath() { +} + +TFsPath::TFsPath(const TString& path) + : Path_(path) +{ + VerifyPath(Path_); +} + +TFsPath::TFsPath(const TStringBuf path) + : Path_(ToString(path)) +{ + VerifyPath(Path_); +} + +TFsPath::TFsPath(const char* path) + : Path_(path) +{ +} + +TFsPath TFsPath::Child(const TString& name) const { + if (!name) { + ythrow TIoException() << "child name must not be empty"; + } + + return *this / name; +} + +struct TClosedir { + static void Destroy(DIR* dir) { + if (dir) { + if (0 != closedir(dir)) { + ythrow TIoSystemError() << "failed to closedir"; + } + } + } +}; + +void TFsPath::ListNames(TVector<TString>& children) const { + CheckDefined(); + THolder<DIR, TClosedir> dir(opendir(this->c_str())); + if (!dir) { + ythrow TIoSystemError() << "failed to opendir " << Path_; + } + + for (;;) { + struct dirent de; + struct dirent* ok; + // TODO(yazevnul|IGNIETFERRO-1070): remove these macroses by replacing `readdir_r` with proper + // alternative + Y_PRAGMA_DIAGNOSTIC_PUSH + Y_PRAGMA_NO_DEPRECATED + int r = readdir_r(dir.Get(), &de, &ok); + Y_PRAGMA_DIAGNOSTIC_POP + if (r != 0) { + ythrow TIoSystemError() << "failed to readdir " << Path_; + } + if (ok == nullptr) { + return; + } + TString name(de.d_name); + if (name == "." || name == "..") { + continue; + } + children.push_back(name); + } +} + +bool TFsPath::Contains(const TString& component) const { + if (!IsDefined()) { + return false; + } + + TFsPath path = *this; + while (path.Parent() != path) { + if (path.GetName() == component) { + return true; + } + + path = path.Parent(); + } + + return false; +} + +void TFsPath::List(TVector<TFsPath>& files) const { + TVector<TString> names; + ListNames(names); + for (auto& name : names) { + files.push_back(Child(name)); + } +} + +void TFsPath::RenameTo(const TString& newPath) const { + CheckDefined(); + if (!newPath) { + ythrow TIoException() << "bad new file name"; + } + if (!NFs::Rename(Path_, newPath)) { + ythrow TIoSystemError() << "failed to rename " << Path_ << " to " << newPath; + } +} + +void TFsPath::RenameTo(const char* newPath) const { + RenameTo(TString(newPath)); +} + +void TFsPath::RenameTo(const TFsPath& newPath) const { + RenameTo(newPath.GetPath()); +} + +void TFsPath::Touch() const { + CheckDefined(); + if (!TFile(*this, OpenAlways).IsOpen()) { + ythrow TIoException() << "failed to touch " << *this; + } +} + +// XXX: move implementation to util/somewhere. +TFsPath TFsPath::RealPath() const { + CheckDefined(); + return ::RealPath(*this); +} + +TFsPath TFsPath::RealLocation() const { + CheckDefined(); + return ::RealLocation(*this); +} + +TFsPath TFsPath::ReadLink() const { + CheckDefined(); + + if (!IsSymlink()) { + ythrow TIoException() << "not a symlink " << *this; + } + + return NFs::ReadLink(*this); +} + +bool TFsPath::Exists() const { + return IsDefined() && NFs::Exists(*this); +} + +void TFsPath::CheckExists() const { + if (!Exists()) { + ythrow TIoException() << "path does not exist " << Path_; + } +} + +bool TFsPath::IsDirectory() const { + return IsDefined() && TFileStat(GetPath().data()).IsDir(); +} + +bool TFsPath::IsFile() const { + return IsDefined() && TFileStat(GetPath().data()).IsFile(); +} + +bool TFsPath::IsSymlink() const { + return IsDefined() && TFileStat(GetPath().data(), true).IsSymlink(); +} + +void TFsPath::DeleteIfExists() const { + if (!IsDefined()) { + return; + } + + ::unlink(this->c_str()); + ::rmdir(this->c_str()); + if (Exists()) { + ythrow TIoException() << "failed to delete " << Path_; + } +} + +void TFsPath::MkDir(const int mode) const { + CheckDefined(); + if (!Exists()) { + int r = Mkdir(this->c_str(), mode); + if (r != 0) { + // TODO (stanly) will still fail on Windows because + // LastSystemError() returns windows specific ERROR_ALREADY_EXISTS + // instead of EEXIST. + if (LastSystemError() != EEXIST) { + ythrow TIoSystemError() << "could not create directory " << Path_; + } + } + } +} + +void TFsPath::MkDirs(const int mode) const { + CheckDefined(); + if (!Exists()) { + Parent().MkDirs(mode); + MkDir(mode); + } +} + +void TFsPath::ForceDelete() const { + if (!IsDefined()) { + return; + } + + TFileStat stat(GetPath().c_str(), true); + if (stat.IsNull()) { + const int err = LastSystemError(); +#ifdef _win_ + if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) { +#else + if (err == ENOENT) { +#endif + return; + } else { + ythrow TIoException() << "failed to stat " << Path_; + } + } + + ClearLastSystemError(); + if (stat.IsDir()) { + TVector<TFsPath> children; + List(children); + for (auto& i : children) { + i.ForceDelete(); + } + ::rmdir(this->c_str()); + } else { + ::unlink(this->c_str()); + } + + if (LastSystemError()) { + ythrow TIoException() << "failed to delete " << Path_; + } +} + +void TFsPath::CopyTo(const TString& newPath, bool force) const { + if (IsDirectory()) { + if (force) { + TFsPath(newPath).MkDirs(); + } else if (!TFsPath(newPath).IsDirectory()) { + ythrow TIoException() << "Target path is not a directory " << newPath; + } + TVector<TFsPath> children; + List(children); + for (auto&& i : children) { + i.CopyTo(newPath + "/" + i.GetName(), force); + } + } else { + if (force) { + TFsPath(newPath).Parent().MkDirs(); + } else { + if (!TFsPath(newPath).Parent().IsDirectory()) { + ythrow TIoException() << "Parent (" << TFsPath(newPath).Parent() << ") of a target path is not a directory " << newPath; + } + if (TFsPath(newPath).Exists()) { + ythrow TIoException() << "Path already exists " << newPath; + } + } + NFs::Copy(Path_, newPath); + } +} + +void TFsPath::ForceRenameTo(const TString& newPath) const { + try { + RenameTo(newPath); + } catch (const TIoSystemError& /* error */) { + CopyTo(newPath, true); + ForceDelete(); + } +} + +TFsPath TFsPath::Cwd() { + return TFsPath(::NFs::CurrentWorkingDirectory()); +} + +const TPathSplit& TFsPath::PathSplit() const { + return GetSplit(); +} + +template <> +void Out<TFsPath>(IOutputStream& os, const TFsPath& f) { + os << f.GetPath(); +} + +template <> +TFsPath FromStringImpl<TFsPath>(const char* s, size_t len) { + return TFsPath{TStringBuf{s, len}}; +} + +template <> +bool TryFromStringImpl(const char* s, size_t len, TFsPath& result) { + try { + result = TStringBuf{s, len}; + return true; + } catch (std::exception&) { + return false; + } +} diff --git a/util/folder/path.h b/util/folder/path.h new file mode 100644 index 0000000000..2fb4d6b4ef --- /dev/null +++ b/util/folder/path.h @@ -0,0 +1,231 @@ +#pragma once + +#include "fwd.h" +#include "pathsplit.h" + +#include <util/generic/ptr.h> +#include <util/generic/strbuf.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> +#include <util/string/cast.h> +#include <util/system/fstat.h> +#include <util/system/platform.h> +#include <util/system/sysstat.h> +#include <util/system/yassert.h> + +#include <utility> + +/** + * Class behaviour is platform-dependent. + * It uses platform-dependent separators for path-reconstructing operations. + */ +class TFsPath { +private: + struct TSplit; + +public: + TFsPath(); + TFsPath(const TString& path); + TFsPath(const TStringBuf path); + TFsPath(const char* path); + + TFsPath(const std::string& path) + : TFsPath(TStringBuf(path)) + { + } + + void CheckDefined() const; + + inline bool IsDefined() const { + return Path_.length() > 0; + } + + inline explicit operator bool() const { + return IsDefined(); + } + + inline const char* c_str() const { + return Path_.c_str(); + } + + inline operator const TString&() const { + return Path_; + } + + inline bool operator==(const TFsPath& that) const { + return Path_ == that.Path_; + } + + inline bool operator!=(const TFsPath& that) const { + return Path_ != that.Path_; + } + + TFsPath& operator/=(const TFsPath& that); + + friend TFsPath operator/(const TFsPath& s, const TFsPath& p) { + TFsPath ret(s); + return ret /= p; + } + + const TPathSplit& PathSplit() const; + + TFsPath& Fix(); + + inline const TString& GetPath() const { + return Path_; + } + + /// last component of path, or "/" if root + TString GetName() const; + + /** + * "a.b.tmp" -> "tmp" + * "a.tmp" -> "tmp" + * ".tmp" -> "" + */ + TString GetExtension() const; + + bool IsAbsolute() const; + bool IsRelative() const; + + /** + * TFsPath("/a/b").IsSubpathOf("/a") -> true + * + * TFsPath("/a").IsSubpathOf("/a") -> false + * + * TFsPath("/a").IsSubpathOf("/other/path") -> false + * @param that - presumable parent path of this + * @return True if this is a subpath of that and false otherwise. + */ + bool IsSubpathOf(const TFsPath& that) const; + + /** + * TFsPath("/a/b").IsNonStrictSubpathOf("/a") -> true + * + * TFsPath("/a").IsNonStrictSubpathOf("/a") -> true + * + * TFsPath("/a").IsNonStrictSubpathOf("/other/path") -> false + * @param that - presumable parent path of this + * @return True if this is a subpath of that or they are equivalent and false otherwise. + */ + bool IsNonStrictSubpathOf(const TFsPath& that) const; + + bool IsContainerOf(const TFsPath& that) const { + return that.IsSubpathOf(*this); + } + + TFsPath RelativeTo(const TFsPath& root) const; //must be subpath of root + + /** + * @returns relative path or empty path if root equals to this. + */ + TFsPath RelativePath(const TFsPath& root) const; //..; for relative paths 1st component must be the same + + /** + * Never fails. Returns this if already a root. + */ + TFsPath Parent() const; + + TString Basename() const { + return GetName(); + } + TString Dirname() const { + return Parent(); + } + + TFsPath Child(const TString& name) const; + + /** + * @brief create this directory + * + * @param mode specifies permissions to use as described in mkdir(2), makes sense only on Unix-like systems. + * + * Nothing to do if dir exists. + */ + void MkDir(const int mode = MODE0777) const; + + /** + * @brief create this directory and all parent directories as needed + * + * @param mode specifies permissions to use as described in mkdir(2), makes sense only on Unix-like systems. + */ + void MkDirs(const int mode = MODE0777) const; + + // XXX: rewrite to return iterator + void List(TVector<TFsPath>& children) const; + void ListNames(TVector<TString>& children) const; + + // Check, if path contains at least one component with a specific name. + bool Contains(const TString& component) const; + + // fails to delete non-empty directory + void DeleteIfExists() const; + // delete recursively. Does nothing if not exists + void ForceDelete() const; + + // XXX: ino + + inline bool Stat(TFileStat& stat) const { + stat = TFileStat(Path_.data()); + + return stat.Mode; + } + + bool Exists() const; + /// false if not exists + bool IsDirectory() const; + /// false if not exists + bool IsFile() const; + /// false if not exists + bool IsSymlink() const; + /// throw TIoException if not exists + void CheckExists() const; + + void RenameTo(const TString& newPath) const; + void RenameTo(const char* newPath) const; + void RenameTo(const TFsPath& newFile) const; + void ForceRenameTo(const TString& newPath) const; + + void CopyTo(const TString& newPath, bool force) const; + + void Touch() const; + + TFsPath RealPath() const; + TFsPath RealLocation() const; + TFsPath ReadLink() const; + + /// always absolute + static TFsPath Cwd(); + + inline void Swap(TFsPath& p) noexcept { + DoSwap(Path_, p.Path_); + Split_.Swap(p.Split_); + } + +private: + void InitSplit() const; + TSplit& GetSplit() const; + +private: + TString Path_; + /// cache + mutable TSimpleIntrusivePtr<TSplit> Split_; +}; + +namespace NPrivate { + inline void AppendToFsPath(TFsPath&) { + } + + template <class T, class... Ts> + void AppendToFsPath(TFsPath& fsPath, const T& arg, Ts&&... args) { + fsPath /= TFsPath(arg); + AppendToFsPath(fsPath, std::forward<Ts>(args)...); + } +} + +template <class... Ts> +TString JoinFsPaths(Ts&&... args) { + TFsPath fsPath; + ::NPrivate::AppendToFsPath(fsPath, std::forward<Ts>(args)...); + return fsPath.GetPath(); +} diff --git a/util/folder/path.pxd b/util/folder/path.pxd new file mode 100644 index 0000000000..85af10d746 --- /dev/null +++ b/util/folder/path.pxd @@ -0,0 +1,93 @@ +from util.generic.string cimport TString, TStringBuf +from util.generic.vector cimport TVector + + +# NOTE (danila-eremin) Currently not possible to use `const` and `except +` at the same time, so some function not marked const +cdef extern from "util/folder/path.h" nogil: + cdef cppclass TFsPath: + TFsPath() except + + TFsPath(const TString&) except + + TFsPath(const TStringBuf) except + + TFsPath(const char*) except + + + void CheckDefined() except + + + bint IsDefined() const + bint operator bool() const + + const char* c_str() const + + bint operator==(const TFsPath&) const + bint operator!=(const TFsPath&) const + + # NOTE (danila-eremin) operator `/=` Not supported + # TFsPath& operator/=(const TFsPath&) const + + TFsPath operator/(const TFsPath&, const TFsPath&) except + + + # NOTE (danila-eremin) TPathSplit needed + # const TPathSplit& PathSplit() const + + TFsPath& Fix() except + + + const TString& GetPath() const + TString GetName() const + + TString GetExtension() const + + bint IsAbsolute() const + bint IsRelative() const + + bint IsSubpathOf(const TFsPath&) const + bint IsNonStrictSubpathOf(const TFsPath&) const + bint IsContainerOf(const TFsPath&) const + + TFsPath RelativeTo(const TFsPath&) except + + TFsPath RelativePath(const TFsPath&) except + + + TFsPath Parent() const + + TString Basename() const + TString Dirname() const + + TFsPath Child(const TString&) except + + + void MkDir() except + + void MkDir(const int) except + + void MkDirs() except + + void MkDirs(const int) except + + + void List(TVector[TFsPath]&) except + + void ListNames(TVector[TString]&) except + + + bint Contains(const TString&) const + + void DeleteIfExists() except + + void ForceDelete() except + + + # NOTE (danila-eremin) TFileStat needed + # bint Stat(TFileStat&) const + + bint Exists() const + bint IsDirectory() const + bint IsFile() const + bint IsSymlink() const + void CheckExists() except + + + void RenameTo(const TString&) except + + void RenameTo(const char*) except + + void RenameTo(const TFsPath&) except + + void ForceRenameTo(const TString&) except + + + void CopyTo(const TString&, bint) except + + + void Touch() except + + + TFsPath RealPath() except + + TFsPath RealLocation() except + + TFsPath ReadLink() except + + + @staticmethod + TFsPath Cwd() except + + + void Swap(TFsPath&) diff --git a/util/folder/path_ut.cpp b/util/folder/path_ut.cpp new file mode 100644 index 0000000000..e6a3451016 --- /dev/null +++ b/util/folder/path_ut.cpp @@ -0,0 +1,812 @@ +#include "path.h" +#include "pathsplit.h" +#include "dirut.h" +#include "tempdir.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/scope.h> +#include <util/system/platform.h> +#include <util/system/yassert.h> +#include <util/stream/output.h> +#include <util/stream/file.h> +#include <util/system/fs.h> + +#include <algorithm> + +#ifdef _win_ + #include <aclapi.h> +#endif + +namespace { + /// empty directory for test that needs filesystem + /// recreates directory in constructor and removes directory in destructor + class TTestDirectory { + private: + TFsPath Path_; + + public: + TTestDirectory(const TString& name); + ~TTestDirectory(); + + TFsPath GetFsPath() const { + return Path_; + } + + TFsPath Child(const TString& name) const { + return Path_.Child(name); + } + }; + + TTestDirectory::TTestDirectory(const TString& name) { + Y_VERIFY(name.length() > 0, "have to specify name"); + Y_VERIFY(name.find('.') == TString::npos, "must be simple name"); + Y_VERIFY(name.find('/') == TString::npos, "must be simple name"); + Y_VERIFY(name.find('\\') == TString::npos, "must be simple name"); + Path_ = TFsPath(name); + + Path_.ForceDelete(); + Path_.MkDir(); + } + + TTestDirectory::~TTestDirectory() { + Path_.ForceDelete(); + } +} + +Y_UNIT_TEST_SUITE(TFsPathTests) { + Y_UNIT_TEST(TestMkDirs) { + const TFsPath path = "a/b/c/d/e/f"; + path.ForceDelete(); + TFsPath current = path; + ui32 checksCounter = 0; + while (current != ".") { + UNIT_ASSERT(!path.Exists()); + ++checksCounter; + current = current.Parent(); + } + UNIT_ASSERT_VALUES_EQUAL(checksCounter, 6); + + path.MkDirs(); + UNIT_ASSERT(path.Exists()); + + current = path; + while (current != ".") { + UNIT_ASSERT(path.Exists()); + current = current.Parent(); + } + } + + Y_UNIT_TEST(MkDirFreak) { + TFsPath path; + UNIT_ASSERT_EXCEPTION(path.MkDir(), TIoException); + UNIT_ASSERT_EXCEPTION(path.MkDirs(), TIoException); + path = "."; + path.MkDir(); + path.MkDirs(); + } + + Y_UNIT_TEST(Parent) { +#ifdef _win_ + UNIT_ASSERT_VALUES_EQUAL(TFsPath("\\etc/passwd").Parent(), TFsPath("\\etc")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("\\etc").Parent(), TFsPath("\\")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("\\").Parent(), TFsPath("\\")); + + UNIT_ASSERT_VALUES_EQUAL(TFsPath("etc\\passwd").Parent(), TFsPath("etc")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("etc").Parent(), TFsPath(".")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath(".\\etc").Parent(), TFsPath(".")); + + UNIT_ASSERT_VALUES_EQUAL(TFsPath("C:\\etc/passwd").Parent(), TFsPath("C:\\etc")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("C:\\etc").Parent(), TFsPath("C:\\")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("C:\\").Parent(), TFsPath("C:\\")); +#else + UNIT_ASSERT_VALUES_EQUAL(TFsPath("/etc/passwd").Parent(), TFsPath("/etc")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("/etc").Parent(), TFsPath("/")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("/").Parent(), TFsPath("/")); + + UNIT_ASSERT_VALUES_EQUAL(TFsPath("etc/passwd").Parent(), TFsPath("etc")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("etc").Parent(), TFsPath(".")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("./etc").Parent(), TFsPath(".")); +#endif + +#if 0 + UNIT_ASSERT_VALUES_EQUAL(TFsPath("./etc/passwd").Parent(), TFsPath("./etc")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("./").Parent(), TFsPath("..")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath(".").Parent(), TFsPath("..")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("..").Parent(), TFsPath("../..")); +#endif + } + + Y_UNIT_TEST(GetName) { + TTestDirectory d("GetName"); + UNIT_ASSERT_VALUES_EQUAL(TString("dfgh"), d.Child("dfgh").GetName()); + + // check does not fail + TFsPath(".").GetName(); + +#ifdef _unix_ + UNIT_ASSERT_VALUES_EQUAL(TString("/"), TFsPath("/").GetName()); +#endif + } + + Y_UNIT_TEST(GetExtension) { + TTestDirectory d("GetExtension"); + UNIT_ASSERT_VALUES_EQUAL("", d.Child("a").GetExtension()); + UNIT_ASSERT_VALUES_EQUAL("", d.Child(".a").GetExtension()); + UNIT_ASSERT_VALUES_EQUAL("", d.Child("zlib").GetExtension()); + UNIT_ASSERT_VALUES_EQUAL("zlib", d.Child("file.zlib").GetExtension()); + UNIT_ASSERT_VALUES_EQUAL("zlib", d.Child("file.ylib.zlib").GetExtension()); + } + + Y_UNIT_TEST(TestRename) { + TTestDirectory xx("TestRename"); + TFsPath f1 = xx.Child("f1"); + TFsPath f2 = xx.Child("f2"); + f1.Touch(); + f1.RenameTo(f2); + UNIT_ASSERT(!f1.Exists()); + UNIT_ASSERT(f2.Exists()); + } + + Y_UNIT_TEST(TestForceRename) { + TTestDirectory xx("TestForceRename"); + TFsPath fMain = xx.Child("main"); + + TFsPath f1 = fMain.Child("f1"); + f1.MkDirs(); + TFsPath f1Child = f1.Child("f1child"); + f1Child.Touch(); + + TFsPath f2 = fMain.Child("f2"); + f2.MkDirs(); + + fMain.ForceRenameTo("TestForceRename/main1"); + + UNIT_ASSERT(!xx.Child("main").Exists()); + UNIT_ASSERT(xx.Child("main1").Child("f1").Exists()); + UNIT_ASSERT(xx.Child("main1").Child("f2").Exists()); + UNIT_ASSERT(xx.Child("main1").Child("f1").Child("f1child").Exists()); + } + + Y_UNIT_TEST(TestRenameFail) { + UNIT_ASSERT_EXCEPTION(TFsPath("sfsfsfsdfsfsdfdf").RenameTo("sdfsdf"), TIoException); + } + +#ifndef _win_ + Y_UNIT_TEST(TestRealPath) { + UNIT_ASSERT(TFsPath(".").RealPath().IsDirectory()); + + TTestDirectory td("TestRealPath"); + TFsPath link = td.Child("link"); + TFsPath target1 = td.Child("target1"); + target1.Touch(); + TFsPath target2 = td.Child("target2"); + target2.Touch(); + UNIT_ASSERT(NFs::SymLink(target1.RealPath(), link.GetPath())); + UNIT_ASSERT_VALUES_EQUAL(link.RealPath(), target1.RealPath()); + UNIT_ASSERT(NFs::Remove(link.GetPath())); + UNIT_ASSERT(NFs::SymLink(target2.RealPath(), link.GetPath())); + UNIT_ASSERT_VALUES_EQUAL(link.RealPath(), target2.RealPath()); // must not cache old value + } +#endif + + Y_UNIT_TEST(TestSlashesAndBasename) { + TFsPath p("/db/BASE/primus121-025-1380131338//"); + UNIT_ASSERT_VALUES_EQUAL(p.Basename(), TString("primus121-025-1380131338")); + TFsPath testP = p / "test"; +#ifdef _win_ + UNIT_ASSERT_VALUES_EQUAL(testP.GetPath(), "\\db\\BASE\\primus121-025-1380131338\\test"); +#else + UNIT_ASSERT_VALUES_EQUAL(testP.GetPath(), "/db/BASE/primus121-025-1380131338/test"); +#endif + } + + Y_UNIT_TEST(TestSlashesAndBasenameWin) { + TFsPath p("\\db\\BASE\\primus121-025-1380131338\\\\"); + TFsPath testP = p / "test"; +#ifdef _win_ + UNIT_ASSERT_VALUES_EQUAL(p.Basename(), TString("primus121-025-1380131338")); + UNIT_ASSERT_VALUES_EQUAL(testP.GetPath(), "\\db\\BASE\\primus121-025-1380131338\\test"); +#else + UNIT_ASSERT_VALUES_EQUAL(p.Basename(), TString("\\db\\BASE\\primus121-025-1380131338\\\\")); + UNIT_ASSERT_VALUES_EQUAL(testP.GetPath(), "\\db\\BASE\\primus121-025-1380131338\\\\/test"); +#endif + } + + Y_UNIT_TEST(TestSlashesAndBasenameWinDrive) { + TFsPath p("C:\\db\\BASE\\primus121-025-1380131338\\\\"); + TFsPath testP = p / "test"; +#ifdef _win_ + UNIT_ASSERT_VALUES_EQUAL(p.Basename(), TString("primus121-025-1380131338")); + UNIT_ASSERT_VALUES_EQUAL(testP.GetPath(), "C:\\db\\BASE\\primus121-025-1380131338\\test"); +#else + UNIT_ASSERT_VALUES_EQUAL(p.Basename(), TString("C:\\db\\BASE\\primus121-025-1380131338\\\\")); + UNIT_ASSERT_VALUES_EQUAL(testP.GetPath(), "C:\\db\\BASE\\primus121-025-1380131338\\\\/test"); +#endif + } + + Y_UNIT_TEST(TestList) { + TTestDirectory td("TestList-dir"); + + TFsPath dir = td.GetFsPath(); + dir.Child("a").Touch(); + dir.Child("b").MkDir(); + dir.Child("b").Child("b-1").Touch(); + dir.Child("c").MkDir(); + dir.Child("d").Touch(); + + TVector<TString> children; + dir.ListNames(children); + std::sort(children.begin(), children.end()); + + TVector<TString> expected; + expected.push_back("a"); + expected.push_back("b"); + expected.push_back("c"); + expected.push_back("d"); + + UNIT_ASSERT_VALUES_EQUAL(expected, children); + } + +#ifdef _unix_ + Y_UNIT_TEST(MkDirMode) { + TTestDirectory td("MkDirMode"); + TFsPath subDir = td.Child("subdir"); + const int mode = MODE0775; + subDir.MkDir(mode); + TFileStat stat; + UNIT_ASSERT(subDir.Stat(stat)); + // mkdir(2) places umask(2) on mode argument. + const int mask = Umask(0); + Umask(mask); + UNIT_ASSERT_VALUES_EQUAL(stat.Mode& MODE0777, mode & ~mask); + } +#endif + + Y_UNIT_TEST(Cwd) { + UNIT_ASSERT_VALUES_EQUAL(TFsPath::Cwd().RealPath(), TFsPath(".").RealPath()); + } + + Y_UNIT_TEST(TestSubpathOf) { + UNIT_ASSERT(TFsPath("/a/b/c/d").IsSubpathOf("/a/b")); + + UNIT_ASSERT(TFsPath("/a").IsSubpathOf("/")); + UNIT_ASSERT(!TFsPath("/").IsSubpathOf("/a")); + UNIT_ASSERT(!TFsPath("/a").IsSubpathOf("/a")); + + UNIT_ASSERT(TFsPath("/a/b").IsSubpathOf("/a")); + UNIT_ASSERT(TFsPath("a/b").IsSubpathOf("a")); + UNIT_ASSERT(!TFsPath("/a/b").IsSubpathOf("/b")); + UNIT_ASSERT(!TFsPath("a/b").IsSubpathOf("b")); + + // mixing absolute/relative + UNIT_ASSERT(!TFsPath("a").IsSubpathOf("/")); + UNIT_ASSERT(!TFsPath("a").IsSubpathOf("/a")); + UNIT_ASSERT(!TFsPath("/a").IsSubpathOf("a")); + UNIT_ASSERT(!TFsPath("a/b").IsSubpathOf("/a")); + UNIT_ASSERT(!TFsPath("/a/b").IsSubpathOf("a")); + +#ifdef _win_ + UNIT_ASSERT(TFsPath("x:/a/b").IsSubpathOf("x:/a")); + UNIT_ASSERT(!TFsPath("x:/a/b").IsSubpathOf("y:/a")); + UNIT_ASSERT(!TFsPath("x:/a/b").IsSubpathOf("a")); +#endif + } + + Y_UNIT_TEST(TestNonStrictSubpathOf) { + UNIT_ASSERT(TFsPath("/a/b/c/d").IsNonStrictSubpathOf("/a/b")); + + UNIT_ASSERT(TFsPath("/a").IsNonStrictSubpathOf("/")); + UNIT_ASSERT(!TFsPath("/").IsNonStrictSubpathOf("/a")); + + UNIT_ASSERT(TFsPath("/a/b").IsNonStrictSubpathOf("/a")); + UNIT_ASSERT(TFsPath("a/b").IsNonStrictSubpathOf("a")); + UNIT_ASSERT(!TFsPath("/a/b").IsNonStrictSubpathOf("/b")); + UNIT_ASSERT(!TFsPath("a/b").IsNonStrictSubpathOf("b")); + + // mixing absolute/relative + UNIT_ASSERT(!TFsPath("a").IsNonStrictSubpathOf("/")); + UNIT_ASSERT(!TFsPath("a").IsNonStrictSubpathOf("/a")); + UNIT_ASSERT(!TFsPath("/a").IsNonStrictSubpathOf("a")); + UNIT_ASSERT(!TFsPath("a/b").IsNonStrictSubpathOf("/a")); + UNIT_ASSERT(!TFsPath("/a/b").IsNonStrictSubpathOf("a")); + + // equal paths + UNIT_ASSERT(TFsPath("").IsNonStrictSubpathOf("")); + UNIT_ASSERT(TFsPath("/").IsNonStrictSubpathOf("/")); + UNIT_ASSERT(TFsPath("a").IsNonStrictSubpathOf("a")); + UNIT_ASSERT(TFsPath("/a").IsNonStrictSubpathOf("/a")); + UNIT_ASSERT(TFsPath("/a").IsNonStrictSubpathOf("/a/")); + UNIT_ASSERT(TFsPath("/a/").IsNonStrictSubpathOf("/a")); + UNIT_ASSERT(TFsPath("/a/").IsNonStrictSubpathOf("/a/")); + +#ifdef _win_ + UNIT_ASSERT(TFsPath("x:/a/b").IsNonStrictSubpathOf("x:/a")); + + UNIT_ASSERT(TFsPath("x:/a").IsNonStrictSubpathOf("x:/a")); + UNIT_ASSERT(TFsPath("x:/a/").IsNonStrictSubpathOf("x:/a")); + UNIT_ASSERT(TFsPath("x:/a").IsNonStrictSubpathOf("x:/a/")); + UNIT_ASSERT(TFsPath("x:/a/").IsNonStrictSubpathOf("x:/a/")); + + UNIT_ASSERT(!TFsPath("x:/").IsNonStrictSubpathOf("y:/")); + UNIT_ASSERT(!TFsPath("x:/a/b").IsNonStrictSubpathOf("y:/a")); + UNIT_ASSERT(!TFsPath("x:/a/b").IsNonStrictSubpathOf("a")); +#endif + } + + Y_UNIT_TEST(TestRelativePath) { + UNIT_ASSERT_VALUES_EQUAL(TFsPath("/a/b/c/d").RelativePath(TFsPath("/a/b")), TFsPath("c/d")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("/a/b/c/d").RelativePath(TFsPath("/a/b/e/f")), TFsPath("../../c/d")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("/").RelativePath(TFsPath("/")), TFsPath()); + UNIT_ASSERT_VALUES_EQUAL(TFsPath(".").RelativePath(TFsPath(".")), TFsPath()); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("/a/c").RelativePath(TFsPath("/a/b/../c")), TFsPath()); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("a/.././b").RelativePath(TFsPath("b/c")), TFsPath("..")); + + UNIT_ASSERT_EXCEPTION(TFsPath("a/b/c").RelativePath(TFsPath("d/e")), TIoException); + } + + Y_UNIT_TEST(TestUndefined) { + UNIT_ASSERT_VALUES_EQUAL(TFsPath(), TFsPath("")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath(), TFsPath().Fix()); + + UNIT_ASSERT_VALUES_EQUAL(TFsPath() / TFsPath(), TFsPath()); +#ifdef _win_ + UNIT_ASSERT_VALUES_EQUAL(TFsPath("a\\b"), TFsPath() / TString("a\\b")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("a\\b"), "a\\b" / TFsPath()); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("\\a\\b"), TFsPath() / "\\a\\b"); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("\\a\\b"), "\\a\\b" / TFsPath()); +#else + UNIT_ASSERT_VALUES_EQUAL(TFsPath("a/b"), TFsPath() / TString("a/b")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("a/b"), "a/b" / TFsPath()); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("/a/b"), TFsPath() / "/a/b"); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("/a/b"), "/a/b" / TFsPath()); +#endif + UNIT_ASSERT_VALUES_EQUAL(TFsPath("."), TFsPath() / "."); + UNIT_ASSERT_VALUES_EQUAL(TFsPath("."), "." / TFsPath()); + + UNIT_ASSERT(TFsPath().PathSplit().empty()); + UNIT_ASSERT(!TFsPath().PathSplit().IsAbsolute); + UNIT_ASSERT(TFsPath().IsRelative()); // undefined path is relative + + UNIT_ASSERT_VALUES_EQUAL(TFsPath().GetPath(), ""); + UNIT_ASSERT_VALUES_EQUAL(TFsPath().GetName(), ""); + UNIT_ASSERT_VALUES_EQUAL(TFsPath().GetExtension(), ""); + + UNIT_ASSERT_VALUES_EQUAL(TFsPath().Parent(), TFsPath()); + UNIT_ASSERT_VALUES_EQUAL(TFsPath().Child("a"), TFsPath("a")); + UNIT_ASSERT_VALUES_EQUAL(TFsPath().Basename(), ""); + UNIT_ASSERT_VALUES_EQUAL(TFsPath().Dirname(), ""); + + UNIT_ASSERT(!TFsPath().IsSubpathOf("a/b")); + UNIT_ASSERT(TFsPath().IsContainerOf("a/b")); + UNIT_ASSERT(!TFsPath().IsContainerOf("/a/b")); +#ifdef _win_ + UNIT_ASSERT_VALUES_EQUAL(TFsPath("a\\b").RelativeTo(TFsPath()), TFsPath("a\\b")); +#else + UNIT_ASSERT_VALUES_EQUAL(TFsPath("a/b").RelativeTo(TFsPath()), TFsPath("a/b")); +#endif + + UNIT_ASSERT(!TFsPath().Exists()); + UNIT_ASSERT(!TFsPath().IsFile()); + UNIT_ASSERT(!TFsPath().IsDirectory()); + TFileStat stat; + UNIT_ASSERT(!TFsPath().Stat(stat)); + } + + Y_UNIT_TEST(TestJoinFsPaths) { +#ifdef _win_ + UNIT_ASSERT_VALUES_EQUAL(JoinFsPaths("a\\b", "c\\d"), "a\\b\\c\\d"); + UNIT_ASSERT_VALUES_EQUAL(JoinFsPaths("a\\b", "..\\c"), "a\\b\\..\\c"); + UNIT_ASSERT_VALUES_EQUAL(JoinFsPaths("a\\b\\..\\c", "d"), "a\\c\\d"); + UNIT_ASSERT_VALUES_EQUAL(JoinFsPaths("a", "b", "c", "d"), "a\\b\\c\\d"); + UNIT_ASSERT_VALUES_EQUAL(JoinFsPaths("a\\b\\..\\c"), "a\\b\\..\\c"); + UNIT_ASSERT_VALUES_EQUAL(JoinFsPaths("a\\b", ""), "a\\b"); +#else + UNIT_ASSERT_VALUES_EQUAL(JoinFsPaths("a/b", "c/d"), "a/b/c/d"); + UNIT_ASSERT_VALUES_EQUAL(JoinFsPaths("a/b", "../c"), "a/b/../c"); + UNIT_ASSERT_VALUES_EQUAL(JoinFsPaths("a/b/../c", "d"), "a/c/d"); + UNIT_ASSERT_VALUES_EQUAL(JoinFsPaths("a", "b", "c", "d"), "a/b/c/d"); + UNIT_ASSERT_VALUES_EQUAL(JoinFsPaths("a/b/../c"), "a/b/../c"); + UNIT_ASSERT_VALUES_EQUAL(JoinFsPaths("a/b", ""), "a/b"); +#endif + } + + Y_UNIT_TEST(TestStringCast) { + TFsPath pathOne; + UNIT_ASSERT(TryFromString<TFsPath>("/a/b", pathOne)); + UNIT_ASSERT_VALUES_EQUAL(pathOne, TFsPath{"/a/b"}); + + TFsPath pathTwo; + UNIT_ASSERT_NO_EXCEPTION(TryFromString<TFsPath>("/a/b", pathTwo)); + + UNIT_ASSERT_VALUES_EQUAL(FromString<TFsPath>("/a/b"), TFsPath{"/a/b"}); + + TFsPath pathThree{"/a/b"}; + UNIT_ASSERT_VALUES_EQUAL(ToString(pathThree), "/a/b"); + } + +#ifdef _unix_ + Y_UNIT_TEST(TestRemoveSymlinkToDir) { + TTempDir tempDir; + TFsPath tempDirPath(tempDir()); + + const TString originDir = tempDirPath.Child("origin"); + MakePathIfNotExist(originDir.c_str()); + + const TString originFile = TFsPath(originDir).Child("data"); + { + TFixedBufferFileOutput out(originFile); + out << "data111!!!"; + } + + const TString link = tempDirPath.Child("origin_symlink"); + NFs::SymLink(originDir, link); + + TFsPath(link).ForceDelete(); + + UNIT_ASSERT(!NFs::Exists(link)); + UNIT_ASSERT(NFs::Exists(originFile)); + UNIT_ASSERT(NFs::Exists(originDir)); + } + + Y_UNIT_TEST(TestRemoveSymlinkToFile) { + TTempDir tempDir; + TFsPath tempDirPath(tempDir()); + + const TString originDir = tempDirPath.Child("origin"); + MakePathIfNotExist(originDir.c_str()); + + const TString originFile = TFsPath(originDir).Child("data"); + { + TFixedBufferFileOutput out(originFile); + out << "data111!!!"; + } + + const TString link = tempDirPath.Child("origin_symlink"); + NFs::SymLink(originFile, link); + + TFsPath(link).ForceDelete(); + + UNIT_ASSERT(!NFs::Exists(link)); + UNIT_ASSERT(NFs::Exists(originFile)); + UNIT_ASSERT(NFs::Exists(originDir)); + } + + Y_UNIT_TEST(TestRemoveDirWithSymlinkToDir) { + TTempDir tempDir; + TFsPath tempDirPath(tempDir()); + + const TString symlinkedDir = tempDirPath.Child("to_remove"); + MakePathIfNotExist(symlinkedDir.c_str()); + + const TString originDir = tempDirPath.Child("origin"); + MakePathIfNotExist(originDir.c_str()); + + const TString originFile = TFsPath(originDir).Child("data"); + { + TFixedBufferFileOutput out(originFile); + out << "data111!!!"; + } + + const TString symlinkedFile = TFsPath(symlinkedDir).Child("origin_symlink"); + NFs::SymLink(originDir, symlinkedFile); + + TFsPath(symlinkedDir).ForceDelete(); + + UNIT_ASSERT(!NFs::Exists(symlinkedFile)); + UNIT_ASSERT(!NFs::Exists(symlinkedDir)); + UNIT_ASSERT(NFs::Exists(originFile)); + UNIT_ASSERT(NFs::Exists(originDir)); + } + + Y_UNIT_TEST(TestRemoveDirWithSymlinkToFile) { + TTempDir tempDir; + TFsPath tempDirPath(tempDir()); + + const TString symlinkedDir = tempDirPath.Child("to_remove"); + MakePathIfNotExist(symlinkedDir.c_str()); + + const TString originDir = tempDirPath.Child("origin"); + MakePathIfNotExist(originDir.c_str()); + + const TString originFile = TFsPath(originDir).Child("data"); + { + TFixedBufferFileOutput out(originFile); + out << "data111!!!"; + } + + const TString symlinkedFile = TFsPath(symlinkedDir).Child("origin_symlink"); + NFs::SymLink(originFile, symlinkedFile); + + TFsPath(symlinkedDir).ForceDelete(); + + UNIT_ASSERT(!NFs::Exists(symlinkedFile)); + UNIT_ASSERT(!NFs::Exists(symlinkedDir)); + UNIT_ASSERT(NFs::Exists(originFile)); + UNIT_ASSERT(NFs::Exists(originDir)); + } +#endif + + Y_UNIT_TEST(TestForceDeleteNonexisting) { + TTempDir tempDir; + TFsPath nonexisting = TFsPath(tempDir()).Child("nonexisting"); + nonexisting.ForceDelete(); + } + + // Here we want to test that all possible errors during TFsPath::ForceDelete + // are properly handled. To do so we have to trigger fs operation errors in + // three points: + // 1. stat/GetFileInformationByHandle + // 2. opendir + // 3. unlink/rmdir + // + // On unix systems we can achieve this by simply setting access rights on + // entry being deleted and its parent. But on windows it is more complicated. + // Current Chmod implementation on windows is not enough as it sets only + // FILE_ATTRIBUTE_READONLY throught SetFileAttributes call. But doing so does + // not affect directory access rights on older versions of Windows and Wine + // that we use to run autocheck tests. + // + // To get required access rights we use DACL in SetSecurityInfo. This is wrapped + // in RAII class that drops requested permissions on file/dir and grantss them + // back in destructor. + // + // Another obstacle is FILE_LIST_DIRECTORY permission when running on Wine. + // Dropping this permission is necessary to provoke error + // in GetFileInformationByHandle. Wine allows dropping this permission, but I + // have not found a way to grant it back. So tests crash during cleanup sequence. + // To make it possible to run this tests natively we detect Wine with special + // registry key and skip these tests only there. + +#ifdef _win_ + struct TLocalFree { + static void Destroy(void* ptr) { + LocalFree((HLOCAL)ptr); + } + }; + + bool IsWine() { + HKEY subKey = nullptr; + LONG result = RegOpenKeyEx(HKEY_CURRENT_USER, "Software\\Wine", 0, KEY_READ, &subKey); + if (result == ERROR_SUCCESS) { + return true; + } + result = RegOpenKeyEx(HKEY_LOCAL_MACHINE, "Software\\Wine", 0, KEY_READ, &subKey); + if (result == ERROR_SUCCESS) { + return true; + } + + HMODULE hntdll = GetModuleHandle("ntdll.dll"); + if (!hntdll) { + return false; + } + + auto func = GetProcAddress(hntdll, "wine_get_version"); + return func != nullptr; + } + + class TWinFileDenyAccessScope { + public: + TWinFileDenyAccessScope(const TFsPath& name, DWORD permissions) + : Name_(name) + , Perms_(permissions) + { + DWORD res = 0; + PACL oldAcl = nullptr; + PSECURITY_DESCRIPTOR sd = nullptr; + + res = GetNamedSecurityInfoA((LPSTR)name.c_str(), + SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION, + nullptr, + nullptr, + &oldAcl, + nullptr, + &sd); + SdHolder_.Reset(sd); + if (res != ERROR_SUCCESS) { + ythrow TSystemError(res) << "error in GetNamedSecurityInfoA"; + } + + Acl_ = SetAcl(oldAcl, DENY_ACCESS); + } + + ~TWinFileDenyAccessScope() { + try { + const TFsPath parent = Name_.Parent(); + Chmod(parent.c_str(), MODE0777); + Chmod(Name_.c_str(), MODE0777); + SetAcl((PACL)Acl_.Get(), GRANT_ACCESS); + } catch (const yexception& ex) { + Cerr << "~TWinFileDenyAccessScope failed: " << ex.AsStrBuf() << Endl; + } + } + + THolder<void, TLocalFree> SetAcl(PACL oldAcl, ACCESS_MODE accessMode) { + DWORD res = 0; + EXPLICIT_ACCESS ea; + PACL newAcl = nullptr; + THolder<void, TLocalFree> newAclHolder; + + memset(&ea, 0, sizeof(EXPLICIT_ACCESS)); + ea.grfAccessPermissions = Perms_; + ea.grfAccessMode = accessMode; + ea.grfInheritance = NO_INHERITANCE; + ea.Trustee.TrusteeForm = TRUSTEE_IS_NAME; + ea.Trustee.ptstrName = (LPSTR) "CURRENT_USER"; + + res = SetEntriesInAcl(1, &ea, oldAcl, &newAcl); + newAclHolder.Reset(newAcl); + if (res != ERROR_SUCCESS) { + ythrow TSystemError(res) << "error in SetEntriesInAcl"; + } + + res = SetNamedSecurityInfoA((LPSTR)Name_.c_str(), + SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION, + nullptr, + nullptr, + newAcl, + nullptr); + if (res != ERROR_SUCCESS) { + ythrow TSystemError(res) << "error in SetNamedSecurityInfoA"; + } + + return std::move(newAclHolder); + } + + private: + const TFsPath Name_; + const DWORD Perms_; + THolder<void, TLocalFree> SdHolder_; + THolder<void, TLocalFree> Acl_; + }; +#endif + + Y_UNIT_TEST(TestForceDeleteErrorUnlink) { + TTempDir tempDir; + + const TFsPath testDir = TFsPath(tempDir()).Child("dir"); + MakePathIfNotExist(testDir.c_str()); + + const TFsPath testFile = testDir.Child("file"); + { + TFixedBufferFileOutput out(testFile); + out << "data111!!!"; + } + +#ifdef _win_ + Chmod(testFile.c_str(), S_IRUSR); + Y_DEFER { + Chmod(testFile.c_str(), MODE0777); + }; +#else + Chmod(testDir.c_str(), S_IRUSR | S_IXUSR); + Y_DEFER { + Chmod(testDir.c_str(), MODE0777); + }; +#endif + + UNIT_ASSERT_EXCEPTION_CONTAINS(testFile.ForceDelete(), TIoException, "failed to delete"); + } + + Y_UNIT_TEST(TestForceDeleteErrorRmdir) { + TTempDir tempDir; + + const TFsPath testDir = TFsPath(tempDir()).Child("dir"); + const TFsPath testSubdir = testDir.Child("file"); + MakePathIfNotExist(testSubdir.c_str()); + +#ifdef _win_ + Chmod(testSubdir.c_str(), 0); + Y_DEFER { + Chmod(testSubdir.c_str(), MODE0777); + }; + TWinFileDenyAccessScope dirAcl(testDir, FILE_WRITE_DATA); +#else + Chmod(testDir.c_str(), S_IRUSR | S_IXUSR); + Y_DEFER { + Chmod(testDir.c_str(), MODE0777); + }; +#endif + + UNIT_ASSERT_EXCEPTION_CONTAINS(testSubdir.ForceDelete(), TIoException, "failed to delete"); + } + + Y_UNIT_TEST(TestForceDeleteErrorStatDir) { + TTempDir tempDir; + + const TFsPath testDir = TFsPath(tempDir()).Child("dir"); + const TFsPath testSubdir = testDir.Child("file"); + MakePathIfNotExist(testSubdir.c_str()); + +#ifdef _win_ + if (IsWine()) { + // FILE_LIST_DIRECTORY seem to be irreversible on wine + return; + } + TWinFileDenyAccessScope subdirAcl(testSubdir, FILE_READ_ATTRIBUTES); + TWinFileDenyAccessScope dirAcl(testDir, FILE_LIST_DIRECTORY); +#else + Chmod(testDir.c_str(), 0); + Y_DEFER { + Chmod(testDir.c_str(), MODE0777); + }; +#endif + + UNIT_ASSERT_EXCEPTION_CONTAINS(testSubdir.ForceDelete(), TIoException, "failed to stat"); + } + + Y_UNIT_TEST(TestForceDeleteErrorStatFile) { + TTempDir tempDir; + + const TFsPath testDir = TFsPath(tempDir()).Child("dir"); + MakePathIfNotExist(testDir.c_str()); + + const TFsPath testFile = testDir.Child("file"); + { + TFixedBufferFileOutput out(testFile); + out << "data111!!!"; + } + +#ifdef _win_ + if (IsWine()) { + // FILE_LIST_DIRECTORY seem to be irreversible on wine + return; + } + TWinFileDenyAccessScope fileAcl(testFile, FILE_READ_ATTRIBUTES); + TWinFileDenyAccessScope dirAcl(testDir, FILE_LIST_DIRECTORY); +#else + Chmod(testDir.c_str(), 0); + Y_DEFER { + Chmod(testDir.c_str(), MODE0777); + }; +#endif + + UNIT_ASSERT_EXCEPTION_CONTAINS(testFile.ForceDelete(), TIoException, "failed to stat"); + } + + Y_UNIT_TEST(TestForceDeleteErrorListDir) { + TTempDir tempDir; + + const TFsPath testDir = TFsPath(tempDir()).Child("dir"); + const TFsPath testSubdir = testDir.Child("file"); + MakePathIfNotExist(testSubdir.c_str()); + +#ifdef _win_ + if (IsWine()) { + // FILE_LIST_DIRECTORY seem to be irreversible on wine + return; + } + TWinFileDenyAccessScope subdirAcl(testSubdir, FILE_LIST_DIRECTORY); +#else + Chmod(testSubdir.c_str(), 0); + Y_DEFER { + Chmod(testSubdir.c_str(), MODE0777); + }; +#endif + + UNIT_ASSERT_EXCEPTION_CONTAINS(testSubdir.ForceDelete(), TIoException, "failed to opendir"); + } + +#ifdef _unix_ + Y_UNIT_TEST(TestForceDeleteErrorSymlink) { + TTempDir tempDir; + + const TFsPath testDir = TFsPath(tempDir()).Child("dir"); + MakePathIfNotExist(testDir.c_str()); + + const TFsPath testSymlink = testDir.Child("symlink"); + NFs::SymLink("something", testSymlink); + + Chmod(testSymlink.c_str(), S_IRUSR); + Chmod(testDir.c_str(), S_IRUSR | S_IXUSR); + Y_DEFER { + Chmod(testDir.c_str(), MODE0777); + Chmod(testSymlink.c_str(), MODE0777); + }; + + UNIT_ASSERT_EXCEPTION_CONTAINS(testSymlink.ForceDelete(), TIoException, "failed to delete"); + } +#endif +} diff --git a/util/folder/path_ut.pyx b/util/folder/path_ut.pyx new file mode 100644 index 0000000000..e2537683ee --- /dev/null +++ b/util/folder/path_ut.pyx @@ -0,0 +1,379 @@ +# cython: c_string_type=str, c_string_encoding=utf8 + +from util.folder.path cimport TFsPath +from util.generic.string cimport TString, TStringBuf +from util.generic.vector cimport TVector + +import unittest +import yatest.common + +import os.path + + +class TestPath(unittest.TestCase): + def test_ctor1(self): + cdef TFsPath path = TFsPath() + self.assertEqual(path.IsDefined(), False) + self.assertEquals(path.c_str(), "") + + def test_ctor2(self): + cdef TString str_path = "/a/b/c" + cdef TFsPath path = TFsPath(str_path) + self.assertEqual(path.IsDefined(), True) + self.assertEquals(path.c_str(), "/a/b/c") + + def test_ctor3(self): + cdef TStringBuf buf_path = "/a/b/c" + cdef TFsPath path = TFsPath(buf_path) + self.assertEqual(path.IsDefined(), True) + self.assertEquals(path.c_str(), "/a/b/c") + + def test_ctor4(self): + cdef char* char_path = "/a/b/c" + cdef TFsPath path = TFsPath(char_path) + self.assertEqual(path.IsDefined(), True) + self.assertEquals(path.c_str(), "/a/b/c") + + def test_assignment(self): + cdef TFsPath path1 = TFsPath("/a/b") + cdef TFsPath path2 = TFsPath("/a/c") + + self.assertEquals(path1.GetPath(), "/a/b") + self.assertEquals(path2.GetPath(), "/a/c") + + path2 = path1 + + self.assertEquals(path1.GetPath(), "/a/b") + self.assertEquals(path2.GetPath(), "/a/b") + + def test_check_defined(self): + cdef TFsPath path1 = TFsPath() + with self.assertRaises(RuntimeError): + path1.CheckDefined() + self.assertEqual(path1.IsDefined(), False) + if path1: + assert False + else: + pass + + cdef TFsPath path2 = TFsPath("") + with self.assertRaises(RuntimeError): + path2.CheckDefined() + self.assertEqual(path2.IsDefined(), False) + if path2: + assert False + else: + pass + + cdef TFsPath path3 = TFsPath("/") + path3.CheckDefined() + self.assertEqual(path3.IsDefined(), True) + if path3: + pass + else: + assert False + + def test_comparison(self): + cdef TFsPath path1 = TFsPath("/a/b") + cdef TFsPath path2 = TFsPath("/a/c") + cdef TFsPath path3 = TFsPath("/a/b") + + self.assertEqual(path1 == path3, True) + self.assertEqual(path1 != path2, True) + self.assertEqual(path3 != path2, True) + + def test_concatenation(self): + cdef TFsPath path1 = TFsPath("/a") + cdef TFsPath path2 = TFsPath("b") + cdef TFsPath path3 = path1 / path2 + cdef TFsPath path4 = TFsPath("/a/b") + + self.assertEqual(path3 == path4, True) + + def test_fix(self): + cdef TFsPath path = TFsPath("test_fix/b/c/../d") + cdef TFsPath fixed = path.Fix() + self.assertEquals(fixed.GetPath(), "test_fix/b/d") + + def test_parts(self): + cdef TFsPath path = TFsPath("/a/b/c") + self.assertEquals(path.GetPath(), "/a/b/c") + self.assertEquals(path.GetName(), "c") + self.assertEquals(path.GetExtension(), "") + self.assertEquals(path.Basename(), "c") + self.assertEquals(path.Dirname(), "/a/b") + + cdef TFsPath path_ext = TFsPath("/a/b/c.ext") + self.assertEquals(path_ext.GetPath(), "/a/b/c.ext") + self.assertEquals(path_ext.GetName(), "c.ext") + self.assertEquals(path_ext.GetExtension(), "ext") + self.assertEquals(path_ext.Basename(), "c.ext") + self.assertEquals(path_ext.Dirname(), "/a/b") + + cdef TFsPath path_only_ext = TFsPath("/a/b/.ext") + self.assertEquals(path_only_ext.GetPath(), "/a/b/.ext") + self.assertEquals(path_only_ext.GetName(), ".ext") + self.assertEquals(path_only_ext.GetExtension(), "") + self.assertEquals(path_only_ext.Basename(), ".ext") + self.assertEquals(path_only_ext.Dirname(), "/a/b") + + cdef TFsPath path_dir = TFsPath("/a/b/") + self.assertEquals(path_dir.GetPath(), "/a/b/") + self.assertEquals(path_dir.GetName(), "b") + self.assertEquals(path_dir.GetExtension(), "") + self.assertEquals(path_dir.Basename(), "b") + self.assertEquals(path_dir.Dirname(), "/a") + + def test_absolute(self): + cdef TFsPath path_absolute = TFsPath("/a/b/c") + self.assertEquals(path_absolute.IsAbsolute(), True) + self.assertEquals(path_absolute.IsRelative(), False) + + self.assertEquals(path_absolute.IsSubpathOf(TFsPath("/a/b")), True) + self.assertEquals(path_absolute.IsNonStrictSubpathOf(TFsPath("/a/b")), True) + self.assertEquals(TFsPath("/a/b").IsContainerOf(path_absolute), True) + + self.assertEquals(path_absolute.IsSubpathOf(TFsPath("/a/b/c")), False) + self.assertEquals(path_absolute.IsNonStrictSubpathOf(TFsPath("/a/b/c")), True) + self.assertEquals(TFsPath("/a/b/c").IsContainerOf(path_absolute), False) + + self.assertEquals(path_absolute.IsSubpathOf(TFsPath("/a/c")), False) + self.assertEquals(path_absolute.IsNonStrictSubpathOf(TFsPath("/a/c")), False) + self.assertEquals(TFsPath("/a/c").IsContainerOf(path_absolute), False) + + with self.assertRaises(RuntimeError): + path_absolute.RelativeTo(TFsPath("/a/c")) + self.assertEquals(path_absolute.RelativePath(TFsPath("/a/с")).GetPath(), "../b/c") + self.assertEquals(path_absolute.RelativeTo(TFsPath("/a")).GetPath(), "b/c") + self.assertEquals(path_absolute.RelativePath(TFsPath("/a")).GetPath(), "b/c") + self.assertEquals(path_absolute.RelativeTo(TFsPath("/")).GetPath(), "a/b/c") + self.assertEquals(path_absolute.RelativePath(TFsPath("/")).GetPath(), "a/b/c") + + with self.assertRaises(RuntimeError): + path_absolute.RelativeTo(TFsPath("./a")) + with self.assertRaises(RuntimeError): + path_absolute.RelativePath(TFsPath("d")) + self.assertEquals(path_absolute.RelativePath(TFsPath("./a")).GetPath(), "b/c") + + self.assertEquals(path_absolute.Parent().GetPath(), "/a/b") + self.assertEquals(path_absolute.Child("d").GetPath(), "/a/b/c/d") + + def test_relative(self): + cdef TFsPath path_relative_1 = TFsPath("a/b/c") + self.assertEquals(path_relative_1.IsAbsolute(), False) + self.assertEquals(path_relative_1.IsRelative(), True) + + self.assertEquals(path_relative_1.IsSubpathOf(TFsPath("a/b")), True) + self.assertEquals(path_relative_1.IsNonStrictSubpathOf(TFsPath("a/b")), True) + self.assertEquals(TFsPath("a/b").IsContainerOf(path_relative_1), True) + + self.assertEquals(path_relative_1.IsSubpathOf(TFsPath("a/b/c")), False) + self.assertEquals(path_relative_1.IsNonStrictSubpathOf(TFsPath("a/b/c")), True) + self.assertEquals(TFsPath("a/b/c").IsContainerOf(path_relative_1), False) + + self.assertEquals(path_relative_1.IsSubpathOf(TFsPath("a/c")), False) + self.assertEquals(path_relative_1.IsNonStrictSubpathOf(TFsPath("a/c")), False) + self.assertEquals(TFsPath("a/c").IsContainerOf(path_relative_1), False) + + self.assertEquals(path_relative_1.Parent().GetPath(), "a/b") + self.assertEquals(path_relative_1.Child("d").GetPath(), "a/b/c/d") + + cdef TFsPath path_relative_2 = TFsPath("./a/b/c") + self.assertEquals(path_relative_2.IsAbsolute(), False) + self.assertEquals(path_relative_2.IsRelative(), True) + + self.assertEquals(path_relative_2.IsSubpathOf(TFsPath("a/b")), True) + self.assertEquals(path_relative_2.IsNonStrictSubpathOf(TFsPath("a/b")), True) + self.assertEquals(TFsPath("a/b").IsContainerOf(path_relative_2), True) + + self.assertEquals(path_relative_2.IsSubpathOf(TFsPath("a/b/c")), False) + self.assertEquals(path_relative_2.IsNonStrictSubpathOf(TFsPath("a/b/c")), True) + self.assertEquals(TFsPath("a/b/c").IsContainerOf(path_relative_2), False) + + self.assertEquals(path_relative_2.IsSubpathOf(TFsPath("a/c")), False) + self.assertEquals(path_relative_2.IsNonStrictSubpathOf(TFsPath("a/c")), False) + self.assertEquals(TFsPath("a/c").IsContainerOf(path_relative_2), False) + + with self.assertRaises(RuntimeError): + path_relative_2.RelativeTo(TFsPath("a/c")) + self.assertEquals(path_relative_2.RelativePath(TFsPath("a/с")).GetPath(), "../b/c") + self.assertEquals(path_relative_2.RelativeTo(TFsPath("a")).GetPath(), "b/c") + self.assertEquals(path_relative_2.RelativePath(TFsPath("a")).GetPath(), "b/c") + self.assertEquals(path_relative_2.RelativeTo(TFsPath("./")).GetPath(), "a/b/c") + self.assertEquals(path_relative_2.RelativePath(TFsPath("/a")).GetPath(), "b/c") + + with self.assertRaises(RuntimeError): + self.assertEquals(path_relative_2.RelativePath(TFsPath("./")).GetPath(), "a/b/c") + + with self.assertRaises(RuntimeError): + path_relative_2.RelativeTo(TFsPath("/d")) + with self.assertRaises(RuntimeError): + path_relative_2.RelativePath(TFsPath("/d")) + with self.assertRaises(RuntimeError): + path_relative_2.RelativePath(TFsPath("/")) + + self.assertEquals(path_relative_2.Parent().GetPath(), "a/b") + self.assertEquals(path_relative_2.Child("d").GetPath(), "a/b/c/d") + + def test_mkdir(self): + cdef TFsPath directory = TFsPath("test_mkdir") + cdef TFsPath full = directory / directory + cdef TFsPath internal = full / directory + with self.assertRaises(RuntimeError): + full.MkDir() + full.MkDirs() + internal.MkDir() + + def test_list(self): + cdef TFsPath dir = TFsPath("test_list") + dir.MkDir() + TFsPath("test_list/b").Touch() + TFsPath("test_list/c").Touch() + + cdef TVector[TFsPath] files + cdef TVector[TString] names + + dir.List(files) + dir.ListNames(names) + + self.assertEquals(files.size(), 2) + self.assertEquals(sorted([files[0].GetPath(), files[1].GetPath()]), ["test_list/b", "test_list/c"]) + self.assertEquals(names.size(), 2) + self.assertEquals(sorted(list(names)), ["b", "c"]) + + def test_contains(self): + cdef TFsPath path = TFsPath("a/b/c") + self.assertEquals(path.Contains("c"), True) + self.assertEquals(path.Contains("b"), True) + self.assertEquals(path.Contains("d"), False) + + def test_delete(self): + cdef TFsPath root = TFsPath("/") + with self.assertRaises(RuntimeError): + root.DeleteIfExists() + with self.assertRaises(RuntimeError): + root.ForceDelete() + + cdef TFsPath directory = TFsPath("test_delete") + cdef TFsPath full = directory / directory + full.MkDirs() + + self.assertEquals(full.Exists(), True) + with self.assertRaises(RuntimeError): + directory.DeleteIfExists() + self.assertEquals(directory.Exists(), True) + directory.ForceDelete() + self.assertEquals(directory.Exists(), False) + + cdef TFsPath local_file = TFsPath("test_delete_1") + self.assertEquals(local_file.Exists(), False) + local_file.DeleteIfExists() + self.assertEquals(local_file.Exists(), False) + local_file.ForceDelete() + self.assertEquals(local_file.Exists(), False) + + local_file.Touch() + self.assertEquals(local_file.Exists(), True) + local_file.DeleteIfExists() + self.assertEquals(local_file.Exists(), False) + + local_file.Touch() + self.assertEquals(local_file.Exists(), True) + local_file.ForceDelete() + self.assertEquals(local_file.Exists(), False) + + full.MkDirs() + self.assertEquals(full.Exists(), True) + full.DeleteIfExists() + self.assertEquals(full.Exists(), False) + self.assertEquals(directory.Exists(), True) + directory.DeleteIfExists() + self.assertEquals(directory.Exists(), False) + + def test_checks(self): + cdef TFsPath local_file = TFsPath("test_checks") + with self.assertRaises(RuntimeError): + local_file.CheckExists() + local_file.Touch() + self.assertEquals(local_file.Exists(), True) + self.assertEquals(local_file.IsDirectory(), False) + self.assertEquals(local_file.IsFile(), True) + self.assertEquals(local_file.IsSymlink(), False) + local_file.CheckExists() + + local_file.DeleteIfExists() + local_file.MkDir() + self.assertEquals(local_file.Exists(), True) + self.assertEquals(local_file.IsDirectory(), True) + self.assertEquals(local_file.IsFile(), False) + self.assertEquals(local_file.IsSymlink(), False) + local_file.CheckExists() + + def test_rename(self): + cdef TFsPath path = TFsPath("test_rename_a") + path.Touch() + + cdef TString path_str = "test_rename_b" + cdef TFsPath path_from_str = TFsPath(path_str) + self.assertEquals(path.Exists(), True) + self.assertEquals(path_from_str.Exists(), False) + path.RenameTo(path_str) + self.assertEquals(path.Exists(), False) + self.assertEquals(path_from_str.Exists(), True) + + cdef const char* path_char = "test_rename_c" + cdef TFsPath path_from_char = TFsPath(path_char) + self.assertEquals(path_from_str.Exists(), True) + self.assertEquals(path_from_char.Exists(), False) + path_from_str.RenameTo(path_char) + self.assertEquals(path_from_str.Exists(), False) + self.assertEquals(path_from_char.Exists(), True) + + path_from_char.RenameTo(path) + + self.assertEquals(path_from_char.Exists(), False) + self.assertEquals(path.Exists(), True) + + path.ForceRenameTo(path_str) + + self.assertEquals(path_from_str.Exists(), True) + self.assertEquals(path.Exists(), False) + + with self.assertRaises(RuntimeError): + path_from_str.RenameTo("") + + def test_copy(self): + cdef TString dst = "test_copy_dst" + cdef TFsPath src_path = TFsPath("test_copy_src") + cdef TFsPath dst_path = TFsPath(dst) + self.assertEquals(src_path.Exists(), False) + src_path.Touch() + self.assertEquals(src_path.Exists(), True) + src_path.CopyTo(dst, False) + self.assertEquals(src_path.Exists(), True) + self.assertEquals(dst_path.Exists(), True) + + def test_real_path(self): + cdef TFsPath path = TFsPath("test_real_path_a") + path.Touch() + self.assertEquals(path.RealPath().GetPath(), os.path.join(yatest.common.work_path(), "test_real_path_a")) + self.assertEquals(path.RealLocation().GetPath(), os.path.join(yatest.common.work_path(), "test_real_path_a")) + with self.assertRaises(RuntimeError): + path.ReadLink() + + def test_cwd(self): + cdef TFsPath path = TFsPath.Cwd() + self.assertEquals(path.GetPath(), yatest.common.work_path()) + + def test_swap(self): + cdef TFsPath first = TFsPath("first") + cdef TFsPath second = TFsPath("second") + + self.assertEquals(first.GetPath(), "first") + self.assertEquals(second.GetPath(), "second") + first.Swap(second) + self.assertEquals(first.GetPath(), "second") + self.assertEquals(second.GetPath(), "first") + second.Swap(first) + self.assertEquals(first.GetPath(), "first") + self.assertEquals(second.GetPath(), "second") diff --git a/util/folder/pathsplit.cpp b/util/folder/pathsplit.cpp new file mode 100644 index 0000000000..81d439a727 --- /dev/null +++ b/util/folder/pathsplit.cpp @@ -0,0 +1,151 @@ +#include "pathsplit.h" + +#include "dirut.h" + +#include <util/stream/output.h> +#include <util/generic/yexception.h> + +template <class T> +static inline size_t ToReserve(const T& t) { + size_t ret = t.size() + 5; + + for (auto it = t.begin(); it != t.end(); ++it) { + ret += it->size(); + } + + return ret; +} + +void TPathSplitTraitsUnix::DoParseFirstPart(const TStringBuf part) { + if (part == TStringBuf(".")) { + push_back(TStringBuf(".")); + + return; + } + + if (IsAbsolutePath(part)) { + IsAbsolute = true; + } + + DoParsePart(part); +} + +void TPathSplitTraitsUnix::DoParsePart(const TStringBuf part0) { + DoAppendHint(part0.size() / 8); + + TStringBuf next(part0); + TStringBuf part; + + while (TStringBuf(next).TrySplit('/', part, next)) { + AppendComponent(part); + } + + AppendComponent(next); +} + +void TPathSplitTraitsWindows::DoParseFirstPart(const TStringBuf part0) { + TStringBuf part(part0); + + if (part == TStringBuf(".")) { + push_back(TStringBuf(".")); + + return; + } + + if (IsAbsolutePath(part)) { + IsAbsolute = true; + + if (part.size() > 1 && part[1] == ':') { + Drive = part.SubStr(0, 2); + part = part.SubStr(2); + } + } + + DoParsePart(part); +} + +void TPathSplitTraitsWindows::DoParsePart(const TStringBuf part0) { + DoAppendHint(part0.size() / 8); + + size_t pos = 0; + TStringBuf part(part0); + + while (pos < part.size()) { + while (pos < part.size() && this->IsPathSep(part[pos])) { + ++pos; + } + + const char* begin = part.data() + pos; + + while (pos < part.size() && !this->IsPathSep(part[pos])) { + ++pos; + } + + AppendComponent(TStringBuf(begin, part.data() + pos)); + } +} + +TString TPathSplitStore::DoReconstruct(const TStringBuf slash) const { + TString r; + + r.reserve(ToReserve(*this)); + + if (IsAbsolute) { + r.AppendNoAlias(Drive); + r.AppendNoAlias(slash); + } + + for (auto i = begin(); i != end(); ++i) { + if (i != begin()) { + r.AppendNoAlias(slash); + } + + r.AppendNoAlias(*i); + } + + return r; +} + +void TPathSplitStore::AppendComponent(const TStringBuf comp) { + if (!comp || comp == TStringBuf(".")) { + ; // ignore + } else if (comp == TStringBuf("..") && !empty() && back() != TStringBuf("..")) { + pop_back(); + } else { + // push back first .. also + push_back(comp); + } +} + +TStringBuf TPathSplitStore::Extension() const { + return size() > 0 ? CutExtension(back()) : TStringBuf(); +} + +template <> +void Out<TPathSplit>(IOutputStream& o, const TPathSplit& ps) { + o << ps.Reconstruct(); +} + +TString JoinPaths(const TPathSplit& p1, const TPathSplit& p2) { + if (p2.IsAbsolute) { + ythrow yexception() << "can not join " << p1 << " and " << p2; + } + + return TPathSplit(p1).AppendMany(p2.begin(), p2.end()).Reconstruct(); +} + +TStringBuf CutExtension(const TStringBuf fileName) { + if (fileName.empty()) { + return fileName; + } + + TStringBuf name; + TStringBuf extension; + fileName.RSplit('.', name, extension); + if (name.empty()) { + // dot at a start or not found + return name; + } else { + return extension; + } +} diff --git a/util/folder/pathsplit.h b/util/folder/pathsplit.h new file mode 100644 index 0000000000..d134338e35 --- /dev/null +++ b/util/folder/pathsplit.h @@ -0,0 +1,113 @@ +#pragma once + +#include <util/generic/vector.h> +#include <util/generic/strbuf.h> +#include <util/generic/string.h> +#include <util/string/ascii.h> + +//do not own any data +struct TPathSplitStore: public TVector<TStringBuf> { + TStringBuf Drive; + bool IsAbsolute = false; + + void AppendComponent(const TStringBuf comp); + TStringBuf Extension() const; + +protected: + TString DoReconstruct(const TStringBuf slash) const; + + inline void DoAppendHint(size_t hint) { + reserve(size() + hint); + } +}; + +struct TPathSplitTraitsUnix: public TPathSplitStore { + static constexpr char MainPathSep = '/'; + + inline TString Reconstruct() const { + return DoReconstruct(TStringBuf("/")); + } + + static constexpr bool IsPathSep(const char c) noexcept { + return c == '/'; + } + + static inline bool IsAbsolutePath(const TStringBuf path) noexcept { + return path && IsPathSep(path[0]); + } + + void DoParseFirstPart(const TStringBuf part); + void DoParsePart(const TStringBuf part); +}; + +struct TPathSplitTraitsWindows: public TPathSplitStore { + static constexpr char MainPathSep = '\\'; + + inline TString Reconstruct() const { + return DoReconstruct(TStringBuf("\\")); + } + + static constexpr bool IsPathSep(char c) noexcept { + return c == '/' || c == '\\'; + } + + static inline bool IsAbsolutePath(const TStringBuf path) noexcept { + return path && (IsPathSep(path[0]) || (path.size() > 1 && path[1] == ':' && IsAsciiAlpha(path[0]) && (path.size() == 2 || IsPathSep(path[2])))); + } + + void DoParseFirstPart(const TStringBuf part); + void DoParsePart(const TStringBuf part); +}; + +#if defined(_unix_) +using TPathSplitTraitsLocal = TPathSplitTraitsUnix; +#else +using TPathSplitTraitsLocal = TPathSplitTraitsWindows; +#endif + +template <class TTraits> +class TPathSplitBase: public TTraits { +public: + inline TPathSplitBase() = default; + + inline TPathSplitBase(const TStringBuf part) { + this->ParseFirstPart(part); + } + + inline TPathSplitBase& AppendHint(size_t hint) { + this->DoAppendHint(hint); + + return *this; + } + + inline TPathSplitBase& ParseFirstPart(const TStringBuf part) { + this->DoParseFirstPart(part); + + return *this; + } + + inline TPathSplitBase& ParsePart(const TStringBuf part) { + this->DoParsePart(part); + + return *this; + } + + template <class It> + inline TPathSplitBase& AppendMany(It b, It e) { + this->AppendHint(e - b); + + while (b != e) { + this->AppendComponent(*b++); + } + + return *this; + } +}; + +using TPathSplit = TPathSplitBase<TPathSplitTraitsLocal>; +using TPathSplitUnix = TPathSplitBase<TPathSplitTraitsUnix>; +using TPathSplitWindows = TPathSplitBase<TPathSplitTraitsWindows>; + +TString JoinPaths(const TPathSplit& p1, const TPathSplit& p2); + +TStringBuf CutExtension(const TStringBuf fileName); diff --git a/util/folder/pathsplit_ut.cpp b/util/folder/pathsplit_ut.cpp new file mode 100644 index 0000000000..0e97afd0d0 --- /dev/null +++ b/util/folder/pathsplit_ut.cpp @@ -0,0 +1,482 @@ +// File includes itself to make multiple passes of its suites with different platform-dependent definitions + +#ifndef PS_INCLUDED +// Outer part + + #include "pathsplit.h" + + #include <library/cpp/testing/unittest/registar.h> + + #define VAR(NAME) Y_CAT(NAME, __LINE__) + + #define PS_CHECK(input, ...) \ + const char* VAR(model)[] = {"", __VA_ARGS__}; \ + UNIT_ASSERT_EQUAL(input.size(), sizeof(VAR(model)) / sizeof(const char*) - 1); \ + for (size_t n = 0; n < input.size(); ++n) { \ + UNIT_ASSERT_STRINGS_EQUAL(input[n], VAR(model)[n + 1]); \ + } + + #define PS_INCLUDED + + #define PSUF(NAME) NAME + #define PSUF_LOCAL(NAME) NAME##Local + #include __FILE__ + #undef PSUF + #undef PSUF_LOCAL + + #define PSUF(NAME) NAME##Unix + #define PSUF_LOCAL(NAME) PSUF(NAME) + #ifdef _win_ + #undef _win_ + #define REVERT_WIN + #endif + #include __FILE__ + #ifdef REVERT_WIN + #define _win_ + #undef REVERT_WIN + #endif + #undef PSUF + #undef PSUF_LOCAL + + #define PSUF(NAME) NAME##Windows + #define PSUF_LOCAL(NAME) PSUF(NAME) + #ifndef _win_ + #define _win_ + #define REVERT_WIN + #endif + #include __FILE__ + #ifdef REVERT_WIN + #undef _win_ + #undef REVERT_WIN + #endif + #undef PSUF + #undef PSUF_LOCAL + + #undef PS_INCLUDED + +#else +// Inner part + + #ifdef _win_ + #define TRUE_ONLY_WIN true + #else + #define TRUE_ONLY_WIN false + #endif + +Y_UNIT_TEST_SUITE(PSUF(PathSplit)) { + Y_UNIT_TEST(Empty) { + PSUF(TPathSplit) + ps; + PS_CHECK(ps); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + } + + Y_UNIT_TEST(Relative) { + PSUF(TPathSplit) + ps("some/usual/path"); + PS_CHECK(ps, "some", "usual", "path"); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + } + + Y_UNIT_TEST(Absolute) { + PSUF(TPathSplit) + ps("/some/usual/path"); + PS_CHECK(ps, "some", "usual", "path"); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, true); + } + + Y_UNIT_TEST(Self) { + PSUF(TPathSplit) + ps("."); + PS_CHECK(ps, "."); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + } + + Y_UNIT_TEST(Parent) { + PSUF(TPathSplit) + ps(".."); + PS_CHECK(ps, ".."); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + } + + Y_UNIT_TEST(Root) { + PSUF(TPathSplit) + ps("/"); + PS_CHECK(ps); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, true); + } + + Y_UNIT_TEST(Reconstruct) { + PSUF(TPathSplit) + ps("some/usual/path/../../other/././//path"); + #ifdef _win_ + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), "some\\other\\path"); + #else + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), "some/other/path"); + #endif + + ps = PSUF(TPathSplit)("/some/usual/path/../../other/././//path"); + #ifdef _win_ + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), "\\some\\other\\path"); + #else + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), "/some/other/path"); + #endif + } + + Y_UNIT_TEST(ParseFirstPart) { + PSUF(TPathSplit) + ps; + ps.ParseFirstPart("some/usual/path"); + PS_CHECK(ps, "some", "usual", "path"); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + + ps = PSUF(TPathSplit)(); + ps.ParseFirstPart("/some/usual/path"); + PS_CHECK(ps, "some", "usual", "path"); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, true); + } + + Y_UNIT_TEST(ParsePart) { + PSUF(TPathSplit) + ps("some/usual/path"); + ps.ParsePart("sub/path"); + PS_CHECK(ps, "some", "usual", "path", "sub", "path"); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + + ps = PSUF(TPathSplit)("some/usual/path"); + ps.ParsePart("/sub/path"); + PS_CHECK(ps, "some", "usual", "path", "sub", "path"); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + } + + Y_UNIT_TEST(ParsePartSelf) { + PSUF(TPathSplit) + ps("some/usual/path"); + ps.ParsePart("."); + PS_CHECK(ps, "some", "usual", "path"); + + ps = PSUF(TPathSplit)("some/usual/path"); + ps.ParsePart("././."); + PS_CHECK(ps, "some", "usual", "path"); + } + + Y_UNIT_TEST(ParsePartParent) { + PSUF(TPathSplit) + ps("some/usual/path"); + ps.ParsePart(".."); + PS_CHECK(ps, "some", "usual"); + + ps = PSUF(TPathSplit)("some/usual/path"); + ps.ParsePart("../.."); + PS_CHECK(ps, "some"); + + ps = PSUF(TPathSplit)("some/usual/path"); + ps.ParsePart("../../.."); + PS_CHECK(ps); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + + ps = PSUF(TPathSplit)("/some/usual/path"); + ps.ParsePart("../../.."); + PS_CHECK(ps); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, true); + } + + Y_UNIT_TEST(ParsePartOverflow) { + PSUF(TPathSplit) + ps("some/usual/path"); + ps.ParsePart("../../../../.."); + PS_CHECK(ps, "..", ".."); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + + ps = PSUF(TPathSplit)("/some/usual/path"); + ps.ParsePart("../../../../.."); + PS_CHECK(ps, "..", ".."); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, true); + } + + Y_UNIT_TEST(WinRelative) { + PSUF(TPathSplit) + ps("some\\usual\\path"); + #ifdef _win_ + PS_CHECK(ps, "some", "usual", "path"); + #else + PS_CHECK(ps, "some\\usual\\path"); + #endif + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + } + + Y_UNIT_TEST(WinAbsolute) { + PSUF(TPathSplit) + ps("\\some\\usual\\path"); + #ifdef _win_ + PS_CHECK(ps, "some", "usual", "path"); + #else + PS_CHECK(ps, "\\some\\usual\\path"); + #endif + UNIT_ASSERT_EQUAL(ps.IsAbsolute, TRUE_ONLY_WIN); + + PSUF(TPathSplit) + psDrive("C:\\some\\usual\\path"); + #ifdef _win_ + PS_CHECK(psDrive, "some", "usual", "path"); + UNIT_ASSERT_EQUAL(psDrive.Drive, "C:"); + #else + PS_CHECK(psDrive, "C:\\some\\usual\\path"); + #endif + UNIT_ASSERT_EQUAL(psDrive.IsAbsolute, TRUE_ONLY_WIN); + + PSUF(TPathSplit) + psDrive2("C:/some/usual/path"); + #ifdef _win_ + PS_CHECK(psDrive2, "some", "usual", "path"); + UNIT_ASSERT_EQUAL(psDrive2.Drive, "C:"); + #else + PS_CHECK(psDrive2, "C:", "some", "usual", "path"); + #endif + UNIT_ASSERT_EQUAL(psDrive2.IsAbsolute, TRUE_ONLY_WIN); + } + + Y_UNIT_TEST(WinRoot) { + PSUF(TPathSplit) + ps("\\"); + #ifdef _win_ + PS_CHECK(ps); + #else + PS_CHECK(ps, "\\"); + #endif + UNIT_ASSERT_EQUAL(ps.IsAbsolute, TRUE_ONLY_WIN); + + PSUF(TPathSplit) + psDrive("C:"); + #ifdef _win_ + PS_CHECK(psDrive); + UNIT_ASSERT_EQUAL(psDrive.Drive, "C:"); + #else + PS_CHECK(psDrive, "C:"); + #endif + UNIT_ASSERT_EQUAL(psDrive.IsAbsolute, TRUE_ONLY_WIN); + } + + Y_UNIT_TEST(WinReconstruct) { + PSUF(TPathSplit) + ps("some\\usual\\path\\..\\..\\other\\.\\.\\\\\\path"); + #ifdef _win_ + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), "some\\other\\path"); + #else + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), "some\\usual\\path\\..\\..\\other\\.\\.\\\\\\path"); + #endif + + ps = PSUF(TPathSplit)("\\some\\usual\\path\\..\\..\\other\\.\\.\\\\\\path"); + #ifdef _win_ + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), "\\some\\other\\path"); + #else + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), "\\some\\usual\\path\\..\\..\\other\\.\\.\\\\\\path"); + #endif + } + + Y_UNIT_TEST(WinParseFirstPart) { + PSUF(TPathSplit) + ps; + ps.ParseFirstPart("some\\usual\\path"); + #ifdef _win_ + PS_CHECK(ps, "some", "usual", "path"); + #else + PS_CHECK(ps, "some\\usual\\path"); + #endif + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + + ps = PSUF(TPathSplit)(); + ps.ParseFirstPart("\\some\\usual\\path"); + #ifdef _win_ + PS_CHECK(ps, "some", "usual", "path"); + #else + PS_CHECK(ps, "\\some\\usual\\path"); + #endif + UNIT_ASSERT_EQUAL(ps.IsAbsolute, TRUE_ONLY_WIN); + } + + Y_UNIT_TEST(WinParsePart) { + PSUF(TPathSplit) + ps("some\\usual\\path"); + ps.ParsePart("sub\\path"); + #ifdef _win_ + PS_CHECK(ps, "some", "usual", "path", "sub", "path"); + #else + PS_CHECK(ps, "some\\usual\\path", "sub\\path"); + #endif + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + + ps = PSUF(TPathSplit)("some\\usual\\path"); + ps.ParsePart("\\sub\\path"); + #ifdef _win_ + PS_CHECK(ps, "some", "usual", "path", "sub", "path"); + #else + PS_CHECK(ps, "some\\usual\\path", "\\sub\\path"); + #endif + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + } + + #ifdef _win_ + Y_UNIT_TEST(WinParsePartSelf) { + PSUF(TPathSplit) + ps("some\\usual\\path"); + ps.ParsePart("."); + PS_CHECK(ps, "some", "usual", "path"); + + ps = PSUF(TPathSplit)("some\\usual\\path"); + ps.ParsePart(".\\.\\."); + PS_CHECK(ps, "some", "usual", "path"); + } + + Y_UNIT_TEST(WinParsePartParent) { + PSUF(TPathSplit) + ps("some\\usual\\path"); + ps.ParsePart(".."); + PS_CHECK(ps, "some", "usual"); + + ps = PSUF(TPathSplit)("some\\usual\\path"); + ps.ParsePart("..\\.."); + PS_CHECK(ps, "some"); + + ps = PSUF(TPathSplit)("some\\usual\\path"); + ps.ParsePart("..\\..\\.."); + PS_CHECK(ps); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + + ps = PSUF(TPathSplit)("\\some\\usual\\path"); + ps.ParsePart("..\\..\\.."); + PS_CHECK(ps); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, true); + + ps = PSUF(TPathSplit)("C:\\some\\usual\\path"); + ps.ParsePart("..\\..\\.."); + PS_CHECK(ps); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, true); + UNIT_ASSERT_EQUAL(ps.Drive, "C:"); + } + + Y_UNIT_TEST(WinParsePartOverflow) { + PSUF(TPathSplit) + ps("some\\usual\\path"); + ps.ParsePart("..\\..\\..\\..\\.."); + PS_CHECK(ps, "..", ".."); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + + ps = PSUF(TPathSplit)("\\some\\usual\\path"); + ps.ParsePart("..\\..\\..\\..\\.."); + PS_CHECK(ps, "..", ".."); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, true); + + ps = PSUF(TPathSplit)("C:\\some\\usual\\path"); + ps.ParsePart("..\\..\\..\\..\\.."); + PS_CHECK(ps, "..", ".."); + UNIT_ASSERT_EQUAL(ps.IsAbsolute, true); + UNIT_ASSERT_EQUAL(ps.Drive, "C:"); + } + #endif + + Y_UNIT_TEST(WinMixed) { + PSUF(TPathSplit) + ps("some\\usual/path"); + #ifdef _win_ + PS_CHECK(ps, "some", "usual", "path"); + #else + PS_CHECK(ps, "some\\usual", "path"); + #endif + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + } + + Y_UNIT_TEST(WinParsePartMixed) { + PSUF(TPathSplit) + ps("some\\usual/path"); + ps.ParsePart("sub/sub\\path"); + #ifdef _win_ + PS_CHECK(ps, "some", "usual", "path", "sub", "sub", "path"); + #else + PS_CHECK(ps, "some\\usual", "path", "sub", "sub\\path"); + #endif + UNIT_ASSERT_EQUAL(ps.IsAbsolute, false); + } + + Y_UNIT_TEST(BeginWithSelf) { + PSUF(TPathSplit) + ps("./some/usual/path"); + PS_CHECK(ps, "some", "usual", "path"); + #ifdef _win_ + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), "some\\usual\\path"); + #else + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), "some/usual/path"); + #endif + } + + Y_UNIT_TEST(BeginWithParent) { + PSUF(TPathSplit) + ps("../some/usual/path"); + PS_CHECK(ps, "..", "some", "usual", "path"); + #ifdef _win_ + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), "..\\some\\usual\\path"); + #else + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), "../some/usual/path"); + #endif + } + + Y_UNIT_TEST(InOut) { + PSUF(TPathSplit) + ps("path/.."); + PS_CHECK(ps); + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), ""); + } + + Y_UNIT_TEST(OutIn) { + PSUF(TPathSplit) + ps("../path"); + PS_CHECK(ps, "..", "path"); + #ifdef _win_ + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), "..\\path"); + #else + UNIT_ASSERT_STRINGS_EQUAL(ps.Reconstruct(), "../path"); + #endif + } +} + +Y_UNIT_TEST_SUITE(PSUF(PathSplitTraits)) { + Y_UNIT_TEST(IsPathSep) { + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsPathSep('/'), true); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsPathSep('\\'), TRUE_ONLY_WIN); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsPathSep(' '), false); + } + + Y_UNIT_TEST(IsAbsolutePath) { + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath(""), false); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("/"), true); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("some/usual/path"), false); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("/some/usual/path"), true); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("."), false); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath(".."), false); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("/."), true); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("/.."), true); + } + + Y_UNIT_TEST(WinIsAbsolutePath) { + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("somepath"), false); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("\\"), TRUE_ONLY_WIN); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("\\somepath"), TRUE_ONLY_WIN); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("\\."), TRUE_ONLY_WIN); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("\\.."), TRUE_ONLY_WIN); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("C"), false); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("C:"), TRUE_ONLY_WIN); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("C:somepath"), false); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("C:\\"), TRUE_ONLY_WIN); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("C:\\somepath"), TRUE_ONLY_WIN); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("C:/"), TRUE_ONLY_WIN); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("C:/somepath"), TRUE_ONLY_WIN); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("#:"), false); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("#:somepath"), false); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("#:\\somepath"), false); + UNIT_ASSERT_EQUAL(PSUF_LOCAL(TPathSplitTraits)::IsAbsolutePath("#:/somepath"), false); + } +} + + #undef TRUE_ONLY_WIN + +#endif diff --git a/util/folder/tempdir.cpp b/util/folder/tempdir.cpp new file mode 100644 index 0000000000..6fdf8f753c --- /dev/null +++ b/util/folder/tempdir.cpp @@ -0,0 +1,44 @@ +#include "tempdir.h" + +#include "dirut.h" + +#include <util/system/fs.h> +#include <util/system/maxlen.h> + +TTempDir::TTempDir() + : TTempDir(nullptr, TCreationToken{}) +{ +} + +TTempDir::TTempDir(const char* prefix, TCreationToken) + : TempDir() + , Remove(true) +{ + char tempDir[MAX_PATH]; + if (MakeTempDir(tempDir, prefix) != 0) { + ythrow TSystemError() << "Can't create temporary directory"; + } + TempDir = tempDir; +} + +TTempDir::TTempDir(const TString& tempDir) + : TempDir(tempDir) + , Remove(true) +{ + NFs::Remove(TempDir); + MakeDirIfNotExist(TempDir.c_str()); +} + +TTempDir TTempDir::NewTempDir(const TString& root) { + return {root.c_str(), TCreationToken{}}; +} + +void TTempDir::DoNotRemove() { + Remove = false; +} + +TTempDir::~TTempDir() { + if (Remove) { + RemoveDirWithContents(TempDir); + } +} diff --git a/util/folder/tempdir.h b/util/folder/tempdir.h new file mode 100644 index 0000000000..ff458f83b9 --- /dev/null +++ b/util/folder/tempdir.h @@ -0,0 +1,43 @@ +#pragma once + +#include "fwd.h" +#include "path.h" +#include <util/generic/string.h> + +class TTempDir { +public: + /// Create new directory in system tmp folder. + TTempDir(); + + /// Create new directory with this fixed name. If it already exists, clear it. + TTempDir(const TString& tempDir); + + ~TTempDir(); + + /// Create new directory in given folder. + static TTempDir NewTempDir(const TString& root); + + const TString& operator()() const { + return Name(); + } + + const TString& Name() const { + return TempDir.GetPath(); + } + + const TFsPath& Path() const { + return TempDir; + } + + void DoNotRemove(); + +private: + struct TCreationToken {}; + + // Prevent people from confusing this ctor with the public one + // by requiring additional fake argument. + TTempDir(const char* prefix, TCreationToken); + + TFsPath TempDir; + bool Remove; +}; diff --git a/util/folder/ut/ya.make b/util/folder/ut/ya.make new file mode 100644 index 0000000000..64877d9b58 --- /dev/null +++ b/util/folder/ut/ya.make @@ -0,0 +1,17 @@ +UNITTEST_FOR(util) + +OWNER(g:util) +SUBSCRIBER(g:util-subscribers) + +SRCS( + folder/dirut_ut.cpp + folder/filelist_ut.cpp + folder/fts_ut.cpp + folder/iterator_ut.cpp + folder/path_ut.cpp + folder/pathsplit_ut.cpp +) + +INCLUDE(${ARCADIA_ROOT}/util/tests/ya_util_tests.inc) + +END() diff --git a/util/folder/ya.make b/util/folder/ya.make new file mode 100644 index 0000000000..79c9498ddd --- /dev/null +++ b/util/folder/ya.make @@ -0,0 +1,6 @@ +OWNER(g:util) +SUBSCRIBER(g:util-subscribers) + +RECURSE_FOR_TESTS( + ut +) |