diff options
author | iddqd <iddqd@yandex-team.ru> | 2022-02-10 16:49:46 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:49:46 +0300 |
commit | ff241e1daf76d79ed38015cdb76d55eb3a33ac27 (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 /util/system | |
parent | 07fce9c5f7771600d0b3d70e1f88fd8a7e164d85 (diff) | |
download | ydb-ff241e1daf76d79ed38015cdb76d55eb3a33ac27.tar.gz |
Restoring authorship annotation for <iddqd@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'util/system')
-rw-r--r-- | util/system/atexit.cpp | 18 | ||||
-rw-r--r-- | util/system/atexit.h | 4 | ||||
-rw-r--r-- | util/system/direct_io.cpp | 256 | ||||
-rw-r--r-- | util/system/direct_io.h | 96 | ||||
-rw-r--r-- | util/system/direct_io_ut.cpp | 8 | ||||
-rw-r--r-- | util/system/error.cpp | 2 | ||||
-rw-r--r-- | util/system/info.cpp | 16 | ||||
-rw-r--r-- | util/system/info.h | 2 | ||||
-rw-r--r-- | util/system/mlock.cpp | 4 |
9 files changed, 203 insertions, 203 deletions
diff --git a/util/system/atexit.cpp b/util/system/atexit.cpp index c816bc5fe0..74fb10b6b1 100644 --- a/util/system/atexit.cpp +++ b/util/system/atexit.cpp @@ -35,7 +35,7 @@ namespace { } inline void Finish() noexcept { - AtomicSet(FinishStarted_, 1); + AtomicSet(FinishStarted_, 1); auto guard = Guard(Lock_); @@ -65,13 +65,13 @@ namespace { } } - inline bool FinishStarted() const { - return AtomicGet(FinishStarted_); - } + inline bool FinishStarted() const { + return AtomicGet(FinishStarted_); + } private: TAdaptiveLock Lock_; - TAtomic FinishStarted_; + TAtomic FinishStarted_; TDeque<TFunc> Store_; TPriorityQueue<TFunc*, TVector<TFunc*>, TCmp> Items_; }; @@ -108,13 +108,13 @@ void ManualRunAtExitFinalizers() { OnExit(); } -bool ExitStarted() { +bool ExitStarted() { if (TAtExit* const atExit = AtomicGet(atExitPtr)) { return atExit->FinishStarted(); - } + } return false; -} - +} + void AtExit(TAtExitFunc func, void* ctx, size_t priority) { Instance()->Register(func, ctx, priority); } diff --git a/util/system/atexit.h b/util/system/atexit.h index f70c2fcd8c..eb3188615c 100644 --- a/util/system/atexit.h +++ b/util/system/atexit.h @@ -10,8 +10,8 @@ void AtExit(TAtExitFunc func, void* ctx, size_t priority); void AtExit(TTraditionalAtExitFunc func); void AtExit(TTraditionalAtExitFunc func, size_t priority); - -bool ExitStarted(); + +bool ExitStarted(); /** * Generally it's a bad idea to call this method except for some rare cases, diff --git a/util/system/direct_io.cpp b/util/system/direct_io.cpp index 3ace0bfc57..f59c54b0cb 100644 --- a/util/system/direct_io.cpp +++ b/util/system/direct_io.cpp @@ -1,36 +1,36 @@ -#include "direct_io.h" - -#include <util/generic/singleton.h> -#include <util/generic/yexception.h> +#include "direct_io.h" + +#include <util/generic/singleton.h> +#include <util/generic/yexception.h> #include <util/system/info.h> #include "align.h" - -#ifdef _linux_ + +#ifdef _linux_ #include <util/string/cast.h> #include <linux/version.h> #include <sys/utsname.h> -#endif +#endif -namespace { +namespace { struct TAlignmentCalcer { inline TAlignmentCalcer() - : Alignment(0) - { -#ifdef _linux_ - utsname sysInfo; + : Alignment(0) + { +#ifdef _linux_ + utsname sysInfo; Y_VERIFY(!uname(&sysInfo), "Error while call uname: %s", LastSystemErrorText()); - TStringBuf release(sysInfo.release); - release = release.substr(0, release.find_first_not_of(".0123456789")); + TStringBuf release(sysInfo.release); + release = release.substr(0, release.find_first_not_of(".0123456789")); - int v1 = FromString<int>(release.NextTok('.')); - int v2 = FromString<int>(release.NextTok('.')); - int v3 = FromString<int>(release.NextTok('.')); - int linuxVersionCode = KERNEL_VERSION(v1, v2, v3); + int v1 = FromString<int>(release.NextTok('.')); + int v2 = FromString<int>(release.NextTok('.')); + int v3 = FromString<int>(release.NextTok('.')); + int linuxVersionCode = KERNEL_VERSION(v1, v2, v3); if (linuxVersionCode < KERNEL_VERSION(2, 4, 10)) { - Alignment = 0; + Alignment = 0; } else if (linuxVersionCode < KERNEL_VERSION(2, 6, 0)) { Alignment = NSystemInfo::GetPageSize(); } else { @@ -39,44 +39,44 @@ namespace { // See IGNIETFERRO-946. Alignment = 4096; } -#endif - } - - size_t Alignment; - }; -} - +#endif + } + + size_t Alignment; + }; +} + TDirectIOBufferedFile::TDirectIOBufferedFile(const TString& path, EOpenMode oMode, size_t buflen /*= 1 << 17*/) - : File(path, oMode) - , Alignment(0) - , DataLen(0) - , ReadPosition(0) - , WritePosition(0) - , DirectIO(false) -{ + : File(path, oMode) + , Alignment(0) + , DataLen(0) + , ReadPosition(0) + , WritePosition(0) + , DirectIO(false) +{ if (buflen == 0) { ythrow TFileError() << "unbuffered usage is not supported"; - } - - if (oMode & Direct) { + } + + if (oMode & Direct) { Alignment = Singleton<TAlignmentCalcer>()->Alignment; - SetDirectIO(true); - } - - WritePosition = File.GetLength(); - FlushedBytes = WritePosition; - FlushedToDisk = FlushedBytes; - BufLen = (!!Alignment) ? AlignUp(buflen, Alignment) : buflen; - BufferStorage.Resize(BufLen + Alignment); - Buffer = (!!Alignment) ? AlignUp(BufferStorage.Data(), Alignment) : BufferStorage.Data(); -} - -#define DIRECT_IO_FLAGS (O_DIRECT | O_SYNC) - -void TDirectIOBufferedFile::SetDirectIO(bool value) { -#ifdef _linux_ + SetDirectIO(true); + } + + WritePosition = File.GetLength(); + FlushedBytes = WritePosition; + FlushedToDisk = FlushedBytes; + BufLen = (!!Alignment) ? AlignUp(buflen, Alignment) : buflen; + BufferStorage.Resize(BufLen + Alignment); + Buffer = (!!Alignment) ? AlignUp(BufferStorage.Data(), Alignment) : BufferStorage.Data(); +} + +#define DIRECT_IO_FLAGS (O_DIRECT | O_SYNC) + +void TDirectIOBufferedFile::SetDirectIO(bool value) { +#ifdef _linux_ if (DirectIO == value) { - return; + return; } if (!!Alignment && value) { @@ -85,73 +85,73 @@ void TDirectIOBufferedFile::SetDirectIO(bool value) { (void)fcntl(File.GetHandle(), F_SETFL, fcntl(File.GetHandle(), F_GETFL) & ~DIRECT_IO_FLAGS); } - DirectIO = value; -#else DirectIO = value; -#endif -} - -TDirectIOBufferedFile::~TDirectIOBufferedFile() { +#else + DirectIO = value; +#endif +} + +TDirectIOBufferedFile::~TDirectIOBufferedFile() { try { Finish(); } catch (...) { } -} - -void TDirectIOBufferedFile::FlushData() { +} + +void TDirectIOBufferedFile::FlushData() { WriteToFile(Buffer, DataLen, FlushedBytes); DataLen = 0; File.FlushData(); -} +} -void TDirectIOBufferedFile::Finish() { +void TDirectIOBufferedFile::Finish() { FlushData(); File.Flush(); File.Close(); -} - +} + void TDirectIOBufferedFile::Write(const void* buffer, size_t byteCount) { WriteToBuffer(buffer, byteCount, DataLen); WritePosition += byteCount; -} - -void TDirectIOBufferedFile::WriteToBuffer(const void* buf, size_t len, ui64 position) { - while (len > 0) { - size_t writeLen = Min<size_t>(BufLen - position, len); - - if (writeLen > 0) { - memcpy((char*)Buffer + position, buf, writeLen); - buf = (char*)buf + writeLen; - len -= writeLen; +} + +void TDirectIOBufferedFile::WriteToBuffer(const void* buf, size_t len, ui64 position) { + while (len > 0) { + size_t writeLen = Min<size_t>(BufLen - position, len); + + if (writeLen > 0) { + memcpy((char*)Buffer + position, buf, writeLen); + buf = (char*)buf + writeLen; + len -= writeLen; DataLen = (size_t)Max(position + writeLen, (ui64)DataLen); - position += writeLen; - } - - if (DataLen == BufLen) { - WriteToFile(Buffer, DataLen, FlushedBytes); - DataLen = 0; - position = 0; - } - } -} - -void TDirectIOBufferedFile::WriteToFile(const void* buf, size_t len, ui64 position) { - if (!!len) { - SetDirectIO(IsAligned(buf) && IsAligned(len) && IsAligned(position)); + position += writeLen; + } + + if (DataLen == BufLen) { + WriteToFile(Buffer, DataLen, FlushedBytes); + DataLen = 0; + position = 0; + } + } +} + +void TDirectIOBufferedFile::WriteToFile(const void* buf, size_t len, ui64 position) { + if (!!len) { + SetDirectIO(IsAligned(buf) && IsAligned(len) && IsAligned(position)); File.Pwrite(buf, len, position); - FlushedBytes = Max(FlushedBytes, position + len); - FlushedToDisk = Min(FlushedToDisk, position); - } -} - + FlushedBytes = Max(FlushedBytes, position + len); + FlushedToDisk = Min(FlushedToDisk, position); + } +} + size_t TDirectIOBufferedFile::PreadSafe(void* buffer, size_t byteCount, ui64 offset) { - if (FlushedToDisk < offset + byteCount) { - File.FlushData(); - FlushedToDisk = FlushedBytes; - } - + if (FlushedToDisk < offset + byteCount) { + File.FlushData(); + FlushedToDisk = FlushedBytes; + } + #ifdef _linux_ ssize_t bytesRead = 0; do { @@ -159,15 +159,15 @@ size_t TDirectIOBufferedFile::PreadSafe(void* buffer, size_t byteCount, ui64 off } while (bytesRead == -1 && errno == EINTR); if (bytesRead < 0) { - ythrow yexception() << "error while pread file: " << LastSystemError() << "(" << LastSystemErrorText() << ")"; + ythrow yexception() << "error while pread file: " << LastSystemError() << "(" << LastSystemErrorText() << ")"; } return bytesRead; #else return File.Pread(buffer, byteCount, offset); #endif -} - +} + size_t TDirectIOBufferedFile::ReadFromFile(void* buffer, size_t byteCount, ui64 offset) { SetDirectIO(true); @@ -187,64 +187,64 @@ size_t TDirectIOBufferedFile::ReadFromFile(void* buffer, size_t byteCount, ui64 break; } } - + if (!byteCount) { return bytesRead; } - ui64 bufSize = AlignUp(Min<size_t>(BufferStorage.Size(), byteCount + (Alignment << 1)), Alignment); - TBuffer readBufferStorage(bufSize + Alignment); - char* readBuffer = AlignUp((char*)readBufferStorage.Data(), Alignment); + ui64 bufSize = AlignUp(Min<size_t>(BufferStorage.Size(), byteCount + (Alignment << 1)), Alignment); + TBuffer readBufferStorage(bufSize + Alignment); + char* readBuffer = AlignUp((char*)readBufferStorage.Data(), Alignment); - while (byteCount) { + while (byteCount) { ui64 begin = AlignDown(offset, (ui64)Alignment); ui64 end = AlignUp(offset + byteCount, (ui64)Alignment); ui64 toRead = Min(end - begin, bufSize); ui64 fromFile = PreadSafe(readBuffer, toRead, begin); if (!fromFile) { - break; + break; } ui64 delta = offset - begin; ui64 count = Min<ui64>(fromFile - delta, byteCount); - memcpy(buffer, readBuffer + delta, count); - buffer = (char*)buffer + count; - byteCount -= count; - offset += count; + memcpy(buffer, readBuffer + delta, count); + buffer = (char*)buffer + count; + byteCount -= count; + offset += count; bytesRead += count; - } + } return bytesRead; -} - +} + size_t TDirectIOBufferedFile::Read(void* buffer, size_t byteCount) { size_t bytesRead = Pread(buffer, byteCount, ReadPosition); ReadPosition += bytesRead; return bytesRead; -} - +} + size_t TDirectIOBufferedFile::Pread(void* buffer, size_t byteCount, ui64 offset) { if (!byteCount) { - return 0; + return 0; } size_t readFromFile = 0; - if (offset < FlushedBytes) { - readFromFile = Min<ui64>(byteCount, FlushedBytes - offset); + if (offset < FlushedBytes) { + readFromFile = Min<ui64>(byteCount, FlushedBytes - offset); size_t bytesRead = ReadFromFile(buffer, readFromFile, offset); if (bytesRead != readFromFile || readFromFile == byteCount) { return bytesRead; } - } - ui64 start = offset > FlushedBytes ? offset - FlushedBytes : 0; + } + ui64 start = offset > FlushedBytes ? offset - FlushedBytes : 0; ui64 count = Min<ui64>(DataLen - start, byteCount - readFromFile); if (count) { - memcpy((char*)buffer + readFromFile, (const char*)Buffer + start, count); + memcpy((char*)buffer + readFromFile, (const char*)Buffer + start, count); } - return count + readFromFile; -} - + return count + readFromFile; +} + void TDirectIOBufferedFile::Pwrite(const void* buffer, size_t byteCount, ui64 offset) { if (offset > WritePosition) { ythrow yexception() << "cannot frite to position" << offset; @@ -262,5 +262,5 @@ void TDirectIOBufferedFile::Pwrite(const void* buffer, size_t byteCount, ui64 of if (writeToBufer > 0) { ui64 bufferOffset = offset + writeToFile - FlushedBytes; WriteToBuffer((const char*)buffer + writeToFile, writeToBufer, bufferOffset); - } -} + } +} diff --git a/util/system/direct_io.h b/util/system/direct_io.h index 351b647501..6a3325a960 100644 --- a/util/system/direct_io.h +++ b/util/system/direct_io.h @@ -1,42 +1,42 @@ -#pragma once - +#pragma once + #include "align.h" #include "file.h" -#include <util/generic/buffer.h> - -// Supports Linux Direct-IO: -// - Simple buffering logic. -// - Default buffer size of 128KB matches VM page writeback granularity, to maximize IO throughput. -// - Supports writing odd sized files by turning off direct IO for the last chunk. -class TDirectIOBufferedFile { -public: +#include <util/generic/buffer.h> + +// Supports Linux Direct-IO: +// - Simple buffering logic. +// - Default buffer size of 128KB matches VM page writeback granularity, to maximize IO throughput. +// - Supports writing odd sized files by turning off direct IO for the last chunk. +class TDirectIOBufferedFile { +public: TDirectIOBufferedFile(const TString& path, EOpenMode oMode, size_t buflen = 1 << 17); - ~TDirectIOBufferedFile(); + ~TDirectIOBufferedFile(); - void FlushData(); - void Finish(); + void FlushData(); + void Finish(); size_t Read(void* buffer, size_t byteCount); void Write(const void* buffer, size_t byteCount); size_t Pread(void* buffer, size_t byteCount, ui64 offset); void Pwrite(const void* buffer, size_t byteCount, ui64 offset); - - inline bool IsOpen() const { + + inline bool IsOpen() const { return true; - } - - inline ui64 GetWritePosition() const { - return WritePosition; - } - - inline ui64 GetLength() const { - return FlushedBytes + DataLen; - } - + } + + inline ui64 GetWritePosition() const { + return WritePosition; + } + + inline ui64 GetLength() const { + return FlushedBytes + DataLen; + } + inline FHANDLE GetHandle() { return File.GetHandle(); - } - + } + inline void FallocateNoResize(ui64 length) { File.FallocateNoResize(length); } @@ -45,31 +45,31 @@ public: File.ShrinkToFit(); } -private: - inline bool IsAligned(i64 value) { +private: + inline bool IsAligned(i64 value) { return Alignment ? value == AlignDown<i64>(value, Alignment) : true; - } + } - inline bool IsAligned(const void* value) { + inline bool IsAligned(const void* value) { return Alignment ? value == AlignDown(value, Alignment) : true; - } - + } + size_t PreadSafe(void* buffer, size_t byteCount, ui64 offset); size_t ReadFromFile(void* buffer, size_t byteCount, ui64 offset); - void WriteToFile(const void* buf, size_t len, ui64 position); - void WriteToBuffer(const void* buf, size_t len, ui64 position); - void SetDirectIO(bool value); - + void WriteToFile(const void* buf, size_t len, ui64 position); + void WriteToBuffer(const void* buf, size_t len, ui64 position); + void SetDirectIO(bool value); + private: TFile File; - size_t Alignment; - size_t BufLen; - size_t DataLen; - void* Buffer; - TBuffer BufferStorage; - ui64 ReadPosition; - ui64 WritePosition; - ui64 FlushedBytes; - ui64 FlushedToDisk; - bool DirectIO; -}; + size_t Alignment; + size_t BufLen; + size_t DataLen; + void* Buffer; + TBuffer BufferStorage; + ui64 ReadPosition; + ui64 WritePosition; + ui64 FlushedBytes; + ui64 FlushedToDisk; + bool DirectIO; +}; diff --git a/util/system/direct_io_ut.cpp b/util/system/direct_io_ut.cpp index f3872df228..839c3de7ca 100644 --- a/util/system/direct_io_ut.cpp +++ b/util/system/direct_io_ut.cpp @@ -1,12 +1,12 @@ #include <library/cpp/testing/unittest/registar.h> - + #include <util/generic/yexception.h> #include <util/system/fs.h> #include <util/system/tempfile.h> -#include <util/random/random.h> +#include <util/random/random.h> #include "direct_io.h" - + static const char* FileName_("./test.file"); Y_UNIT_TEST_SUITE(TDirectIoTestSuite) { @@ -86,7 +86,7 @@ Y_UNIT_TEST_SUITE(TDirectIoTestSuite) { TestHugeFile(5 * 1024 * 1024 * 1024ULL + 1111); } } -} +} Y_UNIT_TEST_SUITE(TDirectIoErrorHandling) { Y_UNIT_TEST(Constructor) { diff --git a/util/system/error.cpp b/util/system/error.cpp index 3396f21d0c..f778ec42cb 100644 --- a/util/system/error.cpp +++ b/util/system/error.cpp @@ -58,7 +58,7 @@ namespace { const char* LastSystemErrorText(int code) { #if defined(_win_) - TErrString& text(*Singleton<TErrString>()); + TErrString& text(*Singleton<TErrString>()); LastSystemErrorText(text.data, sizeof(text.data), code); return text.data; diff --git a/util/system/info.cpp b/util/system/info.cpp index 6cb5ac8b9d..cf6681e89a 100644 --- a/util/system/info.cpp +++ b/util/system/info.cpp @@ -187,7 +187,7 @@ size_t NSystemInfo::GetPageSize() noexcept { #endif } -size_t NSystemInfo::TotalMemorySize() { +size_t NSystemInfo::TotalMemorySize() { #if defined(_linux_) && defined(_64_) try { auto q = FromString<size_t>(StripString(TFileInput("/sys/fs/cgroup/memory/memory.limit_in_bytes").ReadAll())); @@ -200,9 +200,9 @@ size_t NSystemInfo::TotalMemorySize() { #endif #if defined(_linux_) || defined(_cygwin_) - struct sysinfo info; - sysinfo(&info); - return info.totalram; + struct sysinfo info; + sysinfo(&info); + return info.totalram; #elif defined(_darwin_) int mib[2]; int64_t memSize; @@ -223,10 +223,10 @@ size_t NSystemInfo::TotalMemorySize() { ythrow yexception() << "GlobalMemoryStatusEx failed: " << LastSystemErrorText(); } return (size_t)memoryStatusEx.ullTotalPhys; -#else - return 0; -#endif -} +#else + return 0; +#endif +} size_t NSystemInfo::MaxOpenFiles() { #if defined(ANDROID) || defined(__ANDROID__) diff --git a/util/system/info.h b/util/system/info.h index cdf401cdfb..73ebe48a9a 100644 --- a/util/system/info.h +++ b/util/system/info.h @@ -7,6 +7,6 @@ namespace NSystemInfo { size_t CachedNumberOfCpus(); size_t LoadAverage(double* la, size_t len); size_t GetPageSize() noexcept; - size_t TotalMemorySize(); + size_t TotalMemorySize(); size_t MaxOpenFiles(); } diff --git a/util/system/mlock.cpp b/util/system/mlock.cpp index df69fed797..435338c98f 100644 --- a/util/system/mlock.cpp +++ b/util/system/mlock.cpp @@ -26,7 +26,7 @@ void LockMemory(const void* addr, size_t len) { #elif defined(_win_) HANDLE hndl = GetCurrentProcess(); SIZE_T min, max; - if (!GetProcessWorkingSetSize(hndl, &min, &max)) + if (!GetProcessWorkingSetSize(hndl, &min, &max)) ythrow yexception() << LastSystemErrorText(); if (!SetProcessWorkingSetSize(hndl, min + len, max + len)) ythrow yexception() << LastSystemErrorText(); @@ -43,7 +43,7 @@ void UnlockMemory(const void* addr, size_t len) { #elif defined(_win_) HANDLE hndl = GetCurrentProcess(); SIZE_T min, max; - if (!GetProcessWorkingSetSize(hndl, &min, &max)) + if (!GetProcessWorkingSetSize(hndl, &min, &max)) ythrow yexception() << LastSystemErrorText(); if (!SetProcessWorkingSetSize(hndl, min - len, max - len)) ythrow yexception() << LastSystemErrorText(); |