diff options
author | kulikov <kulikov@yandex-team.ru> | 2022-02-10 16:49:34 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:49:34 +0300 |
commit | c707901605d7b7c6cba0998cd52e1ae619c97762 (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 /util/system | |
parent | 65e5266709e7ff94b14ae128309e229de714b0df (diff) | |
download | ydb-c707901605d7b7c6cba0998cd52e1ae619c97762.tar.gz |
Restoring authorship annotation for <kulikov@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'util/system')
-rw-r--r-- | util/system/direct_io.cpp | 104 | ||||
-rw-r--r-- | util/system/direct_io.h | 12 | ||||
-rw-r--r-- | util/system/direct_io_ut.cpp | 80 | ||||
-rw-r--r-- | util/system/file.cpp | 8 | ||||
-rw-r--r-- | util/system/file.h | 4 | ||||
-rw-r--r-- | util/system/file_ut.cpp | 52 |
6 files changed, 130 insertions, 130 deletions
diff --git a/util/system/direct_io.cpp b/util/system/direct_io.cpp index ea4dc39382..f59c54b0cb 100644 --- a/util/system/direct_io.cpp +++ b/util/system/direct_io.cpp @@ -110,7 +110,7 @@ void TDirectIOBufferedFile::Finish() { File.Close(); } -void TDirectIOBufferedFile::Write(const void* buffer, size_t byteCount) { +void TDirectIOBufferedFile::Write(const void* buffer, size_t byteCount) { WriteToBuffer(buffer, byteCount, DataLen); WritePosition += byteCount; } @@ -146,51 +146,51 @@ void TDirectIOBufferedFile::WriteToFile(const void* buf, size_t len, ui64 positi } } -size_t TDirectIOBufferedFile::PreadSafe(void* buffer, size_t byteCount, ui64 offset) { +size_t TDirectIOBufferedFile::PreadSafe(void* buffer, size_t byteCount, ui64 offset) { if (FlushedToDisk < offset + byteCount) { File.FlushData(); FlushedToDisk = FlushedBytes; } -#ifdef _linux_ - ssize_t bytesRead = 0; - do { - bytesRead = pread(File.GetHandle(), buffer, byteCount, offset); - } while (bytesRead == -1 && errno == EINTR); +#ifdef _linux_ + ssize_t bytesRead = 0; + do { + bytesRead = pread(File.GetHandle(), buffer, byteCount, offset); + } while (bytesRead == -1 && errno == EINTR); - if (bytesRead < 0) { + if (bytesRead < 0) { ythrow yexception() << "error while pread file: " << LastSystemError() << "(" << LastSystemErrorText() << ")"; } - return bytesRead; -#else - return File.Pread(buffer, byteCount, offset); -#endif + return bytesRead; +#else + return File.Pread(buffer, byteCount, offset); +#endif } -size_t TDirectIOBufferedFile::ReadFromFile(void* buffer, size_t byteCount, ui64 offset) { - SetDirectIO(true); - - ui64 bytesRead = 0; - - while (byteCount) { - if (!Alignment || IsAligned(buffer) && IsAligned(byteCount) && IsAligned(offset)) { - if (const ui64 fromFile = PreadSafe(buffer, byteCount, offset)) { - buffer = (char*)buffer + fromFile; - byteCount -= fromFile; - offset += fromFile; - bytesRead += fromFile; - } else { - return bytesRead; - } - } else { - break; - } +size_t TDirectIOBufferedFile::ReadFromFile(void* buffer, size_t byteCount, ui64 offset) { + SetDirectIO(true); + + ui64 bytesRead = 0; + + while (byteCount) { + if (!Alignment || IsAligned(buffer) && IsAligned(byteCount) && IsAligned(offset)) { + if (const ui64 fromFile = PreadSafe(buffer, byteCount, offset)) { + buffer = (char*)buffer + fromFile; + byteCount -= fromFile; + offset += fromFile; + bytesRead += fromFile; + } else { + return bytesRead; + } + } else { + break; + } } - if (!byteCount) { - return bytesRead; - } + if (!byteCount) { + return bytesRead; + } ui64 bufSize = AlignUp(Min<size_t>(BufferStorage.Size(), byteCount + (Alignment << 1)), Alignment); TBuffer readBufferStorage(bufSize + Alignment); @@ -199,59 +199,59 @@ size_t TDirectIOBufferedFile::ReadFromFile(void* buffer, size_t byteCount, ui64 while (byteCount) { ui64 begin = AlignDown(offset, (ui64)Alignment); ui64 end = AlignUp(offset + byteCount, (ui64)Alignment); - ui64 toRead = Min(end - begin, bufSize); - ui64 fromFile = PreadSafe(readBuffer, toRead, begin); + ui64 toRead = Min(end - begin, bufSize); + ui64 fromFile = PreadSafe(readBuffer, toRead, begin); if (!fromFile) { break; } - ui64 delta = offset - begin; - ui64 count = Min<ui64>(fromFile - delta, byteCount); + ui64 delta = offset - begin; + ui64 count = Min<ui64>(fromFile - delta, byteCount); memcpy(buffer, readBuffer + delta, count); buffer = (char*)buffer + count; byteCount -= count; offset += count; - bytesRead += count; + bytesRead += count; } - return bytesRead; + return bytesRead; } -size_t TDirectIOBufferedFile::Read(void* buffer, size_t byteCount) { - size_t bytesRead = Pread(buffer, byteCount, ReadPosition); - ReadPosition += bytesRead; - return bytesRead; +size_t TDirectIOBufferedFile::Read(void* buffer, size_t byteCount) { + size_t bytesRead = Pread(buffer, byteCount, ReadPosition); + ReadPosition += bytesRead; + return bytesRead; } -size_t TDirectIOBufferedFile::Pread(void* buffer, size_t byteCount, ui64 offset) { +size_t TDirectIOBufferedFile::Pread(void* buffer, size_t byteCount, ui64 offset) { if (!byteCount) { return 0; } - size_t readFromFile = 0; + size_t readFromFile = 0; if (offset < FlushedBytes) { readFromFile = Min<ui64>(byteCount, FlushedBytes - offset); - size_t bytesRead = ReadFromFile(buffer, readFromFile, offset); - if (bytesRead != readFromFile || readFromFile == byteCount) { - return bytesRead; + size_t bytesRead = ReadFromFile(buffer, readFromFile, offset); + if (bytesRead != readFromFile || readFromFile == byteCount) { + return bytesRead; } } ui64 start = offset > FlushedBytes ? offset - FlushedBytes : 0; - ui64 count = Min<ui64>(DataLen - start, byteCount - readFromFile); + ui64 count = Min<ui64>(DataLen - start, byteCount - readFromFile); if (count) { memcpy((char*)buffer + readFromFile, (const char*)Buffer + start, count); } return count + readFromFile; } -void TDirectIOBufferedFile::Pwrite(const void* buffer, size_t byteCount, ui64 offset) { +void TDirectIOBufferedFile::Pwrite(const void* buffer, size_t byteCount, ui64 offset) { if (offset > WritePosition) { ythrow yexception() << "cannot frite to position" << offset; } - size_t writeToBufer = byteCount; - size_t writeToFile = 0; + size_t writeToBufer = byteCount; + size_t writeToFile = 0; if (FlushedBytes > offset) { writeToFile = Min<ui64>(byteCount, FlushedBytes - offset); diff --git a/util/system/direct_io.h b/util/system/direct_io.h index ec1ff84356..6a3325a960 100644 --- a/util/system/direct_io.h +++ b/util/system/direct_io.h @@ -16,10 +16,10 @@ public: void FlushData(); void Finish(); - size_t Read(void* buffer, size_t byteCount); - void Write(const void* buffer, size_t byteCount); - size_t Pread(void* buffer, size_t byteCount, ui64 offset); - void Pwrite(const void* buffer, size_t byteCount, ui64 offset); + size_t Read(void* buffer, size_t byteCount); + void Write(const void* buffer, size_t byteCount); + size_t Pread(void* buffer, size_t byteCount, ui64 offset); + void Pwrite(const void* buffer, size_t byteCount, ui64 offset); inline bool IsOpen() const { return true; @@ -54,8 +54,8 @@ private: return Alignment ? value == AlignDown(value, Alignment) : true; } - size_t PreadSafe(void* buffer, size_t byteCount, ui64 offset); - size_t ReadFromFile(void* buffer, size_t byteCount, ui64 offset); + size_t PreadSafe(void* buffer, size_t byteCount, ui64 offset); + size_t ReadFromFile(void* buffer, size_t byteCount, ui64 offset); void WriteToFile(const void* buf, size_t len, ui64 position); void WriteToBuffer(const void* buf, size_t len, ui64 position); void SetDirectIO(bool value); diff --git a/util/system/direct_io_ut.cpp b/util/system/direct_io_ut.cpp index 36c1a20d63..839c3de7ca 100644 --- a/util/system/direct_io_ut.cpp +++ b/util/system/direct_io_ut.cpp @@ -2,7 +2,7 @@ #include <util/generic/yexception.h> #include <util/system/fs.h> -#include <util/system/tempfile.h> +#include <util/system/tempfile.h> #include <util/random/random.h> #include "direct_io.h" @@ -13,7 +13,7 @@ Y_UNIT_TEST_SUITE(TDirectIoTestSuite) { Y_UNIT_TEST(TestDirectFile) { TDirectIOBufferedFile file(FileName_, RdWr | Direct | Seq | CreateAlways, 1 << 15); TVector<ui64> data((1 << 15) + 1); - TVector<ui64> readResult(data.size()); + TVector<ui64> readResult(data.size()); for (auto& i : data) { i = RandomNumber<ui64>(); } @@ -24,10 +24,10 @@ Y_UNIT_TEST_SUITE(TDirectIoTestSuite) { size_t readPos = RandomNumber(writePos); size_t readCount = RandomNumber(writePos - readPos); UNIT_ASSERT_VALUES_EQUAL( - file.Pread(&readResult[0], readCount * sizeof(ui64), readPos * sizeof(ui64)), + file.Pread(&readResult[0], readCount * sizeof(ui64), readPos * sizeof(ui64)), readCount * sizeof(ui64)); for (size_t i = 0; i < readCount; ++i) { - UNIT_ASSERT_VALUES_EQUAL(readResult[i], data[i + readPos]); + UNIT_ASSERT_VALUES_EQUAL(readResult[i], data[i + readPos]); } } file.Finish(); @@ -36,56 +36,56 @@ Y_UNIT_TEST_SUITE(TDirectIoTestSuite) { size_t readPos = RandomNumber(data.size()); size_t readCount = RandomNumber(data.size() - readPos); UNIT_ASSERT_VALUES_EQUAL( - fileNew.Pread(&readResult[0], readCount * sizeof(ui64), readPos * sizeof(ui64)), + fileNew.Pread(&readResult[0], readCount * sizeof(ui64), readPos * sizeof(ui64)), readCount * sizeof(ui64)); for (size_t j = 0; j < readCount; ++j) { - UNIT_ASSERT_VALUES_EQUAL(readResult[j], data[j + readPos]); + UNIT_ASSERT_VALUES_EQUAL(readResult[j], data[j + readPos]); } } size_t readCount = data.size(); UNIT_ASSERT_VALUES_EQUAL( - fileNew.Pread(&readResult[0], readCount * sizeof(ui64), 0), + fileNew.Pread(&readResult[0], readCount * sizeof(ui64), 0), readCount * sizeof(ui64)); for (size_t i = 0; i < readCount; ++i) { - UNIT_ASSERT_VALUES_EQUAL(readResult[i], data[i]); + UNIT_ASSERT_VALUES_EQUAL(readResult[i], data[i]); } NFs::Remove(FileName_); } - void TestHugeFile(size_t size) { - TTempFile tmpFile("test.file"); - - { - TDirectIOBufferedFile directIOFile(tmpFile.Name(), WrOnly | CreateAlways | Direct); - TVector<ui8> data(size, 'x'); - directIOFile.Write(&data[0], data.size()); - } - - { - TDirectIOBufferedFile directIOFile(tmpFile.Name(), RdOnly | Direct); - TVector<ui8> data(size + 1, 'y'); - - const size_t readResult = directIOFile.Read(&data[0], data.size()); - - UNIT_ASSERT_VALUES_EQUAL(readResult, size); - - UNIT_ASSERT_VALUES_EQUAL(data[0], 'x'); - UNIT_ASSERT_VALUES_EQUAL(data[size / 2], 'x'); - UNIT_ASSERT_VALUES_EQUAL(data[size - 1], 'x'); - UNIT_ASSERT_VALUES_EQUAL(data[size], 'y'); - } - } - - Y_UNIT_TEST(TestHugeFile1) { + void TestHugeFile(size_t size) { + TTempFile tmpFile("test.file"); + + { + TDirectIOBufferedFile directIOFile(tmpFile.Name(), WrOnly | CreateAlways | Direct); + TVector<ui8> data(size, 'x'); + directIOFile.Write(&data[0], data.size()); + } + + { + TDirectIOBufferedFile directIOFile(tmpFile.Name(), RdOnly | Direct); + TVector<ui8> data(size + 1, 'y'); + + const size_t readResult = directIOFile.Read(&data[0], data.size()); + + UNIT_ASSERT_VALUES_EQUAL(readResult, size); + + UNIT_ASSERT_VALUES_EQUAL(data[0], 'x'); + UNIT_ASSERT_VALUES_EQUAL(data[size / 2], 'x'); + UNIT_ASSERT_VALUES_EQUAL(data[size - 1], 'x'); + UNIT_ASSERT_VALUES_EQUAL(data[size], 'y'); + } + } + + Y_UNIT_TEST(TestHugeFile1) { if constexpr (sizeof(size_t) > 4) { - TestHugeFile(5 * 1024 * 1024 * 1024ULL); - } - } - Y_UNIT_TEST(TestHugeFile2) { + TestHugeFile(5 * 1024 * 1024 * 1024ULL); + } + } + Y_UNIT_TEST(TestHugeFile2) { if constexpr (sizeof(size_t) > 4) { - TestHugeFile(5 * 1024 * 1024 * 1024ULL + 1111); - } - } + TestHugeFile(5 * 1024 * 1024 * 1024ULL + 1111); + } + } } Y_UNIT_TEST_SUITE(TDirectIoErrorHandling) { diff --git a/util/system/file.cpp b/util/system/file.cpp index 6115799e8d..4a261d020c 100644 --- a/util/system/file.cpp +++ b/util/system/file.cpp @@ -1191,10 +1191,10 @@ size_t TFile::Pread(void* buf, size_t len, i64 offset) const { return Impl_->Pread(buf, len, offset); } -i32 TFile::RawPread(void* buf, ui32 len, i64 offset) const { - return Impl_->RawPread(buf, len, offset); -} - +i32 TFile::RawPread(void* buf, ui32 len, i64 offset) const { + return Impl_->RawPread(buf, len, offset); +} + void TFile::Pload(void* buf, size_t len, i64 offset) const { Impl_->Pload(buf, len, offset); } diff --git a/util/system/file.h b/util/system/file.h index 3b18ae07ce..9502e159b6 100644 --- a/util/system/file.h +++ b/util/system/file.h @@ -186,8 +186,8 @@ public: // Retries incomplete reads until EOF, throws on error size_t Pread(void* buf, size_t len, i64 offset) const; - // Single pread call - i32 RawPread(void* buf, ui32 len, i64 offset) const; + // Single pread call + i32 RawPread(void* buf, ui32 len, i64 offset) const; // Reads exactly len bytes, throws on premature EOF or error void Pload(void* buf, size_t len, i64 offset) const; diff --git a/util/system/file_ut.cpp b/util/system/file_ut.cpp index 2d5343c94d..941e6a50f3 100644 --- a/util/system/file_ut.cpp +++ b/util/system/file_ut.cpp @@ -21,7 +21,7 @@ class TFileTest: public TTestBase { UNIT_TEST(TestFlushSpecialFile); UNIT_TEST(TestRawRead); UNIT_TEST(TestRead); - UNIT_TEST(TestRawPread); + UNIT_TEST(TestRawPread); UNIT_TEST(TestPread); UNIT_TEST(TestCache); UNIT_TEST_SUITE_END(); @@ -35,7 +35,7 @@ public: void TestFlushSpecialFile(); void TestRawRead(); void TestRead(); - void TestRawPread(); + void TestRawPread(); void TestPread(); void TestCache(); @@ -281,30 +281,30 @@ void TFileTest::TestRead() { } } -void TFileTest::TestRawPread() { - TTempFile tmp("tmp"); - - { - TFile file(tmp.Name(), OpenAlways | WrOnly); - file.Write("1234567", 7); - file.Flush(); - file.Close(); - } - - { - TFile file(tmp.Name(), OpenExisting | RdOnly); - char buf[7]; - i32 reallyRead = file.RawPread(buf, 3, 1); - Y_ENSURE(0 <= reallyRead && reallyRead <= 3); - Y_ENSURE(TStringBuf(buf, reallyRead) == TStringBuf("234").Head(reallyRead)); - - memset(buf, 0, sizeof(buf)); - reallyRead = file.RawPread(buf, 2, 5); - Y_ENSURE(0 <= reallyRead && reallyRead <= 2); - Y_ENSURE(TStringBuf(buf, reallyRead) == TStringBuf("67").Head(reallyRead)); - } -} - +void TFileTest::TestRawPread() { + TTempFile tmp("tmp"); + + { + TFile file(tmp.Name(), OpenAlways | WrOnly); + file.Write("1234567", 7); + file.Flush(); + file.Close(); + } + + { + TFile file(tmp.Name(), OpenExisting | RdOnly); + char buf[7]; + i32 reallyRead = file.RawPread(buf, 3, 1); + Y_ENSURE(0 <= reallyRead && reallyRead <= 3); + Y_ENSURE(TStringBuf(buf, reallyRead) == TStringBuf("234").Head(reallyRead)); + + memset(buf, 0, sizeof(buf)); + reallyRead = file.RawPread(buf, 2, 5); + Y_ENSURE(0 <= reallyRead && reallyRead <= 2); + Y_ENSURE(TStringBuf(buf, reallyRead) == TStringBuf("67").Head(reallyRead)); + } +} + void TFileTest::TestPread() { TTempFile tmp("tmp"); |