diff options
author | robot-piglet <robot-piglet@yandex-team.com> | 2023-12-01 16:59:11 +0300 |
---|---|---|
committer | robot-piglet <robot-piglet@yandex-team.com> | 2023-12-01 19:54:31 +0300 |
commit | 3715aa9254f65ae1058290101351a72a6d3a67d4 (patch) | |
tree | 9ac5a1cdab42dfc7cd095a06a362e0681cb1482f /library/cpp | |
parent | b20a8c04fb7e595955ca9d1b943033342b6580cb (diff) | |
download | ydb-3715aa9254f65ae1058290101351a72a6d3a67d4.tar.gz |
Intermediate changes
Diffstat (limited to 'library/cpp')
77 files changed, 0 insertions, 9546 deletions
diff --git a/library/cpp/deprecated/autoarray/README.md b/library/cpp/deprecated/autoarray/README.md deleted file mode 100644 index 1d83147cee..0000000000 --- a/library/cpp/deprecated/autoarray/README.md +++ /dev/null @@ -1,3 +0,0 @@ -Pre-C++11 vector-like container. - -Just use std::vector. If you need to fill your vector with custom-constructed data, use reserve+emplace_back (but make sure that your elements are movable). diff --git a/library/cpp/deprecated/autoarray/autoarray.cpp b/library/cpp/deprecated/autoarray/autoarray.cpp deleted file mode 100644 index 15167f27f6..0000000000 --- a/library/cpp/deprecated/autoarray/autoarray.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "autoarray.h" diff --git a/library/cpp/deprecated/autoarray/autoarray.h b/library/cpp/deprecated/autoarray/autoarray.h deleted file mode 100644 index 2aa12c5916..0000000000 --- a/library/cpp/deprecated/autoarray/autoarray.h +++ /dev/null @@ -1,264 +0,0 @@ -#pragma once - -#include <util/system/compat.h> -#include <util/system/yassert.h> -#include <util/system/defaults.h> -#include <util/system/sys_alloc.h> - -#include <util/generic/typetraits.h> -#include <utility> - -#include <new> -#include <util/generic/noncopyable.h> - -struct autoarray_getindex { - autoarray_getindex() = default; -}; - -struct aarr_b0 { - aarr_b0() = default; -}; - -struct aarr_nofill { - aarr_nofill() = default; -}; - -template <typename T> -struct ynd_type_traits { - enum { - empty_destructor = TTypeTraits<T>::IsPod, - }; -}; - -template <class T> -class autoarray : TNonCopyable { -protected: - T* arr; - size_t _size; - -private: - void AllocBuf(size_t siz) { - arr = nullptr; - _size = 0; - if (siz) { - arr = (T*)y_allocate(sizeof(T) * siz); - _size = siz; - } - } - -public: - using value_type = T; - using iterator = T*; - using const_iterator = const T*; - - autoarray() - : arr(nullptr) - , _size(0) - { - } - autoarray(size_t siz) { - AllocBuf(siz); - T* curr = arr; - try { - for (T* end = arr + _size; curr != end; ++curr) - new (curr) T(); - } catch (...) { - for (--curr; curr >= arr; --curr) - curr->~T(); - y_deallocate(arr); - throw; - } - } - template <class A> - explicit autoarray(size_t siz, A& fill) { - AllocBuf(siz); - T* curr = arr; - try { - for (T* end = arr + _size; curr != end; ++curr) - new (curr) T(fill); - } catch (...) { - for (--curr; curr >= arr; --curr) - curr->~T(); - y_deallocate(arr); - throw; - } - } - explicit autoarray(size_t siz, autoarray_getindex) { - AllocBuf(siz); - size_t nCurrent = 0; - try { - for (nCurrent = 0; nCurrent < _size; ++nCurrent) - new (&arr[nCurrent]) T(nCurrent); - } catch (...) { - for (size_t n = 0; n < nCurrent; ++n) - arr[n].~T(); - y_deallocate(arr); - throw; - } - } - explicit autoarray(size_t siz, aarr_b0) { - AllocBuf(siz); - memset(arr, 0, _size * sizeof(T)); - } - explicit autoarray(size_t siz, aarr_nofill) { - AllocBuf(siz); - } - template <class A> - explicit autoarray(const A* fill, size_t siz) { - AllocBuf(siz); - size_t nCurrent = 0; - try { - for (nCurrent = 0; nCurrent < _size; ++nCurrent) - new (&arr[nCurrent]) T(fill[nCurrent]); - } catch (...) { - for (size_t n = 0; n < nCurrent; ++n) - arr[n].~T(); - y_deallocate(arr); - throw; - } - } - template <class A, class B> - explicit autoarray(const A* fill, const B* cfill, size_t siz) { - AllocBuf(siz); - size_t nCurrent = 0; - try { - for (nCurrent = 0; nCurrent < _size; ++nCurrent) - new (&arr[nCurrent]) T(fill[nCurrent], cfill); - } catch (...) { - for (size_t n = 0; n < nCurrent; ++n) - arr[n].~T(); - y_deallocate(arr); - throw; - } - } - template <class A> - explicit autoarray(const A* fill, size_t initsiz, size_t fullsiz) { - AllocBuf(fullsiz); - size_t nCurrent = 0; - try { - for (nCurrent = 0; nCurrent < ((initsiz < _size) ? initsiz : _size); ++nCurrent) - new (&arr[nCurrent]) T(fill[nCurrent]); - for (; nCurrent < _size; ++nCurrent) - new (&arr[nCurrent]) T(); - } catch (...) { - for (size_t n = 0; n < nCurrent; ++n) - arr[n].~T(); - y_deallocate(arr); - throw; - } - } - template <class A> - explicit autoarray(const A* fill, size_t initsiz, size_t fullsiz, const T& dummy) { - AllocBuf(fullsiz); - size_t nCurrent = 0; - try { - for (nCurrent = 0; nCurrent < ((initsiz < _size) ? initsiz : _size); ++nCurrent) - new (&arr[nCurrent]) T(fill[nCurrent]); - for (; nCurrent < _size; ++nCurrent) - new (&arr[nCurrent]) T(dummy); - } catch (...) { - for (size_t n = 0; n < nCurrent; ++n) - arr[n].~T(); - y_deallocate(arr); - throw; - } - } - - template <class... R> - explicit autoarray(size_t siz, R&&... fill) { - AllocBuf(siz); - T* curr = arr; - try { - for (T* end = arr + _size; curr != end; ++curr) - new (curr) T(std::forward<R>(fill)...); - } catch (...) { - for (--curr; curr >= arr; --curr) - curr->~T(); - y_deallocate(arr); - throw; - } - } - ~autoarray() { - if (_size) { - if (!ynd_type_traits<T>::empty_destructor) - for (T *curr = arr, *end = arr + _size; curr != end; ++curr) - curr->~T(); - y_deallocate(arr); - } - } - T& operator[](size_t pos) { - Y_ASSERT(pos < _size); - return arr[pos]; - } - const T& operator[](size_t pos) const { - Y_ASSERT(pos < _size); - return arr[pos]; - } - size_t size() const { - return _size; - } - void swap(autoarray& with) { - T* tmp_arr = arr; - size_t tmp_size = _size; - arr = with.arr; - _size = with._size; - with.arr = tmp_arr; - with._size = tmp_size; - } - void resize(size_t siz) { - autoarray<T> tmp(arr, _size, siz); - swap(tmp); - } - void resize(size_t siz, const T& dummy) { - autoarray<T> tmp(arr, _size, siz, dummy); - swap(tmp); - } - T* rawpointer() { - return arr; - } - const T* operator~() const { - return arr; - } - T* begin() { - return arr; - } - T* end() { - return arr + _size; - } - T& back() { - Y_ASSERT(_size); - return arr[_size - 1]; - } - bool empty() const { - return !_size; - } - bool operator!() const { - return !_size; - } - size_t operator+() const { - return _size; - } - const T* begin() const { - return arr; - } - const T* end() const { - return arr + _size; - } - const T& back() const { - Y_ASSERT(_size); - return arr[_size - 1]; - } - //operator T*() { return arr; } -}; - -template <class T> -inline bool operator==(const autoarray<T>& a, const autoarray<T>& b) { - size_t count = a.size(); - if (count != b.size()) - return false; - for (size_t i = 0; i < count; ++i) { - if (a[i] != b[i]) - return false; - } - return true; -} diff --git a/library/cpp/deprecated/autoarray/ya.make b/library/cpp/deprecated/autoarray/ya.make deleted file mode 100644 index 4b055f8c29..0000000000 --- a/library/cpp/deprecated/autoarray/ya.make +++ /dev/null @@ -1,7 +0,0 @@ -LIBRARY() - -SRCS( - autoarray.cpp -) - -END() diff --git a/library/cpp/deprecated/fgood/README.md b/library/cpp/deprecated/fgood/README.md deleted file mode 100644 index 4f66289657..0000000000 --- a/library/cpp/deprecated/fgood/README.md +++ /dev/null @@ -1,15 +0,0 @@ -Some ancient wrappers on top of FILE*, and some string manupulation functions. - -Alternatives are as follows. - -For TFILEPtr. Use TIFStream or TOFStream if you need IO. For some rare use cases a TFileMap might also do. - -For fput/fget/getline. Use streams API. - -For struct ffb and struct prnstr. Just don't use them. Even if you can figure out what they do. - -For sf family of functions and TLineSplitter. Just use Split* from util/string/split.h - -For TSFReader. Use TMapTsvFile. - -For read_or_die family of functions. Use streams API. diff --git a/library/cpp/deprecated/fgood/ffb.cpp b/library/cpp/deprecated/fgood/ffb.cpp deleted file mode 100644 index aa9da861a6..0000000000 --- a/library/cpp/deprecated/fgood/ffb.cpp +++ /dev/null @@ -1,407 +0,0 @@ -#include "ffb.h" - -#include <util/string/util.h> // str_spn -#include <util/system/compat.h> -#include <util/generic/yexception.h> - -#include <cstdio> -#include <algorithm> - -#include <ctype.h> - -#ifdef _win_ -#include <io.h> -#else -#include <unistd.h> -#endif - -ffb::ffb(FILE* file) - : TFILEPtr(file) -{ - if (file && !isatty(fileno(file)) && BUFSIZ < 512 * 1024) - setvbuf(file, nullptr, _IOFBF, 512 * 1024); -} - -void ffb::operator=(FILE* f) { - TFILEPtr::operator=(f); - if (f && !isatty(fileno(f)) && BUFSIZ < 512 * 1024) - setvbuf(f, nullptr, _IOFBF, 512 * 1024); -} - -void ffb::open(const char* name, const char* mode) { - TFILEPtr::open(name, mode); - if (!isatty(fileno(*this)) && BUFSIZ < 512 * 1024) - setvbuf(*this, nullptr, _IOFBF, 512 * 1024); -} - -int sf(char** fb, char* buf) { //don't want to call sf(fb, buf, 32) - if (!(*buf && *buf != 10)) { - *fb = nullptr; - return 0; - } - int n = 1; - fb[0] = buf; - while (*buf && *buf != 10 && n < 31) { - if (*buf == '\t') { - *buf++ = 0; - fb[n++] = buf; - continue; - } - buf++; - } - if (*buf == 10 && buf[-1] == 13) - buf[-1] = 0; - *buf = 0; - fb[n] = nullptr; - return n; -} - -int sf(char** fb, char* buf, size_t fb_sz) { - if (!(*buf && *buf != 10)) { - *fb = nullptr; - return 0; - } - fb_sz--; - int n = 1; - fb[0] = buf; - while (*buf && *buf != 10 && n < (int)fb_sz) { - if (*buf == '\t') { - *buf++ = 0; - fb[n++] = buf; - continue; - } - buf++; - } - if (*buf == 10 && buf[-1] == 13) - buf[-1] = 0; - *buf = 0; - fb[n] = nullptr; - return n; -} - -inline int sf_blank(char** fb, char* buf, size_t fb_sz) { - while (isspace((ui8)*buf)) - buf++; - if (!*buf) { - *fb = nullptr; - return 0; - } - fb_sz--; - int n = 1; - fb[0] = buf; - while (*buf && *buf != 10 && n < (int)fb_sz) { - if (isspace((ui8)*buf)) { - *buf++ = 0; - while (isspace((ui8)*buf)) - buf++; - if (*buf) - fb[n++] = buf; - continue; - } - buf++; - } - if (*buf == 10 && buf[-1] == 13) - buf[-1] = 0; - *buf = 0; - fb[n] = nullptr; - return n; -} - -int sf(char fs, char** fb, char* buf, size_t fb_sz) { - if (fs == ' ') - return sf_blank(fb, buf, fb_sz); - while (*buf == fs) - buf++; - if (!(*buf && *buf != 10)) { - *fb = nullptr; - return 0; - } - fb_sz--; - int n = 1; - fb[0] = buf; - while (*buf && *buf != 10 && n < (int)fb_sz) { - if (*buf == fs) { - *buf++ = 0; - while (*buf == fs) - buf++; - fb[n++] = buf; - continue; - } - buf++; - } - if (*buf == 10 && buf[-1] == 13) - buf[-1] = 0; - *buf = 0; - fb[n] = nullptr; - return n; -} - -int sf(const char* fs, char** fb, char* buf, size_t fb_sz) { - if (!(*buf && *buf != 10)) { - *fb = nullptr; - return 0; - } - int fs_len = strlen(fs); - fb_sz--; - int n = 1; - fb[0] = buf; - while (*buf && *buf != 10 && n < (int)fb_sz) { - if (*buf == *fs && !strncmp(buf + 1, fs + 1, fs_len - 1)) { - *buf = 0; - buf += fs_len; - fb[n++] = buf; - continue; - } - buf++; - } - if (*buf == 10 && buf[-1] == 13) - buf[-1] = 0; - *buf = 0; - fb[n] = nullptr; - return n; -} - -inline bool is_end(const char* p) { - return !p || !p[0]; -} - -int sf(const char* seps, char* buf, char** fb, size_t fb_sz) { - if (fb_sz < 1 || is_end(buf)) { - *fb = nullptr; - return 0; - } - str_spn sseps(seps); - fb[0] = nullptr; - int n = 0; - // skip leading delimeters - buf = sseps.cbrk(buf); - if (is_end(buf)) - return 0; - // store fields - while (n < (int)fb_sz) { - fb[n++] = buf; - // find delimeters - buf = sseps.brk(buf + 1); - if (is_end(buf)) - break; - *buf = 0; - // skip delimiters - buf = sseps.cbrk(buf + 1); - if (is_end(buf)) - break; - } - fb[n] = nullptr; - return n; -} - -void TLineSplitter::operator()(char* p, TVector<char*>& fields) const { - if (!p || !*p) - return; - char* q = p; - while (1) { - p = Sep.brk(p); - if (q && (p - q || !SkipEmpty())) - fields.push_back(q); - q = nullptr; - if (!*p) - break; - if (SepStrLen == 1 || (SepStrLen > 1 && !strncmp(p + 1, SepStr + 1, SepStrLen - 1))) { - *p = 0; - p += SepStrLen; - q = p; - } else - p++; - } -} - -void TLineSplitter::operator()(const char* p, TVector<std::pair<const char*, size_t>>& fields) const { - if (!p || !*p) - return; - const char* q = p; - while (1) { - p = Sep.brk(p); - if (q && (p - q || !SkipEmpty())) - fields.push_back(std::make_pair(q, p - q)); - q = nullptr; - if (!*p) - break; - if (SepStrLen == 1 || (SepStrLen > 1 && !strncmp(p + 1, SepStr + 1, SepStrLen - 1))) { - p += SepStrLen; - q = p; - } else - p++; - } -} - -TSFReader::TSFReader(const char* fname, char sep, i32 nfrq) // if sep == ' ' isspace will be imitated (for compat) - : Split(str_spn(sep == ' ' ? "\t\n\v\f\r " : TString(1, sep).data()), sep == ' ') - , OpenPipe(false) -{ - Open(fname, nfrq); -} - -TSFReader::TSFReader(const char* fname, const char* sep, i32 nfrq) - : Split(sep, false) - , OpenPipe(false) -{ - Open(fname, nfrq); -} - -TSFReader::TSFReader(const char* fname, const TLineSplitter& spl, i32 nfrq) - : Split(spl) - , OpenPipe(false) -{ - Open(fname, nfrq); -} - -void TSFReader::Open(const char* fname, i32 nfrq, size_t vbuf_size) { - FieldsRequired = nfrq; - NF = NR = 0; - - if (IsOpen()) - File.close(); - - if (!fname) - return; - - if (!strcmp(fname, "/dev/stdin")) { - File.assign(stdin, "/dev/stdin"); - } else { - if (OpenPipe) - File.popen(fname, "r"); - else - File.open(fname, "r"); - } - OpenPipe = false; - if (!isatty(fileno(File))) - setvbuf(File, nullptr, _IOFBF, vbuf_size); -} - -void TSFReader::Popen(const char* pname, i32 nfrq, size_t vbuf_size) { - OpenPipe = true; - Open(pname, nfrq, vbuf_size); -} - -bool TSFReader::NextLine(segmented_string_pool* pool) { - size_t line_len = 0; - -#ifdef __FreeBSD__ - char* ptr = fgetln(File, &line_len); - if (!ptr) - return false; - if (!line_len || ptr[line_len - 1] != '\n') { // last line w/o newline - Buf.AssignNoAlias(ptr, line_len); - ptr = Buf.begin(); - } else { - // can safely replace newline with \0 - ptr[line_len - 1] = 0; - --line_len; - } -#else - if (!getline(File, Buf)) - return false; - char* ptr = Buf.begin(); - line_len = Buf.size(); -#endif - if (line_len && ptr[line_len - 1] == '\r') - ptr[line_len - 1] = 0; - - if (pool) { - char* nptr = pool->append(ptr); - Y_ASSERT(!strcmp(ptr, nptr)); - ptr = nptr; - } - - ++NR; - Fields.clear(); - Split(ptr, Fields); - NF = Fields.size(); - - if (FieldsRequired != -1 && FieldsRequired != (int)NF) - ythrow yexception() << File.name() << " line " << NR << ": " << NF << " fields, expected " << FieldsRequired; - - return true; -} - -int prnstr::f(const char* c, ...) { - va_list params; - int n = asize - pos, k; - va_start(params, c); - while ((k = vsnprintf(buf + pos, n, c, params)) >= n) { - n += asize, asize *= 2; - while (k + pos >= n) - n += asize, asize *= 2; - char* t = new char[asize]; - memcpy(t, buf, pos); - delete[] buf; - buf = t; - va_end(params); - va_start(params, c); - } - pos += k; - va_end(params); - return k; -} -int prnstr::s(const char* c, size_t k) { - if (!c) - return 0; - size_t n = asize - pos; - if (k >= n) { - n += asize, asize *= 2; - while (k + pos >= n) - n += asize, asize *= 2; - char* t = new char[asize]; - memcpy(t, buf, pos); - delete[] buf; - buf = t; - } - memcpy(buf + pos, c, k); - pos += k; - buf[pos] = 0; - return k; -} -void prnstr::clear() { - pos = 0; - if (asize > 32768) { - asize = 32768; - delete[] buf; - buf = new char[asize]; - } -} - -void prnstr::swap(prnstr& w) { - std::swap(buf, w.buf); - std::swap(pos, w.pos); - std::swap(asize, w.asize); -} - -FILE* read_or_die(const char* fname) { - FILE* f = fopen(fname, "rb"); - if (!f) - err(1, "%s", fname); - return f; -} -FILE* write_or_die(const char* fname) { - FILE* f = fopen(fname, "wb"); - if (!f) - err(1, "%s", fname); - return f; -} -FILE* fopen_or_die(const char* fname, const char* mode) { - FILE* f = fopen(fname, mode); - if (!f) - err(1, "%s (mode '%s')", fname, mode); - return f; -} - -FILE* fopen_chk(const char* fname, const char* mode) { - FILE* f = fopen(fname, mode); - if (!f) - ythrow yexception() << fname << " (mode '" << mode << "'): " << LastSystemErrorText(); - return f; -} - -void fclose_chk(FILE* f, const char* fname) { - if (fclose(f)) - ythrow yexception() << "file " << fname << ": " << LastSystemErrorText(); -} diff --git a/library/cpp/deprecated/fgood/ffb.h b/library/cpp/deprecated/fgood/ffb.h deleted file mode 100644 index ca229eb65a..0000000000 --- a/library/cpp/deprecated/fgood/ffb.h +++ /dev/null @@ -1,264 +0,0 @@ -#pragma once - -#include "fgood.h" - -#include <util/string/util.h> // str_spn -#include <util/string/split.h> // str_spn -#include <util/memory/segmented_string_pool.h> -#include <util/generic/string.h> -#include <util/generic/vector.h> -#include <util/generic/noncopyable.h> - -#include <utility> - -#include <cstdarg> -#include <cstring> - -struct ffb: public TFILEPtr { - ffb() { - } - ffb(FILE* file); - ffb(const char* name, const char* mode) { - open(name, mode); - } - void operator=(FILE* f); // take ownership - void open(const char* name, const char* mode); - int f(const char* c, ...) { - va_list args; - va_start(args, c); - return vfprintf(*this, c, args); - } - void s(const char* c) { - fsput(c, strlen(c)); - } - void b(const void* cc, int n) { - fsput((const char*)cc, n); - } - void B(const void* cc, int N) { - fsput((const char*)cc, N); - } - void c(char c) { - fputc(c); - } - void cbe(wchar16 c) { // big endian utf-16 - fputc(char(c >> 8)); //Hi8 - fputc(char(c & 255)); //Lo8 - } - void sbe(const wchar16* c) { - for (; *c; c++) - cbe(*c); - } - void fclose() { - close(); - } -}; - -// split fields of tab-delimited line of text -// here and below fb actual size must be fb_sz + 1 to allow fb[fb_sz] be zero -int sf(char** fb, char* buf, size_t fb_sz); -int sf(char** fb, char* buf /* fb_sz == 32 */); - -// split fields of char-delimited line of text -// Achtung: delim = ' ' imitates awk: initial separators are skipped, -// repeated seps treated as one, all chars less than ' ' treated as separators. -int sf(char fs, char** fb, char* buf, size_t fb_sz = 32); - -// split fields of string-delimited line of text (fs is NOT a regexp) -// (usually fs is "@@") -int sf(const char* fs, char** fb, char* buf, size_t fb_sz = 32); - -// split fields of char-delimited line of text, set of char-separators is given -// Achtung: repeated seps treated as one, initial seps are skipped -// newlines are NOT ignored. -int sf(const char* seps, char* buf, char** fb, size_t fb_sz = 32); - -inline char* chomp(char* buf) { - char* c = buf + strlen(buf); - if (c > buf && c[-1] == '\n') { - *--c = 0; -#ifdef _win32_ - if (c > buf && c[-1] == '\r') - *--c = 0; -#endif - } - return buf; -} - -inline char* chomp_cr(char* buf) { - char* c = buf + strlen(buf); - if (c > buf && c[-1] == '\n') - *--c = 0; - if (c > buf && c[-1] == '\r') - *--c = 0; - return buf; -} - -class TLineSplitter { -protected: - enum { // Default: Split string by SepStr - SplitByAnySep = 1, // Split string by Sep - NoEmptyFields = 2 // Skip all empty fields between separators - }; - -private: - ui32 Flags; - const str_spn Sep; // collection of separators - const char* SepStr; // pointer exact string to separate by - size_t SepStrLen; // length of separator string - -public: - TLineSplitter(const char* sep, bool noEmpty) - : Flags(noEmpty ? NoEmptyFields : 0) - , Sep(TString(sep, 1).data()) - , SepStr(sep) - , SepStrLen(strlen(sep)) - { - } - TLineSplitter(const str_spn& sep, bool noEmpty = false) - : Flags(SplitByAnySep | (noEmpty ? NoEmptyFields : 0)) - , Sep(sep) - , SepStr(nullptr) - , SepStrLen(1) - { - } - bool AnySep() const { - return Flags & SplitByAnySep; - } - bool SkipEmpty() const { - return Flags & NoEmptyFields; - } - /// Separates string onto tokens - /// Expecting a zero-terminated string - /// By default returns empty fields between sequential separators - void operator()(char* p, TVector<char*>& fields) const; - /// Same, but for const string - fills vector of pairs (pointer, length) - void operator()(const char* p, TVector<std::pair<const char*, size_t>>& fields) const; -}; - -/** - * Use library/cpp/map_text_file/map_tsv_file.h instead. - */ -class TSFReader { - TString Buf; // buffer used for non-'\n'-terminated string and for non-freebsd work - TLineSplitter Split; - TVector<char*> Fields; - size_t NF; // Fields.size() - size_t NR; - - TFILEPtr File; - - bool OpenPipe; // internal flag that turns open() to popen() - - i32 FieldsRequired; // if != -1, != nf, terminate program - -public: - // char separator - // Achtung: delim = ' ' imitates awk: initial separators are skipped, - // all chars less than ' ' treated as separators. - TSFReader(const char* fname = nullptr, char sep = '\t', i32 nf_reqired = -1); - // exact string separator - TSFReader(const char* fname, const char* sep, i32 nf_reqired = -1); - // fully customizable - TSFReader(const char* fname, const TLineSplitter& spl, i32 nf_reqired = -1); - - void Open(const char* fname, i32 nf_reqired = -1, size_t vbufsize = 1u << 21); // use "/dev/stdin" for stdin - void Popen(const char* pname, i32 nf_reqired = -1, size_t vbufsize = 1u << 21); - - bool NextLine(segmented_string_pool* pool = nullptr); - - bool IsOpen() const { - return (FILE*)File != nullptr; - } - bool IsEof() const { - return feof(File); - } - void Close() { - File.close(); - } - void Rewind() { - File.seek(0, SEEK_SET); - } - void Seek(i64 offset, int mode = SEEK_SET) { - File.seek(offset, mode); - } - i64 Tell() const { - return ftell(File); - } - char*& operator[](size_t ind) { - //if (ind >= NF) - // throw yexception("Can't return reference to unexisting field %" PRISZT, ind); - return Fields[ind]; - } - const char* operator[](size_t ind) const { - if (ind >= NF) - return nullptr; - return Fields[ind]; - } - operator int() const { // note: empty input line makes 0 fields - return (int)NF; - } - const char* Name() const { - return File.name().data(); - } - size_t Line() const { - return NR; - } - const TVector<char*>& GetFields() const { - return Fields; - } -}; - -struct prnstr { - char* buf; - int pos; - int asize; - prnstr() - : pos(0) - { - asize = 32; - buf = new char[asize]; - } - explicit prnstr(int asz) - : pos(0) - { - asize = asz; - buf = new char[asize]; - } - int f(const char* c, ...); - int s(const char* c1, const char* c2); - int s(const char* c1, const char* c2, const char* c3); - int s(const char* c, size_t len); - //int s(const char *c); - int s(const char* c) { - return c ? s(c, strlen(c)) : 0; - } - int s(const TString& c); - int s_htmesc(const char* c, bool enc_utf = false); - int s_htmesc_w(const char* c); - int c(char c); - int cu(wchar32 c); //for utf-8 - void restart() { - *buf = 0; - pos = 0; - } - const char* operator~() const { - return buf; - } - int operator+() const { - return pos; - } - ~prnstr() { - delete[] buf; - } - void clear(); - void swap(prnstr& w); -}; - -// functions that terminate program upon failure -FILE* read_or_die(const char* fname); -FILE* write_or_die(const char* fname); -FILE* fopen_or_die(const char* fname, const char* mode); - -// functions that throw upon failure -FILE* fopen_chk(const char* fname, const char* mode); -void fclose_chk(FILE* f, const char* fname_dbg); diff --git a/library/cpp/deprecated/fgood/fgood.cpp b/library/cpp/deprecated/fgood/fgood.cpp deleted file mode 100644 index 5d4725bfae..0000000000 --- a/library/cpp/deprecated/fgood/fgood.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include "fgood.h" - -#include <util/generic/cast.h> -#include <util/string/cast.h> -#include <util/system/fstat.h> - -#ifdef _win32_ -#include <io.h> -#endif - -i64 TFILEPtr::length() const { -#ifdef _win32_ - FHANDLE fd = (FHANDLE)_get_osfhandle(fileno(m_file)); -#else - FHANDLE fd = fileno(m_file); -#endif - i64 rv = GetFileLength(fd); - if (rv < 0) - ythrow yexception() << "TFILEPtr::length() " << Name.data() << ": " << LastSystemErrorText(); - return rv; -} - -FILE* OpenFILEOrFail(const TString& name, const char* mode) { - FILE* res = ::fopen(name.data(), mode); - if (!res) { - ythrow yexception() << "can't open \'" << name << "\' with mode \'" << mode << "\': " << LastSystemErrorText(); - } - return res; -} - -void TFILECloser::Destroy(FILE* file) { - ::fclose(file); -} - -#ifdef _freebsd_ // fgetln -#define getline getline_alt_4test -#endif // _freebsd_ - -bool getline(TFILEPtr& f, TString& s) { - char buf[4096]; - char* buf_ptr; - if (s.capacity() > sizeof(buf)) { - s.resize(s.capacity()); - if ((buf_ptr = fgets(s.begin(), IntegerCast<int>(s.capacity()), f)) == nullptr) - return false; - } else { - if ((buf_ptr = fgets(buf, sizeof(buf), f)) == nullptr) - return false; - } - size_t buf_len = strlen(buf_ptr); - bool line_complete = buf_len && buf_ptr[buf_len - 1] == '\n'; - if (line_complete) - buf_len--; - if (buf_ptr == s.begin()) - s.resize(buf_len); - else - s.AssignNoAlias(buf, buf_len); - if (line_complete) - return true; - while (fgets(buf, sizeof(buf), f)) { - size_t buf_len2 = strlen(buf); - if (buf_len2 && buf[buf_len2 - 1] == '\n') { - buf[buf_len2 - 1] = 0; - s.append(buf, buf_len2 - 1); - return true; - } - s.append(buf, buf_len2); - } - return true; -} diff --git a/library/cpp/deprecated/fgood/fgood.h b/library/cpp/deprecated/fgood/fgood.h deleted file mode 100644 index 0aaf910c0f..0000000000 --- a/library/cpp/deprecated/fgood/fgood.h +++ /dev/null @@ -1,328 +0,0 @@ -#pragma once - -#include <util/system/yassert.h> -#include <util/system/defaults.h> -#include <util/generic/string.h> -#include <util/generic/yexception.h> -#include <util/generic/ptr.h> - -#include "fput.h" - -#include <cstdio> - -#include <fcntl.h> - -#ifdef _unix_ -extern "C" int __ungetc(int, FILE*); -#endif - -#if (!defined(__FreeBSD__) && !defined(__linux__) && !defined(_darwin_) && !defined(_cygwin_)) || defined(_bionic_) -#define feof_unlocked(_stream) feof(_stream) -#define ferror_unlocked(_stream) ferror(_stream) -#endif - -#ifndef _unix_ -#if defined(_MSC_VER) && (_MSC_VER < 1900) -#define getc_unlocked(_stream) (--(_stream)->_cnt >= 0 ? 0xff & *(_stream)->_ptr++ : _filbuf(_stream)) -#define putc_unlocked(_c, _stream) (--(_stream)->_cnt >= 0 ? 0xff & (*(_stream)->_ptr++ = (char)(_c)) : _flsbuf((_c), (_stream))) -#else -#define getc_unlocked(_stream) getc(_stream) -#define putc_unlocked(_c, _stream) putc(_c, _stream) -#endif -#endif - -inline bool fgood(FILE* f) { - return !feof_unlocked(f) && !ferror_unlocked(f); -} - -#ifdef _win32_ -// These functions will work only with static MSVC runtime linkage. For dynamic linkage, -// fseeki64.c and ftelli64.c from CRT sources should be included in project -extern "C" int __cdecl _fseeki64(FILE*, __int64, int); -extern "C" __int64 __cdecl _ftelli64(FILE*); - -inline i64 ftello(FILE* stream) { - return _ftelli64(stream); -} - -inline int fseeko(FILE* stream, i64 offset, int origin) { - return _fseeki64(stream, offset, origin); -} -#endif - -class TFILEPtr { -private: - enum { SHOULD_CLOSE = 1, - IS_PIPE = 2 }; - FILE* m_file; - int m_Flags; - TString Name; - -public: - TFILEPtr() noexcept { - m_file = nullptr; - m_Flags = 0; - } - TFILEPtr(const TString& name, const char* mode) { - m_file = nullptr; - m_Flags = 0; - open(name, mode); - } - TFILEPtr(const TFILEPtr& src) noexcept { - m_file = src.m_file; - m_Flags = 0; - } - TFILEPtr& operator=(const TFILEPtr& src) { - if (src.m_file != m_file) { - close(); - m_file = src.m_file; - m_Flags = 0; - } - return *this; - } - explicit TFILEPtr(FILE* f) noexcept { // take ownership - m_file = f; - m_Flags = SHOULD_CLOSE; - } - TFILEPtr& operator=(FILE* f) { // take ownership - if (f != m_file) { - close(); - m_file = f; - m_Flags = SHOULD_CLOSE; - } - return *this; - } - const TString& name() const { - return Name; - } - operator FILE*() const noexcept { - return m_file; - } - FILE* operator->() const noexcept { - return m_file; - } - bool operator!() const noexcept { - return m_file == nullptr; - } - bool operator!=(FILE* f) const noexcept { - return m_file != f; - } - bool operator==(FILE* f) const noexcept { - return m_file == f; - } - ~TFILEPtr() { - close(); - } - void Y_PRINTF_FORMAT(2, 3) check(const char* message, ...) const { - if (Y_UNLIKELY(!fgood(m_file))) { - va_list args; - va_start(args, message); - char buf[512]; - vsnprintf(buf, 512, message, args); - // XXX: errno is undefined here - ythrow yexception() << buf << ": " << LastSystemErrorText() << ", " << Name.data() << " at offset " << (i64)ftell(); - } - } - TFILEPtr& assign(FILE* f, const char* name = nullptr) { // take ownership and have a name - *this = f; - if (name) - Name = name; - return *this; - } - void open(const TString& name, const char* mode) { - Y_ASSERT(!name.empty()); - Y_ASSERT(m_file == nullptr); - m_file = ::fopen(name.data(), mode); - if (!m_file) - ythrow yexception() << "can't open \'" << name << "\' with mode \'" << mode << "\': " << LastSystemErrorText(); - m_Flags = SHOULD_CLOSE; - Name = name; - } - void popen(const TString& command, const char* mode) { - Y_ASSERT(!command.empty()); - Y_ASSERT(m_file == nullptr); - m_file = ::popen(command.data(), mode); - if (!m_file) - ythrow yexception() << "can't execute \'" << command << "\' with mode \'" << mode << "\': " << LastSystemErrorText(); - m_Flags = IS_PIPE | SHOULD_CLOSE; - Name = command; - } - void close() { - if (m_file != nullptr && (m_Flags & SHOULD_CLOSE)) { - if ((m_Flags & IS_PIPE) ? ::pclose(m_file) : ::fclose(m_file)) { - m_file = nullptr; - m_Flags = 0; - if (!UncaughtException()) - ythrow yexception() << "can't close file " << Name.data() << ": " << LastSystemErrorText(); - } - } - m_file = nullptr; - m_Flags = 0; - Name.clear(); - } - size_t write(const void* buffer, size_t size, size_t count) const { - Y_ASSERT(m_file != nullptr); - size_t r = ::fwrite(buffer, size, count, m_file); - check("can't write %lu bytes", (unsigned long)size * count); - return r; - } - size_t read(void* buffer, size_t size, size_t count) const { - Y_ASSERT(m_file != nullptr); - size_t r = ::fread(buffer, size, count, m_file); - if (ferror_unlocked(m_file)) - ythrow yexception() << "can't read " << (unsigned long)size * count << " bytes: " << LastSystemErrorText() << ", " << Name.data() << " at offset " << (i64)ftell(); - return r; - } - char* fgets(char* buffer, int size) const { - Y_ASSERT(m_file != nullptr); - char* r = ::fgets(buffer, size, m_file); - if (ferror_unlocked(m_file)) - ythrow yexception() << "can't read string of maximum size " << size << ": " << LastSystemErrorText() << ", " << Name.data() << " at offset " << (i64)ftell(); - return r; - } - void Y_PRINTF_FORMAT(2, 3) fprintf(const char* format, ...) { - Y_ASSERT(m_file != nullptr); - va_list args; - va_start(args, format); - vfprintf(m_file, format, args); - check("can't write"); - } - void seek(i64 offset, int origin) const { - Y_ASSERT(m_file != nullptr); -#if defined(_unix_) || defined(_win32_) - if (fseeko(m_file, offset, origin) != 0) -#else - Y_ASSERT(offset == (i64)(i32)offset); - if (::fseek(m_file, (long)offset, origin) != 0) -#endif - ythrow yexception() << "can't seek " << Name.data() << " by " << offset << ": " << LastSystemErrorText(); - } - i64 length() const; // uses various system headers -> in fileptr.cpp - - void setDirect() const { -#if !defined(_win_) && !defined(_darwin_) - if (!m_file) - ythrow yexception() << "file not open"; - if (fcntl(fileno(m_file), F_SETFL, O_DIRECT) == -1) - ythrow yexception() << "Cannot set O_DIRECT flag"; -#endif - } - - // for convenience - - i64 ftell() const noexcept { -#if defined(_unix_) || defined(_win32_) - return ftello(m_file); -#else - return ftell(m_file); -#endif - } - bool eof() const noexcept { - Y_ASSERT(m_file != nullptr); - return feof_unlocked(m_file) != 0; - } - int fputc(int c) { - Y_ASSERT(m_file != nullptr); - return putc_unlocked(c, m_file); - } - size_t fputs(const char* buffer) const { - return write(buffer, strlen(buffer), 1); - } - int fgetc() { - Y_ASSERT(m_file != nullptr); - return getc_unlocked(m_file); - } - int ungetc(int c) { - Y_ASSERT(m_file != nullptr); - return ::ungetc(c, m_file); - } - template <class T> - size_t fput(const T& a) { - Y_ASSERT(m_file != nullptr); - return ::fput(m_file, a); - } - template <class T> - size_t fget(T& a) { - Y_ASSERT(m_file != nullptr); - return ::fget(m_file, a); - } - size_t fsput(const char* s, size_t l) { - Y_ASSERT(m_file != nullptr); - return ::fsput(m_file, s, l); - } - size_t fsget(char* s, size_t l) { - Y_ASSERT(m_file != nullptr); - return ::fsget(m_file, s, l); - } - - void fflush() { - ::fflush(m_file); - } - - /* This block contains some TFile/TStream - compatible names */ - size_t Read(void* bufferIn, size_t numBytes) { - size_t r = fsget((char*)bufferIn, numBytes); - if (Y_UNLIKELY(ferror_unlocked(m_file))) - ythrow yexception() << "can't read " << numBytes << " bytes: " << LastSystemErrorText() << ", " << Name << " at offset " << (i64)ftell(); - return r; - } - void Write(const void* buffer, size_t numBytes) { - write(buffer, 1, numBytes); - } - i64 Seek(i64 offset, int origin /*SeekDir*/) { - seek(offset, origin); - return ftell(); - } - i64 GetPosition() const noexcept { - return ftell(); - } - i64 GetLength() const noexcept { - return length(); - } - bool ReadLine(TString& st); - - /* Similar to TAutoPtr::Release - return pointer and forget about it. */ - FILE* Release() noexcept { - FILE* result = m_file; - m_file = nullptr; - m_Flags = 0; - Name.clear(); - return result; - } -}; - -inline void fclose(TFILEPtr& F) { - F.close(); -} - -inline void fseek(const TFILEPtr& F, i64 offset, int whence) { - F.seek(offset, whence); -} - -#ifdef _freebsd_ // fgetln -inline bool getline(TFILEPtr& f, TString& s) { - size_t len; - char* buf = fgetln(f, &len); - if (!buf) - return false; - if (len && buf[len - 1] == '\n') - len--; - s.AssignNoAlias(buf, len); - return true; -} -#else -bool getline(TFILEPtr& f, TString& s); -#endif //_freebsd_ - -inline bool TFILEPtr::ReadLine(TString& st) { - return getline(*this, st); -} - -FILE* OpenFILEOrFail(const TString& name, const char* mode); - -//Should be used with THolder -struct TFILECloser { - static void Destroy(FILE* file); -}; - -using TFILEHolder = THolder<FILE, TFILECloser>; diff --git a/library/cpp/deprecated/fgood/fput.h b/library/cpp/deprecated/fgood/fput.h deleted file mode 100644 index 690b06332d..0000000000 --- a/library/cpp/deprecated/fgood/fput.h +++ /dev/null @@ -1,79 +0,0 @@ -#pragma once - -#include <util/system/defaults.h> -#include <util/system/valgrind.h> - -#include <cstdio> - -#ifdef __FreeBSD__ -#include <cstring> - -template <class T> -Y_FORCE_INLINE size_t fput(FILE* F, const T& a) { - if (Y_LIKELY(F->_w >= int(sizeof(a)))) { - memcpy(F->_p, &a, sizeof(a)); - F->_p += sizeof(a); - F->_w -= sizeof(a); - return 1; - } else { - return fwrite(&a, sizeof(a), 1, F); - } -} - -template <class T> -Y_FORCE_INLINE size_t fget(FILE* F, T& a) { - if (Y_LIKELY(F->_r >= int(sizeof(a)))) { - memcpy(&a, F->_p, sizeof(a)); - F->_p += sizeof(a); - F->_r -= sizeof(a); - return 1; - } else { - return fread(&a, sizeof(a), 1, F); - } -} - -inline size_t fsput(FILE* F, const char* s, size_t l) { - VALGRIND_CHECK_READABLE(s, l); - - if ((size_t)F->_w >= l) { - memcpy(F->_p, s, l); - F->_p += l; - F->_w -= l; - return l; - } else { - return fwrite(s, 1, l, F); - } -} - -inline size_t fsget(FILE* F, char* s, size_t l) { - if ((size_t)F->_r >= l) { - memcpy(s, F->_p, l); - F->_p += l; - F->_r -= l; - return l; - } else { - return fread(s, 1, l, F); - } -} -#else -template <class T> -Y_FORCE_INLINE size_t fput(FILE* F, const T& a) { - return fwrite(&a, sizeof(a), 1, F); -} - -template <class T> -Y_FORCE_INLINE size_t fget(FILE* F, T& a) { - return fread(&a, sizeof(a), 1, F); -} - -inline size_t fsput(FILE* F, const char* s, size_t l) { -#ifdef WITH_VALGRIND - VALGRIND_CHECK_READABLE(s, l); -#endif - return fwrite(s, 1, l, F); -} - -inline size_t fsget(FILE* F, char* s, size_t l) { - return fread(s, 1, l, F); -} -#endif diff --git a/library/cpp/deprecated/fgood/ya.make b/library/cpp/deprecated/fgood/ya.make deleted file mode 100644 index 2394f9ad7a..0000000000 --- a/library/cpp/deprecated/fgood/ya.make +++ /dev/null @@ -1,8 +0,0 @@ -LIBRARY() - -SRCS( - ffb.cpp - fgood.cpp -) - -END() diff --git a/library/cpp/deprecated/mapped_file/mapped_file.cpp b/library/cpp/deprecated/mapped_file/mapped_file.cpp deleted file mode 100644 index b0e4511299..0000000000 --- a/library/cpp/deprecated/mapped_file/mapped_file.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include "mapped_file.h" - -#include <util/generic/yexception.h> -#include <util/system/defaults.h> -#include <util/system/hi_lo.h> -#include <util/system/filemap.h> - -TMappedFile::TMappedFile(TFileMap* map, const char* dbgName) { - Map_ = map; - i64 len = Map_->Length(); - if (Hi32(len) != 0 && sizeof(size_t) <= sizeof(ui32)) - ythrow yexception() << "File '" << dbgName << "' mapping error: " << len << " too large"; - - Map_->Map(0, static_cast<size_t>(len)); -} - -TMappedFile::TMappedFile(const TFile& file, TFileMap::EOpenMode om, const char* dbgName) - : Map_(nullptr) -{ - init(file, om, dbgName); -} - -void TMappedFile::precharge(size_t off, size_t size) const { - if (!Map_) - return; - - Map_->Precharge(off, size); -} - -void TMappedFile::init(const TString& name) { - THolder<TFileMap> map(new TFileMap(name)); - TMappedFile newFile(map.Get(), name.data()); - Y_UNUSED(map.Release()); - newFile.swap(*this); - newFile.term(); -} - -void TMappedFile::init(const TString& name, size_t length, TFileMap::EOpenMode om) { - THolder<TFileMap> map(new TFileMap(name, length, om)); - TMappedFile newFile(map.Get(), name.data()); - Y_UNUSED(map.Release()); - newFile.swap(*this); - newFile.term(); -} - -void TMappedFile::init(const TFile& file, TFileMap::EOpenMode om, const char* dbgName) { - THolder<TFileMap> map(new TFileMap(file, om)); - TMappedFile newFile(map.Get(), dbgName); - Y_UNUSED(map.Release()); - newFile.swap(*this); - newFile.term(); -} - -void TMappedFile::init(const TString& name, TFileMap::EOpenMode om) { - THolder<TFileMap> map(new TFileMap(name, om)); - TMappedFile newFile(map.Get(), name.data()); - Y_UNUSED(map.Release()); - newFile.swap(*this); - newFile.term(); -} - -void TMappedFile::flush() { - Map_->Flush(); -} diff --git a/library/cpp/deprecated/mapped_file/ya.make b/library/cpp/deprecated/mapped_file/ya.make deleted file mode 100644 index 309341f1da..0000000000 --- a/library/cpp/deprecated/mapped_file/ya.make +++ /dev/null @@ -1,7 +0,0 @@ -LIBRARY() - -SRCS( - mapped_file.cpp -) - -END() diff --git a/library/cpp/eventlog/common.h b/library/cpp/eventlog/common.h deleted file mode 100644 index 75c512c13e..0000000000 --- a/library/cpp/eventlog/common.h +++ /dev/null @@ -1,10 +0,0 @@ -#pragma once - -template <class T> -class TPacketInputStream { -public: - virtual bool Avail() const = 0; - virtual T operator*() const = 0; - virtual bool Next() = 0; - virtual ~TPacketInputStream() = default; -}; diff --git a/library/cpp/eventlog/evdecoder.cpp b/library/cpp/eventlog/evdecoder.cpp deleted file mode 100644 index e4413a1b0e..0000000000 --- a/library/cpp/eventlog/evdecoder.cpp +++ /dev/null @@ -1,112 +0,0 @@ -#include <util/memory/tempbuf.h> -#include <util/string/cast.h> -#include <util/stream/output.h> - -#include "evdecoder.h" -#include "logparser.h" - -static const char* const UNKNOWN_EVENT_CLASS = "Unknown event class"; - -static inline void LogError(ui64 frameAddr, const char* msg, bool strict) { - if (!strict) { - Cerr << "EventDecoder warning @" << frameAddr << ": " << msg << Endl; - } else { - ythrow yexception() << "EventDecoder error @" << frameAddr << ": " << msg; - } -} - -static inline bool SkipData(IInputStream& s, size_t amount) { - return (amount == s.Skip(amount)); -} - -// There are 2 log fomats: the one, that allows event skip without event decode (it has stored event length) -// and another, that requires each event decode just to seek over stream. needRead == true means the latter format. -static inline THolder<TEvent> DoDecodeEvent(IInputStream& s, const TEventFilter* const filter, const bool needRead, IEventFactory* fac) { - TEventTimestamp ts; - TEventClass c; - THolder<TEvent> e; - - ::Load(&s, ts); - ::Load(&s, c); - - bool needReturn = false; - - if (!filter || filter->EventAllowed(c)) { - needReturn = true; - } - - if (needRead || needReturn) { - e.Reset(fac->CreateLogEvent(c)); - - if (!!e) { - e->Timestamp = ts; - e->Load(s); - } else if (needReturn) { - e.Reset(new TUnknownEvent(ts, c)); - } - - if (!needReturn) { - e.Reset(nullptr); - } - } - - return e; -} - -THolder<TEvent> DecodeFramed(IInputStream& inp, ui64 frameAddr, const TEventFilter* const filter, IEventFactory* fac, bool strict) { - ui32 len; - ::Load(&inp, len); - - if (len < sizeof(ui32)) { - ythrow TEventDecoderError() << "invalid event length"; - } - - TLengthLimitedInput s(&inp, len - sizeof(ui32)); - - try { - THolder<TEvent> e = DoDecodeEvent(s, filter, false, fac); - if (!!e) { - if (!s.Left()) { - return e; - } else if (e->Class == 0) { - if (!SkipData(s, s.Left())) { - ythrow TEventDecoderError() << "cannot skip bad event"; - } - - return e; - } - - LogError(frameAddr, "Event is not fully read", strict); - } - } catch (const TLoadEOF&) { - if (s.Left()) { - throw; - } - - LogError(frameAddr, "Unexpected event end", strict); - } - - if (!SkipData(s, s.Left())) { - ythrow TEventDecoderError() << "cannot skip bad event"; - } - - return nullptr; -} - -THolder<TEvent> DecodeEvent(IInputStream& s, bool framed, ui64 frameAddr, const TEventFilter* const filter, IEventFactory* fac, bool strict) { - try { - if (framed) { - return DecodeFramed(s, frameAddr, filter, fac, strict); - } else { - THolder<TEvent> e = DoDecodeEvent(s, filter, true, fac); - // e(0) means event, skipped by filter. Not an error. - if (!!e && !e->Class) { - ythrow TEventDecoderError() << UNKNOWN_EVENT_CLASS; - } - - return e; - } - } catch (const TLoadEOF&) { - ythrow TEventDecoderError() << "unexpected frame end"; - } -} diff --git a/library/cpp/eventlog/evdecoder.h b/library/cpp/eventlog/evdecoder.h deleted file mode 100644 index eedfc82174..0000000000 --- a/library/cpp/eventlog/evdecoder.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include <util/generic/yexception.h> -#include <util/generic/ptr.h> - -#include "eventlog.h" - -class TEvent; -class IInputStream; -class TEventFilter; - -struct TEventDecoderError: public yexception { -}; - -THolder<TEvent> DecodeEvent(IInputStream& s, bool framed, ui64 frameAddr, const TEventFilter* const filter, IEventFactory* fac, bool strict = false); -bool AcceptableContent(TEventLogFormat); diff --git a/library/cpp/eventlog/event_field_output.cpp b/library/cpp/eventlog/event_field_output.cpp deleted file mode 100644 index f9d98dac9d..0000000000 --- a/library/cpp/eventlog/event_field_output.cpp +++ /dev/null @@ -1,68 +0,0 @@ -#include "event_field_output.h" - -#include <util/string/split.h> - -namespace { - TString MakeSeparators(EFieldOutputFlags flags) { - TString res; - res.reserve(3); - - if (flags & EFieldOutputFlag::EscapeTab) { - res.append('\t'); - } - if (flags & EFieldOutputFlag::EscapeNewLine) { - res.append('\n'); - res.append('\r'); - } - if (flags & EFieldOutputFlag::EscapeBackSlash) { - res.append('\\'); - } - - return res; - } -} - -TEventFieldOutput::TEventFieldOutput(IOutputStream& output, EFieldOutputFlags flags) - : Output(output) - , Flags(flags) - , Separators(MakeSeparators(flags)) -{ -} - -IOutputStream& TEventFieldOutput::GetOutputStream() { - return Output; -} - -EFieldOutputFlags TEventFieldOutput::GetFlags() const { - return Flags; -} - -void TEventFieldOutput::DoWrite(const void* buf, size_t len) { - if (!Flags) { - Output.Write(buf, len); - return; - } - - TStringBuf chunk{static_cast<const char*>(buf), len}; - - for (const auto part : StringSplitter(chunk).SplitBySet(Separators.data())) { - TStringBuf token = part.Token(); - TStringBuf delim = part.Delim(); - - if (!token.empty()) { - Output.Write(token); - } - if ("\n" == delim) { - Output.Write(TStringBuf("\\n")); - } else if ("\r" == delim) { - Output.Write(TStringBuf("\\r")); - } else if ("\t" == delim) { - Output.Write(TStringBuf("\\t")); - } else if ("\\" == delim) { - Output.Write(TStringBuf("\\\\")); - } else { - Y_ASSERT(delim.empty()); - } - } -} - diff --git a/library/cpp/eventlog/event_field_output.h b/library/cpp/eventlog/event_field_output.h deleted file mode 100644 index ed9db0ae16..0000000000 --- a/library/cpp/eventlog/event_field_output.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include <util/stream/output.h> -#include <util/generic/flags.h> - -enum class EFieldOutputFlag { - EscapeTab = 0x1, // escape \t in field value - EscapeNewLine = 0x2, // escape \n in field value - EscapeBackSlash = 0x4 // escape \ in field value -}; - -Y_DECLARE_FLAGS(EFieldOutputFlags, EFieldOutputFlag); -Y_DECLARE_OPERATORS_FOR_FLAGS(EFieldOutputFlags); - -class TEventFieldOutput: public IOutputStream { -public: - TEventFieldOutput(IOutputStream& output, EFieldOutputFlags flags); - - IOutputStream& GetOutputStream(); - EFieldOutputFlags GetFlags() const; - -protected: - void DoWrite(const void* buf, size_t len) override; - -private: - IOutputStream& Output; - EFieldOutputFlags Flags; - TString Separators; -}; diff --git a/library/cpp/eventlog/event_field_printer.cpp b/library/cpp/eventlog/event_field_printer.cpp deleted file mode 100644 index 29c6b4b661..0000000000 --- a/library/cpp/eventlog/event_field_printer.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "event_field_printer.h" - -#include <library/cpp/protobuf/json/proto2json.h> - -namespace { - - const NProtobufJson::TProto2JsonConfig PROTO_2_JSON_CONFIG = NProtobufJson::TProto2JsonConfig() - .SetMissingRepeatedKeyMode(NProtobufJson::TProto2JsonConfig::MissingKeyDefault) - .AddStringTransform(MakeIntrusive<NProtobufJson::TBase64EncodeBytesTransform>()); - -} // namespace - -TEventProtobufMessageFieldPrinter::TEventProtobufMessageFieldPrinter(EProtobufMessageFieldPrintMode mode) - : Mode(mode) -{} - -template <> -void TEventProtobufMessageFieldPrinter::PrintProtobufMessageFieldToOutput<google::protobuf::Message, false>(const google::protobuf::Message& field, TEventFieldOutput& output) { - switch (Mode) { - case EProtobufMessageFieldPrintMode::DEFAULT: - case EProtobufMessageFieldPrintMode::JSON: { - // Do not use field.PrintJSON() here: IGNIETFERRO-2002 - NProtobufJson::Proto2Json(field, output, PROTO_2_JSON_CONFIG); - break; - } - } -} diff --git a/library/cpp/eventlog/event_field_printer.h b/library/cpp/eventlog/event_field_printer.h deleted file mode 100644 index 835e8f4a85..0000000000 --- a/library/cpp/eventlog/event_field_printer.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include "event_field_output.h" - -#include <google/protobuf/message.h> - -// NB: For historical reasons print code for all primitive types/repeated fields/etc generated by https://a.yandex-team.ru/arc/trunk/arcadia/tools/event2cpp - -enum class EProtobufMessageFieldPrintMode { - // Use <TEventProtobufMessageFieldType>::Print method for fields that has it - // Print json for other fields - DEFAULT = 0, - - JSON = 1, -}; - -class TEventProtobufMessageFieldPrinter { -public: - explicit TEventProtobufMessageFieldPrinter(EProtobufMessageFieldPrintMode mode); - - template <typename TEventProtobufMessageFieldType, bool HasPrintFunction> - void PrintProtobufMessageFieldToOutput(const TEventProtobufMessageFieldType& field, TEventFieldOutput& output) { - if constexpr (HasPrintFunction) { - if (Mode == EProtobufMessageFieldPrintMode::DEFAULT) { - field.Print(output.GetOutputStream(), output.GetFlags()); - return; - } - } - - PrintProtobufMessageFieldToOutput<google::protobuf::Message, false>(field, output); - } - - template <> - void PrintProtobufMessageFieldToOutput<google::protobuf::Message, false>(const google::protobuf::Message& field, TEventFieldOutput& output); - -private: - EProtobufMessageFieldPrintMode Mode; -}; diff --git a/library/cpp/eventlog/eventlog.cpp b/library/cpp/eventlog/eventlog.cpp deleted file mode 100644 index 458a632b4a..0000000000 --- a/library/cpp/eventlog/eventlog.cpp +++ /dev/null @@ -1,554 +0,0 @@ -#include <util/datetime/base.h> -#include <util/stream/zlib.h> -#include <util/stream/length.h> -#include <util/generic/buffer.h> -#include <util/generic/yexception.h> -#include <util/digest/murmur.h> -#include <util/generic/singleton.h> -#include <util/generic/function.h> -#include <util/stream/output.h> -#include <util/stream/format.h> -#include <util/stream/null.h> - -#include <google/protobuf/messagext.h> - -#include "eventlog.h" -#include "events_extension.h" -#include "evdecoder.h" -#include "logparser.h" -#include <library/cpp/eventlog/proto/internal.pb.h> - -#include <library/cpp/json/json_writer.h> -#include <library/cpp/protobuf/json/proto2json.h> - - -TAtomic eventlogFrameCounter = 0; - -namespace { - - const NProtobufJson::TProto2JsonConfig PROTO_2_JSON_CONFIG = NProtobufJson::TProto2JsonConfig() - .SetMissingRepeatedKeyMode(NProtobufJson::TProto2JsonConfig::MissingKeyDefault) - .AddStringTransform(MakeIntrusive<NProtobufJson::TBase64EncodeBytesTransform>()); - - ui32 GenerateFrameId() { - return ui32(AtomicAdd(eventlogFrameCounter, 1)); - } - - inline const NProtoBuf::Message* UnknownEventMessage() { - return Singleton<NEventLogInternal::TUnknownEvent>(); - } - -} // namespace - -void TEvent::Print(IOutputStream& out, const TOutputOptions& options, const TEventState& eventState) const { - if (options.OutputFormat == TOutputFormat::TabSeparatedRaw) { - PrintHeader(out, options, eventState); - DoPrint(out, {}); - } else if (options.OutputFormat == TOutputFormat::TabSeparated) { - PrintHeader(out, options, eventState); - DoPrint( - out, - EFieldOutputFlags{} | EFieldOutputFlag::EscapeNewLine | EFieldOutputFlag::EscapeBackSlash); - } else if (options.OutputFormat == TOutputFormat::Json) { - NJson::TJsonWriterConfig jsonWriterConfig; - jsonWriterConfig.FormatOutput = 0; - NJson::TJsonWriter jsonWriter(&out, jsonWriterConfig); - - jsonWriter.OpenMap(); - PrintJsonHeader(jsonWriter); - DoPrintJson(jsonWriter); - jsonWriter.CloseMap(); - } -} - -void TEvent::PrintHeader(IOutputStream& out, const TOutputOptions& options, const TEventState& eventState) const { - if (options.HumanReadable) { - out << TInstant::MicroSeconds(Timestamp).ToString() << "\t"; - if (Timestamp >= eventState.FrameStartTime) - out << "+" << HumanReadable(TDuration::MicroSeconds(Timestamp - eventState.FrameStartTime)); - else // a bug somewhere? anyway, let's handle it in a nice fashion - out << "-" << HumanReadable(TDuration::MicroSeconds(eventState.FrameStartTime - Timestamp)); - - if (Timestamp >= eventState.PrevEventTime) - out << " (+" << HumanReadable(TDuration::MicroSeconds(Timestamp - eventState.PrevEventTime)) << ")"; - // else: these events are async and out-of-order, relative time diff makes no sense, skip it - - out << "\tF# " << FrameId << '\t'; - } else { - out << static_cast<TEventTimestamp>(Timestamp); - out << '\t' << FrameId << '\t'; - } -} - -void TEvent::PrintJsonHeader(NJson::TJsonWriter& jsonWriter) const { - jsonWriter.Write("Timestamp", Timestamp); - jsonWriter.Write("FrameId", FrameId); -} - -class TProtobufEvent: public TEvent { -public: - TProtobufEvent(TEventTimestamp t, size_t eventId, const NProtoBuf::Message& msg) - : TEvent(eventId, t) - , Message_(&msg) - , EventFactory_(NProtoBuf::TEventFactory::Instance()) - { - } - - TProtobufEvent() - : TEvent(0, 0) - , EventFactory_(NProtoBuf::TEventFactory::Instance()) - { - } - - explicit TProtobufEvent(ui32 id, NProtoBuf::TEventFactory* eventFactory = NProtoBuf::TEventFactory::Instance()) - : TEvent(id, 0) - , EventFactory_(eventFactory) - { - InnerMsg_.Reset(EventFactory_->CreateEvent(Class)); - Message_ = InnerMsg_.Get(); - } - - ui32 Id() const { - return Class; - } - - void Load(IInputStream& in) override { - if (!!InnerMsg_) { - InnerMsg_->ParseFromArcadiaStream(&in); - } else { - TransferData(&in, &Cnull); - } - } - - void Save(IOutputStream& out) const override { - Message_->SerializeToArcadiaStream(&out); - } - - void SaveToBuffer(TBufferOutput& buf) const override { - size_t messageSize = Message_->ByteSize(); - size_t before = buf.Buffer().Size(); - buf.Buffer().Advance(messageSize); - Y_PROTOBUF_SUPPRESS_NODISCARD Message_->SerializeToArray(buf.Buffer().Data() + before, messageSize); - } - - TStringBuf GetName() const override { - return EventFactory_->NameById(Id()); - } - -private: - void DoPrint(IOutputStream& out, EFieldOutputFlags flags) const override { - EventFactory_->PrintEvent(Id(), Message_, out, flags); - } - void DoPrintJson(NJson::TJsonWriter& jsonWriter) const override { - jsonWriter.OpenMap("EventBody"); - jsonWriter.Write("Type", GetName()); - - jsonWriter.Write("Fields"); - NProtobufJson::Proto2Json(*GetProto(), jsonWriter, PROTO_2_JSON_CONFIG); - - jsonWriter.CloseMap(); - } - - const NProtoBuf::Message* GetProto() const override { - if (Message_) { - return Message_; - } - - return UnknownEventMessage(); - } - -private: - const NProtoBuf::Message* Message_ = nullptr; - NProtoBuf::TEventFactory* EventFactory_; - THolder<NProtoBuf::Message> InnerMsg_; - - friend class TEventLogFrame; -}; - -void TEventLogFrame::LogProtobufEvent(size_t eventId, const NProtoBuf::Message& ev) { - TProtobufEvent event(Now().MicroSeconds(), eventId, ev); - - LogEventImpl(event); -} - -void TEventLogFrame::LogProtobufEvent(TEventTimestamp timestamp, size_t eventId, const NProtoBuf::Message& ev) { - TProtobufEvent event(timestamp, eventId, ev); - - LogEventImpl(event); -} - -template <> -void TEventLogFrame::DebugDump(const TProtobufEvent& ev) { - static TMutex lock; - - with_lock (lock) { - Cerr << ev.Timestamp << "\t" << ev.GetName() << "\t"; - ev.GetProto()->PrintJSON(Cerr); - Cerr << Endl; - } -} - -#pragma pack(push, 1) -struct TFrameHeaderData { - char SyncField[COMPRESSED_LOG_FRAME_SYNC_DATA.size()]; - TCompressedFrameBaseHeader Header; - TCompressedFrameHeader2 HeaderEx; -}; -#pragma pack(pop) - -TEventLogFrame::TEventLogFrame(IEventLog& parentLog, bool needAlwaysSafeAdd, TWriteFrameCallbackPtr writeFrameCallback) - : EvLog_(parentLog.HasNullBackend() ? nullptr : &parentLog) - , NeedAlwaysSafeAdd_(needAlwaysSafeAdd) - , ForceDump_(false) - , WriteFrameCallback_(std::move(writeFrameCallback)) -{ - DoInit(); -} - -TEventLogFrame::TEventLogFrame(IEventLog* parentLog, bool needAlwaysSafeAdd, TWriteFrameCallbackPtr writeFrameCallback) - : EvLog_(parentLog) - , NeedAlwaysSafeAdd_(needAlwaysSafeAdd) - , ForceDump_(false) - , WriteFrameCallback_(std::move(writeFrameCallback)) -{ - if (EvLog_ && EvLog_->HasNullBackend()) { - EvLog_ = nullptr; - } - - DoInit(); -} - -TEventLogFrame::TEventLogFrame(bool needAlwaysSafeAdd, TWriteFrameCallbackPtr writeFrameCallback) - : EvLog_(nullptr) - , NeedAlwaysSafeAdd_(needAlwaysSafeAdd) - , ForceDump_(false) - , WriteFrameCallback_(std::move(writeFrameCallback)) -{ - DoInit(); -} - -void TEventLogFrame::Flush() { - if (EvLog_ == nullptr) - return; - - TBuffer& buf = Buf_.Buffer(); - - if (buf.Empty()) { - return; - } - - EvLog_->WriteFrame(buf, StartTimestamp_, EndTimestamp_, WriteFrameCallback_, std::move(MetaFlags_)); - - DoInit(); - - return; -} - -void TEventLogFrame::SafeFlush() { - TGuard<TMutex> g(Mtx_); - Flush(); -} - -void TEventLogFrame::AddEvent(TEventTimestamp timestamp) { - if (timestamp < StartTimestamp_) { - StartTimestamp_ = timestamp; - } - - if (timestamp > EndTimestamp_) { - EndTimestamp_ = timestamp; - } -} - -void TEventLogFrame::DoInit() { - Buf_.Buffer().Clear(); - - StartTimestamp_ = (TEventTimestamp)-1; - EndTimestamp_ = 0; -} - -void TEventLogFrame::VisitEvents(ILogFrameEventVisitor& visitor, IEventFactory* eventFactory) { - const auto doVisit = [this, &visitor, eventFactory]() { - TBuffer& buf = Buf_.Buffer(); - - TBufferInput bufferInput(buf); - TLengthLimitedInput limitedInput(&bufferInput, buf.size()); - - TEventFilter EventFilter(false); - - while (limitedInput.Left()) { - THolder<TEvent> event = DecodeEvent(limitedInput, true, 0, &EventFilter, eventFactory); - - visitor.Visit(*event); - } - }; - if (NeedAlwaysSafeAdd_) { - TGuard<TMutex> g(Mtx_); - doVisit(); - } else { - doVisit(); - } -} - -TSelfFlushLogFrame::TSelfFlushLogFrame(IEventLog& parentLog, bool needAlwaysSafeAdd, TWriteFrameCallbackPtr writeFrameCallback) - : TEventLogFrame(parentLog, needAlwaysSafeAdd, std::move(writeFrameCallback)) -{ -} - -TSelfFlushLogFrame::TSelfFlushLogFrame(IEventLog* parentLog, bool needAlwaysSafeAdd, TWriteFrameCallbackPtr writeFrameCallback) - : TEventLogFrame(parentLog, needAlwaysSafeAdd, std::move(writeFrameCallback)) -{ -} - -TSelfFlushLogFrame::TSelfFlushLogFrame(bool needAlwaysSafeAdd, TWriteFrameCallbackPtr writeFrameCallback) - : TEventLogFrame(needAlwaysSafeAdd, std::move(writeFrameCallback)) -{ -} - -TSelfFlushLogFrame::~TSelfFlushLogFrame() { - try { - Flush(); - } catch (...) { - } -} - -IEventLog::~IEventLog() { -} - -static THolder<TLogBackend> ConstructBackend(const TString& fileName, const TEventLogBackendOptions& backendOpts) { - try { - THolder<TLogBackend> backend; - if (backendOpts.UseSyncPageCacheBackend) { - backend = MakeHolder<TSyncPageCacheFileLogBackend>(fileName, backendOpts.SyncPageCacheBackendBufferSize, backendOpts.SyncPageCacheBackendMaxPendingSize); - } else { - backend = MakeHolder<TFileLogBackend>(fileName); - } - return MakeHolder<TReopenLogBackend>(std::move(backend)); - } catch (...) { - Cdbg << "Warning: Cannot open event log '" << fileName << "': " << CurrentExceptionMessage() << "." << Endl; - } - - return MakeHolder<TNullLogBackend>(); -} - -TEventLog::TEventLog(const TString& fileName, TEventLogFormat contentFormat, const TEventLogBackendOptions& backendOpts, TMaybe<TEventLogFormat> logFormat) - : Log_(ConstructBackend(fileName, backendOpts)) - , ContentFormat_(contentFormat) - , LogFormat_(logFormat.Defined() ? *logFormat : COMPRESSED_LOG_FORMAT_V4) - , HasNullBackend_(Log_.IsNullLog()) - , Lz4hcCodec_(NBlockCodecs::Codec("lz4hc")) - , ZstdCodec_(NBlockCodecs::Codec("zstd_1")) -{ - Y_ENSURE(LogFormat_ == COMPRESSED_LOG_FORMAT_V4 || LogFormat_ == COMPRESSED_LOG_FORMAT_V5); - - if (contentFormat & 0xff000000) { - ythrow yexception() << "wrong compressed event log content format code (" << contentFormat << ")"; - } -} - -TEventLog::TEventLog(const TString& fileName, TEventLogFormat contentFormat, const TEventLogBackendOptions& backendOpts) - : TEventLog(fileName, contentFormat, backendOpts, COMPRESSED_LOG_FORMAT_V4) -{ -} - -TEventLog::TEventLog(const TLog& log, TEventLogFormat contentFormat, TEventLogFormat logFormat) - : Log_(log) - , ContentFormat_(contentFormat) - , LogFormat_(logFormat) - , HasNullBackend_(Log_.IsNullLog()) - , Lz4hcCodec_(NBlockCodecs::Codec("lz4hc")) - , ZstdCodec_(NBlockCodecs::Codec("zstd_1")) -{ - if (contentFormat & 0xff000000) { - ythrow yexception() << "wrong compressed event log content format code (" << contentFormat << ")"; - } -} - -TEventLog::TEventLog(TEventLogFormat contentFormat, TEventLogFormat logFormat) - : Log_(MakeHolder<TNullLogBackend>()) - , ContentFormat_(contentFormat) - , LogFormat_(logFormat) - , HasNullBackend_(true) - , Lz4hcCodec_(NBlockCodecs::Codec("lz4hc")) - , ZstdCodec_(NBlockCodecs::Codec("zstd_1")) -{ - if (contentFormat & 0xff000000) { - ythrow yexception() << "wrong compressed event log content format code (" << contentFormat << ")"; - } -} - -TEventLog::~TEventLog() { -} - -void TEventLog::ReopenLog() { - Log_.ReopenLog(); -} - -void TEventLog::CloseLog() { - Log_.CloseLog(); -} - -void TEventLog::Flush() { -} - -namespace { - class TOnExceptionAction { - public: - TOnExceptionAction(std::function<void()>&& f) - : F_(std::move(f)) - { - } - - ~TOnExceptionAction() { - if (F_ && UncaughtException()) { - try { - F_(); - } catch (...) { - } - } - } - - private: - std::function<void()> F_; - }; -} - -void TEventLog::WriteFrame(TBuffer& buffer, - TEventTimestamp startTimestamp, - TEventTimestamp endTimestamp, - TWriteFrameCallbackPtr writeFrameCallback, - TLogRecord::TMetaFlags metaFlags) { - Y_ENSURE(LogFormat_ == COMPRESSED_LOG_FORMAT_V4 || LogFormat_ == COMPRESSED_LOG_FORMAT_V5); - - TBuffer& b1 = buffer; - - size_t maxCompressedLength = (LogFormat_ == COMPRESSED_LOG_FORMAT_V4) ? b1.Size() + 256 : ZstdCodec_->MaxCompressedLength(b1); - - // Reserve enough memory to minimize reallocs - TBufferOutput outbuf(sizeof(TFrameHeaderData) + maxCompressedLength); - TBuffer& b2 = outbuf.Buffer(); - b2.Proceed(sizeof(TFrameHeaderData)); - - { - TFrameHeaderData& hdr = *reinterpret_cast<TFrameHeaderData*>(b2.data()); - - memcpy(hdr.SyncField, COMPRESSED_LOG_FRAME_SYNC_DATA.data(), COMPRESSED_LOG_FRAME_SYNC_DATA.size()); - hdr.Header.Format = (LogFormat_ << 24) | (ContentFormat_ & 0xffffff); - hdr.Header.FrameId = GenerateFrameId(); - hdr.HeaderEx.UncompressedDatalen = (ui32)b1.Size(); - hdr.HeaderEx.StartTimestamp = startTimestamp; - hdr.HeaderEx.EndTimestamp = endTimestamp; - hdr.HeaderEx.PayloadChecksum = 0; - hdr.HeaderEx.CompressorVersion = 0; - } - - if (LogFormat_ == COMPRESSED_LOG_FORMAT_V4) { - TBuffer encoded(b1.Size() + sizeof(TFrameHeaderData) + 256); - Lz4hcCodec_->Encode(b1, encoded); - - TZLibCompress compr(&outbuf, ZLib::ZLib, 6, 2048); - compr.Write(encoded.data(), encoded.size()); - compr.Finish(); - } else { - b2.Advance(ZstdCodec_->Compress(b1, b2.Pos())); - } - - { - const size_t k = sizeof(TCompressedFrameBaseHeader) + COMPRESSED_LOG_FRAME_SYNC_DATA.size(); - TFrameHeaderData& hdr = *reinterpret_cast<TFrameHeaderData*>(b2.data()); - hdr.Header.Length = static_cast<ui32>(b2.size() - k); - hdr.HeaderEx.PayloadChecksum = MurmurHash<ui32>(b2.data() + sizeof(TFrameHeaderData), b2.size() - sizeof(TFrameHeaderData)); - - const size_t n = sizeof(TFrameHeaderData) - (COMPRESSED_LOG_FRAME_SYNC_DATA.size() + sizeof(hdr.HeaderEx.HeaderChecksum)); - hdr.HeaderEx.HeaderChecksum = MurmurHash<ui32>(b2.data() + COMPRESSED_LOG_FRAME_SYNC_DATA.size(), n); - } - - const TBuffer& frameData = outbuf.Buffer(); - - TOnExceptionAction actionCallback([this] { - if (ErrorCallback_) { - ErrorCallback_->OnWriteError(); - } - }); - - if (writeFrameCallback) { - writeFrameCallback->OnAfterCompress(frameData, startTimestamp, endTimestamp); - } - - Log_.Write(frameData.Data(), frameData.Size(), std::move(metaFlags)); - if (SuccessCallback_) { - SuccessCallback_->OnWriteSuccess(frameData); - } -} - -TEvent* TProtobufEventFactory::CreateLogEvent(TEventClass c) { - return new TProtobufEvent(c, EventFactory_); -} - -TEventClass TProtobufEventFactory::ClassByName(TStringBuf name) const { - return EventFactory_->IdByName(name); -} - -TEventClass TProtobufEventFactory::EventClassBegin() const { - const auto& items = EventFactory_->FactoryItems(); - - if (items.empty()) { - return static_cast<TEventClass>(0); - } - - return static_cast<TEventClass>(items.begin()->first); -} - -TEventClass TProtobufEventFactory::EventClassEnd() const { - const auto& items = EventFactory_->FactoryItems(); - - if (items.empty()) { - return static_cast<TEventClass>(0); - } - - return static_cast<TEventClass>(items.rbegin()->first + 1); -} - -namespace NEvClass { - IEventFactory* Factory() { - return Singleton<TProtobufEventFactory>(); - } - - IEventProcessor* Processor() { - return Singleton<TProtobufEventProcessor>(); - } -} - -const NProtoBuf::Message* TUnknownEvent::GetProto() const { - return UnknownEventMessage(); -} - -TStringBuf TUnknownEvent::GetName() const { - return TStringBuf("UnknownEvent"); -} - -void TUnknownEvent::DoPrintJson(NJson::TJsonWriter& jsonWriter) const { - jsonWriter.OpenMap("EventBody"); - jsonWriter.Write("Type", GetName()); - jsonWriter.Write("EventId", (size_t)Class); - jsonWriter.CloseMap(); -} - -TStringBuf TEndOfFrameEvent::GetName() const { - return TStringBuf("EndOfFrame"); -} - -const NProtoBuf::Message* TEndOfFrameEvent::GetProto() const { - return Singleton<NEventLogInternal::TEndOfFrameEvent>(); -} - -void TEndOfFrameEvent::DoPrintJson(NJson::TJsonWriter& jsonWriter) const { - jsonWriter.OpenMap("EventBody"); - jsonWriter.Write("Type", GetName()); - jsonWriter.OpenMap("Fields"); - jsonWriter.CloseMap(); - jsonWriter.CloseMap(); -} - -THolder<TEvent> MakeProtobufLogEvent(TEventTimestamp ts, TEventClass eventId, google::protobuf::Message& ev) { - return MakeHolder<TProtobufEvent>(ts, eventId, ev); -} diff --git a/library/cpp/eventlog/eventlog.h b/library/cpp/eventlog/eventlog.h deleted file mode 100644 index 45c2dfb17f..0000000000 --- a/library/cpp/eventlog/eventlog.h +++ /dev/null @@ -1,623 +0,0 @@ -#pragma once - -#include "eventlog_int.h" -#include "event_field_output.h" -#include "events_extension.h" - -#include <library/cpp/blockcodecs/codecs.h> -#include <library/cpp/logger/all.h> - -#include <google/protobuf/message.h> - -#include <util/datetime/base.h> -#include <util/generic/ptr.h> -#include <util/generic/string.h> -#include <util/stream/output.h> -#include <util/stream/buffer.h> -#include <util/stream/str.h> -#include <util/system/mutex.h> -#include <util/stream/output.h> -#include <util/system/env.h> -#include <util/system/unaligned_mem.h> -#include <util/ysaveload.h> - -#include <cstdlib> - -namespace NJson { - class TJsonWriter; -} - -class IEventLog; - -class TEvent : public TThrRefBase { -public: - enum class TOutputFormat { - TabSeparated, - TabSeparatedRaw, // disables escaping - Json - }; - - struct TOutputOptions { - TOutputFormat OutputFormat = TOutputFormat::TabSeparated; - // Dump some fields (e.g. timestamp) in more human-readable format - bool HumanReadable = false; - - TOutputOptions(TOutputFormat outputFormat = TOutputFormat::TabSeparated) - : OutputFormat(outputFormat) - { - } - - TOutputOptions(TOutputFormat outputFormat, bool humanReadable) - : OutputFormat(outputFormat) - , HumanReadable(humanReadable) - { - } - }; - - struct TEventState { - TEventTimestamp FrameStartTime = 0; - TEventTimestamp PrevEventTime = 0; - TEventState() { - } - }; - - TEvent(TEventClass c, TEventTimestamp t) - : Class(c) - , Timestamp(t) - { - } - - virtual ~TEvent() = default; - - // Note, that descendants MUST have Save() & Load() methods to alter - // only its new variables, not the base class! - virtual void Save(IOutputStream& out) const = 0; - virtual void SaveToBuffer(TBufferOutput& out) const { - Save(out); - } - - // Note, that descendants MUST have Save() & Load() methods to alter - // only its new variables, not the base class! - virtual void Load(IInputStream& i) = 0; - - virtual TStringBuf GetName() const = 0; - virtual const NProtoBuf::Message* GetProto() const = 0; - - void Print(IOutputStream& out, const TOutputOptions& options = TOutputOptions(), const TEventState& eventState = TEventState()) const; - void PrintHeader(IOutputStream& out, const TOutputOptions& options, const TEventState& eventState) const; - - TString ToString() const { - TStringStream buff; - Print(buff); - return buff.Str(); - } - - void FullSaveToBuffer(TBufferOutput& buf) const { - SaveMessageHeader(buf); - this->SaveToBuffer(buf); - } - - void FullSave(IOutputStream& o) const { - SaveMessageHeader(o); - this->Save(o); - } - - void FullLoad(IInputStream& i) { - ::Load(&i, Timestamp); - ::Load(&i, Class); - this->Load(i); - } - - template <class T> - const T* Get() const { - return static_cast<const T*>(this->GetProto()); - } - - TEventClass Class; - TEventTimestamp Timestamp; - ui32 FrameId = 0; - -private: - void SaveMessageHeader(IOutputStream& out) const { - ::Save(&out, Timestamp); - ::Save(&out, Class); - } - - virtual void DoPrint(IOutputStream& out, EFieldOutputFlags flags) const = 0; - virtual void DoPrintJson(NJson::TJsonWriter& jsonWriter) const = 0; - - void PrintJsonHeader(NJson::TJsonWriter& jsonWriter) const; -}; - -using TEventPtr = TIntrusivePtr<TEvent>; -using TConstEventPtr = TIntrusiveConstPtr<TEvent>; - -class IEventProcessor { -public: - virtual void SetOptions(const TEvent::TOutputOptions& options) { - Options_ = options; - } - virtual void ProcessEvent(const TEvent* ev) = 0; - virtual bool CheckedProcessEvent(const TEvent* ev) { - ProcessEvent(ev); - return true; - } - virtual ~IEventProcessor() = default; - -protected: - TEvent::TOutputOptions Options_; -}; - -class IEventFactory { -public: - virtual TEvent* CreateLogEvent(TEventClass c) = 0; - virtual TEventLogFormat CurrentFormat() = 0; - virtual TEventClass ClassByName(TStringBuf name) const = 0; - virtual TEventClass EventClassBegin() const = 0; - virtual TEventClass EventClassEnd() const = 0; - virtual ~IEventFactory() = default; -}; - -class TUnknownEvent: public TEvent { -public: - TUnknownEvent(TEventTimestamp ts, TEventClass cls) - : TEvent(cls, ts) - { - } - - ~TUnknownEvent() override = default; - - void Save(IOutputStream& /* o */) const override { - ythrow yexception() << "TUnknownEvent cannot be saved"; - } - - void Load(IInputStream& /* i */) override { - ythrow yexception() << "TUnknownEvent cannot be loaded"; - } - - TStringBuf GetName() const override; - -private: - void DoPrint(IOutputStream& out, EFieldOutputFlags) const override { - out << GetName() << "\t" << (size_t)Class; - } - - void DoPrintJson(NJson::TJsonWriter& jsonWriter) const override; - - const NProtoBuf::Message* GetProto() const override; -}; - -class TEndOfFrameEvent: public TEvent { -public: - enum { - EventClass = 0 - }; - - TEndOfFrameEvent(TEventTimestamp ts) - : TEvent(TEndOfFrameEvent::EventClass, ts) - { - } - - ~TEndOfFrameEvent() override = default; - - void Save(IOutputStream& o) const override { - (void)o; - ythrow yexception() << "TEndOfFrameEvent cannot be saved"; - } - - void Load(IInputStream& i) override { - (void)i; - ythrow yexception() << "TEndOfFrameEvent cannot be loaded"; - } - - TStringBuf GetName() const override; - -private: - void DoPrint(IOutputStream& out, EFieldOutputFlags) const override { - out << GetName(); - } - void DoPrintJson(NJson::TJsonWriter& jsonWriter) const override; - - const NProtoBuf::Message* GetProto() const override; -}; - -class ILogFrameEventVisitor { -public: - virtual ~ILogFrameEventVisitor() = default; - - virtual void Visit(const TEvent& event) = 0; -}; - -class IWriteFrameCallback : public TAtomicRefCount<IWriteFrameCallback> { -public: - virtual ~IWriteFrameCallback() = default; - - virtual void OnAfterCompress(const TBuffer& compressedFrame, TEventTimestamp startTimestamp, TEventTimestamp endTimestamp) = 0; -}; - -using TWriteFrameCallbackPtr = TIntrusivePtr<IWriteFrameCallback>; - -class TEventLogFrame { -public: - TEventLogFrame(bool needAlwaysSafeAdd = false, TWriteFrameCallbackPtr writeFrameCallback = nullptr); - TEventLogFrame(IEventLog& parentLog, bool needAlwaysSafeAdd = false, TWriteFrameCallbackPtr writeFrameCallback = nullptr); - TEventLogFrame(IEventLog* parentLog, bool needAlwaysSafeAdd = false, TWriteFrameCallbackPtr writeFrameCallback = nullptr); - - virtual ~TEventLogFrame() = default; - - void Flush(); - void SafeFlush(); - - void ForceDump() { - ForceDump_ = true; - } - - template <class T> - inline void LogEvent(const T& ev) { - if (NeedAlwaysSafeAdd_) { - SafeLogEvent(ev); - } else { - UnSafeLogEvent(ev); - } - } - - template <class T> - inline void LogEvent(TEventTimestamp timestamp, const T& ev) { - if (NeedAlwaysSafeAdd_) { - SafeLogEvent(timestamp, ev); - } else { - UnSafeLogEvent(timestamp, ev); - } - } - - template <class T> - inline void UnSafeLogEvent(const T& ev) { - if (!IsEventIgnored(ev.ID)) - LogProtobufEvent(ev.ID, ev); - } - - template <class T> - inline void UnSafeLogEvent(TEventTimestamp timestamp, const T& ev) { - if (!IsEventIgnored(ev.ID)) - LogProtobufEvent(timestamp, ev.ID, ev); - } - - template <class T> - inline void SafeLogEvent(const T& ev) { - if (!IsEventIgnored(ev.ID)) { - TGuard<TMutex> g(Mtx_); - LogProtobufEvent(ev.ID, ev); - } - } - - template <class T> - inline void SafeLogEvent(TEventTimestamp timestamp, const T& ev) { - if (!IsEventIgnored(ev.ID)) { - TGuard<TMutex> g(Mtx_); - LogProtobufEvent(timestamp, ev.ID, ev); - } - } - - void VisitEvents(ILogFrameEventVisitor& visitor, IEventFactory* eventFactory); - - inline bool IsEventIgnored(size_t eventId) const { - Y_UNUSED(eventId); // in future we might want to selectively discard only some kinds of messages - return !IsDebugModeEnabled() && EvLog_ == nullptr && !ForceDump_; - } - - void Enable(IEventLog& evLog) { - EvLog_ = &evLog; - } - - void Disable() { - EvLog_ = nullptr; - } - - void SetNeedAlwaysSafeAdd(bool val) { - NeedAlwaysSafeAdd_ = val; - } - - void SetWriteFrameCallback(TWriteFrameCallbackPtr writeFrameCallback) { - WriteFrameCallback_ = writeFrameCallback; - } - - void AddMetaFlag(const TString& key, const TString& value) { - if (NeedAlwaysSafeAdd_) { - TGuard<TMutex> g(Mtx_); - MetaFlags_.emplace_back(key, value); - } else { - MetaFlags_.emplace_back(key, value); - } - } - -protected: - void LogProtobufEvent(size_t eventId, const NProtoBuf::Message& ev); - void LogProtobufEvent(TEventTimestamp timestamp, size_t eventId, const NProtoBuf::Message& ev); - -private: - static bool IsDebugModeEnabled() { - static struct TSelector { - bool Flag; - - TSelector() - : Flag(GetEnv("EVLOG_DEBUG") == TStringBuf("1")) - { - } - } selector; - - return selector.Flag; - } - - template <class T> - void DebugDump(const T& ev); - - // T must be a descendant of NEvClass::TEvent - template <class T> - inline void LogEventImpl(const T& ev) { - if (EvLog_ != nullptr || ForceDump_) { - TBuffer& b = Buf_.Buffer(); - size_t lastSize = b.size(); - ::Save(&Buf_, ui32(0)); - ev.FullSaveToBuffer(Buf_); - WriteUnaligned<ui32>(b.data() + lastSize, (ui32)(b.size() - lastSize)); - AddEvent(ev.Timestamp); - } - - if (IsDebugModeEnabled()) { - DebugDump(ev); - } - } - - void AddEvent(TEventTimestamp timestamp); - void DoInit(); - -private: - TBufferOutput Buf_; - TEventTimestamp StartTimestamp_, EndTimestamp_; - IEventLog* EvLog_; - TMutex Mtx_; - bool NeedAlwaysSafeAdd_; - bool ForceDump_; - TWriteFrameCallbackPtr WriteFrameCallback_; - TLogRecord::TMetaFlags MetaFlags_; - friend class TEventRecord; -}; - -class TSelfFlushLogFrame: public TEventLogFrame, public TAtomicRefCount<TSelfFlushLogFrame> { -public: - TSelfFlushLogFrame(bool needAlwaysSafeAdd = false, TWriteFrameCallbackPtr writeFrameCallback = nullptr); - TSelfFlushLogFrame(IEventLog& parentLog, bool needAlwaysSafeAdd = false, TWriteFrameCallbackPtr writeFrameCallback = nullptr); - TSelfFlushLogFrame(IEventLog* parentLog, bool needAlwaysSafeAdd = false, TWriteFrameCallbackPtr writeFrameCallback = nullptr); - - virtual ~TSelfFlushLogFrame(); -}; - -using TSelfFlushLogFramePtr = TIntrusivePtr<TSelfFlushLogFrame>; - -class IEventLog: public TAtomicRefCount<IEventLog> { -public: - class IErrorCallback { - public: - virtual ~IErrorCallback() { - } - - virtual void OnWriteError() = 0; - }; - - class ISuccessCallback { - public: - virtual ~ISuccessCallback() { - } - - virtual void OnWriteSuccess(const TBuffer& frameData) = 0; - }; - - virtual ~IEventLog(); - - virtual void ReopenLog() = 0; - virtual void CloseLog() = 0; - virtual void Flush() = 0; - virtual void SetErrorCallback(IErrorCallback*) { - } - virtual void SetSuccessCallback(ISuccessCallback*) { - } - - template <class T> - void LogEvent(const T& ev) { - TEventLogFrame frame(*this); - frame.LogEvent(ev); - frame.Flush(); - } - - virtual bool HasNullBackend() const = 0; - - virtual void WriteFrame(TBuffer& buffer, - TEventTimestamp startTimestamp, - TEventTimestamp endTimestamp, - TWriteFrameCallbackPtr writeFrameCallback = nullptr, - TLogRecord::TMetaFlags metaFlags = {}) = 0; -}; - -struct TEventLogBackendOptions { - bool UseSyncPageCacheBackend = false; - size_t SyncPageCacheBackendBufferSize = 0; - size_t SyncPageCacheBackendMaxPendingSize = 0; -}; - -class TEventLog: public IEventLog { -public: - /* - * Параметр contentformat указывает формат контента лога, например какие могут в логе - * встретится классы событий, какие параметры у этих событий, и пр. Старший байт параметра - * должен быть нулевым. - */ - TEventLog(const TString& fileName, TEventLogFormat contentFormat, const TEventLogBackendOptions& backendOpts, TMaybe<TEventLogFormat> logFormat); - TEventLog(const TString& fileName, TEventLogFormat contentFormat, const TEventLogBackendOptions& backendOpts = {}); - TEventLog(const TLog& log, TEventLogFormat contentFormat, TEventLogFormat logFormat = COMPRESSED_LOG_FORMAT_V4); - TEventLog(TEventLogFormat contentFormat, TEventLogFormat logFormat = COMPRESSED_LOG_FORMAT_V4); - - ~TEventLog() override; - - void ReopenLog() override; - void CloseLog() override; - void Flush() override; - void SetErrorCallback(IErrorCallback* errorCallback) override { - ErrorCallback_ = errorCallback; - } - void SetSuccessCallback(ISuccessCallback* successCallback) override { - SuccessCallback_ = successCallback; - } - - template <class T> - void LogEvent(const T& ev) { - TEventLogFrame frame(*this); - frame.LogEvent(ev); - frame.Flush(); - } - - bool HasNullBackend() const override { - return HasNullBackend_; - } - - void WriteFrame(TBuffer& buffer, - TEventTimestamp startTimestamp, - TEventTimestamp endTimestamp, - TWriteFrameCallbackPtr writeFrameCallback = nullptr, - TLogRecord::TMetaFlags metaFlags = {}) override; - -private: - mutable TLog Log_; - TEventLogFormat ContentFormat_; - const TEventLogFormat LogFormat_; - bool HasNullBackend_; - const NBlockCodecs::ICodec* const Lz4hcCodec_; - const NBlockCodecs::ICodec* const ZstdCodec_; - IErrorCallback* ErrorCallback_ = nullptr; - ISuccessCallback* SuccessCallback_ = nullptr; -}; - -using TEventLogPtr = TIntrusivePtr<IEventLog>; - -class TEventLogWithSlave: public IEventLog { -public: - TEventLogWithSlave(IEventLog& parentLog) - : Slave_(&parentLog) - { - } - - TEventLogWithSlave(const TEventLogPtr& parentLog) - : SlavePtr_(parentLog) - , Slave_(SlavePtr_.Get()) - { - } - - ~TEventLogWithSlave() override { - try { - Slave().Flush(); - } catch (...) { - } - } - - void Flush() override { - Slave().Flush(); - } - - void ReopenLog() override { - return Slave().ReopenLog(); - } - void CloseLog() override { - return Slave().CloseLog(); - } - - bool HasNullBackend() const override { - return Slave().HasNullBackend(); - } - - void WriteFrame(TBuffer& buffer, - TEventTimestamp startTimestamp, - TEventTimestamp endTimestamp, - TWriteFrameCallbackPtr writeFrameCallback = nullptr, - TLogRecord::TMetaFlags metaFlags = {}) override { - Slave().WriteFrame(buffer, startTimestamp, endTimestamp, writeFrameCallback, std::move(metaFlags)); - } - - void SetErrorCallback(IErrorCallback* errorCallback) override { - Slave().SetErrorCallback(errorCallback); - } - - void SetSuccessCallback(ISuccessCallback* successCallback) override { - Slave().SetSuccessCallback(successCallback); - } - -protected: - inline IEventLog& Slave() const { - return *Slave_; - } - -private: - TEventLogPtr SlavePtr_; - IEventLog* Slave_ = nullptr; -}; - -extern TAtomic eventlogFrameCounter; - -class TProtobufEventProcessor: public IEventProcessor { -public: - void ProcessEvent(const TEvent* ev) override final { - ProcessEvent(ev, &Cout); - } - - void ProcessEvent(const TEvent* ev, IOutputStream *out) { - UpdateEventState(ev); - DoProcessEvent(ev, out); - EventState_.PrevEventTime = ev->Timestamp; - } -protected: - virtual void DoProcessEvent(const TEvent * ev, IOutputStream *out) { - ev->Print(*out, Options_, EventState_); - (*out) << Endl; - } - ui32 CurrentFrameId_ = Max<ui32>(); - TEvent::TEventState EventState_; - -private: - void UpdateEventState(const TEvent *ev) { - if (ev->FrameId != CurrentFrameId_) { - EventState_.FrameStartTime = ev->Timestamp; - EventState_.PrevEventTime = ev->Timestamp; - CurrentFrameId_ = ev->FrameId; - } - } -}; - -class TProtobufEventFactory: public IEventFactory { -public: - TProtobufEventFactory(NProtoBuf::TEventFactory* factory = NProtoBuf::TEventFactory::Instance()) - : EventFactory_(factory) - { - } - - TEvent* CreateLogEvent(TEventClass c) override; - - TEventLogFormat CurrentFormat() override { - return 0; - } - - TEventClass ClassByName(TStringBuf name) const override; - - TEventClass EventClassBegin() const override; - - TEventClass EventClassEnd() const override; - - ~TProtobufEventFactory() override = default; - -private: - NProtoBuf::TEventFactory* EventFactory_; -}; - -THolder<TEvent> MakeProtobufLogEvent(TEventTimestamp ts, TEventClass eventId, google::protobuf::Message& ev); - -namespace NEvClass { - IEventFactory* Factory(); - IEventProcessor* Processor(); -} diff --git a/library/cpp/eventlog/eventlog_int.cpp b/library/cpp/eventlog/eventlog_int.cpp deleted file mode 100644 index faa8c42cbe..0000000000 --- a/library/cpp/eventlog/eventlog_int.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include "eventlog_int.h" - -#include <util/string/cast.h> - -TMaybe<TEventLogFormat> ParseEventLogFormat(TStringBuf str) { - EEventLogFormat format; - if (TryFromString(str, format)) { - return static_cast<TEventLogFormat>(format); - } else { - return {}; - } -} diff --git a/library/cpp/eventlog/eventlog_int.h b/library/cpp/eventlog/eventlog_int.h deleted file mode 100644 index eb00fecfab..0000000000 --- a/library/cpp/eventlog/eventlog_int.h +++ /dev/null @@ -1,72 +0,0 @@ -#pragma once - -#include <util/stream/output.h> -#include <util/generic/maybe.h> -#include <util/generic/utility.h> -#include <util/generic/yexception.h> -#include <util/ysaveload.h> - -using TEventClass = ui32; -using TEventLogFormat = ui32; -using TEventTimestamp = ui64; - -constexpr TStringBuf COMPRESSED_LOG_FRAME_SYNC_DATA = - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\xfe\x00\x00\xff\xff\x00\x00\xff\xff\x00" - "\x00\xff\xff\x00\x00\xff\xff\x00\x00\xff\xff\x00\x00\xff" - "\xff\x00\x00\xff\xff\x00\x00\xff"sv; - -static_assert(COMPRESSED_LOG_FRAME_SYNC_DATA.size() == 64); - -/* - * Коды форматов логов. Форматом лога считается формат служебных - * структур лога. К примеру формат заголовка, наличие компрессии, и т.д. - * Имеет значение только 1 младший байт. - */ - -enum EEventLogFormat : TEventLogFormat { - // Формат версии 1. Используется компрессор LZQ. - COMPRESSED_LOG_FORMAT_V1 = 1, - - // Формат версии 2. Используется компрессор ZLIB. Добавлены CRC заголовка и данных, - // поле типа компрессора. - COMPRESSED_LOG_FORMAT_V2 = 2, - - // Формат версии 3. Используется компрессор ZLIB. В начинке фреймов перед каждым событием добавлен его размер. - COMPRESSED_LOG_FORMAT_V3 = 3, - - // Lz4hc codec + zlib - COMPRESSED_LOG_FORMAT_V4 = 4 /* "zlib_lz4" */, - - // zstd - COMPRESSED_LOG_FORMAT_V5 = 5 /* "zstd" */, -}; - -TMaybe<TEventLogFormat> ParseEventLogFormat(TStringBuf str); - -#pragma pack(push, 1) - -struct TCompressedFrameBaseHeader { - TEventLogFormat Format; - ui32 Length; // Длина остатка фрейма в байтах, после этого заголовка - ui32 FrameId; -}; - -struct TCompressedFrameHeader { - TEventTimestamp StartTimestamp; - TEventTimestamp EndTimestamp; - ui32 UncompressedDatalen; // Длина данных, которые были закомпрессированы - ui32 PayloadChecksum; // В логе версии 1 поле не используется -}; - -struct TCompressedFrameHeader2: public TCompressedFrameHeader { - ui8 CompressorVersion; // Сейчас не используется - ui32 HeaderChecksum; -}; - -#pragma pack(pop) - -Y_DECLARE_PODTYPE(TCompressedFrameBaseHeader); -Y_DECLARE_PODTYPE(TCompressedFrameHeader); -Y_DECLARE_PODTYPE(TCompressedFrameHeader2); diff --git a/library/cpp/eventlog/events_extension.h b/library/cpp/eventlog/events_extension.h deleted file mode 100644 index 0cf062f959..0000000000 --- a/library/cpp/eventlog/events_extension.h +++ /dev/null @@ -1,161 +0,0 @@ -#pragma once - -#include "event_field_output.h" - -#include <google/protobuf/descriptor.h> -#include <google/protobuf/message.h> - -#include <library/cpp/threading/atomic/bool.h> -#include <library/cpp/string_utils/base64/base64.h> - -#include <util/generic/map.h> -#include <util/generic/deque.h> -#include <util/generic/singleton.h> -#include <util/string/hex.h> -#include <util/system/guard.h> -#include <util/system/mutex.h> - -namespace NProtoBuf { - class TEventFactory { - public: - typedef ::google::protobuf::Message Message; - typedef void (*TEventSerializer)(const Message* event, IOutputStream& output, EFieldOutputFlags flags); - typedef void (*TRegistrationFunc)(); - - private: - class TFactoryItem { - public: - TFactoryItem(const Message* prototype, const TEventSerializer serializer) - : Prototype_(prototype) - , Serializer_(serializer) - { - } - - TStringBuf GetName() const { - return Prototype_->GetDescriptor()->name(); - } - - Message* Create() const { - return Prototype_->New(); - } - - void PrintEvent(const Message* event, IOutputStream& out, EFieldOutputFlags flags) const { - (*Serializer_)(event, out, flags); - } - - private: - const Message* Prototype_; - const TEventSerializer Serializer_; - }; - - typedef TMap<size_t, TFactoryItem> TFactoryMap; - - public: - TEventFactory() - : FactoryItems_() - { - } - - void ScheduleRegistration(TRegistrationFunc func) { - EventRegistrators_.push_back(func); - } - - void RegisterEvent(size_t eventId, const Message* prototype, const TEventSerializer serializer) { - FactoryItems_.insert(std::make_pair(eventId, TFactoryItem(prototype, serializer))); - } - - size_t IdByName(TStringBuf eventname) { - DelayedRegistration(); - for (TFactoryMap::const_iterator it = FactoryItems_.begin(); it != FactoryItems_.end(); ++it) { - if (it->second.GetName() == eventname) - return it->first; - } - - ythrow yexception() << "do not know event '" << eventname << "'"; - } - - TStringBuf NameById(size_t id) { - DelayedRegistration(); - TFactoryMap::const_iterator it = FactoryItems_.find(id); - return it != FactoryItems_.end() ? it->second.GetName() : TStringBuf(); - } - - Message* CreateEvent(size_t eventId) { - DelayedRegistration(); - TFactoryMap::const_iterator it = FactoryItems_.find(eventId); - - if (it != FactoryItems_.end()) { - return it->second.Create(); - } - - return nullptr; - } - - const TMap<size_t, TFactoryItem>& FactoryItems() { - DelayedRegistration(); - return FactoryItems_; - } - - void PrintEvent( - size_t eventId, - const Message* event, - IOutputStream& output, - EFieldOutputFlags flags = {}) { - DelayedRegistration(); - TFactoryMap::const_iterator it = FactoryItems_.find(eventId); - - if (it != FactoryItems_.end()) { - it->second.PrintEvent(event, output, flags); - } - } - - static TEventFactory* Instance() { - return Singleton<TEventFactory>(); - } - - private: - void DelayedRegistration() { - if (!DelayedRegistrationDone_) { - TGuard<TMutex> guard(MutexEventRegistrators_); - Y_UNUSED(guard); - while (!EventRegistrators_.empty()) { - EventRegistrators_.front()(); - EventRegistrators_.pop_front(); - } - DelayedRegistrationDone_ = true; - } - } - - private: - TMap<size_t, TFactoryItem> FactoryItems_; - TDeque<TRegistrationFunc> EventRegistrators_; - NAtomic::TBool DelayedRegistrationDone_ = false; - TMutex MutexEventRegistrators_; - }; - - template <typename T> - void PrintAsBytes(const T& obj, IOutputStream& output) { - const ui8* b = reinterpret_cast<const ui8*>(&obj); - const ui8* e = b + sizeof(T); - const char* delim = ""; - - while (b != e) { - output << delim; - output << (int)*b++; - delim = "."; - } - } - - template <typename T> - void PrintAsHex(const T& obj, IOutputStream& output) { - output << "0x"; - output << HexEncode(&obj, sizeof(T)); - } - - inline void PrintAsBase64(TStringBuf data, IOutputStream& output) { - if (!data.empty()) { - output << Base64Encode(data); - } - } - -} diff --git a/library/cpp/eventlog/iterator.cpp b/library/cpp/eventlog/iterator.cpp deleted file mode 100644 index 71f955bca8..0000000000 --- a/library/cpp/eventlog/iterator.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include "iterator.h" - -#include <library/cpp/streams/growing_file_input/growing_file_input.h> - -#include <util/string/cast.h> -#include <util/string/split.h> -#include <util/string/type.h> -#include <util/stream/file.h> - -using namespace NEventLog; - -namespace { - inline TIntrusivePtr<TEventFilter> ConstructEventFilter(bool enableEvents, const TString& evList, IEventFactory* fac) { - if (evList.empty()) { - return nullptr; - } - - TVector<TString> events; - - StringSplitter(evList).Split(',').SkipEmpty().Collect(&events); - if (events.empty()) { - return nullptr; - } - - TIntrusivePtr<TEventFilter> filter(new TEventFilter(enableEvents)); - - for (const auto& event : events) { - if (IsNumber(event)) - filter->AddEventClass(FromString<size_t>(event)); - else - filter->AddEventClass(fac->ClassByName(event)); - } - - return filter; - } - - struct TIterator: public IIterator { - inline TIterator(const TOptions& o, IEventFactory* fac) - : First(true) - { - if (o.FileName.size()) { - if (o.ForceStreamMode || o.TailFMode) { - FileInput.Reset(o.TailFMode ? (IInputStream*)new TGrowingFileInput(o.FileName) : (IInputStream*)new TUnbufferedFileInput(o.FileName)); - FrameStream.Reset(new TFrameStreamer(*FileInput, fac, o.FrameFilter)); - } else { - FrameStream.Reset(new TFrameStreamer(o.FileName, o.StartTime, o.EndTime, o.MaxRequestDuration, fac, o.FrameFilter)); - } - } else { - FrameStream.Reset(new TFrameStreamer(*o.Input, fac, o.FrameFilter)); - } - - EvFilter = ConstructEventFilter(o.EnableEvents, o.EvList, fac); - EventStream.Reset(new TEventStreamer(*FrameStream, o.StartTime, o.EndTime, o.ForceStrongOrdering, EvFilter, o.ForceLosslessStrongOrdering)); - } - - TConstEventPtr Next() override { - if (First) { - First = false; - - if (!EventStream->Avail()) { - return nullptr; - } - } else { - if (!EventStream->Next()) { - return nullptr; - } - } - - return **EventStream; - } - - THolder<IInputStream> FileInput; - THolder<TFrameStreamer> FrameStream; - TIntrusivePtr<TEventFilter> EvFilter; - THolder<TEventStreamer> EventStream; - bool First; - }; -} - -IIterator::~IIterator() = default; - -THolder<IIterator> NEventLog::CreateIterator(const TOptions& o, IEventFactory* fac) { - return MakeHolder<TIterator>(o, fac); -} - -THolder<IIterator> NEventLog::CreateIterator(const TOptions& o) { - return MakeHolder<TIterator>(o, NEvClass::Factory()); -} diff --git a/library/cpp/eventlog/iterator.h b/library/cpp/eventlog/iterator.h deleted file mode 100644 index 71a61ed549..0000000000 --- a/library/cpp/eventlog/iterator.h +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#include <util/stream/input.h> -#include <util/generic/ptr.h> -#include <util/generic/string.h> -#include <util/generic/iterator.h> - -#include "eventlog.h" -#include "logparser.h" - -namespace NEventLog { - struct TOptions { - inline TOptions& SetFileName(const TString& fileName) { - FileName = fileName; - - return *this; - } - - inline TOptions& SetForceStrongOrdering(bool v) { - if(!ForceLosslessStrongOrdering) { - ForceStrongOrdering = v; - } - - return *this; - } - - ui64 StartTime = MIN_START_TIME; - ui64 EndTime = MAX_END_TIME; - ui64 MaxRequestDuration = MAX_REQUEST_DURATION; - TString FileName; - bool ForceStrongOrdering = false; - bool ForceWeakOrdering = false; - bool EnableEvents = true; - TString EvList; - bool ForceStreamMode = false; - bool ForceLosslessStrongOrdering = false; - bool TailFMode = false; - IInputStream* Input = &Cin; - IFrameFilterRef FrameFilter; - }; - - class IIterator: public TInputRangeAdaptor<IIterator> { - public: - virtual ~IIterator(); - - virtual TConstEventPtr Next() = 0; - }; - - THolder<IIterator> CreateIterator(const TOptions& o); - THolder<IIterator> CreateIterator(const TOptions& o, IEventFactory* fac); -} diff --git a/library/cpp/eventlog/logparser.cpp b/library/cpp/eventlog/logparser.cpp deleted file mode 100644 index 6f8959f788..0000000000 --- a/library/cpp/eventlog/logparser.cpp +++ /dev/null @@ -1,814 +0,0 @@ -#include "logparser.h" -#include "evdecoder.h" - -#include <util/stream/output.h> -#include <util/stream/zlib.h> -#include <util/digest/murmur.h> -#include <util/generic/algorithm.h> -#include <util/generic/scope.h> -#include <util/generic/hash_set.h> -#include <util/string/split.h> -#include <util/string/cast.h> -#include <util/string/escape.h> -#include <util/string/builder.h> - -#include <contrib/libs/re2/re2/re2.h> - -#include <algorithm> -#include <array> - -namespace { - bool FastforwardUntilSyncHeader(IInputStream* in) { - // Usually this function finds the correct header at the first hit - std::array<char, COMPRESSED_LOG_FRAME_SYNC_DATA.size()> buffer; - if (in->Load(buffer.data(), buffer.size()) != buffer.size()) { - return false; - } - - auto begin = buffer.begin(); - - for (;;) { - if (std::mismatch( - begin, buffer.end(), - COMPRESSED_LOG_FRAME_SYNC_DATA.begin()).first == buffer.end() && - std::mismatch( - buffer.begin(), begin, - COMPRESSED_LOG_FRAME_SYNC_DATA.begin() + (buffer.end() - begin)).first == begin) { - return true; - } - if (!in->ReadChar(*begin)) { - return false; - } - ++begin; - if (begin == buffer.end()) { - begin = buffer.begin(); - } - } - } - - bool HasCorrectChecksum(const TFrameHeader& header) { - // Calculating hash over all the fields of the read header except for the field with the hash of the header itself. - const size_t baseSize = sizeof(TCompressedFrameBaseHeader) + sizeof(TCompressedFrameHeader2) - sizeof(ui32); - const ui32 checksum = MurmurHash<ui32>(&header.Basehdr, baseSize); - return checksum == header.Framehdr.HeaderChecksum; - } - - TMaybe<TFrameHeader> FindNextFrameHeader(IInputStream* in) { - for (;;) { - if (FastforwardUntilSyncHeader(in)) { - try { - return TFrameHeader(*in); - } catch (const TFrameLoadError& err) { - Cdbg << err.what() << Endl; - in->Skip(err.SkipAfter); - } - } else { - return Nothing(); - } - } - } - - std::pair<TMaybe<TFrameHeader>, TStringBuf> FindNextFrameHeader(TStringBuf span) { - for (;;) { - auto iter = std::search( - span.begin(), span.end(), - COMPRESSED_LOG_FRAME_SYNC_DATA.begin(), COMPRESSED_LOG_FRAME_SYNC_DATA.end()); - const size_t offset = iter - span.begin(); - - if (offset != span.size()) { - span = span.substr(offset); - try { - TMemoryInput in( - span.data() + COMPRESSED_LOG_FRAME_SYNC_DATA.size(), - span.size() - COMPRESSED_LOG_FRAME_SYNC_DATA.size()); - return {TFrameHeader(in), span}; - } catch (const TFrameLoadError& err) { - Cdbg << err.what() << Endl; - span = span.substr(err.SkipAfter); - } - } else { - return {Nothing(), {}}; - } - } - } - - size_t FindFrames(const TStringBuf span, ui64 start, ui64 end, ui64 maxRequestDuration) { - Y_ENSURE(start <= end); - - const auto leftTimeBound = start - Min(start, maxRequestDuration); - const auto rightTimeBound = end + Min(maxRequestDuration, Max<ui64>() - end); - - TStringBuf subspan = span; - TMaybe<TFrameHeader> maybeLeftFrame; - std::tie(maybeLeftFrame, subspan) = FindNextFrameHeader(subspan); - - if (!maybeLeftFrame || maybeLeftFrame->EndTime() > rightTimeBound) { - return span.size(); - } - - if (maybeLeftFrame->StartTime() > leftTimeBound) { - return 0; - } - - while (subspan.size() > maybeLeftFrame->FullLength()) { - const auto mid = subspan.data() + subspan.size() / 2; - auto [midFrame, rightHalfSpan] = FindNextFrameHeader({mid, subspan.data() + subspan.size()}); - if (!midFrame) { - // If mid is in the middle of the last frame, here we will lose it meaning that - // we will find previous frame as the result. - // This is fine because we will iterate frames starting from that. - subspan = subspan.substr(0, subspan.size() / 2); - continue; - } - if (midFrame->StartTime() <= leftTimeBound) { - maybeLeftFrame = midFrame; - subspan = rightHalfSpan; - } else { - subspan = subspan.substr(0, subspan.size() / 2); - } - } - - return subspan.data() - span.data(); - } -} - -TFrameHeader::TFrameHeader(IInputStream& in) { - try { - ::Load(&in, Basehdr); - - Y_ENSURE(Basehdr.Length, "Empty frame additional data"); - - ::Load(&in, Framehdr); - switch (LogFormat()) { - case COMPRESSED_LOG_FORMAT_V1: - break; - - case COMPRESSED_LOG_FORMAT_V2: - case COMPRESSED_LOG_FORMAT_V3: - case COMPRESSED_LOG_FORMAT_V4: - case COMPRESSED_LOG_FORMAT_V5: - Y_ENSURE(!Framehdr.CompressorVersion, "Wrong compressor"); - - Y_ENSURE(HasCorrectChecksum(*this), "Wrong header checksum"); - break; - - default: - ythrow yexception() << "Unsupported log structure format"; - }; - - Y_ENSURE(Framehdr.StartTimestamp <= Framehdr.EndTimestamp, "Wrong start/end timestamps"); - - // Each frame must contain at least one event. - Y_ENSURE(Framehdr.UncompressedDatalen, "Empty frame payload"); - } catch (...) { - TString location = ""; - if (const auto* cnt = dynamic_cast<TCountingInput *>(&in)) { - location = "@ " + ToString(cnt->Counter()); - } - ythrow TFrameLoadError(FrameLength()) << "Frame Load Error" << location << ": " << CurrentExceptionMessage(); - } -} - -TFrame::TFrame(IInputStream& in, TFrameHeader header, IEventFactory* fac) - : TFrameHeader(header) - , Limiter_(MakeHolder<TLengthLimitedInput>(&in, header.FrameLength())) - , Fac_(fac) -{ - if (auto* cnt = dynamic_cast<TCountingInput *>(&in)) { - Address_ = cnt->Counter() - sizeof(TFrameHeader); - } else { - Address_ = 0; - } -} - -TFrame::TIterator TFrame::GetIterator(TIntrusiveConstPtr<TEventFilter> eventFilter) const { - if (EventsCache_.empty()) { - for (TFrameDecoder decoder{*this, eventFilter.Get()}; decoder.Avail(); decoder.Next()) { - EventsCache_.emplace_back(*decoder); - } - } - - return TIterator(*this, eventFilter); -} - -void TFrame::ClearEventsCache() const { - EventsCache_.clear(); -} - -TString TFrame::GetCompressedFrame() const { - const auto left = Limiter_->Left(); - TString payload = Limiter_->ReadAll(); - Y_ENSURE(payload.size() == left, "Could not read frame payload: premature end of stream"); - const ui32 checksum = MurmurHash<ui32>(payload.data(), payload.size()); - Y_ENSURE(checksum == Framehdr.PayloadChecksum, "Invalid frame checksum"); - - return payload; -} - -TString TFrame::GetRawFrame() const { - TString frameBuf = GetCompressedFrame(); - TStringInput sin(frameBuf); - return TZLibDecompress{&sin}.ReadAll(); -} - -TFrame::TIterator::TIterator(const TFrame& frame, TIntrusiveConstPtr<TEventFilter> filter) - : Frame_(frame) - , Size_(frame.EventsCache_.size()) - , Filter_(filter) - , Index_(0) -{ - SkipToValidEvent(); -} - -TConstEventPtr TFrame::TIterator::operator*() const { - return Frame_.GetEvent(Index_); -} - -bool TFrame::TIterator::Next() { - Index_++; - SkipToValidEvent(); - return Index_ < Size_; -} - -void TFrame::TIterator::SkipToValidEvent() { - if (!Filter_) { - return; - } - - for (; Index_ < Size_; ++Index_) { - if (Filter_->EventAllowed(Frame_.GetEvent(Index_)->Class)) { - break; - } - } -} - -TMaybe<TFrame> FindNextFrame(IInputStream* in, IEventFactory* eventFactory) { - if (auto header = FindNextFrameHeader(in)) { - return TFrame{*in, *header, eventFactory}; - } else { - return Nothing(); - } -} - -TContainsEventFrameFilter::TContainsEventFrameFilter(const TString& unparsedMatchGroups, const IEventFactory* eventFactory) { - TVector<TStringBuf> tokens; - - SplitWithEscaping(tokens, unparsedMatchGroups, "/"); - - // Amount of match groups - size_t size = tokens.size(); - MatchGroups.resize(size); - - for (size_t i = 0; i < size; i++) { - TMatchGroup& group = MatchGroups[i]; - TVector<TStringBuf> groupTokens; - SplitWithEscaping(groupTokens, tokens[i], ":"); - - Y_ENSURE(groupTokens.size() == 3); - - try { - group.EventID = eventFactory->ClassByName(groupTokens[0]); - } catch (yexception& e) { - if (!TryFromString<TEventClass>(groupTokens[0], group.EventID)) { - e << "\nAppend:\n" << "Cannot derive EventId from EventType: " << groupTokens[0]; - throw e; - } - } - - group.FieldName = groupTokens[1]; - group.ValueToMatch = UnescapeCharacters(groupTokens[2], "/:"); - } -} - -bool TContainsEventFrameFilter::FrameAllowed(const TFrame& frame) const { - THashSet<size_t> toMatchSet; - for (size_t i = 0; i < MatchGroups.size(); i++) { - toMatchSet.insert(i); - } - - for (auto it = frame.GetIterator(); it.Avail(); it.Next()) { - TConstEventPtr event(*it); - TVector<size_t> indicesToErase; - - if (!toMatchSet.empty()) { - const NProtoBuf::Message* message = event->GetProto(); - const google::protobuf::Descriptor* descriptor = message->GetDescriptor(); - const google::protobuf::Reflection* reflection = message->GetReflection(); - - Y_ENSURE(descriptor); - Y_ENSURE(reflection); - - for (size_t groupIndex : toMatchSet) { - const TMatchGroup& group = MatchGroups[groupIndex]; - - if (event->Class == group.EventID) { - TVector<TString> parts = StringSplitter(group.FieldName).Split('.').ToList<TString>(); - TString lastPart = std::move(parts.back()); - parts.pop_back(); - - for (auto part : parts) { - auto fieldDescriptor = descriptor->FindFieldByName(part); - Y_ENSURE(fieldDescriptor, "Cannot find field \"" + part + "\". Full fieldname is \"" + group.FieldName + "\"."); - - message = &reflection->GetMessage(*message, fieldDescriptor); - descriptor = message->GetDescriptor(); - reflection = message->GetReflection(); - - Y_ENSURE(descriptor); - Y_ENSURE(reflection); - } - - const google::protobuf::FieldDescriptor* fieldDescriptor = descriptor->FindFieldByName(lastPart); - Y_ENSURE(fieldDescriptor, "Cannot find field \"" + lastPart + "\". Full fieldname is \"" + group.FieldName + "\"."); - - TString fieldValue = GetEventFieldAsString(message, fieldDescriptor, reflection); - if (re2::RE2::FullMatch(fieldValue, group.ValueToMatch)) { - indicesToErase.push_back(groupIndex); - } - } - } - - for (size_t idx : indicesToErase) { - toMatchSet.erase(idx); - } - - if (toMatchSet.empty()) { - return true; - } - } - } - - return toMatchSet.empty(); -} - -void SplitWithEscaping(TVector<TStringBuf>& tokens, const TStringBuf& stringToSplit, const TStringBuf& externalCharacterSet) { - size_t tokenStart = 0; - const TString characterSet = TString::Join("\\", externalCharacterSet); - - for (size_t position = stringToSplit.find_first_of(characterSet); position != TString::npos; position = stringToSplit.find_first_of(characterSet, position + 1)) { - if (stringToSplit[position] == '\\') { - position++; - } else { - if (tokenStart != position) { - tokens.push_back(TStringBuf(stringToSplit, tokenStart, position - tokenStart)); - } - tokenStart = position + 1; - } - } - - if (tokenStart < stringToSplit.size()) { - tokens.push_back(TStringBuf(stringToSplit, tokenStart, stringToSplit.size() - tokenStart)); - } -} - -TString UnescapeCharacters(const TStringBuf& stringToUnescape, const TStringBuf& characterSet) { - TStringBuilder stringBuilder; - size_t tokenStart = 0; - - for (size_t position = stringToUnescape.find('\\', 0u); position != TString::npos; position = stringToUnescape.find('\\', position + 2)) { - if (position + 1 < stringToUnescape.size() && characterSet.find(stringToUnescape[position + 1]) != TString::npos) { - stringBuilder << TStringBuf(stringToUnescape, tokenStart, position - tokenStart); - tokenStart = position + 1; - } - } - - if (tokenStart < stringToUnescape.size()) { - stringBuilder << TStringBuf(stringToUnescape, tokenStart, stringToUnescape.size() - tokenStart); - } - - return stringBuilder; -} - -TString GetEventFieldAsString(const NProtoBuf::Message* message, const google::protobuf::FieldDescriptor* fieldDescriptor, const google::protobuf::Reflection* reflection) { - Y_ENSURE(message); - Y_ENSURE(fieldDescriptor); - Y_ENSURE(reflection); - - TString result; - switch (fieldDescriptor->type()) { - case google::protobuf::FieldDescriptor::Type::TYPE_DOUBLE: - result = ToString(reflection->GetDouble(*message, fieldDescriptor)); - break; - case google::protobuf::FieldDescriptor::Type::TYPE_FLOAT: - result = ToString(reflection->GetFloat(*message, fieldDescriptor)); - break; - case google::protobuf::FieldDescriptor::Type::TYPE_BOOL: - result = ToString(reflection->GetBool(*message, fieldDescriptor)); - break; - case google::protobuf::FieldDescriptor::Type::TYPE_INT32: - result = ToString(reflection->GetInt32(*message, fieldDescriptor)); - break; - case google::protobuf::FieldDescriptor::Type::TYPE_UINT32: - result = ToString(reflection->GetUInt32(*message, fieldDescriptor)); - break; - case google::protobuf::FieldDescriptor::Type::TYPE_INT64: - result = ToString(reflection->GetInt64(*message, fieldDescriptor)); - break; - case google::protobuf::FieldDescriptor::Type::TYPE_UINT64: - result = ToString(reflection->GetUInt64(*message, fieldDescriptor)); - break; - case google::protobuf::FieldDescriptor::Type::TYPE_STRING: - result = ToString(reflection->GetString(*message, fieldDescriptor)); - break; - case google::protobuf::FieldDescriptor::Type::TYPE_ENUM: - { - const NProtoBuf::EnumValueDescriptor* enumValueDescriptor = reflection->GetEnum(*message, fieldDescriptor); - result = ToString(enumValueDescriptor->name()); - } - break; - default: - throw yexception() << "GetEventFieldAsString for type " << fieldDescriptor->type_name() << " is not implemented."; - } - return result; -} - -TFrameStreamer::TFrameStreamer(IInputStream& s, IEventFactory* fac, IFrameFilterRef ff) - : In_(&s) - , FrameFilter_(ff) - , EventFactory_(fac) -{ - Frame_ = FindNextFrame(&In_, EventFactory_); - - SkipToAllowedFrame(); -} - -TFrameStreamer::TFrameStreamer( - const TString& fileName, - ui64 startTime, - ui64 endTime, - ui64 maxRequestDuration, - IEventFactory* fac, - IFrameFilterRef ff) - : File_(TBlob::FromFile(fileName)) - , MemoryIn_(File_.Data(), File_.Size()) - , In_(&MemoryIn_) - , StartTime_(startTime) - , EndTime_(endTime) - , CutoffTime_(endTime + Min(maxRequestDuration, Max<ui64>() - endTime)) - , FrameFilter_(ff) - , EventFactory_(fac) -{ - In_.Skip(FindFrames(File_.AsStringBuf(), startTime, endTime, maxRequestDuration)); - Frame_ = FindNextFrame(&In_, fac); - SkipToAllowedFrame(); -} - -TFrameStreamer::~TFrameStreamer() = default; - -bool TFrameStreamer::Avail() const { - return Frame_.Defined(); -} - -const TFrame& TFrameStreamer::operator*() const { - Y_ENSURE(Frame_, "Frame streamer depleted"); - - return *Frame_; -} - -bool TFrameStreamer::Next() { - DoNext(); - SkipToAllowedFrame(); - - return Frame_.Defined(); -} - -bool TFrameStreamer::AllowedTimeRange(const TFrame& frame) const { - const bool allowedStartTime = (StartTime_ == 0) || ((StartTime_ <= frame.StartTime()) && (frame.StartTime() <= EndTime_)); - const bool allowedEndTime = (EndTime_ == 0) || ((StartTime_ <= frame.EndTime()) && (frame.EndTime() <= EndTime_)); - return allowedStartTime || allowedEndTime; -} - -bool TFrameStreamer::DoNext() { - if (!Frame_) { - return false; - } - In_.Skip(Frame_->Limiter_->Left()); - Frame_ = FindNextFrame(&In_, EventFactory_); - - if (Frame_ && CutoffTime_ > 0 && Frame_->EndTime() > CutoffTime_) { - Frame_.Clear(); - } - - return Frame_.Defined(); -} - -namespace { - struct TDecodeBuffer { - TDecodeBuffer(const TString codec, IInputStream& src, size_t bs) { - TBuffer from(bs); - - { - TBufferOutput b(from); - TransferData(&src, &b); - } - - NBlockCodecs::Codec(codec)->Decode(from, DecodeBuffer); - } - - explicit TDecodeBuffer(IInputStream& src) { - TBufferOutput b(DecodeBuffer); - TransferData(&src, &b); - } - - TBuffer DecodeBuffer; - }; - - class TBlockCodecStream: private TDecodeBuffer, public TBufferInput { - public: - TBlockCodecStream(const TString codec, IInputStream& src, size_t bs) - : TDecodeBuffer(codec, src, bs) - , TBufferInput(DecodeBuffer) - {} - - explicit TBlockCodecStream(IInputStream& src) - : TDecodeBuffer(src) - , TBufferInput(DecodeBuffer) - {} - }; -} - -TFrameDecoder::TFrameDecoder(const TFrame& fr, const TEventFilter* const filter, bool strict, bool withRawData) - : Frame_(fr) - , Event_(nullptr) - , Flt_(filter) - , Fac_(fr.Fac_) - , EndOfFrame_(new TEndOfFrameEvent(Frame_.EndTime())) - , Strict_(strict) - , WithRawData_(withRawData) -{ - switch (fr.LogFormat()) { - case COMPRESSED_LOG_FORMAT_V2: - case COMPRESSED_LOG_FORMAT_V3: - case COMPRESSED_LOG_FORMAT_V4: - case COMPRESSED_LOG_FORMAT_V5: { - const auto payload = fr.GetCompressedFrame(); - TMemoryInput payloadInput{payload}; - - if (fr.LogFormat() == COMPRESSED_LOG_FORMAT_V5) { - Decompressor_.Reset(new TBlockCodecStream("zstd_1", payloadInput, payload.size())); - } else { - TZLibDecompress zlib(&payloadInput); - Decompressor_.Reset(new TBlockCodecStream(zlib)); - if (fr.LogFormat() == COMPRESSED_LOG_FORMAT_V4) { - Decompressor_.Reset(new TBlockCodecStream("lz4hc", *Decompressor_, payload.size())); - } - } - - break; - } - - default: - ythrow yexception() << "unsupported log format: " << fr.LogFormat() << Endl; - break; - }; - - if (WithRawData_) { - TBufferOutput out(UncompressedData_); - TLengthLimitedInput limiter(Decompressor_.Get(), fr.Framehdr.UncompressedDatalen); - - TransferData(&limiter, &out); - Decompressor_.Reset(new TMemoryInput(UncompressedData_.data(), UncompressedData_.size())); - } - - Limiter_.Reset(new TLengthLimitedInput(Decompressor_.Get(), fr.Framehdr.UncompressedDatalen)); - - Decode(); -} - -TFrameDecoder::~TFrameDecoder() = default; - -bool TFrameDecoder::Avail() const { - return HaveData(); -} - -TConstEventPtr TFrameDecoder::operator*() const { - Y_ENSURE(HaveData(), "Decoder depleted"); - - return Event_; -} - -bool TFrameDecoder::Next() { - if (HaveData()) { - Decode(); - } - - return HaveData(); -} - -void TFrameDecoder::Decode() { - Event_ = nullptr; - const bool framed = (Frame_.LogFormat() == COMPRESSED_LOG_FORMAT_V3) || (Frame_.LogFormat() == COMPRESSED_LOG_FORMAT_V4 || Frame_.LogFormat() == COMPRESSED_LOG_FORMAT_V5); - - size_t evBegin = 0; - size_t evEnd = 0; - if (WithRawData_) - evBegin = UncompressedData_.Size() - Limiter_->Left(); - - while (Limiter_->Left() && !(Event_ = DecodeEvent(*Limiter_, framed, Frame_.Address(), Flt_, Fac_, Strict_).Release())) { - } - - if (WithRawData_) { - evEnd = UncompressedData_.Size() - Limiter_->Left(); - RawEventData_ = TStringBuf(UncompressedData_.data() + evBegin, UncompressedData_.data() + evEnd); - } - - if (!Event_ && (!Flt_ || (Flt_->EventAllowed(TEndOfFrameEvent::EventClass)))) { - Event_ = EndOfFrame_.Release(); - } - - if (!!Event_) { - Event_->FrameId = Frame_.FrameId(); - } -} - -const TStringBuf TFrameDecoder::GetRawEvent() const { - return RawEventData_; -} - -TEventStreamer::TEventStreamer(TFrameStream& fs, ui64 s, ui64 e, bool strongOrdering, TIntrusivePtr<TEventFilter> filter, bool losslessStrongOrdering) - : Frames_(fs) - , Start_(s) - , End_(e) - , MaxEndTimestamp_(0) - , Frontier_(0) - , StrongOrdering_(strongOrdering) - , LosslessStrongOrdering_(losslessStrongOrdering) - , EventFilter_(filter) -{ - - if (Start_ > End_) { - ythrow yexception() << "Wrong main interval"; - } - - TEventStreamer::Next(); -} - -TEventStreamer::~TEventStreamer() = default; - -bool TEventStreamer::Avail() const { - return Events_.Avail() && (*Events_)->Timestamp <= Frontier_; -} - -TConstEventPtr TEventStreamer::operator*() const { - Y_ENSURE(TEventStreamer::Avail(), "Event streamer depleted"); - - return *Events_; -} - -bool TEventStreamer::Next() { - if (Events_.Avail() && Events_.Next() && (*Events_)->Timestamp <= Frontier_) { - return true; - } - - for (;;) { - if (!LoadMoreEvents()) { - return false; - } - - if (TEventStreamer::Avail()) { - return true; - } - } -} - -/* -Two parameters are used in the function: -Frontier - the moment of time up to which inclusively all the log events made their way - into the buffer (and might have been already extracted out of it). -Horizon - the moment of time, that equals to Frontier + MAX_REQUEST_DURATION. -In order to get all the log events up to the Frontier inclusively, - frames need to be read until "end time" of the current frame exceeds the Horizon. -*/ -bool TEventStreamer::LoadMoreEvents() { - if (!Frames_.Avail()) { - return false; - } - - const TFrame& fr1 = *Frames_; - const ui64 maxRequestDuration = (StrongOrdering_ ? MAX_REQUEST_DURATION : 0); - - if (fr1.EndTime() <= Frontier_ + maxRequestDuration) { - ythrow yexception() << "Wrong frame stream state"; - } - - if (Frontier_ >= End_) { - return false; - } - - const ui64 old_frontier = Frontier_; - Frontier_ = fr1.EndTime(); - - { - Y_DEFER { - Events_.Reorder(StrongOrdering_); - }; - - for (; Frames_.Avail(); Frames_.Next()) { - const TFrame& fr2 = *Frames_; - - // Frames need to start later than the Frontier. - if (StrongOrdering_ && fr2.StartTime() <= old_frontier) { - Cdbg << "Invalid frame encountered" << Endl; - continue; - } - - if (fr2.EndTime() > MaxEndTimestamp_) { - MaxEndTimestamp_ = fr2.EndTime(); - } - - if (fr2.EndTime() > Frontier_ + maxRequestDuration && !LosslessStrongOrdering_) { - return true; - } - - // Checking for the frame to be within the main time borders. - if (fr2.EndTime() >= Start_ && fr2.StartTime() <= End_) { - TransferEvents(fr2); - } - } - } - - Frontier_ = MaxEndTimestamp_; - - return true; -} - -void TEventStreamer::TransferEvents(const TFrame& fr) { - Events_.SetCheckpoint(); - - try { - for (auto it = fr.GetIterator(EventFilter_); it.Avail(); it.Next()) { - TConstEventPtr ev = *it; - - if (ev->Timestamp > fr.EndTime() || ev->Timestamp < fr.StartTime()) { - ythrow TInvalidEventTimestamps() << "Event timestamp out of frame range"; - } - - if (ev->Timestamp >= Start_ && ev->Timestamp <= End_) { - Events_.Append(ev, StrongOrdering_); - } - } - } catch (const TInvalidEventTimestamps& err) { - Events_.Rollback(); - Cdbg << "EventsTransfer error: InvalidEventTimestamps: " << err.what() << Endl; - } catch (const TFrameLoadError& err) { - Events_.Rollback(); - Cdbg << "EventsTransfer error: " << err.what() << Endl; - } catch (const TEventDecoderError& err) { - Events_.Rollback(); - Cdbg << "EventsTransfer error: EventDecoder error: " << err.what() << Endl; - } catch (const TZLibDecompressorError& err) { - Events_.Rollback(); - Cdbg << "EventsTransfer error: ZLibDecompressor error: " << err.what() << Endl; - } catch (...) { - Events_.Rollback(); - throw; - } -} - -void TEventStreamer::TEventBuffer::SetCheckpoint() { - BufLen_ = Buffer_.size(); -} - -void TEventStreamer::TEventBuffer::Rollback() { - Buffer_.resize(BufLen_); -} - -void TEventStreamer::TEventBuffer::Reorder(bool strongOrdering) { - SetCheckpoint(); - - std::reverse(Buffer_.begin(), Buffer_.end()); - - if (strongOrdering) { - StableSort(Buffer_.begin(), Buffer_.end(), [&](const auto& a, const auto& b) { - return (a->Timestamp > b->Timestamp) || - ((a->Timestamp == b->Timestamp) && !a->Class && b->Class); - }); - } -} - -void TEventStreamer::TEventBuffer::Append(TConstEventPtr ev, bool strongOrdering) { - // Events in buffer output must be in an ascending order. - Y_ENSURE(!strongOrdering || ev->Timestamp >= LastTimestamp_, "Trying to append out-of-order event"); - - Buffer_.push_back(std::move(ev)); -} - -bool TEventStreamer::TEventBuffer::Avail() const { - return !Buffer_.empty(); -} - -TConstEventPtr TEventStreamer::TEventBuffer::operator*() const { - Y_ENSURE(!Buffer_.empty(), "Event buffer is empty"); - - return Buffer_.back(); -} - -bool TEventStreamer::TEventBuffer::Next() { - if (!Buffer_.empty()) { - LastTimestamp_ = Buffer_.back()->Timestamp; - Buffer_.pop_back(); - return !Buffer_.empty(); - } else { - return false; - } -} diff --git a/library/cpp/eventlog/logparser.h b/library/cpp/eventlog/logparser.h deleted file mode 100644 index f819e72589..0000000000 --- a/library/cpp/eventlog/logparser.h +++ /dev/null @@ -1,343 +0,0 @@ -#pragma once - -#include <util/generic/ptr.h> -#include <util/generic/yexception.h> -#include <util/generic/vector.h> -#include <util/generic/set.h> -#include <util/generic/maybe.h> -#include <util/memory/blob.h> -#include <util/stream/length.h> -#include <util/stream/mem.h> - -#include "eventlog_int.h" -#include "eventlog.h" -#include "common.h" - -class IInputStream; - -static const ui64 MAX_REQUEST_DURATION = 60'000'000; -static const ui64 MIN_START_TIME = MAX_REQUEST_DURATION; -static const ui64 MAX_END_TIME = ((ui64)-1) - MAX_REQUEST_DURATION; - -class TEventFilter: public TSet<TEventClass>, public TSimpleRefCount<TEventFilter> { -public: - TEventFilter(bool enableEvents) - : Enable_(enableEvents) - { - } - - void AddEventClass(TEventClass cls) { - insert(cls); - } - - bool EventAllowed(TEventClass cls) const { - bool found = (find(cls) != end()); - - return Enable_ == found; - } - -private: - bool Enable_; -}; - -using TEventStream = TPacketInputStream<TConstEventPtr>; - -struct TFrameHeader { - // Reads header from the stream. The caller must make sure that the - // sync data is present just befor the stream position. - explicit TFrameHeader(IInputStream& in); - - ui64 StartTime() const { - return Framehdr.StartTimestamp; - } - - ui64 EndTime() const { - return Framehdr.EndTimestamp; - } - - ui32 FrameId() const { - return Basehdr.FrameId; - } - - ui64 Duration() const { - return EndTime() - StartTime(); - } - - TEventLogFormat ContentFormat() const { - return Basehdr.Format & 0xffffff; - } - - TEventLogFormat LogFormat() const { - return Basehdr.Format >> 24; - } - - ui64 FrameLength() const { - return Basehdr.Length - sizeof(TCompressedFrameHeader2); - } - - // Length including the header - ui64 FullLength() const { - return sizeof(*this) + FrameLength(); - } - - TCompressedFrameBaseHeader Basehdr; - TCompressedFrameHeader2 Framehdr; -}; - -struct TFrameLoadError: public yexception { - explicit TFrameLoadError(size_t skipAfter) - : SkipAfter(skipAfter) - {} - - size_t SkipAfter; -}; - -class TFrame : public TFrameHeader { -public: - // Reads the frame after the header has been read. - TFrame(IInputStream& in, TFrameHeader header, IEventFactory*); - - TString GetRawFrame() const; - TString GetCompressedFrame() const; - - ui64 Address() const { return Address_; } - -private: - const TConstEventPtr& GetEvent(size_t index) const { - return EventsCache_[index]; - } - - void ClearEventsCache() const; - - THolder<TLengthLimitedInput> Limiter_; - mutable TVector<TConstEventPtr> EventsCache_; - - IEventFactory* Fac_; - ui64 Address_; - - friend class TFrameDecoder; - friend class TFrameStreamer; - -private: - class TIterator: TEventStream { - public: - TIterator(const TFrame& frame, TIntrusiveConstPtr<TEventFilter> filter); - ~TIterator() override = default; - - bool Avail() const override { - return Index_ < Size_; - } - - TConstEventPtr operator*() const override; - bool Next() override; - - private: - void SkipToValidEvent(); - - const TFrame& Frame_; - size_t Size_; - TIntrusiveConstPtr<TEventFilter> Filter_; - size_t Index_; - }; - -public: - TFrame::TIterator GetIterator(TIntrusiveConstPtr<TEventFilter> eventFilter = nullptr) const; -}; - -// If `in` is derived from TCountingInput, Frame's address will -// be set accorting to the in->Counter(). Otherwise it will be zeroO -TMaybe<TFrame> FindNextFrame(IInputStream* in, IEventFactory*); - -using TFrameStream = TPacketInputStream<const TFrame&>; - -class IFrameFilter: public TSimpleRefCount<IFrameFilter> { -public: - IFrameFilter() { - } - - virtual ~IFrameFilter() = default; - - virtual bool FrameAllowed(const TFrame& frame) const = 0; -}; - -using IFrameFilterRef = TIntrusivePtr<IFrameFilter>; - -class TDurationFrameFilter: public IFrameFilter { -public: - TDurationFrameFilter(ui64 minFrameDuration, ui64 maxFrameDuration = Max<ui64>()) - : MinDuration_(minFrameDuration) - , MaxDuration_(maxFrameDuration) - { - } - - bool FrameAllowed(const TFrame& frame) const override { - return frame.Duration() >= MinDuration_ && frame.Duration() <= MaxDuration_; - } - -private: - const ui64 MinDuration_; - const ui64 MaxDuration_; -}; - -class TFrameIdFrameFilter: public IFrameFilter { -public: - TFrameIdFrameFilter(ui32 frameId) - : FrameId_(frameId) - { - } - - bool FrameAllowed(const TFrame& frame) const override { - return frame.FrameId() == FrameId_; - } - -private: - const ui32 FrameId_; -}; - -class TContainsEventFrameFilter: public IFrameFilter { -public: - TContainsEventFrameFilter(const TString& args, const IEventFactory* fac); - - bool FrameAllowed(const TFrame& frame) const override; - -private: - struct TMatchGroup { - TEventClass EventID; - TString FieldName; - TString ValueToMatch; - }; - - TVector<TMatchGroup> MatchGroups; -}; - -void SplitWithEscaping(TVector<TStringBuf>& tokens, const TStringBuf& stringToSplit, const TStringBuf& externalCharacterSet); - -TString UnescapeCharacters(const TStringBuf& stringToUnescape, const TStringBuf& characterSet); - -TString GetEventFieldAsString(const NProtoBuf::Message* message, const google::protobuf::FieldDescriptor* fieldDescriptor, const google::protobuf::Reflection* reflection); - -class TFrameStreamer: public TFrameStream { -public: - TFrameStreamer(IInputStream&, IEventFactory* fac, IFrameFilterRef ff = nullptr); - TFrameStreamer( - const TString& fileName, - ui64 startTime, - ui64 endTime, - ui64 maxRequestDuration, - IEventFactory* fac, - IFrameFilterRef ff = nullptr); - ~TFrameStreamer() override; - - bool Avail() const override; - const TFrame& operator*() const override; - bool Next() override; - -private: - bool DoNext(); - bool AllowedTimeRange(const TFrame& frame) const; - - bool AllowedFrame(const TFrame& frame) const { - return AllowedTimeRange(frame) && (!FrameFilter_ || FrameFilter_->FrameAllowed(frame)); - } - - void SkipToAllowedFrame() { - if (Frame_) { - while (!AllowedFrame(*Frame_) && DoNext()) { - //do nothing - } - } - } - - TBlob File_; - TMemoryInput MemoryIn_; - TCountingInput In_; - THolder<IInputStream> Stream_; - ui64 StartTime_ = 0; - ui64 EndTime_ = 0; - ui64 CutoffTime_ = 0; - TMaybe<TFrame> Frame_; - IFrameFilterRef FrameFilter_; - IEventFactory* EventFactory_; -}; - -class TFrameDecoder: TEventStream { -public: - TFrameDecoder(const TFrame&, const TEventFilter* const filter, bool strict = false, bool withRawData = false); - ~TFrameDecoder() override; - - bool Avail() const override; - - TConstEventPtr operator*() const override; - bool Next() override; - - const TStringBuf GetRawEvent() const; - -private: - TFrameDecoder(const TFrameDecoder&); - void operator=(const TFrameDecoder&); - - inline bool HaveData() const { - return Event_ != nullptr; - } - - void Decode(); - -private: - const TFrame& Frame_; - THolder<IInputStream> Decompressor_; - THolder<TLengthLimitedInput> Limiter_; - TEventPtr Event_; - const TEventFilter* const Flt_; - IEventFactory* Fac_; - THolder<TEvent> EndOfFrame_; - bool Strict_; - TBuffer UncompressedData_; - TStringBuf RawEventData_; - bool WithRawData_; -}; - -class TEventStreamer: public TEventStream { -public: - TEventStreamer(TFrameStream&, ui64 start, ui64 end, bool strongOrdering, TIntrusivePtr<TEventFilter> filter, bool losslessStrongOrdering = false); - ~TEventStreamer() override; - - bool Avail() const override; - TConstEventPtr operator*() const override; - bool Next() override; - -private: - class TEventBuffer: public TEventStream { - public: - void SetCheckpoint(); - void Rollback(); - void Reorder(bool strongOrdering); - void Append(TConstEventPtr event, bool strongOrdering); - - bool Avail() const override; - TConstEventPtr operator*() const override; - bool Next() override; - - private: - TVector<TConstEventPtr> Buffer_; - size_t BufLen_ = 0; - ui64 LastTimestamp_ = 0; - }; - -private: - struct TInvalidEventTimestamps: public yexception { - }; - - bool LoadMoreEvents(); - void TransferEvents(const TFrame&); - -private: - TFrameStream& Frames_; - TEventBuffer Events_; - - ui64 Start_, End_; - ui64 MaxEndTimestamp_; - ui64 Frontier_; - bool StrongOrdering_; - bool LosslessStrongOrdering_; - TIntrusivePtr<TEventFilter> EventFilter_; -}; diff --git a/library/cpp/eventlog/proto/events_extension.proto b/library/cpp/eventlog/proto/events_extension.proto deleted file mode 100644 index 7db1af3a59..0000000000 --- a/library/cpp/eventlog/proto/events_extension.proto +++ /dev/null @@ -1,22 +0,0 @@ -import "google/protobuf/descriptor.proto"; - -option go_package = "github.com/ydb-platform/ydb/library/cpp/eventlog/proto;extensions"; -option java_package = "NEventLogEventsExtension"; - -extend google.protobuf.MessageOptions { - optional uint32 message_id = 50001; - optional string realm_name = 50002; -} - -message Repr { - enum ReprType { - none = 0; - as_bytes = 1; // Only for primitive types - as_hex = 2; // Only for primitive types - as_base64 = 3; // Only for 'string' and 'bytes' fields - }; -} - -extend google.protobuf.FieldOptions { - optional Repr.ReprType repr = 55003 [default = none]; -} diff --git a/library/cpp/eventlog/proto/internal.proto b/library/cpp/eventlog/proto/internal.proto deleted file mode 100644 index 8070a09685..0000000000 --- a/library/cpp/eventlog/proto/internal.proto +++ /dev/null @@ -1,9 +0,0 @@ -option go_package = "github.com/ydb-platform/ydb/library/cpp/eventlog/proto;extensions"; - -package NEventLogInternal; - -message TUnknownEvent { -}; - -message TEndOfFrameEvent { -}; diff --git a/library/cpp/eventlog/proto/ya.make b/library/cpp/eventlog/proto/ya.make deleted file mode 100644 index fbf5a6c619..0000000000 --- a/library/cpp/eventlog/proto/ya.make +++ /dev/null @@ -1,12 +0,0 @@ -PROTO_LIBRARY() - -IF (NOT PY_PROTOS_FOR) - INCLUDE_TAGS(GO_PROTO) -ENDIF() - -SRCS( - events_extension.proto - internal.proto -) - -END() diff --git a/library/cpp/eventlog/threaded_eventlog.cpp b/library/cpp/eventlog/threaded_eventlog.cpp deleted file mode 100644 index 67839063fb..0000000000 --- a/library/cpp/eventlog/threaded_eventlog.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "threaded_eventlog.h" diff --git a/library/cpp/eventlog/threaded_eventlog.h b/library/cpp/eventlog/threaded_eventlog.h deleted file mode 100644 index 52382b856d..0000000000 --- a/library/cpp/eventlog/threaded_eventlog.h +++ /dev/null @@ -1,154 +0,0 @@ -#pragma once - -#include "eventlog.h" - -#include <util/generic/string.h> -#include <util/thread/pool.h> - -class TThreadedEventLog: public TEventLogWithSlave { -public: - class TWrapper; - using TOverflowCallback = std::function<void(TWrapper& wrapper)>; - - enum class EDegradationResult { - ShouldWrite, - ShouldDrop, - }; - using TDegradationCallback = std::function<EDegradationResult(float fillFactor)>; - -public: - TThreadedEventLog( - IEventLog& parentLog, - size_t threadCount, - size_t queueSize, - TOverflowCallback cb, - TDegradationCallback degradationCallback = {}) - : TEventLogWithSlave(parentLog) - , LogSaver(TThreadPoolParams().SetThreadName("ThreadedEventLog")) - , ThreadCount(threadCount) - , QueueSize(queueSize) - , OverflowCallback(std::move(cb)) - , DegradationCallback(std::move(degradationCallback)) - { - Init(); - } - - TThreadedEventLog( - const TEventLogPtr& parentLog, - size_t threadCount, - size_t queueSize, - TOverflowCallback cb, - TDegradationCallback degradationCallback = {}) - : TEventLogWithSlave(parentLog) - , LogSaver(TThreadPoolParams().SetThreadName("ThreadedEventLog")) - , ThreadCount(threadCount) - , QueueSize(queueSize) - , OverflowCallback(std::move(cb)) - , DegradationCallback(std::move(degradationCallback)) - { - Init(); - } - - TThreadedEventLog(IEventLog& parentLog) - : TThreadedEventLog(parentLog, 1, 0, TOverflowCallback()) - { - } - - TThreadedEventLog(const TEventLogPtr& parentLog) - : TThreadedEventLog(parentLog, 1, 0, TOverflowCallback()) - { - } - - ~TThreadedEventLog() override { - try { - LogSaver.Stop(); - } catch (...) { - } - } - - void ReopenLog() override { - TEventLogWithSlave::ReopenLog(); - } - - void CloseLog() override { - LogSaver.Stop(); - TEventLogWithSlave::CloseLog(); - } - - void WriteFrame(TBuffer& buffer, - TEventTimestamp startTimestamp, - TEventTimestamp endTimestamp, - TWriteFrameCallbackPtr writeFrameCallback = nullptr, - TLogRecord::TMetaFlags metaFlags = {}) override { - float fillFactor = 0.0f; - if (Y_LIKELY(LogSaver.GetMaxQueueSize() > 0)) { - fillFactor = static_cast<float>(LogSaver.Size()) / LogSaver.GetMaxQueueSize(); - } - - EDegradationResult status = EDegradationResult::ShouldWrite; - if (DegradationCallback) { - status = DegradationCallback(fillFactor); - } - if (Y_UNLIKELY(status == EDegradationResult::ShouldDrop)) { - return; - } - - THolder<TWrapper> wrapped; - wrapped.Reset(new TWrapper(buffer, startTimestamp, endTimestamp, Slave(), writeFrameCallback, std::move(metaFlags))); - - if (LogSaver.Add(wrapped.Get())) { - Y_UNUSED(wrapped.Release()); - } else if (OverflowCallback) { - OverflowCallback(*wrapped); - } - } - -private: - void Init() { - LogSaver.Start(ThreadCount, QueueSize); - } - -public: - class TWrapper: public IObjectInQueue { - public: - TWrapper(TBuffer& buffer, - TEventTimestamp startTimestamp, - TEventTimestamp endTimestamp, - IEventLog& slave, - TWriteFrameCallbackPtr writeFrameCallback = nullptr, - TLogRecord::TMetaFlags metaFlags = {}) - : StartTimestamp(startTimestamp) - , EndTimestamp(endTimestamp) - , Slave(&slave) - , WriteFrameCallback(writeFrameCallback) - , MetaFlags(std::move(metaFlags)) - { - Buffer.Swap(buffer); - } - - void Process(void*) override { - THolder<TWrapper> holder(this); - - WriteFrame(); - } - - void WriteFrame() { - Slave->WriteFrame(Buffer, StartTimestamp, EndTimestamp, WriteFrameCallback, std::move(MetaFlags)); - } - - private: - TBuffer Buffer; - TEventTimestamp StartTimestamp; - TEventTimestamp EndTimestamp; - IEventLog* Slave; - TWriteFrameCallbackPtr WriteFrameCallback; - TLogRecord::TMetaFlags MetaFlags; - }; - -private: - TThreadPool LogSaver; - const size_t ThreadCount; - const size_t QueueSize; - const TOverflowCallback OverflowCallback; - const TDegradationCallback DegradationCallback; -}; diff --git a/library/cpp/eventlog/ya.make b/library/cpp/eventlog/ya.make deleted file mode 100644 index fbbc1eff00..0000000000 --- a/library/cpp/eventlog/ya.make +++ /dev/null @@ -1,29 +0,0 @@ -LIBRARY() - -PEERDIR( - library/cpp/blockcodecs - library/cpp/eventlog/proto - library/cpp/json - library/cpp/logger - library/cpp/protobuf/json - library/cpp/streams/growing_file_input - library/cpp/string_utils/base64 - contrib/libs/re2 -) - -SRCS( - common.h - evdecoder.cpp - event_field_output.cpp - event_field_printer.cpp - eventlog.cpp - eventlog_int.cpp - iterator.cpp - logparser.cpp - threaded_eventlog.cpp -) - -GENERATE_ENUM_SERIALIZATION(eventlog.h) -GENERATE_ENUM_SERIALIZATION(eventlog_int.h) - -END() diff --git a/library/cpp/fieldcalc/field_calc.cpp b/library/cpp/fieldcalc/field_calc.cpp deleted file mode 100644 index 1066b5b5e6..0000000000 --- a/library/cpp/fieldcalc/field_calc.cpp +++ /dev/null @@ -1,1136 +0,0 @@ -#include <cstdio> - -#include <util/str_stl.h> -#include <util/string/subst.h> -#include <util/string/util.h> -#include <util/string/cast.h> -#include <util/stream/printf.h> - -#include "field_calc_int.h" - -using namespace std; - -enum Operators { - OP_ADD, - OP_SUBSTRACT, - OP_MULTIPLY, - OP_DIVIDE, - OP_MODULUS, - OP_REGEXP, - OP_REGEXP_NOT, - OP_LEFT_SHIFT, - OP_RIGHT_SHIFT, - OP_EQUAL, - OP_NOT_EQUAL, - OP_LESS, - OP_LESS_OR_EQUAL, - OP_GREATER, - OP_GREATER_OR_EQUAL, - OP_XOR, - OP_BITWISE_OR, - OP_BITWISE_AND, - OP_LOGICAL_OR, - OP_LOGICAL_AND, - OP_UNARY_NOT, - OP_UNARY_COMPLEMENT, - OP_UNARY_MINUS, - OP_LOG, - OP_LOG10, - OP_ROUND, - OP_ASSIGN, - OP_QUESTION, - OP_COLON, - - OP_UNKNOWN, -}; - -struct calc_op; - -struct calc_elem { - dump_item item; - char oper; - int op_prio; -}; - -struct calc_op { - dump_item Left, Right; - char Oper; - bool force_long; - bool unary; - bool is_variable; - bool string_op; // TODO -> bitop - - // for local vars - mutable bool calculated; - mutable eval_res_type result; - - calc_op(calc_elem& left, calc_elem& right) - : Left(left.item) - , Right(right.item) - , Oper(right.oper) - , is_variable(false) - , calculated(false) - , result(false) - { - force_long = Oper == OP_XOR || Oper == OP_BITWISE_OR || Oper == OP_BITWISE_AND || - Oper == OP_LOGICAL_OR || Oper == OP_LOGICAL_AND || Oper == OP_UNARY_NOT || - Oper == OP_UNARY_COMPLEMENT || Oper == OP_LEFT_SHIFT || Oper == OP_RIGHT_SHIFT || - Oper == OP_MODULUS; - unary = Oper == OP_UNARY_NOT || Oper == OP_UNARY_COMPLEMENT || Oper == OP_UNARY_MINUS || - Oper == OP_LOG || Oper == OP_LOG10 || Oper == OP_ROUND; - string_op = IsStringType(Left.type) && IsStringType(Right.type) && - (Oper == OP_REGEXP || Oper == OP_REGEXP_NOT || Oper == OP_EQUAL || Oper == OP_NOT_EQUAL || - Oper == OP_LESS || Oper == OP_LESS_OR_EQUAL || Oper == OP_GREATER || Oper == OP_GREATER_OR_EQUAL); - if (Oper == OP_REGEXP || Oper == OP_REGEXP_NOT) { - if (!string_op) - ythrow yexception() << "calc-expr: regexp requested for non-strings"; - ythrow yexception() << "calc-expr: regexps currently not supported"; - } - } - - Y_FORCE_INLINE void eval(const char** dd) const { - if (is_variable) { - if (!calculated) { - do_eval(dd); - calculated = true; - } - } else { - do_eval(dd); - } - } - -private: - Y_FORCE_INLINE void do_eval(const char** dd) const; -}; - -void calc_op::do_eval(const char** dd) const { - eval_res_type left1 = unary ? (eval_res_type) false : Left.eval(dd); - if (Oper == OP_QUESTION) { - left1.to_long(); - if (left1.res_long) { - result = Right.eval(dd); - } else { - result = eval_res_type(); // null - } - return; - } else if (Oper == OP_COLON) { - if (left1.is_null()) { - result = Right.eval(dd); - } else { - result = left1; - } - return; - } - - if (Y_UNLIKELY(string_op)) { - TStringBuf left2 = Left.GetStrBuf(dd); - TStringBuf right2 = Right.GetStrBuf(dd); - switch (Oper) { - case OP_REGEXP: - result = false; - break; - case OP_REGEXP_NOT: - result = false; - break; - case OP_EQUAL: - result = left2 == right2; - break; - case OP_NOT_EQUAL: - result = left2 != right2; - break; - case OP_LESS: - result = left2 < right2; - break; - case OP_LESS_OR_EQUAL: - result = left2 <= right2; - break; - case OP_GREATER: - result = left2 > right2; - break; - case OP_GREATER_OR_EQUAL: - result = left2 >= right2; - break; - default: - assert(false); - } - return; - } - - eval_res_type right1 = Right.eval(dd); - if (force_long) { // logical ops will be all long - left1.to_long(); - right1.to_long(); - } - switch (Oper) { - case OP_ADD: - result = left1 + right1; - break; - case OP_SUBSTRACT: - result = left1 - right1; - break; - case OP_MULTIPLY: - result = left1 * right1; - break; - case OP_DIVIDE: - result = left1 / right1; - break; - case OP_MODULUS: - result = left1.res_long ? left1.res_long % right1.res_long : 0; - break; - case OP_LEFT_SHIFT: - result = left1.res_long << right1.res_long; - break; - case OP_RIGHT_SHIFT: - result = left1.res_long >> right1.res_long; - break; - case OP_EQUAL: - result = left1 == right1; - break; - case OP_NOT_EQUAL: - result = !(left1 == right1); - break; - case OP_LESS: - result = left1 < right1; - break; - case OP_LESS_OR_EQUAL: - result = !(right1 < left1); - break; // <= - case OP_GREATER: - result = right1 < left1; - break; - case OP_GREATER_OR_EQUAL: - result = !(left1 < right1); - break; // >= - case OP_XOR: - result = left1.res_long ^ right1.res_long; - break; - case OP_BITWISE_OR: - result = left1.res_long | right1.res_long; - break; - case OP_BITWISE_AND: - result = left1.res_long & right1.res_long; - break; - case OP_LOGICAL_OR: - result = left1.res_long || right1.res_long; - break; - case OP_LOGICAL_AND: - result = left1.res_long && right1.res_long; - break; - case OP_UNARY_NOT: - result = !right1.res_long; - break; - case OP_UNARY_COMPLEMENT: - result = ~right1.res_long; - break; - case OP_UNARY_MINUS: - result = Minus(right1); - break; - case OP_LOG: - result = Log(right1); - break; - case OP_LOG10: - result = Log10(right1); - break; - case OP_ROUND: - result = Round(right1); - break; - default: - assert(false); - } -} - -namespace { - // copy-paste of fcat(TString) - // we don't want it to be too slow, yet we don't want do slow down our - // main functionality, libc fprintf, even a little - size_t Y_PRINTF_FORMAT(2, 3) fprintf(TString* s, const char* c, ...) { - TStringOutput so(*s); - - va_list params; - va_start(params, c); - const size_t ret = Printf(so, c, params); - va_end(params); - - return ret; - } - size_t Y_PRINTF_FORMAT(2, 3) fprintf(IOutputStream* s, const char* c, ...) { - va_list params; - va_start(params, c); - const size_t ret = Printf(*s, c, params); - va_end(params); - - return ret; - } -} - -template <class TOut> -void dump_item::print(TOut* p, const char** dd) const { - const char* d = dd[pack_id]; - const fake* f = reinterpret_cast<const fake*>(d); - - switch (type) { - case DIT_FAKE_ITEM: - assert(false); - break; - case DIT_MATH_RESULT: - assert(false); - break; // must call eval instead - case DIT_NAME: - assert(false); - break; // no op - - case DIT_BOOL_FIELD: - fprintf(p, *(bool*)(d + field_offset) ? "true" : "false"); - break; - case DIT_UI8_FIELD: - fprintf(p, "%u", *(ui8*)(d + field_offset)); - break; - case DIT_UI16_FIELD: - fprintf(p, "%u", *(ui16*)(d + field_offset)); - break; - case DIT_UI32_FIELD: - fprintf(p, "%u", *(ui32*)(d + field_offset)); - break; - case DIT_I64_FIELD: - fprintf(p, "%" PRId64, *(i64*)(d + field_offset)); - break; - case DIT_UI64_FIELD: - fprintf(p, "%" PRIu64, *(ui64*)(d + field_offset)); - break; - case DIT_FLOAT_FIELD: - fprintf(p, "%.4f", *(float*)(d + field_offset)); - break; - case DIT_DOUBLE_FIELD: - fprintf(p, "%.7f", *(double*)(d + field_offset)); - break; - case DIT_TIME_T32_FIELD: - fprintf(p, "%ld", (long)*(time_t32*)(d + field_offset)); - break; - case DIT_PF16UI32_FIELD: - fprintf(p, "%u", (ui32) * (pf16ui32*)(d + field_offset)); - break; - case DIT_PF16FLOAT_FIELD: - fprintf(p, "%.4f", (float)*(pf16float*)(d + field_offset)); - break; - case DIT_SF16FLOAT_FIELD: - fprintf(p, "%.4f", (float)*(sf16float*)(d + field_offset)); - break; - case DIT_STRING_FIELD: - fprintf(p, "%s", (d + field_offset)); - break; - - case DIT_LONG_CONST: - fprintf(p, "%ld", long_const); - break; - case DIT_FLOAT_CONST: - fprintf(p, "%.4f", float_const); - break; - case DIT_STR_CONST: - fprintf(p, "%.*s", (int)the_buf.size(), the_buf.data()); - break; - - case DIT_INT_FUNCTION: - fprintf(p, "%d", (f->*int_fn)()); - break; - case DIT_FLOAT_FUNCTION: - fprintf(p, "%.4f", (f->*float_fn)()); - break; - case DIT_BOOL_FUNCTION: - fprintf(p, "%d", (f->*bool_fn)()); - break; - case DIT_STR_FUNCTION: - fprintf(p, "%s", (f->*str_fn)()); - break; - case DIT_STRBUF_FUNCTION: - the_buf.clear(); - fprintf(p, "%s", (f->*strbuf_2_fn)(the_buf, nullptr)); - break; - - case DIT_UI8_EXT_FUNCTION: - fprintf(p, "%u", (*ui8_ext_fn)(f)); - break; - case DIT_UI16_EXT_FUNCTION: - fprintf(p, "%u", (*ui16_ext_fn)(f)); - break; - case DIT_UI32_EXT_FUNCTION: - fprintf(p, "%u", (*ui32_ext_fn)(f)); - break; - case DIT_UI64_EXT_FUNCTION: - fprintf(p, "%" PRIu64, (*ui64_ext_fn)(f)); - break; - - case DIT_UI8_ENUM_EQ: - fprintf(p, "%d", *(ui8*)(d + field_offset) == enum_val); - break; - case DIT_UI8_ENUM_SET: - fprintf(p, "%d", !!(*(ui8*)(d + field_offset) & enum_val)); - break; - - case DIT_UI16_ENUM_EQ: - fprintf(p, "%d", *(ui16*)(d + field_offset) == enum_val); - break; - case DIT_UI16_ENUM_SET: - fprintf(p, "%d", !!(*(ui16*)(d + field_offset) & enum_val)); - break; - - case DIT_UI32_ENUM_EQ: - fprintf(p, "%d", *(ui32*)(d + field_offset) == enum_val); - break; - case DIT_UI32_ENUM_SET: - fprintf(p, "%d", !!(*(ui32*)(d + field_offset) & enum_val)); - break; - - case DIT_INT_ENUM_FUNCTION_EQ: - fprintf(p, "%d", (ui32)(f->*int_enum_fn)() == enum_val); - break; - case DIT_INT_ENUM_FUNCTION_SET: - fprintf(p, "%d", !!(ui32)((f->*int_enum_fn)() & enum_val)); - break; - - case DIT_BOOL_FUNC_FIXED_STR: - fprintf(p, "%u", (ui32)(f->*bool_strbuf_fn)(the_buf)); - break; - case DIT_UI8_FUNC_FIXED_STR: - fprintf(p, "%u", (ui32)(f->*ui8_strbuf_fn)(the_buf)); - break; - case DIT_UI16_FUNC_FIXED_STR: - fprintf(p, "%u", (ui32)(f->*ui16_strbuf_fn)(the_buf)); - break; - case DIT_UI32_FUNC_FIXED_STR: - fprintf(p, "%u", (f->*ui32_strbuf_fn)(the_buf)); - break; - case DIT_I64_FUNC_FIXED_STR: - fprintf(p, "%" PRId64, (f->*i64_strbuf_fn)(the_buf)); - break; - case DIT_UI64_FUNC_FIXED_STR: - fprintf(p, "%" PRIu64, (f->*ui64_strbuf_fn)(the_buf)); - break; - case DIT_FLOAT_FUNC_FIXED_STR: - fprintf(p, "%.4f", (f->*float_strbuf_fn)(the_buf)); - break; - case DIT_DOUBLE_FUNC_FIXED_STR: - fprintf(p, "%.7f", (f->*double_strbuf_fn)(the_buf)); - break; - - case DIT_RESOLVE_BY_NAME: - fprintf(p, "%s", (f->*resolve_fn)(the_buf).data()); - break; - - default: - assert(false); - break; - } -} - -// instantiate, just for a case -template void dump_item::print<FILE>(FILE* p, const char** dd) const; -template void dump_item::print<TString>(TString* p, const char** dd) const; -template void dump_item::print<IOutputStream>(IOutputStream* p, const char** dd) const; - -TStringBuf dump_item::GetStrBuf(const char** dd) const { - const char* d = dd[pack_id]; - const fake* f = reinterpret_cast<const fake*>(d); - switch (type) { - case DIT_STRING_FIELD: - return d + field_offset; - case DIT_STR_CONST: - return the_buf; - case DIT_STR_FUNCTION: - return (f->*str_fn)(); - case DIT_STRBUF_FUNCTION: - the_buf.clear(); - return (f->*strbuf_2_fn)(the_buf, nullptr); - case DIT_RESOLVE_BY_NAME: - return (f->*resolve_fn)(the_buf); - default: - assert(false); - return TStringBuf(); - } -} - -// recursive -eval_res_type dump_item::eval(const char** dd) const { - const char* d = dd[pack_id]; - const fake* f = reinterpret_cast<const fake*>(d); - - switch (type) { - case DIT_FAKE_ITEM: - assert(false); - return (long int)0; - case DIT_MATH_RESULT: - this->op->eval(dd); - return this->op->result; - case DIT_NAME: - assert(false); - return (long int)0; - - case DIT_BOOL_FIELD: - return (ui32) * (bool*)(d + field_offset); - case DIT_UI8_FIELD: - return (ui32) * (ui8*)(d + field_offset); - case DIT_UI16_FIELD: - return (ui32) * (ui16*)(d + field_offset); - case DIT_UI32_FIELD: - return (ui32) * (ui32*)(d + field_offset); - case DIT_I64_FIELD: - return (long)*(i64*)(d + field_offset); // TODO: 64 bit support in calculator? - case DIT_UI64_FIELD: - return (long)*(ui64*)(d + field_offset); // TODO: 64 bit support in calculator? - case DIT_FLOAT_FIELD: - return (float)*(float*)(d + field_offset); - case DIT_DOUBLE_FIELD: - return *(double*)(d + field_offset); - case DIT_TIME_T32_FIELD: - return (long)*(time_t32*)(d + field_offset); - case DIT_PF16UI32_FIELD: - return (ui32) * (pf16ui32*)(d + field_offset); - case DIT_PF16FLOAT_FIELD: - return (float)*(pf16float*)(d + field_offset); - case DIT_SF16FLOAT_FIELD: - return (float)*(sf16float*)(d + field_offset); - case DIT_STRING_FIELD: - return !!d[field_offset]; // we don't have any string functions, just 0 if empty - - case DIT_LONG_CONST: - return long_const; - case DIT_FLOAT_CONST: - return float_const; - case DIT_STR_CONST: - return !!the_buf; - - case DIT_INT_FUNCTION: - return (long)(f->*int_fn)(); - case DIT_FLOAT_FUNCTION: - return (float)(f->*float_fn)(); - case DIT_BOOL_FUNCTION: - return (long)(f->*bool_fn)(); - case DIT_STR_FUNCTION: - return !!*(f->*str_fn)(); // string -> int - case DIT_STRBUF_FUNCTION: - the_buf.clear(); - return !!*(f->*strbuf_2_fn)(the_buf, nullptr); // string -> 0/1 - - case DIT_UI8_EXT_FUNCTION: - return (ui32)(*ui8_ext_fn)(f); - case DIT_UI16_EXT_FUNCTION: - return (ui32)(*ui16_ext_fn)(f); - case DIT_UI32_EXT_FUNCTION: - return (ui32)(*ui32_ext_fn)(f); - case DIT_UI64_EXT_FUNCTION: - return (long)(*ui64_ext_fn)(f); // TODO: 64 bit support in calculator? - - case DIT_UI8_ENUM_EQ: - return (ui32)(*(ui8*)(d + field_offset) == enum_val); - case DIT_UI8_ENUM_SET: - return !!(ui32)(*(ui8*)(d + field_offset) & enum_val); - - case DIT_UI16_ENUM_EQ: - return (ui32)(*(ui16*)(d + field_offset) == enum_val); - case DIT_UI16_ENUM_SET: - return !!(ui32)(*(ui16*)(d + field_offset) & enum_val); - - case DIT_UI32_ENUM_EQ: - return (ui32)(*(ui32*)(d + field_offset) == enum_val); - case DIT_UI32_ENUM_SET: - return !!(ui32)(*(ui32*)(d + field_offset) & enum_val); - - case DIT_INT_ENUM_FUNCTION_EQ: - return (ui32)((ui32)(f->*int_enum_fn)() == enum_val); - case DIT_INT_ENUM_FUNCTION_SET: - return !!(ui32)((ui32)(f->*int_enum_fn)() & enum_val); - - case DIT_BOOL_FUNC_FIXED_STR: - return (ui32)(f->*bool_strbuf_fn)(the_buf); - case DIT_UI8_FUNC_FIXED_STR: - return (ui32)(f->*ui8_strbuf_fn)(the_buf); - case DIT_UI16_FUNC_FIXED_STR: - return (ui32)(f->*ui16_strbuf_fn)(the_buf); - case DIT_UI32_FUNC_FIXED_STR: - return (ui32)(f->*ui32_strbuf_fn)(the_buf); - case DIT_I64_FUNC_FIXED_STR: - return (long)(f->*i64_strbuf_fn)(the_buf); - case DIT_UI64_FUNC_FIXED_STR: - return (long)(f->*ui64_strbuf_fn)(the_buf); - case DIT_FLOAT_FUNC_FIXED_STR: - return (float)(f->*float_strbuf_fn)(the_buf); - case DIT_DOUBLE_FUNC_FIXED_STR: - return (double)(f->*double_strbuf_fn)(the_buf); - - case DIT_RESOLVE_BY_NAME: - return !!(f->*resolve_fn)(the_buf); - - default: - assert(false); - break; - } - - // unreached - return eval_res_type(false); -} - -void dump_item::set_arrind(int arrind) { - switch (type) { - case DIT_BOOL_FIELD: - field_offset += arrind * sizeof(bool); - break; - case DIT_UI8_FIELD: - field_offset += arrind * sizeof(ui8); - break; - case DIT_UI16_FIELD: - field_offset += arrind * sizeof(ui16); - break; - case DIT_UI32_FIELD: - field_offset += arrind * sizeof(ui32); - break; - case DIT_I64_FIELD: - field_offset += arrind * sizeof(i64); - break; - case DIT_UI64_FIELD: - field_offset += arrind * sizeof(ui64); - break; - case DIT_FLOAT_FIELD: - field_offset += arrind * sizeof(float); - break; - case DIT_DOUBLE_FIELD: - field_offset += arrind * sizeof(double); - break; - case DIT_TIME_T32_FIELD: - field_offset += arrind * sizeof(time_t32); - break; - case DIT_PF16UI32_FIELD: - field_offset += arrind * sizeof(pf16ui32); - break; - case DIT_PF16FLOAT_FIELD: - field_offset += arrind * sizeof(pf16float); - break; - case DIT_SF16FLOAT_FIELD: - field_offset += arrind * sizeof(sf16float); - break; - default: - break; - } -} - -static str_spn FieldNameChars("a-zA-Z0-9_$", true); -static str_spn MathOpChars("-+=*%/&|<>()!~^?:#", true); -static str_spn SpaceChars("\t\n\r ", true); - -TFieldCalculatorBase::TFieldCalculatorBase() { -} - -TFieldCalculatorBase::~TFieldCalculatorBase() = default; - -bool TFieldCalculatorBase::item_by_name(dump_item& it, const char* name) const { - for (size_t i = 0; i < named_dump_items.size(); i++) { - const named_dump_item* list = named_dump_items[i].first; - size_t sz = named_dump_items[i].second; - for (unsigned int n = 0; n < sz; n++) { - if (!stricmp(name, list[n].name)) { - it = list[n].item; - it.pack_id = i; - return true; - } - } - } - return false; -} - -bool TFieldCalculatorBase::get_local_var(dump_item& dst, char* var_name) { - TMap<const char*, dump_item>::const_iterator it = local_vars.find(var_name); - if (it == local_vars.end()) { - // New local variable - dst.type = DIT_LOCAL_VARIABLE; - dst.local_var_name = pool.append(var_name); - return false; - } else { - dst = it->second; - return true; - } -} - -char* TFieldCalculatorBase::get_field(dump_item& dst, char* s) { - if (!stricmp(s, "name")) { - dst.type = DIT_NAME; - return s + 4; // leave there 0 - } - - if (*s == '"' || *s == '\'') { - char* end = strchr(s + 1, *s); - bool hasEsc = false; - while (end && end > s + 1 && end[-1] == '\\') { - end = strchr(end + 1, *s); - hasEsc = true; - } - if (!end) - ythrow yexception() << "calc-expr: unterminated string constant at " << s; - dst.type = DIT_STR_CONST; - dst.the_buf.assign(s + 1, end); - if (hasEsc) - SubstGlobal(dst.the_buf, *s == '"' ? "\\\"" : "\\'", *s == '"' ? "\"" : "'"); - dst.set_arrind(0); // just for a case - return end + 1; - } - - bool is_number = isdigit((ui8)*s) || (*s == '+' || *s == '-') && isdigit((ui8)s[1]), is_float = false; - char* end = FieldNameChars.cbrk(s + is_number); - if (is_number && *end == '.') { - is_float = true; - end = FieldNameChars.cbrk(end + 1); - } - char* next = SpaceChars.cbrk(end); - int arr_index = 0; - bool has_arr_index = false; - if (*next == '[') { - arr_index = atoi(next + 1); - has_arr_index = true; - next = strchr(next, ']'); - if (!next) - ythrow yexception() << "calc-expr: No closing ']' for '" << s << "'"; - next = SpaceChars.cbrk(next + 1); - } - char end_sav = *end; - *end = 0; - - if (!item_by_name(dst, s)) { - if (!is_number) { - get_local_var(dst, s); - } else if (is_float) { - dst = (float)strtod(s, nullptr); - } else - dst = strtol(s, nullptr, 10); - - dst.pack_id = 0; - *end = end_sav; - return next; - } - - // check array/not array - if (has_arr_index && !dst.is_array_field()) - ythrow yexception() << "calc-expr: field " << s << " is not an array"; - - //if (!has_arr_index && dst.is_array_field()) - // yexception("calc-expr: field %s is array, index required", s); - - if (has_arr_index && (arr_index < 0 || arr_index >= dst.arr_length)) - ythrow yexception() << "calc-expr: array index [" << arr_index << "] is out of range for field " << s << " (length is " << dst.arr_length << ")"; - - *end = end_sav; - dst.set_arrind(arr_index); - return next; -} - -// BEGIN Stack calculator functions -inline char* skipspace(char* c, int& bracket_depth) { - while ((ui8)*c <= ' ' && *c || *c == '(' || *c == ')') { - if (*c == '(') - bracket_depth++; - else if (*c == ')') - bracket_depth--; - c++; - } - return c; -} - -void ensure_defined(const dump_item& item) { - if (item.type == DIT_LOCAL_VARIABLE) { - ythrow yexception() << "Usage of non-defined field or local variable '" << item.local_var_name << "'"; - } -} - -void TFieldCalculatorBase::emit_op(TVector<calc_op>& ops, calc_elem& left, calc_elem& right) { - int out_op = ops.size(); - char oper = right.oper; - ensure_defined(right.item); - if (oper == OP_ASSIGN) { - if (left.item.type != DIT_LOCAL_VARIABLE) { - ythrow yexception() << "Assignment only to local variables is allowed"; - } - if (local_vars.find(left.item.local_var_name) != local_vars.end()) { - ythrow yexception() << "Reassignment to the local variable " << left.item.local_var_name << " is not allowed"; - } - local_vars[left.item.local_var_name] = right.item; - if (right.item.type == DIT_MATH_RESULT) { - calc_ops[right.item.arr_ind].is_variable = true; - } - left = right; - } else { - ensure_defined(left.item); - ops.push_back(calc_op(left, right)); - left.item.type = DIT_MATH_RESULT; - left.item.arr_ind = out_op; - } -} - -inline int get_op_prio(char c) { - switch (c) { - case OP_ASSIGN: - return 1; - case OP_QUESTION: - case OP_COLON: - return 2; - case OP_LOGICAL_OR: - return 3; - case OP_LOGICAL_AND: - return 4; - case OP_BITWISE_OR: - return 5; - case OP_XOR: - return 6; - case OP_BITWISE_AND: - return 7; - case OP_EQUAL: - case OP_NOT_EQUAL: - return 8; - case OP_LESS: - case OP_LESS_OR_EQUAL: - case OP_GREATER: - case OP_GREATER_OR_EQUAL: - return 9; - case OP_LEFT_SHIFT: - case OP_RIGHT_SHIFT: - return 10; - case OP_ADD: - case OP_SUBSTRACT: - return 11; - case OP_MULTIPLY: - case OP_DIVIDE: - case OP_MODULUS: - return 12; - case OP_REGEXP: - case OP_REGEXP_NOT: - return 13; - case OP_UNARY_NOT: - case OP_UNARY_COMPLEMENT: - case OP_UNARY_MINUS: - case OP_LOG: - case OP_LOG10: - case OP_ROUND: - return 14; - default: - return 0; - } -} - -Operators get_oper(char*& c, bool unary_op_near) { - Operators cur_oper = OP_UNKNOWN; - switch (*c++) { - case '&': - if (*c == '&') - cur_oper = OP_LOGICAL_AND, c++; - else - cur_oper = OP_BITWISE_AND; - break; - case '|': - if (*c == '|') - cur_oper = OP_LOGICAL_OR, c++; - else - cur_oper = OP_BITWISE_OR; - break; - case '<': - if (*c == '=') - cur_oper = OP_LESS_OR_EQUAL, c++; - else if (*c == '<') - cur_oper = OP_LEFT_SHIFT, c++; - else - cur_oper = OP_LESS; - break; - case '>': - if (*c == '=') - cur_oper = OP_GREATER_OR_EQUAL, c++; - else if (*c == '>') - cur_oper = OP_RIGHT_SHIFT, c++; - else - cur_oper = OP_GREATER; - break; - case '!': - if (*c == '=') - cur_oper = OP_NOT_EQUAL, c++; - else if (*c == '~') - cur_oper = OP_REGEXP_NOT, c++; - else - cur_oper = OP_UNARY_NOT; - break; - case '=': - if (*c == '=') - cur_oper = OP_EQUAL, c++; - else if (*c == '~') - cur_oper = OP_REGEXP, c++; - else - cur_oper = OP_ASSIGN; - break; - case '-': - if (unary_op_near) - cur_oper = OP_UNARY_MINUS; - else - cur_oper = OP_SUBSTRACT; - break; - case '#': - if (!strncmp(c, "LOG#", 4)) { - cur_oper = OP_LOG; - c += 4; - } else if (!strncmp(c, "LOG10#", 6)) { - cur_oper = OP_LOG10; - c += 6; - } else if (!strncmp(c, "ROUND#", 6)) { - cur_oper = OP_ROUND; - c += 6; - } - break; - case '+': - cur_oper = OP_ADD; - break; - case '*': - cur_oper = OP_MULTIPLY; - break; - case '/': - cur_oper = OP_DIVIDE; - break; - case '%': - cur_oper = OP_MODULUS; - break; - case '^': - cur_oper = OP_XOR; - break; - case '~': - cur_oper = OP_UNARY_COMPLEMENT; - break; - case '?': - cur_oper = OP_QUESTION; - break; - case ':': - cur_oper = OP_COLON; - break; - } - return cur_oper; -} -// END Stack calculator functions - -void TFieldCalculatorBase::Compile(char** field_names, int field_count) { - out_el = 0, out_cond = 0; - autoarray<dump_item>(field_count).swap(printouts); - autoarray<dump_item>(field_count).swap(conditions); - local_vars.clear(); - - // parse arguments into calculator's "pseudo-code" - for (int el = 0; el < field_count; el++) { - char* c = field_names[el]; - bool is_expr = !!*MathOpChars.brk(c), is_cond = *c == '?'; - if (is_cond) - c++; - if (!is_expr && !is_cond) { - get_field(printouts[out_el], c); - ensure_defined(printouts[out_el]); - ++out_el; - continue; - } else { // Stack Calculator - const int maxstack = 64; - calc_elem fstack[maxstack]; // calculator's stack - int bdepth = 0; // brackets depth - int stack_cur = -1; - bool unary_op_near = false; // indicates that the next operator in unary - bool had_assignment_out_of_brackets = false; - int uop_seq = 0; // maintains right-to left order for unary operators - while (*(c = skipspace(c, bdepth))) { - /** https://wiki.yandex.ru/JandeksPoisk/Antispam/OwnersData/attselect#calc */ - //printf("1.%i c = '%s'\n", unary_op_near, c); - Operators cur_oper = OP_UNKNOWN; - int op_prio = 0; - if (stack_cur >= 0) { - cur_oper = get_oper(c, unary_op_near); - op_prio = get_op_prio(cur_oper); - if (!op_prio) - ythrow yexception() << "calc-expr: Unsupported operator '" << c[-1] << "'"; - op_prio += bdepth * 256 + uop_seq; - if (unary_op_near) - uop_seq += 20; - while (op_prio <= fstack[stack_cur].op_prio && stack_cur > 0) { - emit_op(calc_ops, fstack[stack_cur - 1], fstack[stack_cur]); - stack_cur--; - } - } - //printf("2.%i c = '%s'\n", unary_op_near, c); - had_assignment_out_of_brackets |= (bdepth == 0 && cur_oper == OP_ASSIGN); - c = skipspace(c, bdepth); - unary_op_near = *c == '-' && !isdigit((ui8)c[1]) || *c == '~' || (*c == '!' && c[1] != '=') || - !strncmp(c, "#LOG#", 5) || !strncmp(c, "#LOG10#", 7) || !strncmp(c, "#ROUND#", 7); - if (!unary_op_near) - uop_seq = 0; - if (stack_cur >= maxstack - 1) - ythrow yexception() << "calc-expr: Math eval stack overflow!\n"; - stack_cur++; - fstack[stack_cur].oper = cur_oper; - fstack[stack_cur].op_prio = op_prio; - //printf("3.%i c = '%s'\n", unary_op_near, c); - if (unary_op_near) - fstack[stack_cur].item = dump_item(); - else - c = get_field(fstack[stack_cur].item, c); - } - while (stack_cur > 0) { - emit_op(calc_ops, fstack[stack_cur - 1], fstack[stack_cur]); - stack_cur--; - } - ensure_defined(fstack[0].item); - if (is_cond) { - if (had_assignment_out_of_brackets) - ythrow yexception() << "Assignment in condition. (Did you mean '==' instead of '='?)"; - if (fstack[0].item.type != DIT_FAKE_ITEM) // Skip empty conditions: "?()". - conditions[out_cond++] = fstack[0].item; - } else if (!had_assignment_out_of_brackets) { - printouts[out_el++] = fstack[0].item; - } - } - } - // calc_ops will not grow any more, so arr_ind -> op - for (int n = 0; n < out_cond; n++) - conditions[n].rewrite_op(calc_ops.data()); - for (int n = 0; n < out_el; n++) - printouts[n].rewrite_op(calc_ops.data()); - for (auto& local_var : local_vars) { - local_var.second.rewrite_op(calc_ops.data()); - } - for (int n = 0; n < (int)calc_ops.size(); n++) { - calc_ops[n].Left.rewrite_op(calc_ops.data()); - calc_ops[n].Right.rewrite_op(calc_ops.data()); - } -} - -void dump_item::rewrite_op(const calc_op* ops) { - if (type == DIT_MATH_RESULT) - op = ops + arr_ind; -} - -void TFieldCalculatorBase::MarkLocalVarsAsUncalculated() { - for (auto& local_var : local_vars) { - if (local_var.second.type == DIT_MATH_RESULT) { - local_var.second.op->calculated = false; - } - } -} - -bool TFieldCalculatorBase::Cond(const char** d) { - MarkLocalVarsAsUncalculated(); - for (int n = 0; n < out_cond; n++) { - /** https://wiki.yandex.ru/JandeksPoisk/Antispam/OwnersData/attselect#conditions */ - eval_res_type res = conditions[n].eval(d); - bool is_true = res.type == 0 ? !!res.res_ui32 : res.type == 1 ? !!res.res_long : !!res.res_dbl; - if (!is_true) - return false; - } - return true; -} - -bool TFieldCalculatorBase::CondById(const char** d, int condNumber) { - MarkLocalVarsAsUncalculated(); - if (condNumber >= out_cond) - return false; - eval_res_type res = conditions[condNumber].eval(d); - bool is_true = res.type == 0 ? !!res.res_ui32 : res.type == 1 ? !!res.res_long : !!res.res_dbl; - if (!is_true) - return false; - return true; -} - -void TFieldCalculatorBase::Print(FILE* p, const char** d, const char* Name) { - for (int n = 0; n < out_el; n++) { - if (printouts[n].type == DIT_NAME) { - fprintf(p, "%s", Name); - } else if (printouts[n].type == DIT_MATH_RESULT) { // calculate - eval_res_type res = printouts[n].eval(d); - switch (res.type) { - case 0: - fprintf(p, "%u", res.res_ui32); - break; - case 1: - fprintf(p, "%ld", res.res_long); - break; - case 2: - fprintf(p, "%f", res.res_dbl); - break; - } - } else { - printouts[n].print(p, d); - } - fprintf(p, n != out_el - 1 ? "\t" : "\n"); - } -} - -void TFieldCalculatorBase::CalcAll(const char** d, TVector<float>& result) const { - result.clear(); - for (int n = 0; n < out_el; ++n) { - if (printouts[n].type == DIT_MATH_RESULT || printouts[n].type == DIT_FLOAT_FIELD) { - eval_res_type res = printouts[n].eval(d); - result.push_back(res.res_dbl); - } - } -} - -void TFieldCalculatorBase::SelfTest() { - if (out_el < 1) - ythrow yexception() << "Please specify conditions for test mode"; - const char* dummy = ""; - eval_res_type res = printouts[0].eval(&dummy); - switch (res.type) { - case 0: - printf("%u\n", res.res_ui32); - break; - case 1: - printf("%ld\n", res.res_long); - break; - case 2: - printf("%f\n", res.res_dbl); - break; - } -} - -void TFieldCalculatorBase::PrintDiff(const char* rec1, const char* rec2) { - for (size_t n = 0; n < named_dump_items[0].second; n++) { - const dump_item& field = named_dump_items[0].first[n].item; - if (!field.is_field()) - continue; // not really a field - for (int ind = 0, arrsz = field.is_array_field() ? field.arr_length : 1; ind < arrsz; ind++) { - intptr_t sav_field_offset = field.field_offset; - const_cast<dump_item&>(field).set_arrind(ind); - if (field.eval(&rec1) == field.eval(&rec2)) { - const_cast<dump_item&>(field).field_offset = sav_field_offset; - continue; - } - if (field.is_array_field()) - printf("\t%s[%i]: ", named_dump_items[0].first[n].name, ind); - else - printf("\t%s: ", named_dump_items[0].first[n].name); - field.print(stdout, &rec1); - printf(" -> "); - field.print(stdout, &rec2); - const_cast<dump_item&>(field).field_offset = sav_field_offset; - } - } -} - -void TFieldCalculatorBase::DumpAll(IOutputStream& s, const char** d, const TStringBuf& delim) { - bool firstPrinted = false; - for (size_t k = 0; k < named_dump_items.size(); k++) { - const named_dump_item* fields = named_dump_items[k].first; - size_t numFields = named_dump_items[k].second; - const char* obj = d[k]; - for (size_t n = 0; n < numFields; n++) { - const dump_item& field = fields[n].item; - if (!field.is_field()) - continue; - for (int ind = 0, arrsz = field.is_array_field() ? field.arr_length : 1; ind < arrsz; ind++) { - if (firstPrinted) - s << delim; - else - firstPrinted = true; - s << fields[n].name; - if (field.is_array_field()) - Printf(s, "[%i]", ind); - s << "="; - intptr_t sav_field_offset = field.field_offset; - const_cast<dump_item&>(field).set_arrind(ind); - field.print(&s, &obj); - const_cast<dump_item&>(field).field_offset = sav_field_offset; - } - } - } -} diff --git a/library/cpp/fieldcalc/field_calc.h b/library/cpp/fieldcalc/field_calc.h deleted file mode 100644 index 46bf371a60..0000000000 --- a/library/cpp/fieldcalc/field_calc.h +++ /dev/null @@ -1,136 +0,0 @@ -#pragma once - -#include <cstdio> - -#include <library/cpp/deprecated/autoarray/autoarray.h> -#include <util/generic/map.h> -#include <util/generic/vector.h> -#include <util/memory/segmented_string_pool.h> - -struct dump_item; -struct calc_op; -struct named_dump_item; -struct calc_elem; -class IOutputStream; - -template <class T> -std::pair<const named_dump_item*, size_t> get_named_dump_items(); - -class TFieldCalculatorBase { -private: - segmented_string_pool pool; - void emit_op(TVector<calc_op>& ops, calc_elem& left, calc_elem& right); - void MarkLocalVarsAsUncalculated(); - -protected: - autoarray<dump_item> printouts, conditions; - int out_el, out_cond; - TVector<calc_op> calc_ops; // operands for calculator, indexed by arr_ind for DIT_math_result - - TVector<std::pair<const named_dump_item*, size_t>> named_dump_items; - TMap<const char*, dump_item> local_vars; - - char* get_field(dump_item& dst, char* s); - bool get_local_var(dump_item& dst, char* s); - virtual bool item_by_name(dump_item& it, const char* name) const; - - TFieldCalculatorBase(); - virtual ~TFieldCalculatorBase(); - - bool Cond(const char** d); - bool CondById(const char** d, int condNumber); - void Print(FILE* p, const char** d, const char* Name); - void Compile(char** field_names, int field_count); - void SelfTest(); - void PrintDiff(const char* d1, const char* d2); - void CalcAll(const char** d, TVector<float>& result) const; - void DumpAll(IOutputStream& s, const char** d, const TStringBuf& delim); -}; - -template <class T> -class TFieldCalculator: protected TFieldCalculatorBase { -public: - TFieldCalculator() { - named_dump_items.push_back(get_named_dump_items<T>()); - } - - ~TFieldCalculator() override = default; - - bool Cond(const T& d) { - const char* dd = reinterpret_cast<const char*>(&d); - return TFieldCalculatorBase::Cond(&dd); - } - - bool CondById(const T& d, int condNumber) { - const char* dd = reinterpret_cast<const char*>(&d); - return TFieldCalculatorBase::CondById(&dd, condNumber); - } - - void Print(const T& d, const char* Name) { - const char* dd = reinterpret_cast<const char*>(&d); - return TFieldCalculatorBase::Print(stdout, &dd, Name); - } - - void Print(FILE* p, const T& d, const char* Name) { - const char* dd = reinterpret_cast<const char*>(&d); - return TFieldCalculatorBase::Print(p, &dd, Name); - } - - size_t Compile(char** field_names, int field_count) { - TFieldCalculatorBase::Compile(field_names, field_count); - return out_el; // number of fields printed - } - - void SelfTest() { - return TFieldCalculatorBase::SelfTest(); - } - - void PrintDiff(const T& d1, const T& d2) { - return TFieldCalculatorBase::PrintDiff((const char*)&d1, (const char*)&d2); - } - - void CalcAll(const T& d, TVector<float>& result) const { - const char* dd = reinterpret_cast<const char*>(&d); - return TFieldCalculatorBase::CalcAll(&dd, result); - } - - // it appends to `result', clear it yourself - void DumpAll(IOutputStream& s, const T& d, const TStringBuf& delim) { - const char* dd = reinterpret_cast<const char*>(&d); - return TFieldCalculatorBase::DumpAll(s, &dd, delim); - } -}; - -template <class T, class T2> -class TFieldCalculator2: protected TFieldCalculator<T> { -public: - TFieldCalculator2() { - TFieldCalculator<T>::named_dump_items.push_back(get_named_dump_items<T2>()); - } - - ~TFieldCalculator2() override = default; - - bool Cond(const T& d, const T2& d2) { - const char* dd[2] = {reinterpret_cast<const char*>(&d), reinterpret_cast<const char*>(&d2)}; - return TFieldCalculatorBase::Cond(dd); - } - - bool CondById(const T& d, const T2& d2, int condNumber) { - const char* dd[2] = {reinterpret_cast<const char*>(&d), reinterpret_cast<const char*>(&d2)}; - return TFieldCalculatorBase::CondById(dd, condNumber); - } - - void Print(const T& d, const T2& d2, const char* Name) { - const char* dd[2] = {reinterpret_cast<const char*>(&d), reinterpret_cast<const char*>(&d2)}; - return TFieldCalculatorBase::Print(stdout, dd, Name); - } - - void Print(FILE* p, const T& d, const T2& d2, const char* Name) { - const char* dd[2] = {reinterpret_cast<const char*>(&d), reinterpret_cast<const char*>(&d2)}; - return TFieldCalculatorBase::Print(p, dd, Name); - } - - size_t Compile(char** field_names, int field_count) { - return TFieldCalculator<T>::Compile(field_names, field_count); - } -}; diff --git a/library/cpp/fieldcalc/field_calc_int.h b/library/cpp/fieldcalc/field_calc_int.h deleted file mode 100644 index 5f71fafbda..0000000000 --- a/library/cpp/fieldcalc/field_calc_int.h +++ /dev/null @@ -1,593 +0,0 @@ -#pragma once - -#include <cmath> - -#include <util/system/defaults.h> -#include <util/system/yassert.h> -#include <util/memory/alloc.h> -#include <util/generic/yexception.h> - -#include "lossy_types.h" -#include "field_calc.h" - -// eval_res_type -struct eval_res_type { - union { - ui32 res_ui32; - long res_long; - double res_dbl; - }; - int type; - eval_res_type(ui32 v) - : res_ui32(v) - , type(0) - { - } - eval_res_type(long v) - : res_long(v) - , type(1) - { - } - eval_res_type(bool v) - : res_long(v) - , type(1) - { - } - eval_res_type(double v) - : res_dbl(v) - , type(2) - { - } - // a special null value for ternary operator - explicit eval_res_type() - : type(3) - { - } - operator ui32() const; - operator long() const; - operator double() const; - void to_long(); - bool is_null() const; -}; - -inline bool eval_res_type::is_null() const { - return type == 3; -} - -inline void eval_res_type::to_long() { - if (type == 0) - res_long = res_ui32; - else if (type == 2) - res_long = (long)res_dbl; - type = 1; -} - -inline eval_res_type::operator ui32() const { - assert(type == 0); - return res_ui32; -} - -inline eval_res_type::operator long() const { - assert(type == 0 || type == 1); - return type == 1 ? res_long : res_ui32; -} - -inline eval_res_type::operator double() const { - return type == 2 ? res_dbl : type == 1 ? (double)res_long : (double)res_ui32; -} - -inline eval_res_type operator+(const eval_res_type& a, const eval_res_type& b) { - switch (std::max(a.type, b.type)) { - case 0: - return (ui32)a + (ui32)b; - case 1: - return (long)a + (long)b; - /*case 2*/ default: - return (double)a + (double)b; - } -} - -inline eval_res_type operator-(const eval_res_type& a, const eval_res_type& b) { - switch (std::max(a.type, b.type)) { - case 0: - case 1: - return (long)a - (long)b; - /*case 2*/ default: - return (double)a - (double)b; - } -} - -inline eval_res_type Minus(const eval_res_type& a) { - switch (a.type) { - case 0: - return -(long)a.res_ui32; - case 1: - return -a.res_long; - /*case 2*/ default: - return -a.res_dbl; - } -} - -inline eval_res_type Log(const eval_res_type& a) { - switch (a.type) { - case 0: - return log(a.res_ui32); - case 1: - return log(a.res_long); - /*case 2*/ default: - return log(a.res_dbl); - } -} - -inline eval_res_type Log10(const eval_res_type& a) { - switch (a.type) { - case 0: - return log10(a.res_ui32); - case 1: - return log10(a.res_long); - /*case 2*/ default: - return log10(a.res_dbl); - } -} - -inline eval_res_type Round(const eval_res_type& a) { - switch (a.type) { - case 0: - return a.res_ui32; - case 1: - return a.res_long; - /*case 2*/ default: - return round(a.res_dbl); - } -} - -inline bool operator==(const eval_res_type& a, const eval_res_type& b) { - switch (std::max(a.type, b.type)) { - case 0: - return (ui32)a == (ui32)b; - case 1: - return (long)a == (long)b; - /*case 2*/ default: - return (double)a == (double)b; - } -} - -inline bool operator<(const eval_res_type& a, const eval_res_type& b) { - switch (std::max(a.type, b.type)) { - case 0: - return (ui32)a < (ui32)b; - case 1: - return (long)a < (long)b; - /*case 2*/ default: - return (double)a < (double)b; - } -} - -inline eval_res_type operator*(const eval_res_type& a, const eval_res_type& b) { - switch (std::max(a.type, b.type)) { - case 0: - return (ui32)a * (ui32)b; - case 1: - return (long)a * (long)b; - /*case 2*/ default: - return (double)a * (double)b; - } -} - -inline double operator/(const eval_res_type& a, const eval_res_type& b) { - double a1 = a, b1 = b; - if (b1 == 0) { - if (a1 == 0) - return 0.; // assume that a should be 0 - ythrow yexception() << "Division by zero"; // TODO: show parameter names - } - return a1 / b1; -} - -// dump_item -enum EDumpItemType { - DIT_FAKE_ITEM, // fake item - value never used - DIT_MATH_RESULT, // eval result - DIT_NAME, - - DIT_FIELDS_START, // Start of item types for real fields - - DIT_BOOL_FIELD, - DIT_UI8_FIELD, - DIT_UI16_FIELD, - DIT_UI32_FIELD, - DIT_I64_FIELD, - DIT_UI64_FIELD, - DIT_FLOAT_FIELD, - DIT_DOUBLE_FIELD, - DIT_TIME_T32_FIELD, - DIT_PF16UI32_FIELD, - DIT_PF16FLOAT_FIELD, - DIT_SF16FLOAT_FIELD, - DIT_STRING_FIELD, // new - - DIT_FIELDS_END, // End of item types for real fields - - DIT_LONG_CONST, - DIT_FLOAT_CONST, - DIT_STR_CONST, - - DIT_INT_FUNCTION, - DIT_FLOAT_FUNCTION, - DIT_BOOL_FUNCTION, - DIT_STR_FUNCTION, // new - DIT_STRBUF_FUNCTION, // new - - DIT_UI8_EXT_FUNCTION, - DIT_UI16_EXT_FUNCTION, - DIT_UI32_EXT_FUNCTION, - DIT_UI64_EXT_FUNCTION, - - DIT_UI8_ENUM_EQ, - DIT_UI8_ENUM_SET, - DIT_UI16_ENUM_EQ, - DIT_UI16_ENUM_SET, - DIT_UI32_ENUM_EQ, - DIT_UI32_ENUM_SET, - DIT_INT_ENUM_FUNCTION_EQ, - DIT_INT_ENUM_FUNCTION_SET, - - DIT_BOOL_FUNC_FIXED_STR, - DIT_UI8_FUNC_FIXED_STR, - DIT_UI16_FUNC_FIXED_STR, - DIT_UI32_FUNC_FIXED_STR, - DIT_I64_FUNC_FIXED_STR, - DIT_UI64_FUNC_FIXED_STR, - DIT_FLOAT_FUNC_FIXED_STR, - DIT_DOUBLE_FUNC_FIXED_STR, - - DIT_RESOLVE_BY_NAME, //new - for external functions - - DIT_LOCAL_VARIABLE -}; - -inline bool IsStringType(EDumpItemType type) { - return type == DIT_STRING_FIELD || type == DIT_STR_CONST || type == DIT_STR_FUNCTION || type == DIT_STRBUF_FUNCTION || type == DIT_RESOLVE_BY_NAME; -} - -struct fake {}; - -struct calc_op; - -typedef int (fake::*int_fn_t)() const; -typedef float (fake::*float_fn_t)() const; -typedef bool (fake::*bool_fn_t)() const; -typedef ui16 (fake::*ui16_fn_t)() const; -typedef ui32 (fake::*ui32_fn_t)() const; -typedef bool (fake::*bool_strbuf_fn_t)(const TStringBuf&) const; // string -> bool -typedef ui8 (fake::*ui8_strbuf_fn_t)(const TStringBuf&) const; // string -> ui8 -typedef ui16 (fake::*ui16_strbuf_fn_t)(const TStringBuf&) const; // string -> ui16 -typedef ui32 (fake::*ui32_strbuf_fn_t)(const TStringBuf&) const; // string -> ui32 -typedef i64 (fake::*i64_strbuf_fn_t)(const TStringBuf&) const; // string -> i64 -typedef ui64 (fake::*ui64_strbuf_fn_t)(const TStringBuf&) const; // string -> ui64 -typedef float (fake::*float_strbuf_fn_t)(const TStringBuf&) const; // string -> float -typedef double (fake::*double_strbuf_fn_t)(const TStringBuf&) const; // string -> double -typedef const char* (fake::*str_fn_t)() const; -typedef const char* (fake::*strbuf_2_fn_t)(TString& buf, const char* nul) const; -typedef TStringBuf (fake::*resolve_fn_t)(const TStringBuf&) const; // string -> string, $var -> "value" - -// note: we can not reuse the above signatures, calling conventions may differ -typedef ui8 (*ui8_ext_fn_t)(const fake*); -typedef ui16 (*ui16_ext_fn_t)(const fake*); -typedef ui32 (*ui32_ext_fn_t)(const fake*); -typedef ui64 (*ui64_ext_fn_t)(const fake*); - -struct dump_item { - EDumpItemType type; - int pack_id = 0; - - union { - // fields - intptr_t field_offset; - - // constants - long long_const; - float float_const; - - // functions - int_fn_t int_fn; - float_fn_t float_fn; - bool_fn_t bool_fn; - str_fn_t str_fn; - strbuf_2_fn_t strbuf_2_fn; - resolve_fn_t resolve_fn; - - bool_strbuf_fn_t bool_strbuf_fn; - ui8_strbuf_fn_t ui8_strbuf_fn; - ui16_strbuf_fn_t ui16_strbuf_fn; - ui32_strbuf_fn_t ui32_strbuf_fn; - i64_strbuf_fn_t i64_strbuf_fn; - ui64_strbuf_fn_t ui64_strbuf_fn; - float_strbuf_fn_t float_strbuf_fn; - double_strbuf_fn_t double_strbuf_fn; - - ui8_ext_fn_t ui8_ext_fn; - ui16_ext_fn_t ui16_ext_fn; - ui32_ext_fn_t ui32_ext_fn; - ui64_ext_fn_t ui64_ext_fn; - - // enum - int_fn_t int_enum_fn; - - // for DIT_MATH_RESULT - const calc_op* op; - }; - - // for enum - ui32 enum_val; - - // for local vars, also used to mark accessor functions to use them in dump - const char* local_var_name = nullptr; - - int arr_ind; // externally initialized! - int arr_length; - - mutable TString the_buf; // buffer for string function, string constants also here - - // Ctors - dump_item() - : type(DIT_FAKE_ITEM) - , field_offset(0) - { - } - - dump_item(bool* ptr, int arrlen = 0) - : type(DIT_BOOL_FIELD) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , arr_length(arrlen) - { - } - dump_item(ui8* ptr, int arrlen = 0) - : type(DIT_UI8_FIELD) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , arr_length(arrlen) - { - } - dump_item(ui16* ptr, int arrlen = 0) - : type(DIT_UI16_FIELD) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , arr_length(arrlen) - { - } - dump_item(ui32* ptr, int arrlen = 0) - : type(DIT_UI32_FIELD) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , arr_length(arrlen) - { - } - dump_item(i64* ptr, int arrlen = 0) - : type(DIT_I64_FIELD) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , arr_length(arrlen) - { - } - dump_item(ui64* ptr, int arrlen = 0) - : type(DIT_UI64_FIELD) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , arr_length(arrlen) - { - } - dump_item(float* ptr, int arrlen = 0) - : type(DIT_FLOAT_FIELD) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , arr_length(arrlen) - { - } - dump_item(double* ptr, int arrlen = 0) - : type(DIT_DOUBLE_FIELD) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , arr_length(arrlen) - { - } - dump_item(time_t32* ptr, int arrlen = 0) - : type(DIT_TIME_T32_FIELD) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , arr_length(arrlen) - { - } - dump_item(pf16ui32* ptr, int arrlen = 0) - : type(DIT_PF16UI32_FIELD) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , arr_length(arrlen) - { - } - dump_item(pf16float* ptr, int arrlen = 0) - : type(DIT_PF16FLOAT_FIELD) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , arr_length(arrlen) - { - } - dump_item(sf16float* ptr, int arrlen = 0) - : type(DIT_SF16FLOAT_FIELD) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , arr_length(arrlen) - { - } - dump_item(char* ptr, int arrlen = 0) - : type(DIT_STRING_FIELD) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , arr_length(arrlen) - { - } - - dump_item(long val) - : type(DIT_LONG_CONST) - , long_const(val) - { - } - dump_item(float val) - : type(DIT_FLOAT_CONST) - , float_const(val) - { - } - dump_item(TString& val) - : type(DIT_STR_CONST) - , the_buf(val) - { - } - - dump_item(int_fn_t fn) - : type(DIT_INT_FUNCTION) - , int_fn(fn) - { - } - dump_item(float_fn_t fn) - : type(DIT_FLOAT_FUNCTION) - , float_fn(fn) - { - } - dump_item(bool_fn_t fn) - : type(DIT_BOOL_FUNCTION) - , bool_fn(fn) - { - } - dump_item(bool_strbuf_fn_t fn, const char* name) - : type(DIT_BOOL_FUNC_FIXED_STR) - , bool_strbuf_fn(fn) - , the_buf(name) - { - } - dump_item(ui8_strbuf_fn_t fn, const char* name) - : type(DIT_UI8_FUNC_FIXED_STR) - , ui8_strbuf_fn(fn) - , the_buf(name) - { - } - dump_item(ui16_strbuf_fn_t fn, const char* name) - : type(DIT_UI16_FUNC_FIXED_STR) - , ui16_strbuf_fn(fn) - , the_buf(name) - { - } - dump_item(ui32_strbuf_fn_t fn, const char* name) - : type(DIT_UI32_FUNC_FIXED_STR) - , ui32_strbuf_fn(fn) - , the_buf(name) - { - } - dump_item(i64_strbuf_fn_t fn, const char* name) - : type(DIT_I64_FUNC_FIXED_STR) - , i64_strbuf_fn(fn) - , the_buf(name) - { - } - dump_item(ui64_strbuf_fn_t fn, const char* name) - : type(DIT_UI64_FUNC_FIXED_STR) - , ui64_strbuf_fn(fn) - , the_buf(name) - { - } - dump_item(float_strbuf_fn_t fn, const char* name) - : type(DIT_FLOAT_FUNC_FIXED_STR) - , float_strbuf_fn(fn) - , the_buf(name) - { - } - dump_item(double_strbuf_fn_t fn, const char* name) - : type(DIT_DOUBLE_FUNC_FIXED_STR) - , double_strbuf_fn(fn) - , the_buf(name) - { - } - dump_item(str_fn_t fn) - : type(DIT_STR_FUNCTION) - , str_fn(fn) - { - } - dump_item(strbuf_2_fn_t fn) - : type(DIT_STRBUF_FUNCTION) - , strbuf_2_fn(fn) - { - } - - dump_item(ui8_ext_fn_t fn, const char* lvn = nullptr) - : type(DIT_UI8_EXT_FUNCTION) - , ui8_ext_fn(fn) - , local_var_name(lvn) - { - } - dump_item(ui16_ext_fn_t fn, const char* lvn = nullptr) - : type(DIT_UI16_EXT_FUNCTION) - , ui16_ext_fn(fn) - , local_var_name(lvn) - { - } - dump_item(ui32_ext_fn_t fn, const char* lvn = nullptr) - : type(DIT_UI32_EXT_FUNCTION) - , ui32_ext_fn(fn) - , local_var_name(lvn) - { - } - dump_item(ui64_ext_fn_t fn, const char* lvn = nullptr) - : type(DIT_UI64_EXT_FUNCTION) - , ui64_ext_fn(fn) - , local_var_name(lvn) - { - } - - dump_item(ui8* ptr, ui32 val, bool bitset) - : type(bitset ? DIT_UI8_ENUM_SET : DIT_UI8_ENUM_EQ) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , enum_val(val) - { - } - - dump_item(ui16* ptr, ui32 val, bool bitset) - : type(bitset ? DIT_UI16_ENUM_SET : DIT_UI16_ENUM_EQ) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , enum_val(val) - { - } - - dump_item(ui32* ptr, ui32 val, bool bitset) - : type(bitset ? DIT_UI32_ENUM_SET : DIT_UI32_ENUM_EQ) - , field_offset(reinterpret_cast<intptr_t>(ptr)) - , enum_val(val) - { - } - - dump_item(int_fn_t fn, ui32 val, bool bitset) - : type(bitset ? DIT_INT_ENUM_FUNCTION_SET : DIT_INT_ENUM_FUNCTION_EQ) - , int_enum_fn(fn) - , enum_val(val) - { - } - - dump_item(resolve_fn_t fn, const char* name) - : type(DIT_RESOLVE_BY_NAME) - , resolve_fn(fn) - , the_buf(name) - { - } //name of variable saved in the_buf - - // Functions - template <class TOut> // implemented for FILE*, TString* (appends) and IOutputStream* - void print(TOut* p, const char** dd) const; - TStringBuf GetStrBuf(const char** dd) const; // for char-types only! - eval_res_type eval(const char** dd) const; - void set_arrind(int arrind); - void rewrite_op(const calc_op* ops); - - bool is_accessor_func() const { - return type >= DIT_INT_FUNCTION && type <= DIT_UI64_EXT_FUNCTION && local_var_name; - } - - bool is_field() const { - return type > DIT_FIELDS_START && type < DIT_FIELDS_END || is_accessor_func(); - } - - bool is_array_field() const { - return is_field() && arr_length > 0; - } -}; - -// named_dump_item -struct named_dump_item { - const char* name; - dump_item item; -}; diff --git a/library/cpp/fieldcalc/lossy_types.h b/library/cpp/fieldcalc/lossy_types.h deleted file mode 100644 index 98acfea902..0000000000 --- a/library/cpp/fieldcalc/lossy_types.h +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once - -#include <util/generic/cast.h> - -// although target value is float, this thing is only used as unsigned int container -struct pf16ui32 { - ui16 val; - pf16ui32() - : val(0) - { - } - void operator=(ui32 t) { - val = static_cast<ui16>(BitCast<ui32>(static_cast<float>(t)) >> 15); - } - operator ui32() const { - return (ui32)BitCast<float>((ui32)(val << 15)); - } -}; - -// unsigned float value -struct pf16float { - ui16 val; - pf16float() - : val(0) - { - } - void operator=(float t) { - assert(t >= 0.); - val = static_cast<ui16>(BitCast<ui32>(t) >> 15); - } - operator float() const { - return BitCast<float>((ui32)(val << 15)); - } -}; - -// signed float value -struct sf16float { - ui16 val; - sf16float() - : val(0) - { - } - void operator=(float t) { - assert(t >= 0.); - val = BitCast<ui32>(t) >> 16; - } - operator float() const { - return BitCast<float>((ui32)(val << 16)); - } -}; - -typedef i32 time_t32; // not really lossy, should be placed somewhere else diff --git a/library/cpp/fieldcalc/ya.make b/library/cpp/fieldcalc/ya.make deleted file mode 100644 index 9796592996..0000000000 --- a/library/cpp/fieldcalc/ya.make +++ /dev/null @@ -1,13 +0,0 @@ -LIBRARY() - -PEERDIR( - library/cpp/deprecated/autoarray -) - -SRCS( - field_calc.cpp - lossy_types.h - field_calc_int.h -) - -END() diff --git a/library/cpp/malloc/galloc/malloc-info.cpp b/library/cpp/malloc/galloc/malloc-info.cpp deleted file mode 100644 index fbcfa7ee06..0000000000 --- a/library/cpp/malloc/galloc/malloc-info.cpp +++ /dev/null @@ -1,9 +0,0 @@ -#include <library/cpp/malloc/api/malloc.h> - -using namespace NMalloc; - -TMallocInfo NMalloc::MallocInfo() { - TMallocInfo r; - r.Name = "tcmalloc"; - return r; -} diff --git a/library/cpp/malloc/galloc/ya.make b/library/cpp/malloc/galloc/ya.make deleted file mode 100644 index b6646a6cf6..0000000000 --- a/library/cpp/malloc/galloc/ya.make +++ /dev/null @@ -1,15 +0,0 @@ -LIBRARY() - -NO_UTIL() -ALLOCATOR_IMPL() - -PEERDIR( - library/cpp/malloc/api - contrib/deprecated/galloc -) - -SRCS( - malloc-info.cpp -) - -END() diff --git a/library/cpp/on_disk/multi_blob/multiblob.cpp b/library/cpp/on_disk/multi_blob/multiblob.cpp deleted file mode 100644 index d92b31e613..0000000000 --- a/library/cpp/on_disk/multi_blob/multiblob.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include <util/generic/yexception.h> -#include <util/system/align.h> - -#include <library/cpp/on_disk/chunks/reader.h> - -#include "multiblob.h" - -void TSubBlobs::ReadMultiBlob(const TBlob& multi) { - if (multi.Size() < sizeof(TMultiBlobHeader)) { - ythrow yexception() << "not a blob, too small"; - } - - Multi = multi; - memcpy((void*)&Header, Multi.Data(), sizeof(TMultiBlobHeader)); - - if (Header.BlobMetaSig != BLOBMETASIG) { - if (Header.BlobRecordSig != TMultiBlobHeader::RecordSig) { - if (ReadChunkedData(multi)) - return; - } - ythrow yexception() << "is not a blob, MetaSig was read: " - << Header.BlobMetaSig - << ", must be" << BLOBMETASIG; - } - - if (Header.BlobRecordSig != TMultiBlobHeader::RecordSig) - ythrow yexception() << "unknown multiblob RecordSig " - << Header.BlobRecordSig; - - reserve(size() + Header.Count); - if (Header.Flags & EMF_INTERLAY) { - size_t pos = Header.HeaderSize(); - for (size_t i = 0; i < Header.Count; ++i) { - pos = AlignUp<ui64>(pos, sizeof(ui64)); - ui64 size = *((ui64*)((const char*)multi.Data() + pos)); - pos = AlignUp<ui64>(pos + sizeof(ui64), Header.Align); - push_back(multi.SubBlob(pos, pos + size)); - pos += size; - } - } else { - const ui64* sizes = Header.Sizes(multi.Data()); - size_t pos = Header.HeaderSize() + Header.Count * sizeof(ui64); - for (size_t i = 0; i < Header.Count; ++i) { - pos = AlignUp<ui64>(pos, Header.Align); - push_back(multi.SubBlob(pos, pos + *sizes)); - pos += *sizes; - sizes++; - } - } -} - -bool TSubBlobs::ReadChunkedData(const TBlob& multi) noexcept { - Multi = multi; - memset((void*)&Header, 0, sizeof(Header)); - - TChunkedDataReader reader(Multi); - Header.Count = reader.GetBlocksCount(); - resize(GetHeader()->Count); - for (size_t i = 0; i < size(); ++i) - // We can use TBlob::NoCopy() because of reader.GetBlock(i) returns - // address into memory of multi blob. - // This knowledge was acquired from implementation of - // TChunkedDataReader, so we need care about any changes that. - (*this)[i] = TBlob::NoCopy(reader.GetBlock(i), reader.GetBlockLen(i)); - Header.Flags |= EMF_CHUNKED_DATA_READER; - return true; -} diff --git a/library/cpp/on_disk/multi_blob/multiblob.h b/library/cpp/on_disk/multi_blob/multiblob.h deleted file mode 100644 index b40a5ae6af..0000000000 --- a/library/cpp/on_disk/multi_blob/multiblob.h +++ /dev/null @@ -1,77 +0,0 @@ -#pragma once - -#include <util/generic/vector.h> -#include <util/memory/blob.h> - -#define BLOBMETASIG 0x3456789Au - -enum E_Multiblob_Flags { - // if EMF_INTERLAY is clear - // multiblob format - // HeaderSize() bytes for TMultiBlobHeader - // Count*sizeof(ui64) bytes for blob sizes - // blob1 - // (alignment) - // blob2 - // (alignment) - // ... - // (alignment) - // blobn - // if EMF_INTERLAY is set - // multiblob format - // HeaderSize() bytes for TMultiBlobHeader - // size1 ui64, the size of 1st blob - // blob1 - // (alignment) - // size2 ui64, the size of 2nd blob - // blob2 - // (alignment) - // ... - // (alignment) - // sizen ui64, the size of n'th blob - // blobn - EMF_INTERLAY = 1, - - // Means that multiblob contains blocks in TChunkedDataReader format - // Legacy, use it only for old files, created for TChunkedDataReader - EMF_CHUNKED_DATA_READER = 2, - - // Flags that may be configured for blobbuilder in client code - EMF_WRITEABLE = EMF_INTERLAY, -}; - -struct TMultiBlobHeader { - // data - ui32 BlobMetaSig; - ui32 BlobRecordSig; - ui64 Count; // count of sub blobs - ui32 Align; // alignment for every subblob - ui32 Flags; - static const ui32 RecordSig = 0x23456789; - static inline size_t HeaderSize() { - return 4 * sizeof(ui64); - } - inline const ui64* Sizes(const void* Data) const { - return (const ui64*)((const char*)Data + HeaderSize()); - } -}; - -class TSubBlobs: public TVector<TBlob> { -public: - TSubBlobs() { - } - TSubBlobs(const TBlob& multi) { - ReadMultiBlob(multi); - } - void ReadMultiBlob(const TBlob& multi); - const TMultiBlobHeader* GetHeader() const { - return (const TMultiBlobHeader*)&Header; - } - -protected: - TMultiBlobHeader Header; - TBlob Multi; - -private: - bool ReadChunkedData(const TBlob& multi) noexcept; -}; diff --git a/library/cpp/on_disk/multi_blob/multiblob_builder.cpp b/library/cpp/on_disk/multi_blob/multiblob_builder.cpp deleted file mode 100644 index 44aa4a6c2f..0000000000 --- a/library/cpp/on_disk/multi_blob/multiblob_builder.cpp +++ /dev/null @@ -1,146 +0,0 @@ -#include <util/memory/tempbuf.h> -#include <util/system/align.h> - -#include "multiblob_builder.h" - -/* - * TBlobSaverMemory - */ -TBlobSaverMemory::TBlobSaverMemory(const void* ptr, size_t size) - : Blob(TBlob::NoCopy(ptr, size)) -{ -} - -TBlobSaverMemory::TBlobSaverMemory(const TBlob& blob) - : Blob(blob) -{ -} - -void TBlobSaverMemory::Save(IOutputStream& output, ui32 /*flags*/) { - output.Write((void*)Blob.Data(), Blob.Length()); -} - -size_t TBlobSaverMemory::GetLength() { - return Blob.Length(); -} - -/* - * TBlobSaverFile - */ - -TBlobSaverFile::TBlobSaverFile(TFile file) - : File(file) -{ - Y_ASSERT(File.IsOpen()); -} - -TBlobSaverFile::TBlobSaverFile(const char* filename, EOpenMode oMode) - : File(filename, oMode) -{ - Y_ASSERT(File.IsOpen()); -} - -void TBlobSaverFile::Save(IOutputStream& output, ui32 /*flags*/) { - TTempBuf buffer(1 << 20); - while (size_t size = File.Read((void*)buffer.Data(), buffer.Size())) - output.Write((void*)buffer.Data(), size); -} - -size_t TBlobSaverFile::GetLength() { - return File.GetLength(); -} - -/* - * TMultiBlobBuilder - */ - -TMultiBlobBuilder::TMultiBlobBuilder(bool isOwn) - : IsOwner(isOwn) -{ -} - -TMultiBlobBuilder::~TMultiBlobBuilder() { - if (IsOwner) - DeleteSubBlobs(); -} - -namespace { - ui64 PadToAlign(IOutputStream& output, ui64 fromPos, ui32 align) { - ui64 toPos = AlignUp<ui64>(fromPos, align); - for (; fromPos < toPos; ++fromPos) { - output << (char)0; - } - return toPos; - } -} - -void TMultiBlobBuilder::Save(IOutputStream& output, ui32 flags) { - TMultiBlobHeader header; - memset((void*)&header, 0, sizeof(header)); - header.BlobMetaSig = BLOBMETASIG; - header.BlobRecordSig = TMultiBlobHeader::RecordSig; - header.Count = Blobs.size(); - header.Align = ALIGN; - header.Flags = flags & EMF_WRITEABLE; - output.Write((void*)&header, sizeof(header)); - for (size_t i = sizeof(header); i < header.HeaderSize(); ++i) - output << (char)0; - ui64 pos = header.HeaderSize(); - if (header.Flags & EMF_INTERLAY) { - for (size_t i = 0; i < Blobs.size(); ++i) { - ui64 size = Blobs[i]->GetLength(); - pos = PadToAlign(output, pos, sizeof(ui64)); // Align size record - output.Write((void*)&size, sizeof(ui64)); - pos = PadToAlign(output, pos + sizeof(ui64), header.Align); // Align blob - Blobs[i]->Save(output, header.Flags); - pos += size; - } - } else { - for (size_t i = 0; i < Blobs.size(); ++i) { - ui64 size = Blobs[i]->GetLength(); - output.Write((void*)&size, sizeof(ui64)); - } - pos += Blobs.size() * sizeof(ui64); - for (size_t i = 0; i < Blobs.size(); ++i) { - pos = PadToAlign(output, pos, header.Align); - Blobs[i]->Save(output, header.Flags); - pos += Blobs[i]->GetLength(); - } - } - // Compensate for imprecise size - for (ui64 len = GetLength(); pos < len; ++pos) { - output << (char)0; - } -} - -size_t TMultiBlobBuilder::GetLength() { - // Sizes may be diferent with and without EMF_INTERLAY, so choose greater of 2 - size_t resNonInter = TMultiBlobHeader::HeaderSize() + Blobs.size() * sizeof(ui64); - size_t resInterlay = TMultiBlobHeader::HeaderSize(); - for (size_t i = 0; i < Blobs.size(); ++i) { - resInterlay = AlignUp<ui64>(resInterlay, sizeof(ui64)) + sizeof(ui64); - resInterlay = AlignUp<ui64>(resInterlay, ALIGN) + Blobs[i]->GetLength(); - resNonInter = AlignUp<ui64>(resNonInter, ALIGN) + Blobs[i]->GetLength(); - } - resInterlay = AlignUp<ui64>(resInterlay, ALIGN); - resNonInter = AlignUp<ui64>(resNonInter, ALIGN); - return Max(resNonInter, resInterlay); -} - -TMultiBlobBuilder::TSavers& TMultiBlobBuilder::GetBlobs() { - return Blobs; -} - -const TMultiBlobBuilder::TSavers& TMultiBlobBuilder::GetBlobs() const { - return Blobs; -} - -void TMultiBlobBuilder::AddBlob(IBlobSaverBase* blob) { - Blobs.push_back(blob); -} - -void TMultiBlobBuilder::DeleteSubBlobs() { - for (size_t i = 0; i < Blobs.size(); ++i) - delete Blobs[i]; - Blobs.clear(); -} diff --git a/library/cpp/on_disk/multi_blob/multiblob_builder.h b/library/cpp/on_disk/multi_blob/multiblob_builder.h deleted file mode 100644 index a8e3c6d35e..0000000000 --- a/library/cpp/on_disk/multi_blob/multiblob_builder.h +++ /dev/null @@ -1,64 +0,0 @@ -#pragma once - -#include <util/system/align.h> -#include <util/stream/output.h> -#include <util/stream/file.h> -#include <util/draft/holder_vector.h> - -#include "multiblob.h" - -class IBlobSaverBase { -public: - virtual ~IBlobSaverBase() { - } - virtual void Save(IOutputStream& output, ui32 flags = 0) = 0; - virtual size_t GetLength() = 0; -}; - -inline void MultiBlobSave(IOutputStream& output, IBlobSaverBase& saver) { - saver.Save(output); -} - -class TBlobSaverMemory: public IBlobSaverBase { -public: - TBlobSaverMemory(const void* ptr, size_t size); - TBlobSaverMemory(const TBlob& blob); - void Save(IOutputStream& output, ui32 flags = 0) override; - size_t GetLength() override; - -private: - TBlob Blob; -}; - -class TBlobSaverFile: public IBlobSaverBase { -public: - TBlobSaverFile(TFile file); - TBlobSaverFile(const char* filename, EOpenMode oMode = RdOnly); - void Save(IOutputStream& output, ui32 flags = 0) override; - size_t GetLength() override; - -protected: - TFile File; -}; - -class TMultiBlobBuilder: public IBlobSaverBase { -protected: - // Data will be stored with default alignment DEVTOOLS-4548 - static const size_t ALIGN = 16; - -public: - typedef TVector<IBlobSaverBase*> TSavers; - - TMultiBlobBuilder(bool isOwn = true); - ~TMultiBlobBuilder() override; - void Save(IOutputStream& output, ui32 flags = 0) override; - size_t GetLength() override; - TSavers& GetBlobs(); - const TSavers& GetBlobs() const; - void AddBlob(IBlobSaverBase* blob); - void DeleteSubBlobs(); - -protected: - TSavers Blobs; - bool IsOwner; -}; diff --git a/library/cpp/on_disk/multi_blob/ya.make b/library/cpp/on_disk/multi_blob/ya.make deleted file mode 100644 index 50615fc901..0000000000 --- a/library/cpp/on_disk/multi_blob/ya.make +++ /dev/null @@ -1,13 +0,0 @@ -LIBRARY() - -SRCS( - multiblob.cpp - multiblob_builder.cpp -) - -PEERDIR( - library/cpp/on_disk/chunks - util/draft -) - -END() diff --git a/library/cpp/on_disk/st_hash/fake.cpp b/library/cpp/on_disk/st_hash/fake.cpp deleted file mode 100644 index ef5af4d432..0000000000 --- a/library/cpp/on_disk/st_hash/fake.cpp +++ /dev/null @@ -1,4 +0,0 @@ -#include "save_stl.h" -#include "static_hash.h" -#include "static_hash_map.h" -#include "sthash_iterators.h" diff --git a/library/cpp/on_disk/st_hash/save_stl.h b/library/cpp/on_disk/st_hash/save_stl.h deleted file mode 100644 index 00f8f0e20d..0000000000 --- a/library/cpp/on_disk/st_hash/save_stl.h +++ /dev/null @@ -1,84 +0,0 @@ -#pragma once - -#include <util/generic/hash.h> -#include <util/system/yassert.h> -#include <util/stream/output.h> - -// this structure might be replaced with sthashtable class -template <class HF, class Eq, class size_type> -struct sthashtable_nvm_sv { - sthashtable_nvm_sv() { - if (sizeof(sthashtable_nvm_sv) != sizeof(HF) + sizeof(Eq) + 3 * sizeof(size_type)) { - memset(this, 0, sizeof(sthashtable_nvm_sv)); - } - } - - sthashtable_nvm_sv(const HF& phf, const Eq& peq, const size_type& pnb, const size_type& pne, const size_type& pnd) - : sthashtable_nvm_sv() - { - hf = phf; - eq = peq; - num_buckets = pnb; - num_elements = pne; - data_end_off = pnd; - } - - HF hf; - Eq eq; - size_type num_buckets; - size_type num_elements; - size_type data_end_off; -}; - -/** - * Some hack to save both THashMap and sthash. - * Working with stHash does not depend on the template parameters, because the content of stHash is not used inside this method. - */ -template <class V, class K, class HF, class Ex, class Eq, class A> -template <class KeySaver> -inline int THashTable<V, K, HF, Ex, Eq, A>::save_for_st(IOutputStream* stream, KeySaver& ks, sthash<int, int, THash<int>, TEqualTo<int>, typename KeySaver::TSizeType>* stHash) const { - Y_ASSERT(!stHash || stHash->bucket_count() == bucket_count()); - typedef sthashtable_nvm_sv<HF, Eq, typename KeySaver::TSizeType> sv_type; - sv_type sv = {this->_get_hash_fun(), this->_get_key_eq(), static_cast<typename KeySaver::TSizeType>(buckets.size()), static_cast<typename KeySaver::TSizeType>(num_elements), 0}; - // to do: m.b. use just the size of corresponding object? - typename KeySaver::TSizeType cur_off = sizeof(sv_type) + - (sv.num_buckets + 1) * sizeof(typename KeySaver::TSizeType); - sv.data_end_off = cur_off; - const_iterator n; - for (n = begin(); n != end(); ++n) { - sv.data_end_off += static_cast<typename KeySaver::TSizeType>(ks.GetRecordSize(*n)); - } - typename KeySaver::TSizeType* sb = stHash ? (typename KeySaver::TSizeType*)(stHash->buckets()) : nullptr; - if (stHash) - sv.data_end_off += static_cast<typename KeySaver::TSizeType>(sb[buckets.size()] - sb[0]); - //saver.Align(sizeof(char*)); - stream->Write(&sv, sizeof(sv)); - - size_type i; - //save vector - for (i = 0; i < buckets.size(); ++i) { - node* cur = buckets[i]; - stream->Write(&cur_off, sizeof(cur_off)); - if (cur) { - while (!((uintptr_t)cur & 1)) { - cur_off += static_cast<typename KeySaver::TSizeType>(ks.GetRecordSize(cur->val)); - cur = cur->next; - } - } - if (stHash) - cur_off += static_cast<typename KeySaver::TSizeType>(sb[i + 1] - sb[i]); - } - stream->Write(&cur_off, sizeof(cur_off)); // end mark - for (i = 0; i < buckets.size(); ++i) { - node* cur = buckets[i]; - if (cur) { - while (!((uintptr_t)cur & 1)) { - ks.SaveRecord(stream, cur->val); - cur = cur->next; - } - } - if (stHash) - stream->Write((const char*)stHash + sb[i], sb[i + 1] - sb[i]); - } - return 0; -} diff --git a/library/cpp/on_disk/st_hash/static_hash.h b/library/cpp/on_disk/st_hash/static_hash.h deleted file mode 100644 index ca7a6ccd36..0000000000 --- a/library/cpp/on_disk/st_hash/static_hash.h +++ /dev/null @@ -1,420 +0,0 @@ -#pragma once - -#include "save_stl.h" -#include "sthash_iterators.h" - -#include <util/generic/hash.h> -#include <util/generic/vector.h> -#include <util/generic/buffer.h> -#include <util/generic/cast.h> -#include <util/generic/yexception.h> // for save/load only -#include <util/stream/file.h> -#include <util/stream/buffer.h> -#include <utility> - -#include <memory> -#include <algorithm> -#include <functional> - -#include <cstdlib> -#include <cstddef> - -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable : 4624) // 'destructor could not be generated because a base class destructor is inaccessible' -#endif - -template <class HashType, class KeySaver> -inline void SaveHashToStreamEx(HashType& hash, IOutputStream* stream) { - KeySaver ks; - if (hash.save_for_st(stream, ks)) - ythrow yexception() << "Could not save hash to stream"; -} - -template <class HashType> -inline void SaveHashToStream(HashType& hash, IOutputStream* stream) { - typedef TSthashWriter<typename HashType::key_type, typename HashType::mapped_type, ui64> KeySaver; - return SaveHashToStreamEx<HashType, KeySaver>(hash, stream); -} - -template <class HashType, class KeySaver> -inline void SaveHashToFileEx(HashType& hash, const char* fileName) { - TFileOutput output(fileName); - SaveHashToStreamEx<HashType, KeySaver>(hash, &output); -} - -template <class HashType> -inline void SaveHashToFile(HashType& hash, const char* fileName) { - typedef TSthashWriter<typename HashType::key_type, typename HashType::mapped_type, ui64> KeySaver; - return SaveHashToFileEx<HashType, KeySaver>(hash, fileName); -} - -template <class HashType> -inline void SaveHashSetToFile(HashType& hash, const char* fileName) { - typedef TSthashSetWriter<typename HashType::key_type, ui64> KeySaver; - return SaveHashToFileEx<HashType, KeySaver>(hash, fileName); -} - -template <class HashType> -inline void SaveHashToFile32(HashType& hash, const char* fileName) { - typedef TSthashWriter<typename HashType::key_type, typename HashType::mapped_type, ui32> KeySaver; - return SaveHashToFileEx<HashType, KeySaver>(hash, fileName); -} - -template <class HashType, class KeySaver> -inline void SaveHashToBufferEx(HashType& hash, TBuffer& buffer, sthash<int, int, THash<int>, TEqualTo<int>, typename KeySaver::TSizeType>* stHash = nullptr) { - TBufferOutput stream(buffer); - KeySaver ks; - if (hash.save_for_st(&stream, ks, stHash)) - ythrow yexception() << "Could not save hash to memory"; -} - -template <class HashType> -inline void SaveHashToBuffer(HashType& hash, TBuffer& buffer) { - typedef TSthashWriter<typename HashType::key_type, typename HashType::mapped_type, ui64> KeySaver; - SaveHashToBufferEx<HashType, KeySaver>(hash, buffer); -} - -/** - * Some hack to save both THashMap and sthash. - * THashMap and sthash must have same bucket_count(). - */ -template <class HashType, class StHashType> -inline void SaveHashToBuffer(HashType& hash, TBuffer& buffer, StHashType* stHash) { - typedef TSthashWriter<typename HashType::key_type, typename HashType::mapped_type, ui64> KeySaver; - typedef sthash<int, int, THash<int>, TEqualTo<int>, typename KeySaver::TSizeType>* SH; - - SH sh = reinterpret_cast<SH>(stHash); - SaveHashToBufferEx<HashType, KeySaver>(hash, buffer, sh); -} - -template <class HashType> -inline void SaveHashToBuffer32(HashType& hash, TBuffer& buffer) { - typedef TSthashWriter<typename HashType::key_type, typename HashType::mapped_type, ui32> KeySaver; - SaveHashToBufferEx<HashType, KeySaver>(hash, buffer); -} - -template <class Iter, typename size_type_f = ui64> -class sthashtable { -public: - typedef typename Iter::TKeyType key_type; - typedef typename Iter::TValueType value_type; - typedef typename Iter::THasherType hasher; - typedef typename Iter::TKeyEqualType key_equal; - - typedef size_type_f size_type; - typedef ptrdiff_t difference_type; - typedef const value_type* const_pointer; - typedef const value_type& const_reference; - - typedef Iter const_iterator; - - const hasher hash_funct() const { - return hash; - } - const key_equal key_eq() const { - return equals; - } - -private: - const hasher hash; - const key_equal equals; - -private: - const_iterator iter_at_bucket(size_type bucket) const { - return (const_iterator)(((char*)this + buckets()[bucket])); - } - - const_iterator iter_at_bucket_or_end(size_type bucket) const { - if (bucket < num_buckets) - return (const_iterator)(((char*)this + buckets()[bucket])); - else - return end(); - } - - const size_type num_buckets; - const size_type num_elements; - const size_type data_end_off; - -protected: //shut up gcc warning - // we can't construct/destroy this object at all! - sthashtable(); - sthashtable(const sthashtable& ht); - ~sthashtable(); - -public: - // const size_type *buckets; - const size_type* buckets() const { - return (size_type*)((char*)this + sizeof(*this)); - } - const size_type buckets(size_type n) const { - return buckets()[n]; - } - - size_type size() const { - return num_elements; - } - size_type max_size() const { - return size_type(-1); - } - bool empty() const { - return size() == 0; - } - - const_iterator begin() const { - return num_buckets ? iter_at_bucket(0) : end(); - } - - const_iterator end() const { - return (const_iterator)(((char*)this + data_end_off)); - } - -public: - size_type size_in_bytes() const { - return data_end_off; - } - - size_type bucket_count() const { - return num_buckets; - } - - size_type elems_in_bucket(size_type bucket) const { - size_type result = 0; - const_iterator first = iter_at_bucket(bucket); - const_iterator last = iter_at_bucket_or_end(bucket + 1); - - for (; first != last; ++first) - ++result; - return result; - } - - template <class TheKey> - const_iterator find(const TheKey& key) const { - size_type n = bkt_num_key(key); - const_iterator first(iter_at_bucket(n)), last(iter_at_bucket_or_end(n + 1)); - for (; - first != last && !first.KeyEquals(equals, key); - ++first) { - } - if (first != last) - return first; - return end(); - } - - size_type count(const key_type& key) const { - const size_type n = bkt_num_key(key); - size_type result = 0; - const_iterator first = iter_at_bucket(n); - const_iterator last = iter_at_bucket_or_end(n + 1); - - for (; first != last; ++first) - if (first.KeyEquals(equals, key)) - ++result; - return result; - } - - std::pair<const_iterator, const_iterator> equal_range(const key_type& key) const; - -private: - template <class TheKey> - size_type bkt_num_key(const TheKey& key) const { - return hash(key) % num_buckets; - } -}; - -template <class I, class size_type_f> -std::pair<I, I> sthashtable<I, size_type_f>::equal_range(const key_type& key) const { - typedef std::pair<const_iterator, const_iterator> pii; - const size_type n = bkt_num_key(key); - const_iterator first = iter_at_bucket(n); - const_iterator last = iter_at_bucket_or_end(n + 1); - - for (; first != last; ++first) { - if (first.KeyEquals(equals, key)) { - const_iterator cur = first; - ++cur; - for (; cur != last; ++cur) - if (!cur.KeyEquals(equals, key)) - return pii(const_iterator(first), - const_iterator(cur)); - return pii(const_iterator(first), - const_iterator(last)); - } - } - return pii(end(), end()); -} - -/* end __SGI_STL_HASHTABLE_H */ - -template <class Key, class T, class HashFcn /*= hash<Key>*/, - class EqualKey = TEqualTo<Key>, typename size_type_f = ui64> -class sthash { -private: - typedef sthashtable<TSthashIterator<const Key, const T, HashFcn, EqualKey>, size_type_f> ht; - ht rep; - -public: - typedef typename ht::key_type key_type; - typedef typename ht::value_type value_type; - typedef typename ht::hasher hasher; - typedef typename ht::key_equal key_equal; - typedef T mapped_type; - - typedef typename ht::size_type size_type; - typedef typename ht::difference_type difference_type; - typedef typename ht::const_pointer const_pointer; - typedef typename ht::const_reference const_reference; - - typedef typename ht::const_iterator const_iterator; - - const hasher hash_funct() const { - return rep.hash_funct(); - } - const key_equal key_eq() const { - return rep.key_eq(); - } - -public: - size_type size() const { - return rep.size(); - } - size_type max_size() const { - return rep.max_size(); - } - bool empty() const { - return rep.empty(); - } - - const_iterator begin() const { - return rep.begin(); - } - const_iterator end() const { - return rep.end(); - } - -public: - template <class TheKey> - const_iterator find(const TheKey& key) const { - return rep.find(key); - } - template <class TheKey> - bool has(const TheKey& key) const { - return rep.find(key) != rep.end(); - } - - size_type count(const key_type& key) const { - return rep.count(key); - } - - std::pair<const_iterator, const_iterator> equal_range(const key_type& key) const { - return rep.equal_range(key); - } - - size_type size_in_bytes() const { - return rep.size_in_bytes(); - } - - size_type bucket_count() const { - return rep.bucket_count(); - } - size_type max_bucket_count() const { - return rep.max_bucket_count(); - } - size_type elems_in_bucket(size_type n) const { - return rep.elems_in_bucket(n); - } - - const size_type* buckets() const { - return rep.buckets(); - } - const size_type buckets(size_type n) const { - return rep.buckets()[n]; - } -}; - -template <class Key, class HashFcn, - class EqualKey = TEqualTo<Key>, typename size_type_f = ui64> -class sthash_set: public sthash<Key, TEmptyValue, HashFcn, EqualKey, size_type_f> { - typedef sthash<Key, TEmptyValue, HashFcn, EqualKey, size_type_f> Base; - -public: - using Base::const_iterator; - using Base::hasher; - using Base::key_equal; - using Base::key_type; - using Base::size_type; - using Base::value_type; -}; - -template <class Key, class T, class HashFcn /*= hash<Key>*/, - class EqualKey = TEqualTo<Key>, typename size_type_f = ui64> -class sthash_mm { -private: - typedef sthashtable<TSthashIterator<const Key, T, HashFcn, EqualKey>, size_type_f> ht; - ht rep; - -public: - typedef typename ht::key_type key_type; - typedef typename ht::value_type value_type; - typedef typename ht::hasher hasher; - typedef typename ht::key_equal key_equal; - typedef T mapped_type; - - typedef typename ht::size_type size_type; - typedef typename ht::difference_type difference_type; - typedef typename ht::const_pointer const_pointer; - typedef typename ht::const_reference const_reference; - - typedef typename ht::const_iterator const_iterator; - - const hasher hash_funct() const { - return rep.hash_funct(); - } - const key_equal key_eq() const { - return rep.key_eq(); - } - -public: - size_type size() const { - return rep.size(); - } - size_type max_size() const { - return rep.max_size(); - } - bool empty() const { - return rep.empty(); - } - - const_iterator begin() const { - return rep.begin(); - } - const_iterator end() const { - return rep.end(); - } - - const_iterator find(const key_type& key) const { - return rep.find(key); - } - - size_type count(const key_type& key) const { - return rep.count(key); - } - - std::pair<const_iterator, const_iterator> equal_range(const key_type& key) const { - return rep.equal_range(key); - } - - size_type bucket_count() const { - return rep.bucket_count(); - } - size_type max_bucket_count() const { - return rep.max_bucket_count(); - } - size_type elems_in_bucket(size_type n) const { - return rep.elems_in_bucket(n); - } -}; - -#ifdef _MSC_VER -#pragma warning(pop) -#endif diff --git a/library/cpp/on_disk/st_hash/static_hash_map.h b/library/cpp/on_disk/st_hash/static_hash_map.h deleted file mode 100644 index 5dc50abd39..0000000000 --- a/library/cpp/on_disk/st_hash/static_hash_map.h +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include "static_hash.h" - -#include <library/cpp/deprecated/mapped_file/mapped_file.h> - -#include <util/system/filemap.h> - -template <class SH> -struct sthash_mapped_c { - typedef SH H; - typedef typename H::const_iterator const_iterator; - TMappedFile M; - H* hsh; - sthash_mapped_c() - : M() - , hsh(nullptr) - { - } - sthash_mapped_c(const char* fname, bool precharge) - : M() - , hsh(nullptr) - { - Open(fname, precharge); - } - void Open(const char* fname, bool precharge) { - M.init(fname); - if (precharge) - M.precharge(); - hsh = (H*)M.getData(); - if (M.getSize() < sizeof(H) || (ssize_t)M.getSize() != hsh->end().Data - (char*)hsh) - ythrow yexception() << "Could not map hash: " << fname << " is damaged"; - } - H* operator->() { - return hsh; - } - const H* operator->() const { - return hsh; - } - H* GetSthash() { - return hsh; - } - const H* GetSthash() const { - return hsh; - } -}; - -template <class Key, class T, class Hash> -struct sthash_mapped: public sthash_mapped_c<sthash<Key, T, Hash>> { - typedef sthash<Key, T, Hash> H; - sthash_mapped(const char* fname, bool precharge) - : sthash_mapped_c<H>(fname, precharge) - { - } - sthash_mapped() - : sthash_mapped_c<H>() - { - } -}; diff --git a/library/cpp/on_disk/st_hash/sthash_iterators.h b/library/cpp/on_disk/st_hash/sthash_iterators.h deleted file mode 100644 index 6a9ebdd6c3..0000000000 --- a/library/cpp/on_disk/st_hash/sthash_iterators.h +++ /dev/null @@ -1,334 +0,0 @@ -#pragma once - -#include "save_stl.h" - -#include <util/system/align.h> - -/** - This file provides functionality for saving some relatively simple THashMap object - to disk in a form that can be mapped read-only (via mmap) at any address. - That saved object is accessed via pointer to sthash object (that must have - the same parameters as original THashMap object) - - If either key or value are variable-sized (i.e. contain pointers), user must - write his own instantiation of TSthashIterator (read iterator for sthash) and - TSthashWriter (write iterator for THashMap). - An example for <const char *, B> pair is in here. -**/ - -// TEmptyValue and SizeOfEx are helpers for sthash_set -struct TEmptyValue { - TEmptyValue() = default; -}; - -template <class T> -inline size_t SizeOfEx() { - return sizeof(T); -} - -template <> -inline size_t SizeOfEx<TEmptyValue>() { - return 0; -} -template <> -inline size_t SizeOfEx<const TEmptyValue>() { - return 0; -} - -template <class TKey, class TValue, class HashFcn, class EqualKey> -struct TSthashIterator { - // Implementation for simple types - typedef const TKey TKeyType; - typedef const TValue TValueType; - typedef EqualKey TKeyEqualType; - typedef HashFcn THasherType; - - const char* Data; - TSthashIterator() - : Data(nullptr) - { - } - explicit TSthashIterator(const char* data) - : Data(data) - { - } - void operator++() { - Data += GetLength(); - } - - bool operator!=(const TSthashIterator& that) const { - return Data != that.Data; - } - bool operator==(const TSthashIterator& that) const { - return Data == that.Data; - } - TKey& Key() const { - return *(TKey*)Data; - } - TValue& Value() { - return *(TValue*)(Data + sizeof(TKey)); - } - const TValue& Value() const { - return *(const TValue*)(Data + sizeof(TKey)); - } - - template <class AnotherKeyType> - bool KeyEquals(const EqualKey& eq, const AnotherKeyType& key) const { - return eq(*(TKey*)Data, key); - } - - size_t GetLength() const { - return sizeof(TKey) + SizeOfEx<TValue>(); - } -}; - -template <class Key, class Value, typename size_type_o = ui64> -struct TSthashWriter { - typedef size_type_o TSizeType; - size_t GetRecordSize(const std::pair<const Key, const Value>&) const { - return sizeof(Key) + SizeOfEx<Value>(); - } - int SaveRecord(IOutputStream* stream, const std::pair<const Key, const Value>& record) const { - stream->Write(&record.first, sizeof(Key)); - stream->Write(&record.second, SizeOfEx<Value>()); - return 0; - } -}; - -// Remember that this simplified implementation makes a copy of `key' in std::make_pair. -// It can also waste some memory on undesired alignment. -template <class Key, typename size_type_o = ui64> -struct TSthashSetWriter: public TSthashWriter<Key, TEmptyValue, size_type_o> { - typedef TSthashWriter<Key, TEmptyValue, size_type_o> MapWriter; - size_t GetRecordSize(const Key& key) const { - return MapWriter::GetRecordSize(std::make_pair(key, TEmptyValue())); - } - int SaveRecord(IOutputStream* stream, const Key& key) const { - return MapWriter::SaveRecord(stream, std::make_pair(key, TEmptyValue())); - } -}; - -// we can't save something with pointers without additional tricks - -template <class A, class B, class HashFcn, class EqualKey> -struct TSthashIterator<A*, B, HashFcn, EqualKey> {}; - -template <class A, class B, class HashFcn, class EqualKey> -struct TSthashIterator<A, B*, HashFcn, EqualKey> {}; - -template <class A, class B, typename size_type_o> -struct TSthashWriter<A*, B*, size_type_o> {}; - -template <class A, class B, typename size_type_o> -struct TSthashWriter<A*, B, size_type_o> {}; - -template <class A, class B, typename size_type_o> -struct TSthashWriter<A, B*, size_type_o> {}; - -template <class T> -inline size_t AlignForChrKey() { - return 4; // TODO: change this (requeres rebuilt of a few existing files) -} - -template <> -inline size_t AlignForChrKey<TEmptyValue>() { - return 1; -} - -template <> -inline size_t AlignForChrKey<const TEmptyValue>() { - return AlignForChrKey<TEmptyValue>(); -} - -// !! note that for char*, physical placement of key and value is swapped -template <class TValue, class HashFcn, class EqualKey> -struct TSthashIterator<const char* const, TValue, HashFcn, EqualKey> { - typedef const TValue TValueType; - typedef const char* TKeyType; - typedef EqualKey TKeyEqualType; - typedef HashFcn THasherType; - - const char* Data; - TSthashIterator() - : Data(nullptr) - { - } - TSthashIterator(const char* data) - : Data(data) - { - } - void operator++() { - Data += GetLength(); - } - - bool operator!=(const TSthashIterator& that) const { - return Data != that.Data; - } - bool operator==(const TSthashIterator& that) const { - return Data == that.Data; - } - const char* Key() const { - return Data + SizeOfEx<TValue>(); - } - TValue& Value() { - return *(TValue*)Data; - } - const TValue& Value() const { - return *(const TValue*)Data; - } - - template <class K> - bool KeyEquals(const EqualKey& eq, const K& k) const { - return eq(Data + SizeOfEx<TValue>(), k); - } - - size_t GetLength() const { - size_t length = strlen(Data + SizeOfEx<TValue>()) + 1 + SizeOfEx<TValue>(); - length = AlignUp(length, AlignForChrKey<TValue>()); - return length; - } -}; - -template <class Value, typename size_type_o> -struct TSthashWriter<const char*, Value, size_type_o> { - typedef size_type_o TSizeType; - size_t GetRecordSize(const std::pair<const char*, const Value>& record) const { - size_t length = strlen(record.first) + 1 + SizeOfEx<Value>(); - length = AlignUp(length, AlignForChrKey<Value>()); - return length; - } - int SaveRecord(IOutputStream* stream, const std::pair<const char*, const Value>& record) const { - const char* alignBuffer = "qqqq"; - stream->Write(&record.second, SizeOfEx<Value>()); - size_t length = strlen(record.first) + 1; - stream->Write(record.first, length); - length = AlignUpSpace(length, AlignForChrKey<Value>()); - if (length) - stream->Write(alignBuffer, length); - return 0; - } -}; - -template <class TKey, class HashFcn, class EqualKey> -struct TSthashIterator<TKey, const char* const, HashFcn, EqualKey> { - typedef const TKey TKeyType; - typedef const char* TValueType; - typedef EqualKey TKeyEqualType; - typedef HashFcn THasherType; - - const char* Data; - TSthashIterator() - : Data(nullptr) - { - } - TSthashIterator(const char* data) - : Data(data) - { - } - void operator++() { - Data += GetLength(); - } - - bool operator!=(const TSthashIterator& that) const { - return Data != that.Data; - } - bool operator==(const TSthashIterator& that) const { - return Data == that.Data; - } - TKey& Key() { - return *(TKey*)Data; - } - const char* Value() const { - return Data + sizeof(TKey); - } - - template <class K> - bool KeyEquals(const EqualKey& eq, const K& k) const { - return eq(*(TKey*)Data, k); - } - - size_t GetLength() const { - size_t length = strlen(Data + sizeof(TKey)) + 1 + sizeof(TKey); - length = AlignUp(length, (size_t)4); - return length; - } -}; - -template <class Key, typename size_type_o> -struct TSthashWriter<Key, const char*, size_type_o> { - typedef size_type_o TSizeType; - size_t GetRecordSize(const std::pair<const Key, const char*>& record) const { - size_t length = strlen(record.second) + 1 + sizeof(Key); - length = AlignUp(length, (size_t)4); - return length; - } - int SaveRecord(IOutputStream* stream, const std::pair<const Key, const char*>& record) const { - const char* alignBuffer = "qqqq"; - stream->Write(&record.first, sizeof(Key)); - size_t length = strlen(record.second) + 1; - stream->Write(record.second, length); - length = AlignUpSpace(length, (size_t)4); - if (length) - stream->Write(alignBuffer, length); - return 0; - } -}; - -template <class HashFcn, class EqualKey> -struct TSthashIterator<const char* const, const char* const, HashFcn, EqualKey> { - typedef const char* TKeyType; - typedef const char* TValueType; - typedef EqualKey TKeyEqualType; - typedef HashFcn THasherType; - - const char* Data; - TSthashIterator() - : Data(nullptr) - { - } - TSthashIterator(const char* data) - : Data(data) - { - } - void operator++() { - Data += GetLength(); - } - - bool operator!=(const TSthashIterator& that) const { - return Data != that.Data; - } - bool operator==(const TSthashIterator& that) const { - return Data == that.Data; - } - const char* Key() const { - return Data; - } - const char* Value() const { - return Data + strlen(Data) + 1; - } - - template <class K> - bool KeyEquals(const EqualKey& eq, const K& k) const { - return eq(Data, k); - } - - size_t GetLength() const { - size_t length = strlen(Data) + 1; - length += strlen(Data + length) + 1; - return length; - } -}; - -template <typename size_type_o> -struct TSthashWriter<const char*, const char*, size_type_o> { - typedef size_type_o TSizeType; - size_t GetRecordSize(const std::pair<const char*, const char*>& record) const { - size_t size = strlen(record.first) + strlen(record.second) + 2; - return size; - } - int SaveRecord(IOutputStream* stream, const std::pair<const char*, const char*>& record) const { - stream->Write(record.first, strlen(record.first) + 1); - stream->Write(record.second, strlen(record.second) + 1); - return 0; - } -}; diff --git a/library/cpp/on_disk/st_hash/ya.make b/library/cpp/on_disk/st_hash/ya.make deleted file mode 100644 index 8c6d05711c..0000000000 --- a/library/cpp/on_disk/st_hash/ya.make +++ /dev/null @@ -1,15 +0,0 @@ -LIBRARY() - -SRCS( - fake.cpp - save_stl.h - static_hash.h - static_hash_map.h - sthash_iterators.h -) - -PEERDIR( - library/cpp/deprecated/mapped_file -) - -END() diff --git a/library/cpp/remmap/remmap.cpp b/library/cpp/remmap/remmap.cpp deleted file mode 100644 index ce72af7352..0000000000 --- a/library/cpp/remmap/remmap.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include <util/system/info.h> -#include <util/system/defaults.h> - -#if defined(_win_) -#include <util/system/winint.h> -#elif defined(_unix_) -#include <sys/types.h> -#include <sys/mman.h> - -#ifndef MAP_NOCORE -#define MAP_NOCORE 0 -#endif -#else -#error todo -#endif - -#include "remmap.h" - -static const size_t REMMAP_PAGESIZE = NSystemInfo::GetPageSize(); - -#if defined(_unix_) -TRemmapAllocation::TRemmapAllocation() - : Ptr_(nullptr) - , Size_(0) -{ -} - -TRemmapAllocation::TRemmapAllocation(size_t size, char* base) - : Ptr_(nullptr) - , Size_(0) -{ - Alloc(size, base); -} - -char* TRemmapAllocation::Alloc(size_t size, char* base) { - assert(Ptr_ == nullptr); - - if (!size) - return nullptr; - - const size_t HUGESIZE = size_t(16) << 30; - Ptr_ = CommonMMap(HUGESIZE, base); - - if (Ptr_ != (char*)MAP_FAILED) - munmap((void*)Ptr_, HUGESIZE); - else - Ptr_ = nullptr; - - Ptr_ = CommonMMap(AlignUp(size, REMMAP_PAGESIZE), Ptr_); - if (Ptr_ == (char*)MAP_FAILED) - Ptr_ = nullptr; - - Size_ = Ptr_ ? size : 0; - return Ptr_; -} - -char* TRemmapAllocation::Realloc(size_t newsize) { - if (Ptr_ == nullptr) - return Alloc(newsize); - - size_t realSize = AlignUp(Size_, REMMAP_PAGESIZE); - size_t needSize = AlignUp(newsize, REMMAP_PAGESIZE); - - if (needSize > realSize) { - char* part = Ptr_ + realSize; - char* bunch = CommonMMap(needSize - realSize, part); - if (bunch != (char*)MAP_FAILED && bunch != part) - munmap(bunch, needSize - realSize); - if (bunch == (char*)MAP_FAILED || bunch != part) - return FullRealloc(newsize); - } else if (needSize < realSize) - munmap(Ptr_ + needSize, realSize - needSize); - - if ((Size_ = newsize) == 0) - Ptr_ = nullptr; - - return Ptr_; -} - -void TRemmapAllocation::Dealloc() { - if (Ptr_ != nullptr) - munmap(Ptr_, AlignUp(Size_, REMMAP_PAGESIZE)); - Ptr_ = nullptr; - Size_ = 0; -} - -char* TRemmapAllocation::FullRealloc(size_t newsize) { - char* newPtr = CommonMMap(newsize); - Y_ABORT_UNLESS(newPtr != MAP_FAILED, "mmap failed"); - - size_t useful = Min(Size_, newsize), cur = 0; - - for (; cur + REMMAP_PAGESIZE < useful; cur += REMMAP_PAGESIZE) { - memcpy((void*)&newPtr[cur], (void*)&Ptr_[cur], REMMAP_PAGESIZE); - munmap((void*)&Ptr_[cur], REMMAP_PAGESIZE); - } - - memcpy((void*)&newPtr[cur], (void*)&Ptr_[cur], useful - cur); - munmap((void*)&Ptr_[cur], AlignUp(Size_ - cur, REMMAP_PAGESIZE)); - - Size_ = newsize; - return (Ptr_ = newPtr); -} - -inline char* TRemmapAllocation::CommonMMap(size_t size, char* base) { - return (char*)mmap((void*)base, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); -} - -#else -TRemmapAllocation::TRemmapAllocation() - : Allocation_(0, false, NULL) -{ -} - -TRemmapAllocation::TRemmapAllocation(size_t size, char* base) - : Allocation_(size, false, (void*)base) -{ -} - -char* TRemmapAllocation::Alloc(size_t size, char* base) { - return (char*)Allocation_.Alloc(size, (void*)base); -} - -char* TRemmapAllocation::Realloc(size_t newsize) { - return FullRealloc(newsize); -} - -void TRemmapAllocation::Dealloc() { - Allocation_.Dealloc(); -} - -char* TRemmapAllocation::FullRealloc(size_t newsize) { - TMappedAllocation other(newsize); - memcpy(other.Ptr(), Allocation_.Ptr(), Min(other.MappedSize(), Allocation_.MappedSize())); - Allocation_.swap(other); - return Data(); -} -#endif diff --git a/library/cpp/remmap/remmap.h b/library/cpp/remmap/remmap.h deleted file mode 100644 index 7cb738f7ae..0000000000 --- a/library/cpp/remmap/remmap.h +++ /dev/null @@ -1,64 +0,0 @@ -#pragma once - -#include <util/system/yassert.h> -#include <util/system/align.h> -#include <util/system/info.h> -#include <util/system/filemap.h> -#include <util/memory/alloc.h> -#include <util/generic/noncopyable.h> - -class TRemmapAllocation : TNonCopyable { -public: - TRemmapAllocation(); - TRemmapAllocation(size_t size, char* base = nullptr); - - ~TRemmapAllocation() { - Dealloc(); - } - - char* Alloc(size_t size, char* base = nullptr); - char* Realloc(size_t newsize); - void Dealloc(); - char* FullRealloc(size_t newsize); - -#if defined(_unix_) -private: - inline char* CommonMMap(size_t size, char* base = nullptr); - - char* Ptr_; - size_t Size_; - -public: - inline void* Ptr() const { - return (void*)Ptr_; - } - inline char* Data(ui32 pos = 0) const { - return Ptr_ + pos; - } - inline size_t Size() const { - return Size_; - } - inline void swap(TRemmapAllocation& other) { - DoSwap(Ptr_, other.Ptr_); - DoSwap(Size_, other.Size_); - } - -#else -private: - TMappedAllocation Allocation_; - -public: - inline void* Ptr() const { - return Allocation_.Ptr(); - } - inline char* Data(ui32 pos = 0) const { - return Allocation_.Data(pos); - } - inline size_t Size() const { - return Allocation_.MappedSize(); - } - inline void swap(TRemmapAllocation& other) { - Allocation_.swap(other.Allocation_); - } -#endif -}; diff --git a/library/cpp/remmap/ya.make b/library/cpp/remmap/ya.make deleted file mode 100644 index 281df6443a..0000000000 --- a/library/cpp/remmap/ya.make +++ /dev/null @@ -1,7 +0,0 @@ -LIBRARY() - -SRCS( - remmap.cpp -) - -END() diff --git a/library/cpp/sqlite3/sqlite.cpp b/library/cpp/sqlite3/sqlite.cpp deleted file mode 100644 index 98e498f76b..0000000000 --- a/library/cpp/sqlite3/sqlite.cpp +++ /dev/null @@ -1,288 +0,0 @@ -#include "sqlite.h" - -#include <util/generic/singleton.h> -#include <util/generic/scope.h> - -#include <cstdlib> - -using namespace NSQLite; - -namespace { - struct TSQLiteInit { - inline TSQLiteInit() { - int ret = sqlite3_config(SQLITE_CONFIG_MULTITHREAD); - - if (ret != SQLITE_OK) { - ythrow TSQLiteError(ret) << "init failure"; - } - } - - static inline void Ensure() { - Singleton<TSQLiteInit>(); - } - }; -} - -namespace NSQLite { - TSQLiteError::TSQLiteError(sqlite3* hndl) - : ErrorCode(sqlite3_errcode(hndl)) - { - *this << sqlite3_errmsg(hndl) << ". "; - } - - TSQLiteError::TSQLiteError(int rc) - : ErrorCode(rc) - { - *this << sqlite3_errstr(rc) << " (" << rc << "). "; - } - - TSQLiteDB::TSQLiteDB(const TString& path) { - TSQLiteInit::Ensure(); - - sqlite3* db = nullptr; - const int rc = sqlite3_open(path.data(), &db); - - H_.Reset(db); - - if (rc) { - ythrow TSQLiteError(Handle()) << "can not init db " << path.Quote(); - } - } - - TSQLiteDB::TSQLiteDB(const TString& path, int flags) { - TSQLiteInit::Ensure(); - - sqlite3* db = nullptr; - const int rc = sqlite3_open_v2(path.data(), &db, flags, nullptr); - - H_.Reset(db); - - if (rc) { - ythrow TSQLiteError(Handle()) << "can not init db " << path.Quote(); - } - } - - sqlite3* TSQLiteDB::Handle() const noexcept { - return H_.Get(); - } - - size_t TSQLiteDB::RowsAffected() const noexcept { - return static_cast<size_t>(sqlite3_changes(H_.Get())); - } - - TSQLiteStatement::TSQLiteStatement(TSQLiteDB& db, const TString& s) - : S_(s) - { - if (!S_.empty() && S_[S_.size() - 1] != ';') { - S_ += ';'; - } - - sqlite3_stmt* st = nullptr; - const char* tail = nullptr; - const int rc = sqlite3_prepare_v2(db.Handle(), S_.data(), S_.size() + 1, &st, &tail); - - H_.Reset(st); - - if (rc != SQLITE_OK) { - ythrow TSQLiteError(db.Handle()) << "can not prepare " << S_.Quote(); - } - } - - void TSQLiteStatement::Execute() { - while (Step()) { - } - - Reset(); - } - - TSQLiteStatement& TSQLiteStatement::Bind(size_t idx, i64 val) { - sqlite3_bind_int64(Handle(), idx, val); - return *this; - } - - TSQLiteStatement& TSQLiteStatement::Bind(size_t idx, int val) { - sqlite3_bind_int(Handle(), idx, val); - return *this; - } - - TSQLiteStatement& TSQLiteStatement::Bind(size_t idx) { - sqlite3_bind_null(Handle(), idx); - return *this; - } - - TSQLiteStatement& TSQLiteStatement::Bind(size_t idx, double val) { - sqlite3_bind_double(Handle(), idx, val); - return *this; - } - - void TSQLiteStatement::BindText(size_t idx, const char* text, size_t len, TFreeFunc func) { - sqlite3_bind_text(Handle(), idx, text, len, func); - } - - TSQLiteStatement& TSQLiteStatement::Bind(size_t idx, TStringBuf str) { - BindText(idx, str.data(), str.size(), SQLITE_STATIC); - return *this; - } - - TSQLiteStatement& TSQLiteStatement::BindBlob(size_t idx, TStringBuf blob) { - sqlite3_bind_blob(Handle(), idx, blob.data(), blob.size(), SQLITE_STATIC); - return *this; - } - - size_t TSQLiteStatement::BoundNamePosition(TStringBuf name) const noexcept { - return sqlite3_bind_parameter_index(Handle(), name.data()); - } - - size_t TSQLiteStatement::BoundParameterCount() const noexcept { - return sqlite3_bind_parameter_count(Handle()); - } - - const char* TSQLiteStatement::BoundParameterName(size_t idx) const noexcept { - return sqlite3_bind_parameter_name(Handle(), idx); - } - - sqlite3_stmt* TSQLiteStatement::Handle() const noexcept { - return H_.Get(); - } - - bool TSQLiteStatement::Step() { - const int rc = sqlite3_step(Handle()); - - switch (rc) { - case SQLITE_ROW: - return true; - - case SQLITE_DONE: - return false; - - default: - break; - } - - char* stmt = rc == SQLITE_CONSTRAINT ? sqlite3_expanded_sql(Handle()) : nullptr; - Y_DEFER { - if (stmt != nullptr) { - sqlite3_free(reinterpret_cast<void*>(stmt)); - stmt = nullptr; - } - }; - if (stmt != nullptr) { - ythrow TSQLiteError(rc) << "step failed: " << stmt; - } else { - ythrow TSQLiteError(rc) << "step failed"; - } - } - - i64 TSQLiteStatement::ColumnInt64(size_t idx) { - return sqlite3_column_int64(Handle(), idx); - } - - double TSQLiteStatement::ColumnDouble(size_t idx) { - return sqlite3_column_double(Handle(), idx); - } - - TStringBuf TSQLiteStatement::ColumnText(size_t idx) { - return reinterpret_cast<const char*>(sqlite3_column_text(Handle(), idx)); - } - - TStringBuf TSQLiteStatement::ColumnBlob(size_t idx) { - const void* blob = sqlite3_column_blob(Handle(), idx); - size_t size = sqlite3_column_bytes(Handle(), idx); - return TStringBuf(static_cast<const char*>(blob), size); - } - - void TSQLiteStatement::ColumnAccept(size_t idx, ISQLiteColumnVisitor& visitor) { - const auto columnType = sqlite3_column_type(Handle(), idx); - switch (columnType) { - case SQLITE_INTEGER: - visitor.OnColumnInt64(ColumnInt64(idx)); - break; - case SQLITE_FLOAT: - visitor.OnColumnDouble(ColumnDouble(idx)); - break; - case SQLITE_TEXT: - visitor.OnColumnText(ColumnText(idx)); - break; - case SQLITE_BLOB: - visitor.OnColumnBlob(ColumnBlob(idx)); - break; - case SQLITE_NULL: - visitor.OnColumnNull(); - break; - } - } - - size_t TSQLiteStatement::ColumnCount() const noexcept { - return static_cast<size_t>(sqlite3_column_count(Handle())); - } - - TStringBuf TSQLiteStatement::ColumnName(size_t idx) const noexcept { - return sqlite3_column_name(Handle(), idx); - } - - void TSQLiteStatement::Reset() { - const int rc = sqlite3_reset(Handle()); - - if (rc != SQLITE_OK) { - ythrow TSQLiteError(rc) << "reset failed"; - } - } - - void TSQLiteStatement::ResetHard() { - (void)sqlite3_reset(Handle()); - } - - void TSQLiteStatement::ClearBindings() noexcept { - // No error is documented. - // sqlite3.c's code always returns SQLITE_OK. - (void)sqlite3_clear_bindings(Handle()); - } - - TSQLiteTransaction::TSQLiteTransaction(TSQLiteDB& db) - : Db(&db) - { - Execute("BEGIN TRANSACTION"); - } - - TSQLiteTransaction::~TSQLiteTransaction() { - if (Db) { - Rollback(); - } - } - - void TSQLiteTransaction::Commit() { - Execute("COMMIT TRANSACTION"); - Db = nullptr; - } - - void TSQLiteTransaction::Rollback() { - Execute("ROLLBACK TRANSACTION"); - Db = nullptr; - } - - void TSQLiteTransaction::Execute(const TString& query) { - Y_ENSURE(Db, "Transaction is already ended"); - TSQLiteStatement st(*Db, query); - st.Execute(); - } - - TSimpleDB::TSimpleDB(const TString& path) - : TSQLiteDB(path) - , Start_(*this, "begin transaction") - , End_(*this, "end transaction") - { - } - - void TSimpleDB::Execute(const TString& statement) { - TSQLiteStatement(*this, statement).Execute(); - } - - void TSimpleDB::Acquire() { - Start_.Execute(); - } - - void TSimpleDB::Release() { - End_.Execute(); - } - -} diff --git a/library/cpp/sqlite3/sqlite.h b/library/cpp/sqlite3/sqlite.h deleted file mode 100644 index 8b35e2606a..0000000000 --- a/library/cpp/sqlite3/sqlite.h +++ /dev/null @@ -1,136 +0,0 @@ -#pragma once - -#include <util/generic/yexception.h> -#include <util/generic/ptr.h> - -#include <contrib/libs/sqlite3/sqlite3.h> - -namespace NSQLite { - class TSQLiteError: public yexception { - public: - TSQLiteError(sqlite3* hndl); - TSQLiteError(int rc); - - int GetErrorCode() const { - return ErrorCode; - } - - private: - int ErrorCode; - }; - - template <class T, int (*Func)(T*)> - struct TCFree { - static void Destroy(T* t) { - Func(t); - } - }; - - class TSQLiteDB { - public: - TSQLiteDB(const TString& path, int flags); - TSQLiteDB(const TString& path); - - sqlite3* Handle() const noexcept; - size_t RowsAffected() const noexcept; - - private: - THolder<sqlite3, TCFree<sqlite3, sqlite3_close>> H_; - }; - - class ISQLiteColumnVisitor { - public: - virtual ~ISQLiteColumnVisitor() = default; - - virtual void OnColumnInt64(i64 value) = 0; - virtual void OnColumnDouble(double value) = 0; - virtual void OnColumnText(TStringBuf value) = 0; - virtual void OnColumnBlob(TStringBuf value) = 0; - virtual void OnColumnNull() = 0; - }; - - class TSQLiteStatement { - public: - TSQLiteStatement(TSQLiteDB& db, const TString& s); - - void Execute(); - TSQLiteStatement& Bind(size_t idx, i64 val); - TSQLiteStatement& Bind(size_t idx, int val); - TSQLiteStatement& Bind(size_t idx); - TSQLiteStatement& Bind(size_t idx, double val); - TSQLiteStatement& Bind(size_t idx, TStringBuf str); - TSQLiteStatement& BindBlob(size_t idx, TStringBuf blob); - template <typename Value> - TSQLiteStatement& Bind(TStringBuf name, Value val) { - size_t idx = BoundNamePosition(name); - Y_ASSERT(idx > 0); - return Bind(idx, val); - } - TSQLiteStatement& BindBlob(TStringBuf name, TStringBuf blob) { - size_t idx = BoundNamePosition(name); - Y_ASSERT(idx > 0); - return BindBlob(idx, blob); - } - TSQLiteStatement& Bind(TStringBuf name) { - size_t idx = BoundNamePosition(name); - Y_ASSERT(idx > 0); - return Bind(idx); - } - size_t BoundNamePosition(TStringBuf name) const noexcept; - size_t BoundParameterCount() const noexcept; - const char* BoundParameterName(size_t idx) const noexcept; - - sqlite3_stmt* Handle() const noexcept; - bool Step(); - i64 ColumnInt64(size_t idx); - double ColumnDouble(size_t idx); - TStringBuf ColumnText(size_t idx); - TStringBuf ColumnBlob(size_t idx); - void ColumnAccept(size_t idx, ISQLiteColumnVisitor& visitor); - size_t ColumnCount() const noexcept; - TStringBuf ColumnName(size_t idx) const noexcept; - void Reset(); - // Ignore last error on this statement - void ResetHard(); - void ClearBindings() noexcept; - - private: - typedef void (*TFreeFunc)(void*); - void BindText(size_t col, const char* text, size_t len, TFreeFunc func); - - private: - TString S_; - THolder<sqlite3_stmt, TCFree<sqlite3_stmt, sqlite3_finalize>> H_; - }; - - /** - * Forces user to commit transaction explicitly, to not get exception in destructor (with all consequences of it). - */ - class TSQLiteTransaction: private TNonCopyable { - private: - TSQLiteDB* Db; - - public: - TSQLiteTransaction(TSQLiteDB& db); - ~TSQLiteTransaction(); - - void Commit(); - void Rollback(); - - private: - void Execute(const TString& query); - }; - - class TSimpleDB: public TSQLiteDB { - public: - TSimpleDB(const TString& path); - - void Execute(const TString& statement); - void Acquire(); - void Release(); - - private: - TSQLiteStatement Start_; - TSQLiteStatement End_; - }; -} diff --git a/library/cpp/sqlite3/ya.make b/library/cpp/sqlite3/ya.make deleted file mode 100644 index 15417e278d..0000000000 --- a/library/cpp/sqlite3/ya.make +++ /dev/null @@ -1,13 +0,0 @@ -LIBRARY() - -SRCS( - sqlite.cpp -) - -PEERDIR( - contrib/libs/sqlite3 -) - -END() - -RECURSE_FOR_TESTS(ut) diff --git a/library/cpp/streams/growing_file_input/growing_file_input.cpp b/library/cpp/streams/growing_file_input/growing_file_input.cpp deleted file mode 100644 index 0bbfa5ade9..0000000000 --- a/library/cpp/streams/growing_file_input/growing_file_input.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include "growing_file_input.h" - -#include <util/datetime/base.h> -#include <util/generic/yexception.h> - -TGrowingFileInput::TGrowingFileInput(const TString& path) - : File_(path, OpenExisting | RdOnly | Seq) -{ - if (!File_.IsOpen()) { - ythrow TIoException() << "file " << path << " not open"; - } - - File_.Seek(0, sEnd); -} - -TGrowingFileInput::TGrowingFileInput(const TFile& file) - : File_(file) -{ - if (!File_.IsOpen()) { - ythrow TIoException() << "file (" << file.GetName() << ") not open"; - } - - File_.Seek(0, sEnd); -} - -size_t TGrowingFileInput::DoRead(void* buf, size_t len) { - for (int sleepTime = 1;;) { - size_t rr = File_.Read(buf, len); - - if (rr != 0) { - return rr; - } - - NanoSleep((ui64)sleepTime * 1000000); - - if (sleepTime < 2000) { - sleepTime <<= 1; - } - } -} diff --git a/library/cpp/streams/growing_file_input/growing_file_input.h b/library/cpp/streams/growing_file_input/growing_file_input.h deleted file mode 100644 index 9054a5f3da..0000000000 --- a/library/cpp/streams/growing_file_input/growing_file_input.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include <util/stream/input.h> -#include <util/system/file.h> - -/** - * Growing file input stream. - * - * File descriptor offsets to the end of the file, when the object is created. - * - * Read function waites for reading at least one byte. - */ -class TGrowingFileInput: public IInputStream { -public: - TGrowingFileInput(const TFile& file); - TGrowingFileInput(const TString& path); - -private: - size_t DoRead(void* buf, size_t len) override; - -private: - TFile File_; -}; diff --git a/library/cpp/streams/growing_file_input/ya.make b/library/cpp/streams/growing_file_input/ya.make deleted file mode 100644 index 69c56fea46..0000000000 --- a/library/cpp/streams/growing_file_input/ya.make +++ /dev/null @@ -1,11 +0,0 @@ -LIBRARY() - -SRCS( - growing_file_input.cpp -) - -END() - -RECURSE_FOR_TESTS( - ut -) diff --git a/library/cpp/string_utils/subst_buf/substbuf.cpp b/library/cpp/string_utils/subst_buf/substbuf.cpp deleted file mode 100644 index f23cb24b19..0000000000 --- a/library/cpp/string_utils/subst_buf/substbuf.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "substbuf.h" diff --git a/library/cpp/string_utils/subst_buf/substbuf.h b/library/cpp/string_utils/subst_buf/substbuf.h deleted file mode 100644 index 357ee68ae3..0000000000 --- a/library/cpp/string_utils/subst_buf/substbuf.h +++ /dev/null @@ -1,63 +0,0 @@ -#pragma once - -#include <util/generic/vector.h> -#include <util/generic/strbuf.h> -#include <util/string/subst.h> - -/// Заменяет в строке одни подстроки на другие. -template <class TBuf, class TPool> -size_t SubstGlobal(TBuf& s, const TBuf& from, const TBuf& to, TPool& pool) { - if (from.empty()) - return 0; - - TVector<size_t> offs; - for (size_t off = 0; (off = s.find(from, off)) != TBuf::npos; off += from.length()) - offs.push_back(off); - if (offs.empty()) - return 0; - - size_t dstSize = s.size() + ssize_t(offs.size()) * ssize_t(to.size() - from.size()); - const size_t charTypeSz = sizeof(typename TBuf::char_type); - typename TBuf::char_type* dst = (typename TBuf::char_type*)pool.Allocate((dstSize + 1) * charTypeSz); - dst[dstSize] = 0; - - typename TBuf::char_type* p = dst; - size_t lastSrc = 0; - for (auto off : offs) { - memcpy(p, s.data() + lastSrc, (off - lastSrc) * charTypeSz); - p += off - lastSrc; - lastSrc = off + from.size(); - memcpy(p, to.data(), to.size() * charTypeSz); - p += to.size(); - } - memcpy(p, s.data() + lastSrc, (s.size() - lastSrc) * charTypeSz); - p += s.size() - lastSrc; - Y_ASSERT(p - dst == (ssize_t)dstSize); - - s = TBuf(dst, dstSize); - return offs.size(); -} - -template <class TPool> -size_t SubstGlobal(TStringBuf& s, const TStringBuf& from, const TStringBuf& to, TPool& pool) { - return SubstGlobal<TStringBuf, TPool>(s, from, to, pool); -} - -/// Заменяет в строке одни подстроки на другие. -template <class TBuf, class TPool> -inline size_t SubstGlobal(TBuf& s, typename TBuf::char_type from, typename TBuf::char_type to, TPool& pool) { - size_t result = 0; - size_t off = s.find(from); - if (off == TBuf::npos) - return 0; - - s = TBuf(pool.Append(s), s.size()); - - for (typename TBuf::char_type* it = const_cast<typename TBuf::char_type*>(s.begin()) + off; it != s.end(); ++it) { - if (*it == from) { - *it = to; - ++result; - } - } - return result; -} diff --git a/library/cpp/string_utils/subst_buf/ya.make b/library/cpp/string_utils/subst_buf/ya.make deleted file mode 100644 index 8b8793f5b3..0000000000 --- a/library/cpp/string_utils/subst_buf/ya.make +++ /dev/null @@ -1,7 +0,0 @@ -LIBRARY() - -SRCS( - substbuf.cpp -) - -END() diff --git a/library/cpp/ucompress/README.md b/library/cpp/ucompress/README.md deleted file mode 100644 index 5a6e9d8f42..0000000000 --- a/library/cpp/ucompress/README.md +++ /dev/null @@ -1 +0,0 @@ -Compatible implementation of library/python/compress (also known as "uc" - uber compressor: tools/uc, ya tool uc). diff --git a/library/cpp/ucompress/common.h b/library/cpp/ucompress/common.h deleted file mode 100644 index d59cde9cf1..0000000000 --- a/library/cpp/ucompress/common.h +++ /dev/null @@ -1,8 +0,0 @@ -#pragma once - - -namespace NUCompress { - // These limitations come from original implementation - library/python/compress - using TBlockLen = ui32; - constexpr TBlockLen MaxCompressedLen = 100000000; -} diff --git a/library/cpp/ucompress/reader.cpp b/library/cpp/ucompress/reader.cpp deleted file mode 100644 index 45a8ca8da2..0000000000 --- a/library/cpp/ucompress/reader.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include "reader.h" -#include "common.h" - -#include <library/cpp/blockcodecs/codecs.h> -#include <library/cpp/json/json_reader.h> - -#include <util/system/byteorder.h> - - -using namespace NUCompress; - -TDecodedInput::TDecodedInput(IInputStream* in) - : S_(in) -{ - Y_ENSURE_EX(S_, TBadArgumentException() << "Null output stream"); -} - -TDecodedInput::~TDecodedInput() = default; - -size_t TDecodedInput::DoUnboundedNext(const void** ptr) { - if (!C_) { - TBlockLen blockLen = 0; - S_->LoadOrFail(&blockLen, sizeof(blockLen)); - blockLen = LittleToHost(blockLen); - Y_ENSURE(blockLen <= MaxCompressedLen, "broken stream"); - - TString buf = TString::Uninitialized(blockLen); - S_->LoadOrFail(buf.Detach(), blockLen); - - NJson::TJsonValue hdr; - Y_ENSURE(NJson::ReadJsonTree(buf, &hdr), "cannot parse header, suspect old format"); - - auto& codecName = hdr["codec"].GetString(); - Y_ENSURE(codecName, "header does not have codec info"); - - // Throws TNotFound - C_ = NBlockCodecs::Codec(codecName); - Y_ASSERT(C_); - } - - TBlockLen blockLen = 0; - size_t actualRead = S_->Load(&blockLen, sizeof(blockLen)); - if (!actualRead) { - // End of stream - return 0; - } - Y_ENSURE(actualRead == sizeof(blockLen), "broken stream: cannot read block length"); - blockLen = LittleToHost(blockLen); - Y_ENSURE(blockLen <= MaxCompressedLen, "broken stream"); - - TBuffer block; - block.Resize(blockLen); - S_->LoadOrFail(block.Data(), blockLen); - - C_->Decode(block, D_); - *ptr = D_.Data(); - return D_.Size(); -} diff --git a/library/cpp/ucompress/reader.h b/library/cpp/ucompress/reader.h deleted file mode 100644 index 5a5d1c9a89..0000000000 --- a/library/cpp/ucompress/reader.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include <util/generic/buffer.h> -#include <util/stream/walk.h> - - -namespace NBlockCodecs { - struct ICodec; -} - -namespace NUCompress { - class TDecodedInput: public IWalkInput { - public: - TDecodedInput(IInputStream* in); - ~TDecodedInput() override; - - private: - size_t DoUnboundedNext(const void** ptr) override; - - private: - IInputStream* const S_; - const NBlockCodecs::ICodec* C_ = nullptr; - TBuffer D_; - }; -} diff --git a/library/cpp/ucompress/writer.cpp b/library/cpp/ucompress/writer.cpp deleted file mode 100644 index 40f8b12108..0000000000 --- a/library/cpp/ucompress/writer.cpp +++ /dev/null @@ -1,95 +0,0 @@ -#include "writer.h" -#include "common.h" - -#include <library/cpp/blockcodecs/codecs.h> -#include <library/cpp/json/writer/json.h> - -#include <util/generic/scope.h> -#include <util/generic/yexception.h> -#include <util/system/byteorder.h> - - -using namespace NUCompress; - -TCodedOutput::TCodedOutput(IOutputStream* out, const NBlockCodecs::ICodec* c, size_t bufLen) - : C_(c) - , D_(bufLen) - , S_(out) -{ - Y_ENSURE_EX(C_, TBadArgumentException() << "Null codec"); - Y_ENSURE_EX(S_, TBadArgumentException() << "Null output stream"); - D_.Resize(bufLen); - Y_ENSURE_EX(C_->MaxCompressedLength(D_) <= MaxCompressedLen, TBadArgumentException() << "Too big buffer size: " << bufLen); - D_.Clear(); -} - -TCodedOutput::~TCodedOutput() { - try { - Finish(); - } catch (...) { - } -} - -void TCodedOutput::DoWrite(const void* buf, size_t len) { - Y_ENSURE(S_, "Stream finished already"); - const char* in = static_cast<const char*>(buf); - - while (len) { - const size_t avail = D_.Avail(); - if (len < avail) { - D_.Append(in, len); - return; - } - - D_.Append(in, avail); - Y_ASSERT(!D_.Avail()); - in += avail; - len -= avail; - - FlushImpl(); - } -} - -void TCodedOutput::FlushImpl() { - if (!HdrWritten) { - NJsonWriter::TBuf jBuf; - jBuf.BeginObject(); - jBuf.WriteKey("codec"); - jBuf.WriteString(C_->Name()); - jBuf.EndObject(); - - TString jStr = jBuf.Str() + '\n'; - const TBlockLen lenToSave = HostToLittle(jStr.length()); - S_->Write(&lenToSave, sizeof(lenToSave)); - S_->Write(jStr.Detach(), jStr.length()); - HdrWritten = true; - } - - O_.Reserve(C_->MaxCompressedLength(D_)); - const size_t oLen = C_->Compress(D_, O_.Data()); - Y_ASSERT(oLen <= MaxCompressedLen); - - const TBlockLen lenToSave = HostToLittle(oLen); - S_->Write(&lenToSave, sizeof(lenToSave)); - S_->Write(O_.Data(), oLen); - - D_.Clear(); - O_.Clear(); -} - -void TCodedOutput::DoFlush() { - if (S_ && D_) { - FlushImpl(); - } -} - -void TCodedOutput::DoFinish() { - if (S_) { - Y_DEFER { - S_ = nullptr; - }; - FlushImpl(); - // Write zero-length block as EOF marker. - FlushImpl(); - } -} diff --git a/library/cpp/ucompress/writer.h b/library/cpp/ucompress/writer.h deleted file mode 100644 index 4d3ae71093..0000000000 --- a/library/cpp/ucompress/writer.h +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include <util/generic/buffer.h> -#include <util/stream/output.h> - - -namespace NBlockCodecs { - struct ICodec; -} - -namespace NUCompress { - class TCodedOutput: public IOutputStream { - public: - TCodedOutput(IOutputStream* out, const NBlockCodecs::ICodec* c, size_t bufLen = 16 << 20); - ~TCodedOutput() override; - - private: - void DoWrite(const void* buf, size_t len) override; - void DoFlush() override; - void DoFinish() override; - - void FlushImpl(); - - private: - const NBlockCodecs::ICodec* const C_; - TBuffer D_; - TBuffer O_; - IOutputStream* S_; - bool HdrWritten = false; - }; -} diff --git a/library/cpp/ucompress/ya.make b/library/cpp/ucompress/ya.make deleted file mode 100644 index 6582dd9a41..0000000000 --- a/library/cpp/ucompress/ya.make +++ /dev/null @@ -1,18 +0,0 @@ -LIBRARY() - -PEERDIR( - library/cpp/blockcodecs - library/cpp/json -) - -SRCS( - reader.cpp - writer.cpp -) - -END() - -RECURSE( - tests - ut -) diff --git a/library/cpp/zipatch/reader.cpp b/library/cpp/zipatch/reader.cpp deleted file mode 100644 index 03ac365da1..0000000000 --- a/library/cpp/zipatch/reader.cpp +++ /dev/null @@ -1,173 +0,0 @@ -#include "reader.h" - -#include <library/cpp/json/json_reader.h> -#include <library/cpp/json/json_value.h> - -#include <util/generic/hash.h> -#include <util/memory/tempbuf.h> - -#include <contrib/libs/libarchive/libarchive/archive.h> -#include <contrib/libs/libarchive/libarchive/archive_entry.h> - -using namespace NJson; - -namespace NZipatch { - -class TReader::TImpl { - - using TEntry = archive_entry; - -public: - TImpl() { - if ((Archive_ = archive_read_new()) == nullptr) { - ythrow yexception() << "can't create archive object"; - } - } - - TImpl(const TFsPath& path) - : TImpl() - { - archive_read_support_filter_all(Archive_); - archive_read_support_format_zip(Archive_); - - if (ARCHIVE_OK != archive_read_open_filename(Archive_, TString(path).c_str(), 10240)) { - ythrow yexception() << "can't open archive path = " << path; - } - - Read(); - } - - TImpl(const TStringBuf buf) - : TImpl() - { - archive_read_support_filter_all(Archive_); - archive_read_support_format_zip(Archive_); - - if (ARCHIVE_OK != archive_read_open_memory(Archive_, buf.data(), buf.size())) { - ythrow yexception() << "can't open in-memory archive"; - } - - Read(); - } - - ~TImpl() { - for (const auto& item : Files_) { - archive_entry_free(item.second.first); - } - if (Archive_) { - archive_read_free(Archive_); - } - } - - void Enumerate(TOnEvent cb) const { - for (const auto& item : Actions_) { - TEvent event; - - event.Action = GetTypeFromString(item["type"].GetStringSafe(TString())); - event.Path = item["path"].GetStringSafe(TString()); - event.Executable = item["executable"].GetBooleanSafe(false); - event.Symlink = false; - - if (event.Action == Copy || event.Action == Move) { - event.Source.Path = item["orig_path"].GetStringSafe(TString()); - event.Source.Revision = item["orig_revision"].GetUIntegerRobust(); - } - if (event.Action == StoreFile) { - auto fi = Files_.find(event.Path); - if (fi == Files_.end()) { - ythrow yexception() << "can't find file; path = " << event.Path; - } - - event.Data = fi->second.second; - event.Symlink = archive_entry_filetype(fi->second.first) == AE_IFLNK; - } - - if (event.Path) { - cb(event); - } - } - } - -private: - EAction GetTypeFromString(const TString& type) const { - if (type == "store_file") { - return StoreFile; - } - if (type == "mkdir") { - return MkDir; - } - if (type == "remove_file" || type == "remove_tree") { - return Remove; - } - if (type == "svn_copy") { - return Copy; - } - return Unknown; - } - - void Read() { - TEntry* current = nullptr; - - while (archive_read_next_header(Archive_, ¤t) == ARCHIVE_OK) { - const TStringBuf path(archive_entry_pathname(current)); - - if (path == "actions.json") { - TJsonValue value; - ReadJsonFastTree(GetData(current), &value, true); - - for (const auto& item : value.GetArraySafe()) { - Actions_.push_back(item); - } - } else if (AsciiHasPrefix(path, "files/")) { - TEntry* entry = archive_entry_clone(current); - - Files_.emplace(path.substr(6), std::make_pair(entry, GetData(current))); - } - } - - archive_read_close(Archive_); - } - - TString GetData(TEntry* current) const { - if (archive_entry_filetype(current) == AE_IFLNK) { - return archive_entry_symlink(current); - } - - if (const auto size = archive_entry_size(current)) { - TTempBuf data(size); - - if (archive_read_data(Archive_, data.Data(), size) != size) { - ythrow yexception() << "can't read entry"; - } - - return TString(data.Data(), size); - } - - return TString(); - } - -private: - struct archive* Archive_; - TVector<TJsonValue> Actions_; - THashMap<TString, std::pair<TEntry*, TString>> Files_; -}; - -TReader::TReader(const TFsPath& path) - : Impl_(new TImpl(path)) -{ -} - -TReader::TReader(const TStringBuf buf) - : Impl_(new TImpl(buf)) -{ -} - -TReader::~TReader() -{ } - -void TReader::Enumerate(TOnEvent cb) const { - Impl_->Enumerate(cb); -} - -} // namespace NZipatch - diff --git a/library/cpp/zipatch/reader.h b/library/cpp/zipatch/reader.h deleted file mode 100644 index a94bc79b71..0000000000 --- a/library/cpp/zipatch/reader.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include <util/folder/path.h> -#include <util/generic/ptr.h> - -namespace NZipatch { - -class TReader { -public: - enum EAction { - Unknown = 0, - Copy, - MkDir, - Move, - Remove, - StoreFile, - }; - - struct TSource { - TString Path; - ui64 Revision; - }; - - struct TEvent { - EAction Action; - TString Path; - TStringBuf Data; - TSource Source; - bool Executable; - bool Symlink; - }; - - using TOnEvent = std::function<void(const TEvent&)>; - -public: - TReader(const TFsPath& path); - TReader(const TStringBuf buf); - ~TReader(); - - void Enumerate(TOnEvent cb) const; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - -} // namespace NZipatch - diff --git a/library/cpp/zipatch/writer.cpp b/library/cpp/zipatch/writer.cpp deleted file mode 100644 index a9ca451b01..0000000000 --- a/library/cpp/zipatch/writer.cpp +++ /dev/null @@ -1,232 +0,0 @@ -#include "writer.h" - -#include <library/cpp/json/json_value.h> -#include <library/cpp/json/json_writer.h> - -#include <util/string/join.h> - -#include <contrib/libs/libarchive/libarchive/archive.h> -#include <contrib/libs/libarchive/libarchive/archive_entry.h> - -using namespace NJson; - -namespace NZipatch { - -class TWriter::TImpl { -public: - TImpl(const TFsPath& path) - : Actions_(new TJsonValue(JSON_ARRAY)) - , Meta_(new TJsonValue(JSON_MAP)) - , Revprops_(new TJsonValue(JSON_MAP)) - , Archive_(nullptr) - { - Archive_ = archive_write_new(); - if (!Archive_) { - ythrow yexception() << "can't create archive object"; - } - archive_write_set_format_zip(Archive_); - archive_write_zip_set_compression_deflate(Archive_); - - if (ARCHIVE_OK != archive_write_open_filename(Archive_, TString(path).c_str())) { - ythrow yexception() << "can't open archive path = " << path; - } - } - - ~TImpl() { - if (Actions_ || Meta_ || Revprops_) { - Finish(); - } - if (Archive_) { - archive_write_free(Archive_); - } - } - - void Finish() { - if (Actions_) { - if (Archive_) { - WriteEntry("actions.json", WriteJson(Actions_.Get(), true, false)); - } - - Actions_.Destroy(); - } - - if (Meta_) { - if (Archive_) { - WriteEntry("meta.json", WriteJson(Meta_.Get(), true)); - } - - Meta_.Destroy(); - } - - if (Revprops_) { - if (Archive_) { - WriteEntry("revprops.json", WriteJson(Revprops_.Get(), true)); - } - - Revprops_.Destroy(); - } - - if (Archive_) { - archive_write_close(Archive_); - } - } - - void Copy(const TString& path, const TOrigin& origin) { - Y_ASSERT(origin.Path); - Y_ASSERT(origin.Revision); - - if (Actions_) { - TJsonValue item; - item["type"] = "svn_copy"; - item["path"] = path; - item["orig_path"] = origin.Path; - item["orig_revision"] = origin.Revision; - Actions_->AppendValue(item); - } - } - - void MkDir(const TString& path) { - if (Actions_) { - TJsonValue item; - item["type"] = "mkdir"; - item["path"] = path; - Actions_->AppendValue(item); - } - } - - void RemoveFile(const TString& path) { - if (Actions_) { - TJsonValue item; - item["type"] = "remove_file"; - item["path"] = path; - Actions_->AppendValue(item); - } - } - - void RemoveTree(const TString& path) { - if (Actions_) { - TJsonValue item; - item["type"] = "remove_tree"; - item["path"] = path; - Actions_->AppendValue(item); - } - } - - void StoreFile( - const TString& path, - const TString& data, - const bool execute, - const bool symlink, - const TMaybe<bool> binaryHint, - const TMaybe<bool> encrypted) - { - if (Actions_) { - const TString file = Join("/", "files", path); - TJsonValue item; - item["type"] = "store_file"; - item["executable"] = execute; - item["path"] = path; - item["file"] = file; - if (binaryHint.Defined()) { - item["binary_hint"] = *binaryHint; - } - if (encrypted.Defined()) { - item["encrypted"] = *encrypted; - } - Actions_->AppendValue(item); - WriteEntry(file, data, symlink); - } - } - - void SetBaseSvnRevision(ui64 revision) { - if (Meta_) { - (*Meta_)["base_svn_revision"] = revision; - } - } - - void AddRevprop(const TString& prop, const TString& value) { - if (Revprops_) { - (*Revprops_)[prop] = value; - } - } - -private: - void WriteEntry( - const TString& path, - const TString& data, - const bool symlink = false) - { - struct archive_entry* const entry = archive_entry_new(); - // Write header. - archive_entry_set_pathname(entry, path.c_str()); - archive_entry_set_size(entry, data.size()); - archive_entry_set_filetype(entry, symlink ? AE_IFLNK : AE_IFREG); - archive_entry_set_perm(entry, 0644); - if (symlink) { - archive_entry_set_symlink(entry, data.c_str()); - } - archive_write_header(Archive_, entry); - // Write data. - // If entry is symlink then entry size become zero. - if (archive_entry_size(entry) > 0) { - archive_write_data(Archive_, data.data(), data.size()); - } - archive_entry_free(entry); - } - -private: - THolder<NJson::TJsonValue> Actions_; - THolder<NJson::TJsonValue> Meta_; - THolder<NJson::TJsonValue> Revprops_; - struct archive* Archive_; -}; - -TWriter::TWriter(const TFsPath& path) - : Impl_(new TImpl(path)) -{ -} - -TWriter::~TWriter() -{ } - -void TWriter::Finish() { - Impl_->Finish(); -} - -void TWriter::SetBaseSvnRevision(ui64 revision) { - Impl_->SetBaseSvnRevision(revision); -} - -void TWriter::AddRevprop(const TString& prop, const TString& value) { - Impl_->AddRevprop(prop, value); -} - -void TWriter::Copy(const TString& path, const TOrigin& origin) { - Impl_->Copy(path, origin); -} - -void TWriter::MkDir(const TString& path) { - Impl_->MkDir(path); -} - -void TWriter::RemoveFile(const TString& path) { - Impl_->RemoveFile(path); -} - -void TWriter::RemoveTree(const TString& path) { - Impl_->RemoveTree(path); -} - -void TWriter::StoreFile( - const TString& path, - const TString& data, - const bool execute, - const bool symlink, - const TMaybe<bool> binaryHint, - const TMaybe<bool> encrypted) -{ - Impl_->StoreFile(path, data, execute, symlink, binaryHint, encrypted); -} - -} // namespace NZipatch - diff --git a/library/cpp/zipatch/writer.h b/library/cpp/zipatch/writer.h deleted file mode 100644 index 75cbe49777..0000000000 --- a/library/cpp/zipatch/writer.h +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#include <util/folder/path.h> -#include <util/generic/ptr.h> -#include <util/generic/maybe.h> - -namespace NZipatch { - -class TWriter { -public: - struct TOrigin { - TString Path; - ui64 Revision; - - inline TOrigin(const TString& path, const ui64 revision) - : Path(path) - , Revision(revision) - { } - }; - - TWriter(const TFsPath& path); - ~TWriter(); - - void Finish(); - - void SetBaseSvnRevision(ui64 revision); - - void AddRevprop(const TString& prop, const TString& value); - - void Copy(const TString& path, const TOrigin& origin); - - void MkDir(const TString& path); - - void RemoveFile(const TString& path); - - void RemoveTree(const TString& path); - - void StoreFile(const TString& path, - const TString& data, - const bool execute, - const bool symlink, - const TMaybe<bool> binaryHint = Nothing(), - const TMaybe<bool> encrypted = Nothing()); - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - -} // namespace NZipatch - diff --git a/library/cpp/zipatch/ya.make b/library/cpp/zipatch/ya.make deleted file mode 100644 index f8fd6006b2..0000000000 --- a/library/cpp/zipatch/ya.make +++ /dev/null @@ -1,16 +0,0 @@ -LIBRARY() - -SRCS( - reader.cpp - writer.cpp -) - -PEERDIR( - contrib/libs/libarchive - library/cpp/json -) - -GENERATE_ENUM_SERIALIZATION(reader.h) - -END() - |