diff options
author | vvvv <vvvv@yandex-team.ru> | 2022-03-04 18:24:57 +0300 |
---|---|---|
committer | vvvv <vvvv@yandex-team.ru> | 2022-03-04 18:24:57 +0300 |
commit | 7a3824569306e0314575376cfaa33b18b2a4b5fa (patch) | |
tree | 4e95341ae02d7e929f30e7e502a1ecbdc296e531 | |
parent | 33a587f1b0e04bd1ff8ab956feb11cd79a1cd9de (diff) | |
download | ydb-7a3824569306e0314575376cfaa33b18b2a4b5fa.tar.gz |
YQL-13710 refactored out low level pickle utils
ref:dfa1f20f7a3843f882d085059bbf42629fdc4f62
5 files changed, 104 insertions, 86 deletions
diff --git a/ydb/library/yql/minikql/computation/CMakeLists.txt b/ydb/library/yql/minikql/computation/CMakeLists.txt index f53c4446c3d..554c25f1564 100644 --- a/ydb/library/yql/minikql/computation/CMakeLists.txt +++ b/ydb/library/yql/minikql/computation/CMakeLists.txt @@ -39,6 +39,7 @@ target_sources(yql-minikql-computation PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_impl.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_custom_list.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder.cpp diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp index 625a269eaf6..e00ed6a2a50 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp @@ -1,4 +1,5 @@ #include "mkql_computation_node_pack.h" +#include "mkql_computation_node_pack_impl.h" #include "mkql_computation_node_holders.h" #include "presort.h" @@ -7,7 +8,6 @@ #include <ydb/library/yql/minikql/defs.h> #include <ydb/library/yql/minikql/pack_num.h> #include <ydb/library/yql/minikql/mkql_string_util.h> -#include <library/cpp/packedtypes/zigzag.h> #include <library/cpp/resource/resource.h> #include <ydb/library/yql/utils/fp_bits.h> @@ -29,91 +29,6 @@ namespace NMiniKQL { using namespace llvm; #endif -namespace NDetails { - -void PackUInt64(ui64 val, TBuffer& buf) { - size_t off = buf.Size(); - buf.Advance(MAX_PACKED64_SIZE); - buf.EraseBack(MAX_PACKED64_SIZE - Pack64(val, buf.Data() + off)); -} - -void PackInt64(i64 val, TBuffer& buf) { - PackUInt64(ZigZagEncode(val), buf); -} - -void PackUInt32(ui32 val, TBuffer& buf) { - size_t off = buf.Size(); - buf.Advance(MAX_PACKED32_SIZE); - buf.EraseBack(MAX_PACKED32_SIZE - Pack32(val, buf.Data() + off)); -} - -void PackInt32(i32 val, TBuffer& buf) { - PackUInt32(ZigZagEncode(val), buf); -} - -void PackUInt16(ui16 val, TBuffer& buf) { - size_t off = buf.Size(); - buf.Advance(MAX_PACKED32_SIZE); - buf.EraseBack(MAX_PACKED32_SIZE - Pack32(val, buf.Data() + off)); -} - -void PackInt16(i16 val, TBuffer& buf) { - PackUInt16(ZigZagEncode(val), buf); -} - -ui64 UnpackUInt64(TStringBuf& buf) { - ui64 res = 0; - size_t read = Unpack64(buf.data(), buf.length(), res); - MKQL_ENSURE(read, "Bad ui64 packed data"); - buf.Skip(read); - return res; -} - -i64 UnpackInt64(TStringBuf& buf) { - return ZigZagDecode(UnpackUInt64(buf)); -} - -ui32 UnpackUInt32(TStringBuf& buf) { - ui32 res = 0; - size_t read = Unpack32(buf.data(), buf.length(), res); - MKQL_ENSURE(read, "Bad ui32 packed data"); - buf.Skip(read); - return res; -} - -i32 UnpackInt32(TStringBuf& buf) { - return ZigZagDecode(UnpackUInt32(buf)); -} - -ui16 UnpackUInt16(TStringBuf& buf) { - ui32 res = 0; - size_t read = Unpack32(buf.data(), buf.length(), res); - MKQL_ENSURE(read, "Bad ui32 packed data"); - buf.Skip(read); - MKQL_ENSURE(res <= Max<ui16>(), "Corrupted data"); - return res; -} - -i16 UnpackInt16(TStringBuf& buf) { - return ZigZagDecode(UnpackUInt16(buf)); -} - -template <typename T> -void PutRawData(T val, TBuffer& buf) { - buf.Append(reinterpret_cast<const char*>(&val), sizeof(T)); -} - -template <typename T> -T GetRawData(TStringBuf& buf) { - MKQL_ENSURE(sizeof(T) <= buf.size(), "Bad packed data. Buffer too small"); - T val = 0; - memcpy(&val, buf.data(), sizeof(T)); - buf.Skip(sizeof(T)); - return val; -} - -} // NDetails - namespace { #ifndef MKQL_DISABLE_CODEGEN TString MakeName(const TStringBuf& common, const TType* type) { diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.cpp new file mode 100644 index 00000000000..f8332b7d478 --- /dev/null +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.cpp @@ -0,0 +1 @@ +#include "mkql_computation_node_pack_impl.h" diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.h b/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.h new file mode 100644 index 00000000000..7cf19e9f3be --- /dev/null +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.h @@ -0,0 +1,99 @@ +#pragma once + +#include <ydb/library/yql/minikql/defs.h> +#include <ydb/library/yql/minikql/pack_num.h> +#include <util/generic/buffer.h> +#include <util/generic/strbuf.h> +#include <library/cpp/packedtypes/zigzag.h> + +namespace NKikimr { +namespace NMiniKQL { + +namespace NDetails { + +inline void PackUInt64(ui64 val, TBuffer& buf) { + size_t off = buf.Size(); + buf.Advance(MAX_PACKED64_SIZE); + buf.EraseBack(MAX_PACKED64_SIZE - Pack64(val, buf.Data() + off)); +} + +inline void PackInt64(i64 val, TBuffer& buf) { + PackUInt64(ZigZagEncode(val), buf); +} + +inline void PackUInt32(ui32 val, TBuffer& buf) { + size_t off = buf.Size(); + buf.Advance(MAX_PACKED32_SIZE); + buf.EraseBack(MAX_PACKED32_SIZE - Pack32(val, buf.Data() + off)); +} + +inline void PackInt32(i32 val, TBuffer& buf) { + PackUInt32(ZigZagEncode(val), buf); +} + +inline void PackUInt16(ui16 val, TBuffer& buf) { + size_t off = buf.Size(); + buf.Advance(MAX_PACKED32_SIZE); + buf.EraseBack(MAX_PACKED32_SIZE - Pack32(val, buf.Data() + off)); +} + +inline void PackInt16(i16 val, TBuffer& buf) { + PackUInt16(ZigZagEncode(val), buf); +} + +inline ui64 UnpackUInt64(TStringBuf& buf) { + ui64 res = 0; + size_t read = Unpack64(buf.data(), buf.length(), res); + MKQL_ENSURE(read, "Bad ui64 packed data"); + buf.Skip(read); + return res; +} + +inline i64 UnpackInt64(TStringBuf& buf) { + return ZigZagDecode(UnpackUInt64(buf)); +} + +inline ui32 UnpackUInt32(TStringBuf& buf) { + ui32 res = 0; + size_t read = Unpack32(buf.data(), buf.length(), res); + MKQL_ENSURE(read, "Bad ui32 packed data"); + buf.Skip(read); + return res; +} + +inline i32 UnpackInt32(TStringBuf& buf) { + return ZigZagDecode(UnpackUInt32(buf)); +} + +inline ui16 UnpackUInt16(TStringBuf& buf) { + ui32 res = 0; + size_t read = Unpack32(buf.data(), buf.length(), res); + MKQL_ENSURE(read, "Bad ui32 packed data"); + buf.Skip(read); + MKQL_ENSURE(res <= Max<ui16>(), "Corrupted data"); + return res; +} + +inline i16 UnpackInt16(TStringBuf& buf) { + return ZigZagDecode(UnpackUInt16(buf)); +} + +template <typename T> +void PutRawData(T val, TBuffer& buf) { + buf.Append(reinterpret_cast<const char*>(&val), sizeof(T)); +} + +template <typename T> +T GetRawData(TStringBuf& buf) { + MKQL_ENSURE(sizeof(T) <= buf.size(), "Bad packed data. Buffer too small"); + T val = 0; + memcpy(&val, buf.data(), sizeof(T)); + buf.Skip(sizeof(T)); + return val; +} + +} // NDetails + + +} +} diff --git a/ydb/library/yql/minikql/computation/ya.make b/ydb/library/yql/minikql/computation/ya.make index ae419b3b232..13f1d12e510 100644 --- a/ydb/library/yql/minikql/computation/ya.make +++ b/ydb/library/yql/minikql/computation/ya.make @@ -20,7 +20,9 @@ SRCS( mkql_computation_node_impl.cpp mkql_computation_node_list.h mkql_computation_node_pack.cpp + mkql_computation_node_pack_impl.cpp mkql_computation_node_pack.h + mkql_computation_node_pack_impl.h mkql_custom_list.cpp mkql_custom_list.h mkql_validate.cpp |