diff options
author | Devtools Arcadia <arcadia-devtools@yandex-team.ru> | 2022-02-07 18:08:42 +0300 |
---|---|---|
committer | Devtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net> | 2022-02-07 18:08:42 +0300 |
commit | 1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch) | |
tree | e26c9fed0de5d9873cce7e00bc214573dc2195b7 /library/cpp/scheme | |
download | ydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz |
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'library/cpp/scheme')
51 files changed, 7597 insertions, 0 deletions
diff --git a/library/cpp/scheme/README b/library/cpp/scheme/README new file mode 100644 index 00000000000..117c81979bd --- /dev/null +++ b/library/cpp/scheme/README @@ -0,0 +1 @@ +see http://wiki.yandex-team.ru/JandeksPoisk/KachestvoPoiska/Rearrange/RealizacijaFormata
\ No newline at end of file diff --git a/library/cpp/scheme/domscheme_traits.h b/library/cpp/scheme/domscheme_traits.h new file mode 100644 index 00000000000..a11c4dd4446 --- /dev/null +++ b/library/cpp/scheme/domscheme_traits.h @@ -0,0 +1,228 @@ +#pragma once + +#include "scheme.h" +#include <util/string/cast.h> + +struct TSchemeTraits { + using TValue = NSc::TValue; + using TValueRef = TValue*; + using TConstValueRef = const TValue*; + using TStringType = TStringBuf; + + // anyvalue defaults + template <class T> + static inline TValue Value(T&& t) { + return TValue(std::forward<T>(t)); + } + + template <class T> + static inline TValue Value(std::initializer_list<T> t) { + return TValue().SetArray().AppendAll(t); + } + + static inline TValueRef Ref(TValue& v) { + return &v; + } + + static inline TConstValueRef Ref(const TValue& v) { + return &v; + } + + // common ops + static inline bool IsNull(TConstValueRef v) { + return v->IsNull(); + } + + static inline TString ToJson(TConstValueRef v) { + return v->ToJson(); + } + + // struct ops + static inline TValueRef GetField(TValueRef v, const TStringBuf& name) { + return &(*v)[name]; + } + + static inline TConstValueRef GetField(TConstValueRef v, const TStringBuf& name) { + return &(*v)[name]; + } + + // array ops + static bool IsArray(TConstValueRef v) { + return v->IsArray(); + } + + static inline void ArrayClear(TValueRef v) { + v->SetArray(); + v->ClearArray(); + } + + using TArrayIterator = size_t; + + static inline TValueRef ArrayElement(TValueRef v, TArrayIterator n) { + return &(*v)[n]; + } + + static inline TConstValueRef ArrayElement(TConstValueRef v, TArrayIterator n) { + return &(*v)[n]; + } + + static inline size_t ArraySize(TConstValueRef v) { + return v->GetArray().size(); + } + + static inline TArrayIterator ArrayBegin(TConstValueRef) { + return 0; + } + + static inline TArrayIterator ArrayEnd(TConstValueRef v) { + return ArraySize(v); + } + + // dict ops + static bool IsDict(TConstValueRef v) { + return v->IsDict(); + } + + static inline void DictClear(TValueRef v) { + v->SetDict(); + v->ClearDict(); + } + + static inline TValueRef DictElement(TValueRef v, TStringBuf key) { + return &(*v)[key]; + } + + static inline TConstValueRef DictElement(TConstValueRef v, TStringBuf key) { + return &(*v)[key]; + } + + static inline size_t DictSize(TConstValueRef v) { + return v->GetDict().size(); + } + + using TDictIterator = NSc::TDict::const_iterator; + + static inline TDictIterator DictBegin(TConstValueRef v) { + return v->GetDict().begin(); + } + + static inline TDictIterator DictEnd(TConstValueRef v) { + return v->GetDict().end(); + } + + static inline TStringBuf DictIteratorKey(TConstValueRef /*dict*/, const TDictIterator& it) { + return it->first; + } + + static inline TConstValueRef DictIteratorValue(TConstValueRef /*dict*/, const TDictIterator& it) { + return &it->second; + } + + // boolean ops + static inline void Get(TConstValueRef v, bool def, bool& b) { + b = def == true ? !v->IsExplicitFalse() : v->IsTrue(); + } + + static inline void Get(TConstValueRef v, bool& b) { + b = v->IsTrue(); + } + + static inline void Set(TValueRef v, bool b) { + v->SetIntNumber(b ? 1 : 0); + } + + static inline bool IsValidPrimitive(const bool&, TConstValueRef v) { + return v->IsTrue() || v->IsExplicitFalse(); + } + +#define INTEGER_OPS_EX(type, min, max, isUnsigned) \ + static inline void Get(TConstValueRef v, type def, type& i) { \ + if (isUnsigned) { \ + i = v->IsNumber() && v->GetIntNumber() >= 0 ? v->GetIntNumber() : def; \ + } else { \ + i = v->IsNumber() ? v->GetIntNumber() : def; \ + } \ + } \ + static inline void Get(TConstValueRef v, type& i) { \ + if (isUnsigned) { \ + i = Max<i64>(0, v->GetIntNumber()); \ + } else { \ + i = v->GetIntNumber(); \ + } \ + } \ + static inline bool IsValidPrimitive(const type&, TConstValueRef v) { \ + return v->IsIntNumber() && \ + v->GetIntNumber() >= min && \ + v->GetIntNumber() <= max; \ + } \ + static inline void Set(TValueRef v, type i) { \ + v->SetIntNumber(i); \ + } + +#define INTEGER_OPS(type, isUnsigned) INTEGER_OPS_EX(type, Min<type>(), Max<type>(), isUnsigned) + + INTEGER_OPS(i8, false) + INTEGER_OPS(i16, false) + INTEGER_OPS(i32, false) + INTEGER_OPS(i64, false) + INTEGER_OPS(ui8, true) + INTEGER_OPS(ui16, true) + INTEGER_OPS(ui32, true) + INTEGER_OPS_EX(ui64, 0, (i64)(Max<i64>() >> 1), true) + +#undef INTEGER_OPS +#undef INTEGER_OPS_EX + + // double ops + static inline bool Get(TConstValueRef v, double def, double& d) { + if (v->IsNumber()) { + d = v->GetNumber(def); + return true; + } + d = def; + return false; + } + + static inline void Get(TConstValueRef v, double& d) { + d = v->GetNumber(); + } + + static inline void Set(TValueRef v, double d) { + v->SetNumber(d); + } + + static inline bool IsValidPrimitive(const double&, TConstValueRef v) { + return v->IsNumber(); + } + + // string ops + static inline void Get(TConstValueRef v, TStringBuf def, TStringBuf& s) { + s = v->GetString(def); + } + + static inline void Get(TConstValueRef v, TStringBuf& s) { + s = v->GetString(); + } + + static inline void Set(TValueRef v, TStringBuf s) { + v->SetString(s); + } + + static inline bool IsValidPrimitive(const TStringBuf&, TConstValueRef v) { + return v->IsString(); + } + + // validation ops + static inline TVector<TString> GetKeys(TConstValueRef v) { + TVector<TString> res; + for (const auto& key : v->DictKeys(true)) { + res.push_back(ToString(key)); + } + return res; + } + + template <typename T> + static inline bool IsValidPrimitive(const T&, TConstValueRef) { + return false; + } +}; diff --git a/library/cpp/scheme/fwd.h b/library/cpp/scheme/fwd.h new file mode 100644 index 00000000000..975a034eb23 --- /dev/null +++ b/library/cpp/scheme/fwd.h @@ -0,0 +1,7 @@ +#pragma once + +namespace NSc { + class TValue; + class TDict; + class TArray; +} diff --git a/library/cpp/scheme/scheme.cpp b/library/cpp/scheme/scheme.cpp new file mode 100644 index 00000000000..3efd116d4f1 --- /dev/null +++ b/library/cpp/scheme/scheme.cpp @@ -0,0 +1,598 @@ +#include "scheme.h" +#include "scimpl_private.h" + +#include <util/generic/algorithm.h> +#include <util/string/cast.h> + +namespace NSc { + TStringBufs& TValue::DictKeys(TStringBufs& vs, bool sorted) const { + if (!IsDict()) { + return vs; + } + + const ::NSc::TDict& dict = GetDict(); + vs.reserve(vs.size() + dict.size()); + for (const auto& it : dict) + vs.push_back(it.first); + + if (sorted) { + Sort(vs.begin(), vs.end()); + } + + return vs; + } + + TStringBufs TValue::DictKeys(bool sorted) const { + TStringBufs bufs; + DictKeys(bufs, sorted); + return bufs; + } + + TValue& TValue::MergeUpdate(const TValue& delta) { + return DoMerge(delta, false); + } + + TValue& TValue::ReverseMerge(const TValue& delta) { + return DoMerge(delta, true); + } + + TValue& TValue::MergeUpdateJson(TStringBuf data) { + return MergeUpdate(FromJson(data)); + } + + TValue& TValue::ReverseMergeJson(TStringBuf data) { + return ReverseMerge(FromJson(data)); + } + + bool TValue::MergeUpdateJson(TValue& v, TStringBuf data) { + NSc::TValue m; + if (!FromJson(m, data)) { + return false; + } + + v.MergeUpdate(m); + return true; + } + + bool TValue::ReverseMergeJson(TValue& v, TStringBuf data) { + NSc::TValue m; + if (!FromJson(m, data)) { + return false; + } + + v.ReverseMerge(m); + return true; + } + + TValue TValue::Clone() const { + return TValue().CopyFrom(*this); + } + + double TValue::ForceNumber(double deflt) const { + const TScCore& core = Core(); + if (core.IsNumber()) { + return core.GetNumber(deflt); + } + + if (TStringBuf str = core.GetString(TStringBuf())) { + { + double result = 0; + if (TryFromString<double>(str, result)) { + return result; + } + } + { + i64 result = 0; + if (TryFromString<i64>(str, result)) { + return result; + } + } + { + ui64 result = 0; + if (TryFromString<ui64>(str, result)) { + return result; + } + } + } + + return deflt; + } + + i64 TValue::ForceIntNumber(i64 deflt) const { + const TScCore& core = Core(); + if (core.IsNumber()) { + return core.GetIntNumber(deflt); + } + + if (TStringBuf str = core.GetString(TStringBuf())) { + { + i64 result = 0; + if (TryFromString<i64>(str, result)) { + return result; + } + } + { + ui64 result = 0; + if (TryFromString<ui64>(str, result)) { + return result; + } + } + { + double result = 0; + if (TryFromString<double>(str, result)) { + return result; + } + } + } + + return deflt; + } + + TString TValue::ForceString(const TString& deflt) const { + const TScCore& core = Core(); + if (core.IsString()) { + return ToString(core.GetString(TStringBuf())); + } + + if (core.IsIntNumber()) { + return ToString(core.GetIntNumber(0)); + } + + if (core.IsNumber()) { + return ToString(core.GetNumber(0)); + } + + return deflt; + } + + TValue& /*this*/ TValue::CopyFrom(const TValue& other) { + if (Same(*this, other)) { + return *this; + } + + using namespace NImpl; + return DoCopyFromImpl(other, GetTlsInstance<TSelfLoopContext>(), GetTlsInstance<TSelfOverrideContext>()); + } + + TValue& TValue::DoCopyFromImpl(const TValue& other, + NImpl::TSelfLoopContext& otherLoopCtx, + NImpl::TSelfOverrideContext& selfOverrideCtx) { + if (Same(*this, other)) { + return *this; + } + + CoreMutableForSet(); // trigger COW + + TScCore& selfCore = *TheCore; + const TScCore& otherCore = other.Core(); + + NImpl::TSelfLoopContext::TGuard loopCheck(otherLoopCtx, otherCore); + NImpl::TSelfOverrideContext::TGuard overrideGuard(selfOverrideCtx, selfCore); + + selfCore.SetNull(); + + if (!loopCheck.Ok) { + return *this; // a loop encountered (and asserted), skip the back reference + } + + switch (otherCore.ValueType) { + default: + Y_ASSERT(false); + [[fallthrough]]; + case EType::Null: + break; + case EType::Bool: + selfCore.SetBool(otherCore.IntNumber); + break; + case EType::IntNumber: + selfCore.SetIntNumber(otherCore.IntNumber); + break; + case EType::FloatNumber: + selfCore.SetNumber(otherCore.FloatNumber); + break; + case EType::String: + if (selfCore.Pool.Get() == otherCore.Pool.Get()) { + selfCore.SetOwnedString(otherCore.String); + } else { + selfCore.SetString(otherCore.String); + } + break; + case EType::Array: + selfCore.SetArray(); + for (const TValue& e : otherCore.GetArray()) { + selfCore.Push().DoCopyFromImpl(e, otherLoopCtx, selfOverrideCtx); + } + break; + case EType::Dict: { + TCorePtr tmp = NewCore(selfCore.Pool); + auto& tmpCore = *tmp; + tmpCore.SetDict(); + const TDict& d = otherCore.GetDict(); + tmpCore.Dict.reserve(d.size()); + for (const TDict::value_type& e : d) { + tmpCore.Add(e.first).DoCopyFromImpl(e.second, otherLoopCtx, selfOverrideCtx); + } + TheCore = std::move(tmp); + break; + } + } + + return *this; + } + + TValue& TValue::Swap(TValue& v) { + DoSwap(TheCore, v.TheCore); + DoSwap(CopyOnWrite, v.CopyOnWrite); + return *this; + } + + bool TValue::Same(const TValue& a, const TValue& b) { + return a.TheCore.Get() == b.TheCore.Get(); + } + + bool TValue::SamePool(const TValue& a, const TValue& b) { + return Same(a, b) || a.TheCore->Pool.Get() == b.TheCore->Pool.Get(); + } + + bool TValue::Equal(const TValue& a, const TValue& b) { + if (Same(a, b)) { + return true; + } + + const NSc::TValue::TScCore& coreA = a.Core(); + const NSc::TValue::TScCore& coreB = b.Core(); + + if (coreA.IsNumber() && coreB.IsNumber()) { + return coreA.GetIntNumber(0) == coreB.GetIntNumber(0) && coreA.GetNumber(0) == coreB.GetNumber(0); + } + + if (coreA.ValueType != coreB.ValueType) { + return false; + } + + if (coreA.IsString()) { + std::string_view strA = coreA.String; + std::string_view strB = coreB.String; + + if (strA != strB) { + return false; + } + } else if (coreA.IsArray()) { + const TArray& arrA = coreA.Array; + const TArray& arrB = coreB.Array; + + if (arrA.size() != arrB.size()) { + return false; + } + + for (size_t i = 0; i < arrA.size(); ++i) { + if (!Equal(arrA[i], arrB[i])) { + return false; + } + } + } else if (coreA.IsDict()) { + const ::NSc::TDict& dictA = coreA.Dict; + const ::NSc::TDict& dictB = coreB.Dict; + + if (dictA.size() != dictB.size()) { + return false; + } + + for (const auto& ita : dictA) { + ::NSc::TDict::const_iterator itb = dictB.find(ita.first); + + if (itb == dictB.end() || !Equal(ita.second, itb->second)) { + return false; + } + } + } + + return true; + } + + TValue& TValue::DoMerge(const TValue& delta, bool lowPriorityDelta) { + if (Same(*this, delta)) { + return *this; + } + + using namespace NImpl; + return DoMergeImpl(delta, lowPriorityDelta, GetTlsInstance<TSelfLoopContext>(), GetTlsInstance<TSelfOverrideContext>()); + } + + TValue& TValue::DoMergeImpl(const TValue& delta, bool lowPriorityDelta, + NImpl::TSelfLoopContext& otherLoopCtx, + NImpl::TSelfOverrideContext& selfOverrideGuard) { + if (Same(*this, delta)) { + return *this; + } + + if (delta.IsDict() && (!lowPriorityDelta || IsDict() || IsNull())) { + TScCore& core = CoreMutable(); + const TScCore& deltaCore = delta.Core(); + + NImpl::TSelfLoopContext::TGuard loopCheck(otherLoopCtx, deltaCore); + + if (!loopCheck.Ok) { + return *this; // a loop encountered (and asserted), skip the back reference + } + + if (!lowPriorityDelta || IsNull()) { + SetDict(); + } + + const TDict& ddelta = deltaCore.Dict; + + for (const auto& dit : ddelta) { + core.GetOrAdd(dit.first).DoMergeImpl(dit.second, lowPriorityDelta, otherLoopCtx, selfOverrideGuard); + } + } else if (!delta.IsNull() && (!lowPriorityDelta || IsNull())) { + DoCopyFromImpl(delta, otherLoopCtx, selfOverrideGuard); + } + + return *this; + } + + NJson::TJsonValue TValue::ToJsonValue() const { + using namespace NImpl; + return ToJsonValueImpl(GetTlsInstance<TSelfLoopContext>()); + } + + NJson::TJsonValue TValue::ToJsonValueImpl(NImpl::TSelfLoopContext& loopCtx) const { + const TScCore& core = Core(); + + switch (core.ValueType) { + default: + case EType::Null: + return NJson::TJsonValue(NJson::JSON_NULL); + case EType::Bool: + return NJson::TJsonValue(core.GetBool()); + case EType::IntNumber: + return NJson::TJsonValue(core.GetIntNumber()); + case EType::FloatNumber: + return NJson::TJsonValue(core.GetNumber()); + case EType::String: + return NJson::TJsonValue(core.String); + case EType::Array: { + NImpl::TSelfLoopContext::TGuard loopGuard(loopCtx, core); + + if (!loopGuard.Ok) { + return NJson::TJsonValue(NJson::JSON_NULL); + } + + NJson::TJsonValue result(NJson::JSON_ARRAY); + const TArray& arr = core.Array; + + for (const auto& item : arr) { + result.AppendValue(NJson::TJsonValue::UNDEFINED) = item.ToJsonValueImpl(loopCtx); + } + + return result; + } + case EType::Dict: { + NImpl::TSelfLoopContext::TGuard loopGuard(loopCtx, core); + + if (!loopGuard.Ok) { + return NJson::TJsonValue(NJson::JSON_NULL); + } + + NJson::TJsonValue result(NJson::JSON_MAP); + const TDict& dict = core.Dict; + + for (const auto& item : dict) { + result.InsertValue(item.first, NJson::TJsonValue::UNDEFINED) = item.second.ToJsonValueImpl(loopCtx); + } + + return result; + } + } + } + + TValue TValue::FromJsonValue(const NJson::TJsonValue& val) { + TValue result; + FromJsonValue(result, val); + return result; + } + + TValue& TValue::FromJsonValue(TValue& res, const NJson::TJsonValue& val) { + TScCore& core = res.CoreMutableForSet(); + core.SetNull(); + + switch (val.GetType()) { + default: + case NJson::JSON_UNDEFINED: + case NJson::JSON_NULL: + break; + case NJson::JSON_BOOLEAN: + core.SetBool(val.GetBoolean()); + break; + case NJson::JSON_INTEGER: + core.SetIntNumber(val.GetInteger()); + break; + case NJson::JSON_UINTEGER: + core.SetIntNumber(val.GetUInteger()); + break; + case NJson::JSON_DOUBLE: + core.SetNumber(val.GetDouble()); + break; + case NJson::JSON_STRING: + core.SetString(val.GetString()); + break; + case NJson::JSON_ARRAY: { + core.SetArray(); + for (const auto& item : val.GetArray()) { + FromJsonValue(core.Push(), item); + } + break; + } + case NJson::JSON_MAP: { + core.SetDict(); + for (const auto& item : val.GetMap()) { + FromJsonValue(core.Add(item.first), item.second); + } + break; + } + } + + return res; + } + + struct TDefaults { + TValue::TPoolPtr Pool = MakeIntrusive<NDefinitions::TPool>(); + TValue::TScCore Core{Pool}; + }; + + const TValue::TScCore& TValue::DefaultCore() { + return Default<TDefaults>().Core; + } + + const TArray& TValue::DefaultArray() { + return Default<TDefaults>().Core.Array; + } + + const TDict& TValue::DefaultDict() { + return Default<TDefaults>().Core.Dict; + } + + const TValue& TValue::DefaultValue() { + return *FastTlsSingleton<TValue>(); + } + + bool TValue::IsSameOrAncestorOf(const TValue& other) const { + using namespace NImpl; + return IsSameOrAncestorOfImpl(other.Core(), GetTlsInstance<TSelfLoopContext>()); + } + + bool TValue::IsSameOrAncestorOfImpl(const TScCore& other, NImpl::TSelfLoopContext& loopCtx) const { + const TScCore& core = Core(); + + if (&core == &other) { + return true; + } + + switch (core.ValueType) { + default: + return false; + case EType::Array: { + NImpl::TSelfLoopContext::TGuard loopGuard(loopCtx, core); + + if (!loopGuard.Ok) { + return false; + } + + for (const auto& item : core.Array) { + if (item.IsSameOrAncestorOfImpl(other, loopCtx)) { + return true; + } + } + + return false; + } + case EType::Dict: { + NImpl::TSelfLoopContext::TGuard loopGuard(loopCtx, core); + + if (!loopGuard.Ok) { + return false; + } + + for (const auto& item : core.Dict) { + if (item.second.IsSameOrAncestorOfImpl(other, loopCtx)) { + return true; + } + } + + return false; + } + } + } + + namespace NPrivate { + int CompareStr(const NSc::TValue& a, TStringBuf b) { + return a.GetString().compare(b); + } + + int CompareInt(const NSc::TValue& a, i64 r) { + i64 l = a.GetIntNumber(); + return l < r ? -1 : l > r ? 1 : 0; + } + + int CompareFloat(const NSc::TValue& a, double r) { + double l = a.GetNumber(); + return l < r ? -1 : l > r ? 1 : 0; + } + + } + + bool operator==(const NSc::TValue& a, const NSc::TValue& b) { + return NSc::TValue::Equal(a, b); + } + + bool operator!=(const NSc::TValue& a, const NSc::TValue& b) { + return !NSc::TValue::Equal(a, b); + } + +#define LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(T, Impl) \ + bool operator==(const NSc::TValue& a, T b) { \ + return NPrivate::Impl(a, b) == 0; \ + } \ + bool operator==(T b, const NSc::TValue& a) { \ + return NPrivate::Impl(a, b) == 0; \ + } \ + bool operator!=(const NSc::TValue& a, T b) { \ + return NPrivate::Impl(a, b) != 0; \ + } \ + bool operator!=(T b, const NSc::TValue& a) { \ + return NPrivate::Impl(a, b) != 0; \ + } \ + bool operator<=(const NSc::TValue& a, T b) { \ + return NPrivate::Impl(a, b) <= 0; \ + } \ + bool operator<=(T b, const NSc::TValue& a) { \ + return NPrivate::Impl(a, b) >= 0; \ + } \ + bool operator>=(const NSc::TValue& a, T b) { \ + return NPrivate::Impl(a, b) >= 0; \ + } \ + bool operator>=(T b, const NSc::TValue& a) { \ + return NPrivate::Impl(a, b) <= 0; \ + } \ + bool operator<(const NSc::TValue& a, T b) { \ + return NPrivate::Impl(a, b) < 0; \ + } \ + bool operator<(T b, const NSc::TValue& a) { \ + return NPrivate::Impl(a, b) > 0; \ + } \ + bool operator>(const NSc::TValue& a, T b) { \ + return NPrivate::Impl(a, b) > 0; \ + } \ + bool operator>(T b, const NSc::TValue& a) { \ + return NPrivate::Impl(a, b) < 0; \ + } + +#define LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(T) \ + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(signed T, CompareInt) \ + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(unsigned T, CompareInt) + + //LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(bool, CompareInt) + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(char, CompareInt) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(char) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(short) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(int) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(long) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(long long) + + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(float, CompareFloat) + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(double, CompareFloat) + + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(TStringBuf, CompareStr) + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(const TString&, CompareStr) + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(const char* const, CompareStr) + +#undef LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL +#undef LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL + +} + +template <> +void Out<NSc::TValue>(IOutputStream& o, TTypeTraits<NSc::TValue>::TFuncParam v) { + o.Write(v.ToJson(true)); +} diff --git a/library/cpp/scheme/scheme.h b/library/cpp/scheme/scheme.h new file mode 100644 index 00000000000..3d7c59f3c97 --- /dev/null +++ b/library/cpp/scheme/scheme.h @@ -0,0 +1,538 @@ +#pragma once + +#include "scimpl_defs.h" + +#include "fwd.h" + +#include <iterator> +#include <utility> + +namespace NSc { +#ifdef _MSC_VER +#pragma warning(disable : 4521 4522) +#endif + + // todo: try to remove some rarely used methods + class TValue { + public: + enum class EType { + Null = 0 /* "Null" */, + Bool /* "Bool" */, + IntNumber /* "Int" */, + FloatNumber /* "Float" */, + String /* "String" */, + Array /* "Array" */, + Dict /* "Dict" */ + }; + + struct TScCore; + using TCorePtr = TIntrusivePtr<TScCore>; + using TPoolPtr = TIntrusivePtr<NDefinitions::TPool>; + + using TArray = ::NSc::TArray; + using TDict = ::NSc::TDict; + + private: // A TValue instance has only these 3 fields + mutable TCorePtr TheCore; // a pointer to a refcounted (kind of) variant + bool CopyOnWrite = false; // a flag that thevalue is a COW shallow copy and should produce a deep copy once modified + + // Thus all copies of a TValue are by default shallow. Use TValue::Clone to force a deep copy. + // A COW copy will see changes in its parent, but no change in the COW copy will propagate to its parent. + + public: + inline TValue(); + inline TValue(TValue& v); + inline TValue(const TValue& v); + inline TValue(TValue&& v) noexcept; + + public: // Operators + inline TValue(double t); + inline TValue(unsigned long long t); + inline TValue(unsigned long t); + inline TValue(unsigned t); + inline TValue(long long t); + inline TValue(long t); + inline TValue(int t); + // inline TValue(bool b); + + inline TValue(TStringBuf t); + inline TValue(const char*); + + inline operator double() const; + inline operator float() const; + inline operator long long() const; + inline operator long() const; + inline operator int() const; + inline operator short() const; + inline operator char() const; + inline operator unsigned long long() const; + inline operator unsigned long() const; + inline operator unsigned() const; + inline operator unsigned short() const; + inline operator unsigned char() const; + inline operator signed char() const; + + inline operator TStringBuf() const; + + inline operator const ::NSc::TArray&() const; + inline operator const ::NSc::TDict&() const; + + inline TValue& operator=(double t); + + inline TValue& operator=(unsigned long long t); + inline TValue& operator=(unsigned long t); + inline TValue& operator=(unsigned t); + + inline TValue& operator=(long long t); + inline TValue& operator=(long t); + inline TValue& operator=(int t); + // inline TValue& operator=(bool t); + + inline TValue& operator=(TStringBuf t); + inline TValue& operator=(const char* t); + + inline TValue& operator=(TValue& v) &; + inline TValue& operator=(const TValue& v) &; + inline TValue& operator=(TValue&& v) & noexcept; + + inline TValue& operator=(TValue& v) && = delete; + inline TValue& operator=(const TValue& v) && = delete; + inline TValue& operator=(TValue&& v) && = delete; + + public: + template <class T> // ui16 or TStringBuf + inline TValue& operator[](const T& idx) { + return GetOrAdd(idx); + } + + template <class T> // ui16 or TStringBuf + inline const TValue& operator[](const T& idx) const { + return Get(idx); + } + + public: // Data methods /////////////////////////////////////////////////////////// + inline EType GetType() const; + + inline bool IsNull() const; + + inline TValue& SetNull(); // returns self, will set type to Null + + TValue& Clear() { + return ClearArray().ClearDict().SetNull(); + } + + public: // Number methods ///////////////////////////////////////////////////////// + // Bool, IntNumber and FloatNumber are all compatible. + // If a TValue node has one of the types it may as well be used as another. + // FloatNumber methods. Forces FloatNumber representation. Compatible with IntNumber and Bool + + inline bool IsNumber() const; // true if any of FloatNumber, IntNumber, Bool + + inline double GetNumber(double defaultval = 0) const; // Compatible with Bool, IntNumber and FloatNumber types + + inline double& GetNumberMutable(double defaultval = 0); // Will switch the type to FloatNumber + + inline TValue& SetNumber(double val = 0); // returns self, will switch the type to FloatNumber + + double ForceNumber(double deflt = 0) const; // Best-effort cast to double (will do TryFromString if applicable) + + // IntNumber methods. Forces integer representation. Compatible with FloatNumber and Bool types. + // Note: if you don't care about distinguishing bools, ints and doubles, use *Number methods above + + inline bool IsIntNumber() const; // true only if IntNumber or Bool + + inline i64 GetIntNumber(i64 defaultval = 0) const; // Compatible with Bool, IntNumber and FloatNumber types + + inline i64& GetIntNumberMutable(i64 defaultval = 0); // Will switch the type to IntNumber + + inline TValue& SetIntNumber(i64 val = 0); // returns self, will switch the type to IntNumber + + i64 ForceIntNumber(i64 deflt = 0) const; // Best-effort cast to i64 (will do TryFromString for String) + + // Bool methods. Forces bool representation. Compatible with Float Number and Int Number methods above. + // Note: if you don't care about distinguishing Bool, IntNumber and FloatNumber, use *Number methods above + + inline bool IsBool() const; // true only if Bool + + inline bool GetBool(bool defaultval = false) const; // Compatible with Bool, IntNumber and FloatNumber types + + inline TValue& SetBool(bool val = false); // returns self, will switch the type to Bool + + public: // Arcadia-specific boolean representation support + // Tests for explicit True, also checks for arcadia-specific boolean representation + bool IsTrue() const { + return IsNumber() ? GetNumber() : ::IsTrue(GetString()); + } + + // Tests for explicit False, also checks for arcadia-specific boolean representation + bool IsExplicitFalse() const { + return IsNumber() ? !GetNumber() : IsFalse(GetString()); + } + + public: // String methods ///////////////////////////////////////////////////////// + inline bool IsString() const; + + inline TStringBuf GetString(TStringBuf defaultval = TStringBuf()) const; + + inline TValue& SetString(TStringBuf val = TStringBuf()); // returns self + + TString ForceString(const TString& deflt = TString()) const; // Best-effort cast to TString (will do ToString for numeric types) + + // todo: remove + inline bool StringEmpty() const; + inline size_t StringSize() const; + + public: // Array methods ////////////////////////////////////////////////////////// + inline bool IsArray() const; + + inline const TArray& GetArray() const; + inline TArray& GetArrayMutable(); + inline TValue& SetArray(); // turns into array if needed, returns self + inline TValue& ClearArray(); + + inline bool Has(size_t idx) const; + + inline const TValue& Get(size_t idx) const; // returns child or default + inline TValue* GetNoAdd(size_t idx); // returns link to existing child or nullptr + + inline TValue& Push(); // returns new child + + template <class T> + TValue& Push(T&& t) { + return Push() = std::forward<T>(t); + } // returns new child + + TValue& Insert(ui16 idx) { + return InsertUnsafe(idx); + } // creates missing values, returns new child + + template <class T> + TValue& Insert(ui16 idx, T&& v) { + return InsertUnsafe(idx, std::forward<T>(v)); + } // creates missing values, returns new child + + template <class TIt> + inline TValue& AppendAll(TIt begin, TIt end); // Append(vec.begin(), vec.end()) + + template <class TColl> + inline TValue& AppendAll(TColl&& coll); // Append(vec) + + inline TValue& AppendAll(std::initializer_list<TValue> coll); + + TValue& GetOrAdd(ui16 idx) { + return GetOrAddUnsafe(idx); + } // creates missing values, returns new child + + inline TValue& InsertUnsafe(size_t idx); // creates missing values, returns new child + + template <class T> + TValue& InsertUnsafe(size_t idx, T&& t) { + return InsertUnsafe(idx) = std::forward<T>(t); + } // creates missing values, returns new child + + inline TValue& GetOrAddUnsafe(size_t idx); // creates missing values, returns new child + + inline TValue Pop(); // returns popped value + inline TValue Delete(size_t idx); // returns deleted value if it existed, NSc::Null() otherwise + + inline TValue& Front() { + return GetOrAdd(0); + } // creates missing value, returns child + inline const TValue& Front() const { + return Get(0); + } // returns child or default + + inline TValue& Back(); // creates missing value, returns child + inline const TValue& Back() const; // returns child or default + + // todo: remove + inline bool ArrayEmpty() const; + inline size_t ArraySize() const; + + public: // Dict methods + inline bool IsDict() const; + + inline const TDict& GetDict() const; + inline TDict& GetDictMutable(); + inline TValue& SetDict(); // turns into dict if not one, returns self + inline TValue& ClearDict(); + + inline bool Has(TStringBuf idx) const; + + inline const TValue& Get(TStringBuf idx) const; + inline TValue* GetNoAdd(TStringBuf idx); // returns link to existing child or nullptr + + TValue& Add(TStringBuf idx) { + return GetOrAdd(idx); + } + + template <class T> + TValue& Add(TStringBuf idx, T&& t) { + return Add(idx) = std::forward<T>(t); + } + + inline TValue& GetOrAdd(TStringBuf idx); // creates missing value, returns child + + inline TValue Delete(TStringBuf idx); // returns deleted value + + inline TValue& AddAll(std::initializer_list<std::pair<TStringBuf, TValue>> t); + + TStringBufs DictKeys(bool sorted = true) const; + TStringBufs& DictKeys(TStringBufs&, bool sorted = true) const; + + // todo: remove + inline bool DictEmpty() const; + inline size_t DictSize() const; + + public: // Json methods //////////////////////////////////////////////// + using TJsonOpts = NSc::TJsonOpts; + using EJsonOpts = TJsonOpts::EJsonOpts; + static const EJsonOpts JO_DEFAULT = TJsonOpts::JO_DEFAULT; + static const EJsonOpts JO_SORT_KEYS = TJsonOpts::JO_SORT_KEYS; + static const EJsonOpts JO_SKIP_UNSAFE = TJsonOpts::JO_SKIP_UNSAFE; // skip non-utf8 strings + static const EJsonOpts JO_PRETTY = TJsonOpts::JO_PRETTY; + static const EJsonOpts JO_SAFE = TJsonOpts::JO_SAFE; // JO_SORT_KEYS | JO_SKIP_UNSAFE + static const EJsonOpts JO_PARSER_STRICT_WITH_COMMENTS = TJsonOpts::JO_PARSER_STRICT_WITH_COMMENTS; // strict json + strict utf8 + static const EJsonOpts JO_PARSER_STRICT = TJsonOpts::JO_PARSER_STRICT; // strict json + strict utf8 + comments are disallowed + static const EJsonOpts JO_PARSER_DISALLOW_DUPLICATE_KEYS = TJsonOpts::JO_PARSER_DISALLOW_DUPLICATE_KEYS; + + static TValue FromJson(TStringBuf, const TJsonOpts& = TJsonOpts()); + static TValue FromJsonThrow(TStringBuf, const TJsonOpts& = TJsonOpts()); + static bool FromJson(TValue&, TStringBuf, const TJsonOpts& = TJsonOpts()); + + // TODO: Переименовать ToJson в ToJsonUnsafe, а ToJsonSafe в ToJson + TString ToJson(const TJsonOpts& = TJsonOpts()) const; + const TValue& ToJson(IOutputStream&, const TJsonOpts& = TJsonOpts()) const; // returns self + + // ToJson(JO_SORT_KEYS | JO_SKIP_UNSAFE) + TString ToJsonSafe(const TJsonOpts& = TJsonOpts()) const; + const TValue& ToJsonSafe(IOutputStream&, const TJsonOpts& = TJsonOpts()) const; + + // ToJson(JO_SORT_KEYS | JO_PRETTY | JO_SKIP_UNSAFE) + TString ToJsonPretty(const TJsonOpts& = TJsonOpts()) const; + const TValue& ToJsonPretty(IOutputStream&, const TJsonOpts& = TJsonOpts()) const; + + NJson::TJsonValue ToJsonValue() const; + + static TValue FromJsonValue(const NJson::TJsonValue&); + static TValue& FromJsonValue(TValue&, const NJson::TJsonValue&); // returns self + + static TJsonOpts MakeOptsSafeForSerializer(TJsonOpts = TJsonOpts()); + static TJsonOpts MakeOptsPrettyForSerializer(TJsonOpts = TJsonOpts()); + + public: // Merge methods //////////////////////////////////////////////// + /* + * LHS.MergeUpdate(RHS): + * 1. Dict <- Dict: + * - Copy all nonconflicting key-value pairs from RHS to LHS. + * - For every pair of conflicting values apply LHS[key].MergeUpdate(RHS[key]). + * 2. Anything <- Null: + * - Do nothing. + * 3. Other conflicts: + * - Copy RHS over LHS. + * + * LHS.ReverseMerge(RHS): + * 1. Dict <- Dict: + * - Copy all nonconflicting key-value pairs from RHS to LHS. + * - For every pair of conflicting values apply LHS[key].ReverseMerge(RHS[key]). + * 2. Null <- Anything: + * - Copy RHS over LHS. + * 3. Other conflicts: + * - Do nothing. + */ + + TValue& MergeUpdateJson(TStringBuf json); // returns self + TValue& ReverseMergeJson(TStringBuf json); // returns self + + static bool MergeUpdateJson(TValue&, TStringBuf json); // returns true unless failed to parse the json + static bool ReverseMergeJson(TValue&, TStringBuf json); // returns true unless failed to parse the json + + TValue& MergeUpdate(const TValue& delta); // return self + TValue& ReverseMerge(const TValue& delta); // return self + + public: // Path methods ///////////////////////////////////////////////////////// + // TODO: add throwing variants + // make sure to properly escape the tokens + + static TString EscapeForPath(TStringBuf rawKey); // converts a raw dict key into a valid token for a selector path + + static bool PathValid(TStringBuf path); // returns true if the path is syntactically valid + + bool PathExists(TStringBuf path) const; // returns true if the path is syntactically valid and the target value exists + + const TValue& TrySelect(TStringBuf path) const; // returns the target value + // if the path is syntactically valid and the target value exists + // otherwise returns NSc::Null() + + TValue* TrySelectOrAdd(TStringBuf path); // returns the target value if it exists or creates if not + // if the path is syntactically valid + // otherwise returns NSc::Null() + + TValue TrySelectAndDelete(TStringBuf path); // deletes and returns the target value + // if the path is syntactically valid and the target value existed + // otherwise returns NSc::Null() + + public: // Copy methods ///////////////////////////////////////////////////////// + TValue Clone() const; // returns deep copy of self (on the separate pool) + TValue& CopyFrom(const TValue& other); // deep copy other value into self, returns self + + TValue& Swap(TValue& v); + + static bool Same(const TValue&, const TValue&); // point to the same core + static bool Equal(const TValue&, const TValue&); // recursively equal + static bool SamePool(const TValue&, const TValue&); // share arena + + public: + // very specific methods useful in very specific corner cases + + static TValue From(const ::google::protobuf::Message&, bool mapAsDict = false); + + void To(::google::protobuf::Message&, const TProtoOpts& opts = {}) const; + + public: + inline explicit TValue(TPoolPtr&); + + static const TScCore& DefaultCore(); + static const TArray& DefaultArray(); + static const TDict& DefaultDict(); + static const TValue& DefaultValue(); + static const TValue& Null() { + return DefaultValue(); + } + + void DoWriteJsonImpl(IOutputStream&, const TJsonOpts&, NImpl::TKeySortContext&, NImpl::TSelfLoopContext&) const; + + bool IsSameOrAncestorOf(const TValue& other) const; + + private: + TValue& DoMerge(const TValue& delta, bool olddelta); + TValue& DoMergeImpl(const TValue& delta, bool olddelta, NImpl::TSelfLoopContext&, NImpl::TSelfOverrideContext&); + TValue& DoCopyFromImpl(const TValue& other, NImpl::TSelfLoopContext&, NImpl::TSelfOverrideContext&); + NJson::TJsonValue ToJsonValueImpl(NImpl::TSelfLoopContext&) const; + + bool IsSameOrAncestorOfImpl(const TScCore& other, NImpl::TSelfLoopContext& loopCtx) const; + + inline TScCore& CoreMutable(); + inline TScCore& CoreMutableForSet(); + inline const TScCore& Core() const; + + static inline TScCore* NewCore(TPoolPtr&); + + static TValue FromField(const ::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*); + static TValue FromRepeatedField(const ::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*, int index); + + void ValueToField(const TValue& value, ::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*, const TProtoOpts& opts) const; + void ToField(::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*, const TProtoOpts& opts) const; + void ToEnumField(::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*, const TProtoOpts& opts) const; + void ToRepeatedField(::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*, const TProtoOpts& opts) const; + void ToMapField(::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*, const TProtoOpts& opts) const; + }; + + + inline const TValue& Null() { + return TValue::DefaultValue(); + } + + + class TArray: public TDeque<TValue, TPoolAllocator>, TNonCopyable { + using TParent = TDeque<TValue, TPoolAllocator>; + + public: + TArray(TMemoryPool* p) + : TParent(p) + { + } + + template <class TIt> + void AppendAll(TIt begin, TIt end) { + TParent::insert(TParent::end(), begin, end); + } + + template <class TColl> + void AppendAll(TColl&& coll) { + AppendAll(std::begin(coll), std::end(coll)); + } + + void AppendAll(std::initializer_list<TValue> coll) { + AppendAll(coll.begin(), coll.end()); + } + + const TValue& operator[](size_t i) const { + return EnsureIndex(i); + } + + TValue& operator[](size_t i) { + return EnsureIndex(i); + } + + const TValue& front() const { + return EnsureIndex(0); + } + + TValue& front() { + return EnsureIndex(0); + } + + const TValue& back() const { + return EnsureIndex(LastIndex()); + } + + TValue& back() { + return EnsureIndex(LastIndex()); + } + + void pop_back() { + if (empty()) + return; + TParent::pop_back(); + } + + void pop_front() { + if (empty()) + return; + TParent::pop_front(); + } + + private: + size_t LastIndex() const { + return ::Max<size_t>(size(), 1) - 1; + } + + TValue& EnsureIndex(size_t i) { + if (i >= size()) + resize(::Min<size_t>(i + 1, ::Max<ui16>()), TValue::DefaultValue()); + return TParent::operator[](i); + } + + const TValue& EnsureIndex(size_t i) const { + return i < size() ? TParent::operator[](i) : TValue::DefaultValue(); + } + }; + + + // todo: densehashtable + // todo: allow insertions + // todo: make TDict methods safe + class TDict: public THashMap<TStringBuf, TValue, THash<TStringBuf>, TEqualTo<TStringBuf>, TPoolAllocator>, TNonCopyable { + using TParent = THashMap<TStringBuf, TValue, THash<TStringBuf>, TEqualTo<TStringBuf>, TPoolAllocator>; + + public: + TDict(TMemoryPool* p) + : TParent(p) + { + } + + template <class TStr> + const TValue& Get(const TStr& key) const { + const_iterator it = find(key); + return it != end() ? it->second : TValue::DefaultValue(); + } + }; +} + +#include "scimpl.h" +#include "scheme_cast.h" + +#ifdef _MSC_VER +#pragma warning(default : 4521 4522) +#endif diff --git a/library/cpp/scheme/scheme_cast.h b/library/cpp/scheme/scheme_cast.h new file mode 100644 index 00000000000..00839e8017c --- /dev/null +++ b/library/cpp/scheme/scheme_cast.h @@ -0,0 +1,321 @@ +#pragma once + +#include <util/generic/set.h> +#include <util/generic/vector.h> +#include <util/generic/map.h> +#include <util/generic/hash.h> +#include <util/generic/hash_set.h> +#include <util/string/cast.h> +#include <util/generic/yexception.h> + +#include "scheme.h" + +namespace NJsonConverters { + class IJsonSerializable { + public: + virtual NSc::TValue ToTValue() const = 0; + virtual void FromTValue(const NSc::TValue&, const bool validate) = 0; + + const TString ToJson(const bool sort = false) const { + return ToTValue().ToJson(sort); + }; + + void FromJson(const TStringBuf& json, const bool validate = false) { + NSc::TValue v = NSc::TValue::FromJson(json); + FromTValue(v, validate); + } + + virtual ~IJsonSerializable(){}; + }; + ////////////////////////////////////////////////////////////////////// + // fwd declarations + ////////////////////////////////////////////////////////////////////// + + //TVector + template <typename T, typename A> + NSc::TValue ToTValue(const TVector<T, A>& x); + template <typename T, typename A> + void FromTValue(const NSc::TValue& x, TVector<T, A>& out, const bool validate); + + //THashMap + template <class Key, class T, class HashFcn, class EqualKey, class Alloc> + NSc::TValue ToTValue(const THashMap<Key, T, HashFcn, EqualKey, Alloc>& x); + template <class Key, class T, class HashFcn, class EqualKey, class Alloc> + void FromTValue(const NSc::TValue& x, THashMap<Key, T, HashFcn, EqualKey, Alloc>& out, const bool validate); + + //TMap + template <class K, class V, class Less, class A> + NSc::TValue ToTValue(const TMap<K, V, Less, A>& x); + template <class K, class V, class Less, class A> + void FromTValue(const NSc::TValue& x, TMap<K, V, Less, A>& out, const bool validate); + + //THashSet + template <class V, class H, class E, class A> + NSc::TValue ToTValue(const THashSet<V, H, E, A>& x); + template <class V, class H, class E, class A> + void FromTValue(const NSc::TValue& x, THashSet<V, H, E, A>& out, const bool validate); + + //TSet + template <class K, class L, class A> + NSc::TValue ToTValue(const TSet<K, L, A>& x); + template <class K, class L, class A> + void FromTValue(const NSc::TValue& x, TSet<K, L, A>& out, const bool validate); + + //std::pair + template <class T1, class T2> + NSc::TValue ToTValue(const std::pair<T1, T2>& x); + template <class T1, class T2> + void FromTValue(const NSc::TValue& x, std::pair<T1, T2>& out, const bool validate); + + ////////////////////////////////////////////////////////////////////// + // simple From, To helpers + ////////////////////////////////////////////////////////////////////// + template <typename T, bool HasSerializer> + struct TValueAndStrokaConv {}; + + template <typename T> + struct TValueAndStrokaConv<T, 0> { + static NSc::TValue ToTValue(const T& x) { + return NSc::TValue(x); + } + + static void FromTValue(const NSc::TValue& x, T& out, const bool) { + out = x; + } + + static TString ToString(const T& x) { + return ::ToString(x); + } + + static void FromString(const TStringBuf& str, T& res, const bool) { + res = ::FromString<T>(str); + } + }; + + template <typename T> + struct TValueAndStrokaConv<T, 1> { + static NSc::TValue ToTValue(const T& x) { + return x.ToTValue(); + } + + static void FromTValue(const NSc::TValue& x, T& out, const bool validate) { + out.FromTValue(x, validate); + } + + static TString ToString(const T& x) { + return x.ToJson(); + } + + static void FromString(const TStringBuf& str, T& res, const bool validate) { + res.FromJson(str, validate); + } + }; + + template <typename T> + NSc::TValue ToTValue(const T& x) { + return TValueAndStrokaConv<T, std::is_base_of<IJsonSerializable, T>::value>::ToTValue(x); + } + + template <typename T> + void FromTValue(const NSc::TValue& x, T& out, const bool validate) { + return TValueAndStrokaConv<T, std::is_base_of<IJsonSerializable, T>::value>::FromTValue(x, out, validate); + } + + template <typename T> + T FromTValue(const NSc::TValue& x, const bool validate) { + T ret; + FromTValue(x, ret, validate); + return ret; + } + + template <typename T> + TString ToString(const T& x) { + return TValueAndStrokaConv<T, std::is_base_of<IJsonSerializable, T>::value>::ToString(x); + } + + template <typename T> + void FromString(const TStringBuf& str, T& res, const bool validate) { + return TValueAndStrokaConv<T, std::is_base_of<IJsonSerializable, T>::value>::FromString(str, res, validate); + } + + template <typename T> + T FromString(const TStringBuf& str, bool validate) { + T ret; + FromString(str, ret, validate); + return ret; + } + + namespace NPrivate { + template <typename T> + NSc::TValue ToTValueDict(const T& dict) { + NSc::TValue out; + out.SetDict(); + for (typename T::const_iterator it = dict.begin(); it != dict.end(); ++it) { + out[ToString(it->first)] = NJsonConverters::ToTValue(it->second); + } + return out; + } + + template <typename T> + void FromTValueDict(const NSc::TValue& x, T& out, bool validate) { + typedef typename T::key_type TKey; + typedef typename T::mapped_type TMapped; + if (validate) + Y_ENSURE(x.IsDict() || x.IsNull(), "not valid input scheme"); + out.clear(); + if (x.IsDict()) { + const NSc::TDict& dict = x.GetDict(); + for (const auto& it : dict) { + TKey key = NJsonConverters::FromString<TKey>(it.first, validate); + TMapped val = NJsonConverters::FromTValue<TMapped>(it.second, validate); + out.insert(std::pair<TKey, TMapped>(key, val)); + } + } + } + + template <typename T> + NSc::TValue ToTValueSet(const T& set) { + NSc::TValue out; + out.SetDict(); + for (typename T::const_iterator it = set.begin(); it != set.end(); ++it) { + out[ToString(*it)] = NSc::Null(); + } + return out; + } + + template <typename T> + void FromTValueSet(const NSc::TValue& x, T& out, const bool validate) { + typedef typename T::key_type TKey; + if (validate) + Y_ENSURE(x.IsDict() || x.IsNull(), "not valid input scheme"); + out.clear(); + if (x.IsDict()) { + const NSc::TDict& dict = x.GetDict(); + for (const auto& it : dict) { + TKey key; + NJsonConverters::FromString<TKey>(it.first, key, validate); + out.insert(key); + } + } + } + } + + ////////////////////////////////////////////////////////////////////// + // TVector + ////////////////////////////////////////////////////////////////////// + template <typename T, typename A> + NSc::TValue ToTValue(const TVector<T, A>& x) { + NSc::TValue out; + out.SetArray(); + for (typename TVector<T, A>::const_iterator it = x.begin(); it != x.end(); ++it) + out.Push(NJsonConverters::ToTValue(*it)); + return out; + } + + template <typename T, typename A> + void FromTValue(const NSc::TValue& x, TVector<T, A>& out, const bool validate) { + if (validate) + Y_ENSURE(x.IsArray() || x.IsNull(), "not valid input scheme"); + out.clear(); + if (x.IsArray()) { + const NSc::TArray& arr = x.GetArray(); + out.reserve(arr.size()); + for (const auto& it : arr) { + T val; + NJsonConverters::FromTValue(it, val, validate); + out.push_back(val); + } + } + } + + ////////////////////////////////////////////////////////////////////// + // THashMap & TMap + ////////////////////////////////////////////////////////////////////// + template <class Key, class T, class HashFcn, class EqualKey, class Alloc> + NSc::TValue ToTValue(const THashMap<Key, T, HashFcn, EqualKey, Alloc>& x) { + return NPrivate::ToTValueDict(x); + } + + template <class Key, class T, class HashFcn, class EqualKey, class Alloc> + void FromTValue(const NSc::TValue& x, THashMap<Key, T, HashFcn, EqualKey, Alloc>& out, const bool validate) { + NPrivate::FromTValueDict(x, out, validate); + } + + template <class K, class V, class Less, class A> + NSc::TValue ToTValue(const TMap<K, V, Less, A>& x) { + return NPrivate::ToTValueDict(x); + } + + template <class K, class V, class Less, class A> + void FromTValue(const NSc::TValue& x, TMap<K, V, Less, A>& out, const bool validate) { + NPrivate::FromTValueDict(x, out, validate); + } + + ////////////////////////////////////////////////////////////////////// + // THashSet & TSet + ////////////////////////////////////////////////////////////////////// + template <class V, class H, class E, class A> + NSc::TValue ToTValue(const THashSet<V, H, E, A>& x) { + return NPrivate::ToTValueSet(x); + } + + template <class V, class H, class E, class A> + void FromTValue(const NSc::TValue& x, THashSet<V, H, E, A>& out, const bool validate) { + NPrivate::FromTValueSet(x, out, validate); + } + + template <class K, class L, class A> + NSc::TValue ToTValue(const TSet<K, L, A>& x) { + return NPrivate::ToTValueSet(x); + } + + template <class K, class L, class A> + void FromTValue(const NSc::TValue& x, TSet<K, L, A>& out, const bool validate) { + NPrivate::FromTValueSet(x, out, validate); + } + + ////////////////////////////////////////////////////////////////////// + // std::pair + ////////////////////////////////////////////////////////////////////// + template <class T1, class T2> + NSc::TValue ToTValue(const std::pair<T1, T2>& x) { + NSc::TValue out; + out.SetArray(); + out.Push(NJsonConverters::ToTValue(x.first)); + out.Push(NJsonConverters::ToTValue(x.second)); + return out; + } + + template <class T1, class T2> + void FromTValue(const NSc::TValue& x, std::pair<T1, T2>& out, const bool validate) { + if (validate) + Y_ENSURE(x.IsArray() || x.IsNull(), "not valid input scheme"); + if (x.IsArray()) { + const NSc::TArray& arr = x.GetArray(); + if (arr.size() == 2) { + T1 val0; + T2 val1; + NJsonConverters::FromTValue(arr[0], val0, validate); + NJsonConverters::FromTValue(arr[1], val1, validate); + out.first = val0; + out.second = val1; + } + } + } + + ////////////////////////////////////////////////////////////////////// + // global user functions + ////////////////////////////////////////////////////////////////////// + template <typename T> + TString ToJson(const T& val, const bool sort = false) { + return NJsonConverters::ToTValue(val).ToJson(sort); + } + + template <typename T> + T FromJson(const TStringBuf& json, bool validate = false) { + NSc::TValue v = NSc::TValue::FromJson(json); + T ret; + NJsonConverters::FromTValue(v, ret, validate); + return ret; + } +} diff --git a/library/cpp/scheme/scimpl.h b/library/cpp/scheme/scimpl.h new file mode 100644 index 00000000000..4f68f16290f --- /dev/null +++ b/library/cpp/scheme/scimpl.h @@ -0,0 +1,858 @@ +#pragma once + +#include "scheme.h" + +#include <util/stream/output.h> + +namespace NSc { + struct TValue::TScCore : TAtomicRefCount<TScCore, TDestructor>, TNonCopyable { + TPoolPtr Pool; + double FloatNumber = 0; + i64 IntNumber = 0; + TStringBuf String; + TDict Dict; + TArray Array; + TValue::EType ValueType = TValue::EType::Null; + + TScCore(TPoolPtr& p) + : Pool(p) + , Dict(Pool->Get()) + , Array(Pool->Get()) + { + } + + bool IsNull() const { + return TValue::EType::Null == ValueType; + } + + bool IsBool() const { + return TValue::EType::Bool == ValueType; + } + + bool IsIntNumber() const { + return TValue::EType::IntNumber == ValueType || IsBool(); + } + + bool IsNumber() const { + return TValue::EType::FloatNumber == ValueType || IsIntNumber(); + } + + bool IsString() const { + return TValue::EType::String == ValueType; + } + + bool IsArray() const { + return TValue::EType::Array == ValueType; + } + + bool IsDict() const { + return TValue::EType::Dict == ValueType; + } + + bool HasChildren() const { + return GetDict().size() + GetArray().size(); + } + + void SetNull() { + ValueType = TValue::EType::Null; + } + + void SetArray() { + if (Y_LIKELY(IsArray())) { + return; + } + + ValueType = TValue::EType::Array; + Array.clear(); + } + + void SetDict() { + if (Y_LIKELY(IsDict())) { + return; + } + + ValueType = TValue::EType::Dict; + Dict.clear(); + } + + void SetNumber(double n) { + ValueType = TValue::EType::FloatNumber; + FloatNumber = n; + } + + void SetIntNumber(i64 n) { + ValueType = TValue::EType::IntNumber; + IntNumber = n; + } + + void SetBool(bool b) { + ValueType = TValue::EType::Bool; + IntNumber = b; + } + + void SetString(TStringBuf s) { + SetOwnedString(Pool->AppendBuf(s)); + } + + void SetOwnedString(TStringBuf s) { + ValueType = TValue::EType::String; + String = s; + } + + double& GetNumberMutable(double defaultnum) { + switch (ValueType) { + case TValue::EType::Bool: + SetNumber(bool(IntNumber)); + break; + case TValue::EType::IntNumber: + SetNumber(IntNumber); + break; + case TValue::EType::FloatNumber: + break; + default: + SetNumber(defaultnum); + break; + } + + return FloatNumber; + } + + i64& GetIntNumberMutable(i64 defaultnum) { + switch (ValueType) { + case TValue::EType::Bool: + SetIntNumber(bool(IntNumber)); + break; + case TValue::EType::IntNumber: + break; + case TValue::EType::FloatNumber: + SetIntNumber(FloatNumber); + break; + default: + SetIntNumber(defaultnum); + break; + } + + return IntNumber; + } + + void ClearArray() { + if (!IsArray()) { + return; + } + + Array.clear(); + } + + void ClearDict() { + if (!IsDict()) { + return; + } + + Dict.clear(); + } + + double GetNumber(double d = 0) const { + switch (ValueType) { + case TValue::EType::Bool: + return (bool)IntNumber; + case TValue::EType::IntNumber: + return IntNumber; + case TValue::EType::FloatNumber: + return FloatNumber; + default: + return d; + } + } + + i64 GetIntNumber(i64 n = 0) const { + switch (ValueType) { + case TValue::EType::Bool: + return (bool)IntNumber; + case TValue::EType::IntNumber: + return IntNumber; + case TValue::EType::FloatNumber: + return FloatNumber; + default: + return n; + } + } + + bool GetBool(bool b = false) const { + return GetIntNumber(b); + } + + TStringBuf GetString(TStringBuf s = TStringBuf()) const { + return IsString() ? String : s; + } + + const TArray& GetArray() const { + return IsArray() ? Array : TValue::DefaultArray(); + } + + TArray& GetArrayMutable() { + SetArray(); + return Array; + } + + TDict& GetDictMutable() { + SetDict(); + return Dict; + } + + const TDict& GetDict() const { + return IsDict() ? Dict : TValue::DefaultDict(); + } + + static void DoPush(TPoolPtr& p, TArray& a) { + a.push_back(TValue(p)); + a.back().CopyOnWrite = false; + } + + TValue& Push() { + SetArray(); + DoPush(Pool, Array); + return Array.back(); + } + + TValue Pop() { + if (!IsArray() || Array.empty()) { + return TValue::DefaultValue(); + } + + TValue v = Array.back(); + Array.pop_back(); + return v; + } + + const TValue& Get(size_t key) const { + return IsArray() && Array.size() > key ? Array[key] : TValue::DefaultValue(); + } + + TValue* GetNoAdd(size_t key) { + return IsArray() && Array.size() > key ? &Array[key] : nullptr; + } + + TValue& GetOrAdd(size_t key) { + SetArray(); + for (size_t i = Array.size(); i <= key; ++i) { + DoPush(Pool, Array); + } + + return Array[key]; + } + + TValue& Back() { + SetArray(); + + if (Array.empty()) { + DoPush(Pool, Array); + } + + return Array.back(); + } + + TValue& Insert(size_t key) { + SetArray(); + + if (Array.size() <= key) { + return GetOrAdd(key); + } else { + Array.insert(Array.begin() + key, TValue(Pool)); + Array[key].CopyOnWrite = false; + return Array[key]; + } + } + + TValue Delete(size_t key) { + if (!IsArray() || Array.size() <= key) { + return TValue::DefaultValue(); + } + + TValue v = Array[key]; + Array.erase(Array.begin() + key); + return v; + } + + const TValue& Get(TStringBuf key) const { + if (!IsDict()) { + return TValue::DefaultValue(); + } + + TDict::const_iterator it = Dict.find(key); + return it != Dict.end() ? it->second : TValue::DefaultValue(); + } + + TValue* GetNoAdd(TStringBuf key) { + if (!IsDict()) { + return nullptr; + } + + return Dict.FindPtr(key); + } + + TValue& Add(TStringBuf key) { + SetDict(); + TDict::iterator it = Dict.insert(std::make_pair(Pool->AppendBuf(key), TValue(Pool))).first; + it->second.CopyOnWrite = false; + return it->second; + } + + TValue& GetOrAdd(TStringBuf key) { + SetDict(); + TDict::insert_ctx ctx; + TDict::iterator it = Dict.find(key, ctx); + + if (it == Dict.end()) { + it = Dict.insert_direct(std::make_pair(Pool->AppendBuf(key), TValue(Pool)), ctx); + it->second.CopyOnWrite = false; + } + + return it->second; + } + + TValue Delete(TStringBuf key) { + if (!IsDict()) { + return TValue::DefaultValue(); + } + + TDict::iterator it = Dict.find(key); + + if (it == Dict.end()) { + return TValue::DefaultValue(); + } + + TValue v = it->second; + Dict.erase(key); + return v; + } + }; + + TValue::TScCore* TValue::NewCore(TPoolPtr& p) { + return new (p->Pool.Allocate<TScCore>()) TScCore(p); + } + + TValue::TValue() { + auto p = TPoolPtr(new NDefinitions::TPool); + TheCore = NewCore(p); + } + + TValue::TValue(double t) + : TValue() + { + SetNumber(t); + } + + TValue::TValue(unsigned long long t) + : TValue() + { + SetIntNumber(t); + } + + TValue::TValue(unsigned long t) + : TValue() + { + SetIntNumber(t); + } + + TValue::TValue(unsigned t) + : TValue() + { + SetIntNumber(t); + } + + TValue::TValue(long long t) + : TValue() + { + SetIntNumber(t); + } + + TValue::TValue(long t) + : TValue() + { + SetIntNumber(t); + } + + TValue::TValue(int t) + : TValue() + { + SetIntNumber(t); + } + + //TValue::TValue(bool t) + // : TValue() + //{ + // SetBool(t); + //} + + TValue::TValue(TStringBuf t) + : TValue() + { + SetString(t); + } + + TValue::TValue(const char* t) + : TValue() + { + SetString(t); + } + + TValue::TValue(TValue& v) + : TheCore(v.TheCore) + , CopyOnWrite(v.CopyOnWrite) + { + } + + TValue::TValue(const TValue& v) + : TheCore(v.TheCore) + , CopyOnWrite(true) + { + } + + TValue::TValue(TValue&& v) noexcept + : TheCore(std::move(v.TheCore)) + , CopyOnWrite(v.CopyOnWrite) + {} + + TValue::operator double() const { + return GetNumber(); + } + + TValue::operator float() const { + return GetNumber(); + } + + TValue::operator long long() const { + return GetIntNumber(); + } + + TValue::operator long() const { + return GetIntNumber(); + } + + TValue::operator int() const { + return GetIntNumber(); + } + + TValue::operator short() const { + return GetIntNumber(); + } + + TValue::operator char() const { + return GetIntNumber(); + } + + TValue::operator unsigned long long() const { + return GetIntNumber(); + } + + TValue::operator unsigned long() const { + return GetIntNumber(); + } + + TValue::operator unsigned() const { + return GetIntNumber(); + } + + TValue::operator unsigned short() const { + return GetIntNumber(); + } + + TValue::operator unsigned char() const { + return GetIntNumber(); + } + + TValue::operator signed char() const { + return GetIntNumber(); + } + + TValue::operator TStringBuf() const { + return GetString(); + } + + TValue::operator const ::NSc::TArray&() const { + return GetArray(); + } + + TValue::operator const ::NSc::TDict&() const { + return GetDict(); + } + + TValue& TValue::operator=(double t) { + return SetNumber(t); + } + + TValue& TValue::operator=(unsigned long long t) { + return SetIntNumber(t); + } + + TValue& TValue::operator=(unsigned long t) { + return SetIntNumber(t); + } + + TValue& TValue::operator=(unsigned t) { + return SetIntNumber(t); + } + + TValue& TValue::operator=(long long t) { + return SetIntNumber(t); + } + + TValue& TValue::operator=(long t) { + return SetIntNumber(t); + } + + TValue& TValue::operator=(int t) { + return SetIntNumber(t); + } + + //TValue& TValue::operator=(bool t) { + // return SetBool(t); + //} + + TValue& TValue::operator=(TStringBuf t) { + return SetString(t); + } + + TValue& TValue::operator=(const char* t) { + return SetString(t); + } + + TValue& TValue::operator=(TValue& v) & { + if (!Same(*this, v)) { + //Extend TheCore lifetime not to trigger possible v deletion via parent-child chain + auto tmpCore = TheCore; + TheCore = v.TheCore; + CopyOnWrite = v.CopyOnWrite; + } + return *this; + } + + TValue& TValue::operator=(const TValue& v) & { + if (!Same(*this, v)) { + //Extend TheCore lifetime not to trigger possible v deletion via parent-child chain + auto tmpCore = TheCore; + TheCore = v.TheCore; + CopyOnWrite = true; + } + return *this; + } + + TValue& TValue::operator=(TValue&& v) & noexcept { + if (!Same(*this, v)) { + //Extend TheCore lifetime not to trigger possible v deletion via parent-child chain + auto tmpCore = TheCore; + TheCore = std::move(v.TheCore); + CopyOnWrite = v.CopyOnWrite; + } + return *this; + } + + bool TValue::Has(size_t idx) const { + return IsArray() && GetArray().size() > idx; + } + + bool TValue::Has(TStringBuf s) const { + return GetDict().contains(s); + } + + TValue& TValue::GetOrAddUnsafe(size_t idx) { + return CoreMutable().GetOrAdd(idx); + } + + TValue& TValue::GetOrAdd(TStringBuf idx) { + return CoreMutable().GetOrAdd(idx); + } + + const TValue& TValue::Get(size_t idx) const { + return Core().Get(idx); + } + + TValue* TValue::GetNoAdd(size_t idx) { + return CoreMutable().GetNoAdd(idx); + } + + const TValue& TValue::Get(TStringBuf idx) const { + return Core().Get(idx); + } + + TValue* TValue::GetNoAdd(TStringBuf key) { + return CoreMutable().GetNoAdd(key); + } + + TValue& TValue::Back() { + return CoreMutable().Back(); + } + + const TValue& TValue::Back() const { + const TArray& arr = GetArray(); + return arr.empty() ? DefaultValue() : arr.back(); + } + + TValue TValue::Delete(size_t idx) { + return CoreMutable().Delete(idx); + } + + TValue TValue::Delete(TStringBuf idx) { + return CoreMutable().Delete(idx); + } + + TValue& TValue::AddAll(std::initializer_list<std::pair<TStringBuf, TValue>> t) { + for (const auto& el : t) { + Add(el.first) = el.second; + } + return *this; + } + + TValue& TValue::InsertUnsafe(size_t idx) { + return CoreMutable().Insert(idx); + } + + template <class TIt> + TValue& TValue::AppendAll(TIt begin, TIt end) { + GetArrayMutable().AppendAll(begin, end); + return *this; + } + + template <class TColl> + TValue& TValue::AppendAll(TColl&& coll) { + return AppendAll(std::begin(coll), std::end(coll)); + } + + TValue& TValue::AppendAll(std::initializer_list<TValue> coll) { + return AppendAll(coll.begin(), coll.end()); + } + + TValue& TValue::Push() { + return CoreMutable().Push(); + } + + TValue TValue::Pop() { + return CoreMutable().Pop(); + } + + TValue::EType TValue::GetType() const { + return Core().ValueType; + } + + bool TValue::IsNull() const { + return Core().IsNull(); + } + + bool TValue::IsNumber() const { + return Core().IsNumber(); + } + + bool TValue::IsIntNumber() const { + return Core().IsIntNumber(); + } + + bool TValue::IsBool() const { + return Core().IsBool(); + } + + bool TValue::IsString() const { + return Core().IsString(); + } + + bool TValue::IsArray() const { + return Core().IsArray(); + } + + bool TValue::IsDict() const { + return Core().IsDict(); + } + + TValue& TValue::SetNumber(double i) { + CoreMutableForSet().SetNumber(i); + return *this; + } + + TValue& TValue::SetIntNumber(i64 n) { + CoreMutableForSet().SetIntNumber(n); + return *this; + } + + TValue& TValue::SetBool(bool val) { + CoreMutableForSet().SetBool(val); + return *this; + } + + TValue& TValue::SetString(TStringBuf s) { + CoreMutableForSet().SetString(s); + return *this; + } + + double TValue::GetNumber(double d) const { + return Core().GetNumber(d); + } + + i64 TValue::GetIntNumber(i64 n) const { + return Core().GetIntNumber(n); + } + + bool TValue::GetBool(bool b) const { + return Core().GetBool(b); + } + + double& TValue::GetNumberMutable(double defaultval) { + return CoreMutable().GetNumberMutable(defaultval); + } + + i64& TValue::GetIntNumberMutable(i64 defaultval) { + return CoreMutable().GetIntNumberMutable(defaultval); + } + + TStringBuf TValue::GetString(TStringBuf d) const { + return Core().GetString(d); + } + + TValue& TValue::SetArray() { + CoreMutable().SetArray(); + return *this; + } + + TValue& TValue::ClearArray() { + CoreMutable().ClearArray(); + return *this; + } + + TValue& TValue::SetDict() { + CoreMutable().SetDict(); + return *this; + } + + TValue& TValue::ClearDict() { + CoreMutable().ClearDict(); + return *this; + } + + const TArray& TValue::GetArray() const { + return Core().GetArray(); + } + + TArray& TValue::GetArrayMutable() { + return CoreMutable().GetArrayMutable(); + } + + const TDict& TValue::GetDict() const { + return Core().GetDict(); + } + + TDict& TValue::GetDictMutable() { + return CoreMutable().GetDictMutable(); + } + + size_t TValue::StringSize() const { + return GetString().size(); + } + + size_t TValue::ArraySize() const { + return GetArray().size(); + } + + size_t TValue::DictSize() const { + return GetDict().size(); + } + + bool TValue::StringEmpty() const { + return GetString().empty(); + } + + bool TValue::ArrayEmpty() const { + return GetArray().empty(); + } + + bool TValue::DictEmpty() const { + return GetDict().empty(); + } + + TValue::TValue(TPoolPtr& p) + : TheCore(NewCore(p)) + { + } + + TValue::TScCore& TValue::CoreMutable() { + if (Y_UNLIKELY(!TheCore)) { + *this = TValue(); + } else if (Y_UNLIKELY(CopyOnWrite) && Y_UNLIKELY(TheCore->RefCount() > 1)) { + *this = Clone(); + } + + CopyOnWrite = false; + + return *TheCore; + } + + TValue::TScCore& TValue::CoreMutableForSet() { + if (Y_UNLIKELY(!TheCore) || Y_UNLIKELY(CopyOnWrite) && Y_UNLIKELY(TheCore->RefCount() > 1)) { + *this = TValue(); + } + + CopyOnWrite = false; + + return *TheCore; + } + + const TValue::TScCore& TValue::Core() const { + return TheCore ? *TheCore : DefaultCore(); + } + + TValue& TValue::SetNull() { + CoreMutableForSet().SetNull(); + return *this; + } + + namespace NPrivate { + int CompareStr(const NSc::TValue& a, TStringBuf b); + + int CompareInt(const NSc::TValue& a, i64 r); + + int CompareFloat(const NSc::TValue& a, double r); + } + + bool operator==(const TValue& a, const TValue& b); + + bool operator!=(const TValue& a, const TValue& b); + + bool operator<=(const TValue&, const TValue&) = delete; + bool operator>=(const TValue&, const TValue&) = delete; + bool operator<(const TValue&, const TValue&) = delete; + bool operator>(const TValue&, const TValue&) = delete; + +#define LIBRARY_SCHEME_DECLARE_TVALUE_OPS(T, Impl) \ + bool operator==(const NSc::TValue& a, T b); \ + bool operator==(T b, const NSc::TValue& a); \ + bool operator!=(const NSc::TValue& a, T b); \ + bool operator!=(T b, const NSc::TValue& a); \ + bool operator<=(const NSc::TValue& a, T b); \ + bool operator<=(T b, const NSc::TValue& a); \ + bool operator>=(const NSc::TValue& a, T b); \ + bool operator>=(T b, const NSc::TValue& a); \ + bool operator<(const NSc::TValue& a, T b); \ + bool operator<(T b, const NSc::TValue& a); \ + bool operator>(const NSc::TValue& a, T b); \ + bool operator>(T b, const NSc::TValue& a); + +#define LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(T) \ + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(signed T, CompareInt) \ + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(unsigned T, CompareInt) + + //LIBRARY_SCHEME_DECLARE_TVALUE_OPS(bool, CompareInt) + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(char, CompareInt) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(char) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(short) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(int) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(long) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(long long) + + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(float, CompareFloat) + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(double, CompareFloat) + + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(TStringBuf, CompareStr) + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(const TString&, CompareStr) + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(const char* const, CompareStr) + +#undef LIBRARY_SCHEME_DECLARE_TVALUE_OPS +#undef LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS + +} diff --git a/library/cpp/scheme/scimpl_defs.h b/library/cpp/scheme/scimpl_defs.h new file mode 100644 index 00000000000..f3dd66b4379 --- /dev/null +++ b/library/cpp/scheme/scimpl_defs.h @@ -0,0 +1,140 @@ +#pragma once + +#include <library/cpp/json/json_reader.h> +#include <library/cpp/json/json_value.h> + +#include <util/system/types.h> +#include <util/memory/pool.h> + +#include <util/generic/deque.h> +#include <util/generic/hash.h> +#include <util/generic/ptr.h> +#include <util/generic/strbuf.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> +#include <functional> +#include <util/string/vector.h> +#include <util/string/type.h> + +namespace NSc { + namespace NDefinitions { + const size_t POOL_BLOCK_SIZE = 4000; // leave 96 bytes for overhead + + struct TPool: public TAtomicRefCount<TPool> { + TMemoryPool Pool; + + TPool(size_t blsz = POOL_BLOCK_SIZE, TMemoryPool::IGrowPolicy* grow = TMemoryPool::TExpGrow::Instance()) + : Pool(blsz, grow) + { + } + + TMemoryPool* Get() { + return &Pool; + } + + TStringBuf AppendBuf(const TStringBuf& strb) { + return Pool.AppendCString(strb); + } + + template <typename T> + T* NewWithPool() { + return new (Pool.Allocate<T>()) T(&Pool); + } + }; + } + + using TStringBufs = TVector<TStringBuf>; + + class TSchemeException : public yexception { + }; + + class TSchemeParseException : public TSchemeException { + public: + size_t Offset = 0; + TString Reason; + + public: + TSchemeParseException() = default; + + TSchemeParseException(size_t off, const TString& reason) + : Offset(off) + , Reason(reason) + { + } + }; + + struct TJsonOpts: public NJson::TJsonReaderConfig { + enum EJsonOpts { + JO_DEFAULT = 0, // just dump json, used to be default, actually + JO_SORT_KEYS = 1, // sort dict keys to make output more predictable + JO_SKIP_UNSAFE = 2, // skip nonunicode data to make external json parsers happy + // will skip nonunicode dict keys and replace nonunicode values with nulls + JO_FORMAT = 8, // format json + + JO_PARSER_STRICT_JSON = 16, // strict standard json + JO_PARSER_STRICT_UTF8 = 32, // strict utf8 + JO_PARSER_DISALLOW_COMMENTS = 64, + JO_PARSER_DISALLOW_DUPLICATE_KEYS = 128, + + JO_PRETTY = JO_FORMAT | JO_SORT_KEYS, // pretty print json + JO_SAFE = JO_SKIP_UNSAFE | JO_SORT_KEYS, // ensure standard parser-safe json + + JO_PARSER_STRICT_WITH_COMMENTS = JO_PARSER_STRICT_JSON | JO_PARSER_STRICT_UTF8, + JO_PARSER_STRICT = JO_PARSER_STRICT_JSON | JO_PARSER_STRICT_UTF8 | JO_PARSER_DISALLOW_COMMENTS, + }; + + public: + TJsonOpts(int opts = JO_SORT_KEYS) + : Opts(opts) + , SortKeys(opts & JO_SORT_KEYS) + , FormatJson(opts & JO_FORMAT) + , StringPolicy((opts & JO_SKIP_UNSAFE) ? StringPolicySafe : StringPolicyUnsafe) + { + AllowComments = !(opts & JO_PARSER_DISALLOW_COMMENTS); + RelaxedJson = !(opts & JO_PARSER_STRICT_JSON); + DontValidateUtf8 = !(opts & JO_PARSER_STRICT_UTF8); + } + + public: + bool RelaxedJson = false; + int Opts = 0; + bool SortKeys = true; + bool FormatJson = false; + + // return true to proceed with output, false to skip, optionally modify value + std::function<bool(double&)> NumberPolicy = NumberPolicySafe; + std::function<bool(TStringBuf&)> StringPolicy = StringPolicyUnsafe; + + public: + static bool NumberPolicySafe(double&); // skip if nan or inf + static bool NumberPolicyUnsafe(double&) { + return true; + } + + static bool StringPolicySafe(TStringBuf&); // skip if not utf8 + static bool StringPolicyUnsafe(TStringBuf&) { + return true; + } + }; + + struct TProtoOpts { + // Serialization throws on unknown enum value if not set, else use default value + bool UnknownEnumValueIsDefault = false; + + // Serialization throws on type mismatch if not set, else leaves protobuf empty + bool SkipTypeMismatch = false; + }; + + namespace NImpl { + class TKeySortContext; + class TSelfLoopContext; + class TSelfOverrideContext; + } +} + +namespace google { + namespace protobuf { + class Message; + class FieldDescriptor; + } +} diff --git a/library/cpp/scheme/scimpl_json_read.cpp b/library/cpp/scheme/scimpl_json_read.cpp new file mode 100644 index 00000000000..8a29cc77391 --- /dev/null +++ b/library/cpp/scheme/scimpl_json_read.cpp @@ -0,0 +1,229 @@ +#include "scimpl.h" + +#include <library/cpp/json/json_reader.h> +#include <util/stream/output.h> +#include <util/generic/maybe.h> + +namespace NSc { + struct TJsonError { + size_t Offset = 0; + TMaybe<TString> Reason; + }; + + struct TJsonDeserializer : NJson::TJsonCallbacks { + struct TContainer { + TValue* Container = nullptr; + TValue* LastValue = nullptr; + bool ExpectKey = false; + + TContainer(TValue& v) + : Container(&v) + , ExpectKey(v.IsDict()) + { + } + + bool Add(TStringBuf v, bool allowDuplicated) { + if (!ExpectKey || Y_UNLIKELY(!Container->IsDict())) + return false; + + if (!allowDuplicated && Y_UNLIKELY(Container->Has(v))) + return false; + + LastValue = &Container->GetOrAdd(v); + ExpectKey = false; + return true; + } + + void Push() { + LastValue = &Container->Push(); + } + + TValue& NextValue() { + if (Container->IsArray()) { + Push(); + } else if (Container->IsDict()) { + ExpectKey = true; + } + + return *(LastValue ? LastValue : Container); + } + + bool IsArray() const { + return !!Container && Container->IsArray(); + } + + bool IsDict() const { + return !!Container && Container->IsDict(); + } + }; + + typedef TVector<TContainer> TStackType; + + public: + TValue& Root; + TJsonError& Error; + const TJsonOpts& Cfg; + + TStackType Stack; + bool Virgin = true; + + public: + TJsonDeserializer(TValue& root, TJsonError& err, const TJsonOpts& cfg) + : Root(root) + , Error(err) + , Cfg(cfg) + { + Root.SetNull(); + Stack.reserve(10); + } + + bool HasNextValue() const { + return Virgin | !Stack.empty(); + } + + TValue& NextValue() { + Virgin = false; + return Stack.empty() ? Root : Stack.back().NextValue(); + } + + bool OnNull() override { + if (Y_UNLIKELY(!HasNextValue())) + return false; + + NextValue().SetNull(); + return true; + } + + bool OnEnd() override { + return Stack.empty(); + } + + template <typename T> + bool OnValue(T v) { + if (Y_UNLIKELY(!HasNextValue())) + return false; + + NextValue() = v; + return true; + } + + template <typename T> + bool OnIntValue(T v) { + if (Y_UNLIKELY(!HasNextValue())) + return false; + + NextValue().SetIntNumber(v); + return true; + } + + bool OnBoolean(bool v) override { + if (Y_UNLIKELY(!HasNextValue())) + return false; + + NextValue().SetBool(v); + return true; + } + + bool OnInteger(long long v) override { + return OnIntValue(v); + } + + bool OnUInteger(unsigned long long v) override { + return OnIntValue(v); + } + + bool OnDouble(double v) override { + return OnValue(v); + } + + bool OnString(const TStringBuf& v) override { + return OnValue(v); + } + + bool OnMapKey(const TStringBuf& k) override { + if (Y_UNLIKELY(Stack.empty())) + return false; + return Stack.back().Add(k, !(Cfg.Opts & TJsonOpts::JO_PARSER_DISALLOW_DUPLICATE_KEYS)); + } + + bool OnOpenMap() override { + if (Y_UNLIKELY(!HasNextValue())) + return false; + Stack.push_back(TContainer(NextValue().SetDict())); + return true; + } + + bool OnCloseMap() override { + if (Y_UNLIKELY(Stack.empty() || !Stack.back().IsDict())) + return false; + Stack.pop_back(); + return true; + } + + bool OnOpenArray() override { + if (Y_UNLIKELY(!HasNextValue())) + return false; + Stack.push_back(TContainer(NextValue().SetArray())); + return true; + } + + bool OnCloseArray() override { + if (Y_UNLIKELY(Stack.empty() || !Stack.back().IsArray())) + return false; + Stack.pop_back(); + return true; + } + + void OnError(size_t off, TStringBuf reason) override { + Error.Offset = off; + Error.Reason = reason; + } + }; + + static bool DoParseFromJson(TValue& res, TJsonError& err, TStringBuf json, const TJsonOpts& cfg) { + TJsonDeserializer d(res, err, cfg); + + if (cfg.RelaxedJson) { + return NJson::ReadJsonFast(json, &d); + } else { + TMemoryInput min(json.data(), json.size()); + return NJson::ReadJson(&min, &cfg, &d); + } + } + + static bool DoParseFromJson(TValue& res, TStringBuf json, const TJsonOpts& cfg) { + TJsonError err; + return DoParseFromJson(res, err, json, cfg); + } + + TValue TValue::FromJson(TStringBuf v, const TJsonOpts& cfg) { + TValue res; + if (FromJson(res, v, cfg)) { + return res; + } else { + return DefaultValue(); + } + } + + TValue TValue::FromJsonThrow(TStringBuf json, const TJsonOpts& cfg) { + TValue res; + TJsonError err; + + if (DoParseFromJson(res, err, json, cfg)) { + return res; + } + + TString reason = err.Reason.Empty() ? "NULL" : *err.Reason; + ythrow TSchemeParseException(err.Offset, reason) << "JSON error at offset " << err.Offset << " (" << reason << ")"; + } + + bool TValue::FromJson(TValue& res, TStringBuf json, const TJsonOpts& cfg) { + if (DoParseFromJson(res, json, cfg)) { + return true; + } + + res.SetNull(); + return false; + } + +} diff --git a/library/cpp/scheme/scimpl_json_write.cpp b/library/cpp/scheme/scimpl_json_write.cpp new file mode 100644 index 00000000000..aadd7e6cd5f --- /dev/null +++ b/library/cpp/scheme/scimpl_json_write.cpp @@ -0,0 +1,193 @@ +#include "scimpl.h" +#include "scimpl_private.h" + +#include <library/cpp/json/json_prettifier.h> +#include <library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h> + +#include <util/charset/utf8.h> +#include <util/generic/algorithm.h> +#include <util/generic/ymath.h> +#include <util/system/tls.h> + +namespace NSc { + bool TJsonOpts::StringPolicySafe(TStringBuf& s) { + return IsUtf(s); + } + + bool TJsonOpts::NumberPolicySafe(double& d) { + return IsFinite(d); + } + + static inline void WriteString(IOutputStream& out, TStringBuf s) { + NEscJ::EscapeJ<true, true>(s, out); + } + + static inline const NSc::TValue& GetValue(size_t, TStringBuf key, const TDict& dict) { + return dict.find(key)->second; + } + + static inline const NSc::TValue& GetValue(TDict::const_iterator it, TStringBuf, const TDict&) { + return it->second; + } + + static inline TStringBuf GetKey(size_t it, const NImpl::TKeySortContext::TGuard& keys) { + return keys.GetVector()[it]; + } + + static inline TStringBuf GetKey(TDict::const_iterator it, const TDict&) { + return it->first; + } + + template <typename TDictKeys> + static inline void WriteDict(IOutputStream& out, const TDictKeys& keys, const TDict& dict, + const TJsonOpts& jopts, NImpl::TKeySortContext& sortCtx, NImpl::TSelfLoopContext& loopCtx) { + using const_iterator = typename TDictKeys::const_iterator; + const_iterator begin = keys.begin(); + const_iterator end = keys.end(); + for (const_iterator it = begin; it != end; ++it) { + TStringBuf key = GetKey(it, keys); + + if (jopts.StringPolicy && !jopts.StringPolicy(key)) { + ++begin; + continue; + } + + if (it != begin) { + out << ','; + } + + NEscJ::EscapeJ<true, true>(key, out); + out << ':'; + + GetValue(it, key, dict).DoWriteJsonImpl(out, jopts, sortCtx, loopCtx); + } + } + + void TValue::DoWriteJsonImpl(IOutputStream& out, const TJsonOpts& jopts, + NImpl::TKeySortContext& sortCtx, NImpl::TSelfLoopContext& loopCtx) const { + const TScCore& core = Core(); + + NImpl::TSelfLoopContext::TGuard loopCheck(loopCtx, core); + + if (!loopCheck.Ok) { + out << TStringBuf("null"); // a loop encountered (and asserted), skip the back reference + return; + } + + switch (core.ValueType) { + default: { + Y_ASSERT(false); + [[fallthrough]]; /* no break */ + } + case EType::Null: { + out << TStringBuf("null"); + break; + } + case EType::Bool: { + out << (core.IntNumber ? TStringBuf("true") : TStringBuf("false")); + break; + } + case EType::IntNumber: { + out << core.IntNumber; + break; + } + case EType::FloatNumber: { + double d = core.FloatNumber; + if (!jopts.NumberPolicy || jopts.NumberPolicy(d)) { + out << d; + } else { + out << TStringBuf("null"); + } + break; + } + case EType::String: { + TStringBuf s = core.String; + if (!jopts.StringPolicy || jopts.StringPolicy(s)) { + WriteString(out, s); + } else { + out << TStringBuf("null"); + } + break; + } + case EType::Array: { + out << '['; + const TArray& a = core.GetArray(); + for (TArray::const_iterator it = a.begin(); it != a.end(); ++it) { + if (it != a.begin()) { + out << ','; + } + + it->DoWriteJsonImpl(out, jopts, sortCtx, loopCtx); + } + out << ']'; + break; + } + case EType::Dict: { + out << '{'; + + const TDict& dict = core.GetDict(); + + if (jopts.SortKeys) { + NImpl::TKeySortContext::TGuard keys(sortCtx, dict); + WriteDict(out, keys, dict, jopts, sortCtx, loopCtx); + } else { + WriteDict(out, dict, dict, jopts, sortCtx, loopCtx); + } + + out << '}'; + break; + } + } + } + + const TValue& TValue::ToJson(IOutputStream& out, const TJsonOpts& jopts) const { + using namespace NImpl; + + if (jopts.FormatJson) { + TStringStream str; + DoWriteJsonImpl(str, jopts, GetTlsInstance<TKeySortContext>(), GetTlsInstance<TSelfLoopContext>()); + NJson::PrettifyJson(str.Str(), out); + } else { + DoWriteJsonImpl(out, jopts, GetTlsInstance<TKeySortContext>(), GetTlsInstance<TSelfLoopContext>()); + } + + return *this; + } + + TString TValue::ToJson(const TJsonOpts& jopts) const { + TString s; + { + TStringOutput out(s); + ToJson(out, jopts); + } + return s; + } + + TJsonOpts TValue::MakeOptsSafeForSerializer(TJsonOpts opts) { + opts.SortKeys = true; + opts.StringPolicy = TJsonOpts::StringPolicySafe; + opts.NumberPolicy = TJsonOpts::NumberPolicySafe; + return opts; + } + + TJsonOpts TValue::MakeOptsPrettyForSerializer(TJsonOpts opts) { + opts.FormatJson = true; + return MakeOptsSafeForSerializer(opts); + } + + TString TValue::ToJsonSafe(const TJsonOpts& jopts) const { + return ToJson(MakeOptsSafeForSerializer(jopts)); + } + + const TValue& TValue::ToJsonSafe(IOutputStream& out, const TJsonOpts& jopts) const { + return ToJson(out, MakeOptsSafeForSerializer(jopts)); + } + + TString TValue::ToJsonPretty(const TJsonOpts& jopts) const { + return ToJson(MakeOptsPrettyForSerializer(jopts)); + } + + const TValue& TValue::ToJsonPretty(IOutputStream& out, const TJsonOpts& jopts) const { + return ToJson(out, MakeOptsPrettyForSerializer(jopts)); + } +} diff --git a/library/cpp/scheme/scimpl_private.cpp b/library/cpp/scheme/scimpl_private.cpp new file mode 100644 index 00000000000..024bf8cc3b2 --- /dev/null +++ b/library/cpp/scheme/scimpl_private.cpp @@ -0,0 +1,74 @@ +#include "scimpl_private.h" + +#include <util/generic/algorithm.h> +#include <utility> + +namespace NSc { + namespace NImpl { + struct TGetKey { + static inline TStringBuf Do(const TDict::value_type& v) { + return v.first; + } + }; + + struct TMoveValue { + static inline TValue&& Do(TDict::value_type& v) { + return std::move(v.second); + } + + static inline TValue&& Do(TArray::value_type& v) { + return std::move(v); + } + }; + + template <typename TAction, typename TElement, typename TColl> + static inline void PutToVector(TVector<TElement>& vector, TColl& coll) { + size_t i = vector.size(); + vector.resize(vector.size() + coll.size()); + + for (auto& item : coll) { + vector[i++] = TAction::Do(item); + } + } + + bool TKeySortContext::Process(const TDict& self) { + size_t oldSz = Vector.size(); + PutToVector<TGetKey>(Vector, self); + Sort(Vector.begin() + oldSz, Vector.end()); + return true; + } + + bool TSelfOverrideContext::Process(TValue::TScCore& self) { + if (self.GetDict().size()) { + PutToVector<TMoveValue>(Vector, self.Dict); + } else if (self.GetArray().size()) { + PutToVector<TMoveValue>(Vector, self.Array); + } + return true; + } + + bool TSelfLoopContext::Process(const TValue::TScCore& self) { + const bool ok = (Vector.end() == Find(Vector.begin(), Vector.end(), &self)); + + if (!ok) { + switch (ReportingMode) { + case EMode::Assert: + Y_ASSERT(false); // make sure the debug build sees this + break; + case EMode::Throw: + ythrow TSchemeException() << "REFERENCE LOOP DETECTED"; + case EMode::Abort: + Y_FAIL("REFERENCE LOOP DETECTED"); + break; + case EMode::Stderr: + Cerr << "REFERENCE LOOP DETECTED: " << JoinStrings(Vector.begin(), Vector.end(), ", ") + << " AND " << ToString((const void*)&self) << Endl; + break; + } + } + + Vector.push_back(&self); + return ok; + } + } +} diff --git a/library/cpp/scheme/scimpl_private.h b/library/cpp/scheme/scimpl_private.h new file mode 100644 index 00000000000..b92badabde1 --- /dev/null +++ b/library/cpp/scheme/scimpl_private.h @@ -0,0 +1,114 @@ +#pragma once + +#include "scheme.h" + +#include <util/thread/singleton.h> + +namespace NSc { + namespace NImpl { + template <typename TContext> + static inline TContext& GetTlsInstance() { + return *FastTlsSingleton<TContext>(); + } + + template <typename TContext> + class TContextGuard : TNonCopyable { + using TElement = typename TContext::TElement; + using TTarget = typename TContext::TTarget; + using TVectorType = TVector<TElement>; + + public: + TContextGuard(TContext& ctx, TTarget& target) + : Ctx(ctx) + , Active(TContext::Needed(target)) + { + if (Active) { + Begin = Ctx.Vector.size(); + Ok = Ctx.Process(target); + End = Ctx.Vector.size(); + } + } + + ~TContextGuard() noexcept { + if (Active) { + Ctx.Vector.resize(Begin); + } + } + + const TVectorType& GetVector() const { + return Ctx.Vector; + } + + using const_iterator = size_t; + + size_t begin() const { + return Begin; + } + + size_t end() const { + return End; + } + + bool Ok = true; + + private: + TContext& Ctx; + size_t Begin = 0; + size_t End = 0; + bool Active = false; + }; + + template <typename TElem, typename TTgt> + class TBasicContext { + public: + using TElement = TElem; + using TTarget = TTgt; + + TBasicContext() { + Vector.reserve(64); + } + + TVector<TElement> Vector; + }; + + class TKeySortContext: public TBasicContext<TStringBuf, const TDict> { + public: + using TGuard = TContextGuard<TKeySortContext>; + + bool Process(const TDict& self); + + static bool Needed(const TDict& self) { + return self.size(); + } + }; + + class TSelfOverrideContext: public TBasicContext<TValue, TValue::TScCore> { + public: + using TGuard = TContextGuard<TSelfOverrideContext>; + + bool Process(TValue::TScCore& self); + + static bool Needed(const TValue::TScCore& self) { + return self.HasChildren(); + } + }; + + class TSelfLoopContext: public TBasicContext<const void*, const TValue::TScCore> { + public: + enum class EMode { + Assert, Throw, Abort, Stderr + }; + + using TGuard = TContextGuard<TSelfLoopContext>; + + bool Process(const TValue::TScCore& self); + + static bool Needed(const TValue::TScCore& self) { + return self.HasChildren(); + } + + public: + EMode ReportingMode = EMode::Assert; + }; + } +} diff --git a/library/cpp/scheme/scimpl_protobuf.cpp b/library/cpp/scheme/scimpl_protobuf.cpp new file mode 100644 index 00000000000..0c99122c69a --- /dev/null +++ b/library/cpp/scheme/scimpl_protobuf.cpp @@ -0,0 +1,329 @@ +#include "scheme.h" + +#include <util/generic/vector.h> +#include <util/generic/yexception.h> + +#include <google/protobuf/descriptor.h> +#include <google/protobuf/message.h> +#include <google/protobuf/reflection.h> + +using namespace google::protobuf; + +namespace NSc { + TValue TValue::From(const Message& msg, bool mapAsDict) { + TValue v; + const Reflection* r = msg.GetReflection(); + TVector<const FieldDescriptor*> fields; + TVector<const FieldDescriptor*>::iterator it; + int i1; + + r->ListFields(msg, &fields); + for (it = fields.begin(), i1 = 0; it != fields.end(); ++it, ++i1) { + const FieldDescriptor* field = *it; + try { + if (field->is_repeated()) { + if (field->is_map() && mapAsDict) { + auto& elem = v[field->name()]; + for (int i2 = 0; i2 < r->FieldSize(msg, field); ++i2) { + auto val = FromRepeatedField(msg, field, i2); + if (val.IsDict()) { + elem[TStringBuf(val["key"])] = val["value"]; + } + } + } else { + for (int i2 = 0; i2 < r->FieldSize(msg, field); ++i2) + v[field->name()][i2] = FromRepeatedField(msg, field, i2); + } + } else { + v[field->name()] = FromField(msg, field); + } + } catch (...) { + /* conversion failed, skip this field */ + } + } + + return v; + } + + TValue TValue::FromField(const Message& msg, const FieldDescriptor* field) { + TValue v; + const Reflection* r = msg.GetReflection(); + + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + v = r->GetInt32(msg, field); + break; + case FieldDescriptor::CPPTYPE_INT64: + v = r->GetInt64(msg, field); + break; + case FieldDescriptor::CPPTYPE_UINT32: + v = r->GetUInt32(msg, field); + break; + case FieldDescriptor::CPPTYPE_UINT64: + v = r->GetUInt64(msg, field); + break; + case FieldDescriptor::CPPTYPE_DOUBLE: + v = r->GetDouble(msg, field); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + v = r->GetFloat(msg, field); + break; + case FieldDescriptor::CPPTYPE_BOOL: + v.SetBool(r->GetBool(msg, field)); + break; + case FieldDescriptor::CPPTYPE_ENUM: + v = r->GetEnum(msg, field)->name(); + break; + case FieldDescriptor::CPPTYPE_STRING: + v = r->GetString(msg, field); + break; + case FieldDescriptor::CPPTYPE_MESSAGE: + v = From(r->GetMessage(msg, field)); + break; + default: + ythrow TSchemeException() << "field " << field->full_name() << " unexpected type " << (int)field->cpp_type(); + } + + return v; + } + + TValue TValue::FromRepeatedField(const Message& msg, const FieldDescriptor* field, int index) { + TValue v; + const Reflection* r = msg.GetReflection(); + + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + v = r->GetRepeatedInt32(msg, field, index); + break; + case FieldDescriptor::CPPTYPE_INT64: + v = r->GetRepeatedInt64(msg, field, index); + break; + case FieldDescriptor::CPPTYPE_UINT32: + v = r->GetRepeatedUInt32(msg, field, index); + break; + case FieldDescriptor::CPPTYPE_UINT64: + v = r->GetRepeatedUInt64(msg, field, index); + break; + case FieldDescriptor::CPPTYPE_DOUBLE: + v = r->GetRepeatedDouble(msg, field, index); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + v = r->GetRepeatedFloat(msg, field, index); + break; + case FieldDescriptor::CPPTYPE_BOOL: + v.SetBool(r->GetRepeatedBool(msg, field, index)); + break; + case FieldDescriptor::CPPTYPE_ENUM: + v = r->GetRepeatedEnum(msg, field, index)->name(); + break; + case FieldDescriptor::CPPTYPE_STRING: + v = r->GetRepeatedString(msg, field, index); + break; + case FieldDescriptor::CPPTYPE_MESSAGE: + v = From(r->GetRepeatedMessage(msg, field, index)); + break; + default: + ythrow TSchemeException() << "field " << field->full_name() << " unexpected type " << (int)field->cpp_type(); + } + + return v; + } + + void TValue::To(Message& msg, const TProtoOpts& opts) const { + msg.Clear(); + + if (IsNull()) { + return; + } + + if (!IsDict()) { + ythrow TSchemeException() << "expected dictionary"; + } + + const Descriptor* descriptor = msg.GetDescriptor(); + for (int i = 0, count = descriptor->field_count(); i < count; ++i) { + const FieldDescriptor* field = descriptor->field(i); + if (field->is_map()) { + ToMapField(msg, field, opts); + } else if (field->is_repeated()) { + ToRepeatedField(msg, field, opts); + } else { + ToField(msg, field, opts); + } + } + } + + void TValue::ValueToField(const TValue& value, Message& msg, const FieldDescriptor* field, const TProtoOpts& opts) const { + const TString& name = field->name(); + if (value.IsNull()) { + if (field->is_required() && !field->has_default_value()) { + ythrow TSchemeException() << "has no value for required field " << name; + } + return; + } + + const Reflection* reflection = msg.GetReflection(); + + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + reflection->SetInt32(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_INT64: + reflection->SetInt64(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_UINT32: + reflection->SetUInt32(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_UINT64: + reflection->SetUInt64(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_DOUBLE: + reflection->SetDouble(&msg, field, value.ForceNumber()); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + reflection->SetFloat(&msg, field, value.ForceNumber()); + break; + case FieldDescriptor::CPPTYPE_BOOL: + reflection->SetBool(&msg, field, value.IsTrue()); + break; + case FieldDescriptor::CPPTYPE_STRING: + reflection->SetString(&msg, field, value.ForceString()); + break; + case FieldDescriptor::CPPTYPE_ENUM: + value.ToEnumField(msg, field, opts); + break; + case FieldDescriptor::CPPTYPE_MESSAGE: + value.To(*reflection->MutableMessage(&msg, field), opts); + break; + default: + ythrow TSchemeException() + << "field " << field->full_name() + << " unexpected type " << (int)field->cpp_type(); + } + } + + void TValue::ToField(Message& msg, const FieldDescriptor* field, const TProtoOpts& opts) const { + const TString& name = field->name(); + const TValue& value = Get(name); + ValueToField(value, msg, field, opts); + } + + void TValue::ToEnumField(Message& msg, const FieldDescriptor* field, const TProtoOpts& opts) const { + const EnumDescriptor* enumField = field->enum_type(); + + const EnumValueDescriptor* enumFieldValue = IsString() + ? enumField->FindValueByName(ForceString()) + : enumField->FindValueByNumber(ForceIntNumber()); + + if (!enumFieldValue) { + if (opts.UnknownEnumValueIsDefault) { + enumFieldValue = field->default_value_enum(); + } else { + ythrow TSchemeException() << "invalid value of enum field " << field->name(); + } + } + + const Reflection* reflection = msg.GetReflection(); + + if (field->is_repeated()) { + reflection->AddEnum(&msg, field, enumFieldValue); + } else { + reflection->SetEnum(&msg, field, enumFieldValue); + } + } + + void TValue::ToRepeatedField(Message& msg, const FieldDescriptor* field, const TProtoOpts& opts) const { + const TString& name = field->name(); + + const TValue& fieldValue = Get(name); + if (fieldValue.IsNull()) { + return; + } + + if (!fieldValue.IsArray()) { + if (opts.SkipTypeMismatch) { + return; // leave repeated field empty + } else { + ythrow TSchemeException() << "invalid type of repeated field " << name << ": not an array"; + } + } + + const Reflection* reflection = msg.GetReflection(); + + for (const TValue& value : fieldValue.GetArray()) { + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + reflection->AddInt32(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_INT64: + reflection->AddInt64(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_UINT32: + reflection->AddUInt32(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_UINT64: + reflection->AddUInt64(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_DOUBLE: + reflection->AddDouble(&msg, field, value.ForceNumber()); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + reflection->AddFloat(&msg, field, value.ForceNumber()); + break; + case FieldDescriptor::CPPTYPE_BOOL: + reflection->AddBool(&msg, field, value.IsTrue()); + break; + case FieldDescriptor::CPPTYPE_STRING: + reflection->AddString(&msg, field, value.ForceString()); + break; + case FieldDescriptor::CPPTYPE_ENUM: + value.ToEnumField(msg, field, opts); + break; + case FieldDescriptor::CPPTYPE_MESSAGE: + value.To(*reflection->AddMessage(&msg, field)); + break; + default: + ythrow TSchemeException() + << "field " << field->full_name() + << " unexpected type " << (int)field->cpp_type(); + } + } + } + + void TValue::ToMapField(Message& msg, const FieldDescriptor* field, const TProtoOpts& opts) const { + const TString& name = field->name(); + + const TValue& fieldValue = Get(name); + if (fieldValue.IsNull()) { + return; + } + + if (fieldValue.IsArray()) { + // read dict from key, value array + ToRepeatedField(msg, field, opts); + return; + } + + if (!fieldValue.IsDict()) { + if (opts.SkipTypeMismatch) { + return; // leave map field empty + } else { + ythrow TSchemeException() << "invalid type of map field " << name << ": not dict or array"; + } + } + + const Reflection* reflection = msg.GetReflection(); + + auto mutableField = reflection->GetMutableRepeatedFieldRef<Message>(&msg, field); + for (const auto& value : fieldValue.GetDict()) { + THolder<Message> entry(mutableField.NewMessage()); + auto entryDesc = entry->GetDescriptor(); + auto keyField = entryDesc->FindFieldByNumber(1); + auto valueField = entryDesc->FindFieldByNumber(2); + auto entryReflection = entry->GetReflection(); + entryReflection->SetString(entry.Get(), keyField, TString(value.first)); + ValueToField(value.second, *entry, valueField, opts); + mutableField.Add(*entry); + } + } +} diff --git a/library/cpp/scheme/scimpl_select.rl6 b/library/cpp/scheme/scimpl_select.rl6 new file mode 100644 index 00000000000..11aa549b78b --- /dev/null +++ b/library/cpp/scheme/scimpl_select.rl6 @@ -0,0 +1,270 @@ +#include <library/cpp/scheme/scimpl.h> + +#include <util/string/cast.h> +#include <util/string/escape.h> +#include <library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h> +#include <util/generic/is_in.h> +#include <util/string/util.h> + +namespace NSc { + + template <typename TValue, typename TGetter> + struct TSelector { + TValue& Root; + TValue* Current = nullptr; + TValue* Parent = nullptr; + TStringBuf CurrentDictKey; + size_t CurrentArrayKey = 0; + size_t Depth = 0; + bool HasArray = false; + bool HasError = false; + + TSelector(TValue& root) + : Root(root) + , Current(&Root) + {} + + template <typename T> + bool Next(T k) { + Depth += 1; + Parent = Current; + Current = TGetter::Next(Current, k); + return Current != nullptr; + } + + bool NextDict(TStringBuf k) { + return Next(CurrentDictKey = k); + } + + bool NextArray(size_t k) { + HasArray = true; + return Next(CurrentArrayKey = k); + } + + bool Error() { + Parent = nullptr; + Current = nullptr; + CurrentArrayKey = 0; + CurrentDictKey = TStringBuf(); + HasError = true; + return false; + } + }; + + template <typename TSelector> + struct TSelectorCtx { + TSelector Selector; + TString Buffer; + + const char* p0 = nullptr; + const char* p = nullptr; + const char* pe = nullptr; + const char* eof = nullptr; + const char* ts = nullptr; + const char* te = nullptr; + int cs = 0; + int act = 0; + + TSelectorCtx(TSelector sel, TStringBuf data) + : Selector(sel) + , p0(data.data()) + , p(data.data()) + , pe(data.end()) + , eof(data.end()) + {} + + bool OnString(TStringBuf s) { + return Selector.NextDict(s); + } + + bool OnInt(size_t k) { + return Selector.NextArray(k); + } + + bool OnStrU() { + return OnString(TStringBuf(ts, te)); + } + + bool OnStrQ() { + return OnString(TStringBuf(ts + 1, te - 1)); + } + + bool OnStrE() { + Buffer.clear(); + Buffer.reserve(te - ts); + UnescapeC(ts + 1, te - ts - 2, Buffer); + return OnString(Buffer); + } + + bool OnIntU() { + return OnInt(FromString<ui32>(TStringBuf(ts, te))); + } + + bool OnIntQ() { + return OnInt(FromString<ui32>(TStringBuf(ts + 1, te - 1))); + } + + bool OnError() { + Selector.Error(); + return false; + } + + bool SelectPath(); + }; + +#if 0 + %%{ + machine schemeselect; + + alphtype char; + + action OnIntU { if (Y_UNLIKELY(!OnIntU())) goto TOKEN_ERROR; } + action OnIntQ { if (Y_UNLIKELY(!OnIntQ())) goto TOKEN_ERROR; } + action OnStrU { if (Y_UNLIKELY(!OnStrU())) goto TOKEN_ERROR; } + action OnStrQ { if (Y_UNLIKELY(!OnStrQ())) goto TOKEN_ERROR; } + action OnStrE { if (Y_UNLIKELY(!OnStrE())) goto TOKEN_ERROR; } + action OnError { goto TOKEN_ERROR; } + + intu = [0-9]+; + intq = '[' intu ']'; + + uchar0 = [a-zA-Z_@$] | (0x80 .. 0xFF); + uchar = uchar0 | digit | [.\-]; + + qchar = [^'\\]; #'; + dchar = [^"\\]; #"; + bchar = [^\]\\]; + + echar = "\\" any; + + qechar = qchar | echar; + dechar = dchar | echar; + bechar = bchar | echar; + + strq = "'" qchar* "'"; + strd = '"' dchar* '"'; + strb = '[' bchar* ']'; + + strqe = "'" qechar* "'"; + strde = '"' dechar* '"'; + strbe = '[' bechar* ']'; + + strU = uchar0 uchar*; + strQ = strq | strd | strb; + strE = strqe | strde | strbe; + + main := |* + intu => OnIntU; + intq => OnIntQ; + + strU => OnStrU; + strQ => OnStrQ; + strE => OnStrE; + + '/'; + + (intu) (any - ('/' | '[' )) => OnError; + + any => OnError; + *|; + }%% +#endif + + template <typename TSelector> + bool TSelectorCtx<TSelector>::SelectPath() { + try { + %%{ + write data noerror nofinal; + write init; + write exec; + }%% + ; + Y_UNUSED(schemeselect_en_main); + } catch (const TFromStringException&) { + return OnError(); + } + + return Selector.Current; + + TOKEN_ERROR: + return OnError(); + } + + template <bool CheckHas> + struct TGetNext { + template <typename TValue, typename TIdx> + static TValue* Next(TValue* val, TIdx idx) { + if (val) { + if (CheckHas && !val->Has(idx)) { + return nullptr; + } else { + return &(*val)[idx]; + } + } else { + return nullptr; + } + } + }; + + const TValue& TValue::TrySelect(TStringBuf path) const { + TSelectorCtx<TSelector<const TValue, TGetNext<true> > > ctx(*this, path); + + if (ctx.SelectPath()) { + return *ctx.Selector.Current; + } + + return DefaultValue(); + } + + TValue* TValue::TrySelectOrAdd(TStringBuf path) { + TSelectorCtx<TSelector<TValue, TGetNext<false> > > ctx(*this, path); + + if (ctx.SelectPath()) { + return ctx.Selector.Current; + } else { + return nullptr; + } + } + + TValue TValue::TrySelectAndDelete(TStringBuf path) { + TSelectorCtx<TSelector<TValue, TGetNext<true> > > ctx(*this, path); + + if (ctx.SelectPath() && ctx.Selector.Parent) { + if (ctx.Selector.Parent->IsArray()) { + return ctx.Selector.Parent->Delete(ctx.Selector.CurrentArrayKey); + } else if (ctx.Selector.Parent->IsDict()) { + return ctx.Selector.Parent->Delete(ctx.Selector.CurrentDictKey); + } else { + Y_ASSERT(false); + return DefaultValue(); + } + } else { + return DefaultValue(); + } + } + + bool TValue::PathExists(TStringBuf path) const { + return TSelectorCtx<TSelector<const TValue, TGetNext<true>>>(*this, path).SelectPath(); + } + + bool TValue::PathValid(TStringBuf path) { + TSelectorCtx<TSelector<const TValue, TGetNext<false>>> ctx(DefaultValue(), path); + return ctx.SelectPath() || !ctx.Selector.HasError; + } + + TString TValue::EscapeForPath(TStringBuf rawKey) { + static const str_spn danger{"/[]"}; + if (!rawKey || danger.brk(rawKey.begin(), rawKey.end()) != rawKey.end()) { + return NEscJ::EscapeJ<true>(rawKey); + } + + TSelectorCtx<TSelector<const TValue, TGetNext<false>>> ctx(DefaultValue(), rawKey); + ctx.SelectPath(); + if (ctx.Selector.HasError || ctx.Selector.Depth > 1 || ctx.Selector.HasArray) { + return NEscJ::EscapeJ<true>(rawKey); + } else { + return ToString(rawKey); + } + } + +} diff --git a/library/cpp/scheme/tests/fuzz_json/fuzz_json.cpp b/library/cpp/scheme/tests/fuzz_json/fuzz_json.cpp new file mode 100644 index 00000000000..8d4c0fa8a08 --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_json/fuzz_json.cpp @@ -0,0 +1,6 @@ +#include <library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.h> + +extern "C" int LLVMFuzzerTestOneInput(const ui8* wireData, const size_t wireSize) { + NSc::NUt::FuzzJson({(const char*)wireData, wireSize}); + return 0; +} diff --git a/library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.cpp b/library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.cpp new file mode 100644 index 00000000000..7c16527c23f --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.cpp @@ -0,0 +1,115 @@ +#include "fuzz_json.h" +#include "util/generic/fwd.h" + +#include <library/cpp/scheme/scheme.h> +#include <util/stream/null.h> + +namespace { + static constexpr size_t MAX_DEPTH = 4; + static constexpr size_t MAX_PATH_LEN = 256; + static constexpr size_t MAX_ITERATIONS = 4; + + void SplitOnDepth(const TStringBuf src, const size_t depth, const size_t maxPathLen, + TStringBuf& left, TStringBuf& right) + { + size_t pos = 0; + size_t prevPos = 0; + for(size_t i = 0; i < depth; ++i) { + if (pos > maxPathLen) { + break; + } + prevPos = pos; + pos = src.find_first_of(TStringBuf("/]"), pos + 1); + if (pos == TStringBuf::npos) { + break; + } + } + if (pos == TStringBuf::npos && prevPos > 0) { + pos = prevPos; + } + if (src.length() > maxPathLen) { + if (pos == TStringBuf::npos || pos > maxPathLen) { + pos = maxPathLen; + } + } + if (pos == TStringBuf::npos || pos == 0) { + left = src; + right = TStringBuf(); + } else { + src.SplitAt(pos + 1, left, right); + } + } + + TString tmp; + //Limit max array size in the path to 256 + TStringBuf ProcessPath(TStringBuf path) { + size_t pos = 0; + while(pos != TStringBuf::npos) { + pos = path.find(']', pos + 1); + if (pos == TStringBuf::npos) { + continue; + } + size_t open = path.rfind('[', pos); + if (open == TStringBuf::npos) { + continue; + } + bool allDigit = true; + for(size_t i = open + 1; i < pos; ++i) { + if (path[i] < '0' || path[i] > '9') { + allDigit = false; + break; + } + } + if (!allDigit) { + continue; + } + if (pos - open > 4) { + TString str = TString::Join(path.Head(open + 1), "256", path.Tail(pos)); + tmp = std::move(str); + path = tmp; + pos = (open + 1) + 3; + continue; + } + } + return path; + } +} + +namespace NSc::NUt { + + + void FuzzJson(TStringBuf wire) { + if (wire.size() < 2) { + return; + } + + + ProcessPath("[123][1234][12][2134][12312312][1][12]"); + ui8 len1 = wire[0]; + ui8 len2 = wire[1]; + wire.Skip(2); + auto json1 = wire.NextTokAt(len1); + auto json2 = wire.NextTokAt(len2); + NSc::TValue val1 = NSc::TValue::FromJson(json1); + NSc::TValue val2 = NSc::TValue::FromJson(json2); + NSc::TValue val3; + val3.MergeUpdate(val1); + + size_t i = 0; + while (!wire.empty()) { + TStringBuf path; + SplitOnDepth(wire, MAX_DEPTH, MAX_PATH_LEN, path, wire); + path = ProcessPath(path); + if (auto* target = val3.TrySelectOrAdd(path)) { + target->MergeUpdate(val2); + } + ++i; + // Release memory since there are up to MAX_DICT_SIZE * MAX_DEPTH elements + if (i > MAX_ITERATIONS) { + Cnull << val3.ToJson(); + val3 = NSc::TValue(); + } + } + Cnull << val3.ToJson(); + } +} diff --git a/library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.h b/library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.h new file mode 100644 index 00000000000..f8cf7a47708 --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.h @@ -0,0 +1,7 @@ +#pragma once + +#include <util/generic/strbuf.h> + +namespace NSc::NUt { + void FuzzJson(TStringBuf wire); +} diff --git a/library/cpp/scheme/tests/fuzz_json/lib/ya.make b/library/cpp/scheme/tests/fuzz_json/lib/ya.make new file mode 100644 index 00000000000..b30a6c93505 --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_json/lib/ya.make @@ -0,0 +1,18 @@ +LIBRARY() + +OWNER( + g:blender + g:middle + g:upper + velavokr +) + +SRCS( + fuzz_json.cpp +) + +PEERDIR( + library/cpp/scheme +) + +END() diff --git a/library/cpp/scheme/tests/fuzz_json/ya.make b/library/cpp/scheme/tests/fuzz_json/ya.make new file mode 100644 index 00000000000..0d91c70585d --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_json/ya.make @@ -0,0 +1,20 @@ +FUZZ() + +OWNER( + g:blender + g:middle + g:upper + velavokr +) + +SIZE(MEDIUM) + +SRCS( + fuzz_json.cpp +) + +PEERDIR( + library/cpp/scheme/tests/fuzz_json/lib +) + +END() diff --git a/library/cpp/scheme/tests/fuzz_ops/fuzz_ops.cpp b/library/cpp/scheme/tests/fuzz_ops/fuzz_ops.cpp new file mode 100644 index 00000000000..facde50f5aa --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_ops/fuzz_ops.cpp @@ -0,0 +1,6 @@ +#include <library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.h> + +extern "C" int LLVMFuzzerTestOneInput(const ui8* wireData, const size_t wireSize) { + NSc::NUt::FuzzOps({(const char*)wireData, wireSize}, false); + return 0; +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.cpp b/library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.cpp new file mode 100644 index 00000000000..8a7facba246 --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.cpp @@ -0,0 +1,37 @@ +#include "fuzz_ops.h" +#include "vm_apply.h" +#include "vm_defs.h" +#include "vm_parse.h" + +#include <library/cpp/bit_io/bitinput.h> + +#include <library/cpp/scheme/scheme.h> +#include <library/cpp/scheme/scimpl_private.h> + +#include <util/generic/maybe.h> + +namespace NSc::NUt { + + void FuzzOps(TStringBuf wire, bool log) { + if (log) { + NImpl::GetTlsInstance<NImpl::TSelfLoopContext>().ReportingMode = NImpl::TSelfLoopContext::EMode::Stderr; + } + + // We start with a single TValue node + TVMState st {wire, 1, 0}; + + while (auto act = ParseNextAction(st)) { + if (log) { + Cerr << " STATE: " << st.ToString() << Endl; + Cerr << "ACTION: " << (act ? act->ToString() : TString("(empty)")) << Endl; + } + + if (!ApplyNextAction(st, *act)) { + break; + } + if (!NSc::TValue::DefaultValue().IsNull()) { + std::terminate(); + } + } + } +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.h b/library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.h new file mode 100644 index 00000000000..26ba42eaef1 --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.h @@ -0,0 +1,7 @@ +#pragma once + +#include <util/generic/strbuf.h> + +namespace NSc::NUt { + void FuzzOps(TStringBuf wire, bool log); +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/vm_apply.cpp b/library/cpp/scheme/tests/fuzz_ops/lib/vm_apply.cpp new file mode 100644 index 00000000000..ada7b8854f4 --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_ops/lib/vm_apply.cpp @@ -0,0 +1,302 @@ +#include "vm_apply.h" + +namespace NSc::NUt { + +#define Y_GEN_TRY_OP(op) \ + if (!op) { return false; } + +#define Y_GEN_SRC_OP(op, arg, st, act) \ + switch (act.GetSrc(arg).Type) { \ + case TSrc::T_LREF__POS: \ + op(st.LRef(act.GetSrc(arg).Pos)); \ + break; \ + case TSrc::T_CREF__POS: \ + op(st.CRef(act.GetSrc(arg).Pos)); \ + break; \ + case TSrc::T_RREF__POS: \ + op(st.RRef(act.GetSrc(arg).Pos)); \ + break; \ + default: \ + Y_FAIL(); \ + } + + +#define Y_GEN_SRC_TRY_OP(op, arg, st, act) \ + if (st.CRef(act.GetSrc(arg).Pos).IsSameOrAncestorOf(st.Current())) { \ + return false; \ + } \ + Y_GEN_SRC_OP(op, arg, st, act); + + +#define Y_GEN_DST_OP(op, arg, st, act) \ + switch (act.GetDst(arg).Type) { \ + case TDst::T_CREATE_BACK_LREF: \ + Y_GEN_TRY_OP(st.TryPushBack(op)) \ + break; \ + case TDst::T_CREATE_BACK_CREF: \ + Y_GEN_TRY_OP(st.TryPushBack(std::as_const(op))) \ + break; \ + case TDst::T_CREATE_BACK_RREF: \ + Y_GEN_TRY_OP(st.TryPushBack(std::move(op))) \ + break; \ + case TDst::T_CREATE_FRONT_LREF: \ + Y_GEN_TRY_OP(st.TryPushFront(op)) \ + break; \ + case TDst::T_CREATE_FRONT_CREF: \ + Y_GEN_TRY_OP(st.TryPushFront(std::as_const(op))) \ + break; \ + case TDst::T_CREATE_FRONT_RREF: \ + Y_GEN_TRY_OP(st.TryPushFront(std::move(op))) \ + break; \ + case TDst::T_LREF__POS: \ + st.LRef(act.GetDst(arg).Pos) = op; \ + break; \ + case TDst::T_CREF__POS: \ + st.LRef(act.GetDst(arg).Pos) = std::as_const(op); \ + break; \ + case TDst::T_RREF__POS: \ + st.LRef(act.GetDst(arg).Pos) = std::move(op); \ + break; \ + default: \ + Y_FAIL(); \ + } + + +#define Y_GEN_REF_OP(op, arg, st, act) \ + switch (act.GetRef(arg).Type) { \ + case TRef::T_CREATE_BACK: \ + Y_GEN_TRY_OP(st.TryPushBack(op)) \ + break; \ + case TRef::T_CREATE_FRONT: \ + Y_GEN_TRY_OP(st.TryPushFront(op)) \ + break; \ + case TRef::T_REF__POS: \ + st.LRef(act.GetRef(arg).Pos) = op; \ + break; \ + default: \ + Y_FAIL(); \ + } + +#define Y_GEN_PTR_OP(op, arg, st, act) \ + if (auto* r = (op)) { \ + switch (act.GetRef(arg).Type) { \ + case TRef::T_CREATE_BACK: \ + Y_GEN_TRY_OP(st.TryPushBack(*r)) \ + break; \ + case TRef::T_CREATE_FRONT: \ + Y_GEN_TRY_OP(st.TryPushFront(*r)) \ + break; \ + case TRef::T_REF__POS: \ + st.LRef(act.GetRef(arg).Pos) = *r; \ + break; \ + default: \ + Y_FAIL(); \ + } \ + } + + bool ApplyNextAction(TVMState& st, TVMAction act) { + switch (act.Type) { + case VMA_JMP__POS: + st.Pos = act.GetPos(0); + return true; + + case VMA_CREATE_BACK: + Y_GEN_TRY_OP(st.TryPushBack()) + return true; + + case VMA_CREATE_BACK__SRC: + switch (act.GetSrc(0).Type) { + case TSrc::T_LREF__POS: + Y_GEN_TRY_OP(st.TryPushBack(st.LRef(act.GetSrc(0).Pos))) + break; + case TSrc::T_CREF__POS: + Y_GEN_TRY_OP(st.TryPushBack(st.CRef(act.GetSrc(0).Pos))) + break; + case TSrc::T_RREF__POS: + Y_GEN_TRY_OP(st.TryPushBack(st.RRef(act.GetSrc(0).Pos))) + break; + default: + Y_FAIL(); + } + + return true; + + case VMA_CREATE_FRONT: + Y_GEN_TRY_OP(st.TryPushFront()) + return true; + + case VMA_CREATE_FRONT__SRC: + switch (act.GetSrc(0).Type) { + case TSrc::T_LREF__POS: + Y_GEN_TRY_OP(st.TryPushFront(st.LRef(act.GetSrc(0).Pos))) + break; + case TSrc::T_CREF__POS: + Y_GEN_TRY_OP(st.TryPushFront(st.CRef(act.GetSrc(0).Pos))) + break; + case TSrc::T_RREF__POS: + Y_GEN_TRY_OP(st.TryPushFront(st.RRef(act.GetSrc(0).Pos))) + break; + default: + Y_FAIL(); + } + return true; + + case VMA_DESTROY_BACK: + return st.TryPopBack(); + + case VMA_DESTROY_FRONT: + return st.TryPopFront(); + + case VMA_ASSIGN__SRC: + Y_GEN_SRC_OP(st.Current() = , 0, st, act); + return true; + + case VMA_SET_STRING__IDX: + st.Current().SetString(act.GetString(0)); + return true; + + case VMA_SET_INT_NUMBER__IDX: + st.Current().SetIntNumber(act.GetIntNumber(0)); + return true; + + case VMA_SET_DICT: + st.Current().SetDict(); + return true; + + case VMA_SET_ARRAY: + st.Current().SetArray(); + return true; + + case VMA_SET_NULL: + st.Current().SetNull(); + return true; + + case VMA_GET_JSON: + st.Current().ToJson(); + return true; + + case VMA_ARRAY_CLEAR: + st.Current().ClearArray(); + return true; + + case VMA_ARRAY_PUSH: + st.Current().Push(); + return true; + + case VMA_ARRAY_PUSH__SRC: + Y_GEN_SRC_TRY_OP(st.Current().Push, 0, st, act); + return true; + + case VMA_ARRAY_PUSH__DST: + Y_GEN_DST_OP(st.Current().Push(), 0, st, act); + return true; + + case VMA_ARRAY_POP__REF: + Y_GEN_REF_OP(st.Current().Pop(), 0, st, act); + return true; + + case VMA_ARRAY_INSERT__IDX: + st.Current().Insert(act.GetIdx(0)); + return true; + + case VMA_ARRAY_INSERT__IDX_SRC: + Y_GEN_SRC_TRY_OP(st.Current().Insert(act.GetIdx(0)) = , 1, st, act); + return true; + + case VMA_ARRAY_INSERT__IDX_DST: + Y_GEN_DST_OP(st.Current().Insert(act.GetIdx(0)), 1, st, act); + return true; + + case VMA_ARRAY_DELETE__IDX_REF: + Y_GEN_REF_OP(st.Current().Delete(act.GetIdx(0)), 1, st, act); + return true; + + case VMA_ARRAY_GET__IDX_REF: + Y_GEN_REF_OP(st.Current().Get(act.GetIdx(0)), 1, st, act); + return true; + + case VMA_ARRAY_GET_OR_ADD__IDX: + st.Current().GetOrAdd(act.GetIdx(0)); + return true; + + case VMA_ARRAY_GET_OR_ADD__IDX_SRC: + Y_GEN_SRC_TRY_OP(st.Current().GetOrAdd(act.GetIdx(0)) = , 1, st, act); + return true; + + case VMA_ARRAY_GET_OR_ADD__IDX_DST: + Y_GEN_DST_OP(st.Current().GetOrAdd(act.GetIdx(0)), 1, st, act); + return true; + + case VMA_ARRAY_GET_NO_ADD__IDX_REF: + Y_GEN_PTR_OP(st.Current().GetNoAdd(act.GetIdx(0)), 1, st, act); + return true; + + case VMA_DICT_CLEAR: + st.Current().ClearDict(); + return true; + + case VMA_DICT_DELETE__IDX: + st.Current().Delete(act.GetKey(0)); + return true; + + case VMA_DICT_DELETE__IDX_REF: + Y_GEN_REF_OP(st.Current().Delete(act.GetKey(0)), 1, st, act); + return true; + + case VMA_DICT_GET__IDX_REF: + Y_GEN_REF_OP(st.Current().Get(act.GetKey(0)), 1, st, act); + return true; + + case VMA_DICT_GET_OR_ADD__IDX: + st.Current().GetOrAdd(act.GetKey(0)); + return true; + + case VMA_DICT_GET_OR_ADD__IDX_SRC: + Y_GEN_SRC_TRY_OP(st.Current().GetOrAdd(act.GetKey(0)) = , 1, st, act); + return true; + + case VMA_DICT_GET_OR_ADD__IDX_DST: + Y_GEN_DST_OP(st.Current().GetOrAdd(act.GetKey(0)), 1, st, act); + return true; + + case VMA_DICT_GET_NO_ADD__IDX_REF: + Y_GEN_PTR_OP(st.Current().GetNoAdd(act.GetKey(0)), 1, st, act); + return true; + + case VMA_MERGE_UPDATE__POS: + st.Current().MergeUpdate(st.LRef(act.GetPos(0))); + return true; + + case VMA_MERGE_REVERSE__POS: + st.Current().ReverseMerge(st.LRef(act.GetPos(0))); + return true; + + case VMA_MERGE_COPY_FROM__POS: + st.Current().CopyFrom(st.LRef(act.GetPos(0))); + return true; + + case VMA_SWAP__POS: + st.Current().Swap(st.LRef(act.GetPos(0))); + return true; + + case VMA_EQUAL__POS_POS: + TValue::Equal(st.CRef(act.GetPos(0)), st.CRef(act.GetPos(1))); + return true; + + case VMA_SELECT_NO_ADD__PATH_REF: + Y_GEN_REF_OP(st.Current().TrySelect(act.GetPath(0)), 1, st, act); + return true; + + case VMA_SELECT_OR_ADD__PATH_REF: + Y_GEN_PTR_OP(st.Current().TrySelectOrAdd(act.GetPath(0)), 1, st, act); + return true; + + case VMA_SELECT_AND_DELETE__PATH_REF: + Y_GEN_REF_OP(st.Current().TrySelectAndDelete(act.GetPath(0)), 1, st, act); + return true; + + default: + Y_FAIL(); + } + } +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/vm_apply.h b/library/cpp/scheme/tests/fuzz_ops/lib/vm_apply.h new file mode 100644 index 00000000000..82906b53fb2 --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_ops/lib/vm_apply.h @@ -0,0 +1,9 @@ +#pragma once + +#include "vm_defs.h" + +namespace NSc::NUt { + + bool ApplyNextAction(TVMState& st, TVMAction act); + +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/vm_defs.cpp b/library/cpp/scheme/tests/fuzz_ops/lib/vm_defs.cpp new file mode 100644 index 00000000000..55a971d9e4f --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_ops/lib/vm_defs.cpp @@ -0,0 +1,168 @@ +#include "vm_defs.h" + +#include <util/generic/xrange.h> +#include <util/string/builder.h> + +namespace NSc::NUt { + namespace { + TStringBuf GetStringByIdx(ui32 idx) { + static const TStringBuf strings[TIdx::ValueCount] {{}, "a", "aa", "aaa"}; + return strings[idx % TIdx::ValueCount]; + } + } + + + TVMState::TVMState(TStringBuf wire, ui32 memSz, ui32 pos) + : Input(wire) + , Memory(memSz) + , Pos(pos) + {} + + bool TVMState::TryPushBack() { + if (MayAddMoreMemory()) { + Memory.emplace_back(); + return true; + } else { + return false; + } + } + + bool TVMState::TryPushFront() { + if (MayAddMoreMemory()) { + Pos += 1; + Memory.emplace_front(); + return true; + } else { + return false; + } + } + + bool TVMState::TryPopBack() { + if (Memory.size() > 1 && Pos < Memory.size() - 1) { + Memory.pop_back(); + return true; + } else { + return false; + } + } + + bool TVMState::TryPopFront() { + if (Memory.size() > 1 && Pos > 0) { + Memory.pop_front(); + Pos -= 1; + return true; + } else { + return false; + } + } + + TString TVMState::ToString() const { + TStringBuilder b; + b << "pos=" << Pos << ";"; + for (const auto i : xrange(Memory.size())) { + b << " " << i << (i == Pos ? "@" : ".") << Memory[i].ToJson().Quote(); + } + + return b; + } + + TString TIdx::ToString() const { + return TStringBuilder() << "IDX:" << Idx; + } + + TString TPos::ToString() const { + return TStringBuilder() << "POS:" << Pos; + } + + template <class T> + TString RenderToString(TStringBuf name, T type, ui32 pos) { + TStringBuilder b; + b << name << ":" << type; + if ((ui32)-1 != pos) { + b << "," << pos; + } + return b; + } + + TString TRef::ToString() const { + return RenderToString("REF", Type, Pos); + } + + TString TSrc::ToString() const { + return RenderToString("SRC", Type, Pos); + } + + TString TDst::ToString() const { + return RenderToString("DST", Type, Pos); + } + + TString TPath::ToString() const { + return TStringBuilder() << "PATH:" << Path.Quote(); + } + + + TRef TVMAction::GetRef(ui32 arg) const noexcept { + return std::get<TRef>(Arg[arg]); + } + + TSrc TVMAction::GetSrc(ui32 arg) const noexcept { + return std::get<TSrc>(Arg[arg]); + } + + TDst TVMAction::GetDst(ui32 arg) const noexcept { + return std::get<TDst>(Arg[arg]); + } + + ui32 TVMAction::GetPos(ui32 arg) const noexcept { + return std::get<TPos>(Arg[arg]).Pos; + } + + ui32 TVMAction::GetIdx(ui32 arg) const noexcept { + return std::get<TIdx>(Arg[arg]).Idx; + } + + TStringBuf TVMAction::GetKey(ui32 arg) const noexcept { + return GetString(arg); + } + + TStringBuf TVMAction::GetString(ui32 arg) const noexcept { + return GetStringByIdx(GetIdx(arg)); + } + + i64 TVMAction::GetIntNumber(ui32 arg) const noexcept { + return GetIdx(arg); + } + + TStringBuf TVMAction::GetPath(ui32 arg) const noexcept { + return std::get<TPath>(Arg[arg]).Path; + } + + + struct TActionPrinter { + TActionPrinter(IOutputStream& out) + : Out(out) + {} + + bool operator()(const std::monostate&) const { + return true; + } + //TIdx, TPos, TRef, TSrc, TDst, TPath + template <class T> + bool operator()(const T& t) const { + Out << "; " << t.ToString(); + return false; + } + IOutputStream& Out; + }; + + TString TVMAction::ToString() const { + TStringBuilder out; + out << Type; + for (const auto& arg : Arg) { + if (std::visit(TActionPrinter(out.Out), arg)) { + break; + } + } + return out; + } +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/vm_defs.h b/library/cpp/scheme/tests/fuzz_ops/lib/vm_defs.h new file mode 100644 index 00000000000..9a0ddf7351a --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_ops/lib/vm_defs.h @@ -0,0 +1,277 @@ +#pragma once + +#include <library/cpp/bit_io/bitinput.h> +#include <library/cpp/scheme/scheme.h> + +#include <util/generic/deque.h> +#include <util/generic/maybe.h> +#include <util/generic/variant.h> +#include <util/string/builder.h> + +namespace NSc::NUt { + + enum EVMAction { + VMA_JMP__POS /* "pos=POS" */, + + VMA_CREATE_BACK /* "TValue()->emplace_back" */, + VMA_CREATE_BACK__SRC /* "TValue(SRC)->emplace_back" */, + VMA_CREATE_FRONT /* "TValue()->emplace_front,pos+=1" */, + VMA_CREATE_FRONT__SRC /* "TValue(SRC)->emplace_front,pos+=1" */, + + VMA_DESTROY_BACK /* "pop_back" */, + VMA_DESTROY_FRONT /* "pop_front,pos-=1" */, + + VMA_ASSIGN__SRC /* "operator=(SRC)" */, + + VMA_SET_STRING__IDX /* "SetString(str[IDX])" */, + VMA_SET_INT_NUMBER__IDX /* "SetIntNumber(IDX)" */, + VMA_SET_DICT /* "SetDict()" */, + VMA_SET_ARRAY /* "SetArray()" */, + VMA_SET_NULL /* "SetNull()" */, + + VMA_GET_JSON /* "ToJson()" */, + + VMA_ARRAY_CLEAR /* "ClearArray()" */, + VMA_ARRAY_PUSH /* "Push()" */, + VMA_ARRAY_PUSH__SRC /* "Push()=SRC" */, + VMA_ARRAY_PUSH__DST /* "Push()->DST" */, + VMA_ARRAY_POP /* "Pop()" */, + VMA_ARRAY_POP__REF /* "Pop()->REF" */, + VMA_ARRAY_INSERT__IDX /* "Insert(IDX)" */, + VMA_ARRAY_INSERT__IDX_SRC /* "Insert(IDX)=SRC" */, + VMA_ARRAY_INSERT__IDX_DST /* "Insert(IDX)->DST" */, + VMA_ARRAY_DELETE__IDX /* "Delete(IDX)" */, + VMA_ARRAY_DELETE__IDX_REF /* "Delete(IDX)->REF" */, + VMA_ARRAY_GET__IDX_REF /* "Get(IDX)->REF" */, + VMA_ARRAY_GET_OR_ADD__IDX /* "GetOrAdd(IDX)" */, + VMA_ARRAY_GET_OR_ADD__IDX_SRC /* "GetOrAdd(IDX)=SRC" */, + VMA_ARRAY_GET_OR_ADD__IDX_DST /* "GetOrAdd(IDX)->DST" */, + VMA_ARRAY_GET_NO_ADD__IDX_REF /* "GetNoAdd(IDX)->REF" */, + + VMA_DICT_CLEAR /* "ClearDict()" */, + VMA_DICT_DELETE__IDX /* "Delete(str[IDX])" */, + VMA_DICT_DELETE__IDX_REF /* "Delete(str[IDX])->REF" */, + VMA_DICT_GET__IDX_REF /* "Get(str[IDX])->REF" */, + VMA_DICT_GET_OR_ADD__IDX /* "GetOrAdd(str[IDX])" */, + VMA_DICT_GET_OR_ADD__IDX_SRC /* "GetOrAdd(str[IDX])=SRC" */, + VMA_DICT_GET_OR_ADD__IDX_DST /* "GetOrAdd(str[IDX])->DST" */, + VMA_DICT_GET_NO_ADD__IDX_REF /* "GetNoAdd(str[IDX])->REF" */, + + VMA_MERGE_UPDATE__POS /* "MergeUpdate(POS)" */, + VMA_MERGE_REVERSE__POS /* "ReverseMerge(POS)" */, + VMA_MERGE_COPY_FROM__POS /* "CopyFrom(POS)" */, + + VMA_SWAP__POS /* "Swap(POS)" */, + VMA_EQUAL__POS_POS /* "Equal(POS, POS)" */, + + VMA_SELECT_NO_ADD__PATH_REF /* "TrySelect(PATH)->REF" */, + VMA_SELECT_OR_ADD__PATH_REF /* "TrySelectOrAdd(PATH)->REF" */, + VMA_SELECT_AND_DELETE__PATH_REF /* "TrySelectAndDelete(PATH)->REF" */, + + VMA_COUNT, + }; + + + struct TVMState { + NBitIO::TBitInput Input; + TDeque<TValue> Memory { 1 }; + ui32 Pos = 0; + + static const ui32 MaxMemory = 16; + + public: + explicit TVMState(TStringBuf wire, ui32 memSz, ui32 pos); + + TValue& Current() { + return Memory[Pos]; + } + + TValue& LRef(ui32 pos) { + return Memory[pos]; + } + + const TValue& CRef(ui32 pos) { + return Memory[pos]; + } + + TValue&& RRef(ui32 pos) { + return std::move(Memory[pos]); + } + + [[nodiscard]] + bool TryPushBack(); + + template <class T> + [[nodiscard]] + bool TryPushBack(T&& t) { + if (MayAddMoreMemory()) { + Memory.emplace_back(std::forward<T>(t)); + return true; + } else { + return false; + } + } + + [[nodiscard]] + bool TryPushFront(); + + template <class T> + [[nodiscard]] + bool TryPushFront(T&& t) { + if (MayAddMoreMemory()) { + Pos += 1; + Memory.emplace_front(std::forward<T>(t)); + return true; + } else { + return false; + } + } + + [[nodiscard]] + bool TryPopBack(); + + [[nodiscard]] + bool TryPopFront(); + + TString ToString() const; + + private: + [[nodiscard]] + bool MayAddMoreMemory() const noexcept { + return Memory.size() < MaxMemory; + } + }; + + + struct TIdx { + ui32 Idx = -1; + static const ui32 ValueCount = 4; + + public: + TString ToString() const; + }; + + + struct TPos { + ui32 Pos = -1; + static const ui32 ValueCount = TVMState::MaxMemory; + + public: + TString ToString() const; + }; + + + struct TRef : public TPos { + enum EType { + T_CREATE_FRONT /* "emplace_front(RES),pos+=1" */, + T_CREATE_BACK /* "emplace_back(RES)" */, + T_REF__POS /* "pos@(RES)" */, + T_COUNT + }; + + EType Type = T_COUNT; + static const ui32 TypeCount = T_COUNT; + + public: + TString ToString() const; + }; + + + struct TSrc : public TPos { + enum EType { + T_LREF__POS /* "pos@(TValue&)" */, + T_CREF__POS /* "pos@(const TValue&)" */, + T_RREF__POS /* "pos@(TValue&&)" */, + T_COUNT + }; + + EType Type = T_COUNT; + static const ui32 TypeCount = T_COUNT; + + public: + TString ToString() const; + }; + + + struct TDst : public TPos { + enum EType { + T_CREATE_FRONT_LREF /* "emplace_front(TValue&),pos+=1" */, + T_CREATE_FRONT_CREF /* "emplace_front(const TValue&),pos+=1" */, + T_CREATE_FRONT_RREF /* "emplace_front(TValue&&),pos+=1" */, + T_CREATE_BACK_LREF /* "emplace_back(TValue&)" */, + T_CREATE_BACK_CREF /* "emplace_back(TValue&),pos+=1" */, + T_CREATE_BACK_RREF /* "emplace_back(TValue&),pos+=1" */, + T_LREF__POS /* "pos@(TValue&)" */, + T_CREF__POS /* "pos@(const TValue&)" */, + T_RREF__POS /* "pos@(TValue&&)" */, + T_COUNT + }; + + EType Type = T_COUNT; + static const ui32 TypeCount = T_COUNT; + + public: + TString ToString() const; + }; + + + struct TPath { + TString Path; + static const ui32 MaxLength = 32; + + public: + TString ToString() const; + }; + + using TArg = std::variant<std::monostate, TIdx, TPos, TRef, TSrc, TDst, TPath>; + + struct TVMAction { + using EType = EVMAction; + EVMAction Type = VMA_COUNT; + static const ui32 TypeCount = VMA_COUNT; + + public: + template <class T> + bool SetArg(TMaybe<T> arg) { + Y_VERIFY(CurrArg < Y_ARRAY_SIZE(Arg)); + if (arg) { + Arg[CurrArg++] = *arg; + return true; + } else { + return false; + } + } + + public: + TRef GetRef(ui32 arg) const noexcept; + + TSrc GetSrc(ui32 arg) const noexcept; + + TDst GetDst(ui32 arg) const noexcept; + + + ui32 GetPos(ui32 arg) const noexcept; + + ui32 GetIdx(ui32 arg) const noexcept; + + TStringBuf GetKey(ui32 arg) const noexcept; + + TStringBuf GetString(ui32 arg) const noexcept; + + i64 GetIntNumber(ui32 arg) const noexcept; + + TStringBuf GetPath(ui32 arg) const noexcept; + + TString ToString() const; + + private: + TArg Arg[2]; + ui32 CurrArg = 0; + }; + + [[nodiscard]] + inline ui32 GetCountWidth(ui32 cnt) { + return MostSignificantBit(cnt - 1) + 1; + } + +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.cpp b/library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.cpp new file mode 100644 index 00000000000..a03f5aef534 --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.cpp @@ -0,0 +1,265 @@ +#include "vm_parse.h" + +namespace NSc::NUt { +#define Y_TRY_READ_BITS(state, out, bits, res) do { if (!state.Input.Read(out, bits)) { return res; } } while (false) +#define Y_TRY_READ_BITS_BOOL(state, out, bits) Y_TRY_READ_BITS(state, out, bits, false) +#define Y_TRY_READ_BITS_MAYBE(state, out, bits) Y_TRY_READ_BITS(state, out, bits, Nothing()) + + TMaybe<TIdx> ParseIdx(TVMState& st) { + static_assert(IsPowerOf2(TIdx::ValueCount)); + static const auto bits = GetCountWidth(TIdx::ValueCount); + + TIdx idx; + if (!st.Input.Read(idx.Idx, bits)) { + return Nothing(); + } + return idx; + } + + namespace { + bool DoParsePos(TVMState& st, TPos& pos) { + static const auto bits = GetCountWidth(TPos::ValueCount); + const ui32 sz = st.Memory.size(); + + ui32 neg = -1; + Y_TRY_READ_BITS_BOOL(st, neg, 1); + + ui32 delta = -1; + Y_TRY_READ_BITS_BOOL(st, delta, bits); + + if (neg) { + if (st.Pos < delta) { + return false; + } + pos.Pos = st.Pos - delta; + } else { + if (st.Pos + delta >= sz) { + return false; + } + pos.Pos = st.Pos + delta; + } + + return true; + } + + template <class T> + bool DoParseType(TVMState& state, T& res) { + static const auto bits = GetCountWidth(T::TypeCount); + + ui32 type = -1; + if (state.Input.Read(type, bits) && type < T::TypeCount) { + res.Type = (typename T::EType)(type); + return true; + } else { + return false; + } + } + } + + TMaybe<TPos> ParsePos(TVMState& state) { + TPos res; + if (DoParsePos(state, res)) { + return res; + } + return Nothing(); + } + + TMaybe<TRef> ParseRef(TVMState& state) { + TRef ref; + if (!DoParseType(state, ref)) { + return Nothing(); + } + + switch (ref.Type) { + case TRef::T_REF__POS: + if (!DoParsePos(state, ref)) { + return Nothing(); + } + [[fallthrough]]; + case TRef::T_CREATE_FRONT: + case TRef::T_CREATE_BACK: + return ref; + default: + Y_FAIL(); + } + } + + TMaybe<TSrc> ParseSrc(TVMState& state) { + TSrc src; + if (!DoParseType(state, src)) { + return Nothing(); + } + + switch (src.Type) { + case TSrc::T_LREF__POS: + case TSrc::T_CREF__POS: + case TSrc::T_RREF__POS: + if (!DoParsePos(state, src)) { + return Nothing(); + } + return src; + default: + Y_FAIL(); + } + } + + TMaybe<TDst> ParseDst(TVMState& state) { + TDst dst; + if (!DoParseType(state, dst)) { + return Nothing(); + } + + switch (dst.Type) { + case TDst::T_LREF__POS: + case TDst::T_CREF__POS: + case TDst::T_RREF__POS: + if (!DoParsePos(state, dst)) { + return Nothing(); + } + [[fallthrough]]; + case TDst::T_CREATE_FRONT_LREF: + case TDst::T_CREATE_FRONT_CREF: + case TDst::T_CREATE_FRONT_RREF: + case TDst::T_CREATE_BACK_LREF: + case TDst::T_CREATE_BACK_CREF: + case TDst::T_CREATE_BACK_RREF: + return dst; + default: + Y_FAIL(); + } + } + + TMaybe<TPath> ParsePath(TVMState& state) { + static const ui32 bits = GetCountWidth(TPath::MaxLength); + TPath path; + + ui32 len = -1; + Y_TRY_READ_BITS_MAYBE(state, len, bits); + while (len--) { + ui8 c; + Y_TRY_READ_BITS_MAYBE(state, c, 8); + path.Path.push_back(c); + } + return path; + } + + TMaybe<TVMAction> ParseNextAction(TVMState& state) { + TVMAction res; + + if (!DoParseType(state, res)) { + return Nothing(); + } + + switch (res.Type) { + case VMA_CREATE_BACK: + case VMA_CREATE_FRONT: + case VMA_DESTROY_BACK: + case VMA_DESTROY_FRONT: + + case VMA_SET_DICT: + case VMA_SET_ARRAY: + case VMA_SET_NULL: + case VMA_GET_JSON: + + case VMA_ARRAY_CLEAR: + case VMA_ARRAY_PUSH: + case VMA_DICT_CLEAR: + return res; + + case VMA_JMP__POS: + case VMA_MERGE_UPDATE__POS: + case VMA_MERGE_REVERSE__POS: + case VMA_MERGE_COPY_FROM__POS: + case VMA_SWAP__POS: + if (res.SetArg(ParsePos(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_ARRAY_POP__REF: + if (res.SetArg(ParseRef(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_SET_STRING__IDX: + case VMA_SET_INT_NUMBER__IDX: + case VMA_ARRAY_INSERT__IDX: + case VMA_ARRAY_GET_OR_ADD__IDX: + case VMA_DICT_GET_OR_ADD__IDX: + if (res.SetArg(ParseIdx(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_CREATE_BACK__SRC: + case VMA_CREATE_FRONT__SRC: + case VMA_ASSIGN__SRC: + case VMA_ARRAY_PUSH__SRC: + if (res.SetArg(ParseSrc(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_ARRAY_PUSH__DST: + if (res.SetArg(ParseDst(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_ARRAY_DELETE__IDX_REF: + case VMA_ARRAY_GET__IDX_REF: + case VMA_ARRAY_GET_NO_ADD__IDX_REF: + case VMA_DICT_DELETE__IDX_REF: + case VMA_DICT_GET__IDX_REF: + case VMA_DICT_GET_NO_ADD__IDX_REF: + if (res.SetArg(ParseIdx(state)) && res.SetArg(ParseRef(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_ARRAY_INSERT__IDX_SRC: + case VMA_ARRAY_GET_OR_ADD__IDX_SRC: + case VMA_DICT_GET_OR_ADD__IDX_SRC: + if (res.SetArg(ParseIdx(state)) && res.SetArg(ParseSrc(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_ARRAY_INSERT__IDX_DST: + case VMA_ARRAY_GET_OR_ADD__IDX_DST: + case VMA_DICT_GET_OR_ADD__IDX_DST: + if (res.SetArg(ParseIdx(state)) && res.SetArg(ParseDst(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_EQUAL__POS_POS: + if (res.SetArg(ParsePos(state)) && res.SetArg(ParsePos(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_SELECT_NO_ADD__PATH_REF: + case VMA_SELECT_OR_ADD__PATH_REF: + case VMA_SELECT_AND_DELETE__PATH_REF: + if (res.SetArg(ParsePath(state)) && res.SetArg(ParseRef(state))) { + return res; + } else { + return Nothing(); + } + + default: + return Nothing(); + } + } +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.h b/library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.h new file mode 100644 index 00000000000..b4aba4e4d4f --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.h @@ -0,0 +1,16 @@ +#pragma once + +#include "vm_defs.h" + +namespace NSc::NUt { + + TMaybe<TIdx> ParseIdx(TVMState& st); + TMaybe<TPos> ParsePos(TVMState& state); + TMaybe<TRef> ParseRef(TVMState& state); + TMaybe<TSrc> ParseSrc(TVMState& state); + TMaybe<TDst> ParseDst(TVMState& state); + TMaybe<TPath> ParsePath(TVMState& state); + + TMaybe<TVMAction> ParseNextAction(TVMState& state); + +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/ya.make b/library/cpp/scheme/tests/fuzz_ops/lib/ya.make new file mode 100644 index 00000000000..279a2ca2d4c --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_ops/lib/ya.make @@ -0,0 +1,23 @@ +LIBRARY() + +OWNER( + g:blender + g:middle + g:upper + velavokr +) + +GENERATE_ENUM_SERIALIZATION(vm_defs.h) + +SRCS( + fuzz_ops.cpp + vm_apply.cpp + vm_defs.cpp + vm_parse.cpp +) + +PEERDIR( + library/cpp/scheme +) + +END() diff --git a/library/cpp/scheme/tests/fuzz_ops/ut/vm_parse_ut.cpp b/library/cpp/scheme/tests/fuzz_ops/ut/vm_parse_ut.cpp new file mode 100644 index 00000000000..ce3786a6714 --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_ops/ut/vm_parse_ut.cpp @@ -0,0 +1,225 @@ +#include <library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.h> +#include <library/cpp/testing/unittest/registar.h> + +Y_UNIT_TEST_SUITE(TestParseNextAction) { + using namespace NSc::NUt; + + Y_UNIT_TEST(TestWidth) { + UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TIdx::ValueCount), 2); + UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TPos::ValueCount), 4); + UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TRef::TypeCount), 2); + UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TSrc::TypeCount), 2); + UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TDst::TypeCount), 4); + UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TPath::MaxLength), 5); + UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TVMAction::TypeCount), 6); + } + + Y_UNIT_TEST(TestParseIdx) { + { + TVMState st{"", 1, 0}; + UNIT_ASSERT(!ParseIdx(st)); + } + { + TVMState st{"\x03", 1, 0}; + auto idx = ParseIdx(st); + UNIT_ASSERT(idx); + UNIT_ASSERT_VALUES_EQUAL(idx->Idx, 3); + } + } + + void DoTestParsePosFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + UNIT_ASSERT(!ParsePos(st)); + } + + [[nodiscard]] + ui32 DoTestParsePosSuccess(TVMState& st) { + const auto pos = ParsePos(st); + UNIT_ASSERT(pos); + return pos->Pos; + } + + [[nodiscard]] + ui32 DoTestParsePosSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + return DoTestParsePosSuccess(st); + } + + Y_UNIT_TEST(TestParsePos) { + DoTestParsePosFailure("", 1, 0); + + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(TStringBuf("\x00"sv), 1, 0), 0); + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(TStringBuf("\x01"sv), 1, 0), 0); + + DoTestParsePosFailure(TStringBuf("\x02"sv), 1, 0); + DoTestParsePosFailure(TStringBuf("\x03"sv), 2, 0); + + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(TStringBuf("\x02"sv), 2, 0), 1); + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(TStringBuf("\x03"sv), 2, 1), 0); + + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(TStringBuf("\x0E"sv), 8, 0), 7); + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(TStringBuf("\x0F"sv), 8, 7), 0); + + { + TVMState st{TStringBuf("\xDE\x7B"), 16, 0}; + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 15); + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 15); + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 15); + UNIT_ASSERT(!ParsePos(st)); + } + { + TVMState st{TStringBuf("\xFF\x7F"), 16, 15}; + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 0); + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 0); + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 0); + UNIT_ASSERT(!ParsePos(st)); + } + } + + void DoTestParseRefFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + UNIT_ASSERT(!ParseRef(st)); + } + + [[nodiscard]] + auto DoTestParseRefSuccess(TVMState& st) { + const auto ref = ParseRef(st); + UNIT_ASSERT(ref); + return std::make_pair(ref->Pos, ref->Type); + } + + [[nodiscard]] + auto DoTestParseRefSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + return DoTestParseRefSuccess(st); + } + + Y_UNIT_TEST(TestParseRef) { + DoTestParseRefFailure("", 1, 0); + + UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(TStringBuf("\x00"sv), 1, 0), std::make_pair((ui32)-1, TRef::T_CREATE_FRONT)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(TStringBuf("\x01"sv), 1, 0), std::make_pair((ui32)-1, TRef::T_CREATE_BACK)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(TStringBuf("\x0A"sv), 2, 0), std::make_pair(1u, TRef::T_REF__POS)); + + DoTestParseRefFailure(TStringBuf("\x12"), 1, 0); + DoTestParseRefFailure(TStringBuf("\x03"sv), 1, 0); + + { + TVMState st{TStringBuf("\x7A\x7D"), 16, 0}; + UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(st), std::make_pair(15u, TRef::T_REF__POS)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(st), std::make_pair(15u, TRef::T_REF__POS)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(st), std::make_pair((ui32)-1, TRef::T_CREATE_BACK)); + UNIT_ASSERT(!ParseRef(st)); + } + } + + void DoTestParseSrcFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + UNIT_ASSERT(!ParseSrc(st)); + } + + [[nodiscard]] + auto DoTestParseSrcSuccess(TVMState& st) { + const auto src = ParseSrc(st); + UNIT_ASSERT(src); + return std::make_pair(src->Pos, src->Type); + } + + [[nodiscard]] + auto DoTestParseSrcSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + return DoTestParseSrcSuccess(st); + } + + Y_UNIT_TEST(TestParseSrc) { + DoTestParseSrcFailure("", 1, 0); + + UNIT_ASSERT_VALUES_EQUAL(DoTestParseSrcSuccess(TStringBuf("\x08"sv), 2, 0), std::make_pair(1u, TSrc::T_LREF__POS)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseSrcSuccess(TStringBuf("\x09"sv), 2, 0), std::make_pair(1u, TSrc::T_CREF__POS)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseSrcSuccess(TStringBuf("\x0A"sv), 2, 0), std::make_pair(1u, TSrc::T_RREF__POS)); + + DoTestParseSrcFailure(TStringBuf("\x03"sv), 1, 0); + + { + TVMState st{TStringBuf("\x7A\x7D"), 16, 0}; + UNIT_ASSERT_VALUES_EQUAL(DoTestParseSrcSuccess(st), std::make_pair(15u, TSrc::T_RREF__POS)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseSrcSuccess(st), std::make_pair(15u, TSrc::T_RREF__POS)); + UNIT_ASSERT(!ParseSrc(st)); + } + } + + void DoTestParseDstFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + UNIT_ASSERT(!ParseDst(st)); + } + + [[nodiscard]] + auto DoTestParseDstSuccess(TVMState& st) { + const auto dst = ParseDst(st); + UNIT_ASSERT(dst); + return std::make_pair(dst->Pos, dst->Type); + } + + [[nodiscard]] + auto DoTestParseDstSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + return DoTestParseDstSuccess(st); + } + + Y_UNIT_TEST(TestParseDst) { + DoTestParseDstFailure("", 1, 0); + + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x00"sv), 1, 0), std::make_pair((ui32)-1, TDst::T_CREATE_FRONT_LREF)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x01"sv), 1, 0), std::make_pair((ui32)-1, TDst::T_CREATE_FRONT_CREF)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x02"sv), 1, 0), std::make_pair((ui32)-1, TDst::T_CREATE_FRONT_RREF)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x03"sv), 1, 0), std::make_pair((ui32)-1, TDst::T_CREATE_BACK_LREF)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x04"sv), 1, 0), std::make_pair((ui32)-1, TDst::T_CREATE_BACK_CREF)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x05"sv), 1, 0), std::make_pair((ui32)-1, TDst::T_CREATE_BACK_RREF)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x26\x00"sv), 2, 0), std::make_pair(1u, TDst::T_LREF__POS)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x27\x00"sv), 2, 0), std::make_pair(1u, TDst::T_CREF__POS)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x28\x00"sv), 2, 0), std::make_pair(1u, TDst::T_RREF__POS)); + + DoTestParseDstFailure(TStringBuf("\x06"sv), 1, 0); + DoTestParseDstFailure(TStringBuf("\x09\x00"sv), 1, 0); + + { + TVMState st{TStringBuf("\x14\xE7\x09"sv), 16, 0}; + // 4=4 + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(st), std::make_pair((ui32)-1, TDst::T_CREATE_BACK_CREF)); + // 4=8 + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(st), std::make_pair((ui32)-1, TDst::T_CREATE_FRONT_CREF)); + // 4+1+4=17 + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(st), std::make_pair(15u, TDst::T_CREF__POS)); + // 4=21 + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(st), std::make_pair((ui32)-1, TDst::T_CREATE_BACK_CREF)); + UNIT_ASSERT(!ParseDst(st)); + } + } + + void DoTestParsePathFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + UNIT_ASSERT(!ParsePath(st)); + } + + [[nodiscard]] + auto DoTestParsePathSuccess(TVMState& st) { + const auto path = ParsePath(st); + UNIT_ASSERT(path); + return path->Path; + } + + [[nodiscard]] + auto DoTestParsePathSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + return DoTestParsePathSuccess(st); + } + + Y_UNIT_TEST(TestParsePath) { + DoTestParsePathFailure("", 1, 0); + + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePathSuccess(TStringBuf("\x00"sv), 1, 0), TStringBuf("")); + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePathSuccess(TStringBuf("\x21\x0C"sv), 1, 0), TStringBuf("a")); + + DoTestParsePathFailure("\x22\x0C", 1, 0); + } +}; diff --git a/library/cpp/scheme/tests/fuzz_ops/ut/ya.make b/library/cpp/scheme/tests/fuzz_ops/ut/ya.make new file mode 100644 index 00000000000..5c933518ea8 --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_ops/ut/ya.make @@ -0,0 +1,15 @@ +UNITTEST() + +OWNER(velavokr) + +PEERDIR( + library/cpp/testing/unittest + library/cpp/scheme + library/cpp/scheme/tests/fuzz_ops/lib +) + +SRCS( + vm_parse_ut.cpp +) + +END() diff --git a/library/cpp/scheme/tests/fuzz_ops/ya.make b/library/cpp/scheme/tests/fuzz_ops/ya.make new file mode 100644 index 00000000000..025241ef205 --- /dev/null +++ b/library/cpp/scheme/tests/fuzz_ops/ya.make @@ -0,0 +1,18 @@ +FUZZ() + +OWNER( + g:blender + g:middle + g:upper + velavokr +) + +SRCS( + fuzz_ops.cpp +) + +PEERDIR( + library/cpp/scheme/tests/fuzz_ops/lib +) + +END() diff --git a/library/cpp/scheme/tests/ut/fuzz_ops_found_bugs_ut.cpp b/library/cpp/scheme/tests/ut/fuzz_ops_found_bugs_ut.cpp new file mode 100644 index 00000000000..a445b0f87c7 --- /dev/null +++ b/library/cpp/scheme/tests/ut/fuzz_ops_found_bugs_ut.cpp @@ -0,0 +1,12 @@ +#include <library/cpp/scheme/scheme.h> +#include <library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.h> +#include <library/cpp/testing/unittest/registar.h> +#include <util/string/hex.h> + +Y_UNIT_TEST_SUITE(TTestSchemeFuzzOpsFoundBugs) { + using namespace NSc::NUt; + + Y_UNIT_TEST(TestBug1) { + FuzzOps(HexDecode("98040129000525"), true); + } +}; diff --git a/library/cpp/scheme/tests/ut/scheme_cast_ut.cpp b/library/cpp/scheme/tests/ut/scheme_cast_ut.cpp new file mode 100644 index 00000000000..4f907157e92 --- /dev/null +++ b/library/cpp/scheme/tests/ut/scheme_cast_ut.cpp @@ -0,0 +1,162 @@ +#include <library/cpp/scheme/scheme.h> +#include <library/cpp/scheme/scheme_cast.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/null.h> +#include <util/string/subst.h> +#include <util/string/util.h> + +using namespace NJsonConverters; + +using TVI = TVector<int>; +using THI = THashMap<int, int>; +using TMI = TMap<int, int>; +using THSI = THashSet<int>; +using TSI = TSet<int>; +using TPI = std::pair<int, int>; + +Y_UNIT_TEST_SUITE(TSchemeCastTest) { + Y_UNIT_TEST(TestYVector) { + TVI v; + for (int i = 0; i < 3; ++i) + v.push_back(i); + + UNIT_ASSERT_VALUES_EQUAL("[0,1,2]", ToJson(v)); + + TVI y(FromJson<TVI>(ToJson(v))); + UNIT_ASSERT_VALUES_EQUAL(v.size(), y.size()); + UNIT_ASSERT(std::equal(v.begin(), v.end(), y.begin())); + } + + Y_UNIT_TEST(TestYHash) { + THI h; + for (int i = 0; i < 3; ++i) + h[i] = i * i; + + const TString etalon = "{\"0\":0,\"1\":1,\"2\":4}"; + UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(h, true)); + + THI h2(FromJson<THI>(ToJson(h))); + UNIT_ASSERT_VALUES_EQUAL(ToJson(h2, true), ToJson(h, true)); + } + + Y_UNIT_TEST(TestYMap) { + TMI h; + for (int i = 0; i < 3; ++i) + h[i] = i * i; + + const TString etalon = "{\"0\":0,\"1\":1,\"2\":4}"; + UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(h, true)); + + TMI h2(FromJson<TMI>(ToJson(h))); + UNIT_ASSERT_VALUES_EQUAL(ToJson(h2, true), ToJson(h, true)); + } + + Y_UNIT_TEST(TestYHashSet) { + THSI h; + for (int i = 0; i < 3; ++i) + h.insert(i * i); + + const TString etalon = "{\"0\":null,\"1\":null,\"4\":null}"; + UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(h, true)); + + THSI h2(FromJson<THSI>(ToJson(h))); + UNIT_ASSERT_VALUES_EQUAL(ToJson(h2, true), ToJson(h, true)); + } + + Y_UNIT_TEST(TestYSet) { + TSI h; + for (int i = 0; i < 3; ++i) + h.insert(i * i); + + const TString etalon = "{\"0\":null,\"1\":null,\"4\":null}"; + UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(h, true)); + + TSI h2(FromJson<TSI>(ToJson(h))); + UNIT_ASSERT_VALUES_EQUAL(ToJson(h2, true), ToJson(h, true)); + } + + Y_UNIT_TEST(TestTPair) { + TPI p(1, 1); + + const TString etalon = "[1,1]"; + UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(p, true)); + + TPI p2(FromJson<TPI>(ToJson(p))); + UNIT_ASSERT_VALUES_EQUAL(ToJson(p2, true), ToJson(p, true)); + } + + struct TCustom: public IJsonSerializable { + int A, B; + TCustom() + : A(0) + , B(0){}; + TCustom(int a, int b) + : A(a) + , B(b) + { + } + NSc::TValue ToTValue() const override { + NSc::TValue res; + res["a"] = A; + res["b"] = B; + return res; + } + void FromTValue(const NSc::TValue& v, const bool) override { + A = v["a"].GetNumber(); + B = v["b"].GetNumber(); + } + + bool operator<(const TCustom& rhs) const { + return A < rhs.A; + } + }; + + Y_UNIT_TEST(TestTCustom) { + TCustom x(2, 3); + + const TString etalon = "{\"a\":2,\"b\":3}"; + UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(x, true)); + + TCustom x2(FromJson<TCustom>(ToJson(x))); + UNIT_ASSERT_VALUES_EQUAL(ToJson(x2, true), ToJson(x, true)); + } + + Y_UNIT_TEST(TestVectorOfPairs) { + typedef TVector<TPI> TVPI; + TVPI v; + + for (int i = 0; i < 3; ++i) + v.push_back(TPI(i, i * i)); + + const TString etalon = "[[0,0],[1,1],[2,4]]"; + UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(v, true)); + + TVPI v2(FromJson<TVPI>(ToJson(v))); + UNIT_ASSERT_VALUES_EQUAL(ToJson(v2, true), ToJson(v, true)); + } + + Y_UNIT_TEST(TestSetOfCustom) { + typedef TSet<TCustom> TSC; + TSC s; + s.insert(TCustom(2, 3)); + + const TString etalon = "{\"{\\\"a\\\":2,\\\"b\\\":3}\":null}"; + UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(s, true)); + + TSC s2(FromJson<TSC>(ToJson(s))); + UNIT_ASSERT_VALUES_EQUAL(ToJson(s2, true), ToJson(s, true)); + } + + Y_UNIT_TEST(TestExceptions) { + NSc::TValue v = 1; + const TString json = v.ToJson(); + UNIT_ASSERT_EXCEPTION(FromJson<TVI>(json, true), yexception); + UNIT_ASSERT_EXCEPTION(FromJson<THI>(json, true), yexception); + UNIT_ASSERT_EXCEPTION(FromJson<TMI>(json, true), yexception); + UNIT_ASSERT_EXCEPTION(FromJson<THSI>(json, true), yexception); + UNIT_ASSERT_EXCEPTION(FromJson<TSI>(json, true), yexception); + UNIT_ASSERT_EXCEPTION(FromJson<TPI>(json, true), yexception); + } +}; diff --git a/library/cpp/scheme/tests/ut/scheme_json_ut.cpp b/library/cpp/scheme/tests/ut/scheme_json_ut.cpp new file mode 100644 index 00000000000..daeb2654f9a --- /dev/null +++ b/library/cpp/scheme/tests/ut/scheme_json_ut.cpp @@ -0,0 +1,161 @@ +#include <library/cpp/scheme/scimpl_private.h> +#include <library/cpp/scheme/ut_utils/scheme_ut_utils.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/null.h> +#include <util/string/subst.h> +#include <util/string/util.h> + +#include <type_traits> +#include <library/cpp/string_utils/quote/quote.h> + +using namespace std::string_view_literals; + +Y_UNIT_TEST_SUITE(TSchemeJsonTest) { + Y_UNIT_TEST(TestJson) { + const char* json = "[\n" + " {\n" + " \"url\":\"foo\",\n" + " \"title\":\"bar\",\n" + " \"passages\":[\"foo\", \"bar\"],\n" + " }\n" + "]"; + + { + const NSc::TValue& v = NSc::TValue::FromJson(json); + UNIT_ASSERT_VALUES_EQUAL("bar", (TStringBuf)v.TrySelect("0/passages/1")); + UNIT_ASSERT(v.PathExists("0/passages/0")); + UNIT_ASSERT(v.PathExists("[0]/[passages]/[0]")); + UNIT_ASSERT(v.PathExists("[0][passages][0]")); + UNIT_ASSERT(v.PathExists("")); + UNIT_ASSERT(!v.PathExists("`")); + UNIT_ASSERT(v.TrySelect("").Has(0)); + UNIT_ASSERT(!v.PathExists("1")); + UNIT_ASSERT(!v.PathExists("0/passages1")); + UNIT_ASSERT(!v.PathExists("0/passages/2")); + UNIT_ASSERT(!v.PathExists("0/passages/2")); + UNIT_ASSERT_VALUES_EQUAL(0, (double)v.TrySelect("0/passages/2")); + UNIT_ASSERT(!v.PathExists("0/passages/2")); + } + { + const NSc::TValue& vv = NSc::TValue::FromJson("[ test ]]"); + UNIT_ASSERT(vv.IsNull()); + } + { + const char* json = "[a,b],[a,b]"; + const NSc::TValue& v = NSc::TValue::FromJson(json); + UNIT_ASSERT(v.IsNull()); + } + { + const char* json = "[null,null]"; + const NSc::TValue& v = NSc::TValue::FromJson(json); + UNIT_ASSERT(v.PathExists("1")); + UNIT_ASSERT(!v.PathExists("2")); + } + { + const char* json = "{ a : b : c }"; + NSc::TValue v; + UNIT_ASSERT(!NSc::TValue::FromJson(v, json)); + UNIT_ASSERT(v.IsNull()); + } + { + const char* json = "[a:b]"; + UNIT_ASSERT(NSc::TValue::FromJson(json).IsNull()); + } + { + UNIT_ASSERT_VALUES_EQUAL("{\n \"a\" : \"b\",\n \"c\" : \"d\"\n}", + NSc::TValue::FromJson("{a:b,c:d}").ToJson(NSc::TValue::JO_PRETTY)); + } + } + + Y_UNIT_TEST(TestSafeJson) { + TString ss; + ss.reserve(256); + + for (int i = 0; i < 256; ++i) { + ss.append((char)i); + } + + NSc::TValue v; + v[ss] = "xxx"; + v["xxx"] = ss; + + UNIT_ASSERT_VALUES_EQUAL("{\"xxx\":null}", v.ToJson(NSc::TValue::JO_SKIP_UNSAFE)); + UNIT_ASSERT_VALUES_EQUAL("{\"xxx\":null}", v.ToJson(NSc::TValue::JO_SAFE)); + + UNIT_ASSERT_VALUES_EQUAL("{" + "\"\\u0000\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\b\\t\\n\\u000B\\f\\r" + "\\u000E\\u000F\\u0010\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018" + "\\u0019\\u001A\\u001B\\u001C\\u001D\\u001E\\u001F !\\\"#$%&'()*+,-./0123456789" + ":;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\u007F" + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A\x8B\x8C\x8D\x8E\x8F\x90\x91\x92\x93" + "\x94\x95\x96\x97\x98\x99\x9A\x9B\x9C\x9D\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7" + "\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB" + "\xBC\xBD\xBE\xBF\\xC0\\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA\xCB\xCC\xCD\xCE" + "\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF\xE0\xE1" + "\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4" + "\\xF5\\xF6\\xF7\\xF8\\xF9\\xFA\\xFB\\xFC\\xFD\\xFE\\xFF\":" + "\"xxx\"," + "\"xxx\":" + "\"\\u0000\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\b\\t\\n\\u000B\\f\\r" + "\\u000E\\u000F\\u0010\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018" + "\\u0019\\u001A\\u001B\\u001C\\u001D\\u001E\\u001F !\\\"#$%&'()*+,-./0123456789" + ":;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\u007F" + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A\x8B\x8C\x8D\x8E\x8F\x90\x91\x92\x93" + "\x94\x95\x96\x97\x98\x99\x9A\x9B\x9C\x9D\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7" + "\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB" + "\xBC\xBD\xBE\xBF\\xC0\\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA\xCB\xCC\xCD\xCE" + "\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF\xE0\xE1" + "\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4" + "\\xF5\\xF6\\xF7\\xF8\\xF9\\xFA\\xFB\\xFC\\xFD\\xFE\\xFF\"" + "}", + v.ToJson(NSc::TValue::JO_SORT_KEYS)); + UNIT_ASSERT(NSc::TValue::Equal(v, NSc::TValue::FromJson(v.ToJson()))); + + { + NSc::TValue value; + TString articleName{"\xC2\xC2\xCF"}; + value["text"] = articleName; + UNIT_ASSERT_VALUES_EQUAL(value.ToJson(), "{\"text\":\"\xC2\xC2\xCF\"}"); + UNIT_ASSERT_VALUES_EQUAL(value.ToJsonSafe(), "{\"text\":null}"); + } + } + + Y_UNIT_TEST(TestJsonEscape) { + NSc::TValue v("\10\7\6\5\4\3\2\1\0"sv); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "\"\\b\\u0007\\u0006\\u0005\\u0004\\u0003\\u0002\\u0001\\u0000\""); + } + + Y_UNIT_TEST(TestStrictJson) { + UNIT_ASSERT_NO_EXCEPTION(NSc::TValue::FromJsonThrow("{a:b}")); + UNIT_ASSERT_EXCEPTION(NSc::TValue::FromJsonThrow("{a:b}", NSc::TValue::JO_PARSER_STRICT), yexception); + UNIT_ASSERT_NO_EXCEPTION(NSc::TValue::FromJsonThrow("{\"a\":\"b\"}", NSc::TValue::JO_PARSER_STRICT)); + } + + Y_UNIT_TEST(TestJsonValue) { + NSc::TValue a = NSc::NUt::AssertFromJson("{a:[null,-1,2,3.4,str,{b:{c:d}}],e:f}"); + NSc::TValue b = NSc::TValue::FromJsonValue(a.ToJsonValue()); + UNIT_ASSERT_JSON_EQ_JSON(a, b); + } + + Y_UNIT_TEST(TestJsonEmptyContainers) { + { + NSc::TValue a = NSc::NUt::AssertFromJson("{a:[]}"); + NSc::TValue b = NSc::TValue::FromJsonValue(a.ToJsonValue()); + UNIT_ASSERT_JSON_EQ_JSON(a, b); + } + { + NSc::TValue a = NSc::NUt::AssertFromJson("{a:{}}"); + NSc::TValue b = NSc::TValue::FromJsonValue(a.ToJsonValue()); + UNIT_ASSERT_JSON_EQ_JSON(a, b); + } + } + + Y_UNIT_TEST(TestDuplicateKeys) { + const TStringBuf duplicatedKeys = "{\"a\":[{\"b\":1, \"b\":42}]}"; + UNIT_ASSERT_NO_EXCEPTION(NSc::TValue::FromJsonThrow(duplicatedKeys)); + UNIT_ASSERT_EXCEPTION(NSc::TValue::FromJsonThrow(duplicatedKeys, NSc::TValue::JO_PARSER_DISALLOW_DUPLICATE_KEYS), yexception); + UNIT_ASSERT(NSc::TValue::FromJson(duplicatedKeys).IsDict()); + UNIT_ASSERT(NSc::TValue::FromJson(duplicatedKeys, NSc::TValue::JO_PARSER_DISALLOW_DUPLICATE_KEYS).IsNull()); + } +}; diff --git a/library/cpp/scheme/tests/ut/scheme_merge_ut.cpp b/library/cpp/scheme/tests/ut/scheme_merge_ut.cpp new file mode 100644 index 00000000000..2a06cf110da --- /dev/null +++ b/library/cpp/scheme/tests/ut/scheme_merge_ut.cpp @@ -0,0 +1,172 @@ +#include <library/cpp/scheme/scimpl_private.h> +#include <library/cpp/scheme/ut_utils/scheme_ut_utils.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/null.h> +#include <library/cpp/string_utils/quote/quote.h> +#include <util/string/subst.h> +#include <util/string/util.h> + +#include <type_traits> + + +Y_UNIT_TEST_SUITE(TSchemeMergeTest) { + + void DoTestReverseMerge(TStringBuf lhs, TStringBuf rhs, TStringBuf res) { + NSc::TValue v = NSc::TValue::FromJson(lhs); + v.ReverseMerge(NSc::TValue::FromJson(rhs)); + UNIT_ASSERT(NSc::TValue::Equal(v, NSc::TValue::FromJson(res))); + } + + Y_UNIT_TEST(TestReverseMerge) { + DoTestReverseMerge("{a:{x:y, b:c}}", "{a:{u:w, b:d}}", "{a:{u:w, x:y, b:c}}"); + DoTestReverseMerge("null", "{x:y}", "{x:y}"); + DoTestReverseMerge("null", "[b]", "[b]"); + DoTestReverseMerge("[a]", "[b]", "[a]"); + DoTestReverseMerge("{x:null}", "{x:b}", "{x:b}"); + } + + Y_UNIT_TEST(TestMerge) { + TStringBuf data = "{ a : [ { b : 1, d : { e : -1.e5 } }, { f : 0, g : [ h, i ] } ] }"; + NSc::TValue v = NSc::TValue::FromJson(data); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(true), v.Clone().ToJson(true)); + UNIT_ASSERT(v.Has("a")); + UNIT_ASSERT(v["a"].Has(1)); + UNIT_ASSERT(v["a"][0].Has("b")); + UNIT_ASSERT(v["a"][0].Has("d")); + UNIT_ASSERT(1 == v["a"][0]["b"]); + UNIT_ASSERT(v["a"][0]["d"].Has("e")); + UNIT_ASSERT(-1.e5 == v["a"][0]["d"]["e"]); + UNIT_ASSERT(v["a"][1].Has("f")); + UNIT_ASSERT(v["a"][1].Has("g")); + UNIT_ASSERT(0. == v["a"][1]["f"]); + UNIT_ASSERT(v["a"][1]["g"].IsArray()); + UNIT_ASSERT(v["a"][1]["g"].Has(1)); + UNIT_ASSERT(TStringBuf("h") == v["a"][1]["g"][0]); + UNIT_ASSERT(TStringBuf("i") == v["a"][1]["g"][1]); + + { + TStringBuf data = "{ a : [ { d : 42 }, { g : [ 3 ] } ], q : r }"; + + NSc::TValue v1 = NSc::TValue::FromJson(data); + UNIT_ASSERT_VALUES_EQUAL(v1.ToJson(true), v1.Clone().ToJson(true)); + UNIT_ASSERT(NSc::TValue::Equal(v1, v1.FromJson(v1.ToJson()))); + + NSc::TValue v2; + v2.MergeUpdate(v["a"]); + UNIT_ASSERT_C(NSc::TValue::Equal(v["a"], v2), Sprintf("\n%s\n!=\n%s\n", v["a"].ToJson().data(), v2.ToJson().data())); + + v.MergeUpdate(v1); + UNIT_ASSERT_C(!NSc::TValue::Equal(v["a"], v2), Sprintf("\n%s\n!=\n%s\n", v["a"].ToJson().data(), v2.ToJson().data())); + v2.MergeUpdate(v1["a"]); + UNIT_ASSERT_C(NSc::TValue::Equal(v["a"], v2), Sprintf("\n%s\n!=\n%s\n", v["a"].ToJson().data(), v2.ToJson().data())); + } + + UNIT_ASSERT(v.Has("a")); + UNIT_ASSERT(v.Has("q")); + UNIT_ASSERT(TStringBuf("r") == v["q"]); + UNIT_ASSERT(v["a"].Has(1)); + UNIT_ASSERT(!v["a"][0].Has("b")); + UNIT_ASSERT(v["a"][0].Has("d")); + UNIT_ASSERT(!v["a"][0]["d"].IsArray()); + UNIT_ASSERT(!v["a"][0]["d"].IsDict()); + UNIT_ASSERT(42 == v["a"][0]["d"]); + UNIT_ASSERT(!v["a"][1].Has("f")); + UNIT_ASSERT(v["a"][1].Has("g")); + UNIT_ASSERT(v["a"][1]["g"].IsArray()); + UNIT_ASSERT(!v["a"][1]["g"].Has(1)); + UNIT_ASSERT(3 == v["a"][1]["g"][0]); + } + + Y_UNIT_TEST(TestMerge1) { + TStringBuf data = "[ { a : { b : d } } ]"; + + NSc::TValue wcopy = NSc::TValue::FromJson(data); + + TStringBuf data1 = "[ { a : { b : c } } ]"; + + wcopy.MergeUpdateJson(data1); + + { + TString json = wcopy.ToJson(true); + SubstGlobal(json, "\"", ""); + UNIT_ASSERT_VALUES_EQUAL(json, "[{a:{b:c}}]"); + } + } + + Y_UNIT_TEST(TestMerge2) { + TStringBuf data = "{ a : { b : c }, q : { x : y } }"; + + NSc::TValue wcopy = NSc::TValue::FromJson(data); + + TStringBuf data1 = "{ a : { e : f } }"; + + wcopy.MergeUpdateJson(data1); + + { + TString json = wcopy.ToJson(true); + SubstGlobal(json, "\"", ""); + UNIT_ASSERT_VALUES_EQUAL(json, "{a:{b:c,e:f},q:{x:y}}"); + } + } + + Y_UNIT_TEST(TestMerge3) { + TStringBuf data = "{ g : { x : { a : { b : c }, q : { x : y } }, y : fff } }"; + + NSc::TValue wcopy = NSc::TValue::FromJson(data); + + TStringBuf data1 = "{ g : { x : { a : { e : f } } } }"; + + wcopy.MergeUpdateJson(data1); + + { + TString json = wcopy.ToJson(true); + SubstGlobal(json, "\"", ""); + UNIT_ASSERT_VALUES_EQUAL(json, "{g:{x:{a:{b:c,e:f},q:{x:y}},y:fff}}"); + } + } + + Y_UNIT_TEST(TestMerge4) { + TStringBuf data = "{ a : 1, b : { c : 2, d : { q : f } } }"; + NSc::TValue val = NSc::TValue::FromJson(data); + + TStringBuf data1 = "{ a : 2, b : { c : 3, d : { q : e }, g : h } }"; + + val.MergeUpdateJson(data1); + + { + TString json = val.ToJson(true); + SubstGlobal(json, "\"", ""); + + UNIT_ASSERT_VALUES_EQUAL(json, "{a:2,b:{c:3,d:{q:e},g:h}}"); + } + } + + Y_UNIT_TEST(TestMerge5) { + NSc::TValue v0; + v0.GetOrAdd("x").MergeUpdate(NSc::TValue(1)); + UNIT_ASSERT_VALUES_EQUAL(v0.ToJson(), "{\"x\":1}"); + } + + Y_UNIT_TEST(TestMerge6) { + NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"abc\",\"y\":\"def\"}"); + NSc::TValue vb = va.Get("y"); + NSc::TValue diff; + diff["y"] = vb; + va.MergeUpdate(diff); + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), "{\"x\":\"abc\",\"y\":\"def\"}"); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), "\"def\""); + UNIT_ASSERT_VALUES_EQUAL(diff.ToJson(), "{\"y\":\"def\"}"); + } + + Y_UNIT_TEST(TestMerge7) { + NSc::TValue v; + v["a"] = NSc::TValue::FromJson("[0.125,0.12,0.1,0.08,0.06]"); + UNIT_ASSERT_JSON_EQ_JSON(v, "{a:[0.125,0.12,0.1,0.08,0.06]}"); + + NSc::TValue a = v.TrySelectOrAdd("a")->MergeUpdateJson("[1,2,3]"); + + UNIT_ASSERT_JSON_EQ_JSON(a, "[1,2,3]"); + UNIT_ASSERT_JSON_EQ_JSON(v, "{a:[1,2,3]}"); + } +}; diff --git a/library/cpp/scheme/tests/ut/scheme_path_ut.cpp b/library/cpp/scheme/tests/ut/scheme_path_ut.cpp new file mode 100644 index 00000000000..0d4d79d4833 --- /dev/null +++ b/library/cpp/scheme/tests/ut/scheme_path_ut.cpp @@ -0,0 +1,159 @@ +#include <library/cpp/scheme/scimpl_private.h> +#include <library/cpp/scheme/ut_utils/scheme_ut_utils.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/null.h> +#include <util/string/subst.h> +#include <util/string/util.h> + +#include <type_traits> +#include <library/cpp/string_utils/quote/quote.h> + +Y_UNIT_TEST_SUITE(TSchemePathTest) { + void DoTestSelect(TStringBuf path, TStringBuf expected, TStringBuf delexpected) { + NSc::TValue v; + UNIT_ASSERT(!v.PathExists(path)); + UNIT_ASSERT(NSc::TValue::PathValid(path)); + UNIT_ASSERT(NSc::TValue::Same(v.TrySelect(path), NSc::Null())); + *v.TrySelectOrAdd(path) = 1; + NSc::NUt::AssertSchemeJson(expected, v); + UNIT_ASSERT(v.PathExists(path)); + UNIT_ASSERT(1 == v.TrySelectOrAdd(path)->GetNumber()); + UNIT_ASSERT(1 == v.TrySelect(path).GetNumber()); + UNIT_ASSERT(1 == v.TrySelectAndDelete(path).GetNumber()); + UNIT_ASSERT(NSc::TValue::Same(v.TrySelectAndDelete(path), NSc::Null())); + NSc::NUt::AssertSchemeJson(delexpected, v); + UNIT_ASSERT(!v.PathExists(path)); + UNIT_ASSERT(NSc::TValue::Same(v.TrySelect(path), NSc::Null())); + } + + Y_UNIT_TEST(TestSelect) { + NSc::TValue v; + UNIT_ASSERT(!v.PathValid(" ")); + UNIT_ASSERT(v.PathExists("")); + UNIT_ASSERT(v.PathExists("//")); + + UNIT_ASSERT(NSc::TValue::Same(v, *v.TrySelectOrAdd("//"))); + NSc::NUt::AssertSchemeJson("null", v); + UNIT_ASSERT(NSc::TValue::Same(v.TrySelectAndDelete("//"), NSc::Null())); + NSc::NUt::AssertSchemeJson("null", v); + + v.SetDict(); + UNIT_ASSERT(NSc::TValue::Same(v, *v.TrySelectOrAdd("//"))); + NSc::NUt::AssertSchemeJson("{}", v); + UNIT_ASSERT(NSc::TValue::Same(v.TrySelectAndDelete("//"), NSc::Null())); + NSc::NUt::AssertSchemeJson("{}", v); + + v.SetArray(); + UNIT_ASSERT(NSc::TValue::Same(v, *v.TrySelectOrAdd("//"))); + NSc::NUt::AssertSchemeJson("[]", v); + UNIT_ASSERT(NSc::TValue::Same(v.TrySelectAndDelete("//"), NSc::Null())); + NSc::NUt::AssertSchemeJson("[]", v); + + DoTestSelect("[]", "{'':1}", "{}"); + DoTestSelect("[ ]", "{' ':1}", "{}"); + DoTestSelect("[0]", "[1]", "[]"); + DoTestSelect("[1]", "[null,1]", "[null]"); + DoTestSelect("foo/[0]/bar", "{foo:[{bar:1}]}", "{foo:[{}]}"); + DoTestSelect("foo/1/bar", "{foo:[null,{bar:1}]}", "{foo:[null,{}]}"); + DoTestSelect("foo[-1]bar", "{foo:{'-1':{bar:1}}}", "{foo:{'-1':{}}}"); + DoTestSelect("'foo'/\"0\"/'bar'", "{foo:{'0':{bar:1}}}", "{foo:{'0':{}}}"); + DoTestSelect("'\\''", "{'\\'':1}", "{}"); + } + + Y_UNIT_TEST(TestSelectAndMerge) { + NSc::TValue v; + v.TrySelectOrAdd("blender/enabled")->MergeUpdateJson("1"); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::FromJson("1").ToJson(), "1"); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"blender\":{\"enabled\":1}}"); + } + + Y_UNIT_TEST(TestPathEscapes) { + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a"), "a"); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath(""), R"=("")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[]"), R"=("[]")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[]ab"), R"=("[]ab")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a[]b"), R"=("a[]b")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab[]"), R"=("ab[]")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[ab]"), R"=("[ab]")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[ab"), R"=("[ab")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a[b"), R"=("a[b")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab["), R"=("ab[")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("]ab"), R"=("]ab")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a]b"), R"=("a]b")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab]"), R"=("ab]")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath(R"=(\)="), R"=("\\")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath(R"=(\\)="), R"=("\\\\")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("/"), R"=("/")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("//"), R"=("//")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("///"), R"=("///")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("/ab"), R"=("/ab")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a/b"), R"=("a/b")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab/"), R"=("ab/")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("//ab"), R"=("//ab")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a//b"), R"=("a//b")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab//"), R"=("ab//")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("6400"), R"=("6400")="); + + { + NSc::TValue val; + *val.TrySelectOrAdd("") = 100; + const TString res = R"=(100)="; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; + *val.TrySelectOrAdd("a") = 100; + const TString res = R"=({"a":100})="; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; + *val.TrySelectOrAdd(R"=(////)=") = 100; + const TString res = R"=(100)="; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; + *val.TrySelectOrAdd(R"=()=") = 100; + const TString res = R"=(100)="; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; + *val.TrySelectOrAdd(R"=("")=") = 100; + const TString res = R"=({"":100})="; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; + *val.TrySelectOrAdd(R"=("[1]")=") = 100; + const TString res = R"=({"[1]":100})="; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; + *val.TrySelectOrAdd(R"=("\"\"")=") = 100; + const TString res = R"=({"\"\"":100})="; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; + *val.TrySelectOrAdd(R"=("/10/")=") = 100; + const TString res = R"=({"/10/":100})="; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; + *val.TrySelectOrAdd(R"=(/"[10]"//""/"\"/10/\""///)=") = 100; + const TString res = R"=({"[10]":{"":{"\"/10/\"":100}}})="; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; + *val.TrySelectOrAdd(R"=(/"[10]"//""/"\"/10/\""///)=") = 100; + const TString res = R"=({"[10]":{"":{"\"/10/\"":100}}})="; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + } +}; diff --git a/library/cpp/scheme/tests/ut/scheme_proto_ut.cpp b/library/cpp/scheme/tests/ut/scheme_proto_ut.cpp new file mode 100644 index 00000000000..e711a0d0925 --- /dev/null +++ b/library/cpp/scheme/tests/ut/scheme_proto_ut.cpp @@ -0,0 +1,220 @@ +#include <library/cpp/protobuf/util/is_equal.h> +#include <library/cpp/scheme/scheme.h> +#include <library/cpp/scheme/tests/ut/scheme_ut.pb.h> +#include <library/cpp/testing/unittest/registar.h> + +Y_UNIT_TEST_SUITE(TSchemeProtoTest) { + void DoTestProtobuf(bool fromProto, bool mapAsDict); + + Y_UNIT_TEST(TestFromProtobuf) { + DoTestProtobuf(true, false); + } + + Y_UNIT_TEST(TestToProtobuf) { + DoTestProtobuf(false, false); + } + + Y_UNIT_TEST(TestFromProtobufWithDict) { + DoTestProtobuf(true, true); + } + + Y_UNIT_TEST(TestToProtobufWithDict) { + DoTestProtobuf(false, true); + } + + template <class T> + void AddMapPair(NSc::TValue& v, TStringBuf key, T value, bool mapAsDict) { + if (mapAsDict) { + v[key] = value; + } else { + auto& newElement = v.Push(); + newElement["key"] = key; + newElement["value"] = value; + } + } + + void DoTestProtobuf(bool fromProto, bool mapAsDict) { + NSc::TMessage m; + NSc::TValue v; + m.SetDouble((double)1 / 3), v["Double"] = (double)1 / 3; + m.SetFloat((float)1 / 3), v["Float"] = (float)1 / 3; + m.SetInt32(1000000), v["Int32"] = 1000000; + m.SetInt64(1000000000000LL), v["Int64"] = 1000000000000LL; + m.SetUInt32(555555), v["UInt32"] = 555555; + m.SetUInt64(555555555555LL), v["UInt64"] = 555555555555LL; + m.SetSInt32(-555555), v["SInt32"] = -555555; + m.SetSInt64(-555555555555LL), v["SInt64"] = -555555555555LL; + m.SetFixed32(123456), v["Fixed32"] = 123456; + m.SetFixed64(123456123456LL), v["Fixed64"] = 123456123456LL; + m.SetSFixed32(-123456), v["SFixed32"] = -123456; + m.SetSFixed64(-123456123456LL), v["SFixed64"] = -123456123456LL; + m.SetBool(true), v["Bool"] = true; + m.SetString("String"), v["String"] = "String"; + m.SetBytes("Bytes"), v["Bytes"] = "Bytes"; + m.SetEnum(NSc::VALUE1), v["Enum"] = "VALUE1"; + + auto& mapDoublesP = *m.mutable_mapdoubles(); + auto& mapDoublesV = v["MapDoubles"]; + mapDoublesP["pi"] = 3.14; + AddMapPair(mapDoublesV, "pi", 3.14, mapAsDict); + if (fromProto && mapAsDict) { + // can not add two entries in dict, as it represent over array with unstable order + mapDoublesP["back_e"] = 1 / 2.7; + AddMapPair(mapDoublesV, "back_e", 1 / 2.7, mapAsDict); + } + + auto& mapInt32sP = *m.mutable_mapint32s(); + auto& mapInt32sV = v["MapInt32s"]; + mapInt32sP["pi"] = -7; + AddMapPair(mapInt32sV, "pi", -7, mapAsDict); + if (fromProto && mapAsDict) { + // can not add two entries in dict, as it represent over array with unstable order + mapInt32sP["back_e"] = 42; + AddMapPair(mapInt32sV, "back_e", 42, mapAsDict); + } + + auto& mapStringP = *m.mutable_mapstring(); + auto& mapStringV = v["MapString"]; + mapStringP["intro"] = "body"; + AddMapPair(mapStringV, "intro", "body", mapAsDict); + if (fromProto && mapAsDict) { + // can not add two entries in dict, as it represent over array with unstable order + mapStringP["deep"] = "blue"; + AddMapPair(mapStringV, "deep", "blue", mapAsDict); + } + + m.AddDoubles((double)1 / 3), v["Doubles"][0] = (double)1 / 3; + m.AddDoubles((double)1 / 4), v["Doubles"][1] = (double)1 / 4; + + m.AddFloats((float)1 / 3), v["Floats"][0] = (float)1 / 3; + m.AddFloats((float)1 / 4), v["Floats"][1] = (float)1 / 4; + + m.AddInt32s(1000000), v["Int32s"][0] = 1000000; + m.AddInt32s(2000000), v["Int32s"][1] = 2000000; + + m.AddInt64s(1000000000000LL), v["Int64s"][0] = 1000000000000LL; + m.AddInt64s(2000000000000LL), v["Int64s"][1] = 2000000000000LL; + + m.AddUInt32s(555555), v["UInt32s"][0] = 555555; + m.AddUInt32s(655555), v["UInt32s"][1] = 655555; + + m.AddUInt64s(555555555555LL); + v["UInt64s"][0] = 555555555555LL; + m.AddUInt64s(655555555555LL); + v["UInt64s"][1] = 655555555555LL; + + m.AddSInt32s(-555555), v["SInt32s"][0] = -555555; + m.AddSInt32s(-655555), v["SInt32s"][1] = -655555; + + m.AddSInt64s(-555555555555LL), v["SInt64s"][0] = -555555555555LL; + m.AddSInt64s(-655555555555LL), v["SInt64s"][1] = -655555555555LL; + + m.AddFixed32s(123456), v["Fixed32s"][0] = 123456; + m.AddFixed32s(223456), v["Fixed32s"][1] = 223456; + + m.AddFixed64s(123456123456LL), v["Fixed64s"][0] = 123456123456LL; + m.AddFixed64s(223456123456LL), v["Fixed64s"][1] = 223456123456LL; + + m.AddSFixed32s(-123456), v["SFixed32s"][0] = -123456; + m.AddSFixed32s(-223456), v["SFixed32s"][1] = -223456; + + m.AddSFixed64s(-123456123456LL), v["SFixed64s"][0] = -123456123456LL; + m.AddSFixed64s(-223456123456LL), v["SFixed64s"][1] = -223456123456LL; + + m.AddBools(false), v["Bools"][0] = false; + m.AddBools(true), v["Bools"][1] = true; + + m.AddStrings("String1"), v["Strings"][0] = "String1"; + m.AddStrings("String2"), v["Strings"][1] = "String2"; + + m.AddBytess("Bytes1"), v["Bytess"][0] = "Bytes1"; + m.AddBytess("Bytes2"), v["Bytess"][1] = "Bytes2"; + + m.AddEnums(NSc::VALUE1), v["Enums"][0] = "VALUE1"; + m.AddEnums(NSc::VALUE2), v["Enums"][1] = "VALUE2"; + + NSc::TMessage2 m2; + NSc::TValue v2; + m2.SetDouble((double)1 / 3), v2["Double"] = (double)1 / 3; + m2.SetFloat((float)1 / 3), v2["Float"] = (float)1 / 3; + m2.SetInt32(1000000), v2["Int32"] = 1000000; + m2.SetInt64(1000000000000LL), v2["Int64"] = 1000000000000LL; + m2.SetUInt32(555555), v2["UInt32"] = 555555; + m2.SetUInt64(555555555555LL), v2["UInt64"] = 555555555555LL; + m2.SetSInt32(-555555), v2["SInt32"] = -555555; + m2.SetSInt64(-555555555555LL), v2["SInt64"] = -555555555555LL; + m2.SetFixed32(123456), v2["Fixed32"] = 123456; + m2.SetFixed64(123456123456LL), v2["Fixed64"] = 123456123456LL; + m2.SetSFixed32(-123456), v2["SFixed32"] = -123456; + m2.SetSFixed64(-123456123456LL), v2["SFixed64"] = -123456123456LL; + m2.SetBool(true), v2["Bool"] = true; + m2.SetString("String"), v2["String"] = "String"; + m2.SetBytes("Bytes"), v2["Bytes"] = "Bytes"; + m2.SetEnum(NSc::VALUE1), v2["Enum"] = "VALUE1"; + + m2.AddDoubles((double)1 / 3), v2["Doubles"][0] = (double)1 / 3; + m2.AddDoubles((double)1 / 4), v2["Doubles"][1] = (double)1 / 4; + + m2.AddFloats((float)1 / 3), v2["Floats"][0] = (float)1 / 3; + m2.AddFloats((float)1 / 4), v2["Floats"][1] = (float)1 / 4; + + m2.AddInt32s(1000000), v2["Int32s"][0] = 1000000; + m2.AddInt32s(2000000), v2["Int32s"][1] = 2000000; + + m2.AddInt64s(1000000000000LL), v2["Int64s"][0] = 1000000000000LL; + m2.AddInt64s(2000000000000LL), v2["Int64s"][1] = 2000000000000LL; + + m2.AddUInt32s(555555), v2["UInt32s"][0] = 555555; + m2.AddUInt32s(655555), v2["UInt32s"][1] = 655555; + + m2.AddUInt64s(555555555555LL); + v2["UInt64s"][0] = 555555555555LL; + m2.AddUInt64s(655555555555LL); + v2["UInt64s"][1] = 655555555555LL; + + m2.AddSInt32s(-555555), v2["SInt32s"][0] = -555555; + m2.AddSInt32s(-655555), v2["SInt32s"][1] = -655555; + + m2.AddSInt64s(-555555555555LL), v2["SInt64s"][0] = -555555555555LL; + m2.AddSInt64s(-655555555555LL), v2["SInt64s"][1] = -655555555555LL; + + m2.AddFixed32s(123456), v2["Fixed32s"][0] = 123456; + m2.AddFixed32s(223456), v2["Fixed32s"][1] = 223456; + + m2.AddFixed64s(123456123456LL), v2["Fixed64s"][0] = 123456123456LL; + m2.AddFixed64s(223456123456LL), v2["Fixed64s"][1] = 223456123456LL; + + m2.AddSFixed32s(-123456), v2["SFixed32s"][0] = -123456; + m2.AddSFixed32s(-223456), v2["SFixed32s"][1] = -223456; + + m2.AddSFixed64s(-123456123456LL), v2["SFixed64s"][0] = -123456123456LL; + m2.AddSFixed64s(-223456123456LL), v2["SFixed64s"][1] = -223456123456LL; + + m2.AddBools(false), v2["Bools"][0] = false; + m2.AddBools(true), v2["Bools"][1] = true; + + m2.AddStrings("String1"), v2["Strings"][0] = "String1"; + m2.AddStrings("String2"), v2["Strings"][1] = "String2"; + + m2.AddBytess("Bytes1"), v2["Bytess"][0] = "Bytes1"; + m2.AddBytess("Bytes2"), v2["Bytess"][1] = "Bytes2"; + + m2.AddEnums(NSc::VALUE1), v2["Enums"][0] = "VALUE1"; + m2.AddEnums(NSc::VALUE2), v2["Enums"][1] = "VALUE2"; + + *(m.MutableMessage()) = m2, v["Message"] = v2; + + *(m.AddMessages()) = m2, v["Messages"][0] = v2; + *(m.AddMessages()) = m2, v["Messages"][1] = v2; + + if (fromProto) { + UNIT_ASSERT(NSc::TValue::Equal(v, NSc::TValue::From(m, mapAsDict))); + } else { + NSc::TMessage proto; + v.To(proto); + + TString differentPath; + UNIT_ASSERT_C(NProtoBuf::IsEqual(m, proto, &differentPath), differentPath); + } + } +}; diff --git a/library/cpp/scheme/tests/ut/scheme_ut.cpp b/library/cpp/scheme/tests/ut/scheme_ut.cpp new file mode 100644 index 00000000000..1a5d07c31bc --- /dev/null +++ b/library/cpp/scheme/tests/ut/scheme_ut.cpp @@ -0,0 +1,879 @@ +#include <library/cpp/scheme/scimpl_private.h> +#include <library/cpp/scheme/ut_utils/scheme_ut_utils.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/null.h> +#include <library/cpp/string_utils/quote/quote.h> +#include <util/string/subst.h> +#include <util/string/util.h> + +#include <type_traits> + +Y_UNIT_TEST_SUITE(TSchemeTest) { + + Y_UNIT_TEST(TestNaN) { + UNIT_ASSERT_VALUES_EQUAL("null", NSc::TValue(std::numeric_limits<double>::quiet_NaN()).ToJson()); + UNIT_ASSERT_VALUES_EQUAL("null", NSc::TValue(-std::numeric_limits<double>::infinity()).ToJson()); + UNIT_ASSERT_VALUES_EQUAL("null", NSc::TValue(std::numeric_limits<double>::infinity()).ToJson()); + UNIT_ASSERT_VALUES_EQUAL("1", NSc::TValue(1.0).ToJson()); + } + + Y_UNIT_TEST(TestNumbers) { + { + NSc::TValue vd; + UNIT_ASSERT_VALUES_EQUAL(2.5, vd.GetNumberMutable(2.5)); + UNIT_ASSERT_VALUES_EQUAL(2, vd.GetIntNumberMutable(-1)); + } + { + NSc::TValue vi; + UNIT_ASSERT_VALUES_EQUAL(2, vi.GetIntNumberMutable(2)); + UNIT_ASSERT_VALUES_EQUAL(2., vi.GetNumberMutable(-1)); + } + { + NSc::TValue vb = NSc::TValue::FromJson("true"); + + UNIT_ASSERT_VALUES_EQUAL("true", vb.ToJson()); + + UNIT_ASSERT(vb.IsBool()); + UNIT_ASSERT(vb.IsIntNumber()); + UNIT_ASSERT(vb.IsNumber()); + UNIT_ASSERT_VALUES_EQUAL(true, vb.GetBool()); + UNIT_ASSERT_VALUES_EQUAL(1, vb.GetIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1.0, vb.GetNumber()); + + NSc::TValue vb1 = vb.Clone(); + + UNIT_ASSERT(NSc::TValue::Equal(vb, vb1)); + + UNIT_ASSERT_VALUES_EQUAL(true, vb.GetBool()); + UNIT_ASSERT_VALUES_EQUAL(1, vb.GetIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1.0, vb.GetNumber()); + UNIT_ASSERT(vb.IsBool()); + UNIT_ASSERT_VALUES_EQUAL(1, vb.GetIntNumberMutable()); + UNIT_ASSERT(!vb.IsBool()); + + UNIT_ASSERT(NSc::TValue::Equal(vb, vb1)); + + UNIT_ASSERT(vb1.IsBool()); + UNIT_ASSERT_VALUES_EQUAL(1.0, vb1.GetNumberMutable()); + UNIT_ASSERT(!vb1.IsBool()); + + vb.SetBool(true); + + UNIT_ASSERT(vb.IsBool()); + UNIT_ASSERT(NSc::TValue::Equal(vb, vb1)); + + vb = NSc::TValue::FromJson("false"); + + UNIT_ASSERT_VALUES_EQUAL("false", vb.ToJson()); + UNIT_ASSERT(!NSc::TValue::Equal(vb, vb1)); + + UNIT_ASSERT(vb.IsBool()); + UNIT_ASSERT(vb.IsIntNumber()); + UNIT_ASSERT(vb.IsNumber()); + UNIT_ASSERT_VALUES_EQUAL(false, vb.GetBool()); + UNIT_ASSERT_VALUES_EQUAL(0.0, vb.GetNumber()); + UNIT_ASSERT_VALUES_EQUAL(0, vb.GetIntNumber()); + + NSc::TValue vd = NSc::TValue::FromJson("1.0"); + + UNIT_ASSERT(vd.IsNumber()); + UNIT_ASSERT(!vd.IsIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1.0, vd.GetNumber()); + UNIT_ASSERT_VALUES_EQUAL(1, vd.GetIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1.0, vd.GetNumberMutable()); + + NSc::TValue vi = NSc::TValue::FromJson("1"); + + UNIT_ASSERT(vi.IsNumber()); + UNIT_ASSERT(vi.IsIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1.0, vi.GetNumber()); + UNIT_ASSERT_VALUES_EQUAL(1, vi.GetIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1, vi.GetIntNumberMutable()); + + UNIT_ASSERT(NSc::TValue::Equal(vd, vi)); + + vd.SetNumber(1.5); + UNIT_ASSERT(vd.IsNumber()); + UNIT_ASSERT(!vd.IsIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1.5, vd.GetNumber()); + UNIT_ASSERT_VALUES_EQUAL(1, vd.GetIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1.5, vd.GetNumberMutable()); + + UNIT_ASSERT(!NSc::TValue::Equal(vd, vi)); + + UNIT_ASSERT_VALUES_EQUAL("1", vi.ToJson()); + UNIT_ASSERT_VALUES_EQUAL("1.5", vd.ToJson()); + + UNIT_ASSERT_VALUES_EQUAL(1, vd.GetIntNumberMutable()); + UNIT_ASSERT(NSc::TValue::Equal(vd, vi)); + vd.SetIntNumber(2); + UNIT_ASSERT(!NSc::TValue::Equal(vd, vi)); + vi.SetNumber(2.); + UNIT_ASSERT(NSc::TValue::Equal(vd, vi)); + vd.SetNumber(2.); + UNIT_ASSERT(NSc::TValue::Equal(vd, vi)); + vi.SetIntNumber(5); + vd.MergeUpdate(vi); + UNIT_ASSERT(vd.IsNumber()); + UNIT_ASSERT(vd.IsIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(5, vd.GetIntNumber()); + vd.SetNumber(3.3); + vi.MergeUpdate(vd); + UNIT_ASSERT(vi.IsNumber()); + UNIT_ASSERT(!vi.IsIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(3.3, vi.GetNumber()); + + vi.SetIntNumber(Max<i64>()); + UNIT_ASSERT_VALUES_EQUAL("9223372036854775807", vi.ToJson()); + } + } + + template <typename T> + void DoTestForce(T t) { + UNIT_ASSERT_VALUES_EQUAL_C(i64(t), NSc::TValue(i64(t)).ForceIntNumber(), ToString(t)); + UNIT_ASSERT_VALUES_EQUAL_C(double(t), NSc::TValue(double(t)).ForceNumber(), ToString(t)); + + UNIT_ASSERT_VALUES_EQUAL_C(i64(t), NSc::TValue(TStringBuf(ToString(i64(t)))).ForceIntNumber(), ToString(t)); + UNIT_ASSERT_VALUES_EQUAL_C(ToString(double(t)), ToString(NSc::TValue(TStringBuf(ToString(double(t)))).ForceNumber()), ToString(t)); + + UNIT_ASSERT_VALUES_EQUAL_C(ToString(i64(t)), NSc::TValue(TStringBuf(ToString(i64(t)))).ForceString(), ToString(t)); + UNIT_ASSERT_VALUES_EQUAL_C(ToString(double(t)), NSc::TValue(TStringBuf(ToString(double(t)))).ForceString(), ToString(t)); + } + + Y_UNIT_TEST(TestForce) { + DoTestForce(Max<i64>()); + DoTestForce(Min<i64>()); + DoTestForce(1.5); + DoTestForce(-1.5); + + UNIT_ASSERT_VALUES_EQUAL(1, NSc::TValue("32a").ForceIntNumber(1)); + UNIT_ASSERT_VALUES_EQUAL(1.5, NSc::TValue("32a").ForceNumber(1.5)); + } + + template <typename T> + void DoCheckRelations(T t, T tless, T tmore, const NSc::TValue& v, TStringBuf ss) { + UNIT_ASSERT_C((t == v), ss); + UNIT_ASSERT_C(!(t != v), ss); + UNIT_ASSERT_C((t <= v), ss); + UNIT_ASSERT_C((t >= v), ss); + UNIT_ASSERT_C(!(t < v), ss); + UNIT_ASSERT_C(!(t > v), ss); + + UNIT_ASSERT_C(!(tless == v), ss); + UNIT_ASSERT_C((tless != v), ss); + UNIT_ASSERT_C((tless <= v), ss); + UNIT_ASSERT_C(!(tless >= v), ss); + UNIT_ASSERT_C((tless < v), ss); + UNIT_ASSERT_C(!(tless > v), ss); + UNIT_ASSERT_C(!(tmore == v), ss); + UNIT_ASSERT_C((tmore != v), ss); + UNIT_ASSERT_C(!(tmore <= v), ss); + UNIT_ASSERT_C((tmore >= v), ss); + UNIT_ASSERT_C(!(tmore < v), ss); + UNIT_ASSERT_C((tmore > v), ss); + } + + void DoCheckRelations(const NSc::TValue& t, const NSc::TValue&, const NSc::TValue&, const NSc::TValue& v, TStringBuf ss) { + UNIT_ASSERT_C((t == v), ss); + UNIT_ASSERT_C(!(t != v), ss); + } + + // void DoCheckRelations(bool t, bool, bool, const NSc::TValue& v, TStringBuf ss) { + // UNIT_ASSERT_C((t == v), ss); + // UNIT_ASSERT_C(!(t != v), ss); + // } + + template <typename T> + void DoCheckAssignment(T t, T tless, T tmore, TStringBuf s, TStringBuf ss) { + bool expectint = std::is_integral<T>::value; + bool expectnum = std::is_arithmetic<T>::value; + bool expectbool = std::is_same<bool, T>::value; + + { + NSc::TValue v(t); + UNIT_ASSERT_VALUES_EQUAL_C(expectnum, v.IsNumber(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(expectint, v.IsIntNumber(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(expectbool, v.IsBool(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(s, v.ToJson(), ss); + DoCheckRelations(t, tless, tmore, v, ss); + } + { + NSc::TValue v; + UNIT_ASSERT(v.IsNull()); + v = t; + UNIT_ASSERT(!v.IsNull()); + UNIT_ASSERT_VALUES_EQUAL_C(expectnum, v.IsNumber(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(expectint, v.IsIntNumber(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(expectbool, v.IsBool(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(s, v.ToJson(), ss); + DoCheckRelations(t, tless, tmore, v, ss); + } + } + + template <size_t N> + void DoCheckAssignmentArr(const char (&t)[N], const char (&tless)[N], const char (&tmore)[N], TStringBuf s, TStringBuf ss) { + { + NSc::TValue v(t); + + UNIT_ASSERT_C(v.IsString(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(s, v.ToJson(), ss); + DoCheckRelations(t, tless, tmore, v, ss); + } + { + NSc::TValue v; + v = t; + UNIT_ASSERT_C(v.IsString(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(s, v.ToJson(), ss); + DoCheckRelations(t, tless, tmore, v, ss); + } + } + + template <typename T> + void DoCheckAssignmentNum(T t, T tless, T tmore, TStringBuf s, TStringBuf ss) { + DoCheckAssignment(t, tless, tmore, s, ss); + { + NSc::TValue v; + T tt = (v = t); + UNIT_ASSERT_VALUES_EQUAL_C(t, tt, ss); + } + } + + Y_UNIT_TEST(TestAssignments) { + for (int i = -2; i < 3; ++i) { + TString ii = ToString(i); + int iless = i - 1; + int imore = i + 1; + DoCheckAssignmentNum<signed char>(i, iless, imore, ii, "schar"); + DoCheckAssignmentNum<short>(i, iless, imore, ii, "short"); + DoCheckAssignmentNum<int>(i, iless, imore, ii, "int"); + DoCheckAssignmentNum<long>(i, iless, imore, ii, "long"); + DoCheckAssignmentNum<long long>(i, iless, imore, ii, "longlong"); + DoCheckAssignmentNum<i8>(i, iless, imore, ii, "i8"); + DoCheckAssignmentNum<i16>(i, iless, imore, ii, "i16"); + DoCheckAssignmentNum<i32>(i, iless, imore, ii, "i32"); + DoCheckAssignmentNum<i64>(i, iless, imore, ii, "i64"); + + DoCheckAssignmentNum<float>(i, iless, imore, ii, "float"); + DoCheckAssignmentNum<double>(i, iless, imore, ii, "double"); + } + + // DoCheckAssignment<bool>(true, true, true, "true", "bool"); + // DoCheckAssignment<bool>(false, false, false, "false", "bool"); + + for (int i = 1; i < 3; ++i) { + TString ii = ToString(i); + int iless = i - 1; + int imore = i + 1; + + DoCheckAssignmentNum<char>(i, iless, imore, ii, "char"); + + DoCheckAssignmentNum<signed char>(i, iless, imore, ii, "schar"); + DoCheckAssignmentNum<short>(i, iless, imore, ii, "short"); + DoCheckAssignmentNum<int>(i, iless, imore, ii, "int"); + DoCheckAssignmentNum<long>(i, iless, imore, ii, "long"); + DoCheckAssignmentNum<long long>(i, iless, imore, ii, "longlong"); + DoCheckAssignmentNum<i8>(i, iless, imore, ii, "i8"); + DoCheckAssignmentNum<i16>(i, iless, imore, ii, "i16"); + DoCheckAssignmentNum<i32>(i, iless, imore, ii, "i32"); + DoCheckAssignmentNum<i64>(i, iless, imore, ii, "i64"); + + DoCheckAssignmentNum<unsigned char>(i, iless, imore, ii, "uchar"); + DoCheckAssignmentNum<unsigned short>(i, iless, imore, ii, "ushort"); + DoCheckAssignmentNum<unsigned int>(i, iless, imore, ii, "uint"); + DoCheckAssignmentNum<unsigned long>(i, iless, imore, ii, "ulong"); + DoCheckAssignmentNum<unsigned long long>(i, iless, imore, ii, "ulonglong"); + DoCheckAssignmentNum<ui8>(i, iless, imore, ii, "ui8"); + DoCheckAssignmentNum<ui16>(i, iless, imore, ii, "ui16"); + DoCheckAssignmentNum<ui32>(i, iless, imore, ii, "ui32"); + DoCheckAssignmentNum<ui64>(i, iless, imore, ii, "ui64"); + + DoCheckAssignmentNum<float>(i, iless, imore, ii, "float"); + DoCheckAssignmentNum<double>(i, iless, imore, ii, "double"); + } + + TString uuu = "uuu"; + TString uua = "uua"; + TString uuz = "uuz"; + DoCheckAssignment<char*>(uuu.begin(), uua.begin(), uuz.begin(), "\"uuu\"", "char*"); + DoCheckAssignment<const char*>("www", "wwa", "wwz", "\"www\"", "const char*"); + DoCheckAssignmentArr("xxx", "xxa", "xxz", "\"xxx\"", "const char[]"); + DoCheckAssignment<TStringBuf>("yyy", "yya", "yyz", "\"yyy\"", "TStringBuf"); + +#if defined(_MSC_VER) + //TODO +#else + DoCheckAssignment<TString>("ttt", "tta", "ttz", "\"ttt\"", "TString"); +#endif + + NSc::TValue v; + v.SetDict(); + DoCheckAssignment<NSc::TValue>(v, v, v, "{}", "TValue"); + DoCheckAssignment<NSc::TValue&>(v, v, v, "{}", "TValue&"); + DoCheckAssignment<const NSc::TValue&>(v, v, v, "{}", "const TValue&"); + + NSc::TValue v1{1}; + UNIT_ASSERT_VALUES_EQUAL(v1.ToJson(), "1"); + } + + Y_UNIT_TEST(TestAssignmentDictChild) { + { + NSc::TValue v; + { + NSc::TValue b; + v["a"] = b; + } + v = v["a"]; + } + { + NSc::TValue v; + { + NSc::TValue b; + v["a"] = b; + } + v = v.Get("a"); + } + { + NSc::TValue v; + { + NSc::TValue b; + v["a"] = b; + } + v = std::move(v["a"]); + } + } + + Y_UNIT_TEST(TestInsert) { + NSc::TValue v; + v.Insert(0, "b"); + v.Insert(0, "a"); + v.Insert(2, "d"); + v.Insert(2, "c"); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d"])"); + + v.AppendAll({1, 2, 3}); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d",1,2,3])"); + + TVector<int> d{4, 5, 6}; + v.AppendAll(d); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d",1,2,3,4,5,6])"); + UNIT_ASSERT_VALUES_EQUAL(d.size(), 3u); + TVector<TStringBuf> s{"x", "y", "z"}; + v.AppendAll(s.begin(), s.end()); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d",1,2,3,4,5,6,"x","y","z"])"); + + UNIT_ASSERT_VALUES_EQUAL(v.Clone().Clear().AppendAll(s).ToJson(), R"(["x","y","z"])"); + UNIT_ASSERT_VALUES_EQUAL(v.Clone().Clear().AppendAll(TVector<TStringBuf>()).ToJson(), R"([])"); + + v.AddAll({{"a", "b"}, {"c", "d"}}); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"({"a":"b","c":"d"})"); + } + + Y_UNIT_TEST(TestFrontBack) { + NSc::TValue v; + const NSc::TValue& vv = v; + UNIT_ASSERT(NSc::TValue::Same(vv.Front(), NSc::Null())); + UNIT_ASSERT(NSc::TValue::Same(vv.Back(), NSc::Null())); + UNIT_ASSERT(!vv.IsArray()); + v.Back() = "a"; + UNIT_ASSERT_VALUES_EQUAL("a", vv.Front().GetString()); + UNIT_ASSERT_VALUES_EQUAL("a", vv.Back().GetString()); + UNIT_ASSERT(vv.IsArray()); + v.Push("b"); + UNIT_ASSERT_VALUES_EQUAL("a", vv.Front().GetString()); + UNIT_ASSERT_VALUES_EQUAL("b", vv.Back().GetString()); + + UNIT_ASSERT_VALUES_EQUAL("b", v.Pop().GetString()); + + UNIT_ASSERT_VALUES_EQUAL("a", vv.Front().GetString()); + UNIT_ASSERT_VALUES_EQUAL("a", vv.Back().GetString()); + + UNIT_ASSERT_VALUES_EQUAL("a", v.Pop().GetString()); + + UNIT_ASSERT(NSc::TValue::Same(vv.Front(), NSc::Null())); + UNIT_ASSERT(NSc::TValue::Same(vv.Back(), NSc::Null())); + + v.Front() = "a"; + UNIT_ASSERT_VALUES_EQUAL("a", vv.Front().GetString()); + UNIT_ASSERT_VALUES_EQUAL("a", vv.Back().GetString()); + } + + Y_UNIT_TEST(TestAssign) { + NSc::TValue v; + v.SetArray(); + v.Push() = "test"; + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "[\"test\"]"); + UNIT_ASSERT(NSc::TValue::SamePool(v[0], v)); + } + + NSc::TValue MutableRef(const NSc::TValue& v) { + return v; + } + + NSc::TValue Clone(const NSc::TValue& v) { + NSc::TValue v1 = v.Clone(); + UNIT_ASSERT_VALUES_EQUAL(v1.ToJson(true), v.ToJson(true)); + return v1; + } + + Y_UNIT_TEST(TestCOW) { + NSc::TValue vd = NSc::TValue::FromJson("{ a : 1, b : c}"); + NSc::TValue va = NSc::TValue::FromJson("[ x, y]"); + NSc::TValue vs = NSc::TValue::FromJson("foo"); + NSc::TValue vn = NSc::TValue::FromJson("1"); + TString sd = "{\"a\":1,\"b\":\"c\"}"; + TString sa = "[\"x\",\"y\"]"; + TString ss = "\"foo\""; + TString sn = "1"; + UNIT_ASSERT_VALUES_EQUAL(sd, vd.ToJson(true)); + UNIT_ASSERT_VALUES_EQUAL(sa, va.ToJson(true)); + UNIT_ASSERT_VALUES_EQUAL(ss, vs.ToJson(true)); + UNIT_ASSERT_VALUES_EQUAL(sn, vn.ToJson(true)); + + { + NSc::TValue v2 = MutableRef(vn); + v2 = -1; + + UNIT_ASSERT_VALUES_EQUAL(sn, vn.ToJson(true)); + + NSc::TValue v3 = Clone(vn); + v3 = -1; + + UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); + } + + { + NSc::TValue v2 = MutableRef(vs); + v2 = "xxx"; + + UNIT_ASSERT_VALUES_EQUAL(ss, vs.ToJson(true)); + + NSc::TValue v3 = Clone(vs); + v3 = "xxx"; + + UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); + } + + { + NSc::TValue v2 = MutableRef(vd); + v2["a"] = "zzz"; + + UNIT_ASSERT_VALUES_EQUAL(sd, vd.ToJson(true)); + + NSc::TValue v3 = Clone(vd); + v3["a"] = "zzz"; + + UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); + } + + { + NSc::TValue v2 = MutableRef(va); + v2[0] = "zzz"; + + UNIT_ASSERT_VALUES_EQUAL(sa, va.ToJson(true)); + + NSc::TValue v3 = Clone(va); + v3[0] = "zzz"; + + UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); + } + + { + NSc::TValue v2 = MutableRef(va); + + UNIT_ASSERT_VALUES_EQUAL(sa, va.ToJson(true)); + + NSc::TValue v3 = Clone(va); + + UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); + } + + { + NSc::TValue v2 = MutableRef(va); + v2.ClearArray(); + + UNIT_ASSERT_VALUES_EQUAL(sa, va.ToJson(true)); + + NSc::TValue v3 = Clone(va); + v3.ClearArray(); + + UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); + } + } + + Y_UNIT_TEST(TestOperators) { + UNIT_ASSERT("test" == NSc::TValue("test")); + UNIT_ASSERT(NSc::TValue("test") == "test"); + + UNIT_ASSERT("test1" != NSc::TValue("test")); + UNIT_ASSERT(NSc::TValue("test") != "test1"); + + UNIT_ASSERT("test1" > NSc::TValue("test")); + UNIT_ASSERT(NSc::TValue("test") < "test1"); + + UNIT_ASSERT("test" < NSc::TValue("test1")); + UNIT_ASSERT(NSc::TValue("test1") > "test"); + + UNIT_ASSERT(1 == NSc::TValue(1)); + UNIT_ASSERT(NSc::TValue(1) == 1); + + UNIT_ASSERT(2 != NSc::TValue(1)); + UNIT_ASSERT(NSc::TValue(1) != 2); + + UNIT_ASSERT(1 < NSc::TValue(2)); + UNIT_ASSERT(NSc::TValue(2) > 1); + + UNIT_ASSERT(2 > NSc::TValue(1)); + UNIT_ASSERT(NSc::TValue(1) < 2); + + UNIT_ASSERT(TString("test") == NSc::TValue("test")); + } + + Y_UNIT_TEST(TestDestructor) { + NSc::TValue v; + const NSc::TValue& v1 = v; + v1.GetString(); + v1.GetArray(); + v1.GetNumber(); + v1.GetDict(); + v.GetString(); + v.GetArray(); + v.GetNumber(); + v.GetDict(); + } + + void DoTestSamePool(TStringBuf json, TStringBuf jpath) { + NSc::TValue v = NSc::TValue::FromJson(json); + UNIT_ASSERT_C(NSc::TValue::SamePool(v, v.TrySelect(jpath)), json); + } + + Y_UNIT_TEST(TestSamePool) { + DoTestSamePool("", ""); + DoTestSamePool("a", ""); + DoTestSamePool("[a]", "0"); + DoTestSamePool("{a:b}", "a"); + DoTestSamePool("{a:{b:c}}", "a/b"); + DoTestSamePool("{a:{b:[c, {}]}}", "a/b/1"); + DoTestSamePool("{a:{b:[c, {d:{e:[]}}]}}", "a/b/1/d/e"); + UNIT_ASSERT(!NSc::TValue::SamePool(NSc::TValue(), NSc::TValue())); + UNIT_ASSERT(!NSc::TValue::SamePool(NSc::Null().Clone(), NSc::Null())); + UNIT_ASSERT(!NSc::TValue::SamePool(NSc::TValue() = 0, NSc::TValue())); + UNIT_ASSERT(!NSc::TValue::SamePool(NSc::TValue::FromJson("a"), NSc::TValue::FromJson("a"))); + NSc::TValue v, vv; + v["x"] = vv; + UNIT_ASSERT(!NSc::TValue::SamePool(v, vv)); + UNIT_ASSERT(!NSc::TValue::SamePool(v, v["x"])); + v = vv; + UNIT_ASSERT(NSc::TValue::SamePool(v, vv)); + } + + Y_UNIT_TEST(TestLoopDetection) { + NSc::NImpl::GetTlsInstance<NSc::NImpl::TSelfLoopContext>().ReportingMode + = NSc::NImpl::TSelfLoopContext::EMode::Stderr; + + NSc::TValue x; + + x["a"]["x"] = x; + x["b"][0] = x; + + UNIT_ASSERT(x.IsSameOrAncestorOf(x)); + UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); + UNIT_ASSERT(x.Get("a").IsSameOrAncestorOf(x)); + + UNIT_ASSERT_VALUES_EQUAL(x.ToJson(), "{\"a\":{\"x\":null},\"b\":[null]}"); + + NSc::TValue y = x.Clone(); + + UNIT_ASSERT(y.Has("a")); + UNIT_ASSERT(y.Get("a").Has("x")); + UNIT_ASSERT(y.Get("a").Get("x").IsNull()); + UNIT_ASSERT(y.Get("a").Get("x").IsNull()); + + UNIT_ASSERT(y.Has("b")); + UNIT_ASSERT(y.Get("b").Has(0)); + UNIT_ASSERT(y.Get("b").Get(0).IsNull()); + + UNIT_ASSERT_VALUES_EQUAL(y.ToJson(), "{\"a\":{\"x\":null},\"b\":[null]}"); + + NSc::TValue z; + z.MergeUpdate(x); + + UNIT_ASSERT(z.Has("a")); + UNIT_ASSERT(z.Get("a").Has("x")); + UNIT_ASSERT(z.Get("a").Get("x").IsNull()); + + UNIT_ASSERT(z.Has("b")); + UNIT_ASSERT(z.Get("b").Has(0)); + UNIT_ASSERT(z.Get("b").Get(0).IsNull()); + + UNIT_ASSERT_VALUES_EQUAL(z.ToJson(), "{\"a\":{\"x\":null},\"b\":[null]}"); + + x["a"].Delete("x"); + x["b"].Delete(0); + + UNIT_ASSERT(x.IsSameOrAncestorOf(x)); + UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); + UNIT_ASSERT(!x.Get("a").IsSameOrAncestorOf(x)); + } + + Y_UNIT_TEST(TestLoopDetectionThrow) { + NSc::NImpl::GetTlsInstance<NSc::NImpl::TSelfLoopContext>().ReportingMode + = NSc::NImpl::TSelfLoopContext::EMode::Throw; + + { + NSc::TValue x; + x["a"]["x"] = x; + + UNIT_ASSERT(x.IsSameOrAncestorOf(x)); + UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); + UNIT_ASSERT(x.Get("a").IsSameOrAncestorOf(x)); + + UNIT_ASSERT_EXCEPTION(x.ToJson(), NSc::TSchemeException); + UNIT_ASSERT_EXCEPTION(x.Clone(), NSc::TSchemeException); + + NSc::TValue z; + UNIT_ASSERT_EXCEPTION(z.MergeUpdate(x), NSc::TSchemeException); + + UNIT_ASSERT_VALUES_EQUAL(z.ToJson(), "{\"a\":{\"x\":null}}"); + + x["a"].Delete("x"); + + UNIT_ASSERT(x.IsSameOrAncestorOf(x)); + UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); + UNIT_ASSERT(!x.Get("a").IsSameOrAncestorOf(x)); + + UNIT_ASSERT_VALUES_EQUAL(x.ToJson(), "{\"a\":{}}"); + } + + { + NSc::TValue x; + x["a"][0] = x; + + UNIT_ASSERT(x.IsSameOrAncestorOf(x)); + UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); + UNIT_ASSERT(x.Get("a").IsSameOrAncestorOf(x)); + + UNIT_ASSERT_EXCEPTION(x.ToJson(), NSc::TSchemeException); + UNIT_ASSERT_EXCEPTION(x.Clone(), NSc::TSchemeException); + + NSc::TValue z; + UNIT_ASSERT_EXCEPTION(z.MergeUpdate(x), NSc::TSchemeException); + + UNIT_ASSERT_VALUES_EQUAL(z.ToJson(), "{\"a\":[null]}"); + + x["a"].Delete(0); + + UNIT_ASSERT(x.IsSameOrAncestorOf(x)); + UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); + UNIT_ASSERT(!x.Get("a").IsSameOrAncestorOf(x)); + + UNIT_ASSERT_VALUES_EQUAL(x.ToJson(), "{\"a\":[]}"); + } + } + + Y_UNIT_TEST(TestIsSameOrAncestorOf) { + NSc::TValue x; + UNIT_ASSERT(x.IsSameOrAncestorOf(x)); + + x["a"] = NSc::Null(); + UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); + + NSc::TValue a = 1; + NSc::TValue b = 2; + NSc::TValue c = 3; + NSc::TValue d = 4; + + x["a"] = a; + x["b"] = b; + x["c"] = a; + UNIT_ASSERT(x.IsSameOrAncestorOf(a)); + UNIT_ASSERT(x.IsSameOrAncestorOf(b)); + UNIT_ASSERT(x.Get("a").IsSameOrAncestorOf(a)); + UNIT_ASSERT(x.Get("b").IsSameOrAncestorOf(b)); + UNIT_ASSERT(x.Get("c").IsSameOrAncestorOf(a)); + + UNIT_ASSERT(!x.Get("a").IsSameOrAncestorOf(b)); + UNIT_ASSERT(!x.Get("b").IsSameOrAncestorOf(a)); + + UNIT_ASSERT(!x.Get("a").Get(0).IsSameOrAncestorOf(a)); + + b.Push() = c; + b.Push() = d; + b.Push() = c; + + UNIT_ASSERT(x.Get("b").IsSameOrAncestorOf(b)); + UNIT_ASSERT(x.IsSameOrAncestorOf(c)); + UNIT_ASSERT(x.IsSameOrAncestorOf(d)); + UNIT_ASSERT(x.Get("b").Get(0).IsSameOrAncestorOf(c)); + UNIT_ASSERT(x.Get("b").Get(1).IsSameOrAncestorOf(d)); + UNIT_ASSERT(x.Get("b").Get(2).IsSameOrAncestorOf(c)); + + UNIT_ASSERT(b.Get(0).IsSameOrAncestorOf(b.Get(2))); + UNIT_ASSERT(b.Get(2).IsSameOrAncestorOf(b.Get(0))); + UNIT_ASSERT(b.Get(0).IsSameOrAncestorOf(c)); + UNIT_ASSERT(b.Get(1).IsSameOrAncestorOf(d)); + + UNIT_ASSERT(!b.Get(0).IsSameOrAncestorOf(d)); + UNIT_ASSERT(!b.Get(1).IsSameOrAncestorOf(c)); + } + + static void ByVal(NSc::TValue v) { + v["VAL"] = 1; + } + + static void ByRef(NSc::TValue& v) { + ByVal(v); + } + + static void ByRefAndModify(NSc::TValue& v) { + v["REF"] = 1; + ByVal(v); + } + + Y_UNIT_TEST(TestMove) { + using namespace NSc; + { + TValue v = TValue::FromJson("{}"); + ByRef(v); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"VAL\":1}"); + } + { + TValue v = TValue::FromJson("{}"); + ByVal(v); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"VAL\":1}"); + } + { + TValue v; + ByVal(v); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"VAL\":1}"); + } + { + TValue v = TValue::FromJson("{}"); + ByRefAndModify(v); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"REF\":1,\"VAL\":1}"); + } + { + TValue v = TValue::FromJson("{foo:bar}"); + TValue w(std::move(v)); + UNIT_ASSERT(v.IsNull()); + v = static_cast<TValue&>(v); + UNIT_ASSERT(v.IsNull()); + UNIT_ASSERT_VALUES_EQUAL(w.Get("foo").GetString(), "bar"); + v = std::move(w); + UNIT_ASSERT_VALUES_EQUAL(v.Get("foo").GetString(), "bar"); + UNIT_ASSERT(w.IsNull()); + UNIT_ASSERT(w.Get("foo").IsNull()); // no crash here + w["foo"] = "baz"; // no crash here + UNIT_ASSERT(w.IsDict()); + UNIT_ASSERT_VALUES_EQUAL(w.Get("foo").GetString(), "baz"); + } + UNIT_ASSERT(NSc::TValue::DefaultValue().IsNull()); + } + + //SPI-25156 + Y_UNIT_TEST(TestMoveNotCorruptingDefault) { + using namespace NSc; + TValue w = TValue::FromJson("{foo:bar}"); + TValue v = std::move(w); + w["foo"] = "baz"; // no crash here + UNIT_ASSERT(NSc::TValue::DefaultValue().IsNull()); + } + + Y_UNIT_TEST(TestCopyFrom) { + { + TString sa = "[1,2]"; + const NSc::TValue& va = NSc::TValue::FromJson(sa); + NSc::TValue vb = va; + vb.CopyFrom(va); + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); + } + { + TString sa = "[1,2]"; + NSc::TValue va = NSc::TValue::FromJson(sa); + NSc::TValue vb = va; + vb.CopyFrom(va); + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); + } + { + TString sa = "[1,2]"; + NSc::TValue va = NSc::TValue::FromJson(sa); + const NSc::TValue& vb = va; + va.CopyFrom(vb); + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); + } + { + TString sa = "[1,2]"; + NSc::TValue va = NSc::TValue::FromJson(sa); + NSc::TValue vb = va; + va.CopyFrom(vb); + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); + } + { + NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"ab\",\"y\":{\"p\":\"cd\",\"q\":\"ef\"}}"); + NSc::TValue vb = va.Get("y"); + va.CopyFrom(vb); + TString sa = "{\"p\":\"cd\",\"q\":\"ef\"}"; + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); + } + { + NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"ab\",\"y\":{\"p\":\"cd\",\"q\":\"ef\"}}"); + const NSc::TValue& vb = va.Get("y"); + va.CopyFrom(vb); + TString sa = "{\"p\":\"cd\",\"q\":\"ef\"}"; + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + } + { + NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"ab\",\"y\":{\"p\":\"cd\",\"q\":\"ef\"}}"); + NSc::TValue vb = va.Get("y"); + va = vb; + TString sa = "{\"p\":\"cd\",\"q\":\"ef\"}"; + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); + } + { + NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"ab\",\"y\":{\"p\":\"cd\",\"q\":\"ef\"}}"); + const NSc::TValue& vb = va.Get("y"); + va = vb; + TString sa = "{\"p\":\"cd\",\"q\":\"ef\"}"; + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); + } + } + + Y_UNIT_TEST(TestCopyingDictIntoSelf) { //Found by fuzzing + NSc::TValue a; + NSc::TValue b = a.GetOrAdd("aa"); + b.CopyFrom(a); + NSc::TValue target = NSc::TValue::FromJsonThrow("{\"aa\":null}"); + UNIT_ASSERT_VALUES_EQUAL(b, target); + UNIT_ASSERT_VALUES_EQUAL(a, target); + } + + Y_UNIT_TEST(TestCopyingDictIntoSelfByRef) { //Found by fuzzing + NSc::TValue a; + NSc::TValue& b = a.GetOrAdd("aa"); + b.CopyFrom(a); + UNIT_ASSERT_VALUES_EQUAL(b, NSc::TValue::FromJsonThrow("{\"aa\":null}")); + UNIT_ASSERT_VALUES_EQUAL(a, NSc::TValue::FromJsonThrow("{\"aa\": {\"aa\": null}}")); + } + + Y_UNIT_TEST(TestGetNoAdd) { + NSc::TValue v = NSc::NUt::AssertFromJson("{a:[null,-1,2,3.4],b:3,c:{d:5}}"); + UNIT_ASSERT(v.GetNoAdd("a") != nullptr); + UNIT_ASSERT(v.GetNoAdd("b") != nullptr); + UNIT_ASSERT(v.GetNoAdd("c") != nullptr); + UNIT_ASSERT(v.GetNoAdd("d") == nullptr); + UNIT_ASSERT(v.GetNoAdd("value") == nullptr); + + NSc::TValue* child = v.GetNoAdd("c"); + UNIT_ASSERT(child != nullptr); + (*child)["e"]["f"] = 42; + const NSc::TValue expectedResult = NSc::NUt::AssertFromJson("{a:[null,-1,2,3.4],b:3,c:{d:5,e:{f:42}}}"); + UNIT_ASSERT_VALUES_EQUAL(v, expectedResult); + } +}; diff --git a/library/cpp/scheme/tests/ut/scheme_ut.proto b/library/cpp/scheme/tests/ut/scheme_ut.proto new file mode 100644 index 00000000000..7981af7eae0 --- /dev/null +++ b/library/cpp/scheme/tests/ut/scheme_ut.proto @@ -0,0 +1,84 @@ +package NSc; + +message TMessage { + optional double Double = 1; + optional float Float = 2; + optional int32 Int32 = 3; + optional int64 Int64 = 4; + optional uint32 UInt32 = 5; + optional uint64 UInt64 = 6; + optional sint32 SInt32 = 7; + optional sint64 SInt64 = 8; + optional fixed32 Fixed32 = 9; + optional fixed64 Fixed64 = 10; + optional sfixed32 SFixed32 = 11; + optional sfixed64 SFixed64 = 12; + optional bool Bool = 13; + optional string String = 14; + optional bytes Bytes = 15; + optional EEnum Enum = 16; + optional TMessage2 Message = 17; + + repeated double Doubles = 18; + repeated float Floats = 19; + repeated int32 Int32s = 20; + repeated int64 Int64s = 21; + repeated uint32 UInt32s = 22; + repeated uint64 UInt64s = 23; + repeated sint32 SInt32s = 24; + repeated sint64 SInt64s = 25; + repeated fixed32 Fixed32s = 26; + repeated fixed64 Fixed64s = 27; + repeated sfixed32 SFixed32s = 28; + repeated sfixed64 SFixed64s = 29; + repeated bool Bools = 30; + repeated string Strings = 31; + repeated bytes Bytess = 32; + repeated EEnum Enums = 33; + repeated TMessage2 Messages = 34; + + map<string, double> MapDoubles = 35; + map<string, int32> MapInt32s = 36; + map<string, string> MapString = 37; +} + +enum EEnum { + VALUE1 = 0; + VALUE2 = 1; +} + +message TMessage2 { + optional double Double = 1; + optional float Float = 2; + optional int32 Int32 = 3; + optional int64 Int64 = 4; + optional uint32 UInt32 = 5; + optional uint64 UInt64 = 6; + optional sint32 SInt32 = 7; + optional sint64 SInt64 = 8; + optional fixed32 Fixed32 = 9; + optional fixed64 Fixed64 = 10; + optional sfixed32 SFixed32 = 11; + optional sfixed64 SFixed64 = 12; + optional bool Bool = 13; + optional string String = 14; + optional bytes Bytes = 15; + optional EEnum Enum = 16; + + repeated double Doubles = 18; + repeated float Floats = 19; + repeated int32 Int32s = 20; + repeated int64 Int64s = 21; + repeated uint32 UInt32s = 22; + repeated uint64 UInt64s = 23; + repeated sint32 SInt32s = 24; + repeated sint64 SInt64s = 25; + repeated fixed32 Fixed32s = 26; + repeated fixed64 Fixed64s = 27; + repeated sfixed32 SFixed32s = 28; + repeated sfixed64 SFixed64s = 29; + repeated bool Bools = 30; + repeated string Strings = 31; + repeated bytes Bytess = 32; + repeated EEnum Enums = 33; +} diff --git a/library/cpp/scheme/tests/ut/ya.make b/library/cpp/scheme/tests/ut/ya.make new file mode 100644 index 00000000000..9f547914146 --- /dev/null +++ b/library/cpp/scheme/tests/ut/ya.make @@ -0,0 +1,24 @@ +UNITTEST() + +OWNER(velavokr) + +PEERDIR( + library/cpp/protobuf/util + library/cpp/scheme/tests/fuzz_ops/lib + library/cpp/scheme/ut_utils + library/cpp/string_utils/quote + library/cpp/testing/unittest +) + +SRCS( + fuzz_ops_found_bugs_ut.cpp + scheme_cast_ut.cpp + scheme_json_ut.cpp + scheme_merge_ut.cpp + scheme_path_ut.cpp + scheme_proto_ut.cpp + scheme_ut.cpp + scheme_ut.proto +) + +END() diff --git a/library/cpp/scheme/tests/ya.make b/library/cpp/scheme/tests/ya.make new file mode 100644 index 00000000000..741cc9a2da6 --- /dev/null +++ b/library/cpp/scheme/tests/ya.make @@ -0,0 +1,13 @@ +OWNER( + g:blender + g:middle + g:upper + velavokr +) + +RECURSE( + fuzz_json + fuzz_ops + fuzz_ops/ut + ut +) diff --git a/library/cpp/scheme/ut_utils/scheme_ut_utils.cpp b/library/cpp/scheme/ut_utils/scheme_ut_utils.cpp new file mode 100644 index 00000000000..0bbdab10e8c --- /dev/null +++ b/library/cpp/scheme/ut_utils/scheme_ut_utils.cpp @@ -0,0 +1,44 @@ +#include "scheme_ut_utils.h" + +#include <library/cpp/colorizer/colors.h> + +#include <util/stream/str.h> + +namespace NSc { + namespace NUt { + NSc::TValue AssertFromJson(TStringBuf val) { + try { + return TValue::FromJsonThrow(val); + } catch (const TSchemeParseException& e) { + TStringStream s; + NColorizer::TColors colors; + s << "\n" + << colors.YellowColor() << "Reason:" << colors.OldColor() << "\n" + << e.Reason; + s << "\n" + << colors.YellowColor() << "Where:" << colors.OldColor() << "\n" + << val.SubStr(0, e.Offset) << colors.RedColor() << val.SubStr(e.Offset) << colors.OldColor() << "\n"; + UNIT_FAIL_IMPL("could not parse json", s.Str()); + return NSc::Null(); + } catch (const yexception& e) { + TStringStream s; + s << '\n' + << val; + UNIT_FAIL_IMPL("could not parse json", s.Str()); + return NSc::Null(); + } + } + + void AssertScheme(const TValue& expected, const TValue& actual) { + UNIT_ASSERT_JSON_EQ_JSON(actual, expected); + } + + void AssertSchemeJson(TStringBuf expected, const NSc::TValue& actual) { + UNIT_ASSERT_JSON_EQ_JSON(actual, expected); + } + + void AssertJsonJson(TStringBuf expected, TStringBuf actual) { + UNIT_ASSERT_JSON_EQ_JSON(actual, expected); + } + } +} diff --git a/library/cpp/scheme/ut_utils/scheme_ut_utils.h b/library/cpp/scheme/ut_utils/scheme_ut_utils.h new file mode 100644 index 00000000000..eb3ea15b2ab --- /dev/null +++ b/library/cpp/scheme/ut_utils/scheme_ut_utils.h @@ -0,0 +1,55 @@ +#pragma once + +#include <library/cpp/json/json_prettifier.h> +#include <library/cpp/scheme/scheme.h> +#include <library/cpp/json/json_value.h> +#include <library/cpp/json/json_writer.h> +#include <library/cpp/testing/unittest/registar.h> +#include <util/string/cast.h> + +namespace NSc { + namespace NUt { + TValue AssertFromJson(TStringBuf json); + + inline TString NormalizeJson(const NSc::TValue& sc) { + return sc.ToJson(true); + } + + inline TString NormalizeJson(const NJson::TJsonValue& sc) { + return NJson::WriteJson(sc, false, true, false); + } + + template <class TStr> + inline TString NormalizeJson(const TStr& val) { + return AssertFromJson(val).ToJson(true); + } + +#define UNIT_ASSERT_JSON_EQ_JSON_C(A, B, c) \ + do { \ + const TString _a = NSc::NUt::NormalizeJson(A); \ + const TString _b = NSc::NUt::NormalizeJson(B); \ + if (_a != _b) { \ + UNIT_FAIL_IMPL( \ + "json values are different (" #A " != " #B ")", \ + Sprintf("%s\n!=\n%s\n%s\n%s", _a.data(), _b.data(), \ + ::NUnitTest::ColoredDiff(NJson::PrettifyJson(_a), NJson::PrettifyJson(_b), " \t\n,:\"{}[]").data(), ToString(c).data())); \ + } \ + } while (false) + +#define UNIT_ASSERT_JSON_EQ_JSON(A, B) UNIT_ASSERT_JSON_EQ_JSON_C(A, B, "") + + inline TString DumpJson(const TValue& json) { + return NJson::CompactifyJson(json.ToJson(true), true, true); + } + + // deprecated + inline TString DumpJsonVS(const TValue& expected, const TValue& fact) { + return DumpJson(expected) + "(expected) != (fact)" + DumpJson(fact); + } + + void AssertScheme(const TValue& expected, const TValue& real); + void AssertSchemeJson(TStringBuf expected, const TValue& real); + void AssertJsonJson(TStringBuf expected, TStringBuf real); + + } +} diff --git a/library/cpp/scheme/ut_utils/ya.make b/library/cpp/scheme/ut_utils/ya.make new file mode 100644 index 00000000000..7661262e1b7 --- /dev/null +++ b/library/cpp/scheme/ut_utils/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +OWNER(velavokr) + +SRCS( + scheme_ut_utils.cpp +) + +PEERDIR( + library/cpp/colorizer + library/cpp/json + library/cpp/scheme + library/cpp/testing/unittest +) + +END() diff --git a/library/cpp/scheme/util/scheme_holder.cpp b/library/cpp/scheme/util/scheme_holder.cpp new file mode 100644 index 00000000000..33232acc500 --- /dev/null +++ b/library/cpp/scheme/util/scheme_holder.cpp @@ -0,0 +1 @@ +#include "scheme_holder.h" diff --git a/library/cpp/scheme/util/scheme_holder.h b/library/cpp/scheme/util/scheme_holder.h new file mode 100644 index 00000000000..f2fa16d1cdb --- /dev/null +++ b/library/cpp/scheme/util/scheme_holder.h @@ -0,0 +1,76 @@ +#pragma once + +#include <library/cpp/scheme/scheme.h> + + +// Scheme adapter that holds referenced value +template <typename TScheme> +class TSchemeHolder { +public: + TSchemeHolder() + : Value_(NSc::Null()) + , Scheme_(&Value_) + { + } + + explicit TSchemeHolder(NSc::TValue&& value) + : Value_(std::move(value)) + , Scheme_(&Value_) + { + } + + explicit TSchemeHolder(const NSc::TValue& value) + : Value_(value) + , Scheme_(&Value_) + { + } + + TSchemeHolder(TSchemeHolder<TScheme>&& rhs) + : Value_(std::move(rhs.Value_)) + , Scheme_(&Value_) + { + } + + TSchemeHolder(const TSchemeHolder<TScheme>& rhs) + : Value_(rhs.Value_) + , Scheme_(&Value_) + { + } + + TSchemeHolder<TScheme>& operator=(TSchemeHolder<TScheme>&& rhs) { + Value_ = std::move(rhs.Value_); + return *this; + } + TSchemeHolder<TScheme>& operator=(const TSchemeHolder<TScheme>& rhs) { + Value_ = rhs.Value_; + return *this; + } + + TScheme& Scheme() { + return Scheme_; + } + const TScheme& Scheme() const { + return Scheme_; + } + TScheme& operator->() { + return Scheme_; + } + const TScheme& operator->() const { + return Scheme_; + } + + NSc::TValue& Value() { + return Value_; + } + const NSc::TValue& Value() const { + return Value_; + } + + bool IsNull() const { + return Value_.IsNull(); + } + +private: + NSc::TValue Value_; + TScheme Scheme_; +}; diff --git a/library/cpp/scheme/util/utils.cpp b/library/cpp/scheme/util/utils.cpp new file mode 100644 index 00000000000..40cceceacd7 --- /dev/null +++ b/library/cpp/scheme/util/utils.cpp @@ -0,0 +1,23 @@ +#include "utils.h" + +#include <library/cpp/string_utils/base64/base64.h> + +namespace NScUtils { + +void CopyField(const NSc::TValue& from, NSc::TValue& to) { + to = from; +} + +} // namespace NScUtils + +void TSerializer<NSc::TValue>::Save(IOutputStream* out, const NSc::TValue& v) { + TString json = Base64Encode(v.ToJson()); + ::Save(out, json); +} + +void TSerializer<NSc::TValue>::Load(IInputStream* in, NSc::TValue& v) { + TString json; + ::Load(in, json); + json = Base64Decode(json); + v = NSc::TValue::FromJsonThrow(json); +} diff --git a/library/cpp/scheme/util/utils.h b/library/cpp/scheme/util/utils.h new file mode 100644 index 00000000000..f7d666f67a2 --- /dev/null +++ b/library/cpp/scheme/util/utils.h @@ -0,0 +1,23 @@ +#pragma once + +#include <library/cpp/scheme/scheme.h> + +#include <util/generic/strbuf.h> +#include <util/ysaveload.h> + +namespace NScUtils { + +void CopyField(const NSc::TValue& from, NSc::TValue& to); + +template <typename... Args> +void CopyField(const NSc::TValue& from, NSc::TValue& to, TStringBuf path, Args... args) { + CopyField(from[path], to[path], args...); +} + +} // namespace NScUtils + +template<> +struct TSerializer<NSc::TValue> { + static void Save(IOutputStream* out, const NSc::TValue& v); + static void Load(IInputStream* in, NSc::TValue& v); +}; diff --git a/library/cpp/scheme/util/ya.make b/library/cpp/scheme/util/ya.make new file mode 100644 index 00000000000..dffd1c80702 --- /dev/null +++ b/library/cpp/scheme/util/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +OWNER(g:cards_serivce) + +SRCS( + scheme_holder.cpp + utils.cpp +) + +PEERDIR( + library/cpp/string_utils/base64 +) + +END() diff --git a/library/cpp/scheme/ya.make b/library/cpp/scheme/ya.make new file mode 100644 index 00000000000..bac08ba5a4a --- /dev/null +++ b/library/cpp/scheme/ya.make @@ -0,0 +1,25 @@ +LIBRARY() + +OWNER(velavokr) + +SRCS( + scheme.cpp + scheme_cast.h + scimpl.h + scimpl_defs.h + scimpl_private.cpp + scimpl_protobuf.cpp + scimpl_select.rl6 + scimpl_json_read.cpp + scimpl_json_write.cpp +) + +PEERDIR( + contrib/libs/protobuf + library/cpp/json + library/cpp/string_utils/relaxed_escaper +) + +GENERATE_ENUM_SERIALIZATION(scheme.h) + +END() |