diff options
| author | robot-piglet <[email protected]> | 2025-08-08 10:48:03 +0300 |
|---|---|---|
| committer | robot-piglet <[email protected]> | 2025-08-08 11:07:59 +0300 |
| commit | 6399e7ae57f1556e7b777a0fe1d6b0bf3b36b90e (patch) | |
| tree | 7991057328ee56fb23dfb43c2f48b72e1ba7413d /library/cpp/json/ordered_maps | |
| parent | 9fc8652742d52b557fedfd87254a6065c37b27aa (diff) | |
Intermediate changes
commit_hash:369029716d3b4afaec45df28e06b27c781a5b564
Diffstat (limited to 'library/cpp/json/ordered_maps')
16 files changed, 5646 insertions, 0 deletions
diff --git a/library/cpp/json/ordered_maps/json_ordered.cpp b/library/cpp/json/ordered_maps/json_ordered.cpp new file mode 100644 index 00000000000..960064a458f --- /dev/null +++ b/library/cpp/json/ordered_maps/json_ordered.cpp @@ -0,0 +1,517 @@ +#include "json_ordered.h" + +#include "json_value_ordered.h" + +#include <util/string/cast.h> +#include <util/string/strspn.h> +#include <util/generic/algorithm.h> +#include <util/generic/ymath.h> +#include <util/generic/singleton.h> + +namespace NJsonOrderedWriter { + TBuf::TBuf(EHtmlEscapeMode mode, IOutputStream* stream) + : Stream(stream) + , NeedComma(false) + , NeedNewline(false) + , EscapeMode(mode) + , IndentSpaces(0) + , WriteNanAsString(false) + { + Y_ASSERT(mode == HEM_DONT_ESCAPE_HTML || + mode == HEM_ESCAPE_HTML || + mode == HEM_RELAXED || + mode == HEM_UNSAFE); + if (!Stream) { + StringStream.Reset(new TStringStream); + Stream = StringStream.Get(); + } + + Stack.reserve(64); // should be enough for most cases + StackPush(JE_OUTER_SPACE); + } + + static TStringBuf EntityToStr(EJsonEntity e) { + switch (e) { + case JE_OUTER_SPACE: + return "JE_OUTER_SPACE"; + case JE_LIST: + return "JE_LIST"; + case JE_OBJECT: + return "JE_OBJECT"; + case JE_PAIR: + return "JE_PAIR"; + default: + return "JE_unknown"; + } + } + + inline void TBuf::StackPush(EJsonEntity e) { + Stack.push_back(e); + } + + inline EJsonEntity TBuf::StackTop() const { + return Stack.back(); + } + + inline void TBuf::StackPop() { + Y_ASSERT(!Stack.empty()); + const EJsonEntity current = StackTop(); + Stack.pop_back(); + switch (current) { + case JE_OUTER_SPACE: + ythrow TError() << "JSON writer: stack empty"; + case JE_LIST: + PrintIndentation(true); + RawWriteChar(']'); + break; + case JE_OBJECT: + PrintIndentation(true); + RawWriteChar('}'); + break; + case JE_PAIR: + break; + } + NeedComma = true; + NeedNewline = true; + } + + inline void TBuf::CheckAndPop(EJsonEntity e) { + if (Y_UNLIKELY(StackTop() != e)) { + ythrow TError() << "JSON writer: unexpected value " + << EntityToStr(StackTop()) << " on the stack"; + } + StackPop(); + } + + void TBuf::PrintIndentation(bool closing) { + if (!IndentSpaces) + return; + const int indentation = IndentSpaces * (Stack.size() - 1); + if (!indentation && !closing) + return; + + PrintWhitespaces(Max(0, indentation), true); + } + + void TBuf::PrintWhitespaces(size_t count, bool prependWithNewLine) { + static constexpr TStringBuf whitespacesTemplate = "\n "; + static_assert(whitespacesTemplate[0] == '\n'); + static_assert(whitespacesTemplate[1] == ' '); + + count += (prependWithNewLine); + do { + const TStringBuf buffer = whitespacesTemplate.SubString(prependWithNewLine ? 0 : 1, count); + count -= buffer.size(); + UnsafeWriteRawBytes(buffer); + prependWithNewLine = false; // skip '\n' in subsequent writes + } while (count > 0); + } + + inline void TBuf::WriteComma() { + if (NeedComma) { + RawWriteChar(','); + } + NeedComma = true; + + if (NeedNewline) { + PrintIndentation(false); + } + NeedNewline = true; + } + + inline void TBuf::BeginValue() { + if (Y_UNLIKELY(KeyExpected())) { + ythrow TError() << "JSON writer: value written, " + "but expected a key:value pair"; + } + WriteComma(); + } + + inline void TBuf::BeginKey() { + if (Y_UNLIKELY(!KeyExpected())) { + ythrow TError() << "JSON writer: key written outside of an object"; + } + WriteComma(); + StackPush(JE_PAIR); + NeedComma = false; + NeedNewline = false; + } + + inline void TBuf::EndValue() { + if (StackTop() == JE_PAIR) { + StackPop(); + } + } + + TValueContext TBuf::BeginList() { + NeedNewline = true; + BeginValue(); + RawWriteChar('['); + StackPush(JE_LIST); + NeedComma = false; + return TValueContext(*this); + } + + TPairContext TBuf::BeginObject() { + NeedNewline = true; + BeginValue(); + RawWriteChar('{'); + StackPush(JE_OBJECT); + NeedComma = false; + return TPairContext(*this); + } + + TAfterColonContext TBuf::UnsafeWriteKey(const TStringBuf& s) { + BeginKey(); + RawWriteChar('"'); + UnsafeWriteRawBytes(s); + UnsafeWriteRawBytes("\":", 2); + return TAfterColonContext(*this); + } + + TAfterColonContext TBuf::WriteKey(const TStringBuf& s) { + // use the default escaping mode for this object + return WriteKey(s, EscapeMode); + } + + TAfterColonContext TBuf::WriteKey(const TStringBuf& s, EHtmlEscapeMode hem) { + BeginKey(); + WriteBareString(s, hem); + RawWriteChar(':'); + return TAfterColonContext(*this); + } + + TAfterColonContext TBuf::CompatWriteKeyWithoutQuotes(const TStringBuf& s) { + BeginKey(); + Y_ASSERT(AllOf(s, [](char x) { return 'a' <= x && x <= 'z'; })); + UnsafeWriteRawBytes(s); + RawWriteChar(':'); + return TAfterColonContext(*this); + } + + TBuf& TBuf::EndList() { + CheckAndPop(JE_LIST); + EndValue(); + return *this; + } + + TBuf& TBuf::EndObject() { + CheckAndPop(JE_OBJECT); + EndValue(); + return *this; + } + + TValueContext TBuf::WriteString(const TStringBuf& s) { + // use the default escaping mode for this object + return WriteString(s, EscapeMode); + } + + TValueContext TBuf::WriteString(const TStringBuf& s, EHtmlEscapeMode hem) { + BeginValue(); + WriteBareString(s, hem); + EndValue(); + return TValueContext(*this); + } + + TValueContext TBuf::WriteNull() { + UnsafeWriteValue(TStringBuf("null")); + return TValueContext(*this); + } + + TValueContext TBuf::WriteBool(bool b) { + constexpr TStringBuf trueVal = "true"; + constexpr TStringBuf falseVal = "false"; + UnsafeWriteValue(b ? trueVal : falseVal); + return TValueContext(*this); + } + + TValueContext TBuf::WriteInt(int i) { + char buf[22]; // enough to hold any 64-bit number + size_t len = ToString(i, buf, sizeof(buf)); + UnsafeWriteValue(buf, len); + return TValueContext(*this); + } + + TValueContext TBuf::WriteLongLong(long long i) { + static_assert(sizeof(long long) <= 8, "expect sizeof(long long) <= 8"); + char buf[22]; // enough to hold any 64-bit number + size_t len = ToString(i, buf, sizeof(buf)); + UnsafeWriteValue(buf, len); + return TValueContext(*this); + } + + TValueContext TBuf::WriteULongLong(unsigned long long i) { + char buf[22]; // enough to hold any 64-bit number + size_t len = ToString(i, buf, sizeof(buf)); + UnsafeWriteValue(buf, len); + return TValueContext(*this); + } + + template <class TFloat> + TValueContext TBuf::WriteFloatImpl(TFloat f, EFloatToStringMode mode, int ndigits) { + char buf[512]; // enough to hold most floats, the same buffer is used in FloatToString implementation + if (Y_UNLIKELY(!IsValidFloat(f))) { + if (WriteNanAsString) { + const size_t size = FloatToString(f, buf, Y_ARRAY_SIZE(buf)); + WriteString(TStringBuf(buf, size)); + return TValueContext(*this); + } else { + ythrow TError() << "JSON writer: invalid float value: " << FloatToString(f); + } + } + size_t len = FloatToString(f, buf, Y_ARRAY_SIZE(buf), mode, ndigits); + UnsafeWriteValue(buf, len); + return TValueContext(*this); + } + + TValueContext TBuf::WriteFloat(float f, EFloatToStringMode mode, int ndigits) { + return WriteFloatImpl(f, mode, ndigits); + } + + TValueContext TBuf::WriteDouble(double f, EFloatToStringMode mode, int ndigits) { + return WriteFloatImpl(f, mode, ndigits); + } + + namespace { + struct TFinder: public TCompactStrSpn { + inline TFinder() + : TCompactStrSpn("\xe2\\\"\b\n\f\r\t<>&\'/") + { + for (ui8 ch = 0; ch < 0x20; ++ch) { + Set(ch); + } + } + }; + } + + inline void TBuf::WriteBareString(const TStringBuf s, EHtmlEscapeMode hem) { + RawWriteChar('"'); + const auto& specialChars = *Singleton<TFinder>(); + const char* b = s.begin(); + const char* e = s.end(); + const char* i = b; + while ((i = specialChars.FindFirstOf(i, e)) != e) { + // U+2028 (line separator) and U+2029 (paragraph separator) are valid string + // contents in JSON, but are treated as line breaks in JavaScript, breaking JSONP. + // In UTF-8, U+2028 is "\xe2\x80\xa8" and U+2029 is "\xe2\x80\xa9". + if (Y_UNLIKELY(e - i >= 3 && i[0] == '\xe2' && i[1] == '\x80' && (i[2] | 1) == '\xa9')) { + UnsafeWriteRawBytes(b, i - b); + UnsafeWriteRawBytes(i[2] == '\xa9' ? "\\u2029" : "\\u2028", 6); + b = i = i + 3; + } else if (EscapedWriteChar(b, i, hem)) { + b = ++i; + } else { + ++i; + } + } + UnsafeWriteRawBytes(b, e - b); + RawWriteChar('"'); + } + + inline void TBuf::RawWriteChar(char c) { + Stream->Write(c); + } + + void TBuf::WriteHexEscape(unsigned char c) { + Y_ASSERT(c < 0x80); + UnsafeWriteRawBytes("\\u00", 4); + static const char hexDigits[] = "0123456789ABCDEF"; + RawWriteChar(hexDigits[(c & 0xf0) >> 4]); + RawWriteChar(hexDigits[(c & 0x0f)]); + } + +#define MATCH(sym, string) \ + case sym: \ + UnsafeWriteRawBytes(beg, cur - beg); \ + UnsafeWriteRawBytes(TStringBuf(string)); \ + return true + + inline bool TBuf::EscapedWriteChar(const char* beg, const char* cur, EHtmlEscapeMode hem) { + unsigned char c = *cur; + if (hem == HEM_ESCAPE_HTML) { + switch (c) { + MATCH('"', """); + MATCH('\'', "'"); + MATCH('<', "<"); + MATCH('>', ">"); + MATCH('&', "&"); + } + //for other characters, we fall through to the non-HTML-escaped part + } + + if (hem == HEM_RELAXED && c == '/') + return false; + + if (hem != HEM_UNSAFE) { + switch (c) { + case '/': + UnsafeWriteRawBytes(beg, cur - beg); + UnsafeWriteRawBytes("\\/", 2); + return true; + case '<': + case '>': + case '\'': + UnsafeWriteRawBytes(beg, cur - beg); + WriteHexEscape(c); + return true; + } + // for other characters, fall through to the non-escaped part + } + + switch (c) { + MATCH('"', "\\\""); + MATCH('\\', "\\\\"); + MATCH('\b', "\\b"); + MATCH('\f', "\\f"); + MATCH('\n', "\\n"); + MATCH('\r', "\\r"); + MATCH('\t', "\\t"); + } + if (c < 0x20) { + UnsafeWriteRawBytes(beg, cur - beg); + WriteHexEscape(c); + return true; + } + + return false; + } + +#undef MATCH + + static bool LessStrPtr(const TString* a, const TString* b) { + return *a < *b; + } + + TValueContext TBuf::WriteJsonValue(const NJson::NOrderedJson::TJsonValue* v, bool sortKeys, EFloatToStringMode mode, int ndigits) { + using namespace NJson::NOrderedJson; + switch (v->GetType()) { + default: + case JSON_NULL: + WriteNull(); + break; + case JSON_BOOLEAN: + WriteBool(v->GetBoolean()); + break; + case JSON_DOUBLE: + WriteDouble(v->GetDouble(), mode, ndigits); + break; + case JSON_INTEGER: + WriteLongLong(v->GetInteger()); + break; + case JSON_UINTEGER: + WriteULongLong(v->GetUInteger()); + break; + case JSON_STRING: + WriteString(v->GetString()); + break; + case JSON_ARRAY: { + BeginList(); + const TJsonValue::TArray& arr = v->GetArray(); + for (const auto& it : arr) + WriteJsonValue(&it, sortKeys, mode, ndigits); + EndList(); + break; + } + case JSON_MAP: { + BeginObject(); + const TJsonValue::TMapType& map = v->GetMap(); + if (sortKeys) { + const size_t oldsz = Keys.size(); + Keys.reserve(map.size() + oldsz); + for (const auto& it : map) { + Keys.push_back(&(it.first)); + } + Sort(Keys.begin() + oldsz, Keys.end(), LessStrPtr); + for (size_t i = oldsz, sz = Keys.size(); i < sz; ++i) { + TJsonValue::TMapType::const_iterator kv = map.find(*Keys[i]); + WriteKey(kv->first); + WriteJsonValue(&kv->second, sortKeys, mode, ndigits); + } + Keys.resize(oldsz); + } else { + for (const auto& it : map) { + WriteKey(it.first); + WriteJsonValue(&it.second, sortKeys, mode, ndigits); + } + } + EndObject(); + break; + } + } + return TValueContext(*this); + } + + TPairContext TBuf::UnsafeWritePair(const TStringBuf& s) { + if (Y_UNLIKELY(StackTop() != JE_OBJECT)) { + ythrow TError() << "JSON writer: key:value pair written outside of an object"; + } + WriteComma(); + UnsafeWriteRawBytes(s); + return TPairContext(*this); + } + + void TBuf::UnsafeWriteValue(const TStringBuf& s) { + BeginValue(); + UnsafeWriteRawBytes(s); + EndValue(); + } + + void TBuf::UnsafeWriteValue(const char* s, size_t len) { + BeginValue(); + UnsafeWriteRawBytes(s, len); + EndValue(); + } + + void TBuf::UnsafeWriteRawBytes(const char* src, size_t len) { + Stream->Write(src, len); + } + + void TBuf::UnsafeWriteRawBytes(const TStringBuf& s) { + UnsafeWriteRawBytes(s.data(), s.size()); + } + + const TString& TBuf::Str() const { + if (!StringStream) { + ythrow TError() << "JSON writer: Str() called " + "but writing to an external stream"; + } + if (!(Stack.size() == 1 && StackTop() == JE_OUTER_SPACE)) { + ythrow TError() << "JSON writer: incomplete object converted to string"; + } + return StringStream->Str(); + } + + void TBuf::FlushTo(IOutputStream* stream) { + if (!StringStream) { + ythrow TError() << "JSON writer: FlushTo() called " + "but writing to an external stream"; + } + stream->Write(StringStream->Str()); + StringStream->Clear(); + } + + TString WrapJsonToCallback(const TBuf& buf, TStringBuf callback) { + if (!callback) { + return buf.Str(); + } else { + return TString::Join(callback, "(", buf.Str(), ")"); + } + } + + TBufState TBuf::State() const { + return TBufState{NeedComma, NeedNewline, Stack}; + } + + void TBuf::Reset(const TBufState& from) { + NeedComma = from.NeedComma; + NeedNewline = from.NeedNewline; + Stack = from.Stack; + } + + void TBuf::Reset(TBufState&& from) { + NeedComma = from.NeedComma; + NeedNewline = from.NeedNewline; + Stack.swap(from.Stack); + } + +} diff --git a/library/cpp/json/ordered_maps/json_ordered.h b/library/cpp/json/ordered_maps/json_ordered.h new file mode 100644 index 00000000000..193858fa51b --- /dev/null +++ b/library/cpp/json/ordered_maps/json_ordered.h @@ -0,0 +1,289 @@ +#pragma once + +#include <util/generic/noncopyable.h> +#include <util/generic/ptr.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> +#include <util/stream/str.h> +#include <util/string/cast.h> + +namespace NJson::NOrderedJson { + class TJsonValue; +} + +namespace NJsonOrderedWriter { + enum EJsonEntity : ui8 { + JE_OUTER_SPACE = 1, + JE_LIST, + JE_OBJECT, + JE_PAIR, + }; + + enum EHtmlEscapeMode { + HEM_ESCAPE_HTML = 1, // Use HTML escaping: < > & \/ + HEM_DONT_ESCAPE_HTML, // Use JSON escaping: \u003C \u003E \u0026 \/ + HEM_RELAXED, // Use JSON escaping: \u003C \u003E \u0026 / + HEM_UNSAFE, // Turn escaping off: < > & / + }; + + class TError: public yexception {}; + + class TValueContext; + class TPairContext; + class TAfterColonContext; + + struct TBufState { + bool NeedComma; + bool NeedNewline; + TVector<EJsonEntity> Stack; + }; + + class TBuf : TNonCopyable { + public: + TBuf(EHtmlEscapeMode mode = HEM_DONT_ESCAPE_HTML, IOutputStream* stream = nullptr); + + TValueContext WriteString(const TStringBuf& s, EHtmlEscapeMode hem); + TValueContext WriteString(const TStringBuf& s); + TValueContext WriteInt(int i); + TValueContext WriteLongLong(long long i); + TValueContext WriteULongLong(unsigned long long i); + TValueContext WriteFloat(float f, EFloatToStringMode mode = PREC_NDIGITS, int ndigits = 6); + TValueContext WriteDouble(double f, EFloatToStringMode mode = PREC_NDIGITS, int ndigits = 10); + TValueContext WriteBool(bool b); + TValueContext WriteNull(); + TValueContext WriteJsonValue(const NJson::NOrderedJson::TJsonValue* value, bool sortKeys = false, EFloatToStringMode mode = PREC_NDIGITS, int ndigits = 10); + + TValueContext BeginList(); + TBuf& EndList(); + + TPairContext BeginObject(); + TAfterColonContext WriteKey(const TStringBuf& key, EHtmlEscapeMode hem); + TAfterColonContext WriteKey(const TStringBuf& key); + TAfterColonContext UnsafeWriteKey(const TStringBuf& key); + bool KeyExpected() const { + return Stack.back() == JE_OBJECT; + } + + //! deprecated, do not use in new code + TAfterColonContext CompatWriteKeyWithoutQuotes(const TStringBuf& key); + + TBuf& EndObject(); + + /*** Indent the resulting JSON with spaces. + * By default (spaces==0) no formatting is done. */ + TBuf& SetIndentSpaces(int spaces) { + IndentSpaces = spaces; + return *this; + } + + /*** NaN and Inf are not valid json values, + * so if WriteNanAsString is set, writer would write string + * intead of throwing exception (default case) */ + TBuf& SetWriteNanAsString(bool writeNanAsString = true) { + WriteNanAsString = writeNanAsString; + return *this; + } + + /*** Return the string formed in the internal TStringStream. + * You may only call it if the `stream' parameter was NULL + * at construction time. */ + const TString& Str() const; + + /*** Dump and forget the string constructed so far. + * You may only call it if the `stream' parameter was NULL + * at construction time. */ + void FlushTo(IOutputStream* stream); + + /*** Write a literal string that represents a JSON value + * (string, number, object, array, bool, or null). + * + * Example: + * j.UnsafeWriteValue("[1, 2, 3, \"o'clock\", 4, \"o'clock rock\"]"); + * + * As in all of the Unsafe* functions, no escaping is done. */ + void UnsafeWriteValue(const TStringBuf& s); + void UnsafeWriteValue(const char* s, size_t len); + + /*** When in the context of an object, write a literal string + * that represents a key:value pair (or several pairs). + * + * Example: + * j.BeginObject(); + * j.UnsafeWritePair("\"adam\": \"male\", \"eve\": \"female\""); + * j.EndObject(); + * + * As in all of the Unsafe* functions, no escaping is done. */ + TPairContext UnsafeWritePair(const TStringBuf& s); + + /*** Copy the supplied string directly into the output stream. */ + void UnsafeWriteRawBytes(const TStringBuf& s); + void UnsafeWriteRawBytes(const char* c, size_t len); + + TBufState State() const; + void Reset(const TBufState& from); + void Reset(TBufState&& from); + + private: + void BeginValue(); + void EndValue(); + void BeginKey(); + void RawWriteChar(char c); + bool EscapedWriteChar(const char* b, const char* c, EHtmlEscapeMode hem); + void WriteBareString(const TStringBuf s, EHtmlEscapeMode hem); + void WriteComma(); + void PrintIndentation(bool closing); + void PrintWhitespaces(size_t count, bool prependWithNewLine); + void WriteHexEscape(unsigned char c); + + void StackPush(EJsonEntity e); + void StackPop(); + void CheckAndPop(EJsonEntity e); + EJsonEntity StackTop() const; + + template <class TFloat> + TValueContext WriteFloatImpl(TFloat f, EFloatToStringMode mode, int ndigits); + + private: + IOutputStream* Stream; + THolder<TStringStream> StringStream; + typedef TVector<const TString*> TKeys; + TKeys Keys; + + TVector<EJsonEntity> Stack; + bool NeedComma; + bool NeedNewline; + const EHtmlEscapeMode EscapeMode; + int IndentSpaces; + bool WriteNanAsString; + }; + + // Please don't try to instantiate the classes declared below this point. + + template <typename TOutContext> + class TValueWriter { + public: + TOutContext WriteNull(); + TOutContext WriteString(const TStringBuf&); + TOutContext WriteString(const TStringBuf& s, EHtmlEscapeMode hem); + TOutContext WriteInt(int); + TOutContext WriteLongLong(long long); + TOutContext WriteULongLong(unsigned long long); + TOutContext WriteBool(bool); + TOutContext WriteFloat(float); + TOutContext WriteFloat(float, EFloatToStringMode, int ndigits); + TOutContext WriteDouble(double); + TOutContext WriteDouble(double, EFloatToStringMode, int ndigits); + TOutContext WriteJsonValue(const NJson::NOrderedJson::TJsonValue* value, bool sortKeys = false); + TOutContext UnsafeWriteValue(const TStringBuf&); + + TValueContext BeginList(); + TPairContext BeginObject(); + + protected: + TValueWriter(TBuf& buf) + : Buf(buf) + { + } + friend class TBuf; + + protected: + TBuf& Buf; + }; + + class TValueContext: public TValueWriter<TValueContext> { + public: + TBuf& EndList() { + return Buf.EndList(); + } + TString Str() const { + return Buf.Str(); + } + + private: + TValueContext(TBuf& buf) + : TValueWriter<TValueContext>(buf) + { + } + friend class TBuf; + friend class TValueWriter<TValueContext>; + }; + + class TAfterColonContext: public TValueWriter<TPairContext> { + private: + TAfterColonContext(TBuf& iBuf) + : TValueWriter<TPairContext>(iBuf) + { + } + friend class TBuf; + friend class TPairContext; + }; + + class TPairContext { + public: + TAfterColonContext WriteKey(const TStringBuf& s, EHtmlEscapeMode hem) { + return Buf.WriteKey(s, hem); + } + TAfterColonContext WriteKey(const TStringBuf& s) { + return Buf.WriteKey(s); + } + TAfterColonContext UnsafeWriteKey(const TStringBuf& s) { + return Buf.UnsafeWriteKey(s); + } + TAfterColonContext CompatWriteKeyWithoutQuotes(const TStringBuf& s) { + return Buf.CompatWriteKeyWithoutQuotes(s); + } + TPairContext UnsafeWritePair(const TStringBuf& s) { + return Buf.UnsafeWritePair(s); + } + TBuf& EndObject() { + return Buf.EndObject(); + } + + private: + TPairContext(TBuf& buf) + : Buf(buf) + { + } + + friend class TBuf; + friend class TValueWriter<TPairContext>; + + private: + TBuf& Buf; + }; + +#define JSON_VALUE_WRITER_WRAP(function, params, args) \ + template <typename TOutContext> \ + TOutContext TValueWriter<TOutContext>::function params { \ + Buf.function args; \ + return TOutContext(Buf); \ + } + + JSON_VALUE_WRITER_WRAP(WriteNull, (), ()) + JSON_VALUE_WRITER_WRAP(WriteString, (const TStringBuf& arg), (arg)) + JSON_VALUE_WRITER_WRAP(WriteString, (const TStringBuf& s, EHtmlEscapeMode hem), (s, hem)) + JSON_VALUE_WRITER_WRAP(WriteInt, (int arg), (arg)) + JSON_VALUE_WRITER_WRAP(WriteLongLong, (long long arg), (arg)) + JSON_VALUE_WRITER_WRAP(WriteULongLong, (unsigned long long arg), (arg)) + JSON_VALUE_WRITER_WRAP(WriteBool, (bool arg), (arg)) + JSON_VALUE_WRITER_WRAP(WriteFloat, (float arg), (arg)) + JSON_VALUE_WRITER_WRAP(WriteFloat, (float arg, EFloatToStringMode mode, int ndigits), (arg, mode, ndigits)) + JSON_VALUE_WRITER_WRAP(WriteDouble, (double arg), (arg)) + JSON_VALUE_WRITER_WRAP(WriteDouble, (double arg, EFloatToStringMode mode, int ndigits), (arg, mode, ndigits)) + JSON_VALUE_WRITER_WRAP(WriteJsonValue, (const NJson::NOrderedJson::TJsonValue* value, bool sortKeys), (value, sortKeys)) + JSON_VALUE_WRITER_WRAP(UnsafeWriteValue, (const TStringBuf& arg), (arg)) +#undef JSON_VALUE_WRITER_WRAP + + template <typename TOutContext> + TValueContext TValueWriter<TOutContext>::BeginList() { + return Buf.BeginList(); + } + + template <typename TOutContext> + TPairContext TValueWriter<TOutContext>::BeginObject() { + return Buf.BeginObject(); + } + + TString WrapJsonToCallback(const TBuf& buf, TStringBuf callback); +} diff --git a/library/cpp/json/ordered_maps/json_reader_ordered.cpp b/library/cpp/json/ordered_maps/json_reader_ordered.cpp new file mode 100644 index 00000000000..babb4201978 --- /dev/null +++ b/library/cpp/json/ordered_maps/json_reader_ordered.cpp @@ -0,0 +1,636 @@ +#include "json_reader_ordered.h" + +#include <library/cpp/json/rapidjson_helpers.h> + +#include <contrib/libs/rapidjson/include/rapidjson/error/en.h> +#include <contrib/libs/rapidjson/include/rapidjson/error/error.h> +#include <contrib/libs/rapidjson/include/rapidjson/reader.h> + +#include <util/generic/stack.h> +#include <util/string/cast.h> +#include <util/system/yassert.h> +#include <util/string/builder.h> + +namespace NJson::NOrderedJson { + namespace { + TString PrintError(const rapidjson::ParseResult& result) { + return TStringBuilder() << TStringBuf("Offset: ") << result.Offset() + << TStringBuf(", Code: ") << (int)result.Code() + << TStringBuf(", Error: ") << GetParseError_En(result.Code()); + } + } + + static const size_t DEFAULT_BUFFER_LEN = 65536; + + bool TParserCallbacks::OpenComplexValue(EJsonValueType type) { + TJsonValue* pvalue; + switch (CurrentState) { + case START: + Value.SetType(type); + ValuesStack.push_back(&Value); + break; + case IN_ARRAY: + pvalue = &ValuesStack.back()->AppendValue(type); + ValuesStack.push_back(pvalue); + break; + case AFTER_MAP_KEY: + pvalue = &ValuesStack.back()->InsertValue(Key, type); + ValuesStack.push_back(pvalue); + CurrentState = IN_MAP; + break; + default: + return false; + } + return true; + } + + bool TParserCallbacks::CloseComplexValue() { + if (ValuesStack.empty()) { + return false; + } + + ValuesStack.pop_back(); + if (!ValuesStack.empty()) { + switch (ValuesStack.back()->GetType()) { + case JSON_ARRAY: + CurrentState = IN_ARRAY; + break; + case JSON_MAP: + CurrentState = IN_MAP; + break; + default: + return false; + } + } else { + CurrentState = FINISH; + } + return true; + } + + TParserCallbacks::TParserCallbacks(TJsonValue& value, bool throwOnError, bool notClosedBracketIsError) + : TJsonCallbacks(throwOnError) + , Value(value) + , NotClosedBracketIsError(notClosedBracketIsError) + , CurrentState(START) + { + } + + bool TParserCallbacks::OnNull() { + return SetValue(JSON_NULL); + } + + bool TParserCallbacks::OnBoolean(bool val) { + return SetValue(val); + } + + bool TParserCallbacks::OnInteger(long long val) { + return SetValue(val); + } + + bool TParserCallbacks::OnUInteger(unsigned long long val) { + return SetValue(val); + } + + bool TParserCallbacks::OnString(const TStringBuf& val) { + return SetValue(val); + } + + bool TParserCallbacks::OnDouble(double val) { + return SetValue(val); + } + + bool TParserCallbacks::OnOpenArray() { + bool res = OpenComplexValue(JSON_ARRAY); + if (res) + CurrentState = IN_ARRAY; + return res; + } + + bool TParserCallbacks::OnCloseArray() { + return CloseComplexValue(); + } + + bool TParserCallbacks::OnOpenMap() { + bool res = OpenComplexValue(JSON_MAP); + if (res) + CurrentState = IN_MAP; + return res; + } + + bool TParserCallbacks::OnCloseMap() { + return CloseComplexValue(); + } + + bool TParserCallbacks::OnMapKey(const TStringBuf& val) { + switch (CurrentState) { + case IN_MAP: + Key = val; + CurrentState = AFTER_MAP_KEY; + break; + default: + return false; + } + return true; + } + + bool TParserCallbacks::OnEnd() { + if (NotClosedBracketIsError){ + return ValuesStack.empty(); + } + return true; + } + + TJsonReaderConfig::TJsonReaderConfig() + : BufferSize(DEFAULT_BUFFER_LEN) + { + } + + void TJsonReaderConfig::SetBufferSize(size_t bufferSize) { + BufferSize = Max((size_t)1, Min(bufferSize, DEFAULT_BUFFER_LEN)); + } + + size_t TJsonReaderConfig::GetBufferSize() const { + return BufferSize; + } + + namespace { + struct TJsonValueBuilderConfig { + ui64 MaxDepth = 0; + }; + + struct TJsonValueBuilder { +#ifdef NDEBUG + using TItem = TJsonValue*; + + inline TJsonValue& Access(TItem& item) const { + return *item; + } +#else + struct TItem { + TJsonValue* V; + size_t DuplicateKeyCount; + + TItem(TJsonValue* v) + : V(v) + , DuplicateKeyCount(0) + { + } + }; + + inline TJsonValue& Access(TItem& item) const { + return *item.V; + } +#endif + + NJson::NOrderedJson::TJsonValue& V; + + TStack<TItem> S; + + TJsonValueBuilderConfig Config; + + TJsonValueBuilder(NJson::NOrderedJson::TJsonValue& v) + : V(v) + { + S.emplace(&V); + } + + TJsonValueBuilder(NJson::NOrderedJson::TJsonValue& v, const TJsonValueBuilderConfig& config) + : V(v) + , Config(config) + { + S.emplace(&V); + } + + template <class T> + void Set(const T& t) { + if (Access(S.top()).IsArray()) { + Access(S.top()).AppendValue(t); + } else { + Access(S.top()) = t; + S.pop(); + } + } + + bool Null() { + Set(NJson::NOrderedJson::JSON_NULL); + return true; + } + + bool Bool(bool b) { + Set(b); + return true; + } + + bool Int(int i) { + Set(i); + return true; + } + + template <class U> + bool ProcessUint(U u) { + if (Y_LIKELY(u <= static_cast<ui64>(Max<i64>()))) { + Set(i64(u)); + } else { + Set(u); + } + return true; + } + + bool Uint(unsigned u) { + return ProcessUint(u); + } + + bool Int64(i64 i) { + Set(i); + return true; + } + + bool Uint64(ui64 u) { + return ProcessUint(u); + } + + bool Double(double d) { + Set(d); + return true; + } + + bool RawNumber(const char* str, rapidjson::SizeType length, bool copy) { + Y_ASSERT(false && "this method should never be called"); + Y_UNUSED(str); + Y_UNUSED(length); + Y_UNUSED(copy); + return true; + } + + bool String(const char* str, rapidjson::SizeType length, bool copy) { + Y_ASSERT(copy); + Set(TStringBuf(str, length)); + return true; + } + + bool StartObject() { + if (Access(S.top()).IsArray()) { + S.emplace(&Access(S.top()).AppendValue(NJson::NOrderedJson::JSON_MAP)); + if (!IsWithinStackBounds()) { + return false; + } + } else { + Access(S.top()).SetType(NJson::NOrderedJson::JSON_MAP); + } + return true; + } + + bool Key(const char* str, rapidjson::SizeType length, bool copy) { + Y_ASSERT(copy); + auto& value = Access(S.top())[TStringBuf(str, length)]; + if (Y_UNLIKELY(value.GetType() != JSON_UNDEFINED)) { +#ifndef NDEBUG + ++S.top().DuplicateKeyCount; +#endif + value.SetType(JSON_UNDEFINED); + } + S.emplace(&value); + if (!IsWithinStackBounds()) { + return false; + } + return true; + } + + inline int GetDuplicateKeyCount() const { +#ifdef NDEBUG + return 0; +#else + return S.top().DuplicateKeyCount; +#endif + } + + bool EndObject(rapidjson::SizeType memberCount) { + Y_ASSERT(memberCount == Access(S.top()).GetMap().size() + GetDuplicateKeyCount()); + S.pop(); + return true; + } + + bool StartArray() { + if (Access(S.top()).IsArray()) { + S.emplace(&Access(S.top()).AppendValue(NJson::NOrderedJson::JSON_ARRAY)); + if (!IsWithinStackBounds()) { + return false; + } + } else { + Access(S.top()).SetType(NJson::NOrderedJson::JSON_ARRAY); + } + return true; + } + + bool EndArray(rapidjson::SizeType elementCount) { + Y_ASSERT(elementCount == Access(S.top()).GetArray().size()); + S.pop(); + return true; + } + + bool IsWithinStackBounds() { + return Config.MaxDepth == 0 || (S.size() <= Config.MaxDepth); + } + }; + + constexpr ui32 ConvertToRapidJsonFlags(ui8 flags) { + ui32 rapidjsonFlags = rapidjson::kParseNoFlags; + + if (flags & ReaderConfigFlags::NANINF) { + rapidjsonFlags |= rapidjson::kParseNanAndInfFlag; + } + if (flags & ReaderConfigFlags::ITERATIVE) { + rapidjsonFlags |= rapidjson::kParseIterativeFlag; + } + + if (flags & ReaderConfigFlags::COMMENTS) { + rapidjsonFlags |= rapidjson::kParseCommentsFlag; + } + + if (flags & ReaderConfigFlags::VALIDATE) { + rapidjsonFlags |= rapidjson::kParseValidateEncodingFlag; + } + + if (flags & ReaderConfigFlags::ESCAPE) { + rapidjsonFlags |= rapidjson::kParseEscapedApostropheFlag; + } + + return rapidjsonFlags; + } + + template <class TRapidJsonCompliantInputStream, class THandler, ui8 currentFlags = 0> + auto ReadWithRuntimeFlags(ui8 runtimeFlags, + rapidjson::Reader& reader, + TRapidJsonCompliantInputStream& is, + THandler& handler) { + if (runtimeFlags == 0) { + return reader.Parse<ConvertToRapidJsonFlags(currentFlags)>(is, handler); + } + +#define TRY_EXTRACT_FLAG(flag) \ + if (runtimeFlags & flag) { \ + return ReadWithRuntimeFlags<TRapidJsonCompliantInputStream, THandler, currentFlags | flag>( \ + runtimeFlags ^ flag, reader, is, handler \ + ); \ + } + + TRY_EXTRACT_FLAG(ReaderConfigFlags::NANINF); + TRY_EXTRACT_FLAG(ReaderConfigFlags::ITERATIVE); + TRY_EXTRACT_FLAG(ReaderConfigFlags::COMMENTS); + TRY_EXTRACT_FLAG(ReaderConfigFlags::VALIDATE); + TRY_EXTRACT_FLAG(ReaderConfigFlags::ESCAPE); + +#undef TRY_EXTRACT_FLAG + + return reader.Parse<ConvertToRapidJsonFlags(currentFlags)>(is, handler); + } + + template <class TRapidJsonCompliantInputStream, class THandler> + auto Read(const TJsonReaderConfig& config, + rapidjson::Reader& reader, + TRapidJsonCompliantInputStream& is, + THandler& handler) { + + // validate by default + ui8 flags = ReaderConfigFlags::VALIDATE; + + if (config.UseIterativeParser) { + flags |= ReaderConfigFlags::ITERATIVE; + } + + if (config.AllowComments) { + flags |= ReaderConfigFlags::COMMENTS; + } + + if (config.DontValidateUtf8) { + flags &= ~(ReaderConfigFlags::VALIDATE); + } + + if (config.AllowEscapedApostrophe) { + flags |= ReaderConfigFlags::ESCAPE; + } + if (config.AllowReadNanInf) { + flags |= ReaderConfigFlags::NANINF; + } + + return ReadWithRuntimeFlags(flags, reader, is, handler); + } + + template <class TRapidJsonCompliantInputStream, class THandler> + bool ReadJson(TRapidJsonCompliantInputStream& is, const TJsonReaderConfig* config, THandler& handler, bool throwOnError) { + rapidjson::Reader reader; + + auto result = Read(*config, reader, is, handler); + + if (result.IsError()) { + if (throwOnError) { + ythrow TJsonException() << PrintError(result); + } else { + return false; + } + } + + return true; + } + + template <class TRapidJsonCompliantInputStream> + bool ReadJsonTree(TRapidJsonCompliantInputStream& is, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError) { + out->SetType(NJson::NOrderedJson::JSON_NULL); + + TJsonValueBuilder handler(*out, { .MaxDepth = config->MaxDepth }); + + return ReadJson(is, config, handler, throwOnError); + } + + template <class TData> + bool ReadJsonTreeImpl(TData* in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError) { + std::conditional_t<std::is_same<TData, TStringBuf>::value, TStringBufStreamWrapper, TInputStreamWrapper> is(*in); + return ReadJsonTree(is, config, out, throwOnError); + } + + template <class TData> + bool ReadJsonTreeImpl(TData* in, bool allowComments, TJsonValue* out, bool throwOnError) { + TJsonReaderConfig config; + config.AllowComments = allowComments; + return ReadJsonTreeImpl(in, &config, out, throwOnError); + } + + template <class TData> + bool ReadJsonTreeImpl(TData* in, TJsonValue* out, bool throwOnError) { + return ReadJsonTreeImpl(in, false, out, throwOnError); + } + } //namespace + + bool ReadJsonTree(TStringBuf in, TJsonValue* out, bool throwOnError) { + return ReadJsonTreeImpl(&in, out, throwOnError); + } + + bool ReadJsonTree(TStringBuf in, bool allowComments, TJsonValue* out, bool throwOnError) { + return ReadJsonTreeImpl(&in, allowComments, out, throwOnError); + } + + bool ReadJsonTree(TStringBuf in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError) { + return ReadJsonTreeImpl(&in, config, out, throwOnError); + } + + bool ReadJsonTree(IInputStream* in, TJsonValue* out, bool throwOnError) { + return ReadJsonTreeImpl(in, out, throwOnError); + } + + bool ReadJsonTree(IInputStream* in, bool allowComments, TJsonValue* out, bool throwOnError) { + return ReadJsonTreeImpl(in, allowComments, out, throwOnError); + } + + bool ReadJsonTree(IInputStream* in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError) { + return ReadJsonTreeImpl(in, config, out, throwOnError); + } + + bool ReadJsonFastTree(TStringBuf in, TJsonValue* out, bool throwOnError, bool notClosedBracketIsError) { + TParserCallbacks cb(*out, throwOnError, notClosedBracketIsError); + + return ReadJsonFast(in, &cb); + } + + TJsonValue ReadJsonFastTree(TStringBuf in, bool notClosedBracketIsError) { + TJsonValue value; + // There is no way to report an error apart from throwing an exception when we return result by value. + ReadJsonFastTree(in, &value, /* throwOnError = */ true, notClosedBracketIsError); + return value; + } + + namespace { + struct TJsonCallbacksWrapper { + TJsonCallbacks& Impl; + + TJsonCallbacksWrapper(TJsonCallbacks& impl) + : Impl(impl) + { + } + + bool Null() { + return Impl.OnNull(); + } + + bool Bool(bool b) { + return Impl.OnBoolean(b); + } + + template <class U> + bool ProcessUint(U u) { + if (Y_LIKELY(u <= ui64(Max<i64>()))) { + return Impl.OnInteger(i64(u)); + } else { + return Impl.OnUInteger(u); + } + } + + bool Int(int i) { + return Impl.OnInteger(i); + } + + bool Uint(unsigned u) { + return ProcessUint(u); + } + + bool Int64(i64 i) { + return Impl.OnInteger(i); + } + + bool Uint64(ui64 u) { + return ProcessUint(u); + } + + bool Double(double d) { + return Impl.OnDouble(d); + } + + bool RawNumber(const char* str, rapidjson::SizeType length, bool copy) { + Y_ASSERT(false && "this method should never be called"); + Y_UNUSED(str); + Y_UNUSED(length); + Y_UNUSED(copy); + return true; + } + + bool String(const char* str, rapidjson::SizeType length, bool copy) { + Y_ASSERT(copy); + return Impl.OnString(TStringBuf(str, length)); + } + + bool StartObject() { + return Impl.OnOpenMap(); + } + + bool Key(const char* str, rapidjson::SizeType length, bool copy) { + Y_ASSERT(copy); + return Impl.OnMapKey(TStringBuf(str, length)); + } + + bool EndObject(rapidjson::SizeType memberCount) { + Y_UNUSED(memberCount); + return Impl.OnCloseMap(); + } + + bool StartArray() { + return Impl.OnOpenArray(); + } + + bool EndArray(rapidjson::SizeType elementCount) { + Y_UNUSED(elementCount); + return Impl.OnCloseArray(); + } + }; + } + + bool ReadJson(IInputStream* in, TJsonCallbacks* cbs) { + return ReadJson(in, false, cbs); + } + + bool ReadJson(IInputStream* in, bool allowComments, TJsonCallbacks* cbs) { + TJsonReaderConfig config; + config.AllowComments = allowComments; + return ReadJson(in, &config, cbs); + } + + bool ReadJson(IInputStream* in, bool allowComments, bool allowEscapedApostrophe, TJsonCallbacks* cbs) { + TJsonReaderConfig config; + config.AllowComments = allowComments; + config.AllowEscapedApostrophe = allowEscapedApostrophe; + return ReadJson(in, &config, cbs); + } + + bool ReadJson(IInputStream* in, const TJsonReaderConfig* config, TJsonCallbacks* cbs) { + TJsonCallbacksWrapper wrapper(*cbs); + TInputStreamWrapper is(*in); + + rapidjson::Reader reader; + auto result = Read(*config, reader, is, wrapper); + + if (result.IsError()) { + cbs->OnError(result.Offset(), PrintError(result)); + + return false; + } + + return cbs->OnEnd(); + } + + TJsonValue ReadJsonTree(IInputStream* in, bool throwOnError) { + TJsonValue out; + ReadJsonTree(in, &out, throwOnError); + return out; + } + + TJsonValue ReadJsonTree(IInputStream* in, bool allowComments, bool throwOnError) { + TJsonValue out; + ReadJsonTree(in, allowComments, &out, throwOnError); + return out; + } + + TJsonValue ReadJsonTree(IInputStream* in, const TJsonReaderConfig* config, bool throwOnError) { + TJsonValue out; + ReadJsonTree(in, config, &out, throwOnError); + return out; + } + +} diff --git a/library/cpp/json/ordered_maps/json_reader_ordered.h b/library/cpp/json/ordered_maps/json_reader_ordered.h new file mode 100644 index 00000000000..019a7cb4b5e --- /dev/null +++ b/library/cpp/json/ordered_maps/json_reader_ordered.h @@ -0,0 +1,137 @@ +#pragma once + +#include "json_value_ordered.h" + +#include <library/cpp/json/common/defs.h> +#include <library/cpp/json/fast_sax/parser.h> + +#include <util/generic/yexception.h> + +#include <util/stream/input.h> +#include <util/stream/str.h> +#include <util/stream/mem.h> + +namespace NJson::NOrderedJson { + struct TJsonReaderConfig { + TJsonReaderConfig(); + + bool UseIterativeParser = false; + // js-style comments (both // and /**/) + bool AllowComments = false; + bool DontValidateUtf8 = false; + bool AllowEscapedApostrophe = false; + // Allow to read Nan and Inf as integer values + bool AllowReadNanInf = false; + + ui64 MaxDepth = 0; + + void SetBufferSize(size_t bufferSize); + size_t GetBufferSize() const; + + private: + size_t BufferSize; + }; + + bool ReadJsonTree(TStringBuf in, TJsonValue* out, bool throwOnError = false); + bool ReadJsonTree(TStringBuf in, bool allowComments, TJsonValue* out, bool throwOnError = false); + bool ReadJsonTree(TStringBuf in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError = false); + + bool ReadJsonTree(IInputStream* in, TJsonValue* out, bool throwOnError = false); + bool ReadJsonTree(IInputStream* in, bool allowComments, TJsonValue* out, bool throwOnError = false); + bool ReadJsonTree(IInputStream* in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError = false); + + TJsonValue ReadJsonTree(IInputStream* in, bool throwOnError = false); + TJsonValue ReadJsonTree(IInputStream* in, bool allowComments, bool throwOnError); + TJsonValue ReadJsonTree(IInputStream* in, const TJsonReaderConfig* config, bool throwOnError = false); + + bool ReadJson(IInputStream* in, TJsonCallbacks* callbacks); + bool ReadJson(IInputStream* in, bool allowComments, TJsonCallbacks* callbacks); + bool ReadJson(IInputStream* in, bool allowComments, bool allowEscapedApostrophe, TJsonCallbacks* callbacks); + bool ReadJson(IInputStream* in, const TJsonReaderConfig* config, TJsonCallbacks* callbacks); + + enum ReaderConfigFlags { + NANINF = 0b10000, + ITERATIVE = 0b1000, + COMMENTS = 0b0100, + VALIDATE = 0b0010, + ESCAPE = 0b0001, + }; + + inline bool ValidateJson(IInputStream* in, const TJsonReaderConfig* config, bool throwOnError = false) { + TJsonCallbacks c(throwOnError); + return ReadJson(in, config, &c); + } + + inline bool ValidateJson(TStringBuf in, const TJsonReaderConfig& config = TJsonReaderConfig(), bool throwOnError = false) { + TMemoryInput min(in.data(), in.size()); + return ValidateJson(&min, &config, throwOnError); + } + + inline bool ValidateJsonThrow(IInputStream* in, const TJsonReaderConfig* config) { + return ValidateJson(in, config, true); + } + + inline bool ValidateJsonThrow(TStringBuf in, const TJsonReaderConfig& config = TJsonReaderConfig()) { + return ValidateJson(in, config, true); + } + + class TParserCallbacks: public TJsonCallbacks { + public: + TParserCallbacks(TJsonValue& value, bool throwOnError = false, bool notClosedBracketIsError = false); + bool OnNull() override; + bool OnBoolean(bool val) override; + bool OnInteger(long long val) override; + bool OnUInteger(unsigned long long val) override; + bool OnString(const TStringBuf& val) override; + bool OnDouble(double val) override; + bool OnOpenArray() override; + bool OnCloseArray() override; + bool OnOpenMap() override; + bool OnCloseMap() override; + bool OnMapKey(const TStringBuf& val) override; + bool OnEnd() override; + + protected: + TJsonValue& Value; + TString Key; + TVector<TJsonValue*> ValuesStack; + bool NotClosedBracketIsError; + + enum { + START, + AFTER_MAP_KEY, + IN_MAP, + IN_ARRAY, + FINISH + } CurrentState; + + template <class T> + bool SetValue(const T& value) { + switch (CurrentState) { + case START: + Value.SetValue(value); + break; + case AFTER_MAP_KEY: + ValuesStack.back()->InsertValue(Key, value); + CurrentState = IN_MAP; + break; + case IN_ARRAY: + ValuesStack.back()->AppendValue(value); + break; + case IN_MAP: + case FINISH: + return false; + default: + ythrow yexception() << "TParserCallbacks::SetValue invalid enum"; + } + return true; + } + + bool OpenComplexValue(EJsonValueType type); + bool CloseComplexValue(); + }; + + //// relaxed json, used in library/cpp/scheme + bool ReadJsonFastTree(TStringBuf in, TJsonValue* out, bool throwOnError = false, bool notClosedBracketIsError = false); + TJsonValue ReadJsonFastTree(TStringBuf in, bool notClosedBracketIsError = false); +} diff --git a/library/cpp/json/ordered_maps/json_value_ordered.cpp b/library/cpp/json/ordered_maps/json_value_ordered.cpp new file mode 100644 index 00000000000..b25ca97b71a --- /dev/null +++ b/library/cpp/json/ordered_maps/json_value_ordered.cpp @@ -0,0 +1,1128 @@ +#include "json_value_ordered.h" +#include "json_ordered.h" + +#include <util/generic/ymath.h> +#include <util/generic/ylimits.h> +#include <util/generic/utility.h> +#include <util/generic/singleton.h> +#include <util/stream/str.h> +#include <util/stream/output.h> +#include <util/string/cast.h> +#include <util/string/type.h> +#include <util/string/vector.h> +#include <util/system/yassert.h> +#include <util/ysaveload.h> +#include <util/generic/yexception.h> + +static bool +AreJsonMapsEqual(const NJson::NOrderedJson::TJsonValue& lhs, const NJson::NOrderedJson::TJsonValue& rhs) { + using namespace NJson::NOrderedJson; + + Y_ABORT_UNLESS(lhs.GetType() == JSON_MAP, "lhs has not a JSON_MAP type."); + + if (rhs.GetType() != JSON_MAP) + return false; + + typedef TJsonValue::TMapType TMapType; + const TMapType& lhsMap = lhs.GetMap(); + const TMapType& rhsMap = rhs.GetMap(); + + if (lhsMap.size() != rhsMap.size()) + return false; + + for (const auto& lhsIt : lhsMap) { + TMapType::const_iterator rhsIt = rhsMap.find(lhsIt.first); + if (rhsIt == rhsMap.end()) + return false; + + if (lhsIt.second != rhsIt->second) + return false; + } + + return true; +} + +static bool +AreJsonArraysEqual(const NJson::NOrderedJson::TJsonValue& lhs, const NJson::NOrderedJson::TJsonValue& rhs) { + using namespace NJson::NOrderedJson; + + Y_ABORT_UNLESS(lhs.GetType() == JSON_ARRAY, "lhs has not a JSON_ARRAY type."); + + if (rhs.GetType() != JSON_ARRAY) + return false; + + typedef TJsonValue::TArray TArray; + const TArray& lhsArray = lhs.GetArray(); + const TArray& rhsArray = rhs.GetArray(); + + if (lhsArray.size() != rhsArray.size()) + return false; + + for (TArray::const_iterator lhsIt = lhsArray.begin(), rhsIt = rhsArray.begin(); + lhsIt != lhsArray.end(); ++lhsIt, ++rhsIt) { + if (*lhsIt != *rhsIt) + return false; + } + + return true; +} + +namespace NJson::NOrderedJson { + const TJsonValue TJsonValue::UNDEFINED{}; + + TJsonValue::TJsonValue(const EJsonValueType type) { + SetType(type); + } + + TJsonValue::TJsonValue(TJsonValue&& vval) noexcept + : Type(JSON_UNDEFINED) + { + vval.SwapWithUndefined(*this); + Zero(vval.Value); + } + + TJsonValue::TJsonValue(const NJson::TJsonValue& vval) { + switch (vval.GetType()) { + case NJson::EJsonValueType::JSON_ARRAY: + SetType(JSON_ARRAY); + for (const auto& item : vval.GetArray()) { + AppendValue(TJsonValue(item)); + } + break; + case NJson::EJsonValueType::JSON_MAP: + SetType(JSON_MAP); + for (const auto& item : vval.GetMap()) { + InsertValue(item.first, TJsonValue(item.second)); + } + break; + case NJson::EJsonValueType::JSON_BOOLEAN: + SetType(JSON_BOOLEAN); + Value.Boolean = vval.GetBoolean(); + break; + case NJson::EJsonValueType::JSON_INTEGER: + SetType(JSON_INTEGER); + Value.Integer = vval.GetInteger(); + break; + case NJson::EJsonValueType::JSON_UINTEGER: + SetType(JSON_UINTEGER); + Value.UInteger = vval.GetUInteger(); + break; + case NJson::EJsonValueType::JSON_STRING: + SetType(JSON_STRING); + Value.String = vval.GetString(); + break; + case NJson::EJsonValueType::JSON_NULL: + SetType(JSON_NULL); + break; + case NJson::EJsonValueType::JSON_UNDEFINED: + SetType(JSON_UNDEFINED); + break; + case NJson::EJsonValueType::JSON_DOUBLE: + SetType(JSON_DOUBLE); + Value.Double = vval.GetDouble(); + break; + } + } + + TJsonValue::TJsonValue(const TJsonValue& val) + : Type(val.Type) + { + switch (Type) { + case JSON_STRING: + new (&Value.String) TString(val.GetString()); + break; + case JSON_MAP: + Value.Map = new TMapType(val.GetMap()); + break; + case JSON_ARRAY: + Value.Array = new TArray(val.GetArray()); + break; + case JSON_UNDEFINED: + case JSON_NULL: + case JSON_BOOLEAN: + case JSON_INTEGER: + case JSON_UINTEGER: + case JSON_DOUBLE: + std::memcpy(&Value, &val.Value, sizeof(Value)); + break; + } + } + + TJsonValue& TJsonValue::operator=(const TJsonValue& val) { + if (this == &val) + return *this; + TJsonValue tmp(val); + tmp.Swap(*this); + return *this; + } + + TJsonValue& TJsonValue::operator=(TJsonValue&& val) noexcept { + if (this == &val) + return *this; + TJsonValue tmp(std::move(val)); + tmp.Swap(*this); + return *this; + } + + TJsonValue::TJsonValue(const bool value) noexcept { + SetType(JSON_BOOLEAN); + Value.Boolean = value; + } + + TJsonValue::TJsonValue(const long long value) noexcept { + SetType(JSON_INTEGER); + Value.Integer = value; + } + + TJsonValue::TJsonValue(const unsigned long long value) noexcept { + SetType(JSON_UINTEGER); + Value.UInteger = value; + } + + TJsonValue::TJsonValue(const int value) noexcept { + SetType(JSON_INTEGER); + Value.Integer = value; + } + + TJsonValue::TJsonValue(const unsigned int value) noexcept { + SetType(JSON_UINTEGER); + Value.UInteger = value; + } + + TJsonValue::TJsonValue(const long value) noexcept { + SetType(JSON_INTEGER); + Value.Integer = value; + } + + TJsonValue::TJsonValue(const unsigned long value) noexcept { + SetType(JSON_UINTEGER); + Value.UInteger = value; + } + + TJsonValue::TJsonValue(const double value) noexcept { + SetType(JSON_DOUBLE); + Value.Double = value; + } + + TJsonValue::TJsonValue(TString value) { + SetType(JSON_STRING); + Value.String = std::move(value); + } + + TJsonValue::TJsonValue(const TStringBuf value) { + SetType(JSON_STRING); + Value.String = value; + } + + TJsonValue::TJsonValue(const char* value) { + SetType(JSON_STRING); + Value.String = value; + } + + EJsonValueType TJsonValue::GetType() const noexcept { + return Type; + } + + TJsonValue& TJsonValue::SetType(const EJsonValueType type) { + if (Type == type) + return *this; + + Clear(); + Type = type; + + switch (Type) { + case JSON_STRING: + new (&Value.String) TString(); + break; + case JSON_MAP: + Value.Map = new TMapType(); + break; + case JSON_ARRAY: + Value.Array = new TArray(); + break; + case JSON_UNDEFINED: + case JSON_NULL: + case JSON_BOOLEAN: + case JSON_INTEGER: + case JSON_UINTEGER: + case JSON_DOUBLE: + break; + } + + return *this; + } + + TJsonValue& TJsonValue::SetValue(const TJsonValue& value) { + return *this = value; + } + + TJsonValue& TJsonValue::SetValue(TJsonValue&& value) { + *this = std::move(value); + return *this; + } + + TJsonValue& TJsonValue::InsertValue(const TString& key, const TJsonValue& value) { + SetType(JSON_MAP); + return (*Value.Map)[key] = value; + } + + TJsonValue& TJsonValue::InsertValue(const TStringBuf key, const TJsonValue& value) { + SetType(JSON_MAP); + return (*Value.Map)[key] = value; + } + + TJsonValue& TJsonValue::InsertValue(const char* key, const TJsonValue& value) { + SetType(JSON_MAP); + return (*Value.Map)[key] = value; + } + + TJsonValue& TJsonValue::InsertValue(const TString& key, TJsonValue&& value) { + SetType(JSON_MAP); + return (*Value.Map)[key] = std::move(value); + } + + TJsonValue& TJsonValue::InsertValue(const TStringBuf key, TJsonValue&& value) { + SetType(JSON_MAP); + return (*Value.Map)[key] = std::move(value); + } + + TJsonValue& TJsonValue::InsertValue(const char* key, TJsonValue&& value) { + SetType(JSON_MAP); + return (*Value.Map)[key] = std::move(value); + } + + TJsonValue& TJsonValue::Back() { + BackChecks(); + return Value.Array->back(); + } + + const TJsonValue& TJsonValue::Back() const { + BackChecks(); + return Value.Array->back(); + } + + TJsonValue& TJsonValue::AppendValue(const TJsonValue& value) { + SetType(JSON_ARRAY); + Value.Array->push_back(value); + return Value.Array->back(); + } + + TJsonValue& TJsonValue::AppendValue(TJsonValue&& value) { + SetType(JSON_ARRAY); + Value.Array->push_back(std::move(value)); + return Value.Array->back(); + } + + void TJsonValue::EraseValue(const TStringBuf key) { + if (IsMap()) { + TMapType::iterator it = Value.Map->find(key); + if (it != Value.Map->end()) + Value.Map->erase(it); + } + } + + void TJsonValue::EraseValue(const size_t index) { + if (IsArray()) { + if (index >= Value.Array->size()) { + return; + } + TArray::iterator it = Value.Array->begin() + index; + Value.Array->erase(it); + } + } + + void TJsonValue::Clear() noexcept { + switch (Type) { + case JSON_STRING: + Value.String.~TString(); + break; + case JSON_MAP: + delete Value.Map; + break; + case JSON_ARRAY: + delete Value.Array; + break; + case JSON_UNDEFINED: + case JSON_NULL: + case JSON_BOOLEAN: + case JSON_INTEGER: + case JSON_UINTEGER: + case JSON_DOUBLE: + break; + } + Zero(Value); + Type = JSON_UNDEFINED; + } + + TJsonValue& TJsonValue::operator[](const size_t idx) { + SetType(JSON_ARRAY); + if (Value.Array->size() <= idx) + Value.Array->resize(idx + 1); + return (*Value.Array)[idx]; + } + + TJsonValue& TJsonValue::operator[](const TStringBuf& key) { + SetType(JSON_MAP); + return (*Value.Map)[key]; + } + + namespace { + struct TDefaultsHolder { + const TString String{}; + const TJsonValue::TMapType Map{}; + const TJsonValue::TArray Array{}; + const TJsonValue Value{}; + }; + } + + const TJsonValue& TJsonValue::operator[](const size_t idx) const noexcept { + const TJsonValue* ret = nullptr; + if (GetValuePointer(idx, &ret)) + return *ret; + + return Singleton<TDefaultsHolder>()->Value; + } + + const TJsonValue& TJsonValue::operator[](const TStringBuf& key) const noexcept { + const TJsonValue* ret = nullptr; + if (GetValuePointer(key, &ret)) + return *ret; + + return Singleton<TDefaultsHolder>()->Value; + } + + bool TJsonValue::GetBoolean() const { + return Type != JSON_BOOLEAN ? false : Value.Boolean; + } + + long long TJsonValue::GetInteger() const { + if (!IsInteger()) + return 0; + + switch (Type) { + case JSON_INTEGER: + return Value.Integer; + + case JSON_UINTEGER: + return Value.UInteger; + + case JSON_DOUBLE: + return Value.Double; + + default: + Y_ASSERT(false && "Unexpected type."); + return 0; + } + } + + unsigned long long TJsonValue::GetUInteger() const { + if (!IsUInteger()) + return 0; + + switch (Type) { + case JSON_UINTEGER: + return Value.UInteger; + + case JSON_INTEGER: + return Value.Integer; + + case JSON_DOUBLE: + return Value.Double; + + default: + Y_ASSERT(false && "Unexpected type."); + return 0; + } + } + + double TJsonValue::GetDouble() const { + if (!IsDouble()) + return 0.0; + + switch (Type) { + case JSON_DOUBLE: + return Value.Double; + + case JSON_INTEGER: + return Value.Integer; + + case JSON_UINTEGER: + return Value.UInteger; + + default: + Y_ASSERT(false && "Unexpected type."); + return 0.0; + } + } + + const TString& TJsonValue::GetString() const { + return Type != JSON_STRING ? Singleton<TDefaultsHolder>()->String : Value.String; + } + + const TJsonValue::TMapType& TJsonValue::GetMap() const { + return Type != JSON_MAP ? Singleton<TDefaultsHolder>()->Map : *Value.Map; + } + + const TJsonValue::TArray& TJsonValue::GetArray() const { + return (Type != JSON_ARRAY) ? Singleton<TDefaultsHolder>()->Array : *Value.Array; + } + + bool TJsonValue::GetBooleanSafe() const { + if (Type != JSON_BOOLEAN) + ythrow TJsonException() << "Not a boolean"; + + return Value.Boolean; + } + + long long TJsonValue::GetIntegerSafe() const { + if (!IsInteger()) + ythrow TJsonException() << "Not an integer"; + + return GetInteger(); + } + + unsigned long long TJsonValue::GetUIntegerSafe() const { + if (!IsUInteger()) + ythrow TJsonException() << "Not an unsigned integer"; + + return GetUInteger(); + } + + double TJsonValue::GetDoubleSafe() const { + if (!IsDouble()) + ythrow TJsonException() << "Not a double"; + + return GetDouble(); + } + + const TString& TJsonValue::GetStringSafe() const { + if (Type != JSON_STRING) + ythrow TJsonException() << "Not a string"; + + return Value.String; + } + + bool TJsonValue::GetBooleanSafe(const bool defaultValue) const { + if (Type == JSON_UNDEFINED) + return defaultValue; + + return GetBooleanSafe(); + } + + long long TJsonValue::GetIntegerSafe(const long long defaultValue) const { + if (Type == JSON_UNDEFINED) + return defaultValue; + + return GetIntegerSafe(); + } + + unsigned long long TJsonValue::GetUIntegerSafe(const unsigned long long defaultValue) const { + if (Type == JSON_UNDEFINED) + return defaultValue; + + return GetUIntegerSafe(); + } + + double TJsonValue::GetDoubleSafe(const double defaultValue) const { + if (Type == JSON_UNDEFINED) + return defaultValue; + + return GetDoubleSafe(); + } + + TString TJsonValue::GetStringSafe(const TString& defaultValue) const { + if (Type == JSON_UNDEFINED) + return defaultValue; + + return GetStringSafe(); + } + + const TJsonValue::TMapType& TJsonValue::GetMapSafe() const { + if (Type != JSON_MAP) + ythrow TJsonException() << "Not a map"; + + return *Value.Map; + } + + TJsonValue::TMapType& TJsonValue::GetMapSafe() { + return const_cast<TJsonValue::TMapType&>(const_cast<const TJsonValue*>(this)->GetMapSafe()); + } + + const TJsonValue::TArray& TJsonValue::GetArraySafe() const { + if (Type != JSON_ARRAY) + ythrow TJsonException() << "Not an array"; + + return *Value.Array; + } + + TJsonValue::TArray& TJsonValue::GetArraySafe() { + return const_cast<TJsonValue::TArray&>(const_cast<const TJsonValue*>(this)->GetArraySafe()); + } + + bool TJsonValue::GetBooleanRobust() const noexcept { + switch (Type) { + case JSON_ARRAY: + return !Value.Array->empty(); + case JSON_MAP: + return !Value.Map->empty(); + case JSON_INTEGER: + case JSON_UINTEGER: + case JSON_DOUBLE: + return GetIntegerRobust(); + case JSON_STRING: + return GetIntegerRobust() || IsTrue(Value.String); + case JSON_NULL: + case JSON_UNDEFINED: + default: + return false; + case JSON_BOOLEAN: + return Value.Boolean; + } + } + + long long TJsonValue::GetIntegerRobust() const noexcept { + switch (Type) { + case JSON_ARRAY: + return Value.Array->size(); + case JSON_MAP: + return Value.Map->size(); + case JSON_BOOLEAN: + return Value.Boolean; + case JSON_DOUBLE: + return GetDoubleRobust(); + case JSON_STRING: + try { + i64 res = 0; + if (Value.String && TryFromString(Value.String, res)) { + return res; + } + } catch (const yexception&) { + } + return 0; + case JSON_NULL: + case JSON_UNDEFINED: + default: + return 0; + case JSON_INTEGER: + case JSON_UINTEGER: + return Value.Integer; + } + } + + unsigned long long TJsonValue::GetUIntegerRobust() const noexcept { + switch (Type) { + case JSON_ARRAY: + return Value.Array->size(); + case JSON_MAP: + return Value.Map->size(); + case JSON_BOOLEAN: + return Value.Boolean; + case JSON_DOUBLE: + return GetDoubleRobust(); + case JSON_STRING: + try { + ui64 res = 0; + if (Value.String && TryFromString(Value.String, res)) { + return res; + } + } catch (const yexception&) { + } + return 0; + case JSON_NULL: + case JSON_UNDEFINED: + default: + return 0; + case JSON_INTEGER: + case JSON_UINTEGER: + return Value.UInteger; + } + } + + double TJsonValue::GetDoubleRobust() const noexcept { + switch (Type) { + case JSON_ARRAY: + return Value.Array->size(); + case JSON_MAP: + return Value.Map->size(); + case JSON_BOOLEAN: + return Value.Boolean; + case JSON_INTEGER: + return Value.Integer; + case JSON_UINTEGER: + return Value.UInteger; + case JSON_STRING: + try { + double res = 0; + if (Value.String && TryFromString(Value.String, res)) { + return res; + } + } catch (const yexception&) { + } + return 0; + case JSON_NULL: + case JSON_UNDEFINED: + default: + return 0; + case JSON_DOUBLE: + return Value.Double; + } + } + + TString TJsonValue::GetStringRobust() const { + switch (Type) { + case JSON_ARRAY: + case JSON_MAP: + case JSON_BOOLEAN: + case JSON_DOUBLE: + case JSON_INTEGER: + case JSON_UINTEGER: + case JSON_NULL: + case JSON_UNDEFINED: + default: { + NJsonOrderedWriter::TBuf sout; + sout.WriteJsonValue(this); + return sout.Str(); + } + case JSON_STRING: + return Value.String; + } + } + + NJson::TJsonValue TJsonValue::GetNonOrderedJsonValue() const { + NJson::TJsonValue res; + switch (Type) { + case JSON_ARRAY: + res.SetType(NJson::EJsonValueType::JSON_ARRAY); + for (const auto& item : *Value.Array) { + res.AppendValue(item.GetNonOrderedJsonValue()); + } + return res; + case JSON_MAP: + res.SetType(NJson::EJsonValueType::JSON_MAP); + for (const auto& item : *Value.Map) { + res.InsertValue(item.first, item.second.GetNonOrderedJsonValue()); + } + return res; + case JSON_BOOLEAN: + return NJson::TJsonValue(Value.Boolean); + case JSON_INTEGER: + return NJson::TJsonValue(Value.Integer); + case JSON_UINTEGER: + return NJson::TJsonValue(Value.UInteger); + case JSON_STRING: + return NJson::TJsonValue(Value.String); + case JSON_NULL: + return NJson::TJsonValue(NJson::EJsonValueType::JSON_NULL); + case JSON_UNDEFINED: + return NJson::TJsonValue(); + case JSON_DOUBLE: + return NJson::TJsonValue(Value.Double); + } + } + + bool TJsonValue::GetBoolean(bool* value) const noexcept { + if (Type != JSON_BOOLEAN) + return false; + + *value = Value.Boolean; + return true; + } + + bool TJsonValue::GetInteger(long long* value) const noexcept { + if (!IsInteger()) + return false; + + *value = GetInteger(); + return true; + } + + bool TJsonValue::GetUInteger(unsigned long long* value) const noexcept { + if (!IsUInteger()) + return false; + + *value = GetUInteger(); + return true; + } + + bool TJsonValue::GetDouble(double* value) const noexcept { + if (!IsDouble()) + return false; + + *value = GetDouble(); + return true; + } + + bool TJsonValue::GetString(TString* value) const { + if (Type != JSON_STRING) + return false; + + *value = Value.String; + return true; + } + + bool TJsonValue::GetMap(TJsonValue::TMapType* value) const { + if (Type != JSON_MAP) + return false; + + *value = *Value.Map; + return true; + } + + bool TJsonValue::GetArray(TJsonValue::TArray* value) const { + if (Type != JSON_ARRAY) + return false; + + *value = *Value.Array; + return true; + } + + bool TJsonValue::GetMapPointer(const TJsonValue::TMapType** value) const noexcept { + if (Type != JSON_MAP) + return false; + + *value = Value.Map; + return true; + } + + bool TJsonValue::GetArrayPointer(const TJsonValue::TArray** value) const noexcept { + if (Type != JSON_ARRAY) + return false; + + *value = Value.Array; + return true; + } + + bool TJsonValue::GetValue(const size_t index, TJsonValue* value) const { + const TJsonValue* tmp = nullptr; + if (GetValuePointer(index, &tmp)) { + *value = *tmp; + return true; + } + return false; + } + + bool TJsonValue::GetValue(const TStringBuf key, TJsonValue* value) const { + const TJsonValue* tmp = nullptr; + if (GetValuePointer(key, &tmp)) { + *value = *tmp; + return true; + } + return false; + } + + bool TJsonValue::GetValuePointer(const size_t index, const TJsonValue** value) const noexcept { + if (Type == JSON_ARRAY && index < Value.Array->size()) { + *value = &(*Value.Array)[index]; + return true; + } + return false; + } + + bool TJsonValue::GetValuePointer(const TStringBuf key, const TJsonValue** value) const noexcept { + if (Type == JSON_MAP) { + const TMapType::const_iterator it = Value.Map->find(key); + if (it != Value.Map->end()) { + *value = &(it->second); + return true; + } + } + return false; + } + + bool TJsonValue::GetValuePointer(const TStringBuf key, TJsonValue** value) noexcept { + return static_cast<const TJsonValue*>(this)->GetValuePointer(key, const_cast<const TJsonValue**>(value)); + } + + bool TJsonValue::IsNull() const noexcept { + return Type == JSON_NULL; + } + + bool TJsonValue::IsBoolean() const noexcept { + return Type == JSON_BOOLEAN; + } + + bool TJsonValue::IsInteger() const noexcept { + switch (Type) { + case JSON_INTEGER: + return true; + + case JSON_UINTEGER: + return (Value.UInteger <= static_cast<unsigned long long>(Max<long long>())); + + case JSON_DOUBLE: + return ((long long)Value.Double == Value.Double); + + default: + return false; + } + } + + bool TJsonValue::IsUInteger() const noexcept { + switch (Type) { + case JSON_UINTEGER: + return true; + + case JSON_INTEGER: + return (Value.Integer >= 0); + + case JSON_DOUBLE: + return ((unsigned long long)Value.Double == Value.Double); + + default: + return false; + } + } + + bool TJsonValue::IsDouble() const noexcept { + // Check whether we can convert integer to floating-point + // without precision loss. + switch (Type) { + case JSON_DOUBLE: + return true; + + case JSON_INTEGER: + return (1ll << std::numeric_limits<double>::digits) >= Abs(Value.Integer); + + case JSON_UINTEGER: + return (1ull << std::numeric_limits<double>::digits) >= Value.UInteger; + + default: + return false; + } + } + + namespace { + template <class TPtr, class T> + TPtr* CreateOrNullptr(TPtr* p, T key, std::true_type /*create*/) { + return &(*p)[key]; + } + + template <class TPtr, class T> + TPtr* CreateOrNullptr(const TPtr* p, T key, std::false_type /*create*/) noexcept { + const TPtr* const next = &(*p)[key]; + return next->IsDefined() ? const_cast<TPtr*>(next) : nullptr; + } + + template <bool Create, class TJsonPtr> + TJsonPtr GetValuePtrByPath(TJsonPtr currentJson, TStringBuf path, char delimiter) noexcept(!Create) { + static_assert( + !(Create && std::is_const<std::remove_pointer_t<TJsonPtr>>::value), + "TJsonPtr must be a `TJsonValue*` if `Create` is true"); + constexpr std::integral_constant<bool, Create> create_tag{}; + + while (!path.empty()) { + size_t index = 0; + const TStringBuf step = path.NextTok(delimiter); + if (step.size() > 2 && *step.begin() == '[' && step.back() == ']' && TryFromString(step.substr(1, step.size() - 2), index)) { + currentJson = CreateOrNullptr(currentJson, index, create_tag); + } else { + currentJson = CreateOrNullptr(currentJson, step, create_tag); + } + + if (!currentJson) { + return nullptr; + } + } + + return currentJson; + } + } // anonymous namespace + + bool TJsonValue::GetValueByPath(const TStringBuf path, TJsonValue& result, char delimiter) const { + const TJsonValue* const ptr = GetValuePtrByPath<false>(this, path, delimiter); + if (ptr) { + result = *ptr; + return true; + } + return false; + } + + bool TJsonValue::SetValueByPath(const TStringBuf path, const TJsonValue& value, char delimiter) { + TJsonValue* const ptr = GetValuePtrByPath<true>(this, path, delimiter); + if (ptr) { + *ptr = value; + return true; + } + return false; + } + + bool TJsonValue::SetValueByPath(const TStringBuf path, TJsonValue&& value, char delimiter) { + TJsonValue* const ptr = GetValuePtrByPath<true>(this, path, delimiter); + if (ptr) { + *ptr = std::move(value); + return true; + } + return false; + } + + const TJsonValue* TJsonValue::GetValueByPath(const TStringBuf key, char delim) const noexcept { + return GetValuePtrByPath<false>(this, key, delim); + } + + TJsonValue* TJsonValue::GetValueByPath(const TStringBuf key, char delim) noexcept { + return GetValuePtrByPath<false>(this, key, delim); + } + + void TJsonValue::DoScan(const TString& path, TJsonValue* parent, IScanCallback& callback) { + if (!callback.Do(path, parent, *this)) { + return; + } + + if (Type == JSON_MAP) { + for (auto&& i : *Value.Map) { + i.second.DoScan(!!path ? TString::Join(path, ".", i.first) : i.first, this, callback); + } + } else if (Type == JSON_ARRAY) { + for (ui32 i = 0; i < Value.Array->size(); ++i) { + (*Value.Array)[i].DoScan(TString::Join(path, "[", ToString(i), "]"), this, callback); + } + } + } + + void TJsonValue::Scan(IScanCallback& callback) { + DoScan("", nullptr, callback); + } + + bool TJsonValue::IsString() const noexcept { + return Type == JSON_STRING; + } + + bool TJsonValue::IsMap() const noexcept { + return Type == JSON_MAP; + } + + bool TJsonValue::IsArray() const noexcept { + return Type == JSON_ARRAY; + } + + bool TJsonValue::Has(const TStringBuf& key) const noexcept { + return Type == JSON_MAP && Value.Map->contains(key); + } + + bool TJsonValue::Has(size_t key) const noexcept { + return Type == JSON_ARRAY && Value.Array->size() > key; + } + + bool TJsonValue::operator==(const TJsonValue& rhs) const { + switch (Type) { + case JSON_UNDEFINED: { + return (rhs.GetType() == JSON_UNDEFINED); + } + + case JSON_NULL: { + return rhs.IsNull(); + } + + case JSON_BOOLEAN: { + return (rhs.IsBoolean() && Value.Boolean == rhs.Value.Boolean); + } + + case JSON_INTEGER: { + return (rhs.IsInteger() && GetInteger() == rhs.GetInteger()); + } + + case JSON_UINTEGER: { + return (rhs.IsUInteger() && GetUInteger() == rhs.GetUInteger()); + } + + case JSON_STRING: { + return (rhs.IsString() && Value.String == rhs.Value.String); + } + + case JSON_DOUBLE: { + return (rhs.IsDouble() && fabs(GetDouble() - rhs.GetDouble()) <= FLT_EPSILON); + } + + case JSON_MAP: + return AreJsonMapsEqual(*this, rhs); + + case JSON_ARRAY: + return AreJsonArraysEqual(*this, rhs); + + default: + Y_ASSERT(false && "Unknown type."); + return false; + } + } + + void TJsonValue::SwapWithUndefined(TJsonValue& output) noexcept { + if (Type == JSON_STRING) { + static_assert(std::is_nothrow_move_constructible<TString>::value, "noexcept violation! Add some try {} catch (...) logic"); + new (&output.Value.String) TString(std::move(Value.String)); + Value.String.~TString(); + } else { + std::memcpy(&output.Value, &Value, sizeof(Value)); + } + + output.Type = Type; + Type = JSON_UNDEFINED; + } + + void TJsonValue::Swap(TJsonValue& rhs) noexcept { + TJsonValue tmp(std::move(*this)); + rhs.SwapWithUndefined(*this); + tmp.SwapWithUndefined(rhs); + } + + void TJsonValue::Save(IOutputStream* s) const { + GetNonOrderedJsonValue().Save(s); + } + + void TJsonValue::Load(IInputStream* s) { + auto temp = GetNonOrderedJsonValue(); + temp.Load(s); + *this = TJsonValue(std::move(temp)); + } + + //**************************************************************** + + bool GetMapPointer(const TJsonValue& jv, const size_t index, const TJsonValue::TMapType** value) { + const TJsonValue* v; + if (!jv.GetValuePointer(index, &v) || !v->IsMap()) + return false; + + *value = &v->GetMap(); + return true; + } + + bool GetArrayPointer(const TJsonValue& jv, const size_t index, const TJsonValue::TArray** value) { + const TJsonValue* v; + if (!jv.GetValuePointer(index, &v) || !v->IsArray()) + return false; + + *value = &v->GetArray(); + return true; + } + + bool GetMapPointer(const TJsonValue& jv, const TStringBuf key, const TJsonValue::TMapType** value) { + const TJsonValue* v; + if (!jv.GetValuePointer(key, &v) || !v->IsMap()) + return false; + + *value = &v->GetMap(); + return true; + } + + bool GetArrayPointer(const TJsonValue& jv, const TStringBuf key, const TJsonValue::TArray** value) { + const TJsonValue* v; + if (!jv.GetValuePointer(key, &v) || !v->IsArray()) + return false; + + *value = &v->GetArray(); + return true; + } + + void TJsonValue::BackChecks() const { + if (Type != JSON_ARRAY) + ythrow TJsonException() << "Not an array"; + + if (Value.Array->empty()) + ythrow TJsonException() << "Get back on empty array"; + } +} + +template <> +void Out<NJson::NOrderedJson::TJsonValue>(IOutputStream& out, const NJson::NOrderedJson::TJsonValue& v) { + NJsonOrderedWriter::TBuf buf(NJsonOrderedWriter::HEM_DONT_ESCAPE_HTML, &out); + buf.WriteJsonValue(&v); +} diff --git a/library/cpp/json/ordered_maps/json_value_ordered.h b/library/cpp/json/ordered_maps/json_value_ordered.h new file mode 100644 index 00000000000..bb7476a552b --- /dev/null +++ b/library/cpp/json/ordered_maps/json_value_ordered.h @@ -0,0 +1,303 @@ +#pragma once + +#include <library/cpp/json/common/defs.h> +#include <library/cpp/containers/ordered_map/ordered_map.h> +#include <library/cpp/json/writer/json_value.h> + +#include <util/generic/string.h> +#include <util/generic/hash.h> +#include <util/generic/vector.h> +#include <util/generic/deque.h> +#include <util/generic/utility.h> +#include <util/generic/yexception.h> +#include <util/system/compiler.h> + +namespace NJson::NOrderedJson { + enum EJsonValueType { + JSON_UNDEFINED /* "Undefined" */, + JSON_NULL /* "Null" */, + JSON_BOOLEAN /* "Boolean" */, + JSON_INTEGER /* "Integer" */, + JSON_DOUBLE /* "Double" */, + JSON_STRING /* "String" */, + JSON_MAP /* "Map" */, + JSON_ARRAY /* "Array" */, + JSON_UINTEGER /* "UInteger" */ + }; + + class TJsonValue; + + class IScanCallback { + public: + virtual ~IScanCallback() = default; + + virtual bool Do(const TString& path, TJsonValue* parent, TJsonValue& value) = 0; + }; + + class TJsonValue { + void Clear() noexcept; + + public: + typedef NOrderedMap::TOrderedMap<TString, TJsonValue> TMapType; + typedef TDeque<TJsonValue> TArray; + + TJsonValue() noexcept = default; + TJsonValue(EJsonValueType type); + TJsonValue(bool value) noexcept; + TJsonValue(int value) noexcept; + TJsonValue(unsigned int value) noexcept; + TJsonValue(long value) noexcept; + TJsonValue(unsigned long value) noexcept; + TJsonValue(long long value) noexcept; + TJsonValue(unsigned long long value) noexcept; + TJsonValue(double value) noexcept; + TJsonValue(TString value); + TJsonValue(const char* value); + template <class T> + TJsonValue(const T*) = delete; + TJsonValue(TStringBuf value); + + TJsonValue(const std::string& s) + : TJsonValue(TStringBuf(s)) + { + } + + TJsonValue(const TJsonValue& vval); + TJsonValue(TJsonValue&& vval) noexcept; + + explicit TJsonValue(const NJson::TJsonValue& vval); + + TJsonValue& operator=(const TJsonValue& val) Y_LIFETIME_BOUND; + TJsonValue& operator=(TJsonValue&& val) noexcept Y_LIFETIME_BOUND; + + ~TJsonValue() { + Clear(); + } + + EJsonValueType GetType() const noexcept; + TJsonValue& SetType(EJsonValueType type) Y_LIFETIME_BOUND; + + TJsonValue& SetValue(const TJsonValue& value) Y_LIFETIME_BOUND; + TJsonValue& SetValue(TJsonValue&& value) Y_LIFETIME_BOUND; + + // for Map + TJsonValue& InsertValue(const TString& key, const TJsonValue& value) Y_LIFETIME_BOUND; + TJsonValue& InsertValue(TStringBuf key, const TJsonValue& value) Y_LIFETIME_BOUND; + TJsonValue& InsertValue(const char* key, const TJsonValue& value) Y_LIFETIME_BOUND; + TJsonValue& InsertValue(const TString& key, TJsonValue&& value) Y_LIFETIME_BOUND; + TJsonValue& InsertValue(TStringBuf key, TJsonValue&& value) Y_LIFETIME_BOUND; + TJsonValue& InsertValue(const char* key, TJsonValue&& value) Y_LIFETIME_BOUND; + + // for Array + TJsonValue& AppendValue(const TJsonValue& value) Y_LIFETIME_BOUND; + TJsonValue& AppendValue(TJsonValue&& value) Y_LIFETIME_BOUND; + TJsonValue& Back() Y_LIFETIME_BOUND; + const TJsonValue& Back() const Y_LIFETIME_BOUND; + + bool GetValueByPath(TStringBuf path, TJsonValue& result, char delimiter = '.') const; + bool SetValueByPath(TStringBuf path, const TJsonValue& value, char delimiter = '.'); + bool SetValueByPath(TStringBuf path, TJsonValue&& value, char delimiter = '.'); + + // returns NULL on failure + const TJsonValue* GetValueByPath(TStringBuf path, char delimiter = '.') const noexcept Y_LIFETIME_BOUND; + TJsonValue* GetValueByPath(TStringBuf path, char delimiter = '.') noexcept Y_LIFETIME_BOUND; + + void EraseValue(TStringBuf key); + void EraseValue(size_t index); + + TJsonValue& operator[](size_t idx) Y_LIFETIME_BOUND; + TJsonValue& operator[](const TStringBuf& key) Y_LIFETIME_BOUND; + const TJsonValue& operator[](size_t idx) const noexcept Y_LIFETIME_BOUND; + const TJsonValue& operator[](const TStringBuf& key) const noexcept Y_LIFETIME_BOUND; + + bool GetBoolean() const; + long long GetInteger() const; + unsigned long long GetUInteger() const; + double GetDouble() const; + const TString& GetString() const Y_LIFETIME_BOUND; + const TMapType& GetMap() const Y_LIFETIME_BOUND; + const TArray& GetArray() const Y_LIFETIME_BOUND; + + //throwing TJsonException possible + bool GetBooleanSafe() const; + long long GetIntegerSafe() const; + unsigned long long GetUIntegerSafe() const; + double GetDoubleSafe() const; + const TString& GetStringSafe() const Y_LIFETIME_BOUND; + const TMapType& GetMapSafe() const Y_LIFETIME_BOUND; + TMapType& GetMapSafe() Y_LIFETIME_BOUND; + const TArray& GetArraySafe() const Y_LIFETIME_BOUND; + TArray& GetArraySafe() Y_LIFETIME_BOUND; + + bool GetBooleanSafe(bool defaultValue) const; + long long GetIntegerSafe(long long defaultValue) const; + unsigned long long GetUIntegerSafe(unsigned long long defaultValue) const; + double GetDoubleSafe(double defaultValue) const; + TString GetStringSafe(const TString& defaultValue) const; + + bool GetBooleanRobust() const noexcept; + long long GetIntegerRobust() const noexcept; + unsigned long long GetUIntegerRobust() const noexcept; + double GetDoubleRobust() const noexcept; + TString GetStringRobust() const; + NJson::TJsonValue GetNonOrderedJsonValue() const; + + // Exception-free accessors + bool GetBoolean(bool* value) const noexcept; + bool GetInteger(long long* value) const noexcept; + bool GetUInteger(unsigned long long* value) const noexcept; + bool GetDouble(double* value) const noexcept; + bool GetMapPointer(const TMapType** value) const noexcept; + bool GetArrayPointer(const TArray** value) const noexcept; + + bool GetString(TString* value) const; + bool GetMap(TMapType* value) const; + bool GetArray(TArray* value) const; + bool GetValue(size_t index, TJsonValue* value) const; + bool GetValue(TStringBuf key, TJsonValue* value) const; + bool GetValuePointer(size_t index, const TJsonValue** value) const noexcept; + bool GetValuePointer(TStringBuf key, const TJsonValue** value) const noexcept; + bool GetValuePointer(TStringBuf key, TJsonValue** value) noexcept; + + // Checking for defined non-null value + bool IsDefined() const noexcept { + return Type != JSON_UNDEFINED && Type != JSON_NULL; + } + + bool IsNull() const noexcept; + bool IsBoolean() const noexcept; + bool IsDouble() const noexcept; + bool IsString() const noexcept; + bool IsMap() const noexcept; + bool IsArray() const noexcept; + + /// @return true if JSON_INTEGER or (JSON_UINTEGER and Value <= Max<long long>) + bool IsInteger() const noexcept; + + /// @return true if JSON_UINTEGER or (JSON_INTEGER and Value >= 0) + bool IsUInteger() const noexcept; + + bool Has(const TStringBuf& key) const noexcept; + bool Has(size_t key) const noexcept; + + void Scan(IScanCallback& callback); + + /// Non-robust comparison. + bool operator==(const TJsonValue& rhs) const; + + void Swap(TJsonValue& rhs) noexcept; + + static const TJsonValue UNDEFINED; + + // save using util/ysaveload.h serialization (not to JSON stream) + void Save(IOutputStream* s) const; + // load using util/ysaveload.h serialization (not as JSON stream) + void Load(IInputStream* s); + + NJson::TJsonValue MakeNonOrderedTJsonValue() const; + + private: + EJsonValueType Type = JSON_UNDEFINED; + union TValueUnion { + bool Boolean; + long long Integer; + unsigned long long UInteger; + double Double; + TString String; + TMapType* Map; + TArray* Array; + + TValueUnion() noexcept { + Zero(*this); + } + ~TValueUnion() noexcept { + } + }; + TValueUnion Value; + void DoScan(const TString& path, TJsonValue* parent, IScanCallback& callback); + void SwapWithUndefined(TJsonValue& output) noexcept; + + /** + @throw yexception if Back shouldn't be called on the object. + */ + void BackChecks() const; + }; + + inline bool GetBoolean(const TJsonValue& jv, size_t index, bool* value) noexcept { + return jv[index].GetBoolean(value); + } + + inline bool GetInteger(const TJsonValue& jv, size_t index, long long* value) noexcept { + return jv[index].GetInteger(value); + } + + inline bool GetUInteger(const TJsonValue& jv, size_t index, unsigned long long* value) noexcept { + return jv[index].GetUInteger(value); + } + + inline bool GetDouble(const TJsonValue& jv, size_t index, double* value) noexcept { + return jv[index].GetDouble(value); + } + + inline bool GetString(const TJsonValue& jv, size_t index, TString* value) { + return jv[index].GetString(value); + } + + bool GetMapPointer(const TJsonValue& jv, size_t index, const TJsonValue::TMapType** value); + bool GetArrayPointer(const TJsonValue& jv, size_t index, const TJsonValue::TArray** value); + + inline bool GetBoolean(const TJsonValue& jv, TStringBuf key, bool* value) noexcept { + return jv[key].GetBoolean(value); + } + + inline bool GetInteger(const TJsonValue& jv, TStringBuf key, long long* value) noexcept { + return jv[key].GetInteger(value); + } + + inline bool GetUInteger(const TJsonValue& jv, TStringBuf key, unsigned long long* value) noexcept { + return jv[key].GetUInteger(value); + } + + inline bool GetDouble(const TJsonValue& jv, TStringBuf key, double* value) noexcept { + return jv[key].GetDouble(value); + } + + inline bool GetString(const TJsonValue& jv, TStringBuf key, TString* value) { + return jv[key].GetString(value); + } + + bool GetMapPointer(const TJsonValue& jv, const TStringBuf key, const TJsonValue::TMapType** value); + bool GetArrayPointer(const TJsonValue& jv, const TStringBuf key, const TJsonValue::TArray** value); + + class TJsonMap: public TJsonValue { + public: + TJsonMap() + : TJsonValue(NJson::NOrderedJson::JSON_MAP) + {} + + TJsonMap(const std::initializer_list<std::pair<TString, TJsonValue>>& list) + : TJsonValue(NJson::NOrderedJson::JSON_MAP) + { + GetMapSafe() = [&list]() { + TJsonValue::TMapType map; + for (const auto& [key, value] : list) { + map[key] = value; + } + return map; + }(); + } + }; + + class TJsonArray: public TJsonValue { + public: + TJsonArray() + : TJsonValue(NJson::NOrderedJson::JSON_ARRAY) + {} + + TJsonArray(const std::initializer_list<TJsonValue>& list) + : TJsonValue(NJson::NOrderedJson::JSON_ARRAY) + { + GetArraySafe() = TJsonValue::TArray(list); + } + }; +} diff --git a/library/cpp/json/ordered_maps/json_writer_ordered.cpp b/library/cpp/json/ordered_maps/json_writer_ordered.cpp new file mode 100644 index 00000000000..39f63a3e567 --- /dev/null +++ b/library/cpp/json/ordered_maps/json_writer_ordered.cpp @@ -0,0 +1,149 @@ +#include "json_writer_ordered.h" + +#include <util/charset/utf8.h> +#include <util/generic/algorithm.h> +#include <util/string/cast.h> +#include <util/system/yassert.h> + +namespace NJson::NOrderedJson { + TJsonWriter::TJsonWriter(IOutputStream* out, bool formatOutput, bool sortkeys, bool validateUtf8) + : Out(out) + , Buf(NJsonOrderedWriter::HEM_UNSAFE) + , DoubleNDigits(TJsonWriterConfig::DefaultDoubleNDigits) + , FloatNDigits(TJsonWriterConfig::DefaultFloatNDigits) + , FloatToStringMode(TJsonWriterConfig::DefaultFloatToStringMode) + , SortKeys(sortkeys) + , ValidateUtf8(validateUtf8) + , DontEscapeStrings(false) + , DontFlushInDestructor(false) + { + Buf.SetIndentSpaces(formatOutput ? 2 : 0); + } + + TJsonWriter::TJsonWriter(IOutputStream* out, const TJsonWriterConfig& config, bool DFID) + : Out(config.Unbuffered ? nullptr : out) + , Buf(NJsonOrderedWriter::HEM_UNSAFE, config.Unbuffered ? out : nullptr) + , DoubleNDigits(config.DoubleNDigits) + , FloatNDigits(config.FloatNDigits) + , FloatToStringMode(config.FloatToStringMode) + , SortKeys(config.SortKeys) + , ValidateUtf8(config.ValidateUtf8) + , DontEscapeStrings(config.DontEscapeStrings) + , DontFlushInDestructor(DFID) + { + Buf.SetIndentSpaces(config.FormatOutput ? 2 : 0); + Buf.SetWriteNanAsString(config.WriteNanAsString); + } + + TJsonWriter::~TJsonWriter() { + // if we write to socket it's possible to get exception here + // don't use exceptions in destructors + if (!DontFlushInDestructor) { + try { + Flush(); + } catch (...) { + } + } + } + + void TJsonWriter::Flush() { + if (Out) { + Buf.FlushTo(Out); + } + } + + void TJsonWriter::OpenMap() { + Buf.BeginObject(); + } + + void TJsonWriter::CloseMap() { + Buf.EndObject(); + } + + void TJsonWriter::OpenArray() { + Buf.BeginList(); + } + + void TJsonWriter::CloseArray() { + Buf.EndList(); + } + + void TJsonWriter::Write(const TStringBuf& value) { + if (ValidateUtf8 && !IsUtf(value)) + throw yexception() << "JSON writer: invalid UTF-8"; + if (Buf.KeyExpected()) { + Buf.WriteKey(value); + } else { + if (DontEscapeStrings) { + Buf.UnsafeWriteValue(TString("\"") + value + '"'); + } else { + Buf.WriteString(value); + } + } + } + + void TJsonWriter::WriteNull() { + Buf.WriteNull(); + } + + void TJsonWriter::Write(float value) { + Buf.WriteFloat(value, FloatToStringMode, FloatNDigits); + } + + void TJsonWriter::Write(double value) { + Buf.WriteDouble(value, FloatToStringMode, DoubleNDigits); + } + + void TJsonWriter::Write(long long value) { + Buf.WriteLongLong(value); + } + + void TJsonWriter::Write(unsigned long long value) { + Buf.WriteULongLong(value); + } + + void TJsonWriter::Write(bool value) { + Buf.WriteBool(value); + } + + namespace { + struct TLessStrPtr { + bool operator()(const TString* a, const TString* b) const { + return *a < *b; + } + }; + } + + void TJsonWriter::Write(const TJsonValue* v) { + Buf.WriteJsonValue(v, SortKeys, FloatToStringMode, DoubleNDigits); + } + + void TJsonWriter::Write(const TJsonValue& v) { + Buf.WriteJsonValue(&v, SortKeys, FloatToStringMode, DoubleNDigits); + } + + TString WriteJson(const TJsonValue* value, bool formatOutput, bool sortkeys, bool validateUtf8) { + TStringStream ss; + WriteJson(&ss, value, formatOutput, sortkeys, validateUtf8); + return ss.Str(); + } + + TString WriteJson(const TJsonValue& value, bool formatOutput, bool sortkeys, bool validateUtf8) { + TStringStream ss; + WriteJson(&ss, &value, formatOutput, sortkeys, validateUtf8); + return ss.Str(); + } + + void WriteJson(IOutputStream* out, const TJsonValue* val, bool formatOutput, bool sortkeys, bool validateUtf8) { + TJsonWriter w(out, formatOutput, sortkeys, validateUtf8); + w.Write(val); + w.Flush(); + } + + void WriteJson(IOutputStream* out, const TJsonValue* val, const TJsonWriterConfig& config) { + TJsonWriter w(out, config, true); + w.Write(val); + w.Flush(); + } + +} diff --git a/library/cpp/json/ordered_maps/json_writer_ordered.h b/library/cpp/json/ordered_maps/json_writer_ordered.h new file mode 100644 index 00000000000..cfb2e30bd49 --- /dev/null +++ b/library/cpp/json/ordered_maps/json_writer_ordered.h @@ -0,0 +1,194 @@ +#pragma once + +#include "json_value_ordered.h" + +#include <library/cpp/json/ordered_maps/json_ordered.h> + +#include <util/stream/output.h> +#include <util/generic/hash.h> +#include <util/generic/maybe.h> +#include <util/generic/strbuf.h> + +namespace NJson::NOrderedJson { + struct TJsonWriterConfig { + constexpr static ui32 DefaultDoubleNDigits = 10; + constexpr static ui32 DefaultFloatNDigits = 6; + constexpr static EFloatToStringMode DefaultFloatToStringMode = PREC_NDIGITS; + + inline TJsonWriterConfig& SetUnbuffered(bool v) noexcept { + Unbuffered = v; + + return *this; + } + + inline TJsonWriterConfig& SetValidateUtf8(bool v) noexcept { + ValidateUtf8 = v; + + return *this; + } + + inline TJsonWriterConfig& SetFormatOutput(bool v) noexcept { + FormatOutput = v; + + return *this; + } + + ui32 DoubleNDigits = DefaultDoubleNDigits; + ui32 FloatNDigits = DefaultFloatNDigits; + EFloatToStringMode FloatToStringMode = DefaultFloatToStringMode; + bool FormatOutput = false; + bool SortKeys = false; + bool ValidateUtf8 = true; + bool DontEscapeStrings = false; + bool Unbuffered = false; + bool WriteNanAsString = false; // NaN and Inf are not valid json values, so if WriteNanAsString is set, writer would write string intead of throwing exception (default case) + }; + + class TJsonWriter { + IOutputStream* Out; + NJsonOrderedWriter::TBuf Buf; + const ui32 DoubleNDigits; + const ui32 FloatNDigits; + const EFloatToStringMode FloatToStringMode; + const bool SortKeys; + const bool ValidateUtf8; + const bool DontEscapeStrings; + const bool DontFlushInDestructor; + + public: + TJsonWriter(IOutputStream* out, bool formatOutput, bool sortkeys = false, bool validateUtf8 = true); + TJsonWriter(IOutputStream* out, const TJsonWriterConfig& config, bool DontFlushInDestructor = false); + ~TJsonWriter(); + + void Flush(); + + void OpenMap(); + void OpenMap(const TStringBuf& key) { + Buf.WriteKey(key); + OpenMap(); + } + void CloseMap(); + + void OpenArray(); + void OpenArray(const TStringBuf& key) { + Buf.WriteKey(key); + OpenArray(); + } + void CloseArray(); + + void WriteNull(); + + void Write(const TStringBuf& value); + void Write(float value); + void Write(double value); + void Write(bool value); + void Write(const TJsonValue* value); + void Write(const TJsonValue& value); + + // must use all variations of integer types since long + // and long long are different types but with same size + void Write(long long value); + void Write(unsigned long long value); + void Write(long value) { + Write((long long)value); + } + void Write(unsigned long value) { + Write((unsigned long long)value); + } + void Write(int value) { + Write((long long)value); + } + void Write(unsigned int value) { + Write((unsigned long long)value); + } + void Write(short value) { + Write((long long)value); + } + void Write(unsigned short value) { + Write((unsigned long long)value); + } + + void Write(const unsigned char* value) { + Write((const char*)value); + } + void Write(const char* value) { + Write(TStringBuf(value)); + } + void Write(const TString& value) { + Write(TStringBuf(value)); + } + void Write(const std::string& value) { + Write(TStringBuf(value)); + } + + // write raw json without checks + void UnsafeWrite(const TStringBuf& value) { + Buf.UnsafeWriteValue(value); + } + + template <typename T> + void Write(const TStringBuf& key, const T& value) { + Buf.WriteKey(key); + Write(value); + } + + // write raw json without checks + void UnsafeWrite(const TStringBuf& key, const TStringBuf& value) { + Buf.WriteKey(key); + UnsafeWrite(value); + } + + void WriteNull(const TStringBuf& key) { + Buf.WriteKey(key); + WriteNull(); + } + + template <typename T> + void WriteOptional(const TStringBuf& key, const TMaybe<T>& value) { + if (value) { + Write(key, *value); + } + } + + void WriteOptional(const TStringBuf&, const TNothing&) { + // nothing to do + } + + void WriteKey(const TStringBuf key) { + Buf.WriteKey(key); + } + + void WriteKey(const unsigned char* key) { + WriteKey((const char*)key); + } + + void WriteKey(const char* key) { + WriteKey(TStringBuf{key}); + } + + void WriteKey(const TString& key) { + WriteKey(TStringBuf{key}); + } + + void WriteKey(const std::string& key) { + WriteKey(TStringBuf{key}); + } + + NJsonOrderedWriter::TBufState State() const { + return Buf.State(); + } + + void Reset(const NJsonOrderedWriter::TBufState& from) { + return Buf.Reset(from); + } + + void Reset(NJsonOrderedWriter::TBufState&& from) { + return Buf.Reset(std::move(from)); + } + }; + + void WriteJson(IOutputStream*, const TJsonValue*, bool formatOutput = false, bool sortkeys = false, bool validateUtf8 = true); + TString WriteJson(const TJsonValue*, bool formatOutput = true, bool sortkeys = false, bool validateUtf8 = false); + TString WriteJson(const TJsonValue&, bool formatOutput = true, bool sortkeys = false, bool validateUtf8 = false); + void WriteJson(IOutputStream*, const TJsonValue*, const TJsonWriterConfig& config); +} diff --git a/library/cpp/json/ordered_maps/ut/json_ordered_ut.cpp b/library/cpp/json/ordered_maps/ut/json_ordered_ut.cpp new file mode 100644 index 00000000000..2607814de3c --- /dev/null +++ b/library/cpp/json/ordered_maps/ut/json_ordered_ut.cpp @@ -0,0 +1,268 @@ +#include <library/cpp/testing/unittest/registar.h> +#include <util/system/sanitizers.h> + +#include <library/cpp/json/ordered_maps/json_ordered.h> +#include <library/cpp/json/ordered_maps/json_value_ordered.h> + +#include <limits> + +Y_UNIT_TEST_SUITE(JsonWriter) { + Y_UNIT_TEST(Struct) { + NJsonOrderedWriter::TBuf w; + w.BeginList(); + w.BeginObject() + .WriteKey("key") + .WriteString("value") + .UnsafeWritePair("\"xk\":13") + .WriteKey("key2") + .BeginList() + .BeginObject() + .EndObject() + .BeginObject() + .EndObject() + .EndList() + .EndObject(); + w.WriteInt(43); + w.UnsafeWriteValue("\"x\""); + w.WriteString("..."); + w.EndList(); + const char* exp = "[{\"key\":\"value\",\"xk\":13,\"key2\":[{},{}]},43,\"x\",\"...\"]"; + UNIT_ASSERT_EQUAL(w.Str(), exp); + } + Y_UNIT_TEST(EscapedString) { + NJsonOrderedWriter::TBuf w(NJsonOrderedWriter::HEM_ESCAPE_HTML); + w.WriteString(" \n \r \t \007 \b \f ' <tag> &ent; \"txt\" "); + TString ws = w.Str(); + const char* exp = "\" \\n \\r \\t \\u0007 \\b \\f ' <tag> &ent; "txt" \""; + UNIT_ASSERT_STRINGS_EQUAL(ws.c_str(), exp); + } + Y_UNIT_TEST(UnescapedString) { + NJsonOrderedWriter::TBuf w; + w.WriteString(" \n \r \t \b \f '; -- <tag> &ent; \"txt\"", NJsonOrderedWriter::HEM_DONT_ESCAPE_HTML); + TString ws = w.Str(); + const char* exp = "\" \\n \\r \\t \\b \\f \\u0027; -- \\u003Ctag\\u003E &ent; \\\"txt\\\"\""; + UNIT_ASSERT_STRINGS_EQUAL(ws.c_str(), exp); + } + Y_UNIT_TEST(UnescapedChaining) { + NJsonOrderedWriter::TBuf w(NJsonOrderedWriter::HEM_DONT_ESCAPE_HTML); + w.UnsafeWriteRawBytes("(", 1); + w.BeginList().WriteString("<>&'\\").BeginList(); + w.EndList().EndList(); + TString ws = w.Str(); + const char* exp = "([\"\\u003C\\u003E&\\u0027\\\\\",[]]"; + UNIT_ASSERT_STRINGS_EQUAL(ws.c_str(), exp); + } + Y_UNIT_TEST(Utf8) { + TString ws = NJsonOrderedWriter::TBuf().WriteString("яЯ σΣ ש א").Str(); + const char* exp = "\"яЯ σΣ ש א\""; + UNIT_ASSERT_STRINGS_EQUAL(ws.c_str(), exp); + } + Y_UNIT_TEST(WrongObject) { + NJsonOrderedWriter::TBuf w; + w.BeginObject(); + UNIT_ASSERT_EXCEPTION(w.WriteString("hehe"), NJsonOrderedWriter::TError); + } + Y_UNIT_TEST(WrongList) { + NJsonOrderedWriter::TBuf w; + w.BeginList(); + UNIT_ASSERT_EXCEPTION(w.WriteKey("hehe"), NJsonOrderedWriter::TError); + } + Y_UNIT_TEST(Incomplete) { + NJsonOrderedWriter::TBuf w; + w.BeginList(); + UNIT_ASSERT_EXCEPTION(w.Str(), NJsonOrderedWriter::TError); + } + Y_UNIT_TEST(BareKey) { + NJsonOrderedWriter::TBuf w; + w.BeginObject() + .CompatWriteKeyWithoutQuotes("p") + .WriteInt(1) + .CompatWriteKeyWithoutQuotes("n") + .WriteInt(0) + .EndObject(); + TString ws = w.Str(); + const char* exp = "{p:1,n:0}"; + UNIT_ASSERT_STRINGS_EQUAL(ws.c_str(), exp); + } + Y_UNIT_TEST(UnescapedStringInObject) { + NJsonOrderedWriter::TBuf w(NJsonOrderedWriter::HEM_DONT_ESCAPE_HTML); + w.BeginObject().WriteKey("key").WriteString("</&>'").EndObject(); + TString ws = w.Str(); + const char* exp = "{\"key\":\"\\u003C\\/&\\u003E\\u0027\"}"; + UNIT_ASSERT_STRINGS_EQUAL(ws.c_str(), exp); + } + Y_UNIT_TEST(ForeignStreamStr) { + NJsonOrderedWriter::TBuf w(NJsonOrderedWriter::HEM_DONT_ESCAPE_HTML, &Cerr); + UNIT_ASSERT_EXCEPTION(w.Str(), NJsonOrderedWriter::TError); + } + Y_UNIT_TEST(ForeignStreamValue) { + TStringStream ss; + NJsonOrderedWriter::TBuf w(NJsonOrderedWriter::HEM_DONT_ESCAPE_HTML, &ss); + w.WriteInt(1543); + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), "1543"); + } + Y_UNIT_TEST(Indentation) { + NJsonOrderedWriter::TBuf w(NJsonOrderedWriter::HEM_DONT_ESCAPE_HTML); + w.SetIndentSpaces(2); + w.BeginList() + .WriteInt(1) + .WriteString("hello") + .BeginObject() + .WriteKey("abc") + .WriteInt(3) + .WriteKey("def") + .WriteInt(4) + .EndObject() + .EndList(); + const char* exp = "[\n" + " 1,\n" + " \"hello\",\n" + " {\n" + " \"abc\":3,\n" + " \"def\":4\n" + " }\n" + "]"; + UNIT_ASSERT_STRINGS_EQUAL(exp, w.Str()); + } + Y_UNIT_TEST(WriteJsonValue) { + using namespace NJson::NOrderedJson; + TJsonValue val; + val.AppendValue(1); + val.AppendValue("2"); + val.AppendValue(3.5); + TJsonValue obj; + obj.InsertValue("key", TJsonValue("value")); + + val.AppendValue(obj); + val.AppendValue(TJsonValue(JSON_NULL)); + + NJsonOrderedWriter::TBuf w(NJsonOrderedWriter::HEM_DONT_ESCAPE_HTML); + w.WriteJsonValue(&val); + + const char exp[] = "[1,\"2\",3.5,{\"key\":\"value\"},null]"; + UNIT_ASSERT_STRINGS_EQUAL(exp, w.Str()); + } + Y_UNIT_TEST(WriteJsonValueSorted) { + using namespace NJson::NOrderedJson; + TJsonValue val; + val.InsertValue("1", TJsonValue(1)); + val.InsertValue("2", TJsonValue(2)); + + TJsonValue obj; + obj.InsertValue("zero", TJsonValue(0)); + obj.InsertValue("succ", TJsonValue(1)); + val.InsertValue("0", obj); + + NJsonOrderedWriter::TBuf w(NJsonOrderedWriter::HEM_DONT_ESCAPE_HTML); + w.WriteJsonValue(&val, true); + + const char exp[] = "{\"0\":{\"succ\":1,\"zero\":0},\"1\":1,\"2\":2}"; + UNIT_ASSERT_STRINGS_EQUAL(exp, w.Str()); + } + Y_UNIT_TEST(Unescaped) { + NJsonOrderedWriter::TBuf buf(NJsonOrderedWriter::HEM_UNSAFE); + buf.WriteString("</security>'"); + UNIT_ASSERT_STRINGS_EQUAL("\"</security>'\"", buf.Str()); + } + Y_UNIT_TEST(LittleBobbyJsonp) { + NJsonOrderedWriter::TBuf buf; + buf.WriteString("hello\xe2\x80\xa8\xe2\x80\xa9stranger"); + UNIT_ASSERT_STRINGS_EQUAL("\"hello\\u2028\\u2029stranger\"", buf.Str()); + } + Y_UNIT_TEST(LittleBobbyInvalid) { + NJsonOrderedWriter::TBuf buf; + TStringBuf incomplete("\xe2\x80\xa8", 2); + buf.WriteString(incomplete); + // garbage in - garbage out + UNIT_ASSERT_STRINGS_EQUAL("\"\xe2\x80\"", buf.Str()); + } + Y_UNIT_TEST(OverlyZealous) { + NJsonOrderedWriter::TBuf buf; + buf.WriteString("—"); + UNIT_ASSERT_STRINGS_EQUAL("\"—\"", buf.Str()); + } + Y_UNIT_TEST(RelaxedEscaping) { + NJsonOrderedWriter::TBuf buf(NJsonOrderedWriter::HEM_RELAXED); + buf.WriteString("</>"); + UNIT_ASSERT_STRINGS_EQUAL("\"\\u003C/\\u003E\"", buf.Str()); + } + + Y_UNIT_TEST(FloatFormatting) { + NJsonOrderedWriter::TBuf buf(NJsonOrderedWriter::HEM_DONT_ESCAPE_HTML); + buf.BeginList() + .WriteFloat(0.12345678987654321f) + .WriteDouble(0.12345678987654321) + .WriteFloat(0.315501, PREC_NDIGITS, 3) + .WriteFloat(244.13854, PREC_NDIGITS, 4) + .WriteFloat(10385.8324, PREC_POINT_DIGITS, 2) + .BeginObject() + .WriteKey("1") + .WriteDouble(1111.71, PREC_POINT_DIGITS, 0) + .WriteKey("2") + .WriteDouble(1111.71, PREC_NDIGITS, 1) + .EndObject() + .EndList(); + const char exp[] = "[0.123457,0.1234567899,0.316,244.1,10385.83,{\"1\":1112,\"2\":1e+03}]"; + UNIT_ASSERT_STRINGS_EQUAL(exp, buf.Str()); + } + + Y_UNIT_TEST(NanFormatting) { + { + NJsonOrderedWriter::TBuf buf; + buf.BeginObject(); + buf.WriteKey("nanvalue"); + UNIT_ASSERT_EXCEPTION(buf.WriteFloat(std::numeric_limits<double>::quiet_NaN()), yexception); + } + + { + NJsonOrderedWriter::TBuf buf; + buf.BeginObject(); + buf.WriteKey("infvalue"); + UNIT_ASSERT_EXCEPTION(buf.WriteFloat(std::numeric_limits<double>::infinity()), yexception); + } + + { + NJsonOrderedWriter::TBuf buf; + buf.BeginList(); + UNIT_ASSERT_EXCEPTION(buf.WriteFloat(std::numeric_limits<double>::quiet_NaN()), yexception); + } + + { + NJsonOrderedWriter::TBuf buf; + buf.BeginList(); + UNIT_ASSERT_EXCEPTION(buf.WriteFloat(std::numeric_limits<double>::infinity()), yexception); + } + + { + NJsonOrderedWriter::TBuf buf; + buf.SetWriteNanAsString(); + + buf.BeginObject() + .WriteKey("nanvalue") + .WriteFloat(std::numeric_limits<double>::quiet_NaN()) + .WriteKey("infvalue") + .WriteFloat(std::numeric_limits<double>::infinity()) + .WriteKey("minus_infvalue") + .WriteFloat(-std::numeric_limits<float>::infinity()) + .WriteKey("l") + .BeginList() + .WriteFloat(std::numeric_limits<float>::quiet_NaN()) + .EndList() + .EndObject(); + + UNIT_ASSERT_STRINGS_EQUAL(buf.Str(), R"raw_json({"nanvalue":"nan","infvalue":"inf","minus_infvalue":"-inf","l":["nan"]})raw_json"); + } + + { + NJsonOrderedWriter::TBuf buf; + buf.BeginObject() + .WriteKey("<>&") + .WriteString("Ololo") + .UnsafeWriteKey("<>&") + .WriteString("Ololo2") + .EndObject(); + + UNIT_ASSERT_STRINGS_EQUAL(buf.Str(), R"({"\u003C\u003E&":"Ololo","<>&":"Ololo2"})"); + } + } +} diff --git a/library/cpp/json/ordered_maps/ut/json_reader_fast_ordered_ut.cpp b/library/cpp/json/ordered_maps/ut/json_reader_fast_ordered_ut.cpp new file mode 100644 index 00000000000..de9702f8775 --- /dev/null +++ b/library/cpp/json/ordered_maps/ut/json_reader_fast_ordered_ut.cpp @@ -0,0 +1,304 @@ +#include <library/cpp/json/ordered_maps/json_reader_ordered.h> +#include <library/cpp/json/json_prettifier.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h> +#include <util/string/cast.h> +#include <util/string/printf.h> + +namespace NJson::NOrderedJson { + namespace NTest { + enum ETestEvent { + E_NO_EVENT = 0, + E_ERROR = 1, + E_DICT_OPEN, + E_DICT_CLOSE, + E_ARR_OPEN, + E_ARR_CLOSE, + E_NULL, + E_BOOL, + E_FLT, + E_INT, + E_LONG_LONG, + E_STR, + E_KEY + }; + + struct TEvent { + ETestEvent Type = E_NO_EVENT; + + i64 INum = 0; + double DNum = 0; + TString Str; + + TEvent(ETestEvent e = E_NO_EVENT) + : Type(e) + { + } + + TEvent(double v, ETestEvent e) + : Type(e) + , DNum(v) + { + } + + TEvent(i64 v, ETestEvent e) + : Type(e) + , INum(v) + { + } + + TEvent(TStringBuf t, ETestEvent e) + : Type(e) + , Str(NEscJ::EscapeJ<true, false>(t)) + { + } + + TString ToString() const { + switch (Type) { + default: + return "YOUFAILED"; + case E_ERROR: + return Sprintf("error: %s", Str.data()); + case E_DICT_OPEN: + return "{"; + case E_DICT_CLOSE: + return "}"; + case E_ARR_OPEN: + return "["; + case E_ARR_CLOSE: + return "]"; + case E_NULL: + return "null"; + case E_BOOL: + return INum ? "true" : "false"; + case E_INT: + return ::ToString(INum); + case E_FLT: + return ::ToString(DNum); + case E_STR: + return Sprintf("%s", Str.data()); + case E_KEY: + return Sprintf("key: %s", Str.data()); + } + } + }; + + using TEvents = TVector<TEvent>; + + struct TTestHandler : TJsonCallbacks { + TEvents Events; + + bool OnOpenMap() override { + Events.push_back(E_DICT_OPEN); + return true; + } + + bool OnCloseMap() override { + Events.push_back(E_DICT_CLOSE); + return true; + } + + bool OnOpenArray() override { + Events.push_back(E_ARR_OPEN); + return true; + } + + bool OnCloseArray() override { + Events.push_back(E_ARR_CLOSE); + return true; + } + + bool OnNull() override { + Events.push_back(E_NULL); + return true; + } + + bool OnBoolean(bool v) override { + Events.push_back(TEvent((i64)v, E_BOOL)); + return true; + } + + bool OnInteger(long long v) override { + Events.push_back(TEvent((i64)v, E_INT)); + return true; + } + + bool OnUInteger(unsigned long long v) override { + return OnInteger(v); + } + + bool OnDouble(double v) override { + Events.push_back(TEvent(v, E_FLT)); + return true; + } + + bool OnString(const TStringBuf& v) override { + Events.push_back(TEvent(v, E_STR)); + return true; + } + + bool OnMapKey(const TStringBuf& v) override { + Events.push_back(TEvent(v, E_KEY)); + return true; + } + + void OnError(size_t, TStringBuf token) override { + Events.push_back(TEvent(token, E_ERROR)); + } + + void Assert(const TEvents& e, TString str) { + try { + UNIT_ASSERT_VALUES_EQUAL_C(e.size(), Events.size(), str); + + for (ui32 i = 0, sz = e.size(); i < sz; ++i) { + UNIT_ASSERT_VALUES_EQUAL_C((int)e[i].Type, (int)Events[i].Type, Sprintf("'%s' %u", str.data(), i)); + UNIT_ASSERT_VALUES_EQUAL_C(e[i].INum, Events[i].INum, Sprintf("'%s' %u", str.data(), i)); + UNIT_ASSERT_VALUES_EQUAL_C(e[i].DNum, Events[i].DNum, Sprintf("'%s' %u", str.data(), i)); + UNIT_ASSERT_VALUES_EQUAL_C(e[i].Str, Events[i].Str, Sprintf("'%s' %u", str.data(), i)); + } + } catch (const yexception&) { + Clog << "Exception at '" << str << "'" << Endl; + for (const auto& event : Events) { + Clog << event.ToString() << Endl; + } + + throw; + } + } + }; + } +} + +class TFastJsonTest: public TTestBase { + UNIT_TEST_SUITE(TFastJsonTest) + UNIT_TEST(TestParse) + UNIT_TEST(TestReadJsonFastTree) + UNIT_TEST(TestNoInlineComment) + UNIT_TEST_SUITE_END(); + +public: + template <bool accept> + void DoTestParse(TStringBuf json, ui32 amount, ...) { + using namespace NJson::NOrderedJson::NTest; + TEvents evs; + va_list vl; + va_start(vl, amount); + for (ui32 i = 0; i < amount; i++) { + ETestEvent e = (ETestEvent)va_arg(vl, int); + + switch ((int)e) { + case E_NO_EVENT: + case E_DICT_OPEN: + case E_DICT_CLOSE: + case E_ARR_OPEN: + case E_ARR_CLOSE: + case E_NULL: + evs.push_back(e); + break; + case E_BOOL: { + bool v = va_arg(vl, int); + evs.push_back(TEvent((i64)v, E_BOOL)); + break; + } + case E_INT: { + i64 i = va_arg(vl, int); + evs.push_back(TEvent(i, E_INT)); + break; + } + case E_LONG_LONG: { + i64 i = va_arg(vl, long long); + evs.push_back(TEvent(i, E_INT)); + break; + } + case E_FLT: { + double f = va_arg(vl, double); + evs.push_back(TEvent(f, E_FLT)); + break; + } + case E_STR: { + const char* s = va_arg(vl, const char*); + evs.push_back(TEvent(TStringBuf(s), E_STR)); + break; + } + case E_KEY: + case E_ERROR: { + const char* s = va_arg(vl, const char*); + evs.push_back(TEvent(TStringBuf(s), e)); + break; + } + } + } + va_end(vl); + + TTestHandler h; + const bool res = ReadJsonFast(json, &h); + UNIT_ASSERT_VALUES_EQUAL_C(res, accept, Sprintf("%s (%s)", ToString(json).data(), h.Events.back().Str.data())); + h.Assert(evs, ToString(json)); + } + + void TestParse() { + using namespace NJson::NOrderedJson::NTest; + + DoTestParse<true>("", 0); + DoTestParse<true>(" \t \t ", 0); + DoTestParse<true>("a-b-c@аб_вгд909AБ", 1, E_STR, "a-b-c@аб_вгд909AБ"); + DoTestParse<true>("'я тестовая строка'", 1, E_STR, "я тестовая строка"); + DoTestParse<true>("\"я тестовая строка\"", 1, E_STR, "я тестовая строка"); + DoTestParse<true>("'\\xA\\xA\\xA'", 1, E_STR, "\n\n\n"); + DoTestParse<true>("12.15", 1, E_FLT, 12.15); + DoTestParse<true>("null", 1, E_NULL); + DoTestParse<true>("true", 1, E_BOOL, true); + DoTestParse<true>("false", 1, E_BOOL, false); + DoTestParse<true>("[]", 2, E_ARR_OPEN, E_ARR_CLOSE); + DoTestParse<true>("[ a ]", 3, E_ARR_OPEN, E_STR, "a", E_ARR_CLOSE); + DoTestParse<true>("[ a, b ]", 4, E_ARR_OPEN, E_STR, "a", E_STR, "b", E_ARR_CLOSE); + DoTestParse<true>("[a,b]", 4, E_ARR_OPEN, E_STR, "a", E_STR, "b", E_ARR_CLOSE); + DoTestParse<false>("[a,b][a,b]", 5, E_ARR_OPEN, E_STR, "a", E_STR, "b", E_ARR_CLOSE, E_ERROR, "invalid syntax at token: '['"); + DoTestParse<false>("[a,,b]", 3, E_ARR_OPEN, E_STR, "a", E_ERROR, "invalid syntax at token: ','"); + DoTestParse<true>("{ k : v }", 4, E_DICT_OPEN, E_KEY, "k", E_STR, "v", E_DICT_CLOSE); + DoTestParse<true>("{a:'\\b'/*comment*/, k /*comment*/\n : v }", 6, E_DICT_OPEN, E_KEY, "a", E_STR, "\b", E_KEY, "k", E_STR, "v", E_DICT_CLOSE); + DoTestParse<true>("{a:.15, k : v }", 6, E_DICT_OPEN, E_KEY, "a", E_FLT, .15, E_KEY, "k", E_STR, "v", E_DICT_CLOSE); + DoTestParse<true>("[ a, -.1e+5, 1E-7]", 5, E_ARR_OPEN, E_STR, "a", E_FLT, -.1e+5, E_FLT, 1e-7, E_ARR_CLOSE); + DoTestParse<true>("{}", 2, E_DICT_OPEN, E_DICT_CLOSE); + DoTestParse<true>("{ a : x, b : [ c, d, ] }", 9, E_DICT_OPEN, E_KEY, "a", E_STR, "x", E_KEY, "b", E_ARR_OPEN, E_STR, "c", E_STR, "d", E_ARR_CLOSE, E_DICT_CLOSE); + DoTestParse<false>("{ a : x, b : [ c, d,, ] }", 8, E_DICT_OPEN, E_KEY, "a", E_STR, "x", E_KEY, "b", E_ARR_OPEN, E_STR, "c", E_STR, "d", E_ERROR, "invalid syntax at token: ','"); + // DoTestParse<false>("{ a : x : y }", 4, E_DICT_OPEN + // , E_KEY, "a", E_STR, "x" + // , E_ERROR + // , ":"); + // DoTestParse<false>("{queries:{ref:[]},{nonref:[]}}", 8, E_DICT_OPEN + // , E_KEY, "queries", E_DICT_OPEN + // , E_KEY, "ref", E_ARR_OPEN, E_ARR_CLOSE + // , E_DICT_CLOSE, E_ERROR, ""); + DoTestParse<true>("'100x00'", 1, E_STR, "100x00"); + DoTestParse<true>("-1", 1, E_INT, -1); + DoTestParse<true>("-9223372036854775808", 1, E_LONG_LONG, (long long)Min<i64>()); + DoTestParse<false>("100x00", 1, E_ERROR, "invalid syntax at token: '100x'"); + DoTestParse<false>("100 200", 2, E_INT, 100, E_ERROR, "invalid syntax at token: '200'"); + DoTestParse<true>("{g:{x:{a:{b:c,e:f},q:{x:y}},y:fff}}", 22, E_DICT_OPEN, E_KEY, "g", E_DICT_OPEN, E_KEY, "x", E_DICT_OPEN, E_KEY, "a", E_DICT_OPEN, E_KEY, "b", E_STR, "c", E_KEY, "e", E_STR, "f", E_DICT_CLOSE, E_KEY, "q", E_DICT_OPEN, E_KEY, "x", E_STR, "y", E_DICT_CLOSE, E_DICT_CLOSE, E_KEY, "y", E_STR, "fff", E_DICT_CLOSE, E_DICT_CLOSE); + } + + void TestReadJsonFastTree() { + const TString json = R"( + { + "a": { + "b": {} + } + }} + )"; + NJson::TJsonValue value; + UNIT_ASSERT(!ReadJsonFastTree(json, &value)); + } + + void TestNoInlineComment() { + using namespace NJson::NOrderedJson::NTest; + DoTestParse<false>("{\"a\":1}//d{\"b\":2}", 5, E_DICT_OPEN, E_KEY, "a", E_INT, 1, E_DICT_CLOSE, E_ERROR, "invalid syntax at token: '/'"); + DoTestParse<false>("{\"a\":1}//d{\"b\":2}\n", 5, E_DICT_OPEN, E_KEY, "a", E_INT, 1, E_DICT_CLOSE, E_ERROR, "invalid syntax at token: '/'"); + DoTestParse<false>("{\"a\":{//d{\"b\":2}\n}}", 4, E_DICT_OPEN, E_KEY, "a", E_DICT_OPEN, E_ERROR, "invalid syntax at token: '/'"); + DoTestParse<false>("{\"a\":{//d{\"b\":2}}}\n", 4, E_DICT_OPEN, E_KEY, "a", E_DICT_OPEN, E_ERROR, "invalid syntax at token: '/'"); + DoTestParse<false>("{\"a\":{//d{\"b\":2}}}", 4, E_DICT_OPEN, E_KEY, "a", E_DICT_OPEN, E_ERROR, "invalid syntax at token: '/'"); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TFastJsonTest) diff --git a/library/cpp/json/ordered_maps/ut/json_reader_nan_ordered_ut.cpp b/library/cpp/json/ordered_maps/ut/json_reader_nan_ordered_ut.cpp new file mode 100644 index 00000000000..71c3cc2e7a6 --- /dev/null +++ b/library/cpp/json/ordered_maps/ut/json_reader_nan_ordered_ut.cpp @@ -0,0 +1,29 @@ +#include <library/cpp/json/ordered_maps/json_reader_ordered.h> +#include <library/cpp/testing/unittest/registar.h> + +using namespace NJson::NOrderedJson; + +namespace { + +constexpr TStringBuf JSON_NAN_TEST = "{ \"Value1\": 0.0, \"Value2\": 1, \"Value3\": NaN }"; + +} + +Y_UNIT_TEST_SUITE(TJsonReaderNanTest) { + Y_UNIT_TEST(WithoutNanTest) { + TJsonReaderConfig cfg; + TJsonValue out; + // This read will fail + UNIT_ASSERT(!ReadJsonTree(JSON_NAN_TEST, &cfg, &out, /* throwOnError */ false)); + + } + Y_UNIT_TEST(WithNanTest) { + TJsonReaderConfig cfg; + cfg.AllowReadNanInf = true; + + TJsonValue out; + // This read will ok + UNIT_ASSERT(ReadJsonTree(JSON_NAN_TEST, &cfg, &out, /* throwOnError */ false)); + } +} + diff --git a/library/cpp/json/ordered_maps/ut/json_reader_ordered_ut.cpp b/library/cpp/json/ordered_maps/ut/json_reader_ordered_ut.cpp new file mode 100644 index 00000000000..bd317cf7a6f --- /dev/null +++ b/library/cpp/json/ordered_maps/ut/json_reader_ordered_ut.cpp @@ -0,0 +1,507 @@ +#include <library/cpp/json/ordered_maps/json_reader_ordered.h> +#include <library/cpp/json/ordered_maps/json_writer_ordered.h> + +#include <library/cpp/testing/unittest/registar.h> +#include <util/stream/str.h> + +using namespace NJson::NOrderedJson; + +class TReformatCallbacks: public NJson::TJsonCallbacks { + TJsonWriter& Writer; + +public: + TReformatCallbacks(TJsonWriter& writer) + : Writer(writer) + { + } + + bool OnBoolean(bool val) override { + Writer.Write(val); + return true; + } + + bool OnInteger(long long val) override { + Writer.Write(val); + return true; + } + + bool OnUInteger(unsigned long long val) override { + Writer.Write(val); + return true; + } + + bool OnString(const TStringBuf& val) override { + Writer.Write(val); + return true; + } + + bool OnDouble(double val) override { + Writer.Write(val); + return true; + } + + bool OnOpenArray() override { + Writer.OpenArray(); + return true; + } + + bool OnCloseArray() override { + Writer.CloseArray(); + return true; + } + + bool OnOpenMap() override { + Writer.OpenArray(); + return true; + } + + bool OnCloseMap() override { + Writer.CloseArray(); + return true; + } + + bool OnMapKey(const TStringBuf& val) override { + Writer.Write(val); + return true; + } +}; + +void GenerateDeepJsonArray(TStringStream& stream, ui64 depth) { + stream << "{\"key\":"; + for (ui32 i = 0; i < depth - 1; ++i) { + stream << "["; + } + for (ui32 i = 0; i < depth - 1; ++i) { + stream << "]"; + } + stream << "}"; +} + +void GenerateDeepJsonDict(TStringStream& stream, ui64 depth) { + for (ui64 i = 0; i < depth - 1; ++i) { + stream << "{\"key\":"; + } + stream << "{}"; + for (ui64 i = 0; i < depth - 1; ++i) { + stream << "}"; + } +} + +Y_UNIT_TEST_SUITE(TJsonReaderTest) { + Y_UNIT_TEST(JsonReformatTest) { + TString data = "{\"null value\": null, \"intkey\": 10, \"double key\": 11.11, \"string key\": \"string\", \"array\": [1,2,3,\"TString\"], \"bool key\": true}"; + + TString result1, result2; + { + TStringStream in; + in << data; + TStringStream out; + TJsonWriter writer(&out, false); + TReformatCallbacks cb(writer); + ReadJson(&in, &cb); + writer.Flush(); + result1 = out.Str(); + } + + { + TStringStream in; + in << result1; + TStringStream out; + TJsonWriter writer(&out, false); + TReformatCallbacks cb(writer); + ReadJson(&in, &cb); + writer.Flush(); + result2 = out.Str(); + } + + UNIT_ASSERT_VALUES_EQUAL(result1, result2); + } + + Y_UNIT_TEST(TJsonEscapedApostrophe) { + TString jsonString = "{ \"foo\" : \"bar\\'buzz\" }"; + { + TStringStream in; + in << jsonString; + TStringStream out; + TJsonWriter writer(&out, false); + TReformatCallbacks cb(writer); + UNIT_ASSERT(!ReadJson(&in, &cb)); + } + + { + TStringStream in; + in << jsonString; + TStringStream out; + TJsonWriter writer(&out, false); + TReformatCallbacks cb(writer); + UNIT_ASSERT(ReadJson(&in, false, true, &cb)); + writer.Flush(); + UNIT_ASSERT_EQUAL(out.Str(), "[\"foo\",\"bar'buzz\"]"); + } + } + + Y_UNIT_TEST(TJsonTreeTest) { + TString data = "{\"intkey\": 10, \"double key\": 11.11, \"null value\":null, \"string key\": \"string\", \"array\": [1,2,3,\"TString\"], \"bool key\": true}"; + TStringStream in; + in << data; + TJsonValue value; + ReadJsonTree(&in, &value); + + UNIT_ASSERT_VALUES_EQUAL(value["intkey"].GetInteger(), 10); + UNIT_ASSERT_DOUBLES_EQUAL(value["double key"].GetDouble(), 11.11, 0.001); + UNIT_ASSERT_VALUES_EQUAL(value["bool key"].GetBoolean(), true); + UNIT_ASSERT_VALUES_EQUAL(value["absent string key"].GetString(), TString("")); + UNIT_ASSERT_VALUES_EQUAL(value["string key"].GetString(), TString("string")); + UNIT_ASSERT_VALUES_EQUAL(value["array"][0].GetInteger(), 1); + UNIT_ASSERT_VALUES_EQUAL(value["array"][1].GetInteger(), 2); + UNIT_ASSERT_VALUES_EQUAL(value["array"][2].GetInteger(), 3); + UNIT_ASSERT_VALUES_EQUAL(value["array"][3].GetString(), TString("TString")); + UNIT_ASSERT(value["null value"].IsNull()); + + // AsString + UNIT_ASSERT_VALUES_EQUAL(value["intkey"].GetStringRobust(), "10"); + UNIT_ASSERT_VALUES_EQUAL(value["double key"].GetStringRobust(), "11.11"); + UNIT_ASSERT_VALUES_EQUAL(value["bool key"].GetStringRobust(), "true"); + UNIT_ASSERT_VALUES_EQUAL(value["string key"].GetStringRobust(), "string"); + UNIT_ASSERT_VALUES_EQUAL(value["array"].GetStringRobust(), "[1,2,3,\"TString\"]"); + UNIT_ASSERT_VALUES_EQUAL(value["null value"].GetStringRobust(), "null"); + + const TJsonValue::TArray* array; + UNIT_ASSERT(GetArrayPointer(value, "array", &array)); + UNIT_ASSERT_VALUES_EQUAL(value["array"].GetArray().size(), array->size()); + UNIT_ASSERT_VALUES_EQUAL(value["array"][0].GetInteger(), (*array)[0].GetInteger()); + UNIT_ASSERT_VALUES_EQUAL(value["array"][1].GetInteger(), (*array)[1].GetInteger()); + UNIT_ASSERT_VALUES_EQUAL(value["array"][2].GetInteger(), (*array)[2].GetInteger()); + UNIT_ASSERT_VALUES_EQUAL(value["array"][3].GetString(), (*array)[3].GetString()); + } + + Y_UNIT_TEST(TJsonRomaTest) { + TString data = "{\"test\": [ {\"name\": \"A\"} ]}"; + + TStringStream in; + in << data; + TJsonValue value; + ReadJsonTree(&in, &value); + + UNIT_ASSERT_VALUES_EQUAL(value["test"][0]["name"].GetString(), TString("A")); + } + + Y_UNIT_TEST(TJsonReadTreeWithComments) { + { + TString leadingCommentData = "{ // \"test\" : 1 \n}"; + { + // No comments allowed + TStringStream in; + in << leadingCommentData; + TJsonValue value; + UNIT_ASSERT(!ReadJsonTree(&in, false, &value)); + } + + { + // Comments allowed + TStringStream in; + in << leadingCommentData; + TJsonValue value; + UNIT_ASSERT(ReadJsonTree(&in, true, &value)); + UNIT_ASSERT(!value.Has("test")); + } + } + + { + TString trailingCommentData = "{ \"test1\" : 1 // \"test2\" : 2 \n }"; + { + // No comments allowed + TStringStream in; + in << trailingCommentData; + TJsonValue value; + UNIT_ASSERT(!ReadJsonTree(&in, false, &value)); + } + + { + // Comments allowed + TStringStream in; + in << trailingCommentData; + TJsonValue value; + UNIT_ASSERT(ReadJsonTree(&in, true, &value)); + UNIT_ASSERT(value.Has("test1")); + UNIT_ASSERT_EQUAL(value["test1"].GetInteger(), 1); + UNIT_ASSERT(!value.Has("test2")); + } + } + } + + Y_UNIT_TEST(TJsonSignedIntegerTest) { + { + TStringStream in; + in << "{ \"test\" : " << Min<i64>() << " }"; + TJsonValue value; + UNIT_ASSERT(ReadJsonTree(&in, &value)); + UNIT_ASSERT(value.Has("test")); + UNIT_ASSERT(value["test"].IsInteger()); + UNIT_ASSERT(!value["test"].IsUInteger()); + UNIT_ASSERT_EQUAL(value["test"].GetInteger(), Min<i64>()); + UNIT_ASSERT_EQUAL(value["test"].GetIntegerRobust(), Min<i64>()); + } // Min<i64>() + + { + TStringStream in; + in << "{ \"test\" : " << Max<i64>() + 1ull << " }"; + TJsonValue value; + UNIT_ASSERT(ReadJsonTree(&in, &value)); + UNIT_ASSERT(value.Has("test")); + UNIT_ASSERT(!value["test"].IsInteger()); + UNIT_ASSERT(value["test"].IsUInteger()); + UNIT_ASSERT_EQUAL(value["test"].GetIntegerRobust(), (i64)(Max<i64>() + 1ull)); + } // Max<i64>() + 1 + } + + Y_UNIT_TEST(TJsonUnsignedIntegerTest) { + { + TStringStream in; + in << "{ \"test\" : 1 }"; + TJsonValue value; + UNIT_ASSERT(ReadJsonTree(&in, &value)); + UNIT_ASSERT(value.Has("test")); + UNIT_ASSERT(value["test"].IsInteger()); + UNIT_ASSERT(value["test"].IsUInteger()); + UNIT_ASSERT_EQUAL(value["test"].GetInteger(), 1); + UNIT_ASSERT_EQUAL(value["test"].GetIntegerRobust(), 1); + UNIT_ASSERT_EQUAL(value["test"].GetUInteger(), 1); + UNIT_ASSERT_EQUAL(value["test"].GetUIntegerRobust(), 1); + } // 1 + + { + TStringStream in; + in << "{ \"test\" : -1 }"; + TJsonValue value; + UNIT_ASSERT(ReadJsonTree(&in, &value)); + UNIT_ASSERT(value.Has("test")); + UNIT_ASSERT(value["test"].IsInteger()); + UNIT_ASSERT(!value["test"].IsUInteger()); + UNIT_ASSERT_EQUAL(value["test"].GetInteger(), -1); + UNIT_ASSERT_EQUAL(value["test"].GetIntegerRobust(), -1); + UNIT_ASSERT_EQUAL(value["test"].GetUInteger(), 0); + UNIT_ASSERT_EQUAL(value["test"].GetUIntegerRobust(), static_cast<unsigned long long>(-1)); + } // -1 + + { + TStringStream in; + in << "{ \"test\" : 18446744073709551615 }"; + TJsonValue value; + UNIT_ASSERT(ReadJsonTree(&in, &value)); + UNIT_ASSERT(value.Has("test")); + UNIT_ASSERT(!value["test"].IsInteger()); + UNIT_ASSERT(value["test"].IsUInteger()); + UNIT_ASSERT_EQUAL(value["test"].GetInteger(), 0); + UNIT_ASSERT_EQUAL(value["test"].GetIntegerRobust(), static_cast<long long>(18446744073709551615ull)); + UNIT_ASSERT_EQUAL(value["test"].GetUInteger(), 18446744073709551615ull); + UNIT_ASSERT_EQUAL(value["test"].GetUIntegerRobust(), 18446744073709551615ull); + } // 18446744073709551615 + + { + TStringStream in; + in << "{ \"test\" : 1.1 }"; + TJsonValue value; + UNIT_ASSERT(ReadJsonTree(&in, &value)); + UNIT_ASSERT(value.Has("test")); + UNIT_ASSERT(!value["test"].IsInteger()); + UNIT_ASSERT(!value["test"].IsUInteger()); + UNIT_ASSERT_EQUAL(value["test"].GetInteger(), 0); + UNIT_ASSERT_EQUAL(value["test"].GetIntegerRobust(), static_cast<long long>(1.1)); + UNIT_ASSERT_EQUAL(value["test"].GetUInteger(), 0); + UNIT_ASSERT_EQUAL(value["test"].GetUIntegerRobust(), static_cast<unsigned long long>(1.1)); + } // 1.1 + + { + TStringStream in; + in << "{ \"test\" : [1, 18446744073709551615] }"; + TJsonValue value; + UNIT_ASSERT(ReadJsonTree(&in, &value)); + UNIT_ASSERT(value.Has("test")); + UNIT_ASSERT(value["test"].IsArray()); + UNIT_ASSERT_EQUAL(value["test"].GetArray().size(), 2); + UNIT_ASSERT(value["test"][0].IsInteger()); + UNIT_ASSERT(value["test"][0].IsUInteger()); + UNIT_ASSERT_EQUAL(value["test"][0].GetInteger(), 1); + UNIT_ASSERT_EQUAL(value["test"][0].GetUInteger(), 1); + UNIT_ASSERT(!value["test"][1].IsInteger()); + UNIT_ASSERT(value["test"][1].IsUInteger()); + UNIT_ASSERT_EQUAL(value["test"][1].GetUInteger(), 18446744073709551615ull); + } + } // TJsonUnsignedIntegerTest + + Y_UNIT_TEST(TJsonDoubleTest) { + { + TStringStream in; + in << "{ \"test\" : 1.0 }"; + TJsonValue value; + UNIT_ASSERT(ReadJsonTree(&in, &value)); + UNIT_ASSERT(value.Has("test")); + UNIT_ASSERT(value["test"].IsDouble()); + UNIT_ASSERT_EQUAL(value["test"].GetDouble(), 1.0); + UNIT_ASSERT_EQUAL(value["test"].GetDoubleRobust(), 1.0); + } // 1.0 + + { + TStringStream in; + in << "{ \"test\" : 1 }"; + TJsonValue value; + UNIT_ASSERT(ReadJsonTree(&in, &value)); + UNIT_ASSERT(value.Has("test")); + UNIT_ASSERT(value["test"].IsDouble()); + UNIT_ASSERT_EQUAL(value["test"].GetDouble(), 1.0); + UNIT_ASSERT_EQUAL(value["test"].GetDoubleRobust(), 1.0); + } // 1 + + { + TStringStream in; + in << "{ \"test\" : -1 }"; + TJsonValue value; + UNIT_ASSERT(ReadJsonTree(&in, &value)); + UNIT_ASSERT(value.Has("test")); + UNIT_ASSERT(value["test"].IsDouble()); + UNIT_ASSERT_EQUAL(value["test"].GetDouble(), -1.0); + UNIT_ASSERT_EQUAL(value["test"].GetDoubleRobust(), -1.0); + } // -1 + + { + TStringStream in; + in << "{ \"test\" : " << Max<ui64>() << " }"; + TJsonValue value; + UNIT_ASSERT(ReadJsonTree(&in, &value)); + UNIT_ASSERT(value.Has("test")); + UNIT_ASSERT(!value["test"].IsDouble()); + UNIT_ASSERT_EQUAL(value["test"].GetDouble(), 0.0); + UNIT_ASSERT_EQUAL(value["test"].GetDoubleRobust(), static_cast<double>(Max<ui64>())); + } // Max<ui64>() + } // TJsonDoubleTest + + Y_UNIT_TEST(TJsonInvalidTest) { + { + // No exceptions mode. + TStringStream in; + in << "{ \"test\" : }"; + TJsonValue value; + UNIT_ASSERT(!ReadJsonTree(&in, &value)); + } + + { + // Exception throwing mode. + TStringStream in; + in << "{ \"test\" : }"; + TJsonValue value; + UNIT_ASSERT_EXCEPTION(ReadJsonTree(&in, &value, true), NJson::TJsonException); + } + } + + Y_UNIT_TEST(TJsonMemoryLeakTest) { + // after https://clubs.at.yandex-team.ru/stackoverflow/3691 + TString s = "."; + NJson::NOrderedJson::TJsonValue json; + try { + TStringInput in(s); + NJson::NOrderedJson::ReadJsonTree(&in, &json, true); + } catch (...) { + } + } // TJsonMemoryLeakTest + + Y_UNIT_TEST(TJsonDuplicateKeysWithNullValuesTest) { + const TString json = "{\"\":null,\"\":\"\"}"; + + TStringInput in(json); + NJson::NOrderedJson::TJsonValue v; + UNIT_ASSERT(ReadJsonTree(&in, &v)); + UNIT_ASSERT(v.IsMap()); + UNIT_ASSERT_VALUES_EQUAL(1, v.GetMap().size()); + UNIT_ASSERT_VALUES_EQUAL("", v.GetMap().begin()->first); + UNIT_ASSERT(v.GetMap().begin()->second.IsString()); + UNIT_ASSERT_VALUES_EQUAL("", v.GetMap().begin()->second.GetString()); + } + + // Parsing an extremely deep json tree would result in stack overflow. + // Not crashing on one is a good indicator of iterative mode. + Y_UNIT_TEST(TJsonIterativeTest) { + constexpr ui32 brackets = static_cast<ui32>(1e5); + + TStringStream jsonStream; + GenerateDeepJsonArray(jsonStream, brackets); + + TJsonReaderConfig config; + config.UseIterativeParser = true; + config.MaxDepth = static_cast<ui32>(1e3); + + TJsonValue v; + UNIT_ASSERT(!ReadJsonTree(&jsonStream, &config, &v)); + } + + Y_UNIT_TEST(TJsonMaxDepthTest) { + constexpr ui32 depth = static_cast<ui32>(1e3); + + { + TStringStream jsonStream; + GenerateDeepJsonArray(jsonStream, depth); + TJsonReaderConfig config; + config.MaxDepth = depth; + TJsonValue v; + UNIT_ASSERT(ReadJsonTree(&jsonStream, &config, &v)); + } + + { + TStringStream jsonStream; + GenerateDeepJsonArray(jsonStream, depth); + TJsonReaderConfig config; + config.MaxDepth = depth - 1; + TJsonValue v; + UNIT_ASSERT(!ReadJsonTree(&jsonStream, &config, &v)); + } + + { + TStringStream jsonStream; + GenerateDeepJsonDict(jsonStream, depth); + TJsonReaderConfig config; + config.MaxDepth = depth; + TJsonValue v; + UNIT_ASSERT(ReadJsonTree(&jsonStream, &config, &v)); + } + + { + TStringStream jsonStream; + GenerateDeepJsonDict(jsonStream, depth); + TJsonReaderConfig config; + config.MaxDepth = depth - 1; + TJsonValue v; + UNIT_ASSERT(!ReadJsonTree(&jsonStream, &config, &v)); + } + } +} + + +static const TString YANDEX_STREAMING_JSON("{\"a\":1}//d{\"b\":2}"); + + +Y_UNIT_TEST_SUITE(TCompareReadJsonFast) { + Y_UNIT_TEST(NoEndl) { + NJson::NOrderedJson::TJsonValue parsed; + + bool success = NJson::NOrderedJson::ReadJsonTree(YANDEX_STREAMING_JSON, &parsed, false); + bool fast_success = NJson::NOrderedJson::ReadJsonFastTree(YANDEX_STREAMING_JSON, &parsed, false); + UNIT_ASSERT(success == fast_success); + } + Y_UNIT_TEST(WithEndl) { + NJson::NOrderedJson::TJsonValue parsed1; + NJson::NOrderedJson::TJsonValue parsed2; + + bool success = NJson::NOrderedJson::ReadJsonTree(YANDEX_STREAMING_JSON + "\n", &parsed1, false); + bool fast_success = NJson::NOrderedJson::ReadJsonFastTree(YANDEX_STREAMING_JSON + "\n", &parsed2, false); + + UNIT_ASSERT_VALUES_EQUAL(success, fast_success); + } + Y_UNIT_TEST(NoQuotes) { + TString streamingJson = "{a:1}"; + NJson::NOrderedJson::TJsonValue parsed; + + bool success = NJson::NOrderedJson::ReadJsonTree(streamingJson, &parsed, false); + bool fast_success = NJson::NOrderedJson::ReadJsonFastTree(streamingJson, &parsed, false); + UNIT_ASSERT(success != fast_success); + } +} diff --git a/library/cpp/json/ordered_maps/ut/json_value_ordered_ut.cpp b/library/cpp/json/ordered_maps/ut/json_value_ordered_ut.cpp new file mode 100644 index 00000000000..5fad165cf37 --- /dev/null +++ b/library/cpp/json/ordered_maps/ut/json_value_ordered_ut.cpp @@ -0,0 +1,920 @@ +#include <library/cpp/json/ordered_maps/json_value_ordered.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/input.h> + +using namespace NJson::NOrderedJson; + +Y_UNIT_TEST_SUITE(TJsonValueTest) { + Y_UNIT_TEST(Equal) { + UNIT_ASSERT(1 == TJsonValue(1)); + UNIT_ASSERT(TJsonValue(1) == 1); + UNIT_ASSERT(2 != TJsonValue(1)); + UNIT_ASSERT(TJsonValue(1) != 2); + } + + Y_UNIT_TEST(UndefTest) { + TJsonValue undef; + TJsonValue null(JSON_NULL); + TJsonValue _false(false); + TJsonValue zeroInt(0); + TJsonValue zeroDouble(0.0); + TJsonValue emptyStr(""); + TJsonValue emptyArray(JSON_ARRAY); + TJsonValue emptyMap(JSON_MAP); + + UNIT_ASSERT(!undef.IsDefined()); + UNIT_ASSERT(!null.IsDefined()); // json NULL is undefined too! + UNIT_ASSERT(_false.IsDefined()); + UNIT_ASSERT(zeroInt.IsDefined()); + UNIT_ASSERT(zeroDouble.IsDefined()); + UNIT_ASSERT(emptyStr.IsDefined()); + UNIT_ASSERT(emptyArray.IsDefined()); + UNIT_ASSERT(emptyMap.IsDefined()); + + UNIT_ASSERT(undef == TJsonValue()); + UNIT_ASSERT(undef != null); + UNIT_ASSERT(undef != _false); + UNIT_ASSERT(undef != zeroInt); + UNIT_ASSERT(undef != zeroDouble); + UNIT_ASSERT(undef != emptyStr); + UNIT_ASSERT(undef != emptyArray); + UNIT_ASSERT(undef != emptyMap); + } + + Y_UNIT_TEST(DefaultCompareTest) { + { + TJsonValue lhs; + TJsonValue rhs; + UNIT_ASSERT(lhs == rhs); + UNIT_ASSERT(rhs == lhs); + } + + { + TJsonValue lhs; + TJsonValue rhs(JSON_NULL); + UNIT_ASSERT(lhs != rhs); + UNIT_ASSERT(rhs != lhs); + } + } + + Y_UNIT_TEST(NullCompareTest) { + TJsonValue lhs(JSON_NULL); + TJsonValue rhs(JSON_NULL); + UNIT_ASSERT(lhs == rhs); + UNIT_ASSERT(rhs == lhs); + } + + Y_UNIT_TEST(StringCompareTest) { + { + TJsonValue lhs(JSON_STRING); + TJsonValue rhs(JSON_STRING); + UNIT_ASSERT(lhs == rhs); + UNIT_ASSERT(rhs == lhs); + } + + { + TJsonValue lhs(""); + TJsonValue rhs(""); + UNIT_ASSERT(lhs == rhs); + UNIT_ASSERT(rhs == lhs); + } + + { + TJsonValue lhs("abc"); + TJsonValue rhs("abc"); + UNIT_ASSERT(lhs == rhs); + UNIT_ASSERT(rhs == lhs); + } + + { + TJsonValue lhs("1"); + TJsonValue rhs(1); + UNIT_ASSERT(lhs != rhs); + UNIT_ASSERT(rhs != lhs); + } + } + + Y_UNIT_TEST(ArrayCompareTest) { + { + TJsonValue lhs(JSON_ARRAY); + TJsonValue rhs(JSON_ARRAY); + UNIT_ASSERT(lhs == rhs); + UNIT_ASSERT(rhs == lhs); + } + + { + TJsonValue lhs; + TJsonValue rhs; + + lhs.AppendValue(TJsonValue()); + + UNIT_ASSERT(lhs != rhs); + UNIT_ASSERT(rhs != lhs); + } + + { + TJsonValue lhs; + TJsonValue rhs; + + lhs.AppendValue(1); + lhs.AppendValue("2"); + lhs.AppendValue(3.0); + lhs.AppendValue(TJsonValue()); + lhs.AppendValue(TJsonValue(JSON_NULL)); + + rhs.AppendValue(1); + rhs.AppendValue("2"); + rhs.AppendValue(3.0); + rhs.AppendValue(TJsonValue()); + rhs.AppendValue(TJsonValue(JSON_NULL)); + + UNIT_ASSERT(lhs == rhs); + UNIT_ASSERT(rhs == lhs); + } + + { + TJsonValue lhs; + TJsonValue rhs; + + lhs.AppendValue(1); + rhs.AppendValue("1"); + + UNIT_ASSERT(lhs != rhs); + UNIT_ASSERT(rhs != lhs); + } + } + + Y_UNIT_TEST(CompareTest) { + { + TJsonValue lhs; + lhs.InsertValue("null value", TJsonValue(JSON_NULL)); + lhs.InsertValue("int key", TJsonValue(10)); + lhs.InsertValue("double key", TJsonValue(11.11)); + lhs.InsertValue("string key", TJsonValue("string")); + + TJsonValue array; + array.AppendValue(1); + array.AppendValue(2); + array.AppendValue(3); + array.AppendValue("string"); + lhs.InsertValue("array", array); + + lhs.InsertValue("bool key", TJsonValue(true)); + + TJsonValue rhs; + rhs = lhs; + + UNIT_ASSERT(lhs == rhs); + UNIT_ASSERT(rhs == lhs); + } + + { + // Insert keys in different orders + const int NUM_KEYS = 1000; + + TJsonValue lhs; + for (int i = 0; i < NUM_KEYS; ++i) + lhs.InsertValue(ToString(i), i); + + TJsonValue rhs; + for (int i = 0; i < NUM_KEYS; i += 2) + rhs.InsertValue(ToString(i), i); + for (int i = 1; i < NUM_KEYS; i += 2) + rhs.InsertValue(ToString(i), i); + + UNIT_ASSERT(lhs == rhs); + UNIT_ASSERT(rhs == lhs); + } + + { + TJsonValue lhs; + lhs.InsertValue("null value", TJsonValue(JSON_NULL)); + lhs.InsertValue("int key", TJsonValue(10)); + lhs.InsertValue("double key", TJsonValue(11.11)); + lhs.InsertValue("string key", TJsonValue("string")); + + TJsonValue array; + array.AppendValue(1); + array.AppendValue(2); + array.AppendValue(3); + array.AppendValue("string"); + lhs.InsertValue("array", array); + + lhs.InsertValue("bool key", TJsonValue(true)); + + TJsonValue rhs; + rhs.InsertValue("null value", TJsonValue(JSON_NULL)); + rhs.InsertValue("int key", TJsonValue(10)); + rhs.InsertValue("double key", TJsonValue(11.11)); + rhs.InsertValue("string key", TJsonValue("string")); + rhs.InsertValue("bool key", TJsonValue(true)); + + UNIT_ASSERT(lhs != rhs); + UNIT_ASSERT(rhs != lhs); + } + } + + Y_UNIT_TEST(SwapTest) { + { + TJsonValue lhs; + lhs.InsertValue("a", "b"); + TJsonValue lhsCopy = lhs; + + TJsonValue rhs(JSON_NULL); + TJsonValue rhsCopy = rhs; + + UNIT_ASSERT(lhs == lhsCopy); + UNIT_ASSERT(rhs == rhsCopy); + + lhs.Swap(rhs); + + UNIT_ASSERT(rhs == lhsCopy); + UNIT_ASSERT(lhs == rhsCopy); + + lhs.Swap(rhs); + + UNIT_ASSERT(lhs == lhsCopy); + UNIT_ASSERT(rhs == rhsCopy); + } + } + + Y_UNIT_TEST(GetValueByPathTest) { + { + TJsonValue lhs; + TJsonValue first; + TJsonValue second; + TJsonValue last; + first.InsertValue("e", "f"); + second.InsertValue("c", first); + last.InsertValue("a", second); + lhs.InsertValue("l", last); + + TJsonValue result; + UNIT_ASSERT(lhs.GetValueByPath("l/a/c/e", result, '/')); + UNIT_ASSERT(result.GetStringRobust() == "f"); + UNIT_ASSERT(!lhs.GetValueByPath("l/a/c/se", result, '/')); + UNIT_ASSERT(lhs.GetValueByPath("l/a/c", result, '/')); + UNIT_ASSERT(result.GetStringRobust() == "{\"e\":\"f\"}"); + + // faster TStringBuf version + UNIT_ASSERT_EQUAL(*lhs.GetValueByPath("l", '/'), last); + UNIT_ASSERT_EQUAL(*lhs.GetValueByPath("l/a", '/'), second); + UNIT_ASSERT_EQUAL(*lhs.GetValueByPath("l/a/c", '/'), first); + UNIT_ASSERT_EQUAL(*lhs.GetValueByPath("l.a.c.e", '.'), "f"); + UNIT_ASSERT_EQUAL(lhs.GetValueByPath("l/a/c/e/x", '/'), NULL); + UNIT_ASSERT_EQUAL(lhs.GetValueByPath("a/c/e/x", '/'), NULL); + UNIT_ASSERT_EQUAL(lhs.GetValueByPath("nokey", '/'), NULL); + UNIT_ASSERT_EQUAL(*lhs.GetValueByPath("", '/'), lhs); // itself + + TJsonValue array; + TJsonValue third; + array[0] = first; + array[1] = second; + third["t"] = array; + + UNIT_ASSERT(array.GetValueByPath("[0].e", result)); + UNIT_ASSERT(result.GetStringRobust() == "f"); + UNIT_ASSERT(third.GetValueByPath("t.[0].e", result)); + UNIT_ASSERT(result.GetStringRobust() == "f"); + UNIT_ASSERT(third.GetValueByPath("t.[1].c.e", result)); + UNIT_ASSERT(result.GetStringRobust() == "f"); + UNIT_ASSERT(!third.GetValueByPath("t.[2]", result)); + + UNIT_ASSERT(third.SetValueByPath("t.[2]", "g")); + UNIT_ASSERT(third.GetValueByPath("t.[2]", result)); + UNIT_ASSERT(result.GetStringRobust() == "g"); + + UNIT_ASSERT(lhs.SetValueByPath("l/a/c/se", "h", '/')); + UNIT_ASSERT(lhs.GetValueByPath("l/a/c/se", result, '/')); + UNIT_ASSERT(result.GetStringRobust() == "h"); + } + } + + Y_UNIT_TEST(GetValueByPathConstTest) { + TJsonValue lhs; + TJsonValue first; + TJsonValue second; + TJsonValue last; + first.InsertValue("e", "f"); + second.InsertValue("c", first); + last.InsertValue("a", second); + lhs.InsertValue("l", last); + + { + const TJsonValue* result = lhs.GetValueByPath("l", '/'); + UNIT_ASSERT_EQUAL(*result, last); + } + { + const TJsonValue* result = lhs.GetValueByPath("l/a", '/'); + UNIT_ASSERT_EQUAL(*result, second); + } + { + const TJsonValue* result = lhs.GetValueByPath("l/a/c", '/'); + UNIT_ASSERT_EQUAL(*result, first); + } + { + const TJsonValue* result = lhs.GetValueByPath("l.a.c.e", '.'); + UNIT_ASSERT_EQUAL(*result, "f"); + } + { + const TJsonValue* result = lhs.GetValueByPath("l/a/c/e/x", '/'); + UNIT_ASSERT_EQUAL(result, nullptr); + } + { + const TJsonValue* result = lhs.GetValueByPath("a/c/e/x", '/'); + UNIT_ASSERT_EQUAL(result, nullptr); + } + { + const TJsonValue* result = lhs.GetValueByPath("nokey", '/'); + UNIT_ASSERT_EQUAL(result, nullptr); + } + { + const TJsonValue* result = lhs.GetValueByPath("", '/'); + UNIT_ASSERT_EQUAL(*result, lhs); // itself + } + + TJsonValue array; + TJsonValue third; + array[0] = first; + array[1] = second; + third["t"] = array; + + UNIT_ASSERT(array.GetValueByPath("[0].e", '.')->GetStringRobust() == "f"); + UNIT_ASSERT(third.GetValueByPath("t.[0].e", '.')->GetStringRobust() == "f"); + UNIT_ASSERT(third.GetValueByPath("t.[1].c.e", '.')->GetStringRobust() == "f"); + } + + Y_UNIT_TEST(EraseValueFromArray) { + { + TJsonValue vec; + vec.AppendValue(TJsonValue(0)); + vec.AppendValue(TJsonValue(1)); + vec.AppendValue(TJsonValue("2")); + vec.AppendValue(TJsonValue("3.14")); + + TJsonValue vec1; + vec1.AppendValue(TJsonValue(0)); + vec1.AppendValue(TJsonValue("2")); + vec1.AppendValue(TJsonValue("3.14")); + + TJsonValue vec2; + vec2.AppendValue(TJsonValue(0)); + vec2.AppendValue(TJsonValue("2")); + + TJsonValue vec3; + vec3.AppendValue(TJsonValue("2")); + + TJsonValue vec4(JSON_ARRAY); + + UNIT_ASSERT(vec.IsArray()); + UNIT_ASSERT(vec.GetArray().size() == 4); + vec.EraseValue(1); + UNIT_ASSERT(vec.GetArray().size() == 3); + UNIT_ASSERT(vec == vec1); + vec.EraseValue(2); + UNIT_ASSERT(vec.GetArray().size() == 2); + UNIT_ASSERT(vec == vec2); + vec.EraseValue(0); + UNIT_ASSERT(vec.GetArray().size() == 1); + UNIT_ASSERT(vec == vec3); + vec.EraseValue(0); + UNIT_ASSERT(vec.GetArray().size() == 0); + UNIT_ASSERT(vec == vec4); + } + } + + Y_UNIT_TEST(NonConstMethodsTest) { + { + TJsonValue src; + TJsonValue value1; + value1.AppendValue(1); + value1.AppendValue(2); + src.InsertValue("key", value1); + src.InsertValue("key1", "HI!"); + TJsonValue dst; + TJsonValue value2; + value2.AppendValue(1); + value2.AppendValue(2); + value2.AppendValue(3); + dst.InsertValue("key", value2); + src.GetValueByPath("key", '.')->AppendValue(3); + src.EraseValue("key1"); + UNIT_ASSERT(src == dst); + dst.GetValueByPath("key", '.')->EraseValue(0); + UNIT_ASSERT(src != dst); + src.GetValueByPath("key", '.')->EraseValue(0); + UNIT_ASSERT(src == dst); + } + + { + TJsonValue src; + TJsonValue value1; + TJsonValue arr1; + value1.InsertValue("key", "value"); + arr1.AppendValue(value1); + arr1.AppendValue(value1); + arr1.AppendValue(value1); + src.InsertValue("arr", arr1); + + TJsonValue dst; + TJsonValue value2; + TJsonValue arr2; + value2.InsertValue("key", "value"); + value2.InsertValue("yek", "eulav"); + arr2.AppendValue(value2); + arr2.AppendValue(value2); + arr2.AppendValue(value2); + arr2.AppendValue(value2); + dst.InsertValue("arr", arr2); + + src["arr"].AppendValue(value1); + for (auto& node : src["arr"].GetArraySafe()) { + node.InsertValue("yek", "eulav"); + } + UNIT_ASSERT(src == dst); + } + + { + TJsonValue src; + TJsonValue value1; + TJsonValue arr1; + value1.InsertValue("key", "value"); + arr1.AppendValue(value1); + arr1.AppendValue(value1); + arr1.AppendValue(value1); + src.InsertValue("arr", arr1); + + TJsonValue dst; + TJsonValue value2; + TJsonValue arr2; + value2.InsertValue("key", "value"); + value2.InsertValue("yek", "eulav"); + arr2.AppendValue(value2); + arr2.AppendValue(value2); + arr2.AppendValue(value2); + arr2.AppendValue(value2); + dst.InsertValue("arr", arr2); + + src["arr"].AppendValue(value1); + for (auto& node : src.GetValueByPath("arr", '.')->GetArraySafe()) { + node.InsertValue("yek", "eulav"); + } + UNIT_ASSERT(src == dst); + } + + { + TJsonValue json; + json.InsertValue("key", "value"); + try { + json.GetArraySafe(); + UNIT_ASSERT(false); + } catch (const NJson::TJsonException&) { + } + + const TJsonValue constJson(json); + try { + constJson.GetArray(); + } catch (...) { + UNIT_ASSERT(false); + } + } + + { + // Check non-const GetArraySafe() + TJsonValue json{JSON_ARRAY}; + json.GetArraySafe().push_back(TJsonValue{"foo"}); + + TJsonValue expectedJson; + expectedJson.AppendValue(TJsonValue{"foo"}); + UNIT_ASSERT(json == expectedJson); + + TJsonValue::TArray jsonArray = std::move(json.GetArraySafe()); + TJsonValue::TArray expectedArray = {TJsonValue{"foo"}}; + UNIT_ASSERT(jsonArray == expectedArray); + } + + { + // Check non-const GetMap() + TJsonValue json{JSON_MAP}; + json.GetMapSafe()["foo"] = "bar"; + + TJsonValue expectedJson; + expectedJson.InsertValue("foo", "bar"); + UNIT_ASSERT(json == expectedJson); + + TJsonValue::TMapType jsonMap = std::move(json.GetMapSafe()); + TJsonValue::TMapType expectedMap; + expectedMap["foo"] = TJsonValue{"bar"}; + UNIT_ASSERT(jsonMap["foo"] == expectedMap["foo"]); + } + } + + Y_UNIT_TEST(NonexistentFieldAccessTest) { + { + TJsonValue json; + json.InsertValue("some", "key"); + + UNIT_ASSERT(!json["some"]["weird"]["access"]["sequence"].Has("value")); + UNIT_ASSERT(!json["some"]["weird"]["access"]["sequence"].IsDefined()); + + UNIT_ASSERT(json["some"].GetType() == JSON_MAP); + } + } + + Y_UNIT_TEST(DefaultValuesTest) { + { + TJsonValue json; + json.InsertValue("some", "key"); + json.InsertValue("existing", 1.2); + + UNIT_ASSERT_VALUES_EQUAL(json["existing"].GetDoubleSafe(), 1.2); + UNIT_ASSERT_VALUES_EQUAL(json["existing"].GetDoubleSafe(15), 1.2); + + UNIT_ASSERT_EXCEPTION(json["some"].GetUIntegerSafe(), yexception); + UNIT_ASSERT_EXCEPTION(json["some"].GetUIntegerSafe(12), yexception); + + UNIT_ASSERT_EXCEPTION(json["nonexistent"].GetUIntegerSafe(), yexception); + UNIT_ASSERT_VALUES_EQUAL(json["nonexistent"].GetUIntegerSafe(12), 12); + UNIT_ASSERT_VALUES_EQUAL(json["nonexistent"]["more_nonexistent"].GetUIntegerSafe(12), 12); + + json.InsertValue("map", TJsonValue(JSON_MAP)); + + UNIT_ASSERT_VALUES_EQUAL(json["map"]["nonexistent"].GetUIntegerSafe(12), 12); + } + } + + Y_UNIT_TEST(GetArrayPointerInArrayTest) { + TJsonValue outer; + { + TJsonValue json; + json.AppendValue(1); + json.AppendValue(2); + json.AppendValue(3); + + outer.AppendValue(json); + } + const TJsonValue::TArray* array = nullptr; + GetArrayPointer(outer, 0, &array); + UNIT_ASSERT_VALUES_EQUAL((*array)[1], 2); + } + + Y_UNIT_TEST(GetArrayPointerInMapTest) { + TJsonValue outer; + { + TJsonValue json; + json.AppendValue(1); + json.AppendValue(2); + json.AppendValue(3); + + outer.InsertValue("x", json); + } + const TJsonValue::TArray* array = nullptr; + GetArrayPointer(outer, "x", &array); + UNIT_ASSERT_VALUES_EQUAL((*array)[1], 2); + } + + Y_UNIT_TEST(GetMapPointerInArrayTest) { + TJsonValue outer; + { + TJsonValue json; + json.InsertValue("a", 1); + json.InsertValue("b", 2); + json.InsertValue("c", 3); + + outer.AppendValue(json); + } + const TJsonValue::TMapType* map = nullptr; + GetMapPointer(outer, 0, &map); + UNIT_ASSERT_VALUES_EQUAL((*map).at("b"), 2); + } + + Y_UNIT_TEST(GetMapPointerInMapTest) { + TJsonValue outer; + { + TJsonValue json; + json.InsertValue("a", 1); + json.InsertValue("b", 2); + json.InsertValue("c", 3); + + outer.InsertValue("x", json); + } + const TJsonValue::TMapType* map = nullptr; + GetMapPointer(outer, "x", &map); + UNIT_ASSERT_VALUES_EQUAL((*map).at("b"), 2); + } + + Y_UNIT_TEST(GetIntegerRobustBignumStringTest) { + TString value = "1626862681464633683"; + TJsonValue json(value); + UNIT_ASSERT_VALUES_EQUAL(json.GetUIntegerRobust(), FromString<ui64>(value)); + UNIT_ASSERT_VALUES_EQUAL(json.GetIntegerRobust(), FromString<i64>(value)); + } + + Y_UNIT_TEST(MoveSubpartToSelf) { + TJsonValue json; + json[0] = "testing 0"; + json[1] = "testing 1"; + json[2] = "testing 2"; + json = std::move(json[1]); + UNIT_ASSERT_VALUES_EQUAL(json.GetString(), "testing 1"); + + const char* longTestString = + "Testing TJsonValue& operator=(TJsonValue&&) subpart self moving " + "after TJsonValue was constrcuted from TString&&."; + + json["hello"] = TString{longTestString}; + json = std::move(json["hello"]); + UNIT_ASSERT_VALUES_EQUAL(json.GetString(), longTestString); + } + + Y_UNIT_TEST(TJsonArrayMapConstructor) { + TJsonMap emptyMap; + UNIT_ASSERT_VALUES_EQUAL(emptyMap.GetType(), JSON_MAP); + UNIT_ASSERT_VALUES_EQUAL(emptyMap.GetMapSafe().size(), 0); + + TJsonArray emptyArray; + UNIT_ASSERT_VALUES_EQUAL(emptyArray.GetType(), JSON_ARRAY); + UNIT_ASSERT_VALUES_EQUAL(emptyArray.GetArraySafe().size(), 0); + + TJsonMap filled = { + {"1", 1}, + {"2", "2"}, + {"3", TJsonArray{3}}, + {"4", TJsonMap{{"5", 5}}}, + }; + UNIT_ASSERT_VALUES_EQUAL(filled.GetType(), JSON_MAP); + UNIT_ASSERT_VALUES_EQUAL(filled["1"], TJsonValue{1}); + UNIT_ASSERT_VALUES_EQUAL(filled["2"], TJsonValue{"2"}); + UNIT_ASSERT_VALUES_EQUAL(filled["3"].GetArraySafe().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(filled["3"][0], TJsonValue{3}); + UNIT_ASSERT_VALUES_EQUAL(filled["4"].GetMapSafe().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(filled["4"]["5"], TJsonValue{5}); + } + + Y_UNIT_TEST(FromRegularJsonValue) { + NJson::TJsonValue regular; + regular["key3"] = 3; + regular["key1"] = 1; + regular["key2"] = "two"; + regular["key4"].AppendValue(42); + regular["key4"].AppendValue("inner"); + + TJsonValue ordered(regular); + + UNIT_ASSERT(ordered.IsMap()); + UNIT_ASSERT_VALUES_EQUAL(ordered.GetMap().size(), 4); + + UNIT_ASSERT(ordered.Has("key1")); + UNIT_ASSERT(ordered.Has("key2")); + UNIT_ASSERT(ordered.Has("key3")); + UNIT_ASSERT(ordered.Has("key4")); + + UNIT_ASSERT_VALUES_EQUAL(ordered["key1"].GetIntegerRobust(), 1); + UNIT_ASSERT_VALUES_EQUAL(ordered["key2"].GetStringRobust(), "two"); + UNIT_ASSERT_VALUES_EQUAL(ordered["key3"].GetIntegerRobust(), 3); + + UNIT_ASSERT(ordered["key4"].IsArray()); + UNIT_ASSERT_VALUES_EQUAL(ordered["key4"].GetArray().size(), 2); + UNIT_ASSERT_VALUES_EQUAL(ordered["key4"][0].GetIntegerRobust(), 42); + UNIT_ASSERT_VALUES_EQUAL(ordered["key4"][1].GetStringRobust(), "inner"); + } + + Y_UNIT_TEST(ConstructFromVariousTypes) { + NJson::TJsonValue nonEmptyArray(NJson::JSON_ARRAY); + nonEmptyArray.AppendValue(42); + nonEmptyArray.AppendValue("test"); + + NJson::TJsonValue nonEmptyMap(NJson::JSON_MAP); + nonEmptyMap["key"] = "value"; + nonEmptyMap["another"] = 3.14; + + NJson::TJsonValue values[] = { + NJson::TJsonValue(), + NJson::TJsonValue(NJson::JSON_NULL), + NJson::TJsonValue(true), + NJson::TJsonValue(42), + NJson::TJsonValue(42u), + NJson::TJsonValue(3.14), + NJson::TJsonValue("test string"), + NJson::TJsonValue(NJson::JSON_ARRAY), + NJson::TJsonValue(NJson::JSON_MAP), + nonEmptyArray, + nonEmptyMap + }; + + for (const auto& val : values) { + TJsonValue ordered(val); + + switch (val.GetType()) { + case NJson::JSON_UNDEFINED: + UNIT_ASSERT(!ordered.IsDefined()); + break; + case NJson::JSON_NULL: + UNIT_ASSERT(ordered.IsNull()); + break; + case NJson::JSON_BOOLEAN: + UNIT_ASSERT(ordered.IsBoolean()); + UNIT_ASSERT_VALUES_EQUAL(ordered.GetBooleanRobust(), val.GetBoolean()); + break; + case NJson::JSON_INTEGER: + UNIT_ASSERT(ordered.IsInteger()); + UNIT_ASSERT_VALUES_EQUAL(ordered.GetIntegerRobust(), val.GetInteger()); + break; + case NJson::JSON_UINTEGER: + UNIT_ASSERT(ordered.IsUInteger()); + UNIT_ASSERT_VALUES_EQUAL(ordered.GetUIntegerRobust(), val.GetUInteger()); + break; + case NJson::JSON_DOUBLE: + UNIT_ASSERT(ordered.IsDouble()); + UNIT_ASSERT_VALUES_EQUAL(ordered.GetDoubleRobust(), val.GetDouble()); + break; + case NJson::JSON_STRING: + UNIT_ASSERT(ordered.IsString()); + UNIT_ASSERT_VALUES_EQUAL(ordered.GetStringRobust(), val.GetString()); + break; + case NJson::JSON_ARRAY: + UNIT_ASSERT(ordered.IsArray()); + UNIT_ASSERT_VALUES_EQUAL(ordered.GetArray().size(), val.GetArray().size()); + + if (!val.GetArray().empty()) { + for (size_t i = 0; i < val.GetArray().size(); ++i) { + UNIT_ASSERT(ordered[i] == TJsonValue(val[i])); + } + } + break; + case NJson::JSON_MAP: + UNIT_ASSERT(ordered.IsMap()); + UNIT_ASSERT_VALUES_EQUAL(ordered.GetMap().size(), val.GetMap().size()); + + if (!val.GetMap().empty()) { + for (const auto& [key, value] : val.GetMap()) { + UNIT_ASSERT(ordered.Has(key)); + UNIT_ASSERT(ordered[key] == TJsonValue(value)); + } + } + break; + } + } + } + + Y_UNIT_TEST(GetNonOrderedJsonValueRobust) { + TJsonValue ordered; + ordered.InsertValue("key3", 3); + ordered.InsertValue("key1", 1); + ordered.InsertValue("key2", "two"); + + TJsonValue arr; + arr.AppendValue(42); + arr.AppendValue("inner"); + ordered.InsertValue("key4", arr); + + NJson::TJsonValue nonOrdered = ordered.GetNonOrderedJsonValue(); + + UNIT_ASSERT(nonOrdered.IsMap()); + UNIT_ASSERT_VALUES_EQUAL(nonOrdered.GetMap().size(), 4); + + UNIT_ASSERT(nonOrdered.Has("key1")); + UNIT_ASSERT(nonOrdered.Has("key2")); + UNIT_ASSERT(nonOrdered.Has("key3")); + UNIT_ASSERT(nonOrdered.Has("key4")); + + UNIT_ASSERT_VALUES_EQUAL(nonOrdered["key1"].GetInteger(), 1); + UNIT_ASSERT_VALUES_EQUAL(nonOrdered["key2"].GetString(), "two"); + UNIT_ASSERT_VALUES_EQUAL(nonOrdered["key3"].GetInteger(), 3); + + UNIT_ASSERT(nonOrdered["key4"].IsArray()); + UNIT_ASSERT_VALUES_EQUAL(nonOrdered["key4"].GetArray().size(), 2); + UNIT_ASSERT_VALUES_EQUAL(nonOrdered["key4"][0].GetInteger(), 42); + UNIT_ASSERT_VALUES_EQUAL(nonOrdered["key4"][1].GetString(), "inner"); + } + + Y_UNIT_TEST(SaveLoadPreservesData) { + TJsonValue original; + original.InsertValue("z_key", 1); + original.InsertValue("a_key", 2); + original.InsertValue("m_key", 3); + + TJsonValue inner; + inner.InsertValue("inner_z", 10); + inner.InsertValue("inner_a", 20); + original.InsertValue("inner_obj", inner); + + TJsonValue arr; + arr.AppendValue("first"); + arr.AppendValue(42); + arr.AppendValue(inner); + original.InsertValue("array", arr); + + TStringStream ss; + original.Save(&ss); + + TJsonValue loaded; + loaded.Load(&ss); + + UNIT_ASSERT(loaded.Has("a_key")); + UNIT_ASSERT(loaded.Has("z_key")); + UNIT_ASSERT(loaded.Has("m_key")); + UNIT_ASSERT(loaded.Has("inner_obj")); + UNIT_ASSERT(loaded.Has("array")); + + UNIT_ASSERT_VALUES_EQUAL(loaded["a_key"], 2); + UNIT_ASSERT_VALUES_EQUAL(loaded["z_key"], 1); + UNIT_ASSERT_VALUES_EQUAL(loaded["m_key"], 3); + + UNIT_ASSERT(loaded["inner_obj"].Has("inner_a")); + UNIT_ASSERT(loaded["inner_obj"].Has("inner_z")); + UNIT_ASSERT_VALUES_EQUAL(loaded["inner_obj"]["inner_a"], 20); + UNIT_ASSERT_VALUES_EQUAL(loaded["inner_obj"]["inner_z"], 10); + + UNIT_ASSERT(loaded["array"].IsArray()); + const auto& loadedArray = loaded["array"].GetArray(); + UNIT_ASSERT_VALUES_EQUAL(loadedArray.size(), 3); + UNIT_ASSERT_VALUES_EQUAL(loadedArray[0], "first"); + UNIT_ASSERT_VALUES_EQUAL(loadedArray[1], 42); + + UNIT_ASSERT(loadedArray[2].Has("inner_a")); + UNIT_ASSERT(loadedArray[2].Has("inner_z")); + UNIT_ASSERT_VALUES_EQUAL(loadedArray[2]["inner_a"], 20); + UNIT_ASSERT_VALUES_EQUAL(loadedArray[2]["inner_z"], 10); + } + + Y_UNIT_TEST(SaveLoadEdgeCases) { + TJsonValue values[] = { + TJsonValue(), + TJsonValue(JSON_NULL), + TJsonValue(true), + TJsonValue(42), + TJsonValue(42u), + TJsonValue(3.14), + TJsonValue("test string"), + TJsonValue(JSON_ARRAY), + TJsonValue(JSON_MAP) + }; + + for (auto& original : values) { + TStringStream ss; + original.Save(&ss); + + TJsonValue loaded; + loaded.Load(&ss); + + if ( + original.IsBoolean() || original.IsInteger() || + original.IsUInteger() || original.IsDouble() || + original.IsString() || original.IsNull() || + !original.IsDefined() + ) { + UNIT_ASSERT(loaded == original); + } else if (original.IsMap()) { + UNIT_ASSERT(loaded.IsMap()); + UNIT_ASSERT_VALUES_EQUAL(loaded.GetMap().size(), original.GetMap().size()); + } else if (original.IsArray()) { + UNIT_ASSERT(loaded.IsArray()); + UNIT_ASSERT_VALUES_EQUAL(loaded.GetArray().size(), original.GetArray().size()); + } + } + } + + Y_UNIT_TEST(SaveLoadLargeData) { + TJsonValue original; + TJsonValue largeArray; + + for (int i = 0; i < 1000; ++i) { + largeArray.AppendValue(i); + } + original.InsertValue("large_array", largeArray); + + TJsonValue largeMap; + for (int i = 999; i >= 0; --i) { + largeMap.InsertValue(ToString(i), i); + } + original.InsertValue("large_map", largeMap); + + TStringStream ss; + original.Save(&ss); + + TJsonValue loaded; + loaded.Load(&ss); + + UNIT_ASSERT(loaded.Has("large_array")); + UNIT_ASSERT(loaded["large_array"].IsArray()); + const auto& loadedArray = loaded["large_array"].GetArray(); + UNIT_ASSERT_VALUES_EQUAL(loadedArray.size(), 1000); + for (size_t i = 0; i < loadedArray.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(loadedArray[i].GetIntegerRobust(), i); + } + + UNIT_ASSERT(loaded.Has("large_map")); + UNIT_ASSERT(loaded["large_map"].IsMap()); + const auto& loadedMap = loaded["large_map"].GetMap(); + UNIT_ASSERT_VALUES_EQUAL(loadedMap.size(), 1000); + for (int i = 0; i < 1000; ++i) { + const TString key = ToString(i); + auto it = loadedMap.find(key); + UNIT_ASSERT(it != loadedMap.end()); + UNIT_ASSERT_VALUES_EQUAL(it->second.GetIntegerRobust(), i); + } + } +} // TJsonValueTest diff --git a/library/cpp/json/ordered_maps/ut/json_writer_ordered_ut.cpp b/library/cpp/json/ordered_maps/ut/json_writer_ordered_ut.cpp new file mode 100644 index 00000000000..adf2e6f6230 --- /dev/null +++ b/library/cpp/json/ordered_maps/ut/json_writer_ordered_ut.cpp @@ -0,0 +1,228 @@ +#include <library/cpp/json/ordered_maps/json_writer_ordered.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/str.h> + +using namespace NJson::NOrderedJson; + +Y_UNIT_TEST_SUITE(TJsonWriterTest) { + Y_UNIT_TEST(SimpleWriteTest) { + TString expected1 = "{\"key1\":1,\"key2\":2,\"key3\":3"; + TString expected2 = expected1 + ",\"array\":[\"stroka\",false]"; + TString expected3 = expected2 + "}"; + + TStringStream out; + + TJsonWriter json(&out, false); + json.OpenMap(); + json.Write("key1", (ui16)1); + json.WriteKey("key2"); + json.Write((i32)2); + json.Write("key3", (ui64)3); + + UNIT_ASSERT(out.Empty()); + json.Flush(); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected1); + + json.Write("array"); + json.OpenArray(); + json.Write("stroka"); + json.Write(false); + json.CloseArray(); + + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected1); + json.Flush(); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected2); + + json.CloseMap(); + + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected2); + json.Flush(); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected3); + } + + Y_UNIT_TEST(SimpleWriteValueTest) { + TString expected = "{\"key1\":null,\"key2\":{\"subkey1\":[1,{\"subsubkey\":\"test2\"},null,true],\"subkey2\":\"test\"}}"; + TJsonValue v; + v["key1"] = JSON_NULL; + v["key2"]["subkey1"].AppendValue(1); + v["key2"]["subkey1"].AppendValue(JSON_MAP)["subsubkey"] = "test2"; + v["key2"]["subkey1"].AppendValue(JSON_NULL); + v["key2"]["subkey1"].AppendValue(true); + v["key2"]["subkey2"] = "test"; + TStringStream out; + WriteJson(&out, &v); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected); + } + + Y_UNIT_TEST(FormatOutput) { + TString expected = "{\n \"key1\":null,\n \"key2\":\n {\n \"subkey1\":\n [\n 1,\n {\n \"subsubkey\":\"test2\"\n },\n null,\n true\n ],\n \"subkey2\":\"test\"\n }\n}"; + TJsonValue v; + v["key1"] = JSON_NULL; + v["key2"]["subkey1"].AppendValue(1); + v["key2"]["subkey1"].AppendValue(JSON_MAP)["subsubkey"] = "test2"; + v["key2"]["subkey1"].AppendValue(JSON_NULL); + v["key2"]["subkey1"].AppendValue(true); + v["key2"]["subkey2"] = "test"; + TStringStream out; + WriteJson(&out, &v, true); + UNIT_ASSERT_STRINGS_EQUAL(out.Str(), expected); + } + + Y_UNIT_TEST(SortKeys) { + TString expected = "{\"a\":null,\"j\":null,\"n\":null,\"y\":null,\"z\":null}"; + TJsonValue v; + v["z"] = JSON_NULL; + v["n"] = JSON_NULL; + v["a"] = JSON_NULL; + v["y"] = JSON_NULL; + v["j"] = JSON_NULL; + TStringStream out; + WriteJson(&out, &v, false, true); + UNIT_ASSERT_STRINGS_EQUAL(out.Str(), expected); + } + + Y_UNIT_TEST(SimpleUnsignedIntegerWriteTest) { + { + TString expected = "{\"test\":1}"; + TJsonValue v; + v.InsertValue("test", 1ull); + TStringStream out; + WriteJson(&out, &v); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected); + } // 1 + + { + TString expected = "{\"test\":-1}"; + TJsonValue v; + v.InsertValue("test", -1); + TStringStream out; + WriteJson(&out, &v); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected); + } // -1 + + { + TString expected = "{\"test\":18446744073709551615}"; + TJsonValue v; + v.InsertValue("test", 18446744073709551615ull); + TStringStream out; + WriteJson(&out, &v); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected); + } // 18446744073709551615 + + { + TString expected = "{\"test\":[1,18446744073709551615]}"; + TJsonValue v; + v.InsertValue("test", TJsonValue()); + v["test"].AppendValue(1); + v["test"].AppendValue(18446744073709551615ull); + TStringStream out; + WriteJson(&out, &v); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected); + } // 18446744073709551615 + } // SimpleUnsignedIntegerWriteTest + + Y_UNIT_TEST(WriteOptionalTest) { + { + TString expected = "{\"test\":1}"; + + TStringStream out; + + TJsonWriter json(&out, false); + json.OpenMap(); + json.WriteOptional("test", MakeMaybe<int>(1)); + json.CloseMap(); + json.Flush(); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected); + } + + { + TString expected = "{}"; + + TStringStream out; + + TMaybe<int> nothing = Nothing(); + + TJsonWriter json(&out, false); + json.OpenMap(); + json.WriteOptional("test", nothing); + json.CloseMap(); + json.Flush(); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected); + } + + { + TString expected = "{}"; + + TStringStream out; + + TMaybe<int> empty; + + TJsonWriter json(&out, false); + json.OpenMap(); + json.WriteOptional("test", empty); + json.CloseMap(); + json.Flush(); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected); + } + + { + TString expected = "{}"; + + TStringStream out; + + TJsonWriter json(&out, false); + json.OpenMap(); + json.WriteOptional("test", Nothing()); + json.CloseMap(); + json.Flush(); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected); + } + } + + Y_UNIT_TEST(Callback) { + NJsonOrderedWriter::TBuf json; + json.WriteString("A"); + UNIT_ASSERT_VALUES_EQUAL(json.Str(), "\"A\""); + UNIT_ASSERT_VALUES_EQUAL(WrapJsonToCallback(json, ""), "\"A\""); + UNIT_ASSERT_VALUES_EQUAL(WrapJsonToCallback(json, "Foo"), "Foo(\"A\")"); + } + + Y_UNIT_TEST(FloatPrecision) { + const double value = 1517933989.4242; + const NJson::NOrderedJson::TJsonValue json(value); + NJson::NOrderedJson::TJsonWriterConfig config; + { + TString expected = "1517933989"; + TString actual = NJson::NOrderedJson::WriteJson(json); + UNIT_ASSERT_VALUES_EQUAL(actual, expected); + } + { + TString expected = "1517933989"; + + TStringStream ss; + NJson::NOrderedJson::WriteJson(&ss, &json, config); + TString actual = ss.Str(); + UNIT_ASSERT_VALUES_EQUAL(actual, expected); + } + { + config.DoubleNDigits = 13; + TString expected = "1517933989.424"; + + TStringStream ss; + NJson::NOrderedJson::WriteJson(&ss, &json, config); + TString actual = ss.Str(); + UNIT_ASSERT_VALUES_EQUAL(actual, expected); + } + { + config.DoubleNDigits = 6; + config.FloatToStringMode = PREC_POINT_DIGITS; + TString expected = "1517933989.424200"; + + TStringStream ss; + NJson::NOrderedJson::WriteJson(&ss, &json, config); + TString actual = ss.Str(); + UNIT_ASSERT_VALUES_EQUAL(actual, expected); + } + } +} diff --git a/library/cpp/json/ordered_maps/ut/ya.make b/library/cpp/json/ordered_maps/ut/ya.make new file mode 100644 index 00000000000..0a92c1a4093 --- /dev/null +++ b/library/cpp/json/ordered_maps/ut/ya.make @@ -0,0 +1,17 @@ +UNITTEST_FOR(library/cpp/json/ordered_maps) + + +PEERDIR( + library/cpp/string_utils/relaxed_escaper +) + +SRCS( + json_value_ordered_ut.cpp + json_reader_fast_ordered_ut.cpp + json_reader_nan_ordered_ut.cpp + json_reader_ordered_ut.cpp + json_writer_ordered_ut.cpp + json_ordered_ut.cpp +) + +END() diff --git a/library/cpp/json/ordered_maps/ya.make b/library/cpp/json/ordered_maps/ya.make new file mode 100644 index 00000000000..9699638f173 --- /dev/null +++ b/library/cpp/json/ordered_maps/ya.make @@ -0,0 +1,20 @@ +LIBRARY() + +PEERDIR( + library/cpp/json +) + +SRCS( + json_ordered.cpp + json_value_ordered.cpp + json_reader_ordered.cpp + json_writer_ordered.cpp +) + +GENERATE_ENUM_SERIALIZATION(json_value_ordered.h) + +END() + +RECURSE_FOR_TESTS( + ut +) |
