diff options
author | Daniil Cherednik <dan.cherednik@gmail.com> | 2023-05-05 11:09:01 +0300 |
---|---|---|
committer | Daniil Cherednik <dan.cherednik@gmail.com> | 2023-05-05 11:09:01 +0300 |
commit | b5a989b16cafa8a3b3bc076f1097a0eda6f48c06 (patch) | |
tree | 4da744117a5aab37758921fa43b95a3068e5aec1 /library/cpp | |
parent | fc1cffcfa7f0497a1f97b384a24bcbf23362f3be (diff) | |
download | ydb-b5a989b16cafa8a3b3bc076f1097a0eda6f48c06.tar.gz |
Ydb stable 23-1-2623.1.26
x-stable-origin-commit: 22184a7e157553d447f17a2dffc4ea2d32dfd74d
Diffstat (limited to 'library/cpp')
28 files changed, 3978 insertions, 66 deletions
diff --git a/library/cpp/actors/core/actor.h b/library/cpp/actors/core/actor.h index 9ed3608223..b67c04b09b 100644 --- a/library/cpp/actors/core/actor.h +++ b/library/cpp/actors/core/actor.h @@ -29,6 +29,7 @@ namespace NActors { ui32 CapturedActivation = 0; ESendingType CapturedType = ESendingType::Lazy; ESendingType SendingType = ESendingType::Common; + bool IsEnoughCpu = true; }; extern Y_POD_THREAD(TThreadContext*) TlsThreadContext; @@ -436,20 +437,24 @@ namespace NActors { // must be called to wrap any call trasitions from one actor to another template<typename TActor, typename TMethod, typename... TArgs> - static decltype((std::declval<TActor>().*std::declval<TMethod>())(std::declval<TArgs>()...)) - InvokeOtherActor(TActor& actor, TMethod&& method, TArgs&&... args) { - struct TRecurseContext: TActorContext { - TActivationContext* Prev; + static std::invoke_result_t<TMethod, TActor, TArgs...> InvokeOtherActor(TActor& actor, TMethod&& method, TArgs&&... args) { + struct TRecurseContext : TActorContext { + TActivationContext* const Prev; + TRecurseContext(const TActorId& actorId) : TActorContext(TActivationContext::ActorContextFor(actorId)) - , Prev(TlsActivationContext) { + , Prev(TlsActivationContext) + { TlsActivationContext = this; } + ~TRecurseContext() { + Y_VERIFY(TlsActivationContext == this, "TlsActivationContext mismatch; probably InvokeOtherActor was invoked from a coroutine"); TlsActivationContext = Prev; } } context(actor.SelfId()); - return (actor.*method)(std::forward<TArgs>(args)...); + + return std::invoke(std::forward<TMethod>(method), actor, std::forward<TArgs>(args)...); } virtual void Registered(TActorSystem* sys, const TActorId& owner); @@ -486,6 +491,12 @@ namespace NActors { } protected: + void SetEnoughCpu(bool isEnough) { + if (TlsThreadContext) { + TlsThreadContext->IsEnoughCpu = isEnough; + } + } + void Describe(IOutputStream&) const noexcept override; bool Send(const TActorId& recipient, IEventBase* ev, ui32 flags = 0, ui64 cookie = 0, NWilson::TTraceId traceId = {}) const noexcept final; bool Send(const TActorId& recipient, THolder<IEventBase> ev, ui32 flags = 0, ui64 cookie = 0, NWilson::TTraceId traceId = {}) const { diff --git a/library/cpp/actors/core/actorsystem.cpp b/library/cpp/actors/core/actorsystem.cpp index b8896acb34..21cca94d40 100644 --- a/library/cpp/actors/core/actorsystem.cpp +++ b/library/cpp/actors/core/actorsystem.cpp @@ -217,6 +217,11 @@ namespace NActors { CpuManager->GetPoolStats(poolId, poolStats, statsCopy); } + THarmonizerStats TActorSystem::GetHarmonizerStats() const { + return CpuManager->GetHarmonizerStats(); + + } + void TActorSystem::Start() { Y_VERIFY(StartExecuted == false); StartExecuted = true; diff --git a/library/cpp/actors/core/actorsystem.h b/library/cpp/actors/core/actorsystem.h index cd2cfda1bb..9f2483245a 100644 --- a/library/cpp/actors/core/actorsystem.h +++ b/library/cpp/actors/core/actorsystem.h @@ -299,6 +299,8 @@ namespace NActors { void GetPoolStats(ui32 poolId, TExecutorPoolStats& poolStats, TVector<TExecutorThreadStats>& statsCopy) const; + THarmonizerStats GetHarmonizerStats() const; + void DeferPreStop(std::function<void()> fn) { DeferredPreStop.push_back(std::move(fn)); } diff --git a/library/cpp/actors/core/cpu_manager.h b/library/cpp/actors/core/cpu_manager.h index e2e3861a3b..c3a7588639 100644 --- a/library/cpp/actors/core/cpu_manager.h +++ b/library/cpp/actors/core/cpu_manager.h @@ -40,6 +40,13 @@ namespace NActors { } } + THarmonizerStats GetHarmonizerStats() const { + if (Harmonizer) { + return Harmonizer->GetStats(); + } + return {}; + } + private: IExecutorPool* CreateExecutorPool(ui32 poolId); }; diff --git a/library/cpp/actors/core/event.h b/library/cpp/actors/core/event.h index 6b92edaf41..90ae16ac26 100644 --- a/library/cpp/actors/core/event.h +++ b/library/cpp/actors/core/event.h @@ -80,7 +80,8 @@ namespace NActors { Y_FAIL("Event type %" PRIu32 " doesn't match the expected type %" PRIu32, Type, TEventType::EventType); if (!Event) { - Event.Reset(TEventType::Load(Buffer.Get())); + static TEventSerializedData empty; + Event.Reset(TEventType::Load(Buffer ? Buffer.Get() : &empty)); } if (Event) { diff --git a/library/cpp/actors/core/event_pb.h b/library/cpp/actors/core/event_pb.h index 2d388fceeb..38058df749 100644 --- a/library/cpp/actors/core/event_pb.h +++ b/library/cpp/actors/core/event_pb.h @@ -228,7 +228,7 @@ namespace NActors { return result; } - static IEventBase* Load(TIntrusivePtr<TEventSerializedData> input) { + static IEventBase* Load(TEventSerializedData *input) { THolder<TEventPBBase> ev(new TEv()); if (!input->GetSize()) { Y_PROTOBUF_SUPPRESS_NODISCARD ev->Record.ParseFromString(TString()); diff --git a/library/cpp/actors/core/event_pb_payload_ut.cpp b/library/cpp/actors/core/event_pb_payload_ut.cpp index eab007bc15..7024d338d5 100644 --- a/library/cpp/actors/core/event_pb_payload_ut.cpp +++ b/library/cpp/actors/core/event_pb_payload_ut.cpp @@ -66,7 +66,7 @@ Y_UNIT_TEST_SUITE(TEventProtoWithPayload) { } UNIT_ASSERT_VALUES_EQUAL(chunkerRes, ser); - THolder<IEventBase> ev2 = THolder(TEventTo::Load(buffers)); + THolder<IEventBase> ev2 = THolder(TEventTo::Load(buffers.Get())); TEventTo& msg2 = static_cast<TEventTo&>(*ev2); UNIT_ASSERT_VALUES_EQUAL(msg2.Record.GetMeta(), msg.Record.GetMeta()); UNIT_ASSERT_EQUAL(msg2.GetPayload(msg2.Record.GetPayloadId(0)), msg.GetPayload(msg.Record.GetPayloadId(0))); @@ -142,7 +142,7 @@ Y_UNIT_TEST_SUITE(TEventProtoWithPayload) { // deserialize auto data = MakeIntrusive<TEventSerializedData>(ser1, false); - THolder<TEvMessageWithPayloadPreSerialized> parsedEvent(static_cast<TEvMessageWithPayloadPreSerialized*>(TEvMessageWithPayloadPreSerialized::Load(data))); + THolder<TEvMessageWithPayloadPreSerialized> parsedEvent(static_cast<TEvMessageWithPayloadPreSerialized*>(TEvMessageWithPayloadPreSerialized::Load(data.Get()))); UNIT_ASSERT_VALUES_EQUAL(parsedEvent->PreSerializedData, ""); // this field is empty after deserialization auto& record = parsedEvent->GetRecord(); UNIT_ASSERT_VALUES_EQUAL(record.GetMeta(), msg.GetMeta()); diff --git a/library/cpp/actors/core/executor_pool.h b/library/cpp/actors/core/executor_pool.h index c7c85e61fd..f4def74077 100644 --- a/library/cpp/actors/core/executor_pool.h +++ b/library/cpp/actors/core/executor_pool.h @@ -13,6 +13,13 @@ namespace NActors { struct TCpuConsumption { double ConsumedUs = 0; double BookedUs = 0; + ui64 NotEnoughCpuExecutions = 0; + + void Add(const TCpuConsumption& other) { + ConsumedUs += other.ConsumedUs; + BookedUs += other.BookedUs; + NotEnoughCpuExecutions += other.NotEnoughCpuExecutions; + } }; class IExecutorPool : TNonCopyable { diff --git a/library/cpp/actors/core/executor_pool_basic.cpp b/library/cpp/actors/core/executor_pool_basic.cpp index de04105991..fc87daed77 100644 --- a/library/cpp/actors/core/executor_pool_basic.cpp +++ b/library/cpp/actors/core/executor_pool_basic.cpp @@ -337,7 +337,7 @@ namespace NActors { poolStats.DefaultThreadCount = DefaultThreadCount; poolStats.MaxThreadCount = MaxThreadCount; if (Harmonizer) { - TPoolHarmonizedStats stats = Harmonizer->GetPoolStats(PoolId); + TPoolHarmonizerStats stats = Harmonizer->GetPoolStats(PoolId); poolStats.IsNeedy = stats.IsNeedy; poolStats.IsStarved = stats.IsStarved; poolStats.IsHoggish = stats.IsHoggish; @@ -345,6 +345,10 @@ namespace NActors { poolStats.DecreasingThreadsByStarvedState = stats.DecreasingThreadsByStarvedState; poolStats.DecreasingThreadsByHoggishState = stats.DecreasingThreadsByHoggishState; poolStats.PotentialMaxThreadCount = stats.PotentialMaxThreadCount; + poolStats.MaxConsumedCpuUs = stats.MaxConsumedCpu; + poolStats.MinConsumedCpuUs = stats.MinConsumedCpu; + poolStats.MaxBookedCpuUs = stats.MaxBookedCpu; + poolStats.MinBookedCpuUs = stats.MinBookedCpu; } statsCopy.resize(PoolThreads + 1); @@ -500,7 +504,7 @@ namespace NActors { TThreadCtx& threadCtx = Threads[threadIdx]; TExecutorThreadStats stats; threadCtx.Thread->GetCurrentStats(stats); - return {Ts2Us(stats.SafeElapsedTicks), static_cast<double>(stats.CpuUs)}; + return {Ts2Us(stats.SafeElapsedTicks), static_cast<double>(stats.CpuUs), stats.NotEnoughCpuExecutions}; } i16 TBasicExecutorPool::GetBlockingThreadCount() const { diff --git a/library/cpp/actors/core/executor_thread.cpp b/library/cpp/actors/core/executor_thread.cpp index 3c5dc2c2b4..f05b1d4479 100644 --- a/library/cpp/actors/core/executor_thread.cpp +++ b/library/cpp/actors/core/executor_thread.cpp @@ -364,7 +364,8 @@ namespace NActors { } #undef EXECUTE_MAILBOX hpnow = GetCycleCountFast(); - execCycles += hpnow - hpprev; + i64 currentExecCycles = hpnow - hpprev; + execCycles += currentExecCycles; hpprev = hpnow; execCount++; if (execCycles + nonExecCycles > 39000000) { // every 15 ms at 2.6GHz, so 1000 items is 15 sec (solomon interval) @@ -377,6 +378,11 @@ namespace NActors { nonExecCycles = 0; Ctx.UpdateThreadTime(); } + + if (!TlsThreadContext->IsEnoughCpu) { + Ctx.IncreaseNotEnoughCpuExecutions(); + TlsThreadContext->IsEnoughCpu = true; + } } } LWTRACK(ActivationEnd, Ctx.Orbit, Ctx.CpuId, Ctx.PoolId, Ctx.WorkerId); diff --git a/library/cpp/actors/core/harmonizer.cpp b/library/cpp/actors/core/harmonizer.cpp index e2fd0c5f24..a1838ee2b0 100644 --- a/library/cpp/actors/core/harmonizer.cpp +++ b/library/cpp/actors/core/harmonizer.cpp @@ -19,8 +19,8 @@ constexpr bool CheckBinaryPower(ui64 value) { return !(value & (value - 1)); } +template <ui8 HistoryBufferSize = 8> struct TValueHistory { - static constexpr ui64 HistoryBufferSize = 8; static_assert(CheckBinaryPower(HistoryBufferSize)); double History[HistoryBufferSize] = {0.0}; @@ -31,32 +31,87 @@ struct TValueHistory { ui64 AccumulatedTs = 0; template <bool WithTail=false> - double GetAvgPartForLastSeconds(ui8 seconds) { - double sum = AccumulatedUs; + double Accumulate(auto op, auto comb, ui8 seconds) { + double acc = AccumulatedUs; size_t idx = HistoryIdx; ui8 leftSeconds = seconds; + if constexpr (!WithTail) { + idx--; + leftSeconds--; + if (idx >= HistoryBufferSize) { + idx = HistoryBufferSize - 1; + } + acc = History[idx]; + } do { idx--; leftSeconds--; if (idx >= HistoryBufferSize) { idx = HistoryBufferSize - 1; } - if (WithTail || leftSeconds) { - sum += History[idx]; + if constexpr (WithTail) { + acc = op(acc, History[idx]); + } else if (leftSeconds) { + acc = op(acc, History[idx]); } else { ui64 tsInSecond = Us2Ts(1'000'000.0); - sum += History[idx] * (tsInSecond - AccumulatedTs) / tsInSecond; + acc = op(acc, History[idx] * (tsInSecond - AccumulatedTs) / tsInSecond); } } while (leftSeconds); - double duration = 1'000'000.0 * seconds + (WithTail ? Ts2Us(AccumulatedTs): 0.0); - double avg = sum / duration; - return avg; + double duration = 1'000'000.0 * seconds; + if constexpr (WithTail) { + duration += Ts2Us(AccumulatedTs); + } + return comb(acc, duration); + } + + template <bool WithTail=false> + double GetAvgPartForLastSeconds(ui8 seconds) { + auto sum = [](double acc, double value) { + return acc + value; + }; + auto avg = [](double sum, double duration) { + return sum / duration; + }; + return Accumulate<WithTail>(sum, avg, seconds); } double GetAvgPart() { return GetAvgPartForLastSeconds<true>(HistoryBufferSize); } + double GetMaxForLastSeconds(ui8 seconds) { + auto max = [](const double& acc, const double& value) { + return Max(acc, value); + }; + auto fst = [](const double& value, const double&) { return value; }; + return Accumulate<false>(max, fst, seconds); + } + + double GetMax() { + return GetMaxForLastSeconds(HistoryBufferSize); + } + + i64 GetMaxInt() { + return static_cast<i64>(GetMax()); + } + + double GetMinForLastSeconds(ui8 seconds) { + auto min = [](const double& acc, const double& value) { + return Min(acc, value); + }; + auto fst = [](const double& value, const double&) { return value; }; + return Accumulate<false>(min, fst, seconds); + } + + double GetMin() { + return GetMinForLastSeconds(HistoryBufferSize); + } + + i64 GetMinInt() { + return static_cast<i64>(GetMin()); + } + void Register(ui64 ts, double valueUs) { if (ts < LastTs) { LastTs = ts; @@ -101,8 +156,8 @@ struct TValueHistory { }; struct TThreadInfo { - TValueHistory Consumed; - TValueHistory Booked; + TValueHistory<8> Consumed; + TValueHistory<8> Booked; }; struct TPoolInfo { @@ -116,6 +171,8 @@ struct TPoolInfo { NMonitoring::TDynamicCounters::TCounterPtr AvgPingCounterWithSmallWindow; ui32 MaxAvgPingUs = 0; ui64 LastUpdateTs = 0; + ui64 NotEnoughCpuExecutions = 0; + ui64 NewNotEnoughCpuExecutions = 0; TAtomic LastFlags = 0; // 0 - isNeedy; 1 - isStarved; 2 - isHoggish TAtomic IncreasingThreadsByNeedyState = 0; @@ -123,12 +180,20 @@ struct TPoolInfo { TAtomic DecreasingThreadsByHoggishState = 0; TAtomic PotentialMaxThreadCount = 0; + TValueHistory<16> Consumed; + TValueHistory<16> Booked; + + TAtomic MaxConsumedCpu = 0; + TAtomic MinConsumedCpu = 0; + TAtomic MaxBookedCpu = 0; + TAtomic MinBookedCpu = 0; + bool IsBeingStopped(i16 threadIdx); double GetBooked(i16 threadIdx); double GetlastSecondPoolBooked(i16 threadIdx); double GetConsumed(i16 threadIdx); double GetlastSecondPoolConsumed(i16 threadIdx); - void PullStats(ui64 ts); + TCpuConsumption PullStats(ui64 ts); i16 GetThreadCount(); void SetThreadCount(i16 threadCount); bool IsAvgPingGood(); @@ -167,15 +232,26 @@ double TPoolInfo::GetlastSecondPoolConsumed(i16 threadIdx) { } #define UNROLL_HISTORY(history) (history)[0], (history)[1], (history)[2], (history)[3], (history)[4], (history)[5], (history)[6], (history)[7] -void TPoolInfo::PullStats(ui64 ts) { +TCpuConsumption TPoolInfo::PullStats(ui64 ts) { + TCpuConsumption acc; for (i16 threadIdx = 0; threadIdx < MaxThreadCount; ++threadIdx) { TThreadInfo &threadInfo = ThreadInfo[threadIdx]; TCpuConsumption cpuConsumption = Pool->GetThreadCpuConsumption(threadIdx); + acc.Add(cpuConsumption); threadInfo.Consumed.Register(ts, cpuConsumption.ConsumedUs); LWPROBE(SavedValues, Pool->PoolId, Pool->GetName(), "consumed", UNROLL_HISTORY(threadInfo.Consumed.History)); threadInfo.Booked.Register(ts, cpuConsumption.BookedUs); LWPROBE(SavedValues, Pool->PoolId, Pool->GetName(), "booked", UNROLL_HISTORY(threadInfo.Booked.History)); } + Consumed.Register(ts, acc.ConsumedUs); + RelaxedStore(&MaxConsumedCpu, Consumed.GetMaxInt()); + RelaxedStore(&MinConsumedCpu, Consumed.GetMinInt()); + Booked.Register(ts, acc.BookedUs); + RelaxedStore(&MaxBookedCpu, Booked.GetMaxInt()); + RelaxedStore(&MinBookedCpu, Booked.GetMinInt()); + NewNotEnoughCpuExecutions = acc.NotEnoughCpuExecutions - NotEnoughCpuExecutions; + NotEnoughCpuExecutions = acc.NotEnoughCpuExecutions; + return acc; } #undef UNROLL_HISTORY @@ -206,6 +282,14 @@ private: std::vector<TPoolInfo> Pools; std::vector<ui16> PriorityOrder; + TValueHistory<16> Consumed; + TValueHistory<16> Booked; + + TAtomic MaxConsumedCpu = 0; + TAtomic MinConsumedCpu = 0; + TAtomic MaxBookedCpu = 0; + TAtomic MinBookedCpu = 0; + void PullStats(ui64 ts); void HarmonizeImpl(ui64 ts); void CalculatePriorityOrder(); @@ -217,7 +301,8 @@ public: void DeclareEmergency(ui64 ts) override; void AddPool(IExecutorPool* pool, TSelfPingInfo *pingInfo) override; void Enable(bool enable) override; - TPoolHarmonizedStats GetPoolStats(i16 poolId) const override; + TPoolHarmonizerStats GetPoolStats(i16 poolId) const override; + THarmonizerStats GetStats() const override; }; THarmonizer::THarmonizer(ui64 ts) { @@ -232,13 +317,21 @@ double THarmonizer::Rescale(double value) const { } void THarmonizer::PullStats(ui64 ts) { + TCpuConsumption acc; for (TPoolInfo &pool : Pools) { - pool.PullStats(ts); + TCpuConsumption consumption = pool.PullStats(ts); + acc.Add(consumption); } + Consumed.Register(ts, acc.ConsumedUs); + RelaxedStore(&MaxConsumedCpu, Consumed.GetMaxInt()); + RelaxedStore(&MinConsumedCpu, Consumed.GetMinInt()); + Booked.Register(ts, acc.BookedUs); + RelaxedStore(&MaxBookedCpu, Booked.GetMaxInt()); + RelaxedStore(&MinBookedCpu, Booked.GetMinInt()); } Y_FORCE_INLINE bool IsStarved(double consumed, double booked) { - return consumed < booked * 0.7; + return Max(consumed, booked) > 0.1 && consumed < booked * 0.7; } Y_FORCE_INLINE bool IsHoggish(double booked, ui16 currentThreadCount) { @@ -273,7 +366,7 @@ void THarmonizer::HarmonizeImpl(ui64 ts) { isStarvedPresent = true; } ui32 currentThreadCount = pool.GetThreadCount(); - bool isNeedy = pool.IsAvgPingGood() && poolBooked >= currentThreadCount; + bool isNeedy = (pool.IsAvgPingGood() || pool.NewNotEnoughCpuExecutions) && poolBooked >= currentThreadCount; if (pool.AvgPingCounter) { if (pool.LastUpdateTs + Us2Ts(3'000'000ull) > ts) { isNeedy = false; @@ -304,6 +397,9 @@ void THarmonizer::HarmonizeImpl(ui64 ts) { AtomicSet(pool.PotentialMaxThreadCount, Min(pool.MaxThreadCount, budgetInt)); } double overbooked = consumed - booked; + if (overbooked < 0) { + isStarvedPresent = false; + } if (isStarvedPresent) { // last_starved_at_consumed_value = сумма по всем пулам consumed; // TODO(cthulhu): использовать как лимит планвно устремлять этот лимит к total, @@ -319,7 +415,7 @@ void THarmonizer::HarmonizeImpl(ui64 ts) { TPoolInfo &pool = Pools[poolIdx]; i64 threadCount = pool.GetThreadCount(); while (threadCount > pool.DefaultThreadCount) { - pool.SetThreadCount(threadCount - 1); + pool.SetThreadCount(--threadCount); AtomicIncrement(pool.DecreasingThreadsByStarvedState); overbooked--; LWPROBE(HarmonizeOperation, poolIdx, pool.Pool->GetName(), "decrease", threadCount - 1, pool.DefaultThreadCount, pool.MaxThreadCount); @@ -425,13 +521,17 @@ IHarmonizer* MakeHarmonizer(ui64 ts) { return new THarmonizer(ts); } -TPoolHarmonizedStats THarmonizer::GetPoolStats(i16 poolId) const { +TPoolHarmonizerStats THarmonizer::GetPoolStats(i16 poolId) const { const TPoolInfo &pool = Pools[poolId]; ui64 flags = RelaxedLoad(&pool.LastFlags); - return TPoolHarmonizedStats { + return TPoolHarmonizerStats{ .IncreasingThreadsByNeedyState = static_cast<ui64>(RelaxedLoad(&pool.IncreasingThreadsByNeedyState)), .DecreasingThreadsByStarvedState = static_cast<ui64>(RelaxedLoad(&pool.DecreasingThreadsByStarvedState)), .DecreasingThreadsByHoggishState = static_cast<ui64>(RelaxedLoad(&pool.DecreasingThreadsByHoggishState)), + .MaxConsumedCpu = static_cast<i64>(RelaxedLoad(&pool.MaxConsumedCpu)), + .MinConsumedCpu = static_cast<i64>(RelaxedLoad(&pool.MinConsumedCpu)), + .MaxBookedCpu = static_cast<i64>(RelaxedLoad(&pool.MaxBookedCpu)), + .MinBookedCpu = static_cast<i64>(RelaxedLoad(&pool.MinBookedCpu)), .PotentialMaxThreadCount = static_cast<i16>(RelaxedLoad(&pool.PotentialMaxThreadCount)), .IsNeedy = static_cast<bool>(flags & 1), .IsStarved = static_cast<bool>(flags & 2), @@ -439,4 +539,13 @@ TPoolHarmonizedStats THarmonizer::GetPoolStats(i16 poolId) const { }; } +THarmonizerStats THarmonizer::GetStats() const { + return THarmonizerStats{ + .MaxConsumedCpu = static_cast<i64>(RelaxedLoad(&MaxConsumedCpu)), + .MinConsumedCpu = static_cast<i64>(RelaxedLoad(&MinConsumedCpu)), + .MaxBookedCpu = static_cast<i64>(RelaxedLoad(&MaxBookedCpu)), + .MinBookedCpu = static_cast<i64>(RelaxedLoad(&MinBookedCpu)), + }; +} + } diff --git a/library/cpp/actors/core/harmonizer.h b/library/cpp/actors/core/harmonizer.h index bc6b938fe8..7c66ff54c6 100644 --- a/library/cpp/actors/core/harmonizer.h +++ b/library/cpp/actors/core/harmonizer.h @@ -6,16 +6,27 @@ namespace NActors { class IExecutorPool; - struct TPoolHarmonizedStats { + struct TPoolHarmonizerStats { ui64 IncreasingThreadsByNeedyState = 0; ui64 DecreasingThreadsByStarvedState = 0; ui64 DecreasingThreadsByHoggishState = 0; + i64 MaxConsumedCpu = 0.0; + i64 MinConsumedCpu = 0.0; + i64 MaxBookedCpu = 0.0; + i64 MinBookedCpu = 0.0; i16 PotentialMaxThreadCount = 0; bool IsNeedy = false; bool IsStarved = false; bool IsHoggish = false; }; + struct THarmonizerStats { + i64 MaxConsumedCpu = 0.0; + i64 MinConsumedCpu = 0.0; + i64 MaxBookedCpu = 0.0; + i64 MinBookedCpu = 0.0; + }; + // Pool cpu harmonizer class IHarmonizer { public: @@ -24,7 +35,8 @@ namespace NActors { virtual void DeclareEmergency(ui64 ts) = 0; virtual void AddPool(IExecutorPool* pool, TSelfPingInfo *pingInfo = nullptr) = 0; virtual void Enable(bool enable) = 0; - virtual TPoolHarmonizedStats GetPoolStats(i16 poolId) const = 0; + virtual TPoolHarmonizerStats GetPoolStats(i16 poolId) const = 0; + virtual THarmonizerStats GetStats() const = 0; }; IHarmonizer* MakeHarmonizer(ui64 ts); diff --git a/library/cpp/actors/core/mon_stats.h b/library/cpp/actors/core/mon_stats.h index 4c664a964a..5d61c9f87c 100644 --- a/library/cpp/actors/core/mon_stats.h +++ b/library/cpp/actors/core/mon_stats.h @@ -63,6 +63,10 @@ namespace NActors { ui64 IncreasingThreadsByNeedyState = 0; ui64 DecreasingThreadsByStarvedState = 0; ui64 DecreasingThreadsByHoggishState = 0; + i64 MaxConsumedCpuUs = 0; + i64 MinConsumedCpuUs = 0; + i64 MaxBookedCpuUs = 0; + i64 MinBookedCpuUs = 0; i16 WrongWakenedThreadCount = 0; i16 CurrentThreadCount = 0; i16 PotentialMaxThreadCount = 0; @@ -100,6 +104,7 @@ namespace NActors { ui64 MailboxPushedOutBySoftPreemption = 0; ui64 MailboxPushedOutByTime = 0; ui64 MailboxPushedOutByEventCount = 0; + ui64 NotEnoughCpuExecutions = 0; TExecutorThreadStats(size_t activityVecSize = 5) // must be not empty as 0 used as default : ElapsedTicksByActivity(activityVecSize) @@ -136,6 +141,7 @@ namespace NActors { MailboxPushedOutBySoftPreemption += RelaxedLoad(&other.MailboxPushedOutBySoftPreemption); MailboxPushedOutByTime += RelaxedLoad(&other.MailboxPushedOutByTime); MailboxPushedOutByEventCount += RelaxedLoad(&other.MailboxPushedOutByEventCount); + NotEnoughCpuExecutions += RelaxedLoad(&other.NotEnoughCpuExecutions); ActivationTimeHistogram.Aggregate(other.ActivationTimeHistogram); EventDeliveryTimeHistogram.Aggregate(other.EventDeliveryTimeHistogram); diff --git a/library/cpp/actors/core/worker_context.h b/library/cpp/actors/core/worker_context.h index c3a2947df1..cc8da2ff77 100644 --- a/library/cpp/actors/core/worker_context.h +++ b/library/cpp/actors/core/worker_context.h @@ -104,7 +104,7 @@ namespace NActors { i64 ts = deliveredTs > scheduleTs ? deliveredTs - scheduleTs : 0; double usec = NHPTimer::GetSeconds(ts) * 1000000.0; Stats->ActivationTimeHistogram.Add(usec); - Stats->WorstActivationTimeUs = Max(Stats->WorstActivationTimeUs, (ui64)usec); + RelaxedStore(&Stats->WorstActivationTimeUs, Max(Stats->WorstActivationTimeUs, (ui64)usec)); return usec; } @@ -140,6 +140,11 @@ namespace NActors { RelaxedStore(&WorkerStats.SafeElapsedTicks, (ui64)RelaxedLoad(&WorkerStats.ElapsedTicks)); RelaxedStore(&WorkerStats.CpuUs, ThreadCPUTime()); } + + void IncreaseNotEnoughCpuExecutions() { + RelaxedStore(&WorkerStats.NotEnoughCpuExecutions, + (ui64)RelaxedLoad(&WorkerStats.NotEnoughCpuExecutions) + 1); + } #else void GetCurrentStats(TExecutorThreadStats&) const {} inline void AddElapsedCycles(ui32, i64) {} @@ -159,6 +164,7 @@ namespace NActors { i64 AddEventProcessingStats(i64, i64, ui32, ui64) { return 0; } void UpdateActorsStats(size_t, IExecutorPool*) {} void UpdateThreadTime() {} + void IncreaseNotEnoughCpuExecutions() {} #endif void Switch(IExecutorPool* executor, diff --git a/library/cpp/actors/helpers/pool_stats_collector.h b/library/cpp/actors/helpers/pool_stats_collector.h index d80951827d..51ace0e3cc 100644 --- a/library/cpp/actors/helpers/pool_stats_collector.h +++ b/library/cpp/actors/helpers/pool_stats_collector.h @@ -135,6 +135,11 @@ private: NMonitoring::TDynamicCounters::TCounterPtr IncreasingThreadsByNeedyState; NMonitoring::TDynamicCounters::TCounterPtr DecreasingThreadsByStarvedState; NMonitoring::TDynamicCounters::TCounterPtr DecreasingThreadsByHoggishState; + NMonitoring::TDynamicCounters::TCounterPtr NotEnoughCpuExecutions; + NMonitoring::TDynamicCounters::TCounterPtr MaxConsumedCpu; + NMonitoring::TDynamicCounters::TCounterPtr MinConsumedCpu; + NMonitoring::TDynamicCounters::TCounterPtr MaxBookedCpu; + NMonitoring::TDynamicCounters::TCounterPtr MinBookedCpu; THistogramCounters LegacyActivationTimeHistogram; @@ -190,6 +195,11 @@ private: IncreasingThreadsByNeedyState = PoolGroup->GetCounter("IncreasingThreadsByNeedyState", true); DecreasingThreadsByStarvedState = PoolGroup->GetCounter("DecreasingThreadsByStarvedState", true); DecreasingThreadsByHoggishState = PoolGroup->GetCounter("DecreasingThreadsByHoggishState", true); + NotEnoughCpuExecutions = PoolGroup->GetCounter("NotEnoughCpuExecutions", true); + MaxConsumedCpu = PoolGroup->GetCounter("MaxConsumedCpuByPool", false); + MinConsumedCpu = PoolGroup->GetCounter("MinConsumedCpuByPool", false); + MaxBookedCpu = PoolGroup->GetCounter("MaxBookedCpuByPool", false); + MinBookedCpu = PoolGroup->GetCounter("MinBookedCpuByPool", false); LegacyActivationTimeHistogram.Init(PoolGroup.Get(), "ActivationTime", "usec", 5*1000*1000); ActivationTimeHistogram = PoolGroup->GetHistogram( @@ -237,6 +247,7 @@ private: *IncreasingThreadsByNeedyState = poolStats.IncreasingThreadsByNeedyState; *DecreasingThreadsByStarvedState = poolStats.DecreasingThreadsByStarvedState; *DecreasingThreadsByHoggishState = poolStats.DecreasingThreadsByHoggishState; + *NotEnoughCpuExecutions = stats.NotEnoughCpuExecutions; LegacyActivationTimeHistogram.Set(stats.ActivationTimeHistogram); ActivationTimeHistogram->Reset(); @@ -281,6 +292,38 @@ private: } }; + struct TActorSystemCounters { + TIntrusivePtr<NMonitoring::TDynamicCounters> Group; + + NMonitoring::TDynamicCounters::TCounterPtr MaxConsumedCpu; + NMonitoring::TDynamicCounters::TCounterPtr MinConsumedCpu; + NMonitoring::TDynamicCounters::TCounterPtr MaxBookedCpu; + NMonitoring::TDynamicCounters::TCounterPtr MinBookedCpu; + + void Init(NMonitoring::TDynamicCounters* group) { + Group = group; + + MaxConsumedCpu = Group->GetCounter("MaxConsumedCpu", false); + MinConsumedCpu = Group->GetCounter("MinConsumedCpu", false); + MaxBookedCpu = Group->GetCounter("MaxBookedCpu", false); + MinBookedCpu = Group->GetCounter("MinBookedCpu", false); + } + + void Set(const THarmonizerStats& harmonizerStats) { +#ifdef ACTORSLIB_COLLECT_EXEC_STATS + *MaxConsumedCpu = harmonizerStats.MaxConsumedCpu; + *MinConsumedCpu = harmonizerStats.MinConsumedCpu; + *MaxBookedCpu = harmonizerStats.MaxBookedCpu; + *MinBookedCpu = harmonizerStats.MinBookedCpu; +#else + Y_UNUSED(poolStats); + Y_UNUSED(stats); + Y_UNUSED(numThreads); +#endif + } + + }; + public: static constexpr IActor::EActivityType ActorActivityType() { return IActor::ACTORLIB_STATS; @@ -297,6 +340,7 @@ public: for (size_t poolId = 0; poolId < PoolCounters.size(); ++poolId) { PoolCounters[poolId].Init(Counters.Get(), setup.GetPoolName(poolId), setup.GetThreads(poolId)); } + ActorSystemCounters.Init(Counters.Get()); } void Bootstrap(const TActorContext& ctx) { @@ -322,6 +366,8 @@ private: ctx.ExecutorThread.ActorSystem->GetPoolStats(poolId, poolStats, stats); SetAggregatedCounters(PoolCounters[poolId], poolStats, stats); } + THarmonizerStats harmonizerStats = ctx.ExecutorThread.ActorSystem->GetHarmonizerStats(); + ActorSystemCounters.Set(harmonizerStats); OnWakeup(ctx); @@ -343,6 +389,7 @@ protected: NMonitoring::TDynamicCounterPtr Counters; TVector<TExecutorPoolCounters> PoolCounters; + TActorSystemCounters ActorSystemCounters; }; } // NActors diff --git a/library/cpp/actors/interconnect/poller_actor.cpp b/library/cpp/actors/interconnect/poller_actor.cpp index e75cbcaef4..04e17c24ab 100644 --- a/library/cpp/actors/interconnect/poller_actor.cpp +++ b/library/cpp/actors/interconnect/poller_actor.cpp @@ -146,8 +146,7 @@ namespace NActors { wrapper.Wait(); } - bool DrainReadEnd() { - size_t totalRead = 0; + void DrainReadEnd() { char buffer[4096]; for (;;) { ssize_t n = ReadEnd.Read(buffer, sizeof(buffer)); @@ -162,37 +161,38 @@ namespace NActors { } } else { Y_VERIFY(n); - totalRead += n; } } - return totalRead; } bool ProcessSyncOpQueue() { - if (DrainReadEnd()) { - Y_VERIFY(!SyncOperationsQ.IsEmpty()); - do { - TPollerSyncOperationWrapper *op = SyncOperationsQ.Top(); - if (auto *unregister = std::get_if<TPollerUnregisterSocket>(&op->Operation)) { - static_cast<TDerived&>(*this).UnregisterSocketInLoop(unregister->Socket); - op->SignalDone(); - } else if (std::get_if<TPollerExitThread>(&op->Operation)) { - op->SignalDone(); - return false; // terminate the thread - } else if (std::get_if<TPollerWakeup>(&op->Operation)) { - op->SignalDone(); - } else { - Y_FAIL(); - } - } while (SyncOperationsQ.Pop()); - } + Y_VERIFY(!SyncOperationsQ.IsEmpty()); + do { + TPollerSyncOperationWrapper *op = SyncOperationsQ.Top(); + if (auto *unregister = std::get_if<TPollerUnregisterSocket>(&op->Operation)) { + static_cast<TDerived&>(*this).UnregisterSocketInLoop(unregister->Socket); + op->SignalDone(); + } else if (std::get_if<TPollerExitThread>(&op->Operation)) { + op->SignalDone(); + return false; // terminate the thread + } else if (std::get_if<TPollerWakeup>(&op->Operation)) { + op->SignalDone(); + } else { + Y_FAIL(); + } + } while (SyncOperationsQ.Pop()); return true; } void *ThreadProc() override { SetCurrentThreadName("network poller"); - while (ProcessSyncOpQueue()) { - static_cast<TDerived&>(*this).ProcessEventsInLoop(); + for (;;) { + if (static_cast<TDerived&>(*this).ProcessEventsInLoop()) { // need to process the queue + DrainReadEnd(); + if (!ProcessSyncOpQueue()) { + break; + } + } } return nullptr; } diff --git a/library/cpp/actors/interconnect/poller_actor_darwin.h b/library/cpp/actors/interconnect/poller_actor_darwin.h index 4cb0a58f8d..31c1144794 100644 --- a/library/cpp/actors/interconnect/poller_actor_darwin.h +++ b/library/cpp/actors/interconnect/poller_actor_darwin.h @@ -45,18 +45,20 @@ namespace NActors { close(KqDescriptor); } - void ProcessEventsInLoop() { + bool ProcessEventsInLoop() { std::array<struct kevent, 256> events; int numReady = kevent(KqDescriptor, nullptr, 0, events.data(), events.size(), nullptr); if (numReady == -1) { if (errno == EINTR) { - return; + return false; } else { Y_FAIL("kevent() failed with %s", strerror(errno)); } } + bool res = false; + for (int i = 0; i < numReady; ++i) { const struct kevent& ev = events[i]; if (ev.udata) { @@ -65,8 +67,12 @@ namespace NActors { const bool read = error || ev.filter == EVFILT_READ; const bool write = error || ev.filter == EVFILT_WRITE; Notify(it, read, write); + } else { + res = true; } } + + return res; } void UnregisterSocketInLoop(const TIntrusivePtr<TSharedDescriptor>& socket) { diff --git a/library/cpp/actors/interconnect/poller_actor_linux.h b/library/cpp/actors/interconnect/poller_actor_linux.h index dd4f7c0124..6bd2cc258f 100644 --- a/library/cpp/actors/interconnect/poller_actor_linux.h +++ b/library/cpp/actors/interconnect/poller_actor_linux.h @@ -30,7 +30,7 @@ namespace NActors { close(EpollDescriptor); } - void ProcessEventsInLoop() { + bool ProcessEventsInLoop() { // preallocated array for events std::array<epoll_event, 256> events; @@ -42,12 +42,14 @@ namespace NActors { // check return status for any errors if (numReady == -1) { if (errno == EINTR) { - return; // restart the call a bit later + return false; // restart the call a bit later } else { Y_FAIL("epoll_wait() failed with %s", strerror(errno)); } } + bool res = false; + for (int i = 0; i < numReady; ++i) { const epoll_event& ev = events[i]; if (auto *record = static_cast<TSocketRecord*>(ev.data.ptr)) { @@ -73,8 +75,12 @@ namespace NActors { // issue notifications Notify(record, read, write); + } else { + res = true; } } + + return res; } void UnregisterSocketInLoop(const TIntrusivePtr<TSharedDescriptor>& socket) { @@ -110,5 +116,5 @@ namespace NActors { }; using TPollerThread = TEpollThread; - + } // namespace NActors diff --git a/library/cpp/actors/interconnect/poller_actor_win.h b/library/cpp/actors/interconnect/poller_actor_win.h index 4b4caa0ebd..e593cbafd1 100644 --- a/library/cpp/actors/interconnect/poller_actor_win.h +++ b/library/cpp/actors/interconnect/poller_actor_win.h @@ -23,7 +23,7 @@ namespace NActors { Stop(); } - void ProcessEventsInLoop() { + bool ProcessEventsInLoop() { fd_set readfds, writefds, exceptfds; FD_ZERO(&readfds); @@ -51,12 +51,14 @@ namespace NActors { if (res == -1) { const int err = LastSocketError(); if (err == EINTR) { - return; // try a bit later + return false; // try a bit later } else { Y_FAIL("select() failed with %s", strerror(err)); } } + bool flag = false; + with_lock (Mutex) { for (const auto& [fd, record] : Descriptors) { if (record) { @@ -70,9 +72,13 @@ namespace NActors { record->Flags &= ~WRITE; } Notify(record.Get(), read, write); + } else { + flag = true; } } } + + return flag; } void UnregisterSocketInLoop(const TIntrusivePtr<TSharedDescriptor>& socket) { diff --git a/library/cpp/yaml/CMakeLists.txt b/library/cpp/yaml/CMakeLists.txt index df58c83133..bf8d0de6d5 100644 --- a/library/cpp/yaml/CMakeLists.txt +++ b/library/cpp/yaml/CMakeLists.txt @@ -7,3 +7,4 @@ add_subdirectory(as) +add_subdirectory(fyamlcpp) diff --git a/library/cpp/yaml/fyamlcpp/CMakeLists.darwin.txt b/library/cpp/yaml/fyamlcpp/CMakeLists.darwin.txt new file mode 100644 index 0000000000..4db87e9d0d --- /dev/null +++ b/library/cpp/yaml/fyamlcpp/CMakeLists.darwin.txt @@ -0,0 +1,18 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yaml-fyamlcpp) +target_link_libraries(cpp-yaml-fyamlcpp PUBLIC + contrib-libs-cxxsupp + yutil + contrib-libs-libfyaml +) +target_sources(cpp-yaml-fyamlcpp PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp +) diff --git a/library/cpp/yaml/fyamlcpp/CMakeLists.linux-aarch64.txt b/library/cpp/yaml/fyamlcpp/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..d13ed8d7f1 --- /dev/null +++ b/library/cpp/yaml/fyamlcpp/CMakeLists.linux-aarch64.txt @@ -0,0 +1,19 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yaml-fyamlcpp) +target_link_libraries(cpp-yaml-fyamlcpp PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + contrib-libs-libfyaml +) +target_sources(cpp-yaml-fyamlcpp PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp +) diff --git a/library/cpp/yaml/fyamlcpp/CMakeLists.linux.txt b/library/cpp/yaml/fyamlcpp/CMakeLists.linux.txt new file mode 100644 index 0000000000..d13ed8d7f1 --- /dev/null +++ b/library/cpp/yaml/fyamlcpp/CMakeLists.linux.txt @@ -0,0 +1,19 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-yaml-fyamlcpp) +target_link_libraries(cpp-yaml-fyamlcpp PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + contrib-libs-libfyaml +) +target_sources(cpp-yaml-fyamlcpp PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp +) diff --git a/library/cpp/yaml/fyamlcpp/CMakeLists.txt b/library/cpp/yaml/fyamlcpp/CMakeLists.txt new file mode 100644 index 0000000000..3e0811fb22 --- /dev/null +++ b/library/cpp/yaml/fyamlcpp/CMakeLists.txt @@ -0,0 +1,15 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux-aarch64.txt) +elseif (APPLE) + include(CMakeLists.darwin.txt) +elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux.txt) +endif() diff --git a/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp b/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp new file mode 100644 index 0000000000..b8d8b17596 --- /dev/null +++ b/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp @@ -0,0 +1,969 @@ +#include "fyamlcpp.h" + +#include <contrib/libs/libfyaml/include/libfyaml.h> + +#include <util/digest/murmur.h> + +namespace NFyaml { + +#define ENSURE_NODE_NOT_EMPTY(NODE) Y_ENSURE_EX(NODE, TFyamlEx() << "Expected non-empty Node") +#define ENSURE_DOCUMENT_NOT_EMPTY(NODE) Y_ENSURE_EX(NODE, TFyamlEx() << "Expected non-empty Document") + +const char* zstr = ""; + +enum class EErrorType { + Debug = FYET_DEBUG, + Info = FYET_INFO, + Notice = FYET_NOTICE, + Warning = FYET_WARNING, + Error = FYET_ERROR, + Max = FYET_MAX, +}; + +enum class EErrorModule { + Unknown = FYEM_UNKNOWN, + Atom = FYEM_ATOM, + Scan = FYEM_SCAN, + Parse = FYEM_PARSE, + Doc = FYEM_DOC, + Build = FYEM_BUILD, + Internal = FYEM_INTERNAL, + System = FYEM_SYSTEM, + Max = FYEM_MAX, +}; + +enum class EParseCfgFlags { + Quiet = FYPCF_QUIET, + CollectDiag = FYPCF_COLLECT_DIAG, + ResolveDocument = FYPCF_RESOLVE_DOCUMENT, + DisableMmapOpt = FYPCF_DISABLE_MMAP_OPT, + DisableRecycling = FYPCF_DISABLE_RECYCLING, + ParseComments = FYPCF_PARSE_COMMENTS, + DisableDepth_limit = FYPCF_DISABLE_DEPTH_LIMIT, + DisableAccelerators = FYPCF_DISABLE_ACCELERATORS, + DisableBuffering = FYPCF_DISABLE_BUFFERING, + DefaultVersionAuto = FYPCF_DEFAULT_VERSION_AUTO, + DefaultVersion1_1 = FYPCF_DEFAULT_VERSION_1_1, + DefaultVersion1_2 = FYPCF_DEFAULT_VERSION_1_2, + DefaultVersion1_3 = FYPCF_DEFAULT_VERSION_1_3, + SloppyFlowIndentation = FYPCF_SLOPPY_FLOW_INDENTATION, + PreferRecursive = FYPCF_PREFER_RECURSIVE, + JsonAuto = FYPCF_JSON_AUTO, + JsonNone = FYPCF_JSON_NONE, + JsonForce = FYPCF_JSON_FORCE, + YpathAliases = FYPCF_YPATH_ALIASES, + AllowDuplicateKeys = FYPCF_ALLOW_DUPLICATE_KEYS, +}; + +enum class EEventType { + None = FYET_NONE, + StreamStart = FYET_STREAM_START, + StreamEnd = FYET_STREAM_END, + DocumentStart = FYET_DOCUMENT_START, + DocumentEnd = FYET_DOCUMENT_END, + MappingStart = FYET_MAPPING_START, + MappingEnd = FYET_MAPPING_END, + SequenceStart = FYET_SEQUENCE_START, + SequenceEnd = FYET_SEQUENCE_END, + Scalar = FYET_SCALAR, + Alias = FYET_ALIAS, +}; + +enum class EScalarStyle { + Any = FYSS_ANY, + Plain = FYSS_PLAIN, + SingleQuoted = FYSS_SINGLE_QUOTED, + DoubleQuoted = FYSS_DOUBLE_QUOTED, + Literal = FYSS_LITERAL, + Folded = FYSS_FOLDED, + Max = FYSS_MAX, +}; + +enum class EEmitterWriteType { + DocumentIndicator = fyewt_document_indicator, + TagDirective = fyewt_tag_directive, + VersionDirective = fyewt_version_directive, + Indent = fyewt_indent, + Indicator = fyewt_indicator, + Whitespace = fyewt_whitespace, + PlainScalar = fyewt_plain_scalar, + SingleQuotedScalar = fyewt_single_quoted_scalar, + DoubleQuotedScalar = fyewt_double_quoted_scalar, + LiteralScalar = fyewt_literal_scalar, + FoldedScalar = fyewt_folded_scalar, + Anchor = fyewt_anchor, + Tag = fyewt_tag, + Linebreak = fyewt_linebreak, + Alias = fyewt_alias, + TerminatingZero = fyewt_terminating_zero, + PlainScalarKey = fyewt_plain_scalar_key, + SingleQuotedScalarKey = fyewt_single_quoted_scalar_key, + DoubleQuotedScalarKey = fyewt_double_quoted_scalar_key, + Comment = fyewt_comment, +}; + +enum class ECommentPlacement { + Top = fycp_top, + Right = fycp_right, + Bottom = fycp_bottom, +}; + +enum EEmitterCfgFlags { + SortKeys = FYECF_SORT_KEYS, + OutputComments = FYECF_OUTPUT_COMMENTS, + StripLabels = FYECF_STRIP_LABELS, + StripTags = FYECF_STRIP_TAGS, + StripDoc = FYECF_STRIP_DOC, + NoEndingNewline = FYECF_NO_ENDING_NEWLINE, + StripEmptyKv = FYECF_STRIP_EMPTY_KV, + IndentDefault = FYECF_INDENT_DEFAULT, + Indent1 = FYECF_INDENT_1, + Indent2 = FYECF_INDENT_2, + Indent3 = FYECF_INDENT_3, + Indent4 = FYECF_INDENT_4, + Indent5 = FYECF_INDENT_5, + Indent6 = FYECF_INDENT_6, + Indent7 = FYECF_INDENT_7, + Indent8 = FYECF_INDENT_8, + Indent9 = FYECF_INDENT_9, + WidthDefault = FYECF_WIDTH_DEFAULT, + Width80 = FYECF_WIDTH_80, + Width132 = FYECF_WIDTH_132, + WidthInf = FYECF_WIDTH_INF, + ModeOriginal = FYECF_MODE_ORIGINAL, + ModeBlock = FYECF_MODE_BLOCK, + ModeFlow = FYECF_MODE_FLOW, + ModeFlowOneline = FYECF_MODE_FLOW_ONELINE, + ModeJson = FYECF_MODE_JSON, + ModeJsonTp = FYECF_MODE_JSON_TP, + ModeJsonOneline = FYECF_MODE_JSON_ONELINE, + ModeDejson = FYECF_MODE_DEJSON, + ModePretty = FYECF_MODE_PRETTY, + DocStartMarkAuto = FYECF_DOC_START_MARK_AUTO, + DocStartMarkOff = FYECF_DOC_START_MARK_OFF, + DocStartMarkOn = FYECF_DOC_START_MARK_ON, + DocEndMarkAuto = FYECF_DOC_END_MARK_AUTO, + DocEndMarkOff = FYECF_DOC_END_MARK_OFF, + DocEndMarkOn = FYECF_DOC_END_MARK_ON, + VersionDirAuto = FYECF_VERSION_DIR_AUTO, + VersionDirOff = FYECF_VERSION_DIR_OFF, + VersionDirOn = FYECF_VERSION_DIR_ON, + TagDirAuto = FYECF_TAG_DIR_AUTO, + TagDirOff = FYECF_TAG_DIR_OFF, + TagDirOn = FYECF_TAG_DIR_ON, + + Default = FYECF_DEFAULT, +}; + +enum class ENodeStyle { + Any = FYNS_ANY, + Flow = FYNS_FLOW, + Block = FYNS_BLOCK, + Plain = FYNS_PLAIN, + SingleQuoted = FYNS_SINGLE_QUOTED, + DoubleQuoted = FYNS_DOUBLE_QUOTED, + Literal = FYNS_LITERAL, + Folded = FYNS_FOLDED, + Alias = FYNS_ALIAS, +}; + +enum class ENodeWalkFlags { + DontFollow = FYNWF_DONT_FOLLOW, + Follow = FYNWF_FOLLOW, + PtrYaml = FYNWF_PTR_YAML, + PtrJson = FYNWF_PTR_JSON, + PtrReljson = FYNWF_PTR_RELJSON, + PtrYpath = FYNWF_PTR_YPATH, + UriEncoded = FYNWF_URI_ENCODED, + MaxdepthDefault = FYNWF_MAXDEPTH_DEFAULT, + MarkerDefault = FYNWF_MARKER_DEFAULT, + PtrDefault = FYNWF_PTR_DEFAULT, +}; + +enum class EPathParseCfgFlags { + Quiet = FYPPCF_QUIET, + DisableRecycling = FYPPCF_DISABLE_RECYCLING, + DisableAccelerators = FYPPCF_DISABLE_ACCELERATORS, +}; + +enum class EPathExecCfgFlags { + Quiet = FYPXCF_QUIET, + DisableRecycling = FYPXCF_DISABLE_RECYCLING, + DisableAccelerators = FYPXCF_DISABLE_ACCELERATORS, +}; + +enum class ETokenType { + /* non-content token types */ + None = FYTT_NONE, + StreamStart = FYTT_STREAM_START, + StreamEnd = FYTT_STREAM_END, + VersionDirective = FYTT_VERSION_DIRECTIVE, + TagDirective = FYTT_TAG_DIRECTIVE, + DocumentStart = FYTT_DOCUMENT_START, + DocumentEnd = FYTT_DOCUMENT_END, + /* content token types */ + BlockSequenceStart = FYTT_BLOCK_SEQUENCE_START, + BlockMappingStart = FYTT_BLOCK_MAPPING_START, + BlockEnd = FYTT_BLOCK_END, + FlowSequenceStart = FYTT_FLOW_SEQUENCE_START, + FlowSequenceEnd = FYTT_FLOW_SEQUENCE_END, + FlowMappingStart = FYTT_FLOW_MAPPING_START, + FlowMappingEnd = FYTT_FLOW_MAPPING_END, + BlockEntry = FYTT_BLOCK_ENTRY, + FlowEntry = FYTT_FLOW_ENTRY, + Key = FYTT_KEY, + Value = FYTT_VALUE, + Alias = FYTT_ALIAS, + Anchor = FYTT_ANCHOR, + Tag = FYTT_TAG, + Scalar = FYTT_SCALAR, + + /* special error reporting */ + Input_marker = FYTT_INPUT_MARKER, + + /* path expression tokens */ + PeSlash = FYTT_PE_SLASH, + PeRoot = FYTT_PE_ROOT, + PeThis = FYTT_PE_THIS, + PeParent = FYTT_PE_PARENT, + PeMapKey = FYTT_PE_MAP_KEY, + PeSeqIndex = FYTT_PE_SEQ_INDEX, + PeSeqSlice = FYTT_PE_SEQ_SLICE, + PeScalarFilter = FYTT_PE_SCALAR_FILTER, + PeCollectionFilter = FYTT_PE_COLLECTION_FILTER, + PeSeqFilter = FYTT_PE_SEQ_FILTER, + PeMapFilter = FYTT_PE_MAP_FILTER, + PeUniqueFilter = FYTT_PE_UNIQUE_FILTER, + PeEveryChild = FYTT_PE_EVERY_CHILD, + PeEveryChildR = FYTT_PE_EVERY_CHILD_R, + PeAlias = FYTT_PE_ALIAS, + PeSibling = FYTT_PE_SIBLING, + PeComma = FYTT_PE_COMMA, + PeBarbar = FYTT_PE_BARBAR, + PeAmpamp = FYTT_PE_AMPAMP, + PeLparen = FYTT_PE_LPAREN, + PeRparen = FYTT_PE_RPAREN, + + /* comparison operators */ + PeEqeq = FYTT_PE_EQEQ, + PeNoteq = FYTT_PE_NOTEQ, + PeLt = FYTT_PE_LT, + PeGt = FYTT_PE_GT, + PeLte = FYTT_PE_LTE, + PeGte = FYTT_PE_GTE, + + /* scalar expression tokens */ + SePlus = FYTT_SE_PLUS, + SeMinus = FYTT_SE_MINUS, + SeMult = FYTT_SE_MULT, + SeDiv = FYTT_SE_DIV, + + PeMethod = FYTT_PE_METHOD, + SeMethod = FYTT_SE_METHOD, +}; + +enum class EComposerReturn { + OkContinue = FYCR_OK_CONTINUE, + OkStop = FYCR_OK_STOP, + OkStartSkip = FYCR_OK_START_SKIP, + OkStopSkip = FYCR_OK_STOP_SKIP, + Error = FYCR_ERROR, +}; + +TDocumentIterator::TDocumentIterator(fy_document_iterator* iterator) + : Iterator_(iterator, fy_document_iterator_destroy) +{} + +TNode TNodeRef::CreateReference() const { + ENSURE_NODE_NOT_EMPTY(Node_); + return TNode(fy_node_create_reference(Node_)); +} + +TNode TNodeRef::Copy() const { + ENSURE_NODE_NOT_EMPTY(Node_); + return TNode(fy_node_copy(fy_node_document(Node_), Node_)); +} + +TNode TNodeRef::Copy(TDocument& to) const { + ENSURE_NODE_NOT_EMPTY(Node_); + auto* fromDoc = fy_node_document(Node_); + auto& fromUserdata = *reinterpret_cast<THashSet<TSimpleSharedPtr<TString>, TStringPtrHashT>*>(fy_document_get_userdata(fromDoc)); + auto& toUserdata = *reinterpret_cast<THashSet<TSimpleSharedPtr<TString>, TStringPtrHashT>*>(fy_document_get_userdata(to.Document_.get())); + toUserdata.insert(fromUserdata.begin(), fromUserdata.end()); + return TNode(fy_node_copy(to.Document_.get(), Node_)); +} + +TString TNodeRef::Path() const { + ENSURE_NODE_NOT_EMPTY(Node_); + char* path = fy_node_get_path(Node_); + + if (path) { + TString str(path); + free(path); + return str; + } + + return {}; +} + +ENodeType TNodeRef::Type() const { + ENSURE_NODE_NOT_EMPTY(Node_); + return static_cast<ENodeType>(fy_node_get_type(Node_)); +} + +bool TNodeRef::IsAlias() const { + ENSURE_NODE_NOT_EMPTY(Node_); + return fy_node_is_alias(Node_); +} + +TNodeRef TNodeRef::ResolveAlias() const { + ENSURE_NODE_NOT_EMPTY(Node_); + Y_VERIFY_DEBUG(IsAlias()); + return TNodeRef(fy_node_resolve_alias(Node_)); +} + +TString TNodeRef::Scalar() const { + ENSURE_NODE_NOT_EMPTY(Node_); + Y_ENSURE_EX(fy_node_is_scalar(Node_), TFyamlEx() << "Node is not Scalar: " << Path()); + size_t size; + const char* text = fy_node_get_scalar(Node_, &size); + return TString(text, size); +} + +TMapping TNodeRef::Map() const { + ENSURE_NODE_NOT_EMPTY(Node_); + Y_ENSURE_EX(fy_node_is_mapping(Node_), TFyamlEx() << "Node is not Mapping: " << Path()); + return TMapping(*this); +} + +TSequence TNodeRef::Sequence() const { + ENSURE_NODE_NOT_EMPTY(Node_); + Y_ENSURE_EX(fy_node_is_sequence(Node_), TFyamlEx() << "Node is not Sequence: " << Path()); + return TSequence(*this); +} + +void TNodeRef::Insert(const TNodeRef& node) { + ENSURE_NODE_NOT_EMPTY(Node_); + NDetail::RethrowOnError(fy_node_insert(Node_, node.Node_), Node_); +} + +std::optional<TString> TNodeRef::Tag() const { + ENSURE_NODE_NOT_EMPTY(Node_); + size_t len = 0; + const char* tag = fy_node_get_tag(Node_, &len); + + if (tag) { + return TString(tag, len); + } + + return std::nullopt; +} + +void TNodeRef::SetTag(const TString& tag) { + ENSURE_NODE_NOT_EMPTY(Node_); + auto* str = new TString(std::move(tag)); + auto* data = new NDetail::TUserDataHolder(UserData(), str); + SetUserData(data); + NDetail::RethrowOnError(fy_node_set_tag(Node_, str->c_str(), str->length()), Node_); +} + +bool TNodeRef::RemoveTag() { + ENSURE_NODE_NOT_EMPTY(Node_); + bool ret = fy_node_remove_tag(Node_); + ClearUserData(); + return ret; +} + +bool TNodeRef::HasAnchor() const { + ENSURE_NODE_NOT_EMPTY(Node_); + return fy_node_get_anchor(Node_) != nullptr; +} + +void TNodeRef::SetAnchor(const TString& anchor) { + auto* str = new TString(anchor); + auto* data = new NDetail::TUserDataHolder(UserData(), str); + SetUserData(data); + NDetail::RethrowOnError(fy_node_set_anchor(Node_, str->c_str(), str->length()), Node_); +} + +bool TNodeRef::DeepEqual(const TNodeRef& other) { + ENSURE_NODE_NOT_EMPTY(Node_); + ENSURE_NODE_NOT_EMPTY(other.Node_); + return fy_node_compare(Node_, other.Node_); +} + +std::unique_ptr<char, void(*)(char*)> TNodeRef::EmitToCharArray() const { + std::unique_ptr<char, void(*)(char*)> res( + fy_emit_node_to_string( + Node_, + (fy_emitter_cfg_flags)(FYECF_DEFAULT)), &NDetail::FreeChar); + return res; +} + +void TNodeRef::SetUserData(NDetail::IBasicUserData* data) { + ENSURE_NODE_NOT_EMPTY(Node_); + fy_node_set_meta(Node_, data); +} + +NDetail::IBasicUserData* TNodeRef::UserData() const { + ENSURE_NODE_NOT_EMPTY(Node_); + return reinterpret_cast<NDetail::IBasicUserData* >(fy_node_get_meta(Node_)); +} + +void TNodeRef::ClearUserData() { + ENSURE_NODE_NOT_EMPTY(Node_); + fy_node_clear_meta(Node_); +} + +TNode& TNode::operator=(fy_node* node) { + Node_.reset(node, fy_node_free); + return *this; +} + +TNode::TNode(fy_node* node) + : Node_(node, fy_node_free) +{} + +TNodeRef TNodePairRef::Key() const { + ENSURE_NODE_NOT_EMPTY(Pair_); + return TNodeRef(fy_node_pair_key(Pair_)); +} + +void TNodePairRef::SetKey(const TNodeRef& node) { + ENSURE_NODE_NOT_EMPTY(Pair_); + ENSURE_NODE_NOT_EMPTY(node); + NDetail::RethrowOnError(fy_node_pair_set_key(Pair_, node.Node_), Pair_); +} + +TNodeRef TNodePairRef::Value() const { + ENSURE_NODE_NOT_EMPTY(Pair_); + return TNodeRef(fy_node_pair_value(Pair_)); +} + +void TNodePairRef::SetValue(const TNodeRef& node) { + ENSURE_NODE_NOT_EMPTY(Pair_); + ENSURE_NODE_NOT_EMPTY(node); + NDetail::RethrowOnError(fy_node_pair_set_value(Pair_, node.Node_), Pair_); +} + +TMappingIterator::TMappingIterator(const TNodeRef& node, bool end) + : Node_(node) +{ + if (!end) { + NodePair_ = TNodePairRef(fy_node_mapping_iterate(Node_.Node_, reinterpret_cast<void**>(&NodePair_.Pair_))); + } +} + +TMappingIterator& TMappingIterator::operator++() { + NodePair_ = TNodePairRef(fy_node_mapping_iterate(Node_.Node_, reinterpret_cast<void**>(&NodePair_.Pair_))); + return *this; +} + +TReverseMappingIterator::TReverseMappingIterator(const TNodeRef& node, bool end) + : Node_(node) +{ + if (!end) { + NodePair_ = TNodePairRef(fy_node_mapping_reverse_iterate(Node_.Node_, reinterpret_cast<void**>(&NodePair_.Pair_))); + } +} + +TReverseMappingIterator& TReverseMappingIterator::operator++() { + NodePair_ = TNodePairRef(fy_node_mapping_reverse_iterate(Node_.Node_, reinterpret_cast<void**>(&NodePair_.Pair_))); + return *this; +} + +size_t TMapping::size() const { + ENSURE_NODE_NOT_EMPTY(Node_); + return fy_node_mapping_item_count(Node_); +} + +size_t TMapping::empty() const { + ENSURE_NODE_NOT_EMPTY(Node_); + return fy_node_mapping_is_empty(Node_); +} + +TNodePairRef TMapping::at(int index) const { + ENSURE_NODE_NOT_EMPTY(Node_); + auto res = fy_node_mapping_get_by_index(Node_, index); + Y_ENSURE_EX(res, TFyamlEx() << "No such child: " << Path() << "/" << index); + return TNodePairRef(res); +} + +TNodePairRef TMapping::operator[](int index) const { + ENSURE_NODE_NOT_EMPTY(Node_); + return TNodePairRef(fy_node_mapping_get_by_index(Node_, index)); +} + +TNodeRef TMapping::at(const TString& index) const { + ENSURE_NODE_NOT_EMPTY(Node_); + auto res = fy_node_mapping_lookup_by_string(Node_, index.data(), index.size()); + Y_ENSURE_EX(res, TFyamlEx() << "No such child: " << Path() << "/" << index); + return TNodeRef(res); +} + +TNodePairRef TMapping::pair_at(const TString& index) const { + ENSURE_NODE_NOT_EMPTY(Node_); + auto res = fy_node_mapping_lookup_pair_by_string(Node_, index.data(), index.size()); + Y_ENSURE_EX(res, TFyamlEx() << "No such child: " << Path() << "/" << index); + return TNodePairRef(res); +} + +TNodePairRef TMapping::pair_at_opt(const TString& index) const { + ENSURE_NODE_NOT_EMPTY(Node_); + return TNodePairRef(fy_node_mapping_lookup_pair_by_string(Node_, index.data(), index.size())); +} + +TNodeRef TMapping::operator[](const TString& index) const { + ENSURE_NODE_NOT_EMPTY(Node_); + return TNodeRef(fy_node_mapping_lookup_by_string(Node_, index.data(), index.size())); +} + +TNodeRef TMapping::operator[](const char* str) const { + ENSURE_NODE_NOT_EMPTY(Node_); + TString index(str); + return TNodeRef(fy_node_mapping_lookup_by_string(Node_, index.data(), index.size())); +} + +void TMapping::Append(const TNodeRef& key, const TNodeRef& value) { + ENSURE_NODE_NOT_EMPTY(Node_); + ENSURE_NODE_NOT_EMPTY(key); + ENSURE_NODE_NOT_EMPTY(value); + NDetail::RethrowOnError(fy_node_mapping_append(Node_, key.Node_, value.Node_), Node_); +} + +void TMapping::Prepend(const TNodeRef& key, const TNodeRef& value) { + ENSURE_NODE_NOT_EMPTY(Node_); + ENSURE_NODE_NOT_EMPTY(key); + ENSURE_NODE_NOT_EMPTY(value); + NDetail::RethrowOnError(fy_node_mapping_prepend(Node_, key.Node_, value.Node_), Node_); +} + +void TMapping::Remove(const TNodePairRef& toRemove) { + ENSURE_NODE_NOT_EMPTY(Node_); + ENSURE_NODE_NOT_EMPTY(toRemove); + NDetail::RethrowOnError(fy_node_mapping_remove(Node_, toRemove.Pair_), Node_); +} + +TMappingIterator TMapping::Remove(const TMappingIterator& toRemove) { + ENSURE_NODE_NOT_EMPTY(Node_); + Y_VERIFY_DEBUG(Node_ == toRemove.Node_); + TMappingIterator ret = toRemove; + ++ret; + fy_node_mapping_remove(Node_, toRemove.NodePair_.Pair_); + return ret; +} + +void TMapping::Remove(const TNodeRef& key) { + ENSURE_NODE_NOT_EMPTY(Node_); + fy_node_free(fy_node_mapping_remove_by_key(Node_, key.Node_)); +} + +TSequenceIterator::TSequenceIterator(const TNodeRef& node, bool end) + : Node_(node) +{ + if (!end) { + IterNode_ = TNodeRef(fy_node_sequence_iterate(Node_.Node_, &Iter_)); + } +} + +TSequenceIterator& TSequenceIterator::operator++() { + IterNode_ = TNodeRef(fy_node_sequence_iterate(Node_.Node_, &Iter_)); + return *this; +} + +void TSequenceIterator::InsertBefore(const TNodeRef& node) { + ENSURE_NODE_NOT_EMPTY(Node_); + ENSURE_NODE_NOT_EMPTY(IterNode_); + ENSURE_NODE_NOT_EMPTY(node); + NDetail::RethrowOnError(fy_node_sequence_insert_before(Node_.Node_, IterNode_.Node_, node.Node_), Node_.Node_); +} + +void TSequenceIterator::InsertAfter(const TNodeRef& node) { + ENSURE_NODE_NOT_EMPTY(Node_); + ENSURE_NODE_NOT_EMPTY(IterNode_); + ENSURE_NODE_NOT_EMPTY(node); + NDetail::RethrowOnError(fy_node_sequence_insert_after(Node_.Node_, IterNode_.Node_, node.Node_), Node_.Node_); +} + +TReverseSequenceIterator::TReverseSequenceIterator(const TNodeRef& node, bool end) + : Node_(node) +{ + if (!end) { + IterNode_ = TNodeRef(fy_node_sequence_reverse_iterate(Node_.Node_, &Iter_)); + } +} + +TReverseSequenceIterator& TReverseSequenceIterator::operator++() { + IterNode_ = TNodeRef(fy_node_sequence_reverse_iterate(Node_.Node_, &Iter_)); + return *this; +} + +void TReverseSequenceIterator::InsertBefore(const TNodeRef& node) { + ENSURE_NODE_NOT_EMPTY(Node_); + ENSURE_NODE_NOT_EMPTY(IterNode_); + ENSURE_NODE_NOT_EMPTY(node); + NDetail::RethrowOnError(fy_node_sequence_insert_after(Node_.Node_, IterNode_.Node_, node.Node_), Node_.Node_); +} + +void TReverseSequenceIterator::InsertAfter(const TNodeRef& node) { + ENSURE_NODE_NOT_EMPTY(Node_); + ENSURE_NODE_NOT_EMPTY(IterNode_); + ENSURE_NODE_NOT_EMPTY(node); + NDetail::RethrowOnError(fy_node_sequence_insert_before(Node_.Node_, IterNode_.Node_, node.Node_), Node_.Node_); +} + +size_t TSequence::size() const { + ENSURE_NODE_NOT_EMPTY(Node_); + return fy_node_sequence_item_count(Node_); +} + +size_t TSequence::empty() const { + ENSURE_NODE_NOT_EMPTY(Node_); + return fy_node_sequence_is_empty(Node_); +} + +TNodeRef TSequence::at(int index) const { + ENSURE_NODE_NOT_EMPTY(Node_); + auto res = fy_node_sequence_get_by_index(Node_, index); + Y_ENSURE_EX(res, TFyamlEx() << "No such index: " << Path() << "/" << index); + return TNodeRef(res); +} + +TNodeRef TSequence::operator[](int index) const { + ENSURE_NODE_NOT_EMPTY(Node_); + return TNodeRef(fy_node_sequence_get_by_index(Node_, index)); +} + +void TSequence::Append(const TNodeRef& node) { + ENSURE_NODE_NOT_EMPTY(Node_); + ENSURE_NODE_NOT_EMPTY(node); + NDetail::RethrowOnError(fy_node_sequence_append(Node_, node.Node_), Node_); +} + +void TSequence::Prepend(const TNodeRef& node) { + ENSURE_NODE_NOT_EMPTY(Node_); + ENSURE_NODE_NOT_EMPTY(node); + NDetail::RethrowOnError(fy_node_sequence_prepend(Node_, node.Node_), Node_); +} + +void TSequence::InsertBefore(const TNodeRef& mark, const TNodeRef& node) { + ENSURE_NODE_NOT_EMPTY(Node_); + ENSURE_NODE_NOT_EMPTY(mark); + ENSURE_NODE_NOT_EMPTY(node); + NDetail::RethrowOnError(fy_node_sequence_insert_before(Node_, mark.Node_, node.Node_), Node_); +} + +void TSequence::InsertAfter(const TNodeRef& mark, const TNodeRef& node) { + ENSURE_NODE_NOT_EMPTY(Node_); + ENSURE_NODE_NOT_EMPTY(mark); + ENSURE_NODE_NOT_EMPTY(node); + NDetail::RethrowOnError(fy_node_sequence_insert_after(Node_, mark.Node_, node.Node_), Node_); +} + +TNode TSequence::Remove(const TNodeRef& toRemove) { + ENSURE_NODE_NOT_EMPTY(Node_); + ENSURE_NODE_NOT_EMPTY(toRemove.Node_); + return TNode(fy_node_sequence_remove(Node_, toRemove.Node_)); +} + +TSequenceIterator TSequence::Remove(const TSequenceIterator& toRemove) { + ENSURE_NODE_NOT_EMPTY(Node_); + Y_VERIFY_DEBUG(Node_ == toRemove.Node_); + ENSURE_NODE_NOT_EMPTY(toRemove.IterNode_); + TSequenceIterator ret = toRemove; + ++ret; + fy_node_sequence_remove(Node_, toRemove.IterNode_.Node_); + fy_node_free(toRemove.IterNode_.Node_); // TODO add extract + return ret; +} + +TReverseSequenceIterator TSequence::Remove(const TReverseSequenceIterator& toRemove) { + ENSURE_NODE_NOT_EMPTY(Node_); + Y_VERIFY_DEBUG(Node_ == toRemove.Node_); + ENSURE_NODE_NOT_EMPTY(toRemove.IterNode_); + TReverseSequenceIterator ret = toRemove; + ++ret; + fy_node_sequence_remove(Node_, toRemove.IterNode_.Node_); + fy_node_free(toRemove.IterNode_.Node_); // TODO add extract + return ret; +} + +TDocumentNodeIterator::TDocumentNodeIterator(TNodeRef&& node) + : Node_(node) +{ + if (node) { + Iterator_ = {fy_document_iterator_create(), fy_document_iterator_destroy}; + fy_document_iterator_node_start(Iterator_.get(), node.Node_); + } +} + +TDocumentNodeIterator& TDocumentNodeIterator::operator++() { + Node_ = fy_document_iterator_node_next(Iterator_.get()); + return *this; +} + +TDocument::TDocument(TString str, fy_document* doc, fy_diag* diag) + : Document_(doc, fy_document_destroy) + , Diag_(diag, fy_diag_destroy) +{ + auto* userdata = new THashSet<TSimpleSharedPtr<TString>, TStringPtrHashT>({MakeSimpleShared<TString>(std::move(str))}); + fy_document_set_userdata(doc, userdata); + fy_document_register_on_destroy(doc, &DestroyDocumentStrings); + RegisterUserDataCleanup(); +} + +TDocument::TDocument(fy_document* doc, fy_diag* diag) + : Document_(doc, fy_document_destroy) + , Diag_(diag, fy_diag_destroy) +{ + RegisterUserDataCleanup(); +} + + +TDocument TDocument::Parse(TString str) { + const char* cstr = str.empty() ? zstr : str.cbegin(); + fy_diag_cfg dcfg; + fy_diag_cfg_default(&dcfg); + std::unique_ptr<fy_diag, void(*)(fy_diag*)> diag(fy_diag_create(&dcfg), fy_diag_destroy); + fy_diag_set_collect_errors(diag.get(), true); + fy_parse_cfg cfg{ + "", + // FYPCF_PARSE_COMMENTS, + FYPCF_QUIET, + nullptr, + diag.get() + }; + fy_document* doc = fy_document_build_from_string(&cfg, cstr, FY_NT); + if (!doc) { + fy_diag_error* err; + void *iter = nullptr; + while ((err = fy_diag_errors_iterate(diag.get(), &iter)) != nullptr) { + ythrow yexception() << err->file << ":" << err->line << ":" << err->column << " " << err->msg; + } + } + return TDocument(std::move(str), doc, diag.release()); +} + +TDocument TDocument::Clone() const { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + fy_document* doc = fy_document_clone(Document_.get()); + fy_document_set_userdata( + doc, + new THashSet<TSimpleSharedPtr<TString>, TStringPtrHashT>( + *reinterpret_cast<THashSet<TSimpleSharedPtr<TString>, TStringPtrHashT>*>(fy_document_get_userdata(Document_.get())) + ) + ); + fy_document_register_on_destroy(doc, &DestroyDocumentStrings); + return TDocument(doc, fy_document_get_diag(doc)); +} + +void TDocument::InsertAt(const char* path, const TNodeRef& node) { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + NDetail::RethrowOnError(fy_document_insert_at(Document_.get(), path, FY_NT, node.Node_), Diag_.get()); +} + +TNodeRef TDocument::Buildf(const char* content) { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + return fy_node_build_from_string(Document_.get(), content, strlen(content)); +} + +void TDocument::Resolve() { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + if (fy_document_resolve(Document_.get()) != 0) { + fy_diag_error* err; + void *iter = nullptr; + while ((err = fy_diag_errors_iterate(Diag_.get(), &iter)) != nullptr) { + ythrow yexception() << err->line << ":" << err->column << " " << err->msg; + } + } +} + +bool TDocument::HasDirectives() { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + return fy_document_has_directives(Document_.get()); +} + +bool TDocument::HasExplicitDocumentStart() { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + return fy_document_has_explicit_document_start(Document_.get()); +} + +bool TDocument::HasExplicitDocumentEnd() { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + return fy_document_has_explicit_document_end(Document_.get()); +} + +void TDocument::SetParent(const TDocument& doc) { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + ENSURE_DOCUMENT_NOT_EMPTY(doc.Document_); + NDetail::RethrowOnError(fy_document_set_parent(doc.Document_.get(), Document_.release()), Diag_.get()); +} + +TNodeRef TDocument::Root() { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + return fy_document_root(Document_.get()); +} + +void TDocument::SetRoot(const TNodeRef& node) { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + ENSURE_NODE_NOT_EMPTY(node.Node_); + NDetail::RethrowOnError(fy_document_set_root(Document_.get(), node.Node_), Diag_.get()); +} + +TNodeRef TDocument::CreateAlias(const TString& name) { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + return TNodeRef(fy_node_create_alias_copy(Document_.get(), name.c_str(), name.length())); +} + +std::unique_ptr<char, void(*)(char*)> TDocument::EmitToCharArray() const { + std::unique_ptr<char, void(*)(char*)> res( + fy_emit_document_to_string( + Document_.get(), + (fy_emitter_cfg_flags)(FYECF_DEFAULT | FYECF_OUTPUT_COMMENTS)), &NDetail::FreeChar); + return res; +} + +bool TDocument::RegisterUserDataCleanup() { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + return fy_document_register_meta(Document_.get(), &DestroyUserData, nullptr) == 0; +} + +void TDocument::UnregisterUserDataCleanup() { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + fy_document_unregister_meta(Document_.get()); +} + +TMark TDocument::BeginMark() const { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + auto* fyds = fy_document_get_document_state(Document_.get()); + auto* mark = fy_document_state_start_mark(fyds); + return TMark{ + mark->input_pos, + mark->line, + mark->column, + }; +} + +TMark TDocument::EndMark() const { + ENSURE_DOCUMENT_NOT_EMPTY(Document_); + auto* fyds = fy_document_get_document_state(Document_.get()); + auto* mark = fy_document_state_end_mark(fyds); + return TMark{ + mark->input_pos, + mark->line, + mark->column, + }; +} + +std::unique_ptr<char, void(*)(char*)> TJsonEmitter::EmitToCharArray() const { + std::unique_ptr<char, void(*)(char*)> res( + fy_emit_node_to_string( + Node_.Node_, + (fy_emitter_cfg_flags)(FYECF_DEFAULT | FYECF_SORT_KEYS | FYECF_MODE_JSON_TP)), &NDetail::FreeChar); + return res; +} + +TParser::TParser(TString rawStream, fy_parser* parser, fy_diag* diag) + : RawDocumentStream_(std::move(rawStream)) + , Parser_(parser, fy_parser_destroy) + , Diag_(diag, fy_diag_destroy) +{} + +TParser TParser::Create(TString str) +{ + const char* stream = str.empty() ? zstr : str.cbegin(); + fy_diag_cfg dcfg; + fy_diag_cfg_default(&dcfg); + std::unique_ptr<fy_diag, void(*)(fy_diag*)> diag(fy_diag_create(&dcfg), fy_diag_destroy); + fy_diag_set_collect_errors(diag.get(), true); + fy_parse_cfg cfg{ + "", + // FYPCF_PARSE_COMMENTS, + FYPCF_QUIET, + nullptr, + diag.get() + }; + auto* parser = fy_parser_create(&cfg); + if (!parser) { + fy_diag_error* err; + void *iter = nullptr; + while ((err = fy_diag_errors_iterate(diag.get(), &iter)) != nullptr) { + ythrow yexception() << err->file << ":" << err->line << ":" << err->column << " " << err->msg; + } + } + + fy_parser_set_string(parser, stream, -1); + + return TParser(std::move(str), parser, diag.release()); +} + +std::optional<TDocument> TParser::NextDocument() { + auto* doc = fy_parse_load_document(Parser_.get()); + if (!doc) { + return std::nullopt; + } + + return TDocument(RawDocumentStream_, doc, fy_document_get_diag(doc)); +} + +namespace NDetail { + +void RethrowError(fy_diag* diag) { + void *iter = nullptr; + fy_diag_error* err; + TStringStream ss; + while ((err = fy_diag_errors_iterate(diag, &iter)) != nullptr) { + ss << err->line << ":" << err->column << " " << err->msg << "\n"; + } + ythrow yexception() << ss.Str(); +} + +void RethrowOnError(bool isError, fy_node* node) { + if (!isError) { + return; + } + + std::unique_ptr<fy_diag, void(*)(fy_diag*)> diag(fy_document_get_diag(fy_node_document(node)), fy_diag_unref); + RethrowError(diag.get()); +} + +void RethrowOnError(bool isError, fy_node_pair* pair) { + if (!isError) { + return; + } + + std::unique_ptr<fy_diag, void(*)(fy_diag*)> diag(fy_document_get_diag(fy_node_document(fy_node_pair_key(pair))), fy_diag_unref); + RethrowError(diag.get()); +} + +void RethrowOnError(bool isError, fy_diag* diag) { + if (!isError) { + return; + } + + RethrowError(diag); +} + + +void FreeChar(char* mem) { + free(mem); +} + +} // namespace NDetail + +} // namespace NFyaml + +template <> +void Out<NFyaml::TDocument>(IOutputStream& out, const NFyaml::TDocument& value) { + out << value.EmitToCharArray().get(); +} + +template <> +void Out<NFyaml::TNodeRef>(IOutputStream& out, const NFyaml::TNodeRef& value) { + out << value.EmitToCharArray().get(); +} + +template <> +void Out<NFyaml::TJsonEmitter>(IOutputStream& out, const NFyaml::TJsonEmitter& value) { + out << value.EmitToCharArray().get(); +} diff --git a/library/cpp/yaml/fyamlcpp/fyamlcpp.h b/library/cpp/yaml/fyamlcpp/fyamlcpp.h new file mode 100644 index 0000000000..bcf9362d73 --- /dev/null +++ b/library/cpp/yaml/fyamlcpp/fyamlcpp.h @@ -0,0 +1,636 @@ +#pragma once + +#include <util/generic/yexception.h> +#include <util/system/compiler.h> +#include <util/system/yassert.h> +#include <util/stream/str.h> +#include <util/generic/hash_set.h> + +#include <memory> +#include <optional> + +struct fy_parser; +struct fy_node; +struct fy_document; +struct fy_diag; +struct fy_document_iterator; +struct fy_node_pair; + +namespace NFyaml { + +struct TStringPtrHashT { + size_t operator()(const TSimpleSharedPtr<TString>& str) const { + return (size_t)str.Get(); + } +}; + +struct TFyamlEx : public yexception {}; + +enum class ENodeType { + Scalar, + Sequence, + Mapping, +}; + +namespace NDetail { + +class IBasicUserData { +public: + virtual ~IBasicUserData() = default; +}; + +template <class T> +class TUserDataHolder : public IBasicUserData { +public: + TUserDataHolder(IBasicUserData* next, T* data) + : Next_(next) + , Data_(data) + {} + +private: + std::unique_ptr<IBasicUserData> Next_ = nullptr; + std::unique_ptr<T> Data_ = nullptr; +}; + +void RethrowError(fy_diag* diag); + +void RethrowOnError(bool isError, fy_node* node); + +void RethrowOnError(bool isError, fy_node_pair* pair); + +void RethrowOnError(bool isError, fy_diag* diag); + +void FreeChar(char* mem); + +} // namespace NDetail + +class TNodeRef; +class TDocumentIterator; +class TDocument; +class TNode; +class TMappingIterator; +class TReverseMappingIterator; +class TMapping; +class TSequenceIterator; +class TReverseSequenceIterator; +class TSequence; +class TJsonEmitter; +class TParser; +struct TMark; + +class TDocumentIterator { + friend class TDocument; +public: + TDocumentIterator(fy_document_iterator* iterator = nullptr); + +protected: + std::unique_ptr<fy_document_iterator, void(*)(fy_document_iterator*)> Iterator_; +}; + +class TNodeRef { + friend class TDocument; + friend class TDocumentNodeIterator; + friend class TNode; + friend class TMapping; + friend class TMappingIterator; + friend class TReverseMappingIterator; + friend class TNodePairRef; + friend class TSequence; + friend class TSequenceIterator; + friend class TReverseSequenceIterator; + friend class TJsonEmitter; + + TNodeRef(fy_node* node) + : Node_(node) + {} + +public: + TNodeRef() {}; + + TNodeRef(const TNodeRef& other) { Node_ = other.Node_; } + + TNodeRef& operator=(const TNodeRef& other) { Node_ = other.Node_; return *this; } + + TNodeRef& operator=(fy_node* node) { Node_ = node; return *this; } + + bool operator==(const TNodeRef& other) const { return Node_ == other.Node_; } + + explicit operator bool() const { return Node_ != nullptr; } + + TString Path() const; + + ENodeType Type() const; + + TNode Copy() const; + + TNode Copy(TDocument& to) const; + + bool IsAlias() const; + + TNodeRef ResolveAlias() const; + + TNode CreateReference() const; + + TSequence Sequence() const; + + TMapping Map() const; + + TString Scalar() const; + + void Insert(const TNodeRef& node); + + bool Empty() const { return Node_ == nullptr; } + + std::optional<TString> Tag() const; + + void SetTag(const TString& tag); + + bool RemoveTag(); + + bool HasAnchor() const; + + void SetAnchor(const TString& anchor); + + bool DeepEqual(const TNodeRef& other); + + std::unique_ptr<char, void(*)(char*)> EmitToCharArray() const; + +protected: + fy_node* Node_ = nullptr; + + void SetUserData(NDetail::IBasicUserData* data); + NDetail::IBasicUserData* UserData() const; + void ClearUserData(); +}; + +class TNode { + friend class TDocument; + friend class TDocumentNodeIterator; + friend class TNodeRef; + friend class TSequence; + + TNode& operator=(fy_node* node); +public: + TNode(fy_node* node = nullptr); + + bool operator==(const TNode& other) const { return Node_ == other.Node_; } + + explicit operator bool() { return Node_ != nullptr; } + + TNodeRef Ref() { return TNodeRef(Node_.get()); } + +private: + std::shared_ptr<fy_node> Node_; +}; + +class TNodePairRef { + friend class TMappingIterator; + friend class TReverseMappingIterator; + friend class TMapping; +public: + TNodePairRef(fy_node_pair* pair = nullptr) + : Pair_(pair) + {} + + bool operator==(const TNodePairRef& other) const { return Pair_ == other.Pair_; } + + explicit operator bool() const { return Pair_ != nullptr; } + + TNodeRef Key() const; + + void SetKey(const TNodeRef& node); + + TNodeRef Value() const; + + void SetValue(const TNodeRef& node); + +private: + fy_node_pair* Pair_ = nullptr; +}; + +class TMappingIterator { + friend class TMapping; +public: + TMappingIterator(const TNodeRef& node, bool end = false); + + TMappingIterator(const TMappingIterator& other) { + Node_ = other.Node_; + NodePair_ = other.NodePair_; + } + + TMappingIterator& operator=(const TMappingIterator& other) { + Node_ = other.Node_; + NodePair_ = other.NodePair_; + return *this; + } + + TMappingIterator& operator++(); + + const TNodePairRef* operator->() const { return &NodePair_; } + + TMappingIterator operator++(int) { + TMappingIterator retval = *this; + ++(*this); + return retval; + } + + bool operator==(TMappingIterator other) const { return Node_ == other.Node_ && NodePair_ == other.NodePair_; } + + const TNodePairRef& operator*() const { return NodePair_; } + +private: + TNodeRef Node_; + TNodePairRef NodePair_; +}; + +class TReverseMappingIterator { + friend class TMapping; +public: + TReverseMappingIterator(const TNodeRef& node, bool end = false); + + TReverseMappingIterator(const TReverseMappingIterator& other) { + Node_ = other.Node_; + NodePair_ = other.NodePair_; + } + + TReverseMappingIterator& operator=(const TReverseMappingIterator& other) { + Node_ = other.Node_; + NodePair_ = other.NodePair_; + return *this; + } + + TReverseMappingIterator& operator++(); + + const TNodePairRef* operator->() const { return &NodePair_; } + + TReverseMappingIterator operator++(int) { + TReverseMappingIterator retval = *this; + ++(*this); + return retval; + } + + bool operator==(TReverseMappingIterator other) const { return Node_ == other.Node_ && NodePair_ == other.NodePair_; } + + bool operator!=(TReverseMappingIterator other) const { return !(*this == other); } + + const TNodePairRef& operator*() const { return NodePair_; } + +private: + TNodeRef Node_; + TNodePairRef NodePair_; +}; + +class TMapping : public TNodeRef { +public: + explicit TMapping(const TNodeRef& node) + : TNodeRef(node) + { + Y_VERIFY_DEBUG(Type() == ENodeType::Mapping); + } + + TMappingIterator begin() const { + return TMappingIterator(Node_); + } + + TMappingIterator end() const { + return TMappingIterator(Node_, true); + } + + TReverseMappingIterator rbegin() const { + return TReverseMappingIterator(Node_); + } + + TReverseMappingIterator rend() const { + return TReverseMappingIterator(Node_, true); + } + + size_t size() const; + + size_t empty() const; + + TNodePairRef at(int index) const; + + TNodePairRef operator[](int index) const; + + TNodeRef at(const TString& index) const; + + TNodePairRef pair_at(const TString& index) const; + + TNodePairRef pair_at_opt(const TString& index) const; + + TNodeRef operator[](const TString& index) const; + + TNodeRef operator[](const char* str) const; + + void Append(const TNodeRef& key, const TNodeRef& value); + + void Prepend(const TNodeRef& key, const TNodeRef& value); + + void Remove(const TNodePairRef& toRemove); + + TMappingIterator Remove(const TMappingIterator& toRemove); + + void Remove(const TNodeRef& key); +}; + +class TSequenceIterator { + friend class TSequence; +public: + TSequenceIterator(const TNodeRef& node, bool end = false); + + TSequenceIterator(const TSequenceIterator& other) { + Node_ = other.Node_; + IterNode_ = other.IterNode_; + Iter_ = other.Iter_; + } + + TSequenceIterator& operator=(const TSequenceIterator& other) { + Node_ = other.Node_; + IterNode_ = other.IterNode_; + Iter_ = other.Iter_; + return *this; + } + + TSequenceIterator& operator++(); + + const TNodeRef* operator->() const { + return &IterNode_; + } + + TSequenceIterator operator++(int) { + TSequenceIterator retval = *this; + ++(*this); + return retval; + } + + bool operator==(TSequenceIterator other) const { return Node_ == other.Node_ && Iter_ == other.Iter_; } + + bool operator!=(TSequenceIterator other) const { return !(*this == other); } + + const TNodeRef& operator*() const { return IterNode_; } + + void InsertBefore(const TNodeRef& node); + + void InsertAfter(const TNodeRef& node); + +private: + TNodeRef Node_; + TNodeRef IterNode_; + void* Iter_ = nullptr; +}; + +class TReverseSequenceIterator { + friend class TSequence; +public: + TReverseSequenceIterator(const TNodeRef& node, bool end = false); + + TReverseSequenceIterator(const TReverseSequenceIterator& other) { + Node_ = other.Node_; + IterNode_ = other.IterNode_; + Iter_ = other.Iter_; + } + + TReverseSequenceIterator& operator=(const TReverseSequenceIterator& other) { + Node_ = other.Node_; + IterNode_ = other.IterNode_; + Iter_ = other.Iter_; + return *this; + } + + TReverseSequenceIterator& operator++(); + + const TNodeRef* operator->() const { + return &IterNode_; + } + + TReverseSequenceIterator operator++(int) { + TReverseSequenceIterator retval = *this; + ++(*this); + return retval; + } + + bool operator==(TReverseSequenceIterator other) const { return Node_ == other.Node_ && Iter_ == other.Iter_; } + + bool operator!=(TReverseSequenceIterator other) const { return !(*this == other); } + + const TNodeRef& operator*() const { return IterNode_; } + + void InsertBefore(const TNodeRef& node); + + void InsertAfter(const TNodeRef& node); + +private: + TNodeRef Node_; + TNodeRef IterNode_; + void* Iter_ = nullptr; +}; + +class TSequence : public TNodeRef { +public: + explicit TSequence(const TNodeRef& node) + : TNodeRef(node) + { + Y_VERIFY_DEBUG(Type() == ENodeType::Sequence); + } + + TSequenceIterator begin() const { + return TSequenceIterator(Node_); + } + + TSequenceIterator end() const { + return TSequenceIterator(Node_, true); + } + + TReverseSequenceIterator rbegin() const { + return TReverseSequenceIterator(Node_); + } + + TReverseSequenceIterator rend() const { + return TReverseSequenceIterator(Node_, true); + } + + size_t size() const; + + size_t empty() const; + + TNodeRef at(int index) const; + + TNodeRef operator[](int index) const; + + void Append(const TNodeRef& node); + + void Prepend(const TNodeRef& node); + + void InsertBefore(const TNodeRef& mark, const TNodeRef& node); + + void InsertAfter(const TNodeRef& mark, const TNodeRef& node); + + TNode Remove(const TNodeRef& toRemove); + + TSequenceIterator Remove(const TSequenceIterator& toRemove); + + TReverseSequenceIterator Remove(const TReverseSequenceIterator& toRemove); +}; + +class TDocumentNodeIterator + : public TDocumentIterator +{ +public: + TDocumentNodeIterator(TNodeRef&& node); + + TDocumentNodeIterator(const TDocumentNodeIterator& other) + : TDocumentIterator(other.Iterator_.get()) + , Node_(other.Node_) + {} + + TDocumentNodeIterator& operator=(const TDocumentNodeIterator& other) { + Iterator_.reset(other.Iterator_.get()); + Node_ = other.Node_; + return *this; + } + + TDocumentNodeIterator& operator++(); + + TNodeRef* operator->() { + return &Node_; + } + + TDocumentNodeIterator operator++(int) { + TDocumentNodeIterator retval = *this; + ++(*this); + return retval; + } + + bool operator==(TDocumentNodeIterator other) const { return Node_ == other.Node_; } + + bool operator!=(TDocumentNodeIterator other) const { return !(*this == other); } + + TNodeRef& operator*() { return Node_; } + +private: + TNodeRef Node_; +}; + +class TDocument { + friend class TNode; + friend class TNodeRef; + friend class TJsonEmitter; + friend class TParser; + friend class TMapping; + + TDocument(TString str, fy_document* doc = nullptr, fy_diag* diag = nullptr); + TDocument(fy_document* doc = nullptr, fy_diag* diag = nullptr); + +public: + TDocument(TDocument&& other) + : Document_(std::move(other.Document_)) + , Diag_(std::move(other.Diag_)) + {} + + static TDocument Parse(TString cstr); + + TDocument Clone() const; + + template <class... Args> + size_t Scanf(const char* fmt, Args&& ...args) { + Y_VERIFY_DEBUG(Document_); + return fy_document_scanf(Document_.get(), fmt, std::forward<Args>(args)...); + } + + void InsertAt(const char* path, const TNodeRef& node); + + template <class... Args> + TNodeRef Buildf(const char* fmt, Args&& ...args) { + Y_VERIFY_DEBUG(Document_); + return fy_node_buildf(Document_.get(), fmt, std::forward<Args>(args)...); + } + + TNodeRef Buildf(const char* content); + + void Resolve(); + + bool HasDirectives(); + + bool HasExplicitDocumentStart(); + + bool HasExplicitDocumentEnd(); + + void SetParent(const TDocument& doc); + + TNodeRef Root(); + + void SetRoot(const TNodeRef& node); + + TDocumentNodeIterator begin() { + auto it = TDocumentNodeIterator(Root()); + ++it; + return it; + } + + TDocumentNodeIterator end() { + return TDocumentNodeIterator(TNodeRef(nullptr)); + } + + TNodeRef CreateAlias(const TString& name); + + std::unique_ptr<char, void(*)(char*)> EmitToCharArray() const; + + TMark BeginMark() const; + + TMark EndMark() const; + +private: + std::unique_ptr<fy_document, void(*)(fy_document*)> Document_; + std::unique_ptr<fy_diag, void(*)(fy_diag*)> Diag_; + + static void DestroyUserData(fy_node *fyn, void *meta, void *user) { + Y_UNUSED(fyn); + Y_UNUSED(user); + if (meta) { + auto* data = reinterpret_cast<NDetail::IBasicUserData*>(meta); + delete data; + } + } + + static void DestroyDocumentStrings(fy_document *fyd, void *user) { + Y_UNUSED(fyd); + if (user) { + auto* data = reinterpret_cast<THashSet<TSimpleSharedPtr<TString>, TStringPtrHashT>*>(user); + delete data; + } + } + + bool RegisterUserDataCleanup(); + void UnregisterUserDataCleanup(); +}; + +class TJsonEmitter { +public: + TJsonEmitter(TDocument& doc) : Node_(doc.Root()) {} + TJsonEmitter(const TNodeRef& node) : Node_(node) {} + + std::unique_ptr<char, void(*)(char*)> EmitToCharArray() const; + +private: + const TNodeRef Node_; +}; + +class TParser { + TParser(TString rawStream, fy_parser* doc, fy_diag* diag); +public: + static TParser Create(TString str); + + std::optional<TDocument> NextDocument(); +private: + TString RawDocumentStream_; + std::unique_ptr<fy_parser, void(*)(fy_parser*)> Parser_; + std::unique_ptr<fy_diag, void(*)(fy_diag*)> Diag_; +}; + +struct TMark { + size_t InputPos; + int Line; + int Column; +}; + +} // namesapce NFyaml diff --git a/library/cpp/yaml/fyamlcpp/fyamlcpp_ut.cpp b/library/cpp/yaml/fyamlcpp/fyamlcpp_ut.cpp new file mode 100644 index 0000000000..ffe1d4368c --- /dev/null +++ b/library/cpp/yaml/fyamlcpp/fyamlcpp_ut.cpp @@ -0,0 +1,156 @@ +#include "fyamlcpp.h" + +#include <contrib/libs/libfyaml/include/libfyaml.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/str.h> + +Y_UNIT_TEST_SUITE(FYamlCpp) { + Y_UNIT_TEST(EnumEquals) { + UNIT_ASSERT_EQUAL((int)NFyaml::ENodeType::Scalar, FYNT_SCALAR); + UNIT_ASSERT_EQUAL((int)NFyaml::ENodeType::Sequence, FYNT_SEQUENCE); + UNIT_ASSERT_EQUAL((int)NFyaml::ENodeType::Mapping, FYNT_MAPPING); + } + + Y_UNIT_TEST(ErrorHandling) { + { + const char *yaml = R"( +config: a +config: b +)"; + UNIT_ASSERT_EXCEPTION_CONTAINS( + NFyaml::TDocument::Parse(yaml), + yexception, + "3:1 duplicate key"); + } + + { + const char *yaml = R"( +anchor: *does_not_exists +)"; + auto doc = NFyaml::TDocument::Parse(yaml); + UNIT_ASSERT_EXCEPTION_CONTAINS( + doc.Resolve(), + yexception, + "2:10 invalid alias"); + } + } + + Y_UNIT_TEST(Out) { + const char *yaml = R"( +test: output +)"; + + auto doc = NFyaml::TDocument::Parse(yaml); + + TStringStream ss; + + ss << doc; + + UNIT_ASSERT_VALUES_EQUAL(ss.Str(), "test: output\n"); + } + + Y_UNIT_TEST(Parser) { + const char *yaml = R"( +test: a +--- +test: b +)"; + auto parser = NFyaml::TParser::Create(yaml); + + TStringStream ss; + + auto docOpt = parser.NextDocument(); + UNIT_ASSERT(docOpt); + ss << *docOpt; + UNIT_ASSERT_VALUES_EQUAL(ss.Str(), "test: a\n"); + auto beginMark = docOpt->BeginMark(); + UNIT_ASSERT_VALUES_EQUAL(beginMark.InputPos, 1); + auto endMark = docOpt->EndMark(); + UNIT_ASSERT_VALUES_EQUAL(endMark.InputPos, 12); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(yaml).SubStr(beginMark.InputPos, endMark.InputPos - 4), ss.Str()); + + ss.clear(); + + auto docOpt2 = parser.NextDocument(); + UNIT_ASSERT(docOpt2); + ss << *docOpt2; + UNIT_ASSERT_VALUES_EQUAL(ss.Str(), "---\ntest: b\n"); + beginMark = docOpt2->BeginMark(); + UNIT_ASSERT_VALUES_EQUAL(beginMark.InputPos, 9); + endMark = docOpt2->EndMark(); + UNIT_ASSERT_VALUES_EQUAL(endMark.InputPos, 21); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(yaml).SubStr(beginMark.InputPos, endMark.InputPos), ss.Str()); + + auto docOpt3 = parser.NextDocument(); + UNIT_ASSERT(!docOpt3); + } + + Y_UNIT_TEST(Leak) { + std::optional<NFyaml::TDocument> doc; + + { + std::optional<NFyaml::TDocument> item1; + std::optional<NFyaml::TDocument> item2; + { + const TString items = R"( +item: + x: 1 + y: 2 +--- +test: + a: noop + b: + - 1 + - 2 + - 3 +--- +x: b +)"; + auto parser = NFyaml::TParser::Create(items); + + item1.emplace(*parser.NextDocument()); + item2.emplace(*parser.NextDocument()); + parser.NextDocument(); + parser.NextDocument(); + } + + { + const TString config = R"( +test: a +--- +test: [] +--- +x: b +)"; + auto parser = NFyaml::TParser::Create(config); + + parser.NextDocument(); + doc.emplace(*parser.NextDocument()); + parser.NextDocument(); + parser.NextDocument(); + } + + { + auto item1NodeRef = item1->Root().Map().at("item"); + auto item2NodeRef = item2->Root().Map().at("test"); + auto docNodeRef = doc->Root().Map().at("test"); + auto node1 = item1NodeRef.Copy(*doc); + auto node2 = item2NodeRef.Copy(*doc); + docNodeRef.Sequence().Append(node1.Ref()); + docNodeRef.Sequence().Append(node2.Ref()); + item1.reset(); + item2.reset(); + } + } + + auto seq = doc->Root().Map().at("test").Sequence(); + UNIT_ASSERT_VALUES_EQUAL(seq[0].Map().at("x").Scalar(), "1"); + UNIT_ASSERT_VALUES_EQUAL(seq[0].Map().at("y").Scalar(), "2"); + UNIT_ASSERT_VALUES_EQUAL(seq[1].Map().at("a").Scalar(), "noop"); + UNIT_ASSERT_VALUES_EQUAL(seq[1].Map().at("b").Sequence().at(0).Scalar(), "1"); + UNIT_ASSERT_VALUES_EQUAL(seq[1].Map().at("b").Sequence().at(1).Scalar(), "2"); + UNIT_ASSERT_VALUES_EQUAL(seq[1].Map().at("b").Sequence().at(2).Scalar(), "3"); + } +} diff --git a/library/cpp/yaml/fyamlcpp/libfyaml_ut.cpp b/library/cpp/yaml/fyamlcpp/libfyaml_ut.cpp new file mode 100644 index 0000000000..d5e28cf98e --- /dev/null +++ b/library/cpp/yaml/fyamlcpp/libfyaml_ut.cpp @@ -0,0 +1,1838 @@ +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <assert.h> +#include <limits.h> + +#include <span> + +#include <contrib/libs/libfyaml/include/libfyaml.h> +#include <library/cpp/testing/unittest/registar.h> + +/* +These tests are ported from https://github.com/pantoniou/libfyaml/blob/master/test/libfyaml-test-core.c +To check windows compatibility and avoid possible internal patch issues +*/ + +Y_UNIT_TEST_SUITE(LibFyamlCore) { + +Y_UNIT_TEST(doc_build_simple) { + struct fy_document *fyd; + + /* setup */ + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* cleanup */ + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_build_parse_check) { + struct fy_document *fyd; + char *buf; + + /* build document (with comments, newlines etc) */ + fyd = fy_document_build_from_string(NULL, "#comment\n[ 42, \n 12 ] # comment\n", FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* convert to string */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(buf, NULL); + + /* compare with expected result */ + UNIT_ASSERT_VALUES_EQUAL(buf, TString("[42, 12]\n")); + + free(buf); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_build_scalar) { + struct fy_document *fyd; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "plain scalar # comment", FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* compare with expected result */ + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_document_root(fyd)), TString("plain scalar")); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_build_sequence) { + struct fy_document *fyd; + struct fy_node *fyn; + int count; + void *iter; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "[ 10, 11, foo ] # comment", FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* check for correct count value */ + count = fy_node_sequence_item_count(fy_document_root(fyd)); + UNIT_ASSERT_EQUAL(count, 3); + + /* try forward iterator first */ + iter = NULL; + + /* 0 */ + fyn = fy_node_sequence_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fyn), TString("10")); + + /* 1 */ + fyn = fy_node_sequence_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fyn), TString("11")); + + /* 2 */ + fyn = fy_node_sequence_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fyn), TString("foo")); + + /* final, iterator must return NULL */ + fyn = fy_node_sequence_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_EQUAL(fyn, NULL); + + /* reverse iterator */ + iter = NULL; + + /* 2 */ + fyn = fy_node_sequence_reverse_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fyn), TString("foo")); + + /* 1 */ + fyn = fy_node_sequence_reverse_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fyn), TString("11")); + + /* 0 */ + fyn = fy_node_sequence_reverse_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fyn), TString("10")); + + /* final, iterator must return NULL */ + fyn = fy_node_sequence_reverse_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_EQUAL(fyn, NULL); + + /* do forward index based accesses */ + + /* 0 */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), 0); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fyn), TString("10")); + + /* 1 */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), 1); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fyn), TString("11")); + + /* 2 */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), 2); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fyn), TString("foo")); + + /* 3, it must not exist */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), 3); + UNIT_ASSERT_EQUAL(fyn, NULL); + + /* do backward index based accesses */ + + /* 2 */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -1); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fyn), TString("foo")); + + /* 1 */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -2); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fyn), TString("11")); + + /* 0 */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -3); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fyn), TString("10")); + + /* -1, it must not exist */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -4); + UNIT_ASSERT_EQUAL(fyn, NULL); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_build_mapping) { + struct fy_document *fyd; + struct fy_node_pair *fynp; + int count; + void *iter; + + fyd = fy_document_build_from_string(NULL, "{ foo: 10, bar : 20, baz: [100, 101], [frob, 1]: boo }", FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* check for correct count value */ + count = fy_node_mapping_item_count(fy_document_root(fyd)); + UNIT_ASSERT_EQUAL(count, 4); + + /* forward iterator first */ + iter = NULL; + + /* 0 */ + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_key(fynp)), TString("foo")); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_value(fynp)), TString("10")); + + /* 1 */ + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_key(fynp)), TString("bar")); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_value(fynp)), TString("20")); + + /* 2 */ + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_key(fynp)), TString("baz")); + UNIT_ASSERT_EQUAL(fy_node_sequence_item_count(fy_node_pair_value(fynp)), 2); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_value(fynp), 0)), TString("100")); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_value(fynp), 1)), TString("101")); + + /* 3 */ + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_EQUAL(fy_node_sequence_item_count(fy_node_pair_key(fynp)), 2); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_key(fynp), 0)), TString("frob")); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_key(fynp), 1)), TString("1")); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_value(fynp)), TString("boo")); + + /* 4, it must not exist */ + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_EQUAL(fynp, NULL); + + /* reverse iterator */ + iter = NULL; + + /* 3 */ + fynp = fy_node_mapping_reverse_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_EQUAL(fy_node_sequence_item_count(fy_node_pair_key(fynp)), 2); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_key(fynp), 0)), TString("frob")); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_key(fynp), 1)), TString("1")); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_value(fynp)), TString("boo")); + + /* 2 */ + fynp = fy_node_mapping_reverse_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_key(fynp)), TString("baz")); + UNIT_ASSERT_EQUAL(fy_node_sequence_item_count(fy_node_pair_value(fynp)), 2); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_value(fynp), 0)), TString("100")); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_value(fynp), 1)), TString("101")); + + /* 1 */ + fynp = fy_node_mapping_reverse_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_key(fynp)), TString("bar")); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_value(fynp)), TString("20")); + + /* 0 */ + fynp = fy_node_mapping_reverse_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_key(fynp)), TString("foo")); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_value(fynp)), TString("10")); + + /* -1, it must not exist */ + fynp = fy_node_mapping_reverse_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_EQUAL(fynp, NULL); + + /* key lookups (note how only the contents are compared) */ + UNIT_ASSERT(fy_node_compare_string(fy_node_mapping_lookup_by_string(fy_document_root(fyd), "foo", FY_NT), "10", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_mapping_lookup_by_string(fy_document_root(fyd), "bar", FY_NT), "20", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_mapping_lookup_by_string(fy_document_root(fyd), "baz", FY_NT), "- 100\n- 101", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_mapping_lookup_by_string(fy_document_root(fyd), "- 'frob'\n- \"\x31\"", FY_NT), "boo", FY_NT) == true); + + fy_document_destroy(fyd); + fyd = NULL; +} + +Y_UNIT_TEST(doc_path_access) { + struct fy_document *fyd; + struct fy_node *fyn; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "{ " + "foo: 10, bar : 20, baz:{ frob: boo }, " + "frooz: [ seq1, { key: value} ], \"zero\\0zero\" : 0, " + "{ key2: value2 }: { key3: value3 } " + "}", FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* check that getting root node works */ + fyn = fy_node_by_path(fy_document_root(fyd), "/", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_EQUAL(fyn, fy_document_root(fyd)); + + /* check access to scalars */ + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW), "10", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "bar", FY_NT, FYNWF_DONT_FOLLOW), "20", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "baz/frob", FY_NT, FYNWF_DONT_FOLLOW), "boo", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/frooz/0", FY_NT, FYNWF_DONT_FOLLOW), "seq1", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/frooz/1/key", FY_NT, FYNWF_DONT_FOLLOW), "value", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "\"zero\\0zero\"", FY_NT, FYNWF_DONT_FOLLOW), "0", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/{ key2: value2 }/key3", FY_NT, FYNWF_DONT_FOLLOW), "value3", FY_NT) == true); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_path_node) { + struct fy_document *fyd; + char *path; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "{ " + "foo: 10, bar : 20, baz:{ frob: boo }, " + "frooz: [ seq1, { key: value} ], \"zero\\0zero\" : 0, " + "{ key2: value2 }: { key3: value3 } " + "}", FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + path = fy_node_get_path(fy_node_by_path(fy_document_root(fyd), "/", FY_NT, FYNWF_DONT_FOLLOW)); + UNIT_ASSERT_VALUES_EQUAL(path, TString("/")); + free(path); + + path = fy_node_get_path(fy_node_by_path(fy_document_root(fyd), "/frooz", FY_NT, FYNWF_DONT_FOLLOW)); + UNIT_ASSERT_VALUES_EQUAL(path, TString("/frooz")); + free(path); + + path = fy_node_get_path(fy_node_by_path(fy_document_root(fyd), "/frooz/0", FY_NT, FYNWF_DONT_FOLLOW)); + UNIT_ASSERT_VALUES_EQUAL(path, TString("/frooz/0")); + free(path); + + path = fy_node_get_path(fy_node_by_path(fy_document_root(fyd), "/{ key2: value2 }/key3", FY_NT, FYNWF_DONT_FOLLOW)); + UNIT_ASSERT_VALUES_EQUAL(path, TString("/{key2: value2}/key3")); + free(path); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_path_parent) { + struct fy_document *fyd; + struct fy_node *fyn_root, *fyn_foo, *fyn_bar, *fyn_baz, *fyn_frob, *fyn_ten; + struct fy_node *fyn_deep, *fyn_deeper; + char *path; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "{ " + "foo: 10, bar : [ ten, 20 ], baz:{ frob: boo, deep: { deeper: yet } }, " + "}", FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn_root = fy_document_root(fyd); + UNIT_ASSERT_UNEQUAL(fyn_root, NULL); + + fyn_foo = fy_node_by_path(fyn_root, "/foo", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_foo, NULL); + + fyn_bar = fy_node_by_path(fyn_root, "/bar", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_bar, NULL); + + fyn_baz = fy_node_by_path(fyn_root, "/baz", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_baz, NULL); + + fyn_frob = fy_node_by_path(fyn_root, "/baz/frob", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_frob, NULL); + + fyn_ten = fy_node_by_path(fyn_root, "/bar/0", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_ten, NULL); + + fyn_deep = fy_node_by_path(fyn_root, "/baz/deep", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_deep, NULL); + + fyn_deeper = fy_node_by_path(fyn_root, "/baz/deep/deeper", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_deeper, NULL); + + /* check parent paths of foo, frob, ten */ + path = fy_node_get_parent_address(fyn_foo); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("foo")); + free(path); + + path = fy_node_get_parent_address(fyn_frob); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("frob")); + free(path); + + path = fy_node_get_parent_address(fyn_ten); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("0")); + free(path); + + /* check relative paths to root */ + path = fy_node_get_path_relative_to(NULL, fyn_foo); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("foo")); + free(path); + + path = fy_node_get_path_relative_to(fyn_root, fyn_foo); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("foo")); + free(path); + + path = fy_node_get_path_relative_to(fyn_root, fyn_frob); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("baz/frob")); + free(path); + + path = fy_node_get_path_relative_to(fyn_root, fyn_ten); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("bar/0")); + free(path); + + /* check relative paths to other parents */ + path = fy_node_get_path_relative_to(fyn_baz, fyn_frob); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("frob")); + free(path); + + path = fy_node_get_path_relative_to(fyn_bar, fyn_ten); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("0")); + free(path); + + path = fy_node_get_path_relative_to(fyn_baz, fyn_deeper); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("deep/deeper")); + free(path); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_short_path) { + struct fy_document *fyd; + struct fy_node *fyn_root, *fyn_foo, *fyn_notfoo, *fyn_bar, *fyn_baz; + const char *str; + + /* build document */ + fyd = fy_document_build_from_string(NULL, + "--- &r\n" + " foo: &f\n" + " bar: [ 0, two, baz: what ]\n" + " frob: true\n" + " notfoo: false\n" + , FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn_root = fy_document_root(fyd); + UNIT_ASSERT_UNEQUAL(fyn_root, NULL); + + fyn_foo = fy_node_by_path(fyn_root, "/foo", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_foo, NULL); + + fyn_notfoo = fy_node_by_path(fyn_root, "/notfoo", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_notfoo, NULL); + + fyn_bar = fy_node_by_path(fyn_root, "/foo/bar", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_bar, NULL); + + fyn_baz = fy_node_by_path(fyn_root, "/foo/bar/2/baz", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_baz, NULL); + + str = fy_node_get_short_path(fyn_root); + UNIT_ASSERT_UNEQUAL(str, NULL); + UNIT_ASSERT_VALUES_EQUAL(str, "*r"); + UNIT_ASSERT_EQUAL(fy_node_by_path(fy_document_root(fyd), str, FY_NT, FYNWF_FOLLOW), fyn_root); + free((void*)str); + + str = fy_node_get_short_path(fyn_foo); + UNIT_ASSERT_UNEQUAL(str, NULL); + UNIT_ASSERT_VALUES_EQUAL(str, "*f"); + UNIT_ASSERT_EQUAL(fy_node_by_path(fy_document_root(fyd), str, FY_NT, FYNWF_FOLLOW), fyn_foo); + free((void*)str); + + str = fy_node_get_short_path(fyn_notfoo); + UNIT_ASSERT_UNEQUAL(str, NULL); + UNIT_ASSERT_VALUES_EQUAL(str, "*r/notfoo"); + UNIT_ASSERT_EQUAL(fy_node_by_path(fy_document_root(fyd), str, FY_NT, FYNWF_FOLLOW), fyn_notfoo); + free((void*)str); + + str = fy_node_get_short_path(fyn_bar); + UNIT_ASSERT_UNEQUAL(str, NULL); + UNIT_ASSERT_VALUES_EQUAL(str, "*f/bar"); + UNIT_ASSERT_EQUAL(fy_node_by_path(fy_document_root(fyd), str, FY_NT, FYNWF_FOLLOW), fyn_bar); + free((void*)str); + + str = fy_node_get_short_path(fyn_baz); + UNIT_ASSERT_UNEQUAL(str, NULL); + UNIT_ASSERT_VALUES_EQUAL(str, "*f/bar/2/baz"); + UNIT_ASSERT_EQUAL(fy_node_by_path(fy_document_root(fyd), str, FY_NT, FYNWF_FOLLOW), fyn_baz); + free((void*)str); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_scalar_path) { + struct fy_document *fyd; + struct fy_node *fyn_root, *fyn_foo; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "--- foo\n", FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn_root = fy_document_root(fyd); + UNIT_ASSERT_UNEQUAL(fyn_root, NULL); + + /* get the scalar root and verify */ + fyn_foo = fy_node_by_path(fyn_root, "/", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_foo, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fyn_foo), "foo"); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_scalar_path_array) { + struct fy_document *fyd; + struct fy_node *fyn_root, *fynt; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "--- [ foo, bar, baz ]\n", FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn_root = fy_document_root(fyd); + UNIT_ASSERT_UNEQUAL(fyn_root, NULL); + + /* get the scalars in the array and verify */ + fynt = fy_node_by_path(fyn_root, "/0", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fynt, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fynt), "foo"); + + fynt = fy_node_by_path(fyn_root, "/1", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fynt, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fynt), "bar"); + + fynt = fy_node_by_path(fyn_root, "/2", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fynt, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fynt), "baz"); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_nearest_anchor) { + struct fy_document *fyd; + struct fy_node *fyn, *fyn_root, *fyn_foo, *fyn_notfoo, *fyn_bar, *fyn_baz; + + /* build document */ + fyd = fy_document_build_from_string(NULL, + "--- &r\n" + " foo: &f\n" + " bar: [ 0, two, baz: what ]\n" + " frob: true\n" + " notfoo: false\n" + , FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn_root = fy_document_root(fyd); + UNIT_ASSERT_UNEQUAL(fyn_root, NULL); + + fyn_foo = fy_node_by_path(fyn_root, "/foo", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_foo, NULL); + + fyn_notfoo = fy_node_by_path(fyn_root, "/notfoo", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_notfoo, NULL); + + fyn_bar = fy_node_by_path(fyn_root, "/foo/bar", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_bar, NULL); + + fyn_baz = fy_node_by_path(fyn_root, "/foo/bar/2/baz", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_baz, NULL); + + /* get nearest anchor of root (is root) */ + fyn = fy_anchor_node(fy_node_get_nearest_anchor(fyn_root)); + UNIT_ASSERT_EQUAL(fyn, fyn_root); + + /* get nearest anchor of notfoo (is root) */ + fyn = fy_anchor_node(fy_node_get_nearest_anchor(fyn_notfoo)); + UNIT_ASSERT_EQUAL(fyn, fyn_root); + + /* get nearest anchor of baz (is foo) */ + fyn = fy_anchor_node(fy_node_get_nearest_anchor(fyn_baz)); + UNIT_ASSERT_EQUAL(fyn, fyn_foo); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_references) { + struct fy_document *fyd; + struct fy_node *fyn, *fyn_ref, *fyn_root, *fyn_foo, *fyn_notfoo, *fyn_bar, *fyn_baz; + char *path; + + /* build document */ + fyd = fy_document_build_from_string(NULL, + "---\n" + " foo: &f\n" + " bar: [ 0, two, baz: what ]\n" + " frob: true\n" + " notfoo: false\n" + , FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn_root = fy_document_root(fyd); + UNIT_ASSERT_UNEQUAL(fyn_root, NULL); + + fyn_foo = fy_node_by_path(fyn_root, "/foo", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_foo, NULL); + + fyn_notfoo = fy_node_by_path(fyn_root, "/notfoo", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_notfoo, NULL); + + fyn_bar = fy_node_by_path(fyn_root, "/foo/bar", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_bar, NULL); + + fyn_baz = fy_node_by_path(fyn_root, "/foo/bar/2/baz", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_baz, NULL); + + /* get reference to root */ + path = fy_node_get_reference(fyn_root); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("*/")); + free(path); + + /* get reference to /foo */ + path = fy_node_get_reference(fyn_foo); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("*f")); + free(path); + + /* get reference to /notfoo */ + path = fy_node_get_reference(fyn_notfoo); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("*/notfoo")); + free(path); + + /* get reference to /foo/bar/2/baz */ + path = fy_node_get_reference(fyn_baz); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("*/foo/bar/2/baz")); + free(path); + + /* create reference to root and verify it points there */ + fyn_ref = fy_node_create_reference(fyn_root); + UNIT_ASSERT_UNEQUAL(fyn_ref, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_EQUAL(fyn, fyn_root); + fy_node_free(fyn_ref); + + /* get reference to /foo */ + fyn_ref = fy_node_create_reference(fyn_foo); + UNIT_ASSERT_UNEQUAL(path, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_EQUAL(fyn, fyn_foo); + fy_node_free(fyn_ref); + + /* get reference to /notfoo */ + fyn_ref = fy_node_create_reference(fyn_notfoo); + UNIT_ASSERT_UNEQUAL(path, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_EQUAL(fyn, fyn_notfoo); + fy_node_free(fyn_ref); + + /* get reference to /bar */ + fyn_ref = fy_node_create_reference(fyn_bar); + UNIT_ASSERT_UNEQUAL(path, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_EQUAL(fyn, fyn_bar); + fy_node_free(fyn_ref); + + /* get reference to /baz */ + fyn_ref = fy_node_create_reference(fyn_baz); + UNIT_ASSERT_UNEQUAL(path, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_EQUAL(fyn, fyn_baz); + fy_node_free(fyn_ref); + + /* get relative reference to /foo starting at / */ + path = fy_node_get_relative_reference(fyn_root, fyn_foo); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("*/foo")); + free(path); + + /* get relative reference to /foo/bar/2/baz starting at / */ + path = fy_node_get_relative_reference(fyn_root, fyn_baz); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("*/foo/bar/2/baz")); + free(path); + + /* get relative reference to /foo/bar/2/baz starting at /foo */ + path = fy_node_get_relative_reference(fyn_foo, fyn_baz); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("*f/bar/2/baz")); + free(path); + + /* get relative reference to /notfoo at /foo (will return absolute) */ + path = fy_node_get_relative_reference(fyn_foo, fyn_notfoo); + UNIT_ASSERT_UNEQUAL(path, NULL); + UNIT_ASSERT_VALUES_EQUAL(path, TString("*/notfoo")); + free(path); + + /* create relative reference to /foo starting at / */ + fyn_ref = fy_node_create_relative_reference(fyn_root, fyn_foo); + UNIT_ASSERT_UNEQUAL(fyn_ref, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_EQUAL(fyn, fyn_foo); + fy_node_free(fyn_ref); + + /* create relative reference to /foo/bar/2/baz starting at / */ + fyn_ref = fy_node_create_relative_reference(fyn_root, fyn_baz); + UNIT_ASSERT_UNEQUAL(fyn_ref, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_EQUAL(fyn, fyn_baz); + fy_node_free(fyn_ref); + + /* create relative reference to /foo/bar/2/baz starting at /foo */ + fyn_ref = fy_node_create_relative_reference(fyn_foo, fyn_baz); + UNIT_ASSERT_UNEQUAL(fyn_ref, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_EQUAL(fyn, fyn_baz); + fy_node_free(fyn_ref); + + /* create relative reference to /notfoo starting at /foo (will use absolute) */ + fyn_ref = fy_node_create_relative_reference(fyn_foo, fyn_notfoo); + UNIT_ASSERT_UNEQUAL(fyn_ref, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_EQUAL(fyn, fyn_notfoo); + fy_node_free(fyn_ref); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_nearest_child_of) { + struct fy_document *fyd; + struct fy_node *fyn, *fyn_root, *fyn_foo, *fyn_bar, *fyn_baz; + + /* build document */ + fyd = fy_document_build_from_string(NULL, + "foo:\n" + " bar:\n" + " barz: [ zero, baz: true ]\n" + " frooz: notfoo\n" + , FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn_root = fy_document_root(fyd); + UNIT_ASSERT_UNEQUAL(fyn_root, NULL); + + fyn_foo = fy_node_by_path(fyn_root, "/foo", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_foo, NULL); + + fyn_bar = fy_node_by_path(fyn_root, "/foo/bar", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_bar, NULL); + + fyn_baz = fy_node_by_path(fyn_root, "/foo/bar/barz/1/baz", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_baz, NULL); + + /* nearest child to the root of /foo is /foo */ + fyn = fy_node_get_nearest_child_of(fyn_root, fyn_foo); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_EQUAL(fyn, fyn_foo); + + /* nearest child to the root of /foo/bar/barz/1/baz is /foo */ + fyn = fy_node_get_nearest_child_of(fyn_root, fyn_baz); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_EQUAL(fyn, fyn_foo); + + /* nearest child to foo of /foo/bar/barz/1/baz is /foo/bar */ + fyn = fy_node_get_nearest_child_of(fyn_foo, fyn_baz); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + UNIT_ASSERT_EQUAL(fyn, fyn_bar); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_create_empty_seq1) { + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn = fy_node_build_from_string(fyd, "[ ]", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* convert to string */ + buf = fy_emit_node_to_string(fy_document_root(fyd), FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(buf, NULL); + + /* compare with expected result */ + UNIT_ASSERT_VALUES_EQUAL(buf, TString("[]")); + + free(buf); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_create_empty_seq2) { + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn = fy_node_create_sequence(fyd); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* convert to string */ + buf = fy_emit_node_to_string(fy_document_root(fyd), FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(buf, NULL); + + /* compare with expected result */ + UNIT_ASSERT_VALUES_EQUAL(buf, TString("[]")); + + free(buf); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_create_empty_map1) { + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn = fy_node_build_from_string(fyd, "{ }", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* convert to string */ + buf = fy_emit_node_to_string(fy_document_root(fyd), FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(buf, NULL); + + /* compare with expected result */ + UNIT_ASSERT_VALUES_EQUAL(buf, TString("{}")); + + free(buf); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_create_empty_map2) { + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn = fy_node_create_mapping(fyd); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* convert to string */ + buf = fy_emit_node_to_string(fy_document_root(fyd), FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(buf, NULL); + + /* compare with expected result */ + UNIT_ASSERT_VALUES_EQUAL(buf, TString("{}")); + + free(buf); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_create_test_seq1) { + struct fy_document *fyd; + struct fy_node *fyn; + int ret; + + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn = fy_node_create_sequence(fyd); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + ret = fy_node_sequence_append(fyn, fy_node_create_scalar(fyd, "foo", FY_NT)); + UNIT_ASSERT_EQUAL(ret, 0); + + ret = fy_node_sequence_append(fyn, fy_node_create_scalar(fyd, "bar", FY_NT)); + UNIT_ASSERT_EQUAL(ret, 0); + + ret = fy_node_sequence_append(fyn, fy_node_build_from_string(fyd, "{ baz: frooz }", FY_NT)); + UNIT_ASSERT_EQUAL(ret, 0); + + fy_document_set_root(fyd, fyn); + + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/0", FY_NT, FYNWF_DONT_FOLLOW), "foo", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/1", FY_NT, FYNWF_DONT_FOLLOW), "bar", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/2/baz", FY_NT, FYNWF_DONT_FOLLOW), "frooz", FY_NT) == true); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_create_test_map1) { + struct fy_document *fyd; + struct fy_node *fyn, *fyn1, *fyn2, *fyn3; + int ret; + + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn = fy_node_create_mapping(fyd); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + ret = fy_node_mapping_append(fyn, + fy_node_build_from_string(fyd, "seq", FY_NT), + fy_node_build_from_string(fyd, "[ zero, one ]", FY_NT)); + UNIT_ASSERT_EQUAL(ret, 0); + + ret = fy_node_mapping_append(fyn, NULL, + fy_node_build_from_string(fyd, "value-of-null-key", FY_NT)); + UNIT_ASSERT_EQUAL(ret, 0); + + ret = fy_node_mapping_append(fyn, + fy_node_build_from_string(fyd, "key-of-null-value", FY_NT), NULL); + UNIT_ASSERT_EQUAL(ret, 0); + + fy_document_set_root(fyd, fyn); + + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/seq/0", FY_NT, FYNWF_DONT_FOLLOW), "zero", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/seq/1", FY_NT, FYNWF_DONT_FOLLOW), "one", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/''", FY_NT, FYNWF_DONT_FOLLOW), "value-of-null-key", FY_NT) == true); + + fyn1 = fy_node_by_path(fy_document_root(fyd), "/key-of-null-value", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_EQUAL(fyn1, NULL); + + /* try to append duplicate key (it should fail) */ + fyn2 = fy_node_build_from_string(fyd, "seq", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn2, NULL); + fyn3 = fy_node_create_scalar(fyd, "dupl", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn3, NULL); + ret = fy_node_mapping_append(fyn, fyn2, fyn3); + UNIT_ASSERT_UNEQUAL(ret, 0); + + fy_node_free(fyn3); + fy_node_free(fyn2); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_insert_remove_seq) { + struct fy_document *fyd; + struct fy_node *fyn; + int ret; + + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fy_document_set_root(fyd, fy_node_build_from_string(fyd, "[ one, two, four ]", FY_NT)); + + /* check that the order is correct */ + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/0", FY_NT, FYNWF_DONT_FOLLOW), "one", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/1", FY_NT, FYNWF_DONT_FOLLOW), "two", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/2", FY_NT, FYNWF_DONT_FOLLOW), "four", FY_NT) == true); + + ret = fy_node_sequence_append(fy_document_root(fyd), fy_node_build_from_string(fyd, "five", FY_NT)); + UNIT_ASSERT_EQUAL(ret, 0); + + ret = fy_node_sequence_prepend(fy_document_root(fyd), fy_node_build_from_string(fyd, "zero", FY_NT)); + UNIT_ASSERT_EQUAL(ret, 0); + + ret = fy_node_sequence_insert_after(fy_document_root(fyd), + fy_node_by_path(fy_document_root(fyd), "/2", FY_NT, FYNWF_DONT_FOLLOW), + fy_node_build_from_string(fyd, "three", FY_NT)); + UNIT_ASSERT_EQUAL(ret, 0); + + ret = fy_node_sequence_insert_before(fy_document_root(fyd), + fy_node_by_path(fy_document_root(fyd), "/3", FY_NT, FYNWF_DONT_FOLLOW), + fy_node_build_from_string(fyd, "two-and-a-half", FY_NT)); + UNIT_ASSERT_EQUAL(ret, 0); + + fyn = fy_node_sequence_remove(fy_document_root(fyd), + fy_node_by_path(fy_document_root(fyd), "/3", FY_NT, FYNWF_DONT_FOLLOW)); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + fy_node_free(fyn); + + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/0", FY_NT, FYNWF_DONT_FOLLOW), "zero", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/1", FY_NT, FYNWF_DONT_FOLLOW), "one", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/2", FY_NT, FYNWF_DONT_FOLLOW), "two", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/3", FY_NT, FYNWF_DONT_FOLLOW), "three", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/4", FY_NT, FYNWF_DONT_FOLLOW), "four", FY_NT) == true); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_insert_remove_map) { + struct fy_document *fyd; + struct fy_node *fyn; + int ret; + + fyd = fy_document_build_from_string(NULL, "{ one: 1, two: 2, four: 4 }", FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* check that the order is correct */ + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/one", FY_NT, FYNWF_DONT_FOLLOW), "1", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/two", FY_NT, FYNWF_DONT_FOLLOW), "2", FY_NT) == true); + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/four", FY_NT, FYNWF_DONT_FOLLOW), "4", FY_NT) == true); + + ret = fy_node_mapping_append(fy_document_root(fyd), + fy_node_build_from_string(fyd, "three", FY_NT), + fy_node_build_from_string(fyd, "3", FY_NT)); + UNIT_ASSERT_EQUAL(ret, 0); + + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/three", FY_NT, FYNWF_DONT_FOLLOW), "3", FY_NT) == true); + + ret = fy_node_mapping_prepend(fy_document_root(fyd), + fy_node_build_from_string(fyd, "zero", FY_NT), + fy_node_build_from_string(fyd, "0", FY_NT)); + UNIT_ASSERT_EQUAL(ret, 0); + + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/zero", FY_NT, FYNWF_DONT_FOLLOW), "0", FY_NT) == true); + + ret = fy_node_mapping_append(fy_document_root(fyd), + fy_node_build_from_string(fyd, "two-and-a-half", FY_NT), + fy_node_build_from_string(fyd, "2.5", FY_NT)); + UNIT_ASSERT_EQUAL(ret, 0); + + UNIT_ASSERT(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/two-and-a-half", FY_NT, FYNWF_DONT_FOLLOW), "2.5", FY_NT) == true); + + fyn = fy_node_mapping_remove_by_key(fy_document_root(fyd), + fy_node_build_from_string(fyd, "two-and-a-half", FY_NT)); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + fy_node_free(fyn); + + /* it must be removed */ + fyn = fy_node_by_path(fy_document_root(fyd), "/two-and-a-half", FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_EQUAL(fyn, NULL); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_sort) { + struct fy_document *fyd; + fy_node_pair *fynp; + void *iter; + int ret, count; + + fyd = fy_document_build_from_string(NULL, "{ a: 5, { z: bar }: 1, z: 7, " + "[ a, b, c] : 3, { a: whee } : 2 , " + "b: 6, [ z ]: 4 }", FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + ret = fy_node_sort(fy_document_root(fyd), NULL, NULL); + UNIT_ASSERT_EQUAL(ret, 0); + + /* check for correct count value */ + count = fy_node_mapping_item_count(fy_document_root(fyd)); + UNIT_ASSERT_EQUAL(count, 7); + + /* forward iterator first */ + iter = NULL; + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_value(fynp)), "1"); + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_value(fynp)), "2"); + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_value(fynp)), "3"); + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_value(fynp)), "4"); + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_value(fynp)), "5"); + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_value(fynp)), "6"); + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_pair_value(fynp)), "7"); + + fy_document_destroy(fyd); +} + +static char *join_docs(const char *tgt_text, const char *tgt_path, + const char *src_text, const char *src_path, + const char *emit_path) +{ + struct fy_document *fyd_tgt, *fyd_src; + struct fy_node *fyn_tgt, *fyn_src, *fyn_emit; + char *output; + int ret; + + /* insert which overwrites root ( <map> <- <scalar> ) */ + fyd_tgt = fy_document_build_from_string(NULL, tgt_text, FY_NT); + UNIT_ASSERT_UNEQUAL(fyd_tgt, NULL); + + fyd_src = fy_document_build_from_string(NULL, src_text, FY_NT); + UNIT_ASSERT_UNEQUAL(fyd_src, NULL); + + fyn_tgt = fy_node_by_path(fy_document_root(fyd_tgt), tgt_path, FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_tgt, NULL); + + fyn_src = fy_node_by_path(fy_document_root(fyd_src), src_path, FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_src, NULL); + + ret = fy_node_insert(fyn_tgt, fyn_src); + UNIT_ASSERT_EQUAL(ret, 0); + + ret = fy_document_set_parent(fyd_tgt, fyd_src); + UNIT_ASSERT_EQUAL(ret, 0); + + fyn_emit = fy_node_by_path(fy_document_root(fyd_tgt), emit_path, FY_NT, FYNWF_DONT_FOLLOW); + UNIT_ASSERT_UNEQUAL(fyn_emit, NULL); + + output = fy_emit_node_to_string(fyn_emit, (fy_emitter_cfg_flags) (FYECF_MODE_FLOW_ONELINE | FYECF_WIDTH_INF)); + UNIT_ASSERT_UNEQUAL(output, NULL); + + fy_document_destroy(fyd_tgt); + + return output; +} + +Y_UNIT_TEST(doc_join_scalar_to_scalar) { + char *output; + + output = join_docs( + "foo", "/", /* target */ + "bar", "/", /* source */ + "/"); /* emit path */ + + UNIT_ASSERT_VALUES_EQUAL(output, TString("bar")); + free(output); +} + +Y_UNIT_TEST(doc_join_scalar_to_map) { + char *output; + + output = join_docs( + "{ foo: baz }", "/", /* target */ + "bar", "/", /* source */ + "/"); /* emit path */ + + UNIT_ASSERT_VALUES_EQUAL(output, TString("bar")); + free(output); +} + +Y_UNIT_TEST(doc_join_scalar_to_seq) { + char *output; + + output = join_docs( + "[ foo, baz ]", "/", /* target */ + "bar", "/", /* source */ + "/"); /* emit path */ + + UNIT_ASSERT_VALUES_EQUAL(output, TString("bar")); + free(output); +} + +Y_UNIT_TEST(doc_join_map_to_scalar) { + char *output; + + output = join_docs( + "foo", "/", /* target */ + "{bar: baz}", "/", /* source */ + "/"); /* emit path */ + + UNIT_ASSERT_VALUES_EQUAL(output, TString("{bar: baz}")); + free(output); +} + +Y_UNIT_TEST(doc_join_map_to_seq) { + char *output; + + output = join_docs( + "[foo, frooz]", "/", /* target */ + "{bar: baz}", "/", /* source */ + "/"); /* emit path */ + + UNIT_ASSERT_VALUES_EQUAL(output, TString("{bar: baz}")); + free(output); +} + +Y_UNIT_TEST(doc_join_map_to_map) { + char *output; + + output = join_docs( + "{foo: frooz}", "/", /* target */ + "{bar: baz}", "/", /* source */ + "/"); /* emit path */ + + UNIT_ASSERT_VALUES_EQUAL(output, TString("{foo: frooz, bar: baz}")); + free(output); +} + +Y_UNIT_TEST(doc_join_seq_to_scalar) { + char *output; + + output = join_docs( + "foo", "/", /* target */ + "[bar, baz]", "/", /* source */ + "/"); /* emit path */ + + UNIT_ASSERT_VALUES_EQUAL(output, TString("[bar, baz]")); + free(output); +} + +Y_UNIT_TEST(doc_join_seq_to_seq) { + char *output; + + output = join_docs( + "[foo, frooz]", "/", /* target */ + "[bar, baz]", "/", /* source */ + "/"); /* emit path */ + + UNIT_ASSERT_VALUES_EQUAL(output, TString("[foo, frooz, bar, baz]")); + free(output); +} + +Y_UNIT_TEST(doc_join_seq_to_map) { + char *output; + + output = join_docs( + "{foo: frooz}", "/", /* target */ + "[bar, baz]", "/", /* source */ + "/"); /* emit path */ + + UNIT_ASSERT_VALUES_EQUAL(output, TString("[bar, baz]")); + free(output); +} + +Y_UNIT_TEST(doc_join_tags) { + char *output; + + output = join_docs( + "%TAG !a! tag:a.com,2019:\n" + "---\n" + "- !a!foo\n" + " foo: bar\n", "/", + "%TAG !b! tag:b.com,2019:\n" + "---\n" + "- !b!bar\n" + " something: other\n", "/", + "/"); + + UNIT_ASSERT_VALUES_EQUAL(output, TString("[!a!foo {foo: bar}, !b!bar {something: other}]")); + free(output); +} + +Y_UNIT_TEST(doc_build_with_tags) { + struct fy_document *fyd; + struct fy_node *fyn; + struct fy_token *fyt; + char *buf; + int rc; + + /* build document */ + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* create a sequence and set it as root */ + fyn = fy_node_create_sequence(fyd); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + fy_document_set_root(fyd, fyn); + fyn = NULL; + + /* create a node, containing a new tag */ + fyn = fy_node_build_from_string(fyd, "%TAG !e! tag:example.com,2000:app/\n---\n- foo\n- !e!foo bar\n", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + /* append it to the root of the document */ + rc = fy_node_sequence_append(fy_document_root(fyd), fyn); + UNIT_ASSERT_EQUAL(rc, 0); + fyn = NULL; + + /* there must be a new tag */ + fyt = fy_document_tag_directive_lookup(fyd, "!e!"); + UNIT_ASSERT_UNEQUAL(fyt, NULL); + + /* try to build another, but with a different !e! prefix, it must fail */ + fyn = fy_node_build_from_string(fyd, "%TAG !e! tag:example.com,2019:app/\n---\n- foo\n- !e!foo bar\n", FY_NT); + UNIT_ASSERT_EQUAL(fyn, NULL); + + /* manually add a tag */ + rc = fy_document_tag_directive_add(fyd, "!f!", "tag:example.com,2019:f/"); + UNIT_ASSERT_EQUAL(rc, 0); + + /* build a node with a tag that's already in the document */ + fyn = fy_node_build_from_string(fyd, "!f!whiz frooz\n", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + /* append it to the root of the document */ + rc = fy_node_sequence_append(fy_document_root(fyd), fyn); + UNIT_ASSERT_EQUAL(rc, 0); + fyn = NULL; + + /* convert to string */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(buf, NULL); + + free(buf); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(doc_attach_check) { + struct fy_document *fyd; + struct fy_node *fyn, *fyn_seq, *fyn_map; + struct fy_node *fyn_foo, *fyn_foo2, *fyn_bar, *fyn_baz; + struct fy_node_pair *fynp; + int rc; + + /* build document */ + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* create a sequence */ + fyn_seq = fy_node_create_sequence(fyd); + UNIT_ASSERT_UNEQUAL(fyn_seq, NULL); + + /* create a mapping */ + fyn_map = fy_node_create_mapping(fyd); + UNIT_ASSERT_UNEQUAL(fyn_map, NULL); + + /* create a simple scalar node foo */ + fyn_foo = fy_node_build_from_string(fyd, "foo", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn_foo, NULL); + + /* create another simple scalar node bar */ + fyn_bar = fy_node_build_from_string(fyd, "bar", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn_bar, NULL); + + /* create another simple scalar node baz */ + fyn_baz = fy_node_build_from_string(fyd, "baz", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn_baz, NULL); + + /* create a scalar node with the same content as foo */ + fyn_foo2 = fy_node_build_from_string(fyd, "foo", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn_foo2, NULL); + + /* set the root as the sequence */ + rc = fy_document_set_root(fyd, fyn_seq); + UNIT_ASSERT_EQUAL(rc, 0); + + /* should fail since fyn_seq is now attached */ + rc = fy_document_set_root(fyd, fyn_seq); + UNIT_ASSERT_UNEQUAL(rc, 0); + + /* freeing should fail, since it's attached too */ + rc = fy_node_free(fyn_seq); + UNIT_ASSERT_UNEQUAL(rc, 0); + + /* append it to the sequence */ + rc = fy_node_sequence_append(fyn_seq, fyn_foo); + UNIT_ASSERT_EQUAL(rc, 0); + + /* freeing should fail, since it's attached to the seq */ + rc = fy_node_free(fyn_foo); + UNIT_ASSERT_UNEQUAL(rc, 0); + + /* trying to append it to the sequence again should fail */ + rc = fy_node_sequence_append(fyn_seq, fyn_foo); + UNIT_ASSERT_UNEQUAL(rc, 0); + + /* append the mapping to the sequence */ + rc = fy_node_sequence_append(fyn_seq, fyn_map); + UNIT_ASSERT_EQUAL(rc, 0); + + /* this should fail, since foo is attached to the sequence */ + rc = fy_node_mapping_append(fyn_map, fyn_foo, fyn_bar); + UNIT_ASSERT_UNEQUAL(rc, 0); + + /* this should be OK, since foo2 is not attached */ + rc = fy_node_mapping_append(fyn_map, fyn_foo2, fyn_bar); + UNIT_ASSERT_EQUAL(rc, 0); + + /* remove foo from the sequence */ + fyn = fy_node_sequence_remove(fyn_seq, fyn_foo); + UNIT_ASSERT_EQUAL(fyn, fyn_foo); + + /* trying to append the same key should fail */ + rc = fy_node_mapping_append(fyn_map, fyn_foo, NULL); + UNIT_ASSERT_UNEQUAL(rc, 0); + + /* append the baz: NULL mapping */ + rc = fy_node_mapping_append(fyn_map, fyn_baz, NULL); + UNIT_ASSERT_EQUAL(rc, 0); + + /* get the baz: null node pair */ + fynp = fy_node_mapping_lookup_pair(fyn_map, fyn_baz); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_EQUAL(fy_node_pair_key(fynp), fyn_baz); + UNIT_ASSERT_EQUAL(fy_node_pair_value(fynp), NULL); + + /* trying to set the same key in the mapping should fail */ + rc = fy_node_pair_set_key(fynp, fyn_foo); + UNIT_ASSERT_UNEQUAL(rc, 0); + + /* get the foo: bar node pair */ + fynp = fy_node_mapping_lookup_pair(fyn_map, fyn_foo); + UNIT_ASSERT_UNEQUAL(fynp, NULL); + UNIT_ASSERT_EQUAL(fy_node_pair_key(fynp), fyn_foo2); + UNIT_ASSERT_EQUAL(fy_node_pair_value(fynp), fyn_bar); + + /* we're setting the same key to the mapping, but that's OK + * since the key is replaced */ + rc = fy_node_pair_set_key(fynp, fyn_foo); + UNIT_ASSERT_EQUAL(rc, 0); + + /* fyn_foo has been freed */ + fyn_foo = NULL; + + /* convert to string */ + rc = fy_emit_document_to_fp(fyd, FYECF_MODE_FLOW_ONELINE, stderr); + UNIT_ASSERT_EQUAL(rc, 0); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(manual_scalar_esc) { + // FIXME removed \e because it works incorrectly in MS C + const std::span MANUAL_SCALAR_ESC = "\\\"\0\a\b\t\v\f\r\xc2\x85\xc2\xa0\xe2\x80\xa8\xe2\x80\xa9"; + const TStringBuf MANUAL_SCALAR_ESC_TXT = "\"\\\\\\\"\\0\\a\\b\\t\\v\\f\\r\\N\\_\\L\\P\""; + const char *what = MANUAL_SCALAR_ESC.data(); + size_t what_sz = MANUAL_SCALAR_ESC.size() - 1; + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + const char *buf2; + size_t sz2; + int rc; + + /* build document */ + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* create a manual scalar with all the escapes */ + fyn = fy_node_create_scalar(fyd, what, what_sz); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + fy_document_set_root(fyd, fyn); + fyn = NULL; + + /* emit to a buffer */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(buf, NULL); + + /* destroy the old document */ + fy_document_destroy(fyd); + fyd = NULL; + + /* verify that the resulting document is in the escaped form */ + UNIT_ASSERT_VALUES_EQUAL(buf, TString(MANUAL_SCALAR_ESC_TXT) + "\n"); + + /* now load the result back and verify that it contains exactly the same */ + fyd = fy_document_build_from_string(NULL, buf, FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* get the scalar content */ + buf2 = fy_node_get_scalar(fy_document_root(fyd), &sz2); + UNIT_ASSERT_UNEQUAL(buf2, NULL); + + /* sizes must match */ + UNIT_ASSERT_EQUAL(what_sz, sz2); + + /* and the strings too */ + rc = memcmp(what, buf2, what_sz); + UNIT_ASSERT_EQUAL(rc, 0); + + /* free the document */ + fy_document_destroy(fyd); + fyd = NULL; + + free(buf); +} + +Y_UNIT_TEST(manual_scalar_quoted) { + const TStringBuf MANUAL_SCALAR_QUOTED = "&foo"; + const TStringBuf MANUAL_SCALAR_QUOTED_TXT = "\"&foo\""; + const char *what = MANUAL_SCALAR_QUOTED.data(); + size_t what_sz = MANUAL_SCALAR_QUOTED.length(); + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + const char *buf2; + size_t sz2; + int rc; + + /* build document */ + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* create a manual scalar with all the escapes */ + fyn = fy_node_create_scalar(fyd, what, what_sz); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + fy_document_set_root(fyd, fyn); + fyn = NULL; + + /* emit to a buffer */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(buf, NULL); + + /* destroy the old document */ + fy_document_destroy(fyd); + fyd = NULL; + + /* verify that the resulting document is in the escaped form */ + UNIT_ASSERT_VALUES_EQUAL(buf, TString(MANUAL_SCALAR_QUOTED_TXT) + "\n"); + + /* now load the result back and verify that it contains exactly the same */ + fyd = fy_document_build_from_string(NULL, buf, FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* get the scalar content */ + buf2 = fy_node_get_scalar(fy_document_root(fyd), &sz2); + UNIT_ASSERT_UNEQUAL(buf2, NULL); + + /* sizes must match */ + UNIT_ASSERT_EQUAL(what_sz, sz2); + + /* and the strings too */ + rc = memcmp(what, buf2, what_sz); + UNIT_ASSERT_EQUAL(rc, 0); + + /* free the document */ + fy_document_destroy(fyd); + fyd = NULL; + + free(buf); +} + +Y_UNIT_TEST(manual_scalar_copy) { + const TStringBuf MANUAL_SCALAR_COPY = "foo"; + const char *what = MANUAL_SCALAR_COPY.data(); + size_t what_sz = MANUAL_SCALAR_COPY.length(); + char *what_copy; + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + /* build document */ + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + what_copy = (char*)malloc(what_sz); + UNIT_ASSERT_UNEQUAL(what_copy, NULL); + memcpy(what_copy, what, what_sz); + + /* create a manual scalar with all the escapes */ + fyn = fy_node_create_scalar_copy(fyd, what_copy, what_sz); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + /* free the data */ + free(what_copy); + + fy_document_set_root(fyd, fyn); + fyn = NULL; + + /* emit to a buffer */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(buf, NULL); + + /* verify that the resulting document is the one we used + '\n' */ + UNIT_ASSERT_VALUES_EQUAL(buf, TString(MANUAL_SCALAR_COPY) + "\n"); + + /* destroy the old document */ + fy_document_destroy(fyd); + fyd = NULL; + + free(buf); + +} + +Y_UNIT_TEST(manual_scalarf) { + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + /* build document */ + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* create a manual scalar using the printf interface */ + fyn = fy_node_create_scalarf(fyd, "foo%d", 13); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + fy_document_set_root(fyd, fyn); + fyn = NULL; + + /* emit to a buffer */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(buf, NULL); + + /* verify that the resulting document is the one we used + '\n' */ + UNIT_ASSERT_VALUES_EQUAL(buf, TString("foo13\n")); + + /* destroy the old document */ + fy_document_destroy(fyd); + fyd = NULL; + + free(buf); +} + +Y_UNIT_TEST(manual_valid_anchor) { + struct fy_document *fyd; + struct fy_node *fyn; + int ret; + + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn = fy_node_create_sequence(fyd); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + fyn = fy_node_create_scalar(fyd, "foo", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + ret = fy_node_sequence_append(fy_document_root(fyd), fyn); + UNIT_ASSERT_EQUAL(ret, 0); + + /* create a valid anchor */ + ret = fy_node_set_anchor(fyn, "foo", FY_NT); + UNIT_ASSERT_EQUAL(ret, 0); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(manual_invalid_anchor) { + struct fy_document *fyd; + struct fy_node *fyn; + int ret; + + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn = fy_node_create_sequence(fyd); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + fyn = fy_node_create_scalar(fyd, "foo", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + ret = fy_node_sequence_append(fy_document_root(fyd), fyn); + UNIT_ASSERT_EQUAL(ret, 0); + + /* create an invalid anchor */ + ret = fy_node_set_anchor(fyn, "*foo", FY_NT); + UNIT_ASSERT_UNEQUAL(ret, 0); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(manual_anchor_removal) { + struct fy_document *fyd; + struct fy_node *fyn; + int ret; + + fyd = fy_document_create(NULL); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn = fy_node_create_sequence(fyd); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + fyn = fy_node_create_scalar(fyd, "foo", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn, NULL); + + ret = fy_node_sequence_append(fy_document_root(fyd), fyn); + UNIT_ASSERT_EQUAL(ret, 0); + + /* create a valid anchor */ + ret = fy_node_set_anchor(fyn, "foo", FY_NT); + UNIT_ASSERT_EQUAL(ret, 0); + + fprintf(stderr, "---\n# with anchor\n"); + fy_emit_document_to_fp(fyd, FYECF_MODE_FLOW_ONELINE, stderr); + + /* should fail (an anchor already exists) */ + ret = fy_node_set_anchor(fyn, "bar", FY_NT); + UNIT_ASSERT_UNEQUAL(ret, 0); + + /* should succeed */ + ret = fy_node_remove_anchor(fyn); + UNIT_ASSERT_EQUAL(ret, 0); + + fprintf(stderr, "---\n# without anchor\n"); + fy_emit_document_to_fp(fyd, FYECF_MODE_FLOW_ONELINE, stderr); + + fy_document_destroy(fyd); +} + +Y_UNIT_TEST(manual_block_flow_mix) { + struct fy_document *fyd; + struct fy_node *fyn_mapping, *fyn_key, *fyn_value; + char *buf; + int ret; + + fyd = fy_document_build_from_string(NULL, "--- &root\n{\n}\n", FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn_mapping = fy_document_root(fyd); + UNIT_ASSERT_UNEQUAL(fyn_mapping, NULL); + + UNIT_ASSERT(fy_node_is_mapping(fyn_mapping) == true); + + fyn_key = fy_node_create_scalar(fyd, "key", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn_key, NULL); + + fyn_value = fy_node_build_from_string(fyd, "|\n literal\n", FY_NT); + UNIT_ASSERT_UNEQUAL(fyn_value, NULL); + + ret = fy_node_mapping_append(fyn_mapping, fyn_key, fyn_value); + UNIT_ASSERT_EQUAL(ret, 0); + + /* emit document to buffer */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(buf, NULL); + + /* destroy the first document */ + fy_document_destroy(fyd); + fyd = NULL; + + /* read the emitted document back */ + fyd = fy_document_build_from_string(NULL, buf, FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* compare with expected result */ + UNIT_ASSERT_VALUES_EQUAL(fy_node_get_scalar0(fy_node_by_path(fy_document_root(fyd), "/key", FY_NT, FYNWF_DONT_FOLLOW)), "literal\n"); + + /* destroy the second document */ + fy_document_destroy(fyd); + fyd = NULL; + + free(buf); + +} + +/* FIXME + * This test is disabled because we can't make compatible + * alloca-based API on windows because original library + * uses gcc inline block extensions it will be uncommented + * when new API will be introduced + */ +#if false +Y_UNIT_TEST(alloca_check) { + struct fy_document *fyd; + char *buf; + const char *abuf; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "{ " + "foo: 10, bar : [ ten, 20 ], baz:{ frob: boo, deep: { deeper: yet } }, " + "}", FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + /* fy_emit_document_to_string*() */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(buf, NULL); + abuf = fy_emit_document_to_string_alloca(fyd, FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(abuf, NULL); + UNIT_ASSERT_VALUES_EQUAL(buf, abuf); + free(buf); + + /* fy_emit_node_to_string*() */ + buf = fy_emit_node_to_string(fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW), FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(buf, NULL); + abuf = fy_emit_node_to_string_alloca(fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW), FYECF_MODE_FLOW_ONELINE); + UNIT_ASSERT_UNEQUAL(abuf, NULL); + UNIT_ASSERT_VALUES_EQUAL(buf, abuf); + free(buf); + + /* path check eq */ + buf = fy_node_get_path(fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW)); + UNIT_ASSERT_UNEQUAL(buf, NULL); + UNIT_ASSERT_VALUES_EQUAL(buf, "/foo"); + abuf = fy_node_get_path_alloca(fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW)); + UNIT_ASSERT_UNEQUAL(abuf, NULL); + UNIT_ASSERT_VALUES_EQUAL(abuf, "/foo"); + UNIT_ASSERT_VALUES_EQUAL(buf, abuf); + free(buf); + + /* check that a bad path is "" */ + abuf = fy_node_get_path_alloca(NULL); + UNIT_ASSERT_UNEQUAL(abuf, NULL); + UNIT_ASSERT_VALUES_EQUAL(abuf, ""); + + fy_document_destroy(fyd); + +} +#endif + +Y_UNIT_TEST(scanf_check) { + struct fy_document *fyd; + struct fy_node *fyn_root; + int ret, ival; + char sval[256]; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "{ " + "foo: 10, bar : 20, baz:{ frob: boo }, " + "frooz: [ 1, { key: value }, three ]" + "}", FY_NT); + UNIT_ASSERT_UNEQUAL(fyd, NULL); + + fyn_root = fy_document_root(fyd); + UNIT_ASSERT_UNEQUAL(fyn_root, NULL); + + /* check scanf accesses to scalars */ + ret = fy_node_scanf(fyn_root, "/foo %d", &ival); + UNIT_ASSERT_EQUAL(ret, 1); + UNIT_ASSERT_EQUAL(ival, 10); + + ret = fy_node_scanf(fyn_root, "/bar %d", &ival); + UNIT_ASSERT_EQUAL(ret, 1); + UNIT_ASSERT_EQUAL(ival, 20); + + ret = fy_node_scanf(fyn_root, "/baz/frob %s", sval); + UNIT_ASSERT_EQUAL(ret, 1); + UNIT_ASSERT_VALUES_EQUAL(sval, "boo"); + + ret = fy_node_scanf(fyn_root, "/frooz/0 %d", &ival); + UNIT_ASSERT_EQUAL(ret, 1); + UNIT_ASSERT_EQUAL(ival, 1); + + ret = fy_node_scanf(fyn_root, "/frooz/1/key %s", sval); + UNIT_ASSERT_EQUAL(ret, 1); + UNIT_ASSERT_VALUES_EQUAL(sval, "value"); + + ret = fy_node_scanf(fyn_root, "/frooz/2 %s", sval); + UNIT_ASSERT_EQUAL(ret, 1); + UNIT_ASSERT_VALUES_EQUAL(sval, "three"); + + fy_document_destroy(fyd); +} + +} |