validate canons without yatest_common

6 files changed, 534 insertions, 0 deletions

diff --git a/library/cpp/eventlog/dumper/common.cpp b/library/cpp/eventlog/dumper/common.cpp
new file mode 100644
index 0000000000..eebe3b6ee3
--- /dev/null
+++ b/library/cpp/eventlog/dumper/common.cpp
@@ -0,0 +1,81 @@
+#include "common.h"
+
+#include <contrib/libs/re2/re2/re2.h>
+
+#include <util/datetime/base.h>
+#include <util/datetime/parser.h>
+#include <util/string/cast.h>
+
+static time_t RecentTime(int h, int m, int s) {
+    time_t now = time(nullptr);
+    tm tmTmp;
+    localtime_r(&now, &tmTmp);
+    tmTmp.tm_hour = h;
+    tmTmp.tm_min = m;
+    tmTmp.tm_sec = s;
+    time_t today = mktime(&tmTmp);
+    tmTmp.tm_mday -= 1;
+    time_t yesterday = mktime(&tmTmp);
+    return today <= now ? today : yesterday;
+}
+
+static bool ParseRecentTime(const TString& str, time_t& result) {
+    RE2 RecentTimePattern("(\\d{1,2}):(\\d{2})(?::(\\d{2}))?");
+    re2::StringPiece hStr, mStr, sStr;
+    if (!RE2::FullMatch({str.data(), str.size()}, RecentTimePattern, &hStr, &mStr, &sStr)) {
+        return false;
+    }
+    int h = FromString<int>(hStr.data(), hStr.length());
+    int m = FromString<int>(mStr.data(), mStr.length());
+    int s = FromString<int>(sStr.data(), sStr.length(), 0);
+    if (h > 23 || m > 59 || s > 59) {
+        return false;
+    }
+    result = RecentTime(h, m, s);
+    return true;
+}
+
+namespace {
+    class TDefaultOffset8601Parser: public TIso8601DateTimeParser {
+    public:
+        TDefaultOffset8601Parser(int offsetHours) {
+            DateTimeFields.ZoneOffsetMinutes = offsetHours * 60;
+        }
+    };
+} // namespace
+
+static bool ParseISO8601DateTimeWithDefaultOffset(TStringBuf str, int offsetHours, time_t& result) {
+    TDefaultOffset8601Parser parser{offsetHours};
+
+    if (!parser.ParsePart(str.data(), str.size())) {
+        return false;
+    }
+
+    const TInstant instant = parser.GetResult(TInstant::Max());
+    if (instant == TInstant::Max()) {
+        return false;
+    }
+
+    result = instant.TimeT();
+    return true;
+}
+
+ui64 ParseTime(const TString& str, ui64 defValue, int offset) {
+    if (!str) {
+        return defValue;
+    }
+
+    time_t utcTime;
+
+    if (ParseISO8601DateTimeWithDefaultOffset(str, offset, utcTime)) {
+        return (ui64)utcTime * 1000000;
+    }
+
+    if (ParseRecentTime(str, utcTime)) {
+        return (ui64)utcTime * 1000000;
+    }
+
+    // if conversion fails, TryFromString leaves defValue unchanged
+    TryFromString<ui64>(str, defValue);
+    return defValue;
+}
diff --git a/library/cpp/eventlog/dumper/common.h b/library/cpp/eventlog/dumper/common.h
new file mode 100644
index 0000000000..839fc7b221
--- /dev/null
+++ b/library/cpp/eventlog/dumper/common.h
@@ -0,0 +1,8 @@
+#pragma once
+
+#include <util/generic/string.h>
+
+/**
+ * Returns microseconds since the epoch.
+ */
+ui64 ParseTime(const TString& str, ui64 defValue, int offset /* timezone offset, default MSK */ = 3);
diff --git a/library/cpp/eventlog/dumper/evlogdump.cpp b/library/cpp/eventlog/dumper/evlogdump.cpp
new file mode 100644
index 0000000000..003b412265
--- /dev/null
+++ b/library/cpp/eventlog/dumper/evlogdump.cpp
@@ -0,0 +1,414 @@
+#include "common.h"
+#include "evlogdump.h"
+
+#include <library/cpp/eventlog/evdecoder.h>
+#include <library/cpp/eventlog/iterator.h>
+#include <library/cpp/eventlog/logparser.h>
+
+#include <library/cpp/getopt/last_getopt.h>
+
+#ifndef NO_SVN_DEPEND
+#include <library/cpp/svnversion/svnversion.h>
+#endif
+
+#include <util/datetime/base.h>
+#include <util/generic/ptr.h>
+#include <util/stream/file.h>
+#include <util/string/cast.h>
+#include <util/string/type.h>
+
+using namespace NEventLog;
+namespace nlg = NLastGetopt;
+
+IFrameFilterRef CreateDurationFrameFilter(const TString& params) {
+    TStringBuf d(params);
+    TStringBuf l, r;
+    d.Split(':', l, r);
+    TDuration min = l.empty() ? TDuration::Zero() : FromString<TDuration>(l);
+    TDuration max = r.empty() ? TDuration::Max() : FromString<TDuration>(r);
+
+    if (min > TDuration::Zero() || max < TDuration::Max()) {
+        return new TDurationFrameFilter(min.GetValue(), max.GetValue());
+    }
+
+    return nullptr;
+}
+
+IFrameFilterRef CreateFrameIdFilter(const ui32 frameId) {
+    return new TFrameIdFrameFilter(frameId);
+}
+
+IFrameFilterRef CreateContainsEventFrameFilter(const TString& args, const IEventFactory* factory) {
+    return new TContainsEventFrameFilter(args, factory);
+}
+
+void ListAllEvents(IEventFactory* factory, IOutputStream* out) {
+    Y_ENSURE(out);
+
+    for (TEventClass eventClass = factory->EventClassBegin(); eventClass < factory->EventClassEnd(); eventClass++) {
+        THolder<TEvent> event(factory->CreateLogEvent(eventClass));
+
+        const TStringBuf& name = event->GetName();
+        if (name) {
+            (*out) << name << "\n";
+        }
+    }
+}
+
+void ListEventFieldsByEventId(const TEventClass eventClass, IEventFactory* factory, IOutputStream* out) {
+    THolder<TEvent> event(factory->CreateLogEvent(eventClass));
+    const NProtoBuf::Message* message = event->GetProto();
+    const google::protobuf::Descriptor* descriptor = message->GetDescriptor();
+
+    (*out) << descriptor->DebugString();
+}
+
+void ListEventFields(const TString& eventName, IEventFactory* factory, IOutputStream* out) {
+    Y_ENSURE(out);
+
+    TEventClass eventClass;
+
+    try {
+        eventClass = factory->ClassByName(eventName);
+    } catch (yexception& e) {
+        if (!TryFromString<TEventClass>(eventName, eventClass)) {
+            (*out) << "Cannot dervie event class from event name: " << eventName << Endl;
+            return;
+        }
+    }
+
+     ListEventFieldsByEventId(eventClass, factory, out);
+}
+
+int PrintHelpEvents(const TString& helpEvents, IEventFactory* factory) {
+    if (helpEvents == "all") {
+        ListAllEvents(factory, &Cout);
+    } else if (IsNumber(helpEvents)) {
+        ListEventFieldsByEventId(IntFromString<ui32, 10>(helpEvents), factory, &Cout);
+    } else {
+        ListEventFields(helpEvents, factory, &Cout);
+    }
+
+    return 0;
+}
+
+int IterateEventLog(IEventFactory* fac, IEventProcessor* proc, int argc, const char** argv) {
+    class TProxy: public ITunableEventProcessor {
+    public:
+        TProxy(IEventProcessor* proc)
+            : Processor_(proc)
+        {
+        }
+
+        void AddOptions(NLastGetopt::TOpts&) override {
+        }
+
+        void SetOptions(const TEvent::TOutputOptions& options) override {
+            Processor_->SetOptions(options);
+        }
+
+        void ProcessEvent(const TEvent* ev) override {
+            Processor_->ProcessEvent(ev);
+        }
+
+        bool CheckedProcessEvent(const TEvent* ev) override {
+            return Processor_->CheckedProcessEvent(ev);
+        }
+
+    private:
+        IEventProcessor* Processor_ = nullptr;
+    };
+
+    TProxy proxy(proc);
+    return IterateEventLog(fac, &proxy, argc, argv);
+}
+
+int IterateEventLog(IEventFactory* fac, ITunableEventProcessor* proc, int argc, const char** argv) {
+    nlg::TOpts opts;
+    opts.AddHelpOption('?');
+    opts.AddHelpOption('h');
+    opts.SetTitle(
+        "Search EventLog dumper\n\n"
+        "Examples:\n"
+        "evlogdump -s 1228484839332219 -e 1228484840332219 event_log\n"
+        "evlogdump -s 2008-12-12T12:00:00+03 -e 2008-12-12T12:05:00+03 event_log\n"
+        "evlogdump -s 12:40 -e 12:55 event_log\n"
+        "evlogdump -i 301,302,303 event_log\n"
+        "evlogdump -i SubSourceResponse,SubSourceOk,SubSourceError event_log\n"
+        "evlogdump -d 40ms:60ms event_log\n"
+        "evlogdump -c ReqId:ReqId:1563185655856304-30407004790538173600035-vla1-0671-p1\n"
+        "\n"
+        "Fine manual: https://nda.ya.ru/3Tpo3L\n"
+        "\n"
+        "If in trouble using this (or bugs encountered),\n"
+        "please don't hesitate to ask middlesearch/basesearch dev team\n"
+        "(vmordovin@, mvel@, etc)\n");
+
+    TString start;
+    opts.AddLongOption(
+            's', "start-time",
+            "Start time (Unix time in microseconds, ISO8601, or HH:MM[:SS] in the last 24 hours).\n"
+            "Long frames can produce inaccurate cropping and event loss in dump (see SEARCH-5576)")
+        .Optional()
+        .StoreResult(&start);
+
+    TString end;
+    opts.AddLongOption(
+            'e', "end-time",
+            "End time (Unix time in microseconds, ISO8601, or HH:MM[:SS] in the last 24 hours)")
+        .Optional()
+        .StoreResult(&end);
+
+    TOptions o;
+
+    opts.AddLongOption(
+            'm', "max-request-duration",
+            "Max duration of a request in the log, in us")
+        .Optional()
+        .StoreResult(&o.MaxRequestDuration);
+
+    bool oneFrame = false;
+    opts.AddLongOption(
+            'f', "one-frame",
+            "Treat input file as single frame dump (e.g. from gdb)")
+        .NoArgument()
+        .Optional()
+        .StoreValue(&oneFrame, true);
+
+    opts.AddLongOption(
+            'v', "version",
+            "Print program version")
+        .NoArgument()
+        .Optional()
+        .Handler(&nlg::PrintVersionAndExit);
+
+    TString includeEventList;
+    opts.AddLongOption(
+            'i', "event-list",
+            "Comma-separated list of included event class IDs or names "
+            "(e.g. 265,287 or MainTaskError,ContextCreated)")
+        .Optional()
+        .StoreResult(&includeEventList)
+        .StoreValue(&o.EnableEvents, true);
+
+    TString durationFilterStr;
+    opts.AddLongOption(
+            'd', "duration",
+            "DurationMin[:DurationMax] (values must contain a unit, valid examples are: 50us, 50ms, 50s)\n"
+            "(show frames with duration greater or equal DurationMin [and less or equal than DurationMax])")
+        .Optional()
+        .StoreResult(&durationFilterStr);
+
+    const ui32 INVALID_FRAME_ID = ui32(-1);
+    ui32 frameIdFilter = INVALID_FRAME_ID;
+    opts.AddLongOption(
+            'n', "frame-id",
+            "Filter frame with given id\n")
+        .Optional()
+        .StoreResult(&frameIdFilter);
+
+    TString excludeEventList;
+    opts.AddLongOption(
+            'x', "exclude-list",
+            "Comma-separated list of excluded event class IDs or names (see also --event-list option)")
+        .Optional()
+        .StoreResult(&excludeEventList)
+        .StoreValue(&o.EnableEvents, false);
+
+    opts.AddLongOption(
+            'o', "frame-order",
+            "Order events by time only inside a frame (faster)")
+        .NoArgument()
+        .Optional()
+        .StoreValue(&o.ForceWeakOrdering, true);
+
+    opts.AddLongOption(
+            'O', "global-order",
+            "Globally but lossy (may lose some frames) order events by time only (slower)")
+        .NoArgument()
+        .Optional()
+        .StoreValue(&o.ForceStrongOrdering, true);
+
+    opts.AddLongOption(
+        "lossless-strong-order",
+        "Globally order events by time only (super slow and memory heavy)")
+        .NoArgument()
+        .Optional()
+        .StoreValue(&o.ForceLosslessStrongOrdering, true)
+        .StoreValue(&o.ForceStrongOrdering, true);
+
+    opts.AddLongOption(
+            'S', "stream",
+            "Force stream mode (for log files with invalid ending)")
+        .NoArgument()
+        .Optional()
+        .StoreValue(&o.ForceStreamMode, true);
+
+    opts.AddLongOption(
+            't', "tail",
+            "Open file like `tail -f` does")
+        .NoArgument()
+        .Optional()
+        .StoreValue(&o.TailFMode, true);
+
+    bool unescaped = false;
+    opts.AddLongOption(
+            'u', "unescaped",
+            "Don't escape \\t, \\n chars in message fields")
+        .NoArgument()
+        .StoreValue(&unescaped, true);
+
+    bool json = false;
+    opts.AddLongOption(
+            'j', "json",
+            "Print messages as JSON values")
+        .NoArgument()
+        .StoreValue(&json, true);
+
+    TEvent::TOutputOptions outputOptions;
+    opts.AddLongOption(
+            'r', "human-readable",
+            "Print some fields (e.g. timestamp) in human-readable format, add time offsets")
+        .NoArgument()
+        .StoreValue(&outputOptions.HumanReadable, true);
+
+    //Supports only fields of type string
+    TString containsEvent;
+    opts.AddLongOption(
+            'c', "contains-event",
+            "Only print frames that contain events whose particular field's value matches given value.\n"
+            "Match group should be provided in format: EventName:FieldName:ValueToMatch\n"
+            "If more than one match group is provided, they should be separated by / delimiter.\n"
+            "Nested fields are supported through \'.\' in FieldName.\n"
+            "Symbols \'/\' and \':\' in EventName, FieldName or ValueToMatch must be escaped.\n"
+            "NOTE: bool values are 1 and 0.")
+        .Optional()
+        .StoreResult(&containsEvent);
+
+    TString helpEvents;
+    opts.AddLongOption("help-events")
+        .RequiredArgument("EVENT, EVENT_ID or string \"all\"")
+        .Optional()
+        .StoreResult(&helpEvents);
+
+    proc->AddOptions(opts);
+
+    opts.SetFreeArgsMin(0);
+    opts.SetFreeArgsMax(1);
+    opts.SetFreeArgTitle(0, "<eventlog>", "Event log file");
+
+    try {
+        const nlg::TOptsParseResult optsRes(&opts, argc, argv);
+
+        if (helpEvents) {
+            return PrintHelpEvents(helpEvents, fac);
+        }
+
+        TVector<TString> freeArgs = optsRes.GetFreeArgs();
+        if (freeArgs) {
+            o.FileName = freeArgs[0];
+        }
+
+        ui32 conditionsHappened = 0;
+
+        if (frameIdFilter != INVALID_FRAME_ID) {
+            ++conditionsHappened;
+        }
+        if (containsEvent) {
+            ++conditionsHappened;
+        }
+        if (durationFilterStr) {
+            ++conditionsHappened;
+        }
+
+        if (conditionsHappened > 1) {
+            throw nlg::TUsageException() << "You can only use no more than one of frame id, duration or contains event frame filters.";
+        }
+
+        if (frameIdFilter != INVALID_FRAME_ID) {
+            o.FrameFilter = CreateFrameIdFilter(frameIdFilter);
+        } else if (durationFilterStr) {
+            o.FrameFilter = CreateDurationFrameFilter(durationFilterStr);
+        } else if (containsEvent) {
+            o.FrameFilter = CreateContainsEventFrameFilter(containsEvent, fac);
+        }
+
+        // handle some inconsistencies
+        if (o.ForceWeakOrdering && o.ForceStrongOrdering) {
+            throw nlg::TUsageException() << "You can use either strong (-O) or weak (-o) ordering. ";
+        }
+        if (includeEventList && excludeEventList) {
+            throw nlg::TUsageException() << "You can use either include (-i) or exclude (-x) events list. ";
+        }
+        if (unescaped && json) {
+            throw nlg::TUsageException() << "You can use either unescaped (-u) or json (-j) output format. ";
+        }
+
+        proc->CheckOptions();
+    } catch (...) {
+        Cerr << "Usage error: " << CurrentExceptionMessage() << Endl;
+        return 1;
+    }
+
+    o.EvList = o.EnableEvents ? includeEventList : excludeEventList;
+    if (json) {
+        outputOptions.OutputFormat = TEvent::TOutputFormat::Json;
+    } else if (unescaped) {
+        outputOptions.OutputFormat = TEvent::TOutputFormat::TabSeparatedRaw;
+    } else {
+        outputOptions.OutputFormat = TEvent::TOutputFormat::TabSeparated;
+    }
+
+    IEventProcessor* eventProcessor = NEvClass::Processor();
+    // FIXME(mvel): A little of hell here: `proc` and `eventProcessor` are `IEventProcessor`s
+    // So we need to set options for BOTH!
+
+    eventProcessor->SetOptions(outputOptions);
+    proc->SetOptions(outputOptions);
+    proc->SetEventProcessor(eventProcessor);
+
+    if (oneFrame) {
+        THolder<IInputStream> fileInput;
+
+        // this is for coredumps analysis, usage:
+        // gdb: dump binary memory framefile Data_ Data_ + Pos_
+        // evlogdump -f framefile
+        IInputStream* usedInput = nullptr;
+
+        if (o.FileName.size()) {
+            fileInput.Reset(new TUnbufferedFileInput(o.FileName));
+            usedInput = fileInput.Get();
+        } else {
+            usedInput = &Cin;
+        }
+
+        try {
+            for (;;) {
+                proc->ProcessEvent(DecodeEvent(*usedInput, true, 0, nullptr, fac).Get());
+            }
+        } catch (...) {
+        }
+
+        return 0;
+    }
+
+    o.StartTime = ParseTime(start, MIN_START_TIME);
+    o.EndTime = ParseTime(end, MAX_END_TIME);
+
+    try {
+        THolder<IIterator> it = CreateIterator(o, fac);
+
+        while (const auto ev = it->Next()) {
+            if (!proc->CheckedProcessEvent(ev.Get())) {
+                break;
+            }
+        }
+
+        return 0;
+    } catch (...) {
+        Cout.Flush();
+        Cerr << "Error occured: " << CurrentExceptionMessage() << Endl;
+    }
+
+    return 1;
+}
diff --git a/library/cpp/eventlog/dumper/evlogdump.h b/library/cpp/eventlog/dumper/evlogdump.h
new file mode 100644
index 0000000000..eb15057374
--- /dev/null
+++ b/library/cpp/eventlog/dumper/evlogdump.h
@@ -0,0 +1,11 @@
+#pragma once
+
+#include "tunable_event_processor.h"
+
+#include <library/cpp/eventlog/eventlog.h>
+
+int IterateEventLog(IEventFactory* fac, IEventProcessor* proc, int argc, const char** argv);
+int IterateEventLog(IEventFactory* fac, ITunableEventProcessor* proc, int argc, const char** argv);
+
+// added for using in infra/libs/logger/log_printer.cpp
+int PrintHelpEvents(const TString& helpEvents, IEventFactory* factory);
diff --git a/library/cpp/eventlog/dumper/tunable_event_processor.cpp b/library/cpp/eventlog/dumper/tunable_event_processor.cpp
new file mode 100644
index 0000000000..8333089cb5
--- /dev/null
+++ b/library/cpp/eventlog/dumper/tunable_event_processor.cpp
@@ -0,0 +1 @@
+#include "tunable_event_processor.h"
diff --git a/library/cpp/eventlog/dumper/tunable_event_processor.h b/library/cpp/eventlog/dumper/tunable_event_processor.h
new file mode 100644
index 0000000000..b56eae9222
--- /dev/null
+++ b/library/cpp/eventlog/dumper/tunable_event_processor.h
@@ -0,0 +1,19 @@
+#pragma once
+
+#include <library/cpp/eventlog/eventlog.h>
+
+namespace NLastGetopt {
+    class TOpts;
+}
+
+class ITunableEventProcessor: public IEventProcessor {
+public:
+    virtual void SetEventProcessor(IEventProcessor* /*processor*/) {
+    }
+
+    virtual void AddOptions(NLastGetopt::TOpts& opts) = 0;
+    virtual void CheckOptions() {
+    }
+    virtual ~ITunableEventProcessor() {
+    }
+};