diff options
author | max42 <max42@yandex-team.com> | 2023-07-29 00:02:16 +0300 |
---|---|---|
committer | max42 <max42@yandex-team.com> | 2023-07-29 00:02:16 +0300 |
commit | 73b89de71748a21e102d27b9f3ed1bf658766cb5 (patch) | |
tree | 188bbd2d622fa91cdcbb1b6d6d77fbc84a0646f5 /yt/cpp/mapreduce/interface/job_statistics.h | |
parent | 528e321bcc2a2b67b53aeba58c3bd88305a141ee (diff) | |
download | ydb-73b89de71748a21e102d27b9f3ed1bf658766cb5.tar.gz |
YT-19210: expose YQL shared library for YT.
After this, a new target libyqlplugin.so appears. in open-source cmake build.
Diff in open-source YDB repo looks like the following: https://paste.yandex-team.ru/f302bdb4-7ef2-4362-91c7-6ca45f329264
Diffstat (limited to 'yt/cpp/mapreduce/interface/job_statistics.h')
-rw-r--r-- | yt/cpp/mapreduce/interface/job_statistics.h | 268 |
1 files changed, 268 insertions, 0 deletions
diff --git a/yt/cpp/mapreduce/interface/job_statistics.h b/yt/cpp/mapreduce/interface/job_statistics.h new file mode 100644 index 0000000000..8af751604f --- /dev/null +++ b/yt/cpp/mapreduce/interface/job_statistics.h @@ -0,0 +1,268 @@ +#pragma once + +/// +/// @file yt/cpp/mapreduce/interface/job_statistics.h +/// +/// Header containing classes and utility functions to work with +/// [job statistics](https://docs.yandex-team.ru/yt/problems/jobstatistics). + +#include "fwd.h" + +#include <library/cpp/yson/node/node.h> + +#include <util/system/defaults.h> +#include <util/generic/maybe.h> +#include <util/generic/ptr.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////// + +/// +/// @brief Convert i64 representation of statistics to other type. +/// +/// Library defines this template for types TDuration and i64. +/// Users may define it for their types. +/// +/// @see @ref NYT::TJobStatistics::GetStatisticsAs method. +template <typename T> +T ConvertJobStatisticsEntry(i64 value); + +//////////////////////////////////////////////////////////////////// + +/// Class representing a collection of job statistics. +class TJobStatistics +{ +public: + /// + /// Construct empty statistics. + TJobStatistics(); + + /// + /// Construct statistics from statistics node. + TJobStatistics(const NYT::TNode& statistics); + + TJobStatistics(const TJobStatistics& jobStatistics); + TJobStatistics(TJobStatistics&& jobStatistics); + + TJobStatistics& operator=(const TJobStatistics& jobStatistics); + TJobStatistics& operator=(TJobStatistics&& jobStatistics); + + ~TJobStatistics(); + + /// + /// @brief Filter statistics by task name. + /// + /// @param taskNames What task names to include (empty means all). + TJobStatistics TaskName(TVector<TTaskName> taskNames) const; + + /// + /// @brief Filter statistics by job state. + /// + /// @param filter What job states to include (empty means all). + /// + /// @note Default statistics include only (successfully) completed jobs. + TJobStatistics JobState(TVector<EJobState> filter) const; + + /// + /// @brief Filter statistics by job type. + /// + /// @param filter What job types to include (empty means all). + /// + /// @deprecated Use @ref TJobStatistics::TaskName instead. + /// + /// @see https://yt.yandex-team.ru/docs/description/mr/jobs#obshaya-shema + TJobStatistics JobType(TVector<EJobType> filter) const; + + /// + /// @brief Check that given statistics exist. + /// + /// @param name Slash separated statistics name, e.g. "time/total" (like it appears in web interface). + bool HasStatistics(TStringBuf name) const; + + /// + /// @brief Get statistics by name. + /// + /// @param name Slash separated statistics name, e.g. "time/total" (like it appears in web interface). + /// + /// @note If statistics is missing an exception is thrown. If because of filters + /// no fields remain the returned value is empty (all fields are `Nothing`). + /// + /// @note We don't use `TMaybe<TJobStatisticsEntry>` here; + /// instead, @ref NYT::TJobStatisticsEntry methods return `TMaybe<i64>`, + /// so user easier use `.GetOrElse`: + /// ``` + /// jobStatistics.GetStatistics("some/statistics/name").Max().GetOrElse(0); + /// ``` + TJobStatisticsEntry<i64> GetStatistics(TStringBuf name) const; + + /// + /// @brief Get statistics by name. + /// + /// @param name Slash separated statistics name, e.g. "time/total" (like it appears in web interface). + /// + /// @note In order to use `GetStatisticsAs` method, @ref NYT::ConvertJobStatisticsEntry function must be defined + /// (the library defines it for `i64` and `TDuration`, user may define it for other types). + template <typename T> + TJobStatisticsEntry<T> GetStatisticsAs(TStringBuf name) const; + + /// + /// Get (slash separated) names of statistics. + TVector<TString> GetStatisticsNames() const; + + /// + /// @brief Check if given custom statistics exists. + /// + /// @param name Slash separated custom statistics name. + bool HasCustomStatistics(TStringBuf name) const; + + /// + /// @brief Get custom statistics (those the user can write in job with @ref NYT::WriteCustomStatistics). + /// + /// @param name Slash separated custom statistics name. + TJobStatisticsEntry<i64> GetCustomStatistics(TStringBuf name) const; + + /// + /// @brief Get custom statistics (those the user can write in job with @ref NYT::WriteCustomStatistics). + /// + /// @param name Slash separated custom statistics name. + template <typename T> + TJobStatisticsEntry<T> GetCustomStatisticsAs(TStringBuf name) const; + + /// + /// Get names of all custom statistics. + TVector<TString> GetCustomStatisticsNames() const; + +private: + class TData; + struct TFilter; + + struct TDataEntry { + i64 Max; + i64 Min; + i64 Sum; + i64 Count; + }; + + static const TString CustomStatisticsNamePrefix_; + +private: + TJobStatistics(::TIntrusivePtr<TData> data, ::TIntrusivePtr<TFilter> filter); + + TMaybe<TDataEntry> GetStatisticsImpl(TStringBuf name) const; + +private: + ::TIntrusivePtr<TData> Data_; + ::TIntrusivePtr<TFilter> Filter_; + +private: + template<typename T> + friend class TJobStatisticsEntry; +}; + +//////////////////////////////////////////////////////////////////// + +/// Class representing single statistic. +template <typename T> +class TJobStatisticsEntry +{ +public: + TJobStatisticsEntry(TMaybe<TJobStatistics::TDataEntry> data) + : Data_(std::move(data)) + { } + + /// Sum of the statistic over all jobs. + TMaybe<T> Sum() const + { + if (Data_) { + return ConvertJobStatisticsEntry<T>(Data_->Sum); + } + return Nothing(); + } + + /// @brief Average of the statistic over all jobs. + /// + /// @note Only jobs that emitted statistics are taken into account. + TMaybe<T> Avg() const + { + if (Data_ && Data_->Count) { + return ConvertJobStatisticsEntry<T>(Data_->Sum / Data_->Count); + } + return Nothing(); + } + + /// @brief Number of jobs that emitted this statistic. + TMaybe<T> Count() const + { + if (Data_) { + return ConvertJobStatisticsEntry<T>(Data_->Count); + } + return Nothing(); + } + + /// @brief Maximum value of the statistic over all jobs. + TMaybe<T> Max() const + { + if (Data_) { + return ConvertJobStatisticsEntry<T>(Data_->Max); + } + return Nothing(); + } + + /// @brief Minimum value of the statistic over all jobs. + TMaybe<T> Min() const + { + if (Data_) { + return ConvertJobStatisticsEntry<T>(Data_->Min); + } + return Nothing(); + } + +private: + TMaybe<TJobStatistics::TDataEntry> Data_; + +private: + friend class TJobStatistics; +}; + +//////////////////////////////////////////////////////////////////// + +template <typename T> +TJobStatisticsEntry<T> TJobStatistics::GetStatisticsAs(TStringBuf name) const +{ + return TJobStatisticsEntry<T>(GetStatisticsImpl(name)); +} + +template <typename T> +TJobStatisticsEntry<T> TJobStatistics::GetCustomStatisticsAs(TStringBuf name) const +{ + return TJobStatisticsEntry<T>(GetStatisticsImpl(CustomStatisticsNamePrefix_ + name)); +} + +//////////////////////////////////////////////////////////////////// + +/// +/// @brief Write [custom statistics](https://yt.yandex-team.ru/docs/description/mr/jobs#user_stats). +/// +/// @param path Slash-separated path (length must not exceed 512 bytes). +/// @param value Value of the statistic. +/// +/// @note The function must be called in job. +/// Total number of statistics (with different paths) must not exceed 128. +void WriteCustomStatistics(TStringBuf path, i64 value); + +/// +/// @brief Write several [custom statistics](https://yt.yandex-team.ru/docs/description/mr/jobs#user_stats) at once. +/// +/// @param statistics A tree of map nodes with leaves of type `i64`. +/// +/// @note The call is equivalent to calling @ref NYT::WriteCustomStatistics(TStringBuf, i64) for every path in the given map. +void WriteCustomStatistics(const TNode& statistics); + +/// +/// @brief Flush [custom statistics stream](https://yt.yandex-team.ru/docs/description/mr/jobs#user_stats) +/// +void FlushCustomStatisticsStream(); +//////////////////////////////////////////////////////////////////// + +} // namespace NYT |