aboutsummaryrefslogtreecommitdiffstats
path: root/yt/cpp/mapreduce/interface/job_statistics.h
diff options
context:
space:
mode:
authormax42 <max42@yandex-team.com>2023-07-29 00:02:16 +0300
committermax42 <max42@yandex-team.com>2023-07-29 00:02:16 +0300
commit73b89de71748a21e102d27b9f3ed1bf658766cb5 (patch)
tree188bbd2d622fa91cdcbb1b6d6d77fbc84a0646f5 /yt/cpp/mapreduce/interface/job_statistics.h
parent528e321bcc2a2b67b53aeba58c3bd88305a141ee (diff)
downloadydb-73b89de71748a21e102d27b9f3ed1bf658766cb5.tar.gz
YT-19210: expose YQL shared library for YT.
After this, a new target libyqlplugin.so appears. in open-source cmake build. Diff in open-source YDB repo looks like the following: https://paste.yandex-team.ru/f302bdb4-7ef2-4362-91c7-6ca45f329264
Diffstat (limited to 'yt/cpp/mapreduce/interface/job_statistics.h')
-rw-r--r--yt/cpp/mapreduce/interface/job_statistics.h268
1 files changed, 268 insertions, 0 deletions
diff --git a/yt/cpp/mapreduce/interface/job_statistics.h b/yt/cpp/mapreduce/interface/job_statistics.h
new file mode 100644
index 0000000000..8af751604f
--- /dev/null
+++ b/yt/cpp/mapreduce/interface/job_statistics.h
@@ -0,0 +1,268 @@
+#pragma once
+
+///
+/// @file yt/cpp/mapreduce/interface/job_statistics.h
+///
+/// Header containing classes and utility functions to work with
+/// [job statistics](https://docs.yandex-team.ru/yt/problems/jobstatistics).
+
+#include "fwd.h"
+
+#include <library/cpp/yson/node/node.h>
+
+#include <util/system/defaults.h>
+#include <util/generic/maybe.h>
+#include <util/generic/ptr.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////
+
+///
+/// @brief Convert i64 representation of statistics to other type.
+///
+/// Library defines this template for types TDuration and i64.
+/// Users may define it for their types.
+///
+/// @see @ref NYT::TJobStatistics::GetStatisticsAs method.
+template <typename T>
+T ConvertJobStatisticsEntry(i64 value);
+
+////////////////////////////////////////////////////////////////////
+
+/// Class representing a collection of job statistics.
+class TJobStatistics
+{
+public:
+ ///
+ /// Construct empty statistics.
+ TJobStatistics();
+
+ ///
+ /// Construct statistics from statistics node.
+ TJobStatistics(const NYT::TNode& statistics);
+
+ TJobStatistics(const TJobStatistics& jobStatistics);
+ TJobStatistics(TJobStatistics&& jobStatistics);
+
+ TJobStatistics& operator=(const TJobStatistics& jobStatistics);
+ TJobStatistics& operator=(TJobStatistics&& jobStatistics);
+
+ ~TJobStatistics();
+
+ ///
+ /// @brief Filter statistics by task name.
+ ///
+ /// @param taskNames What task names to include (empty means all).
+ TJobStatistics TaskName(TVector<TTaskName> taskNames) const;
+
+ ///
+ /// @brief Filter statistics by job state.
+ ///
+ /// @param filter What job states to include (empty means all).
+ ///
+ /// @note Default statistics include only (successfully) completed jobs.
+ TJobStatistics JobState(TVector<EJobState> filter) const;
+
+ ///
+ /// @brief Filter statistics by job type.
+ ///
+ /// @param filter What job types to include (empty means all).
+ ///
+ /// @deprecated Use @ref TJobStatistics::TaskName instead.
+ ///
+ /// @see https://yt.yandex-team.ru/docs/description/mr/jobs#obshaya-shema
+ TJobStatistics JobType(TVector<EJobType> filter) const;
+
+ ///
+ /// @brief Check that given statistics exist.
+ ///
+ /// @param name Slash separated statistics name, e.g. "time/total" (like it appears in web interface).
+ bool HasStatistics(TStringBuf name) const;
+
+ ///
+ /// @brief Get statistics by name.
+ ///
+ /// @param name Slash separated statistics name, e.g. "time/total" (like it appears in web interface).
+ ///
+ /// @note If statistics is missing an exception is thrown. If because of filters
+ /// no fields remain the returned value is empty (all fields are `Nothing`).
+ ///
+ /// @note We don't use `TMaybe<TJobStatisticsEntry>` here;
+ /// instead, @ref NYT::TJobStatisticsEntry methods return `TMaybe<i64>`,
+ /// so user easier use `.GetOrElse`:
+ /// ```
+ /// jobStatistics.GetStatistics("some/statistics/name").Max().GetOrElse(0);
+ /// ```
+ TJobStatisticsEntry<i64> GetStatistics(TStringBuf name) const;
+
+ ///
+ /// @brief Get statistics by name.
+ ///
+ /// @param name Slash separated statistics name, e.g. "time/total" (like it appears in web interface).
+ ///
+ /// @note In order to use `GetStatisticsAs` method, @ref NYT::ConvertJobStatisticsEntry function must be defined
+ /// (the library defines it for `i64` and `TDuration`, user may define it for other types).
+ template <typename T>
+ TJobStatisticsEntry<T> GetStatisticsAs(TStringBuf name) const;
+
+ ///
+ /// Get (slash separated) names of statistics.
+ TVector<TString> GetStatisticsNames() const;
+
+ ///
+ /// @brief Check if given custom statistics exists.
+ ///
+ /// @param name Slash separated custom statistics name.
+ bool HasCustomStatistics(TStringBuf name) const;
+
+ ///
+ /// @brief Get custom statistics (those the user can write in job with @ref NYT::WriteCustomStatistics).
+ ///
+ /// @param name Slash separated custom statistics name.
+ TJobStatisticsEntry<i64> GetCustomStatistics(TStringBuf name) const;
+
+ ///
+ /// @brief Get custom statistics (those the user can write in job with @ref NYT::WriteCustomStatistics).
+ ///
+ /// @param name Slash separated custom statistics name.
+ template <typename T>
+ TJobStatisticsEntry<T> GetCustomStatisticsAs(TStringBuf name) const;
+
+ ///
+ /// Get names of all custom statistics.
+ TVector<TString> GetCustomStatisticsNames() const;
+
+private:
+ class TData;
+ struct TFilter;
+
+ struct TDataEntry {
+ i64 Max;
+ i64 Min;
+ i64 Sum;
+ i64 Count;
+ };
+
+ static const TString CustomStatisticsNamePrefix_;
+
+private:
+ TJobStatistics(::TIntrusivePtr<TData> data, ::TIntrusivePtr<TFilter> filter);
+
+ TMaybe<TDataEntry> GetStatisticsImpl(TStringBuf name) const;
+
+private:
+ ::TIntrusivePtr<TData> Data_;
+ ::TIntrusivePtr<TFilter> Filter_;
+
+private:
+ template<typename T>
+ friend class TJobStatisticsEntry;
+};
+
+////////////////////////////////////////////////////////////////////
+
+/// Class representing single statistic.
+template <typename T>
+class TJobStatisticsEntry
+{
+public:
+ TJobStatisticsEntry(TMaybe<TJobStatistics::TDataEntry> data)
+ : Data_(std::move(data))
+ { }
+
+ /// Sum of the statistic over all jobs.
+ TMaybe<T> Sum() const
+ {
+ if (Data_) {
+ return ConvertJobStatisticsEntry<T>(Data_->Sum);
+ }
+ return Nothing();
+ }
+
+ /// @brief Average of the statistic over all jobs.
+ ///
+ /// @note Only jobs that emitted statistics are taken into account.
+ TMaybe<T> Avg() const
+ {
+ if (Data_ && Data_->Count) {
+ return ConvertJobStatisticsEntry<T>(Data_->Sum / Data_->Count);
+ }
+ return Nothing();
+ }
+
+ /// @brief Number of jobs that emitted this statistic.
+ TMaybe<T> Count() const
+ {
+ if (Data_) {
+ return ConvertJobStatisticsEntry<T>(Data_->Count);
+ }
+ return Nothing();
+ }
+
+ /// @brief Maximum value of the statistic over all jobs.
+ TMaybe<T> Max() const
+ {
+ if (Data_) {
+ return ConvertJobStatisticsEntry<T>(Data_->Max);
+ }
+ return Nothing();
+ }
+
+ /// @brief Minimum value of the statistic over all jobs.
+ TMaybe<T> Min() const
+ {
+ if (Data_) {
+ return ConvertJobStatisticsEntry<T>(Data_->Min);
+ }
+ return Nothing();
+ }
+
+private:
+ TMaybe<TJobStatistics::TDataEntry> Data_;
+
+private:
+ friend class TJobStatistics;
+};
+
+////////////////////////////////////////////////////////////////////
+
+template <typename T>
+TJobStatisticsEntry<T> TJobStatistics::GetStatisticsAs(TStringBuf name) const
+{
+ return TJobStatisticsEntry<T>(GetStatisticsImpl(name));
+}
+
+template <typename T>
+TJobStatisticsEntry<T> TJobStatistics::GetCustomStatisticsAs(TStringBuf name) const
+{
+ return TJobStatisticsEntry<T>(GetStatisticsImpl(CustomStatisticsNamePrefix_ + name));
+}
+
+////////////////////////////////////////////////////////////////////
+
+///
+/// @brief Write [custom statistics](https://yt.yandex-team.ru/docs/description/mr/jobs#user_stats).
+///
+/// @param path Slash-separated path (length must not exceed 512 bytes).
+/// @param value Value of the statistic.
+///
+/// @note The function must be called in job.
+/// Total number of statistics (with different paths) must not exceed 128.
+void WriteCustomStatistics(TStringBuf path, i64 value);
+
+///
+/// @brief Write several [custom statistics](https://yt.yandex-team.ru/docs/description/mr/jobs#user_stats) at once.
+///
+/// @param statistics A tree of map nodes with leaves of type `i64`.
+///
+/// @note The call is equivalent to calling @ref NYT::WriteCustomStatistics(TStringBuf, i64) for every path in the given map.
+void WriteCustomStatistics(const TNode& statistics);
+
+///
+/// @brief Flush [custom statistics stream](https://yt.yandex-team.ru/docs/description/mr/jobs#user_stats)
+///
+void FlushCustomStatisticsStream();
+////////////////////////////////////////////////////////////////////
+
+} // namespace NYT