path: root/ydb/library/yql/dq/actors/protos/dq_stats.proto

syntax = "proto3";

package NYql.NDqProto;

option cc_enable_arenas = true;

import "google/protobuf/any.proto";

enum EDqStatsMode {
    DQ_STATS_MODE_UNSPECIFIED = 0;
    DQ_STATS_MODE_NONE    = 10; // no statistics is expected, it will be ignored anyway
    DQ_STATS_MODE_BASIC   = 20; // per graph statistics
    DQ_STATS_MODE_FULL    = 25; // per stage/connection statistics
    DQ_STATS_MODE_PROFILE = 30; // per task statistics
}

// XxxxxMs means time period in ms since epoch

message TDqAsyncBufferStats {
    // Data
    uint64 Bytes  = 1; // physical bytes
    uint64 Rows   = 2; // logical rows (if applicable)
    uint64 Chunks = 3; // chunk is group of sent/received bytes in single batch
    uint64 Splits = 4; // logical partitioning (if applicable)
    // Time
    uint64 FirstMessageMs  = 5;  // first message processed
    uint64 PauseMessageMs  = 6;  // first blocking time on empty/overflow condition
    uint64 ResumeMessageMs = 7;  // last unblocking time
    uint64 LastMessageMs   = 8;  // last message processed
    uint64 WaitTimeUs      = 9;  // SUM(Resume_i - Pause_i) in us
    uint64 WaitPeriods     = 10; // COUNT(Resume_i - Pause_i)
}

message TDqAsyncInputBufferStats {
    uint64 InputIndex = 1;
    string IngressName = 9;

    TDqAsyncBufferStats Ingress = 10;
    TDqAsyncBufferStats Push = 11;
    TDqAsyncBufferStats Pop = 12;

    // profile stats
    uint64 MaxMemoryUsage = 6;
    uint32 ErrorsCount = 7;

    google.protobuf.Any Extra = 100;

    // deprecated
    uint64 Chunks_Deprecated = 2;
    uint64 Bytes_Deprecated = 3;
    uint64 RowsIn_Deprecated = 4;
    uint64 RowsOut_Deprecated = 5;
    uint64 IngressBytes_Deprecated = 8;
}

message TDqInputChannelStats {
    uint64 ChannelId = 1;
    uint32 SrcStageId = 12;

    TDqAsyncBufferStats Push = 13;
    TDqAsyncBufferStats Pop = 14;

    // profile stats
    uint64 DeserializationTimeUs = 6;
    uint64 MaxMemoryUsage = 7;
    uint32 PollRequests = 8;
    uint32 ResentMessages = 9;

    google.protobuf.Any Extra = 100;

    // deprecated
    uint64 Chunks_Deprecated = 2;
    uint64 Bytes_Deprecated = 3;
    uint64 RowsIn_Deprecated = 4;
    uint64 RowsOut_Deprecated = 5;
    uint64 IdleTimeUs_Deprecated = 11;
    uint64 WaitTimeUs_Deprecated = 10;
    uint64 FirstMessageMs_Deprecated = 30;
    uint64 LastMessageMs_Deprecated = 31;
}

message TDqAsyncOutputBufferStats {
    uint64 OutputIndex = 1;
    string EgressName = 9;

    TDqAsyncBufferStats Push = 10;
    TDqAsyncBufferStats Pop = 11;
    TDqAsyncBufferStats Egress = 12;

    // profile stats
    uint64 MaxMemoryUsage = 6;
    uint32 ErrorsCount = 7;

    google.protobuf.Any Extra = 100;

    // deprecated
    uint64 Chunks_Deprecated = 2;
    uint64 Bytes_Deprecated = 3;
    uint64 RowsIn_Deprecated = 4;
    uint64 RowsOut_Deprecated = 5;
    uint64 EgressBytes_Deprecated = 8;
}

message TDqOutputChannelStats {
    uint64 ChannelId = 1;
    uint32 DstStageId = 20;

    TDqAsyncBufferStats Push = 16;
    TDqAsyncBufferStats Pop = 17;

    // profile stats
    uint64 SerializationTimeUs = 6;
    uint32 ResentMessages = 8;
    uint32 NoDstActorId = 9;
    uint32 MaxRowsInMemory = 10;
    uint64 MaxMemoryUsage = 11;
    uint64 SpilledBytes = 12;
    uint64 SpilledRows = 13;
    uint64 SpilledBlobs = 14;

    google.protobuf.Any Extra = 100;

    // deprecated
    uint64 Chunks_Deprecated = 2;
    uint64 Bytes_Deprecated = 3;
    uint64 RowsIn_Deprecated = 4;
    uint64 RowsOut_Deprecated = 5;
    uint32 BlockedByCapacity_Deprecated = 7;
    uint64 BlockedTimeUs_Deprecated = 15;
    uint64 FirstMessageMs_Deprecated = 30;
    uint64 LastMessageMs_Deprecated = 31;
}

message TDqTableStats {
    string TablePath = 1;

    uint64 ReadRows = 2;
    uint64 ReadBytes = 3;
    uint64 WriteRows = 4;
    uint64 WriteBytes = 5;
    uint64 EraseRows = 6;
    uint64 EraseBytes = 7;

    uint32 AffectedPartitions = 8; // TODO: move it to the Extra

    google.protobuf.Any Extra = 100;
}

message TDqMkqlStat {
    string Name = 1;
    int64 Value = 2;
    bool Deriv = 3;
}

message TDqDataProviderStats {
    string Name   = 1;
    uint64 Bytes  = 2;
    uint64 Rows   = 3;
    uint64 Chunks = 4;
    uint64 Splits = 5;
}

message TDqTaskStats {
    reserved 154;

    // basic stats
    uint64 TaskId = 1;
    uint32 StageId = 2;
    uint64 CpuTimeUs = 3;        // total cpu time (build & compute)
    uint64 SourceCpuTimeUs = 15; // time consumed in source(s)
    uint64 BuildCpuTimeUs = 103;   // prepare task time: build computation graph, prepare parameters, ...
    uint64 ComputeCpuTimeUs = 102; // compute time only

    repeated TDqDataProviderStats Ingress = 13;
    repeated TDqDataProviderStats Egress = 14;

    uint64 InputRows = 6;
    uint64 InputBytes = 7;
    uint64 OutputRows = 8;
    uint64 OutputBytes = 9;

    // full stats
    repeated TDqAsyncInputBufferStats Sources = 150;
    repeated TDqInputChannelStats InputChannels = 151;
    repeated TDqAsyncOutputBufferStats Sinks = 152;
    repeated TDqOutputChannelStats OutputChannels = 153;
    repeated TDqAsyncInputBufferStats InputTransforms = 155;

    // profile stats
    repeated TDqTableStats Tables = 10;

    repeated TDqMkqlStat MkqlStats = 110; // stats from mkql
    message THistBucket {
        double Bound = 1;
        uint64 Value = 2;
    }
    repeated THistBucket ComputeCpuTimeByRun = 106;

    string HostName = 156;
    uint32 NodeId = 157;

    google.protobuf.Any Extra = 200;

    // to be reviewed
    uint64 StartTimeMs = 158;
    uint64 FinishTimeMs = 5;     // task finish time, timestamp in millis
    uint64 FirstRowTimeMs = 4;   // first row time, timestamp in millis
    reserved 104;
    uint64 WaitInputTimeUs = 111;  // wait input wall time  (any input: channels, source, ...)
    uint64 WaitOutputTimeUs = 105; // wait output wall time (any output: channels, sinks, ...)
    reserved 107;
    reserved 108;
    reserved 109;
}

message TDqComputeActorStats {
    // basic stats
    uint64 CpuTimeUs = 1;  // total cpu time: tasks cpu time + self cpu time
    uint64 DurationUs = 2; // compute actor duration, wall time (from FirstRowTime to FinishTime)
    repeated TDqTaskStats Tasks = 3; // in the BASIC_MODE only basic fields are used
    uint64 MaxMemoryUsage = 4;

    // profile stats
    uint64 MkqlMaxMemoryUsage = 102;       // MKQL allocations stats
    uint64 MkqlExtraMemoryBytes = 103;
    uint32 MkqlExtraMemoryRequests = 104;

    google.protobuf.Any Extra = 200;
}

message TDqStatsAggr {
    uint64 Min = 1;
    uint64 Max = 2;
    uint64 Sum = 3;
    uint64 Cnt = 4;
}

message TExtraStats {
    map<string, TDqStatsAggr> Stats = 1;
}

message TDqStatsMinMax {
    uint64 Min = 1;
    uint64 Max = 2;
}

message TDqAsyncStatsAggr {
    // Data
    TDqStatsAggr Bytes  = 1;
    TDqStatsAggr Rows   = 2;
    TDqStatsAggr Chunks = 3;
    TDqStatsAggr Splits = 4;
    // Time
    TDqStatsAggr FirstMessageMs  = 5;
    TDqStatsAggr PauseMessageMs  = 6;
    TDqStatsAggr ResumeMessageMs = 7;
    TDqStatsAggr LastMessageMs   = 8;
    TDqStatsAggr WaitTimeUs      = 9;
    TDqStatsAggr WaitPeriods     = 10;
    TDqStatsAggr ActiveTimeUs    = 11;
}

message TDqAsyncBufferStatsAggr {
    TDqAsyncStatsAggr Ingress = 1; // for Ingress only
    TDqAsyncStatsAggr Push    = 2;
    TDqAsyncStatsAggr Pop     = 3;
    TDqAsyncStatsAggr Egress  = 4; // for Egress only
}

message TDqTableAggrStats {
    string TablePath = 1;

    TDqStatsAggr ReadRows = 2;
    TDqStatsAggr ReadBytes = 3;
    TDqStatsAggr WriteRows = 4;
    TDqStatsAggr WriteBytes = 5;
    TDqStatsAggr EraseRows = 6;
    TDqStatsAggr EraseBytes = 7;

    uint32 AffectedPartitions = 8;

    google.protobuf.Any Extra = 100;
}

// aggregated stats for all tasks of the stage
message TDqStageStats {
    uint32 StageId = 1;
    string StageGuid = 2;
    string Program = 3;

    uint32 TotalTasksCount = 5;
    uint32 FailedTasksCount = 6;

    TDqStatsAggr CpuTimeUs = 8;
    TDqStatsAggr SourceCpuTimeUs = 25;
    TDqStatsAggr InputRows = 9;
    TDqStatsAggr InputBytes = 10;
    TDqStatsAggr OutputRows = 11;
    TDqStatsAggr OutputBytes = 12;

    TDqStatsMinMax FirstRowTimeMs = 13;
    TDqStatsMinMax FinishTimeMs = 14;
    TDqStatsMinMax StartTimeMs = 21;
    uint64 DurationUs = 15; // microseconds from min(task_first_row_time) to max(task_finish_time)

    repeated TDqTableAggrStats Tables = 16; // is it required?

    repeated TDqComputeActorStats ComputeActors = 17; // more detailed stats
    oneof LlvmOptions {
        bool UseLlvm = 18;
    }

    // currently only 1 source/sink per stage is possible, but format is ready for multiples
    map<string, TDqAsyncBufferStatsAggr> Ingress = 19; // ingress from external source, per provider
    map<string, TDqAsyncBufferStatsAggr> Egress = 20;  // egress to external consumer, per provider
    map<uint32, TDqAsyncBufferStatsAggr> Input = 22;   // input from other stage, per stage
    map<uint32, TDqAsyncBufferStatsAggr> Output = 23;  // output to other stage, per stage
    TDqStatsAggr MaxMemoryUsage = 24;

    google.protobuf.Any Extra = 100;
}

// graph execution stats
message TDqExecutionStats {
    // basic stats
    uint64 CpuTimeUs = 1;  // total cpu time, executer + compute actors + ...
    uint64 DurationUs = 2; // execution wall time

    uint64 ResultRows = 3;
    uint64 ResultBytes = 4;

    repeated TDqTableStats Tables = 6;

    // profile stats
    uint64 ExecuterCpuTimeUs = 10; // executer self cpu time, microseconds
    uint64 StartTimeMs = 11;       // executer start timestamp, milliseconds
    uint64 FinishTimeMs = 12;      // executer finish timestamp, milliseconds
    uint64 FirstRowTimeMs = 13;    // first result row timestamp, milliseconds

    repeated TDqStageStats Stages = 14;
    repeated string TxPlansWithStats = 15;

    google.protobuf.Any Extra = 100;
}