aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorkbalakirev <kbalakirev@yandex-team.ru>2022-02-10 16:48:58 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:48:58 +0300
commit498a47e48d41e5ec64ee3aa622a76a80274f35bd (patch)
tree5d5cb817648f650d76cf1076100726fd9b8448e8
parent1906a186042870fd03a12180acd1a6fcee045e42 (diff)
downloadydb-498a47e48d41e5ec64ee3aa622a76a80274f35bd.tar.gz
Restoring authorship annotation for <kbalakirev@yandex-team.ru>. Commit 2 of 2.
-rw-r--r--contrib/python/ya.make2
-rw-r--r--library/cpp/actors/util/rope.h8
-rw-r--r--library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp12
-rw-r--r--library/cpp/monlib/encode/buffered/buffered_encoder_base.h4
-rw-r--r--library/cpp/monlib/encode/fake/fake.cpp12
-rw-r--r--library/cpp/monlib/encode/json/json_decoder.cpp86
-rw-r--r--library/cpp/monlib/encode/json/json_encoder.cpp98
-rw-r--r--library/cpp/monlib/encode/json/json_ut.cpp544
-rw-r--r--library/cpp/monlib/encode/json/typed_point.h16
-rw-r--r--library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json92
-rw-r--r--library/cpp/monlib/encode/json/ut/log_histogram_value.json50
-rw-r--r--library/cpp/monlib/encode/json/ut/summary_inf.json40
-rw-r--r--library/cpp/monlib/encode/json/ut/summary_timeseries.json72
-rw-r--r--library/cpp/monlib/encode/json/ut/summary_value.json40
-rw-r--r--library/cpp/monlib/encode/json/ut/ya.make10
-rw-r--r--library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp50
-rw-r--r--library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp36
-rw-r--r--library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp82
-rw-r--r--library/cpp/monlib/encode/protobuf/protos/samples.proto42
-rw-r--r--library/cpp/monlib/encode/spack/spack_v1_decoder.cpp66
-rw-r--r--library/cpp/monlib/encode/spack/spack_v1_encoder.cpp66
-rw-r--r--library/cpp/monlib/encode/spack/spack_v1_ut.cpp202
-rw-r--r--library/cpp/monlib/encode/text/text_encoder.cpp20
-rw-r--r--library/cpp/monlib/encode/text/text_encoder_ut.cpp34
-rw-r--r--library/cpp/monlib/encode/unistat/unistat_decoder.cpp2
-rw-r--r--library/cpp/monlib/metrics/histogram_snapshot.cpp18
-rw-r--r--library/cpp/monlib/metrics/labels.h10
-rw-r--r--library/cpp/monlib/metrics/log_histogram_collector.h302
-rw-r--r--library/cpp/monlib/metrics/log_histogram_collector_ut.cpp76
-rw-r--r--library/cpp/monlib/metrics/log_histogram_snapshot.h138
-rw-r--r--library/cpp/monlib/metrics/metric_consumer.h6
-rw-r--r--library/cpp/monlib/metrics/metric_registry_ut.cpp12
-rw-r--r--library/cpp/monlib/metrics/metric_type.h2
-rw-r--r--library/cpp/monlib/metrics/metric_value.cpp14
-rw-r--r--library/cpp/monlib/metrics/metric_value.h202
-rw-r--r--library/cpp/monlib/metrics/metric_value_type.h2
-rw-r--r--library/cpp/monlib/metrics/metric_value_ut.cpp406
-rw-r--r--library/cpp/monlib/metrics/summary_collector.cpp2
-rw-r--r--library/cpp/monlib/metrics/summary_collector.h194
-rw-r--r--library/cpp/monlib/metrics/summary_collector_ut.cpp126
-rw-r--r--library/cpp/monlib/metrics/summary_snapshot.cpp14
-rw-r--r--library/cpp/monlib/metrics/summary_snapshot.h140
-rw-r--r--library/cpp/monlib/metrics/ut/ya.make2
-rw-r--r--ydb/core/erasure/erasure.cpp16
-rw-r--r--ydb/core/erasure/erasure_rope.cpp5492
-rw-r--r--ydb/core/erasure/erasure_rope.h998
-rw-r--r--ydb/core/erasure/erasure_rope_ut.cpp1476
-rw-r--r--ydb/core/erasure/ut_perf/ya.make28
-rw-r--r--ydb/core/erasure/ut_rope/ya.make42
-rw-r--r--ydb/core/erasure/ya.make16
-rw-r--r--ydb/services/ydb/ydb_stats_ut.cpp8
51 files changed, 5714 insertions, 5714 deletions
diff --git a/contrib/python/ya.make b/contrib/python/ya.make
index b2c994e2c0..d01ced9f3a 100644
--- a/contrib/python/ya.make
+++ b/contrib/python/ya.make
@@ -1103,7 +1103,7 @@ RECURSE(
tzlocal
ua-parser
udatetime
- uhashring
+ uhashring
ujson
ulid2
umalqurra
diff --git a/library/cpp/actors/util/rope.h b/library/cpp/actors/util/rope.h
index 012a6bc194..f5595efbaa 100644
--- a/library/cpp/actors/util/rope.h
+++ b/library/cpp/actors/util/rope.h
@@ -461,10 +461,10 @@ private:
return Ptr ? Iter->End - Ptr : 0;
}
- size_t ChunkOffset() const {
- return Ptr ? Ptr - Iter->Begin : 0;
- }
-
+ size_t ChunkOffset() const {
+ return Ptr ? Ptr - Iter->Begin : 0;
+ }
+
// Advance to next contiguous block of data.
void AdvanceToNextContiguousBlock() {
CheckValid();
diff --git a/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp b/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp
index b58d9ff475..87c832d642 100644
--- a/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp
+++ b/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp
@@ -132,18 +132,18 @@ void TBufferedEncoderBase::OnHistogram(TInstant time, IHistogramSnapshotPtr s) {
metric.TimeSeries.Add(time, s.Get());
}
-void TBufferedEncoderBase::OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr s) {
+void TBufferedEncoderBase::OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr s) {
State_.Expect(TEncoderState::EState::METRIC);
TMetric& metric = Metrics_.back();
metric.TimeSeries.Add(time, s.Get());
-}
-
-void TBufferedEncoderBase::OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr s) {
+}
+
+void TBufferedEncoderBase::OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr s) {
State_.Expect(TEncoderState::EState::METRIC);
TMetric& metric = Metrics_.back();
metric.TimeSeries.Add(time, s.Get());
-}
-
+}
+
TString TBufferedEncoderBase::FormatLabels(const TPooledLabels& labels) const {
auto formattedLabels = TVector<TString>(Reserve(labels.size() + CommonLabels_.size()));
auto addLabel = [&](const TPooledLabel& l) {
diff --git a/library/cpp/monlib/encode/buffered/buffered_encoder_base.h b/library/cpp/monlib/encode/buffered/buffered_encoder_base.h
index 1c717fb299..fe3714e58f 100644
--- a/library/cpp/monlib/encode/buffered/buffered_encoder_base.h
+++ b/library/cpp/monlib/encode/buffered/buffered_encoder_base.h
@@ -34,8 +34,8 @@ public:
void OnUint64(TInstant time, ui64 value) override;
void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override;
- void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override;
- void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override;
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override;
+ void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override;
protected:
using TPooledStr = TStringPoolBuilder::TValue;
diff --git a/library/cpp/monlib/encode/fake/fake.cpp b/library/cpp/monlib/encode/fake/fake.cpp
index e8b35c6fa7..69d691361a 100644
--- a/library/cpp/monlib/encode/fake/fake.cpp
+++ b/library/cpp/monlib/encode/fake/fake.cpp
@@ -35,12 +35,12 @@ namespace NMonitoring {
void OnHistogram(TInstant, IHistogramSnapshotPtr) override {
}
- void OnSummaryDouble(TInstant, ISummaryDoubleSnapshotPtr) override {
- }
-
- void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override {
- }
-
+ void OnSummaryDouble(TInstant, ISummaryDoubleSnapshotPtr) override {
+ }
+
+ void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override {
+ }
+
void Close() override {
}
};
diff --git a/library/cpp/monlib/encode/json/json_decoder.cpp b/library/cpp/monlib/encode/json/json_decoder.cpp
index ec4520d7c0..d44ff5fd28 100644
--- a/library/cpp/monlib/encode/json/json_decoder.cpp
+++ b/library/cpp/monlib/encode/json/json_decoder.cpp
@@ -73,38 +73,38 @@ private:
bool InfPresented_ = false;
TBucketValue InfValue_;
};
-
+
class TSummaryDoubleBuilder {
public:
ISummaryDoubleSnapshotPtr Build() const {
return MakeIntrusive<TSummaryDoubleSnapshot>(Sum_, Min_, Max_, Last_, Count_);
}
-
+
void SetSum(double sum) {
Empty_ = false;
Sum_ = sum;
}
-
+
void SetMin(double min) {
Empty_ = false;
Min_ = min;
}
-
+
void SetMax(double max) {
Empty_ = false;
Max_ = max;
}
-
+
void SetLast(double last) {
Empty_ = false;
Last_ = last;
}
-
+
void SetCount(ui64 count) {
Empty_ = false;
Count_ = count;
}
-
+
void Clear() {
Empty_ = true;
Sum_ = 0;
@@ -113,11 +113,11 @@ public:
Last_ = 0;
Count_ = 0;
}
-
+
bool Empty() const {
return Empty_;
}
-
+
private:
double Sum_ = 0;
double Min_ = 0;
@@ -126,51 +126,51 @@ private:
ui64 Count_ = 0;
bool Empty_ = true;
};
-
+
class TLogHistogramBuilder {
public:
void SetBase(double base) {
DECODE_ENSURE(base > 0, "base must be positive");
Base_ = base;
}
-
+
void SetZerosCount(ui64 zerosCount) {
DECODE_ENSURE(zerosCount >= 0, "zeros count must be positive");
ZerosCount_ = zerosCount;
}
-
+
void SetStartPower(int startPower) {
StartPower_ = startPower;
}
-
+
void AddBucketValue(double value) {
DECODE_ENSURE(value > 0.0, "bucket values must be positive");
DECODE_ENSURE(value < std::numeric_limits<double>::max(), "bucket values must be finite");
Buckets_.push_back(value);
}
-
+
void Clear() {
Buckets_.clear();
Base_ = 1.5;
ZerosCount_ = 0;
StartPower_ = 0;
}
-
+
bool Empty() const {
return Buckets_.empty() && ZerosCount_ == 0;
}
-
+
TLogHistogramSnapshotPtr Build() {
return MakeIntrusive<TLogHistogramSnapshot>(Base_, ZerosCount_, StartPower_, std::move(Buckets_));
}
-
+
private:
double Base_ = 1.5;
ui64 ZerosCount_ = 0;
int StartPower_ = 0;
TVector<double> Buckets_;
};
-
+
std::pair<double, bool> ParseSpecDouble(TStringBuf string) {
if (string == TStringBuf("nan") || string == TStringBuf("NaN")) {
return {std::numeric_limits<double>::quiet_NaN(), true};
@@ -542,7 +542,7 @@ if (Y_UNLIKELY(!(CONDITION))) { \
LastMetric_.SummaryBuilder.SetCount(value);
State_.ToPrev();
break;
-
+
case TState::METRIC_DSUMMARY_SUM:
LastMetric_.SummaryBuilder.SetSum(value);
State_.ToPrev();
@@ -559,26 +559,26 @@ if (Y_UNLIKELY(!(CONDITION))) { \
LastMetric_.SummaryBuilder.SetLast(value);
State_.ToPrev();
break;
-
+
case TState::METRIC_LOG_HIST_BASE:
LastMetric_.LogHistBuilder.SetBase(value);
State_.ToPrev();
break;
-
+
case TState::METRIC_LOG_HIST_ZEROS:
LastMetric_.LogHistBuilder.SetZerosCount(value);
State_.ToPrev();
break;
-
+
case TState::METRIC_LOG_HIST_START_POWER:
LastMetric_.LogHistBuilder.SetStartPower(value);
State_.ToPrev();
break;
-
+
case TState::METRIC_LOG_HIST_BUCKETS:
LastMetric_.LogHistBuilder.AddBucketValue(value);
break;
-
+
default:
return false;
}
@@ -627,7 +627,7 @@ if (Y_UNLIKELY(!(CONDITION))) { \
LastMetric_.SummaryBuilder.SetCount(value);
State_.ToPrev();
break;
-
+
case TState::METRIC_DSUMMARY_SUM:
LastMetric_.SummaryBuilder.SetSum(value);
State_.ToPrev();
@@ -644,26 +644,26 @@ if (Y_UNLIKELY(!(CONDITION))) { \
LastMetric_.SummaryBuilder.SetLast(value);
State_.ToPrev();
break;
-
+
case TState::METRIC_LOG_HIST_BASE:
LastMetric_.LogHistBuilder.SetBase(value);
State_.ToPrev();
break;
-
+
case TState::METRIC_LOG_HIST_ZEROS:
LastMetric_.LogHistBuilder.SetZerosCount(value);
State_.ToPrev();
break;
-
+
case TState::METRIC_LOG_HIST_START_POWER:
LastMetric_.LogHistBuilder.SetStartPower(value);
State_.ToPrev();
break;
-
+
case TState::METRIC_LOG_HIST_BUCKETS:
LastMetric_.LogHistBuilder.AddBucketValue(value);
break;
-
+
default:
return false;
}
@@ -697,16 +697,16 @@ if (Y_UNLIKELY(!(CONDITION))) { \
LastMetric_.SummaryBuilder.SetLast(value);
State_.ToPrev();
break;
-
+
case TState::METRIC_LOG_HIST_BASE:
LastMetric_.LogHistBuilder.SetBase(value);
State_.ToPrev();
break;
-
+
case TState::METRIC_LOG_HIST_BUCKETS:
LastMetric_.LogHistBuilder.AddBucketValue(value);
break;
-
+
default:
return false;
}
@@ -810,7 +810,7 @@ if (Y_UNLIKELY(!(CONDITION))) { \
return true;
}
-
+
bool OnMapKey(const TStringBuf& key) override {
switch (State_.Current()) {
case TState::ROOT_OBJECT:
@@ -822,12 +822,12 @@ if (Y_UNLIKELY(!(CONDITION))) { \
State_.ToNext(TState::METRICS_ARRAY);
}
break;
-
+
case TState::COMMON_LABELS:
case TState::METRIC_LABELS:
LastLabelName_ = key;
break;
-
+
case TState::METRIC_OBJECT:
if (key == TStringBuf("labels")) {
State_.ToNext(TState::METRIC_LABELS);
@@ -865,7 +865,7 @@ if (Y_UNLIKELY(!(CONDITION))) { \
return false;
}
break;
-
+
case TState::METRIC_TIMESERIES:
if (key == TStringBuf("ts")) {
State_.ToNext(TState::METRIC_TS);
@@ -924,22 +924,22 @@ if (Y_UNLIKELY(!(CONDITION))) { \
return true;
}
-
+
bool OnOpenMap() override {
switch (State_.Current()) {
case TState::ROOT_OBJECT:
MetricConsumer_->OnStreamBegin();
break;
-
+
case TState::COMMON_LABELS:
MetricConsumer_->OnLabelsBegin();
break;
-
+
case TState::METRICS_ARRAY:
State_.ToNext(TState::METRIC_OBJECT);
LastMetric_.Clear();
break;
-
+
default:
break;
}
@@ -1147,7 +1147,7 @@ void DecodeJson(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel)
// no need to check a return value. If there is an error, a TJsonDecodeError is thrown
NJson::ReadJson(&memIn, &decoder);
}
-
+
TCommonPartsProxy commonPartsProxy(std::move(commonPartsCollector.CommonParts()), c);
{
TMemoryInput memIn(data);
@@ -1156,7 +1156,7 @@ void DecodeJson(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel)
NJson::ReadJson(&memIn, &decoder);
}
}
-
+
#undef DECODE_ENSURE
}
diff --git a/library/cpp/monlib/encode/json/json_encoder.cpp b/library/cpp/monlib/encode/json/json_encoder.cpp
index 7e61d0443a..20d2bb6283 100644
--- a/library/cpp/monlib/encode/json/json_encoder.cpp
+++ b/library/cpp/monlib/encode/json/json_encoder.cpp
@@ -93,55 +93,55 @@ namespace NMonitoring {
Buf_.EndObject();
}
- void WriteValue(ISummaryDoubleSnapshot* s) {
+ void WriteValue(ISummaryDoubleSnapshot* s) {
Y_ENSURE(Style_ == EJsonStyle::Solomon);
Buf_.WriteKey(TStringBuf("summary"));
- Buf_.BeginObject();
-
+ Buf_.BeginObject();
+
Buf_.WriteKey(TStringBuf("sum"));
- Buf_.WriteDouble(s->GetSum());
-
+ Buf_.WriteDouble(s->GetSum());
+
Buf_.WriteKey(TStringBuf("min"));
- Buf_.WriteDouble(s->GetMin());
-
+ Buf_.WriteDouble(s->GetMin());
+
Buf_.WriteKey(TStringBuf("max"));
- Buf_.WriteDouble(s->GetMax());
-
+ Buf_.WriteDouble(s->GetMax());
+
Buf_.WriteKey(TStringBuf("last"));
- Buf_.WriteDouble(s->GetLast());
-
+ Buf_.WriteDouble(s->GetLast());
+
Buf_.WriteKey(TStringBuf("count"));
- Buf_.WriteULongLong(s->GetCount());
-
- Buf_.EndObject();
- }
-
- void WriteValue(TLogHistogramSnapshot* s) {
+ Buf_.WriteULongLong(s->GetCount());
+
+ Buf_.EndObject();
+ }
+
+ void WriteValue(TLogHistogramSnapshot* s) {
Y_ENSURE(Style_ == EJsonStyle::Solomon);
Buf_.WriteKey(TStringBuf("log_hist"));
- Buf_.BeginObject();
-
+ Buf_.BeginObject();
+
Buf_.WriteKey(TStringBuf("base"));
- Buf_.WriteDouble(s->Base());
-
+ Buf_.WriteDouble(s->Base());
+
Buf_.WriteKey(TStringBuf("zeros_count"));
- Buf_.WriteULongLong(s->ZerosCount());
-
+ Buf_.WriteULongLong(s->ZerosCount());
+
Buf_.WriteKey(TStringBuf("start_power"));
- Buf_.WriteInt(s->StartPower());
-
+ Buf_.WriteInt(s->StartPower());
+
Buf_.WriteKey(TStringBuf("buckets"));
- Buf_.BeginList();
- for (size_t i = 0; i < s->Count(); ++i) {
- Buf_.WriteDouble(s->Bucket(i));
- }
- Buf_.EndList();
-
- Buf_.EndObject();
- }
-
+ Buf_.BeginList();
+ for (size_t i = 0; i < s->Count(); ++i) {
+ Buf_.WriteDouble(s->Bucket(i));
+ }
+ Buf_.EndList();
+
+ Buf_.EndObject();
+ }
+
void WriteValue(EMetricValueType type, TMetricValue value) {
switch (type) {
case EMetricValueType::DOUBLE:
@@ -161,13 +161,13 @@ namespace NMonitoring {
break;
case EMetricValueType::SUMMARY:
- WriteValue(value.AsSummaryDouble());
- break;
-
- case EMetricValueType::LOGHISTOGRAM:
- WriteValue(value.AsLogHistogram());
- break;
-
+ WriteValue(value.AsSummaryDouble());
+ break;
+
+ case EMetricValueType::LOGHISTOGRAM:
+ WriteValue(value.AsLogHistogram());
+ break;
+
case EMetricValueType::UNKNOWN:
ythrow yexception() << "unknown metric value type";
}
@@ -358,16 +358,16 @@ namespace NMonitoring {
Write<IHistogramSnapshot*>(time, snapshot.Get());
}
- void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
State_.Expect(TEncoderState::EState::METRIC);
- Write<ISummaryDoubleSnapshot*>(time, snapshot.Get());
- }
-
- void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override {
+ Write<ISummaryDoubleSnapshot*>(time, snapshot.Get());
+ }
+
+ void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override {
State_.Expect(TEncoderState::EState::METRIC);
- Write<TLogHistogramSnapshot*>(time, snapshot.Get());
- }
-
+ Write<TLogHistogramSnapshot*>(time, snapshot.Get());
+ }
+
template <typename T>
void Write(TInstant time, T value) {
State_.Expect(TEncoderState::EState::METRIC);
diff --git a/library/cpp/monlib/encode/json/json_ut.cpp b/library/cpp/monlib/encode/json/json_ut.cpp
index 5260def99d..09e7909289 100644
--- a/library/cpp/monlib/encode/json/json_ut.cpp
+++ b/library/cpp/monlib/encode/json/json_ut.cpp
@@ -101,34 +101,34 @@ namespace {
}
}
- void AssertPointEqual(const NProto::TPoint& p, TInstant time, const TLogHistogramSnapshot& expected) {
- UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
- UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kLogHistogram);
-
- const double eps = 1e-10;
- const NProto::TLogHistogram& h = p.GetLogHistogram();
-
- UNIT_ASSERT_DOUBLES_EQUAL(h.GetBase(), expected.Base(), eps);
- UNIT_ASSERT_VALUES_EQUAL(h.GetZerosCount(), expected.ZerosCount());
- UNIT_ASSERT_VALUES_EQUAL(h.GetStartPower(), expected.StartPower());
- UNIT_ASSERT_VALUES_EQUAL(h.BucketsSize(), expected.Count());
- for (size_t i = 0; i < expected.Count(); ++i) {
- UNIT_ASSERT_DOUBLES_EQUAL(h.GetBuckets(i), expected.Bucket(i), eps);
- }
- }
-
- void AssertPointEqual(const NProto::TPoint& p, TInstant time, const ISummaryDoubleSnapshot& expected) {
- UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
- UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kSummaryDouble);
- auto actual = p.GetSummaryDouble();
- const double eps = 1e-10;
- UNIT_ASSERT_DOUBLES_EQUAL(actual.GetSum(), expected.GetSum(), eps);
- UNIT_ASSERT_DOUBLES_EQUAL(actual.GetMin(), expected.GetMin(), eps);
- UNIT_ASSERT_DOUBLES_EQUAL(actual.GetMax(), expected.GetMax(), eps);
- UNIT_ASSERT_DOUBLES_EQUAL(actual.GetLast(), expected.GetLast(), eps);
- UNIT_ASSERT_VALUES_EQUAL(actual.GetCount(), expected.GetCount());
- }
-
+ void AssertPointEqual(const NProto::TPoint& p, TInstant time, const TLogHistogramSnapshot& expected) {
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kLogHistogram);
+
+ const double eps = 1e-10;
+ const NProto::TLogHistogram& h = p.GetLogHistogram();
+
+ UNIT_ASSERT_DOUBLES_EQUAL(h.GetBase(), expected.Base(), eps);
+ UNIT_ASSERT_VALUES_EQUAL(h.GetZerosCount(), expected.ZerosCount());
+ UNIT_ASSERT_VALUES_EQUAL(h.GetStartPower(), expected.StartPower());
+ UNIT_ASSERT_VALUES_EQUAL(h.BucketsSize(), expected.Count());
+ for (size_t i = 0; i < expected.Count(); ++i) {
+ UNIT_ASSERT_DOUBLES_EQUAL(h.GetBuckets(i), expected.Bucket(i), eps);
+ }
+ }
+
+ void AssertPointEqual(const NProto::TPoint& p, TInstant time, const ISummaryDoubleSnapshot& expected) {
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kSummaryDouble);
+ auto actual = p.GetSummaryDouble();
+ const double eps = 1e-10;
+ UNIT_ASSERT_DOUBLES_EQUAL(actual.GetSum(), expected.GetSum(), eps);
+ UNIT_ASSERT_DOUBLES_EQUAL(actual.GetMin(), expected.GetMin(), eps);
+ UNIT_ASSERT_DOUBLES_EQUAL(actual.GetMax(), expected.GetMax(), eps);
+ UNIT_ASSERT_DOUBLES_EQUAL(actual.GetLast(), expected.GetLast(), eps);
+ UNIT_ASSERT_VALUES_EQUAL(actual.GetCount(), expected.GetCount());
+ }
+
} // namespace
@@ -734,223 +734,223 @@ Y_UNIT_TEST_SUITE(TJsonTest) {
return out.Str();
}
- Y_UNIT_TEST(SummaryValueEncode) {
+ Y_UNIT_TEST(SummaryValueEncode) {
auto writeDocument = [](IMetricEncoder* e) {
- e->OnStreamBegin();
- {
+ e->OnStreamBegin();
+ {
e->OnMetricBegin(EMetricType::DSUMMARY);
- {
- e->OnLabelsBegin();
+ {
+ e->OnLabelsBegin();
e->OnLabel("metric", "temperature");
- e->OnLabelsEnd();
- }
-
- e->OnSummaryDouble(now, MakeIntrusive<TSummaryDoubleSnapshot>(10., -0.5, 0.5, 0.3, 30u));
+ e->OnLabelsEnd();
+ }
+
+ e->OnSummaryDouble(now, MakeIntrusive<TSummaryDoubleSnapshot>(10., -0.5, 0.5, 0.3, 30u));
e->OnMetricEnd();
- }
- e->OnStreamEnd();
- };
-
- TString result1 = EncodeToString(EncoderJson, writeDocument);
- UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_value.json"));
-
- TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
- UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_value.json"));
- }
-
- ISummaryDoubleSnapshotPtr TestInfSummary() {
- return MakeIntrusive<TSummaryDoubleSnapshot>(
- std::numeric_limits<double>::quiet_NaN(),
- -std::numeric_limits<double>::infinity(),
- std::numeric_limits<double>::infinity(),
- 0.3,
- 30u);
- }
-
- Y_UNIT_TEST(SummaryInfEncode) {
+ }
+ e->OnStreamEnd();
+ };
+
+ TString result1 = EncodeToString(EncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_value.json"));
+
+ TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_value.json"));
+ }
+
+ ISummaryDoubleSnapshotPtr TestInfSummary() {
+ return MakeIntrusive<TSummaryDoubleSnapshot>(
+ std::numeric_limits<double>::quiet_NaN(),
+ -std::numeric_limits<double>::infinity(),
+ std::numeric_limits<double>::infinity(),
+ 0.3,
+ 30u);
+ }
+
+ Y_UNIT_TEST(SummaryInfEncode) {
auto writeDocument = [](IMetricEncoder* e) {
- e->OnStreamBegin();
- {
+ e->OnStreamBegin();
+ {
e->OnMetricBegin(EMetricType::DSUMMARY);
- {
- e->OnLabelsBegin();
+ {
+ e->OnLabelsBegin();
e->OnLabel("metric", "temperature");
- e->OnLabelsEnd();
- }
-
- e->OnSummaryDouble(now, TestInfSummary());
+ e->OnLabelsEnd();
+ }
+
+ e->OnSummaryDouble(now, TestInfSummary());
e->OnMetricEnd();
- }
- e->OnStreamEnd();
- };
-
- TString result1 = EncodeToString(EncoderJson, writeDocument);
- UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_inf.json"));
-
- TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
- UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_inf.json"));
- }
-
- Y_UNIT_TEST(SummaryInfDecode) {
- NProto::TMultiSamplesList samples;
- {
+ }
+ e->OnStreamEnd();
+ };
+
+ TString result1 = EncodeToString(EncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_inf.json"));
+
+ TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_inf.json"));
+ }
+
+ Y_UNIT_TEST(SummaryInfDecode) {
+ NProto::TMultiSamplesList samples;
+ {
IMetricEncoderPtr e = EncoderProtobuf(&samples);
-
- TString testJson = NResource::Find("/summary_inf.json");
- DecodeJson(testJson, e.Get());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
- const NProto::TMultiSample& s = samples.GetSamples(0);
-
+
+ TString testJson = NResource::Find("/summary_inf.json");
+ DecodeJson(testJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+
UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY);
- UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
AssertLabelEqual(s.GetLabels(0), "metric", "temperature");
-
- UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
-
- auto actual = s.GetPoints(0).GetSummaryDouble();
- UNIT_ASSERT(std::isnan(actual.GetSum()));
- UNIT_ASSERT(actual.GetMin() < 0);
- UNIT_ASSERT(std::isinf(actual.GetMin()));
- UNIT_ASSERT(actual.GetMax() > 0);
- UNIT_ASSERT(std::isinf(actual.GetMax()));
- }
-
- Y_UNIT_TEST(SummaryValueDecode) {
- NProto::TMultiSamplesList samples;
- {
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+
+ auto actual = s.GetPoints(0).GetSummaryDouble();
+ UNIT_ASSERT(std::isnan(actual.GetSum()));
+ UNIT_ASSERT(actual.GetMin() < 0);
+ UNIT_ASSERT(std::isinf(actual.GetMin()));
+ UNIT_ASSERT(actual.GetMax() > 0);
+ UNIT_ASSERT(std::isinf(actual.GetMax()));
+ }
+
+ Y_UNIT_TEST(SummaryValueDecode) {
+ NProto::TMultiSamplesList samples;
+ {
IMetricEncoderPtr e = EncoderProtobuf(&samples);
-
- TString testJson = NResource::Find("/summary_value.json");
- DecodeJson(testJson, e.Get());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
- const NProto::TMultiSample& s = samples.GetSamples(0);
-
+
+ TString testJson = NResource::Find("/summary_value.json");
+ DecodeJson(testJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+
UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY);
- UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
AssertLabelEqual(s.GetLabels(0), "metric", "temperature");
-
- UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
-
- auto snapshot = TSummaryDoubleSnapshot(10., -0.5, 0.5, 0.3, 30u);
- AssertPointEqual(s.GetPoints(0), now, snapshot);
- }
-
- Y_UNIT_TEST(SummaryTimeSeriesEncode) {
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+
+ auto snapshot = TSummaryDoubleSnapshot(10., -0.5, 0.5, 0.3, 30u);
+ AssertPointEqual(s.GetPoints(0), now, snapshot);
+ }
+
+ Y_UNIT_TEST(SummaryTimeSeriesEncode) {
auto writeDocument = [](IMetricEncoder* e) {
- e->OnStreamBegin();
- {
+ e->OnStreamBegin();
+ {
e->OnMetricBegin(EMetricType::DSUMMARY);
- {
- e->OnLabelsBegin();
+ {
+ e->OnLabelsBegin();
e->OnLabel("metric", "temperature");
- e->OnLabelsEnd();
- }
-
- TSummaryDoubleCollector summary;
- summary.Collect(0.3);
- summary.Collect(-0.5);
- summary.Collect(1.);
-
- e->OnSummaryDouble(now, summary.Snapshot());
-
- summary.Collect(-1.5);
- summary.Collect(0.01);
-
- e->OnSummaryDouble(now + TDuration::Seconds(15), summary.Snapshot());
-
+ e->OnLabelsEnd();
+ }
+
+ TSummaryDoubleCollector summary;
+ summary.Collect(0.3);
+ summary.Collect(-0.5);
+ summary.Collect(1.);
+
+ e->OnSummaryDouble(now, summary.Snapshot());
+
+ summary.Collect(-1.5);
+ summary.Collect(0.01);
+
+ e->OnSummaryDouble(now + TDuration::Seconds(15), summary.Snapshot());
+
e->OnMetricEnd();
- }
- e->OnStreamEnd();
- };
-
- TString result1 = EncodeToString(EncoderJson, writeDocument);
- UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_timeseries.json"));
-
- TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
- UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_timeseries.json"));
- }
-
- Y_UNIT_TEST(SummaryTimeSeriesDecode) {
- NProto::TMultiSamplesList samples;
- {
+ }
+ e->OnStreamEnd();
+ };
+
+ TString result1 = EncodeToString(EncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_timeseries.json"));
+
+ TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_timeseries.json"));
+ }
+
+ Y_UNIT_TEST(SummaryTimeSeriesDecode) {
+ NProto::TMultiSamplesList samples;
+ {
IMetricEncoderPtr e = EncoderProtobuf(&samples);
-
- TString testJson = NResource::Find("/summary_timeseries.json");
- DecodeJson(testJson, e.Get());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
- const NProto::TMultiSample& s = samples.GetSamples(0);
-
+
+ TString testJson = NResource::Find("/summary_timeseries.json");
+ DecodeJson(testJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+
UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY);
- UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
AssertLabelEqual(s.GetLabels(0), "metric", "temperature");
-
- UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
-
- TSummaryDoubleCollector summary;
- summary.Collect(0.3);
- summary.Collect(-0.5);
- summary.Collect(1.);
-
- AssertPointEqual(s.GetPoints(0), now, *summary.Snapshot());
-
- summary.Collect(-1.5);
- summary.Collect(0.01);
-
- AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *summary.Snapshot());
- }
-
- Y_UNIT_TEST(LogHistogramValueEncode) {
- auto writeDocument = [](IMetricEncoder* e) {
- e->OnStreamBegin();
- {
- e->OnMetricBegin(EMetricType::LOGHIST);
- {
- e->OnLabelsBegin();
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
+
+ TSummaryDoubleCollector summary;
+ summary.Collect(0.3);
+ summary.Collect(-0.5);
+ summary.Collect(1.);
+
+ AssertPointEqual(s.GetPoints(0), now, *summary.Snapshot());
+
+ summary.Collect(-1.5);
+ summary.Collect(0.01);
+
+ AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *summary.Snapshot());
+ }
+
+ Y_UNIT_TEST(LogHistogramValueEncode) {
+ auto writeDocument = [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::LOGHIST);
+ {
+ e->OnLabelsBegin();
e->OnLabel("metric", "ms");
- e->OnLabelsEnd();
- }
-
- e->OnLogHistogram(now, TestLogHistogram());
- e->OnMetricEnd();
- }
- e->OnStreamEnd();
- };
-
- TString result1 = EncodeToString(EncoderJson, writeDocument);
- UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/log_histogram_value.json"));
-
- TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
- UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/log_histogram_value.json"));
- }
-
- Y_UNIT_TEST(LogHistogramValueDecode) {
- NProto::TMultiSamplesList samples;
- {
- IMetricEncoderPtr e = EncoderProtobuf(&samples);
-
- TString testJson = NResource::Find("/log_histogram_value.json");
- DecodeJson(testJson, e.Get());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
- const NProto::TMultiSample& s = samples.GetSamples(0);
-
- UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM);
- UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ e->OnLabelsEnd();
+ }
+
+ e->OnLogHistogram(now, TestLogHistogram());
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ };
+
+ TString result1 = EncodeToString(EncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/log_histogram_value.json"));
+
+ TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/log_histogram_value.json"));
+ }
+
+ Y_UNIT_TEST(LogHistogramValueDecode) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TString testJson = NResource::Find("/log_histogram_value.json");
+ DecodeJson(testJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
AssertLabelEqual(s.GetLabels(0), "metric", "ms");
-
- UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
-
- auto snapshot = TestLogHistogram();
- AssertPointEqual(s.GetPoints(0), now, *snapshot);
- }
-
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+
+ auto snapshot = TestLogHistogram();
+ AssertPointEqual(s.GetPoints(0), now, *snapshot);
+ }
+
Y_UNIT_TEST(HistogramValueEncode) {
auto writeDocument = [](IMetricEncoder* e) {
e->OnStreamBegin();
@@ -981,58 +981,58 @@ Y_UNIT_TEST_SUITE(TJsonTest) {
UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_value.json"));
}
- Y_UNIT_TEST(LogHistogramTimeSeriesEncode) {
- auto writeDocument = [](IMetricEncoder* e) {
- e->OnStreamBegin();
- {
- e->OnMetricBegin(EMetricType::LOGHIST);
- {
- e->OnLabelsBegin();
+ Y_UNIT_TEST(LogHistogramTimeSeriesEncode) {
+ auto writeDocument = [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::LOGHIST);
+ {
+ e->OnLabelsBegin();
e->OnLabel("metric", "ms");
- e->OnLabelsEnd();
- }
-
- e->OnLogHistogram(now, TestLogHistogram(1));;
-
- e->OnLogHistogram(now + TDuration::Seconds(15), TestLogHistogram(2));
-
- e->OnMetricEnd();
- }
- e->OnStreamEnd();
- };
-
- TString result1 = EncodeToString(EncoderJson, writeDocument);
- UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/log_histogram_timeseries.json"));
-
- TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
- UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/log_histogram_timeseries.json"));
- }
-
- Y_UNIT_TEST(LogHistogramTimeSeriesDecode) {
- NProto::TMultiSamplesList samples;
- {
- IMetricEncoderPtr e = EncoderProtobuf(&samples);
-
- TString testJson = NResource::Find("/log_histogram_timeseries.json");
- DecodeJson(testJson, e.Get());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
- const NProto::TMultiSample& s = samples.GetSamples(0);
-
- UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM);
- UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ e->OnLabelsEnd();
+ }
+
+ e->OnLogHistogram(now, TestLogHistogram(1));;
+
+ e->OnLogHistogram(now + TDuration::Seconds(15), TestLogHistogram(2));
+
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ };
+
+ TString result1 = EncodeToString(EncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/log_histogram_timeseries.json"));
+
+ TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/log_histogram_timeseries.json"));
+ }
+
+ Y_UNIT_TEST(LogHistogramTimeSeriesDecode) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TString testJson = NResource::Find("/log_histogram_timeseries.json");
+ DecodeJson(testJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
AssertLabelEqual(s.GetLabels(0), "metric", "ms");
-
- UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
-
- auto logHist = TestLogHistogram(1);
- AssertPointEqual(s.GetPoints(0), now, *logHist);
-
- logHist = TestLogHistogram(2);
- AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *logHist);
- }
-
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
+
+ auto logHist = TestLogHistogram(1);
+ AssertPointEqual(s.GetPoints(0), now, *logHist);
+
+ logHist = TestLogHistogram(2);
+ AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *logHist);
+ }
+
void HistogramValueDecode(const TString& filePath) {
NProto::TMultiSamplesList samples;
{
diff --git a/library/cpp/monlib/encode/json/typed_point.h b/library/cpp/monlib/encode/json/typed_point.h
index 5edeaab8be..fbaa840c4b 100644
--- a/library/cpp/monlib/encode/json/typed_point.h
+++ b/library/cpp/monlib/encode/json/typed_point.h
@@ -98,20 +98,20 @@ namespace NMonitoring {
if (ValueType_ == EMetricValueType::HISTOGRAM) {
Value_.AsHistogram()->Ref();
} else if (ValueType_ == EMetricValueType::SUMMARY) {
- Value_.AsSummaryDouble()->Ref();
- } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
- Value_.AsLogHistogram()->Ref();
- }
+ Value_.AsSummaryDouble()->Ref();
+ } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
+ Value_.AsLogHistogram()->Ref();
+ }
}
void UnRef() {
if (ValueType_ == EMetricValueType::HISTOGRAM) {
Value_.AsHistogram()->UnRef();
} else if (ValueType_ == EMetricValueType::SUMMARY) {
- Value_.AsSummaryDouble()->UnRef();
- } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
- Value_.AsLogHistogram()->UnRef();
- }
+ Value_.AsSummaryDouble()->UnRef();
+ } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
+ Value_.AsLogHistogram()->UnRef();
+ }
}
private:
diff --git a/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json b/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json
index 6a1e1947d1..e811a2cc57 100644
--- a/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json
+++ b/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json
@@ -1,47 +1,47 @@
-{
- "sensors":
- [
- {
- "kind":"LOGHIST",
- "labels":
- {
+{
+ "sensors":
+ [
+ {
+ "kind":"LOGHIST",
+ "labels":
+ {
"metric":"ms"
- },
- "timeseries":
- [
- {
- "ts":1509843723,
- "log_hist":
- {
- "base":1.5,
- "zeros_count":1,
- "start_power":0,
- "buckets":
- [
- 0.5,
- 0.25,
- 0.25,
- 0.5
- ]
- }
- },
- {
- "ts":1509843738,
- "log_hist":
- {
- "base":1.5,
- "zeros_count":1,
- "start_power":0,
- "buckets":
- [
- 1,
- 0.5,
- 0.5,
- 1
- ]
- }
- }
- ]
- }
- ]
-}
+ },
+ "timeseries":
+ [
+ {
+ "ts":1509843723,
+ "log_hist":
+ {
+ "base":1.5,
+ "zeros_count":1,
+ "start_power":0,
+ "buckets":
+ [
+ 0.5,
+ 0.25,
+ 0.25,
+ 0.5
+ ]
+ }
+ },
+ {
+ "ts":1509843738,
+ "log_hist":
+ {
+ "base":1.5,
+ "zeros_count":1,
+ "start_power":0,
+ "buckets":
+ [
+ 1,
+ 0.5,
+ 0.5,
+ 1
+ ]
+ }
+ }
+ ]
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/log_histogram_value.json b/library/cpp/monlib/encode/json/ut/log_histogram_value.json
index 919d709549..002478293b 100644
--- a/library/cpp/monlib/encode/json/ut/log_histogram_value.json
+++ b/library/cpp/monlib/encode/json/ut/log_histogram_value.json
@@ -1,26 +1,26 @@
-{
- "sensors":
- [
- {
- "kind":"LOGHIST",
- "labels":
- {
+{
+ "sensors":
+ [
+ {
+ "kind":"LOGHIST",
+ "labels":
+ {
"metric":"ms"
- },
- "ts":1509843723,
- "log_hist":
- {
- "base":1.5,
- "zeros_count":1,
- "start_power":0,
- "buckets":
- [
- 0.5,
- 0.25,
- 0.25,
- 0.5
- ]
- }
- }
- ]
-}
+ },
+ "ts":1509843723,
+ "log_hist":
+ {
+ "base":1.5,
+ "zeros_count":1,
+ "start_power":0,
+ "buckets":
+ [
+ 0.5,
+ 0.25,
+ 0.25,
+ 0.5
+ ]
+ }
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/summary_inf.json b/library/cpp/monlib/encode/json/ut/summary_inf.json
index 9d335a4d8a..625a6cd8ad 100644
--- a/library/cpp/monlib/encode/json/ut/summary_inf.json
+++ b/library/cpp/monlib/encode/json/ut/summary_inf.json
@@ -1,21 +1,21 @@
-{
- "sensors":
- [
- {
- "kind":"DSUMMARY",
- "labels":
- {
+{
+ "sensors":
+ [
+ {
+ "kind":"DSUMMARY",
+ "labels":
+ {
"metric":"temperature"
- },
- "ts":1509843723,
- "summary":
- {
- "sum":"nan",
- "min":"-inf",
- "max":"inf",
- "last":0.3,
- "count":30
- }
- }
- ]
-}
+ },
+ "ts":1509843723,
+ "summary":
+ {
+ "sum":"nan",
+ "min":"-inf",
+ "max":"inf",
+ "last":0.3,
+ "count":30
+ }
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/summary_timeseries.json b/library/cpp/monlib/encode/json/ut/summary_timeseries.json
index 82704a7b72..92007af3e6 100644
--- a/library/cpp/monlib/encode/json/ut/summary_timeseries.json
+++ b/library/cpp/monlib/encode/json/ut/summary_timeseries.json
@@ -1,37 +1,37 @@
-{
- "sensors":
- [
- {
- "kind":"DSUMMARY",
- "labels":
- {
+{
+ "sensors":
+ [
+ {
+ "kind":"DSUMMARY",
+ "labels":
+ {
"metric":"temperature"
- },
- "timeseries":
- [
- {
- "ts":1509843723,
- "summary":
- {
- "sum":0.8,
- "min":-0.5,
- "max":1,
- "last":1,
- "count":3
- }
- },
- {
- "ts":1509843738,
- "summary":
- {
- "sum":-0.69,
- "min":-1.5,
- "max":1,
- "last":0.01,
- "count":5
- }
- }
- ]
- }
- ]
-}
+ },
+ "timeseries":
+ [
+ {
+ "ts":1509843723,
+ "summary":
+ {
+ "sum":0.8,
+ "min":-0.5,
+ "max":1,
+ "last":1,
+ "count":3
+ }
+ },
+ {
+ "ts":1509843738,
+ "summary":
+ {
+ "sum":-0.69,
+ "min":-1.5,
+ "max":1,
+ "last":0.01,
+ "count":5
+ }
+ }
+ ]
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/summary_value.json b/library/cpp/monlib/encode/json/ut/summary_value.json
index 7eaf355d18..366394c5e1 100644
--- a/library/cpp/monlib/encode/json/ut/summary_value.json
+++ b/library/cpp/monlib/encode/json/ut/summary_value.json
@@ -1,21 +1,21 @@
-{
- "sensors":
- [
- {
- "kind":"DSUMMARY",
- "labels":
- {
+{
+ "sensors":
+ [
+ {
+ "kind":"DSUMMARY",
+ "labels":
+ {
"metric":"temperature"
- },
- "ts":1509843723,
- "summary":
- {
- "sum":10,
- "min":-0.5,
- "max":0.5,
- "last":0.3,
- "count":30
- }
- }
- ]
-}
+ },
+ "ts":1509843723,
+ "summary":
+ {
+ "sum":10,
+ "min":-0.5,
+ "max":0.5,
+ "last":0.3,
+ "count":30
+ }
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/ya.make b/library/cpp/monlib/encode/json/ut/ya.make
index 70f9e4c917..e50c4f4903 100644
--- a/library/cpp/monlib/encode/json/ut/ya.make
+++ b/library/cpp/monlib/encode/json/ut/ya.make
@@ -29,11 +29,11 @@ RESOURCE(
test_decode_to_encode.json /test_decode_to_encode.json
crash.json /crash.json
hist_crash.json /hist_crash.json
- summary_value.json /summary_value.json
- summary_inf.json /summary_inf.json
- summary_timeseries.json /summary_timeseries.json
- log_histogram_value.json /log_histogram_value.json
- log_histogram_timeseries.json /log_histogram_timeseries.json
+ summary_value.json /summary_value.json
+ summary_inf.json /summary_inf.json
+ summary_timeseries.json /summary_timeseries.json
+ log_histogram_value.json /log_histogram_value.json
+ log_histogram_timeseries.json /log_histogram_timeseries.json
)
PEERDIR(
diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp
index f77c4a3137..15efeb8c03 100644
--- a/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp
+++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp
@@ -45,11 +45,11 @@ namespace NMonitoring {
case EMetricType::HIST_RATE:
Out_->Write("histogram");
break;
- case EMetricType::LOGHIST:
- // TODO(@kbalakirev): implement this case
- break;
+ case EMetricType::LOGHIST:
+ // TODO(@kbalakirev): implement this case
+ break;
case EMetricType::DSUMMARY:
- ythrow yexception() << "writing summary type is forbiden";
+ ythrow yexception() << "writing summary type is forbiden";
case EMetricType::UNKNOWN:
ythrow yexception() << "unknown metric type: " << MetricTypeToStr(type)
<< ", name: " << name;
@@ -90,14 +90,14 @@ namespace NMonitoring {
WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, totalCount);
}
- void WriteSummaryDouble(TStringBuf name, const TLabels& labels, TInstant time, ISummaryDoubleSnapshot* s) {
- WriteValue(name, NPrometheus::SUM_SUFFIX, labels, "", "", time, s->GetSum());
- WriteValue(name, NPrometheus::MIN_SUFFIX, labels, "", "", time, s->GetMin());
- WriteValue(name, NPrometheus::MAX_SUFFIX, labels, "", "", time, s->GetMax());
- WriteValue(name, NPrometheus::LAST_SUFFIX, labels, "", "", time, s->GetLast());
- WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, s->GetCount());
- }
-
+ void WriteSummaryDouble(TStringBuf name, const TLabels& labels, TInstant time, ISummaryDoubleSnapshot* s) {
+ WriteValue(name, NPrometheus::SUM_SUFFIX, labels, "", "", time, s->GetSum());
+ WriteValue(name, NPrometheus::MIN_SUFFIX, labels, "", "", time, s->GetMin());
+ WriteValue(name, NPrometheus::MAX_SUFFIX, labels, "", "", time, s->GetMax());
+ WriteValue(name, NPrometheus::LAST_SUFFIX, labels, "", "", time, s->GetLast());
+ WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, s->GetCount());
+ }
+
void WriteLn() {
Out_->Write('\n');
}
@@ -220,7 +220,7 @@ namespace NMonitoring {
if (ValueType == EMetricValueType::HISTOGRAM) {
Value.AsHistogram()->UnRef();
} else if (ValueType == EMetricValueType::SUMMARY) {
- Value.AsSummaryDouble()->UnRef();
+ Value.AsSummaryDouble()->UnRef();
}
ValueType = EMetricValueType::UNKNOWN;
Value = {};
@@ -232,14 +232,14 @@ namespace NMonitoring {
if (ValueType == EMetricValueType::HISTOGRAM) {
Value.AsHistogram()->UnRef();
} else if (ValueType == EMetricValueType::SUMMARY) {
- Value.AsSummaryDouble()->UnRef();
+ Value.AsSummaryDouble()->UnRef();
}
ValueType = TValueType<T>::Type;
Value = TMetricValue(value);
if (ValueType == EMetricValueType::HISTOGRAM) {
Value.AsHistogram()->Ref();
} else if (ValueType == EMetricValueType::SUMMARY) {
- Value.AsSummaryDouble()->Ref();
+ Value.AsSummaryDouble()->Ref();
}
}
};
@@ -335,16 +335,16 @@ namespace NMonitoring {
MetricState_.SetValue(snapshot.Get());
}
- void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
State_.Expect(TEncoderState::EState::METRIC);
MetricState_.Time = time;
MetricState_.SetValue(snapshot.Get());
- }
-
- void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override {
- // TODO(@kbalakirev): implement this function
- }
-
+ }
+
+ void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override {
+ // TODO(@kbalakirev): implement this function
+ }
+
void Close() override {
}
@@ -366,7 +366,7 @@ namespace NMonitoring {
const TString& metricName = ToString(nameLabel->Value());
if (MetricState_.Type != EMetricType::DSUMMARY) {
Writer_.WriteType(MetricState_.Type, metricName);
- }
+ }
if (MetricState_.Time == TInstant::Zero()) {
MetricState_.Time = CommonTime_;
@@ -382,8 +382,8 @@ namespace NMonitoring {
MetricState_.Time,
MetricState_.Value.AsHistogram());
} else if (type == EMetricType::DSUMMARY) {
- Writer_.WriteSummaryDouble(
- metricName,
+ Writer_.WriteSummaryDouble(
+ metricName,
MetricState_.Labels,
MetricState_.Time,
MetricState_.Value.AsSummaryDouble());
diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp
index 562dec43cd..fd9debb060 100644
--- a/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp
+++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp
@@ -20,10 +20,10 @@ Y_UNIT_TEST_SUITE(TPrometheusEncoderTest) {
return ss.Str();
}
- ISummaryDoubleSnapshotPtr TestSummaryDouble() {
- return MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u);
- }
-
+ ISummaryDoubleSnapshotPtr TestSummaryDouble() {
+ return MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u);
+ }
+
Y_UNIT_TEST(Empty) {
auto result = EncodeToString([](IMetricEncoder* e) {
e->OnStreamBegin();
@@ -112,17 +112,17 @@ Y_UNIT_TEST_SUITE(TPrometheusEncoderTest) {
e->OnDouble(TInstant::Zero(), INFINITY);
e->OnMetricEnd();
}
- {
+ {
e->OnMetricBegin(EMetricType::DSUMMARY);
- {
- e->OnLabelsBegin();
- e->OnLabel("sensor", "seconds");
- e->OnLabel("disk", "sdb1");
- e->OnLabelsEnd();
- }
- e->OnSummaryDouble(TInstant::Zero(), TestSummaryDouble());
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "seconds");
+ e->OnLabel("disk", "sdb1");
+ e->OnLabelsEnd();
+ }
+ e->OnSummaryDouble(TInstant::Zero(), TestSummaryDouble());
e->OnMetricEnd();
- }
+ }
e->OnStreamEnd();
});
@@ -138,11 +138,11 @@ Y_UNIT_TEST_SUITE(TPrometheusEncoderTest) {
"nanValue nan\n"
"# TYPE infValue gauge\n"
"infValue inf\n"
- "seconds_sum{disk=\"sdb1\", } 10.1\n"
- "seconds_min{disk=\"sdb1\", } -0.45\n"
- "seconds_max{disk=\"sdb1\", } 0.478\n"
- "seconds_last{disk=\"sdb1\", } 0.3\n"
- "seconds_count{disk=\"sdb1\", } 30\n"
+ "seconds_sum{disk=\"sdb1\", } 10.1\n"
+ "seconds_min{disk=\"sdb1\", } -0.45\n"
+ "seconds_max{disk=\"sdb1\", } 0.478\n"
+ "seconds_last{disk=\"sdb1\", } 0.3\n"
+ "seconds_count{disk=\"sdb1\", } 30\n"
"\n");
}
diff --git a/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp b/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp
index b7e2fca629..2d11b9d5ba 100644
--- a/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp
+++ b/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp
@@ -19,9 +19,9 @@ namespace NMonitoring {
case EMetricType::HIST_RATE:
return NProto::HIST_RATE;
case EMetricType::DSUMMARY:
- return NProto::DSUMMARY;
- case EMetricType::LOGHIST:
- return NProto::LOGHISTOGRAM;
+ return NProto::DSUMMARY;
+ case EMetricType::LOGHIST:
+ return NProto::LOGHISTOGRAM;
case EMetricType::UNKNOWN:
return NProto::UNKNOWN;
}
@@ -37,23 +37,23 @@ namespace NMonitoring {
}
}
- void FillSummaryDouble(const ISummaryDoubleSnapshot& snapshot, NProto::TSummaryDouble* summary) {
- summary->SetSum(snapshot.GetSum());
- summary->SetMin(snapshot.GetMin());
- summary->SetMax(snapshot.GetMax());
- summary->SetLast(snapshot.GetLast());
- summary->SetCount(snapshot.GetCount());
- }
-
- void FillLogHistogram(const TLogHistogramSnapshot& snapshot, NProto::TLogHistogram* logHist) {
- logHist->SetBase(snapshot.Base());
- logHist->SetZerosCount(snapshot.ZerosCount());
- logHist->SetStartPower(snapshot.StartPower());
- for (ui32 i = 0; i < snapshot.Count(); ++i) {
- logHist->AddBuckets(snapshot.Bucket(i));
- }
- }
-
+ void FillSummaryDouble(const ISummaryDoubleSnapshot& snapshot, NProto::TSummaryDouble* summary) {
+ summary->SetSum(snapshot.GetSum());
+ summary->SetMin(snapshot.GetMin());
+ summary->SetMax(snapshot.GetMax());
+ summary->SetLast(snapshot.GetLast());
+ summary->SetCount(snapshot.GetCount());
+ }
+
+ void FillLogHistogram(const TLogHistogramSnapshot& snapshot, NProto::TLogHistogram* logHist) {
+ logHist->SetBase(snapshot.Base());
+ logHist->SetZerosCount(snapshot.ZerosCount());
+ logHist->SetStartPower(snapshot.StartPower());
+ for (ui32 i = 0; i < snapshot.Count(); ++i) {
+ logHist->AddBuckets(snapshot.Bucket(i));
+ }
+ }
+
///////////////////////////////////////////////////////////////////////////////
// TSingleamplesEncoder
///////////////////////////////////////////////////////////////////////////////
@@ -121,18 +121,18 @@ namespace NMonitoring {
FillHistogram(*snapshot, Sample_->MutableHistogram());
}
- void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
Y_ENSURE(Sample_, "metric not started");
- Sample_->SetTime(time.MilliSeconds());
- FillSummaryDouble(*snapshot, Sample_->MutableSummaryDouble());
- }
-
- void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override {
+ Sample_->SetTime(time.MilliSeconds());
+ FillSummaryDouble(*snapshot, Sample_->MutableSummaryDouble());
+ }
+
+ void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override {
Y_ENSURE(Sample_, "metric not started");
- Sample_->SetTime(time.MilliSeconds());
- FillLogHistogram(*snapshot, Sample_->MutableLogHistogram());
- }
-
+ Sample_->SetTime(time.MilliSeconds());
+ FillLogHistogram(*snapshot, Sample_->MutableLogHistogram());
+ }
+
void Close() override {
}
@@ -215,18 +215,18 @@ namespace NMonitoring {
void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
Y_ENSURE(Sample_, "metric not started");
- NProto::TPoint* point = Sample_->AddPoints();
- point->SetTime(time.MilliSeconds());
- FillSummaryDouble(*snapshot, point->MutableSummaryDouble());
- }
-
- void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override {
+ NProto::TPoint* point = Sample_->AddPoints();
+ point->SetTime(time.MilliSeconds());
+ FillSummaryDouble(*snapshot, point->MutableSummaryDouble());
+ }
+
+ void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override {
Y_ENSURE(Sample_, "metric not started");
- NProto::TPoint* point = Sample_->AddPoints();
- point->SetTime(time.MilliSeconds());
- FillLogHistogram(*snapshot, point->MutableLogHistogram());
- }
-
+ NProto::TPoint* point = Sample_->AddPoints();
+ point->SetTime(time.MilliSeconds());
+ FillLogHistogram(*snapshot, point->MutableLogHistogram());
+ }
+
void Close() override {
}
diff --git a/library/cpp/monlib/encode/protobuf/protos/samples.proto b/library/cpp/monlib/encode/protobuf/protos/samples.proto
index 6e6646303a..371f4181d2 100644
--- a/library/cpp/monlib/encode/protobuf/protos/samples.proto
+++ b/library/cpp/monlib/encode/protobuf/protos/samples.proto
@@ -19,8 +19,8 @@ enum EMetricType {
RATE = 4;
HISTOGRAM = 5;
HIST_RATE = 6;
- DSUMMARY = 7;
- LOGHISTOGRAM = 8;
+ DSUMMARY = 7;
+ LOGHISTOGRAM = 8;
}
message THistogram {
@@ -28,21 +28,21 @@ message THistogram {
repeated uint64 Values = 2; // values stored in each bucket
}
-message TLogHistogram {
- double Base = 1;
- uint64 ZerosCount = 2;
- int32 StartPower = 3;
- repeated double Buckets = 4;
-}
-
-message TSummaryDouble {
- double Sum = 1;
- double Min = 2;
- double Max = 3;
- double Last = 4;
- uint64 Count = 5;
-}
-
+message TLogHistogram {
+ double Base = 1;
+ uint64 ZerosCount = 2;
+ int32 StartPower = 3;
+ repeated double Buckets = 4;
+}
+
+message TSummaryDouble {
+ double Sum = 1;
+ double Min = 2;
+ double Max = 3;
+ double Last = 4;
+ uint64 Count = 5;
+}
+
// see TSingleSample
message TPoint {
uint64 Time = 1;
@@ -51,8 +51,8 @@ message TPoint {
fixed64 Uint64 = 3;
double Float64 = 4;
THistogram Histogram = 5;
- TSummaryDouble SummaryDouble = 6;
- TLogHistogram LogHistogram = 7;
+ TSummaryDouble SummaryDouble = 6;
+ TLogHistogram LogHistogram = 7;
}
}
@@ -67,8 +67,8 @@ message TSingleSample {
fixed64 Uint64 = 5;
double Float64 = 6;
THistogram Histogram = 7;
- TSummaryDouble SummaryDouble = 8;
- TLogHistogram LogHistogram = 9;
+ TSummaryDouble SummaryDouble = 8;
+ TLogHistogram LogHistogram = 9;
}
}
diff --git a/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp
index cffcf94df1..1f445fc80d 100644
--- a/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp
+++ b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp
@@ -167,49 +167,49 @@ namespace NMonitoring {
break;
case EMetricType::DSUMMARY:
- c->OnSummaryDouble(time, ReadSummaryDouble());
- break;
-
+ c->OnSummaryDouble(time, ReadSummaryDouble());
+ break;
+
case EMetricType::HIST:
case EMetricType::HIST_RATE:
c->OnHistogram(time, ReadHistogram());
break;
- case EMetricType::LOGHIST:
- c->OnLogHistogram(time, ReadLogHistogram());
- break;
-
+ case EMetricType::LOGHIST:
+ c->OnLogHistogram(time, ReadLogHistogram());
+ break;
+
default:
throw TSpackDecodeError() << "Unsupported metric type: " << metricType;
}
}
- ISummaryDoubleSnapshotPtr ReadSummaryDouble() {
- ui64 count = ReadFixed<ui64>();
- double sum = ReadFixed<double>();
- double min = ReadFixed<double>();
- double max = ReadFixed<double>();
- double last = ReadFixed<double>();
- return MakeIntrusive<TSummaryDoubleSnapshot>(sum, min, max, last, count);
- }
-
- TLogHistogramSnapshotPtr ReadLogHistogram() {
- double base = ReadFixed<double>();
- ui64 zerosCount = ReadFixed<ui64>();
- int startPower = static_cast<int>(ReadVarint());
- ui32 count = ReadVarint();
+ ISummaryDoubleSnapshotPtr ReadSummaryDouble() {
+ ui64 count = ReadFixed<ui64>();
+ double sum = ReadFixed<double>();
+ double min = ReadFixed<double>();
+ double max = ReadFixed<double>();
+ double last = ReadFixed<double>();
+ return MakeIntrusive<TSummaryDoubleSnapshot>(sum, min, max, last, count);
+ }
+
+ TLogHistogramSnapshotPtr ReadLogHistogram() {
+ double base = ReadFixed<double>();
+ ui64 zerosCount = ReadFixed<ui64>();
+ int startPower = static_cast<int>(ReadVarint());
+ ui32 count = ReadVarint();
// see https://a.yandex-team.ru/arc/trunk/arcadia/infra/yasm/stockpile_client/points.cpp?rev=r8593154#L31
// and https://a.yandex-team.ru/arc/trunk/arcadia/infra/yasm/common/points/hgram/normal/normal.h?rev=r8268697#L9
// TODO: share this constant value
Y_ENSURE(count <= 100u, "more than 100 buckets in log histogram: " << count);
- TVector<double> buckets;
- buckets.reserve(count);
- for (ui32 i = 0; i < count; ++i) {
- buckets.emplace_back(ReadFixed<double>());
- }
- return MakeIntrusive<TLogHistogramSnapshot>(base, zerosCount, startPower, std::move(buckets));
- }
-
+ TVector<double> buckets;
+ buckets.reserve(count);
+ for (ui32 i = 0; i < count; ++i) {
+ buckets.emplace_back(ReadFixed<double>());
+ }
+ return MakeIntrusive<TLogHistogramSnapshot>(base, zerosCount, startPower, std::move(buckets));
+ }
+
IHistogramSnapshotPtr ReadHistogram() {
ui32 bucketsCount = ReadVarint();
auto s = TExplicitHistogramSnapshot::New(bucketsCount);
@@ -245,9 +245,9 @@ namespace NMonitoring {
IMetricConsumer* c)
{
for (ui32 i = 0; i < count; i++) {
- auto nameIdx = ReadVarint();
- auto valueIdx = ReadVarint();
- c->OnLabel(labelNames.Get(nameIdx), labelValues.Get(valueIdx));
+ auto nameIdx = ReadVarint();
+ auto valueIdx = ReadVarint();
+ c->OnLabel(labelNames.Get(nameIdx), labelValues.Get(valueIdx));
}
}
@@ -385,7 +385,7 @@ namespace NMonitoring {
*result = EMetricType::DSUMMARY;
}
return true;
- } else if (byte == EncodeMetricType(EMetricType::LOGHIST)) {
+ } else if (byte == EncodeMetricType(EMetricType::LOGHIST)) {
if (result) {
*result = EMetricType::LOGHIST;
}
diff --git a/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp
index 727c772bc2..a2b0bb5f50 100644
--- a/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp
+++ b/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp
@@ -60,14 +60,14 @@ namespace NMonitoring {
TBufferedEncoderBase::OnHistogram(time, snapshot);
}
- void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
- TBufferedEncoderBase::OnSummaryDouble(time, snapshot);
- }
-
- void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override {
- TBufferedEncoderBase::OnLogHistogram(time, snapshot);
- }
-
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
+ TBufferedEncoderBase::OnSummaryDouble(time, snapshot);
+ }
+
+ void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override {
+ TBufferedEncoderBase::OnLogHistogram(time, snapshot);
+ }
+
void Close() override {
if (Closed_) {
return;
@@ -222,13 +222,13 @@ namespace NMonitoring {
break;
case EMetricType::DSUMMARY:
- WriteSummaryDouble(*value.AsSummaryDouble());
- break;
-
- case EMetricType::LOGHIST:
- WriteLogHistogram(*value.AsLogHistogram());
- break;
-
+ WriteSummaryDouble(*value.AsSummaryDouble());
+ break;
+
+ case EMetricType::LOGHIST:
+ WriteLogHistogram(*value.AsLogHistogram());
+ break;
+
default:
ythrow yexception() << "unsupported metric type: " << metricType;
}
@@ -267,24 +267,24 @@ namespace NMonitoring {
}
}
- void WriteLogHistogram(const TLogHistogramSnapshot& logHist) {
- WriteFixed(logHist.Base());
- WriteFixed(logHist.ZerosCount());
- WriteVarUInt32(Out_, static_cast<ui32>(logHist.StartPower()));
- WriteVarUInt32(Out_, logHist.Count());
- for (ui32 i = 0; i < logHist.Count(); ++i) {
- WriteFixed(logHist.Bucket(i));
- }
- }
-
- void WriteSummaryDouble(const ISummaryDoubleSnapshot& summary) {
- WriteFixed(summary.GetCount());
- WriteFixed(summary.GetSum());
- WriteFixed(summary.GetMin());
- WriteFixed(summary.GetMax());
- WriteFixed(summary.GetLast());
- }
-
+ void WriteLogHistogram(const TLogHistogramSnapshot& logHist) {
+ WriteFixed(logHist.Base());
+ WriteFixed(logHist.ZerosCount());
+ WriteVarUInt32(Out_, static_cast<ui32>(logHist.StartPower()));
+ WriteVarUInt32(Out_, logHist.Count());
+ for (ui32 i = 0; i < logHist.Count(); ++i) {
+ WriteFixed(logHist.Bucket(i));
+ }
+ }
+
+ void WriteSummaryDouble(const ISummaryDoubleSnapshot& summary) {
+ WriteFixed(summary.GetCount());
+ WriteFixed(summary.GetSum());
+ WriteFixed(summary.GetMin());
+ WriteFixed(summary.GetMax());
+ WriteFixed(summary.GetLast());
+ }
+
private:
IOutputStream* Out_;
ETimePrecision TimePrecision_;
diff --git a/library/cpp/monlib/encode/spack/spack_v1_ut.cpp b/library/cpp/monlib/encode/spack/spack_v1_ut.cpp
index 2d8a8fbb51..fe778eb7e0 100644
--- a/library/cpp/monlib/encode/spack/spack_v1_ut.cpp
+++ b/library/cpp/monlib/encode/spack/spack_v1_ut.cpp
@@ -57,9 +57,9 @@ Y_UNIT_TEST_SUITE(TSpackTest) {
0x00, // time precision (fixed ui8)
0x00, // compression algorithm (fixed ui8)
0x0d, 0x00, 0x00, 0x00, // label names size (fixed ui32)
- 0x40, 0x00, 0x00, 0x00, // labels values size (fixed ui32)
+ 0x40, 0x00, 0x00, 0x00, // labels values size (fixed ui32)
0x08, 0x00, 0x00, 0x00, // metric count (fixed ui32)
- 0x08, 0x00, 0x00, 0x00, // points count (fixed ui32)
+ 0x08, 0x00, 0x00, 0x00, // points count (fixed ui32)
};
ui8 expectedHeader[] = {
@@ -70,9 +70,9 @@ Y_UNIT_TEST_SUITE(TSpackTest) {
0x00, // time precision (fixed ui8)
0x00, // compression algorithm (fixed ui8)
0x0d, 0x00, 0x00, 0x00, // label names size (fixed ui32)
- 0x40, 0x00, 0x00, 0x00, // labels values size (fixed ui32)
+ 0x40, 0x00, 0x00, 0x00, // labels values size (fixed ui32)
0x08, 0x00, 0x00, 0x00, // metric count (fixed ui32)
- 0x08, 0x00, 0x00, 0x00, // points count (fixed ui32)
+ 0x08, 0x00, 0x00, 0x00, // points count (fixed ui32)
};
ui8 expectedStringPools[] = {
@@ -86,7 +86,7 @@ Y_UNIT_TEST_SUITE(TSpackTest) {
0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, // "responseTimeMillis\0"
0x54, 0x69, 0x6d, 0x65, 0x4d, 0x69, 0x6c, 0x6c,
0x69, 0x73, 0x00,
- 0x62, 0x79, 0x74, 0x65, 0x73, 0x00, // "bytes\0"
+ 0x62, 0x79, 0x74, 0x65, 0x73, 0x00, // "bytes\0"
0x74, 0x65, 0x6D, 0x70, 0x65, 0x72, 0x61, 0x74, // "temperature\0"
0x75, 0x72, 0x65, 0x00,
0x6d, 0x73, 0x00, // "ms\0"
@@ -191,42 +191,42 @@ Y_UNIT_TEST_SUITE(TSpackTest) {
0x00, // flags (fixed ui8)
0x01, // metric labels count (varint)
0x00, // label name index (varint)
- 0x06, // label value index (varint)
+ 0x06, // label value index (varint)
0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32)
0x39, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed i64)
};
ui8 expectedMetric7[] = {
- 0x1e, // types (DSUMMARY | ONE_WITH_TS) (fixed ui8)
- 0x00, // flags (fixed ui8)
+ 0x1e, // types (DSUMMARY | ONE_WITH_TS) (fixed ui8)
+ 0x00, // flags (fixed ui8)
0x01, // metric labels count (varint)
- 0x00, // label name index (varint)
+ 0x00, // label name index (varint)
0x07, // label value index (varint)
- 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32)
- 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // count (fixed ui64)
- 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x24, 0x40, // sum (fixed double)
- 0xcd, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xdc, 0xbf, // min (fixed double)
- 0x64, 0x3b, 0xdf, 0x4f, 0x8d, 0x97, 0xde, 0x3f, // max (fixed double)
- 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0xd3, 0x3f, // last (fixed double)
- };
-
+ 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32)
+ 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // count (fixed ui64)
+ 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x24, 0x40, // sum (fixed double)
+ 0xcd, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xdc, 0xbf, // min (fixed double)
+ 0x64, 0x3b, 0xdf, 0x4f, 0x8d, 0x97, 0xde, 0x3f, // max (fixed double)
+ 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0xd3, 0x3f, // last (fixed double)
+ };
+
ui8 expectedMetric8[] = {
0x26, // types (LOGHIST | ONE_WITH_TS) (fixed ui8)
- 0x00, // flags (fixed ui8)
+ 0x00, // flags (fixed ui8)
0x01, // metric labels count (variant)
- 0x00, // label name index (variant)
- 0x08, // label value index (variant)
- 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32)
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF8, 0x3F, // base (fixed double)
- 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // zerosCount (fixed ui64)
- 0x00, // startPower (variant)
- 0x04, // buckets count (variant)
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x3F,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0x3F, // bucket values
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0x3F,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x3F,
- };
-
+ 0x00, // label name index (variant)
+ 0x08, // label value index (variant)
+ 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32)
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF8, 0x3F, // base (fixed double)
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // zerosCount (fixed ui64)
+ 0x00, // startPower (variant)
+ 0x04, // buckets count (variant)
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x3F,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0x3F, // bucket values
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0x3F,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x3F,
+ };
+
const size_t expectedSize =
Y_ARRAY_SIZE(expectedHeader) +
Y_ARRAY_SIZE(expectedStringPools) +
@@ -252,15 +252,15 @@ Y_UNIT_TEST_SUITE(TSpackTest) {
return h->Snapshot();
}
- TLogHistogramSnapshotPtr TestLogHistogram() {
- TVector buckets{0.5, 0.25, 0.25, 0.5};
- return MakeIntrusive<TLogHistogramSnapshot>(1.5, 1u, 0, std::move(buckets));
- }
-
- ISummaryDoubleSnapshotPtr TestSummaryDouble() {
- return MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u);
- }
-
+ TLogHistogramSnapshotPtr TestLogHistogram() {
+ TVector buckets{0.5, 0.25, 0.25, 0.5};
+ return MakeIntrusive<TLogHistogramSnapshot>(1.5, 1u, 0, std::move(buckets));
+ }
+
+ ISummaryDoubleSnapshotPtr TestSummaryDouble() {
+ return MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u);
+ }
+
Y_UNIT_TEST(Encode) {
TBuffer buffer;
TBufferOutput out(buffer);
@@ -345,29 +345,29 @@ Y_UNIT_TEST_SUITE(TSpackTest) {
}
{ // metric 7
e->OnMetricBegin(EMetricType::DSUMMARY);
- {
- e->OnLabelsBegin();
+ {
+ e->OnLabelsBegin();
e->OnLabel("name", "temperature");
- e->OnLabelsEnd();
- }
- e->OnSummaryDouble(now, TestSummaryDouble());
+ e->OnLabelsEnd();
+ }
+ e->OnSummaryDouble(now, TestSummaryDouble());
e->OnMetricEnd();
- }
+ }
{ // metric 8
- e->OnMetricBegin(EMetricType::LOGHIST);
- {
- e->OnLabelsBegin();
+ e->OnMetricBegin(EMetricType::LOGHIST);
+ {
+ e->OnLabelsBegin();
e->OnLabel("name", "ms");
- e->OnLabelsEnd();
- }
- e->OnLogHistogram(now, TestLogHistogram());
- e->OnMetricEnd();
- }
+ e->OnLabelsEnd();
+ }
+ e->OnLogHistogram(now, TestLogHistogram());
+ e->OnMetricEnd();
+ }
e->OnStreamEnd();
e->Close();
- // Cout << "encoded: " << HexEncode(buffer.Data(), buffer.Size()) << Endl;
- // Cout << "size: " << buffer.Size() << Endl;
+ // Cout << "encoded: " << HexEncode(buffer.Data(), buffer.Size()) << Endl;
+ // Cout << "size: " << buffer.Size() << Endl;
UNIT_ASSERT_VALUES_EQUAL(buffer.Size(), expectedSize);
@@ -401,10 +401,10 @@ Y_UNIT_TEST_SUITE(TSpackTest) {
UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric6);
p += Y_ARRAY_SIZE(expectedMetric6);
-
+
UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric7);
p += Y_ARRAY_SIZE(expectedMetric7);
-
+
UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric8);
p += Y_ARRAY_SIZE(expectedMetric8);
}
@@ -509,7 +509,7 @@ Y_UNIT_TEST_SUITE(TSpackTest) {
UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1);
AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon");
- UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 8);
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 8);
{
const NProto::TMultiSample& s = samples.GetSamples(0);
UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE);
@@ -576,51 +576,51 @@ Y_UNIT_TEST_SUITE(TSpackTest) {
UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
AssertPointEqual(s.GetPoints(0), now, i64(1337));
}
- {
- const NProto::TMultiSample& s = samples.GetSamples(6);
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(6);
UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY);
- UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
AssertLabelEqual(s.GetLabels(0), "name", "temperature");
-
- UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
-
- const NProto::TPoint& p = s.GetPoints(0);
- UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds());
- UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kSummaryDouble);
-
- auto expected = TestSummaryDouble();
-
- auto actual = p.GetSummaryDouble();
-
- UNIT_ASSERT_VALUES_EQUAL(expected->GetSum(), actual.GetSum());
- UNIT_ASSERT_VALUES_EQUAL(expected->GetMin(), actual.GetMin());
- UNIT_ASSERT_VALUES_EQUAL(expected->GetMax(), actual.GetMax());
- UNIT_ASSERT_VALUES_EQUAL(expected->GetLast(), actual.GetLast());
- UNIT_ASSERT_VALUES_EQUAL(expected->GetCount(), actual.GetCount());
- }
- {
- const NProto::TMultiSample& s = samples.GetSamples(7);
- UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM);
- UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+
+ const NProto::TPoint& p = s.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kSummaryDouble);
+
+ auto expected = TestSummaryDouble();
+
+ auto actual = p.GetSummaryDouble();
+
+ UNIT_ASSERT_VALUES_EQUAL(expected->GetSum(), actual.GetSum());
+ UNIT_ASSERT_VALUES_EQUAL(expected->GetMin(), actual.GetMin());
+ UNIT_ASSERT_VALUES_EQUAL(expected->GetMax(), actual.GetMax());
+ UNIT_ASSERT_VALUES_EQUAL(expected->GetLast(), actual.GetLast());
+ UNIT_ASSERT_VALUES_EQUAL(expected->GetCount(), actual.GetCount());
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(7);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
AssertLabelEqual(s.GetLabels(0), "name", "ms");
-
- UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
-
- const NProto::TPoint& p = s.GetPoints(0);
- UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds());
- UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kLogHistogram);
-
- auto expected = TestLogHistogram();
- auto actual = p.GetLogHistogram();
-
- UNIT_ASSERT_VALUES_EQUAL(expected->ZerosCount(), actual.GetZerosCount());
- UNIT_ASSERT_VALUES_EQUAL(expected->Base(), actual.GetBase());
- UNIT_ASSERT_VALUES_EQUAL(expected->StartPower(), actual.GetStartPower());
- UNIT_ASSERT_VALUES_EQUAL(expected->Count(), actual.BucketsSize());
- for (size_t i = 0; i < expected->Count(); ++i) {
- UNIT_ASSERT_VALUES_EQUAL(expected->Bucket(i), actual.GetBuckets(i));
- }
- }
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+
+ const NProto::TPoint& p = s.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kLogHistogram);
+
+ auto expected = TestLogHistogram();
+ auto actual = p.GetLogHistogram();
+
+ UNIT_ASSERT_VALUES_EQUAL(expected->ZerosCount(), actual.GetZerosCount());
+ UNIT_ASSERT_VALUES_EQUAL(expected->Base(), actual.GetBase());
+ UNIT_ASSERT_VALUES_EQUAL(expected->StartPower(), actual.GetStartPower());
+ UNIT_ASSERT_VALUES_EQUAL(expected->Count(), actual.BucketsSize());
+ for (size_t i = 0; i < expected->Count(); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(expected->Bucket(i), actual.GetBuckets(i));
+ }
+ }
}
void TestCompression(ECompression alg) {
diff --git a/library/cpp/monlib/encode/text/text_encoder.cpp b/library/cpp/monlib/encode/text/text_encoder.cpp
index 8aca133542..10336261f0 100644
--- a/library/cpp/monlib/encode/text/text_encoder.cpp
+++ b/library/cpp/monlib/encode/text/text_encoder.cpp
@@ -94,16 +94,16 @@ namespace NMonitoring {
TimeSeries_.Add(time, snapshot.Get());
}
- void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
State_.Expect(TEncoderState::EState::METRIC);
- TimeSeries_.Add(time, snapshot.Get());
- }
-
+ TimeSeries_.Add(time, snapshot.Get());
+ }
+
void OnLogHistogram(TInstant ts, TLogHistogramSnapshotPtr snapshot) override {
State_.Expect(TEncoderState::EState::METRIC);
TimeSeries_.Add(ts, snapshot.Get());
- }
-
+ }
+
void Close() override {
}
@@ -132,11 +132,11 @@ namespace NMonitoring {
(*Out_) << *value.AsHistogram();
break;
case EMetricValueType::SUMMARY:
- (*Out_) << *value.AsSummaryDouble();
- break;
- case EMetricValueType::LOGHISTOGRAM:
+ (*Out_) << *value.AsSummaryDouble();
+ break;
+ case EMetricValueType::LOGHISTOGRAM:
(*Out_) << *value.AsLogHistogram();
- break;
+ break;
case EMetricValueType::UNKNOWN:
ythrow yexception() << "unknown metric value type";
}
diff --git a/library/cpp/monlib/encode/text/text_encoder_ut.cpp b/library/cpp/monlib/encode/text/text_encoder_ut.cpp
index 09418cff96..554b6f5fa9 100644
--- a/library/cpp/monlib/encode/text/text_encoder_ut.cpp
+++ b/library/cpp/monlib/encode/text/text_encoder_ut.cpp
@@ -260,24 +260,24 @@ Y_UNIT_TEST_SUITE(TTextText) {
" HIST readTimeMillis{} [{1: 0, 2: 0, 3: 1, 4: 0, 5: 7, inf: 1}]\n"
"HIST_RATE writeTimeMillis{} [{1: 0, 2: 0, 3: 1, 4: 0, 5: 7, inf: 1}]\n");
}
-
- Y_UNIT_TEST(Summary) {
- auto s = MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u);
+
+ Y_UNIT_TEST(Summary) {
+ auto s = MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u);
TString result = EncodeToString(true, [s](IMetricEncoder* e) {
- e->OnStreamBegin();
- {
+ e->OnStreamBegin();
+ {
e->OnMetricBegin(EMetricType::DSUMMARY);
- {
- e->OnLabelsBegin();
- e->OnLabel("sensor", "temperature");
- e->OnLabelsEnd();
- }
- e->OnSummaryDouble(TInstant::Zero(), s);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "temperature");
+ e->OnLabelsEnd();
+ }
+ e->OnSummaryDouble(TInstant::Zero(), s);
e->OnMetricEnd();
- }
- e->OnStreamEnd();
- });
- UNIT_ASSERT_STRINGS_EQUAL(result,
- " DSUMMARY temperature{} [{sum: 10.1, min: -0.45, max: 0.478, last: 0.3, count: 30}]\n");
- }
+ }
+ e->OnStreamEnd();
+ });
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+ " DSUMMARY temperature{} [{sum: 10.1, min: -0.45, max: 0.478, last: 0.3, count: 30}]\n");
+ }
}
diff --git a/library/cpp/monlib/encode/unistat/unistat_decoder.cpp b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp
index d8ba7d80af..b2344b0905 100644
--- a/library/cpp/monlib/encode/unistat/unistat_decoder.cpp
+++ b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp
@@ -211,7 +211,7 @@ namespace NMonitoring {
case EMetricType::HIST_RATE:
Consumer_->OnHistogram(Timestamp_, MetricContext_.Value.AsHistogram());
break;
- case EMetricType::LOGHIST:
+ case EMetricType::LOGHIST:
case EMetricType::DSUMMARY:
case EMetricType::IGAUGE:
case EMetricType::COUNTER:
diff --git a/library/cpp/monlib/metrics/histogram_snapshot.cpp b/library/cpp/monlib/metrics/histogram_snapshot.cpp
index 7df9762c1a..75b5811546 100644
--- a/library/cpp/monlib/metrics/histogram_snapshot.cpp
+++ b/library/cpp/monlib/metrics/histogram_snapshot.cpp
@@ -1,7 +1,7 @@
#include "histogram_snapshot.h"
-#include <util/stream/output.h>
-
+#include <util/stream/output.h>
+
#include <iostream>
@@ -23,16 +23,16 @@ namespace NMonitoring {
}
} // namespace NMonitoring
-
+
namespace {
-
+
template <typename TStream>
auto& Output(TStream& os, const NMonitoring::IHistogramSnapshot& hist) {
os << TStringBuf("{");
-
+
ui32 i = 0;
ui32 count = hist.Count();
-
+
if (count > 0) {
for (; i < count - 1; ++i) {
os << hist.UpperBound(i) << TStringBuf(": ") << hist.Value(i);
@@ -44,12 +44,12 @@ auto& Output(TStream& os, const NMonitoring::IHistogramSnapshot& hist) {
} else {
os << hist.UpperBound(i) << TStringBuf(": ") << hist.Value(i);
}
- }
-
+ }
+
os << TStringBuf("}");
return os;
-}
+}
} // namespace
diff --git a/library/cpp/monlib/metrics/labels.h b/library/cpp/monlib/metrics/labels.h
index 80a78efa6c..63dc997c28 100644
--- a/library/cpp/monlib/metrics/labels.h
+++ b/library/cpp/monlib/metrics/labels.h
@@ -292,12 +292,12 @@ namespace NMonitoring {
return tmp;
}
- void SortByName() {
+ void SortByName() {
std::sort(Labels_.begin(), Labels_.end(), [](const auto& lhs, const auto& rhs) {
- return lhs.Name() < rhs.Name();
- });
- }
-
+ return lhs.Name() < rhs.Name();
+ });
+ }
+
inline size_t Hash() const noexcept override {
return TSimpleRangeHash()(Labels_);
}
diff --git a/library/cpp/monlib/metrics/log_histogram_collector.h b/library/cpp/monlib/metrics/log_histogram_collector.h
index 9d51c887bc..b81f84ebf3 100644
--- a/library/cpp/monlib/metrics/log_histogram_collector.h
+++ b/library/cpp/monlib/metrics/log_histogram_collector.h
@@ -1,158 +1,158 @@
-#pragma once
-
-#include "log_histogram_snapshot.h"
-
-#include <util/generic/algorithm.h>
-#include <util/generic/utility.h>
-#include <util/generic/yexception.h>
-
-#include <mutex>
-#include <cmath>
-
-namespace NMonitoring {
-
- class TLogHistogramCollector {
- public:
- static constexpr int DEFAULT_START_POWER = -1;
-
- explicit TLogHistogramCollector(int startPower = DEFAULT_START_POWER)
- : StartPower_(startPower)
- , CountZero_(0u)
- {}
-
- void Collect(TLogHistogramSnapshot* logHist) {
- std::lock_guard guard(Mutex_);
- Merge(logHist);
- }
-
+#pragma once
+
+#include "log_histogram_snapshot.h"
+
+#include <util/generic/algorithm.h>
+#include <util/generic/utility.h>
+#include <util/generic/yexception.h>
+
+#include <mutex>
+#include <cmath>
+
+namespace NMonitoring {
+
+ class TLogHistogramCollector {
+ public:
+ static constexpr int DEFAULT_START_POWER = -1;
+
+ explicit TLogHistogramCollector(int startPower = DEFAULT_START_POWER)
+ : StartPower_(startPower)
+ , CountZero_(0u)
+ {}
+
+ void Collect(TLogHistogramSnapshot* logHist) {
+ std::lock_guard guard(Mutex_);
+ Merge(logHist);
+ }
+
bool Collect(double value) {
- std::lock_guard guard(Mutex_);
+ std::lock_guard guard(Mutex_);
return CollectDouble(value);
- }
-
- TLogHistogramSnapshotPtr Snapshot() const {
- std::lock_guard guard(Mutex_);
- return MakeIntrusive<TLogHistogramSnapshot>(BASE, CountZero_, StartPower_, Buckets_);
- }
-
- void AddZeros(ui64 zerosCount) noexcept {
- std::lock_guard guard(Mutex_);
- CountZero_ += zerosCount;
- }
-
- private:
- int StartPower_;
- ui64 CountZero_;
- TVector<double> Buckets_;
- mutable std::mutex Mutex_;
-
- static constexpr size_t MAX_BUCKETS = LOG_HIST_MAX_BUCKETS;
- static constexpr double BASE = 1.5;
-
- private:
- int EstimateBucketIndex(double value) const {
- return (int) (std::floor(std::log(value) / std::log(BASE)) - StartPower_);
- }
-
- void CollectPositiveDouble(double value) {
- ssize_t idx = std::floor(std::log(value) / std::log(BASE)) - StartPower_;
- if (idx >= Buckets_.ysize()) {
- idx = ExtendUp(idx);
- } else if (idx <= 0) {
- idx = Max<ssize_t>(0, ExtendDown(idx, 1));
- }
- ++Buckets_[idx];
- }
-
+ }
+
+ TLogHistogramSnapshotPtr Snapshot() const {
+ std::lock_guard guard(Mutex_);
+ return MakeIntrusive<TLogHistogramSnapshot>(BASE, CountZero_, StartPower_, Buckets_);
+ }
+
+ void AddZeros(ui64 zerosCount) noexcept {
+ std::lock_guard guard(Mutex_);
+ CountZero_ += zerosCount;
+ }
+
+ private:
+ int StartPower_;
+ ui64 CountZero_;
+ TVector<double> Buckets_;
+ mutable std::mutex Mutex_;
+
+ static constexpr size_t MAX_BUCKETS = LOG_HIST_MAX_BUCKETS;
+ static constexpr double BASE = 1.5;
+
+ private:
+ int EstimateBucketIndex(double value) const {
+ return (int) (std::floor(std::log(value) / std::log(BASE)) - StartPower_);
+ }
+
+ void CollectPositiveDouble(double value) {
+ ssize_t idx = std::floor(std::log(value) / std::log(BASE)) - StartPower_;
+ if (idx >= Buckets_.ysize()) {
+ idx = ExtendUp(idx);
+ } else if (idx <= 0) {
+ idx = Max<ssize_t>(0, ExtendDown(idx, 1));
+ }
+ ++Buckets_[idx];
+ }
+
bool CollectDouble(double value) {
if (Y_UNLIKELY(std::isnan(value) || std::isinf(value))) {
return false;
}
- if (value <= 0.0) {
- ++CountZero_;
- } else {
- CollectPositiveDouble(value);
- }
+ if (value <= 0.0) {
+ ++CountZero_;
+ } else {
+ CollectPositiveDouble(value);
+ }
return true;
- }
-
- void Merge(TLogHistogramSnapshot* logHist) {
- CountZero_ += logHist->ZerosCount();
- const i32 firstIdxBeforeExtend = logHist->StartPower() - StartPower_;
- const i32 lastIdxBeforeExtend = firstIdxBeforeExtend + logHist->Count() - 1;
- if (firstIdxBeforeExtend > Max<i16>() || firstIdxBeforeExtend < Min<i16>()) {
- ythrow yexception() << "i16 overflow on first index";
- }
- if (lastIdxBeforeExtend > Max<i16>() || lastIdxBeforeExtend < Min<i16>()) {
- ythrow yexception() << "i16 overflow on last index";
- }
- i64 firstIdx = ExtendBounds(firstIdxBeforeExtend, lastIdxBeforeExtend, 0).first;
- size_t toMerge = std::min<ui32>(std::max<i64>(-firstIdx, (i64) 0), logHist->Count());
- if (toMerge) {
- for (size_t i = 0; i < toMerge; ++i) {
- Buckets_[0] += logHist->Bucket(i);
- }
- firstIdx = 0;
- }
- for (size_t i = toMerge; i != logHist->Count(); ++i) {
- Buckets_[firstIdx] += logHist->Bucket(i);
- ++firstIdx;
- }
- }
-
- int ExtendUp(int expectedIndex) {
- Y_VERIFY_DEBUG(expectedIndex >= (int) Buckets_.size());
- const size_t toAdd = expectedIndex - Buckets_.size() + 1;
- const size_t newSize = Buckets_.size() + toAdd;
- if (newSize <= MAX_BUCKETS) {
- Buckets_.resize(newSize, 0.0);
- return expectedIndex;
- }
-
- const size_t toRemove = newSize - MAX_BUCKETS;
- const size_t actualToRemove = std::min<size_t>(toRemove, Buckets_.size());
- if (actualToRemove > 0) {
- const double firstWeight = std::accumulate(Buckets_.cbegin(), Buckets_.cbegin() + actualToRemove, 0.0);
- Buckets_.erase(Buckets_.cbegin(), Buckets_.cbegin() + actualToRemove);
- if (Buckets_.empty()) {
- Buckets_.push_back(firstWeight);
- } else {
- Buckets_[0] = firstWeight;
- }
- }
- Buckets_.resize(MAX_BUCKETS, 0.0);
- StartPower_ += toRemove;
- return expectedIndex - toRemove;
- }
-
- int ExtendDown(int expectedIndex, int margin) {
- Y_VERIFY_DEBUG(expectedIndex <= 0);
- int toAdd = std::min<int>(MAX_BUCKETS - Buckets_.size(), margin - expectedIndex);
- if (toAdd > 0) {
- Buckets_.insert(Buckets_.begin(), toAdd, 0.0);
- StartPower_ -= toAdd;
- }
- return expectedIndex + toAdd;
- }
-
- std::pair<ssize_t, ssize_t> ExtendBounds(ssize_t startIdx, ssize_t endIdx, ui8 margin) {
- ssize_t realEndIdx;
- ssize_t realStartIdx;
- if (endIdx >= Buckets_.ysize()) {
- Buckets_.reserve(std::max<size_t>(std::min<ui32>(endIdx - startIdx + 1ul, MAX_BUCKETS), 0ul));
- realEndIdx = ExtendUp(endIdx);
- startIdx += realEndIdx - endIdx;
- } else {
- realEndIdx = endIdx;
- }
- if (startIdx < 1) {
- realStartIdx = ExtendDown(startIdx, margin);
- realEndIdx += realStartIdx - startIdx;
- } else {
- realStartIdx = startIdx;
- }
- return std::make_pair(realStartIdx, realEndIdx);
- }
- };
-
-} // namespace NMonitoring
+ }
+
+ void Merge(TLogHistogramSnapshot* logHist) {
+ CountZero_ += logHist->ZerosCount();
+ const i32 firstIdxBeforeExtend = logHist->StartPower() - StartPower_;
+ const i32 lastIdxBeforeExtend = firstIdxBeforeExtend + logHist->Count() - 1;
+ if (firstIdxBeforeExtend > Max<i16>() || firstIdxBeforeExtend < Min<i16>()) {
+ ythrow yexception() << "i16 overflow on first index";
+ }
+ if (lastIdxBeforeExtend > Max<i16>() || lastIdxBeforeExtend < Min<i16>()) {
+ ythrow yexception() << "i16 overflow on last index";
+ }
+ i64 firstIdx = ExtendBounds(firstIdxBeforeExtend, lastIdxBeforeExtend, 0).first;
+ size_t toMerge = std::min<ui32>(std::max<i64>(-firstIdx, (i64) 0), logHist->Count());
+ if (toMerge) {
+ for (size_t i = 0; i < toMerge; ++i) {
+ Buckets_[0] += logHist->Bucket(i);
+ }
+ firstIdx = 0;
+ }
+ for (size_t i = toMerge; i != logHist->Count(); ++i) {
+ Buckets_[firstIdx] += logHist->Bucket(i);
+ ++firstIdx;
+ }
+ }
+
+ int ExtendUp(int expectedIndex) {
+ Y_VERIFY_DEBUG(expectedIndex >= (int) Buckets_.size());
+ const size_t toAdd = expectedIndex - Buckets_.size() + 1;
+ const size_t newSize = Buckets_.size() + toAdd;
+ if (newSize <= MAX_BUCKETS) {
+ Buckets_.resize(newSize, 0.0);
+ return expectedIndex;
+ }
+
+ const size_t toRemove = newSize - MAX_BUCKETS;
+ const size_t actualToRemove = std::min<size_t>(toRemove, Buckets_.size());
+ if (actualToRemove > 0) {
+ const double firstWeight = std::accumulate(Buckets_.cbegin(), Buckets_.cbegin() + actualToRemove, 0.0);
+ Buckets_.erase(Buckets_.cbegin(), Buckets_.cbegin() + actualToRemove);
+ if (Buckets_.empty()) {
+ Buckets_.push_back(firstWeight);
+ } else {
+ Buckets_[0] = firstWeight;
+ }
+ }
+ Buckets_.resize(MAX_BUCKETS, 0.0);
+ StartPower_ += toRemove;
+ return expectedIndex - toRemove;
+ }
+
+ int ExtendDown(int expectedIndex, int margin) {
+ Y_VERIFY_DEBUG(expectedIndex <= 0);
+ int toAdd = std::min<int>(MAX_BUCKETS - Buckets_.size(), margin - expectedIndex);
+ if (toAdd > 0) {
+ Buckets_.insert(Buckets_.begin(), toAdd, 0.0);
+ StartPower_ -= toAdd;
+ }
+ return expectedIndex + toAdd;
+ }
+
+ std::pair<ssize_t, ssize_t> ExtendBounds(ssize_t startIdx, ssize_t endIdx, ui8 margin) {
+ ssize_t realEndIdx;
+ ssize_t realStartIdx;
+ if (endIdx >= Buckets_.ysize()) {
+ Buckets_.reserve(std::max<size_t>(std::min<ui32>(endIdx - startIdx + 1ul, MAX_BUCKETS), 0ul));
+ realEndIdx = ExtendUp(endIdx);
+ startIdx += realEndIdx - endIdx;
+ } else {
+ realEndIdx = endIdx;
+ }
+ if (startIdx < 1) {
+ realStartIdx = ExtendDown(startIdx, margin);
+ realEndIdx += realStartIdx - startIdx;
+ } else {
+ realStartIdx = startIdx;
+ }
+ return std::make_pair(realStartIdx, realEndIdx);
+ }
+ };
+
+} // namespace NMonitoring
diff --git a/library/cpp/monlib/metrics/log_histogram_collector_ut.cpp b/library/cpp/monlib/metrics/log_histogram_collector_ut.cpp
index 560ec83714..ac9a3522ce 100644
--- a/library/cpp/monlib/metrics/log_histogram_collector_ut.cpp
+++ b/library/cpp/monlib/metrics/log_histogram_collector_ut.cpp
@@ -1,38 +1,38 @@
-#include "log_histogram_collector.h"
-
-#include <library/cpp/testing/unittest/registar.h>
-
-Y_UNIT_TEST_SUITE(LogHistogramCollector) {
-
- Y_UNIT_TEST(ExtendUpEmpty) {
- NMonitoring::TLogHistogramCollector collector(-1);
- collector.Collect(4.1944122207138854e+17);
- auto s = collector.Snapshot();
- UNIT_ASSERT_EQUAL(s->ZerosCount(), 0);
- UNIT_ASSERT_EQUAL(s->StartPower(), 1);
- UNIT_ASSERT_EQUAL(s->Count(), 100);
- UNIT_ASSERT_EQUAL(s->Bucket(s->Count() - 1), 1);
- }
-
- Y_UNIT_TEST(ExtendUpNonEmpty) {
- NMonitoring::TLogHistogramCollector collector(-1);
- collector.Collect(0.0);
- collector.Collect(1/(1.5*1.5*1.5));
- collector.Collect(1/1.5);
- auto s = collector.Snapshot();
-
- UNIT_ASSERT_EQUAL(s->ZerosCount(), 1);
- UNIT_ASSERT_EQUAL(s->StartPower(), -4);
- UNIT_ASSERT_EQUAL(s->Count(), 3);
- UNIT_ASSERT_EQUAL(s->Bucket(1), 1);
- UNIT_ASSERT_EQUAL(s->Bucket(2), 1);
-
- collector.Collect(4.1944122207138854e+17);
- s = collector.Snapshot();
- UNIT_ASSERT_EQUAL(s->ZerosCount(), 1);
- UNIT_ASSERT_EQUAL(s->StartPower(), 1);
- UNIT_ASSERT_EQUAL(s->Count(), 100);
- UNIT_ASSERT_EQUAL(s->Bucket(0), 2);
- UNIT_ASSERT_EQUAL(s->Bucket(99), 1);
- }
-}
+#include "log_histogram_collector.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(LogHistogramCollector) {
+
+ Y_UNIT_TEST(ExtendUpEmpty) {
+ NMonitoring::TLogHistogramCollector collector(-1);
+ collector.Collect(4.1944122207138854e+17);
+ auto s = collector.Snapshot();
+ UNIT_ASSERT_EQUAL(s->ZerosCount(), 0);
+ UNIT_ASSERT_EQUAL(s->StartPower(), 1);
+ UNIT_ASSERT_EQUAL(s->Count(), 100);
+ UNIT_ASSERT_EQUAL(s->Bucket(s->Count() - 1), 1);
+ }
+
+ Y_UNIT_TEST(ExtendUpNonEmpty) {
+ NMonitoring::TLogHistogramCollector collector(-1);
+ collector.Collect(0.0);
+ collector.Collect(1/(1.5*1.5*1.5));
+ collector.Collect(1/1.5);
+ auto s = collector.Snapshot();
+
+ UNIT_ASSERT_EQUAL(s->ZerosCount(), 1);
+ UNIT_ASSERT_EQUAL(s->StartPower(), -4);
+ UNIT_ASSERT_EQUAL(s->Count(), 3);
+ UNIT_ASSERT_EQUAL(s->Bucket(1), 1);
+ UNIT_ASSERT_EQUAL(s->Bucket(2), 1);
+
+ collector.Collect(4.1944122207138854e+17);
+ s = collector.Snapshot();
+ UNIT_ASSERT_EQUAL(s->ZerosCount(), 1);
+ UNIT_ASSERT_EQUAL(s->StartPower(), 1);
+ UNIT_ASSERT_EQUAL(s->Count(), 100);
+ UNIT_ASSERT_EQUAL(s->Bucket(0), 2);
+ UNIT_ASSERT_EQUAL(s->Bucket(99), 1);
+ }
+}
diff --git a/library/cpp/monlib/metrics/log_histogram_snapshot.h b/library/cpp/monlib/metrics/log_histogram_snapshot.h
index 7f7826d49d..7673b43751 100644
--- a/library/cpp/monlib/metrics/log_histogram_snapshot.h
+++ b/library/cpp/monlib/metrics/log_histogram_snapshot.h
@@ -1,71 +1,71 @@
-#pragma once
-
-#include <util/generic/ptr.h>
-#include <util/generic/vector.h>
-
-#include <cmath>
-
-namespace NMonitoring {
-
- constexpr ui32 LOG_HIST_MAX_BUCKETS = 100;
-
- class TLogHistogramSnapshot: public TAtomicRefCount<TLogHistogramSnapshot> {
- public:
- TLogHistogramSnapshot(double base, ui64 zerosCount, int startPower, TVector<double> buckets)
- : Base_(base)
- , ZerosCount_(zerosCount)
- , StartPower_(startPower)
- , Buckets_(std::move(buckets)) {
- }
-
- /**
- * @return buckets count.
- */
- ui32 Count() const noexcept {
- return Buckets_.size();
- }
-
- /**
- * @return upper bound for the bucket with particular index.
- */
- double UpperBound(int index) const noexcept {
- return std::pow(Base_, StartPower_ + index);
- }
-
- /**
- * @return value stored in the bucket with particular index.
- */
- double Bucket(ui32 index) const noexcept {
- return Buckets_[index];
- }
-
- /**
- * @return nonpositive values count
- */
- ui64 ZerosCount() const noexcept {
- return ZerosCount_;
- }
-
- double Base() const noexcept {
- return Base_;
- }
-
- int StartPower() const noexcept {
- return StartPower_;
- }
-
- ui64 MemorySizeBytes() const noexcept {
- return sizeof(*this) + Buckets_.capacity() * sizeof(double);
- }
-
- private:
- double Base_;
- ui64 ZerosCount_;
- int StartPower_;
- TVector<double> Buckets_;
- };
-
- using TLogHistogramSnapshotPtr = TIntrusivePtr<TLogHistogramSnapshot>;
-}
+#pragma once
+
+#include <util/generic/ptr.h>
+#include <util/generic/vector.h>
+
+#include <cmath>
+
+namespace NMonitoring {
+
+ constexpr ui32 LOG_HIST_MAX_BUCKETS = 100;
+
+ class TLogHistogramSnapshot: public TAtomicRefCount<TLogHistogramSnapshot> {
+ public:
+ TLogHistogramSnapshot(double base, ui64 zerosCount, int startPower, TVector<double> buckets)
+ : Base_(base)
+ , ZerosCount_(zerosCount)
+ , StartPower_(startPower)
+ , Buckets_(std::move(buckets)) {
+ }
+
+ /**
+ * @return buckets count.
+ */
+ ui32 Count() const noexcept {
+ return Buckets_.size();
+ }
+
+ /**
+ * @return upper bound for the bucket with particular index.
+ */
+ double UpperBound(int index) const noexcept {
+ return std::pow(Base_, StartPower_ + index);
+ }
+
+ /**
+ * @return value stored in the bucket with particular index.
+ */
+ double Bucket(ui32 index) const noexcept {
+ return Buckets_[index];
+ }
+
+ /**
+ * @return nonpositive values count
+ */
+ ui64 ZerosCount() const noexcept {
+ return ZerosCount_;
+ }
+
+ double Base() const noexcept {
+ return Base_;
+ }
+
+ int StartPower() const noexcept {
+ return StartPower_;
+ }
+
+ ui64 MemorySizeBytes() const noexcept {
+ return sizeof(*this) + Buckets_.capacity() * sizeof(double);
+ }
+
+ private:
+ double Base_;
+ ui64 ZerosCount_;
+ int StartPower_;
+ TVector<double> Buckets_;
+ };
+
+ using TLogHistogramSnapshotPtr = TIntrusivePtr<TLogHistogramSnapshot>;
+}
std::ostream& operator<<(std::ostream& os, const NMonitoring::TLogHistogramSnapshot& hist);
diff --git a/library/cpp/monlib/metrics/metric_consumer.h b/library/cpp/monlib/metrics/metric_consumer.h
index 7049bce68c..f7a727585a 100644
--- a/library/cpp/monlib/metrics/metric_consumer.h
+++ b/library/cpp/monlib/metrics/metric_consumer.h
@@ -2,8 +2,8 @@
#include "metric_type.h"
#include "histogram_collector.h"
-#include "summary_collector.h"
-#include "log_histogram_snapshot.h"
+#include "summary_collector.h"
+#include "log_histogram_snapshot.h"
class TInstant;
@@ -31,7 +31,7 @@ namespace NMonitoring {
virtual void OnUint64(TInstant time, ui64 value) = 0;
virtual void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) = 0;
- virtual void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) = 0;
+ virtual void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) = 0;
virtual void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) = 0;
};
diff --git a/library/cpp/monlib/metrics/metric_registry_ut.cpp b/library/cpp/monlib/metrics/metric_registry_ut.cpp
index b27214e62a..86d9a52ec0 100644
--- a/library/cpp/monlib/metrics/metric_registry_ut.cpp
+++ b/library/cpp/monlib/metrics/metric_registry_ut.cpp
@@ -25,12 +25,12 @@ void Out<NMonitoring::NProto::TSingleSample::ValueCase>(IOutputStream& os, NMoni
case NMonitoring::NProto::TSingleSample::ValueCase::kFloat64:
os << "Float64";
break;
- case NMonitoring::NProto::TSingleSample::ValueCase::kSummaryDouble:
- os << "DSummary";
- break;
- case NMonitoring::NProto::TSingleSample::ValueCase::kLogHistogram:
- os << "LogHistogram";
- break;
+ case NMonitoring::NProto::TSingleSample::ValueCase::kSummaryDouble:
+ os << "DSummary";
+ break;
+ case NMonitoring::NProto::TSingleSample::ValueCase::kLogHistogram:
+ os << "LogHistogram";
+ break;
case NMonitoring::NProto::TSingleSample::ValueCase::VALUE_NOT_SET:
os << "NOT SET";
break;
diff --git a/library/cpp/monlib/metrics/metric_type.h b/library/cpp/monlib/metrics/metric_type.h
index 63ee9b8d1b..1984c42c1e 100644
--- a/library/cpp/monlib/metrics/metric_type.h
+++ b/library/cpp/monlib/metrics/metric_type.h
@@ -14,7 +14,7 @@ namespace NMonitoring {
IGAUGE = 4,
HIST = 5,
HIST_RATE = 6,
- DSUMMARY = 7,
+ DSUMMARY = 7,
// ISUMMARY = 8, reserved
LOGHIST = 9,
};
diff --git a/library/cpp/monlib/metrics/metric_value.cpp b/library/cpp/monlib/metrics/metric_value.cpp
index 9a7d6fe1a1..b95d7011c6 100644
--- a/library/cpp/monlib/metrics/metric_value.cpp
+++ b/library/cpp/monlib/metrics/metric_value.cpp
@@ -12,15 +12,15 @@ namespace NMonitoring {
SnapshotUnRef<EMetricValueType::HISTOGRAM>(p);
}
} else if (ValueType_ == EMetricValueType::SUMMARY) {
- for (TPoint& p: Points_) {
+ for (TPoint& p: Points_) {
SnapshotUnRef<EMetricValueType::SUMMARY>(p);
- }
- } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
- for (TPoint& p: Points_) {
- SnapshotUnRef<EMetricValueType::LOGHISTOGRAM>(p);
- }
+ }
+ } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
+ for (TPoint& p: Points_) {
+ SnapshotUnRef<EMetricValueType::LOGHISTOGRAM>(p);
+ }
}
-
+
Points_.clear();
ValueType_ = EMetricValueType::UNKNOWN;
}
diff --git a/library/cpp/monlib/metrics/metric_value.h b/library/cpp/monlib/metrics/metric_value.h
index a9d15e75f6..607fcc8602 100644
--- a/library/cpp/monlib/metrics/metric_value.h
+++ b/library/cpp/monlib/metrics/metric_value.h
@@ -2,8 +2,8 @@
#include "histogram_collector.h"
#include "metric_value_type.h"
-#include "summary_collector.h"
-#include "log_histogram_snapshot.h"
+#include "summary_collector.h"
+#include "log_histogram_snapshot.h"
#include <util/datetime/base.h>
#include <util/generic/algorithm.h>
@@ -43,20 +43,20 @@ namespace NMonitoring {
static constexpr auto Type = EMetricValueType::UINT64;
};
- template <>
- struct TValueType<TLogHistogramSnapshot*> {
- static constexpr auto Type = EMetricValueType::LOGHISTOGRAM;
- };
-
+ template <>
+ struct TValueType<TLogHistogramSnapshot*> {
+ static constexpr auto Type = EMetricValueType::LOGHISTOGRAM;
+ };
+
template <typename T>
struct TValueType<T*, typename std::enable_if_t<std::is_base_of<IHistogramSnapshot, T>::value>> {
static constexpr auto Type = EMetricValueType::HISTOGRAM;
};
- template <typename T>
- struct TValueType<T*, typename std::enable_if_t<std::is_base_of<ISummaryDoubleSnapshot, T>::value>> {
+ template <typename T>
+ struct TValueType<T*, typename std::enable_if_t<std::is_base_of<ISummaryDoubleSnapshot, T>::value>> {
static constexpr auto Type = EMetricValueType::SUMMARY;
- };
+ };
///////////////////////////////////////////////////////////////////////////
// TMetricValue
@@ -90,13 +90,13 @@ namespace NMonitoring {
}
explicit TMetricValue(ISummaryDoubleSnapshot* summary) noexcept {
- Value_.Summary = summary;
- }
-
- explicit TMetricValue(TLogHistogramSnapshot* logHist) noexcept {
- Value_.LogHistogram = logHist;
- }
-
+ Value_.Summary = summary;
+ }
+
+ explicit TMetricValue(TLogHistogramSnapshot* logHist) noexcept {
+ Value_.LogHistogram = logHist;
+ }
+
double AsDouble() const noexcept {
return Value_.Double;
}
@@ -114,9 +114,9 @@ namespace NMonitoring {
case EMetricValueType::HISTOGRAM:
ythrow yexception() << "histogram cannot be casted to Double";
case EMetricValueType::SUMMARY:
- ythrow yexception() << "summary cannot be casted to Double";
- case EMetricValueType::LOGHISTOGRAM:
- ythrow yexception() << "loghistogram cannot be casted to Double";
+ ythrow yexception() << "summary cannot be casted to Double";
+ case EMetricValueType::LOGHISTOGRAM:
+ ythrow yexception() << "loghistogram cannot be casted to Double";
case EMetricValueType::UNKNOWN:
ythrow yexception() << "unknown value type";
}
@@ -140,9 +140,9 @@ namespace NMonitoring {
case EMetricValueType::HISTOGRAM:
ythrow yexception() << "histogram cannot be casted to Uint64";
case EMetricValueType::SUMMARY:
- ythrow yexception() << "summary cannot be casted to Uint64";
- case EMetricValueType::LOGHISTOGRAM:
- ythrow yexception() << "loghistogram cannot be casted to Uint64";
+ ythrow yexception() << "summary cannot be casted to Uint64";
+ case EMetricValueType::LOGHISTOGRAM:
+ ythrow yexception() << "loghistogram cannot be casted to Uint64";
case EMetricValueType::UNKNOWN:
ythrow yexception() << "unknown value type";
}
@@ -166,9 +166,9 @@ namespace NMonitoring {
case EMetricValueType::HISTOGRAM:
ythrow yexception() << "histogram cannot be casted to Int64";
case EMetricValueType::SUMMARY:
- ythrow yexception() << "summary cannot be casted to Int64";
- case EMetricValueType::LOGHISTOGRAM:
- ythrow yexception() << "loghistogram cannot be casted to Int64";
+ ythrow yexception() << "summary cannot be casted to Int64";
+ case EMetricValueType::LOGHISTOGRAM:
+ ythrow yexception() << "loghistogram cannot be casted to Int64";
case EMetricValueType::UNKNOWN:
ythrow yexception() << "unknown value type";
}
@@ -187,10 +187,10 @@ namespace NMonitoring {
return Value_.Histogram;
}
- ISummaryDoubleSnapshot* AsSummaryDouble() const noexcept {
- return Value_.Summary;
- }
-
+ ISummaryDoubleSnapshot* AsSummaryDouble() const noexcept {
+ return Value_.Summary;
+ }
+
ISummaryDoubleSnapshot* AsSummaryDouble(EMetricValueType type) const {
if (type != EMetricValueType::SUMMARY) {
ythrow yexception() << type << " cannot be casted to SummaryDouble";
@@ -199,26 +199,26 @@ namespace NMonitoring {
return Value_.Summary;
}
- TLogHistogramSnapshot* AsLogHistogram() const noexcept {
- return Value_.LogHistogram;
- }
-
- TLogHistogramSnapshot* AsLogHistogram(EMetricValueType type) const {
- if (type != EMetricValueType::LOGHISTOGRAM) {
- ythrow yexception() << type << " cannot be casted to LogHistogram";
- }
-
- return Value_.LogHistogram;
- }
-
+ TLogHistogramSnapshot* AsLogHistogram() const noexcept {
+ return Value_.LogHistogram;
+ }
+
+ TLogHistogramSnapshot* AsLogHistogram(EMetricValueType type) const {
+ if (type != EMetricValueType::LOGHISTOGRAM) {
+ ythrow yexception() << type << " cannot be casted to LogHistogram";
+ }
+
+ return Value_.LogHistogram;
+ }
+
protected:
union {
double Double;
i64 Int64;
ui64 Uint64;
IHistogramSnapshot* Histogram;
- ISummaryDoubleSnapshot* Summary;
- TLogHistogramSnapshot* LogHistogram;
+ ISummaryDoubleSnapshot* Summary;
+ TLogHistogramSnapshot* LogHistogram;
} Value_;
};
@@ -290,18 +290,18 @@ namespace NMonitoring {
return TBase::AsSummaryDouble(ValueType_);
}
- TLogHistogramSnapshot* AsLogHistogram() const {
- return TBase::AsLogHistogram(ValueType_);
- }
-
+ TLogHistogramSnapshot* AsLogHistogram() const {
+ return TBase::AsLogHistogram(ValueType_);
+ }
+
private:
void Ref() {
if (ValueType_ == EMetricValueType::SUMMARY) {
TBase::AsSummaryDouble()->Ref();
} else if (ValueType_ == EMetricValueType::HISTOGRAM) {
TBase::AsHistogram()->Ref();
- } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
- TBase::AsLogHistogram()->Ref();
+ } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
+ TBase::AsLogHistogram()->Ref();
}
}
@@ -310,8 +310,8 @@ namespace NMonitoring {
TBase::AsSummaryDouble()->UnRef();
} else if (ValueType_ == EMetricValueType::HISTOGRAM) {
TBase::AsHistogram()->UnRef();
- } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
- TBase::AsLogHistogram()->UnRef();
+ } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
+ TBase::AsLogHistogram()->UnRef();
}
}
@@ -401,7 +401,7 @@ namespace NMonitoring {
} else if (ValueType_ == EMetricValueType::HISTOGRAM) {
TPoint& p = Points_.back();
p.GetValue().AsHistogram()->Ref();
- } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
+ } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
TPoint& p = Points_.back();
p.GetValue().AsLogHistogram()->Ref();
}
@@ -424,16 +424,16 @@ namespace NMonitoring {
point.GetValue().AsHistogram()->Ref();
}
} else if (ValueType_ == EMetricValueType::SUMMARY) {
- for (size_t i = prevSize; i < Points_.size(); ++i) {
- TPoint& point = Points_[i];
- point.GetValue().AsSummaryDouble()->Ref();
- }
- } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
- for (size_t i = prevSize; i < Points_.size(); ++i) {
- TPoint& point = Points_[i];
- point.GetValue().AsLogHistogram()->Ref();
- }
- }
+ for (size_t i = prevSize; i < Points_.size(); ++i) {
+ TPoint& point = Points_[i];
+ point.GetValue().AsSummaryDouble()->Ref();
+ }
+ } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
+ for (size_t i = prevSize; i < Points_.size(); ++i) {
+ TPoint& point = Points_[i];
+ point.GetValue().AsLogHistogram()->Ref();
+ }
+ }
}
template <typename TConsumer>
@@ -480,50 +480,50 @@ namespace NMonitoring {
};
template <EMetricValueType valueType, typename TPoint>
- static inline void SnapshotUnRef(TPoint& point) {
+ static inline void SnapshotUnRef(TPoint& point) {
if constexpr (valueType == EMetricValueType::HISTOGRAM) {
- if (auto* hist = point.GetValue().AsHistogram()) {
- hist->UnRef();
- }
+ if (auto* hist = point.GetValue().AsHistogram()) {
+ hist->UnRef();
+ }
} else if constexpr (valueType == EMetricValueType::SUMMARY) {
- if (auto* summary = point.GetValue().AsSummaryDouble()) {
- summary->UnRef();
- }
- } else if constexpr (valueType == EMetricValueType::LOGHISTOGRAM) {
- if (auto* logHist = point.GetValue().AsLogHistogram()) {
- logHist->UnRef();
- }
- }
- }
-
+ if (auto* summary = point.GetValue().AsSummaryDouble()) {
+ summary->UnRef();
+ }
+ } else if constexpr (valueType == EMetricValueType::LOGHISTOGRAM) {
+ if (auto* logHist = point.GetValue().AsLogHistogram()) {
+ logHist->UnRef();
+ }
+ }
+ }
+
template <EMetricValueType valueType, typename TPoint>
- static void EraseDuplicates(TVector<TPoint>& points) {
- // we have to manually clean reference to a snapshot from point
- // while removing duplicates
- auto result = points.rbegin();
- for (auto it = result + 1; it != points.rend(); ++it) {
- if (result->GetTime() != it->GetTime() && ++result != it) {
+ static void EraseDuplicates(TVector<TPoint>& points) {
+ // we have to manually clean reference to a snapshot from point
+ // while removing duplicates
+ auto result = points.rbegin();
+ for (auto it = result + 1; it != points.rend(); ++it) {
+ if (result->GetTime() != it->GetTime() && ++result != it) {
SnapshotUnRef<valueType>(*result);
- *result = *it; // (2) copy
- it->ClearValue(); // (3) clean pointer in the source
- }
- }
-
- // erase tail points
- for (auto it = result + 1; it != points.rend(); ++it) {
+ *result = *it; // (2) copy
+ it->ClearValue(); // (3) clean pointer in the source
+ }
+ }
+
+ // erase tail points
+ for (auto it = result + 1; it != points.rend(); ++it) {
SnapshotUnRef<valueType>(*it);
- }
- points.erase(points.begin(), (result + 1).base());
- }
-
- template <typename TPoint>
+ }
+ points.erase(points.begin(), (result + 1).base());
+ }
+
+ template <typename TPoint>
void SortPointsByTs(EMetricValueType valueType, TVector<TPoint>& points) {
if (points.size() < 2) {
return;
}
- if (valueType != EMetricValueType::HISTOGRAM && valueType != EMetricValueType::SUMMARY
- && valueType != EMetricValueType::LOGHISTOGRAM) {
+ if (valueType != EMetricValueType::HISTOGRAM && valueType != EMetricValueType::SUMMARY
+ && valueType != EMetricValueType::LOGHISTOGRAM) {
// Stable sort + saving only the last point inside a group of duplicates
StableSortBy(points, NPrivate::POINT_KEY_FN);
auto it = UniqueBy(points.rbegin(), points.rend(), NPrivate::POINT_KEY_FN);
@@ -532,11 +532,11 @@ namespace NMonitoring {
StableSortBy(points, NPrivate::POINT_KEY_FN);
if (valueType == EMetricValueType::HISTOGRAM) {
EraseDuplicates<EMetricValueType::HISTOGRAM>(points);
- } else if (valueType == EMetricValueType::LOGHISTOGRAM) {
- EraseDuplicates<EMetricValueType::LOGHISTOGRAM>(points);
- } else {
+ } else if (valueType == EMetricValueType::LOGHISTOGRAM) {
+ EraseDuplicates<EMetricValueType::LOGHISTOGRAM>(points);
+ } else {
EraseDuplicates<EMetricValueType::SUMMARY>(points);
- }
+ }
}
}
}
diff --git a/library/cpp/monlib/metrics/metric_value_type.h b/library/cpp/monlib/metrics/metric_value_type.h
index 500e9770b5..ab30a958c2 100644
--- a/library/cpp/monlib/metrics/metric_value_type.h
+++ b/library/cpp/monlib/metrics/metric_value_type.h
@@ -10,7 +10,7 @@ enum class EMetricValueType {
UINT64,
HISTOGRAM,
SUMMARY,
- LOGHISTOGRAM,
+ LOGHISTOGRAM,
};
} // namespace NMonitoring
diff --git a/library/cpp/monlib/metrics/metric_value_ut.cpp b/library/cpp/monlib/metrics/metric_value_ut.cpp
index 261807c904..49b47c4057 100644
--- a/library/cpp/monlib/metrics/metric_value_ut.cpp
+++ b/library/cpp/monlib/metrics/metric_value_ut.cpp
@@ -32,18 +32,18 @@ Y_UNIT_TEST_SUITE(TMetricValueTest) {
return MakeIntrusive<TTestHistogram>();
}
- ISummaryDoubleSnapshotPtr MakeSummarySnapshot(ui64 count = 0u) {
- return MakeIntrusive<TSummaryDoubleSnapshot>(0.0, 0.0, 0.0, 0.0, count);
- }
-
- TLogHistogramSnapshotPtr MakeLogHistogram(ui64 count = 0) {
- TVector<double> buckets;
- for (ui64 i = 0; i < count; ++i) {
- buckets.push_back(i);
- }
- return MakeIntrusive<TLogHistogramSnapshot>(1.5, 0u, 0, buckets);
- }
-
+ ISummaryDoubleSnapshotPtr MakeSummarySnapshot(ui64 count = 0u) {
+ return MakeIntrusive<TSummaryDoubleSnapshot>(0.0, 0.0, 0.0, 0.0, count);
+ }
+
+ TLogHistogramSnapshotPtr MakeLogHistogram(ui64 count = 0) {
+ TVector<double> buckets;
+ for (ui64 i = 0; i < count; ++i) {
+ buckets.push_back(i);
+ }
+ return MakeIntrusive<TLogHistogramSnapshot>(1.5, 0u, 0, buckets);
+ }
+
Y_UNIT_TEST(Sorted) {
auto ts1 = TInstant::Now();
auto ts2 = ts1 + TDuration::Seconds(1);
@@ -78,30 +78,30 @@ Y_UNIT_TEST_SUITE(TMetricValueTest) {
UNIT_ASSERT_VALUES_EQUAL(1, histogram->RefCount());
}
- Y_UNIT_TEST(Summary) {
- auto ts = TInstant::Now();
- auto summary = MakeSummarySnapshot();
- UNIT_ASSERT_VALUES_EQUAL(1, summary->RefCount());
- {
+ Y_UNIT_TEST(Summary) {
+ auto ts = TInstant::Now();
+ auto summary = MakeSummarySnapshot();
+ UNIT_ASSERT_VALUES_EQUAL(1, summary->RefCount());
+ {
TMetricTimeSeries timeSeries;
- timeSeries.Add(ts, summary.Get());
- UNIT_ASSERT_VALUES_EQUAL(2, summary->RefCount());
- }
- UNIT_ASSERT_VALUES_EQUAL(1, summary->RefCount());
- }
-
- Y_UNIT_TEST(LogHistogram) {
- auto ts = TInstant::Now();
- auto logHist = MakeLogHistogram();
- UNIT_ASSERT_VALUES_EQUAL(1, logHist->RefCount());
- {
- TMetricTimeSeries timeSeries;
- timeSeries.Add(ts, logHist.Get());
- UNIT_ASSERT_VALUES_EQUAL(2, logHist->RefCount());
- }
- UNIT_ASSERT_VALUES_EQUAL(1, logHist->RefCount());
- }
-
+ timeSeries.Add(ts, summary.Get());
+ UNIT_ASSERT_VALUES_EQUAL(2, summary->RefCount());
+ }
+ UNIT_ASSERT_VALUES_EQUAL(1, summary->RefCount());
+ }
+
+ Y_UNIT_TEST(LogHistogram) {
+ auto ts = TInstant::Now();
+ auto logHist = MakeLogHistogram();
+ UNIT_ASSERT_VALUES_EQUAL(1, logHist->RefCount());
+ {
+ TMetricTimeSeries timeSeries;
+ timeSeries.Add(ts, logHist.Get());
+ UNIT_ASSERT_VALUES_EQUAL(2, logHist->RefCount());
+ }
+ UNIT_ASSERT_VALUES_EQUAL(1, logHist->RefCount());
+ }
+
Y_UNIT_TEST(TimeSeriesMovable) {
auto ts = TInstant::Now();
auto histogram = MakeIntrusive<TTestHistogram>();
@@ -170,98 +170,98 @@ Y_UNIT_TEST_SUITE(TMetricValueTest) {
UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount());
}
- Y_UNIT_TEST(LogHistogramsUnique) {
- auto ts1 = TInstant::Now();
- auto ts2 = ts1 + TDuration::Seconds(1);
- auto ts3 = ts2 + TDuration::Seconds(1);
-
- auto h1 = MakeLogHistogram();
- auto h2 = MakeLogHistogram();
- auto h3 = MakeLogHistogram();
-
- UNIT_ASSERT_VALUES_EQUAL(1, h1->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount());
-
- {
- TMetricTimeSeries timeSeries;
- timeSeries.Add(ts1, h1.Get()); // drop at the head
- timeSeries.Add(ts1, h1.Get());
- timeSeries.Add(ts1, h1.Get());
-
- timeSeries.Add(ts2, h2.Get()); // drop in the middle
- timeSeries.Add(ts2, h2.Get());
- timeSeries.Add(ts2, h2.Get());
-
- timeSeries.Add(ts3, h3.Get()); // drop at the end
- timeSeries.Add(ts3, h3.Get());
- timeSeries.Add(ts3, h3.Get());
-
- UNIT_ASSERT_EQUAL(timeSeries.Size(), 9);
-
- UNIT_ASSERT_VALUES_EQUAL(4, h1->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(4, h2->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(4, h3->RefCount());
-
- timeSeries.SortByTs();
- UNIT_ASSERT_EQUAL(timeSeries.Size(), 3);
-
- UNIT_ASSERT_VALUES_EQUAL(2, h1->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(2, h2->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(2, h3->RefCount());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(1, h1->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount());
- }
-
- Y_UNIT_TEST(SummaryUnique) {
- auto ts1 = TInstant::Now();
- auto ts2 = ts1 + TDuration::Seconds(1);
- auto ts3 = ts2 + TDuration::Seconds(1);
-
- auto h1 = MakeSummarySnapshot();
- auto h2 = MakeSummarySnapshot();
- auto h3 = MakeSummarySnapshot();
-
- UNIT_ASSERT_VALUES_EQUAL(1, h1->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount());
-
- {
+ Y_UNIT_TEST(LogHistogramsUnique) {
+ auto ts1 = TInstant::Now();
+ auto ts2 = ts1 + TDuration::Seconds(1);
+ auto ts3 = ts2 + TDuration::Seconds(1);
+
+ auto h1 = MakeLogHistogram();
+ auto h2 = MakeLogHistogram();
+ auto h3 = MakeLogHistogram();
+
+ UNIT_ASSERT_VALUES_EQUAL(1, h1->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount());
+
+ {
TMetricTimeSeries timeSeries;
- timeSeries.Add(ts1, h1.Get()); // drop at the head
- timeSeries.Add(ts1, h1.Get());
- timeSeries.Add(ts1, h1.Get());
-
- timeSeries.Add(ts2, h2.Get()); // drop in the middle
- timeSeries.Add(ts2, h2.Get());
- timeSeries.Add(ts2, h2.Get());
-
- timeSeries.Add(ts3, h3.Get()); // drop at the end
- timeSeries.Add(ts3, h3.Get());
- timeSeries.Add(ts3, h3.Get());
-
- UNIT_ASSERT_EQUAL(timeSeries.Size(), 9);
-
- UNIT_ASSERT_VALUES_EQUAL(4, h1->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(4, h2->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(4, h3->RefCount());
-
- timeSeries.SortByTs();
- UNIT_ASSERT_EQUAL(timeSeries.Size(), 3);
-
- UNIT_ASSERT_VALUES_EQUAL(2, h1->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(2, h2->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(2, h3->RefCount());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(1, h1->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount());
- UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount());
- }
-
+ timeSeries.Add(ts1, h1.Get()); // drop at the head
+ timeSeries.Add(ts1, h1.Get());
+ timeSeries.Add(ts1, h1.Get());
+
+ timeSeries.Add(ts2, h2.Get()); // drop in the middle
+ timeSeries.Add(ts2, h2.Get());
+ timeSeries.Add(ts2, h2.Get());
+
+ timeSeries.Add(ts3, h3.Get()); // drop at the end
+ timeSeries.Add(ts3, h3.Get());
+ timeSeries.Add(ts3, h3.Get());
+
+ UNIT_ASSERT_EQUAL(timeSeries.Size(), 9);
+
+ UNIT_ASSERT_VALUES_EQUAL(4, h1->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(4, h2->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(4, h3->RefCount());
+
+ timeSeries.SortByTs();
+ UNIT_ASSERT_EQUAL(timeSeries.Size(), 3);
+
+ UNIT_ASSERT_VALUES_EQUAL(2, h1->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(2, h2->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(2, h3->RefCount());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, h1->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount());
+ }
+
+ Y_UNIT_TEST(SummaryUnique) {
+ auto ts1 = TInstant::Now();
+ auto ts2 = ts1 + TDuration::Seconds(1);
+ auto ts3 = ts2 + TDuration::Seconds(1);
+
+ auto h1 = MakeSummarySnapshot();
+ auto h2 = MakeSummarySnapshot();
+ auto h3 = MakeSummarySnapshot();
+
+ UNIT_ASSERT_VALUES_EQUAL(1, h1->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount());
+
+ {
+ TMetricTimeSeries timeSeries;
+ timeSeries.Add(ts1, h1.Get()); // drop at the head
+ timeSeries.Add(ts1, h1.Get());
+ timeSeries.Add(ts1, h1.Get());
+
+ timeSeries.Add(ts2, h2.Get()); // drop in the middle
+ timeSeries.Add(ts2, h2.Get());
+ timeSeries.Add(ts2, h2.Get());
+
+ timeSeries.Add(ts3, h3.Get()); // drop at the end
+ timeSeries.Add(ts3, h3.Get());
+ timeSeries.Add(ts3, h3.Get());
+
+ UNIT_ASSERT_EQUAL(timeSeries.Size(), 9);
+
+ UNIT_ASSERT_VALUES_EQUAL(4, h1->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(4, h2->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(4, h3->RefCount());
+
+ timeSeries.SortByTs();
+ UNIT_ASSERT_EQUAL(timeSeries.Size(), 3);
+
+ UNIT_ASSERT_VALUES_EQUAL(2, h1->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(2, h2->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(2, h3->RefCount());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, h1->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount());
+ UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount());
+ }
+
Y_UNIT_TEST(HistogramsUnique2) {
auto ts1 = TInstant::Now();
auto ts2 = ts1 + TDuration::Seconds(1);
@@ -300,84 +300,84 @@ Y_UNIT_TEST_SUITE(TMetricValueTest) {
UNIT_ASSERT_EQUAL(timeSeries[4].GetValue().AsHistogram()->Count(), 7);
}
}
-
- Y_UNIT_TEST(LogHistogramsUnique2) {
- auto ts1 = TInstant::Now();
- auto ts2 = ts1 + TDuration::Seconds(1);
- auto ts3 = ts2 + TDuration::Seconds(1);
- auto ts4 = ts3 + TDuration::Seconds(1);
- auto ts5 = ts4 + TDuration::Seconds(1);
-
- auto h1 = MakeLogHistogram(1u);
- auto h2 = MakeLogHistogram(2u);
- auto h3 = MakeLogHistogram(3u);
- auto h4 = MakeLogHistogram(4u);
- auto h5 = MakeLogHistogram(5u);
- auto h6 = MakeLogHistogram(6u);
- auto h7 = MakeLogHistogram(7u);
-
- {
- TMetricTimeSeries timeSeries;
- timeSeries.Add(ts1, h1.Get());
- timeSeries.Add(ts1, h2.Get());
-
- timeSeries.Add(ts2, h3.Get());
-
- timeSeries.Add(ts3, h4.Get());
- timeSeries.Add(ts3, h5.Get());
-
- timeSeries.Add(ts4, h6.Get());
- timeSeries.Add(ts5, h7.Get());
-
- timeSeries.SortByTs();
-
- UNIT_ASSERT_EQUAL(timeSeries.Size(), 5);
- UNIT_ASSERT_EQUAL(timeSeries[0].GetValue().AsLogHistogram()->Count(), 2);
- UNIT_ASSERT_EQUAL(timeSeries[1].GetValue().AsLogHistogram()->Count(), 3);
- UNIT_ASSERT_EQUAL(timeSeries[2].GetValue().AsLogHistogram()->Count(), 5);
- UNIT_ASSERT_EQUAL(timeSeries[3].GetValue().AsLogHistogram()->Count(), 6);
- UNIT_ASSERT_EQUAL(timeSeries[4].GetValue().AsLogHistogram()->Count(), 7);
- }
- }
-
- Y_UNIT_TEST(SummaryUnique2) {
- auto ts1 = TInstant::Now();
- auto ts2 = ts1 + TDuration::Seconds(1);
- auto ts3 = ts2 + TDuration::Seconds(1);
- auto ts4 = ts3 + TDuration::Seconds(1);
- auto ts5 = ts4 + TDuration::Seconds(1);
-
- auto h1 = MakeSummarySnapshot(1u);
- auto h2 = MakeSummarySnapshot(2u);
- auto h3 = MakeSummarySnapshot(3u);
- auto h4 = MakeSummarySnapshot(4u);
- auto h5 = MakeSummarySnapshot(5u);
- auto h6 = MakeSummarySnapshot(6u);
- auto h7 = MakeSummarySnapshot(7u);
-
- {
+
+ Y_UNIT_TEST(LogHistogramsUnique2) {
+ auto ts1 = TInstant::Now();
+ auto ts2 = ts1 + TDuration::Seconds(1);
+ auto ts3 = ts2 + TDuration::Seconds(1);
+ auto ts4 = ts3 + TDuration::Seconds(1);
+ auto ts5 = ts4 + TDuration::Seconds(1);
+
+ auto h1 = MakeLogHistogram(1u);
+ auto h2 = MakeLogHistogram(2u);
+ auto h3 = MakeLogHistogram(3u);
+ auto h4 = MakeLogHistogram(4u);
+ auto h5 = MakeLogHistogram(5u);
+ auto h6 = MakeLogHistogram(6u);
+ auto h7 = MakeLogHistogram(7u);
+
+ {
TMetricTimeSeries timeSeries;
- timeSeries.Add(ts1, h1.Get());
- timeSeries.Add(ts1, h2.Get());
-
- timeSeries.Add(ts2, h3.Get());
-
- timeSeries.Add(ts3, h4.Get());
- timeSeries.Add(ts3, h5.Get());
-
- timeSeries.Add(ts4, h6.Get());
- timeSeries.Add(ts5, h7.Get());
-
- timeSeries.SortByTs();
-
- UNIT_ASSERT_EQUAL(timeSeries.Size(), 5);
- UNIT_ASSERT_EQUAL(timeSeries[0].GetValue().AsSummaryDouble()->GetCount(), 2);
- UNIT_ASSERT_EQUAL(timeSeries[1].GetValue().AsSummaryDouble()->GetCount(), 3);
- UNIT_ASSERT_EQUAL(timeSeries[2].GetValue().AsSummaryDouble()->GetCount(), 5);
- UNIT_ASSERT_EQUAL(timeSeries[3].GetValue().AsSummaryDouble()->GetCount(), 6);
- UNIT_ASSERT_EQUAL(timeSeries[4].GetValue().AsSummaryDouble()->GetCount(), 7);
- }
- }
+ timeSeries.Add(ts1, h1.Get());
+ timeSeries.Add(ts1, h2.Get());
+
+ timeSeries.Add(ts2, h3.Get());
+
+ timeSeries.Add(ts3, h4.Get());
+ timeSeries.Add(ts3, h5.Get());
+
+ timeSeries.Add(ts4, h6.Get());
+ timeSeries.Add(ts5, h7.Get());
+
+ timeSeries.SortByTs();
+
+ UNIT_ASSERT_EQUAL(timeSeries.Size(), 5);
+ UNIT_ASSERT_EQUAL(timeSeries[0].GetValue().AsLogHistogram()->Count(), 2);
+ UNIT_ASSERT_EQUAL(timeSeries[1].GetValue().AsLogHistogram()->Count(), 3);
+ UNIT_ASSERT_EQUAL(timeSeries[2].GetValue().AsLogHistogram()->Count(), 5);
+ UNIT_ASSERT_EQUAL(timeSeries[3].GetValue().AsLogHistogram()->Count(), 6);
+ UNIT_ASSERT_EQUAL(timeSeries[4].GetValue().AsLogHistogram()->Count(), 7);
+ }
+ }
+
+ Y_UNIT_TEST(SummaryUnique2) {
+ auto ts1 = TInstant::Now();
+ auto ts2 = ts1 + TDuration::Seconds(1);
+ auto ts3 = ts2 + TDuration::Seconds(1);
+ auto ts4 = ts3 + TDuration::Seconds(1);
+ auto ts5 = ts4 + TDuration::Seconds(1);
+
+ auto h1 = MakeSummarySnapshot(1u);
+ auto h2 = MakeSummarySnapshot(2u);
+ auto h3 = MakeSummarySnapshot(3u);
+ auto h4 = MakeSummarySnapshot(4u);
+ auto h5 = MakeSummarySnapshot(5u);
+ auto h6 = MakeSummarySnapshot(6u);
+ auto h7 = MakeSummarySnapshot(7u);
+
+ {
+ TMetricTimeSeries timeSeries;
+ timeSeries.Add(ts1, h1.Get());
+ timeSeries.Add(ts1, h2.Get());
+
+ timeSeries.Add(ts2, h3.Get());
+
+ timeSeries.Add(ts3, h4.Get());
+ timeSeries.Add(ts3, h5.Get());
+
+ timeSeries.Add(ts4, h6.Get());
+ timeSeries.Add(ts5, h7.Get());
+
+ timeSeries.SortByTs();
+
+ UNIT_ASSERT_EQUAL(timeSeries.Size(), 5);
+ UNIT_ASSERT_EQUAL(timeSeries[0].GetValue().AsSummaryDouble()->GetCount(), 2);
+ UNIT_ASSERT_EQUAL(timeSeries[1].GetValue().AsSummaryDouble()->GetCount(), 3);
+ UNIT_ASSERT_EQUAL(timeSeries[2].GetValue().AsSummaryDouble()->GetCount(), 5);
+ UNIT_ASSERT_EQUAL(timeSeries[3].GetValue().AsSummaryDouble()->GetCount(), 6);
+ UNIT_ASSERT_EQUAL(timeSeries[4].GetValue().AsSummaryDouble()->GetCount(), 7);
+ }
+ }
Y_UNIT_TEST(TMetricValueWithType) {
// correct usage
diff --git a/library/cpp/monlib/metrics/summary_collector.cpp b/library/cpp/monlib/metrics/summary_collector.cpp
index 7496ee17b0..cae8560891 100644
--- a/library/cpp/monlib/metrics/summary_collector.cpp
+++ b/library/cpp/monlib/metrics/summary_collector.cpp
@@ -1 +1 @@
-#include "summary_collector.h"
+#include "summary_collector.h"
diff --git a/library/cpp/monlib/metrics/summary_collector.h b/library/cpp/monlib/metrics/summary_collector.h
index f09f2cfcf7..acba0fddf9 100644
--- a/library/cpp/monlib/metrics/summary_collector.h
+++ b/library/cpp/monlib/metrics/summary_collector.h
@@ -1,104 +1,104 @@
-#pragma once
-
-#include "summary_snapshot.h"
-
-#include <atomic>
-#include <limits>
-#include <cmath>
-
-namespace NMonitoring {
-
- class ISummaryDoubleCollector {
- public:
- virtual ~ISummaryDoubleCollector() = default;
-
- virtual void Collect(double value) = 0;
-
- virtual ISummaryDoubleSnapshotPtr Snapshot() const = 0;
+#pragma once
+
+#include "summary_snapshot.h"
+
+#include <atomic>
+#include <limits>
+#include <cmath>
+
+namespace NMonitoring {
+
+ class ISummaryDoubleCollector {
+ public:
+ virtual ~ISummaryDoubleCollector() = default;
+
+ virtual void Collect(double value) = 0;
+
+ virtual ISummaryDoubleSnapshotPtr Snapshot() const = 0;
virtual size_t SizeBytes() const = 0;
- };
-
- using ISummaryDoubleCollectorPtr = THolder<ISummaryDoubleCollector>;
-
- class TSummaryDoubleCollector final: public ISummaryDoubleCollector {
- public:
- TSummaryDoubleCollector() {
- Sum_.store(0, std::memory_order_relaxed);
- Min_.store(std::numeric_limits<double>::max(), std::memory_order_relaxed);
- Max_.store(std::numeric_limits<double>::lowest(), std::memory_order_relaxed);
- Count_.store(0, std::memory_order_relaxed);
- }
-
- void Collect(double value) noexcept override {
- if (std::isnan(value)) {
- return;
- }
- UpdateSum(value);
- UpdateMin(value);
- UpdateMax(value);
- Last_.store(value, std::memory_order_relaxed);
- Count_.fetch_add(1ul, std::memory_order_relaxed);
- }
-
+ };
+
+ using ISummaryDoubleCollectorPtr = THolder<ISummaryDoubleCollector>;
+
+ class TSummaryDoubleCollector final: public ISummaryDoubleCollector {
+ public:
+ TSummaryDoubleCollector() {
+ Sum_.store(0, std::memory_order_relaxed);
+ Min_.store(std::numeric_limits<double>::max(), std::memory_order_relaxed);
+ Max_.store(std::numeric_limits<double>::lowest(), std::memory_order_relaxed);
+ Count_.store(0, std::memory_order_relaxed);
+ }
+
+ void Collect(double value) noexcept override {
+ if (std::isnan(value)) {
+ return;
+ }
+ UpdateSum(value);
+ UpdateMin(value);
+ UpdateMax(value);
+ Last_.store(value, std::memory_order_relaxed);
+ Count_.fetch_add(1ul, std::memory_order_relaxed);
+ }
+
ISummaryDoubleSnapshotPtr Snapshot() const override {
- return new TSummaryDoubleSnapshot(
- Sum_.load(std::memory_order_relaxed),
- Min_.load(std::memory_order_relaxed),
- Max_.load(std::memory_order_relaxed),
- Last_.load(std::memory_order_relaxed),
- Count_.load(std::memory_order_relaxed));
- }
-
+ return new TSummaryDoubleSnapshot(
+ Sum_.load(std::memory_order_relaxed),
+ Min_.load(std::memory_order_relaxed),
+ Max_.load(std::memory_order_relaxed),
+ Last_.load(std::memory_order_relaxed),
+ Count_.load(std::memory_order_relaxed));
+ }
+
size_t SizeBytes() const override {
return sizeof(*this);
}
- private:
- std::atomic<double> Sum_;
- std::atomic<double> Min_;
- std::atomic<double> Max_;
- std::atomic<double> Last_;
- std::atomic_uint64_t Count_;
-
- void UpdateSum(double add) noexcept {
- double newValue;
- double oldValue = Sum_.load(std::memory_order_relaxed);
- do {
- newValue = oldValue + add;
- } while (!Sum_.compare_exchange_weak(
- oldValue,
- newValue,
- std::memory_order_release,
- std::memory_order_consume));
- }
-
- void UpdateMin(double candidate) noexcept {
- double oldValue = Min_.load(std::memory_order_relaxed);
- do {
- if (oldValue <= candidate) {
- break;
- }
- } while (!Min_.compare_exchange_weak(
- oldValue,
- candidate,
- std::memory_order_release,
- std::memory_order_consume));
- }
-
- void UpdateMax(double candidate) noexcept {
- double oldValue = Max_.load(std::memory_order_relaxed);
- do {
- if (oldValue >= candidate) {
- break;
- }
- } while (!Max_.compare_exchange_weak(
- oldValue,
- candidate,
- std::memory_order_release,
- std::memory_order_consume));
- }
-
- };
-
-}
+ private:
+ std::atomic<double> Sum_;
+ std::atomic<double> Min_;
+ std::atomic<double> Max_;
+ std::atomic<double> Last_;
+ std::atomic_uint64_t Count_;
+
+ void UpdateSum(double add) noexcept {
+ double newValue;
+ double oldValue = Sum_.load(std::memory_order_relaxed);
+ do {
+ newValue = oldValue + add;
+ } while (!Sum_.compare_exchange_weak(
+ oldValue,
+ newValue,
+ std::memory_order_release,
+ std::memory_order_consume));
+ }
+
+ void UpdateMin(double candidate) noexcept {
+ double oldValue = Min_.load(std::memory_order_relaxed);
+ do {
+ if (oldValue <= candidate) {
+ break;
+ }
+ } while (!Min_.compare_exchange_weak(
+ oldValue,
+ candidate,
+ std::memory_order_release,
+ std::memory_order_consume));
+ }
+
+ void UpdateMax(double candidate) noexcept {
+ double oldValue = Max_.load(std::memory_order_relaxed);
+ do {
+ if (oldValue >= candidate) {
+ break;
+ }
+ } while (!Max_.compare_exchange_weak(
+ oldValue,
+ candidate,
+ std::memory_order_release,
+ std::memory_order_consume));
+ }
+
+ };
+
+}
diff --git a/library/cpp/monlib/metrics/summary_collector_ut.cpp b/library/cpp/monlib/metrics/summary_collector_ut.cpp
index 600148f814..191929550f 100644
--- a/library/cpp/monlib/metrics/summary_collector_ut.cpp
+++ b/library/cpp/monlib/metrics/summary_collector_ut.cpp
@@ -1,64 +1,64 @@
-#include "summary_collector.h"
-
+#include "summary_collector.h"
+
#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/random/random.h>
-
-#include <numeric>
-#include <algorithm>
-
-namespace NMonitoring {
-
-Y_UNIT_TEST_SUITE(SummaryCollectorTest) {
-
- void CheckSnapshot(ISummaryDoubleSnapshotPtr snapshot, const TVector<double> values) {
- const double eps = 1e-9;
-
- double sum = std::accumulate(values.begin(), values.end(), 0.0);
- double min = *std::min_element(values.begin(), values.end());
- double max = *std::max_element(values.begin(), values.end());
- double last = values.back();
- ui64 count = values.size();
-
- UNIT_ASSERT_DOUBLES_EQUAL(snapshot->GetSum(), sum, eps);
- UNIT_ASSERT_DOUBLES_EQUAL(snapshot->GetMin(), min, eps);
- UNIT_ASSERT_DOUBLES_EQUAL(snapshot->GetMax(), max, eps);
- UNIT_ASSERT_DOUBLES_EQUAL(snapshot->GetLast(), last, eps);
- UNIT_ASSERT_EQUAL(snapshot->GetCount(), count);
- }
-
- Y_UNIT_TEST(Simple) {
- {
- TVector<double> test{05, -1.5, 0.0, 2.5, 0.25, -1.0};
- TSummaryDoubleCollector summary;
- for (auto value : test) {
- summary.Collect(value);
- }
- CheckSnapshot(summary.Snapshot(), test);
- }
- {
- TVector<double> test{-1.0, 1.0, 9.0, -5000.0, 5000.0, 5.0, -5.0};
- TSummaryDoubleCollector summary;
- for (auto value : test) {
- summary.Collect(value);
- }
- CheckSnapshot(summary.Snapshot(), test);
- }
- }
-
- Y_UNIT_TEST(RandomStressTest) {
- const ui32 attemts = 100;
- for (ui32 i = 0; i < attemts; ++i) {
- const ui32 size = 100;
- TVector<double> values(size);
- TSummaryDoubleCollector summary;
- for (auto& value : values) {
- value = RandomNumber<double>() - 0.5;
- summary.Collect(value);
- }
- CheckSnapshot(summary.Snapshot(), values);
- }
- }
-}
-
-}
+
+#include <util/random/random.h>
+
+#include <numeric>
+#include <algorithm>
+
+namespace NMonitoring {
+
+Y_UNIT_TEST_SUITE(SummaryCollectorTest) {
+
+ void CheckSnapshot(ISummaryDoubleSnapshotPtr snapshot, const TVector<double> values) {
+ const double eps = 1e-9;
+
+ double sum = std::accumulate(values.begin(), values.end(), 0.0);
+ double min = *std::min_element(values.begin(), values.end());
+ double max = *std::max_element(values.begin(), values.end());
+ double last = values.back();
+ ui64 count = values.size();
+
+ UNIT_ASSERT_DOUBLES_EQUAL(snapshot->GetSum(), sum, eps);
+ UNIT_ASSERT_DOUBLES_EQUAL(snapshot->GetMin(), min, eps);
+ UNIT_ASSERT_DOUBLES_EQUAL(snapshot->GetMax(), max, eps);
+ UNIT_ASSERT_DOUBLES_EQUAL(snapshot->GetLast(), last, eps);
+ UNIT_ASSERT_EQUAL(snapshot->GetCount(), count);
+ }
+
+ Y_UNIT_TEST(Simple) {
+ {
+ TVector<double> test{05, -1.5, 0.0, 2.5, 0.25, -1.0};
+ TSummaryDoubleCollector summary;
+ for (auto value : test) {
+ summary.Collect(value);
+ }
+ CheckSnapshot(summary.Snapshot(), test);
+ }
+ {
+ TVector<double> test{-1.0, 1.0, 9.0, -5000.0, 5000.0, 5.0, -5.0};
+ TSummaryDoubleCollector summary;
+ for (auto value : test) {
+ summary.Collect(value);
+ }
+ CheckSnapshot(summary.Snapshot(), test);
+ }
+ }
+
+ Y_UNIT_TEST(RandomStressTest) {
+ const ui32 attemts = 100;
+ for (ui32 i = 0; i < attemts; ++i) {
+ const ui32 size = 100;
+ TVector<double> values(size);
+ TSummaryDoubleCollector summary;
+ for (auto& value : values) {
+ value = RandomNumber<double>() - 0.5;
+ summary.Collect(value);
+ }
+ CheckSnapshot(summary.Snapshot(), values);
+ }
+ }
+}
+
+}
diff --git a/library/cpp/monlib/metrics/summary_snapshot.cpp b/library/cpp/monlib/metrics/summary_snapshot.cpp
index 39f34902f3..0b13263337 100644
--- a/library/cpp/monlib/metrics/summary_snapshot.cpp
+++ b/library/cpp/monlib/metrics/summary_snapshot.cpp
@@ -1,9 +1,9 @@
-#include "summary_snapshot.h"
-
-#include <util/stream/output.h>
-
+#include "summary_snapshot.h"
+
+#include <util/stream/output.h>
+
#include <iostream>
-
+
namespace {
@@ -16,11 +16,11 @@ auto& Output(TStream& o, const NMonitoring::ISummaryDoubleSnapshot& s) {
o << TStringBuf("max: ") << s.GetMax() << TStringBuf(", ");
o << TStringBuf("last: ") << s.GetLast() << TStringBuf(", ");
o << TStringBuf("count: ") << s.GetCount();
-
+
o << TStringBuf("}");
return o;
-}
+}
} // namespace
diff --git a/library/cpp/monlib/metrics/summary_snapshot.h b/library/cpp/monlib/metrics/summary_snapshot.h
index d90b5f1b01..afcc895fd3 100644
--- a/library/cpp/monlib/metrics/summary_snapshot.h
+++ b/library/cpp/monlib/metrics/summary_snapshot.h
@@ -1,72 +1,72 @@
-#pragma once
-
-#include <util/generic/ptr.h>
-
-namespace NMonitoring {
-
- class ISummaryDoubleSnapshot: public TAtomicRefCount<ISummaryDoubleSnapshot> {
- public:
- virtual ~ISummaryDoubleSnapshot() = default;
-
- // TODO: write documentation
-
- virtual ui64 GetCount() const = 0;
-
- virtual double GetSum() const = 0;
-
- virtual double GetMin() const = 0;
-
- virtual double GetMax() const = 0;
-
- virtual double GetLast() const = 0;
-
- virtual ui64 MemorySizeBytes() const = 0;
- };
-
- using ISummaryDoubleSnapshotPtr = TIntrusivePtr<ISummaryDoubleSnapshot>;
-
- class TSummaryDoubleSnapshot final: public ISummaryDoubleSnapshot {
- public:
- TSummaryDoubleSnapshot(double sum, double min, double max, double last, ui64 count)
- : Sum_(sum)
- , Min_(min)
- , Max_(max)
- , Last_(last)
- , Count_(count)
- {}
-
- ui64 GetCount() const noexcept override {
- return Count_;
- }
-
- double GetSum() const noexcept override {
- return Sum_;
- }
-
- double GetMin() const noexcept override {
- return Min_;
- }
-
- double GetMax() const noexcept override {
- return Max_;
- }
-
- virtual double GetLast() const noexcept override {
- return Last_;
- }
-
- ui64 MemorySizeBytes() const noexcept override {
- return sizeof(*this);
- }
-
- private:
- double Sum_;
- double Min_;
- double Max_;
- double Last_;
- ui64 Count_;
- };
-
-}
+#pragma once
+
+#include <util/generic/ptr.h>
+
+namespace NMonitoring {
+
+ class ISummaryDoubleSnapshot: public TAtomicRefCount<ISummaryDoubleSnapshot> {
+ public:
+ virtual ~ISummaryDoubleSnapshot() = default;
+
+ // TODO: write documentation
+
+ virtual ui64 GetCount() const = 0;
+
+ virtual double GetSum() const = 0;
+
+ virtual double GetMin() const = 0;
+
+ virtual double GetMax() const = 0;
+
+ virtual double GetLast() const = 0;
+
+ virtual ui64 MemorySizeBytes() const = 0;
+ };
+
+ using ISummaryDoubleSnapshotPtr = TIntrusivePtr<ISummaryDoubleSnapshot>;
+
+ class TSummaryDoubleSnapshot final: public ISummaryDoubleSnapshot {
+ public:
+ TSummaryDoubleSnapshot(double sum, double min, double max, double last, ui64 count)
+ : Sum_(sum)
+ , Min_(min)
+ , Max_(max)
+ , Last_(last)
+ , Count_(count)
+ {}
+
+ ui64 GetCount() const noexcept override {
+ return Count_;
+ }
+
+ double GetSum() const noexcept override {
+ return Sum_;
+ }
+
+ double GetMin() const noexcept override {
+ return Min_;
+ }
+
+ double GetMax() const noexcept override {
+ return Max_;
+ }
+
+ virtual double GetLast() const noexcept override {
+ return Last_;
+ }
+
+ ui64 MemorySizeBytes() const noexcept override {
+ return sizeof(*this);
+ }
+
+ private:
+ double Sum_;
+ double Min_;
+ double Max_;
+ double Last_;
+ ui64 Count_;
+ };
+
+}
std::ostream& operator<<(std::ostream& os, const NMonitoring::ISummaryDoubleSnapshot& s);
diff --git a/library/cpp/monlib/metrics/ut/ya.make b/library/cpp/monlib/metrics/ut/ya.make
index cb55da7822..aec9974fbd 100644
--- a/library/cpp/monlib/metrics/ut/ya.make
+++ b/library/cpp/monlib/metrics/ut/ya.make
@@ -10,7 +10,7 @@ SRCS(
fake_ut.cpp
histogram_collector_ut.cpp
labels_ut.cpp
- log_histogram_collector_ut.cpp
+ log_histogram_collector_ut.cpp
metric_registry_ut.cpp
metric_sub_registry_ut.cpp
metric_value_ut.cpp
diff --git a/ydb/core/erasure/erasure.cpp b/ydb/core/erasure/erasure.cpp
index 8e299fcc5b..a41b027932 100644
--- a/ydb/core/erasure/erasure.cpp
+++ b/ydb/core/erasure/erasure.cpp
@@ -1547,10 +1547,10 @@ void EoBlockRestore(TErasureType::ECrcMode crcMode, const TErasureType &type, TD
// The (f1) case, but no full data needed, only parts
TRACE("case# f1" << Endl);
VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
- if (isStripe) {
- Refurbish(buffer, dataSize);
- p.PrepareInputDataPointers<isStripe>(buffer.Detach());
- }
+ if (isStripe) {
+ Refurbish(buffer, dataSize);
+ p.PrepareInputDataPointers<isStripe>(buffer.Detach());
+ }
p.XorRestorePart<isStripe, true, false, false>(partSet, missingDataPartIdxA);
TRACE("case# f1 split" << Endl);
p.EoSplit<isStripe, true>(partSet);
@@ -1592,9 +1592,9 @@ void EoBlockRestore(TErasureType::ECrcMode crcMode, const TErasureType &type, TD
return;
}
TRACE(__LINE__ << Endl);
- if (isStripe) {
- Refurbish(buffer, dataSize);
- }
+ if (isStripe) {
+ Refurbish(buffer, dataSize);
+ }
}
if (isStripe) {
TRACE(__LINE__ << Endl);
@@ -2575,7 +2575,7 @@ void MirrorRestore(TErasureType::ECrcMode crcMode, const TErasureType &type, TDa
}
-static void VerifyPartSizes(TDataPartSet& partSet, size_t definedPartEndIdx) {
+static void VerifyPartSizes(TDataPartSet& partSet, size_t definedPartEndIdx) {
size_t partSize = partSet.Parts[0].size();
for (size_t idx = 0; idx < partSet.Parts.size(); ++idx) {
Y_VERIFY(partSet.Parts[idx].size() == partSize);
diff --git a/ydb/core/erasure/erasure_rope.cpp b/ydb/core/erasure/erasure_rope.cpp
index ca8399d86f..b1de833d93 100644
--- a/ydb/core/erasure/erasure_rope.cpp
+++ b/ydb/core/erasure/erasure_rope.cpp
@@ -1,2808 +1,2808 @@
-#include "erasure_rope.h"
-
-#include <util/generic/yexception.h>
-#include <util/system/unaligned_mem.h>
+#include "erasure_rope.h"
+
+#include <util/generic/yexception.h>
+#include <util/system/unaligned_mem.h>
#include <library/cpp/containers/stack_vector/stack_vec.h>
-
-#define MAX_TOTAL_PARTS 8
-#define MAX_LINES_IN_BLOCK 8
-
-#define IS_VERBOSE 0
-#define IS_TRACE 0
-
-#if IS_VERBOSE
-# include <util/stream/str.h>
-# define VERBOSE_COUT(a) \
- Cerr << a
-
-static TString DebugFormatBits(ui64 value) {
- TStringStream s;
- for (size_t i = 7; i >=4; --i) {
- s << ((value >> i) & 1);
- }
- s << "_";
- for (size_t i = 3; i <= 3; --i) {
- s << ((value >> i) & 1);
- }
- return s.Str();
-}
-#else
-# define VERBOSE_COUT(a) \
- do { \
- } while (false)
-#endif
-
-#if IS_TRACE
-# define TRACE(a) \
- Cerr << a
-#else
-# define TRACE(a) \
- do { \
- } while (false)
-#endif
-
-namespace NKikimr {
+
+#define MAX_TOTAL_PARTS 8
+#define MAX_LINES_IN_BLOCK 8
+
+#define IS_VERBOSE 0
+#define IS_TRACE 0
+
+#if IS_VERBOSE
+# include <util/stream/str.h>
+# define VERBOSE_COUT(a) \
+ Cerr << a
+
+static TString DebugFormatBits(ui64 value) {
+ TStringStream s;
+ for (size_t i = 7; i >=4; --i) {
+ s << ((value >> i) & 1);
+ }
+ s << "_";
+ for (size_t i = 3; i <= 3; --i) {
+ s << ((value >> i) & 1);
+ }
+ return s.Str();
+}
+#else
+# define VERBOSE_COUT(a) \
+ do { \
+ } while (false)
+#endif
+
+#if IS_TRACE
+# define TRACE(a) \
+ Cerr << a
+#else
+# define TRACE(a) \
+ do { \
+ } while (false)
+#endif
+
+namespace NKikimr {
namespace NErasureRope {
-
-static void Refurbish(TRope &str, ui64 size) {
- if (str.GetSize() != size) {
+
+static void Refurbish(TRope &str, ui64 size) {
+ if (str.GetSize() != size) {
str = TRopeHelpers::RopeUninitialized(size);
- }
-}
-
-static void Refurbish(TPartFragment &fragment, ui64 size) {
- if (fragment.size() != size) {
- TRACE("Refurbish fragment size# " << fragment.size() << " to size# " << size << Endl);
- fragment.UninitializedOwnedWhole(size);
- }
-}
-
-const char *TRopeErasureType::ErasureSpeciesToStr(TRopeErasureType::EErasureSpecies es) {
- switch (es) {
- case ErasureNone: return "None";
- case ErasureMirror3: return "Mirror3";
- case Erasure3Plus1Block: return "3Plus1Block";
- case Erasure3Plus1Stripe: return "3Plus1Stripe";
- case Erasure4Plus2Block: return "4Plus2Block";
- case Erasure3Plus2Block: return "3Plus2Block";
- case Erasure4Plus2Stripe: return "4Plus2Stripe";
- case Erasure3Plus2Stripe: return "3Plus2Stripe";
- case ErasureMirror3Plus2: return "Mirror3Plus2";
- case ErasureMirror3dc: return "Mirror3dc";
- case Erasure4Plus3Block: return "4Plus3Block";
- case Erasure4Plus3Stripe: return "4Plus3Stripe";
- case Erasure3Plus3Block: return "3Plus3Block";
- case Erasure3Plus3Stripe: return "3Plus3Stripe";
- case Erasure2Plus3Block: return "2Plus3Block";
- case Erasure2Plus3Stripe: return "2Plus3Stripe";
- case Erasure2Plus2Block: return "2Plus2Block";
- case Erasure2Plus2Stripe: return "2Plus2Stripe";
+ }
+}
+
+static void Refurbish(TPartFragment &fragment, ui64 size) {
+ if (fragment.size() != size) {
+ TRACE("Refurbish fragment size# " << fragment.size() << " to size# " << size << Endl);
+ fragment.UninitializedOwnedWhole(size);
+ }
+}
+
+const char *TRopeErasureType::ErasureSpeciesToStr(TRopeErasureType::EErasureSpecies es) {
+ switch (es) {
+ case ErasureNone: return "None";
+ case ErasureMirror3: return "Mirror3";
+ case Erasure3Plus1Block: return "3Plus1Block";
+ case Erasure3Plus1Stripe: return "3Plus1Stripe";
+ case Erasure4Plus2Block: return "4Plus2Block";
+ case Erasure3Plus2Block: return "3Plus2Block";
+ case Erasure4Plus2Stripe: return "4Plus2Stripe";
+ case Erasure3Plus2Stripe: return "3Plus2Stripe";
+ case ErasureMirror3Plus2: return "Mirror3Plus2";
+ case ErasureMirror3dc: return "Mirror3dc";
+ case Erasure4Plus3Block: return "4Plus3Block";
+ case Erasure4Plus3Stripe: return "4Plus3Stripe";
+ case Erasure3Plus3Block: return "3Plus3Block";
+ case Erasure3Plus3Stripe: return "3Plus3Stripe";
+ case Erasure2Plus3Block: return "2Plus3Block";
+ case Erasure2Plus3Stripe: return "2Plus3Stripe";
+ case Erasure2Plus2Block: return "2Plus2Block";
+ case Erasure2Plus2Stripe: return "2Plus2Stripe";
case ErasureMirror3of4: return "Mirror3of4";
- default: return "UNKNOWN";
- }
-}
-
-struct TErasureParameters {
- TRopeErasureType::EErasureFamily ErasureFamily;
- ui32 DataParts; // for parity - number of data parts, for mirror - 1
- ui32 ParityParts; // for parity - number of parity parts (1 | 2 | 3), for mirror - number of additional copies
- ui32 Prime; // for parity - smallest prime number >= DataParts, for mirror - 1
-};
-
-static const std::array<TErasureParameters, TRopeErasureType::ErasureSpeciesCount> ErasureSpeciesParameters{{
- {TRopeErasureType::ErasureMirror, 1, 0, 1} // 0 = ErasureSpicies::ErasureNone
- ,{TRopeErasureType::ErasureMirror, 1, 2, 1} // 1 = ErasureSpicies::ErasureMirror3
- ,{TRopeErasureType::ErasureParityBlock, 3, 1, 3} // 2 = ErasureSpicies::Erasure3Plus1Block
- ,{TRopeErasureType::ErasureParityStripe, 3, 1, 3} // 3 = ErasureSpicies::Erasure3Plus1Stipe
- ,{TRopeErasureType::ErasureParityBlock, 4, 2, 5} // 4 = ErasureSpicies::Erasure4Plus2Block
- ,{TRopeErasureType::ErasureParityBlock, 3, 2, 3} // 5 = ErasureSpicies::Erasure3Plus2Block
- ,{TRopeErasureType::ErasureParityStripe, 4, 2, 5} // 6 = ErasureSpicies::Erasure4Plus2Stipe
- ,{TRopeErasureType::ErasureParityStripe, 3, 2, 3} // 7 = ErasureSpicies::Erasure3Plus2Stipe
- ,{TRopeErasureType::ErasureMirror, 1, 2, 1} // 8 = ErasureSpicies::ErasureMirror3Plus2
- ,{TRopeErasureType::ErasureMirror, 1, 2, 1} // 9 = ErasureSpicies::ErasureMirror3dc
- ,{TRopeErasureType::ErasureParityBlock, 4, 3, 5} // 10 = ErasureSpicies::Erasure4Plus3Block
- ,{TRopeErasureType::ErasureParityStripe, 4, 3, 5} // 11 = ErasureSpicies::Erasure4Plus3Stripe
- ,{TRopeErasureType::ErasureParityBlock, 3, 3, 3} // 12 = ErasureSpicies::Erasure3Plus3Block
- ,{TRopeErasureType::ErasureParityStripe, 3, 3, 3} // 13 = ErasureSpicies::Erasure3Plus3Stripe
- ,{TRopeErasureType::ErasureParityBlock, 2, 3, 3} // 14 = ErasureSpicies::Erasure2Plus3Block
- ,{TRopeErasureType::ErasureParityStripe, 2, 3, 3} // 15 = ErasureSpicies::Erasure2Plus3Stripe
- ,{TRopeErasureType::ErasureParityBlock, 2, 2, 3} // 16 = ErasureSpicies::Erasure2Plus2Block
- ,{TRopeErasureType::ErasureParityStripe, 2, 2, 3} // 17 = ErasureSpicies::Erasure2Plus2Stripe
+ default: return "UNKNOWN";
+ }
+}
+
+struct TErasureParameters {
+ TRopeErasureType::EErasureFamily ErasureFamily;
+ ui32 DataParts; // for parity - number of data parts, for mirror - 1
+ ui32 ParityParts; // for parity - number of parity parts (1 | 2 | 3), for mirror - number of additional copies
+ ui32 Prime; // for parity - smallest prime number >= DataParts, for mirror - 1
+};
+
+static const std::array<TErasureParameters, TRopeErasureType::ErasureSpeciesCount> ErasureSpeciesParameters{{
+ {TRopeErasureType::ErasureMirror, 1, 0, 1} // 0 = ErasureSpicies::ErasureNone
+ ,{TRopeErasureType::ErasureMirror, 1, 2, 1} // 1 = ErasureSpicies::ErasureMirror3
+ ,{TRopeErasureType::ErasureParityBlock, 3, 1, 3} // 2 = ErasureSpicies::Erasure3Plus1Block
+ ,{TRopeErasureType::ErasureParityStripe, 3, 1, 3} // 3 = ErasureSpicies::Erasure3Plus1Stipe
+ ,{TRopeErasureType::ErasureParityBlock, 4, 2, 5} // 4 = ErasureSpicies::Erasure4Plus2Block
+ ,{TRopeErasureType::ErasureParityBlock, 3, 2, 3} // 5 = ErasureSpicies::Erasure3Plus2Block
+ ,{TRopeErasureType::ErasureParityStripe, 4, 2, 5} // 6 = ErasureSpicies::Erasure4Plus2Stipe
+ ,{TRopeErasureType::ErasureParityStripe, 3, 2, 3} // 7 = ErasureSpicies::Erasure3Plus2Stipe
+ ,{TRopeErasureType::ErasureMirror, 1, 2, 1} // 8 = ErasureSpicies::ErasureMirror3Plus2
+ ,{TRopeErasureType::ErasureMirror, 1, 2, 1} // 9 = ErasureSpicies::ErasureMirror3dc
+ ,{TRopeErasureType::ErasureParityBlock, 4, 3, 5} // 10 = ErasureSpicies::Erasure4Plus3Block
+ ,{TRopeErasureType::ErasureParityStripe, 4, 3, 5} // 11 = ErasureSpicies::Erasure4Plus3Stripe
+ ,{TRopeErasureType::ErasureParityBlock, 3, 3, 3} // 12 = ErasureSpicies::Erasure3Plus3Block
+ ,{TRopeErasureType::ErasureParityStripe, 3, 3, 3} // 13 = ErasureSpicies::Erasure3Plus3Stripe
+ ,{TRopeErasureType::ErasureParityBlock, 2, 3, 3} // 14 = ErasureSpicies::Erasure2Plus3Block
+ ,{TRopeErasureType::ErasureParityStripe, 2, 3, 3} // 15 = ErasureSpicies::Erasure2Plus3Stripe
+ ,{TRopeErasureType::ErasureParityBlock, 2, 2, 3} // 16 = ErasureSpicies::Erasure2Plus2Block
+ ,{TRopeErasureType::ErasureParityStripe, 2, 2, 3} // 17 = ErasureSpicies::Erasure2Plus2Stripe
,{TRopeErasureType::ErasureMirror, 1, 2, 1} // 18 = ErasureSpicies::ErasureMirror3of4
-}};
-
+}};
+
void PadAndCrcAtTheEnd(TRopeHelpers::Iterator data, ui64 dataSize, ui64 bufferSize) {
- ui64 marginSize = bufferSize - dataSize - sizeof(ui32);
- if (marginSize) {
+ ui64 marginSize = bufferSize - dataSize - sizeof(ui32);
+ if (marginSize) {
TRopeUtils::Memset(data + dataSize, 0, marginSize);
- }
+ }
ui32 hash = TRopeHelpers::GetCrc32c(data, dataSize);
TRopeUtils::Memcpy(data + (bufferSize - sizeof(ui32)), (const char *)&hash, sizeof(ui32));
-}
-
-bool CheckCrcAtTheEnd(TRopeErasureType::ECrcMode crcMode, const TRope& buf) {
- switch (crcMode) {
- case TRopeErasureType::CrcModeNone:
- return true;
- case TRopeErasureType::CrcModeWholePart:
- if (buf.GetSize() == 0) {
- return true;
- } else {
- Y_VERIFY(buf.GetSize() >= sizeof(ui32), "Error in CheckWholeBlobCrc: blob part size# %" PRIu64
- " is less then crcSize# %" PRIu64, (ui64)buf.GetSize(), (ui64)sizeof(ui32));
+}
+
+bool CheckCrcAtTheEnd(TRopeErasureType::ECrcMode crcMode, const TRope& buf) {
+ switch (crcMode) {
+ case TRopeErasureType::CrcModeNone:
+ return true;
+ case TRopeErasureType::CrcModeWholePart:
+ if (buf.GetSize() == 0) {
+ return true;
+ } else {
+ Y_VERIFY(buf.GetSize() >= sizeof(ui32), "Error in CheckWholeBlobCrc: blob part size# %" PRIu64
+ " is less then crcSize# %" PRIu64, (ui64)buf.GetSize(), (ui64)sizeof(ui32));
ui32 crc = TRopeHelpers::GetCrc32c(buf.Begin(), buf.GetSize() - sizeof(ui32));
- TString expectedStringCrc = TRope(buf.Begin() + buf.GetSize() - sizeof(ui32), buf.End()).ConvertToString();
- ui32 expectedCrc = ReadUnaligned<ui32>(expectedStringCrc.data());
- return crc == expectedCrc;
- }
- }
- ythrow TWithBackTrace<yexception>() << "Unknown crcMode = " << (i32)crcMode;
-}
-
-class TBlockParams {
-public:
- ui64 DataSize;
- ui64 PartUserSize;
- ui64 PartContainerSize;
- ui32 DataParts;
-
- ui64 BlockSize; // Whole data is split into blocks of BlockSize bytes
- ui64 ColumnSize; // Each block consists of DataParts columns, each containing ColumnSize bytes
- ui64 WholeColumns;
- ui64 LineCount;
-
- ui64 SmallPartColumns;
- ui64 LargePartColumns;
-
- ui64 LastPartTailSize;
-
- ui64 SmallPartSize;
- ui64 LargePartSize;
-
- ui32 FirstSmallPartIdx;
-
- ui32 TotalParts;
- ui64 WholeBlocks; // Data consists of (WhloeBlocks * BlockSize + TailSize) bytes
- ui32 TailSize;
-
- ui32 Prime;
- TRopeErasureType::ECrcMode CrcMode;
-
+ TString expectedStringCrc = TRope(buf.Begin() + buf.GetSize() - sizeof(ui32), buf.End()).ConvertToString();
+ ui32 expectedCrc = ReadUnaligned<ui32>(expectedStringCrc.data());
+ return crc == expectedCrc;
+ }
+ }
+ ythrow TWithBackTrace<yexception>() << "Unknown crcMode = " << (i32)crcMode;
+}
+
+class TBlockParams {
+public:
+ ui64 DataSize;
+ ui64 PartUserSize;
+ ui64 PartContainerSize;
+ ui32 DataParts;
+
+ ui64 BlockSize; // Whole data is split into blocks of BlockSize bytes
+ ui64 ColumnSize; // Each block consists of DataParts columns, each containing ColumnSize bytes
+ ui64 WholeColumns;
+ ui64 LineCount;
+
+ ui64 SmallPartColumns;
+ ui64 LargePartColumns;
+
+ ui64 LastPartTailSize;
+
+ ui64 SmallPartSize;
+ ui64 LargePartSize;
+
+ ui32 FirstSmallPartIdx;
+
+ ui32 TotalParts;
+ ui64 WholeBlocks; // Data consists of (WhloeBlocks * BlockSize + TailSize) bytes
+ ui32 TailSize;
+
+ ui32 Prime;
+ TRopeErasureType::ECrcMode CrcMode;
+
using TBufferDataPart = TStackVec<TRopeHelpers::TRopeFastView, MAX_TOTAL_PARTS>;
- TBufferDataPart BufferDataPart;
+ TBufferDataPart BufferDataPart;
TRopeHelpers::TRopeFastView Data;
-
- TBlockParams(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, ui64 dataSize) {
- DataSize = dataSize;
- PartUserSize = type.PartUserSize(dataSize);
- PartContainerSize = type.PartSize(crcMode, dataSize);
- DataParts = type.DataParts();
-
- BlockSize = type.MinimalBlockSize();
- ColumnSize = BlockSize / DataParts;
- WholeColumns = dataSize / ColumnSize;
- LineCount = ColumnSize / sizeof(ui64);
-
- SmallPartColumns = WholeColumns / DataParts;
- LargePartColumns = SmallPartColumns + 1;
-
- LastPartTailSize = DataSize - WholeColumns * ColumnSize;
-
- SmallPartSize = SmallPartColumns * ColumnSize;
- LargePartSize = LargePartColumns * ColumnSize;
-
- FirstSmallPartIdx = WholeColumns % DataParts;
-
- TotalParts = type.TotalPartCount();
- WholeBlocks = DataSize / BlockSize;
- TailSize = (ui32)(DataSize % BlockSize);
-
- Prime = type.Prime();
- CrcMode = crcMode;
- }
-
+
+ TBlockParams(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, ui64 dataSize) {
+ DataSize = dataSize;
+ PartUserSize = type.PartUserSize(dataSize);
+ PartContainerSize = type.PartSize(crcMode, dataSize);
+ DataParts = type.DataParts();
+
+ BlockSize = type.MinimalBlockSize();
+ ColumnSize = BlockSize / DataParts;
+ WholeColumns = dataSize / ColumnSize;
+ LineCount = ColumnSize / sizeof(ui64);
+
+ SmallPartColumns = WholeColumns / DataParts;
+ LargePartColumns = SmallPartColumns + 1;
+
+ LastPartTailSize = DataSize - WholeColumns * ColumnSize;
+
+ SmallPartSize = SmallPartColumns * ColumnSize;
+ LargePartSize = LargePartColumns * ColumnSize;
+
+ FirstSmallPartIdx = WholeColumns % DataParts;
+
+ TotalParts = type.TotalPartCount();
+ WholeBlocks = DataSize / BlockSize;
+ TailSize = (ui32)(DataSize % BlockSize);
+
+ Prime = type.Prime();
+ CrcMode = crcMode;
+ }
+
inline void PrepareInputDataPointers(TRopeHelpers::Iterator data) {
- BufferDataPart.resize(DataParts);
- for (ui32 i = 0; i < FirstSmallPartIdx; ++i) {
+ BufferDataPart.resize(DataParts);
+ for (ui32 i = 0; i < FirstSmallPartIdx; ++i) {
BufferDataPart[i] = TRopeHelpers::TRopeFastView(data);
- data += LargePartSize;
- }
- for (ui32 i = FirstSmallPartIdx; i < DataParts; ++i) {
+ data += LargePartSize;
+ }
+ for (ui32 i = FirstSmallPartIdx; i < DataParts; ++i) {
BufferDataPart[i] = TRopeHelpers::TRopeFastView(data);
- data += SmallPartSize;
- }
- }
-
- inline void XorSplitWhole(TBufferDataPart &bufferDataPart,
- TDataPartSet &outPartSet, ui64 writePosition, ui32 blocks) {
- ui64 readPosition = 0;
- VERBOSE_COUT("XorSplitWhole:" << Endl);
- for (ui64 blockIdx = 0; blockIdx < blocks; ++blockIdx) {
- for (ui64 lineIdx = 0; lineIdx < LineCount; ++lineIdx) {
- ui64 xored = 0;
- for (ui32 part = 0; part < DataParts; ++part) {
- ui64 sourceData;
- sourceData = bufferDataPart[part].At64(readPosition);
- xored ^= sourceData;
- VERBOSE_COUT(DebugFormatBits(sourceData) << ", ");
- *(ui64*)(outPartSet.Parts[part].GetDataAt(writePosition)) = sourceData;
- }
- *(ui64*)(outPartSet.Parts[DataParts].GetDataAt(writePosition)) = xored;
- VERBOSE_COUT(DebugFormatBits(xored) << Endl);
- writePosition += sizeof(ui64);
- ++readPosition;
- }
- }
- VERBOSE_COUT(Endl);
- }
-
- inline void XorSplit(TDataPartSet &outPartSet) {
- VERBOSE_COUT("XorSplit:" << Endl);
- // Write data and parity
- XorSplitWhole(BufferDataPart, outPartSet, 0ull, WholeBlocks);
-
- // Use the remaining parts to fill in the last block
- // Write the tail of the data
- if (TailSize) {
+ data += SmallPartSize;
+ }
+ }
+
+ inline void XorSplitWhole(TBufferDataPart &bufferDataPart,
+ TDataPartSet &outPartSet, ui64 writePosition, ui32 blocks) {
+ ui64 readPosition = 0;
+ VERBOSE_COUT("XorSplitWhole:" << Endl);
+ for (ui64 blockIdx = 0; blockIdx < blocks; ++blockIdx) {
+ for (ui64 lineIdx = 0; lineIdx < LineCount; ++lineIdx) {
+ ui64 xored = 0;
+ for (ui32 part = 0; part < DataParts; ++part) {
+ ui64 sourceData;
+ sourceData = bufferDataPart[part].At64(readPosition);
+ xored ^= sourceData;
+ VERBOSE_COUT(DebugFormatBits(sourceData) << ", ");
+ *(ui64*)(outPartSet.Parts[part].GetDataAt(writePosition)) = sourceData;
+ }
+ *(ui64*)(outPartSet.Parts[DataParts].GetDataAt(writePosition)) = xored;
+ VERBOSE_COUT(DebugFormatBits(xored) << Endl);
+ writePosition += sizeof(ui64);
+ ++readPosition;
+ }
+ }
+ VERBOSE_COUT(Endl);
+ }
+
+ inline void XorSplit(TDataPartSet &outPartSet) {
+ VERBOSE_COUT("XorSplit:" << Endl);
+ // Write data and parity
+ XorSplitWhole(BufferDataPart, outPartSet, 0ull, WholeBlocks);
+
+ // Use the remaining parts to fill in the last block
+ // Write the tail of the data
+ if (TailSize) {
TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64));
- TBufferDataPart bufferDataPart;
- PrepareLastBlockData(lastBlockSource.Begin(), bufferDataPart);
-
- XorSplitWhole(bufferDataPart, outPartSet, WholeBlocks * ColumnSize, 1);
- }
- }
-
-#if IS_VERBOSE
-# define VERBOSE_COUT_BLOCK(IS_FULL_DATA, FULL_DATA_ELEM, PART_ELEM, COL_M, COL_M1) \
- do { \
- for (ui32 row = 0; row < LineCount; ++row) { \
- VERBOSE_COUT(Endl); \
- for (ui32 col = 0; col < DataParts; ++col) { \
- if (IS_FULL_DATA) { \
- VERBOSE_COUT(DebugFormatBits(FULL_DATA_ELEM(row, col)) << ", "); \
- } else { \
- VERBOSE_COUT(DebugFormatBits(PART_ELEM(row, col)) << ", "); \
- } \
- } \
- VERBOSE_COUT(DebugFormatBits(COL_M(row)) << ", "); \
- VERBOSE_COUT(DebugFormatBits(COL_M1(row))); \
- } \
- VERBOSE_COUT(Endl); \
- } while (false)
-# define VERBOSE_COUT_BLOCK_M2(IS_FULL_DATA, FULL_DATA_ELEM, PART_ELEM, COL_M, COL_M1, COL_M2) \
- do { \
- for (ui32 row = 0; row < LineCount; ++row) { \
- VERBOSE_COUT(Endl); \
- for (ui32 col = 0; col < DataParts; ++col) { \
- if (IS_FULL_DATA) { \
- VERBOSE_COUT(DebugFormatBits(FULL_DATA_ELEM(row, col)) << ", "); \
- } else { \
- VERBOSE_COUT(DebugFormatBits(PART_ELEM(row, col)) << ", "); \
- } \
- } \
- VERBOSE_COUT(DebugFormatBits(COL_M(row)) << ", "); \
- VERBOSE_COUT(DebugFormatBits(COL_M1(row)) << ", "); \
- VERBOSE_COUT(DebugFormatBits(COL_M2(row))); \
- } \
- VERBOSE_COUT(Endl); \
- } while (false)
-#else
-# define VERBOSE_COUT_BLOCK(IS_FULL_DATA, FULL_DATA_ELEM, PART_ELEM, COL_M, COL_M1) \
- do { \
- } while (false)
-# define VERBOSE_COUT_BLOCK_M2(IS_FULL_DATA, FULL_DATA_ELEM, PART_ELEM, COL_M, COL_M1, COL_M2) \
- do { \
- } while (false)
-#endif
-
-
-
- template <bool isFastMode, class TData>
- class EoView {
- private:
- TData& Data;
- ui64 Offset;
-
- public:
- EoView(TData& data, ui64 offset)
- : Data(data)
- , Offset(offset) {
- }
-
- ui64& At(ui64 row, ui64 column) {
- if constexpr (std::is_same_v<TDataPartSet, TData>) {
- if constexpr (isFastMode) {
- return *(ui64*)(Data.Parts[column].FastDataPtrAt(Offset + row * sizeof(ui64)));
- } else {
- return *(ui64*)(Data.Parts[column].GetDataAt(Offset + row * sizeof(ui64)));
- }
- } else {
- return *(ui64*) (Data[column] + row * sizeof(ui64));
- }
- }
- };
-
- template <bool isFastMode, class TData>
- void EoSplitLoop(TDataPartSet& outPartSet, ui64 writePosition, ui32 blocks, TData& inPartSet) {
-
- const ui32 m = Prime;
-
-
- for (ui64 blockIdx = 0; blockIdx < blocks; ++blockIdx) {
-
- EoView<isFastMode, TData> in(inPartSet, writePosition);
- EoView<isFastMode, TDataPartSet> out(outPartSet, writePosition);
-
- VERBOSE_COUT_BLOCK(true, IN_EL_BLOCK, IN_EL_BLOCK, OUT_M, OUT_M1);
-
- ui64 adj = 0;
- const ui32 mint = (m - 2 < LineCount ? 1 : m - 2 - LineCount);
- VERBOSE_COUT("mint = " << mint << " m - 1 - t = " << (m - 1 - mint) << Endl);
- for (ui32 t = mint; t < DataParts; ++t) {
- adj ^= in.At(m - 1 - t, t);
- VERBOSE_COUT("s: " << adj << " el[" << (m - 1 - t) << ", " << t << "]: " <<
- DebugFormatBits(in.At(m - 1 - t, t)) << Endl);
- }
- for (ui32 l = 0; l < LineCount; ++l) {
- ui64 sourceData = in.At(l, 0);
- out.At(l, DataParts + 1) = adj ^ sourceData;
- out.At(l, DataParts) = sourceData;
- }
- for (ui32 t = 1; t < DataParts; ++t) {
- for (ui32 l = 0; l < LineCount; ++l) {
- out.At(l, DataParts) ^= in.At(l, t);
- VERBOSE_COUT("OUT_M(" << l << ") = " <<
- DebugFormatBits(out.At(l, DataParts)) << Endl);
- }
- }
- for (ui32 t = 1; t < DataParts; ++t) {
- for (ui32 l = 0; l < LineCount - t; ++l) {
- ui32 row = l + t;
- out.At(row, DataParts + 1) ^= in.At(l, t);
- VERBOSE_COUT(DebugFormatBits(in.At(row, t)) << Endl);
- }
- for (ui32 l = LineCount - t + 1; l < LineCount; ++l) {
- ui32 row = l + t - m;
- out.At(row, DataParts + 1) ^= in.At(l, t);
- VERBOSE_COUT(DebugFormatBits(in.At(row, t)) << Endl);
- }
- }
-
- VERBOSE_COUT_BLOCK(true, OUT_EL, OUT_EL, OUT_M, OUT_M1);
- writePosition += ColumnSize;
- }
- }
-
- void EoSplitWhole(TDataPartSet &outPartSet, ui64 writePosition, ui32 blocks) {
- for (ui64 blockIdx = 0; blockIdx < blocks; ) {
- ui64 contiguousSize = PartContainerSize;
- for (ui32 i = 0; i < DataParts + 2; ++i) {
- contiguousSize = Min(contiguousSize, outPartSet.Parts[i].GetContiguousSize(writePosition));
- }
- ui64 contiguousBlocks = contiguousSize / ColumnSize;
- contiguousBlocks = Min(contiguousBlocks, blocks - blockIdx);
-
- EoSplitLoop<true>(outPartSet, writePosition, contiguousBlocks, outPartSet);
- writePosition += contiguousBlocks * ColumnSize;
- blockIdx += contiguousBlocks;
-
- if (blockIdx == blocks) {
- break;
- }
-
- bool isAligned = true;
- for (ui32 i = 0; i < DataParts; ++i) {
- ui64 currentSize = outPartSet.Parts[i].GetContiguousSize(writePosition);
- if (currentSize < ColumnSize && currentSize % sizeof(ui64)) {
- isAligned = false;
- break;
- }
- }
-
- if (isAligned) {
- EoSplitLoop<false>(outPartSet, writePosition, 1u, outPartSet);
- } else {
- char buffer[MAX_TOTAL_PARTS][(MAX_TOTAL_PARTS - 2) * sizeof(ui64)];
- for (ui32 i = 0; i < DataParts; ++i) {
+ TBufferDataPart bufferDataPart;
+ PrepareLastBlockData(lastBlockSource.Begin(), bufferDataPart);
+
+ XorSplitWhole(bufferDataPart, outPartSet, WholeBlocks * ColumnSize, 1);
+ }
+ }
+
+#if IS_VERBOSE
+# define VERBOSE_COUT_BLOCK(IS_FULL_DATA, FULL_DATA_ELEM, PART_ELEM, COL_M, COL_M1) \
+ do { \
+ for (ui32 row = 0; row < LineCount; ++row) { \
+ VERBOSE_COUT(Endl); \
+ for (ui32 col = 0; col < DataParts; ++col) { \
+ if (IS_FULL_DATA) { \
+ VERBOSE_COUT(DebugFormatBits(FULL_DATA_ELEM(row, col)) << ", "); \
+ } else { \
+ VERBOSE_COUT(DebugFormatBits(PART_ELEM(row, col)) << ", "); \
+ } \
+ } \
+ VERBOSE_COUT(DebugFormatBits(COL_M(row)) << ", "); \
+ VERBOSE_COUT(DebugFormatBits(COL_M1(row))); \
+ } \
+ VERBOSE_COUT(Endl); \
+ } while (false)
+# define VERBOSE_COUT_BLOCK_M2(IS_FULL_DATA, FULL_DATA_ELEM, PART_ELEM, COL_M, COL_M1, COL_M2) \
+ do { \
+ for (ui32 row = 0; row < LineCount; ++row) { \
+ VERBOSE_COUT(Endl); \
+ for (ui32 col = 0; col < DataParts; ++col) { \
+ if (IS_FULL_DATA) { \
+ VERBOSE_COUT(DebugFormatBits(FULL_DATA_ELEM(row, col)) << ", "); \
+ } else { \
+ VERBOSE_COUT(DebugFormatBits(PART_ELEM(row, col)) << ", "); \
+ } \
+ } \
+ VERBOSE_COUT(DebugFormatBits(COL_M(row)) << ", "); \
+ VERBOSE_COUT(DebugFormatBits(COL_M1(row)) << ", "); \
+ VERBOSE_COUT(DebugFormatBits(COL_M2(row))); \
+ } \
+ VERBOSE_COUT(Endl); \
+ } while (false)
+#else
+# define VERBOSE_COUT_BLOCK(IS_FULL_DATA, FULL_DATA_ELEM, PART_ELEM, COL_M, COL_M1) \
+ do { \
+ } while (false)
+# define VERBOSE_COUT_BLOCK_M2(IS_FULL_DATA, FULL_DATA_ELEM, PART_ELEM, COL_M, COL_M1, COL_M2) \
+ do { \
+ } while (false)
+#endif
+
+
+
+ template <bool isFastMode, class TData>
+ class EoView {
+ private:
+ TData& Data;
+ ui64 Offset;
+
+ public:
+ EoView(TData& data, ui64 offset)
+ : Data(data)
+ , Offset(offset) {
+ }
+
+ ui64& At(ui64 row, ui64 column) {
+ if constexpr (std::is_same_v<TDataPartSet, TData>) {
+ if constexpr (isFastMode) {
+ return *(ui64*)(Data.Parts[column].FastDataPtrAt(Offset + row * sizeof(ui64)));
+ } else {
+ return *(ui64*)(Data.Parts[column].GetDataAt(Offset + row * sizeof(ui64)));
+ }
+ } else {
+ return *(ui64*) (Data[column] + row * sizeof(ui64));
+ }
+ }
+ };
+
+ template <bool isFastMode, class TData>
+ void EoSplitLoop(TDataPartSet& outPartSet, ui64 writePosition, ui32 blocks, TData& inPartSet) {
+
+ const ui32 m = Prime;
+
+
+ for (ui64 blockIdx = 0; blockIdx < blocks; ++blockIdx) {
+
+ EoView<isFastMode, TData> in(inPartSet, writePosition);
+ EoView<isFastMode, TDataPartSet> out(outPartSet, writePosition);
+
+ VERBOSE_COUT_BLOCK(true, IN_EL_BLOCK, IN_EL_BLOCK, OUT_M, OUT_M1);
+
+ ui64 adj = 0;
+ const ui32 mint = (m - 2 < LineCount ? 1 : m - 2 - LineCount);
+ VERBOSE_COUT("mint = " << mint << " m - 1 - t = " << (m - 1 - mint) << Endl);
+ for (ui32 t = mint; t < DataParts; ++t) {
+ adj ^= in.At(m - 1 - t, t);
+ VERBOSE_COUT("s: " << adj << " el[" << (m - 1 - t) << ", " << t << "]: " <<
+ DebugFormatBits(in.At(m - 1 - t, t)) << Endl);
+ }
+ for (ui32 l = 0; l < LineCount; ++l) {
+ ui64 sourceData = in.At(l, 0);
+ out.At(l, DataParts + 1) = adj ^ sourceData;
+ out.At(l, DataParts) = sourceData;
+ }
+ for (ui32 t = 1; t < DataParts; ++t) {
+ for (ui32 l = 0; l < LineCount; ++l) {
+ out.At(l, DataParts) ^= in.At(l, t);
+ VERBOSE_COUT("OUT_M(" << l << ") = " <<
+ DebugFormatBits(out.At(l, DataParts)) << Endl);
+ }
+ }
+ for (ui32 t = 1; t < DataParts; ++t) {
+ for (ui32 l = 0; l < LineCount - t; ++l) {
+ ui32 row = l + t;
+ out.At(row, DataParts + 1) ^= in.At(l, t);
+ VERBOSE_COUT(DebugFormatBits(in.At(row, t)) << Endl);
+ }
+ for (ui32 l = LineCount - t + 1; l < LineCount; ++l) {
+ ui32 row = l + t - m;
+ out.At(row, DataParts + 1) ^= in.At(l, t);
+ VERBOSE_COUT(DebugFormatBits(in.At(row, t)) << Endl);
+ }
+ }
+
+ VERBOSE_COUT_BLOCK(true, OUT_EL, OUT_EL, OUT_M, OUT_M1);
+ writePosition += ColumnSize;
+ }
+ }
+
+ void EoSplitWhole(TDataPartSet &outPartSet, ui64 writePosition, ui32 blocks) {
+ for (ui64 blockIdx = 0; blockIdx < blocks; ) {
+ ui64 contiguousSize = PartContainerSize;
+ for (ui32 i = 0; i < DataParts + 2; ++i) {
+ contiguousSize = Min(contiguousSize, outPartSet.Parts[i].GetContiguousSize(writePosition));
+ }
+ ui64 contiguousBlocks = contiguousSize / ColumnSize;
+ contiguousBlocks = Min(contiguousBlocks, blocks - blockIdx);
+
+ EoSplitLoop<true>(outPartSet, writePosition, contiguousBlocks, outPartSet);
+ writePosition += contiguousBlocks * ColumnSize;
+ blockIdx += contiguousBlocks;
+
+ if (blockIdx == blocks) {
+ break;
+ }
+
+ bool isAligned = true;
+ for (ui32 i = 0; i < DataParts; ++i) {
+ ui64 currentSize = outPartSet.Parts[i].GetContiguousSize(writePosition);
+ if (currentSize < ColumnSize && currentSize % sizeof(ui64)) {
+ isAligned = false;
+ break;
+ }
+ }
+
+ if (isAligned) {
+ EoSplitLoop<false>(outPartSet, writePosition, 1u, outPartSet);
+ } else {
+ char buffer[MAX_TOTAL_PARTS][(MAX_TOTAL_PARTS - 2) * sizeof(ui64)];
+ for (ui32 i = 0; i < DataParts; ++i) {
TRopeUtils::Memcpy(buffer[i], outPartSet.Parts[i].FastViewer.GetCurrent(writePosition), ColumnSize);
- }
- EoSplitLoop<false>(outPartSet, writePosition, 1u, buffer);
- }
-
- writePosition += ColumnSize;
- blockIdx += 1;
- }
- }
-
- template <bool isFromDataParts>
- void StarSplitWhole(TBufferDataPart &bufferDataPart, TDataPartSet &outPartSet,
- ui64 writePosition, ui32 blocks) {
- const ui32 m = Prime;
-#define IN_EL_SB(row, column) bufferDataPart[column].At64(blockIdx * LineCount + (row))
-#define OUT_EL(row, column) *((ui64*)(outPartSet.Parts[column].GetDataAt(writePosition + (row) * sizeof(ui64))))
-#define IN_EL(row, column) (isFromDataParts ? OUT_EL(row, column) : IN_EL_SB(row, column))
-#define OUT_M(row) *((ui64*)(outPartSet.Parts[DataParts].GetDataAt(writePosition + (row) * sizeof(ui64))))
-#define OUT_M1(row) *((ui64*)(outPartSet.Parts[DataParts + 1].GetDataAt(writePosition + (row) * sizeof(ui64))))
-#define OUT_M2(row) *((ui64*)(outPartSet.Parts[DataParts + 2].GetDataAt(writePosition + (row) * sizeof(ui64))))
- for (ui64 blockIdx = 0; blockIdx < blocks; ++blockIdx) {
- VERBOSE_COUT_BLOCK(true, IN_EL_BLOCK, IN_EL_BLOCK, OUT_M, OUT_M1);
- ui64 s1 = 0;
- const ui32 mint = (m - 2 < LineCount ? 1 : m - 2 - LineCount);
- VERBOSE_COUT("mint = " << mint << " m - 1 - t = " << (m - 1 - mint) << Endl);
- for (ui32 t = mint; t < DataParts; ++t) {
- s1 ^= IN_EL(m - 1 - t, t);
- VERBOSE_COUT("s1: " << s1 << " el[" << (m - 1 - t) << ", " << t << "]: " <<
- DebugFormatBits(IN_EL_BLOCK(m - 1 - t, t)) << Endl);
- }
- ui64 s2 = 0;
- for (ui32 t = 1; t < DataParts; ++t) {
- s2 ^= IN_EL(t - 1, t);
- VERBOSE_COUT("s2: " << s2 << " el[" << (t - 1) << ", " << t << "]: " <<
- DebugFormatBits(IN_EL(t - 1, t)) << Endl);
- }
- for (ui32 l = 0; l < LineCount; ++l) {
- ui64 dataIN_EL = IN_EL(l, 0);
- OUT_M(l) = dataIN_EL;
- OUT_M1(l) = s1 ^ dataIN_EL;
- OUT_M2(l) = s2 ^ dataIN_EL;
- if (!isFromDataParts) {
- OUT_EL(l, 0) = dataIN_EL;
- }
- }
- for (ui32 t = 1; t < DataParts; ++t) {
- for (ui32 l = 0; l < LineCount; ++l) {
- ui64 dataIN_EL = IN_EL(l, t);
- ui32 row1 = (l + t) % m;
- OUT_M(l) ^= dataIN_EL;
- if (row1 < LineCount) {
- OUT_M1(row1) ^= dataIN_EL;
- VERBOSE_COUT(IN_EL(row1, t) << Endl);
- }
- ui32 row2 = (m + l - t) % m;
- if (row2 < LineCount) {
- OUT_M2(row2) ^= dataIN_EL;
- VERBOSE_COUT(IN_EL(row2, t) << Endl);
- }
- if (!isFromDataParts) {
- OUT_EL(l, t) = dataIN_EL;
- }
- }
- }
-#if IS_VERBOSE
- for (ui32 l = 0; l < LineCount; ++l) {
- VERBOSE_COUT("OUT_M1(" << l << ") = " << DebugFormatBits(OUT_M1(l)) << Endl);
- }
- VERBOSE_COUT_BLOCK_M2(true, OUT_EL, OUT_EL, OUT_M, OUT_M1, OUT_M2);
-#endif
- writePosition += ColumnSize;
- }
-#undef OUT_M2
-#undef OUT_M1
-#undef OUT_M
-#undef OUT_EL
-#undef IN_EL
- }
-
+ }
+ EoSplitLoop<false>(outPartSet, writePosition, 1u, buffer);
+ }
+
+ writePosition += ColumnSize;
+ blockIdx += 1;
+ }
+ }
+
+ template <bool isFromDataParts>
+ void StarSplitWhole(TBufferDataPart &bufferDataPart, TDataPartSet &outPartSet,
+ ui64 writePosition, ui32 blocks) {
+ const ui32 m = Prime;
+#define IN_EL_SB(row, column) bufferDataPart[column].At64(blockIdx * LineCount + (row))
+#define OUT_EL(row, column) *((ui64*)(outPartSet.Parts[column].GetDataAt(writePosition + (row) * sizeof(ui64))))
+#define IN_EL(row, column) (isFromDataParts ? OUT_EL(row, column) : IN_EL_SB(row, column))
+#define OUT_M(row) *((ui64*)(outPartSet.Parts[DataParts].GetDataAt(writePosition + (row) * sizeof(ui64))))
+#define OUT_M1(row) *((ui64*)(outPartSet.Parts[DataParts + 1].GetDataAt(writePosition + (row) * sizeof(ui64))))
+#define OUT_M2(row) *((ui64*)(outPartSet.Parts[DataParts + 2].GetDataAt(writePosition + (row) * sizeof(ui64))))
+ for (ui64 blockIdx = 0; blockIdx < blocks; ++blockIdx) {
+ VERBOSE_COUT_BLOCK(true, IN_EL_BLOCK, IN_EL_BLOCK, OUT_M, OUT_M1);
+ ui64 s1 = 0;
+ const ui32 mint = (m - 2 < LineCount ? 1 : m - 2 - LineCount);
+ VERBOSE_COUT("mint = " << mint << " m - 1 - t = " << (m - 1 - mint) << Endl);
+ for (ui32 t = mint; t < DataParts; ++t) {
+ s1 ^= IN_EL(m - 1 - t, t);
+ VERBOSE_COUT("s1: " << s1 << " el[" << (m - 1 - t) << ", " << t << "]: " <<
+ DebugFormatBits(IN_EL_BLOCK(m - 1 - t, t)) << Endl);
+ }
+ ui64 s2 = 0;
+ for (ui32 t = 1; t < DataParts; ++t) {
+ s2 ^= IN_EL(t - 1, t);
+ VERBOSE_COUT("s2: " << s2 << " el[" << (t - 1) << ", " << t << "]: " <<
+ DebugFormatBits(IN_EL(t - 1, t)) << Endl);
+ }
+ for (ui32 l = 0; l < LineCount; ++l) {
+ ui64 dataIN_EL = IN_EL(l, 0);
+ OUT_M(l) = dataIN_EL;
+ OUT_M1(l) = s1 ^ dataIN_EL;
+ OUT_M2(l) = s2 ^ dataIN_EL;
+ if (!isFromDataParts) {
+ OUT_EL(l, 0) = dataIN_EL;
+ }
+ }
+ for (ui32 t = 1; t < DataParts; ++t) {
+ for (ui32 l = 0; l < LineCount; ++l) {
+ ui64 dataIN_EL = IN_EL(l, t);
+ ui32 row1 = (l + t) % m;
+ OUT_M(l) ^= dataIN_EL;
+ if (row1 < LineCount) {
+ OUT_M1(row1) ^= dataIN_EL;
+ VERBOSE_COUT(IN_EL(row1, t) << Endl);
+ }
+ ui32 row2 = (m + l - t) % m;
+ if (row2 < LineCount) {
+ OUT_M2(row2) ^= dataIN_EL;
+ VERBOSE_COUT(IN_EL(row2, t) << Endl);
+ }
+ if (!isFromDataParts) {
+ OUT_EL(l, t) = dataIN_EL;
+ }
+ }
+ }
+#if IS_VERBOSE
+ for (ui32 l = 0; l < LineCount; ++l) {
+ VERBOSE_COUT("OUT_M1(" << l << ") = " << DebugFormatBits(OUT_M1(l)) << Endl);
+ }
+ VERBOSE_COUT_BLOCK_M2(true, OUT_EL, OUT_EL, OUT_M, OUT_M1, OUT_M2);
+#endif
+ writePosition += ColumnSize;
+ }
+#undef OUT_M2
+#undef OUT_M1
+#undef OUT_M
+#undef OUT_EL
+#undef IN_EL
+ }
+
void PrepareLastBlockData(TRopeHelpers::Iterator lastBlockSource, TBufferDataPart &bufferDataPart) {
- bufferDataPart.resize(DataParts);
- for (ui32 i = 0; i < FirstSmallPartIdx; ++i) {
+ bufferDataPart.resize(DataParts);
+ for (ui32 i = 0; i < FirstSmallPartIdx; ++i) {
bufferDataPart[i] = TRopeHelpers::TRopeFastView(lastBlockSource);
TRopeUtils::Memcpy(bufferDataPart[i].GetBegin(), BufferDataPart[i].GetBegin() + WholeBlocks * ColumnSize,
- ColumnSize);
- lastBlockSource += ColumnSize;
- }
- for (ui32 i = FirstSmallPartIdx; i < DataParts - 1; ++i) {
+ ColumnSize);
+ lastBlockSource += ColumnSize;
+ }
+ for (ui32 i = FirstSmallPartIdx; i < DataParts - 1; ++i) {
bufferDataPart[i] = TRopeHelpers::TRopeFastView(lastBlockSource);
TRopeUtils::Memset(bufferDataPart[i].GetBegin(), 0, ColumnSize);
- lastBlockSource += ColumnSize;
- }
+ lastBlockSource += ColumnSize;
+ }
bufferDataPart[DataParts - 1] = TRopeHelpers::TRopeFastView(lastBlockSource);
- if (LastPartTailSize) {
+ if (LastPartTailSize) {
TRopeUtils::Memcpy(bufferDataPart[DataParts - 1].GetBegin(), BufferDataPart[DataParts - 1].GetBegin() + WholeBlocks * ColumnSize,
- LastPartTailSize);
- }
+ LastPartTailSize);
+ }
TRopeUtils::Memset(bufferDataPart[DataParts - 1].GetBegin() + LastPartTailSize, 0, ColumnSize - LastPartTailSize);
- }
-
+ }
+
void PrepareLastBlockPointers(TRopeHelpers::Iterator lastBlockSource, TBufferDataPart &bufferDataPart) {
- bufferDataPart.resize(DataParts);
- for (ui32 i = 0; i < DataParts; ++i) {
+ bufferDataPart.resize(DataParts);
+ for (ui32 i = 0; i < DataParts; ++i) {
bufferDataPart[i] = TRopeHelpers::TRopeFastView(lastBlockSource);
- lastBlockSource += ColumnSize;
- }
- }
-
- void PlaceLastBlock(TBufferDataPart& bufferDataPart) {
- for (ui32 i = 0; i < FirstSmallPartIdx; ++i) {
+ lastBlockSource += ColumnSize;
+ }
+ }
+
+ void PlaceLastBlock(TBufferDataPart& bufferDataPart) {
+ for (ui32 i = 0; i < FirstSmallPartIdx; ++i) {
TRopeUtils::Memcpy(BufferDataPart[i].GetBegin() + WholeBlocks * ColumnSize, bufferDataPart[i].GetBegin(), ColumnSize);
- }
+ }
TRopeUtils::Memcpy(BufferDataPart[DataParts - 1].GetBegin() + WholeBlocks * ColumnSize,
- bufferDataPart[DataParts - 1].GetBegin(), LastPartTailSize);
- }
-
- template <bool isFromDataParts>
- void StarSplit(TDataPartSet &outPartSet) {
- // Use all whole columns of all the parts
- StarSplitWhole<isFromDataParts>(BufferDataPart, outPartSet, 0ull, WholeBlocks);
-
- // Use the remaining parts to fill in the last block
- // Write the tail of the data
- if (TailSize) {
+ bufferDataPart[DataParts - 1].GetBegin(), LastPartTailSize);
+ }
+
+ template <bool isFromDataParts>
+ void StarSplit(TDataPartSet &outPartSet) {
+ // Use all whole columns of all the parts
+ StarSplitWhole<isFromDataParts>(BufferDataPart, outPartSet, 0ull, WholeBlocks);
+
+ // Use the remaining parts to fill in the last block
+ // Write the tail of the data
+ if (TailSize) {
TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64));
- TBufferDataPart bufferDataPart;
- if (!isFromDataParts) {
- PrepareLastBlockData(lastBlockSource.Begin(), bufferDataPart);
- }
-
- StarSplitWhole<isFromDataParts>(bufferDataPart, outPartSet, WholeBlocks * ColumnSize, 1);
- }
- }
-
- void EoSplit(TDataPartSet &outPartSet) {
- Y_VERIFY(outPartSet.Parts[0].Offset % ColumnSize == 0);
- ui64 readPosition = outPartSet.Parts[0].Offset;
- ui64 wholeBlocks = Min(WholeBlocks - readPosition / ColumnSize, outPartSet.Parts[0].Size / ColumnSize);
- bool hasTail = TailSize && (outPartSet.Parts[0].Size + readPosition > WholeBlocks * ColumnSize);
-
- TRACE(__LINE__ << " EoSplit readPosition# " << readPosition
- << " wholeBlocks# " << wholeBlocks
- << " hasTail# " << hasTail
- << " fullDataSize# " << outPartSet.FullDataSize
- << Endl);
- // Use all columns of all the parts
- EoSplitWhole(outPartSet, readPosition, wholeBlocks + hasTail);
- }
-
- template <bool restoreParts>
- void GlueOutBuffer(TRope& dst, const TDataPartSet& partSet, ui32 missingPartIdxA, ui32 missingPartIdxB) {
- Y_VERIFY(dst.GetSize() == 0);
- for (ui32 i = 0; i < DataParts; ++i) {
- const TPartFragment& part = partSet.Parts[i];
- Y_VERIFY(part.Offset == 0 && part.Size == part.PartSize);
- ui64 partSize = i < FirstSmallPartIdx ? LargePartSize : SmallPartSize;
- partSize = i == DataParts - 1 ? SmallPartSize + LastPartTailSize : partSize;
- if (!restoreParts && (i == missingPartIdxA || i == missingPartIdxB)) {
+ TBufferDataPart bufferDataPart;
+ if (!isFromDataParts) {
+ PrepareLastBlockData(lastBlockSource.Begin(), bufferDataPart);
+ }
+
+ StarSplitWhole<isFromDataParts>(bufferDataPart, outPartSet, WholeBlocks * ColumnSize, 1);
+ }
+ }
+
+ void EoSplit(TDataPartSet &outPartSet) {
+ Y_VERIFY(outPartSet.Parts[0].Offset % ColumnSize == 0);
+ ui64 readPosition = outPartSet.Parts[0].Offset;
+ ui64 wholeBlocks = Min(WholeBlocks - readPosition / ColumnSize, outPartSet.Parts[0].Size / ColumnSize);
+ bool hasTail = TailSize && (outPartSet.Parts[0].Size + readPosition > WholeBlocks * ColumnSize);
+
+ TRACE(__LINE__ << " EoSplit readPosition# " << readPosition
+ << " wholeBlocks# " << wholeBlocks
+ << " hasTail# " << hasTail
+ << " fullDataSize# " << outPartSet.FullDataSize
+ << Endl);
+ // Use all columns of all the parts
+ EoSplitWhole(outPartSet, readPosition, wholeBlocks + hasTail);
+ }
+
+ template <bool restoreParts>
+ void GlueOutBuffer(TRope& dst, const TDataPartSet& partSet, ui32 missingPartIdxA, ui32 missingPartIdxB) {
+ Y_VERIFY(dst.GetSize() == 0);
+ for (ui32 i = 0; i < DataParts; ++i) {
+ const TPartFragment& part = partSet.Parts[i];
+ Y_VERIFY(part.Offset == 0 && part.Size == part.PartSize);
+ ui64 partSize = i < FirstSmallPartIdx ? LargePartSize : SmallPartSize;
+ partSize = i == DataParts - 1 ? SmallPartSize + LastPartTailSize : partSize;
+ if (!restoreParts && (i == missingPartIdxA || i == missingPartIdxB)) {
dst.Insert(dst.End(), TRopeHelpers::RopeUninitialized(partSize));
- } else {
- dst.Insert(dst.End(), TRope(part.OwnedRope.Begin(), part.OwnedRope.Begin() + partSize));
- }
- }
- }
-
+ } else {
+ dst.Insert(dst.End(), TRope(part.OwnedRope.Begin(), part.OwnedRope.Begin() + partSize));
+ }
+ }
+ }
+
void GlueBlockPartsMemcpy(TRopeHelpers::Iterator dst, const TDataPartSet& partSet) const {
- if (LargePartSize) {
- for (ui32 i = 0; i < FirstSmallPartIdx; ++i) {
+ if (LargePartSize) {
+ for (ui32 i = 0; i < FirstSmallPartIdx; ++i) {
TRopeUtils::Memcpy(dst, partSet.Parts[i].OwnedRope.Begin(), LargePartSize);
- dst += LargePartSize;
- }
- if (SmallPartSize) {
- for (ui32 i = FirstSmallPartIdx; i < DataParts - 1; ++i) {
+ dst += LargePartSize;
+ }
+ if (SmallPartSize) {
+ for (ui32 i = FirstSmallPartIdx; i < DataParts - 1; ++i) {
TRopeUtils::Memcpy(dst, partSet.Parts[i].OwnedRope.Begin(), SmallPartSize);
- dst += SmallPartSize;
- }
- }
- }
- if (SmallPartSize + LastPartTailSize) {
+ dst += SmallPartSize;
+ }
+ }
+ }
+ if (SmallPartSize + LastPartTailSize) {
TRopeUtils::Memcpy(dst, partSet.Parts[DataParts - 1].OwnedRope.Begin(), SmallPartSize + LastPartTailSize);
- }
- return;
- }
-
- // s = a[(m + missingDataPartIdx - 1) % m][m + 1];
- // for (l = 0; l < m; ++l) {
- // s ^= a[(m + missingDataPartIdx - l - 1) % m][l];
- // }
- // for (k = 0; k < m - 1; ++k) {
- // ui64 res = s;
- // for (l = 0; l < missingDataPartIdx; ++l) {
- // res ^= a[(m + k + missingDataPartIdx - l) % m][l];
- // }
- // for (l = missingDataPartIdx + 1; l < m; ++l) {
- // res ^= a[(m + k + missingDataPartIdx - l) % m][l];
- // }
- // a[k][missingDataPartIdx] = res;
- // }
- template <bool restoreParts, bool restoreFullData, bool reversed, bool restoreParityParts>
- void EoDiagonalRestorePartWhole(TBufferDataPart &bufferDataPart, TDataPartSet &partSet, ui64 readPosition,
- ui32 beginBlockIdx, ui32 endBlockIdx, ui32 missingDataPartIdx) {
- ui32 lastColumn = reversed ? DataParts + 2 : DataParts + 1;
- const ui32 m = Prime;
- // Use all whole columns of all the parts
- ui64 fullDataBuffer[MAX_TOTAL_PARTS][MAX_TOTAL_PARTS - 2];
- char partsBuffer[MAX_TOTAL_PARTS][(MAX_TOTAL_PARTS - 2) * sizeof(ui64)];
-
- for (ui64 blockIdx = beginBlockIdx; blockIdx < endBlockIdx; ++blockIdx) {
-#define RIGHT_ROW(row) (reversed ? LineCount - 1 - (row) : (row))
-#define OUT_EL_BLOCK(row, column) fullDataBuffer[column][RIGHT_ROW(row)]
-#define IN_EL(row, column) *((ui64*)(partsBuffer[column] + RIGHT_ROW(row) * sizeof(ui64)))
-#define IN_M(row) *((ui64*)(partSet.Parts[DataParts].GetDataAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
-#define IN_M12(row) *((ui64*)(partsBuffer[lastColumn] + RIGHT_ROW(row) * sizeof(ui64)))
-
- ui64 readContiguousSize = DataSize;
- ui64 writeContiguousSize = DataSize;
- if (restoreFullData) {
- for (ui32 i = 0; i < DataParts; ++i) {
- writeContiguousSize = std::min(writeContiguousSize,
- bufferDataPart[i].GetContiguousSize(blockIdx * LineCount * sizeof(ui64)));
- }
- }
- for (ui32 i = 0; i < DataParts; ++i) {
- if (restoreParts || i != missingDataPartIdx) {
- readContiguousSize = std::min(readContiguousSize,
- partSet.Parts[i].GetContiguousSize(readPosition));
- }
- }
- if (restoreParityParts) {
- readContiguousSize = std::min(readContiguousSize,
- partSet.Parts[DataParts].GetContiguousSize(readPosition));
- }
- readContiguousSize = std::min(readContiguousSize,
- partSet.Parts[lastColumn].GetContiguousSize(readPosition));
-
- readContiguousSize /= ColumnSize;
- writeContiguousSize /= LineCount * sizeof(ui64);
- ui64 contiguousSize = std::min(readContiguousSize, writeContiguousSize);
-
- for (ui32 i = 0; i < contiguousSize && blockIdx < endBlockIdx; ++i, ++blockIdx) {
-
-#define FAST_OUT_EL_BLOCK(row, column) bufferDataPart[column].FastAt64(blockIdx * LineCount + RIGHT_ROW(row))
-#define FAST_IN_EL(row, column) *((ui64*)(partSet.Parts[column].FastDataPtrAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
-#define FAST_IN_M(row) *((ui64*)(partSet.Parts[DataParts].FastDataPtrAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
-#define FAST_IN_M12(row) *((ui64*)(partSet.Parts[lastColumn].FastDataPtrAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
-
- VERBOSE_COUT_BLOCK(true, FAST_IN_EL, FAST_IN_EL, FAST_IN_M, FAST_IN_M12);
- ui64 s = 0;
- ui32 colLimit = DataParts;
- ui32 rowLimit = LineCount;
- {
- ui32 idx = (m + missingDataPartIdx - 1) % m;
- if (idx < rowLimit) {
- s = FAST_IN_M12(idx);
- VERBOSE_COUT("s(" << idx << ", m1): " << DebugFormatBits(s) << Endl);
- }
- }
- for (ui32 l = 0; l < colLimit; ++l) {
- ui32 idx = (m + missingDataPartIdx - l - 1) % m;
- if (idx < LineCount) {
- ui64 value = FAST_IN_EL(idx, l);
- s ^= value;
- if (restoreFullData) {
- VERBOSE_COUT("a [" << idx << ", " << l << "] = " << DebugFormatBits(value) << Endl);
- FAST_OUT_EL_BLOCK(idx, l) = value;
- }
- }
- }
- VERBOSE_COUT("s: " << DebugFormatBits(s) << Endl);
- for (ui32 k = 0; k < LineCount; ++k) {
- ui64 res = s;
- for (ui32 l = 0; l < missingDataPartIdx; ++l) {
- ui32 idx = (m + k + missingDataPartIdx - l) % m;
- if (idx < LineCount) {
- ui64 value = FAST_IN_EL(idx, l);
- res ^= value;
- if (restoreFullData) {
- VERBOSE_COUT("b [" << idx << ", " << l << "] = " << DebugFormatBits(value) << Endl);
- FAST_OUT_EL_BLOCK(idx, l) = value;
- }
- }
- }
- for (ui32 l = missingDataPartIdx + 1; l < colLimit; ++l) {
- ui32 idx = (m + k + missingDataPartIdx - l) % m;
- if (idx < LineCount) {
- ui64 value = FAST_IN_EL(idx, l);
- res ^= value;
- if (restoreFullData) {
- VERBOSE_COUT("c [" << idx << ", " << l << "] = " << DebugFormatBits(value) << Endl);
- FAST_OUT_EL_BLOCK(idx, l) = value;
- }
- }
- }
- ui32 idx = (m + k + missingDataPartIdx) % m;
- if (idx < LineCount) {
- VERBOSE_COUT("idx = " << idx);
- res ^= FAST_IN_M12(idx); // This is missing in the article!
- }
- if (restoreFullData) {
- VERBOSE_COUT("out [" << k << ", " << missingDataPartIdx << "] = " << DebugFormatBits(res) << Endl);
- FAST_OUT_EL_BLOCK(k, missingDataPartIdx) = res;
- }
- if (restoreParts) {
- FAST_IN_EL(k, missingDataPartIdx) = res;
- if (restoreParityParts) {
- ui64 tmp = 0;
- for (ui32 l = 0; l < DataParts; ++l) {
- tmp ^= FAST_IN_EL(k, l);
- }
- FAST_IN_M(k) = tmp;
- }
- }
- }
-
- VERBOSE_COUT_BLOCK(restoreFullData, FAST_OUT_EL_BLOCK, FAST_IN_EL, FAST_IN_M, FAST_IN_M12);
- readPosition += ColumnSize;
-#undef FAST_OUT_EL_BLOCK
-#undef FAST_IN_EL
-#undef FAST_IN_M
-#undef FAST_IN_M12
- }
- if (blockIdx == endBlockIdx) {
- break;
- }
-
- for (ui32 i = 0; i < DataParts; ++i) {
- if (i != missingDataPartIdx) {
+ }
+ return;
+ }
+
+ // s = a[(m + missingDataPartIdx - 1) % m][m + 1];
+ // for (l = 0; l < m; ++l) {
+ // s ^= a[(m + missingDataPartIdx - l - 1) % m][l];
+ // }
+ // for (k = 0; k < m - 1; ++k) {
+ // ui64 res = s;
+ // for (l = 0; l < missingDataPartIdx; ++l) {
+ // res ^= a[(m + k + missingDataPartIdx - l) % m][l];
+ // }
+ // for (l = missingDataPartIdx + 1; l < m; ++l) {
+ // res ^= a[(m + k + missingDataPartIdx - l) % m][l];
+ // }
+ // a[k][missingDataPartIdx] = res;
+ // }
+ template <bool restoreParts, bool restoreFullData, bool reversed, bool restoreParityParts>
+ void EoDiagonalRestorePartWhole(TBufferDataPart &bufferDataPart, TDataPartSet &partSet, ui64 readPosition,
+ ui32 beginBlockIdx, ui32 endBlockIdx, ui32 missingDataPartIdx) {
+ ui32 lastColumn = reversed ? DataParts + 2 : DataParts + 1;
+ const ui32 m = Prime;
+ // Use all whole columns of all the parts
+ ui64 fullDataBuffer[MAX_TOTAL_PARTS][MAX_TOTAL_PARTS - 2];
+ char partsBuffer[MAX_TOTAL_PARTS][(MAX_TOTAL_PARTS - 2) * sizeof(ui64)];
+
+ for (ui64 blockIdx = beginBlockIdx; blockIdx < endBlockIdx; ++blockIdx) {
+#define RIGHT_ROW(row) (reversed ? LineCount - 1 - (row) : (row))
+#define OUT_EL_BLOCK(row, column) fullDataBuffer[column][RIGHT_ROW(row)]
+#define IN_EL(row, column) *((ui64*)(partsBuffer[column] + RIGHT_ROW(row) * sizeof(ui64)))
+#define IN_M(row) *((ui64*)(partSet.Parts[DataParts].GetDataAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
+#define IN_M12(row) *((ui64*)(partsBuffer[lastColumn] + RIGHT_ROW(row) * sizeof(ui64)))
+
+ ui64 readContiguousSize = DataSize;
+ ui64 writeContiguousSize = DataSize;
+ if (restoreFullData) {
+ for (ui32 i = 0; i < DataParts; ++i) {
+ writeContiguousSize = std::min(writeContiguousSize,
+ bufferDataPart[i].GetContiguousSize(blockIdx * LineCount * sizeof(ui64)));
+ }
+ }
+ for (ui32 i = 0; i < DataParts; ++i) {
+ if (restoreParts || i != missingDataPartIdx) {
+ readContiguousSize = std::min(readContiguousSize,
+ partSet.Parts[i].GetContiguousSize(readPosition));
+ }
+ }
+ if (restoreParityParts) {
+ readContiguousSize = std::min(readContiguousSize,
+ partSet.Parts[DataParts].GetContiguousSize(readPosition));
+ }
+ readContiguousSize = std::min(readContiguousSize,
+ partSet.Parts[lastColumn].GetContiguousSize(readPosition));
+
+ readContiguousSize /= ColumnSize;
+ writeContiguousSize /= LineCount * sizeof(ui64);
+ ui64 contiguousSize = std::min(readContiguousSize, writeContiguousSize);
+
+ for (ui32 i = 0; i < contiguousSize && blockIdx < endBlockIdx; ++i, ++blockIdx) {
+
+#define FAST_OUT_EL_BLOCK(row, column) bufferDataPart[column].FastAt64(blockIdx * LineCount + RIGHT_ROW(row))
+#define FAST_IN_EL(row, column) *((ui64*)(partSet.Parts[column].FastDataPtrAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
+#define FAST_IN_M(row) *((ui64*)(partSet.Parts[DataParts].FastDataPtrAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
+#define FAST_IN_M12(row) *((ui64*)(partSet.Parts[lastColumn].FastDataPtrAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
+
+ VERBOSE_COUT_BLOCK(true, FAST_IN_EL, FAST_IN_EL, FAST_IN_M, FAST_IN_M12);
+ ui64 s = 0;
+ ui32 colLimit = DataParts;
+ ui32 rowLimit = LineCount;
+ {
+ ui32 idx = (m + missingDataPartIdx - 1) % m;
+ if (idx < rowLimit) {
+ s = FAST_IN_M12(idx);
+ VERBOSE_COUT("s(" << idx << ", m1): " << DebugFormatBits(s) << Endl);
+ }
+ }
+ for (ui32 l = 0; l < colLimit; ++l) {
+ ui32 idx = (m + missingDataPartIdx - l - 1) % m;
+ if (idx < LineCount) {
+ ui64 value = FAST_IN_EL(idx, l);
+ s ^= value;
+ if (restoreFullData) {
+ VERBOSE_COUT("a [" << idx << ", " << l << "] = " << DebugFormatBits(value) << Endl);
+ FAST_OUT_EL_BLOCK(idx, l) = value;
+ }
+ }
+ }
+ VERBOSE_COUT("s: " << DebugFormatBits(s) << Endl);
+ for (ui32 k = 0; k < LineCount; ++k) {
+ ui64 res = s;
+ for (ui32 l = 0; l < missingDataPartIdx; ++l) {
+ ui32 idx = (m + k + missingDataPartIdx - l) % m;
+ if (idx < LineCount) {
+ ui64 value = FAST_IN_EL(idx, l);
+ res ^= value;
+ if (restoreFullData) {
+ VERBOSE_COUT("b [" << idx << ", " << l << "] = " << DebugFormatBits(value) << Endl);
+ FAST_OUT_EL_BLOCK(idx, l) = value;
+ }
+ }
+ }
+ for (ui32 l = missingDataPartIdx + 1; l < colLimit; ++l) {
+ ui32 idx = (m + k + missingDataPartIdx - l) % m;
+ if (idx < LineCount) {
+ ui64 value = FAST_IN_EL(idx, l);
+ res ^= value;
+ if (restoreFullData) {
+ VERBOSE_COUT("c [" << idx << ", " << l << "] = " << DebugFormatBits(value) << Endl);
+ FAST_OUT_EL_BLOCK(idx, l) = value;
+ }
+ }
+ }
+ ui32 idx = (m + k + missingDataPartIdx) % m;
+ if (idx < LineCount) {
+ VERBOSE_COUT("idx = " << idx);
+ res ^= FAST_IN_M12(idx); // This is missing in the article!
+ }
+ if (restoreFullData) {
+ VERBOSE_COUT("out [" << k << ", " << missingDataPartIdx << "] = " << DebugFormatBits(res) << Endl);
+ FAST_OUT_EL_BLOCK(k, missingDataPartIdx) = res;
+ }
+ if (restoreParts) {
+ FAST_IN_EL(k, missingDataPartIdx) = res;
+ if (restoreParityParts) {
+ ui64 tmp = 0;
+ for (ui32 l = 0; l < DataParts; ++l) {
+ tmp ^= FAST_IN_EL(k, l);
+ }
+ FAST_IN_M(k) = tmp;
+ }
+ }
+ }
+
+ VERBOSE_COUT_BLOCK(restoreFullData, FAST_OUT_EL_BLOCK, FAST_IN_EL, FAST_IN_M, FAST_IN_M12);
+ readPosition += ColumnSize;
+#undef FAST_OUT_EL_BLOCK
+#undef FAST_IN_EL
+#undef FAST_IN_M
+#undef FAST_IN_M12
+ }
+ if (blockIdx == endBlockIdx) {
+ break;
+ }
+
+ for (ui32 i = 0; i < DataParts; ++i) {
+ if (i != missingDataPartIdx) {
TRopeUtils::Memcpy(partsBuffer[i], partSet.Parts[i].FastViewer.GetCurrent(readPosition), ColumnSize);
- }
- }
-
- if (restoreParts) {
+ }
+ }
+
+ if (restoreParts) {
TRopeUtils::Memcpy(partsBuffer[missingDataPartIdx],
- partSet.Parts[missingDataPartIdx].FastViewer.GetCurrent(readPosition), ColumnSize);
- }
-
+ partSet.Parts[missingDataPartIdx].FastViewer.GetCurrent(readPosition), ColumnSize);
+ }
+
TRopeUtils::Memcpy(partsBuffer[lastColumn],
- partSet.Parts[lastColumn].FastViewer.GetCurrent(readPosition), ColumnSize);
-
- VERBOSE_COUT_BLOCK(true, IN_EL, IN_EL, IN_M, IN_M12);
- ui64 s = 0;
- ui32 colLimit = DataParts;
- ui32 rowLimit = LineCount;
- {
- ui32 idx = (m + missingDataPartIdx - 1) % m;
- if (idx < rowLimit) {
- s = IN_M12(idx);
- VERBOSE_COUT("s(" << idx << ", m1): " << DebugFormatBits(s) << Endl);
- }
- }
- for (ui32 l = 0; l < colLimit; ++l) {
- ui32 idx = (m + missingDataPartIdx - l - 1) % m;
- if (idx < LineCount) {
- ui64 value = IN_EL(idx, l);
- s ^= value;
- if (restoreFullData) {
- VERBOSE_COUT("a [" << idx << ", " << l << "] = " << DebugFormatBits(value) << Endl);
- OUT_EL_BLOCK(idx, l) = value;
- }
- }
- }
- VERBOSE_COUT("s: " << DebugFormatBits(s) << Endl);
- for (ui32 k = 0; k < LineCount; ++k) {
- ui64 res = s;
- for (ui32 l = 0; l < missingDataPartIdx; ++l) {
- ui32 idx = (m + k + missingDataPartIdx - l) % m;
- if (idx < LineCount) {
- ui64 value = IN_EL(idx, l);
- res ^= value;
- if (restoreFullData) {
- VERBOSE_COUT("b [" << idx << ", " << l << "] = " << DebugFormatBits(value) << Endl);
- OUT_EL_BLOCK(idx, l) = value;
- }
- }
- }
- for (ui32 l = missingDataPartIdx + 1; l < colLimit; ++l) {
- ui32 idx = (m + k + missingDataPartIdx - l) % m;
- if (idx < LineCount) {
- ui64 value = IN_EL(idx, l);
- res ^= value;
- if (restoreFullData) {
- VERBOSE_COUT("c [" << idx << ", " << l << "] = " << DebugFormatBits(value) << Endl);
- OUT_EL_BLOCK(idx, l) = value;
- }
- }
- }
- ui32 idx = (m + k + missingDataPartIdx) % m;
- if (idx < LineCount) {
- VERBOSE_COUT("idx = " << idx);
- res ^= IN_M12(idx); // This is missing in the article!
- }
- if (restoreFullData) {
- VERBOSE_COUT("out [" << k << ", " << missingDataPartIdx << "] = " << DebugFormatBits(res) << Endl);
- OUT_EL_BLOCK(k, missingDataPartIdx) = res;
- }
- if (restoreParts) {
- IN_EL(k, missingDataPartIdx) = res;
- if (restoreParityParts) {
- ui64 tmp = 0;
- for (ui32 l = 0; l < DataParts; ++l) {
- tmp ^= IN_EL(k, l);
- }
- IN_M(k) = tmp;
- }
- }
- }
-
- VERBOSE_COUT_BLOCK(restoreFullData, OUT_EL_BLOCK, IN_EL, IN_M, IN_M12);
-
- if (restoreParts) {
+ partSet.Parts[lastColumn].FastViewer.GetCurrent(readPosition), ColumnSize);
+
+ VERBOSE_COUT_BLOCK(true, IN_EL, IN_EL, IN_M, IN_M12);
+ ui64 s = 0;
+ ui32 colLimit = DataParts;
+ ui32 rowLimit = LineCount;
+ {
+ ui32 idx = (m + missingDataPartIdx - 1) % m;
+ if (idx < rowLimit) {
+ s = IN_M12(idx);
+ VERBOSE_COUT("s(" << idx << ", m1): " << DebugFormatBits(s) << Endl);
+ }
+ }
+ for (ui32 l = 0; l < colLimit; ++l) {
+ ui32 idx = (m + missingDataPartIdx - l - 1) % m;
+ if (idx < LineCount) {
+ ui64 value = IN_EL(idx, l);
+ s ^= value;
+ if (restoreFullData) {
+ VERBOSE_COUT("a [" << idx << ", " << l << "] = " << DebugFormatBits(value) << Endl);
+ OUT_EL_BLOCK(idx, l) = value;
+ }
+ }
+ }
+ VERBOSE_COUT("s: " << DebugFormatBits(s) << Endl);
+ for (ui32 k = 0; k < LineCount; ++k) {
+ ui64 res = s;
+ for (ui32 l = 0; l < missingDataPartIdx; ++l) {
+ ui32 idx = (m + k + missingDataPartIdx - l) % m;
+ if (idx < LineCount) {
+ ui64 value = IN_EL(idx, l);
+ res ^= value;
+ if (restoreFullData) {
+ VERBOSE_COUT("b [" << idx << ", " << l << "] = " << DebugFormatBits(value) << Endl);
+ OUT_EL_BLOCK(idx, l) = value;
+ }
+ }
+ }
+ for (ui32 l = missingDataPartIdx + 1; l < colLimit; ++l) {
+ ui32 idx = (m + k + missingDataPartIdx - l) % m;
+ if (idx < LineCount) {
+ ui64 value = IN_EL(idx, l);
+ res ^= value;
+ if (restoreFullData) {
+ VERBOSE_COUT("c [" << idx << ", " << l << "] = " << DebugFormatBits(value) << Endl);
+ OUT_EL_BLOCK(idx, l) = value;
+ }
+ }
+ }
+ ui32 idx = (m + k + missingDataPartIdx) % m;
+ if (idx < LineCount) {
+ VERBOSE_COUT("idx = " << idx);
+ res ^= IN_M12(idx); // This is missing in the article!
+ }
+ if (restoreFullData) {
+ VERBOSE_COUT("out [" << k << ", " << missingDataPartIdx << "] = " << DebugFormatBits(res) << Endl);
+ OUT_EL_BLOCK(k, missingDataPartIdx) = res;
+ }
+ if (restoreParts) {
+ IN_EL(k, missingDataPartIdx) = res;
+ if (restoreParityParts) {
+ ui64 tmp = 0;
+ for (ui32 l = 0; l < DataParts; ++l) {
+ tmp ^= IN_EL(k, l);
+ }
+ IN_M(k) = tmp;
+ }
+ }
+ }
+
+ VERBOSE_COUT_BLOCK(restoreFullData, OUT_EL_BLOCK, IN_EL, IN_M, IN_M12);
+
+ if (restoreParts) {
TRopeUtils::Memcpy(partSet.Parts[missingDataPartIdx].FastViewer.GetCurrent(readPosition),
- partsBuffer[missingDataPartIdx], ColumnSize);
- }
-
- if (restoreFullData) {
- for (ui32 i = 0; i < DataParts; ++i) {
+ partsBuffer[missingDataPartIdx], ColumnSize);
+ }
+
+ if (restoreFullData) {
+ for (ui32 i = 0; i < DataParts; ++i) {
TRopeUtils::Memcpy(bufferDataPart[i].GetCurrent(blockIdx * LineCount * sizeof(ui64)),
- (const char*)fullDataBuffer[i], LineCount * sizeof(ui64));
- }
- }
-
-#undef IN_M12
-#undef IN_M
-#undef IN_EL
-#undef OUT_EL_BLOCK
-#undef RIGHT_ROW
-
- readPosition += ColumnSize;
- }
- }
-
- template <bool restoreParts, bool restoreFullData, bool reversed, bool restoreParityParts>
- void EoDiagonalRestorePart(TDataPartSet& partSet, ui32 missingDataPartIdx) {
- TRACE("Line# " << __LINE__ << " Diagonal restore: LineCount=" << LineCount << Endl);
-
- TRACE("EoDiagonalRestorePart fullSize# " << partSet.FullDataSize
- << " partSet p0 Size# " << partSet.Parts[0].Size
- << " p1 Size# " << partSet.Parts[1].Size
- << " p2 Size# " << partSet.Parts[2].Size << Endl);
- ui32 presentPartIdx = (missingDataPartIdx == 0 ? 1 : 0);
- Y_VERIFY(partSet.Parts[presentPartIdx].Offset % ColumnSize == 0);
- ui64 readPosition = partSet.Parts[presentPartIdx].Offset;
- ui64 wholeBlocks = Min(WholeBlocks - readPosition / ColumnSize, partSet.Parts[presentPartIdx].Size / ColumnSize);
-
- ui64 beginBlock = readPosition / ColumnSize;
- ui64 endBlock = beginBlock + wholeBlocks;
- TRACE("wholeBlocks# " << wholeBlocks << " blockSize# " << BlockSize << Endl);
-
- EoDiagonalRestorePartWhole<restoreParts, restoreFullData, reversed, restoreParityParts>(
- BufferDataPart, partSet, readPosition, beginBlock, endBlock, missingDataPartIdx);
-
- // Read the tail of the data
- if (TailSize && (partSet.Parts[presentPartIdx].Size + readPosition > WholeBlocks * ColumnSize)) {
- TRACE("EoDiagonalRestorePart tail" << Endl);
+ (const char*)fullDataBuffer[i], LineCount * sizeof(ui64));
+ }
+ }
+
+#undef IN_M12
+#undef IN_M
+#undef IN_EL
+#undef OUT_EL_BLOCK
+#undef RIGHT_ROW
+
+ readPosition += ColumnSize;
+ }
+ }
+
+ template <bool restoreParts, bool restoreFullData, bool reversed, bool restoreParityParts>
+ void EoDiagonalRestorePart(TDataPartSet& partSet, ui32 missingDataPartIdx) {
+ TRACE("Line# " << __LINE__ << " Diagonal restore: LineCount=" << LineCount << Endl);
+
+ TRACE("EoDiagonalRestorePart fullSize# " << partSet.FullDataSize
+ << " partSet p0 Size# " << partSet.Parts[0].Size
+ << " p1 Size# " << partSet.Parts[1].Size
+ << " p2 Size# " << partSet.Parts[2].Size << Endl);
+ ui32 presentPartIdx = (missingDataPartIdx == 0 ? 1 : 0);
+ Y_VERIFY(partSet.Parts[presentPartIdx].Offset % ColumnSize == 0);
+ ui64 readPosition = partSet.Parts[presentPartIdx].Offset;
+ ui64 wholeBlocks = Min(WholeBlocks - readPosition / ColumnSize, partSet.Parts[presentPartIdx].Size / ColumnSize);
+
+ ui64 beginBlock = readPosition / ColumnSize;
+ ui64 endBlock = beginBlock + wholeBlocks;
+ TRACE("wholeBlocks# " << wholeBlocks << " blockSize# " << BlockSize << Endl);
+
+ EoDiagonalRestorePartWhole<restoreParts, restoreFullData, reversed, restoreParityParts>(
+ BufferDataPart, partSet, readPosition, beginBlock, endBlock, missingDataPartIdx);
+
+ // Read the tail of the data
+ if (TailSize && (partSet.Parts[presentPartIdx].Size + readPosition > WholeBlocks * ColumnSize)) {
+ TRACE("EoDiagonalRestorePart tail" << Endl);
TRope lastBlock = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64));
- TBufferDataPart bufferDataPart;
- PrepareLastBlockPointers(lastBlock.Begin(), bufferDataPart);
-
- EoDiagonalRestorePartWhole<restoreParts, restoreFullData, reversed, restoreParityParts>(bufferDataPart,
- partSet, WholeBlocks * ColumnSize, 0, 1, missingDataPartIdx);
-
- if (restoreFullData) {
- PlaceLastBlock(bufferDataPart);
- }
- }
- if (restoreParts && missingDataPartIdx < partSet.Parts.size()) {
- PadAndCrcPart(partSet, missingDataPartIdx);
- }
- }
-
- template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
- void StarMainRestorePartsWholeSymmetric(TBufferDataPart &bufferDataPart, TDataPartSet& partSet,
- ui64 readPosition, ui32 endBlockIdx, ui32 missingDataPartIdxA, ui32 missingDataPartIdxB,
- ui32 missingDataPartIdxC) {
- VERBOSE_COUT("Start of StarMainRestorePartsWholeSymmetric for blocks " << missingDataPartIdxA
- << " " << missingDataPartIdxB << " " <<missingDataPartIdxC << Endl);
- // Notation used in this function is taken from article
- // Cheng Huang, Lihao Xu (2005, 4th USENIX Conf.) - STAR: An Efficient Coding Scheme...
- ui64 readPositionStart = readPosition;
- const ui32 m = Prime;
- const ui32 r = missingDataPartIdxA;
- const ui32 s = missingDataPartIdxB;
- const ui32 t = missingDataPartIdxC;
- const ui32 dr = (m + s - r) % m;
- // Use all whole columns of all the parts
-#define OUT_EL(row, column) bufferDataPart[column].At64(blockIdx * LineCount + (row))
-#define IN_EL(row, column) *((ui64*)(partSet.Parts[column].GetDataAt(readPosition + (row) * sizeof(ui64))))
-#define IN_M(row) *((ui64*)(partSet.Parts[DataParts].GetDataAt(readPosition + (row) * sizeof(ui64))))
-#define IN_M1(row) *((ui64*)(partSet.Parts[DataParts + 1].GetDataAt(readPosition + (row) * sizeof(ui64))))
-#define IN_M2(row) *((ui64*)(partSet.Parts[DataParts + 2].GetDataAt(readPosition + (row) * sizeof(ui64))))
- for (ui64 blockIdx = 0; blockIdx < endBlockIdx; ++blockIdx) {
- VERBOSE_COUT_BLOCK_M2(true, IN_EL, IN_EL, IN_M, IN_M1, IN_M2);
- // 1) Adjusters recovery adj0 is for S0
- ui64 adj0 = 0;
- ui64 adj1 = 0;
- ui64 adj2 = 0;
- for (ui32 i = 0; i < LineCount; ++i) {
- adj0 ^= IN_M(i);
- adj1 ^= IN_M1(i);
- adj2 ^= IN_M2(i);
- }
- adj1 = adj0 ^ adj1;
- adj2 = adj0 ^ adj2;
- // 2) Syndrome calculation
- ui64 s0[MAX_LINES_IN_BLOCK];
- ui64 s1[MAX_LINES_IN_BLOCK];
- ui64 s2[MAX_LINES_IN_BLOCK];
- ui32 row;
- for (ui32 i = 0; i < LineCount; ++i) {
- s0[i] = IN_M(i);
- s1[i] = IN_M1(i) ^ adj1;
- s2[i] = IN_M2(i) ^ adj2;
- VERBOSE_COUT("IN_M[" << i << "] = " << DebugFormatBits(IN_M(i)) << ", ");
- VERBOSE_COUT("IN_M1[" << i << "] ^ adj1 = " << DebugFormatBits(IN_M1(i) ^ adj1) << ", ");
- VERBOSE_COUT("IN_M2[" << i << "] ^ adj2 = " << DebugFormatBits(IN_M2(i) ^ adj2) << Endl);
- }
- s0[m - 1] = 0;
- s1[m - 1] = adj1;
- s2[m - 1] = adj2;
- for (ui32 j = 0; j < DataParts; ++j) {
- if (j == r || j == s || j == t) {
- continue;
- }
- for (ui32 i = 0; i < LineCount; ++i) {
- ui64 data_tmp = IN_EL(i, j);
- if (restoreFullData) {
- OUT_EL(i, j) = data_tmp;
- }
- s0[i] ^= data_tmp;
- row = (i + j) % m;
- if (row < m) {
- s1[row] ^= IN_EL(i, j);
- }
- row = (m + i - j) % m;
- if (row < m) {
- s2[row] ^= IN_EL(i, j);
- VERBOSE_COUT("s2[" << i << "] ^= IN_EL(" << row << "," << j << ");" << Endl;);
-
- }
- }
- }
- for (ui32 i = 0; i < m; ++i) {
- VERBOSE_COUT("s0[" << i << "] = " << DebugFormatBits(s0[i]) << ", ");
- VERBOSE_COUT("s1[" << i << "] = " << DebugFormatBits(s1[i]) << ", ");
- VERBOSE_COUT("s2[" << i << "] = " << DebugFormatBits(s2[i]) << Endl);
- }
- // 3) Compute all rows in s
- ui32 row1 = (m - 1 + r) % m;
- ui32 row2 = (m + m - 1 - 2*dr - r) % m;
- ui32 row01 = (m + row1 - r) % m;
- ui32 row02 = (row2 + r) % m;
- ui64 res = 0;
- for (ui32 i = 0; i < LineCount; ++i) {
- res = s0[row01] ^ s1[row1] ^ s0[row02] ^ s2[row2] ^ res;
- if (restoreFullData) {
- OUT_EL(row02, s) = res;
- }
- IN_EL(row02, s) = res;
- VERBOSE_COUT("IN_EL(" << row02 << ", " << s << ") = " << DebugFormatBits(res) << Endl);
- row1 = (m + row1 - 2*dr) % m;
- row2 = (m + row2 - 2*dr) % m;
- row01 = (m + row1 - r) % m;
- row02 = (row2 + r) % m;
- }
- VERBOSE_COUT_BLOCK_M2(true, IN_EL, IN_EL, IN_M, IN_M1, IN_M2);
- readPosition += ColumnSize;
- }
- VERBOSE_COUT("End of StarMainRestorePartsWholeSymmetric" << Endl);
- EoMainRestorePartsWhole<restoreParts, restoreFullData, false, restoreParityParts>(bufferDataPart,
- partSet, readPositionStart, endBlockIdx, Min(r, t), Max(r,t));
-#undef IN_M2
-#undef IN_M1
-#undef IN_M
-#undef IN_EL
-#undef OUT_EL
- }
-
- template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
- void StarRestoreHorizontalPartWhole(TBufferDataPart &bufferDataPart, TDataPartSet& partSet,
- ui64 readPosition, ui32 endBlockIdx, ui32 missingDataPartIdxA, ui32 missingDataPartIdxB) {
- VERBOSE_COUT("Start of StarRestoreHorizontalPartWhole for blocks "
- << missingDataPartIdxA << " " << missingDataPartIdxB << Endl);
- // Notation ised in this function is taken from article
- // Cheng Huang, Lihao Xu (2005, 4th USENIX Conf.) - STAR: An Efficient Coding Scheme...
- ui64 readPositionStart = readPosition;
- const ui32 m = Prime;
- const ui32 r = missingDataPartIdxA;
- const ui32 s = missingDataPartIdxB;
- const ui32 dr = (m + s - r) % m;
- // Use all whole columns of all the parts
-#define OUT_EL(row, column) bufferDataPart[column].At64(blockIdx * LineCount + (row))
-#define IN_EL(row, column) *((ui64*)(partSet.Parts[column].GetDataAt(readPosition + (row) * sizeof(ui64))))
-#define IN_M(row) *((ui64*)(partSet.Parts[DataParts].GetDataAt(readPosition + (row) * sizeof(ui64))))
-#define IN_M1(row) *((ui64*)(partSet.Parts[DataParts + 1].GetDataAt(readPosition + (row) * sizeof(ui64))))
-#define IN_M2(row) *((ui64*)(partSet.Parts[DataParts + 2].GetDataAt(readPosition + (row) * sizeof(ui64))))
- for (ui64 blockIdx = 0; blockIdx < endBlockIdx; ++blockIdx) {
- VERBOSE_COUT_BLOCK_M2(true, IN_EL, IN_EL, IN_M, IN_M1, IN_M2);
- // 1) Adjusters recovery
- ui64 adj12 = 0;
- for (ui32 i = 0; i < LineCount; ++i) {
- adj12 ^= IN_M1(i) ^ IN_M2(i);
- }
- VERBOSE_COUT("adj12# " << DebugFormatBits(adj12) << Endl);
- // 2) Syndrome calculation
- ui64 s1[MAX_LINES_IN_BLOCK];
- ui64 s2[MAX_LINES_IN_BLOCK];
- //ui32 row_adj;
- for (ui32 i = 0; i < LineCount; ++i) {
- IN_M(i) = 0;
- s1[i] = IN_M1(i);
- s2[i] = IN_M2(i);
- }
- s1[m - 1] = 0;
- s2[m - 1] = 0;
- ui32 row;
- for (ui32 j = 0; j < DataParts; ++j) {
- if (j == r || j == s) {
- continue;
- }
- for (ui32 i = 0; i < LineCount; ++i) {
- ui64 data_tmp = IN_EL(i, j);
- IN_M(i) ^= data_tmp; // Store horizontal syndrome directly in M-column
- if (restoreFullData) {
- OUT_EL(i, j) = data_tmp;
- }
- row = (i + j) % m;
- s1[row] ^= data_tmp;
- row = (m + i - j) % m;
- s2[row] ^= data_tmp;
- }
- }
- for (ui32 i = 0; i < m; ++i) {
- VERBOSE_COUT("s1[" << i << "] = " << DebugFormatBits(s1[i]) << ", ");
- VERBOSE_COUT("s2[" << i << "] = " << DebugFormatBits(s2[i]) << Endl);
- }
- // 3) Compute all row pairs
- ui32 row1 = (m - 1 + r) % m;
- ui32 row2 = (m + m - 1 - r - dr) % m;
- ui32 row3 = (row2 + r) % m;
- ui64 res = 0;
- for (ui32 i = 0; i < LineCount; ++i) {
- res = s1[row1] ^ s2[row2] ^ adj12 ^ res;
- IN_M(row3) ^= res;
- VERBOSE_COUT("IN_M(" << row3 << ") = " << DebugFormatBits(IN_M(row3)) << Endl);
- //row1 = (m + row1 - dr) % m;
- VERBOSE_COUT("row1,2,3# " << row1 << " " << row2 << " " << row3 << Endl);
- row1 = (m + row1 - dr) % m;
- row2 = (m + row2 - dr) % m;
- row3 = (m + row3 - dr) % m;
- }
- VERBOSE_COUT_BLOCK_M2(true, IN_EL, IN_EL, IN_M, IN_M1, IN_M2);
- readPosition += ColumnSize;
- }
- VERBOSE_COUT("End of StarRestoreHorizontalPartWhole" << Endl);
- EoMainRestorePartsWhole<restoreParts, restoreFullData, false, restoreParityParts>(bufferDataPart,
- partSet, readPositionStart, endBlockIdx, r, s);
-#undef IN_M2
-#undef IN_M1
-#undef IN_M
-#undef IN_EL
-#undef OUT_EL
- }
-
-
- template <bool restoreParts, bool restoreFullData, bool reversed, bool restoreParityParts>
- void EoMainRestorePartsWhole(TBufferDataPart &bufferDataPart, TDataPartSet& partSet, ui64 readPosition,
- ui32 endBlockIdx, ui32 missingDataPartIdxA, ui32 missingDataPartIdxB) {
- VERBOSE_COUT("Start of EoMainRestorePartsWhole" << Endl);
- ui32 lastColumn = reversed ? DataParts + 2 : DataParts + 1;
- const ui32 m = Prime;
- ui64 fullDataBuffer[MAX_TOTAL_PARTS][MAX_TOTAL_PARTS - 2];
- char partsBuffer[MAX_TOTAL_PARTS][(MAX_TOTAL_PARTS - 2) * sizeof(ui64)];
- // Use all whole columns of all the parts
-#define RIGHT_ROW(row) (reversed ? LineCount - 1 - (row) : (row))
-#define OUT_EL(row, column) fullDataBuffer[column][RIGHT_ROW(row)]
-#define IN_EL(row, column) *((ui64*)(partsBuffer[column] + RIGHT_ROW(row) * sizeof(ui64)))
-#define IN_EL_P(row, column) *((ui64*)(partSet.Parts[column].GetDataAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
-#define IN_M(row) *((ui64*)(partsBuffer[DataParts] + RIGHT_ROW(row) * sizeof(ui64)))
-#define IN_M12(row) *((ui64*)(partsBuffer[lastColumn] + RIGHT_ROW(row) * sizeof(ui64)))
- for (ui64 blockIdx = 0; blockIdx < endBlockIdx; ++blockIdx) {
-
- ui64 readContiguousSize = DataSize;
- ui64 writeContiguousSize = DataSize;
- if (restoreFullData) {
- for (ui32 i = 0; i < DataParts; ++i) {
- writeContiguousSize = std::min(writeContiguousSize,
- bufferDataPart[i].GetContiguousSize(blockIdx * LineCount * sizeof(ui64)));
- }
- }
- for (ui32 i = 0; i <= DataParts; ++i) {
- if (restoreParts || (i != missingDataPartIdxA && i != missingDataPartIdxB)) {
- readContiguousSize = std::min(readContiguousSize, partSet.Parts[i].GetContiguousSize(readPosition));
- }
- }
-
- readContiguousSize = std::min(readContiguousSize, partSet.Parts[lastColumn].GetContiguousSize(readPosition));
-
- readContiguousSize /= ColumnSize;
- writeContiguousSize /= LineCount * sizeof(ui64);
- ui64 contiguousSize = std::min(readContiguousSize, writeContiguousSize);
-
- for (ui32 i = 0; i < contiguousSize && blockIdx < endBlockIdx; ++i, ++blockIdx) {
-
-#define FAST_OUT_EL(row, column) bufferDataPart[column].FastAt64(blockIdx * LineCount + RIGHT_ROW(row))
-#define FAST_IN_EL(row, column) *((ui64*)(partSet.Parts[column].FastDataPtrAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
-#define FAST_IN_M(row) *((ui64*)(partSet.Parts[DataParts].FastDataPtrAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
-#define FAST_IN_M12(row) *((ui64*)(partSet.Parts[lastColumn].FastDataPtrAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
-
- VERBOSE_COUT_BLOCK(true, IN_EL, IN_EL, IN_M, IN_M12);
- // compute diagonal partiy s
- ui64 s = 0;
- ui64 s0[MAX_LINES_IN_BLOCK];
- for (ui32 l = 0; l < LineCount; ++l) {
- ui64 tmp = FAST_IN_M(l);
- s0[l] = tmp;
- s ^= tmp;
- s ^= FAST_IN_M12(l);
- VERBOSE_COUT("Diag [l,m] s:" << DebugFormatBits(s) << Endl);
- }
-
- // compute horizontal syndromes s0
- for (ui32 t = 0; t < DataParts; ++t) {
- if (t == missingDataPartIdxA || t == missingDataPartIdxB) {
- continue;
- }
- for (ui32 l = 0; l < LineCount; ++l) {
- ui64 val = FAST_IN_EL(l, t);
- s0[l] ^= val;
- if (restoreFullData) {
- FAST_OUT_EL(l, t) = val;
- }
- }
- }
-
- // compute diagonal syndromes s1
- ui64 s1[MAX_LINES_IN_BLOCK];
- for (ui32 u = 0; u < m; ++u) {
- s1[u] = s;
- VERBOSE_COUT("S1 = s = " << DebugFormatBits(s1[u]) << Endl);
- if (u < LineCount) {
- s1[u] ^= FAST_IN_M12(u);
- VERBOSE_COUT("S1 ^= a[" << u << ", m+1] = " << DebugFormatBits(s1[u]) << Endl);
- }
- for (ui32 l = 0; l < missingDataPartIdxA; ++l) {
- ui32 idx = (m + u - l) % m;
- if (idx < LineCount) {
- ui64 val = FAST_IN_EL(idx, l);
- s1[u] ^= val;
- }
- VERBOSE_COUT("S1 ^= a[" << idx << ", " << l << "] = " << DebugFormatBits(s1[u]) << Endl);
- }
- for (ui32 l = missingDataPartIdxA + 1; l < missingDataPartIdxB; ++l) {
- ui32 idx = (m + u - l) % m;
- if (idx < LineCount) {
- ui64 val = FAST_IN_EL(idx, l);
- s1[u] ^= val;
- }
- VERBOSE_COUT("S1 ^= a[" << idx << ", " << l << "] = " << DebugFormatBits(s1[u]) << Endl);
- }
- for (ui32 l = missingDataPartIdxB + 1; l < DataParts; ++l) {
- ui32 idx = (m + u - l) % m;
- if (idx < LineCount) {
- ui64 val = FAST_IN_EL(idx, l);
- s1[u] ^= val;
- }
- VERBOSE_COUT("S1 ^= a[" << idx << ", " << l << "] = " << DebugFormatBits(s1[u]) << Endl);
- }
- VERBOSE_COUT("S1[" << u << "] = " << DebugFormatBits(s1[u]) << Endl);
- }
-
- s = (m - (missingDataPartIdxB - missingDataPartIdxA) - 1) % m;
- ui64 aVal = 0;
- do {
- if (s < LineCount) {
- ui64 bVal = s1[(missingDataPartIdxB + s) % m];
- VERBOSE_COUT("bVal = s1[" << ((missingDataPartIdxB + s ) % m) << "] = " << DebugFormatBits(bVal)
- << Endl);
- ui32 bRow = (m + s + (missingDataPartIdxB - missingDataPartIdxA)) % m;
- if (bRow < LineCount) {
- VERBOSE_COUT("read [" << bRow << ", " << missingDataPartIdxA << "] = ");
- bVal ^= aVal;
- if (restoreParts) {
- VERBOSE_COUT("i " << DebugFormatBits(IN_EL(bRow, missingDataPartIdxA)) << Endl);
- } else {
- VERBOSE_COUT("o " << DebugFormatBits(OUT_EL_STRIPE(bRow,missingDataPartIdxA)) << Endl);
- }
- }
- if (restoreParts) {
- FAST_IN_EL(s, missingDataPartIdxB) = bVal;
- VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxB << "] = " << DebugFormatBits(bVal)
- << Endl);
- }
- if (restoreFullData) {
- FAST_OUT_EL(s, missingDataPartIdxB) = bVal;
- VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxB << "] = " << DebugFormatBits(bVal)
- << Endl);
- }
-
- aVal = s0[s];
- VERBOSE_COUT("aVal = s0[" << s << "] = " << DebugFormatBits(aVal) << Endl);
- VERBOSE_COUT("read [" << s << ", " << missingDataPartIdxB << "] = ");
- aVal ^= bVal;
- if (restoreParts) {
- VERBOSE_COUT("i " << DebugFormatBits(IN_EL(s,missingDataPartIdxB)) << Endl);
- } else {
- VERBOSE_COUT("o " << DebugFormatBits(OUT_EL_STRIPE(s,missingDataPartIdxB)) << Endl);
- }
-
- if (restoreParts) {
- FAST_IN_EL(s, missingDataPartIdxA) = aVal;
- VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxA << "] = " << DebugFormatBits(bVal)
- << Endl);
- }
- if (restoreFullData) {
- FAST_OUT_EL(s, missingDataPartIdxA) = aVal;
- VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxA << "] = " << DebugFormatBits(bVal)
- << Endl);
- }
- }
-
- s = (m + s - (missingDataPartIdxB - missingDataPartIdxA)) % m;
- } while (s != m - 1);
- VERBOSE_COUT_BLOCK(restoreFullData, OUT_EL, IN_EL, IN_M, IN_M12);
- readPosition += ColumnSize;
-
-#undef FAST_OUT_EL
-#undef FAST_IN_EL
-#undef FAST_IN_M
-#undef FAST_IN_M12
-
- }
-
- if (blockIdx == endBlockIdx) {
- break;
- }
-
- for (ui32 i = 0; i <= DataParts; ++i) {
- if (i != missingDataPartIdxA && i != missingDataPartIdxB) {
+ TBufferDataPart bufferDataPart;
+ PrepareLastBlockPointers(lastBlock.Begin(), bufferDataPart);
+
+ EoDiagonalRestorePartWhole<restoreParts, restoreFullData, reversed, restoreParityParts>(bufferDataPart,
+ partSet, WholeBlocks * ColumnSize, 0, 1, missingDataPartIdx);
+
+ if (restoreFullData) {
+ PlaceLastBlock(bufferDataPart);
+ }
+ }
+ if (restoreParts && missingDataPartIdx < partSet.Parts.size()) {
+ PadAndCrcPart(partSet, missingDataPartIdx);
+ }
+ }
+
+ template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
+ void StarMainRestorePartsWholeSymmetric(TBufferDataPart &bufferDataPart, TDataPartSet& partSet,
+ ui64 readPosition, ui32 endBlockIdx, ui32 missingDataPartIdxA, ui32 missingDataPartIdxB,
+ ui32 missingDataPartIdxC) {
+ VERBOSE_COUT("Start of StarMainRestorePartsWholeSymmetric for blocks " << missingDataPartIdxA
+ << " " << missingDataPartIdxB << " " <<missingDataPartIdxC << Endl);
+ // Notation used in this function is taken from article
+ // Cheng Huang, Lihao Xu (2005, 4th USENIX Conf.) - STAR: An Efficient Coding Scheme...
+ ui64 readPositionStart = readPosition;
+ const ui32 m = Prime;
+ const ui32 r = missingDataPartIdxA;
+ const ui32 s = missingDataPartIdxB;
+ const ui32 t = missingDataPartIdxC;
+ const ui32 dr = (m + s - r) % m;
+ // Use all whole columns of all the parts
+#define OUT_EL(row, column) bufferDataPart[column].At64(blockIdx * LineCount + (row))
+#define IN_EL(row, column) *((ui64*)(partSet.Parts[column].GetDataAt(readPosition + (row) * sizeof(ui64))))
+#define IN_M(row) *((ui64*)(partSet.Parts[DataParts].GetDataAt(readPosition + (row) * sizeof(ui64))))
+#define IN_M1(row) *((ui64*)(partSet.Parts[DataParts + 1].GetDataAt(readPosition + (row) * sizeof(ui64))))
+#define IN_M2(row) *((ui64*)(partSet.Parts[DataParts + 2].GetDataAt(readPosition + (row) * sizeof(ui64))))
+ for (ui64 blockIdx = 0; blockIdx < endBlockIdx; ++blockIdx) {
+ VERBOSE_COUT_BLOCK_M2(true, IN_EL, IN_EL, IN_M, IN_M1, IN_M2);
+ // 1) Adjusters recovery adj0 is for S0
+ ui64 adj0 = 0;
+ ui64 adj1 = 0;
+ ui64 adj2 = 0;
+ for (ui32 i = 0; i < LineCount; ++i) {
+ adj0 ^= IN_M(i);
+ adj1 ^= IN_M1(i);
+ adj2 ^= IN_M2(i);
+ }
+ adj1 = adj0 ^ adj1;
+ adj2 = adj0 ^ adj2;
+ // 2) Syndrome calculation
+ ui64 s0[MAX_LINES_IN_BLOCK];
+ ui64 s1[MAX_LINES_IN_BLOCK];
+ ui64 s2[MAX_LINES_IN_BLOCK];
+ ui32 row;
+ for (ui32 i = 0; i < LineCount; ++i) {
+ s0[i] = IN_M(i);
+ s1[i] = IN_M1(i) ^ adj1;
+ s2[i] = IN_M2(i) ^ adj2;
+ VERBOSE_COUT("IN_M[" << i << "] = " << DebugFormatBits(IN_M(i)) << ", ");
+ VERBOSE_COUT("IN_M1[" << i << "] ^ adj1 = " << DebugFormatBits(IN_M1(i) ^ adj1) << ", ");
+ VERBOSE_COUT("IN_M2[" << i << "] ^ adj2 = " << DebugFormatBits(IN_M2(i) ^ adj2) << Endl);
+ }
+ s0[m - 1] = 0;
+ s1[m - 1] = adj1;
+ s2[m - 1] = adj2;
+ for (ui32 j = 0; j < DataParts; ++j) {
+ if (j == r || j == s || j == t) {
+ continue;
+ }
+ for (ui32 i = 0; i < LineCount; ++i) {
+ ui64 data_tmp = IN_EL(i, j);
+ if (restoreFullData) {
+ OUT_EL(i, j) = data_tmp;
+ }
+ s0[i] ^= data_tmp;
+ row = (i + j) % m;
+ if (row < m) {
+ s1[row] ^= IN_EL(i, j);
+ }
+ row = (m + i - j) % m;
+ if (row < m) {
+ s2[row] ^= IN_EL(i, j);
+ VERBOSE_COUT("s2[" << i << "] ^= IN_EL(" << row << "," << j << ");" << Endl;);
+
+ }
+ }
+ }
+ for (ui32 i = 0; i < m; ++i) {
+ VERBOSE_COUT("s0[" << i << "] = " << DebugFormatBits(s0[i]) << ", ");
+ VERBOSE_COUT("s1[" << i << "] = " << DebugFormatBits(s1[i]) << ", ");
+ VERBOSE_COUT("s2[" << i << "] = " << DebugFormatBits(s2[i]) << Endl);
+ }
+ // 3) Compute all rows in s
+ ui32 row1 = (m - 1 + r) % m;
+ ui32 row2 = (m + m - 1 - 2*dr - r) % m;
+ ui32 row01 = (m + row1 - r) % m;
+ ui32 row02 = (row2 + r) % m;
+ ui64 res = 0;
+ for (ui32 i = 0; i < LineCount; ++i) {
+ res = s0[row01] ^ s1[row1] ^ s0[row02] ^ s2[row2] ^ res;
+ if (restoreFullData) {
+ OUT_EL(row02, s) = res;
+ }
+ IN_EL(row02, s) = res;
+ VERBOSE_COUT("IN_EL(" << row02 << ", " << s << ") = " << DebugFormatBits(res) << Endl);
+ row1 = (m + row1 - 2*dr) % m;
+ row2 = (m + row2 - 2*dr) % m;
+ row01 = (m + row1 - r) % m;
+ row02 = (row2 + r) % m;
+ }
+ VERBOSE_COUT_BLOCK_M2(true, IN_EL, IN_EL, IN_M, IN_M1, IN_M2);
+ readPosition += ColumnSize;
+ }
+ VERBOSE_COUT("End of StarMainRestorePartsWholeSymmetric" << Endl);
+ EoMainRestorePartsWhole<restoreParts, restoreFullData, false, restoreParityParts>(bufferDataPart,
+ partSet, readPositionStart, endBlockIdx, Min(r, t), Max(r,t));
+#undef IN_M2
+#undef IN_M1
+#undef IN_M
+#undef IN_EL
+#undef OUT_EL
+ }
+
+ template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
+ void StarRestoreHorizontalPartWhole(TBufferDataPart &bufferDataPart, TDataPartSet& partSet,
+ ui64 readPosition, ui32 endBlockIdx, ui32 missingDataPartIdxA, ui32 missingDataPartIdxB) {
+ VERBOSE_COUT("Start of StarRestoreHorizontalPartWhole for blocks "
+ << missingDataPartIdxA << " " << missingDataPartIdxB << Endl);
+ // Notation ised in this function is taken from article
+ // Cheng Huang, Lihao Xu (2005, 4th USENIX Conf.) - STAR: An Efficient Coding Scheme...
+ ui64 readPositionStart = readPosition;
+ const ui32 m = Prime;
+ const ui32 r = missingDataPartIdxA;
+ const ui32 s = missingDataPartIdxB;
+ const ui32 dr = (m + s - r) % m;
+ // Use all whole columns of all the parts
+#define OUT_EL(row, column) bufferDataPart[column].At64(blockIdx * LineCount + (row))
+#define IN_EL(row, column) *((ui64*)(partSet.Parts[column].GetDataAt(readPosition + (row) * sizeof(ui64))))
+#define IN_M(row) *((ui64*)(partSet.Parts[DataParts].GetDataAt(readPosition + (row) * sizeof(ui64))))
+#define IN_M1(row) *((ui64*)(partSet.Parts[DataParts + 1].GetDataAt(readPosition + (row) * sizeof(ui64))))
+#define IN_M2(row) *((ui64*)(partSet.Parts[DataParts + 2].GetDataAt(readPosition + (row) * sizeof(ui64))))
+ for (ui64 blockIdx = 0; blockIdx < endBlockIdx; ++blockIdx) {
+ VERBOSE_COUT_BLOCK_M2(true, IN_EL, IN_EL, IN_M, IN_M1, IN_M2);
+ // 1) Adjusters recovery
+ ui64 adj12 = 0;
+ for (ui32 i = 0; i < LineCount; ++i) {
+ adj12 ^= IN_M1(i) ^ IN_M2(i);
+ }
+ VERBOSE_COUT("adj12# " << DebugFormatBits(adj12) << Endl);
+ // 2) Syndrome calculation
+ ui64 s1[MAX_LINES_IN_BLOCK];
+ ui64 s2[MAX_LINES_IN_BLOCK];
+ //ui32 row_adj;
+ for (ui32 i = 0; i < LineCount; ++i) {
+ IN_M(i) = 0;
+ s1[i] = IN_M1(i);
+ s2[i] = IN_M2(i);
+ }
+ s1[m - 1] = 0;
+ s2[m - 1] = 0;
+ ui32 row;
+ for (ui32 j = 0; j < DataParts; ++j) {
+ if (j == r || j == s) {
+ continue;
+ }
+ for (ui32 i = 0; i < LineCount; ++i) {
+ ui64 data_tmp = IN_EL(i, j);
+ IN_M(i) ^= data_tmp; // Store horizontal syndrome directly in M-column
+ if (restoreFullData) {
+ OUT_EL(i, j) = data_tmp;
+ }
+ row = (i + j) % m;
+ s1[row] ^= data_tmp;
+ row = (m + i - j) % m;
+ s2[row] ^= data_tmp;
+ }
+ }
+ for (ui32 i = 0; i < m; ++i) {
+ VERBOSE_COUT("s1[" << i << "] = " << DebugFormatBits(s1[i]) << ", ");
+ VERBOSE_COUT("s2[" << i << "] = " << DebugFormatBits(s2[i]) << Endl);
+ }
+ // 3) Compute all row pairs
+ ui32 row1 = (m - 1 + r) % m;
+ ui32 row2 = (m + m - 1 - r - dr) % m;
+ ui32 row3 = (row2 + r) % m;
+ ui64 res = 0;
+ for (ui32 i = 0; i < LineCount; ++i) {
+ res = s1[row1] ^ s2[row2] ^ adj12 ^ res;
+ IN_M(row3) ^= res;
+ VERBOSE_COUT("IN_M(" << row3 << ") = " << DebugFormatBits(IN_M(row3)) << Endl);
+ //row1 = (m + row1 - dr) % m;
+ VERBOSE_COUT("row1,2,3# " << row1 << " " << row2 << " " << row3 << Endl);
+ row1 = (m + row1 - dr) % m;
+ row2 = (m + row2 - dr) % m;
+ row3 = (m + row3 - dr) % m;
+ }
+ VERBOSE_COUT_BLOCK_M2(true, IN_EL, IN_EL, IN_M, IN_M1, IN_M2);
+ readPosition += ColumnSize;
+ }
+ VERBOSE_COUT("End of StarRestoreHorizontalPartWhole" << Endl);
+ EoMainRestorePartsWhole<restoreParts, restoreFullData, false, restoreParityParts>(bufferDataPart,
+ partSet, readPositionStart, endBlockIdx, r, s);
+#undef IN_M2
+#undef IN_M1
+#undef IN_M
+#undef IN_EL
+#undef OUT_EL
+ }
+
+
+ template <bool restoreParts, bool restoreFullData, bool reversed, bool restoreParityParts>
+ void EoMainRestorePartsWhole(TBufferDataPart &bufferDataPart, TDataPartSet& partSet, ui64 readPosition,
+ ui32 endBlockIdx, ui32 missingDataPartIdxA, ui32 missingDataPartIdxB) {
+ VERBOSE_COUT("Start of EoMainRestorePartsWhole" << Endl);
+ ui32 lastColumn = reversed ? DataParts + 2 : DataParts + 1;
+ const ui32 m = Prime;
+ ui64 fullDataBuffer[MAX_TOTAL_PARTS][MAX_TOTAL_PARTS - 2];
+ char partsBuffer[MAX_TOTAL_PARTS][(MAX_TOTAL_PARTS - 2) * sizeof(ui64)];
+ // Use all whole columns of all the parts
+#define RIGHT_ROW(row) (reversed ? LineCount - 1 - (row) : (row))
+#define OUT_EL(row, column) fullDataBuffer[column][RIGHT_ROW(row)]
+#define IN_EL(row, column) *((ui64*)(partsBuffer[column] + RIGHT_ROW(row) * sizeof(ui64)))
+#define IN_EL_P(row, column) *((ui64*)(partSet.Parts[column].GetDataAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
+#define IN_M(row) *((ui64*)(partsBuffer[DataParts] + RIGHT_ROW(row) * sizeof(ui64)))
+#define IN_M12(row) *((ui64*)(partsBuffer[lastColumn] + RIGHT_ROW(row) * sizeof(ui64)))
+ for (ui64 blockIdx = 0; blockIdx < endBlockIdx; ++blockIdx) {
+
+ ui64 readContiguousSize = DataSize;
+ ui64 writeContiguousSize = DataSize;
+ if (restoreFullData) {
+ for (ui32 i = 0; i < DataParts; ++i) {
+ writeContiguousSize = std::min(writeContiguousSize,
+ bufferDataPart[i].GetContiguousSize(blockIdx * LineCount * sizeof(ui64)));
+ }
+ }
+ for (ui32 i = 0; i <= DataParts; ++i) {
+ if (restoreParts || (i != missingDataPartIdxA && i != missingDataPartIdxB)) {
+ readContiguousSize = std::min(readContiguousSize, partSet.Parts[i].GetContiguousSize(readPosition));
+ }
+ }
+
+ readContiguousSize = std::min(readContiguousSize, partSet.Parts[lastColumn].GetContiguousSize(readPosition));
+
+ readContiguousSize /= ColumnSize;
+ writeContiguousSize /= LineCount * sizeof(ui64);
+ ui64 contiguousSize = std::min(readContiguousSize, writeContiguousSize);
+
+ for (ui32 i = 0; i < contiguousSize && blockIdx < endBlockIdx; ++i, ++blockIdx) {
+
+#define FAST_OUT_EL(row, column) bufferDataPart[column].FastAt64(blockIdx * LineCount + RIGHT_ROW(row))
+#define FAST_IN_EL(row, column) *((ui64*)(partSet.Parts[column].FastDataPtrAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
+#define FAST_IN_M(row) *((ui64*)(partSet.Parts[DataParts].FastDataPtrAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
+#define FAST_IN_M12(row) *((ui64*)(partSet.Parts[lastColumn].FastDataPtrAt(readPosition + RIGHT_ROW(row) * sizeof(ui64))))
+
+ VERBOSE_COUT_BLOCK(true, IN_EL, IN_EL, IN_M, IN_M12);
+ // compute diagonal partiy s
+ ui64 s = 0;
+ ui64 s0[MAX_LINES_IN_BLOCK];
+ for (ui32 l = 0; l < LineCount; ++l) {
+ ui64 tmp = FAST_IN_M(l);
+ s0[l] = tmp;
+ s ^= tmp;
+ s ^= FAST_IN_M12(l);
+ VERBOSE_COUT("Diag [l,m] s:" << DebugFormatBits(s) << Endl);
+ }
+
+ // compute horizontal syndromes s0
+ for (ui32 t = 0; t < DataParts; ++t) {
+ if (t == missingDataPartIdxA || t == missingDataPartIdxB) {
+ continue;
+ }
+ for (ui32 l = 0; l < LineCount; ++l) {
+ ui64 val = FAST_IN_EL(l, t);
+ s0[l] ^= val;
+ if (restoreFullData) {
+ FAST_OUT_EL(l, t) = val;
+ }
+ }
+ }
+
+ // compute diagonal syndromes s1
+ ui64 s1[MAX_LINES_IN_BLOCK];
+ for (ui32 u = 0; u < m; ++u) {
+ s1[u] = s;
+ VERBOSE_COUT("S1 = s = " << DebugFormatBits(s1[u]) << Endl);
+ if (u < LineCount) {
+ s1[u] ^= FAST_IN_M12(u);
+ VERBOSE_COUT("S1 ^= a[" << u << ", m+1] = " << DebugFormatBits(s1[u]) << Endl);
+ }
+ for (ui32 l = 0; l < missingDataPartIdxA; ++l) {
+ ui32 idx = (m + u - l) % m;
+ if (idx < LineCount) {
+ ui64 val = FAST_IN_EL(idx, l);
+ s1[u] ^= val;
+ }
+ VERBOSE_COUT("S1 ^= a[" << idx << ", " << l << "] = " << DebugFormatBits(s1[u]) << Endl);
+ }
+ for (ui32 l = missingDataPartIdxA + 1; l < missingDataPartIdxB; ++l) {
+ ui32 idx = (m + u - l) % m;
+ if (idx < LineCount) {
+ ui64 val = FAST_IN_EL(idx, l);
+ s1[u] ^= val;
+ }
+ VERBOSE_COUT("S1 ^= a[" << idx << ", " << l << "] = " << DebugFormatBits(s1[u]) << Endl);
+ }
+ for (ui32 l = missingDataPartIdxB + 1; l < DataParts; ++l) {
+ ui32 idx = (m + u - l) % m;
+ if (idx < LineCount) {
+ ui64 val = FAST_IN_EL(idx, l);
+ s1[u] ^= val;
+ }
+ VERBOSE_COUT("S1 ^= a[" << idx << ", " << l << "] = " << DebugFormatBits(s1[u]) << Endl);
+ }
+ VERBOSE_COUT("S1[" << u << "] = " << DebugFormatBits(s1[u]) << Endl);
+ }
+
+ s = (m - (missingDataPartIdxB - missingDataPartIdxA) - 1) % m;
+ ui64 aVal = 0;
+ do {
+ if (s < LineCount) {
+ ui64 bVal = s1[(missingDataPartIdxB + s) % m];
+ VERBOSE_COUT("bVal = s1[" << ((missingDataPartIdxB + s ) % m) << "] = " << DebugFormatBits(bVal)
+ << Endl);
+ ui32 bRow = (m + s + (missingDataPartIdxB - missingDataPartIdxA)) % m;
+ if (bRow < LineCount) {
+ VERBOSE_COUT("read [" << bRow << ", " << missingDataPartIdxA << "] = ");
+ bVal ^= aVal;
+ if (restoreParts) {
+ VERBOSE_COUT("i " << DebugFormatBits(IN_EL(bRow, missingDataPartIdxA)) << Endl);
+ } else {
+ VERBOSE_COUT("o " << DebugFormatBits(OUT_EL_STRIPE(bRow,missingDataPartIdxA)) << Endl);
+ }
+ }
+ if (restoreParts) {
+ FAST_IN_EL(s, missingDataPartIdxB) = bVal;
+ VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxB << "] = " << DebugFormatBits(bVal)
+ << Endl);
+ }
+ if (restoreFullData) {
+ FAST_OUT_EL(s, missingDataPartIdxB) = bVal;
+ VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxB << "] = " << DebugFormatBits(bVal)
+ << Endl);
+ }
+
+ aVal = s0[s];
+ VERBOSE_COUT("aVal = s0[" << s << "] = " << DebugFormatBits(aVal) << Endl);
+ VERBOSE_COUT("read [" << s << ", " << missingDataPartIdxB << "] = ");
+ aVal ^= bVal;
+ if (restoreParts) {
+ VERBOSE_COUT("i " << DebugFormatBits(IN_EL(s,missingDataPartIdxB)) << Endl);
+ } else {
+ VERBOSE_COUT("o " << DebugFormatBits(OUT_EL_STRIPE(s,missingDataPartIdxB)) << Endl);
+ }
+
+ if (restoreParts) {
+ FAST_IN_EL(s, missingDataPartIdxA) = aVal;
+ VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxA << "] = " << DebugFormatBits(bVal)
+ << Endl);
+ }
+ if (restoreFullData) {
+ FAST_OUT_EL(s, missingDataPartIdxA) = aVal;
+ VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxA << "] = " << DebugFormatBits(bVal)
+ << Endl);
+ }
+ }
+
+ s = (m + s - (missingDataPartIdxB - missingDataPartIdxA)) % m;
+ } while (s != m - 1);
+ VERBOSE_COUT_BLOCK(restoreFullData, OUT_EL, IN_EL, IN_M, IN_M12);
+ readPosition += ColumnSize;
+
+#undef FAST_OUT_EL
+#undef FAST_IN_EL
+#undef FAST_IN_M
+#undef FAST_IN_M12
+
+ }
+
+ if (blockIdx == endBlockIdx) {
+ break;
+ }
+
+ for (ui32 i = 0; i <= DataParts; ++i) {
+ if (i != missingDataPartIdxA && i != missingDataPartIdxB) {
TRopeUtils::Memcpy(partsBuffer[i],
- partSet.Parts[i].FastViewer.GetCurrent(readPosition), ColumnSize);
- }
- }
-
+ partSet.Parts[i].FastViewer.GetCurrent(readPosition), ColumnSize);
+ }
+ }
+
TRopeUtils::Memcpy(partsBuffer[lastColumn],
- partSet.Parts[lastColumn].FastViewer.GetCurrent(readPosition), ColumnSize);
-
- VERBOSE_COUT_BLOCK(true, IN_EL, IN_EL, IN_M, IN_M12);
- // compute diagonal partiy s
- ui64 s = 0;
- ui64 s0[MAX_LINES_IN_BLOCK];
- for (ui32 l = 0; l < LineCount; ++l) {
- ui64 tmp = IN_M(l);
- s0[l] = tmp;
- s ^= tmp;
- s ^= IN_M12(l);
- VERBOSE_COUT("Diag [l,m] s:" << DebugFormatBits(s) << Endl);
- }
-
- // compute horizontal syndromes s0
- for (ui32 t = 0; t < DataParts; ++t) {
- if (t == missingDataPartIdxA || t == missingDataPartIdxB) {
- continue;
- }
- for (ui32 l = 0; l < LineCount; ++l) {
- ui64 val = IN_EL(l, t);
- s0[l] ^= val;
- if (restoreFullData) {
- OUT_EL(l, t) = val;
- }
- }
- }
-
- // compute diagonal syndromes s1
- ui64 s1[MAX_LINES_IN_BLOCK];
- for (ui32 u = 0; u < m; ++u) {
- s1[u] = s;
- VERBOSE_COUT("S1 = s = " << DebugFormatBits(s1[u]) << Endl);
- if (u < LineCount) {
- s1[u] ^= IN_M12(u);
- VERBOSE_COUT("S1 ^= a[" << u << ", m+1] = " << DebugFormatBits(s1[u]) << Endl);
- }
- for (ui32 l = 0; l < missingDataPartIdxA; ++l) {
- ui32 idx = (m + u - l) % m;
- if (idx < LineCount) {
- ui64 val = IN_EL(idx, l);
- s1[u] ^= val;
- }
- VERBOSE_COUT("S1 ^= a[" << idx << ", " << l << "] = " << DebugFormatBits(s1[u]) << Endl);
- }
- for (ui32 l = missingDataPartIdxA + 1; l < missingDataPartIdxB; ++l) {
- ui32 idx = (m + u - l) % m;
- if (idx < LineCount) {
- ui64 val = IN_EL(idx, l);
- s1[u] ^= val;
- }
- VERBOSE_COUT("S1 ^= a[" << idx << ", " << l << "] = " << DebugFormatBits(s1[u]) << Endl);
- }
- for (ui32 l = missingDataPartIdxB + 1; l < DataParts; ++l) {
- ui32 idx = (m + u - l) % m;
- if (idx < LineCount) {
- ui64 val = IN_EL(idx, l);
- s1[u] ^= val;
- }
- VERBOSE_COUT("S1 ^= a[" << idx << ", " << l << "] = " << DebugFormatBits(s1[u]) << Endl);
- }
- VERBOSE_COUT("S1[" << u << "] = " << DebugFormatBits(s1[u]) << Endl);
- }
-
- s = (m - (missingDataPartIdxB - missingDataPartIdxA) - 1) % m;
- ui64 aVal = 0;
- do {
- if (s < LineCount) {
- ui64 bVal = s1[(missingDataPartIdxB + s) % m];
- VERBOSE_COUT("bVal = s1[" << ((missingDataPartIdxB + s ) % m) << "] = " << DebugFormatBits(bVal)
- << Endl);
- ui32 bRow = (m + s + (missingDataPartIdxB - missingDataPartIdxA)) % m;
- if (bRow < LineCount) {
- VERBOSE_COUT("read [" << bRow << ", " << missingDataPartIdxA << "] = ");
- bVal ^= aVal;
- if (restoreParts) {
- VERBOSE_COUT("i " << DebugFormatBits(IN_EL(bRow, missingDataPartIdxA)) << Endl);
- } else {
- VERBOSE_COUT("o " << DebugFormatBits(OUT_EL_STRIPE(bRow,missingDataPartIdxA)) << Endl);
- }
- }
- if (restoreParts) {
- IN_EL_P(s, missingDataPartIdxB) = bVal;
- VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxB << "] = " << DebugFormatBits(bVal)
- << Endl);
- }
- if (restoreFullData) {
- OUT_EL(s, missingDataPartIdxB) = bVal;
- VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxB << "] = " << DebugFormatBits(bVal)
- << Endl);
- }
-
- aVal = s0[s];
- VERBOSE_COUT("aVal = s0[" << s << "] = " << DebugFormatBits(aVal) << Endl);
- VERBOSE_COUT("read [" << s << ", " << missingDataPartIdxB << "] = ");
- aVal ^= bVal;
- if (restoreParts) {
- VERBOSE_COUT("i " << DebugFormatBits(IN_EL(s,missingDataPartIdxB)) << Endl);
- } else {
- VERBOSE_COUT("o " << DebugFormatBits(OUT_EL_STRIPE(s,missingDataPartIdxB)) << Endl);
- }
-
- if (restoreParts) {
- IN_EL_P(s, missingDataPartIdxA) = aVal;
- VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxA << "] = " << DebugFormatBits(bVal)
- << Endl);
- }
- if (restoreFullData) {
- OUT_EL(s, missingDataPartIdxA) = aVal;
- VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxA << "] = " << DebugFormatBits(bVal)
- << Endl);
- }
- }
-
- s = (m + s - (missingDataPartIdxB - missingDataPartIdxA)) % m;
- } while (s != m - 1);
- VERBOSE_COUT_BLOCK(restoreFullData, OUT_EL, IN_EL, IN_M, IN_M12);
-#undef IN_M12
-#undef IN_M
-#undef IN_EL
-#undef IN_EL_P
-
- if (restoreFullData) {
- for (ui32 i = 0; i < DataParts; ++i) {
+ partSet.Parts[lastColumn].FastViewer.GetCurrent(readPosition), ColumnSize);
+
+ VERBOSE_COUT_BLOCK(true, IN_EL, IN_EL, IN_M, IN_M12);
+ // compute diagonal partiy s
+ ui64 s = 0;
+ ui64 s0[MAX_LINES_IN_BLOCK];
+ for (ui32 l = 0; l < LineCount; ++l) {
+ ui64 tmp = IN_M(l);
+ s0[l] = tmp;
+ s ^= tmp;
+ s ^= IN_M12(l);
+ VERBOSE_COUT("Diag [l,m] s:" << DebugFormatBits(s) << Endl);
+ }
+
+ // compute horizontal syndromes s0
+ for (ui32 t = 0; t < DataParts; ++t) {
+ if (t == missingDataPartIdxA || t == missingDataPartIdxB) {
+ continue;
+ }
+ for (ui32 l = 0; l < LineCount; ++l) {
+ ui64 val = IN_EL(l, t);
+ s0[l] ^= val;
+ if (restoreFullData) {
+ OUT_EL(l, t) = val;
+ }
+ }
+ }
+
+ // compute diagonal syndromes s1
+ ui64 s1[MAX_LINES_IN_BLOCK];
+ for (ui32 u = 0; u < m; ++u) {
+ s1[u] = s;
+ VERBOSE_COUT("S1 = s = " << DebugFormatBits(s1[u]) << Endl);
+ if (u < LineCount) {
+ s1[u] ^= IN_M12(u);
+ VERBOSE_COUT("S1 ^= a[" << u << ", m+1] = " << DebugFormatBits(s1[u]) << Endl);
+ }
+ for (ui32 l = 0; l < missingDataPartIdxA; ++l) {
+ ui32 idx = (m + u - l) % m;
+ if (idx < LineCount) {
+ ui64 val = IN_EL(idx, l);
+ s1[u] ^= val;
+ }
+ VERBOSE_COUT("S1 ^= a[" << idx << ", " << l << "] = " << DebugFormatBits(s1[u]) << Endl);
+ }
+ for (ui32 l = missingDataPartIdxA + 1; l < missingDataPartIdxB; ++l) {
+ ui32 idx = (m + u - l) % m;
+ if (idx < LineCount) {
+ ui64 val = IN_EL(idx, l);
+ s1[u] ^= val;
+ }
+ VERBOSE_COUT("S1 ^= a[" << idx << ", " << l << "] = " << DebugFormatBits(s1[u]) << Endl);
+ }
+ for (ui32 l = missingDataPartIdxB + 1; l < DataParts; ++l) {
+ ui32 idx = (m + u - l) % m;
+ if (idx < LineCount) {
+ ui64 val = IN_EL(idx, l);
+ s1[u] ^= val;
+ }
+ VERBOSE_COUT("S1 ^= a[" << idx << ", " << l << "] = " << DebugFormatBits(s1[u]) << Endl);
+ }
+ VERBOSE_COUT("S1[" << u << "] = " << DebugFormatBits(s1[u]) << Endl);
+ }
+
+ s = (m - (missingDataPartIdxB - missingDataPartIdxA) - 1) % m;
+ ui64 aVal = 0;
+ do {
+ if (s < LineCount) {
+ ui64 bVal = s1[(missingDataPartIdxB + s) % m];
+ VERBOSE_COUT("bVal = s1[" << ((missingDataPartIdxB + s ) % m) << "] = " << DebugFormatBits(bVal)
+ << Endl);
+ ui32 bRow = (m + s + (missingDataPartIdxB - missingDataPartIdxA)) % m;
+ if (bRow < LineCount) {
+ VERBOSE_COUT("read [" << bRow << ", " << missingDataPartIdxA << "] = ");
+ bVal ^= aVal;
+ if (restoreParts) {
+ VERBOSE_COUT("i " << DebugFormatBits(IN_EL(bRow, missingDataPartIdxA)) << Endl);
+ } else {
+ VERBOSE_COUT("o " << DebugFormatBits(OUT_EL_STRIPE(bRow,missingDataPartIdxA)) << Endl);
+ }
+ }
+ if (restoreParts) {
+ IN_EL_P(s, missingDataPartIdxB) = bVal;
+ VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxB << "] = " << DebugFormatBits(bVal)
+ << Endl);
+ }
+ if (restoreFullData) {
+ OUT_EL(s, missingDataPartIdxB) = bVal;
+ VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxB << "] = " << DebugFormatBits(bVal)
+ << Endl);
+ }
+
+ aVal = s0[s];
+ VERBOSE_COUT("aVal = s0[" << s << "] = " << DebugFormatBits(aVal) << Endl);
+ VERBOSE_COUT("read [" << s << ", " << missingDataPartIdxB << "] = ");
+ aVal ^= bVal;
+ if (restoreParts) {
+ VERBOSE_COUT("i " << DebugFormatBits(IN_EL(s,missingDataPartIdxB)) << Endl);
+ } else {
+ VERBOSE_COUT("o " << DebugFormatBits(OUT_EL_STRIPE(s,missingDataPartIdxB)) << Endl);
+ }
+
+ if (restoreParts) {
+ IN_EL_P(s, missingDataPartIdxA) = aVal;
+ VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxA << "] = " << DebugFormatBits(bVal)
+ << Endl);
+ }
+ if (restoreFullData) {
+ OUT_EL(s, missingDataPartIdxA) = aVal;
+ VERBOSE_COUT("write [" << s << ", " << missingDataPartIdxA << "] = " << DebugFormatBits(bVal)
+ << Endl);
+ }
+ }
+
+ s = (m + s - (missingDataPartIdxB - missingDataPartIdxA)) % m;
+ } while (s != m - 1);
+ VERBOSE_COUT_BLOCK(restoreFullData, OUT_EL, IN_EL, IN_M, IN_M12);
+#undef IN_M12
+#undef IN_M
+#undef IN_EL
+#undef IN_EL_P
+
+ if (restoreFullData) {
+ for (ui32 i = 0; i < DataParts; ++i) {
TRopeUtils::Memcpy(bufferDataPart[i].GetCurrent(blockIdx * LineCount * sizeof(ui64)),
- (const char *) fullDataBuffer[i], LineCount * sizeof(ui64));
- }
- }
-
- readPosition += ColumnSize;
- }
- }
-
- template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
- void StarRestoreHorizontalPart(TDataPartSet& partSet, ui32 missingDataPartIdxA,
- ui32 missingDataPartIdxB) {
- // Read data and parity
- VERBOSE_COUT("StarRestoreHorizontalPart for " << missingDataPartIdxA << " " << missingDataPartIdxB << Endl);
- StarRestoreHorizontalPartWhole<restoreParts, restoreFullData, restoreParityParts>(BufferDataPart,
- partSet, 0ull, WholeBlocks, missingDataPartIdxA, missingDataPartIdxB);
-
- if (TailSize) {
+ (const char *) fullDataBuffer[i], LineCount * sizeof(ui64));
+ }
+ }
+
+ readPosition += ColumnSize;
+ }
+ }
+
+ template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
+ void StarRestoreHorizontalPart(TDataPartSet& partSet, ui32 missingDataPartIdxA,
+ ui32 missingDataPartIdxB) {
+ // Read data and parity
+ VERBOSE_COUT("StarRestoreHorizontalPart for " << missingDataPartIdxA << " " << missingDataPartIdxB << Endl);
+ StarRestoreHorizontalPartWhole<restoreParts, restoreFullData, restoreParityParts>(BufferDataPart,
+ partSet, 0ull, WholeBlocks, missingDataPartIdxA, missingDataPartIdxB);
+
+ if (TailSize) {
TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64));
- TBufferDataPart bufferDataPart;
- PrepareLastBlockPointers(lastBlockSource.Begin(), bufferDataPart);
-
- StarRestoreHorizontalPartWhole<restoreParts, restoreFullData, restoreParityParts>(
- bufferDataPart, partSet, WholeBlocks * ColumnSize, 1, missingDataPartIdxA,
- missingDataPartIdxB);
-
- if (restoreFullData) {
- PlaceLastBlock(bufferDataPart);
- }
- }
- if (restoreParts) {
- if (missingDataPartIdxA < partSet.Parts.size()) {
- PadAndCrcPart(partSet, missingDataPartIdxA);
- }
- if (missingDataPartIdxB < partSet.Parts.size()) {
- PadAndCrcPart(partSet, missingDataPartIdxB);
- }
- }
- }
-
-
- template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
- void StarMainRestorePartsSymmetric(TDataPartSet& partSet, ui32 missingDataPartIdxA,
- ui32 missingDataPartIdxB, ui32 missingDataPartIdxC) {
- // Read data and parity
- VERBOSE_COUT("StarMainRestorePartsSymmetric" << Endl);
- StarMainRestorePartsWholeSymmetric<restoreParts, restoreFullData, restoreParityParts>(BufferDataPart,
- partSet, 0ull, WholeBlocks, missingDataPartIdxA, missingDataPartIdxB, missingDataPartIdxC);
-
- if (TailSize) {
+ TBufferDataPart bufferDataPart;
+ PrepareLastBlockPointers(lastBlockSource.Begin(), bufferDataPart);
+
+ StarRestoreHorizontalPartWhole<restoreParts, restoreFullData, restoreParityParts>(
+ bufferDataPart, partSet, WholeBlocks * ColumnSize, 1, missingDataPartIdxA,
+ missingDataPartIdxB);
+
+ if (restoreFullData) {
+ PlaceLastBlock(bufferDataPart);
+ }
+ }
+ if (restoreParts) {
+ if (missingDataPartIdxA < partSet.Parts.size()) {
+ PadAndCrcPart(partSet, missingDataPartIdxA);
+ }
+ if (missingDataPartIdxB < partSet.Parts.size()) {
+ PadAndCrcPart(partSet, missingDataPartIdxB);
+ }
+ }
+ }
+
+
+ template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
+ void StarMainRestorePartsSymmetric(TDataPartSet& partSet, ui32 missingDataPartIdxA,
+ ui32 missingDataPartIdxB, ui32 missingDataPartIdxC) {
+ // Read data and parity
+ VERBOSE_COUT("StarMainRestorePartsSymmetric" << Endl);
+ StarMainRestorePartsWholeSymmetric<restoreParts, restoreFullData, restoreParityParts>(BufferDataPart,
+ partSet, 0ull, WholeBlocks, missingDataPartIdxA, missingDataPartIdxB, missingDataPartIdxC);
+
+ if (TailSize) {
TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64));
- TBufferDataPart bufferDataPart;
- PrepareLastBlockPointers(lastBlockSource.Begin(), bufferDataPart);
-
- StarMainRestorePartsWholeSymmetric<restoreParts, restoreFullData, restoreParityParts>(
- bufferDataPart, partSet, WholeBlocks * ColumnSize, 1, missingDataPartIdxA,
- missingDataPartIdxB, missingDataPartIdxC);
-
- if (restoreFullData) {
- PlaceLastBlock(bufferDataPart);
- }
- }
- if (restoreParts) {
- if (missingDataPartIdxA < partSet.Parts.size()) {
- PadAndCrcPart(partSet, missingDataPartIdxA);
- }
- if (missingDataPartIdxB < partSet.Parts.size()) {
- PadAndCrcPart(partSet, missingDataPartIdxB);
- }
- if (missingDataPartIdxC < partSet.Parts.size()) {
- PadAndCrcPart(partSet, missingDataPartIdxC);
- }
- }
- }
-
- template <bool restoreParts, bool restoreFullData, bool reversed, bool restoreParityParts>
- void EoMainRestoreParts(TDataPartSet& partSet, ui32 missingDataPartIdxA, ui32 missingDataPartIdxB) {
- // Read data and parity
- VERBOSE_COUT("EoMainRestorePart" << Endl);
- TRACE("EoMainRestorePart fullSize# " << partSet.FullDataSize
- << " partSet p0 Size# " << partSet.Parts[0].Size
- << " p1 Size# " << partSet.Parts[1].Size
- << " p2 Size# " << partSet.Parts[2].Size << Endl);
- ui32 presentPartIdx = (missingDataPartIdxA == 0 ?
- (missingDataPartIdxB == 1 ? 2 : 1) : \
- (missingDataPartIdxB == 0 ? 1 : 0));
- Y_VERIFY(partSet.Parts[presentPartIdx].Offset % ColumnSize == 0);
- ui64 readPosition = partSet.Parts[presentPartIdx].Offset;
- ui64 wholeBlocks = Min(WholeBlocks - readPosition / ColumnSize, partSet.Parts[presentPartIdx].Size / ColumnSize);
-
- TRACE("wholeBlocks# " << wholeBlocks << " blockSize# " << BlockSize << Endl);
- EoMainRestorePartsWhole<restoreParts, restoreFullData, reversed, restoreParityParts>(BufferDataPart,
- partSet, readPosition, wholeBlocks, missingDataPartIdxA, missingDataPartIdxB);
-
- if (TailSize && (partSet.Parts[presentPartIdx].Size + readPosition > WholeBlocks * ColumnSize)) {
- TRACE("EoMainRestoreParts restore tail" << Endl);
+ TBufferDataPart bufferDataPart;
+ PrepareLastBlockPointers(lastBlockSource.Begin(), bufferDataPart);
+
+ StarMainRestorePartsWholeSymmetric<restoreParts, restoreFullData, restoreParityParts>(
+ bufferDataPart, partSet, WholeBlocks * ColumnSize, 1, missingDataPartIdxA,
+ missingDataPartIdxB, missingDataPartIdxC);
+
+ if (restoreFullData) {
+ PlaceLastBlock(bufferDataPart);
+ }
+ }
+ if (restoreParts) {
+ if (missingDataPartIdxA < partSet.Parts.size()) {
+ PadAndCrcPart(partSet, missingDataPartIdxA);
+ }
+ if (missingDataPartIdxB < partSet.Parts.size()) {
+ PadAndCrcPart(partSet, missingDataPartIdxB);
+ }
+ if (missingDataPartIdxC < partSet.Parts.size()) {
+ PadAndCrcPart(partSet, missingDataPartIdxC);
+ }
+ }
+ }
+
+ template <bool restoreParts, bool restoreFullData, bool reversed, bool restoreParityParts>
+ void EoMainRestoreParts(TDataPartSet& partSet, ui32 missingDataPartIdxA, ui32 missingDataPartIdxB) {
+ // Read data and parity
+ VERBOSE_COUT("EoMainRestorePart" << Endl);
+ TRACE("EoMainRestorePart fullSize# " << partSet.FullDataSize
+ << " partSet p0 Size# " << partSet.Parts[0].Size
+ << " p1 Size# " << partSet.Parts[1].Size
+ << " p2 Size# " << partSet.Parts[2].Size << Endl);
+ ui32 presentPartIdx = (missingDataPartIdxA == 0 ?
+ (missingDataPartIdxB == 1 ? 2 : 1) : \
+ (missingDataPartIdxB == 0 ? 1 : 0));
+ Y_VERIFY(partSet.Parts[presentPartIdx].Offset % ColumnSize == 0);
+ ui64 readPosition = partSet.Parts[presentPartIdx].Offset;
+ ui64 wholeBlocks = Min(WholeBlocks - readPosition / ColumnSize, partSet.Parts[presentPartIdx].Size / ColumnSize);
+
+ TRACE("wholeBlocks# " << wholeBlocks << " blockSize# " << BlockSize << Endl);
+ EoMainRestorePartsWhole<restoreParts, restoreFullData, reversed, restoreParityParts>(BufferDataPart,
+ partSet, readPosition, wholeBlocks, missingDataPartIdxA, missingDataPartIdxB);
+
+ if (TailSize && (partSet.Parts[presentPartIdx].Size + readPosition > WholeBlocks * ColumnSize)) {
+ TRACE("EoMainRestoreParts restore tail" << Endl);
TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64));
- TBufferDataPart bufferDataPart;
- PrepareLastBlockPointers(lastBlockSource.Begin(), bufferDataPart);
-
- EoMainRestorePartsWhole<restoreParts, restoreFullData, reversed, restoreParityParts>(
- bufferDataPart, partSet, WholeBlocks * ColumnSize, 1, missingDataPartIdxA, missingDataPartIdxB);
-
- if (restoreFullData) {
- PlaceLastBlock(bufferDataPart);
- }
- }
-
- if (restoreParts) {
- if (missingDataPartIdxA < partSet.Parts.size()) {
- PadAndCrcPart(partSet, missingDataPartIdxA);
- }
- if (missingDataPartIdxB < partSet.Parts.size()) {
- PadAndCrcPart(partSet, missingDataPartIdxB);
- }
- }
- }
-
- template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
- void XorRestorePartWhole(TBufferDataPart &bufferDataPart, TDataPartSet& partSet,
- ui64 readPosition, ui32 beginBlockIdx, ui32 endBlockIdx, ui32 missingDataPartIdx) {
- VERBOSE_COUT("XorRestorePartWhole: read:" << readPosition << " LineCount: " << LineCount << Endl);
- ui64 writePosition = 0;
- ui64 partsBuffer[MAX_TOTAL_PARTS][MAX_TOTAL_PARTS - 2];
- ui64 fullDataBuffer[MAX_TOTAL_PARTS][MAX_TOTAL_PARTS - 2];
- for (ui64 blockIdx = beginBlockIdx; blockIdx < endBlockIdx; ++blockIdx) {
-#if IS_VERBOSE
- for (ui64 lineIdx = 0; lineIdx < LineCount; ++lineIdx) {
- for (ui32 part = 0; part <= DataParts; ++part) {
- if (part != missingDataPartIdx) {
- ui64 partData = *reinterpret_cast<const ui64*>(partSet.Parts[part].GetDataAt(readPosition +
- lineIdx * sizeof(ui64)));
- VERBOSE_COUT(DebugFormatBits(partData) << ", ");
- } else {
- VERBOSE_COUT(", ");
- }
- }
- VERBOSE_COUT(Endl);
- }
- VERBOSE_COUT(Endl);
-#endif
-
- ui64 contiguousSize = DataSize;
-
- if (restoreFullData) {
- for (ui32 i = 0; i < DataParts; ++i) {
- contiguousSize = std::min(contiguousSize,
- bufferDataPart[i].GetContiguousSize(sizeof(ui64) * writePosition));
- }
- }
-
- for (ui32 i = 0; i <= DataParts; ++i) {
- if (restoreParts || i != missingDataPartIdx) {
- contiguousSize = std::min(contiguousSize, partSet.Parts[i].GetContiguousSize(readPosition));
- }
- }
-
- if (restoreParts) {
- contiguousSize = std::min(contiguousSize, partSet.Parts[missingDataPartIdx].GetContiguousSize(readPosition));
- }
-
- contiguousSize /= sizeof(ui64) * LineCount;
-
- for (ui64 i = 0; i < contiguousSize && blockIdx < endBlockIdx; ++i, ++blockIdx) {
-
- for (ui64 lineIdx = 0; lineIdx < LineCount; ++lineIdx) {
- ui64 restoredData = 0;
- for (ui32 part = 0; part < DataParts; ++part) {
- if (part != missingDataPartIdx) {
- ui64 partData = *reinterpret_cast<const ui64*>(partSet.Parts[part].FastDataPtrAt(readPosition));
- restoredData ^= partData;
- if (restoreFullData) {
- bufferDataPart[part].FastAt64(writePosition) = partData;
- }
- }
- }
- if (missingDataPartIdx < DataParts) {
- ui64 partData = *reinterpret_cast<const ui64*>(partSet.Parts[DataParts].FastDataPtrAt(readPosition));
- restoredData ^= partData;
- if (restoreFullData) {
- bufferDataPart[missingDataPartIdx].FastAt64(writePosition) = restoredData;
- }
- if (restoreParts) {
- *reinterpret_cast<ui64*>(partSet.Parts[missingDataPartIdx].FastDataPtrAt(readPosition)) =
- restoredData;
- }
- } else if (restoreParts && missingDataPartIdx == DataParts) {
- *reinterpret_cast<ui64*>(partSet.Parts[DataParts].FastDataPtrAt(readPosition)) = restoredData;
- }
- readPosition += sizeof(ui64);
- if (restoreFullData) {
- ++writePosition;
- }
- }
-
- }
-
- if (blockIdx == endBlockIdx) {
- break;
- }
-
- for (ui32 i = 0; i <= DataParts; ++i) {
- if (i != missingDataPartIdx) {
+ TBufferDataPart bufferDataPart;
+ PrepareLastBlockPointers(lastBlockSource.Begin(), bufferDataPart);
+
+ EoMainRestorePartsWhole<restoreParts, restoreFullData, reversed, restoreParityParts>(
+ bufferDataPart, partSet, WholeBlocks * ColumnSize, 1, missingDataPartIdxA, missingDataPartIdxB);
+
+ if (restoreFullData) {
+ PlaceLastBlock(bufferDataPart);
+ }
+ }
+
+ if (restoreParts) {
+ if (missingDataPartIdxA < partSet.Parts.size()) {
+ PadAndCrcPart(partSet, missingDataPartIdxA);
+ }
+ if (missingDataPartIdxB < partSet.Parts.size()) {
+ PadAndCrcPart(partSet, missingDataPartIdxB);
+ }
+ }
+ }
+
+ template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
+ void XorRestorePartWhole(TBufferDataPart &bufferDataPart, TDataPartSet& partSet,
+ ui64 readPosition, ui32 beginBlockIdx, ui32 endBlockIdx, ui32 missingDataPartIdx) {
+ VERBOSE_COUT("XorRestorePartWhole: read:" << readPosition << " LineCount: " << LineCount << Endl);
+ ui64 writePosition = 0;
+ ui64 partsBuffer[MAX_TOTAL_PARTS][MAX_TOTAL_PARTS - 2];
+ ui64 fullDataBuffer[MAX_TOTAL_PARTS][MAX_TOTAL_PARTS - 2];
+ for (ui64 blockIdx = beginBlockIdx; blockIdx < endBlockIdx; ++blockIdx) {
+#if IS_VERBOSE
+ for (ui64 lineIdx = 0; lineIdx < LineCount; ++lineIdx) {
+ for (ui32 part = 0; part <= DataParts; ++part) {
+ if (part != missingDataPartIdx) {
+ ui64 partData = *reinterpret_cast<const ui64*>(partSet.Parts[part].GetDataAt(readPosition +
+ lineIdx * sizeof(ui64)));
+ VERBOSE_COUT(DebugFormatBits(partData) << ", ");
+ } else {
+ VERBOSE_COUT(", ");
+ }
+ }
+ VERBOSE_COUT(Endl);
+ }
+ VERBOSE_COUT(Endl);
+#endif
+
+ ui64 contiguousSize = DataSize;
+
+ if (restoreFullData) {
+ for (ui32 i = 0; i < DataParts; ++i) {
+ contiguousSize = std::min(contiguousSize,
+ bufferDataPart[i].GetContiguousSize(sizeof(ui64) * writePosition));
+ }
+ }
+
+ for (ui32 i = 0; i <= DataParts; ++i) {
+ if (restoreParts || i != missingDataPartIdx) {
+ contiguousSize = std::min(contiguousSize, partSet.Parts[i].GetContiguousSize(readPosition));
+ }
+ }
+
+ if (restoreParts) {
+ contiguousSize = std::min(contiguousSize, partSet.Parts[missingDataPartIdx].GetContiguousSize(readPosition));
+ }
+
+ contiguousSize /= sizeof(ui64) * LineCount;
+
+ for (ui64 i = 0; i < contiguousSize && blockIdx < endBlockIdx; ++i, ++blockIdx) {
+
+ for (ui64 lineIdx = 0; lineIdx < LineCount; ++lineIdx) {
+ ui64 restoredData = 0;
+ for (ui32 part = 0; part < DataParts; ++part) {
+ if (part != missingDataPartIdx) {
+ ui64 partData = *reinterpret_cast<const ui64*>(partSet.Parts[part].FastDataPtrAt(readPosition));
+ restoredData ^= partData;
+ if (restoreFullData) {
+ bufferDataPart[part].FastAt64(writePosition) = partData;
+ }
+ }
+ }
+ if (missingDataPartIdx < DataParts) {
+ ui64 partData = *reinterpret_cast<const ui64*>(partSet.Parts[DataParts].FastDataPtrAt(readPosition));
+ restoredData ^= partData;
+ if (restoreFullData) {
+ bufferDataPart[missingDataPartIdx].FastAt64(writePosition) = restoredData;
+ }
+ if (restoreParts) {
+ *reinterpret_cast<ui64*>(partSet.Parts[missingDataPartIdx].FastDataPtrAt(readPosition)) =
+ restoredData;
+ }
+ } else if (restoreParts && missingDataPartIdx == DataParts) {
+ *reinterpret_cast<ui64*>(partSet.Parts[DataParts].FastDataPtrAt(readPosition)) = restoredData;
+ }
+ readPosition += sizeof(ui64);
+ if (restoreFullData) {
+ ++writePosition;
+ }
+ }
+
+ }
+
+ if (blockIdx == endBlockIdx) {
+ break;
+ }
+
+ for (ui32 i = 0; i <= DataParts; ++i) {
+ if (i != missingDataPartIdx) {
TRopeUtils::Memcpy((char*)partsBuffer[i],
- partSet.Parts[i].FastViewer.GetCurrent(readPosition), LineCount * sizeof(ui64));
- }
- }
-
- for (ui64 lineIdx = 0; lineIdx < LineCount; ++lineIdx) {
- ui64 restoredData = 0;
- for (ui32 part = 0; part < DataParts; ++part) {
- if (part != missingDataPartIdx) {
- ui64 partData = partsBuffer[part][lineIdx];
- restoredData ^= partData;
- if (restoreFullData) {
- fullDataBuffer[part][lineIdx] = partData;
- }
- }
- }
- if (missingDataPartIdx < DataParts) {
- ui64 partData = partsBuffer[DataParts][lineIdx];
- restoredData ^= partData;
- if (restoreFullData) {
- fullDataBuffer[missingDataPartIdx][lineIdx] = restoredData;
- }
- if (restoreParts) {
- *reinterpret_cast<ui64*>(partSet.Parts[missingDataPartIdx].GetDataAt(readPosition)) =
- restoredData;
- }
- } else if (restoreParts && missingDataPartIdx == DataParts) {
- *reinterpret_cast<ui64*>(partSet.Parts[DataParts].GetDataAt(readPosition)) = restoredData;
- }
- readPosition += sizeof(ui64);
- if (restoreFullData) {
- ++writePosition;
- }
- }
-
- if (restoreFullData) {
- for (ui32 i = 0; i < DataParts; ++i) {
+ partSet.Parts[i].FastViewer.GetCurrent(readPosition), LineCount * sizeof(ui64));
+ }
+ }
+
+ for (ui64 lineIdx = 0; lineIdx < LineCount; ++lineIdx) {
+ ui64 restoredData = 0;
+ for (ui32 part = 0; part < DataParts; ++part) {
+ if (part != missingDataPartIdx) {
+ ui64 partData = partsBuffer[part][lineIdx];
+ restoredData ^= partData;
+ if (restoreFullData) {
+ fullDataBuffer[part][lineIdx] = partData;
+ }
+ }
+ }
+ if (missingDataPartIdx < DataParts) {
+ ui64 partData = partsBuffer[DataParts][lineIdx];
+ restoredData ^= partData;
+ if (restoreFullData) {
+ fullDataBuffer[missingDataPartIdx][lineIdx] = restoredData;
+ }
+ if (restoreParts) {
+ *reinterpret_cast<ui64*>(partSet.Parts[missingDataPartIdx].GetDataAt(readPosition)) =
+ restoredData;
+ }
+ } else if (restoreParts && missingDataPartIdx == DataParts) {
+ *reinterpret_cast<ui64*>(partSet.Parts[DataParts].GetDataAt(readPosition)) = restoredData;
+ }
+ readPosition += sizeof(ui64);
+ if (restoreFullData) {
+ ++writePosition;
+ }
+ }
+
+ if (restoreFullData) {
+ for (ui32 i = 0; i < DataParts; ++i) {
TRopeUtils::Memcpy(bufferDataPart[i].GetCurrent((writePosition - LineCount) * sizeof(ui64)),
- (const char*)fullDataBuffer[i], LineCount * sizeof(ui64));
- }
- }
-
-#if IS_VERBOSE
- VERBOSE_COUT("Out: " << Endl);
- for (ui64 lineIdx = 0; lineIdx < LineCount; ++lineIdx) {
- for (ui32 part = 0; part <= DataParts; ++part) {
- ui64 partData = *reinterpret_cast<const ui64*>(
- partSet.Parts[part].GetDataAt(readPosition - ColumnSize + lineIdx * sizeof(ui64)));
- VERBOSE_COUT(DebugFormatBits(partData) << ", ");
- }
- VERBOSE_COUT(Endl);
- }
- VERBOSE_COUT(Endl);
-#endif
- }
- }
-
- template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
- void XorRestorePart(TDataPartSet &partSet, ui32 missingDataPartIdx) {
- // Read data and parity
- VERBOSE_COUT("XorRestorePart" << Endl);
- TRACE("XorRestorePart partSet p0 Size# " << partSet.Parts[0].Size
- << " p1 Size# " << partSet.Parts[1].Size << Endl);
- ui32 presentPartIdx = (missingDataPartIdx == 0 ? 1 : 0);
- Y_VERIFY(partSet.Parts[presentPartIdx].Offset % ColumnSize == 0);
- ui64 readPosition = partSet.Parts[presentPartIdx].Offset;
- ui64 beginBlockIdx = readPosition / ColumnSize;
- ui64 wholeBlocks = Min(WholeBlocks - readPosition / ColumnSize, partSet.Parts[presentPartIdx].Size / ColumnSize);
- TRACE("XorRestore beginBlockIdx# " << beginBlockIdx << " wholeBlocks# " << wholeBlocks << Endl);
- XorRestorePartWhole<restoreParts, restoreFullData, restoreParityParts>(BufferDataPart, partSet, readPosition,
- beginBlockIdx, beginBlockIdx + wholeBlocks, missingDataPartIdx);
-
- if (TailSize && (partSet.Parts[presentPartIdx].Size + readPosition > WholeBlocks * ColumnSize)) {
- TRACE("Restore tail, restoreFullData# " << restoreFullData << " resotreParts# " << restoreParts << Endl);
+ (const char*)fullDataBuffer[i], LineCount * sizeof(ui64));
+ }
+ }
+
+#if IS_VERBOSE
+ VERBOSE_COUT("Out: " << Endl);
+ for (ui64 lineIdx = 0; lineIdx < LineCount; ++lineIdx) {
+ for (ui32 part = 0; part <= DataParts; ++part) {
+ ui64 partData = *reinterpret_cast<const ui64*>(
+ partSet.Parts[part].GetDataAt(readPosition - ColumnSize + lineIdx * sizeof(ui64)));
+ VERBOSE_COUT(DebugFormatBits(partData) << ", ");
+ }
+ VERBOSE_COUT(Endl);
+ }
+ VERBOSE_COUT(Endl);
+#endif
+ }
+ }
+
+ template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
+ void XorRestorePart(TDataPartSet &partSet, ui32 missingDataPartIdx) {
+ // Read data and parity
+ VERBOSE_COUT("XorRestorePart" << Endl);
+ TRACE("XorRestorePart partSet p0 Size# " << partSet.Parts[0].Size
+ << " p1 Size# " << partSet.Parts[1].Size << Endl);
+ ui32 presentPartIdx = (missingDataPartIdx == 0 ? 1 : 0);
+ Y_VERIFY(partSet.Parts[presentPartIdx].Offset % ColumnSize == 0);
+ ui64 readPosition = partSet.Parts[presentPartIdx].Offset;
+ ui64 beginBlockIdx = readPosition / ColumnSize;
+ ui64 wholeBlocks = Min(WholeBlocks - readPosition / ColumnSize, partSet.Parts[presentPartIdx].Size / ColumnSize);
+ TRACE("XorRestore beginBlockIdx# " << beginBlockIdx << " wholeBlocks# " << wholeBlocks << Endl);
+ XorRestorePartWhole<restoreParts, restoreFullData, restoreParityParts>(BufferDataPart, partSet, readPosition,
+ beginBlockIdx, beginBlockIdx + wholeBlocks, missingDataPartIdx);
+
+ if (TailSize && (partSet.Parts[presentPartIdx].Size + readPosition > WholeBlocks * ColumnSize)) {
+ TRACE("Restore tail, restoreFullData# " << restoreFullData << " resotreParts# " << restoreParts << Endl);
TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64));
- TBufferDataPart bufferDataPart;
- PrepareLastBlockPointers(lastBlockSource.Begin(), bufferDataPart);
-
- XorRestorePartWhole<restoreParts, restoreFullData, restoreParityParts>(bufferDataPart,
- partSet, WholeBlocks * ColumnSize, WholeBlocks, WholeBlocks + 1, missingDataPartIdx);
-
- if (restoreFullData) {
- PlaceLastBlock(bufferDataPart);
- }
- }
-
- if (restoreParts && missingDataPartIdx < partSet.Parts.size()) {
- if (restoreParityParts || missingDataPartIdx < DataParts) {
- PadAndCrcPart(partSet, missingDataPartIdx);
- }
- }
- }
-
- void PadAndCrcPart(TDataPartSet &inOutPartSet, ui32 partIdx) {
- if (inOutPartSet.IsFragment) {
- return;
- }
- switch (CrcMode) {
- case TRopeErasureType::CrcModeNone:
- return;
- case TRopeErasureType::CrcModeWholePart:
- if (DataSize) {
- PadAndCrcAtTheEnd(inOutPartSet.Parts[partIdx].OwnedRope.Begin(), PartUserSize, PartContainerSize);
- } else {
+ TBufferDataPart bufferDataPart;
+ PrepareLastBlockPointers(lastBlockSource.Begin(), bufferDataPart);
+
+ XorRestorePartWhole<restoreParts, restoreFullData, restoreParityParts>(bufferDataPart,
+ partSet, WholeBlocks * ColumnSize, WholeBlocks, WholeBlocks + 1, missingDataPartIdx);
+
+ if (restoreFullData) {
+ PlaceLastBlock(bufferDataPart);
+ }
+ }
+
+ if (restoreParts && missingDataPartIdx < partSet.Parts.size()) {
+ if (restoreParityParts || missingDataPartIdx < DataParts) {
+ PadAndCrcPart(partSet, missingDataPartIdx);
+ }
+ }
+ }
+
+ void PadAndCrcPart(TDataPartSet &inOutPartSet, ui32 partIdx) {
+ if (inOutPartSet.IsFragment) {
+ return;
+ }
+ switch (CrcMode) {
+ case TRopeErasureType::CrcModeNone:
+ return;
+ case TRopeErasureType::CrcModeWholePart:
+ if (DataSize) {
+ PadAndCrcAtTheEnd(inOutPartSet.Parts[partIdx].OwnedRope.Begin(), PartUserSize, PartContainerSize);
+ } else {
TRopeUtils::Memset(inOutPartSet.Parts[partIdx].OwnedRope.Begin(), 0, PartContainerSize);
- }
- return;
- }
- ythrow TWithBackTrace<yexception>() << "Unknown crcMode = " << (i32)CrcMode;
- }
-};
-
-void PadAndCrcParts(TRopeErasureType::ECrcMode crcMode, const TBlockParams &p, TDataPartSet &inOutPartSet) {
- if (inOutPartSet.IsFragment) {
- return;
- }
- switch (crcMode) {
- case TRopeErasureType::CrcModeNone:
- return;
- case TRopeErasureType::CrcModeWholePart:
- if (p.DataSize) {
- for (ui32 i = 0; i < p.TotalParts; ++i) {
- PadAndCrcAtTheEnd(inOutPartSet.Parts[i].OwnedRope.Begin(), p.PartUserSize, p.PartContainerSize);
- }
- } else {
- for (ui32 i = 0; i < p.TotalParts; ++i) {
+ }
+ return;
+ }
+ ythrow TWithBackTrace<yexception>() << "Unknown crcMode = " << (i32)CrcMode;
+ }
+};
+
+void PadAndCrcParts(TRopeErasureType::ECrcMode crcMode, const TBlockParams &p, TDataPartSet &inOutPartSet) {
+ if (inOutPartSet.IsFragment) {
+ return;
+ }
+ switch (crcMode) {
+ case TRopeErasureType::CrcModeNone:
+ return;
+ case TRopeErasureType::CrcModeWholePart:
+ if (p.DataSize) {
+ for (ui32 i = 0; i < p.TotalParts; ++i) {
+ PadAndCrcAtTheEnd(inOutPartSet.Parts[i].OwnedRope.Begin(), p.PartUserSize, p.PartContainerSize);
+ }
+ } else {
+ for (ui32 i = 0; i < p.TotalParts; ++i) {
TRopeUtils::Memset(inOutPartSet.Parts[i].OwnedRope.Begin(), 0, p.PartContainerSize);
- }
- }
- return;
- }
- ythrow TWithBackTrace<yexception>() << "Unknown crcMode = " << (i32)crcMode;
-}
-
-inline void StarBlockSplit(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, const TRope &buffer,
- TDataPartSet &outPartSet) {
+ }
+ }
+ return;
+ }
+ ythrow TWithBackTrace<yexception>() << "Unknown crcMode = " << (i32)crcMode;
+}
+
+inline void StarBlockSplit(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, const TRope &buffer,
+ TDataPartSet &outPartSet) {
Y_VERIFY(TRopeHelpers::Is8Aligned(buffer));
- TBlockParams p(crcMode, type, buffer.GetSize());
-
- // Prepare input data pointers
- p.PrepareInputDataPointers(buffer.Begin());
-
- outPartSet.FullDataSize = buffer.GetSize();
- outPartSet.PartsMask = ~((~(ui32)0) << p.TotalParts);
- outPartSet.Parts.resize(p.TotalParts);
- for (ui32 i = 0; i < p.TotalParts; ++i) {
- TRACE("Line# " << __LINE__ << Endl);
- Refurbish(outPartSet.Parts[i], p.PartContainerSize);
- }
- outPartSet.MemoryConsumed = p.TotalParts * outPartSet.Parts[0].MemoryConsumed();
-
- p.StarSplit<false>(outPartSet);
- PadAndCrcParts(crcMode, p, outPartSet);
-}
-
-inline void EoBlockSplit(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, const TRope &buffer,
- TDataPartSet &outPartSet) {
- TBlockParams p(crcMode, type, buffer.GetSize());
-
- outPartSet.FullDataSize = buffer.GetSize();
- outPartSet.PartsMask = ~((~(ui32)0) << p.TotalParts);
- outPartSet.Parts.resize(p.TotalParts);
-
- TRope::TConstIterator iterator = buffer.Begin();
- for (ui32 i = 0; i < p.DataParts; ++i) {
- size_t size = p.SmallPartSize + (i < p.FirstSmallPartIdx) * p.ColumnSize;
- outPartSet.Parts[i].OwnedRope = TRope();
- TRope &rope = outPartSet.Parts[i].OwnedRope;
-
- if (size) {
- rope = TRope(iterator, iterator + size);
- iterator += size;
- }
-
- if (i == p.DataParts - 1 && iterator != buffer.End()) {
- rope.Insert(rope.End(), TRope(iterator, buffer.End()));
- }
-
+ TBlockParams p(crcMode, type, buffer.GetSize());
+
+ // Prepare input data pointers
+ p.PrepareInputDataPointers(buffer.Begin());
+
+ outPartSet.FullDataSize = buffer.GetSize();
+ outPartSet.PartsMask = ~((~(ui32)0) << p.TotalParts);
+ outPartSet.Parts.resize(p.TotalParts);
+ for (ui32 i = 0; i < p.TotalParts; ++i) {
+ TRACE("Line# " << __LINE__ << Endl);
+ Refurbish(outPartSet.Parts[i], p.PartContainerSize);
+ }
+ outPartSet.MemoryConsumed = p.TotalParts * outPartSet.Parts[0].MemoryConsumed();
+
+ p.StarSplit<false>(outPartSet);
+ PadAndCrcParts(crcMode, p, outPartSet);
+}
+
+inline void EoBlockSplit(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, const TRope &buffer,
+ TDataPartSet &outPartSet) {
+ TBlockParams p(crcMode, type, buffer.GetSize());
+
+ outPartSet.FullDataSize = buffer.GetSize();
+ outPartSet.PartsMask = ~((~(ui32)0) << p.TotalParts);
+ outPartSet.Parts.resize(p.TotalParts);
+
+ TRope::TConstIterator iterator = buffer.Begin();
+ for (ui32 i = 0; i < p.DataParts; ++i) {
+ size_t size = p.SmallPartSize + (i < p.FirstSmallPartIdx) * p.ColumnSize;
+ outPartSet.Parts[i].OwnedRope = TRope();
+ TRope &rope = outPartSet.Parts[i].OwnedRope;
+
+ if (size) {
+ rope = TRope(iterator, iterator + size);
+ iterator += size;
+ }
+
+ if (i == p.DataParts - 1 && iterator != buffer.End()) {
+ rope.Insert(rope.End(), TRope(iterator, buffer.End()));
+ }
+
TRopeHelpers::Resize(rope, p.PartContainerSize);
- outPartSet.Parts[i].ReferenceTo(rope);
- }
-
- for (ui32 i = p.DataParts; i < p.TotalParts; ++i) {
- TRACE("Line# " << __LINE__ << Endl);
- Refurbish(outPartSet.Parts[i], p.PartContainerSize);
- }
-
- outPartSet.MemoryConsumed = p.TotalParts * outPartSet.Parts[0].MemoryConsumed();
-
- p.EoSplit(outPartSet);
-
- PadAndCrcParts(crcMode, p, outPartSet);
-}
-
-inline void XorBlockSplit(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, const TRope& buffer,
- TDataPartSet& outPartSet) {
+ outPartSet.Parts[i].ReferenceTo(rope);
+ }
+
+ for (ui32 i = p.DataParts; i < p.TotalParts; ++i) {
+ TRACE("Line# " << __LINE__ << Endl);
+ Refurbish(outPartSet.Parts[i], p.PartContainerSize);
+ }
+
+ outPartSet.MemoryConsumed = p.TotalParts * outPartSet.Parts[0].MemoryConsumed();
+
+ p.EoSplit(outPartSet);
+
+ PadAndCrcParts(crcMode, p, outPartSet);
+}
+
+inline void XorBlockSplit(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, const TRope& buffer,
+ TDataPartSet& outPartSet) {
Y_VERIFY(TRopeHelpers::Is8Aligned(buffer));
- TBlockParams p(crcMode, type, buffer.GetSize());
-
- // Prepare input data pointers
- p.PrepareInputDataPointers(buffer.Begin());
-
- outPartSet.FullDataSize = buffer.GetSize();
- outPartSet.PartsMask = ~((~(ui32)0) << p.TotalParts);
- outPartSet.Parts.resize(p.TotalParts);
- for (ui32 i = 0; i < p.TotalParts; ++i) {
- TRACE("Line# " << __LINE__ << Endl);
- Refurbish(outPartSet.Parts[i], p.PartContainerSize);
- }
- outPartSet.MemoryConsumed = p.TotalParts * outPartSet.Parts[0].MemoryConsumed();
-
- p.XorSplit(outPartSet);
- PadAndCrcParts(crcMode, p, outPartSet);
-}
-
-template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
-void EoBlockRestore(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, TDataPartSet& partSet) {
- TRope &outBuffer = partSet.FullDataFragment.OwnedRope;
- ui32 totalParts = type.TotalPartCount();
- Y_VERIFY(partSet.Parts.size() >= totalParts);
-
- if (outBuffer.GetSize()) {
- outBuffer = TRope();
- }
-
- ui32 missingDataPartIdxA = totalParts;
- ui32 missingDataPartIdxB = totalParts;
- ui32 missingDataPartCount = 0;
- ui64 expectedPartSize = type.PartSize(crcMode, partSet.FullDataSize);
-
- ui32 i = 0;
- for (; i < totalParts; ++i) {
- if (!(partSet.PartsMask & (1 << i))) {
- missingDataPartIdxA = i;
- ++missingDataPartCount;
- break;
- } else {
- Y_VERIFY(partSet.Parts[i].size() == expectedPartSize, "partSet.Parts[%" PRIu32 "].size(): %" PRIu64
- " expectedPartSize: %" PRIu64 " erasure: %s partSet.FullDataSize: %" PRIu64,
- (ui32)i, (ui64)partSet.Parts[i].size(), expectedPartSize, type.ErasureName[type.GetErasure()].data(),
- (ui64)partSet.FullDataSize);
- }
- }
- ++i;
- for (; i < totalParts; ++i) {
- if (!(partSet.PartsMask & (1 << i))) {
- missingDataPartIdxB = i;
- ++missingDataPartCount;
- } else {
- Y_VERIFY(partSet.Parts[i].size() == expectedPartSize, "partSet.Parts[%" PRIu32 "].size()# %" PRIu32
- " != expectedPartSize# %" PRIu32 " erasure: %s partSet.FullDataSize: %" PRIu64,
- (ui32)i, (ui32)partSet.Parts[i].size(), (ui32)expectedPartSize, type.ErasureName[type.GetErasure()].data(),
- (ui64)partSet.FullDataSize);
- }
- }
- Y_VERIFY(missingDataPartCount <= 2);
-
- ui64 dataSize = partSet.FullDataSize;
- if (restoreParts) {
- if (missingDataPartIdxA != totalParts) {
- TRACE("Line# " << __LINE__ << Endl);
- Refurbish(partSet.Parts[missingDataPartIdxA], expectedPartSize);
- }
- if (missingDataPartIdxB != totalParts) {
- TRACE("Line# " << __LINE__ << Endl);
- Refurbish(partSet.Parts[missingDataPartIdxB], expectedPartSize);
- }
- }
-
- TBlockParams p(crcMode, type, dataSize);
-
- if (restoreFullData) {
- p.GlueOutBuffer<restoreParts>(outBuffer, partSet, missingDataPartIdxA, missingDataPartIdxB);
- } else if (missingDataPartCount == 0) {
- return;
- }
-
- if (missingDataPartCount == 2) {
- VERBOSE_COUT("missing parts " << missingDataPartIdxA << " and " << missingDataPartIdxB << Endl);
- } else if (missingDataPartCount == 1) {
- VERBOSE_COUT("missing part " << missingDataPartIdxA << Endl);
- }
-
- // Restore the fast way if all data parts are present
- if (missingDataPartCount == 0 ||
- (!restoreParts && missingDataPartIdxA >= p.TotalParts - 2)) {
- VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
- return;
- }
-
- // Prepare output data pointers
- if (restoreFullData) {
- p.PrepareInputDataPointers(outBuffer.Begin());
- }
-
- // Consider failed disk cases
- // a) < m
- // b) m
- // 'xor-restore'
- // d) m, m+1
- // TODO: 1-pass
- // just glue the data
- // use 'eo split' to restore the remaining parts
- // f) <m, m+1
- // use 'xor-restore' to restore the data
- // TODO: use 2-nd part of 'eo-split' to restore m+1 part
- // TODO: 1-pass
- if (missingDataPartIdxA <= p.DataParts && missingDataPartIdxB >= p.TotalParts - 1) {
- if (!restoreFullData && restoreParts && missingDataPartIdxB == p.TotalParts - 1) {
- // The (f1) case, but no full data needed, only parts
- TRACE("case# f1" << Endl);
- VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
-
- p.XorRestorePart<true, false, false>(partSet, missingDataPartIdxA);
- TRACE("case# f1 split" << Endl);
- p.EoSplit(partSet);
- p.PadAndCrcPart(partSet, missingDataPartIdxA);
- p.PadAndCrcPart(partSet, missingDataPartIdxB);
- } else {
- // Cases (a), (b) and (d2), case (f2) with full data and maybe parts needed
- TRACE("case# a b d2 f2" << Endl);
- VERBOSE_COUT(__LINE__ << " of " << __FILE__ << " missing " << missingDataPartIdxA << Endl);
- p.XorRestorePart<restoreParts, restoreFullData, restoreParityParts>(partSet, missingDataPartIdxA);
- if (restoreParts && missingDataPartIdxB == p.TotalParts - 1 && restoreParityParts) {
- // The (d2a) or (f2a) case with full data and parts needed
- TRACE("case# d2a f2a" << Endl);
- VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
- p.EoSplit(partSet);
- p.PadAndCrcPart(partSet, missingDataPartIdxB);
- }
- if (restoreParts) {
- p.PadAndCrcPart(partSet, missingDataPartIdxA);
- }
- }
- return;
- }
-
- // c) m+1
- // TODO: use 2-nd part of 'eo-split' to restore m+1 part, while glueing the data
- // TODO: 1-pass
- // just glue the data
- // use 'eo split' to restore the missing part
- if (missingDataPartIdxA == p.TotalParts - 1 && missingDataPartIdxB == p.TotalParts) {
- TRACE("case# c" << Endl);
- VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
- if (!restoreFullData) {
- TRACE(__LINE__ << Endl);
- if (!restoreParityParts) {
- TRACE(__LINE__ << Endl);
- return;
- }
- TRACE(__LINE__ << Endl);
- }
- if (restoreParts) {
- TRACE(__LINE__ << Endl);
- VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
- p.EoSplit(partSet);
- p.PadAndCrcPart(partSet, missingDataPartIdxA);
- }
- return;
- }
-
- // e) <m, m
- // TODO: 1-pass
- // use diagonal-sums to restore the data
- // use 'xor restore' with 'restore part' to restore m part
- if (missingDataPartIdxA < p.DataParts && missingDataPartIdxB == p.DataParts) {
- TRACE(__LINE__ << " of " << __FILE__ << " case# e restore part missing# " << missingDataPartIdxA << ", " << missingDataPartIdxB <<
- " restoreParts# " << restoreParts
- << " restoreParityParts# " << restoreParityParts
- << " restoreFullData# " << restoreFullData << Endl);
- p.EoDiagonalRestorePart<restoreParts, restoreFullData, false, restoreParityParts>(partSet, missingDataPartIdxA);
- if (restoreParts) {
- p.PadAndCrcPart(partSet, missingDataPartIdxA);
- if (restoreParityParts) {
- p.PadAndCrcPart(partSet, missingDataPartIdxB);
- }
- }
- return;
- }
-
- // g) <m, <m
- // the main case :(
- TRACE("case# g" << Endl);
- VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
- Y_VERIFY(missingDataPartIdxA < p.DataParts && missingDataPartIdxB < p.DataParts);
- p.EoMainRestoreParts<restoreParts, restoreFullData, false, restoreParityParts>(partSet, missingDataPartIdxA,
- missingDataPartIdxB);
- if (restoreParts) {
- p.PadAndCrcPart(partSet, missingDataPartIdxA);
- p.PadAndCrcPart(partSet, missingDataPartIdxB);
- }
-}
-
-// restorePartiyParts may be set only togehter with restore parts
-template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
-void StarBlockRestore(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, TDataPartSet& partSet) {
- Y_VERIFY(partSet.Is8Aligned());
- TRope &outBuffer = partSet.FullDataFragment.OwnedRope;
-
- ui32 totalParts = type.TotalPartCount();
- Y_VERIFY(partSet.Parts.size() == totalParts);
-
- ui32 missingDataPartIdxA = totalParts;
- ui32 missingDataPartIdxB = totalParts;
- ui32 missingDataPartIdxC = totalParts;
- ui32 missingDataPartCount = 0;
- ui64 expectedPartSize = type.PartSize(crcMode, partSet.FullDataSize); // ???
- ui32 i = 0;
- for (; i < totalParts; ++i) {
- if (!(partSet.PartsMask & (1 << i))) {
- missingDataPartIdxA = i;
- ++missingDataPartCount;
- break;
- } else {
- Y_VERIFY(partSet.Parts[i].size() == expectedPartSize, "partSet.Parts[%" PRIu32 "].size(): %" PRIu64
- " expectedPartSize: %" PRIu64 " erasure: %s partSet.FullDataSize: %" PRIu64,
- (ui32)i, (ui64)partSet.Parts[i].size(), expectedPartSize, type.ErasureName[type.GetErasure()].data(),
- (ui64)partSet.FullDataSize);
- }
- }
- ++i;
- for (; i < totalParts; ++i) {
- if (!(partSet.PartsMask & (1 << i))) {
- missingDataPartIdxB = i;
- ++missingDataPartCount;
- break;
- } else {
- Y_VERIFY(partSet.Parts[i].size() == expectedPartSize, "partSet.Parts[%" PRIu32 "].size()# %" PRIu32
- " != expectedPartSize# %" PRIu32 " erasure: %s partSet.FullDataSize: %" PRIu64,
- (ui32)i, (ui32)partSet.Parts[i].size(), (ui32)expectedPartSize, type.ErasureName[type.GetErasure()].data(),
- (ui64)partSet.FullDataSize);
- }
- }
- ++i;
- for (; i < totalParts; ++i) {
- if (!(partSet.PartsMask & (1 << i))) {
- missingDataPartIdxC = i;
- ++missingDataPartCount;
- break;
- } else {
- Y_VERIFY(partSet.Parts[i].size() == expectedPartSize, "partSet.Parts[%" PRIu32 "].size()# %" PRIu32
- " != expectedPartSize# %" PRIu32 " erasure: %s partSet.FullDataSize: %" PRIu64,
- (ui32)i, (ui32)partSet.Parts[i].size(), (ui32)expectedPartSize, type.ErasureName[type.GetErasure()].data(),
- (ui64)partSet.FullDataSize);
- }
- }
- Y_VERIFY(missingDataPartCount <= 3);
-
- if (restoreParts) {
- if (missingDataPartIdxA != totalParts) {
- TRACE("Line# " << __LINE__ << Endl);
- Refurbish(partSet.Parts[missingDataPartIdxA], expectedPartSize);
- }
- if (missingDataPartIdxB != totalParts) {
- TRACE("Line# " << __LINE__ << Endl);
- Refurbish(partSet.Parts[missingDataPartIdxB], expectedPartSize);
- }
- if (missingDataPartIdxC != totalParts) {
- TRACE("Line# " << __LINE__ << Endl);
- Refurbish(partSet.Parts[missingDataPartIdxC], expectedPartSize);
- }
- }
- if (missingDataPartCount == 3) {
- VERBOSE_COUT("missing parts " << missingDataPartIdxA << " and " << missingDataPartIdxB <<
- " and " << missingDataPartIdxC << Endl);
- } else if (missingDataPartCount == 2) {
- VERBOSE_COUT("missing parts " << missingDataPartIdxA << " and " << missingDataPartIdxB << Endl);
- } else if (missingDataPartCount == 1) {
- VERBOSE_COUT("missing part " << missingDataPartIdxA << Endl);
- }
-
- ui64 dataSize = partSet.FullDataSize;
- TBlockParams p(crcMode, type, dataSize);
- if (restoreFullData) {
- Refurbish(outBuffer, dataSize);
- p.PrepareInputDataPointers(outBuffer.Begin());
- } else if (missingDataPartCount == 0) {
- return;
- }
-
- // Restore the fast way if all data parts are present
- if (missingDataPartCount == 0 ||
- (!restoreParts && missingDataPartIdxA >= p.DataParts)) {
- VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
- p.GlueBlockPartsMemcpy(outBuffer.Begin(), partSet);
- return;
- }
-
-
- // All possible failures of 2 disks which EVENODD capable to handle
- if (missingDataPartCount <= 2 && missingDataPartIdxA != p.TotalParts - 1
- && missingDataPartIdxB != p.TotalParts - 1) {
- if (p.DataParts == 4) {
- TRopeErasureType typeEO(TRopeErasureType::EErasureSpecies::Erasure4Plus2Block);
- EoBlockRestore<restoreParts, restoreFullData, restoreParityParts>(crcMode, typeEO, partSet);
- } else if (p.DataParts == 3) {
- TRopeErasureType typeEO(TRopeErasureType::EErasureSpecies::Erasure3Plus2Block);
- EoBlockRestore<restoreParts, restoreFullData, restoreParityParts>(crcMode, typeEO, partSet);
- } else if (p.DataParts == 2) {
- TRopeErasureType typeEO(TRopeErasureType::EErasureSpecies::Erasure2Plus2Block);
- EoBlockRestore<restoreParts, restoreFullData, restoreParityParts>(crcMode, typeEO, partSet);
- }
- return;
- }
- if (missingDataPartIdxA == p.TotalParts - 1
- || missingDataPartIdxB == p.TotalParts - 1
- || missingDataPartIdxC == p.TotalParts - 1) {
- // Possible combinations handled in this branch
- // '+' stands for part, which is present for sure,
- // '-' stands for part, which is missing for sure,
- // series of 0, 1 and 2 means that there are n missing parts in this region
- // 0 0 0 0 0 0 - or 1 1 1 1 1 1 - or 2 2 2 2 2 2 -
- if (p.DataParts == 4) {
- TRopeErasureType typeEO(TRopeErasureType::EErasureSpecies::Erasure4Plus2Block);
- EoBlockRestore<restoreParts, restoreFullData, restoreParityParts>(crcMode, typeEO, partSet);
- } else if (p.DataParts == 3) {
- TRopeErasureType typeEO(TRopeErasureType::EErasureSpecies::Erasure3Plus2Block);
- EoBlockRestore<restoreParts, restoreFullData, restoreParityParts>(crcMode, typeEO, partSet);
- } else if (p.DataParts == 2) {
- TRopeErasureType typeEO(TRopeErasureType::EErasureSpecies::Erasure2Plus2Block);
- EoBlockRestore<restoreParts, restoreFullData, restoreParityParts>(crcMode, typeEO, partSet);
- }
- if (restoreParts) {
- if (restoreParityParts) {
- p.StarSplit<true>(partSet);
- }
- if (missingDataPartIdxA < (restoreParityParts ? p.TotalParts : p.DataParts)) {
- p.PadAndCrcPart(partSet, missingDataPartIdxA);
- }
- if (missingDataPartIdxB < (restoreParityParts ? p.TotalParts : p.DataParts)) {
- p.PadAndCrcPart(partSet, missingDataPartIdxB);
- }
- if (missingDataPartIdxC < (restoreParityParts ? p.TotalParts : p.DataParts)) {
- p.PadAndCrcPart(partSet, missingDataPartIdxC);
- }
- }
- return;
- }
- // There are remain only cases with missingDataPartCount == 3
- if ( missingDataPartIdxC == p.DataParts + 1) {
- if (missingDataPartIdxB < p.DataParts) {
- // 2 2 2 2 + - +
- // "It can be decoded with slightly modification of the EVENODD decoding" (c)
- p.EoMainRestoreParts<restoreParts, restoreFullData, true, restoreParityParts>(partSet, missingDataPartIdxA,
- missingDataPartIdxB);
- } else {
- // 1 1 1 1 - - +
- p.EoDiagonalRestorePart<restoreParts, restoreFullData, true, restoreParityParts>(partSet, missingDataPartIdxA);
- }
- if (restoreParts) {
- if (restoreParityParts) {
- p.StarSplit<!restoreFullData>(partSet);
- }
- if (missingDataPartIdxA < (restoreParityParts ? p.TotalParts : p.DataParts)) {
- p.PadAndCrcPart(partSet, missingDataPartIdxA);
- }
- if (missingDataPartIdxB < (restoreParityParts ? p.TotalParts : p.DataParts)) {
- p.PadAndCrcPart(partSet, missingDataPartIdxB);
- }
- if (missingDataPartIdxC < (restoreParityParts ? p.TotalParts : p.DataParts)) {
- p.PadAndCrcPart(partSet, missingDataPartIdxC);
- }
- }
- return;
- }
- if (missingDataPartIdxC == p.DataParts) {
- // 2 2 2 2 - + +
- if (! restoreParts) {
- TRACE("Line# " << __LINE__ << Endl);
- Refurbish(partSet.Parts[missingDataPartIdxC], expectedPartSize);
- }
- p.StarRestoreHorizontalPart<restoreParts, restoreFullData, restoreParityParts>(partSet,
- missingDataPartIdxA, missingDataPartIdxB);
- if (restoreParts) {
- if (missingDataPartIdxA < (restoreParityParts ? p.TotalParts : p.DataParts)) {
- p.PadAndCrcPart(partSet, missingDataPartIdxA);
- }
- if (missingDataPartIdxB < (restoreParityParts ? p.TotalParts : p.DataParts)) {
- p.PadAndCrcPart(partSet, missingDataPartIdxB);
- }
- if (missingDataPartIdxC < (restoreParityParts ? p.TotalParts : p.DataParts)) {
- p.PadAndCrcPart(partSet, missingDataPartIdxC);
- }
- }
- return;
- }
-
- VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
- Y_VERIFY(missingDataPartIdxA < p.DataParts && missingDataPartIdxB < p.DataParts
- && missingDataPartIdxC < p.DataParts);
- // Two possible cases:
- // - Symmetric
- // - Asymmetric
- // But for m = 5 it is always possible to change asymmetric to symmetric by shifting
- ui32 m = ErasureSpeciesParameters[TRopeErasureType::EErasureSpecies::Erasure4Plus3Block].Prime;
- while ((m + missingDataPartIdxB - missingDataPartIdxA) % m != (m + missingDataPartIdxC - missingDataPartIdxB) % m ) {
- ui32 tmp = missingDataPartIdxA;
- missingDataPartIdxA = missingDataPartIdxB;
- missingDataPartIdxB = missingDataPartIdxC;
- missingDataPartIdxC = tmp;
- }
- if (! restoreParts) {
- TRACE("Line# " << __LINE__ << Endl);
- Refurbish(partSet.Parts[missingDataPartIdxB], expectedPartSize);
- }
- p.StarMainRestorePartsSymmetric<restoreParts, restoreFullData, restoreParityParts>(partSet,
- missingDataPartIdxA, missingDataPartIdxB, missingDataPartIdxC);
- if (restoreParts) {
- if (missingDataPartIdxA < (restoreParityParts ? p.TotalParts : p.DataParts)) {
- p.PadAndCrcPart(partSet, missingDataPartIdxA);
- }
- if (missingDataPartIdxB < (restoreParityParts ? p.TotalParts : p.DataParts)) {
- p.PadAndCrcPart(partSet, missingDataPartIdxB);
- }
- if (missingDataPartIdxC < (restoreParityParts ? p.TotalParts : p.DataParts)) {
- p.PadAndCrcPart(partSet, missingDataPartIdxC);
- }
- }
-}
-
-template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
-void XorBlockRestore(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, TDataPartSet &partSet) {
- TRope &outBuffer = partSet.FullDataFragment.OwnedRope;
- ui32 totalParts = type.TotalPartCount();
- Y_VERIFY(partSet.Parts.size() == totalParts,
- "partSet.Parts.size(): %" PRIu64 " totalParts: %" PRIu32 " erasure: %s",
- (ui64)partSet.Parts.size(), (ui32)totalParts, type.ErasureName[type.GetErasure()].data());
-
- ui32 missingDataPartIdx = totalParts;
- ui32 missingDataPartCount = 0;
- ui64 expectedPartSize = type.PartSize(crcMode, partSet.FullDataSize);
- for (ui32 i = 0; i < totalParts; ++i) {
- if (!(partSet.PartsMask & (1 << i))) {
- missingDataPartIdx = i;
- ++missingDataPartCount;
- } else {
- Y_VERIFY(partSet.Parts[i].size() == expectedPartSize, "partSet.Parts[%" PRIu32 "].size(): %" PRIu64
- " expectedPartSize: %" PRIu64 " erasure: %s partSet.FullDataSize: %" PRIu64,
- (ui32)i, (ui64)partSet.Parts[i].size(), expectedPartSize, type.ErasureName[type.GetErasure()].data(),
- (ui64)partSet.FullDataSize);
- }
- }
- Y_VERIFY(missingDataPartCount <= 1);
-
- ui64 dataSize = partSet.FullDataSize;
- if (restoreParts && missingDataPartIdx != totalParts) {
- TRACE("Line# " << __LINE__ << Endl);
- Refurbish(partSet.Parts[missingDataPartIdx], partSet.Parts[missingDataPartIdx == 0 ? 1 : 0].size());
- }
- if (restoreFullData) {
- Refurbish(outBuffer, dataSize);
- } else if (missingDataPartCount == 0) {
- return;
- }
-
- TBlockParams p(crcMode, type, dataSize);
-
- // Restore the fast way if all data parts are present
- if (missingDataPartCount == 0 ||
- (missingDataPartCount == 1 && !restoreParts && missingDataPartIdx == p.TotalParts - 1)) {
- p.GlueBlockPartsMemcpy(outBuffer.Begin(), partSet);
- return;
- }
- // Prepare output data pointers
- if (restoreFullData) {
- p.PrepareInputDataPointers(outBuffer.Begin());
- }
-
- p.XorRestorePart<restoreParts, restoreFullData, restoreParityParts>(partSet, missingDataPartIdx);
-}
-
-const std::array<TString, TRopeErasureType::ErasureSpeciesCount> TRopeErasureType::ErasureName{{
- "none",
- "mirror-3",
- "block-3-1",
- "stripe-3-1",
- "block-4-2",
- "block-3-2",
- "stripe-4-2",
- "stripe-3-2",
- "mirror-3-2",
- "mirror-3-dc",
- "block-4-3",
- "stripe-4-3",
- "block-3-3",
- "stripe-3-3",
- "block-2-3",
- "stripe-2-3",
- "block-2-2",
- "stripe-2-2",
+ TBlockParams p(crcMode, type, buffer.GetSize());
+
+ // Prepare input data pointers
+ p.PrepareInputDataPointers(buffer.Begin());
+
+ outPartSet.FullDataSize = buffer.GetSize();
+ outPartSet.PartsMask = ~((~(ui32)0) << p.TotalParts);
+ outPartSet.Parts.resize(p.TotalParts);
+ for (ui32 i = 0; i < p.TotalParts; ++i) {
+ TRACE("Line# " << __LINE__ << Endl);
+ Refurbish(outPartSet.Parts[i], p.PartContainerSize);
+ }
+ outPartSet.MemoryConsumed = p.TotalParts * outPartSet.Parts[0].MemoryConsumed();
+
+ p.XorSplit(outPartSet);
+ PadAndCrcParts(crcMode, p, outPartSet);
+}
+
+template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
+void EoBlockRestore(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, TDataPartSet& partSet) {
+ TRope &outBuffer = partSet.FullDataFragment.OwnedRope;
+ ui32 totalParts = type.TotalPartCount();
+ Y_VERIFY(partSet.Parts.size() >= totalParts);
+
+ if (outBuffer.GetSize()) {
+ outBuffer = TRope();
+ }
+
+ ui32 missingDataPartIdxA = totalParts;
+ ui32 missingDataPartIdxB = totalParts;
+ ui32 missingDataPartCount = 0;
+ ui64 expectedPartSize = type.PartSize(crcMode, partSet.FullDataSize);
+
+ ui32 i = 0;
+ for (; i < totalParts; ++i) {
+ if (!(partSet.PartsMask & (1 << i))) {
+ missingDataPartIdxA = i;
+ ++missingDataPartCount;
+ break;
+ } else {
+ Y_VERIFY(partSet.Parts[i].size() == expectedPartSize, "partSet.Parts[%" PRIu32 "].size(): %" PRIu64
+ " expectedPartSize: %" PRIu64 " erasure: %s partSet.FullDataSize: %" PRIu64,
+ (ui32)i, (ui64)partSet.Parts[i].size(), expectedPartSize, type.ErasureName[type.GetErasure()].data(),
+ (ui64)partSet.FullDataSize);
+ }
+ }
+ ++i;
+ for (; i < totalParts; ++i) {
+ if (!(partSet.PartsMask & (1 << i))) {
+ missingDataPartIdxB = i;
+ ++missingDataPartCount;
+ } else {
+ Y_VERIFY(partSet.Parts[i].size() == expectedPartSize, "partSet.Parts[%" PRIu32 "].size()# %" PRIu32
+ " != expectedPartSize# %" PRIu32 " erasure: %s partSet.FullDataSize: %" PRIu64,
+ (ui32)i, (ui32)partSet.Parts[i].size(), (ui32)expectedPartSize, type.ErasureName[type.GetErasure()].data(),
+ (ui64)partSet.FullDataSize);
+ }
+ }
+ Y_VERIFY(missingDataPartCount <= 2);
+
+ ui64 dataSize = partSet.FullDataSize;
+ if (restoreParts) {
+ if (missingDataPartIdxA != totalParts) {
+ TRACE("Line# " << __LINE__ << Endl);
+ Refurbish(partSet.Parts[missingDataPartIdxA], expectedPartSize);
+ }
+ if (missingDataPartIdxB != totalParts) {
+ TRACE("Line# " << __LINE__ << Endl);
+ Refurbish(partSet.Parts[missingDataPartIdxB], expectedPartSize);
+ }
+ }
+
+ TBlockParams p(crcMode, type, dataSize);
+
+ if (restoreFullData) {
+ p.GlueOutBuffer<restoreParts>(outBuffer, partSet, missingDataPartIdxA, missingDataPartIdxB);
+ } else if (missingDataPartCount == 0) {
+ return;
+ }
+
+ if (missingDataPartCount == 2) {
+ VERBOSE_COUT("missing parts " << missingDataPartIdxA << " and " << missingDataPartIdxB << Endl);
+ } else if (missingDataPartCount == 1) {
+ VERBOSE_COUT("missing part " << missingDataPartIdxA << Endl);
+ }
+
+ // Restore the fast way if all data parts are present
+ if (missingDataPartCount == 0 ||
+ (!restoreParts && missingDataPartIdxA >= p.TotalParts - 2)) {
+ VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
+ return;
+ }
+
+ // Prepare output data pointers
+ if (restoreFullData) {
+ p.PrepareInputDataPointers(outBuffer.Begin());
+ }
+
+ // Consider failed disk cases
+ // a) < m
+ // b) m
+ // 'xor-restore'
+ // d) m, m+1
+ // TODO: 1-pass
+ // just glue the data
+ // use 'eo split' to restore the remaining parts
+ // f) <m, m+1
+ // use 'xor-restore' to restore the data
+ // TODO: use 2-nd part of 'eo-split' to restore m+1 part
+ // TODO: 1-pass
+ if (missingDataPartIdxA <= p.DataParts && missingDataPartIdxB >= p.TotalParts - 1) {
+ if (!restoreFullData && restoreParts && missingDataPartIdxB == p.TotalParts - 1) {
+ // The (f1) case, but no full data needed, only parts
+ TRACE("case# f1" << Endl);
+ VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
+
+ p.XorRestorePart<true, false, false>(partSet, missingDataPartIdxA);
+ TRACE("case# f1 split" << Endl);
+ p.EoSplit(partSet);
+ p.PadAndCrcPart(partSet, missingDataPartIdxA);
+ p.PadAndCrcPart(partSet, missingDataPartIdxB);
+ } else {
+ // Cases (a), (b) and (d2), case (f2) with full data and maybe parts needed
+ TRACE("case# a b d2 f2" << Endl);
+ VERBOSE_COUT(__LINE__ << " of " << __FILE__ << " missing " << missingDataPartIdxA << Endl);
+ p.XorRestorePart<restoreParts, restoreFullData, restoreParityParts>(partSet, missingDataPartIdxA);
+ if (restoreParts && missingDataPartIdxB == p.TotalParts - 1 && restoreParityParts) {
+ // The (d2a) or (f2a) case with full data and parts needed
+ TRACE("case# d2a f2a" << Endl);
+ VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
+ p.EoSplit(partSet);
+ p.PadAndCrcPart(partSet, missingDataPartIdxB);
+ }
+ if (restoreParts) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxA);
+ }
+ }
+ return;
+ }
+
+ // c) m+1
+ // TODO: use 2-nd part of 'eo-split' to restore m+1 part, while glueing the data
+ // TODO: 1-pass
+ // just glue the data
+ // use 'eo split' to restore the missing part
+ if (missingDataPartIdxA == p.TotalParts - 1 && missingDataPartIdxB == p.TotalParts) {
+ TRACE("case# c" << Endl);
+ VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
+ if (!restoreFullData) {
+ TRACE(__LINE__ << Endl);
+ if (!restoreParityParts) {
+ TRACE(__LINE__ << Endl);
+ return;
+ }
+ TRACE(__LINE__ << Endl);
+ }
+ if (restoreParts) {
+ TRACE(__LINE__ << Endl);
+ VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
+ p.EoSplit(partSet);
+ p.PadAndCrcPart(partSet, missingDataPartIdxA);
+ }
+ return;
+ }
+
+ // e) <m, m
+ // TODO: 1-pass
+ // use diagonal-sums to restore the data
+ // use 'xor restore' with 'restore part' to restore m part
+ if (missingDataPartIdxA < p.DataParts && missingDataPartIdxB == p.DataParts) {
+ TRACE(__LINE__ << " of " << __FILE__ << " case# e restore part missing# " << missingDataPartIdxA << ", " << missingDataPartIdxB <<
+ " restoreParts# " << restoreParts
+ << " restoreParityParts# " << restoreParityParts
+ << " restoreFullData# " << restoreFullData << Endl);
+ p.EoDiagonalRestorePart<restoreParts, restoreFullData, false, restoreParityParts>(partSet, missingDataPartIdxA);
+ if (restoreParts) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxA);
+ if (restoreParityParts) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxB);
+ }
+ }
+ return;
+ }
+
+ // g) <m, <m
+ // the main case :(
+ TRACE("case# g" << Endl);
+ VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
+ Y_VERIFY(missingDataPartIdxA < p.DataParts && missingDataPartIdxB < p.DataParts);
+ p.EoMainRestoreParts<restoreParts, restoreFullData, false, restoreParityParts>(partSet, missingDataPartIdxA,
+ missingDataPartIdxB);
+ if (restoreParts) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxA);
+ p.PadAndCrcPart(partSet, missingDataPartIdxB);
+ }
+}
+
+// restorePartiyParts may be set only togehter with restore parts
+template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
+void StarBlockRestore(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, TDataPartSet& partSet) {
+ Y_VERIFY(partSet.Is8Aligned());
+ TRope &outBuffer = partSet.FullDataFragment.OwnedRope;
+
+ ui32 totalParts = type.TotalPartCount();
+ Y_VERIFY(partSet.Parts.size() == totalParts);
+
+ ui32 missingDataPartIdxA = totalParts;
+ ui32 missingDataPartIdxB = totalParts;
+ ui32 missingDataPartIdxC = totalParts;
+ ui32 missingDataPartCount = 0;
+ ui64 expectedPartSize = type.PartSize(crcMode, partSet.FullDataSize); // ???
+ ui32 i = 0;
+ for (; i < totalParts; ++i) {
+ if (!(partSet.PartsMask & (1 << i))) {
+ missingDataPartIdxA = i;
+ ++missingDataPartCount;
+ break;
+ } else {
+ Y_VERIFY(partSet.Parts[i].size() == expectedPartSize, "partSet.Parts[%" PRIu32 "].size(): %" PRIu64
+ " expectedPartSize: %" PRIu64 " erasure: %s partSet.FullDataSize: %" PRIu64,
+ (ui32)i, (ui64)partSet.Parts[i].size(), expectedPartSize, type.ErasureName[type.GetErasure()].data(),
+ (ui64)partSet.FullDataSize);
+ }
+ }
+ ++i;
+ for (; i < totalParts; ++i) {
+ if (!(partSet.PartsMask & (1 << i))) {
+ missingDataPartIdxB = i;
+ ++missingDataPartCount;
+ break;
+ } else {
+ Y_VERIFY(partSet.Parts[i].size() == expectedPartSize, "partSet.Parts[%" PRIu32 "].size()# %" PRIu32
+ " != expectedPartSize# %" PRIu32 " erasure: %s partSet.FullDataSize: %" PRIu64,
+ (ui32)i, (ui32)partSet.Parts[i].size(), (ui32)expectedPartSize, type.ErasureName[type.GetErasure()].data(),
+ (ui64)partSet.FullDataSize);
+ }
+ }
+ ++i;
+ for (; i < totalParts; ++i) {
+ if (!(partSet.PartsMask & (1 << i))) {
+ missingDataPartIdxC = i;
+ ++missingDataPartCount;
+ break;
+ } else {
+ Y_VERIFY(partSet.Parts[i].size() == expectedPartSize, "partSet.Parts[%" PRIu32 "].size()# %" PRIu32
+ " != expectedPartSize# %" PRIu32 " erasure: %s partSet.FullDataSize: %" PRIu64,
+ (ui32)i, (ui32)partSet.Parts[i].size(), (ui32)expectedPartSize, type.ErasureName[type.GetErasure()].data(),
+ (ui64)partSet.FullDataSize);
+ }
+ }
+ Y_VERIFY(missingDataPartCount <= 3);
+
+ if (restoreParts) {
+ if (missingDataPartIdxA != totalParts) {
+ TRACE("Line# " << __LINE__ << Endl);
+ Refurbish(partSet.Parts[missingDataPartIdxA], expectedPartSize);
+ }
+ if (missingDataPartIdxB != totalParts) {
+ TRACE("Line# " << __LINE__ << Endl);
+ Refurbish(partSet.Parts[missingDataPartIdxB], expectedPartSize);
+ }
+ if (missingDataPartIdxC != totalParts) {
+ TRACE("Line# " << __LINE__ << Endl);
+ Refurbish(partSet.Parts[missingDataPartIdxC], expectedPartSize);
+ }
+ }
+ if (missingDataPartCount == 3) {
+ VERBOSE_COUT("missing parts " << missingDataPartIdxA << " and " << missingDataPartIdxB <<
+ " and " << missingDataPartIdxC << Endl);
+ } else if (missingDataPartCount == 2) {
+ VERBOSE_COUT("missing parts " << missingDataPartIdxA << " and " << missingDataPartIdxB << Endl);
+ } else if (missingDataPartCount == 1) {
+ VERBOSE_COUT("missing part " << missingDataPartIdxA << Endl);
+ }
+
+ ui64 dataSize = partSet.FullDataSize;
+ TBlockParams p(crcMode, type, dataSize);
+ if (restoreFullData) {
+ Refurbish(outBuffer, dataSize);
+ p.PrepareInputDataPointers(outBuffer.Begin());
+ } else if (missingDataPartCount == 0) {
+ return;
+ }
+
+ // Restore the fast way if all data parts are present
+ if (missingDataPartCount == 0 ||
+ (!restoreParts && missingDataPartIdxA >= p.DataParts)) {
+ VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
+ p.GlueBlockPartsMemcpy(outBuffer.Begin(), partSet);
+ return;
+ }
+
+
+ // All possible failures of 2 disks which EVENODD capable to handle
+ if (missingDataPartCount <= 2 && missingDataPartIdxA != p.TotalParts - 1
+ && missingDataPartIdxB != p.TotalParts - 1) {
+ if (p.DataParts == 4) {
+ TRopeErasureType typeEO(TRopeErasureType::EErasureSpecies::Erasure4Plus2Block);
+ EoBlockRestore<restoreParts, restoreFullData, restoreParityParts>(crcMode, typeEO, partSet);
+ } else if (p.DataParts == 3) {
+ TRopeErasureType typeEO(TRopeErasureType::EErasureSpecies::Erasure3Plus2Block);
+ EoBlockRestore<restoreParts, restoreFullData, restoreParityParts>(crcMode, typeEO, partSet);
+ } else if (p.DataParts == 2) {
+ TRopeErasureType typeEO(TRopeErasureType::EErasureSpecies::Erasure2Plus2Block);
+ EoBlockRestore<restoreParts, restoreFullData, restoreParityParts>(crcMode, typeEO, partSet);
+ }
+ return;
+ }
+ if (missingDataPartIdxA == p.TotalParts - 1
+ || missingDataPartIdxB == p.TotalParts - 1
+ || missingDataPartIdxC == p.TotalParts - 1) {
+ // Possible combinations handled in this branch
+ // '+' stands for part, which is present for sure,
+ // '-' stands for part, which is missing for sure,
+ // series of 0, 1 and 2 means that there are n missing parts in this region
+ // 0 0 0 0 0 0 - or 1 1 1 1 1 1 - or 2 2 2 2 2 2 -
+ if (p.DataParts == 4) {
+ TRopeErasureType typeEO(TRopeErasureType::EErasureSpecies::Erasure4Plus2Block);
+ EoBlockRestore<restoreParts, restoreFullData, restoreParityParts>(crcMode, typeEO, partSet);
+ } else if (p.DataParts == 3) {
+ TRopeErasureType typeEO(TRopeErasureType::EErasureSpecies::Erasure3Plus2Block);
+ EoBlockRestore<restoreParts, restoreFullData, restoreParityParts>(crcMode, typeEO, partSet);
+ } else if (p.DataParts == 2) {
+ TRopeErasureType typeEO(TRopeErasureType::EErasureSpecies::Erasure2Plus2Block);
+ EoBlockRestore<restoreParts, restoreFullData, restoreParityParts>(crcMode, typeEO, partSet);
+ }
+ if (restoreParts) {
+ if (restoreParityParts) {
+ p.StarSplit<true>(partSet);
+ }
+ if (missingDataPartIdxA < (restoreParityParts ? p.TotalParts : p.DataParts)) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxA);
+ }
+ if (missingDataPartIdxB < (restoreParityParts ? p.TotalParts : p.DataParts)) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxB);
+ }
+ if (missingDataPartIdxC < (restoreParityParts ? p.TotalParts : p.DataParts)) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxC);
+ }
+ }
+ return;
+ }
+ // There are remain only cases with missingDataPartCount == 3
+ if ( missingDataPartIdxC == p.DataParts + 1) {
+ if (missingDataPartIdxB < p.DataParts) {
+ // 2 2 2 2 + - +
+ // "It can be decoded with slightly modification of the EVENODD decoding" (c)
+ p.EoMainRestoreParts<restoreParts, restoreFullData, true, restoreParityParts>(partSet, missingDataPartIdxA,
+ missingDataPartIdxB);
+ } else {
+ // 1 1 1 1 - - +
+ p.EoDiagonalRestorePart<restoreParts, restoreFullData, true, restoreParityParts>(partSet, missingDataPartIdxA);
+ }
+ if (restoreParts) {
+ if (restoreParityParts) {
+ p.StarSplit<!restoreFullData>(partSet);
+ }
+ if (missingDataPartIdxA < (restoreParityParts ? p.TotalParts : p.DataParts)) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxA);
+ }
+ if (missingDataPartIdxB < (restoreParityParts ? p.TotalParts : p.DataParts)) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxB);
+ }
+ if (missingDataPartIdxC < (restoreParityParts ? p.TotalParts : p.DataParts)) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxC);
+ }
+ }
+ return;
+ }
+ if (missingDataPartIdxC == p.DataParts) {
+ // 2 2 2 2 - + +
+ if (! restoreParts) {
+ TRACE("Line# " << __LINE__ << Endl);
+ Refurbish(partSet.Parts[missingDataPartIdxC], expectedPartSize);
+ }
+ p.StarRestoreHorizontalPart<restoreParts, restoreFullData, restoreParityParts>(partSet,
+ missingDataPartIdxA, missingDataPartIdxB);
+ if (restoreParts) {
+ if (missingDataPartIdxA < (restoreParityParts ? p.TotalParts : p.DataParts)) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxA);
+ }
+ if (missingDataPartIdxB < (restoreParityParts ? p.TotalParts : p.DataParts)) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxB);
+ }
+ if (missingDataPartIdxC < (restoreParityParts ? p.TotalParts : p.DataParts)) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxC);
+ }
+ }
+ return;
+ }
+
+ VERBOSE_COUT(__LINE__ << " of " << __FILE__ << Endl);
+ Y_VERIFY(missingDataPartIdxA < p.DataParts && missingDataPartIdxB < p.DataParts
+ && missingDataPartIdxC < p.DataParts);
+ // Two possible cases:
+ // - Symmetric
+ // - Asymmetric
+ // But for m = 5 it is always possible to change asymmetric to symmetric by shifting
+ ui32 m = ErasureSpeciesParameters[TRopeErasureType::EErasureSpecies::Erasure4Plus3Block].Prime;
+ while ((m + missingDataPartIdxB - missingDataPartIdxA) % m != (m + missingDataPartIdxC - missingDataPartIdxB) % m ) {
+ ui32 tmp = missingDataPartIdxA;
+ missingDataPartIdxA = missingDataPartIdxB;
+ missingDataPartIdxB = missingDataPartIdxC;
+ missingDataPartIdxC = tmp;
+ }
+ if (! restoreParts) {
+ TRACE("Line# " << __LINE__ << Endl);
+ Refurbish(partSet.Parts[missingDataPartIdxB], expectedPartSize);
+ }
+ p.StarMainRestorePartsSymmetric<restoreParts, restoreFullData, restoreParityParts>(partSet,
+ missingDataPartIdxA, missingDataPartIdxB, missingDataPartIdxC);
+ if (restoreParts) {
+ if (missingDataPartIdxA < (restoreParityParts ? p.TotalParts : p.DataParts)) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxA);
+ }
+ if (missingDataPartIdxB < (restoreParityParts ? p.TotalParts : p.DataParts)) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxB);
+ }
+ if (missingDataPartIdxC < (restoreParityParts ? p.TotalParts : p.DataParts)) {
+ p.PadAndCrcPart(partSet, missingDataPartIdxC);
+ }
+ }
+}
+
+template <bool restoreParts, bool restoreFullData, bool restoreParityParts>
+void XorBlockRestore(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, TDataPartSet &partSet) {
+ TRope &outBuffer = partSet.FullDataFragment.OwnedRope;
+ ui32 totalParts = type.TotalPartCount();
+ Y_VERIFY(partSet.Parts.size() == totalParts,
+ "partSet.Parts.size(): %" PRIu64 " totalParts: %" PRIu32 " erasure: %s",
+ (ui64)partSet.Parts.size(), (ui32)totalParts, type.ErasureName[type.GetErasure()].data());
+
+ ui32 missingDataPartIdx = totalParts;
+ ui32 missingDataPartCount = 0;
+ ui64 expectedPartSize = type.PartSize(crcMode, partSet.FullDataSize);
+ for (ui32 i = 0; i < totalParts; ++i) {
+ if (!(partSet.PartsMask & (1 << i))) {
+ missingDataPartIdx = i;
+ ++missingDataPartCount;
+ } else {
+ Y_VERIFY(partSet.Parts[i].size() == expectedPartSize, "partSet.Parts[%" PRIu32 "].size(): %" PRIu64
+ " expectedPartSize: %" PRIu64 " erasure: %s partSet.FullDataSize: %" PRIu64,
+ (ui32)i, (ui64)partSet.Parts[i].size(), expectedPartSize, type.ErasureName[type.GetErasure()].data(),
+ (ui64)partSet.FullDataSize);
+ }
+ }
+ Y_VERIFY(missingDataPartCount <= 1);
+
+ ui64 dataSize = partSet.FullDataSize;
+ if (restoreParts && missingDataPartIdx != totalParts) {
+ TRACE("Line# " << __LINE__ << Endl);
+ Refurbish(partSet.Parts[missingDataPartIdx], partSet.Parts[missingDataPartIdx == 0 ? 1 : 0].size());
+ }
+ if (restoreFullData) {
+ Refurbish(outBuffer, dataSize);
+ } else if (missingDataPartCount == 0) {
+ return;
+ }
+
+ TBlockParams p(crcMode, type, dataSize);
+
+ // Restore the fast way if all data parts are present
+ if (missingDataPartCount == 0 ||
+ (missingDataPartCount == 1 && !restoreParts && missingDataPartIdx == p.TotalParts - 1)) {
+ p.GlueBlockPartsMemcpy(outBuffer.Begin(), partSet);
+ return;
+ }
+ // Prepare output data pointers
+ if (restoreFullData) {
+ p.PrepareInputDataPointers(outBuffer.Begin());
+ }
+
+ p.XorRestorePart<restoreParts, restoreFullData, restoreParityParts>(partSet, missingDataPartIdx);
+}
+
+const std::array<TString, TRopeErasureType::ErasureSpeciesCount> TRopeErasureType::ErasureName{{
+ "none",
+ "mirror-3",
+ "block-3-1",
+ "stripe-3-1",
+ "block-4-2",
+ "block-3-2",
+ "stripe-4-2",
+ "stripe-3-2",
+ "mirror-3-2",
+ "mirror-3-dc",
+ "block-4-3",
+ "stripe-4-3",
+ "block-3-3",
+ "stripe-3-3",
+ "block-2-3",
+ "stripe-2-3",
+ "block-2-2",
+ "stripe-2-2",
"mirror-3of4",
-}};
-
-ui32 TRopeErasureType::ParityParts() const {
- const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
- return erasure.ParityParts;
-}
-
-ui32 TRopeErasureType::DataParts() const {
- const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
- return erasure.DataParts;
-}
-
-ui32 TRopeErasureType::TotalPartCount() const {
- const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
- return erasure.DataParts + erasure.ParityParts;
-}
-
-ui32 TRopeErasureType::MinimalRestorablePartCount() const {
- const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
- return erasure.DataParts;
-}
-
-/*
-ui32 TRopeErasureType::PartialRestoreStep() const {
- const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
- switch (erasure.ErasureFamily) {
- case TRopeErasureType::ErasureMirror:
- return 1;
- case TRopeErasureType::ErasureParityStripe:
- if (erasure.ParityParts == 1) {
- return erasure.DataParts * sizeof(ui64);
- }
- return erasure.DataParts * (erasure.Prime - 1) * sizeof(ui64);
- case TRopeErasureType::ErasureParityBlock:
- if (erasure.ParityParts == 1) {
- return sizeof(ui64);
- }
- return (erasure.Prime - 1) * sizeof(ui64);
- }
- ythrow TWithBackTrace<yexception>() << "Unknown ErasureFamily = " << (i32)erasure.ErasureFamily;
-}*/
-
-ui32 TRopeErasureType::MinimalBlockSize() const {
- const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
- switch (erasure.ErasureFamily) {
- case TRopeErasureType::ErasureMirror:
- return 1;
- case TRopeErasureType::ErasureParityBlock:
- if (erasure.ParityParts == 1) {
- return erasure.DataParts * sizeof(ui64);
- }
- if (erasure.ParityParts == 2) {
- return (erasure.Prime - 1) * erasure.DataParts * sizeof(ui64);
- }
- if (erasure.ParityParts == 3) {
- return (erasure.Prime - 1) * erasure.DataParts * sizeof(ui64);
- }
- ythrow TWithBackTrace<yexception>() << "Unsupported partiy part count = " << erasure.ParityParts <<
- " for ErasureFamily = " << (i32)erasure.ErasureFamily;
- default:
- ythrow TWithBackTrace<yexception>() << "Unknown ErasureFamily = " << (i32)erasure.ErasureFamily;
- }
-}
-
-ui64 TRopeErasureType::PartUserSize(ui64 dataSize) const {
- const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
- switch (erasure.ErasureFamily) {
- case TRopeErasureType::ErasureMirror:
- return dataSize;
- case TRopeErasureType::ErasureParityBlock:
- {
- ui32 blockSize = MinimalBlockSize();
- ui64 dataSizeBlocks = (dataSize + blockSize - 1) / blockSize;
- ui64 partSize = dataSizeBlocks * sizeof(ui64) * (erasure.ParityParts == 1 ? 1 : (erasure.Prime - 1));
- return partSize;
- }
- default:
- ythrow TWithBackTrace<yexception>() << "Unknown ErasureFamily = " << (i32)erasure.ErasureFamily;
- }
-}
-
-ui64 TRopeErasureType::PartSize(ECrcMode crcMode, ui64 dataSize) const {
- const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
- switch (erasure.ErasureFamily) {
- case TRopeErasureType::ErasureMirror:
- switch (crcMode) {
- case CrcModeNone:
- return dataSize;
- case CrcModeWholePart:
- if (dataSize) {
- return dataSize + sizeof(ui32);
- } else {
- return 0;
- }
- }
- ythrow TWithBackTrace<yexception>() << "Unknown crcMode = " << (i32)crcMode;
- case TRopeErasureType::ErasureParityBlock:
- {
- ui32 blockSize = MinimalBlockSize();
- ui64 dataSizeBlocks = (dataSize + blockSize - 1) / blockSize;
- ui64 partSize = dataSizeBlocks * sizeof(ui64) * (erasure.ParityParts == 1 ? 1 : (erasure.Prime - 1));
- switch (crcMode) {
- case CrcModeNone:
- return partSize;
- case CrcModeWholePart:
- return partSize + sizeof(ui32);
- }
- ythrow TWithBackTrace<yexception>() << "Unknown crcMode = " << (i32)crcMode;
- }
- default:
- ythrow TWithBackTrace<yexception>() << "Unknown ErasureFamily = " << (i32)erasure.ErasureFamily;
- }
-}
-
-ui32 TRopeErasureType::Prime() const {
- const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
- return erasure.Prime;
-}
-
-// Block consists of columns.
-// block = [column1, column2, ... ,columnN], where N == erasure.DataParts
-//
-// Input partitioning:
-// | large, ... | small, ... | small + tail |
-
-void TRopeErasureType::BlockSplitRange(ECrcMode crcMode, ui64 blobSize, ui64 wholeBegin, ui64 wholeEnd,
- TBlockSplitRange *outRange) const {
- Y_VERIFY(wholeBegin <= wholeEnd && outRange, "wholeBegin# %" PRIu64 " wholeEnd# %" PRIu64 " outRange# %" PRIu64,
- wholeBegin, wholeEnd, (ui64)(intptr_t)outRange);
- Y_UNUSED(crcMode);
- const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
- const ui64 blockSize = MinimalBlockSize();
- const ui64 dataParts = erasure.DataParts;
- const ui64 columnSize = blockSize / dataParts;
-
- const ui64 wholeColumns = blobSize / columnSize;
-
- const ui64 smallPartColumns = wholeColumns / dataParts;
- const ui64 largePartColumns = smallPartColumns + 1;
-
- const ui64 smallPartSize = smallPartColumns * columnSize;
- const ui64 largePartSize = largePartColumns * columnSize;
-
- const ui32 firstSmallPartIdx = wholeColumns % dataParts;
- const ui64 lastPartSize = blobSize
- - largePartSize * firstSmallPartIdx
- - smallPartSize * (dataParts - firstSmallPartIdx - 1);
- const ui64 alignedLastPartSize = ((lastPartSize + columnSize - 1) / columnSize) * columnSize;
-
- const ui64 alignedPartSize = Max<ui64>(alignedLastPartSize, (firstSmallPartIdx ? largePartSize : smallPartSize));
-
- outRange->BeginPartIdx = Max<ui64>();
- outRange->EndPartIdx = Max<ui64>();
- outRange->PartRanges.resize(dataParts);
- ui64 partBeginWholeOffset = 0;
- ui64 partEndWholeOffset = 0;
- ui32 partIdx = 0;
- for (; partIdx < firstSmallPartIdx; ++partIdx) {
- TPartOffsetRange &out = outRange->PartRanges[partIdx];
- partEndWholeOffset = partBeginWholeOffset + largePartSize;
- if (!largePartSize) {
- out.Reset();
- } else {
- // pb----pe pb-----pe pb-----pe pb----pe pb-----pe
- // wb----------------we
- // wb < pe && we > pb
- if (wholeBegin < partEndWholeOffset && wholeEnd > partBeginWholeOffset) {
- if (wholeBegin >= partBeginWholeOffset) {
- // first part
- outRange->BeginPartIdx = partIdx;
- out.Begin = wholeBegin - partBeginWholeOffset;
- out.AlignedBegin = (out.Begin / columnSize) * columnSize;
- out.WholeBegin = wholeBegin;
- out.AlignedWholeBegin = wholeBegin + out.AlignedBegin - out.Begin;
- } else {
- // not first part
- out.Begin = 0;
- out.AlignedBegin = 0;
- out.WholeBegin = partBeginWholeOffset;
- out.AlignedWholeBegin = partBeginWholeOffset;
- }
-
- if (wholeEnd <= partEndWholeOffset) {
- // last part
- outRange->EndPartIdx = partIdx + 1;
- out.End = wholeEnd - partBeginWholeOffset;
- out.AlignedEnd = ((out.End + columnSize - 1) / columnSize) * columnSize;
- out.WholeEnd = wholeEnd;
- } else {
- // not last part
- out.End = largePartSize;
- out.AlignedEnd = largePartSize;
- out.WholeEnd = partEndWholeOffset;
- }
- } else {
- out.Reset();
- }
- }
- partBeginWholeOffset = partEndWholeOffset;
- }
- for (; partIdx < dataParts - 1; ++partIdx) {
- TPartOffsetRange &out = outRange->PartRanges[partIdx];
- partEndWholeOffset = partBeginWholeOffset + smallPartSize;
- if (!smallPartSize) {
- out.Reset();
- } else {
- // pb----pe pb-----pe pb-----pe pb----pe pb-----pe
- // wb----------------we
- // wb < pe && we > pb
- if (wholeBegin < partEndWholeOffset && wholeEnd > partBeginWholeOffset) {
- if (wholeBegin >= partBeginWholeOffset) {
- // first part
- outRange->BeginPartIdx = partIdx;
- out.Begin = wholeBegin - partBeginWholeOffset;
- out.AlignedBegin = (out.Begin / columnSize) * columnSize;
- out.WholeBegin = wholeBegin;
- out.AlignedWholeBegin = wholeBegin + out.AlignedBegin - out.Begin;
- } else {
- // not first part
- out.Begin = 0;
- out.AlignedBegin = 0;
- out.WholeBegin = partBeginWholeOffset;
- out.AlignedWholeBegin = partBeginWholeOffset;
- }
-
- if (wholeEnd <= partEndWholeOffset) {
- // last part
- outRange->EndPartIdx = partIdx + 1;
- out.End = wholeEnd - partBeginWholeOffset;
- out.AlignedEnd = ((out.End + columnSize - 1) / columnSize) * columnSize;
- out.WholeEnd = wholeEnd;
- } else {
- // not last part
- out.End = smallPartSize;
- // Align up to the large part size for restoration purposes
- out.AlignedEnd = alignedPartSize;
- out.WholeEnd = partEndWholeOffset;
- }
- } else {
- out.Reset();
- }
- }
- partBeginWholeOffset = partEndWholeOffset;
- }
- // last part
- {
- TPartOffsetRange &out = outRange->PartRanges[partIdx];
- partEndWholeOffset = blobSize;
- if (!lastPartSize) {
- out.Reset();
- } else {
- // pb----pe pb-----pe pb-----pe pb----pe pb-----pe
- // wb----------------we
- // wb < pe && we > pb
- if (wholeBegin < partEndWholeOffset && wholeEnd > partBeginWholeOffset) {
- if (wholeBegin >= partBeginWholeOffset) {
- // first part
- outRange->BeginPartIdx = partIdx;
- out.Begin = wholeBegin - partBeginWholeOffset;
- out.AlignedBegin = (out.Begin / columnSize) * columnSize;
- out.WholeBegin = wholeBegin;
- out.AlignedWholeBegin = wholeBegin + out.AlignedBegin - out.Begin;
- } else {
- // not first part
- out.Begin = 0;
- out.AlignedBegin = 0;
- out.WholeBegin = partBeginWholeOffset;
- out.AlignedWholeBegin = partBeginWholeOffset;
- }
-
- if (wholeBegin >= partBeginWholeOffset || wholeEnd < partEndWholeOffset) {
- // part of the last part
- out.End = wholeEnd - partBeginWholeOffset;
- out.AlignedEnd = ((out.End + columnSize - 1) / columnSize) * columnSize;
- out.WholeEnd = wholeEnd;
- } else {
- // up to the end
- out.End = lastPartSize;
- out.AlignedEnd = alignedPartSize;
- out.WholeEnd = partEndWholeOffset;
- }
- // It IS the LAST data part of the blob
- outRange->EndPartIdx = partIdx + 1;
- } else {
- out.Reset();
- }
- }
- }
- Y_VERIFY_DEBUG(outRange->EndPartIdx != Max<ui64>());
-}
-
-void MirrorSplit(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, const TRope& buffer,
- TDataPartSet& outPartSet) {
- outPartSet.FullDataSize = buffer.GetSize();
- outPartSet.Parts.resize(type.TotalPartCount());
- TString partBuffer;
- ui32 parityParts = type.ParityParts();
- switch (crcMode) {
- case TRopeErasureType::CrcModeNone:
- for (ui32 partIdx = 0; partIdx <= parityParts; ++partIdx) {
- outPartSet.Parts[partIdx].ReferenceTo(buffer);
- outPartSet.PartsMask |= (1 << partIdx);
- }
- outPartSet.MemoryConsumed = buffer.GetSize();
- return;
- case TRopeErasureType::CrcModeWholePart:
- {
- ui64 partSize = type.PartSize(crcMode, buffer.GetSize());
- TRope& part = outPartSet.FullDataFragment.OwnedRope;
- part = buffer;
+}};
+
+ui32 TRopeErasureType::ParityParts() const {
+ const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
+ return erasure.ParityParts;
+}
+
+ui32 TRopeErasureType::DataParts() const {
+ const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
+ return erasure.DataParts;
+}
+
+ui32 TRopeErasureType::TotalPartCount() const {
+ const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
+ return erasure.DataParts + erasure.ParityParts;
+}
+
+ui32 TRopeErasureType::MinimalRestorablePartCount() const {
+ const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
+ return erasure.DataParts;
+}
+
+/*
+ui32 TRopeErasureType::PartialRestoreStep() const {
+ const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
+ switch (erasure.ErasureFamily) {
+ case TRopeErasureType::ErasureMirror:
+ return 1;
+ case TRopeErasureType::ErasureParityStripe:
+ if (erasure.ParityParts == 1) {
+ return erasure.DataParts * sizeof(ui64);
+ }
+ return erasure.DataParts * (erasure.Prime - 1) * sizeof(ui64);
+ case TRopeErasureType::ErasureParityBlock:
+ if (erasure.ParityParts == 1) {
+ return sizeof(ui64);
+ }
+ return (erasure.Prime - 1) * sizeof(ui64);
+ }
+ ythrow TWithBackTrace<yexception>() << "Unknown ErasureFamily = " << (i32)erasure.ErasureFamily;
+}*/
+
+ui32 TRopeErasureType::MinimalBlockSize() const {
+ const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
+ switch (erasure.ErasureFamily) {
+ case TRopeErasureType::ErasureMirror:
+ return 1;
+ case TRopeErasureType::ErasureParityBlock:
+ if (erasure.ParityParts == 1) {
+ return erasure.DataParts * sizeof(ui64);
+ }
+ if (erasure.ParityParts == 2) {
+ return (erasure.Prime - 1) * erasure.DataParts * sizeof(ui64);
+ }
+ if (erasure.ParityParts == 3) {
+ return (erasure.Prime - 1) * erasure.DataParts * sizeof(ui64);
+ }
+ ythrow TWithBackTrace<yexception>() << "Unsupported partiy part count = " << erasure.ParityParts <<
+ " for ErasureFamily = " << (i32)erasure.ErasureFamily;
+ default:
+ ythrow TWithBackTrace<yexception>() << "Unknown ErasureFamily = " << (i32)erasure.ErasureFamily;
+ }
+}
+
+ui64 TRopeErasureType::PartUserSize(ui64 dataSize) const {
+ const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
+ switch (erasure.ErasureFamily) {
+ case TRopeErasureType::ErasureMirror:
+ return dataSize;
+ case TRopeErasureType::ErasureParityBlock:
+ {
+ ui32 blockSize = MinimalBlockSize();
+ ui64 dataSizeBlocks = (dataSize + blockSize - 1) / blockSize;
+ ui64 partSize = dataSizeBlocks * sizeof(ui64) * (erasure.ParityParts == 1 ? 1 : (erasure.Prime - 1));
+ return partSize;
+ }
+ default:
+ ythrow TWithBackTrace<yexception>() << "Unknown ErasureFamily = " << (i32)erasure.ErasureFamily;
+ }
+}
+
+ui64 TRopeErasureType::PartSize(ECrcMode crcMode, ui64 dataSize) const {
+ const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
+ switch (erasure.ErasureFamily) {
+ case TRopeErasureType::ErasureMirror:
+ switch (crcMode) {
+ case CrcModeNone:
+ return dataSize;
+ case CrcModeWholePart:
+ if (dataSize) {
+ return dataSize + sizeof(ui32);
+ } else {
+ return 0;
+ }
+ }
+ ythrow TWithBackTrace<yexception>() << "Unknown crcMode = " << (i32)crcMode;
+ case TRopeErasureType::ErasureParityBlock:
+ {
+ ui32 blockSize = MinimalBlockSize();
+ ui64 dataSizeBlocks = (dataSize + blockSize - 1) / blockSize;
+ ui64 partSize = dataSizeBlocks * sizeof(ui64) * (erasure.ParityParts == 1 ? 1 : (erasure.Prime - 1));
+ switch (crcMode) {
+ case CrcModeNone:
+ return partSize;
+ case CrcModeWholePart:
+ return partSize + sizeof(ui32);
+ }
+ ythrow TWithBackTrace<yexception>() << "Unknown crcMode = " << (i32)crcMode;
+ }
+ default:
+ ythrow TWithBackTrace<yexception>() << "Unknown ErasureFamily = " << (i32)erasure.ErasureFamily;
+ }
+}
+
+ui32 TRopeErasureType::Prime() const {
+ const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
+ return erasure.Prime;
+}
+
+// Block consists of columns.
+// block = [column1, column2, ... ,columnN], where N == erasure.DataParts
+//
+// Input partitioning:
+// | large, ... | small, ... | small + tail |
+
+void TRopeErasureType::BlockSplitRange(ECrcMode crcMode, ui64 blobSize, ui64 wholeBegin, ui64 wholeEnd,
+ TBlockSplitRange *outRange) const {
+ Y_VERIFY(wholeBegin <= wholeEnd && outRange, "wholeBegin# %" PRIu64 " wholeEnd# %" PRIu64 " outRange# %" PRIu64,
+ wholeBegin, wholeEnd, (ui64)(intptr_t)outRange);
+ Y_UNUSED(crcMode);
+ const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
+ const ui64 blockSize = MinimalBlockSize();
+ const ui64 dataParts = erasure.DataParts;
+ const ui64 columnSize = blockSize / dataParts;
+
+ const ui64 wholeColumns = blobSize / columnSize;
+
+ const ui64 smallPartColumns = wholeColumns / dataParts;
+ const ui64 largePartColumns = smallPartColumns + 1;
+
+ const ui64 smallPartSize = smallPartColumns * columnSize;
+ const ui64 largePartSize = largePartColumns * columnSize;
+
+ const ui32 firstSmallPartIdx = wholeColumns % dataParts;
+ const ui64 lastPartSize = blobSize
+ - largePartSize * firstSmallPartIdx
+ - smallPartSize * (dataParts - firstSmallPartIdx - 1);
+ const ui64 alignedLastPartSize = ((lastPartSize + columnSize - 1) / columnSize) * columnSize;
+
+ const ui64 alignedPartSize = Max<ui64>(alignedLastPartSize, (firstSmallPartIdx ? largePartSize : smallPartSize));
+
+ outRange->BeginPartIdx = Max<ui64>();
+ outRange->EndPartIdx = Max<ui64>();
+ outRange->PartRanges.resize(dataParts);
+ ui64 partBeginWholeOffset = 0;
+ ui64 partEndWholeOffset = 0;
+ ui32 partIdx = 0;
+ for (; partIdx < firstSmallPartIdx; ++partIdx) {
+ TPartOffsetRange &out = outRange->PartRanges[partIdx];
+ partEndWholeOffset = partBeginWholeOffset + largePartSize;
+ if (!largePartSize) {
+ out.Reset();
+ } else {
+ // pb----pe pb-----pe pb-----pe pb----pe pb-----pe
+ // wb----------------we
+ // wb < pe && we > pb
+ if (wholeBegin < partEndWholeOffset && wholeEnd > partBeginWholeOffset) {
+ if (wholeBegin >= partBeginWholeOffset) {
+ // first part
+ outRange->BeginPartIdx = partIdx;
+ out.Begin = wholeBegin - partBeginWholeOffset;
+ out.AlignedBegin = (out.Begin / columnSize) * columnSize;
+ out.WholeBegin = wholeBegin;
+ out.AlignedWholeBegin = wholeBegin + out.AlignedBegin - out.Begin;
+ } else {
+ // not first part
+ out.Begin = 0;
+ out.AlignedBegin = 0;
+ out.WholeBegin = partBeginWholeOffset;
+ out.AlignedWholeBegin = partBeginWholeOffset;
+ }
+
+ if (wholeEnd <= partEndWholeOffset) {
+ // last part
+ outRange->EndPartIdx = partIdx + 1;
+ out.End = wholeEnd - partBeginWholeOffset;
+ out.AlignedEnd = ((out.End + columnSize - 1) / columnSize) * columnSize;
+ out.WholeEnd = wholeEnd;
+ } else {
+ // not last part
+ out.End = largePartSize;
+ out.AlignedEnd = largePartSize;
+ out.WholeEnd = partEndWholeOffset;
+ }
+ } else {
+ out.Reset();
+ }
+ }
+ partBeginWholeOffset = partEndWholeOffset;
+ }
+ for (; partIdx < dataParts - 1; ++partIdx) {
+ TPartOffsetRange &out = outRange->PartRanges[partIdx];
+ partEndWholeOffset = partBeginWholeOffset + smallPartSize;
+ if (!smallPartSize) {
+ out.Reset();
+ } else {
+ // pb----pe pb-----pe pb-----pe pb----pe pb-----pe
+ // wb----------------we
+ // wb < pe && we > pb
+ if (wholeBegin < partEndWholeOffset && wholeEnd > partBeginWholeOffset) {
+ if (wholeBegin >= partBeginWholeOffset) {
+ // first part
+ outRange->BeginPartIdx = partIdx;
+ out.Begin = wholeBegin - partBeginWholeOffset;
+ out.AlignedBegin = (out.Begin / columnSize) * columnSize;
+ out.WholeBegin = wholeBegin;
+ out.AlignedWholeBegin = wholeBegin + out.AlignedBegin - out.Begin;
+ } else {
+ // not first part
+ out.Begin = 0;
+ out.AlignedBegin = 0;
+ out.WholeBegin = partBeginWholeOffset;
+ out.AlignedWholeBegin = partBeginWholeOffset;
+ }
+
+ if (wholeEnd <= partEndWholeOffset) {
+ // last part
+ outRange->EndPartIdx = partIdx + 1;
+ out.End = wholeEnd - partBeginWholeOffset;
+ out.AlignedEnd = ((out.End + columnSize - 1) / columnSize) * columnSize;
+ out.WholeEnd = wholeEnd;
+ } else {
+ // not last part
+ out.End = smallPartSize;
+ // Align up to the large part size for restoration purposes
+ out.AlignedEnd = alignedPartSize;
+ out.WholeEnd = partEndWholeOffset;
+ }
+ } else {
+ out.Reset();
+ }
+ }
+ partBeginWholeOffset = partEndWholeOffset;
+ }
+ // last part
+ {
+ TPartOffsetRange &out = outRange->PartRanges[partIdx];
+ partEndWholeOffset = blobSize;
+ if (!lastPartSize) {
+ out.Reset();
+ } else {
+ // pb----pe pb-----pe pb-----pe pb----pe pb-----pe
+ // wb----------------we
+ // wb < pe && we > pb
+ if (wholeBegin < partEndWholeOffset && wholeEnd > partBeginWholeOffset) {
+ if (wholeBegin >= partBeginWholeOffset) {
+ // first part
+ outRange->BeginPartIdx = partIdx;
+ out.Begin = wholeBegin - partBeginWholeOffset;
+ out.AlignedBegin = (out.Begin / columnSize) * columnSize;
+ out.WholeBegin = wholeBegin;
+ out.AlignedWholeBegin = wholeBegin + out.AlignedBegin - out.Begin;
+ } else {
+ // not first part
+ out.Begin = 0;
+ out.AlignedBegin = 0;
+ out.WholeBegin = partBeginWholeOffset;
+ out.AlignedWholeBegin = partBeginWholeOffset;
+ }
+
+ if (wholeBegin >= partBeginWholeOffset || wholeEnd < partEndWholeOffset) {
+ // part of the last part
+ out.End = wholeEnd - partBeginWholeOffset;
+ out.AlignedEnd = ((out.End + columnSize - 1) / columnSize) * columnSize;
+ out.WholeEnd = wholeEnd;
+ } else {
+ // up to the end
+ out.End = lastPartSize;
+ out.AlignedEnd = alignedPartSize;
+ out.WholeEnd = partEndWholeOffset;
+ }
+ // It IS the LAST data part of the blob
+ outRange->EndPartIdx = partIdx + 1;
+ } else {
+ out.Reset();
+ }
+ }
+ }
+ Y_VERIFY_DEBUG(outRange->EndPartIdx != Max<ui64>());
+}
+
+void MirrorSplit(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, const TRope& buffer,
+ TDataPartSet& outPartSet) {
+ outPartSet.FullDataSize = buffer.GetSize();
+ outPartSet.Parts.resize(type.TotalPartCount());
+ TString partBuffer;
+ ui32 parityParts = type.ParityParts();
+ switch (crcMode) {
+ case TRopeErasureType::CrcModeNone:
+ for (ui32 partIdx = 0; partIdx <= parityParts; ++partIdx) {
+ outPartSet.Parts[partIdx].ReferenceTo(buffer);
+ outPartSet.PartsMask |= (1 << partIdx);
+ }
+ outPartSet.MemoryConsumed = buffer.GetSize();
+ return;
+ case TRopeErasureType::CrcModeWholePart:
+ {
+ ui64 partSize = type.PartSize(crcMode, buffer.GetSize());
+ TRope& part = outPartSet.FullDataFragment.OwnedRope;
+ part = buffer;
TRopeHelpers::Resize(part, partSize);
- if (buffer.GetSize() || part.GetSize()) {
- Y_VERIFY(part.GetSize() >= buffer.GetSize() + sizeof(ui32), "Part size too small, buffer size# %" PRIu64
- " partSize# %" PRIu64, (ui64)buffer.GetSize(), (ui64)partSize);
- PadAndCrcAtTheEnd(part.Begin(), buffer.GetSize(), part.GetSize());
- }
- for (ui32 partIdx = 0; partIdx <= parityParts; ++partIdx) {
- outPartSet.Parts[partIdx].ReferenceTo(part);
- outPartSet.PartsMask |= (1 << partIdx);
- }
- outPartSet.MemoryConsumed = part.GetSize();
- }
- return;
- }
- ythrow TWithBackTrace<yexception>() << "Unknown crcMode = " << (i32)crcMode;
-
-}
-
-template <bool restoreParts, bool restoreFullData>
-void MirrorRestore(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, TDataPartSet& partSet) {
- ui32 totalParts = type.TotalPartCount();
- for (ui32 partIdx = 0; partIdx < totalParts; ++partIdx) {
- if (partSet.PartsMask & (1 << partIdx)) {
- if (restoreParts) {
- for (ui32 i = 0; i < totalParts; ++i) {
- if (!(partSet.PartsMask & (1 << i))) {
- partSet.Parts[i] = partSet.Parts[partIdx];
- }
- }
- partSet.MemoryConsumed = partSet.Parts[partIdx].MemoryConsumed();
- }
- if (restoreFullData) {
- switch (crcMode) {
- case TRopeErasureType::CrcModeNone:
- partSet.FullDataFragment.ReferenceTo(partSet.Parts[partIdx].OwnedRope);
- return;
- case TRopeErasureType::CrcModeWholePart:
- TRope outBuffer = partSet.Parts[partIdx].OwnedRope;
- Y_VERIFY(outBuffer.GetSize() >= partSet.FullDataSize, "Unexpected outBuffer.size# %" PRIu64
- " fullDataSize# %" PRIu64, (ui64)outBuffer.GetSize(), (ui64)partSet.FullDataSize);
+ if (buffer.GetSize() || part.GetSize()) {
+ Y_VERIFY(part.GetSize() >= buffer.GetSize() + sizeof(ui32), "Part size too small, buffer size# %" PRIu64
+ " partSize# %" PRIu64, (ui64)buffer.GetSize(), (ui64)partSize);
+ PadAndCrcAtTheEnd(part.Begin(), buffer.GetSize(), part.GetSize());
+ }
+ for (ui32 partIdx = 0; partIdx <= parityParts; ++partIdx) {
+ outPartSet.Parts[partIdx].ReferenceTo(part);
+ outPartSet.PartsMask |= (1 << partIdx);
+ }
+ outPartSet.MemoryConsumed = part.GetSize();
+ }
+ return;
+ }
+ ythrow TWithBackTrace<yexception>() << "Unknown crcMode = " << (i32)crcMode;
+
+}
+
+template <bool restoreParts, bool restoreFullData>
+void MirrorRestore(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, TDataPartSet& partSet) {
+ ui32 totalParts = type.TotalPartCount();
+ for (ui32 partIdx = 0; partIdx < totalParts; ++partIdx) {
+ if (partSet.PartsMask & (1 << partIdx)) {
+ if (restoreParts) {
+ for (ui32 i = 0; i < totalParts; ++i) {
+ if (!(partSet.PartsMask & (1 << i))) {
+ partSet.Parts[i] = partSet.Parts[partIdx];
+ }
+ }
+ partSet.MemoryConsumed = partSet.Parts[partIdx].MemoryConsumed();
+ }
+ if (restoreFullData) {
+ switch (crcMode) {
+ case TRopeErasureType::CrcModeNone:
+ partSet.FullDataFragment.ReferenceTo(partSet.Parts[partIdx].OwnedRope);
+ return;
+ case TRopeErasureType::CrcModeWholePart:
+ TRope outBuffer = partSet.Parts[partIdx].OwnedRope;
+ Y_VERIFY(outBuffer.GetSize() >= partSet.FullDataSize, "Unexpected outBuffer.size# %" PRIu64
+ " fullDataSize# %" PRIu64, (ui64)outBuffer.GetSize(), (ui64)partSet.FullDataSize);
TRopeHelpers::Resize(outBuffer, partSet.FullDataSize); // To pad with zeroes!
- partSet.FullDataFragment.ReferenceTo(outBuffer);
- return;
- }
- ythrow TWithBackTrace<yexception>() << "Unknown crcMode = " << (i32)crcMode;
- }
- return;
- }
- }
- ythrow TWithBackTrace<yexception>() << "No data in part set";
-
-
-}
-static void VerifyPartSizes(TDataPartSet& partSet, size_t definedPartEndIdx) {
- size_t partSize = partSet.Parts[0].size();
- for (size_t idx = 0; idx < partSet.Parts.size(); ++idx) {
- Y_VERIFY(partSet.Parts[idx].size() == partSize);
- if (partSize && idx < definedPartEndIdx) {
+ partSet.FullDataFragment.ReferenceTo(outBuffer);
+ return;
+ }
+ ythrow TWithBackTrace<yexception>() << "Unknown crcMode = " << (i32)crcMode;
+ }
+ return;
+ }
+ }
+ ythrow TWithBackTrace<yexception>() << "No data in part set";
+
+
+}
+static void VerifyPartSizes(TDataPartSet& partSet, size_t definedPartEndIdx) {
+ size_t partSize = partSet.Parts[0].size();
+ for (size_t idx = 0; idx < partSet.Parts.size(); ++idx) {
+ Y_VERIFY(partSet.Parts[idx].size() == partSize);
+ if (partSize && idx < definedPartEndIdx) {
CHECK_ROPE_IS_DEFINED(partSet.Parts[idx].FastViewer.GetCurrent(partSet.Parts[idx].Offset),
- partSet.Parts[idx].Size);
- }
- }
-}
-
-void TRopeErasureType::SplitData(ECrcMode crcMode, const TRope& buffer, TDataPartSet& outPartSet) const {
- const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
- for (size_t i = 0; i < outPartSet.Parts.size(); ++i) {
- outPartSet.Parts[i].OwnedRope = TRope();
- }
- switch (erasure.ErasureFamily) {
- case TRopeErasureType::ErasureMirror:
- MirrorSplit(crcMode, *this, buffer, outPartSet);
- VerifyPartSizes(outPartSet, Max<size_t>());
- break;
- case TRopeErasureType::ErasureParityBlock:
- switch (erasure.ParityParts) {
- case 1:
- XorBlockSplit(crcMode, *this, buffer, outPartSet);
- VerifyPartSizes(outPartSet, Max<size_t>());
- break;
- case 2:
- EoBlockSplit(crcMode, *this, buffer, outPartSet);
- VerifyPartSizes(outPartSet, Max<size_t>());
- break;
- case 3:
- StarBlockSplit(crcMode, *this, buffer, outPartSet);
- VerifyPartSizes(outPartSet, Max<size_t>());
- break;
- default:
- ythrow TWithBackTrace<yexception>() << "Unsupported number of parity parts: "
- << erasure.ParityParts;
- break;
- }
- break;
- default:
- ythrow TWithBackTrace<yexception>() << "Unknown ErasureFamily = " << (i32)erasure.ErasureFamily;
- break;
- }
-}
-
-void TRopeErasureType::RestoreData(ECrcMode crcMode, TDataPartSet& partSet, TRope& outBuffer, bool restoreParts,
- bool restoreFullData, bool restoreParityParts) const {
- partSet.FullDataFragment.ReferenceTo(outBuffer);
- RestoreData(crcMode, partSet, restoreParts, restoreFullData, restoreParityParts);
- outBuffer = partSet.FullDataFragment.OwnedRope;
-}
-
-void TRopeErasureType::RestoreData(ECrcMode crcMode, TDataPartSet& partSet, bool restoreParts, bool restoreFullData,
- bool restoreParityParts) const {
- if (restoreParityParts) {
- restoreParts = true;
- }
- const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
- ui32 totalParts = TotalPartCount();
- if (partSet.Parts.size() != totalParts) {
- ythrow TWithBackTrace<yexception>() << "Incorrect partSet size, received " << partSet.Parts.size()
- << " while expected " << (erasure.DataParts + erasure.ParityParts);
- }
- Y_VERIFY_DEBUG(restoreFullData || restoreParts);
- Y_VERIFY_DEBUG(erasure.Prime <= MAX_LINES_IN_BLOCK);
- switch (erasure.ErasureFamily) {
- case TRopeErasureType::ErasureMirror:
- if (restoreParts) {
- if (restoreFullData) {
- MirrorRestore<true, true>(crcMode, *this, partSet);
- } else {
- MirrorRestore<true, false>(crcMode, *this, partSet);
- }
- VerifyPartSizes(partSet, Max<size_t>());
- } else if (restoreFullData) {
- MirrorRestore<false, true>(crcMode, *this, partSet);
- }
- if (restoreFullData) {
- Y_VERIFY(partSet.FullDataSize == partSet.FullDataFragment.PartSize,
- "Incorrect data part size = %" PRIu64 ", expected size = %" PRIu64,
- (ui64)partSet.FullDataFragment.PartSize, (ui64)partSet.FullDataSize);
- }
- break;
- case TRopeErasureType::ErasureParityBlock:
- switch (erasure.ParityParts) {
- case 1:
- if (restoreParts) {
- if (restoreFullData) {
- if (restoreParityParts) {
- XorBlockRestore<true, true, true>(crcMode, *this, partSet);
- VerifyPartSizes(partSet, Max<size_t>());
- } else {
- XorBlockRestore<true, true, false>(crcMode, *this, partSet);
- VerifyPartSizes(partSet, erasure.DataParts);
- }
- } else {
- if (restoreParityParts) {
- XorBlockRestore<true, false, true>(crcMode, *this, partSet);
- VerifyPartSizes(partSet, Max<size_t>());
- } else {
- XorBlockRestore<true, false, false>(crcMode, *this, partSet);
- VerifyPartSizes(partSet, erasure.DataParts);
- }
- }
- partSet.MemoryConsumed = partSet.Parts[0].MemoryConsumed() * partSet.Parts.size();
- } else if (restoreFullData) {
- XorBlockRestore<false, true, false>(crcMode, *this, partSet);
- }
- break;
- case 2:
- if (restoreParts) {
- if (restoreFullData) {
- if (restoreParityParts) {
- EoBlockRestore<true, true, true>(crcMode, *this, partSet);
- VerifyPartSizes(partSet, Max<size_t>());
- } else {
- EoBlockRestore<true, true, false>(crcMode, *this, partSet);
- VerifyPartSizes(partSet, erasure.DataParts);
- }
- } else {
- if (restoreParityParts) {
- EoBlockRestore<true, false, true>(crcMode, *this, partSet);
- VerifyPartSizes(partSet, Max<size_t>());
- } else {
- EoBlockRestore<true, false, false>(crcMode, *this, partSet);
- VerifyPartSizes(partSet, erasure.DataParts);
- }
- }
- partSet.MemoryConsumed = partSet.Parts[0].MemoryConsumed() * partSet.Parts.size();
- } else if (restoreFullData) {
- EoBlockRestore<false, true, false>(crcMode, *this, partSet);
- }
- break;
- case 3:
- if (restoreParts) {
- if (restoreFullData) {
- if (restoreParityParts) {
- // restoreParts, restoreFullData, restoreParityParts
- StarBlockRestore<true, true, true>(crcMode, *this, partSet);
- VerifyPartSizes(partSet, Max<size_t>());
- } else {
- StarBlockRestore<true, true, false>(crcMode, *this, partSet);
- VerifyPartSizes(partSet, erasure.DataParts);
- }
- } else {
- if (restoreParityParts) {
- StarBlockRestore<true, false, true>(crcMode, *this, partSet);
- VerifyPartSizes(partSet, Max<size_t>());
- } else {
- StarBlockRestore<true, false, false>(crcMode, *this, partSet);
- VerifyPartSizes(partSet, erasure.DataParts);
- }
- }
- partSet.MemoryConsumed = partSet.Parts[0].MemoryConsumed() * partSet.Parts.size();
- } else if (restoreFullData) {
- StarBlockRestore<false, true, false>(crcMode, *this, partSet);
- }
- break;
- default:
- ythrow TWithBackTrace<yexception>() << "Unsupported number of parity parts: "
- << erasure.ParityParts;
- }
- break;
- default:
- ythrow TWithBackTrace<yexception>() << "Unsupported erasure family";
- }
-}
-
-} // NKikimr
+ partSet.Parts[idx].Size);
+ }
+ }
+}
+
+void TRopeErasureType::SplitData(ECrcMode crcMode, const TRope& buffer, TDataPartSet& outPartSet) const {
+ const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
+ for (size_t i = 0; i < outPartSet.Parts.size(); ++i) {
+ outPartSet.Parts[i].OwnedRope = TRope();
+ }
+ switch (erasure.ErasureFamily) {
+ case TRopeErasureType::ErasureMirror:
+ MirrorSplit(crcMode, *this, buffer, outPartSet);
+ VerifyPartSizes(outPartSet, Max<size_t>());
+ break;
+ case TRopeErasureType::ErasureParityBlock:
+ switch (erasure.ParityParts) {
+ case 1:
+ XorBlockSplit(crcMode, *this, buffer, outPartSet);
+ VerifyPartSizes(outPartSet, Max<size_t>());
+ break;
+ case 2:
+ EoBlockSplit(crcMode, *this, buffer, outPartSet);
+ VerifyPartSizes(outPartSet, Max<size_t>());
+ break;
+ case 3:
+ StarBlockSplit(crcMode, *this, buffer, outPartSet);
+ VerifyPartSizes(outPartSet, Max<size_t>());
+ break;
+ default:
+ ythrow TWithBackTrace<yexception>() << "Unsupported number of parity parts: "
+ << erasure.ParityParts;
+ break;
+ }
+ break;
+ default:
+ ythrow TWithBackTrace<yexception>() << "Unknown ErasureFamily = " << (i32)erasure.ErasureFamily;
+ break;
+ }
+}
+
+void TRopeErasureType::RestoreData(ECrcMode crcMode, TDataPartSet& partSet, TRope& outBuffer, bool restoreParts,
+ bool restoreFullData, bool restoreParityParts) const {
+ partSet.FullDataFragment.ReferenceTo(outBuffer);
+ RestoreData(crcMode, partSet, restoreParts, restoreFullData, restoreParityParts);
+ outBuffer = partSet.FullDataFragment.OwnedRope;
+}
+
+void TRopeErasureType::RestoreData(ECrcMode crcMode, TDataPartSet& partSet, bool restoreParts, bool restoreFullData,
+ bool restoreParityParts) const {
+ if (restoreParityParts) {
+ restoreParts = true;
+ }
+ const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies];
+ ui32 totalParts = TotalPartCount();
+ if (partSet.Parts.size() != totalParts) {
+ ythrow TWithBackTrace<yexception>() << "Incorrect partSet size, received " << partSet.Parts.size()
+ << " while expected " << (erasure.DataParts + erasure.ParityParts);
+ }
+ Y_VERIFY_DEBUG(restoreFullData || restoreParts);
+ Y_VERIFY_DEBUG(erasure.Prime <= MAX_LINES_IN_BLOCK);
+ switch (erasure.ErasureFamily) {
+ case TRopeErasureType::ErasureMirror:
+ if (restoreParts) {
+ if (restoreFullData) {
+ MirrorRestore<true, true>(crcMode, *this, partSet);
+ } else {
+ MirrorRestore<true, false>(crcMode, *this, partSet);
+ }
+ VerifyPartSizes(partSet, Max<size_t>());
+ } else if (restoreFullData) {
+ MirrorRestore<false, true>(crcMode, *this, partSet);
+ }
+ if (restoreFullData) {
+ Y_VERIFY(partSet.FullDataSize == partSet.FullDataFragment.PartSize,
+ "Incorrect data part size = %" PRIu64 ", expected size = %" PRIu64,
+ (ui64)partSet.FullDataFragment.PartSize, (ui64)partSet.FullDataSize);
+ }
+ break;
+ case TRopeErasureType::ErasureParityBlock:
+ switch (erasure.ParityParts) {
+ case 1:
+ if (restoreParts) {
+ if (restoreFullData) {
+ if (restoreParityParts) {
+ XorBlockRestore<true, true, true>(crcMode, *this, partSet);
+ VerifyPartSizes(partSet, Max<size_t>());
+ } else {
+ XorBlockRestore<true, true, false>(crcMode, *this, partSet);
+ VerifyPartSizes(partSet, erasure.DataParts);
+ }
+ } else {
+ if (restoreParityParts) {
+ XorBlockRestore<true, false, true>(crcMode, *this, partSet);
+ VerifyPartSizes(partSet, Max<size_t>());
+ } else {
+ XorBlockRestore<true, false, false>(crcMode, *this, partSet);
+ VerifyPartSizes(partSet, erasure.DataParts);
+ }
+ }
+ partSet.MemoryConsumed = partSet.Parts[0].MemoryConsumed() * partSet.Parts.size();
+ } else if (restoreFullData) {
+ XorBlockRestore<false, true, false>(crcMode, *this, partSet);
+ }
+ break;
+ case 2:
+ if (restoreParts) {
+ if (restoreFullData) {
+ if (restoreParityParts) {
+ EoBlockRestore<true, true, true>(crcMode, *this, partSet);
+ VerifyPartSizes(partSet, Max<size_t>());
+ } else {
+ EoBlockRestore<true, true, false>(crcMode, *this, partSet);
+ VerifyPartSizes(partSet, erasure.DataParts);
+ }
+ } else {
+ if (restoreParityParts) {
+ EoBlockRestore<true, false, true>(crcMode, *this, partSet);
+ VerifyPartSizes(partSet, Max<size_t>());
+ } else {
+ EoBlockRestore<true, false, false>(crcMode, *this, partSet);
+ VerifyPartSizes(partSet, erasure.DataParts);
+ }
+ }
+ partSet.MemoryConsumed = partSet.Parts[0].MemoryConsumed() * partSet.Parts.size();
+ } else if (restoreFullData) {
+ EoBlockRestore<false, true, false>(crcMode, *this, partSet);
+ }
+ break;
+ case 3:
+ if (restoreParts) {
+ if (restoreFullData) {
+ if (restoreParityParts) {
+ // restoreParts, restoreFullData, restoreParityParts
+ StarBlockRestore<true, true, true>(crcMode, *this, partSet);
+ VerifyPartSizes(partSet, Max<size_t>());
+ } else {
+ StarBlockRestore<true, true, false>(crcMode, *this, partSet);
+ VerifyPartSizes(partSet, erasure.DataParts);
+ }
+ } else {
+ if (restoreParityParts) {
+ StarBlockRestore<true, false, true>(crcMode, *this, partSet);
+ VerifyPartSizes(partSet, Max<size_t>());
+ } else {
+ StarBlockRestore<true, false, false>(crcMode, *this, partSet);
+ VerifyPartSizes(partSet, erasure.DataParts);
+ }
+ }
+ partSet.MemoryConsumed = partSet.Parts[0].MemoryConsumed() * partSet.Parts.size();
+ } else if (restoreFullData) {
+ StarBlockRestore<false, true, false>(crcMode, *this, partSet);
+ }
+ break;
+ default:
+ ythrow TWithBackTrace<yexception>() << "Unsupported number of parity parts: "
+ << erasure.ParityParts;
+ }
+ break;
+ default:
+ ythrow TWithBackTrace<yexception>() << "Unsupported erasure family";
+ }
+}
+
+} // NKikimr
} // NErasureRope
-
+
Y_DECLARE_OUT_SPEC(, NKikimr::NErasureRope::TRopeErasureType::EErasureSpecies, stream, value) {
stream << NKikimr::NErasureRope::TRopeErasureType::ErasureSpeciesToStr(value);
-}
+}
diff --git a/ydb/core/erasure/erasure_rope.h b/ydb/core/erasure/erasure_rope.h
index 98c495bf83..a4835ce834 100644
--- a/ydb/core/erasure/erasure_rope.h
+++ b/ydb/core/erasure/erasure_rope.h
@@ -1,526 +1,526 @@
-#pragma once
-
-#include <array>
-
+#pragma once
+
+#include <array>
+
#include <ydb/core/debug/valgrind_check.h>
-
-#include <util/stream/str.h>
-#include <util/generic/string.h>
-#include <util/generic/bt_exception.h>
-#include <util/string/builder.h>
-
-#include <util/generic/list.h>
+
+#include <util/stream/str.h>
+#include <util/generic/string.h>
+#include <util/generic/bt_exception.h>
+#include <util/string/builder.h>
+
+#include <util/generic/list.h>
#include <library/cpp/containers/stack_vector/stack_vec.h>
#include <library/cpp/actors/util/rope.h>
#include <library/cpp/digest/crc32c/crc32c.h>
-
-namespace NKikimr {
+
+namespace NKikimr {
namespace NErasureRope {
-
+
class TRopeHelpers {
-public:
- using Iterator = TRope::TConstIterator;
-
- static const ui32 RopeBlockSize = 32 * 1024 - 128;
-
- static TRope CreateRope(size_t size, char c = '\0') {
- TRope rope;
- for (size_t i = 0; i < size / RopeBlockSize; ++i) {
- rope.Insert(rope.End(), RopeFromStringReference(TString(RopeBlockSize, c)));
- }
-
- if (rope.GetSize() < size) {
- rope.Insert(rope.End(), RopeFromStringReference(TString(size - rope.GetSize(), c)));
- }
-
- return rope;
- }
-
- static TRope RopeUninitialized(size_t size) {
- TRope rope;
- for (size_t i = 0; i < size / RopeBlockSize; ++i) {
- rope.Insert(rope.End(), RopeFromStringReference(TString::Uninitialized(RopeBlockSize)));
- }
-
- if (rope.GetSize() < size) {
- rope.Insert(rope.End(),RopeFromStringReference(TString::Uninitialized(size - rope.GetSize())));
- }
-
- return rope;
- }
-
- static TRope RopeCopy(const TRope& src) {
- TRope copy = RopeUninitialized(src.GetSize());
+public:
+ using Iterator = TRope::TConstIterator;
+
+ static const ui32 RopeBlockSize = 32 * 1024 - 128;
+
+ static TRope CreateRope(size_t size, char c = '\0') {
+ TRope rope;
+ for (size_t i = 0; i < size / RopeBlockSize; ++i) {
+ rope.Insert(rope.End(), RopeFromStringReference(TString(RopeBlockSize, c)));
+ }
+
+ if (rope.GetSize() < size) {
+ rope.Insert(rope.End(), RopeFromStringReference(TString(size - rope.GetSize(), c)));
+ }
+
+ return rope;
+ }
+
+ static TRope RopeUninitialized(size_t size) {
+ TRope rope;
+ for (size_t i = 0; i < size / RopeBlockSize; ++i) {
+ rope.Insert(rope.End(), RopeFromStringReference(TString::Uninitialized(RopeBlockSize)));
+ }
+
+ if (rope.GetSize() < size) {
+ rope.Insert(rope.End(),RopeFromStringReference(TString::Uninitialized(size - rope.GetSize())));
+ }
+
+ return rope;
+ }
+
+ static TRope RopeCopy(const TRope& src) {
+ TRope copy = RopeUninitialized(src.GetSize());
TRopeUtils::Memcpy(copy.Begin(), src.Begin(), src.GetSize());
- return copy;
- }
-
- static TRope RopeFromStringMemcpy(const TString& string) {
- TRope rope = RopeUninitialized(string.size());
+ return copy;
+ }
+
+ static TRope RopeFromStringMemcpy(const TString& string) {
+ TRope rope = RopeUninitialized(string.size());
TRopeUtils::Memcpy(rope.Begin(), string.data(), string.size());
- return rope;
- }
-
- static TRope RopeFromStringReference(TString string) {
- TRope rope;
- if (string.Empty()) {
- return rope;
- }
+ return rope;
+ }
+
+ static TRope RopeFromStringReference(TString string) {
+ TRope rope;
+ if (string.Empty()) {
+ return rope;
+ }
rope.Insert(rope.End(), TRope(std::move(string)));
- return rope;
- }
-
- static TRope& ResetWithFullCopy(TRope& rope) {
- rope = RopeCopy(rope);
- return rope;
- }
-
- static TRope& Resize(TRope& rope, size_t size) {
- if (size > rope.GetSize()) {
- rope.Insert(rope.End(), CreateRope(size - rope.GetSize()));
- } else {
- rope.EraseBack(rope.GetSize() - size);
- }
- return rope;
- }
-
- static bool Is8Aligned(const TRope& buffer) {
- Iterator begin = buffer.Begin();
- while (begin.Valid()) {
- intptr_t address = (intptr_t) begin.ContiguousData();
- ui64 size = begin.ContiguousSize();
- begin.AdvanceToNextContiguousBlock();
- if (address % 8 != 0 || (begin.Valid() && size % 8 != 0)) {
- return false;
- }
- }
- return true;
- }
-
- static ui32 GetCrc32c(Iterator begin, size_t size) {
- ui32 hash = 0;
- while (size) {
- Y_VERIFY(begin.Valid());
- size_t len = std::min(size, begin.ContiguousSize());
- hash = Crc32cExtend(hash, begin.ContiguousData(), len);
- begin += len;
- size -= len;
- }
- return hash;
- }
-
- class TRopeFastView {
- private:
- Iterator Begin;
- Iterator Current;
-
- char* BlockData = nullptr;
- char* Boost = nullptr;
- ui64 BlockBeginIndx = 0;
- ui64 BlockEndIndx = 0;
- ui64 Offset = 0;
-
- public:
- explicit TRopeFastView(const TRope& rope)
- : TRopeFastView(rope.Begin()) {
- }
-
- explicit TRopeFastView(Iterator begin)
- : Begin(begin)
- , Current(Begin)
- , BlockData(const_cast<char*>(Begin.ContiguousData()))
- , Boost(BlockData)
- , BlockBeginIndx(0)
- , BlockEndIndx(Begin.ContiguousSize())
- , Offset(0) {
- }
-
- TRopeFastView() = default;
-
- ui64 GetContiguousSize(size_t pos) {
- Y_VERIFY_DEBUG(pos >= Offset);
- return (BlockBeginIndx <= (pos - Offset) && (pos - Offset) < BlockEndIndx)
- ? (BlockEndIndx - pos + Offset) : 0;
- }
-
- char* DataPtrAt(size_t pos) {
- Y_VERIFY_DEBUG(pos >= Offset);
- return (BlockBeginIndx <= (pos - Offset) && (pos - Offset) < BlockEndIndx)
- ? BlockData + (pos - BlockBeginIndx) - Offset : UpdateCurrent(pos - Offset);
- }
-
- char* FastDataPtrAt(size_t pos) {
- return Boost + pos;
- }
-
- ui8& At8(size_t pos) {
- return *(ui8*)DataPtrAt(pos);
- }
-
- ui64& At64(size_t pos) {
- return *(ui64*)DataPtrAt(pos * sizeof(ui64));
- }
-
- ui64& FastAt64(size_t pos) {
- return *(ui64*)FastDataPtrAt(pos * sizeof(ui64));
- }
-
- Iterator GetBegin() {
- return Begin;
- }
-
- void SetOffset(ui64 offset) {
- Offset = offset;
- if (BlockData) {
- Boost = BlockData - BlockBeginIndx - Offset;
- }
- }
-
- Iterator GetCurrent(size_t pos) {
- if (BlockBeginIndx > pos - Offset || BlockEndIndx <= pos - Offset) {
- UpdateCurrent(pos - Offset);
- }
- if (Current.ContiguousData() > Boost + pos) {
- Current -= Current.ContiguousData() - (Boost + pos);
- } else {
- Current += (Boost + pos) - Current.ContiguousData();
- }
- Y_VERIFY(Current.ContiguousData() == Boost + pos);
- return Current;
- }
-
- private:
-
- char* UpdateCurrent(size_t pos) {
- if (pos >= BlockEndIndx) {
- while (pos >= BlockEndIndx) {
- Current.AdvanceToNextContiguousBlock();
- BlockBeginIndx = BlockEndIndx;
- BlockEndIndx += Current.ContiguousSize();
- }
- Y_VERIFY(BlockBeginIndx <= pos && pos < BlockEndIndx);
- BlockData = const_cast<char*>(Current.ContiguousData());
- Boost = BlockData - BlockBeginIndx - Offset;
- return BlockData + (pos - BlockBeginIndx);
- }
-
- if (pos < BlockBeginIndx - pos) {
- Current = Begin + pos;
- } else {
- Current -= Current.ChunkOffset();
- Current -= BlockBeginIndx - pos;
- }
-
- BlockData = const_cast<char*>(Current.ContiguousData()) - Current.ChunkOffset();
- if (BlockData == Begin.ContiguousData() - Begin.ChunkOffset()) {
- BlockData = const_cast<char*>(Begin.ContiguousData());
- }
-
- ui64 offset = Current.ContiguousData() - BlockData;
- Y_VERIFY(pos >= offset);
- BlockBeginIndx = pos - offset;
- BlockEndIndx = pos + Current.ContiguousSize();
- Boost = BlockData - BlockBeginIndx - Offset;
- Y_VERIFY(pos >= BlockBeginIndx && pos < BlockEndIndx);
-
- return BlockData + (pos - BlockBeginIndx);
- }
- };
-};
-
-// Part fragment, contains only some data
-struct TPartFragment {
- TRope OwnedRope;
- ui64 Offset = 0; // Relative to part beginning
- ui64 Size = 0;
- ui64 PartSize = 0; // Full size of the part
-
+ return rope;
+ }
+
+ static TRope& ResetWithFullCopy(TRope& rope) {
+ rope = RopeCopy(rope);
+ return rope;
+ }
+
+ static TRope& Resize(TRope& rope, size_t size) {
+ if (size > rope.GetSize()) {
+ rope.Insert(rope.End(), CreateRope(size - rope.GetSize()));
+ } else {
+ rope.EraseBack(rope.GetSize() - size);
+ }
+ return rope;
+ }
+
+ static bool Is8Aligned(const TRope& buffer) {
+ Iterator begin = buffer.Begin();
+ while (begin.Valid()) {
+ intptr_t address = (intptr_t) begin.ContiguousData();
+ ui64 size = begin.ContiguousSize();
+ begin.AdvanceToNextContiguousBlock();
+ if (address % 8 != 0 || (begin.Valid() && size % 8 != 0)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ static ui32 GetCrc32c(Iterator begin, size_t size) {
+ ui32 hash = 0;
+ while (size) {
+ Y_VERIFY(begin.Valid());
+ size_t len = std::min(size, begin.ContiguousSize());
+ hash = Crc32cExtend(hash, begin.ContiguousData(), len);
+ begin += len;
+ size -= len;
+ }
+ return hash;
+ }
+
+ class TRopeFastView {
+ private:
+ Iterator Begin;
+ Iterator Current;
+
+ char* BlockData = nullptr;
+ char* Boost = nullptr;
+ ui64 BlockBeginIndx = 0;
+ ui64 BlockEndIndx = 0;
+ ui64 Offset = 0;
+
+ public:
+ explicit TRopeFastView(const TRope& rope)
+ : TRopeFastView(rope.Begin()) {
+ }
+
+ explicit TRopeFastView(Iterator begin)
+ : Begin(begin)
+ , Current(Begin)
+ , BlockData(const_cast<char*>(Begin.ContiguousData()))
+ , Boost(BlockData)
+ , BlockBeginIndx(0)
+ , BlockEndIndx(Begin.ContiguousSize())
+ , Offset(0) {
+ }
+
+ TRopeFastView() = default;
+
+ ui64 GetContiguousSize(size_t pos) {
+ Y_VERIFY_DEBUG(pos >= Offset);
+ return (BlockBeginIndx <= (pos - Offset) && (pos - Offset) < BlockEndIndx)
+ ? (BlockEndIndx - pos + Offset) : 0;
+ }
+
+ char* DataPtrAt(size_t pos) {
+ Y_VERIFY_DEBUG(pos >= Offset);
+ return (BlockBeginIndx <= (pos - Offset) && (pos - Offset) < BlockEndIndx)
+ ? BlockData + (pos - BlockBeginIndx) - Offset : UpdateCurrent(pos - Offset);
+ }
+
+ char* FastDataPtrAt(size_t pos) {
+ return Boost + pos;
+ }
+
+ ui8& At8(size_t pos) {
+ return *(ui8*)DataPtrAt(pos);
+ }
+
+ ui64& At64(size_t pos) {
+ return *(ui64*)DataPtrAt(pos * sizeof(ui64));
+ }
+
+ ui64& FastAt64(size_t pos) {
+ return *(ui64*)FastDataPtrAt(pos * sizeof(ui64));
+ }
+
+ Iterator GetBegin() {
+ return Begin;
+ }
+
+ void SetOffset(ui64 offset) {
+ Offset = offset;
+ if (BlockData) {
+ Boost = BlockData - BlockBeginIndx - Offset;
+ }
+ }
+
+ Iterator GetCurrent(size_t pos) {
+ if (BlockBeginIndx > pos - Offset || BlockEndIndx <= pos - Offset) {
+ UpdateCurrent(pos - Offset);
+ }
+ if (Current.ContiguousData() > Boost + pos) {
+ Current -= Current.ContiguousData() - (Boost + pos);
+ } else {
+ Current += (Boost + pos) - Current.ContiguousData();
+ }
+ Y_VERIFY(Current.ContiguousData() == Boost + pos);
+ return Current;
+ }
+
+ private:
+
+ char* UpdateCurrent(size_t pos) {
+ if (pos >= BlockEndIndx) {
+ while (pos >= BlockEndIndx) {
+ Current.AdvanceToNextContiguousBlock();
+ BlockBeginIndx = BlockEndIndx;
+ BlockEndIndx += Current.ContiguousSize();
+ }
+ Y_VERIFY(BlockBeginIndx <= pos && pos < BlockEndIndx);
+ BlockData = const_cast<char*>(Current.ContiguousData());
+ Boost = BlockData - BlockBeginIndx - Offset;
+ return BlockData + (pos - BlockBeginIndx);
+ }
+
+ if (pos < BlockBeginIndx - pos) {
+ Current = Begin + pos;
+ } else {
+ Current -= Current.ChunkOffset();
+ Current -= BlockBeginIndx - pos;
+ }
+
+ BlockData = const_cast<char*>(Current.ContiguousData()) - Current.ChunkOffset();
+ if (BlockData == Begin.ContiguousData() - Begin.ChunkOffset()) {
+ BlockData = const_cast<char*>(Begin.ContiguousData());
+ }
+
+ ui64 offset = Current.ContiguousData() - BlockData;
+ Y_VERIFY(pos >= offset);
+ BlockBeginIndx = pos - offset;
+ BlockEndIndx = pos + Current.ContiguousSize();
+ Boost = BlockData - BlockBeginIndx - Offset;
+ Y_VERIFY(pos >= BlockBeginIndx && pos < BlockEndIndx);
+
+ return BlockData + (pos - BlockBeginIndx);
+ }
+ };
+};
+
+// Part fragment, contains only some data
+struct TPartFragment {
+ TRope OwnedRope;
+ ui64 Offset = 0; // Relative to part beginning
+ ui64 Size = 0;
+ ui64 PartSize = 0; // Full size of the part
+
mutable TRopeHelpers::TRopeFastView FastViewer;
-
- TPartFragment() = default;
-
- TPartFragment(const TPartFragment& lhs) {
- OwnedRope = lhs.OwnedRope;
- Offset = lhs.Offset;
- Size = lhs.Size;
- PartSize = lhs.PartSize;
+
+ TPartFragment() = default;
+
+ TPartFragment(const TPartFragment& lhs) {
+ OwnedRope = lhs.OwnedRope;
+ Offset = lhs.Offset;
+ Size = lhs.Size;
+ PartSize = lhs.PartSize;
FastViewer = TRopeHelpers::TRopeFastView(OwnedRope);
- FastViewer.SetOffset(Offset);
- }
-
- TPartFragment& operator=(const TPartFragment& lhs) {
- if (this == &lhs) {
- return *this;
- }
- OwnedRope = lhs.OwnedRope;
- Offset = lhs.Offset;
- Size = lhs.Size;
- PartSize = lhs.PartSize;
+ FastViewer.SetOffset(Offset);
+ }
+
+ TPartFragment& operator=(const TPartFragment& lhs) {
+ if (this == &lhs) {
+ return *this;
+ }
+ OwnedRope = lhs.OwnedRope;
+ Offset = lhs.Offset;
+ Size = lhs.Size;
+ PartSize = lhs.PartSize;
FastViewer = TRopeHelpers::TRopeFastView(OwnedRope);
- FastViewer.SetOffset(Offset);
- return *this;
- }
-
- ui64 size() const {
- return PartSize;
- }
-
- void clear() {
- OwnedRope = TRope();
- Offset = 0;
- Size = 0;
- PartSize = 0;
+ FastViewer.SetOffset(Offset);
+ return *this;
+ }
+
+ ui64 size() const {
+ return PartSize;
+ }
+
+ void clear() {
+ OwnedRope = TRope();
+ Offset = 0;
+ Size = 0;
+ PartSize = 0;
FastViewer = TRopeHelpers::TRopeFastView(OwnedRope);
- }
-
- void UninitializedOwnedWhole(ui64 size) {
+ }
+
+ void UninitializedOwnedWhole(ui64 size) {
OwnedRope = TRopeHelpers::RopeUninitialized(size);
FastViewer = TRopeHelpers::TRopeFastView(OwnedRope);
- Offset = 0;
- Size = size;
- PartSize = size;
- }
-
- void ResetToWhole(const TRope& whole) {
+ Offset = 0;
+ Size = size;
+ PartSize = size;
+ }
+
+ void ResetToWhole(const TRope& whole) {
OwnedRope = TRopeHelpers::RopeCopy(whole);
FastViewer = TRopeHelpers::TRopeFastView(OwnedRope);
- Offset = 0;
- Size = whole.GetSize();
- PartSize = Size;
- }
-
- void ReferenceTo(const TRope& whole) {
- OwnedRope = whole;
+ Offset = 0;
+ Size = whole.GetSize();
+ PartSize = Size;
+ }
+
+ void ReferenceTo(const TRope& whole) {
+ OwnedRope = whole;
FastViewer = TRopeHelpers::TRopeFastView(OwnedRope);
- Offset = 0;
- Size = whole.GetSize();
- PartSize = Size;
- }
-
- void ReferenceTo(const TRope &piece, ui64 offset, ui64 size, ui64 partSize) {
- OwnedRope = piece;
+ Offset = 0;
+ Size = whole.GetSize();
+ PartSize = Size;
+ }
+
+ void ReferenceTo(const TRope &piece, ui64 offset, ui64 size, ui64 partSize) {
+ OwnedRope = piece;
FastViewer = TRopeHelpers::TRopeFastView(OwnedRope);
- Offset = offset;
- Y_VERIFY(size <= piece.GetSize());
- Size = size;
- Y_VERIFY(offset + size <= partSize);
- PartSize = partSize;
- FastViewer.SetOffset(Offset);
- }
-
- char *GetDataAt(ui64 getOffset) const {
- Y_VERIFY_DEBUG(Size);
- Y_VERIFY_DEBUG(getOffset >= Offset, "%s", (TStringBuilder() << "get_offset# " << getOffset
- << " Offset# " << Offset << " Size# " << Size << " capacity# " << OwnedRope.GetSize()).c_str());
- Y_VERIFY_DEBUG(getOffset < Offset + Size, "%s", (TStringBuilder() << "get_offset# " << getOffset
- << " Offset# " << Offset << " Size# " << Size << " capacity# " << OwnedRope.GetSize()).c_str());
- return FastViewer.DataPtrAt(getOffset);
- }
-
- char *GetDataAtSafe(ui64 getOffset) const {
- if (getOffset < Offset || getOffset >= Offset + Size) {
- return nullptr;
- }
- return FastViewer.DataPtrAt(getOffset);
- }
-
- ui64 MemoryConsumed() const {
- return OwnedRope.GetSize();
- }
-
- void ResetWithFullCopy() {
+ Offset = offset;
+ Y_VERIFY(size <= piece.GetSize());
+ Size = size;
+ Y_VERIFY(offset + size <= partSize);
+ PartSize = partSize;
+ FastViewer.SetOffset(Offset);
+ }
+
+ char *GetDataAt(ui64 getOffset) const {
+ Y_VERIFY_DEBUG(Size);
+ Y_VERIFY_DEBUG(getOffset >= Offset, "%s", (TStringBuilder() << "get_offset# " << getOffset
+ << " Offset# " << Offset << " Size# " << Size << " capacity# " << OwnedRope.GetSize()).c_str());
+ Y_VERIFY_DEBUG(getOffset < Offset + Size, "%s", (TStringBuilder() << "get_offset# " << getOffset
+ << " Offset# " << Offset << " Size# " << Size << " capacity# " << OwnedRope.GetSize()).c_str());
+ return FastViewer.DataPtrAt(getOffset);
+ }
+
+ char *GetDataAtSafe(ui64 getOffset) const {
+ if (getOffset < Offset || getOffset >= Offset + Size) {
+ return nullptr;
+ }
+ return FastViewer.DataPtrAt(getOffset);
+ }
+
+ ui64 MemoryConsumed() const {
+ return OwnedRope.GetSize();
+ }
+
+ void ResetWithFullCopy() {
OwnedRope = TRopeHelpers::RopeCopy(OwnedRope);
FastViewer = TRopeHelpers::TRopeFastView(OwnedRope);
- }
-
- ui64 GetContiguousSize(ui64 pos) const {
- return FastViewer.GetContiguousSize(pos);
- }
-
- char* FastDataPtrAt(size_t pos) {
- return FastViewer.FastDataPtrAt(pos);
- }
-};
-
-struct TDataPartSet {
- ui64 FullDataSize;
- ui32 PartsMask;
- TStackVec<TPartFragment, 8> Parts;
- TPartFragment FullDataFragment;
- ui64 MemoryConsumed;
- bool IsFragment;
-
- TDataPartSet()
- : FullDataSize(0)
- , PartsMask(0)
- , MemoryConsumed(0)
- , IsFragment(false)
- {}
-
- bool Is8Aligned() const {
- for (ui32 i = 0; i < 8u; ++i) {
+ }
+
+ ui64 GetContiguousSize(ui64 pos) const {
+ return FastViewer.GetContiguousSize(pos);
+ }
+
+ char* FastDataPtrAt(size_t pos) {
+ return FastViewer.FastDataPtrAt(pos);
+ }
+};
+
+struct TDataPartSet {
+ ui64 FullDataSize;
+ ui32 PartsMask;
+ TStackVec<TPartFragment, 8> Parts;
+ TPartFragment FullDataFragment;
+ ui64 MemoryConsumed;
+ bool IsFragment;
+
+ TDataPartSet()
+ : FullDataSize(0)
+ , PartsMask(0)
+ , MemoryConsumed(0)
+ , IsFragment(false)
+ {}
+
+ bool Is8Aligned() const {
+ for (ui32 i = 0; i < 8u; ++i) {
if ((PartsMask & (1u << i)) && !TRopeHelpers::Is8Aligned(Parts[i].OwnedRope)) {
- return false;
- }
- }
- return true;
- }
-
- void ResetWithFullCopy() {
- for (size_t i = 0; i < Parts.size(); ++i) {
- Parts[i].ResetWithFullCopy();
- }
- FullDataFragment.ResetWithFullCopy();
- }
-};
-
-struct TPartOffsetRange { // [Begin, End)
- ui64 Begin = 0;
- ui64 End = 0;
-
- ui64 WholeBegin = 0;
- ui64 WholeEnd = 0;
-
- ui64 AlignedBegin = 0;
- ui64 AlignedEnd = 0;
-
- // AlignedWholeEnd does not always exist because part are zero-padded
- ui64 AlignedWholeBegin = 0;
-
- bool IsEmpty() {
- return (End == 0);
- }
-
- void Reset() {
- Begin = 0;
- End = 0;
-
- WholeBegin = 0;
- WholeEnd = 0;
-
- AlignedBegin = 0;
- AlignedEnd = 0;
-
- AlignedWholeBegin = 0;
- }
-};
-
-struct TBlockSplitRange {
- ui64 BeginPartIdx = 0;
- ui64 EndPartIdx = 0;
- TStackVec<TPartOffsetRange, 8> PartRanges;
-};
-
-struct TErasureParameters;
-
-struct TRopeErasureType {
-
- enum EErasureSpecies {
- ErasureNone = 0,
- ErasureMirror3 = 1,
- Erasure3Plus1Block = 2,
- Erasure3Plus1Stripe = 3, // Not implemented in TRope version of erasure
-
- Erasure4Plus2Block = 4,
- Erasure3Plus2Block = 5,
- Erasure4Plus2Stripe = 6, // Not implemented in TRope version of erasure
- Erasure3Plus2Stripe = 7, // Not implemented in TRope version of erasure
-
- ErasureMirror3Plus2 = 8,
- ErasureMirror3dc = 9,
-
- Erasure4Plus3Block = 10,
- Erasure4Plus3Stripe = 11, // Not implemented in TRope version of erasure
- Erasure3Plus3Block = 12,
- Erasure3Plus3Stripe = 13, // Not implemented in TRope version of erasure
- Erasure2Plus3Block = 14,
- Erasure2Plus3Stripe = 15, // Not implemented in TRope version of erasure
-
- Erasure2Plus2Block = 16,
- Erasure2Plus2Stripe = 17, // Not implemented in TRope version of erasure
-
+ return false;
+ }
+ }
+ return true;
+ }
+
+ void ResetWithFullCopy() {
+ for (size_t i = 0; i < Parts.size(); ++i) {
+ Parts[i].ResetWithFullCopy();
+ }
+ FullDataFragment.ResetWithFullCopy();
+ }
+};
+
+struct TPartOffsetRange { // [Begin, End)
+ ui64 Begin = 0;
+ ui64 End = 0;
+
+ ui64 WholeBegin = 0;
+ ui64 WholeEnd = 0;
+
+ ui64 AlignedBegin = 0;
+ ui64 AlignedEnd = 0;
+
+ // AlignedWholeEnd does not always exist because part are zero-padded
+ ui64 AlignedWholeBegin = 0;
+
+ bool IsEmpty() {
+ return (End == 0);
+ }
+
+ void Reset() {
+ Begin = 0;
+ End = 0;
+
+ WholeBegin = 0;
+ WholeEnd = 0;
+
+ AlignedBegin = 0;
+ AlignedEnd = 0;
+
+ AlignedWholeBegin = 0;
+ }
+};
+
+struct TBlockSplitRange {
+ ui64 BeginPartIdx = 0;
+ ui64 EndPartIdx = 0;
+ TStackVec<TPartOffsetRange, 8> PartRanges;
+};
+
+struct TErasureParameters;
+
+struct TRopeErasureType {
+
+ enum EErasureSpecies {
+ ErasureNone = 0,
+ ErasureMirror3 = 1,
+ Erasure3Plus1Block = 2,
+ Erasure3Plus1Stripe = 3, // Not implemented in TRope version of erasure
+
+ Erasure4Plus2Block = 4,
+ Erasure3Plus2Block = 5,
+ Erasure4Plus2Stripe = 6, // Not implemented in TRope version of erasure
+ Erasure3Plus2Stripe = 7, // Not implemented in TRope version of erasure
+
+ ErasureMirror3Plus2 = 8,
+ ErasureMirror3dc = 9,
+
+ Erasure4Plus3Block = 10,
+ Erasure4Plus3Stripe = 11, // Not implemented in TRope version of erasure
+ Erasure3Plus3Block = 12,
+ Erasure3Plus3Stripe = 13, // Not implemented in TRope version of erasure
+ Erasure2Plus3Block = 14,
+ Erasure2Plus3Stripe = 15, // Not implemented in TRope version of erasure
+
+ Erasure2Plus2Block = 16,
+ Erasure2Plus2Stripe = 17, // Not implemented in TRope version of erasure
+
ErasureMirror3of4 = 18,
ErasureSpeciesCount = 19
- };
-
- static const char *ErasureSpeciesToStr(EErasureSpecies es);
-
- enum EErasureFamily {
- ErasureMirror,
- ErasureParityStripe,
- ErasureParityBlock
- };
-
- enum ECrcMode {
- CrcModeNone = 0,
- CrcModeWholePart = 1
- };
-
- TRopeErasureType(EErasureSpecies s = ErasureNone)
- : ErasureSpecies(s)
- {}
-
- virtual ~TRopeErasureType() = default;
- TRopeErasureType(const TRopeErasureType &) = default;
- TRopeErasureType &operator =(const TRopeErasureType &) = default;
-
- EErasureSpecies GetErasure() const {
- return ErasureSpecies;
- }
-
- TString ToString() const {
- Y_VERIFY((ui64)ErasureSpecies < ErasureSpeciesCount);
- return ErasureName[ErasureSpecies];
- }
-
- static TString ErasureSpeciesName(ui32 erasureSpecies) {
- if (erasureSpecies < ErasureSpeciesCount) {
- return ErasureName[erasureSpecies];
- }
- TStringStream str;
- str << "Unknown" << erasureSpecies;
- return str.Str();
- }
-
- static EErasureSpecies ErasureSpeciesByName(TString name) {
- for (ui32 species = 0; species < TRopeErasureType::ErasureSpeciesCount; ++species) {
- if (TRopeErasureType::ErasureName[species] == name) {
- return TRopeErasureType::EErasureSpecies(species);
- }
- }
- return TRopeErasureType::ErasureSpeciesCount;
- }
-
- ui32 ParityParts() const; // 4 + _2_
- ui32 DataParts() const; // _4_ + 2
- ui32 TotalPartCount() const; // _4_+_2_
- ui32 MinimalRestorablePartCount() const; // ? _4_ + 2
- ui32 MinimalBlockSize() const;
- // Size of user data contained in the part.
- ui64 PartUserSize(ui64 dataSize) const;
- // Size of the part including user data and crcs
- ui64 PartSize(ECrcMode crcMode, ui64 dataSize) const;
- ui32 Prime() const;
-
- void SplitData(ECrcMode crcMode, const TRope& buffer, TDataPartSet& outPartSet) const;
-
- void RestoreData(ECrcMode crcMode, TDataPartSet& partSet, TRope& outBuffer, bool restoreParts,
- bool restoreFullData, bool restoreParityParts) const;
-
- void RestoreData(ECrcMode crcMode, TDataPartSet& partSet, bool restoreParts, bool restoreFullData,
- bool restoreParityParts) const;
-
- void BlockSplitRange(ECrcMode crcMode, ui64 blobSize, ui64 wholeBegin, ui64 wholeEnd,
- TBlockSplitRange *outRange) const;
-
- static const std::array<TString, ErasureSpeciesCount> ErasureName;
-protected:
- EErasureSpecies ErasureSpecies;
-};
-
-bool CheckCrcAtTheEnd(TRopeErasureType::ECrcMode crcMode, const TRope& buf);
-
+ };
+
+ static const char *ErasureSpeciesToStr(EErasureSpecies es);
+
+ enum EErasureFamily {
+ ErasureMirror,
+ ErasureParityStripe,
+ ErasureParityBlock
+ };
+
+ enum ECrcMode {
+ CrcModeNone = 0,
+ CrcModeWholePart = 1
+ };
+
+ TRopeErasureType(EErasureSpecies s = ErasureNone)
+ : ErasureSpecies(s)
+ {}
+
+ virtual ~TRopeErasureType() = default;
+ TRopeErasureType(const TRopeErasureType &) = default;
+ TRopeErasureType &operator =(const TRopeErasureType &) = default;
+
+ EErasureSpecies GetErasure() const {
+ return ErasureSpecies;
+ }
+
+ TString ToString() const {
+ Y_VERIFY((ui64)ErasureSpecies < ErasureSpeciesCount);
+ return ErasureName[ErasureSpecies];
+ }
+
+ static TString ErasureSpeciesName(ui32 erasureSpecies) {
+ if (erasureSpecies < ErasureSpeciesCount) {
+ return ErasureName[erasureSpecies];
+ }
+ TStringStream str;
+ str << "Unknown" << erasureSpecies;
+ return str.Str();
+ }
+
+ static EErasureSpecies ErasureSpeciesByName(TString name) {
+ for (ui32 species = 0; species < TRopeErasureType::ErasureSpeciesCount; ++species) {
+ if (TRopeErasureType::ErasureName[species] == name) {
+ return TRopeErasureType::EErasureSpecies(species);
+ }
+ }
+ return TRopeErasureType::ErasureSpeciesCount;
+ }
+
+ ui32 ParityParts() const; // 4 + _2_
+ ui32 DataParts() const; // _4_ + 2
+ ui32 TotalPartCount() const; // _4_+_2_
+ ui32 MinimalRestorablePartCount() const; // ? _4_ + 2
+ ui32 MinimalBlockSize() const;
+ // Size of user data contained in the part.
+ ui64 PartUserSize(ui64 dataSize) const;
+ // Size of the part including user data and crcs
+ ui64 PartSize(ECrcMode crcMode, ui64 dataSize) const;
+ ui32 Prime() const;
+
+ void SplitData(ECrcMode crcMode, const TRope& buffer, TDataPartSet& outPartSet) const;
+
+ void RestoreData(ECrcMode crcMode, TDataPartSet& partSet, TRope& outBuffer, bool restoreParts,
+ bool restoreFullData, bool restoreParityParts) const;
+
+ void RestoreData(ECrcMode crcMode, TDataPartSet& partSet, bool restoreParts, bool restoreFullData,
+ bool restoreParityParts) const;
+
+ void BlockSplitRange(ECrcMode crcMode, ui64 blobSize, ui64 wholeBegin, ui64 wholeEnd,
+ TBlockSplitRange *outRange) const;
+
+ static const std::array<TString, ErasureSpeciesCount> ErasureName;
+protected:
+ EErasureSpecies ErasureSpecies;
+};
+
+bool CheckCrcAtTheEnd(TRopeErasureType::ECrcMode crcMode, const TRope& buf);
+
} // NKikimr
} // NErasureRope
-
+
diff --git a/ydb/core/erasure/erasure_rope_ut.cpp b/ydb/core/erasure/erasure_rope_ut.cpp
index 91ee7d1c6a..62a6242ea9 100644
--- a/ydb/core/erasure/erasure_rope_ut.cpp
+++ b/ydb/core/erasure/erasure_rope_ut.cpp
@@ -1,662 +1,662 @@
-#include "erasure_rope.h"
+#include "erasure_rope.h"
#include "ut_util.h"
-
-
-namespace NKikimr {
+
+
+namespace NKikimr {
namespace NErasureRope {
-
+
TRope GenerateRandomRope(size_t dataSize) {
- NPrivate::TMersenne64 randGen(Seed());
+ NPrivate::TMersenne64 randGen(Seed());
return TRopeHelpers::RopeFromStringMemcpy(GenerateRandomString(randGen, dataSize));
-}
-
-void TestMissingPartWithRandomData(TRopeErasureType &groupType, ui32 *missingPartIdx, ui32 missingParts,
- ui32 dataSize, bool isRestoreParts,bool isRestoreFullData,
- TString &info, std::function<TRope(size_t)> ropeGen = GenerateRandomRope) {
-
- ui32 partMask = ~(ui32)0;
- for (ui32 i = 0; i < missingParts; ++i) {
- partMask &= ~(ui32)(1ul << missingPartIdx[i]);
- }
-
- TString mode = Sprintf(" restoreParts=%s restoreFullData=%s ",
- (isRestoreParts ? "true" : "false"),
- (isRestoreFullData ? "true" : "false"));
- VERBOSE_COUT(" dataSize# " << dataSize << Endl);
-
- TRope testRope = ropeGen(dataSize);
- TString testString = testRope.ConvertToString();
-
- // Split the data into parts
- TDataPartSet partSet;
- groupType.SplitData(TRopeErasureType::CrcModeNone, testRope, partSet);
- ui64 partSize = groupType.PartSize(TRopeErasureType::CrcModeNone, dataSize);
- for (ui32 part = 0; part < groupType.TotalPartCount(); ++part) {
- UNIT_ASSERT_EQUAL(partSize, partSet.Parts[part].size());
- }
-
- // Save the original parts for the future checks
- TDataPartSet originalPartSet = partSet;
-
- // Remove the 'missing' parts
- partSet.PartsMask &= partMask;
- for (ui32 i = 0; i < missingParts; ++i) {
- partSet.Parts[missingPartIdx[i]].clear();
- }
- // Restore the data
- TRope restoredRope;
- groupType.RestoreData(TRopeErasureType::CrcModeNone, partSet, restoredRope,
- isRestoreParts, isRestoreFullData, isRestoreParts);
- TString restoredString = restoredRope.ConvertToString();
- // Make sure the restored data matches the original
- TString errorInfo = Sprintf("dataSize=%d partMask=0x%x", dataSize, partMask);
- if (isRestoreFullData) {
- UNIT_ASSERT_EQUAL_C(testString.size(), restoredString.size(), errorInfo);
- UNIT_ASSERT_EQUAL(testString, restoredString);
- }
-
- if (isRestoreParts) {
- for (ui32 idx = 0; idx < missingParts; ++idx) {
- if (missingPartIdx[idx] < partSet.Parts.size()) {
- UNIT_ASSERT_EQUAL_C(partSet.Parts[missingPartIdx[idx]].size(),
- originalPartSet.Parts[missingPartIdx[idx]].size(), info + errorInfo);
- ui32 size = (ui32)originalPartSet.Parts[missingPartIdx[idx]].size();
- TString restored = partSet.Parts[missingPartIdx[idx]].OwnedRope.ConvertToString();
- TString original = originalPartSet.Parts[missingPartIdx[idx]].OwnedRope.ConvertToString();
- for (ui32 i = 0; i < size; ++i) {
- UNIT_ASSERT_EQUAL_C(restored[i], original[i],
- (info + errorInfo + mode + Sprintf(" (part %d byte %d)", missingPartIdx[idx], i)));
- }
- }
- }
- }
-}
-
-template <ui32 maxMissingParts>
-void TestAllLossesDifferentSizes(TRopeErasureType &groupType, ui32 maxParts,
- std::function<TRope(size_t)> ropeGen = GenerateRandomRope) {
- for (ui32 missingParts = 0; missingParts <= maxMissingParts; ++missingParts) {
- ui32 missingPartIdx[maxMissingParts];
- GenFirstCombination(&missingPartIdx[0], missingParts);
- ui32 maxMissingVariants = Fact(maxParts)/Fact(missingParts)/Fact(maxParts-missingParts);
- //printf("k=%u, n=%u, variants=%u\n", missingParts, maxParts, maxMissingVariants);
- for (ui32 missingVariant = 0; missingVariant < maxMissingVariants; ++missingVariant) {
- VERBOSE_COUT(PrintArr(missingPartIdx, missingParts));
- ui32 partMask = ~(ui32)0;
- for (ui32 i = 0; i < missingParts; ++i) {
- partMask &= ~(ui32)(1ul << missingPartIdx[i]);
- }
- for (ui32 dataSize = 1; dataSize < 600; ++dataSize) {
- VERBOSE_COUT("dataSize# " << dataSize << Endl);
- for (ui32 type = 0; type < 3; ++type) {
- bool isRestoreParts = false;
- bool isRestoreFullData = false;
- switch (type) {
- case 0:
- isRestoreParts = true;
- isRestoreFullData = true;
- break;
- case 1:
- isRestoreFullData = true;
- break;
- case 2:
- isRestoreParts = true;
- break;
- }
- TStringStream info;
- info << "Type# " << groupType.ToString() << " ";
- info << "maxMissingParts# " << maxMissingParts << " ";
- info << "missingVariant# " << missingVariant << " ";
- info << "dataSize# " << dataSize << " ";
- info << "case# " << BoolToStr(isRestoreParts) << "," << BoolToStr(isRestoreFullData) << " ";
- VERBOSE_COUT(info.Str() << Endl);
- TestMissingPartWithRandomData(groupType, missingPartIdx, missingParts, dataSize,
- isRestoreParts, isRestoreFullData, info.Str(), ropeGen);
- }
- } // dataSize
- GenNextCombination(&missingPartIdx[0], missingParts, maxParts);
- }
- } // missingVariant
-}
-
-Y_UNIT_TEST_SUITE(TErasureTypeTest) {
-// Test if new version is capable to restore data splited by current version (which is right by definition)
- Y_UNIT_TEST(isSplittedDataEqualsToOldVerion) {
- TVector<TVector<ui8>> dataPool {
- {49,184,130,19,181,231,130},
-
- {249,122,57,146,140,30,69,51,88,81,92,29,220,192,18,14,195,162,244,139,59,141,161,14,
- 202,194,28,123,179,195,60,101,56,157,176,150,23,105,123,62,101,19,56,168,222,81,172,
- 251,199,223,85,60,99,184,45,90,84,68,1,131,199,36,64,103,150,221,18,236,86,15,142},
-
- {46,173,157,247,36,205,150,116,82,10,212,7,45,29,93,90,49,233,170,207,198,219,215,
- 187,220,220,48,228,83,53,50,37,153,214,149,28,231,171,92,176,230,139,168,126,
- 138,227,106,92,38,23,87,62,20,192,151,15,170,34,248,199,220,250,108,47,54,217,36,
- 56,146,224,21,148,133,155,49,199,101,250,173,93,104,205,67,222,132,104,187,231,53,
- 206,247,46,22,73,11,70,87,124,4,242,9,165,99,82,83,40,165,55,53,187,238,96,248,16,
- 103,197,132,216,107,191,229,140,90,129,81,63,232,85,19,232,59,96,193,5,133,139,251,
- 148,144,0,147,22,247,36,221,244,117,144,98,173,40} };
- TVector<TVector<TVector<ui8>>> partsPool {
- {
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
- {49,184,130,19,181,231,130,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
- {49,184,130,19,181,231,130,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49,184,130,19,181,231,130,0, },
- },{
- {249,122,57,146,140,30,69,51,88,81,92,29,220,192,18,14,195,162,244,139,59,141,161,14,202,
- 194,28,123,179,195,60,101,},
- {56,157,176,150,23,105,123,62,101,19,56,168,222,81,172,251,199,223,85,60,99,184,45,90,84,
- 68,1,131,199,36,64,103,},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
-
- {150,221,18,236,86,15,142,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
- {87,58,155,232,205,120,176,13,61,66,100,181,2,145,190,245,4,125,161,183,88,53,140,84,
- 158,134,29,248,116,231,124,2,},
- {173,62,56,17,75,58,5,84,52,136,237,8,12,141,41,87,242,245,205,160,34,248,77,146,207,
- 132,90,40,65,80,223,88,},
- },{
- {46,173,157,247,36,205,150,116,82,10,212,7,45,29,93,90,49,233,170,207,198,219,215,187,220,220,
- 48,228,83,53,50,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
- {153,214,149,28,231,171,92,176,230,139,168,126,138,227,106,92,38,23,87,62,20,192,151,15,170,
- 34,248,199,220,250,108,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
- {54,217,36,56,146,224,21,148,133,155,49,199,101,250,173,93,104,205,67,222,132,104,187,231,53,
- 206,247,46,22,73,11,70,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
- {87,124,4,242,9,165,99,82,83,40,165,55,53,187,238,96,248,16,103,197,132,216,107,191,229,140,
- 90,129,81,63,232,85,19,232,59,96,193,5,133,139,251,148,144,0,147,22,247,36,221,244,117,144,
- 98,173,40,0,0,0,0,0,0,0,0,0,},
- {214,222,40,33,88,35,188,2,98,50,232,137,247,191,116,59,135,35,217,234,210,171,144,236,166,
- 188,101,140,200,185,189,25,19,232,59,96,193,5,133,139,251,148,144,0,147,22,247,36,221,
- 244,117,144,98,173,40,0,0,0,0,0,0,0,0,0,},
- {114,180,19,50,219,117,207,37,191,151,5,180,246,160,208,23,112,124,56,167,179,241,145,219,
- 185,235,76,193,70,131,82,141,38,96,229,144,241,187,223,36,251,148,144,0,147,22,247,36,
- 251,148,144,0,147,22,247,36,232,124,171,96,82,19,114,175,},
- }
- };
- TRopeErasureType type(TRopeErasureType::EErasureSpecies::Erasure4Plus2Block);
- for (ui32 variant = 0; variant < dataPool.size(); ++variant) {
- TVector<ui8> &data = dataPool[variant];
- TVector<TVector<ui8>> &expectedParts = partsPool[variant];
- TString testString;
- testString.resize(data.size());
- for (ui32 i = 0; i < testString.size(); ++i) {
- testString[i] = (char)data[i];
- }
- TDataPartSet partSet;
+}
+
+void TestMissingPartWithRandomData(TRopeErasureType &groupType, ui32 *missingPartIdx, ui32 missingParts,
+ ui32 dataSize, bool isRestoreParts,bool isRestoreFullData,
+ TString &info, std::function<TRope(size_t)> ropeGen = GenerateRandomRope) {
+
+ ui32 partMask = ~(ui32)0;
+ for (ui32 i = 0; i < missingParts; ++i) {
+ partMask &= ~(ui32)(1ul << missingPartIdx[i]);
+ }
+
+ TString mode = Sprintf(" restoreParts=%s restoreFullData=%s ",
+ (isRestoreParts ? "true" : "false"),
+ (isRestoreFullData ? "true" : "false"));
+ VERBOSE_COUT(" dataSize# " << dataSize << Endl);
+
+ TRope testRope = ropeGen(dataSize);
+ TString testString = testRope.ConvertToString();
+
+ // Split the data into parts
+ TDataPartSet partSet;
+ groupType.SplitData(TRopeErasureType::CrcModeNone, testRope, partSet);
+ ui64 partSize = groupType.PartSize(TRopeErasureType::CrcModeNone, dataSize);
+ for (ui32 part = 0; part < groupType.TotalPartCount(); ++part) {
+ UNIT_ASSERT_EQUAL(partSize, partSet.Parts[part].size());
+ }
+
+ // Save the original parts for the future checks
+ TDataPartSet originalPartSet = partSet;
+
+ // Remove the 'missing' parts
+ partSet.PartsMask &= partMask;
+ for (ui32 i = 0; i < missingParts; ++i) {
+ partSet.Parts[missingPartIdx[i]].clear();
+ }
+ // Restore the data
+ TRope restoredRope;
+ groupType.RestoreData(TRopeErasureType::CrcModeNone, partSet, restoredRope,
+ isRestoreParts, isRestoreFullData, isRestoreParts);
+ TString restoredString = restoredRope.ConvertToString();
+ // Make sure the restored data matches the original
+ TString errorInfo = Sprintf("dataSize=%d partMask=0x%x", dataSize, partMask);
+ if (isRestoreFullData) {
+ UNIT_ASSERT_EQUAL_C(testString.size(), restoredString.size(), errorInfo);
+ UNIT_ASSERT_EQUAL(testString, restoredString);
+ }
+
+ if (isRestoreParts) {
+ for (ui32 idx = 0; idx < missingParts; ++idx) {
+ if (missingPartIdx[idx] < partSet.Parts.size()) {
+ UNIT_ASSERT_EQUAL_C(partSet.Parts[missingPartIdx[idx]].size(),
+ originalPartSet.Parts[missingPartIdx[idx]].size(), info + errorInfo);
+ ui32 size = (ui32)originalPartSet.Parts[missingPartIdx[idx]].size();
+ TString restored = partSet.Parts[missingPartIdx[idx]].OwnedRope.ConvertToString();
+ TString original = originalPartSet.Parts[missingPartIdx[idx]].OwnedRope.ConvertToString();
+ for (ui32 i = 0; i < size; ++i) {
+ UNIT_ASSERT_EQUAL_C(restored[i], original[i],
+ (info + errorInfo + mode + Sprintf(" (part %d byte %d)", missingPartIdx[idx], i)));
+ }
+ }
+ }
+ }
+}
+
+template <ui32 maxMissingParts>
+void TestAllLossesDifferentSizes(TRopeErasureType &groupType, ui32 maxParts,
+ std::function<TRope(size_t)> ropeGen = GenerateRandomRope) {
+ for (ui32 missingParts = 0; missingParts <= maxMissingParts; ++missingParts) {
+ ui32 missingPartIdx[maxMissingParts];
+ GenFirstCombination(&missingPartIdx[0], missingParts);
+ ui32 maxMissingVariants = Fact(maxParts)/Fact(missingParts)/Fact(maxParts-missingParts);
+ //printf("k=%u, n=%u, variants=%u\n", missingParts, maxParts, maxMissingVariants);
+ for (ui32 missingVariant = 0; missingVariant < maxMissingVariants; ++missingVariant) {
+ VERBOSE_COUT(PrintArr(missingPartIdx, missingParts));
+ ui32 partMask = ~(ui32)0;
+ for (ui32 i = 0; i < missingParts; ++i) {
+ partMask &= ~(ui32)(1ul << missingPartIdx[i]);
+ }
+ for (ui32 dataSize = 1; dataSize < 600; ++dataSize) {
+ VERBOSE_COUT("dataSize# " << dataSize << Endl);
+ for (ui32 type = 0; type < 3; ++type) {
+ bool isRestoreParts = false;
+ bool isRestoreFullData = false;
+ switch (type) {
+ case 0:
+ isRestoreParts = true;
+ isRestoreFullData = true;
+ break;
+ case 1:
+ isRestoreFullData = true;
+ break;
+ case 2:
+ isRestoreParts = true;
+ break;
+ }
+ TStringStream info;
+ info << "Type# " << groupType.ToString() << " ";
+ info << "maxMissingParts# " << maxMissingParts << " ";
+ info << "missingVariant# " << missingVariant << " ";
+ info << "dataSize# " << dataSize << " ";
+ info << "case# " << BoolToStr(isRestoreParts) << "," << BoolToStr(isRestoreFullData) << " ";
+ VERBOSE_COUT(info.Str() << Endl);
+ TestMissingPartWithRandomData(groupType, missingPartIdx, missingParts, dataSize,
+ isRestoreParts, isRestoreFullData, info.Str(), ropeGen);
+ }
+ } // dataSize
+ GenNextCombination(&missingPartIdx[0], missingParts, maxParts);
+ }
+ } // missingVariant
+}
+
+Y_UNIT_TEST_SUITE(TErasureTypeTest) {
+// Test if new version is capable to restore data splited by current version (which is right by definition)
+ Y_UNIT_TEST(isSplittedDataEqualsToOldVerion) {
+ TVector<TVector<ui8>> dataPool {
+ {49,184,130,19,181,231,130},
+
+ {249,122,57,146,140,30,69,51,88,81,92,29,220,192,18,14,195,162,244,139,59,141,161,14,
+ 202,194,28,123,179,195,60,101,56,157,176,150,23,105,123,62,101,19,56,168,222,81,172,
+ 251,199,223,85,60,99,184,45,90,84,68,1,131,199,36,64,103,150,221,18,236,86,15,142},
+
+ {46,173,157,247,36,205,150,116,82,10,212,7,45,29,93,90,49,233,170,207,198,219,215,
+ 187,220,220,48,228,83,53,50,37,153,214,149,28,231,171,92,176,230,139,168,126,
+ 138,227,106,92,38,23,87,62,20,192,151,15,170,34,248,199,220,250,108,47,54,217,36,
+ 56,146,224,21,148,133,155,49,199,101,250,173,93,104,205,67,222,132,104,187,231,53,
+ 206,247,46,22,73,11,70,87,124,4,242,9,165,99,82,83,40,165,55,53,187,238,96,248,16,
+ 103,197,132,216,107,191,229,140,90,129,81,63,232,85,19,232,59,96,193,5,133,139,251,
+ 148,144,0,147,22,247,36,221,244,117,144,98,173,40} };
+ TVector<TVector<TVector<ui8>>> partsPool {
+ {
+ {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
+ {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
+ {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
+ {49,184,130,19,181,231,130,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
+ {49,184,130,19,181,231,130,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
+ {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49,184,130,19,181,231,130,0, },
+ },{
+ {249,122,57,146,140,30,69,51,88,81,92,29,220,192,18,14,195,162,244,139,59,141,161,14,202,
+ 194,28,123,179,195,60,101,},
+ {56,157,176,150,23,105,123,62,101,19,56,168,222,81,172,251,199,223,85,60,99,184,45,90,84,
+ 68,1,131,199,36,64,103,},
+ {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
+
+ {150,221,18,236,86,15,142,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
+ {87,58,155,232,205,120,176,13,61,66,100,181,2,145,190,245,4,125,161,183,88,53,140,84,
+ 158,134,29,248,116,231,124,2,},
+ {173,62,56,17,75,58,5,84,52,136,237,8,12,141,41,87,242,245,205,160,34,248,77,146,207,
+ 132,90,40,65,80,223,88,},
+ },{
+ {46,173,157,247,36,205,150,116,82,10,212,7,45,29,93,90,49,233,170,207,198,219,215,187,220,220,
+ 48,228,83,53,50,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
+ {153,214,149,28,231,171,92,176,230,139,168,126,138,227,106,92,38,23,87,62,20,192,151,15,170,
+ 34,248,199,220,250,108,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
+ {54,217,36,56,146,224,21,148,133,155,49,199,101,250,173,93,104,205,67,222,132,104,187,231,53,
+ 206,247,46,22,73,11,70,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,},
+ {87,124,4,242,9,165,99,82,83,40,165,55,53,187,238,96,248,16,103,197,132,216,107,191,229,140,
+ 90,129,81,63,232,85,19,232,59,96,193,5,133,139,251,148,144,0,147,22,247,36,221,244,117,144,
+ 98,173,40,0,0,0,0,0,0,0,0,0,},
+ {214,222,40,33,88,35,188,2,98,50,232,137,247,191,116,59,135,35,217,234,210,171,144,236,166,
+ 188,101,140,200,185,189,25,19,232,59,96,193,5,133,139,251,148,144,0,147,22,247,36,221,
+ 244,117,144,98,173,40,0,0,0,0,0,0,0,0,0,},
+ {114,180,19,50,219,117,207,37,191,151,5,180,246,160,208,23,112,124,56,167,179,241,145,219,
+ 185,235,76,193,70,131,82,141,38,96,229,144,241,187,223,36,251,148,144,0,147,22,247,36,
+ 251,148,144,0,147,22,247,36,232,124,171,96,82,19,114,175,},
+ }
+ };
+ TRopeErasureType type(TRopeErasureType::EErasureSpecies::Erasure4Plus2Block);
+ for (ui32 variant = 0; variant < dataPool.size(); ++variant) {
+ TVector<ui8> &data = dataPool[variant];
+ TVector<TVector<ui8>> &expectedParts = partsPool[variant];
+ TString testString;
+ testString.resize(data.size());
+ for (ui32 i = 0; i < testString.size(); ++i) {
+ testString[i] = (char)data[i];
+ }
+ TDataPartSet partSet;
type.SplitData(TRopeErasureType::CrcModeNone, TRopeHelpers::RopeFromStringMemcpy(testString), partSet);
- for (ui32 i = 0; i < 6; ++i) {
- UNIT_ASSERT_EQUAL_C(partSet.Parts[i].size(), expectedParts[i].size(), Sprintf("%lu == %lu",
- partSet.Parts[i].size(), expectedParts[i].size()));
- for (ui32 j = 0; j < partSet.Parts[i].size(); ++j) {
- UNIT_ASSERT_EQUAL( (ui8)partSet.Parts[i].FastViewer.At8(j), expectedParts[i][j]);
- }
- }
- }
- }
-
- Y_UNIT_TEST(SpecialTest) {
-
- }
-
- Y_UNIT_TEST(TestTRopeErasureType) {
- ui64 dataSize = 1024;
- TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::ErasureMirror3);
-
- // for full coverage
- // TRopeErasureType::ErasureSpeciesName(TRopeErasureType::ErasureSpeciesCount);
-
- UNIT_ASSERT_EQUAL(dataSize, groupType.PartUserSize(dataSize));
- }
-
- Y_UNIT_TEST(TestEo) {
- ui32 species = (ui32)TRopeErasureType::Erasure4Plus2Block;
- {
- TRopeErasureType groupType((TRopeErasureType::EErasureSpecies)species);
-
- ui32 startingDataSize = 248;
-
- ui32 dataSize = startingDataSize;
- {
- const ui32 maxMissingParts = 4;
- ui32 missingPartIdx[maxMissingParts];
- for (ui32 i = 0; i < maxMissingParts; ++i) {
- missingPartIdx[i] = groupType.TotalPartCount();
- }
- missingPartIdx[0] = 2;
- missingPartIdx[1] = 3;
-
- ui32 maxMissingPartsTolerable = groupType.TotalPartCount() - groupType.MinimalRestorablePartCount();
- {
- ui32 partMask = ~(ui32)0;
- for (ui32 idx = maxMissingPartsTolerable - 1; idx != (ui32)-1; --idx) {
- partMask &= ~(ui32)(1 << missingPartIdx[idx]);
- }
- char mask[33];
- for (ui32 idx = 0; idx < 32; ++idx) {
- mask[idx] = (partMask & ((1ul << 31) >> idx)) ? '1' : '0';
- }
- mask[32] = 0;
-
- TString errorInfo = Sprintf("species=%d (%s) dataSize=%d partMask=0x%x (%s)", species,
- TRopeErasureType::ErasureSpeciesName(species).c_str(), dataSize, partMask, mask);
-
- TString testString;
- testString.resize(dataSize);
- for (ui32 i = 0; i < testString.size(); ++i) {
- ui32 col = (i / 8) % 4;
- ui32 row = (i / (2 * 8 * 4)) % 4;
- ui8 val = ui8(1 << col) | ui8(1 << (row + 4));
- ((char*)testString.data())[i] = val;
- }
- TDataPartSet partSet;
- try {
+ for (ui32 i = 0; i < 6; ++i) {
+ UNIT_ASSERT_EQUAL_C(partSet.Parts[i].size(), expectedParts[i].size(), Sprintf("%lu == %lu",
+ partSet.Parts[i].size(), expectedParts[i].size()));
+ for (ui32 j = 0; j < partSet.Parts[i].size(); ++j) {
+ UNIT_ASSERT_EQUAL( (ui8)partSet.Parts[i].FastViewer.At8(j), expectedParts[i][j]);
+ }
+ }
+ }
+ }
+
+ Y_UNIT_TEST(SpecialTest) {
+
+ }
+
+ Y_UNIT_TEST(TestTRopeErasureType) {
+ ui64 dataSize = 1024;
+ TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::ErasureMirror3);
+
+ // for full coverage
+ // TRopeErasureType::ErasureSpeciesName(TRopeErasureType::ErasureSpeciesCount);
+
+ UNIT_ASSERT_EQUAL(dataSize, groupType.PartUserSize(dataSize));
+ }
+
+ Y_UNIT_TEST(TestEo) {
+ ui32 species = (ui32)TRopeErasureType::Erasure4Plus2Block;
+ {
+ TRopeErasureType groupType((TRopeErasureType::EErasureSpecies)species);
+
+ ui32 startingDataSize = 248;
+
+ ui32 dataSize = startingDataSize;
+ {
+ const ui32 maxMissingParts = 4;
+ ui32 missingPartIdx[maxMissingParts];
+ for (ui32 i = 0; i < maxMissingParts; ++i) {
+ missingPartIdx[i] = groupType.TotalPartCount();
+ }
+ missingPartIdx[0] = 2;
+ missingPartIdx[1] = 3;
+
+ ui32 maxMissingPartsTolerable = groupType.TotalPartCount() - groupType.MinimalRestorablePartCount();
+ {
+ ui32 partMask = ~(ui32)0;
+ for (ui32 idx = maxMissingPartsTolerable - 1; idx != (ui32)-1; --idx) {
+ partMask &= ~(ui32)(1 << missingPartIdx[idx]);
+ }
+ char mask[33];
+ for (ui32 idx = 0; idx < 32; ++idx) {
+ mask[idx] = (partMask & ((1ul << 31) >> idx)) ? '1' : '0';
+ }
+ mask[32] = 0;
+
+ TString errorInfo = Sprintf("species=%d (%s) dataSize=%d partMask=0x%x (%s)", species,
+ TRopeErasureType::ErasureSpeciesName(species).c_str(), dataSize, partMask, mask);
+
+ TString testString;
+ testString.resize(dataSize);
+ for (ui32 i = 0; i < testString.size(); ++i) {
+ ui32 col = (i / 8) % 4;
+ ui32 row = (i / (2 * 8 * 4)) % 4;
+ ui8 val = ui8(1 << col) | ui8(1 << (row + 4));
+ ((char*)testString.data())[i] = val;
+ }
+ TDataPartSet partSet;
+ try {
groupType.SplitData(TRopeErasureType::CrcModeNone, TRopeHelpers::RopeFromStringMemcpy(testString), partSet);
- } catch (yexception ex) {
- ex << " [in SplitData while testing " << errorInfo << "]";
- throw ex;
- }
-
- ui64 partSize = groupType.PartSize(TRopeErasureType::CrcModeNone, dataSize);
- for (ui32 part = 0; part < groupType.TotalPartCount(); ++part) {
- UNIT_ASSERT_EQUAL_C(partSize, partSet.Parts[part].size(), errorInfo);
- }
-
- TDataPartSet originalPartSet = partSet;
-
- // Restore full data
- for (int type = 0; type < 1; ++type) {
- bool isRestoreFullData = false;
- bool isRestoreParts = false;
- switch (type) {
- case 0:
- isRestoreFullData = true;
- break;
- case 1:
- isRestoreParts = true;
- break;
- case 2:
- isRestoreFullData = true;
- isRestoreParts = true;
- break;
- default:
- Y_FAIL();
- }
-
- partSet = originalPartSet;
- for (ui32 idx = maxMissingPartsTolerable - 1; idx != (ui32)-1; --idx) {
- if (missingPartIdx[idx] < partSet.Parts.size()) {
- partSet.PartsMask &= partMask;
- partSet.Parts[missingPartIdx[idx]].clear();
- }
- }
-
- TString mode = Sprintf(" restoreParts=%s restoreFullData=%s ",
- (isRestoreParts ? "true" : "false"),
- (isRestoreFullData ? "true" : "false"));
-
- TRope restoredRope;
- try {
- groupType.RestoreData(TRopeErasureType::CrcModeNone, partSet, restoredRope,
- isRestoreParts, isRestoreFullData, isRestoreParts);
- } catch (yexception ex) {
- ex << " [in RestoreData while testing " << errorInfo << mode << "]";
- throw ex;
- }
- TString restoredString = restoredRope.ConvertToString();
-
- VERBOSE_COUT("testing " << errorInfo << mode << " (full data)" << Endl);
- if (isRestoreFullData) {
- UNIT_ASSERT_EQUAL_C(testString.size(), restoredString.size(), errorInfo);
- for (ui32 i = 0; i < testString.size(); ++i) {
- UNIT_ASSERT_EQUAL_C(((char*)testString.data())[i], ((char*)restoredString.data())[i],
- (errorInfo + mode + " (full data)"));
- if (((char*)testString.data())[i] != ((char*)restoredString.data())[i]) {
- VERBOSE_COUT("mismatch " << errorInfo << mode << " (full data)" << Endl);
- break;
- }
- }
- }
- if (isRestoreParts) {
- for (ui32 idx = maxMissingPartsTolerable - 1; idx != (ui32)-1; --idx) {
- if (missingPartIdx[idx] < partSet.Parts.size()) {
- UNIT_ASSERT_EQUAL_C(partSet.Parts[missingPartIdx[idx]].size(),
- originalPartSet.Parts[missingPartIdx[idx]].size(), errorInfo);
- ui32 size = (ui32)originalPartSet.Parts[missingPartIdx[idx]].size();
- TString restored = partSet.Parts[missingPartIdx[idx]].OwnedRope.ConvertToString();
- TString original = originalPartSet.Parts[missingPartIdx[idx]].OwnedRope.ConvertToString();
- for (ui32 i = 0; i < size; ++i) {
- UNIT_ASSERT_EQUAL_C(restored[i], original[i],
- (errorInfo + mode + Sprintf(" (part %d byte %d)", missingPartIdx[idx], i)));
- if (restored[i] != original[i]) {
- VERBOSE_COUT(" wrong part " << errorInfo << mode <<
- Sprintf(" (part %d byte %d)", missingPartIdx[idx], i) << Endl);
- break;
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
-
- // Mirror tests
- Y_UNIT_TEST(TestMirror3LossOfAllPossible3) {
- // Set up the erasure
- TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::ErasureMirror3);
- constexpr ui32 maxMissingParts = 2;
- constexpr ui32 maxParts = 1 + 2;
- TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
- }
-
- // Block tests
- Y_UNIT_TEST(TestBlock31LossOfAllPossible1) {
- // Set up the erasure
- TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure3Plus1Block);
- constexpr ui32 maxMissingParts = 1;
- constexpr ui32 maxParts = 3 + 1;
- TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
- }
-
- Y_UNIT_TEST(TestBlock42LossOfAllPossible2) {
- // Set up the erasure
- TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure4Plus2Block);
- // Specify
- constexpr ui32 maxMissingParts = 2;
- constexpr ui32 maxParts = 4 + 2;
- TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
- }
-
-// Y_UNIT_TEST(TestBlock42Not8AlignedSizes) {
-// TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure4Plus2Block);
-// constexpr ui32 maxMissingParts = 2;
-// constexpr ui32 maxParts = 4 + 2;
-//
-// {
-// auto ropeGen = [](size_t dataSize) {
-// TRope rope;
-// for (size_t i = 0; i < dataSize; ++i) {
-// TString base = GenerateRandomString(1);
+ } catch (yexception ex) {
+ ex << " [in SplitData while testing " << errorInfo << "]";
+ throw ex;
+ }
+
+ ui64 partSize = groupType.PartSize(TRopeErasureType::CrcModeNone, dataSize);
+ for (ui32 part = 0; part < groupType.TotalPartCount(); ++part) {
+ UNIT_ASSERT_EQUAL_C(partSize, partSet.Parts[part].size(), errorInfo);
+ }
+
+ TDataPartSet originalPartSet = partSet;
+
+ // Restore full data
+ for (int type = 0; type < 1; ++type) {
+ bool isRestoreFullData = false;
+ bool isRestoreParts = false;
+ switch (type) {
+ case 0:
+ isRestoreFullData = true;
+ break;
+ case 1:
+ isRestoreParts = true;
+ break;
+ case 2:
+ isRestoreFullData = true;
+ isRestoreParts = true;
+ break;
+ default:
+ Y_FAIL();
+ }
+
+ partSet = originalPartSet;
+ for (ui32 idx = maxMissingPartsTolerable - 1; idx != (ui32)-1; --idx) {
+ if (missingPartIdx[idx] < partSet.Parts.size()) {
+ partSet.PartsMask &= partMask;
+ partSet.Parts[missingPartIdx[idx]].clear();
+ }
+ }
+
+ TString mode = Sprintf(" restoreParts=%s restoreFullData=%s ",
+ (isRestoreParts ? "true" : "false"),
+ (isRestoreFullData ? "true" : "false"));
+
+ TRope restoredRope;
+ try {
+ groupType.RestoreData(TRopeErasureType::CrcModeNone, partSet, restoredRope,
+ isRestoreParts, isRestoreFullData, isRestoreParts);
+ } catch (yexception ex) {
+ ex << " [in RestoreData while testing " << errorInfo << mode << "]";
+ throw ex;
+ }
+ TString restoredString = restoredRope.ConvertToString();
+
+ VERBOSE_COUT("testing " << errorInfo << mode << " (full data)" << Endl);
+ if (isRestoreFullData) {
+ UNIT_ASSERT_EQUAL_C(testString.size(), restoredString.size(), errorInfo);
+ for (ui32 i = 0; i < testString.size(); ++i) {
+ UNIT_ASSERT_EQUAL_C(((char*)testString.data())[i], ((char*)restoredString.data())[i],
+ (errorInfo + mode + " (full data)"));
+ if (((char*)testString.data())[i] != ((char*)restoredString.data())[i]) {
+ VERBOSE_COUT("mismatch " << errorInfo << mode << " (full data)" << Endl);
+ break;
+ }
+ }
+ }
+ if (isRestoreParts) {
+ for (ui32 idx = maxMissingPartsTolerable - 1; idx != (ui32)-1; --idx) {
+ if (missingPartIdx[idx] < partSet.Parts.size()) {
+ UNIT_ASSERT_EQUAL_C(partSet.Parts[missingPartIdx[idx]].size(),
+ originalPartSet.Parts[missingPartIdx[idx]].size(), errorInfo);
+ ui32 size = (ui32)originalPartSet.Parts[missingPartIdx[idx]].size();
+ TString restored = partSet.Parts[missingPartIdx[idx]].OwnedRope.ConvertToString();
+ TString original = originalPartSet.Parts[missingPartIdx[idx]].OwnedRope.ConvertToString();
+ for (ui32 i = 0; i < size; ++i) {
+ UNIT_ASSERT_EQUAL_C(restored[i], original[i],
+ (errorInfo + mode + Sprintf(" (part %d byte %d)", missingPartIdx[idx], i)));
+ if (restored[i] != original[i]) {
+ VERBOSE_COUT(" wrong part " << errorInfo << mode <<
+ Sprintf(" (part %d byte %d)", missingPartIdx[idx], i) << Endl);
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // Mirror tests
+ Y_UNIT_TEST(TestMirror3LossOfAllPossible3) {
+ // Set up the erasure
+ TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::ErasureMirror3);
+ constexpr ui32 maxMissingParts = 2;
+ constexpr ui32 maxParts = 1 + 2;
+ TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
+ }
+
+ // Block tests
+ Y_UNIT_TEST(TestBlock31LossOfAllPossible1) {
+ // Set up the erasure
+ TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure3Plus1Block);
+ constexpr ui32 maxMissingParts = 1;
+ constexpr ui32 maxParts = 3 + 1;
+ TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
+ }
+
+ Y_UNIT_TEST(TestBlock42LossOfAllPossible2) {
+ // Set up the erasure
+ TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure4Plus2Block);
+ // Specify
+ constexpr ui32 maxMissingParts = 2;
+ constexpr ui32 maxParts = 4 + 2;
+ TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
+ }
+
+// Y_UNIT_TEST(TestBlock42Not8AlignedSizes) {
+// TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure4Plus2Block);
+// constexpr ui32 maxMissingParts = 2;
+// constexpr ui32 maxParts = 4 + 2;
+//
+// {
+// auto ropeGen = [](size_t dataSize) {
+// TRope rope;
+// for (size_t i = 0; i < dataSize; ++i) {
+// TString base = GenerateRandomString(1);
// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base));
-// }
-// Y_VERIFY(rope.GetSize() == dataSize);
-// return rope;
-// };
-// TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts, ropeGen);
-// }
-//
-// {
-// auto ropeGen = [](size_t dataSize) {
-// TRope rope;
-// for (size_t i = 1; i < dataSize && rope.GetSize() + i <= dataSize; ++i) {
-// TString base = GenerateRandomString(i);
+// }
+// Y_VERIFY(rope.GetSize() == dataSize);
+// return rope;
+// };
+// TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts, ropeGen);
+// }
+//
+// {
+// auto ropeGen = [](size_t dataSize) {
+// TRope rope;
+// for (size_t i = 1; i < dataSize && rope.GetSize() + i <= dataSize; ++i) {
+// TString base = GenerateRandomString(i);
// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base));
-// }
-// if (rope.GetSize() < dataSize) {
-// TString base = GenerateRandomString(dataSize - rope.GetSize());
+// }
+// if (rope.GetSize() < dataSize) {
+// TString base = GenerateRandomString(dataSize - rope.GetSize());
// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base));
-// }
-// Y_VERIFY(rope.GetSize() == dataSize);
-// return rope;
-// };
-// TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts, ropeGen);
-// }
-//
-// {
-// NPrivate::TMersenne64 randGen(Seed());
-// auto ropeGen = [&randGen](size_t dataSize) {
-// ui32 maxSize = (ui32) std::sqrt(dataSize) + 1;
-// maxSize = std::min(dataSize, (size_t) maxSize);
-// TRope rope;
-// while (rope.GetSize() < dataSize) {
-// ui64 size = (ui64) randGen.GenRand() % maxSize;
-// if (rope.GetSize() + size <= dataSize) {
-// TString base = GenerateRandomString(size);
+// }
+// Y_VERIFY(rope.GetSize() == dataSize);
+// return rope;
+// };
+// TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts, ropeGen);
+// }
+//
+// {
+// NPrivate::TMersenne64 randGen(Seed());
+// auto ropeGen = [&randGen](size_t dataSize) {
+// ui32 maxSize = (ui32) std::sqrt(dataSize) + 1;
+// maxSize = std::min(dataSize, (size_t) maxSize);
+// TRope rope;
+// while (rope.GetSize() < dataSize) {
+// ui64 size = (ui64) randGen.GenRand() % maxSize;
+// if (rope.GetSize() + size <= dataSize) {
+// TString base = GenerateRandomString(size);
// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base));
-// } else {
-// TString base = GenerateRandomString(dataSize - rope.GetSize());
+// } else {
+// TString base = GenerateRandomString(dataSize - rope.GetSize());
// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base));
-// }
-// }
-// Y_VERIFY(rope.GetSize() == dataSize);
-// return rope;
-// };
-// TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts, ropeGen);
-// }
-// }
-
- Y_UNIT_TEST(TestBlock32LossOfAllPossible2) {
- // Set up the erasure
- TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure3Plus2Block);
- constexpr ui32 maxMissingParts = 2;
- constexpr ui32 maxParts = 3 + 2;
- TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
- }
-
- Y_UNIT_TEST(TestBlock43LossOfAllPossible3) {
- // Set up the erasure
- TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure4Plus3Block);
- constexpr ui32 maxMissingParts = 3;
- constexpr ui32 maxParts = 4 + 3;
- TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
- }
-
- Y_UNIT_TEST(TestBlock33LossOfAllPossible3) {
- // Set up the erasure
- TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure3Plus3Block);
- constexpr ui32 maxMissingParts = 3;
- constexpr ui32 maxParts = 3 + 3;
- TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
- }
-
- Y_UNIT_TEST(TestBlock23LossOfAllPossible3) {
- // Set up the erasure
- TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure2Plus3Block);
- constexpr ui32 maxMissingParts = 3;
- constexpr ui32 maxParts = 2 + 3;
- TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
- }
-
- Y_UNIT_TEST(TestBlock22LossOfAllPossible2) {
- // Set up the erasure
- TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure2Plus2Block);
- constexpr ui32 maxMissingParts = 2;
- constexpr ui32 maxParts = 2 + 2;
- TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
- }
-
-
- void TestErasureMain(ui32 dataSize, TRopeErasureType::ECrcMode crcMode, ui32 species, const TRope& inp = TRope()) {
- TRopeErasureType groupType((TRopeErasureType::EErasureSpecies)species);
- TString erasureName = TRopeErasureType::ErasureName[species];
-
- const ui32 maxMissingParts = 4;
- ui32 missingPartIdx[maxMissingParts];
- for (ui32 i = 0; i < maxMissingParts; ++i) {
- missingPartIdx[i] = groupType.TotalPartCount();
- }
-
- ui32 maxMissingPartsTolerable = groupType.TotalPartCount() - groupType.MinimalRestorablePartCount();
- bool isComplete = false;
- while (!isComplete) {
- ui32 partMask = ~(ui32)0;
- for (ui32 idx = maxMissingPartsTolerable - 1; idx != (ui32)-1; --idx) {
- partMask &= ~(ui32)(1 << missingPartIdx[idx]);
- }
- char mask[33];
- for (ui32 idx = 0; idx < 32; ++idx) {
- mask[idx] = (partMask & ((1ul << 31) >> idx)) ? '1' : '0';
- }
- mask[32] = 0;
-
- TString errorInfo = Sprintf("crcMode=%d species=%d (%s) dataSize=%d partMask=0x%x (%s)",
- (i32)crcMode, species, TRopeErasureType::ErasureSpeciesName(species).c_str(),
- dataSize, partMask, mask);
-
- TString testString;
-
-
- TRope inputBuffer;
- if (inp.GetSize() == dataSize) {
- inputBuffer = inp;
- testString = inp.ConvertToString();
- } else {
- testString.resize(dataSize);
- for (ui32 i = 0; i < testString.size(); ++i) {
- ((char*)testString.data())[i] = (char)(i % 10) + '0';
- }
-
+// }
+// }
+// Y_VERIFY(rope.GetSize() == dataSize);
+// return rope;
+// };
+// TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts, ropeGen);
+// }
+// }
+
+ Y_UNIT_TEST(TestBlock32LossOfAllPossible2) {
+ // Set up the erasure
+ TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure3Plus2Block);
+ constexpr ui32 maxMissingParts = 2;
+ constexpr ui32 maxParts = 3 + 2;
+ TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
+ }
+
+ Y_UNIT_TEST(TestBlock43LossOfAllPossible3) {
+ // Set up the erasure
+ TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure4Plus3Block);
+ constexpr ui32 maxMissingParts = 3;
+ constexpr ui32 maxParts = 4 + 3;
+ TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
+ }
+
+ Y_UNIT_TEST(TestBlock33LossOfAllPossible3) {
+ // Set up the erasure
+ TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure3Plus3Block);
+ constexpr ui32 maxMissingParts = 3;
+ constexpr ui32 maxParts = 3 + 3;
+ TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
+ }
+
+ Y_UNIT_TEST(TestBlock23LossOfAllPossible3) {
+ // Set up the erasure
+ TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure2Plus3Block);
+ constexpr ui32 maxMissingParts = 3;
+ constexpr ui32 maxParts = 2 + 3;
+ TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
+ }
+
+ Y_UNIT_TEST(TestBlock22LossOfAllPossible2) {
+ // Set up the erasure
+ TRopeErasureType groupType(TRopeErasureType::EErasureSpecies::Erasure2Plus2Block);
+ constexpr ui32 maxMissingParts = 2;
+ constexpr ui32 maxParts = 2 + 2;
+ TestAllLossesDifferentSizes<maxMissingParts>(groupType, maxParts);
+ }
+
+
+ void TestErasureMain(ui32 dataSize, TRopeErasureType::ECrcMode crcMode, ui32 species, const TRope& inp = TRope()) {
+ TRopeErasureType groupType((TRopeErasureType::EErasureSpecies)species);
+ TString erasureName = TRopeErasureType::ErasureName[species];
+
+ const ui32 maxMissingParts = 4;
+ ui32 missingPartIdx[maxMissingParts];
+ for (ui32 i = 0; i < maxMissingParts; ++i) {
+ missingPartIdx[i] = groupType.TotalPartCount();
+ }
+
+ ui32 maxMissingPartsTolerable = groupType.TotalPartCount() - groupType.MinimalRestorablePartCount();
+ bool isComplete = false;
+ while (!isComplete) {
+ ui32 partMask = ~(ui32)0;
+ for (ui32 idx = maxMissingPartsTolerable - 1; idx != (ui32)-1; --idx) {
+ partMask &= ~(ui32)(1 << missingPartIdx[idx]);
+ }
+ char mask[33];
+ for (ui32 idx = 0; idx < 32; ++idx) {
+ mask[idx] = (partMask & ((1ul << 31) >> idx)) ? '1' : '0';
+ }
+ mask[32] = 0;
+
+ TString errorInfo = Sprintf("crcMode=%d species=%d (%s) dataSize=%d partMask=0x%x (%s)",
+ (i32)crcMode, species, TRopeErasureType::ErasureSpeciesName(species).c_str(),
+ dataSize, partMask, mask);
+
+ TString testString;
+
+
+ TRope inputBuffer;
+ if (inp.GetSize() == dataSize) {
+ inputBuffer = inp;
+ testString = inp.ConvertToString();
+ } else {
+ testString.resize(dataSize);
+ for (ui32 i = 0; i < testString.size(); ++i) {
+ ((char*)testString.data())[i] = (char)(i % 10) + '0';
+ }
+
inputBuffer = TRopeHelpers::RopeFromStringMemcpy(testString);
- }
-
- TDataPartSet partSet;
- try {
- VERBOSE_COUT("SplitData " << errorInfo << Endl);
- groupType.SplitData(crcMode, inputBuffer, partSet);
- } catch (yexception ex) {
- ex << " [in SplitData while testing " << errorInfo << "]";
- throw ex;
- }
-
- ui64 partSize = groupType.PartSize(crcMode, dataSize);
- for (ui32 part = 0; part < groupType.TotalPartCount(); ++part) {
- UNIT_ASSERT_EQUAL_C(partSize, partSet.Parts[part].size(), errorInfo);
- UNIT_ASSERT(CheckCrcAtTheEnd(crcMode, partSet.Parts[part].OwnedRope));
- }
-
- TDataPartSet originalPartSet = partSet;
-
- // Restore full data
- for (int type = 0; type < 5; ++type) {
- bool isRestoreFullData = false;
- bool isRestoreParts = false;
- bool isRestoreParityParts = false;
- switch (type) {
- case 0:
- isRestoreFullData = true;
- break;
- case 1:
- isRestoreParts = true;
- break;
- case 2:
- isRestoreFullData = true;
- isRestoreParts = true;
- break;
- case 3:
- isRestoreParts = true;
- isRestoreParityParts = true;
- break;
- case 4:
- isRestoreFullData = true;
- isRestoreParts = true;
- isRestoreParityParts = true;
- break;
- default:
- Y_FAIL();
- }
- partSet = originalPartSet;
- partSet.ResetWithFullCopy();
- for (ui32 idx = maxMissingPartsTolerable - 1; idx != (ui32)-1; --idx) {
- if (missingPartIdx[idx] < partSet.Parts.size()) {
- partSet.PartsMask &= partMask;
- partSet.Parts[missingPartIdx[idx]].clear();
- }
- }
- partSet.FullDataFragment.UninitializedOwnedWhole(dataSize);
-
-
- TString mode = Sprintf(" restoreParts=%s isRestoreParityParts=%s restoreFullData=%s ",
- (isRestoreParts ? "true" : "false"),
- (isRestoreParityParts ? "true" : "false"),
- (isRestoreFullData ? "true" : "false"));
-
- VERBOSE_COUT("RestoreData " << errorInfo << Endl);
- TRope restoredRope;
- try {
- groupType.RestoreData(crcMode, partSet, restoredRope,
- isRestoreParts, isRestoreFullData, isRestoreParityParts);
- } catch (yexception ex) {
- ex << " [in RestoreData while testing " << errorInfo << mode << "]";
- throw ex;
- }
- for (ui32 part = 0; part < groupType.TotalPartCount(); ++part) {
- if (part < groupType.DataParts() && isRestoreParts) {
- Y_VERIFY(CheckCrcAtTheEnd(crcMode, partSet.Parts[part].OwnedRope));
- } else if (part >= groupType.DataParts() && isRestoreParityParts) {
- Y_VERIFY(CheckCrcAtTheEnd(crcMode, partSet.Parts[part].OwnedRope));
- }
- }
- TString restoredString = restoredRope.ConvertToString();
- VERBOSE_COUT("testing " << errorInfo << mode << " (full data)" << Endl);
- if (isRestoreFullData) {
- UNIT_ASSERT_EQUAL_C(testString.size(), restoredString.size(), errorInfo);
- for (ui32 i = 0; i < testString.size(); ++i) {
- UNIT_ASSERT_EQUAL_C(((char*)testString.data())[i], ((char*)restoredString.data())[i],
- (errorInfo + erasureName + mode + " (full data)"));
- }
- }
- if (isRestoreParts) {
- for (ui32 idx = maxMissingPartsTolerable - 1; idx != (ui32)-1; --idx) {
- ui32 missingIdx = missingPartIdx[idx];
- if (missingIdx < partSet.Parts.size() &&
- (isRestoreParityParts || missingIdx < groupType.DataParts())) {
- UNIT_ASSERT_EQUAL_C(partSet.Parts[missingIdx].size(),
- originalPartSet.Parts[missingIdx].size(), errorInfo);
- ui32 size = (ui32)originalPartSet.Parts[missingIdx].size();
- if (size) {
- TString restored = partSet.Parts[missingIdx].OwnedRope.ConvertToString();
- TString original = originalPartSet.Parts[missingIdx].OwnedRope.ConvertToString();
- for (ui32 i = 0; i < size; ++i) {
- UNIT_ASSERT_EQUAL_C(restored[i], original[i],
- (errorInfo + erasureName + mode +
- Sprintf(" (part idx# %d of %d byte i# %d size# %d restored# %d original# %d)",
- missingIdx, (ui32)groupType.TotalPartCount(), i, size, (ui32)restored[i], (ui32)original[i])));
- }
- } else {
- UNIT_ASSERT(partSet.Parts[missingIdx].size() == 0);
- UNIT_ASSERT(originalPartSet.Parts[missingIdx].size() == 0);
- }
- }
- }
- }
- }
-
- if (maxMissingPartsTolerable == 0) {
- isComplete = true;
- }
- for (ui32 idx = maxMissingPartsTolerable - 1; idx != (ui32)-1; --idx) {
- missingPartIdx[idx]--;
- if (missingPartIdx[idx] != (ui32)-1) {
- break;
- }
- if (idx == 0) {
- isComplete = true;
- }
- missingPartIdx[idx] = groupType.TotalPartCount() - 1;
- }
- } // while !isComplete
- }
-
+ }
+
+ TDataPartSet partSet;
+ try {
+ VERBOSE_COUT("SplitData " << errorInfo << Endl);
+ groupType.SplitData(crcMode, inputBuffer, partSet);
+ } catch (yexception ex) {
+ ex << " [in SplitData while testing " << errorInfo << "]";
+ throw ex;
+ }
+
+ ui64 partSize = groupType.PartSize(crcMode, dataSize);
+ for (ui32 part = 0; part < groupType.TotalPartCount(); ++part) {
+ UNIT_ASSERT_EQUAL_C(partSize, partSet.Parts[part].size(), errorInfo);
+ UNIT_ASSERT(CheckCrcAtTheEnd(crcMode, partSet.Parts[part].OwnedRope));
+ }
+
+ TDataPartSet originalPartSet = partSet;
+
+ // Restore full data
+ for (int type = 0; type < 5; ++type) {
+ bool isRestoreFullData = false;
+ bool isRestoreParts = false;
+ bool isRestoreParityParts = false;
+ switch (type) {
+ case 0:
+ isRestoreFullData = true;
+ break;
+ case 1:
+ isRestoreParts = true;
+ break;
+ case 2:
+ isRestoreFullData = true;
+ isRestoreParts = true;
+ break;
+ case 3:
+ isRestoreParts = true;
+ isRestoreParityParts = true;
+ break;
+ case 4:
+ isRestoreFullData = true;
+ isRestoreParts = true;
+ isRestoreParityParts = true;
+ break;
+ default:
+ Y_FAIL();
+ }
+ partSet = originalPartSet;
+ partSet.ResetWithFullCopy();
+ for (ui32 idx = maxMissingPartsTolerable - 1; idx != (ui32)-1; --idx) {
+ if (missingPartIdx[idx] < partSet.Parts.size()) {
+ partSet.PartsMask &= partMask;
+ partSet.Parts[missingPartIdx[idx]].clear();
+ }
+ }
+ partSet.FullDataFragment.UninitializedOwnedWhole(dataSize);
+
+
+ TString mode = Sprintf(" restoreParts=%s isRestoreParityParts=%s restoreFullData=%s ",
+ (isRestoreParts ? "true" : "false"),
+ (isRestoreParityParts ? "true" : "false"),
+ (isRestoreFullData ? "true" : "false"));
+
+ VERBOSE_COUT("RestoreData " << errorInfo << Endl);
+ TRope restoredRope;
+ try {
+ groupType.RestoreData(crcMode, partSet, restoredRope,
+ isRestoreParts, isRestoreFullData, isRestoreParityParts);
+ } catch (yexception ex) {
+ ex << " [in RestoreData while testing " << errorInfo << mode << "]";
+ throw ex;
+ }
+ for (ui32 part = 0; part < groupType.TotalPartCount(); ++part) {
+ if (part < groupType.DataParts() && isRestoreParts) {
+ Y_VERIFY(CheckCrcAtTheEnd(crcMode, partSet.Parts[part].OwnedRope));
+ } else if (part >= groupType.DataParts() && isRestoreParityParts) {
+ Y_VERIFY(CheckCrcAtTheEnd(crcMode, partSet.Parts[part].OwnedRope));
+ }
+ }
+ TString restoredString = restoredRope.ConvertToString();
+ VERBOSE_COUT("testing " << errorInfo << mode << " (full data)" << Endl);
+ if (isRestoreFullData) {
+ UNIT_ASSERT_EQUAL_C(testString.size(), restoredString.size(), errorInfo);
+ for (ui32 i = 0; i < testString.size(); ++i) {
+ UNIT_ASSERT_EQUAL_C(((char*)testString.data())[i], ((char*)restoredString.data())[i],
+ (errorInfo + erasureName + mode + " (full data)"));
+ }
+ }
+ if (isRestoreParts) {
+ for (ui32 idx = maxMissingPartsTolerable - 1; idx != (ui32)-1; --idx) {
+ ui32 missingIdx = missingPartIdx[idx];
+ if (missingIdx < partSet.Parts.size() &&
+ (isRestoreParityParts || missingIdx < groupType.DataParts())) {
+ UNIT_ASSERT_EQUAL_C(partSet.Parts[missingIdx].size(),
+ originalPartSet.Parts[missingIdx].size(), errorInfo);
+ ui32 size = (ui32)originalPartSet.Parts[missingIdx].size();
+ if (size) {
+ TString restored = partSet.Parts[missingIdx].OwnedRope.ConvertToString();
+ TString original = originalPartSet.Parts[missingIdx].OwnedRope.ConvertToString();
+ for (ui32 i = 0; i < size; ++i) {
+ UNIT_ASSERT_EQUAL_C(restored[i], original[i],
+ (errorInfo + erasureName + mode +
+ Sprintf(" (part idx# %d of %d byte i# %d size# %d restored# %d original# %d)",
+ missingIdx, (ui32)groupType.TotalPartCount(), i, size, (ui32)restored[i], (ui32)original[i])));
+ }
+ } else {
+ UNIT_ASSERT(partSet.Parts[missingIdx].size() == 0);
+ UNIT_ASSERT(originalPartSet.Parts[missingIdx].size() == 0);
+ }
+ }
+ }
+ }
+ }
+
+ if (maxMissingPartsTolerable == 0) {
+ isComplete = true;
+ }
+ for (ui32 idx = maxMissingPartsTolerable - 1; idx != (ui32)-1; --idx) {
+ missingPartIdx[idx]--;
+ if (missingPartIdx[idx] != (ui32)-1) {
+ break;
+ }
+ if (idx == 0) {
+ isComplete = true;
+ }
+ missingPartIdx[idx] = groupType.TotalPartCount() - 1;
+ }
+ } // while !isComplete
+ }
+
void TestErasure(TRopeErasureType::ECrcMode crcMode, ui32 species, ui32 dataSizeOffset = 0, ui32 dataSizeStep = 1) {
- ui32 startingDataSize = 0;
- TRopeErasureType groupType((TRopeErasureType::EErasureSpecies)species);
- ui32 maxDataSize = groupType.MinimalBlockSize() * 8;
-
+ ui32 startingDataSize = 0;
+ TRopeErasureType groupType((TRopeErasureType::EErasureSpecies)species);
+ ui32 maxDataSize = groupType.MinimalBlockSize() * 8;
+
for (ui32 dataSize = startingDataSize + dataSizeOffset; dataSize < maxDataSize; dataSize += dataSizeStep) {
- TestErasureMain(dataSize, crcMode, species);
- }
- }
-
- void TestErasure(TRopeErasureType::ECrcMode crcMode, ui32 species, const TVector<ui32>& sizes) {
- for (ui32 dataSize : sizes) {
- TestErasureMain(dataSize, crcMode, species);
- }
- }
-
- ui32 SpeciesForTest[] {
+ TestErasureMain(dataSize, crcMode, species);
+ }
+ }
+
+ void TestErasure(TRopeErasureType::ECrcMode crcMode, ui32 species, const TVector<ui32>& sizes) {
+ for (ui32 dataSize : sizes) {
+ TestErasureMain(dataSize, crcMode, species);
+ }
+ }
+
+ ui32 SpeciesForTest[] {
TRopeErasureType::Erasure4Plus2Block, // 1
TRopeErasureType::Erasure2Plus2Block, // 2
TRopeErasureType::Erasure3Plus2Block, // 3
@@ -668,16 +668,16 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) {
TRopeErasureType::ErasureMirror3, // 1
TRopeErasureType::ErasureMirror3Plus2, // 2
TRopeErasureType::ErasureMirror3dc // 3
- };
-
- ui32 NumberOfTestSpecies = sizeof(SpeciesForTest) / sizeof(ui32);
-
+ };
+
+ ui32 NumberOfTestSpecies = sizeof(SpeciesForTest) / sizeof(ui32);
+
Y_UNIT_TEST(TestAllSpeciesCrcWhole1of4) {
for (ui32 species = 0; species < NumberOfTestSpecies; species += 4) {
- TestErasure(TRopeErasureType::CrcModeWholePart, SpeciesForTest[species]);
- }
- }
-
+ TestErasure(TRopeErasureType::CrcModeWholePart, SpeciesForTest[species]);
+ }
+ }
+
Y_UNIT_TEST(TestAllSpeciesCrcWhole2of4) {
for (ui32 species = 1; species < NumberOfTestSpecies; species += 4) {
TestErasure(TRopeErasureType::CrcModeWholePart, SpeciesForTest[species]);
@@ -714,19 +714,19 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) {
}
}
- Y_UNIT_TEST(TestBigSizes) {
- TVector<ui32> sizes{1234567};
- for (ui32 species = 0; species < NumberOfTestSpecies; ++species) {
- TestErasure(TRopeErasureType::CrcModeNone, SpeciesForTest[species], sizes);
- }
- }
-
+ Y_UNIT_TEST(TestBigSizes) {
+ TVector<ui32> sizes{1234567};
+ for (ui32 species = 0; species < NumberOfTestSpecies; ++species) {
+ TestErasure(TRopeErasureType::CrcModeNone, SpeciesForTest[species], sizes);
+ }
+ }
+
Y_UNIT_TEST(TestAllSpecies1of2) {
for (ui32 species = 0; species < NumberOfTestSpecies; species += 4) {
- TestErasure(TRopeErasureType::CrcModeNone, SpeciesForTest[species]);
- }
- }
-
+ TestErasure(TRopeErasureType::CrcModeNone, SpeciesForTest[species]);
+ }
+ }
+
Y_UNIT_TEST(TestAllSpecies2of4) {
for (ui32 species = 1; species < NumberOfTestSpecies; species += 4) {
TestErasure(TRopeErasureType::CrcModeNone, SpeciesForTest[species]);
@@ -747,68 +747,68 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) {
- Y_UNIT_TEST(TestBlockByteOrder) {
- ui32 species = (ui32)TRopeErasureType::Erasure4Plus2Block;
- TRopeErasureType groupType((TRopeErasureType::EErasureSpecies)species);
- TString erasureName = TRopeErasureType::ErasureName[species];
-
- for (ui32 dataSize = 0; dataSize <= 256; ++dataSize) {
- TString testString;
- testString.resize(dataSize);
- for (ui32 i = 0; i < testString.size(); ++i) {
- ((ui8*)testString.data())[i] = (ui8)i;
- }
- TDataPartSet partSet;
+ Y_UNIT_TEST(TestBlockByteOrder) {
+ ui32 species = (ui32)TRopeErasureType::Erasure4Plus2Block;
+ TRopeErasureType groupType((TRopeErasureType::EErasureSpecies)species);
+ TString erasureName = TRopeErasureType::ErasureName[species];
+
+ for (ui32 dataSize = 0; dataSize <= 256; ++dataSize) {
+ TString testString;
+ testString.resize(dataSize);
+ for (ui32 i = 0; i < testString.size(); ++i) {
+ ((ui8*)testString.data())[i] = (ui8)i;
+ }
+ TDataPartSet partSet;
groupType.SplitData(TRopeErasureType::CrcModeNone, TRopeHelpers::RopeFromStringMemcpy(testString), partSet);
- for (ui32 p = 0; p < groupType.DataParts(); ++p) {
- auto &part = partSet.Parts[p];
- VERBOSE_COUT("Part# " << p << " Size# " << part.size() << " Data# ");
- if (part.size() == 0) {
- VERBOSE_COUT(" --- ");
- } else {
- ui32 begin = (ui32)*(ui8*)part.GetDataAt(0);
- ui32 prev = (ui32)*(ui8*)part.GetDataAt(0);
- for (ui32 i = 1; i < part.size(); ++i) {
- ui32 cur = (ui32)*(ui8*)part.GetDataAt(i);
- if (cur == prev + 1) {
- prev = cur;
- } else {
- if (begin == prev) {
- VERBOSE_COUT(begin << " ");
- } else {
- VERBOSE_COUT(begin << ".." << prev << " ");
- }
- begin = cur;
- prev = cur;
- }
- }
- if (begin == prev) {
- VERBOSE_COUT(begin << " ");
- } else {
- VERBOSE_COUT(begin << ".." << prev << " ");
- }
- }
- VERBOSE_COUT(" ");
- }
- VERBOSE_COUT(Endl);
- }
- }
-
-
+ for (ui32 p = 0; p < groupType.DataParts(); ++p) {
+ auto &part = partSet.Parts[p];
+ VERBOSE_COUT("Part# " << p << " Size# " << part.size() << " Data# ");
+ if (part.size() == 0) {
+ VERBOSE_COUT(" --- ");
+ } else {
+ ui32 begin = (ui32)*(ui8*)part.GetDataAt(0);
+ ui32 prev = (ui32)*(ui8*)part.GetDataAt(0);
+ for (ui32 i = 1; i < part.size(); ++i) {
+ ui32 cur = (ui32)*(ui8*)part.GetDataAt(i);
+ if (cur == prev + 1) {
+ prev = cur;
+ } else {
+ if (begin == prev) {
+ VERBOSE_COUT(begin << " ");
+ } else {
+ VERBOSE_COUT(begin << ".." << prev << " ");
+ }
+ begin = cur;
+ prev = cur;
+ }
+ }
+ if (begin == prev) {
+ VERBOSE_COUT(begin << " ");
+ } else {
+ VERBOSE_COUT(begin << ".." << prev << " ");
+ }
+ }
+ VERBOSE_COUT(" ");
+ }
+ VERBOSE_COUT(Endl);
+ }
+ }
+
+
void TestBlock42PartialRestore(ui32 missingVariant) {
- // Set up the erasure
- TRopeErasureType groupType(TRopeErasureType::Erasure4Plus2Block);
-
- // Specify the missing part indexes
- const ui32 maxMissingParts = 2;
- ui32 missingPartsToTest[] = {0, 1, 2, 4, 3, 5, 4, 5};
+ // Set up the erasure
+ TRopeErasureType groupType(TRopeErasureType::Erasure4Plus2Block);
+
+ // Specify the missing part indexes
+ const ui32 maxMissingParts = 2;
+ ui32 missingPartsToTest[] = {0, 1, 2, 4, 3, 5, 4, 5};
ui32 missingPartIdx[maxMissingParts];
missingPartIdx[0] = missingPartsToTest[missingVariant * 2];
missingPartIdx[1] = missingPartsToTest[missingVariant * 2 + 1];
ui32 partMask = ~(ui32) 0;
partMask &= ~(ui32) (1 << missingPartIdx[0]);
partMask &= ~(ui32) (1 << missingPartIdx[1]);
-
+
// Prepare the test data
TString testString;
for (ui32 dataSize = 1; dataSize < 600; ++dataSize) {
@@ -820,7 +820,7 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) {
for (ui32 i = 0; i < testString.size(); ++i) {
((char *) testString.data())[i] = (char) (i % 10) + '0';
}
-
+
// Split the data into parts
TDataPartSet partSet;
groupType.SplitData(TRopeErasureType::CrcModeNone, TRopeHelpers::RopeFromStringMemcpy(testString), partSet);
@@ -828,12 +828,12 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) {
for (ui32 part = 0; part < groupType.TotalPartCount(); ++part) {
UNIT_ASSERT_EQUAL(partSize, partSet.Parts[part].size());
}
-
+
// Save the original parts for the future checks
TDataPartSet originalPartSet = partSet;
originalPartSet.ResetWithFullCopy();
-
-
+
+
// TODO: Test different offsets and sizes
for (ui64 partialSize = 1; partialSize < Min((ui32) dataSize, (ui32) 512); ++partialSize) {
VERBOSE_COUT( "partialSize# " << partialSize << Endl);
@@ -844,7 +844,7 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) {
ui64 specialCount = sizeof(partialShiftSpecials) / sizeof(partialShiftSpecials[0]);
ui64 normalCount = 70;
ui64 totalCount = normalCount + specialCount;
-
+
for (ui64 caseIdx = 0; caseIdx < totalCount; ++caseIdx) {
ui64 partialShift = caseIdx < normalCount ?
caseIdx : partialShiftSpecials[caseIdx - normalCount];
@@ -853,13 +853,13 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) {
}
VERBOSE_COUT( "partialShift# " << partialShift << Endl);
partSet = originalPartSet;
-
+
ui64 shift = Max<ui64>();
ui64 size = Max<ui64>();
-
+
ui64 needBegin = Max<ui64>();
ui64 needEnd = Max<ui64>();
-
+
TBlockSplitRange range1;
groupType.BlockSplitRange(TRopeErasureType::CrcModeNone, dataSize, partialShift,
partialShift + partialSize, &range1);
@@ -867,22 +867,22 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) {
TPartOffsetRange &r = range1.PartRanges[partIdx];
if (shift == Max<ui64>() || shift > r.AlignedWholeBegin) {
shift = r.AlignedWholeBegin;
- }
+ }
if (size == Max<ui64>() || size < r.AlignedWholeBegin + r.AlignedEnd - r.AlignedBegin) {
size = r.AlignedWholeBegin + r.AlignedEnd - r.AlignedBegin;
- }
+ }
if (needBegin == Max<ui64>() || needBegin > r.AlignedBegin) {
needBegin = r.AlignedBegin;
- }
+ }
if (needEnd == Max<ui64>() || needEnd < r.AlignedEnd) {
needEnd = r.AlignedEnd;
- }
+ }
}
if (size > dataSize) {
size = dataSize;
}
size -= shift;
-
+
ui64 partSize = groupType.PartSize(TRopeErasureType::CrcModeNone, dataSize);
for (ui32 idx = 0; idx < partSet.Parts.size(); ++idx) {
ui32 cutBegin = Min(partSize, needBegin);
@@ -891,7 +891,7 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) {
partSet.Parts[idx].OwnedRope.ConvertToString().substr(
cutBegin, cutSize)), cutBegin, cutSize, partSize);
}
-
+
// Remove the 'missing' parts
partSet.PartsMask &= partMask;
for (ui32 i = 0; i < 2; ++i) {
@@ -926,13 +926,13 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) {
TString string = partSet.Parts[partIdx].OwnedRope.ConvertToString();
const char* actual = string.data() + partRange.AlignedBegin - partSet.Parts[partIdx].Offset;
UNIT_ASSERT(memcmp(expected, actual, checkSize) == 0);
- }
+ }
}
-
+
} // partialShift
} // partialSize
} // dataSize
- }
+ }
Y_UNIT_TEST(TestBlock42PartialRestore0) {
TestBlock42PartialRestore(0);
@@ -949,7 +949,7 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) {
Y_UNIT_TEST(TestBlock42PartialRestore3) {
TestBlock42PartialRestore(3);
}
-}
-
-} // namespace NKikimr
+}
+
+} // namespace NKikimr
} // NErasureRope
diff --git a/ydb/core/erasure/ut_perf/ya.make b/ydb/core/erasure/ut_perf/ya.make
index 683b053669..c7d6e3a810 100644
--- a/ydb/core/erasure/ut_perf/ya.make
+++ b/ydb/core/erasure/ut_perf/ya.make
@@ -1,15 +1,15 @@
UNITTEST_FOR(ydb/core/erasure)
-
-FORK_SUBTESTS()
-SPLIT_FACTOR(30)
-
-TIMEOUT(60)
-SIZE(SMALL)
-
-OWNER(ddoarn cthulhu fomichev g:kikimr)
-
-SRCS(
- erasure_perf_test.cpp
-)
-
-END()
+
+FORK_SUBTESTS()
+SPLIT_FACTOR(30)
+
+TIMEOUT(60)
+SIZE(SMALL)
+
+OWNER(ddoarn cthulhu fomichev g:kikimr)
+
+SRCS(
+ erasure_perf_test.cpp
+)
+
+END()
diff --git a/ydb/core/erasure/ut_rope/ya.make b/ydb/core/erasure/ut_rope/ya.make
index b8fa413b4d..bcba5b5729 100644
--- a/ydb/core/erasure/ut_rope/ya.make
+++ b/ydb/core/erasure/ut_rope/ya.make
@@ -1,32 +1,32 @@
UNITTEST_FOR(ydb/core/erasure)
-
-FORK_SUBTESTS()
-SPLIT_FACTOR(30)
-
+FORK_SUBTESTS()
+
+SPLIT_FACTOR(30)
+
IF (WITH_VALGRIND OR SANITIZER_TYPE == "thread")
- TIMEOUT(1800)
- SIZE(LARGE)
- TAG(ya:fat)
-ELSE()
- TIMEOUT(600)
- SIZE(MEDIUM)
-ENDIF()
-
+ TIMEOUT(1800)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
OWNER(
cthulhu
ddoarn
fomichev
g:kikimr
)
-
-PEERDIR(
+
+PEERDIR(
library/cpp/digest/crc32c
ydb/core/erasure
-)
-
-SRCS(
- erasure_rope_ut.cpp
-)
-
-END()
+)
+
+SRCS(
+ erasure_rope_ut.cpp
+)
+
+END()
diff --git a/ydb/core/erasure/ya.make b/ydb/core/erasure/ya.make
index 9cd55e0447..63483c1b96 100644
--- a/ydb/core/erasure/ya.make
+++ b/ydb/core/erasure/ya.make
@@ -11,8 +11,8 @@ OWNER(
SRCS(
erasure.cpp
erasure.h
- erasure_rope.cpp
- erasure_rope.h
+ erasure_rope.cpp
+ erasure_rope.h
erasure_perf_test.cpp
)
@@ -31,9 +31,9 @@ IF (MSVC)
ENDIF()
END()
-
-RECURSE_FOR_TESTS(
- ut
- ut_rope
- ut_perf
-)
+
+RECURSE_FOR_TESTS(
+ ut
+ ut_rope
+ ut_perf
+)
diff --git a/ydb/services/ydb/ydb_stats_ut.cpp b/ydb/services/ydb/ydb_stats_ut.cpp
index e8919bf702..b40e22fc42 100644
--- a/ydb/services/ydb/ydb_stats_ut.cpp
+++ b/ydb/services/ydb/ydb_stats_ut.cpp
@@ -145,10 +145,10 @@ public:
void OnHistogram(TInstant, NMonitoring::IHistogramSnapshotPtr) override { }
- void OnLogHistogram(TInstant, NMonitoring::TLogHistogramSnapshotPtr) override { }
-
- void OnSummaryDouble(TInstant, NMonitoring::ISummaryDoubleSnapshotPtr) override { }
-
+ void OnLogHistogram(TInstant, NMonitoring::TLogHistogramSnapshotPtr) override { }
+
+ void OnSummaryDouble(TInstant, NMonitoring::ISummaryDoubleSnapshotPtr) override { }
+
ECounterType State = ECounterType::UNKNOWN;
TStatCounters Counters;
};