library/cpp/codecs/static/tools/common/ct_common.cpp


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

#include "ct_common.h"

#include <library/cpp/codecs/codecs.h>
#include <library/cpp/codecs/static/static_codec_info.pb.h>
#include <library/cpp/string_utils/base64/base64.h>

#include <util/stream/output.h>
#include <util/string/builder.h>
#include <util/system/hp_timer.h>

namespace NCodecs {
    TString TComprStats::Format(const TStaticCodecInfo& info, bool checkMode) const {
        TStringBuilder s;
        s << "raw size/item:      " << RawSizePerRecord() << Endl;
        s << "enc.size/item:      " << EncSizePerRecord() << Endl;
        if (checkMode) {
            s << "orig.enc.size/item: " << OldEncSizePerRecord(info.GetDebugInfo().GetCompression()) << Endl;
        }
        s << "enc time us/item:   " << EncTimePerRecordUS() << Endl;
        s << "dec time us/item:   " << DecTimePerRecordUS() << Endl;
        s << "dict size:          " << info.GetStoredCodec().Size() << Endl;
        s << "compression:        " << AsPercent(Compression()) << " %" << Endl;
        if (checkMode) {
            s << "orig.compression:   " << AsPercent(info.GetDebugInfo().GetCompression()) << " %" << Endl;
        }
        return s;
    }

    TComprStats TestCodec(const ICodec& c, const TVector<TString>& input) {
        TComprStats stats;

        TBuffer encodeBuffer;
        TBuffer decodeBuffer;
        for (const auto& data : input) {
            encodeBuffer.Clear();
            decodeBuffer.Clear();

            stats.Records += 1;
            stats.RawSize += data.size(); 

            THPTimer timer;
            c.Encode(data, encodeBuffer);
            stats.EncSize += encodeBuffer.size(); 
            stats.EncSeconds += timer.PassedReset();

            c.Decode(TStringBuf{encodeBuffer.data(), encodeBuffer.size()}, decodeBuffer); 
            stats.DecSeconds += timer.PassedReset();
            Y_ENSURE(data == TStringBuf(decodeBuffer.data(), decodeBuffer.size()), "invalid encoding at record " << stats.Records); 
        }

        return stats;
    }

    void ParseBlob(TVector<TString>& result, EDataStreamFormat fmt, const TBlob& blob) {
        TStringBuf bin(blob.AsCharPtr(), blob.Size()); 
        TStringBuf line;
        TString buffer;
        while (bin.ReadLine(line)) {
            if (DSF_BASE64_LF == fmt) {
                Base64Decode(line, buffer);
                line = buffer;
            }
            if (!line) {
                continue;
            }
            result.emplace_back(line.data(), line.size()); 
        }
    }

    TBlob GetInputBlob(const TString& dataFile) {
        return dataFile && dataFile != "-" ? TBlob::FromFile(dataFile) : TBlob::FromStream(Cin);
    }

}