1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
|
#include <library/cpp/comptable/comptable.h>
#include <library/cpp/testing/unittest/registar.h>
#include <util/random/random.h>
#include <util/random/fast.h>
using namespace NCompTable;
template <bool HQ>
void DoTest(const TCompressorTable& table, const TVector<TString>& lines) {
TVector<char> compressed;
TVector<char> decompressed;
TChunkCompressor compressor(HQ, table);
TStringStream tmp;
Save(&tmp, table);
TCompressorTable tableLoaded;
Load(&tmp, tableLoaded);
UNIT_ASSERT(memcmp(&table, &tableLoaded, sizeof(table)) == 0);
TChunkDecompressor deCompressor(HQ, tableLoaded);
size_t origSize = 0;
size_t compSize = 0;
for (size_t i = 0; i < lines.size(); ++i) {
const TString& line = lines[i];
compressor.Compress(line, &compressed);
origSize += line.size();
compSize += compressed.size();
TStringBuf in(compressed.data(), compressed.size());
deCompressor.Decompress(in, &decompressed);
UNIT_ASSERT(decompressed.size() == line.size() && memcmp(decompressed.data(), line.data(), decompressed.size()) == 0);
}
UNIT_ASSERT_EQUAL(origSize, 45491584);
if (HQ) {
UNIT_ASSERT_EQUAL(compSize, 11074583);
} else {
UNIT_ASSERT_EQUAL(compSize, 17459336);
}
UNIT_ASSERT(compSize < origSize);
}
Y_UNIT_TEST_SUITE(TestComptable) {
Y_UNIT_TEST(TestComptableCompressDecompress) {
TReallyFastRng32 rr(17);
TVector<TString> lines;
for (size_t i = 0; i < 1000000; ++i) {
size_t size = rr.Uniform(32);
TString res = "www.yandex.ru/yandsearch?text=";
for (size_t j = 0; j < size; ++j) {
res += "qwer"[rr.Uniform(4)];
}
lines.push_back(res);
}
THolder<TDataSampler> sampler(new TDataSampler);
for (size_t i = 0; i < lines.size(); ++i) {
sampler->AddStat(lines[i]);
}
TCompressorTable table;
sampler->BuildTable(table);
DoTest<true>(table, lines);
DoTest<false>(table, lines);
}
}
|
