1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
|
#include "codecs.h"
#include "common.h"
#include "register.h"
#include <util/ysaveload.h>
#include <util/stream/null.h>
#include <util/stream/mem.h>
#include <util/string/cast.h>
#include <util/string/join.h>
#include <util/system/align.h>
#include <util/system/unaligned_mem.h>
#include <util/generic/hash.h>
#include <util/generic/cast.h>
#include <util/generic/deque.h>
#include <util/generic/buffer.h>
#include <util/generic/array_ref.h>
#include <util/generic/singleton.h>
#include <util/generic/algorithm.h>
#include <util/generic/mem_copy.h>
using namespace NBlockCodecs;
namespace {
struct TCodecFactory {
inline TCodecFactory() {
Add(&Null);
}
inline const ICodec* Find(const TStringBuf& name) const {
auto it = Registry.find(name);
if (it == Registry.end()) {
ythrow TNotFound() << "can not found " << name << " codec";
}
return it->second;
}
inline void ListCodecs(TCodecList& lst) const {
for (const auto& it : Registry) {
lst.push_back(it.first);
}
Sort(lst.begin(), lst.end());
}
inline void Add(ICodec* codec) {
Registry[codec->Name()] = codec;
}
inline void Add(TCodecPtr codec) {
Codecs.push_back(std::move(codec));
Add(Codecs.back().Get());
}
inline void Alias(TStringBuf from, TStringBuf to) {
Tmp.emplace_back(from);
Registry[Tmp.back()] = Registry[to];
}
TDeque<TString> Tmp;
TNullCodec Null;
TVector<TCodecPtr> Codecs;
typedef THashMap<TStringBuf, ICodec*> TRegistry;
TRegistry Registry;
// SEARCH-8344: Global decompressed size limiter (to prevent remote DoS)
size_t MaxPossibleDecompressedLength = Max<size_t>();
};
}
const ICodec* NBlockCodecs::Codec(const TStringBuf& name) {
return Singleton<TCodecFactory>()->Find(name);
}
TCodecList NBlockCodecs::ListAllCodecs() {
TCodecList ret;
Singleton<TCodecFactory>()->ListCodecs(ret);
return ret;
}
TString NBlockCodecs::ListAllCodecsAsString() {
return JoinSeq(TStringBuf(","), ListAllCodecs());
}
void NBlockCodecs::RegisterCodec(TCodecPtr codec) {
Singleton<TCodecFactory>()->Add(std::move(codec));
}
void NBlockCodecs::RegisterAlias(TStringBuf from, TStringBuf to) {
Singleton<TCodecFactory>()->Alias(from, to);
}
void NBlockCodecs::SetMaxPossibleDecompressedLength(size_t maxPossibleDecompressedLength) {
Singleton<TCodecFactory>()->MaxPossibleDecompressedLength = maxPossibleDecompressedLength;
}
size_t NBlockCodecs::GetMaxPossibleDecompressedLength() {
return Singleton<TCodecFactory>()->MaxPossibleDecompressedLength;
}
size_t ICodec::GetDecompressedLength(const TData& in) const {
const size_t len = DecompressedLength(in);
Y_ENSURE(
len <= NBlockCodecs::GetMaxPossibleDecompressedLength(),
"Attempt to decompress the block that is larger than maximum possible decompressed length, "
"see SEARCH-8344 for details. "
);
return len;
}
void ICodec::Encode(const TData& in, TBuffer& out) const {
const size_t maxLen = MaxCompressedLength(in);
out.Reserve(maxLen);
out.Resize(Compress(in, out.Data()));
}
void ICodec::Decode(const TData& in, TBuffer& out) const {
const size_t len = GetDecompressedLength(in);
out.Reserve(len);
out.Resize(Decompress(in, out.Data()));
}
void ICodec::Encode(const TData& in, TString& out) const {
const size_t maxLen = MaxCompressedLength(in);
out.ReserveAndResize(maxLen);
size_t actualLen = Compress(in, out.begin());
Y_ASSERT(actualLen <= maxLen);
out.resize(actualLen);
}
void ICodec::Decode(const TData& in, TString& out) const {
const size_t maxLen = GetDecompressedLength(in);
out.ReserveAndResize(maxLen);
size_t actualLen = Decompress(in, out.begin());
Y_ASSERT(actualLen <= maxLen);
out.resize(actualLen);
}
ICodec::~ICodec() = default;
|
