1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
#pragma once
#include <base/types.h>
#include <Interpreters/BloomFilter.h>
#include <Interpreters/GinFilter.h>
namespace DB
{
/// Interface for string parsers.
struct ITokenExtractor
{
virtual ~ITokenExtractor() = default;
/// Fast inplace implementation for regular use.
/// Gets string (data ptr and len) and start position for extracting next token (state of extractor).
/// Returns false if parsing is finished, otherwise returns true.
virtual bool nextInString(const char * data, size_t length, size_t * __restrict pos, size_t * __restrict token_start, size_t * __restrict token_length) const = 0;
/// Optimized version that can assume at least 15 padding bytes after data + len (as our Columns provide).
virtual bool nextInStringPadded(const char * data, size_t length, size_t * __restrict pos, size_t * __restrict token_start, size_t * __restrict token_length) const
{
return nextInString(data, length, pos, token_start, token_length);
}
/// Special implementation for creating bloom filter for LIKE function.
/// It skips unescaped `%` and `_` and supports escaping symbols, but it is less lightweight.
virtual bool nextInStringLike(const char * data, size_t length, size_t * pos, String & out) const = 0;
virtual void stringToBloomFilter(const char * data, size_t length, BloomFilter & bloom_filter) const = 0;
virtual void stringPaddedToBloomFilter(const char * data, size_t length, BloomFilter & bloom_filter) const
{
return stringToBloomFilter(data, length, bloom_filter);
}
virtual void stringLikeToBloomFilter(const char * data, size_t length, BloomFilter & bloom_filter) const = 0;
virtual void stringToGinFilter(const char * data, size_t length, GinFilter & gin_filter) const = 0;
virtual void stringPaddedToGinFilter(const char * data, size_t length, GinFilter & gin_filter) const
{
return stringToGinFilter(data, length, gin_filter);
}
virtual void stringLikeToGinFilter(const char * data, size_t length, GinFilter & gin_filter) const = 0;
};
using TokenExtractorPtr = const ITokenExtractor *;
template <typename Derived>
class ITokenExtractorHelper : public ITokenExtractor
{
void stringToBloomFilter(const char * data, size_t length, BloomFilter & bloom_filter) const override
{
size_t cur = 0;
size_t token_start = 0;
size_t token_len = 0;
while (cur < length && static_cast<const Derived *>(this)->nextInString(data, length, &cur, &token_start, &token_len))
bloom_filter.add(data + token_start, token_len);
}
void stringPaddedToBloomFilter(const char * data, size_t length, BloomFilter & bloom_filter) const override
{
size_t cur = 0;
size_t token_start = 0;
size_t token_len = 0;
while (cur < length && static_cast<const Derived *>(this)->nextInStringPadded(data, length, &cur, &token_start, &token_len))
bloom_filter.add(data + token_start, token_len);
}
void stringLikeToBloomFilter(const char * data, size_t length, BloomFilter & bloom_filter) const override
{
size_t cur = 0;
String token;
while (cur < length && static_cast<const Derived *>(this)->nextInStringLike(data, length, &cur, token))
bloom_filter.add(token.c_str(), token.size());
}
void stringToGinFilter(const char * data, size_t length, GinFilter & gin_filter) const override
{
gin_filter.setQueryString(data, length);
size_t cur = 0;
size_t token_start = 0;
size_t token_len = 0;
while (cur < length && static_cast<const Derived *>(this)->nextInString(data, length, &cur, &token_start, &token_len))
gin_filter.addTerm(data + token_start, token_len);
}
void stringPaddedToGinFilter(const char * data, size_t length, GinFilter & gin_filter) const override
{
gin_filter.setQueryString(data, length);
size_t cur = 0;
size_t token_start = 0;
size_t token_len = 0;
while (cur < length && static_cast<const Derived *>(this)->nextInStringPadded(data, length, &cur, &token_start, &token_len))
gin_filter.addTerm(data + token_start, token_len);
}
void stringLikeToGinFilter(const char * data, size_t length, GinFilter & gin_filter) const override
{
gin_filter.setQueryString(data, length);
size_t cur = 0;
String token;
while (cur < length && static_cast<const Derived *>(this)->nextInStringLike(data, length, &cur, token))
gin_filter.addTerm(token.c_str(), token.size());
}
};
/// Parser extracting all ngrams from string.
struct NgramTokenExtractor final : public ITokenExtractorHelper<NgramTokenExtractor>
{
explicit NgramTokenExtractor(size_t n_) : n(n_) {}
static const char * getName() { return "ngrambf_v1"; }
bool nextInString(const char * data, size_t length, size_t * __restrict pos, size_t * __restrict token_start, size_t * __restrict token_length) const override;
bool nextInStringLike(const char * data, size_t length, size_t * pos, String & token) const override;
size_t getN() const { return n; }
private:
size_t n;
};
/// Parser extracting tokens (sequences of numbers and ascii letters).
struct SplitTokenExtractor final : public ITokenExtractorHelper<SplitTokenExtractor>
{
static const char * getName() { return "tokenbf_v1"; }
bool nextInString(const char * data, size_t length, size_t * __restrict pos, size_t * __restrict token_start, size_t * __restrict token_length) const override;
bool nextInStringPadded(const char * data, size_t length, size_t * __restrict pos, size_t * __restrict token_start, size_t * __restrict token_length) const override;
bool nextInStringLike(const char * data, size_t length, size_t * __restrict pos, String & token) const override;
};
}
|
