1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
#include "clickhouse_config.h"
#if USE_NLP
#include <Columns/ColumnArray.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnVector.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeString.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionHelpers.h>
#include <Functions/IFunction.h>
#include <Interpreters/Context.h>
#include <Interpreters/SynonymsExtensions.h>
#include <string_view>
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_COLUMN;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int SUPPORT_IS_DISABLED;
}
class FunctionSynonyms : public IFunction
{
public:
static constexpr auto name = "synonyms";
static FunctionPtr create(ContextPtr context)
{
if (!context->getSettingsRef().allow_experimental_nlp_functions)
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
"Natural language processing function '{}' is experimental. "
"Set `allow_experimental_nlp_functions` setting to enable it", name);
return std::make_shared<FunctionSynonyms>(context->getSynonymsExtensions());
}
private:
SynonymsExtensions & extensions;
public:
explicit FunctionSynonyms(SynonymsExtensions & extensions_)
: extensions(extensions_) {}
String getName() const override { return name; }
size_t getNumberOfArguments() const override { return 2; }
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (!isString(arguments[0]))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}",
arguments[0]->getName(), getName());
if (!isString(arguments[1]))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}",
arguments[1]->getName(), getName());
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
}
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
const auto & extcolumn = arguments[0].column;
const auto & strcolumn = arguments[1].column;
const ColumnConst * ext_col = checkAndGetColumn<ColumnConst>(extcolumn.get());
const ColumnString * word_col = checkAndGetColumn<ColumnString>(strcolumn.get());
if (!ext_col)
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}",
arguments[0].column->getName(), getName());
if (!word_col)
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}",
arguments[1].column->getName(), getName());
String ext_name = ext_col->getValue<String>();
auto extension = extensions.getExtension(ext_name);
/// Create and fill the result array.
const DataTypePtr & elem_type = static_cast<const DataTypeArray &>(*result_type).getNestedType();
auto out = ColumnArray::create(elem_type->createColumn());
IColumn & out_data = out->getData();
IColumn::Offsets & out_offsets = out->getOffsets();
const ColumnString::Chars & data = word_col->getChars();
const ColumnString::Offsets & offsets = word_col->getOffsets();
out_data.reserve(input_rows_count);
out_offsets.resize(input_rows_count);
IColumn::Offset current_offset = 0;
for (size_t i = 0; i < offsets.size(); ++i)
{
std::string_view word(reinterpret_cast<const char *>(data.data() + offsets[i - 1]), offsets[i] - offsets[i - 1] - 1);
const auto * synset = extension->getSynonyms(word);
if (synset)
{
for (const auto & token : *synset)
out_data.insert(Field(token.data(), token.size()));
current_offset += synset->size();
}
out_offsets[i] = current_offset;
}
return out;
}
};
REGISTER_FUNCTION(Synonyms)
{
factory.registerFunction<FunctionSynonyms>({}, FunctionFactory::CaseInsensitive);
}
}
#endif
|
