aboutsummaryrefslogtreecommitdiffstats
path: root/library/cpp/tokenizer/multitokenutil.h
diff options
context:
space:
mode:
authorqrort <qrort@yandex-team.com>2022-12-02 11:31:25 +0300
committerqrort <qrort@yandex-team.com>2022-12-02 11:31:25 +0300
commitb1f4ffc9c8abff3ba58dc1ec9a9f92d2f0de6806 (patch)
tree2a23209faf0fea5586a6d4b9cee60d1b318d29fe /library/cpp/tokenizer/multitokenutil.h
parent559174a9144de40d6bb3997ea4073c82289b4974 (diff)
downloadydb-b1f4ffc9c8abff3ba58dc1ec9a9f92d2f0de6806.tar.gz
remove kikimr/driver DEPENDS
Diffstat (limited to 'library/cpp/tokenizer/multitokenutil.h')
-rw-r--r--library/cpp/tokenizer/multitokenutil.h22
1 files changed, 0 insertions, 22 deletions
diff --git a/library/cpp/tokenizer/multitokenutil.h b/library/cpp/tokenizer/multitokenutil.h
deleted file mode 100644
index 19781f67858..00000000000
--- a/library/cpp/tokenizer/multitokenutil.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#pragma once
-
-#include <library/cpp/token/nlptypes.h>
-#include <library/cpp/token/token_structure.h>
-
-void CorrectDelimiters(TCharSpan& prevtok, wchar16 suffixChar, TCharSpan& lasttok, wchar16 prefixChar);
-
-//! removes hyphenations and replaces unicode delimiters
-//! @return new length of multitoken
-NLP_TYPE PrepareMultitoken(TTokenStructure& subtokens, wchar16* buffer, size_t buflen, const wchar16* entry, size_t& len);
-
-//! cuts off the subtokens according to the specified maximum length
-//! @return new length of the subtokens
-size_t AdjustSubtokens(TTokenStructure& subtokens, size_t maxLen);
-
-//! corrects positions of subtokens and cuts off their length according to the specified maximum
-//! @note the first @c n characters are accents
-//! @return new length of the subtokens
-size_t AdjustSubtokens(TTokenStructure& subtokens, size_t n, size_t maxLen);
-
-//! for debugging purposes only
-bool CheckMultitoken(const TWideToken& tok);