diff options
author | qrort <qrort@yandex-team.com> | 2022-12-02 11:31:25 +0300 |
---|---|---|
committer | qrort <qrort@yandex-team.com> | 2022-12-02 11:31:25 +0300 |
commit | b1f4ffc9c8abff3ba58dc1ec9a9f92d2f0de6806 (patch) | |
tree | 2a23209faf0fea5586a6d4b9cee60d1b318d29fe /library/cpp/tokenizer/multitokenutil.h | |
parent | 559174a9144de40d6bb3997ea4073c82289b4974 (diff) | |
download | ydb-b1f4ffc9c8abff3ba58dc1ec9a9f92d2f0de6806.tar.gz |
remove kikimr/driver DEPENDS
Diffstat (limited to 'library/cpp/tokenizer/multitokenutil.h')
-rw-r--r-- | library/cpp/tokenizer/multitokenutil.h | 22 |
1 files changed, 0 insertions, 22 deletions
diff --git a/library/cpp/tokenizer/multitokenutil.h b/library/cpp/tokenizer/multitokenutil.h deleted file mode 100644 index 19781f67858..00000000000 --- a/library/cpp/tokenizer/multitokenutil.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include <library/cpp/token/nlptypes.h> -#include <library/cpp/token/token_structure.h> - -void CorrectDelimiters(TCharSpan& prevtok, wchar16 suffixChar, TCharSpan& lasttok, wchar16 prefixChar); - -//! removes hyphenations and replaces unicode delimiters -//! @return new length of multitoken -NLP_TYPE PrepareMultitoken(TTokenStructure& subtokens, wchar16* buffer, size_t buflen, const wchar16* entry, size_t& len); - -//! cuts off the subtokens according to the specified maximum length -//! @return new length of the subtokens -size_t AdjustSubtokens(TTokenStructure& subtokens, size_t maxLen); - -//! corrects positions of subtokens and cuts off their length according to the specified maximum -//! @note the first @c n characters are accents -//! @return new length of the subtokens -size_t AdjustSubtokens(TTokenStructure& subtokens, size_t n, size_t maxLen); - -//! for debugging purposes only -bool CheckMultitoken(const TWideToken& tok); |