diff options
author | qrort <qrort@yandex-team.com> | 2022-12-02 11:31:25 +0300 |
---|---|---|
committer | qrort <qrort@yandex-team.com> | 2022-12-02 11:31:25 +0300 |
commit | b1f4ffc9c8abff3ba58dc1ec9a9f92d2f0de6806 (patch) | |
tree | 2a23209faf0fea5586a6d4b9cee60d1b318d29fe /library/cpp/token/token_util.h | |
parent | 559174a9144de40d6bb3997ea4073c82289b4974 (diff) | |
download | ydb-b1f4ffc9c8abff3ba58dc1ec9a9f92d2f0de6806.tar.gz |
remove kikimr/driver DEPENDS
Diffstat (limited to 'library/cpp/token/token_util.h')
-rw-r--r-- | library/cpp/token/token_util.h | 30 |
1 files changed, 0 insertions, 30 deletions
diff --git a/library/cpp/token/token_util.h b/library/cpp/token/token_util.h deleted file mode 100644 index 7383bdc840e..00000000000 --- a/library/cpp/token/token_util.h +++ /dev/null @@ -1,30 +0,0 @@ -#pragma once - -#include "token_structure.h" - -#include <util/system/yassert.h> -#include <util/generic/string.h> - -TUtf16String RemoveWideTokenPrefix(TWideToken& token); -TUtf16String RemoveWideTokenSuffix(TWideToken& token); - -// Check if we can split wide-token after specified sub-token. -// The function does't allow to split on dash and apostrophe betwenn normal words -bool CheckWideTokenSplit(const TWideToken& token, size_t pos); -// Check if we can split wide-token after specified sub-token by dot delimiter. -// The function verifies the following condition: -// <word> <dot> <word with uppercased first character or number> -// <number> <dot> <word with uppercased first character> -bool CheckWideTokenDotSplit(const TWideToken& token, size_t pos); - -// Check if we can split wide-token after specified sub-token. -// The function uses rich-tree specific heuristics -bool CheckWideTokenReqSplit(const TTokenStructure& subtokens, size_t pos); - -inline size_t GetSubTokenOffset(const TWideToken& tok, size_t subToken) { - Y_ASSERT(subToken < tok.SubTokens.size()); - return tok.SubTokens[subToken].Pos - tok.SubTokens[subToken].PrefixLen; -} - -// Create a new wide-token with the specified inclusive [start, end] sub-token range -TWideToken ExtractWideTokenRange(const TWideToken& tok, size_t start, size_t end); |