aboutsummaryrefslogtreecommitdiffstats
path: root/library/cpp/token/token_util.h
diff options
context:
space:
mode:
authorqrort <qrort@yandex-team.com>2022-12-02 11:31:25 +0300
committerqrort <qrort@yandex-team.com>2022-12-02 11:31:25 +0300
commitb1f4ffc9c8abff3ba58dc1ec9a9f92d2f0de6806 (patch)
tree2a23209faf0fea5586a6d4b9cee60d1b318d29fe /library/cpp/token/token_util.h
parent559174a9144de40d6bb3997ea4073c82289b4974 (diff)
downloadydb-b1f4ffc9c8abff3ba58dc1ec9a9f92d2f0de6806.tar.gz
remove kikimr/driver DEPENDS
Diffstat (limited to 'library/cpp/token/token_util.h')
-rw-r--r--library/cpp/token/token_util.h30
1 files changed, 0 insertions, 30 deletions
diff --git a/library/cpp/token/token_util.h b/library/cpp/token/token_util.h
deleted file mode 100644
index 7383bdc840e..00000000000
--- a/library/cpp/token/token_util.h
+++ /dev/null
@@ -1,30 +0,0 @@
-#pragma once
-
-#include "token_structure.h"
-
-#include <util/system/yassert.h>
-#include <util/generic/string.h>
-
-TUtf16String RemoveWideTokenPrefix(TWideToken& token);
-TUtf16String RemoveWideTokenSuffix(TWideToken& token);
-
-// Check if we can split wide-token after specified sub-token.
-// The function does't allow to split on dash and apostrophe betwenn normal words
-bool CheckWideTokenSplit(const TWideToken& token, size_t pos);
-// Check if we can split wide-token after specified sub-token by dot delimiter.
-// The function verifies the following condition:
-// <word> <dot> <word with uppercased first character or number>
-// <number> <dot> <word with uppercased first character>
-bool CheckWideTokenDotSplit(const TWideToken& token, size_t pos);
-
-// Check if we can split wide-token after specified sub-token.
-// The function uses rich-tree specific heuristics
-bool CheckWideTokenReqSplit(const TTokenStructure& subtokens, size_t pos);
-
-inline size_t GetSubTokenOffset(const TWideToken& tok, size_t subToken) {
- Y_ASSERT(subToken < tok.SubTokens.size());
- return tok.SubTokens[subToken].Pos - tok.SubTokens[subToken].PrefixLen;
-}
-
-// Create a new wide-token with the specified inclusive [start, end] sub-token range
-TWideToken ExtractWideTokenRange(const TWideToken& tok, size_t start, size_t end);