path: root/util/charset/wide.h

#pragma once

#include "recode_result.h"
#include "unidata.h"
#include "utf8.h"
#include "wide_specific.h"

#include <util/generic/algorithm.h>
#include <util/generic/string.h>
#include <util/generic/yexception.h>
#include <util/memory/tempbuf.h>
#include <util/system/compiler.h>
#include <util/system/cpu_id.h>
#include <util/system/yassert.h>

#include <cstring>

#ifdef _sse2_
    #include <emmintrin.h>
#endif

template <class T>
class TTempArray;
using TCharTemp = TTempArray<wchar16>;

namespace NDetail {
    inline TString InStringMsg(const char* s, size_t len) {
        return (len <= 50) ? " in string " + TString(s, len).Quote() : TString();
    }

    template <bool isPointer>
    struct TSelector;

    template <>
    struct TSelector<false> {
        template <class T>
        static inline void WriteSymbol(wchar16 s, T& dest) noexcept {
            dest.push_back(s);
        }
    };

    template <>
    struct TSelector<true> {
        template <class T>
        static inline void WriteSymbol(wchar16 s, T& dest) noexcept {
            *(dest++) = s;
        }
    };

    inline wchar32 ReadSurrogatePair(const wchar16* chars) noexcept {
        const wchar32 SURROGATE_OFFSET = static_cast<wchar32>(0x10000 - (0xD800 << 10) - 0xDC00);
        wchar16 lead = chars[0];
        wchar16 tail = chars[1];

        Y_ASSERT(IsW16SurrogateLead(lead));
        Y_ASSERT(IsW16SurrogateTail(tail));

        return (static_cast<wchar32>(lead) << 10) + tail + SURROGATE_OFFSET;
    }

    template <class T>
    inline void WriteSurrogatePair(wchar32 s, T& dest) noexcept;

}

inline wchar16* SkipSymbol(wchar16* begin, const wchar16* end) noexcept {
    return begin + W16SymbolSize(begin, end);
}
inline const wchar16* SkipSymbol(const wchar16* begin, const wchar16* end) noexcept {
    return begin + W16SymbolSize(begin, end);
}
inline wchar32* SkipSymbol(wchar32* begin, const wchar32* end) noexcept {
    Y_ASSERT(begin < end);
    return begin + 1;
}
inline const wchar32* SkipSymbol(const wchar32* begin, const wchar32* end) noexcept {
    Y_ASSERT(begin < end);
    return begin + 1;
}

inline wchar32 ReadSymbol(const wchar16* begin, const wchar16* end) noexcept {
    Y_ASSERT(begin < end);
    if (IsW16SurrogateLead(*begin)) {
        if (begin + 1 < end && IsW16SurrogateTail(*(begin + 1)))
            return ::NDetail::ReadSurrogatePair(begin);

        return BROKEN_RUNE;
    } else if (IsW16SurrogateTail(*begin)) {
        return BROKEN_RUNE;
    }

    return *begin;
}

inline wchar32 ReadSymbol(const wchar32* begin, const wchar32* end) noexcept {
    Y_ASSERT(begin < end);
    return *begin;
}

//! presuming input data is either big enought of null terminated
inline wchar32 ReadSymbolAndAdvance(const wchar16*& begin) noexcept {
    Y_ASSERT(*begin);
    if (IsW16SurrogateLead(begin[0])) {
        if (IsW16SurrogateTail(begin[1])) {
            Y_ASSERT(begin[1] != 0);
            const wchar32 c = ::NDetail::ReadSurrogatePair(begin);
            begin += 2;
            return c;
        }
        ++begin;
        return BROKEN_RUNE;
    } else if (IsW16SurrogateTail(begin[0])) {
        ++begin;
        return BROKEN_RUNE;
    }
    return *(begin++);
}

//! presuming input data is either big enought of null terminated
inline wchar32 ReadSymbolAndAdvance(const wchar32*& begin) noexcept {
    Y_ASSERT(*begin);
    return *(begin++);
}

inline wchar32 ReadSymbolAndAdvance(const wchar16*& begin, const wchar16* end) noexcept {
    Y_ASSERT(begin < end);
    if (IsW16SurrogateLead(begin[0])) {
        if (begin + 1 != end && IsW16SurrogateTail(begin[1])) {
            const wchar32 c = ::NDetail::ReadSurrogatePair(begin);
            begin += 2;
            return c;
        }
        ++begin;
        return BROKEN_RUNE;
    } else if (IsW16SurrogateTail(begin[0])) {
        ++begin;
        return BROKEN_RUNE;
    }
    return *(begin++);
}

inline wchar32 ReadSymbolAndAdvance(const wchar32*& begin, const wchar32* end) noexcept {
    Y_ASSERT(begin < end);
    return *(begin++);
}

template <class T>
inline size_t WriteSymbol(wchar16 s, T& dest) noexcept {
    ::NDetail::TSelector<std::is_pointer<T>::value>::WriteSymbol(s, dest); 
    return 1;
}

template <class T>
inline size_t WriteSymbol(wchar32 s, T& dest) noexcept {
    if (s > 0xFFFF) {
        if (s >= ::NUnicode::UnicodeInstancesLimit()) {
            return WriteSymbol(static_cast<wchar16>(BROKEN_RUNE), dest);
        }

        ::NDetail::WriteSurrogatePair(s, dest);
        return 2;
    }

    return WriteSymbol(static_cast<wchar16>(s), dest);
}

inline bool WriteSymbol(wchar32 s, wchar16*& dest, const wchar16* destEnd) noexcept {
    Y_ASSERT(dest < destEnd);

    if (s > 0xFFFF) {
        if (s >= NUnicode::UnicodeInstancesLimit()) {
            *(dest++) = static_cast<wchar16>(BROKEN_RUNE);
            return true;
        }

        if (dest + 2 > destEnd)
            return false;

        ::NDetail::WriteSurrogatePair(s, dest);
    } else {
        *(dest++) = static_cast<wchar16>(s);
    }

    return true;
}

inline size_t WriteSymbol(wchar32 s, wchar32*& dest) noexcept {
    *(dest++) = s;
    return 1;
}

inline bool WriteSymbol(wchar32 s, wchar32*& dest, const wchar32* destEnd) noexcept {
    Y_ASSERT(dest < destEnd);

    *(dest++) = s;

    return true;
}

template <class T>
inline void ::NDetail::WriteSurrogatePair(wchar32 s, T& dest) noexcept {
    const wchar32 LEAD_OFFSET = 0xD800 - (0x10000 >> 10);
    Y_ASSERT(s > 0xFFFF && s < ::NUnicode::UnicodeInstancesLimit());

    wchar16 lead = LEAD_OFFSET + (static_cast<wchar16>(s >> 10));
    wchar16 tail = 0xDC00 + static_cast<wchar16>(s & 0x3FF);
    Y_ASSERT(IsW16SurrogateLead(lead));
    Y_ASSERT(IsW16SurrogateTail(tail));

    WriteSymbol(lead, dest);
    WriteSymbol(tail, dest);
}

class TCharIterator {
private:
    const wchar16* Begin;
    const wchar16* End;

public:
    inline explicit TCharIterator(const wchar16* end)
        : Begin(end)
        , End(end)
    {
    }

    inline TCharIterator(const wchar16* begin, const wchar16* end)
        : Begin(begin)
        , End(end)
    {
    }

    inline TCharIterator& operator++() {
        Begin = SkipSymbol(Begin, End);

        return *this;
    }

    inline bool operator==(const wchar16* other) const {
        return Begin == other;
    }
    inline bool operator!=(const wchar16* other) const {
        return !(*this == other);
    }

    inline bool operator==(const TCharIterator& other) const {
        return *this == other.Begin;
    }
    inline bool operator!=(const TCharIterator& other) const {
        return *this != other.Begin;
    }

    inline wchar32 operator*() const {
        return ReadSymbol(Begin, End);
    }

    inline const wchar16* Get() const {
        return Begin;
    }
};

namespace NDetail {
    template <bool robust, typename TCharType>
    inline void UTF8ToWideImplScalar(const unsigned char*& cur, const unsigned char* last, TCharType*& dest) noexcept {
        wchar32 rune = BROKEN_RUNE;

        while (cur != last) {
            if (ReadUTF8CharAndAdvance(rune, cur, last) != RECODE_OK) {
                if (robust) {
                    rune = BROKEN_RUNE;
                    ++cur;
                } else {
                    break;
                }
            }

            Y_ASSERT(cur <= last);
            WriteSymbol(rune, dest);
        }
    }

    template <typename TCharType>
    inline void UTF16ToUTF32ImplScalar(const wchar16* cur, const wchar16* last, TCharType*& dest) noexcept {
        wchar32 rune = BROKEN_RUNE;

        while (cur != last) {
            rune = ReadSymbolAndAdvance(cur, last);
            Y_ASSERT(cur <= last);
            WriteSymbol(rune, dest);
        }
    }

    template <class TCharType>
    inline void UTF8ToWideImplSSE41(const unsigned char*& /*cur*/, const unsigned char* /*last*/, TCharType*& /*dest*/) noexcept {
    }

    void UTF8ToWideImplSSE41(const unsigned char*& cur, const unsigned char* last, wchar16*& dest) noexcept;

    void UTF8ToWideImplSSE41(const unsigned char*& cur, const unsigned char* last, wchar32*& dest) noexcept;
}

//! @return len if robust and position where encoding stopped if not
template <bool robust, typename TCharType>
inline size_t UTF8ToWideImpl(const char* text, size_t len, TCharType* dest, size_t& written) noexcept {
    const unsigned char* cur = reinterpret_cast<const unsigned char*>(text);
    const unsigned char* last = cur + len;
    TCharType* p = dest;
#ifdef _sse_ //can't check for sse4, as we build most of arcadia without sse4 support even on platforms that support it
    if (cur + 16 <= last && NX86::CachedHaveSSE41()) {
        ::NDetail::UTF8ToWideImplSSE41(cur, last, p);
    }
#endif

    ::NDetail::UTF8ToWideImplScalar<robust>(cur, last, p);
    written = p - dest;
    return cur - reinterpret_cast<const unsigned char*>(text);
}

template <typename TCharType>
inline size_t UTF8ToWideImpl(const char* text, size_t len, TCharType* dest, size_t& written) {
    return UTF8ToWideImpl<false>(text, len, dest, written);
}

template <bool robust>
inline TUtf16String UTF8ToWide(const char* text, size_t len) {
    TUtf16String w = TUtf16String::Uninitialized(len);
    size_t written;
    size_t pos = UTF8ToWideImpl<robust>(text, len, w.begin(), written);
    if (pos != len)
        ythrow yexception() << "failed to decode UTF-8 string at pos " << pos << ::NDetail::InStringMsg(text, len);
    Y_ASSERT(w.size() >= written);
    w.remove(written);
    return w;
}

template <bool robust, typename TCharType>
inline bool UTF8ToWide(const char* text, size_t len, TCharType* dest, size_t& written) noexcept {
    return UTF8ToWideImpl<robust>(text, len, dest, written) == len;
}

//! converts text from UTF8 to unicode, stops immediately it UTF8 byte sequence is wrong
//! @attention destination buffer must be long enough to fit all characters of the text,
//!            conversion stops if a broken symbol is met
//! @return @c true if all the text converted successfully, @c false - a broken symbol was found
template <typename TCharType>
inline bool UTF8ToWide(const char* text, size_t len, TCharType* dest, size_t& written) noexcept {
    return UTF8ToWide<false>(text, len, dest, written);
}

template <bool robust>
inline TWtringBuf UTF8ToWide(const TStringBuf src, TUtf16String& dst) {
    dst.ReserveAndResize(src.size());
    size_t written = 0;
    UTF8ToWideImpl<robust>(src.data(), src.size(), dst.begin(), written);
    dst.resize(written);
    return dst;
}

//! if not robust will stop at first error position
template <bool robust>
inline TUtf32StringBuf UTF8ToUTF32(const TStringBuf src, TUtf32String& dst) {
    dst.ReserveAndResize(src.size());
    size_t written = 0;
    UTF8ToWideImpl<robust>(src.data(), src.size(), dst.begin(), written);
    dst.resize(written);
    return dst;
}

inline TWtringBuf UTF8ToWide(const TStringBuf src, TUtf16String& dst) {
    return UTF8ToWide<false>(src, dst);
}

inline TUtf16String UTF8ToWide(const char* text, size_t len) {
    return UTF8ToWide<false>(text, len);
}

template <bool robust>
inline TUtf16String UTF8ToWide(const TStringBuf s) {
    return UTF8ToWide<robust>(s.data(), s.size());
}

template <bool robust>
inline TUtf32String UTF8ToUTF32(const TStringBuf s) {
    TUtf32String r;
    UTF8ToUTF32<robust>(s, r);
    return r;
}

inline TUtf16String UTF8ToWide(const TStringBuf s) {
    return UTF8ToWide<false>(s.data(), s.size());
}

//! converts text from unicode to UTF8
//! @attention destination buffer must be long enough to fit all characters of the text,
//!            @c WriteUTF8Char converts @c wchar32 into maximum 4 bytes of UTF8 so
//!            destination buffer must have length equal to <tt> len * 4 </tt>
template <typename TCharType>
inline void WideToUTF8(const TCharType* text, size_t len, char* dest, size_t& written) {
    const TCharType* const last = text + len;
    unsigned char* p = reinterpret_cast<unsigned char*>(dest);
    size_t runeLen;
    for (const TCharType* cur = text; cur != last;) {
        WriteUTF8Char(ReadSymbolAndAdvance(cur, last), runeLen, p);
        Y_ASSERT(runeLen <= 4);
        p += runeLen;
    }
    written = p - reinterpret_cast<unsigned char*>(dest);
}

constexpr size_t WideToUTF8BufferSize(const size_t inputStringSize) noexcept {
    return inputStringSize * 4; // * 4 because the conversion functions can convert unicode character into maximum 4 bytes of UTF8
}

inline TStringBuf WideToUTF8(const TWtringBuf src, TString& dst) {
    dst.ReserveAndResize(WideToUTF8BufferSize(src.size()));
    size_t written = 0;
    WideToUTF8(src.data(), src.size(), dst.begin(), written);
    Y_ASSERT(dst.size() >= written);
    dst.remove(written);
    return dst;
}

inline TString WideToUTF8(const wchar16* text, size_t len) {
    TString s = TString::Uninitialized(WideToUTF8BufferSize(len));
    size_t written = 0;
    WideToUTF8(text, len, s.begin(), written);
    Y_ASSERT(s.size() >= written);
    s.remove(written);
    return s;
}

inline TString WideToUTF8(const wchar32* text, size_t len) {
    TString s = TString::Uninitialized(WideToUTF8BufferSize(len));
    size_t written = 0;
    WideToUTF8(text, len, s.begin(), written);
    Y_ASSERT(s.size() >= written);
    s.remove(written);
    return s;
}

inline TString WideToUTF8(const TWtringBuf w) {
    return WideToUTF8(w.data(), w.size());
}

inline TString WideToUTF8(const TUtf32StringBuf w) {
    return WideToUTF8(w.data(), w.size());
}

inline TUtf16String UTF32ToWide(const wchar32* begin, size_t len) {
    TUtf16String res;
    res.reserve(len);

    const wchar32* end = begin + len;
    for (const wchar32* i = begin; i != end; ++i) {
        WriteSymbol(*i, res);
    }

    return res;
}

// adopted from https://chromium.googlesource.com/chromium/src/+/master/base/strings/string_util.cc
// Assuming that a pointer is the size of a "machine word", then
// uintptr_t is an integer type that is also a machine word.

namespace NDetail {
    using TMachineWord = uintptr_t;
    const uintptr_t kMachineWordAlignmentMask = sizeof(TMachineWord) - 1;

    inline bool IsAlignedToMachineWord(const void* pointer) {
        return !(reinterpret_cast<TMachineWord>(pointer) & kMachineWordAlignmentMask);
    }

    template <typename T>
    inline T* AlignToMachineWord(T* pointer) {
        return reinterpret_cast<T*>(reinterpret_cast<TMachineWord>(pointer) & ~kMachineWordAlignmentMask);
    }

    template <size_t size, typename CharacterType>
    struct NonASCIIMask;

    template <>
    struct
        NonASCIIMask<4, wchar16> {
        static constexpr ui32 Value() {
            return 0xFF80FF80U;
        }
    };

    template <>
    struct
        NonASCIIMask<4, char> {
        static constexpr ui32 Value() {
            return 0x80808080U;
        }
    };

    template <>
    struct
        NonASCIIMask<8, wchar16> {
        static constexpr ui64 Value() {
            return 0xFF80FF80FF80FF80ULL;
        }
    };

    template <>
    struct
        NonASCIIMask<8, char> {
        static constexpr ui64 Value() {
            return 0x8080808080808080ULL;
        }
    };

    template <typename TChar>
    inline bool DoIsStringASCIISlow(const TChar* first, const TChar* last) {
        using TUnsignedChar = std::make_unsigned_t<TChar>;
        Y_ASSERT(first <= last);
        for (; first != last; ++first) {
            if (static_cast<TUnsignedChar>(*first) > 0x7F) {
                return false;
            }
        }
        return true;
    }

    template <typename TChar>
    inline bool DoIsStringASCII(const TChar* first, const TChar* last) {
        if (last - first < 10) {
            return DoIsStringASCIISlow(first, last);
        }
        TMachineWord allCharBits = 0;
        TMachineWord nonAsciiBitMask = NonASCIIMask<sizeof(TMachineWord), TChar>::Value();

        // Prologue: align the input.
        while (!IsAlignedToMachineWord(first) && first != last) {
            allCharBits |= *first;
            ++first;
        }

        // Compare the values of CPU word size.
        const TChar* word_end = AlignToMachineWord(last);
        const size_t loopIncrement = sizeof(TMachineWord) / sizeof(TChar);
        while (first < word_end) {
            allCharBits |= *(reinterpret_cast<const TMachineWord*>(first));
            first += loopIncrement;

            // fast exit
            if (allCharBits & nonAsciiBitMask) {
                return false;
            }
        }

        // Process the remaining bytes.
        while (first != last) {
            allCharBits |= *first;
            ++first;
        }

        return !(allCharBits & nonAsciiBitMask);
    }

#ifdef _sse2_
    inline bool DoIsStringASCIISSE(const unsigned char* first, const unsigned char* last) {
        //scalar version for short strings
        if (first + 8 > last) {
            return ::NDetail::DoIsStringASCIISlow(first, last);
        }

        alignas(16) unsigned char buf[16];

        while (first + 16 <= last) {
            memcpy(buf, first, 16);
            __m128i chunk = _mm_load_si128(reinterpret_cast<__m128i*>(buf));

            int asciiMask = _mm_movemask_epi8(chunk);
            if (asciiMask) {
                return false;
            }
            first += 16;
        }

        if (first + 8 <= last) {
            memcpy(buf, first, 8);
            __m128i chunk = _mm_loadl_epi64(reinterpret_cast<__m128i*>(buf));

            int asciiMask = _mm_movemask_epi8(chunk);
            if (asciiMask) {
                return false;
            }
            first += 8;
        }

        return ::NDetail::DoIsStringASCIISlow(first, last);
    }
#endif //_sse2_

}

//! returns @c true if character sequence has no symbols with value greater than 0x7F
template <typename TChar>
inline bool IsStringASCII(const TChar* first, const TChar* last) {
    return ::NDetail::DoIsStringASCII(first, last);
}

#ifdef _sse2_
template <>
inline bool IsStringASCII<unsigned char>(const unsigned char* first, const unsigned char* last) {
    return ::NDetail::DoIsStringASCIISSE(first, last);
}
template <>
inline bool IsStringASCII<char>(const char* first, const char* last) {
    return ::NDetail::DoIsStringASCIISSE(reinterpret_cast<const unsigned char*>(first), reinterpret_cast<const unsigned char*>(last));
}
#endif

//! copies elements from one character sequence to another using memcpy
//! for compatibility only
template <typename TChar>
inline void Copy(const TChar* first, size_t len, TChar* result) {
    memcpy(result, first, len * sizeof(TChar));
}

template <typename TChar1, typename TChar2>
inline void Copy(const TChar1* first, size_t len, TChar2* result) {
    Copy(first, first + len, result);
}

//! copies symbols from one character sequence to another without any conversion
//! @note this function can be used instead of the template constructor of @c std::basic_string:
//!       template <typename InputIterator>
//!       basic_string(InputIterator begin, InputIterator end, const Allocator& a = Allocator());
//!       and the family of template member functions: append, assign, insert, replace.
template <typename TStringType, typename TChar>
inline TStringType CopyTo(const TChar* first, const TChar* last) {
    Y_ASSERT(first <= last);
    TStringType str = TStringType::Uninitialized(last - first);
    Copy(first, last, str.begin());
    return str;
}

template <typename TStringType, typename TChar>
inline TStringType CopyTo(const TChar* s, size_t n) {
    TStringType str = TStringType::Uninitialized(n);
    Copy(s, n, str.begin());
    return str;
}

inline TString WideToASCII(const TWtringBuf w) {
    Y_ASSERT(IsStringASCII(w.begin(), w.end()));
    return CopyTo<TString>(w.begin(), w.end());
}

inline TUtf16String ASCIIToWide(const TStringBuf s) {
    Y_ASSERT(IsStringASCII(s.begin(), s.end()));
    return CopyTo<TUtf16String>(s.begin(), s.end());
}

inline TUtf32String ASCIIToUTF32(const TStringBuf s) {
    Y_ASSERT(IsStringASCII(s.begin(), s.end()));
    return CopyTo<TUtf32String>(s.begin(), s.end());
}

//! returns @c true if string contains whitespace characters only
inline bool IsSpace(const wchar16* s, size_t n) {
    if (n == 0)
        return false;

    Y_ASSERT(s);

    const wchar16* const e = s + n;
    for (const wchar16* p = s; p != e; ++p) {
        if (!IsWhitespace(*p))
            return false;
    }
    return true;
}

//! returns @c true if string contains whitespace characters only
inline bool IsSpace(const TWtringBuf s) {
    return IsSpace(s.data(), s.length());
}

//! replaces multiple sequential whitespace characters with a single space character
void Collapse(TUtf16String& w);

//! @return new length
size_t Collapse(wchar16* s, size_t n);

//! Removes leading whitespace characters
TWtringBuf StripLeft(const TWtringBuf text) noexcept Y_WARN_UNUSED_RESULT;
void StripLeft(TUtf16String& text);

//! Removes trailing whitespace characters
TWtringBuf StripRight(const TWtringBuf text) noexcept Y_WARN_UNUSED_RESULT;
void StripRight(TUtf16String& text);

//! Removes leading and trailing whitespace characters
TWtringBuf Strip(const TWtringBuf text) noexcept Y_WARN_UNUSED_RESULT;
void Strip(TUtf16String& text);

/* Check if given word is lowercase/uppercase. Will return false if string contains any
 * non-alphabetical symbols. It is expected that `text` is a correct UTF-16 string.
 *
 * For example `IsLowerWord("hello")` will return `true`, when `IsLowerWord("hello there")` will
 * return false because of the space in the middle of the string. Empty string is also considered
 * lowercase.
 */
bool IsLowerWord(const TWtringBuf text) noexcept;
bool IsUpperWord(const TWtringBuf text) noexcept;

/* Will check if given word starts with capital letter and the rest of the word is lowercase. Will
 * return `false` for empty string. See also `IsLowerWord`.
 */
bool IsTitleWord(const TWtringBuf text) noexcept;

/* Check if given string is lowercase/uppercase. Will return `true` if all alphabetic symbols are
 * in proper case, all other symbols are ignored. It is expected that `text` is a correct UTF-16
 * string.
 *
 * For example `IsLowerWord("hello")` will return `true` and `IsLowerWord("hello there")` will
 * also return true because. Empty string is also considered lowercase.
 *
 * NOTE: for any case where `IsLowerWord` returns `true` `IsLower` will also return `true`.
 */
bool IsLower(const TWtringBuf text) noexcept;
bool IsUpper(const TWtringBuf text) noexcept;

/* Lowercase/uppercase given string inplace. Any alphabetic symbol will be converted to a proper
 * case, the rest of the symbols will be kept the same. It is expected that `text` is a correct
 * UTF-16 string.
 *
 * For example `ToLower("heLLo")` will return `"hello"`.
 *
 * @param text      String to modify
 * @param pos       Position of the first character to modify
 * @param count     Length of the substring
 * @returns         `true` if `text` was changed
 *
 * NOTE: `pos` and `count` are measured in `wchar16`, not in codepoints.
 */
bool ToLower(TUtf16String& text, size_t pos = 0, size_t count = TUtf16String::npos);
bool ToUpper(TUtf16String& text, size_t pos = 0, size_t count = TUtf16String::npos);

/* Lowercase/uppercase given string inplace. Any alphabetic symbol will be converted to a proper
* case, the rest of the symbols will be kept the same. It is expected that `text` is a correct
* UTF-32 string.
*
* For example `ToLower("heLLo")` will return `"hello"`.
*
* @param text      String to modify
* @param pos       Position of the first character to modify
* @param count     Length of the substring
* @returns         `true` if `text` was changed
*
* NOTE: `pos` and `count` are measured in `wchar16`, not in codepoints.
*/
bool ToLower(TUtf32String& /*text*/, size_t /*pos*/ = 0, size_t /*count*/ = TUtf16String::npos);
bool ToUpper(TUtf32String& /*text*/, size_t /*pos*/ = 0, size_t /*count*/ = TUtf16String::npos);

/* Titlecase first symbol and lowercase the rest, see `ToLower` for more details.
 */
bool ToTitle(TUtf16String& text, size_t pos = 0, size_t count = TUtf16String::npos);

/* Titlecase first symbol and lowercase the rest, see `ToLower` for more details.
*/
bool ToTitle(TUtf32String& /*text*/, size_t /*pos*/ = 0, size_t /*count*/ = TUtf16String::npos);

/* @param text      Pointer to the string to modify
 * @param length    Length of the string to modify
 * @param out       Pointer to the character array to write to
 *
 * NOTE: [text, text+length) and [out, out+length) should not interleave.
 *
 * TODO(yazevnul): replace these functions with `bool(const TWtringBuf, const TArrayRef<wchar16>)`
 * overload.
 */
bool ToLower(const wchar16* text, size_t length, wchar16* out) noexcept;
bool ToUpper(const wchar16* text, size_t length, wchar16* out) noexcept;
bool ToTitle(const wchar16* text, size_t length, wchar16* out) noexcept;

bool ToLower(const wchar32* text, size_t length, wchar32* out) noexcept;
bool ToUpper(const wchar32* text, size_t length, wchar32* out) noexcept;
bool ToTitle(const wchar32* text, size_t length, wchar32* out) noexcept;

/* @param text      Pointer to the string to modify
 * @param length    Length of the string to modify
 *
 * TODO(yazevnul): replace these functions with `bool(const TArrayRef<wchar16>)` overload.
 */
bool ToLower(wchar16* text, size_t length) noexcept;
bool ToUpper(wchar16* text, size_t length) noexcept;
bool ToTitle(wchar16* text, size_t length) noexcept;

bool ToLower(wchar32* text, size_t length) noexcept;
bool ToUpper(wchar32* text, size_t length) noexcept;
bool ToTitle(wchar32* text, size_t length) noexcept;

/* Convenience wrappers for `ToLower`, `ToUpper` and `ToTitle`.
 */
TUtf16String ToLowerRet(TUtf16String text, size_t pos = 0, size_t count = TUtf16String::npos) Y_WARN_UNUSED_RESULT;
TUtf16String ToUpperRet(TUtf16String text, size_t pos = 0, size_t count = TUtf16String::npos) Y_WARN_UNUSED_RESULT;
TUtf16String ToTitleRet(TUtf16String text, size_t pos = 0, size_t count = TUtf16String::npos) Y_WARN_UNUSED_RESULT;

TUtf16String ToLowerRet(const TWtringBuf text, size_t pos = 0, size_t count = TWtringBuf::npos) Y_WARN_UNUSED_RESULT;
TUtf16String ToUpperRet(const TWtringBuf text, size_t pos = 0, size_t count = TWtringBuf::npos) Y_WARN_UNUSED_RESULT;
TUtf16String ToTitleRet(const TWtringBuf text, size_t pos = 0, size_t count = TWtringBuf::npos) Y_WARN_UNUSED_RESULT;

TUtf32String ToLowerRet(const TUtf32StringBuf text, size_t pos = 0, size_t count = TWtringBuf::npos) Y_WARN_UNUSED_RESULT;
TUtf32String ToUpperRet(const TUtf32StringBuf text, size_t pos = 0, size_t count = TWtringBuf::npos) Y_WARN_UNUSED_RESULT;
TUtf32String ToTitleRet(const TUtf32StringBuf text, size_t pos = 0, size_t count = TWtringBuf::npos) Y_WARN_UNUSED_RESULT;

//! replaces the '<', '>' and '&' characters in string with '&lt;', '&gt;' and '&amp;' respectively
// insertBr=true - replace '\r' and '\n' with "<BR>"
template <bool insertBr>
void EscapeHtmlChars(TUtf16String& str);

//! returns number of characters in range. Handle surrogate pairs as one character.
inline size_t CountWideChars(const wchar16* b, const wchar16* e) {
    size_t count = 0;
    Y_ENSURE(b <= e, TStringBuf("invalid iterators"));
    while (b < e) {
        b = SkipSymbol(b, e);
        ++count;
    }
    return count;
}

inline size_t CountWideChars(const TWtringBuf str) {
    return CountWideChars(str.begin(), str.end());
}

//! checks whether the range is valid UTF-16 sequence
inline bool IsValidUTF16(const wchar16* b, const wchar16* e) {
    Y_ENSURE(b <= e, TStringBuf("invalid iterators"));
    while (b < e) {
        wchar32 symbol = ReadSymbolAndAdvance(b, e);
        if (symbol == BROKEN_RUNE)
            return false;
    }
    return true;
}

inline bool IsValidUTF16(const TWtringBuf str) {
    return IsValidUTF16(str.begin(), str.end());
}