#include "wide.h"
#include "codepage.h"
#include "recyr.hh"
#include <library/cpp/testing/unittest/registar.h>
#include <util/charset/utf8.h>
#include <util/digest/numeric.h>
#include <util/generic/hash_set.h>
#include <algorithm>
namespace {
//! three UTF8 encoded russian letters (A, B, V)
const char yandexCyrillicAlphabet[] =
"\xC0\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA\xCB\xCC\xCD\xCE\xCF" // A - P
"\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF" // R - YA
"\xE0\xE1\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF" // a - p
"\xF0\xF1\xF2\xF3\xF4\xF5\xF6\xF7\xF8\xF9\xFA\xFB\xFC\xFD\xFE\xFF"; // r - ya
const wchar16 wideCyrillicAlphabet[] = {
0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F,
0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F,
0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F,
0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, 0x00};
const char utf8CyrillicAlphabet[] =
"\xd0\x90\xd0\x91\xd0\x92\xd0\x93\xd0\x94\xd0\x95\xd0\x96\xd0\x97"
"\xd0\x98\xd0\x99\xd0\x9a\xd0\x9b\xd0\x9c\xd0\x9d\xd0\x9e\xd0\x9f"
"\xd0\xa0\xd0\xa1\xd0\xa2\xd0\xa3\xd0\xa4\xd0\xa5\xd0\xa6\xd0\xa7"
"\xd0\xa8\xd0\xa9\xd0\xaa\xd0\xab\xd0\xac\xd0\xad\xd0\xae\xd0\xaf"
"\xd0\xb0\xd0\xb1\xd0\xb2\xd0\xb3\xd0\xb4\xd0\xb5\xd0\xb6\xd0\xb7"
"\xd0\xb8\xd0\xb9\xd0\xba\xd0\xbb\xd0\xbc\xd0\xbd\xd0\xbe\xd0\xbf"
"\xd1\x80\xd1\x81\xd1\x82\xd1\x83\xd1\x84\xd1\x85\xd1\x86\xd1\x87"
"\xd1\x88\xd1\x89\xd1\x8a\xd1\x8b\xd1\x8c\xd1\x8d\xd1\x8e\xd1\x8f";
TString CreateYandexText() {
const int len = 256;
char text[len] = {0};
for (int i = 0; i < len; ++i) {
text[i] = static_cast<char>(i);
}
return TString(text, len);
}
TUtf16String CreateUnicodeText() {
const int len = 256;
wchar16 text[len] = {
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, // 0x00 - 0x0F
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, // 0x10 - 0x1F
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, // 0x20 - 0x2F
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, // 0x30 - 0x3F
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, // 0x40 - 0x4F
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, // 0x50 - 0x5F
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, // 0x60 - 0x6F
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, // 0x70 - 0x7F
0x0301, 0x00C4, 0x00D6, 0x00DC, 0x0104, 0x0106, 0x0118, 0x0141, 0x00E0, 0x00E2, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x0490, 0x00AD, // 0x80 - 0x8F
0x00DF, 0x00E4, 0x00F6, 0x00FC, 0x0105, 0x0107, 0x0119, 0x0142, 0x00EB, 0x00EE, 0x00EF, 0x00F4, 0x00F9, 0x00FB, 0x0491, 0x92CF, // 0x90 - 0x9F
0x00A0, 0x0143, 0x00D3, 0x015A, 0x017B, 0x0179, 0x046C, 0x00A7, 0x0401, 0x0462, 0x0472, 0x0474, 0x040E, 0x0406, 0x0404, 0x0407, // 0xA0 - 0xAF
0x00B0, 0x0144, 0x00F3, 0x015B, 0x017C, 0x017A, 0x046D, 0x2116, 0x0451, 0x0463, 0x0473, 0x0475, 0x045E, 0x0456, 0x0454, 0x0457 // 0xB0 - 0xBF
};
for (int i = 0; i < len; ++i) {
if (i <= 0x7F) { // ASCII characters without 0x7 and 0x1B
text[i] = static_cast<wchar16>(i);
} else if (i >= 0xC0 && i <= 0xFF) { // russian characters (without YO and yo)
text[i] = static_cast<wchar16>(i + 0x0350); // 0x0410 - 0x044F
}
}
return TUtf16String(text, len);
}
TString CreateUTF8Text() {
char text[] = {
'\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', '\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', '\x0f',
'\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', '\x18', '\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f',
'\x20', '\x21', '\x22', '\x23', '\x24', '\x25', '\x26', '\x27', '\x28', '\x29', '\x2a', '\x2b', '\x2c', '\x2d', '\x2e', '\x2f',
'\x30', '\x31', '\x32', '\x33', '\x34', '\x35', '\x36', '\x37', '\x38', '\x39', '\x3a', '\x3b', '\x3c', '\x3d', '\x3e', '\x3f',
'\x40', '\x41', '\x42', '\x43', '\x44', '\x45', '\x46', '\x47', '\x48', '\x49', '\x4a', '\x4b', '\x4c', '\x4d', '\x4e', '\x4f',
'\x50', '\x51', '\x52', '\x53', '\x54', '\x55', '\x56', '\x57', '\x58', '\x59', '\x5a', '\x5b', '\x5c', '\x5d', '\x5e', '\x5f',
'\x60', '\x61', '\x62', '\x63', '\x64', '\x65', '\x66', '\x67', '\x68', '\x69', '\x6a', '\x6b', '\x6c', '\x6d', '\x6e', '\x6f',
'\x70', '\x71', '\x72', '\x73', '\x74', '\x75', '\x76', '\x77', '\x78', '\x79', '\x7a', '\x7b', '\x7c', '\x7d', '\x7e', '\x7f',
'\xcc', '\x81', '\xc3', '\x84', '\xc3', '\x96', '\xc3', '\x9c', '\xc4', '\x84', '\xc4', '\x86', '\xc4', '\x98', '\xc5', '\x81',
'\xc3', '\xa0', '\xc3', '\xa2', '\xc3', '\xa7', '\xc3', '\xa8', '\xc3', '\xa9', '\xc3', '\xaa', '\xd2', '\x90', '\xc2', '\xad',
'\xc3', '\x9f', '\xc3', '\xa4', '\xc3', '\xb6', '\xc3', '\xbc', '\xc4', '\x85', '\xc4', '\x87', '\xc4', '\x99', '\xc5', '\x82',
'\xc3', '\xab', '\xc3', '\xae', '\xc3', '\xaf', '\xc3', '\xb4', '\xc3', '\xb9', '\xc3', '\xbb', '\xd2', '\x91', '\xe9', '\x8b',
'\x8f', '\xc2', '\xa0', '\xc5', '\x83', '\xc3', '\x93', '\xc5', '\x9a', '\xc5', '\xbb', '\xc5', '\xb9', '\xd1', '\xac', '\xc2',
'\xa7', '\xd0', '\x81', '\xd1', '\xa2', '\xd1', '\xb2', '\xd1', '\xb4', '\xd0', '\x8e', '\xd0', '\x86', '\xd0', '\x84', '\xd0',
'\x87', '\xc2', '\xb0', '\xc5', '\x84', '\xc3', '\xb3', '\xc5', '\x9b', '\xc5', '\xbc', '\xc5', '\xba', '\xd1', '\xad', '\xe2',
'\x84', '\x96', '\xd1', '\x91', '\xd1', '\xa3', '\xd1', '\xb3', '\xd1', '\xb5', '\xd1', '\x9e', '\xd1', '\x96', '\xd1', '\x94',
'\xd1', '\x97', '\xd0', '\x90', '\xd0', '\x91', '\xd0', '\x92', '\xd0', '\x93', '\xd0', '\x94', '\xd0', '\x95', '\xd0', '\x96',
'\xd0', '\x97', '\xd0', '\x98', '\xd0', '\x99', '\xd0', '\x9a', '\xd0', '\x9b', '\xd0', '\x9c', '\xd0', '\x9d', '\xd0', '\x9e',
'\xd0', '\x9f', '\xd0', '\xa0', '\xd0', '\xa1', '\xd0', '\xa2', '\xd0', '\xa3', '\xd0', '\xa4', '\xd0', '\xa5', '\xd0', '\xa6',
'\xd0', '\xa7', '\xd0', '\xa8', '\xd0', '\xa9', '\xd0', '\xaa', '\xd0', '\xab', '\xd0', '\xac', '\xd0', '\xad', '\xd0', '\xae',
'\xd0', '\xaf', '\xd0', '\xb0', '\xd0', '\xb1', '\xd0', '\xb2', '\xd0', '\xb3', '\xd0', '\xb4', '\xd0', '\xb5', '\xd0', '\xb6',
'\xd0', '\xb7', '\xd0', '\xb8', '\xd0', '\xb9', '\xd0', '\xba', '\xd0', '\xbb', '\xd0', '\xbc', '\xd0', '\xbd', '\xd0', '\xbe',
'\xd0', '\xbf', '\xd1', '\x80', '\xd1', '\x81', '\xd1', '\x82', '\xd1', '\x83', '\xd1', '\x84', '\xd1', '\x85', '\xd1', '\x86',
'\xd1', '\x87', '\xd1', '\x88', '\xd1', '\x89', '\xd1', '\x8a', '\xd1', '\x8b', '\xd1', '\x8c', '\xd1', '\x8d', '\xd1', '\x8e',
'\xd1', '\x8f'};
return TString(text, Y_ARRAY_SIZE(text));
}
//! use this function to dump UTF8 text into a file in case of any changes
// void DumpUTF8Text() {
// TString s = WideToUTF8(UnicodeText);
// std::ofstream f("utf8.txt");
// f << std::hex;
// for (int i = 0; i < (int)s.size(); ++i) {
// f << "0x" << std::setw(2) << std::setfill('0') << (int)(ui8)s[i] << ", ";
// if ((i + 1) % 16 == 0)
// f << std::endl;
// }
// }
}
//! this unit tests ensure validity of Yandex-Unicode and UTF8-Unicode conversions
//! @note only those conversions are verified because they are used in index
class TConversionTest: public TTestBase {
private:
//! @note every of the text can have zeros in the middle
const TString YandexText;
const TUtf16String UnicodeText;
const TString UTF8Text;
private:
UNIT_TEST_SUITE(TConversionTest);
UNIT_TEST(TestCharToWide);
UNIT_TEST(TestWideToChar);
UNIT_TEST(TestYandexEncoding);
UNIT_TEST(TestRecodeIntoString);
UNIT_TEST(TestRecodeAppend);
UNIT_TEST(TestRecode);
UNIT_TEST(TestUnicodeLimit);
UNIT_TEST_SUITE_END();
public:
TConversionTest()
: YandexText(CreateYandexText())
, UnicodeText(CreateUnicodeText())
, UTF8Text(CreateUTF8Text())
{
}
void TestCharToWide();
void TestWideToChar();
void TestYandexEncoding();
void TestRecodeIntoString();
void TestRecodeAppend();
void TestRecode();
void TestUnicodeLimit();
};
UNIT_TEST_SUITE_REGISTRATION(TConversionTest);
// test conversions (char -> wchar32), (wchar32 -> char) and (wchar32 -> wchar16)
#define TEST_WCHAR32(sbuf, wbuf, enc) \
do { \
/* convert char to wchar32 */ \
TTempBuf tmpbuf1(sbuf.length() * sizeof(wchar32)); \
const TBasicStringBuf<wchar32> s4buf = NDetail::NBaseOps::Recode<char>(sbuf, reinterpret_cast<wchar32*>(tmpbuf1.Data()), enc); \
\
/* convert wchar32 to char */ \
TTempBuf tmpbuf2(s4buf.length() * 4); \
const TStringBuf s1buf = NDetail::NBaseOps::Recode(s4buf, tmpbuf2.Data(), enc); \
\
/* convert wchar32 to wchar16 */ \
const TUtf16String wstr2 = UTF32ToWide(s4buf.data(), s4buf.length()); \
\
/* test conversions */ \
UNIT_ASSERT_VALUES_EQUAL(sbuf, s1buf); \
UNIT_ASSERT_VALUES_EQUAL(wbuf, wstr2); \
} while (false)
void TConversionTest::TestCharToWide() {
TUtf16String w = CharToWide(YandexText, CODES_YANDEX);
UNIT_ASSERT(w.size() == 256);
UNIT_ASSERT(w.size() == UnicodeText.size());
for (int i = 0; i < 256; ++i) {
UNIT_ASSERT_VALUES_EQUAL(w[i], UnicodeText[i]);
}
}
void TConversionTest::TestWideToChar() {
TString s = WideToChar(UnicodeText, CODES_YANDEX);
UNIT_ASSERT(s.size() == 256);
UNIT_ASSERT(s.size() == YandexText.size());
for (int i = 0; i < 256; ++i) {
UNIT_ASSERT_VALUES_EQUAL(s[i], YandexText[i]);
}
}
static void TestSurrogates(const char* str, const wchar16* wide, size_t wideSize, ECharset enc) {
TUtf16String w = UTF8ToWide(str);
UNIT_ASSERT(w.size() == wideSize);
UNIT_ASSERT(!memcmp(w.c_str(), wide, wideSize));
TString s = WideToChar(w, enc);
UNIT_ASSERT(s == str);
}
void TConversionTest::TestYandexEncoding() {
TUtf16String w = UTF8ToWide(utf8CyrillicAlphabet, strlen(utf8CyrillicAlphabet), csYandex);
UNIT_ASSERT(w == wideCyrillicAlphabet);
w = UTF8ToWide(yandexCyrillicAlphabet, strlen(yandexCyrillicAlphabet), csYandex);
UNIT_ASSERT(w == wideCyrillicAlphabet);
const char* utf8NonBMP2 = "ab\xf4\x80\x89\x87n";
wchar16 wNonBMPDummy2[] = {'a', 'b', 0xDBC0, 0xDE47, 'n'};
TestSurrogates(utf8NonBMP2, wNonBMPDummy2, Y_ARRAY_SIZE(wNonBMPDummy2), CODES_UTF8);
{
const char* yandexNonBMP2 = "ab?n";
UNIT_ASSERT(yandexNonBMP2 == WideToChar(wNonBMPDummy2, Y_ARRAY_SIZE(wNonBMPDummy2), CODES_YANDEX));
TString temp;
temp.resize(Y_ARRAY_SIZE(wNonBMPDummy2));
size_t read = 0;
size_t written = 0;
RecodeFromUnicode(CODES_YANDEX, wNonBMPDummy2, temp.begin(), Y_ARRAY_SIZE(wNonBMPDummy2), temp.size(), read, written);
temp.remove(written);
UNIT_ASSERT(yandexNonBMP2 == temp);
}
}
void TConversionTest::TestRecodeIntoString() {
TString sYandex(UnicodeText.size() * 4, 'x');
const char* sdata = sYandex.data();
TStringBuf sres = NDetail::Recode<wchar16>(UnicodeText, sYandex, CODES_YANDEX);
UNIT_ASSERT(sYandex == YandexText); // same content
UNIT_ASSERT(sYandex.data() == sdata); // reserved buffer reused
UNIT_ASSERT(sYandex.data() == sres.data()); // same buffer
UNIT_ASSERT(sYandex.size() == sres.size()); // same size
TEST_WCHAR32(sYandex, UnicodeText, CODES_YANDEX);
TUtf16String sUnicode;
sUnicode.reserve(YandexText.size() * 4);
const wchar16* wdata = sUnicode.data();
TWtringBuf wres = NDetail::Recode<char>(YandexText, sUnicode, CODES_YANDEX);
UNIT_ASSERT(sUnicode == UnicodeText); // same content
UNIT_ASSERT(sUnicode.data() == wdata); // reserved buffer reused
UNIT_ASSERT(sUnicode.data() == wres.data()); // same buffer
UNIT_ASSERT(sUnicode.size() == wres.size()); // same size
TString sUtf8 = " ";
size_t scap = sUtf8.capacity();
sres = NDetail::Recode<wchar16>(UnicodeText, sUtf8, CODES_UTF8);
UNIT_ASSERT(sUtf8 == UTF8Text); // same content
UNIT_ASSERT(sUtf8.capacity() > scap); // increased buffer capacity (supplied was too small)
UNIT_ASSERT(sUtf8.data() == sres.data()); // same buffer
UNIT_ASSERT(sUtf8.size() == sres.size()); // same size
TEST_WCHAR32(sUtf8, UnicodeText, CODES_UTF8);
sUnicode.clear();
wdata = sUnicode.data();
TUtf16String copy = sUnicode; // increase ref-counter
wres = NDetail::Recode<char>(UTF8Text, sUnicode, CODES_UTF8);
UNIT_ASSERT(sUnicode == UnicodeText); // same content
#ifndef TSTRING_IS_STD_STRING
UNIT_ASSERT(sUnicode.data() != wdata); // re-allocated (shared buffer supplied)
UNIT_ASSERT(sUnicode.data() == wres.data()); // same buffer
#endif
UNIT_ASSERT(sUnicode.size() == wres.size()); // same content
}
static TString GenerateJunk(size_t seed) {
TString res;
size_t hash = NumericHash(seed);
size_t size = hash % 1024;
res.reserve(size);
for (size_t i = 0; i < size; ++i)
res += static_cast<char>(NumericHash(hash + i) % 256);
return res;
}
void TConversionTest::TestRecodeAppend() {
{
TString s1, s2;
NDetail::RecodeAppend<wchar16>(TUtf16String(), s1, CODES_YANDEX);
UNIT_ASSERT(s1.empty());
NDetail::RecodeAppend<wchar16>(UnicodeText, s1, CODES_WIN);
s2 += WideToChar(UnicodeText, CODES_WIN);
UNIT_ASSERT_EQUAL(s1, s2);
NDetail::RecodeAppend<wchar16>(UnicodeText, s1, CODES_YANDEX);
s2 += WideToChar(UnicodeText, CODES_YANDEX);
UNIT_ASSERT_EQUAL(s1, s2);
NDetail::RecodeAppend<wchar16>(TUtf16String(), s1, CODES_YANDEX);
UNIT_ASSERT_EQUAL(s1, s2);
NDetail::RecodeAppend<wchar16>(UnicodeText, s1, CODES_UTF8);
s2 += WideToUTF8(UnicodeText);
UNIT_ASSERT_EQUAL(s1, s2);
for (size_t i = 0; i < 100; ++i) {
TUtf16String junk = CharToWide(GenerateJunk(i), CODES_YANDEX);
NDetail::RecodeAppend<wchar16>(junk, s1, CODES_UTF8);
s2 += WideToUTF8(junk);
UNIT_ASSERT_EQUAL(s1, s2);
}
}
{
TUtf16String s1, s2;
NDetail::RecodeAppend<char>(TString(), s1, CODES_YANDEX);
UNIT_ASSERT(s1.empty());
NDetail::RecodeAppend<char>(YandexText, s1, CODES_WIN);
s2 += CharToWide(YandexText, CODES_WIN);
UNIT_ASSERT_EQUAL(s1, s2);
NDetail::RecodeAppend<char>(YandexText, s1, CODES_YANDEX);
s2 += CharToWide(YandexText, CODES_YANDEX);
UNIT_ASSERT_EQUAL(s1, s2);
NDetail::RecodeAppend<char>(TString(), s1, CODES_YANDEX);
UNIT_ASSERT_EQUAL(s1, s2);
NDetail::RecodeAppend<char>(UTF8Text, s1, CODES_UTF8);
s2 += UTF8ToWide(UTF8Text);
UNIT_ASSERT_EQUAL(s1, s2);
for (size_t i = 0; i < 100; ++i) {
TString junk = GenerateJunk(i);
NDetail::RecodeAppend<char>(junk, s1, CODES_YANDEX);
s2 += CharToWide(junk, CODES_YANDEX);
UNIT_ASSERT_EQUAL(s1, s2);
}
}
}
template <>
void Out<RECODE_RESULT>(IOutputStream& out, RECODE_RESULT val) {
out << int(val);
}
void TConversionTest::TestRecode() {
for (int c = 0; c != CODES_MAX; ++c) {
ECharset enc = static_cast<ECharset>(c);
if (!SingleByteCodepage(enc))
continue;
using THash = THashSet<char>;
THash hash;
for (int i = 0; i != 256; ++i) {
char ch = static_cast<char>(i);
wchar32 wch;
size_t read = 0;
size_t written = 0;
RECODE_RESULT res = RECODE_ERROR;
res = RecodeToUnicode(enc, &ch, &wch, 1, 1, read, written);
UNIT_ASSERT(res == RECODE_OK);
if (wch == BROKEN_RUNE)
continue;
char rch = 0;
res = RecodeFromUnicode(enc, &wch, &rch, 1, 1, read, written);
UNIT_ASSERT(res == RECODE_OK);
char rch2 = 0;
UNIT_ASSERT_VALUES_EQUAL(RECODE_OK, RecodeFromUnicode(enc, wch, &rch2, 1, written));
UNIT_ASSERT_VALUES_EQUAL(size_t(1), written);
UNIT_ASSERT_VALUES_EQUAL(rch2, rch);
if (hash.contains(rch)) { // there are some stupid encodings with duplicate characters
continue;
} else {
hash.insert(rch);
}
UNIT_ASSERT(ch == rch);
}
}
}
void TConversionTest::TestUnicodeLimit() {
for (int i = 0; i != CODES_MAX; ++i) {
ECharset code = static_cast<ECharset>(i);
if (!SingleByteCodepage(code))
continue;
const CodePage* page = CodePageByCharset(code);
Y_ASSERT(page);
for (int c = 0; c < 256; ++c) {
UNIT_ASSERT(page->unicode[c] < 1 << 16);
}
}
}