path: root/contrib/libs/icu/i18n/uspoof_conf.cpp

// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html 
/* 
****************************************************************************** 
* 
*   Copyright (C) 2008-2015, International Business Machines 
*   Corporation and others.  All Rights Reserved. 
* 
****************************************************************************** 
*   file name:  uspoof_conf.cpp 
*   encoding:   UTF-8
*   tab size:   8 (not used) 
*   indentation:4 
* 
*   created on: 2009Jan05  (refactoring earlier files) 
*   created by: Andy Heninger 
* 
*   Internal classes for compililing confusable data into its binary (runtime) form. 
*/ 
 
#include "unicode/utypes.h" 
#include "unicode/uspoof.h" 
#if !UCONFIG_NO_REGULAR_EXPRESSIONS 
#if !UCONFIG_NO_NORMALIZATION 
 
#include "unicode/unorm.h" 
#include "unicode/uregex.h" 
#include "unicode/ustring.h" 
#include "cmemory.h" 
#include "uspoof_impl.h" 
#include "uhash.h" 
#include "uvector.h" 
#include "uassert.h" 
#include "uarrsort.h" 
#include "uspoof_conf.h" 
 
U_NAMESPACE_USE 
 
 
//--------------------------------------------------------------------- 
// 
//  buildConfusableData   Compile the source confusable data, as defined by 
//                        the Unicode data file confusables.txt, into the binary 
//                        structures used by the confusable detector. 
// 
//                        The binary structures are described in uspoof_impl.h 
// 
//     1.  Parse the data, making a hash table mapping from a UChar32 to a String. 
// 
//     2.  Sort all of the strings encountered by length, since they will need to 
//         be stored in that order in the final string table. 
//         TODO: Sorting these strings by length is no longer needed since the removal of 
//         the string lengths table.  This logic can be removed to save processing time 
//         when building confusables data. 
// 
//     3.  Build a list of keys (UChar32s) from the four mapping tables.  Sort the 
//         list because that will be the ordering of our runtime table. 
// 
//     4.  Generate the run time string table.  This is generated before the key & value 
//         tables because we need the string indexes when building those tables. 
// 
//     5.  Build the run-time key and value tables.  These are parallel tables, and are built 
//         at the same time 
// 
 
SPUString::SPUString(UnicodeString *s) { 
    fStr = s; 
    fCharOrStrTableIndex = 0; 
} 
 
 
SPUString::~SPUString() { 
    delete fStr; 
} 
 
 
SPUStringPool::SPUStringPool(UErrorCode &status) : fVec(NULL), fHash(NULL) { 
    fVec = new UVector(status); 
    if (fVec == NULL) {
        status = U_MEMORY_ALLOCATION_ERROR;
        return;
    }
    fHash = uhash_open(uhash_hashUnicodeString,           // key hash function 
                       uhash_compareUnicodeString,        // Key Comparator 
                       NULL,                              // Value Comparator 
                       &status); 
} 
 
 
SPUStringPool::~SPUStringPool() { 
    int i; 
    for (i=fVec->size()-1; i>=0; i--) { 
        SPUString *s = static_cast<SPUString *>(fVec->elementAt(i)); 
        delete s; 
    } 
    delete fVec; 
    uhash_close(fHash); 
} 
 
 
int32_t SPUStringPool::size() { 
    return fVec->size(); 
} 
 
SPUString *SPUStringPool::getByIndex(int32_t index) { 
    SPUString *retString = (SPUString *)fVec->elementAt(index); 
    return retString; 
} 
 
 
// Comparison function for ordering strings in the string pool. 
// Compare by length first, then, within a group of the same length, 
// by code point order. 
// Conforms to the type signature for a USortComparator in uvector.h 
 
static int8_t U_CALLCONV SPUStringCompare(UHashTok left, UHashTok right) { 
	const SPUString *sL = const_cast<const SPUString *>( 
        static_cast<SPUString *>(left.pointer)); 
 	const SPUString *sR = const_cast<const SPUString *>( 
 	    static_cast<SPUString *>(right.pointer)); 
    int32_t lenL = sL->fStr->length(); 
    int32_t lenR = sR->fStr->length(); 
    if (lenL < lenR) { 
        return -1; 
    } else if (lenL > lenR) { 
        return 1; 
    } else { 
        return sL->fStr->compare(*(sR->fStr)); 
    } 
} 
 
void SPUStringPool::sort(UErrorCode &status) { 
    fVec->sort(SPUStringCompare, status); 
} 
 
 
SPUString *SPUStringPool::addString(UnicodeString *src, UErrorCode &status) { 
    SPUString *hashedString = static_cast<SPUString *>(uhash_get(fHash, src)); 
    if (hashedString != NULL) { 
        delete src; 
    } else { 
        hashedString = new SPUString(src); 
        if (hashedString == NULL) {
            status = U_MEMORY_ALLOCATION_ERROR;
            return NULL;
        }
        uhash_put(fHash, src, hashedString, &status); 
        fVec->addElement(hashedString, status); 
    } 
    return hashedString; 
} 
 
 
 
ConfusabledataBuilder::ConfusabledataBuilder(SpoofImpl *spImpl, UErrorCode &status) : 
    fSpoofImpl(spImpl), 
    fInput(NULL), 
    fTable(NULL), 
    fKeySet(NULL), 
    fKeyVec(NULL), 
    fValueVec(NULL), 
    fStringTable(NULL), 
    stringPool(NULL), 
    fParseLine(NULL), 
    fParseHexNum(NULL), 
    fLineNum(0) 
{ 
    if (U_FAILURE(status)) { 
        return; 
    } 

    fTable = uhash_open(uhash_hashLong, uhash_compareLong, NULL, &status);

    fKeySet = new UnicodeSet();
    if (fKeySet == NULL) {
        status = U_MEMORY_ALLOCATION_ERROR;
        return;
    }

    fKeyVec = new UVector(status);
    if (fKeyVec == NULL) {
        status = U_MEMORY_ALLOCATION_ERROR;
        return;
    }

    fValueVec = new UVector(status);
    if (fValueVec == NULL) {
        status = U_MEMORY_ALLOCATION_ERROR;
        return;
    }

    stringPool = new SPUStringPool(status); 
    if (stringPool == NULL) {
        status = U_MEMORY_ALLOCATION_ERROR;
        return;
    }
} 
 
 
ConfusabledataBuilder::~ConfusabledataBuilder() { 
    uprv_free(fInput); 
    uregex_close(fParseLine); 
    uregex_close(fParseHexNum); 
    uhash_close(fTable); 
    delete fKeySet; 
    delete fKeyVec; 
    delete fStringTable; 
    delete fValueVec; 
    delete stringPool; 
} 
 
 
void ConfusabledataBuilder::buildConfusableData(SpoofImpl * spImpl, const char * confusables, 
    int32_t confusablesLen, int32_t *errorType, UParseError *pe, UErrorCode &status) { 
 
    if (U_FAILURE(status)) { 
        return; 
    } 
    ConfusabledataBuilder builder(spImpl, status); 
    builder.build(confusables, confusablesLen, status); 
    if (U_FAILURE(status) && errorType != NULL) { 
        *errorType = USPOOF_SINGLE_SCRIPT_CONFUSABLE; 
        pe->line = builder.fLineNum; 
    } 
} 
 
 
void ConfusabledataBuilder::build(const char * confusables, int32_t confusablesLen, 
               UErrorCode &status) { 
 
    // Convert the user input data from UTF-8 to UChar (UTF-16) 
    int32_t inputLen = 0; 
    if (U_FAILURE(status)) { 
        return; 
    } 
    u_strFromUTF8(NULL, 0, &inputLen, confusables, confusablesLen, &status); 
    if (status != U_BUFFER_OVERFLOW_ERROR) { 
        return; 
    } 
    status = U_ZERO_ERROR; 
    fInput = static_cast<UChar *>(uprv_malloc((inputLen+1) * sizeof(UChar))); 
    if (fInput == NULL) { 
        status = U_MEMORY_ALLOCATION_ERROR; 
        return; 
    } 
    u_strFromUTF8(fInput, inputLen+1, NULL, confusables, confusablesLen, &status); 
 
 
    // Regular Expression to parse a line from Confusables.txt.  The expression will match 
    // any line.  What was matched is determined by examining which capture groups have a match. 
    //   Capture Group 1:  the source char 
    //   Capture Group 2:  the replacement chars 
    //   Capture Group 3-6  the table type, SL, SA, ML, or MA (deprecated) 
    //   Capture Group 7:  A blank or comment only line. 
    //   Capture Group 8:  A syntactically invalid line.  Anything that didn't match before. 
    // Example Line from the confusables.txt source file: 
    //   "1D702 ;	006E 0329 ;	SL	# MATHEMATICAL ITALIC SMALL ETA ... " 
    UnicodeString pattern( 
        "(?m)^[ \\t]*([0-9A-Fa-f]+)[ \\t]+;"      // Match the source char 
        "[ \\t]*([0-9A-Fa-f]+"                    // Match the replacement char(s) 
           "(?:[ \\t]+[0-9A-Fa-f]+)*)[ \\t]*;"    //     (continued) 
        "\\s*(?:(SL)|(SA)|(ML)|(MA))"             // Match the table type 
        "[ \\t]*(?:#.*?)?$"                       // Match any trailing #comment 
        "|^([ \\t]*(?:#.*?)?)$"       // OR match empty lines or lines with only a #comment 
        "|^(.*?)$", -1, US_INV);      // OR match any line, which catches illegal lines. 
    // TODO: Why are we using the regex C API here? C++ would just take UnicodeString... 
    fParseLine = uregex_open(pattern.getBuffer(), pattern.length(), 0, NULL, &status); 
 
    // Regular expression for parsing a hex number out of a space-separated list of them. 
    //   Capture group 1 gets the number, with spaces removed. 
    pattern = UNICODE_STRING_SIMPLE("\\s*([0-9A-F]+)"); 
    fParseHexNum = uregex_open(pattern.getBuffer(), pattern.length(), 0, NULL, &status); 
 
    // Zap any Byte Order Mark at the start of input.  Changing it to a space is benign 
    //   given the syntax of the input. 
    if (*fInput == 0xfeff) { 
        *fInput = 0x20; 
    } 
 
    // Parse the input, one line per iteration of this loop. 
    uregex_setText(fParseLine, fInput, inputLen, &status); 
    while (uregex_findNext(fParseLine, &status)) { 
        fLineNum++; 
        if (uregex_start(fParseLine, 7, &status) >= 0) { 
            // this was a blank or comment line. 
            continue; 
        } 
        if (uregex_start(fParseLine, 8, &status) >= 0) { 
            // input file syntax error. 
            status = U_PARSE_ERROR; 
            return; 
        } 
 
        // We have a good input line.  Extract the key character and mapping string, and 
        //    put them into the appropriate mapping table. 
        UChar32 keyChar = SpoofImpl::ScanHex(fInput, uregex_start(fParseLine, 1, &status), 
                          uregex_end(fParseLine, 1, &status), status); 
 
        int32_t mapStringStart = uregex_start(fParseLine, 2, &status); 
        int32_t mapStringLength = uregex_end(fParseLine, 2, &status) - mapStringStart; 
        uregex_setText(fParseHexNum, &fInput[mapStringStart], mapStringLength, &status); 
 
        UnicodeString  *mapString = new UnicodeString(); 
        if (mapString == NULL) { 
            status = U_MEMORY_ALLOCATION_ERROR; 
            return; 
        } 
        while (uregex_findNext(fParseHexNum, &status)) { 
            UChar32 c = SpoofImpl::ScanHex(&fInput[mapStringStart], uregex_start(fParseHexNum, 1, &status), 
                                 uregex_end(fParseHexNum, 1, &status), status); 
            mapString->append(c); 
        } 
        U_ASSERT(mapString->length() >= 1); 
 
        // Put the map (value) string into the string pool 
        // This a little like a Java intern() - any duplicates will be eliminated. 
        SPUString *smapString = stringPool->addString(mapString, status); 
 
        // Add the UChar32 -> string mapping to the table. 
        // For Unicode 8, the SL, SA and ML tables have been discontinued. 
        //                All input data from confusables.txt is tagged MA. 
        uhash_iput(fTable, keyChar, smapString, &status); 
        if (U_FAILURE(status)) { return; } 
        fKeySet->add(keyChar); 
    } 
 
    // Input data is now all parsed and collected. 
    // Now create the run-time binary form of the data. 
    // 
    // This is done in two steps.  First the data is assembled into vectors and strings, 
    //   for ease of construction, then the contents of these collections are dumped 
    //   into the actual raw-bytes data storage. 
 
    // Build up the string array, and record the index of each string therein 
    //  in the (build time only) string pool. 
    // Strings of length one are not entered into the strings array. 
    // (Strings in the table are sorted by length) 
    stringPool->sort(status); 
    fStringTable = new UnicodeString(); 
    int32_t poolSize = stringPool->size(); 
    int32_t i; 
    for (i=0; i<poolSize; i++) { 
        SPUString *s = stringPool->getByIndex(i); 
        int32_t strLen = s->fStr->length(); 
        int32_t strIndex = fStringTable->length(); 
        if (strLen == 1) { 
            // strings of length one do not get an entry in the string table. 
            // Keep the single string character itself here, which is the same 
            //  convention that is used in the final run-time string table index. 
            s->fCharOrStrTableIndex = s->fStr->charAt(0); 
        } else { 
            s->fCharOrStrTableIndex = strIndex; 
            fStringTable->append(*(s->fStr)); 
        } 
    } 
 
    // Construct the compile-time Key and Value tables 
    // 
    // For each key code point, check which mapping tables it applies to, 
    //   and create the final data for the key & value structures. 
    // 
    //   The four logical mapping tables are conflated into one combined table. 
    //   If multiple logical tables have the same mapping for some key, they 
    //     share a single entry in the combined table. 
    //   If more than one mapping exists for the same key code point, multiple 
    //     entries will be created in the table 
 
    for (int32_t range=0; range<fKeySet->getRangeCount(); range++) { 
        // It is an oddity of the UnicodeSet API that simply enumerating the contained 
        //   code points requires a nested loop. 
        for (UChar32 keyChar=fKeySet->getRangeStart(range); 
                keyChar <= fKeySet->getRangeEnd(range); keyChar++) { 
            SPUString *targetMapping = static_cast<SPUString *>(uhash_iget(fTable, keyChar)); 
            U_ASSERT(targetMapping != NULL); 
 
            // Set an error code if trying to consume a long string.  Otherwise, 
            // codePointAndLengthToKey will abort on a U_ASSERT. 
            if (targetMapping->fStr->length() > 256) { 
                status = U_ILLEGAL_ARGUMENT_ERROR; 
                return; 
            } 
 
            int32_t key = ConfusableDataUtils::codePointAndLengthToKey(keyChar, 
                targetMapping->fStr->length()); 
            int32_t value = targetMapping->fCharOrStrTableIndex; 
 
            fKeyVec->addElement(key, status); 
            fValueVec->addElement(value, status); 
        } 
    } 
 
    // Put the assembled data into the flat runtime array 
    outputData(status); 
 
    // All of the intermediate allocated data belongs to the ConfusabledataBuilder 
    //  object  (this), and is deleted in the destructor. 
    return; 
} 
 
// 
// outputData     The confusable data has been compiled and stored in intermediate 
//                collections and strings.  Copy it from there to the final flat 
//                binary array. 
// 
//                Note that as each section is added to the output data, the 
//                expand (reserveSpace() function will likely relocate it in memory. 
//                Be careful with pointers. 
// 
void ConfusabledataBuilder::outputData(UErrorCode &status) { 
 
    U_ASSERT(fSpoofImpl->fSpoofData->fDataOwned == TRUE); 
 
    //  The Key Table 
    //     While copying the keys to the runtime array, 
    //       also sanity check that they are sorted. 
 
    int32_t numKeys = fKeyVec->size(); 
    int32_t *keys = 
        static_cast<int32_t *>(fSpoofImpl->fSpoofData->reserveSpace(numKeys*sizeof(int32_t), status)); 
    if (U_FAILURE(status)) { 
        return; 
    } 
    int i; 
    UChar32 previousCodePoint = 0; 
    for (i=0; i<numKeys; i++) { 
        int32_t key =  fKeyVec->elementAti(i); 
        UChar32 codePoint = ConfusableDataUtils::keyToCodePoint(key); 
        (void)previousCodePoint;    // Suppress unused variable warning.
        // strictly greater because there can be only one entry per code point 
        U_ASSERT(codePoint > previousCodePoint); 
        keys[i] = key; 
        previousCodePoint = codePoint; 
    } 
    SpoofDataHeader *rawData = fSpoofImpl->fSpoofData->fRawData; 
    rawData->fCFUKeys = (int32_t)((char *)keys - (char *)rawData); 
    rawData->fCFUKeysSize = numKeys; 
    fSpoofImpl->fSpoofData->fCFUKeys = keys; 
 
 
    // The Value Table, parallels the key table 
    int32_t numValues = fValueVec->size(); 
    U_ASSERT(numKeys == numValues); 
    uint16_t *values = 
        static_cast<uint16_t *>(fSpoofImpl->fSpoofData->reserveSpace(numKeys*sizeof(uint16_t), status)); 
    if (U_FAILURE(status)) { 
        return; 
    } 
    for (i=0; i<numValues; i++) { 
        uint32_t value = static_cast<uint32_t>(fValueVec->elementAti(i)); 
        U_ASSERT(value < 0xffff); 
        values[i] = static_cast<uint16_t>(value); 
    } 
    rawData = fSpoofImpl->fSpoofData->fRawData; 
    rawData->fCFUStringIndex = (int32_t)((char *)values - (char *)rawData); 
    rawData->fCFUStringIndexSize = numValues; 
    fSpoofImpl->fSpoofData->fCFUValues = values; 
 
    // The Strings Table. 
 
    uint32_t stringsLength = fStringTable->length(); 
    // Reserve an extra space so the string will be nul-terminated.  This is 
    // only a convenience, for when debugging; it is not needed otherwise. 
    UChar *strings = 
        static_cast<UChar *>(fSpoofImpl->fSpoofData->reserveSpace(stringsLength*sizeof(UChar)+2, status)); 
    if (U_FAILURE(status)) { 
        return; 
    } 
    fStringTable->extract(strings, stringsLength+1, status); 
    rawData = fSpoofImpl->fSpoofData->fRawData; 
    U_ASSERT(rawData->fCFUStringTable == 0); 
    rawData->fCFUStringTable = (int32_t)((char *)strings - (char *)rawData); 
    rawData->fCFUStringTableLen = stringsLength; 
    fSpoofImpl->fSpoofData->fCFUStrings = strings; 
} 
 
#endif 
#endif // !UCONFIG_NO_REGULAR_EXPRESSIONS