path: root/contrib/libs/icu/i18n/cpdtrans.cpp

// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html 
/* 
********************************************************************** 
*   Copyright (C) 1999-2011, International Business Machines 
*   Corporation and others.  All Rights Reserved. 
********************************************************************** 
*   Date        Name        Description 
*   11/17/99    aliu        Creation. 
********************************************************************** 
*/ 
 
#include "unicode/utypes.h" 
 
#if !UCONFIG_NO_TRANSLITERATION 
 
#include "unicode/unifilt.h" 
#include "unicode/uniset.h" 
#include "cpdtrans.h" 
#include "uvector.h" 
#include "tridpars.h" 
#include "cmemory.h" 
 
// keep in sync with Transliterator 
//static const UChar ID_SEP   = 0x002D; /*-*/ 
static const UChar ID_DELIM = 0x003B; /*;*/ 
static const UChar NEWLINE  = 10; 
 
static const UChar COLON_COLON[] = {0x3A, 0x3A, 0}; //"::" 
 
U_NAMESPACE_BEGIN 
 
const UChar CompoundTransliterator::PASS_STRING[] = { 0x0025, 0x0050, 0x0061, 0x0073, 0x0073, 0 }; // "%Pass" 
 
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CompoundTransliterator) 
 
/** 
 * Constructs a new compound transliterator given an array of 
 * transliterators.  The array of transliterators may be of any 
 * length, including zero or one, however, useful compound 
 * transliterators have at least two components. 
 * @param transliterators array of <code>Transliterator</code> 
 * objects 
 * @param transliteratorCount The number of 
 * <code>Transliterator</code> objects in transliterators. 
 * @param filter the filter.  Any character for which 
 * <tt>filter.contains()</tt> returns <tt>false</tt> will not be 
 * altered by this transliterator.  If <tt>filter</tt> is 
 * <tt>null</tt> then no filtering is applied. 
 */ 
CompoundTransliterator::CompoundTransliterator( 
                           Transliterator* const transliterators[], 
                           int32_t transliteratorCount, 
                           UnicodeFilter* adoptedFilter) : 
    Transliterator(joinIDs(transliterators, transliteratorCount), adoptedFilter), 
    trans(0), count(0), numAnonymousRBTs(0)  { 
    setTransliterators(transliterators, transliteratorCount); 
} 
 
/** 
 * Splits an ID of the form "ID;ID;..." into a compound using each 
 * of the IDs.  
 * @param id of above form 
 * @param forward if false, does the list in reverse order, and 
 * takes the inverse of each ID. 
 */ 
CompoundTransliterator::CompoundTransliterator(const UnicodeString& id, 
                              UTransDirection direction, 
                              UnicodeFilter* adoptedFilter, 
                              UParseError& /*parseError*/, 
                              UErrorCode& status) : 
    Transliterator(id, adoptedFilter), 
    trans(0), numAnonymousRBTs(0) { 
    // TODO add code for parseError...currently unused, but 
    // later may be used by parsing code... 
    init(id, direction, TRUE, status); 
} 
 
CompoundTransliterator::CompoundTransliterator(const UnicodeString& id, 
                              UParseError& /*parseError*/, 
                              UErrorCode& status) : 
    Transliterator(id, 0), // set filter to 0 here! 
    trans(0), numAnonymousRBTs(0) { 
    // TODO add code for parseError...currently unused, but 
    // later may be used by parsing code... 
    init(id, UTRANS_FORWARD, TRUE, status); 
} 
 
 
/** 
 * Private constructor for use of TransliteratorAlias 
 */ 
CompoundTransliterator::CompoundTransliterator(const UnicodeString& newID, 
                                              UVector& list, 
                                              UnicodeFilter* adoptedFilter, 
                                              int32_t anonymousRBTs, 
                                              UParseError& /*parseError*/, 
                                              UErrorCode& status) : 
    Transliterator(newID, adoptedFilter), 
    trans(0), numAnonymousRBTs(anonymousRBTs) 
{ 
    init(list, UTRANS_FORWARD, FALSE, status); 
} 
 
/** 
 * Private constructor for Transliterator from a vector of 
 * transliterators.  The caller is responsible for fixing up the 
 * ID. 
 */ 
CompoundTransliterator::CompoundTransliterator(UVector& list, 
                                               UParseError& /*parseError*/, 
                                               UErrorCode& status) : 
    Transliterator(UnicodeString(), NULL), 
    trans(0), numAnonymousRBTs(0) 
{ 
    // TODO add code for parseError...currently unused, but 
    // later may be used by parsing code... 
    init(list, UTRANS_FORWARD, FALSE, status); 
    // assume caller will fixup ID 
} 
 
CompoundTransliterator::CompoundTransliterator(UVector& list, 
                                               int32_t anonymousRBTs, 
                                               UParseError& /*parseError*/, 
                                               UErrorCode& status) : 
    Transliterator(UnicodeString(), NULL), 
    trans(0), numAnonymousRBTs(anonymousRBTs) 
{ 
    init(list, UTRANS_FORWARD, FALSE, status); 
} 
 
/** 
 * Finish constructing a transliterator: only to be called by 
 * constructors.  Before calling init(), set trans and filter to NULL. 
 * @param id the id containing ';'-separated entries 
 * @param direction either FORWARD or REVERSE 
 * @param idSplitPoint the index into id at which the 
 * adoptedSplitTransliterator should be inserted, if there is one, or 
 * -1 if there is none. 
 * @param adoptedSplitTransliterator a transliterator to be inserted 
 * before the entry at offset idSplitPoint in the id string.  May be 
 * NULL to insert no entry. 
 * @param fixReverseID if TRUE, then reconstruct the ID of reverse 
 * entries by calling getID() of component entries.  Some constructors 
 * do not require this because they apply a facade ID anyway. 
 * @param status the error code indicating success or failure 
 */ 
void CompoundTransliterator::init(const UnicodeString& id, 
                                  UTransDirection direction, 
                                  UBool fixReverseID, 
                                  UErrorCode& status) { 
    // assert(trans == 0); 
 
    if (U_FAILURE(status)) { 
        return; 
    } 
 
    UVector list(status); 
    UnicodeSet* compoundFilter = NULL; 
    UnicodeString regenID; 
    if (!TransliteratorIDParser::parseCompoundID(id, direction, 
                                      regenID, list, compoundFilter)) { 
        status = U_INVALID_ID; 
        delete compoundFilter; 
        return; 
    } 
 
    TransliteratorIDParser::instantiateList(list, status); 
 
    init(list, direction, fixReverseID, status); 
 
    if (compoundFilter != NULL) { 
        adoptFilter(compoundFilter); 
    } 
} 
 
/** 
 * Finish constructing a transliterator: only to be called by 
 * constructors.  Before calling init(), set trans and filter to NULL. 
 * @param list a vector of transliterator objects to be adopted.  It 
 * should NOT be empty.  The list should be in declared order.  That 
 * is, it should be in the FORWARD order; if direction is REVERSE then 
 * the list order will be reversed. 
 * @param direction either FORWARD or REVERSE 
 * @param fixReverseID if TRUE, then reconstruct the ID of reverse 
 * entries by calling getID() of component entries.  Some constructors 
 * do not require this because they apply a facade ID anyway. 
 * @param status the error code indicating success or failure 
 */ 
void CompoundTransliterator::init(UVector& list, 
                                  UTransDirection direction, 
                                  UBool fixReverseID, 
                                  UErrorCode& status) { 
    // assert(trans == 0); 
 
    // Allocate array 
    if (U_SUCCESS(status)) { 
        count = list.size(); 
        trans = (Transliterator **)uprv_malloc(count * sizeof(Transliterator *)); 
        /* test for NULL */ 
        if (trans == 0) { 
            status = U_MEMORY_ALLOCATION_ERROR; 
            return; 
        } 
    } 
 
    if (U_FAILURE(status) || trans == 0) { 
         // assert(trans == 0); 
        return; 
    } 
 
    // Move the transliterators from the vector into an array. 
    // Reverse the order if necessary. 
    int32_t i; 
    for (i=0; i<count; ++i) { 
        int32_t j = (direction == UTRANS_FORWARD) ? i : count - 1 - i; 
        trans[i] = (Transliterator*) list.elementAt(j); 
    } 
 
    // If the direction is UTRANS_REVERSE then we may need to fix the 
    // ID. 
    if (direction == UTRANS_REVERSE && fixReverseID) { 
        UnicodeString newID; 
        for (i=0; i<count; ++i) { 
            if (i > 0) { 
                newID.append(ID_DELIM); 
            } 
            newID.append(trans[i]->getID()); 
        } 
        setID(newID); 
    } 
 
    computeMaximumContextLength(); 
} 
 
/** 
 * Return the IDs of the given list of transliterators, concatenated 
 * with ID_DELIM delimiting them.  Equivalent to the perlish expression 
 * join(ID_DELIM, map($_.getID(), transliterators). 
 */ 
UnicodeString CompoundTransliterator::joinIDs(Transliterator* const transliterators[], 
                                              int32_t transCount) { 
    UnicodeString id; 
    for (int32_t i=0; i<transCount; ++i) { 
        if (i > 0) { 
            id.append(ID_DELIM); 
        } 
        id.append(transliterators[i]->getID()); 
    } 
    return id; // Return temporary 
} 
 
/** 
 * Copy constructor. 
 */ 
CompoundTransliterator::CompoundTransliterator(const CompoundTransliterator& t) : 
    Transliterator(t), trans(0), count(0), numAnonymousRBTs(-1) { 
    *this = t; 
} 
 
/** 
 * Destructor 
 */ 
CompoundTransliterator::~CompoundTransliterator() { 
    freeTransliterators(); 
} 
 
void CompoundTransliterator::freeTransliterators(void) { 
    if (trans != 0) { 
        for (int32_t i=0; i<count; ++i) { 
            delete trans[i]; 
        } 
        uprv_free(trans); 
    } 
    trans = 0; 
    count = 0; 
} 
 
/** 
 * Assignment operator. 
 */ 
CompoundTransliterator& CompoundTransliterator::operator=( 
                                             const CompoundTransliterator& t) 
{ 
    Transliterator::operator=(t); 
    int32_t i = 0; 
    UBool failed = FALSE; 
    if (trans != NULL) { 
        for (i=0; i<count; ++i) { 
            delete trans[i]; 
            trans[i] = 0; 
        } 
    } 
    if (t.count > count) { 
        if (trans != NULL) { 
            uprv_free(trans); 
        } 
        trans = (Transliterator **)uprv_malloc(t.count * sizeof(Transliterator *)); 
    } 
    count = t.count; 
    if (trans != NULL) { 
        for (i=0; i<count; ++i) { 
            trans[i] = t.trans[i]->clone(); 
            if (trans[i] == NULL) { 
                failed = TRUE; 
                break; 
            } 
        } 
    } 
 
    // if memory allocation failed delete backwards trans array 
    if (failed && i > 0) { 
        int32_t n; 
        for (n = i-1; n >= 0; n--) { 
            uprv_free(trans[n]); 
            trans[n] = NULL; 
        } 
    } 
    numAnonymousRBTs = t.numAnonymousRBTs; 
    return *this; 
} 
 
/** 
 * Transliterator API. 
 */ 
CompoundTransliterator* CompoundTransliterator::clone() const {
    return new CompoundTransliterator(*this); 
} 
 
/** 
 * Returns the number of transliterators in this chain. 
 * @return number of transliterators in this chain. 
 */ 
int32_t CompoundTransliterator::getCount(void) const { 
    return count; 
} 
 
/** 
 * Returns the transliterator at the given index in this chain. 
 * @param index index into chain, from 0 to <code>getCount() - 1</code> 
 * @return transliterator at the given index 
 */ 
const Transliterator& CompoundTransliterator::getTransliterator(int32_t index) const { 
    return *trans[index]; 
} 
 
void CompoundTransliterator::setTransliterators(Transliterator* const transliterators[], 
                                                int32_t transCount) { 
    Transliterator** a = (Transliterator **)uprv_malloc(transCount * sizeof(Transliterator *)); 
    if (a == NULL) { 
        return; 
    } 
    int32_t i = 0; 
    UBool failed = FALSE; 
    for (i=0; i<transCount; ++i) { 
        a[i] = transliterators[i]->clone(); 
        if (a[i] == NULL) { 
            failed = TRUE; 
            break; 
        } 
    } 
    if (failed && i > 0) { 
        int32_t n; 
        for (n = i-1; n >= 0; n--) { 
            uprv_free(a[n]); 
            a[n] = NULL; 
        } 
        return; 
    } 
    adoptTransliterators(a, transCount); 
} 
 
void CompoundTransliterator::adoptTransliterators(Transliterator* adoptedTransliterators[], 
                                                  int32_t transCount) { 
    // First free trans[] and set count to zero.  Once this is done, 
    // orphan the filter.  Set up the new trans[]. 
    freeTransliterators(); 
    trans = adoptedTransliterators; 
    count = transCount; 
    computeMaximumContextLength(); 
    setID(joinIDs(trans, count)); 
} 
 
/** 
 * Append c to buf, unless buf is empty or buf already ends in c. 
 */ 
static void _smartAppend(UnicodeString& buf, UChar c) { 
    if (buf.length() != 0 && 
        buf.charAt(buf.length() - 1) != c) { 
        buf.append(c); 
    } 
} 
 
UnicodeString& CompoundTransliterator::toRules(UnicodeString& rulesSource, 
                                               UBool escapeUnprintable) const { 
    // We do NOT call toRules() on our component transliterators, in 
    // general.  If we have several rule-based transliterators, this 
    // yields a concatenation of the rules -- not what we want.  We do 
    // handle compound RBT transliterators specially -- those for which 
    // compoundRBTIndex >= 0.  For the transliterator at compoundRBTIndex, 
    // we do call toRules() recursively. 
    rulesSource.truncate(0); 
    if (numAnonymousRBTs >= 1 && getFilter() != NULL) { 
        // If we are a compound RBT and if we have a global 
        // filter, then emit it at the top. 
        UnicodeString pat; 
        rulesSource.append(COLON_COLON, 2).append(getFilter()->toPattern(pat, escapeUnprintable)).append(ID_DELIM); 
    } 
    for (int32_t i=0; i<count; ++i) { 
        UnicodeString rule; 
 
        // Anonymous RuleBasedTransliterators (inline rules and 
        // ::BEGIN/::END blocks) are given IDs that begin with 
        // "%Pass": use toRules() to write all the rules to the output 
        // (and insert "::Null;" if we have two in a row) 
        if (trans[i]->getID().startsWith(PASS_STRING, 5)) { 
            trans[i]->toRules(rule, escapeUnprintable); 
            if (numAnonymousRBTs > 1 && i > 0 && trans[i - 1]->getID().startsWith(PASS_STRING, 5)) 
                rule = UNICODE_STRING_SIMPLE("::Null;") + rule; 
 
        // we also use toRules() on CompoundTransliterators (which we 
        // check for by looking for a semicolon in the ID)-- this gets 
        // the list of their child transliterators output in the right 
        // format 
        } else if (trans[i]->getID().indexOf(ID_DELIM) >= 0) { 
            trans[i]->toRules(rule, escapeUnprintable); 
 
        // for everything else, use Transliterator::toRules() 
        } else { 
            trans[i]->Transliterator::toRules(rule, escapeUnprintable); 
        } 
        _smartAppend(rulesSource, NEWLINE); 
        rulesSource.append(rule); 
        _smartAppend(rulesSource, ID_DELIM); 
    } 
    return rulesSource; 
} 
 
/** 
 * Implement Transliterator framework 
 */ 
void CompoundTransliterator::handleGetSourceSet(UnicodeSet& result) const { 
    UnicodeSet set; 
    result.clear(); 
    for (int32_t i=0; i<count; ++i) { 
    result.addAll(trans[i]->getSourceSet(set)); 
    // Take the example of Hiragana-Latin.  This is really 
    // Hiragana-Katakana; Katakana-Latin.  The source set of 
    // these two is roughly [:Hiragana:] and [:Katakana:]. 
    // But the source set for the entire transliterator is 
    // actually [:Hiragana:] ONLY -- that is, the first 
    // non-empty source set. 
 
    // This is a heuristic, and not 100% reliable. 
    if (!result.isEmpty()) { 
        break; 
    } 
    } 
} 
 
/** 
 * Override Transliterator framework 
 */ 
UnicodeSet& CompoundTransliterator::getTargetSet(UnicodeSet& result) const { 
    UnicodeSet set; 
    result.clear(); 
    for (int32_t i=0; i<count; ++i) { 
    // This is a heuristic, and not 100% reliable. 
    result.addAll(trans[i]->getTargetSet(set)); 
    } 
    return result; 
} 
 
/** 
 * Implements {@link Transliterator#handleTransliterate}. 
 */ 
void CompoundTransliterator::handleTransliterate(Replaceable& text, UTransPosition& index, 
                                                 UBool incremental) const { 
    /* Call each transliterator with the same contextStart and 
     * start, but with the limit as modified 
     * by preceding transliterators.  The start index must be 
     * reset for each transliterator to give each a chance to 
     * transliterate the text.  The initial contextStart index is known 
     * to still point to the same place after each transliterator 
     * is called because each transliterator will not change the 
     * text between contextStart and the initial start index. 
     * 
     * IMPORTANT: After the first transliterator, each subsequent 
     * transliterator only gets to transliterate text committed by 
     * preceding transliterators; that is, the start (output 
     * value) of transliterator i becomes the limit (input value) 
     * of transliterator i+1.  Finally, the overall limit is fixed 
     * up before we return. 
     * 
     * Assumptions we make here: 
     * (1) contextStart <= start <= limit <= contextLimit <= text.length() 
     * (2) start <= start' <= limit'  ;cursor doesn't move back 
     * (3) start <= limit'            ;text before cursor unchanged 
     * - start' is the value of start after calling handleKT 
     * - limit' is the value of limit after calling handleKT 
     */ 
     
    /** 
     * Example: 3 transliterators.  This example illustrates the 
     * mechanics we need to implement.  C, S, and L are the contextStart, 
     * start, and limit.  gl is the globalLimit.  contextLimit is 
     * equal to limit throughout. 
     * 
     * 1. h-u, changes hex to Unicode 
     * 
     *    4  7  a  d  0      4  7  a 
     *    abc/u0061/u    =>  abca/u     
     *    C  S       L       C   S L   gl=f->a 
     * 
     * 2. upup, changes "x" to "XX" 
     * 
     *    4  7  a       4  7  a 
     *    abca/u    =>  abcAA/u     
     *    C  SL         C    S    
     *                       L    gl=a->b 
     * 3. u-h, changes Unicode to hex 
     * 
     *    4  7  a        4  7  a  d  0  3 
     *    abcAA/u    =>  abc/u0041/u0041/u     
     *    C  S L         C              S 
     *                                  L   gl=b->15 
     * 4. return 
     * 
     *    4  7  a  d  0  3 
     *    abc/u0041/u0041/u     
     *    C S L 
     */ 
 
    if (count < 1) { 
        index.start = index.limit; 
        return; // Short circuit for empty compound transliterators 
    } 
 
    // compoundLimit is the limit value for the entire compound 
    // operation.  We overwrite index.limit with the previous 
    // index.start.  After each transliteration, we update 
    // compoundLimit for insertions or deletions that have happened. 
    int32_t compoundLimit = index.limit; 
 
    // compoundStart is the start for the entire compound 
    // operation. 
    int32_t compoundStart = index.start; 
     
    int32_t delta = 0; // delta in length 
 
    // Give each transliterator a crack at the run of characters. 
    // See comments at the top of the method for more detail. 
    for (int32_t i=0; i<count; ++i) { 
        index.start = compoundStart; // Reset start 
        int32_t limit = index.limit; 
         
        if (index.start == index.limit) { 
            // Short circuit for empty range 
            break; 
        } 
 
        trans[i]->filteredTransliterate(text, index, incremental); 
         
        // In a properly written transliterator, start == limit after 
        // handleTransliterate() returns when incremental is false. 
        // Catch cases where the subclass doesn't do this, and throw 
        // an exception.  (Just pinning start to limit is a bad idea, 
        // because what's probably happening is that the subclass 
        // isn't transliterating all the way to the end, and it should 
        // in non-incremental mode.) 
        if (!incremental && index.start != index.limit) { 
            // We can't throw an exception, so just fudge things 
            index.start = index.limit; 
        } 
 
        // Cumulative delta for insertions/deletions 
        delta += index.limit - limit; 
         
        if (incremental) { 
            // In the incremental case, only allow subsequent 
            // transliterators to modify what has already been 
            // completely processed by prior transliterators.  In the 
            // non-incrmental case, allow each transliterator to 
            // process the entire text. 
            index.limit = index.start; 
        } 
    } 
 
    compoundLimit += delta; 
 
    // Start is good where it is -- where the last transliterator left 
    // it.  Limit needs to be put back where it was, modulo 
    // adjustments for deletions/insertions. 
    index.limit = compoundLimit; 
} 
 
/** 
 * Sets the length of the longest context required by this transliterator. 
 * This is <em>preceding</em> context. 
 */ 
void CompoundTransliterator::computeMaximumContextLength(void) { 
    int32_t max = 0; 
    for (int32_t i=0; i<count; ++i) { 
        int32_t len = trans[i]->getMaximumContextLength(); 
        if (len > max) { 
            max = len; 
        } 
    } 
    setMaximumContextLength(max); 
} 
 
U_NAMESPACE_END 
 
#endif /* #if !UCONFIG_NO_TRANSLITERATION */ 
 
/* eof */