aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/libs/icu/i18n/numparse_affixes.cpp
blob: 8333600f9e2cdd3027eb30688111150bf44597f2 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
// © 2018 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html

#include "unicode/utypes.h"

#if !UCONFIG_NO_FORMATTING

// Allow implicit conversion from char16_t* to UnicodeString for this file:
// Helpful in toString methods and elsewhere.
#define UNISTR_FROM_STRING_EXPLICIT

#include "numparse_types.h"
#include "numparse_affixes.h"
#include "numparse_utils.h"
#include "number_utils.h"
#include "string_segment.h"

using namespace icu;
using namespace icu::numparse;
using namespace icu::numparse::impl;
using namespace icu::number;
using namespace icu::number::impl;


namespace {

/**
 * Helper method to return whether the given AffixPatternMatcher equals the given pattern string.
 * Either both arguments must be null or the pattern string inside the AffixPatternMatcher must equal
 * the given pattern string.
 */
bool matched(const AffixPatternMatcher* affix, const UnicodeString& patternString) {
    return (affix == nullptr && patternString.isBogus()) ||
           (affix != nullptr && affix->getPattern() == patternString);
}

/**
 * Helper method to return the length of the given AffixPatternMatcher. Returns 0 for null.
 */
int32_t length(const AffixPatternMatcher* matcher) {
    return matcher == nullptr ? 0 : matcher->getPattern().length();
}

/**
 * Helper method to return whether (1) both lhs and rhs are null/invalid, or (2) if they are both
 * valid, whether they are equal according to operator==.  Similar to Java Objects.equals()
 */
bool equals(const AffixPatternMatcher* lhs, const AffixPatternMatcher* rhs) {
    if (lhs == nullptr && rhs == nullptr) {
        return true;
    }
    if (lhs == nullptr || rhs == nullptr) {
        return false;
    }
    return *lhs == *rhs;
}

}


AffixPatternMatcherBuilder::AffixPatternMatcherBuilder(const UnicodeString& pattern,
                                                       AffixTokenMatcherWarehouse& warehouse,
                                                       IgnorablesMatcher* ignorables)
        : fMatchersLen(0),
          fLastTypeOrCp(0),
          fPattern(pattern),
          fWarehouse(warehouse),
          fIgnorables(ignorables) {}

void AffixPatternMatcherBuilder::consumeToken(AffixPatternType type, UChar32 cp, UErrorCode& status) {
    // This is called by AffixUtils.iterateWithConsumer() for each token.

    // Add an ignorables matcher between tokens except between two literals, and don't put two
    // ignorables matchers in a row.
    if (fIgnorables != nullptr && fMatchersLen > 0 &&
        (fLastTypeOrCp < 0 || !fIgnorables->getSet()->contains(fLastTypeOrCp))) {
        addMatcher(*fIgnorables);
    }

    if (type != TYPE_CODEPOINT) {
        // Case 1: the token is a symbol.
        switch (type) {
            case TYPE_MINUS_SIGN:
                addMatcher(fWarehouse.minusSign());
                break;
            case TYPE_PLUS_SIGN:
                addMatcher(fWarehouse.plusSign());
                break;
            case TYPE_PERCENT:
                addMatcher(fWarehouse.percent());
                break;
            case TYPE_PERMILLE:
                addMatcher(fWarehouse.permille());
                break;
            case TYPE_CURRENCY_SINGLE:
            case TYPE_CURRENCY_DOUBLE:
            case TYPE_CURRENCY_TRIPLE:
            case TYPE_CURRENCY_QUAD:
            case TYPE_CURRENCY_QUINT:
                // All currency symbols use the same matcher
                addMatcher(fWarehouse.currency(status));
                break;
            default:
                UPRV_UNREACHABLE_EXIT;
        }

    } else if (fIgnorables != nullptr && fIgnorables->getSet()->contains(cp)) {
        // Case 2: the token is an ignorable literal.
        // No action necessary: the ignorables matcher has already been added.

    } else {
        // Case 3: the token is a non-ignorable literal.
        if (auto* ptr = fWarehouse.nextCodePointMatcher(cp, status)) {
            addMatcher(*ptr);
        } else {
            // OOM; unwind the stack
            return;
        }
    }
    fLastTypeOrCp = type != TYPE_CODEPOINT ? type : cp;
}

void AffixPatternMatcherBuilder::addMatcher(NumberParseMatcher& matcher) {
    if (fMatchersLen >= fMatchers.getCapacity()) {
        fMatchers.resize(fMatchersLen * 2, fMatchersLen);
    }
    fMatchers[fMatchersLen++] = &matcher;
}

AffixPatternMatcher AffixPatternMatcherBuilder::build(UErrorCode& status) {
    return AffixPatternMatcher(fMatchers, fMatchersLen, fPattern, status);
}

AffixTokenMatcherWarehouse::AffixTokenMatcherWarehouse(const AffixTokenMatcherSetupData* setupData)
        : fSetupData(setupData) {}

NumberParseMatcher& AffixTokenMatcherWarehouse::minusSign() {
    return fMinusSign = {fSetupData->dfs, true};
}

NumberParseMatcher& AffixTokenMatcherWarehouse::plusSign() {
    return fPlusSign = {fSetupData->dfs, true};
}

NumberParseMatcher& AffixTokenMatcherWarehouse::percent() {
    return fPercent = {fSetupData->dfs};
}

NumberParseMatcher& AffixTokenMatcherWarehouse::permille() {
    return fPermille = {fSetupData->dfs};
}

NumberParseMatcher& AffixTokenMatcherWarehouse::currency(UErrorCode& status) {
    return fCurrency = {fSetupData->currencySymbols, fSetupData->dfs, fSetupData->parseFlags, status};
}

IgnorablesMatcher& AffixTokenMatcherWarehouse::ignorables() {
    return fSetupData->ignorables;
}

NumberParseMatcher* AffixTokenMatcherWarehouse::nextCodePointMatcher(UChar32 cp, UErrorCode& status) {
    if (U_FAILURE(status)) {
        return nullptr;
    }
    auto* result = fCodePoints.create(cp);
    if (result == nullptr) {
        status = U_MEMORY_ALLOCATION_ERROR;
    }
    return result;
}

bool AffixTokenMatcherWarehouse::hasEmptyCurrencySymbol() const {
    return fSetupData->currencySymbols.hasEmptyCurrencySymbol();
}


CodePointMatcher::CodePointMatcher(UChar32 cp)
        : fCp(cp) {}

bool CodePointMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode&) const {
    if (segment.startsWith(fCp)) {
        segment.adjustOffsetByCodePoint();
        result.setCharsConsumed(segment);
    }
    return false;
}

bool CodePointMatcher::smokeTest(const StringSegment& segment) const {
    return segment.startsWith(fCp);
}

UnicodeString CodePointMatcher::toString() const {
    return u"<CodePoint>";
}


AffixPatternMatcher AffixPatternMatcher::fromAffixPattern(const UnicodeString& affixPattern,
                                                          AffixTokenMatcherWarehouse& tokenWarehouse,
                                                          parse_flags_t parseFlags, bool* success,
                                                          UErrorCode& status) {
    if (affixPattern.isEmpty()) {
        *success = false;
        return {};
    }
    *success = true;

    IgnorablesMatcher* ignorables;
    if (0 != (parseFlags & PARSE_FLAG_EXACT_AFFIX)) {
        ignorables = nullptr;
    } else {
        ignorables = &tokenWarehouse.ignorables();
    }

    AffixPatternMatcherBuilder builder(affixPattern, tokenWarehouse, ignorables);
    AffixUtils::iterateWithConsumer(affixPattern, builder, status);
    return builder.build(status);
}

AffixPatternMatcher::AffixPatternMatcher(MatcherArray& matchers, int32_t matchersLen,
                                         const UnicodeString& pattern, UErrorCode& status)
    : ArraySeriesMatcher(matchers, matchersLen), fPattern(pattern, status) {
}

UnicodeString AffixPatternMatcher::getPattern() const {
    return fPattern.toAliasedUnicodeString();
}

bool AffixPatternMatcher::operator==(const AffixPatternMatcher& other) const {
    return fPattern == other.fPattern;
}


AffixMatcherWarehouse::AffixMatcherWarehouse(AffixTokenMatcherWarehouse* tokenWarehouse)
        : fTokenWarehouse(tokenWarehouse) {
}

bool AffixMatcherWarehouse::isInteresting(const AffixPatternProvider& patternInfo,
                                          const IgnorablesMatcher& ignorables, parse_flags_t parseFlags,
                                          UErrorCode& status) {
    UnicodeString posPrefixString = patternInfo.getString(AffixPatternProvider::AFFIX_POS_PREFIX);
    UnicodeString posSuffixString = patternInfo.getString(AffixPatternProvider::AFFIX_POS_SUFFIX);
    UnicodeString negPrefixString;
    UnicodeString negSuffixString;
    if (patternInfo.hasNegativeSubpattern()) {
        negPrefixString = patternInfo.getString(AffixPatternProvider::AFFIX_NEG_PREFIX);
        negSuffixString = patternInfo.getString(AffixPatternProvider::AFFIX_NEG_SUFFIX);
    }

    if (0 == (parseFlags & PARSE_FLAG_USE_FULL_AFFIXES) &&
        AffixUtils::containsOnlySymbolsAndIgnorables(posPrefixString, *ignorables.getSet(), status) &&
        AffixUtils::containsOnlySymbolsAndIgnorables(posSuffixString, *ignorables.getSet(), status) &&
        AffixUtils::containsOnlySymbolsAndIgnorables(negPrefixString, *ignorables.getSet(), status) &&
        AffixUtils::containsOnlySymbolsAndIgnorables(negSuffixString, *ignorables.getSet(), status)
        // HACK: Plus and minus sign are a special case: we accept them trailing only if they are
        // trailing in the pattern string.
        && !AffixUtils::containsType(posSuffixString, TYPE_PLUS_SIGN, status) &&
        !AffixUtils::containsType(posSuffixString, TYPE_MINUS_SIGN, status) &&
        !AffixUtils::containsType(negSuffixString, TYPE_PLUS_SIGN, status) &&
        !AffixUtils::containsType(negSuffixString, TYPE_MINUS_SIGN, status)) {
        // The affixes contain only symbols and ignorables.
        // No need to generate affix matchers.
        return false;
    }
    return true;
}

void AffixMatcherWarehouse::createAffixMatchers(const AffixPatternProvider& patternInfo,
                                                MutableMatcherCollection& output,
                                                const IgnorablesMatcher& ignorables,
                                                parse_flags_t parseFlags, UErrorCode& status) {
    if (!isInteresting(patternInfo, ignorables, parseFlags, status)) {
        return;
    }

    // The affixes have interesting characters, or we are in strict mode.
    // Use initial capacity of 6, the highest possible number of AffixMatchers.
    UnicodeString sb;
    bool includeUnpaired = 0 != (parseFlags & PARSE_FLAG_INCLUDE_UNPAIRED_AFFIXES);

    int32_t numAffixMatchers = 0;
    int32_t numAffixPatternMatchers = 0;

    AffixPatternMatcher* posPrefix = nullptr;
    AffixPatternMatcher* posSuffix = nullptr;

    // Pre-process the affix strings to resolve LDML rules like sign display.
    for (int8_t typeInt = 0; typeInt < PATTERN_SIGN_TYPE_COUNT * 2; typeInt++) {
        auto type = static_cast<PatternSignType>(typeInt / 2);
        bool dropCurrencySymbols = (typeInt % 2) == 1;

        if (dropCurrencySymbols && !patternInfo.hasCurrencySign()) {
            continue;
        }
        if (dropCurrencySymbols && !fTokenWarehouse->hasEmptyCurrencySymbol()) {
            continue;
        }

        // Skip affixes in some cases
        if (type == PATTERN_SIGN_TYPE_POS
                && 0 != (parseFlags & PARSE_FLAG_PLUS_SIGN_ALLOWED)) {
            continue;
        }
        if (type == PATTERN_SIGN_TYPE_POS_SIGN
                && 0 == (parseFlags & PARSE_FLAG_PLUS_SIGN_ALLOWED)) {
            continue;
        }

        // Generate Prefix
        // TODO: Handle approximately sign?
        bool hasPrefix = false;
        PatternStringUtils::patternInfoToStringBuilder(
                patternInfo, true, type, false, StandardPlural::OTHER, false, dropCurrencySymbols, sb);
        fAffixPatternMatchers[numAffixPatternMatchers] = AffixPatternMatcher::fromAffixPattern(
                sb, *fTokenWarehouse, parseFlags, &hasPrefix, status);
        AffixPatternMatcher* prefix = hasPrefix ? &fAffixPatternMatchers[numAffixPatternMatchers++]
                                                : nullptr;

        // Generate Suffix
        // TODO: Handle approximately sign?
        bool hasSuffix = false;
        PatternStringUtils::patternInfoToStringBuilder(
                patternInfo, false, type, false, StandardPlural::OTHER, false, dropCurrencySymbols, sb);
        fAffixPatternMatchers[numAffixPatternMatchers] = AffixPatternMatcher::fromAffixPattern(
                sb, *fTokenWarehouse, parseFlags, &hasSuffix, status);
        AffixPatternMatcher* suffix = hasSuffix ? &fAffixPatternMatchers[numAffixPatternMatchers++]
                                                : nullptr;

        if (type == PATTERN_SIGN_TYPE_POS) {
            posPrefix = prefix;
            posSuffix = suffix;
        } else if (equals(prefix, posPrefix) && equals(suffix, posSuffix)) {
            // Skip adding these matchers (we already have equivalents)
            continue;
        }

        // Flags for setting in the ParsedNumber; the token matchers may add more.
        int flags = (type == PATTERN_SIGN_TYPE_NEG) ? FLAG_NEGATIVE : 0;

        // Note: it is indeed possible for posPrefix and posSuffix to both be null.
        // We still need to add that matcher for strict mode to work.
        fAffixMatchers[numAffixMatchers++] = {prefix, suffix, flags};
        if (includeUnpaired && prefix != nullptr && suffix != nullptr) {
            // The following if statements are designed to prevent adding two identical matchers.
            if (type == PATTERN_SIGN_TYPE_POS || !equals(prefix, posPrefix)) {
                fAffixMatchers[numAffixMatchers++] = {prefix, nullptr, flags};
            }
            if (type == PATTERN_SIGN_TYPE_POS || !equals(suffix, posSuffix)) {
                fAffixMatchers[numAffixMatchers++] = {nullptr, suffix, flags};
            }
        }
    }

    // Put the AffixMatchers in order, and then add them to the output.
    // Since there are at most 9 elements, do a simple-to-implement bubble sort.
    bool madeChanges;
    do {
        madeChanges = false;
        for (int32_t i = 1; i < numAffixMatchers; i++) {
            if (fAffixMatchers[i - 1].compareTo(fAffixMatchers[i]) > 0) {
                madeChanges = true;
                AffixMatcher temp = std::move(fAffixMatchers[i - 1]);
                fAffixMatchers[i - 1] = std::move(fAffixMatchers[i]);
                fAffixMatchers[i] = std::move(temp);
            }
        }
    } while (madeChanges);

    for (int32_t i = 0; i < numAffixMatchers; i++) {
        // Enable the following line to debug affixes
        //std::cout << "Adding affix matcher: " << CStr(fAffixMatchers[i].toString())() << std::endl;
        output.addMatcher(fAffixMatchers[i]);
    }
}


AffixMatcher::AffixMatcher(AffixPatternMatcher* prefix, AffixPatternMatcher* suffix, result_flags_t flags)
        : fPrefix(prefix), fSuffix(suffix), fFlags(flags) {}

bool AffixMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const {
    if (!result.seenNumber()) {
        // Prefix
        // Do not match if:
        // 1. We have already seen a prefix (result.prefix != null)
        // 2. The prefix in this AffixMatcher is empty (prefix == null)
        if (!result.prefix.isBogus() || fPrefix == nullptr) {
            return false;
        }

        // Attempt to match the prefix.
        int initialOffset = segment.getOffset();
        bool maybeMore = fPrefix->match(segment, result, status);
        if (initialOffset != segment.getOffset()) {
            result.prefix = fPrefix->getPattern();
        }
        return maybeMore;

    } else {
        // Suffix
        // Do not match if:
        // 1. We have already seen a suffix (result.suffix != null)
        // 2. The suffix in this AffixMatcher is empty (suffix == null)
        // 3. The matched prefix does not equal this AffixMatcher's prefix
        if (!result.suffix.isBogus() || fSuffix == nullptr || !matched(fPrefix, result.prefix)) {
            return false;
        }

        // Attempt to match the suffix.
        int initialOffset = segment.getOffset();
        bool maybeMore = fSuffix->match(segment, result, status);
        if (initialOffset != segment.getOffset()) {
            result.suffix = fSuffix->getPattern();
        }
        return maybeMore;
    }
}

bool AffixMatcher::smokeTest(const StringSegment& segment) const {
    return (fPrefix != nullptr && fPrefix->smokeTest(segment)) ||
           (fSuffix != nullptr && fSuffix->smokeTest(segment));
}

void AffixMatcher::postProcess(ParsedNumber& result) const {
    // Check to see if our affix is the one that was matched. If so, set the flags in the result.
    if (matched(fPrefix, result.prefix) && matched(fSuffix, result.suffix)) {
        // Fill in the result prefix and suffix with non-null values (empty string).
        // Used by strict mode to determine whether an entire affix pair was matched.
        if (result.prefix.isBogus()) {
            result.prefix = UnicodeString();
        }
        if (result.suffix.isBogus()) {
            result.suffix = UnicodeString();
        }
        result.flags |= fFlags;
        if (fPrefix != nullptr) {
            fPrefix->postProcess(result);
        }
        if (fSuffix != nullptr) {
            fSuffix->postProcess(result);
        }
    }
}

int8_t AffixMatcher::compareTo(const AffixMatcher& rhs) const {
    const AffixMatcher& lhs = *this;
    if (length(lhs.fPrefix) != length(rhs.fPrefix)) {
        return length(lhs.fPrefix) > length(rhs.fPrefix) ? -1 : 1;
    } else if (length(lhs.fSuffix) != length(rhs.fSuffix)) {
        return length(lhs.fSuffix) > length(rhs.fSuffix) ? -1 : 1;
    } else {
        return 0;
    }
}

UnicodeString AffixMatcher::toString() const {
    bool isNegative = 0 != (fFlags & FLAG_NEGATIVE);
    return UnicodeString(u"<Affix") + (isNegative ? u":negative " : u" ") +
           (fPrefix ? fPrefix->getPattern() : u"null") + u"#" +
           (fSuffix ? fSuffix->getPattern() : u"null") + u">";

}


#endif /* #if !UCONFIG_NO_FORMATTING */