summaryrefslogtreecommitdiffstats
path: root/contrib/libs/icu/common/normalizer2impl.cpp
diff options
context:
space:
mode:
authorneksard <[email protected]>2022-02-10 16:45:33 +0300
committerDaniil Cherednik <[email protected]>2022-02-10 16:45:33 +0300
commit1d9c550e7c38e051d7961f576013a482003a70d9 (patch)
treeb2cc84ee7850122e7ccf51d0ea21e4fa7e7a5685 /contrib/libs/icu/common/normalizer2impl.cpp
parent8f7cf138264e0caa318144bf8a2c950e0b0a8593 (diff)
Restoring authorship annotation for <[email protected]>. Commit 2 of 2.
Diffstat (limited to 'contrib/libs/icu/common/normalizer2impl.cpp')
-rw-r--r--contrib/libs/icu/common/normalizer2impl.cpp2834
1 files changed, 1417 insertions, 1417 deletions
diff --git a/contrib/libs/icu/common/normalizer2impl.cpp b/contrib/libs/icu/common/normalizer2impl.cpp
index 48d587b2227..cbf6b4d9804 100644
--- a/contrib/libs/icu/common/normalizer2impl.cpp
+++ b/contrib/libs/icu/common/normalizer2impl.cpp
@@ -1,49 +1,49 @@
// © 2016 and later: Unicode, Inc. and others.
-// License & terms of use: http://www.unicode.org/copyright.html
-/*
-*******************************************************************************
-*
-* Copyright (C) 2009-2014, International Business Machines
-* Corporation and others. All Rights Reserved.
-*
-*******************************************************************************
-* file name: normalizer2impl.cpp
+// License & terms of use: http://www.unicode.org/copyright.html
+/*
+*******************************************************************************
+*
+* Copyright (C) 2009-2014, International Business Machines
+* Corporation and others. All Rights Reserved.
+*
+*******************************************************************************
+* file name: normalizer2impl.cpp
* encoding: UTF-8
-* tab size: 8 (not used)
-* indentation:4
-*
-* created on: 2009nov22
-* created by: Markus W. Scherer
-*/
-
+* tab size: 8 (not used)
+* indentation:4
+*
+* created on: 2009nov22
+* created by: Markus W. Scherer
+*/
+
// #define UCPTRIE_DEBUG
-#include "unicode/utypes.h"
-
-#if !UCONFIG_NO_NORMALIZATION
-
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_NORMALIZATION
+
#include "unicode/bytestream.h"
#include "unicode/edits.h"
-#include "unicode/normalizer2.h"
+#include "unicode/normalizer2.h"
#include "unicode/stringoptions.h"
#include "unicode/ucptrie.h"
-#include "unicode/udata.h"
+#include "unicode/udata.h"
#include "unicode/umutablecptrie.h"
-#include "unicode/ustring.h"
-#include "unicode/utf16.h"
+#include "unicode/ustring.h"
+#include "unicode/utf16.h"
#include "unicode/utf8.h"
#include "bytesinkutil.h"
-#include "cmemory.h"
-#include "mutex.h"
-#include "normalizer2impl.h"
-#include "putilimp.h"
-#include "uassert.h"
+#include "cmemory.h"
+#include "mutex.h"
+#include "normalizer2impl.h"
+#include "putilimp.h"
+#include "uassert.h"
#include "ucptrie_impl.h"
-#include "uset_imp.h"
-#include "uvector.h"
-
-U_NAMESPACE_BEGIN
-
+#include "uset_imp.h"
+#include "uvector.h"
+
+U_NAMESPACE_BEGIN
+
namespace {
/**
@@ -166,8 +166,8 @@ appendCodePointDelta(const uint8_t *cpStart, const uint8_t *cpLimit, int32_t del
} // namespace
-// ReorderingBuffer -------------------------------------------------------- ***
-
+// ReorderingBuffer -------------------------------------------------------- ***
+
ReorderingBuffer::ReorderingBuffer(const Normalizer2Impl &ni, UnicodeString &dest,
UErrorCode &errorCode) :
impl(ni), str(dest),
@@ -179,38 +179,38 @@ ReorderingBuffer::ReorderingBuffer(const Normalizer2Impl &ni, UnicodeString &des
}
}
-UBool ReorderingBuffer::init(int32_t destCapacity, UErrorCode &errorCode) {
- int32_t length=str.length();
- start=str.getBuffer(destCapacity);
- if(start==NULL) {
- // getBuffer() already did str.setToBogus()
- errorCode=U_MEMORY_ALLOCATION_ERROR;
- return FALSE;
- }
- limit=start+length;
- remainingCapacity=str.getCapacity()-length;
- reorderStart=start;
- if(start==limit) {
- lastCC=0;
- } else {
- setIterator();
- lastCC=previousCC();
- // Set reorderStart after the last code point with cc<=1 if there is one.
- if(lastCC>1) {
- while(previousCC()>1) {}
- }
- reorderStart=codePointLimit;
- }
- return TRUE;
-}
-
-UBool ReorderingBuffer::equals(const UChar *otherStart, const UChar *otherLimit) const {
- int32_t length=(int32_t)(limit-start);
- return
- length==(int32_t)(otherLimit-otherStart) &&
- 0==u_memcmp(start, otherStart, length);
-}
-
+UBool ReorderingBuffer::init(int32_t destCapacity, UErrorCode &errorCode) {
+ int32_t length=str.length();
+ start=str.getBuffer(destCapacity);
+ if(start==NULL) {
+ // getBuffer() already did str.setToBogus()
+ errorCode=U_MEMORY_ALLOCATION_ERROR;
+ return FALSE;
+ }
+ limit=start+length;
+ remainingCapacity=str.getCapacity()-length;
+ reorderStart=start;
+ if(start==limit) {
+ lastCC=0;
+ } else {
+ setIterator();
+ lastCC=previousCC();
+ // Set reorderStart after the last code point with cc<=1 if there is one.
+ if(lastCC>1) {
+ while(previousCC()>1) {}
+ }
+ reorderStart=codePointLimit;
+ }
+ return TRUE;
+}
+
+UBool ReorderingBuffer::equals(const UChar *otherStart, const UChar *otherLimit) const {
+ int32_t length=(int32_t)(limit-start);
+ return
+ length==(int32_t)(otherLimit-otherStart) &&
+ 0==u_memcmp(start, otherStart, length);
+}
+
UBool ReorderingBuffer::equals(const uint8_t *otherStart, const uint8_t *otherLimit) const {
U_ASSERT((otherLimit - otherStart) <= INT32_MAX); // ensured by caller
int32_t length = (int32_t)(limit - start);
@@ -237,202 +237,202 @@ UBool ReorderingBuffer::equals(const uint8_t *otherStart, const uint8_t *otherLi
}
}
-UBool ReorderingBuffer::appendSupplementary(UChar32 c, uint8_t cc, UErrorCode &errorCode) {
- if(remainingCapacity<2 && !resize(2, errorCode)) {
- return FALSE;
- }
- if(lastCC<=cc || cc==0) {
- limit[0]=U16_LEAD(c);
- limit[1]=U16_TRAIL(c);
- limit+=2;
- lastCC=cc;
- if(cc<=1) {
- reorderStart=limit;
- }
- } else {
- insert(c, cc);
- }
- remainingCapacity-=2;
- return TRUE;
-}
-
+UBool ReorderingBuffer::appendSupplementary(UChar32 c, uint8_t cc, UErrorCode &errorCode) {
+ if(remainingCapacity<2 && !resize(2, errorCode)) {
+ return FALSE;
+ }
+ if(lastCC<=cc || cc==0) {
+ limit[0]=U16_LEAD(c);
+ limit[1]=U16_TRAIL(c);
+ limit+=2;
+ lastCC=cc;
+ if(cc<=1) {
+ reorderStart=limit;
+ }
+ } else {
+ insert(c, cc);
+ }
+ remainingCapacity-=2;
+ return TRUE;
+}
+
UBool ReorderingBuffer::append(const UChar *s, int32_t length, UBool isNFD,
- uint8_t leadCC, uint8_t trailCC,
- UErrorCode &errorCode) {
- if(length==0) {
- return TRUE;
- }
- if(remainingCapacity<length && !resize(length, errorCode)) {
- return FALSE;
- }
- remainingCapacity-=length;
- if(lastCC<=leadCC || leadCC==0) {
- if(trailCC<=1) {
- reorderStart=limit+length;
- } else if(leadCC<=1) {
- reorderStart=limit+1; // Ok if not a code point boundary.
- }
- const UChar *sLimit=s+length;
- do { *limit++=*s++; } while(s!=sLimit);
- lastCC=trailCC;
- } else {
- int32_t i=0;
- UChar32 c;
- U16_NEXT(s, i, length, c);
- insert(c, leadCC); // insert first code point
- while(i<length) {
- U16_NEXT(s, i, length, c);
- if(i<length) {
+ uint8_t leadCC, uint8_t trailCC,
+ UErrorCode &errorCode) {
+ if(length==0) {
+ return TRUE;
+ }
+ if(remainingCapacity<length && !resize(length, errorCode)) {
+ return FALSE;
+ }
+ remainingCapacity-=length;
+ if(lastCC<=leadCC || leadCC==0) {
+ if(trailCC<=1) {
+ reorderStart=limit+length;
+ } else if(leadCC<=1) {
+ reorderStart=limit+1; // Ok if not a code point boundary.
+ }
+ const UChar *sLimit=s+length;
+ do { *limit++=*s++; } while(s!=sLimit);
+ lastCC=trailCC;
+ } else {
+ int32_t i=0;
+ UChar32 c;
+ U16_NEXT(s, i, length, c);
+ insert(c, leadCC); // insert first code point
+ while(i<length) {
+ U16_NEXT(s, i, length, c);
+ if(i<length) {
if (isNFD) {
leadCC = Normalizer2Impl::getCCFromYesOrMaybe(impl.getRawNorm16(c));
} else {
leadCC = impl.getCC(impl.getNorm16(c));
}
- } else {
- leadCC=trailCC;
- }
- append(c, leadCC, errorCode);
- }
- }
- return TRUE;
-}
-
-UBool ReorderingBuffer::appendZeroCC(UChar32 c, UErrorCode &errorCode) {
- int32_t cpLength=U16_LENGTH(c);
- if(remainingCapacity<cpLength && !resize(cpLength, errorCode)) {
- return FALSE;
- }
- remainingCapacity-=cpLength;
- if(cpLength==1) {
- *limit++=(UChar)c;
- } else {
- limit[0]=U16_LEAD(c);
- limit[1]=U16_TRAIL(c);
- limit+=2;
- }
- lastCC=0;
- reorderStart=limit;
- return TRUE;
-}
-
-UBool ReorderingBuffer::appendZeroCC(const UChar *s, const UChar *sLimit, UErrorCode &errorCode) {
- if(s==sLimit) {
- return TRUE;
- }
- int32_t length=(int32_t)(sLimit-s);
- if(remainingCapacity<length && !resize(length, errorCode)) {
- return FALSE;
- }
- u_memcpy(limit, s, length);
- limit+=length;
- remainingCapacity-=length;
- lastCC=0;
- reorderStart=limit;
- return TRUE;
-}
-
-void ReorderingBuffer::remove() {
- reorderStart=limit=start;
- remainingCapacity=str.getCapacity();
- lastCC=0;
-}
-
-void ReorderingBuffer::removeSuffix(int32_t suffixLength) {
- if(suffixLength<(limit-start)) {
- limit-=suffixLength;
- remainingCapacity+=suffixLength;
- } else {
- limit=start;
- remainingCapacity=str.getCapacity();
- }
- lastCC=0;
- reorderStart=limit;
-}
-
-UBool ReorderingBuffer::resize(int32_t appendLength, UErrorCode &errorCode) {
- int32_t reorderStartIndex=(int32_t)(reorderStart-start);
- int32_t length=(int32_t)(limit-start);
- str.releaseBuffer(length);
- int32_t newCapacity=length+appendLength;
- int32_t doubleCapacity=2*str.getCapacity();
- if(newCapacity<doubleCapacity) {
- newCapacity=doubleCapacity;
- }
- if(newCapacity<256) {
- newCapacity=256;
- }
- start=str.getBuffer(newCapacity);
- if(start==NULL) {
- // getBuffer() already did str.setToBogus()
- errorCode=U_MEMORY_ALLOCATION_ERROR;
- return FALSE;
- }
- reorderStart=start+reorderStartIndex;
- limit=start+length;
- remainingCapacity=str.getCapacity()-length;
- return TRUE;
-}
-
-void ReorderingBuffer::skipPrevious() {
- codePointLimit=codePointStart;
- UChar c=*--codePointStart;
- if(U16_IS_TRAIL(c) && start<codePointStart && U16_IS_LEAD(*(codePointStart-1))) {
- --codePointStart;
- }
-}
-
-uint8_t ReorderingBuffer::previousCC() {
- codePointLimit=codePointStart;
- if(reorderStart>=codePointStart) {
- return 0;
- }
- UChar32 c=*--codePointStart;
- UChar c2;
- if(U16_IS_TRAIL(c) && start<codePointStart && U16_IS_LEAD(c2=*(codePointStart-1))) {
- --codePointStart;
- c=U16_GET_SUPPLEMENTARY(c2, c);
- }
+ } else {
+ leadCC=trailCC;
+ }
+ append(c, leadCC, errorCode);
+ }
+ }
+ return TRUE;
+}
+
+UBool ReorderingBuffer::appendZeroCC(UChar32 c, UErrorCode &errorCode) {
+ int32_t cpLength=U16_LENGTH(c);
+ if(remainingCapacity<cpLength && !resize(cpLength, errorCode)) {
+ return FALSE;
+ }
+ remainingCapacity-=cpLength;
+ if(cpLength==1) {
+ *limit++=(UChar)c;
+ } else {
+ limit[0]=U16_LEAD(c);
+ limit[1]=U16_TRAIL(c);
+ limit+=2;
+ }
+ lastCC=0;
+ reorderStart=limit;
+ return TRUE;
+}
+
+UBool ReorderingBuffer::appendZeroCC(const UChar *s, const UChar *sLimit, UErrorCode &errorCode) {
+ if(s==sLimit) {
+ return TRUE;
+ }
+ int32_t length=(int32_t)(sLimit-s);
+ if(remainingCapacity<length && !resize(length, errorCode)) {
+ return FALSE;
+ }
+ u_memcpy(limit, s, length);
+ limit+=length;
+ remainingCapacity-=length;
+ lastCC=0;
+ reorderStart=limit;
+ return TRUE;
+}
+
+void ReorderingBuffer::remove() {
+ reorderStart=limit=start;
+ remainingCapacity=str.getCapacity();
+ lastCC=0;
+}
+
+void ReorderingBuffer::removeSuffix(int32_t suffixLength) {
+ if(suffixLength<(limit-start)) {
+ limit-=suffixLength;
+ remainingCapacity+=suffixLength;
+ } else {
+ limit=start;
+ remainingCapacity=str.getCapacity();
+ }
+ lastCC=0;
+ reorderStart=limit;
+}
+
+UBool ReorderingBuffer::resize(int32_t appendLength, UErrorCode &errorCode) {
+ int32_t reorderStartIndex=(int32_t)(reorderStart-start);
+ int32_t length=(int32_t)(limit-start);
+ str.releaseBuffer(length);
+ int32_t newCapacity=length+appendLength;
+ int32_t doubleCapacity=2*str.getCapacity();
+ if(newCapacity<doubleCapacity) {
+ newCapacity=doubleCapacity;
+ }
+ if(newCapacity<256) {
+ newCapacity=256;
+ }
+ start=str.getBuffer(newCapacity);
+ if(start==NULL) {
+ // getBuffer() already did str.setToBogus()
+ errorCode=U_MEMORY_ALLOCATION_ERROR;
+ return FALSE;
+ }
+ reorderStart=start+reorderStartIndex;
+ limit=start+length;
+ remainingCapacity=str.getCapacity()-length;
+ return TRUE;
+}
+
+void ReorderingBuffer::skipPrevious() {
+ codePointLimit=codePointStart;
+ UChar c=*--codePointStart;
+ if(U16_IS_TRAIL(c) && start<codePointStart && U16_IS_LEAD(*(codePointStart-1))) {
+ --codePointStart;
+ }
+}
+
+uint8_t ReorderingBuffer::previousCC() {
+ codePointLimit=codePointStart;
+ if(reorderStart>=codePointStart) {
+ return 0;
+ }
+ UChar32 c=*--codePointStart;
+ UChar c2;
+ if(U16_IS_TRAIL(c) && start<codePointStart && U16_IS_LEAD(c2=*(codePointStart-1))) {
+ --codePointStart;
+ c=U16_GET_SUPPLEMENTARY(c2, c);
+ }
return impl.getCCFromYesOrMaybeCP(c);
-}
-
-// Inserts c somewhere before the last character.
-// Requires 0<cc<lastCC which implies reorderStart<limit.
-void ReorderingBuffer::insert(UChar32 c, uint8_t cc) {
- for(setIterator(), skipPrevious(); previousCC()>cc;) {}
- // insert c at codePointLimit, after the character with prevCC<=cc
- UChar *q=limit;
- UChar *r=limit+=U16_LENGTH(c);
- do {
- *--r=*--q;
- } while(codePointLimit!=q);
- writeCodePoint(q, c);
- if(cc<=1) {
- reorderStart=r;
- }
-}
-
-// Normalizer2Impl --------------------------------------------------------- ***
-
-struct CanonIterData : public UMemory {
- CanonIterData(UErrorCode &errorCode);
- ~CanonIterData();
- void addToStartSet(UChar32 origin, UChar32 decompLead, UErrorCode &errorCode);
+}
+
+// Inserts c somewhere before the last character.
+// Requires 0<cc<lastCC which implies reorderStart<limit.
+void ReorderingBuffer::insert(UChar32 c, uint8_t cc) {
+ for(setIterator(), skipPrevious(); previousCC()>cc;) {}
+ // insert c at codePointLimit, after the character with prevCC<=cc
+ UChar *q=limit;
+ UChar *r=limit+=U16_LENGTH(c);
+ do {
+ *--r=*--q;
+ } while(codePointLimit!=q);
+ writeCodePoint(q, c);
+ if(cc<=1) {
+ reorderStart=r;
+ }
+}
+
+// Normalizer2Impl --------------------------------------------------------- ***
+
+struct CanonIterData : public UMemory {
+ CanonIterData(UErrorCode &errorCode);
+ ~CanonIterData();
+ void addToStartSet(UChar32 origin, UChar32 decompLead, UErrorCode &errorCode);
UMutableCPTrie *mutableTrie;
UCPTrie *trie;
- UVector canonStartSets; // contains UnicodeSet *
-};
-
-Normalizer2Impl::~Normalizer2Impl() {
- delete fCanonIterData;
-}
-
-void
+ UVector canonStartSets; // contains UnicodeSet *
+};
+
+Normalizer2Impl::~Normalizer2Impl() {
+ delete fCanonIterData;
+}
+
+void
Normalizer2Impl::init(const int32_t *inIndexes, const UCPTrie *inTrie,
- const uint16_t *inExtraData, const uint8_t *inSmallFCD) {
+ const uint16_t *inExtraData, const uint8_t *inSmallFCD) {
minDecompNoCP = static_cast<UChar>(inIndexes[IX_MIN_DECOMP_NO_CP]);
minCompNoMaybeCP = static_cast<UChar>(inIndexes[IX_MIN_COMP_NO_MAYBE_CP]);
minLcccCP = static_cast<UChar>(inIndexes[IX_MIN_LCCC_CP]);
-
+
minYesNo = static_cast<uint16_t>(inIndexes[IX_MIN_YES_NO]);
minYesNoMappingsOnly = static_cast<uint16_t>(inIndexes[IX_MIN_YES_NO_MAPPINGS_ONLY]);
minNoNo = static_cast<uint16_t>(inIndexes[IX_MIN_NO_NO]);
@@ -443,26 +443,26 @@ Normalizer2Impl::init(const int32_t *inIndexes, const UCPTrie *inTrie,
minMaybeYes = static_cast<uint16_t>(inIndexes[IX_MIN_MAYBE_YES]);
U_ASSERT((minMaybeYes & 7) == 0); // 8-aligned for noNoDelta bit fields
centerNoNoDelta = (minMaybeYes >> DELTA_SHIFT) - MAX_DELTA - 1;
-
- normTrie=inTrie;
-
- maybeYesCompositions=inExtraData;
+
+ normTrie=inTrie;
+
+ maybeYesCompositions=inExtraData;
extraData=maybeYesCompositions+((MIN_NORMAL_MAYBE_YES-minMaybeYes)>>OFFSET_SHIFT);
-
- smallFCD=inSmallFCD;
-}
-
-U_CDECL_BEGIN
-
-static uint32_t U_CALLCONV
-segmentStarterMapper(const void * /*context*/, uint32_t value) {
- return value&CANON_NOT_SEGMENT_STARTER;
-}
-
-U_CDECL_END
-
-void
-Normalizer2Impl::addLcccChars(UnicodeSet &set) const {
+
+ smallFCD=inSmallFCD;
+}
+
+U_CDECL_BEGIN
+
+static uint32_t U_CALLCONV
+segmentStarterMapper(const void * /*context*/, uint32_t value) {
+ return value&CANON_NOT_SEGMENT_STARTER;
+}
+
+U_CDECL_END
+
+void
+Normalizer2Impl::addLcccChars(UnicodeSet &set) const {
UChar32 start = 0, end;
uint32_t norm16;
while ((end = ucptrie_getRange(normTrie, start, UCPMAP_RANGE_FIXED_LEAD_SURROGATES, INERT,
@@ -476,10 +476,10 @@ Normalizer2Impl::addLcccChars(UnicodeSet &set) const {
}
start = end + 1;
}
-}
-
-void
-Normalizer2Impl::addPropertyStarts(const USetAdder *sa, UErrorCode & /*errorCode*/) const {
+}
+
+void
+Normalizer2Impl::addPropertyStarts(const USetAdder *sa, UErrorCode & /*errorCode*/) const {
// Add the start code point of each same-value range of the trie.
UChar32 start = 0, end;
uint32_t value;
@@ -501,17 +501,17 @@ Normalizer2Impl::addPropertyStarts(const USetAdder *sa, UErrorCode & /*errorCode
}
start = end + 1;
}
-
- /* add Hangul LV syllables and LV+1 because of skippables */
- for(UChar c=Hangul::HANGUL_BASE; c<Hangul::HANGUL_LIMIT; c+=Hangul::JAMO_T_COUNT) {
- sa->add(sa->set, c);
- sa->add(sa->set, c+1);
- }
- sa->add(sa->set, Hangul::HANGUL_LIMIT); /* add Hangul+1 to continue with other properties */
-}
-
-void
-Normalizer2Impl::addCanonIterPropertyStarts(const USetAdder *sa, UErrorCode &errorCode) const {
+
+ /* add Hangul LV syllables and LV+1 because of skippables */
+ for(UChar c=Hangul::HANGUL_BASE; c<Hangul::HANGUL_LIMIT; c+=Hangul::JAMO_T_COUNT) {
+ sa->add(sa->set, c);
+ sa->add(sa->set, c+1);
+ }
+ sa->add(sa->set, Hangul::HANGUL_LIMIT); /* add Hangul+1 to continue with other properties */
+}
+
+void
+Normalizer2Impl::addCanonIterPropertyStarts(const USetAdder *sa, UErrorCode &errorCode) const {
// Add the start code point of each same-value range of the canonical iterator data trie.
if (!ensureCanonIterData(errorCode)) { return; }
// Currently only used for the SEGMENT_STARTER property.
@@ -521,99 +521,99 @@ Normalizer2Impl::addCanonIterPropertyStarts(const USetAdder *sa, UErrorCode &err
segmentStarterMapper, nullptr, &value)) >= 0) {
sa->add(sa->set, start);
start = end + 1;
- }
-}
-
-const UChar *
-Normalizer2Impl::copyLowPrefixFromNulTerminated(const UChar *src,
- UChar32 minNeedDataCP,
- ReorderingBuffer *buffer,
- UErrorCode &errorCode) const {
- // Make some effort to support NUL-terminated strings reasonably.
- // Take the part of the fast quick check loop that does not look up
- // data and check the first part of the string.
- // After this prefix, determine the string length to simplify the rest
- // of the code.
- const UChar *prevSrc=src;
- UChar c;
- while((c=*src++)<minNeedDataCP && c!=0) {}
- // Back out the last character for full processing.
- // Copy this prefix.
- if(--src!=prevSrc) {
- if(buffer!=NULL) {
- buffer->appendZeroCC(prevSrc, src, errorCode);
- }
- }
- return src;
-}
-
-UnicodeString &
-Normalizer2Impl::decompose(const UnicodeString &src, UnicodeString &dest,
- UErrorCode &errorCode) const {
- if(U_FAILURE(errorCode)) {
- dest.setToBogus();
- return dest;
- }
- const UChar *sArray=src.getBuffer();
- if(&dest==&src || sArray==NULL) {
- errorCode=U_ILLEGAL_ARGUMENT_ERROR;
- dest.setToBogus();
- return dest;
- }
- decompose(sArray, sArray+src.length(), dest, src.length(), errorCode);
- return dest;
-}
-
-void
-Normalizer2Impl::decompose(const UChar *src, const UChar *limit,
- UnicodeString &dest,
- int32_t destLengthEstimate,
- UErrorCode &errorCode) const {
- if(destLengthEstimate<0 && limit!=NULL) {
- destLengthEstimate=(int32_t)(limit-src);
- }
- dest.remove();
- ReorderingBuffer buffer(*this, dest);
- if(buffer.init(destLengthEstimate, errorCode)) {
- decompose(src, limit, &buffer, errorCode);
- }
-}
-
-// Dual functionality:
-// buffer!=NULL: normalize
-// buffer==NULL: isNormalized/spanQuickCheckYes
-const UChar *
-Normalizer2Impl::decompose(const UChar *src, const UChar *limit,
- ReorderingBuffer *buffer,
- UErrorCode &errorCode) const {
- UChar32 minNoCP=minDecompNoCP;
- if(limit==NULL) {
- src=copyLowPrefixFromNulTerminated(src, minNoCP, buffer, errorCode);
- if(U_FAILURE(errorCode)) {
- return src;
- }
- limit=u_strchr(src, 0);
- }
-
- const UChar *prevSrc;
- UChar32 c=0;
- uint16_t norm16=0;
-
- // only for quick check
- const UChar *prevBoundary=src;
- uint8_t prevCC=0;
-
- for(;;) {
- // count code units below the minimum or with irrelevant data for the quick check
- for(prevSrc=src; src!=limit;) {
- if( (c=*src)<minNoCP ||
+ }
+}
+
+const UChar *
+Normalizer2Impl::copyLowPrefixFromNulTerminated(const UChar *src,
+ UChar32 minNeedDataCP,
+ ReorderingBuffer *buffer,
+ UErrorCode &errorCode) const {
+ // Make some effort to support NUL-terminated strings reasonably.
+ // Take the part of the fast quick check loop that does not look up
+ // data and check the first part of the string.
+ // After this prefix, determine the string length to simplify the rest
+ // of the code.
+ const UChar *prevSrc=src;
+ UChar c;
+ while((c=*src++)<minNeedDataCP && c!=0) {}
+ // Back out the last character for full processing.
+ // Copy this prefix.
+ if(--src!=prevSrc) {
+ if(buffer!=NULL) {
+ buffer->appendZeroCC(prevSrc, src, errorCode);
+ }
+ }
+ return src;
+}
+
+UnicodeString &
+Normalizer2Impl::decompose(const UnicodeString &src, UnicodeString &dest,
+ UErrorCode &errorCode) const {
+ if(U_FAILURE(errorCode)) {
+ dest.setToBogus();
+ return dest;
+ }
+ const UChar *sArray=src.getBuffer();
+ if(&dest==&src || sArray==NULL) {
+ errorCode=U_ILLEGAL_ARGUMENT_ERROR;
+ dest.setToBogus();
+ return dest;
+ }
+ decompose(sArray, sArray+src.length(), dest, src.length(), errorCode);
+ return dest;
+}
+
+void
+Normalizer2Impl::decompose(const UChar *src, const UChar *limit,
+ UnicodeString &dest,
+ int32_t destLengthEstimate,
+ UErrorCode &errorCode) const {
+ if(destLengthEstimate<0 && limit!=NULL) {
+ destLengthEstimate=(int32_t)(limit-src);
+ }
+ dest.remove();
+ ReorderingBuffer buffer(*this, dest);
+ if(buffer.init(destLengthEstimate, errorCode)) {
+ decompose(src, limit, &buffer, errorCode);
+ }
+}
+
+// Dual functionality:
+// buffer!=NULL: normalize
+// buffer==NULL: isNormalized/spanQuickCheckYes
+const UChar *
+Normalizer2Impl::decompose(const UChar *src, const UChar *limit,
+ ReorderingBuffer *buffer,
+ UErrorCode &errorCode) const {
+ UChar32 minNoCP=minDecompNoCP;
+ if(limit==NULL) {
+ src=copyLowPrefixFromNulTerminated(src, minNoCP, buffer, errorCode);
+ if(U_FAILURE(errorCode)) {
+ return src;
+ }
+ limit=u_strchr(src, 0);
+ }
+
+ const UChar *prevSrc;
+ UChar32 c=0;
+ uint16_t norm16=0;
+
+ // only for quick check
+ const UChar *prevBoundary=src;
+ uint8_t prevCC=0;
+
+ for(;;) {
+ // count code units below the minimum or with irrelevant data for the quick check
+ for(prevSrc=src; src!=limit;) {
+ if( (c=*src)<minNoCP ||
isMostDecompYesAndZeroCC(norm16=UCPTRIE_FAST_BMP_GET(normTrie, UCPTRIE_16, c))
- ) {
- ++src;
+ ) {
+ ++src;
} else if(!U16_IS_LEAD(c)) {
- break;
- } else {
- UChar c2;
+ break;
+ } else {
+ UChar c2;
if((src+1)!=limit && U16_IS_TRAIL(c2=src[1])) {
c=U16_GET_SUPPLEMENTARY(c, c2);
norm16=UCPTRIE_FAST_SUPP_GET(normTrie, UCPTRIE_16, c);
@@ -621,54 +621,54 @@ Normalizer2Impl::decompose(const UChar *src, const UChar *limit,
src+=2;
} else {
break;
- }
- } else {
+ }
+ } else {
++src; // unpaired lead surrogate: inert
- }
- }
- }
- // copy these code units all at once
- if(src!=prevSrc) {
- if(buffer!=NULL) {
- if(!buffer->appendZeroCC(prevSrc, src, errorCode)) {
- break;
- }
- } else {
- prevCC=0;
- prevBoundary=src;
- }
- }
- if(src==limit) {
- break;
- }
-
- // Check one above-minimum, relevant code point.
- src+=U16_LENGTH(c);
- if(buffer!=NULL) {
- if(!decompose(c, norm16, *buffer, errorCode)) {
- break;
- }
- } else {
- if(isDecompYes(norm16)) {
- uint8_t cc=getCCFromYesOrMaybe(norm16);
- if(prevCC<=cc || cc==0) {
- prevCC=cc;
- if(cc<=1) {
- prevBoundary=src;
- }
- continue;
- }
- }
- return prevBoundary; // "no" or cc out of order
- }
- }
- return src;
-}
-
-// Decompose a short piece of text which is likely to contain characters that
-// fail the quick check loop and/or where the quick check loop's overhead
-// is unlikely to be amortized.
-// Called by the compose() and makeFCD() implementations.
+ }
+ }
+ }
+ // copy these code units all at once
+ if(src!=prevSrc) {
+ if(buffer!=NULL) {
+ if(!buffer->appendZeroCC(prevSrc, src, errorCode)) {
+ break;
+ }
+ } else {
+ prevCC=0;
+ prevBoundary=src;
+ }
+ }
+ if(src==limit) {
+ break;
+ }
+
+ // Check one above-minimum, relevant code point.
+ src+=U16_LENGTH(c);
+ if(buffer!=NULL) {
+ if(!decompose(c, norm16, *buffer, errorCode)) {
+ break;
+ }
+ } else {
+ if(isDecompYes(norm16)) {
+ uint8_t cc=getCCFromYesOrMaybe(norm16);
+ if(prevCC<=cc || cc==0) {
+ prevCC=cc;
+ if(cc<=1) {
+ prevBoundary=src;
+ }
+ continue;
+ }
+ }
+ return prevBoundary; // "no" or cc out of order
+ }
+ }
+ return src;
+}
+
+// Decompose a short piece of text which is likely to contain characters that
+// fail the quick check loop and/or where the quick check loop's overhead
+// is unlikely to be amortized.
+// Called by the compose() and makeFCD() implementations.
const UChar *
Normalizer2Impl::decomposeShort(const UChar *src, const UChar *limit,
UBool stopAtCompBoundary, UBool onlyContiguous,
@@ -676,34 +676,34 @@ Normalizer2Impl::decomposeShort(const UChar *src, const UChar *limit,
if (U_FAILURE(errorCode)) {
return nullptr;
}
- while(src<limit) {
+ while(src<limit) {
if (stopAtCompBoundary && *src < minCompNoMaybeCP) {
return src;
}
const UChar *prevSrc = src;
- UChar32 c;
- uint16_t norm16;
+ UChar32 c;
+ uint16_t norm16;
UCPTRIE_FAST_U16_NEXT(normTrie, UCPTRIE_16, src, limit, c, norm16);
if (stopAtCompBoundary && norm16HasCompBoundaryBefore(norm16)) {
return prevSrc;
}
- if(!decompose(c, norm16, buffer, errorCode)) {
+ if(!decompose(c, norm16, buffer, errorCode)) {
return nullptr;
- }
+ }
if (stopAtCompBoundary && norm16HasCompBoundaryAfter(norm16, onlyContiguous)) {
return src;
}
- }
+ }
return src;
-}
-
-UBool Normalizer2Impl::decompose(UChar32 c, uint16_t norm16,
- ReorderingBuffer &buffer,
- UErrorCode &errorCode) const {
+}
+
+UBool Normalizer2Impl::decompose(UChar32 c, uint16_t norm16,
+ ReorderingBuffer &buffer,
+ UErrorCode &errorCode) const {
// get the decomposition and the lead and trail cc's
if (norm16 >= limitNoNo) {
if (isMaybeOrNonZeroCC(norm16)) {
- return buffer.append(c, getCCFromYesOrMaybe(norm16), errorCode);
+ return buffer.append(c, getCCFromYesOrMaybe(norm16), errorCode);
}
// Maps to an isCompYesAndZeroCC.
c=mapAlgorithmic(c, norm16);
@@ -778,7 +778,7 @@ Normalizer2Impl::decomposeShort(const uint8_t *src, const uint8_t *limit,
return nullptr;
}
} else if (isHangulLV(norm16) || isHangulLVT(norm16)) {
- // Hangul syllable: decompose algorithmically
+ // Hangul syllable: decompose algorithmically
if (c < 0) {
c = codePointFromValidUTF8(prevSrc, src);
}
@@ -786,7 +786,7 @@ Normalizer2Impl::decomposeShort(const uint8_t *src, const uint8_t *limit,
if (!buffer.appendZeroCC(jamos, jamos+Hangul::decompose(c, jamos), errorCode)) {
return nullptr;
}
- } else {
+ } else {
// The character decomposes, get everything from the variable-length extra data.
const uint16_t *mapping = getMapping(norm16);
uint16_t firstUnit = *mapping;
@@ -795,27 +795,27 @@ Normalizer2Impl::decomposeShort(const uint8_t *src, const uint8_t *limit,
uint8_t leadCC;
if (firstUnit & MAPPING_HAS_CCC_LCCC_WORD) {
leadCC = (uint8_t)(*(mapping-1) >> 8);
- } else {
+ } else {
leadCC = 0;
- }
+ }
if (!buffer.append((const char16_t *)mapping+1, length, TRUE, leadCC, trailCC, errorCode)) {
return nullptr;
}
- }
+ }
if (stopAtCompBoundary && norm16HasCompBoundaryAfter(norm16, onlyContiguous)) {
return src;
}
- }
+ }
return src;
-}
-
-const UChar *
-Normalizer2Impl::getDecomposition(UChar32 c, UChar buffer[4], int32_t &length) const {
- uint16_t norm16;
+}
+
+const UChar *
+Normalizer2Impl::getDecomposition(UChar32 c, UChar buffer[4], int32_t &length) const {
+ uint16_t norm16;
if(c<minDecompNoCP || isMaybeOrNonZeroCC(norm16=getNorm16(c))) {
// c does not decompose
return nullptr;
- }
+ }
const UChar *decomp = nullptr;
if(isDecompNoAlgorithmic(norm16)) {
// Maps to an isCompYesAndZeroCC.
@@ -837,28 +837,28 @@ Normalizer2Impl::getDecomposition(UChar32 c, UChar buffer[4], int32_t &length) c
const uint16_t *mapping=getMapping(norm16);
length=*mapping&MAPPING_LENGTH_MASK;
return (const UChar *)mapping+1;
-}
-
-// The capacity of the buffer must be 30=MAPPING_LENGTH_MASK-1
-// so that a raw mapping fits that consists of one unit ("rm0")
-// plus all but the first two code units of the normal mapping.
-// The maximum length of a normal mapping is 31=MAPPING_LENGTH_MASK.
-const UChar *
-Normalizer2Impl::getRawDecomposition(UChar32 c, UChar buffer[30], int32_t &length) const {
- uint16_t norm16;
- if(c<minDecompNoCP || isDecompYes(norm16=getNorm16(c))) {
- // c does not decompose
- return NULL;
+}
+
+// The capacity of the buffer must be 30=MAPPING_LENGTH_MASK-1
+// so that a raw mapping fits that consists of one unit ("rm0")
+// plus all but the first two code units of the normal mapping.
+// The maximum length of a normal mapping is 31=MAPPING_LENGTH_MASK.
+const UChar *
+Normalizer2Impl::getRawDecomposition(UChar32 c, UChar buffer[30], int32_t &length) const {
+ uint16_t norm16;
+ if(c<minDecompNoCP || isDecompYes(norm16=getNorm16(c))) {
+ // c does not decompose
+ return NULL;
} else if(isHangulLV(norm16) || isHangulLVT(norm16)) {
- // Hangul syllable: decompose algorithmically
- Hangul::getRawDecomposition(c, buffer);
- length=2;
- return buffer;
- } else if(isDecompNoAlgorithmic(norm16)) {
- c=mapAlgorithmic(c, norm16);
- length=0;
- U16_APPEND_UNSAFE(buffer, length, c);
- return buffer;
+ // Hangul syllable: decompose algorithmically
+ Hangul::getRawDecomposition(c, buffer);
+ length=2;
+ return buffer;
+ } else if(isDecompNoAlgorithmic(norm16)) {
+ c=mapAlgorithmic(c, norm16);
+ length=0;
+ U16_APPEND_UNSAFE(buffer, length, c);
+ return buffer;
}
// c decomposes, get everything from the variable-length extra data
const uint16_t *mapping=getMapping(norm16);
@@ -872,30 +872,30 @@ Normalizer2Impl::getRawDecomposition(UChar32 c, UChar buffer[30], int32_t &lengt
if(rm0<=MAPPING_LENGTH_MASK) {
length=rm0;
return (const UChar *)rawMapping-rm0;
- } else {
+ } else {
// Copy the normal mapping and replace its first two code units with rm0.
buffer[0]=(UChar)rm0;
u_memcpy(buffer+1, (const UChar *)mapping+1+2, mLength-2);
length=mLength-1;
return buffer;
- }
+ }
} else {
length=mLength;
return (const UChar *)mapping+1;
- }
-}
-
-void Normalizer2Impl::decomposeAndAppend(const UChar *src, const UChar *limit,
- UBool doDecompose,
- UnicodeString &safeMiddle,
- ReorderingBuffer &buffer,
- UErrorCode &errorCode) const {
- buffer.copyReorderableSuffixTo(safeMiddle);
- if(doDecompose) {
- decompose(src, limit, &buffer, errorCode);
- return;
- }
- // Just merge the strings at the boundary.
+ }
+}
+
+void Normalizer2Impl::decomposeAndAppend(const UChar *src, const UChar *limit,
+ UBool doDecompose,
+ UnicodeString &safeMiddle,
+ ReorderingBuffer &buffer,
+ UErrorCode &errorCode) const {
+ buffer.copyReorderableSuffixTo(safeMiddle);
+ if(doDecompose) {
+ decompose(src, limit, &buffer, errorCode);
+ return;
+ }
+ // Just merge the strings at the boundary.
bool isFirst = true;
uint8_t firstCC = 0, prevCC = 0, cc;
const UChar *p = src;
@@ -914,15 +914,15 @@ void Normalizer2Impl::decomposeAndAppend(const UChar *src, const UChar *limit,
}
prevCC = cc;
}
- if(limit==NULL) { // appendZeroCC() needs limit!=NULL
+ if(limit==NULL) { // appendZeroCC() needs limit!=NULL
limit=u_strchr(p, 0);
- }
-
+ }
+
if (buffer.append(src, (int32_t)(p - src), FALSE, firstCC, prevCC, errorCode)) {
buffer.appendZeroCC(p, limit, errorCode);
- }
-}
-
+ }
+}
+
UBool Normalizer2Impl::hasDecompBoundaryBefore(UChar32 c) const {
return c < minLcccCP || (c <= 0xffff && !singleLeadMightHaveNonZeroFCD16(c)) ||
norm16HasDecompBoundaryBefore(getNorm16(c));
@@ -959,10 +959,10 @@ UBool Normalizer2Impl::norm16HasDecompBoundaryAfter(uint16_t norm16) const {
if (norm16 >= limitNoNo) {
if (isMaybeOrNonZeroCC(norm16)) {
return norm16 <= MIN_NORMAL_MAYBE_YES || norm16 == JAMO_VT;
- }
+ }
// Maps to an isCompYesAndZeroCC.
return (norm16 & DELTA_TCCC_MASK) <= DELTA_TCCC_1;
- }
+ }
// c decomposes, get everything from the variable-length extra data
const uint16_t *mapping=getMapping(norm16);
uint16_t firstUnit=*mapping;
@@ -977,344 +977,344 @@ UBool Normalizer2Impl::norm16HasDecompBoundaryAfter(uint16_t norm16) const {
// if(trailCC==1) test leadCC==0, same as checking for before-boundary
// TRUE if leadCC==0 (hasFCDBoundaryBefore())
return (firstUnit&MAPPING_HAS_CCC_LCCC_WORD)==0 || (*(mapping-1)&0xff00)==0;
-}
-
-/*
- * Finds the recomposition result for
- * a forward-combining "lead" character,
- * specified with a pointer to its compositions list,
- * and a backward-combining "trail" character.
- *
- * If the lead and trail characters combine, then this function returns
- * the following "compositeAndFwd" value:
- * Bits 21..1 composite character
- * Bit 0 set if the composite is a forward-combining starter
- * otherwise it returns -1.
- *
- * The compositions list has (trail, compositeAndFwd) pair entries,
- * encoded as either pairs or triples of 16-bit units.
- * The last entry has the high bit of its first unit set.
- *
- * The list is sorted by ascending trail characters (there are no duplicates).
- * A linear search is used.
- *
- * See normalizer2impl.h for a more detailed description
- * of the compositions list format.
- */
-int32_t Normalizer2Impl::combine(const uint16_t *list, UChar32 trail) {
- uint16_t key1, firstUnit;
- if(trail<COMP_1_TRAIL_LIMIT) {
- // trail character is 0..33FF
- // result entry may have 2 or 3 units
- key1=(uint16_t)(trail<<1);
- while(key1>(firstUnit=*list)) {
- list+=2+(firstUnit&COMP_1_TRIPLE);
- }
- if(key1==(firstUnit&COMP_1_TRAIL_MASK)) {
- if(firstUnit&COMP_1_TRIPLE) {
- return ((int32_t)list[1]<<16)|list[2];
- } else {
- return list[1];
- }
- }
- } else {
- // trail character is 3400..10FFFF
- // result entry has 3 units
- key1=(uint16_t)(COMP_1_TRAIL_LIMIT+
- (((trail>>COMP_1_TRAIL_SHIFT))&
- ~COMP_1_TRIPLE));
- uint16_t key2=(uint16_t)(trail<<COMP_2_TRAIL_SHIFT);
- uint16_t secondUnit;
- for(;;) {
- if(key1>(firstUnit=*list)) {
- list+=2+(firstUnit&COMP_1_TRIPLE);
- } else if(key1==(firstUnit&COMP_1_TRAIL_MASK)) {
- if(key2>(secondUnit=list[1])) {
- if(firstUnit&COMP_1_LAST_TUPLE) {
- break;
- } else {
- list+=3;
- }
- } else if(key2==(secondUnit&COMP_2_TRAIL_MASK)) {
- return ((int32_t)(secondUnit&~COMP_2_TRAIL_MASK)<<16)|list[2];
- } else {
- break;
- }
- } else {
- break;
- }
- }
- }
- return -1;
-}
-
-/**
- * @param list some character's compositions list
- * @param set recursively receives the composites from these compositions
- */
-void Normalizer2Impl::addComposites(const uint16_t *list, UnicodeSet &set) const {
- uint16_t firstUnit;
- int32_t compositeAndFwd;
- do {
- firstUnit=*list;
- if((firstUnit&COMP_1_TRIPLE)==0) {
- compositeAndFwd=list[1];
- list+=2;
- } else {
- compositeAndFwd=(((int32_t)list[1]&~COMP_2_TRAIL_MASK)<<16)|list[2];
- list+=3;
- }
- UChar32 composite=compositeAndFwd>>1;
- if((compositeAndFwd&1)!=0) {
+}
+
+/*
+ * Finds the recomposition result for
+ * a forward-combining "lead" character,
+ * specified with a pointer to its compositions list,
+ * and a backward-combining "trail" character.
+ *
+ * If the lead and trail characters combine, then this function returns
+ * the following "compositeAndFwd" value:
+ * Bits 21..1 composite character
+ * Bit 0 set if the composite is a forward-combining starter
+ * otherwise it returns -1.
+ *
+ * The compositions list has (trail, compositeAndFwd) pair entries,
+ * encoded as either pairs or triples of 16-bit units.
+ * The last entry has the high bit of its first unit set.
+ *
+ * The list is sorted by ascending trail characters (there are no duplicates).
+ * A linear search is used.
+ *
+ * See normalizer2impl.h for a more detailed description
+ * of the compositions list format.
+ */
+int32_t Normalizer2Impl::combine(const uint16_t *list, UChar32 trail) {
+ uint16_t key1, firstUnit;
+ if(trail<COMP_1_TRAIL_LIMIT) {
+ // trail character is 0..33FF
+ // result entry may have 2 or 3 units
+ key1=(uint16_t)(trail<<1);
+ while(key1>(firstUnit=*list)) {
+ list+=2+(firstUnit&COMP_1_TRIPLE);
+ }
+ if(key1==(firstUnit&COMP_1_TRAIL_MASK)) {
+ if(firstUnit&COMP_1_TRIPLE) {
+ return ((int32_t)list[1]<<16)|list[2];
+ } else {
+ return list[1];
+ }
+ }
+ } else {
+ // trail character is 3400..10FFFF
+ // result entry has 3 units
+ key1=(uint16_t)(COMP_1_TRAIL_LIMIT+
+ (((trail>>COMP_1_TRAIL_SHIFT))&
+ ~COMP_1_TRIPLE));
+ uint16_t key2=(uint16_t)(trail<<COMP_2_TRAIL_SHIFT);
+ uint16_t secondUnit;
+ for(;;) {
+ if(key1>(firstUnit=*list)) {
+ list+=2+(firstUnit&COMP_1_TRIPLE);
+ } else if(key1==(firstUnit&COMP_1_TRAIL_MASK)) {
+ if(key2>(secondUnit=list[1])) {
+ if(firstUnit&COMP_1_LAST_TUPLE) {
+ break;
+ } else {
+ list+=3;
+ }
+ } else if(key2==(secondUnit&COMP_2_TRAIL_MASK)) {
+ return ((int32_t)(secondUnit&~COMP_2_TRAIL_MASK)<<16)|list[2];
+ } else {
+ break;
+ }
+ } else {
+ break;
+ }
+ }
+ }
+ return -1;
+}
+
+/**
+ * @param list some character's compositions list
+ * @param set recursively receives the composites from these compositions
+ */
+void Normalizer2Impl::addComposites(const uint16_t *list, UnicodeSet &set) const {
+ uint16_t firstUnit;
+ int32_t compositeAndFwd;
+ do {
+ firstUnit=*list;
+ if((firstUnit&COMP_1_TRIPLE)==0) {
+ compositeAndFwd=list[1];
+ list+=2;
+ } else {
+ compositeAndFwd=(((int32_t)list[1]&~COMP_2_TRAIL_MASK)<<16)|list[2];
+ list+=3;
+ }
+ UChar32 composite=compositeAndFwd>>1;
+ if((compositeAndFwd&1)!=0) {
addComposites(getCompositionsListForComposite(getRawNorm16(composite)), set);
- }
- set.add(composite);
- } while((firstUnit&COMP_1_LAST_TUPLE)==0);
-}
-
-/*
- * Recomposes the buffer text starting at recomposeStartIndex
- * (which is in NFD - decomposed and canonically ordered),
- * and truncates the buffer contents.
- *
- * Note that recomposition never lengthens the text:
- * Any character consists of either one or two code units;
- * a composition may contain at most one more code unit than the original starter,
- * while the combining mark that is removed has at least one code unit.
- */
-void Normalizer2Impl::recompose(ReorderingBuffer &buffer, int32_t recomposeStartIndex,
- UBool onlyContiguous) const {
- UChar *p=buffer.getStart()+recomposeStartIndex;
- UChar *limit=buffer.getLimit();
- if(p==limit) {
- return;
- }
-
- UChar *starter, *pRemove, *q, *r;
- const uint16_t *compositionsList;
- UChar32 c, compositeAndFwd;
- uint16_t norm16;
- uint8_t cc, prevCC;
- UBool starterIsSupplementary;
-
- // Some of the following variables are not used until we have a forward-combining starter
- // and are only initialized now to avoid compiler warnings.
- compositionsList=NULL; // used as indicator for whether we have a forward-combining starter
- starter=NULL;
- starterIsSupplementary=FALSE;
- prevCC=0;
-
- for(;;) {
+ }
+ set.add(composite);
+ } while((firstUnit&COMP_1_LAST_TUPLE)==0);
+}
+
+/*
+ * Recomposes the buffer text starting at recomposeStartIndex
+ * (which is in NFD - decomposed and canonically ordered),
+ * and truncates the buffer contents.
+ *
+ * Note that recomposition never lengthens the text:
+ * Any character consists of either one or two code units;
+ * a composition may contain at most one more code unit than the original starter,
+ * while the combining mark that is removed has at least one code unit.
+ */
+void Normalizer2Impl::recompose(ReorderingBuffer &buffer, int32_t recomposeStartIndex,
+ UBool onlyContiguous) const {
+ UChar *p=buffer.getStart()+recomposeStartIndex;
+ UChar *limit=buffer.getLimit();
+ if(p==limit) {
+ return;
+ }
+
+ UChar *starter, *pRemove, *q, *r;
+ const uint16_t *compositionsList;
+ UChar32 c, compositeAndFwd;
+ uint16_t norm16;
+ uint8_t cc, prevCC;
+ UBool starterIsSupplementary;
+
+ // Some of the following variables are not used until we have a forward-combining starter
+ // and are only initialized now to avoid compiler warnings.
+ compositionsList=NULL; // used as indicator for whether we have a forward-combining starter
+ starter=NULL;
+ starterIsSupplementary=FALSE;
+ prevCC=0;
+
+ for(;;) {
UCPTRIE_FAST_U16_NEXT(normTrie, UCPTRIE_16, p, limit, c, norm16);
- cc=getCCFromYesOrMaybe(norm16);
- if( // this character combines backward and
- isMaybe(norm16) &&
- // we have seen a starter that combines forward and
- compositionsList!=NULL &&
- // the backward-combining character is not blocked
- (prevCC<cc || prevCC==0)
- ) {
- if(isJamoVT(norm16)) {
- // c is a Jamo V/T, see if we can compose it with the previous character.
- if(c<Hangul::JAMO_T_BASE) {
- // c is a Jamo Vowel, compose with previous Jamo L and following Jamo T.
- UChar prev=(UChar)(*starter-Hangul::JAMO_L_BASE);
- if(prev<Hangul::JAMO_L_COUNT) {
- pRemove=p-1;
- UChar syllable=(UChar)
- (Hangul::HANGUL_BASE+
- (prev*Hangul::JAMO_V_COUNT+(c-Hangul::JAMO_V_BASE))*
- Hangul::JAMO_T_COUNT);
- UChar t;
- if(p!=limit && (t=(UChar)(*p-Hangul::JAMO_T_BASE))<Hangul::JAMO_T_COUNT) {
- ++p;
- syllable+=t; // The next character was a Jamo T.
- }
- *starter=syllable;
- // remove the Jamo V/T
- q=pRemove;
- r=p;
- while(r<limit) {
- *q++=*r++;
- }
- limit=q;
- p=pRemove;
- }
- }
- /*
- * No "else" for Jamo T:
- * Since the input is in NFD, there are no Hangul LV syllables that
- * a Jamo T could combine with.
- * All Jamo Ts are combined above when handling Jamo Vs.
- */
- if(p==limit) {
- break;
- }
- compositionsList=NULL;
- continue;
- } else if((compositeAndFwd=combine(compositionsList, c))>=0) {
- // The starter and the combining mark (c) do combine.
- UChar32 composite=compositeAndFwd>>1;
-
- // Replace the starter with the composite, remove the combining mark.
- pRemove=p-U16_LENGTH(c); // pRemove & p: start & limit of the combining mark
- if(starterIsSupplementary) {
- if(U_IS_SUPPLEMENTARY(composite)) {
- // both are supplementary
- starter[0]=U16_LEAD(composite);
- starter[1]=U16_TRAIL(composite);
- } else {
- *starter=(UChar)composite;
- // The composite is shorter than the starter,
- // move the intermediate characters forward one.
- starterIsSupplementary=FALSE;
- q=starter+1;
- r=q+1;
- while(r<pRemove) {
- *q++=*r++;
- }
- --pRemove;
- }
- } else if(U_IS_SUPPLEMENTARY(composite)) {
- // The composite is longer than the starter,
- // move the intermediate characters back one.
- starterIsSupplementary=TRUE;
- ++starter; // temporarily increment for the loop boundary
- q=pRemove;
- r=++pRemove;
- while(starter<q) {
- *--r=*--q;
- }
- *starter=U16_TRAIL(composite);
- *--starter=U16_LEAD(composite); // undo the temporary increment
- } else {
- // both are on the BMP
- *starter=(UChar)composite;
- }
-
- /* remove the combining mark by moving the following text over it */
- if(pRemove<p) {
- q=pRemove;
- r=p;
- while(r<limit) {
- *q++=*r++;
- }
- limit=q;
- p=pRemove;
- }
- // Keep prevCC because we removed the combining mark.
-
- if(p==limit) {
- break;
- }
- // Is the composite a starter that combines forward?
- if(compositeAndFwd&1) {
- compositionsList=
+ cc=getCCFromYesOrMaybe(norm16);
+ if( // this character combines backward and
+ isMaybe(norm16) &&
+ // we have seen a starter that combines forward and
+ compositionsList!=NULL &&
+ // the backward-combining character is not blocked
+ (prevCC<cc || prevCC==0)
+ ) {
+ if(isJamoVT(norm16)) {
+ // c is a Jamo V/T, see if we can compose it with the previous character.
+ if(c<Hangul::JAMO_T_BASE) {
+ // c is a Jamo Vowel, compose with previous Jamo L and following Jamo T.
+ UChar prev=(UChar)(*starter-Hangul::JAMO_L_BASE);
+ if(prev<Hangul::JAMO_L_COUNT) {
+ pRemove=p-1;
+ UChar syllable=(UChar)
+ (Hangul::HANGUL_BASE+
+ (prev*Hangul::JAMO_V_COUNT+(c-Hangul::JAMO_V_BASE))*
+ Hangul::JAMO_T_COUNT);
+ UChar t;
+ if(p!=limit && (t=(UChar)(*p-Hangul::JAMO_T_BASE))<Hangul::JAMO_T_COUNT) {
+ ++p;
+ syllable+=t; // The next character was a Jamo T.
+ }
+ *starter=syllable;
+ // remove the Jamo V/T
+ q=pRemove;
+ r=p;
+ while(r<limit) {
+ *q++=*r++;
+ }
+ limit=q;
+ p=pRemove;
+ }
+ }
+ /*
+ * No "else" for Jamo T:
+ * Since the input is in NFD, there are no Hangul LV syllables that
+ * a Jamo T could combine with.
+ * All Jamo Ts are combined above when handling Jamo Vs.
+ */
+ if(p==limit) {
+ break;
+ }
+ compositionsList=NULL;
+ continue;
+ } else if((compositeAndFwd=combine(compositionsList, c))>=0) {
+ // The starter and the combining mark (c) do combine.
+ UChar32 composite=compositeAndFwd>>1;
+
+ // Replace the starter with the composite, remove the combining mark.
+ pRemove=p-U16_LENGTH(c); // pRemove & p: start & limit of the combining mark
+ if(starterIsSupplementary) {
+ if(U_IS_SUPPLEMENTARY(composite)) {
+ // both are supplementary
+ starter[0]=U16_LEAD(composite);
+ starter[1]=U16_TRAIL(composite);
+ } else {
+ *starter=(UChar)composite;
+ // The composite is shorter than the starter,
+ // move the intermediate characters forward one.
+ starterIsSupplementary=FALSE;
+ q=starter+1;
+ r=q+1;
+ while(r<pRemove) {
+ *q++=*r++;
+ }
+ --pRemove;
+ }
+ } else if(U_IS_SUPPLEMENTARY(composite)) {
+ // The composite is longer than the starter,
+ // move the intermediate characters back one.
+ starterIsSupplementary=TRUE;
+ ++starter; // temporarily increment for the loop boundary
+ q=pRemove;
+ r=++pRemove;
+ while(starter<q) {
+ *--r=*--q;
+ }
+ *starter=U16_TRAIL(composite);
+ *--starter=U16_LEAD(composite); // undo the temporary increment
+ } else {
+ // both are on the BMP
+ *starter=(UChar)composite;
+ }
+
+ /* remove the combining mark by moving the following text over it */
+ if(pRemove<p) {
+ q=pRemove;
+ r=p;
+ while(r<limit) {
+ *q++=*r++;
+ }
+ limit=q;
+ p=pRemove;
+ }
+ // Keep prevCC because we removed the combining mark.
+
+ if(p==limit) {
+ break;
+ }
+ // Is the composite a starter that combines forward?
+ if(compositeAndFwd&1) {
+ compositionsList=
getCompositionsListForComposite(getRawNorm16(composite));
- } else {
- compositionsList=NULL;
- }
-
- // We combined; continue with looking for compositions.
- continue;
- }
- }
-
- // no combination this time
- prevCC=cc;
- if(p==limit) {
- break;
- }
-
- // If c did not combine, then check if it is a starter.
- if(cc==0) {
- // Found a new starter.
- if((compositionsList=getCompositionsListForDecompYes(norm16))!=NULL) {
- // It may combine with something, prepare for it.
- if(U_IS_BMP(c)) {
- starterIsSupplementary=FALSE;
- starter=p-1;
- } else {
- starterIsSupplementary=TRUE;
- starter=p-2;
- }
- }
- } else if(onlyContiguous) {
- // FCC: no discontiguous compositions; any intervening character blocks.
- compositionsList=NULL;
- }
- }
- buffer.setReorderingLimit(limit);
-}
-
-UChar32
-Normalizer2Impl::composePair(UChar32 a, UChar32 b) const {
+ } else {
+ compositionsList=NULL;
+ }
+
+ // We combined; continue with looking for compositions.
+ continue;
+ }
+ }
+
+ // no combination this time
+ prevCC=cc;
+ if(p==limit) {
+ break;
+ }
+
+ // If c did not combine, then check if it is a starter.
+ if(cc==0) {
+ // Found a new starter.
+ if((compositionsList=getCompositionsListForDecompYes(norm16))!=NULL) {
+ // It may combine with something, prepare for it.
+ if(U_IS_BMP(c)) {
+ starterIsSupplementary=FALSE;
+ starter=p-1;
+ } else {
+ starterIsSupplementary=TRUE;
+ starter=p-2;
+ }
+ }
+ } else if(onlyContiguous) {
+ // FCC: no discontiguous compositions; any intervening character blocks.
+ compositionsList=NULL;
+ }
+ }
+ buffer.setReorderingLimit(limit);
+}
+
+UChar32
+Normalizer2Impl::composePair(UChar32 a, UChar32 b) const {
uint16_t norm16=getNorm16(a); // maps an out-of-range 'a' to inert norm16
- const uint16_t *list;
- if(isInert(norm16)) {
- return U_SENTINEL;
- } else if(norm16<minYesNoMappingsOnly) {
+ const uint16_t *list;
+ if(isInert(norm16)) {
+ return U_SENTINEL;
+ } else if(norm16<minYesNoMappingsOnly) {
// a combines forward.
- if(isJamoL(norm16)) {
- b-=Hangul::JAMO_V_BASE;
- if(0<=b && b<Hangul::JAMO_V_COUNT) {
- return
- (Hangul::HANGUL_BASE+
- ((a-Hangul::JAMO_L_BASE)*Hangul::JAMO_V_COUNT+b)*
- Hangul::JAMO_T_COUNT);
- } else {
- return U_SENTINEL;
- }
+ if(isJamoL(norm16)) {
+ b-=Hangul::JAMO_V_BASE;
+ if(0<=b && b<Hangul::JAMO_V_COUNT) {
+ return
+ (Hangul::HANGUL_BASE+
+ ((a-Hangul::JAMO_L_BASE)*Hangul::JAMO_V_COUNT+b)*
+ Hangul::JAMO_T_COUNT);
+ } else {
+ return U_SENTINEL;
+ }
} else if(isHangulLV(norm16)) {
- b-=Hangul::JAMO_T_BASE;
+ b-=Hangul::JAMO_T_BASE;
if(0<b && b<Hangul::JAMO_T_COUNT) { // not b==0!
- return a+b;
- } else {
- return U_SENTINEL;
- }
- } else {
- // 'a' has a compositions list in extraData
+ return a+b;
+ } else {
+ return U_SENTINEL;
+ }
+ } else {
+ // 'a' has a compositions list in extraData
list=getMapping(norm16);
- if(norm16>minYesNo) { // composite 'a' has both mapping & compositions list
- list+= // mapping pointer
+ if(norm16>minYesNo) { // composite 'a' has both mapping & compositions list
+ list+= // mapping pointer
1+ // +1 to skip the first unit with the mapping length
- (*list&MAPPING_LENGTH_MASK); // + mapping length
- }
- }
- } else if(norm16<minMaybeYes || MIN_NORMAL_MAYBE_YES<=norm16) {
- return U_SENTINEL;
- } else {
+ (*list&MAPPING_LENGTH_MASK); // + mapping length
+ }
+ }
+ } else if(norm16<minMaybeYes || MIN_NORMAL_MAYBE_YES<=norm16) {
+ return U_SENTINEL;
+ } else {
list=getCompositionsListForMaybe(norm16);
- }
- if(b<0 || 0x10ffff<b) { // combine(list, b) requires a valid code point b
- return U_SENTINEL;
- }
-#if U_SIGNED_RIGHT_SHIFT_IS_ARITHMETIC
- return combine(list, b)>>1;
-#else
- int32_t compositeAndFwd=combine(list, b);
- return compositeAndFwd>=0 ? compositeAndFwd>>1 : U_SENTINEL;
-#endif
-}
-
-// Very similar to composeQuickCheck(): Make the same changes in both places if relevant.
-// doCompose: normalize
-// !doCompose: isNormalized (buffer must be empty and initialized)
-UBool
-Normalizer2Impl::compose(const UChar *src, const UChar *limit,
- UBool onlyContiguous,
- UBool doCompose,
- ReorderingBuffer &buffer,
- UErrorCode &errorCode) const {
- const UChar *prevBoundary=src;
- UChar32 minNoMaybeCP=minCompNoMaybeCP;
- if(limit==NULL) {
- src=copyLowPrefixFromNulTerminated(src, minNoMaybeCP,
- doCompose ? &buffer : NULL,
- errorCode);
- if(U_FAILURE(errorCode)) {
- return FALSE;
- }
+ }
+ if(b<0 || 0x10ffff<b) { // combine(list, b) requires a valid code point b
+ return U_SENTINEL;
+ }
+#if U_SIGNED_RIGHT_SHIFT_IS_ARITHMETIC
+ return combine(list, b)>>1;
+#else
+ int32_t compositeAndFwd=combine(list, b);
+ return compositeAndFwd>=0 ? compositeAndFwd>>1 : U_SENTINEL;
+#endif
+}
+
+// Very similar to composeQuickCheck(): Make the same changes in both places if relevant.
+// doCompose: normalize
+// !doCompose: isNormalized (buffer must be empty and initialized)
+UBool
+Normalizer2Impl::compose(const UChar *src, const UChar *limit,
+ UBool onlyContiguous,
+ UBool doCompose,
+ ReorderingBuffer &buffer,
+ UErrorCode &errorCode) const {
+ const UChar *prevBoundary=src;
+ UChar32 minNoMaybeCP=minCompNoMaybeCP;
+ if(limit==NULL) {
+ src=copyLowPrefixFromNulTerminated(src, minNoMaybeCP,
+ doCompose ? &buffer : NULL,
+ errorCode);
+ if(U_FAILURE(errorCode)) {
+ return FALSE;
+ }
limit=u_strchr(src, 0);
if (prevBoundary != src) {
if (hasCompBoundaryAfter(*(src-1), onlyContiguous)) {
@@ -1323,9 +1323,9 @@ Normalizer2Impl::compose(const UChar *src, const UChar *limit,
buffer.removeSuffix(1);
prevBoundary = --src;
}
- }
- }
-
+ }
+ }
+
for (;;) {
// Fast path: Scan over a sequence of characters below the minimum "no or maybe" code point,
// or with (compYes && ccc==0) properties.
@@ -1339,11 +1339,11 @@ Normalizer2Impl::compose(const UChar *src, const UChar *limit,
}
return TRUE;
}
- if( (c=*src)<minNoMaybeCP ||
+ if( (c=*src)<minNoMaybeCP ||
isCompYesAndZeroCC(norm16=UCPTRIE_FAST_BMP_GET(normTrie, UCPTRIE_16, c))
- ) {
- ++src;
- } else {
+ ) {
+ ++src;
+ } else {
prevSrc = src++;
if(!U16_IS_LEAD(c)) {
break;
@@ -1351,15 +1351,15 @@ Normalizer2Impl::compose(const UChar *src, const UChar *limit,
UChar c2;
if(src!=limit && U16_IS_TRAIL(c2=*src)) {
++src;
- c=U16_GET_SUPPLEMENTARY(c, c2);
+ c=U16_GET_SUPPLEMENTARY(c, c2);
norm16=UCPTRIE_FAST_SUPP_GET(normTrie, UCPTRIE_16, c);
if(!isCompYesAndZeroCC(norm16)) {
break;
}
- }
- }
- }
- }
+ }
+ }
+ }
+ }
// isCompYesAndZeroCC(norm16) is false, that is, norm16>=minNoNo.
// The current character is either a "noNo" (has a mapping)
// or a "maybeYes" (combines backward)
@@ -1386,7 +1386,7 @@ Normalizer2Impl::compose(const UChar *src, const UChar *limit,
}
prevBoundary = src;
continue;
- }
+ }
} else if (norm16 < minNoNoCompBoundaryBefore) {
// The mapping is comp-normalized which also implies hasCompBoundaryBefore.
if (norm16HasCompBoundaryAfter(norm16, onlyContiguous) ||
@@ -1414,25 +1414,25 @@ Normalizer2Impl::compose(const UChar *src, const UChar *limit,
prevBoundary = src;
continue;
}
- }
+ }
// Other "noNo" type, or need to examine more text around this character:
// Fall through to the slow path.
} else if (isJamoVT(norm16) && prevBoundary != prevSrc) {
- UChar prev=*(prevSrc-1);
- if(c<Hangul::JAMO_T_BASE) {
+ UChar prev=*(prevSrc-1);
+ if(c<Hangul::JAMO_T_BASE) {
// The current character is a Jamo Vowel,
// compose with previous Jamo L and following Jamo T.
UChar l = (UChar)(prev-Hangul::JAMO_L_BASE);
if(l<Hangul::JAMO_L_COUNT) {
if (!doCompose) {
- return FALSE;
- }
+ return FALSE;
+ }
int32_t t;
if (src != limit &&
0 < (t = ((int32_t)*src - Hangul::JAMO_T_BASE)) &&
t < Hangul::JAMO_T_COUNT) {
// The next character is a Jamo T.
- ++src;
+ ++src;
} else if (hasCompBoundaryBefore(src, limit)) {
// No Jamo T follows, not even via decomposition.
t = 0;
@@ -1451,33 +1451,33 @@ Normalizer2Impl::compose(const UChar *src, const UChar *limit,
break;
}
prevBoundary = src;
- continue;
- }
- // If we see L+V+x where x!=T then we drop to the slow path,
- // decompose and recompose.
- // This is to deal with NFKC finding normal L and V but a
+ continue;
+ }
+ // If we see L+V+x where x!=T then we drop to the slow path,
+ // decompose and recompose.
+ // This is to deal with NFKC finding normal L and V but a
// compatibility variant of a T.
// We need to either fully compose that combination here
// (which would complicate the code and may not work with strange custom data)
// or use the slow path.
- }
+ }
} else if (Hangul::isHangulLV(prev)) {
// The current character is a Jamo Trailing consonant,
- // compose with previous Hangul LV that does not contain a Jamo T.
+ // compose with previous Hangul LV that does not contain a Jamo T.
if (!doCompose) {
- return FALSE;
- }
+ return FALSE;
+ }
UChar32 syllable = prev + c - Hangul::JAMO_T_BASE;
--prevSrc; // Replace the Hangul LV as well.
if (prevBoundary != prevSrc && !buffer.appendZeroCC(prevBoundary, prevSrc, errorCode)) {
break;
- }
+ }
if(!buffer.appendBMP((UChar)syllable, 0, errorCode)) {
break;
}
prevBoundary = src;
- continue;
- }
+ continue;
+ }
// No matching context, or may need to decompose surrounding text first:
// Fall through to the slow path.
} else if (norm16 > JAMO_VT) { // norm16 >= MIN_YES_YES_WITH_CC
@@ -1486,10 +1486,10 @@ Normalizer2Impl::compose(const UChar *src, const UChar *limit,
// if followed by a character with a boundary-before.
uint8_t cc = getCCFromNormalYesOrMaybe(norm16); // cc!=0
if (onlyContiguous /* FCC */ && getPreviousTrailCC(prevBoundary, prevSrc) > cc) {
- // Fails FCD test, need to decompose and contiguously recompose.
+ // Fails FCD test, need to decompose and contiguously recompose.
if (!doCompose) {
- return FALSE;
- }
+ return FALSE;
+ }
} else {
// If !onlyContiguous (not FCC), then we ignore the tccc of
// the previous character which passed the quick check "yes && ccc==0" test.
@@ -1517,7 +1517,7 @@ Normalizer2Impl::compose(const UChar *src, const UChar *limit,
break;
}
src = nextSrc;
- }
+ }
// src is after the last in-order combining mark.
// If there is a boundary here, then we continue with no change.
if (norm16HasCompBoundaryBefore(n16)) {
@@ -1527,9 +1527,9 @@ Normalizer2Impl::compose(const UChar *src, const UChar *limit,
continue;
}
// Use the slow path. There is no boundary in [prevSrc, src[.
- }
- }
-
+ }
+ }
+
// Slow path: Find the nearest boundaries around the current character,
// decompose and recompose.
if (prevBoundary != prevSrc && !norm16HasCompBoundaryBefore(norm16)) {
@@ -1538,11 +1538,11 @@ Normalizer2Impl::compose(const UChar *src, const UChar *limit,
if (!norm16HasCompBoundaryAfter(norm16, onlyContiguous)) {
prevSrc = p;
}
- }
+ }
if (doCompose && prevBoundary != prevSrc && !buffer.appendZeroCC(prevBoundary, prevSrc, errorCode)) {
break;
}
- int32_t recomposeStartIndex=buffer.length();
+ int32_t recomposeStartIndex=buffer.length();
// We know there is not a boundary here.
decomposeShort(prevSrc, src, FALSE /* !stopAtCompBoundary */, onlyContiguous,
buffer, errorCode);
@@ -1550,36 +1550,36 @@ Normalizer2Impl::compose(const UChar *src, const UChar *limit,
src = decomposeShort(src, limit, TRUE /* stopAtCompBoundary */, onlyContiguous,
buffer, errorCode);
if (U_FAILURE(errorCode)) {
- break;
- }
+ break;
+ }
if ((src - prevSrc) > INT32_MAX) { // guard before buffer.equals()
errorCode = U_INDEX_OUTOFBOUNDS_ERROR;
return TRUE;
}
- recompose(buffer, recomposeStartIndex, onlyContiguous);
- if(!doCompose) {
+ recompose(buffer, recomposeStartIndex, onlyContiguous);
+ if(!doCompose) {
if(!buffer.equals(prevSrc, src)) {
- return FALSE;
- }
- buffer.remove();
- }
- prevBoundary=src;
- }
- return TRUE;
-}
-
-// Very similar to compose(): Make the same changes in both places if relevant.
-// pQCResult==NULL: spanQuickCheckYes
-// pQCResult!=NULL: quickCheck (*pQCResult must be UNORM_YES)
-const UChar *
-Normalizer2Impl::composeQuickCheck(const UChar *src, const UChar *limit,
- UBool onlyContiguous,
- UNormalizationCheckResult *pQCResult) const {
- const UChar *prevBoundary=src;
- UChar32 minNoMaybeCP=minCompNoMaybeCP;
- if(limit==NULL) {
- UErrorCode errorCode=U_ZERO_ERROR;
- src=copyLowPrefixFromNulTerminated(src, minNoMaybeCP, NULL, errorCode);
+ return FALSE;
+ }
+ buffer.remove();
+ }
+ prevBoundary=src;
+ }
+ return TRUE;
+}
+
+// Very similar to compose(): Make the same changes in both places if relevant.
+// pQCResult==NULL: spanQuickCheckYes
+// pQCResult!=NULL: quickCheck (*pQCResult must be UNORM_YES)
+const UChar *
+Normalizer2Impl::composeQuickCheck(const UChar *src, const UChar *limit,
+ UBool onlyContiguous,
+ UNormalizationCheckResult *pQCResult) const {
+ const UChar *prevBoundary=src;
+ UChar32 minNoMaybeCP=minCompNoMaybeCP;
+ if(limit==NULL) {
+ UErrorCode errorCode=U_ZERO_ERROR;
+ src=copyLowPrefixFromNulTerminated(src, minNoMaybeCP, NULL, errorCode);
limit=u_strchr(src, 0);
if (prevBoundary != src) {
if (hasCompBoundaryAfter(*(src-1), onlyContiguous)) {
@@ -1587,24 +1587,24 @@ Normalizer2Impl::composeQuickCheck(const UChar *src, const UChar *limit,
} else {
prevBoundary = --src;
}
- }
- }
-
- for(;;) {
+ }
+ }
+
+ for(;;) {
// Fast path: Scan over a sequence of characters below the minimum "no or maybe" code point,
// or with (compYes && ccc==0) properties.
const UChar *prevSrc;
UChar32 c = 0;
uint16_t norm16 = 0;
for (;;) {
- if(src==limit) {
- return src;
- }
- if( (c=*src)<minNoMaybeCP ||
+ if(src==limit) {
+ return src;
+ }
+ if( (c=*src)<minNoMaybeCP ||
isCompYesAndZeroCC(norm16=UCPTRIE_FAST_BMP_GET(normTrie, UCPTRIE_16, c))
- ) {
- ++src;
- } else {
+ ) {
+ ++src;
+ } else {
prevSrc = src++;
if(!U16_IS_LEAD(c)) {
break;
@@ -1612,13 +1612,13 @@ Normalizer2Impl::composeQuickCheck(const UChar *src, const UChar *limit,
UChar c2;
if(src!=limit && U16_IS_TRAIL(c2=*src)) {
++src;
- c=U16_GET_SUPPLEMENTARY(c, c2);
+ c=U16_GET_SUPPLEMENTARY(c, c2);
norm16=UCPTRIE_FAST_SUPP_GET(normTrie, UCPTRIE_16, c);
if(!isCompYesAndZeroCC(norm16)) {
break;
}
- }
- }
+ }
+ }
}
}
// isCompYesAndZeroCC(norm16) is false, that is, norm16>=minNoNo.
@@ -1637,15 +1637,15 @@ Normalizer2Impl::composeQuickCheck(const UChar *src, const UChar *limit,
UCPTRIE_FAST_U16_PREV(normTrie, UCPTRIE_16, prevBoundary, p, c, n16);
if (norm16HasCompBoundaryAfter(n16, onlyContiguous)) {
prevBoundary = prevSrc;
- } else {
+ } else {
prevBoundary = p;
prevNorm16 = n16;
- }
- }
- }
-
- if(isMaybeOrNonZeroCC(norm16)) {
- uint8_t cc=getCCFromYesOrMaybe(norm16);
+ }
+ }
+ }
+
+ if(isMaybeOrNonZeroCC(norm16)) {
+ uint8_t cc=getCCFromYesOrMaybe(norm16);
if (onlyContiguous /* FCC */ && cc != 0 &&
getTrailCCFromCompYesAndZeroCC(prevNorm16) > cc) {
// The [prevBoundary..prevSrc[ character
@@ -1674,61 +1674,61 @@ Normalizer2Impl::composeQuickCheck(const UChar *src, const UChar *limit,
if (!(prevCC <= cc || cc == 0)) {
break;
}
- } else {
+ } else {
break;
- }
+ }
src = nextSrc;
- }
+ }
// src is after the last in-order combining mark.
if (isCompYesAndZeroCC(norm16)) {
prevBoundary = src;
src = nextSrc;
continue;
}
- }
- }
- if(pQCResult!=NULL) {
- *pQCResult=UNORM_NO;
- }
- return prevBoundary;
- }
-}
-
-void Normalizer2Impl::composeAndAppend(const UChar *src, const UChar *limit,
- UBool doCompose,
- UBool onlyContiguous,
- UnicodeString &safeMiddle,
- ReorderingBuffer &buffer,
- UErrorCode &errorCode) const {
- if(!buffer.isEmpty()) {
+ }
+ }
+ if(pQCResult!=NULL) {
+ *pQCResult=UNORM_NO;
+ }
+ return prevBoundary;
+ }
+}
+
+void Normalizer2Impl::composeAndAppend(const UChar *src, const UChar *limit,
+ UBool doCompose,
+ UBool onlyContiguous,
+ UnicodeString &safeMiddle,
+ ReorderingBuffer &buffer,
+ UErrorCode &errorCode) const {
+ if(!buffer.isEmpty()) {
const UChar *firstStarterInSrc=findNextCompBoundary(src, limit, onlyContiguous);
- if(src!=firstStarterInSrc) {
- const UChar *lastStarterInDest=findPreviousCompBoundary(buffer.getStart(),
+ if(src!=firstStarterInSrc) {
+ const UChar *lastStarterInDest=findPreviousCompBoundary(buffer.getStart(),
buffer.getLimit(), onlyContiguous);
- int32_t destSuffixLength=(int32_t)(buffer.getLimit()-lastStarterInDest);
- UnicodeString middle(lastStarterInDest, destSuffixLength);
- buffer.removeSuffix(destSuffixLength);
- safeMiddle=middle;
- middle.append(src, (int32_t)(firstStarterInSrc-src));
- const UChar *middleStart=middle.getBuffer();
- compose(middleStart, middleStart+middle.length(), onlyContiguous,
- TRUE, buffer, errorCode);
- if(U_FAILURE(errorCode)) {
- return;
- }
- src=firstStarterInSrc;
- }
- }
- if(doCompose) {
- compose(src, limit, onlyContiguous, TRUE, buffer, errorCode);
- } else {
- if(limit==NULL) { // appendZeroCC() needs limit!=NULL
- limit=u_strchr(src, 0);
- }
- buffer.appendZeroCC(src, limit, errorCode);
- }
-}
-
+ int32_t destSuffixLength=(int32_t)(buffer.getLimit()-lastStarterInDest);
+ UnicodeString middle(lastStarterInDest, destSuffixLength);
+ buffer.removeSuffix(destSuffixLength);
+ safeMiddle=middle;
+ middle.append(src, (int32_t)(firstStarterInSrc-src));
+ const UChar *middleStart=middle.getBuffer();
+ compose(middleStart, middleStart+middle.length(), onlyContiguous,
+ TRUE, buffer, errorCode);
+ if(U_FAILURE(errorCode)) {
+ return;
+ }
+ src=firstStarterInSrc;
+ }
+ }
+ if(doCompose) {
+ compose(src, limit, onlyContiguous, TRUE, buffer, errorCode);
+ } else {
+ if(limit==NULL) { // appendZeroCC() needs limit!=NULL
+ limit=u_strchr(src, 0);
+ }
+ buffer.appendZeroCC(src, limit, errorCode);
+ }
+}
+
UBool
Normalizer2Impl::composeUTF8(uint32_t options, UBool onlyContiguous,
const uint8_t *src, const uint8_t *limit,
@@ -1964,58 +1964,58 @@ Normalizer2Impl::composeUTF8(uint32_t options, UBool onlyContiguous,
}
if ((src - prevSrc) > INT32_MAX) { // guard before buffer.equals()
errorCode = U_INDEX_OUTOFBOUNDS_ERROR;
- return TRUE;
+ return TRUE;
}
recompose(buffer, 0, onlyContiguous);
if (!buffer.equals(prevSrc, src)) {
if (sink == nullptr) {
- return FALSE;
- }
+ return FALSE;
+ }
if (prevBoundary != prevSrc &&
!ByteSinkUtil::appendUnchanged(prevBoundary, prevSrc,
*sink, options, edits, errorCode)) {
break;
- }
+ }
if (!ByteSinkUtil::appendChange(prevSrc, src, buffer.getStart(), buffer.length(),
*sink, edits, errorCode)) {
break;
}
prevBoundary = src;
- }
- }
+ }
+ }
return TRUE;
-}
-
+}
+
UBool Normalizer2Impl::hasCompBoundaryBefore(const UChar *src, const UChar *limit) const {
if (src == limit || *src < minCompNoMaybeCP) {
return TRUE;
- }
+ }
UChar32 c;
uint16_t norm16;
UCPTRIE_FAST_U16_NEXT(normTrie, UCPTRIE_16, src, limit, c, norm16);
return norm16HasCompBoundaryBefore(norm16);
-}
-
+}
+
UBool Normalizer2Impl::hasCompBoundaryBefore(const uint8_t *src, const uint8_t *limit) const {
if (src == limit) {
return TRUE;
}
- uint16_t norm16;
+ uint16_t norm16;
UCPTRIE_FAST_U8_NEXT(normTrie, UCPTRIE_16, src, limit, norm16);
return norm16HasCompBoundaryBefore(norm16);
-}
-
+}
+
UBool Normalizer2Impl::hasCompBoundaryAfter(const UChar *start, const UChar *p,
UBool onlyContiguous) const {
if (start == p) {
return TRUE;
}
UChar32 c;
- uint16_t norm16;
+ uint16_t norm16;
UCPTRIE_FAST_U16_PREV(normTrie, UCPTRIE_16, start, p, c, norm16);
return norm16HasCompBoundaryAfter(norm16, onlyContiguous);
-}
-
+}
+
UBool Normalizer2Impl::hasCompBoundaryAfter(const uint8_t *start, const uint8_t *p,
UBool onlyContiguous) const {
if (start == p) {
@@ -2080,14 +2080,14 @@ uint8_t Normalizer2Impl::getPreviousTrailCC(const uint8_t *start, const uint8_t
return (uint8_t)getFCD16(c);
}
-// Note: normalizer2impl.cpp r30982 (2011-nov-27)
-// still had getFCDTrie() which built and cached an FCD trie.
-// That provided faster access to FCD data than getFCD16FromNormData()
-// but required synchronization and consumed some 10kB of heap memory
-// in any process that uses FCD (e.g., via collation).
+// Note: normalizer2impl.cpp r30982 (2011-nov-27)
+// still had getFCDTrie() which built and cached an FCD trie.
+// That provided faster access to FCD data than getFCD16FromNormData()
+// but required synchronization and consumed some 10kB of heap memory
+// in any process that uses FCD (e.g., via collation).
// minDecompNoCP etc. and smallFCD[] are intended to help with any loss of performance,
// at least for ASCII & CJK.
-
+
// Ticket 20907 - The optimizer in MSVC/Visual Studio versions below 16.4 has trouble with this
// function on Windows ARM64. As a work-around, we disable optimizations for this function.
// This work-around could/should be removed once the following versions of Visual Studio are no
@@ -2095,26 +2095,26 @@ uint8_t Normalizer2Impl::getPreviousTrailCC(const uint8_t *start, const uint8_t
#if (defined(_MSC_VER) && (defined(_M_ARM64)) && (_MSC_VER < 1924))
#pragma optimize( "", off )
#endif
-// Gets the FCD value from the regular normalization data.
-uint16_t Normalizer2Impl::getFCD16FromNormData(UChar32 c) const {
+// Gets the FCD value from the regular normalization data.
+uint16_t Normalizer2Impl::getFCD16FromNormData(UChar32 c) const {
uint16_t norm16=getNorm16(c);
if (norm16 >= limitNoNo) {
if(norm16>=MIN_NORMAL_MAYBE_YES) {
- // combining mark
+ // combining mark
norm16=getCCFromNormalYesOrMaybe(norm16);
- return norm16|(norm16<<8);
- } else if(norm16>=minMaybeYes) {
- return 0;
+ return norm16|(norm16<<8);
+ } else if(norm16>=minMaybeYes) {
+ return 0;
} else { // isDecompNoAlgorithmic(norm16)
uint16_t deltaTrailCC = norm16 & DELTA_TCCC_MASK;
if (deltaTrailCC <= DELTA_TCCC_1) {
return deltaTrailCC >> OFFSET_SHIFT;
}
// Maps to an isCompYesAndZeroCC.
- c=mapAlgorithmic(c, norm16);
+ c=mapAlgorithmic(c, norm16);
norm16=getRawNorm16(c);
- }
- }
+ }
+ }
if(norm16<=minYesNo || isHangulLVT(norm16)) {
// no decomposition or Hangul syllable, all zeros
return 0;
@@ -2127,86 +2127,86 @@ uint16_t Normalizer2Impl::getFCD16FromNormData(UChar32 c) const {
norm16|=*(mapping-1)&0xff00; // lccc
}
return norm16;
-}
+}
#if (defined(_MSC_VER) && (defined(_M_ARM64)) && (_MSC_VER < 1924))
#pragma optimize( "", on )
#endif
-
-// Dual functionality:
-// buffer!=NULL: normalize
-// buffer==NULL: isNormalized/quickCheck/spanQuickCheckYes
-const UChar *
-Normalizer2Impl::makeFCD(const UChar *src, const UChar *limit,
- ReorderingBuffer *buffer,
- UErrorCode &errorCode) const {
- // Tracks the last FCD-safe boundary, before lccc=0 or after properly-ordered tccc<=1.
- // Similar to the prevBoundary in the compose() implementation.
- const UChar *prevBoundary=src;
- int32_t prevFCD16=0;
- if(limit==NULL) {
+
+// Dual functionality:
+// buffer!=NULL: normalize
+// buffer==NULL: isNormalized/quickCheck/spanQuickCheckYes
+const UChar *
+Normalizer2Impl::makeFCD(const UChar *src, const UChar *limit,
+ ReorderingBuffer *buffer,
+ UErrorCode &errorCode) const {
+ // Tracks the last FCD-safe boundary, before lccc=0 or after properly-ordered tccc<=1.
+ // Similar to the prevBoundary in the compose() implementation.
+ const UChar *prevBoundary=src;
+ int32_t prevFCD16=0;
+ if(limit==NULL) {
src=copyLowPrefixFromNulTerminated(src, minLcccCP, buffer, errorCode);
- if(U_FAILURE(errorCode)) {
- return src;
- }
- if(prevBoundary<src) {
- prevBoundary=src;
- // We know that the previous character's lccc==0.
- // Fetching the fcd16 value was deferred for this below-U+0300 code point.
- prevFCD16=getFCD16(*(src-1));
- if(prevFCD16>1) {
- --prevBoundary;
- }
- }
- limit=u_strchr(src, 0);
- }
-
- // Note: In this function we use buffer->appendZeroCC() because we track
- // the lead and trail combining classes here, rather than leaving it to
- // the ReorderingBuffer.
- // The exception is the call to decomposeShort() which uses the buffer
- // in the normal way.
-
- const UChar *prevSrc;
- UChar32 c=0;
- uint16_t fcd16=0;
-
- for(;;) {
- // count code units with lccc==0
- for(prevSrc=src; src!=limit;) {
+ if(U_FAILURE(errorCode)) {
+ return src;
+ }
+ if(prevBoundary<src) {
+ prevBoundary=src;
+ // We know that the previous character's lccc==0.
+ // Fetching the fcd16 value was deferred for this below-U+0300 code point.
+ prevFCD16=getFCD16(*(src-1));
+ if(prevFCD16>1) {
+ --prevBoundary;
+ }
+ }
+ limit=u_strchr(src, 0);
+ }
+
+ // Note: In this function we use buffer->appendZeroCC() because we track
+ // the lead and trail combining classes here, rather than leaving it to
+ // the ReorderingBuffer.
+ // The exception is the call to decomposeShort() which uses the buffer
+ // in the normal way.
+
+ const UChar *prevSrc;
+ UChar32 c=0;
+ uint16_t fcd16=0;
+
+ for(;;) {
+ // count code units with lccc==0
+ for(prevSrc=src; src!=limit;) {
if((c=*src)<minLcccCP) {
- prevFCD16=~c;
- ++src;
- } else if(!singleLeadMightHaveNonZeroFCD16(c)) {
- prevFCD16=0;
- ++src;
- } else {
+ prevFCD16=~c;
+ ++src;
+ } else if(!singleLeadMightHaveNonZeroFCD16(c)) {
+ prevFCD16=0;
+ ++src;
+ } else {
if(U16_IS_LEAD(c)) {
- UChar c2;
+ UChar c2;
if((src+1)!=limit && U16_IS_TRAIL(c2=src[1])) {
c=U16_GET_SUPPLEMENTARY(c, c2);
- }
- }
- if((fcd16=getFCD16FromNormData(c))<=0xff) {
- prevFCD16=fcd16;
- src+=U16_LENGTH(c);
- } else {
- break;
- }
- }
- }
- // copy these code units all at once
- if(src!=prevSrc) {
- if(buffer!=NULL && !buffer->appendZeroCC(prevSrc, src, errorCode)) {
- break;
- }
- if(src==limit) {
- break;
- }
- prevBoundary=src;
- // We know that the previous character's lccc==0.
- if(prevFCD16<0) {
+ }
+ }
+ if((fcd16=getFCD16FromNormData(c))<=0xff) {
+ prevFCD16=fcd16;
+ src+=U16_LENGTH(c);
+ } else {
+ break;
+ }
+ }
+ }
+ // copy these code units all at once
+ if(src!=prevSrc) {
+ if(buffer!=NULL && !buffer->appendZeroCC(prevSrc, src, errorCode)) {
+ break;
+ }
+ if(src==limit) {
+ break;
+ }
+ prevBoundary=src;
+ // We know that the previous character's lccc==0.
+ if(prevFCD16<0) {
// Fetching the fcd16 value was deferred for this below-minLcccCP code point.
- UChar32 prev=~prevFCD16;
+ UChar32 prev=~prevFCD16;
if(prev<minDecompNoCP) {
prevFCD16=0;
} else {
@@ -2214,102 +2214,102 @@ Normalizer2Impl::makeFCD(const UChar *src, const UChar *limit,
if(prevFCD16>1) {
--prevBoundary;
}
- }
- } else {
- const UChar *p=src-1;
- if(U16_IS_TRAIL(*p) && prevSrc<p && U16_IS_LEAD(*(p-1))) {
- --p;
- // Need to fetch the previous character's FCD value because
- // prevFCD16 was just for the trail surrogate code point.
- prevFCD16=getFCD16FromNormData(U16_GET_SUPPLEMENTARY(p[0], p[1]));
- // Still known to have lccc==0 because its lead surrogate unit had lccc==0.
- }
- if(prevFCD16>1) {
- prevBoundary=p;
- }
- }
- // The start of the current character (c).
- prevSrc=src;
- } else if(src==limit) {
- break;
- }
-
- src+=U16_LENGTH(c);
- // The current character (c) at [prevSrc..src[ has a non-zero lead combining class.
- // Check for proper order, and decompose locally if necessary.
- if((prevFCD16&0xff)<=(fcd16>>8)) {
- // proper order: prev tccc <= current lccc
- if((fcd16&0xff)<=1) {
- prevBoundary=src;
- }
- if(buffer!=NULL && !buffer->appendZeroCC(c, errorCode)) {
- break;
- }
- prevFCD16=fcd16;
- continue;
- } else if(buffer==NULL) {
- return prevBoundary; // quick check "no"
- } else {
- /*
- * Back out the part of the source that we copied or appended
- * already but is now going to be decomposed.
- * prevSrc is set to after what was copied/appended.
- */
- buffer->removeSuffix((int32_t)(prevSrc-prevBoundary));
- /*
- * Find the part of the source that needs to be decomposed,
- * up to the next safe boundary.
- */
- src=findNextFCDBoundary(src, limit);
- /*
- * The source text does not fulfill the conditions for FCD.
- * Decompose and reorder a limited piece of the text.
- */
+ }
+ } else {
+ const UChar *p=src-1;
+ if(U16_IS_TRAIL(*p) && prevSrc<p && U16_IS_LEAD(*(p-1))) {
+ --p;
+ // Need to fetch the previous character's FCD value because
+ // prevFCD16 was just for the trail surrogate code point.
+ prevFCD16=getFCD16FromNormData(U16_GET_SUPPLEMENTARY(p[0], p[1]));
+ // Still known to have lccc==0 because its lead surrogate unit had lccc==0.
+ }
+ if(prevFCD16>1) {
+ prevBoundary=p;
+ }
+ }
+ // The start of the current character (c).
+ prevSrc=src;
+ } else if(src==limit) {
+ break;
+ }
+
+ src+=U16_LENGTH(c);
+ // The current character (c) at [prevSrc..src[ has a non-zero lead combining class.
+ // Check for proper order, and decompose locally if necessary.
+ if((prevFCD16&0xff)<=(fcd16>>8)) {
+ // proper order: prev tccc <= current lccc
+ if((fcd16&0xff)<=1) {
+ prevBoundary=src;
+ }
+ if(buffer!=NULL && !buffer->appendZeroCC(c, errorCode)) {
+ break;
+ }
+ prevFCD16=fcd16;
+ continue;
+ } else if(buffer==NULL) {
+ return prevBoundary; // quick check "no"
+ } else {
+ /*
+ * Back out the part of the source that we copied or appended
+ * already but is now going to be decomposed.
+ * prevSrc is set to after what was copied/appended.
+ */
+ buffer->removeSuffix((int32_t)(prevSrc-prevBoundary));
+ /*
+ * Find the part of the source that needs to be decomposed,
+ * up to the next safe boundary.
+ */
+ src=findNextFCDBoundary(src, limit);
+ /*
+ * The source text does not fulfill the conditions for FCD.
+ * Decompose and reorder a limited piece of the text.
+ */
decomposeShort(prevBoundary, src, FALSE, FALSE, *buffer, errorCode);
if (U_FAILURE(errorCode)) {
- break;
- }
- prevBoundary=src;
- prevFCD16=0;
- }
- }
- return src;
-}
-
-void Normalizer2Impl::makeFCDAndAppend(const UChar *src, const UChar *limit,
- UBool doMakeFCD,
- UnicodeString &safeMiddle,
- ReorderingBuffer &buffer,
- UErrorCode &errorCode) const {
- if(!buffer.isEmpty()) {
- const UChar *firstBoundaryInSrc=findNextFCDBoundary(src, limit);
- if(src!=firstBoundaryInSrc) {
- const UChar *lastBoundaryInDest=findPreviousFCDBoundary(buffer.getStart(),
- buffer.getLimit());
- int32_t destSuffixLength=(int32_t)(buffer.getLimit()-lastBoundaryInDest);
- UnicodeString middle(lastBoundaryInDest, destSuffixLength);
- buffer.removeSuffix(destSuffixLength);
- safeMiddle=middle;
- middle.append(src, (int32_t)(firstBoundaryInSrc-src));
- const UChar *middleStart=middle.getBuffer();
- makeFCD(middleStart, middleStart+middle.length(), &buffer, errorCode);
- if(U_FAILURE(errorCode)) {
- return;
- }
- src=firstBoundaryInSrc;
- }
- }
- if(doMakeFCD) {
- makeFCD(src, limit, &buffer, errorCode);
- } else {
- if(limit==NULL) { // appendZeroCC() needs limit!=NULL
- limit=u_strchr(src, 0);
- }
- buffer.appendZeroCC(src, limit, errorCode);
- }
-}
-
-const UChar *Normalizer2Impl::findPreviousFCDBoundary(const UChar *start, const UChar *p) const {
+ break;
+ }
+ prevBoundary=src;
+ prevFCD16=0;
+ }
+ }
+ return src;
+}
+
+void Normalizer2Impl::makeFCDAndAppend(const UChar *src, const UChar *limit,
+ UBool doMakeFCD,
+ UnicodeString &safeMiddle,
+ ReorderingBuffer &buffer,
+ UErrorCode &errorCode) const {
+ if(!buffer.isEmpty()) {
+ const UChar *firstBoundaryInSrc=findNextFCDBoundary(src, limit);
+ if(src!=firstBoundaryInSrc) {
+ const UChar *lastBoundaryInDest=findPreviousFCDBoundary(buffer.getStart(),
+ buffer.getLimit());
+ int32_t destSuffixLength=(int32_t)(buffer.getLimit()-lastBoundaryInDest);
+ UnicodeString middle(lastBoundaryInDest, destSuffixLength);
+ buffer.removeSuffix(destSuffixLength);
+ safeMiddle=middle;
+ middle.append(src, (int32_t)(firstBoundaryInSrc-src));
+ const UChar *middleStart=middle.getBuffer();
+ makeFCD(middleStart, middleStart+middle.length(), &buffer, errorCode);
+ if(U_FAILURE(errorCode)) {
+ return;
+ }
+ src=firstBoundaryInSrc;
+ }
+ }
+ if(doMakeFCD) {
+ makeFCD(src, limit, &buffer, errorCode);
+ } else {
+ if(limit==NULL) { // appendZeroCC() needs limit!=NULL
+ limit=u_strchr(src, 0);
+ }
+ buffer.appendZeroCC(src, limit, errorCode);
+ }
+}
+
+const UChar *Normalizer2Impl::findPreviousFCDBoundary(const UChar *start, const UChar *p) const {
while(start<p) {
const UChar *codePointLimit = p;
UChar32 c;
@@ -2322,88 +2322,88 @@ const UChar *Normalizer2Impl::findPreviousFCDBoundary(const UChar *start, const
return p;
}
}
- return p;
-}
-
-const UChar *Normalizer2Impl::findNextFCDBoundary(const UChar *p, const UChar *limit) const {
- while(p<limit) {
- const UChar *codePointStart=p;
+ return p;
+}
+
+const UChar *Normalizer2Impl::findNextFCDBoundary(const UChar *p, const UChar *limit) const {
+ while(p<limit) {
+ const UChar *codePointStart=p;
UChar32 c;
uint16_t norm16;
UCPTRIE_FAST_U16_NEXT(normTrie, UCPTRIE_16, p, limit, c, norm16);
if (c < minLcccCP || norm16HasDecompBoundaryBefore(norm16)) {
- return codePointStart;
- }
+ return codePointStart;
+ }
if (norm16HasDecompBoundaryAfter(norm16)) {
return p;
}
- }
- return p;
-}
-
-// CanonicalIterator data -------------------------------------------------- ***
-
-CanonIterData::CanonIterData(UErrorCode &errorCode) :
+ }
+ return p;
+}
+
+// CanonicalIterator data -------------------------------------------------- ***
+
+CanonIterData::CanonIterData(UErrorCode &errorCode) :
mutableTrie(umutablecptrie_open(0, 0, &errorCode)), trie(nullptr),
- canonStartSets(uprv_deleteUObject, NULL, errorCode) {}
-
-CanonIterData::~CanonIterData() {
+ canonStartSets(uprv_deleteUObject, NULL, errorCode) {}
+
+CanonIterData::~CanonIterData() {
umutablecptrie_close(mutableTrie);
ucptrie_close(trie);
-}
-
-void CanonIterData::addToStartSet(UChar32 origin, UChar32 decompLead, UErrorCode &errorCode) {
+}
+
+void CanonIterData::addToStartSet(UChar32 origin, UChar32 decompLead, UErrorCode &errorCode) {
uint32_t canonValue = umutablecptrie_get(mutableTrie, decompLead);
- if((canonValue&(CANON_HAS_SET|CANON_VALUE_MASK))==0 && origin!=0) {
- // origin is the first character whose decomposition starts with
- // the character for which we are setting the value.
+ if((canonValue&(CANON_HAS_SET|CANON_VALUE_MASK))==0 && origin!=0) {
+ // origin is the first character whose decomposition starts with
+ // the character for which we are setting the value.
umutablecptrie_set(mutableTrie, decompLead, canonValue|origin, &errorCode);
- } else {
- // origin is not the first character, or it is U+0000.
- UnicodeSet *set;
- if((canonValue&CANON_HAS_SET)==0) {
- set=new UnicodeSet;
- if(set==NULL) {
- errorCode=U_MEMORY_ALLOCATION_ERROR;
- return;
- }
- UChar32 firstOrigin=(UChar32)(canonValue&CANON_VALUE_MASK);
- canonValue=(canonValue&~CANON_VALUE_MASK)|CANON_HAS_SET|(uint32_t)canonStartSets.size();
+ } else {
+ // origin is not the first character, or it is U+0000.
+ UnicodeSet *set;
+ if((canonValue&CANON_HAS_SET)==0) {
+ set=new UnicodeSet;
+ if(set==NULL) {
+ errorCode=U_MEMORY_ALLOCATION_ERROR;
+ return;
+ }
+ UChar32 firstOrigin=(UChar32)(canonValue&CANON_VALUE_MASK);
+ canonValue=(canonValue&~CANON_VALUE_MASK)|CANON_HAS_SET|(uint32_t)canonStartSets.size();
umutablecptrie_set(mutableTrie, decompLead, canonValue, &errorCode);
- canonStartSets.addElement(set, errorCode);
- if(firstOrigin!=0) {
- set->add(firstOrigin);
- }
- } else {
- set=(UnicodeSet *)canonStartSets[(int32_t)(canonValue&CANON_VALUE_MASK)];
- }
- set->add(origin);
- }
-}
-
+ canonStartSets.addElement(set, errorCode);
+ if(firstOrigin!=0) {
+ set->add(firstOrigin);
+ }
+ } else {
+ set=(UnicodeSet *)canonStartSets[(int32_t)(canonValue&CANON_VALUE_MASK)];
+ }
+ set->add(origin);
+ }
+}
+
// C++ class for friend access to private Normalizer2Impl members.
class InitCanonIterData {
public:
static void doInit(Normalizer2Impl *impl, UErrorCode &errorCode);
};
-U_CDECL_BEGIN
-
+U_CDECL_BEGIN
+
// UInitOnce instantiation function for CanonIterData
static void U_CALLCONV
initCanonIterData(Normalizer2Impl *impl, UErrorCode &errorCode) {
InitCanonIterData::doInit(impl, errorCode);
-}
-
+}
+
U_CDECL_END
-
+
void InitCanonIterData::doInit(Normalizer2Impl *impl, UErrorCode &errorCode) {
- U_ASSERT(impl->fCanonIterData == NULL);
- impl->fCanonIterData = new CanonIterData(errorCode);
- if (impl->fCanonIterData == NULL) {
- errorCode=U_MEMORY_ALLOCATION_ERROR;
- }
- if (U_SUCCESS(errorCode)) {
+ U_ASSERT(impl->fCanonIterData == NULL);
+ impl->fCanonIterData = new CanonIterData(errorCode);
+ if (impl->fCanonIterData == NULL) {
+ errorCode=U_MEMORY_ALLOCATION_ERROR;
+ }
+ if (U_SUCCESS(errorCode)) {
UChar32 start = 0, end;
uint32_t value;
while ((end = ucptrie_getRange(impl->normTrie, start,
@@ -2422,183 +2422,183 @@ void InitCanonIterData::doInit(Normalizer2Impl *impl, UErrorCode &errorCode) {
impl->fCanonIterData->mutableTrie, UCPTRIE_TYPE_SMALL, UCPTRIE_VALUE_BITS_32, &errorCode);
umutablecptrie_close(impl->fCanonIterData->mutableTrie);
impl->fCanonIterData->mutableTrie = nullptr;
- }
- if (U_FAILURE(errorCode)) {
- delete impl->fCanonIterData;
- impl->fCanonIterData = NULL;
- }
-}
-
+ }
+ if (U_FAILURE(errorCode)) {
+ delete impl->fCanonIterData;
+ impl->fCanonIterData = NULL;
+ }
+}
+
void Normalizer2Impl::makeCanonIterDataFromNorm16(UChar32 start, UChar32 end, const uint16_t norm16,
- CanonIterData &newData,
- UErrorCode &errorCode) const {
+ CanonIterData &newData,
+ UErrorCode &errorCode) const {
if(isInert(norm16) || (minYesNo<=norm16 && norm16<minNoNo)) {
- // Inert, or 2-way mapping (including Hangul syllable).
- // We do not write a canonStartSet for any yesNo character.
- // Composites from 2-way mappings are added at runtime from the
- // starter's compositions list, and the other characters in
- // 2-way mappings get CANON_NOT_SEGMENT_STARTER set because they are
- // "maybe" characters.
- return;
- }
- for(UChar32 c=start; c<=end; ++c) {
+ // Inert, or 2-way mapping (including Hangul syllable).
+ // We do not write a canonStartSet for any yesNo character.
+ // Composites from 2-way mappings are added at runtime from the
+ // starter's compositions list, and the other characters in
+ // 2-way mappings get CANON_NOT_SEGMENT_STARTER set because they are
+ // "maybe" characters.
+ return;
+ }
+ for(UChar32 c=start; c<=end; ++c) {
uint32_t oldValue = umutablecptrie_get(newData.mutableTrie, c);
- uint32_t newValue=oldValue;
+ uint32_t newValue=oldValue;
if(isMaybeOrNonZeroCC(norm16)) {
- // not a segment starter if it occurs in a decomposition or has cc!=0
- newValue|=CANON_NOT_SEGMENT_STARTER;
- if(norm16<MIN_NORMAL_MAYBE_YES) {
- newValue|=CANON_HAS_COMPOSITIONS;
- }
- } else if(norm16<minYesNo) {
- newValue|=CANON_HAS_COMPOSITIONS;
- } else {
- // c has a one-way decomposition
- UChar32 c2=c;
+ // not a segment starter if it occurs in a decomposition or has cc!=0
+ newValue|=CANON_NOT_SEGMENT_STARTER;
+ if(norm16<MIN_NORMAL_MAYBE_YES) {
+ newValue|=CANON_HAS_COMPOSITIONS;
+ }
+ } else if(norm16<minYesNo) {
+ newValue|=CANON_HAS_COMPOSITIONS;
+ } else {
+ // c has a one-way decomposition
+ UChar32 c2=c;
// Do not modify the whole-range norm16 value.
- uint16_t norm16_2=norm16;
+ uint16_t norm16_2=norm16;
if (isDecompNoAlgorithmic(norm16_2)) {
// Maps to an isCompYesAndZeroCC.
c2 = mapAlgorithmic(c2, norm16_2);
norm16_2 = getRawNorm16(c2);
// No compatibility mappings for the CanonicalIterator.
U_ASSERT(!(isHangulLV(norm16_2) || isHangulLVT(norm16_2)));
- }
+ }
if (norm16_2 > minYesNo) {
- // c decomposes, get everything from the variable-length extra data
- const uint16_t *mapping=getMapping(norm16_2);
- uint16_t firstUnit=*mapping;
- int32_t length=firstUnit&MAPPING_LENGTH_MASK;
- if((firstUnit&MAPPING_HAS_CCC_LCCC_WORD)!=0) {
- if(c==c2 && (*(mapping-1)&0xff)!=0) {
- newValue|=CANON_NOT_SEGMENT_STARTER; // original c has cc!=0
- }
- }
- // Skip empty mappings (no characters in the decomposition).
- if(length!=0) {
- ++mapping; // skip over the firstUnit
- // add c to first code point's start set
- int32_t i=0;
- U16_NEXT_UNSAFE(mapping, i, c2);
- newData.addToStartSet(c, c2, errorCode);
- // Set CANON_NOT_SEGMENT_STARTER for each remaining code point of a
- // one-way mapping. A 2-way mapping is possible here after
- // intermediate algorithmic mapping.
- if(norm16_2>=minNoNo) {
- while(i<length) {
- U16_NEXT_UNSAFE(mapping, i, c2);
+ // c decomposes, get everything from the variable-length extra data
+ const uint16_t *mapping=getMapping(norm16_2);
+ uint16_t firstUnit=*mapping;
+ int32_t length=firstUnit&MAPPING_LENGTH_MASK;
+ if((firstUnit&MAPPING_HAS_CCC_LCCC_WORD)!=0) {
+ if(c==c2 && (*(mapping-1)&0xff)!=0) {
+ newValue|=CANON_NOT_SEGMENT_STARTER; // original c has cc!=0
+ }
+ }
+ // Skip empty mappings (no characters in the decomposition).
+ if(length!=0) {
+ ++mapping; // skip over the firstUnit
+ // add c to first code point's start set
+ int32_t i=0;
+ U16_NEXT_UNSAFE(mapping, i, c2);
+ newData.addToStartSet(c, c2, errorCode);
+ // Set CANON_NOT_SEGMENT_STARTER for each remaining code point of a
+ // one-way mapping. A 2-way mapping is possible here after
+ // intermediate algorithmic mapping.
+ if(norm16_2>=minNoNo) {
+ while(i<length) {
+ U16_NEXT_UNSAFE(mapping, i, c2);
uint32_t c2Value = umutablecptrie_get(newData.mutableTrie, c2);
- if((c2Value&CANON_NOT_SEGMENT_STARTER)==0) {
+ if((c2Value&CANON_NOT_SEGMENT_STARTER)==0) {
umutablecptrie_set(newData.mutableTrie, c2,
c2Value|CANON_NOT_SEGMENT_STARTER, &errorCode);
- }
- }
- }
- }
- } else {
- // c decomposed to c2 algorithmically; c has cc==0
- newData.addToStartSet(c, c2, errorCode);
- }
- }
- if(newValue!=oldValue) {
+ }
+ }
+ }
+ }
+ } else {
+ // c decomposed to c2 algorithmically; c has cc==0
+ newData.addToStartSet(c, c2, errorCode);
+ }
+ }
+ if(newValue!=oldValue) {
umutablecptrie_set(newData.mutableTrie, c, newValue, &errorCode);
- }
- }
-}
-
-UBool Normalizer2Impl::ensureCanonIterData(UErrorCode &errorCode) const {
- // Logically const: Synchronized instantiation.
- Normalizer2Impl *me=const_cast<Normalizer2Impl *>(this);
- umtx_initOnce(me->fCanonIterDataInitOnce, &initCanonIterData, me, errorCode);
- return U_SUCCESS(errorCode);
-}
-
-int32_t Normalizer2Impl::getCanonValue(UChar32 c) const {
+ }
+ }
+}
+
+UBool Normalizer2Impl::ensureCanonIterData(UErrorCode &errorCode) const {
+ // Logically const: Synchronized instantiation.
+ Normalizer2Impl *me=const_cast<Normalizer2Impl *>(this);
+ umtx_initOnce(me->fCanonIterDataInitOnce, &initCanonIterData, me, errorCode);
+ return U_SUCCESS(errorCode);
+}
+
+int32_t Normalizer2Impl::getCanonValue(UChar32 c) const {
return (int32_t)ucptrie_get(fCanonIterData->trie, c);
-}
-
-const UnicodeSet &Normalizer2Impl::getCanonStartSet(int32_t n) const {
- return *(const UnicodeSet *)fCanonIterData->canonStartSets[n];
-}
-
-UBool Normalizer2Impl::isCanonSegmentStarter(UChar32 c) const {
- return getCanonValue(c)>=0;
-}
-
-UBool Normalizer2Impl::getCanonStartSet(UChar32 c, UnicodeSet &set) const {
- int32_t canonValue=getCanonValue(c)&~CANON_NOT_SEGMENT_STARTER;
- if(canonValue==0) {
- return FALSE;
- }
- set.clear();
- int32_t value=canonValue&CANON_VALUE_MASK;
- if((canonValue&CANON_HAS_SET)!=0) {
- set.addAll(getCanonStartSet(value));
- } else if(value!=0) {
- set.add(value);
- }
- if((canonValue&CANON_HAS_COMPOSITIONS)!=0) {
+}
+
+const UnicodeSet &Normalizer2Impl::getCanonStartSet(int32_t n) const {
+ return *(const UnicodeSet *)fCanonIterData->canonStartSets[n];
+}
+
+UBool Normalizer2Impl::isCanonSegmentStarter(UChar32 c) const {
+ return getCanonValue(c)>=0;
+}
+
+UBool Normalizer2Impl::getCanonStartSet(UChar32 c, UnicodeSet &set) const {
+ int32_t canonValue=getCanonValue(c)&~CANON_NOT_SEGMENT_STARTER;
+ if(canonValue==0) {
+ return FALSE;
+ }
+ set.clear();
+ int32_t value=canonValue&CANON_VALUE_MASK;
+ if((canonValue&CANON_HAS_SET)!=0) {
+ set.addAll(getCanonStartSet(value));
+ } else if(value!=0) {
+ set.add(value);
+ }
+ if((canonValue&CANON_HAS_COMPOSITIONS)!=0) {
uint16_t norm16=getRawNorm16(c);
- if(norm16==JAMO_L) {
- UChar32 syllable=
- (UChar32)(Hangul::HANGUL_BASE+(c-Hangul::JAMO_L_BASE)*Hangul::JAMO_VT_COUNT);
- set.add(syllable, syllable+Hangul::JAMO_VT_COUNT-1);
- } else {
- addComposites(getCompositionsList(norm16), set);
- }
- }
- return TRUE;
-}
-
-U_NAMESPACE_END
-
-// Normalizer2 data swapping ----------------------------------------------- ***
-
-U_NAMESPACE_USE
-
-U_CAPI int32_t U_EXPORT2
-unorm2_swap(const UDataSwapper *ds,
- const void *inData, int32_t length, void *outData,
- UErrorCode *pErrorCode) {
- const UDataInfo *pInfo;
- int32_t headerSize;
-
- const uint8_t *inBytes;
- uint8_t *outBytes;
-
- const int32_t *inIndexes;
+ if(norm16==JAMO_L) {
+ UChar32 syllable=
+ (UChar32)(Hangul::HANGUL_BASE+(c-Hangul::JAMO_L_BASE)*Hangul::JAMO_VT_COUNT);
+ set.add(syllable, syllable+Hangul::JAMO_VT_COUNT-1);
+ } else {
+ addComposites(getCompositionsList(norm16), set);
+ }
+ }
+ return TRUE;
+}
+
+U_NAMESPACE_END
+
+// Normalizer2 data swapping ----------------------------------------------- ***
+
+U_NAMESPACE_USE
+
+U_CAPI int32_t U_EXPORT2
+unorm2_swap(const UDataSwapper *ds,
+ const void *inData, int32_t length, void *outData,
+ UErrorCode *pErrorCode) {
+ const UDataInfo *pInfo;
+ int32_t headerSize;
+
+ const uint8_t *inBytes;
+ uint8_t *outBytes;
+
+ const int32_t *inIndexes;
int32_t indexes[Normalizer2Impl::IX_TOTAL_SIZE+1];
-
- int32_t i, offset, nextOffset, size;
-
- /* udata_swapDataHeader checks the arguments */
- headerSize=udata_swapDataHeader(ds, inData, length, outData, pErrorCode);
- if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
- return 0;
- }
-
- /* check data format and format version */
- pInfo=(const UDataInfo *)((const char *)inData+4);
+
+ int32_t i, offset, nextOffset, size;
+
+ /* udata_swapDataHeader checks the arguments */
+ headerSize=udata_swapDataHeader(ds, inData, length, outData, pErrorCode);
+ if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
+ return 0;
+ }
+
+ /* check data format and format version */
+ pInfo=(const UDataInfo *)((const char *)inData+4);
uint8_t formatVersion0=pInfo->formatVersion[0];
- if(!(
- pInfo->dataFormat[0]==0x4e && /* dataFormat="Nrm2" */
- pInfo->dataFormat[1]==0x72 &&
- pInfo->dataFormat[2]==0x6d &&
- pInfo->dataFormat[3]==0x32 &&
+ if(!(
+ pInfo->dataFormat[0]==0x4e && /* dataFormat="Nrm2" */
+ pInfo->dataFormat[1]==0x72 &&
+ pInfo->dataFormat[2]==0x6d &&
+ pInfo->dataFormat[3]==0x32 &&
(1<=formatVersion0 && formatVersion0<=4)
- )) {
- udata_printError(ds, "unorm2_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized as Normalizer2 data\n",
- pInfo->dataFormat[0], pInfo->dataFormat[1],
- pInfo->dataFormat[2], pInfo->dataFormat[3],
- pInfo->formatVersion[0]);
- *pErrorCode=U_UNSUPPORTED_ERROR;
- return 0;
- }
-
- inBytes=(const uint8_t *)inData+headerSize;
- outBytes=(uint8_t *)outData+headerSize;
-
- inIndexes=(const int32_t *)inBytes;
+ )) {
+ udata_printError(ds, "unorm2_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized as Normalizer2 data\n",
+ pInfo->dataFormat[0], pInfo->dataFormat[1],
+ pInfo->dataFormat[2], pInfo->dataFormat[3],
+ pInfo->formatVersion[0]);
+ *pErrorCode=U_UNSUPPORTED_ERROR;
+ return 0;
+ }
+
+ inBytes=(const uint8_t *)inData+headerSize;
+ outBytes=(uint8_t *)outData+headerSize;
+
+ inIndexes=(const int32_t *)inBytes;
int32_t minIndexesLength;
if(formatVersion0==1) {
minIndexesLength=Normalizer2Impl::IX_MIN_MAYBE_YES+1;
@@ -2607,63 +2607,63 @@ unorm2_swap(const UDataSwapper *ds,
} else {
minIndexesLength=Normalizer2Impl::IX_MIN_LCCC_CP+1;
}
-
- if(length>=0) {
- length-=headerSize;
+
+ if(length>=0) {
+ length-=headerSize;
if(length<minIndexesLength*4) {
- udata_printError(ds, "unorm2_swap(): too few bytes (%d after header) for Normalizer2 data\n",
- length);
- *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
- return 0;
- }
- }
-
- /* read the first few indexes */
+ udata_printError(ds, "unorm2_swap(): too few bytes (%d after header) for Normalizer2 data\n",
+ length);
+ *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
+ return 0;
+ }
+ }
+
+ /* read the first few indexes */
for(i=0; i<UPRV_LENGTHOF(indexes); ++i) {
- indexes[i]=udata_readInt32(ds, inIndexes[i]);
- }
-
- /* get the total length of the data */
- size=indexes[Normalizer2Impl::IX_TOTAL_SIZE];
-
- if(length>=0) {
- if(length<size) {
- udata_printError(ds, "unorm2_swap(): too few bytes (%d after header) for all of Normalizer2 data\n",
- length);
- *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
- return 0;
- }
-
- /* copy the data for inaccessible bytes */
- if(inBytes!=outBytes) {
- uprv_memcpy(outBytes, inBytes, size);
- }
-
- offset=0;
-
- /* swap the int32_t indexes[] */
- nextOffset=indexes[Normalizer2Impl::IX_NORM_TRIE_OFFSET];
- ds->swapArray32(ds, inBytes, nextOffset-offset, outBytes, pErrorCode);
- offset=nextOffset;
-
+ indexes[i]=udata_readInt32(ds, inIndexes[i]);
+ }
+
+ /* get the total length of the data */
+ size=indexes[Normalizer2Impl::IX_TOTAL_SIZE];
+
+ if(length>=0) {
+ if(length<size) {
+ udata_printError(ds, "unorm2_swap(): too few bytes (%d after header) for all of Normalizer2 data\n",
+ length);
+ *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
+ return 0;
+ }
+
+ /* copy the data for inaccessible bytes */
+ if(inBytes!=outBytes) {
+ uprv_memcpy(outBytes, inBytes, size);
+ }
+
+ offset=0;
+
+ /* swap the int32_t indexes[] */
+ nextOffset=indexes[Normalizer2Impl::IX_NORM_TRIE_OFFSET];
+ ds->swapArray32(ds, inBytes, nextOffset-offset, outBytes, pErrorCode);
+ offset=nextOffset;
+
/* swap the trie */
- nextOffset=indexes[Normalizer2Impl::IX_EXTRA_DATA_OFFSET];
+ nextOffset=indexes[Normalizer2Impl::IX_EXTRA_DATA_OFFSET];
utrie_swapAnyVersion(ds, inBytes+offset, nextOffset-offset, outBytes+offset, pErrorCode);
- offset=nextOffset;
-
- /* swap the uint16_t extraData[] */
- nextOffset=indexes[Normalizer2Impl::IX_SMALL_FCD_OFFSET];
- ds->swapArray16(ds, inBytes+offset, nextOffset-offset, outBytes+offset, pErrorCode);
- offset=nextOffset;
-
- /* no need to swap the uint8_t smallFCD[] (new in formatVersion 2) */
- nextOffset=indexes[Normalizer2Impl::IX_SMALL_FCD_OFFSET+1];
- offset=nextOffset;
-
- U_ASSERT(offset==size);
- }
-
- return headerSize+size;
-}
-
-#endif // !UCONFIG_NO_NORMALIZATION
+ offset=nextOffset;
+
+ /* swap the uint16_t extraData[] */
+ nextOffset=indexes[Normalizer2Impl::IX_SMALL_FCD_OFFSET];
+ ds->swapArray16(ds, inBytes+offset, nextOffset-offset, outBytes+offset, pErrorCode);
+ offset=nextOffset;
+
+ /* no need to swap the uint8_t smallFCD[] (new in formatVersion 2) */
+ nextOffset=indexes[Normalizer2Impl::IX_SMALL_FCD_OFFSET+1];
+ offset=nextOffset;
+
+ U_ASSERT(offset==size);
+ }
+
+ return headerSize+size;
+}
+
+#endif // !UCONFIG_NO_NORMALIZATION