aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/libs/icu/i18n/collationkeys.cpp
diff options
context:
space:
mode:
authorneksard <neksard@yandex-team.ru>2022-02-10 16:45:33 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:45:33 +0300
commit1d9c550e7c38e051d7961f576013a482003a70d9 (patch)
treeb2cc84ee7850122e7ccf51d0ea21e4fa7e7a5685 /contrib/libs/icu/i18n/collationkeys.cpp
parent8f7cf138264e0caa318144bf8a2c950e0b0a8593 (diff)
downloadydb-1d9c550e7c38e051d7961f576013a482003a70d9.tar.gz
Restoring authorship annotation for <neksard@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'contrib/libs/icu/i18n/collationkeys.cpp')
-rw-r--r--contrib/libs/icu/i18n/collationkeys.cpp1332
1 files changed, 666 insertions, 666 deletions
diff --git a/contrib/libs/icu/i18n/collationkeys.cpp b/contrib/libs/icu/i18n/collationkeys.cpp
index 576cb93fe0..b5c322fb44 100644
--- a/contrib/libs/icu/i18n/collationkeys.cpp
+++ b/contrib/libs/icu/i18n/collationkeys.cpp
@@ -1,673 +1,673 @@
// © 2016 and later: Unicode, Inc. and others.
-// License & terms of use: http://www.unicode.org/copyright.html
-/*
-*******************************************************************************
-* Copyright (C) 2012-2015, International Business Machines
-* Corporation and others. All Rights Reserved.
-*******************************************************************************
-* collationkeys.cpp
-*
-* created on: 2012sep02
-* created by: Markus W. Scherer
-*/
-
-#include "unicode/utypes.h"
-
-#if !UCONFIG_NO_COLLATION
-
-#include "unicode/bytestream.h"
-#include "collation.h"
-#include "collationiterator.h"
-#include "collationkeys.h"
-#include "collationsettings.h"
-#include "uassert.h"
-
-U_NAMESPACE_BEGIN
-
-SortKeyByteSink::~SortKeyByteSink() {}
-
-void
-SortKeyByteSink::Append(const char *bytes, int32_t n) {
- if (n <= 0 || bytes == NULL) {
- return;
- }
- if (ignore_ > 0) {
- int32_t ignoreRest = ignore_ - n;
- if (ignoreRest >= 0) {
- ignore_ = ignoreRest;
- return;
- } else {
- bytes += ignore_;
- n = -ignoreRest;
- ignore_ = 0;
- }
- }
- int32_t length = appended_;
- appended_ += n;
- if ((buffer_ + length) == bytes) {
- return; // the caller used GetAppendBuffer() and wrote the bytes already
- }
- int32_t available = capacity_ - length;
- if (n <= available) {
- uprv_memcpy(buffer_ + length, bytes, n);
- } else {
- AppendBeyondCapacity(bytes, n, length);
- }
-}
-
-char *
-SortKeyByteSink::GetAppendBuffer(int32_t min_capacity,
- int32_t desired_capacity_hint,
- char *scratch,
- int32_t scratch_capacity,
- int32_t *result_capacity) {
- if (min_capacity < 1 || scratch_capacity < min_capacity) {
- *result_capacity = 0;
- return NULL;
- }
- if (ignore_ > 0) {
- // Do not write ignored bytes right at the end of the buffer.
- *result_capacity = scratch_capacity;
- return scratch;
- }
- int32_t available = capacity_ - appended_;
- if (available >= min_capacity) {
- *result_capacity = available;
- return buffer_ + appended_;
- } else if (Resize(desired_capacity_hint, appended_)) {
- *result_capacity = capacity_ - appended_;
- return buffer_ + appended_;
- } else {
- *result_capacity = scratch_capacity;
- return scratch;
- }
-}
-
-namespace {
-
-/**
- * uint8_t byte buffer, similar to CharString but simpler.
- */
-class SortKeyLevel : public UMemory {
-public:
- SortKeyLevel() : len(0), ok(TRUE) {}
- ~SortKeyLevel() {}
-
- /** @return FALSE if memory allocation failed */
- UBool isOk() const { return ok; }
- UBool isEmpty() const { return len == 0; }
- int32_t length() const { return len; }
- const uint8_t *data() const { return buffer.getAlias(); }
- uint8_t operator[](int32_t index) const { return buffer[index]; }
-
- uint8_t *data() { return buffer.getAlias(); }
-
- void appendByte(uint32_t b);
- void appendWeight16(uint32_t w);
- void appendWeight32(uint32_t w);
- void appendReverseWeight16(uint32_t w);
-
- /** Appends all but the last byte to the sink. The last byte should be the 01 terminator. */
- void appendTo(ByteSink &sink) const {
- U_ASSERT(len > 0 && buffer[len - 1] == 1);
- sink.Append(reinterpret_cast<const char *>(buffer.getAlias()), len - 1);
- }
-
-private:
- MaybeStackArray<uint8_t, 40> buffer;
- int32_t len;
- UBool ok;
-
- UBool ensureCapacity(int32_t appendCapacity);
-
- SortKeyLevel(const SortKeyLevel &other); // forbid copying of this class
- SortKeyLevel &operator=(const SortKeyLevel &other); // forbid copying of this class
-};
-
-void SortKeyLevel::appendByte(uint32_t b) {
- if(len < buffer.getCapacity() || ensureCapacity(1)) {
- buffer[len++] = (uint8_t)b;
- }
-}
-
-void
-SortKeyLevel::appendWeight16(uint32_t w) {
- U_ASSERT((w & 0xffff) != 0);
- uint8_t b0 = (uint8_t)(w >> 8);
- uint8_t b1 = (uint8_t)w;
- int32_t appendLength = (b1 == 0) ? 1 : 2;
- if((len + appendLength) <= buffer.getCapacity() || ensureCapacity(appendLength)) {
- buffer[len++] = b0;
- if(b1 != 0) {
- buffer[len++] = b1;
- }
- }
-}
-
-void
-SortKeyLevel::appendWeight32(uint32_t w) {
- U_ASSERT(w != 0);
- uint8_t bytes[4] = { (uint8_t)(w >> 24), (uint8_t)(w >> 16), (uint8_t)(w >> 8), (uint8_t)w };
- int32_t appendLength = (bytes[1] == 0) ? 1 : (bytes[2] == 0) ? 2 : (bytes[3] == 0) ? 3 : 4;
- if((len + appendLength) <= buffer.getCapacity() || ensureCapacity(appendLength)) {
- buffer[len++] = bytes[0];
- if(bytes[1] != 0) {
- buffer[len++] = bytes[1];
- if(bytes[2] != 0) {
- buffer[len++] = bytes[2];
- if(bytes[3] != 0) {
- buffer[len++] = bytes[3];
- }
- }
- }
- }
-}
-
-void
-SortKeyLevel::appendReverseWeight16(uint32_t w) {
- U_ASSERT((w & 0xffff) != 0);
- uint8_t b0 = (uint8_t)(w >> 8);
- uint8_t b1 = (uint8_t)w;
- int32_t appendLength = (b1 == 0) ? 1 : 2;
- if((len + appendLength) <= buffer.getCapacity() || ensureCapacity(appendLength)) {
- if(b1 == 0) {
- buffer[len++] = b0;
- } else {
- buffer[len] = b1;
- buffer[len + 1] = b0;
- len += 2;
- }
- }
-}
-
-UBool SortKeyLevel::ensureCapacity(int32_t appendCapacity) {
- if(!ok) {
- return FALSE;
- }
- int32_t newCapacity = 2 * buffer.getCapacity();
- int32_t altCapacity = len + 2 * appendCapacity;
- if (newCapacity < altCapacity) {
- newCapacity = altCapacity;
- }
- if (newCapacity < 200) {
- newCapacity = 200;
- }
- if(buffer.resize(newCapacity, len)==NULL) {
- return ok = FALSE;
- }
- return TRUE;
-}
-
-} // namespace
-
-CollationKeys::LevelCallback::~LevelCallback() {}
-
-UBool
-CollationKeys::LevelCallback::needToWrite(Collation::Level /*level*/) { return TRUE; }
-
-/**
- * Map from collation strength (UColAttributeValue)
- * to a mask of Collation::Level bits up to that strength,
- * excluding the CASE_LEVEL which is independent of the strength,
- * and excluding IDENTICAL_LEVEL which this function does not write.
- */
-static const uint32_t levelMasks[UCOL_STRENGTH_LIMIT] = {
- 2, // UCOL_PRIMARY -> PRIMARY_LEVEL
- 6, // UCOL_SECONDARY -> up to SECONDARY_LEVEL
- 0x16, // UCOL_TERTIARY -> up to TERTIARY_LEVEL
- 0x36, // UCOL_QUATERNARY -> up to QUATERNARY_LEVEL
- 0, 0, 0, 0,
- 0, 0, 0, 0,
- 0, 0, 0,
- 0x36 // UCOL_IDENTICAL -> up to QUATERNARY_LEVEL
-};
-
-void
-CollationKeys::writeSortKeyUpToQuaternary(CollationIterator &iter,
- const UBool *compressibleBytes,
- const CollationSettings &settings,
- SortKeyByteSink &sink,
- Collation::Level minLevel, LevelCallback &callback,
- UBool preflight, UErrorCode &errorCode) {
- if(U_FAILURE(errorCode)) { return; }
-
- int32_t options = settings.options;
- // Set of levels to process and write.
- uint32_t levels = levelMasks[CollationSettings::getStrength(options)];
- if((options & CollationSettings::CASE_LEVEL) != 0) {
- levels |= Collation::CASE_LEVEL_FLAG;
- }
- // Minus the levels below minLevel.
- levels &= ~(((uint32_t)1 << minLevel) - 1);
- if(levels == 0) { return; }
-
- uint32_t variableTop;
- if((options & CollationSettings::ALTERNATE_MASK) == 0) {
- variableTop = 0;
- } else {
- // +1 so that we can use "<" and primary ignorables test out early.
- variableTop = settings.variableTop + 1;
- }
-
- uint32_t tertiaryMask = CollationSettings::getTertiaryMask(options);
-
- SortKeyLevel cases;
- SortKeyLevel secondaries;
- SortKeyLevel tertiaries;
- SortKeyLevel quaternaries;
-
- uint32_t prevReorderedPrimary = 0; // 0==no compression
- int32_t commonCases = 0;
- int32_t commonSecondaries = 0;
- int32_t commonTertiaries = 0;
- int32_t commonQuaternaries = 0;
-
- uint32_t prevSecondary = 0;
- int32_t secSegmentStart = 0;
-
- for(;;) {
- // No need to keep all CEs in the buffer when we write a sort key.
- iter.clearCEsIfNoneRemaining();
- int64_t ce = iter.nextCE(errorCode);
- uint32_t p = (uint32_t)(ce >> 32);
- if(p < variableTop && p > Collation::MERGE_SEPARATOR_PRIMARY) {
- // Variable CE, shift it to quaternary level.
- // Ignore all following primary ignorables, and shift further variable CEs.
- if(commonQuaternaries != 0) {
- --commonQuaternaries;
- while(commonQuaternaries >= QUAT_COMMON_MAX_COUNT) {
- quaternaries.appendByte(QUAT_COMMON_MIDDLE);
- commonQuaternaries -= QUAT_COMMON_MAX_COUNT;
- }
- // Shifted primary weights are lower than the common weight.
- quaternaries.appendByte(QUAT_COMMON_LOW + commonQuaternaries);
- commonQuaternaries = 0;
- }
- do {
- if((levels & Collation::QUATERNARY_LEVEL_FLAG) != 0) {
- if(settings.hasReordering()) {
- p = settings.reorder(p);
- }
- if((p >> 24) >= QUAT_SHIFTED_LIMIT_BYTE) {
- // Prevent shifted primary lead bytes from
- // overlapping with the common compression range.
- quaternaries.appendByte(QUAT_SHIFTED_LIMIT_BYTE);
- }
- quaternaries.appendWeight32(p);
- }
- do {
- ce = iter.nextCE(errorCode);
- p = (uint32_t)(ce >> 32);
- } while(p == 0);
- } while(p < variableTop && p > Collation::MERGE_SEPARATOR_PRIMARY);
- }
- // ce could be primary ignorable, or NO_CE, or the merge separator,
- // or a regular primary CE, but it is not variable.
- // If ce==NO_CE, then write nothing for the primary level but
- // terminate compression on all levels and then exit the loop.
- if(p > Collation::NO_CE_PRIMARY && (levels & Collation::PRIMARY_LEVEL_FLAG) != 0) {
- // Test the un-reordered primary for compressibility.
- UBool isCompressible = compressibleBytes[p >> 24];
- if(settings.hasReordering()) {
- p = settings.reorder(p);
- }
- uint32_t p1 = p >> 24;
- if(!isCompressible || p1 != (prevReorderedPrimary >> 24)) {
- if(prevReorderedPrimary != 0) {
- if(p < prevReorderedPrimary) {
- // No primary compression terminator
- // at the end of the level or merged segment.
- if(p1 > Collation::MERGE_SEPARATOR_BYTE) {
- sink.Append(Collation::PRIMARY_COMPRESSION_LOW_BYTE);
- }
- } else {
- sink.Append(Collation::PRIMARY_COMPRESSION_HIGH_BYTE);
- }
- }
- sink.Append(p1);
- if(isCompressible) {
- prevReorderedPrimary = p;
- } else {
- prevReorderedPrimary = 0;
- }
- }
- char p2 = (char)(p >> 16);
- if(p2 != 0) {
- char buffer[3] = { p2, (char)(p >> 8), (char)p };
- sink.Append(buffer, (buffer[1] == 0) ? 1 : (buffer[2] == 0) ? 2 : 3);
- }
- // Optimization for internalNextSortKeyPart():
- // When the primary level overflows we can stop because we need not
- // calculate (preflight) the whole sort key length.
- if(!preflight && sink.Overflowed()) {
- if(U_SUCCESS(errorCode) && !sink.IsOk()) {
- errorCode = U_MEMORY_ALLOCATION_ERROR;
- }
- return;
- }
- }
-
- uint32_t lower32 = (uint32_t)ce;
- if(lower32 == 0) { continue; } // completely ignorable, no secondary/case/tertiary/quaternary
-
- if((levels & Collation::SECONDARY_LEVEL_FLAG) != 0) {
- uint32_t s = lower32 >> 16;
- if(s == 0) {
- // secondary ignorable
- } else if(s == Collation::COMMON_WEIGHT16 &&
- ((options & CollationSettings::BACKWARD_SECONDARY) == 0 ||
- p != Collation::MERGE_SEPARATOR_PRIMARY)) {
- // s is a common secondary weight, and
- // backwards-secondary is off or the ce is not the merge separator.
- ++commonSecondaries;
- } else if((options & CollationSettings::BACKWARD_SECONDARY) == 0) {
- if(commonSecondaries != 0) {
- --commonSecondaries;
- while(commonSecondaries >= SEC_COMMON_MAX_COUNT) {
- secondaries.appendByte(SEC_COMMON_MIDDLE);
- commonSecondaries -= SEC_COMMON_MAX_COUNT;
- }
- uint32_t b;
- if(s < Collation::COMMON_WEIGHT16) {
- b = SEC_COMMON_LOW + commonSecondaries;
- } else {
- b = SEC_COMMON_HIGH - commonSecondaries;
- }
- secondaries.appendByte(b);
- commonSecondaries = 0;
- }
- secondaries.appendWeight16(s);
- } else {
- if(commonSecondaries != 0) {
- --commonSecondaries;
- // Append reverse weights. The level will be re-reversed later.
- int32_t remainder = commonSecondaries % SEC_COMMON_MAX_COUNT;
- uint32_t b;
- if(prevSecondary < Collation::COMMON_WEIGHT16) {
- b = SEC_COMMON_LOW + remainder;
- } else {
- b = SEC_COMMON_HIGH - remainder;
- }
- secondaries.appendByte(b);
- commonSecondaries -= remainder;
- // commonSecondaries is now a multiple of SEC_COMMON_MAX_COUNT.
- while(commonSecondaries > 0) { // same as >= SEC_COMMON_MAX_COUNT
- secondaries.appendByte(SEC_COMMON_MIDDLE);
- commonSecondaries -= SEC_COMMON_MAX_COUNT;
- }
- // commonSecondaries == 0
- }
- if(0 < p && p <= Collation::MERGE_SEPARATOR_PRIMARY) {
- // The backwards secondary level compares secondary weights backwards
- // within segments separated by the merge separator (U+FFFE).
- uint8_t *secs = secondaries.data();
- int32_t last = secondaries.length() - 1;
- if(secSegmentStart < last) {
+// License & terms of use: http://www.unicode.org/copyright.html
+/*
+*******************************************************************************
+* Copyright (C) 2012-2015, International Business Machines
+* Corporation and others. All Rights Reserved.
+*******************************************************************************
+* collationkeys.cpp
+*
+* created on: 2012sep02
+* created by: Markus W. Scherer
+*/
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_COLLATION
+
+#include "unicode/bytestream.h"
+#include "collation.h"
+#include "collationiterator.h"
+#include "collationkeys.h"
+#include "collationsettings.h"
+#include "uassert.h"
+
+U_NAMESPACE_BEGIN
+
+SortKeyByteSink::~SortKeyByteSink() {}
+
+void
+SortKeyByteSink::Append(const char *bytes, int32_t n) {
+ if (n <= 0 || bytes == NULL) {
+ return;
+ }
+ if (ignore_ > 0) {
+ int32_t ignoreRest = ignore_ - n;
+ if (ignoreRest >= 0) {
+ ignore_ = ignoreRest;
+ return;
+ } else {
+ bytes += ignore_;
+ n = -ignoreRest;
+ ignore_ = 0;
+ }
+ }
+ int32_t length = appended_;
+ appended_ += n;
+ if ((buffer_ + length) == bytes) {
+ return; // the caller used GetAppendBuffer() and wrote the bytes already
+ }
+ int32_t available = capacity_ - length;
+ if (n <= available) {
+ uprv_memcpy(buffer_ + length, bytes, n);
+ } else {
+ AppendBeyondCapacity(bytes, n, length);
+ }
+}
+
+char *
+SortKeyByteSink::GetAppendBuffer(int32_t min_capacity,
+ int32_t desired_capacity_hint,
+ char *scratch,
+ int32_t scratch_capacity,
+ int32_t *result_capacity) {
+ if (min_capacity < 1 || scratch_capacity < min_capacity) {
+ *result_capacity = 0;
+ return NULL;
+ }
+ if (ignore_ > 0) {
+ // Do not write ignored bytes right at the end of the buffer.
+ *result_capacity = scratch_capacity;
+ return scratch;
+ }
+ int32_t available = capacity_ - appended_;
+ if (available >= min_capacity) {
+ *result_capacity = available;
+ return buffer_ + appended_;
+ } else if (Resize(desired_capacity_hint, appended_)) {
+ *result_capacity = capacity_ - appended_;
+ return buffer_ + appended_;
+ } else {
+ *result_capacity = scratch_capacity;
+ return scratch;
+ }
+}
+
+namespace {
+
+/**
+ * uint8_t byte buffer, similar to CharString but simpler.
+ */
+class SortKeyLevel : public UMemory {
+public:
+ SortKeyLevel() : len(0), ok(TRUE) {}
+ ~SortKeyLevel() {}
+
+ /** @return FALSE if memory allocation failed */
+ UBool isOk() const { return ok; }
+ UBool isEmpty() const { return len == 0; }
+ int32_t length() const { return len; }
+ const uint8_t *data() const { return buffer.getAlias(); }
+ uint8_t operator[](int32_t index) const { return buffer[index]; }
+
+ uint8_t *data() { return buffer.getAlias(); }
+
+ void appendByte(uint32_t b);
+ void appendWeight16(uint32_t w);
+ void appendWeight32(uint32_t w);
+ void appendReverseWeight16(uint32_t w);
+
+ /** Appends all but the last byte to the sink. The last byte should be the 01 terminator. */
+ void appendTo(ByteSink &sink) const {
+ U_ASSERT(len > 0 && buffer[len - 1] == 1);
+ sink.Append(reinterpret_cast<const char *>(buffer.getAlias()), len - 1);
+ }
+
+private:
+ MaybeStackArray<uint8_t, 40> buffer;
+ int32_t len;
+ UBool ok;
+
+ UBool ensureCapacity(int32_t appendCapacity);
+
+ SortKeyLevel(const SortKeyLevel &other); // forbid copying of this class
+ SortKeyLevel &operator=(const SortKeyLevel &other); // forbid copying of this class
+};
+
+void SortKeyLevel::appendByte(uint32_t b) {
+ if(len < buffer.getCapacity() || ensureCapacity(1)) {
+ buffer[len++] = (uint8_t)b;
+ }
+}
+
+void
+SortKeyLevel::appendWeight16(uint32_t w) {
+ U_ASSERT((w & 0xffff) != 0);
+ uint8_t b0 = (uint8_t)(w >> 8);
+ uint8_t b1 = (uint8_t)w;
+ int32_t appendLength = (b1 == 0) ? 1 : 2;
+ if((len + appendLength) <= buffer.getCapacity() || ensureCapacity(appendLength)) {
+ buffer[len++] = b0;
+ if(b1 != 0) {
+ buffer[len++] = b1;
+ }
+ }
+}
+
+void
+SortKeyLevel::appendWeight32(uint32_t w) {
+ U_ASSERT(w != 0);
+ uint8_t bytes[4] = { (uint8_t)(w >> 24), (uint8_t)(w >> 16), (uint8_t)(w >> 8), (uint8_t)w };
+ int32_t appendLength = (bytes[1] == 0) ? 1 : (bytes[2] == 0) ? 2 : (bytes[3] == 0) ? 3 : 4;
+ if((len + appendLength) <= buffer.getCapacity() || ensureCapacity(appendLength)) {
+ buffer[len++] = bytes[0];
+ if(bytes[1] != 0) {
+ buffer[len++] = bytes[1];
+ if(bytes[2] != 0) {
+ buffer[len++] = bytes[2];
+ if(bytes[3] != 0) {
+ buffer[len++] = bytes[3];
+ }
+ }
+ }
+ }
+}
+
+void
+SortKeyLevel::appendReverseWeight16(uint32_t w) {
+ U_ASSERT((w & 0xffff) != 0);
+ uint8_t b0 = (uint8_t)(w >> 8);
+ uint8_t b1 = (uint8_t)w;
+ int32_t appendLength = (b1 == 0) ? 1 : 2;
+ if((len + appendLength) <= buffer.getCapacity() || ensureCapacity(appendLength)) {
+ if(b1 == 0) {
+ buffer[len++] = b0;
+ } else {
+ buffer[len] = b1;
+ buffer[len + 1] = b0;
+ len += 2;
+ }
+ }
+}
+
+UBool SortKeyLevel::ensureCapacity(int32_t appendCapacity) {
+ if(!ok) {
+ return FALSE;
+ }
+ int32_t newCapacity = 2 * buffer.getCapacity();
+ int32_t altCapacity = len + 2 * appendCapacity;
+ if (newCapacity < altCapacity) {
+ newCapacity = altCapacity;
+ }
+ if (newCapacity < 200) {
+ newCapacity = 200;
+ }
+ if(buffer.resize(newCapacity, len)==NULL) {
+ return ok = FALSE;
+ }
+ return TRUE;
+}
+
+} // namespace
+
+CollationKeys::LevelCallback::~LevelCallback() {}
+
+UBool
+CollationKeys::LevelCallback::needToWrite(Collation::Level /*level*/) { return TRUE; }
+
+/**
+ * Map from collation strength (UColAttributeValue)
+ * to a mask of Collation::Level bits up to that strength,
+ * excluding the CASE_LEVEL which is independent of the strength,
+ * and excluding IDENTICAL_LEVEL which this function does not write.
+ */
+static const uint32_t levelMasks[UCOL_STRENGTH_LIMIT] = {
+ 2, // UCOL_PRIMARY -> PRIMARY_LEVEL
+ 6, // UCOL_SECONDARY -> up to SECONDARY_LEVEL
+ 0x16, // UCOL_TERTIARY -> up to TERTIARY_LEVEL
+ 0x36, // UCOL_QUATERNARY -> up to QUATERNARY_LEVEL
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0,
+ 0x36 // UCOL_IDENTICAL -> up to QUATERNARY_LEVEL
+};
+
+void
+CollationKeys::writeSortKeyUpToQuaternary(CollationIterator &iter,
+ const UBool *compressibleBytes,
+ const CollationSettings &settings,
+ SortKeyByteSink &sink,
+ Collation::Level minLevel, LevelCallback &callback,
+ UBool preflight, UErrorCode &errorCode) {
+ if(U_FAILURE(errorCode)) { return; }
+
+ int32_t options = settings.options;
+ // Set of levels to process and write.
+ uint32_t levels = levelMasks[CollationSettings::getStrength(options)];
+ if((options & CollationSettings::CASE_LEVEL) != 0) {
+ levels |= Collation::CASE_LEVEL_FLAG;
+ }
+ // Minus the levels below minLevel.
+ levels &= ~(((uint32_t)1 << minLevel) - 1);
+ if(levels == 0) { return; }
+
+ uint32_t variableTop;
+ if((options & CollationSettings::ALTERNATE_MASK) == 0) {
+ variableTop = 0;
+ } else {
+ // +1 so that we can use "<" and primary ignorables test out early.
+ variableTop = settings.variableTop + 1;
+ }
+
+ uint32_t tertiaryMask = CollationSettings::getTertiaryMask(options);
+
+ SortKeyLevel cases;
+ SortKeyLevel secondaries;
+ SortKeyLevel tertiaries;
+ SortKeyLevel quaternaries;
+
+ uint32_t prevReorderedPrimary = 0; // 0==no compression
+ int32_t commonCases = 0;
+ int32_t commonSecondaries = 0;
+ int32_t commonTertiaries = 0;
+ int32_t commonQuaternaries = 0;
+
+ uint32_t prevSecondary = 0;
+ int32_t secSegmentStart = 0;
+
+ for(;;) {
+ // No need to keep all CEs in the buffer when we write a sort key.
+ iter.clearCEsIfNoneRemaining();
+ int64_t ce = iter.nextCE(errorCode);
+ uint32_t p = (uint32_t)(ce >> 32);
+ if(p < variableTop && p > Collation::MERGE_SEPARATOR_PRIMARY) {
+ // Variable CE, shift it to quaternary level.
+ // Ignore all following primary ignorables, and shift further variable CEs.
+ if(commonQuaternaries != 0) {
+ --commonQuaternaries;
+ while(commonQuaternaries >= QUAT_COMMON_MAX_COUNT) {
+ quaternaries.appendByte(QUAT_COMMON_MIDDLE);
+ commonQuaternaries -= QUAT_COMMON_MAX_COUNT;
+ }
+ // Shifted primary weights are lower than the common weight.
+ quaternaries.appendByte(QUAT_COMMON_LOW + commonQuaternaries);
+ commonQuaternaries = 0;
+ }
+ do {
+ if((levels & Collation::QUATERNARY_LEVEL_FLAG) != 0) {
+ if(settings.hasReordering()) {
+ p = settings.reorder(p);
+ }
+ if((p >> 24) >= QUAT_SHIFTED_LIMIT_BYTE) {
+ // Prevent shifted primary lead bytes from
+ // overlapping with the common compression range.
+ quaternaries.appendByte(QUAT_SHIFTED_LIMIT_BYTE);
+ }
+ quaternaries.appendWeight32(p);
+ }
+ do {
+ ce = iter.nextCE(errorCode);
+ p = (uint32_t)(ce >> 32);
+ } while(p == 0);
+ } while(p < variableTop && p > Collation::MERGE_SEPARATOR_PRIMARY);
+ }
+ // ce could be primary ignorable, or NO_CE, or the merge separator,
+ // or a regular primary CE, but it is not variable.
+ // If ce==NO_CE, then write nothing for the primary level but
+ // terminate compression on all levels and then exit the loop.
+ if(p > Collation::NO_CE_PRIMARY && (levels & Collation::PRIMARY_LEVEL_FLAG) != 0) {
+ // Test the un-reordered primary for compressibility.
+ UBool isCompressible = compressibleBytes[p >> 24];
+ if(settings.hasReordering()) {
+ p = settings.reorder(p);
+ }
+ uint32_t p1 = p >> 24;
+ if(!isCompressible || p1 != (prevReorderedPrimary >> 24)) {
+ if(prevReorderedPrimary != 0) {
+ if(p < prevReorderedPrimary) {
+ // No primary compression terminator
+ // at the end of the level or merged segment.
+ if(p1 > Collation::MERGE_SEPARATOR_BYTE) {
+ sink.Append(Collation::PRIMARY_COMPRESSION_LOW_BYTE);
+ }
+ } else {
+ sink.Append(Collation::PRIMARY_COMPRESSION_HIGH_BYTE);
+ }
+ }
+ sink.Append(p1);
+ if(isCompressible) {
+ prevReorderedPrimary = p;
+ } else {
+ prevReorderedPrimary = 0;
+ }
+ }
+ char p2 = (char)(p >> 16);
+ if(p2 != 0) {
+ char buffer[3] = { p2, (char)(p >> 8), (char)p };
+ sink.Append(buffer, (buffer[1] == 0) ? 1 : (buffer[2] == 0) ? 2 : 3);
+ }
+ // Optimization for internalNextSortKeyPart():
+ // When the primary level overflows we can stop because we need not
+ // calculate (preflight) the whole sort key length.
+ if(!preflight && sink.Overflowed()) {
+ if(U_SUCCESS(errorCode) && !sink.IsOk()) {
+ errorCode = U_MEMORY_ALLOCATION_ERROR;
+ }
+ return;
+ }
+ }
+
+ uint32_t lower32 = (uint32_t)ce;
+ if(lower32 == 0) { continue; } // completely ignorable, no secondary/case/tertiary/quaternary
+
+ if((levels & Collation::SECONDARY_LEVEL_FLAG) != 0) {
+ uint32_t s = lower32 >> 16;
+ if(s == 0) {
+ // secondary ignorable
+ } else if(s == Collation::COMMON_WEIGHT16 &&
+ ((options & CollationSettings::BACKWARD_SECONDARY) == 0 ||
+ p != Collation::MERGE_SEPARATOR_PRIMARY)) {
+ // s is a common secondary weight, and
+ // backwards-secondary is off or the ce is not the merge separator.
+ ++commonSecondaries;
+ } else if((options & CollationSettings::BACKWARD_SECONDARY) == 0) {
+ if(commonSecondaries != 0) {
+ --commonSecondaries;
+ while(commonSecondaries >= SEC_COMMON_MAX_COUNT) {
+ secondaries.appendByte(SEC_COMMON_MIDDLE);
+ commonSecondaries -= SEC_COMMON_MAX_COUNT;
+ }
+ uint32_t b;
+ if(s < Collation::COMMON_WEIGHT16) {
+ b = SEC_COMMON_LOW + commonSecondaries;
+ } else {
+ b = SEC_COMMON_HIGH - commonSecondaries;
+ }
+ secondaries.appendByte(b);
+ commonSecondaries = 0;
+ }
+ secondaries.appendWeight16(s);
+ } else {
+ if(commonSecondaries != 0) {
+ --commonSecondaries;
+ // Append reverse weights. The level will be re-reversed later.
+ int32_t remainder = commonSecondaries % SEC_COMMON_MAX_COUNT;
+ uint32_t b;
+ if(prevSecondary < Collation::COMMON_WEIGHT16) {
+ b = SEC_COMMON_LOW + remainder;
+ } else {
+ b = SEC_COMMON_HIGH - remainder;
+ }
+ secondaries.appendByte(b);
+ commonSecondaries -= remainder;
+ // commonSecondaries is now a multiple of SEC_COMMON_MAX_COUNT.
+ while(commonSecondaries > 0) { // same as >= SEC_COMMON_MAX_COUNT
+ secondaries.appendByte(SEC_COMMON_MIDDLE);
+ commonSecondaries -= SEC_COMMON_MAX_COUNT;
+ }
+ // commonSecondaries == 0
+ }
+ if(0 < p && p <= Collation::MERGE_SEPARATOR_PRIMARY) {
+ // The backwards secondary level compares secondary weights backwards
+ // within segments separated by the merge separator (U+FFFE).
+ uint8_t *secs = secondaries.data();
+ int32_t last = secondaries.length() - 1;
+ if(secSegmentStart < last) {
uint8_t *q = secs + secSegmentStart;
uint8_t *r = secs + last;
- do {
+ do {
uint8_t b = *q;
*q++ = *r;
*r-- = b;
} while(q < r);
- }
- secondaries.appendByte(p == Collation::NO_CE_PRIMARY ?
- Collation::LEVEL_SEPARATOR_BYTE : Collation::MERGE_SEPARATOR_BYTE);
- prevSecondary = 0;
- secSegmentStart = secondaries.length();
- } else {
- secondaries.appendReverseWeight16(s);
- prevSecondary = s;
- }
- }
- }
-
- if((levels & Collation::CASE_LEVEL_FLAG) != 0) {
- if((CollationSettings::getStrength(options) == UCOL_PRIMARY) ?
- p == 0 : lower32 <= 0xffff) {
- // Primary+caseLevel: Ignore case level weights of primary ignorables.
- // Otherwise: Ignore case level weights of secondary ignorables.
- // For details see the comments in the CollationCompare class.
- } else {
- uint32_t c = (lower32 >> 8) & 0xff; // case bits & tertiary lead byte
- U_ASSERT((c & 0xc0) != 0xc0);
- if((c & 0xc0) == 0 && c > Collation::LEVEL_SEPARATOR_BYTE) {
- ++commonCases;
- } else {
- if((options & CollationSettings::UPPER_FIRST) == 0) {
- // lowerFirst: Compress common weights to nibbles 1..7..13, mixed=14, upper=15.
- // If there are only common (=lowest) weights in the whole level,
- // then we need not write anything.
- // Level length differences are handled already on the next-higher level.
- if(commonCases != 0 &&
- (c > Collation::LEVEL_SEPARATOR_BYTE || !cases.isEmpty())) {
- --commonCases;
- while(commonCases >= CASE_LOWER_FIRST_COMMON_MAX_COUNT) {
- cases.appendByte(CASE_LOWER_FIRST_COMMON_MIDDLE << 4);
- commonCases -= CASE_LOWER_FIRST_COMMON_MAX_COUNT;
- }
- uint32_t b;
- if(c <= Collation::LEVEL_SEPARATOR_BYTE) {
- b = CASE_LOWER_FIRST_COMMON_LOW + commonCases;
- } else {
- b = CASE_LOWER_FIRST_COMMON_HIGH - commonCases;
- }
- cases.appendByte(b << 4);
- commonCases = 0;
- }
- if(c > Collation::LEVEL_SEPARATOR_BYTE) {
- c = (CASE_LOWER_FIRST_COMMON_HIGH + (c >> 6)) << 4; // 14 or 15
- }
- } else {
- // upperFirst: Compress common weights to nibbles 3..15, mixed=2, upper=1.
- // The compressed common case weights only go up from the "low" value
- // because with upperFirst the common weight is the highest one.
- if(commonCases != 0) {
- --commonCases;
- while(commonCases >= CASE_UPPER_FIRST_COMMON_MAX_COUNT) {
- cases.appendByte(CASE_UPPER_FIRST_COMMON_LOW << 4);
- commonCases -= CASE_UPPER_FIRST_COMMON_MAX_COUNT;
- }
- cases.appendByte((CASE_UPPER_FIRST_COMMON_LOW + commonCases) << 4);
- commonCases = 0;
- }
- if(c > Collation::LEVEL_SEPARATOR_BYTE) {
- c = (CASE_UPPER_FIRST_COMMON_LOW - (c >> 6)) << 4; // 2 or 1
- }
- }
- // c is a separator byte 01,
- // or a left-shifted nibble 0x10, 0x20, ... 0xf0.
- cases.appendByte(c);
- }
- }
- }
-
- if((levels & Collation::TERTIARY_LEVEL_FLAG) != 0) {
- uint32_t t = lower32 & tertiaryMask;
- U_ASSERT((lower32 & 0xc000) != 0xc000);
- if(t == Collation::COMMON_WEIGHT16) {
- ++commonTertiaries;
- } else if((tertiaryMask & 0x8000) == 0) {
- // Tertiary weights without case bits.
- // Move lead bytes 06..3F to C6..FF for a large common-weight range.
- if(commonTertiaries != 0) {
- --commonTertiaries;
- while(commonTertiaries >= TER_ONLY_COMMON_MAX_COUNT) {
- tertiaries.appendByte(TER_ONLY_COMMON_MIDDLE);
- commonTertiaries -= TER_ONLY_COMMON_MAX_COUNT;
- }
- uint32_t b;
- if(t < Collation::COMMON_WEIGHT16) {
- b = TER_ONLY_COMMON_LOW + commonTertiaries;
- } else {
- b = TER_ONLY_COMMON_HIGH - commonTertiaries;
- }
- tertiaries.appendByte(b);
- commonTertiaries = 0;
- }
- if(t > Collation::COMMON_WEIGHT16) { t += 0xc000; }
- tertiaries.appendWeight16(t);
- } else if((options & CollationSettings::UPPER_FIRST) == 0) {
- // Tertiary weights with caseFirst=lowerFirst.
- // Move lead bytes 06..BF to 46..FF for the common-weight range.
- if(commonTertiaries != 0) {
- --commonTertiaries;
- while(commonTertiaries >= TER_LOWER_FIRST_COMMON_MAX_COUNT) {
- tertiaries.appendByte(TER_LOWER_FIRST_COMMON_MIDDLE);
- commonTertiaries -= TER_LOWER_FIRST_COMMON_MAX_COUNT;
- }
- uint32_t b;
- if(t < Collation::COMMON_WEIGHT16) {
- b = TER_LOWER_FIRST_COMMON_LOW + commonTertiaries;
- } else {
- b = TER_LOWER_FIRST_COMMON_HIGH - commonTertiaries;
- }
- tertiaries.appendByte(b);
- commonTertiaries = 0;
- }
- if(t > Collation::COMMON_WEIGHT16) { t += 0x4000; }
- tertiaries.appendWeight16(t);
- } else {
- // Tertiary weights with caseFirst=upperFirst.
- // Do not change the artificial uppercase weight of a tertiary CE (0.0.ut),
- // to keep tertiary CEs well-formed.
- // Their case+tertiary weights must be greater than those of
- // primary and secondary CEs.
- //
- // Separator 01 -> 01 (unchanged)
- // Lowercase 02..04 -> 82..84 (includes uncased)
- // Common weight 05 -> 85..C5 (common-weight compression range)
- // Lowercase 06..3F -> C6..FF
- // Mixed case 42..7F -> 42..7F
- // Uppercase 82..BF -> 02..3F
- // Tertiary CE 86..BF -> C6..FF
- if(t <= Collation::NO_CE_WEIGHT16) {
- // Keep separators unchanged.
- } else if(lower32 > 0xffff) {
- // Invert case bits of primary & secondary CEs.
- t ^= 0xc000;
- if(t < (TER_UPPER_FIRST_COMMON_HIGH << 8)) {
- t -= 0x4000;
- }
- } else {
- // Keep uppercase bits of tertiary CEs.
- U_ASSERT(0x8600 <= t && t <= 0xbfff);
- t += 0x4000;
- }
- if(commonTertiaries != 0) {
- --commonTertiaries;
- while(commonTertiaries >= TER_UPPER_FIRST_COMMON_MAX_COUNT) {
- tertiaries.appendByte(TER_UPPER_FIRST_COMMON_MIDDLE);
- commonTertiaries -= TER_UPPER_FIRST_COMMON_MAX_COUNT;
- }
- uint32_t b;
- if(t < (TER_UPPER_FIRST_COMMON_LOW << 8)) {
- b = TER_UPPER_FIRST_COMMON_LOW + commonTertiaries;
- } else {
- b = TER_UPPER_FIRST_COMMON_HIGH - commonTertiaries;
- }
- tertiaries.appendByte(b);
- commonTertiaries = 0;
- }
- tertiaries.appendWeight16(t);
- }
- }
-
- if((levels & Collation::QUATERNARY_LEVEL_FLAG) != 0) {
- uint32_t q = lower32 & 0xffff;
- if((q & 0xc0) == 0 && q > Collation::NO_CE_WEIGHT16) {
- ++commonQuaternaries;
- } else if(q == Collation::NO_CE_WEIGHT16 &&
- (options & CollationSettings::ALTERNATE_MASK) == 0 &&
- quaternaries.isEmpty()) {
- // If alternate=non-ignorable and there are only common quaternary weights,
- // then we need not write anything.
- // The only weights greater than the merge separator and less than the common weight
- // are shifted primary weights, which are not generated for alternate=non-ignorable.
- // There are also exactly as many quaternary weights as tertiary weights,
- // so level length differences are handled already on tertiary level.
- // Any above-common quaternary weight will compare greater regardless.
- quaternaries.appendByte(Collation::LEVEL_SEPARATOR_BYTE);
- } else {
- if(q == Collation::NO_CE_WEIGHT16) {
- q = Collation::LEVEL_SEPARATOR_BYTE;
- } else {
- q = 0xfc + ((q >> 6) & 3);
- }
- if(commonQuaternaries != 0) {
- --commonQuaternaries;
- while(commonQuaternaries >= QUAT_COMMON_MAX_COUNT) {
- quaternaries.appendByte(QUAT_COMMON_MIDDLE);
- commonQuaternaries -= QUAT_COMMON_MAX_COUNT;
- }
- uint32_t b;
- if(q < QUAT_COMMON_LOW) {
- b = QUAT_COMMON_LOW + commonQuaternaries;
- } else {
- b = QUAT_COMMON_HIGH - commonQuaternaries;
- }
- quaternaries.appendByte(b);
- commonQuaternaries = 0;
- }
- quaternaries.appendByte(q);
- }
- }
-
- if((lower32 >> 24) == Collation::LEVEL_SEPARATOR_BYTE) { break; } // ce == NO_CE
- }
-
- if(U_FAILURE(errorCode)) { return; }
-
- // Append the beyond-primary levels.
- UBool ok = TRUE;
- if((levels & Collation::SECONDARY_LEVEL_FLAG) != 0) {
- if(!callback.needToWrite(Collation::SECONDARY_LEVEL)) { return; }
- ok &= secondaries.isOk();
- sink.Append(Collation::LEVEL_SEPARATOR_BYTE);
- secondaries.appendTo(sink);
- }
-
- if((levels & Collation::CASE_LEVEL_FLAG) != 0) {
- if(!callback.needToWrite(Collation::CASE_LEVEL)) { return; }
- ok &= cases.isOk();
- sink.Append(Collation::LEVEL_SEPARATOR_BYTE);
- // Write pairs of nibbles as bytes, except separator bytes as themselves.
- int32_t length = cases.length() - 1; // Ignore the trailing NO_CE.
- uint8_t b = 0;
- for(int32_t i = 0; i < length; ++i) {
- uint8_t c = (uint8_t)cases[i];
- U_ASSERT((c & 0xf) == 0 && c != 0);
- if(b == 0) {
- b = c;
- } else {
- sink.Append(b | (c >> 4));
- b = 0;
- }
- }
- if(b != 0) {
- sink.Append(b);
- }
- }
-
- if((levels & Collation::TERTIARY_LEVEL_FLAG) != 0) {
- if(!callback.needToWrite(Collation::TERTIARY_LEVEL)) { return; }
- ok &= tertiaries.isOk();
- sink.Append(Collation::LEVEL_SEPARATOR_BYTE);
- tertiaries.appendTo(sink);
- }
-
- if((levels & Collation::QUATERNARY_LEVEL_FLAG) != 0) {
- if(!callback.needToWrite(Collation::QUATERNARY_LEVEL)) { return; }
- ok &= quaternaries.isOk();
- sink.Append(Collation::LEVEL_SEPARATOR_BYTE);
- quaternaries.appendTo(sink);
- }
-
- if(!ok || !sink.IsOk()) {
- errorCode = U_MEMORY_ALLOCATION_ERROR;
- }
-}
-
-U_NAMESPACE_END
-
-#endif // !UCONFIG_NO_COLLATION
+ }
+ secondaries.appendByte(p == Collation::NO_CE_PRIMARY ?
+ Collation::LEVEL_SEPARATOR_BYTE : Collation::MERGE_SEPARATOR_BYTE);
+ prevSecondary = 0;
+ secSegmentStart = secondaries.length();
+ } else {
+ secondaries.appendReverseWeight16(s);
+ prevSecondary = s;
+ }
+ }
+ }
+
+ if((levels & Collation::CASE_LEVEL_FLAG) != 0) {
+ if((CollationSettings::getStrength(options) == UCOL_PRIMARY) ?
+ p == 0 : lower32 <= 0xffff) {
+ // Primary+caseLevel: Ignore case level weights of primary ignorables.
+ // Otherwise: Ignore case level weights of secondary ignorables.
+ // For details see the comments in the CollationCompare class.
+ } else {
+ uint32_t c = (lower32 >> 8) & 0xff; // case bits & tertiary lead byte
+ U_ASSERT((c & 0xc0) != 0xc0);
+ if((c & 0xc0) == 0 && c > Collation::LEVEL_SEPARATOR_BYTE) {
+ ++commonCases;
+ } else {
+ if((options & CollationSettings::UPPER_FIRST) == 0) {
+ // lowerFirst: Compress common weights to nibbles 1..7..13, mixed=14, upper=15.
+ // If there are only common (=lowest) weights in the whole level,
+ // then we need not write anything.
+ // Level length differences are handled already on the next-higher level.
+ if(commonCases != 0 &&
+ (c > Collation::LEVEL_SEPARATOR_BYTE || !cases.isEmpty())) {
+ --commonCases;
+ while(commonCases >= CASE_LOWER_FIRST_COMMON_MAX_COUNT) {
+ cases.appendByte(CASE_LOWER_FIRST_COMMON_MIDDLE << 4);
+ commonCases -= CASE_LOWER_FIRST_COMMON_MAX_COUNT;
+ }
+ uint32_t b;
+ if(c <= Collation::LEVEL_SEPARATOR_BYTE) {
+ b = CASE_LOWER_FIRST_COMMON_LOW + commonCases;
+ } else {
+ b = CASE_LOWER_FIRST_COMMON_HIGH - commonCases;
+ }
+ cases.appendByte(b << 4);
+ commonCases = 0;
+ }
+ if(c > Collation::LEVEL_SEPARATOR_BYTE) {
+ c = (CASE_LOWER_FIRST_COMMON_HIGH + (c >> 6)) << 4; // 14 or 15
+ }
+ } else {
+ // upperFirst: Compress common weights to nibbles 3..15, mixed=2, upper=1.
+ // The compressed common case weights only go up from the "low" value
+ // because with upperFirst the common weight is the highest one.
+ if(commonCases != 0) {
+ --commonCases;
+ while(commonCases >= CASE_UPPER_FIRST_COMMON_MAX_COUNT) {
+ cases.appendByte(CASE_UPPER_FIRST_COMMON_LOW << 4);
+ commonCases -= CASE_UPPER_FIRST_COMMON_MAX_COUNT;
+ }
+ cases.appendByte((CASE_UPPER_FIRST_COMMON_LOW + commonCases) << 4);
+ commonCases = 0;
+ }
+ if(c > Collation::LEVEL_SEPARATOR_BYTE) {
+ c = (CASE_UPPER_FIRST_COMMON_LOW - (c >> 6)) << 4; // 2 or 1
+ }
+ }
+ // c is a separator byte 01,
+ // or a left-shifted nibble 0x10, 0x20, ... 0xf0.
+ cases.appendByte(c);
+ }
+ }
+ }
+
+ if((levels & Collation::TERTIARY_LEVEL_FLAG) != 0) {
+ uint32_t t = lower32 & tertiaryMask;
+ U_ASSERT((lower32 & 0xc000) != 0xc000);
+ if(t == Collation::COMMON_WEIGHT16) {
+ ++commonTertiaries;
+ } else if((tertiaryMask & 0x8000) == 0) {
+ // Tertiary weights without case bits.
+ // Move lead bytes 06..3F to C6..FF for a large common-weight range.
+ if(commonTertiaries != 0) {
+ --commonTertiaries;
+ while(commonTertiaries >= TER_ONLY_COMMON_MAX_COUNT) {
+ tertiaries.appendByte(TER_ONLY_COMMON_MIDDLE);
+ commonTertiaries -= TER_ONLY_COMMON_MAX_COUNT;
+ }
+ uint32_t b;
+ if(t < Collation::COMMON_WEIGHT16) {
+ b = TER_ONLY_COMMON_LOW + commonTertiaries;
+ } else {
+ b = TER_ONLY_COMMON_HIGH - commonTertiaries;
+ }
+ tertiaries.appendByte(b);
+ commonTertiaries = 0;
+ }
+ if(t > Collation::COMMON_WEIGHT16) { t += 0xc000; }
+ tertiaries.appendWeight16(t);
+ } else if((options & CollationSettings::UPPER_FIRST) == 0) {
+ // Tertiary weights with caseFirst=lowerFirst.
+ // Move lead bytes 06..BF to 46..FF for the common-weight range.
+ if(commonTertiaries != 0) {
+ --commonTertiaries;
+ while(commonTertiaries >= TER_LOWER_FIRST_COMMON_MAX_COUNT) {
+ tertiaries.appendByte(TER_LOWER_FIRST_COMMON_MIDDLE);
+ commonTertiaries -= TER_LOWER_FIRST_COMMON_MAX_COUNT;
+ }
+ uint32_t b;
+ if(t < Collation::COMMON_WEIGHT16) {
+ b = TER_LOWER_FIRST_COMMON_LOW + commonTertiaries;
+ } else {
+ b = TER_LOWER_FIRST_COMMON_HIGH - commonTertiaries;
+ }
+ tertiaries.appendByte(b);
+ commonTertiaries = 0;
+ }
+ if(t > Collation::COMMON_WEIGHT16) { t += 0x4000; }
+ tertiaries.appendWeight16(t);
+ } else {
+ // Tertiary weights with caseFirst=upperFirst.
+ // Do not change the artificial uppercase weight of a tertiary CE (0.0.ut),
+ // to keep tertiary CEs well-formed.
+ // Their case+tertiary weights must be greater than those of
+ // primary and secondary CEs.
+ //
+ // Separator 01 -> 01 (unchanged)
+ // Lowercase 02..04 -> 82..84 (includes uncased)
+ // Common weight 05 -> 85..C5 (common-weight compression range)
+ // Lowercase 06..3F -> C6..FF
+ // Mixed case 42..7F -> 42..7F
+ // Uppercase 82..BF -> 02..3F
+ // Tertiary CE 86..BF -> C6..FF
+ if(t <= Collation::NO_CE_WEIGHT16) {
+ // Keep separators unchanged.
+ } else if(lower32 > 0xffff) {
+ // Invert case bits of primary & secondary CEs.
+ t ^= 0xc000;
+ if(t < (TER_UPPER_FIRST_COMMON_HIGH << 8)) {
+ t -= 0x4000;
+ }
+ } else {
+ // Keep uppercase bits of tertiary CEs.
+ U_ASSERT(0x8600 <= t && t <= 0xbfff);
+ t += 0x4000;
+ }
+ if(commonTertiaries != 0) {
+ --commonTertiaries;
+ while(commonTertiaries >= TER_UPPER_FIRST_COMMON_MAX_COUNT) {
+ tertiaries.appendByte(TER_UPPER_FIRST_COMMON_MIDDLE);
+ commonTertiaries -= TER_UPPER_FIRST_COMMON_MAX_COUNT;
+ }
+ uint32_t b;
+ if(t < (TER_UPPER_FIRST_COMMON_LOW << 8)) {
+ b = TER_UPPER_FIRST_COMMON_LOW + commonTertiaries;
+ } else {
+ b = TER_UPPER_FIRST_COMMON_HIGH - commonTertiaries;
+ }
+ tertiaries.appendByte(b);
+ commonTertiaries = 0;
+ }
+ tertiaries.appendWeight16(t);
+ }
+ }
+
+ if((levels & Collation::QUATERNARY_LEVEL_FLAG) != 0) {
+ uint32_t q = lower32 & 0xffff;
+ if((q & 0xc0) == 0 && q > Collation::NO_CE_WEIGHT16) {
+ ++commonQuaternaries;
+ } else if(q == Collation::NO_CE_WEIGHT16 &&
+ (options & CollationSettings::ALTERNATE_MASK) == 0 &&
+ quaternaries.isEmpty()) {
+ // If alternate=non-ignorable and there are only common quaternary weights,
+ // then we need not write anything.
+ // The only weights greater than the merge separator and less than the common weight
+ // are shifted primary weights, which are not generated for alternate=non-ignorable.
+ // There are also exactly as many quaternary weights as tertiary weights,
+ // so level length differences are handled already on tertiary level.
+ // Any above-common quaternary weight will compare greater regardless.
+ quaternaries.appendByte(Collation::LEVEL_SEPARATOR_BYTE);
+ } else {
+ if(q == Collation::NO_CE_WEIGHT16) {
+ q = Collation::LEVEL_SEPARATOR_BYTE;
+ } else {
+ q = 0xfc + ((q >> 6) & 3);
+ }
+ if(commonQuaternaries != 0) {
+ --commonQuaternaries;
+ while(commonQuaternaries >= QUAT_COMMON_MAX_COUNT) {
+ quaternaries.appendByte(QUAT_COMMON_MIDDLE);
+ commonQuaternaries -= QUAT_COMMON_MAX_COUNT;
+ }
+ uint32_t b;
+ if(q < QUAT_COMMON_LOW) {
+ b = QUAT_COMMON_LOW + commonQuaternaries;
+ } else {
+ b = QUAT_COMMON_HIGH - commonQuaternaries;
+ }
+ quaternaries.appendByte(b);
+ commonQuaternaries = 0;
+ }
+ quaternaries.appendByte(q);
+ }
+ }
+
+ if((lower32 >> 24) == Collation::LEVEL_SEPARATOR_BYTE) { break; } // ce == NO_CE
+ }
+
+ if(U_FAILURE(errorCode)) { return; }
+
+ // Append the beyond-primary levels.
+ UBool ok = TRUE;
+ if((levels & Collation::SECONDARY_LEVEL_FLAG) != 0) {
+ if(!callback.needToWrite(Collation::SECONDARY_LEVEL)) { return; }
+ ok &= secondaries.isOk();
+ sink.Append(Collation::LEVEL_SEPARATOR_BYTE);
+ secondaries.appendTo(sink);
+ }
+
+ if((levels & Collation::CASE_LEVEL_FLAG) != 0) {
+ if(!callback.needToWrite(Collation::CASE_LEVEL)) { return; }
+ ok &= cases.isOk();
+ sink.Append(Collation::LEVEL_SEPARATOR_BYTE);
+ // Write pairs of nibbles as bytes, except separator bytes as themselves.
+ int32_t length = cases.length() - 1; // Ignore the trailing NO_CE.
+ uint8_t b = 0;
+ for(int32_t i = 0; i < length; ++i) {
+ uint8_t c = (uint8_t)cases[i];
+ U_ASSERT((c & 0xf) == 0 && c != 0);
+ if(b == 0) {
+ b = c;
+ } else {
+ sink.Append(b | (c >> 4));
+ b = 0;
+ }
+ }
+ if(b != 0) {
+ sink.Append(b);
+ }
+ }
+
+ if((levels & Collation::TERTIARY_LEVEL_FLAG) != 0) {
+ if(!callback.needToWrite(Collation::TERTIARY_LEVEL)) { return; }
+ ok &= tertiaries.isOk();
+ sink.Append(Collation::LEVEL_SEPARATOR_BYTE);
+ tertiaries.appendTo(sink);
+ }
+
+ if((levels & Collation::QUATERNARY_LEVEL_FLAG) != 0) {
+ if(!callback.needToWrite(Collation::QUATERNARY_LEVEL)) { return; }
+ ok &= quaternaries.isOk();
+ sink.Append(Collation::LEVEL_SEPARATOR_BYTE);
+ quaternaries.appendTo(sink);
+ }
+
+ if(!ok || !sink.IsOk()) {
+ errorCode = U_MEMORY_ALLOCATION_ERROR;
+ }
+}
+
+U_NAMESPACE_END
+
+#endif // !UCONFIG_NO_COLLATION