collationbuilder.cpp | Explore in Territory

// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2014, International Business Machines
* Corporation and others.  All Rights Reserved.
*******************************************************************************
* collationbuilder.cpp
*
* (replaced the former ucol_bld.cpp)
*
* created on: 2013may06
* created by: Markus W. Scherer
*/

#ifdef DEBUG_COLLATION_BUILDER
#include <stdio.h>
#endif

#include "unicode/utypes.h"

#if !UCONFIG_NO_COLLATION

#include "unicode/caniter.h"
#include "unicode/normalizer2.h"
#include "unicode/tblcoll.h"
#include "unicode/parseerr.h"
#include "unicode/uchar.h"
#include "unicode/ucol.h"
#include "unicode/unistr.h"
#include "unicode/usetiter.h"
#include "unicode/utf16.h"
#include "unicode/uversion.h"
#include "cmemory.h"
#include "collation.h"
#include "collationbuilder.h"
#include "collationdata.h"
#include "collationdatabuilder.h"
#include "collationfastlatin.h"
#include "collationroot.h"
#include "collationrootelements.h"
#include "collationruleparser.h"
#include "collationsettings.h"
#include "collationtailoring.h"
#include "collationweights.h"
#include "normalizer2impl.h"
#include "uassert.h"
#include "ucol_imp.h"
#include "utf16collationiterator.h"

U_NAMESPACE_BEGIN

namespace {

class BundleImporter : public CollationRuleParser::Importer { … };

BundleImporter::~BundleImporter() { … }

void
BundleImporter::getRules(
        const char *localeID, const char *collationType,
        UnicodeString &rules,
        const char *& /*errorReason*/, UErrorCode &errorCode) { … }

}  // namespace

// RuleBasedCollator implementation ---------------------------------------- ***

// These methods are here, rather than in rulebasedcollator.cpp,
// for modularization:
// Most code using Collator does not need to build a Collator from rules.
// By moving these constructors and helper methods to a separate file,
// most code will not have a static dependency on the builder code.

RuleBasedCollator::RuleBasedCollator()
        : … { … }

RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, UErrorCode &errorCode)
        : … { … }

RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, ECollationStrength strength,
                                     UErrorCode &errorCode)
        : … { … }

RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules,
                                     UColAttributeValue decompositionMode,
                                     UErrorCode &errorCode)
        : … { … }

RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules,
                                     ECollationStrength strength,
                                     UColAttributeValue decompositionMode,
                                     UErrorCode &errorCode)
        : … { … }

RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules,
                                     UParseError &parseError, UnicodeString &reason,
                                     UErrorCode &errorCode)
        : … { … }

void
RuleBasedCollator::internalBuildTailoring(const UnicodeString &rules,
                                          int32_t strength,
                                          UColAttributeValue decompositionMode,
                                          UParseError *outParseError, UnicodeString *outReason,
                                          UErrorCode &errorCode) { … }

// CollationBuilder implementation ----------------------------------------- ***

CollationBuilder::CollationBuilder(const CollationTailoring *b, UBool icu4xMode, UErrorCode &errorCode)
        : … { … }

CollationBuilder::CollationBuilder(const CollationTailoring *b, UErrorCode &errorCode)
  : … { … }

CollationBuilder::~CollationBuilder() { … }

CollationTailoring *
CollationBuilder::parseAndBuild(const UnicodeString &ruleString,
                                const UVersionInfo rulesVersion,
                                CollationRuleParser::Importer *importer,
                                UParseError *outParseError,
                                UErrorCode &errorCode) { … }

void
CollationBuilder::addReset(int32_t strength, const UnicodeString &str,
                           const char *&parserErrorReason, UErrorCode &errorCode) { … }

uint32_t
CollationBuilder::getWeight16Before(int32_t index, int64_t node, int32_t level) { … }

int64_t
CollationBuilder::getSpecialResetPosition(const UnicodeString &str,
                                          const char *&parserErrorReason, UErrorCode &errorCode) { … }

void
CollationBuilder::addRelation(int32_t strength, const UnicodeString &prefix,
                              const UnicodeString &str, const UnicodeString &extension,
                              const char *&parserErrorReason, UErrorCode &errorCode) { … }

int32_t
CollationBuilder::findOrInsertNodeForCEs(int32_t strength, const char *&parserErrorReason,
                                         UErrorCode &errorCode) { … }

int32_t
CollationBuilder::findOrInsertNodeForRootCE(int64_t ce, int32_t strength, UErrorCode &errorCode) { … }

namespace {

/**
 * Like Java Collections.binarySearch(List, key, Comparator).
 *
 * @return the index>=0 where the item was found,
 *         or the index<0 for inserting the string at ~index in sorted order
 *         (index into rootPrimaryIndexes)
 */
int32_t
binarySearchForRootPrimaryNode(const int32_t *rootPrimaryIndexes, int32_t length,
                               const int64_t *nodes, uint32_t p) { … }

}  // namespace

int32_t
CollationBuilder::findOrInsertNodeForPrimary(uint32_t p, UErrorCode &errorCode) { … }

int32_t
CollationBuilder::findOrInsertWeakNode(int32_t index, uint32_t weight16, int32_t level, UErrorCode &errorCode) { … }

int32_t
CollationBuilder::insertTailoredNodeAfter(int32_t index, int32_t strength, UErrorCode &errorCode) { … }

int32_t
CollationBuilder::insertNodeBetween(int32_t index, int32_t nextIndex, int64_t node,
                                    UErrorCode &errorCode) { … }

int32_t
CollationBuilder::findCommonNode(int32_t index, int32_t strength) const { … }

void
CollationBuilder::setCaseBits(const UnicodeString &nfdString,
                              const char *&parserErrorReason, UErrorCode &errorCode) { … }

void
CollationBuilder::suppressContractions(const UnicodeSet &set, const char *&parserErrorReason,
                                       UErrorCode &errorCode) { … }

void
CollationBuilder::optimize(const UnicodeSet &set, const char *& /* parserErrorReason */,
                           UErrorCode &errorCode) { … }

uint32_t
CollationBuilder::addWithClosure(const UnicodeString &nfdPrefix, const UnicodeString &nfdString,
                                 const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32,
                                 UErrorCode &errorCode) { … }

// ICU-22517
// This constant defines a limit for the addOnlyClosure to return
// error, to avoid taking a long time for canonical closure expansion.
// Please let us know if you have a reasonable use case that needed
// for a practical Collation rule that needs to increase this limit.
// This value is needed for compiling a rule with eight Hangul syllables such as
// "&a=b쫊쫊쫊쫊쫊쫊쫊쫊" without error, which should be more than realistic
// usage.
static constexpr int32_t kClosureLoopLimit = …;

uint32_t
CollationBuilder::addOnlyClosure(const UnicodeString &nfdPrefix, const UnicodeString &nfdString,
                                 const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32,
                                 UErrorCode &errorCode) { … }

void
CollationBuilder::addTailComposites(const UnicodeString &nfdPrefix, const UnicodeString &nfdString,
                                    UErrorCode &errorCode) { … }

UBool
CollationBuilder::mergeCompositeIntoString(const UnicodeString &nfdString,
                                           int32_t indexAfterLastStarter,
                                           UChar32 composite, const UnicodeString &decomp,
                                           UnicodeString &newNFDString, UnicodeString &newString,
                                           UErrorCode &errorCode) const { … }

UBool
CollationBuilder::ignorePrefix(const UnicodeString &s, UErrorCode &errorCode) const { … }

UBool
CollationBuilder::ignoreString(const UnicodeString &s, UErrorCode &errorCode) const { … }

UBool
CollationBuilder::isFCD(const UnicodeString &s, UErrorCode &errorCode) const { … }

void
CollationBuilder::closeOverComposites(UErrorCode &errorCode) { … }

uint32_t
CollationBuilder::addIfDifferent(const UnicodeString &prefix, const UnicodeString &str,
                                 const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32,
                                 UErrorCode &errorCode) { … }

UBool
CollationBuilder::sameCEs(const int64_t ces1[], int32_t ces1Length,
                          const int64_t ces2[], int32_t ces2Length) { … }

#ifdef DEBUG_COLLATION_BUILDER

uint32_t
alignWeightRight(uint32_t w) {
    if(w != 0) {
        while((w & 0xff) == 0) { w >>= 8; }
    }
    return w;
}

#endif

void
CollationBuilder::makeTailoredCEs(UErrorCode &errorCode) { … }

int32_t
CollationBuilder::countTailoredNodes(const int64_t *nodesArray, int32_t i, int32_t strength) { … }

class CEFinalizer : public CollationDataBuilder::CEModifier { … };

CEFinalizer::~CEFinalizer() { … }

void
CollationBuilder::finalizeCEs(UErrorCode &errorCode) { … }

int32_t
CollationBuilder::ceStrength(int64_t ce) { … }

U_NAMESPACE_END

U_NAMESPACE_USE

U_CAPI UCollator * U_EXPORT2
ucol_openRules(const char16_t *rules, int32_t rulesLength,
               UColAttributeValue normalizationMode, UCollationStrength strength,
               UParseError *parseError, UErrorCode *pErrorCode) { … }

static const int32_t internalBufferSize = …;

// The @internal ucol_getUnsafeSet() was moved here from ucol_sit.cpp
// because it calls UnicodeSet "builder" code that depends on all Unicode properties,
// and the rest of the collation "runtime" code only depends on normalization.
// This function is not related to the collation builder,
// but it did not seem worth moving it into its own .cpp file,
// nor rewriting it to use lower-level UnicodeSet and Normalizer2Impl methods.
U_CAPI int32_t U_EXPORT2
ucol_getUnsafeSet( const UCollator *coll,
                  USet *unsafe,
                  UErrorCode *status)
{ … }

#endif  // !UCONFIG_NO_COLLATION
chromium/third_party/icu/source/i18n/collationbuilder.cpp
