// © 2016 and later: Unicode, Inc. and others. // License & terms of use: http://www.unicode.org/copyright.html /* ******************************************************************************* * * Copyright (C) 2009-2014, International Business Machines * Corporation and others. All Rights Reserved. * ******************************************************************************* * file name: normalizer2impl.cpp * encoding: UTF-8 * tab size: 8 (not used) * indentation:4 * * created on: 2009nov22 * created by: Markus W. Scherer */ // #define UCPTRIE_DEBUG #include "unicode/utypes.h" #if !UCONFIG_NO_NORMALIZATION #include "unicode/bytestream.h" #include "unicode/edits.h" #include "unicode/normalizer2.h" #include "unicode/stringoptions.h" #include "unicode/ucptrie.h" #include "unicode/udata.h" #include "unicode/umutablecptrie.h" #include "unicode/ustring.h" #include "unicode/utf16.h" #include "unicode/utf8.h" #include "bytesinkutil.h" #include "cmemory.h" #include "mutex.h" #include "normalizer2impl.h" #include "putilimp.h" #include "uassert.h" #include "ucptrie_impl.h" #include "uset_imp.h" #include "uvector.h" U_NAMESPACE_BEGIN namespace { /** * UTF-8 lead byte for minNoMaybeCP. * Can be lower than the actual lead byte for c. * Typically U+0300 for NFC/NFD, U+00A0 for NFKC/NFKD, U+0041 for NFKC_Casefold. */ inline uint8_t leadByteForCP(UChar32 c) { … } /** * Returns the code point from one single well-formed UTF-8 byte sequence * between cpStart and cpLimit. * * Trie UTF-8 macros do not assemble whole code points (for efficiency). * When we do need the code point, we call this function. * We should not need it for normalization-inert data (norm16==0). * Illegal sequences yield the error value norm16==0 just like real normalization-inert code points. */ UChar32 codePointFromValidUTF8(const uint8_t *cpStart, const uint8_t *cpLimit) { … } /** * Returns the last code point in [start, p[ if it is valid and in U+1000..U+D7FF. * Otherwise returns a negative value. */ UChar32 previousHangulOrJamo(const uint8_t *start, const uint8_t *p) { … } /** * Returns the offset from the Jamo T base if [src, limit[ starts with a single Jamo T code point. * Otherwise returns a negative value. */ int32_t getJamoTMinusBase(const uint8_t *src, const uint8_t *limit) { … } void appendCodePointDelta(const uint8_t *cpStart, const uint8_t *cpLimit, int32_t delta, ByteSink &sink, Edits *edits) { … } } // namespace // ReorderingBuffer -------------------------------------------------------- *** ReorderingBuffer::ReorderingBuffer(const Normalizer2Impl &ni, UnicodeString &dest, UErrorCode &errorCode) : … { … } UBool ReorderingBuffer::init(int32_t destCapacity, UErrorCode &errorCode) { … } UBool ReorderingBuffer::equals(const char16_t *otherStart, const char16_t *otherLimit) const { … } UBool ReorderingBuffer::equals(const uint8_t *otherStart, const uint8_t *otherLimit) const { … } UBool ReorderingBuffer::appendSupplementary(UChar32 c, uint8_t cc, UErrorCode &errorCode) { … } UBool ReorderingBuffer::append(const char16_t *s, int32_t length, UBool isNFD, uint8_t leadCC, uint8_t trailCC, UErrorCode &errorCode) { … } UBool ReorderingBuffer::appendZeroCC(UChar32 c, UErrorCode &errorCode) { … } UBool ReorderingBuffer::appendZeroCC(const char16_t *s, const char16_t *sLimit, UErrorCode &errorCode) { … } void ReorderingBuffer::remove() { … } void ReorderingBuffer::removeSuffix(int32_t suffixLength) { … } UBool ReorderingBuffer::resize(int32_t appendLength, UErrorCode &errorCode) { … } void ReorderingBuffer::skipPrevious() { … } uint8_t ReorderingBuffer::previousCC() { … } // Inserts c somewhere before the last character. // Requires 0<cc<lastCC which implies reorderStart<limit. void ReorderingBuffer::insert(UChar32 c, uint8_t cc) { … } // Normalizer2Impl --------------------------------------------------------- *** struct CanonIterData : public UMemory { … }; Normalizer2Impl::~Normalizer2Impl() { … } void Normalizer2Impl::init(const int32_t *inIndexes, const UCPTrie *inTrie, const uint16_t *inExtraData, const uint8_t *inSmallFCD) { … } U_CDECL_BEGIN static uint32_t U_CALLCONV segmentStarterMapper(const void * /*context*/, uint32_t value) { … } U_CDECL_END void Normalizer2Impl::addLcccChars(UnicodeSet &set) const { … } void Normalizer2Impl::addPropertyStarts(const USetAdder *sa, UErrorCode & /*errorCode*/) const { … } void Normalizer2Impl::addCanonIterPropertyStarts(const USetAdder *sa, UErrorCode &errorCode) const { … } const char16_t * Normalizer2Impl::copyLowPrefixFromNulTerminated(const char16_t *src, UChar32 minNeedDataCP, ReorderingBuffer *buffer, UErrorCode &errorCode) const { … } UnicodeString & Normalizer2Impl::decompose(const UnicodeString &src, UnicodeString &dest, UErrorCode &errorCode) const { … } void Normalizer2Impl::decompose(const char16_t *src, const char16_t *limit, UnicodeString &dest, int32_t destLengthEstimate, UErrorCode &errorCode) const { … } // Dual functionality: // buffer!=nullptr: normalize // buffer==nullptr: isNormalized/spanQuickCheckYes const char16_t * Normalizer2Impl::decompose(const char16_t *src, const char16_t *limit, ReorderingBuffer *buffer, UErrorCode &errorCode) const { … } // Decompose a short piece of text which is likely to contain characters that // fail the quick check loop and/or where the quick check loop's overhead // is unlikely to be amortized. // Called by the compose() and makeFCD() implementations. const char16_t * Normalizer2Impl::decomposeShort(const char16_t *src, const char16_t *limit, UBool stopAtCompBoundary, UBool onlyContiguous, ReorderingBuffer &buffer, UErrorCode &errorCode) const { … } UBool Normalizer2Impl::decompose(UChar32 c, uint16_t norm16, ReorderingBuffer &buffer, UErrorCode &errorCode) const { … } // Dual functionality: // sink != nullptr: normalize // sink == nullptr: isNormalized/spanQuickCheckYes const uint8_t * Normalizer2Impl::decomposeUTF8(uint32_t options, const uint8_t *src, const uint8_t *limit, ByteSink *sink, Edits *edits, UErrorCode &errorCode) const { … } const uint8_t * Normalizer2Impl::decomposeShort(const uint8_t *src, const uint8_t *limit, StopAt stopAt, UBool onlyContiguous, ReorderingBuffer &buffer, UErrorCode &errorCode) const { … } const char16_t * Normalizer2Impl::getDecomposition(UChar32 c, char16_t buffer[4], int32_t &length) const { … } // The capacity of the buffer must be 30=MAPPING_LENGTH_MASK-1 // so that a raw mapping fits that consists of one unit ("rm0") // plus all but the first two code units of the normal mapping. // The maximum length of a normal mapping is 31=MAPPING_LENGTH_MASK. const char16_t * Normalizer2Impl::getRawDecomposition(UChar32 c, char16_t buffer[30], int32_t &length) const { … } void Normalizer2Impl::decomposeAndAppend(const char16_t *src, const char16_t *limit, UBool doDecompose, UnicodeString &safeMiddle, ReorderingBuffer &buffer, UErrorCode &errorCode) const { … } UBool Normalizer2Impl::hasDecompBoundaryBefore(UChar32 c) const { … } UBool Normalizer2Impl::norm16HasDecompBoundaryBefore(uint16_t norm16) const { … } UBool Normalizer2Impl::hasDecompBoundaryAfter(UChar32 c) const { … } UBool Normalizer2Impl::norm16HasDecompBoundaryAfter(uint16_t norm16) const { … } /* * Finds the recomposition result for * a forward-combining "lead" character, * specified with a pointer to its compositions list, * and a backward-combining "trail" character. * * If the lead and trail characters combine, then this function returns * the following "compositeAndFwd" value: * Bits 21..1 composite character * Bit 0 set if the composite is a forward-combining starter * otherwise it returns -1. * * The compositions list has (trail, compositeAndFwd) pair entries, * encoded as either pairs or triples of 16-bit units. * The last entry has the high bit of its first unit set. * * The list is sorted by ascending trail characters (there are no duplicates). * A linear search is used. * * See normalizer2impl.h for a more detailed description * of the compositions list format. */ int32_t Normalizer2Impl::combine(const uint16_t *list, UChar32 trail) { … } /** * @param list some character's compositions list * @param set recursively receives the composites from these compositions */ void Normalizer2Impl::addComposites(const uint16_t *list, UnicodeSet &set) const { … } /* * Recomposes the buffer text starting at recomposeStartIndex * (which is in NFD - decomposed and canonically ordered), * and truncates the buffer contents. * * Note that recomposition never lengthens the text: * Any character consists of either one or two code units; * a composition may contain at most one more code unit than the original starter, * while the combining mark that is removed has at least one code unit. */ void Normalizer2Impl::recompose(ReorderingBuffer &buffer, int32_t recomposeStartIndex, UBool onlyContiguous) const { … } UChar32 Normalizer2Impl::composePair(UChar32 a, UChar32 b) const { … } // Very similar to composeQuickCheck(): Make the same changes in both places if relevant. // doCompose: normalize // !doCompose: isNormalized (buffer must be empty and initialized) UBool Normalizer2Impl::compose(const char16_t *src, const char16_t *limit, UBool onlyContiguous, UBool doCompose, ReorderingBuffer &buffer, UErrorCode &errorCode) const { … } // Very similar to compose(): Make the same changes in both places if relevant. // pQCResult==nullptr: spanQuickCheckYes // pQCResult!=nullptr: quickCheck (*pQCResult must be UNORM_YES) const char16_t * Normalizer2Impl::composeQuickCheck(const char16_t *src, const char16_t *limit, UBool onlyContiguous, UNormalizationCheckResult *pQCResult) const { … } void Normalizer2Impl::composeAndAppend(const char16_t *src, const char16_t *limit, UBool doCompose, UBool onlyContiguous, UnicodeString &safeMiddle, ReorderingBuffer &buffer, UErrorCode &errorCode) const { … } UBool Normalizer2Impl::composeUTF8(uint32_t options, UBool onlyContiguous, const uint8_t *src, const uint8_t *limit, ByteSink *sink, Edits *edits, UErrorCode &errorCode) const { … } UBool Normalizer2Impl::hasCompBoundaryBefore(const char16_t *src, const char16_t *limit) const { … } UBool Normalizer2Impl::hasCompBoundaryBefore(const uint8_t *src, const uint8_t *limit) const { … } UBool Normalizer2Impl::hasCompBoundaryAfter(const char16_t *start, const char16_t *p, UBool onlyContiguous) const { … } UBool Normalizer2Impl::hasCompBoundaryAfter(const uint8_t *start, const uint8_t *p, UBool onlyContiguous) const { … } const char16_t *Normalizer2Impl::findPreviousCompBoundary(const char16_t *start, const char16_t *p, UBool onlyContiguous) const { … } const char16_t *Normalizer2Impl::findNextCompBoundary(const char16_t *p, const char16_t *limit, UBool onlyContiguous) const { … } uint8_t Normalizer2Impl::getPreviousTrailCC(const char16_t *start, const char16_t *p) const { … } uint8_t Normalizer2Impl::getPreviousTrailCC(const uint8_t *start, const uint8_t *p) const { … } // Note: normalizer2impl.cpp r30982 (2011-nov-27) // still had getFCDTrie() which built and cached an FCD trie. // That provided faster access to FCD data than getFCD16FromNormData() // but required synchronization and consumed some 10kB of heap memory // in any process that uses FCD (e.g., via collation). // minDecompNoCP etc. and smallFCD[] are intended to help with any loss of performance, // at least for ASCII & CJK. // Ticket 20907 - The optimizer in MSVC/Visual Studio versions below 16.4 has trouble with this // function on Windows ARM64. As a work-around, we disable optimizations for this function. // This work-around could/should be removed once the following versions of Visual Studio are no // longer supported: All versions of VS2017, and versions of VS2019 below 16.4. #if (defined(_MSC_VER) && (defined(_M_ARM64)) && (_MSC_VER < 1924)) #pragma optimize( "", off ) #endif // Gets the FCD value from the regular normalization data. uint16_t Normalizer2Impl::getFCD16FromNormData(UChar32 c) const { … } #if (defined(_MSC_VER) && (defined(_M_ARM64)) && (_MSC_VER < 1924)) #pragma optimize( "", on ) #endif // Dual functionality: // buffer!=nullptr: normalize // buffer==nullptr: isNormalized/quickCheck/spanQuickCheckYes const char16_t * Normalizer2Impl::makeFCD(const char16_t *src, const char16_t *limit, ReorderingBuffer *buffer, UErrorCode &errorCode) const { … } void Normalizer2Impl::makeFCDAndAppend(const char16_t *src, const char16_t *limit, UBool doMakeFCD, UnicodeString &safeMiddle, ReorderingBuffer &buffer, UErrorCode &errorCode) const { … } const char16_t *Normalizer2Impl::findPreviousFCDBoundary(const char16_t *start, const char16_t *p) const { … } const char16_t *Normalizer2Impl::findNextFCDBoundary(const char16_t *p, const char16_t *limit) const { … } // CanonicalIterator data -------------------------------------------------- *** CanonIterData::CanonIterData(UErrorCode &errorCode) : … { … } CanonIterData::~CanonIterData() { … } void CanonIterData::addToStartSet(UChar32 origin, UChar32 decompLead, UErrorCode &errorCode) { … } // C++ class for friend access to private Normalizer2Impl members. class InitCanonIterData { … }; U_CDECL_BEGIN // UInitOnce instantiation function for CanonIterData static void U_CALLCONV initCanonIterData(Normalizer2Impl *impl, UErrorCode &errorCode) { … } U_CDECL_END void InitCanonIterData::doInit(Normalizer2Impl *impl, UErrorCode &errorCode) { … } void Normalizer2Impl::makeCanonIterDataFromNorm16(UChar32 start, UChar32 end, const uint16_t norm16, CanonIterData &newData, UErrorCode &errorCode) const { … } UBool Normalizer2Impl::ensureCanonIterData(UErrorCode &errorCode) const { … } int32_t Normalizer2Impl::getCanonValue(UChar32 c) const { … } const UnicodeSet &Normalizer2Impl::getCanonStartSet(int32_t n) const { … } UBool Normalizer2Impl::isCanonSegmentStarter(UChar32 c) const { … } UBool Normalizer2Impl::getCanonStartSet(UChar32 c, UnicodeSet &set) const { … } U_NAMESPACE_END // Normalizer2 data swapping ----------------------------------------------- *** U_NAMESPACE_USE U_CAPI int32_t U_EXPORT2 unorm2_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, UErrorCode *pErrorCode) { … } #endif // !UCONFIG_NO_NORMALIZATION
Codebase Browser
chromium