// © 2016 and later: Unicode, Inc. and others. // License & terms of use: http://www.unicode.org/copyright.html /* ********************************************************************** * Copyright (C) 2010-2015, International Business Machines * Corporation and others. All Rights Reserved. ********************************************************************** * file name: ucnv_ct.c * encoding: UTF-8 * tab size: 8 (not used) * indentation:4 * * created on: 2010Dec09 * created by: Michael Ow */ #include "unicode/utypes.h" #if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION && !UCONFIG_ONLY_HTML_CONVERSION #include "unicode/ucnv.h" #include "unicode/uset.h" #include "unicode/ucnv_err.h" #include "unicode/ucnv_cb.h" #include "unicode/utf16.h" #include "ucnv_imp.h" #include "ucnv_bld.h" #include "ucnv_cnv.h" #include "ucnvmbcs.h" #include "cstring.h" #include "cmemory.h" typedef enum { INVALID = -2, DO_SEARCH = -1, COMPOUND_TEXT_SINGLE_0 = 0, COMPOUND_TEXT_SINGLE_1 = 1, COMPOUND_TEXT_SINGLE_2 = 2, COMPOUND_TEXT_SINGLE_3 = 3, COMPOUND_TEXT_DOUBLE_1 = 4, COMPOUND_TEXT_DOUBLE_2 = 5, COMPOUND_TEXT_DOUBLE_3 = 6, COMPOUND_TEXT_DOUBLE_4 = 7, COMPOUND_TEXT_DOUBLE_5 = 8, COMPOUND_TEXT_DOUBLE_6 = 9, COMPOUND_TEXT_DOUBLE_7 = 10, COMPOUND_TEXT_TRIPLE_DOUBLE = 11, IBM_915 = 12, IBM_916 = 13, IBM_914 = 14, IBM_874 = 15, IBM_912 = 16, IBM_913 = 17, ISO_8859_14 = 18, IBM_923 = 19, NUM_OF_CONVERTERS = 20 } COMPOUND_TEXT_CONVERTERS; #define SEARCH_LENGTH … static const uint8_t escSeqCompoundText[NUM_OF_CONVERTERS][5] = { /* Single */ { 0x1B, 0x2D, 0x41, 0, 0 }, { 0x1B, 0x2D, 0x4D, 0, 0 }, { 0x1B, 0x2D, 0x46, 0, 0 }, { 0x1B, 0x2D, 0x47, 0, 0 }, /* Double */ { 0x1B, 0x24, 0x29, 0x41, 0 }, { 0x1B, 0x24, 0x29, 0x42, 0 }, { 0x1B, 0x24, 0x29, 0x43, 0 }, { 0x1B, 0x24, 0x29, 0x44, 0 }, { 0x1B, 0x24, 0x29, 0x47, 0 }, { 0x1B, 0x24, 0x29, 0x48, 0 }, { 0x1B, 0x24, 0x29, 0x49, 0 }, /* Triple/Double */ { 0x1B, 0x25, 0x47, 0, 0 }, /*IBM-915*/ { 0x1B, 0x2D, 0x4C, 0, 0 }, /*IBM-916*/ { 0x1B, 0x2D, 0x48, 0, 0 }, /*IBM-914*/ { 0x1B, 0x2D, 0x44, 0, 0 }, /*IBM-874*/ { 0x1B, 0x2D, 0x54, 0, 0 }, /*IBM-912*/ { 0x1B, 0x2D, 0x42, 0, 0 }, /* IBM-913 */ { 0x1B, 0x2D, 0x43, 0, 0 }, /* ISO-8859_14 */ { 0x1B, 0x2D, 0x5F, 0, 0 }, /* IBM-923 */ { 0x1B, 0x2D, 0x62, 0, 0 }, }; #define ESC_START … #define isASCIIRange … #define isIBM915 … #define isIBM916 … #define isCompoundS3 … #define isCompoundS2 … #define isIBM914 … #define isIBM874 … #define isIBM912 … #define isIBM913 … #define isCompoundS1 … #define isISO8859_14 … #define isIBM923 … typedef struct{ UConverterSharedData *myConverterArray[NUM_OF_CONVERTERS]; COMPOUND_TEXT_CONVERTERS state; } UConverterDataCompoundText; /*********** Compound Text Converter Protos ***********/ U_CDECL_BEGIN static void U_CALLCONV _CompoundTextOpen(UConverter *cnv, UConverterLoadArgs *pArgs, UErrorCode *errorCode); static void U_CALLCONV _CompoundTextClose(UConverter *converter); static void U_CALLCONV _CompoundTextReset(UConverter *converter, UConverterResetChoice choice); static const char* U_CALLCONV _CompoundTextgetName(const UConverter* cnv); static int32_t findNextEsc(const char *source, const char *sourceLimit) { int32_t length = static_cast<int32_t>(sourceLimit - source); int32_t i; for (i = 1; i < length; i++) { if (*(source + i) == 0x1B) { return i; } } return length; } static COMPOUND_TEXT_CONVERTERS getState(int codepoint) { COMPOUND_TEXT_CONVERTERS state = DO_SEARCH; if (isASCIIRange(codepoint)) { state = COMPOUND_TEXT_SINGLE_0; } else if (isIBM912(codepoint)) { state = IBM_912; }else if (isIBM913(codepoint)) { state = IBM_913; } else if (isISO8859_14(codepoint)) { state = ISO_8859_14; } else if (isIBM923(codepoint)) { state = IBM_923; } else if (isIBM874(codepoint)) { state = IBM_874; } else if (isIBM914(codepoint)) { state = IBM_914; } else if (isCompoundS2(codepoint)) { state = COMPOUND_TEXT_SINGLE_2; } else if (isCompoundS3(codepoint)) { state = COMPOUND_TEXT_SINGLE_3; } else if (isIBM916(codepoint)) { state = IBM_916; } else if (isIBM915(codepoint)) { state = IBM_915; } else if (isCompoundS1(codepoint)) { state = COMPOUND_TEXT_SINGLE_1; } return state; } static COMPOUND_TEXT_CONVERTERS findStateFromEscSeq(const char* source, const char* sourceLimit, const uint8_t* toUBytesBuffer, int32_t toUBytesBufferLength, UErrorCode *err) { COMPOUND_TEXT_CONVERTERS state = INVALID; UBool matchFound = false; int32_t i, n, offset = toUBytesBufferLength; for (i = 0; i < NUM_OF_CONVERTERS; i++) { matchFound = true; for (n = 0; escSeqCompoundText[i][n] != 0; n++) { if (n < toUBytesBufferLength) { if (toUBytesBuffer[n] != escSeqCompoundText[i][n]) { matchFound = false; break; } } else if ((source + (n - offset)) >= sourceLimit) { *err = U_TRUNCATED_CHAR_FOUND; matchFound = false; break; } else if (*(source + (n - offset)) != escSeqCompoundText[i][n]) { matchFound = false; break; } } if (matchFound) { break; } } if (matchFound) { state = (COMPOUND_TEXT_CONVERTERS)i; } return state; } static void U_CALLCONV _CompoundTextOpen(UConverter *cnv, UConverterLoadArgs *pArgs, UErrorCode *errorCode){ cnv->extraInfo = uprv_malloc (sizeof (UConverterDataCompoundText)); if (cnv->extraInfo != nullptr) { UConverterDataCompoundText *myConverterData = (UConverterDataCompoundText *) cnv->extraInfo; UConverterNamePieces stackPieces; UConverterLoadArgs stackArgs=UCNV_LOAD_ARGS_INITIALIZER; myConverterData->myConverterArray[COMPOUND_TEXT_SINGLE_0] = nullptr; myConverterData->myConverterArray[COMPOUND_TEXT_SINGLE_1] = ucnv_loadSharedData("icu-internal-compound-s1", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[COMPOUND_TEXT_SINGLE_2] = ucnv_loadSharedData("icu-internal-compound-s2", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[COMPOUND_TEXT_SINGLE_3] = ucnv_loadSharedData("icu-internal-compound-s3", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[COMPOUND_TEXT_DOUBLE_1] = ucnv_loadSharedData("icu-internal-compound-d1", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[COMPOUND_TEXT_DOUBLE_2] = ucnv_loadSharedData("icu-internal-compound-d2", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[COMPOUND_TEXT_DOUBLE_3] = ucnv_loadSharedData("icu-internal-compound-d3", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[COMPOUND_TEXT_DOUBLE_4] = ucnv_loadSharedData("icu-internal-compound-d4", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[COMPOUND_TEXT_DOUBLE_5] = ucnv_loadSharedData("icu-internal-compound-d5", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[COMPOUND_TEXT_DOUBLE_6] = ucnv_loadSharedData("icu-internal-compound-d6", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[COMPOUND_TEXT_DOUBLE_7] = ucnv_loadSharedData("icu-internal-compound-d7", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[COMPOUND_TEXT_TRIPLE_DOUBLE] = ucnv_loadSharedData("icu-internal-compound-t", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[IBM_915] = ucnv_loadSharedData("ibm-915_P100-1995", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[IBM_916] = ucnv_loadSharedData("ibm-916_P100-1995", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[IBM_914] = ucnv_loadSharedData("ibm-914_P100-1995", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[IBM_874] = ucnv_loadSharedData("ibm-874_P100-1995", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[IBM_912] = ucnv_loadSharedData("ibm-912_P100-1995", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[IBM_913] = ucnv_loadSharedData("ibm-913_P100-2000", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[ISO_8859_14] = ucnv_loadSharedData("iso-8859_14-1998", &stackPieces, &stackArgs, errorCode); myConverterData->myConverterArray[IBM_923] = ucnv_loadSharedData("ibm-923_P100-1998", &stackPieces, &stackArgs, errorCode); if (U_FAILURE(*errorCode) || pArgs->onlyTestIsLoadable) { _CompoundTextClose(cnv); return; } myConverterData->state = (COMPOUND_TEXT_CONVERTERS)0; } else { *errorCode = U_MEMORY_ALLOCATION_ERROR; } } static void U_CALLCONV _CompoundTextClose(UConverter *converter) { UConverterDataCompoundText* myConverterData = (UConverterDataCompoundText*)(converter->extraInfo); int32_t i; if (converter->extraInfo != nullptr) { /*close the array of converter pointers and free the memory*/ for (i = 0; i < NUM_OF_CONVERTERS; i++) { if (myConverterData->myConverterArray[i] != nullptr) { ucnv_unloadSharedDataIfReady(myConverterData->myConverterArray[i]); } } uprv_free(converter->extraInfo); converter->extraInfo = nullptr; } } static void U_CALLCONV _CompoundTextReset(UConverter *converter, UConverterResetChoice choice) { (void)converter; (void)choice; } static const char* U_CALLCONV _CompoundTextgetName(const UConverter* cnv){ (void)cnv; return "x11-compound-text"; } static void U_CALLCONV UConverter_fromUnicode_CompoundText_OFFSETS(UConverterFromUnicodeArgs* args, UErrorCode* err){ UConverter *cnv = args->converter; uint8_t *target = (uint8_t *) args->target; const uint8_t *targetLimit = (const uint8_t *) args->targetLimit; const char16_t* source = args->source; const char16_t* sourceLimit = args->sourceLimit; /* int32_t* offsets = args->offsets; */ UChar32 sourceChar; UBool useFallback = cnv->useFallback; uint8_t tmpTargetBuffer[7]; int32_t tmpTargetBufferLength = 0; COMPOUND_TEXT_CONVERTERS currentState, tmpState; uint32_t pValue; int32_t pValueLength = 0; int32_t i, n, j; UConverterDataCompoundText *myConverterData = (UConverterDataCompoundText *) cnv->extraInfo; currentState = myConverterData->state; /* check if the last codepoint of previous buffer was a lead surrogate*/ if((sourceChar = cnv->fromUChar32)!=0 && target< targetLimit) { goto getTrail; } while( source < sourceLimit){ if(target < targetLimit){ sourceChar = *(source++); /*check if the char is a First surrogate*/ if(U16_IS_SURROGATE(sourceChar)) { if(U16_IS_SURROGATE_LEAD(sourceChar)) { getTrail: /*look ahead to find the trail surrogate*/ if(source < sourceLimit) { /* test the following code unit */ char16_t trail=(char16_t) *source; if(U16_IS_TRAIL(trail)) { source++; sourceChar=U16_GET_SUPPLEMENTARY(sourceChar, trail); cnv->fromUChar32=0x00; /* convert this supplementary code point */ /* exit this condition tree */ } else { /* this is an unmatched lead code unit (1st surrogate) */ /* callback(illegal) */ *err=U_ILLEGAL_CHAR_FOUND; cnv->fromUChar32=sourceChar; break; } } else { /* no more input */ cnv->fromUChar32=sourceChar; break; } } else { /* this is an unmatched trail code unit (2nd surrogate) */ /* callback(illegal) */ *err=U_ILLEGAL_CHAR_FOUND; cnv->fromUChar32=sourceChar; break; } } tmpTargetBufferLength = 0; tmpState = getState(sourceChar); if (tmpState != DO_SEARCH && currentState != tmpState) { /* Get escape sequence if necessary */ currentState = tmpState; for (i = 0; escSeqCompoundText[currentState][i] != 0; i++) { tmpTargetBuffer[tmpTargetBufferLength++] = escSeqCompoundText[currentState][i]; } } if (tmpState == DO_SEARCH) { /* Test all available converters */ for (i = 1; i < SEARCH_LENGTH; i++) { pValueLength = ucnv_MBCSFromUChar32(myConverterData->myConverterArray[i], sourceChar, &pValue, useFallback); if (pValueLength > 0) { tmpState = (COMPOUND_TEXT_CONVERTERS)i; if (currentState != tmpState) { currentState = tmpState; for (j = 0; escSeqCompoundText[currentState][j] != 0; j++) { tmpTargetBuffer[tmpTargetBufferLength++] = escSeqCompoundText[currentState][j]; } } for (n = (pValueLength - 1); n >= 0; n--) { tmpTargetBuffer[tmpTargetBufferLength++] = (uint8_t)(pValue >> (n * 8)); } break; } } } else if (tmpState == COMPOUND_TEXT_SINGLE_0) { tmpTargetBuffer[tmpTargetBufferLength++] = (uint8_t)sourceChar; } else { pValueLength = ucnv_MBCSFromUChar32(myConverterData->myConverterArray[currentState], sourceChar, &pValue, useFallback); if (pValueLength > 0) { for (n = (pValueLength - 1); n >= 0; n--) { tmpTargetBuffer[tmpTargetBufferLength++] = (uint8_t)(pValue >> (n * 8)); } } } for (i = 0; i < tmpTargetBufferLength; i++) { if (target < targetLimit) { *target++ = tmpTargetBuffer[i]; } else { *err = U_BUFFER_OVERFLOW_ERROR; break; } } if (*err == U_BUFFER_OVERFLOW_ERROR) { for (; i < tmpTargetBufferLength; i++) { args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = tmpTargetBuffer[i]; } } } else { *err = U_BUFFER_OVERFLOW_ERROR; break; } } /*save the state and return */ myConverterData->state = currentState; args->source = source; args->target = (char*)target; } static void U_CALLCONV UConverter_toUnicode_CompoundText_OFFSETS(UConverterToUnicodeArgs *args, UErrorCode* err){ const char *mySource = (char *) args->source; char16_t *myTarget = args->target; const char *mySourceLimit = args->sourceLimit; const char *tmpSourceLimit = mySourceLimit; uint32_t mySourceChar = 0x0000; COMPOUND_TEXT_CONVERTERS currentState, tmpState; int32_t sourceOffset = 0; UConverterDataCompoundText *myConverterData = (UConverterDataCompoundText *) args->converter->extraInfo; UConverterSharedData* savedSharedData = nullptr; UConverterToUnicodeArgs subArgs; int32_t minArgsSize; /* set up the subconverter arguments */ if(args->size<sizeof(UConverterToUnicodeArgs)) { minArgsSize = args->size; } else { minArgsSize = (int32_t)sizeof(UConverterToUnicodeArgs); } uprv_memcpy(&subArgs, args, minArgsSize); subArgs.size = (uint16_t)minArgsSize; currentState = tmpState = myConverterData->state; while(mySource < mySourceLimit){ if(myTarget < args->targetLimit){ if (args->converter->toULength > 0) { mySourceChar = args->converter->toUBytes[0]; } else { mySourceChar = (uint8_t)*mySource; } if (mySourceChar == ESC_START) { tmpState = findStateFromEscSeq(mySource, mySourceLimit, args->converter->toUBytes, args->converter->toULength, err); if (*err == U_TRUNCATED_CHAR_FOUND) { for (; mySource < mySourceLimit;) { args->converter->toUBytes[args->converter->toULength++] = *mySource++; } *err = U_ZERO_ERROR; break; } else if (tmpState == INVALID) { if (args->converter->toULength == 0) { mySource++; /* skip over the 0x1b byte */ } *err = U_ILLEGAL_CHAR_FOUND; break; } if (tmpState != currentState) { currentState = tmpState; } sourceOffset = static_cast<int32_t>(uprv_strlen((char*)escSeqCompoundText[currentState]) - args->converter->toULength); mySource += sourceOffset; args->converter->toULength = 0; } if (currentState == COMPOUND_TEXT_SINGLE_0) { while (mySource < mySourceLimit) { if (*mySource == ESC_START) { break; } if (myTarget < args->targetLimit) { *myTarget++ = 0x00ff&(*mySource++); } else { *err = U_BUFFER_OVERFLOW_ERROR; break; } } } else if (mySource < mySourceLimit){ sourceOffset = findNextEsc(mySource, mySourceLimit); tmpSourceLimit = mySource + sourceOffset; subArgs.source = mySource; subArgs.sourceLimit = tmpSourceLimit; subArgs.target = myTarget; savedSharedData = subArgs.converter->sharedData; subArgs.converter->sharedData = myConverterData->myConverterArray[currentState]; ucnv_MBCSToUnicodeWithOffsets(&subArgs, err); subArgs.converter->sharedData = savedSharedData; mySource = subArgs.source; myTarget = subArgs.target; if (U_FAILURE(*err)) { if(*err == U_BUFFER_OVERFLOW_ERROR) { if(subArgs.converter->UCharErrorBufferLength > 0) { uprv_memcpy(args->converter->UCharErrorBuffer, subArgs.converter->UCharErrorBuffer, subArgs.converter->UCharErrorBufferLength); } args->converter->UCharErrorBufferLength=subArgs.converter->UCharErrorBufferLength; subArgs.converter->UCharErrorBufferLength = 0; } break; } } } else { *err = U_BUFFER_OVERFLOW_ERROR; break; } } myConverterData->state = currentState; args->target = myTarget; args->source = mySource; } static void U_CALLCONV _CompoundText_GetUnicodeSet(const UConverter *cnv, const USetAdder *sa, UConverterUnicodeSet which, UErrorCode *pErrorCode) { UConverterDataCompoundText *myConverterData = (UConverterDataCompoundText *)cnv->extraInfo; int32_t i; for (i = 1; i < NUM_OF_CONVERTERS; i++) { ucnv_MBCSGetUnicodeSetForUnicode(myConverterData->myConverterArray[i], sa, which, pErrorCode); } sa->add(sa->set, 0x0000); sa->add(sa->set, 0x0009); sa->add(sa->set, 0x000A); sa->addRange(sa->set, 0x0020, 0x007F); sa->addRange(sa->set, 0x00A0, 0x00FF); } U_CDECL_END static const UConverterImpl _CompoundTextImpl = { UCNV_COMPOUND_TEXT, nullptr, nullptr, _CompoundTextOpen, _CompoundTextClose, _CompoundTextReset, UConverter_toUnicode_CompoundText_OFFSETS, UConverter_toUnicode_CompoundText_OFFSETS, UConverter_fromUnicode_CompoundText_OFFSETS, UConverter_fromUnicode_CompoundText_OFFSETS, nullptr, nullptr, _CompoundTextgetName, nullptr, nullptr, _CompoundText_GetUnicodeSet, nullptr, nullptr }; static const UConverterStaticData _CompoundTextStaticData = { sizeof(UConverterStaticData), "COMPOUND_TEXT", 0, UCNV_IBM, UCNV_COMPOUND_TEXT, 1, 6, { 0xef, 0, 0, 0 }, 1, false, false, 0, 0, { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ }; const UConverterSharedData _CompoundTextData = UCNV_IMMUTABLE_SHARED_DATA_INITIALIZER(&_CompoundTextStaticData, &_CompoundTextImpl); #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */
Codebase Browser
godot