/*===--- ConvertUTF.c - Universal Character Names conversions ---------------=== * * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. * See https://llvm.org/LICENSE.txt for license information. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * *===------------------------------------------------------------------------=*/ /* * Copyright © 1991-2015 Unicode, Inc. All rights reserved. * Distributed under the Terms of Use in * http://www.unicode.org/copyright.html. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of the Unicode data files and any associated documentation * (the "Data Files") or Unicode software and any associated documentation * (the "Software") to deal in the Data Files or Software * without restriction, including without limitation the rights to use, * copy, modify, merge, publish, distribute, and/or sell copies of * the Data Files or Software, and to permit persons to whom the Data Files * or Software are furnished to do so, provided that * (a) this copyright and permission notice appear with all copies * of the Data Files or Software, * (b) this copyright and permission notice appear in associated * documentation, and * (c) there is clear notice in each modified Data File or in the Software * as well as in the documentation associated with the Data File(s) or * Software that the data or software has been modified. * * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF * ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT OF THIRD PARTY RIGHTS. * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS * NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL * DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR * PERFORMANCE OF THE DATA FILES OR SOFTWARE. * * Except as contained in this notice, the name of a copyright holder * shall not be used in advertising or otherwise to promote the sale, * use or other dealings in these Data Files or Software without prior * written authorization of the copyright holder. */ /* --------------------------------------------------------------------- Conversions between UTF32, UTF-16, and UTF-8. Source code file. Author: Mark E. Davis, 1994. Rev History: Rick McGowan, fixes & updates May 2001. Sept 2001: fixed const & error conditions per mods suggested by S. Parent & A. Lillich. June 2002: Tim Dodd added detection and handling of incomplete source sequences, enhanced error detection, added casts to eliminate compiler warnings. July 2003: slight mods to back out aggressive FFFE detection. Jan 2004: updated switches in from-UTF8 conversions. Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions. See the header file "ConvertUTF.h" for complete documentation. ------------------------------------------------------------------------ */ #include "llvm/Support/ConvertUTF.h" #ifdef CVTUTF_DEBUG #include <stdio.h> #endif #include <assert.h> /* * This code extensively uses fall-through switches. * Keep the compiler from warning about that. */ #if defined(__clang__) && defined(__has_warning) # if __has_warning("-Wimplicit-fallthrough") #define ConvertUTF_DISABLE_WARNINGS … #define ConvertUTF_RESTORE_WARNINGS … # endif #elif defined(__GNUC__) && __GNUC__ > 6 #define ConvertUTF_DISABLE_WARNINGS … #define ConvertUTF_RESTORE_WARNINGS … #endif #ifndef ConvertUTF_DISABLE_WARNINGS #define ConvertUTF_DISABLE_WARNINGS #endif #ifndef ConvertUTF_RESTORE_WARNINGS #define ConvertUTF_RESTORE_WARNINGS #endif ConvertUTF_DISABLE_WARNINGS namespace llvm { static const int halfShift = …; /* used for shifting by 10 bits */ static const UTF32 halfBase = …; static const UTF32 halfMask = …; #define UNI_SUR_HIGH_START … #define UNI_SUR_HIGH_END … #define UNI_SUR_LOW_START … #define UNI_SUR_LOW_END … /* --------------------------------------------------------------------- */ /* * Index into the table below with the first byte of a UTF-8 sequence to * get the number of trailing bytes that are supposed to follow it. * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is * left as-is for anyone who may want to do such conversion, which was * allowed in earlier algorithms. */ static const char trailingBytesForUTF8[256] = …; /* * Magic values subtracted from a buffer value during UTF8 conversion. * This table contains as many values as there might be trailing bytes * in a UTF-8 sequence. */ static const UTF32 offsetsFromUTF8[6] = …; /* * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed * into the first byte, depending on how many bytes follow. There are * as many entries in this table as there are UTF-8 sequence types. * (I.e., one byte sequence, two byte... etc.). Remember that sequencs * for *legal* UTF-8 will be 4 or fewer bytes total. */ static const UTF8 firstByteMark[7] = …; /* --------------------------------------------------------------------- */ /* The interface converts a whole buffer to avoid function-call overhead. * Constants have been gathered. Loops & conditionals have been removed as * much as possible for efficiency, in favor of drop-through switches. * (See "Note A" at the bottom of the file for equivalent code.) * If your compiler supports it, the "isLegalUTF8" call can be turned * into an inline function. */ /* --------------------------------------------------------------------- */ ConversionResult ConvertUTF32toUTF16 ( const UTF32** sourceStart, const UTF32* sourceEnd, UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { … } /* --------------------------------------------------------------------- */ ConversionResult ConvertUTF16toUTF32 ( const UTF16** sourceStart, const UTF16* sourceEnd, UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { … } ConversionResult ConvertUTF16toUTF8 ( const UTF16** sourceStart, const UTF16* sourceEnd, UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { … } /* --------------------------------------------------------------------- */ ConversionResult ConvertUTF32toUTF8 ( const UTF32** sourceStart, const UTF32* sourceEnd, UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { … } /* --------------------------------------------------------------------- */ /* * Utility routine to tell whether a sequence of bytes is legal UTF-8. * This must be called with the length pre-determined by the first byte. * If not calling this from ConvertUTF8to*, then the length can be set by: * length = trailingBytesForUTF8[*source]+1; * and the sequence is illegal right away if there aren't that many bytes * available. * If presented with a length > 4, this returns false. The Unicode * definition of UTF-8 goes up to 4-byte sequences. */ static Boolean isLegalUTF8(const UTF8 *source, int length) { … } /* --------------------------------------------------------------------- */ /* * Exported function to return whether a UTF-8 sequence is legal or not. * This is not used here; it's just exported. */ Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) { … } /* * Exported function to return the size of the first utf-8 code unit sequence, * Or 0 if the sequence is not valid; */ unsigned getUTF8SequenceSize(const UTF8 *source, const UTF8 *sourceEnd) { … } /* --------------------------------------------------------------------- */ static unsigned findMaximalSubpartOfIllFormedUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) { … } /* --------------------------------------------------------------------- */ /* * Exported function to return the total number of bytes in a codepoint * represented in UTF-8, given the value of the first byte. */ unsigned getNumBytesForUTF8(UTF8 first) { … } /* --------------------------------------------------------------------- */ /* * Exported function to return whether a UTF-8 string is legal or not. * This is not used here; it's just exported. */ Boolean isLegalUTF8String(const UTF8 **source, const UTF8 *sourceEnd) { … } /* --------------------------------------------------------------------- */ ConversionResult ConvertUTF8toUTF16 ( const UTF8** sourceStart, const UTF8* sourceEnd, UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { … } /* --------------------------------------------------------------------- */ static ConversionResult ConvertUTF8toUTF32Impl( const UTF8** sourceStart, const UTF8* sourceEnd, UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags, Boolean InputIsPartial) { … } ConversionResult ConvertUTF8toUTF32Partial(const UTF8 **sourceStart, const UTF8 *sourceEnd, UTF32 **targetStart, UTF32 *targetEnd, ConversionFlags flags) { … } ConversionResult ConvertUTF8toUTF32(const UTF8 **sourceStart, const UTF8 *sourceEnd, UTF32 **targetStart, UTF32 *targetEnd, ConversionFlags flags) { … } /* --------------------------------------------------------------------- Note A. The fall-through switches in UTF-8 reading code save a temp variable, some decrements & conditionals. The switches are equivalent to the following loop: { int tmpBytesToRead = extraBytesToRead+1; do { ch += *source++; --tmpBytesToRead; if (tmpBytesToRead) ch <<= 6; } while (tmpBytesToRead > 0); } In UTF-8 writing code, the switches on "bytesToWrite" are similarly unrolled loops. --------------------------------------------------------------------- */ } // namespace llvm ConvertUTF_RESTORE_WARNINGS
Codebase Browser
llvm