csrsbcs.cpp | Explore in Territory

// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
 **********************************************************************
 *   Copyright (C) 2005-2016, International Business Machines
 *   Corporation and others.  All Rights Reserved.
 **********************************************************************
 */

#include "unicode/utypes.h"

#include "cmemory.h"

#if !UCONFIG_NO_CONVERSION
#include "csrsbcs.h"
#include "csmatch.h"

#define N_GRAM_SIZE …
#define N_GRAM_MASK …

U_NAMESPACE_BEGIN

NGramParser::NGramParser(const int32_t *theNgramList, const uint8_t *theCharMap)
 : … { … }

NGramParser::~NGramParser()
{ … }

/*
 * Binary search for value in table, which must have exactly 64 entries.
 */

int32_t NGramParser::search(const int32_t *table, int32_t value)
{ … }

void NGramParser::lookup(int32_t thisNgram)
{ … }

void NGramParser::addByte(int32_t b)
{ … }

int32_t NGramParser::nextByte(InputText *det)
{ … }

void NGramParser::parseCharacters(InputText *det)
{ … }

int32_t NGramParser::parse(InputText *det)
{ … }

#if !UCONFIG_ONLY_HTML_CONVERSION
static const uint8_t unshapeMap_IBM420[] = {
/*           -0    -1    -2    -3    -4    -5    -6    -7    -8    -9    -A    -B    -C    -D    -E    -F   */
/* 0- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 1- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 2- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 3- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 4- */    0x40, 0x40, 0x42, 0x42, 0x44, 0x45, 0x46, 0x47, 0x47, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 
/* 5- */    0x50, 0x49, 0x52, 0x53, 0x54, 0x55, 0x56, 0x56, 0x58, 0x58, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, 
/* 6- */    0x60, 0x61, 0x62, 0x63, 0x63, 0x65, 0x65, 0x67, 0x67, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 
/* 7- */    0x69, 0x71, 0x71, 0x73, 0x74, 0x75, 0x76, 0x77, 0x77, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, 
/* 8- */    0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x80, 0x8B, 0x8B, 0x8D, 0x8D, 0x8F, 
/* 9- */    0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9A, 0x9A, 0x9A, 0x9E, 0x9E, 
/* A- */    0x9E, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x9E, 0xAB, 0xAB, 0xAD, 0xAD, 0xAF, 
/* B- */    0xAF, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xB1, 0xBB, 0xBB, 0xBD, 0xBD, 0xBF, 
/* C- */    0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xBF, 0xCC, 0xBF, 0xCE, 0xCF, 
/* D- */    0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDA, 0xDC, 0xDC, 0xDC, 0xDF, 
/* E- */    0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 
/* F- */    0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 
};

NGramParser_IBM420::NGramParser_IBM420(const int32_t *theNgramList, const uint8_t *theCharMap):NGramParser(theNgramList, theCharMap)
{
	alef = 0x00;
}

NGramParser_IBM420::~NGramParser_IBM420() {}

int32_t NGramParser_IBM420::isLamAlef(int32_t b)
{
	if(b == 0xB2 || b == 0xB3){
         	return 0x47;        		
        }else if(b == 0xB4 || b == 0xB5){
         	return 0x49;
        }else if(b == 0xB8 || b == 0xB9){
         	return 0x56;
        }else
         	return 0x00;
}

/*
* Arabic shaping needs to be done manually. Cannot call ArabicShaping class
* because CharsetDetector is dealing with bytes not Unicode code points. We could
* convert the bytes to Unicode code points but that would leave us dependent
* on CharsetICU which we try to avoid. IBM420 converter amongst different versions
* of JDK can produce different results and therefore is also avoided.
*/ 
int32_t NGramParser_IBM420::nextByte(InputText *det)
{
	
    if (byteIndex >= det->fInputLen || det->fInputBytes[byteIndex] == 0) {
        return -1;
    }              
    int next;
             
    alef = isLamAlef(det->fInputBytes[byteIndex]);
    if(alef != 0x00)
        next = 0xB1 & 0xFF;
    else
        next = unshapeMap_IBM420[det->fInputBytes[byteIndex]& 0xFF] & 0xFF;
            
    byteIndex++;
             
    return next;
}

void NGramParser_IBM420::parseCharacters(InputText *det)
{
	int32_t b;
    bool ignoreSpace = false;

    while ((b = nextByte(det)) >= 0) {
        uint8_t mb = charMap[b];

        // TODO: 0x20 might not be a space in all character sets...
        if (mb != 0) {
            if (!(mb == 0x20 && ignoreSpace)) {
                addByte(mb);
            }
            ignoreSpace = (mb == 0x20);
        }
		
		if(alef != 0x00){
            mb = charMap[alef & 0xFF];
                     
            // TODO: 0x20 might not be a space in all character sets...
            if (mb != 0) {
                if (!(mb == 0x20 && ignoreSpace)) {
                    addByte(mb);                    
                }
                         
                ignoreSpace = (mb == 0x20);
            }
                	 
        }
    }
}
#endif

CharsetRecog_sbcs::CharsetRecog_sbcs()
{ … }

CharsetRecog_sbcs::~CharsetRecog_sbcs()
{ … }

int32_t CharsetRecog_sbcs::match_sbcs(InputText *det, const int32_t ngrams[],  const uint8_t byteMap[]) const
{ … }

static const uint8_t charMap_8859_1[] = …;

static const uint8_t charMap_8859_2[] = …;

static const uint8_t charMap_8859_5[] = …;

static const uint8_t charMap_8859_6[] = …;

static const uint8_t charMap_8859_7[] = …;

static const uint8_t charMap_8859_8[] = …;

static const uint8_t charMap_8859_9[] = …;

static const int32_t ngrams_windows_1251[] = …;

static const uint8_t charMap_windows_1251[] = …;

static const int32_t ngrams_windows_1256[] = …;

static const uint8_t charMap_windows_1256[] = …;

static const int32_t ngrams_KOI8_R[] = …;

static const uint8_t charMap_KOI8_R[] = …;

#if !UCONFIG_ONLY_HTML_CONVERSION
static const int32_t ngrams_IBM424_he_rtl[] = {
    0x404146, 0x404148, 0x404151, 0x404171, 0x404251, 0x404256, 0x404541, 0x404546, 0x404551, 0x404556, 0x404562, 0x404569, 0x404571, 0x405441, 0x405445, 0x405641, 
    0x406254, 0x406954, 0x417140, 0x454041, 0x454042, 0x454045, 0x454054, 0x454056, 0x454069, 0x454641, 0x464140, 0x465540, 0x465740, 0x466840, 0x467140, 0x514045, 
    0x514540, 0x514671, 0x515155, 0x515540, 0x515740, 0x516840, 0x517140, 0x544041, 0x544045, 0x544140, 0x544540, 0x554041, 0x554042, 0x554045, 0x554054, 0x554056, 
    0x554069, 0x564540, 0x574045, 0x584540, 0x585140, 0x585155, 0x625440, 0x684045, 0x685155, 0x695440, 0x714041, 0x714042, 0x714045, 0x714054, 0x714056, 0x714069, 
};

static const int32_t ngrams_IBM424_he_ltr[] = {
    0x404146, 0x404154, 0x404551, 0x404554, 0x404556, 0x404558, 0x405158, 0x405462, 0x405469, 0x405546, 0x405551, 0x405746, 0x405751, 0x406846, 0x406851, 0x407141,
    0x407146, 0x407151, 0x414045, 0x414054, 0x414055, 0x414071, 0x414540, 0x414645, 0x415440, 0x415640, 0x424045, 0x424055, 0x424071, 0x454045, 0x454051, 0x454054,
    0x454055, 0x454057, 0x454068, 0x454071, 0x455440, 0x464140, 0x464540, 0x484140, 0x514140, 0x514240, 0x514540, 0x544045, 0x544055, 0x544071, 0x546240, 0x546940,
    0x555151, 0x555158, 0x555168, 0x564045, 0x564055, 0x564071, 0x564240, 0x564540, 0x624540, 0x694045, 0x694055, 0x694071, 0x694540, 0x714140, 0x714540, 0x714651,
};

static const uint8_t charMap_IBM424_he[] = {
/*           -0    -1    -2    -3    -4    -5    -6    -7    -8    -9    -A    -B    -C    -D    -E    -F   */
/* 0- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 1- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 2- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 3- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 4- */    0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 5- */    0x40, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 6- */    0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 7- */    0x40, 0x71, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x00, 0x40, 0x40, 
/* 8- */    0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 9- */    0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* A- */    0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* B- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* C- */    0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* D- */    0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* E- */    0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* F- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
};

static const int32_t ngrams_IBM420_ar_rtl[] = {
    0x4056B1, 0x4056BD, 0x405856, 0x409AB1, 0x40ABDC, 0x40B1B1, 0x40BBBD, 0x40CF56, 0x564056, 0x564640, 0x566340, 0x567540, 0x56B140, 0x56B149, 0x56B156, 0x56B158,
    0x56B163, 0x56B167, 0x56B169, 0x56B173, 0x56B178, 0x56B19A, 0x56B1AD, 0x56B1BB, 0x56B1CF, 0x56B1DC, 0x56BB40, 0x56BD40, 0x56BD63, 0x584056, 0x624056, 0x6240AB,
    0x6240B1, 0x6240BB, 0x6240CF, 0x634056, 0x734056, 0x736240, 0x754056, 0x756240, 0x784056, 0x9A4056, 0x9AB1DA, 0xABDC40, 0xB14056, 0xB16240, 0xB1DA40, 0xB1DC40,
    0xBB4056, 0xBB5640, 0xBB6240, 0xBBBD40, 0xBD4056, 0xBF4056, 0xBF5640, 0xCF56B1, 0xCFBD40, 0xDA4056, 0xDC4056, 0xDC40BB, 0xDC40CF, 0xDC6240, 0xDC7540, 0xDCBD40,
};

static const int32_t ngrams_IBM420_ar_ltr[] = {
    0x404656, 0x4056BB, 0x4056BF, 0x406273, 0x406275, 0x4062B1, 0x4062BB, 0x4062DC, 0x406356, 0x407556, 0x4075DC, 0x40B156, 0x40BB56, 0x40BD56, 0x40BDBB, 0x40BDCF, 
    0x40BDDC, 0x40DAB1, 0x40DCAB, 0x40DCB1, 0x49B156, 0x564056, 0x564058, 0x564062, 0x564063, 0x564073, 0x564075, 0x564078, 0x56409A, 0x5640B1, 0x5640BB, 0x5640BD,
    0x5640BF, 0x5640DA, 0x5640DC, 0x565840, 0x56B156, 0x56CF40, 0x58B156, 0x63B156, 0x63BD56, 0x67B156, 0x69B156, 0x73B156, 0x78B156, 0x9AB156, 0xAB4062, 0xADB156,
    0xB14062, 0xB15640, 0xB156CF, 0xB19A40, 0xB1B140, 0xBB4062, 0xBB40DC, 0xBBB156, 0xBD5640, 0xBDBB40, 0xCF4062, 0xCF40DC, 0xCFB156, 0xDAB19A, 0xDCAB40, 0xDCB156
};

static const uint8_t charMap_IBM420_ar[]= {
/*           -0    -1    -2    -3    -4    -5    -6    -7    -8    -9    -A    -B    -C    -D    -E    -F   */
/* 0- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 1- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 2- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 3- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 4- */    0x40, 0x40, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 5- */    0x40, 0x51, 0x52, 0x40, 0x40, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 6- */    0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 7- */    0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 
/* 8- */    0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, 
/* 9- */    0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F, 
/* A- */    0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 
/* B- */    0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0x40, 0x40, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, 
/* C- */    0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0xCB, 0x40, 0xCD, 0x40, 0xCF, 
/* D- */    0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 
/* E- */    0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xEA, 0xEB, 0x40, 0xED, 0xEE, 0xEF, 
/* F- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0xFB, 0xFC, 0xFD, 0xFE, 0x40, 
};
#endif

//ISO-8859-1,2,5,6,7,8,9 Ngrams

struct NGramsPlusLang { … };

static const NGramsPlusLang ngrams_8859_1[] = …;


static const NGramsPlusLang ngrams_8859_2[] = …;

static const int32_t ngrams_8859_5_ru[] = …;

static const int32_t ngrams_8859_6_ar[] = …;

static const int32_t ngrams_8859_7_el[] = …;

static const int32_t ngrams_8859_8_I_he[] = …;

static const int32_t ngrams_8859_8_he[] = …;

static const int32_t ngrams_8859_9_tr[] = …;

CharsetRecog_8859_1::~CharsetRecog_8859_1()
{ … }

UBool CharsetRecog_8859_1::match(InputText *textIn, CharsetMatch *results) const { … }

const char *CharsetRecog_8859_1::getName() const
{ … }


CharsetRecog_8859_2::~CharsetRecog_8859_2()
{ … }

UBool CharsetRecog_8859_2::match(InputText *textIn, CharsetMatch *results) const { … }

const char *CharsetRecog_8859_2::getName() const
{ … }


CharsetRecog_8859_5::~CharsetRecog_8859_5()
{ … }

const char *CharsetRecog_8859_5::getName() const
{ … }

CharsetRecog_8859_5_ru::~CharsetRecog_8859_5_ru()
{ … }

const char *CharsetRecog_8859_5_ru::getLanguage() const
{ … }

UBool CharsetRecog_8859_5_ru::match(InputText *textIn, CharsetMatch *results) const
{ … }

CharsetRecog_8859_6::~CharsetRecog_8859_6()
{ … }

const char *CharsetRecog_8859_6::getName() const
{ … }

CharsetRecog_8859_6_ar::~CharsetRecog_8859_6_ar()
{ … }

const char *CharsetRecog_8859_6_ar::getLanguage() const
{ … }

UBool CharsetRecog_8859_6_ar::match(InputText *textIn, CharsetMatch *results) const
{ … }

CharsetRecog_8859_7::~CharsetRecog_8859_7()
{ … }

const char *CharsetRecog_8859_7::getName() const
{ … }

CharsetRecog_8859_7_el::~CharsetRecog_8859_7_el()
{ … }

const char *CharsetRecog_8859_7_el::getLanguage() const
{ … }

UBool CharsetRecog_8859_7_el::match(InputText *textIn, CharsetMatch *results) const
{ … }

CharsetRecog_8859_8::~CharsetRecog_8859_8()
{ … }

const char *CharsetRecog_8859_8::getName() const 
{ … }

CharsetRecog_8859_8_I_he::~CharsetRecog_8859_8_I_he ()
{ … }

const char *CharsetRecog_8859_8_I_he::getName() const
{ … }

const char *CharsetRecog_8859_8_I_he::getLanguage() const
{ … }

UBool CharsetRecog_8859_8_I_he::match(InputText *textIn, CharsetMatch *results) const
{ … }

CharsetRecog_8859_8_he::~CharsetRecog_8859_8_he()
{ … }

const char *CharsetRecog_8859_8_he::getLanguage() const
{ … }

UBool CharsetRecog_8859_8_he::match(InputText *textIn, CharsetMatch *results) const
{ … }

CharsetRecog_8859_9::~CharsetRecog_8859_9()
{ … }

const char *CharsetRecog_8859_9::getName() const
{ … }

CharsetRecog_8859_9_tr::~CharsetRecog_8859_9_tr ()
{ … }

const char *CharsetRecog_8859_9_tr::getLanguage() const
{ … }

UBool CharsetRecog_8859_9_tr::match(InputText *textIn, CharsetMatch *results) const
{ … }

CharsetRecog_windows_1256::~CharsetRecog_windows_1256()
{ … }

const char *CharsetRecog_windows_1256::getName() const
{ … }

const char *CharsetRecog_windows_1256::getLanguage() const
{ … }

UBool CharsetRecog_windows_1256::match(InputText *textIn, CharsetMatch *results) const
{ … }

CharsetRecog_windows_1251::~CharsetRecog_windows_1251()
{ … }

const char *CharsetRecog_windows_1251::getName() const
{ … }

const char *CharsetRecog_windows_1251::getLanguage() const
{ … }

UBool CharsetRecog_windows_1251::match(InputText *textIn, CharsetMatch *results) const
{ … }

CharsetRecog_KOI8_R::~CharsetRecog_KOI8_R()
{ … }

const char *CharsetRecog_KOI8_R::getName() const
{ … }

const char *CharsetRecog_KOI8_R::getLanguage() const
{ … }

UBool CharsetRecog_KOI8_R::match(InputText *textIn, CharsetMatch *results) const
{ … }

#if !UCONFIG_ONLY_HTML_CONVERSION
CharsetRecog_IBM424_he::~CharsetRecog_IBM424_he()
{
    // nothing to do
}

const char *CharsetRecog_IBM424_he::getLanguage() const
{
    return "he";
}

CharsetRecog_IBM424_he_rtl::~CharsetRecog_IBM424_he_rtl()
{
    // nothing to do
}

const char *CharsetRecog_IBM424_he_rtl::getName() const
{
    return  "IBM424_rtl";
}

UBool CharsetRecog_IBM424_he_rtl::match(InputText *textIn, CharsetMatch *results) const
{
    int32_t confidence = match_sbcs(textIn, ngrams_IBM424_he_rtl, charMap_IBM424_he);
    results->set(textIn, this, confidence);
    return (confidence > 0);
}

CharsetRecog_IBM424_he_ltr::~CharsetRecog_IBM424_he_ltr()
{
    // nothing to do
}

const char *CharsetRecog_IBM424_he_ltr::getName() const
{
    return  "IBM424_ltr";
}

UBool CharsetRecog_IBM424_he_ltr::match(InputText *textIn, CharsetMatch *results) const
{
    int32_t confidence = match_sbcs(textIn, ngrams_IBM424_he_ltr, charMap_IBM424_he);
    results->set(textIn, this, confidence);
    return (confidence > 0);
}

CharsetRecog_IBM420_ar::~CharsetRecog_IBM420_ar()
{
    // nothing to do
}

const char *CharsetRecog_IBM420_ar::getLanguage() const
{
    return "ar";
}

    
int32_t CharsetRecog_IBM420_ar::match_sbcs(InputText *det, const int32_t ngrams[],  const uint8_t byteMap[]) const
{
    NGramParser_IBM420 parser(ngrams, byteMap);
    int32_t result;
    
    result = parser.parse(det);
        
    return result;
}

CharsetRecog_IBM420_ar_rtl::~CharsetRecog_IBM420_ar_rtl()
{
    // nothing to do
}

const char *CharsetRecog_IBM420_ar_rtl::getName() const
{
    return  "IBM420_rtl";
}

UBool CharsetRecog_IBM420_ar_rtl::match(InputText *textIn, CharsetMatch *results) const
{
    int32_t confidence = match_sbcs(textIn, ngrams_IBM420_ar_rtl, charMap_IBM420_ar);
    results->set(textIn, this, confidence);
    return (confidence > 0);
}

CharsetRecog_IBM420_ar_ltr::~CharsetRecog_IBM420_ar_ltr()
{
    // nothing to do
}

const char *CharsetRecog_IBM420_ar_ltr::getName() const
{
    return  "IBM420_ltr";
}

UBool CharsetRecog_IBM420_ar_ltr::match(InputText *textIn, CharsetMatch *results) const
{
    int32_t confidence = match_sbcs(textIn, ngrams_IBM420_ar_ltr, charMap_IBM420_ar);
    results->set(textIn, this, confidence);
    return (confidence > 0);
}
#endif

U_NAMESPACE_END
#endif
chromium/third_party/icu/source/i18n/csrsbcs.cpp
