godot/thirdparty/icu4c/common/lstmbe.cpp

// © 2021 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html

#include <complex>
#include <utility>

#include "unicode/utypes.h"

#if !UCONFIG_NO_BREAK_ITERATION

#include "brkeng.h"
#include "charstr.h"
#include "cmemory.h"
#include "lstmbe.h"
#include "putilimp.h"
#include "uassert.h"
#include "ubrkimpl.h"
#include "uresimp.h"
#include "uvectr32.h"
#include "uvector.h"

#include "unicode/brkiter.h"
#include "unicode/resbund.h"
#include "unicode/ubrk.h"
#include "unicode/uniset.h"
#include "unicode/ustring.h"
#include "unicode/utf.h"

U_NAMESPACE_BEGIN

// Uncomment the following #define to debug.
// #define LSTM_DEBUG 1
// #define LSTM_VECTORIZER_DEBUG 1

/**
 * Interface for reading 1D array.
 */
class ReadArray1D { … };

ReadArray1D::~ReadArray1D()
{ … }

/**
 * Interface for reading 2D array.
 */
class ReadArray2D { … };

ReadArray2D::~ReadArray2D()
{ … }

/**
 * A class to index a float array as a 1D Array without owning the pointer or
 * copy the data.
 */
class ConstArray1D : public ReadArray1D { … };

ConstArray1D::~ConstArray1D()
{ … }

/**
 * A class to index a float array as a 2D Array without owning the pointer or
 * copy the data.
 */
class ConstArray2D : public ReadArray2D { … };

ConstArray2D::~ConstArray2D()
{ … }

/**
 * A class to allocate data as a writable 1D array.
 * This is the main class implement matrix operation.
 */
class Array1D : public ReadArray1D { … };

Array1D::~Array1D()
{ … }

class Array2D : public ReadArray2D { … };

Array2D::~Array2D()
{ … }

LSTMClass;

EmbeddingType;

struct LSTMData : public UMemory { … };

LSTMData::LSTMData(UResourceBundle* rb, UErrorCode &status)
    : … { … }

LSTMData::~LSTMData() { … }

class Vectorizer : public UMemory { … };

Vectorizer::~Vectorizer()
{ … }

class CodePointsVectorizer : public Vectorizer { … };

CodePointsVectorizer::~CodePointsVectorizer()
{ … }

void CodePointsVectorizer::vectorize(
    UText *text, int32_t startPos, int32_t endPos,
    UVector32 &offsets, UVector32 &indices, UErrorCode &status) const
{ … }

class GraphemeClusterVectorizer : public Vectorizer { … };

GraphemeClusterVectorizer::~GraphemeClusterVectorizer()
{ … }

constexpr int32_t MAX_GRAPHEME_CLSTER_LENGTH = …;

void GraphemeClusterVectorizer::vectorize(
    UText *text, int32_t startPos, int32_t endPos,
    UVector32 &offsets, UVector32 &indices, UErrorCode &status) const
{ … }

// Computing LSTM as stated in
// https://en.wikipedia.org/wiki/Long_short-term_memory#LSTM_with_a_forget_gate
// ifco is temp array allocate outside which does not need to be
// input/output value but could avoid unnecessary memory alloc/free if passing
// in.
void compute(
    int32_t hunits,
    const ReadArray2D& W, const ReadArray2D& U, const ReadArray1D& b,
    const ReadArray1D& x, Array1D& h, Array1D& c,
    Array1D& ifco)
{ … }

// Minimum word size
static const int32_t MIN_WORD = …;

// Minimum number of characters for two words
static const int32_t MIN_WORD_SPAN = …;

int32_t
LSTMBreakEngine::divideUpDictionaryRange( UText *text,
                                                int32_t startPos,
                                                int32_t endPos,
                                                UVector32 &foundBreaks,
                                                UBool /* isPhraseBreaking */,
                                                UErrorCode& status) const { … }

Vectorizer* createVectorizer(const LSTMData* data, UErrorCode &status) { … }

LSTMBreakEngine::LSTMBreakEngine(const LSTMData* data, const UnicodeSet& set, UErrorCode &status)
    : … { … }

LSTMBreakEngine::~LSTMBreakEngine() { … }

const char16_t* LSTMBreakEngine::name() const { … }

UnicodeString defaultLSTM(UScriptCode script, UErrorCode& status) { … }

U_CAPI const LSTMData* U_EXPORT2 CreateLSTMDataForScript(UScriptCode script, UErrorCode& status)
{ … }

U_CAPI const LSTMData* U_EXPORT2 CreateLSTMData(UResourceBundle* rb, UErrorCode& status)
{ … }

U_CAPI const LanguageBreakEngine* U_EXPORT2
CreateLSTMBreakEngine(UScriptCode script, const LSTMData* data, UErrorCode& status)
{ … }

U_CAPI void U_EXPORT2 DeleteLSTMData(const LSTMData* data)
{ … }

U_CAPI const char16_t* U_EXPORT2 LSTMDataName(const LSTMData* data)
{ … }

U_NAMESPACE_END

#endif /* #if !UCONFIG_NO_BREAK_ITERATION */