chromium/third_party/sentencepiece/src/src/util.h

// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.!

#ifndef UTIL_H_
#define UTIL_H_

#include <stdio.h>
#include <string.h>

#include <algorithm>
#include <functional>
#include <memory>
#include <random>
#include <sstream>
#include <string>
#include <thread>
#include <utility>
#include <vector>

#include "absl/strings/string_view.h"
#include "common.h"
#include "sentencepiece_processor.h"

#ifdef SPM_NO_THREADLOCAL
#include <pthread.h>
#endif

namespace sentencepiece {
template <typename T>
std::ostream &operator<<(std::ostream &out, const std::vector<T> &v) { … }

uint32 GetRandomGeneratorSeed();

// String utilities
namespace string_util {

template <typename Target>
inline bool lexical_cast(absl::string_view arg, Target *result) { … }

template <>
inline bool lexical_cast(absl::string_view arg, bool *result) { … }

template <>
inline bool lexical_cast(absl::string_view arg, std::string *result) { … }

template <typename T>
inline bool DecodePOD(absl::string_view str, T *result) { … }

template <typename T>
inline std::string EncodePOD(const T &value) { … }

template <typename T>
inline std::string IntToHex(T value) { … }

template <typename T>
inline T HexToInt(absl::string_view value) { … }

template <typename T>
inline size_t Itoa(T val, char *s) { … }

template <typename T>
std::string SimpleItoa(T val) { … }

// Return length of a single UTF-8 source character
inline size_t OneCharLen(const char *src) { … }

// Return (x & 0xC0) == 0x80;
// Since trail bytes are always in [0x80, 0xBF], we can optimize:
inline bool IsTrailByte(char x) { … }

inline bool IsValidCodepoint(char32 c) { … }

bool IsStructurallyValid(absl::string_view str);

UnicodeText;

char32 DecodeUTF8(const char *begin, const char *end, size_t *mblen);

inline char32 DecodeUTF8(absl::string_view input, size_t *mblen) { … }

inline bool IsValidDecodeUTF8(absl::string_view input, size_t *mblen) { … }

size_t EncodeUTF8(char32 c, char *output);

std::string UnicodeCharToUTF8(const char32 c);

UnicodeText UTF8ToUnicodeText(absl::string_view utf8);

std::string UnicodeTextToUTF8(const UnicodeText &utext);

}  // namespace string_util

// other map/ptr utilties
namespace port {

template <class Collection, class Key>
bool ContainsKey(const Collection &collection, const Key &key) { … }

template <class Collection>
const typename Collection::value_type::second_type &FindOrDie(
    const Collection &collection,
    const typename Collection::value_type::first_type &key) { … }

template <class Collection>
const typename Collection::value_type::second_type &FindWithDefault(
    const Collection &collection,
    const typename Collection::value_type::first_type &key,
    const typename Collection::value_type::second_type &value) { … }

template <class Collection>
bool InsertIfNotPresent(Collection *const collection,
                        const typename Collection::value_type &vt) { … }

template <class Collection>
bool InsertIfNotPresent(
    Collection *const collection,
    const typename Collection::value_type::first_type &key,
    const typename Collection::value_type::second_type &value) { … }

template <class Collection>
void InsertOrDie(Collection *const collection,
                 const typename Collection::value_type::first_type &key,
                 const typename Collection::value_type::second_type &data) { … }

// hash
inline void mix(uint64 &a, uint64 &b, uint64 &c) { … }

inline uint64 FingerprintCat(uint64 x, uint64 y) { … }

}  // namespace port

namespace random {

std::mt19937 *GetRandomGenerator();

template <typename T>
class ReservoirSampler { … };

}  // namespace random

util  // namespace util

namespace port {
template <typename T>
void STLDeleteElements(std::vector<T*>* vec) { … }
}  // namespace port

class ThreadPool { … };
}  // namespace sentencepiece
#endif  // UTIL_H_