chromium/third_party/sentencepiece/src/src/util.cc

// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.!

#include "util.h"

#include <atomic>
#include <iostream>

namespace sentencepiece {

namespace {
constexpr unsigned int kDefaultSeed = …;
static std::atomic<unsigned int> g_seed = …;
static std::atomic<int> g_minloglevel = …;
}  // namespace

void SetRandomGeneratorSeed(unsigned int seed) { … }

uint32 GetRandomGeneratorSeed() { … }

namespace logging {
int GetMinLogLevel() { … }
void SetMinLogLevel(int v) { … }
}  // namespace logging

namespace string_util {

// mblen sotres the number of bytes consumed after decoding.
char32 DecodeUTF8(const char *begin, const char *end, size_t *mblen) { … }

bool IsStructurallyValid(absl::string_view str) { … }

size_t EncodeUTF8(char32 c, char *output) { … }

std::string UnicodeCharToUTF8(const char32 c) { … }

UnicodeText UTF8ToUnicodeText(absl::string_view utf8) { … }

std::string UnicodeTextToUTF8(const UnicodeText &utext) { … }
}  // namespace string_util

namespace random {
#ifdef SPM_NO_THREADLOCAL
namespace {
class RandomGeneratorStorage {
 public:
  RandomGeneratorStorage() {
    pthread_key_create(&key_, &RandomGeneratorStorage::Delete);
  }
  virtual ~RandomGeneratorStorage() { pthread_key_delete(key_); }

  std::mt19937 *Get() {
    auto *result = static_cast<std::mt19937 *>(pthread_getspecific(key_));
    if (result == nullptr) {
      result = new std::mt19937(GetRandomGeneratorSeed());
      pthread_setspecific(key_, result);
    }
    return result;
  }

 private:
  static void Delete(void *value) { delete static_cast<std::mt19937 *>(value); }
  pthread_key_t key_;
};
}  // namespace

std::mt19937 *GetRandomGenerator() {
  static RandomGeneratorStorage *storage = new RandomGeneratorStorage;
  return storage->Get();
}
#else
std::mt19937 *GetRandomGenerator() { … }
#endif
}  // namespace random

namespace util {

std::string StrError(int errnum) { … }

std::vector<std::string> StrSplitAsCSV(absl::string_view text) { … }

#ifdef OS_WIN
std::wstring Utf8ToWide(absl::string_view input) {
  int output_length = ::MultiByteToWideChar(
      CP_UTF8, 0, input.data(), static_cast<int>(input.size()), nullptr, 0);
  if (output_length == 0) {
    return L"";
  }
  std::wstring output(output_length, 0);
  const int result = ::MultiByteToWideChar(CP_UTF8, 0, input.data(),
                                           static_cast<int>(input.size()),
                                           output.data(), output.size());
  return result == output_length ? output : L"";
}
#endif
}  // namespace util
}  // namespace sentencepiece