chromium/third_party/sentencepiece/src/src/normalizer.cc

// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.!

#include "normalizer.h"

#include <utility>
#include <vector>

#include "absl/memory/memory.h"
#include "absl/strings/match.h"
#include "absl/strings/string_view.h"
#include "absl/strings/strip.h"
#include "common.h"
#include "third_party/darts_clone/darts.h"
#include "util.h"

namespace sentencepiece {
namespace normalizer {

constexpr int Normalizer::kMaxTrieResultsSize;

Normalizer::Normalizer(const NormalizerSpec& spec,
                       const TrainerSpec& trainer_spec)
    : … { … }

Normalizer::Normalizer(const NormalizerSpec& spec)
    : … { … }

Normalizer::~Normalizer() { … }

void Normalizer::Init() { … }

util::Status Normalizer::Normalize(absl::string_view input,
                                   std::string* normalized,
                                   std::vector<size_t>* norm_to_orig) const { … }

std::string Normalizer::Normalize(absl::string_view input) const { … }

std::pair<absl::string_view, int> Normalizer::NormalizePrefix(
    absl::string_view input) const { … }

// static
std::string Normalizer::EncodePrecompiledCharsMap(
    absl::string_view trie_blob, absl::string_view normalized) { … }

// static
util::Status Normalizer::DecodePrecompiledCharsMap(
    absl::string_view blob,
    absl::string_view* trie_blob,
    absl::string_view* normalized,
    std::string* buffer) { … }

PrefixMatcher::PrefixMatcher(const std::set<absl::string_view> &dic) { … }

int PrefixMatcher::PrefixMatch(absl::string_view w, bool *found) const { … }

std::string PrefixMatcher::GlobalReplace(absl::string_view w,
                                         absl::string_view out) const { … }

}  // namespace normalizer
}  // namespace sentencepiece