chromium/third_party/sentencepiece/src/src/model_interface.cc

// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.!

#include <algorithm>

#include "absl/memory/memory.h"
#include "absl/strings/str_format.h"
#include "model_interface.h"
#include "sentencepiece_model.pb.h"
#include "util.h"

namespace sentencepiece {

ModelInterface::ModelInterface(const ModelProto& model_proto)
    : … { … }
ModelInterface::~ModelInterface() { … }

#define RETURN_PIECE …

absl::string_view ModelInterface::unk_piece() const { … }

absl::string_view ModelInterface::bos_piece() const { … }

absl::string_view ModelInterface::eos_piece() const { … }

absl::string_view ModelInterface::pad_piece() const { … }

#undef RETURN_PIECE

int ModelInterface::PieceToId(absl::string_view piece) const { … }

void ModelInterface::InitializePieces() { … }

std::vector<absl::string_view> SplitIntoWords(absl::string_view text,
                                              bool treat_ws_as_suffix,
                                              bool allow_ws_only_pieces) { … }

std::string ByteToPiece(unsigned char c) { … }

int PieceToByte(absl::string_view piece) { … }

}  // namespace sentencepiece