// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: sentencepiece_model.proto
#include "sentencepiece_model.pb.h"
#include <algorithm>
#include <google/protobuf/extension_set.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
#include <google/protobuf/wire_format_lite.h>
// @@protoc_insertion_point(includes)
#include <google/protobuf/port_def.inc>
extern PROTOBUF_INTERNAL_EXPORT_sentencepiece_5fmodel_2eproto ::
PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0>
scc_info_ModelProto_SentencePiece_sentencepiece_5fmodel_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_sentencepiece_5fmodel_2eproto ::
PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0>
scc_info_NormalizerSpec_sentencepiece_5fmodel_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_sentencepiece_5fmodel_2eproto ::
PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1>
scc_info_SelfTestData_sentencepiece_5fmodel_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_sentencepiece_5fmodel_2eproto ::
PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0>
scc_info_SelfTestData_Sample_sentencepiece_5fmodel_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_sentencepiece_5fmodel_2eproto ::
PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0>
scc_info_TrainerSpec_sentencepiece_5fmodel_2eproto;
namespace sentencepiece {
class TrainerSpecDefaultTypeInternal {
public:
::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed<TrainerSpec>
_instance;
} _TrainerSpec_default_instance_;
class NormalizerSpecDefaultTypeInternal {
public:
::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed<NormalizerSpec>
_instance;
} _NormalizerSpec_default_instance_;
class SelfTestData_SampleDefaultTypeInternal {
public:
::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed<SelfTestData_Sample>
_instance;
} _SelfTestData_Sample_default_instance_;
class SelfTestDataDefaultTypeInternal {
public:
::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed<SelfTestData>
_instance;
} _SelfTestData_default_instance_;
class ModelProto_SentencePieceDefaultTypeInternal {
public:
::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed<
ModelProto_SentencePiece>
_instance;
} _ModelProto_SentencePiece_default_instance_;
class ModelProtoDefaultTypeInternal {
public:
::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed<ModelProto>
_instance;
} _ModelProto_default_instance_;
} // namespace sentencepiece
static void InitDefaultsscc_info_ModelProto_sentencepiece_5fmodel_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
{
void* ptr = &::sentencepiece::_ModelProto_default_instance_;
new (ptr)::sentencepiece::ModelProto();
::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
}
}
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<4>
scc_info_ModelProto_sentencepiece_5fmodel_2eproto = {
{ATOMIC_VAR_INIT(
::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized),
4, 0, InitDefaultsscc_info_ModelProto_sentencepiece_5fmodel_2eproto},
{
&scc_info_ModelProto_SentencePiece_sentencepiece_5fmodel_2eproto
.base,
&scc_info_TrainerSpec_sentencepiece_5fmodel_2eproto.base,
&scc_info_NormalizerSpec_sentencepiece_5fmodel_2eproto.base,
&scc_info_SelfTestData_sentencepiece_5fmodel_2eproto.base,
}};
static void
InitDefaultsscc_info_ModelProto_SentencePiece_sentencepiece_5fmodel_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
{
void* ptr = &::sentencepiece::_ModelProto_SentencePiece_default_instance_;
new (ptr)::sentencepiece::ModelProto_SentencePiece();
::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
}
}
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0>
scc_info_ModelProto_SentencePiece_sentencepiece_5fmodel_2eproto = {
{ATOMIC_VAR_INIT(
::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized),
0, 0,
InitDefaultsscc_info_ModelProto_SentencePiece_sentencepiece_5fmodel_2eproto},
{}};
static void
InitDefaultsscc_info_NormalizerSpec_sentencepiece_5fmodel_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
{
void* ptr = &::sentencepiece::_NormalizerSpec_default_instance_;
new (ptr)::sentencepiece::NormalizerSpec();
::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
}
}
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0>
scc_info_NormalizerSpec_sentencepiece_5fmodel_2eproto = {
{ATOMIC_VAR_INIT(
::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized),
0, 0,
InitDefaultsscc_info_NormalizerSpec_sentencepiece_5fmodel_2eproto},
{}};
static void InitDefaultsscc_info_SelfTestData_sentencepiece_5fmodel_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
{
void* ptr = &::sentencepiece::_SelfTestData_default_instance_;
new (ptr)::sentencepiece::SelfTestData();
::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
}
}
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1>
scc_info_SelfTestData_sentencepiece_5fmodel_2eproto = {
{ATOMIC_VAR_INIT(
::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized),
1, 0, InitDefaultsscc_info_SelfTestData_sentencepiece_5fmodel_2eproto},
{
&scc_info_SelfTestData_Sample_sentencepiece_5fmodel_2eproto.base,
}};
static void
InitDefaultsscc_info_SelfTestData_Sample_sentencepiece_5fmodel_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
{
void* ptr = &::sentencepiece::_SelfTestData_Sample_default_instance_;
new (ptr)::sentencepiece::SelfTestData_Sample();
::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
}
}
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0>
scc_info_SelfTestData_Sample_sentencepiece_5fmodel_2eproto = {
{ATOMIC_VAR_INIT(
::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized),
0, 0,
InitDefaultsscc_info_SelfTestData_Sample_sentencepiece_5fmodel_2eproto},
{}};
static void InitDefaultsscc_info_TrainerSpec_sentencepiece_5fmodel_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
{
void* ptr = &::sentencepiece::_TrainerSpec_default_instance_;
new (ptr)::sentencepiece::TrainerSpec();
::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
}
}
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0>
scc_info_TrainerSpec_sentencepiece_5fmodel_2eproto = {
{ATOMIC_VAR_INIT(
::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized),
0, 0, InitDefaultsscc_info_TrainerSpec_sentencepiece_5fmodel_2eproto},
{}};
namespace sentencepiece {
bool TrainerSpec_ModelType_IsValid(int value) {
switch (value) {
case 1:
case 2:
case 3:
case 4:
return true;
default:
return false;
}
}
static ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed<std::string>
TrainerSpec_ModelType_strings[4] = {};
static const char TrainerSpec_ModelType_names[] =
"BPE"
"CHAR"
"UNIGRAM"
"WORD";
static const ::PROTOBUF_NAMESPACE_ID::internal::EnumEntry
TrainerSpec_ModelType_entries[] = {
{{TrainerSpec_ModelType_names + 0, 3}, 2},
{{TrainerSpec_ModelType_names + 3, 4}, 4},
{{TrainerSpec_ModelType_names + 7, 7}, 1},
{{TrainerSpec_ModelType_names + 14, 4}, 3},
};
static const int TrainerSpec_ModelType_entries_by_number[] = {
2, // 1 -> UNIGRAM
0, // 2 -> BPE
3, // 3 -> WORD
1, // 4 -> CHAR
};
const std::string& TrainerSpec_ModelType_Name(TrainerSpec_ModelType value) {
static const bool dummy =
::PROTOBUF_NAMESPACE_ID::internal::InitializeEnumStrings(
TrainerSpec_ModelType_entries,
TrainerSpec_ModelType_entries_by_number, 4,
TrainerSpec_ModelType_strings);
(void)dummy;
int idx = ::PROTOBUF_NAMESPACE_ID::internal::LookUpEnumName(
TrainerSpec_ModelType_entries, TrainerSpec_ModelType_entries_by_number, 4,
value);
return idx == -1 ? ::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString()
: TrainerSpec_ModelType_strings[idx].get();
}
bool TrainerSpec_ModelType_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name,
TrainerSpec_ModelType* value) {
int int_value;
bool success = ::PROTOBUF_NAMESPACE_ID::internal::LookUpEnumValue(
TrainerSpec_ModelType_entries, 4, name, &int_value);
if (success) {
*value = static_cast<TrainerSpec_ModelType>(int_value);
}
return success;
}
#if (__cplusplus < 201703) && (!defined(_MSC_VER) || _MSC_VER >= 1900)
constexpr TrainerSpec_ModelType TrainerSpec::UNIGRAM;
constexpr TrainerSpec_ModelType TrainerSpec::BPE;
constexpr TrainerSpec_ModelType TrainerSpec::WORD;
constexpr TrainerSpec_ModelType TrainerSpec::CHAR;
constexpr TrainerSpec_ModelType TrainerSpec::ModelType_MIN;
constexpr TrainerSpec_ModelType TrainerSpec::ModelType_MAX;
constexpr int TrainerSpec::ModelType_ARRAYSIZE;
#endif // (__cplusplus < 201703) && (!defined(_MSC_VER) || _MSC_VER >= 1900)
bool ModelProto_SentencePiece_Type_IsValid(int value) {
switch (value) {
case 1:
case 2:
case 3:
case 4:
case 5:
case 6:
return true;
default:
return false;
}
}
static ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed<std::string>
ModelProto_SentencePiece_Type_strings[6] = {};
static const char ModelProto_SentencePiece_Type_names[] =
"BYTE"
"CONTROL"
"NORMAL"
"UNKNOWN"
"UNUSED"
"USER_DEFINED";
static const ::PROTOBUF_NAMESPACE_ID::internal::EnumEntry
ModelProto_SentencePiece_Type_entries[] = {
{{ModelProto_SentencePiece_Type_names + 0, 4}, 6},
{{ModelProto_SentencePiece_Type_names + 4, 7}, 3},
{{ModelProto_SentencePiece_Type_names + 11, 6}, 1},
{{ModelProto_SentencePiece_Type_names + 17, 7}, 2},
{{ModelProto_SentencePiece_Type_names + 24, 6}, 5},
{{ModelProto_SentencePiece_Type_names + 30, 12}, 4},
};
static const int ModelProto_SentencePiece_Type_entries_by_number[] = {
2, // 1 -> NORMAL
3, // 2 -> UNKNOWN
1, // 3 -> CONTROL
5, // 4 -> USER_DEFINED
4, // 5 -> UNUSED
0, // 6 -> BYTE
};
const std::string& ModelProto_SentencePiece_Type_Name(
ModelProto_SentencePiece_Type value) {
static const bool dummy =
::PROTOBUF_NAMESPACE_ID::internal::InitializeEnumStrings(
ModelProto_SentencePiece_Type_entries,
ModelProto_SentencePiece_Type_entries_by_number, 6,
ModelProto_SentencePiece_Type_strings);
(void)dummy;
int idx = ::PROTOBUF_NAMESPACE_ID::internal::LookUpEnumName(
ModelProto_SentencePiece_Type_entries,
ModelProto_SentencePiece_Type_entries_by_number, 6, value);
return idx == -1 ? ::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString()
: ModelProto_SentencePiece_Type_strings[idx].get();
}
bool ModelProto_SentencePiece_Type_Parse(
::PROTOBUF_NAMESPACE_ID::ConstStringParam name,
ModelProto_SentencePiece_Type* value) {
int int_value;
bool success = ::PROTOBUF_NAMESPACE_ID::internal::LookUpEnumValue(
ModelProto_SentencePiece_Type_entries, 6, name, &int_value);
if (success) {
*value = static_cast<ModelProto_SentencePiece_Type>(int_value);
}
return success;
}
#if (__cplusplus < 201703) && (!defined(_MSC_VER) || _MSC_VER >= 1900)
constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::NORMAL;
constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::UNKNOWN;
constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::CONTROL;
constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::USER_DEFINED;
constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::BYTE;
constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::UNUSED;
constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::Type_MIN;
constexpr ModelProto_SentencePiece_Type ModelProto_SentencePiece::Type_MAX;
constexpr int ModelProto_SentencePiece::Type_ARRAYSIZE;
#endif // (__cplusplus < 201703) && (!defined(_MSC_VER) || _MSC_VER >= 1900)
// ===================================================================
class TrainerSpec::_Internal {
public:
using HasBits = decltype(std::declval<TrainerSpec>()._has_bits_);
static void set_has_input_format(HasBits* has_bits) { (*has_bits)[0] |= 2u; }
static void set_has_model_prefix(HasBits* has_bits) { (*has_bits)[0] |= 1u; }
static void set_has_model_type(HasBits* has_bits) {
(*has_bits)[0] |= 8388608u;
}
static void set_has_vocab_size(HasBits* has_bits) {
(*has_bits)[0] |= 16777216u;
}
static void set_has_self_test_sample_size(HasBits* has_bits) {
(*has_bits)[0] |= 512u;
}
static void set_has_enable_differential_privacy(HasBits* has_bits) {
(*has_bits)[0] |= 8192u;
}
static void set_has_differential_privacy_noise_level(HasBits* has_bits) {
(*has_bits)[0] |= 2097152u;
}
static void set_has_differential_privacy_clipping_threshold(
HasBits* has_bits) {
(*has_bits)[0] |= 4194304u;
}
static void set_has_character_coverage(HasBits* has_bits) {
(*has_bits)[0] |= 33554432u;
}
static void set_has_input_sentence_size(HasBits* has_bits) {
(*has_bits)[0] |= 2048u;
}
static void set_has_shuffle_input_sentence(HasBits* has_bits) {
(*has_bits)[1] |= 1u;
}
static void set_has_mining_sentence_size(HasBits* has_bits) {
(*has_bits)[0] |= 1024u;
}
static void set_has_training_sentence_size(HasBits* has_bits) {
(*has_bits)[0] |= 4096u;
}
static void set_has_seed_sentencepiece_size(HasBits* has_bits) {
(*has_bits)[0] |= 67108864u;
}
static void set_has_shrinking_factor(HasBits* has_bits) {
(*has_bits)[0] |= 134217728u;
}
static void set_has_max_sentence_length(HasBits* has_bits) {
(*has_bits)[0] |= 1073741824u;
}
static void set_has_num_threads(HasBits* has_bits) {
(*has_bits)[0] |= 268435456u;
}
static void set_has_num_sub_iterations(HasBits* has_bits) {
(*has_bits)[0] |= 536870912u;
}
static void set_has_max_sentencepiece_length(HasBits* has_bits) {
(*has_bits)[0] |= 2147483648u;
}
static void set_has_split_by_unicode_script(HasBits* has_bits) {
(*has_bits)[1] |= 2u;
}
static void set_has_split_by_number(HasBits* has_bits) {
(*has_bits)[1] |= 4u;
}
static void set_has_split_by_whitespace(HasBits* has_bits) {
(*has_bits)[1] |= 8u;
}
static void set_has_treat_whitespace_as_suffix(HasBits* has_bits) {
(*has_bits)[0] |= 16384u;
}
static void set_has_allow_whitespace_only_pieces(HasBits* has_bits) {
(*has_bits)[0] |= 32768u;
}
static void set_has_split_digits(HasBits* has_bits) {
(*has_bits)[0] |= 65536u;
}
static void set_has_pretokenization_delimiter(HasBits* has_bits) {
(*has_bits)[0] |= 256u;
}
static void set_has_required_chars(HasBits* has_bits) {
(*has_bits)[0] |= 4u;
}
static void set_has_byte_fallback(HasBits* has_bits) {
(*has_bits)[0] |= 131072u;
}
static void set_has_vocabulary_output_piece_score(HasBits* has_bits) {
(*has_bits)[1] |= 16u;
}
static void set_has_hard_vocab_limit(HasBits* has_bits) {
(*has_bits)[1] |= 32u;
}
static void set_has_use_all_vocab(HasBits* has_bits) {
(*has_bits)[0] |= 262144u;
}
static void set_has_unk_id(HasBits* has_bits) { (*has_bits)[0] |= 1048576u; }
static void set_has_bos_id(HasBits* has_bits) { (*has_bits)[1] |= 64u; }
static void set_has_eos_id(HasBits* has_bits) { (*has_bits)[1] |= 128u; }
static void set_has_pad_id(HasBits* has_bits) { (*has_bits)[1] |= 256u; }
static void set_has_unk_piece(HasBits* has_bits) { (*has_bits)[0] |= 16u; }
static void set_has_bos_piece(HasBits* has_bits) { (*has_bits)[0] |= 32u; }
static void set_has_eos_piece(HasBits* has_bits) { (*has_bits)[0] |= 64u; }
static void set_has_pad_piece(HasBits* has_bits) { (*has_bits)[0] |= 128u; }
static void set_has_unk_surface(HasBits* has_bits) { (*has_bits)[0] |= 8u; }
static void set_has_train_extremely_large_corpus(HasBits* has_bits) {
(*has_bits)[0] |= 524288u;
}
};
const ::PROTOBUF_NAMESPACE_ID::internal::LazyString
TrainerSpec::_i_give_permission_to_break_this_code_default_unk_piece_{
{{"<unk>", 5}},
{nullptr}};
const ::PROTOBUF_NAMESPACE_ID::internal::LazyString
TrainerSpec::_i_give_permission_to_break_this_code_default_bos_piece_{
{{"<s>", 3}},
{nullptr}};
const ::PROTOBUF_NAMESPACE_ID::internal::LazyString
TrainerSpec::_i_give_permission_to_break_this_code_default_eos_piece_{
{{"</s>", 4}},
{nullptr}};
const ::PROTOBUF_NAMESPACE_ID::internal::LazyString
TrainerSpec::_i_give_permission_to_break_this_code_default_pad_piece_{
{{"<pad>", 5}},
{nullptr}};
const ::PROTOBUF_NAMESPACE_ID::internal::LazyString
TrainerSpec::_i_give_permission_to_break_this_code_default_unk_surface_{
{{" \342\201\207 ", 5}},
{nullptr}};
TrainerSpec::TrainerSpec(::PROTOBUF_NAMESPACE_ID::Arena* arena)
: ::PROTOBUF_NAMESPACE_ID::MessageLite(arena),
_extensions_(arena),
input_(arena),
accept_language_(arena),
control_symbols_(arena),
user_defined_symbols_(arena) {
SharedCtor();
RegisterArenaDtor(arena);
// @@protoc_insertion_point(arena_constructor:sentencepiece.TrainerSpec)
}
TrainerSpec::TrainerSpec(const TrainerSpec& from)
: ::PROTOBUF_NAMESPACE_ID::MessageLite(),
_has_bits_(from._has_bits_),
input_(from.input_),
accept_language_(from.accept_language_),
control_symbols_(from.control_symbols_),
user_defined_symbols_(from.user_defined_symbols_) {
_internal_metadata_.MergeFrom<std::string>(from._internal_metadata_);
_extensions_.MergeFrom(from._extensions_);
model_prefix_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (from._internal_has_model_prefix()) {
model_prefix_.Set(
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{},
from._internal_model_prefix(), GetArena());
}
input_format_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (from._internal_has_input_format()) {
input_format_.Set(
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{},
from._internal_input_format(), GetArena());
}
required_chars_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (from._internal_has_required_chars()) {
required_chars_.Set(
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{},
from._internal_required_chars(), GetArena());
}
unk_surface_.UnsafeSetDefault(nullptr);
if (from._internal_has_unk_surface()) {
unk_surface_.Set(
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::NonEmptyDefault{},
from._internal_unk_surface(), GetArena());
}
unk_piece_.UnsafeSetDefault(nullptr);
if (from._internal_has_unk_piece()) {
unk_piece_.Set(
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::NonEmptyDefault{},
from._internal_unk_piece(), GetArena());
}
bos_piece_.UnsafeSetDefault(nullptr);
if (from._internal_has_bos_piece()) {
bos_piece_.Set(
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::NonEmptyDefault{},
from._internal_bos_piece(), GetArena());
}
eos_piece_.UnsafeSetDefault(nullptr);
if (from._internal_has_eos_piece()) {
eos_piece_.Set(
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::NonEmptyDefault{},
from._internal_eos_piece(), GetArena());
}
pad_piece_.UnsafeSetDefault(nullptr);
if (from._internal_has_pad_piece()) {
pad_piece_.Set(
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::NonEmptyDefault{},
from._internal_pad_piece(), GetArena());
}
pretokenization_delimiter_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (from._internal_has_pretokenization_delimiter()) {
pretokenization_delimiter_.Set(
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{},
from._internal_pretokenization_delimiter(), GetArena());
}
::memcpy(
&self_test_sample_size_, &from.self_test_sample_size_,
static_cast<size_t>(reinterpret_cast<char*>(&pad_id_) -
reinterpret_cast<char*>(&self_test_sample_size_)) +
sizeof(pad_id_));
// @@protoc_insertion_point(copy_constructor:sentencepiece.TrainerSpec)
}
void TrainerSpec::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(
&scc_info_TrainerSpec_sentencepiece_5fmodel_2eproto.base);
model_prefix_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
input_format_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
required_chars_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
unk_surface_.UnsafeSetDefault(nullptr);
unk_piece_.UnsafeSetDefault(nullptr);
bos_piece_.UnsafeSetDefault(nullptr);
eos_piece_.UnsafeSetDefault(nullptr);
pad_piece_.UnsafeSetDefault(nullptr);
pretokenization_delimiter_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
::memset(
reinterpret_cast<char*>(this) +
static_cast<size_t>(reinterpret_cast<char*>(&self_test_sample_size_) -
reinterpret_cast<char*>(this)),
0,
static_cast<size_t>(
reinterpret_cast<char*>(&differential_privacy_clipping_threshold_) -
reinterpret_cast<char*>(&self_test_sample_size_)) +
sizeof(differential_privacy_clipping_threshold_));
model_type_ = 1;
vocab_size_ = 8000;
character_coverage_ = 0.9995f;
seed_sentencepiece_size_ = 1000000;
shrinking_factor_ = 0.75f;
num_threads_ = 16;
num_sub_iterations_ = 2;
max_sentence_length_ = 4192;
max_sentencepiece_length_ = 16;
shuffle_input_sentence_ = true;
split_by_unicode_script_ = true;
split_by_number_ = true;
split_by_whitespace_ = true;
vocabulary_output_piece_score_ = true;
hard_vocab_limit_ = true;
bos_id_ = 1;
eos_id_ = 2;
pad_id_ = -1;
}
TrainerSpec::~TrainerSpec() {
// @@protoc_insertion_point(destructor:sentencepiece.TrainerSpec)
SharedDtor();
_internal_metadata_.Delete<std::string>();
}
void TrainerSpec::SharedDtor() {
GOOGLE_DCHECK(GetArena() == nullptr);
model_prefix_.DestroyNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
input_format_.DestroyNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
required_chars_.DestroyNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
unk_surface_.DestroyNoArena(nullptr);
unk_piece_.DestroyNoArena(nullptr);
bos_piece_.DestroyNoArena(nullptr);
eos_piece_.DestroyNoArena(nullptr);
pad_piece_.DestroyNoArena(nullptr);
pretokenization_delimiter_.DestroyNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
void TrainerSpec::ArenaDtor(void* object) {
TrainerSpec* _this = reinterpret_cast<TrainerSpec*>(object);
(void)_this;
}
void TrainerSpec::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) {}
void TrainerSpec::SetCachedSize(int size) const {
_cached_size_.Set(size);
}
const TrainerSpec& TrainerSpec::default_instance() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(
&::scc_info_TrainerSpec_sentencepiece_5fmodel_2eproto.base);
return *internal_default_instance();
}
void TrainerSpec::Clear() {
// @@protoc_insertion_point(message_clear_start:sentencepiece.TrainerSpec)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void)cached_has_bits;
_extensions_.Clear();
input_.Clear();
accept_language_.Clear();
control_symbols_.Clear();
user_defined_symbols_.Clear();
cached_has_bits = _has_bits_[0];
if (cached_has_bits & 0x000000ffu) {
if (cached_has_bits & 0x00000001u) {
model_prefix_.ClearNonDefaultToEmpty();
}
if (cached_has_bits & 0x00000002u) {
input_format_.ClearNonDefaultToEmpty();
}
if (cached_has_bits & 0x00000004u) {
required_chars_.ClearNonDefaultToEmpty();
}
if (cached_has_bits & 0x00000008u) {
unk_surface_.ClearToDefault(
::sentencepiece::TrainerSpec::
_i_give_permission_to_break_this_code_default_unk_surface_,
GetArena());
}
if (cached_has_bits & 0x00000010u) {
unk_piece_.ClearToDefault(
::sentencepiece::TrainerSpec::
_i_give_permission_to_break_this_code_default_unk_piece_,
GetArena());
}
if (cached_has_bits & 0x00000020u) {
bos_piece_.ClearToDefault(
::sentencepiece::TrainerSpec::
_i_give_permission_to_break_this_code_default_bos_piece_,
GetArena());
}
if (cached_has_bits & 0x00000040u) {
eos_piece_.ClearToDefault(
::sentencepiece::TrainerSpec::
_i_give_permission_to_break_this_code_default_eos_piece_,
GetArena());
}
if (cached_has_bits & 0x00000080u) {
pad_piece_.ClearToDefault(
::sentencepiece::TrainerSpec::
_i_give_permission_to_break_this_code_default_pad_piece_,
GetArena());
}
}
if (cached_has_bits & 0x00000100u) {
pretokenization_delimiter_.ClearNonDefaultToEmpty();
}
if (cached_has_bits & 0x0000fe00u) {
::memset(&self_test_sample_size_, 0,
static_cast<size_t>(
reinterpret_cast<char*>(&allow_whitespace_only_pieces_) -
reinterpret_cast<char*>(&self_test_sample_size_)) +
sizeof(allow_whitespace_only_pieces_));
}
if (cached_has_bits & 0x00ff0000u) {
::memset(
&split_digits_, 0,
static_cast<size_t>(
reinterpret_cast<char*>(&differential_privacy_clipping_threshold_) -
reinterpret_cast<char*>(&split_digits_)) +
sizeof(differential_privacy_clipping_threshold_));
model_type_ = 1;
}
if (cached_has_bits & 0xff000000u) {
vocab_size_ = 8000;
character_coverage_ = 0.9995f;
seed_sentencepiece_size_ = 1000000;
shrinking_factor_ = 0.75f;
num_threads_ = 16;
num_sub_iterations_ = 2;
max_sentence_length_ = 4192;
max_sentencepiece_length_ = 16;
}
cached_has_bits = _has_bits_[1];
if (cached_has_bits & 0x000000ffu) {
shuffle_input_sentence_ = true;
split_by_unicode_script_ = true;
split_by_number_ = true;
split_by_whitespace_ = true;
vocabulary_output_piece_score_ = true;
hard_vocab_limit_ = true;
bos_id_ = 1;
eos_id_ = 2;
}
pad_id_ = -1;
_has_bits_.Clear();
_internal_metadata_.Clear<std::string>();
}
const char* TrainerSpec::_InternalParse(
const char* ptr,
::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) {
#define CHK_(x) \
if (PROTOBUF_PREDICT_FALSE(!(x))) \
goto failure
while (!ctx->Done(&ptr)) {
::PROTOBUF_NAMESPACE_ID::uint32 tag;
ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag);
CHK_(ptr);
switch (tag >> 3) {
// repeated string input = 1;
case 1:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) {
ptr -= 1;
do {
ptr += 1;
auto str = _internal_add_input();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
if (!ctx->DataAvailable(ptr)) {
break;
}
} while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr));
} else {
goto handle_unusual;
}
continue;
// optional string model_prefix = 2;
case 2:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) {
auto str = _internal_mutable_model_prefix();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional .sentencepiece.TrainerSpec.ModelType model_type = 3 [default =
// UNIGRAM];
case 3:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) {
::PROTOBUF_NAMESPACE_ID::uint64 val =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
if (PROTOBUF_PREDICT_TRUE(
::sentencepiece::TrainerSpec_ModelType_IsValid(val))) {
_internal_set_model_type(
static_cast<::sentencepiece::TrainerSpec_ModelType>(val));
} else {
::PROTOBUF_NAMESPACE_ID::internal::WriteVarint(
3, val, mutable_unknown_fields());
}
} else {
goto handle_unusual;
}
continue;
// optional int32 vocab_size = 4 [default = 8000];
case 4:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) {
_Internal::set_has_vocab_size(&_has_bits_);
vocab_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// repeated string accept_language = 5;
case 5:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 42)) {
ptr -= 1;
do {
ptr += 1;
auto str = _internal_add_accept_language();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
if (!ctx->DataAvailable(ptr)) {
break;
}
} while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<42>(ptr));
} else {
goto handle_unusual;
}
continue;
// optional int32 self_test_sample_size = 6 [default = 0];
case 6:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 48)) {
_Internal::set_has_self_test_sample_size(&_has_bits_);
self_test_sample_size_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional string input_format = 7;
case 7:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 58)) {
auto str = _internal_mutable_input_format();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional float character_coverage = 10 [default = 0.9995];
case 10:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 85)) {
_Internal::set_has_character_coverage(&_has_bits_);
character_coverage_ =
::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<float>(ptr);
ptr += sizeof(float);
} else {
goto handle_unusual;
}
continue;
// optional uint64 input_sentence_size = 11 [default = 0];
case 11:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 88)) {
_Internal::set_has_input_sentence_size(&_has_bits_);
input_sentence_size_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional int32 mining_sentence_size = 12 [deprecated = true];
case 12:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 96)) {
_Internal::set_has_mining_sentence_size(&_has_bits_);
mining_sentence_size_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional int32 training_sentence_size = 13 [deprecated = true];
case 13:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 104)) {
_Internal::set_has_training_sentence_size(&_has_bits_);
training_sentence_size_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional int32 seed_sentencepiece_size = 14 [default = 1000000];
case 14:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 112)) {
_Internal::set_has_seed_sentencepiece_size(&_has_bits_);
seed_sentencepiece_size_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional float shrinking_factor = 15 [default = 0.75];
case 15:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 125)) {
_Internal::set_has_shrinking_factor(&_has_bits_);
shrinking_factor_ =
::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<float>(ptr);
ptr += sizeof(float);
} else {
goto handle_unusual;
}
continue;
// optional int32 num_threads = 16 [default = 16];
case 16:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 128)) {
_Internal::set_has_num_threads(&_has_bits_);
num_threads_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional int32 num_sub_iterations = 17 [default = 2];
case 17:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 136)) {
_Internal::set_has_num_sub_iterations(&_has_bits_);
num_sub_iterations_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional int32 max_sentence_length = 18 [default = 4192];
case 18:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 144)) {
_Internal::set_has_max_sentence_length(&_has_bits_);
max_sentence_length_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool shuffle_input_sentence = 19 [default = true];
case 19:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 152)) {
_Internal::set_has_shuffle_input_sentence(&_has_bits_);
shuffle_input_sentence_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional int32 max_sentencepiece_length = 20 [default = 16];
case 20:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 160)) {
_Internal::set_has_max_sentencepiece_length(&_has_bits_);
max_sentencepiece_length_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool split_by_unicode_script = 21 [default = true];
case 21:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 168)) {
_Internal::set_has_split_by_unicode_script(&_has_bits_);
split_by_unicode_script_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool split_by_whitespace = 22 [default = true];
case 22:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 176)) {
_Internal::set_has_split_by_whitespace(&_has_bits_);
split_by_whitespace_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool split_by_number = 23 [default = true];
case 23:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 184)) {
_Internal::set_has_split_by_number(&_has_bits_);
split_by_number_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool treat_whitespace_as_suffix = 24 [default = false];
case 24:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 192)) {
_Internal::set_has_treat_whitespace_as_suffix(&_has_bits_);
treat_whitespace_as_suffix_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool split_digits = 25 [default = false];
case 25:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 200)) {
_Internal::set_has_split_digits(&_has_bits_);
split_digits_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool allow_whitespace_only_pieces = 26 [default = false];
case 26:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 208)) {
_Internal::set_has_allow_whitespace_only_pieces(&_has_bits_);
allow_whitespace_only_pieces_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// repeated string control_symbols = 30;
case 30:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 242)) {
ptr -= 2;
do {
ptr += 2;
auto str = _internal_add_control_symbols();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
if (!ctx->DataAvailable(ptr)) {
break;
}
} while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<242>(ptr));
} else {
goto handle_unusual;
}
continue;
// repeated string user_defined_symbols = 31;
case 31:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 250)) {
ptr -= 2;
do {
ptr += 2;
auto str = _internal_add_user_defined_symbols();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
if (!ctx->DataAvailable(ptr)) {
break;
}
} while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<250>(ptr));
} else {
goto handle_unusual;
}
continue;
// optional bool vocabulary_output_piece_score = 32 [default = true];
case 32:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 0)) {
_Internal::set_has_vocabulary_output_piece_score(&_has_bits_);
vocabulary_output_piece_score_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool hard_vocab_limit = 33 [default = true];
case 33:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 8)) {
_Internal::set_has_hard_vocab_limit(&_has_bits_);
hard_vocab_limit_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool use_all_vocab = 34 [default = false];
case 34:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) {
_Internal::set_has_use_all_vocab(&_has_bits_);
use_all_vocab_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool byte_fallback = 35 [default = false];
case 35:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) {
_Internal::set_has_byte_fallback(&_has_bits_);
byte_fallback_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional string required_chars = 36;
case 36:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 34)) {
auto str = _internal_mutable_required_chars();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional int32 unk_id = 40 [default = 0];
case 40:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 64)) {
_Internal::set_has_unk_id(&_has_bits_);
unk_id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional int32 bos_id = 41 [default = 1];
case 41:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 72)) {
_Internal::set_has_bos_id(&_has_bits_);
bos_id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional int32 eos_id = 42 [default = 2];
case 42:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 80)) {
_Internal::set_has_eos_id(&_has_bits_);
eos_id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional int32 pad_id = 43 [default = -1];
case 43:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 88)) {
_Internal::set_has_pad_id(&_has_bits_);
pad_id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional string unk_surface = 44 [default = " \342\201\207 "];
case 44:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 98)) {
auto str = _internal_mutable_unk_surface();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional string unk_piece = 45 [default = "<unk>"];
case 45:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 106)) {
auto str = _internal_mutable_unk_piece();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional string bos_piece = 46 [default = "<s>"];
case 46:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 114)) {
auto str = _internal_mutable_bos_piece();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional string eos_piece = 47 [default = "</s>"];
case 47:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 122)) {
auto str = _internal_mutable_eos_piece();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional string pad_piece = 48 [default = "<pad>"];
case 48:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 130)) {
auto str = _internal_mutable_pad_piece();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool train_extremely_large_corpus = 49 [default = false];
case 49:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 136)) {
_Internal::set_has_train_extremely_large_corpus(&_has_bits_);
train_extremely_large_corpus_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool enable_differential_privacy = 50 [default = false];
case 50:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 144)) {
_Internal::set_has_enable_differential_privacy(&_has_bits_);
enable_differential_privacy_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional float differential_privacy_noise_level = 51 [default = 0];
case 51:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 157)) {
_Internal::set_has_differential_privacy_noise_level(&_has_bits_);
differential_privacy_noise_level_ =
::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<float>(ptr);
ptr += sizeof(float);
} else {
goto handle_unusual;
}
continue;
// optional uint64 differential_privacy_clipping_threshold = 52 [default =
// 0];
case 52:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 160)) {
_Internal::set_has_differential_privacy_clipping_threshold(
&_has_bits_);
differential_privacy_clipping_threshold_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional string pretokenization_delimiter = 53 [default = ""];
case 53:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 170)) {
auto str = _internal_mutable_pretokenization_delimiter();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
ctx->SetLastTag(tag);
goto success;
}
if ((1600u <= tag)) {
ptr = _extensions_.ParseField(tag, ptr, internal_default_instance(),
&_internal_metadata_, ctx);
CHK_(ptr != nullptr);
continue;
}
ptr = UnknownFieldParse(
tag, _internal_metadata_.mutable_unknown_fields<std::string>(), ptr,
ctx);
CHK_(ptr != nullptr);
continue;
}
} // switch
} // while
success:
return ptr;
failure:
ptr = nullptr;
goto success;
#undef CHK_
}
::PROTOBUF_NAMESPACE_ID::uint8* TrainerSpec::_InternalSerialize(
::PROTOBUF_NAMESPACE_ID::uint8* target,
::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const {
// @@protoc_insertion_point(serialize_to_array_start:sentencepiece.TrainerSpec)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void)cached_has_bits;
// repeated string input = 1;
for (int i = 0, n = this->_internal_input_size(); i < n; i++) {
const auto& s = this->_internal_input(i);
target = stream->WriteString(1, s, target);
}
cached_has_bits = _has_bits_[0];
// optional string model_prefix = 2;
if (cached_has_bits & 0x00000001u) {
target = stream->WriteStringMaybeAliased(2, this->_internal_model_prefix(),
target);
}
// optional .sentencepiece.TrainerSpec.ModelType model_type = 3 [default =
// UNIGRAM];
if (cached_has_bits & 0x00800000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
3, this->_internal_model_type(), target);
}
// optional int32 vocab_size = 4 [default = 8000];
if (cached_has_bits & 0x01000000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(
4, this->_internal_vocab_size(), target);
}
// repeated string accept_language = 5;
for (int i = 0, n = this->_internal_accept_language_size(); i < n; i++) {
const auto& s = this->_internal_accept_language(i);
target = stream->WriteString(5, s, target);
}
// optional int32 self_test_sample_size = 6 [default = 0];
if (cached_has_bits & 0x00000200u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(
6, this->_internal_self_test_sample_size(), target);
}
// optional string input_format = 7;
if (cached_has_bits & 0x00000002u) {
target = stream->WriteStringMaybeAliased(7, this->_internal_input_format(),
target);
}
// optional float character_coverage = 10 [default = 0.9995];
if (cached_has_bits & 0x02000000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteFloatToArray(
10, this->_internal_character_coverage(), target);
}
// optional uint64 input_sentence_size = 11 [default = 0];
if (cached_has_bits & 0x00000800u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteUInt64ToArray(
11, this->_internal_input_sentence_size(), target);
}
// optional int32 mining_sentence_size = 12 [deprecated = true];
if (cached_has_bits & 0x00000400u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(
12, this->_internal_mining_sentence_size(), target);
}
// optional int32 training_sentence_size = 13 [deprecated = true];
if (cached_has_bits & 0x00001000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(
13, this->_internal_training_sentence_size(), target);
}
// optional int32 seed_sentencepiece_size = 14 [default = 1000000];
if (cached_has_bits & 0x04000000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(
14, this->_internal_seed_sentencepiece_size(), target);
}
// optional float shrinking_factor = 15 [default = 0.75];
if (cached_has_bits & 0x08000000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteFloatToArray(
15, this->_internal_shrinking_factor(), target);
}
// optional int32 num_threads = 16 [default = 16];
if (cached_has_bits & 0x10000000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(
16, this->_internal_num_threads(), target);
}
// optional int32 num_sub_iterations = 17 [default = 2];
if (cached_has_bits & 0x20000000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(
17, this->_internal_num_sub_iterations(), target);
}
// optional int32 max_sentence_length = 18 [default = 4192];
if (cached_has_bits & 0x40000000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(
18, this->_internal_max_sentence_length(), target);
}
cached_has_bits = _has_bits_[1];
// optional bool shuffle_input_sentence = 19 [default = true];
if (cached_has_bits & 0x00000001u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
19, this->_internal_shuffle_input_sentence(), target);
}
cached_has_bits = _has_bits_[0];
// optional int32 max_sentencepiece_length = 20 [default = 16];
if (cached_has_bits & 0x80000000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(
20, this->_internal_max_sentencepiece_length(), target);
}
cached_has_bits = _has_bits_[1];
// optional bool split_by_unicode_script = 21 [default = true];
if (cached_has_bits & 0x00000002u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
21, this->_internal_split_by_unicode_script(), target);
}
// optional bool split_by_whitespace = 22 [default = true];
if (cached_has_bits & 0x00000008u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
22, this->_internal_split_by_whitespace(), target);
}
// optional bool split_by_number = 23 [default = true];
if (cached_has_bits & 0x00000004u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
23, this->_internal_split_by_number(), target);
}
cached_has_bits = _has_bits_[0];
// optional bool treat_whitespace_as_suffix = 24 [default = false];
if (cached_has_bits & 0x00004000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
24, this->_internal_treat_whitespace_as_suffix(), target);
}
// optional bool split_digits = 25 [default = false];
if (cached_has_bits & 0x00010000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
25, this->_internal_split_digits(), target);
}
// optional bool allow_whitespace_only_pieces = 26 [default = false];
if (cached_has_bits & 0x00008000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
26, this->_internal_allow_whitespace_only_pieces(), target);
}
// repeated string control_symbols = 30;
for (int i = 0, n = this->_internal_control_symbols_size(); i < n; i++) {
const auto& s = this->_internal_control_symbols(i);
target = stream->WriteString(30, s, target);
}
// repeated string user_defined_symbols = 31;
for (int i = 0, n = this->_internal_user_defined_symbols_size(); i < n; i++) {
const auto& s = this->_internal_user_defined_symbols(i);
target = stream->WriteString(31, s, target);
}
cached_has_bits = _has_bits_[1];
// optional bool vocabulary_output_piece_score = 32 [default = true];
if (cached_has_bits & 0x00000010u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
32, this->_internal_vocabulary_output_piece_score(), target);
}
// optional bool hard_vocab_limit = 33 [default = true];
if (cached_has_bits & 0x00000020u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
33, this->_internal_hard_vocab_limit(), target);
}
cached_has_bits = _has_bits_[0];
// optional bool use_all_vocab = 34 [default = false];
if (cached_has_bits & 0x00040000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
34, this->_internal_use_all_vocab(), target);
}
// optional bool byte_fallback = 35 [default = false];
if (cached_has_bits & 0x00020000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
35, this->_internal_byte_fallback(), target);
}
// optional string required_chars = 36;
if (cached_has_bits & 0x00000004u) {
target = stream->WriteStringMaybeAliased(
36, this->_internal_required_chars(), target);
}
// optional int32 unk_id = 40 [default = 0];
if (cached_has_bits & 0x00100000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(
40, this->_internal_unk_id(), target);
}
cached_has_bits = _has_bits_[1];
// optional int32 bos_id = 41 [default = 1];
if (cached_has_bits & 0x00000040u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(
41, this->_internal_bos_id(), target);
}
// optional int32 eos_id = 42 [default = 2];
if (cached_has_bits & 0x00000080u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(
42, this->_internal_eos_id(), target);
}
// optional int32 pad_id = 43 [default = -1];
if (cached_has_bits & 0x00000100u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(
43, this->_internal_pad_id(), target);
}
cached_has_bits = _has_bits_[0];
// optional string unk_surface = 44 [default = " \342\201\207 "];
if (cached_has_bits & 0x00000008u) {
target = stream->WriteStringMaybeAliased(44, this->_internal_unk_surface(),
target);
}
// optional string unk_piece = 45 [default = "<unk>"];
if (cached_has_bits & 0x00000010u) {
target = stream->WriteStringMaybeAliased(45, this->_internal_unk_piece(),
target);
}
// optional string bos_piece = 46 [default = "<s>"];
if (cached_has_bits & 0x00000020u) {
target = stream->WriteStringMaybeAliased(46, this->_internal_bos_piece(),
target);
}
// optional string eos_piece = 47 [default = "</s>"];
if (cached_has_bits & 0x00000040u) {
target = stream->WriteStringMaybeAliased(47, this->_internal_eos_piece(),
target);
}
// optional string pad_piece = 48 [default = "<pad>"];
if (cached_has_bits & 0x00000080u) {
target = stream->WriteStringMaybeAliased(48, this->_internal_pad_piece(),
target);
}
// optional bool train_extremely_large_corpus = 49 [default = false];
if (cached_has_bits & 0x00080000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
49, this->_internal_train_extremely_large_corpus(), target);
}
// optional bool enable_differential_privacy = 50 [default = false];
if (cached_has_bits & 0x00002000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
50, this->_internal_enable_differential_privacy(), target);
}
// optional float differential_privacy_noise_level = 51 [default = 0];
if (cached_has_bits & 0x00200000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteFloatToArray(
51, this->_internal_differential_privacy_noise_level(), target);
}
// optional uint64 differential_privacy_clipping_threshold = 52 [default = 0];
if (cached_has_bits & 0x00400000u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteUInt64ToArray(
52, this->_internal_differential_privacy_clipping_threshold(),
target);
}
// optional string pretokenization_delimiter = 53 [default = ""];
if (cached_has_bits & 0x00000100u) {
target = stream->WriteStringMaybeAliased(
53, this->_internal_pretokenization_delimiter(), target);
}
// Extension range [200, 536870912)
target = _extensions_._InternalSerialize(200, 536870912, target, stream);
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
target = stream->WriteRaw(
_internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.data(),
static_cast<int>(
_internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.size()),
target);
}
// @@protoc_insertion_point(serialize_to_array_end:sentencepiece.TrainerSpec)
return target;
}
size_t TrainerSpec::ByteSizeLong() const {
// @@protoc_insertion_point(message_byte_size_start:sentencepiece.TrainerSpec)
size_t total_size = 0;
total_size += _extensions_.ByteSize();
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void)cached_has_bits;
// repeated string input = 1;
total_size +=
1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(input_.size());
for (int i = 0, n = input_.size(); i < n; i++) {
total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
input_.Get(i));
}
// repeated string accept_language = 5;
total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(
accept_language_.size());
for (int i = 0, n = accept_language_.size(); i < n; i++) {
total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
accept_language_.Get(i));
}
// repeated string control_symbols = 30;
total_size += 2 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(
control_symbols_.size());
for (int i = 0, n = control_symbols_.size(); i < n; i++) {
total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
control_symbols_.Get(i));
}
// repeated string user_defined_symbols = 31;
total_size += 2 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(
user_defined_symbols_.size());
for (int i = 0, n = user_defined_symbols_.size(); i < n; i++) {
total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
user_defined_symbols_.Get(i));
}
cached_has_bits = _has_bits_[0];
if (cached_has_bits & 0x000000ffu) {
// optional string model_prefix = 2;
if (cached_has_bits & 0x00000001u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->_internal_model_prefix());
}
// optional string input_format = 7;
if (cached_has_bits & 0x00000002u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->_internal_input_format());
}
// optional string required_chars = 36;
if (cached_has_bits & 0x00000004u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->_internal_required_chars());
}
// optional string unk_surface = 44 [default = " \342\201\207 "];
if (cached_has_bits & 0x00000008u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->_internal_unk_surface());
}
// optional string unk_piece = 45 [default = "<unk>"];
if (cached_has_bits & 0x00000010u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->_internal_unk_piece());
}
// optional string bos_piece = 46 [default = "<s>"];
if (cached_has_bits & 0x00000020u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->_internal_bos_piece());
}
// optional string eos_piece = 47 [default = "</s>"];
if (cached_has_bits & 0x00000040u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->_internal_eos_piece());
}
// optional string pad_piece = 48 [default = "<pad>"];
if (cached_has_bits & 0x00000080u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->_internal_pad_piece());
}
}
if (cached_has_bits & 0x0000ff00u) {
// optional string pretokenization_delimiter = 53 [default = ""];
if (cached_has_bits & 0x00000100u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->_internal_pretokenization_delimiter());
}
// optional int32 self_test_sample_size = 6 [default = 0];
if (cached_has_bits & 0x00000200u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->_internal_self_test_sample_size());
}
// optional int32 mining_sentence_size = 12 [deprecated = true];
if (cached_has_bits & 0x00000400u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->_internal_mining_sentence_size());
}
// optional uint64 input_sentence_size = 11 [default = 0];
if (cached_has_bits & 0x00000800u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::UInt64Size(
this->_internal_input_sentence_size());
}
// optional int32 training_sentence_size = 13 [deprecated = true];
if (cached_has_bits & 0x00001000u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->_internal_training_sentence_size());
}
// optional bool enable_differential_privacy = 50 [default = false];
if (cached_has_bits & 0x00002000u) {
total_size += 2 + 1;
}
// optional bool treat_whitespace_as_suffix = 24 [default = false];
if (cached_has_bits & 0x00004000u) {
total_size += 2 + 1;
}
// optional bool allow_whitespace_only_pieces = 26 [default = false];
if (cached_has_bits & 0x00008000u) {
total_size += 2 + 1;
}
}
if (cached_has_bits & 0x00ff0000u) {
// optional bool split_digits = 25 [default = false];
if (cached_has_bits & 0x00010000u) {
total_size += 2 + 1;
}
// optional bool byte_fallback = 35 [default = false];
if (cached_has_bits & 0x00020000u) {
total_size += 2 + 1;
}
// optional bool use_all_vocab = 34 [default = false];
if (cached_has_bits & 0x00040000u) {
total_size += 2 + 1;
}
// optional bool train_extremely_large_corpus = 49 [default = false];
if (cached_has_bits & 0x00080000u) {
total_size += 2 + 1;
}
// optional int32 unk_id = 40 [default = 0];
if (cached_has_bits & 0x00100000u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->_internal_unk_id());
}
// optional float differential_privacy_noise_level = 51 [default = 0];
if (cached_has_bits & 0x00200000u) {
total_size += 2 + 4;
}
// optional uint64 differential_privacy_clipping_threshold = 52 [default =
// 0];
if (cached_has_bits & 0x00400000u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::UInt64Size(
this->_internal_differential_privacy_clipping_threshold());
}
// optional .sentencepiece.TrainerSpec.ModelType model_type = 3 [default =
// UNIGRAM];
if (cached_has_bits & 0x00800000u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::EnumSize(
this->_internal_model_type());
}
}
if (cached_has_bits & 0xff000000u) {
// optional int32 vocab_size = 4 [default = 8000];
if (cached_has_bits & 0x01000000u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->_internal_vocab_size());
}
// optional float character_coverage = 10 [default = 0.9995];
if (cached_has_bits & 0x02000000u) {
total_size += 1 + 4;
}
// optional int32 seed_sentencepiece_size = 14 [default = 1000000];
if (cached_has_bits & 0x04000000u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->_internal_seed_sentencepiece_size());
}
// optional float shrinking_factor = 15 [default = 0.75];
if (cached_has_bits & 0x08000000u) {
total_size += 1 + 4;
}
// optional int32 num_threads = 16 [default = 16];
if (cached_has_bits & 0x10000000u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->_internal_num_threads());
}
// optional int32 num_sub_iterations = 17 [default = 2];
if (cached_has_bits & 0x20000000u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->_internal_num_sub_iterations());
}
// optional int32 max_sentence_length = 18 [default = 4192];
if (cached_has_bits & 0x40000000u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->_internal_max_sentence_length());
}
// optional int32 max_sentencepiece_length = 20 [default = 16];
if (cached_has_bits & 0x80000000u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->_internal_max_sentencepiece_length());
}
}
cached_has_bits = _has_bits_[1];
if (cached_has_bits & 0x000000ffu) {
// optional bool shuffle_input_sentence = 19 [default = true];
if (cached_has_bits & 0x00000001u) {
total_size += 2 + 1;
}
// optional bool split_by_unicode_script = 21 [default = true];
if (cached_has_bits & 0x00000002u) {
total_size += 2 + 1;
}
// optional bool split_by_number = 23 [default = true];
if (cached_has_bits & 0x00000004u) {
total_size += 2 + 1;
}
// optional bool split_by_whitespace = 22 [default = true];
if (cached_has_bits & 0x00000008u) {
total_size += 2 + 1;
}
// optional bool vocabulary_output_piece_score = 32 [default = true];
if (cached_has_bits & 0x00000010u) {
total_size += 2 + 1;
}
// optional bool hard_vocab_limit = 33 [default = true];
if (cached_has_bits & 0x00000020u) {
total_size += 2 + 1;
}
// optional int32 bos_id = 41 [default = 1];
if (cached_has_bits & 0x00000040u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->_internal_bos_id());
}
// optional int32 eos_id = 42 [default = 2];
if (cached_has_bits & 0x00000080u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->_internal_eos_id());
}
}
// optional int32 pad_id = 43 [default = -1];
if (cached_has_bits & 0x00000100u) {
total_size +=
2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->_internal_pad_id());
}
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
total_size += _internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.size();
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
}
void TrainerSpec::CheckTypeAndMergeFrom(
const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) {
MergeFrom(
*::PROTOBUF_NAMESPACE_ID::internal::DownCast<const TrainerSpec*>(&from));
}
void TrainerSpec::MergeFrom(const TrainerSpec& from) {
// @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.TrainerSpec)
GOOGLE_DCHECK_NE(&from, this);
_extensions_.MergeFrom(from._extensions_);
_internal_metadata_.MergeFrom<std::string>(from._internal_metadata_);
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void)cached_has_bits;
input_.MergeFrom(from.input_);
accept_language_.MergeFrom(from.accept_language_);
control_symbols_.MergeFrom(from.control_symbols_);
user_defined_symbols_.MergeFrom(from.user_defined_symbols_);
cached_has_bits = from._has_bits_[0];
if (cached_has_bits & 0x000000ffu) {
if (cached_has_bits & 0x00000001u) {
_internal_set_model_prefix(from._internal_model_prefix());
}
if (cached_has_bits & 0x00000002u) {
_internal_set_input_format(from._internal_input_format());
}
if (cached_has_bits & 0x00000004u) {
_internal_set_required_chars(from._internal_required_chars());
}
if (cached_has_bits & 0x00000008u) {
_internal_set_unk_surface(from._internal_unk_surface());
}
if (cached_has_bits & 0x00000010u) {
_internal_set_unk_piece(from._internal_unk_piece());
}
if (cached_has_bits & 0x00000020u) {
_internal_set_bos_piece(from._internal_bos_piece());
}
if (cached_has_bits & 0x00000040u) {
_internal_set_eos_piece(from._internal_eos_piece());
}
if (cached_has_bits & 0x00000080u) {
_internal_set_pad_piece(from._internal_pad_piece());
}
}
if (cached_has_bits & 0x0000ff00u) {
if (cached_has_bits & 0x00000100u) {
_internal_set_pretokenization_delimiter(
from._internal_pretokenization_delimiter());
}
if (cached_has_bits & 0x00000200u) {
self_test_sample_size_ = from.self_test_sample_size_;
}
if (cached_has_bits & 0x00000400u) {
mining_sentence_size_ = from.mining_sentence_size_;
}
if (cached_has_bits & 0x00000800u) {
input_sentence_size_ = from.input_sentence_size_;
}
if (cached_has_bits & 0x00001000u) {
training_sentence_size_ = from.training_sentence_size_;
}
if (cached_has_bits & 0x00002000u) {
enable_differential_privacy_ = from.enable_differential_privacy_;
}
if (cached_has_bits & 0x00004000u) {
treat_whitespace_as_suffix_ = from.treat_whitespace_as_suffix_;
}
if (cached_has_bits & 0x00008000u) {
allow_whitespace_only_pieces_ = from.allow_whitespace_only_pieces_;
}
_has_bits_[0] |= cached_has_bits;
}
if (cached_has_bits & 0x00ff0000u) {
if (cached_has_bits & 0x00010000u) {
split_digits_ = from.split_digits_;
}
if (cached_has_bits & 0x00020000u) {
byte_fallback_ = from.byte_fallback_;
}
if (cached_has_bits & 0x00040000u) {
use_all_vocab_ = from.use_all_vocab_;
}
if (cached_has_bits & 0x00080000u) {
train_extremely_large_corpus_ = from.train_extremely_large_corpus_;
}
if (cached_has_bits & 0x00100000u) {
unk_id_ = from.unk_id_;
}
if (cached_has_bits & 0x00200000u) {
differential_privacy_noise_level_ =
from.differential_privacy_noise_level_;
}
if (cached_has_bits & 0x00400000u) {
differential_privacy_clipping_threshold_ =
from.differential_privacy_clipping_threshold_;
}
if (cached_has_bits & 0x00800000u) {
model_type_ = from.model_type_;
}
_has_bits_[0] |= cached_has_bits;
}
if (cached_has_bits & 0xff000000u) {
if (cached_has_bits & 0x01000000u) {
vocab_size_ = from.vocab_size_;
}
if (cached_has_bits & 0x02000000u) {
character_coverage_ = from.character_coverage_;
}
if (cached_has_bits & 0x04000000u) {
seed_sentencepiece_size_ = from.seed_sentencepiece_size_;
}
if (cached_has_bits & 0x08000000u) {
shrinking_factor_ = from.shrinking_factor_;
}
if (cached_has_bits & 0x10000000u) {
num_threads_ = from.num_threads_;
}
if (cached_has_bits & 0x20000000u) {
num_sub_iterations_ = from.num_sub_iterations_;
}
if (cached_has_bits & 0x40000000u) {
max_sentence_length_ = from.max_sentence_length_;
}
if (cached_has_bits & 0x80000000u) {
max_sentencepiece_length_ = from.max_sentencepiece_length_;
}
_has_bits_[0] |= cached_has_bits;
}
cached_has_bits = from._has_bits_[1];
if (cached_has_bits & 0x000000ffu) {
if (cached_has_bits & 0x00000001u) {
shuffle_input_sentence_ = from.shuffle_input_sentence_;
}
if (cached_has_bits & 0x00000002u) {
split_by_unicode_script_ = from.split_by_unicode_script_;
}
if (cached_has_bits & 0x00000004u) {
split_by_number_ = from.split_by_number_;
}
if (cached_has_bits & 0x00000008u) {
split_by_whitespace_ = from.split_by_whitespace_;
}
if (cached_has_bits & 0x00000010u) {
vocabulary_output_piece_score_ = from.vocabulary_output_piece_score_;
}
if (cached_has_bits & 0x00000020u) {
hard_vocab_limit_ = from.hard_vocab_limit_;
}
if (cached_has_bits & 0x00000040u) {
bos_id_ = from.bos_id_;
}
if (cached_has_bits & 0x00000080u) {
eos_id_ = from.eos_id_;
}
_has_bits_[1] |= cached_has_bits;
}
if (cached_has_bits & 0x00000100u) {
_internal_set_pad_id(from._internal_pad_id());
}
}
void TrainerSpec::CopyFrom(const TrainerSpec& from) {
// @@protoc_insertion_point(class_specific_copy_from_start:sentencepiece.TrainerSpec)
if (&from == this) {
return;
}
Clear();
MergeFrom(from);
}
bool TrainerSpec::IsInitialized() const {
if (!_extensions_.IsInitialized()) {
return false;
}
return true;
}
void TrainerSpec::InternalSwap(TrainerSpec* other) {
using std::swap;
_extensions_.Swap(&other->_extensions_);
_internal_metadata_.Swap<std::string>(&other->_internal_metadata_);
swap(_has_bits_[0], other->_has_bits_[0]);
swap(_has_bits_[1], other->_has_bits_[1]);
input_.InternalSwap(&other->input_);
accept_language_.InternalSwap(&other->accept_language_);
control_symbols_.InternalSwap(&other->control_symbols_);
user_defined_symbols_.InternalSwap(&other->user_defined_symbols_);
model_prefix_.Swap(
&other->model_prefix_,
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
GetArena());
input_format_.Swap(
&other->input_format_,
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
GetArena());
required_chars_.Swap(
&other->required_chars_,
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
GetArena());
unk_surface_.Swap(&other->unk_surface_, nullptr, GetArena());
unk_piece_.Swap(&other->unk_piece_, nullptr, GetArena());
bos_piece_.Swap(&other->bos_piece_, nullptr, GetArena());
eos_piece_.Swap(&other->eos_piece_, nullptr, GetArena());
pad_piece_.Swap(&other->pad_piece_, nullptr, GetArena());
pretokenization_delimiter_.Swap(
&other->pretokenization_delimiter_,
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
GetArena());
::PROTOBUF_NAMESPACE_ID::internal::memswap<
PROTOBUF_FIELD_OFFSET(TrainerSpec,
differential_privacy_clipping_threshold_) +
sizeof(TrainerSpec::differential_privacy_clipping_threshold_) -
PROTOBUF_FIELD_OFFSET(TrainerSpec, self_test_sample_size_)>(
reinterpret_cast<char*>(&self_test_sample_size_),
reinterpret_cast<char*>(&other->self_test_sample_size_));
swap(model_type_, other->model_type_);
swap(vocab_size_, other->vocab_size_);
swap(character_coverage_, other->character_coverage_);
swap(seed_sentencepiece_size_, other->seed_sentencepiece_size_);
swap(shrinking_factor_, other->shrinking_factor_);
swap(num_threads_, other->num_threads_);
swap(num_sub_iterations_, other->num_sub_iterations_);
swap(max_sentence_length_, other->max_sentence_length_);
swap(max_sentencepiece_length_, other->max_sentencepiece_length_);
swap(shuffle_input_sentence_, other->shuffle_input_sentence_);
swap(split_by_unicode_script_, other->split_by_unicode_script_);
swap(split_by_number_, other->split_by_number_);
swap(split_by_whitespace_, other->split_by_whitespace_);
swap(vocabulary_output_piece_score_, other->vocabulary_output_piece_score_);
swap(hard_vocab_limit_, other->hard_vocab_limit_);
swap(bos_id_, other->bos_id_);
swap(eos_id_, other->eos_id_);
swap(pad_id_, other->pad_id_);
}
std::string TrainerSpec::GetTypeName() const {
return "sentencepiece.TrainerSpec";
}
// ===================================================================
class NormalizerSpec::_Internal {
public:
using HasBits = decltype(std::declval<NormalizerSpec>()._has_bits_);
static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; }
static void set_has_precompiled_charsmap(HasBits* has_bits) {
(*has_bits)[0] |= 2u;
}
static void set_has_add_dummy_prefix(HasBits* has_bits) {
(*has_bits)[0] |= 8u;
}
static void set_has_remove_extra_whitespaces(HasBits* has_bits) {
(*has_bits)[0] |= 16u;
}
static void set_has_escape_whitespaces(HasBits* has_bits) {
(*has_bits)[0] |= 32u;
}
static void set_has_normalization_rule_tsv(HasBits* has_bits) {
(*has_bits)[0] |= 4u;
}
};
NormalizerSpec::NormalizerSpec(::PROTOBUF_NAMESPACE_ID::Arena* arena)
: ::PROTOBUF_NAMESPACE_ID::MessageLite(arena), _extensions_(arena) {
SharedCtor();
RegisterArenaDtor(arena);
// @@protoc_insertion_point(arena_constructor:sentencepiece.NormalizerSpec)
}
NormalizerSpec::NormalizerSpec(const NormalizerSpec& from)
: ::PROTOBUF_NAMESPACE_ID::MessageLite(), _has_bits_(from._has_bits_) {
_internal_metadata_.MergeFrom<std::string>(from._internal_metadata_);
_extensions_.MergeFrom(from._extensions_);
name_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (from._internal_has_name()) {
name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{},
from._internal_name(), GetArena());
}
precompiled_charsmap_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (from._internal_has_precompiled_charsmap()) {
precompiled_charsmap_.Set(
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{},
from._internal_precompiled_charsmap(), GetArena());
}
normalization_rule_tsv_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (from._internal_has_normalization_rule_tsv()) {
normalization_rule_tsv_.Set(
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{},
from._internal_normalization_rule_tsv(), GetArena());
}
::memcpy(&add_dummy_prefix_, &from.add_dummy_prefix_,
static_cast<size_t>(reinterpret_cast<char*>(&escape_whitespaces_) -
reinterpret_cast<char*>(&add_dummy_prefix_)) +
sizeof(escape_whitespaces_));
// @@protoc_insertion_point(copy_constructor:sentencepiece.NormalizerSpec)
}
void NormalizerSpec::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(
&scc_info_NormalizerSpec_sentencepiece_5fmodel_2eproto.base);
name_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
precompiled_charsmap_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
normalization_rule_tsv_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
add_dummy_prefix_ = true;
remove_extra_whitespaces_ = true;
escape_whitespaces_ = true;
}
NormalizerSpec::~NormalizerSpec() {
// @@protoc_insertion_point(destructor:sentencepiece.NormalizerSpec)
SharedDtor();
_internal_metadata_.Delete<std::string>();
}
void NormalizerSpec::SharedDtor() {
GOOGLE_DCHECK(GetArena() == nullptr);
name_.DestroyNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
precompiled_charsmap_.DestroyNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
normalization_rule_tsv_.DestroyNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
void NormalizerSpec::ArenaDtor(void* object) {
NormalizerSpec* _this = reinterpret_cast<NormalizerSpec*>(object);
(void)_this;
}
void NormalizerSpec::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) {}
void NormalizerSpec::SetCachedSize(int size) const {
_cached_size_.Set(size);
}
const NormalizerSpec& NormalizerSpec::default_instance() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(
&::scc_info_NormalizerSpec_sentencepiece_5fmodel_2eproto.base);
return *internal_default_instance();
}
void NormalizerSpec::Clear() {
// @@protoc_insertion_point(message_clear_start:sentencepiece.NormalizerSpec)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void)cached_has_bits;
_extensions_.Clear();
cached_has_bits = _has_bits_[0];
if (cached_has_bits & 0x0000003fu) {
if (cached_has_bits & 0x00000001u) {
name_.ClearNonDefaultToEmpty();
}
if (cached_has_bits & 0x00000002u) {
precompiled_charsmap_.ClearNonDefaultToEmpty();
}
if (cached_has_bits & 0x00000004u) {
normalization_rule_tsv_.ClearNonDefaultToEmpty();
}
add_dummy_prefix_ = true;
remove_extra_whitespaces_ = true;
escape_whitespaces_ = true;
}
_has_bits_.Clear();
_internal_metadata_.Clear<std::string>();
}
const char* NormalizerSpec::_InternalParse(
const char* ptr,
::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) {
#define CHK_(x) \
if (PROTOBUF_PREDICT_FALSE(!(x))) \
goto failure
_Internal::HasBits has_bits{};
while (!ctx->Done(&ptr)) {
::PROTOBUF_NAMESPACE_ID::uint32 tag;
ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag);
CHK_(ptr);
switch (tag >> 3) {
// optional string name = 1;
case 1:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) {
auto str = _internal_mutable_name();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bytes precompiled_charsmap = 2;
case 2:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) {
auto str = _internal_mutable_precompiled_charsmap();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool add_dummy_prefix = 3 [default = true];
case 3:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) {
_Internal::set_has_add_dummy_prefix(&has_bits);
add_dummy_prefix_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool remove_extra_whitespaces = 4 [default = true];
case 4:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) {
_Internal::set_has_remove_extra_whitespaces(&has_bits);
remove_extra_whitespaces_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional bool escape_whitespaces = 5 [default = true];
case 5:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 40)) {
_Internal::set_has_escape_whitespaces(&has_bits);
escape_whitespaces_ =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional string normalization_rule_tsv = 6;
case 6:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 50)) {
auto str = _internal_mutable_normalization_rule_tsv();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
ctx->SetLastTag(tag);
goto success;
}
if ((1600u <= tag)) {
ptr = _extensions_.ParseField(tag, ptr, internal_default_instance(),
&_internal_metadata_, ctx);
CHK_(ptr != nullptr);
continue;
}
ptr = UnknownFieldParse(
tag, _internal_metadata_.mutable_unknown_fields<std::string>(), ptr,
ctx);
CHK_(ptr != nullptr);
continue;
}
} // switch
} // while
success:
_has_bits_.Or(has_bits);
return ptr;
failure:
ptr = nullptr;
goto success;
#undef CHK_
}
::PROTOBUF_NAMESPACE_ID::uint8* NormalizerSpec::_InternalSerialize(
::PROTOBUF_NAMESPACE_ID::uint8* target,
::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const {
// @@protoc_insertion_point(serialize_to_array_start:sentencepiece.NormalizerSpec)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void)cached_has_bits;
cached_has_bits = _has_bits_[0];
// optional string name = 1;
if (cached_has_bits & 0x00000001u) {
target = stream->WriteStringMaybeAliased(1, this->_internal_name(), target);
}
// optional bytes precompiled_charsmap = 2;
if (cached_has_bits & 0x00000002u) {
target = stream->WriteBytesMaybeAliased(
2, this->_internal_precompiled_charsmap(), target);
}
// optional bool add_dummy_prefix = 3 [default = true];
if (cached_has_bits & 0x00000008u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
3, this->_internal_add_dummy_prefix(), target);
}
// optional bool remove_extra_whitespaces = 4 [default = true];
if (cached_has_bits & 0x00000010u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
4, this->_internal_remove_extra_whitespaces(), target);
}
// optional bool escape_whitespaces = 5 [default = true];
if (cached_has_bits & 0x00000020u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(
5, this->_internal_escape_whitespaces(), target);
}
// optional string normalization_rule_tsv = 6;
if (cached_has_bits & 0x00000004u) {
target = stream->WriteStringMaybeAliased(
6, this->_internal_normalization_rule_tsv(), target);
}
// Extension range [200, 536870912)
target = _extensions_._InternalSerialize(200, 536870912, target, stream);
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
target = stream->WriteRaw(
_internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.data(),
static_cast<int>(
_internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.size()),
target);
}
// @@protoc_insertion_point(serialize_to_array_end:sentencepiece.NormalizerSpec)
return target;
}
size_t NormalizerSpec::ByteSizeLong() const {
// @@protoc_insertion_point(message_byte_size_start:sentencepiece.NormalizerSpec)
size_t total_size = 0;
total_size += _extensions_.ByteSize();
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void)cached_has_bits;
cached_has_bits = _has_bits_[0];
if (cached_has_bits & 0x0000003fu) {
// optional string name = 1;
if (cached_has_bits & 0x00000001u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->_internal_name());
}
// optional bytes precompiled_charsmap = 2;
if (cached_has_bits & 0x00000002u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::BytesSize(
this->_internal_precompiled_charsmap());
}
// optional string normalization_rule_tsv = 6;
if (cached_has_bits & 0x00000004u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->_internal_normalization_rule_tsv());
}
// optional bool add_dummy_prefix = 3 [default = true];
if (cached_has_bits & 0x00000008u) {
total_size += 1 + 1;
}
// optional bool remove_extra_whitespaces = 4 [default = true];
if (cached_has_bits & 0x00000010u) {
total_size += 1 + 1;
}
// optional bool escape_whitespaces = 5 [default = true];
if (cached_has_bits & 0x00000020u) {
total_size += 1 + 1;
}
}
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
total_size += _internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.size();
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
}
void NormalizerSpec::CheckTypeAndMergeFrom(
const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) {
MergeFrom(*::PROTOBUF_NAMESPACE_ID::internal::DownCast<const NormalizerSpec*>(
&from));
}
void NormalizerSpec::MergeFrom(const NormalizerSpec& from) {
// @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.NormalizerSpec)
GOOGLE_DCHECK_NE(&from, this);
_extensions_.MergeFrom(from._extensions_);
_internal_metadata_.MergeFrom<std::string>(from._internal_metadata_);
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void)cached_has_bits;
cached_has_bits = from._has_bits_[0];
if (cached_has_bits & 0x0000003fu) {
if (cached_has_bits & 0x00000001u) {
_internal_set_name(from._internal_name());
}
if (cached_has_bits & 0x00000002u) {
_internal_set_precompiled_charsmap(from._internal_precompiled_charsmap());
}
if (cached_has_bits & 0x00000004u) {
_internal_set_normalization_rule_tsv(
from._internal_normalization_rule_tsv());
}
if (cached_has_bits & 0x00000008u) {
add_dummy_prefix_ = from.add_dummy_prefix_;
}
if (cached_has_bits & 0x00000010u) {
remove_extra_whitespaces_ = from.remove_extra_whitespaces_;
}
if (cached_has_bits & 0x00000020u) {
escape_whitespaces_ = from.escape_whitespaces_;
}
_has_bits_[0] |= cached_has_bits;
}
}
void NormalizerSpec::CopyFrom(const NormalizerSpec& from) {
// @@protoc_insertion_point(class_specific_copy_from_start:sentencepiece.NormalizerSpec)
if (&from == this) {
return;
}
Clear();
MergeFrom(from);
}
bool NormalizerSpec::IsInitialized() const {
if (!_extensions_.IsInitialized()) {
return false;
}
return true;
}
void NormalizerSpec::InternalSwap(NormalizerSpec* other) {
using std::swap;
_extensions_.Swap(&other->_extensions_);
_internal_metadata_.Swap<std::string>(&other->_internal_metadata_);
swap(_has_bits_[0], other->_has_bits_[0]);
name_.Swap(&other->name_,
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
GetArena());
precompiled_charsmap_.Swap(
&other->precompiled_charsmap_,
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
GetArena());
normalization_rule_tsv_.Swap(
&other->normalization_rule_tsv_,
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
GetArena());
swap(add_dummy_prefix_, other->add_dummy_prefix_);
swap(remove_extra_whitespaces_, other->remove_extra_whitespaces_);
swap(escape_whitespaces_, other->escape_whitespaces_);
}
std::string NormalizerSpec::GetTypeName() const {
return "sentencepiece.NormalizerSpec";
}
// ===================================================================
class SelfTestData_Sample::_Internal {
public:
using HasBits = decltype(std::declval<SelfTestData_Sample>()._has_bits_);
static void set_has_input(HasBits* has_bits) { (*has_bits)[0] |= 1u; }
static void set_has_expected(HasBits* has_bits) { (*has_bits)[0] |= 2u; }
};
SelfTestData_Sample::SelfTestData_Sample(::PROTOBUF_NAMESPACE_ID::Arena* arena)
: ::PROTOBUF_NAMESPACE_ID::MessageLite(arena) {
SharedCtor();
RegisterArenaDtor(arena);
// @@protoc_insertion_point(arena_constructor:sentencepiece.SelfTestData.Sample)
}
SelfTestData_Sample::SelfTestData_Sample(const SelfTestData_Sample& from)
: ::PROTOBUF_NAMESPACE_ID::MessageLite(), _has_bits_(from._has_bits_) {
_internal_metadata_.MergeFrom<std::string>(from._internal_metadata_);
input_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (from._internal_has_input()) {
input_.Set(
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{},
from._internal_input(), GetArena());
}
expected_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (from._internal_has_expected()) {
expected_.Set(
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{},
from._internal_expected(), GetArena());
}
// @@protoc_insertion_point(copy_constructor:sentencepiece.SelfTestData.Sample)
}
void SelfTestData_Sample::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(
&scc_info_SelfTestData_Sample_sentencepiece_5fmodel_2eproto.base);
input_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
expected_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
SelfTestData_Sample::~SelfTestData_Sample() {
// @@protoc_insertion_point(destructor:sentencepiece.SelfTestData.Sample)
SharedDtor();
_internal_metadata_.Delete<std::string>();
}
void SelfTestData_Sample::SharedDtor() {
GOOGLE_DCHECK(GetArena() == nullptr);
input_.DestroyNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
expected_.DestroyNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
void SelfTestData_Sample::ArenaDtor(void* object) {
SelfTestData_Sample* _this = reinterpret_cast<SelfTestData_Sample*>(object);
(void)_this;
}
void SelfTestData_Sample::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) {}
void SelfTestData_Sample::SetCachedSize(int size) const {
_cached_size_.Set(size);
}
const SelfTestData_Sample& SelfTestData_Sample::default_instance() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(
&::scc_info_SelfTestData_Sample_sentencepiece_5fmodel_2eproto.base);
return *internal_default_instance();
}
void SelfTestData_Sample::Clear() {
// @@protoc_insertion_point(message_clear_start:sentencepiece.SelfTestData.Sample)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void)cached_has_bits;
cached_has_bits = _has_bits_[0];
if (cached_has_bits & 0x00000003u) {
if (cached_has_bits & 0x00000001u) {
input_.ClearNonDefaultToEmpty();
}
if (cached_has_bits & 0x00000002u) {
expected_.ClearNonDefaultToEmpty();
}
}
_has_bits_.Clear();
_internal_metadata_.Clear<std::string>();
}
const char* SelfTestData_Sample::_InternalParse(
const char* ptr,
::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) {
#define CHK_(x) \
if (PROTOBUF_PREDICT_FALSE(!(x))) \
goto failure
_Internal::HasBits has_bits{};
while (!ctx->Done(&ptr)) {
::PROTOBUF_NAMESPACE_ID::uint32 tag;
ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag);
CHK_(ptr);
switch (tag >> 3) {
// optional string input = 1;
case 1:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) {
auto str = _internal_mutable_input();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional string expected = 2;
case 2:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) {
auto str = _internal_mutable_expected();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
ctx->SetLastTag(tag);
goto success;
}
ptr = UnknownFieldParse(
tag, _internal_metadata_.mutable_unknown_fields<std::string>(), ptr,
ctx);
CHK_(ptr != nullptr);
continue;
}
} // switch
} // while
success:
_has_bits_.Or(has_bits);
return ptr;
failure:
ptr = nullptr;
goto success;
#undef CHK_
}
::PROTOBUF_NAMESPACE_ID::uint8* SelfTestData_Sample::_InternalSerialize(
::PROTOBUF_NAMESPACE_ID::uint8* target,
::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const {
// @@protoc_insertion_point(serialize_to_array_start:sentencepiece.SelfTestData.Sample)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void)cached_has_bits;
cached_has_bits = _has_bits_[0];
// optional string input = 1;
if (cached_has_bits & 0x00000001u) {
target =
stream->WriteStringMaybeAliased(1, this->_internal_input(), target);
}
// optional string expected = 2;
if (cached_has_bits & 0x00000002u) {
target =
stream->WriteStringMaybeAliased(2, this->_internal_expected(), target);
}
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
target = stream->WriteRaw(
_internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.data(),
static_cast<int>(
_internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.size()),
target);
}
// @@protoc_insertion_point(serialize_to_array_end:sentencepiece.SelfTestData.Sample)
return target;
}
size_t SelfTestData_Sample::ByteSizeLong() const {
// @@protoc_insertion_point(message_byte_size_start:sentencepiece.SelfTestData.Sample)
size_t total_size = 0;
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void)cached_has_bits;
cached_has_bits = _has_bits_[0];
if (cached_has_bits & 0x00000003u) {
// optional string input = 1;
if (cached_has_bits & 0x00000001u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->_internal_input());
}
// optional string expected = 2;
if (cached_has_bits & 0x00000002u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->_internal_expected());
}
}
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
total_size += _internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.size();
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
}
void SelfTestData_Sample::CheckTypeAndMergeFrom(
const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) {
MergeFrom(
*::PROTOBUF_NAMESPACE_ID::internal::DownCast<const SelfTestData_Sample*>(
&from));
}
void SelfTestData_Sample::MergeFrom(const SelfTestData_Sample& from) {
// @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.SelfTestData.Sample)
GOOGLE_DCHECK_NE(&from, this);
_internal_metadata_.MergeFrom<std::string>(from._internal_metadata_);
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void)cached_has_bits;
cached_has_bits = from._has_bits_[0];
if (cached_has_bits & 0x00000003u) {
if (cached_has_bits & 0x00000001u) {
_internal_set_input(from._internal_input());
}
if (cached_has_bits & 0x00000002u) {
_internal_set_expected(from._internal_expected());
}
}
}
void SelfTestData_Sample::CopyFrom(const SelfTestData_Sample& from) {
// @@protoc_insertion_point(class_specific_copy_from_start:sentencepiece.SelfTestData.Sample)
if (&from == this) {
return;
}
Clear();
MergeFrom(from);
}
bool SelfTestData_Sample::IsInitialized() const {
return true;
}
void SelfTestData_Sample::InternalSwap(SelfTestData_Sample* other) {
using std::swap;
_internal_metadata_.Swap<std::string>(&other->_internal_metadata_);
swap(_has_bits_[0], other->_has_bits_[0]);
input_.Swap(&other->input_,
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
GetArena());
expected_.Swap(
&other->expected_,
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
GetArena());
}
std::string SelfTestData_Sample::GetTypeName() const {
return "sentencepiece.SelfTestData.Sample";
}
// ===================================================================
class SelfTestData::_Internal {
public:
};
SelfTestData::SelfTestData(::PROTOBUF_NAMESPACE_ID::Arena* arena)
: ::PROTOBUF_NAMESPACE_ID::MessageLite(arena),
_extensions_(arena),
samples_(arena) {
SharedCtor();
RegisterArenaDtor(arena);
// @@protoc_insertion_point(arena_constructor:sentencepiece.SelfTestData)
}
SelfTestData::SelfTestData(const SelfTestData& from)
: ::PROTOBUF_NAMESPACE_ID::MessageLite(), samples_(from.samples_) {
_internal_metadata_.MergeFrom<std::string>(from._internal_metadata_);
_extensions_.MergeFrom(from._extensions_);
// @@protoc_insertion_point(copy_constructor:sentencepiece.SelfTestData)
}
void SelfTestData::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(
&scc_info_SelfTestData_sentencepiece_5fmodel_2eproto.base);
}
SelfTestData::~SelfTestData() {
// @@protoc_insertion_point(destructor:sentencepiece.SelfTestData)
SharedDtor();
_internal_metadata_.Delete<std::string>();
}
void SelfTestData::SharedDtor() {
GOOGLE_DCHECK(GetArena() == nullptr);
}
void SelfTestData::ArenaDtor(void* object) {
SelfTestData* _this = reinterpret_cast<SelfTestData*>(object);
(void)_this;
}
void SelfTestData::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) {}
void SelfTestData::SetCachedSize(int size) const {
_cached_size_.Set(size);
}
const SelfTestData& SelfTestData::default_instance() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(
&::scc_info_SelfTestData_sentencepiece_5fmodel_2eproto.base);
return *internal_default_instance();
}
void SelfTestData::Clear() {
// @@protoc_insertion_point(message_clear_start:sentencepiece.SelfTestData)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void)cached_has_bits;
_extensions_.Clear();
samples_.Clear();
_internal_metadata_.Clear<std::string>();
}
const char* SelfTestData::_InternalParse(
const char* ptr,
::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) {
#define CHK_(x) \
if (PROTOBUF_PREDICT_FALSE(!(x))) \
goto failure
while (!ctx->Done(&ptr)) {
::PROTOBUF_NAMESPACE_ID::uint32 tag;
ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag);
CHK_(ptr);
switch (tag >> 3) {
// repeated .sentencepiece.SelfTestData.Sample samples = 1;
case 1:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) {
ptr -= 1;
do {
ptr += 1;
ptr = ctx->ParseMessage(_internal_add_samples(), ptr);
CHK_(ptr);
if (!ctx->DataAvailable(ptr)) {
break;
}
} while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr));
} else {
goto handle_unusual;
}
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
ctx->SetLastTag(tag);
goto success;
}
if ((1600u <= tag)) {
ptr = _extensions_.ParseField(tag, ptr, internal_default_instance(),
&_internal_metadata_, ctx);
CHK_(ptr != nullptr);
continue;
}
ptr = UnknownFieldParse(
tag, _internal_metadata_.mutable_unknown_fields<std::string>(), ptr,
ctx);
CHK_(ptr != nullptr);
continue;
}
} // switch
} // while
success:
return ptr;
failure:
ptr = nullptr;
goto success;
#undef CHK_
}
::PROTOBUF_NAMESPACE_ID::uint8* SelfTestData::_InternalSerialize(
::PROTOBUF_NAMESPACE_ID::uint8* target,
::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const {
// @@protoc_insertion_point(serialize_to_array_start:sentencepiece.SelfTestData)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void)cached_has_bits;
// repeated .sentencepiece.SelfTestData.Sample samples = 1;
for (unsigned int i = 0, n = static_cast<unsigned int>(
this->_internal_samples_size());
i < n; i++) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::InternalWriteMessage(
1, this->_internal_samples(i), target, stream);
}
// Extension range [200, 536870912)
target = _extensions_._InternalSerialize(200, 536870912, target, stream);
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
target = stream->WriteRaw(
_internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.data(),
static_cast<int>(
_internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.size()),
target);
}
// @@protoc_insertion_point(serialize_to_array_end:sentencepiece.SelfTestData)
return target;
}
size_t SelfTestData::ByteSizeLong() const {
// @@protoc_insertion_point(message_byte_size_start:sentencepiece.SelfTestData)
size_t total_size = 0;
total_size += _extensions_.ByteSize();
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void)cached_has_bits;
// repeated .sentencepiece.SelfTestData.Sample samples = 1;
total_size += 1UL * this->_internal_samples_size();
for (const auto& msg : this->samples_) {
total_size +=
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg);
}
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
total_size += _internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.size();
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
}
void SelfTestData::CheckTypeAndMergeFrom(
const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) {
MergeFrom(
*::PROTOBUF_NAMESPACE_ID::internal::DownCast<const SelfTestData*>(&from));
}
void SelfTestData::MergeFrom(const SelfTestData& from) {
// @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.SelfTestData)
GOOGLE_DCHECK_NE(&from, this);
_extensions_.MergeFrom(from._extensions_);
_internal_metadata_.MergeFrom<std::string>(from._internal_metadata_);
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void)cached_has_bits;
samples_.MergeFrom(from.samples_);
}
void SelfTestData::CopyFrom(const SelfTestData& from) {
// @@protoc_insertion_point(class_specific_copy_from_start:sentencepiece.SelfTestData)
if (&from == this) {
return;
}
Clear();
MergeFrom(from);
}
bool SelfTestData::IsInitialized() const {
if (!_extensions_.IsInitialized()) {
return false;
}
return true;
}
void SelfTestData::InternalSwap(SelfTestData* other) {
using std::swap;
_extensions_.Swap(&other->_extensions_);
_internal_metadata_.Swap<std::string>(&other->_internal_metadata_);
samples_.InternalSwap(&other->samples_);
}
std::string SelfTestData::GetTypeName() const {
return "sentencepiece.SelfTestData";
}
// ===================================================================
class ModelProto_SentencePiece::_Internal {
public:
using HasBits = decltype(std::declval<ModelProto_SentencePiece>()._has_bits_);
static void set_has_piece(HasBits* has_bits) { (*has_bits)[0] |= 1u; }
static void set_has_score(HasBits* has_bits) { (*has_bits)[0] |= 2u; }
static void set_has_type(HasBits* has_bits) { (*has_bits)[0] |= 4u; }
};
ModelProto_SentencePiece::ModelProto_SentencePiece(
::PROTOBUF_NAMESPACE_ID::Arena* arena)
: ::PROTOBUF_NAMESPACE_ID::MessageLite(arena), _extensions_(arena) {
SharedCtor();
RegisterArenaDtor(arena);
// @@protoc_insertion_point(arena_constructor:sentencepiece.ModelProto.SentencePiece)
}
ModelProto_SentencePiece::ModelProto_SentencePiece(
const ModelProto_SentencePiece& from)
: ::PROTOBUF_NAMESPACE_ID::MessageLite(), _has_bits_(from._has_bits_) {
_internal_metadata_.MergeFrom<std::string>(from._internal_metadata_);
_extensions_.MergeFrom(from._extensions_);
piece_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (from._internal_has_piece()) {
piece_.Set(
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{},
from._internal_piece(), GetArena());
}
::memcpy(&score_, &from.score_,
static_cast<size_t>(reinterpret_cast<char*>(&type_) -
reinterpret_cast<char*>(&score_)) +
sizeof(type_));
// @@protoc_insertion_point(copy_constructor:sentencepiece.ModelProto.SentencePiece)
}
void ModelProto_SentencePiece::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(
&scc_info_ModelProto_SentencePiece_sentencepiece_5fmodel_2eproto.base);
piece_.UnsafeSetDefault(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
score_ = 0;
type_ = 1;
}
ModelProto_SentencePiece::~ModelProto_SentencePiece() {
// @@protoc_insertion_point(destructor:sentencepiece.ModelProto.SentencePiece)
SharedDtor();
_internal_metadata_.Delete<std::string>();
}
void ModelProto_SentencePiece::SharedDtor() {
GOOGLE_DCHECK(GetArena() == nullptr);
piece_.DestroyNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
void ModelProto_SentencePiece::ArenaDtor(void* object) {
ModelProto_SentencePiece* _this =
reinterpret_cast<ModelProto_SentencePiece*>(object);
(void)_this;
}
void ModelProto_SentencePiece::RegisterArenaDtor(
::PROTOBUF_NAMESPACE_ID::Arena*) {}
void ModelProto_SentencePiece::SetCachedSize(int size) const {
_cached_size_.Set(size);
}
const ModelProto_SentencePiece& ModelProto_SentencePiece::default_instance() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(
&::scc_info_ModelProto_SentencePiece_sentencepiece_5fmodel_2eproto.base);
return *internal_default_instance();
}
void ModelProto_SentencePiece::Clear() {
// @@protoc_insertion_point(message_clear_start:sentencepiece.ModelProto.SentencePiece)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void)cached_has_bits;
_extensions_.Clear();
cached_has_bits = _has_bits_[0];
if (cached_has_bits & 0x00000001u) {
piece_.ClearNonDefaultToEmpty();
}
if (cached_has_bits & 0x00000006u) {
score_ = 0;
type_ = 1;
}
_has_bits_.Clear();
_internal_metadata_.Clear<std::string>();
}
const char* ModelProto_SentencePiece::_InternalParse(
const char* ptr,
::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) {
#define CHK_(x) \
if (PROTOBUF_PREDICT_FALSE(!(x))) \
goto failure
_Internal::HasBits has_bits{};
while (!ctx->Done(&ptr)) {
::PROTOBUF_NAMESPACE_ID::uint32 tag;
ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag);
CHK_(ptr);
switch (tag >> 3) {
// optional string piece = 1;
case 1:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) {
auto str = _internal_mutable_piece();
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(
str, ptr, ctx);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional float score = 2;
case 2:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 21)) {
_Internal::set_has_score(&has_bits);
score_ = ::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<float>(ptr);
ptr += sizeof(float);
} else {
goto handle_unusual;
}
continue;
// optional .sentencepiece.ModelProto.SentencePiece.Type type = 3 [default
// = NORMAL];
case 3:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) {
::PROTOBUF_NAMESPACE_ID::uint64 val =
::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
if (PROTOBUF_PREDICT_TRUE(
::sentencepiece::ModelProto_SentencePiece_Type_IsValid(
val))) {
_internal_set_type(
static_cast<::sentencepiece::ModelProto_SentencePiece_Type>(
val));
} else {
::PROTOBUF_NAMESPACE_ID::internal::WriteVarint(
3, val, mutable_unknown_fields());
}
} else {
goto handle_unusual;
}
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
ctx->SetLastTag(tag);
goto success;
}
if ((1600u <= tag)) {
ptr = _extensions_.ParseField(tag, ptr, internal_default_instance(),
&_internal_metadata_, ctx);
CHK_(ptr != nullptr);
continue;
}
ptr = UnknownFieldParse(
tag, _internal_metadata_.mutable_unknown_fields<std::string>(), ptr,
ctx);
CHK_(ptr != nullptr);
continue;
}
} // switch
} // while
success:
_has_bits_.Or(has_bits);
return ptr;
failure:
ptr = nullptr;
goto success;
#undef CHK_
}
::PROTOBUF_NAMESPACE_ID::uint8* ModelProto_SentencePiece::_InternalSerialize(
::PROTOBUF_NAMESPACE_ID::uint8* target,
::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const {
// @@protoc_insertion_point(serialize_to_array_start:sentencepiece.ModelProto.SentencePiece)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void)cached_has_bits;
cached_has_bits = _has_bits_[0];
// optional string piece = 1;
if (cached_has_bits & 0x00000001u) {
target =
stream->WriteStringMaybeAliased(1, this->_internal_piece(), target);
}
// optional float score = 2;
if (cached_has_bits & 0x00000002u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteFloatToArray(
2, this->_internal_score(), target);
}
// optional .sentencepiece.ModelProto.SentencePiece.Type type = 3 [default =
// NORMAL];
if (cached_has_bits & 0x00000004u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
3, this->_internal_type(), target);
}
// Extension range [200, 536870912)
target = _extensions_._InternalSerialize(200, 536870912, target, stream);
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
target = stream->WriteRaw(
_internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.data(),
static_cast<int>(
_internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.size()),
target);
}
// @@protoc_insertion_point(serialize_to_array_end:sentencepiece.ModelProto.SentencePiece)
return target;
}
size_t ModelProto_SentencePiece::ByteSizeLong() const {
// @@protoc_insertion_point(message_byte_size_start:sentencepiece.ModelProto.SentencePiece)
size_t total_size = 0;
total_size += _extensions_.ByteSize();
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void)cached_has_bits;
cached_has_bits = _has_bits_[0];
if (cached_has_bits & 0x00000007u) {
// optional string piece = 1;
if (cached_has_bits & 0x00000001u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->_internal_piece());
}
// optional float score = 2;
if (cached_has_bits & 0x00000002u) {
total_size += 1 + 4;
}
// optional .sentencepiece.ModelProto.SentencePiece.Type type = 3 [default =
// NORMAL];
if (cached_has_bits & 0x00000004u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::EnumSize(
this->_internal_type());
}
}
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
total_size += _internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.size();
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
}
void ModelProto_SentencePiece::CheckTypeAndMergeFrom(
const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) {
MergeFrom(*::PROTOBUF_NAMESPACE_ID::internal::DownCast<
const ModelProto_SentencePiece*>(&from));
}
void ModelProto_SentencePiece::MergeFrom(const ModelProto_SentencePiece& from) {
// @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.ModelProto.SentencePiece)
GOOGLE_DCHECK_NE(&from, this);
_extensions_.MergeFrom(from._extensions_);
_internal_metadata_.MergeFrom<std::string>(from._internal_metadata_);
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void)cached_has_bits;
cached_has_bits = from._has_bits_[0];
if (cached_has_bits & 0x00000007u) {
if (cached_has_bits & 0x00000001u) {
_internal_set_piece(from._internal_piece());
}
if (cached_has_bits & 0x00000002u) {
score_ = from.score_;
}
if (cached_has_bits & 0x00000004u) {
type_ = from.type_;
}
_has_bits_[0] |= cached_has_bits;
}
}
void ModelProto_SentencePiece::CopyFrom(const ModelProto_SentencePiece& from) {
// @@protoc_insertion_point(class_specific_copy_from_start:sentencepiece.ModelProto.SentencePiece)
if (&from == this) {
return;
}
Clear();
MergeFrom(from);
}
bool ModelProto_SentencePiece::IsInitialized() const {
if (!_extensions_.IsInitialized()) {
return false;
}
return true;
}
void ModelProto_SentencePiece::InternalSwap(ModelProto_SentencePiece* other) {
using std::swap;
_extensions_.Swap(&other->_extensions_);
_internal_metadata_.Swap<std::string>(&other->_internal_metadata_);
swap(_has_bits_[0], other->_has_bits_[0]);
piece_.Swap(&other->piece_,
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
GetArena());
swap(score_, other->score_);
swap(type_, other->type_);
}
std::string ModelProto_SentencePiece::GetTypeName() const {
return "sentencepiece.ModelProto.SentencePiece";
}
// ===================================================================
class ModelProto::_Internal {
public:
using HasBits = decltype(std::declval<ModelProto>()._has_bits_);
static const ::sentencepiece::TrainerSpec& trainer_spec(
const ModelProto* msg);
static void set_has_trainer_spec(HasBits* has_bits) { (*has_bits)[0] |= 1u; }
static const ::sentencepiece::NormalizerSpec& normalizer_spec(
const ModelProto* msg);
static void set_has_normalizer_spec(HasBits* has_bits) {
(*has_bits)[0] |= 2u;
}
static const ::sentencepiece::SelfTestData& self_test_data(
const ModelProto* msg);
static void set_has_self_test_data(HasBits* has_bits) {
(*has_bits)[0] |= 4u;
}
static const ::sentencepiece::NormalizerSpec& denormalizer_spec(
const ModelProto* msg);
static void set_has_denormalizer_spec(HasBits* has_bits) {
(*has_bits)[0] |= 8u;
}
};
const ::sentencepiece::TrainerSpec& ModelProto::_Internal::trainer_spec(
const ModelProto* msg) {
return *msg->trainer_spec_;
}
const ::sentencepiece::NormalizerSpec& ModelProto::_Internal::normalizer_spec(
const ModelProto* msg) {
return *msg->normalizer_spec_;
}
const ::sentencepiece::SelfTestData& ModelProto::_Internal::self_test_data(
const ModelProto* msg) {
return *msg->self_test_data_;
}
const ::sentencepiece::NormalizerSpec& ModelProto::_Internal::denormalizer_spec(
const ModelProto* msg) {
return *msg->denormalizer_spec_;
}
ModelProto::ModelProto(::PROTOBUF_NAMESPACE_ID::Arena* arena)
: ::PROTOBUF_NAMESPACE_ID::MessageLite(arena),
_extensions_(arena),
pieces_(arena) {
SharedCtor();
RegisterArenaDtor(arena);
// @@protoc_insertion_point(arena_constructor:sentencepiece.ModelProto)
}
ModelProto::ModelProto(const ModelProto& from)
: ::PROTOBUF_NAMESPACE_ID::MessageLite(),
_has_bits_(from._has_bits_),
pieces_(from.pieces_) {
_internal_metadata_.MergeFrom<std::string>(from._internal_metadata_);
_extensions_.MergeFrom(from._extensions_);
if (from._internal_has_trainer_spec()) {
trainer_spec_ = new ::sentencepiece::TrainerSpec(*from.trainer_spec_);
} else {
trainer_spec_ = nullptr;
}
if (from._internal_has_normalizer_spec()) {
normalizer_spec_ =
new ::sentencepiece::NormalizerSpec(*from.normalizer_spec_);
} else {
normalizer_spec_ = nullptr;
}
if (from._internal_has_self_test_data()) {
self_test_data_ = new ::sentencepiece::SelfTestData(*from.self_test_data_);
} else {
self_test_data_ = nullptr;
}
if (from._internal_has_denormalizer_spec()) {
denormalizer_spec_ =
new ::sentencepiece::NormalizerSpec(*from.denormalizer_spec_);
} else {
denormalizer_spec_ = nullptr;
}
// @@protoc_insertion_point(copy_constructor:sentencepiece.ModelProto)
}
void ModelProto::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(
&scc_info_ModelProto_sentencepiece_5fmodel_2eproto.base);
::memset(reinterpret_cast<char*>(this) +
static_cast<size_t>(reinterpret_cast<char*>(&trainer_spec_) -
reinterpret_cast<char*>(this)),
0,
static_cast<size_t>(reinterpret_cast<char*>(&denormalizer_spec_) -
reinterpret_cast<char*>(&trainer_spec_)) +
sizeof(denormalizer_spec_));
}
ModelProto::~ModelProto() {
// @@protoc_insertion_point(destructor:sentencepiece.ModelProto)
SharedDtor();
_internal_metadata_.Delete<std::string>();
}
void ModelProto::SharedDtor() {
GOOGLE_DCHECK(GetArena() == nullptr);
if (this != internal_default_instance()) {
delete trainer_spec_;
}
if (this != internal_default_instance()) {
delete normalizer_spec_;
}
if (this != internal_default_instance()) {
delete self_test_data_;
}
if (this != internal_default_instance()) {
delete denormalizer_spec_;
}
}
void ModelProto::ArenaDtor(void* object) {
ModelProto* _this = reinterpret_cast<ModelProto*>(object);
(void)_this;
}
void ModelProto::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) {}
void ModelProto::SetCachedSize(int size) const {
_cached_size_.Set(size);
}
const ModelProto& ModelProto::default_instance() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(
&::scc_info_ModelProto_sentencepiece_5fmodel_2eproto.base);
return *internal_default_instance();
}
void ModelProto::Clear() {
// @@protoc_insertion_point(message_clear_start:sentencepiece.ModelProto)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void)cached_has_bits;
_extensions_.Clear();
pieces_.Clear();
cached_has_bits = _has_bits_[0];
if (cached_has_bits & 0x0000000fu) {
if (cached_has_bits & 0x00000001u) {
GOOGLE_DCHECK(trainer_spec_ != nullptr);
trainer_spec_->Clear();
}
if (cached_has_bits & 0x00000002u) {
GOOGLE_DCHECK(normalizer_spec_ != nullptr);
normalizer_spec_->Clear();
}
if (cached_has_bits & 0x00000004u) {
GOOGLE_DCHECK(self_test_data_ != nullptr);
self_test_data_->Clear();
}
if (cached_has_bits & 0x00000008u) {
GOOGLE_DCHECK(denormalizer_spec_ != nullptr);
denormalizer_spec_->Clear();
}
}
_has_bits_.Clear();
_internal_metadata_.Clear<std::string>();
}
const char* ModelProto::_InternalParse(
const char* ptr,
::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) {
#define CHK_(x) \
if (PROTOBUF_PREDICT_FALSE(!(x))) \
goto failure
_Internal::HasBits has_bits{};
while (!ctx->Done(&ptr)) {
::PROTOBUF_NAMESPACE_ID::uint32 tag;
ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag);
CHK_(ptr);
switch (tag >> 3) {
// repeated .sentencepiece.ModelProto.SentencePiece pieces = 1;
case 1:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) {
ptr -= 1;
do {
ptr += 1;
ptr = ctx->ParseMessage(_internal_add_pieces(), ptr);
CHK_(ptr);
if (!ctx->DataAvailable(ptr)) {
break;
}
} while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr));
} else {
goto handle_unusual;
}
continue;
// optional .sentencepiece.TrainerSpec trainer_spec = 2;
case 2:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) {
ptr = ctx->ParseMessage(_internal_mutable_trainer_spec(), ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional .sentencepiece.NormalizerSpec normalizer_spec = 3;
case 3:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) {
ptr = ctx->ParseMessage(_internal_mutable_normalizer_spec(), ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional .sentencepiece.SelfTestData self_test_data = 4;
case 4:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 34)) {
ptr = ctx->ParseMessage(_internal_mutable_self_test_data(), ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
// optional .sentencepiece.NormalizerSpec denormalizer_spec = 5;
case 5:
if (PROTOBUF_PREDICT_TRUE(
static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 42)) {
ptr = ctx->ParseMessage(_internal_mutable_denormalizer_spec(), ptr);
CHK_(ptr);
} else {
goto handle_unusual;
}
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
ctx->SetLastTag(tag);
goto success;
}
if ((1600u <= tag)) {
ptr = _extensions_.ParseField(tag, ptr, internal_default_instance(),
&_internal_metadata_, ctx);
CHK_(ptr != nullptr);
continue;
}
ptr = UnknownFieldParse(
tag, _internal_metadata_.mutable_unknown_fields<std::string>(), ptr,
ctx);
CHK_(ptr != nullptr);
continue;
}
} // switch
} // while
success:
_has_bits_.Or(has_bits);
return ptr;
failure:
ptr = nullptr;
goto success;
#undef CHK_
}
::PROTOBUF_NAMESPACE_ID::uint8* ModelProto::_InternalSerialize(
::PROTOBUF_NAMESPACE_ID::uint8* target,
::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const {
// @@protoc_insertion_point(serialize_to_array_start:sentencepiece.ModelProto)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void)cached_has_bits;
// repeated .sentencepiece.ModelProto.SentencePiece pieces = 1;
for (unsigned int i = 0, n = static_cast<unsigned int>(
this->_internal_pieces_size());
i < n; i++) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::InternalWriteMessage(
1, this->_internal_pieces(i), target, stream);
}
cached_has_bits = _has_bits_[0];
// optional .sentencepiece.TrainerSpec trainer_spec = 2;
if (cached_has_bits & 0x00000001u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::InternalWriteMessage(
2, _Internal::trainer_spec(this), target, stream);
}
// optional .sentencepiece.NormalizerSpec normalizer_spec = 3;
if (cached_has_bits & 0x00000002u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::InternalWriteMessage(
3, _Internal::normalizer_spec(this), target, stream);
}
// optional .sentencepiece.SelfTestData self_test_data = 4;
if (cached_has_bits & 0x00000004u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::InternalWriteMessage(
4, _Internal::self_test_data(this), target, stream);
}
// optional .sentencepiece.NormalizerSpec denormalizer_spec = 5;
if (cached_has_bits & 0x00000008u) {
target = stream->EnsureSpace(target);
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::InternalWriteMessage(
5, _Internal::denormalizer_spec(this), target, stream);
}
// Extension range [200, 536870912)
target = _extensions_._InternalSerialize(200, 536870912, target, stream);
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
target = stream->WriteRaw(
_internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.data(),
static_cast<int>(
_internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.size()),
target);
}
// @@protoc_insertion_point(serialize_to_array_end:sentencepiece.ModelProto)
return target;
}
size_t ModelProto::ByteSizeLong() const {
// @@protoc_insertion_point(message_byte_size_start:sentencepiece.ModelProto)
size_t total_size = 0;
total_size += _extensions_.ByteSize();
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void)cached_has_bits;
// repeated .sentencepiece.ModelProto.SentencePiece pieces = 1;
total_size += 1UL * this->_internal_pieces_size();
for (const auto& msg : this->pieces_) {
total_size +=
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg);
}
cached_has_bits = _has_bits_[0];
if (cached_has_bits & 0x0000000fu) {
// optional .sentencepiece.TrainerSpec trainer_spec = 2;
if (cached_has_bits & 0x00000001u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(
*trainer_spec_);
}
// optional .sentencepiece.NormalizerSpec normalizer_spec = 3;
if (cached_has_bits & 0x00000002u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(
*normalizer_spec_);
}
// optional .sentencepiece.SelfTestData self_test_data = 4;
if (cached_has_bits & 0x00000004u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(
*self_test_data_);
}
// optional .sentencepiece.NormalizerSpec denormalizer_spec = 5;
if (cached_has_bits & 0x00000008u) {
total_size +=
1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(
*denormalizer_spec_);
}
}
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
total_size += _internal_metadata_
.unknown_fields<std::string>(
::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString)
.size();
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
}
void ModelProto::CheckTypeAndMergeFrom(
const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) {
MergeFrom(
*::PROTOBUF_NAMESPACE_ID::internal::DownCast<const ModelProto*>(&from));
}
void ModelProto::MergeFrom(const ModelProto& from) {
// @@protoc_insertion_point(class_specific_merge_from_start:sentencepiece.ModelProto)
GOOGLE_DCHECK_NE(&from, this);
_extensions_.MergeFrom(from._extensions_);
_internal_metadata_.MergeFrom<std::string>(from._internal_metadata_);
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void)cached_has_bits;
pieces_.MergeFrom(from.pieces_);
cached_has_bits = from._has_bits_[0];
if (cached_has_bits & 0x0000000fu) {
if (cached_has_bits & 0x00000001u) {
_internal_mutable_trainer_spec()->::sentencepiece::TrainerSpec::MergeFrom(
from._internal_trainer_spec());
}
if (cached_has_bits & 0x00000002u) {
_internal_mutable_normalizer_spec()
->::sentencepiece::NormalizerSpec::MergeFrom(
from._internal_normalizer_spec());
}
if (cached_has_bits & 0x00000004u) {
_internal_mutable_self_test_data()
->::sentencepiece::SelfTestData::MergeFrom(
from._internal_self_test_data());
}
if (cached_has_bits & 0x00000008u) {
_internal_mutable_denormalizer_spec()
->::sentencepiece::NormalizerSpec::MergeFrom(
from._internal_denormalizer_spec());
}
}
}
void ModelProto::CopyFrom(const ModelProto& from) {
// @@protoc_insertion_point(class_specific_copy_from_start:sentencepiece.ModelProto)
if (&from == this) {
return;
}
Clear();
MergeFrom(from);
}
bool ModelProto::IsInitialized() const {
if (!_extensions_.IsInitialized()) {
return false;
}
if (!::PROTOBUF_NAMESPACE_ID::internal::AllAreInitialized(pieces_)) {
return false;
}
if (_internal_has_trainer_spec()) {
if (!trainer_spec_->IsInitialized()) {
return false;
}
}
if (_internal_has_normalizer_spec()) {
if (!normalizer_spec_->IsInitialized()) {
return false;
}
}
if (_internal_has_self_test_data()) {
if (!self_test_data_->IsInitialized()) {
return false;
}
}
if (_internal_has_denormalizer_spec()) {
if (!denormalizer_spec_->IsInitialized()) {
return false;
}
}
return true;
}
void ModelProto::InternalSwap(ModelProto* other) {
using std::swap;
_extensions_.Swap(&other->_extensions_);
_internal_metadata_.Swap<std::string>(&other->_internal_metadata_);
swap(_has_bits_[0], other->_has_bits_[0]);
pieces_.InternalSwap(&other->pieces_);
::PROTOBUF_NAMESPACE_ID::internal::memswap<
PROTOBUF_FIELD_OFFSET(ModelProto, denormalizer_spec_) +
sizeof(ModelProto::denormalizer_spec_) -
PROTOBUF_FIELD_OFFSET(ModelProto, trainer_spec_)>(
reinterpret_cast<char*>(&trainer_spec_),
reinterpret_cast<char*>(&other->trainer_spec_));
}
std::string ModelProto::GetTypeName() const {
return "sentencepiece.ModelProto";
}
// @@protoc_insertion_point(namespace_scope)
} // namespace sentencepiece
PROTOBUF_NAMESPACE_OPEN
template <>
PROTOBUF_NOINLINE ::sentencepiece::TrainerSpec*
Arena::CreateMaybeMessage<::sentencepiece::TrainerSpec>(Arena* arena) {
return Arena::CreateMessageInternal<::sentencepiece::TrainerSpec>(arena);
}
template <>
PROTOBUF_NOINLINE ::sentencepiece::NormalizerSpec*
Arena::CreateMaybeMessage<::sentencepiece::NormalizerSpec>(Arena* arena) {
return Arena::CreateMessageInternal<::sentencepiece::NormalizerSpec>(arena);
}
template <>
PROTOBUF_NOINLINE ::sentencepiece::SelfTestData_Sample*
Arena::CreateMaybeMessage<::sentencepiece::SelfTestData_Sample>(Arena* arena) {
return Arena::CreateMessageInternal<::sentencepiece::SelfTestData_Sample>(
arena);
}
template <>
PROTOBUF_NOINLINE ::sentencepiece::SelfTestData*
Arena::CreateMaybeMessage<::sentencepiece::SelfTestData>(Arena* arena) {
return Arena::CreateMessageInternal<::sentencepiece::SelfTestData>(arena);
}
template <>
PROTOBUF_NOINLINE ::sentencepiece::ModelProto_SentencePiece*
Arena::CreateMaybeMessage<::sentencepiece::ModelProto_SentencePiece>(
Arena* arena) {
return Arena::CreateMessageInternal<
::sentencepiece::ModelProto_SentencePiece>(arena);
}
template <>
PROTOBUF_NOINLINE ::sentencepiece::ModelProto*
Arena::CreateMaybeMessage<::sentencepiece::ModelProto>(Arena* arena) {
return Arena::CreateMessageInternal<::sentencepiece::ModelProto>(arena);
}
PROTOBUF_NAMESPACE_CLOSE
// @@protoc_insertion_point(global_scope)
#include <google/protobuf/port_undef.inc>
Codebase Browser
chromium