chromium/components/omnibox/browser/on_device_tail_tokenizer.cc

// Copyright 2022 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "components/omnibox/browser/on_device_tail_tokenizer.h"

#include <algorithm>
#include <fstream>
#include <sstream>
#include <string>
#include <utility>
#include <vector>

#include "base/containers/flat_map.h"
#include "base/containers/flat_set.h"
#include "base/files/file_util.h"
#include "base/logging.h"
#include "base/strings/string_util.h"
#include "components/omnibox/browser/omnibox_field_trial.h"

namespace {
// Maximum vocabulary file size that will be loaded in bytes.
static constexpr size_t kVocabFileSizeLimit = …;

// The max num of single char tokens where token IDs are directly mapped to
// ASCII characters.
// Token IDs greater than kNumSingleChar are special control tokens or
// multi-char tokens specified by the given vocabulary file.
static constexpr size_t kNumSingleChar = …;

// Special control tokens.
static constexpr char kBeginQueryToken[] = …;
static constexpr char kEndQueryToken[] = …;
static constexpr char kEmptyPreviousQueryToken[] = …;
static constexpr char kUnknownToken[] = …;

std::ostream& operator<<(std::ostream& os,
                         const base::flat_set<std::string>& tokens) { … }

}  // namespace

OnDeviceTailTokenizer::Tokenization::Tokenization() = default;

OnDeviceTailTokenizer::Tokenization::~Tokenization() = default;

OnDeviceTailTokenizer::OnDeviceTailTokenizer() = default;

OnDeviceTailTokenizer::~OnDeviceTailTokenizer() = default;

bool OnDeviceTailTokenizer::Init(const base::FilePath& vocabulary_filepath) { … }

bool OnDeviceTailTokenizer::IsReady() const { … }

void OnDeviceTailTokenizer::Reset() { … }

std::string OnDeviceTailTokenizer::IdToToken(const TokenId token_id) const { … }

OnDeviceTailTokenizer::TokenId OnDeviceTailTokenizer::TokenToId(
    const std::string& token) const { … }

void OnDeviceTailTokenizer::InitAmbiguousMap() { … }

bool OnDeviceTailTokenizer::IsBeginQueryTokenId(TokenId token_id) const { … }

bool OnDeviceTailTokenizer::IsEndQueryTokenId(TokenId token_id) const { … }

OnDeviceTailTokenizer::TokenId OnDeviceTailTokenizer::GetEndQueryTokenId()
    const { … }

bool OnDeviceTailTokenizer::IsAmbiguousToken(const std::string& token) const { … }

bool OnDeviceTailTokenizer::IsTokenPrintable(TokenId token_id) const { … }

void OnDeviceTailTokenizer::EncodeRawString(
    const std::string& raw_string,
    std::vector<std::pair<std::string, TokenId>>* token_and_ids) const { … }

void OnDeviceTailTokenizer::TokenizePrevQuery(
    const std::string& prev_query,
    TokenIds* prev_query_token_ids) const { … }

void OnDeviceTailTokenizer::CreatePrefixTokenization(
    const std::string& prefix,
    Tokenization* tokenization) const { … }

void OnDeviceTailTokenizer::InsertTokenToMaps(const std::string& token) { … }