chromium/third_party/tensorflow_models/src/research/seq_flow_lite/tf_ops/projection_normalizer_util.cc

/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tf_ops/projection_normalizer_util.h"  // seq_flow_lite

#include <algorithm>
#include <cstddef>
#include <memory>
#include <sstream>
#include <utility>

#include "icu4c/source/common/unicode/utf8.h"
#include "tf_ops/projection_util.h"  // seq_flow_lite

// Returns true if the given text contains a number.
bool IsDigit(const std::string& text) { … }

// Gets the string containing |num_chars| characters from |start| position.
std::string GetCharToken(const std::vector<std::string>& char_tokens,
                         size_t start, size_t num_chars) { … }

// Counts how many times |pattern| appeared from |start| position.
int GetNumPattern(const std::vector<std::string>& char_tokens, size_t start,
                  size_t num_chars, const std::string& pattern) { … }

std::string ContractToken(const char* input_ptr, size_t len, size_t num_chars) { … }

void NormalizeSpaces(std::string& input) { … }

void ProjectionNormalizer::InitializeSeparators(const std::string& separators) { … }

std::string ProjectionNormalizer::NormalizeInternal(const char* input_ptr,
                                                    size_t len) { … }

std::string ProjectionNormalizer::Normalize(const std::string& input,
                                            size_t max_input) { … }

std::string ProjectionNormalizer::Normalize(const char* input_ptr, size_t len,
                                            size_t max_input) { … }