language_detection_util.cc | Explore in Territory

// Copyright 2014 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifdef UNSAFE_BUFFERS_BUILD
// TODO(crbug.com/40285824): Remove this and convert code to safer constructs.
#pragma allow_unsafe_buffers
#endif

#include "components/translate/core/language_detection/language_detection_util.h"

#include <stddef.h>

#include <string_view>

#include "base/containers/fixed_flat_set.h"
#include "base/logging.h"
#include "base/metrics/histogram_base.h"
#include "base/metrics/histogram_functions.h"
#include "base/metrics/histogram_macros.h"
#include "base/metrics/metrics_hashes.h"
#include "base/strings/string_split.h"
#include "base/strings/string_util.h"
#include "base/strings/utf_string_conversions.h"
#include "components/language/core/common/language_util.h"
#include "components/translate/core/common/translate_constants.h"
#include "components/translate/core/common/translate_metrics.h"
#include "components/translate/core/language_detection/chinese_script_classifier.h"
#include "third_party/cld_3/src/src/nnet_language_identifier.h"

namespace {

// Similar language code list. Some languages are very similar and difficult
// for CLD to distinguish.
struct SimilarLanguageCode { … };

const SimilarLanguageCode kSimilarLanguageCodes[] = …;

// Checks |kSimilarLanguageCodes| and returns group code.
int GetSimilarLanguageGroupCode(const std::string& language) { … }

// Applies a series of language code modification in proper order.
void ApplyLanguageCodeCorrection(std::string* code) { … }

// Get page language from html language code if it is not empty, otherwise get
// page language from Content-Language code. Returns an empty string when
// Content-Language code is empty.
std::string GetHTMLOrHTTPContentLanguage(const std::string& content_lang,
                                         const std::string& html_lang) { … }

// Checks if the model can complement a sub code when the page language doesn't
// know the sub code.
bool CanModelComplementSubCode(const std::string& page_language,
                               const std::string& model_detected_language) { … }

}  // namespace

namespace translate {

std::string FilterDetectedLanguage(const std::string& utf8_text,
                                   const std::string& detected_language,
                                   bool is_detection_reliable) { … }

// Returns the ISO 639 language code of the specified |utf8_text|, or 'unknown'
// if it failed. |is_model_reliable| will be set as true if CLD says the
// detection is reliable and |model_reliability_score| will provide the model's
// confidence in that prediction.
std::string DetermineTextLanguage(const std::string& utf8_text,
                                  bool* is_model_reliable,
                                  float& model_reliability_score) { … }

std::string DeterminePageLanguage(const std::string& code,
                                  const std::string& html_lang,
                                  const std::u16string& contents,
                                  std::string* model_detected_language,
                                  bool* is_model_reliable,
                                  float& model_reliability_score) { … }

std::string DeterminePageLanguageNoModel(
    const std::string& code,
    const std::string& html_lang,
    LanguageVerificationType language_verification_type) { … }

// Now consider the web page language details along with the contents language.
std::string DeterminePageLanguage(const std::string& code,
                                  const std::string& html_lang,
                                  const std::string& model_detected_language,
                                  bool is_model_reliable) { … }

void CorrectLanguageCodeTypo(std::string* code) { … }

bool IsValidLanguageCode(const std::string& code) { … }

bool IsSameOrSimilarLanguages(const std::string& page_language,
                              const std::string& model_detected_language) { … }

bool IsServerWrongConfigurationLanguage(const std::string& language_code) { … }

bool MaybeServerWrongConfiguration(const std::string& page_language,
                                   const std::string& model_detected_language) { … }

}  // namespace translate
chromium/components/translate/core/language_detection/language_detection_util.cc
