annotator.cc | Explore in Territory

// Copyright 2019 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "services/image_annotation/annotator.h"

#include <string_view>
#include <tuple>
#include <utility>
#include <vector>

#include "base/base64.h"
#include "base/containers/contains.h"
#include "base/feature_list.h"
#include "base/functional/bind.h"
#include "base/json/json_reader.h"
#include "base/json/json_writer.h"
#include "base/location.h"
#include "base/logging.h"
#include "base/no_destructor.h"
#include "base/ranges/algorithm.h"
#include "base/strings/string_split.h"
#include "base/time/time.h"
#include "base/values.h"
#include "components/google/core/common/google_util.h"
#include "components/manta/anchovy/anchovy_requests.h"
#include "components/manta/manta_status.h"
#include "net/base/load_flags.h"
#include "net/base/net_errors.h"
#include "net/traffic_annotation/network_traffic_annotation.h"
#include "services/image_annotation/image_annotation_metrics.h"
#include "services/image_annotation/public/mojom/image_annotation.mojom-forward.h"
#include "services/network/public/cpp/resource_request.h"
#include "services/network/public/mojom/url_response_head.mojom.h"
#include "ui/accessibility/accessibility_features.h"
#include "url/gurl.h"

namespace image_annotation {

namespace {

constexpr size_t kImageAnnotationMaxResponseSize = …;  // 1MB.
constexpr size_t kServerLangsMaxResponseSize = …;             // 1KB.

const std::map<std::string, mojom::AnnotationType>& annotation_types() { … }

net::NetworkTrafficAnnotationTag GetTrafficAnnotation() { … }

// For a given source ID and requested description language, returns the unique
// image ID string that can be used to look up results from a server response.
std::string MakeImageId(const std::string& source_id,
                        const std::string& desc_lang_tag) { … }

std::string NormalizeLanguageCode(std::string language) { … }

// The server returns separate OCR results for each region of the image; we
// naively concatenate these into one response string.
//
// Returns a null pointer if there is any unexpected structure to the
// annotations message.
mojom::AnnotationPtr ParseJsonOcrAnnotation(const base::Value& ocr_engine,
                                            const double min_ocr_confidence) { … }

// Extracts annotations from the given description engine result into the second
// element of the return tuple.
//
// The first element of the return tuple will be true if the image was
// classified as containing adult content.
std::tuple<bool, std::vector<mojom::AnnotationPtr>> ParseJsonDescAnnotations(
    const base::Value& desc_engine) { … }

// Extracts annotations from the given icon engine result.
mojom::AnnotationPtr ParseJsonIconAnnotations(const base::Value& icon_engine) { … }

// Returns the integer status code for this engine, or -1 if no status can be
// extracted.
int ExtractStatusCode(const base::Value::Dict* const status_dict) { … }

// Attempts to extract annotation results from the server response, returning a
// map from each image ID to its annotations (if successfully extracted).
std::map<std::string, mojom::AnnotateImageResultPtr> UnpackJsonResponse(
    const base::Value& json_data,
    const double min_ocr_confidence) { … }

mojom::AnnotationPtr CreateAnnotationFromMantaResponse(
    const base::Value::Dict& result_data) { … }

}  // namespace

constexpr char Annotator::kGoogApiKeyHeader[];

static_assert …;
static_assert …;
static_assert …;

Annotator::ClientRequestInfo::ClientRequestInfo(
    mojo::PendingRemote<mojom::ImageProcessor> in_image_processor,
    AnnotateImageCallback in_callback)
    : … { … }

Annotator::ClientRequestInfo::~ClientRequestInfo() = default;

Annotator::ServerRequestInfo::ServerRequestInfo(
    const std::string& in_source_id,
    const bool in_desc_requested,
    const bool in_icon_requested,
    const std::string& in_desc_lang_tag,
    const std::vector<uint8_t>& in_image_bytes)
    : … { … }

Annotator::ServerRequestInfo& Annotator::ServerRequestInfo::operator=(
    ServerRequestInfo&& other) = default;

Annotator::ServerRequestInfo::~ServerRequestInfo() = default;

Annotator::Annotator(
    GURL pixels_server_url,
    GURL langs_server_url,
    std::string api_key,
    const base::TimeDelta throttle,
    const int batch_size,
    const double min_ocr_confidence,
    scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
    std::unique_ptr<manta::AnchovyProvider> anchovy_provider,
    std::unique_ptr<Client> client)
    : … { … }

Annotator::~Annotator() { … }

void Annotator::BindReceiver(mojo::PendingReceiver<mojom::Annotator> receiver) { … }

void Annotator::AnnotateImage(
    const std::string& source_id,
    const std::string& page_language,
    mojo::PendingRemote<mojom::ImageProcessor> image_processor,
    AnnotateImageCallback callback) { … }

// static
bool Annotator::IsWithinDescPolicy(const int32_t width, const int32_t height) { … }

// static
bool Annotator::IsWithinIconPolicy(const int32_t width, const int32_t height) { … }

// static
std::string Annotator::FormatJsonRequest(
    const std::deque<ServerRequestInfo>::iterator begin,
    const std::deque<ServerRequestInfo>::iterator end) { … }

// static
std::unique_ptr<network::SimpleURLLoader> Annotator::MakeRequestLoader(
    const GURL& server_url,
    const std::string& api_key) { … }

void Annotator::OnJpgImageDataReceived(
    const RequestKey& request_key,
    const std::list<ClientRequestInfo>::iterator request_info_it,
    const std::vector<uint8_t>& image_bytes,
    const int32_t width,
    const int32_t height) { … }

void Annotator::SendRequestBatchToServer() { … }

void Annotator::OnMantaResponseReceived(const RequestKey& request_key,
                                        const base::Time request_time,
                                        base::Value::Dict dict,
                                        manta::MantaStatus status) { … }

void Annotator::OnServerResponseReceived(
    const std::set<RequestKey>& request_keys,
    const UrlLoaderList::iterator server_request_it,
    const std::unique_ptr<std::string> json_response) { … }

void Annotator::OnResponseJsonParsed(const std::set<RequestKey>& request_keys,
                                     const std::optional<base::Value> json_data,
                                     const std::optional<std::string>& error) { … }

void Annotator::ProcessResult(
    const RequestKey& request_key,
    const std::map<std::string, mojom::AnnotateImageResultPtr>& results) { … }

void Annotator::ProcessResults(
    const std::set<RequestKey>& request_keys,
    const std::map<std::string, mojom::AnnotateImageResultPtr>& results) { … }

data_decoder::mojom::JsonParser* Annotator::GetJsonParser() { … }

void Annotator::RemoveRequestInfo(
    const RequestKey& request_key,
    const std::list<ClientRequestInfo>::iterator request_info_it,
    const bool canceled) { … }

std::string Annotator::ComputePreferredLanguage(
    const std::string& in_page_language) const { … }

void Annotator::FetchServerLanguages() { … }

void Annotator::OnServerLangsResponseReceived(
    const std::unique_ptr<std::string> json_response) { … }

void Annotator::OnServerLangsResponseJsonParsed(
    std::optional<base::Value> json_data,
    const std::optional<std::string>& error) { … }

}  // namespace image_annotation
chromium/services/image_annotation/annotator.cc
