url_loader.cc | Explore in Territory

// Copyright 2017 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "services/network/url_loader.h"

#include <algorithm>
#include <limits>
#include <map>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <vector>

#include "base/command_line.h"
#include "base/containers/fixed_flat_set.h"
#include "base/files/file.h"
#include "base/functional/bind.h"
#include "base/functional/callback_helpers.h"
#include "base/logging.h"
#include "base/memory/raw_ptr.h"
#include "base/memory/scoped_refptr.h"
#include "base/memory/weak_ptr.h"
#include "base/metrics/histogram_functions.h"
#include "base/metrics/histogram_macros.h"
#include "base/not_fatal_until.h"
#include "base/numerics/safe_conversions.h"
#include "base/ranges/algorithm.h"
#include "base/sequence_checker.h"
#include "base/strings/strcat.h"
#include "base/strings/string_util.h"
#include "base/task/sequenced_task_runner.h"
#include "base/task/single_thread_task_runner.h"
#include "base/task/thread_pool.h"
#include "base/thread_annotations.h"
#include "base/time/time.h"
#include "base/trace_event/trace_event.h"
#include "base/trace_event/typed_macros.h"
#include "build/build_config.h"
#include "mojo/public/cpp/system/simple_watcher.h"
#include "net/base/elements_upload_data_stream.h"
#include "net/base/isolation_info.h"
#include "net/base/load_flags.h"
#include "net/base/load_timing_info.h"
#include "net/base/mime_sniffer.h"
#include "net/base/proxy_chain.h"
#include "net/base/schemeful_site.h"
#include "net/base/transport_info.h"
#include "net/base/upload_bytes_element_reader.h"
#include "net/base/upload_file_element_reader.h"
#include "net/cookies/canonical_cookie.h"
#include "net/cookies/cookie_inclusion_status.h"
#include "net/cookies/cookie_setting_override.h"
#include "net/cookies/cookie_store.h"
#include "net/cookies/cookie_util.h"
#include "net/cookies/site_for_cookies.h"
#include "net/cookies/static_cookie_policy.h"
#include "net/dns/public/secure_dns_policy.h"
#include "net/http/http_connection_info.h"
#include "net/http/http_request_headers.h"
#include "net/http/http_util.h"
#include "net/log/net_log_source_type.h"
#include "net/log/net_log_with_source.h"
#include "net/ssl/client_cert_store.h"
#include "net/ssl/ssl_connection_status_flags.h"
#include "net/ssl/ssl_private_key.h"
#include "net/traffic_annotation/network_traffic_annotation.h"
#include "net/url_request/redirect_info.h"
#include "net/url_request/url_request_context.h"
#include "net/url_request/url_request_context_getter.h"
#include "services/network/ad_heuristic_cookie_overrides.h"
#include "services/network/attribution/attribution_request_helper.h"
#include "services/network/chunked_data_pipe_upload_data_stream.h"
#include "services/network/data_pipe_element_reader.h"
#include "services/network/orb/orb_impl.h"
#include "services/network/public/cpp/client_hints.h"
#include "services/network/public/cpp/constants.h"
#include "services/network/public/cpp/cors/cors.h"
#include "services/network/public/cpp/cors/origin_access_list.h"
#include "services/network/public/cpp/cross_origin_resource_policy.h"
#include "services/network/public/cpp/empty_url_loader_client.h"
#include "services/network/public/cpp/features.h"
#include "services/network/public/cpp/header_util.h"
#include "services/network/public/cpp/ip_address_space_util.h"
#include "services/network/public/cpp/net_adapters.h"
#include "services/network/public/cpp/network_switches.h"
#include "services/network/public/cpp/parsed_headers.h"
#include "services/network/public/cpp/resource_request.h"
#include "services/network/public/mojom/client_security_state.mojom-forward.h"
#include "services/network/public/mojom/cookie_access_observer.mojom-forward.h"
#include "services/network/public/mojom/cookie_access_observer.mojom.h"
#include "services/network/public/mojom/cookie_manager.mojom.h"
#include "services/network/public/mojom/devtools_observer.mojom.h"
#include "services/network/public/mojom/early_hints.mojom.h"
#include "services/network/public/mojom/fetch_api.mojom.h"
#include "services/network/public/mojom/http_raw_headers.mojom.h"
#include "services/network/public/mojom/network_context_client.mojom.h"
#include "services/network/public/mojom/url_loader_factory.mojom.h"
#include "services/network/public/mojom/url_response_head.mojom.h"
#include "services/network/resource_scheduler/resource_scheduler_client.h"
#include "services/network/sec_header_helpers.h"
#include "services/network/shared_dictionary/shared_dictionary_access_checker.h"
#include "services/network/shared_dictionary/shared_dictionary_manager.h"
#include "services/network/shared_dictionary/shared_dictionary_storage.h"
#include "services/network/shared_storage/shared_storage_request_helper.h"
#include "services/network/slop_bucket.h"
#include "services/network/throttling/scoped_throttling_token.h"
#include "services/network/trust_tokens/trust_token_request_helper.h"
#include "services/network/url_loader_factory.h"
#include "url/origin.h"

namespace network {

namespace {

// Cannot use 0, because this means "default" in
// mojo::core::Core::CreateDataPipe
constexpr size_t kBlockedBodyAllocationSize = …;

// Size to allocate for `discard_buffer_`.
constexpr size_t kDiscardBufferSize = …;

// A subclass of net::UploadBytesElementReader which owns
// ResourceRequestBody.
class BytesElementReader : public net::UploadBytesElementReader { … };

// A subclass of net::UploadFileElementReader which owns
// ResourceRequestBody.
// This class is necessary to ensure the BlobData and any attached shareable
// files survive until upload completion.
class FileElementReader : public net::UploadFileElementReader { … };

std::unique_ptr<net::UploadDataStream> CreateUploadDataStream(
    ResourceRequestBody* body,
    std::vector<base::File>& opened_files,
    base::SequencedTaskRunner* file_task_runner) { … }

class SSLPrivateKeyInternal : public net::SSLPrivateKey { … };

bool ShouldNotifyAboutCookie(net::CookieInclusionStatus status) { … }

// Parses AcceptCHFrame and removes client hints already in the headers.
std::vector<mojom::WebClientHintsType> ComputeAcceptCHFrameHints(
    const std::string& accept_ch_frame,
    const net::HttpRequestHeaders& headers) { … }

// Returns true if the |credentials_mode| of the request allows sending
// credentials.
bool ShouldAllowCredentials(mojom::CredentialsMode credentials_mode) { … }

// Returns true when the |credentials_mode| of the request allows sending client
// certificates.
bool ShouldSendClientCertificates(mojom::CredentialsMode credentials_mode) { … }

template <typename T>
T* PtrOrFallback(const mojo::Remote<T>& remote, T* fallback) { … }

// Retrieves the Cookie header from either `cors_exempt_headers` or `headers`.
std::string GetCookiesFromHeaders(
    const net::HttpRequestHeaders& headers,
    const net::HttpRequestHeaders& cors_exempt_headers) { … }

net::HttpRequestHeaders AttachCookies(const net::HttpRequestHeaders& headers,
                                      const std::string& cookies_from_browser) { … }

const char* GetDestinationTypePartString(
    network::mojom::RequestDestination destination) { … }

const char* GetCertStatePartString(const net::SSLInfo& ssl_info) { … }

void MaybeRecordSharedDictionaryUsedResponseMetrics(
    int error_code,
    network::mojom::RequestDestination destination,
    const net::HttpResponseInfo& response_info,
    bool shared_dictionary_allowed_check_passed) { … }

std::vector<network::mojom::HttpRawHeaderPairPtr>
ResponseHeaderToRawHeaderPairs(
    const net::HttpResponseHeaders& response_headers) { … }

}  // namespace

URLLoader::MaybeSyncURLLoaderClient::MaybeSyncURLLoaderClient(
    mojo::PendingRemote<mojom::URLLoaderClient> mojo_client,
    base::WeakPtr<mojom::URLLoaderClient> sync_client)
    : … { … }

URLLoader::MaybeSyncURLLoaderClient::~MaybeSyncURLLoaderClient() = default;

void URLLoader::MaybeSyncURLLoaderClient::Reset() { … }

mojo::PendingReceiver<mojom::URLLoaderClient>
URLLoader::MaybeSyncURLLoaderClient::BindNewPipeAndPassReceiver() { … }

mojom::URLLoaderClient* URLLoader::MaybeSyncURLLoaderClient::Get() { … }

URLLoader::PartialLoadInfo::PartialLoadInfo(net::LoadStateWithParam load_state,
                                            net::UploadProgress upload_progress)
    : … { … }

URLLoader::URLLoader(
    URLLoaderContext& context,
    DeleteCallback delete_callback,
    mojo::PendingReceiver<mojom::URLLoader> url_loader_receiver,
    int32_t options,
    const ResourceRequest& request,
    mojo::PendingRemote<mojom::URLLoaderClient> url_loader_client,
    base::WeakPtr<mojom::URLLoaderClient> sync_url_loader_client,
    const net::NetworkTrafficAnnotationTag& traffic_annotation,
    base::StrictNumeric<int32_t> request_id,
    int keepalive_request_size,
    base::WeakPtr<KeepaliveStatisticsRecorder> keepalive_statistics_recorder,
    std::unique_ptr<TrustTokenRequestHelperFactory> trust_token_helper_factory,
    SharedDictionaryManager* shared_dictionary_manager,
    std::unique_ptr<SharedDictionaryAccessChecker> shared_dictionary_checker,
    mojo::PendingRemote<mojom::CookieAccessObserver> cookie_observer,
    mojo::PendingRemote<mojom::TrustTokenAccessObserver> trust_token_observer,
    mojo::PendingRemote<mojom::URLLoaderNetworkServiceObserver>
        url_loader_network_observer,
    mojo::PendingRemote<mojom::DevToolsObserver> devtools_observer,
    mojo::PendingRemote<mojom::AcceptCHFrameObserver> accept_ch_frame_observer,
    net::CookieSettingOverrides cookie_setting_overrides,
    std::unique_ptr<AttributionRequestHelper> attribution_request_helper,
    bool shared_storage_writable_eligible)
    : … { … }

// This class is used to manage the queue of pending file upload operations
// initiated by the URLLoader::OpenFilesForUpload().
class URLLoader::FileOpenerForUpload { … };

void URLLoader::OpenFilesForUpload(const ResourceRequest& request) { … }

void URLLoader::SetUpUpload(const ResourceRequest& request,
                            int error_code,
                            std::vector<base::File> opened_files) { … }

void URLLoader::ProcessOutboundSharedStorageInterceptor() { … }

// TODO(crbug.com/40254265): Parallelize Private State Tokens and
// Attribution operations.
void URLLoader::ProcessOutboundAttributionInterceptor() { … }

void URLLoader::ProcessOutboundTrustTokenInterceptor(
    const ResourceRequest& request) { … }

void URLLoader::OnDoneConstructingTrustTokenHelper(
    mojom::TrustTokenOperationType operation,
    TrustTokenStatusOrRequestHelper status_or_helper) { … }

void URLLoader::OnDoneBeginningTrustTokenOperation(
    std::optional<net::HttpRequestHeaders> headers,
    mojom::TrustTokenOperationStatus status) { … }

void URLLoader::ScheduleStart() { … }

URLLoader::~URLLoader() { … }

// static
const void* const URLLoader::kUserDataKey = …;

void URLLoader::FollowRedirect(
    const std::vector<std::string>& removed_headers,
    const net::HttpRequestHeaders& modified_headers,
    const net::HttpRequestHeaders& modified_cors_exempt_headers,
    const std::optional<GURL>& new_url) { … }

void URLLoader::SetPriority(net::RequestPriority priority,
                            int32_t intra_priority_value) { … }

void URLLoader::PauseReadingBodyFromNet() { … }

void URLLoader::ResumeReadingBodyFromNet() { … }

PrivateNetworkAccessCheckResult URLLoader::PrivateNetworkAccessCheck(
    const net::TransportInfo& transport_info) { … }

int URLLoader::OnConnected(net::URLRequest* url_request,
                           const net::TransportInfo& info,
                           net::CompletionOnceCallback callback) { … }

mojom::URLResponseHeadPtr URLLoader::BuildResponseHead() const { … }

void URLLoader::OnReceivedRedirect(net::URLRequest* url_request,
                                   const net::RedirectInfo& redirect_info,
                                   bool* defer_redirect) { … }

void URLLoader::ProcessInboundSharedStorageInterceptorOnReceivedRedirect(
    const ::net::RedirectInfo& redirect_info,
    mojom::URLResponseHeadPtr response) { … }

void URLLoader::ProcessInboundAttributionInterceptorOnReceivedRedirect(
    const net::RedirectInfo& redirect_info,
    mojom::URLResponseHeadPtr response) { … }

void URLLoader::ContinueOnReceiveRedirect(
    const net::RedirectInfo& redirect_info,
    uint64_t response_index) { … }

// static
bool URLLoader::HasFetchStreamingUploadBody(const ResourceRequest* request) { … }

// static
std::optional<net::IsolationInfo> URLLoader::GetIsolationInfo(
    const net::IsolationInfo& factory_isolation_info,
    bool automatically_assign_isolation_info,
    const ResourceRequest& request) { … }

void URLLoader::OnAuthRequired(net::URLRequest* url_request,
                               const net::AuthChallengeInfo& auth_info) { … }

void URLLoader::OnCertificateRequested(net::URLRequest* unused,
                                       net::SSLCertRequestInfo* cert_info) { … }

void URLLoader::OnSSLCertificateError(net::URLRequest* request,
                                      int net_error,
                                      const net::SSLInfo& ssl_info,
                                      bool fatal) { … }

void URLLoader::ProcessInboundSharedStorageInterceptorOnResponseStarted() { … }

void URLLoader::ProcessInboundAttributionInterceptorOnResponseStarted() { … }

void URLLoader::OnResponseStarted(net::URLRequest* url_request, int net_error) { … }

void URLLoader::OnDoneFinalizingTrustTokenOperation(
    mojom::TrustTokenOperationStatus status) { … }

void URLLoader::MaybeSendTrustTokenOperationResultToDevTools() { … }

void URLLoader::ContinueOnResponseStarted() { … }

void URLLoader::ReadMore() { … }

// Handles the completion of a read. `num_bytes` is the number of bytes read, 0
// if we reached the end of the response body, or a net::Error otherwise.
// `completed_synchronously` is true if the call to URLRequest::Read did not
// return net::ERR_IO_PENDING. `into_slop_bucket` is true if this was actually a
// read into `slop_bucket_` and not into our own buffer.
void URLLoader::DidRead(int num_bytes,
                        bool completed_synchronously,
                        bool into_slop_bucket) { … }

void URLLoader::OnReadCompleted(net::URLRequest* url_request, int bytes_read) { … }

int URLLoader::OnBeforeStartTransaction(
    const net::HttpRequestHeaders& headers,
    net::NetworkDelegate::OnBeforeStartTransactionCallback callback) { … }

int URLLoader::OnHeadersReceived(
    net::CompletionOnceCallback callback,
    const net::HttpResponseHeaders* original_response_headers,
    scoped_refptr<net::HttpResponseHeaders>* override_response_headers,
    const net::IPEndPoint& endpoint,
    std::optional<GURL>* preserve_fragment_on_redirect_url) { … }

URLLoader::PartialLoadInfo URLLoader::GetPartialLoadInfo() const { … }

mojom::LoadInfoPtr URLLoader::CreateLoadInfo(
    const PartialLoadInfo& partial_load_info) { … }

net::LoadState URLLoader::GetLoadState() const { … }

net::UploadProgress URLLoader::GetUploadProgress() const { … }

int32_t URLLoader::GetProcessId() const { … }

void URLLoader::SetEnableReportingRawHeaders(bool allow) { … }

uint32_t URLLoader::GetResourceType() const { … }

bool URLLoader::CookiesDisabled() const { … }

bool URLLoader::AllowCookie(const net::CanonicalCookie& cookie,
                            const GURL& url,
                            const net::SiteForCookies& site_for_cookies) const { … }

bool URLLoader::AllowFullCookies(
    const GURL& url,
    const net::SiteForCookies& site_for_cookies) const { … }

// static
URLLoader* URLLoader::ForRequest(const net::URLRequest& request) { … }

void URLLoader::OnAuthCredentials(
    const std::optional<net::AuthCredentials>& credentials) { … }

void URLLoader::ContinueWithCertificate(
    const scoped_refptr<net::X509Certificate>& x509_certificate,
    const std::string& provider_name,
    const std::vector<uint16_t>& algorithm_preferences,
    mojo::PendingRemote<mojom::SSLPrivateKey> ssl_private_key) { … }

void URLLoader::ContinueWithoutCertificate() { … }

void URLLoader::CancelRequest() { … }

void URLLoader::CancelRequestIfNonceMatchesAndUrlNotExempted(
    const base::UnguessableToken& nonce,
    const std::set<GURL>& exemptions) { … }

void URLLoader::NotifyCompleted(int error_code) { … }

void URLLoader::OnMojoDisconnect() { … }

void URLLoader::OnResponseBodyStreamConsumerClosed(MojoResult result) { … }

void URLLoader::OnResponseBodyStreamReady(MojoResult result) { … }

void URLLoader::DeleteSelf() { … }

void URLLoader::SendResponseToClient() { … }

void URLLoader::CompletePendingWrite(bool success) { … }

void URLLoader::SetRawResponseHeaders(
    scoped_refptr<const net::HttpResponseHeaders> headers) { … }

void URLLoader::NotifyEarlyResponse(
    scoped_refptr<const net::HttpResponseHeaders> headers) { … }

void URLLoader::MaybeNotifyEarlyResponseToDevtools(
    const net::HttpResponseHeaders& headers) { … }

void URLLoader::SetRawRequestHeadersAndNotify(
    net::HttpRawRequestHeaders headers) { … }

bool URLLoader::IsSharedDictionaryReadAllowed() { … }

void URLLoader::DispatchOnRawRequest(
    std::vector<network::mojom::HttpRawHeaderPairPtr> headers) { … }

bool URLLoader::DispatchOnRawResponse() { … }

void URLLoader::SendUploadProgress(const net::UploadProgress& progress) { … }

void URLLoader::OnUploadProgressACK() { … }

void URLLoader::OnSSLCertificateErrorResponse(const net::SSLInfo& ssl_info,
                                              int net_error) { … }

bool URLLoader::HasDataPipe() const { … }

void URLLoader::ResumeStart() { … }

void URLLoader::OnBeforeSendHeadersComplete(
    net::NetworkDelegate::OnBeforeStartTransactionCallback callback,
    int result,
    const std::optional<net::HttpRequestHeaders>& headers) { … }

void URLLoader::OnHeadersReceivedComplete(
    net::CompletionOnceCallback callback,
    scoped_refptr<net::HttpResponseHeaders>* out_headers,
    std::optional<GURL>* out_preserve_fragment_on_redirect_url,
    int result,
    const std::optional<std::string>& headers,
    const std::optional<GURL>& preserve_fragment_on_redirect_url) { … }

void URLLoader::CompleteBlockedResponse(
    int error_code,
    bool should_report_orb_blocking,
    std::optional<mojom::BlockedByResponseReason> reason) { … }

URLLoader::BlockResponseForOrbResult URLLoader::BlockResponseForOrb() { … }

bool URLLoader::MaybeBlockResponseForOrb(
    orb::ResponseAnalyzer::Decision orb_decision) { … }

void URLLoader::ReportFlaggedResponseCookies(bool call_cookie_observer) { … }

void URLLoader::StartReading() { … }

bool URLLoader::ShouldForceIgnoreSiteForCookies(
    const ResourceRequest& request) { … }

void URLLoader::SetRequestCredentials(const GURL& url) { … }

// [spec]:
// https://fetch.spec.whatwg.org/#cross-origin-embedder-policy-allows-credentials
bool URLLoader::CoepAllowCredentials(const GURL& url) { … }

bool URLLoader::ShouldSendTransferSizeUpdated() const { … }

}  // namespace network
chromium/services/network/url_loader.cc
