storage_queue.cc | Explore in Territory

// Copyright 2020 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifdef UNSAFE_BUFFERS_BUILD
// TODO(crbug.com/40285824): Remove this and convert code to safer constructs.
#pragma allow_unsafe_buffers
#endif

#include "components/reporting/storage/storage_queue.h"

#include <algorithm>
#include <cstring>
#include <iterator>
#include <limits>
#include <list>
#include <map>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <vector>

#include "base/containers/adapters.h"
#include "base/containers/flat_set.h"
#include "base/files/file.h"
#include "base/files/file_enumerator.h"
#include "base/files/file_path.h"
#include "base/files/file_util.h"
#include "base/functional/bind.h"
#include "base/functional/callback.h"
#include "base/functional/callback_helpers.h"
#include "base/hash/hash.h"
#include "base/logging.h"
#include "base/memory/ptr_util.h"
#include "base/memory/scoped_refptr.h"
#include "base/memory/weak_ptr.h"
#include "base/metrics/histogram_functions.h"
#include "base/rand_util.h"
#include "base/sequence_checker.h"
#include "base/strings/strcat.h"
#include "base/strings/string_number_conversions.h"
#include "base/strings/string_split.h"
#include "base/strings/string_util.h"
#include "base/task/bind_post_task.h"
#include "base/task/sequenced_task_runner.h"
#include "base/task/task_runner.h"
#include "base/task/thread_pool.h"
#include "base/thread_annotations.h"
#include "base/types/expected.h"
#include "base/types/expected_macros.h"
#include "components/reporting/compression/compression_module.h"
#include "components/reporting/encryption/encryption_module_interface.h"
#include "components/reporting/proto/synced/record.pb.h"
#include "components/reporting/resources/resource_managed_buffer.h"
#include "components/reporting/resources/resource_manager.h"
#include "components/reporting/storage/storage_configuration.h"
#include "components/reporting/storage/storage_uploader_interface.h"
#include "components/reporting/util/file.h"
#include "components/reporting/util/refcounted_closure_list.h"
#include "components/reporting/util/reporting_errors.h"
#include "components/reporting/util/status.h"
#include "components/reporting/util/status_macros.h"
#include "components/reporting/util/statusor.h"
#include "components/reporting/util/task_runner_context.h"
#include "crypto/random.h"
#include "crypto/sha2.h"
#include "third_party/protobuf/src/google/protobuf/io/zero_copy_stream_impl_lite.h"

namespace reporting {

// Temporary: enable/disable Storage degradation.
// When enabled: If there is no disk space available at the time of writing a
// new record then the storage queue will try to shed files sequentially from
// lowest to highest priority.
BASE_FEATURE(…);

namespace {

// Metadata file name prefix.
const base::FilePath::CharType METADATA_NAME[] = …);

// The size in bytes that all files and records are rounded to (for privacy:
// make it harder to differ between kinds of records).
constexpr size_t FRAME_SIZE = …;

// Helper functions for FRAME_SIZE alignment support.
size_t RoundUpToFrameSize(size_t size) { … }

// Internal structure of the record header. Must fit in FRAME_SIZE.
struct RecordHeader { … };
}  // namespace

// static
void StorageQueue::Create(
    const QueueOptions& options,
    UploaderInterface::AsyncStartUploaderCb async_start_upload_cb,
    scoped_refptr<EncryptionModuleInterface> encryption_module,
    scoped_refptr<CompressionModule> compression_module,
    base::OnceCallback<void(StatusOr<scoped_refptr<StorageQueue>>)>
        completion_cb) { … }

StorageQueue::StorageQueue(
    scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner,
    const QueueOptions& options,
    UploaderInterface::AsyncStartUploaderCb async_start_upload_cb,
    scoped_refptr<EncryptionModuleInterface> encryption_module,
    scoped_refptr<CompressionModule> compression_module)
    : … { … }

StorageQueue::~StorageQueue() { … }

void StorageQueue::AssignDegradationQueues(
    const std::vector<scoped_refptr<StorageQueue>>& degradation_queues) { … }

Status StorageQueue::Init() { … }

std::optional<std::string> StorageQueue::GetLastRecordDigest() const { … }

Status StorageQueue::SetOrConfirmGenerationId(const base::FilePath& full_name) { … }

StatusOr<int64_t> StorageQueue::GetFileSequenceIdFromPath(
    const base::FilePath& file_name) { … }

StatusOr<int64_t> StorageQueue::AddDataFile(
    const base::FilePath& full_name,
    const base::FileEnumerator::FileInfo& file_info) { … }

Status StorageQueue::EnumerateDataFiles(
    base::flat_set<base::FilePath>* used_files_set) { … }

Status StorageQueue::ScanLastFile() { … }

StatusOr<scoped_refptr<StorageQueue::SingleFile>> StorageQueue::AssignLastFile(
    size_t size) { … }

StatusOr<scoped_refptr<StorageQueue::SingleFile>>
StorageQueue::OpenNewWriteableFile() { … }

Status StorageQueue::WriteHeaderAndBlock(
    std::string_view data,
    std::string_view current_record_digest,
    scoped_refptr<StorageQueue::SingleFile> file) { … }

Status StorageQueue::WriteMetadata(std::string_view current_record_digest) { … }

Status StorageQueue::ReadMetadata(
    const base::FilePath& meta_file_path,
    size_t size,
    int64_t sequencing_id,
    base::flat_set<base::FilePath>* used_files_set) { … }

Status StorageQueue::RestoreMetadata(
    base::flat_set<base::FilePath>* used_files_set) { … }  // namespace reporting

void StorageQueue::DeleteUnusedFiles(
    const base::flat_set<base::FilePath>& used_files_set) const { … }

void StorageQueue::DeleteOutdatedMetadata(int64_t sequencing_id_to_keep) const { … }

// Context for uploading data from the queue in proper sequence.
// Runs on a storage_queue->sequenced_task_runner_
// Makes necessary calls to the provided |UploaderInterface|:
// repeatedly to ProcessRecord/ProcessGap, and Completed at the end.
// Sets references to potentially used files aside, and increments
// active_read_operations_ to make sure confirmation will not trigger
// files deletion. Decrements it upon completion (when this counter
// is zero, RemoveConfirmedData can delete the unused files).
// Returns result through `completion_cb`.
class StorageQueue::ReadContext : public TaskRunnerContext<Status> { … };

class StorageQueue::WriteContext : public TaskRunnerContext<Status> { … };

void StorageQueue::Write(Record record,
                         base::OnceCallback<void(Status)> completion_cb) { … }

Status StorageQueue::ReserveNewRecordDiskSpace(const size_t total_size) { … }

void StorageQueue::RecordsShedding(
    base::span<scoped_refptr<StorageQueue>> degradation_queues,
    scoped_refptr<StorageQueue> writing_storage_queue,
    const size_t space_to_recover,
    base::OnceClosure resume_writing_cb,
    base::OnceClosure writing_failure_cb) { … }

void StorageQueue::RecordsSheddingHelper(const size_t space_to_recover,
                                         base::OnceClosure resume_writing_cb,
                                         base::OnceClosure writing_failure_cb) { … }

bool StorageQueue::FilesShedding(const size_t space_to_recover) { … }

Status StorageQueue::SwitchLastFileIfNotEmpty() { … }

std::map<int64_t, scoped_refptr<StorageQueue::SingleFile>>
StorageQueue::CollectFilesForUpload(int64_t sequencing_id) const { … }

class StorageQueue::ConfirmContext : public TaskRunnerContext<Status> { … };

void StorageQueue::Confirm(SequenceInformation sequence_information,
                           bool force,
                           base::OnceCallback<void(Status)> completion_cb) { … }

Status StorageQueue::RemoveConfirmedData(int64_t sequencing_id) { … }

void StorageQueue::CheckBackUpload(Status status, int64_t next_sequencing_id) { … }

void StorageQueue::PeriodicUpload() { … }

void StorageQueue::Flush(base::OnceCallback<void(Status)> completion_cb) { … }

void StorageQueue::ReleaseAllFileInstances() { … }

void StorageQueue::RegisterCompletionCallback(base::OnceClosure callback) { … }

void StorageQueue::TestInjectErrorsForOperation(
    test::ErrorInjectionHandlerType handler) { … }

//
// SingleFile implementation
//
StatusOr<scoped_refptr<StorageQueue::SingleFile>>
StorageQueue::SingleFile::Create(
    const base::FilePath& filename,
    int64_t size,
    scoped_refptr<ResourceManager> memory_resource,
    scoped_refptr<ResourceManager> disk_space_resource,
    scoped_refptr<RefCountedClosureList> completion_closure_list) { … }

StorageQueue::SingleFile::SingleFile(
    const base::FilePath& filename,
    int64_t size,
    scoped_refptr<ResourceManager> memory_resource,
    scoped_refptr<ResourceManager> disk_space_resource,
    scoped_refptr<RefCountedClosureList> completion_closure_list)
    : … { … }

StorageQueue::SingleFile::~SingleFile() { … }

Status StorageQueue::SingleFile::Open(bool read_only) { … }

void StorageQueue::SingleFile::Close() { … }

void StorageQueue::SingleFile::DeleteWarnIfFailed() { … }

StatusOr<std::string_view> StorageQueue::SingleFile::Read(
    uint32_t pos,
    uint32_t size,
    size_t max_buffer_size,
    bool expect_readonly) { … }

StatusOr<uint32_t> StorageQueue::SingleFile::Append(std::string_view data) { … }
}  // namespace reporting
chromium/components/reporting/storage/storage_queue.cc
