chromium/v8/src/heap/cppgc/sweeper.cc

// Copyright 2020 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/heap/cppgc/sweeper.h"

#include <algorithm>
#include <atomic>
#include <cstdint>
#include <memory>
#include <optional>
#include <vector>

#include "include/cppgc/platform.h"
#include "src/base/platform/mutex.h"
#include "src/base/platform/time.h"
#include "src/heap/cppgc/free-list.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-base.h"
#include "src/heap/cppgc/heap-config.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/heap-space.h"
#include "src/heap/cppgc/heap-visitor.h"
#include "src/heap/cppgc/memory.h"
#include "src/heap/cppgc/object-poisoner.h"
#include "src/heap/cppgc/object-start-bitmap.h"
#include "src/heap/cppgc/page-memory.h"
#include "src/heap/cppgc/raw-heap.h"
#include "src/heap/cppgc/stats-collector.h"
#include "src/heap/cppgc/task-handle.h"

namespace cppgc::internal {

namespace {

class DeadlineChecker final { … };

enum class MutatorThreadSweepingMode { … };

constexpr const char* ToString(MutatorThreadSweepingMode sweeping_mode) { … }

enum class StickyBits : uint8_t { … };

class ObjectStartBitmapVerifier
    : private HeapVisitor<ObjectStartBitmapVerifier> { … };

class FreeHandlerBase { … };

class DiscardingFreeHandler : public FreeHandlerBase { … };

class RegularFreeHandler : public FreeHandlerBase { … };

template <typename T>
class ThreadSafeStack { … };

struct SweepingState { … };

SpaceStates;

void StickyUnmark(HeapObjectHeader* header, StickyBits sticky_bits) { … }

class InlinedFinalizationBuilderBase { … };

// Builder that finalizes objects and adds freelist entries right away.
template <typename FreeHandler>
class InlinedFinalizationBuilder final : public InlinedFinalizationBuilderBase,
                                         public FreeHandler { … };

// Builder that produces results for deferred processing.
template <typename FreeHandler>
class DeferredFinalizationBuilder final : public FreeHandler { … };

template <typename FinalizationBuilder>
typename FinalizationBuilder::ResultType SweepNormalPage(
    NormalPage* page, PageAllocator& page_allocator, StickyBits sticky_bits) { … }

constexpr BaseSpace* kSweepWithoutSpaceAssignment = …;

enum class EmptyPageHandling { … };

// SweepFinalizer is responsible for heap/space/page finalization. Finalization
// is defined as a step following concurrent sweeping which:
// - calls finalizers;
// - returns (unmaps) empty pages;
// - merges freelists to the space's freelist.
class SweepFinalizer final { … };

class MutatorThreadSweeper final : private HeapVisitor<MutatorThreadSweeper> { … };

class ConcurrentSweepTask final : public cppgc::JobTask,
                                  private HeapVisitor<ConcurrentSweepTask> { … };

// This visitor:
// - clears free lists for all spaces;
// - moves all Heap pages to local Sweeper's state (SpaceStates).
// - ASAN: Poisons all unmarked object payloads.
class PrepareForSweepVisitor final
    : protected HeapVisitor<PrepareForSweepVisitor> { … };

}  // namespace

class Sweeper::SweeperImpl final { … };

Sweeper::Sweeper(HeapBase& heap)
    : … { … }

Sweeper::~Sweeper() = default;

void Sweeper::Start(SweepingConfig config) { … }

bool Sweeper::FinishIfRunning() { … }

void Sweeper::FinishIfOutOfWork() { … }

void Sweeper::WaitForConcurrentSweepingForTesting() { … }

bool Sweeper::SweepForAllocationIfRunning(NormalPageSpace* space, size_t size,
                                          v8::base::TimeDelta max_duration) { … }

bool Sweeper::IsSweepingOnMutatorThread() const { … }

bool Sweeper::IsSweepingInProgress() const { … }

bool Sweeper::PerformSweepOnMutatorThread(v8::base::TimeDelta max_duration,
                                          StatsCollector::ScopeId scope_id) { … }

Sweeper::SweepingOnMutatorThreadObserver::SweepingOnMutatorThreadObserver(
    Sweeper& sweeper)
    : … { … }

Sweeper::SweepingOnMutatorThreadObserver::~SweepingOnMutatorThreadObserver() { … }

}  // namespace cppgc::internal