chromium/v8/src/wasm/module-compiler.cc

// Copyright 2017 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/wasm/module-compiler.h"

#include <algorithm>
#include <atomic>
#include <memory>
#include <queue>

#include "src/api/api-inl.h"
#include "src/base/enum-set.h"
#include "src/base/platform/mutex.h"
#include "src/base/platform/semaphore.h"
#include "src/base/platform/time.h"
#include "src/codegen/compiler.h"
#include "src/compiler/wasm-compiler.h"
#include "src/debug/debug.h"
#include "src/handles/global-handles-inl.h"
#include "src/logging/counters-scopes.h"
#include "src/logging/metrics.h"
#include "src/tracing/trace-event.h"
#include "src/wasm/code-space-access.h"
#include "src/wasm/compilation-environment-inl.h"
#include "src/wasm/jump-table-assembler.h"
#include "src/wasm/module-decoder.h"
#include "src/wasm/pgo.h"
#include "src/wasm/std-object-sizes.h"
#include "src/wasm/streaming-decoder.h"
#include "src/wasm/wasm-code-manager.h"
#include "src/wasm/wasm-engine.h"
#include "src/wasm/wasm-feature-flags.h"
#include "src/wasm/wasm-import-wrapper-cache.h"
#include "src/wasm/wasm-js.h"
#include "src/wasm/wasm-limits.h"
#include "src/wasm/wasm-objects-inl.h"
#include "src/wasm/wasm-result.h"
#include "src/wasm/wasm-serialization.h"

#define TRACE_COMPILE …

#define TRACE_STREAMING …

#define TRACE_LAZY …

namespace v8::internal::wasm {

namespace {

enum class CompileStrategy : uint8_t { … };

class CompilationStateImpl;
class CompilationUnitBuilder;

class V8_NODISCARD BackgroundCompileScope { … };

enum CompilationTier { … };

// A set of work-stealing queues (vectors of units). Each background compile
// task owns one of the queues and steals from all others once its own queue
// runs empty.
class CompilationUnitQueues { … };

size_t CompilationUnitQueues::EstimateCurrentMemoryConsumption() const { … }

bool CompilationUnitQueues::Queue::ShouldPublish(
    int num_processed_units) const { … }

// The {CompilationStateImpl} keeps track of the compilation state of the
// owning NativeModule, i.e. which functions are left to be compiled.
// It contains a task manager to allow parallel and asynchronous background
// compilation of functions.
// Its public interface {CompilationState} lives in compilation-environment.h.
class CompilationStateImpl { … };

CompilationStateImpl* Impl(CompilationState* compilation_state) { … }
const CompilationStateImpl* Impl(const CompilationState* compilation_state) { … }

CompilationStateImpl* BackgroundCompileScope::compilation_state() const { … }

size_t CompilationStateImpl::EstimateCurrentMemoryConsumption() const { … }

bool BackgroundCompileScope::cancelled() const { … }

}  // namespace

//////////////////////////////////////////////////////
// PIMPL implementation of {CompilationState}.

CompilationState::~CompilationState() { … }

void CompilationState::InitCompileJob() { … }

void CompilationState::CancelCompilation() { … }

void CompilationState::CancelInitialCompilation() { … }

void CompilationState::SetError() { … }

void CompilationState::SetWireBytesStorage(
    std::shared_ptr<WireBytesStorage> wire_bytes_storage) { … }

std::shared_ptr<WireBytesStorage> CompilationState::GetWireBytesStorage()
    const { … }

void CompilationState::AddCallback(
    std::unique_ptr<CompilationEventCallback> callback) { … }

void CompilationState::TierUpAllFunctions() { … }

void CompilationState::AllowAnotherTopTierJob(uint32_t func_index) { … }

void CompilationState::AllowAnotherTopTierJobForAllFunctions() { … }

void CompilationState::InitializeAfterDeserialization(
    base::Vector<const int> lazy_functions,
    base::Vector<const int> eager_functions) { … }

bool CompilationState::failed() const { … }

bool CompilationState::baseline_compilation_finished() const { … }

void CompilationState::set_compilation_id(int compilation_id) { … }

DynamicTiering CompilationState::dynamic_tiering() const { … }

size_t CompilationState::EstimateCurrentMemoryConsumption() const { … }

std::vector<WasmCode*> CompilationState::PublishCode(
    base::Vector<std::unique_ptr<WasmCode>> unpublished_code) { … }

// static
std::unique_ptr<CompilationState> CompilationState::New(
    const std::shared_ptr<NativeModule>& native_module,
    std::shared_ptr<Counters> async_counters, DynamicTiering dynamic_tiering) { … }

// End of PIMPL implementation of {CompilationState}.
//////////////////////////////////////////////////////

namespace {

ExecutionTier ApplyHintToExecutionTier(WasmCompilationHintTier hint,
                                       ExecutionTier default_tier) { … }

const WasmCompilationHint* GetCompilationHint(const WasmModule* module,
                                              uint32_t func_index) { … }

CompileStrategy GetCompileStrategy(const WasmModule* module,
                                   WasmEnabledFeatures enabled_features,
                                   uint32_t func_index, bool lazy_module) { … }

struct ExecutionTierPair { … };

// Pass the debug state as a separate parameter to avoid data races: the debug
// state may change between its use here and its use at the call site. To have
// a consistent view on the debug state, the caller reads the debug state once
// and then passes it to this function.
ExecutionTierPair GetDefaultTiersPerModule(NativeModule* native_module,
                                           DynamicTiering dynamic_tiering,
                                           DebugState is_in_debug_state,
                                           bool lazy_module) { … }

ExecutionTierPair GetLazyCompilationTiers(NativeModule* native_module,
                                          uint32_t func_index,
                                          DebugState is_in_debug_state) { … }

// The {CompilationUnitBuilder} builds compilation units and stores them in an
// internal buffer. The buffer is moved into the working queue of the
// {CompilationStateImpl} when {Commit} is called.
class CompilationUnitBuilder { … };

DecodeResult ValidateSingleFunction(Zone* zone, const WasmModule* module,
                                    int func_index,
                                    base::Vector<const uint8_t> code,
                                    WasmEnabledFeatures enabled_features) { … }

enum OnlyLazyFunctions : bool { … };

bool IsLazyModule(const WasmModule* module) { … }

class CompileLazyTimingScope { … };

}  // namespace

bool CompileLazy(Isolate* isolate,
                 Tagged<WasmTrustedInstanceData> trusted_instance_data,
                 int func_index) { … }

void ThrowLazyCompilationError(Isolate* isolate,
                               const NativeModule* native_module,
                               int func_index) { … }

// The main purpose of this class is to copy the feedback vectors that live in
// `FixedArray`s on the JavaScript heap to a C++ datastructure on the `module`
// that is accessible to the background compilation threads.
// While we are at it, we also do some light processing here, e.g., mapping the
// feedback to functions, identified by their function index, and filtering out
// feedback for calls to imported functions (which we currently don't inline).
class TransitiveTypeFeedbackProcessor { … };

class FeedbackMaker { … };

void TransitiveTypeFeedbackProcessor::ProcessFunction(int func_index) { … }

void TriggerTierUp(Isolate* isolate,
                   Tagged<WasmTrustedInstanceData> trusted_instance_data,
                   int func_index) { … }

void TierUpNowForTesting(Isolate* isolate,
                         Tagged<WasmTrustedInstanceData> trusted_instance_data,
                         int func_index) { … }

void TierUpAllForTesting(
    Isolate* isolate, Tagged<WasmTrustedInstanceData> trusted_instance_data) { … }

namespace {

bool IsI16Array(wasm::ValueType type, const WasmModule* module) { … }

bool IsI8Array(wasm::ValueType type, const WasmModule* module,
               bool allow_nullable) { … }

// Returns the start offset of a given import, for use in error messages.
// The module_name payload is preceded by an i32v giving its length. That i32v
// is preceded by another i32v, which is either a type index (specifying the
// type of the previous import) or the imports count (in case of the first
// import). So we scan backwards as long as we find non-last LEB bytes there.
uint32_t ImportStartOffset(base::Vector<const uint8_t> wire_bytes,
                           uint32_t module_name_start) { … }

}  // namespace

// Validates the signatures of recognized compile-time imports, and stores
// them on the {module}'s {well_known_imports} list.
WasmError ValidateAndSetBuiltinImports(const WasmModule* module,
                                       base::Vector<const uint8_t> wire_bytes,
                                       const CompileTimeImports& imports) { … }

namespace {

void RecordStats(Tagged<Code> code, Counters* counters) { … }

enum CompilationExecutionResult : int8_t { … };

const char* GetCompilationEventName(const WasmCompilationUnit& unit,
                                    const CompilationEnv& env) { … }

constexpr uint8_t kMainTaskId = …;

// Run by the {BackgroundCompileJob} (on any thread).
CompilationExecutionResult ExecuteCompilationUnits(
    std::weak_ptr<NativeModule> native_module, Counters* counters,
    JobDelegate* delegate, CompilationTier tier) { … }

// Returns the number of units added.
int AddExportWrapperUnits(Isolate* isolate, NativeModule* native_module,
                          CompilationUnitBuilder* builder) { … }

std::unique_ptr<CompilationUnitBuilder> InitializeCompilation(
    Isolate* isolate, NativeModule* native_module,
    ProfileInformation* pgo_info) { … }

bool MayCompriseLazyFunctions(const WasmModule* module,
                              WasmEnabledFeatures enabled_features) { … }

class CompilationTimeCallback : public CompilationEventCallback { … };

WasmError ValidateFunctions(const WasmModule* module,
                            base::Vector<const uint8_t> wire_bytes,
                            WasmEnabledFeatures enabled_features,
                            OnlyLazyFunctions only_lazy_functions) { … }

WasmError ValidateFunctions(const NativeModule& native_module,
                            OnlyLazyFunctions only_lazy_functions) { … }

void CompileNativeModule(Isolate* isolate,
                         v8::metrics::Recorder::ContextId context_id,
                         ErrorThrower* thrower,
                         std::shared_ptr<NativeModule> native_module,
                         ProfileInformation* pgo_info) { … }

class BaseCompileJSToWasmWrapperJob : public JobTask { … };

class AsyncCompileJSToWasmWrapperJob final
    : public BaseCompileJSToWasmWrapperJob { … };

class BackgroundCompileJob final : public JobTask { … };

}  // namespace

std::shared_ptr<NativeModule> CompileToNativeModule(
    Isolate* isolate, WasmEnabledFeatures enabled_features,
    CompileTimeImports compile_imports, ErrorThrower* thrower,
    std::shared_ptr<const WasmModule> module, ModuleWireBytes wire_bytes,
    int compilation_id, v8::metrics::Recorder::ContextId context_id,
    ProfileInformation* pgo_info) { … }

AsyncCompileJob::AsyncCompileJob(
    Isolate* isolate, WasmEnabledFeatures enabled_features,
    CompileTimeImports compile_imports, base::OwnedVector<const uint8_t> bytes,
    DirectHandle<Context> context,
    DirectHandle<NativeContext> incumbent_context, const char* api_method_name,
    std::shared_ptr<CompilationResultResolver> resolver, int compilation_id)
    : … { … }

void AsyncCompileJob::Start() { … }

void AsyncCompileJob::Abort() { … }

// {ValidateFunctionsStreamingJobData} holds information that is shared between
// the {AsyncStreamingProcessor} and the {ValidateFunctionsStreamingJob}. It
// lives in the {AsyncStreamingProcessor} and is updated from both classes.
struct ValidateFunctionsStreamingJobData { … };

class ValidateFunctionsStreamingJob final : public JobTask { … };

class AsyncStreamingProcessor final : public StreamingProcessor { … };

std::shared_ptr<StreamingDecoder> AsyncCompileJob::CreateStreamingDecoder() { … }

AsyncCompileJob::~AsyncCompileJob() { … }

void AsyncCompileJob::CreateNativeModule(
    std::shared_ptr<const WasmModule> module, size_t code_size_estimate) { … }

bool AsyncCompileJob::GetOrCreateNativeModule(
    std::shared_ptr<const WasmModule> module, size_t code_size_estimate) { … }

void AsyncCompileJob::PrepareRuntimeObjects() { … }

// This function assumes that it is executed in a HandleScope, and that a
// context is set on the isolate.
void AsyncCompileJob::FinishCompile(bool is_after_cache_hit) { … }

void AsyncCompileJob::Failed() { … }

class AsyncCompileJob::CompilationStateCallback
    : public CompilationEventCallback { … };

// A closure to run a compilation step (either as foreground or background
// task) and schedule the next step(s), if any.
class AsyncCompileJob::CompileStep { … };

class AsyncCompileJob::CompileTask : public CancelableTask { … };

void AsyncCompileJob::StartForegroundTask() { … }

void AsyncCompileJob::ExecuteForegroundTaskImmediately() { … }

void AsyncCompileJob::CancelPendingForegroundTask() { … }

void AsyncCompileJob::StartBackgroundTask() { … }

template <typename Step,
          AsyncCompileJob::UseExistingForegroundTask use_existing_fg_task,
          typename... Args>
void AsyncCompileJob::DoSync(Args&&... args) { … }

template <typename Step, typename... Args>
void AsyncCompileJob::DoImmediately(Args&&... args) { … }

template <typename Step, typename... Args>
void AsyncCompileJob::DoAsync(Args&&... args) { … }

template <typename Step, typename... Args>
void AsyncCompileJob::NextStep(Args&&... args) { … }

//==========================================================================
// Step 1: (async) Decode the module.
//==========================================================================
class AsyncCompileJob::DecodeModule : public AsyncCompileJob::CompileStep { … };

//==========================================================================
// Step 2 (sync): Create heap-allocated data and start compilation.
//==========================================================================
class AsyncCompileJob::PrepareAndStartCompile : public CompileStep { … };

//==========================================================================
// Step 3 (sync): Compilation finished.
//==========================================================================
class AsyncCompileJob::FinishCompilation : public CompileStep { … };

//==========================================================================
// Step 4 (sync): Decoding or compilation failed.
//==========================================================================
class AsyncCompileJob::Fail : public CompileStep { … };

void AsyncCompileJob::FinishSuccessfully() { … }

AsyncStreamingProcessor::AsyncStreamingProcessor(AsyncCompileJob* job)
    : … { … }

// Process the module header.
bool AsyncStreamingProcessor::ProcessModuleHeader(
    base::Vector<const uint8_t> bytes) { … }

// Process all sections except for the code section.
bool AsyncStreamingProcessor::ProcessSection(SectionCode section_code,
                                             base::Vector<const uint8_t> bytes,
                                             uint32_t offset) { … }

// Start the code section.
bool AsyncStreamingProcessor::ProcessCodeSectionHeader(
    int num_functions, uint32_t functions_mismatch_error_offset,
    std::shared_ptr<WireBytesStorage> wire_bytes_storage,
    int code_section_start, int code_section_length) { … }

// Process a function body.
bool AsyncStreamingProcessor::ProcessFunctionBody(
    base::Vector<const uint8_t> bytes, uint32_t offset) { … }

void AsyncStreamingProcessor::CommitCompilationUnits() { … }

void AsyncStreamingProcessor::OnFinishedChunk() { … }

// Finish the processing of the stream.
void AsyncStreamingProcessor::OnFinishedStream(
    base::OwnedVector<const uint8_t> bytes, bool after_error) { … }

void AsyncStreamingProcessor::OnAbort() { … }

bool AsyncStreamingProcessor::Deserialize(
    base::Vector<const uint8_t> module_bytes,
    base::Vector<const uint8_t> wire_bytes) { … }

CompilationStateImpl::CompilationStateImpl(
    const std::shared_ptr<NativeModule>& native_module,
    std::shared_ptr<Counters> async_counters, DynamicTiering dynamic_tiering)
    : … { … }

void CompilationStateImpl::InitCompileJob() { … }

void CompilationStateImpl::CancelCompilation(
    CompilationStateImpl::CancellationPolicy cancellation_policy) { … }

bool CompilationStateImpl::cancelled() const { … }

void CompilationStateImpl::ApplyCompilationHintToInitialProgress(
    const WasmCompilationHint& hint, size_t hint_idx) { … }

void CompilationStateImpl::ApplyPgoInfoToInitialProgress(
    ProfileInformation* pgo_info) { … }

void CompilationStateImpl::ApplyPgoInfoLate(ProfileInformation* pgo_info) { … }

void CompilationStateImpl::InitializeCompilationProgress(
    int num_import_wrappers, int num_export_wrappers,
    ProfileInformation* pgo_info) { … }

void CompilationStateImpl::AddCompilationUnitInternal(
    CompilationUnitBuilder* builder, int function_index,
    uint8_t function_progress) { … }

void CompilationStateImpl::InitializeCompilationUnits(
    std::unique_ptr<CompilationUnitBuilder> builder) { … }

void CompilationStateImpl::AddCompilationUnit(CompilationUnitBuilder* builder,
                                              int func_index) { … }

void CompilationStateImpl::InitializeCompilationProgressAfterDeserialization(
    base::Vector<const int> lazy_functions,
    base::Vector<const int> eager_functions) { … }

void CompilationStateImpl::AddCallback(
    std::unique_ptr<CompilationEventCallback> callback) { … }

void CompilationStateImpl::CommitCompilationUnits(
    base::Vector<WasmCompilationUnit> baseline_units,
    base::Vector<WasmCompilationUnit> top_tier_units,
    base::Vector<JSToWasmWrapperCompilationUnit> js_to_wasm_wrapper_units) { … }

void CompilationStateImpl::CommitTopTierCompilationUnit(
    WasmCompilationUnit unit) { … }

void CompilationStateImpl::AddTopTierPriorityCompilationUnit(
    WasmCompilationUnit unit, size_t priority) { … }

JSToWasmWrapperCompilationUnit*
CompilationStateImpl::GetJSToWasmWrapperCompilationUnit(size_t index) { … }

void CompilationStateImpl::FinalizeJSToWasmWrappers(Isolate* isolate,
                                                    const WasmModule* module) { … }

CompilationUnitQueues::Queue* CompilationStateImpl::GetQueueForCompileTask(
    int task_id) { … }

std::optional<WasmCompilationUnit> CompilationStateImpl::GetNextCompilationUnit(
    CompilationUnitQueues::Queue* queue, CompilationTier tier) { … }

void CompilationStateImpl::OnFinishedUnits(
    base::Vector<WasmCode*> code_vector) { … }

void CompilationStateImpl::OnFinishedJSToWasmWrapperUnits() { … }

namespace {
class TriggerCodeCachingAfterTimeoutTask : public v8::Task { … };
}  // namespace

void CompilationStateImpl::TriggerOutstandingCallbacks() { … }

void CompilationStateImpl::TriggerCallbacks(
    base::EnumSet<CompilationEvent> events) { … }

void CompilationStateImpl::TriggerCachingAfterTimeout() { … }

void CompilationStateImpl::OnCompilationStopped(WasmDetectedFeatures detected) { … }

void CompilationStateImpl::PublishDetectedFeaturesAfterCompilation(
    Isolate* isolate) { … }

void CompilationStateImpl::PublishCompilationResults(
    std::vector<std::unique_ptr<WasmCode>> unpublished_code) { … }

std::vector<WasmCode*> CompilationStateImpl::PublishCode(
    base::Vector<std::unique_ptr<WasmCode>> code) { … }

void CompilationStateImpl::SchedulePublishCompilationResults(
    std::vector<std::unique_ptr<WasmCode>> unpublished_code,
    CompilationTier tier) { … }

size_t CompilationStateImpl::NumOutstandingCompilations(
    CompilationTier tier) const { … }

void CompilationStateImpl::SetError() { … }

void CompilationStateImpl::WaitForCompilationEvent(
    CompilationEvent expect_event) { … }

void CompilationStateImpl::TierUpAllFunctions() { … }

namespace {
using JSToWasmWrapperSet = std::unordered_set<uint32_t>;
using JSToWasmWrapperUnitVector = std::vector<
    std::pair<uint32_t, std::unique_ptr<JSToWasmWrapperCompilationUnit>>>;

class CompileJSToWasmWrapperJob final : public BaseCompileJSToWasmWrapperJob { … };
}  // namespace

void CompileJsToWasmWrappers(Isolate* isolate, const WasmModule* module) { … }

WasmCode* CompileImportWrapperForTest(NativeModule* native_module,
                                      Counters* counters, ImportCallKind kind,
                                      const FunctionSig* sig,
                                      uint32_t canonical_type_index,
                                      int expected_arity, Suspend suspend) { … }

}  // namespace v8::internal::wasm

#undef TRACE_COMPILE
#undef TRACE_STREAMING
#undef TRACE_LAZY