video_effects_processor_webgpu.cc | Explore in Territory

// Copyright 2024 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "services/video_effects/video_effects_processor_webgpu.h"

#include <memory>
#include <numbers>

#include "base/bit_cast.h"
#include "base/functional/bind.h"
#include "base/memory/ptr_util.h"
#include "base/memory/scoped_refptr.h"
#include "base/memory/weak_ptr.h"
#include "base/numerics/safe_conversions.h"
#include "base/sequence_checker.h"
#include "base/strings/stringprintf.h"
#include "gpu/GLES2/gl2extchromium.h"
#include "gpu/command_buffer/client/client_shared_image.h"
#include "gpu/command_buffer/client/context_support.h"
#include "gpu/command_buffer/client/raster_interface.h"
#include "gpu/command_buffer/client/shared_image_interface.h"
#include "gpu/command_buffer/client/webgpu_interface.h"
#include "gpu/command_buffer/common/shared_image_usage.h"
#include "gpu/ipc/client/client_shared_image_interface.h"
#include "gpu/webgpu/callback.h"
#include "media/base/media_switches.h"
#include "media/base/video_types.h"
#include "services/video_effects/public/mojom/video_effects_processor.mojom-shared.h"
#include "services/video_effects/public/mojom/video_effects_processor.mojom.h"
#include "services/viz/public/cpp/gpu/context_provider_command_buffer.h"
#include "third_party/dawn/include/dawn/dawn_proc.h"
#include "third_party/dawn/include/dawn/dawn_proc_table.h"
#include "third_party/dawn/include/dawn/webgpu.h"
#include "third_party/dawn/include/dawn/webgpu_cpp.h"
#include "third_party/dawn/include/dawn/webgpu_cpp_print.h"
#include "third_party/dawn/include/dawn/wire/WireClient.h"

namespace {

scoped_refptr<gpu::ClientSharedImage> CreateSharedImageRGBA(
    gpu::SharedImageInterface* sii,
    const media::mojom::VideoFrameInfo& frame_info,
    gpu::SharedImageUsageSet gpu_usage) { … }

}  // namespace

namespace video_effects {

// Must be kept in sync w/ the struct Uniforms in compute shader below.
// See `VideoEffectsProcessorWebGpu::CreateComputePipeline()`.
struct Uniforms { … };

VideoEffectsProcessorWebGpu::VideoEffectsProcessorWebGpu(
    scoped_refptr<viz::ContextProviderCommandBuffer> context_provider,
    scoped_refptr<viz::RasterContextProvider> raster_interface_context_provider,
    scoped_refptr<gpu::ClientSharedImageInterface> shared_image_interface,
    base::OnceClosure on_unrecoverable_error)
    : … { … }

VideoEffectsProcessorWebGpu::~VideoEffectsProcessorWebGpu() = default;

bool VideoEffectsProcessorWebGpu::Initialize() { … }

// `VideoEffectsProcessorWebGpu::PostProcess()` runs the simple shader on top of
// video frame provided to us from Video Capture Service. We use 3 different IPC
// interfaces to talk to GPU service, hence we need to rely on sync tokens for
// synchronizing different GPU contexts. The high-level strategy is: before
// using a resource on a given interface, wait on a sync token that was
// generated by a context on which the resource originated from. Additionally,
// the token must've been generated _after_ the said resource was created.
//
// Key:
// - SII - SharedImageInterface
// - RI - RasterInterface
// - WGPU - WebGPUInterface
// - s1 - SI created by Video Capture Service for input frame, available in
//      `input_frame_data->get_shared_image_handle()->shared_image`
// - t1 - sync token created by Video Capture Service, available in
//      `input_frame_data->get_shared_image_handle()->sync_token`
// - s2 - SI created by Video Capture Service for output frame, available in
//      `result_frame_data->get_shared_image_handle()->shared_image`
// - t2 - sync token created by Video Capture Service, available in
//      `result_frame_data->get_shared_image_handle()->sync_token`
//
// clang-format off
//
//   Video Capture Service                Video Effects Service
//
//           SII                        SII         RI          WGPU
//            |                          │           │            │
//       s1=CreateSI()                   │           │            │
//       t1=GenSyncToken()               │           │            │
//            │                          │           │            │
//       s2=CreateSI()                   │           │            │
//       t2=GenSyncToken()               │           │            │
//            │                          │           │            │
//            │                   WaitSyncToken(t1)  │            │
//            │                   ImportSI(s1)       │            │
//            │                          |           │            │
//            │                   s3=CreateSI()      │            │
//            │                   t3=GenSyncToken()  │            │
//            │                          |           │            │
//            │                   WaitSyncToken(t2)  │            │
//            │                   ImportSI(s2)       │            │
//            │                          |           │            │
//            │                   s4=CreateSI()      │            │
//            │                   t4=GenSyncToken()  │            │
//            │                          |           │            │
//            │                          |    WaitSyncToken(t1)   │
//            │                          |    WaitSyncToken(t3)   │
//            │                          |           |            │
//            │                          |    s3<-CopySI(s1)      │
//            │                          |    t5=GenSyncToken()   │
//            │                          |           │            │
//            │                          |           │     WaitSyncToken(t3)
//            │                          │           |     WaitSyncToken(t5)
//            │                          │           │     w1=ImportImage(s3)
//            |                          |           |            |
//            |                          |           |     WaitSyncToken(t4)
//            |                          |           |     w2=ImportImage(s4)
//            |                          |           |            |
//            |                          |           |     w2<-RunPipeline(w1)
//            |                          |           |     t6=GenSyncToken()
//            |                          |           |            |
//            |                          |    WaitSyncToken(t2)   |
//            |                          |    WaitSyncToken(t4)   |
//            |                          |    WaitSyncToken(t6)   |
//            |                          |           |            |
//            |                          |    s2<-CopySI(s4)      |
//            |                          |           |            |
//            |                          |    ScheduleCallback()  |
//            |                          |           |            |
//
// clang-format on
//
// The method body is annotated with the comments taken from the diagram above
// to make it more clear which part of the code corresponds to which step from
// the diagram.
void VideoEffectsProcessorWebGpu::PostProcess(
    media::mojom::VideoBufferHandlePtr input_frame_data,
    media::mojom::VideoFrameInfoPtr input_frame_info,
    media::mojom::VideoBufferHandlePtr result_frame_data,
    media::VideoPixelFormat result_pixel_format,
    mojom::VideoEffectsProcessor::PostProcessCallback post_process_cb) { … }

void VideoEffectsProcessorWebGpu::QueryDone(
    GLuint query_id,
    uint64_t trace_id,
    media::mojom::VideoBufferHandlePtr input_frame_data,
    media::mojom::VideoFrameInfoPtr input_frame_info,
    media::mojom::VideoBufferHandlePtr result_frame_data,
    media::VideoPixelFormat result_pixel_format,
    mojom::VideoEffectsProcessor::PostProcessCallback post_process_cb) { … }

void VideoEffectsProcessorWebGpu::OnRequestAdapter(
    wgpu::RequestAdapterStatus status,
    wgpu::Adapter adapter,
    char const* message) { … }

void VideoEffectsProcessorWebGpu::OnRequestDevice(
    wgpu::RequestDeviceStatus status,
    wgpu::Device device,
    char const* message) { … }

wgpu::ComputePipeline VideoEffectsProcessorWebGpu::CreateComputePipeline() { … }

void VideoEffectsProcessorWebGpu::OnDeviceLost(WGPUDeviceLostReason reason,
                                               char const* message) { … }

void VideoEffectsProcessorWebGpu::EnsureFlush() { … }

void VideoEffectsProcessorWebGpu::MaybeCallOnUnrecoverableError() { … }

// static
void VideoEffectsProcessorWebGpu::ErrorCallback(WGPUErrorType type,
                                                char const* message,
                                                void* userdata) { … }

// static
void VideoEffectsProcessorWebGpu::LoggingCallback(WGPULoggingType type,
                                                  char const* message,
                                                  void* userdata) { … }

}  // namespace video_effects
chromium/services/video_effects/video_effects_processor_webgpu.cc
