camera_video_frame_handler.cc | Explore in Territory

// Copyright 2022 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "components/capture_mode/camera_video_frame_handler.h"

#include "base/check.h"
#include "base/functional/bind.h"
#include "base/functional/callback.h"
#include "base/functional/callback_helpers.h"
#include "base/logging.h"
#include "base/memory/scoped_refptr.h"
#include "base/memory/shared_memory_mapping.h"
#include "base/memory/unsafe_shared_memory_region.h"
#include "base/system/sys_info.h"
#include "components/viz/common/gpu/context_lost_observer.h"
#include "components/viz/common/gpu/context_provider.h"
#include "gpu/command_buffer/client/client_shared_image.h"
#include "gpu/command_buffer/client/shared_image_interface.h"
#include "gpu/command_buffer/common/shared_image_usage.h"
#include "media/base/media_switches.h"
#include "media/base/video_frame.h"
#include "media/base/video_types.h"
#include "media/capture/video_capture_types.h"
#include "mojo/public/cpp/system/buffer.h"
#include "ui/compositor/compositor.h"
#include "ui/gfx/buffer_types.h"
#include "ui/gfx/geometry/size.h"
#include "ui/gfx/gpu_memory_buffer.h"

#if BUILDFLAG(IS_WIN)
#include "gpu/command_buffer/common/shared_image_capabilities.h"
#endif

#if BUILDFLAG(IS_MAC)
#include "media/capture/video/apple/video_capture_device_factory_apple.h"
#endif

namespace capture_mode {

namespace {

// The `kGpuMemoryBuffer` type is requested only when running on an actual
// device. This allows force-requesting them when testing in which case
// SharedMemory GMBs are used.
bool g_force_use_gpu_memory_buffer_for_test = …;

// A constant flag that describes which APIs the shared images created
// for the video frames will be used with. They will be read via the raster
// interface (which will be going over GLES2 if OOP-R is not enabled), sent
// to the display compositor, and may be used as overlays.
constexpr gpu::SharedImageUsageSet kSharedImageUsage = …;

viz::SharedImageFormat GetSharedImageFormat() { … }

#if BUILDFLAG(IS_CHROMEOS)
// Adjusts the requested video capture `params` depending on whether we're
// running on an actual device or the linux-chromeos build.
void AdjustParamsForCurrentConfig(media::VideoCaptureParams* params) {
  DCHECK(params);

  // The default params are good enough when running on linux-chromeos.
  if (!base::SysInfo::IsRunningOnChromeOS() &&
      !g_force_use_gpu_memory_buffer_for_test) {
    DCHECK_EQ(params->buffer_type,
              media::VideoCaptureBufferType::kSharedMemory);
    return;
  }

  // On an actual device, the camera HAL only supports NV12 pixel formats in a
  // GPU memory buffer.
  params->requested_format.pixel_format = media::PIXEL_FORMAT_NV12;
  params->buffer_type = media::VideoCaptureBufferType::kGpuMemoryBuffer;
}
#endif

bool IsFatalError(media::VideoCaptureError error) { … }

#if BUILDFLAG(IS_MAC) || BUILDFLAG(IS_WIN)
bool IsGpuRasterizationSupported(ui::ContextFactory* context_factory) {
  DCHECK(context_factory);
  auto provider = context_factory->SharedMainThreadRasterContextProvider();
  return provider && provider->ContextCapabilities().gpu_rasterization;
}
#endif

#if BUILDFLAG(IS_WIN)
bool IsD3DSharedImageSupported(ui::ContextFactory* context_factory) {
  DCHECK(context_factory);
  if (auto provider =
          context_factory->SharedMainThreadRasterContextProvider()) {
    auto* shared_image_interface = provider->SharedImageInterface();
    return shared_image_interface &&
           shared_image_interface->GetCapabilities().shared_image_d3d;
  }
  return false;
}

void AdjustWinParamsForCurrentConfig(media::VideoCaptureParams* params,
                                     ui::ContextFactory* context_factory) {
  if (media::IsMediaFoundationD3D11VideoCaptureEnabled() &&
      params->requested_format.pixel_format == media::PIXEL_FORMAT_NV12) {
    if (IsGpuRasterizationSupported(context_factory)) {
      params->buffer_type = media::VideoCaptureBufferType::kGpuMemoryBuffer;
    } else {
      params->requested_format.pixel_format = media::PIXEL_FORMAT_I420;
    }
  }
}
#endif

#if BUILDFLAG(IS_MAC)
void AdjustMacParamsForCurrentConfig(media::VideoCaptureParams* params,
                                     const std::string& device_id) {
  if (media::ShouldEnableGpuMemoryBuffer(device_id)) {
    params->buffer_type = media::VideoCaptureBufferType::kGpuMemoryBuffer;
  }
}
#endif

// -----------------------------------------------------------------------------
// SharedMemoryBufferHandleHolder:

// Defines an implementation for a `BufferHandleHolder` that can extract a video
// frame that is backed by a `kSharedMemory` buffer type.
class SharedMemoryBufferHandleHolder : public BufferHandleHolder { … };

// -----------------------------------------------------------------------------
// GpuMemoryBufferHandleHolder:

// Defines an implementation for a `BufferHandleHolder` that can extract a video
// frame that is backed by a `kGpuMemoryBuffer` buffer type.
class GpuMemoryBufferHandleHolder : public BufferHandleHolder,
                                    public viz::ContextLostObserver { … };

#if BUILDFLAG(IS_MAC)
// -----------------------------------------------------------------------------
// MacGpuMemoryBufferHandleHolder:

// Defines an implementation for a `BufferHandleHolder` that can extract a video
// frame that is backed by an IOSurface on mac. It allows the possibility to map
// the memory and produce a software video frame.
class MacGpuMemoryBufferHandleHolder : public BufferHandleHolder {
 public:
  explicit MacGpuMemoryBufferHandleHolder(
      media::mojom::VideoBufferHandlePtr buffer_handle,
      ui::ContextFactory* context_factory)
      : context_factory_(context_factory),
        gmb_holder_(std::move(buffer_handle), context_factory) {}
  MacGpuMemoryBufferHandleHolder(const MacGpuMemoryBufferHandleHolder&) =
      delete;
  MacGpuMemoryBufferHandleHolder& operator=(
      const MacGpuMemoryBufferHandleHolder&) = delete;
  ~MacGpuMemoryBufferHandleHolder() override = default;

  // BufferHandleHolder:
  scoped_refptr<media::VideoFrame> OnFrameReadyInBuffer(
      video_capture::mojom::ReadyFrameInBufferPtr buffer) override {
    // TODO: This isn't required to be checked every frame and only changes
    // after context loss.
    if (IsGpuRasterizationSupported(context_factory_)) {
      return gmb_holder_.OnFrameReadyInBuffer(std::move(buffer));
    }
    auto frame = media::VideoFrame::WrapUnacceleratedIOSurface(
        gmb_holder_.GetGpuMemoryBufferHandle().Clone(),
        buffer->frame_info->visible_rect, buffer->frame_info->timestamp);

    if (!frame) {
      VLOG(0) << "Failed to create a video frame.";
    }
    return frame;
  }

 private:
  const raw_ptr<ui::ContextFactory> context_factory_;

  // A GMB handle holder used to extract and return the video frame when Gpu
  // rasterization is supported.
  GpuMemoryBufferHandleHolder gmb_holder_;
};
#endif

#if BUILDFLAG(IS_WIN)
// -----------------------------------------------------------------------------
// WinGpuMemoryBufferHandleHolder:

// Defines an implementation for a `BufferHandleHolder` that can extract a video
// frame that is backed by a `kGpuMemoryBuffer` buffer type on Win. It allows
// the possibility to map the memory and produce a software video frame.
class WinGpuMemoryBufferHandleHolder : public BufferHandleHolder {
 public:
  explicit WinGpuMemoryBufferHandleHolder(
      base::RepeatingClosure require_mapped_frame_callback,
      media::mojom::VideoBufferHandlePtr buffer_handle,
      ui::ContextFactory* context_factory)
      : context_factory_(context_factory),
        gmb_holder_(std::move(buffer_handle), context_factory),
        sh_mem_holder_(gmb_holder_.TakeGpuMemoryBufferHandleRegion()),
        require_mapped_frame_callback_(
            std::move(require_mapped_frame_callback)) {
    CHECK_EQ(gmb_holder_.GetGpuMemoryBufferHandle().type,
             gfx::DXGI_SHARED_HANDLE);
    CHECK(require_mapped_frame_callback_);
  }
  WinGpuMemoryBufferHandleHolder(const WinGpuMemoryBufferHandleHolder&) =
      delete;
  WinGpuMemoryBufferHandleHolder& operator=(
      const WinGpuMemoryBufferHandleHolder&) = delete;
  ~WinGpuMemoryBufferHandleHolder() override = default;

  // BufferHandleHolder:
  scoped_refptr<media::VideoFrame> OnFrameReadyInBuffer(
      video_capture::mojom::ReadyFrameInBufferPtr buffer) override {
    // TODO: This isn't required to be checked every frame and only changes
    // after context loss.
    if (IsGpuRasterizationSupported(context_factory_) &&
        IsD3DSharedImageSupported(context_factory_)) {
      return gmb_holder_.OnFrameReadyInBuffer(std::move(buffer));
    }
    if (buffer->frame_info->is_premapped) {
      return sh_mem_holder_.OnFrameReadyInBuffer(std::move(buffer));
    }
    require_mapped_frame_callback_.Run();
    return {};
  }

 private:
  const raw_ptr<ui::ContextFactory> context_factory_;

  // A GMB handle holder used to extract and return the video frame when both
  // Gpu rasterization and D3dSharedImage are supported.
  GpuMemoryBufferHandleHolder gmb_holder_;

  // A shared memory handle holder used to extract and return the video frame
  // when frame is pre-mapped.
  SharedMemoryBufferHandleHolder sh_mem_holder_;

  // A callback that is used to request pre-mapped frames.
  const base::RepeatingClosure require_mapped_frame_callback_;
};
#endif

// Notifies the passed `VideoFrameAccessHandler` that the handler is done using
// the buffer.
void OnFrameGone(
    mojo::SharedRemote<video_capture::mojom::VideoFrameAccessHandler>
        video_frame_access_handler_remote,
    const int buffer_id) { … }

}  // namespace

// -----------------------------------------------------------------------------
// BufferHandleHolder:

BufferHandleHolder::~BufferHandleHolder() = default;

// static
std::unique_ptr<BufferHandleHolder> BufferHandleHolder::Create(
    media::mojom::VideoBufferHandlePtr buffer_handle,
    ui::ContextFactory* context_factory,
    base::RepeatingClosure require_mapped_frame_callback) { … }

// -----------------------------------------------------------------------------
// CameraVideoFrameHandler:

CameraVideoFrameHandler::CameraVideoFrameHandler(
    ui::ContextFactory* context_factory,
    mojo::Remote<video_capture::mojom::VideoSource> camera_video_source,
    const media::VideoCaptureFormat& capture_format,
    const std::string& device_id)
    : … { … }

CameraVideoFrameHandler::~CameraVideoFrameHandler() = default;

void CameraVideoFrameHandler::StartHandlingFrames(Delegate* delegate) { … }

void CameraVideoFrameHandler::Close(base::OnceClosure close_complete_callback) { … }

void CameraVideoFrameHandler::OnCaptureConfigurationChanged() { … }

void CameraVideoFrameHandler::OnNewBuffer(
    int buffer_id,
    media::mojom::VideoBufferHandlePtr buffer_handle) { … }

void CameraVideoFrameHandler::OnFrameAccessHandlerReady(
    mojo::PendingRemote<video_capture::mojom::VideoFrameAccessHandler>
        pending_frame_access_handler) { … }

void CameraVideoFrameHandler::OnFrameReadyInBuffer(
    video_capture::mojom::ReadyFrameInBufferPtr buffer) { … }

void CameraVideoFrameHandler::OnBufferRetired(int buffer_id) { … }

void CameraVideoFrameHandler::OnError(media::VideoCaptureError error) { … }

void CameraVideoFrameHandler::OnFrameDropped(
    media::VideoCaptureFrameDropReason reason) { … }

void CameraVideoFrameHandler::OnNewSubCaptureTargetVersion(
    uint32_t sub_capture_target_version) { … }

void CameraVideoFrameHandler::OnFrameWithEmptyRegionCapture() { … }

void CameraVideoFrameHandler::OnLog(const std::string& message) { … }

void CameraVideoFrameHandler::OnStarted() { … }

void CameraVideoFrameHandler::OnStartedUsingGpuDecode() { … }

void CameraVideoFrameHandler::OnStopped() { … }

// static
void CameraVideoFrameHandler::SetForceUseGpuMemoryBufferForTest(bool value) { … }

void CameraVideoFrameHandler::OnSubscriptionCreationResult(
    video_capture::mojom::CreatePushSubscriptionResultCodePtr result_code,
    const media::VideoCaptureParams& actual_params) { … }

void CameraVideoFrameHandler::OnVideoFrameGone(int buffer_id) { … }

void CameraVideoFrameHandler::OnFatalErrorOrDisconnection() { … }

void CameraVideoFrameHandler::RequireMappedFrame() { … }

}  // namespace capture_mode
chromium/components/capture_mode/camera_video_frame_handler.cc
