chromium/media/base/video_frame.cc

// Copyright 2012 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifdef UNSAFE_BUFFERS_BUILD
// TODO(crbug.com/40285824): Remove this and convert code to safer constructs.
#pragma allow_unsafe_buffers
#endif

#include "media/base/video_frame.h"

#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>

#include <algorithm>
#include <atomic>
#include <climits>
#include <numeric>
#include <string_view>
#include <utility>

#include "base/bits.h"
#include "base/functional/bind.h"
#include "base/functional/callback_helpers.h"
#include "base/logging.h"
#include "base/numerics/safe_conversions.h"
#include "base/process/memory.h"
#include "base/strings/stringprintf.h"
#include "base/synchronization/lock.h"
#include "build/build_config.h"
#include "components/viz/common/resources/shared_image_format_utils.h"
#include "media/base/color_plane_layout.h"
#include "media/base/format_utils.h"
#include "media/base/limits.h"
#include "media/base/media_switches.h"
#include "media/base/timestamp_constants.h"
#include "media/base/video_util.h"
#include "ui/gfx/buffer_format_util.h"
#include "ui/gfx/geometry/point.h"
#include "ui/gfx/gpu_memory_buffer.h"
#if BUILDFLAG(IS_APPLE)
#include "ui/gfx/mac/io_surface.h"
#endif

namespace media {

namespace {

VideoFrame::ID GetNextID() { … }

// Helper to provide gfx::Rect::Intersect() as an expression.
gfx::Rect Intersection(gfx::Rect a, const gfx::Rect& b) { … }

void ReleaseMailboxAndDropGpuMemoryBuffer(
    VideoFrame::ReleaseMailboxCB cb,
    const gpu::SyncToken& sync_token,
    std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer) { … }

VideoFrame::ReleaseMailboxAndGpuMemoryBufferCB WrapReleaseMailboxCB(
    VideoFrame::ReleaseMailboxCB cb) { … }

}  // namespace

// static
std::string VideoFrame::StorageTypeToString(
    const VideoFrame::StorageType storage_type) { … }

// static
bool VideoFrame::IsStorageTypeMappable(VideoFrame::StorageType storage_type) { … }

// static
bool VideoFrame::IsValidPlane(VideoPixelFormat format, size_t plane) { … }

// static
gfx::Size VideoFrame::SampleSize(VideoPixelFormat format, size_t plane) { … }

// Checks if |source_format| can be wrapped into a |target_format| frame.
static bool AreValidPixelFormatsForWrap(VideoPixelFormat source_format,
                                        VideoPixelFormat target_format) { … }

// Creates VideoFrameLayout for tightly packed frame.
static std::optional<VideoFrameLayout> GetDefaultLayout(
    VideoPixelFormat format,
    const gfx::Size& coded_size) { … }

// static
bool VideoFrame::IsValidConfig(VideoPixelFormat format,
                               StorageType storage_type,
                               const gfx::Size& coded_size,
                               const gfx::Rect& visible_rect,
                               const gfx::Size& natural_size) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::CreateFrame(VideoPixelFormat format,
                                                  const gfx::Size& coded_size,
                                                  const gfx::Rect& visible_rect,
                                                  const gfx::Size& natural_size,
                                                  base::TimeDelta timestamp) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::CreateVideoHoleFrame(
    const base::UnguessableToken& overlay_plane_id,
    const gfx::Size& natural_size,
    base::TimeDelta timestamp) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::CreateZeroInitializedFrame(
    VideoPixelFormat format,
    const gfx::Size& coded_size,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    base::TimeDelta timestamp) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::CreateFrameForNativeTexturesInternal(
    VideoPixelFormat format,
    const gfx::Size& coded_size,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    base::TimeDelta timestamp) { … }

scoped_refptr<VideoFrame>
VideoFrame::CreateFrameForGpuMemoryBufferOrMappableSIInternal(
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer,
    scoped_refptr<gpu::ClientSharedImage> shared_image,
    const bool enable_mappable_si,
    ReleaseMailboxAndGpuMemoryBufferCB mailbox_holder_and_gmb_release_cb,
    base::TimeDelta timestamp) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::WrapNativeTextures(
    VideoPixelFormat format,
    const gpu::MailboxHolder (&mailbox_holders)[kMaxPlanes],
    ReleaseMailboxCB mailbox_holder_release_cb,
    const gfx::Size& coded_size,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    base::TimeDelta timestamp) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::WrapSharedImage(
    VideoPixelFormat format,
    scoped_refptr<gpu::ClientSharedImage> shared_image,
    gpu::SyncToken sync_token,
    uint32_t texture_target,
    ReleaseMailboxCB mailbox_holder_release_cb,
    const gfx::Size& coded_size,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    base::TimeDelta timestamp) { … }

scoped_refptr<VideoFrame> VideoFrame::WrapMappableSharedImage(
    scoped_refptr<gpu::ClientSharedImage> shared_image,
    gpu::SyncToken sync_token,
    uint32_t texture_target,
    ReleaseMailboxAndGpuMemoryBufferCB mailbox_holder_and_gmb_release_cb,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    base::TimeDelta timestamp) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::WrapExternalData(
    VideoPixelFormat format,
    const gfx::Size& coded_size,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    const uint8_t* data,
    size_t data_size,
    base::TimeDelta timestamp) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::WrapExternalDataWithLayout(
    const VideoFrameLayout& layout,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    const uint8_t* data,
    size_t data_size,
    base::TimeDelta timestamp) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::WrapExternalYuvData(
    VideoPixelFormat format,
    const gfx::Size& coded_size,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    int32_t y_stride,
    int32_t u_stride,
    int32_t v_stride,
    const uint8_t* y_data,
    const uint8_t* u_data,
    const uint8_t* v_data,
    base::TimeDelta timestamp) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::WrapExternalYuvDataWithLayout(
    const VideoFrameLayout& layout,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    const uint8_t* y_data,
    const uint8_t* u_data,
    const uint8_t* v_data,
    base::TimeDelta timestamp) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::WrapExternalYuvaData(
    VideoPixelFormat format,
    const gfx::Size& coded_size,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    int32_t y_stride,
    int32_t u_stride,
    int32_t v_stride,
    int32_t a_stride,
    const uint8_t* y_data,
    const uint8_t* u_data,
    const uint8_t* v_data,
    const uint8_t* a_data,
    base::TimeDelta timestamp) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::WrapExternalYuvData(
    VideoPixelFormat format,
    const gfx::Size& coded_size,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    int32_t y_stride,
    int32_t uv_stride,
    const uint8_t* y_data,
    const uint8_t* uv_data,
    base::TimeDelta timestamp) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::WrapExternalGpuMemoryBuffer(
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer,
    base::TimeDelta timestamp) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::WrapExternalGpuMemoryBuffer(
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer,
    scoped_refptr<gpu::ClientSharedImage> shared_image,
    const gpu::SyncToken& sync_token,
    uint32_t texture_target,
    ReleaseMailboxAndGpuMemoryBufferCB mailbox_holder_and_gmb_release_cb,
    base::TimeDelta timestamp) { … }

#if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
// static
scoped_refptr<VideoFrame> VideoFrame::WrapExternalDmabufs(
    const VideoFrameLayout& layout,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    std::vector<base::ScopedFD> dmabuf_fds,
    base::TimeDelta timestamp) { … }
#endif

#if BUILDFLAG(IS_APPLE)
// static
scoped_refptr<VideoFrame> VideoFrame::WrapUnacceleratedIOSurface(
    gfx::GpuMemoryBufferHandle handle,
    const gfx::Rect& visible_rect,
    base::TimeDelta timestamp) {
  if (handle.type != gfx::GpuMemoryBufferType::IO_SURFACE_BUFFER) {
    DLOG(ERROR) << "Non-IOSurface handle.";
    return nullptr;
  }
  gfx::ScopedIOSurface io_surface = handle.io_surface;
  if (!io_surface) {
    return nullptr;
  }

  // Only support NV12 IOSurfaces.
  const OSType cv_pixel_format = IOSurfaceGetPixelFormat(io_surface.get());
  if (cv_pixel_format != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
    DLOG(ERROR) << "Invalid (non-NV12) pixel format.";
    return nullptr;
  }
  const VideoPixelFormat pixel_format = PIXEL_FORMAT_NV12;

  // Retrieve the layout parameters for |io_surface_|.
  const size_t num_planes = IOSurfaceGetPlaneCount(io_surface.get());
  const gfx::Size size(IOSurfaceGetWidth(io_surface.get()),
                       IOSurfaceGetHeight(io_surface.get()));
  std::vector<int32_t> strides;
  for (size_t i = 0; i < num_planes; ++i)
    strides.push_back(IOSurfaceGetBytesPerRowOfPlane(io_surface.get(), i));
  std::optional<VideoFrameLayout> layout =
      media::VideoFrameLayout::CreateWithStrides(pixel_format, size, strides);
  if (!layout) {
    DLOG(ERROR) << "Invalid layout.";
    return nullptr;
  }

  const StorageType storage_type = STORAGE_UNOWNED_MEMORY;
  if (!IsValidConfig(pixel_format, storage_type, size, visible_rect, size)) {
    DLOG(ERROR) << "Invalid config.";
    return nullptr;
  }

  // Lock the IOSurface for CPU read access. After the VideoFrame is created,
  // add a destruction callback to unlock the IOSurface.
  kern_return_t lock_result =
      IOSurfaceLock(io_surface.get(), kIOSurfaceLockReadOnly, nullptr);
  if (lock_result != kIOReturnSuccess) {
    DLOG(ERROR) << "Failed to lock IOSurface.";
    return nullptr;
  }
  auto unlock_lambda =
      [](base::apple::ScopedCFTypeRef<IOSurfaceRef> io_surface) {
        IOSurfaceUnlock(io_surface.get(), kIOSurfaceLockReadOnly, nullptr);
      };

  scoped_refptr<VideoFrame> frame =
      new VideoFrame(*layout, storage_type, visible_rect, size, timestamp);
  for (size_t i = 0; i < num_planes; ++i) {
    frame->data_[i] = reinterpret_cast<uint8_t*>(
        IOSurfaceGetBaseAddressOfPlane(io_surface.get(), i));
  }
  frame->AddDestructionObserver(
      base::BindOnce(unlock_lambda, std::move(io_surface)));
  return frame;
}

// static
scoped_refptr<VideoFrame> VideoFrame::WrapCVPixelBuffer(
    CVPixelBufferRef cv_pixel_buffer,
    base::TimeDelta timestamp) {
  DCHECK(cv_pixel_buffer);
  DCHECK(CFGetTypeID(cv_pixel_buffer) == CVPixelBufferGetTypeID());

  const OSType cv_format = CVPixelBufferGetPixelFormatType(cv_pixel_buffer);
  VideoPixelFormat format;
  // There are very few compatible CV pixel formats, so just check each.
  if (cv_format == kCVPixelFormatType_420YpCbCr8Planar) {
    format = PIXEL_FORMAT_I420;
  } else if (cv_format == kCVPixelFormatType_444YpCbCr8) {
    format = PIXEL_FORMAT_I444;
  } else if (cv_format == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
    format = PIXEL_FORMAT_NV12;
  } else if (cv_format ==
             kCVPixelFormatType_420YpCbCr8VideoRange_8A_TriPlanar) {
    format = PIXEL_FORMAT_NV12A;
  } else {
    DLOG(ERROR) << "CVPixelBuffer format not supported: " << cv_format;
    return nullptr;
  }

  const gfx::Size coded_size(CVImageBufferGetEncodedSize(cv_pixel_buffer));
  const gfx::Rect visible_rect(CVImageBufferGetCleanRect(cv_pixel_buffer));
  const gfx::Size natural_size(CVImageBufferGetDisplaySize(cv_pixel_buffer));
  const StorageType storage = STORAGE_UNOWNED_MEMORY;

  if (!IsValidConfig(format, storage, coded_size, visible_rect, natural_size)) {
    DLOG(ERROR) << __func__ << " Invalid config."
                << ConfigToString(format, storage, coded_size, visible_rect,
                                  natural_size);
    return nullptr;
  }

  auto layout = VideoFrameLayout::Create(format, coded_size);
  if (!layout) {
    DLOG(ERROR) << "Invalid layout.";
    return nullptr;
  }

  scoped_refptr<VideoFrame> frame(
      new VideoFrame(*layout, storage, visible_rect, natural_size, timestamp));

  frame->cv_pixel_buffer_.reset(cv_pixel_buffer, base::scoped_policy::RETAIN);
  return frame;
}
#endif

// static
scoped_refptr<VideoFrame> VideoFrame::WrapVideoFrame(
    scoped_refptr<VideoFrame> frame,
    VideoPixelFormat format,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::CreateEOSFrame() { … }

// static
scoped_refptr<VideoFrame> VideoFrame::CreateColorFrame(
    const gfx::Size& size,
    uint8_t y,
    uint8_t u,
    uint8_t v,
    base::TimeDelta timestamp) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::CreateBlackFrame(const gfx::Size& size) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::CreateTransparentFrame(
    const gfx::Size& size) { … }

// static
size_t VideoFrame::NumPlanes(VideoPixelFormat format) { … }

// static
size_t VideoFrame::AllocationSize(VideoPixelFormat format,
                                  const gfx::Size& coded_size) { … }

// static
gfx::Size VideoFrame::PlaneSize(VideoPixelFormat format,
                                size_t plane,
                                const gfx::Size& coded_size) { … }

// static
gfx::Size VideoFrame::PlaneSizeInSamples(VideoPixelFormat format,
                                         size_t plane,
                                         const gfx::Size& coded_size) { … }

// static
int VideoFrame::PlaneHorizontalBitsPerPixel(VideoPixelFormat format,
                                            size_t plane) { … }

// static
int VideoFrame::PlaneBitsPerPixel(VideoPixelFormat format, size_t plane) { … }

// static
size_t VideoFrame::RowBytes(size_t plane, VideoPixelFormat format, int width) { … }

// static
int VideoFrame::BytesPerElement(VideoPixelFormat format, size_t plane) { … }

// static
std::vector<int32_t> VideoFrame::ComputeStrides(VideoPixelFormat format,
                                                const gfx::Size& coded_size) { … }

// static
size_t VideoFrame::Rows(size_t plane, VideoPixelFormat format, int height) { … }

// static
size_t VideoFrame::Columns(size_t plane, VideoPixelFormat format, int width) { … }

// static
void VideoFrame::HashFrameForTesting(base::MD5Context* context,
                                     const VideoFrame& frame) { … }

void VideoFrame::BackWithSharedMemory(
    const base::ReadOnlySharedMemoryRegion* region) { … }

void VideoFrame::BackWithOwnedSharedMemory(
    base::ReadOnlySharedMemoryRegion region,
    base::ReadOnlySharedMemoryMapping mapping) { … }

bool VideoFrame::IsMappable() const { … }

bool VideoFrame::HasTextures() const { … }

size_t VideoFrame::NumTextures() const { … }

bool VideoFrame::HasSharedImage() const { … }

bool VideoFrame::HasMappableGpuBuffer() const { … }

bool VideoFrame::HasNativeGpuMemoryBuffer() const { … }

gfx::GpuMemoryBuffer* VideoFrame::GetGpuMemoryBuffer() const { … }

std::unique_ptr<VideoFrame::ScopedMapping> VideoFrame::MapGMBOrSharedImage()
    const { … }

gfx::GpuMemoryBufferHandle VideoFrame::GetGpuMemoryBufferHandle() const { … }

bool VideoFrame::IsSameAllocation(VideoPixelFormat format,
                                  const gfx::Size& coded_size,
                                  const gfx::Rect& visible_rect,
                                  const gfx::Size& natural_size) const { … }

gfx::ColorSpace VideoFrame::ColorSpace() const { … }

gfx::ColorSpace VideoFrame::CompatRGBColorSpace() const { … }

bool VideoFrame::RequiresExternalSampler() const { … }

int VideoFrame::row_bytes(size_t plane) const { … }

int VideoFrame::rows(size_t plane) const { … }

int VideoFrame::columns(size_t plane) const { … }

template <typename T>
T VideoFrame::GetVisibleDataInternal(T data, size_t plane) const { … }

const uint8_t* VideoFrame::visible_data(size_t plane) const { … }

uint8_t* VideoFrame::GetWritableVisibleData(size_t plane) { … }

const gpu::MailboxHolder& VideoFrame::mailbox_holder(
    size_t texture_index) const { … }

scoped_refptr<gpu::ClientSharedImage> VideoFrame::shared_image() const { … }

#if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
size_t VideoFrame::NumDmabufFds() const { … }

bool VideoFrame::HasDmaBufs() const { … }

int VideoFrame::GetDmabufFd(size_t i) const { … }
#endif

#if BUILDFLAG(IS_APPLE)
CVPixelBufferRef VideoFrame::CvPixelBuffer() const {
  return cv_pixel_buffer_.get();
}
#endif

void VideoFrame::SetReleaseMailboxCB(ReleaseMailboxCB release_mailbox_cb) { … }

void VideoFrame::SetReleaseMailboxAndGpuMemoryBufferCB(
    ReleaseMailboxAndGpuMemoryBufferCB release_mailbox_cb) { … }

bool VideoFrame::HasReleaseMailboxCB() const { … }

void VideoFrame::AddDestructionObserver(base::OnceClosure callback) { … }

gpu::SyncToken VideoFrame::UpdateReleaseSyncToken(SyncTokenClient* client) { … }

gpu::SyncToken VideoFrame::UpdateMailboxHolderSyncToken(
    size_t plane,
    SyncTokenClient* client) { … }

std::string VideoFrame::AsHumanReadableString() const { … }

size_t VideoFrame::BitDepth() const { … }

VideoFrame::VideoFrame(const VideoFrameLayout& layout,
                       StorageType storage_type,
                       const gfx::Rect& visible_rect,
                       const gfx::Size& natural_size,
                       base::TimeDelta timestamp,
                       FrameControlType frame_control_type)
    : … { … }

VideoFrame::~VideoFrame() { … }

// static
std::string VideoFrame::ConfigToString(const VideoPixelFormat format,
                                       const StorageType storage_type,
                                       const gfx::Size& coded_size,
                                       const gfx::Rect& visible_rect,
                                       const gfx::Size& natural_size) { … }

// static
gfx::Size VideoFrame::DetermineAlignedSize(VideoPixelFormat format,
                                           const gfx::Size& dimensions) { … }

// static
bool VideoFrame::IsValidSize(const gfx::Size& coded_size,
                             const gfx::Rect& visible_rect,
                             const gfx::Size& natural_size) { … }

// static
bool VideoFrame::IsValidCodedSize(const gfx::Size& size) { … }

// static
bool VideoFrame::IsValidConfigInternal(VideoPixelFormat format,
                                       FrameControlType frame_control_type,
                                       const gfx::Size& coded_size,
                                       const gfx::Rect& visible_rect,
                                       const gfx::Size& natural_size) { … }

// static
std::optional<VideoFrameLayout>
VideoFrame::CreateFullySpecifiedLayoutWithStrides(VideoPixelFormat format,
                                                  const gfx::Size& coded_size) { … }

// static
scoped_refptr<VideoFrame> VideoFrame::CreateFrameInternal(
    VideoPixelFormat format,
    const gfx::Size& coded_size,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    base::TimeDelta timestamp,
    bool zero_initialize_memory) { … }

scoped_refptr<VideoFrame> VideoFrame::CreateFrameWithLayout(
    const VideoFrameLayout& layout,
    const gfx::Rect& visible_rect,
    const gfx::Size& natural_size,
    base::TimeDelta timestamp,
    bool zero_initialize_memory) { … }

// static
gfx::Size VideoFrame::CommonAlignment(VideoPixelFormat format) { … }

bool VideoFrame::AllocateMemory(bool zero_initialize_memory) { … }

bool VideoFrame::IsValidSharedMemoryFrame() const { … }

// static
std::vector<size_t> VideoFrame::CalculatePlaneSize(
    const VideoFrameLayout& layout) { … }

std::vector<size_t> VideoFrame::CalculatePlaneSize() const { … }

VideoFrame::ScopedMapping::ScopedMapping(
    gfx::GpuMemoryBuffer* gpu_memory_buffer,
    std::unique_ptr<gpu::ClientSharedImage::ScopedMapping> scoped_mapping)
    : … { … }

VideoFrame::ScopedMapping::~ScopedMapping() { … }

uint8_t* VideoFrame::ScopedMapping::Memory(uint32_t plane_index) { … }

size_t VideoFrame::ScopedMapping::Stride(uint32_t plane_index) { … }

gfx::Size VideoFrame::ScopedMapping::Size() { … }

}  // namespace media