// Copyright 2023 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "media/gpu/mac/video_toolbox_decompression_session_manager.h"
#include <Foundation/Foundation.h>
#include <memory>
#include "base/apple/bridging.h"
#include "base/functional/bind.h"
#include "base/logging.h"
#include "media/base/media_log.h"
#include "media/base/video_types.h"
#include "media/gpu/mac/video_toolbox_decompression_metadata.h"
#include "media/gpu/mac/video_toolbox_decompression_session.h"
using base::apple::CFToNSPtrCast;
using base::apple::NSToCFPtrCast;
namespace media {
VideoToolboxDecompressionSessionManager::
VideoToolboxDecompressionSessionManager(
scoped_refptr<base::SequencedTaskRunner> task_runner,
std::unique_ptr<MediaLog> media_log,
OutputCB output_cb,
ErrorCB error_cb)
: task_runner_(std::move(task_runner)),
media_log_(std::move(media_log)),
output_cb_(std::move(output_cb)),
error_cb_(std::move(error_cb)) {
DVLOG(1) << __func__;
DCHECK(error_cb_);
weak_this_ = weak_this_factory_.GetWeakPtr();
decompression_session_ =
std::make_unique<VideoToolboxDecompressionSessionImpl>(
task_runner_, media_log_->Clone(),
base::BindRepeating(
&VideoToolboxDecompressionSessionManager::OnOutput, weak_this_));
}
VideoToolboxDecompressionSessionManager::
~VideoToolboxDecompressionSessionManager() {
DVLOG(1) << __func__;
DCHECK(task_runner_->RunsTasksInCurrentSequence());
}
void VideoToolboxDecompressionSessionManager::Decode(
base::apple::ScopedCFTypeRef<CMSampleBufferRef> sample,
std::unique_ptr<VideoToolboxDecodeMetadata> metadata) {
DVLOG(3) << __func__;
DCHECK(task_runner_->RunsTasksInCurrentSequence());
if (has_error_) {
return;
}
pending_decodes_.emplace(std::move(sample), std::move(metadata));
if (!Process()) {
NotifyError(DecoderStatus::Codes::kPlatformDecodeFailure);
return;
}
}
void VideoToolboxDecompressionSessionManager::Reset() {
DVLOG(2) << __func__;
DCHECK(task_runner_->RunsTasksInCurrentSequence());
if (has_error_) {
return;
}
pending_decodes_ = {};
// Discard active decodes when they complete. In most cases this is faster
// than destroying the session.
for (auto& it : active_decodes_) {
it.second->discard = true;
}
// If we are draining, it means that there was a pending decode with a
// different format. Since that was erased, there is no need to drain.
draining_ = false;
}
size_t VideoToolboxDecompressionSessionManager::NumDecodes() {
DVLOG(4) << __func__;
DCHECK(task_runner_->RunsTasksInCurrentSequence());
size_t num_decodes = pending_decodes_.size();
// Only non-discarded decodes are counted because the caller won't be
// notified when discarded decodes complete.
for (auto& it : active_decodes_) {
if (!it.second->discard) {
++num_decodes;
}
}
return num_decodes;
}
void VideoToolboxDecompressionSessionManager::NotifyError(
DecoderStatus status) {
DVLOG(1) << __func__;
DCHECK(task_runner_->RunsTasksInCurrentSequence());
DCHECK(!has_error_);
has_error_ = true;
pending_decodes_ = {};
DestroySession();
// We may still be executing inside Decode() and don't want to make a
// re-entrant call.
task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&VideoToolboxDecompressionSessionManager::CallErrorCB,
weak_this_, std::move(error_cb_), std::move(status)));
}
void VideoToolboxDecompressionSessionManager::CallErrorCB(
ErrorCB error_cb,
DecoderStatus status) {
DVLOG(4) << __func__;
DCHECK(task_runner_->RunsTasksInCurrentSequence());
std::move(error_cb).Run(std::move(status));
}
bool VideoToolboxDecompressionSessionManager::Process() {
DVLOG(4) << __func__;
DCHECK(task_runner_->RunsTasksInCurrentSequence());
DCHECK(!has_error_);
if (draining_) {
return true;
}
while (!pending_decodes_.empty()) {
base::apple::ScopedCFTypeRef<CMSampleBufferRef>& sample =
pending_decodes_.front().first;
std::unique_ptr<VideoToolboxDecodeMetadata>& metadata =
pending_decodes_.front().second;
CMFormatDescriptionRef format =
CMSampleBufferGetFormatDescription(sample.get());
// Handle format changes.
if (decompression_session_->IsValid() && format != active_format_.get()) {
if (decompression_session_->CanAcceptFormat(format)) {
active_format_.reset(format, base::scoped_policy::RETAIN);
} else {
// Destroy the active session so that it can be replaced.
if (!active_decodes_.empty()) {
// Wait for the active session to drain before destroying it.
draining_ = true;
return true;
}
DestroySession();
}
}
// Create a new session if necessary.
if (!decompression_session_->IsValid()) {
if (!CreateSession(format, metadata->session_metadata)) {
return false;
}
}
// Submit the sample for decoding.
uintptr_t context = reinterpret_cast<uintptr_t>(metadata.get());
if (!decompression_session_->DecodeFrame(sample.get(), context)) {
return false;
}
// Update state. The pop() must come second because it destructs `metadata`.
active_decodes_[context] = std::move(metadata);
pending_decodes_.pop();
}
return true;
}
bool VideoToolboxDecompressionSessionManager::CreateSession(
CMFormatDescriptionRef format,
const VideoToolboxDecompressionSessionMetadata& session_metadata) {
DVLOG(2) << __func__;
DCHECK(task_runner_->RunsTasksInCurrentSequence());
DCHECK(!decompression_session_->IsValid());
// Build video decoder specification.
NSDictionary* decoder_config = nil;
#if BUILDFLAG(IS_MAC)
decoder_config = @{
CFToNSPtrCast(
kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder) :
@YES,
CFToNSPtrCast(
kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder) :
!session_metadata.allow_software_decoding
? @YES
: @NO
};
#else
decoder_config = @{};
#endif
// Build destination image buffer attributes.
// TODO(crbug.com/40227557): Also set size using the visible rect.
// It is possible to create a decompression session with no destination image
// buffer attributes, but then we must be able to handle any kind of pixel
// format that VideoToolbox can produce, and there is no definitive list.
//
// Some formats that have been seen include:
// - 12-bit YUV: 'tv20', 'tv22', 'tv44'
// - 10-bit YUV: 'p420', 'p422', 'p444'
// - 8-bit YUV: '420v', '422v', '444v'
//
// Other plausible formats include RGB, monochrome, and versions of the above
// with alpha (eg. 'v0a8') and/or full-range (eg. '420f').
//
// Rather than explicitly handling every possible format in
// VideoToolboxFrameConverter, it may be possible to introspect the IOSurfaces
// at run time and map them to viz formats.
//
// TODO(crbug.com/40227557): Do not create an image config for known-supported
// formats, and add full-range versions as supported formats.
FourCharCode pixel_format;
if (session_metadata.chroma_sampling == VideoChromaSampling::k444) {
pixel_format = session_metadata.bit_depth > 8
? kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange
: kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange;
} else if (session_metadata.chroma_sampling == VideoChromaSampling::k422) {
pixel_format = session_metadata.bit_depth > 8
? kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange
: kCVPixelFormatType_422YpCbCr8BiPlanarVideoRange;
} else {
pixel_format = session_metadata.bit_depth > 8
? kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange
: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
}
if (session_metadata.has_alpha) {
pixel_format = kCVPixelFormatType_420YpCbCr8VideoRange_8A_TriPlanar;
}
NSDictionary* image_config =
@{CFToNSPtrCast(kCVPixelBufferPixelFormatTypeKey) : @(pixel_format)};
// Create the session.
if (!decompression_session_->Create(format, NSToCFPtrCast(decoder_config),
NSToCFPtrCast(image_config))) {
return false;
}
// Update saved state.
active_format_.reset(format, base::scoped_policy::RETAIN);
return true;
}
void VideoToolboxDecompressionSessionManager::DestroySession() {
DVLOG(2) << __func__;
DCHECK(task_runner_->RunsTasksInCurrentSequence());
if (!decompression_session_->IsValid()) {
return;
}
decompression_session_->Invalidate();
active_format_.reset();
active_decodes_.clear();
draining_ = false;
}
void VideoToolboxDecompressionSessionManager::OnOutput(
uintptr_t context,
OSStatus status,
VTDecodeInfoFlags flags,
base::apple::ScopedCFTypeRef<CVImageBufferRef> image) {
DVLOG(4) << __func__;
DCHECK(task_runner_->RunsTasksInCurrentSequence());
if (!error_cb_) {
return;
}
if (status != noErr) {
OSSTATUS_MEDIA_LOG(ERROR, status, media_log_.get())
<< "VTDecompressionOutputCallback";
NotifyError(DecoderStatus::Codes::kPlatformDecodeFailure);
return;
}
if (flags & kVTDecodeInfo_FrameDropped) {
CHECK(!image);
} else if (!image || CFGetTypeID(image.get()) != CVPixelBufferGetTypeID()) {
MEDIA_LOG(ERROR, media_log_.get())
<< "Decoded image is not a CVPixelBuffer";
NotifyError(DecoderStatus::Codes::kPlatformDecodeFailure);
return;
}
auto metadata_it = active_decodes_.find(context);
if (metadata_it == active_decodes_.end()) {
MEDIA_LOG(ERROR, media_log_.get()) << "Unknown decode context";
NotifyError(DecoderStatus::Codes::kPlatformDecodeFailure);
return;
}
std::unique_ptr<VideoToolboxDecodeMetadata> metadata =
std::move(metadata_it->second);
active_decodes_.erase(metadata_it);
// If we are draining and the session is now empty, complete the drain. This
// happens before output so that we don't need to consider what the output
// callback might do synchronously.
if (draining_ && active_decodes_.empty()) {
DestroySession();
if (!Process()) {
NotifyError(DecoderStatus::Codes::kPlatformDecodeFailure);
return;
}
}
if (!metadata->discard) {
// OnOutput() was posted, so this is never re-entrant.
output_cb_.Run(std::move(image), std::move(metadata));
}
}
void VideoToolboxDecompressionSessionManager::SetDecompressionSessionForTesting(
std::unique_ptr<VideoToolboxDecompressionSession> decompression_session) {
decompression_session_ = std::move(decompression_session);
}
} // namespace media
Codebase Browser
chromium