// Copyright 2022 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "content/browser/media/capture/screen_capture_kit_device_mac.h"
#import <ScreenCaptureKit/ScreenCaptureKit.h>
#include <optional>
#include "base/apple/bridging.h"
#include "base/apple/foundation_util.h"
#include "base/task/bind_post_task.h"
#import "base/task/single_thread_task_runner.h"
#include "base/threading/thread_checker.h"
#include "base/timer/timer.h"
#include "content/browser/media/capture/io_surface_capture_device_base_mac.h"
#include "content/browser/media/capture/screen_capture_kit_fullscreen_module.h"
#include "content/public/common/content_features.h"
#include "third_party/webrtc/modules/desktop_capture/desktop_capture_types.h"
#include "ui/gfx/native_widget_types.h"
using SampleCallback = base::RepeatingCallback<void(gfx::ScopedInUseIOSurface,
std::optional<gfx::Size>,
std::optional<gfx::Rect>)>;
using ErrorCallback = base::RepeatingClosure;
API_AVAILABLE(macos(12.3))
@interface ScreenCaptureKitDeviceHelper
: NSObject <SCStreamDelegate, SCStreamOutput>
- (instancetype)initWithSampleCallback:(SampleCallback)sampleCallback
errorCallback:(ErrorCallback)errorCallback;
@end
@implementation ScreenCaptureKitDeviceHelper {
SampleCallback _sampleCallback;
ErrorCallback _errorCallback;
}
- (instancetype)initWithSampleCallback:(SampleCallback)sampleCallback
errorCallback:(ErrorCallback)errorCallback {
if (self = [super init]) {
_sampleCallback = sampleCallback;
_errorCallback = errorCallback;
}
return self;
}
- (void)stream:(SCStream*)stream
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
ofType:(SCStreamOutputType)type {
CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
if (!pixelBuffer)
return;
// Read out width, height and scaling from metadata to determine
// |contentSize|, which is the size of the content on screen, and
// |visibleRect|, which is the region of the IOSurface that contains the
// captured content. |contentSize| is used to detect when a captured window is
// resized so that the stream configuration can be updated and |visibleRect|
// is needed because the IOSurface may be larger than the captured content.
std::optional<gfx::Size> contentSize;
std::optional<gfx::Rect> visibleRect;
CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(
sampleBuffer, /*createIfNecessary=*/false);
if (attachmentsArray && CFArrayGetCount(attachmentsArray) > 0) {
CFDictionaryRef attachment = base::apple::CFCast<CFDictionaryRef>(
CFArrayGetValueAtIndex(attachmentsArray, 0));
if (attachment) {
CFDictionaryRef contentRectValue = base::apple::CFCast<CFDictionaryRef>(
CFDictionaryGetValue(attachment, base::apple::NSToCFPtrCast(
SCStreamFrameInfoContentRect)));
CFNumberRef scaleFactorValue = base::apple::CFCast<CFNumberRef>(
CFDictionaryGetValue(attachment, base::apple::NSToCFPtrCast(
SCStreamFrameInfoScaleFactor)));
CFNumberRef contentScaleValue = base::apple::CFCast<CFNumberRef>(
CFDictionaryGetValue(attachment, base::apple::NSToCFPtrCast(
SCStreamFrameInfoContentScale)));
if (contentRectValue && scaleFactorValue && contentScaleValue) {
CGRect contentRect = {};
bool succeed = CGRectMakeWithDictionaryRepresentation(contentRectValue,
&contentRect);
float scaleFactor = 1.0f;
succeed &= CFNumberGetValue(scaleFactorValue, kCFNumberFloatType,
&scaleFactor);
float contentScale = 1.0f;
succeed &= CFNumberGetValue(contentScaleValue, kCFNumberFloatType,
&contentScale);
if (succeed) {
contentRect.size.width *= scaleFactor;
contentRect.size.height *= scaleFactor;
visibleRect.emplace(contentRect);
contentSize.emplace(round(contentRect.size.width / contentScale),
round(contentRect.size.height / contentScale));
}
}
}
}
IOSurfaceRef ioSurface = CVPixelBufferGetIOSurface(pixelBuffer);
if (!ioSurface)
return;
_sampleCallback.Run(
gfx::ScopedInUseIOSurface(ioSurface, base::scoped_policy::RETAIN),
contentSize, visibleRect);
}
- (void)stream:(SCStream*)stream didStopWithError:(NSError*)error {
_errorCallback.Run();
}
+ (SCStreamConfiguration*)streamConfigurationWithFrameSize:(gfx::Size)frameSize
destRectInFrame:
(gfx::RectF)destRectInFrame
frameRate:(float)frameRate {
SCStreamConfiguration* config = [[SCStreamConfiguration alloc] init];
config.width = frameSize.width();
config.height = frameSize.height();
config.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
config.destinationRect = destRectInFrame.ToCGRect();
config.backgroundColor = CGColorGetConstantColor(kCGColorBlack);
config.scalesToFit = YES;
config.showsCursor = YES;
config.colorSpaceName = kCGColorSpaceSRGB;
config.minimumFrameInterval =
CMTimeMake(media::kFrameRatePrecision,
static_cast<int>(frameRate * media::kFrameRatePrecision));
return config;
}
@end
namespace content {
namespace {
BASE_FEATURE(kScreenCaptureKitFullDesktopFallback,
"ScreenCaptureKitFullDesktopFallback",
base::FEATURE_ENABLED_BY_DEFAULT);
class API_AVAILABLE(macos(12.3)) ScreenCaptureKitDeviceMac
: public IOSurfaceCaptureDeviceBase,
public ScreenCaptureKitResetStreamInterface {
public:
explicit ScreenCaptureKitDeviceMac(const DesktopMediaID& source,
SCContentFilter* filter)
: source_(source),
filter_(filter),
device_task_runner_(base::SingleThreadTaskRunner::GetCurrentDefault()) {
SampleCallback sample_callback = base::BindPostTask(
device_task_runner_,
base::BindRepeating(&ScreenCaptureKitDeviceMac::OnStreamSample,
weak_factory_.GetWeakPtr()));
ErrorCallback error_callback = base::BindPostTask(
device_task_runner_,
base::BindRepeating(&ScreenCaptureKitDeviceMac::OnStreamError,
weak_factory_.GetWeakPtr()));
helper_ = [[ScreenCaptureKitDeviceHelper alloc]
initWithSampleCallback:sample_callback
errorCallback:error_callback];
}
ScreenCaptureKitDeviceMac(const ScreenCaptureKitDeviceMac&) = delete;
ScreenCaptureKitDeviceMac& operator=(const ScreenCaptureKitDeviceMac&) =
delete;
~ScreenCaptureKitDeviceMac() override = default;
void OnShareableContentCreated(SCShareableContent* content) {
DCHECK(device_task_runner_->RunsTasksInCurrentSequence());
if (!content) {
client()->OnError(
media::VideoCaptureError::kScreenCaptureKitFailedGetShareableContent,
FROM_HERE, "Failed getShareableContentWithCompletionHandler");
return;
}
SCContentFilter* filter;
switch (source_.type) {
case DesktopMediaID::TYPE_SCREEN:
for (SCDisplay* display in content.displays) {
// There's currently no support for stitching desktops together as
// requested by kFullDesktopScreenId. Capture the first display as a
// fallback. See https://crbug.com/325530044.
if (source_.id == display.displayID ||
source_.id == webrtc::kFullDesktopScreenId) {
filter = [[SCContentFilter alloc] initWithDisplay:display
excludingWindows:@[]];
stream_config_content_size_ =
gfx::Size(display.width, display.height);
break;
}
}
break;
case DesktopMediaID::TYPE_WINDOW:
for (SCWindow* window in content.windows) {
if (source_.id == window.windowID) {
filter = [[SCContentFilter alloc]
initWithDesktopIndependentWindow:window];
CGRect frame = window.frame;
stream_config_content_size_ = gfx::Size(frame.size);
if (!fullscreen_module_) {
fullscreen_module_ = MaybeCreateScreenCaptureKitFullscreenModule(
device_task_runner_, *this, window);
}
}
}
break;
default:
NOTREACHED_IN_MIGRATION();
break;
}
CreateStream(filter);
}
void CreateStream(SCContentFilter* filter) {
DCHECK(device_task_runner_->RunsTasksInCurrentSequence());
if (!filter) {
client()->OnError(
media::VideoCaptureError::kScreenCaptureKitFailedToFindSCDisplay,
FROM_HERE, "Failed to find SCDisplay");
return;
}
if (@available(macOS 14.0, *)) {
// Update the content size. This step is neccessary when used together
// with SCContentSharingPicker. If the Chrome picker is used, it will
// change to retina resolution if applicable.
stream_config_content_size_ =
gfx::Size(filter.contentRect.size.width * filter.pointPixelScale,
filter.contentRect.size.height * filter.pointPixelScale);
}
gfx::RectF dest_rect_in_frame;
actual_capture_format_ = capture_params().requested_format;
actual_capture_format_.pixel_format = media::PIXEL_FORMAT_NV12;
ComputeFrameSizeAndDestRect(stream_config_content_size_,
actual_capture_format_.frame_size,
dest_rect_in_frame);
SCStreamConfiguration* config = [ScreenCaptureKitDeviceHelper
streamConfigurationWithFrameSize:actual_capture_format_.frame_size
destRectInFrame:dest_rect_in_frame
frameRate:actual_capture_format_.frame_rate];
stream_ = [[SCStream alloc] initWithFilter:filter
configuration:config
delegate:helper_];
{
NSError* error = nil;
bool add_stream_output_result =
[stream_ addStreamOutput:helper_
type:SCStreamOutputTypeScreen
sampleHandlerQueue:dispatch_get_main_queue()
error:&error];
if (!add_stream_output_result) {
stream_ = nil;
client()->OnError(
media::VideoCaptureError::kScreenCaptureKitFailedAddStreamOutput,
FROM_HERE, "Failed addStreamOutput");
return;
}
}
auto stream_started_callback = base::BindPostTask(
device_task_runner_,
base::BindRepeating(&ScreenCaptureKitDeviceMac::OnStreamStarted,
weak_factory_.GetWeakPtr()));
auto handler = ^(NSError* error) {
stream_started_callback.Run(!!error);
};
[stream_ startCaptureWithCompletionHandler:handler];
}
void OnStreamStarted(bool error) {
DCHECK(device_task_runner_->RunsTasksInCurrentSequence());
if (error) {
client()->OnError(
media::VideoCaptureError::kScreenCaptureKitFailedStartCapture,
FROM_HERE, "Failed startCaptureWithCompletionHandler");
return;
}
client()->OnStarted();
if (fullscreen_module_) {
fullscreen_module_->Start();
}
}
void OnStreamStopped(bool error) {
DCHECK(device_task_runner_->RunsTasksInCurrentSequence());
if (error) {
client()->OnError(
media::VideoCaptureError::kScreenCaptureKitFailedStopCapture,
FROM_HERE, "Failed stopCaptureWithCompletionHandler");
return;
}
}
void OnStreamSample(gfx::ScopedInUseIOSurface io_surface,
std::optional<gfx::Size> content_size,
std::optional<gfx::Rect> visible_rect) {
DCHECK(device_task_runner_->RunsTasksInCurrentSequence());
if (requested_capture_format_) {
// Does the size of io_surface match the requested format?
size_t io_surface_width = IOSurfaceGetWidth(io_surface.get());
size_t io_surface_height = IOSurfaceGetHeight(io_surface.get());
DVLOG(3) << "Waiting for new capture format, "
<< requested_capture_format_->frame_size.width() << " x "
<< requested_capture_format_->frame_size.height()
<< ". IO surface size " << io_surface_width << " x "
<< io_surface_height;
if (static_cast<size_t>(requested_capture_format_->frame_size.width()) ==
io_surface_width &&
static_cast<size_t>(requested_capture_format_->frame_size.height()) ==
io_surface_height) {
actual_capture_format_ = requested_capture_format_.value();
requested_capture_format_.reset();
}
} else {
// No current request for new capture format. Check to see if content_size
// has changed and requires an updated configuration. We only track the
// content size for window capturing since the resolution does not
// normally change during a session and because the content scale is wrong
// for retina displays.
if (source_.type == DesktopMediaID::TYPE_WINDOW && content_size &&
(stream_config_content_size_.width() != content_size->width() ||
stream_config_content_size_.height() != content_size->height())) {
DVLOG(3) << "Content size changed to " << content_size->width() << " x "
<< content_size->height() << ". It was "
<< stream_config_content_size_.width() << " x "
<< stream_config_content_size_.height();
stream_config_content_size_ = content_size.value();
gfx::RectF dest_rect_in_frame;
gfx::Size new_frame_size;
ComputeFrameSizeAndDestRect(stream_config_content_size_, new_frame_size,
dest_rect_in_frame);
if (new_frame_size.width() !=
actual_capture_format_.frame_size.width() ||
new_frame_size.height() !=
actual_capture_format_.frame_size.height()) {
DVLOG(3) << "Calling updateConfiguration with new frame size: "
<< new_frame_size.width() << " x "
<< new_frame_size.height();
requested_capture_format_ = actual_capture_format_;
requested_capture_format_->frame_size = new_frame_size;
// Update stream configuration.
SCStreamConfiguration* config = [ScreenCaptureKitDeviceHelper
streamConfigurationWithFrameSize:requested_capture_format_
->frame_size
destRectInFrame:dest_rect_in_frame
frameRate:requested_capture_format_->
frame_rate];
__block base::OnceCallback<void()> on_update_configuration_error =
base::BindPostTask(
device_task_runner_,
base::BindOnce(
&ScreenCaptureKitDeviceMac::OnUpdateConfigurationError,
weak_factory_.GetWeakPtr()));
[stream_
updateConfiguration:config
completionHandler:^(NSError* _Nullable error) {
if (error) {
std::move(on_update_configuration_error).Run();
}
}];
}
}
}
// The IO surface may be larger than the actual content size. Pass on
// visible rect to be able to render/encode the frame correctly.
OnReceivedIOSurfaceFromStream(
io_surface, actual_capture_format_,
visible_rect.value_or(gfx::Rect(actual_capture_format_.frame_size)));
}
void OnStreamError() {
DCHECK(device_task_runner_->RunsTasksInCurrentSequence());
if (is_resetting_ || (fullscreen_module_ &&
fullscreen_module_->is_fullscreen_window_active())) {
// Clear `is_resetting_` because the completion handler in ResetStreamTo()
// may not be called if there's an error.
is_resetting_ = false;
// The stream_ is no longer valid. Restart the stream from scratch.
if (fullscreen_module_) {
fullscreen_module_->Reset();
}
OnStart();
} else {
client()->OnError(media::VideoCaptureError::kScreenCaptureKitStreamError,
FROM_HERE, "Stream delegate called didStopWithError");
}
}
void OnUpdateContentFilterCompleted(NSError* _Nullable error) {
DCHECK(device_task_runner_->RunsTasksInCurrentSequence());
is_resetting_ = false;
if (error) {
client()->OnError(media::VideoCaptureError::kScreenCaptureKitStreamError,
FROM_HERE,
"Error on updateContentFilter (fullscreen window).");
}
}
void OnUpdateConfigurationError() {
DCHECK(device_task_runner_->RunsTasksInCurrentSequence());
client()->OnError(media::VideoCaptureError::kScreenCaptureKitStreamError,
FROM_HERE, "Error on updateConfiguration");
}
// IOSurfaceCaptureDeviceBase:
void OnStart() override {
DCHECK(device_task_runner_->RunsTasksInCurrentSequence());
if (filter_) {
// SCContentSharingPicker is used where filter_ is set on creation.
CreateStream(filter_);
} else {
// Chrome picker is used.
auto content_callback = base::BindPostTask(
device_task_runner_,
base::BindRepeating(
&ScreenCaptureKitDeviceMac::OnShareableContentCreated,
weak_factory_.GetWeakPtr()));
auto handler = ^(SCShareableContent* content, NSError* error) {
content_callback.Run(content);
};
[SCShareableContent getShareableContentWithCompletionHandler:handler];
}
}
void OnStop() override {
DCHECK(device_task_runner_->RunsTasksInCurrentSequence());
if (stream_) {
auto stream_stopped_callback = base::BindPostTask(
device_task_runner_,
base::BindRepeating(&ScreenCaptureKitDeviceMac::OnStreamStopped,
weak_factory_.GetWeakPtr()));
auto handler = ^(NSError* error) {
stream_stopped_callback.Run(!!error);
};
[stream_ stopCaptureWithCompletionHandler:handler];
NSError* error = nil;
bool remove_stream_output_result =
[stream_ removeStreamOutput:helper_
type:SCStreamOutputTypeScreen
error:&error];
if (!remove_stream_output_result) {
DLOG(ERROR) << "Failed removeStreamOutput";
}
}
weak_factory_.InvalidateWeakPtrs();
stream_ = nil;
}
// ScreenCaptureKitResetStreamInterface.
void ResetStreamTo(SCWindow* window) override {
DCHECK(device_task_runner_->RunsTasksInCurrentSequence());
if (!window || is_resetting_) {
client()->OnError(
media::VideoCaptureError::kScreenCaptureKitResetStreamError,
FROM_HERE, "Error on ResetStreamTo.");
return;
}
is_resetting_ = true;
SCContentFilter* filter =
[[SCContentFilter alloc] initWithDesktopIndependentWindow:window];
__block base::OnceCallback<void(NSError*)>
on_update_content_filter_completed = base::BindPostTask(
device_task_runner_,
base::BindOnce(
&ScreenCaptureKitDeviceMac::OnUpdateContentFilterCompleted,
weak_factory_.GetWeakPtr()));
[stream_ updateContentFilter:filter
completionHandler:^(NSError* _Nullable error) {
std::move(on_update_content_filter_completed).Run(error);
}];
}
private:
const DesktopMediaID source_;
SCContentFilter* const filter_;
const scoped_refptr<base::SingleThreadTaskRunner> device_task_runner_;
// The actual format of the video frames that are sent to `client`.
media::VideoCaptureFormat actual_capture_format_;
// The requested format if a request to update the configuration has been
// sent.
std::optional<media::VideoCaptureFormat> requested_capture_format_;
// The size of the content at the time that we configured the stream.
gfx::Size stream_config_content_size_;
// Helper class that acts as output and delegate for `stream_`.
ScreenCaptureKitDeviceHelper* __strong helper_;
// This is used to detect when a captured presentation enters fullscreen mode.
// If this happens, the module will call the ResetStreamTo function.
std::unique_ptr<ScreenCaptureKitFullscreenModule> fullscreen_module_;
bool is_resetting_ = false;
// The stream that does the capturing.
SCStream* __strong stream_;
base::WeakPtrFactory<ScreenCaptureKitDeviceMac> weak_factory_{this};
};
} // namespace
// Although ScreenCaptureKit is available in 12.3 there were some bugs that
// were not fixed until 13.2.
API_AVAILABLE(macos(13.2))
std::unique_ptr<media::VideoCaptureDevice> CreateScreenCaptureKitDeviceMac(
const DesktopMediaID& source,
SCContentFilter* filter) {
switch (source.type) {
case DesktopMediaID::TYPE_SCREEN:
// ScreenCaptureKitDeviceMac only supports a single display at a time.
// It will not stitch desktops together. If
// kScreenCaptureKitFullDesktopFallback is enabled, we will fallback to
// capturing the first display in the list returned from
// getShareableContent. https://crbug.com/1178360 and
// https://crbug.com/325530044
if ((source.id == webrtc::kFullDesktopScreenId &&
!base::FeatureList::IsEnabled(
kScreenCaptureKitFullDesktopFallback)) ||
source.id == webrtc::kInvalidScreenId) {
return nullptr;
}
break;
case DesktopMediaID::TYPE_WINDOW:
break;
default:
// ScreenCaptureKitDeviceMac supports only TYPE_SCREEN and TYPE_WINDOW.
// https://crbug.com/1176900
return nullptr;
}
IncrementDesktopCaptureCounter(SCREEN_CAPTURER_CREATED);
IncrementDesktopCaptureCounter(source.audio_share
? SCREEN_CAPTURER_CREATED_WITH_AUDIO
: SCREEN_CAPTURER_CREATED_WITHOUT_AUDIO);
return std::make_unique<ScreenCaptureKitDeviceMac>(source, filter);
}
} // namespace content
Codebase Browser
chromium