Source code
Revision control
Copy as Markdown
Other Tools
From: Andreas Pehrson <apehrson@mozilla.com>
Date: Thu, 12 Sep 2024 22:36:00 +0000
resolution. r=webrtc-reviewers,ng
If the resolution is smaller than the allocated surface, we crop.
If the resolution is larger than the allocated surface, we reconfigure with a
larger surface.
The allocated surface has the size we told it to have in the
SCStreamConfiguration, in pixels.
If the source is a screen, the size is pretty static. Changing display settings
could affect it.
If the source is a window, the size is initially the size of the window. The
user resizing the window affects the size.
If the source is multiple windows, the size initially seems to be that of the
display they're sitting on. Windows across multiple displays are not captured
into the same surface, as of macOS 14.
Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/bc6215142feaab1a3f7820006129219504851bc8
---
.../mac/desktop_frame_iosurface.h | 9 +-
.../mac/desktop_frame_iosurface.mm | 37 ++++++--
.../mac/screen_capturer_sck.mm | 89 +++++++++++++++++--
3 files changed, 120 insertions(+), 15 deletions(-)
diff --git a/modules/desktop_capture/mac/desktop_frame_iosurface.h b/modules/desktop_capture/mac/desktop_frame_iosurface.h
index 73da0f693c..ed90e40993 100644
--- a/modules/desktop_capture/mac/desktop_frame_iosurface.h
+++ b/modules/desktop_capture/mac/desktop_frame_iosurface.h
@@ -26,7 +26,7 @@ class DesktopFrameIOSurface final : public DesktopFrame {
// Lock an IOSurfaceRef containing a snapshot of a display. Return NULL if
// failed to lock.
static std::unique_ptr<DesktopFrameIOSurface> Wrap(
- rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface);
+ rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface, CGRect rect = {});
~DesktopFrameIOSurface() override;
@@ -35,7 +35,12 @@ class DesktopFrameIOSurface final : public DesktopFrame {
private:
// This constructor expects `io_surface` to hold a non-null IOSurfaceRef.
- explicit DesktopFrameIOSurface(rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface);
+ DesktopFrameIOSurface(
+ rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface,
+ uint8_t* data,
+ int32_t width,
+ int32_t height,
+ int32_t stride);
const rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface_;
};
diff --git a/modules/desktop_capture/mac/desktop_frame_iosurface.mm b/modules/desktop_capture/mac/desktop_frame_iosurface.mm
index b59b319db9..7a5b595f19 100644
--- a/modules/desktop_capture/mac/desktop_frame_iosurface.mm
+++ b/modules/desktop_capture/mac/desktop_frame_iosurface.mm
@@ -17,7 +17,7 @@ namespace webrtc {
// static
std::unique_ptr<DesktopFrameIOSurface> DesktopFrameIOSurface::Wrap(
- rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface) {
+ rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface, CGRect rect) {
if (!io_surface) {
return nullptr;
}
@@ -40,14 +40,39 @@ std::unique_ptr<DesktopFrameIOSurface> DesktopFrameIOSurface::Wrap(
return nullptr;
}
- return std::unique_ptr<DesktopFrameIOSurface>(new DesktopFrameIOSurface(io_surface));
+ size_t surfaceWidth = IOSurfaceGetWidth(io_surface.get());
+ size_t surfaceHeight = IOSurfaceGetHeight(io_surface.get());
+ uint8_t* data = static_cast<uint8_t*>(IOSurfaceGetBaseAddress(io_surface.get()));
+ size_t offset = 0;
+ size_t width = surfaceWidth;
+ size_t height = surfaceHeight;
+ size_t offsetColumns = 0;
+ size_t offsetRows = 0;
+ int32_t stride = IOSurfaceGetBytesPerRow(io_surface.get());
+ if (rect.size.width > 0 && rect.size.height > 0) {
+ width = std::floor(rect.size.width);
+ height = std::floor(rect.size.height);
+ offsetColumns = std::ceil(rect.origin.x);
+ offsetRows = std::ceil(rect.origin.y);
+ RTC_CHECK_GE(surfaceWidth, offsetColumns + width);
+ RTC_CHECK_GE(surfaceHeight, offsetRows + height);
+ offset = stride * offsetRows + bytes_per_pixel * offsetColumns;
+ }
+
+ RTC_LOG(LS_VERBOSE) << "DesktopFrameIOSurface wrapping IOSurface with size " << surfaceWidth << "x"
+ << surfaceHeight << ". Cropping to (" << offsetColumns << "," << offsetRows << "; "
+ << width << "x" << height << "). Stride=" << stride / bytes_per_pixel
+ << ", buffer-offset-px=" << offset / bytes_per_pixel << ", buffer-offset-bytes=" << offset;
+
+ return std::unique_ptr<DesktopFrameIOSurface>(new DesktopFrameIOSurface(io_surface, data + offset, width, height, stride));
}
-DesktopFrameIOSurface::DesktopFrameIOSurface(rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface)
+DesktopFrameIOSurface::DesktopFrameIOSurface(
+ rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface, uint8_t* data, int32_t width, int32_t height, int32_t stride)
: DesktopFrame(
- DesktopSize(IOSurfaceGetWidth(io_surface.get()), IOSurfaceGetHeight(io_surface.get())),
- IOSurfaceGetBytesPerRow(io_surface.get()),
- static_cast<uint8_t*>(IOSurfaceGetBaseAddress(io_surface.get())),
+ DesktopSize(width, height),
+ stride,
+ data,
nullptr),
io_surface_(io_surface) {
RTC_DCHECK(io_surface_);
diff --git a/modules/desktop_capture/mac/screen_capturer_sck.mm b/modules/desktop_capture/mac/screen_capturer_sck.mm
index 96d54c5c03..f331bafdfd 100644
--- a/modules/desktop_capture/mac/screen_capturer_sck.mm
+++ b/modules/desktop_capture/mac/screen_capturer_sck.mm
@@ -169,6 +169,12 @@ class API_AVAILABLE(macos(14.0)) ScreenCapturerSck final : public DesktopCapture
// TODO: crbug.com/327458809 - Replace this flag with ScreenCapturerHelper to more accurately
// track the dirty rectangles from the SCStreamFrameInfoDirtyRects attachment.
bool frame_is_dirty_ RTC_GUARDED_BY(latest_frame_lock_) = true;
+
+ // Tracks whether a reconfigure is needed.
+ bool frame_needs_reconfigure_ RTC_GUARDED_BY(latest_frame_lock_) = false;
+ // If a reconfigure is needed, this will be set to the size in pixels required to fit the entire
+ // source without downscaling.
+ std::optional<CGSize> frame_reconfigure_img_size_ RTC_GUARDED_BY(latest_frame_lock_);
};
ScreenCapturerSck::ScreenCapturerSck(const DesktopCaptureOptions& options)
@@ -240,6 +246,7 @@ void ScreenCapturerSck::CaptureFrame() {
}
std::unique_ptr<DesktopFrame> frame;
+ bool needs_reconfigure = false;
{
MutexLock lock(&latest_frame_lock_);
if (latest_frame_) {
@@ -250,6 +257,8 @@ void ScreenCapturerSck::CaptureFrame() {
frame_is_dirty_ = false;
}
}
+ needs_reconfigure = frame_needs_reconfigure_;
+ frame_needs_reconfigure_ = false;
}
if (frame) {
@@ -260,6 +269,10 @@ void ScreenCapturerSck::CaptureFrame() {
RTC_LOG(LS_VERBOSE) << "ScreenCapturerSck " << this << " CaptureFrame() -> ERROR_TEMPORARY";
callback_->OnCaptureResult(Result::ERROR_TEMPORARY, nullptr);
}
+
+ if (needs_reconfigure) {
+ StartOrReconfigureCapturer();
+ }
}
void ScreenCapturerSck::EnsureVisible() {
@@ -283,6 +296,9 @@ void ScreenCapturerSck::EnsureVisible() {
stream = stream_;
stream_ = nil;
filter_ = nil;
+ MutexLock lock2(&latest_frame_lock_);
+ frame_needs_reconfigure_ = false;
+ frame_reconfigure_img_size_ = std::nullopt;
}
[stream removeStreamOutput:helper_ type:SCStreamOutputTypeScreen error:nil];
[stream stopCaptureWithCompletionHandler:nil];
@@ -439,12 +455,21 @@ void ScreenCapturerSck::StartWithFilter(SCContentFilter* __strong filter) {
{
MutexLock lock(&latest_frame_lock_);
latest_frame_dpi_ = filter.pointPixelScale * kStandardDPI;
+ if (filter_ != filter) {
+ frame_reconfigure_img_size_ = std::nullopt;
+ }
+ auto sourceImgRect = frame_reconfigure_img_size_.value_or(CGSizeMake(
+ filter.contentRect.size.width * filter.pointPixelScale,
+ filter.contentRect.size.height * filter.pointPixelScale));
+ config.width = sourceImgRect.width;
+ config.height = sourceImgRect.height;
}
filter_ = filter;
if (stream_) {
- RTC_LOG(LS_INFO) << "ScreenCapturerSck " << this << " Updating stream configuration.";
+ RTC_LOG(LS_INFO) << "ScreenCapturerSck " << this << " Updating stream configuration to size="
+ << config.width << "x" << config.height << ".";
[stream_ updateContentFilter:filter completionHandler:nil];
[stream_ updateConfiguration:config completionHandler:nil];
} else {
@@ -481,20 +506,63 @@ void ScreenCapturerSck::StartWithFilter(SCContentFilter* __strong filter) {
}
void ScreenCapturerSck::OnNewIOSurface(IOSurfaceRef io_surface, NSDictionary* attachment) {
- RTC_LOG(LS_VERBOSE) << "ScreenCapturerSck " << this << " " << __func__
- << " width=" << IOSurfaceGetWidth(io_surface)
- << ", height=" << IOSurfaceGetHeight(io_surface) << ".";
-
+ double scaleFactor = 1;
+ double contentScale = 1;
+ CGRect contentRect = {};
+ CGRect boundingRect = {};
+ CGRect overlayRect = {};
const auto* dirty_rects = (NSArray*)attachment[SCStreamFrameInfoDirtyRects];
+ if (auto factor = (NSNumber *)attachment[SCStreamFrameInfoScaleFactor]) {
+ scaleFactor = [factor floatValue];
+ }
+ if (auto scale = (NSNumber *)attachment[SCStreamFrameInfoContentScale]) {
+ contentScale = [scale floatValue];
+ }
+ if (const auto* rectDict = (__bridge CFDictionaryRef)attachment[SCStreamFrameInfoContentRect]) {
+ if (!CGRectMakeWithDictionaryRepresentation(rectDict, &contentRect)) {
+ contentRect = CGRect();
+ }
+ }
+ if (const auto* rectDict = (__bridge CFDictionaryRef)attachment[SCStreamFrameInfoBoundingRect]) {
+ if (!CGRectMakeWithDictionaryRepresentation(rectDict, &boundingRect)) {
+ boundingRect = CGRect();
+ }
+ }
+ if (@available(macOS 14.2, *)) {
+ if (const auto* rectDict = (__bridge CFDictionaryRef)attachment[SCStreamFrameInfoPresenterOverlayContentRect ]) {
+ if (!CGRectMakeWithDictionaryRepresentation(rectDict, &overlayRect)) {
+ overlayRect = CGRect();
+ }
+ }
+ }
+
+
+ auto imgBoundingRect = CGRectMake(
+ scaleFactor * boundingRect.origin.x,
+ scaleFactor * boundingRect.origin.y,
+ scaleFactor * boundingRect.size.width,
+ scaleFactor * boundingRect.size.height);
rtc::ScopedCFTypeRef<IOSurfaceRef> scoped_io_surface(io_surface, rtc::RetainPolicy::RETAIN);
std::unique_ptr<DesktopFrameIOSurface> desktop_frame_io_surface =
- DesktopFrameIOSurface::Wrap(scoped_io_surface);
+ DesktopFrameIOSurface::Wrap(scoped_io_surface, imgBoundingRect);
if (!desktop_frame_io_surface) {
RTC_LOG(LS_ERROR) << "Failed to lock IOSurface.";
return;
}
+ const size_t width = IOSurfaceGetWidth(io_surface);
+ const size_t height = IOSurfaceGetHeight(io_surface);
+
+ RTC_LOG(LS_VERBOSE) << "ScreenCapturerSck " << this << " " << __func__
+ << ". New surface: width=" << width << ", height=" << height
+ << ", contentRect=" << NSStringFromRect(contentRect).UTF8String
+ << ", boundingRect=" << NSStringFromRect(boundingRect).UTF8String
+ << ", overlayRect=(" << NSStringFromRect(overlayRect).UTF8String
+ << ", scaleFactor=" << scaleFactor << ", contentScale=" << contentScale
+ << ". Cropping to rect " << NSStringFromRect(imgBoundingRect).UTF8String
+ << ".";
+
std::unique_ptr<SharedDesktopFrame> frame =
SharedDesktopFrame::Wrap(std::move(desktop_frame_io_surface));
@@ -529,8 +597,15 @@ void ScreenCapturerSck::OnNewIOSurface(IOSurfaceRef io_surface, NSDictionary* at
}
}
+ MutexLock lock(&latest_frame_lock_);
+ if (contentScale > 0 && contentScale < 1) {
+ frame_needs_reconfigure_ = true;
+ double scale = 1 / contentScale;
+ frame_reconfigure_img_size_ = CGSizeMake(
+ std::ceil(scale * width),
+ std::ceil(scale * height));
+ }
if (dirty) {
- MutexLock lock(&latest_frame_lock_);
frame_is_dirty_ = true;
std::swap(latest_frame_, frame);
}