diff --git a/cmake/compile_definitions/macos.cmake b/cmake/compile_definitions/macos.cmake index bb478dcfe8c..97d5c03a5d8 100644 --- a/cmake/compile_definitions/macos.cmake +++ b/cmake/compile_definitions/macos.cmake @@ -37,6 +37,7 @@ list(APPEND SUNSHINE_EXTERNAL_LIBRARIES ${FOUNDATION_LIBRARY} ${IOKIT_LIBRARY} ${USER_NOTIFICATIONS_LIBRARY} + ${SCREEN_CAPTURE_KIT_LIBRARY} ${VIDEO_TOOLBOX_LIBRARY}) set(APPLE_PLIST_TEMPLATE "${SUNSHINE_SOURCE_ASSETS_DIR}/macos/build/Info.plist.in") @@ -58,6 +59,10 @@ set(PLATFORM_TARGET_FILES "${CMAKE_SOURCE_DIR}/src/platform/macos/nv12_zero_device.cpp" "${CMAKE_SOURCE_DIR}/src/platform/macos/nv12_zero_device.h" "${CMAKE_SOURCE_DIR}/src/platform/macos/publish.cpp" + "${CMAKE_SOURCE_DIR}/src/platform/macos/sc_capture.h" + "${CMAKE_SOURCE_DIR}/src/platform/macos/sc_capture.mm" + "${CMAKE_SOURCE_DIR}/src/platform/macos/sc_capture_loop.cpp" + "${CMAKE_SOURCE_DIR}/src/platform/macos/sc_capture_loop.h" "${CMAKE_SOURCE_DIR}/third-party/TPCircularBuffer/TPCircularBuffer.c" "${CMAKE_SOURCE_DIR}/third-party/TPCircularBuffer/TPCircularBuffer.h" ${APPLE_ENTITLEMENTS_FILE} diff --git a/cmake/dependencies/macos.cmake b/cmake/dependencies/macos.cmake index 3dc396fe27c..85eda37d21b 100644 --- a/cmake/dependencies/macos.cmake +++ b/cmake/dependencies/macos.cmake @@ -11,6 +11,7 @@ FIND_LIBRARY(CORE_VIDEO_LIBRARY CoreVideo) FIND_LIBRARY(FOUNDATION_LIBRARY Foundation) FIND_LIBRARY(IOKIT_LIBRARY IOKit) FIND_LIBRARY(USER_NOTIFICATIONS_LIBRARY UserNotifications) +FIND_LIBRARY(SCREEN_CAPTURE_KIT_LIBRARY ScreenCaptureKit) FIND_LIBRARY(VIDEO_TOOLBOX_LIBRARY VideoToolbox) if(SUNSHINE_ENABLE_TRAY) diff --git a/src/platform/macos/display.mm b/src/platform/macos/display.mm index 4aec4ebedd5..72fbf0280ad 100644 --- a/src/platform/macos/display.mm +++ b/src/platform/macos/display.mm @@ -4,11 +4,14 @@ */ // standard includes +#include #include #include #include +#include #include #include +#include #include // local includes @@ -20,6 +23,9 @@ #include "src/platform/macos/av_video.h" #include "src/platform/macos/misc.h" #include "src/platform/macos/nv12_zero_device.h" +#import "src/platform/macos/sc_capture.h" +#include "src/platform/macos/sc_capture_loop.h" +#include "src/utility.h" // Avoid conflict between AVFoundation and libavutil both defining AVMediaType /** @@ -78,6 +84,218 @@ OSType videotoolbox_pixel_format(const video::config_t &config) { const auto colorspace {video::colorspace_from_client_config(config, false)}; return colorspace.bit_depth == 10 ? kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; } + + /** + * @brief Derive a steady_clock capture timestamp from a sample buffer's presentation timestamp. + * + * The PTS is on the mach host clock, so back-dating steady_clock::now() by the frame's age on + * that clock includes AVFoundation's internal queueing delay in the reported frame latency. + * @param sample_buffer Sample buffer received from AVFoundation. + * @return Capture timestamp for the frame. + */ + std::chrono::steady_clock::time_point frame_timestamp_from_pts(CMSampleBufferRef sample_buffer) { + auto timestamp = std::chrono::steady_clock::now(); + const CMTime pts = CMSampleBufferGetPresentationTimeStamp(sample_buffer); + if (CMTIME_IS_NUMERIC(pts)) { + const CMTime frame_age = CMTimeSubtract(CMClockGetTime(CMClockGetHostTimeClock()), pts); + const double frame_age_s = CMTimeGetSeconds(frame_age); + if (frame_age_s > 0) { + timestamp -= std::chrono::duration_cast(std::chrono::duration(frame_age_s)); + } + } + return timestamp; + } + + // Upper bound on how long the ScreenCaptureKit capture loop waits for a frame + // before reporting a timeout and checking for interrupt requests again. + constexpr auto SCKIT_FRAME_POLL_INTERVAL = std::chrono::nanoseconds {std::chrono::seconds {1}} / 60; + + /** + * @brief Wrap a captured sample buffer into a Sunshine image. + * + * @param sample_buffer Sample buffer received from the capture backend. + * @param img Image object to populate. + * @return `true` when the sample buffer contained a usable pixel buffer. + */ + bool process_frame(CMSampleBufferRef sample_buffer, img_t *img) { + auto pixel_buffer = CMSampleBufferGetImageBuffer(sample_buffer); + if (!pixel_buffer) { + return false; + } + + auto new_sample_buffer = std::make_shared(sample_buffer); + auto new_pixel_buffer = std::make_shared(new_sample_buffer->buf); + + auto av_img = (av_img_t *) img; + + auto old_data_retainer = std::make_shared( + av_img->sample_buffer, + av_img->pixel_buffer, + img->data + ); + + av_img->sample_buffer = new_sample_buffer; + av_img->pixel_buffer = new_pixel_buffer; + img->data = new_pixel_buffer->data(); + + img->width = (int) CVPixelBufferGetWidth(new_pixel_buffer->buf); + img->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf); + img->row_pitch = CVPixelBufferIsPlanar(new_pixel_buffer->buf) ? + (int) CVPixelBufferGetBytesPerRowOfPlane(new_pixel_buffer->buf, 0) : + (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf); + img->pixel_pitch = img->row_pitch / img->width; + img->frame_timestamp = frame_timestamp_from_pts(sample_buffer); + + old_data_retainer = nullptr; + return true; + } + + /** + * @brief Zero-fill all planes of a pixel buffer. + * + * @param pixel_buffer Pixel buffer to clear. + */ + void clear_pixel_buffer(CVPixelBufferRef pixel_buffer) { + CVPixelBufferLockBaseAddress(pixel_buffer, 0); + + if (CVPixelBufferIsPlanar(pixel_buffer)) { + for (size_t plane = 0; plane < CVPixelBufferGetPlaneCount(pixel_buffer); ++plane) { + auto *base = static_cast(CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, plane)); + auto bytes_per_row = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, plane); + auto height = CVPixelBufferGetHeightOfPlane(pixel_buffer, plane); + std::memset(base, 0, bytes_per_row * height); + } + } else { + auto *base = static_cast(CVPixelBufferGetBaseAddress(pixel_buffer)); + std::memset(base, 0, CVPixelBufferGetBytesPerRow(pixel_buffer) * CVPixelBufferGetHeight(pixel_buffer)); + } + + CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); + } + + /** + * @brief Synthesize a blank frame for capture backends that only deliver frames on screen changes. + * + * @param img Image object to populate. + * @param width Frame width in pixels. + * @param height Frame height in pixels. + * @param pixel_format Pixel format of the synthesized frame. + * @param backend_name Capture backend name used in log messages. + * @return 0 on success; nonzero on failure. + */ + int make_dummy_img(img_t *img, int width, int height, OSType pixel_format, std::string_view backend_name) { + CVPixelBufferRef pixel_buffer = nullptr; + NSDictionary *attrs = @{ + (NSString *) kCVPixelBufferIOSurfacePropertiesKey: @ {}, + }; + + auto status = CVPixelBufferCreate( + kCFAllocatorDefault, + width, + height, + pixel_format, + (__bridge CFDictionaryRef) attrs, + &pixel_buffer + ); + + if (status != kCVReturnSuccess || !pixel_buffer) { + BOOST_LOG(error) << backend_name << " dummy_img: failed to create pixel buffer"sv; + return 1; + } + + clear_pixel_buffer(pixel_buffer); + + CMVideoFormatDescriptionRef format_desc = nullptr; + status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixel_buffer, &format_desc); + if (status != noErr || !format_desc) { + CVPixelBufferRelease(pixel_buffer); + BOOST_LOG(error) << backend_name << " dummy_img: failed to create format description"sv; + return 1; + } + + CMSampleTimingInfo timing = {kCMTimeInvalid, kCMTimeInvalid, kCMTimeInvalid}; + CMSampleBufferRef sample_buffer = nullptr; + status = CMSampleBufferCreateForImageBuffer(kCFAllocatorDefault, pixel_buffer, YES, nullptr, nullptr, format_desc, &timing, &sample_buffer); + CFRelease(format_desc); + + if (status != noErr || !sample_buffer) { + CVPixelBufferRelease(pixel_buffer); + BOOST_LOG(error) << backend_name << " dummy_img: failed to create sample buffer"sv; + return 1; + } + + auto ret = process_frame(sample_buffer, img) ? 0 : 1; + CFRelease(sample_buffer); + CVPixelBufferRelease(pixel_buffer); + + return ret; + } + + /** + * @brief Frame source adapter exposing an SCCapture controller to the capture loop. + */ + class sc_frame_source_t: public sckit::frame_source_t { + public: + /** + * @brief Wrap a capture controller. + * + * @param capture Controller owned by the display; it must outlive the adapter. + */ + explicit sc_frame_source_t(SCCapture *capture): + capture(capture) { + } + + ~sc_frame_source_t() override { + release_frame_signal(); + } + + bool start() override { + if (![capture captureVideo]) { + BOOST_LOG(error) << "SCCapture failed to start the capture stream"sv; + return false; + } + + frame_signal = capture.frameSignal; + if (!frame_signal) { + BOOST_LOG(error) << "SCCapture failed to create frame signal"sv; + [capture stopCapture]; + return false; + } + dispatch_retain(frame_signal); + return true; + } + + void wait_frame(std::chrono::nanoseconds timeout) override { + dispatch_semaphore_wait(frame_signal, dispatch_time(DISPATCH_TIME_NOW, timeout.count())); + } + + [[nodiscard]] bool stopped() const override { + return capture.streamFailed; + } + + CMSampleBufferRef copy_latest_sample_buffer() override { + return [capture copyLatestSampleBuffer]; + } + + void stop() override { + [capture stopCapture]; + release_frame_signal(); + } + + private: + /** + * @brief Drop the retained frame semaphore, if any. + */ + void release_frame_signal() { + if (frame_signal) { + dispatch_release(frame_signal); + frame_signal = nullptr; + } + } + + SCCapture *capture; ///< Borrowed capture controller. + dispatch_semaphore_t frame_signal {}; ///< Frame-arrival semaphore, retained while the stream runs. + }; } // namespace /** @@ -94,6 +312,7 @@ OSType videotoolbox_pixel_format(const video::config_t &config) { capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override { auto signal = [av_capture capture:^(CMSampleBufferRef sampleBuffer) { + const auto frame_timestamp = frame_timestamp_from_pts(sampleBuffer); auto new_sample_buffer = std::make_shared(sampleBuffer); auto new_pixel_buffer = std::make_shared(new_sample_buffer->buf); @@ -119,6 +338,7 @@ capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const img_out->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf); img_out->row_pitch = (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf); img_out->pixel_pitch = img_out->row_pitch / img_out->width; + img_out->frame_timestamp = frame_timestamp; old_data_retainer = nullptr; @@ -160,7 +380,16 @@ capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const } else if (pix_fmt == pix_fmt_e::nv12 || pix_fmt == pix_fmt_e::p010) { auto device = std::make_unique(); - device->init(av_capture, pix_fmt, setResolution, setPixelFormat); + auto *capture = av_capture; + device->init( + pix_fmt, + [capture](int width, int height) { + [capture setFrameWidth:width frameHeight:height]; + }, + [capture](int pixel_format) { + capture.pixelFormat = pixel_format; + } + ); return device; } else { @@ -214,29 +443,92 @@ int dummy_img(img_t *img) override { return 0; } + }; + + /** + * @brief ScreenCaptureKit-based macOS display capture source. + */ + struct sc_display_t: public display_t { + SCCapture *sc_capture {}; ///< ScreenCaptureKit capture controller. + CGDirectDisplayID display_id {}; ///< Display ID. + std::unique_ptr display_power_guard; ///< Display power guard. + + ~sc_display_t() override { + [sc_capture stopCapture]; + [sc_capture release]; + } + + capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override { + sc_frame_source_t source {sc_capture}; + auto &policy = sckit::backend_policy(); + + const auto status = sckit::run_capture_loop(source, policy, push_captured_image_cb, pull_free_image_cb, process_frame, SCKIT_FRAME_POLL_INTERVAL); + if (status == capture_e::reinit) { + if (policy.stream_start_failed()) { + BOOST_LOG(warning) << "SCCapture stream could not be started; reinitializing capture on AVFoundation"sv; + } else { + BOOST_LOG(warning) << "SCCapture stream stopped unexpectedly; reinitializing capture"sv; + } + } + return status; + } /** - * A bridge from the pure C++ code of the hwdevice_t class to the pure Objective C code. + * @brief Allocate an image buffer compatible with this display backend. * - * display --> an opaque pointer to an object of this class - * width --> the intended capture width - * height --> the intended capture height - * @param display Display object or identifier associated with the operation. - * @param width Frame or display width in pixels. - * @param height Frame or display height in pixels. + * @return Allocated img object, or null when unavailable. */ - static void setResolution(AVVideo *display, int width, int height) { - [display setFrameWidth:width frameHeight:height]; + std::shared_ptr alloc_img() override { + return std::make_shared(); } /** - * @brief Set pixel format. + * @brief Create AVCodec encode device. * - * @param display Display object or identifier associated with the operation. - * @param pixelFormat Pixel format. + * @param pix_fmt Sunshine pixel format to convert or allocate for. + * @return Constructed AVCodec encode device object. */ - static void setPixelFormat(AVVideo *display, OSType pixelFormat) { - display.pixelFormat = pixelFormat; + std::unique_ptr make_avcodec_encode_device(pix_fmt_e pix_fmt) override { + if (pix_fmt == pix_fmt_e::yuv420p) { + sc_capture.pixelFormat = kCVPixelFormatType_32BGRA; + + return std::make_unique(); + } else if (pix_fmt == pix_fmt_e::nv12 || pix_fmt == pix_fmt_e::p010) { + auto device = std::make_unique(); + + auto *capture = sc_capture; + device->init( + pix_fmt, + [capture](int width, int height) { + [capture setFrameWidth:width frameHeight:height]; + }, + [capture](int pixel_format) { + capture.pixelFormat = pixel_format; + } + ); + + return device; + } else { + BOOST_LOG(error) << "Unsupported Pixel Format."sv; + return nullptr; + } + } + + /** + * @brief Populate a fallback image when real capture data is unavailable. + * + * @param img Image or frame object to read from or populate. + * @return Capture status reported to the streaming pipeline. + */ + int dummy_img(img_t *img) override { + if (!platf::is_screen_capture_allowed()) { + // A non-zero return value indicates failure to the calling function. + return 1; + } + + // ScreenCaptureKit only delivers frames on screen updates, so synthesize a blank frame + // instead of waiting for one. + return make_dummy_img(img, sc_capture.frameWidth, sc_capture.frameHeight, sc_capture.pixelFormat, "SCCapture"sv); } }; @@ -246,29 +538,27 @@ static void setPixelFormat(AVVideo *display, OSType pixelFormat) { return nullptr; } - auto display = std::make_shared(); - BOOST_LOG(debug) << "Waking display for capture selector ["sv << display_name << ']'; if (!display_device::wake_display(display_name, 1s)) { BOOST_LOG(debug) << "Display wake attempt did not expose the requested display ["sv << display_name << ']'; } - display->display_power_guard = display_device::keep_display_awake("Sunshine display capture"); - if (display->display_power_guard) { + auto display_power_guard = display_device::keep_display_awake("Sunshine display capture"); + if (display_power_guard) { BOOST_LOG(debug) << "Keeping display awake for capture"sv; } else { BOOST_LOG(debug) << "Unable to create display sleep prevention assertion"sv; } // Default to main display - display->display_id = CGMainDisplayID(); + CGDirectDisplayID display_id = CGMainDisplayID(); if (const auto configured_display_id {parse_display_id(display_name)}) { - display->display_id = *configured_display_id; + display_id = *configured_display_id; } else if (!display_name.empty()) { BOOST_LOG(warning) << "Configured display ["sv << display_name << "] is not a valid macOS capture display id. Falling back to main display ["sv - << display->display_id << "]."sv; + << display_id << "]."sv; } // Print all displays available with their names and ids @@ -282,7 +572,44 @@ static void setPixelFormat(AVVideo *display, OSType pixelFormat) { << " (id: "sv << device.m_display_name << ") connected: true"sv; } - BOOST_LOG(info) << "Configuring selected display ("sv << display->display_id << ") to stream"sv; + BOOST_LOG(info) << "Configuring selected display ("sv << display_id << ") to stream"sv; + + // Prefer ScreenCaptureKit: it composites the cursor into frames server-side, which avoids + // an AVCaptureScreenInput bug where a hidden cursor never reappears in the captured frames. + if (@available(macOS 14.0, *)) { + auto &policy = sckit::backend_policy(); + if (policy.stream_start_failed()) { + BOOST_LOG(info) << "ScreenCaptureKit disabled after a stream start failure; using AVFoundation"sv; + } else if (policy.prefer_sckit([SCCapture isAvailable])) { + auto display = std::make_shared(); + display->display_id = display_id; + display->display_power_guard = std::move(display_power_guard); + display->sc_capture = [[SCCapture alloc] initWithDisplay:display_id frameRate:config.framerate]; + + if (display->sc_capture) { + display->width = display->sc_capture.frameWidth; + display->height = display->sc_capture.frameHeight; + // We also need set env_width and env_height for absolute mouse coordinates + display->env_width = display->width; + display->env_height = display->height; + + if (hwdevice_type == platf::mem_type_e::videotoolbox) { + const auto pixel_format {videotoolbox_pixel_format(config)}; + [display->sc_capture setFrameWidth:config.width frameHeight:config.height]; + display->sc_capture.pixelFormat = pixel_format; + } + + return display; + } + + display_power_guard = std::move(display->display_power_guard); + BOOST_LOG(error) << "SCCapture setup failed, trying AVFoundation..."sv; + } + } + + auto display = std::make_shared(); + display->display_id = display_id; + display->display_power_guard = std::move(display_power_guard); const auto selected_display_bounds = CGDisplayBounds(display->display_id); const auto desktop_bounds = active_desktop_bounds(display->display_id); diff --git a/src/platform/macos/nv12_zero_device.cpp b/src/platform/macos/nv12_zero_device.cpp index 80b98e60a1b..cde7955b208 100644 --- a/src/platform/macos/nv12_zero_device.cpp +++ b/src/platform/macos/nv12_zero_device.cpp @@ -59,15 +59,14 @@ namespace platf { av_frame.reset(new_frame); - resolution_fn(this->display, new_frame->width, new_frame->height); + resolution_fn(new_frame->width, new_frame->height); return 0; } - int nv12_zero_device::init(AVVideo *capture_display, pix_fmt_e pix_fmt, resolution_fn_t set_resolution, const pixel_format_fn_t &pixel_format_fn) { - pixel_format_fn(capture_display, pix_fmt == pix_fmt_e::nv12 ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange : kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange); + int nv12_zero_device::init(pix_fmt_e pix_fmt, resolution_fn_t set_resolution, const pixel_format_fn_t &pixel_format_fn) { + pixel_format_fn(pix_fmt == pix_fmt_e::nv12 ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange : kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange); - this->display = capture_display; this->resolution_fn = std::move(set_resolution); hardware = true; diff --git a/src/platform/macos/nv12_zero_device.h b/src/platform/macos/nv12_zero_device.h index 693ab6f4f7d..c3fc17c9c5c 100644 --- a/src/platform/macos/nv12_zero_device.h +++ b/src/platform/macos/nv12_zero_device.h @@ -9,15 +9,6 @@ struct AVFrame; -#ifdef __OBJC__ -@class AVVideo; -#else -/** - * @brief Opaque AVFoundation capture controller, defined in Objective-C. - */ -struct AVVideo; -#endif - namespace platf { /** * @brief Release an FFmpeg frame allocated by the capture or conversion backend. @@ -27,34 +18,31 @@ namespace platf { void free_frame(AVFrame *frame); /** - * @brief macOS zero-copy encode device that forwards NV12 frames from AVFoundation to FFmpeg. + * @brief macOS zero-copy encode device that forwards NV12 frames from the capture backend to FFmpeg. + * + * The capture controller (AVFoundation or ScreenCaptureKit) is Objective-C and its headers + * collide with FFmpeg's, so the device never sees it: the backend binds it into the callbacks. */ class nv12_zero_device: public avcodec_encode_device_t { - AVVideo *display = nullptr; ///< Borrowed AVFoundation controller, forward-declared to avoid FFmpeg name conflicts. - public: - // this function is used to set the resolution on an av_video object that we cannot - // call directly because of namespace collisions between AVFoundation and FFMPEG /** - * @brief Callback signature used to update the opaque AVFoundation display resolution. + * @brief Callback signature used to resize the capture output. */ - using resolution_fn_t = std::function; - resolution_fn_t resolution_fn; ///< Callback stored for later AVFoundation resolution updates. + using resolution_fn_t = std::function; + resolution_fn_t resolution_fn; ///< Callback invoked with the encoder's frame size when a frame is attached. /** - * @brief Callback signature used to update the opaque AVFoundation pixel format. + * @brief Callback signature used to select the capture pixel format. */ - using pixel_format_fn_t = std::function; - + using pixel_format_fn_t = std::function; /** - * @brief Initialize zero-copy NV12 encoding for an AVFoundation display. + * @brief Initialize zero-copy NV12 encoding for a capture backend. * - * @param capture_display Display object or identifier associated with the operation. * @param pix_fmt Sunshine pixel format to convert or allocate for. - * @param set_resolution Callback used to resize the AVFoundation capture output. - * @param pixel_format_fn Pixel format. + * @param set_resolution Callback used to resize the capture output. + * @param pixel_format_fn Callback used to select the capture pixel format; invoked immediately. * @return 0 on success; nonzero or negative platform status on failure. */ - int init(AVVideo *capture_display, pix_fmt_e pix_fmt, resolution_fn_t set_resolution, const pixel_format_fn_t &pixel_format_fn); + int init(pix_fmt_e pix_fmt, resolution_fn_t set_resolution, const pixel_format_fn_t &pixel_format_fn); /** * @brief Forward a captured AVFoundation frame to FFmpeg without copying. diff --git a/src/platform/macos/sc_capture.h b/src/platform/macos/sc_capture.h new file mode 100644 index 00000000000..51cb93c7091 --- /dev/null +++ b/src/platform/macos/sc_capture.h @@ -0,0 +1,149 @@ +/** + * @file src/platform/macos/sc_capture.h + * @brief Declarations for ScreenCaptureKit-based display capture on macOS. + */ +#pragma once + +// platform includes +#import +#import +#import +#import + +/** + * @brief ScreenCaptureKit display capture controller used by the macOS backend. + * + * Frames are delivered by an SCStream and latched into a single latest-wins slot; the + * capture loop consumes the newest frame and the encoder duplicates the previous frame + * when the screen is static. Delivery reliability depends on configuration discipline + * (matching OBS's): `minimumFrameInterval` must sit ~10% below the target interval or + * frames get suppressed by clock beating, the pipeline must hold well fewer sample + * buffers at once than `queueDepth` (capped at 8 by the API) or WindowServer stalls + * delivery until one is returned, and frames must never be filtered by SCFrameStatus. + */ +API_AVAILABLE(macos(14.0)) +@interface SCCapture: NSObject + +/** + * @def kMaxDisplays + * @brief Maximum number of displays queried from CoreGraphics. + */ +#define kMaxDisplays 32 + +/** + * @brief Display ID property. + */ +@property (nonatomic, assign) CGDirectDisplayID displayID; +/** + * @brief Capture frame rate property. + */ +@property (nonatomic, assign) int frameRate; +/** + * @brief Output pixel format property. + */ +@property (nonatomic, assign) OSType pixelFormat; +/** + * @brief Output frame width property. + */ +@property (nonatomic, assign) int frameWidth; +/** + * @brief Output frame height property. + */ +@property (nonatomic, assign) int frameHeight; + +/** + * @brief Content filter selecting the captured display. + */ +@property (nonatomic, strong) SCContentFilter *contentFilter; +/** + * @brief Stream configuration used for the capture stream. + */ +@property (nonatomic, strong) SCStreamConfiguration *streamConfiguration; +/** + * @brief Active capture stream delivering frames, or nil when stopped. + */ +@property (nonatomic, strong) SCStream *stream; +/** + * @brief Cached shareable content used for display lookup. + */ +@property (nonatomic, strong) SCShareableContent *shareableContent; + +/** + * @brief Semaphore signalled when capture stops. + */ +@property (nonatomic, assign) dispatch_semaphore_t captureSignal; +/** + * @brief Semaphore signalled when a new frame is available. + */ +@property (nonatomic, assign) dispatch_semaphore_t frameSignal; +/** + * @brief Whether the capture is shutting down. + */ +@property (nonatomic, assign) BOOL stopping; +/** + * @brief Whether the stream stopped on its own since capture started. + */ +@property (nonatomic, assign) BOOL streamFailed; +/** + * @brief Latest captured sample buffer, replaced as newer frames arrive. + */ +@property (nonatomic, assign) CMSampleBufferRef latestSampleBuffer; + +/** + * @brief Check whether ScreenCaptureKit screenshot capture is available on this system. + * + * @return `YES` when the running macOS version supports the capture backend. + */ ++ (BOOL)isAvailable; +/** + * @brief Enumerate active displays. + * + * @return Array of dictionaries describing each active display. + */ ++ (NSArray *)displayNames; +/** + * @brief Look up the localized name of a display. + * + * @param displayID Display to look up. + * @return Localized display name, or nil when unknown. + */ ++ (NSString *)getDisplayName:(CGDirectDisplayID)displayID; + +/** + * @brief Initialize capture for a display. + * + * @param displayID Display to capture. + * @param frameRate Requested capture frame rate. + * @return Initialized instance, or nil on failure. + */ +- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID + frameRate:(int)frameRate; + +/** + * @brief Set the capture output dimensions. + * + * @param frameWidth Output frame width in pixels. + * @param frameHeight Output frame height in pixels. + */ +- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight; +/** + * @brief Create, configure, and start the capture stream. + * + * When a 10-bit capture format is rejected by the runtime, capture falls back to + * 8-bit and the encoder continues the negotiated session with 8-bit-sourced frames. + * + * @return Semaphore signalled when capture stops, or nil on failure. + */ +- (dispatch_semaphore_t)captureVideo; +/** + * @brief Take ownership of the most recent captured frame. + * + * @return Retained sample buffer, or NULL when no new frame is available. + */ +- (CMSampleBufferRef)copyLatestSampleBuffer; +/** + * @brief Stop capture and release capture state. + */ +- (void)stopCapture; + +@end diff --git a/src/platform/macos/sc_capture.mm b/src/platform/macos/sc_capture.mm new file mode 100644 index 00000000000..8cbf90e2ed0 --- /dev/null +++ b/src/platform/macos/sc_capture.mm @@ -0,0 +1,377 @@ +/** + * @file src/platform/macos/sc_capture.mm + * @brief ScreenCaptureKit-based display capture implementation. + */ +#import "sc_capture.h" + +// local includes +#include "src/logging.h" + +using namespace std::literals; + +/** + * @brief Describe an NSError for logging. + * + * @param error Error to describe, or nil. + * @return UTF-8 description of the error. + */ +static const char *error_description(NSError *error) { + return error && error.localizedDescription ? error.localizedDescription.UTF8String : "unknown error"; +} + +static SCShareableContent *copyShareableContent(void) { + __block SCShareableContent *shareableContent = nil; + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + + [SCShareableContent getShareableContentWithCompletionHandler:^(SCShareableContent *content, NSError *content_error) { + if (content_error) { + BOOST_LOG(error) << "SCCapture failed to get shareable content: "sv << error_description(content_error); + } else { + shareableContent = [content retain]; + } + + dispatch_semaphore_signal(semaphore); + }]; + + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + dispatch_release(semaphore); + + return shareableContent; +} + +static BOOL isUsableImageSampleBuffer(CMSampleBufferRef sampleBuffer) { + return sampleBuffer && CMSampleBufferIsValid(sampleBuffer) && CMSampleBufferGetImageBuffer(sampleBuffer); +} + +API_AVAILABLE(macos(14.0)) +@implementation SCCapture + ++ (BOOL)isAvailable { + if (@available(macOS 14.0, *)) { + return YES; + } + return NO; +} + ++ (NSArray *)displayNames { + CGDirectDisplayID displays[kMaxDisplays]; + uint32_t count; + if (CGGetActiveDisplayList(kMaxDisplays, displays, &count) != kCGErrorSuccess) { + return [NSArray array]; + } + + NSMutableArray *result = [NSMutableArray array]; + + for (uint32_t i = 0; i < count; i++) { + [result addObject:@{ + @"id": [NSNumber numberWithUnsignedInt:displays[i]], + @"name": [NSString stringWithFormat:@"%d", displays[i]], + @"displayName": [self getDisplayName:displays[i]] ?: @"Unknown Display", + }]; + } + + return [NSArray arrayWithArray:result]; +} + ++ (NSString *)getDisplayName:(CGDirectDisplayID)displayID { + for (NSScreen *screen in [NSScreen screens]) { + if ([screen.deviceDescription[@"NSScreenNumber"] isEqualToNumber:[NSNumber numberWithUnsignedInt:displayID]]) { + return screen.localizedName; + } + } + + return nil; +} + +- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID + frameRate:(int)frameRate { + self = [super init]; + if (self) { + CGDisplayModeRef mode = CGDisplayCopyDisplayMode(displayID); + + self.displayID = displayID; + self.frameRate = frameRate; + self.pixelFormat = kCVPixelFormatType_32BGRA; + + if (mode) { + self.frameWidth = (int) CGDisplayModeGetPixelWidth(mode); + self.frameHeight = (int) CGDisplayModeGetPixelHeight(mode); + CFRelease(mode); + } else { + self.frameWidth = (int) CGDisplayPixelsWide(displayID); + self.frameHeight = (int) CGDisplayPixelsHigh(displayID); + } + + SCShareableContent *content = copyShareableContent(); + if (!content) { + [self release]; + return nil; + } + + self.shareableContent = content; + [content release]; + } + + return self; +} + +- (void)dealloc { + [self stopCapture]; + self.shareableContent = nil; + + [super dealloc]; +} + +- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight { + self.frameWidth = frameWidth; + self.frameHeight = frameHeight; +} + +- (SCDisplay *)findDisplayWithID:(CGDirectDisplayID)displayID { + for (SCDisplay *display in self.shareableContent.displays) { + if (display.displayID == displayID) { + return display; + } + } + + return nil; +} + +- (SCDisplay *)findDisplayWithIDRetrying:(CGDirectDisplayID)displayID { + SCDisplay *display = [self findDisplayWithID:displayID]; + if (display) { + return display; + } + + for (int attempt = 1; attempt <= 3; attempt++) { + BOOST_LOG(warning) << "SCCapture display "sv << displayID << " not found in SCShareableContent, refreshing (attempt "sv << attempt << "/3)"sv; + [NSThread sleepForTimeInterval:1.0]; + + SCShareableContent *content = copyShareableContent(); + if (!content) { + continue; + } + + self.shareableContent = content; + [content release]; + + display = [self findDisplayWithID:displayID]; + if (display) { + BOOST_LOG(info) << "SCCapture found display "sv << displayID << " after refresh"sv; + return display; + } + } + + return nil; +} + +- (void)releaseCaptureSignals { + if (self.frameSignal) { + dispatch_semaphore_signal(self.frameSignal); + dispatch_release(self.frameSignal); + self.frameSignal = NULL; + } + + if (self.captureSignal) { + dispatch_semaphore_signal(self.captureSignal); + dispatch_release(self.captureSignal); + self.captureSignal = NULL; + } +} + +- (void)clearLatestSampleBuffer { + if (self.latestSampleBuffer) { + CFRelease(self.latestSampleBuffer); + self.latestSampleBuffer = NULL; + } +} + +- (void)storeSampleBuffer:(CMSampleBufferRef)sampleBuffer { + @synchronized(self) { + if (self.stopping) { + return; + } + + BOOL shouldSignal = self.latestSampleBuffer == NULL; + + [self clearLatestSampleBuffer]; + self.latestSampleBuffer = (CMSampleBufferRef) CFRetain(sampleBuffer); + + if (shouldSignal && self.frameSignal) { + dispatch_semaphore_signal(self.frameSignal); + } + } +} + +/** + * @brief Create and start a capture stream for the current filter and configuration. + * + * @return Started stream (retained), or nil on failure. + */ +- (SCStream *)startConfiguredStream { + SCStream *stream = [[SCStream alloc] initWithFilter:self.contentFilter configuration:self.streamConfiguration delegate:self]; + NSError *output_error = nil; + if (!stream || ![stream addStreamOutput:self type:SCStreamOutputTypeScreen sampleHandlerQueue:nil error:&output_error]) { + BOOST_LOG(error) << "SCCapture failed to create stream output: "sv << error_description(output_error); + [stream release]; + return nil; + } + + __block NSError *start_error = nil; + dispatch_semaphore_t started = dispatch_semaphore_create(0); + [stream startCaptureWithCompletionHandler:^(NSError *completion_error) { + start_error = [completion_error retain]; + dispatch_semaphore_signal(started); + }]; + dispatch_semaphore_wait(started, DISPATCH_TIME_FOREVER); + dispatch_release(started); + + if (start_error) { + BOOST_LOG(error) << "SCCapture failed to start capture stream: "sv << error_description(start_error); + [start_error release]; + [stream release]; + return nil; + } + + return stream; +} + +- (dispatch_semaphore_t)captureVideo { + dispatch_semaphore_t capture_signal = nil; + + @synchronized(self) { + [self clearLatestSampleBuffer]; + [self releaseCaptureSignals]; + + self.stopping = NO; + self.streamFailed = NO; + self.captureSignal = dispatch_semaphore_create(0); + self.frameSignal = dispatch_semaphore_create(0); + + SCDisplay *display = [self findDisplayWithIDRetrying:self.displayID]; + if (!display) { + BOOST_LOG(error) << "SCCapture display not found after retries: "sv << self.displayID; + [self releaseCaptureSignals]; + return nil; + } + + SCContentFilter *filter = [[SCContentFilter alloc] initWithDisplay:display excludingWindows:@[]]; + self.contentFilter = filter; + [filter release]; + + SCStreamConfiguration *config = [[SCStreamConfiguration alloc] init]; + config.width = self.frameWidth; + config.height = self.frameHeight; + config.pixelFormat = self.pixelFormat; + config.showsCursor = YES; + config.captureResolution = SCCaptureResolutionBest; + config.preservesAspectRatio = YES; + // Slightly below the target interval: an interval of exactly 1/fps beats against + // WindowServer's delivery clock and suppresses frames that arrive marginally early. + const int fps = MAX(self.frameRate, 1); + config.minimumFrameInterval = CMTimeMake(9, fps * 10); + // Request the documented maximum explicitly (the default has varied across macOS + // releases). WindowServer only delivers while it has a free surface, so the pipeline + // must keep the number of sample buffers it holds at once (latest-wins slot, image in + // flight, encoder reference) well below this, or delivery stalls until one is returned. + config.queueDepth = 8; + self.streamConfiguration = config; + [config release]; + + capture_signal = self.captureSignal; + } + + // Start the stream outside the lock: delivery callbacks synchronize on self. + SCStream *stream = [self startConfiguredStream]; + + // Apple documents only BGRA, l10r, 420v and 420f for SCStreamConfiguration.pixelFormat; + // 10-bit sessions request the undocumented (but in practice accepted) x420. If the + // runtime rejects it, retry with 8-bit 420v: VideoToolbox accepts 8-bit input into a + // Main10 encode session, so the negotiated stream survives with 8-bit-sourced content. + if (!stream && self.pixelFormat == kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange) { + BOOST_LOG(warning) << "SCCapture 10-bit capture format was rejected, falling back to 8-bit capture"sv; + @synchronized(self) { + self.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; + self.streamConfiguration.pixelFormat = self.pixelFormat; + } + stream = [self startConfiguredStream]; + } + + if (!stream) { + @synchronized(self) { + [self releaseCaptureSignals]; + } + return nil; + } + + self.stream = stream; + [stream release]; + + BOOST_LOG(info) << "SCCapture stream configured: "sv << self.frameWidth << 'x' << self.frameHeight << " @ "sv << self.frameRate << " fps"sv; + + return capture_signal; +} + +- (CMSampleBufferRef)copyLatestSampleBuffer { + @synchronized(self) { + CMSampleBufferRef sampleBuffer = self.latestSampleBuffer; + self.latestSampleBuffer = NULL; + return sampleBuffer; + } +} + +- (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer ofType:(SCStreamOutputType)type { + if (type != SCStreamOutputTypeScreen) { + return; + } + + // No SCFrameStatus filtering: any sample buffer carrying pixels is a real update. + if (!isUsableImageSampleBuffer(sampleBuffer)) { + return; + } + + [self storeSampleBuffer:sampleBuffer]; +} + +- (void)stream:(SCStream *)stream didStopWithError:(NSError *)stop_error { + @synchronized(self) { + if (self.stopping) { + return; + } + + BOOST_LOG(error) << "SCCapture stream stopped: "sv << error_description(stop_error); + self.streamFailed = YES; + + // Wake the capture loop so the pipeline tears down and reinitializes. + if (self.frameSignal) { + dispatch_semaphore_signal(self.frameSignal); + } + if (self.captureSignal) { + dispatch_semaphore_signal(self.captureSignal); + } + } +} + +- (void)stopCapture { + SCStream *stream = nil; + + @synchronized(self) { + self.stopping = YES; + + stream = [self.stream retain]; + self.stream = nil; + self.contentFilter = nil; + self.streamConfiguration = nil; + + [self clearLatestSampleBuffer]; + [self releaseCaptureSignals]; + } + + if (stream) { + [stream stopCaptureWithCompletionHandler:^(NSError *stop_error) { + }]; + [stream release]; + } +} + +@end diff --git a/src/platform/macos/sc_capture_loop.cpp b/src/platform/macos/sc_capture_loop.cpp new file mode 100644 index 00000000000..a240762d577 --- /dev/null +++ b/src/platform/macos/sc_capture_loop.cpp @@ -0,0 +1,91 @@ +/** + * @file src/platform/macos/sc_capture_loop.cpp + * @brief Definitions for the ScreenCaptureKit capture loop and backend fallback policy on macOS. + */ +// standard includes +#include +#include + +// local includes +#include "src/platform/macos/sc_capture_loop.h" +#include "src/utility.h" + +namespace platf::sckit { + void backend_policy_t::note_stream_start_failure() { + _stream_start_failed = true; + } + + bool backend_policy_t::stream_start_failed() const { + return _stream_start_failed; + } + + bool backend_policy_t::prefer_sckit(bool sckit_available) const { + return sckit_available && !_stream_start_failed; + } + + backend_policy_t &backend_policy() { + static backend_policy_t policy; + return policy; + } + + capture_e run_capture_loop( + frame_source_t &source, + backend_policy_t &policy, + const display_t::push_captured_image_cb_t &push_captured_image_cb, + const display_t::pull_free_image_cb_t &pull_free_image_cb, + const process_frame_fn_t &process_frame, + std::chrono::nanoseconds poll_interval + ) { + if (!source.start()) { + // The display was built for ScreenCaptureKit but the stream never came up: ask the + // capture thread to rebuild the display, which the policy now steers to AVFoundation. + policy.note_stream_start_failure(); + return capture_e::reinit; + } + + auto stop_source = util::fail_guard([&source]() { + source.stop(); + }); + + while (true) { + // The stream pushes frames on its own; wait for the next one, waking periodically + // to report a timeout and to notice interrupt requests. + source.wait_frame(poll_interval); + + if (source.stopped()) { + // The stream died underneath us (display change, permission revoked, ...); + // only a display rebuild gets capture going again. + return capture_e::reinit; + } + + CMSampleBufferRef sample_buffer = source.copy_latest_sample_buffer(); + + if (!sample_buffer) { + // No new frame within the poll interval: report a timeout like the other backends + // so the pipeline can repeat the previous frame. + std::shared_ptr img_out; + if (!pull_free_image_cb(img_out) || !push_captured_image_cb(std::move(img_out), false)) { + return capture_e::ok; + } + continue; + } + + auto release_sample_buffer = util::fail_guard([sample_buffer]() { + CFRelease(sample_buffer); + }); + + std::shared_ptr img_out; + if (!pull_free_image_cb(img_out)) { + return capture_e::ok; + } + + if (!process_frame(sample_buffer, img_out.get())) { + continue; + } + + if (!push_captured_image_cb(std::move(img_out), true)) { + return capture_e::ok; + } + } + } +} // namespace platf::sckit diff --git a/src/platform/macos/sc_capture_loop.h b/src/platform/macos/sc_capture_loop.h new file mode 100644 index 00000000000..5bc85a00988 --- /dev/null +++ b/src/platform/macos/sc_capture_loop.h @@ -0,0 +1,133 @@ +/** + * @file src/platform/macos/sc_capture_loop.h + * @brief Declarations for the ScreenCaptureKit capture loop and backend fallback policy on macOS. + */ +#pragma once + +// standard includes +#include +#include +#include + +// platform includes +#include + +// local includes +#include "src/platform/common.h" + +namespace platf::sckit { + /** + * @brief Process-wide policy deciding whether ScreenCaptureKit is still the preferred capture backend. + * + * ScreenCaptureKit is selected when a display is created, but its stream can only be started once + * capture begins. A stream that fails to start flips this policy so that the display is rebuilt + * on AVFoundation instead. + */ + class backend_policy_t { + public: + /** + * @brief Record that an SCStream could not be started. + */ + void note_stream_start_failure(); + + /** + * @brief Check whether an SCStream previously failed to start in this process. + * + * @return `true` once a stream start failure was recorded. + */ + [[nodiscard]] bool stream_start_failed() const; + + /** + * @brief Decide whether ScreenCaptureKit should be used for a new display. + * + * @param sckit_available Whether the running system supports ScreenCaptureKit capture. + * @return `true` when ScreenCaptureKit is available and has not failed to start a stream. + */ + [[nodiscard]] bool prefer_sckit(bool sckit_available) const; + + private: + std::atomic _stream_start_failed {false}; ///< Set once an SCStream fails to start. + }; + + /** + * @brief Access the process-wide backend policy. + * + * @return Shared policy instance. + */ + backend_policy_t &backend_policy(); + + /** + * @brief Minimal view of an SCStream as seen by the capture loop. + * + * The loop only needs to start and stop the stream, wait for and take the latest frame, and + * learn when the stream stopped on its own, so tests can drive it with a scripted source. + */ + class frame_source_t { + public: + virtual ~frame_source_t() = default; + + /** + * @brief Start the stream. + * + * @return `true` when frames may be expected; `false` when the stream could not be started. + */ + virtual bool start() = 0; + + /** + * @brief Wait until a new frame is available or the timeout expires. + * + * @param timeout Maximum time to wait. + */ + virtual void wait_frame(std::chrono::nanoseconds timeout) = 0; + + /** + * @brief Check whether the stream stopped without being asked to. + * + * @return `true` once the stream reported an unexpected stop. + */ + [[nodiscard]] virtual bool stopped() const = 0; + + /** + * @brief Take ownership of the most recent frame. + * + * @return Retained sample buffer, or `NULL` when no new frame arrived. + */ + virtual CMSampleBufferRef copy_latest_sample_buffer() = 0; + + /** + * @brief Stop the stream and release its resources. + */ + virtual void stop() = 0; + }; + + /** + * @brief Callback that converts a sample buffer into a Sunshine image. + * + * Returns `false` when the sample buffer carried no usable pixel buffer. + */ + using process_frame_fn_t = std::function; + + /** + * @brief Run the ScreenCaptureKit capture loop until the pipeline interrupts it or the stream fails. + * + * Frames are pushed as they arrive; a poll interval without a frame is reported as a timeout so + * the pipeline can repeat the previous frame. A stream that fails to start or stops on its own + * yields `capture_e::reinit`, which makes the capture thread rebuild the display; a start failure + * also updates the backend policy, so that rebuild falls back to AVFoundation. + * @param source Stream to capture from; started here and stopped before returning unless the start failed. + * @param policy Backend policy updated when the stream cannot be started. + * @param push_captured_image_cb Pipeline callback receiving captured or repeated images. + * @param pull_free_image_cb Pipeline callback providing free images. + * @param process_frame Converts a sample buffer into the pulled image. + * @param poll_interval Longest time to wait for a frame before reporting a timeout. + * @return `capture_e::ok` when the pipeline interrupted capture; `capture_e::reinit` when the stream failed. + */ + capture_e run_capture_loop( + frame_source_t &source, + backend_policy_t &policy, + const display_t::push_captured_image_cb_t &push_captured_image_cb, + const display_t::pull_free_image_cb_t &pull_free_image_cb, + const process_frame_fn_t &process_frame, + std::chrono::nanoseconds poll_interval + ); +} // namespace platf::sckit diff --git a/src/video.cpp b/src/video.cpp index 13ece17213d..12deae6d8d2 100644 --- a/src/video.cpp +++ b/src/video.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -476,6 +477,7 @@ namespace video { replacements = std::move(other.replacements); sps = std::move(other.sps); vps = std::move(other.vps); + frame_timestamps = std::move(other.frame_timestamps); inject = other.inject; @@ -538,6 +540,8 @@ namespace video { // inject sps/vps data into idr pictures int inject; ///< Number of upcoming IDR frames that should receive rewritten parameter sets. + + frame_timestamp_queue_t frame_timestamps; ///< Capture timestamps of in-flight frames keyed by pts, awaiting their encoded packets. }; /** @@ -1817,6 +1821,31 @@ namespace video { } } + void remember_frame_timestamp(frame_timestamp_queue_t ×tamps, int64_t frame_nr, const std::optional &frame_timestamp) { + if (!frame_timestamp) { + return; + } + + timestamps.emplace_back(frame_nr, *frame_timestamp); + if (timestamps.size() > 128) { + timestamps.pop_front(); + } + } + + std::optional match_frame_timestamp(frame_timestamp_queue_t ×tamps, int64_t pts) { + while (!timestamps.empty() && timestamps.front().first < pts) { + timestamps.pop_front(); + } + + if (!timestamps.empty() && timestamps.front().first == pts) { + auto timestamp = timestamps.front().second; + timestamps.pop_front(); + return timestamp; + } + + return std::nullopt; + } + /** * @brief Drain encoded packets from an FFmpeg encoder session. * @@ -1831,6 +1860,8 @@ namespace video { auto &frame = session.device->frame; frame->pts = frame_nr; + remember_frame_timestamp(session.frame_timestamps, frame_nr, frame_timestamp); + auto &ctx = session.avcodec_ctx; auto &sps = session.sps; @@ -1889,9 +1920,7 @@ namespace video { ); } - if (av_packet && av_packet->pts == frame_nr) { - packet->frame_timestamp = frame_timestamp; - } + packet->frame_timestamp = match_frame_timestamp(session.frame_timestamps, av_packet->pts); packet->replacements = &session.replacements; packet->channel_data = channel_data; diff --git a/src/video.h b/src/video.h index 2092fcacf0c..c94a068e38a 100644 --- a/src/video.h +++ b/src/video.h @@ -6,7 +6,10 @@ // standard includes #include +#include +#include #include +#include // local includes #include "input.h" @@ -191,6 +194,34 @@ namespace video { int offsetH; ///< Offset h. }; + /** + * @brief Capture timestamps of in-flight frames keyed by pts, awaiting their encoded packets. + */ + using frame_timestamp_queue_t = std::deque>; + + /** + * @brief Remember a submitted frame's capture timestamp until its encoded packet surfaces. + * + * Encoders may deliver packets one or more frames behind submission, so the timestamp is + * kept keyed by pts until the corresponding packet arrives. The queue is bounded: once + * 128 timestamps are outstanding, the oldest entry is evicted per insertion. + * @param timestamps Queue of outstanding capture timestamps. + * @param frame_nr Monotonic frame index assigned by the video pipeline, used as the frame pts. + * @param frame_timestamp Capture timestamp associated with the frame, when known. + */ + void remember_frame_timestamp(frame_timestamp_queue_t ×tamps, int64_t frame_nr, const std::optional &frame_timestamp); + + /** + * @brief Match an encoded packet to the capture timestamp of the frame it encodes. + * + * Entries for older frames whose packets never surfaced (e.g. dropped by the encoder) + * are discarded; a packet is never paired with another frame's timestamp. + * @param timestamps Queue of outstanding capture timestamps. + * @param pts Presentation timestamp of the received packet. + * @return Capture timestamp of the packet's frame, when known. + */ + std::optional match_frame_timestamp(frame_timestamp_queue_t ×tamps, int64_t pts); + /** * @brief Pixel formats supported by one encoder backend. */ diff --git a/tests/unit/platform/macos/test_sc_capture_loop.mm b/tests/unit/platform/macos/test_sc_capture_loop.mm new file mode 100644 index 00000000000..4837e9f7c68 --- /dev/null +++ b/tests/unit/platform/macos/test_sc_capture_loop.mm @@ -0,0 +1,247 @@ +/** + * @file tests/unit/platform/macos/test_sc_capture_loop.mm + * @brief Unit tests for src/platform/macos/sc_capture_loop.* and the SCCapture stop delegate. + */ +// Only compile these tests on macOS +#ifdef __APPLE__ + // test includes + #include "../../../tests_common.h" + + // standard includes + #include + #include + #include + #include + + // platform includes + #import + #import + #import + + // local includes + #import + #include + +namespace { + using namespace std::chrono_literals; + + /** + * @brief Build a small sample buffer that needs no capture permission. + * + * @return Retained sample buffer wrapping a blank NV12 pixel buffer. + */ + CMSampleBufferRef make_sample_buffer() { + CVPixelBufferRef pixel_buffer = nullptr; + CVPixelBufferCreate(kCFAllocatorDefault, 16, 16, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, nullptr, &pixel_buffer); + + CMVideoFormatDescriptionRef format_desc = nullptr; + CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixel_buffer, &format_desc); + + CMSampleTimingInfo timing = {kCMTimeInvalid, kCMTimeInvalid, kCMTimeInvalid}; + CMSampleBufferRef sample_buffer = nullptr; + CMSampleBufferCreateForImageBuffer(kCFAllocatorDefault, pixel_buffer, YES, nullptr, nullptr, format_desc, &timing, &sample_buffer); + + CFRelease(format_desc); + CVPixelBufferRelease(pixel_buffer); + return sample_buffer; + } + + /** + * @brief Image type handed out by the fake pipeline. + */ + struct test_img_t: platf::img_t {}; + + /** + * @brief Scripted stand-in for an SCStream. + * + * Each wait consumes the next event: a frame delivers a sample buffer, a timeout delivers + * nothing, and a stop makes the source report an unexpected stream stop. + */ + class fake_frame_source_t: public platf::sckit::frame_source_t { + public: + /** + * @brief Scripted stream events. + */ + enum class event_e { + timeout, ///< No frame arrived within the poll interval. + frame, ///< A frame is available. + stop ///< The stream stopped on its own. + }; + + bool start_result = true; ///< Result returned by start(). + std::deque events; ///< Events replayed in order. + int start_calls = 0; ///< Number of start() calls. + int stop_calls = 0; ///< Number of stop() calls. + int waits = 0; ///< Number of wait_frame() calls. + + bool start() override { + ++start_calls; + return start_result; + } + + void wait_frame(std::chrono::nanoseconds) override { + ++waits; + } + + [[nodiscard]] bool stopped() const override { + return !events.empty() && events.front() == event_e::stop; + } + + CMSampleBufferRef copy_latest_sample_buffer() override { + if (events.empty()) { + return nullptr; + } + const auto event = events.front(); + events.pop_front(); + return event == event_e::frame ? make_sample_buffer() : nullptr; + } + + void stop() override { + ++stop_calls; + } + }; + + /** + * @brief Fake pipeline recording what the loop pushes and controlling when it interrupts. + */ + struct fake_pipeline_t { + bool pull_ok = true; ///< Whether pull_free_image_cb hands out an image. + bool process_ok = true; ///< Whether process_frame accepts the sample buffer. + int push_limit = 1000; ///< push_captured_image_cb interrupts once this many pushes happened. + int processed = 0; ///< Number of process_frame calls. + std::vector pushed; ///< frame_captured flag of every push, in order. + + /** + * @brief Run the loop against this pipeline. + * + * @param source Scripted stream. + * @param policy Backend policy under test. + * @return Loop status. + */ + platf::capture_e run(fake_frame_source_t &source, platf::sckit::backend_policy_t &policy) { + return platf::sckit::run_capture_loop( + source, + policy, + [this](std::shared_ptr &&, bool frame_captured) { + pushed.push_back(frame_captured); + return static_cast(pushed.size()) < push_limit; + }, + [this](std::shared_ptr &img_out) { + img_out = std::make_shared(); + return pull_ok; + }, + [this](CMSampleBufferRef, platf::img_t *) { + ++processed; + return process_ok; + }, + 1ms + ); + } + }; +} // namespace + +TEST(SCCaptureLoop, StreamStartFailureReturnsReinitAndDisablesSckit) { + fake_frame_source_t source; + source.start_result = false; + platf::sckit::backend_policy_t policy; + fake_pipeline_t pipeline; + + EXPECT_EQ(pipeline.run(source, policy), platf::capture_e::reinit); + EXPECT_TRUE(policy.stream_start_failed()); + EXPECT_FALSE(policy.prefer_sckit(true)); + EXPECT_EQ(source.start_calls, 1); + EXPECT_EQ(source.stop_calls, 0); + EXPECT_TRUE(pipeline.pushed.empty()); +} + +TEST(SCCaptureLoop, UnexpectedStreamStopReturnsReinit) { + using event_e = fake_frame_source_t::event_e; + fake_frame_source_t source; + source.events = {event_e::frame, event_e::frame, event_e::stop}; + platf::sckit::backend_policy_t policy; + fake_pipeline_t pipeline; + + EXPECT_EQ(pipeline.run(source, policy), platf::capture_e::reinit); + EXPECT_EQ(pipeline.pushed, (std::vector {true, true})); + EXPECT_EQ(pipeline.processed, 2); + EXPECT_EQ(source.stop_calls, 1); + EXPECT_FALSE(policy.stream_start_failed()); + EXPECT_TRUE(policy.prefer_sckit(true)); +} + +TEST(SCCaptureLoop, PollTimeoutReportsRepeatFrame) { + using event_e = fake_frame_source_t::event_e; + fake_frame_source_t source; + source.events = {event_e::timeout, event_e::frame}; + platf::sckit::backend_policy_t policy; + fake_pipeline_t pipeline; + pipeline.push_limit = 2; + + EXPECT_EQ(pipeline.run(source, policy), platf::capture_e::ok); + EXPECT_EQ(pipeline.pushed, (std::vector {false, true})); + EXPECT_EQ(pipeline.processed, 1); + EXPECT_EQ(source.stop_calls, 1); +} + +TEST(SCCaptureLoop, PipelineInterruptOnPullReturnsOk) { + using event_e = fake_frame_source_t::event_e; + fake_frame_source_t source; + source.events = {event_e::frame}; + platf::sckit::backend_policy_t policy; + fake_pipeline_t pipeline; + pipeline.pull_ok = false; + + EXPECT_EQ(pipeline.run(source, policy), platf::capture_e::ok); + EXPECT_TRUE(pipeline.pushed.empty()); + EXPECT_EQ(pipeline.processed, 0); + EXPECT_EQ(source.stop_calls, 1); +} + +TEST(SCCaptureLoop, UnusableFrameIsSkipped) { + using event_e = fake_frame_source_t::event_e; + fake_frame_source_t source; + source.events = {event_e::frame, event_e::stop}; + platf::sckit::backend_policy_t policy; + fake_pipeline_t pipeline; + pipeline.process_ok = false; + + EXPECT_EQ(pipeline.run(source, policy), platf::capture_e::reinit); + EXPECT_EQ(pipeline.processed, 1); + EXPECT_TRUE(pipeline.pushed.empty()); + EXPECT_EQ(source.stop_calls, 1); +} + +TEST(SCCaptureLoop, BackendPolicyPrefersSckitUntilAStreamFailsToStart) { + platf::sckit::backend_policy_t policy; + + EXPECT_FALSE(policy.stream_start_failed()); + EXPECT_TRUE(policy.prefer_sckit(true)); + EXPECT_FALSE(policy.prefer_sckit(false)); + + policy.note_stream_start_failure(); + + EXPECT_TRUE(policy.stream_start_failed()); + EXPECT_FALSE(policy.prefer_sckit(true)); + EXPECT_FALSE(policy.prefer_sckit(false)); +} + +TEST(SCCaptureLoop, StopDelegateMarksStreamFailedUnlessStopping) { + if (@available(macOS 14.0, *)) { + SCCapture *capture = [[SCCapture alloc] init]; + ASSERT_NE(capture, nil); + + SCStream *stream = nil; + NSError *error = [NSError errorWithDomain:@"test" code:1 userInfo:nil]; + [capture stream:stream didStopWithError:error]; + EXPECT_TRUE(capture.streamFailed); + + // A stop requested by the backend itself must not look like a failure. + capture.streamFailed = NO; + [capture stopCapture]; + [capture stream:stream didStopWithError:error]; + EXPECT_FALSE(capture.streamFailed); + + [capture release]; + } +} +#endif diff --git a/tests/unit/test_video.cpp b/tests/unit/test_video.cpp index 01046dd8857..06534cfaf84 100644 --- a/tests/unit/test_video.cpp +++ b/tests/unit/test_video.cpp @@ -540,3 +540,95 @@ TEST(SoftwareEncoderConversion, Bgr0AndNv12) { fallback_nv12_img.row_pitch = w; EXPECT_EQ(device.convert(fallback_nv12_img), 0); } + +/** + * @brief Fixture for the frame timestamp bookkeeping helpers. + */ +struct FrameTimestampQueueTest: testing::Test { + /** + * @brief Produce a distinct timestamp for a frame number. + * + * @param frame_nr Frame number to derive the timestamp from. + * @return Timestamp unique to the frame number. + */ + static std::chrono::steady_clock::time_point ts(int64_t frame_nr) { + return std::chrono::steady_clock::time_point {std::chrono::milliseconds {frame_nr}}; + } + + video::frame_timestamp_queue_t queue; +}; + +TEST_F(FrameTimestampQueueTest, MatchesInOrder) { + for (int64_t frame = 1; frame <= 4; ++frame) { + video::remember_frame_timestamp(queue, frame, ts(frame)); + EXPECT_EQ(ts(frame), video::match_frame_timestamp(queue, frame)); + } + EXPECT_TRUE(queue.empty()); +} + +TEST_F(FrameTimestampQueueTest, MatchesDelayedPackets) { + // Encoders may deliver packets several frames behind submission. + video::remember_frame_timestamp(queue, 1, ts(1)); + video::remember_frame_timestamp(queue, 2, ts(2)); + video::remember_frame_timestamp(queue, 3, ts(3)); + + EXPECT_EQ(ts(1), video::match_frame_timestamp(queue, 1)); + + video::remember_frame_timestamp(queue, 4, ts(4)); + + EXPECT_EQ(ts(2), video::match_frame_timestamp(queue, 2)); + EXPECT_EQ(ts(3), video::match_frame_timestamp(queue, 3)); + EXPECT_EQ(ts(4), video::match_frame_timestamp(queue, 4)); +} + +TEST_F(FrameTimestampQueueTest, DiscardsDroppedPts) { + // Frames whose packets never surface must be discarded, and a stale pts + // must never be paired with another frame's timestamp. + video::remember_frame_timestamp(queue, 1, ts(1)); + video::remember_frame_timestamp(queue, 2, ts(2)); + video::remember_frame_timestamp(queue, 3, ts(3)); + + EXPECT_EQ(ts(3), video::match_frame_timestamp(queue, 3)); + EXPECT_EQ(std::nullopt, video::match_frame_timestamp(queue, 1)); + EXPECT_EQ(std::nullopt, video::match_frame_timestamp(queue, 2)); +} + +TEST_F(FrameTimestampQueueTest, IgnoresUnknownPts) { + video::remember_frame_timestamp(queue, 5, ts(5)); + + EXPECT_EQ(std::nullopt, video::match_frame_timestamp(queue, 6)); + EXPECT_TRUE(queue.empty()); +} + +TEST_F(FrameTimestampQueueTest, SkipsMissingTimestamps) { + // Duplicate or synthesized frames carry no capture timestamp; they must not + // enqueue anything or disturb their neighbors. + video::remember_frame_timestamp(queue, 1, std::nullopt); + video::remember_frame_timestamp(queue, 2, ts(2)); + + EXPECT_EQ(std::nullopt, video::match_frame_timestamp(queue, 1)); + EXPECT_EQ(ts(2), video::match_frame_timestamp(queue, 2)); +} + +TEST_F(FrameTimestampQueueTest, EvictsOldestBeyondCap) { + // A wedged encoder must not grow the queue without bound. + for (int64_t frame = 1; frame <= 200; ++frame) { + video::remember_frame_timestamp(queue, frame, ts(frame)); + } + + EXPECT_EQ(128u, queue.size()); + EXPECT_EQ(std::nullopt, video::match_frame_timestamp(queue, 1)); // evicted + EXPECT_EQ(ts(73), video::match_frame_timestamp(queue, 73)); // oldest survivor + EXPECT_EQ(ts(200), video::match_frame_timestamp(queue, 200)); +} + +TEST_F(FrameTimestampQueueTest, EmptyQueueYieldsNothing) { + EXPECT_EQ(std::nullopt, video::match_frame_timestamp(queue, 1)); +} + +TEST_F(FrameTimestampQueueTest, ConsumesEntriesOnce) { + video::remember_frame_timestamp(queue, 1, ts(1)); + + EXPECT_EQ(ts(1), video::match_frame_timestamp(queue, 1)); + EXPECT_EQ(std::nullopt, video::match_frame_timestamp(queue, 1)); +}