feat(video,platform): declare explicit capture pixel formats

src/platform/common.h

@@ -345,6 +345,13 @@ namespace platf {
     yuv444p16,  ///< Planar 10-bit (shifted to 16-bit) YUV 4:4:4
     yuv444p,  ///< Planar 8-bit YUV 4:4:4
     y410,  ///< Y410
+    bgr0,  ///< Packed 8-bit B,G,R,0
+    bgra,  ///< Packed 8-bit B,G,R,A
+    xbgr2101010,  ///< Packed 10-bit X,B,G,R
+    bgra1010102,  ///< Packed 10-bit B,G,R,A
+    rgba1010102,  ///< Packed 10-bit R,G,B,A
+    abgr2101010,  ///< Packed 10-bit A,B,G,R
+    argb2101010,  ///< Packed 10-bit A,R,G,B
     unknown  ///< Unknown
   };
 
@@ -370,13 +377,34 @@ namespace platf {
       _CONVERT(yuv444p16);
       _CONVERT(yuv444p);
       _CONVERT(y410);
+      _CONVERT(bgr0);
+      _CONVERT(bgra);
+      _CONVERT(xbgr2101010);
+      _CONVERT(bgra1010102);
+      _CONVERT(rgba1010102);
+      _CONVERT(abgr2101010);
+      _CONVERT(argb2101010);
       _CONVERT(unknown);
     }
 #undef _CONVERT
 
     return "unknown"sv;
   }
 
+  /**
+   * @brief Check whether a capture pixel format can be uploaded as packed 8-bit BGR.
+   * @note Unknown formats are treated as BGR0, the historical assumption for
+   * capture buffers that do not declare a format.
+   *
+   * @param pix_fmt Capture pixel format to check.
+   * @return True when the format is representable as packed 8-bit BGR.
+   */
+  inline bool is_bgr_capture_format(pix_fmt_e pix_fmt) {
+    using enum pix_fmt_e;
+
+    return pix_fmt == unknown || pix_fmt == bgr0 || pix_fmt == bgra;
+  }
+
   // Dimensions for touchscreen input
   /**
    * @brief Touchscreen coordinate bounds used to scale absolute input.
@@ -536,6 +564,7 @@ namespace platf {
     std::int32_t height {};  ///< Image height in pixels.
     std::int32_t pixel_pitch {};  ///< Bytes per pixel in the image buffer.
     std::int32_t row_pitch {};  ///< Bytes between consecutive image rows.
+    pix_fmt_e pixel_format {pix_fmt_e::unknown};  ///< Declared pixel format of the captured image.
 
     std::optional<std::chrono::steady_clock::time_point> frame_timestamp;  ///< Capture timestamp associated with the frame.
 

src/platform/linux/cuda.cpp

@@ -243,6 +243,7 @@ namespace cuda {
       img.height = height;
       img.pixel_pitch = 4;
       img.row_pitch = img.width * img.pixel_pitch;
+      img.pixel_format = platf::pix_fmt_e::bgr0;
 
       std::vector<std::uint8_t> image_data;
       image_data.resize(img.row_pitch * img.height);
@@ -296,6 +297,14 @@ namespace cuda {
      * @return Conversion status.
      */
     int convert(platf::img_t &img) override {
+      // This copy path only understands packed 8-bit BGR captures. Fail loudly
+      // for declared formats it cannot represent (e.g. NV12 or 10-bit formats)
+      // instead of silently copying them as 8-bit RGBA.
+      if (!platf::is_bgr_capture_format(img.pixel_format)) {
+        BOOST_LOG(error) << "RAM capture conversion does not support pixel format: "sv << platf::from_pix_fmt(img.pixel_format);
+        return -1;
+      }
+
       if (is_yuv444) {
         return sws.load_ram(img, tex.array) || sws.convert_yuv444(frame->data[0], frame->data[1], frame->data[2], frame->linesize[0], tex_obj(tex), stream.get());
       }
@@ -1227,6 +1236,7 @@ namespace cuda {
         img->height = height;
         img->pixel_pitch = 4;
         img->row_pitch = img->width * img->pixel_pitch;
+        img->pixel_format = platf::pix_fmt_e::bgr0;
 
         auto tex_opt = tex_t::make(height, width * img->pixel_pitch);
         if (!tex_opt) {

src/platform/linux/cuda.cu

@@ -49,13 +49,36 @@ using namespace std::literals;
  * Not pretty and extremely error-prone, fix at earliest convenience.
  */
 namespace platf {
+  /**
+   * @brief Keep in sync with src/platform/common.h:529 platf::img_t.
+   */
+  enum class pix_fmt_e {
+    yuv420p,
+    yuv420p10,
+    nv12,
+    p010,
+    ayuv,
+    yuv444p16,
+    yuv444p,
+    y410,
+    bgr0,
+    bgra,
+    xbgr2101010,
+    bgra1010102,
+    rgba1010102,
+    abgr2101010,
+    argb2101010,
+    unknown
+  };
+
   struct img_t: std::enable_shared_from_this<img_t> {
   public:
     std::uint8_t *data {};
     std::int32_t width {};
     std::int32_t height {};
     std::int32_t pixel_pitch {};
     std::int32_t row_pitch {};
+    pix_fmt_e pixel_format {pix_fmt_e::unknown};  ///< Keep in sync with src/platform/common.h:529
 
     std::optional<std::chrono::steady_clock::time_point> frame_timestamp;
 
@@ -65,6 +88,8 @@ namespace platf {
 
 // End special declarations
 
+static_assert(sizeof(platf::pix_fmt_e) == 4, "pix_fmt_e must be 4 bytes");
+
 namespace cuda {
 
   struct alignas(16) cuda_color_t {

src/platform/linux/graphics.cpp

@@ -1258,11 +1258,21 @@ namespace egl {
     return make_nv12(in_width, in_height, out_width, out_height, std::move(tex));
   }
 
-  void sws_t::load_ram(platf::img_t &img) {
+  int sws_t::load_ram(platf::img_t &img) {
+    // This upload path only understands packed 8-bit BGR captures. Fail loudly
+    // for declared formats it cannot represent (e.g. NV12 or 10-bit formats)
+    // instead of silently uploading them as BGR0.
+    if (!platf::is_bgr_capture_format(img.pixel_format)) {
+      BOOST_LOG(error) << "RAM capture conversion does not support pixel format: "sv << platf::from_pix_fmt(img.pixel_format);
+      return -1;
+    }
+
     loaded_texture = tex[0];
 
     gl::ctx.BindTexture(GL_TEXTURE_2D, loaded_texture);
     gl::ctx.TexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, img.width, img.height, GL_BGRA, GL_UNSIGNED_BYTE, img.data);
+
+    return 0;
   }
 
   void sws_t::load_vram(img_descriptor_t &img, int offset_x, int offset_y, int texture, bool is_yuv444) {

src/platform/linux/graphics.h

@@ -721,10 +721,13 @@ namespace egl {
 
     /**
      * @brief Load ram data from the backing API or store.
+     * @note Only packed 8-bit BGR captures are supported; other declared
+     * formats fail with a negative status instead of being misread.
      *
      * @param img Image or frame object to read from or populate.
+     * @return 0 on success; negative on unsupported formats.
      */
-    void load_ram(platf::img_t &img);
+    int load_ram(platf::img_t &img);
     /**
      * @brief Load vram data from the backing API or store.
      *

src/platform/linux/kmsgrab.cpp

@@ -1699,6 +1699,7 @@ namespace platf {
         img->height = height;
         img->pixel_pitch = 4;
         img->row_pitch = img->pixel_pitch * width;
+        img->pixel_format = platf::pix_fmt_e::bgr0;
         img->data = new std::uint8_t[height * img->row_pitch];
 
         return img;
@@ -1775,6 +1776,7 @@ namespace platf {
         img->serial = std::numeric_limits<decltype(img->serial)>::max();
         img->data = nullptr;
         img->pixel_pitch = 4;
+        img->pixel_format = platf::pix_fmt_e::bgr0;
 
         img->sequence = 0;
         std::fill_n(img->sd.fds, 4, -1);
@@ -1877,6 +1879,7 @@ namespace platf {
           img->height = captured_cursor.dst_h;
           img->pixel_pitch = 4;
           img->row_pitch = img->pixel_pitch * img->width;
+          img->pixel_format = platf::pix_fmt_e::bgra;
           img->data = img->buffer.data();
         } else {
           img->data = nullptr;

src/platform/linux/pipewire.cpp

@@ -50,6 +50,37 @@ namespace {
   constexpr int MAX_PARAMS = 200;
   constexpr int MAX_DMABUF_FORMATS = 200;
   constexpr int MAX_DMABUF_MODIFIERS = 200;
+
+  /**
+   * @brief Map a PipeWire SPA video format to Sunshine's pixel format enum.
+   *
+   * @param spa_format PipeWire SPA video format identifier.
+   * @return Sunshine pixel format; unknown for unmapped formats.
+   */
+  platf::pix_fmt_e map_spa_pix_fmt(int32_t spa_format) {
+    using enum platf::pix_fmt_e;
+
+    switch (spa_format) {
+      case SPA_VIDEO_FORMAT_NV12:
+        return nv12;
+      case SPA_VIDEO_FORMAT_BGRx:
+        return bgr0;
+      case SPA_VIDEO_FORMAT_BGRA:
+        return bgra;
+      case SPA_VIDEO_FORMAT_xBGR_210LE:
+        return xbgr2101010;
+      case SPA_VIDEO_FORMAT_ARGB_210LE:
+        return argb2101010;
+      case SPA_VIDEO_FORMAT_ABGR_210LE:
+        return abgr2101010;
+      case SPA_VIDEO_FORMAT_RGBA_102LE:
+        return rgba1010102;
+      case SPA_VIDEO_FORMAT_BGRA_102LE:
+        return bgra1010102;
+      default:
+        return unknown;
+    }
+  }
 }  // namespace
 
 using namespace std::literals;
@@ -63,12 +94,13 @@ namespace pipewire {
     int32_t pw_format;  ///< Matching PipeWire SPA video format.
   };
 
-  static constexpr std::array<format_map_t, 7> format_map = {{
+  static constexpr std::array<format_map_t, 8> format_map = {{
+    {DRM_FORMAT_NV12, SPA_VIDEO_FORMAT_NV12},
     {DRM_FORMAT_XBGR2101010, SPA_VIDEO_FORMAT_xBGR_210LE},
-    {DRM_FORMAT_BGRA1010102, SPA_VIDEO_FORMAT_ARGB_210LE},
-    {DRM_FORMAT_RGBA1010102, SPA_VIDEO_FORMAT_ABGR_210LE},
-    {DRM_FORMAT_ABGR2101010, SPA_VIDEO_FORMAT_RGBA_102LE},
-    {DRM_FORMAT_ARGB2101010, SPA_VIDEO_FORMAT_BGRA_102LE},
+    {DRM_FORMAT_BGRA1010102, SPA_VIDEO_FORMAT_BGRA_102LE},
+    {DRM_FORMAT_RGBA1010102, SPA_VIDEO_FORMAT_RGBA_102LE},
+    {DRM_FORMAT_ABGR2101010, SPA_VIDEO_FORMAT_ABGR_210LE},
+    {DRM_FORMAT_ARGB2101010, SPA_VIDEO_FORMAT_ARGB_210LE},
     {DRM_FORMAT_ARGB8888, SPA_VIDEO_FORMAT_BGRA},
     {DRM_FORMAT_XRGB8888, SPA_VIDEO_FORMAT_BGRx},
   }};
@@ -447,6 +479,7 @@ namespace pipewire {
           // NV12 is the only 1-byte-per-pixel format delivered on the memory
           // path; every other negotiated format is packed 4 bytes per pixel.
           img->pixel_pitch = (stream_data.format.info.raw.format == SPA_VIDEO_FORMAT_NV12) ? 1 : 4;
+          img->pixel_format = map_spa_pix_fmt(stream_data.format.info.raw.format);
         }
       }
 
@@ -955,6 +988,7 @@ namespace pipewire {
       img->height = height;
       img->pixel_pitch = 4;
       img->row_pitch = img->pixel_pitch * width;
+      img->pixel_format = platf::pix_fmt_e::bgr0;
       img->sequence = 0;
       img->serial = std::numeric_limits<decltype(img->serial)>::max();
       img->data = nullptr;
@@ -1095,6 +1129,7 @@ namespace pipewire {
           static_cast<img_descriptor_t *>(img)->data_owned = true;
           img->row_pitch = w * 4;
           img->pixel_pitch = 4;
+          img->pixel_format = platf::pix_fmt_e::bgr0;
         }
       }
       return 0;

src/platform/linux/vaapi.cpp

@@ -531,7 +531,9 @@ namespace va {
      * @return Conversion status.
      */
     int convert(platf::img_t &img) override {
-      sws.load_ram(img);
+      if (sws.load_ram(img) < 0) {
+        return -1;
+      }
 
       sws.convert_nv12(nv12->buf);
       return 0;

src/platform/linux/wlgrab.cpp

@@ -348,6 +348,7 @@ namespace wl {
       img->height = height;
       img->pixel_pitch = 4;
       img->row_pitch = img->pixel_pitch * width;
+      img->pixel_format = platf::pix_fmt_e::bgr0;
       img->data = new std::uint8_t[height * img->row_pitch];
 
       return img;

src/platform/linux/x11grab.cpp

@@ -635,6 +635,7 @@ namespace platf {
       img->data = (uint8_t *) x_img->data;
       img->row_pitch = x_img->bytes_per_line;
       img->pixel_pitch = x_img->bits_per_pixel / 8;
+      img->pixel_format = (img->pixel_pitch == 4) ? platf::pix_fmt_e::bgr0 : platf::pix_fmt_e::unknown;
       img->img.reset(x_img);
 
       if (cursor) {
@@ -830,6 +831,7 @@ namespace platf {
       img->height = height;
       img->pixel_pitch = 4;
       img->row_pitch = img->pixel_pitch * width;
+      img->pixel_format = platf::pix_fmt_e::bgr0;
       img->data = new std::uint8_t[height * img->row_pitch];
 
       return img;
@@ -1052,6 +1054,7 @@ namespace platf {
       img.y = xcursor->y - xcursor->yhot;
       img.pixel_pitch = 4;
       img.row_pitch = img.pixel_pitch * img.width;
+      img.pixel_format = platf::pix_fmt_e::bgr0;
       img.serial = xcursor->cursor_serial;
     }
 

src/platform/macos/display.mm

@@ -52,6 +52,25 @@ OSType videotoolbox_pixel_format(const video::config_t &config) {
       const auto colorspace {video::colorspace_from_client_config(config, false)};
       return colorspace.bit_depth == 10 ? kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
     }
+
+    /**
+     * @brief Map a Core Video pixel format to Sunshine's pixel format enum.
+     *
+     * @param cv_format Core Video pixel format type.
+     * @return Sunshine pixel format; unknown for unmapped formats.
+     */
+    platf::pix_fmt_e pix_fmt_from_cv_pixel_format(OSType cv_format) {
+      switch (cv_format) {
+        case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
+          return platf::pix_fmt_e::nv12;
+        case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
+          return platf::pix_fmt_e::p010;
+        case kCVPixelFormatType_32BGRA:
+          return platf::pix_fmt_e::bgra;
+        default:
+          return platf::pix_fmt_e::unknown;
+      }
+    }
   }  // namespace
 
   /**
@@ -93,6 +112,7 @@ capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const
         img_out->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf);
         img_out->row_pitch = (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf);
         img_out->pixel_pitch = img_out->row_pitch / img_out->width;
+        img_out->pixel_format = pix_fmt_from_cv_pixel_format(av_capture.pixelFormat);
 
         old_data_retainer = nullptr;
 
@@ -177,6 +197,7 @@ int dummy_img(img_t *img) override {
         img->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf);
         img->row_pitch = (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf);
         img->pixel_pitch = img->row_pitch / img->width;
+        img->pixel_format = pix_fmt_from_cv_pixel_format(av_capture.pixelFormat);
 
         old_data_retainer = nullptr;
 

src/platform/windows/display_ram.cpp

@@ -374,6 +374,7 @@ namespace platf::dxgi {
     }
 
     img->pixel_pitch = get_pixel_pitch();
+    img->pixel_format = (img->pixel_pitch == 4) ? platf::pix_fmt_e::bgr0 : platf::pix_fmt_e::unknown;
 
     if (dummy && !img->row_pitch) {
       // Assume our dummy image will have no padding

src/platform/windows/display_vram.cpp

@@ -1947,6 +1947,7 @@ namespace platf::dxgi {
 
     // Initialize format-dependent fields
     img->pixel_pitch = get_pixel_pitch();
+    img->pixel_format = (img->pixel_pitch == 4) ? platf::pix_fmt_e::bgr0 : platf::pix_fmt_e::unknown;
     img->row_pitch = img->pixel_pitch * img->width;
     img->dummy = dummy;
     img->format = (capture_format == DXGI_FORMAT_UNKNOWN) ? DXGI_FORMAT_B8G8R8A8_UNORM : capture_format;