portal: use cursor metadata instead of embedded, to allow the wayland compositor to do a direct scanout

3 files changed, 63 insertions, 54 deletions

diff --git a/src/capture/portal.c b/src/capture/portal.c
index c6f058d..a40c42a 100644
--- a/src/capture/portal.c
+++ b/src/capture/portal.c
@@ -201,7 +201,7 @@ static int gsr_capture_portal_setup_dbus(gsr_capture_portal *self, int *pipewire
     }
 
     fprintf(stderr, "gsr info: gsr_capture_portal_setup_dbus: SelectSources\n");
-    response_status = gsr_dbus_screencast_select_sources(&self->dbus, self->session_handle, GSR_PORTAL_CAPTURE_TYPE_ALL, self->params.record_cursor ? GSR_PORTAL_CURSOR_MODE_EMBEDDED : GSR_PORTAL_CURSOR_MODE_HIDDEN);
+    response_status = gsr_dbus_screencast_select_sources(&self->dbus, self->session_handle, GSR_PORTAL_CAPTURE_TYPE_ALL, self->params.record_cursor ? GSR_PORTAL_CURSOR_MODE_METADATA : GSR_PORTAL_CURSOR_MODE_HIDDEN);
     if(response_status != 0) {
         fprintf(stderr, "gsr error: gsr_capture_portal_setup_dbus: SelectSources failed\n");
         return response_status;
@@ -347,8 +347,6 @@ static void gsr_capture_portal_pre_capture(gsr_capture *cap, gsr_capture_metadat
                 self->capture_size.y = self->pipewire_data.region.height;
                 color_conversion->schedule_clear = true;
             }
-        } else {
-            return;
         }
     }
 
diff --git a/src/dbus.c b/src/dbus.c
index 8ba58bc..617245d 100644
--- a/src/dbus.c
+++ b/src/dbus.c
@@ -650,8 +650,16 @@ int gsr_dbus_screencast_select_sources(gsr_dbus *self, const char *session_handl
     uint32_t available_cursor_modes = 0;
     gsr_dbus_desktop_portal_get_property(self, "org.freedesktop.portal.ScreenCast", "AvailableCursorModes", &available_cursor_modes);
     if(available_cursor_modes == 0)
-        fprintf(stderr, "gsr error: gsr_dbus_screencast_select_sources: no cursors modes are available\n");
+        fprintf(stderr, "gsr warning: gsr_dbus_screencast_select_sources: no cursors modes are available\n");
+
+    if(cursor_mode == GSR_PORTAL_CURSOR_MODE_METADATA && !(available_cursor_modes & GSR_PORTAL_CURSOR_MODE_METADATA)) {
+        fprintf(stderr, "gsr warning: gsr_dbus_screencast_select_sources: cursor mode metadata is not available, using cursor mode embedded instead\n");
+        fprintf(stderr, "gsr warning: this may disable direct scanout. Capture a monitor directly with -w <monitor> instead of -w portal if you experience performance issues\n");
+        cursor_mode = GSR_PORTAL_CURSOR_MODE_EMBEDDED;
+    }
     cursor_mode = unset_unsupported_cursor_modes(cursor_mode, available_cursor_modes);
+    if(cursor_mode == 0)
+        cursor_mode = GSR_PORTAL_CURSOR_MODE_HIDDEN;
 
     char handle_token[64];
     gsr_dbus_portal_get_unique_handle_token(self, handle_token, sizeof(handle_token));
diff --git a/src/pipewire_video.c b/src/pipewire_video.c
index 077e733..520ff69 100644
--- a/src/pipewire_video.c
+++ b/src/pipewire_video.c
@@ -138,43 +138,6 @@ static void on_process_cb(void *user_data) {
 
     pthread_mutex_lock(&self->mutex);
 
-    bool buffer_updated = false;
-    if(has_buffer && buffer->datas[0].type == SPA_DATA_DmaBuf) {
-        for(size_t i = 0; i < self->dmabuf_num_planes; ++i) {
-            if(self->dmabuf_data[i].fd > 0) {
-                close(self->dmabuf_data[i].fd);
-                self->dmabuf_data[i].fd = -1;
-            }
-        }
-
-        self->dmabuf_num_planes = buffer->n_datas;
-        if(self->dmabuf_num_planes > GSR_PIPEWIRE_VIDEO_DMABUF_MAX_PLANES)
-            self->dmabuf_num_planes = GSR_PIPEWIRE_VIDEO_DMABUF_MAX_PLANES;
-
-        for(size_t i = 0; i < self->dmabuf_num_planes; ++i) {
-            self->dmabuf_data[i].fd = dup(buffer->datas[i].fd);
-            self->dmabuf_data[i].offset = buffer->datas[i].chunk->offset;
-            self->dmabuf_data[i].stride = buffer->datas[i].chunk->stride;
-        }
-
-        buffer_updated = true;
-    }
-
-    // TODO: Move down to read_metadata
-    struct spa_meta_region *region = spa_buffer_find_meta_data(buffer, SPA_META_VideoCrop, sizeof(*region));
-    if(region && spa_meta_region_is_valid(region)) {
-        // fprintf(stderr, "gsr info: pipewire: crop Region available (%dx%d+%d+%d)\n",
-        //      region->region.position.x, region->region.position.y,
-        //      region->region.size.width, region->region.size.height);
-        self->crop.x = region->region.position.x;
-        self->crop.y = region->region.position.y;
-        self->crop.width = region->region.size.width;
-        self->crop.height = region->region.size.height;
-        self->crop.valid = true;
-    } else {
-        self->crop.valid = false;
-    }
-
     struct spa_meta_videotransform *video_transform = spa_buffer_find_meta_data(buffer, SPA_META_VideoTransform, sizeof(*video_transform));
     enum spa_meta_videotransform_value transform = SPA_META_TRANSFORMATION_None;
     if(video_transform)
@@ -196,6 +159,7 @@ static void on_process_cb(void *user_data) {
             break;
     }
 
+    bool this_frame_damaged = false;
     const struct spa_meta *video_damage = spa_buffer_find_meta(buffer, SPA_META_VideoDamage);
     if(video_damage) {
         struct spa_meta_region *meta_region = NULL;
@@ -204,10 +168,45 @@ static void on_process_cb(void *user_data) {
                 continue;
 
             self->damaged = true;
+            this_frame_damaged = true;
             break;
         }
-    } else if(buffer_updated) {
+    } else {
         self->damaged = true;
+        this_frame_damaged = true;
+    }
+
+    if(this_frame_damaged && has_buffer && buffer->datas[0].type == SPA_DATA_DmaBuf) {
+        for(size_t i = 0; i < self->dmabuf_num_planes; ++i) {
+            if(self->dmabuf_data[i].fd > 0) {
+                close(self->dmabuf_data[i].fd);
+                self->dmabuf_data[i].fd = -1;
+            }
+        }
+
+        self->dmabuf_num_planes = buffer->n_datas;
+        if(self->dmabuf_num_planes > GSR_PIPEWIRE_VIDEO_DMABUF_MAX_PLANES)
+            self->dmabuf_num_planes = GSR_PIPEWIRE_VIDEO_DMABUF_MAX_PLANES;
+
+        for(size_t i = 0; i < self->dmabuf_num_planes; ++i) {
+            self->dmabuf_data[i].fd = dup(buffer->datas[i].fd);
+            self->dmabuf_data[i].offset = buffer->datas[i].chunk->offset;
+            self->dmabuf_data[i].stride = buffer->datas[i].chunk->stride;
+        }
+    }
+
+    struct spa_meta_region *region = spa_buffer_find_meta_data(buffer, SPA_META_VideoCrop, sizeof(*region));
+    if(region && spa_meta_region_is_valid(region)) {
+        // fprintf(stderr, "gsr info: pipewire: crop Region available (%dx%d+%d+%d)\n",
+        //      region->region.position.x, region->region.position.y,
+        //      region->region.size.width, region->region.size.height);
+        self->crop.x = region->region.position.x;
+        self->crop.y = region->region.position.y;
+        self->crop.width = region->region.size.width;
+        self->crop.height = region->region.size.height;
+        self->crop.valid = true;
+    } else {
+        self->crop.valid = false;
     }
 
     const struct spa_meta_cursor *cursor = spa_buffer_find_meta_data(buffer, SPA_META_Cursor, sizeof(*cursor));
@@ -221,8 +220,8 @@ static void on_process_cb(void *user_data) {
         // TODO: Maybe check if the cursor is actually visible by checking if there are visible pixels
         if (bitmap && bitmap->size.width > 0 && bitmap->size.height > 0 && is_cursor_format_supported(bitmap->format)) {
             const uint8_t *bitmap_data = SPA_MEMBER(bitmap, bitmap->offset, uint8_t);
-            fprintf(stderr, "gsr info: pipewire: cursor bitmap update, size: %dx%d, format: %s\n",
-                (int)bitmap->size.width, (int)bitmap->size.height, spa_debug_type_find_name(spa_type_video_format, bitmap->format));
+            //fprintf(stderr, "gsr info: pipewire: cursor bitmap update, size: %dx%d, format: %s\n",
+            //    (int)bitmap->size.width, (int)bitmap->size.height, spa_debug_type_find_name(spa_type_video_format, bitmap->format));
 
             const size_t bitmap_size = bitmap->size.width * bitmap->size.height * 4;
             uint8_t *new_bitmap_data = realloc(self->cursor.data, bitmap_size);
@@ -859,7 +858,20 @@ bool gsr_pipewire_video_map_texture(gsr_pipewire_video *self, gsr_texture_map te
     output->rotation = GSR_MONITOR_ROT_0;
     pthread_mutex_lock(&self->mutex);
 
-    if(!self->negotiated || !self->streaming || self->dmabuf_data[0].fd <= 0) {
+    if(!self->negotiated || !self->streaming) {
+        pthread_mutex_unlock(&self->mutex);
+        return false;
+    }
+
+    gsr_pipewire_video_update_cursor_texture(self, texture_map);
+
+    output->cursor_region.x = self->cursor.x - self->cursor.hotspot_x;
+    output->cursor_region.y = self->cursor.y - self->cursor.hotspot_y;
+
+    output->cursor_region.width = self->cursor.width;
+    output->cursor_region.height = self->cursor.height;
+
+    if(self->dmabuf_data[0].fd <= 0) {
         pthread_mutex_unlock(&self->mutex);
         return false;
     }
@@ -874,8 +886,6 @@ bool gsr_pipewire_video_map_texture(gsr_pipewire_video *self, gsr_texture_map te
     output->using_external_image = self->external_texture_fallback;
     self->egl->eglDestroyImage(self->egl->egl_display, image);
 
-    gsr_pipewire_video_update_cursor_texture(self, texture_map);
-
     output->texture_width = self->format.info.raw.size.width;
     output->texture_height = self->format.info.raw.size.height;
 
@@ -900,13 +910,6 @@ bool gsr_pipewire_video_map_texture(gsr_pipewire_video *self, gsr_texture_map te
         output->region.height = temp;
     }
 
-    /* TODO: Test if cursor hotspot is correct */
-    output->cursor_region.x = self->cursor.x - self->cursor.hotspot_x;
-    output->cursor_region.y = self->cursor.y - self->cursor.hotspot_y;
-
-    output->cursor_region.width = self->cursor.width;
-    output->cursor_region.height = self->cursor.height;
-
     for(size_t i = 0; i < self->dmabuf_num_planes; ++i) {
         output->dmabuf_data[i] = self->dmabuf_data[i];
         self->dmabuf_data[i].fd = -1;