diff options
| -rw-r--r-- | TODO | 4 | ||||
| -rw-r--r-- | include/pipewire_video.h | 7 | ||||
| -rw-r--r-- | src/capture/portal.c | 12 | ||||
| -rw-r--r-- | src/dbus.c | 9 | ||||
| -rw-r--r-- | src/pipewire_video.c | 187 |
5 files changed, 152 insertions, 67 deletions
@@ -187,7 +187,7 @@ Always disable prime run/dri prime and list all monitors to record from from all automatically use the associated gpu card. Allow flv av1 if recent ffmpeg version and streaming to youtube (and twitch?) and for custom services. -Use explicit sync in pipewire video code: https://docs.pipewire.org/page_dma_buf.html. +Use explicit sync in pipewire video code: https://docs.pipewire.org/page_dma_buf.html. This should fix cross gpu pipewire capture. Replay (and recording?) fails to save properly sometimes (especially for long videos). This is noticable with mp4 files since they get corrupt and become unplayable. The entire video does seem to get saved (it's a large video file) and it seems to have the correct headers but it's not playable. @@ -427,3 +427,5 @@ Add the option to use application audio with exectable name, or automatically ch In the ui it should also list the application name beside the app audio. Region capture works incorrectly on gnome with hidpi, especially with 150% scaling. Same in gsr ui. + +Support overlay planes with kms capture. COSMIC uses an overlay plane for the top panel, so its not visible in the video. diff --git a/include/pipewire_video.h b/include/pipewire_video.h index 38ec3d6..00228c5 100644 --- a/include/pipewire_video.h +++ b/include/pipewire_video.h @@ -85,6 +85,7 @@ typedef struct { struct { bool visible; bool valid; + bool updated; uint8_t *data; int x, y; int hotspot_x, hotspot_y; @@ -126,6 +127,12 @@ bool gsr_pipewire_video_init(gsr_pipewire_video *self, int pipewire_fd, uint32_t void gsr_pipewire_video_deinit(gsr_pipewire_video *self); bool gsr_pipewire_video_map_texture(gsr_pipewire_video *self, gsr_texture_map texture_map, gsr_map_texture_output *output); +/* + Returns true if the cursor has changed (moved, changed image or changed visibility) since the last call to + gsr_pipewire_video_map_texture or gsr_pipewire_video_update_cursor. This only happens when the wayland compositor + supports cursor metadata mode. |cursor_region| and the cursor texture in |texture_map| are updated with the latest cursor state. +*/ +bool gsr_pipewire_video_update_cursor(gsr_pipewire_video *self, gsr_texture_map texture_map, gsr_pipewire_video_region *cursor_region); bool gsr_pipewire_video_is_damaged(gsr_pipewire_video *self); void gsr_pipewire_video_clear_damage(gsr_pipewire_video *self); bool gsr_pipewire_video_should_restart(gsr_pipewire_video *self); diff --git a/src/capture/portal.c b/src/capture/portal.c index c6f058d..fd1caec 100644 --- a/src/capture/portal.c +++ b/src/capture/portal.c @@ -201,7 +201,7 @@ static int gsr_capture_portal_setup_dbus(gsr_capture_portal *self, int *pipewire } fprintf(stderr, "gsr info: gsr_capture_portal_setup_dbus: SelectSources\n"); - response_status = gsr_dbus_screencast_select_sources(&self->dbus, self->session_handle, GSR_PORTAL_CAPTURE_TYPE_ALL, self->params.record_cursor ? GSR_PORTAL_CURSOR_MODE_EMBEDDED : GSR_PORTAL_CURSOR_MODE_HIDDEN); + response_status = gsr_dbus_screencast_select_sources(&self->dbus, self->session_handle, GSR_PORTAL_CAPTURE_TYPE_ALL, self->params.record_cursor ? GSR_PORTAL_CURSOR_MODE_METADATA : GSR_PORTAL_CURSOR_MODE_HIDDEN); if(response_status != 0) { fprintf(stderr, "gsr error: gsr_capture_portal_setup_dbus: SelectSources failed\n"); return response_status; @@ -347,7 +347,10 @@ static void gsr_capture_portal_pre_capture(gsr_capture *cap, gsr_capture_metadat self->capture_size.y = self->pipewire_data.region.height; color_conversion->schedule_clear = true; } - } else { + } else if(self->pipewire_data.texture_width == 0 || !gsr_pipewire_video_update_cursor(&self->pipewire, self->texture_map, &self->pipewire_data.cursor_region)) { + /* Theres no new video frame to capture and the cursor hasn't changed (which only happens in cursor metadata mode). + If the cursor has changed then the latest video frame is redrawn with the new cursor state, otherwise nothing is captured. + This is needed on gnome which doesn't send new video frames when capturing a window and only the cursor moves. */ return; } } @@ -387,9 +390,10 @@ static int gsr_capture_portal_capture(gsr_capture *cap, gsr_capture_metadata *ca self->capture_size.y == 0 ? 0 : (double)output_size.y / (double)self->capture_size.y }; + /* The cursor position is relative to the video buffer, remove the crop offset to make it relative to the visible video content */ const vec2i cursor_pos = { - target_pos.x + (self->pipewire_data.cursor_region.x * scale.x), - target_pos.y + (self->pipewire_data.cursor_region.y * scale.y) + target_pos.x + ((self->pipewire_data.cursor_region.x - self->pipewire_data.region.x) * scale.x), + target_pos.y + ((self->pipewire_data.cursor_region.y - self->pipewire_data.region.y) * scale.y) }; self->params.egl->glEnable(GL_SCISSOR_TEST); @@ -650,8 +650,15 @@ int gsr_dbus_screencast_select_sources(gsr_dbus *self, const char *session_handl uint32_t available_cursor_modes = 0; gsr_dbus_desktop_portal_get_property(self, "org.freedesktop.portal.ScreenCast", "AvailableCursorModes", &available_cursor_modes); if(available_cursor_modes == 0) - fprintf(stderr, "gsr error: gsr_dbus_screencast_select_sources: no cursors modes are available\n"); + fprintf(stderr, "gsr warning: gsr_dbus_screencast_select_sources: no cursors modes are available\n"); + + if(cursor_mode == GSR_PORTAL_CURSOR_MODE_METADATA && !(available_cursor_modes & GSR_PORTAL_CURSOR_MODE_METADATA)) { + fprintf(stderr, "gsr warning: gsr_dbus_screencast_select_sources: cursor mode metadata is not available, using cursor mode embedded instead\n"); + cursor_mode = GSR_PORTAL_CURSOR_MODE_EMBEDDED; + } cursor_mode = unset_unsupported_cursor_modes(cursor_mode, available_cursor_modes); + if(cursor_mode == 0) + cursor_mode = GSR_PORTAL_CURSOR_MODE_HIDDEN; char handle_token[64]; gsr_dbus_portal_get_unique_handle_token(self, handle_token, sizeof(handle_token)); diff --git a/src/pipewire_video.c b/src/pipewire_video.c index 30e00a8..360c8b5 100644 --- a/src/pipewire_video.c +++ b/src/pipewire_video.c @@ -114,32 +114,111 @@ static const struct pw_core_events core_events = { .error = on_core_error_cb, }; +static void gsr_pipewire_video_read_cursor_metadata(gsr_pipewire_video *self, struct spa_buffer *buffer) { + const struct spa_meta_cursor *cursor = spa_buffer_find_meta_data(buffer, SPA_META_Cursor, sizeof(*cursor)); + if(!cursor) + return; + + /* The cursor can become invalid when it moves off the captured window/monitor, in which case it should be hidden */ + const bool cursor_valid = spa_meta_cursor_is_valid(cursor); + if(cursor_valid != self->cursor.valid) { + self->cursor.updated = true; + self->damaged = true; + } + self->cursor.valid = cursor_valid; + + if(!self->cursor.visible || !self->cursor.valid) + return; + + struct spa_meta_bitmap *bitmap = NULL; + if(cursor->bitmap_offset) + bitmap = SPA_MEMBER(cursor, cursor->bitmap_offset, struct spa_meta_bitmap); + + // TODO: Maybe check if the cursor is actually visible by checking if there are visible pixels + if(bitmap && bitmap->size.width > 0 && bitmap->size.height > 0 && is_cursor_format_supported(bitmap->format)) { + /* Animated cursors update the bitmap for every animation frame, only log when the size changes */ + if((int)bitmap->size.width != self->cursor.width || (int)bitmap->size.height != self->cursor.height) { + fprintf(stderr, "gsr info: pipewire: cursor bitmap update, size: %dx%d, format: %s\n", + (int)bitmap->size.width, (int)bitmap->size.height, spa_debug_type_find_name(spa_type_video_format, bitmap->format)); + } + + const uint8_t *bitmap_data = SPA_MEMBER(bitmap, bitmap->offset, uint8_t); + const size_t bitmap_size = bitmap->size.width * bitmap->size.height * 4; + uint8_t *new_bitmap_data = realloc(self->cursor.data, bitmap_size); + if(new_bitmap_data) { + self->cursor.data = new_bitmap_data; + /* TODO: Convert bgr and other image formats to rgb here */ + memcpy(self->cursor.data, bitmap_data, bitmap_size); + + self->cursor.hotspot_x = cursor->hotspot.x; + self->cursor.hotspot_y = cursor->hotspot.y; + self->cursor.width = bitmap->size.width; + self->cursor.height = bitmap->size.height; + self->cursor.updated = true; + self->damaged = true; + } + } + + /* Position changes for a cursor that cant be drawn (no bitmap received yet) shouldn't trigger a redraw */ + if(self->cursor.width > 0 && (cursor->position.x != self->cursor.x || cursor->position.y != self->cursor.y)) { + self->cursor.updated = true; + self->damaged = true; + } + + self->cursor.x = cursor->position.x; + self->cursor.y = cursor->position.y; + + //fprintf(stderr, "gsr info: pipewire: cursor: %d %d %d %d\n", cursor->hotspot.x, cursor->hotspot.y, cursor->position.x, cursor->position.y); +} + +static bool buffer_has_video_content(const struct spa_buffer *buffer) { + /* Cursor-only updates in cursor metadata mode are sent as buffers without valid video content + (chunk size 0 on gnome, chunk marked as corrupted on kde plasma) */ + return buffer->n_datas > 0 && buffer->datas[0].chunk->size != 0 && !(buffer->datas[0].chunk->flags & SPA_CHUNK_FLAG_CORRUPTED); +} + static void on_process_cb(void *user_data) { gsr_pipewire_video *self = user_data; - /* Find the most recent buffer */ + /* Find the most recent buffer with video content. The cursor metadata is read from all buffers in the order they arrive + because a batch of buffers can contain both video buffers and cursor-only buffers (in cursor metadata mode) + and a video buffer shouldn't be discarded just because a cursor-only buffer arrived after it. */ + bool got_buffer = false; struct pw_buffer *pw_buf = NULL; for(;;) { struct pw_buffer *aux = pw_stream_dequeue_buffer(self->stream); if(!aux) break; - if(pw_buf) - pw_stream_queue_buffer(self->stream, pw_buf); - pw_buf = aux; + + got_buffer = true; + pthread_mutex_lock(&self->mutex); + gsr_pipewire_video_read_cursor_metadata(self, aux->buffer); + pthread_mutex_unlock(&self->mutex); + + if(buffer_has_video_content(aux->buffer)) { + if(pw_buf) + pw_stream_queue_buffer(self->stream, pw_buf); + pw_buf = aux; + } else { + pw_stream_queue_buffer(self->stream, aux); + } } - if(!pw_buf) { + if(!got_buffer) { fprintf(stderr, "gsr info: pipewire: out of buffers!\n"); return; } + /* Only cursor-only buffers arrived, theres no new video frame to process */ + if(!pw_buf) + return; + struct spa_buffer *buffer = pw_buf->buffer; - const bool has_buffer = buffer->n_datas > 0 && buffer->datas[0].chunk->size != 0; pthread_mutex_lock(&self->mutex); bool buffer_updated = false; - if(has_buffer && buffer->datas[0].type == SPA_DATA_DmaBuf) { + if(buffer->datas[0].type == SPA_DATA_DmaBuf) { for(size_t i = 0; i < self->dmabuf_num_planes; ++i) { if(self->dmabuf_data[i].fd > 0) { close(self->dmabuf_data[i].fd); @@ -210,44 +289,6 @@ static void on_process_cb(void *user_data) { self->damaged = true; } - const struct spa_meta_cursor *cursor = spa_buffer_find_meta_data(buffer, SPA_META_Cursor, sizeof(*cursor)); - self->cursor.valid = cursor && spa_meta_cursor_is_valid(cursor); - - if (self->cursor.visible && self->cursor.valid) { - struct spa_meta_bitmap *bitmap = NULL; - if (cursor->bitmap_offset) - bitmap = SPA_MEMBER(cursor, cursor->bitmap_offset, struct spa_meta_bitmap); - - // TODO: Maybe check if the cursor is actually visible by checking if there are visible pixels - if (bitmap && bitmap->size.width > 0 && bitmap->size.height > 0 && is_cursor_format_supported(bitmap->format)) { - const uint8_t *bitmap_data = SPA_MEMBER(bitmap, bitmap->offset, uint8_t); - fprintf(stderr, "gsr info: pipewire: cursor bitmap update, size: %dx%d, format: %s\n", - (int)bitmap->size.width, (int)bitmap->size.height, spa_debug_type_find_name(spa_type_video_format, bitmap->format)); - - const size_t bitmap_size = bitmap->size.width * bitmap->size.height * 4; - uint8_t *new_bitmap_data = realloc(self->cursor.data, bitmap_size); - if(new_bitmap_data) { - self->cursor.data = new_bitmap_data; - /* TODO: Convert bgr and other image formats to rgb here */ - memcpy(self->cursor.data, bitmap_data, bitmap_size); - } - - self->cursor.hotspot_x = cursor->hotspot.x; - self->cursor.hotspot_y = cursor->hotspot.y; - self->cursor.width = bitmap->size.width; - self->cursor.height = bitmap->size.height; - self->damaged = true; - } - - if(cursor->position.x != self->cursor.x || cursor->position.y != self->cursor.y) - self->damaged = true; - - self->cursor.x = cursor->position.x; - self->cursor.y = cursor->position.y; - - //fprintf(stderr, "gsr info: pipewire: cursor: %d %d %d %d\n", cursor->hotspot.x, cursor->hotspot.y, cursor->position.x, cursor->position.y); - } - pthread_mutex_unlock(&self->mutex); pw_stream_queue_buffer(self->stream, pw_buf); } @@ -840,15 +881,26 @@ static void gsr_pipewire_video_update_cursor_texture(gsr_pipewire_video *self, g self->cursor.data = NULL; } -bool gsr_pipewire_video_map_texture(gsr_pipewire_video *self, gsr_texture_map texture_map, gsr_map_texture_output *output) { - for(int i = 0; i < GSR_PIPEWIRE_VIDEO_DMABUF_MAX_PLANES; ++i) { - memset(&output->dmabuf_data[i], 0, sizeof(gsr_pipewire_video_dmabuf_data)); +static void gsr_pipewire_video_update_cursor_data(gsr_pipewire_video *self, gsr_texture_map texture_map, gsr_pipewire_video_region *cursor_region) { + self->cursor.updated = false; + gsr_pipewire_video_update_cursor_texture(self, texture_map); + + if(self->cursor.valid) { + cursor_region->x = self->cursor.x - self->cursor.hotspot_x; + cursor_region->y = self->cursor.y - self->cursor.hotspot_y; + + cursor_region->width = self->cursor.width; + cursor_region->height = self->cursor.height; + } else { + cursor_region->x = 0; + cursor_region->y = 0; + + cursor_region->width = 0; + cursor_region->height = 0; } - output->num_dmabuf_data = 0; - output->using_external_image = self->external_texture_fallback; - output->fourcc = 0; - output->modifiers = 0; - output->rotation = GSR_MONITOR_ROT_0; +} + +bool gsr_pipewire_video_map_texture(gsr_pipewire_video *self, gsr_texture_map texture_map, gsr_map_texture_output *output) { pthread_mutex_lock(&self->mutex); if(!self->negotiated || !self->streaming || self->dmabuf_data[0].fd <= 0) { @@ -862,12 +914,16 @@ bool gsr_pipewire_video_map_texture(gsr_pipewire_video *self, gsr_texture_map te return false; } + /* |output| is only written to from this point on. When this function returns false the caller + can keep using the data from the previous successful call to redraw the previous video frame. */ + for(int i = 0; i < GSR_PIPEWIRE_VIDEO_DMABUF_MAX_PLANES; ++i) { + memset(&output->dmabuf_data[i], 0, sizeof(gsr_pipewire_video_dmabuf_data)); + } + gsr_pipewire_video_bind_image_to_texture_with_fallback(self, texture_map, image); output->using_external_image = self->external_texture_fallback; self->egl->eglDestroyImage(self->egl->egl_display, image); - gsr_pipewire_video_update_cursor_texture(self, texture_map); - output->texture_width = self->format.info.raw.size.width; output->texture_height = self->format.info.raw.size.height; @@ -892,12 +948,7 @@ bool gsr_pipewire_video_map_texture(gsr_pipewire_video *self, gsr_texture_map te output->region.height = temp; } - /* TODO: Test if cursor hotspot is correct */ - output->cursor_region.x = self->cursor.x - self->cursor.hotspot_x; - output->cursor_region.y = self->cursor.y - self->cursor.hotspot_y; - - output->cursor_region.width = self->cursor.width; - output->cursor_region.height = self->cursor.height; + gsr_pipewire_video_update_cursor_data(self, texture_map, &output->cursor_region); for(size_t i = 0; i < self->dmabuf_num_planes; ++i) { output->dmabuf_data[i] = self->dmabuf_data[i]; @@ -913,6 +964,20 @@ bool gsr_pipewire_video_map_texture(gsr_pipewire_video *self, gsr_texture_map te return true; } +bool gsr_pipewire_video_update_cursor(gsr_pipewire_video *self, gsr_texture_map texture_map, gsr_pipewire_video_region *cursor_region) { + pthread_mutex_lock(&self->mutex); + + if(!self->negotiated || !self->streaming || !self->cursor.updated) { + pthread_mutex_unlock(&self->mutex); + return false; + } + + gsr_pipewire_video_update_cursor_data(self, texture_map, cursor_region); + + pthread_mutex_unlock(&self->mutex); + return true; +} + bool gsr_pipewire_video_is_damaged(gsr_pipewire_video *self) { if(!self->mutex_initialized) return false; |