Support overlay plane

3 files changed, 211 insertions, 55 deletions

diff --git a/kms/kms_shared.h b/kms/kms_shared.h
index c982723..81e0d77 100644
--- a/kms/kms_shared.h
+++ b/kms/kms_shared.h
@@ -5,7 +5,7 @@
 #include <stdbool.h>
 #include <drm_mode.h>
 
-#define GSR_KMS_PROTOCOL_VERSION 5
+#define GSR_KMS_PROTOCOL_VERSION 6
 
 #define GSR_KMS_MAX_ITEMS 8
 #define GSR_KMS_MAX_DMA_BUFS 4
@@ -46,6 +46,12 @@ typedef enum {
     KMS_ROT_270
 } gsr_kms_rotation;
 
+typedef enum {
+    KMS_PLANE_TYPE_PRIMARY,
+    KMS_PLANE_TYPE_CURSOR,
+    KMS_PLANE_TYPE_OVERLAY
+} gsr_kms_plane_type;
+
 struct gsr_kms_response_item {
     gsr_kms_response_dma_buf dma_buf[GSR_KMS_MAX_DMA_BUFS];
     int num_dma_bufs;
@@ -54,13 +60,20 @@ struct gsr_kms_response_item {
     uint32_t pixel_format;
     uint64_t modifier;
     uint32_t connector_id; /* 0 if unknown */
-    bool is_cursor;
+    gsr_kms_plane_type plane_type;
     bool has_hdr_metadata;
     gsr_kms_rotation rotation;
-    int x;
-    int y;
+    /* The region in the framebuffer that is displayed */
+    int src_x;
+    int src_y;
     int src_w;
     int src_h;
+    /* The region on the monitor (crtc) where the framebuffer region is displayed */
+    int dst_x;
+    int dst_y;
+    int dst_w;
+    int dst_h;
+    int zpos;
     struct hdr_output_metadata hdr_metadata;
 };
 
diff --git a/kms/server/kms_server.c b/kms/server/kms_server.c
index 17b8ba5..c41f8bd 100644
--- a/kms/server/kms_server.c
+++ b/kms/server/kms_server.c
@@ -145,17 +145,23 @@ typedef enum {
     PLANE_PROPERTY_IS_CURSOR  = 1 << 6,
     PLANE_PROPERTY_IS_PRIMARY = 1 << 7,
     PLANE_PROPERTY_ROTATION   = 1 << 8,
+    PLANE_PROPERTY_W          = 1 << 9,
+    PLANE_PROPERTY_H          = 1 << 10,
+    PLANE_PROPERTY_IS_OVERLAY = 1 << 11,
+    PLANE_PROPERTY_ZPOS       = 1 << 12,
 } plane_property_mask;
 
+typedef struct {
+    int x, y, w, h;             /* Region on the crtc (monitor) where the plane is displayed (CRTC_* properties) */
+    int src_x, src_y, src_w, src_h; /* Region in the framebuffer that is displayed (SRC_* properties) */
+    int zpos;
+    gsr_kms_rotation rotation;
+} plane_properties;
+
 /* Returns plane_property_mask */
-static uint32_t plane_get_properties(int drmfd, uint32_t plane_id, int *x, int *y, int *src_x, int *src_y, int *src_w, int *src_h, gsr_kms_rotation *rotation) {
-    *x = 0;
-    *y = 0;
-    *src_x = 0;
-    *src_y = 0;
-    *src_w = 0;
-    *src_h = 0;
-    *rotation = KMS_ROT_0;
+static uint32_t plane_get_properties(int drmfd, uint32_t plane_id, plane_properties *properties) {
+    memset(properties, 0, sizeof(*properties));
+    properties->rotation = KMS_ROT_0;
 
     plane_property_mask property_mask = 0;
 
@@ -172,23 +178,35 @@ static uint32_t plane_get_properties(int drmfd, uint32_t plane_id, int *x, int *
         // SRC_* values are fixed 16.16 points
         const uint32_t type = prop->flags & (DRM_MODE_PROP_LEGACY_TYPE | DRM_MODE_PROP_EXTENDED_TYPE);
         if((type & DRM_MODE_PROP_SIGNED_RANGE) && strcmp(prop->name, "CRTC_X") == 0) {
-            *x = (int)props->prop_values[i];
+            properties->x = (int)props->prop_values[i];
             property_mask |= PLANE_PROPERTY_X;
         } else if((type & DRM_MODE_PROP_SIGNED_RANGE) && strcmp(prop->name, "CRTC_Y") == 0) {
-            *y = (int)props->prop_values[i];
+            properties->y = (int)props->prop_values[i];
             property_mask |= PLANE_PROPERTY_Y;
+        } else if((type & DRM_MODE_PROP_RANGE) && strcmp(prop->name, "CRTC_W") == 0) {
+            properties->w = (int)props->prop_values[i];
+            property_mask |= PLANE_PROPERTY_W;
+        } else if((type & DRM_MODE_PROP_RANGE) && strcmp(prop->name, "CRTC_H") == 0) {
+            properties->h = (int)props->prop_values[i];
+            property_mask |= PLANE_PROPERTY_H;
         } else if((type & DRM_MODE_PROP_RANGE) && strcmp(prop->name, "SRC_X") == 0) {
-            *src_x = (int)(props->prop_values[i] >> 16);
+            properties->src_x = (int)(props->prop_values[i] >> 16);
             property_mask |= PLANE_PROPERTY_SRC_X;
         } else if((type & DRM_MODE_PROP_RANGE) && strcmp(prop->name, "SRC_Y") == 0) {
-            *src_y = (int)(props->prop_values[i] >> 16);
+            properties->src_y = (int)(props->prop_values[i] >> 16);
             property_mask |= PLANE_PROPERTY_SRC_Y;
         } else if((type & DRM_MODE_PROP_RANGE) && strcmp(prop->name, "SRC_W") == 0) {
-            *src_w = (int)(props->prop_values[i] >> 16);
+            properties->src_w = (int)(props->prop_values[i] >> 16);
             property_mask |= PLANE_PROPERTY_SRC_W;
         } else if((type & DRM_MODE_PROP_RANGE) && strcmp(prop->name, "SRC_H") == 0) {
-            *src_h = (int)(props->prop_values[i] >> 16);
+            properties->src_h = (int)(props->prop_values[i] >> 16);
             property_mask |= PLANE_PROPERTY_SRC_H;
+        } else if(((type & DRM_MODE_PROP_RANGE) || (type & DRM_MODE_PROP_SIGNED_RANGE)) && (strcmp(prop->name, "zpos") == 0 || strcmp(prop->name, "ZPOS") == 0)) {
+            if(type & DRM_MODE_PROP_SIGNED_RANGE)
+                properties->zpos = (int)(int64_t)props->prop_values[i];
+            else
+                properties->zpos = (int)props->prop_values[i];
+            property_mask |= PLANE_PROPERTY_ZPOS;
         } else if((type & DRM_MODE_PROP_ENUM) && strcmp(prop->name, "type") == 0) {
             const uint64_t current_enum_value = props->prop_values[i];
             for(int j = 0; j < prop->count_enums; ++j) {
@@ -198,17 +216,20 @@ static uint32_t plane_get_properties(int drmfd, uint32_t plane_id, int *x, int *
                 } else if(prop->enums[j].value == current_enum_value && strcmp(prop->enums[j].name, "Cursor") == 0) {
                     property_mask |= PLANE_PROPERTY_IS_CURSOR;
                     break;
+                } else if(prop->enums[j].value == current_enum_value && strcmp(prop->enums[j].name, "Overlay") == 0) {
+                    property_mask |= PLANE_PROPERTY_IS_OVERLAY;
+                    break;
                 }
             }
         } else if((type & DRM_MODE_PROP_BITMASK) && strcmp(prop->name, "rotation") == 0) {
             const uint64_t rotation_bitmask = props->prop_values[i];
-            *rotation = KMS_ROT_0;
+            properties->rotation = KMS_ROT_0;
             if(rotation_bitmask & 2)
-                *rotation = (*rotation + KMS_ROT_90) % 4;
+                properties->rotation = (properties->rotation + KMS_ROT_90) % 4;
             if(rotation_bitmask & 4)
-                *rotation = (*rotation + KMS_ROT_180) % 4;
+                properties->rotation = (properties->rotation + KMS_ROT_180) % 4;
             if(rotation_bitmask & 8)
-                *rotation = (*rotation + KMS_ROT_270) % 4;
+                properties->rotation = (properties->rotation + KMS_ROT_270) % 4;
         }
 
         drmModeFreeProperty(prop);
@@ -350,12 +371,42 @@ static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response) {
         // TODO: Check if dimensions have changed by comparing width and height to previous time this was called.
         // TODO: Support other plane formats than rgb (with multiple planes, such as direct YUV420 on wayland).
 
-        int x = 0, y = 0, src_x = 0, src_y = 0, src_w = 0, src_h = 0;
-        gsr_kms_rotation rotation = KMS_ROT_0;
-        const uint32_t property_mask = plane_get_properties(drm->drmfd, plane->plane_id, &x, &y, &src_x, &src_y, &src_w, &src_h, &rotation);
-        if(!(property_mask & PLANE_PROPERTY_IS_PRIMARY) && !(property_mask & PLANE_PROPERTY_IS_CURSOR))
+        plane_properties properties;
+        const uint32_t property_mask = plane_get_properties(drm->drmfd, plane->plane_id, &properties);
+        if(!(property_mask & (PLANE_PROPERTY_IS_PRIMARY | PLANE_PROPERTY_IS_CURSOR | PLANE_PROPERTY_IS_OVERLAY)))
             goto cleanup_handles;
 
+        gsr_kms_plane_type plane_type = KMS_PLANE_TYPE_OVERLAY;
+        if(property_mask & PLANE_PROPERTY_IS_PRIMARY)
+            plane_type = KMS_PLANE_TYPE_PRIMARY;
+        else if(property_mask & PLANE_PROPERTY_IS_CURSOR)
+            plane_type = KMS_PLANE_TYPE_CURSOR;
+
+        /* These properties are not available when the drm driver doesn't support atomic modesetting */
+        if(!(property_mask & PLANE_PROPERTY_SRC_W))
+            properties.src_w = drmfb->width;
+        if(!(property_mask & PLANE_PROPERTY_SRC_H))
+            properties.src_h = drmfb->height;
+        if(!(property_mask & PLANE_PROPERTY_W))
+            properties.w = properties.src_w;
+        if(!(property_mask & PLANE_PROPERTY_H))
+            properties.h = properties.src_h;
+
+        /* Not all drm drivers support zpos. In that case assume that the planes are stacked in the order primary, overlay, cursor (bottom to top) */
+        if(!(property_mask & PLANE_PROPERTY_ZPOS)) {
+            switch(plane_type) {
+                case KMS_PLANE_TYPE_PRIMARY:
+                    properties.zpos = 0;
+                    break;
+                case KMS_PLANE_TYPE_OVERLAY:
+                    properties.zpos = 1;
+                    break;
+                case KMS_PLANE_TYPE_CURSOR:
+                    properties.zpos = 2;
+                    break;
+            }
+        }
+
         int fb_fds[GSR_KMS_MAX_DMA_BUFS];
         const int num_fb_fds = drm_prime_handles_to_fds(drm, drmfb, fb_fds);
         if(num_fb_fds == 0) {
@@ -386,19 +437,17 @@ static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response) {
         response->items[item_index].pixel_format = drmfb->pixel_format;
         response->items[item_index].modifier = drmfb->flags & DRM_MODE_FB_MODIFIERS ? drmfb->modifier : DRM_FORMAT_MOD_INVALID;
         response->items[item_index].connector_id = crtc_pair ? crtc_pair->connector_id : 0;
-        response->items[item_index].rotation = rotation;
-        response->items[item_index].is_cursor = property_mask & PLANE_PROPERTY_IS_CURSOR;
-        if(property_mask & PLANE_PROPERTY_IS_CURSOR) {
-            response->items[item_index].x = x;
-            response->items[item_index].y = y;
-            response->items[item_index].src_w = 0;
-            response->items[item_index].src_h = 0;
-        } else {
-            response->items[item_index].x = src_x;
-            response->items[item_index].y = src_y;
-            response->items[item_index].src_w = src_w;
-            response->items[item_index].src_h = src_h;
-        }
+        response->items[item_index].rotation = properties.rotation;
+        response->items[item_index].plane_type = plane_type;
+        response->items[item_index].src_x = properties.src_x;
+        response->items[item_index].src_y = properties.src_y;
+        response->items[item_index].src_w = properties.src_w;
+        response->items[item_index].src_h = properties.src_h;
+        response->items[item_index].dst_x = properties.x;
+        response->items[item_index].dst_y = properties.y;
+        response->items[item_index].dst_w = properties.w;
+        response->items[item_index].dst_h = properties.h;
+        response->items[item_index].zpos = properties.zpos;
         ++response->num_items;
 
         cleanup_handles:
diff --git a/src/capture/kms.c b/src/capture/kms.c
index fe4eb40..1544e2c 100644
--- a/src/capture/kms.c
+++ b/src/capture/kms.c
@@ -218,7 +218,7 @@ static int gsr_capture_kms_start(gsr_capture *cap, gsr_capture_metadata *capture
 
 static gsr_kms_response_item* find_drm_by_connector_id(gsr_kms_response *kms_response, uint32_t connector_id) {
     for(int i = 0; i < kms_response->num_items; ++i) {
-        if(kms_response->items[i].connector_id == connector_id && !kms_response->items[i].is_cursor)
+        if(kms_response->items[i].connector_id == connector_id && kms_response->items[i].plane_type == KMS_PLANE_TYPE_PRIMARY)
             return &kms_response->items[i];
     }
     return NULL;
@@ -232,7 +232,7 @@ static gsr_kms_response_item* find_largest_drm(gsr_kms_response *kms_response) {
     gsr_kms_response_item *largest_drm = &kms_response->items[0];
     for(int i = 0; i < kms_response->num_items; ++i) {
         const int64_t size = (int64_t)kms_response->items[i].width * (int64_t)kms_response->items[i].height;
-        if(size > largest_size && !kms_response->items[i].is_cursor) {
+        if(size > largest_size && kms_response->items[i].plane_type == KMS_PLANE_TYPE_PRIMARY) {
             largest_size = size;
             largest_drm = &kms_response->items[i];
         }
@@ -243,7 +243,7 @@ static gsr_kms_response_item* find_largest_drm(gsr_kms_response *kms_response) {
 static gsr_kms_response_item* find_cursor_drm(gsr_kms_response *kms_response, uint32_t connector_id) {
     gsr_kms_response_item *cursor_drm = NULL;
     for(int i = 0; i < kms_response->num_items; ++i) {
-        if(kms_response->items[i].is_cursor) {
+        if(kms_response->items[i].plane_type == KMS_PLANE_TYPE_CURSOR) {
             cursor_drm = &kms_response->items[i];
             if(kms_response->items[i].connector_id == connector_id)
                 break;
@@ -406,7 +406,7 @@ static void render_drm_cursor(gsr_capture_kms *self, gsr_color_conversion *color
     const gsr_monitor_rotation cursor_plane_rotation = kms_rotation_to_gsr_monitor_rotation(cursor_drm_fd->rotation);
     const gsr_monitor_rotation rotation = sub_rotations(self->display_server_monitor_rotation, cursor_plane_rotation);
 
-    vec2i cursor_pos = {cursor_drm_fd->x, cursor_drm_fd->y};
+    vec2i cursor_pos = {cursor_drm_fd->dst_x, cursor_drm_fd->dst_y};
     switch(rotation) {
         case GSR_MONITOR_ROT_0:
             break;
@@ -584,23 +584,14 @@ static void gsr_capture_kms_pre_capture(gsr_capture *cap, gsr_capture_metadata *
     gsr_capture_kms_update_capture_size_change(self, color_conversion, self->target_pos, self->drm_fd);
 }
 
-static int gsr_capture_kms_capture(gsr_capture *cap, gsr_capture_metadata *capture_metadata, gsr_color_conversion *color_conversion) {
-    (void)capture_metadata;
-    gsr_capture_kms *self = cap->priv;
-
-    if(!self->drm_fd || self->params.kms_response->num_items == 0)
-        return -1;
-
+static void render_monitor_plane(gsr_capture_kms *self, gsr_color_conversion *color_conversion, gsr_capture_metadata *capture_metadata) {
     vec2i capture_pos = self->capture_pos;
     if(!self->capture_is_combined_plane)
-        capture_pos = (vec2i){self->drm_fd->x, self->drm_fd->y};
+        capture_pos = (vec2i){self->drm_fd->src_x, self->drm_fd->src_y};
 
     capture_pos.x += self->params.region_position.x;
     capture_pos.y += self->params.region_position.y;
 
-    //self->params.egl->glFlush();
-    //self->params.egl->glFinish();
-
     EGLImage image = gsr_capture_kms_create_egl_image_with_fallback(self, self->drm_fd);
     if(image) {
         gsr_capture_kms_bind_image_to_input_texture_with_fallback(self, image);
@@ -611,6 +602,110 @@ static int gsr_capture_kms_capture(gsr_capture *cap, gsr_capture_metadata *captu
         self->target_pos, self->output_size,
         capture_pos, self->capture_size, (vec2i){ self->drm_fd->width, self->drm_fd->height },
         gsr_monitor_rotation_to_rotation(self->final_monitor_rotation), capture_metadata->flip, GSR_SOURCE_COLOR_RGB, self->external_texture_fallback);
+}
+
+/* Renders an overlay plane on top of (or below, depending on render order) the monitor plane, scaled to the output */
+static void render_drm_plane(gsr_capture_kms *self, gsr_color_conversion *color_conversion, gsr_capture_metadata *capture_metadata, const gsr_kms_response_item *plane_drm_fd, vec2i target_pos, vec2i output_size, vec2i framebuffer_size) {
+    const vec2d scale = {
+        self->capture_size.x == 0 ? 0 : (double)output_size.x / (double)self->capture_size.x,
+        self->capture_size.y == 0 ? 0 : (double)output_size.y / (double)self->capture_size.y
+    };
+
+    const gsr_monitor_rotation plane_rotation = kms_rotation_to_gsr_monitor_rotation(plane_drm_fd->rotation);
+    const gsr_monitor_rotation rotation = sub_rotations(self->display_server_monitor_rotation, plane_rotation);
+
+    const vec2i plane_size = {plane_drm_fd->dst_w, plane_drm_fd->dst_h};
+    vec2i plane_pos = {plane_drm_fd->dst_x, plane_drm_fd->dst_y};
+    switch(rotation) {
+        case GSR_MONITOR_ROT_0:
+            break;
+        case GSR_MONITOR_ROT_90:
+            plane_pos = swap_vec2i(plane_pos);
+            plane_pos.x = framebuffer_size.x - plane_pos.x;
+            // TODO: Remove this horrible hack
+            plane_pos.x -= plane_size.x;
+            break;
+        case GSR_MONITOR_ROT_180:
+            plane_pos.x = framebuffer_size.x - plane_pos.x;
+            plane_pos.y = framebuffer_size.y - plane_pos.y;
+            // TODO: Remove this horrible hack
+            plane_pos.x -= plane_size.x;
+            plane_pos.y -= plane_size.y;
+            break;
+        case GSR_MONITOR_ROT_270:
+            plane_pos = swap_vec2i(plane_pos);
+            plane_pos.y = framebuffer_size.y - plane_pos.y;
+            // TODO: Remove this horrible hack
+            plane_pos.y -= plane_size.y;
+            break;
+    }
+
+    plane_pos.x -= self->params.region_position.x;
+    plane_pos.y -= self->params.region_position.y;
+
+    plane_pos.x *= scale.x;
+    plane_pos.y *= scale.y;
+
+    plane_pos.x += target_pos.x;
+    plane_pos.y += target_pos.y;
+
+    EGLImage image = gsr_capture_kms_create_egl_image_with_fallback(self, plane_drm_fd);
+    if(!image)
+        return;
+
+    gsr_capture_kms_bind_image_to_input_texture_with_fallback(self, image);
+    self->params.egl->eglDestroyImage(self->params.egl->egl_display, image);
+
+    self->params.egl->glEnable(GL_SCISSOR_TEST);
+    self->params.egl->glScissor(target_pos.x, target_pos.y, output_size.x, output_size.y);
+
+    gsr_color_conversion_draw(color_conversion, self->external_texture_fallback ? self->external_input_texture_id : self->input_texture_id,
+        plane_pos, (vec2i){plane_size.x * scale.x, plane_size.y * scale.y},
+        (vec2i){plane_drm_fd->src_x, plane_drm_fd->src_y}, (vec2i){plane_drm_fd->src_w, plane_drm_fd->src_h}, (vec2i){plane_drm_fd->width, plane_drm_fd->height},
+        gsr_monitor_rotation_to_rotation(rotation), capture_metadata->flip, GSR_SOURCE_COLOR_RGB, self->external_texture_fallback);
+
+    self->params.egl->glDisable(GL_SCISSOR_TEST);
+}
+
+static int gsr_capture_kms_capture(gsr_capture *cap, gsr_capture_metadata *capture_metadata, gsr_color_conversion *color_conversion) {
+    gsr_capture_kms *self = cap->priv;
+
+    if(!self->drm_fd || self->params.kms_response->num_items == 0)
+        return -1;
+
+    const vec2i framebuffer_size = rotate_capture_size_if_rotated(self, (vec2i){ self->drm_fd->src_w, self->drm_fd->src_h }, self->final_monitor_rotation);
+
+    //self->params.egl->glFlush();
+    //self->params.egl->glFinish();
+
+    /* Gather all planes that are displayed on the captured monitor. Overlay planes are not used on x11 (combined plane) */
+    const gsr_kms_response_item *planes[GSR_KMS_MAX_ITEMS];
+    int num_planes = 0;
+    planes[num_planes++] = self->drm_fd;
+    if(!self->capture_is_combined_plane) {
+        for(int i = 0; i < self->params.kms_response->num_items && num_planes < GSR_KMS_MAX_ITEMS; ++i) {
+            const gsr_kms_response_item *item = &self->params.kms_response->items[i];
+            if(item->plane_type == KMS_PLANE_TYPE_OVERLAY && item->connector_id == self->drm_fd->connector_id)
+                planes[num_planes++] = item;
+        }
+    }
+
+    /* Sort the planes by zpos, from bottom to top. Insertion sort to keep planes with the same zpos in the order the drm driver returned them (stable) */
+    for(int i = 1; i < num_planes; ++i) {
+        const gsr_kms_response_item *plane = planes[i];
+        int j = i - 1;
+        for(; j >= 0 && planes[j]->zpos > plane->zpos; --j) {
+            planes[j + 1] = planes[j];
+        }
+        planes[j + 1] = plane;
+    }
+
+    for(int i = 0; i < num_planes; ++i) {
+        if(planes[i] == self->drm_fd)
+            render_monitor_plane(self, color_conversion, capture_metadata);
+        else
+            render_drm_plane(self, color_conversion, capture_metadata, planes[i], self->target_pos, self->output_size, framebuffer_size);
+    }
 
     if(self->params.record_cursor) {
         gsr_kms_response_item *cursor_drm_fd = find_cursor_drm_if_on_monitor(self, self->drm_fd->connector_id, self->capture_is_combined_plane);
@@ -624,7 +719,6 @@ static int gsr_capture_kms_capture(gsr_capture *cap, gsr_capture_metadata *captu
             cursor_monitor_offset.y += self->params.region_position.y;
             render_x11_cursor(self, color_conversion, capture_metadata, cursor_monitor_offset, self->target_pos, self->output_size);
         } else if(cursor_drm_fd) {
-            const vec2i framebuffer_size = rotate_capture_size_if_rotated(self, (vec2i){ self->drm_fd->src_w, self->drm_fd->src_h }, self->final_monitor_rotation);
             render_drm_cursor(self, color_conversion, capture_metadata, cursor_drm_fd, self->target_pos, self->output_size, framebuffer_size);
         }
     }