backend/wayland: implement the output layer API

The output layer API is implemented using subsurfaces. I chose to
implement this API in the Wayland backend before doing so in the DRM
backend, because it's way easier on Wayland. On DRM, one needs to figure
out how buffers can be mapped to KMS planes (libliftoff can help) and
perform atomic test-only commits (our current DRM backend isn't ready
for this).

backend/wayland/backend.c

@@ -102,6 +102,9 @@ static void registry_global(void *data, struct wl_registry *registry,
 	if (strcmp(iface, wl_compositor_interface.name) == 0) {
 		wl->compositor = wl_registry_bind(registry, name,
 			&wl_compositor_interface, 4);
+	} else if (strcmp(iface, wl_subcompositor_interface.name) == 0) {
+		wl->subcompositor = wl_registry_bind(registry, name,
+			&wl_subcompositor_interface, 1);
 	} else if (strcmp(iface, wl_seat_interface.name) == 0) {
 		wl->seat = wl_registry_bind(registry, name,
 			&wl_seat_interface, 5);

backend/wayland/output.c

@@ -135,7 +135,9 @@ static struct wlr_wl_buffer *create_wl_buffer(struct wlr_wl_backend *wl,
 		return NULL;
 	}
 
-	if (attribs.width != required_width || attribs.height != required_height) {
+	if (required_width > 0 && required_height > 0 &&
+			(attribs.width != required_width ||
+			attribs.height != required_height)) {
 		return NULL;
 	}
 
@@ -216,6 +218,48 @@ static bool output_commit(struct wlr_output *wlr_output) {
 		}
 	}
 
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_LAYERS) {
+		struct wlr_wl_output_layer *layer, *prev = NULL;
+		wl_list_for_each(layer, &wlr_output->pending.layers, base.pending.link) {
+			if (prev != NULL) {
+				wl_subsurface_place_above(layer->subsurface, prev->surface);
+			}
+
+			prev = layer;
+		}
+	}
+
+	struct wlr_wl_output_layer *layer;
+	wl_list_for_each(layer, &wlr_output->pending.layers, base.pending.link) {
+		if (layer->base.pending.committed & WLR_OUTPUT_LAYER_STATE_BUFFER) {
+			layer->base.accepted = false;
+
+			struct wlr_buffer *wlr_buffer = layer->base.pending.buffer;
+			if (wlr_buffer == NULL) {
+				wl_surface_attach(layer->surface, NULL, 0, 0);
+				layer->base.accepted = true;
+			} else {
+				struct wlr_wl_buffer *buffer =
+					create_wl_buffer(output->backend, wlr_buffer, 0, 0);
+				if (buffer != NULL) {
+					wl_surface_attach(layer->surface,
+						buffer->wl_buffer, 0, 0);
+					layer->base.accepted = true;
+				}
+			}
+		}
+
+		if (layer->base.pending.committed & WLR_OUTPUT_LAYER_STATE_POSITION) {
+			wl_subsurface_set_position(layer->subsurface,
+				layer->base.pending.x, layer->base.pending.y);
+		}
+
+		if (layer->base.pending.committed != 0) {
+			// TODO: make sure to commit the parent surface too
+			wl_surface_commit(layer->surface);
+		}
+	}
+
 	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_BUFFER) {
 		struct wp_presentation_feedback *wp_feedback = NULL;
 		if (output->backend->presentation != NULL) {
@@ -415,13 +459,49 @@ static bool output_move_cursor(struct wlr_output *_output, int x, int y) {
 	return true;
 }
 
+static struct wlr_output_layer *output_create_layer(
+		struct wlr_output *wlr_output) {
+	struct wlr_wl_output *output = get_wl_output_from_output(wlr_output);
+	struct wlr_wl_backend *backend = output->backend;
+
+	struct wlr_wl_output_layer *layer = calloc(1, sizeof(*layer));
+	if (layer == NULL) {
+		return NULL;
+	}
+	wlr_output_layer_init(&layer->base, wlr_output);
+	layer->surface = wl_compositor_create_surface(backend->compositor);
+	layer->subsurface = wl_subcompositor_get_subsurface(backend->subcompositor,
+		layer->surface, output->surface);
+
+	struct wl_region *empty_region =
+		wl_compositor_create_region(backend->compositor);
+	wl_surface_set_input_region(layer->surface, empty_region);
+	wl_region_destroy(empty_region);
+
+	// Initially layers have no buffer, so there's nothing special that needs
+	// to be done to display them
+	layer->base.accepted = true;
+
+	return &layer->base;
+}
+
+static void output_destroy_layer(struct wlr_output_layer *wlr_layer) {
+	struct wlr_wl_output_layer *layer =
+		(struct wlr_wl_output_layer *)wlr_layer;
+	wl_subsurface_destroy(layer->subsurface);
+	wl_surface_destroy(layer->surface);
+	free(layer);
+}
+
 static const struct wlr_output_impl output_impl = {
 	.destroy = output_destroy,
 	.attach_render = output_attach_render,
 	.attach_buffer  = output_attach_buffer,
 	.commit = output_commit,
 	.set_cursor = output_set_cursor,
 	.move_cursor = output_move_cursor,
+	.create_layer = output_create_layer,
+	.destroy_layer = output_destroy_layer,
 };
 
 bool wlr_output_is_wl(struct wlr_output *wlr_output) {

include/backend/wayland.h

@@ -9,10 +9,11 @@
 #include <wayland-util.h>
 
 #include <wlr/backend/wayland.h>
+#include <wlr/render/drm_format_set.h>
 #include <wlr/render/egl.h>
 #include <wlr/render/wlr_renderer.h>
 #include <wlr/types/wlr_box.h>
-#include <wlr/render/drm_format_set.h>
+#include <wlr/types/wlr_output_layer.h>
 
 struct wlr_wl_backend {
 	struct wlr_backend backend;
@@ -32,6 +33,7 @@ struct wlr_wl_backend {
 	struct wl_event_source *remote_display_src;
 	struct wl_registry *registry;
 	struct wl_compositor *compositor;
+	struct wl_subcompositor *subcompositor;
 	struct xdg_wm_base *xdg_wm_base;
 	struct zxdg_decoration_manager_v1 *zxdg_decoration_manager_v1;
 	struct zwp_pointer_gestures_v1 *zwp_pointer_gestures_v1;
@@ -85,6 +87,12 @@ struct wlr_wl_output {
 	} cursor;
 };
 
+struct wlr_wl_output_layer {
+	struct wlr_output_layer base;
+	struct wl_surface *surface;
+	struct wl_subsurface *subsurface;
+};
+
 struct wlr_wl_input_device {
 	struct wlr_input_device wlr_input_device;
 	uint32_t fingers;