render: unify texture format enumeration

Instead of having separate get_shm_texture_formats and
get_dmabuf_texture_formats functions, have a single unified
get_texture_formats function. This brings the renderer API in line
with wlr_backend_impl.

backend/drm/backend.c

@@ -244,7 +244,7 @@ struct wlr_backend *wlr_drm_backend_create(struct wl_display *display,
 		// to be able to texture from them
 		struct wlr_renderer *renderer = drm->mgpu_renderer.wlr_rend;
 		const struct wlr_drm_format_set *texture_formats =
-			wlr_renderer_get_dmabuf_texture_formats(renderer);
+			wlr_renderer_get_texture_formats(renderer, WLR_BUFFER_CAP_DMABUF);
 		if (texture_formats == NULL) {
 			wlr_log(WLR_ERROR, "Failed to query renderer texture formats");
 			goto error_mgpu_renderer;

include/render/gles2.h

@@ -73,6 +73,8 @@ struct wlr_gles2_renderer {
 
 	struct wlr_gles2_buffer *current_buffer;
 	uint32_t viewport_width, viewport_height;
+
+	struct wlr_drm_format_set data_ptr_texture_formats;
 };
 
 struct wlr_gles2_buffer {
@@ -115,8 +117,7 @@ bool is_gles2_pixel_format_supported(const struct wlr_gles2_renderer *renderer,
 const struct wlr_gles2_pixel_format *get_gles2_format_from_drm(uint32_t fmt);
 const struct wlr_gles2_pixel_format *get_gles2_format_from_gl(
 	GLint gl_format, GLint gl_type, bool alpha);
-const uint32_t *get_gles2_shm_formats(const struct wlr_gles2_renderer *renderer,
+void init_gles2_data_ptr_formats(struct wlr_gles2_renderer *renderer);
-	size_t *len);
 
 struct wlr_gles2_renderer *gles2_get_renderer(
 	struct wlr_renderer *wlr_renderer);

include/render/pixman.h

@@ -50,6 +50,6 @@ struct wlr_pixman_texture {
 
 pixman_format_code_t get_pixman_format_from_drm(uint32_t fmt);
 uint32_t get_drm_format_from_pixman(pixman_format_code_t fmt);
-const uint32_t *get_pixman_drm_formats(size_t *len);
+void init_pixman_formats(struct wlr_pixman_renderer *renderer);
 
 #endif

include/wlr/render/interface.h

@@ -32,10 +32,6 @@ struct wlr_renderer_impl {
 		const float matrix[static 9], float alpha);
 	void (*render_quad_with_matrix)(struct wlr_renderer *renderer,
 		const float color[static 4], const float matrix[static 9]);
-	const uint32_t *(*get_shm_texture_formats)(
-		struct wlr_renderer *renderer, size_t *len);
-	const struct wlr_drm_format_set *(*get_dmabuf_texture_formats)(
-		struct wlr_renderer *renderer);
 	const struct wlr_drm_format_set *(*get_render_formats)(
 		struct wlr_renderer *renderer);
 	uint32_t (*preferred_read_format)(struct wlr_renderer *renderer);
@@ -46,6 +42,8 @@ struct wlr_renderer_impl {
 	void (*destroy)(struct wlr_renderer *renderer);
 	int (*get_drm_fd)(struct wlr_renderer *renderer);
 	uint32_t (*get_render_buffer_caps)(struct wlr_renderer *renderer);
+	const struct wlr_drm_format_set *(*get_texture_formats)(
+		struct wlr_renderer *renderer, uint32_t buffer_caps);
 	struct wlr_texture *(*texture_from_buffer)(struct wlr_renderer *renderer,
 		struct wlr_buffer *buffer);
 };

include/wlr/render/wlr_renderer.h

@@ -75,18 +75,12 @@ void wlr_render_rect(struct wlr_renderer *r, const struct wlr_box *box,
  */
 void wlr_render_quad_with_matrix(struct wlr_renderer *r,
 	const float color[static 4], const float matrix[static 9]);
-/**
- * Get the shared-memory formats supporting import usage. Buffers allocated
- * with a format from this list may be imported via wlr_texture_from_pixels.
- */
-const uint32_t *wlr_renderer_get_shm_texture_formats(
-	struct wlr_renderer *r, size_t *len);
 /**
  * Get the DMA-BUF formats supporting sampling usage. Buffers allocated with
- * a format from this list may be imported via wlr_texture_from_dmabuf.
+ * a format from this list may be imported via wlr_texture_from_buffer.
  */
-const struct wlr_drm_format_set *wlr_renderer_get_dmabuf_texture_formats(
+const struct wlr_drm_format_set *wlr_renderer_get_texture_formats(
-	struct wlr_renderer *renderer);
+	struct wlr_renderer *renderer, uint32_t buffer_caps);
 /**
  * Reads out of pixels of the currently bound surface into data. `stride` is in
  * bytes.

render/gles2/pixel_format.c

@@ -136,16 +136,12 @@ const struct wlr_gles2_pixel_format *get_gles2_format_from_gl(
 	return NULL;
 }
 
-const uint32_t *get_gles2_shm_formats(const struct wlr_gles2_renderer *renderer,
+void init_gles2_data_ptr_formats(struct wlr_gles2_renderer *renderer) {
-		size_t *len) {
-	static uint32_t shm_formats[sizeof(formats) / sizeof(formats[0])];
-	size_t j = 0;
 	for (size_t i = 0; i < sizeof(formats) / sizeof(formats[0]); i++) {
 		if (!is_gles2_pixel_format_supported(renderer, &formats[i])) {
 			continue;
 		}
-		shm_formats[j++] = formats[i].drm_format;
+		wlr_drm_format_set_add(&renderer->data_ptr_texture_formats,
+			formats[i].drm_format, DRM_FORMAT_MOD_LINEAR);
 	}
-	*len = j;
-	return shm_formats;
 }

render/gles2/renderer.c

@@ -378,16 +378,16 @@ static void gles2_render_quad_with_matrix(struct wlr_renderer *wlr_renderer,
 	pop_gles2_debug(renderer);
 }
 
-static const uint32_t *gles2_get_shm_texture_formats(
+static const struct wlr_drm_format_set *gles2_get_texture_formats(
-		struct wlr_renderer *wlr_renderer, size_t *len) {
+		struct wlr_renderer *wlr_renderer, uint32_t buffer_caps) {
-	struct wlr_gles2_renderer *renderer = gles2_get_renderer(wlr_renderer);
-	return get_gles2_shm_formats(renderer, len);
-}
-
-static const struct wlr_drm_format_set *gles2_get_dmabuf_texture_formats(
-		struct wlr_renderer *wlr_renderer) {
 	struct wlr_gles2_renderer *renderer = gles2_get_renderer(wlr_renderer);
+	if (buffer_caps & WLR_BUFFER_CAP_DMABUF) {
 		return wlr_egl_get_dmabuf_texture_formats(renderer->egl);
+	} else if (buffer_caps & WLR_BUFFER_CAP_DATA_PTR) {
+		return &renderer->data_ptr_texture_formats;
+	} else {
+		return NULL;
+	}
 }
 
 static const struct wlr_drm_format_set *gles2_get_render_formats(
@@ -529,6 +529,8 @@ static void gles2_destroy(struct wlr_renderer *wlr_renderer) {
 	wlr_egl_unset_current(renderer->egl);
 	wlr_egl_destroy(renderer->egl);
 
+	wlr_drm_format_set_finish(&renderer->data_ptr_texture_formats);
+
 	if (renderer->drm_fd >= 0) {
 		close(renderer->drm_fd);
 	}
@@ -545,8 +547,7 @@ static const struct wlr_renderer_impl renderer_impl = {
 	.scissor = gles2_scissor,
 	.render_subtexture_with_matrix = gles2_render_subtexture_with_matrix,
 	.render_quad_with_matrix = gles2_render_quad_with_matrix,
-	.get_shm_texture_formats = gles2_get_shm_texture_formats,
+	.get_texture_formats = gles2_get_texture_formats,
-	.get_dmabuf_texture_formats = gles2_get_dmabuf_texture_formats,
 	.get_render_formats = gles2_get_render_formats,
 	.preferred_read_format = gles2_preferred_read_format,
 	.read_pixels = gles2_read_pixels,
@@ -791,6 +792,8 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 			GL_DEBUG_TYPE_PUSH_GROUP_KHR, GL_DONT_CARE, 0, NULL, GL_FALSE);
 	}
 
+	init_gles2_data_ptr_formats(renderer);
+
 	push_gles2_debug(renderer);
 
 	GLuint prog;

render/pixman/pixel_format.c

@@ -121,11 +121,9 @@ uint32_t get_drm_format_from_pixman(pixman_format_code_t fmt) {
 	return DRM_FORMAT_INVALID;
 }
 
-const uint32_t *get_pixman_drm_formats(size_t *len) {
+void init_pixman_formats(struct wlr_pixman_renderer *renderer) {
-	static uint32_t drm_formats[sizeof(formats) / sizeof(formats[0])];
-	*len = sizeof(formats) / sizeof(formats[0]);
 	for (size_t i = 0; i < sizeof(formats) / sizeof(formats[0]); i++) {
-		drm_formats[i] = formats[i].drm_format;
+		wlr_drm_format_set_add(&renderer->drm_formats, formats[i].drm_format,
+			DRM_FORMAT_MOD_INVALID);
 	}
-	return drm_formats;
 }

render/pixman/renderer.c

@@ -329,9 +329,14 @@ static void pixman_render_quad_with_matrix(struct wlr_renderer *wlr_renderer,
 	pixman_image_unref(image);
 }
 
-static const uint32_t *pixman_get_shm_texture_formats(
+static const struct wlr_drm_format_set *pixman_get_texture_formats(
-		struct wlr_renderer *wlr_renderer, size_t *len) {
+		struct wlr_renderer *wlr_renderer, uint32_t buffer_caps) {
-	return get_pixman_drm_formats(len);
+	struct wlr_pixman_renderer *renderer = get_renderer(wlr_renderer);
+	if (buffer_caps & WLR_BUFFER_CAP_DATA_PTR) {
+		return &renderer->drm_formats;
+	} else {
+		return NULL;
+	}
 }
 
 static const struct wlr_drm_format_set *pixman_get_render_formats(
@@ -501,7 +506,7 @@ static const struct wlr_renderer_impl renderer_impl = {
 	.scissor = pixman_scissor,
 	.render_subtexture_with_matrix = pixman_render_subtexture_with_matrix,
 	.render_quad_with_matrix = pixman_render_quad_with_matrix,
-	.get_shm_texture_formats = pixman_get_shm_texture_formats,
+	.get_texture_formats = pixman_get_texture_formats,
 	.get_render_formats = pixman_get_render_formats,
 	.texture_from_buffer = pixman_texture_from_buffer,
 	.bind_buffer = pixman_bind_buffer,
@@ -523,13 +528,7 @@ struct wlr_renderer *wlr_pixman_renderer_create(void) {
 	wl_list_init(&renderer->buffers);
 	wl_list_init(&renderer->textures);
 
-	size_t len = 0;
+	init_pixman_formats(renderer);
-	const uint32_t *formats = get_pixman_drm_formats(&len);
-
-	for (size_t i = 0; i < len; ++i) {
-		wlr_drm_format_set_add(&renderer->drm_formats, formats[i],
-				DRM_FORMAT_MOD_INVALID);
-	}
 
 	return &renderer->wlr_renderer;
 }

render/wlr_renderer.c

@@ -37,7 +37,7 @@ void wlr_renderer_init(struct wlr_renderer *renderer,
 	assert(impl->scissor);
 	assert(impl->render_subtexture_with_matrix);
 	assert(impl->render_quad_with_matrix);
-	assert(impl->get_shm_texture_formats);
+	assert(impl->get_texture_formats);
 	assert(impl->get_render_buffer_caps);
 	renderer->impl = impl;
 
@@ -166,17 +166,9 @@ void wlr_render_quad_with_matrix(struct wlr_renderer *r,
 	r->impl->render_quad_with_matrix(r, color, matrix);
 }
 
-const uint32_t *wlr_renderer_get_shm_texture_formats(struct wlr_renderer *r,
+const struct wlr_drm_format_set *wlr_renderer_get_texture_formats(
-		size_t *len) {
+		struct wlr_renderer *r, uint32_t buffer_caps) {
-	return r->impl->get_shm_texture_formats(r, len);
+	return r->impl->get_texture_formats(r, buffer_caps);
-}
-
-const struct wlr_drm_format_set *wlr_renderer_get_dmabuf_texture_formats(
-		struct wlr_renderer *r) {
-	if (!r->impl->get_dmabuf_texture_formats) {
-		return NULL;
-	}
-	return r->impl->get_dmabuf_texture_formats(r);
 }
 
 const struct wlr_drm_format_set *wlr_renderer_get_render_formats(
@@ -209,19 +201,19 @@ bool wlr_renderer_init_wl_shm(struct wlr_renderer *r,
 		return false;
 	}
 
-	size_t len;
+	const struct wlr_drm_format_set *shm_formats =
-	const uint32_t *formats = wlr_renderer_get_shm_texture_formats(r, &len);
+		wlr_renderer_get_texture_formats(r, WLR_BUFFER_CAP_DATA_PTR);
-	if (formats == NULL) {
+	if (shm_formats == NULL) {
-		wlr_log(WLR_ERROR, "Failed to initialize wl_shm: "
+		wlr_log(WLR_ERROR, "Failed to initialize shm: cannot get formats");
-			"cannot get renderer formats");
 		return false;
 	}
 
 	bool argb8888 = false, xrgb8888 = false;
-	for (size_t i = 0; i < len; ++i) {
+	for (size_t i = 0; i < shm_formats->len; ++i) {
 		// ARGB8888 and XRGB8888 must be supported and are implicitly
 		// advertised by wl_display_init_shm
-		enum wl_shm_format fmt = convert_drm_format_to_wl_shm(formats[i]);
+		enum wl_shm_format fmt =
+			convert_drm_format_to_wl_shm(shm_formats->formats[i]->format);
 		switch (fmt) {
 		case WL_SHM_FORMAT_ARGB8888:
 			argb8888 = true;
@@ -248,7 +240,7 @@ bool wlr_renderer_init_wl_display(struct wlr_renderer *r,
 		return false;
 	}
 
-	if (wlr_renderer_get_dmabuf_texture_formats(r) != NULL) {
+	if (wlr_renderer_get_texture_formats(r, WLR_BUFFER_CAP_DMABUF) != NULL) {
 		if (wlr_renderer_get_drm_fd(r) >= 0) {
 			if (wlr_drm_create(wl_display, r) == NULL) {
 				return false;

types/wlr_drm.c

@@ -160,7 +160,7 @@ static void drm_bind(struct wl_client *client, void *data,
 	wl_drm_send_capabilities(resource, WL_DRM_CAPABILITY_PRIME);
 
 	const struct wlr_drm_format_set *formats =
-		wlr_renderer_get_dmabuf_texture_formats(drm->renderer);
+		wlr_renderer_get_texture_formats(drm->renderer, WLR_BUFFER_CAP_DMABUF);
 	if (formats == NULL) {
 		return;
 	}

types/wlr_linux_dmabuf_v1.c

@@ -879,8 +879,8 @@ static void linux_dmabuf_send_modifiers(struct wl_resource *resource,
 
 static void linux_dmabuf_send_formats(struct wlr_linux_dmabuf_v1 *linux_dmabuf,
 		struct wl_resource *resource) {
-	const struct wlr_drm_format_set *formats =
+	const struct wlr_drm_format_set *formats = wlr_renderer_get_texture_formats(
-		wlr_renderer_get_dmabuf_texture_formats(linux_dmabuf->renderer);
+		linux_dmabuf->renderer, WLR_BUFFER_CAP_DMABUF);
 	if (formats == NULL) {
 		return;
 	}