render/color: introduce color_transform_compose

It flattens color transform pipelines, and facilitates building
pipelines of variable content

include/render/color.h

@@ -87,6 +87,16 @@ struct wlr_color_transform_inverse_eotf *wlr_color_transform_inverse_eotf_from_b
 struct wlr_color_transform_lut_3x1d *color_transform_lut_3x1d_from_base(
 	struct wlr_color_transform *tr);
 
+/**
+ * Create a simplified / normalized wlr_color_transform pipeline.
+ * `transforms` may contain NULL transforms, they will be interpreted as the
+ * identity transform, and removed.
+ * `*result` may be set to a tranform of a type different from
+ * `wlr_color_transform_pipeline`, or to NULL if all input transforms are NULL
+ */
+bool color_transform_compose(struct wlr_color_transform **result,
+	struct wlr_color_transform **transforms, size_t len);
+
 /**
  * Compute the matrix to convert RGB color values to CIE 1931 XYZ.
  */

render/color.c

@@ -268,6 +268,71 @@ void wlr_color_transform_eval(struct wlr_color_transform *tr,
 	}
 }
 
+static size_t color_transform_compose_collect(struct wlr_color_transform **out,
+		size_t out_capacity, struct wlr_color_transform **transforms, size_t len) {
+	size_t count = 0;
+	for (size_t i = 0; i < len; i++) {
+		struct wlr_color_transform *transform = transforms[i];
+		if (transform == NULL) {
+			continue;
+		}
+
+		if (transform->type == COLOR_TRANSFORM_PIPELINE) {
+			struct wlr_color_transform_pipeline *pipeline = wl_container_of(transform,
+				pipeline, base);
+			count += color_transform_compose_collect(out, out_capacity,
+				pipeline->transforms, pipeline->len);
+		} else {
+			if (out_capacity > 0) {
+				*out = wlr_color_transform_ref(transform);
+				out++;
+				out_capacity--;
+			}
+			count++;
+		}
+	}
+	return count;
+}
+
+bool color_transform_compose(struct wlr_color_transform **result,
+		struct wlr_color_transform **transforms, size_t len) {
+	// The normalized form has the following properties :
+	// - No NULL transform in a pipeline
+	// - No pipeline of length 1
+	// - No nested pipelines
+	bool status = false;
+
+	size_t result_len = color_transform_compose_collect(NULL, 0, transforms, len);
+	if (result_len == 0) {
+		*result = NULL;
+		return true;
+	}
+
+	struct wlr_color_transform **result_transforms = calloc(result_len,
+		sizeof(result_transforms[0]));
+	if (result_transforms == NULL) {
+		return false;
+	}
+	color_transform_compose_collect(result_transforms, result_len, transforms, len);
+
+	if (result_len == 1) {
+		*result = wlr_color_transform_ref(result_transforms[0]);
+	} else {
+		*result = wlr_color_transform_init_pipeline(result_transforms, result_len);
+		if (*result == NULL) {
+			goto cleanup_transforms;
+		}
+	}
+	status = true;
+
+cleanup_transforms:
+	for (size_t i = 0; i < result_len; i++) {
+		wlr_color_transform_unref(result_transforms[i]);
+	}
+	free(result_transforms);
+	return status;
+}
+
 void wlr_color_primaries_from_named(struct wlr_color_primaries *out,
 		enum wlr_color_named_primaries named) {
 	switch (named) {

types/scene/wlr_scene.c

@@ -2159,9 +2159,10 @@ static void scene_output_state_attempt_gamma(struct wlr_scene_output *scene_outp
 static bool scene_output_combine_color_transforms(
 		struct wlr_scene_output *scene_output, struct wlr_color_transform *supplied,
 		const struct wlr_output_image_description *img_desc, bool render_gamma_lut) {
-	struct wlr_color_transform *transforms[3] = {0};
+	bool result = false;
-	const size_t transforms_cap = sizeof(transforms) / sizeof(transforms[0]);
+	struct wlr_color_transform *color_matrix = NULL;
-	size_t transforms_len = 0;
+	struct wlr_color_transform *inv_eotf = NULL;
+	struct wlr_color_transform *user_gamma = NULL;
 
 	if (img_desc != NULL) {
 		assert(supplied == NULL);
@@ -2171,42 +2172,35 @@ static bool scene_output_combine_color_transforms(
 		wlr_color_primaries_from_named(&primaries, img_desc->primaries);
 		float matrix[9];
 		wlr_color_primaries_transform_absolute_colorimetric(&primaries_srgb, &primaries, matrix);
-		assert(transforms_len < transforms_cap);
+		color_matrix = wlr_color_transform_init_matrix(matrix);
-		transforms[transforms_len++] =	wlr_color_transform_init_matrix(matrix);
+		inv_eotf = wlr_color_transform_init_linear_to_inverse_eotf(img_desc->transfer_function);
-		assert(transforms_len < transforms_cap);
+		if (color_matrix == NULL || inv_eotf == NULL) {
-		transforms[transforms_len++] = wlr_color_transform_init_linear_to_inverse_eotf(
+			goto cleanup_transforms;
-			img_desc->transfer_function);
+		}
 	} else if (supplied != NULL) {
-		assert(transforms_len < transforms_cap);
+		inv_eotf = wlr_color_transform_ref(supplied);
-		transforms[transforms_len++] = wlr_color_transform_ref(supplied);
 	} else {
-		assert(transforms_len < transforms_cap);
+		inv_eotf = wlr_color_transform_init_linear_to_inverse_eotf(
-		transforms[transforms_len++] = wlr_color_transform_init_linear_to_inverse_eotf(
 			WLR_COLOR_TRANSFER_FUNCTION_GAMMA22);
+		if (inv_eotf == NULL) {
+			goto cleanup_transforms;
+		}
 	}
 
 	struct wlr_color_transform *gamma_lut = scene_output->gamma_lut_color_transform;
 	if (gamma_lut != NULL && render_gamma_lut) {
-		assert(transforms_len < transforms_cap);
+		user_gamma = wlr_color_transform_ref(gamma_lut);
-		transforms[transforms_len++] = wlr_color_transform_ref(gamma_lut);
 	}
 
-	for (size_t i = 0; i < transforms_len; ++i) {
-		if (transforms[i] == NULL) {
-			goto err_transforms;
-		}
-	}
 	struct wlr_color_transform *combined;
-	if (transforms_len == 1) {
+	struct wlr_color_transform *transforms[] = {
-		combined = wlr_color_transform_ref(transforms[0]);
+		color_matrix,
-	} else {
+		inv_eotf,
-		combined = wlr_color_transform_init_pipeline(transforms, transforms_len);
+		user_gamma,
-	}
+	};
-	if (combined == NULL) {
+	const size_t transforms_len = sizeof(transforms) / sizeof(transforms[0]);
-		goto err_transforms;
+	if (!color_transform_compose(&combined, transforms, transforms_len)) {
-	}
+		goto cleanup_transforms;
-	for (size_t i = 0; i < transforms_len; ++i) {
-		wlr_color_transform_unref(transforms[i]);
 	}
 
 	wlr_color_transform_unref(scene_output->prev_gamma_lut_color_transform);
@@ -2216,13 +2210,13 @@ static bool scene_output_combine_color_transforms(
 	wlr_color_transform_unref(scene_output->combined_color_transform);
 	scene_output->combined_color_transform = combined;
 
-	return true;
+	result = true;
 
-err_transforms:
+cleanup_transforms:
-	for (size_t i = 0; i < transforms_len; ++i) {
+	wlr_color_transform_unref(color_matrix);
-		wlr_color_transform_unref(transforms[i]);
+	wlr_color_transform_unref(inv_eotf);
-	}
+	wlr_color_transform_unref(user_gamma);
-	return false;
+	return result;
 }
 
 bool wlr_scene_output_build_state(struct wlr_scene_output *scene_output,