drm: avoid dma-buf feedback endless loop

Currently we have the following situation: we add a scanout tranche
because if the client re-allocates with another format/modifier, the
chances of being placed in a DRM/KMS plane is higher.

But then we run out of overlay planes. So we remove the scanout tranche,
because the format/modifier available in the renderer tranche are
optimal for rendering.

Now Weston detects again that the format/modifier is what may be
avoiding the view being place in a plane, re-adding the scanout tranche.
And we have an endless loop.

To avoid this, let's accumulate the reasons why placing the view in a
place failed. So if we detect that we don't have planes available, no
matter the format/modifier, we won't add the scanout tranche.

Signed-off-by: Leandro Ribeiro <leandro.ribeiro@collabora.com>
(cherry picked from commit 2d88b9b03a8f4d32a145ae8e34d90a6a9206d9b0)

libweston/backend-drm/state-propose.c

@@ -394,6 +394,7 @@ drm_output_find_plane_for_view(struct drm_output_state *state,
 
 	bool view_matches_entire_output, scanout_has_view_assigned;
 	uint32_t possible_plane_mask = 0;
+	bool any_candidate_picked = false;
 
 	pnode->try_view_on_plane_failure_reasons = FAILURE_REASONS_NONE;
 
@@ -408,23 +409,19 @@ drm_output_find_plane_for_view(struct drm_output_state *state,
 	if (buffer->type == WESTON_BUFFER_SOLID) {
 		pnode->try_view_on_plane_failure_reasons |=
 			FAILURE_REASONS_BUFFER_TYPE;
-		return NULL;
 	} else if (buffer->type == WESTON_BUFFER_SHM) {
-		if (!output->cursor_plane || device->cursors_are_broken) {
+		if (!output->cursor_plane || device->cursors_are_broken)
 			pnode->try_view_on_plane_failure_reasons |=
 				FAILURE_REASONS_BUFFER_TYPE;
-			return NULL;
-		}
 
-		/* Even though this is a SHM buffer, pixel_format stores the
+		/* Even though this is a SHM buffer, pixel_format stores
-		 * format code as DRM FourCC */
+		 * the format code as DRM FourCC */
 		if (buffer->pixel_format->format != DRM_FORMAT_ARGB8888) {
 			drm_debug(b, "\t\t\t\t[view] not placing view %p on "
 				     "plane; SHM buffers must be ARGB8888 for "
 				     "cursor view\n", ev);
 			pnode->try_view_on_plane_failure_reasons |=
 				FAILURE_REASONS_FB_FORMAT_INCOMPATIBLE;
-			return NULL;
 		}
 
 		if (buffer->width > device->cursor_width ||
@@ -434,9 +431,9 @@ drm_output_find_plane_for_view(struct drm_output_state *state,
 				     ev, buffer->width, buffer->height);
 			pnode->try_view_on_plane_failure_reasons |=
 				FAILURE_REASONS_BUFFER_TOO_BIG;
-			return NULL;
 		}
 
+		if (pnode->try_view_on_plane_failure_reasons == FAILURE_REASONS_NONE)
 			possible_plane_mask = (1 << output->cursor_plane->plane_idx);
 	} else {
 		if (mode == DRM_OUTPUT_PROPOSE_STATE_RENDERER_ONLY) {
@@ -453,20 +450,16 @@ drm_output_find_plane_for_view(struct drm_output_state *state,
 				possible_plane_mask |= 1 << plane->plane_idx;
 		}
 
-		if (!possible_plane_mask) {
+		if (!possible_plane_mask)
 			pnode->try_view_on_plane_failure_reasons |=
 				FAILURE_REASONS_INCOMPATIBLE_TRANSFORM;
-			return NULL;
-		}
 
 		fb = drm_fb_get_from_paint_node(state, pnode);
-		if (!fb) {
+		if (fb)
+			possible_plane_mask &= fb->plane_mask;
+		else
 			drm_debug(b, "\t\t\t[view] couldn't get FB for view: 0x%lx\n",
 				     (unsigned long) pnode->try_view_on_plane_failure_reasons);
-			return NULL;
-		}
-
-		possible_plane_mask &= fb->plane_mask;
 	}
 
 	view_matches_entire_output =
@@ -494,7 +487,6 @@ drm_output_find_plane_for_view(struct drm_output_state *state,
 			assert(plane == output->cursor_plane);
 			break;
 		case WDRM_PLANE_TYPE_PRIMARY:
-			assert(fb);
 			if (plane != output->scanout_plane)
 				continue;
 			if (mode != DRM_OUTPUT_PROPOSE_STATE_PLANES_ONLY)
@@ -503,7 +495,6 @@ drm_output_find_plane_for_view(struct drm_output_state *state,
 				continue;
 			break;
 		case WDRM_PLANE_TYPE_OVERLAY:
-			assert(fb);
 			assert(mode != DRM_OUTPUT_PROPOSE_STATE_RENDERER_ONLY);
 			/* if the view covers the whole output, put it in the
 			 * scanout plane, not overlay */
@@ -570,6 +561,7 @@ drm_output_find_plane_for_view(struct drm_output_state *state,
 		else
 			zpos = MIN(current_lowest_zpos - 1, plane->zpos_max);
 
+		any_candidate_picked = true;
 		drm_debug(b, "\t\t\t\t[plane] plane %d picked "
 			     "from candidate list, type: %s\n",
 			     plane->plane_id, p_name);
@@ -577,6 +569,7 @@ drm_output_find_plane_for_view(struct drm_output_state *state,
 		if (plane->type == WDRM_PLANE_TYPE_CURSOR) {
 			ps = drm_output_prepare_cursor_paint_node(state, pnode, zpos);
 		} else {
+			if (fb)
 				ps = drm_output_try_paint_node_on_plane(plane, state,
 									pnode, mode,
 									fb, zpos);
@@ -593,11 +586,9 @@ drm_output_find_plane_for_view(struct drm_output_state *state,
 			FAILURE_REASONS_PLANES_REJECTED;
 	}
 
-	if (!ps &&
+	if (!any_candidate_picked)
-	    pnode->try_view_on_plane_failure_reasons == FAILURE_REASONS_NONE) {
 		pnode->try_view_on_plane_failure_reasons |=
 			FAILURE_REASONS_NO_PLANES_AVAILABLE;
-	}
 
 	/* if we have a plane state, it has its own ref to the fb; if not then
 	 * we drop ours here */