frontends/va: Add postproc support for converting to full range

Acked-by: Ruijing Dong <ruijing.dong@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24174>

src/gallium/auxiliary/vl/vl_compositor.c

@@ -382,7 +382,8 @@ set_yuv_layer(struct vl_compositor_state *s, struct vl_compositor *c,
 static void
 set_rgb_to_yuv_layer(struct vl_compositor_state *s, struct vl_compositor *c,
                      unsigned layer, struct pipe_sampler_view *v,
-                     struct u_rect *src_rect, struct u_rect *dst_rect, bool y)
+                     struct u_rect *src_rect, struct u_rect *dst_rect, bool y,
+                     bool full_range)
 {
    vl_csc_matrix csc_matrix;
 
@@ -394,7 +395,7 @@ set_rgb_to_yuv_layer(struct vl_compositor_state *s, struct vl_compositor *c,
 
    s->layers[layer].fs = y? c->fs_rgb_yuv.y : c->fs_rgb_yuv.uv;
 
-   vl_csc_get_matrix(VL_CSC_COLOR_STANDARD_BT_709_REV, NULL, false, &csc_matrix);
+   vl_csc_get_matrix(VL_CSC_COLOR_STANDARD_BT_709_REV, NULL, full_range, &csc_matrix);
    vl_compositor_set_csc_matrix(s, (const vl_csc_matrix *)&csc_matrix, 1.0f, 0.0f);
 
    s->layers[layer].samplers[0] = c->sampler_linear;
@@ -725,7 +726,8 @@ vl_compositor_convert_rgb_to_yuv(struct vl_compositor_state *s,
                                  struct pipe_resource *src_res,
                                  struct pipe_video_buffer *dst,
                                  struct u_rect *src_rect,
-                                 struct u_rect *dst_rect)
+                                 struct u_rect *dst_rect,
+                                 bool full_range)
 {
    struct pipe_sampler_view *sv, sv_templ;
    struct pipe_surface **dst_surfaces;
@@ -738,7 +740,7 @@ vl_compositor_convert_rgb_to_yuv(struct vl_compositor_state *s,
 
    vl_compositor_clear_layers(s);
 
-   set_rgb_to_yuv_layer(s, c, 0, sv, src_rect, NULL, true);
+   set_rgb_to_yuv_layer(s, c, 0, sv, src_rect, NULL, true, full_range);
    vl_compositor_set_layer_dst_area(s, 0, dst_rect);
    vl_compositor_render(s, c, dst_surfaces[0], NULL, false);
 
@@ -747,7 +749,7 @@ vl_compositor_convert_rgb_to_yuv(struct vl_compositor_state *s,
       dst_rect->y1 /= 2;
    }
 
-   set_rgb_to_yuv_layer(s, c, 0, sv, src_rect, NULL, false);
+   set_rgb_to_yuv_layer(s, c, 0, sv, src_rect, NULL, false, full_range);
    vl_compositor_set_layer_dst_area(s, 0, dst_rect);
    vl_compositor_render(s, c, dst_surfaces[1], NULL, false);
    pipe_sampler_view_reference(&sv, NULL);

src/gallium/auxiliary/vl/vl_compositor.h

@@ -302,7 +302,8 @@ vl_compositor_convert_rgb_to_yuv(struct vl_compositor_state *state,
                                  struct pipe_resource *src_res,
                                  struct pipe_video_buffer *dst,
                                  struct u_rect *src_rect,
-                                 struct u_rect *dst_rect);
+                                 struct u_rect *dst_rect,
+                                 bool full_range);
 
 /*@}*/
 

src/gallium/auxiliary/vl/vl_csc.c

@@ -137,6 +137,12 @@ static const vl_csc_matrix bt_709_rev  = {
    { 0.439f, -0.399f, -0.040f, 0.5f   }
 };
 
+static const vl_csc_matrix bt_709_full  = {
+   { 0.0f,  1.0f, 0.0f, 0.0f },
+   { 0.0f, -0.5f, 0.5f, 0.5f },
+   { 0.5f, -0.5f, 0.0f, 0.5f }
+};
+
 static const vl_csc_matrix identity =
 {
    { 1.0f, 0.0f, 0.0f, 0.0f, },
@@ -191,7 +197,7 @@ void vl_csc_get_matrix(enum VL_CSC_COLOR_STANDARD cs,
          cstd = &smpte240m;
          break;
       case VL_CSC_COLOR_STANDARD_BT_709_REV:
-         memcpy(matrix, bt_709_rev, sizeof(vl_csc_matrix));
+         memcpy(matrix, full_range ? bt_709_full : bt_709_rev, sizeof(vl_csc_matrix));
          return;
       case VL_CSC_COLOR_STANDARD_IDENTITY:
       default:

src/gallium/frontends/va/picture.c

@@ -1089,6 +1089,7 @@ vlVaEndPicture(VADriverContextP ctx, VAContextID context_id)
       }
 
       context->desc.base.input_format = surf->buffer->buffer_format;
+      context->desc.base.input_full_range = surf->full_range;
       context->desc.base.output_format = surf->encoder_format;
 
       context->decoder->begin_frame(context->decoder, context->target, &context->desc.base);

src/gallium/frontends/va/postproc.c

@@ -191,7 +191,8 @@ static VAStatus vlVaPostProcBlit(vlVaDriver *drv, vlVaContext *context,
                                  const VARectangle *dst_region,
                                  struct pipe_video_buffer *src,
                                  struct pipe_video_buffer *dst,
-                                 enum vl_compositor_deinterlace deinterlace)
+                                 enum vl_compositor_deinterlace deinterlace,
+                                 bool full_range)
 {
    struct pipe_surface **src_surfaces;
    struct pipe_surface **dst_surfaces;
@@ -248,7 +249,7 @@ static VAStatus vlVaPostProcBlit(vlVaDriver *drv, vlVaContext *context,
    if (grab) {
       vl_compositor_convert_rgb_to_yuv(&drv->cstate, &drv->compositor, 0,
                                        ((struct vl_video_buffer *)src)->resources[0],
-                                       dst, &src_rect, &dst_rect);
+                                       dst, &src_rect, &dst_rect, full_range);
 
       return VA_STATUS_SUCCESS;
    }
@@ -419,6 +420,13 @@ vlVaHandleVAProcPipelineParameterBufferType(vlVaDriver *drv, vlVaContext *contex
    if (!src_surface->buffer || !dst_surface->buffer)
       return VA_STATUS_ERROR_INVALID_SURFACE;
 
+   /* Assume full range input when not set */
+   src_surface->full_range =
+      param->input_color_properties.color_range != VA_SOURCE_RANGE_REDUCED;
+   /* Assume limited range output when not set */
+   dst_surface->full_range =
+      param->output_color_properties.color_range == VA_SOURCE_RANGE_FULL;
+
    pscreen = drv->vscreen->pscreen;
 
    src_region = vlVaRegionDefault(param->surface_region, src_surface, &def_src_region);
@@ -539,5 +547,6 @@ vlVaHandleVAProcPipelineParameterBufferType(vlVaDriver *drv, vlVaContext *contex
                                     src, context->target, deinterlace);
    else
       return vlVaPostProcBlit(drv, context, src_region, dst_region,
-                              src, context->target, deinterlace);
+                              src, context->target, deinterlace,
+                              dst_surface->full_range);
 }

src/gallium/frontends/va/va_private.h

@@ -391,6 +391,7 @@ typedef struct {
    bool force_flushed;
    struct pipe_video_buffer *obsolete_buf;
    enum pipe_format encoder_format;
+   bool full_range;
    struct pipe_fence_handle *fence;
 } vlVaSurface;
 

src/gallium/include/pipe/p_video_state.h

@@ -186,6 +186,7 @@ struct pipe_picture_desc
    uint8_t *decrypt_key;
    uint32_t key_size;
    enum pipe_format input_format;
+   bool input_full_range;
    enum pipe_format output_format;
    /* A fence used on PIPE_VIDEO_ENTRYPOINT_DECODE/PROCESSING to signal job completion */
    struct pipe_fence_handle **fence;