frontends/va: improve enc quality interface change

[why]
to enhance va video encoding quality

[how]
use va encoding quality_level interface, and provide
default value and encoding quality adjustment options,
so that users can finetune encoding quality and performance
from va quality interface. (limited to VCNs)

There are 3 settings added:
- preset modes: speed, balance, quality
  they are using different encoding strategies
- vbaq modes:
  vbaq mode is using variance based strategy
  to improve the subjective image quality
- pre-encoding modes:
  Using scaled down input image for pre-encoding to have
  better rate-control reaction and consume more memory
  in the same time. Only preencoding-4x mode is enabled.

Signed-off-by: Ruijing Dong <ruijing.dong@amd.com>
Reviewed-by: Boyuan Zhang <boyuan.zhang@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17994>

src/gallium/drivers/radeonsi/si_get.c

@@ -618,6 +618,11 @@ static int si_get_video_param(struct pipe_screen *screen, enum pipe_video_profil
             return 4;
          else
             return 0;
+      case PIPE_VIDEO_CAP_ENC_QUALITY_LEVEL:
+         if (sscreen->info.family >= CHIP_RAVEN)
+            return 32;
+         else
+            return 0;
       default:
          return 0;
       }

src/gallium/frontends/va/config.c

@@ -207,6 +207,18 @@ vlVaGetConfigAttributes(VADriverContextP ctx, VAProfile profile, VAEntrypoint en
             else
                value = supportedSliceStructuresFlagSet;
          } break;
+         case VAConfigAttribEncQualityRange:
+         {
+            /*
+             * this quality range provides different options within the range; and it isn't strictly
+             * faster when higher value used.
+             * 0, not used; 1, default value; others are using vlVaQualityBits for different modes.
+             */
+            int quality_range = pscreen->get_video_param(pscreen, ProfileToPipe(profile),
+                                 PIPE_VIDEO_ENTRYPOINT_ENCODE,
+                                 PIPE_VIDEO_CAP_ENC_QUALITY_LEVEL);
+            value = quality_range ? quality_range : VA_ATTRIB_NOT_SUPPORTED;
+         } break;
          default:
             value = VA_ATTRIB_NOT_SUPPORTED;
             break;

src/gallium/frontends/va/picture.c

@@ -113,6 +113,52 @@ vlVaGetReferenceFrame(vlVaDriver *drv, VASurfaceID surface_id,
    else
       *ref_frame = NULL;
 }
+/*
+ * in->quality = 0; without any settings, it is using speed preset
+ *                  and no preencode and no vbaq. It is the fastest setting.
+ * in->quality = 1; suggested setting, with balanced preset, and
+ *                  preencode and vbaq
+ * in->quality = others; it is the customized setting
+ *                  with valid bit (bit #0) set to "1"
+ *                  for example:
+ *
+ *                  0x3  (balance preset, no pre-encoding, no vbaq)
+ *                  0x13 (balanced preset, no pre-encoding, vbaq)
+ *                  0x13 (balanced preset, no pre-encoding, vbaq)
+ *                  0x9  (speed preset, pre-encoding, no vbaq)
+ *                  0x19 (speed preset, pre-encoding, vbaq)
+ *
+ *                  The quality value has to be treated as a combination
+ *                  of preset mode, pre-encoding and vbaq settings.
+ *                  The quality and speed could be vary according to
+ *                  different settings,
+ */
+void
+vlVaHandleVAEncMiscParameterTypeQualityLevel(struct pipe_enc_quality_modes *p, vlVaQualityBits *in)
+{
+   if (!in->quality) {
+      p->level = 0;
+      p->preset_mode = PRESET_MODE_SPEED;
+      p->pre_encode_mode = PREENCODING_MODE_DISABLE;
+      p->vbaq_mode = VBAQ_DISABLE;
+
+      return;
+   }
+
+   if (p->level != in->quality) {
+      if (in->quality == 1) {
+         p->preset_mode = PRESET_MODE_BALANCE;
+         p->pre_encode_mode = PREENCODING_MODE_DEFAULT;
+         p->vbaq_mode = VBAQ_AUTO;
+      } else {
+         p->preset_mode = in->preset_mode > PRESET_MODE_QUALITY
+            ? PRESET_MODE_QUALITY : in->preset_mode;
+         p->pre_encode_mode = in->pre_encode_mode;
+         p->vbaq_mode = in->vbaq_mode;
+      }
+   }
+   p->level = in->quality;
+}
 
 static VAStatus
 handlePictureParameterBuffer(vlVaDriver *drv, vlVaContext *context, vlVaBuffer *buf)
@@ -456,6 +502,27 @@ handleVAEncSequenceParameterBufferType(vlVaDriver *drv, vlVaContext *context, vl
    return status;
 }
 
+static VAStatus
+handleVAEncMiscParameterTypeQualityLevel(vlVaContext *context, VAEncMiscParameterBuffer *misc)
+{
+   VAStatus status = VA_STATUS_SUCCESS;
+
+   switch (u_reduce_video_profile(context->templat.profile)) {
+   case PIPE_VIDEO_FORMAT_MPEG4_AVC:
+      status = vlVaHandleVAEncMiscParameterTypeQualityLevelH264(context, misc);
+      break;
+
+   case PIPE_VIDEO_FORMAT_HEVC:
+      status = vlVaHandleVAEncMiscParameterTypeQualityLevelHEVC(context, misc);
+      break;
+
+   default:
+      break;
+   }
+
+   return status;
+}
+
 static VAStatus
 handleVAEncMiscParameterBufferType(vlVaContext *context, vlVaBuffer *buf)
 {
@@ -476,6 +543,10 @@ handleVAEncMiscParameterBufferType(vlVaContext *context, vlVaBuffer *buf)
       vaStatus = handleVAEncMiscParameterTypeTemporalLayer(context, misc);
       break;
 
+   case VAEncMiscParameterTypeQualityLevel:
+      vaStatus = handleVAEncMiscParameterTypeQualityLevel(context, misc);
+      break;
+
    default:
       break;
    }

src/gallium/frontends/va/picture_h264_enc.c

@@ -247,6 +247,16 @@ vlVaHandleVAEncMiscParameterTypeTemporalLayerH264(vlVaContext *context, VAEncMis
    return VA_STATUS_SUCCESS;
 }
 
+VAStatus
+vlVaHandleVAEncMiscParameterTypeQualityLevelH264(vlVaContext *context, VAEncMiscParameterBuffer *misc)
+{
+   VAEncMiscParameterBufferQualityLevel *ql = (VAEncMiscParameterBufferQualityLevel *)misc->data;
+   vlVaHandleVAEncMiscParameterTypeQualityLevel(&context->desc.h264enc.quality_modes,
+                               (vlVaQualityBits *)&ql->quality_level);
+
+   return VA_STATUS_SUCCESS;
+}
+
 void getEncParamPresetH264(vlVaContext *context)
 {
    //motion estimation preset

src/gallium/frontends/va/picture_hevc_enc.c

@@ -197,6 +197,16 @@ vlVaHandleVAEncMiscParameterTypeFrameRateHEVC(vlVaContext *context, VAEncMiscPar
    return VA_STATUS_SUCCESS;
 }
 
+VAStatus
+vlVaHandleVAEncMiscParameterTypeQualityLevelHEVC(vlVaContext *context, VAEncMiscParameterBuffer *misc)
+{
+   VAEncMiscParameterBufferQualityLevel *ql = (VAEncMiscParameterBufferQualityLevel *)misc->data;
+   vlVaHandleVAEncMiscParameterTypeQualityLevel(&context->desc.h265enc.quality_modes,
+                               (vlVaQualityBits *)&ql->quality_level);
+
+   return VA_STATUS_SUCCESS;
+}
+
 static void profile_tier(struct vl_rbsp *rbsp)
 {
    vl_rbsp_u(rbsp, 2); /* general_profile_space */

src/gallium/frontends/va/va_private.h

@@ -67,6 +67,16 @@
 #define SOS (8 + 4 * 2)
 #define MAX_MJPEG_SLICE_HEADER_SIZE (SOI + DQT + DHT + DRI + SOF + SOS)
 
+#define PRESET_MODE_SPEED   (0)
+#define PRESET_MODE_BALANCE (1)
+#define PRESET_MODE_QUALITY (2)
+
+#define PREENCODING_MODE_DISABLE (0)
+#define PREENCODING_MODE_DEFAULT (1)
+
+#define VBAQ_DISABLE (0)
+#define VBAQ_AUTO    (1)
+
 static inline enum pipe_video_chroma_format
 ChromaToPipe(int format)
 {
@@ -342,6 +352,19 @@ typedef struct {
    enum pipe_format encoder_format;
 } vlVaSurface;
 
+typedef struct {
+   union {
+      unsigned int quality;
+      struct {
+         unsigned int valid_setting: 1;
+         unsigned int preset_mode: 2;
+         unsigned int pre_encode_mode: 1;
+         unsigned int vbaq_mode: 1;
+         unsigned int reservered: 27;
+      };
+   };
+} vlVaQualityBits;
+
 // Public functions:
 VAStatus VA_DRIVER_INIT_FUNC(VADriverContextP ctx);
 
@@ -467,16 +490,19 @@ void vlVaHandlePictureParameterBufferAV1(vlVaDriver *drv, vlVaContext *context,
 void vlVaHandleSliceParameterBufferAV1(vlVaContext *context, vlVaBuffer *buf, unsigned int num);
 void getEncParamPresetH264(vlVaContext *context);
 void getEncParamPresetH265(vlVaContext *context);
+void vlVaHandleVAEncMiscParameterTypeQualityLevel(struct pipe_enc_quality_modes *p, vlVaQualityBits *in);
 VAStatus vlVaHandleVAEncPictureParameterBufferTypeH264(vlVaDriver *drv, vlVaContext *context, vlVaBuffer *buf);
 VAStatus vlVaHandleVAEncSliceParameterBufferTypeH264(vlVaDriver *drv, vlVaContext *context, vlVaBuffer *buf);
 VAStatus vlVaHandleVAEncSequenceParameterBufferTypeH264(vlVaDriver *drv, vlVaContext *context, vlVaBuffer *buf);
 VAStatus vlVaHandleVAEncMiscParameterTypeRateControlH264(vlVaContext *context, VAEncMiscParameterBuffer *buf);
 VAStatus vlVaHandleVAEncMiscParameterTypeFrameRateH264(vlVaContext *context, VAEncMiscParameterBuffer *buf);
 VAStatus vlVaHandleVAEncMiscParameterTypeTemporalLayerH264(vlVaContext *context, VAEncMiscParameterBuffer *buf);
+VAStatus vlVaHandleVAEncMiscParameterTypeQualityLevelH264(vlVaContext *context, VAEncMiscParameterBuffer *buf);
 VAStatus vlVaHandleVAEncPictureParameterBufferTypeHEVC(vlVaDriver *drv, vlVaContext *context, vlVaBuffer *buf);
 VAStatus vlVaHandleVAEncSliceParameterBufferTypeHEVC(vlVaDriver *drv, vlVaContext *context, vlVaBuffer *buf);
 VAStatus vlVaHandleVAEncSequenceParameterBufferTypeHEVC(vlVaDriver *drv, vlVaContext *context, vlVaBuffer *buf);
 VAStatus vlVaHandleVAEncMiscParameterTypeRateControlHEVC(vlVaContext *context, VAEncMiscParameterBuffer *buf);
 VAStatus vlVaHandleVAEncMiscParameterTypeFrameRateHEVC(vlVaContext *context, VAEncMiscParameterBuffer *buf);
 VAStatus vlVaHandleVAEncPackedHeaderDataBufferTypeHEVC(vlVaContext *context, vlVaBuffer *buf);
+VAStatus vlVaHandleVAEncMiscParameterTypeQualityLevelHEVC(vlVaContext *context, VAEncMiscParameterBuffer *buf);
 #endif //VA_PRIVATE_H

src/gallium/include/pipe/p_video_enums.h

@@ -105,6 +105,7 @@ enum pipe_video_cap
    PIPE_VIDEO_CAP_VPP_MAX_OUTPUT_HEIGHT = 23,
    PIPE_VIDEO_CAP_VPP_MIN_OUTPUT_WIDTH = 24,
    PIPE_VIDEO_CAP_VPP_MIN_OUTPUT_HEIGHT = 25,
+   PIPE_VIDEO_CAP_ENC_QUALITY_LEVEL = 26,
 };
 
 /* To be used with PIPE_VIDEO_CAP_VPP_ORIENTATION_MODES and for VPP state*/

src/gallium/include/pipe/p_video_state.h

@@ -362,6 +362,14 @@ struct pipe_h264_picture_desc
    void    *priv;
 };
 
+struct pipe_enc_quality_modes
+{
+   unsigned int level;
+   unsigned int preset_mode;
+   unsigned int pre_encode_mode;
+   unsigned int vbaq_mode;
+};
+
 struct pipe_h264_enc_rate_control
 {
    enum pipe_h2645_enc_rate_control_method rate_ctrl_method;
@@ -440,6 +448,7 @@ struct pipe_h264_enc_picture_desc
    unsigned gop_size;
    unsigned ref_pic_mode;
    unsigned num_temporal_layers;
+   struct pipe_enc_quality_modes quality_modes;
 
    bool not_referenced;
    bool enable_vui;
@@ -531,6 +540,7 @@ struct pipe_h265_enc_picture_desc
    unsigned pic_order_cnt_type;
    unsigned ref_idx_l0;
    unsigned ref_idx_l1;
+   struct pipe_enc_quality_modes quality_modes;
    bool not_referenced;
    struct hash_table *frame_idx;
 };