st/va: add encode entrypoint v2

VAAPI passes PIPE_VIDEO_ENTRYPOINT_ENCODE as entry point for encoding case. We
will save this encode entry point in config. config_id was used as profile
previously. Now, config has both profile and entrypoint field, and config_id is
used to get the config object. Later on, we pass this entrypoint to
context->templat.entrypoint instead of always hardcoded to
PIPE_VIDEO_ENTRYPOINT_BITSTREAM for decoding case previously. Encode entrypoint
is not accepted by driver until we enable Vaapi encode in later patch.

v2 (chk): fix commit message to match 80 chars, use switch instead of ifs,
	  fix memory leaks in the error path, implement vlVaQueryConfigEntrypoints
	  as well, drop VAEntrypointEncPicture (only used for JPEG).

Signed-off-by: Boyuan Zhang <boyuan.zhang@amd.com>
Signed-off-by: Christian König <christian.koenig@amd.com>

src/gallium/state_trackers/va/config.c

@@ -34,6 +34,8 @@
 
 #include "va_private.h"
 
+#include "util/u_handle_table.h"
+
 DEBUG_GET_ONCE_BOOL_OPTION(mpeg4, "VAAPI_MPEG4_ENABLED", false)
 
 VAStatus
@@ -88,11 +90,19 @@ vlVaQueryConfigEntrypoints(VADriverContextP ctx, VAProfile profile,
       return VA_STATUS_ERROR_UNSUPPORTED_PROFILE;
 
    pscreen = VL_VA_PSCREEN(ctx);
-   if (!pscreen->get_video_param(pscreen, p, PIPE_VIDEO_ENTRYPOINT_BITSTREAM, PIPE_VIDEO_CAP_SUPPORTED))
+   if (pscreen->get_video_param(pscreen, p, PIPE_VIDEO_ENTRYPOINT_BITSTREAM,
-      return VA_STATUS_ERROR_UNSUPPORTED_PROFILE;
+				PIPE_VIDEO_CAP_SUPPORTED))
-
       entrypoint_list[(*num_entrypoints)++] = VAEntrypointVLD;
 
+#if 0
+   if (pscreen->get_video_param(pscreen, p, PIPE_VIDEO_ENTRYPOINT_ENCODE,
+				PIPE_VIDEO_CAP_SUPPORTED))
+      entrypoint_list[(*num_entrypoints)++] = VAEntrypointEncSlice;
+#endif
+
+   if (num_entrypoints == 0)
+      return VA_STATUS_ERROR_UNSUPPORTED_PROFILE;
+
    return VA_STATUS_SUCCESS;
 }
 
@@ -128,29 +138,73 @@ VAStatus
 vlVaCreateConfig(VADriverContextP ctx, VAProfile profile, VAEntrypoint entrypoint,
                  VAConfigAttrib *attrib_list, int num_attribs, VAConfigID *config_id)
 {
+   vlVaDriver *drv;
+   vlVaConfig *config;
    struct pipe_screen *pscreen;
    enum pipe_video_profile p;
 
    if (!ctx)
       return VA_STATUS_ERROR_INVALID_CONTEXT;
 
+   drv = VL_VA_DRIVER(ctx);
+
+   if (!drv)
+      return VA_STATUS_ERROR_INVALID_CONTEXT;
+
+   config = CALLOC(1, sizeof(vlVaConfig));
+   if (!config)
+      return VA_STATUS_ERROR_ALLOCATION_FAILED;
+
    if (profile == VAProfileNone && entrypoint == VAEntrypointVideoProc) {
-      *config_id = PIPE_VIDEO_PROFILE_UNKNOWN;
+      config->entrypoint = VAEntrypointVideoProc;
+      config->profile = PIPE_VIDEO_PROFILE_UNKNOWN;
+      pipe_mutex_lock(drv->mutex);
+      *config_id = handle_table_add(drv->htab, config);
+      pipe_mutex_unlock(drv->mutex);
       return VA_STATUS_SUCCESS;
    }
 
    p = ProfileToPipe(profile);
-   if (p == PIPE_VIDEO_PROFILE_UNKNOWN)
+   if (p == PIPE_VIDEO_PROFILE_UNKNOWN) {
+      FREE(config);
       return VA_STATUS_ERROR_UNSUPPORTED_PROFILE;
+   }
 
    pscreen = VL_VA_PSCREEN(ctx);
-   if (!pscreen->get_video_param(pscreen, p, PIPE_VIDEO_ENTRYPOINT_BITSTREAM, PIPE_VIDEO_CAP_SUPPORTED))
+
+   switch (entrypoint) {
+   case VAEntrypointVLD:
+      if (!pscreen->get_video_param(pscreen, p, PIPE_VIDEO_ENTRYPOINT_BITSTREAM,
+				    PIPE_VIDEO_CAP_SUPPORTED)) {
+         FREE(config);
          return VA_STATUS_ERROR_UNSUPPORTED_PROFILE;
+      }
 
-   if (entrypoint != VAEntrypointVLD)
+      config->entrypoint = PIPE_VIDEO_ENTRYPOINT_BITSTREAM;
+      break;
+
+#if 0
+   case VAEntrypointEncSlice:
+      if (!pscreen->get_video_param(pscreen, p, PIPE_VIDEO_ENTRYPOINT_ENCODE,
+				    PIPE_VIDEO_CAP_SUPPORTED)) {
+         FREE(config);
+         return VA_STATUS_ERROR_UNSUPPORTED_PROFILE;
+      }
+
+      config->entrypoint = PIPE_VIDEO_ENTRYPOINT_ENCODE;
+      break;
+#endif
+
+   default:
+      FREE(config);
       return VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT;
+   }
 
-   *config_id = p;
+   config->profile = p;
+
+   pipe_mutex_lock(drv->mutex);
+   *config_id = handle_table_add(drv->htab, config);
+   pipe_mutex_unlock(drv->mutex);
 
    return VA_STATUS_SUCCESS;
 }
@@ -158,9 +212,27 @@ vlVaCreateConfig(VADriverContextP ctx, VAProfile profile, VAEntrypoint entrypoin
 VAStatus
 vlVaDestroyConfig(VADriverContextP ctx, VAConfigID config_id)
 {
+   vlVaDriver *drv;
+   vlVaConfig *config;
+
    if (!ctx)
       return VA_STATUS_ERROR_INVALID_CONTEXT;
 
+   drv = VL_VA_DRIVER(ctx);
+
+   if (!drv)
+      return VA_STATUS_ERROR_INVALID_CONTEXT;
+
+   pipe_mutex_lock(drv->mutex);
+   config = handle_table_get(drv->htab, config_id);
+
+   if (!config)
+      return VA_STATUS_ERROR_INVALID_CONFIG;
+
+   FREE(config);
+   handle_table_remove(drv->htab, config_id);
+   pipe_mutex_unlock(drv->mutex);
+
    return VA_STATUS_SUCCESS;
 }
 
@@ -168,18 +240,33 @@ VAStatus
 vlVaQueryConfigAttributes(VADriverContextP ctx, VAConfigID config_id, VAProfile *profile,
                           VAEntrypoint *entrypoint, VAConfigAttrib *attrib_list, int *num_attribs)
 {
+   vlVaDriver *drv;
+   vlVaConfig *config;
+
    if (!ctx)
       return VA_STATUS_ERROR_INVALID_CONTEXT;
 
-   *profile = PipeToProfile(config_id);
+   drv = VL_VA_DRIVER(ctx);
 
-   if (config_id == PIPE_VIDEO_PROFILE_UNKNOWN) {
+   if (!drv)
+      return VA_STATUS_ERROR_INVALID_CONTEXT;
+
+   pipe_mutex_lock(drv->mutex);
+   config = handle_table_get(drv->htab, config_id);
+   pipe_mutex_unlock(drv->mutex);
+
+   if (!config)
+      return VA_STATUS_ERROR_INVALID_CONFIG;
+
+   *profile = PipeToProfile(config->profile);
+
+   if (config->profile == PIPE_VIDEO_PROFILE_UNKNOWN) {
       *entrypoint = VAEntrypointVideoProc;
       *num_attribs = 0;
       return VA_STATUS_SUCCESS;
    }
 
-   *entrypoint = VAEntrypointVLD;
+   *entrypoint = config->entrypoint;
 
    *num_attribs = 1;
    attrib_list[0].type = VAConfigAttribRTFormat;

src/gallium/state_trackers/va/context.c

@@ -195,18 +195,23 @@ vlVaCreateContext(VADriverContextP ctx, VAConfigID config_id, int picture_width,
 {
    vlVaDriver *drv;
    vlVaContext *context;
+   vlVaConfig *config;
    int is_vpp;
 
    if (!ctx)
       return VA_STATUS_ERROR_INVALID_CONTEXT;
 
-   is_vpp = config_id == PIPE_VIDEO_PROFILE_UNKNOWN && !picture_width &&
+   drv = VL_VA_DRIVER(ctx);
+   pipe_mutex_lock(drv->mutex);
+   config = handle_table_get(drv->htab, config_id);
+   pipe_mutex_unlock(drv->mutex);
+
+   is_vpp = config->profile == PIPE_VIDEO_PROFILE_UNKNOWN && !picture_width &&
             !picture_height && !flag && !render_targets && !num_render_targets;
 
    if (!(picture_width && picture_height) && !is_vpp)
       return VA_STATUS_ERROR_INVALID_IMAGE_FORMAT;
 
-   drv = VL_VA_DRIVER(ctx);
    context = CALLOC(1, sizeof(vlVaContext));
    if (!context)
       return VA_STATUS_ERROR_ALLOCATION_FAILED;
@@ -218,8 +223,8 @@ vlVaCreateContext(VADriverContextP ctx, VAConfigID config_id, int picture_width,
          return VA_STATUS_ERROR_INVALID_CONTEXT;
       }
    } else {
-      context->templat.profile = config_id;
+      context->templat.profile = config->profile;
-      context->templat.entrypoint = PIPE_VIDEO_ENTRYPOINT_BITSTREAM;
+      context->templat.entrypoint = config->entrypoint;
       context->templat.chroma_format = PIPE_VIDEO_CHROMA_FORMAT_420;
       context->templat.width = picture_width;
       context->templat.height = picture_height;
@@ -234,6 +239,7 @@ vlVaCreateContext(VADriverContextP ctx, VAConfigID config_id, int picture_width,
 
       case PIPE_VIDEO_FORMAT_MPEG4_AVC:
          context->templat.max_references = 0;
+         if (config->entrypoint != PIPE_VIDEO_ENTRYPOINT_ENCODE) {
             context->desc.h264.pps = CALLOC_STRUCT(pipe_h264_pps);
             if (!context->desc.h264.pps) {
                FREE(context);
@@ -245,6 +251,7 @@ vlVaCreateContext(VADriverContextP ctx, VAConfigID config_id, int picture_width,
                FREE(context);
                return VA_STATUS_ERROR_ALLOCATION_FAILED;
             }
+         }
          break;
 
      case PIPE_VIDEO_FORMAT_HEVC:
@@ -267,7 +274,9 @@ vlVaCreateContext(VADriverContextP ctx, VAConfigID config_id, int picture_width,
       }
    }
 
-   context->desc.base.profile = config_id;
+   context->desc.base.profile = config->profile;
+   context->desc.base.entry_point = config->entrypoint;
+
    pipe_mutex_lock(drv->mutex);
    *context_id = handle_table_add(drv->htab, context);
    pipe_mutex_unlock(drv->mutex);
@@ -293,6 +302,7 @@ vlVaDestroyContext(VADriverContextP ctx, VAContextID context_id)
    }
 
    if (context->decoder) {
+      if (context->desc.base.entry_point != PIPE_VIDEO_ENTRYPOINT_ENCODE) {
          if (u_reduce_video_profile(context->decoder->profile) ==
                PIPE_VIDEO_FORMAT_MPEG4_AVC) {
             FREE(context->desc.h264.pps->sps);
@@ -303,6 +313,7 @@ vlVaDestroyContext(VADriverContextP ctx, VAContextID context_id)
             FREE(context->desc.h265.pps->sps);
             FREE(context->desc.h265.pps);
          }
+      }
       context->decoder->destroy(context->decoder);
    }
    if (context->deint) {

src/gallium/state_trackers/va/surface.c

@@ -319,17 +319,18 @@ vlVaUnlockSurface(VADriverContextP ctx, VASurfaceID surface)
 }
 
 VAStatus
-vlVaQuerySurfaceAttributes(VADriverContextP ctx, VAConfigID config,
+vlVaQuerySurfaceAttributes(VADriverContextP ctx, VAConfigID config_id,
                            VASurfaceAttrib *attrib_list, unsigned int *num_attribs)
 {
    vlVaDriver *drv;
+   vlVaConfig *config;
    VASurfaceAttrib *attribs;
    struct pipe_screen *pscreen;
    int i, j;
 
    STATIC_ASSERT(ARRAY_SIZE(vpp_surface_formats) <= VL_VA_MAX_IMAGE_FORMATS);
 
-   if (config == VA_INVALID_ID)
+   if (config_id == VA_INVALID_ID)
       return VA_STATUS_ERROR_INVALID_CONFIG;
 
    if (!attrib_list && !num_attribs)
@@ -348,6 +349,13 @@ vlVaQuerySurfaceAttributes(VADriverContextP ctx, VAConfigID config,
    if (!drv)
       return VA_STATUS_ERROR_INVALID_CONTEXT;
 
+   pipe_mutex_lock(drv->mutex);
+   config = handle_table_get(drv->htab, config_id);
+   pipe_mutex_unlock(drv->mutex);
+
+   if (!config)
+      return VA_STATUS_ERROR_INVALID_CONFIG;
+
    pscreen = VL_VA_PSCREEN(ctx);
 
    if (!pscreen)
@@ -363,7 +371,7 @@ vlVaQuerySurfaceAttributes(VADriverContextP ctx, VAConfigID config,
 
    /* vlVaCreateConfig returns PIPE_VIDEO_PROFILE_UNKNOWN
     * only for VAEntrypointVideoProc. */
-   if (config == PIPE_VIDEO_PROFILE_UNKNOWN) {
+   if (config->profile == PIPE_VIDEO_PROFILE_UNKNOWN) {
       for (j = 0; j < ARRAY_SIZE(vpp_surface_formats); ++j) {
          attribs[i].type = VASurfaceAttribPixelFormat;
          attribs[i].value.type = VAGenericValueTypeInteger;

src/gallium/state_trackers/va/va_private.h

@@ -243,6 +243,11 @@ typedef struct {
    struct vl_deint_filter *deint;
 } vlVaContext;
 
+typedef struct {
+   enum pipe_video_profile profile;
+   enum pipe_video_entrypoint entrypoint;
+} vlVaConfig;
+
 typedef struct {
    VABufferType type;
    unsigned int size;