diff --git a/src/gallium/frontends/va/picture.c b/src/gallium/frontends/va/picture.c
index e7d5cbb86df..ded801fd881 100644
--- a/src/gallium/frontends/va/picture.c
+++ b/src/gallium/frontends/va/picture.c
@@ -1291,9 +1291,18 @@ vlVaEndPicture(VADriverContextP ctx, VAContextID context_id)
             getEncParamPresetH265(context);
       }
 
-      context->desc.base.input_format = surf->buffer->buffer_format;
+      if (surf->efc_surface) {
+         assert(surf == drv->last_efc_surface);
+         context->target = surf->efc_surface->buffer;
+         context->desc.base.input_format = surf->efc_surface->buffer->buffer_format;
+         context->desc.base.output_format = surf->buffer->buffer_format;
+         surf->efc_surface = NULL;
+         drv->last_efc_surface = NULL;
+      } else {
+         context->desc.base.input_format = surf->buffer->buffer_format;
+         context->desc.base.output_format = surf->buffer->buffer_format;
+      }
       context->desc.base.input_full_range = surf->full_range;
-      context->desc.base.output_format = surf->encoder_format;
 
       int driver_metadata_support = drv->pipe->screen->get_video_param(drv->pipe->screen,
                                                                        context->decoder->profile,
diff --git a/src/gallium/frontends/va/postproc.c b/src/gallium/frontends/va/postproc.c
index b3c92218c69..47efbad8e70 100644
--- a/src/gallium/frontends/va/postproc.c
+++ b/src/gallium/frontends/va/postproc.c
@@ -560,7 +560,17 @@ vlVaHandleVAProcPipelineParameterBufferType(vlVaDriver *drv, vlVaContext *contex
    src_region = vlVaRegionDefault(param->surface_region, src_surface, &def_src_region);
    dst_region = vlVaRegionDefault(param->output_region, dst_surface, &def_dst_region);
 
-   if (!param->num_filters &&
+   /* EFC can only do one conversion, and it must be the last postproc
+    * operation immediately before encoding.
+    * Disable EFC completely if this is not the case. */
+   if (drv->last_efc_surface) {
+      vlVaSurface *surf = drv->last_efc_surface;
+      surf->efc_surface = NULL;
+      drv->last_efc_surface = NULL;
+      drv->efc_count = -1;
+   }
+
+   if (drv->efc_count >= 0 && !param->num_filters &&
        src_region->width == dst_region->width &&
        src_region->height == dst_region->height &&
        src_region->x == dst_region->x &&
@@ -572,24 +582,16 @@ vlVaHandleVAProcPipelineParameterBufferType(vlVaDriver *drv, vlVaContext *contex
                                                  PIPE_VIDEO_PROFILE_UNKNOWN,
                                                  PIPE_VIDEO_ENTRYPOINT_ENCODE)) {
 
-      vlVaSurface *surf = dst_surface;
+      dst_surface->efc_surface = src_surface;
+      drv->last_efc_surface = dst_surface;
 
-      // EFC will convert the buffer to a format the encoder accepts
-      if (src_surface->buffer->buffer_format != surf->buffer->buffer_format) {
-         surf->encoder_format = surf->buffer->buffer_format;
-
-         surf->templat.interlaced = src_surface->templat.interlaced;
-         surf->templat.buffer_format = src_surface->templat.buffer_format;
-         surf->buffer->destroy(surf->buffer);
-
-         if (vlVaHandleSurfaceAllocate(drv, surf, &surf->templat, NULL, 0) != VA_STATUS_SUCCESS)
-            return VA_STATUS_ERROR_ALLOCATION_FAILED;
-      }
-
-      pipe_resource_reference(&(((struct vl_video_buffer *)(surf->buffer))->resources[0]), ((struct vl_video_buffer *)(src_surface->buffer))->resources[0]);
-      context->target = surf->buffer;
-
-      return VA_STATUS_SUCCESS;
+      /* Do the blit for first few conversions as a fallback in case EFC
+       * could not be used (see above), after that assume EFC can always
+       * be used and skip the blit. */
+      if (drv->efc_count < 16)
+         drv->efc_count++;
+      else
+         return VA_STATUS_SUCCESS;
    }
 
    src = src_surface->buffer;
diff --git a/src/gallium/frontends/va/surface.c b/src/gallium/frontends/va/surface.c
index 1e78cc874cd..a46c264db72 100644
--- a/src/gallium/frontends/va/surface.c
+++ b/src/gallium/frontends/va/surface.c
@@ -90,6 +90,8 @@ vlVaDestroySurfaces(VADriverContextP ctx, VASurfaceID *surface_list, int num_sur
          if (surf->fence && surf->ctx->decoder && surf->ctx->decoder->destroy_fence)
             surf->ctx->decoder->destroy_fence(surf->ctx->decoder, surf->fence);
       }
+      if (drv->last_efc_surface == surf)
+         drv->last_efc_surface = NULL;
       util_dynarray_fini(&surf->subpics);
       FREE(surf);
       handle_table_remove(drv->htab, surface_list[i]);
diff --git a/src/gallium/frontends/va/va_private.h b/src/gallium/frontends/va/va_private.h
index 3486fd92767..17ec33439d6 100644
--- a/src/gallium/frontends/va/va_private.h
+++ b/src/gallium/frontends/va/va_private.h
@@ -303,6 +303,9 @@ typedef struct {
    char vendor_string[256];
 
    bool has_external_handles;
+
+   int efc_count;
+   void *last_efc_surface;
 } vlVaDriver;
 
 typedef struct {
@@ -399,7 +402,7 @@ typedef struct {
    unsigned int rt_format;
 } vlVaConfig;
 
-typedef struct {
+typedef struct vlVaSurface {
    struct pipe_video_buffer templat, *buffer;
    struct util_dynarray subpics; /* vlVaSubpicture */
    vlVaContext *ctx;
@@ -408,9 +411,9 @@ typedef struct {
    unsigned int frame_num_cnt;
    bool force_flushed;
    struct pipe_video_buffer *obsolete_buf;
-   enum pipe_format encoder_format;
    bool full_range;
    struct pipe_fence_handle *fence;
+   struct vlVaSurface *efc_surface; /* input surface for EFC */
 } vlVaSurface;
 
 typedef struct {