diff --git a/src/gallium/frontends/va/context.c b/src/gallium/frontends/va/context.c
index f56ddb98a99..6e52986711b 100644
--- a/src/gallium/frontends/va/context.c
+++ b/src/gallium/frontends/va/context.c
@@ -496,12 +496,22 @@ vlVaDestroyContext(VADriverContextP ctx, VAContextID context_id)
              PIPE_VIDEO_FORMAT_MPEG4_AVC) {
             if (context->desc.h264enc.frame_idx)
                _mesa_hash_table_destroy(context->desc.h264enc.frame_idx, NULL);
+            for (uint32_t i = 0; i < ARRAY_SIZE(context->desc.h264enc.dpb); i++) {
+               struct pipe_video_buffer *buf = context->desc.h264enc.dpb[i].buffer;
+               if (buf && !context->desc.h264enc.dpb[i].id)
+                  buf->destroy(buf);
+            }
             util_dynarray_fini(&context->desc.h264enc.raw_headers);
          }
          if (u_reduce_video_profile(context->decoder->profile) ==
              PIPE_VIDEO_FORMAT_HEVC) {
             if (context->desc.h265enc.frame_idx)
                _mesa_hash_table_destroy(context->desc.h265enc.frame_idx, NULL);
+            for (uint32_t i = 0; i < ARRAY_SIZE(context->desc.h265enc.dpb); i++) {
+               struct pipe_video_buffer *buf = context->desc.h265enc.dpb[i].buffer;
+               if (buf && !context->desc.h265enc.dpb[i].id)
+                  buf->destroy(buf);
+            }
             util_dynarray_fini(&context->desc.h265enc.raw_headers);
          }
          if (u_reduce_video_profile(context->decoder->profile) ==
diff --git a/src/gallium/frontends/va/picture_h264_enc.c b/src/gallium/frontends/va/picture_h264_enc.c
index 6464c2ce0d6..91b76d2de40 100644
--- a/src/gallium/frontends/va/picture_h264_enc.c
+++ b/src/gallium/frontends/va/picture_h264_enc.c
@@ -37,7 +37,7 @@ vlVaHandleVAEncPictureParameterBufferTypeH264(vlVaDriver *drv, vlVaContext *cont
    VAEncPictureParameterBufferH264 *h264;
    vlVaBuffer *coded_buf;
    vlVaSurface *surf;
-   unsigned i;
+   unsigned i, j;
 
    h264 = buf->data;
    if (h264->pic_fields.bits.idr_pic_flag == 1)
@@ -52,6 +52,30 @@ vlVaHandleVAEncPictureParameterBufferTypeH264(vlVaDriver *drv, vlVaContext *cont
    else if (context->desc.h264enc.frame_num == 1)
       context->desc.h264enc.i_remain--;
 
+   /* Evict unused surfaces */
+   for (i = 0; i < context->desc.h264enc.dpb_size; i++) {
+      struct pipe_h264_enc_dpb_entry *dpb = &context->desc.h264enc.dpb[i];
+      if (!dpb->id || dpb->id == h264->CurrPic.picture_id)
+         continue;
+      for (j = 0; j < ARRAY_SIZE(h264->ReferenceFrames); j++) {
+         if (h264->ReferenceFrames[j].picture_id == dpb->id) {
+            dpb->evict = false;
+            break;
+         }
+      }
+      if (j == ARRAY_SIZE(h264->ReferenceFrames)) {
+         if (dpb->evict) {
+            surf = handle_table_get(drv->htab, dpb->id);
+            assert(surf);
+            surf->is_dpb = false;
+            surf->buffer = NULL;
+            /* Keep the buffer for reuse later */
+            dpb->id = 0;
+         }
+         dpb->evict = !dpb->evict;
+      }
+   }
+
    surf = handle_table_get(drv->htab, h264->CurrPic.picture_id);
    if (!surf)
       return VA_STATUS_ERROR_INVALID_PARAMETER;
@@ -61,17 +85,32 @@ vlVaHandleVAEncPictureParameterBufferTypeH264(vlVaDriver *drv, vlVaContext *cont
          assert(surf->is_dpb);
          break;
       }
-      if (!context->desc.h264enc.dpb[i].id) {
-         assert(!surf->is_dpb);
+      if (!surf->is_dpb && !context->desc.h264enc.dpb[i].id) {
          surf->is_dpb = true;
          if (surf->buffer) {
             surf->buffer->destroy(surf->buffer);
             surf->buffer = NULL;
          }
-         if (context->decoder && context->decoder->create_dpb_buffer)
-            surf->buffer = context->decoder->create_dpb_buffer(context->decoder, &context->desc.base, &surf->templat);
+         if (context->decoder->create_dpb_buffer) {
+            struct pipe_video_buffer *buffer = context->desc.h264enc.dpb[i].buffer;
+            if (!buffer) {
+               /* Find unused buffer */
+               for (j = 0; j < context->desc.h264enc.dpb_size; j++) {
+                  struct pipe_h264_enc_dpb_entry *dpb = &context->desc.h264enc.dpb[j];
+                  if (!dpb->id && dpb->buffer) {
+                     buffer = dpb->buffer;
+                     dpb->buffer = NULL;
+                     break;
+                  }
+               }
+            }
+            if (!buffer)
+               buffer = context->decoder->create_dpb_buffer(context->decoder, &context->desc.base, &surf->templat);
+            surf->buffer = buffer;
+         }
          vlVaSetSurfaceContext(drv, surf, context);
-         context->desc.h264enc.dpb_size++;
+         if (i == context->desc.h264enc.dpb_size)
+            context->desc.h264enc.dpb_size++;
          break;
       }
    }
@@ -83,6 +122,7 @@ vlVaHandleVAEncPictureParameterBufferTypeH264(vlVaDriver *drv, vlVaContext *cont
    context->desc.h264enc.dpb[i].pic_order_cnt = h264->CurrPic.TopFieldOrderCnt;
    context->desc.h264enc.dpb[i].is_ltr = h264->CurrPic.flags & VA_PICTURE_H264_LONG_TERM_REFERENCE;
    context->desc.h264enc.dpb[i].buffer = surf->buffer;
+   context->desc.h264enc.dpb[i].evict = false;
 
    context->desc.h264enc.p_remain = context->desc.h264enc.gop_size - context->desc.h264enc.gop_cnt - context->desc.h264enc.i_remain;
 
@@ -181,20 +221,26 @@ vlVaHandleVAEncSliceParameterBufferTypeH264(vlVaDriver *drv, vlVaContext *contex
       context->desc.h264enc.num_ref_idx_l1_active_minus1 = h264->num_ref_idx_l1_active_minus1;
    }
 
-   for (int i = 0; i < 32; i++) {
-      if (h264->RefPicList0[i].picture_id != VA_INVALID_ID) {
-         context->desc.h264enc.ref_list0[i] = vlVaDpbIndex(context, h264->RefPicList0[i].picture_id);
-               context->desc.h264enc.ref_idx_l0_list[i] = PTR_TO_UINT(util_hash_table_get(context->desc.h264enc.frame_idx,
-                                 UINT_TO_PTR(h264->RefPicList0[i].picture_id + 1)));
-               context->desc.h264enc.l0_is_long_term[i] = h264->RefPicList0[i].flags &
-		       					  VA_PICTURE_H264_LONG_TERM_REFERENCE;
-      }
-      if (h264->RefPicList1[i].picture_id != VA_INVALID_ID && h264->slice_type == 1) {
-         context->desc.h264enc.ref_list1[i] = vlVaDpbIndex(context, h264->RefPicList1[i].picture_id);
+   if (h264->slice_type != PIPE_H264_SLICE_TYPE_I && h264->slice_type != PIPE_H264_SLICE_TYPE_SI) {
+      for (int i = 0; i < 32; i++) {
+         if (h264->RefPicList0[i].picture_id != VA_INVALID_ID) {
+            context->desc.h264enc.ref_list0[i] = vlVaDpbIndex(context, h264->RefPicList0[i].picture_id);
+            if (context->desc.h264enc.ref_list0[i] == PIPE_H2645_LIST_REF_INVALID_ENTRY)
+               return VA_STATUS_ERROR_INVALID_PARAMETER;
+
+            context->desc.h264enc.ref_idx_l0_list[i] = PTR_TO_UINT(util_hash_table_get(context->desc.h264enc.frame_idx,
+                                    UINT_TO_PTR(h264->RefPicList0[i].picture_id + 1)));
+            context->desc.h264enc.l0_is_long_term[i] = h264->RefPicList0[i].flags & VA_PICTURE_H264_LONG_TERM_REFERENCE;
+         }
+         if (h264->RefPicList1[i].picture_id != VA_INVALID_ID && h264->slice_type == PIPE_H264_SLICE_TYPE_B) {
+            context->desc.h264enc.ref_list1[i] = vlVaDpbIndex(context, h264->RefPicList1[i].picture_id);
+            if (context->desc.h264enc.ref_list1[i] == PIPE_H2645_LIST_REF_INVALID_ENTRY)
+               return VA_STATUS_ERROR_INVALID_PARAMETER;
+
             context->desc.h264enc.ref_idx_l1_list[i] = PTR_TO_UINT(util_hash_table_get(context->desc.h264enc.frame_idx,
-               			 UINT_TO_PTR(h264->RefPicList1[i].picture_id + 1)));
-            context->desc.h264enc.l1_is_long_term[i] = h264->RefPicList1[i].flags &
-		    				       VA_PICTURE_H264_LONG_TERM_REFERENCE;
+                                    UINT_TO_PTR(h264->RefPicList1[i].picture_id + 1)));
+            context->desc.h264enc.l1_is_long_term[i] = h264->RefPicList1[i].flags & VA_PICTURE_H264_LONG_TERM_REFERENCE;
+         }
       }
    }
 
diff --git a/src/gallium/frontends/va/picture_hevc_enc.c b/src/gallium/frontends/va/picture_hevc_enc.c
index 61a9d83ed7c..98c2f744555 100644
--- a/src/gallium/frontends/va/picture_hevc_enc.c
+++ b/src/gallium/frontends/va/picture_hevc_enc.c
@@ -42,7 +42,7 @@ vlVaHandleVAEncPictureParameterBufferTypeHEVC(vlVaDriver *drv, vlVaContext *cont
    VAEncPictureParameterBufferHEVC *h265;
    vlVaBuffer *coded_buf;
    vlVaSurface *surf;
-   int i;
+   int i, j;
 
    h265 = buf->data;
    context->desc.h265enc.decoded_curr_pic = h265->decoded_curr_pic.picture_id;
@@ -51,6 +51,30 @@ vlVaHandleVAEncPictureParameterBufferTypeHEVC(vlVaDriver *drv, vlVaContext *cont
    for (i = 0; i < 15; i++)
       context->desc.h265enc.reference_frames[i] = h265->reference_frames[i].picture_id;
 
+   /* Evict unused surfaces */
+   for (i = 0; i < context->desc.h265enc.dpb_size; i++) {
+      struct pipe_h265_enc_dpb_entry *dpb = &context->desc.h265enc.dpb[i];
+      if (!dpb->id || dpb->id == h265->decoded_curr_pic.picture_id)
+         continue;
+      for (j = 0; j < ARRAY_SIZE(h265->reference_frames); j++) {
+         if (h265->reference_frames[j].picture_id == dpb->id) {
+            dpb->evict = false;
+            break;
+         }
+      }
+      if (j == ARRAY_SIZE(h265->reference_frames)) {
+         if (dpb->evict) {
+            surf = handle_table_get(drv->htab, dpb->id);
+            assert(surf);
+            surf->is_dpb = false;
+            surf->buffer = NULL;
+            /* Keep the buffer for reuse later */
+            dpb->id = 0;
+         }
+         dpb->evict = !dpb->evict;
+      }
+   }
+
    surf = handle_table_get(drv->htab, h265->decoded_curr_pic.picture_id);
    if (!surf)
       return VA_STATUS_ERROR_INVALID_PARAMETER;
@@ -60,27 +84,43 @@ vlVaHandleVAEncPictureParameterBufferTypeHEVC(vlVaDriver *drv, vlVaContext *cont
          assert(surf->is_dpb);
          break;
       }
-      if (!context->desc.h265enc.dpb[i].id) {
-         assert(!surf->is_dpb);
+      if (!surf->is_dpb && !context->desc.h265enc.dpb[i].id) {
          surf->is_dpb = true;
          if (surf->buffer) {
             surf->buffer->destroy(surf->buffer);
             surf->buffer = NULL;
          }
-         if (context->decoder && context->decoder->create_dpb_buffer)
-            surf->buffer = context->decoder->create_dpb_buffer(context->decoder, &context->desc.base, &surf->templat);
+         if (context->decoder->create_dpb_buffer) {
+            struct pipe_video_buffer *buffer = context->desc.h265enc.dpb[i].buffer;
+            if (!buffer) {
+               /* Find unused buffer */
+               for (j = 0; j < context->desc.h265enc.dpb_size; j++) {
+                  struct pipe_h265_enc_dpb_entry *dpb = &context->desc.h265enc.dpb[j];
+                  if (!dpb->id && dpb->buffer) {
+                     buffer = dpb->buffer;
+                     dpb->buffer = NULL;
+                     break;
+                  }
+               }
+            }
+            if (!buffer)
+               buffer = context->decoder->create_dpb_buffer(context->decoder, &context->desc.base, &surf->templat);
+            surf->buffer = buffer;
+         }
          vlVaSetSurfaceContext(drv, surf, context);
-         context->desc.h265enc.dpb_size++;
+         if (i == context->desc.h265enc.dpb_size)
+            context->desc.h265enc.dpb_size++;
          break;
       }
    }
-   if (i == ARRAY_SIZE(context->desc.h264enc.dpb))
+   if (i == ARRAY_SIZE(context->desc.h265enc.dpb))
       return VA_STATUS_ERROR_INVALID_PARAMETER;
    context->desc.h265enc.dpb_curr_pic = i;
    context->desc.h265enc.dpb[i].id = h265->decoded_curr_pic.picture_id;
    context->desc.h265enc.dpb[i].pic_order_cnt = h265->decoded_curr_pic.pic_order_cnt;
    context->desc.h265enc.dpb[i].is_ltr = h265->decoded_curr_pic.flags & VA_PICTURE_HEVC_LONG_TERM_REFERENCE;
    context->desc.h265enc.dpb[i].buffer = surf->buffer;
+   context->desc.h265enc.dpb[i].evict = false;
 
    context->desc.h265enc.pic_order_cnt = h265->decoded_curr_pic.pic_order_cnt;
    coded_buf = handle_table_get(drv->htab, h265->coded_buf);
@@ -178,16 +218,24 @@ vlVaHandleVAEncSliceParameterBufferTypeHEVC(vlVaDriver *drv, vlVaContext *contex
       context->desc.h265enc.num_ref_idx_l1_active_minus1 = h265->num_ref_idx_l1_active_minus1;
    }
 
-   for (int i = 0; i < 15; i++) {
-      if (h265->ref_pic_list0[i].picture_id != VA_INVALID_ID) {
-         context->desc.h265enc.ref_list0[i] = vlVaDpbIndex(context, h265->ref_pic_list0[i].picture_id);
-         context->desc.h265enc.ref_idx_l0_list[i] = PTR_TO_UINT(util_hash_table_get(context->desc.h265enc.frame_idx,
-                                 UINT_TO_PTR(h265->ref_pic_list0[i].picture_id + 1)));
-      }
-      if (h265->ref_pic_list1[i].picture_id != VA_INVALID_ID && h265->slice_type == PIPE_H265_SLICE_TYPE_B) {
-         context->desc.h265enc.ref_list1[i] = vlVaDpbIndex(context, h265->ref_pic_list1[i].picture_id);
-         context->desc.h265enc.ref_idx_l1_list[i] = PTR_TO_UINT(util_hash_table_get(context->desc.h265enc.frame_idx,
-                                 UINT_TO_PTR(h265->ref_pic_list1[i].picture_id + 1)));
+   if (h265->slice_type != PIPE_H265_SLICE_TYPE_I) {
+      for (int i = 0; i < 15; i++) {
+         if (h265->ref_pic_list0[i].picture_id != VA_INVALID_ID) {
+            context->desc.h265enc.ref_list0[i] = vlVaDpbIndex(context, h265->ref_pic_list0[i].picture_id);
+            if (context->desc.h265enc.ref_list0[i] == PIPE_H2645_LIST_REF_INVALID_ENTRY)
+               return VA_STATUS_ERROR_INVALID_PARAMETER;
+
+            context->desc.h265enc.ref_idx_l0_list[i] = PTR_TO_UINT(util_hash_table_get(context->desc.h265enc.frame_idx,
+                                    UINT_TO_PTR(h265->ref_pic_list0[i].picture_id + 1)));
+         }
+         if (h265->ref_pic_list1[i].picture_id != VA_INVALID_ID && h265->slice_type == PIPE_H265_SLICE_TYPE_B) {
+            context->desc.h265enc.ref_list1[i] = vlVaDpbIndex(context, h265->ref_pic_list1[i].picture_id);
+            if (context->desc.h265enc.ref_list1[i] == PIPE_H2645_LIST_REF_INVALID_ENTRY)
+               return VA_STATUS_ERROR_INVALID_PARAMETER;
+
+            context->desc.h265enc.ref_idx_l1_list[i] = PTR_TO_UINT(util_hash_table_get(context->desc.h265enc.frame_idx,
+                                    UINT_TO_PTR(h265->ref_pic_list1[i].picture_id + 1)));
+         }
       }
    }
 
diff --git a/src/gallium/include/pipe/p_video_state.h b/src/gallium/include/pipe/p_video_state.h
index c57eaf03b27..0247e228938 100644
--- a/src/gallium/include/pipe/p_video_state.h
+++ b/src/gallium/include/pipe/p_video_state.h
@@ -761,6 +761,7 @@ struct pipe_h264_enc_dpb_entry
    uint32_t temporal_id;
    bool is_ltr;
    struct pipe_video_buffer *buffer;
+   bool evict;
 };
 
 struct pipe_h264_enc_picture_desc
@@ -1162,6 +1163,7 @@ struct pipe_h265_enc_dpb_entry
    uint32_t temporal_id;
    bool is_ltr;
    struct pipe_video_buffer *buffer;
+   bool evict;
 };
 
 struct pipe_h265_enc_picture_desc