diff --git a/src/gallium/frontends/va/picture.c b/src/gallium/frontends/va/picture.c
index 84cc16ccb01..ba8f9de64dc 100644
--- a/src/gallium/frontends/va/picture.c
+++ b/src/gallium/frontends/va/picture.c
@@ -813,6 +813,8 @@ vlVaEndPicture(VADriverContextP ctx, VAContextID context_id)
             context->first_single_submitted = false;
          surf->force_flushed = true;
       }
+      if (!context->desc.h264enc.not_referenced)
+         context->desc.h264enc.frame_num++;
    } else if (context->decoder->entrypoint == PIPE_VIDEO_ENTRYPOINT_ENCODE &&
               u_reduce_video_profile(context->templat.profile) == PIPE_VIDEO_FORMAT_HEVC)
       context->desc.h265enc.frame_num++;
diff --git a/src/gallium/frontends/va/picture_h264_enc.c b/src/gallium/frontends/va/picture_h264_enc.c
index 3f9b3b199b7..1da11d29188 100644
--- a/src/gallium/frontends/va/picture_h264_enc.c
+++ b/src/gallium/frontends/va/picture_h264_enc.c
@@ -36,7 +36,8 @@ vlVaHandleVAEncPictureParameterBufferTypeH264(vlVaDriver *drv, vlVaContext *cont
    vlVaBuffer *coded_buf;
 
    h264 = buf->data;
-   context->desc.h264enc.frame_num = h264->frame_num;
+   if (h264->pic_fields.bits.idr_pic_flag == 1)
+      context->desc.h264enc.frame_num = 0;
    context->desc.h264enc.not_referenced = !h264->pic_fields.bits.reference_pic_flag;
    context->desc.h264enc.pic_order_cnt = h264->CurrPic.TopFieldOrderCnt;
    if (context->desc.h264enc.gop_cnt == 0)
@@ -54,7 +55,7 @@ vlVaHandleVAEncPictureParameterBufferTypeH264(vlVaDriver *drv, vlVaContext *cont
 
    _mesa_hash_table_insert(context->desc.h264enc.frame_idx,
 		       UINT_TO_PTR(h264->CurrPic.picture_id + 1),
-		       UINT_TO_PTR(h264->frame_num));
+		       UINT_TO_PTR(context->desc.h264enc.frame_num));
 
    if (h264->pic_fields.bits.idr_pic_flag == 1)
       context->desc.h264enc.picture_type = PIPE_H2645_ENC_PICTURE_TYPE_IDR;