diff --git a/src/gallium/drivers/radeonsi/radeon_vcn_enc.c b/src/gallium/drivers/radeonsi/radeon_vcn_enc.c
index 066103747b6..fbf25490040 100644
--- a/src/gallium/drivers/radeonsi/radeon_vcn_enc.c
+++ b/src/gallium/drivers/radeonsi/radeon_vcn_enc.c
@@ -1565,7 +1565,7 @@ static void radeon_enc_begin_frame(struct pipe_video_codec *encoder,
       dpb_slots = 0;
 
    radeon_vcn_enc_get_param(enc, picture);
-   if (dpb_slots && !enc->dpb && setup_dpb(enc, dpb_slots)) {
+   if (enc->first_frame && setup_dpb(enc, dpb_slots)) {
       enc->dpb = si_resource(pipe_buffer_create(enc->screen, 0, PIPE_USAGE_DEFAULT, enc->dpb_size));
       if (!enc->dpb) {
          RADEON_ENC_ERR("Can't create DPB buffer.\n");
@@ -1935,6 +1935,8 @@ static int radeon_enc_end_frame(struct pipe_video_codec *encoder, struct pipe_vi
    if (enc->error)
       return -1;
 
+   enc->first_frame = false;
+
    return flush(enc, picture->flush_flags, picture->out_fence);
 }
 
@@ -2201,6 +2203,8 @@ struct pipe_video_codec *radeon_create_encoder(struct pipe_context *context,
       radeon_enc_1_2_init(enc);
    }
 
+   enc->first_frame = true;
+
    return &enc->base;
 
 error:
diff --git a/src/gallium/drivers/radeonsi/radeon_vcn_enc.h b/src/gallium/drivers/radeonsi/radeon_vcn_enc.h
index 1c234f75d43..75f39f91e6f 100644
--- a/src/gallium/drivers/radeonsi/radeon_vcn_enc.h
+++ b/src/gallium/drivers/radeonsi/radeon_vcn_enc.h
@@ -262,6 +262,7 @@ struct radeon_encoder {
    unsigned metadata_size;
 
    bool error;
+   bool first_frame;
 
    enum {
       DPB_LEGACY = 0,