diff --git a/src/gallium/drivers/radeonsi/radeon_vcn_enc.c b/src/gallium/drivers/radeonsi/radeon_vcn_enc.c
index 4bd1f9b468b..9d0aa41e342 100644
--- a/src/gallium/drivers/radeonsi/radeon_vcn_enc.c
+++ b/src/gallium/drivers/radeonsi/radeon_vcn_enc.c
@@ -1042,23 +1042,23 @@ static void radeon_enc_begin_frame(struct pipe_video_codec *encoder,
 {
    struct radeon_encoder *enc = (struct radeon_encoder *)encoder;
    struct vl_video_buffer *vid_buf = (struct vl_video_buffer *)source;
-   bool need_rate_control = false;
+   enc->need_rate_control = false;
 
    if (u_reduce_video_profile(enc->base.profile) == PIPE_VIDEO_FORMAT_MPEG4_AVC) {
       struct pipe_h264_enc_picture_desc *pic = (struct pipe_h264_enc_picture_desc *)picture;
-      need_rate_control =
+      enc->need_rate_control =
          (enc->enc_pic.rc_layer_init[0].target_bit_rate != pic->rate_ctrl[0].target_bitrate) ||
          (enc->enc_pic.rc_layer_init[0].frame_rate_num != pic->rate_ctrl[0].frame_rate_num) ||
          (enc->enc_pic.rc_layer_init[0].frame_rate_den != pic->rate_ctrl[0].frame_rate_den);
    } else if (u_reduce_video_profile(picture->profile) == PIPE_VIDEO_FORMAT_HEVC) {
       struct pipe_h265_enc_picture_desc *pic = (struct pipe_h265_enc_picture_desc *)picture;
-      need_rate_control =
+      enc->need_rate_control =
          (enc->enc_pic.rc_layer_init[0].target_bit_rate != pic->rc.target_bitrate) ||
          (enc->enc_pic.rc_layer_init[0].frame_rate_num != pic->rc.frame_rate_num) ||
          (enc->enc_pic.rc_layer_init[0].frame_rate_den != pic->rc.frame_rate_den);
    } else if (u_reduce_video_profile(picture->profile) == PIPE_VIDEO_FORMAT_AV1) {
       struct pipe_av1_enc_picture_desc *pic = (struct pipe_av1_enc_picture_desc *)picture;
-      need_rate_control =
+      enc->need_rate_control =
          (enc->enc_pic.rc_layer_init[0].target_bit_rate != pic->rc[0].target_bitrate) ||
          (enc->enc_pic.rc_layer_init[0].frame_rate_num != pic->rc[0].frame_rate_num) ||
          (enc->enc_pic.rc_layer_init[0].frame_rate_den != pic->rc[0].frame_rate_den);
@@ -1113,23 +1113,22 @@ static void radeon_enc_begin_frame(struct pipe_video_codec *encoder,
 
    enc->need_feedback = false;
 
-   if (!enc->stream_handle || need_rate_control) {
+   if (!enc->stream_handle) {
       struct rvid_buffer fb;
-      if (!enc->stream_handle) {
-         enc->stream_handle = si_vid_alloc_stream_handle();
-         enc->si = CALLOC_STRUCT(rvid_buffer);
-         if (!enc->si ||
-             !enc->stream_handle ||
-             !si_vid_create_buffer(enc->screen, enc->si, 128 * 1024, PIPE_USAGE_STAGING)) {
-            RVID_ERR("Can't create session buffer.\n");
-            goto error;
-         }
+      enc->stream_handle = si_vid_alloc_stream_handle();
+      enc->si = CALLOC_STRUCT(rvid_buffer);
+      if (!enc->si ||
+          !enc->stream_handle ||
+          !si_vid_create_buffer(enc->screen, enc->si, 128 * 1024, PIPE_USAGE_STAGING)) {
+         RVID_ERR("Can't create session buffer.\n");
+         goto error;
       }
       si_vid_create_buffer(enc->screen, &fb, 4096, PIPE_USAGE_STAGING);
       enc->fb = &fb;
       enc->begin(enc);
       flush(enc);
       si_vid_destroy_buffer(&fb);
+      enc->need_rate_control = false;
    }
 
    return;
diff --git a/src/gallium/drivers/radeonsi/radeon_vcn_enc.h b/src/gallium/drivers/radeonsi/radeon_vcn_enc.h
index e75479d7106..114315ac948 100644
--- a/src/gallium/drivers/radeonsi/radeon_vcn_enc.h
+++ b/src/gallium/drivers/radeonsi/radeon_vcn_enc.h
@@ -262,6 +262,7 @@ struct radeon_encoder {
 
    bool emulation_prevention;
    bool need_feedback;
+   bool need_rate_control;
    unsigned dpb_size;
    unsigned roi_size;
    rvcn_enc_picture_info_t dpb_info[RENCODE_MAX_NUM_RECONSTRUCTED_PICTURES];
diff --git a/src/gallium/drivers/radeonsi/radeon_vcn_enc_1_2.c b/src/gallium/drivers/radeonsi/radeon_vcn_enc_1_2.c
index 03e8c382abb..cb1b2957c49 100644
--- a/src/gallium/drivers/radeonsi/radeon_vcn_enc_1_2.c
+++ b/src/gallium/drivers/radeonsi/radeon_vcn_enc_1_2.c
@@ -1392,11 +1392,22 @@ static void radeon_enc_headers_hevc(struct radeon_encoder *enc)
 
 static void encode(struct radeon_encoder *enc)
 {
+   unsigned i;
+
    enc->before_encode(enc);
    enc->session_info(enc);
    enc->total_task_size = 0;
    enc->task_info(enc, enc->need_feedback);
 
+   if (enc->need_rate_control) {
+      i = 0;
+      do {
+         enc->enc_pic.temporal_id = i;
+         enc->layer_select(enc);
+         enc->rc_layer_init(enc);
+      } while (++i < enc->enc_pic.num_temporal_layers);
+   }
+
    enc->encode_headers(enc);
    enc->ctx(enc);
    enc->bitstream(enc);
diff --git a/src/gallium/drivers/radeonsi/radeon_vcn_enc_2_0.c b/src/gallium/drivers/radeonsi/radeon_vcn_enc_2_0.c
index 958178e35df..aefbd3192e8 100644
--- a/src/gallium/drivers/radeonsi/radeon_vcn_enc_2_0.c
+++ b/src/gallium/drivers/radeonsi/radeon_vcn_enc_2_0.c
@@ -505,11 +505,22 @@ static void radeon_enc_ctx(struct radeon_encoder *enc)
 }
 static void encode(struct radeon_encoder *enc)
 {
+   unsigned i;
+
    enc->before_encode(enc);
    enc->session_info(enc);
    enc->total_task_size = 0;
    enc->task_info(enc, enc->need_feedback);
 
+   if (enc->need_rate_control) {
+      i = 0;
+      do {
+         enc->enc_pic.temporal_id = i;
+         enc->layer_select(enc);
+         enc->rc_layer_init(enc);
+      } while (++i < enc->enc_pic.num_temporal_layers);
+   }
+
    enc->encode_headers(enc);
    enc->ctx(enc);
    enc->bitstream(enc);