From 7df39b802d78fd93162df1cf63c717787fe691db Mon Sep 17 00:00:00 2001 From: Sil Vilerino Date: Mon, 19 Aug 2024 10:26:56 -0400 Subject: [PATCH] d3d12: Support for on demand vps, sps, pps, aud headers requests from frontend Part-of: --- src/gallium/drivers/d3d12/d3d12_video_enc.cpp | 3 +-- src/gallium/drivers/d3d12/d3d12_video_enc.h | 2 ++ .../drivers/d3d12/d3d12_video_enc_h264.cpp | 23 +++++++++++------ .../drivers/d3d12/d3d12_video_enc_hevc.cpp | 25 +++++++++++++------ ...2_video_encoder_bitstream_builder_h264.cpp | 3 +-- ...d12_video_encoder_bitstream_builder_h264.h | 5 +--- 6 files changed, 38 insertions(+), 23 deletions(-) diff --git a/src/gallium/drivers/d3d12/d3d12_video_enc.cpp b/src/gallium/drivers/d3d12/d3d12_video_enc.cpp index 0b18fc8c876..1eb9d9a298b 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_enc.cpp +++ b/src/gallium/drivers/d3d12/d3d12_video_enc.cpp @@ -597,8 +597,7 @@ d3d12_video_encoder_create_reference_picture_manager(struct d3d12_video_encoder case PIPE_VIDEO_FORMAT_MPEG4_AVC: { pD3D12Enc->m_upDPBManager = std::make_unique(); - struct pipe_h264_enc_picture_desc *pH264Pic = (struct pipe_h264_enc_picture_desc *) picture; - pD3D12Enc->m_upBitstreamBuilder = std::make_unique(pH264Pic->insert_aud_nalu); + pD3D12Enc->m_upBitstreamBuilder = std::make_unique(); } break; #endif #if VIDEO_CODEC_H265ENC diff --git a/src/gallium/drivers/d3d12/d3d12_video_enc.h b/src/gallium/drivers/d3d12/d3d12_video_enc.h index aa33cd61c36..184c4ebd6d7 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_enc.h +++ b/src/gallium/drivers/d3d12/d3d12_video_enc.h @@ -134,6 +134,8 @@ enum d3d12_video_encoder_config_dirty_flags d3d12_video_encoder_config_dirty_flag_sequence_header = 0x400, d3d12_video_encoder_config_dirty_flag_intra_refresh = 0x800, d3d12_video_encoder_config_dirty_flag_video_header = 0x1000, + d3d12_video_encoder_config_dirty_flag_picture_header = 0x2000, + d3d12_video_encoder_config_dirty_flag_aud_header = 0x4000, }; DEFINE_ENUM_FLAG_OPERATORS(d3d12_video_encoder_config_dirty_flags); diff --git a/src/gallium/drivers/d3d12/d3d12_video_enc_h264.cpp b/src/gallium/drivers/d3d12/d3d12_video_enc_h264.cpp index 591bb4dbfe4..e89331062a1 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_enc_h264.cpp +++ b/src/gallium/drivers/d3d12/d3d12_video_enc_h264.cpp @@ -923,9 +923,15 @@ d3d12_video_encoder_update_current_encoder_config_state_h264(struct d3d12_video_ } pD3D12Enc->m_currentEncodeConfig.m_encoderCodecSpecificSequenceStateDescH264 = h264Pic->seq; - if ((h264Pic->picture_type == PIPE_H2645_ENC_PICTURE_TYPE_IDR) && - (h264Pic->renew_headers_on_idr)) - pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags |= d3d12_video_encoder_config_dirty_flag_sequence_header; + // Iterate over the headers the app requested and set flags to emit those for this frame + util_dynarray_foreach(&h264Pic->raw_headers, struct pipe_enc_raw_header, header) { + if (header->type == PIPE_H264_NAL_SPS) + pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags |= d3d12_video_encoder_config_dirty_flag_sequence_header; + else if (header->type == PIPE_H264_NAL_PPS) + pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags |= d3d12_video_encoder_config_dirty_flag_picture_header; + else if (header->type == PIPE_H264_NAL_AUD) + pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags |= d3d12_video_encoder_config_dirty_flag_aud_header; + } // Set input format DXGI_FORMAT targetFmt = d3d12_convert_pipe_video_profile_to_dxgi_format(pD3D12Enc->base.profile); @@ -1163,7 +1169,9 @@ d3d12_video_encoder_build_codec_headers_h264(struct d3d12_video_encoder *pD3D12E size_t writtenAUDBytesCount = 0; pWrittenCodecUnitsSizes.clear(); - if (pH264BitstreamBuilder->insert_aud_nalu_requested()) + + bool forceWriteAUD = (pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags & d3d12_video_encoder_config_dirty_flag_aud_header); + if (forceWriteAUD) { pH264BitstreamBuilder->write_aud(pD3D12Enc->m_BitstreamHeadersBuffer, pD3D12Enc->m_BitstreamHeadersBuffer.begin(), @@ -1172,11 +1180,11 @@ d3d12_video_encoder_build_codec_headers_h264(struct d3d12_video_encoder *pD3D12E } bool isFirstFrame = (pD3D12Enc->m_fenceValue == 1); + bool forceWriteSPS = (pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags & d3d12_video_encoder_config_dirty_flag_sequence_header); bool writeNewSPS = isFirstFrame // on first frame || ((pD3D12Enc->m_currentEncodeConfig.m_seqFlags & // also on resolution change D3D12_VIDEO_ENCODER_SEQUENCE_CONTROL_FLAG_RESOLUTION_CHANGE) != 0) - // Also on input format dirty flag for new SPS, VUI etc - || (pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags & d3d12_video_encoder_config_dirty_flag_sequence_header); + || forceWriteSPS; uint32_t active_seq_parameter_set_id = pH264BitstreamBuilder->get_active_sps().seq_parameter_set_id; @@ -1209,7 +1217,8 @@ d3d12_video_encoder_build_codec_headers_h264(struct d3d12_video_encoder *pD3D12E writtenPPSBytesCount); const H264_PPS &active_pps = pH264BitstreamBuilder->get_active_pps(); - if (d3d12_video_encoder_needs_new_pps_h264(pD3D12Enc, writeNewSPS, tentative_pps, active_pps)) { + bool forceWritePPS = (pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags & d3d12_video_encoder_config_dirty_flag_picture_header); + if (forceWritePPS || d3d12_video_encoder_needs_new_pps_h264(pD3D12Enc, writeNewSPS, tentative_pps, active_pps)) { pH264BitstreamBuilder->set_active_pps(tentative_pps); pD3D12Enc->m_BitstreamHeadersBuffer.resize(writtenAUDBytesCount + writtenSPSBytesCount + writtenPPSBytesCount); memcpy(&pD3D12Enc->m_BitstreamHeadersBuffer.data()[writtenAUDBytesCount + writtenSPSBytesCount], pD3D12Enc->m_StagingHeadersBuffer.data(), writtenPPSBytesCount); diff --git a/src/gallium/drivers/d3d12/d3d12_video_enc_hevc.cpp b/src/gallium/drivers/d3d12/d3d12_video_enc_hevc.cpp index 7165baff13b..5837d463f03 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_enc_hevc.cpp +++ b/src/gallium/drivers/d3d12/d3d12_video_enc_hevc.cpp @@ -811,9 +811,17 @@ d3d12_video_encoder_update_current_encoder_config_state_hevc(struct d3d12_video_ } pD3D12Enc->m_currentEncodeConfig.m_encoderCodecSpecificSequenceStateDescH265 = hevcPic->seq; - if ((hevcPic->picture_type == PIPE_H2645_ENC_PICTURE_TYPE_IDR) && - (hevcPic->renew_headers_on_idr)) - pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags |= d3d12_video_encoder_config_dirty_flag_sequence_header; + // Iterate over the headers the app requested and set flags to emit those for this frame + util_dynarray_foreach(&hevcPic->raw_headers, struct pipe_enc_raw_header, header) { + if (header->type == PIPE_H265_NAL_VPS) + pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags |= d3d12_video_encoder_config_dirty_flag_video_header; + else if (header->type == PIPE_H265_NAL_SPS) + pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags |= d3d12_video_encoder_config_dirty_flag_sequence_header; + else if (header->type == PIPE_H265_NAL_PPS) + pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags |= d3d12_video_encoder_config_dirty_flag_picture_header; + else if (header->type == PIPE_H265_NAL_AUD) + pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags |= d3d12_video_encoder_config_dirty_flag_aud_header; + } // Set input format DXGI_FORMAT targetFmt = d3d12_convert_pipe_video_profile_to_dxgi_format(pD3D12Enc->base.profile); @@ -1066,8 +1074,8 @@ d3d12_video_encoder_build_codec_headers_hevc(struct d3d12_video_encoder *pD3D12E uint32_t active_seq_parameter_set_id = pHEVCBitstreamBuilder->get_active_sps().sps_seq_parameter_set_id; uint32_t active_video_parameter_set_id = pHEVCBitstreamBuilder->get_active_vps().vps_video_parameter_set_id; - bool writeNewVPS = isFirstFrame; - + bool writeNewVPS = isFirstFrame || (pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags & d3d12_video_encoder_config_dirty_flag_video_header); + size_t writtenVPSBytesCount = 0; if (writeNewVPS) { bool gopHasBFrames = (pD3D12Enc->m_currentEncodeConfig.m_encoderGOPConfigDesc.m_HEVCGroupOfPictures.PPicturePeriod > 1); @@ -1084,11 +1092,11 @@ d3d12_video_encoder_build_codec_headers_hevc(struct d3d12_video_encoder *pD3D12E pWrittenCodecUnitsSizes.push_back(writtenVPSBytesCount); } + bool forceWriteSPS = (pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags & d3d12_video_encoder_config_dirty_flag_sequence_header); bool writeNewSPS = writeNewVPS // on new VPS written || ((pD3D12Enc->m_currentEncodeConfig.m_seqFlags & // also on resolution change D3D12_VIDEO_ENCODER_SEQUENCE_CONTROL_FLAG_RESOLUTION_CHANGE) != 0) - // Also on input format dirty flag for new SPS, VUI etc - || (pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags & d3d12_video_encoder_config_dirty_flag_sequence_header); + || forceWriteSPS; size_t writtenSPSBytesCount = 0; if (writeNewSPS) { @@ -1118,7 +1126,8 @@ d3d12_video_encoder_build_codec_headers_hevc(struct d3d12_video_encoder *pD3D12E writtenPPSBytesCount); const HevcPicParameterSet &active_pps = pHEVCBitstreamBuilder->get_active_pps(); - if (d3d12_video_encoder_needs_new_pps_hevc(pD3D12Enc, writeNewSPS, tentative_pps, active_pps)) { + bool forceWritePPS = (pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags & d3d12_video_encoder_config_dirty_flag_picture_header); + if (forceWritePPS || d3d12_video_encoder_needs_new_pps_hevc(pD3D12Enc, writeNewSPS, tentative_pps, active_pps)) { pHEVCBitstreamBuilder->set_active_pps(tentative_pps); pD3D12Enc->m_BitstreamHeadersBuffer.resize(writtenSPSBytesCount + writtenVPSBytesCount + writtenPPSBytesCount); memcpy(&pD3D12Enc->m_BitstreamHeadersBuffer.data()[(writtenSPSBytesCount + writtenVPSBytesCount)], pD3D12Enc->m_StagingHeadersBuffer.data(), writtenPPSBytesCount); diff --git a/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.cpp b/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.cpp index 151767ab838..66e7fcf4ef1 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.cpp +++ b/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.cpp @@ -26,8 +26,7 @@ #include #include "util/u_video.h" -d3d12_video_bitstream_builder_h264::d3d12_video_bitstream_builder_h264(bool insert_aud_nalu) - : m_insert_aud_nalu(insert_aud_nalu) +d3d12_video_bitstream_builder_h264::d3d12_video_bitstream_builder_h264() { } inline H264_SPEC_PROFILES diff --git a/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.h b/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.h index 62e3fb0f98d..d78775b93c0 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.h +++ b/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.h @@ -31,7 +31,7 @@ class d3d12_video_bitstream_builder_h264 : public d3d12_video_bitstream_builder_ { public: - d3d12_video_bitstream_builder_h264(bool insert_aud_nalu = false); + d3d12_video_bitstream_builder_h264(); ~d3d12_video_bitstream_builder_h264() {}; H264_SPS build_sps(const struct pipe_h264_enc_seq_param & seqData, @@ -88,13 +88,10 @@ class d3d12_video_bitstream_builder_h264 : public d3d12_video_bitstream_builder_ m_activePPSStructure = active_pps; }; - bool insert_aud_nalu_requested() { return m_insert_aud_nalu; } - private: d3d12_video_nalu_writer_h264 m_h264Encoder; H264_SPS m_activeSPSStructure = {}; H264_PPS m_activePPSStructure = {}; - bool m_insert_aud_nalu = false; }; #endif