diff --git a/src/amd/common/ac_cmdbuf_sdma.c b/src/amd/common/ac_cmdbuf_sdma.c new file mode 100644 index 00000000000..fe21a2f2985 --- /dev/null +++ b/src/amd/common/ac_cmdbuf_sdma.c @@ -0,0 +1,66 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * Copyright 2025 Valve Corporation + * + * SPDX-License-Identifier: MIT + */ + +#include "ac_cmdbuf.h" +#include "ac_cmdbuf_sdma.h" +#include "sid.h" + +#include "util/u_math.h" + +void +ac_emit_sdma_nop(struct ac_cmdbuf *cs) +{ + /* SDMA NOP acts as a fence command and causes the SDMA engine to wait for pending copy operations. */ + ac_cmdbuf_begin(cs); + ac_cmdbuf_emit(SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); + ac_cmdbuf_end(); +} + +void +ac_emit_sdma_write_timestamp(struct ac_cmdbuf *cs, uint64_t va) +{ + ac_cmdbuf_begin(cs); + ac_cmdbuf_emit(SDMA_PACKET(SDMA_OPCODE_TIMESTAMP, SDMA_TS_SUB_OPCODE_GET_GLOBAL_TIMESTAMP, 0)); + ac_cmdbuf_emit(va); + ac_cmdbuf_emit(va >> 32); + ac_cmdbuf_end(); +} + +void +ac_emit_sdma_fence(struct ac_cmdbuf *cs, uint64_t va, uint32_t fence) +{ + ac_cmdbuf_begin(cs); + ac_cmdbuf_emit(SDMA_PACKET(SDMA_OPCODE_FENCE, 0, SDMA_FENCE_MTYPE_UC)); + ac_cmdbuf_emit(va); + ac_cmdbuf_emit(va >> 32); + ac_cmdbuf_emit(fence); + ac_cmdbuf_end(); +} + +void +ac_emit_sdma_wait_mem(struct ac_cmdbuf *cs, uint32_t op, uint64_t va, uint32_t ref, uint32_t mask) +{ + ac_cmdbuf_begin(cs); + ac_cmdbuf_emit(SDMA_PACKET(SDMA_OPCODE_POLL_REGMEM, 0, 0) | op << 28 | SDMA_POLL_MEM); + ac_cmdbuf_emit(va); + ac_cmdbuf_emit(va >> 32); + ac_cmdbuf_emit(ref); + ac_cmdbuf_emit(mask); + ac_cmdbuf_emit(SDMA_POLL_INTERVAL_160_CLK | SDMA_POLL_RETRY_INDEFINITELY << 16); + ac_cmdbuf_end(); +} + +void +ac_emit_sdma_write_data_head(struct ac_cmdbuf *cs, uint64_t va, uint32_t count) +{ + ac_cmdbuf_begin(cs); + ac_cmdbuf_emit(SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0)); + ac_cmdbuf_emit(va); + ac_cmdbuf_emit(va >> 32); + ac_cmdbuf_emit(count - 1); + ac_cmdbuf_end(); +} diff --git a/src/amd/common/ac_cmdbuf_sdma.h b/src/amd/common/ac_cmdbuf_sdma.h new file mode 100644 index 00000000000..b23b4676aee --- /dev/null +++ b/src/amd/common/ac_cmdbuf_sdma.h @@ -0,0 +1,31 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * Copyright 2025 Valve Corporation + * + * SPDX-License-Identifier: MIT + */ + +#ifndef AC_CMDBUF_SDMA_H +#define AC_CMDBUF_SDMA_H + +struct ac_cmdbuf; + +#ifdef __cplusplus +extern "C" { +#endif + +void ac_emit_sdma_nop(struct ac_cmdbuf *cs); + +void ac_emit_sdma_write_timestamp(struct ac_cmdbuf *cs, uint64_t va); + +void ac_emit_sdma_fence(struct ac_cmdbuf *cs, uint64_t va, uint32_t fence); + +void ac_emit_sdma_wait_mem(struct ac_cmdbuf *cs, uint32_t op, uint64_t va, uint32_t ref, uint32_t mask); + +void ac_emit_sdma_write_data_head(struct ac_cmdbuf *cs, uint64_t va, uint32_t count); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/amd/common/meson.build b/src/amd/common/meson.build index a5ba590968c..fb28b475f0e 100644 --- a/src/amd/common/meson.build +++ b/src/amd/common/meson.build @@ -56,6 +56,8 @@ amd_common_files = files( 'ac_binary.h', 'ac_cmdbuf.c', 'ac_cmdbuf.h', + 'ac_cmdbuf_sdma.c', + 'ac_cmdbuf_sdma.h', 'ac_shader_args.c', 'ac_shader_args.h', 'ac_shader_util.c', diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c index cb714a3cb43..9a0efca1856 100644 --- a/src/amd/vulkan/radv_cmd_buffer.c +++ b/src/amd/vulkan/radv_cmd_buffer.c @@ -15121,7 +15121,7 @@ radv_CmdWriteBufferMarker2AMD(VkCommandBuffer commandBuffer, VkPipelineStageFlag if (cmd_buffer->qf == RADV_QUEUE_TRANSFER) { radeon_check_space(device->ws, cs->b, 4); - radv_sdma_emit_fence(cs, va, marker); + ac_emit_sdma_fence(cmd_buffer->cs->b, va, marker); return; } diff --git a/src/amd/vulkan/radv_cs.c b/src/amd/vulkan/radv_cs.c index 6c3055e2caf..d1e7fa837e1 100644 --- a/src/amd/vulkan/radv_cs.c +++ b/src/amd/vulkan/radv_cs.c @@ -22,7 +22,7 @@ radv_cs_emit_write_event_eop(struct radv_cmd_stream *cs, enum amd_gfx_level gfx_ uint32_t new_fence, uint64_t gfx9_eop_bug_va) { if (cs->hw_ip == AMD_IP_SDMA) { - radv_sdma_emit_fence(cs, va, new_fence); + ac_emit_sdma_fence(cs->b, va, new_fence); return; } diff --git a/src/amd/vulkan/radv_cs.h b/src/amd/vulkan/radv_cs.h index f373d164700..e50718ab531 100644 --- a/src/amd/vulkan/radv_cs.h +++ b/src/amd/vulkan/radv_cs.h @@ -17,6 +17,8 @@ #include "radv_sdma.h" #include "sid.h" +#include "ac_cmdbuf_sdma.h" + static inline unsigned radeon_check_space(struct radeon_winsys *ws, struct ac_cmdbuf *cs, unsigned needed) { @@ -337,7 +339,7 @@ radv_cp_wait_mem(struct radv_cmd_stream *cs, const uint32_t op, const uint64_t v if (cs->hw_ip == AMD_IP_GFX || cs->hw_ip == AMD_IP_COMPUTE) { ac_emit_cp_wait_mem(cs->b, va, ref, mask, op); } else if (cs->hw_ip == AMD_IP_SDMA) { - radv_sdma_emit_wait_mem(cs, op, va, ref, mask); + ac_emit_sdma_wait_mem(cs->b, op, va, ref, mask); } else { UNREACHABLE("unsupported queue family"); } @@ -353,7 +355,7 @@ radv_cs_write_data_head(const struct radv_device *device, struct radv_cmd_stream if (cs->hw_ip == AMD_IP_COMPUTE || cs->hw_ip == AMD_IP_GFX) { ac_emit_cp_write_data_head(cs->b, engine_sel, V_370_MEM, va, count, predicating); } else if (cs->hw_ip == AMD_IP_SDMA) { - radv_sdma_emit_write_data_head(cs, va, count); + ac_emit_sdma_write_data_head(cs->b, va, count); } else { UNREACHABLE("unsupported queue family"); } diff --git a/src/amd/vulkan/radv_query.c b/src/amd/vulkan/radv_query.c index d20cc8aaa09..7d1413db7d9 100644 --- a/src/amd/vulkan/radv_query.c +++ b/src/amd/vulkan/radv_query.c @@ -2747,7 +2747,7 @@ radv_CmdWriteTimestamp2(VkCommandBuffer commandBuffer, VkPipelineStageFlags2 sta for (unsigned i = 0; i < num_queries; ++i, query_va += pool->stride) { radeon_check_space(device->ws, cs->b, 3); - radv_sdma_emit_write_timestamp(cs, query_va); + ac_emit_sdma_write_timestamp(cs->b, query_va); } return; } diff --git a/src/amd/vulkan/radv_sdma.c b/src/amd/vulkan/radv_sdma.c index 64e25ba28c9..4de933c4e9f 100644 --- a/src/amd/vulkan/radv_sdma.c +++ b/src/amd/vulkan/radv_sdma.c @@ -13,6 +13,7 @@ #include "radv_cs.h" #include "radv_formats.h" +#include "ac_cmdbuf_sdma.h" #include "ac_formats.h" struct radv_sdma_chunked_copy_info { @@ -348,56 +349,8 @@ radv_sdma_get_surf(const struct radv_device *const device, const struct radv_ima void radv_sdma_emit_nop(const struct radv_device *device, struct radv_cmd_stream *cs) { - /* SDMA NOP acts as a fence command and causes the SDMA engine to wait for pending copy operations. */ radeon_check_space(device->ws, cs->b, 1); - radeon_begin(cs); - radeon_emit(SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); - radeon_end(); -} - -void -radv_sdma_emit_write_timestamp(struct radv_cmd_stream *cs, uint64_t va) -{ - radeon_begin(cs); - radeon_emit(SDMA_PACKET(SDMA_OPCODE_TIMESTAMP, SDMA_TS_SUB_OPCODE_GET_GLOBAL_TIMESTAMP, 0)); - radeon_emit(va); - radeon_emit(va >> 32); - radeon_end(); -} - -void -radv_sdma_emit_fence(struct radv_cmd_stream *cs, uint64_t va, uint32_t fence) -{ - radeon_begin(cs); - radeon_emit(SDMA_PACKET(SDMA_OPCODE_FENCE, 0, SDMA_FENCE_MTYPE_UC)); - radeon_emit(va); - radeon_emit(va >> 32); - radeon_emit(fence); - radeon_end(); -} - -void -radv_sdma_emit_wait_mem(struct radv_cmd_stream *cs, uint32_t op, uint64_t va, uint32_t ref, uint32_t mask) -{ - radeon_begin(cs); - radeon_emit(SDMA_PACKET(SDMA_OPCODE_POLL_REGMEM, 0, 0) | op << 28 | SDMA_POLL_MEM); - radeon_emit(va); - radeon_emit(va >> 32); - radeon_emit(ref); - radeon_emit(mask); - radeon_emit(SDMA_POLL_INTERVAL_160_CLK | SDMA_POLL_RETRY_INDEFINITELY << 16); - radeon_end(); -} - -void -radv_sdma_emit_write_data_head(struct radv_cmd_stream *cs, uint64_t va, uint32_t count) -{ - radeon_begin(cs); - radeon_emit(SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0)); - radeon_emit(va); - radeon_emit(va >> 32); - radeon_emit(count - 1); - radeon_end(); + ac_emit_sdma_nop(cs->b); } void diff --git a/src/amd/vulkan/radv_sdma.h b/src/amd/vulkan/radv_sdma.h index 7d10fb70548..62818ee2ad2 100644 --- a/src/amd/vulkan/radv_sdma.h +++ b/src/amd/vulkan/radv_sdma.h @@ -82,14 +82,6 @@ void radv_sdma_fill_memory(const struct radv_device *device, struct radv_cmd_str void radv_sdma_emit_nop(const struct radv_device *device, struct radv_cmd_stream *cs); -void radv_sdma_emit_write_timestamp(struct radv_cmd_stream *cs, uint64_t va); - -void radv_sdma_emit_fence(struct radv_cmd_stream *cs, uint64_t va, uint32_t fence); - -void radv_sdma_emit_wait_mem(struct radv_cmd_stream *cs, uint32_t op, uint64_t va, uint32_t ref, uint32_t mask); - -void radv_sdma_emit_write_data_head(struct radv_cmd_stream *cs, uint64_t va, uint32_t count); - #ifdef __cplusplus } #endif