From 2e85076b1d64747322e31ce86a5c2715bb05305d Mon Sep 17 00:00:00 2001 From: Faith Ekstrand Date: Wed, 28 May 2025 15:19:11 -0400 Subject: [PATCH] nak: Set cache ops on surface load/store ops Part-of: --- src/nouveau/compiler/nak/from_nir.rs | 19 +++++++++++++++++++ src/nouveau/compiler/nak/ir.rs | 2 ++ src/nouveau/compiler/nak/sm20.rs | 4 ++-- src/nouveau/compiler/nak/sm32.rs | 8 ++++---- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/nouveau/compiler/nak/from_nir.rs b/src/nouveau/compiler/nak/from_nir.rs index c44ab2857ad..acf0b8e6398 100644 --- a/src/nouveau/compiler/nak/from_nir.rs +++ b/src/nouveau/compiler/nak/from_nir.rs @@ -2672,6 +2672,17 @@ impl<'a> ShaderFromNir<'a> { _ => panic!("Invalid suldga flags"), }; + let mem_order = if (intrin.access() & ACCESS_CAN_REORDER) != 0 { + MemOrder::Constant + } else { + MemOrder::Strong(MemScope::GPU) + }; + let cache_op = LdCacheOp::select( + MemSpace::Global(MemAddrType::A64), + mem_order, + self.get_eviction_priority(intrin.access()), + ); + let dst = b.alloc_ssa_vec(RegFile::GPR, comps); b.push_op(OpSuLdGa { dst: dst.clone().into(), @@ -2680,6 +2691,7 @@ impl<'a> ShaderFromNir<'a> { out_of_bounds, mem_type, offset_mode, + cache_op, }); self.set_dst(&intrin.def, dst); } @@ -2775,6 +2787,12 @@ impl<'a> ShaderFromNir<'a> { _ => panic!("Invalid sustga flags"), }; + let cache_op = StCacheOp::select( + MemSpace::Global(MemAddrType::A64), + MemOrder::Strong(MemScope::GPU), + self.get_eviction_priority(intrin.access()), + ); + b.push_op(OpSuStGa { addr, format, @@ -2782,6 +2800,7 @@ impl<'a> ShaderFromNir<'a> { out_of_bounds, image_access, offset_mode, + cache_op, }); } nir_intrinsic_bindless_image_store => { diff --git a/src/nouveau/compiler/nak/ir.rs b/src/nouveau/compiler/nak/ir.rs index 25d9bb9a4b7..d6990fe9e2c 100644 --- a/src/nouveau/compiler/nak/ir.rs +++ b/src/nouveau/compiler/nak/ir.rs @@ -5848,6 +5848,7 @@ pub struct OpSuLdGa { pub mem_type: MemType, pub offset_mode: SuGaOffsetMode, + pub cache_op: LdCacheOp, /// Format for the loaded data, passed directly from the descriptor. #[src_type(GPR)] @@ -5886,6 +5887,7 @@ impl_display_for_op!(OpSuLdGa); pub struct OpSuStGa { pub image_access: ImageAccess, pub offset_mode: SuGaOffsetMode, + pub cache_op: StCacheOp, #[src_type(GPR)] pub format: Src, diff --git a/src/nouveau/compiler/nak/sm20.rs b/src/nouveau/compiler/nak/sm20.rs index 40ad3202e36..6d58d1bc3e4 100644 --- a/src/nouveau/compiler/nak/sm20.rs +++ b/src/nouveau/compiler/nak/sm20.rs @@ -2212,7 +2212,7 @@ impl SM20Op for OpSuLdGa { e.set_opcode(SM20Unit::Mem, 0x35); e.set_mem_type(5..8, self.mem_type); - e.set_field(8..10, 0_u8); // 0: .ca, 1: none, 2: .cs, 3: .cv + e.set_ld_cache_op(8..10, self.cache_op); e.set_dst(14..20, &self.dst); e.set_reg_src(20..26, &self.addr); @@ -2258,7 +2258,7 @@ impl SM20Op for OpSuStGa { e.set_field(54..58, channel_mask.to_bits()); } } - e.set_field(8..10, 0_u8); // 0: .wb, 1: none, 2: .cs, 3: .wt + e.set_st_cache_op(8..10, self.cache_op); e.set_reg_src(14..20, &self.data); e.set_reg_src(20..26, &self.addr); diff --git a/src/nouveau/compiler/nak/sm32.rs b/src/nouveau/compiler/nak/sm32.rs index c85fc3eed9c..8dab38aaf1d 100644 --- a/src/nouveau/compiler/nak/sm32.rs +++ b/src/nouveau/compiler/nak/sm32.rs @@ -2404,14 +2404,14 @@ impl SM32Op for OpSuLdGa { e.set_opcode(0x300, 2); e.set_mem_type(56..59, self.mem_type); - // 54..56 cache hints (.ca, .cg, .cs, .cv) + e.set_ld_cache_op(54..56, self.cache_op); e.set_src_cbuf(23..42, &cb); } SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x798, 2); e.set_mem_type(33..36, self.mem_type); - // 31..33 cache hints (.ca, .cg, .cs, .cv) + e.set_ld_cache_op(31..33, self.cache_op); e.set_reg_src(23..31, &self.format); } _ => panic!("Unhandled format src type"), @@ -2459,7 +2459,7 @@ impl SM32Op for OpSuStGa { e.set_su_ga_offset_mode(8..10, self.offset_mode); e.set_src_cbuf(23..42, &cb); - // 54..56 cache hints (.wb, .cg, .cs, .wt) + e.set_st_cache_op(54..56, self.cache_op); } SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x79c, 2); @@ -2481,7 +2481,7 @@ impl SM32Op for OpSuStGa { }; e.set_su_ga_offset_mode(29..31, self.offset_mode); - // 31..33 cache hints (.wb, ??, .cs, .wt) + e.set_st_cache_op(31..33, self.cache_op); } _ => panic!("Unhandled format src type"), }