nak: Set cache ops on surface load/store ops

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/35217>

src/nouveau/compiler/nak/from_nir.rs

@@ -2672,6 +2672,17 @@ impl<'a> ShaderFromNir<'a> {
                     _ => panic!("Invalid suldga flags"),
                 };
 
+                let mem_order = if (intrin.access() & ACCESS_CAN_REORDER) != 0 {
+                    MemOrder::Constant
+                } else {
+                    MemOrder::Strong(MemScope::GPU)
+                };
+                let cache_op = LdCacheOp::select(
+                    MemSpace::Global(MemAddrType::A64),
+                    mem_order,
+                    self.get_eviction_priority(intrin.access()),
+                );
+
                 let dst = b.alloc_ssa_vec(RegFile::GPR, comps);
                 b.push_op(OpSuLdGa {
                     dst: dst.clone().into(),
@@ -2680,6 +2691,7 @@ impl<'a> ShaderFromNir<'a> {
                     out_of_bounds,
                     mem_type,
                     offset_mode,
+                    cache_op,
                 });
                 self.set_dst(&intrin.def, dst);
             }
@@ -2775,6 +2787,12 @@ impl<'a> ShaderFromNir<'a> {
                     _ => panic!("Invalid sustga flags"),
                 };
 
+                let cache_op = StCacheOp::select(
+                    MemSpace::Global(MemAddrType::A64),
+                    MemOrder::Strong(MemScope::GPU),
+                    self.get_eviction_priority(intrin.access()),
+                );
+
                 b.push_op(OpSuStGa {
                     addr,
                     format,
@@ -2782,6 +2800,7 @@ impl<'a> ShaderFromNir<'a> {
                     out_of_bounds,
                     image_access,
                     offset_mode,
+                    cache_op,
                 });
             }
             nir_intrinsic_bindless_image_store => {

src/nouveau/compiler/nak/ir.rs

@@ -5848,6 +5848,7 @@ pub struct OpSuLdGa {
 
     pub mem_type: MemType,
     pub offset_mode: SuGaOffsetMode,
+    pub cache_op: LdCacheOp,
 
     /// Format for the loaded data, passed directly from the descriptor.
     #[src_type(GPR)]
@@ -5886,6 +5887,7 @@ impl_display_for_op!(OpSuLdGa);
 pub struct OpSuStGa {
     pub image_access: ImageAccess,
     pub offset_mode: SuGaOffsetMode,
+    pub cache_op: StCacheOp,
 
     #[src_type(GPR)]
     pub format: Src,

src/nouveau/compiler/nak/sm20.rs

@@ -2212,7 +2212,7 @@ impl SM20Op for OpSuLdGa {
 
         e.set_opcode(SM20Unit::Mem, 0x35);
         e.set_mem_type(5..8, self.mem_type);
-        e.set_field(8..10, 0_u8); // 0: .ca, 1: none, 2: .cs, 3: .cv
+        e.set_ld_cache_op(8..10, self.cache_op);
         e.set_dst(14..20, &self.dst);
         e.set_reg_src(20..26, &self.addr);
 
@@ -2258,7 +2258,7 @@ impl SM20Op for OpSuStGa {
                 e.set_field(54..58, channel_mask.to_bits());
             }
         }
-        e.set_field(8..10, 0_u8); // 0: .wb, 1: none, 2: .cs, 3: .wt
+        e.set_st_cache_op(8..10, self.cache_op);
         e.set_reg_src(14..20, &self.data);
         e.set_reg_src(20..26, &self.addr);
 

src/nouveau/compiler/nak/sm32.rs

@@ -2404,14 +2404,14 @@ impl SM32Op for OpSuLdGa {
                 e.set_opcode(0x300, 2);
                 e.set_mem_type(56..59, self.mem_type);
 
-                // 54..56 cache hints (.ca, .cg, .cs, .cv)
+                e.set_ld_cache_op(54..56, self.cache_op);
                 e.set_src_cbuf(23..42, &cb);
             }
             SrcRef::Zero | SrcRef::Reg(_) => {
                 e.set_opcode(0x798, 2);
                 e.set_mem_type(33..36, self.mem_type);
 
-                // 31..33 cache hints (.ca, .cg, .cs, .cv)
+                e.set_ld_cache_op(31..33, self.cache_op);
                 e.set_reg_src(23..31, &self.format);
             }
             _ => panic!("Unhandled format src type"),
@@ -2459,7 +2459,7 @@ impl SM32Op for OpSuStGa {
 
                 e.set_su_ga_offset_mode(8..10, self.offset_mode);
                 e.set_src_cbuf(23..42, &cb);
-                // 54..56 cache hints (.wb, .cg, .cs, .wt)
+                e.set_st_cache_op(54..56, self.cache_op);
             }
             SrcRef::Zero | SrcRef::Reg(_) => {
                 e.set_opcode(0x79c, 2);
@@ -2481,7 +2481,7 @@ impl SM32Op for OpSuStGa {
                 };
 
                 e.set_su_ga_offset_mode(29..31, self.offset_mode);
-                // 31..33 cache hints (.wb, ??, .cs, .wt)
+                e.set_st_cache_op(31..33, self.cache_op);
             }
             _ => panic!("Unhandled format src type"),
         }