intel/mi_builder: Add load/store_offest on GFX 12.5+

Reviewed-by: Lionel Landwerlin <lionel.g.landwerlin@intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/9445>

src/intel/common/mi_builder.h

@@ -1180,4 +1180,63 @@ _mi_resolve_address_token(struct mi_builder *b,
 
 #endif /* MI_BUILDER_CAN_WRITE_BATCH */
 
+#if GEN_VERSIONx10 >= 125
+
+/*
+ * Indirect load/store.  Only available on GFX 12.5+
+ */
+
+MUST_CHECK static inline struct mi_value
+mi_load_mem64_offset(struct mi_builder *b,
+                     __gen_address_type addr, struct mi_value offset)
+{
+   uint64_t addr_u64 = __gen_combine_address(b->user_data, NULL, addr, 0);
+   struct mi_value addr_val = mi_imm(addr_u64);
+
+   struct mi_value dst = mi_new_gpr(b);
+
+   uint32_t dw[5];
+   dw[0] = _mi_math_load_src(b, MI_ALU_SRCA, &addr_val);
+   dw[1] = _mi_math_load_src(b, MI_ALU_SRCB, &offset);
+   dw[2] = _mi_pack_alu(MI_ALU_ADD, 0, 0);
+   dw[3] = _mi_pack_alu(MI_ALU_LOADIND, _mi_value_as_gpr(dst), MI_ALU_ACCU);
+   dw[4] = _mi_pack_alu(MI_ALU_FENCE_RD, 0, 0);
+   _mi_builder_push_math(b, dw, 5);
+
+   mi_value_unref(b, addr_val);
+   mi_value_unref(b, offset);
+
+   return dst;
+}
+
+static inline void
+mi_store_mem64_offset(struct mi_builder *b,
+                          __gen_address_type addr, struct mi_value offset,
+                          struct mi_value data)
+{
+   uint64_t addr_u64 = __gen_combine_address(b->user_data, NULL, addr, 0);
+   struct mi_value addr_val = mi_imm(addr_u64);
+
+   data = mi_value_to_gpr(b, mi_resolve_invert(b, data));
+
+   uint32_t dw[5];
+   dw[0] = _mi_math_load_src(b, MI_ALU_SRCA, &addr_val);
+   dw[1] = _mi_math_load_src(b, MI_ALU_SRCB, &offset);
+   dw[2] = _mi_pack_alu(MI_ALU_ADD, 0, 0);
+   dw[3] = _mi_pack_alu(MI_ALU_STOREIND, MI_ALU_ACCU, _mi_value_as_gpr(data));
+   dw[4] = _mi_pack_alu(MI_ALU_FENCE_WR, 0, 0);
+   _mi_builder_push_math(b, dw, 5);
+
+   mi_value_unref(b, addr_val);
+   mi_value_unref(b, offset);
+   mi_value_unref(b, data);
+
+   /* This is the only math case which has side-effects outside of regular
+    * registers to flush math afterwards so we don't confuse anyone.
+    */
+   mi_builder_flush_math(b);
+}
+
+#endif /* GEN_VERSIONx10 >= 125 */
+
 #endif /* MI_BUILDER_H */

src/intel/common/tests/mi_builder_test.cpp

@@ -896,3 +896,66 @@ TEST_F(mi_builder_test, store_if)
 }
 
 #endif /* GEN_GEN >= 8 || GEN_IS_HASWELL */
+
+#if GEN_VERSIONx10 >= 125
+
+/*
+ * Indirect load/store tests.  Only available on GFX 12.5+
+ */
+
+TEST_F(mi_builder_test, load_mem64_offset)
+{
+   uint64_t values[8] = {
+      0x0123456789abcdef,
+      0xdeadbeefac0ffee2,
+      (uint64_t)-1,
+      1,
+      0,
+      1049571,
+      (uint64_t)-240058,
+      20204184,
+   };
+   memcpy(input, values, sizeof(values));
+
+   uint32_t offsets[8] = { 0, 40, 24, 48, 56, 8, 32, 16 };
+   memcpy(input + 64, offsets, sizeof(offsets));
+
+   for (unsigned i = 0; i < ARRAY_SIZE(offsets); i++) {
+      mi_store(&b, out_mem64(i * 8),
+               mi_load_mem64_offset(&b, in_addr(0), in_mem32(i * 4 + 64)));
+   }
+
+   submit_batch();
+
+   for (unsigned i = 0; i < ARRAY_SIZE(offsets); i++)
+      EXPECT_EQ(*(uint64_t *)(output + i * 8), values[offsets[i] / 8]);
+}
+
+TEST_F(mi_builder_test, store_mem64_offset)
+{
+   uint64_t values[8] = {
+      0x0123456789abcdef,
+      0xdeadbeefac0ffee2,
+      (uint64_t)-1,
+      1,
+      0,
+      1049571,
+      (uint64_t)-240058,
+      20204184,
+   };
+   memcpy(input, values, sizeof(values));
+
+   uint32_t offsets[8] = { 0, 40, 24, 48, 56, 8, 32, 16 };
+   memcpy(input + 64, offsets, sizeof(offsets));
+
+   for (unsigned i = 0; i < ARRAY_SIZE(offsets); i++) {
+      mi_store_mem64_offset(&b, out_addr(0), in_mem32(i * 4 + 64),
+                                in_mem64(i * 8));
+   }
+
+   submit_batch();
+
+   for (unsigned i = 0; i < ARRAY_SIZE(offsets); i++)
+      EXPECT_EQ(*(uint64_t *)(output + offsets[i]), values[i]);
+}
+#endif /* GEN_VERSIONx10 >= 125 */

src/intel/genxml/gen125.xml

@@ -650,10 +650,13 @@
     </field>
     <field name="ALU Opcode" start="20" end="31" type="uint" prefix="MI_ALU">
       <value name="NOOP" value="0x000"/>
+      <value name="FENCE_RD" value="0x001"/>
+      <value name="FENCE_WR" value="0x002"/>
       <value name="LOAD" value="0x080"/>
       <value name="LOADINV" value="0x480"/>
       <value name="LOAD0" value="0x081"/>
       <value name="LOAD1" value="0x481"/>
+      <value name="LOADIND" value="0x082"/>
       <value name="ADD" value="0x100"/>
       <value name="SUB" value="0x101"/>
       <value name="AND" value="0x102"/>
@@ -664,6 +667,7 @@
       <value name="SAR" value="0x107"/>
       <value name="STORE" value="0x180"/>
       <value name="STOREINV" value="0x580"/>
+      <value name="STOREIND" value="0x181"/>
     </field>
   </struct>