r600/sfn: Add GDS instructions

Signed-off-by: Gert Wollny <gert.wollny@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/merge_requests/3225>

src/gallium/drivers/r600/meson.build

@@ -131,6 +131,8 @@ files_r600 = files(
   'sfn/sfn_instruction_export.h',
   'sfn/sfn_instruction_fetch.cpp',
   'sfn/sfn_instruction_fetch.h',
+  'sfn/sfn_instruction_gds.cpp',
+  'sfn/sfn_instruction_gds.h',
   'sfn/sfn_instruction_misc.cpp',
   'sfn/sfn_instruction_misc.h',
   'sfn/sfn_instruction_tex.cpp',

src/gallium/drivers/r600/sfn/sfn_instruction_gds.cpp

@@ -0,0 +1,145 @@
+/* -*- mesa-c++  -*-
+ *
+ * Copyright (c) 2019 Collabora LTD
+ *
+ * Author: Gert Wollny <gert.wollny@collabora.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * on the rights to use, copy, modify, merge, publish, distribute, sub
+ * license, and/or sell copies of the Software, and to permit persons to whom
+ * the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "sfn_instruction_gds.h"
+#include "sfn_liverange.h"
+
+namespace  r600 {
+
+GDSInstr::GDSInstr(ESDOp op, const GPRVector& dest,  const PValue& value,
+                   const PValue& value2, const PValue& uav_id, int uav_base):
+   Instruction(gds),
+   m_op(op),
+   m_src(value),
+   m_src2(value2),
+   m_dest(dest),
+   m_dest_swizzle({PIPE_SWIZZLE_X,7,7,7}),
+   m_src_swizzle({PIPE_SWIZZLE_0, PIPE_SWIZZLE_X, 7}),
+   m_buffer_index_mode(bim_none),
+   m_uav_id(uav_id),
+   m_uav_base(uav_base),
+   m_flags(0)
+{
+   add_remappable_src_value(&m_src);
+   add_remappable_src_value(&m_uav_id);
+   add_remappable_dst_value(&m_dest);
+}
+
+GDSInstr::GDSInstr(ESDOp op, const GPRVector& dest,  const PValue& value,
+                   const PValue& uav_id, int uav_base):
+   GDSInstr(op, dest,  value, PValue(), uav_id, uav_base)
+{
+}
+
+GDSInstr::GDSInstr(ESDOp op, const GPRVector& dest,
+                   const PValue& uav_id, int uav_base):
+   GDSInstr(op, dest,  PValue(), PValue(), uav_id, uav_base)
+{
+   m_src_swizzle[1] = PIPE_SWIZZLE_1;
+}
+
+bool GDSInstr::is_equal_to(UNUSED const Instruction& lhs) const
+{
+   return false;
+}
+
+void GDSInstr::do_print(std::ostream& os) const
+{
+   const char *swz = "xyzw01?_";
+   os << lds_ops.at(m_op).name << " R" << m_dest.sel() << ".";
+   for (int i = 0; i < 4; ++i) {
+      os << swz[m_dest_swizzle[i]];
+   }
+   if (m_src)
+      os << " " << *m_src;
+
+   os << " UAV:" << *m_uav_id;
+}
+
+RatInstruction::RatInstruction(ECFOpCode cf_opcode, ERatOp rat_op,
+                               const GPRVector& data, const GPRVector& index,
+                               int rat_id, const PValue& rat_id_offset,
+                               int burst_count, int comp_mask, int element_size, bool ack):
+   Instruction(rat),
+   m_cf_opcode(cf_opcode),
+   m_rat_op(rat_op),
+   m_data(data),
+   m_index(index),
+   m_rat_id(rat_id),
+   m_rat_id_offset(rat_id_offset),
+   m_burst_count(burst_count),
+   m_comp_mask(comp_mask),
+   m_element_size(element_size),
+   m_need_ack(ack)
+{
+   add_remappable_src_value(&m_data);
+   add_remappable_src_value(&m_rat_id_offset);
+   add_remappable_src_value(&m_index);
+}
+
+bool RatInstruction::is_equal_to(UNUSED const Instruction& lhs) const
+{
+   return false;
+}
+
+void RatInstruction::do_print(std::ostream& os) const
+{
+   os << "MEM_RAT RAT(" << m_rat_id;
+   if (m_rat_id_offset)
+      os << "+" << *m_rat_id_offset;
+   os << ") @" << m_index << " OP:" << m_rat_op << " " << m_data;
+   os << " BC:" << m_burst_count
+      << " MASK:" << m_comp_mask
+      << " ES:" << m_element_size;
+   if (m_need_ack)
+      os << " ACK";
+}
+
+RatInstruction::ERatOp RatInstruction::opcode(nir_intrinsic_op opcode)
+{
+   switch (opcode) {
+   case nir_intrinsic_ssbo_atomic_add:
+      return ADD_RTN;
+   case nir_intrinsic_ssbo_atomic_and:
+      return AND_RTN;
+   case nir_intrinsic_ssbo_atomic_exchange:
+      return XCHG_RTN;
+   case nir_intrinsic_ssbo_atomic_umax:
+      return MAX_UINT_RTN;
+   case nir_intrinsic_ssbo_atomic_umin:
+      return MIN_UINT_RTN;
+   case nir_intrinsic_ssbo_atomic_imax:
+      return MAX_INT_RTN;
+   case nir_intrinsic_ssbo_atomic_imin:
+      return MIN_INT_RTN;
+   case nir_intrinsic_ssbo_atomic_xor:
+      return XOR_RTN;
+   default:
+      return UNSUPPORTED;
+   }
+}
+
+}

src/gallium/drivers/r600/sfn/sfn_instruction_gds.h

@@ -0,0 +1,195 @@
+/* -*- mesa-c++  -*-
+ *
+ * Copyright (c) 2018-2019 Collabora LTD
+ *
+ * Author: Gert Wollny <gert.wollny@collabora.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * on the rights to use, copy, modify, merge, publish, distribute, sub
+ * license, and/or sell copies of the Software, and to permit persons to whom
+ * the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef SFN_GDSINSTR_H
+#define SFN_GDSINSTR_H
+
+#include "sfn_instruction_base.h"
+
+#include <bitset>
+
+namespace r600 {
+
+class GDSInstr : public Instruction
+{
+public:
+   GDSInstr(ESDOp op, const GPRVector& dest,  const PValue& value,
+            const PValue &uav_id, int uav_base);
+   GDSInstr(ESDOp op, const GPRVector& dest,  const PValue& value,
+            const PValue& value2, const PValue &uav_id, int uav_base);
+   GDSInstr(ESDOp op, const GPRVector& dest,  const PValue &uav_id, int uav_base);
+
+   ESDOp op() const {return m_op;}
+
+   int src_sel() const {
+      if (!m_src)
+         return 0;
+
+      assert(m_src->type() == Value::gpr);
+      return m_src->sel();
+   }
+
+   int src2_chan() const {
+      if (!m_src2)
+         return 0;
+
+      assert(m_src->type() == Value::gpr);
+      return m_src->chan();
+   }
+
+   int src_swizzle(int idx) const {assert(idx < 3); return m_src_swizzle[idx];}
+
+   int dest_sel() const {
+      return m_dest.sel();
+   }
+
+   int dest_swizzle(int i) const {
+      if (i < 4)
+         return m_dest_swizzle[i];
+      return 7;
+   }
+
+   void set_dest_swizzle(const std::array<int,4>& swz) {
+      m_dest_swizzle = swz;
+   }
+
+   PValue uav_id() const {return m_uav_id;}
+   int uav_base() const {return m_uav_base;}
+
+private:
+
+   bool is_equal_to(const Instruction& lhs) const override;
+   void do_print(std::ostream& os) const override;
+
+   ESDOp m_op;
+
+   PValue m_src;
+   PValue m_src2;
+   GPRVector m_dest;
+   std::array <int, 4> m_dest_swizzle;
+   std::array <int, 3> m_src_swizzle;
+
+   EBufferIndexMode m_buffer_index_mode;
+   PValue m_uav_id;
+   int m_uav_base;
+   std::bitset<8> m_flags;
+
+};
+
+class RatInstruction : public Instruction {
+
+public:
+   enum ERatOp {
+      NOP,
+      STORE_TYPED,
+      STORE_RAW,
+      STORE_RAW_FDENORM,
+      CMPXCHG_INT,
+      CMPXCHG_FLT,
+      CMPXCHG_FDENORM,
+      ADD,
+      SUB,
+      RSUB,
+      MIN_INT,
+      MIN_UINT,
+      MAX_INT,
+      MAX_UINT,
+      AND,
+      OR,
+      XOR,
+      MSKOR,
+      INC_UINT,
+      DEC_UINT,
+      NOP_RTN = 32,
+      XCHG_RTN = 34,
+      XCHG_FDENORM_RTN,
+      CMPXCHG_INT_RTN,
+      CMPXCHG_FLT_RTN,
+      CMPXCHG_FDENORM_RTN,
+      ADD_RTN,
+      SUB_RTN,
+      RSUB_RTN,
+      MIN_INT_RTN,
+      MIN_UINT_RTN,
+      MAX_INT_RTN,
+      MAX_UINT_RTN,
+      AND_RTN,
+      OR_RTN,
+      XOR_RTN,
+      MSKOR_RTN,
+      UINT_RTN,
+      UNSUPPORTED
+   };
+
+   RatInstruction(ECFOpCode cf_opcode, ERatOp rat_op,
+                  const GPRVector& data, const GPRVector& index,
+                  int rat_id, const PValue& rat_id_offset,
+                  int burst_count, int comp_mask, int element_size,
+                  bool ack);
+
+   PValue rat_id_offset() const { return m_rat_id_offset;}
+
+   ERatOp rat_op() const {return m_rat_op;}
+
+   int data_gpr() const {return m_data.sel();}
+   int index_gpr() const {return m_index.sel();}
+   int elm_size() const {return m_element_size;}
+
+   int comp_mask() const {return m_comp_mask;}
+
+   bool need_ack() const {return m_need_ack;}
+   int burst_count() const {return m_burst_count;}
+
+   static ERatOp opcode(nir_intrinsic_op opcode);
+
+   int data_swz(int chan) const {return m_data.chan_i(chan);}
+
+private:
+
+   bool is_equal_to(const Instruction& lhs) const override;
+   void do_print(std::ostream& os) const override;
+
+   ECFOpCode m_cf_opcode;
+   ERatOp m_rat_op;
+
+   GPRVector m_data;
+   GPRVector m_index;
+
+   int m_rat_id;
+   PValue m_rat_id_offset;
+   int m_burst_count;
+   int m_comp_mask;
+   int m_element_size;
+
+   std::bitset<8> m_flags;
+
+   bool m_need_ack;
+
+};
+
+}
+
+#endif // SFN_GDSINSTR_H

src/gallium/drivers/r600/sfn/sfn_ir_to_assembly.cpp

@@ -27,6 +27,7 @@
 #include "sfn_ir_to_assembly.h"
 #include "sfn_conditionaljumptracker.h"
 #include "sfn_callstack.h"
+#include "sfn_instruction_gds.h"
 #include "sfn_instruction_misc.h"
 #include "sfn_instruction_fetch.h"
 
@@ -61,6 +62,8 @@ private:
    bool emit_loop_continue(const LoopContInstruction& instr);
    bool emit_wait_ack(const WaitAck& instr);
    bool emit_wr_scratch(const WriteScratchInstruction& instr);
+   bool emit_gds(const GDSInstr& instr);
+   bool emit_rat(const RatInstruction& instr);
 
    bool emit_load_addr(PValue addr);
    bool emit_fs_pixel_export(const ExportInstruction & exi);
@@ -176,6 +179,10 @@ bool AssemblyFromShaderLegacyImpl::emit(const Instruction::Pointer i)
       return emit_wait_ack(static_cast<const WaitAck&>(*i));
    case Instruction::mem_wr_scratch:
       return emit_wr_scratch(static_cast<const WriteScratchInstruction&>(*i));
+   case Instruction::gds:
+      return emit_gds(static_cast<const GDSInstr&>(*i));
+   case Instruction::rat:
+      return emit_rat(static_cast<const RatInstruction&>(*i));
    default:
       return false;
    }
@@ -837,6 +844,161 @@ bool AssemblyFromShaderLegacyImpl::emit_wr_scratch(const WriteScratchInstruction
 
 extern const std::map<ESDOp, int> ds_opcode_map;
 
+bool AssemblyFromShaderLegacyImpl::emit_gds(const GDSInstr& instr)
+{
+   struct r600_bytecode_gds gds;
+
+   int uav_idx = -1;
+   auto addr = instr.uav_id();
+   if (addr->type() != Value::literal) {
+      if (!m_bc->index_loaded[1] || m_loop_nesting ||
+          m_bc->index_reg[1] != addr->sel()) {
+         struct r600_bytecode_alu alu;
+
+         memset(&alu, 0, sizeof(alu));
+         alu.op = opcode_map.at(op2_lshr_int);
+         alu.dst.sel = addr->sel();
+         alu.dst.chan = addr->chan();
+         alu.src[0].sel = addr->sel();
+         alu.src[0].chan = addr->chan();
+         alu.src[1].sel = ALU_SRC_LITERAL;
+         alu.src[1].value = 2;
+         alu.last = 1;
+         alu.dst.write = 1;
+         int r = r600_bytecode_add_alu(m_bc, &alu);
+         if (r)
+            return false;
+
+         memset(&alu, 0, sizeof(alu));
+         alu.op = opcode_map.at(op1_mova_int);
+         alu.dst.chan = 0;
+         alu.src[0].sel = addr->sel();
+         alu.src[0].chan = addr->chan();
+         alu.last = 1;
+         r = r600_bytecode_add_alu(m_bc, &alu);
+         if (r)
+            return false;
+
+         m_bc->ar_loaded = 0;
+
+         alu.op = opcode_map.at(op1_set_cf_idx1);
+         alu.dst.chan = 0;
+         alu.src[0].sel = 0;
+         alu.src[0].chan = 0;
+         alu.last = 1;
+
+         r = r600_bytecode_add_alu(m_bc, &alu);
+         if (r)
+            return false;
+
+         m_bc->index_reg[1] = addr->sel();
+         m_bc->index_loaded[1] = true;
+      }
+   } else {
+      const LiteralValue& addr_reg = dynamic_cast<const LiteralValue&>(*addr);
+      uav_idx = addr_reg.value() >> 2;
+   }
+
+   memset(&gds, 0, sizeof(struct r600_bytecode_gds));
+
+   gds.op = ds_opcode_map.at(instr.op());
+	gds.dst_gpr = instr.dest_sel();
+	gds.uav_id = (uav_idx >= 0 ? uav_idx : 0) + instr.uav_base();
+	gds.uav_index_mode = uav_idx >= 0 ? bim_none : bim_one;
+	gds.src_gpr = instr.src_sel();
+
+   if (instr.op() == DS_OP_CMP_XCHG_RET) {
+      gds.src_sel_z = 1;
+   } else {
+      gds.src_sel_z = 7;
+   }
+
+	gds.src_sel_x = instr.src_swizzle(0);
+	gds.src_sel_y = instr.src_swizzle(1);
+
+	gds.dst_sel_x = 0;
+	gds.dst_sel_y = 7;
+	gds.dst_sel_z = 7;
+	gds.dst_sel_w = 7;
+	gds.src_gpr2 = 0;
+	gds.alloc_consume = 1; // Not Cayman
+
+   int r = r600_bytecode_add_gds(m_bc, &gds);
+	if (r)
+		return false;
+	m_bc->cf_last->vpm = 1;
+   return true;
+}
+
+
+bool AssemblyFromShaderLegacyImpl::emit_rat(const RatInstruction& instr)
+{
+   struct r600_bytecode_gds gds;
+
+   int rat_idx = -1;
+   EBufferIndexMode rat_index_mode = bim_none;
+   auto addr = instr.rat_id_offset();
+
+   if (addr) {
+      if (addr->type() != Value::literal) {
+         rat_index_mode = bim_one;
+         if (!m_bc->index_loaded[1] || m_loop_nesting ||  m_bc->index_reg[1] != addr->sel()) {
+            struct r600_bytecode_alu alu;
+
+            memset(&alu, 0, sizeof(alu));
+            alu.op = opcode_map.at(op1_mova_int);
+            alu.dst.chan = 0;
+            alu.src[0].sel = addr->sel();
+            alu.src[0].chan = addr->chan();
+            alu.last = 1;
+            int r = r600_bytecode_add_alu(m_bc, &alu);
+            if (r)
+               return false;
+
+            m_bc->ar_loaded = 0;
+
+            alu.op = opcode_map.at(op1_set_cf_idx1);
+            alu.dst.chan = 0;
+            alu.src[0].sel = 0;
+            alu.src[0].chan = 0;
+            alu.last = 1;
+
+            r = r600_bytecode_add_alu(m_bc, &alu);
+            if (r)
+               return false;
+
+            m_bc->index_reg[1] = addr->sel();
+            m_bc->index_loaded[1] = true;
+
+         }
+      } else {
+         const LiteralValue& addr_reg = dynamic_cast<const LiteralValue&>(*addr);
+         rat_idx = addr_reg.value();
+      }
+   }
+   memset(&gds, 0, sizeof(struct r600_bytecode_gds));
+
+   r600_bytecode_add_cfinst(m_bc, CF_OP_MEM_RAT);
+   auto cf = m_bc->cf_last;
+   cf->rat.id = rat_idx + m_shader->rat_base;
+   cf->rat.inst = instr.rat_op();
+   cf->rat.index_mode = rat_index_mode;
+   cf->output.type = instr.need_ack() ? 3 : 1;
+   cf->output.gpr = instr.data_gpr();
+   cf->output.index_gpr = instr.index_gpr();
+   cf->output.comp_mask = instr.comp_mask();
+   cf->output.burst_count = instr.burst_count();
+   cf->output.swizzle_x = instr.data_swz(0);
+   cf->output.swizzle_y = instr.data_swz(1);
+   cf->output.swizzle_z = instr.data_swz(2);
+   cf->output.swizzle_w = instr.data_swz(3);
+   cf->vpm = 1;
+   cf->barrier = 1;
+   cf->mark = instr.need_ack();
+   cf->output.elem_size = instr.elm_size();
+   return true;
+}
+
 bool AssemblyFromShaderLegacyImpl::copy_dst(r600_bytecode_alu_dst& dst,
                                             const Value& d)
 {