mirror of
https://gitlab.freedesktop.org/mesa/mesa.git
synced 2026-05-18 09:28:06 +02:00
1f754b7aae
This allows it to be built independently of the gallium driver. Acked-by: Karol Herbst <kherbst@redhat.com> Reviewed-by: Emma Anholt <emma@anholt.net> Reviewed-by: Yusuf Khan<yusisamerican@gmail.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/16763>
2812 lines
66 KiB
C++
2812 lines
66 KiB
C++
/*
|
|
* Copyright 2012 Christoph Bumiller
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
* to deal in the Software without restriction, including without limitation
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
|
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
|
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
* OTHER DEALINGS IN THE SOFTWARE.
|
|
*/
|
|
|
|
#include "nv50_ir_target_nvc0.h"
|
|
|
|
// CodeEmitter for GK110 encoding of the Fermi/Kepler ISA.
|
|
|
|
namespace nv50_ir {
|
|
|
|
class CodeEmitterGK110 : public CodeEmitter
|
|
{
|
|
public:
|
|
CodeEmitterGK110(const TargetNVC0 *, Program::Type);
|
|
|
|
virtual bool emitInstruction(Instruction *);
|
|
virtual uint32_t getMinEncodingSize(const Instruction *) const;
|
|
virtual void prepareEmission(Function *);
|
|
|
|
private:
|
|
const TargetNVC0 *targNVC0;
|
|
|
|
Program::Type progType;
|
|
|
|
const bool writeIssueDelays;
|
|
|
|
private:
|
|
void emitForm_21(const Instruction *, uint32_t opc2, uint32_t opc1);
|
|
void emitForm_C(const Instruction *, uint32_t opc, uint8_t ctg);
|
|
void emitForm_L(const Instruction *, uint32_t opc, uint8_t ctg, Modifier, int sCount = 3);
|
|
|
|
void emitPredicate(const Instruction *);
|
|
|
|
void setCAddress14(const ValueRef&);
|
|
void setShortImmediate(const Instruction *, const int s);
|
|
void setImmediate32(const Instruction *, const int s, Modifier);
|
|
void setSUConst16(const Instruction *, const int s);
|
|
|
|
void modNegAbsF32_3b(const Instruction *, const int s);
|
|
|
|
void emitCondCode(CondCode cc, int pos, uint8_t mask);
|
|
void emitInterpMode(const Instruction *);
|
|
void emitLoadStoreType(DataType ty, const int pos);
|
|
void emitCachingMode(CacheMode c, const int pos);
|
|
void emitSUGType(DataType, const int pos);
|
|
void emitSUCachingMode(CacheMode c);
|
|
|
|
inline uint8_t getSRegEncoding(const ValueRef&);
|
|
|
|
void emitRoundMode(RoundMode, const int pos, const int rintPos);
|
|
void emitRoundModeF(RoundMode, const int pos);
|
|
void emitRoundModeI(RoundMode, const int pos);
|
|
|
|
void emitNegAbs12(const Instruction *);
|
|
|
|
void emitNOP(const Instruction *);
|
|
|
|
void emitLOAD(const Instruction *);
|
|
void emitSTORE(const Instruction *);
|
|
void emitMOV(const Instruction *);
|
|
void emitATOM(const Instruction *);
|
|
void emitCCTL(const Instruction *);
|
|
|
|
void emitINTERP(const Instruction *);
|
|
void emitAFETCH(const Instruction *);
|
|
void emitPFETCH(const Instruction *);
|
|
void emitVFETCH(const Instruction *);
|
|
void emitEXPORT(const Instruction *);
|
|
void emitOUT(const Instruction *);
|
|
|
|
void emitUADD(const Instruction *);
|
|
void emitFADD(const Instruction *);
|
|
void emitDADD(const Instruction *);
|
|
void emitIMUL(const Instruction *);
|
|
void emitFMUL(const Instruction *);
|
|
void emitDMUL(const Instruction *);
|
|
void emitIMAD(const Instruction *);
|
|
void emitISAD(const Instruction *);
|
|
void emitSHLADD(const Instruction *);
|
|
void emitFMAD(const Instruction *);
|
|
void emitDMAD(const Instruction *);
|
|
void emitMADSP(const Instruction *i);
|
|
|
|
void emitNOT(const Instruction *);
|
|
void emitLogicOp(const Instruction *, uint8_t subOp);
|
|
void emitPOPC(const Instruction *);
|
|
void emitINSBF(const Instruction *);
|
|
void emitEXTBF(const Instruction *);
|
|
void emitBFIND(const Instruction *);
|
|
void emitPERMT(const Instruction *);
|
|
void emitShift(const Instruction *);
|
|
void emitShift64(const Instruction *);
|
|
|
|
void emitSFnOp(const Instruction *, uint8_t subOp);
|
|
|
|
void emitCVT(const Instruction *);
|
|
void emitMINMAX(const Instruction *);
|
|
void emitPreOp(const Instruction *);
|
|
|
|
void emitSET(const CmpInstruction *);
|
|
void emitSLCT(const CmpInstruction *);
|
|
void emitSELP(const Instruction *);
|
|
|
|
void emitTEXBAR(const Instruction *);
|
|
void emitTEX(const TexInstruction *);
|
|
void emitTEXCSAA(const TexInstruction *);
|
|
void emitTXQ(const TexInstruction *);
|
|
|
|
void emitQUADOP(const Instruction *, uint8_t qOp, uint8_t laneMask);
|
|
|
|
void emitPIXLD(const Instruction *);
|
|
|
|
void emitBAR(const Instruction *);
|
|
void emitMEMBAR(const Instruction *);
|
|
|
|
void emitFlow(const Instruction *);
|
|
|
|
void emitSHFL(const Instruction *);
|
|
|
|
void emitVOTE(const Instruction *);
|
|
|
|
void emitSULDGB(const TexInstruction *);
|
|
void emitSUSTGx(const TexInstruction *);
|
|
void emitSUCLAMPMode(uint16_t);
|
|
void emitSUCalc(Instruction *);
|
|
|
|
void emitVSHL(const Instruction *);
|
|
void emitVectorSubOp(const Instruction *);
|
|
|
|
inline void defId(const ValueDef&, const int pos);
|
|
inline void srcId(const ValueRef&, const int pos);
|
|
inline void srcId(const ValueRef *, const int pos);
|
|
inline void srcId(const Instruction *, int s, const int pos);
|
|
|
|
inline void srcAddr32(const ValueRef&, const int pos); // address / 4
|
|
|
|
inline bool isLIMM(const ValueRef&, DataType ty, bool mod = false);
|
|
};
|
|
|
|
#define GK110_GPR_ZERO 255
|
|
|
|
#define NEG_(b, s) \
|
|
if (i->src(s).mod.neg()) code[(0x##b) / 32] |= 1 << ((0x##b) % 32)
|
|
#define ABS_(b, s) \
|
|
if (i->src(s).mod.abs()) code[(0x##b) / 32] |= 1 << ((0x##b) % 32)
|
|
|
|
#define NOT_(b, s) if (i->src(s).mod & Modifier(NV50_IR_MOD_NOT)) \
|
|
code[(0x##b) / 32] |= 1 << ((0x##b) % 32)
|
|
|
|
#define FTZ_(b) if (i->ftz) code[(0x##b) / 32] |= 1 << ((0x##b) % 32)
|
|
#define DNZ_(b) if (i->dnz) code[(0x##b) / 32] |= 1 << ((0x##b) % 32)
|
|
|
|
#define SAT_(b) if (i->saturate) code[(0x##b) / 32] |= 1 << ((0x##b) % 32)
|
|
|
|
#define RND_(b, t) emitRoundMode##t(i->rnd, 0x##b)
|
|
|
|
#define SDATA(a) ((a).rep()->reg.data)
|
|
#define DDATA(a) ((a).rep()->reg.data)
|
|
|
|
void CodeEmitterGK110::srcId(const ValueRef& src, const int pos)
|
|
{
|
|
code[pos / 32] |= (src.get() ? SDATA(src).id : GK110_GPR_ZERO) << (pos % 32);
|
|
}
|
|
|
|
void CodeEmitterGK110::srcId(const ValueRef *src, const int pos)
|
|
{
|
|
code[pos / 32] |= (src ? SDATA(*src).id : GK110_GPR_ZERO) << (pos % 32);
|
|
}
|
|
|
|
void CodeEmitterGK110::srcId(const Instruction *insn, int s, int pos)
|
|
{
|
|
int r = insn->srcExists(s) ? SDATA(insn->src(s)).id : GK110_GPR_ZERO;
|
|
code[pos / 32] |= r << (pos % 32);
|
|
}
|
|
|
|
void CodeEmitterGK110::srcAddr32(const ValueRef& src, const int pos)
|
|
{
|
|
code[pos / 32] |= (SDATA(src).offset >> 2) << (pos % 32);
|
|
}
|
|
|
|
void CodeEmitterGK110::defId(const ValueDef& def, const int pos)
|
|
{
|
|
code[pos / 32] |= (def.get() && def.getFile() != FILE_FLAGS ? DDATA(def).id : GK110_GPR_ZERO) << (pos % 32);
|
|
}
|
|
|
|
bool CodeEmitterGK110::isLIMM(const ValueRef& ref, DataType ty, bool mod)
|
|
{
|
|
const ImmediateValue *imm = ref.get()->asImm();
|
|
|
|
if (ty == TYPE_F32)
|
|
return imm && imm->reg.data.u32 & 0xfff;
|
|
else
|
|
return imm && (imm->reg.data.s32 > 0x7ffff ||
|
|
imm->reg.data.s32 < -0x80000);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitRoundMode(RoundMode rnd, const int pos, const int rintPos)
|
|
{
|
|
bool rint = false;
|
|
uint8_t n;
|
|
|
|
switch (rnd) {
|
|
case ROUND_MI: rint = true; FALLTHROUGH; case ROUND_M: n = 1; break;
|
|
case ROUND_PI: rint = true; FALLTHROUGH; case ROUND_P: n = 2; break;
|
|
case ROUND_ZI: rint = true; FALLTHROUGH; case ROUND_Z: n = 3; break;
|
|
default:
|
|
rint = rnd == ROUND_NI;
|
|
n = 0;
|
|
assert(rnd == ROUND_N || rnd == ROUND_NI);
|
|
break;
|
|
}
|
|
code[pos / 32] |= n << (pos % 32);
|
|
if (rint && rintPos >= 0)
|
|
code[rintPos / 32] |= 1 << (rintPos % 32);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitRoundModeF(RoundMode rnd, const int pos)
|
|
{
|
|
uint8_t n;
|
|
|
|
switch (rnd) {
|
|
case ROUND_M: n = 1; break;
|
|
case ROUND_P: n = 2; break;
|
|
case ROUND_Z: n = 3; break;
|
|
default:
|
|
n = 0;
|
|
assert(rnd == ROUND_N);
|
|
break;
|
|
}
|
|
code[pos / 32] |= n << (pos % 32);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitRoundModeI(RoundMode rnd, const int pos)
|
|
{
|
|
uint8_t n;
|
|
|
|
switch (rnd) {
|
|
case ROUND_MI: n = 1; break;
|
|
case ROUND_PI: n = 2; break;
|
|
case ROUND_ZI: n = 3; break;
|
|
default:
|
|
n = 0;
|
|
assert(rnd == ROUND_NI);
|
|
break;
|
|
}
|
|
code[pos / 32] |= n << (pos % 32);
|
|
}
|
|
|
|
void CodeEmitterGK110::emitCondCode(CondCode cc, int pos, uint8_t mask)
|
|
{
|
|
uint8_t n;
|
|
|
|
switch (cc) {
|
|
case CC_FL: n = 0x00; break;
|
|
case CC_LT: n = 0x01; break;
|
|
case CC_EQ: n = 0x02; break;
|
|
case CC_LE: n = 0x03; break;
|
|
case CC_GT: n = 0x04; break;
|
|
case CC_NE: n = 0x05; break;
|
|
case CC_GE: n = 0x06; break;
|
|
case CC_LTU: n = 0x09; break;
|
|
case CC_EQU: n = 0x0a; break;
|
|
case CC_LEU: n = 0x0b; break;
|
|
case CC_GTU: n = 0x0c; break;
|
|
case CC_NEU: n = 0x0d; break;
|
|
case CC_GEU: n = 0x0e; break;
|
|
case CC_TR: n = 0x0f; break;
|
|
case CC_NO: n = 0x10; break;
|
|
case CC_NC: n = 0x11; break;
|
|
case CC_NS: n = 0x12; break;
|
|
case CC_NA: n = 0x13; break;
|
|
case CC_A: n = 0x14; break;
|
|
case CC_S: n = 0x15; break;
|
|
case CC_C: n = 0x16; break;
|
|
case CC_O: n = 0x17; break;
|
|
default:
|
|
n = 0;
|
|
assert(!"invalid condition code");
|
|
break;
|
|
}
|
|
code[pos / 32] |= (n & mask) << (pos % 32);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitPredicate(const Instruction *i)
|
|
{
|
|
if (i->predSrc >= 0) {
|
|
srcId(i->src(i->predSrc), 18);
|
|
if (i->cc == CC_NOT_P)
|
|
code[0] |= 8 << 18; // negate
|
|
assert(i->getPredicate()->reg.file == FILE_PREDICATE);
|
|
} else {
|
|
code[0] |= 7 << 18;
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::setCAddress14(const ValueRef& src)
|
|
{
|
|
const Storage& res = src.get()->asSym()->reg;
|
|
const int32_t addr = res.data.offset / 4;
|
|
|
|
code[0] |= (addr & 0x01ff) << 23;
|
|
code[1] |= (addr & 0x3e00) >> 9;
|
|
code[1] |= res.fileIndex << 5;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::setShortImmediate(const Instruction *i, const int s)
|
|
{
|
|
const uint32_t u32 = i->getSrc(s)->asImm()->reg.data.u32;
|
|
const uint64_t u64 = i->getSrc(s)->asImm()->reg.data.u64;
|
|
|
|
if (i->sType == TYPE_F32) {
|
|
assert(!(u32 & 0x00000fff));
|
|
code[0] |= ((u32 & 0x001ff000) >> 12) << 23;
|
|
code[1] |= ((u32 & 0x7fe00000) >> 21);
|
|
code[1] |= ((u32 & 0x80000000) >> 4);
|
|
} else
|
|
if (i->sType == TYPE_F64) {
|
|
assert(!(u64 & 0x00000fffffffffffULL));
|
|
code[0] |= ((u64 & 0x001ff00000000000ULL) >> 44) << 23;
|
|
code[1] |= ((u64 & 0x7fe0000000000000ULL) >> 53);
|
|
code[1] |= ((u64 & 0x8000000000000000ULL) >> 36);
|
|
} else {
|
|
assert((u32 & 0xfff80000) == 0 || (u32 & 0xfff80000) == 0xfff80000);
|
|
code[0] |= (u32 & 0x001ff) << 23;
|
|
code[1] |= (u32 & 0x7fe00) >> 9;
|
|
code[1] |= (u32 & 0x80000) << 8;
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::setImmediate32(const Instruction *i, const int s,
|
|
Modifier mod)
|
|
{
|
|
uint32_t u32 = i->getSrc(s)->asImm()->reg.data.u32;
|
|
|
|
if (mod) {
|
|
ImmediateValue imm(i->getSrc(s)->asImm(), i->sType);
|
|
mod.applyTo(imm);
|
|
u32 = imm.reg.data.u32;
|
|
}
|
|
|
|
code[0] |= u32 << 23;
|
|
code[1] |= u32 >> 9;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitForm_L(const Instruction *i, uint32_t opc, uint8_t ctg,
|
|
Modifier mod, int sCount)
|
|
{
|
|
code[0] = ctg;
|
|
code[1] = opc << 20;
|
|
|
|
emitPredicate(i);
|
|
|
|
defId(i->def(0), 2);
|
|
|
|
for (int s = 0; s < sCount && i->srcExists(s); ++s) {
|
|
switch (i->src(s).getFile()) {
|
|
case FILE_GPR:
|
|
srcId(i->src(s), s ? 42 : 10);
|
|
break;
|
|
case FILE_IMMEDIATE:
|
|
setImmediate32(i, s, mod);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
void
|
|
CodeEmitterGK110::emitForm_C(const Instruction *i, uint32_t opc, uint8_t ctg)
|
|
{
|
|
code[0] = ctg;
|
|
code[1] = opc << 20;
|
|
|
|
emitPredicate(i);
|
|
|
|
defId(i->def(0), 2);
|
|
|
|
switch (i->src(0).getFile()) {
|
|
case FILE_MEMORY_CONST:
|
|
code[1] |= 0x4 << 28;
|
|
setCAddress14(i->src(0));
|
|
break;
|
|
case FILE_GPR:
|
|
code[1] |= 0xc << 28;
|
|
srcId(i->src(0), 23);
|
|
break;
|
|
default:
|
|
assert(0);
|
|
break;
|
|
}
|
|
}
|
|
|
|
// 0x2 for GPR, c[] and 0x1 for short immediate
|
|
void
|
|
CodeEmitterGK110::emitForm_21(const Instruction *i, uint32_t opc2,
|
|
uint32_t opc1)
|
|
{
|
|
const bool imm = i->srcExists(1) && i->src(1).getFile() == FILE_IMMEDIATE;
|
|
|
|
int s1 = 23;
|
|
if (i->srcExists(2) && i->src(2).getFile() == FILE_MEMORY_CONST)
|
|
s1 = 42;
|
|
|
|
if (imm) {
|
|
code[0] = 0x1;
|
|
code[1] = opc1 << 20;
|
|
} else {
|
|
code[0] = 0x2;
|
|
code[1] = (0xc << 28) | (opc2 << 20);
|
|
}
|
|
|
|
emitPredicate(i);
|
|
|
|
defId(i->def(0), 2);
|
|
|
|
for (int s = 0; s < 3 && i->srcExists(s); ++s) {
|
|
switch (i->src(s).getFile()) {
|
|
case FILE_MEMORY_CONST:
|
|
code[1] &= (s == 2) ? ~(0x4 << 28) : ~(0x8 << 28);
|
|
setCAddress14(i->src(s));
|
|
break;
|
|
case FILE_IMMEDIATE:
|
|
setShortImmediate(i, s);
|
|
break;
|
|
case FILE_GPR:
|
|
srcId(i->src(s), s ? ((s == 2) ? 42 : s1) : 10);
|
|
break;
|
|
default:
|
|
if (i->op == OP_SELP) {
|
|
assert(s == 2 && i->src(s).getFile() == FILE_PREDICATE);
|
|
srcId(i->src(s), 42);
|
|
}
|
|
// ignore here, can be predicate or flags, but must not be address
|
|
break;
|
|
}
|
|
}
|
|
// 0x0 = invalid
|
|
// 0xc = rrr
|
|
// 0x8 = rrc
|
|
// 0x4 = rcr
|
|
assert(imm || (code[1] & (0xc << 28)));
|
|
}
|
|
|
|
inline void
|
|
CodeEmitterGK110::modNegAbsF32_3b(const Instruction *i, const int s)
|
|
{
|
|
if (i->src(s).mod.abs()) code[1] &= ~(1 << 27);
|
|
if (i->src(s).mod.neg()) code[1] ^= (1 << 27);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitNOP(const Instruction *i)
|
|
{
|
|
code[0] = 0x00003c02;
|
|
code[1] = 0x85800000;
|
|
|
|
if (i)
|
|
emitPredicate(i);
|
|
else
|
|
code[0] = 0x001c3c02;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitFMAD(const Instruction *i)
|
|
{
|
|
bool neg1 = (i->src(0).mod ^ i->src(1).mod).neg();
|
|
|
|
if (isLIMM(i->src(1), TYPE_F32)) {
|
|
assert(i->getDef(0)->reg.data.id == i->getSrc(2)->reg.data.id);
|
|
|
|
// last source is dst, so force 2 sources
|
|
emitForm_L(i, 0x600, 0x0, 0, 2);
|
|
|
|
if (i->flagsDef >= 0)
|
|
code[1] |= 1 << 23;
|
|
|
|
SAT_(3a);
|
|
NEG_(3c, 2);
|
|
|
|
if (neg1) {
|
|
code[1] |= 1 << 27;
|
|
}
|
|
} else {
|
|
emitForm_21(i, 0x0c0, 0x940);
|
|
|
|
NEG_(34, 2);
|
|
SAT_(35);
|
|
RND_(36, F);
|
|
|
|
if (code[0] & 0x1) {
|
|
if (neg1)
|
|
code[1] ^= 1 << 27;
|
|
} else
|
|
if (neg1) {
|
|
code[1] |= 1 << 19;
|
|
}
|
|
}
|
|
|
|
FTZ_(38);
|
|
DNZ_(39);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitDMAD(const Instruction *i)
|
|
{
|
|
assert(!i->saturate);
|
|
assert(!i->ftz);
|
|
|
|
emitForm_21(i, 0x1b8, 0xb38);
|
|
|
|
NEG_(34, 2);
|
|
RND_(36, F);
|
|
|
|
bool neg1 = (i->src(0).mod ^ i->src(1).mod).neg();
|
|
|
|
if (code[0] & 0x1) {
|
|
if (neg1)
|
|
code[1] ^= 1 << 27;
|
|
} else
|
|
if (neg1) {
|
|
code[1] |= 1 << 19;
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitMADSP(const Instruction *i)
|
|
{
|
|
emitForm_21(i, 0x140, 0xa40);
|
|
|
|
if (i->subOp == NV50_IR_SUBOP_MADSP_SD) {
|
|
code[1] |= 0x00c00000;
|
|
} else {
|
|
code[1] |= (i->subOp & 0x00f) << 19; // imadp1
|
|
code[1] |= (i->subOp & 0x0f0) << 20; // imadp2
|
|
code[1] |= (i->subOp & 0x100) << 11; // imadp3
|
|
code[1] |= (i->subOp & 0x200) << 15; // imadp3
|
|
code[1] |= (i->subOp & 0xc00) << 12; // imadp3
|
|
}
|
|
|
|
if (i->flagsDef >= 0)
|
|
code[1] |= 1 << 18;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitFMUL(const Instruction *i)
|
|
{
|
|
bool neg = (i->src(0).mod ^ i->src(1).mod).neg();
|
|
|
|
assert(i->postFactor >= -3 && i->postFactor <= 3);
|
|
|
|
if (isLIMM(i->src(1), TYPE_F32)) {
|
|
emitForm_L(i, 0x200, 0x2, Modifier(0));
|
|
|
|
FTZ_(38);
|
|
DNZ_(39);
|
|
SAT_(3a);
|
|
if (neg)
|
|
code[1] ^= 1 << 22;
|
|
|
|
assert(i->postFactor == 0);
|
|
} else {
|
|
emitForm_21(i, 0x234, 0xc34);
|
|
code[1] |= ((i->postFactor > 0) ?
|
|
(7 - i->postFactor) : (0 - i->postFactor)) << 12;
|
|
|
|
RND_(2a, F);
|
|
FTZ_(2f);
|
|
DNZ_(30);
|
|
SAT_(35);
|
|
|
|
if (code[0] & 0x1) {
|
|
if (neg)
|
|
code[1] ^= 1 << 27;
|
|
} else
|
|
if (neg) {
|
|
code[1] |= 1 << 19;
|
|
}
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitDMUL(const Instruction *i)
|
|
{
|
|
bool neg = (i->src(0).mod ^ i->src(1).mod).neg();
|
|
|
|
assert(!i->postFactor);
|
|
assert(!i->saturate);
|
|
assert(!i->ftz);
|
|
assert(!i->dnz);
|
|
|
|
emitForm_21(i, 0x240, 0xc40);
|
|
|
|
RND_(2a, F);
|
|
|
|
if (code[0] & 0x1) {
|
|
if (neg)
|
|
code[1] ^= 1 << 27;
|
|
} else
|
|
if (neg) {
|
|
code[1] |= 1 << 19;
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitIMUL(const Instruction *i)
|
|
{
|
|
assert(!i->src(0).mod.neg() && !i->src(1).mod.neg());
|
|
assert(!i->src(0).mod.abs() && !i->src(1).mod.abs());
|
|
|
|
if (isLIMM(i->src(1), TYPE_S32)) {
|
|
emitForm_L(i, 0x280, 2, Modifier(0));
|
|
|
|
if (i->subOp == NV50_IR_SUBOP_MUL_HIGH)
|
|
code[1] |= 1 << 24;
|
|
if (i->sType == TYPE_S32)
|
|
code[1] |= 3 << 25;
|
|
} else {
|
|
emitForm_21(i, 0x21c, 0xc1c);
|
|
|
|
if (i->subOp == NV50_IR_SUBOP_MUL_HIGH)
|
|
code[1] |= 1 << 10;
|
|
if (i->sType == TYPE_S32)
|
|
code[1] |= 3 << 11;
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitFADD(const Instruction *i)
|
|
{
|
|
if (isLIMM(i->src(1), TYPE_F32)) {
|
|
assert(i->rnd == ROUND_N);
|
|
assert(!i->saturate);
|
|
|
|
Modifier mod = i->src(1).mod ^
|
|
Modifier(i->op == OP_SUB ? NV50_IR_MOD_NEG : 0);
|
|
|
|
emitForm_L(i, 0x400, 0, mod);
|
|
|
|
FTZ_(3a);
|
|
NEG_(3b, 0);
|
|
ABS_(39, 0);
|
|
} else {
|
|
emitForm_21(i, 0x22c, 0xc2c);
|
|
|
|
FTZ_(2f);
|
|
RND_(2a, F);
|
|
ABS_(31, 0);
|
|
NEG_(33, 0);
|
|
SAT_(35);
|
|
|
|
if (code[0] & 0x1) {
|
|
modNegAbsF32_3b(i, 1);
|
|
if (i->op == OP_SUB) code[1] ^= 1 << 27;
|
|
} else {
|
|
ABS_(34, 1);
|
|
NEG_(30, 1);
|
|
if (i->op == OP_SUB) code[1] ^= 1 << 16;
|
|
}
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitDADD(const Instruction *i)
|
|
{
|
|
assert(!i->saturate);
|
|
assert(!i->ftz);
|
|
|
|
emitForm_21(i, 0x238, 0xc38);
|
|
RND_(2a, F);
|
|
ABS_(31, 0);
|
|
NEG_(33, 0);
|
|
if (code[0] & 0x1) {
|
|
modNegAbsF32_3b(i, 1);
|
|
if (i->op == OP_SUB) code[1] ^= 1 << 27;
|
|
} else {
|
|
NEG_(30, 1);
|
|
ABS_(34, 1);
|
|
if (i->op == OP_SUB) code[1] ^= 1 << 16;
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitUADD(const Instruction *i)
|
|
{
|
|
uint8_t addOp = (i->src(0).mod.neg() << 1) | i->src(1).mod.neg();
|
|
|
|
if (i->op == OP_SUB)
|
|
addOp ^= 1;
|
|
|
|
assert(!i->src(0).mod.abs() && !i->src(1).mod.abs());
|
|
|
|
if (isLIMM(i->src(1), TYPE_S32)) {
|
|
emitForm_L(i, 0x400, 1, Modifier((addOp & 1) ? NV50_IR_MOD_NEG : 0));
|
|
|
|
if (addOp & 2)
|
|
code[1] |= 1 << 27;
|
|
|
|
assert(i->flagsDef < 0);
|
|
assert(i->flagsSrc < 0);
|
|
|
|
SAT_(39);
|
|
} else {
|
|
emitForm_21(i, 0x208, 0xc08);
|
|
|
|
assert(addOp != 3); // would be add-plus-one
|
|
|
|
code[1] |= addOp << 19;
|
|
|
|
if (i->flagsDef >= 0)
|
|
code[1] |= 1 << 18; // write carry
|
|
if (i->flagsSrc >= 0)
|
|
code[1] |= 1 << 14; // add carry
|
|
|
|
SAT_(35);
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitIMAD(const Instruction *i)
|
|
{
|
|
uint8_t addOp =
|
|
i->src(2).mod.neg() | ((i->src(0).mod.neg() ^ i->src(1).mod.neg()) << 1);
|
|
|
|
emitForm_21(i, 0x100, 0xa00);
|
|
|
|
assert(addOp != 3);
|
|
code[1] |= addOp << 26;
|
|
|
|
if (i->sType == TYPE_S32)
|
|
code[1] |= (1 << 19) | (1 << 24);
|
|
|
|
if (i->subOp == NV50_IR_SUBOP_MUL_HIGH)
|
|
code[1] |= 1 << 25;
|
|
|
|
if (i->flagsDef >= 0) code[1] |= 1 << 18;
|
|
if (i->flagsSrc >= 0) code[1] |= 1 << 20;
|
|
|
|
SAT_(35);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitISAD(const Instruction *i)
|
|
{
|
|
assert(i->dType == TYPE_S32 || i->dType == TYPE_U32);
|
|
|
|
emitForm_21(i, 0x1f4, 0xb74);
|
|
|
|
if (i->dType == TYPE_S32)
|
|
code[1] |= 1 << 19;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitSHLADD(const Instruction *i)
|
|
{
|
|
uint8_t addOp = (i->src(0).mod.neg() << 1) | i->src(2).mod.neg();
|
|
const ImmediateValue *imm = i->src(1).get()->asImm();
|
|
assert(imm);
|
|
|
|
if (i->src(2).getFile() == FILE_IMMEDIATE) {
|
|
code[0] = 0x1;
|
|
code[1] = 0xc0c << 20;
|
|
} else {
|
|
code[0] = 0x2;
|
|
code[1] = 0x20c << 20;
|
|
}
|
|
code[1] |= addOp << 19;
|
|
|
|
emitPredicate(i);
|
|
|
|
defId(i->def(0), 2);
|
|
srcId(i->src(0), 10);
|
|
|
|
if (i->flagsDef >= 0)
|
|
code[1] |= 1 << 18;
|
|
|
|
assert(!(imm->reg.data.u32 & 0xffffffe0));
|
|
code[1] |= imm->reg.data.u32 << 10;
|
|
|
|
switch (i->src(2).getFile()) {
|
|
case FILE_GPR:
|
|
assert(code[0] & 0x2);
|
|
code[1] |= 0xc << 28;
|
|
srcId(i->src(2), 23);
|
|
break;
|
|
case FILE_MEMORY_CONST:
|
|
assert(code[0] & 0x2);
|
|
code[1] |= 0x4 << 28;
|
|
setCAddress14(i->src(2));
|
|
break;
|
|
case FILE_IMMEDIATE:
|
|
assert(code[0] & 0x1);
|
|
setShortImmediate(i, 2);
|
|
break;
|
|
default:
|
|
assert(!"bad src2 file");
|
|
break;
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitNOT(const Instruction *i)
|
|
{
|
|
code[0] = 0x0003fc02; // logop(mov2) dst, 0, not src
|
|
code[1] = 0x22003800;
|
|
|
|
emitPredicate(i);
|
|
|
|
defId(i->def(0), 2);
|
|
|
|
switch (i->src(0).getFile()) {
|
|
case FILE_GPR:
|
|
code[1] |= 0xc << 28;
|
|
srcId(i->src(0), 23);
|
|
break;
|
|
case FILE_MEMORY_CONST:
|
|
code[1] |= 0x4 << 28;
|
|
setCAddress14(i->src(0));
|
|
break;
|
|
default:
|
|
assert(0);
|
|
break;
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitLogicOp(const Instruction *i, uint8_t subOp)
|
|
{
|
|
if (i->def(0).getFile() == FILE_PREDICATE) {
|
|
code[0] = 0x00000002 | (subOp << 27);
|
|
code[1] = 0x84800000;
|
|
|
|
emitPredicate(i);
|
|
|
|
defId(i->def(0), 5);
|
|
srcId(i->src(0), 14);
|
|
if (i->src(0).mod == Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 17;
|
|
srcId(i->src(1), 32);
|
|
if (i->src(1).mod == Modifier(NV50_IR_MOD_NOT)) code[1] |= 1 << 3;
|
|
|
|
if (i->defExists(1)) {
|
|
defId(i->def(1), 2);
|
|
} else {
|
|
code[0] |= 7 << 2;
|
|
}
|
|
// (a OP b) OP c
|
|
if (i->predSrc != 2 && i->srcExists(2)) {
|
|
code[1] |= subOp << 16;
|
|
srcId(i->src(2), 42);
|
|
if (i->src(2).mod == Modifier(NV50_IR_MOD_NOT)) code[1] |= 1 << 13;
|
|
} else {
|
|
code[1] |= 7 << 10;
|
|
}
|
|
} else
|
|
if (isLIMM(i->src(1), TYPE_S32)) {
|
|
emitForm_L(i, 0x200, 0, i->src(1).mod);
|
|
code[1] |= subOp << 24;
|
|
NOT_(3a, 0);
|
|
} else {
|
|
emitForm_21(i, 0x220, 0xc20);
|
|
code[1] |= subOp << 12;
|
|
NOT_(2a, 0);
|
|
NOT_(2b, 1);
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitPOPC(const Instruction *i)
|
|
{
|
|
assert(!isLIMM(i->src(1), TYPE_S32, true));
|
|
|
|
emitForm_21(i, 0x204, 0xc04);
|
|
|
|
NOT_(2a, 0);
|
|
if (!(code[0] & 0x1))
|
|
NOT_(2b, 1);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitINSBF(const Instruction *i)
|
|
{
|
|
emitForm_21(i, 0x1f8, 0xb78);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitEXTBF(const Instruction *i)
|
|
{
|
|
emitForm_21(i, 0x600, 0xc00);
|
|
|
|
if (i->dType == TYPE_S32)
|
|
code[1] |= 0x80000;
|
|
if (i->subOp == NV50_IR_SUBOP_EXTBF_REV)
|
|
code[1] |= 0x800;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitBFIND(const Instruction *i)
|
|
{
|
|
emitForm_C(i, 0x218, 0x2);
|
|
|
|
if (i->dType == TYPE_S32)
|
|
code[1] |= 0x80000;
|
|
if (i->src(0).mod == Modifier(NV50_IR_MOD_NOT))
|
|
code[1] |= 0x800;
|
|
if (i->subOp == NV50_IR_SUBOP_BFIND_SAMT)
|
|
code[1] |= 0x1000;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitPERMT(const Instruction *i)
|
|
{
|
|
emitForm_21(i, 0x1e0, 0xb60);
|
|
|
|
code[1] |= i->subOp << 19;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitShift(const Instruction *i)
|
|
{
|
|
if (i->op == OP_SHR) {
|
|
emitForm_21(i, 0x214, 0xc14);
|
|
if (isSignedType(i->dType))
|
|
code[1] |= 1 << 19;
|
|
} else {
|
|
emitForm_21(i, 0x224, 0xc24);
|
|
}
|
|
|
|
if (i->subOp == NV50_IR_SUBOP_SHIFT_WRAP)
|
|
code[1] |= 1 << 10;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitShift64(const Instruction *i)
|
|
{
|
|
if (i->op == OP_SHR) {
|
|
emitForm_21(i, 0x27c, 0xc7c);
|
|
if (isSignedType(i->sType))
|
|
code[1] |= 0x100;
|
|
if (i->subOp & NV50_IR_SUBOP_SHIFT_HIGH)
|
|
code[1] |= 1 << 19;
|
|
} else {
|
|
emitForm_21(i, 0xdfc, 0xf7c);
|
|
}
|
|
code[1] |= 0x200;
|
|
|
|
if (i->subOp & NV50_IR_SUBOP_SHIFT_WRAP)
|
|
code[1] |= 1 << 21;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitPreOp(const Instruction *i)
|
|
{
|
|
emitForm_C(i, 0x248, 0x2);
|
|
|
|
if (i->op == OP_PREEX2)
|
|
code[1] |= 1 << 10;
|
|
|
|
NEG_(30, 0);
|
|
ABS_(34, 0);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitSFnOp(const Instruction *i, uint8_t subOp)
|
|
{
|
|
code[0] = 0x00000002 | (subOp << 23);
|
|
code[1] = 0x84000000;
|
|
|
|
emitPredicate(i);
|
|
|
|
defId(i->def(0), 2);
|
|
srcId(i->src(0), 10);
|
|
|
|
NEG_(33, 0);
|
|
ABS_(31, 0);
|
|
SAT_(35);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitMINMAX(const Instruction *i)
|
|
{
|
|
uint32_t op2, op1;
|
|
|
|
switch (i->dType) {
|
|
case TYPE_U32:
|
|
case TYPE_S32:
|
|
op2 = 0x210;
|
|
op1 = 0xc10;
|
|
break;
|
|
case TYPE_F32:
|
|
op2 = 0x230;
|
|
op1 = 0xc30;
|
|
break;
|
|
case TYPE_F64:
|
|
op2 = 0x228;
|
|
op1 = 0xc28;
|
|
break;
|
|
default:
|
|
assert(0);
|
|
op2 = 0;
|
|
op1 = 0;
|
|
break;
|
|
}
|
|
emitForm_21(i, op2, op1);
|
|
|
|
if (i->dType == TYPE_S32)
|
|
code[1] |= 1 << 19;
|
|
code[1] |= (i->op == OP_MIN) ? 0x1c00 : 0x3c00; // [!]pt
|
|
code[1] |= i->subOp << 14;
|
|
if (i->flagsDef >= 0)
|
|
code[1] |= i->subOp << 18;
|
|
|
|
FTZ_(2f);
|
|
ABS_(31, 0);
|
|
NEG_(33, 0);
|
|
if (code[0] & 0x1) {
|
|
modNegAbsF32_3b(i, 1);
|
|
} else {
|
|
ABS_(34, 1);
|
|
NEG_(30, 1);
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitCVT(const Instruction *i)
|
|
{
|
|
const bool f2f = isFloatType(i->dType) && isFloatType(i->sType);
|
|
const bool f2i = !isFloatType(i->dType) && isFloatType(i->sType);
|
|
const bool i2f = isFloatType(i->dType) && !isFloatType(i->sType);
|
|
|
|
bool sat = i->saturate;
|
|
bool abs = i->src(0).mod.abs();
|
|
bool neg = i->src(0).mod.neg();
|
|
|
|
RoundMode rnd = i->rnd;
|
|
|
|
switch (i->op) {
|
|
case OP_CEIL: rnd = f2f ? ROUND_PI : ROUND_P; break;
|
|
case OP_FLOOR: rnd = f2f ? ROUND_MI : ROUND_M; break;
|
|
case OP_TRUNC: rnd = f2f ? ROUND_ZI : ROUND_Z; break;
|
|
case OP_SAT: sat = true; break;
|
|
case OP_NEG: neg = !neg; break;
|
|
case OP_ABS: abs = true; neg = false; break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
DataType dType;
|
|
|
|
if (i->op == OP_NEG && i->dType == TYPE_U32)
|
|
dType = TYPE_S32;
|
|
else
|
|
dType = i->dType;
|
|
|
|
|
|
uint32_t op;
|
|
|
|
if (f2f) op = 0x254;
|
|
else if (f2i) op = 0x258;
|
|
else if (i2f) op = 0x25c;
|
|
else op = 0x260;
|
|
|
|
emitForm_C(i, op, 0x2);
|
|
|
|
FTZ_(2f);
|
|
if (neg) code[1] |= 1 << 16;
|
|
if (abs) code[1] |= 1 << 20;
|
|
if (sat) code[1] |= 1 << 21;
|
|
|
|
emitRoundMode(rnd, 32 + 10, f2f ? (32 + 13) : -1);
|
|
|
|
code[0] |= typeSizeofLog2(dType) << 10;
|
|
code[0] |= typeSizeofLog2(i->sType) << 12;
|
|
code[1] |= i->subOp << 12;
|
|
|
|
if (isSignedIntType(dType))
|
|
code[0] |= 0x4000;
|
|
if (isSignedIntType(i->sType))
|
|
code[0] |= 0x8000;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitSET(const CmpInstruction *i)
|
|
{
|
|
uint16_t op1, op2;
|
|
|
|
if (i->def(0).getFile() == FILE_PREDICATE) {
|
|
switch (i->sType) {
|
|
case TYPE_F32: op2 = 0x1d8; op1 = 0xb58; break;
|
|
case TYPE_F64: op2 = 0x1c0; op1 = 0xb40; break;
|
|
default:
|
|
op2 = 0x1b0;
|
|
op1 = 0xb30;
|
|
break;
|
|
}
|
|
emitForm_21(i, op2, op1);
|
|
|
|
NEG_(2e, 0);
|
|
ABS_(9, 0);
|
|
if (!(code[0] & 0x1)) {
|
|
NEG_(8, 1);
|
|
ABS_(2f, 1);
|
|
} else {
|
|
modNegAbsF32_3b(i, 1);
|
|
}
|
|
FTZ_(32);
|
|
|
|
// normal DST field is negated predicate result
|
|
code[0] = (code[0] & ~0xfc) | ((code[0] << 3) & 0xe0);
|
|
if (i->defExists(1))
|
|
defId(i->def(1), 2);
|
|
else
|
|
code[0] |= 0x1c;
|
|
} else {
|
|
switch (i->sType) {
|
|
case TYPE_F32: op2 = 0x000; op1 = 0x800; break;
|
|
case TYPE_F64: op2 = 0x080; op1 = 0x900; break;
|
|
default:
|
|
op2 = 0x1a8;
|
|
op1 = 0xb28;
|
|
break;
|
|
}
|
|
emitForm_21(i, op2, op1);
|
|
|
|
NEG_(2e, 0);
|
|
ABS_(39, 0);
|
|
if (!(code[0] & 0x1)) {
|
|
NEG_(38, 1);
|
|
ABS_(2f, 1);
|
|
} else {
|
|
modNegAbsF32_3b(i, 1);
|
|
}
|
|
FTZ_(3a);
|
|
|
|
if (i->dType == TYPE_F32) {
|
|
if (isFloatType(i->sType))
|
|
code[1] |= 1 << 23;
|
|
else
|
|
code[1] |= 1 << 15;
|
|
}
|
|
}
|
|
if (i->sType == TYPE_S32)
|
|
code[1] |= 1 << 19;
|
|
|
|
if (i->op != OP_SET) {
|
|
switch (i->op) {
|
|
case OP_SET_AND: code[1] |= 0x0 << 16; break;
|
|
case OP_SET_OR: code[1] |= 0x1 << 16; break;
|
|
case OP_SET_XOR: code[1] |= 0x2 << 16; break;
|
|
default:
|
|
assert(0);
|
|
break;
|
|
}
|
|
srcId(i->src(2), 0x2a);
|
|
} else {
|
|
code[1] |= 0x7 << 10;
|
|
}
|
|
if (i->flagsSrc >= 0)
|
|
code[1] |= 1 << 14;
|
|
emitCondCode(i->setCond,
|
|
isFloatType(i->sType) ? 0x33 : 0x34,
|
|
isFloatType(i->sType) ? 0xf : 0x7);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitSLCT(const CmpInstruction *i)
|
|
{
|
|
CondCode cc = i->setCond;
|
|
if (i->src(2).mod.neg())
|
|
cc = reverseCondCode(cc);
|
|
|
|
if (i->dType == TYPE_F32) {
|
|
emitForm_21(i, 0x1d0, 0xb50);
|
|
FTZ_(32);
|
|
emitCondCode(cc, 0x33, 0xf);
|
|
} else {
|
|
emitForm_21(i, 0x1a0, 0xb20);
|
|
emitCondCode(cc, 0x34, 0x7);
|
|
if (i->dType == TYPE_S32)
|
|
code[1] |= 1 << 19;
|
|
}
|
|
}
|
|
|
|
void
|
|
gk110_selpFlip(const FixupEntry *entry, uint32_t *code, const FixupData& data)
|
|
{
|
|
int loc = entry->loc;
|
|
bool val = false;
|
|
switch (entry->ipa) {
|
|
case 0:
|
|
val = data.force_persample_interp;
|
|
break;
|
|
case 1:
|
|
val = data.msaa;
|
|
break;
|
|
}
|
|
if (val)
|
|
code[loc + 1] |= 1 << 13;
|
|
else
|
|
code[loc + 1] &= ~(1 << 13);
|
|
}
|
|
|
|
void CodeEmitterGK110::emitSELP(const Instruction *i)
|
|
{
|
|
emitForm_21(i, 0x250, 0x050);
|
|
|
|
if (i->src(2).mod & Modifier(NV50_IR_MOD_NOT))
|
|
code[1] |= 1 << 13;
|
|
|
|
if (i->subOp >= 1) {
|
|
addInterp(i->subOp - 1, 0, gk110_selpFlip);
|
|
}
|
|
}
|
|
|
|
void CodeEmitterGK110::emitTEXBAR(const Instruction *i)
|
|
{
|
|
code[0] = 0x0000003e | (i->subOp << 23);
|
|
code[1] = 0x77000000;
|
|
|
|
emitPredicate(i);
|
|
}
|
|
|
|
void CodeEmitterGK110::emitTEXCSAA(const TexInstruction *i)
|
|
{
|
|
code[0] = 0x00000002;
|
|
code[1] = 0x76c00000;
|
|
|
|
code[1] |= i->tex.r << 9;
|
|
// code[1] |= i->tex.s << (9 + 8);
|
|
|
|
if (i->tex.liveOnly)
|
|
code[0] |= 0x80000000;
|
|
|
|
defId(i->def(0), 2);
|
|
srcId(i->src(0), 10);
|
|
}
|
|
|
|
static inline bool
|
|
isNextIndependentTex(const TexInstruction *i)
|
|
{
|
|
if (!i->next || !isTextureOp(i->next->op))
|
|
return false;
|
|
if (i->getDef(0)->interfers(i->next->getSrc(0)))
|
|
return false;
|
|
return !i->next->srcExists(1) || !i->getDef(0)->interfers(i->next->getSrc(1));
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitTEX(const TexInstruction *i)
|
|
{
|
|
const bool ind = i->tex.rIndirectSrc >= 0;
|
|
|
|
if (ind) {
|
|
code[0] = 0x00000002;
|
|
switch (i->op) {
|
|
case OP_TXD:
|
|
code[1] = 0x7e000000;
|
|
break;
|
|
case OP_TXLQ:
|
|
code[1] = 0x7e800000;
|
|
break;
|
|
case OP_TXF:
|
|
code[1] = 0x78000000;
|
|
break;
|
|
case OP_TXG:
|
|
code[1] = 0x7dc00000;
|
|
break;
|
|
default:
|
|
code[1] = 0x7d800000;
|
|
break;
|
|
}
|
|
} else {
|
|
switch (i->op) {
|
|
case OP_TXD:
|
|
code[0] = 0x00000002;
|
|
code[1] = 0x76000000;
|
|
code[1] |= i->tex.r << 9;
|
|
break;
|
|
case OP_TXLQ:
|
|
code[0] = 0x00000002;
|
|
code[1] = 0x76800000;
|
|
code[1] |= i->tex.r << 9;
|
|
break;
|
|
case OP_TXF:
|
|
code[0] = 0x00000002;
|
|
code[1] = 0x70000000;
|
|
code[1] |= i->tex.r << 13;
|
|
break;
|
|
case OP_TXG:
|
|
code[0] = 0x00000001;
|
|
code[1] = 0x70000000;
|
|
code[1] |= i->tex.r << 15;
|
|
break;
|
|
default:
|
|
code[0] = 0x00000001;
|
|
code[1] = 0x60000000;
|
|
code[1] |= i->tex.r << 15;
|
|
break;
|
|
}
|
|
}
|
|
|
|
code[1] |= isNextIndependentTex(i) ? 0x1 : 0x2; // t : p mode
|
|
|
|
if (i->tex.liveOnly)
|
|
code[0] |= 0x80000000;
|
|
|
|
switch (i->op) {
|
|
case OP_TEX: break;
|
|
case OP_TXB: code[1] |= 0x2000; break;
|
|
case OP_TXL: code[1] |= 0x3000; break;
|
|
case OP_TXF: break;
|
|
case OP_TXG: break;
|
|
case OP_TXD: break;
|
|
case OP_TXLQ: break;
|
|
default:
|
|
assert(!"invalid texture op");
|
|
break;
|
|
}
|
|
|
|
if (i->op == OP_TXF) {
|
|
if (!i->tex.levelZero)
|
|
code[1] |= 0x1000;
|
|
} else
|
|
if (i->tex.levelZero) {
|
|
code[1] |= 0x1000;
|
|
}
|
|
|
|
if (i->op != OP_TXD && i->tex.derivAll)
|
|
code[1] |= 0x200;
|
|
|
|
emitPredicate(i);
|
|
|
|
code[1] |= i->tex.mask << 2;
|
|
|
|
const int src1 = (i->predSrc == 1) ? 2 : 1; // if predSrc == 1, !srcExists(2)
|
|
|
|
defId(i->def(0), 2);
|
|
srcId(i->src(0), 10);
|
|
srcId(i, src1, 23);
|
|
|
|
if (i->op == OP_TXG) code[1] |= i->tex.gatherComp << 13;
|
|
|
|
// texture target:
|
|
code[1] |= (i->tex.target.isCube() ? 3 : (i->tex.target.getDim() - 1)) << 7;
|
|
if (i->tex.target.isArray())
|
|
code[1] |= 0x40;
|
|
if (i->tex.target.isShadow())
|
|
code[1] |= 0x400;
|
|
if (i->tex.target == TEX_TARGET_2D_MS ||
|
|
i->tex.target == TEX_TARGET_2D_MS_ARRAY)
|
|
code[1] |= 0x800;
|
|
|
|
if (i->srcExists(src1) && i->src(src1).getFile() == FILE_IMMEDIATE) {
|
|
// ?
|
|
}
|
|
|
|
if (i->tex.useOffsets == 1) {
|
|
switch (i->op) {
|
|
case OP_TXF: code[1] |= 0x200; break;
|
|
case OP_TXD: code[1] |= 0x00400000; break;
|
|
default: code[1] |= 0x800; break;
|
|
}
|
|
}
|
|
if (i->tex.useOffsets == 4)
|
|
code[1] |= 0x1000;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitTXQ(const TexInstruction *i)
|
|
{
|
|
code[0] = 0x00000002;
|
|
code[1] = 0x75400001;
|
|
|
|
switch (i->tex.query) {
|
|
case TXQ_DIMS: code[0] |= 0x01 << 25; break;
|
|
case TXQ_TYPE: code[0] |= 0x02 << 25; break;
|
|
case TXQ_SAMPLE_POSITION: code[0] |= 0x05 << 25; break;
|
|
case TXQ_FILTER: code[0] |= 0x10 << 25; break;
|
|
case TXQ_LOD: code[0] |= 0x12 << 25; break;
|
|
case TXQ_BORDER_COLOUR: code[0] |= 0x16 << 25; break;
|
|
default:
|
|
assert(!"invalid texture query");
|
|
break;
|
|
}
|
|
|
|
code[1] |= i->tex.mask << 2;
|
|
code[1] |= i->tex.r << 9;
|
|
if (/*i->tex.sIndirectSrc >= 0 || */i->tex.rIndirectSrc >= 0)
|
|
code[1] |= 0x08000000;
|
|
|
|
defId(i->def(0), 2);
|
|
srcId(i->src(0), 10);
|
|
|
|
emitPredicate(i);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitQUADOP(const Instruction *i, uint8_t qOp, uint8_t laneMask)
|
|
{
|
|
code[0] = 0x00000002 | ((qOp & 1) << 31);
|
|
code[1] = 0x7fc00200 | (qOp >> 1) | (laneMask << 12); // dall
|
|
|
|
defId(i->def(0), 2);
|
|
srcId(i->src(0), 10);
|
|
srcId((i->srcExists(1) && i->predSrc != 1) ? i->src(1) : i->src(0), 23);
|
|
|
|
emitPredicate(i);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitPIXLD(const Instruction *i)
|
|
{
|
|
emitForm_L(i, 0x7f4, 2, Modifier(0));
|
|
code[1] |= i->subOp << 2;
|
|
code[1] |= 0x00070000;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitBAR(const Instruction *i)
|
|
{
|
|
code[0] = 0x00000002;
|
|
code[1] = 0x85400000;
|
|
|
|
switch (i->subOp) {
|
|
case NV50_IR_SUBOP_BAR_ARRIVE: code[1] |= 0x08; break;
|
|
case NV50_IR_SUBOP_BAR_RED_AND: code[1] |= 0x50; break;
|
|
case NV50_IR_SUBOP_BAR_RED_OR: code[1] |= 0x90; break;
|
|
case NV50_IR_SUBOP_BAR_RED_POPC: code[1] |= 0x10; break;
|
|
default:
|
|
assert(i->subOp == NV50_IR_SUBOP_BAR_SYNC);
|
|
break;
|
|
}
|
|
|
|
emitPredicate(i);
|
|
|
|
// barrier id
|
|
if (i->src(0).getFile() == FILE_GPR) {
|
|
srcId(i->src(0), 10);
|
|
} else {
|
|
ImmediateValue *imm = i->getSrc(0)->asImm();
|
|
assert(imm);
|
|
code[0] |= imm->reg.data.u32 << 10;
|
|
code[1] |= 0x8000;
|
|
}
|
|
|
|
// thread count
|
|
if (i->src(1).getFile() == FILE_GPR) {
|
|
srcId(i->src(1), 23);
|
|
} else {
|
|
ImmediateValue *imm = i->getSrc(0)->asImm();
|
|
assert(imm);
|
|
assert(imm->reg.data.u32 <= 0xfff);
|
|
code[0] |= imm->reg.data.u32 << 23;
|
|
code[1] |= imm->reg.data.u32 >> 9;
|
|
code[1] |= 0x4000;
|
|
}
|
|
|
|
if (i->srcExists(2) && (i->predSrc != 2)) {
|
|
srcId(i->src(2), 32 + 10);
|
|
if (i->src(2).mod == Modifier(NV50_IR_MOD_NOT))
|
|
code[1] |= 1 << 13;
|
|
} else {
|
|
code[1] |= 7 << 10;
|
|
}
|
|
}
|
|
|
|
void CodeEmitterGK110::emitMEMBAR(const Instruction *i)
|
|
{
|
|
code[0] = 0x00000002 | NV50_IR_SUBOP_MEMBAR_SCOPE(i->subOp) << 8;
|
|
code[1] = 0x7cc00000;
|
|
|
|
emitPredicate(i);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitFlow(const Instruction *i)
|
|
{
|
|
const FlowInstruction *f = i->asFlow();
|
|
|
|
unsigned mask; // bit 0: predicate, bit 1: target
|
|
|
|
code[0] = 0x00000000;
|
|
|
|
switch (i->op) {
|
|
case OP_BRA:
|
|
code[1] = f->absolute ? 0x10800000 : 0x12000000;
|
|
if (i->srcExists(0) && i->src(0).getFile() == FILE_MEMORY_CONST)
|
|
code[0] |= 0x80;
|
|
mask = 3;
|
|
break;
|
|
case OP_CALL:
|
|
code[1] = f->absolute ? 0x11000000 : 0x13000000;
|
|
if (i->srcExists(0) && i->src(0).getFile() == FILE_MEMORY_CONST)
|
|
code[0] |= 0x80;
|
|
mask = 2;
|
|
break;
|
|
|
|
case OP_EXIT: code[1] = 0x18000000; mask = 1; break;
|
|
case OP_RET: code[1] = 0x19000000; mask = 1; break;
|
|
case OP_DISCARD: code[1] = 0x19800000; mask = 1; break;
|
|
case OP_BREAK: code[1] = 0x1a000000; mask = 1; break;
|
|
case OP_CONT: code[1] = 0x1a800000; mask = 1; break;
|
|
|
|
case OP_JOINAT: code[1] = 0x14800000; mask = 2; break;
|
|
case OP_PREBREAK: code[1] = 0x15000000; mask = 2; break;
|
|
case OP_PRECONT: code[1] = 0x15800000; mask = 2; break;
|
|
case OP_PRERET: code[1] = 0x13800000; mask = 2; break;
|
|
|
|
case OP_QUADON: code[1] = 0x1b800000; mask = 0; break;
|
|
case OP_QUADPOP: code[1] = 0x1c000000; mask = 0; break;
|
|
case OP_BRKPT: code[1] = 0x00000000; mask = 0; break;
|
|
default:
|
|
assert(!"invalid flow operation");
|
|
return;
|
|
}
|
|
|
|
if (mask & 1) {
|
|
emitPredicate(i);
|
|
if (i->flagsSrc < 0)
|
|
code[0] |= 0x3c;
|
|
}
|
|
|
|
if (!f)
|
|
return;
|
|
|
|
if (f->allWarp)
|
|
code[0] |= 1 << 9;
|
|
if (f->limit)
|
|
code[0] |= 1 << 8;
|
|
|
|
if (f->op == OP_CALL) {
|
|
if (f->builtin) {
|
|
assert(f->absolute);
|
|
uint32_t pcAbs = targNVC0->getBuiltinOffset(f->target.builtin);
|
|
addReloc(RelocEntry::TYPE_BUILTIN, 0, pcAbs, 0xff800000, 23);
|
|
addReloc(RelocEntry::TYPE_BUILTIN, 1, pcAbs, 0x007fffff, -9);
|
|
} else {
|
|
assert(!f->absolute);
|
|
int32_t pcRel = f->target.fn->binPos - (codeSize + 8);
|
|
code[0] |= (pcRel & 0x1ff) << 23;
|
|
code[1] |= (pcRel >> 9) & 0x7fff;
|
|
}
|
|
} else
|
|
if (mask & 2) {
|
|
int32_t pcRel = f->target.bb->binPos - (codeSize + 8);
|
|
if (writeIssueDelays && !(f->target.bb->binPos & 0x3f))
|
|
pcRel += 8;
|
|
// currently we don't want absolute branches
|
|
assert(!f->absolute);
|
|
code[0] |= (pcRel & 0x1ff) << 23;
|
|
code[1] |= (pcRel >> 9) & 0x7fff;
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitSHFL(const Instruction *i)
|
|
{
|
|
const ImmediateValue *imm;
|
|
|
|
code[0] = 0x00000002;
|
|
code[1] = 0x78800000 | (i->subOp << 1);
|
|
|
|
emitPredicate(i);
|
|
|
|
defId(i->def(0), 2);
|
|
srcId(i->src(0), 10);
|
|
|
|
switch (i->src(1).getFile()) {
|
|
case FILE_GPR:
|
|
srcId(i->src(1), 23);
|
|
break;
|
|
case FILE_IMMEDIATE:
|
|
imm = i->getSrc(1)->asImm();
|
|
assert(imm && imm->reg.data.u32 < 0x20);
|
|
code[0] |= imm->reg.data.u32 << 23;
|
|
code[0] |= 1 << 31;
|
|
break;
|
|
default:
|
|
assert(!"invalid src1 file");
|
|
break;
|
|
}
|
|
|
|
switch (i->src(2).getFile()) {
|
|
case FILE_GPR:
|
|
srcId(i->src(2), 42);
|
|
break;
|
|
case FILE_IMMEDIATE:
|
|
imm = i->getSrc(2)->asImm();
|
|
assert(imm && imm->reg.data.u32 < 0x2000);
|
|
code[1] |= imm->reg.data.u32 << 5;
|
|
code[1] |= 1;
|
|
break;
|
|
default:
|
|
assert(!"invalid src2 file");
|
|
break;
|
|
}
|
|
|
|
if (!i->defExists(1))
|
|
code[1] |= 7 << 19;
|
|
else {
|
|
assert(i->def(1).getFile() == FILE_PREDICATE);
|
|
defId(i->def(1), 51);
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitVOTE(const Instruction *i)
|
|
{
|
|
const ImmediateValue *imm;
|
|
uint32_t u32;
|
|
|
|
code[0] = 0x00000002;
|
|
code[1] = 0x86c00000 | (i->subOp << 19);
|
|
|
|
emitPredicate(i);
|
|
|
|
unsigned rp = 0;
|
|
for (int d = 0; i->defExists(d); d++) {
|
|
if (i->def(d).getFile() == FILE_PREDICATE) {
|
|
assert(!(rp & 2));
|
|
rp |= 2;
|
|
defId(i->def(d), 48);
|
|
} else if (i->def(d).getFile() == FILE_GPR) {
|
|
assert(!(rp & 1));
|
|
rp |= 1;
|
|
defId(i->def(d), 2);
|
|
} else {
|
|
assert(!"Unhandled def");
|
|
}
|
|
}
|
|
if (!(rp & 1))
|
|
code[0] |= 255 << 2;
|
|
if (!(rp & 2))
|
|
code[1] |= 7 << 16;
|
|
|
|
switch (i->src(0).getFile()) {
|
|
case FILE_PREDICATE:
|
|
if (i->src(0).mod == Modifier(NV50_IR_MOD_NOT))
|
|
code[0] |= 1 << 13;
|
|
srcId(i->src(0), 42);
|
|
break;
|
|
case FILE_IMMEDIATE:
|
|
imm = i->getSrc(0)->asImm();
|
|
assert(imm);
|
|
u32 = imm->reg.data.u32;
|
|
assert(u32 == 0 || u32 == 1);
|
|
code[1] |= (u32 == 1 ? 0x7 : 0xf) << 10;
|
|
break;
|
|
default:
|
|
assert(!"Unhandled src");
|
|
break;
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitSUGType(DataType ty, const int pos)
|
|
{
|
|
uint8_t n = 0;
|
|
|
|
switch (ty) {
|
|
case TYPE_S32: n = 1; break;
|
|
case TYPE_U8: n = 2; break;
|
|
case TYPE_S8: n = 3; break;
|
|
default:
|
|
assert(ty == TYPE_U32);
|
|
break;
|
|
}
|
|
code[pos / 32] |= n << (pos % 32);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitSUCachingMode(CacheMode c)
|
|
{
|
|
uint8_t n = 0;
|
|
|
|
switch (c) {
|
|
case CACHE_CA:
|
|
// case CACHE_WB:
|
|
n = 0;
|
|
break;
|
|
case CACHE_CG:
|
|
n = 1;
|
|
break;
|
|
case CACHE_CS:
|
|
n = 2;
|
|
break;
|
|
case CACHE_CV:
|
|
// case CACHE_WT:
|
|
n = 3;
|
|
break;
|
|
default:
|
|
assert(!"invalid caching mode");
|
|
break;
|
|
}
|
|
code[0] |= (n & 1) << 31;
|
|
code[1] |= (n & 2) >> 1;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::setSUConst16(const Instruction *i, const int s)
|
|
{
|
|
const uint32_t offset = i->getSrc(s)->reg.data.offset;
|
|
|
|
assert(offset == (offset & 0xfffc));
|
|
|
|
code[0] |= offset << 21;
|
|
code[1] |= offset >> 11;
|
|
code[1] |= i->getSrc(s)->reg.fileIndex << 5;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitSULDGB(const TexInstruction *i)
|
|
{
|
|
code[0] = 0x00000002;
|
|
code[1] = 0x30000000 | (i->subOp << 14);
|
|
|
|
if (i->src(1).getFile() == FILE_MEMORY_CONST) {
|
|
emitLoadStoreType(i->dType, 0x38);
|
|
emitCachingMode(i->cache, 0x36);
|
|
|
|
// format
|
|
setSUConst16(i, 1);
|
|
} else {
|
|
assert(i->src(1).getFile() == FILE_GPR);
|
|
code[1] |= 0x49800000;
|
|
|
|
emitLoadStoreType(i->dType, 0x21);
|
|
emitSUCachingMode(i->cache);
|
|
|
|
srcId(i->src(1), 23);
|
|
}
|
|
|
|
emitSUGType(i->sType, 0x34);
|
|
|
|
emitPredicate(i);
|
|
defId(i->def(0), 2); // destination
|
|
srcId(i->src(0), 10); // address
|
|
|
|
// surface predicate
|
|
if (!i->srcExists(2) || (i->predSrc == 2)) {
|
|
code[1] |= 0x7 << 10;
|
|
} else {
|
|
if (i->src(2).mod == Modifier(NV50_IR_MOD_NOT))
|
|
code[1] |= 1 << 13;
|
|
srcId(i->src(2), 32 + 10);
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitSUSTGx(const TexInstruction *i)
|
|
{
|
|
assert(i->op == OP_SUSTP);
|
|
|
|
code[0] = 0x00000002;
|
|
code[1] = 0x38000000;
|
|
|
|
if (i->src(1).getFile() == FILE_MEMORY_CONST) {
|
|
code[0] |= i->subOp << 2;
|
|
|
|
if (i->op == OP_SUSTP)
|
|
code[0] |= i->tex.mask << 4;
|
|
|
|
emitSUGType(i->sType, 0x8);
|
|
emitCachingMode(i->cache, 0x36);
|
|
|
|
// format
|
|
setSUConst16(i, 1);
|
|
} else {
|
|
assert(i->src(1).getFile() == FILE_GPR);
|
|
|
|
code[0] |= i->subOp << 23;
|
|
code[1] |= 0x41c00000;
|
|
|
|
if (i->op == OP_SUSTP)
|
|
code[0] |= i->tex.mask << 25;
|
|
|
|
emitSUGType(i->sType, 0x1d);
|
|
emitSUCachingMode(i->cache);
|
|
|
|
srcId(i->src(1), 2);
|
|
}
|
|
|
|
emitPredicate(i);
|
|
srcId(i->src(0), 10); // address
|
|
srcId(i->src(3), 42); // values
|
|
|
|
// surface predicate
|
|
if (!i->srcExists(2) || (i->predSrc == 2)) {
|
|
code[1] |= 0x7 << 18;
|
|
} else {
|
|
if (i->src(2).mod == Modifier(NV50_IR_MOD_NOT))
|
|
code[1] |= 1 << 21;
|
|
srcId(i->src(2), 32 + 18);
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitSUCLAMPMode(uint16_t subOp)
|
|
{
|
|
uint8_t m;
|
|
switch (subOp & ~NV50_IR_SUBOP_SUCLAMP_2D) {
|
|
case NV50_IR_SUBOP_SUCLAMP_SD(0, 1): m = 0; break;
|
|
case NV50_IR_SUBOP_SUCLAMP_SD(1, 1): m = 1; break;
|
|
case NV50_IR_SUBOP_SUCLAMP_SD(2, 1): m = 2; break;
|
|
case NV50_IR_SUBOP_SUCLAMP_SD(3, 1): m = 3; break;
|
|
case NV50_IR_SUBOP_SUCLAMP_SD(4, 1): m = 4; break;
|
|
case NV50_IR_SUBOP_SUCLAMP_PL(0, 1): m = 5; break;
|
|
case NV50_IR_SUBOP_SUCLAMP_PL(1, 1): m = 6; break;
|
|
case NV50_IR_SUBOP_SUCLAMP_PL(2, 1): m = 7; break;
|
|
case NV50_IR_SUBOP_SUCLAMP_PL(3, 1): m = 8; break;
|
|
case NV50_IR_SUBOP_SUCLAMP_PL(4, 1): m = 9; break;
|
|
case NV50_IR_SUBOP_SUCLAMP_BL(0, 1): m = 10; break;
|
|
case NV50_IR_SUBOP_SUCLAMP_BL(1, 1): m = 11; break;
|
|
case NV50_IR_SUBOP_SUCLAMP_BL(2, 1): m = 12; break;
|
|
case NV50_IR_SUBOP_SUCLAMP_BL(3, 1): m = 13; break;
|
|
case NV50_IR_SUBOP_SUCLAMP_BL(4, 1): m = 14; break;
|
|
default:
|
|
return;
|
|
}
|
|
code[1] |= m << 20;
|
|
if (subOp & NV50_IR_SUBOP_SUCLAMP_2D)
|
|
code[1] |= 1 << 24;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitSUCalc(Instruction *i)
|
|
{
|
|
ImmediateValue *imm = NULL;
|
|
uint64_t opc1, opc2;
|
|
|
|
if (i->srcExists(2)) {
|
|
imm = i->getSrc(2)->asImm();
|
|
if (imm)
|
|
i->setSrc(2, NULL); // special case, make emitForm_21 not assert
|
|
}
|
|
|
|
switch (i->op) {
|
|
case OP_SUCLAMP: opc1 = 0xb00; opc2 = 0x580; break;
|
|
case OP_SUBFM: opc1 = 0xb68; opc2 = 0x1e8; break;
|
|
case OP_SUEAU: opc1 = 0xb6c; opc2 = 0x1ec; break;
|
|
default:
|
|
assert(0);
|
|
return;
|
|
}
|
|
emitForm_21(i, opc2, opc1);
|
|
|
|
if (i->op == OP_SUCLAMP) {
|
|
if (i->dType == TYPE_S32)
|
|
code[1] |= 1 << 19;
|
|
emitSUCLAMPMode(i->subOp);
|
|
}
|
|
|
|
if (i->op == OP_SUBFM && i->subOp == NV50_IR_SUBOP_SUBFM_3D)
|
|
code[1] |= 1 << 18;
|
|
|
|
if (i->op != OP_SUEAU) {
|
|
const uint8_t pos = i->op == OP_SUBFM ? 19 : 16;
|
|
if (i->def(0).getFile() == FILE_PREDICATE) { // p, #
|
|
code[0] |= 255 << 2;
|
|
code[1] |= i->getDef(1)->reg.data.id << pos;
|
|
} else
|
|
if (i->defExists(1)) { // r, p
|
|
assert(i->def(1).getFile() == FILE_PREDICATE);
|
|
code[1] |= i->getDef(1)->reg.data.id << pos;
|
|
} else { // r, #
|
|
code[1] |= 7 << pos;
|
|
}
|
|
}
|
|
|
|
if (imm) {
|
|
assert(i->op == OP_SUCLAMP);
|
|
i->setSrc(2, imm);
|
|
code[1] |= (imm->reg.data.u32 & 0x3f) << 10; // sint6
|
|
}
|
|
}
|
|
|
|
|
|
void
|
|
CodeEmitterGK110::emitVectorSubOp(const Instruction *i)
|
|
{
|
|
switch (NV50_IR_SUBOP_Vn(i->subOp)) {
|
|
case 0:
|
|
code[1] |= (i->subOp & 0x000f) << 7; // vsrc1
|
|
code[1] |= (i->subOp & 0x00e0) >> 6; // vsrc2
|
|
code[1] |= (i->subOp & 0x0100) << 13; // vsrc2
|
|
code[1] |= (i->subOp & 0x3c00) << 12; // vdst
|
|
break;
|
|
default:
|
|
assert(0);
|
|
break;
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitVSHL(const Instruction *i)
|
|
{
|
|
code[0] = 0x00000002;
|
|
code[1] = 0xb8000000;
|
|
|
|
assert(NV50_IR_SUBOP_Vn(i->subOp) == 0);
|
|
|
|
if (isSignedType(i->dType)) code[1] |= 1 << 25;
|
|
if (isSignedType(i->sType)) code[1] |= 1 << 19;
|
|
|
|
emitVectorSubOp(i);
|
|
|
|
emitPredicate(i);
|
|
defId(i->def(0), 2);
|
|
srcId(i->src(0), 10);
|
|
|
|
if (i->getSrc(1)->reg.file == FILE_IMMEDIATE) {
|
|
ImmediateValue *imm = i->getSrc(1)->asImm();
|
|
assert(imm);
|
|
code[0] |= (imm->reg.data.u32 & 0x01ff) << 23;
|
|
code[1] |= (imm->reg.data.u32 & 0xfe00) >> 9;
|
|
} else {
|
|
assert(i->getSrc(1)->reg.file == FILE_GPR);
|
|
code[1] |= 1 << 21;
|
|
srcId(i->src(1), 23);
|
|
}
|
|
srcId(i->src(2), 42);
|
|
|
|
if (i->saturate)
|
|
code[0] |= 1 << 22;
|
|
if (i->flagsDef >= 0)
|
|
code[1] |= 1 << 18;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitAFETCH(const Instruction *i)
|
|
{
|
|
uint32_t offset = i->src(0).get()->reg.data.offset & 0x7ff;
|
|
|
|
code[0] = 0x00000002 | (offset << 23);
|
|
code[1] = 0x7d000000 | (offset >> 9);
|
|
|
|
if (i->getSrc(0)->reg.file == FILE_SHADER_OUTPUT)
|
|
code[1] |= 0x8;
|
|
|
|
emitPredicate(i);
|
|
|
|
defId(i->def(0), 2);
|
|
srcId(i->src(0).getIndirect(0), 10);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitPFETCH(const Instruction *i)
|
|
{
|
|
uint32_t prim = i->src(0).get()->reg.data.u32;
|
|
|
|
code[0] = 0x00000002 | ((prim & 0xff) << 23);
|
|
code[1] = 0x7f800000;
|
|
|
|
emitPredicate(i);
|
|
|
|
const int src1 = (i->predSrc == 1) ? 2 : 1; // if predSrc == 1, !srcExists(2)
|
|
|
|
defId(i->def(0), 2);
|
|
srcId(i, src1, 10);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitVFETCH(const Instruction *i)
|
|
{
|
|
unsigned int size = typeSizeof(i->dType);
|
|
uint32_t offset = i->src(0).get()->reg.data.offset;
|
|
|
|
code[0] = 0x00000002 | (offset << 23);
|
|
code[1] = 0x7ec00000 | (offset >> 9);
|
|
code[1] |= (size / 4 - 1) << 18;
|
|
|
|
if (i->perPatch)
|
|
code[1] |= 0x4;
|
|
if (i->getSrc(0)->reg.file == FILE_SHADER_OUTPUT)
|
|
code[1] |= 0x8; // yes, TCPs can read from *outputs* of other threads
|
|
|
|
emitPredicate(i);
|
|
|
|
defId(i->def(0), 2);
|
|
srcId(i->src(0).getIndirect(0), 10);
|
|
srcId(i->src(0).getIndirect(1), 32 + 10); // vertex address
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitEXPORT(const Instruction *i)
|
|
{
|
|
unsigned int size = typeSizeof(i->dType);
|
|
uint32_t offset = i->src(0).get()->reg.data.offset;
|
|
|
|
code[0] = 0x00000002 | (offset << 23);
|
|
code[1] = 0x7f000000 | (offset >> 9);
|
|
code[1] |= (size / 4 - 1) << 18;
|
|
|
|
if (i->perPatch)
|
|
code[1] |= 0x4;
|
|
|
|
emitPredicate(i);
|
|
|
|
assert(i->src(1).getFile() == FILE_GPR);
|
|
|
|
srcId(i->src(0).getIndirect(0), 10);
|
|
srcId(i->src(0).getIndirect(1), 32 + 10); // vertex base address
|
|
srcId(i->src(1), 2);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitOUT(const Instruction *i)
|
|
{
|
|
assert(i->src(0).getFile() == FILE_GPR);
|
|
|
|
emitForm_21(i, 0x1f0, 0xb70);
|
|
|
|
if (i->op == OP_EMIT)
|
|
code[1] |= 1 << 10;
|
|
if (i->op == OP_RESTART || i->subOp == NV50_IR_SUBOP_EMIT_RESTART)
|
|
code[1] |= 1 << 11;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitInterpMode(const Instruction *i)
|
|
{
|
|
code[1] |= (i->ipa & 0x3) << 21; // TODO: INTERP_SAMPLEID
|
|
code[1] |= (i->ipa & 0xc) << (19 - 2);
|
|
}
|
|
|
|
void
|
|
gk110_interpApply(const struct FixupEntry *entry, uint32_t *code, const FixupData& data)
|
|
{
|
|
int ipa = entry->ipa;
|
|
int reg = entry->reg;
|
|
int loc = entry->loc;
|
|
|
|
if (data.flatshade &&
|
|
(ipa & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_SC) {
|
|
ipa = NV50_IR_INTERP_FLAT;
|
|
reg = 0xff;
|
|
} else if (data.force_persample_interp &&
|
|
(ipa & NV50_IR_INTERP_SAMPLE_MASK) == NV50_IR_INTERP_DEFAULT &&
|
|
(ipa & NV50_IR_INTERP_MODE_MASK) != NV50_IR_INTERP_FLAT) {
|
|
ipa |= NV50_IR_INTERP_CENTROID;
|
|
}
|
|
code[loc + 1] &= ~(0xf << 19);
|
|
code[loc + 1] |= (ipa & 0x3) << 21;
|
|
code[loc + 1] |= (ipa & 0xc) << (19 - 2);
|
|
code[loc + 0] &= ~(0xff << 23);
|
|
code[loc + 0] |= reg << 23;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitINTERP(const Instruction *i)
|
|
{
|
|
const uint32_t base = i->getSrc(0)->reg.data.offset;
|
|
|
|
code[0] = 0x00000002 | (base << 31);
|
|
code[1] = 0x74800000 | (base >> 1);
|
|
|
|
if (i->saturate)
|
|
code[1] |= 1 << 18;
|
|
|
|
if (i->op == OP_PINTERP) {
|
|
srcId(i->src(1), 23);
|
|
addInterp(i->ipa, SDATA(i->src(1)).id, gk110_interpApply);
|
|
} else {
|
|
code[0] |= 0xff << 23;
|
|
addInterp(i->ipa, 0xff, gk110_interpApply);
|
|
}
|
|
|
|
srcId(i->src(0).getIndirect(0), 10);
|
|
emitInterpMode(i);
|
|
|
|
emitPredicate(i);
|
|
defId(i->def(0), 2);
|
|
|
|
if (i->getSampleMode() == NV50_IR_INTERP_OFFSET)
|
|
srcId(i->src(i->op == OP_PINTERP ? 2 : 1), 32 + 10);
|
|
else
|
|
code[1] |= 0xff << 10;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitLoadStoreType(DataType ty, const int pos)
|
|
{
|
|
uint8_t n;
|
|
|
|
switch (ty) {
|
|
case TYPE_U8:
|
|
n = 0;
|
|
break;
|
|
case TYPE_S8:
|
|
n = 1;
|
|
break;
|
|
case TYPE_U16:
|
|
n = 2;
|
|
break;
|
|
case TYPE_S16:
|
|
n = 3;
|
|
break;
|
|
case TYPE_F32:
|
|
case TYPE_U32:
|
|
case TYPE_S32:
|
|
n = 4;
|
|
break;
|
|
case TYPE_F64:
|
|
case TYPE_U64:
|
|
case TYPE_S64:
|
|
n = 5;
|
|
break;
|
|
case TYPE_B128:
|
|
n = 6;
|
|
break;
|
|
default:
|
|
n = 0;
|
|
assert(!"invalid ld/st type");
|
|
break;
|
|
}
|
|
code[pos / 32] |= n << (pos % 32);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitCachingMode(CacheMode c, const int pos)
|
|
{
|
|
uint8_t n;
|
|
|
|
switch (c) {
|
|
case CACHE_CA:
|
|
// case CACHE_WB:
|
|
n = 0;
|
|
break;
|
|
case CACHE_CG:
|
|
n = 1;
|
|
break;
|
|
case CACHE_CS:
|
|
n = 2;
|
|
break;
|
|
case CACHE_CV:
|
|
// case CACHE_WT:
|
|
n = 3;
|
|
break;
|
|
default:
|
|
n = 0;
|
|
assert(!"invalid caching mode");
|
|
break;
|
|
}
|
|
code[pos / 32] |= n << (pos % 32);
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitSTORE(const Instruction *i)
|
|
{
|
|
int32_t offset = SDATA(i->src(0)).offset;
|
|
|
|
switch (i->src(0).getFile()) {
|
|
case FILE_MEMORY_GLOBAL: code[1] = 0xe0000000; code[0] = 0x00000000; break;
|
|
case FILE_MEMORY_LOCAL: code[1] = 0x7a800000; code[0] = 0x00000002; break;
|
|
case FILE_MEMORY_SHARED:
|
|
code[0] = 0x00000002;
|
|
if (i->subOp == NV50_IR_SUBOP_STORE_UNLOCKED)
|
|
code[1] = 0x78400000;
|
|
else
|
|
code[1] = 0x7ac00000;
|
|
break;
|
|
default:
|
|
assert(!"invalid memory file");
|
|
break;
|
|
}
|
|
|
|
if (code[0] & 0x2) {
|
|
offset &= 0xffffff;
|
|
emitLoadStoreType(i->dType, 0x33);
|
|
if (i->src(0).getFile() == FILE_MEMORY_LOCAL)
|
|
emitCachingMode(i->cache, 0x2f);
|
|
} else {
|
|
emitLoadStoreType(i->dType, 0x38);
|
|
emitCachingMode(i->cache, 0x3b);
|
|
}
|
|
code[0] |= offset << 23;
|
|
code[1] |= offset >> 9;
|
|
|
|
// Unlocked store on shared memory can fail.
|
|
if (i->src(0).getFile() == FILE_MEMORY_SHARED &&
|
|
i->subOp == NV50_IR_SUBOP_STORE_UNLOCKED) {
|
|
assert(i->defExists(0));
|
|
defId(i->def(0), 32 + 16);
|
|
}
|
|
|
|
emitPredicate(i);
|
|
|
|
srcId(i->src(1), 2);
|
|
srcId(i->src(0).getIndirect(0), 10);
|
|
if (i->src(0).getFile() == FILE_MEMORY_GLOBAL &&
|
|
i->src(0).isIndirect(0) &&
|
|
i->getIndirect(0, 0)->reg.size == 8)
|
|
code[1] |= 1 << 23;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitLOAD(const Instruction *i)
|
|
{
|
|
int32_t offset = SDATA(i->src(0)).offset;
|
|
|
|
switch (i->src(0).getFile()) {
|
|
case FILE_MEMORY_GLOBAL: code[1] = 0xc0000000; code[0] = 0x00000000; break;
|
|
case FILE_MEMORY_LOCAL: code[1] = 0x7a000000; code[0] = 0x00000002; break;
|
|
case FILE_MEMORY_SHARED:
|
|
code[0] = 0x00000002;
|
|
if (i->subOp == NV50_IR_SUBOP_LOAD_LOCKED)
|
|
code[1] = 0x77400000;
|
|
else
|
|
code[1] = 0x7a400000;
|
|
break;
|
|
case FILE_MEMORY_CONST:
|
|
if (!i->src(0).isIndirect(0) && typeSizeof(i->dType) == 4) {
|
|
emitMOV(i);
|
|
return;
|
|
}
|
|
offset &= 0xffff;
|
|
code[0] = 0x00000002;
|
|
code[1] = 0x7c800000 | (i->src(0).get()->reg.fileIndex << 7);
|
|
code[1] |= i->subOp << 15;
|
|
break;
|
|
default:
|
|
assert(!"invalid memory file");
|
|
break;
|
|
}
|
|
|
|
if (code[0] & 0x2) {
|
|
offset &= 0xffffff;
|
|
emitLoadStoreType(i->dType, 0x33);
|
|
if (i->src(0).getFile() == FILE_MEMORY_LOCAL)
|
|
emitCachingMode(i->cache, 0x2f);
|
|
} else {
|
|
emitLoadStoreType(i->dType, 0x38);
|
|
emitCachingMode(i->cache, 0x3b);
|
|
}
|
|
code[0] |= offset << 23;
|
|
code[1] |= offset >> 9;
|
|
|
|
// Locked store on shared memory can fail.
|
|
int r = 0, p = -1;
|
|
if (i->src(0).getFile() == FILE_MEMORY_SHARED &&
|
|
i->subOp == NV50_IR_SUBOP_LOAD_LOCKED) {
|
|
if (i->def(0).getFile() == FILE_PREDICATE) { // p, #
|
|
r = -1;
|
|
p = 0;
|
|
} else if (i->defExists(1)) { // r, p
|
|
p = 1;
|
|
} else {
|
|
assert(!"Expected predicate dest for load locked");
|
|
}
|
|
}
|
|
|
|
emitPredicate(i);
|
|
|
|
if (r >= 0)
|
|
defId(i->def(r), 2);
|
|
else
|
|
code[0] |= 255 << 2;
|
|
|
|
if (p >= 0)
|
|
defId(i->def(p), 32 + 16);
|
|
|
|
if (i->getIndirect(0, 0)) {
|
|
srcId(i->src(0).getIndirect(0), 10);
|
|
if (i->getIndirect(0, 0)->reg.size == 8)
|
|
code[1] |= 1 << 23;
|
|
} else {
|
|
code[0] |= 255 << 10;
|
|
}
|
|
}
|
|
|
|
uint8_t
|
|
CodeEmitterGK110::getSRegEncoding(const ValueRef& ref)
|
|
{
|
|
switch (SDATA(ref).sv.sv) {
|
|
case SV_LANEID: return 0x00;
|
|
case SV_PHYSID: return 0x03;
|
|
case SV_VERTEX_COUNT: return 0x10;
|
|
case SV_INVOCATION_ID: return 0x11;
|
|
case SV_YDIR: return 0x12;
|
|
case SV_THREAD_KILL: return 0x13;
|
|
case SV_COMBINED_TID: return 0x20;
|
|
case SV_TID: return 0x21 + SDATA(ref).sv.index;
|
|
case SV_CTAID: return 0x25 + SDATA(ref).sv.index;
|
|
case SV_NTID: return 0x29 + SDATA(ref).sv.index;
|
|
case SV_GRIDID: return 0x2c;
|
|
case SV_NCTAID: return 0x2d + SDATA(ref).sv.index;
|
|
case SV_LBASE: return 0x34;
|
|
case SV_SBASE: return 0x30;
|
|
case SV_LANEMASK_EQ: return 0x38;
|
|
case SV_LANEMASK_LT: return 0x39;
|
|
case SV_LANEMASK_LE: return 0x3a;
|
|
case SV_LANEMASK_GT: return 0x3b;
|
|
case SV_LANEMASK_GE: return 0x3c;
|
|
case SV_CLOCK: return 0x50 + SDATA(ref).sv.index;
|
|
default:
|
|
assert(!"no sreg for system value");
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitMOV(const Instruction *i)
|
|
{
|
|
if (i->def(0).getFile() == FILE_PREDICATE) {
|
|
if (i->src(0).getFile() == FILE_GPR) {
|
|
// Use ISETP.NE.AND dst, PT, src, RZ, PT
|
|
code[0] = 0x00000002;
|
|
code[1] = 0xdb500000;
|
|
|
|
code[0] |= 0x7 << 2;
|
|
code[0] |= 0xff << 23;
|
|
code[1] |= 0x7 << 10;
|
|
srcId(i->src(0), 10);
|
|
} else
|
|
if (i->src(0).getFile() == FILE_PREDICATE) {
|
|
// Use PSETP.AND.AND dst, PT, src, PT, PT
|
|
code[0] = 0x00000002;
|
|
code[1] = 0x84800000;
|
|
|
|
code[0] |= 0x7 << 2;
|
|
code[1] |= 0x7 << 0;
|
|
code[1] |= 0x7 << 10;
|
|
|
|
srcId(i->src(0), 14);
|
|
} else {
|
|
assert(!"Unexpected source for predicate destination");
|
|
emitNOP(i);
|
|
}
|
|
emitPredicate(i);
|
|
defId(i->def(0), 5);
|
|
} else
|
|
if (i->src(0).getFile() == FILE_SYSTEM_VALUE) {
|
|
code[0] = 0x00000002 | (getSRegEncoding(i->src(0)) << 23);
|
|
code[1] = 0x86400000;
|
|
emitPredicate(i);
|
|
defId(i->def(0), 2);
|
|
} else
|
|
if (i->src(0).getFile() == FILE_IMMEDIATE) {
|
|
code[0] = 0x00000002 | (i->lanes << 14);
|
|
code[1] = 0x74000000;
|
|
emitPredicate(i);
|
|
defId(i->def(0), 2);
|
|
setImmediate32(i, 0, Modifier(0));
|
|
} else
|
|
if (i->src(0).getFile() == FILE_PREDICATE) {
|
|
code[0] = 0x00000002;
|
|
code[1] = 0x84401c07;
|
|
emitPredicate(i);
|
|
defId(i->def(0), 2);
|
|
srcId(i->src(0), 14);
|
|
} else {
|
|
emitForm_C(i, 0x24c, 2);
|
|
code[1] |= i->lanes << 10;
|
|
}
|
|
}
|
|
|
|
static inline bool
|
|
uses64bitAddress(const Instruction *ldst)
|
|
{
|
|
return ldst->src(0).getFile() == FILE_MEMORY_GLOBAL &&
|
|
ldst->src(0).isIndirect(0) &&
|
|
ldst->getIndirect(0, 0)->reg.size == 8;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitATOM(const Instruction *i)
|
|
{
|
|
const bool hasDst = i->defExists(0);
|
|
const bool exch = i->subOp == NV50_IR_SUBOP_ATOM_EXCH;
|
|
|
|
code[0] = 0x00000002;
|
|
if (i->subOp == NV50_IR_SUBOP_ATOM_CAS)
|
|
code[1] = 0x77800000;
|
|
else
|
|
code[1] = 0x68000000;
|
|
|
|
switch (i->subOp) {
|
|
case NV50_IR_SUBOP_ATOM_CAS: break;
|
|
case NV50_IR_SUBOP_ATOM_EXCH: code[1] |= 0x04000000; break;
|
|
default: code[1] |= i->subOp << 23; break;
|
|
}
|
|
|
|
switch (i->dType) {
|
|
case TYPE_U32: break;
|
|
case TYPE_S32: code[1] |= 0x00100000; break;
|
|
case TYPE_U64: code[1] |= 0x00200000; break;
|
|
case TYPE_F32: code[1] |= 0x00300000; break;
|
|
case TYPE_B128: code[1] |= 0x00400000; break; /* TODO: U128 */
|
|
case TYPE_S64: code[1] |= 0x00500000; break;
|
|
default: assert(!"unsupported type"); break;
|
|
}
|
|
|
|
emitPredicate(i);
|
|
|
|
/* TODO: cas: check that src regs line up */
|
|
/* TODO: cas: flip bits if $r255 is used */
|
|
srcId(i->src(1), 23);
|
|
|
|
if (hasDst) {
|
|
defId(i->def(0), 2);
|
|
} else
|
|
if (!exch) {
|
|
code[0] |= 255 << 2;
|
|
}
|
|
|
|
if (hasDst || !exch) {
|
|
const int32_t offset = SDATA(i->src(0)).offset;
|
|
assert(offset < 0x80000 && offset >= -0x80000);
|
|
code[0] |= (offset & 1) << 31;
|
|
code[1] |= (offset & 0xffffe) >> 1;
|
|
} else {
|
|
srcAddr32(i->src(0), 31);
|
|
}
|
|
|
|
if (i->getIndirect(0, 0)) {
|
|
srcId(i->getIndirect(0, 0), 10);
|
|
if (i->getIndirect(0, 0)->reg.size == 8)
|
|
code[1] |= 1 << 19;
|
|
} else {
|
|
code[0] |= 255 << 10;
|
|
}
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::emitCCTL(const Instruction *i)
|
|
{
|
|
int32_t offset = SDATA(i->src(0)).offset;
|
|
|
|
code[0] = 0x00000002 | (i->subOp << 2);
|
|
|
|
if (i->src(0).getFile() == FILE_MEMORY_GLOBAL) {
|
|
code[1] = 0x7b000000;
|
|
} else {
|
|
code[1] = 0x7c000000;
|
|
offset &= 0xffffff;
|
|
}
|
|
code[0] |= offset << 23;
|
|
code[1] |= offset >> 9;
|
|
|
|
if (uses64bitAddress(i))
|
|
code[1] |= 1 << 23;
|
|
srcId(i->src(0).getIndirect(0), 10);
|
|
|
|
emitPredicate(i);
|
|
}
|
|
|
|
bool
|
|
CodeEmitterGK110::emitInstruction(Instruction *insn)
|
|
{
|
|
const unsigned int size = (writeIssueDelays && !(codeSize & 0x3f)) ? 16 : 8;
|
|
|
|
if (insn->encSize != 8) {
|
|
ERROR("skipping unencodable instruction: ");
|
|
insn->print();
|
|
return false;
|
|
} else
|
|
if (codeSize + size > codeSizeLimit) {
|
|
ERROR("code emitter output buffer too small\n");
|
|
return false;
|
|
}
|
|
|
|
if (writeIssueDelays) {
|
|
int id = (codeSize & 0x3f) / 8 - 1;
|
|
if (id < 0) {
|
|
id += 1;
|
|
code[0] = 0x00000000; // cf issue delay "instruction"
|
|
code[1] = 0x08000000;
|
|
code += 2;
|
|
codeSize += 8;
|
|
}
|
|
uint32_t *data = code - (id * 2 + 2);
|
|
|
|
switch (id) {
|
|
case 0: data[0] |= insn->sched << 2; break;
|
|
case 1: data[0] |= insn->sched << 10; break;
|
|
case 2: data[0] |= insn->sched << 18; break;
|
|
case 3: data[0] |= insn->sched << 26; data[1] |= insn->sched >> 6; break;
|
|
case 4: data[1] |= insn->sched << 2; break;
|
|
case 5: data[1] |= insn->sched << 10; break;
|
|
case 6: data[1] |= insn->sched << 18; break;
|
|
default:
|
|
assert(0);
|
|
break;
|
|
}
|
|
}
|
|
|
|
// assert that instructions with multiple defs don't corrupt registers
|
|
for (int d = 0; insn->defExists(d); ++d)
|
|
assert(insn->asTex() || insn->def(d).rep()->reg.data.id >= 0);
|
|
|
|
switch (insn->op) {
|
|
case OP_MOV:
|
|
case OP_RDSV:
|
|
emitMOV(insn);
|
|
break;
|
|
case OP_NOP:
|
|
break;
|
|
case OP_LOAD:
|
|
emitLOAD(insn);
|
|
break;
|
|
case OP_STORE:
|
|
emitSTORE(insn);
|
|
break;
|
|
case OP_LINTERP:
|
|
case OP_PINTERP:
|
|
emitINTERP(insn);
|
|
break;
|
|
case OP_VFETCH:
|
|
emitVFETCH(insn);
|
|
break;
|
|
case OP_EXPORT:
|
|
emitEXPORT(insn);
|
|
break;
|
|
case OP_AFETCH:
|
|
emitAFETCH(insn);
|
|
break;
|
|
case OP_PFETCH:
|
|
emitPFETCH(insn);
|
|
break;
|
|
case OP_EMIT:
|
|
case OP_RESTART:
|
|
emitOUT(insn);
|
|
break;
|
|
case OP_ADD:
|
|
case OP_SUB:
|
|
if (insn->dType == TYPE_F64)
|
|
emitDADD(insn);
|
|
else if (isFloatType(insn->dType))
|
|
emitFADD(insn);
|
|
else
|
|
emitUADD(insn);
|
|
break;
|
|
case OP_MUL:
|
|
if (insn->dType == TYPE_F64)
|
|
emitDMUL(insn);
|
|
else if (isFloatType(insn->dType))
|
|
emitFMUL(insn);
|
|
else
|
|
emitIMUL(insn);
|
|
break;
|
|
case OP_MAD:
|
|
case OP_FMA:
|
|
if (insn->dType == TYPE_F64)
|
|
emitDMAD(insn);
|
|
else if (isFloatType(insn->dType))
|
|
emitFMAD(insn);
|
|
else
|
|
emitIMAD(insn);
|
|
break;
|
|
case OP_MADSP:
|
|
emitMADSP(insn);
|
|
break;
|
|
case OP_SAD:
|
|
emitISAD(insn);
|
|
break;
|
|
case OP_SHLADD:
|
|
emitSHLADD(insn);
|
|
break;
|
|
case OP_NOT:
|
|
emitNOT(insn);
|
|
break;
|
|
case OP_AND:
|
|
emitLogicOp(insn, 0);
|
|
break;
|
|
case OP_OR:
|
|
emitLogicOp(insn, 1);
|
|
break;
|
|
case OP_XOR:
|
|
emitLogicOp(insn, 2);
|
|
break;
|
|
case OP_SHL:
|
|
case OP_SHR:
|
|
if (typeSizeof(insn->sType) == 8)
|
|
emitShift64(insn);
|
|
else
|
|
emitShift(insn);
|
|
break;
|
|
case OP_SET:
|
|
case OP_SET_AND:
|
|
case OP_SET_OR:
|
|
case OP_SET_XOR:
|
|
emitSET(insn->asCmp());
|
|
break;
|
|
case OP_SELP:
|
|
emitSELP(insn);
|
|
break;
|
|
case OP_SLCT:
|
|
emitSLCT(insn->asCmp());
|
|
break;
|
|
case OP_MIN:
|
|
case OP_MAX:
|
|
emitMINMAX(insn);
|
|
break;
|
|
case OP_ABS:
|
|
case OP_NEG:
|
|
case OP_CEIL:
|
|
case OP_FLOOR:
|
|
case OP_TRUNC:
|
|
case OP_SAT:
|
|
emitCVT(insn);
|
|
break;
|
|
case OP_CVT:
|
|
if (insn->def(0).getFile() == FILE_PREDICATE ||
|
|
insn->src(0).getFile() == FILE_PREDICATE)
|
|
emitMOV(insn);
|
|
else
|
|
emitCVT(insn);
|
|
break;
|
|
case OP_RSQ:
|
|
emitSFnOp(insn, 5 + 2 * insn->subOp);
|
|
break;
|
|
case OP_RCP:
|
|
emitSFnOp(insn, 4 + 2 * insn->subOp);
|
|
break;
|
|
case OP_LG2:
|
|
emitSFnOp(insn, 3);
|
|
break;
|
|
case OP_EX2:
|
|
emitSFnOp(insn, 2);
|
|
break;
|
|
case OP_SIN:
|
|
emitSFnOp(insn, 1);
|
|
break;
|
|
case OP_COS:
|
|
emitSFnOp(insn, 0);
|
|
break;
|
|
case OP_PRESIN:
|
|
case OP_PREEX2:
|
|
emitPreOp(insn);
|
|
break;
|
|
case OP_TEX:
|
|
case OP_TXB:
|
|
case OP_TXL:
|
|
case OP_TXD:
|
|
case OP_TXF:
|
|
case OP_TXG:
|
|
case OP_TXLQ:
|
|
emitTEX(insn->asTex());
|
|
break;
|
|
case OP_TXQ:
|
|
emitTXQ(insn->asTex());
|
|
break;
|
|
case OP_TEXBAR:
|
|
emitTEXBAR(insn);
|
|
break;
|
|
case OP_PIXLD:
|
|
emitPIXLD(insn);
|
|
break;
|
|
case OP_BRA:
|
|
case OP_CALL:
|
|
case OP_PRERET:
|
|
case OP_RET:
|
|
case OP_DISCARD:
|
|
case OP_EXIT:
|
|
case OP_PRECONT:
|
|
case OP_CONT:
|
|
case OP_PREBREAK:
|
|
case OP_BREAK:
|
|
case OP_JOINAT:
|
|
case OP_BRKPT:
|
|
case OP_QUADON:
|
|
case OP_QUADPOP:
|
|
emitFlow(insn);
|
|
break;
|
|
case OP_QUADOP:
|
|
emitQUADOP(insn, insn->subOp, insn->lanes);
|
|
break;
|
|
case OP_DFDX:
|
|
emitQUADOP(insn, insn->src(0).mod.neg() ? 0x66 : 0x99, 0x4);
|
|
break;
|
|
case OP_DFDY:
|
|
emitQUADOP(insn, insn->src(0).mod.neg() ? 0x5a : 0xa5, 0x5);
|
|
break;
|
|
case OP_POPCNT:
|
|
emitPOPC(insn);
|
|
break;
|
|
case OP_INSBF:
|
|
emitINSBF(insn);
|
|
break;
|
|
case OP_EXTBF:
|
|
emitEXTBF(insn);
|
|
break;
|
|
case OP_BFIND:
|
|
emitBFIND(insn);
|
|
break;
|
|
case OP_PERMT:
|
|
emitPERMT(insn);
|
|
break;
|
|
case OP_JOIN:
|
|
emitNOP(insn);
|
|
insn->join = 1;
|
|
break;
|
|
case OP_BAR:
|
|
emitBAR(insn);
|
|
break;
|
|
case OP_MEMBAR:
|
|
emitMEMBAR(insn);
|
|
break;
|
|
case OP_ATOM:
|
|
emitATOM(insn);
|
|
break;
|
|
case OP_CCTL:
|
|
emitCCTL(insn);
|
|
break;
|
|
case OP_SHFL:
|
|
emitSHFL(insn);
|
|
break;
|
|
case OP_VOTE:
|
|
emitVOTE(insn);
|
|
break;
|
|
case OP_SULDB:
|
|
emitSULDGB(insn->asTex());
|
|
break;
|
|
case OP_SUSTB:
|
|
case OP_SUSTP:
|
|
emitSUSTGx(insn->asTex());
|
|
break;
|
|
case OP_SUBFM:
|
|
case OP_SUCLAMP:
|
|
case OP_SUEAU:
|
|
emitSUCalc(insn);
|
|
break;
|
|
case OP_VSHL:
|
|
emitVSHL(insn);
|
|
break;
|
|
case OP_PHI:
|
|
case OP_UNION:
|
|
case OP_CONSTRAINT:
|
|
ERROR("operation should have been eliminated");
|
|
return false;
|
|
case OP_EXP:
|
|
case OP_LOG:
|
|
case OP_SQRT:
|
|
case OP_POW:
|
|
ERROR("operation should have been lowered\n");
|
|
return false;
|
|
default:
|
|
ERROR("unknown op: %u\n", insn->op);
|
|
return false;
|
|
}
|
|
|
|
if (insn->join)
|
|
code[0] |= 1 << 22;
|
|
|
|
code += 2;
|
|
codeSize += 8;
|
|
return true;
|
|
}
|
|
|
|
uint32_t
|
|
CodeEmitterGK110::getMinEncodingSize(const Instruction *i) const
|
|
{
|
|
// No more short instruction encodings.
|
|
return 8;
|
|
}
|
|
|
|
void
|
|
CodeEmitterGK110::prepareEmission(Function *func)
|
|
{
|
|
const Target *targ = func->getProgram()->getTarget();
|
|
|
|
CodeEmitter::prepareEmission(func);
|
|
|
|
if (targ->hasSWSched)
|
|
calculateSchedDataNVC0(targ, func);
|
|
}
|
|
|
|
CodeEmitterGK110::CodeEmitterGK110(const TargetNVC0 *target, Program::Type type)
|
|
: CodeEmitter(target),
|
|
targNVC0(target),
|
|
progType(type),
|
|
writeIssueDelays(target->hasSWSched)
|
|
{
|
|
code = NULL;
|
|
codeSize = codeSizeLimit = 0;
|
|
relocInfo = NULL;
|
|
}
|
|
|
|
CodeEmitter *
|
|
TargetNVC0::createCodeEmitterGK110(Program::Type type)
|
|
{
|
|
CodeEmitterGK110 *emit = new CodeEmitterGK110(this, type);
|
|
return emit;
|
|
}
|
|
|
|
} // namespace nv50_ir
|