hwintrinsiccodegenxarch.cpp

// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

/*XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
XX XX
XX Intel hardware intrinsic Code Generator XX
XX XX
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
*/
#include"jitpch.h"
#ifdef _MSC_VER
#pragma hdrstop
#endif

#ifdef FEATURE_HW_INTRINSICS

#include"emit.h"
#include"codegen.h"
#include"sideeffects.h"
#include"lower.h"
#include"gcinfo.h"
#include"gcinfoencoder.h"

//------------------------------------------------------------------------
// assertIsContainableHWIntrinsicOp: Asserts that op is containable by node
//
// Arguments:
// lowering - The lowering phase from the compiler
// node - The HWIntrinsic node that has the contained node
// op - The op that is contained
//
staticvoidassertIsContainableHWIntrinsicOp(Lowering* lowering, GenTreeHWIntrinsic* node, GenTree* op)
{
#if DEBUG
// The Lowering::IsContainableHWIntrinsicOp call is not quite right, since it follows pre-register allocation
// logic. However, this check is still important due to the various containment rules that SIMD intrinsics follow.
//
// We use isContainable to track the special HWIntrinsic node containment rules (for things like LoadAligned and
// LoadUnaligned) and we use the supportsRegOptional check to support general-purpose loads (both from stack
// spillage
// and for isUsedFromMemory contained nodes, in the case where the register allocator decided to not allocate a
// register
// in the first place).

bool supportsRegOptional = false;
bool isContainable = lowering->IsContainableHWIntrinsicOp(node, op, &supportsRegOptional);
assert(isContainable || supportsRegOptional);
#endif// DEBUG
}

//------------------------------------------------------------------------
// genIsTableDrivenHWIntrinsic:
//
// Arguments:
// category - category of a HW intrinsic
//
// Return Value:
// returns true if this category can be table-driven in CodeGen
//
staticboolgenIsTableDrivenHWIntrinsic(NamedIntrinsic intrinsicId, HWIntrinsicCategory category)
{
// TODO - make more categories to the table-driven framework
// HW_Category_Helper and HW_Flag_MultiIns/HW_Flag_SpecialCodeGen usually need manual codegen
constbool tableDrivenCategory =
 (category != HW_Category_Special) && (category != HW_Category_Scalar) && (category != HW_Category_Helper);
constbool tableDrivenFlag =
 !HWIntrinsicInfo::GeneratesMultipleIns(intrinsicId) && !HWIntrinsicInfo::HasSpecialCodegen(intrinsicId);
return tableDrivenCategory && tableDrivenFlag;
}

//------------------------------------------------------------------------
// genHWIntrinsic: Generates the code for a given hardware intrinsic node.
//
// Arguments:
// node - The hardware intrinsic node
//
voidCodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
{
 NamedIntrinsic intrinsicId = node->gtHWIntrinsicId;
 InstructionSet isa = HWIntrinsicInfo::lookupIsa(intrinsicId);
 HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsicId);
int ival = HWIntrinsicInfo::lookupIval(intrinsicId);
int numArgs = HWIntrinsicInfo::lookupNumArgs(node);

assert(HWIntrinsicInfo::RequiresCodegen(intrinsicId));

if (genIsTableDrivenHWIntrinsic(intrinsicId, category))
 {
 GenTree* op1 = node->gtGetOp1();
 GenTree* op2 = node->gtGetOp2();
 regNumber targetReg = node->GetRegNum();
 var_types targetType = node->TypeGet();
 var_types baseType = node->gtSIMDBaseType;

 regNumber op1Reg = REG_NA;
 regNumber op2Reg = REG_NA;
 emitter* emit = GetEmitter();

assert(numArgs >= 0);
 instruction ins = HWIntrinsicInfo::lookupIns(intrinsicId, baseType);
assert(ins != INS_invalid);
 emitAttr simdSize = EA_ATTR(node->gtSIMDSize);
assert(simdSize != 0);

switch (numArgs)
 {
case1:
 {
if (node->OperIsMemoryLoad())
 {
genConsumeAddress(op1);
// Until we improve the handling of addressing modes in the emitter, we'll create a
// temporary GT_IND to generate code with.
 GenTreeIndir load = indirForm(node->TypeGet(), op1);
 emit->emitInsLoadInd(ins, simdSize, node->GetRegNum(), &load);
 }
else
 {
genConsumeRegs(op1);
 op1Reg = op1->GetRegNum();

if ((ival != -1) && varTypeIsFloating(baseType))
 {
assert((ival >= 0) && (ival <= 127));
if ((category == HW_Category_SIMDScalar) && HWIntrinsicInfo::CopiesUpperBits(intrinsicId))
 {
assert(!op1->isContained());
 emit->emitIns_SIMD_R_R_R_I(ins, simdSize, targetReg, op1Reg, op1Reg,
static_cast<int8_t>(ival));
 }
else
 {
genHWIntrinsic_R_RM_I(node, ins, static_cast<int8_t>(ival));
 }
 }
elseif ((category == HW_Category_SIMDScalar) && HWIntrinsicInfo::CopiesUpperBits(intrinsicId))
 {
 emit->emitIns_SIMD_R_R_R(ins, simdSize, targetReg, op1Reg, op1Reg);
 }
else
 {
genHWIntrinsic_R_RM(node, ins, simdSize, targetReg, op1);
 }
 }
break;
 }

case2:
 {
if (category == HW_Category_MemoryStore)
 {
genConsumeAddress(op1);

if (((intrinsicId == NI_SSE_Store) || (intrinsicId == NI_SSE2_Store)) && op2->isContained())
 {
 GenTreeHWIntrinsic* extract = op2->AsHWIntrinsic();

assert((extract->gtHWIntrinsicId == NI_AVX_ExtractVector128) ||
 (extract->gtHWIntrinsicId == NI_AVX2_ExtractVector128));

 regNumber regData = genConsumeReg(extract->gtGetOp1());

 ins = HWIntrinsicInfo::lookupIns(extract->gtHWIntrinsicId, extract->gtSIMDBaseType);
 ival = static_cast<int>(extract->gtGetOp2()->AsIntCon()->IconValue());

 GenTreeIndir indir = indirForm(TYP_SIMD16, op1);
 emit->emitIns_A_R_I(ins, EA_32BYTE, &indir, regData, ival);
 }
else
 {
genConsumeReg(op2);
// Until we improve the handling of addressing modes in the emitter, we'll create a
// temporary GT_STORE_IND to generate code with.
 GenTreeStoreInd store = storeIndirForm(node->TypeGet(), op1, op2);
 emit->emitInsStoreInd(ins, simdSize, &store);
 }
break;
 }
genConsumeRegs(op1);
genConsumeRegs(op2);

 op1Reg = op1->GetRegNum();
 op2Reg = op2->GetRegNum();

if ((op1Reg != targetReg) && (op2Reg == targetReg) && node->isRMWHWIntrinsic(compiler))
 {
// We have "reg2 = reg1 op reg2" where "reg1 != reg2" on a RMW intrinsic.
//
// For non-commutative intrinsics, we should have ensured that op2 was marked
// delay free in order to prevent it from getting assigned the same register
// as target. However, for commutative intrinsics, we can just swap the operands
// in order to have "reg2 = reg2 op reg1" which will end up producing the right code.

noway_assert(node->OperIsCommutative());
 op2Reg = op1Reg;
 op1Reg = targetReg;
 }

if ((ival != -1) && varTypeIsFloating(baseType))
 {
assert((ival >= 0) && (ival <= 127));
genHWIntrinsic_R_R_RM_I(node, ins, static_cast<int8_t>(ival));
 }
elseif (category == HW_Category_MemoryLoad)
 {
// Get the address and the 'other' register.
 GenTree* addr;
 regNumber otherReg;
if (intrinsicId == NI_AVX_MaskLoad || intrinsicId == NI_AVX2_MaskLoad)
 {
 addr = op1;
 otherReg = op2Reg;
 }
else
 {
 addr = op2;
 otherReg = op1Reg;
 }
// Until we improve the handling of addressing modes in the emitter, we'll create a
// temporary GT_IND to generate code with.
 GenTreeIndir load = indirForm(node->TypeGet(), addr);
genHWIntrinsic_R_R_RM(node, ins, simdSize, targetReg, otherReg, &load);
 }
elseif (HWIntrinsicInfo::isImmOp(intrinsicId, op2))
 {
assert(ival == -1);

if (intrinsicId == NI_SSE2_Extract)
 {
// extract instructions return to GP-registers, so it needs int size as the emitsize
 simdSize = emitTypeSize(TYP_INT);
 }

auto emitSwCase = [&](int8_t i) { genHWIntrinsic_R_RM_I(node, ins, i); };

if (op2->IsCnsIntOrI())
 {
ssize_t ival = op2->AsIntCon()->IconValue();
assert((ival >= 0) && (ival <= 255));
emitSwCase((int8_t)ival);
 }
else
 {
// We emit a fallback case for the scenario when the imm-op is not a constant. This should
// normally happen when the intrinsic is called indirectly, such as via Reflection. However, it
// can also occur if the consumer calls it directly and just doesn't pass a constant value.
 regNumber baseReg = node->ExtractTempReg();
 regNumber offsReg = node->GetSingleTempReg();
genHWIntrinsicJumpTableFallback(intrinsicId, op2Reg, baseReg, offsReg, emitSwCase);
 }
 }
elseif (node->TypeGet() == TYP_VOID)
 {
genHWIntrinsic_R_RM(node, ins, EA_ATTR(node->gtSIMDSize), op1Reg, op2);
 }
else
 {
genHWIntrinsic_R_R_RM(node, ins, EA_ATTR(node->gtSIMDSize));
 }
break;
 }

case3:
 {
 GenTreeArgList* argList = op1->AsArgList();
 op1 = argList->Current();
genConsumeRegs(op1);
 op1Reg = op1->GetRegNum();

 argList = argList->Rest();
 op2 = argList->Current();
genConsumeRegs(op2);
 op2Reg = op2->GetRegNum();

 argList = argList->Rest();
 GenTree* op3 = argList->Current();
genConsumeRegs(op3);
 regNumber op3Reg = op3->GetRegNum();

if (HWIntrinsicInfo::isImmOp(intrinsicId, op3))
 {
assert(ival == -1);

auto emitSwCase = [&](int8_t i) { genHWIntrinsic_R_R_RM_I(node, ins, i); };

if (op3->IsCnsIntOrI())
 {
ssize_t ival = op3->AsIntCon()->IconValue();
assert((ival >= 0) && (ival <= 255));
emitSwCase((int8_t)ival);
 }
else
 {
// We emit a fallback case for the scenario when the imm-op is not a constant. This should
// normally happen when the intrinsic is called indirectly, such as via Reflection. However, it
// can also occur if the consumer calls it directly and just doesn't pass a constant value.
 regNumber baseReg = node->ExtractTempReg();
 regNumber offsReg = node->GetSingleTempReg();
genHWIntrinsicJumpTableFallback(intrinsicId, op3Reg, baseReg, offsReg, emitSwCase);
 }
 }
elseif (category == HW_Category_MemoryStore)
 {
// The Mask instructions do not currently support containment of the address.
assert(!op2->isContained());
if (intrinsicId == NI_AVX_MaskStore || intrinsicId == NI_AVX2_MaskStore)
 {
 emit->emitIns_AR_R_R(ins, simdSize, op2Reg, op3Reg, op1Reg, 0);
 }
else
 {
assert(intrinsicId == NI_SSE2_MaskMove);
assert(targetReg == REG_NA);

// SSE2 MaskMove hardcodes the destination (op3) in DI/EDI/RDI
if (op3Reg != REG_EDI)
 {
 emit->emitIns_R_R(INS_mov, EA_PTRSIZE, REG_EDI, op3Reg);
 }
 emit->emitIns_R_R(ins, simdSize, op1Reg, op2Reg);
 }
 }
else
 {
switch (intrinsicId)
 {
case NI_SSE41_BlendVariable:
case NI_AVX_BlendVariable:
case NI_AVX2_BlendVariable:
 {
genHWIntrinsic_R_R_RM_R(node, ins);
break;
 }

default:
 {
unreached();
break;
 };
 }
 }
break;
 }

default:
unreached();
break;
 }
genProduceReg(node);
return;
 }

switch (isa)
 {
case InstructionSet_Vector128:
case InstructionSet_Vector256:
genBaseIntrinsic(node);
break;
case InstructionSet_SSE:
case InstructionSet_SSE_X64:
genSSEIntrinsic(node);
break;
case InstructionSet_SSE2:
case InstructionSet_SSE2_X64:
genSSE2Intrinsic(node);
break;
case InstructionSet_SSE41:
case InstructionSet_SSE41_X64:
genSSE41Intrinsic(node);
break;
case InstructionSet_SSE42:
case InstructionSet_SSE42_X64:
genSSE42Intrinsic(node);
break;
case InstructionSet_AVX:
case InstructionSet_AVX2:
genAvxOrAvx2Intrinsic(node);
break;
case InstructionSet_AES:
genAESIntrinsic(node);
break;
case InstructionSet_BMI1:
case InstructionSet_BMI1_X64:
case InstructionSet_BMI2:
case InstructionSet_BMI2_X64:
genBMI1OrBMI2Intrinsic(node);
break;
case InstructionSet_FMA:
genFMAIntrinsic(node);
break;
case InstructionSet_LZCNT:
case InstructionSet_LZCNT_X64:
genLZCNTIntrinsic(node);
break;
case InstructionSet_PCLMULQDQ:
genPCLMULQDQIntrinsic(node);
break;
case InstructionSet_POPCNT:
case InstructionSet_POPCNT_X64:
genPOPCNTIntrinsic(node);
break;
default:
unreached();
break;
 }
}

//------------------------------------------------------------------------
// genHWIntrinsic_R_RM: Generates code for a hardware intrinsic node that takes a
// register operand and a register/memory operand.
//
// Arguments:
// node - The hardware intrinsic node
// ins - The instruction being generated
// attr - The emit attribute for the instruciton being generated
// reg - The register
// rmOp - The register/memory operand node
//
voidCodeGen::genHWIntrinsic_R_RM(
 GenTreeHWIntrinsic* node, instruction ins, emitAttr attr, regNumber reg, GenTree* rmOp)
{
 emitter* emit = GetEmitter();

assert(reg != REG_NA);

if (rmOp->isContained() || rmOp->isUsedFromSpillTemp())
 {
assert(HWIntrinsicInfo::SupportsContainment(node->gtHWIntrinsicId));
assertIsContainableHWIntrinsicOp(compiler->m_pLowering, node, rmOp);

 TempDsc* tmpDsc = nullptr;
unsigned varNum = BAD_VAR_NUM;
unsigned offset = (unsigned)-1;

if (rmOp->isUsedFromSpillTemp())
 {
assert(rmOp->IsRegOptional());

 tmpDsc = getSpillTempDsc(rmOp);
 varNum = tmpDsc->tdTempNum();
 offset = 0;

 regSet.tmpRlsTemp(tmpDsc);
 }
elseif (rmOp->isIndir() || rmOp->OperIsHWIntrinsic())
 {
 GenTree* addr;
 GenTreeIndir* memIndir = nullptr;

if (rmOp->isIndir())
 {
 memIndir = rmOp->AsIndir();
 addr = memIndir->Addr();
 }
else
 {
assert(rmOp->AsHWIntrinsic()->OperIsMemoryLoad());
assert(HWIntrinsicInfo::lookupNumArgs(rmOp->AsHWIntrinsic()) == 1);
 addr = rmOp->gtGetOp1();
 }

switch (addr->OperGet())
 {
case GT_LCL_VAR_ADDR:
 {
 varNum = addr->AsLclVarCommon()->GetLclNum();
 offset = 0;
break;
 }

case GT_CLS_VAR_ADDR:
 {
 emit->emitIns_R_C(ins, attr, reg, addr->AsClsVar()->gtClsVarHnd, 0);
return;
 }

default:
 {
 GenTreeIndir load = indirForm(rmOp->TypeGet(), addr);

if (memIndir == nullptr)
 {
// This is the HW intrinsic load case.
// Until we improve the handling of addressing modes in the emitter, we'll create a
// temporary GT_IND to generate code with.
 memIndir = &load;
 }
 emit->emitIns_R_A(ins, attr, reg, memIndir);
return;
 }
 }
 }
else
 {
switch (rmOp->OperGet())
 {
case GT_LCL_FLD:
 {
 GenTreeLclFld* lclField = rmOp->AsLclFld();

 varNum = lclField->GetLclNum();
 offset = lclField->gtLclOffs;
break;
 }

case GT_LCL_VAR:
 {
assert(rmOp->IsRegOptional() || !compiler->lvaGetDesc(rmOp->AsLclVar())->lvIsRegCandidate());
 varNum = rmOp->AsLclVar()->GetLclNum();
 offset = 0;
break;
 }

default:
 {
unreached();
break;
 }
 }
 }

// Ensure we got a good varNum and offset.
// We also need to check for `tmpDsc != nullptr` since spill temp numbers
// are negative and start with -1, which also happens to be BAD_VAR_NUM.
assert((varNum != BAD_VAR_NUM) || (tmpDsc != nullptr));
assert(offset != (unsigned)-1);

 emit->emitIns_R_S(ins, attr, reg, varNum, offset);
 }
else
 {
 emit->emitIns_R_R(ins, attr, reg, rmOp->GetRegNum());
 }
}

//------------------------------------------------------------------------
// genHWIntrinsic_R_RM_I: Generates the code for a hardware intrinsic node that takes a register/memory operand,
// an immediate operand, and that returns a value in register
//
// Arguments:
// node - The hardware intrinsic node
// ins - The instruction being generated
// ival - The immediate value
//
voidCodeGen::genHWIntrinsic_R_RM_I(GenTreeHWIntrinsic* node, instruction ins, int8_t ival)
{
 var_types targetType = node->TypeGet();
 regNumber targetReg = node->GetRegNum();
 GenTree* op1 = node->gtGetOp1();
 emitAttr simdSize = EA_ATTR(node->gtSIMDSize);
 emitter* emit = GetEmitter();

// TODO-XArch-CQ: Commutative operations can have op1 be contained
// TODO-XArch-CQ: Non-VEX encoded instructions can have both ops contained

assert(targetReg != REG_NA);
assert(!node->OperIsCommutative()); // One operand intrinsics cannot be commutative

if (op1->isContained() || op1->isUsedFromSpillTemp())
 {
assert(HWIntrinsicInfo::SupportsContainment(node->gtHWIntrinsicId));
assertIsContainableHWIntrinsicOp(compiler->m_pLowering, node, op1);
 }
inst_RV_TT_IV(ins, simdSize, targetReg, op1, ival);
}

//------------------------------------------------------------------------
// genHWIntrinsic_R_R_RM: Generates the code for a hardware intrinsic node that takes a register operand, a
// register/memory operand, and that returns a value in register
//
// Arguments:
// node - The hardware intrinsic node
// ins - The instruction being generated
// attr - The emit attribute for the instruciton being generated
//
voidCodeGen::genHWIntrinsic_R_R_RM(GenTreeHWIntrinsic* node, instruction ins, emitAttr attr)
{
 regNumber targetReg = node->GetRegNum();
 GenTree* op1 = node->gtGetOp1();
 GenTree* op2 = node->gtGetOp2();
 regNumber op1Reg = op1->GetRegNum();

assert(targetReg != REG_NA);
assert(op1Reg != REG_NA);

genHWIntrinsic_R_R_RM(node, ins, attr, targetReg, op1Reg, op2);
}

//------------------------------------------------------------------------
// genHWIntrinsic_R_R_RM: Generates the code for a hardware intrinsic node that takes a register operand, a
// register/memory operand, and that returns a value in register
//
// Arguments:
// node - The hardware intrinsic node
// ins - The instruction being generated
// attr - The emit attribute for the instruciton being generated
// targetReg - The register allocated to the result
// op1Reg - The register allocated to the first operand
// op2 - Another operand that maybe in register or memory
//
voidCodeGen::genHWIntrinsic_R_R_RM(
 GenTreeHWIntrinsic* node, instruction ins, emitAttr attr, regNumber targetReg, regNumber op1Reg, GenTree* op2)
{
 emitter* emit = GetEmitter();

// TODO-XArch-CQ: Commutative operations can have op1 be contained
// TODO-XArch-CQ: Non-VEX encoded instructions can have both ops contained

assert(targetReg != REG_NA);
assert(op1Reg != REG_NA);

if (op2->isContained() || op2->isUsedFromSpillTemp())
 {
assert(HWIntrinsicInfo::SupportsContainment(node->gtHWIntrinsicId));
assertIsContainableHWIntrinsicOp(compiler->m_pLowering, node, op2);

 TempDsc* tmpDsc = nullptr;
unsigned varNum = BAD_VAR_NUM;
unsigned offset = (unsigned)-1;

if (op2->isUsedFromSpillTemp())
 {
assert(op2->IsRegOptional());

 tmpDsc = getSpillTempDsc(op2);
 varNum = tmpDsc->tdTempNum();
 offset = 0;

 regSet.tmpRlsTemp(tmpDsc);
 }
elseif (op2->isIndir() || op2->OperIsHWIntrinsic())
 {
 GenTree* addr;
 GenTreeIndir* memIndir = nullptr;

if (op2->isIndir())
 {
 memIndir = op2->AsIndir();
 addr = memIndir->Addr();
 }
else
 {
assert(op2->AsHWIntrinsic()->OperIsMemoryLoad());
assert(HWIntrinsicInfo::lookupNumArgs(op2->AsHWIntrinsic()) == 1);
 addr = op2->gtGetOp1();
 }

switch (addr->OperGet())
 {
case GT_LCL_VAR_ADDR:
 {
 varNum = addr->AsLclVarCommon()->GetLclNum();
 offset = 0;
break;
 }

case GT_CLS_VAR_ADDR:
 {
 emit->emitIns_SIMD_R_R_C(ins, attr, targetReg, op1Reg, addr->AsClsVar()->gtClsVarHnd, 0);
return;
 }

default:
 {
 GenTreeIndir load = indirForm(op2->TypeGet(), addr);

if (memIndir == nullptr)
 {
// This is the HW intrinsic load case.
// Until we improve the handling of addressing modes in the emitter, we'll create a
// temporary GT_IND to generate code with.
 memIndir = &load;
 }
 emit->emitIns_SIMD_R_R_A(ins, attr, targetReg, op1Reg, memIndir);
return;
 }
 }
 }
else
 {
switch (op2->OperGet())
 {
case GT_LCL_FLD:
 {
 GenTreeLclFld* lclField = op2->AsLclFld();

 varNum = lclField->GetLclNum();
 offset = lclField->AsLclFld()->gtLclOffs;
break;
 }

case GT_LCL_VAR:
 {
assert(op2->IsRegOptional() ||
 !compiler->lvaTable[op2->AsLclVar()->GetLclNum()].lvIsRegCandidate());
 varNum = op2->AsLclVar()->GetLclNum();
 offset = 0;
break;
 }

default:
unreached();
break;
 }
 }

// Ensure we got a good varNum and offset.
// We also need to check for `tmpDsc != nullptr` since spill temp numbers
// are negative and start with -1, which also happens to be BAD_VAR_NUM.
assert((varNum != BAD_VAR_NUM) || (tmpDsc != nullptr));
assert(offset != (unsigned)-1);

 emit->emitIns_SIMD_R_R_S(ins, attr, targetReg, op1Reg, varNum, offset);
 }
else
 {
 regNumber op2Reg = op2->GetRegNum();

if ((op1Reg != targetReg) && (op2Reg == targetReg) && node->isRMWHWIntrinsic(compiler))
 {
// We have "reg2 = reg1 op reg2" where "reg1 != reg2" on a RMW intrinsic.
//
// For non-commutative intrinsics, we should have ensured that op2 was marked
// delay free in order to prevent it from getting assigned the same register
// as target. However, for commutative intrinsics, we can just swap the operands
// in order to have "reg2 = reg2 op reg1" which will end up producing the right code.

noway_assert(node->OperIsCommutative());
 op2Reg = op1Reg;
 op1Reg = targetReg;
 }

 emit->emitIns_SIMD_R_R_R(ins, attr, targetReg, op1Reg, op2Reg);
 }
}

//------------------------------------------------------------------------
// genHWIntrinsic_R_R_RM_I: Generates the code for a hardware intrinsic node that takes a register operand, a
// register/memory operand, an immediate operand, and that returns a value in register
//
// Arguments:
// node - The hardware intrinsic node
// ins - The instruction being generated
// ival - The immediate value
//
voidCodeGen::genHWIntrinsic_R_R_RM_I(GenTreeHWIntrinsic* node, instruction ins, int8_t ival)
{
 var_types targetType = node->TypeGet();
 regNumber targetReg = node->GetRegNum();
 GenTree* op1 = node->gtGetOp1();
 GenTree* op2 = node->gtGetOp2();
 emitAttr simdSize = EA_ATTR(node->gtSIMDSize);
 emitter* emit = GetEmitter();

// TODO-XArch-CQ: Commutative operations can have op1 be contained
// TODO-XArch-CQ: Non-VEX encoded instructions can have both ops contained

if (op1->OperIsList())
 {
assert(op2 == nullptr);

 GenTreeArgList* argList = op1->AsArgList();

 op1 = argList->Current();
 argList = argList->Rest();

 op2 = argList->Current();
 argList = argList->Rest();

assert(argList->Current() != nullptr);
assert(argList->Rest() == nullptr);
 }

 regNumber op1Reg = op1->GetRegNum();

assert(targetReg != REG_NA);
assert(op1Reg != REG_NA);

if (op2->isContained() || op2->isUsedFromSpillTemp())
 {
assert(HWIntrinsicInfo::SupportsContainment(node->gtHWIntrinsicId));
assertIsContainableHWIntrinsicOp(compiler->m_pLowering, node, op2);

 TempDsc* tmpDsc = nullptr;
unsigned varNum = BAD_VAR_NUM;
unsigned offset = (unsigned)-1;

if (op2->isUsedFromSpillTemp())
 {
assert(op2->IsRegOptional());

 tmpDsc = getSpillTempDsc(op2);
 varNum = tmpDsc->tdTempNum();
 offset = 0;

 regSet.tmpRlsTemp(tmpDsc);
 }
elseif (op2->isIndir() || op2->OperIsHWIntrinsic())
 {
 GenTree* addr;
 GenTreeIndir* memIndir = nullptr;

if (op2->isIndir())
 {
 memIndir = op2->AsIndir();
 addr = memIndir->Addr();
 }
else
 {
assert(op2->AsHWIntrinsic()->OperIsMemoryLoad());
assert(HWIntrinsicInfo::lookupNumArgs(op2->AsHWIntrinsic()) == 1);
 addr = op2->gtGetOp1();
 }

switch (addr->OperGet())
 {
case GT_LCL_VAR_ADDR:
 {
 varNum = addr->AsLclVarCommon()->GetLclNum();
 offset = 0;
break;
 }

case GT_CLS_VAR_ADDR:
 {
 emit->emitIns_SIMD_R_R_C_I(ins, simdSize, targetReg, op1Reg, addr->AsClsVar()->gtClsVarHnd, 0,
 ival);
return;
 }

default:
 {
 GenTreeIndir load = indirForm(op2->TypeGet(), addr);

if (memIndir == nullptr)
 {
// This is the HW intrinsic load case.
// Until we improve the handling of addressing modes in the emitter, we'll create a
// temporary GT_IND to generate code with.
 memIndir = &load;
 }
 emit->emitIns_SIMD_R_R_A_I(ins, simdSize, targetReg, op1Reg, memIndir, ival);
return;
 }
 }
 }
else
 {
switch (op2->OperGet())
 {
case GT_LCL_FLD:
 {
 GenTreeLclFld* lclField = op2->AsLclFld();

 varNum = lclField->GetLclNum();
 offset = lclField->AsLclFld()->gtLclOffs;
break;
 }

case GT_LCL_VAR:
 {
assert(op2->IsRegOptional() ||
 !compiler->lvaTable[op2->AsLclVar()->GetLclNum()].lvIsRegCandidate());
 varNum = op2->AsLclVar()->GetLclNum();
 offset = 0;
break;
 }

default:
unreached();
break;
 }
 }

// Ensure we got a good varNum and offset.
// We also need to check for `tmpDsc != nullptr` since spill temp numbers
// are negative and start with -1, which also happens to be BAD_VAR_NUM.
assert((varNum != BAD_VAR_NUM) || (tmpDsc != nullptr));
assert(offset != (unsigned)-1);

 emit->emitIns_SIMD_R_R_S_I(ins, simdSize, targetReg, op1Reg, varNum, offset, ival);
 }
else
 {
 regNumber op2Reg = op2->GetRegNum();

if ((op1Reg != targetReg) && (op2Reg == targetReg) && node->isRMWHWIntrinsic(compiler))
 {
// We have "reg2 = reg1 op reg2" where "reg1 != reg2" on a RMW intrinsic.
//
// For non-commutative intrinsics, we should have ensured that op2 was marked
// delay free in order to prevent it from getting assigned the same register
// as target. However, for commutative intrinsics, we can just swap the operands
// in order to have "reg2 = reg2 op reg1" which will end up producing the right code.

noway_assert(node->OperIsCommutative());
 op2Reg = op1Reg;
 op1Reg = targetReg;
 }

 emit->emitIns_SIMD_R_R_R_I(ins, simdSize, targetReg, op1Reg, op2Reg, ival);
 }
}

//------------------------------------------------------------------------
// genHWIntrinsic_R_R_RM_R: Generates the code for a hardware intrinsic node that takes a register operand, a
// register/memory operand, another register operand, and that returns a value in register
//
// Arguments:
// node - The hardware intrinsic node
// ins - The instruction being generated
//
voidCodeGen::genHWIntrinsic_R_R_RM_R(GenTreeHWIntrinsic* node, instruction ins)
{
 var_types targetType = node->TypeGet();
 regNumber targetReg = node->GetRegNum();
 GenTree* op1 = node->gtGetOp1();
 GenTree* op2 = node->gtGetOp2();
 GenTree* op3 = nullptr;
 emitAttr simdSize = EA_ATTR(node->gtSIMDSize);
 emitter* emit = GetEmitter();

assert(op1->OperIsList());
assert(op2 == nullptr);

 GenTreeArgList* argList = op1->AsArgList();

 op1 = argList->Current();
 argList = argList->Rest();

 op2 = argList->Current();
 argList = argList->Rest();

 op3 = argList->Current();
assert(argList->Rest() == nullptr);

 regNumber op1Reg = op1->GetRegNum();
 regNumber op3Reg = op3->GetRegNum();

assert(targetReg != REG_NA);
assert(op1Reg != REG_NA);
assert(op3Reg != REG_NA);

if (op2->isContained() || op2->isUsedFromSpillTemp())
 {
assert(HWIntrinsicInfo::SupportsContainment(node->gtHWIntrinsicId));
assertIsContainableHWIntrinsicOp(compiler->m_pLowering, node, op2);

 TempDsc* tmpDsc = nullptr;
unsigned varNum = BAD_VAR_NUM;
unsigned offset = (unsigned)-1;

if (op2->isUsedFromSpillTemp())
 {
assert(op2->IsRegOptional());

// TODO-XArch-Cleanup: The getSpillTempDsc...tempRlsTemp code is a fairly common
// pattern. It could probably be extracted to its own method.
 tmpDsc = getSpillTempDsc(op2);
 varNum = tmpDsc->tdTempNum();
 offset = 0;

 regSet.tmpRlsTemp(tmpDsc);
 }
elseif (op2->isIndir() || op2->OperIsHWIntrinsic())
 {
 GenTree* addr;
 GenTreeIndir* memIndir = nullptr;

if (op2->isIndir())
 {
 memIndir = op2->AsIndir();
 addr = memIndir->Addr();
 }
else
 {
assert(op2->AsHWIntrinsic()->OperIsMemoryLoad());
assert(HWIntrinsicInfo::lookupNumArgs(op2->AsHWIntrinsic()) == 1);
 addr = op2->gtGetOp1();
 }

switch (addr->OperGet())
 {
case GT_LCL_VAR_ADDR:
 {
 varNum = addr->AsLclVarCommon()->GetLclNum();
 offset = 0;
break;
 }

case GT_CLS_VAR_ADDR:
 {
 emit->emitIns_SIMD_R_R_C_R(ins, simdSize, targetReg, op1Reg, op3Reg, addr->AsClsVar()->gtClsVarHnd,
0);
return;
 }