Commit 8add0ee9 authored by Hans Wennborg's avatar Hans Wennborg
Browse files

Merging r226808: 

------------------------------------------------------------------------
r226808 | delena | 2015-01-22 04:07:59 -0800 (Thu, 22 Jan 2015) | 10 lines

Fixed a bug in type legalizer for masked load/store intrinsics.
The problem occurs when after vectorization we have type
<2 x i32>. This type is promoted to <2 x i64> and then requires
additional efforts for expanding loads and truncating stores.
I added EXPAND / TRUNCATE attributes to the masked load/store
SDNodes. The code now contains additional shuffles.
I've prepared changes in the cost estimation for masked memory
operations, it will be submitted separately.


------------------------------------------------------------------------

llvm-svn: 229561
parent 5ba91fb1

llvm/include/llvm/CodeGen/SelectionDAG.h

+4 −2
Original line number Diff line number Diff line
@@ -867,9 +867,11 @@ public: 
                           SDValue Offset, ISD::MemIndexedMode AM);



  SDValue getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr,

                        SDValue Mask, SDValue Src0, MachineMemOperand *MMO);

                        SDValue Mask, SDValue Src0, EVT MemVT,

                        MachineMemOperand *MMO, ISD::LoadExtType);

  SDValue getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val,

                         SDValue Ptr, SDValue Mask, MachineMemOperand *MMO);

                         SDValue Ptr, SDValue Mask, EVT MemVT,

                         MachineMemOperand *MMO, bool IsTrunc);

  /// getSrcValue - Construct a node to track a Value* through the backend.

  SDValue getSrcValue(const Value *v);

llvm/include/llvm/CodeGen/SelectionDAGNodes.h

+20 −11
Original line number Diff line number Diff line
@@ -1970,13 +1970,17 @@ public: 
class MaskedLoadSDNode : public MaskedLoadStoreSDNode {

public:

  friend class SelectionDAG;

  MaskedLoadSDNode(unsigned Order, DebugLoc dl,

                   SDValue *Operands, unsigned numOperands, 

                   SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)

  MaskedLoadSDNode(unsigned Order, DebugLoc dl, SDValue *Operands,

                   unsigned numOperands, SDVTList VTs, ISD::LoadExtType ETy,

                   EVT MemVT, MachineMemOperand *MMO)

    : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, Operands, numOperands,

                            VTs, MemVT, MMO) 

  {}

                            VTs, MemVT, MMO) {

    SubclassData |= (unsigned short)ETy;

  }



  ISD::LoadExtType getExtensionType() const {

    return ISD::LoadExtType(SubclassData & 3);

  } 

  const SDValue &getSrc0() const { return getOperand(3); }

  static bool classof(const SDNode *N) {

    return N->getOpcode() == ISD::MLOAD;
@@ -1989,14 +1993,19 @@ class MaskedStoreSDNode : public MaskedLoadStoreSDNode { 


public:

  friend class SelectionDAG;

  MaskedStoreSDNode(unsigned Order, DebugLoc dl,

                   SDValue *Operands, unsigned numOperands, 

                   SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)

  MaskedStoreSDNode(unsigned Order, DebugLoc dl, SDValue *Operands,

                    unsigned numOperands, SDVTList VTs, bool isTrunc, EVT MemVT,

                    MachineMemOperand *MMO)

    : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, Operands, numOperands,

                            VTs, MemVT, MMO) 

  {}

                            VTs, MemVT, MMO) {

      SubclassData |= (unsigned short)isTrunc;

  }

  /// isTruncatingStore - Return true if the op does a truncation before store.

  /// For integers this is the same as doing a TRUNCATE and storing the result.

  /// For floats, it is the same as doing an FP_ROUND and storing the result.

  bool isTruncatingStore() const { return SubclassData & 1; }



  const SDValue &getData() const { return getOperand(3); }

  const SDValue &getValue() const { return getOperand(3); }



  static bool classof(const SDNode *N) {

    return N->getOpcode() == ISD::MSTORE;

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

+11 −5
Original line number Diff line number Diff line
@@ -4842,7 +4842,7 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) { 


  MaskedStoreSDNode *MST = dyn_cast<MaskedStoreSDNode>(N);

  SDValue Mask = MST->getMask();

  SDValue Data  = MST->getData();

  SDValue Data  = MST->getValue();

  SDLoc DL(N);



  // If the MSTORE data type requires splitting and the mask is provided by a
@@ -4885,7 +4885,8 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) { 
                           MachineMemOperand::MOStore,  LoMemVT.getStoreSize(),

                           Alignment, MST->getAAInfo(), MST->getRanges());



    Lo = DAG.getMaskedStore(Chain, DL, DataLo, Ptr, MaskLo, MMO);

    Lo = DAG.getMaskedStore(Chain, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,

                            MST->isTruncatingStore());



    unsigned IncrementSize = LoMemVT.getSizeInBits()/8;

    Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
@@ -4897,7 +4898,8 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) { 
                           SecondHalfAlignment, MST->getAAInfo(),

                           MST->getRanges());



    Hi = DAG.getMaskedStore(Chain, DL, DataHi, Ptr, MaskHi, MMO);

    Hi = DAG.getMaskedStore(Chain, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,

                            MST->isTruncatingStore());



    AddToWorklist(Lo.getNode());

    AddToWorklist(Hi.getNode());
@@ -4958,7 +4960,8 @@ SDValue DAGCombiner::visitMLOAD(SDNode *N) { 
                         MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),

                         Alignment, MLD->getAAInfo(), MLD->getRanges());



    Lo = DAG.getMaskedLoad(LoVT, DL, Chain, Ptr, MaskLo, Src0Lo, MMO);

    Lo = DAG.getMaskedLoad(LoVT, DL, Chain, Ptr, MaskLo, Src0Lo, LoMemVT, MMO,

                           ISD::NON_EXTLOAD);



    unsigned IncrementSize = LoMemVT.getSizeInBits()/8;

    Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
@@ -4969,7 +4972,8 @@ SDValue DAGCombiner::visitMLOAD(SDNode *N) { 
                         MachineMemOperand::MOLoad,  HiMemVT.getStoreSize(),

                         SecondHalfAlignment, MLD->getAAInfo(), MLD->getRanges());



    Hi = DAG.getMaskedLoad(HiVT, DL, Chain, Ptr, MaskHi, Src0Hi, MMO);

    Hi = DAG.getMaskedLoad(HiVT, DL, Chain, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,

                           ISD::NON_EXTLOAD);



    AddToWorklist(Lo.getNode());

    AddToWorklist(Hi.getNode());
@@ -9482,6 +9486,8 @@ SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) { 
    unsigned MSB = BitWidth - Imm.countLeadingZeros() - 1;

    unsigned NewBW = NextPowerOf2(MSB - ShAmt);

    EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW);

    // The narrowing should be profitable, the load/store operation should be

    // legal (or custom) and the store size should be equal to the NewVT width.

    while (NewBW < BitWidth &&

           !(TLI.isOperationLegalOrCustom(Opc, NewVT) &&

             TLI.isNarrowingProfitable(VT, NewVT))) {

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

+13 −12
Original line number Diff line number Diff line
@@ -458,16 +458,16 @@ SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 
SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {

  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));

  SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0());

  SDValue ExtMask = PromoteTargetBoolean(N->getMask(), NVT);

  SDLoc dl(N);



  MachineMemOperand *MMO = DAG.getMachineFunction().

    getMachineMemOperand(N->getPointerInfo(),

                         MachineMemOperand::MOLoad,  NVT.getStoreSize(),

                         N->getAlignment(), N->getAAInfo(), N->getRanges());

  SDValue Mask = N->getMask();

  EVT NewMaskVT = getSetCCResultType(NVT);

  if (NewMaskVT != N->getMask().getValueType())

    Mask = PromoteTargetBoolean(Mask, NewMaskVT);

  SDLoc dl(N);



  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),

                                  ExtMask, ExtSrc0, MMO);

                                  Mask, ExtSrc0, N->getMemoryVT(),

                                  N->getMemOperand(), ISD::SEXTLOAD);

  // Legalized the chain result - switch anything that used the old chain to

  // use the new one.

  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
@@ -1117,16 +1117,18 @@ SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 
SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo){



  assert(OpNo == 2 && "Only know how to promote the mask!");

  SDValue DataOp = N->getData();

  SDValue DataOp = N->getValue();

  EVT DataVT = DataOp.getValueType();

  SDValue Mask = N->getMask();

  EVT MaskVT = Mask.getValueType();

  SDLoc dl(N);



  bool TruncateStore = false;

  if (!TLI.isTypeLegal(DataVT)) {

    if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger) {

      DataOp = GetPromotedInteger(DataOp);

      Mask = PromoteTargetBoolean(Mask, DataOp.getValueType());

      TruncateStore = true;

    }

    else {

      assert(getTypeAction(DataVT) == TargetLowering::TypeWidenVector &&
@@ -1156,10 +1158,9 @@ SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpN 
  }

  else

    Mask = PromoteTargetBoolean(N->getMask(), DataOp.getValueType());

  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());

  NewOps[2] = Mask;

  NewOps[3] = DataOp;

  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);

  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,

                            N->getMemoryVT(), N->getMemOperand(),

                            TruncateStore);

}



SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo){

llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

+1 −0
Original line number Diff line number Diff line
@@ -659,6 +659,7 @@ private: 
  SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);

  SDValue WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N);

  SDValue WidenVecOp_STORE(SDNode* N);

  SDValue WidenVecOp_MSTORE(SDNode* N, unsigned OpNo);

  SDValue WidenVecOp_SETCC(SDNode* N);



  SDValue WidenVecOp_Convert(SDNode *N);