Commit 79f2422d authored by Sander de Smalen's avatar Sander de Smalen
Browse files

[Aarch64][SVE] Add intrinsics for gather loads (vector + imm) 

This patch adds intrinsics for SVE gather loads from memory addresses generated by a vector base plus immediate index:
  * @llvm.aarch64.sve.ld1.gather.imm

This intrinsics maps 1-1 to the corresponding SVE instruction (example for half-words):
  * ld1h { z0.d }, p0/z, [z0.d, #16]

Committed on behalf of Andrzej Warzynski (andwar)

Reviewers: sdesmalen, huntergr, kmclaughlin, eli.friedman, rengolin, rovka, dancgr, mgudim, efriedma

Reviewed By: sdesmalen

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D70806
parent 877ffa71

llvm/include/llvm/IR/IntrinsicsAArch64.td

+12 −0
Original line number Diff line number Diff line
@@ -990,6 +990,15 @@ class AdvSIMD_GatherLoad_32bitOffset_Intrinsic 


let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".



class AdvSIMD_GatherLoad_VecTorBase_Intrinsic

    : Intrinsic<[llvm_anyvector_ty],

                [

                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                  llvm_anyvector_ty,

                  llvm_i64_ty

                ],

                [IntrReadMem, IntrArgMemOnly]>;



//

// Integer arithmetic

//
@@ -1229,6 +1238,9 @@ def int_aarch64_sve_ld1_gather_uxtw : AdvSIMD_GatherLoad_32bitOffset_Intrinsic; 
def int_aarch64_sve_ld1_gather_sxtw_index : AdvSIMD_GatherLoad_32bitOffset_Intrinsic;

def int_aarch64_sve_ld1_gather_uxtw_index : AdvSIMD_GatherLoad_32bitOffset_Intrinsic;



// vector base + immediate index

def int_aarch64_sve_ld1_gather_imm : AdvSIMD_GatherLoad_VecTorBase_Intrinsic;



//

// SVE2 - Non-widening pairwise arithmetic

//

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

+3 −0
Original line number Diff line number Diff line
@@ -1342,6 +1342,7 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const { 
  case AArch64ISD::GLD1_UXTW:         return "AArch64ISD::GLD1_UXTW";

  case AArch64ISD::GLD1_SXTW_SCALED:  return "AArch64ISD::GLD1_SXTW_SCALED";

  case AArch64ISD::GLD1_UXTW_SCALED:  return "AArch64ISD::GLD1_UXTW_SCALED";

  case AArch64ISD::GLD1_IMM:          return "AArch64ISD::GLD1_IMM";

  }

  return nullptr;

}
@@ -11943,6 +11944,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N, 
      return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_SXTW_SCALED);

    case Intrinsic::aarch64_sve_ld1_gather_uxtw_index:

      return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_UXTW_SCALED);

    case Intrinsic::aarch64_sve_ld1_gather_imm:

      return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_IMM);

    default:

      break;

    }

llvm/lib/Target/AArch64/AArch64ISelLowering.h

+1 −0
Original line number Diff line number Diff line
@@ -205,6 +205,7 @@ enum NodeType : unsigned { 
  GLD1_SXTW,

  GLD1_UXTW_SCALED,

  GLD1_SXTW_SCALED,

  GLD1_IMM,



  // NEON Load/Store with post-increment base updates

  LD2post = ISD::FIRST_TARGET_MEMORY_OPCODE,

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

+33 −27
Original line number Diff line number Diff line
@@ -15,12 +15,18 @@ def SDT_AArch64_GLD1 : SDTypeProfile<1, 4, [ 
  SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>

]>;



def SDT_AArch64_GLD1_IMM : SDTypeProfile<1, 4, [

  SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisInt<3>, SDTCisVT<4, OtherVT>,

  SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>

]>;



def AArch64ld1_gather                : SDNode<"AArch64ISD::GLD1",               SDT_AArch64_GLD1,     [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;

def AArch64ld1_gather_scaled         : SDNode<"AArch64ISD::GLD1_SCALED",        SDT_AArch64_GLD1,     [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;

def AArch64ld1_gather_uxtw           : SDNode<"AArch64ISD::GLD1_UXTW",          SDT_AArch64_GLD1,     [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;

def AArch64ld1_gather_sxtw           : SDNode<"AArch64ISD::GLD1_SXTW",          SDT_AArch64_GLD1,     [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;

def AArch64ld1_gather_uxtw_scaled    : SDNode<"AArch64ISD::GLD1_UXTW_SCALED",   SDT_AArch64_GLD1,     [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;

def AArch64ld1_gather_sxtw_scaled    : SDNode<"AArch64ISD::GLD1_SXTW_SCALED",   SDT_AArch64_GLD1,     [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;

def AArch64ld1_gather_imm            : SDNode<"AArch64ISD::GLD1_IMM",           SDT_AArch64_GLD1_IMM, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;



let Predicates = [HasSVE] in {
@@ -434,35 +440,35 @@ let Predicates = [HasSVE] in { 
  defm GLD1W      : sve_mem_32b_gld_sv_32_scaled<0b1010, "ld1w",    AArch64ld1_gather_sxtw_scaled,  AArch64ld1_gather_uxtw_scaled,  ZPR32ExtSXTW32, ZPR32ExtUXTW32, nxv4i32>;

  defm GLDFF1W    : sve_mem_32b_gld_sv_32_scaled<0b1011, "ldff1w",  null_frag,                      null_frag,                      ZPR32ExtSXTW32, ZPR32ExtUXTW32, nxv4i32>;



  // Gathers using scaled 32-bit pointers with offset, e.g.

  // Gathers using 32-bit pointers with scaled offset, e.g.

  //    ld1h z0.s, p0/z, [z0.s, #16]

  defm GLD1SB_S   : sve_mem_32b_gld_vi_32_ptrs<0b0000, "ld1sb",   imm0_31>;

  defm GLDFF1SB_S : sve_mem_32b_gld_vi_32_ptrs<0b0001, "ldff1sb", imm0_31>;

  defm GLD1B_S    : sve_mem_32b_gld_vi_32_ptrs<0b0010, "ld1b",    imm0_31>;

  defm GLDFF1B_S  : sve_mem_32b_gld_vi_32_ptrs<0b0011, "ldff1b",  imm0_31>;

  defm GLD1SH_S   : sve_mem_32b_gld_vi_32_ptrs<0b0100, "ld1sh",   uimm5s2>;

  defm GLDFF1SH_S : sve_mem_32b_gld_vi_32_ptrs<0b0101, "ldff1sh", uimm5s2>;

  defm GLD1H_S    : sve_mem_32b_gld_vi_32_ptrs<0b0110, "ld1h",    uimm5s2>;

  defm GLDFF1H_S  : sve_mem_32b_gld_vi_32_ptrs<0b0111, "ldff1h",  uimm5s2>;

  defm GLD1W      : sve_mem_32b_gld_vi_32_ptrs<0b1010, "ld1w",    uimm5s4>;

  defm GLDFF1W    : sve_mem_32b_gld_vi_32_ptrs<0b1011, "ldff1w",  uimm5s4>;



  // Gathers using scaled 64-bit pointers with offset, e.g.

  defm GLD1SB_S   : sve_mem_32b_gld_vi_32_ptrs<0b0000, "ld1sb",   imm0_31, null_frag,               nxv4i8>;

  defm GLDFF1SB_S : sve_mem_32b_gld_vi_32_ptrs<0b0001, "ldff1sb", imm0_31, null_frag,               nxv4i8>;

  defm GLD1B_S    : sve_mem_32b_gld_vi_32_ptrs<0b0010, "ld1b",    imm0_31, AArch64ld1_gather_imm,   nxv4i8>;

  defm GLDFF1B_S  : sve_mem_32b_gld_vi_32_ptrs<0b0011, "ldff1b",  imm0_31, null_frag,               nxv4i8>;

  defm GLD1SH_S   : sve_mem_32b_gld_vi_32_ptrs<0b0100, "ld1sh",   uimm5s2, null_frag,               nxv4i16>;

  defm GLDFF1SH_S : sve_mem_32b_gld_vi_32_ptrs<0b0101, "ldff1sh", uimm5s2, null_frag,               nxv4i16>;

  defm GLD1H_S    : sve_mem_32b_gld_vi_32_ptrs<0b0110, "ld1h",    uimm5s2, AArch64ld1_gather_imm,   nxv4i16>;

  defm GLDFF1H_S  : sve_mem_32b_gld_vi_32_ptrs<0b0111, "ldff1h",  uimm5s2, null_frag,               nxv4i16>;

  defm GLD1W      : sve_mem_32b_gld_vi_32_ptrs<0b1010, "ld1w",    uimm5s4, AArch64ld1_gather_imm,   nxv4i32>;

  defm GLDFF1W    : sve_mem_32b_gld_vi_32_ptrs<0b1011, "ldff1w",  uimm5s4, null_frag,               nxv4i32>;



  // Gathers using 64-bit pointers with scaled offset, e.g.

  //    ld1h z0.d, p0/z, [z0.d, #16]

  defm GLD1SB_D   : sve_mem_64b_gld_vi_64_ptrs<0b0000, "ld1sb",   imm0_31>;

  defm GLDFF1SB_D : sve_mem_64b_gld_vi_64_ptrs<0b0001, "ldff1sb", imm0_31>;

  defm GLD1B_D    : sve_mem_64b_gld_vi_64_ptrs<0b0010, "ld1b",    imm0_31>;

  defm GLDFF1B_D  : sve_mem_64b_gld_vi_64_ptrs<0b0011, "ldff1b",  imm0_31>;

  defm GLD1SH_D   : sve_mem_64b_gld_vi_64_ptrs<0b0100, "ld1sh",   uimm5s2>;

  defm GLDFF1SH_D : sve_mem_64b_gld_vi_64_ptrs<0b0101, "ldff1sh", uimm5s2>;

  defm GLD1H_D    : sve_mem_64b_gld_vi_64_ptrs<0b0110, "ld1h",    uimm5s2>;

  defm GLDFF1H_D  : sve_mem_64b_gld_vi_64_ptrs<0b0111, "ldff1h",  uimm5s2>;

  defm GLD1SW_D   : sve_mem_64b_gld_vi_64_ptrs<0b1000, "ld1sw",   uimm5s4>;

  defm GLDFF1SW_D : sve_mem_64b_gld_vi_64_ptrs<0b1001, "ldff1sw", uimm5s4>;

  defm GLD1W_D    : sve_mem_64b_gld_vi_64_ptrs<0b1010, "ld1w",    uimm5s4>;

  defm GLDFF1W_D  : sve_mem_64b_gld_vi_64_ptrs<0b1011, "ldff1w",  uimm5s4>;

  defm GLD1D      : sve_mem_64b_gld_vi_64_ptrs<0b1110, "ld1d",    uimm5s8>;

  defm GLDFF1D    : sve_mem_64b_gld_vi_64_ptrs<0b1111, "ldff1d",  uimm5s8>;

  defm GLD1SB_D   : sve_mem_64b_gld_vi_64_ptrs<0b0000, "ld1sb",   imm0_31, null_frag,             nxv2i8>;

  defm GLDFF1SB_D : sve_mem_64b_gld_vi_64_ptrs<0b0001, "ldff1sb", imm0_31, null_frag,             nxv2i8>;

  defm GLD1B_D    : sve_mem_64b_gld_vi_64_ptrs<0b0010, "ld1b",    imm0_31, AArch64ld1_gather_imm, nxv2i8>;

  defm GLDFF1B_D  : sve_mem_64b_gld_vi_64_ptrs<0b0011, "ldff1b",  imm0_31, null_frag,             nxv2i8>;

  defm GLD1SH_D   : sve_mem_64b_gld_vi_64_ptrs<0b0100, "ld1sh",   uimm5s2, null_frag,             nxv2i16>;

  defm GLDFF1SH_D : sve_mem_64b_gld_vi_64_ptrs<0b0101, "ldff1sh", uimm5s2, null_frag,             nxv2i16>;

  defm GLD1H_D    : sve_mem_64b_gld_vi_64_ptrs<0b0110, "ld1h",    uimm5s2, AArch64ld1_gather_imm, nxv2i16>;

  defm GLDFF1H_D  : sve_mem_64b_gld_vi_64_ptrs<0b0111, "ldff1h",  uimm5s2, null_frag,             nxv2i16>;

  defm GLD1SW_D   : sve_mem_64b_gld_vi_64_ptrs<0b1000, "ld1sw",   uimm5s4, null_frag,             nxv2i32>;

  defm GLDFF1SW_D : sve_mem_64b_gld_vi_64_ptrs<0b1001, "ldff1sw", uimm5s4, null_frag,             nxv2i32>;

  defm GLD1W_D    : sve_mem_64b_gld_vi_64_ptrs<0b1010, "ld1w",    uimm5s4, AArch64ld1_gather_imm, nxv2i32>;

  defm GLDFF1W_D  : sve_mem_64b_gld_vi_64_ptrs<0b1011, "ldff1w",  uimm5s4, null_frag,             nxv2i32>;

  defm GLD1D      : sve_mem_64b_gld_vi_64_ptrs<0b1110, "ld1d",    uimm5s8, AArch64ld1_gather_imm, nxv2i64>;

  defm GLDFF1D    : sve_mem_64b_gld_vi_64_ptrs<0b1111, "ldff1d",  uimm5s8, null_frag,             nxv2i64>;



  // Gathers using unscaled 64-bit offsets, e.g.

  //    ld1h z0.d, p0/z, [x0, z0.d]

llvm/lib/Target/AArch64/SVEInstrFormats.td

+10 −2
Original line number Diff line number Diff line
@@ -5357,7 +5357,8 @@ class sve_mem_32b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 
  let Uses = !if(!eq(opc{0}, 1), [FFR], []);

}



multiclass sve_mem_32b_gld_vi_32_ptrs<bits<4> opc, string asm, Operand imm_ty> {

multiclass sve_mem_32b_gld_vi_32_ptrs<bits<4> opc, string asm, Operand imm_ty,

                                      SDPatternOperator op, ValueType vt> {

  def _IMM_REAL : sve_mem_32b_gld_vi<opc, asm, imm_ty>;



  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",
@@ -5366,6 +5367,9 @@ multiclass sve_mem_32b_gld_vi_32_ptrs<bits<4> opc, string asm, Operand imm_ty> { 
                  (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>;

  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",

                  (!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>;



  def : Pat<(nxv4i32 (op (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt)),

            (!cast<Instruction>(NAME # _IMM_REAL) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>;

}



class sve_mem_prfm_si<bits<2> msz, string asm>
@@ -5687,7 +5691,8 @@ class sve_mem_64b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 
  let Uses = !if(!eq(opc{0}, 1), [FFR], []);

}



multiclass sve_mem_64b_gld_vi_64_ptrs<bits<4> opc, string asm, Operand imm_ty> {

multiclass sve_mem_64b_gld_vi_64_ptrs<bits<4> opc, string asm, Operand imm_ty,

                                      SDPatternOperator op, ValueType vt> {

  def _IMM_REAL : sve_mem_64b_gld_vi<opc, asm, imm_ty>;



  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",
@@ -5696,6 +5701,9 @@ multiclass sve_mem_64b_gld_vi_64_ptrs<bits<4> opc, string asm, Operand imm_ty> { 
                 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>;

  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",

                  (!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>;



  def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt)),

            (!cast<Instruction>(NAME # _IMM_REAL) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>;

}



// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl)