Commit 5e1d7bb6 authored by Kerry McLaughlin's avatar Kerry McLaughlin
Browse files

[AArch64][SVE] Add SVE2 intrinsics for widening DSP operations 

Summary:
Implements the following intrinsics:

 - @llvm.aarch64.sve.[s|u]abalb
 - @llvm.aarch64.sve.[s|u]abalt
 - @llvm.aarch64.sve.[s|u]addlb
 - @llvm.aarch64.sve.[s|u]addlt
 - @llvm.aarch64.sve.[s|u]sublb
 - @llvm.aarch64.sve.[s|u]sublt
 - @llvm.aarch64.sve.[s|u]abdlb
 - @llvm.aarch64.sve.[s|u]abdlt
 - @llvm.aarch64.sve.sqdmullb
 - @llvm.aarch64.sve.sqdmullt
 - @llvm.aarch64.sve.[s|u]mullb
 - @llvm.aarch64.sve.[s|u]mullt

Reviewers: sdesmalen, dancgr, efriedma, cameron.mcinally, rengolin

Reviewed By: sdesmalen

Subscribers: tschuett, kristof.beyls, hiraditya, rkruppe, psnobl, cfe-commits, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D73719
parent da52b9c1

llvm/include/llvm/IR/IntrinsicsAArch64.td

+33 −0
Original line number Diff line number Diff line
@@ -1040,6 +1040,12 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 
                 LLVMVectorOfBitcastsToInt<0>],

                [IntrNoMem]>;



  class SVE2_2VectorArg_Long_Intrinsic

    : Intrinsic<[llvm_anyvector_ty],

                [LLVMSubdivide2VectorType<0>,

                 LLVMSubdivide2VectorType<0>],

                [IntrNoMem]>;



  class SVE2_2VectorArg_Pred_Long_Intrinsic

    : Intrinsic<[llvm_anyvector_ty],

                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
@@ -1721,6 +1727,33 @@ def int_aarch64_sve_ursra : AdvSIMD_2VectorArgIndexed_Intrinsic; 
def int_aarch64_sve_usqadd        : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_usra          : AdvSIMD_2VectorArgIndexed_Intrinsic;



//

// SVE2 - Widening DSP operations

//



def int_aarch64_sve_sabalb   : SVE2_3VectorArg_Long_Intrinsic;

def int_aarch64_sve_sabalt   : SVE2_3VectorArg_Long_Intrinsic;

def int_aarch64_sve_sabdlb   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_sabdlt   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_saddlb   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_saddlt   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_smullb   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_smullt   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_sqdmullb : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_sqdmullt : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_ssublb   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_ssublt   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_uabalb   : SVE2_3VectorArg_Long_Intrinsic;

def int_aarch64_sve_uabalt   : SVE2_3VectorArg_Long_Intrinsic;

def int_aarch64_sve_uabdlb   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_uabdlt   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_uaddlb   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_uaddlt   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_umullb   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_umullt   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_usublb   : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_usublt   : SVE2_2VectorArg_Long_Intrinsic;



//

// SVE2 - Non-widening pairwise arithmetic

//

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

+22 −22
Original line number Diff line number Diff line
@@ -1561,18 +1561,18 @@ let Predicates = [HasSVE2] in { 
  defm SQSHLU_ZPmI : sve2_int_bin_pred_shift_imm_left< 0b1111, "sqshlu", int_aarch64_sve_sqshlu>;



  // SVE2 integer add/subtract long

  defm SADDLB_ZZZ : sve2_wide_int_arith_long<0b00000, "saddlb">;

  defm SADDLT_ZZZ : sve2_wide_int_arith_long<0b00001, "saddlt">;

  defm UADDLB_ZZZ : sve2_wide_int_arith_long<0b00010, "uaddlb">;

  defm UADDLT_ZZZ : sve2_wide_int_arith_long<0b00011, "uaddlt">;

  defm SSUBLB_ZZZ : sve2_wide_int_arith_long<0b00100, "ssublb">;

  defm SSUBLT_ZZZ : sve2_wide_int_arith_long<0b00101, "ssublt">;

  defm USUBLB_ZZZ : sve2_wide_int_arith_long<0b00110, "usublb">;

  defm USUBLT_ZZZ : sve2_wide_int_arith_long<0b00111, "usublt">;

  defm SABDLB_ZZZ : sve2_wide_int_arith_long<0b01100, "sabdlb">;

  defm SABDLT_ZZZ : sve2_wide_int_arith_long<0b01101, "sabdlt">;

  defm UABDLB_ZZZ : sve2_wide_int_arith_long<0b01110, "uabdlb">;

  defm UABDLT_ZZZ : sve2_wide_int_arith_long<0b01111, "uabdlt">;

  defm SADDLB_ZZZ : sve2_wide_int_arith_long<0b00000, "saddlb", int_aarch64_sve_saddlb>;

  defm SADDLT_ZZZ : sve2_wide_int_arith_long<0b00001, "saddlt", int_aarch64_sve_saddlt>;

  defm UADDLB_ZZZ : sve2_wide_int_arith_long<0b00010, "uaddlb", int_aarch64_sve_uaddlb>;

  defm UADDLT_ZZZ : sve2_wide_int_arith_long<0b00011, "uaddlt", int_aarch64_sve_uaddlt>;

  defm SSUBLB_ZZZ : sve2_wide_int_arith_long<0b00100, "ssublb", int_aarch64_sve_ssublb>;

  defm SSUBLT_ZZZ : sve2_wide_int_arith_long<0b00101, "ssublt", int_aarch64_sve_ssublt>;

  defm USUBLB_ZZZ : sve2_wide_int_arith_long<0b00110, "usublb", int_aarch64_sve_usublb>;

  defm USUBLT_ZZZ : sve2_wide_int_arith_long<0b00111, "usublt", int_aarch64_sve_usublt>;

  defm SABDLB_ZZZ : sve2_wide_int_arith_long<0b01100, "sabdlb", int_aarch64_sve_sabdlb>;

  defm SABDLT_ZZZ : sve2_wide_int_arith_long<0b01101, "sabdlt", int_aarch64_sve_sabdlt>;

  defm UABDLB_ZZZ : sve2_wide_int_arith_long<0b01110, "uabdlb", int_aarch64_sve_uabdlb>;

  defm UABDLT_ZZZ : sve2_wide_int_arith_long<0b01111, "uabdlt", int_aarch64_sve_uabdlt>;



  // SVE2 integer add/subtract wide

  defm SADDWB_ZZZ : sve2_wide_int_arith_wide<0b000, "saddwb">;
@@ -1585,12 +1585,12 @@ let Predicates = [HasSVE2] in { 
  defm USUBWT_ZZZ : sve2_wide_int_arith_wide<0b111, "usubwt">;



  // SVE2 integer multiply long

  defm SQDMULLB_ZZZ : sve2_wide_int_arith_long<0b11000, "sqdmullb">;

  defm SQDMULLT_ZZZ : sve2_wide_int_arith_long<0b11001, "sqdmullt">;

  defm SMULLB_ZZZ   : sve2_wide_int_arith_long<0b11100, "smullb">;

  defm SMULLT_ZZZ   : sve2_wide_int_arith_long<0b11101, "smullt">;

  defm UMULLB_ZZZ   : sve2_wide_int_arith_long<0b11110, "umullb">;

  defm UMULLT_ZZZ   : sve2_wide_int_arith_long<0b11111, "umullt">;

  defm SQDMULLB_ZZZ : sve2_wide_int_arith_long<0b11000, "sqdmullb", int_aarch64_sve_sqdmullb>;

  defm SQDMULLT_ZZZ : sve2_wide_int_arith_long<0b11001, "sqdmullt", int_aarch64_sve_sqdmullt>;

  defm SMULLB_ZZZ   : sve2_wide_int_arith_long<0b11100, "smullb",   int_aarch64_sve_smullb>;

  defm SMULLT_ZZZ   : sve2_wide_int_arith_long<0b11101, "smullt",   int_aarch64_sve_smullt>;

  defm UMULLB_ZZZ   : sve2_wide_int_arith_long<0b11110, "umullb",   int_aarch64_sve_umullb>;

  defm UMULLT_ZZZ   : sve2_wide_int_arith_long<0b11111, "umullt",   int_aarch64_sve_umullt>;

  defm PMULLB_ZZZ   : sve2_pmul_long<0b0, "pmullb">;

  defm PMULLT_ZZZ   : sve2_pmul_long<0b1, "pmullt">;
@@ -1613,10 +1613,10 @@ let Predicates = [HasSVE2] in { 
  defm UABA_ZZZ : sve2_int_absdiff_accum<0b1, "uaba", int_aarch64_sve_uaba>;



  // SVE2 integer absolute difference and accumulate long

  defm SABALB_ZZZ : sve2_int_absdiff_accum_long<0b00, "sabalb">;

  defm SABALT_ZZZ : sve2_int_absdiff_accum_long<0b01, "sabalt">;

  defm UABALB_ZZZ : sve2_int_absdiff_accum_long<0b10, "uabalb">;

  defm UABALT_ZZZ : sve2_int_absdiff_accum_long<0b11, "uabalt">;

  defm SABALB_ZZZ : sve2_int_absdiff_accum_long<0b00, "sabalb", int_aarch64_sve_sabalb>;

  defm SABALT_ZZZ : sve2_int_absdiff_accum_long<0b01, "sabalt", int_aarch64_sve_sabalt>;

  defm UABALB_ZZZ : sve2_int_absdiff_accum_long<0b10, "uabalb", int_aarch64_sve_uabalb>;

  defm UABALT_ZZZ : sve2_int_absdiff_accum_long<0b11, "uabalt", int_aarch64_sve_uabalt>;



  // SVE2 integer add/subtract long with carry

  defm ADCLB_ZZZ : sve2_int_addsub_long_carry<0b00, "adclb">;

llvm/lib/Target/AArch64/SVEInstrFormats.td

+12 −2
Original line number Diff line number Diff line
@@ -2855,10 +2855,15 @@ class sve2_wide_int_arith<bits<2> sz, bits<5> opc, string asm, 
  let Inst{4-0}   = Zd;

}



multiclass sve2_wide_int_arith_long<bits<5> opc, string asm> {

multiclass sve2_wide_int_arith_long<bits<5> opc, string asm,

                                    SDPatternOperator op> {

  def _H : sve2_wide_int_arith<0b01, opc, asm, ZPR16, ZPR8, ZPR8>;

  def _S : sve2_wide_int_arith<0b10, opc, asm, ZPR32, ZPR16, ZPR16>;

  def _D : sve2_wide_int_arith<0b11, opc, asm, ZPR64, ZPR32, ZPR32>;



  def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>;

  def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>;

  def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>;

}



multiclass sve2_wide_int_arith_wide<bits<3> opc, string asm> {
@@ -3135,10 +3140,15 @@ multiclass sve2_int_absdiff_accum<bit opc, string asm, SDPatternOperator op> { 
  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;

}



multiclass sve2_int_absdiff_accum_long<bits<2> opc, string asm> {

multiclass sve2_int_absdiff_accum_long<bits<2> opc, string asm,

                                       SDPatternOperator op> {

  def _H : sve2_int_absdiff_accum<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>;

  def _S : sve2_int_absdiff_accum<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>;

  def _D : sve2_int_absdiff_accum<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>;



  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>;

  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>;

  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>;

}



multiclass sve2_int_addsub_long_carry<bits<2> opc, string asm> {

llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-dsp.ll

0 → 100644
+783 −0

File added.

Preview size limit exceeded, changes collapsed.