Commit 3cf80822 authored by Kerry McLaughlin's avatar Kerry McLaughlin
Browse files

[AArch64][SVE] Add SVE2 intrinsics for uniform DSP operations 

Summary:
Implements the following intrinsics:
 - sqrdmlah, sqrdmlsh, sqrdmulh & sqdmulh
 - [s|u]hadd, [s|u]hsub, [s|u]rhadd & [s|u]hsubr
 - urecpe, ursqrte, sqabs & sqneg

Reviewers: sdesmalen, efriedma, dancgr, cameron.mcinally

Reviewed By: efriedma

Subscribers: tschuett, kristof.beyls, hiraditya, rkruppe, psnobl, cfe-commits, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D73493
parent dc0d84f0

llvm/include/llvm/IR/IntrinsicsAArch64.td

+25 −2
Original line number Diff line number Diff line
@@ -1208,8 +1208,6 @@ def int_aarch64_sve_sub : AdvSIMD_Pred2VectorArg_Intrinsic; 
def int_aarch64_sve_subr  : AdvSIMD_Pred2VectorArg_Intrinsic;



def int_aarch64_sve_pmul       : AdvSIMD_2VectorArg_Intrinsic;

def int_aarch64_sve_sqdmulh    : AdvSIMD_2VectorArg_Intrinsic;

def int_aarch64_sve_sqrdmulh   : AdvSIMD_2VectorArg_Intrinsic;



def int_aarch64_sve_mul        : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_smulh      : AdvSIMD_Pred2VectorArg_Intrinsic;
@@ -1665,6 +1663,31 @@ def int_aarch64_sve_st1_scatter_uxtw_index 


def int_aarch64_sve_st1_scatter_scalar_offset : AdvSIMD_ScatterStore_VectorBase_Intrinsic;



//

// SVE2 - Uniform DSP operations

//



def int_aarch64_sve_shadd         : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_shsub         : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_shsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_sqabs         : AdvSIMD_Merged1VectorArg_Intrinsic;

def int_aarch64_sve_sqdmulh       : AdvSIMD_2VectorArg_Intrinsic;

def int_aarch64_sve_sqdmulh_lane  : AdvSIMD_2VectorArgIndexed_Intrinsic;

def int_aarch64_sve_sqneg         : AdvSIMD_Merged1VectorArg_Intrinsic;

def int_aarch64_sve_sqrdmlah      : AdvSIMD_3VectorArg_Intrinsic;

def int_aarch64_sve_sqrdmlah_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;

def int_aarch64_sve_sqrdmlsh      : AdvSIMD_3VectorArg_Intrinsic;

def int_aarch64_sve_sqrdmlsh_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;

def int_aarch64_sve_sqrdmulh      : AdvSIMD_2VectorArg_Intrinsic;

def int_aarch64_sve_sqrdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic;

def int_aarch64_sve_srhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_uhadd         : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_uhsub         : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_uhsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_urecpe        : AdvSIMD_Merged1VectorArg_Intrinsic;

def int_aarch64_sve_urhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_ursqrte       : AdvSIMD_Merged1VectorArg_Intrinsic;



//

// SVE2 - Non-widening pairwise arithmetic

//

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

+21 −21
Original line number Diff line number Diff line
@@ -1396,23 +1396,23 @@ let Predicates = [HasSVE] in { 


let Predicates = [HasSVE2] in {

  // SVE2 integer multiply-add (indexed)

  defm MLA_ZZZI : sve2_int_mla_by_indexed_elem<0b01, 0b0, "mla">;

  defm MLS_ZZZI : sve2_int_mla_by_indexed_elem<0b01, 0b1, "mls">;

  defm MLA_ZZZI : sve2_int_mla_by_indexed_elem<0b01, 0b0, "mla", null_frag>;

  defm MLS_ZZZI : sve2_int_mla_by_indexed_elem<0b01, 0b1, "mls", null_frag>;



  // SVE2 saturating multiply-add high (indexed)

  defm SQRDMLAH_ZZZI : sve2_int_mla_by_indexed_elem<0b10, 0b0, "sqrdmlah">;

  defm SQRDMLSH_ZZZI : sve2_int_mla_by_indexed_elem<0b10, 0b1, "sqrdmlsh">;

  defm SQRDMLAH_ZZZI : sve2_int_mla_by_indexed_elem<0b10, 0b0, "sqrdmlah", int_aarch64_sve_sqrdmlah_lane>;

  defm SQRDMLSH_ZZZI : sve2_int_mla_by_indexed_elem<0b10, 0b1, "sqrdmlsh", int_aarch64_sve_sqrdmlsh_lane>;



  // SVE2 saturating multiply-add high (vectors, unpredicated)

  defm SQRDMLAH_ZZZ : sve2_int_mla<0b0, "sqrdmlah">;

  defm SQRDMLSH_ZZZ : sve2_int_mla<0b1, "sqrdmlsh">;

  defm SQRDMLAH_ZZZ : sve2_int_mla<0b0, "sqrdmlah", int_aarch64_sve_sqrdmlah>;

  defm SQRDMLSH_ZZZ : sve2_int_mla<0b1, "sqrdmlsh", int_aarch64_sve_sqrdmlsh>;



  // SVE2 integer multiply (indexed)

  defm MUL_ZZZI : sve2_int_mul_by_indexed_elem<0b1110, "mul">;

  defm MUL_ZZZI : sve2_int_mul_by_indexed_elem<0b1110, "mul", null_frag>;



  // SVE2 saturating multiply high (indexed)

  defm SQDMULH_ZZZI  : sve2_int_mul_by_indexed_elem<0b1100, "sqdmulh">;

  defm SQRDMULH_ZZZI : sve2_int_mul_by_indexed_elem<0b1101, "sqrdmulh">;

  defm SQDMULH_ZZZI  : sve2_int_mul_by_indexed_elem<0b1100, "sqdmulh",  int_aarch64_sve_sqdmulh_lane>;

  defm SQRDMULH_ZZZI : sve2_int_mul_by_indexed_elem<0b1101, "sqrdmulh", int_aarch64_sve_sqrdmulh_lane>;



  // SVE2 signed saturating doubling multiply high (unpredicated)

  defm SQDMULH_ZZZ  : sve2_int_mul<0b100, "sqdmulh",  int_aarch64_sve_sqdmulh>;
@@ -1503,14 +1503,14 @@ let Predicates = [HasSVE2] in { 
  defm SQDMLSLBT_ZZZ : sve2_int_mla_long<0b00011, "sqdmlslbt">;



  // SVE2 integer halving add/subtract (predicated)

  defm SHADD_ZPmZ  : sve2_int_arith_pred<0b100000, "shadd",  null_frag>;

  defm UHADD_ZPmZ  : sve2_int_arith_pred<0b100010, "uhadd",  null_frag>;

  defm SHSUB_ZPmZ  : sve2_int_arith_pred<0b100100, "shsub",  null_frag>;

  defm UHSUB_ZPmZ  : sve2_int_arith_pred<0b100110, "uhsub",  null_frag>;

  defm SRHADD_ZPmZ : sve2_int_arith_pred<0b101000, "srhadd", null_frag>;

  defm URHADD_ZPmZ : sve2_int_arith_pred<0b101010, "urhadd", null_frag>;

  defm SHSUBR_ZPmZ : sve2_int_arith_pred<0b101100, "shsubr", null_frag>;

  defm UHSUBR_ZPmZ : sve2_int_arith_pred<0b101110, "uhsubr", null_frag>;

  defm SHADD_ZPmZ  : sve2_int_arith_pred<0b100000, "shadd",  int_aarch64_sve_shadd>;

  defm UHADD_ZPmZ  : sve2_int_arith_pred<0b100010, "uhadd",  int_aarch64_sve_uhadd>;

  defm SHSUB_ZPmZ  : sve2_int_arith_pred<0b100100, "shsub",  int_aarch64_sve_shsub>;

  defm UHSUB_ZPmZ  : sve2_int_arith_pred<0b100110, "uhsub",  int_aarch64_sve_uhsub>;

  defm SRHADD_ZPmZ : sve2_int_arith_pred<0b101000, "srhadd", int_aarch64_sve_srhadd>;

  defm URHADD_ZPmZ : sve2_int_arith_pred<0b101010, "urhadd", int_aarch64_sve_urhadd>;

  defm SHSUBR_ZPmZ : sve2_int_arith_pred<0b101100, "shsubr", int_aarch64_sve_shsubr>;

  defm UHSUBR_ZPmZ : sve2_int_arith_pred<0b101110, "uhsubr", int_aarch64_sve_uhsubr>;



  // SVE2 integer pairwise add and accumulate long

  defm SADALP_ZPmZ : sve2_int_sadd_long_accum_pairwise<0, "sadalp", int_aarch64_sve_sadalp>;
@@ -1524,10 +1524,10 @@ let Predicates = [HasSVE2] in { 
  defm UMINP_ZPmZ : sve2_int_arith_pred<0b101111, "uminp", int_aarch64_sve_uminp>;



  // SVE2 integer unary operations (predicated)

  defm URECPE_ZPmZ  : sve2_int_un_pred_arit_s<0b000, "urecpe">;

  defm URSQRTE_ZPmZ : sve2_int_un_pred_arit_s<0b001, "ursqrte">;

  defm SQABS_ZPmZ   : sve2_int_un_pred_arit<0b100, "sqabs">;

  defm SQNEG_ZPmZ   : sve2_int_un_pred_arit<0b101, "sqneg">;

  defm URECPE_ZPmZ  : sve2_int_un_pred_arit_s<0b000, "urecpe",  int_aarch64_sve_urecpe>;

  defm URSQRTE_ZPmZ : sve2_int_un_pred_arit_s<0b001, "ursqrte", int_aarch64_sve_ursqrte>;

  defm SQABS_ZPmZ   : sve2_int_un_pred_arit<0b100,   "sqabs",   int_aarch64_sve_sqabs>;

  defm SQNEG_ZPmZ   : sve2_int_un_pred_arit<0b101,   "sqneg",   int_aarch64_sve_sqneg>;



  // SVE2 saturating add/subtract

  defm SQADD_ZPmZ  : sve2_int_arith_pred<0b110000, "sqadd",  null_frag>;

llvm/lib/Target/AArch64/SVEInstrFormats.td

+33 −11
Original line number Diff line number Diff line
@@ -2340,11 +2340,16 @@ class sve2_int_mla<bits<2> sz, bits<5> opc, string asm, 
  let ElementSize = ElementSizeNone;

}



multiclass sve2_int_mla<bit S, string asm> {

multiclass sve2_int_mla<bit S, string asm, SDPatternOperator op> {

  def _B : sve2_int_mla<0b00, { 0b1110, S }, asm, ZPR8, ZPR8>;

  def _H : sve2_int_mla<0b01, { 0b1110, S }, asm, ZPR16, ZPR16>;

  def _S : sve2_int_mla<0b10, { 0b1110, S }, asm, ZPR32, ZPR32>;

  def _D : sve2_int_mla<0b11, { 0b1110, S }, asm, ZPR64, ZPR64>;



  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;

  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;

  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;

  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;

}



multiclass sve2_int_mla_long<bits<5> opc, string asm> {
@@ -2376,26 +2381,31 @@ class sve2_int_mla_by_indexed_elem<bits<2> sz, bits<6> opc, string asm, 
  let ElementSize = ElementSizeNone;

}



multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm> {

  def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH> {

multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm,

                                        SDPatternOperator op> {

  def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> {

    bits<3> Zm;

    bits<3> iop;

    let Inst{22} = iop{2};

    let Inst{20-19} = iop{1-0};

    let Inst{18-16} = Zm;

  }

  def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS> {

  def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> {

    bits<3> Zm;

    bits<2> iop;

    let Inst{20-19} = iop;

    let Inst{18-16} = Zm;

  }

  def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD> {

  def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> {

    bits<4> Zm;

    bit iop;

    let Inst{20} = iop;

    let Inst{19-16} = Zm;

  }



  def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;

  def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>;

  def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>;

}



//===----------------------------------------------------------------------===//
@@ -2662,26 +2672,31 @@ class sve2_int_mul_by_indexed_elem<bits<2> sz, bits<4> opc, string asm, 
  let Inst{4-0}   = Zd;

}



multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm> {

  def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH> {

multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm,

                                        SDPatternOperator op> {

  def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> {

    bits<3> Zm;

    bits<3> iop;

    let Inst{22} = iop{2};

    let Inst{20-19} = iop{1-0};

    let Inst{18-16} = Zm;

  }

  def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS> {

  def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> {

    bits<3> Zm;

    bits<2> iop;

    let Inst{20-19} = iop;

    let Inst{18-16} = Zm;

  }

  def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD> {

  def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> {

    bits<4> Zm;

    bit iop;

    let Inst{20} = iop;

    let Inst{19-16} = Zm;

  }



  def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;

  def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>;

  def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>;

}



multiclass sve2_int_mul_long_by_indexed_elem<bits<3> opc, string asm> {
@@ -2797,15 +2812,22 @@ class sve2_int_un_pred_arit<bits<2> sz, bit Q, bits<2> opc, 
  let ElementSize = zprty.ElementSize;

}



multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm> {

multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm,

                                   SDPatternOperator op> {

  def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>;

  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;

}



multiclass sve2_int_un_pred_arit<bits<3> opc, string asm> {

multiclass sve2_int_un_pred_arit<bits<3> opc, string asm, SDPatternOperator op> {

  def _B : sve2_int_un_pred_arit<0b00, opc{2}, opc{1-0}, asm, ZPR8>;

  def _H : sve2_int_un_pred_arit<0b01, opc{2}, opc{1-0}, asm, ZPR16>;

  def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>;

  def _D : sve2_int_un_pred_arit<0b11, opc{2}, opc{1-0}, asm, ZPR64>;



  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;

  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;

  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;

  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;

}



//===----------------------------------------------------------------------===//

llvm/test/CodeGen/AArch64/sve2-intrinsics-uniform-dsp.ll

0 → 100644
+869 −0

File added.

Preview size limit exceeded, changes collapsed.