Commit 3cf80822 authored by Kerry McLaughlin's avatar Kerry McLaughlin
Browse files

[AArch64][SVE] Add SVE2 intrinsics for uniform DSP operations

Summary:
Implements the following intrinsics:
 - sqrdmlah, sqrdmlsh, sqrdmulh & sqdmulh
 - [s|u]hadd, [s|u]hsub, [s|u]rhadd & [s|u]hsubr
 - urecpe, ursqrte, sqabs & sqneg

Reviewers: sdesmalen, efriedma, dancgr, cameron.mcinally

Reviewed By: efriedma

Subscribers: tschuett, kristof.beyls, hiraditya, rkruppe, psnobl, cfe-commits, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D73493
parent dc0d84f0
Loading
Loading
Loading
Loading
+25 −2
Original line number Diff line number Diff line
@@ -1208,8 +1208,6 @@ def int_aarch64_sve_sub : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_subr  : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_pmul       : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_sqdmulh    : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_sqrdmulh   : AdvSIMD_2VectorArg_Intrinsic;

def int_aarch64_sve_mul        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_smulh      : AdvSIMD_Pred2VectorArg_Intrinsic;
@@ -1665,6 +1663,31 @@ def int_aarch64_sve_st1_scatter_uxtw_index

def int_aarch64_sve_st1_scatter_scalar_offset : AdvSIMD_ScatterStore_VectorBase_Intrinsic;

//
// SVE2 - Uniform DSP operations
//

def int_aarch64_sve_shadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_shsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_shsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_sqabs         : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_sqdmulh       : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_sqdmulh_lane  : AdvSIMD_2VectorArgIndexed_Intrinsic;
def int_aarch64_sve_sqneg         : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_sqrdmlah      : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_sqrdmlah_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
def int_aarch64_sve_sqrdmlsh      : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_sqrdmlsh_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
def int_aarch64_sve_sqrdmulh      : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_sqrdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic;
def int_aarch64_sve_srhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_uhadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_uhsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_uhsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_urecpe        : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_urhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_ursqrte       : AdvSIMD_Merged1VectorArg_Intrinsic;

//
// SVE2 - Non-widening pairwise arithmetic
//
+21 −21
Original line number Diff line number Diff line
@@ -1396,23 +1396,23 @@ let Predicates = [HasSVE] in {

let Predicates = [HasSVE2] in {
  // SVE2 integer multiply-add (indexed)
  defm MLA_ZZZI : sve2_int_mla_by_indexed_elem<0b01, 0b0, "mla">;
  defm MLS_ZZZI : sve2_int_mla_by_indexed_elem<0b01, 0b1, "mls">;
  defm MLA_ZZZI : sve2_int_mla_by_indexed_elem<0b01, 0b0, "mla", null_frag>;
  defm MLS_ZZZI : sve2_int_mla_by_indexed_elem<0b01, 0b1, "mls", null_frag>;

  // SVE2 saturating multiply-add high (indexed)
  defm SQRDMLAH_ZZZI : sve2_int_mla_by_indexed_elem<0b10, 0b0, "sqrdmlah">;
  defm SQRDMLSH_ZZZI : sve2_int_mla_by_indexed_elem<0b10, 0b1, "sqrdmlsh">;
  defm SQRDMLAH_ZZZI : sve2_int_mla_by_indexed_elem<0b10, 0b0, "sqrdmlah", int_aarch64_sve_sqrdmlah_lane>;
  defm SQRDMLSH_ZZZI : sve2_int_mla_by_indexed_elem<0b10, 0b1, "sqrdmlsh", int_aarch64_sve_sqrdmlsh_lane>;

  // SVE2 saturating multiply-add high (vectors, unpredicated)
  defm SQRDMLAH_ZZZ : sve2_int_mla<0b0, "sqrdmlah">;
  defm SQRDMLSH_ZZZ : sve2_int_mla<0b1, "sqrdmlsh">;
  defm SQRDMLAH_ZZZ : sve2_int_mla<0b0, "sqrdmlah", int_aarch64_sve_sqrdmlah>;
  defm SQRDMLSH_ZZZ : sve2_int_mla<0b1, "sqrdmlsh", int_aarch64_sve_sqrdmlsh>;

  // SVE2 integer multiply (indexed)
  defm MUL_ZZZI : sve2_int_mul_by_indexed_elem<0b1110, "mul">;
  defm MUL_ZZZI : sve2_int_mul_by_indexed_elem<0b1110, "mul", null_frag>;

  // SVE2 saturating multiply high (indexed)
  defm SQDMULH_ZZZI  : sve2_int_mul_by_indexed_elem<0b1100, "sqdmulh">;
  defm SQRDMULH_ZZZI : sve2_int_mul_by_indexed_elem<0b1101, "sqrdmulh">;
  defm SQDMULH_ZZZI  : sve2_int_mul_by_indexed_elem<0b1100, "sqdmulh",  int_aarch64_sve_sqdmulh_lane>;
  defm SQRDMULH_ZZZI : sve2_int_mul_by_indexed_elem<0b1101, "sqrdmulh", int_aarch64_sve_sqrdmulh_lane>;

  // SVE2 signed saturating doubling multiply high (unpredicated)
  defm SQDMULH_ZZZ  : sve2_int_mul<0b100, "sqdmulh",  int_aarch64_sve_sqdmulh>;
@@ -1503,14 +1503,14 @@ let Predicates = [HasSVE2] in {
  defm SQDMLSLBT_ZZZ : sve2_int_mla_long<0b00011, "sqdmlslbt">;

  // SVE2 integer halving add/subtract (predicated)
  defm SHADD_ZPmZ  : sve2_int_arith_pred<0b100000, "shadd",  null_frag>;
  defm UHADD_ZPmZ  : sve2_int_arith_pred<0b100010, "uhadd",  null_frag>;
  defm SHSUB_ZPmZ  : sve2_int_arith_pred<0b100100, "shsub",  null_frag>;
  defm UHSUB_ZPmZ  : sve2_int_arith_pred<0b100110, "uhsub",  null_frag>;
  defm SRHADD_ZPmZ : sve2_int_arith_pred<0b101000, "srhadd", null_frag>;
  defm URHADD_ZPmZ : sve2_int_arith_pred<0b101010, "urhadd", null_frag>;
  defm SHSUBR_ZPmZ : sve2_int_arith_pred<0b101100, "shsubr", null_frag>;
  defm UHSUBR_ZPmZ : sve2_int_arith_pred<0b101110, "uhsubr", null_frag>;
  defm SHADD_ZPmZ  : sve2_int_arith_pred<0b100000, "shadd",  int_aarch64_sve_shadd>;
  defm UHADD_ZPmZ  : sve2_int_arith_pred<0b100010, "uhadd",  int_aarch64_sve_uhadd>;
  defm SHSUB_ZPmZ  : sve2_int_arith_pred<0b100100, "shsub",  int_aarch64_sve_shsub>;
  defm UHSUB_ZPmZ  : sve2_int_arith_pred<0b100110, "uhsub",  int_aarch64_sve_uhsub>;
  defm SRHADD_ZPmZ : sve2_int_arith_pred<0b101000, "srhadd", int_aarch64_sve_srhadd>;
  defm URHADD_ZPmZ : sve2_int_arith_pred<0b101010, "urhadd", int_aarch64_sve_urhadd>;
  defm SHSUBR_ZPmZ : sve2_int_arith_pred<0b101100, "shsubr", int_aarch64_sve_shsubr>;
  defm UHSUBR_ZPmZ : sve2_int_arith_pred<0b101110, "uhsubr", int_aarch64_sve_uhsubr>;

  // SVE2 integer pairwise add and accumulate long
  defm SADALP_ZPmZ : sve2_int_sadd_long_accum_pairwise<0, "sadalp", int_aarch64_sve_sadalp>;
@@ -1524,10 +1524,10 @@ let Predicates = [HasSVE2] in {
  defm UMINP_ZPmZ : sve2_int_arith_pred<0b101111, "uminp", int_aarch64_sve_uminp>;

  // SVE2 integer unary operations (predicated)
  defm URECPE_ZPmZ  : sve2_int_un_pred_arit_s<0b000, "urecpe">;
  defm URSQRTE_ZPmZ : sve2_int_un_pred_arit_s<0b001, "ursqrte">;
  defm SQABS_ZPmZ   : sve2_int_un_pred_arit<0b100, "sqabs">;
  defm SQNEG_ZPmZ   : sve2_int_un_pred_arit<0b101, "sqneg">;
  defm URECPE_ZPmZ  : sve2_int_un_pred_arit_s<0b000, "urecpe",  int_aarch64_sve_urecpe>;
  defm URSQRTE_ZPmZ : sve2_int_un_pred_arit_s<0b001, "ursqrte", int_aarch64_sve_ursqrte>;
  defm SQABS_ZPmZ   : sve2_int_un_pred_arit<0b100,   "sqabs",   int_aarch64_sve_sqabs>;
  defm SQNEG_ZPmZ   : sve2_int_un_pred_arit<0b101,   "sqneg",   int_aarch64_sve_sqneg>;

  // SVE2 saturating add/subtract
  defm SQADD_ZPmZ  : sve2_int_arith_pred<0b110000, "sqadd",  null_frag>;
+33 −11
Original line number Diff line number Diff line
@@ -2340,11 +2340,16 @@ class sve2_int_mla<bits<2> sz, bits<5> opc, string asm,
  let ElementSize = ElementSizeNone;
}

multiclass sve2_int_mla<bit S, string asm> {
multiclass sve2_int_mla<bit S, string asm, SDPatternOperator op> {
  def _B : sve2_int_mla<0b00, { 0b1110, S }, asm, ZPR8, ZPR8>;
  def _H : sve2_int_mla<0b01, { 0b1110, S }, asm, ZPR16, ZPR16>;
  def _S : sve2_int_mla<0b10, { 0b1110, S }, asm, ZPR32, ZPR32>;
  def _D : sve2_int_mla<0b11, { 0b1110, S }, asm, ZPR64, ZPR64>;

  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
}

multiclass sve2_int_mla_long<bits<5> opc, string asm> {
@@ -2376,26 +2381,31 @@ class sve2_int_mla_by_indexed_elem<bits<2> sz, bits<6> opc, string asm,
  let ElementSize = ElementSizeNone;
}

multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm> {
  def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH> {
multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm,
                                        SDPatternOperator op> {
  def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> {
    bits<3> Zm;
    bits<3> iop;
    let Inst{22} = iop{2};
    let Inst{20-19} = iop{1-0};
    let Inst{18-16} = Zm;
  }
  def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS> {
  def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> {
    bits<3> Zm;
    bits<2> iop;
    let Inst{20-19} = iop;
    let Inst{18-16} = Zm;
  }
  def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD> {
  def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> {
    bits<4> Zm;
    bit iop;
    let Inst{20} = iop;
    let Inst{19-16} = Zm;
  }

  def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;
  def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>;
  def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>;
}

//===----------------------------------------------------------------------===//
@@ -2662,26 +2672,31 @@ class sve2_int_mul_by_indexed_elem<bits<2> sz, bits<4> opc, string asm,
  let Inst{4-0}   = Zd;
}

multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm> {
  def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH> {
multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm,
                                        SDPatternOperator op> {
  def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> {
    bits<3> Zm;
    bits<3> iop;
    let Inst{22} = iop{2};
    let Inst{20-19} = iop{1-0};
    let Inst{18-16} = Zm;
  }
  def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS> {
  def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> {
    bits<3> Zm;
    bits<2> iop;
    let Inst{20-19} = iop;
    let Inst{18-16} = Zm;
  }
  def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD> {
  def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> {
    bits<4> Zm;
    bit iop;
    let Inst{20} = iop;
    let Inst{19-16} = Zm;
  }

  def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;
  def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>;
  def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>;
}

multiclass sve2_int_mul_long_by_indexed_elem<bits<3> opc, string asm> {
@@ -2797,15 +2812,22 @@ class sve2_int_un_pred_arit<bits<2> sz, bit Q, bits<2> opc,
  let ElementSize = zprty.ElementSize;
}

multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm> {
multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm,
                                   SDPatternOperator op> {
  def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>;
  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;
}

multiclass sve2_int_un_pred_arit<bits<3> opc, string asm> {
multiclass sve2_int_un_pred_arit<bits<3> opc, string asm, SDPatternOperator op> {
  def _B : sve2_int_un_pred_arit<0b00, opc{2}, opc{1-0}, asm, ZPR8>;
  def _H : sve2_int_un_pred_arit<0b01, opc{2}, opc{1-0}, asm, ZPR16>;
  def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>;
  def _D : sve2_int_un_pred_arit<0b11, opc{2}, opc{1-0}, asm, ZPR64>;

  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
}

//===----------------------------------------------------------------------===//
+869 −0

File added.

Preview size limit exceeded, changes collapsed.