Unverified Commit 40a426fa authored by Pierre van Houtryve's avatar Pierre van Houtryve Committed by GitHub
Browse files

[AMDGPU] Constant fold FMAD_FTZ (#69443) 

Solves #68315
parent 25002b7a

llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

+30 −0
Original line number Diff line number Diff line
@@ -5041,6 +5041,36 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N, 
    return performAssertSZExtCombine(N, DCI);

  case ISD::INTRINSIC_WO_CHAIN:

    return performIntrinsicWOChainCombine(N, DCI);

  case AMDGPUISD::FMAD_FTZ: {

    SDValue N0 = N->getOperand(0);

    SDValue N1 = N->getOperand(1);

    SDValue N2 = N->getOperand(2);

    EVT VT = N->getValueType(0);



    // FMAD_FTZ is a FMAD + flush denormals to zero.

    // We flush the inputs, the intermediate step, and the output.

    ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);

    ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);

    ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);

    if (N0CFP && N1CFP && N2CFP) {

      const auto FTZ = [](const APFloat &V) {

        if (V.isDenormal()) {

          APFloat Zero(V.getSemantics(), 0);

          return V.isNegative() ? -Zero : Zero;

        }

        return V;

      };



      APFloat V0 = FTZ(N0CFP->getValueAPF());

      APFloat V1 = FTZ(N1CFP->getValueAPF());

      APFloat V2 = FTZ(N2CFP->getValueAPF());

      V0.multiply(V1, APFloat::rmNearestTiesToEven);

      V0 = FTZ(V0);

      V0.add(V2, APFloat::rmNearestTiesToEven);

      return DAG.getConstantFP(FTZ(V0), DL, VT);

    }

    break;

  }

  }

  return SDValue();

}

llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll

+419 −732

File changed.

Preview size limit exceeded, changes collapsed.

llvm/test/CodeGen/AMDGPU/udiv.ll

+38 −78
Original line number Diff line number Diff line
@@ -2520,59 +2520,38 @@ define i64 @v_test_udiv64_mulhi_fold(i64 %arg) { 
; SI-LABEL: v_test_udiv64_mulhi_fold:

; SI:       ; %bb.0:

; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)

; SI-NEXT:    v_mov_b32_e32 v2, 0x4f800000

; SI-NEXT:    v_madak_f32 v2, 0, v2, 0x47c35000

; SI-NEXT:    v_rcp_f32_e32 v2, v2

; SI-NEXT:    s_mov_b32 s4, 0x346d900

; SI-NEXT:    s_add_u32 s4, 0x4237, s4

; SI-NEXT:    v_mov_b32_e32 v2, 0xa9000000

; SI-NEXT:    v_add_i32_e32 v2, vcc, s4, v2

; SI-NEXT:    s_addc_u32 s5, 0, 0

; SI-NEXT:    s_or_b32 s4, vcc_lo, vcc_hi

; SI-NEXT:    s_cmp_lg_u32 s4, 0

; SI-NEXT:    s_mov_b32 s4, 0xfffe7960

; SI-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2

; SI-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2

; SI-NEXT:    v_trunc_f32_e32 v3, v3

; SI-NEXT:    v_madmk_f32 v2, v3, 0xcf800000, v2

; SI-NEXT:    v_cvt_u32_f32_e32 v2, v2

; SI-NEXT:    v_cvt_u32_f32_e32 v3, v3

; SI-NEXT:    v_mul_hi_u32 v4, v2, s4

; SI-NEXT:    v_mul_lo_u32 v6, v3, s4

; SI-NEXT:    v_mul_lo_u32 v5, v2, s4

; SI-NEXT:    v_sub_i32_e32 v4, vcc, v4, v2

; SI-NEXT:    v_add_i32_e32 v4, vcc, v4, v6

; SI-NEXT:    v_mul_hi_u32 v7, v2, v5

; SI-NEXT:    v_mul_lo_u32 v6, v2, v4

; SI-NEXT:    v_mul_hi_u32 v8, v2, v4

; SI-NEXT:    v_mul_hi_u32 v9, v3, v4

; SI-NEXT:    v_mul_lo_u32 v4, v3, v4

; SI-NEXT:    v_add_i32_e32 v6, vcc, v7, v6

; SI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc

; SI-NEXT:    v_mul_lo_u32 v8, v3, v5

; SI-NEXT:    v_mul_hi_u32 v5, v3, v5

; SI-NEXT:    v_add_i32_e32 v6, vcc, v6, v8

; SI-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc

; SI-NEXT:    v_addc_u32_e32 v6, vcc, 0, v9, vcc

; SI-NEXT:    v_add_i32_e32 v4, vcc, v5, v4

; SI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc

; SI-NEXT:    v_add_i32_e32 v2, vcc, v2, v4

; SI-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc

; SI-NEXT:    v_mul_hi_u32 v4, v2, s4

; SI-NEXT:    v_mul_lo_u32 v5, v3, s4

; SI-NEXT:    v_mul_lo_u32 v6, v2, s4

; SI-NEXT:    v_mul_hi_u32 v3, v2, s4

; SI-NEXT:    v_mul_lo_u32 v4, v2, s4

; SI-NEXT:    s_addc_u32 s5, s5, 0xa7c5

; SI-NEXT:    s_mul_i32 s6, s5, 0xfffe7960

; SI-NEXT:    v_sub_i32_e32 v3, vcc, v3, v2

; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3

; SI-NEXT:    v_mul_lo_u32 v5, v2, v3

; SI-NEXT:    v_mul_hi_u32 v6, v2, v4

; SI-NEXT:    v_mul_hi_u32 v7, v2, v3

; SI-NEXT:    v_mul_hi_u32 v8, s5, v3

; SI-NEXT:    v_mul_lo_u32 v3, s5, v3

; SI-NEXT:    v_add_i32_e32 v5, vcc, v6, v5

; SI-NEXT:    v_addc_u32_e32 v6, vcc, 0, v7, vcc

; SI-NEXT:    v_mul_lo_u32 v7, s5, v4

; SI-NEXT:    v_mul_hi_u32 v4, s5, v4

; SI-NEXT:    s_mov_b32 s4, 0x186a0

; SI-NEXT:    v_sub_i32_e32 v4, vcc, v4, v2

; SI-NEXT:    v_add_i32_e32 v4, vcc, v5, v4

; SI-NEXT:    v_mul_lo_u32 v5, v2, v4

; SI-NEXT:    v_mul_hi_u32 v7, v2, v6

; SI-NEXT:    v_mul_hi_u32 v8, v2, v4

; SI-NEXT:    v_mul_hi_u32 v9, v3, v4

; SI-NEXT:    v_mul_lo_u32 v4, v3, v4

; SI-NEXT:    v_add_i32_e32 v5, vcc, v7, v5

; SI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc

; SI-NEXT:    v_mul_lo_u32 v8, v3, v6

; SI-NEXT:    v_mul_hi_u32 v6, v3, v6

; SI-NEXT:    v_add_i32_e32 v5, vcc, v5, v8

; SI-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc

; SI-NEXT:    v_addc_u32_e32 v6, vcc, 0, v9, vcc

; SI-NEXT:    v_add_i32_e32 v4, vcc, v5, v4

; SI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc

; SI-NEXT:    v_add_i32_e32 v2, vcc, v2, v4

; SI-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc

; SI-NEXT:    v_add_i32_e32 v5, vcc, v5, v7

; SI-NEXT:    v_addc_u32_e32 v4, vcc, v6, v4, vcc

; SI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v8, vcc

; SI-NEXT:    v_add_i32_e32 v3, vcc, v4, v3

; SI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v5, vcc

; SI-NEXT:    v_mov_b32_e32 v5, s5

; SI-NEXT:    v_add_i32_e32 v2, vcc, v2, v3

; SI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc

; SI-NEXT:    v_mul_lo_u32 v4, v0, v3

; SI-NEXT:    v_mul_hi_u32 v5, v0, v2

; SI-NEXT:    v_mul_hi_u32 v6, v0, v3
@@ -2687,39 +2666,20 @@ define i64 @v_test_udiv64_mulhi_fold(i64 %arg) { 
; GCN-LABEL: v_test_udiv64_mulhi_fold:

; GCN:       ; %bb.0:

; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)

; GCN-NEXT:    v_mov_b32_e32 v2, 0x4f800000

; GCN-NEXT:    v_madak_f32 v2, 0, v2, 0x47c35000

; GCN-NEXT:    v_rcp_f32_e32 v2, v2

; GCN-NEXT:    v_mov_b32_e32 v4, 0xa7c5

; GCN-NEXT:    v_mul_u32_u24_e32 v3, 0x500, v4

; GCN-NEXT:    v_mul_hi_u32_u24_e32 v2, 0x500, v4

; GCN-NEXT:    v_add_u32_e32 v3, vcc, 0x4237, v3

; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v2, vcc

; GCN-NEXT:    v_add_u32_e32 v6, vcc, 0xa9000000, v3

; GCN-NEXT:    s_mov_b32 s6, 0xfffe7960

; GCN-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2

; GCN-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2

; GCN-NEXT:    v_trunc_f32_e32 v3, v3

; GCN-NEXT:    v_madmk_f32 v2, v3, 0xcf800000, v2

; GCN-NEXT:    v_cvt_u32_f32_e32 v6, v2

; GCN-NEXT:    v_cvt_u32_f32_e32 v7, v3

; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0

; GCN-NEXT:    v_mul_lo_u32 v4, v7, s6

; GCN-NEXT:    v_sub_u32_e32 v3, vcc, v3, v6

; GCN-NEXT:    v_add_u32_e32 v8, vcc, v3, v4

; GCN-NEXT:    v_mul_hi_u32 v5, v6, v2

; GCN-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v6, v8, 0

; GCN-NEXT:    v_add_u32_e32 v9, vcc, v5, v3

; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0

; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v4, vcc

; GCN-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v7, v8, 0

; GCN-NEXT:    v_add_u32_e32 v2, vcc, v9, v2

; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v10, v3, vcc

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc

; GCN-NEXT:    v_add_u32_e32 v2, vcc, v2, v4

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc

; GCN-NEXT:    v_add_u32_e32 v6, vcc, v6, v2

; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v7, v3, vcc

; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0

; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v5, v4, vcc

; GCN-NEXT:    v_mul_lo_u32 v4, v7, s6

; GCN-NEXT:    v_sub_u32_e32 v3, vcc, v3, v6

; GCN-NEXT:    v_mul_hi_u32 v8, v6, v2

; GCN-NEXT:    v_add_u32_e32 v5, vcc, v4, v3

; GCN-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0

; GCN-NEXT:    v_mul_hi_u32 v8, v6, v2

; GCN-NEXT:    v_add_u32_e32 v8, vcc, v8, v3

; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0

; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v4, vcc

llvm/test/CodeGen/AMDGPU/udiv64.ll

+55 −105
Original line number Diff line number Diff line
@@ -1326,64 +1326,37 @@ define i64 @v_test_udiv_pow2_k_den_i64(i64 %x) { 
define amdgpu_kernel void @s_test_udiv_k_den_i64(ptr addrspace(1) %out, i64 %x) {

; GCN-LABEL: s_test_udiv_k_den_i64:

; GCN:       ; %bb.0:

; GCN-NEXT:    v_mov_b32_e32 v0, 0x4f800000

; GCN-NEXT:    v_madak_f32 v0, 0, v0, 0x41c00000

; GCN-NEXT:    v_rcp_f32_e32 v0, v0

; GCN-NEXT:    s_movk_i32 s8, 0xffe8

; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9

; GCN-NEXT:    s_mov_b32 s3, 0xf000

; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0

; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0

; GCN-NEXT:    v_trunc_f32_e32 v1, v1

; GCN-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0

; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0

; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1

; GCN-NEXT:    s_mov_b32 s2, -1

; GCN-NEXT:    s_add_u32 s1, 0, 0xaaaa0000

; GCN-NEXT:    v_mov_b32_e32 v0, 0xffffffe8

; GCN-NEXT:    v_mul_hi_u32 v0, s1, v0

; GCN-NEXT:    s_addc_u32 s8, 0, 42

; GCN-NEXT:    s_add_i32 s8, s8, 0xaaaaa80

; GCN-NEXT:    s_waitcnt lgkmcnt(0)

; GCN-NEXT:    s_mov_b32 s0, s4

; GCN-NEXT:    v_mul_hi_u32 v2, v0, s8

; GCN-NEXT:    v_mul_lo_u32 v4, v1, s8

; GCN-NEXT:    v_mul_lo_u32 v3, v0, s8

; GCN-NEXT:    s_mov_b32 s1, s5

; GCN-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0

; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4

; GCN-NEXT:    v_mul_hi_u32 v5, v0, v3

; GCN-NEXT:    v_mul_lo_u32 v4, v0, v2

; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2

; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2

; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2

; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4

; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc

; GCN-NEXT:    v_mul_lo_u32 v6, v1, v3

; GCN-NEXT:    v_mul_hi_u32 v3, v1, v3

; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc

; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc

; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc

; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2

; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc

; GCN-NEXT:    v_mul_hi_u32 v2, v0, s8

; GCN-NEXT:    v_mul_lo_u32 v3, v1, s8

; GCN-NEXT:    v_mul_lo_u32 v4, v0, s8

; GCN-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0

; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2

; GCN-NEXT:    v_mul_lo_u32 v3, v0, v2

; GCN-NEXT:    v_mul_hi_u32 v5, v0, v4

; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2

; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2

; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2

; GCN-NEXT:    v_add_i32_e32 v3, vcc, v5, v3

; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc

; GCN-NEXT:    v_mul_lo_u32 v6, v1, v4

; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4

; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v6

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc

; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc

; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2

; GCN-NEXT:    s_mul_i32 s4, s1, 0xffffffe8

; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s1, v0

; GCN-NEXT:    s_mul_i32 s9, s8, 0xffffffe8

; GCN-NEXT:    v_mov_b32_e32 v1, s4

; GCN-NEXT:    v_add_i32_e32 v0, vcc, s9, v0

; GCN-NEXT:    v_mul_hi_u32 v2, s8, v1

; GCN-NEXT:    v_mul_lo_u32 v3, s1, v0

; GCN-NEXT:    v_mul_hi_u32 v1, s1, v1

; GCN-NEXT:    v_mul_hi_u32 v4, s1, v0

; GCN-NEXT:    s_mul_i32 s4, s8, s4

; GCN-NEXT:    s_mov_b32 s3, 0xf000

; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc

; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2

; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc

; GCN-NEXT:    v_mul_hi_u32 v4, s8, v0

; GCN-NEXT:    v_mul_lo_u32 v0, s8, v0

; GCN-NEXT:    v_add_i32_e32 v1, vcc, s4, v1

; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v3, v2, vcc

; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v4, vcc

; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0

; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc

; GCN-NEXT:    v_mov_b32_e32 v2, s8

; GCN-NEXT:    v_add_i32_e32 v0, vcc, s1, v0

; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc

; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1

; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0

; GCN-NEXT:    v_mul_hi_u32 v4, s6, v1
@@ -1393,6 +1366,8 @@ define amdgpu_kernel void @s_test_udiv_k_den_i64(ptr addrspace(1) %out, i64 %x) 
; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc

; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0

; GCN-NEXT:    v_mul_hi_u32 v0, s7, v0

; GCN-NEXT:    s_mov_b32 s2, -1

; GCN-NEXT:    s_mov_b32 s1, s5

; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4

; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc

; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v5, vcc
@@ -1502,58 +1477,33 @@ define i64 @v_test_udiv_k_den_i64(i64 %x) { 
; GCN-LABEL: v_test_udiv_k_den_i64:

; GCN:       ; %bb.0:

; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)

; GCN-NEXT:    v_mov_b32_e32 v2, 0x4f800000

; GCN-NEXT:    v_madak_f32 v2, 0, v2, 0x41c00000

; GCN-NEXT:    v_rcp_f32_e32 v2, v2

; GCN-NEXT:    s_movk_i32 s4, 0xffe8

; GCN-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2

; GCN-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2

; GCN-NEXT:    v_trunc_f32_e32 v3, v3

; GCN-NEXT:    v_madmk_f32 v2, v3, 0xcf800000, v2

; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2

; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3

; GCN-NEXT:    v_mul_hi_u32 v4, v2, s4

; GCN-NEXT:    v_mul_lo_u32 v6, v3, s4

; GCN-NEXT:    v_mul_lo_u32 v5, v2, s4

; GCN-NEXT:    v_sub_i32_e32 v4, vcc, v4, v2

; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6

; GCN-NEXT:    v_mul_hi_u32 v7, v2, v5

; GCN-NEXT:    v_mul_lo_u32 v6, v2, v4

; GCN-NEXT:    v_mul_hi_u32 v8, v2, v4

; GCN-NEXT:    v_mul_hi_u32 v9, v3, v4

; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4

; GCN-NEXT:    v_add_i32_e32 v6, vcc, v7, v6

; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc

; GCN-NEXT:    v_mul_lo_u32 v8, v3, v5

; GCN-NEXT:    v_mul_hi_u32 v5, v3, v5

; GCN-NEXT:    v_add_i32_e32 v6, vcc, v6, v8

; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc

; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v9, vcc

; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4

; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc

; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc

; GCN-NEXT:    v_mul_hi_u32 v4, v2, s4

; GCN-NEXT:    v_mul_lo_u32 v5, v3, s4

; GCN-NEXT:    v_mul_lo_u32 v6, v2, s4

; GCN-NEXT:    v_sub_i32_e32 v4, vcc, v4, v2

; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4

; GCN-NEXT:    v_mul_lo_u32 v5, v2, v4

; GCN-NEXT:    v_mul_hi_u32 v7, v2, v6

; GCN-NEXT:    v_mul_hi_u32 v8, v2, v4

; GCN-NEXT:    v_mul_hi_u32 v9, v3, v4

; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4

; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5

; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc

; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6

; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6

; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v8

; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc

; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v9, vcc

; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4

; GCN-NEXT:    s_add_u32 s4, 0, 0xaaaa0000

; GCN-NEXT:    v_mov_b32_e32 v2, 0xffffffe8

; GCN-NEXT:    v_mul_hi_u32 v2, s4, v2

; GCN-NEXT:    s_addc_u32 s5, 0, 42

; GCN-NEXT:    s_add_i32 s5, s5, 0xaaaaa80

; GCN-NEXT:    s_mul_i32 s6, s4, 0xffffffe8

; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s4, v2

; GCN-NEXT:    s_mul_i32 s7, s5, 0xffffffe8

; GCN-NEXT:    v_mov_b32_e32 v3, s6

; GCN-NEXT:    v_add_i32_e32 v2, vcc, s7, v2

; GCN-NEXT:    v_mul_hi_u32 v4, s5, v3

; GCN-NEXT:    v_mul_lo_u32 v5, s4, v2

; GCN-NEXT:    v_mul_hi_u32 v3, s4, v3

; GCN-NEXT:    v_mul_hi_u32 v6, s4, v2

; GCN-NEXT:    s_mul_i32 s6, s5, s6

; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5

; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc

; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc

; GCN-NEXT:    v_mul_hi_u32 v6, s5, v2

; GCN-NEXT:    v_mul_lo_u32 v2, s5, v2

; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc

; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v6, vcc

; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc

; GCN-NEXT:    v_mov_b32_e32 v4, s5

; GCN-NEXT:    v_add_i32_e32 v2, vcc, s4, v2

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v3, vcc

; GCN-NEXT:    v_mul_lo_u32 v4, v0, v3

; GCN-NEXT:    v_mul_hi_u32 v5, v0, v2

; GCN-NEXT:    v_mul_hi_u32 v6, v0, v3

llvm/test/CodeGen/AMDGPU/urem64.ll

+30 −55
Original line number Diff line number Diff line
@@ -894,64 +894,36 @@ define amdgpu_kernel void @s_test_urem_k_num_i64(ptr addrspace(1) %out, i64 %x) 
define amdgpu_kernel void @s_test_urem_k_den_i64(ptr addrspace(1) %out, i64 %x) {

; GCN-LABEL: s_test_urem_k_den_i64:

; GCN:       ; %bb.0:

; GCN-NEXT:    v_mov_b32_e32 v0, 0x4f800000

; GCN-NEXT:    v_madak_f32 v0, 0, v0, 0x41c00000

; GCN-NEXT:    v_rcp_f32_e32 v0, v0

; GCN-NEXT:    s_movk_i32 s2, 0xffe8

; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9

; GCN-NEXT:    s_add_u32 s0, 0, 0xaaaa0000

; GCN-NEXT:    v_mov_b32_e32 v0, 0xffffffe8

; GCN-NEXT:    v_mul_hi_u32 v0, s0, v0

; GCN-NEXT:    s_addc_u32 s1, 0, 42

; GCN-NEXT:    s_add_i32 s1, s1, 0xaaaaa80

; GCN-NEXT:    s_mul_i32 s8, s0, 0xffffffe8

; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0

; GCN-NEXT:    s_mul_i32 s9, s1, 0xffffffe8

; GCN-NEXT:    v_mov_b32_e32 v1, s8

; GCN-NEXT:    v_add_i32_e32 v0, vcc, s9, v0

; GCN-NEXT:    v_mul_hi_u32 v2, s1, v1

; GCN-NEXT:    v_mul_lo_u32 v3, s0, v0

; GCN-NEXT:    v_mul_hi_u32 v1, s0, v1

; GCN-NEXT:    v_mul_hi_u32 v4, s0, v0

; GCN-NEXT:    s_mul_i32 s8, s1, s8

; GCN-NEXT:    s_mov_b32 s3, 0xf000

; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0

; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0

; GCN-NEXT:    v_trunc_f32_e32 v1, v1

; GCN-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0

; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0

; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1

; GCN-NEXT:    s_waitcnt lgkmcnt(0)

; GCN-NEXT:    s_mov_b32 s0, s4

; GCN-NEXT:    s_mov_b32 s1, s5

; GCN-NEXT:    v_mul_hi_u32 v2, v0, s2

; GCN-NEXT:    v_mul_lo_u32 v4, v1, s2

; GCN-NEXT:    v_mul_lo_u32 v3, v0, s2

; GCN-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0

; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4

; GCN-NEXT:    v_mul_hi_u32 v5, v0, v3

; GCN-NEXT:    v_mul_lo_u32 v4, v0, v2

; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2

; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2

; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2

; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4

; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc

; GCN-NEXT:    v_mul_lo_u32 v6, v1, v3

; GCN-NEXT:    v_mul_hi_u32 v3, v1, v3

; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc

; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc

; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2

; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc

; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2

; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc

; GCN-NEXT:    v_mul_hi_u32 v2, v0, s2

; GCN-NEXT:    v_mul_lo_u32 v3, v1, s2

; GCN-NEXT:    v_mul_lo_u32 v4, v0, s2

; GCN-NEXT:    s_mov_b32 s2, -1

; GCN-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0

; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2

; GCN-NEXT:    v_mul_lo_u32 v3, v0, v2

; GCN-NEXT:    v_mul_hi_u32 v5, v0, v4

; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2

; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2

; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2

; GCN-NEXT:    v_add_i32_e32 v3, vcc, v5, v3

; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc

; GCN-NEXT:    v_mul_lo_u32 v6, v1, v4

; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4

; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v6

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc

; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc

; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2

; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc

; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2

; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc

; GCN-NEXT:    v_mul_hi_u32 v4, s1, v0

; GCN-NEXT:    v_mul_lo_u32 v0, s1, v0

; GCN-NEXT:    v_add_i32_e32 v1, vcc, s8, v1

; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v3, v2, vcc

; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v4, vcc

; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0

; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc

; GCN-NEXT:    v_mov_b32_e32 v2, s1

; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v0

; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc

; GCN-NEXT:    s_waitcnt lgkmcnt(0)

; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1

; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0

; GCN-NEXT:    v_mul_hi_u32 v4, s6, v1
@@ -961,6 +933,8 @@ define amdgpu_kernel void @s_test_urem_k_den_i64(ptr addrspace(1) %out, i64 %x) 
; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc

; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0

; GCN-NEXT:    v_mul_hi_u32 v0, s7, v0

; GCN-NEXT:    s_mov_b32 s2, -1

; GCN-NEXT:    s_mov_b32 s0, s4

; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4

; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc

; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v5, vcc
@@ -969,6 +943,7 @@ define amdgpu_kernel void @s_test_urem_k_den_i64(ptr addrspace(1) %out, i64 %x) 
; GCN-NEXT:    v_mul_lo_u32 v1, v1, 24

; GCN-NEXT:    v_mul_hi_u32 v2, v0, 24

; GCN-NEXT:    v_mul_lo_u32 v0, v0, 24

; GCN-NEXT:    s_mov_b32 s1, s5

; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v2

; GCN-NEXT:    v_mov_b32_e32 v2, s7

; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0