Thanks to visit codestin.com
Credit goes to github.com

Skip to content

[AArch64][SVE] Add patterns for bit-select instructions. #138689

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

[AArch64][SVE] Add patterns for bit-select instructions. #138689

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6f92bd5
Add tests.
rj-jesus May 1, 2025
c25e7c9
[AArch64][SVE] Add patterns for bit-select instructions.
rj-jesus May 1, 2025
e075270
Lower bit-sel intrinsics to AArch64ISD::BSP.
rj-jesus May 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22042,6 +22042,30 @@ SDValue tryLowerPartialReductionToWideAdd(SDNode *N,
return DAG.getNode(TopOpcode, DL, AccVT, BottomNode, ExtOp);
}

static SDValue combineSVEBitSel(unsigned IID, SDNode *N, SelectionDAG &DAG) {
SDLoc DL(N);
EVT VT = N->getValueType(0);
SDValue Op1 = N->getOperand(1);
SDValue Op2 = N->getOperand(2);
SDValue Op3 = N->getOperand(3);

switch (IID) {
default:
llvm_unreachable("Called with wrong intrinsic!");
case Intrinsic::aarch64_sve_bsl:
return DAG.getNode(AArch64ISD::BSP, DL, VT, Op3, Op1, Op2);
case Intrinsic::aarch64_sve_bsl1n:
return DAG.getNode(AArch64ISD::BSP, DL, VT, Op3, DAG.getNOT(DL, Op1, VT),
Op2);
case Intrinsic::aarch64_sve_bsl2n:
return DAG.getNode(AArch64ISD::BSP, DL, VT, Op3, Op1,
DAG.getNOT(DL, Op2, VT));
case Intrinsic::aarch64_sve_nbsl:
return DAG.getNOT(DL, DAG.getNode(AArch64ISD::BSP, DL, VT, Op3, Op1, Op2),
VT);
}
}

static SDValue performIntrinsicCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const AArch64Subtarget *Subtarget) {
Expand Down Expand Up @@ -22364,6 +22388,11 @@ static SDValue performIntrinsicCombine(SDNode *N,
AArch64CC::LAST_ACTIVE);
case Intrinsic::aarch64_sve_whilelo:
return tryCombineWhileLo(N, DCI, Subtarget);
case Intrinsic::aarch64_sve_bsl:
case Intrinsic::aarch64_sve_bsl1n:
case Intrinsic::aarch64_sve_bsl2n:
case Intrinsic::aarch64_sve_nbsl:
return combineSVEBitSel(IID, N, DAG);
}
return SDValue();
}
Expand Down
20 changes: 13 additions & 7 deletions llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -545,12 +545,18 @@ def AArch64umulh : PatFrag<(ops node:$op1, node:$op2),


def AArch64bsl : PatFrags<(ops node:$Op1, node:$Op2, node:$Op3),
[(int_aarch64_sve_bsl node:$Op1, node:$Op2, node:$Op3),
(AArch64bsp node:$Op3, node:$Op1, node:$Op2)]>;
[(AArch64bsp node:$Op3, node:$Op1, node:$Op2),
(or (and node:$Op1, node:$Op3), (and node:$Op2, (vnot node:$Op3)))]>;

def AArch64nbsl : PatFrags<(ops node:$Op1, node:$Op2, node:$Op3),
[(int_aarch64_sve_nbsl node:$Op1, node:$Op2, node:$Op3),
(vnot (AArch64bsp node:$Op3, node:$Op1, node:$Op2))]>;
def AArch64bsl1n : PatFrag<(ops node:$Op1, node:$Op2, node:$Op3),
(AArch64bsl (vnot node:$Op1), node:$Op2, node:$Op3)>;

def AArch64bsl2n : PatFrags<(ops node:$Op1, node:$Op2, node:$Op3),
[(AArch64bsl node:$Op1, (vnot node:$Op2), node:$Op3),
(or (and node:$Op1, node:$Op3), (vnot (or node:$Op2, node:$Op3)))]>;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That De Morgan has a lot to answer for :(


def AArch64nbsl : PatFrag<(ops node:$Op1, node:$Op2, node:$Op3),
(vnot (AArch64bsl node:$Op1, node:$Op2, node:$Op3))>;


let Predicates = [HasSVE] in {
Expand Down Expand Up @@ -3923,8 +3929,8 @@ let Predicates = [HasSVE2_or_SME] in {
defm EOR3_ZZZZ : sve2_int_bitwise_ternary_op<0b000, "eor3", AArch64eor3>;
defm BCAX_ZZZZ : sve2_int_bitwise_ternary_op<0b010, "bcax", AArch64bcax>;
defm BSL_ZZZZ : sve2_int_bitwise_ternary_op<0b001, "bsl", AArch64bsl>;
defm BSL1N_ZZZZ : sve2_int_bitwise_ternary_op<0b011, "bsl1n", int_aarch64_sve_bsl1n>;
defm BSL2N_ZZZZ : sve2_int_bitwise_ternary_op<0b101, "bsl2n", int_aarch64_sve_bsl2n>;
defm BSL1N_ZZZZ : sve2_int_bitwise_ternary_op<0b011, "bsl1n", AArch64bsl1n>;
defm BSL2N_ZZZZ : sve2_int_bitwise_ternary_op<0b101, "bsl2n", AArch64bsl2n>;
defm NBSL_ZZZZ : sve2_int_bitwise_ternary_op<0b111, "nbsl", AArch64nbsl>;

// SVE2 bitwise xor and rotate right by immediate
Expand Down
206 changes: 206 additions & 0 deletions llvm/test/CodeGen/AArch64/sve2-bsl.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,3 +93,209 @@ define <vscale x 2 x i64> @nbsl_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b
%4 = xor <vscale x 2 x i64> %3, splat(i64 -1)
ret <vscale x 2 x i64> %4
}

; Test BSL/NBSL/BSL1N/BSL2N code generation for:
; #define BSL(x,y,z) ( ((x) & (z)) | ( (y) & ~(z)))
; #define NBSL(x,y,z) (~(((x) & (z)) | ( (y) & ~(z))))
; #define BSL1N(x,y,z) ( (~(x) & (z)) | ( (y) & ~(z)))
; #define BSL2N(x,y,z) ( ((x) & (z)) | (~(y) & ~(z)))

define <vscale x 16 x i8> @codegen_bsl_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <vscale x 16 x i8> %2) {
; CHECK-LABEL: codegen_bsl_i8:
; CHECK: // %bb.0:
; CHECK-NEXT: bsl z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = and <vscale x 16 x i8> %2, %0
%5 = xor <vscale x 16 x i8> %2, splat (i8 -1)
%6 = and <vscale x 16 x i8> %1, %5
%7 = or <vscale x 16 x i8> %4, %6
ret <vscale x 16 x i8> %7
}

define <vscale x 16 x i8> @codegen_nbsl_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <vscale x 16 x i8> %2) {
; CHECK-LABEL: codegen_nbsl_i8:
; CHECK: // %bb.0:
; CHECK-NEXT: nbsl z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = and <vscale x 16 x i8> %2, %0
%5 = xor <vscale x 16 x i8> %2, splat (i8 -1)
%6 = and <vscale x 16 x i8> %1, %5
%7 = or <vscale x 16 x i8> %4, %6
%8 = xor <vscale x 16 x i8> %7, splat (i8 -1)
ret <vscale x 16 x i8> %8
}

define <vscale x 16 x i8> @codegen_bsl1n_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <vscale x 16 x i8> %2) {
; CHECK-LABEL: codegen_bsl1n_i8:
; CHECK: // %bb.0:
; CHECK-NEXT: bsl1n z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = xor <vscale x 16 x i8> %0, splat (i8 -1)
%5 = and <vscale x 16 x i8> %2, %4
%6 = xor <vscale x 16 x i8> %2, splat (i8 -1)
%7 = and <vscale x 16 x i8> %1, %6
%8 = or <vscale x 16 x i8> %5, %7
ret <vscale x 16 x i8> %8
}

define <vscale x 16 x i8> @codegen_bsl2n_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <vscale x 16 x i8> %2) {
; CHECK-LABEL: codegen_bsl2n_i8:
; CHECK: // %bb.0:
; CHECK-NEXT: bsl2n z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = and <vscale x 16 x i8> %2, %0
%5 = or <vscale x 16 x i8> %2, %1
%6 = xor <vscale x 16 x i8> %5, splat (i8 -1)
%7 = or <vscale x 16 x i8> %4, %6
ret <vscale x 16 x i8> %7
}

define <vscale x 8 x i16> @codegen_bsl_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <vscale x 8 x i16> %2) {
; CHECK-LABEL: codegen_bsl_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: bsl z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = and <vscale x 8 x i16> %2, %0
%5 = xor <vscale x 8 x i16> %2, splat (i16 -1)
%6 = and <vscale x 8 x i16> %1, %5
%7 = or <vscale x 8 x i16> %4, %6
ret <vscale x 8 x i16> %7
}

define <vscale x 8 x i16> @codegen_nbsl_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <vscale x 8 x i16> %2) {
; CHECK-LABEL: codegen_nbsl_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: nbsl z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = and <vscale x 8 x i16> %2, %0
%5 = xor <vscale x 8 x i16> %2, splat (i16 -1)
%6 = and <vscale x 8 x i16> %1, %5
%7 = or <vscale x 8 x i16> %4, %6
%8 = xor <vscale x 8 x i16> %7, splat (i16 -1)
ret <vscale x 8 x i16> %8
}

define <vscale x 8 x i16> @codegen_bsl1n_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <vscale x 8 x i16> %2) {
; CHECK-LABEL: codegen_bsl1n_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: bsl1n z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = xor <vscale x 8 x i16> %0, splat (i16 -1)
%5 = and <vscale x 8 x i16> %2, %4
%6 = xor <vscale x 8 x i16> %2, splat (i16 -1)
%7 = and <vscale x 8 x i16> %1, %6
%8 = or <vscale x 8 x i16> %5, %7
ret <vscale x 8 x i16> %8
}

define <vscale x 8 x i16> @codegen_bsl2n_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <vscale x 8 x i16> %2) {
; CHECK-LABEL: codegen_bsl2n_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: bsl2n z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = and <vscale x 8 x i16> %2, %0
%5 = or <vscale x 8 x i16> %2, %1
%6 = xor <vscale x 8 x i16> %5, splat (i16 -1)
%7 = or <vscale x 8 x i16> %4, %6
ret <vscale x 8 x i16> %7
}

define <vscale x 4 x i32> @codegen_bsl_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <vscale x 4 x i32> %2) {
; CHECK-LABEL: codegen_bsl_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: bsl z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = and <vscale x 4 x i32> %2, %0
%5 = xor <vscale x 4 x i32> %2, splat (i32 -1)
%6 = and <vscale x 4 x i32> %1, %5
%7 = or <vscale x 4 x i32> %4, %6
ret <vscale x 4 x i32> %7
}

define <vscale x 4 x i32> @codegen_nbsl_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <vscale x 4 x i32> %2) {
; CHECK-LABEL: codegen_nbsl_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: nbsl z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = and <vscale x 4 x i32> %2, %0
%5 = xor <vscale x 4 x i32> %2, splat (i32 -1)
%6 = and <vscale x 4 x i32> %1, %5
%7 = or <vscale x 4 x i32> %4, %6
%8 = xor <vscale x 4 x i32> %7, splat (i32 -1)
ret <vscale x 4 x i32> %8
}

define <vscale x 4 x i32> @codegen_bsl1n_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <vscale x 4 x i32> %2) {
; CHECK-LABEL: codegen_bsl1n_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: bsl1n z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = xor <vscale x 4 x i32> %0, splat (i32 -1)
%5 = and <vscale x 4 x i32> %2, %4
%6 = xor <vscale x 4 x i32> %2, splat (i32 -1)
%7 = and <vscale x 4 x i32> %1, %6
%8 = or <vscale x 4 x i32> %5, %7
ret <vscale x 4 x i32> %8
}

define <vscale x 4 x i32> @codegen_bsl2n_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <vscale x 4 x i32> %2) {
; CHECK-LABEL: codegen_bsl2n_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: bsl2n z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = and <vscale x 4 x i32> %2, %0
%5 = or <vscale x 4 x i32> %2, %1
%6 = xor <vscale x 4 x i32> %5, splat (i32 -1)
%7 = or <vscale x 4 x i32> %4, %6
ret <vscale x 4 x i32> %7
}

define <vscale x 2 x i64> @codegen_bsl_i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <vscale x 2 x i64> %2) {
; CHECK-LABEL: codegen_bsl_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: bsl z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = and <vscale x 2 x i64> %2, %0
%5 = xor <vscale x 2 x i64> %2, splat (i64 -1)
%6 = and <vscale x 2 x i64> %1, %5
%7 = or <vscale x 2 x i64> %4, %6
ret <vscale x 2 x i64> %7
}

define <vscale x 2 x i64> @codegen_nbsl_i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <vscale x 2 x i64> %2) {
; CHECK-LABEL: codegen_nbsl_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: nbsl z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = and <vscale x 2 x i64> %2, %0
%5 = xor <vscale x 2 x i64> %2, splat (i64 -1)
%6 = and <vscale x 2 x i64> %1, %5
%7 = or <vscale x 2 x i64> %4, %6
%8 = xor <vscale x 2 x i64> %7, splat (i64 -1)
ret <vscale x 2 x i64> %8
}

define <vscale x 2 x i64> @codegen_bsl1n_i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <vscale x 2 x i64> %2) {
; CHECK-LABEL: codegen_bsl1n_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: bsl1n z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = xor <vscale x 2 x i64> %0, splat (i64 -1)
%5 = and <vscale x 2 x i64> %2, %4
%6 = xor <vscale x 2 x i64> %2, splat (i64 -1)
%7 = and <vscale x 2 x i64> %1, %6
%8 = or <vscale x 2 x i64> %5, %7
ret <vscale x 2 x i64> %8
}

define <vscale x 2 x i64> @codegen_bsl2n_i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <vscale x 2 x i64> %2) {
; CHECK-LABEL: codegen_bsl2n_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: bsl2n z0.d, z0.d, z1.d, z2.d
; CHECK-NEXT: ret
%4 = and <vscale x 2 x i64> %2, %0
%5 = or <vscale x 2 x i64> %2, %1
%6 = xor <vscale x 2 x i64> %5, splat (i64 -1)
%7 = or <vscale x 2 x i64> %4, %6
ret <vscale x 2 x i64> %7
}