From 1008f900f6a30e284424e36aba432289722f6f6a Mon Sep 17 00:00:00 2001 From: Jorropo Date: Tue, 3 May 2022 12:24:25 +0200 Subject: [PATCH] cmd/compile: ssa rewrite x-(x&y) => x&^y This allows to rewrite: AND BX, AX SUB AX, BX Into (with GOAMD64=v3): ANDN AX, BX, AX Which is twice as fast, and smaller. Into (without GOAMD64=v3): NOT BX AND AX, BX Which if AX's dependency isn't resolved yet but BX is in most cases twice as fast because now the NOT BX and AX's depency can execute in parallel. In other cases, it has negligible positive impact. Other instructions set will also benefit from the pipelining advantage And also benefit from the smaller ANDN instruction if they have something similar. Help reduce ChaCha20 and Poly1305 to #52563 Change-Id: I0a0a3fe553b857e13c61fbf52f9765377615a898 Reviewed-on: https://go-review.googlesource.com/c/go/+/786700 Reviewed-by: Keith Randall Reviewed-by: Keith Randall Auto-Submit: Jorropo LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com Reviewed-by: Carlos Amedee --- src/cmd/compile/internal/ssa/_gen/AMD64.rules | 1 + .../compile/internal/ssa/_gen/generic.rules | 3 + src/cmd/compile/internal/ssa/rewriteAMD64.go | 50 ++++++++++ .../compile/internal/ssa/rewritegeneric.go | 96 +++++++++++++++++++ test/codegen/bmi.go | 12 +++ 5 files changed, 162 insertions(+) diff --git a/src/cmd/compile/internal/ssa/_gen/AMD64.rules b/src/cmd/compile/internal/ssa/_gen/AMD64.rules index f9adaf8fc4..4988af6626 100644 --- a/src/cmd/compile/internal/ssa/_gen/AMD64.rules +++ b/src/cmd/compile/internal/ssa/_gen/AMD64.rules @@ -1602,6 +1602,7 @@ // CPUID feature: BMI1. (AND(Q|L) x (NOT(Q|L) y)) && buildcfg.GOAMD64 >= 3 => (ANDN(Q|L) x y) +(SUB(Q|L) x (AND(Q|L) x y)) && buildcfg.GOAMD64 >= 3 => (ANDN(Q|L) x y) (AND(Q|L) x (NEG(Q|L) x)) && buildcfg.GOAMD64 >= 3 => (BLSI(Q|L) x) (XOR(Q|L) x (ADD(Q|L)const [-1] x)) && buildcfg.GOAMD64 >= 3 => (BLSMSK(Q|L) x) (AND(Q|L) x (ADD(Q|L)const [-1] x)) && buildcfg.GOAMD64 >= 3 => (Select0 (BLSR(Q|L) x)) diff --git a/src/cmd/compile/internal/ssa/_gen/generic.rules b/src/cmd/compile/internal/ssa/_gen/generic.rules index d70496c278..ea5a7f6e44 100644 --- a/src/cmd/compile/internal/ssa/_gen/generic.rules +++ b/src/cmd/compile/internal/ssa/_gen/generic.rules @@ -708,6 +708,9 @@ (Or(64|32|16|8) x (Or(64|32|16|8) x y)) => (Or(64|32|16|8) x y) (Xor(64|32|16|8) x (Xor(64|32|16|8) x y)) => y +// x-(x&y) == x&^y +(Sub(64|32|16|8) x (And(64|32|16|8) x y)) => (And(64|32|16|8) x (Com(64|32|16|8) y)) + // Fold comparisons with numeric bounds (Less(64|32|16|8)U _ (Const(64|32|16|8) [0])) => (ConstBool [false]) (Leq(64|32|16|8)U (Const(64|32|16|8) [0]) _) => (ConstBool [true]) diff --git a/src/cmd/compile/internal/ssa/rewriteAMD64.go b/src/cmd/compile/internal/ssa/rewriteAMD64.go index d59a29af93..8f501c19d2 100644 --- a/src/cmd/compile/internal/ssa/rewriteAMD64.go +++ b/src/cmd/compile/internal/ssa/rewriteAMD64.go @@ -40193,6 +40193,31 @@ func rewriteValueAMD64_OpAMD64SUBL(v *Value) bool { v.AddArg3(x, ptr, mem) return true } + // match: (SUBL x (ANDL x y)) + // cond: buildcfg.GOAMD64 >= 3 + // result: (ANDNL x y) + for { + x := v_0 + if v_1.Op != OpAMD64ANDL { + break + } + _ = v_1.Args[1] + v_1_0 := v_1.Args[0] + v_1_1 := v_1.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_1_0, v_1_1 = _i0+1, v_1_1, v_1_0 { + if x != v_1_0 { + continue + } + y := v_1_1 + if !(buildcfg.GOAMD64 >= 3) { + continue + } + v.reset(OpAMD64ANDNL) + v.AddArg2(x, y) + return true + } + break + } return false } func rewriteValueAMD64_OpAMD64SUBLconst(v *Value) bool { @@ -40435,6 +40460,31 @@ func rewriteValueAMD64_OpAMD64SUBQ(v *Value) bool { v.AddArg3(x, ptr, mem) return true } + // match: (SUBQ x (ANDQ x y)) + // cond: buildcfg.GOAMD64 >= 3 + // result: (ANDNQ x y) + for { + x := v_0 + if v_1.Op != OpAMD64ANDQ { + break + } + _ = v_1.Args[1] + v_1_0 := v_1.Args[0] + v_1_1 := v_1.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_1_0, v_1_1 = _i0+1, v_1_1, v_1_0 { + if x != v_1_0 { + continue + } + y := v_1_1 + if !(buildcfg.GOAMD64 >= 3) { + continue + } + v.reset(OpAMD64ANDNQ) + v.AddArg2(x, y) + return true + } + break + } return false } func rewriteValueAMD64_OpAMD64SUBQborrow(v *Value) bool { diff --git a/src/cmd/compile/internal/ssa/rewritegeneric.go b/src/cmd/compile/internal/ssa/rewritegeneric.go index 89d58f7db0..24d80d9aa7 100644 --- a/src/cmd/compile/internal/ssa/rewritegeneric.go +++ b/src/cmd/compile/internal/ssa/rewritegeneric.go @@ -35967,6 +35967,30 @@ func rewriteValuegeneric_OpSub16(v *Value) bool { } break } + // match: (Sub16 x (And16 x y)) + // result: (And16 x (Com16 y)) + for { + x := v_0 + if v_1.Op != OpAnd16 { + break + } + t := v_1.Type + _ = v_1.Args[1] + v_1_0 := v_1.Args[0] + v_1_1 := v_1.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_1_0, v_1_1 = _i0+1, v_1_1, v_1_0 { + if x != v_1_0 { + continue + } + y := v_1_1 + v.reset(OpAnd16) + v0 := b.NewValue0(v.Pos, OpCom16, t) + v0.AddArg(y) + v.AddArg2(x, v0) + return true + } + break + } // match: (Sub16 (Add16 x y) x) // result: y for { @@ -36340,6 +36364,30 @@ func rewriteValuegeneric_OpSub32(v *Value) bool { } break } + // match: (Sub32 x (And32 x y)) + // result: (And32 x (Com32 y)) + for { + x := v_0 + if v_1.Op != OpAnd32 { + break + } + t := v_1.Type + _ = v_1.Args[1] + v_1_0 := v_1.Args[0] + v_1_1 := v_1.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_1_0, v_1_1 = _i0+1, v_1_1, v_1_0 { + if x != v_1_0 { + continue + } + y := v_1_1 + v.reset(OpAnd32) + v0 := b.NewValue0(v.Pos, OpCom32, t) + v0.AddArg(y) + v.AddArg2(x, v0) + return true + } + break + } // match: (Sub32 (Add32 x y) x) // result: y for { @@ -36737,6 +36785,30 @@ func rewriteValuegeneric_OpSub64(v *Value) bool { } break } + // match: (Sub64 x (And64 x y)) + // result: (And64 x (Com64 y)) + for { + x := v_0 + if v_1.Op != OpAnd64 { + break + } + t := v_1.Type + _ = v_1.Args[1] + v_1_0 := v_1.Args[0] + v_1_1 := v_1.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_1_0, v_1_1 = _i0+1, v_1_1, v_1_0 { + if x != v_1_0 { + continue + } + y := v_1_1 + v.reset(OpAnd64) + v0 := b.NewValue0(v.Pos, OpCom64, t) + v0.AddArg(y) + v.AddArg2(x, v0) + return true + } + break + } // match: (Sub64 (Add64 x y) x) // result: y for { @@ -37134,6 +37206,30 @@ func rewriteValuegeneric_OpSub8(v *Value) bool { } break } + // match: (Sub8 x (And8 x y)) + // result: (And8 x (Com8 y)) + for { + x := v_0 + if v_1.Op != OpAnd8 { + break + } + t := v_1.Type + _ = v_1.Args[1] + v_1_0 := v_1.Args[0] + v_1_1 := v_1.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_1_0, v_1_1 = _i0+1, v_1_1, v_1_0 { + if x != v_1_0 { + continue + } + y := v_1_1 + v.reset(OpAnd8) + v0 := b.NewValue0(v.Pos, OpCom8, t) + v0.AddArg(y) + v.AddArg2(x, v0) + return true + } + break + } // match: (Sub8 (Add8 x y) x) // result: y for { diff --git a/test/codegen/bmi.go b/test/codegen/bmi.go index c50f15a7e0..3c00dd328b 100644 --- a/test/codegen/bmi.go +++ b/test/codegen/bmi.go @@ -207,3 +207,15 @@ func shlrx32_load(x []uint32, i int, s uint32) uint32 { s = x[i+1] << s return s } + +func XSubXandYAndn64(x, y uint64) uint64 { + // amd64/v1:"ANDQ" "NOTQ" -"SUBQ" + // amd64/v3:"ANDNQ" -"SUBQ" -"ANDQ" + return x - (x & y) +} + +func XSubXandYAndn32(x, y uint32) uint32 { + // amd64/v1:"ANDL" "NOTL" -"SUBL" + // amd64/v3:"ANDNL" -"SUBL" -"ANDL" + return x - (x & y) +}