951 lines
25 KiB
Go
951 lines
25 KiB
Go
package arm64
|
|
|
|
import (
|
|
"encoding/hex"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
|
|
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
|
|
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
|
|
"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
|
|
"github.com/tetratelabs/wazero/internal/testing/require"
|
|
)
|
|
|
|
func TestMachine_LowerConditionalBranch(t *testing.T) {
|
|
cmpInSameGroupFromParams := func(
|
|
brz bool, intCond ssa.IntegerCmpCond, floatCond ssa.FloatCmpCond,
|
|
ctx *mockCompiler, builder ssa.Builder, m *machine,
|
|
) (instr *ssa.Instruction, verify func(t *testing.T)) {
|
|
m.StartLoweringFunction(10)
|
|
entry := builder.CurrentBlock()
|
|
isInt := intCond != ssa.IntegerCmpCondInvalid
|
|
|
|
var val1, val2 ssa.Value
|
|
if isInt {
|
|
val1 = entry.AddParam(builder, ssa.TypeI64)
|
|
val2 = entry.AddParam(builder, ssa.TypeI64)
|
|
ctx.vRegMap[val1], ctx.vRegMap[val2] = regToVReg(x1).SetRegType(regalloc.RegTypeInt), regToVReg(x2).SetRegType(regalloc.RegTypeInt)
|
|
} else {
|
|
val1 = entry.AddParam(builder, ssa.TypeF64)
|
|
val2 = entry.AddParam(builder, ssa.TypeF64)
|
|
ctx.vRegMap[val1], ctx.vRegMap[val2] = regToVReg(v1).SetRegType(regalloc.RegTypeFloat), regToVReg(v2).SetRegType(regalloc.RegTypeFloat)
|
|
}
|
|
|
|
var cmpInstr *ssa.Instruction
|
|
if isInt {
|
|
cmpInstr = builder.AllocateInstruction()
|
|
cmpInstr.AsIcmp(val1, val2, intCond)
|
|
builder.InsertInstruction(cmpInstr)
|
|
} else {
|
|
cmpInstr = builder.AllocateInstruction()
|
|
cmpInstr.AsFcmp(val1, val2, floatCond)
|
|
builder.InsertInstruction(cmpInstr)
|
|
}
|
|
|
|
cmpVal := cmpInstr.Return()
|
|
ctx.vRegMap[cmpVal] = 3
|
|
|
|
ctx.definitions[val1] = &backend.SSAValueDefinition{BlkParamVReg: ctx.vRegMap[val1], BlockParamValue: val1}
|
|
ctx.definitions[val2] = &backend.SSAValueDefinition{BlkParamVReg: ctx.vRegMap[val2], BlockParamValue: val2}
|
|
ctx.definitions[cmpVal] = &backend.SSAValueDefinition{Instr: cmpInstr}
|
|
b := builder.AllocateInstruction()
|
|
if brz {
|
|
b.AsBrz(cmpVal, nil, builder.AllocateBasicBlock())
|
|
} else {
|
|
b.AsBrnz(cmpVal, nil, builder.AllocateBasicBlock())
|
|
}
|
|
builder.InsertInstruction(b)
|
|
return b, func(t *testing.T) {
|
|
require.True(t, cmpInstr.Lowered())
|
|
}
|
|
}
|
|
|
|
icmpInSameGroupFromParamAndImm12 := func(brz bool, ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
|
|
m.StartLoweringFunction(10)
|
|
entry := builder.CurrentBlock()
|
|
v1 := entry.AddParam(builder, ssa.TypeI32)
|
|
|
|
iconst := builder.AllocateInstruction()
|
|
iconst.AsIconst32(0x4d2)
|
|
builder.InsertInstruction(iconst)
|
|
v2 := iconst.Return()
|
|
|
|
// Constant can be referenced from different groups because we inline it.
|
|
builder.SetCurrentBlock(builder.AllocateBasicBlock())
|
|
|
|
icmp := builder.AllocateInstruction()
|
|
icmp.AsIcmp(v1, v2, ssa.IntegerCmpCondEqual)
|
|
builder.InsertInstruction(icmp)
|
|
icmpVal := icmp.Return()
|
|
ctx.definitions[v1] = &backend.SSAValueDefinition{BlkParamVReg: intToVReg(1), BlockParamValue: v1}
|
|
ctx.definitions[v2] = &backend.SSAValueDefinition{Instr: iconst}
|
|
ctx.definitions[icmpVal] = &backend.SSAValueDefinition{Instr: icmp}
|
|
ctx.vRegMap[v1], ctx.vRegMap[v2], ctx.vRegMap[icmpVal] = intToVReg(1), intToVReg(2), intToVReg(3)
|
|
b := builder.AllocateInstruction()
|
|
if brz {
|
|
b.AsBrz(icmpVal, nil, builder.AllocateBasicBlock())
|
|
} else {
|
|
b.AsBrnz(icmpVal, nil, builder.AllocateBasicBlock())
|
|
}
|
|
builder.InsertInstruction(b)
|
|
return b, func(t *testing.T) {
|
|
require.True(t, icmp.Lowered())
|
|
}
|
|
}
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
setup func(*mockCompiler, ssa.Builder, *machine) (instr *ssa.Instruction, verify func(t *testing.T))
|
|
instructions []string
|
|
}{
|
|
{
|
|
name: "icmp in different group",
|
|
setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
|
|
m.StartLoweringFunction(10)
|
|
entry := builder.CurrentBlock()
|
|
v1, v2 := entry.AddParam(builder, ssa.TypeI64), entry.AddParam(builder, ssa.TypeI64)
|
|
|
|
icmp := builder.AllocateInstruction()
|
|
icmp.AsIcmp(v1, v2, ssa.IntegerCmpCondEqual)
|
|
builder.InsertInstruction(icmp)
|
|
icmpVal := icmp.Return()
|
|
ctx.definitions[icmpVal] = &backend.SSAValueDefinition{Instr: icmp}
|
|
ctx.vRegMap[v1], ctx.vRegMap[v2], ctx.vRegMap[icmpVal] = intToVReg(1), intToVReg(2), intToVReg(3)
|
|
|
|
brz := builder.AllocateInstruction()
|
|
brz.AsBrz(icmpVal, nil, builder.AllocateBasicBlock())
|
|
builder.InsertInstruction(brz)
|
|
|
|
// Indicate that currently compiling in the different group.
|
|
ctx.currentGID = 1000
|
|
return brz, func(t *testing.T) {
|
|
require.False(t, icmp.Lowered())
|
|
}
|
|
},
|
|
instructions: []string{"cbz w3?, (L1)"},
|
|
},
|
|
{
|
|
name: "brz / icmp in the same group / params",
|
|
setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
|
|
return cmpInSameGroupFromParams(true, ssa.IntegerCmpCondUnsignedGreaterThan, ssa.FloatCmpCondInvalid, ctx, builder, m)
|
|
},
|
|
instructions: []string{
|
|
"subs xzr, x1, x2",
|
|
"b.ls L1",
|
|
},
|
|
},
|
|
{
|
|
name: "brnz / icmp in the same group / params",
|
|
setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
|
|
return cmpInSameGroupFromParams(false, ssa.IntegerCmpCondEqual, ssa.FloatCmpCondInvalid, ctx, builder, m)
|
|
},
|
|
instructions: []string{
|
|
"subs xzr, x1, x2",
|
|
"b.eq L1",
|
|
},
|
|
},
|
|
{
|
|
name: "brz / fcmp in the same group / params",
|
|
setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
|
|
return cmpInSameGroupFromParams(true, ssa.IntegerCmpCondInvalid, ssa.FloatCmpCondEqual, ctx, builder, m)
|
|
},
|
|
instructions: []string{
|
|
"fcmp d1, d2",
|
|
"b.ne L1",
|
|
},
|
|
},
|
|
{
|
|
name: "brnz / fcmp in the same group / params",
|
|
setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
|
|
return cmpInSameGroupFromParams(false, ssa.IntegerCmpCondInvalid, ssa.FloatCmpCondGreaterThan, ctx, builder, m)
|
|
},
|
|
instructions: []string{
|
|
"fcmp d1, d2",
|
|
"b.gt L1",
|
|
},
|
|
},
|
|
{
|
|
name: "brz / icmp in the same group / params",
|
|
setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
|
|
return icmpInSameGroupFromParamAndImm12(true, ctx, builder, m)
|
|
},
|
|
instructions: []string{
|
|
"subs wzr, w1?, #0x4d2",
|
|
"b.ne L1",
|
|
},
|
|
},
|
|
{
|
|
name: "brz / icmp in the same group / params",
|
|
setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
|
|
return icmpInSameGroupFromParamAndImm12(false, ctx, builder, m)
|
|
},
|
|
instructions: []string{
|
|
"subs wzr, w1?, #0x4d2",
|
|
"b.eq L1",
|
|
},
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
ctx, b, m := newSetupWithMockContext()
|
|
instr, verify := tc.setup(ctx, b, m)
|
|
m.LowerConditionalBranch(instr)
|
|
verify(t)
|
|
require.Equal(t, strings.Join(tc.instructions, "\n"),
|
|
formatEmittedInstructionsInCurrentBlock(m))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMachine_LowerSingleBranch(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
setup func(*mockCompiler, ssa.Builder, *machine) (instr *ssa.Instruction)
|
|
instructions []string
|
|
}{
|
|
{
|
|
name: "jump-fallthrough",
|
|
setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction) {
|
|
jump := builder.AllocateInstruction()
|
|
jump.AsJump(nil, builder.AllocateBasicBlock())
|
|
builder.InsertInstruction(jump)
|
|
jump.AsFallthroughJump()
|
|
return jump
|
|
},
|
|
instructions: []string{}, // Fallthrough jump should be optimized out.
|
|
},
|
|
{
|
|
name: "b",
|
|
setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction) {
|
|
m.StartLoweringFunction(10)
|
|
jump := builder.AllocateInstruction()
|
|
jump.AsJump(nil, builder.AllocateBasicBlock())
|
|
builder.InsertInstruction(jump)
|
|
return jump
|
|
},
|
|
instructions: []string{"b L1"},
|
|
},
|
|
{
|
|
name: "ret",
|
|
setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction) {
|
|
m.StartLoweringFunction(10)
|
|
jump := builder.AllocateInstruction()
|
|
jump.AsJump(nil, builder.ReturnBlock())
|
|
builder.InsertInstruction(jump)
|
|
return jump
|
|
},
|
|
// Jump which targets the return block should be translated as "ret".
|
|
instructions: []string{"ret"},
|
|
},
|
|
} {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
ctx, b, m := newSetupWithMockContext()
|
|
instr := tc.setup(ctx, b, m)
|
|
m.LowerSingleBranch(instr)
|
|
require.Equal(t, strings.Join(tc.instructions, "\n"), formatEmittedInstructionsInCurrentBlock(m))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMachine_InsertMove(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
src, dst regalloc.VReg
|
|
typ ssa.Type
|
|
instruction string
|
|
}{
|
|
{
|
|
name: "int",
|
|
src: regalloc.VReg(1).SetRegType(regalloc.RegTypeInt),
|
|
dst: regalloc.VReg(2).SetRegType(regalloc.RegTypeInt),
|
|
instruction: "mov x1?, x2?",
|
|
typ: ssa.TypeI64,
|
|
},
|
|
{
|
|
name: "float",
|
|
src: regalloc.VReg(1).SetRegType(regalloc.RegTypeFloat),
|
|
dst: regalloc.VReg(2).SetRegType(regalloc.RegTypeFloat),
|
|
instruction: "mov v1?.8b, v2?.8b",
|
|
typ: ssa.TypeF64,
|
|
},
|
|
{
|
|
name: "vector",
|
|
src: regalloc.VReg(1).SetRegType(regalloc.RegTypeFloat),
|
|
dst: regalloc.VReg(2).SetRegType(regalloc.RegTypeFloat),
|
|
instruction: "mov v1?.16b, v2?.16b",
|
|
typ: ssa.TypeV128,
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
_, _, m := newSetupWithMockContext()
|
|
m.InsertMove(tc.src, tc.dst, tc.typ)
|
|
require.Equal(t, tc.instruction, formatEmittedInstructionsInCurrentBlock(m))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMachine_lowerIDiv(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
_64bit bool
|
|
signed bool
|
|
exp string
|
|
}{
|
|
{
|
|
name: "32bit unsigned", _64bit: false, signed: false,
|
|
exp: `
|
|
udiv w1?, w2?, w3?
|
|
cbnz w3?, L1
|
|
movz x1?, #0xa, lsl 0
|
|
str w1?, [x65535?]
|
|
mov x2?, sp
|
|
str x2?, [x65535?, #0x38]
|
|
adr x3?, #0x0
|
|
str x3?, [x65535?, #0x30]
|
|
exit_sequence x65535?
|
|
L1:
|
|
`,
|
|
},
|
|
{name: "32bit signed", _64bit: false, signed: true, exp: `
|
|
sdiv w1?, w2?, w3?
|
|
cbnz w3?, L1
|
|
movz x1?, #0xa, lsl 0
|
|
str w1?, [x65535?]
|
|
mov x2?, sp
|
|
str x2?, [x65535?, #0x38]
|
|
adr x3?, #0x0
|
|
str x3?, [x65535?, #0x30]
|
|
exit_sequence x65535?
|
|
L1:
|
|
adds wzr, w3?, #0x1
|
|
ccmp w2?, #0x1, #0x0, eq
|
|
b.vc L2
|
|
movz x4?, #0xb, lsl 0
|
|
str w4?, [x65535?]
|
|
mov x5?, sp
|
|
str x5?, [x65535?, #0x38]
|
|
adr x6?, #0x0
|
|
str x6?, [x65535?, #0x30]
|
|
exit_sequence x65535?
|
|
L2:
|
|
`},
|
|
{name: "64bit unsigned", _64bit: true, signed: false, exp: `
|
|
udiv x1?, x2?, x3?
|
|
cbnz x3?, L1
|
|
movz x1?, #0xa, lsl 0
|
|
str w1?, [x65535?]
|
|
mov x2?, sp
|
|
str x2?, [x65535?, #0x38]
|
|
adr x3?, #0x0
|
|
str x3?, [x65535?, #0x30]
|
|
exit_sequence x65535?
|
|
L1:
|
|
`},
|
|
{name: "64bit signed", _64bit: true, signed: true, exp: `
|
|
sdiv x1?, x2?, x3?
|
|
cbnz x3?, L1
|
|
movz x1?, #0xa, lsl 0
|
|
str w1?, [x65535?]
|
|
mov x2?, sp
|
|
str x2?, [x65535?, #0x38]
|
|
adr x3?, #0x0
|
|
str x3?, [x65535?, #0x30]
|
|
exit_sequence x65535?
|
|
L1:
|
|
adds xzr, x3?, #0x1
|
|
ccmp x2?, #0x1, #0x0, eq
|
|
b.vc L2
|
|
movz x4?, #0xb, lsl 0
|
|
str w4?, [x65535?]
|
|
mov x5?, sp
|
|
str x5?, [x65535?, #0x38]
|
|
adr x6?, #0x0
|
|
str x6?, [x65535?, #0x30]
|
|
exit_sequence x65535?
|
|
L2:
|
|
`},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
execCtx := regalloc.VReg(0xffff).SetRegType(regalloc.RegTypeInt)
|
|
rd, rn, rm := regalloc.VReg(1).SetRegType(regalloc.RegTypeInt),
|
|
regalloc.VReg(2).SetRegType(regalloc.RegTypeInt),
|
|
regalloc.VReg(3).SetRegType(regalloc.RegTypeInt)
|
|
_, _, m := newSetupWithMockContext()
|
|
m.lowerIDiv(execCtx, operandNR(rd), operandNR(rn), operandNR(rm), tc._64bit, tc.signed)
|
|
require.Equal(t, tc.exp, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMachine_exitWithCode(t *testing.T) {
|
|
_, _, m := newSetupWithMockContext()
|
|
m.lowerExitWithCode(x1VReg, wazevoapi.ExitCodeGrowStack)
|
|
m.FlushPendingInstructions()
|
|
m.encode(m.perBlockHead)
|
|
require.Equal(t, `
|
|
movz x1?, #0x1, lsl 0
|
|
str w1?, [x1]
|
|
mov x2?, sp
|
|
str x2?, [x1, #0x38]
|
|
adr x3?, #0x0
|
|
str x3?, [x1, #0x30]
|
|
exit_sequence x1
|
|
`, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
|
|
}
|
|
|
|
func TestMachine_lowerFpuToInt(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
nontrapping bool
|
|
expectedAsm string
|
|
}{
|
|
{
|
|
name: "trapping",
|
|
nontrapping: false,
|
|
expectedAsm: `
|
|
msr fpsr, xzr
|
|
fcvtzu w1, s2
|
|
mrs x1? fpsr
|
|
subs xzr, x1?, #0x1
|
|
b.ne L2
|
|
fcmp w2, w2
|
|
b.vc L1
|
|
movz x2?, #0xc, lsl 0
|
|
str w2?, [x15]
|
|
mov x3?, sp
|
|
str x3?, [x15, #0x38]
|
|
adr x4?, #0x0
|
|
str x4?, [x15, #0x30]
|
|
exit_sequence x15
|
|
L1:
|
|
movz x5?, #0xb, lsl 0
|
|
str w5?, [x15]
|
|
mov x6?, sp
|
|
str x6?, [x15, #0x38]
|
|
adr x7?, #0x0
|
|
str x7?, [x15, #0x30]
|
|
exit_sequence x15
|
|
L2:
|
|
`,
|
|
},
|
|
{
|
|
name: "nontrapping",
|
|
nontrapping: true,
|
|
expectedAsm: `
|
|
fcvtzu w1, s2
|
|
`,
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
_, _, m := newSetupWithMockContext()
|
|
m.lowerFpuToInt(operandNR(x1VReg), operandNR(x2VReg), x15VReg, false, false, false, tc.nontrapping)
|
|
require.Equal(t, tc.expectedAsm, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
|
|
|
|
m.FlushPendingInstructions()
|
|
m.encode(m.perBlockHead)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMachine_lowerVIMul(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
expectedAsm string
|
|
arrangement vecArrangement
|
|
expectedBytes string
|
|
}{
|
|
{
|
|
name: "2D",
|
|
arrangement: vecArrangement2D,
|
|
expectedAsm: `
|
|
rev64 v2?.4s, x15.4s
|
|
mul v2?.4s, v2?.4s, x2.4s
|
|
xtn v1?.2s, x2.2s
|
|
addp v2?.4s, v2?.4s, v2?.4s
|
|
xtn v3?.2s, x15.2s
|
|
shll x1.2s, v2?.2s
|
|
umlal x1.2s, v3?.2s, v1?.2s
|
|
`,
|
|
expectedBytes: "e009a04e009ca24e4028a10e00bca04ee029a10e0138a12e0180a02e",
|
|
},
|
|
{
|
|
name: "8B",
|
|
arrangement: vecArrangement8B,
|
|
expectedAsm: `
|
|
mul x1.8b, x2.8b, x15.8b
|
|
`,
|
|
expectedBytes: "419c2f0e",
|
|
},
|
|
{
|
|
name: "16B",
|
|
arrangement: vecArrangement16B,
|
|
expectedAsm: `
|
|
mul x1.16b, x2.16b, x15.16b
|
|
`,
|
|
expectedBytes: "419c2f4e",
|
|
},
|
|
{
|
|
name: "4H",
|
|
arrangement: vecArrangement4H,
|
|
expectedAsm: `
|
|
mul x1.4h, x2.4h, x15.4h
|
|
`,
|
|
expectedBytes: "419c6f0e",
|
|
},
|
|
{
|
|
name: "8H",
|
|
arrangement: vecArrangement8H,
|
|
expectedAsm: `
|
|
mul x1.8h, x2.8h, x15.8h
|
|
`,
|
|
expectedBytes: "419c6f4e",
|
|
},
|
|
{
|
|
name: "2S",
|
|
arrangement: vecArrangement2S,
|
|
expectedAsm: `
|
|
mul x1.2s, x2.2s, x15.2s
|
|
`,
|
|
expectedBytes: "419caf0e",
|
|
},
|
|
{
|
|
name: "4S",
|
|
arrangement: vecArrangement4S,
|
|
expectedAsm: `
|
|
mul x1.4s, x2.4s, x15.4s
|
|
`,
|
|
expectedBytes: "419caf4e",
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
_, _, m := newSetupWithMockContext()
|
|
m.lowerVIMul(operandNR(x1VReg), operandNR(x2VReg), operandNR(x15VReg), tc.arrangement)
|
|
require.Equal(t, tc.expectedAsm, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
|
|
|
|
m.FlushPendingInstructions()
|
|
m.encode(m.perBlockHead)
|
|
buf := m.compiler.Buf()
|
|
require.Equal(t, tc.expectedBytes, hex.EncodeToString(buf))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMachine_lowerVcheckTrue(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
op ssa.Opcode
|
|
expectedAsm string
|
|
arrangement vecArrangement
|
|
expectedBytes string
|
|
}{
|
|
{
|
|
name: "anyTrue",
|
|
op: ssa.OpcodeVanyTrue,
|
|
expectedAsm: `
|
|
umaxp v1?.16b, x1.16b, x1.16b
|
|
mov x15, v1?.d[0]
|
|
ccmp x15, #0x0, #0x0, al
|
|
cset x15, ne
|
|
`,
|
|
expectedBytes: "20a4216e0f3c084ee0e940faef079f9a",
|
|
},
|
|
{
|
|
name: "allTrue 2D",
|
|
op: ssa.OpcodeVallTrue,
|
|
arrangement: vecArrangement2D,
|
|
expectedAsm: `
|
|
cmeq v1?.2d, x1.2d, #0
|
|
addp v1?.2d, v1?.2d, v1?.2d
|
|
fcmp d1?, d1?
|
|
cset x15, eq
|
|
`,
|
|
expectedBytes: "2098e04e00bce04e0020601eef179f9a",
|
|
},
|
|
{
|
|
name: "allTrue 8B",
|
|
arrangement: vecArrangement8B,
|
|
op: ssa.OpcodeVallTrue,
|
|
expectedAsm: `
|
|
uminv h1?, x1.8b
|
|
mov x15, v1?.d[0]
|
|
ccmp x15, #0x0, #0x0, al
|
|
cset x15, ne
|
|
`,
|
|
expectedBytes: "20a8312e0f3c084ee0e940faef079f9a",
|
|
},
|
|
{
|
|
name: "allTrue 16B",
|
|
arrangement: vecArrangement16B,
|
|
op: ssa.OpcodeVallTrue,
|
|
expectedAsm: `
|
|
uminv h1?, x1.16b
|
|
mov x15, v1?.d[0]
|
|
ccmp x15, #0x0, #0x0, al
|
|
cset x15, ne
|
|
`,
|
|
expectedBytes: "20a8316e0f3c084ee0e940faef079f9a",
|
|
},
|
|
{
|
|
name: "allTrue 4H",
|
|
arrangement: vecArrangement4H,
|
|
op: ssa.OpcodeVallTrue,
|
|
expectedAsm: `
|
|
uminv s1?, x1.4h
|
|
mov x15, v1?.d[0]
|
|
ccmp x15, #0x0, #0x0, al
|
|
cset x15, ne
|
|
`,
|
|
expectedBytes: "20a8712e0f3c084ee0e940faef079f9a",
|
|
},
|
|
{
|
|
name: "allTrue 8H",
|
|
arrangement: vecArrangement8H,
|
|
op: ssa.OpcodeVallTrue,
|
|
expectedAsm: `
|
|
uminv s1?, x1.8h
|
|
mov x15, v1?.d[0]
|
|
ccmp x15, #0x0, #0x0, al
|
|
cset x15, ne
|
|
`,
|
|
expectedBytes: "20a8716e0f3c084ee0e940faef079f9a",
|
|
},
|
|
{
|
|
name: "allTrue 4S",
|
|
arrangement: vecArrangement4S,
|
|
op: ssa.OpcodeVallTrue,
|
|
expectedAsm: `
|
|
uminv d1?, x1.4s
|
|
mov x15, v1?.d[0]
|
|
ccmp x15, #0x0, #0x0, al
|
|
cset x15, ne
|
|
`,
|
|
expectedBytes: "20a8b16e0f3c084ee0e940faef079f9a",
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
_, _, m := newSetupWithMockContext()
|
|
m.lowerVcheckTrue(tc.op, operandNR(x1VReg), operandNR(x15VReg), tc.arrangement)
|
|
require.Equal(t, tc.expectedAsm, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
|
|
|
|
m.FlushPendingInstructions()
|
|
m.encode(m.perBlockHead)
|
|
buf := m.compiler.Buf()
|
|
require.Equal(t, tc.expectedBytes, hex.EncodeToString(buf))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMachine_lowerVhighBits(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
expectedAsm string
|
|
arrangement vecArrangement
|
|
expectedBytes string
|
|
}{
|
|
{
|
|
name: "16B",
|
|
arrangement: vecArrangement16B,
|
|
expectedAsm: `
|
|
sshr v3?.16b, x1.16b, #7
|
|
movz x1?, #0x201, lsl 0
|
|
movk x1?, #0x804, lsl 16
|
|
movk x1?, #0x2010, lsl 32
|
|
movk x1?, #0x8040, lsl 48
|
|
dup v2?.2d, x1?
|
|
and v3?.16b, v3?.16b, v2?.16b
|
|
ext v2?.16b, v3?.16b, v3?.16b, #8
|
|
zip1 v2?.16b, v3?.16b, v2?.16b
|
|
addv s2?, v2?.8h
|
|
umov w15, v2?.h[0]
|
|
`,
|
|
expectedBytes: "2004094f204080d28000a1f20002c4f20008f0f2000c084e001c204e0040006e0038004e00b8714e0f3c020e",
|
|
},
|
|
{
|
|
name: "8H",
|
|
arrangement: vecArrangement8H,
|
|
expectedAsm: `
|
|
sshr v3?.8h, x1.8h, #15
|
|
movz x1?, #0x1, lsl 0
|
|
movk x1?, #0x2, lsl 16
|
|
movk x1?, #0x4, lsl 32
|
|
movk x1?, #0x8, lsl 48
|
|
dup v2?.2d, x1?
|
|
lsl x1?, x1?, 0x4
|
|
ins v2?.d[1], x1?
|
|
and v2?.16b, v3?.16b, v2?.16b
|
|
addv s2?, v2?.8h
|
|
umov w15, v2?.h[0]
|
|
`,
|
|
expectedBytes: "2004114f200080d24000a0f28000c0f20001e0f2000c084e00ec7cd3001c184e001c204e00b8714e0f3c020e",
|
|
},
|
|
{
|
|
name: "4S",
|
|
arrangement: vecArrangement4S,
|
|
expectedAsm: `
|
|
sshr v3?.4s, x1.4s, #31
|
|
movz x1?, #0x1, lsl 0
|
|
movk x1?, #0x2, lsl 32
|
|
dup v2?.2d, x1?
|
|
lsl x1?, x1?, 0x2
|
|
ins v2?.d[1], x1?
|
|
and v2?.16b, v3?.16b, v2?.16b
|
|
addv d2?, v2?.4s
|
|
umov w15, v2?.s[0]
|
|
`,
|
|
expectedBytes: "2004214f200080d24000c0f2000c084e00f47ed3001c184e001c204e00b8b14e0f3c040e",
|
|
},
|
|
{
|
|
name: "2D",
|
|
arrangement: vecArrangement2D,
|
|
expectedAsm: `
|
|
mov x15, x1.d[0]
|
|
mov x1?, x1.d[1]
|
|
lsr x1?, x1?, 0x3f
|
|
lsr x15, x15, 0x3f
|
|
add w15, w15, w1?, lsl #1
|
|
`,
|
|
expectedBytes: "2f3c084e203c184e00fc7fd3effd7fd3ef05000b",
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
_, _, m := newSetupWithMockContext()
|
|
m.lowerVhighBits(operandNR(x1VReg), operandNR(x15VReg), tc.arrangement)
|
|
require.Equal(t, tc.expectedAsm, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
|
|
|
|
m.FlushPendingInstructions()
|
|
m.encode(m.perBlockHead)
|
|
buf := m.compiler.Buf()
|
|
require.Equal(t, tc.expectedBytes, hex.EncodeToString(buf))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMachine_lowerShuffle(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
lanes []uint64
|
|
expectedAsm string
|
|
expectedBytes string
|
|
}{
|
|
{
|
|
name: "lanes 0..15",
|
|
lanes: []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
|
|
expectedAsm: `
|
|
mov v29.16b, x2.16b
|
|
mov v30.16b, x15.16b
|
|
ldr q1?, #8; b 32; data.v128 0706050403020100 0f0e0d0c0b0a0908
|
|
tbl x1.16b, { v29.16b, v30.16b }, v1?.16b
|
|
`,
|
|
expectedBytes: "5d1ca24efe1daf4e4000009c05000014000102030405060708090a0b0c0d0e0fa123004e",
|
|
},
|
|
{
|
|
name: "lanes 0101...",
|
|
lanes: []uint64{0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1},
|
|
expectedAsm: `
|
|
mov v29.16b, x2.16b
|
|
mov v30.16b, x15.16b
|
|
ldr q1?, #8; b 32; data.v128 0100010001000100 0100010001000100
|
|
tbl x1.16b, { v29.16b, v30.16b }, v1?.16b
|
|
`,
|
|
expectedBytes: "5d1ca24efe1daf4e4000009c0500001400010001000100010001000100010001a123004e",
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
_, _, m := newSetupWithMockContext()
|
|
lanes := tc.lanes
|
|
|
|
// Encode the 16 bytes as 8 bytes in u1, and 8 bytes in u2.
|
|
lane1 := lanes[7]<<56 | lanes[6]<<48 | lanes[5]<<40 | lanes[4]<<32 | lanes[3]<<24 | lanes[2]<<16 | lanes[1]<<8 | lanes[0]
|
|
lane2 := lanes[15]<<56 | lanes[14]<<48 | lanes[13]<<40 | lanes[12]<<32 | lanes[11]<<24 | lanes[10]<<16 | lanes[9]<<8 | lanes[8]
|
|
|
|
m.lowerShuffle(operandNR(x1VReg), operandNR(x2VReg), operandNR(x15VReg), lane1, lane2)
|
|
require.Equal(t, tc.expectedAsm, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
|
|
|
|
m.FlushPendingInstructions()
|
|
m.encode(m.perBlockHead)
|
|
buf := m.compiler.Buf()
|
|
require.Equal(t, tc.expectedBytes, hex.EncodeToString(buf))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMachine_lowerVShift(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
expectedAsm string
|
|
op ssa.Opcode
|
|
arrangement vecArrangement
|
|
expectedBytes string
|
|
}{
|
|
{
|
|
name: "VIshl",
|
|
op: ssa.OpcodeVIshl,
|
|
arrangement: vecArrangement16B,
|
|
expectedAsm: `
|
|
and x1?, x15, #0x7
|
|
dup v2?.16b, x1?
|
|
sshl x1.16b, x2.16b, v2?.16b
|
|
`,
|
|
expectedBytes: "e0094092000c014e4144204e",
|
|
},
|
|
{
|
|
name: "VSshr",
|
|
op: ssa.OpcodeVSshr,
|
|
arrangement: vecArrangement16B,
|
|
expectedAsm: `
|
|
and x1?, x15, #0x7
|
|
sub x1?, xzr, x1?
|
|
dup v2?.16b, x1?
|
|
sshl x1.16b, x2.16b, v2?.16b
|
|
`,
|
|
expectedBytes: "e0094092e00300cb000c014e4144204e",
|
|
},
|
|
{
|
|
name: "VUshr",
|
|
op: ssa.OpcodeVUshr,
|
|
arrangement: vecArrangement16B,
|
|
expectedAsm: `
|
|
and x1?, x15, #0x7
|
|
sub x1?, xzr, x1?
|
|
dup v2?.16b, x1?
|
|
ushl x1.16b, x2.16b, v2?.16b
|
|
`,
|
|
expectedBytes: "e0094092e00300cb000c014e4144206e",
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
_, _, m := newSetupWithMockContext()
|
|
m.lowerVShift(tc.op, operandNR(x1VReg), operandNR(x2VReg), operandNR(x15VReg), tc.arrangement)
|
|
require.Equal(t, tc.expectedAsm, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
|
|
|
|
m.FlushPendingInstructions()
|
|
m.encode(m.perBlockHead)
|
|
buf := m.compiler.Buf()
|
|
require.Equal(t, tc.expectedBytes, hex.EncodeToString(buf))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMachine_lowerSelectVec(t *testing.T) {
|
|
_, _, m := newSetupWithMockContext()
|
|
c := operandNR(m.compiler.AllocateVReg(ssa.TypeI32))
|
|
rn := operandNR(m.compiler.AllocateVReg(ssa.TypeV128))
|
|
rm := operandNR(m.compiler.AllocateVReg(ssa.TypeV128))
|
|
rd := operandNR(m.compiler.AllocateVReg(ssa.TypeV128))
|
|
|
|
require.Equal(t, 1, int(c.reg().ID()))
|
|
require.Equal(t, 2, int(rn.reg().ID()))
|
|
require.Equal(t, 3, int(rm.reg().ID()))
|
|
require.Equal(t, 4, int(rd.reg().ID()))
|
|
|
|
m.lowerSelectVec(c, rn, rm, rd)
|
|
require.Equal(t, `
|
|
cbnz x1?, L1
|
|
mov v4?.16b, v3?.16b
|
|
b L2
|
|
L1:
|
|
mov v4?.16b, v2?.16b
|
|
L2:
|
|
`, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
|
|
}
|
|
|
|
func TestMachine_lowerFcopysign(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
_64bit bool
|
|
exp string
|
|
}{
|
|
{
|
|
_64bit: false,
|
|
exp: `
|
|
movz w1?, #0x8000, lsl 16
|
|
ins v2?.s[0], w1?
|
|
mov v6?.8b, v3?.8b
|
|
bit v6?.8b, v4?.8b, v2?.8b
|
|
mov v5?.8b, v6?.8b
|
|
`,
|
|
},
|
|
{
|
|
_64bit: true,
|
|
exp: `
|
|
movz x1?, #0x8000, lsl 48
|
|
ins v2?.d[0], x1?
|
|
mov v6?.8b, v3?.8b
|
|
bit v6?.8b, v4?.8b, v2?.8b
|
|
mov v5?.8b, v6?.8b
|
|
`,
|
|
},
|
|
} {
|
|
t.Run(fmt.Sprintf("64bit=%v", tc._64bit), func(t *testing.T) {
|
|
_, _, m := newSetupWithMockContext()
|
|
var typ, ftyp ssa.Type
|
|
if tc._64bit {
|
|
typ = ssa.TypeI64
|
|
ftyp = ssa.TypeF64
|
|
} else {
|
|
typ = ssa.TypeI32
|
|
ftyp = ssa.TypeF32
|
|
}
|
|
tmpI := operandNR(m.compiler.AllocateVReg(typ))
|
|
tmpF := operandNR(m.compiler.AllocateVReg(ftyp))
|
|
rn := operandNR(m.compiler.AllocateVReg(ftyp))
|
|
rm := operandNR(m.compiler.AllocateVReg(ftyp))
|
|
rd := operandNR(m.compiler.AllocateVReg(ftyp))
|
|
|
|
require.Equal(t, 1, int(tmpI.reg().ID()))
|
|
require.Equal(t, 2, int(tmpF.reg().ID()))
|
|
require.Equal(t, 3, int(rn.reg().ID()))
|
|
require.Equal(t, 4, int(rm.reg().ID()))
|
|
require.Equal(t, 5, int(rd.reg().ID()))
|
|
|
|
m.lowerFcopysignImpl(rd, rn, rm, tmpI, tmpF, tc._64bit)
|
|
require.Equal(t, tc.exp, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMachine_lowerRotl(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
_64bit bool
|
|
exp string
|
|
}{
|
|
{
|
|
_64bit: false,
|
|
exp: `
|
|
sub w1?, wzr, w3?
|
|
ror w4?, w2?, w1?
|
|
`,
|
|
},
|
|
{
|
|
_64bit: true,
|
|
exp: `
|
|
sub x1?, xzr, x3?
|
|
ror x4?, x2?, x1?
|
|
`,
|
|
},
|
|
} {
|
|
t.Run(fmt.Sprintf("64bit=%v", tc._64bit), func(t *testing.T) {
|
|
_, _, m := newSetupWithMockContext()
|
|
var typ ssa.Type
|
|
if tc._64bit {
|
|
typ = ssa.TypeI64
|
|
} else {
|
|
typ = ssa.TypeI32
|
|
}
|
|
tmpI := operandNR(m.compiler.AllocateVReg(typ))
|
|
rn := operandNR(m.compiler.AllocateVReg(typ))
|
|
rm := operandNR(m.compiler.AllocateVReg(typ))
|
|
rd := operandNR(m.compiler.AllocateVReg(typ))
|
|
|
|
require.Equal(t, 1, int(tmpI.reg().ID()))
|
|
require.Equal(t, 2, int(rn.reg().ID()))
|
|
require.Equal(t, 3, int(rm.reg().ID()))
|
|
require.Equal(t, 4, int(rd.reg().ID()))
|
|
|
|
m.lowerRotlImpl(rd, rn, rm, tmpI, tc._64bit)
|
|
require.Equal(t, tc.exp, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
|
|
})
|
|
}
|
|
}
|