wazevo(amd64): encode unaryRmR, not, neg, mulHi, shiftR (#1916)

Signed-off-by: Edoardo Vacchi <evacchi@users.noreply.github.com>
2024-01-15 02:13:31 +01:00
parent 201154c3c8
commit 0dbc86fe06
3 changed files with 646 additions and 33 deletions
--- a/internal/engine/wazevo/backend/isa/amd64/instr.go
+++ b/internal/engine/wazevo/backend/isa/amd64/instr.go
@@ -68,15 +68,45 @@ func (i *instruction) String() string {
 	case xmmUnaryRmR:
 		return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(i.b1), i.op2.format(i.b1))
 	case unaryRmR:
-		panic("TODO")
+		var suffix string
+		if i.b1 {
+			suffix = "q"
+		} else {
+			suffix = "l"
+		}
+		return fmt.Sprintf("%s%s %s, %s", unaryRmROpcode(i.u1), suffix, i.op1.format(i.b1), i.op2.format(i.b1))
 	case not:
-		panic("TODO")
+		var op string
+		if i.b1 {
+			op = "notq"
+		} else {
+			op = "notl"
+		}
+		return fmt.Sprintf("%s %s", op, i.op1.format(i.b1))
 	case neg:
-		panic("TODO")
+		var op string
+		if i.b1 {
+			op = "negq"
+		} else {
+			op = "negl"
+		}
+		return fmt.Sprintf("%s %s", op, i.op1.format(i.b1))
 	case div:
 		panic("TODO")
 	case mulHi:
-		panic("TODO")
+		signed, _64 := i.u1 != 0, i.b1
+		var op string
+		switch {
+		case signed && _64:
+			op = "imulq"
+		case !signed && _64:
+			op = "mulq"
+		case signed && !_64:
+			op = "imull"
+		case !signed && !_64:
+			op = "mull"
+		}
+		return fmt.Sprintf("%s %s", op, i.op1.format(i.b1))
 	case checkedDivOrRemSeq:
 		panic("TODO")
 	case signExtendData:
@@ -103,7 +133,13 @@ func (i *instruction) String() string {
 		}
 		return fmt.Sprintf("mov.%s %s, %s", suffix, i.op1.format(true), i.op2.format(true))
 	case shiftR:
-		panic("TODO")
+		var suffix string
+		if i.b1 {
+			suffix = "q"
+		} else {
+			suffix = "l"
+		}
+		return fmt.Sprintf("%s%s %s, %s", shiftROp(i.u1), suffix, i.op1.format(false), i.op2.format(i.b1))
 	case xmmRmiReg:
 		panic("TODO")
 	case cmpRmiR:
@@ -693,6 +729,63 @@ func (i *instruction) asMovRR(rm, rd regalloc.VReg, _64 bool) *instruction {
 	return i
 }

+func (i *instruction) asNot(rm operand, _64 bool) *instruction {
+	if rm.kind != operandKindReg && rm.kind != operandKindMem {
+		panic("BUG")
+	}
+	i.kind = not
+	i.op1 = rm
+	i.b1 = _64
+	return i
+}
+
+func (i *instruction) asNeg(rm operand, _64 bool) *instruction {
+	if rm.kind != operandKindReg && rm.kind != operandKindMem {
+		panic("BUG")
+	}
+	i.kind = neg
+	i.op1 = rm
+	i.b1 = _64
+	return i
+}
+
+func (i *instruction) asMulHi(rm operand, signed, _64 bool) *instruction {
+	if rm.kind != operandKindReg && (rm.kind != operandKindMem) {
+		panic("BUG")
+	}
+	i.kind = mulHi
+	i.op1 = rm
+	i.b1 = _64
+	if signed {
+		i.u1 = 1
+	}
+	return i
+}
+
+func (i *instruction) asUnaryRmR(op unaryRmROpcode, rm operand, rd regalloc.VReg, _64 bool) *instruction {
+	if rm.kind != operandKindReg && rm.kind != operandKindMem {
+		panic("BUG")
+	}
+	i.kind = unaryRmR
+	i.op1 = rm
+	i.op2 = newOperandReg(rd)
+	i.u1 = uint64(op)
+	i.b1 = _64
+	return i
+}
+
+func (i *instruction) asShiftR(op shiftROp, amount operand, rd regalloc.VReg, _64 bool) *instruction {
+	if amount.kind != operandKindReg && amount.kind != operandKindImm32 {
+		panic("BUG")
+	}
+	i.kind = shiftR
+	i.op1 = amount
+	i.op2 = newOperandReg(rd)
+	i.u1 = uint64(op)
+	i.b1 = _64
+	return i
+}
+
 func (i *instruction) asXmmUnaryRmR(op sseOpcode, rm operand, rd regalloc.VReg, _64 bool) *instruction {
 	if rm.kind != operandKindReg && rm.kind != operandKindMem {
 		panic("BUG")
@@ -731,6 +824,60 @@ func (i *instruction) asPush64(op operand) *instruction {
 	return i
 }

+type unaryRmROpcode byte
+
+const (
+	unaryRmROpcodeBsr unaryRmROpcode = iota
+	unaryRmROpcodeBsf
+	unaryRmROpcodeLzcnt
+	unaryRmROpcodeTzcnt
+	unaryRmROpcodePopcnt
+)
+
+func (u unaryRmROpcode) String() string {
+	switch u {
+	case unaryRmROpcodeBsr:
+		return "bsr"
+	case unaryRmROpcodeBsf:
+		return "bsf"
+	case unaryRmROpcodeLzcnt:
+		return "lzcnt"
+	case unaryRmROpcodeTzcnt:
+		return "tzcnt"
+	case unaryRmROpcodePopcnt:
+		return "popcnt"
+	default:
+		panic("BUG")
+	}
+}
+
+type shiftROp byte
+
+const (
+	shiftROpRotateLeft           = 0
+	shiftROpRotateRight          = 1
+	shiftROpShiftLeft            = 4
+	shiftROpShiftRightLogical    = 5
+	shiftROpShiftRightArithmetic = 7
+)
+
+func (s shiftROp) String() string {
+	switch s {
+	case shiftROpRotateLeft:
+		return "rol"
+	case shiftROpRotateRight:
+		return "ror"
+	case shiftROpShiftLeft:
+		return "shl"
+	case shiftROpShiftRightLogical:
+		return "shr"
+	case shiftROpShiftRightArithmetic:
+		return "sar"
+	default:
+		panic("BUG")
+	}
+}
+
 type sseOpcode byte

 const (
--- a/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go
+++ b/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go
@@ -452,15 +452,102 @@ func (i *instruction) encode(c backend.Compiler) (needsLabelResolution bool) {
 		}

 	case unaryRmR:
-		panic("TODO")
+		var prefix legacyPrefixes
+		var opcode uint32
+		var opcodeNum uint32
+		op := unaryRmROpcode(i.u1)
+		// We assume size is either 32 or 64.
+		switch op {
+		case unaryRmROpcodeBsr:
+			prefix, opcode, opcodeNum = legacyPrefixesNone, 0x0fbd, 2
+		case unaryRmROpcodeBsf:
+			prefix, opcode, opcodeNum = legacyPrefixesNone, 0x0fbc, 2
+		case unaryRmROpcodeLzcnt:
+			prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0fbd, 2
+		case unaryRmROpcodeTzcnt:
+			prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0fbc, 2
+		case unaryRmROpcodePopcnt:
+			prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0fb8, 2
+		default:
+			panic(fmt.Sprintf("Unsupported unaryRmROpcode: %s", op))
+		}
+
+		dst := regEncodings[i.op2.r.RealReg()]
+
+		rex := rexInfo(0)
+		if i.b1 { // 64 bit.
+			rex = rexInfo(0).setW()
+		} else {
+			rex = rexInfo(0).clearW()
+		}
+		op1 := i.op1
+		if op1.kind == operandKindReg {
+			src := regEncodings[op1.r.RealReg()]
+			encodeRegReg(c, prefix, opcode, opcodeNum, dst, src, rex)
+		} else if i.op1.kind == operandKindMem {
+			m := i.op1.amode
+			encodeRegMem(c, prefix, opcode, opcodeNum, dst, m, rex)
+		} else {
+			panic("BUG: invalid operand kind")
+		}
+
 	case not:
-		panic("TODO")
+		var prefix legacyPrefixes
+		src := regEncodings[i.op1.r.RealReg()]
+		rex := rexInfo(0)
+		if i.b1 { // 64 bit.
+			rex = rexInfo(0).setW()
+		} else {
+			rex = rexInfo(0).clearW()
+		}
+		subopcode := uint8(2)
+		encodeEncEnc(c, prefix, 0xf7, 1, subopcode, uint8(src), rex)
+
 	case neg:
-		panic("TODO")
+		var prefix legacyPrefixes
+		src := regEncodings[i.op1.r.RealReg()]
+		rex := rexInfo(0)
+		if i.b1 { // 64 bit.
+			rex = rexInfo(0).setW()
+		} else {
+			rex = rexInfo(0).clearW()
+		}
+		subopcode := uint8(3)
+		encodeEncEnc(c, prefix, 0xf7, 1, subopcode, uint8(src), rex)
+
 	case div:
 		panic("TODO")
 	case mulHi:
-		panic("TODO")
+		var prefix legacyPrefixes
+		rex := rexInfo(0)
+		if i.b1 { // 64 bit.
+			rex = rexInfo(0).setW()
+		} else {
+			rex = rexInfo(0).clearW()
+		}
+
+		signed := i.u1 != 0
+		var subopcode uint8
+		if signed {
+			subopcode = 5
+		} else {
+			subopcode = 4
+		}
+
+		// src1 is implicitly rax,
+		// dst_lo is implicitly rax,
+		// dst_hi is implicitly rdx.
+		src2 := i.op1
+		if src2.kind == operandKindReg {
+			src := regEncodings[src2.r.RealReg()]
+			encodeEncEnc(c, prefix, 0xf7, 1, subopcode, uint8(src), rex)
+		} else if src2.kind == operandKindMem {
+			m := src2.amode
+			encodeEncMem(c, prefix, 0xf7, 1, subopcode, m, rex)
+		} else {
+			panic("BUG: invalid operand kind")
+		}
+
 	case checkedDivOrRemSeq:
 		panic("TODO")
 	case signExtendData:
@@ -558,7 +645,32 @@ func (i *instruction) encode(c backend.Compiler) (needsLabelResolution bool) {
 		}

 	case shiftR:
-		panic("TODO")
+		src := regEncodings[i.op2.r.RealReg()]
+		amount := i.op1
+
+		var opcode uint32
+		var prefix legacyPrefixes
+		rex := rexInfo(0)
+		if i.b1 { // 64 bit.
+			rex = rexInfo(0).setW()
+		} else {
+			rex = rexInfo(0).clearW()
+		}
+
+		switch amount.kind {
+		case operandKindReg:
+			if amount.r != rcxVReg {
+				panic("BUG: invalid reg operand: must be rcx")
+			}
+			opcode, prefix = 0xd3, legacyPrefixesNone
+			encodeEncEnc(c, prefix, opcode, 1, uint8(i.u1), uint8(src), rex)
+		case operandKindImm32:
+			opcode, prefix = 0xc1, legacyPrefixesNone
+			encodeEncEnc(c, prefix, opcode, 1, uint8(i.u1), uint8(src), rex)
+			c.EmitByte(byte(amount.imm32))
+		default:
+			panic("BUG: invalid operand kind")
+		}
 	case xmmRmiReg:
 		panic("TODO")
 	case cmpRmiR:
@@ -741,6 +853,25 @@ func (i *instruction) encode(c backend.Compiler) (needsLabelResolution bool) {
 	return
 }

+func encodeEncEnc(
+	c backend.Compiler,
+	legPrefixes legacyPrefixes,
+	opcodes uint32,
+	opcodeNum uint32,
+	r uint8,
+	rm uint8,
+	rex rexInfo,
+) {
+	legPrefixes.encode(c)
+	rex.encode(c, r>>3, rm>>3)
+
+	for opcodeNum > 0 {
+		opcodeNum--
+		c.EmitByte(byte((opcodes >> (opcodeNum << 3)) & 0xff))
+	}
+	c.EmitByte(encodeModRM(3, r&0x7, rm&0x7))
+}
+
 func encodeRegReg(
 	c backend.Compiler,
 	legPrefixes legacyPrefixes,
@@ -750,14 +881,7 @@ func encodeRegReg(
 	rm regEnc,
 	rex rexInfo,
 ) {
-	legPrefixes.encode(c)
-	rex.encode(c, r, rm)
-
-	for opcodeNum > 0 {
-		opcodeNum--
-		c.EmitByte(byte((opcodes >> (opcodeNum << 3)) & 0xff))
-	}
-	c.EmitByte(encodeModRM(3, r.encoding(), rm.encoding()))
+	encodeEncEnc(c, legPrefixes, opcodes, opcodeNum, uint8(r), uint8(rm), rex)
 }

 func encodeModRM(mod byte, reg byte, rm byte) byte {
@@ -770,6 +894,12 @@ func encodeSIB(shift byte, encIndex byte, encBase byte) byte {

 func encodeRegMem(
 	c backend.Compiler, legPrefixes legacyPrefixes, opcodes uint32, opcodeNum uint32, r regEnc, m amode, rex rexInfo,
+) {
+	encodeEncMem(c, legPrefixes, opcodes, opcodeNum, uint8(r), m, rex)
+}
+
+func encodeEncMem(
+	c backend.Compiler, legPrefixes legacyPrefixes, opcodes uint32, opcodeNum uint32, r uint8, m amode, rex rexInfo,
 ) {
 	legPrefixes.encode(c)

@@ -786,7 +916,7 @@ func encodeRegMem(
 		base := m.base.RealReg()
 		baseEnc := regEncodings[base]

-		rex.encode(c, r, baseEnc)
+		rex.encode(c, regRexBit(r), baseEnc.rexBit())

 		for opcodeNum > 0 {
 			opcodeNum--
@@ -801,18 +931,18 @@ func encodeRegMem(
 		rspOrR12 := base == rsp || base == r12

 		if immZero && !baseRbp && !baseR13 { // rbp or r13 can't be used as base for without displacement encoding.
-			c.EmitByte(encodeModRM(modNoDisplacement, r.encoding(), baseEnc.encoding()))
+			c.EmitByte(encodeModRM(modNoDisplacement, regEncoding(r), baseEnc.encoding()))
 			if rspOrR12 {
 				c.EmitByte(sibByte)
 			}
 		} else if short { // Note: this includes the case where m.imm32 == 0 && base == rbp || base == r13.
-			c.EmitByte(encodeModRM(modShortDisplacement, r.encoding(), baseEnc.encoding()))
+			c.EmitByte(encodeModRM(modShortDisplacement, regEncoding(r), baseEnc.encoding()))
 			if rspOrR12 {
 				c.EmitByte(sibByte)
 			}
 			c.EmitByte(byte(m.imm32))
 		} else {
-			c.EmitByte(encodeModRM(modLongDisplacement, r.encoding(), baseEnc.encoding()))
+			c.EmitByte(encodeModRM(modLongDisplacement, regEncoding(r), baseEnc.encoding()))
 			if rspOrR12 {
 				c.EmitByte(sibByte)
 			}
@@ -829,7 +959,7 @@ func encodeRegMem(
 			panic("BUG: rsp can't be used as index of addressing mode")
 		}

-		rex.encodeForIndex(c, r, indexEnc, baseEnc)
+		rex.encodeForIndex(c, regEnc(r), indexEnc, baseEnc)

 		for opcodeNum > 0 {
 			opcodeNum--
@@ -838,14 +968,14 @@ func encodeRegMem(

 		immZero, baseRbp, baseR13 := m.imm32 == 0, base == rbp, base == r13
 		if immZero && !baseRbp && !baseR13 { // rbp or r13 can't be used as base for without displacement encoding. (curious why? because it's interpreted as RIP relative addressing).
-			c.EmitByte(encodeModRM(modNoDisplacement, r.encoding(), useSBI))
+			c.EmitByte(encodeModRM(modNoDisplacement, regEncoding(r), useSBI))
 			c.EmitByte(encodeSIB(m.shift, indexEnc.encoding(), baseEnc.encoding()))
 		} else if lower8willSignExtendTo32(m.imm32) {
-			c.EmitByte(encodeModRM(modShortDisplacement, r.encoding(), useSBI))
+			c.EmitByte(encodeModRM(modShortDisplacement, regEncoding(r), useSBI))
 			c.EmitByte(encodeSIB(m.shift, indexEnc.encoding(), baseEnc.encoding()))
 			c.EmitByte(byte(m.imm32))
 		} else {
-			c.EmitByte(encodeModRM(modLongDisplacement, r.encoding(), useSBI))
+			c.EmitByte(encodeModRM(modLongDisplacement, regEncoding(r), useSBI))
 			c.EmitByte(encodeSIB(m.shift, indexEnc.encoding(), baseEnc.encoding()))
 			c.Emit4Bytes(m.imm32)
 		}
@@ -855,7 +985,7 @@ func encodeRegMem(
 			panic("BUG: label must be resolved for amodeRipRelative at this point")
 		}

-		rex.encode(c, r, 0)
+		rex.encode(c, regRexBit(r), 0)
 		for opcodeNum > 0 {
 			opcodeNum--
 			c.EmitByte(byte((opcodes >> (opcodeNum << 3)) & 0xff))
@@ -863,7 +993,7 @@ func encodeRegMem(

 		// Indicate "LEAQ [RIP + 32bit displacement].
 		// https://wiki.osdev.org/X86-64_Instruction_Encoding#32.2F64-bit_addressing
-		c.EmitByte(encodeModRM(0b00, r.encoding(), 0b101))
+		c.EmitByte(encodeModRM(0b00, regEncoding(r), 0b101))
 		c.Emit4Bytes(m.imm32)
 	}
 }
@@ -895,13 +1025,11 @@ func (ri rexInfo) notAlways() rexInfo { //nolint
 	return ri & 0x01
 }

-func (ri rexInfo) encode(c backend.Compiler, encR regEnc, encRM regEnc) {
+func (ri rexInfo) encode(c backend.Compiler, r uint8, b uint8) {
 	var w byte = 0
 	if ri&0x01 != 0 {
 		w = 0x01
 	}
-	r := encR.rexBit()
-	b := encRM.rexBit()
 	rex := rexEncodingDefault | w<<3 | r<<2 | b
 	if rex != rexEncodingDefault || ri&0x02 == 1 {
 		c.EmitByte(rex)
@@ -925,11 +1053,19 @@ func (ri rexInfo) encodeForIndex(c backend.Compiler, encR regEnc, encIndex regEn
 type regEnc byte

 func (r regEnc) rexBit() byte {
-	return byte(r) >> 3
+	return regRexBit(byte(r))
 }

 func (r regEnc) encoding() byte {
-	return byte(r) & 0x07
+	return regEncoding(byte(r))
+}
+
+func regRexBit(r byte) byte {
+	return r >> 3
+}
+
+func regEncoding(r byte) byte {
+	return r & 0x07
 }

 var regEncodings = [...]regEnc{
--- a/internal/engine/wazevo/backend/isa/amd64/instr_encoding_test.go
+++ b/internal/engine/wazevo/backend/isa/amd64/instr_encoding_test.go
@@ -100,6 +100,196 @@ func TestInstruction_format_encode(t *testing.T) {
 			want:       "4d89dc",
 			wantFormat: "movq %r11, %r12",
 		},
+		{
+			setup:      func(i *instruction) { i.asNot(newOperandReg(raxVReg), false) },
+			want:       "f7d0",
+			wantFormat: "notl %eax",
+		},
+		{
+			setup:      func(i *instruction) { i.asNot(newOperandReg(raxVReg), true) },
+			want:       "48f7d0",
+			wantFormat: "notq %rax",
+		},
+		{
+			setup:      func(i *instruction) { i.asNeg(newOperandReg(raxVReg), false) },
+			want:       "f7d8",
+			wantFormat: "negl %eax",
+		},
+		{
+			setup:      func(i *instruction) { i.asNeg(newOperandReg(raxVReg), true) },
+			want:       "48f7d8",
+			wantFormat: "negq %rax",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandReg(rsiVReg), true, false) },
+			want:       "f7ee",
+			wantFormat: "imull %esi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandReg(r14VReg), false, false) },
+			want:       "41f7e6",
+			wantFormat: "mull %r14d",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandReg(r15VReg), true, true) },
+			want:       "49f7ef",
+			wantFormat: "imulq %r15",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandReg(rdiVReg), false, true) },
+			want:       "48f7e7",
+			wantFormat: "mulq %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandMem(newAmodeImmReg(123, raxVReg)), true, false) },
+			want:       "f7687b",
+			wantFormat: "imull 123(%rax)",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandMem(newAmodeImmReg(123, raxVReg)), false, false) },
+			want:       "f7607b",
+			wantFormat: "mull 123(%rax)",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandMem(newAmodeImmReg(123, raxVReg)), true, true) },
+			want:       "48f7687b",
+			wantFormat: "imulq 123(%rax)",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandMem(newAmodeImmReg(123, raxVReg)), false, true) },
+			want:       "48f7607b",
+			wantFormat: "mulq 123(%rax)",
+		},
+		// bsr
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeBsr, newOperandReg(raxVReg), rdiVReg, false) },
+			want:       "0fbdf8",
+			wantFormat: "bsrl %eax, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeBsr, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, false)
+			},
+			want:       "0fbd787b",
+			wantFormat: "bsrl 123(%rax), %edi",
+		},
+		// bsf
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeBsf, newOperandReg(raxVReg), rdiVReg, false) },
+			want:       "0fbcf8",
+			wantFormat: "bsfl %eax, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeBsf, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, false)
+			},
+			want:       "0fbc787b",
+			wantFormat: "bsfl 123(%rax), %edi",
+		},
+		// tzcnt
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeTzcnt, newOperandReg(raxVReg), rdiVReg, false) },
+			want:       "f30fbcf8",
+			wantFormat: "tzcntl %eax, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeTzcnt, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, false)
+			},
+			want:       "f30fbc787b",
+			wantFormat: "tzcntl 123(%rax), %edi",
+		},
+		// lzcnt
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeLzcnt, newOperandReg(raxVReg), rdiVReg, false) },
+			want:       "f30fbdf8",
+			wantFormat: "lzcntl %eax, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeLzcnt, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, false)
+			},
+			want:       "f30fbd787b",
+			wantFormat: "lzcntl 123(%rax), %edi",
+		},
+		// popcnt
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodePopcnt, newOperandReg(raxVReg), rdiVReg, false) },
+			want:       "f30fb8f8",
+			wantFormat: "popcntl %eax, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodePopcnt, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, false)
+			},
+			want:       "f30fb8787b",
+			wantFormat: "popcntl 123(%rax), %edi",
+		},
+		// bsr
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeBsr, newOperandReg(raxVReg), rdiVReg, true) },
+			want:       "480fbdf8",
+			wantFormat: "bsrq %rax, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeBsr, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, true)
+			},
+			want:       "480fbd787b",
+			wantFormat: "bsrq 123(%rax), %rdi",
+		},
+		// bsf
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeBsf, newOperandReg(raxVReg), rdiVReg, true) },
+			want:       "480fbcf8",
+			wantFormat: "bsfq %rax, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeBsf, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, true)
+			},
+			want:       "480fbc787b",
+			wantFormat: "bsfq 123(%rax), %rdi",
+		},
+		// tzcnt
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeTzcnt, newOperandReg(raxVReg), rdiVReg, true) },
+			want:       "f3480fbcf8",
+			wantFormat: "tzcntq %rax, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeTzcnt, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, true)
+			},
+			want:       "f3480fbc787b",
+			wantFormat: "tzcntq 123(%rax), %rdi",
+		},
+		// lzcnt
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeLzcnt, newOperandReg(raxVReg), rdiVReg, true) },
+			want:       "f3480fbdf8",
+			wantFormat: "lzcntq %rax, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeLzcnt, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, true)
+			},
+			want:       "f3480fbd787b",
+			wantFormat: "lzcntq 123(%rax), %rdi",
+		},
+		// popcnt
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodePopcnt, newOperandReg(raxVReg), rdiVReg, true) },
+			want:       "f3480fb8f8",
+			wantFormat: "popcntq %rax, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodePopcnt, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, true)
+			},
+			want:       "f3480fb8787b",
+			wantFormat: "popcntq 123(%rax), %rdi",
+		},
 		// addss
 		{
 			setup:      func(i *instruction) { i.asXmmRmR(sseOpcodeAddss, newOperandReg(xmm1VReg), xmm0VReg, true) },
@@ -1775,6 +1965,146 @@ func TestInstruction_format_encode(t *testing.T) {
 			want:       "66440f11797b",
 			wantFormat: "movupd %xmm15, 123(%rcx)",
 		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateLeft, newOperandReg(rcxVReg), rdiVReg, false)
+			},
+			want:       "d3c7",
+			wantFormat: "roll %ecx, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateLeft, newOperandImm32(128), rdiVReg, false)
+			},
+			want:       "c1c780",
+			wantFormat: "roll $128, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateLeft, newOperandReg(rcxVReg), rdiVReg, true)
+			},
+			want:       "48d3c7",
+			wantFormat: "rolq %ecx, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateLeft, newOperandImm32(128), rdiVReg, true)
+			},
+			want:       "48c1c780",
+			wantFormat: "rolq $128, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateRight, newOperandReg(rcxVReg), rdiVReg, false)
+			},
+			want:       "d3cf",
+			wantFormat: "rorl %ecx, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateRight, newOperandImm32(128), rdiVReg, false)
+			},
+			want:       "c1cf80",
+			wantFormat: "rorl $128, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateRight, newOperandReg(rcxVReg), rdiVReg, true)
+			},
+			want:       "48d3cf",
+			wantFormat: "rorq %ecx, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateRight, newOperandImm32(128), rdiVReg, true)
+			},
+			want:       "48c1cf80",
+			wantFormat: "rorq $128, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftLeft, newOperandReg(rcxVReg), rdiVReg, false)
+			},
+			want:       "d3e7",
+			wantFormat: "shll %ecx, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftLeft, newOperandImm32(128), rdiVReg, false)
+			},
+			want:       "c1e780",
+			wantFormat: "shll $128, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftLeft, newOperandReg(rcxVReg), rdiVReg, true)
+			},
+			want:       "48d3e7",
+			wantFormat: "shlq %ecx, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftLeft, newOperandImm32(128), rdiVReg, true)
+			},
+			want:       "48c1e780",
+			wantFormat: "shlq $128, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightLogical, newOperandReg(rcxVReg), rdiVReg, false)
+			},
+			want:       "d3ef",
+			wantFormat: "shrl %ecx, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightLogical, newOperandImm32(128), rdiVReg, false)
+			},
+			want:       "c1ef80",
+			wantFormat: "shrl $128, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightLogical, newOperandReg(rcxVReg), rdiVReg, true)
+			},
+			want:       "48d3ef",
+			wantFormat: "shrq %ecx, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightLogical, newOperandImm32(128), rdiVReg, true)
+			},
+			want:       "48c1ef80",
+			wantFormat: "shrq $128, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightArithmetic, newOperandReg(rcxVReg), rdiVReg, false)
+			},
+			want:       "d3ff",
+			wantFormat: "sarl %ecx, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightArithmetic, newOperandImm32(128), rdiVReg, false)
+			},
+			want:       "c1ff80",
+			wantFormat: "sarl $128, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightArithmetic, newOperandReg(rcxVReg), rdiVReg, true)
+			},
+			want:       "48d3ff",
+			wantFormat: "sarq %ecx, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightArithmetic, newOperandImm32(128), rdiVReg, true)
+			},
+			want:       "48c1ff80",
+			wantFormat: "sarq $128, %rdi",
+		},
 	} {
 		tc := tc
 		t.Run(tc.wantFormat, func(t *testing.T) {