x86: allow decoding of AVX instructions

x86/x86asm/decode_test.go

@@ -6,14 +6,14 @@ package x86asm
 
 import (
 	"encoding/hex"
-	"io/ioutil"
+	"os"
 	"strconv"
 	"strings"
 	"testing"
 )
 
 func TestDecode(t *testing.T) {
-	data, err := ioutil.ReadFile("testdata/decode.txt")
+	data, err := os.ReadFile("testdata/decode.txt")
 	if err != nil {
 		t.Fatal(err)
 	}

x86/x86asm/gnu.go

@@ -534,8 +534,6 @@ func gnuArg(inst *Inst, pc uint64, symname SymLookup, x Arg, usedPrefixes *bool)
 			if x == DX {
 				return "(%dx)"
 			}
-		case VMOVDQA, VMOVDQU, VMOVNTDQA, VMOVNTDQ:
-			return strings.Replace(gccRegName[x], "xmm", "ymm", -1)
 		}
 		return gccRegName[x]
 	case Mem:
@@ -787,6 +785,22 @@ var gccRegName = [...]string{
 	X13:  "%xmm13",
 	X14:  "%xmm14",
 	X15:  "%xmm15",
+	Y0:   "%ymm0",
+	Y1:   "%ymm1",
+	Y2:   "%ymm2",
+	Y3:   "%ymm3",
+	Y4:   "%ymm4",
+	Y5:   "%ymm5",
+	Y6:   "%ymm6",
+	Y7:   "%ymm7",
+	Y8:   "%ymm8",
+	Y9:   "%ymm9",
+	Y10:  "%ymm10",
+	Y11:  "%ymm11",
+	Y12:  "%ymm12",
+	Y13:  "%ymm13",
+	Y14:  "%ymm14",
+	Y15:  "%ymm15",
 	CS:   "%cs",
 	SS:   "%ss",
 	DS:   "%ds",

x86/x86asm/inst.go

@@ -72,13 +72,22 @@ const (
 
 	// The REX prefixes must be in the range [PrefixREX, PrefixREX+0x10).
 	// the other bits are set or not according to the intended use.
-	PrefixREX       Prefix = 0x40 // REX 64-bit extension prefix
-	PrefixREXW      Prefix = 0x08 // extension bit W (64-bit instruction width)
-	PrefixREXR      Prefix = 0x04 // extension bit R (r field in modrm)
-	PrefixREXX      Prefix = 0x02 // extension bit X (index field in sib)
-	PrefixREXB      Prefix = 0x01 // extension bit B (r/m field in modrm or base field in sib)
-	PrefixVEX2Bytes Prefix = 0xC5 // Short form of vex prefix
-	PrefixVEX3Bytes Prefix = 0xC4 // Long form of vex prefix
+	PrefixREX  Prefix = 0x40 // REX 64-bit extension prefix
+	PrefixREXW Prefix = 0x08 // extension bit W (64-bit instruction width)
+	PrefixREXR Prefix = 0x04 // extension bit R (r field in modrm)
+	PrefixREXX Prefix = 0x02 // extension bit X (index field in sib)
+	PrefixREXB Prefix = 0x01 // extension bit B (r/m field in modrm or base field in sib)
+
+	PrefixVEX2    Prefix = 0xC5 // Short form of VEX prefix
+	PrefixVEX3    Prefix = 0xC4 // Long form of VEX prefix
+	PrefixVEXnotR Prefix = 0x80 // VEX not R
+	PrefixVEXnotX Prefix = 0x40 // VEX not X
+	PrefixVEXnotB Prefix = 0x20 // VEX not B
+	PrefixVEXM    Prefix = 0x1f // VEX M
+	PrefixVEXW    Prefix = 0x80 // VEX W
+	PrefixVEXnotV Prefix = 0x78 // VEX not vvvv
+	PrefixVEXL    Prefix = 0x04 // VEX L
+	PrefixVEXP    Prefix = 0x03 // VEX PP
 )
 
 // IsREX reports whether p is a REX prefix byte.
@@ -87,7 +96,7 @@ func (p Prefix) IsREX() bool {
 }
 
 func (p Prefix) IsVEX() bool {
-	return p&0xFF == PrefixVEX2Bytes || p&0xFF == PrefixVEX3Bytes
+	return p&0xFF == PrefixVEX2 || p&0xFF == PrefixVEX3
 }
 
 func (p Prefix) String() string {
@@ -269,6 +278,24 @@ const (
 	X14
 	X15
 
+	// YMM registers.
+	Y0
+	Y1
+	Y2
+	Y3
+	Y4
+	Y5
+	Y6
+	Y7
+	Y8
+	Y9
+	Y10
+	Y11
+	Y12
+	Y13
+	Y14
+	Y15
+
 	// Segment registers.
 	ES
 	CS
@@ -595,6 +622,22 @@ var regNames = [...]string{
 	X13:  "X13",
 	X14:  "X14",
 	X15:  "X15",
+	Y0:   "Y0",
+	Y1:   "Y1",
+	Y2:   "Y2",
+	Y3:   "Y3",
+	Y4:   "Y4",
+	Y5:   "Y5",
+	Y6:   "Y6",
+	Y7:   "Y7",
+	Y8:   "Y8",
+	Y9:   "Y9",
+	Y10:  "Y10",
+	Y11:  "Y11",
+	Y12:  "Y12",
+	Y13:  "Y13",
+	Y14:  "Y14",
+	Y15:  "Y15",
 	CS:   "CS",
 	SS:   "SS",
 	DS:   "DS",

x86/x86asm/intel.go

@@ -93,7 +93,7 @@ func IntelSyntax(inst Inst, pc uint64, symname SymLookup) string {
 				inst.Prefix[i] |= PrefixImplicit
 			}
 			if p.IsVEX() {
-				if p == PrefixVEX3Bytes {
+				if p == PrefixVEX3 {
 					inst.Prefix[i+2] |= PrefixImplicit
 				}
 				inst.Prefix[i] |= PrefixImplicit
@@ -471,12 +471,7 @@ func intelArg(inst *Inst, pc uint64, symname SymLookup, arg Arg) string {
 		}
 	case Reg:
 		if int(a) < len(intelReg) && intelReg[a] != "" {
-			switch inst.Op {
-			case VMOVDQA, VMOVDQU, VMOVNTDQA, VMOVNTDQ:
-				return strings.Replace(intelReg[a], "xmm", "ymm", -1)
-			default:
-				return intelReg[a]
-			}
+			return intelReg[a]
 		}
 	}
 	return strings.ToLower(arg.String())
@@ -542,6 +537,22 @@ var intelReg = [...]string{
 	X13: "xmm13",
 	X14: "xmm14",
 	X15: "xmm15",
+	Y0:  "ymm0",
+	Y1:  "ymm1",
+	Y2:  "ymm2",
+	Y3:  "ymm3",
+	Y4:  "ymm4",
+	Y5:  "ymm5",
+	Y6:  "ymm6",
+	Y7:  "ymm7",
+	Y8:  "ymm8",
+	Y9:  "ymm9",
+	Y10: "ymm10",
+	Y11: "ymm11",
+	Y12: "ymm12",
+	Y13: "ymm13",
+	Y14: "ymm14",
+	Y15: "ymm15",
 
 	// TODO: Maybe the constants are named wrong.
 	SPB: "spl",

x86/x86asm/plan9x.go

@@ -338,6 +338,22 @@ var plan9Reg = [...]string{
 	X13:  "X13",
 	X14:  "X14",
 	X15:  "X15",
+	Y0:   "Y0",
+	Y1:   "Y1",
+	Y2:   "Y2",
+	Y3:   "Y3",
+	Y4:   "Y4",
+	Y5:   "Y5",
+	Y6:   "Y6",
+	Y7:   "Y7",
+	Y8:   "Y8",
+	Y9:   "Y9",
+	Y10:  "Y10",
+	Y11:  "Y11",
+	Y12:  "Y12",
+	Y13:  "Y13",
+	Y14:  "Y14",
+	Y15:  "Y15",
 	CS:   "CS",
 	SS:   "SS",
 	DS:   "DS",