avx.go

     1  // Copyright 2026 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package x86asm
     6  
     7  import (
     8  	"encoding/binary"
     9  	"errors"
    10  )
    11  
    12  // This file contains the handling of AVX instructions, based on
    13  // tables (avx_tables.go) generated from the XED data.
    14  
    15  //go:generate go run _gen/genavx.go -o avx_tables.go
    16  
    17  // decodeAVX decodes AVX/AVX2/AVX-512 instructions.
    18  // It is called from decode1 when a VEX or EVEX prefix is detected.
    19  func decodeAVX(src []byte, pos int, vex Prefix, vexIndex int, inst Inst, mode int) (Inst, error) {
    20  	var vexP, vexL, vexW uint8
    21  	var mapSelect uint8
    22  	var vvvv uint8
    23  	var vexR, vexX, vexB uint8 // Inverted from VEX/EVEX
    24  	var evex bool
    25  	var evexR_prime, evexV_prime uint8 // Inverted
    26  	var evex_aaa, evex_z uint8
    27  	var evex_b uint8
    28  
    29  	vexR = 1
    30  	vexX = 1
    31  	vexB = 1 // Default to 1 (inactive inverted)
    32  	evexR_prime = 1
    33  	evexV_prime = 1
    34  
    35  	if vex == 0xC5 { // 2-byte VEX
    36  		b1 := uint8(inst.Prefix[vexIndex+1])
    37  		vexR = (b1 >> 7) & 1
    38  		vvvv = (b1 >> 3) & 0xF
    39  		vexL = (b1 >> 2) & 1
    40  		vexP = b1 & 3
    41  		mapSelect = 1 // 0F
    42  	} else if vex == 0xC4 { // 3-byte VEX
    43  		b1 := uint8(inst.Prefix[vexIndex+1])
    44  		b2 := uint8(inst.Prefix[vexIndex+2])
    45  		vexR = (b1 >> 7) & 1
    46  		vexX = (b1 >> 6) & 1
    47  		vexB = (b1 >> 5) & 1
    48  		mapSelect = b1 & 0x1F
    49  
    50  		vexW = (b2 >> 7) & 1
    51  		vvvv = (b2 >> 3) & 0xF
    52  		vexL = (b2 >> 2) & 1
    53  		vexP = b2 & 3
    54  	} else if vex == 0x62 { // EVEX
    55  		evex = true
    56  		b1 := uint8(inst.Prefix[vexIndex+1])
    57  		b2 := uint8(inst.Prefix[vexIndex+2])
    58  		b3 := uint8(inst.Prefix[vexIndex+3])
    59  
    60  		vexR = (b1 >> 7) & 1
    61  		vexX = (b1 >> 6) & 1
    62  		vexB = (b1 >> 5) & 1
    63  		evexR_prime = (b1 >> 4) & 1
    64  		mapSelect = b1 & 3
    65  
    66  		vexW = (b2 >> 7) & 1
    67  		vvvv = (b2 >> 3) & 0xF
    68  		vexP = b2 & 3
    69  
    70  		evex_z = (b3 >> 7) & 1
    71  		vexL = (b3 >> 5) & 3
    72  		evex_b = (b3 >> 4) & 1
    73  		evexV_prime = (b3 >> 3) & 1
    74  		evex_aaa = b3 & 7
    75  	}
    76  
    77  	_ = evex_z // TODO: use zeroing mask if needed for output
    78  
    79  	opbyte := src[pos]
    80  	pos++
    81  
    82  	var candidates []*avxOptab
    83  	switch mapSelect {
    84  	case 1:
    85  		candidates = avxMap0F[opbyte]
    86  	case 2:
    87  		candidates = avxMap0F38[opbyte]
    88  	case 3:
    89  		candidates = avxMap0F3A[opbyte]
    90  	}
    91  
    92  	if len(candidates) == 0 {
    93  		return inst, errors.New("unknown AVX Opcode")
    94  	}
    95  
    96  	var modrm uint8
    97  	var haveModRM bool
    98  	if pos < len(src) {
    99  		modrm = src[pos]
   100  		haveModRM = true
   101  	}
   102  
   103  	var match *avxOptab
   104  
   105  	for i := range candidates {
   106  		c := candidates[i]
   107  		if evex != c.evex {
   108  			continue
   109  		}
   110  		c_vexP := c.vexP
   111  		p_match := false
   112  		switch c_vexP {
   113  		case 0:
   114  			p_match = vexP == 0
   115  		case 1:
   116  			p_match = vexP == 1
   117  		case 2:
   118  			p_match = vexP == 3
   119  		case 3:
   120  			p_match = vexP == 2
   121  		}
   122  		if !p_match {
   123  			continue
   124  		}
   125  
   126  		match_vexL := vexL
   127  		if evex && evex_b != 0 && haveModRM && (modrm>>6) == 3 {
   128  			hasZmm := false
   129  			for j := range candidates {
   130  				if candidates[j].evex == c.evex && candidates[j].vexP == c.vexP && candidates[j].vexW == c.vexW && candidates[j].vexL == 2 {
   131  					hasZmm = true
   132  					break
   133  				}
   134  			}
   135  			if hasZmm {
   136  				match_vexL = 2
   137  			} else {
   138  				match_vexL = 0
   139  			}
   140  		}
   141  		if c.vexL != match_vexL {
   142  			continue
   143  		}
   144  		if c.vexW != vexW {
   145  			continue
   146  		}
   147  
   148  		if haveModRM {
   149  			mod := modrm >> 6
   150  			reg := (modrm >> 3) & 7
   151  			if c.opdigit != -1 && reg != uint8(c.opdigit) {
   152  				continue
   153  			}
   154  			if c.ismem == 1 && mod == 3 {
   155  				continue
   156  			}
   157  			if c.ismem == 0 && mod != 3 {
   158  				continue
   159  			}
   160  		}
   161  		match = c
   162  		break
   163  	}
   164  
   165  	if match == nil {
   166  		return Inst{Len: 1}, ErrUnrecognized
   167  	}
   168  
   169  	inst.Op = match.op
   170  
   171  	var mod, reg, rm uint8
   172  	var sib uint8
   173  	var haveSIB bool
   174  	var mem Mem
   175  	var addrMode = mode
   176  
   177  	if haveModRM {
   178  		mod = modrm >> 6
   179  		reg = (modrm >> 3) & 7
   180  		rm = modrm & 7
   181  		pos++
   182  
   183  		if mod != 3 && rm == 4 {
   184  			if pos >= len(src) {
   185  				return inst, errors.New("truncated")
   186  			}
   187  			sib = src[pos]
   188  			haveSIB = true
   189  			pos++
   190  		}
   191  
   192  		var disp int64
   193  		if mod == 0 && (rm == 5 || (haveSIB && (sib&7) == 5)) || mod == 2 {
   194  			if pos+4 > len(src) {
   195  				return inst, errors.New("truncated")
   196  			}
   197  			disp = int64(int32(binary.LittleEndian.Uint32(src[pos:])))
   198  			pos += 4
   199  		} else if mod == 1 {
   200  			if pos >= len(src) {
   201  				return inst, errors.New("truncated")
   202  			}
   203  			disp = int64(int8(src[pos]))
   204  			pos++
   205  			if evex && match.dispScale > 0 {
   206  				scale := match.dispScale
   207  				if evex_b != 0 && match.bcstScale > 0 {
   208  					scale = match.bcstScale
   209  				}
   210  				disp *= int64(scale)
   211  			}
   212  		}
   213  		mem.Disp = disp
   214  
   215  		if haveSIB {
   216  			scale := sib >> 6
   217  			index := (sib >> 3) & 7
   218  			base := sib & 7
   219  
   220  			if vexX == 0 {
   221  				index |= 8
   222  			}
   223  			if vexB == 0 {
   224  				base |= 8
   225  			}
   226  
   227  			mem.Scale = 1 << uint(scale)
   228  			if index != 4 {
   229  				mem.Index = baseRegForBits(addrMode) + Reg(index)
   230  			}
   231  			if base&7 != 5 || mod != 0 {
   232  				mem.Base = baseRegForBits(addrMode) + Reg(base)
   233  			}
   234  		} else {
   235  			if vexB == 0 {
   236  				rm |= 8
   237  			}
   238  
   239  			if mod != 3 {
   240  				if !(mod == 0 && rm&7 == 5) {
   241  					mem.Base = baseRegForBits(addrMode) + Reg(rm)
   242  				}
   243  			}
   244  		}
   245  	}
   246  
   247  	// Decode Args
   248  	for i, argType := range match.args {
   249  		if argType == argNone {
   250  			continue
   251  		}
   252  		var arg Arg
   253  
   254  		switch argType {
   255  		case argImm8:
   256  			if pos >= len(src) {
   257  				return inst, errors.New("truncated")
   258  			}
   259  			arg = Imm(src[pos])
   260  			pos++
   261  		case argImm8u:
   262  			if pos >= len(src) {
   263  				return inst, errors.New("truncated")
   264  			}
   265  			arg = Imm(src[pos])
   266  			pos++
   267  		case argXmm_SE, argYmm_SE:
   268  			if pos >= len(src) {
   269  				return inst, errors.New("truncated")
   270  			}
   271  			idx := (src[pos] >> 4) & 0xF
   272  			if argType == argXmm_SE {
   273  				arg = X0 + Reg(idx)
   274  			} else {
   275  				arg = Y0 + Reg(idx)
   276  			}
   277  			pos++
   278  		case argGPR_R, argGPR32_R, argGPR64_R:
   279  			idx := reg
   280  			if vexR == 0 {
   281  				idx |= 8
   282  			}
   283  			base := baseRegForBits(mode)
   284  			if argType == argGPR32_R {
   285  				base = EAX
   286  			} else if argType == argGPR64_R {
   287  				base = RAX
   288  			}
   289  			arg = base + Reg(idx)
   290  		case argGPR_N, argGPR32_N, argGPR64_N:
   291  			idx := ^vvvv & 15 // 1s complement
   292  			base := baseRegForBits(mode)
   293  			if argType == argGPR32_N {
   294  				base = EAX
   295  			} else if argType == argGPR64_N {
   296  				base = RAX
   297  			} else if vexW == 1 {
   298  				base = RAX
   299  			}
   300  			arg = base + Reg(idx)
   301  		case argGPR_B, argGPR32_B, argGPR64_B:
   302  			idx := rm
   303  			if vexB == 0 {
   304  				idx |= 8
   305  			}
   306  			base := baseRegForBits(mode)
   307  			if argType == argGPR32_B {
   308  				base = EAX
   309  			} else if argType == argGPR64_B {
   310  				base = RAX
   311  			}
   312  			arg = base + Reg(idx)
   313  		// VEX/EVEX encoding uses inverted bits for register specifiers (0 means bit is set).
   314  		case argXmm_R, argXmmEvex_R:
   315  			idx := reg
   316  			if vexR == 0 {
   317  				idx |= 8
   318  			}
   319  			if evex && evexR_prime == 0 {
   320  				idx |= 16
   321  			}
   322  			arg = X0 + Reg(idx)
   323  		case argXmm_B, argXmmEvex_B:
   324  			idx := rm
   325  			if vexB == 0 {
   326  				idx |= 8
   327  			}
   328  			if evex && vexX == 0 {
   329  				idx |= 16
   330  			}
   331  			arg = X0 + Reg(idx)
   332  		case argXmm_N, argXmmEvex_N:
   333  			idx := 15 - vvvv
   334  			if evex && evexV_prime == 0 {
   335  				idx |= 16
   336  			}
   337  			arg = X0 + Reg(idx)
   338  		case argYmm_R, argYmmEvex_R:
   339  			idx := reg
   340  			if vexR == 0 {
   341  				idx |= 8
   342  			}
   343  			if evex && evexR_prime == 0 {
   344  				idx |= 16
   345  			}
   346  			arg = Y0 + Reg(idx)
   347  		case argYmm_B, argYmmEvex_B:
   348  			idx := rm
   349  			if vexB == 0 {
   350  				idx |= 8
   351  			}
   352  			if evex && vexX == 0 {
   353  				idx |= 16
   354  			}
   355  			arg = Y0 + Reg(idx)
   356  		case argYmm_N, argYmmEvex_N:
   357  			idx := 15 - vvvv
   358  			if evex && evexV_prime == 0 {
   359  				idx |= 16
   360  			}
   361  			arg = Y0 + Reg(idx)
   362  		case argZmm_R:
   363  			vl := vexL
   364  			if evex && evex_b != 0 && match.ismem == 0 {
   365  				vl = 2 // RC / SAE implies 512-bit vector length
   366  			}
   367  			idx := reg
   368  			if vexR == 0 {
   369  				idx |= 8
   370  			}
   371  			if evexR_prime == 0 {
   372  				idx |= 16
   373  			}
   374  			if vl == 0 {
   375  				arg = X0 + Reg(idx)
   376  			} else if vl == 1 {
   377  				arg = Y0 + Reg(idx)
   378  			} else {
   379  				arg = Z0 + Reg(idx)
   380  			}
   381  		case argZmm_B:
   382  			vl := vexL
   383  			if evex && evex_b != 0 && match.ismem == 0 {
   384  				vl = 2
   385  			}
   386  			if match.ismem != 0 {
   387  				arg = mem
   388  			} else {
   389  				idx := rm
   390  				if vexB == 0 {
   391  					idx |= 8
   392  				}
   393  				if vexX == 0 {
   394  					idx |= 16
   395  				}
   396  				if vl == 0 {
   397  					arg = X0 + Reg(idx)
   398  				} else if vl == 1 {
   399  					arg = Y0 + Reg(idx)
   400  				} else {
   401  					arg = Z0 + Reg(idx)
   402  				}
   403  			}
   404  		case argZmm_N:
   405  			vl := vexL
   406  			if evex && evex_b != 0 && match.ismem == 0 {
   407  				vl = 2
   408  			}
   409  			idx := 15 - vvvv
   410  			if evexV_prime == 0 {
   411  				idx |= 16
   412  			}
   413  			if vl == 0 {
   414  				arg = X0 + Reg(idx)
   415  			} else if vl == 1 {
   416  				arg = Y0 + Reg(idx)
   417  			} else {
   418  				arg = Z0 + Reg(idx)
   419  			}
   420  		case argK_R:
   421  			arg = K0 + Reg(reg&7)
   422  		case argK_B:
   423  			arg = K0 + Reg(rm&7)
   424  		case argK_N:
   425  			arg = K0 + Reg((15-vvvv)&7)
   426  		case argKmask:
   427  			if evex_aaa != 0 {
   428  				arg = K0 + Reg(evex_aaa)
   429  			}
   430  		case argKnot0:
   431  			if evex_aaa == 0 {
   432  				return inst, errors.New("k0 mask not allowed")
   433  			}
   434  			arg = K0 + Reg(evex_aaa)
   435  		case argM:
   436  			arg = mem
   437  		}
   438  
   439  		if arg != nil {
   440  			inst.Args[i] = arg
   441  		}
   442  	}
   443  
   444  	n := 0
   445  	for i := range len(inst.Args) {
   446  		if inst.Args[i] != nil {
   447  			if n != i {
   448  				inst.Args[n] = inst.Args[i]
   449  				inst.Args[i] = nil
   450  			}
   451  			n++
   452  		}
   453  	}
   454  	inst.MemBytes = int(match.memBytes)
   455  	if inst.MemBytes == 0 && match.ismem != 0 && match.dispScale != 0 {
   456  		inst.MemBytes = int(match.dispScale)
   457  	}
   458  	if evex {
   459  		inst.Zeroing = evex_z != 0
   460  		if evex_b != 0 {
   461  			if match.bcstScale > 0 {
   462  				inst.Broadcast = true
   463  				inst.MemBytes = int(match.bcstScale)
   464  			} else if match.ismem == 0 {
   465  				inst.SAE = true
   466  				inst.Rounding = int8(vexL)
   467  			}
   468  		}
   469  	}
   470  	inst.Len = pos
   471  
   472  	if match.vsib && haveSIB {
   473  		fixVSIB(&inst, vexL, evex, evexV_prime, vexX, sib)
   474  	}
   475  
   476  	return inst, nil
   477  }
   478  
   479  // fixVSIB calculates the correct vector register size based on data and index element sizes.
   480  func fixVSIB(inst *Inst, vexL uint8, evex bool, evexV_prime uint8, vexX uint8, sib uint8) {
   481  	var indexElemBits, dataElemBits int
   482  	switch inst.Op {
   483  	case VPGATHERDD, VGATHERDPS, VPSCATTERDD, VSCATTERDPS:
   484  		indexElemBits = 32
   485  		dataElemBits = 32
   486  	case VPGATHERDQ, VGATHERDPD, VPSCATTERDQ, VSCATTERDPD:
   487  		indexElemBits = 32
   488  		dataElemBits = 64
   489  	case VPGATHERQD, VGATHERQPS, VPSCATTERQD, VSCATTERQPS:
   490  		indexElemBits = 64
   491  		dataElemBits = 32
   492  	case VPGATHERQQ, VGATHERQPD, VPSCATTERQQ, VSCATTERQPD:
   493  		indexElemBits = 64
   494  		dataElemBits = 64
   495  	case VGATHERPF0DPS, VGATHERPF1DPS, VSCATTERPF0DPS, VSCATTERPF1DPS:
   496  		indexElemBits = 32
   497  		dataElemBits = 32
   498  	case VGATHERPF0DPD, VGATHERPF1DPD, VSCATTERPF0DPD, VSCATTERPF1DPD:
   499  		indexElemBits = 32
   500  		dataElemBits = 64
   501  	case VGATHERPF0QPS, VGATHERPF1QPS, VSCATTERPF0QPS, VSCATTERPF1QPS:
   502  		indexElemBits = 64
   503  		dataElemBits = 32
   504  	case VGATHERPF0QPD, VGATHERPF1QPD, VSCATTERPF0QPD, VSCATTERPF1QPD:
   505  		indexElemBits = 64
   506  		dataElemBits = 64
   507  	default:
   508  		return
   509  	}
   510  
   511  	maxBits := 128 << vexL
   512  
   513  	var destBits, indexVectorBits int
   514  	if indexElemBits > dataElemBits {
   515  		indexVectorBits = maxBits
   516  		numElements := indexVectorBits / indexElemBits
   517  		destBits = numElements * dataElemBits
   518  	} else if dataElemBits > indexElemBits {
   519  		destBits = maxBits
   520  		numElements := destBits / dataElemBits
   521  		indexVectorBits = numElements * indexElemBits
   522  	} else {
   523  		indexVectorBits = maxBits
   524  		destBits = maxBits
   525  	}
   526  
   527  	// Override MemBytes to match objdump's output expectation (memory accessed is based on dest size)
   528  	inst.MemBytes = destBits / 8
   529  
   530  	if indexVectorBits < 128 {
   531  		indexVectorBits = 128
   532  	}
   533  
   534  	var baseReg Reg
   535  	switch indexVectorBits {
   536  	case 128:
   537  		baseReg = X0
   538  	case 256:
   539  		baseReg = Y0
   540  	case 512:
   541  		baseReg = Z0
   542  	default:
   543  		baseReg = X0
   544  	}
   545  
   546  	for i, arg := range inst.Args {
   547  		if mem, ok := arg.(Mem); ok {
   548  			idx := (sib >> 3) & 7
   549  			if vexX == 0 {
   550  				idx |= 8
   551  			}
   552  			if evex && evexV_prime == 0 {
   553  				idx |= 16
   554  			}
   555  			mem.Index = baseReg + Reg(idx)
   556  			inst.Args[i] = mem
   557  			break
   558  		}
   559  	}
   560  }
   561  
   562  // argType defines how to decode an argument.
   563  // It corresponds to the arg type notation in XED.
   564  type argType uint8
   565  
   566  const (
   567  	argNone argType = iota
   568  	argImm8
   569  	argImm8u
   570  	argImm16
   571  	argImm32
   572  	argImm64
   573  
   574  	// GPRs
   575  	argGPR_R   // ModRM.reg (default mode size)
   576  	argGPR_B   // ModRM.rm (default mode size)
   577  	argGPR_N   // VEX.vvvv (default mode size)
   578  	argGPR32_R // ModRM.reg (32-bit forced)
   579  	argGPR32_B // ModRM.rm (32-bit forced)
   580  	argGPR32_N // VEX.vvvv (32-bit forced)
   581  	argGPR64_R // ModRM.reg (64-bit forced)
   582  	argGPR64_B // ModRM.rm (64-bit forced)
   583  	argGPR64_N // VEX.vvvv (64-bit forced)
   584  
   585  	// XMM
   586  	argXmm_R
   587  	argXmm_B
   588  	argXmm_N
   589  	argXmmEvex_R
   590  	argXmmEvex_B
   591  	argXmmEvex_N
   592  	argXmm_SE // is4 immediate
   593  
   594  	// YMM
   595  	argYmm_R
   596  	argYmm_B
   597  	argYmm_N
   598  	argYmmEvex_R
   599  	argYmmEvex_B
   600  	argYmmEvex_N
   601  	argYmm_SE
   602  
   603  	// ZMM
   604  	argZmm_R
   605  	argZmm_B
   606  	argZmm_N
   607  
   608  	// Mask
   609  	argK_R
   610  	argK_B
   611  	argK_N
   612  
   613  	argM     // Memory operand (ModRM.rm)
   614  	argKnot0 // Mask register k1-k7
   615  	argKmask // Mask register k0-k7
   616  )
   617  
   618  // hasRC returns true if the instruction supports static rounding control in AVX-512.
   619  func hasRC(op Op) bool {
   620  	switch op {
   621  	case VADDPD, VADDPS, VADDSD, VADDSS,
   622  		VSUBPD, VSUBPS, VSUBSD, VSUBSS,
   623  		VMULPD, VMULPS, VMULSD, VMULSS,
   624  		VDIVPD, VDIVPS, VDIVSD, VDIVSS,
   625  		VSQRTPD, VSQRTPS, VSQRTSD, VSQRTSS,
   626  		VSCALEFPD, VSCALEFPS, VSCALEFSD, VSCALEFSS,
   627  		VFMADD132PD, VFMADD132PS, VFMADD132SD, VFMADD132SS,
   628  		VFMADD213PD, VFMADD213PS, VFMADD213SD, VFMADD213SS,
   629  		VFMADD231PD, VFMADD231PS, VFMADD231SD, VFMADD231SS,
   630  		VFMSUB132PD, VFMSUB132PS, VFMSUB132SD, VFMSUB132SS,
   631  		VFMSUB213PD, VFMSUB213PS, VFMSUB213SD, VFMSUB213SS,
   632  		VFMSUB231PD, VFMSUB231PS, VFMSUB231SD, VFMSUB231SS,
   633  		VFNMADD132PD, VFNMADD132PS, VFNMADD132SD, VFNMADD132SS,
   634  		VFNMADD213PD, VFNMADD213PS, VFNMADD213SD, VFNMADD213SS,
   635  		VFNMADD231PD, VFNMADD231PS, VFNMADD231SD, VFNMADD231SS,
   636  		VFNMSUB132PD, VFNMSUB132PS, VFNMSUB132SD, VFNMSUB132SS,
   637  		VFNMSUB213PD, VFNMSUB213PS, VFNMSUB213SD, VFNMSUB213SS,
   638  		VFNMSUB231PD, VFNMSUB231PS, VFNMSUB231SD, VFNMSUB231SS,
   639  		VFMADDSUB132PD, VFMADDSUB132PS, VFMADDSUB213PD, VFMADDSUB213PS,
   640  		VFMADDSUB231PD, VFMADDSUB231PS, VFMSUBADD132PD, VFMSUBADD132PS,
   641  		VFMSUBADD213PD, VFMSUBADD213PS, VFMSUBADD231PD, VFMSUBADD231PS,
   642  		VCVTPS2DQ, VCVTPD2DQ, VCVTPS2UDQ, VCVTPD2UDQ,
   643  		VCVTPS2QQ, VCVTPD2QQ, VCVTPS2UQQ, VCVTPD2UQQ,
   644  		VCVTUDQ2PS, VCVTUDQ2PD, VCVTQQ2PS, VCVTQQ2PD,
   645  		VCVTUQQ2PS, VCVTUQQ2PD, VCVTDQ2PS, VCVTDQ2PD,
   646  		VCVTPS2PD, VCVTPD2PS, VCVTSS2SD, VCVTSD2SS,
   647  		VCVTUSI2SS, VCVTUSI2SD, VCVTSI2SS, VCVTSI2SD,
   648  		VCVTSS2USI, VCVTSD2USI, VCVTSS2SI, VCVTSD2SI,
   649  		VPERMT2PD, VPERMT2PS, VPERMI2PD, VPERMI2PS:
   650  		return true
   651  	}
   652  	return false
   653  }
   654
View as plain text