instruction.go

     1  // Copyright 2026 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package sve loads ARM64 SVE / SVE2 instruction definitions from the ARM A64
     6  // ISA XML files and emits them as simdgen unify values.
     7  // TODO: merge with the arm64 package, the approach taken here should take over
     8  // the NEON loader.
     9  // TODO: merge with x/arch/arm64/instgen?
    10  //
    11  // SVE registers are "scalable": their total bit width is the hardware
    12  // implementation-defined vector length rather than a fixed 128/256/512 bits. So
    13  // emitted vector operands carry only a base type and an element width, without a
    14  // fixed bits/lanes count.
    15  //
    16  // Arrangement is per-operand. An SVE instruction template such as
    17  //
    18  //	ADD  <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
    19  //
    20  // stands for a family of concrete instructions, one per value of the <T>
    21  // arrangement symbol. simdgen enumerates them by resolving each operand's
    22  // arrangement symbol from the section's explanations. Different symbols can be
    23  // encoded in the same instruction field but interpreted differently, the
    24  // loader also takes care of this.
    25  //
    26  // It emits register, mask, immediate, memory and special operands.
    27  // Memory and special operands are opaque at this moment.
    28  // Register-list operands are not modeled yet, except for single-register lists,
    29  // so instructions carrying one are skipped (TODO); see classify.
    30  //
    31  // TODO: Peepholes might need the structure of memory operands, implement it?
    32  // TODO: special operands are like registers with indexing, prefetch ops, etc.
    33  // They seem too specialized that we might want to manually implment them instead
    34  // of via simdgen, but we can revisit this.
    35  package sve
    36  
    37  import (
    38  	"fmt"
    39  	"regexp"
    40  	"strings"
    41  
    42  	"golang.org/x/arch/arm64/instgen/xmlspec"
    43  )
    44  
    45  // signedImmRe matches an [Instruction.brief] that describes a signed/unsigned *immediate*
    46  // (e.g. DUP/CPY "Move signed integer immediate ..."). There the signedness is a
    47  // property of the immediate encoding, not of the vector lane, so such ops are
    48  // signedness-agnostic.
    49  var signedImmRe = regexp.MustCompile(`(un)?signed(\s+\w+)?\s+immediate`)
    50  
    51  // reZReg and rePReg detect a Z (scalable vector) or P (predicate) register in an
    52  // assembly template, used to choose the Go opcode prefix (see goOpPrefix). The
    53  // [^/] guard excludes the /<ZM> predication qualifier, which is not a Z register.
    54  // Copied from x/arch/arm64/instgen/xmlspec.
    55  var (
    56  	reZReg = regexp.MustCompile(`(^|[^/])<Z[A-Za-z1-9]+>`)
    57  	rePReg = regexp.MustCompile(`<P[A-Za-z1-9]+>`)
    58  )
    59  
    60  // Instruction is a *logical* SVE instruction, one per iclass.
    61  type Instruction struct {
    62  	xmlspec.Instruction
    63  	// iclass is the specific class this logical instruction represents.
    64  	// A raw xmlspec.Instruction can hold several iclasses with distinct mnemonics.
    65  	// If nil, the first iclass is used.
    66  	iclass        *xmlspec.Iclass
    67  	mnemonicCache string
    68  	// predVariants is set on the unpredicated instruction of a
    69  	// predicated/unpredicated pair (see [groupPredicationForms]), one entry per
    70  	// predicated machine op the pair implies. It is nil for an instruction that
    71  	// comes in one form only.
    72  	predVariants []predVariant
    73  }
    74  
    75  // predVariant is one predicated encoding of an operation, as seen from its
    76  // unpredicated sibling: the governing-predicate qualifiers it offers ("M", "Z",
    77  // or "MZ" for an encoding written <Pg>/<ZM>, which supports either) and its
    78  // register symbols, in the same order as the sibling's own results and
    79  // non-predicate inputs.
    80  //
    81  // One encoding can imply several machine ops — one per qualifier — but they
    82  // share these symbols, because they are the same encoding. A second entry would
    83  // mean a genuinely separate predicated encoding, which no paired operation in
    84  // the ISA has today; the list exists so that such an encoding could be
    85  // described with its own symbols rather than collapsed onto the first one's.
    86  type predVariant struct {
    87  	quals       string
    88  	outRegNames []string
    89  	inRegNames  []string
    90  	// cpuFeature is the predicated sibling encoding's own feature level. It can
    91  	// sit below the carrier's: the unpredicated integer MUL is SVE2 while its
    92  	// predicated sibling is baseline SVE, making the operation available on SVE
    93  	// with the unpredicated encoding a feature-gated upgrade.
    94  	cpuFeature string
    95  	// predAsmPos is the assembly position of the encoding's governing
    96  	// predicate: 1 on every encoding grouped today, but recorded rather than
    97  	// assumed — PTEST, with no destination, governs from position 0.
    98  	predAsmPos int
    99  }
   100  
   101  // ic returns the iclass this logical instruction represents, defaulting to the
   102  // first iclass of the section.
   103  func (inst *Instruction) ic() *xmlspec.Iclass {
   104  	if inst.iclass != nil {
   105  		return inst.iclass
   106  	}
   107  	if len(inst.Classes.Iclass) > 0 {
   108  		return &inst.Classes.Iclass[0]
   109  	}
   110  	return nil
   111  }
   112  
   113  // extractDocVar returns the value of the named docvar, searching from most to
   114  // least specific: this iclass, its encodings, then the section top level.
   115  func (inst *Instruction) extractDocVar(key string) string {
   116  	if ic := inst.ic(); ic != nil {
   117  		for _, dv := range ic.DocVars {
   118  			if dv.Key == key {
   119  				return dv.Value
   120  			}
   121  		}
   122  		for _, enc := range ic.Encodings {
   123  			for _, dv := range enc.DocVars {
   124  				if dv.Key == key {
   125  					return dv.Value
   126  				}
   127  			}
   128  		}
   129  	}
   130  	for _, dv := range inst.DocVars {
   131  		if dv.Key == key {
   132  			return dv.Value
   133  		}
   134  	}
   135  	return ""
   136  }
   137  
   138  // mnemonic returns the instruction mnemonic, e.g. "ADD", "FADD", "SQADD".
   139  func (inst *Instruction) mnemonic() string {
   140  	if inst.mnemonicCache != "" {
   141  		return inst.mnemonicCache
   142  	}
   143  	m := inst.extractDocVar("mnemonic")
   144  	if inst.isAlias() {
   145  		m = inst.extractDocVar("alias_mnemonic")
   146  	}
   147  	inst.mnemonicCache = m
   148  	return m
   149  }
   150  
   151  // isAlias reports whether this XML entry describes an alias of another
   152  // instruction.
   153  func (inst *Instruction) isAlias() bool {
   154  	return inst.Type == "alias"
   155  }
   156  
   157  // instrClass returns the instruction class docvar, e.g. "sve" or "sve2".
   158  func (inst *Instruction) instrClass() string {
   159  	return inst.extractDocVar("instr-class")
   160  }
   161  
   162  // isSVE reports whether this is an SVE or SVE2 instruction.
   163  func (inst *Instruction) isSVE() bool {
   164  	switch inst.instrClass() {
   165  	case "sve", "sve2":
   166  		return true
   167  	}
   168  	return false
   169  }
   170  
   171  // cpuFeature returns the simdgen cpuFeature string for this instruction.
   172  func (inst *Instruction) cpuFeature() string {
   173  	switch inst.instrClass() {
   174  	case "sve2":
   175  		return "SVE2"
   176  	default:
   177  		return "SVE"
   178  	}
   179  }
   180  
   181  // goOpPrefix returns the Go opcode prefix: "Z" if the instruction uses a
   182  // scalable vector register, else "P" if it uses a predicate register, else "".
   183  // So the Go opcode is goOpPrefix()+mnemonic, e.g. ZADD but PPTRUE. Matches
   184  // x/arch/arm64/instgen/xmlspec.goOpcodePrefix.
   185  func (inst *Instruction) goOpPrefix() string {
   186  	ic := inst.ic()
   187  	if ic == nil {
   188  		return ""
   189  	}
   190  	hasZ, hasP := false, false
   191  	for _, enc := range ic.Encodings {
   192  		s := asmTemplateToString(enc.AsmTemplate)
   193  		hasZ = hasZ || reZReg.MatchString(s)
   194  		hasP = hasP || rePReg.MatchString(s)
   195  	}
   196  	switch {
   197  	case hasZ:
   198  		return "Z"
   199  	case hasP:
   200  		return "P"
   201  	default:
   202  		return ""
   203  	}
   204  }
   205  
   206  // laneIsFloat reports whether the given operand's vector lane holds
   207  // floating-point values.
   208  //
   209  // The int<->float conversions have different lane types on input and output, and the
   210  // operand's role selects which side this is:
   211  //
   212  //   - int->float (SCVTF/SCVTFLT, UCVTF/UCVTFLT): destination float, source int.
   213  //   - float->int (FCVTZS/FCVTZU and narrowing, FLOGB): destination int, source
   214  //     float.
   215  //
   216  // Every other instruction is uniform, i.e. all lanes the same type.
   217  func (inst *Instruction) laneIsFloat(op *Operand) bool {
   218  	switch op.Class {
   219  	case "vreg", "greg":
   220  		// has a lane
   221  	default:
   222  		// mask lanes are always integer; mem/immediate/special have no lane.
   223  		return false
   224  	}
   225  	dst := op.role == "destination"
   226  	switch inst.mnemonic() {
   227  	case "SCVTF", "SCVTFLT", "UCVTF", "UCVTFLT": // integer -> floating point
   228  		return dst
   229  	case "FCVTZS", "FCVTZSN", "FCVTZU", "FCVTZUN", "FLOGB": // floating point -> integer
   230  		return !dst
   231  	}
   232  	return isFloatBrief(inst.brief())
   233  }
   234  
   235  // bitwise reports whether this instruction is a bitwise operation, which the
   236  // spec's brief description spells with a "Bitwise " prefix (mirroring the NEON
   237  // loader's test). A bitwise vector encoding is written .D but is element-width
   238  // agnostic: any lane view of it is valid.
   239  func (inst *Instruction) bitwise() bool {
   240  	return strings.HasPrefix(inst.brief(), "Bitwise ")
   241  }
   242  
   243  // isFloatBrief reports whether a brief description names a floating-point type.
   244  // SVE spells these as "floating-point", "bfloat", or an "X-precision" (half /
   245  // single / double / 8-bit) qualifier.
   246  func isFloatBrief(brief string) bool {
   247  	b := strings.ToLower(brief)
   248  	return strings.Contains(b, "floating-point") ||
   249  		strings.Contains(b, "bfloat") ||
   250  		strings.Contains(b, "precision")
   251  }
   252  
   253  // signedness reports whether an integer instruction interprets its lanes as
   254  // signed, unsigned, or agnostic, so the loader emits only the signedness the
   255  // hardware actually implements, not spurious values. Many low-half/bitwise
   256  // ops, e.g. ADD, SUB, MUL, EOR, etc., are genuinely agnostic.
   257  // others are signedness-specific, e.g. SMAX vs UMAX, SDIV vs UDIV,
   258  // the int<->float converts, etc.
   259  //
   260  // The signal is the instruction's brief description, which names the signedness
   261  // for the specific ops ("Signed maximum", "Unsigned divide", "Signed integer
   262  // convert ...") and omits it for the agnostic ones.
   263  //
   264  // Two adjustments: a brief describing a signed/unsigned *immediate*
   265  // (DUP/CPY) is about the immediate, not the lane, so it stays agnostic; and the
   266  // shift-right family and FLOGB name their signedness differently (arithmetic vs
   267  // logical shift; "logarithm as integer") and are handled explicitly.
   268  func (inst *Instruction) signedness() string {
   269  	switch inst.mnemonic() {
   270  	case "ASR", "ASRD", "ASRR", "FLOGB": // arithmetic (sign-propagating) / signed exponent
   271  		return "int"
   272  	case "LSR", "LSRR": // logical (zero-filling) shift right
   273  		return "uint"
   274  	}
   275  	b := strings.ToLower(inst.brief())
   276  	if signedImmRe.MatchString(b) {
   277  		return ""
   278  	}
   279  	switch {
   280  	case strings.Contains(b, "unsigned"):
   281  		return "uint"
   282  	case strings.Contains(b, "signed"): // "unsigned" already handled, so this is the word "signed"
   283  		return "int"
   284  	}
   285  	return ""
   286  }
   287  
   288  // integerSignedness returns the signed/unsigned base variants to enumerate for
   289  // the instruction's integer lanes: the single value fixed by signedness for a
   290  // signedness-specific op, both {"int","uint"} for an agnostic op with an integer
   291  // lane (simdgen narrows later via the Go op definitions), or a single no-op pass
   292  // when there are no integer lanes.
   293  func (inst *Instruction) integerSignedness(ops []Operand) []string {
   294  	switch inst.signedness() {
   295  	case "int":
   296  		return []string{"int"}
   297  	case "uint":
   298  		return []string{"uint"}
   299  	}
   300  	for i := range ops {
   301  		if c := ops[i].Class; (c == "vreg" || c == "greg") && !inst.laneIsFloat(&ops[i]) {
   302  			return []string{"int", "uint"}
   303  		}
   304  	}
   305  	return []string{""}
   306  }
   307  
   308  // brief returns the instruction's short human-readable description, e.g. "Signed
   309  // maximum (predicated)".
   310  func (inst *Instruction) brief() string {
   311  	if len(inst.Desc.Brief.Para) > 0 {
   312  		return strings.TrimSpace(inst.Desc.Brief.Para[0].Text)
   313  	}
   314  	return ""
   315  }
   316  
   317  // findExplanation returns the explanation whose symbol is encoded with the
   318  // given link, or nil.
   319  func (inst *Instruction) findExplanation(link string) *xmlspec.Explanation {
   320  	for i := range inst.Explanations.Explanations {
   321  		if inst.Explanations.Explanations[i].Symbol.Link == link {
   322  			return &inst.Explanations.Explanations[i]
   323  		}
   324  	}
   325  	return nil
   326  }
   327  
   328  // symbolIsGoverning reports whether this instruction's explanation for
   329  // register symbol name (e.g. "Pg") describes it as the governing predicate —
   330  // the spec writes "the governing scalable predicate register" for exactly the
   331  // symbols with that role. found reports whether any explanation names the
   332  // symbol at all. This is the authoritative classification; [buildOperandList]
   333  // cross-checks it against the syntactic <Pg>/qualifier signal.
   334  func (inst *Instruction) symbolIsGoverning(name string) (governing, found bool) {
   335  	want := "<" + name + ">"
   336  	for i := range inst.Explanations.Explanations {
   337  		e := &inst.Explanations.Explanations[i]
   338  		if strings.TrimSpace(e.Symbol.Value) != want {
   339  			continue
   340  		}
   341  		found = true
   342  		if strings.Contains(strings.ToLower(e.Account.Intro), "governing") {
   343  			return true, true
   344  		}
   345  	}
   346  	return false, found
   347  }
   348  
   349  // arngRow is one row of an arrangement size table: the encoding value of the
   350  // size field and the resulting element width in bits.
   351  type arngRow struct {
   352  	size string // the size bitfield value, e.g. "01"; the shared key across symbols
   353  	bits int    // element width for this size (8/16/32/64)
   354  }
   355  
   356  // resolveArrangementTable returns the (size -> element width) rows for the
   357  // arrangement symbol encoded with the given link, read from its definition
   358  // table in encoding order. RESERVED and header rows (no valid element letter)
   359  // are dropped.
   360  //
   361  // Crucially, the size key is the shared encoding field, so different symbols
   362  // (<T> and <Tb>) that select on the same field line up by size. That is what
   363  // lets non-uniform (widening/narrowing) instructions like SUNPKHI give each
   364  // operand its own element width for the same encoded instruction.
   365  func (inst *Instruction) resolveArrangementTable(link string) []arngRow {
   366  	exp := inst.findExplanation(link)
   367  	if exp == nil {
   368  		return nil
   369  	}
   370  	var rows []arngRow
   371  	for i, row := range exp.Definition.Table.TGroup.TBody.Row {
   372  		var size string
   373  		bits := 0
   374  		for _, entry := range row.Entries {
   375  			switch entry.Class {
   376  			case "bitfield":
   377  				size = strings.TrimSpace(entry.Value)
   378  			case "symbol":
   379  				bits = elemLetterBits(strings.TrimSpace(entry.Value))
   380  			}
   381  		}
   382  		if bits == 0 {
   383  			continue // header or RESERVED row
   384  		}
   385  		if size == "" {
   386  			size = fmt.Sprintf("#%d", i) // single-column table: key by position
   387  		}
   388  		rows = append(rows, arngRow{size: size, bits: bits})
   389  	}
   390  	return rows
   391  }
   392  
   393  // arngLinks returns the distinct arrangement-symbol links used by ops, with the
   394  // destination's link first (it is the primary size driver), preserving order.
   395  func arngLinks(ops []Operand) []string {
   396  	seen := map[string]bool{}
   397  	var links []string
   398  	add := func(l string) {
   399  		if l != "" && !seen[l] {
   400  			seen[l] = true
   401  			links = append(links, l)
   402  		}
   403  	}
   404  	for _, op := range ops {
   405  		if op.role == "destination" {
   406  			add(op.arngLink)
   407  		}
   408  	}
   409  	for _, op := range ops {
   410  		add(op.arngLink)
   411  	}
   412  	return links
   413  }
   414  
   415  // elemLetterBits maps an SVE element specifier letter to its bit width.
   416  func elemLetterBits(letter string) int {
   417  	switch letter {
   418  	case "B":
   419  		return 8
   420  	case "H":
   421  		return 16
   422  	case "S":
   423  		return 32
   424  	case "D":
   425  		return 64
   426  	default:
   427  		return 0
   428  	}
   429  }
   430  
   431  // elemLetter is the inverse of elemLetterBits: it maps a bit width to its SVE
   432  // element specifier letter (used as the arrangement in emitted defs).
   433  func elemLetter(bits int) string {
   434  	switch bits {
   435  	case 8:
   436  		return "B"
   437  	case 16:
   438  		return "H"
   439  	case 32:
   440  		return "S"
   441  	case 64:
   442  		return "D"
   443  	default:
   444  		return ""
   445  	}
   446  }
   447  
   448  // allEncodingOperands returns the operand list of every distinct encoding of this iclass.
   449  func (inst *Instruction) allEncodingOperands() [][]Operand {
   450  	ic := inst.ic()
   451  	if ic == nil {
   452  		return nil
   453  	}
   454  	seen := map[string]bool{}
   455  	var out [][]Operand
   456  	for _, enc := range ic.Encodings {
   457  		s := asmTemplateToString(enc.AsmTemplate)
   458  		if s == "" || seen[s] {
   459  			continue
   460  		}
   461  		seen[s] = true
   462  		ops := func() []Operand {
   463  			// A classification panic names only the operand; add which
   464  			// instruction and template it came from.
   465  			defer func() {
   466  				if r := recover(); r != nil {
   467  					panic(fmt.Sprintf("%v\n  in %q template %q", r, inst.Title, s))
   468  				}
   469  			}()
   470  			return operandsFromTextA(enc.AsmTemplate.TextA, inst.symbolIsGoverning)
   471  		}()
   472  		if len(ops) > 0 {
   473  			inst.fixMemoryDirection(ops)
   474  			out = append(out, ops)
   475  		}
   476  	}
   477  	return out
   478  }
   479  
   480  // fixMemoryDirection re-roles a load/store's data direction, which the operand
   481  // order does not reveal on its own. A store's destination is its memory operand
   482  // (unusually, at the end of the syntax, e.g. ST1B {<Zt>.<T>}, <Pg>, [<Xn|SP>]);
   483  // a load's destination is the transferred vector register (the memory is then a
   484  // source). Load/store is read from the brief description.
   485  func (inst *Instruction) fixMemoryDirection(ops []Operand) {
   486  	b := strings.ToLower(inst.brief())
   487  	store := strings.Contains(b, "store")
   488  	load := strings.Contains(b, "load")
   489  	if !store && !load {
   490  		return
   491  	}
   492  	for i := range ops {
   493  		switch {
   494  		case store && ops[i].Class == "mem":
   495  			ops[i].role = "destination"
   496  		case load && ops[i].Class == "vreg":
   497  			ops[i].role = "destination"
   498  		}
   499  	}
   500  }
   501  
   502  // operands parses the operands of this instruction's first encoding form. Most
   503  // instructions have exactly one; use templates for the complete set.
   504  func (inst *Instruction) operands() []Operand {
   505  	if ops := inst.allEncodingOperands(); len(ops) > 0 {
   506  		return ops[0]
   507  	}
   508  	return nil
   509  }
   510  
   511  // hasClass reports whether any operand has the given class.
   512  func hasClass(ops []Operand, class string) bool {
   513  	for _, op := range ops {
   514  		if op.Class == class {
   515  			return true
   516  		}
   517  	}
   518  	return false
   519  }
   520  
   521  // predicationVariants returns the governing-predicate qualifiers to emit for a
   522  // template: the predicate operand's own qualifier ("M" or "Z"), both when a
   523  // single encoding written "<Pg>/<ZM>" (MOVPRFX) selects merging or zeroing via a
   524  // bit, or a single no-op pass when the template has no governing predicate.
   525  func predicationVariants(ops []Operand) []string {
   526  	for i := range ops {
   527  		if ops[i].governing {
   528  			if ops[i].Predication == "MZ" {
   529  				return []string{"M", "Z"}
   530  			}
   531  			return []string{ops[i].Predication}
   532  		}
   533  	}
   534  	return []string{""}
   535  }
   536  
   537  // predicationForm reports whether this encoding is the predicated or the
   538  // unpredicated form of an operation, as "predicated" / "unpredicated".
   539  //
   540  // It reads the encoding rather than the title: an encoding that takes a
   541  // governing predicate is the predicated one. SVE does also spell this out in
   542  // the title of an operation that has both forms ("ADD (vectors, predicated)"
   543  // and "ADD (vectors, unpredicated)"), and [predicationGroupKey] uses that to pair
   544  // them, but an operation that only comes predicated says nothing in its title —
   545  // both of ABS's encodings are titled plain "ABS".
   546  func (inst *Instruction) predicationForm() string {
   547  	for _, ops := range inst.allEncodingOperands() {
   548  		for i := range ops {
   549  			if ops[i].governing {
   550  				return "predicated"
   551  			}
   552  		}
   553  	}
   554  	return "unpredicated"
   555  }
   556  
   557  // predicationGroupKey returns the key that groups the encodings of one
   558  // operation: the title with any predicated/unpredicated qualifier removed, e.g.
   559  // both "ADD (vectors, predicated)" and "ADD (vectors, unpredicated)" yield "add
   560  // (vectors)", and both of ABS's encodings yield "abs".
   561  //
   562  // Encodings that are not variations on one another keep distinct titles — "ADD
   563  // (immediate)", "ADD (extended register)" — so they land in groups of their own,
   564  // which groupPredicationForms then leaves alone.
   565  func (inst *Instruction) predicationGroupKey() string {
   566  	t := strings.ToLower(inst.Title)
   567  	t = strings.ReplaceAll(t, "unpredicated", "")
   568  	t = strings.ReplaceAll(t, "predicated", "")
   569  	// Tidy the separator the qualifier left behind: "(vectors, )" -> "(vectors)".
   570  	t = strings.ReplaceAll(t, ", )", ")")
   571  	t = strings.ReplaceAll(t, "( ", "(")
   572  	return strings.Join(strings.Fields(t), " ")
   573  }
   574  
   575  // documentation returns a one-line description of the instruction.
   576  func (inst *Instruction) documentation() string {
   577  	if len(inst.Desc.Authored.Paragraphs) > 0 {
   578  		return inst.Desc.Authored.Paragraphs[0].Text
   579  	}
   580  	return inst.Title
   581  }
   582  
   583  // asmTemplateToString flattens an AsmTemplate to its text.
   584  func asmTemplateToString(t xmlspec.AsmTemplate) string {
   585  	var b strings.Builder
   586  	for _, ta := range t.TextA {
   587  		b.WriteString(ta.Value)
   588  	}
   589  	return b.String()
   590  }
   591
View as plain text