emit.go

     1  // Copyright 2026 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package sve
     6  
     7  import (
     8  	"cmp"
     9  	"fmt"
    10  	"log"
    11  	"slices"
    12  	"strings"
    13  
    14  	"simd/archsimd/_gen/unify"
    15  )
    16  
    17  // asComment wraps text into // comment lines of at most width columns.
    18  func asComment(text string, width int) string {
    19  	text = strings.TrimSpace(text)
    20  	text = strings.ReplaceAll(text, "&amp;", "&")
    21  	text = strings.ReplaceAll(text, "\n", " ")
    22  	words := strings.Fields(text)
    23  	var lines []string
    24  	line := ""
    25  	for _, w := range words {
    26  		if line != "" {
    27  			line += " "
    28  		}
    29  		line += w
    30  		if len(line) >= width {
    31  			lines = append(lines, "// "+line)
    32  			line = ""
    33  		}
    34  	}
    35  	if line != "" {
    36  		lines = append(lines, "// "+line)
    37  	}
    38  	return strings.Join(lines, "\n")
    39  }
    40  
    41  // mixedWidthLogged dedupes the mixed-element-width warning by mnemonic, so a
    42  // conversion family with many encodings logs once per generate run.
    43  var mixedWidthLogged = map[string]bool{}
    44  
    45  // emit renders an operand as a unify value. Z-vectors and predicates are
    46  // scalable (a base type and per-operand element width, no fixed bits/lanes);
    47  // mem, immediate and special operands are opaque (class and position only).
    48  func (op *Operand) emit() *unify.Value {
    49  	var db unify.DefBuilder
    50  	db.Add("class", unify.NewValue(unify.NewStringExact(op.Class)))
    51  	if op.BaseType != "" {
    52  		db.Add("base", unify.NewValue(unify.NewStringExact(op.BaseType)))
    53  	}
    54  	switch {
    55  	case op.Bits > 0:
    56  		// A fixed-width SIMD&FP scalar (OperandVFP): a real bit width and lanes.
    57  		db.Add("bits", unify.NewValue(unify.NewStringExact(fmt.Sprint(op.Bits))))
    58  		if op.Lanes > 0 {
    59  			db.Add("lanes", unify.NewValue(unify.NewStringExact(fmt.Sprint(op.Lanes))))
    60  		}
    61  	case op.Class == "vreg" || op.Class == "mask":
    62  		// SVE vectors and predicates are scalable: no fixed total bit width.
    63  		// The literal "scalable" both marks that and, because it conflicts with
    64  		// any numeric bits, keeps these operands from unifying with the
    65  		// fixed-width (NEON/AVX) types that share types.yaml.
    66  		db.Add("bits", unify.NewValue(unify.NewStringExact("scalable")))
    67  	}
    68  	if op.ElemBits > 0 {
    69  		db.Add("elemBits", unify.NewValue(unify.NewStringExact(fmt.Sprint(op.ElemBits))))
    70  	}
    71  	if op.Predication != "" {
    72  		// "M" (merging) or "Z" (zeroing) for a governing predicate. Some SVE
    73  		// instructions support only one; this records which.
    74  		db.Add("predication", unify.NewValue(unify.NewStringExact(op.Predication)))
    75  	}
    76  	if op.governing {
    77  		// This operand is a governing predicate.
    78  		db.Add("governing", unify.NewValue(unify.NewStringExact("true")))
    79  	}
    80  	if op.isList {
    81  		// This register came from a single-register list ("{ <Zt>.<T> }"), a
    82  		// distinct assembler encoding from a bare register.
    83  		db.Add("listNumber", unify.NewValue(unify.NewStringExact("0")))
    84  	}
    85  	if op.regName != "" {
    86  		// The assembly template's register symbol, e.g. "Zdn", "Zn", "Pg".
    87  		db.Add("regName", unify.NewValue(unify.NewStringExact(op.regName)))
    88  	}
    89  	// The symbol this operand has in each predicated encoding, indexed to
    90  	// match the def's inVariant. The symbols can differ from the unpredicated
    91  	// ones to predicated ones:
    92  	// ADD <Zd>, <Zn>, <Zm> unpredicated
    93  	// ADD <Zdn>, <Pg>/M, <Zdn>, <Zm> predicated
    94  	//
    95  	// [groupPredicationForms] folds the two into one def.
    96  	// simdgen needs these symbols to recognize resultInArg0.
    97  	names := make([]*unify.Value, len(op.predRegName))
    98  	for i, n := range op.predRegName {
    99  		names[i] = unify.NewValue(unify.NewStringExact(n))
   100  	}
   101  	db.Add("predRegName", unify.NewValue(unify.NewTuple(names...)))
   102  	db.Add("asmPos", unify.NewValue(unify.NewStringExact(fmt.Sprint(op.AsmPos))))
   103  	return unify.NewValue(db.Build())
   104  }
   105  
   106  // pickRegNames returns operand idx's symbol in each predicated encoding, in
   107  // variant order. The encodings passed [sameOperandShape], so idx addresses the
   108  // matching operand in every one of them.
   109  func pickRegNames(variants []predVariant, idx int, sel func(predVariant) []string) []string {
   110  	if len(variants) == 0 {
   111  		return nil
   112  	}
   113  	out := make([]string, len(variants))
   114  	for i, pv := range variants {
   115  		names := sel(pv)
   116  		if idx >= len(names) {
   117  			panic(fmt.Sprintf("operand %d has no counterpart in predicated encoding %d", idx, i))
   118  		}
   119  		out[i] = names[idx]
   120  	}
   121  	return out
   122  }
   123  
   124  // emitOne emits a single instruction def from a fully-instantiated operand list:
   125  // the destination is the output, every other operand (including a governing
   126  // predicate) is a literal input.
   127  //
   128  // An SVE predicate is a mandatory input, not an optional AVX-512-style K-mask, so
   129  // it goes in `in`; inVariant is emitted empty just to satisfy the types.yaml schema.
   130  func (inst *Instruction) emitOne(asm string, ops []Operand, widthAgnostic bool) *unify.Value {
   131  	var db unify.DefBuilder
   132  	db.Add("asm", unify.NewValue(unify.NewStringExact(asm)))
   133  	db.Add("goarch", unify.NewValue(unify.NewStringExact("arm64")))
   134  	// The operation's feature level is the floor across its encodings: an
   135  	// operation whose predicated sibling is baseline SVE is available on SVE
   136  	// even when its unpredicated carrier needs SVE2 — the carrier is then a
   137  	// feature-gated upgrade, recorded as unpredCpuFeature for the rules.
   138  	feature := inst.cpuFeature()
   139  	unpred := ""
   140  	for _, pv := range inst.predVariants {
   141  		if pv.cpuFeature == "SVE" && feature == "SVE2" {
   142  			unpred = feature
   143  			feature = pv.cpuFeature
   144  		}
   145  	}
   146  	db.Add("cpuFeature", unify.NewValue(unify.NewStringExact(feature)))
   147  	if unpred != "" {
   148  		db.Add("unpredCPUFeature", unify.NewValue(unify.NewStringExact(unpred)))
   149  	}
   150  	if doc := inst.documentation(); doc != "" {
   151  		db.Add("details", unify.NewValue(unify.NewStringExact(asComment(doc, 80))))
   152  	}
   153  	if widthAgnostic {
   154  		db.Add("widthAgnostic", unify.NewValue(unify.NewStringExact("true")))
   155  	}
   156  
   157  	// One def can describe several encodings of one operation, grouped by
   158  	// [groupPredicationForms] or [groupPredicatedOnly], so each operand also
   159  	// carries the symbol it has in each predicated encoding. The symbols are
   160  	// matched up in template order, so they must be attached before the sort
   161  	// below reorders the inputs.
   162  	var inOps, outOps []Operand
   163  	var outIdx, inIdx int
   164  	for _, op := range ops {
   165  		switch {
   166  		case op.governing:
   167  			// The governing predicate is the operand the paired encodings differ in, so
   168  			// it is not one of the symbols they are matched up by.
   169  			inOps = append(inOps, op)
   170  		case op.role == "destination":
   171  			op.predRegName = pickRegNames(inst.predVariants, outIdx, func(pv predVariant) []string { return pv.outRegNames })
   172  			outIdx++
   173  			outOps = append(outOps, op)
   174  		default:
   175  			op.predRegName = pickRegNames(inst.predVariants, inIdx, func(pv predVariant) []string { return pv.inRegNames })
   176  			inIdx++
   177  			inOps = append(inOps, op)
   178  		}
   179  	}
   180  	priority := map[string]int{"immediate": 0, "vreg": 1, "greg": 1, "memory": 1, "mask": 2}
   181  	slices.SortStableFunc(inOps, func(a, b Operand) int {
   182  		pa := priority[a.Class]
   183  		pb := priority[b.Class]
   184  		if pa != pb {
   185  			return cmp.Compare(pa, pb)
   186  		}
   187  		return cmp.Compare(a.AsmPos, b.AsmPos)
   188  	})
   189  
   190  	var ins, outs []*unify.Value
   191  	for i := range inOps {
   192  		ins = append(ins, inOps[i].emit())
   193  	}
   194  	for i := range outOps {
   195  		outs = append(outs, outOps[i].emit())
   196  	}
   197  	db.Add("in", unify.NewValue(unify.NewTuple(ins...)))
   198  	var inVar []*unify.Value
   199  	for _, pv := range inst.predVariants {
   200  		// The governing predicate of the paired predicated encoding.
   201  		var pdb unify.DefBuilder
   202  		pdb.Add("class", unify.NewValue(unify.NewStringExact("mask")))
   203  		pdb.Add("bits", unify.NewValue(unify.NewStringExact("scalable")))
   204  		pdb.Add("predication", unify.NewValue(unify.NewStringExact(pv.quals)))
   205  		pdb.Add("asmPos", unify.NewValue(unify.NewStringExact(fmt.Sprint(pv.predAsmPos))))
   206  		inVar = append(inVar, unify.NewValue(pdb.Build()))
   207  	}
   208  	db.Add("inVariant", unify.NewValue(unify.NewTuple(inVar...)))
   209  	db.Add("out", unify.NewValue(unify.NewTuple(outs...)))
   210  	return unify.NewValue(db.Build())
   211  }
   212  
   213  // emitAll emits the unify defs for this instruction — the concrete variants of
   214  // the source template. See classify (used by both emitAll and analyze) for the
   215  // full disposition.
   216  func (inst *Instruction) emitAll() []*unify.Value {
   217  	// emitAll doesn't check the anomalies, that would be done by
   218  	// a full-corpus test in analyze_test.go.
   219  	defs, _, _ := inst.classify()
   220  	return defs
   221  }
   222  
   223  // lookup returns the element width for the given size key in a table.
   224  func lookup(rows []arngRow, size string) (int, bool) {
   225  	for _, r := range rows {
   226  		if r.size == size {
   227  			return r.bits, true
   228  		}
   229  	}
   230  	return 0, false
   231  }
   232  
   233  // emitVariants emits one def per (integer signedness × arrangement row ×
   234  // predication). Each operand's element width comes from its own arrangement
   235  // symbol's table, keyed by the shared size field, so uniform and non-uniform
   236  // (widening/narrowing) forms are handled the same way; operands with no
   237  // arrangement stay unsized. Each operand's base type is resolved per operand
   238  // (laneIsFloat) — floating-point lanes are always "float", integer lanes take
   239  // the signedness of the current variant — so this naturally extends to
   240  // conversions, whose lanes will differ.
   241  func (inst *Instruction) emitVariants(template []Operand) []*unify.Value {
   242  	asm := inst.goOpPrefix() + inst.mnemonic()
   243  
   244  	links := arngLinks(template)
   245  	tables := map[string][]arngRow{}
   246  	for _, l := range links {
   247  		tables[l] = inst.resolveArrangementTable(l)
   248  	}
   249  
   250  	// Rows to iterate: the primary (destination-first) symbol's size keys, or a
   251  	// single pass when there is no variable arrangement.
   252  	var sizes []string
   253  	if len(links) > 0 {
   254  		for _, r := range tables[links[0]] {
   255  			sizes = append(sizes, r.size)
   256  		}
   257  	} else {
   258  		sizes = []string{""}
   259  	}
   260  
   261  	signs := inst.integerSignedness(template)
   262  
   263  	// Governing-predicate qualifier(s) for this template: /M, /Z, both (a /<ZM>
   264  	// encoding), or a single no-op pass when there is no governing predicate.
   265  	preds := predicationVariants(template)
   266  
   267  	// A bitwise operation with no variable arrangement is width-agnostic: the
   268  	// encoding is written .D, but any element view of it computes the same
   269  	// bits, and its predicated sibling is a per-<T> encoding. Emit a def per
   270  	// element width so every Go type gets the API, marked so that simdgen
   271  	// collapses the unpredicated machine op back to the single .D instruction.
   272  	widths := []int{0}
   273  	widthAgnostic := len(links) == 0 && inst.bitwise()
   274  	if widthAgnostic {
   275  		widths = []int{8, 16, 32, 64}
   276  	}
   277  
   278  	var defs []*unify.Value
   279  	for _, sign := range signs {
   280  		for _, size := range sizes {
   281  			ops := make([]Operand, len(template))
   282  			copy(ops, template)
   283  			skip := false
   284  			for i := range ops {
   285  				eb := ops[i].fixedElem
   286  				if ops[i].fixedBits > 0 {
   287  					// SIMD&FP scalar with a fixed width letter (<Dd> = 64), the
   288  					// same for every arrangement row.
   289  					eb = ops[i].fixedBits
   290  				} else if l := ops[i].arngLink; l != "" {
   291  					b, ok := lookup(tables[l], size)
   292  					if !ok {
   293  						// This operand's symbol has no element for this size
   294  						// (e.g. a RESERVED row on one side of a widening op).
   295  						skip = true
   296  						break
   297  					}
   298  					eb = b
   299  				}
   300  				base := sign
   301  				if inst.laneIsFloat(&ops[i]) {
   302  					base = "float"
   303  					if eb > 0 && eb < 16 {
   304  						// No half/quarter-word floating-point Go types.
   305  						skip = true
   306  						break
   307  					}
   308  				}
   309  				ops[i].instantiate(base, eb)
   310  			}
   311  			if skip {
   312  				continue
   313  			}
   314  			for _, pred := range preds {
   315  				variant := make([]Operand, len(ops))
   316  				copy(variant, ops)
   317  				elem := 0
   318  				mixedWidths := false
   319  				for i := range variant {
   320  					if variant[i].Class == "vreg" && variant[i].ElemBits > 0 {
   321  						if elem == 0 {
   322  							elem = variant[i].ElemBits
   323  						} else if variant[i].ElemBits != elem {
   324  							mixedWidths = true
   325  						}
   326  					}
   327  				}
   328  				for i := range variant {
   329  					if variant[i].Class != "mask" {
   330  						continue
   331  					}
   332  					if variant[i].governing {
   333  						variant[i].Predication = pred
   334  					}
   335  					if variant[i].ElemBits == 0 {
   336  						// This predicate doesn't come with an arrangement (which is usual).
   337  						// Get it from its peer data operand.
   338  						if mixedWidths && !mixedWidthLogged[inst.mnemonic()] {
   339  							mixedWidthLogged[inst.mnemonic()] = true
   340  							log.Printf("sve: %s: operands have mixed element widths; predicate width provisionally %d — derive esize from the pseudocode before generating an API from this def",
   341  								inst.mnemonic(), elem)
   342  						}
   343  						variant[i].ElemBits = elem
   344  					}
   345  				}
   346  				for _, w := range widths {
   347  					v := variant
   348  					if w > 0 {
   349  						v = make([]Operand, len(variant))
   350  						copy(v, variant)
   351  						for i := range v {
   352  							if v[i].Class == "vreg" || v[i].Class == "mask" {
   353  								v[i].ElemBits = w
   354  							}
   355  						}
   356  					}
   357  					defs = append(defs, inst.emitOne(asm, v, widthAgnostic))
   358  				}
   359  			}
   360  		}
   361  	}
   362  	return defs
   363  }
   364
View as plain text