godefs.go

     1  // Copyright 2025 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"fmt"
     9  	"log"
    10  	"regexp"
    11  	"slices"
    12  	"strconv"
    13  	"strings"
    14  	"unicode"
    15  
    16  	"simd/archsimd/_gen/unify"
    17  )
    18  
    19  type Operation struct {
    20  	rawOperation
    21  
    22  	// Go is the Go method name of this operation.
    23  	//
    24  	// It is derived from the raw Go method name by adding optional suffixes.
    25  	// Currently, "Masked" is the only suffix.
    26  	Go string
    27  
    28  	// Documentation is the doc string for this API.
    29  	//
    30  	// It is computed from the raw documentation:
    31  	//
    32  	// - "NAME" is replaced by the Go method name.
    33  	//
    34  	// - For masked operation, a sentence about masking is added.
    35  	Documentation string
    36  
    37  	// In is the sequence of parameters to the Go method.
    38  	//
    39  	// For masked operations, this will have the mask operand appended.
    40  	In []Operand
    41  }
    42  
    43  // rawOperation is the unifier representation of an [Operation]. It is
    44  // translated into a more parsed form after unifier decoding.
    45  type rawOperation struct {
    46  	Go string // Base Go method name
    47  
    48  	GoArch       string  // GOARCH for this definition
    49  	Asm          string  // Assembly mnemonic
    50  	OperandOrder *string // optional Operand order for better Go declarations
    51  	// Optional tag to indicate this operation is paired with special generic->machine ssa lowering rules.
    52  	// Should be paired with special templates in gen_simdrules.go
    53  	SpecialLower *string
    54  
    55  	In              []Operand // Parameters
    56  	InVariant       []Operand // Optional parameters
    57  	Out             []Operand // Results
    58  	MemFeatures     *string   // The memory operand feature this operation supports
    59  	MemFeaturesData *string   // Additional data associated with MemFeatures
    60  	Commutative     bool      // Commutativity
    61  	CPUFeature      string    // CPUID/Has* feature name
    62  	Zeroing         *bool     // nil => use asm suffix ".Z"; false => do not use asm suffix ".Z"
    63  	Documentation   *string   // Documentation will be appended to the stubs comments.
    64  	AddDoc          *string   // Additional doc to be appended.
    65  	// ConstMask is a hack to reduce the size of defs the user writes for const-immediate
    66  	// If present, it will be copied to [In[0].Const].
    67  	ConstImm *string
    68  	// NameAndSizeCheck is used to check [BWDQ] maps to (8|16|32|64) elemBits.
    69  	NameAndSizeCheck *bool
    70  	// If non-nil, all generation in gen_simdTypes.go and gen_intrinsics will be skipped.
    71  	NoTypes *string
    72  	// If non-nil, all generation in gen_simdGenericOps and gen_simdrules will be skipped.
    73  	NoGenericOps *string
    74  	// If non-nil, this string will be attached to the machine ssa op name.  E.g. "const"
    75  	SSAVariant *string
    76  	// If true, do not emit method declarations, generic ops, or intrinsics for masked variants
    77  	// DO emit the architecture-specific opcodes and optimizations.
    78  	HideMaskMethods *bool
    79  }
    80  
    81  func (o *Operation) IsMasked() bool {
    82  	if len(o.InVariant) == 0 {
    83  		return false
    84  	}
    85  	if len(o.InVariant) == 1 && o.InVariant[0].Class == "mask" {
    86  		return true
    87  	}
    88  	panic(fmt.Errorf("unknown inVariant"))
    89  }
    90  
    91  func (o *Operation) SkipMaskedMethod() bool {
    92  	if o.HideMaskMethods == nil {
    93  		return false
    94  	}
    95  	if *o.HideMaskMethods && o.IsMasked() {
    96  		return true
    97  	}
    98  	return false
    99  }
   100  
   101  var reForName = regexp.MustCompile(`\bNAME\b`)
   102  
   103  func (o *Operation) DecodeUnified(v *unify.Value) error {
   104  	if err := v.Decode(&o.rawOperation); err != nil {
   105  		return err
   106  	}
   107  
   108  	isMasked := o.IsMasked()
   109  
   110  	// Compute full Go method name.
   111  	o.Go = o.rawOperation.Go
   112  	if isMasked {
   113  		o.Go += "Masked"
   114  	}
   115  
   116  	// Compute doc string.
   117  	if o.rawOperation.Documentation != nil {
   118  		o.Documentation = *o.rawOperation.Documentation
   119  	} else {
   120  		o.Documentation = "// UNDOCUMENTED"
   121  	}
   122  	o.Documentation = reForName.ReplaceAllString(o.Documentation, o.Go)
   123  	if isMasked {
   124  		o.Documentation += "\n//\n// This operation is applied selectively under a write mask."
   125  		// Suppress generic op and method declaration for exported methods, if a mask is present.
   126  		if unicode.IsUpper([]rune(o.Go)[0]) {
   127  			trueVal := "true"
   128  			o.NoGenericOps = &trueVal
   129  			o.NoTypes = &trueVal
   130  		}
   131  	}
   132  	if o.rawOperation.AddDoc != nil {
   133  		o.Documentation += "\n" + reForName.ReplaceAllString(*o.rawOperation.AddDoc, o.Go)
   134  	}
   135  
   136  	o.In = append(o.rawOperation.In, o.rawOperation.InVariant...)
   137  
   138  	// For down conversions, the high elements are zeroed if the result has more elements.
   139  	// TODO: we should encode this logic in the YAML file, instead of hardcoding it here.
   140  	if len(o.In) > 0 && len(o.Out) > 0 {
   141  		inLanes := o.In[0].Lanes
   142  		outLanes := o.Out[0].Lanes
   143  		if inLanes != nil && outLanes != nil && *inLanes < *outLanes {
   144  			if (strings.Contains(o.Go, "Saturate") || strings.Contains(o.Go, "Truncate")) &&
   145  				!strings.Contains(o.Go, "Concat") {
   146  				o.Documentation += "\n// Results are packed to low elements in the returned vector, its upper elements are zeroed."
   147  			}
   148  		}
   149  	}
   150  
   151  	return nil
   152  }
   153  
   154  func (o *Operation) VectorWidth() int {
   155  	out := o.Out[0]
   156  	if out.Class == "vreg" {
   157  		return *out.Bits
   158  	} else if out.Class == "greg" || out.Class == "mask" {
   159  		for i := range o.In {
   160  			if o.In[i].Class == "vreg" {
   161  				return *o.In[i].Bits
   162  			}
   163  		}
   164  	}
   165  	panic(fmt.Errorf("Figure out what the vector width is for %v and implement it", *o))
   166  }
   167  
   168  // Right now simdgen computes the machine op name for most instructions
   169  // as $Name$OutputSize, by this denotation, these instructions are "overloaded".
   170  // for example:
   171  // (Uint16x8) ConvertToInt8
   172  // (Uint16x16) ConvertToInt8
   173  // are both VPMOVWB128.
   174  // To make them distinguishable we need to append the input size to them as well.
   175  // TODO: document them well in the generated code.
   176  var demotingConvertOps = map[string]bool{
   177  	"VPMOVQD128": true, "VPMOVSQD128": true, "VPMOVUSQD128": true, "VPMOVQW128": true, "VPMOVSQW128": true,
   178  	"VPMOVUSQW128": true, "VPMOVDW128": true, "VPMOVSDW128": true, "VPMOVUSDW128": true, "VPMOVQB128": true,
   179  	"VPMOVSQB128": true, "VPMOVUSQB128": true, "VPMOVDB128": true, "VPMOVSDB128": true, "VPMOVUSDB128": true,
   180  	"VPMOVWB128": true, "VPMOVSWB128": true, "VPMOVUSWB128": true,
   181  	"VPMOVQDMasked128": true, "VPMOVSQDMasked128": true, "VPMOVUSQDMasked128": true, "VPMOVQWMasked128": true, "VPMOVSQWMasked128": true,
   182  	"VPMOVUSQWMasked128": true, "VPMOVDWMasked128": true, "VPMOVSDWMasked128": true, "VPMOVUSDWMasked128": true, "VPMOVQBMasked128": true,
   183  	"VPMOVSQBMasked128": true, "VPMOVUSQBMasked128": true, "VPMOVDBMasked128": true, "VPMOVSDBMasked128": true, "VPMOVUSDBMasked128": true,
   184  	"VPMOVWBMasked128": true, "VPMOVSWBMasked128": true, "VPMOVUSWBMasked128": true,
   185  }
   186  
   187  func machineOpName(maskType maskShape, gOp Operation) string {
   188  	asm := gOp.Asm
   189  	if maskType == OneMask {
   190  		asm += "Masked"
   191  	}
   192  	asm = fmt.Sprintf("%s%d", asm, gOp.VectorWidth())
   193  	if gOp.SSAVariant != nil {
   194  		asm += *gOp.SSAVariant
   195  	}
   196  	if demotingConvertOps[asm] {
   197  		// Need to append the size of the source as well.
   198  		// TODO: should be "%sto%d".
   199  		asm = fmt.Sprintf("%s_%d", asm, *gOp.In[0].Bits)
   200  	}
   201  	return asm
   202  }
   203  
   204  func compareStringPointers(x, y *string) int {
   205  	if x != nil && y != nil {
   206  		return compareNatural(*x, *y)
   207  	}
   208  	if x == nil && y == nil {
   209  		return 0
   210  	}
   211  	if x == nil {
   212  		return -1
   213  	}
   214  	return 1
   215  }
   216  
   217  func compareIntPointers(x, y *int) int {
   218  	if x != nil && y != nil {
   219  		return *x - *y
   220  	}
   221  	if x == nil && y == nil {
   222  		return 0
   223  	}
   224  	if x == nil {
   225  		return -1
   226  	}
   227  	return 1
   228  }
   229  
   230  func compareOperations(x, y Operation) int {
   231  	if c := compareNatural(x.Go, y.Go); c != 0 {
   232  		return c
   233  	}
   234  	xIn, yIn := x.In, y.In
   235  
   236  	if len(xIn) > len(yIn) && xIn[len(xIn)-1].Class == "mask" {
   237  		xIn = xIn[:len(xIn)-1]
   238  	} else if len(xIn) < len(yIn) && yIn[len(yIn)-1].Class == "mask" {
   239  		yIn = yIn[:len(yIn)-1]
   240  	}
   241  
   242  	if len(xIn) < len(yIn) {
   243  		return -1
   244  	}
   245  	if len(xIn) > len(yIn) {
   246  		return 1
   247  	}
   248  	if len(x.Out) < len(y.Out) {
   249  		return -1
   250  	}
   251  	if len(x.Out) > len(y.Out) {
   252  		return 1
   253  	}
   254  	for i := range xIn {
   255  		ox, oy := &xIn[i], &yIn[i]
   256  		if c := compareOperands(ox, oy); c != 0 {
   257  			return c
   258  		}
   259  	}
   260  	return 0
   261  }
   262  
   263  func compareOperands(x, y *Operand) int {
   264  	if c := compareNatural(x.Class, y.Class); c != 0 {
   265  		return c
   266  	}
   267  	if x.Class == "immediate" {
   268  		return compareStringPointers(x.ImmOffset, y.ImmOffset)
   269  	} else {
   270  		if c := compareStringPointers(x.Base, y.Base); c != 0 {
   271  			return c
   272  		}
   273  		if c := compareIntPointers(x.ElemBits, y.ElemBits); c != 0 {
   274  			return c
   275  		}
   276  		if c := compareIntPointers(x.Bits, y.Bits); c != 0 {
   277  			return c
   278  		}
   279  		return 0
   280  	}
   281  }
   282  
   283  type Operand struct {
   284  	Class string // One of "mask", "immediate", "vreg", "greg", and "mem"
   285  
   286  	Go     *string // Go type of this operand
   287  	AsmPos int     // Position of this operand in the assembly instruction
   288  
   289  	Base     *string // Base Go type ("int", "uint", "float")
   290  	ElemBits *int    // Element bit width
   291  	Bits     *int    // Total vector bit width
   292  
   293  	Const *string // Optional constant value for immediates.
   294  	// Optional immediate arg offsets. If this field is non-nil,
   295  	// This operand will be an immediate operand:
   296  	// The compiler will right-shift the user-passed value by ImmOffset and set it as the AuxInt
   297  	// field of the operation.
   298  	ImmOffset *string
   299  	Name      *string // optional name in the Go intrinsic declaration
   300  	Lanes     *int    // *Lanes equals Bits/ElemBits except for scalars, when *Lanes == 1
   301  	// TreatLikeAScalarOfSize means only the lower $TreatLikeAScalarOfSize bits of the vector
   302  	// is used, so at the API level we can make it just a scalar value of this size; Then we
   303  	// can overwrite it to a vector of the right size during intrinsics stage.
   304  	TreatLikeAScalarOfSize *int
   305  	// If non-nil, it means the [Class] field is overwritten here, right now this is used to
   306  	// overwrite the results of AVX2 compares to masks.
   307  	OverwriteClass *string
   308  	// If non-nil, it means the [Base] field is overwritten here. This field exist solely
   309  	// because Intel's XED data is inconsistent. e.g. VANDNP[SD] marks its operand int.
   310  	OverwriteBase *string
   311  	// If non-nil, it means the [ElementBits] field is overwritten. This field exist solely
   312  	// because Intel's XED data is inconsistent. e.g. AVX512 VPMADDUBSW marks its operand
   313  	// elemBits 16, which should be 8.
   314  	OverwriteElementBits *int
   315  	// FixedReg is the name of the fixed registers
   316  	FixedReg *string
   317  }
   318  
   319  // isDigit returns true if the byte is an ASCII digit.
   320  func isDigit(b byte) bool {
   321  	return b >= '0' && b <= '9'
   322  }
   323  
   324  // compareNatural performs a "natural sort" comparison of two strings.
   325  // It compares non-digit sections lexicographically and digit sections
   326  // numerically.  In the case of string-unequal "equal" strings like
   327  // "a01b" and "a1b", strings.Compare breaks the tie.
   328  //
   329  // It returns:
   330  //
   331  //	-1 if s1 < s2
   332  //	 0 if s1 == s2
   333  //	+1 if s1 > s2
   334  func compareNatural(s1, s2 string) int {
   335  	i, j := 0, 0
   336  	len1, len2 := len(s1), len(s2)
   337  
   338  	for i < len1 && j < len2 {
   339  		// Find a non-digit segment or a number segment in both strings.
   340  		if isDigit(s1[i]) && isDigit(s2[j]) {
   341  			// Number segment comparison.
   342  			numStart1 := i
   343  			for i < len1 && isDigit(s1[i]) {
   344  				i++
   345  			}
   346  			num1, _ := strconv.Atoi(s1[numStart1:i])
   347  
   348  			numStart2 := j
   349  			for j < len2 && isDigit(s2[j]) {
   350  				j++
   351  			}
   352  			num2, _ := strconv.Atoi(s2[numStart2:j])
   353  
   354  			if num1 < num2 {
   355  				return -1
   356  			}
   357  			if num1 > num2 {
   358  				return 1
   359  			}
   360  			// If numbers are equal, continue to the next segment.
   361  		} else {
   362  			// Non-digit comparison.
   363  			if s1[i] < s2[j] {
   364  				return -1
   365  			}
   366  			if s1[i] > s2[j] {
   367  				return 1
   368  			}
   369  			i++
   370  			j++
   371  		}
   372  	}
   373  
   374  	// deal with a01b vs a1b; there needs to be an order.
   375  	return strings.Compare(s1, s2)
   376  }
   377  
   378  const generatedHeader = `// Code generated by 'simdgen -o godefs -goroot $GOROOT -xedPath $XED_PATH go.yaml types.yaml categories.yaml'; DO NOT EDIT.
   379  `
   380  
   381  func writeGoDefs(path string, cl unify.Closure) error {
   382  	// TODO: Merge operations with the same signature but multiple
   383  	// implementations (e.g., SSE vs AVX)
   384  	var ops []Operation
   385  	for def := range cl.All() {
   386  		var op Operation
   387  		if !def.Exact() {
   388  			continue
   389  		}
   390  		if err := def.Decode(&op); err != nil {
   391  			log.Println(err.Error())
   392  			log.Println(def)
   393  			continue
   394  		}
   395  		// TODO: verify that this is safe.
   396  		op.sortOperand()
   397  		op.adjustAsm()
   398  		ops = append(ops, op)
   399  	}
   400  	slices.SortFunc(ops, compareOperations)
   401  	// The parsed XED data might contain duplicates, like
   402  	// 512 bits VPADDP.
   403  	deduped := dedup(ops)
   404  	slices.SortFunc(deduped, compareOperations)
   405  
   406  	if *Verbose {
   407  		log.Printf("dedup len: %d\n", len(ops))
   408  	}
   409  	var err error
   410  	if err = overwrite(deduped); err != nil {
   411  		return err
   412  	}
   413  	if *Verbose {
   414  		log.Printf("dedup len: %d\n", len(deduped))
   415  	}
   416  	if !*FlagNoDedup {
   417  		// TODO: This can hide mistakes in the API definitions, especially when
   418  		// multiple patterns result in the same API unintentionally. Make it stricter.
   419  		if deduped, err = dedupGodef(deduped); err != nil {
   420  			return err
   421  		}
   422  	}
   423  	if *Verbose {
   424  		log.Printf("dedup len: %d\n", len(deduped))
   425  	}
   426  	if !*FlagNoConstImmPorting {
   427  		if err = copyConstImm(deduped); err != nil {
   428  			return err
   429  		}
   430  	}
   431  	if *Verbose {
   432  		log.Printf("dedup len: %d\n", len(deduped))
   433  	}
   434  	reportXEDInconsistency(deduped)
   435  	typeMap := parseSIMDTypes(deduped)
   436  
   437  	formatWriteAndClose(writeSIMDTypes(typeMap), path, "src/"+simdPackage+"/types_amd64.go")
   438  	formatWriteAndClose(writeSIMDFeatures(deduped), path, "src/"+simdPackage+"/cpu.go")
   439  	f, fI := writeSIMDStubs(deduped, typeMap)
   440  	formatWriteAndClose(f, path, "src/"+simdPackage+"/ops_amd64.go")
   441  	formatWriteAndClose(fI, path, "src/"+simdPackage+"/ops_internal_amd64.go")
   442  	formatWriteAndClose(writeSIMDIntrinsics(deduped, typeMap), path, "src/cmd/compile/internal/ssagen/simdintrinsics.go")
   443  	formatWriteAndClose(writeSIMDGenericOps(deduped), path, "src/cmd/compile/internal/ssa/_gen/simdgenericOps.go")
   444  	formatWriteAndClose(writeSIMDMachineOps(deduped), path, "src/cmd/compile/internal/ssa/_gen/simdAMD64ops.go")
   445  	formatWriteAndClose(writeSIMDSSA(deduped), path, "src/cmd/compile/internal/amd64/simdssa.go")
   446  	writeAndClose(writeSIMDRules(deduped).Bytes(), path, "src/cmd/compile/internal/ssa/_gen/simdAMD64.rules")
   447  
   448  	return nil
   449  }
   450
View as plain text