generate_test.go

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // This file implements a custom generator to create various go/types
     6  // source files from the corresponding types2 files.
     7  
     8  package types_test
     9  
    10  import (
    11  	"bytes"
    12  	"flag"
    13  	"fmt"
    14  	"go/ast"
    15  	"go/format"
    16  	"go/parser"
    17  	"go/token"
    18  	"internal/diff"
    19  	"os"
    20  	"path/filepath"
    21  	"runtime"
    22  	"strings"
    23  	"testing"
    24  )
    25  
    26  var filesToWrite = flag.String("write", "", `go/types files to generate, or "all" for all files`)
    27  
    28  const (
    29  	srcDir = "/src/cmd/compile/internal/types2/"
    30  	dstDir = "/src/go/types/"
    31  )
    32  
    33  // TestGenerate verifies that generated files in go/types match their types2
    34  // counterpart. If -write is set, this test actually writes the expected
    35  // content to go/types; otherwise, it just compares with the existing content.
    36  func TestGenerate(t *testing.T) {
    37  	// If filesToWrite is set, write the generated content to disk.
    38  	// In the special case of "all", write all files in filemap.
    39  	write := *filesToWrite != ""
    40  	var files []string // files to process
    41  	if *filesToWrite != "" && *filesToWrite != "all" {
    42  		files = strings.Split(*filesToWrite, ",")
    43  	} else {
    44  		for file := range filemap {
    45  			files = append(files, file)
    46  		}
    47  	}
    48  
    49  	for _, filename := range files {
    50  		generate(t, filename, write)
    51  	}
    52  }
    53  
    54  func generate(t *testing.T, filename string, write bool) {
    55  	// parse src (cmd/compile/internal/types2)
    56  	srcFilename := filepath.FromSlash(runtime.GOROOT() + srcDir + filename)
    57  	file, err := parser.ParseFile(fset, srcFilename, nil, parser.ParseComments)
    58  	if err != nil {
    59  		t.Fatal(err)
    60  	}
    61  
    62  	// fix package name
    63  	file.Name.Name = strings.ReplaceAll(file.Name.Name, "types2", "types")
    64  
    65  	// rewrite AST as needed
    66  	if action := filemap[filename]; action != nil {
    67  		action(file)
    68  	}
    69  
    70  	// format AST
    71  	var buf bytes.Buffer
    72  	rel, _ := filepath.Rel(dstDir, srcDir)
    73  	fmt.Fprintf(&buf, "// Code generated by \"go test -run=Generate -write=all\"; DO NOT EDIT.\n")
    74  	fmt.Fprintf(&buf, "// Source: %s/%s\n\n", filepath.ToSlash(rel), filename)
    75  	if err := format.Node(&buf, fset, file); err != nil {
    76  		t.Fatal(err)
    77  	}
    78  	generatedContent := buf.Bytes()
    79  
    80  	// read dst (go/types)
    81  	dstFilename := filepath.FromSlash(runtime.GOROOT() + dstDir + filename)
    82  	onDiskContent, err := os.ReadFile(dstFilename)
    83  	if err != nil {
    84  		t.Fatalf("reading %q: %v", filename, err)
    85  	}
    86  
    87  	// compare on-disk dst with buffer generated from src.
    88  	if d := diff.Diff(filename+" (on disk in "+dstDir+")", onDiskContent, filename+" (generated from "+srcDir+")", generatedContent); d != nil {
    89  		if write {
    90  			t.Logf("applying change:\n%s", d)
    91  			if err := os.WriteFile(dstFilename, generatedContent, 0o644); err != nil {
    92  				t.Fatalf("writing %q: %v", filename, err)
    93  			}
    94  		} else {
    95  			t.Errorf("file on disk in %s is stale:\n%s", dstDir, d)
    96  		}
    97  	}
    98  }
    99  
   100  type action func(in *ast.File)
   101  
   102  var filemap = map[string]action{
   103  	"alias.go": nil,
   104  	"assignments.go": func(f *ast.File) {
   105  		renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`)
   106  		renameSelectorExprs(f, "syntax.Name->ast.Ident", "ident.Value->ident.Name", "ast.Pos->token.Pos") // must happen before renaming identifiers
   107  		renameIdents(f, "syntax->ast", "poser->positioner", "nopos->noposn")
   108  	},
   109  	"array.go":          nil,
   110  	"api_predicates.go": nil,
   111  	"basic.go":          nil,
   112  	"builtins.go": func(f *ast.File) {
   113  		renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`)
   114  		renameIdents(f, "syntax->ast")
   115  		renameSelectors(f, "ArgList->Args")
   116  		fixSelValue(f)
   117  		fixAtPosCall(f)
   118  	},
   119  	"builtins_test.go": func(f *ast.File) {
   120  		renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`, `"cmd/compile/internal/types2"->"go/types"`)
   121  		renameSelectorExprs(f, "syntax.Name->ast.Ident", "p.Value->p.Name") // must happen before renaming identifiers
   122  		renameIdents(f, "syntax->ast")
   123  	},
   124  	"chan.go":         nil,
   125  	"const.go":        fixTokenPos,
   126  	"context.go":      nil,
   127  	"context_test.go": nil,
   128  	"conversions.go":  nil,
   129  	"errors_test.go":  func(f *ast.File) { renameIdents(f, "nopos->noposn") },
   130  	"errsupport.go":   nil,
   131  	"gccgosizes.go":   nil,
   132  	"gcsizes.go":      func(f *ast.File) { renameIdents(f, "IsSyncAtomicAlign64->_IsSyncAtomicAlign64") },
   133  	"hilbert_test.go": func(f *ast.File) { renameImportPath(f, `"cmd/compile/internal/types2"->"go/types"`) },
   134  	"infer.go":        func(f *ast.File) { fixTokenPos(f); fixInferSig(f) },
   135  	"initorder.go":    nil,
   136  	// "initorder.go": fixErrErrorfCall, // disabled for now due to unresolved error_ use implications for gopls
   137  	"instantiate.go":      func(f *ast.File) { fixTokenPos(f); fixCheckErrorfCall(f) },
   138  	"instantiate_test.go": func(f *ast.File) { renameImportPath(f, `"cmd/compile/internal/types2"->"go/types"`) },
   139  	"lookup.go":           func(f *ast.File) { fixTokenPos(f) },
   140  	"main_test.go":        nil,
   141  	"map.go":              nil,
   142  	"mono.go": func(f *ast.File) {
   143  		fixTokenPos(f)
   144  		insertImportPath(f, `"go/ast"`)
   145  		renameSelectorExprs(f, "syntax.Expr->ast.Expr")
   146  	},
   147  	"named.go":  func(f *ast.File) { fixTokenPos(f); renameSelectors(f, "Trace->_Trace") },
   148  	"object.go": func(f *ast.File) { fixTokenPos(f); renameIdents(f, "NewTypeNameLazy->_NewTypeNameLazy") },
   149  	// TODO(gri) needs adjustments for TestObjectString - disabled for now
   150  	// "object_test.go": func(f *ast.File) { renameImportPath(f, `"cmd/compile/internal/types2"->"go/types"`) },
   151  	"objset.go": nil,
   152  	"operand.go": func(f *ast.File) {
   153  		insertImportPath(f, `"go/token"`)
   154  		renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`)
   155  		renameSelectorExprs(f,
   156  			"syntax.Pos->token.Pos", "syntax.LitKind->token.Token",
   157  			"syntax.IntLit->token.INT", "syntax.FloatLit->token.FLOAT",
   158  			"syntax.ImagLit->token.IMAG", "syntax.RuneLit->token.CHAR",
   159  			"syntax.StringLit->token.STRING") // must happen before renaming identifiers
   160  		renameIdents(f, "syntax->ast")
   161  	},
   162  	"package.go":       nil,
   163  	"pointer.go":       nil,
   164  	"predicates.go":    nil,
   165  	"scope.go":         func(f *ast.File) { fixTokenPos(f); renameIdents(f, "Squash->squash", "InsertLazy->_InsertLazy") },
   166  	"selection.go":     nil,
   167  	"sizes.go":         func(f *ast.File) { renameIdents(f, "IsSyncAtomicAlign64->_IsSyncAtomicAlign64") },
   168  	"slice.go":         nil,
   169  	"subst.go":         func(f *ast.File) { fixTokenPos(f); renameSelectors(f, "Trace->_Trace") },
   170  	"termlist.go":      nil,
   171  	"termlist_test.go": nil,
   172  	"tuple.go":         nil,
   173  	"typelists.go":     nil,
   174  	"typeset.go":       func(f *ast.File) { fixTokenPos(f); renameSelectors(f, "Trace->_Trace") },
   175  	"typeparam.go":     nil,
   176  	"typeterm_test.go": nil,
   177  	"typeterm.go":      nil,
   178  	"typestring.go":    nil,
   179  	"under.go":         nil,
   180  	"unify.go":         fixSprintf,
   181  	"universe.go":      fixGlobalTypVarDecl,
   182  	"util_test.go":     fixTokenPos,
   183  	"validtype.go":     func(f *ast.File) { fixTokenPos(f); renameSelectors(f, "Trace->_Trace") },
   184  }
   185  
   186  // TODO(gri) We should be able to make these rewriters more configurable/composable.
   187  //           For now this is a good starting point.
   188  
   189  // A renameMap maps old strings to new strings.
   190  type renameMap map[string]string
   191  
   192  // makeRenameMap returns a renameMap populates from renames entries of the form "from->to".
   193  func makeRenameMap(renames ...string) renameMap {
   194  	m := make(renameMap)
   195  	for _, r := range renames {
   196  		s := strings.Split(r, "->")
   197  		if len(s) != 2 {
   198  			panic("invalid rename entry: " + r)
   199  		}
   200  		m[s[0]] = s[1]
   201  	}
   202  	return m
   203  }
   204  
   205  // rename renames the given string s if a corresponding rename exists in m.
   206  func (m renameMap) rename(s *string) {
   207  	if r, ok := m[*s]; ok {
   208  		*s = r
   209  	}
   210  }
   211  
   212  // renameSel renames a selector expression of the form a.x to b.x (where a, b are identifiers)
   213  // if m contains the ("a.x" : "b.y") key-value pair.
   214  func (m renameMap) renameSel(n *ast.SelectorExpr) {
   215  	if a, _ := n.X.(*ast.Ident); a != nil {
   216  		a_x := a.Name + "." + n.Sel.Name
   217  		if r, ok := m[a_x]; ok {
   218  			b_y := strings.Split(r, ".")
   219  			if len(b_y) != 2 {
   220  				panic("invalid selector expression: " + r)
   221  			}
   222  			a.Name = b_y[0]
   223  			n.Sel.Name = b_y[1]
   224  		}
   225  	}
   226  }
   227  
   228  // renameIdents renames identifiers: each renames entry is of the form "from->to".
   229  // Note: This doesn't change the use of the identifiers in comments.
   230  func renameIdents(f *ast.File, renames ...string) {
   231  	m := makeRenameMap(renames...)
   232  	ast.Inspect(f, func(n ast.Node) bool {
   233  		switch n := n.(type) {
   234  		case *ast.Ident:
   235  			m.rename(&n.Name)
   236  			return false
   237  		}
   238  		return true
   239  	})
   240  }
   241  
   242  // renameSelectors is like renameIdents but only looks at selectors.
   243  func renameSelectors(f *ast.File, renames ...string) {
   244  	m := makeRenameMap(renames...)
   245  	ast.Inspect(f, func(n ast.Node) bool {
   246  		switch n := n.(type) {
   247  		case *ast.SelectorExpr:
   248  			m.rename(&n.Sel.Name)
   249  			return false
   250  		}
   251  		return true
   252  	})
   253  
   254  }
   255  
   256  // renameSelectorExprs is like renameIdents but only looks at selector expressions.
   257  // Each renames entry must be of the form "x.a->y.b".
   258  func renameSelectorExprs(f *ast.File, renames ...string) {
   259  	m := makeRenameMap(renames...)
   260  	ast.Inspect(f, func(n ast.Node) bool {
   261  		switch n := n.(type) {
   262  		case *ast.SelectorExpr:
   263  			m.renameSel(n)
   264  			return false
   265  		}
   266  		return true
   267  	})
   268  }
   269  
   270  // renameImportPath is like renameIdents but renames import paths.
   271  func renameImportPath(f *ast.File, renames ...string) {
   272  	m := makeRenameMap(renames...)
   273  	ast.Inspect(f, func(n ast.Node) bool {
   274  		switch n := n.(type) {
   275  		case *ast.ImportSpec:
   276  			if n.Path.Kind != token.STRING {
   277  				panic("invalid import path")
   278  			}
   279  			m.rename(&n.Path.Value)
   280  			return false
   281  		}
   282  		return true
   283  	})
   284  }
   285  
   286  // insertImportPath inserts the given import path.
   287  // There must be at least one import declaration present already.
   288  func insertImportPath(f *ast.File, path string) {
   289  	for _, d := range f.Decls {
   290  		if g, _ := d.(*ast.GenDecl); g != nil && g.Tok == token.IMPORT {
   291  			g.Specs = append(g.Specs, &ast.ImportSpec{Path: &ast.BasicLit{ValuePos: g.End(), Kind: token.STRING, Value: path}})
   292  			return
   293  		}
   294  	}
   295  	panic("no import declaration present")
   296  }
   297  
   298  // fixTokenPos changes imports of "cmd/compile/internal/syntax" to "go/token",
   299  // uses of syntax.Pos to token.Pos, and calls to x.IsKnown() to x.IsValid().
   300  func fixTokenPos(f *ast.File) {
   301  	m := makeRenameMap(`"cmd/compile/internal/syntax"->"go/token"`, "syntax.Pos->token.Pos", "IsKnown->IsValid")
   302  	ast.Inspect(f, func(n ast.Node) bool {
   303  		switch n := n.(type) {
   304  		case *ast.ImportSpec:
   305  			// rewrite import path "cmd/compile/internal/syntax" to "go/token"
   306  			if n.Path.Kind != token.STRING {
   307  				panic("invalid import path")
   308  			}
   309  			m.rename(&n.Path.Value)
   310  			return false
   311  		case *ast.SelectorExpr:
   312  			// rewrite syntax.Pos to token.Pos
   313  			m.renameSel(n)
   314  		case *ast.CallExpr:
   315  			// rewrite x.IsKnown() to x.IsValid()
   316  			if fun, _ := n.Fun.(*ast.SelectorExpr); fun != nil && len(n.Args) == 0 {
   317  				m.rename(&fun.Sel.Name)
   318  				return false
   319  			}
   320  		}
   321  		return true
   322  	})
   323  }
   324  
   325  // fixSelValue updates the selector x.Sel.Value to x.Sel.Name.
   326  func fixSelValue(f *ast.File) {
   327  	ast.Inspect(f, func(n ast.Node) bool {
   328  		switch n := n.(type) {
   329  		case *ast.SelectorExpr:
   330  			if n.Sel.Name == "Value" {
   331  				if selx, _ := n.X.(*ast.SelectorExpr); selx != nil && selx.Sel.Name == "Sel" {
   332  					n.Sel.Name = "Name"
   333  					return false
   334  				}
   335  			}
   336  		}
   337  		return true
   338  	})
   339  }
   340  
   341  // fixInferSig updates the Checker.infer signature to use a positioner instead of a token.Position
   342  // as first argument, renames the argument from "pos" to "posn", and updates a few internal uses of
   343  // "pos" to "posn" and "posn.Pos()" respectively.
   344  func fixInferSig(f *ast.File) {
   345  	ast.Inspect(f, func(n ast.Node) bool {
   346  		switch n := n.(type) {
   347  		case *ast.FuncDecl:
   348  			if n.Name.Name == "infer" {
   349  				// rewrite (pos token.Pos, ...) to (posn positioner, ...)
   350  				par := n.Type.Params.List[0]
   351  				if len(par.Names) == 1 && par.Names[0].Name == "pos" {
   352  					par.Names[0] = newIdent(par.Names[0].Pos(), "posn")
   353  					par.Type = newIdent(par.Type.Pos(), "positioner")
   354  					return true
   355  				}
   356  			}
   357  		case *ast.CallExpr:
   358  			if selx, _ := n.Fun.(*ast.SelectorExpr); selx != nil {
   359  				switch selx.Sel.Name {
   360  				case "renameTParams":
   361  					// rewrite check.renameTParams(pos, ... ) to check.renameTParams(posn.Pos(), ... )
   362  					if isIdent(n.Args[0], "pos") {
   363  						pos := n.Args[0].Pos()
   364  						fun := &ast.SelectorExpr{X: newIdent(pos, "posn"), Sel: newIdent(pos, "Pos")}
   365  						arg := &ast.CallExpr{Fun: fun, Lparen: pos, Args: nil, Ellipsis: token.NoPos, Rparen: pos}
   366  						n.Args[0] = arg
   367  						return false
   368  					}
   369  				case "addf":
   370  					// rewrite err.addf(pos, ...) to err.addf(posn, ...)
   371  					if isIdent(n.Args[0], "pos") {
   372  						pos := n.Args[0].Pos()
   373  						arg := newIdent(pos, "posn")
   374  						n.Args[0] = arg
   375  						return false
   376  					}
   377  				case "allowVersion":
   378  					// rewrite check.allowVersion(pos, ...) to check.allowVersion(posn, ...)
   379  					if isIdent(n.Args[0], "pos") {
   380  						pos := n.Args[0].Pos()
   381  						arg := newIdent(pos, "posn")
   382  						n.Args[0] = arg
   383  						return false
   384  					}
   385  				}
   386  			}
   387  		}
   388  		return true
   389  	})
   390  }
   391  
   392  // fixAtPosCall updates calls of the form atPos(x) to x.Pos() in argument lists of (check).dump calls.
   393  // TODO(gri) can we avoid this and just use atPos consistently in go/types and types2?
   394  func fixAtPosCall(f *ast.File) {
   395  	ast.Inspect(f, func(n ast.Node) bool {
   396  		switch n := n.(type) {
   397  		case *ast.CallExpr:
   398  			if selx, _ := n.Fun.(*ast.SelectorExpr); selx != nil && selx.Sel.Name == "dump" {
   399  				for i, arg := range n.Args {
   400  					if call, _ := arg.(*ast.CallExpr); call != nil {
   401  						// rewrite xxx.dump(..., atPos(x), ...) to xxx.dump(..., x.Pos(), ...)
   402  						if isIdent(call.Fun, "atPos") {
   403  							pos := call.Args[0].Pos()
   404  							fun := &ast.SelectorExpr{X: call.Args[0], Sel: newIdent(pos, "Pos")}
   405  							n.Args[i] = &ast.CallExpr{Fun: fun, Lparen: pos, Rparen: pos}
   406  							return false
   407  						}
   408  					}
   409  				}
   410  			}
   411  		}
   412  		return true
   413  	})
   414  }
   415  
   416  // fixErrErrorfCall updates calls of the form err.addf(obj, ...) to err.addf(obj.Pos(), ...).
   417  func fixErrErrorfCall(f *ast.File) {
   418  	ast.Inspect(f, func(n ast.Node) bool {
   419  		switch n := n.(type) {
   420  		case *ast.CallExpr:
   421  			if selx, _ := n.Fun.(*ast.SelectorExpr); selx != nil {
   422  				if isIdent(selx.X, "err") {
   423  					switch selx.Sel.Name {
   424  					case "errorf":
   425  						// rewrite err.addf(obj, ... ) to err.addf(obj.Pos(), ... )
   426  						if ident, _ := n.Args[0].(*ast.Ident); ident != nil && ident.Name == "obj" {
   427  							pos := n.Args[0].Pos()
   428  							fun := &ast.SelectorExpr{X: ident, Sel: newIdent(pos, "Pos")}
   429  							n.Args[0] = &ast.CallExpr{Fun: fun, Lparen: pos, Rparen: pos}
   430  							return false
   431  						}
   432  					}
   433  				}
   434  			}
   435  		}
   436  		return true
   437  	})
   438  }
   439  
   440  // fixCheckErrorfCall updates calls of the form check.errorf(pos, ...) to check.errorf(atPos(pos), ...).
   441  func fixCheckErrorfCall(f *ast.File) {
   442  	ast.Inspect(f, func(n ast.Node) bool {
   443  		switch n := n.(type) {
   444  		case *ast.CallExpr:
   445  			if selx, _ := n.Fun.(*ast.SelectorExpr); selx != nil {
   446  				if isIdent(selx.X, "check") {
   447  					switch selx.Sel.Name {
   448  					case "errorf":
   449  						// rewrite check.errorf(pos, ... ) to check.errorf(atPos(pos), ... )
   450  						if ident := asIdent(n.Args[0], "pos"); ident != nil {
   451  							pos := n.Args[0].Pos()
   452  							fun := newIdent(pos, "atPos")
   453  							n.Args[0] = &ast.CallExpr{Fun: fun, Lparen: pos, Args: []ast.Expr{ident}, Rparen: pos}
   454  							return false
   455  						}
   456  					}
   457  				}
   458  			}
   459  		}
   460  		return true
   461  	})
   462  }
   463  
   464  // fixGlobalTypVarDecl changes the global Typ variable from an array to a slice
   465  // (in types2 we use an array for efficiency, in go/types it's a slice and we
   466  // cannot change that).
   467  func fixGlobalTypVarDecl(f *ast.File) {
   468  	ast.Inspect(f, func(n ast.Node) bool {
   469  		switch n := n.(type) {
   470  		case *ast.ValueSpec:
   471  			// rewrite type Typ = [...]Type{...} to type Typ = []Type{...}
   472  			if len(n.Names) == 1 && n.Names[0].Name == "Typ" && len(n.Values) == 1 {
   473  				n.Values[0].(*ast.CompositeLit).Type.(*ast.ArrayType).Len = nil
   474  				return false
   475  			}
   476  		}
   477  		return true
   478  	})
   479  }
   480  
   481  // fixSprintf adds an extra nil argument for the *token.FileSet parameter in sprintf calls.
   482  func fixSprintf(f *ast.File) {
   483  	ast.Inspect(f, func(n ast.Node) bool {
   484  		switch n := n.(type) {
   485  		case *ast.CallExpr:
   486  			if isIdent(n.Fun, "sprintf") && len(n.Args) >= 4 /* ... args */ {
   487  				n.Args = insert(n.Args, 1, newIdent(n.Args[1].Pos(), "nil"))
   488  				return false
   489  			}
   490  		}
   491  		return true
   492  	})
   493  }
   494  
   495  // asIdent returns x as *ast.Ident if it is an identifier with the given name.
   496  func asIdent(x ast.Node, name string) *ast.Ident {
   497  	if ident, _ := x.(*ast.Ident); ident != nil && ident.Name == name {
   498  		return ident
   499  	}
   500  	return nil
   501  }
   502  
   503  // isIdent reports whether x is an identifier with the given name.
   504  func isIdent(x ast.Node, name string) bool {
   505  	return asIdent(x, name) != nil
   506  }
   507  
   508  // newIdent returns a new identifier with the given position and name.
   509  func newIdent(pos token.Pos, name string) *ast.Ident {
   510  	id := ast.NewIdent(name)
   511  	id.NamePos = pos
   512  	return id
   513  }
   514  
   515  // insert inserts x at list[at] and moves the remaining elements up.
   516  func insert(list []ast.Expr, at int, x ast.Expr) []ast.Expr {
   517  	list = append(list, nil)
   518  	copy(list[at+1:], list[at:])
   519  	list[at] = x
   520  	return list
   521  }
   522
View as plain text