unified.go

     1  // Copyright 2021 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package noder
     6  
     7  import (
     8  	"cmp"
     9  	"fmt"
    10  	"internal/buildcfg"
    11  	"internal/pkgbits"
    12  	"internal/types/errors"
    13  	"io"
    14  	"runtime"
    15  	"slices"
    16  	"strings"
    17  
    18  	"cmd/compile/internal/base"
    19  	"cmd/compile/internal/inline"
    20  	"cmd/compile/internal/ir"
    21  	"cmd/compile/internal/pgoir"
    22  	"cmd/compile/internal/typecheck"
    23  	"cmd/compile/internal/types"
    24  	"cmd/compile/internal/types2"
    25  	"cmd/internal/src"
    26  )
    27  
    28  // uirVersion is the unified IR version to use for encoding/decoding.
    29  // Use V4 for generic methods if the GOEXPERIMENT is enabled.
    30  var uirVersion = func() pkgbits.Version {
    31  	if buildcfg.Experiment.GenericMethods {
    32  		return pkgbits.V4
    33  	}
    34  	return pkgbits.V3
    35  }()
    36  
    37  // localPkgReader holds the package reader used for reading the local
    38  // package. It exists so the unified IR linker can refer back to it
    39  // later.
    40  var localPkgReader *pkgReader
    41  
    42  // LookupFunc returns the ir.Func for an arbitrary full symbol name if
    43  // that function exists in the set of available export data.
    44  //
    45  // This allows lookup of arbitrary functions and methods that aren't otherwise
    46  // referenced by the local package and thus haven't been read yet.
    47  //
    48  // TODO(prattmic): Does not handle instantiation of generic types. Currently
    49  // profiles don't contain the original type arguments, so we won't be able to
    50  // create the runtime dictionaries.
    51  //
    52  // TODO(prattmic): Hit rate of this function is usually fairly low, and errors
    53  // are only used when debug logging is enabled. Consider constructing cheaper
    54  // errors by default.
    55  func LookupFunc(fullName string) (*ir.Func, error) {
    56  	pkgPath, symName, err := ir.ParseLinkFuncName(fullName)
    57  	if err != nil {
    58  		return nil, fmt.Errorf("error parsing symbol name %q: %v", fullName, err)
    59  	}
    60  
    61  	pkg, ok := types.PkgMap()[pkgPath]
    62  	if !ok {
    63  		return nil, fmt.Errorf("pkg %s doesn't exist in %v", pkgPath, types.PkgMap())
    64  	}
    65  
    66  	// Symbol naming is ambiguous. We can't necessarily distinguish between
    67  	// a method and a closure. e.g., is foo.Bar.func1 a closure defined in
    68  	// function Bar, or a method on type Bar? Thus we must simply attempt
    69  	// to lookup both.
    70  
    71  	fn, err := lookupFunction(pkg, symName)
    72  	if err == nil {
    73  		return fn, nil
    74  	}
    75  
    76  	fn, mErr := lookupMethod(pkg, symName)
    77  	if mErr == nil {
    78  		return fn, nil
    79  	}
    80  
    81  	return nil, fmt.Errorf("%s is not a function (%v) or method (%v)", fullName, err, mErr)
    82  }
    83  
    84  // PostLookupCleanup performs cleanup operations needed
    85  // after a series of calls to LookupFunc, specifically invoking
    86  // readBodies to post-process any funcs on the "todoBodies" list
    87  // that were added as a result of the lookup operations.
    88  func PostLookupCleanup() {
    89  	readBodies(typecheck.Target, false)
    90  }
    91  
    92  func lookupFunction(pkg *types.Pkg, symName string) (*ir.Func, error) {
    93  	sym := pkg.Lookup(symName)
    94  
    95  	// TODO(prattmic): Enclosed functions (e.g., foo.Bar.func1) are not
    96  	// present in objReader, only as OCLOSURE nodes in the enclosing
    97  	// function.
    98  	pri, ok := objReader[sym]
    99  	if !ok {
   100  		return nil, fmt.Errorf("func sym %v missing objReader", sym)
   101  	}
   102  
   103  	node, err := pri.pr.objIdxMayFail(pri.idx, nil, nil, false)
   104  	if err != nil {
   105  		return nil, fmt.Errorf("func sym %v lookup error: %w", sym, err)
   106  	}
   107  	name := node.(*ir.Name)
   108  	if name.Op() != ir.ONAME || name.Class != ir.PFUNC {
   109  		return nil, fmt.Errorf("func sym %v refers to non-function name: %v", sym, name)
   110  	}
   111  	return name.Func, nil
   112  }
   113  
   114  func lookupMethod(pkg *types.Pkg, symName string) (*ir.Func, error) {
   115  	// N.B. readPackage creates a Sym for every object in the package to
   116  	// initialize objReader and importBodyReader, even if the object isn't
   117  	// read.
   118  	//
   119  	// However, objReader is only initialized for top-level objects, so we
   120  	// must first lookup the type and use that to find the method rather
   121  	// than looking for the method directly.
   122  	typ, meth, err := ir.LookupMethodSelector(pkg, symName)
   123  	if err != nil {
   124  		return nil, fmt.Errorf("error looking up method symbol %q: %v", symName, err)
   125  	}
   126  
   127  	pri, ok := objReader[typ]
   128  	if !ok {
   129  		return nil, fmt.Errorf("type sym %v missing objReader", typ)
   130  	}
   131  
   132  	node, err := pri.pr.objIdxMayFail(pri.idx, nil, nil, false)
   133  	if err != nil {
   134  		return nil, fmt.Errorf("func sym %v lookup error: %w", typ, err)
   135  	}
   136  	name := node.(*ir.Name)
   137  	if name.Op() != ir.OTYPE {
   138  		return nil, fmt.Errorf("type sym %v refers to non-type name: %v", typ, name)
   139  	}
   140  	if name.Alias() {
   141  		return nil, fmt.Errorf("type sym %v refers to alias", typ)
   142  	}
   143  	if name.Type().IsInterface() {
   144  		return nil, fmt.Errorf("type sym %v refers to interface type", typ)
   145  	}
   146  
   147  	for _, m := range name.Type().Methods() {
   148  		if m.Sym == meth {
   149  			fn := m.Nname.(*ir.Name).Func
   150  			return fn, nil
   151  		}
   152  	}
   153  
   154  	return nil, fmt.Errorf("method %s missing from method set of %v", symName, typ)
   155  }
   156  
   157  // unified constructs the local package's Internal Representation (IR)
   158  // from its syntax tree (AST).
   159  //
   160  // The pipeline contains 2 steps:
   161  //
   162  //  1. Generate the export data "stub".
   163  //
   164  //  2. Generate the IR from the export data above.
   165  //
   166  // The package data "stub" at step (1) contains everything from the local package,
   167  // but nothing that has been imported. When we're actually writing out export data
   168  // to the output files (see writeNewExport), we run the "linker", which:
   169  //
   170  //   - Updates compiler extensions data (e.g. inlining cost, escape analysis results).
   171  //
   172  //   - Handles re-exporting any transitive dependencies.
   173  //
   174  //   - Prunes out any unnecessary details (e.g. non-inlineable functions, because any
   175  //     downstream importers only care about inlinable functions).
   176  //
   177  // The source files are typechecked twice: once before writing the export data
   178  // using types2, and again after reading the export data using gc/typecheck.
   179  // The duplication of work will go away once we only use the types2 type checker,
   180  // removing the gc/typecheck step. For now, it is kept because:
   181  //
   182  //   - It reduces the engineering costs in maintaining a fork of typecheck
   183  //     (e.g. no need to backport fixes like CL 327651).
   184  //
   185  //   - It makes it easier to pass toolstash -cmp.
   186  //
   187  //   - Historically, we would always re-run the typechecker after importing a package,
   188  //     even though we know the imported data is valid. It's not ideal, but it's
   189  //     not causing any problems either.
   190  //
   191  //   - gc/typecheck is still in charge of some transformations, such as rewriting
   192  //     multi-valued function calls or transforming ir.OINDEX to ir.OINDEXMAP.
   193  //
   194  // Using the syntax tree with types2, which has a complete representation of generics,
   195  // the unified IR has the full typed AST needed for introspection during step (1).
   196  // In other words, we have all the necessary information to build the generic IR form
   197  // (see writer.captureVars for an example).
   198  func unified(m posMap, noders []*noder) {
   199  	inline.InlineCall = unifiedInlineCall
   200  	typecheck.HaveInlineBody = unifiedHaveInlineBody
   201  	pgoir.LookupFunc = LookupFunc
   202  	pgoir.PostLookupCleanup = PostLookupCleanup
   203  
   204  	data := writePkgStub(m, noders)
   205  
   206  	target := typecheck.Target
   207  
   208  	localPkgReader = newPkgReader(pkgbits.NewPkgDecoder(types.LocalPkg.Path, data))
   209  	readPackage(localPkgReader, types.LocalPkg, true)
   210  
   211  	r := localPkgReader.newReader(pkgbits.SectionMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
   212  	r.pkgInit(types.LocalPkg, target)
   213  
   214  	readBodies(target, false)
   215  
   216  	// Check that nothing snuck past typechecking.
   217  	for _, fn := range target.Funcs {
   218  		if fn.Typecheck() == 0 {
   219  			base.FatalfAt(fn.Pos(), "missed typecheck: %v", fn)
   220  		}
   221  
   222  		// For functions, check that at least their first statement (if
   223  		// any) was typechecked too.
   224  		if len(fn.Body) != 0 {
   225  			if stmt := fn.Body[0]; stmt.Typecheck() == 0 {
   226  				base.FatalfAt(stmt.Pos(), "missed typecheck: %v", stmt)
   227  			}
   228  		}
   229  	}
   230  
   231  	// For functions originally came from package runtime,
   232  	// mark as norace to prevent instrumenting, see issue #60439.
   233  	for _, fn := range target.Funcs {
   234  		if !base.Flag.CompilingRuntime && types.RuntimeSymName(fn.Sym()) != "" {
   235  			fn.Pragma |= ir.Norace
   236  		}
   237  	}
   238  
   239  	base.ExitIfErrors() // just in case
   240  }
   241  
   242  // readBodies iteratively expands all pending dictionaries and
   243  // function bodies.
   244  //
   245  // If duringInlining is true, then the inline.InlineDecls is called as
   246  // necessary on instantiations of imported generic functions, so their
   247  // inlining costs can be computed.
   248  func readBodies(target *ir.Package, duringInlining bool) {
   249  	var inlDecls []*ir.Func
   250  
   251  	// Don't use range--bodyIdx can add closures to todoBodies.
   252  	for {
   253  		// The order we expand dictionaries and bodies doesn't matter, so
   254  		// pop from the end to reduce todoBodies reallocations if it grows
   255  		// further.
   256  		//
   257  		// However, we do at least need to flush any pending dictionaries
   258  		// before reading bodies, because bodies might reference the
   259  		// dictionaries.
   260  
   261  		if len(todoDicts) > 0 {
   262  			fn := todoDicts[len(todoDicts)-1]
   263  			todoDicts = todoDicts[:len(todoDicts)-1]
   264  			fn()
   265  			continue
   266  		}
   267  
   268  		if len(todoBodies) > 0 {
   269  			fn := todoBodies[len(todoBodies)-1]
   270  			todoBodies = todoBodies[:len(todoBodies)-1]
   271  
   272  			pri, ok := bodyReader[fn]
   273  			assert(ok)
   274  			pri.funcBody(fn)
   275  
   276  			// Instantiated generic function: add to Decls for typechecking
   277  			// and compilation.
   278  			if fn.OClosure == nil && len(pri.dict.targs) != 0 {
   279  				// cmd/link does not support a type symbol referencing a method symbol
   280  				// across DSO boundary, so force re-compiling methods on a generic type
   281  				// even it was seen from imported package in linkshared mode, see #58966.
   282  				canSkipNonGenericMethod := !(base.Ctxt.Flag_linkshared && ir.IsMethod(fn))
   283  				if duringInlining && canSkipNonGenericMethod {
   284  					inlDecls = append(inlDecls, fn)
   285  				} else {
   286  					target.Funcs = append(target.Funcs, fn)
   287  				}
   288  			}
   289  
   290  			continue
   291  		}
   292  
   293  		break
   294  	}
   295  
   296  	todoDicts = nil
   297  	todoBodies = nil
   298  
   299  	if len(inlDecls) != 0 {
   300  		// If we instantiated any generic functions during inlining, we need
   301  		// to call CanInline on them so they'll be transitively inlined
   302  		// correctly (#56280).
   303  		//
   304  		// We know these functions were already compiled in an imported
   305  		// package though, so we don't need to actually apply InlineCalls or
   306  		// save the function bodies any further than this.
   307  		//
   308  		// We can also lower the -m flag to 0, to suppress duplicate "can
   309  		// inline" diagnostics reported against the imported package. Again,
   310  		// we already reported those diagnostics in the original package, so
   311  		// it's pointless repeating them here.
   312  
   313  		oldLowerM := base.Flag.LowerM
   314  		base.Flag.LowerM = 0
   315  		inline.CanInlineFuncs(inlDecls, nil)
   316  		base.Flag.LowerM = oldLowerM
   317  
   318  		for _, fn := range inlDecls {
   319  			fn.Body = nil // free memory
   320  		}
   321  	}
   322  }
   323  
   324  // writePkgStub type checks the given parsed source files,
   325  // writes an export data package stub representing them,
   326  // and returns the result.
   327  func writePkgStub(m posMap, noders []*noder) string {
   328  	pkg, info, otherInfo := checkFiles(m, noders)
   329  
   330  	pw := newPkgWriter(m, pkg, info, otherInfo)
   331  
   332  	pw.collectDecls(noders)
   333  
   334  	publicRootWriter := pw.newWriter(pkgbits.SectionMeta, pkgbits.SyncPublic)
   335  	privateRootWriter := pw.newWriter(pkgbits.SectionMeta, pkgbits.SyncPrivate)
   336  
   337  	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)
   338  	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)
   339  
   340  	{
   341  		w := publicRootWriter
   342  		w.pkg(pkg)
   343  
   344  		if w.Version().Has(pkgbits.HasInit) {
   345  			w.Bool(false)
   346  		}
   347  
   348  		scope := pkg.Scope()
   349  		names := scope.Names()
   350  		w.Len(len(names))
   351  		for _, name := range names {
   352  			w.obj(scope.Lookup(name), nil)
   353  		}
   354  
   355  		w.Sync(pkgbits.SyncEOF)
   356  		w.Flush()
   357  	}
   358  
   359  	{
   360  		w := privateRootWriter
   361  		w.pkgInit(noders)
   362  		w.Flush()
   363  	}
   364  
   365  	var sb strings.Builder
   366  	pw.DumpTo(&sb)
   367  
   368  	// At this point, we're done with types2. Make sure the package is
   369  	// garbage collected.
   370  	freePackage(pkg)
   371  
   372  	return sb.String()
   373  }
   374  
   375  // freePackage ensures the given package is garbage collected.
   376  func freePackage(pkg *types2.Package) {
   377  	// The GC test below relies on a precise GC that runs finalizers as
   378  	// soon as objects are unreachable. Our implementation provides
   379  	// this, but other/older implementations may not (e.g., Go 1.4 does
   380  	// not because of #22350). To avoid imposing unnecessary
   381  	// restrictions on the GOROOT_BOOTSTRAP toolchain, we skip the test
   382  	// during bootstrapping.
   383  	if base.CompilerBootstrap || base.Debug.GCCheck == 0 {
   384  		*pkg = types2.Package{}
   385  		return
   386  	}
   387  
   388  	// Set a finalizer on pkg so we can detect if/when it's collected.
   389  	done := make(chan struct{})
   390  	runtime.SetFinalizer(pkg, func(*types2.Package) { close(done) })
   391  
   392  	// Important: objects involved in cycles are not finalized, so zero
   393  	// out pkg to break its cycles and allow the finalizer to run.
   394  	*pkg = types2.Package{}
   395  
   396  	// It typically takes just 1 or 2 cycles to release pkg, but it
   397  	// doesn't hurt to try a few more times.
   398  	for i := 0; i < 10; i++ {
   399  		select {
   400  		case <-done:
   401  			return
   402  		default:
   403  			runtime.GC()
   404  		}
   405  	}
   406  
   407  	base.Fatalf("package never finalized")
   408  }
   409  
   410  // readPackage reads package export data from pr to populate
   411  // importpkg.
   412  //
   413  // localStub indicates whether pr is reading the stub export data for
   414  // the local package, as opposed to relocated export data for an
   415  // import.
   416  func readPackage(pr *pkgReader, importpkg *types.Pkg, localStub bool) {
   417  	{
   418  		r := pr.newReader(pkgbits.SectionMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
   419  
   420  		pkg := r.pkg()
   421  		// This error can happen if "go tool compile" is called with wrong "-p" flag, see issue #54542.
   422  		if pkg != importpkg {
   423  			base.ErrorfAt(base.AutogeneratedPos, errors.BadImportPath, "mismatched import path, have %q (%p), want %q (%p)", pkg.Path, pkg, importpkg.Path, importpkg)
   424  			base.ErrorExit()
   425  		}
   426  
   427  		if r.Version().Has(pkgbits.HasInit) {
   428  			r.Bool()
   429  		}
   430  
   431  		for i, n := 0, r.Len(); i < n; i++ {
   432  			r.Sync(pkgbits.SyncObject)
   433  			if r.Version().Has(pkgbits.DerivedFuncInstance) {
   434  				assert(!r.Bool())
   435  			}
   436  			idx := r.Reloc(pkgbits.SectionObj)
   437  			assert(r.Len() == 0)
   438  
   439  			path, name, code := r.p.PeekObj(idx)
   440  			if code != pkgbits.ObjStub {
   441  				objReader[types.NewPkg(path, "").Lookup(name)] = pkgReaderIndex{pr, idx, nil, nil, nil}
   442  			}
   443  		}
   444  
   445  		r.Sync(pkgbits.SyncEOF)
   446  	}
   447  
   448  	if !localStub {
   449  		r := pr.newReader(pkgbits.SectionMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
   450  
   451  		if r.Bool() {
   452  			sym := importpkg.Lookup(".inittask")
   453  			task := ir.NewNameAt(src.NoXPos, sym, nil)
   454  			task.Class = ir.PEXTERN
   455  			sym.Def = task
   456  		}
   457  
   458  		for i, n := 0, r.Len(); i < n; i++ {
   459  			path := r.String()
   460  			name := r.String()
   461  			idx := r.Reloc(pkgbits.SectionBody)
   462  
   463  			sym := types.NewPkg(path, "").Lookup(name)
   464  			if _, ok := importBodyReader[sym]; !ok {
   465  				importBodyReader[sym] = pkgReaderIndex{pr, idx, nil, nil, nil}
   466  			}
   467  		}
   468  
   469  		r.Sync(pkgbits.SyncEOF)
   470  	}
   471  }
   472  
   473  // writeUnifiedExport writes to `out` the finalized, self-contained
   474  // Unified IR export data file for the current compilation unit.
   475  func writeUnifiedExport(out io.Writer) {
   476  	l := linker{
   477  		pw: pkgbits.NewPkgEncoder(uirVersion, base.Debug.SyncFrames),
   478  
   479  		pkgs:   make(map[string]index),
   480  		decls:  make(map[*types.Sym]index),
   481  		bodies: make(map[*types.Sym]index),
   482  	}
   483  
   484  	publicRootWriter := l.pw.NewEncoder(pkgbits.SectionMeta, pkgbits.SyncPublic)
   485  	privateRootWriter := l.pw.NewEncoder(pkgbits.SectionMeta, pkgbits.SyncPrivate)
   486  	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)
   487  	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)
   488  
   489  	var selfPkgIdx index
   490  
   491  	{
   492  		pr := localPkgReader
   493  		r := pr.NewDecoder(pkgbits.SectionMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
   494  
   495  		r.Sync(pkgbits.SyncPkg)
   496  		selfPkgIdx = l.relocIdx(pr, pkgbits.SectionPkg, r.Reloc(pkgbits.SectionPkg))
   497  
   498  		// Versions must match.
   499  		// TODO: It seems that we should be able to use r.Version() for NewPkgEncoder
   500  		// instead of passing uirVersion, but NewPkgEncoder is created before r.
   501  		// If that is correct, we should make that happen.
   502  		assert(r.Version() == uirVersion)
   503  
   504  		if r.Version().Has(pkgbits.HasInit) {
   505  			r.Bool()
   506  		}
   507  
   508  		for i, n := 0, r.Len(); i < n; i++ {
   509  			r.Sync(pkgbits.SyncObject)
   510  			if r.Version().Has(pkgbits.DerivedFuncInstance) {
   511  				assert(!r.Bool())
   512  			}
   513  			idx := r.Reloc(pkgbits.SectionObj)
   514  			assert(r.Len() == 0)
   515  
   516  			xpath, xname, xtag := pr.PeekObj(idx)
   517  			assert(xpath == pr.PkgPath())
   518  			assert(xtag != pkgbits.ObjStub)
   519  
   520  			if types.IsExported(xname) {
   521  				l.relocIdx(pr, pkgbits.SectionObj, idx)
   522  			}
   523  		}
   524  
   525  		r.Sync(pkgbits.SyncEOF)
   526  	}
   527  
   528  	{
   529  		var idxs []index
   530  		for _, idx := range l.decls {
   531  			idxs = append(idxs, idx)
   532  		}
   533  		slices.Sort(idxs)
   534  
   535  		w := publicRootWriter
   536  
   537  		w.Sync(pkgbits.SyncPkg)
   538  		w.Reloc(pkgbits.SectionPkg, selfPkgIdx)
   539  
   540  		if w.Version().Has(pkgbits.HasInit) {
   541  			w.Bool(false)
   542  		}
   543  
   544  		w.Len(len(idxs))
   545  		for _, idx := range idxs {
   546  			w.Sync(pkgbits.SyncObject)
   547  			if w.Version().Has(pkgbits.DerivedFuncInstance) {
   548  				w.Bool(false)
   549  			}
   550  			w.Reloc(pkgbits.SectionObj, idx)
   551  			w.Len(0)
   552  		}
   553  
   554  		w.Sync(pkgbits.SyncEOF)
   555  		w.Flush()
   556  	}
   557  
   558  	{
   559  		type symIdx struct {
   560  			sym *types.Sym
   561  			idx index
   562  		}
   563  		var bodies []symIdx
   564  		for sym, idx := range l.bodies {
   565  			bodies = append(bodies, symIdx{sym, idx})
   566  		}
   567  		slices.SortFunc(bodies, func(a, b symIdx) int { return cmp.Compare(a.idx, b.idx) })
   568  
   569  		w := privateRootWriter
   570  
   571  		w.Bool(typecheck.Lookup(".inittask").Def != nil)
   572  
   573  		w.Len(len(bodies))
   574  		for _, body := range bodies {
   575  			w.String(body.sym.Pkg.Path)
   576  			w.String(body.sym.Name)
   577  			w.Reloc(pkgbits.SectionBody, body.idx)
   578  		}
   579  
   580  		w.Sync(pkgbits.SyncEOF)
   581  		w.Flush()
   582  	}
   583  
   584  	base.Ctxt.Fingerprint = l.pw.DumpTo(out)
   585  }
   586
View as plain text