unified.go

     1  // Copyright 2021 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package noder
     6  
     7  import (
     8  	"cmp"
     9  	"fmt"
    10  	"internal/pkgbits"
    11  	"internal/types/errors"
    12  	"io"
    13  	"runtime"
    14  	"slices"
    15  	"strings"
    16  
    17  	"cmd/compile/internal/base"
    18  	"cmd/compile/internal/inline"
    19  	"cmd/compile/internal/ir"
    20  	"cmd/compile/internal/pgoir"
    21  	"cmd/compile/internal/typecheck"
    22  	"cmd/compile/internal/types"
    23  	"cmd/compile/internal/types2"
    24  	"cmd/internal/src"
    25  )
    26  
    27  // localPkgReader holds the package reader used for reading the local
    28  // package. It exists so the unified IR linker can refer back to it
    29  // later.
    30  var localPkgReader *pkgReader
    31  
    32  // LookupFunc returns the ir.Func for an arbitrary full symbol name if
    33  // that function exists in the set of available export data.
    34  //
    35  // This allows lookup of arbitrary functions and methods that aren't otherwise
    36  // referenced by the local package and thus haven't been read yet.
    37  //
    38  // TODO(prattmic): Does not handle instantiation of generic types. Currently
    39  // profiles don't contain the original type arguments, so we won't be able to
    40  // create the runtime dictionaries.
    41  //
    42  // TODO(prattmic): Hit rate of this function is usually fairly low, and errors
    43  // are only used when debug logging is enabled. Consider constructing cheaper
    44  // errors by default.
    45  func LookupFunc(fullName string) (*ir.Func, error) {
    46  	pkgPath, symName, err := ir.ParseLinkFuncName(fullName)
    47  	if err != nil {
    48  		return nil, fmt.Errorf("error parsing symbol name %q: %v", fullName, err)
    49  	}
    50  
    51  	pkg, ok := types.PkgMap()[pkgPath]
    52  	if !ok {
    53  		return nil, fmt.Errorf("pkg %s doesn't exist in %v", pkgPath, types.PkgMap())
    54  	}
    55  
    56  	// Symbol naming is ambiguous. We can't necessarily distinguish between
    57  	// a method and a closure. e.g., is foo.Bar.func1 a closure defined in
    58  	// function Bar, or a method on type Bar? Thus we must simply attempt
    59  	// to lookup both.
    60  
    61  	fn, err := lookupFunction(pkg, symName)
    62  	if err == nil {
    63  		return fn, nil
    64  	}
    65  
    66  	fn, mErr := lookupMethod(pkg, symName)
    67  	if mErr == nil {
    68  		return fn, nil
    69  	}
    70  
    71  	return nil, fmt.Errorf("%s is not a function (%v) or method (%v)", fullName, err, mErr)
    72  }
    73  
    74  // PostLookupCleanup performs cleanup operations needed
    75  // after a series of calls to LookupFunc, specifically invoking
    76  // readBodies to post-process any funcs on the "todoBodies" list
    77  // that were added as a result of the lookup operations.
    78  func PostLookupCleanup() {
    79  	readBodies(typecheck.Target, false)
    80  }
    81  
    82  func lookupFunction(pkg *types.Pkg, symName string) (*ir.Func, error) {
    83  	sym := pkg.Lookup(symName)
    84  
    85  	// TODO(prattmic): Enclosed functions (e.g., foo.Bar.func1) are not
    86  	// present in objReader, only as OCLOSURE nodes in the enclosing
    87  	// function.
    88  	pri, ok := objReader[sym]
    89  	if !ok {
    90  		return nil, fmt.Errorf("func sym %v missing objReader", sym)
    91  	}
    92  
    93  	node, err := pri.pr.objIdxMayFail(pri.idx, nil, nil, false)
    94  	if err != nil {
    95  		return nil, fmt.Errorf("func sym %v lookup error: %w", sym, err)
    96  	}
    97  	name := node.(*ir.Name)
    98  	if name.Op() != ir.ONAME || name.Class != ir.PFUNC {
    99  		return nil, fmt.Errorf("func sym %v refers to non-function name: %v", sym, name)
   100  	}
   101  	return name.Func, nil
   102  }
   103  
   104  func lookupMethod(pkg *types.Pkg, symName string) (*ir.Func, error) {
   105  	// N.B. readPackage creates a Sym for every object in the package to
   106  	// initialize objReader and importBodyReader, even if the object isn't
   107  	// read.
   108  	//
   109  	// However, objReader is only initialized for top-level objects, so we
   110  	// must first lookup the type and use that to find the method rather
   111  	// than looking for the method directly.
   112  	typ, meth, err := ir.LookupMethodSelector(pkg, symName)
   113  	if err != nil {
   114  		return nil, fmt.Errorf("error looking up method symbol %q: %v", symName, err)
   115  	}
   116  
   117  	pri, ok := objReader[typ]
   118  	if !ok {
   119  		return nil, fmt.Errorf("type sym %v missing objReader", typ)
   120  	}
   121  
   122  	node, err := pri.pr.objIdxMayFail(pri.idx, nil, nil, false)
   123  	if err != nil {
   124  		return nil, fmt.Errorf("func sym %v lookup error: %w", typ, err)
   125  	}
   126  	name := node.(*ir.Name)
   127  	if name.Op() != ir.OTYPE {
   128  		return nil, fmt.Errorf("type sym %v refers to non-type name: %v", typ, name)
   129  	}
   130  	if name.Alias() {
   131  		return nil, fmt.Errorf("type sym %v refers to alias", typ)
   132  	}
   133  	if name.Type().IsInterface() {
   134  		return nil, fmt.Errorf("type sym %v refers to interface type", typ)
   135  	}
   136  
   137  	for _, m := range name.Type().Methods() {
   138  		if m.Sym == meth {
   139  			fn := m.Nname.(*ir.Name).Func
   140  			return fn, nil
   141  		}
   142  	}
   143  
   144  	return nil, fmt.Errorf("method %s missing from method set of %v", symName, typ)
   145  }
   146  
   147  // unified constructs the local package's Internal Representation (IR)
   148  // from its syntax tree (AST).
   149  //
   150  // The pipeline contains 2 steps:
   151  //
   152  //  1. Generate the export data "stub".
   153  //
   154  //  2. Generate the IR from the export data above.
   155  //
   156  // The package data "stub" at step (1) contains everything from the local package,
   157  // but nothing that has been imported. When we're actually writing out export data
   158  // to the output files (see writeNewExport), we run the "linker", which:
   159  //
   160  //   - Updates compiler extensions data (e.g. inlining cost, escape analysis results).
   161  //
   162  //   - Handles re-exporting any transitive dependencies.
   163  //
   164  //   - Prunes out any unnecessary details (e.g. non-inlineable functions, because any
   165  //     downstream importers only care about inlinable functions).
   166  //
   167  // The source files are typechecked twice: once before writing the export data
   168  // using types2, and again after reading the export data using gc/typecheck.
   169  // The duplication of work will go away once we only use the types2 type checker,
   170  // removing the gc/typecheck step. For now, it is kept because:
   171  //
   172  //   - It reduces the engineering costs in maintaining a fork of typecheck
   173  //     (e.g. no need to backport fixes like CL 327651).
   174  //
   175  //   - It makes it easier to pass toolstash -cmp.
   176  //
   177  //   - Historically, we would always re-run the typechecker after importing a package,
   178  //     even though we know the imported data is valid. It's not ideal, but it's
   179  //     not causing any problems either.
   180  //
   181  //   - gc/typecheck is still in charge of some transformations, such as rewriting
   182  //     multi-valued function calls or transforming ir.OINDEX to ir.OINDEXMAP.
   183  //
   184  // Using the syntax tree with types2, which has a complete representation of generics,
   185  // the unified IR has the full typed AST needed for introspection during step (1).
   186  // In other words, we have all the necessary information to build the generic IR form
   187  // (see writer.captureVars for an example).
   188  func unified(m posMap, noders []*noder) {
   189  	inline.InlineCall = unifiedInlineCall
   190  	typecheck.HaveInlineBody = unifiedHaveInlineBody
   191  	pgoir.LookupFunc = LookupFunc
   192  	pgoir.PostLookupCleanup = PostLookupCleanup
   193  
   194  	data := writePkgStub(m, noders)
   195  
   196  	target := typecheck.Target
   197  
   198  	localPkgReader = newPkgReader(pkgbits.NewPkgDecoder(types.LocalPkg.Path, data))
   199  	readPackage(localPkgReader, types.LocalPkg, true)
   200  
   201  	r := localPkgReader.newReader(pkgbits.SectionMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
   202  	r.pkgInit(types.LocalPkg, target)
   203  
   204  	readBodies(target, false)
   205  
   206  	// Check that nothing snuck past typechecking.
   207  	for _, fn := range target.Funcs {
   208  		if fn.Typecheck() == 0 {
   209  			base.FatalfAt(fn.Pos(), "missed typecheck: %v", fn)
   210  		}
   211  
   212  		// For functions, check that at least their first statement (if
   213  		// any) was typechecked too.
   214  		if len(fn.Body) != 0 {
   215  			if stmt := fn.Body[0]; stmt.Typecheck() == 0 {
   216  				base.FatalfAt(stmt.Pos(), "missed typecheck: %v", stmt)
   217  			}
   218  		}
   219  	}
   220  
   221  	// For functions originally came from package runtime,
   222  	// mark as norace to prevent instrumenting, see issue #60439.
   223  	for _, fn := range target.Funcs {
   224  		if !base.Flag.CompilingRuntime && types.RuntimeSymName(fn.Sym()) != "" {
   225  			fn.Pragma |= ir.Norace
   226  		}
   227  	}
   228  
   229  	base.ExitIfErrors() // just in case
   230  }
   231  
   232  // readBodies iteratively expands all pending dictionaries and
   233  // function bodies.
   234  //
   235  // If duringInlining is true, then the inline.InlineDecls is called as
   236  // necessary on instantiations of imported generic functions, so their
   237  // inlining costs can be computed.
   238  func readBodies(target *ir.Package, duringInlining bool) {
   239  	var inlDecls []*ir.Func
   240  
   241  	// Don't use range--bodyIdx can add closures to todoBodies.
   242  	for {
   243  		// The order we expand dictionaries and bodies doesn't matter, so
   244  		// pop from the end to reduce todoBodies reallocations if it grows
   245  		// further.
   246  		//
   247  		// However, we do at least need to flush any pending dictionaries
   248  		// before reading bodies, because bodies might reference the
   249  		// dictionaries.
   250  
   251  		if len(todoDicts) > 0 {
   252  			fn := todoDicts[len(todoDicts)-1]
   253  			todoDicts = todoDicts[:len(todoDicts)-1]
   254  			fn()
   255  			continue
   256  		}
   257  
   258  		if len(todoBodies) > 0 {
   259  			fn := todoBodies[len(todoBodies)-1]
   260  			todoBodies = todoBodies[:len(todoBodies)-1]
   261  
   262  			pri, ok := bodyReader[fn]
   263  			assert(ok)
   264  			pri.funcBody(fn)
   265  
   266  			// Instantiated generic function: add to Decls for typechecking
   267  			// and compilation.
   268  			if fn.OClosure == nil && len(pri.dict.targs) != 0 {
   269  				// cmd/link does not support a type symbol referencing a method symbol
   270  				// across DSO boundary, so force re-compiling methods on a generic type
   271  				// even it was seen from imported package in linkshared mode, see #58966.
   272  				canSkipNonGenericMethod := !(base.Ctxt.Flag_linkshared && ir.IsMethod(fn))
   273  				if duringInlining && canSkipNonGenericMethod {
   274  					inlDecls = append(inlDecls, fn)
   275  				} else {
   276  					target.Funcs = append(target.Funcs, fn)
   277  				}
   278  			}
   279  
   280  			continue
   281  		}
   282  
   283  		break
   284  	}
   285  
   286  	todoDicts = nil
   287  	todoBodies = nil
   288  
   289  	if len(inlDecls) != 0 {
   290  		// If we instantiated any generic functions during inlining, we need
   291  		// to call CanInline on them so they'll be transitively inlined
   292  		// correctly (#56280).
   293  		//
   294  		// We know these functions were already compiled in an imported
   295  		// package though, so we don't need to actually apply InlineCalls or
   296  		// save the function bodies any further than this.
   297  		//
   298  		// We can also lower the -m flag to 0, to suppress duplicate "can
   299  		// inline" diagnostics reported against the imported package. Again,
   300  		// we already reported those diagnostics in the original package, so
   301  		// it's pointless repeating them here.
   302  
   303  		oldLowerM := base.Flag.LowerM
   304  		base.Flag.LowerM = 0
   305  		inline.CanInlineFuncs(inlDecls, nil)
   306  		base.Flag.LowerM = oldLowerM
   307  
   308  		for _, fn := range inlDecls {
   309  			fn.Body = nil // free memory
   310  		}
   311  	}
   312  }
   313  
   314  // writePkgStub type checks the given parsed source files,
   315  // writes an export data package stub representing them,
   316  // and returns the result.
   317  func writePkgStub(m posMap, noders []*noder) string {
   318  	pkg, info, otherInfo := checkFiles(m, noders)
   319  
   320  	pw := newPkgWriter(m, pkg, info, otherInfo)
   321  
   322  	pw.collectDecls(noders)
   323  
   324  	publicRootWriter := pw.newWriter(pkgbits.SectionMeta, pkgbits.SyncPublic)
   325  	privateRootWriter := pw.newWriter(pkgbits.SectionMeta, pkgbits.SyncPrivate)
   326  
   327  	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)
   328  	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)
   329  
   330  	{
   331  		w := publicRootWriter
   332  		w.pkg(pkg)
   333  
   334  		if w.Version().Has(pkgbits.HasInit) {
   335  			w.Bool(false)
   336  		}
   337  
   338  		scope := pkg.Scope()
   339  		names := scope.Names()
   340  		w.Len(len(names))
   341  		for _, name := range names {
   342  			w.obj(scope.Lookup(name), nil)
   343  		}
   344  
   345  		w.Sync(pkgbits.SyncEOF)
   346  		w.Flush()
   347  	}
   348  
   349  	{
   350  		w := privateRootWriter
   351  		w.pkgInit(noders)
   352  		w.Flush()
   353  	}
   354  
   355  	var sb strings.Builder
   356  	pw.DumpTo(&sb)
   357  
   358  	// At this point, we're done with types2. Make sure the package is
   359  	// garbage collected.
   360  	freePackage(pkg)
   361  
   362  	return sb.String()
   363  }
   364  
   365  // freePackage ensures the given package is garbage collected.
   366  func freePackage(pkg *types2.Package) {
   367  	// The GC test below relies on a precise GC that runs finalizers as
   368  	// soon as objects are unreachable. Our implementation provides
   369  	// this, but other/older implementations may not (e.g., Go 1.4 does
   370  	// not because of #22350). To avoid imposing unnecessary
   371  	// restrictions on the GOROOT_BOOTSTRAP toolchain, we skip the test
   372  	// during bootstrapping.
   373  	if base.CompilerBootstrap || base.Debug.GCCheck == 0 {
   374  		*pkg = types2.Package{}
   375  		return
   376  	}
   377  
   378  	// Set a finalizer on pkg so we can detect if/when it's collected.
   379  	done := make(chan struct{})
   380  	runtime.SetFinalizer(pkg, func(*types2.Package) { close(done) })
   381  
   382  	// Important: objects involved in cycles are not finalized, so zero
   383  	// out pkg to break its cycles and allow the finalizer to run.
   384  	*pkg = types2.Package{}
   385  
   386  	// It typically takes just 1 or 2 cycles to release pkg, but it
   387  	// doesn't hurt to try a few more times.
   388  	for i := 0; i < 10; i++ {
   389  		select {
   390  		case <-done:
   391  			return
   392  		default:
   393  			runtime.GC()
   394  		}
   395  	}
   396  
   397  	base.Fatalf("package never finalized")
   398  }
   399  
   400  // readPackage reads package export data from pr to populate
   401  // importpkg.
   402  //
   403  // localStub indicates whether pr is reading the stub export data for
   404  // the local package, as opposed to relocated export data for an
   405  // import.
   406  func readPackage(pr *pkgReader, importpkg *types.Pkg, localStub bool) {
   407  	{
   408  		r := pr.newReader(pkgbits.SectionMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
   409  
   410  		pkg := r.pkg()
   411  		// This error can happen if "go tool compile" is called with wrong "-p" flag, see issue #54542.
   412  		if pkg != importpkg {
   413  			base.ErrorfAt(base.AutogeneratedPos, errors.BadImportPath, "mismatched import path, have %q (%p), want %q (%p)", pkg.Path, pkg, importpkg.Path, importpkg)
   414  			base.ErrorExit()
   415  		}
   416  
   417  		if r.Version().Has(pkgbits.HasInit) {
   418  			r.Bool()
   419  		}
   420  
   421  		for i, n := 0, r.Len(); i < n; i++ {
   422  			r.Sync(pkgbits.SyncObject)
   423  			if r.Version().Has(pkgbits.DerivedFuncInstance) {
   424  				assert(!r.Bool())
   425  			}
   426  			idx := r.Reloc(pkgbits.SectionObj)
   427  			assert(r.Len() == 0)
   428  
   429  			path, name, code := r.p.PeekObj(idx)
   430  			if code != pkgbits.ObjStub {
   431  				objReader[types.NewPkg(path, "").Lookup(name)] = pkgReaderIndex{pr, idx, nil, nil, nil}
   432  			}
   433  		}
   434  
   435  		r.Sync(pkgbits.SyncEOF)
   436  	}
   437  
   438  	if !localStub {
   439  		r := pr.newReader(pkgbits.SectionMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
   440  
   441  		if r.Bool() {
   442  			sym := importpkg.Lookup(".inittask")
   443  			task := ir.NewNameAt(src.NoXPos, sym, nil)
   444  			task.Class = ir.PEXTERN
   445  			sym.Def = task
   446  		}
   447  
   448  		for i, n := 0, r.Len(); i < n; i++ {
   449  			path := r.String()
   450  			name := r.String()
   451  			idx := r.Reloc(pkgbits.SectionBody)
   452  
   453  			sym := types.NewPkg(path, "").Lookup(name)
   454  			if _, ok := importBodyReader[sym]; !ok {
   455  				importBodyReader[sym] = pkgReaderIndex{pr, idx, nil, nil, nil}
   456  			}
   457  		}
   458  
   459  		r.Sync(pkgbits.SyncEOF)
   460  	}
   461  }
   462  
   463  // writeUnifiedExport writes to `out` the finalized, self-contained
   464  // Unified IR export data file for the current compilation unit.
   465  func writeUnifiedExport(out io.Writer) {
   466  	// Use V2 as the encoded version for aliastypeparams.
   467  	version := pkgbits.V2
   468  	l := linker{
   469  		pw: pkgbits.NewPkgEncoder(version, base.Debug.SyncFrames),
   470  
   471  		pkgs:   make(map[string]index),
   472  		decls:  make(map[*types.Sym]index),
   473  		bodies: make(map[*types.Sym]index),
   474  	}
   475  
   476  	publicRootWriter := l.pw.NewEncoder(pkgbits.SectionMeta, pkgbits.SyncPublic)
   477  	privateRootWriter := l.pw.NewEncoder(pkgbits.SectionMeta, pkgbits.SyncPrivate)
   478  	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)
   479  	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)
   480  
   481  	var selfPkgIdx index
   482  
   483  	{
   484  		pr := localPkgReader
   485  		r := pr.NewDecoder(pkgbits.SectionMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
   486  
   487  		r.Sync(pkgbits.SyncPkg)
   488  		selfPkgIdx = l.relocIdx(pr, pkgbits.SectionPkg, r.Reloc(pkgbits.SectionPkg))
   489  
   490  		if r.Version().Has(pkgbits.HasInit) {
   491  			r.Bool()
   492  		}
   493  
   494  		for i, n := 0, r.Len(); i < n; i++ {
   495  			r.Sync(pkgbits.SyncObject)
   496  			if r.Version().Has(pkgbits.DerivedFuncInstance) {
   497  				assert(!r.Bool())
   498  			}
   499  			idx := r.Reloc(pkgbits.SectionObj)
   500  			assert(r.Len() == 0)
   501  
   502  			xpath, xname, xtag := pr.PeekObj(idx)
   503  			assert(xpath == pr.PkgPath())
   504  			assert(xtag != pkgbits.ObjStub)
   505  
   506  			if types.IsExported(xname) {
   507  				l.relocIdx(pr, pkgbits.SectionObj, idx)
   508  			}
   509  		}
   510  
   511  		r.Sync(pkgbits.SyncEOF)
   512  	}
   513  
   514  	{
   515  		var idxs []index
   516  		for _, idx := range l.decls {
   517  			idxs = append(idxs, idx)
   518  		}
   519  		slices.Sort(idxs)
   520  
   521  		w := publicRootWriter
   522  
   523  		w.Sync(pkgbits.SyncPkg)
   524  		w.Reloc(pkgbits.SectionPkg, selfPkgIdx)
   525  
   526  		if w.Version().Has(pkgbits.HasInit) {
   527  			w.Bool(false)
   528  		}
   529  
   530  		w.Len(len(idxs))
   531  		for _, idx := range idxs {
   532  			w.Sync(pkgbits.SyncObject)
   533  			if w.Version().Has(pkgbits.DerivedFuncInstance) {
   534  				w.Bool(false)
   535  			}
   536  			w.Reloc(pkgbits.SectionObj, idx)
   537  			w.Len(0)
   538  		}
   539  
   540  		w.Sync(pkgbits.SyncEOF)
   541  		w.Flush()
   542  	}
   543  
   544  	{
   545  		type symIdx struct {
   546  			sym *types.Sym
   547  			idx index
   548  		}
   549  		var bodies []symIdx
   550  		for sym, idx := range l.bodies {
   551  			bodies = append(bodies, symIdx{sym, idx})
   552  		}
   553  		slices.SortFunc(bodies, func(a, b symIdx) int { return cmp.Compare(a.idx, b.idx) })
   554  
   555  		w := privateRootWriter
   556  
   557  		w.Bool(typecheck.Lookup(".inittask").Def != nil)
   558  
   559  		w.Len(len(bodies))
   560  		for _, body := range bodies {
   561  			w.String(body.sym.Pkg.Path)
   562  			w.String(body.sym.Name)
   563  			w.Reloc(pkgbits.SectionBody, body.idx)
   564  		}
   565  
   566  		w.Sync(pkgbits.SyncEOF)
   567  		w.Flush()
   568  	}
   569  
   570  	base.Ctxt.Fingerprint = l.pw.DumpTo(out)
   571  }
   572
View as plain text