Source file src/cmd/dist/buildtool.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Build toolchain using Go bootstrap version.
     6  //
     7  // The general strategy is to copy the source files we need into
     8  // a new GOPATH workspace, adjust import paths appropriately,
     9  // invoke the Go bootstrap toolchains go command to build those sources,
    10  // and then copy the binaries back.
    11  
    12  package main
    13  
    14  import (
    15  	"fmt"
    16  	"go/version"
    17  	"os"
    18  	"path/filepath"
    19  	"regexp"
    20  	"strings"
    21  )
    22  
    23  // bootstrapDirs is a list of directories holding code that must be
    24  // compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
    25  // All directories in this list are relative to and must be below $GOROOT/src.
    26  //
    27  // The list has two kinds of entries: names beginning with cmd/ with
    28  // no other slashes, which are commands, and other paths, which are packages
    29  // supporting the commands. Packages in the standard library can be listed
    30  // if a newer copy needs to be substituted for the Go bootstrap copy when used
    31  // by the command packages. Paths ending with /... automatically
    32  // include all packages within subdirectories as well.
    33  // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
    34  // We also import packages in cmd/vendor/name as bootstrap/name,
    35  // like bootstrap/golang.org/x/mod/zip.
    36  var bootstrapDirs = []string{
    37  	"cmp",
    38  	"cmd/asm",
    39  	"cmd/asm/internal/...",
    40  	"cmd/cgo",
    41  	"cmd/compile",
    42  	"cmd/compile/internal/...",
    43  	"cmd/go",
    44  	"cmd/go/internal/base",
    45  	"cmd/go/internal/bug",
    46  	"cmd/go/internal/build",
    47  	"cmd/go/internal/cache",
    48  	"cmd/go/internal/cacheprog",
    49  	"cmd/go/internal/cfg",
    50  	"cmd/go/internal/clean",
    51  	"cmd/go/internal/cmdflag",
    52  	"cmd/go/internal/doc",
    53  	"cmd/go/internal/envcmd",
    54  	"cmd/go/internal/fips140",
    55  	"cmd/go/internal/fmtcmd",
    56  	"cmd/go/internal/fsys",
    57  	"cmd/go/internal/generate",
    58  	"cmd/go/internal/gover",
    59  	"cmd/go/internal/help",
    60  	"cmd/go/internal/imports",
    61  	"cmd/go/internal/list",
    62  	"cmd/go/internal/load",
    63  	"cmd/go/internal/lockedfile",
    64  	"cmd/go/internal/lockedfile/internal/filelock",
    65  	"cmd/go/internal/mmap",
    66  	"cmd/go/internal/modcmd",
    67  	"cmd/go/internal/modfetch",
    68  	"cmd/go/internal/modfetch/codehost",
    69  	"cmd/go/internal/modget",
    70  	"cmd/go/internal/modindex",
    71  	"cmd/go/internal/modinfo",
    72  	"cmd/go/internal/modload",
    73  	"cmd/go/internal/mvs",
    74  	"cmd/go/internal/run",
    75  	"cmd/go/internal/search",
    76  	"cmd/go/internal/str",
    77  	"cmd/go/internal/telemetrycmd",
    78  	"cmd/go/internal/telemetrystats",
    79  	"cmd/go/internal/test",
    80  	"cmd/go/internal/tool",
    81  	"cmd/go/internal/toolchain",
    82  	"cmd/go/internal/trace",
    83  	"cmd/go/internal/vcs",
    84  	"cmd/go/internal/version",
    85  	"cmd/go/internal/vet",
    86  	"cmd/go/internal/web",
    87  	"cmd/go/internal/work",
    88  	"cmd/go/internal/workcmd",
    89  	"cmd/internal/archive",
    90  	"cmd/internal/bio",
    91  	"cmd/internal/buildid",
    92  	"cmd/internal/codesign",
    93  	"cmd/internal/dwarf",
    94  	"cmd/internal/edit",
    95  	"cmd/internal/gcprog",
    96  	"cmd/internal/goobj",
    97  	"cmd/internal/hash",
    98  	"cmd/internal/macho",
    99  	"cmd/internal/obj/...",
   100  	"cmd/internal/objabi",
   101  	"cmd/internal/par",
   102  	"cmd/internal/pathcache",
   103  	"cmd/internal/pgo",
   104  	"cmd/internal/pkgpath",
   105  	"cmd/internal/pkgpattern",
   106  	"cmd/internal/quoted",
   107  	"cmd/internal/robustio",
   108  	"cmd/internal/src",
   109  	"cmd/internal/sys",
   110  	"cmd/internal/telemetry",
   111  	"cmd/internal/telemetry/counter",
   112  	"cmd/internal/test2json",
   113  	"cmd/link",
   114  	"cmd/link/internal/...",
   115  	"cmd/preprofile",
   116  	"cmd/vendor/golang.org/x/mod/internal/lazyregexp",
   117  	"cmd/vendor/golang.org/x/mod/modfile",
   118  	"cmd/vendor/golang.org/x/mod/module",
   119  	"cmd/vendor/golang.org/x/mod/semver",
   120  	"cmd/vendor/golang.org/x/mod/sumdb/dirhash",
   121  	"cmd/vendor/golang.org/x/mod/zip",
   122  	"cmd/vendor/golang.org/x/sync/semaphore",
   123  	"compress/flate",
   124  	"compress/zlib",
   125  	"container/heap",
   126  	"debug/dwarf",
   127  	"debug/elf",
   128  	"debug/macho",
   129  	"debug/pe",
   130  	"go/build",
   131  	"go/build/constraint",
   132  	"go/constant",
   133  	"go/version",
   134  	"internal/abi",
   135  	"internal/coverage",
   136  	"cmd/internal/cov/covcmd",
   137  	"internal/bisect",
   138  	"internal/buildcfg",
   139  	"internal/cfg",
   140  	"internal/diff",
   141  	"internal/exportdata",
   142  	"internal/goarch",
   143  	"internal/godebugs",
   144  	"internal/goexperiment",
   145  	"internal/goroot",
   146  	"internal/gover",
   147  	"internal/goversion",
   148  	// internal/lazyregexp is provided by Go 1.17, which permits it to
   149  	// be imported by other packages in this list, but is not provided
   150  	// by the Go 1.17 version of gccgo. It's on this list only to
   151  	// support gccgo, and can be removed if we require gccgo 14 or later.
   152  	"internal/lazyregexp",
   153  	"internal/lazytemplate",
   154  	"internal/oserror",
   155  	"internal/pkgbits",
   156  	"internal/platform",
   157  	"internal/profile",
   158  	"internal/race",
   159  	"internal/runtime/gc",
   160  	"internal/saferio",
   161  	"internal/singleflight",
   162  	"internal/strconv",
   163  	"internal/syslist",
   164  	"internal/trace/traceviewer/format",
   165  	"internal/types/errors",
   166  	"internal/unsafeheader",
   167  	"internal/xcoff",
   168  	"internal/zstd",
   169  	"math/bits",
   170  	"sort",
   171  }
   172  
   173  // File prefixes that are ignored by go/build anyway, and cause
   174  // problems with editor generated temporary files (#18931).
   175  var ignorePrefixes = []string{
   176  	".",
   177  	"_",
   178  	"#",
   179  }
   180  
   181  // File suffixes that use build tags introduced since Go 1.17.
   182  // These must not be copied into the bootstrap build directory.
   183  // Also ignore test files.
   184  var ignoreSuffixes = []string{
   185  	"_test.s",
   186  	"_test.go",
   187  	// Skip PGO profile. No need to build toolchain1 compiler
   188  	// with PGO. And as it is not a text file the import path
   189  	// rewrite will break it.
   190  	".pgo",
   191  	// Skip editor backup files.
   192  	"~",
   193  }
   194  
   195  const minBootstrap = "go1.26.0"
   196  
   197  var tryDirs = []string{
   198  	"sdk/" + minBootstrap,
   199  	minBootstrap,
   200  }
   201  
   202  func bootstrapBuildTools() {
   203  	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
   204  	if goroot_bootstrap == "" {
   205  		home := os.Getenv("HOME")
   206  		goroot_bootstrap = pathf("%s/go1.4", home)
   207  		for _, d := range tryDirs {
   208  			if p := pathf("%s/%s", home, d); isdir(p) {
   209  				goroot_bootstrap = p
   210  			}
   211  		}
   212  	}
   213  
   214  	// Unset GOWASM before invoking the bootstrap toolchain, which may not
   215  	// recognize newly added GOWASM features.
   216  	// This must happen before the "go env GOVERSION" check below (rather than
   217  	// in the os.Setenv block further down) because cmd/go eagerly parses
   218  	// GOWASM in internal/buildcfg at startup and "go env" calls
   219  	// buildcfg.Check(), which exits with a fatal error on unknown GOWASM
   220  	// values.
   221  	defer os.Setenv("GOWASM", os.Getenv("GOWASM"))
   222  	os.Setenv("GOWASM", "")
   223  
   224  	// check bootstrap version.
   225  	ver := run(pathf("%s/bin", goroot_bootstrap), CheckExit, pathf("%s/bin/go", goroot_bootstrap), "env", "GOVERSION")
   226  	// go env GOVERSION output like "go1.22.6\n" or "devel go1.24-ffb3e574 Thu Aug 29 20:16:26 2024 +0000\n".
   227  	ver = ver[:len(ver)-1]
   228  	if version.Compare(ver, version.Lang(minBootstrap)) > 0 && version.Compare(ver, minBootstrap) < 0 {
   229  		fatalf("%s does not meet the minimum bootstrap requirement of %s or later", ver, minBootstrap)
   230  	}
   231  
   232  	xprintf("Building Go toolchain1 and bootstrap cmd/go (go_bootstrap) using %s.\n", goroot_bootstrap)
   233  
   234  	mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
   235  	mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
   236  
   237  	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
   238  	// We use a subdirectory of $GOROOT/pkg because that's the
   239  	// space within $GOROOT where we store all generated objects.
   240  	// We could use a temporary directory outside $GOROOT instead,
   241  	// but it is easier to debug on failure if the files are in a known location.
   242  	workspace := pathf("%s/pkg/bootstrap", goroot)
   243  	xremoveall(workspace)
   244  	xatexit(func() { xremoveall(workspace) })
   245  	base := pathf("%s/src/bootstrap", workspace)
   246  	xmkdirall(base)
   247  
   248  	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
   249  	copySpan := startSpan("copy bootstrap sources")
   250  	minBootstrapVers := requiredBootstrapVersion(goModVersion()) // require the minimum required go version to build this go version in the go.mod file
   251  	writefile("module bootstrap\ngo "+minBootstrapVers+"\n", pathf("%s/%s", base, "go.mod"), 0)
   252  	for _, dir := range bootstrapDirs {
   253  		recurse := strings.HasSuffix(dir, "/...")
   254  		dir = strings.TrimSuffix(dir, "/...")
   255  		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
   256  			if err != nil {
   257  				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
   258  			}
   259  
   260  			name := filepath.Base(path)
   261  			src := pathf("%s/src/%s", goroot, path)
   262  			dst := pathf("%s/%s", base, strings.TrimPrefix(filepath.ToSlash(path), "cmd/vendor/"))
   263  
   264  			if info.IsDir() {
   265  				if !recurse && path != dir || name == "testdata" {
   266  					return filepath.SkipDir
   267  				}
   268  
   269  				xmkdirall(dst)
   270  				if path == "cmd/cgo" {
   271  					// Write to src because we need the file both for bootstrap
   272  					// and for later in the main build.
   273  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
   274  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
   275  				}
   276  				return nil
   277  			}
   278  
   279  			for _, pre := range ignorePrefixes {
   280  				if strings.HasPrefix(name, pre) {
   281  					return nil
   282  				}
   283  			}
   284  			for _, suf := range ignoreSuffixes {
   285  				if strings.HasSuffix(name, suf) {
   286  					return nil
   287  				}
   288  			}
   289  
   290  			text := bootstrapRewriteFile(src)
   291  			writefile(text, dst, 0)
   292  			return nil
   293  		})
   294  	}
   295  	copySpan.done()
   296  
   297  	// Set up environment for invoking Go bootstrap toolchains go command.
   298  	// GOROOT points at Go bootstrap GOROOT,
   299  	// GOPATH points at our bootstrap workspace,
   300  	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
   301  	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
   302  	// so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
   303  	// Set GOFIPS140=off to work with the purego build tag.
   304  	// Restore GOROOT, GOPATH, GOBIN, and GOFIPS140 when done.
   305  	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
   306  	// because setup will take care of those when bootstrapBuildTools returns.
   307  
   308  	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
   309  	os.Setenv("GOROOT", goroot_bootstrap)
   310  
   311  	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
   312  	os.Setenv("GOPATH", workspace)
   313  
   314  	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
   315  	os.Setenv("GOBIN", "")
   316  
   317  	defer os.Setenv("GOFIPS140", os.Getenv("GOFIPS140"))
   318  	os.Setenv("GOFIPS140", "off")
   319  
   320  	os.Setenv("GOOS", gohostos)
   321  	os.Setenv("GOHOSTOS", "")
   322  	os.Setenv("GOARCH", gohostarch)
   323  	os.Setenv("GOHOSTARCH", "")
   324  
   325  	// Run Go bootstrap to build binaries.
   326  	bindir := pathf("%s/bin", workspace)
   327  	xmkdirall(bindir)
   328  	cmd := []string{
   329  		pathf("%s/bin/go", goroot_bootstrap),
   330  		"build",
   331  		"-o", bindir,
   332  		"-tags=compiler_bootstrap,cmd_go_bootstrap",
   333  	}
   334  	if vflag > 0 {
   335  		cmd = append(cmd, "-v")
   336  	}
   337  	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
   338  		cmd = append(cmd, "-toolexec="+tool)
   339  	}
   340  	cmd = append(cmd, maybeTraceFlag("toolchain1")...)
   341  	cmd = append(cmd, "bootstrap/cmd/...")
   342  	toolchain1Span := startSpan("toolchain1")
   343  	run(base, ShowOutput|CheckExit, cmd...)
   344  	toolchain1Span.done()
   345  
   346  	// Copy binaries into tool binary directory.
   347  	copyBinSpan := startSpan("copy toolchain1 binaries")
   348  	for _, name := range bootstrapDirs {
   349  		if !strings.HasPrefix(name, "cmd/") {
   350  			continue
   351  		}
   352  		name = name[len("cmd/"):]
   353  		if !strings.Contains(name, "/") {
   354  			tool := name
   355  			if name == "go" {
   356  				tool = "go_bootstrap"
   357  			}
   358  			copyfile(pathf("%s/%s%s", tooldir, tool, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
   359  		}
   360  	}
   361  	copyBinSpan.done()
   362  
   363  	if vflag > 0 {
   364  		xprintf("\n")
   365  	}
   366  }
   367  
   368  var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssacompile/rewrite")
   369  
   370  func init() {
   371  	if ssaRewriteSplitPackages {
   372  		ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
   373  	}
   374  }
   375  
   376  // isUnneededSSARewriteFile reports whether srcFile is a
   377  // src/cmd/compile/internal/ssa/rewrite/ARCH/rewriteARCHNAME.go file or a
   378  // src/cmd/compile/internal/ssacompile/rewriteARCHNAME.go file for an
   379  // architecture that isn't for the given GOARCH.
   380  //
   381  // When unneeded is true archCaps is the rewrite base filename without
   382  // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
   383  func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
   384  	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
   385  		return "", false
   386  	}
   387  	if ssaRewriteSplitPackages && !strings.HasPrefix(filepath.Base(srcFile), "rewrite") {
   388  		// Don't rewrite any helpers in the package.
   389  		// TODO(matloob): we should be able to clear those too, but we'd have
   390  		// to write a file with just a package declaration.
   391  		return "", false
   392  	}
   393  	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
   394  	if fileArch == "" {
   395  		return "", false
   396  	}
   397  	b := fileArch[0]
   398  	if b == '_' || ('a' <= b && b <= 'z') {
   399  		return "", false
   400  	}
   401  	archCaps = fileArch
   402  	fileArch = strings.ToLower(fileArch)
   403  	fileArch = strings.TrimSuffix(fileArch, "splitload")
   404  	fileArch = strings.TrimSuffix(fileArch, "latelower")
   405  	if fileArch == goArch {
   406  		return "", false
   407  	}
   408  	if fileArch == strings.TrimSuffix(goArch, "le") {
   409  		return "", false
   410  	}
   411  	return archCaps, true
   412  }
   413  
   414  func bootstrapRewriteFile(srcFile string) string {
   415  	// During bootstrap, generate dummy rewrite files for
   416  	// irrelevant architectures. We only need to build a bootstrap
   417  	// binary that works for the current gohostarch.
   418  	// This saves 6+ seconds of bootstrap.
   419  	rewrite := "rewrite"
   420  	if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
   421  		if ssaRewriteSplitPackages {
   422  			rewrite = "Rewrite"
   423  			archCaps = "" // Remove redundant arch suffix
   424  		}
   425  		pkg := filepath.Base(filepath.Dir(srcFile))
   426  		return fmt.Sprintf(`%spackage %s
   427  
   428  import "bootstrap/cmd/compile/internal/ssa"
   429  
   430  func %sValue%s(v *ssa.Value) bool { panic("unused during bootstrap") }
   431  func %sBlock%s(b *ssa.Block) bool { panic("unused during bootstrap") }
   432  `, generatedHeader, pkg, rewrite, archCaps, rewrite, archCaps)
   433  	}
   434  
   435  	return bootstrapFixImports(srcFile)
   436  }
   437  
   438  var ssaRewriteSplitPackages = true
   439  
   440  var (
   441  	importRE      = regexp.MustCompile(`\Aimport\s+(\.|[A-Za-z0-9_]+)?\s*"([^"]+)"\s*(//.*)?\n\z`)
   442  	importBlockRE = regexp.MustCompile(`\A\s*(?:(\.|[A-Za-z0-9_]+)?\s*"([^"]+)")?\s*(//.*)?\n\z`)
   443  )
   444  
   445  func bootstrapFixImports(srcFile string) string {
   446  	text := readfile(srcFile)
   447  	lines := strings.SplitAfter(text, "\n")
   448  	inBlock := false
   449  	inComment := false
   450  	sawImport := false
   451  	for i, line := range lines {
   452  		if strings.HasSuffix(line, "*/\n") {
   453  			inComment = false
   454  		}
   455  		if strings.HasSuffix(line, "/*\n") {
   456  			inComment = true
   457  		}
   458  		if inComment {
   459  			continue
   460  		}
   461  		if strings.HasPrefix(line, "import (") {
   462  			inBlock = true
   463  			sawImport = true
   464  			continue
   465  		}
   466  		if inBlock && strings.HasPrefix(line, ")") {
   467  			inBlock = false
   468  			continue
   469  		}
   470  
   471  		var m []string
   472  		if !inBlock {
   473  			if !strings.HasPrefix(line, "import ") {
   474  				if sawImport && strings.TrimSpace(line) != "" && !strings.HasPrefix(line, "//") {
   475  					break
   476  				}
   477  				continue
   478  			}
   479  			sawImport = true
   480  			m = importRE.FindStringSubmatch(line)
   481  			if m == nil {
   482  				fatalf("%s:%d: invalid import declaration: %q", srcFile, i+1, line)
   483  			}
   484  		} else {
   485  			m = importBlockRE.FindStringSubmatch(line)
   486  			if m == nil {
   487  				fatalf("%s:%d: invalid import block line", srcFile, i+1)
   488  			}
   489  			if m[2] == "" {
   490  				continue
   491  			}
   492  		}
   493  
   494  		path := m[2]
   495  		if strings.HasPrefix(path, "cmd/") {
   496  			path = "bootstrap/" + path
   497  		} else {
   498  			for _, dir := range bootstrapDirs {
   499  				if path == dir || "cmd/vendor/"+path == dir {
   500  					path = "bootstrap/" + path
   501  					break
   502  				}
   503  			}
   504  		}
   505  
   506  		// Ignore imports of cmd/go internal dependencies that have linknames.
   507  		// Their behavior won't affect anything meaningful in the bootstrap toolchain.
   508  		// TODO(matloob): it would be even better to stub out their imports.
   509  		if path == "internal/godebug" || path == "internal/syscall/unix" || path == "internal/syscall/windows" {
   510  			continue
   511  		}
   512  
   513  		// Rewrite use of internal/reflectlite to be plain reflect.
   514  		if path == "internal/reflectlite" {
   515  			lines[i] = strings.ReplaceAll(line, `"reflect"`, `reflectlite "reflect"`)
   516  			continue
   517  		}
   518  
   519  		// Otherwise, reject direct imports of internal packages,
   520  		// since that implies knowledge of internal details that might
   521  		// change from one bootstrap toolchain to the next.
   522  		// There are many internal packages that are listed in
   523  		// bootstrapDirs and made into bootstrap copies based on the
   524  		// current repo's source code. Those are fine; this is catching
   525  		// references to internal packages in the older bootstrap toolchain.
   526  		if strings.HasPrefix(path, "internal/") {
   527  			fatalf("%s:%d: bootstrap-copied source file cannot import %s", srcFile, i+1, path)
   528  		}
   529  		if path != m[2] {
   530  			lines[i] = strings.ReplaceAll(line, `"`+m[2]+`"`, `"`+path+`"`)
   531  		}
   532  	}
   533  
   534  	lines[0] = generatedHeader + "// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
   535  
   536  	return strings.Join(lines, "")
   537  }
   538  

View as plain text