buildtool.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Build toolchain using Go bootstrap version.
     6  //
     7  // The general strategy is to copy the source files we need into
     8  // a new GOPATH workspace, adjust import paths appropriately,
     9  // invoke the Go bootstrap toolchains go command to build those sources,
    10  // and then copy the binaries back.
    11  
    12  package main
    13  
    14  import (
    15  	"fmt"
    16  	"go/version"
    17  	"os"
    18  	"path/filepath"
    19  	"regexp"
    20  	"strings"
    21  )
    22  
    23  // bootstrapDirs is a list of directories holding code that must be
    24  // compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
    25  // All directories in this list are relative to and must be below $GOROOT/src.
    26  //
    27  // The list has two kinds of entries: names beginning with cmd/ with
    28  // no other slashes, which are commands, and other paths, which are packages
    29  // supporting the commands. Packages in the standard library can be listed
    30  // if a newer copy needs to be substituted for the Go bootstrap copy when used
    31  // by the command packages. Paths ending with /... automatically
    32  // include all packages within subdirectories as well.
    33  // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
    34  var bootstrapDirs = []string{
    35  	"cmp",
    36  	"cmd/asm",
    37  	"cmd/asm/internal/...",
    38  	"cmd/cgo",
    39  	"cmd/compile",
    40  	"cmd/compile/internal/...",
    41  	"cmd/internal/archive",
    42  	"cmd/internal/bio",
    43  	"cmd/internal/codesign",
    44  	"cmd/internal/dwarf",
    45  	"cmd/internal/edit",
    46  	"cmd/internal/gcprog",
    47  	"cmd/internal/goobj",
    48  	"cmd/internal/hash",
    49  	"cmd/internal/macho",
    50  	"cmd/internal/obj/...",
    51  	"cmd/internal/objabi",
    52  	"cmd/internal/par",
    53  	"cmd/internal/pgo",
    54  	"cmd/internal/pkgpath",
    55  	"cmd/internal/quoted",
    56  	"cmd/internal/src",
    57  	"cmd/internal/sys",
    58  	"cmd/internal/telemetry",
    59  	"cmd/internal/telemetry/counter",
    60  	"cmd/link",
    61  	"cmd/link/internal/...",
    62  	"compress/flate",
    63  	"compress/zlib",
    64  	"container/heap",
    65  	"debug/dwarf",
    66  	"debug/elf",
    67  	"debug/macho",
    68  	"debug/pe",
    69  	"go/build/constraint",
    70  	"go/constant",
    71  	"go/version",
    72  	"internal/abi",
    73  	"internal/coverage",
    74  	"cmd/internal/cov/covcmd",
    75  	"internal/bisect",
    76  	"internal/buildcfg",
    77  	"internal/exportdata",
    78  	"internal/goarch",
    79  	"internal/godebugs",
    80  	"internal/goexperiment",
    81  	"internal/goroot",
    82  	"internal/gover",
    83  	"internal/goversion",
    84  	// internal/lazyregexp is provided by Go 1.17, which permits it to
    85  	// be imported by other packages in this list, but is not provided
    86  	// by the Go 1.17 version of gccgo. It's on this list only to
    87  	// support gccgo, and can be removed if we require gccgo 14 or later.
    88  	"internal/lazyregexp",
    89  	"internal/pkgbits",
    90  	"internal/platform",
    91  	"internal/profile",
    92  	"internal/race",
    93  	"internal/runtime/gc",
    94  	"internal/saferio",
    95  	"internal/syscall/unix",
    96  	"internal/types/errors",
    97  	"internal/unsafeheader",
    98  	"internal/xcoff",
    99  	"internal/zstd",
   100  	"math/bits",
   101  	"sort",
   102  }
   103  
   104  // File prefixes that are ignored by go/build anyway, and cause
   105  // problems with editor generated temporary files (#18931).
   106  var ignorePrefixes = []string{
   107  	".",
   108  	"_",
   109  	"#",
   110  }
   111  
   112  // File suffixes that use build tags introduced since Go 1.17.
   113  // These must not be copied into the bootstrap build directory.
   114  // Also ignore test files.
   115  var ignoreSuffixes = []string{
   116  	"_test.s",
   117  	"_test.go",
   118  	// Skip PGO profile. No need to build toolchain1 compiler
   119  	// with PGO. And as it is not a text file the import path
   120  	// rewrite will break it.
   121  	".pgo",
   122  	// Skip editor backup files.
   123  	"~",
   124  }
   125  
   126  const minBootstrap = "go1.24.6"
   127  
   128  var tryDirs = []string{
   129  	"sdk/" + minBootstrap,
   130  	minBootstrap,
   131  }
   132  
   133  func bootstrapBuildTools() {
   134  	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
   135  	if goroot_bootstrap == "" {
   136  		home := os.Getenv("HOME")
   137  		goroot_bootstrap = pathf("%s/go1.4", home)
   138  		for _, d := range tryDirs {
   139  			if p := pathf("%s/%s", home, d); isdir(p) {
   140  				goroot_bootstrap = p
   141  			}
   142  		}
   143  	}
   144  
   145  	// check bootstrap version.
   146  	ver := run(pathf("%s/bin", goroot_bootstrap), CheckExit, pathf("%s/bin/go", goroot_bootstrap), "env", "GOVERSION")
   147  	// go env GOVERSION output like "go1.22.6\n" or "devel go1.24-ffb3e574 Thu Aug 29 20:16:26 2024 +0000\n".
   148  	ver = ver[:len(ver)-1]
   149  	if version.Compare(ver, version.Lang(minBootstrap)) > 0 && version.Compare(ver, minBootstrap) < 0 {
   150  		fatalf("%s does not meet the minimum bootstrap requirement of %s or later", ver, minBootstrap)
   151  	}
   152  
   153  	xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
   154  
   155  	mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
   156  	mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
   157  
   158  	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
   159  	// We use a subdirectory of $GOROOT/pkg because that's the
   160  	// space within $GOROOT where we store all generated objects.
   161  	// We could use a temporary directory outside $GOROOT instead,
   162  	// but it is easier to debug on failure if the files are in a known location.
   163  	workspace := pathf("%s/pkg/bootstrap", goroot)
   164  	xremoveall(workspace)
   165  	xatexit(func() { xremoveall(workspace) })
   166  	base := pathf("%s/src/bootstrap", workspace)
   167  	xmkdirall(base)
   168  
   169  	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
   170  	minBootstrapVers := requiredBootstrapVersion(goModVersion()) // require the minimum required go version to build this go version in the go.mod file
   171  	writefile("module bootstrap\ngo "+minBootstrapVers+"\n", pathf("%s/%s", base, "go.mod"), 0)
   172  	for _, dir := range bootstrapDirs {
   173  		recurse := strings.HasSuffix(dir, "/...")
   174  		dir = strings.TrimSuffix(dir, "/...")
   175  		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
   176  			if err != nil {
   177  				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
   178  			}
   179  
   180  			name := filepath.Base(path)
   181  			src := pathf("%s/src/%s", goroot, path)
   182  			dst := pathf("%s/%s", base, path)
   183  
   184  			if info.IsDir() {
   185  				if !recurse && path != dir || name == "testdata" {
   186  					return filepath.SkipDir
   187  				}
   188  
   189  				xmkdirall(dst)
   190  				if path == "cmd/cgo" {
   191  					// Write to src because we need the file both for bootstrap
   192  					// and for later in the main build.
   193  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
   194  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
   195  				}
   196  				return nil
   197  			}
   198  
   199  			for _, pre := range ignorePrefixes {
   200  				if strings.HasPrefix(name, pre) {
   201  					return nil
   202  				}
   203  			}
   204  			for _, suf := range ignoreSuffixes {
   205  				if strings.HasSuffix(name, suf) {
   206  					return nil
   207  				}
   208  			}
   209  
   210  			text := bootstrapRewriteFile(src)
   211  			writefile(text, dst, 0)
   212  			return nil
   213  		})
   214  	}
   215  
   216  	// Set up environment for invoking Go bootstrap toolchains go command.
   217  	// GOROOT points at Go bootstrap GOROOT,
   218  	// GOPATH points at our bootstrap workspace,
   219  	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
   220  	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
   221  	// so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
   222  	// Restore GOROOT, GOPATH, and GOBIN when done.
   223  	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
   224  	// because setup will take care of those when bootstrapBuildTools returns.
   225  
   226  	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
   227  	os.Setenv("GOROOT", goroot_bootstrap)
   228  
   229  	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
   230  	os.Setenv("GOPATH", workspace)
   231  
   232  	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
   233  	os.Setenv("GOBIN", "")
   234  
   235  	os.Setenv("GOOS", "")
   236  	os.Setenv("GOHOSTOS", "")
   237  	os.Setenv("GOARCH", "")
   238  	os.Setenv("GOHOSTARCH", "")
   239  
   240  	// Run Go bootstrap to build binaries.
   241  	// Use the math_big_pure_go build tag to disable the assembly in math/big
   242  	// which may contain unsupported instructions.
   243  	// Use the purego build tag to disable other assembly code.
   244  	cmd := []string{
   245  		pathf("%s/bin/go", goroot_bootstrap),
   246  		"install",
   247  		"-tags=math_big_pure_go compiler_bootstrap purego",
   248  	}
   249  	if vflag > 0 {
   250  		cmd = append(cmd, "-v")
   251  	}
   252  	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
   253  		cmd = append(cmd, "-toolexec="+tool)
   254  	}
   255  	cmd = append(cmd, "bootstrap/cmd/...")
   256  	run(base, ShowOutput|CheckExit, cmd...)
   257  
   258  	// Copy binaries into tool binary directory.
   259  	for _, name := range bootstrapDirs {
   260  		if !strings.HasPrefix(name, "cmd/") {
   261  			continue
   262  		}
   263  		name = name[len("cmd/"):]
   264  		if !strings.Contains(name, "/") {
   265  			copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
   266  		}
   267  	}
   268  
   269  	if vflag > 0 {
   270  		xprintf("\n")
   271  	}
   272  }
   273  
   274  var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
   275  
   276  // isUnneededSSARewriteFile reports whether srcFile is a
   277  // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
   278  // architecture that isn't for the given GOARCH.
   279  //
   280  // When unneeded is true archCaps is the rewrite base filename without
   281  // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
   282  func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
   283  	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
   284  		return "", false
   285  	}
   286  	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
   287  	if fileArch == "" {
   288  		return "", false
   289  	}
   290  	b := fileArch[0]
   291  	if b == '_' || ('a' <= b && b <= 'z') {
   292  		return "", false
   293  	}
   294  	archCaps = fileArch
   295  	fileArch = strings.ToLower(fileArch)
   296  	fileArch = strings.TrimSuffix(fileArch, "splitload")
   297  	fileArch = strings.TrimSuffix(fileArch, "latelower")
   298  	if fileArch == goArch {
   299  		return "", false
   300  	}
   301  	if fileArch == strings.TrimSuffix(goArch, "le") {
   302  		return "", false
   303  	}
   304  	return archCaps, true
   305  }
   306  
   307  func bootstrapRewriteFile(srcFile string) string {
   308  	// During bootstrap, generate dummy rewrite files for
   309  	// irrelevant architectures. We only need to build a bootstrap
   310  	// binary that works for the current gohostarch.
   311  	// This saves 6+ seconds of bootstrap.
   312  	if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
   313  		return fmt.Sprintf(`%spackage ssa
   314  
   315  func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
   316  func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
   317  `, generatedHeader, archCaps, archCaps)
   318  	}
   319  
   320  	return bootstrapFixImports(srcFile)
   321  }
   322  
   323  var (
   324  	importRE      = regexp.MustCompile(`\Aimport\s+(\.|[A-Za-z0-9_]+)?\s*"([^"]+)"\s*(//.*)?\n\z`)
   325  	importBlockRE = regexp.MustCompile(`\A\s*(?:(\.|[A-Za-z0-9_]+)?\s*"([^"]+)")?\s*(//.*)?\n\z`)
   326  )
   327  
   328  func bootstrapFixImports(srcFile string) string {
   329  	text := readfile(srcFile)
   330  	lines := strings.SplitAfter(text, "\n")
   331  	inBlock := false
   332  	inComment := false
   333  	for i, line := range lines {
   334  		if strings.HasSuffix(line, "*/\n") {
   335  			inComment = false
   336  		}
   337  		if strings.HasSuffix(line, "/*\n") {
   338  			inComment = true
   339  		}
   340  		if inComment {
   341  			continue
   342  		}
   343  		if strings.HasPrefix(line, "import (") {
   344  			inBlock = true
   345  			continue
   346  		}
   347  		if inBlock && strings.HasPrefix(line, ")") {
   348  			inBlock = false
   349  			continue
   350  		}
   351  
   352  		var m []string
   353  		if !inBlock {
   354  			if !strings.HasPrefix(line, "import ") {
   355  				continue
   356  			}
   357  			m = importRE.FindStringSubmatch(line)
   358  			if m == nil {
   359  				fatalf("%s:%d: invalid import declaration: %q", srcFile, i+1, line)
   360  			}
   361  		} else {
   362  			m = importBlockRE.FindStringSubmatch(line)
   363  			if m == nil {
   364  				fatalf("%s:%d: invalid import block line", srcFile, i+1)
   365  			}
   366  			if m[2] == "" {
   367  				continue
   368  			}
   369  		}
   370  
   371  		path := m[2]
   372  		if strings.HasPrefix(path, "cmd/") {
   373  			path = "bootstrap/" + path
   374  		} else {
   375  			for _, dir := range bootstrapDirs {
   376  				if path == dir {
   377  					path = "bootstrap/" + dir
   378  					break
   379  				}
   380  			}
   381  		}
   382  
   383  		// Rewrite use of internal/reflectlite to be plain reflect.
   384  		if path == "internal/reflectlite" {
   385  			lines[i] = strings.ReplaceAll(line, `"reflect"`, `reflectlite "reflect"`)
   386  			continue
   387  		}
   388  
   389  		// Otherwise, reject direct imports of internal packages,
   390  		// since that implies knowledge of internal details that might
   391  		// change from one bootstrap toolchain to the next.
   392  		// There are many internal packages that are listed in
   393  		// bootstrapDirs and made into bootstrap copies based on the
   394  		// current repo's source code. Those are fine; this is catching
   395  		// references to internal packages in the older bootstrap toolchain.
   396  		if strings.HasPrefix(path, "internal/") {
   397  			fatalf("%s:%d: bootstrap-copied source file cannot import %s", srcFile, i+1, path)
   398  		}
   399  		if path != m[2] {
   400  			lines[i] = strings.ReplaceAll(line, `"`+m[2]+`"`, `"`+path+`"`)
   401  		}
   402  	}
   403  
   404  	lines[0] = generatedHeader + "// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
   405  
   406  	return strings.Join(lines, "")
   407  }
   408
View as plain text