buildtool.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Build toolchain using Go bootstrap version.
     6  //
     7  // The general strategy is to copy the source files we need into
     8  // a new GOPATH workspace, adjust import paths appropriately,
     9  // invoke the Go bootstrap toolchains go command to build those sources,
    10  // and then copy the binaries back.
    11  
    12  package main
    13  
    14  import (
    15  	"fmt"
    16  	"go/version"
    17  	"os"
    18  	"path/filepath"
    19  	"regexp"
    20  	"strings"
    21  )
    22  
    23  // bootstrapDirs is a list of directories holding code that must be
    24  // compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
    25  // All directories in this list are relative to and must be below $GOROOT/src.
    26  //
    27  // The list has two kinds of entries: names beginning with cmd/ with
    28  // no other slashes, which are commands, and other paths, which are packages
    29  // supporting the commands. Packages in the standard library can be listed
    30  // if a newer copy needs to be substituted for the Go bootstrap copy when used
    31  // by the command packages. Paths ending with /... automatically
    32  // include all packages within subdirectories as well.
    33  // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
    34  var bootstrapDirs = []string{
    35  	"cmp",
    36  	"cmd/asm",
    37  	"cmd/asm/internal/...",
    38  	"cmd/cgo",
    39  	"cmd/compile",
    40  	"cmd/compile/internal/...",
    41  	"cmd/internal/archive",
    42  	"cmd/internal/bio",
    43  	"cmd/internal/codesign",
    44  	"cmd/internal/dwarf",
    45  	"cmd/internal/edit",
    46  	"cmd/internal/gcprog",
    47  	"cmd/internal/goobj",
    48  	"cmd/internal/hash",
    49  	"cmd/internal/macho",
    50  	"cmd/internal/obj/...",
    51  	"cmd/internal/objabi",
    52  	"cmd/internal/par",
    53  	"cmd/internal/pgo",
    54  	"cmd/internal/pkgpath",
    55  	"cmd/internal/quoted",
    56  	"cmd/internal/src",
    57  	"cmd/internal/sys",
    58  	"cmd/internal/telemetry",
    59  	"cmd/internal/telemetry/counter",
    60  	"cmd/link",
    61  	"cmd/link/internal/...",
    62  	"compress/flate",
    63  	"compress/zlib",
    64  	"container/heap",
    65  	"debug/dwarf",
    66  	"debug/elf",
    67  	"debug/macho",
    68  	"debug/pe",
    69  	"go/build/constraint",
    70  	"go/constant",
    71  	"go/version",
    72  	"internal/abi",
    73  	"internal/coverage",
    74  	"cmd/internal/cov/covcmd",
    75  	"internal/bisect",
    76  	"internal/buildcfg",
    77  	"internal/exportdata",
    78  	"internal/goarch",
    79  	"internal/godebugs",
    80  	"internal/goexperiment",
    81  	"internal/goroot",
    82  	"internal/gover",
    83  	"internal/goversion",
    84  	// internal/lazyregexp is provided by Go 1.17, which permits it to
    85  	// be imported by other packages in this list, but is not provided
    86  	// by the Go 1.17 version of gccgo. It's on this list only to
    87  	// support gccgo, and can be removed if we require gccgo 14 or later.
    88  	"internal/lazyregexp",
    89  	"internal/pkgbits",
    90  	"internal/platform",
    91  	"internal/profile",
    92  	"internal/race",
    93  	"internal/saferio",
    94  	"internal/syscall/unix",
    95  	"internal/types/errors",
    96  	"internal/unsafeheader",
    97  	"internal/xcoff",
    98  	"internal/zstd",
    99  	"math/bits",
   100  	"sort",
   101  }
   102  
   103  // File prefixes that are ignored by go/build anyway, and cause
   104  // problems with editor generated temporary files (#18931).
   105  var ignorePrefixes = []string{
   106  	".",
   107  	"_",
   108  	"#",
   109  }
   110  
   111  // File suffixes that use build tags introduced since Go 1.17.
   112  // These must not be copied into the bootstrap build directory.
   113  // Also ignore test files.
   114  var ignoreSuffixes = []string{
   115  	"_test.s",
   116  	"_test.go",
   117  	// Skip PGO profile. No need to build toolchain1 compiler
   118  	// with PGO. And as it is not a text file the import path
   119  	// rewrite will break it.
   120  	".pgo",
   121  	// Skip editor backup files.
   122  	"~",
   123  }
   124  
   125  const minBootstrap = "go1.24.6"
   126  
   127  var tryDirs = []string{
   128  	"sdk/" + minBootstrap,
   129  	minBootstrap,
   130  }
   131  
   132  func bootstrapBuildTools() {
   133  	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
   134  	if goroot_bootstrap == "" {
   135  		home := os.Getenv("HOME")
   136  		goroot_bootstrap = pathf("%s/go1.4", home)
   137  		for _, d := range tryDirs {
   138  			if p := pathf("%s/%s", home, d); isdir(p) {
   139  				goroot_bootstrap = p
   140  			}
   141  		}
   142  	}
   143  
   144  	// check bootstrap version.
   145  	ver := run(pathf("%s/bin", goroot_bootstrap), CheckExit, pathf("%s/bin/go", goroot_bootstrap), "env", "GOVERSION")
   146  	// go env GOVERSION output like "go1.22.6\n" or "devel go1.24-ffb3e574 Thu Aug 29 20:16:26 2024 +0000\n".
   147  	ver = ver[:len(ver)-1]
   148  	if version.Compare(ver, version.Lang(minBootstrap)) > 0 && version.Compare(ver, minBootstrap) < 0 {
   149  		fatalf("%s does not meet the minimum bootstrap requirement of %s or later", ver, minBootstrap)
   150  	}
   151  
   152  	xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
   153  
   154  	mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
   155  	mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
   156  
   157  	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
   158  	// We use a subdirectory of $GOROOT/pkg because that's the
   159  	// space within $GOROOT where we store all generated objects.
   160  	// We could use a temporary directory outside $GOROOT instead,
   161  	// but it is easier to debug on failure if the files are in a known location.
   162  	workspace := pathf("%s/pkg/bootstrap", goroot)
   163  	xremoveall(workspace)
   164  	xatexit(func() { xremoveall(workspace) })
   165  	base := pathf("%s/src/bootstrap", workspace)
   166  	xmkdirall(base)
   167  
   168  	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
   169  	minBootstrapVers := requiredBootstrapVersion(goModVersion()) // require the minimum required go version to build this go version in the go.mod file
   170  	writefile("module bootstrap\ngo "+minBootstrapVers+"\n", pathf("%s/%s", base, "go.mod"), 0)
   171  	for _, dir := range bootstrapDirs {
   172  		recurse := strings.HasSuffix(dir, "/...")
   173  		dir = strings.TrimSuffix(dir, "/...")
   174  		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
   175  			if err != nil {
   176  				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
   177  			}
   178  
   179  			name := filepath.Base(path)
   180  			src := pathf("%s/src/%s", goroot, path)
   181  			dst := pathf("%s/%s", base, path)
   182  
   183  			if info.IsDir() {
   184  				if !recurse && path != dir || name == "testdata" {
   185  					return filepath.SkipDir
   186  				}
   187  
   188  				xmkdirall(dst)
   189  				if path == "cmd/cgo" {
   190  					// Write to src because we need the file both for bootstrap
   191  					// and for later in the main build.
   192  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
   193  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
   194  				}
   195  				return nil
   196  			}
   197  
   198  			for _, pre := range ignorePrefixes {
   199  				if strings.HasPrefix(name, pre) {
   200  					return nil
   201  				}
   202  			}
   203  			for _, suf := range ignoreSuffixes {
   204  				if strings.HasSuffix(name, suf) {
   205  					return nil
   206  				}
   207  			}
   208  
   209  			text := bootstrapRewriteFile(src)
   210  			writefile(text, dst, 0)
   211  			return nil
   212  		})
   213  	}
   214  
   215  	// Set up environment for invoking Go bootstrap toolchains go command.
   216  	// GOROOT points at Go bootstrap GOROOT,
   217  	// GOPATH points at our bootstrap workspace,
   218  	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
   219  	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
   220  	// so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
   221  	// Restore GOROOT, GOPATH, and GOBIN when done.
   222  	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
   223  	// because setup will take care of those when bootstrapBuildTools returns.
   224  
   225  	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
   226  	os.Setenv("GOROOT", goroot_bootstrap)
   227  
   228  	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
   229  	os.Setenv("GOPATH", workspace)
   230  
   231  	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
   232  	os.Setenv("GOBIN", "")
   233  
   234  	os.Setenv("GOOS", "")
   235  	os.Setenv("GOHOSTOS", "")
   236  	os.Setenv("GOARCH", "")
   237  	os.Setenv("GOHOSTARCH", "")
   238  
   239  	// Run Go bootstrap to build binaries.
   240  	// Use the math_big_pure_go build tag to disable the assembly in math/big
   241  	// which may contain unsupported instructions.
   242  	// Use the purego build tag to disable other assembly code.
   243  	cmd := []string{
   244  		pathf("%s/bin/go", goroot_bootstrap),
   245  		"install",
   246  		"-tags=math_big_pure_go compiler_bootstrap purego",
   247  	}
   248  	if vflag > 0 {
   249  		cmd = append(cmd, "-v")
   250  	}
   251  	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
   252  		cmd = append(cmd, "-toolexec="+tool)
   253  	}
   254  	cmd = append(cmd, "bootstrap/cmd/...")
   255  	run(base, ShowOutput|CheckExit, cmd...)
   256  
   257  	// Copy binaries into tool binary directory.
   258  	for _, name := range bootstrapDirs {
   259  		if !strings.HasPrefix(name, "cmd/") {
   260  			continue
   261  		}
   262  		name = name[len("cmd/"):]
   263  		if !strings.Contains(name, "/") {
   264  			copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
   265  		}
   266  	}
   267  
   268  	if vflag > 0 {
   269  		xprintf("\n")
   270  	}
   271  }
   272  
   273  var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
   274  
   275  // isUnneededSSARewriteFile reports whether srcFile is a
   276  // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
   277  // architecture that isn't for the given GOARCH.
   278  //
   279  // When unneeded is true archCaps is the rewrite base filename without
   280  // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
   281  func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
   282  	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
   283  		return "", false
   284  	}
   285  	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
   286  	if fileArch == "" {
   287  		return "", false
   288  	}
   289  	b := fileArch[0]
   290  	if b == '_' || ('a' <= b && b <= 'z') {
   291  		return "", false
   292  	}
   293  	archCaps = fileArch
   294  	fileArch = strings.ToLower(fileArch)
   295  	fileArch = strings.TrimSuffix(fileArch, "splitload")
   296  	fileArch = strings.TrimSuffix(fileArch, "latelower")
   297  	if fileArch == goArch {
   298  		return "", false
   299  	}
   300  	if fileArch == strings.TrimSuffix(goArch, "le") {
   301  		return "", false
   302  	}
   303  	return archCaps, true
   304  }
   305  
   306  func bootstrapRewriteFile(srcFile string) string {
   307  	// During bootstrap, generate dummy rewrite files for
   308  	// irrelevant architectures. We only need to build a bootstrap
   309  	// binary that works for the current gohostarch.
   310  	// This saves 6+ seconds of bootstrap.
   311  	if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
   312  		return fmt.Sprintf(`%spackage ssa
   313  
   314  func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
   315  func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
   316  `, generatedHeader, archCaps, archCaps)
   317  	}
   318  
   319  	return bootstrapFixImports(srcFile)
   320  }
   321  
   322  var (
   323  	importRE      = regexp.MustCompile(`\Aimport\s+(\.|[A-Za-z0-9_]+)?\s*"([^"]+)"\s*(//.*)?\n\z`)
   324  	importBlockRE = regexp.MustCompile(`\A\s*(?:(\.|[A-Za-z0-9_]+)?\s*"([^"]+)")?\s*(//.*)?\n\z`)
   325  )
   326  
   327  func bootstrapFixImports(srcFile string) string {
   328  	text := readfile(srcFile)
   329  	lines := strings.SplitAfter(text, "\n")
   330  	inBlock := false
   331  	inComment := false
   332  	for i, line := range lines {
   333  		if strings.HasSuffix(line, "*/\n") {
   334  			inComment = false
   335  		}
   336  		if strings.HasSuffix(line, "/*\n") {
   337  			inComment = true
   338  		}
   339  		if inComment {
   340  			continue
   341  		}
   342  		if strings.HasPrefix(line, "import (") {
   343  			inBlock = true
   344  			continue
   345  		}
   346  		if inBlock && strings.HasPrefix(line, ")") {
   347  			inBlock = false
   348  			continue
   349  		}
   350  
   351  		var m []string
   352  		if !inBlock {
   353  			if !strings.HasPrefix(line, "import ") {
   354  				continue
   355  			}
   356  			m = importRE.FindStringSubmatch(line)
   357  			if m == nil {
   358  				fatalf("%s:%d: invalid import declaration: %q", srcFile, i+1, line)
   359  			}
   360  		} else {
   361  			m = importBlockRE.FindStringSubmatch(line)
   362  			if m == nil {
   363  				fatalf("%s:%d: invalid import block line", srcFile, i+1)
   364  			}
   365  			if m[2] == "" {
   366  				continue
   367  			}
   368  		}
   369  
   370  		path := m[2]
   371  		if strings.HasPrefix(path, "cmd/") {
   372  			path = "bootstrap/" + path
   373  		} else {
   374  			for _, dir := range bootstrapDirs {
   375  				if path == dir {
   376  					path = "bootstrap/" + dir
   377  					break
   378  				}
   379  			}
   380  		}
   381  
   382  		// Rewrite use of internal/reflectlite to be plain reflect.
   383  		if path == "internal/reflectlite" {
   384  			lines[i] = strings.ReplaceAll(line, `"reflect"`, `reflectlite "reflect"`)
   385  			continue
   386  		}
   387  
   388  		// Otherwise, reject direct imports of internal packages,
   389  		// since that implies knowledge of internal details that might
   390  		// change from one bootstrap toolchain to the next.
   391  		// There are many internal packages that are listed in
   392  		// bootstrapDirs and made into bootstrap copies based on the
   393  		// current repo's source code. Those are fine; this is catching
   394  		// references to internal packages in the older bootstrap toolchain.
   395  		if strings.HasPrefix(path, "internal/") {
   396  			fatalf("%s:%d: bootstrap-copied source file cannot import %s", srcFile, i+1, path)
   397  		}
   398  		if path != m[2] {
   399  			lines[i] = strings.ReplaceAll(line, `"`+m[2]+`"`, `"`+path+`"`)
   400  		}
   401  	}
   402  
   403  	lines[0] = generatedHeader + "// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
   404  
   405  	return strings.Join(lines, "")
   406  }
   407
View as plain text