buildtool.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Build toolchain using Go bootstrap version.
     6  //
     7  // The general strategy is to copy the source files we need into
     8  // a new GOPATH workspace, adjust import paths appropriately,
     9  // invoke the Go bootstrap toolchains go command to build those sources,
    10  // and then copy the binaries back.
    11  
    12  package main
    13  
    14  import (
    15  	"fmt"
    16  	"os"
    17  	"path/filepath"
    18  	"regexp"
    19  	"strings"
    20  )
    21  
    22  // bootstrapDirs is a list of directories holding code that must be
    23  // compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
    24  // All directories in this list are relative to and must be below $GOROOT/src.
    25  //
    26  // The list has two kinds of entries: names beginning with cmd/ with
    27  // no other slashes, which are commands, and other paths, which are packages
    28  // supporting the commands. Packages in the standard library can be listed
    29  // if a newer copy needs to be substituted for the Go bootstrap copy when used
    30  // by the command packages. Paths ending with /... automatically
    31  // include all packages within subdirectories as well.
    32  // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
    33  var bootstrapDirs = []string{
    34  	"cmp",
    35  	"cmd/asm",
    36  	"cmd/asm/internal/...",
    37  	"cmd/cgo",
    38  	"cmd/compile",
    39  	"cmd/compile/internal/...",
    40  	"cmd/internal/archive",
    41  	"cmd/internal/bio",
    42  	"cmd/internal/codesign",
    43  	"cmd/internal/dwarf",
    44  	"cmd/internal/edit",
    45  	"cmd/internal/gcprog",
    46  	"cmd/internal/goobj",
    47  	"cmd/internal/notsha256",
    48  	"cmd/internal/obj/...",
    49  	"cmd/internal/objabi",
    50  	"cmd/internal/pgo",
    51  	"cmd/internal/pkgpath",
    52  	"cmd/internal/quoted",
    53  	"cmd/internal/src",
    54  	"cmd/internal/sys",
    55  	"cmd/link",
    56  	"cmd/link/internal/...",
    57  	"compress/flate",
    58  	"compress/zlib",
    59  	"container/heap",
    60  	"debug/dwarf",
    61  	"debug/elf",
    62  	"debug/macho",
    63  	"debug/pe",
    64  	"go/build/constraint",
    65  	"go/constant",
    66  	"go/version",
    67  	"internal/abi",
    68  	"internal/coverage",
    69  	"cmd/internal/cov/covcmd",
    70  	"internal/bisect",
    71  	"internal/buildcfg",
    72  	"internal/goarch",
    73  	"internal/godebugs",
    74  	"internal/goexperiment",
    75  	"internal/goroot",
    76  	"internal/gover",
    77  	"internal/goversion",
    78  	// internal/lazyregexp is provided by Go 1.17, which permits it to
    79  	// be imported by other packages in this list, but is not provided
    80  	// by the Go 1.17 version of gccgo. It's on this list only to
    81  	// support gccgo, and can be removed if we require gccgo 14 or later.
    82  	"internal/lazyregexp",
    83  	"internal/pkgbits",
    84  	"internal/platform",
    85  	"internal/profile",
    86  	"internal/race",
    87  	"internal/saferio",
    88  	"internal/syscall/unix",
    89  	"internal/types/errors",
    90  	"internal/unsafeheader",
    91  	"internal/xcoff",
    92  	"internal/zstd",
    93  	"math/bits",
    94  	"sort",
    95  }
    96  
    97  // File prefixes that are ignored by go/build anyway, and cause
    98  // problems with editor generated temporary files (#18931).
    99  var ignorePrefixes = []string{
   100  	".",
   101  	"_",
   102  	"#",
   103  }
   104  
   105  // File suffixes that use build tags introduced since Go 1.17.
   106  // These must not be copied into the bootstrap build directory.
   107  // Also ignore test files.
   108  var ignoreSuffixes = []string{
   109  	"_test.s",
   110  	"_test.go",
   111  	// Skip PGO profile. No need to build toolchain1 compiler
   112  	// with PGO. And as it is not a text file the import path
   113  	// rewrite will break it.
   114  	".pgo",
   115  }
   116  
   117  var tryDirs = []string{
   118  	"sdk/go1.17",
   119  	"go1.17",
   120  }
   121  
   122  func bootstrapBuildTools() {
   123  	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
   124  	if goroot_bootstrap == "" {
   125  		home := os.Getenv("HOME")
   126  		goroot_bootstrap = pathf("%s/go1.4", home)
   127  		for _, d := range tryDirs {
   128  			if p := pathf("%s/%s", home, d); isdir(p) {
   129  				goroot_bootstrap = p
   130  			}
   131  		}
   132  	}
   133  	xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
   134  
   135  	mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
   136  	mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
   137  
   138  	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
   139  	// We use a subdirectory of $GOROOT/pkg because that's the
   140  	// space within $GOROOT where we store all generated objects.
   141  	// We could use a temporary directory outside $GOROOT instead,
   142  	// but it is easier to debug on failure if the files are in a known location.
   143  	workspace := pathf("%s/pkg/bootstrap", goroot)
   144  	xremoveall(workspace)
   145  	xatexit(func() { xremoveall(workspace) })
   146  	base := pathf("%s/src/bootstrap", workspace)
   147  	xmkdirall(base)
   148  
   149  	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
   150  	writefile("module bootstrap\ngo 1.20\n", pathf("%s/%s", base, "go.mod"), 0)
   151  	for _, dir := range bootstrapDirs {
   152  		recurse := strings.HasSuffix(dir, "/...")
   153  		dir = strings.TrimSuffix(dir, "/...")
   154  		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
   155  			if err != nil {
   156  				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
   157  			}
   158  
   159  			name := filepath.Base(path)
   160  			src := pathf("%s/src/%s", goroot, path)
   161  			dst := pathf("%s/%s", base, path)
   162  
   163  			if info.IsDir() {
   164  				if !recurse && path != dir || name == "testdata" {
   165  					return filepath.SkipDir
   166  				}
   167  
   168  				xmkdirall(dst)
   169  				if path == "cmd/cgo" {
   170  					// Write to src because we need the file both for bootstrap
   171  					// and for later in the main build.
   172  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
   173  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
   174  				}
   175  				return nil
   176  			}
   177  
   178  			for _, pre := range ignorePrefixes {
   179  				if strings.HasPrefix(name, pre) {
   180  					return nil
   181  				}
   182  			}
   183  			for _, suf := range ignoreSuffixes {
   184  				if strings.HasSuffix(name, suf) {
   185  					return nil
   186  				}
   187  			}
   188  
   189  			text := bootstrapRewriteFile(src)
   190  			writefile(text, dst, 0)
   191  			return nil
   192  		})
   193  	}
   194  
   195  	// Set up environment for invoking Go bootstrap toolchains go command.
   196  	// GOROOT points at Go bootstrap GOROOT,
   197  	// GOPATH points at our bootstrap workspace,
   198  	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
   199  	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
   200  	// so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
   201  	// Restore GOROOT, GOPATH, and GOBIN when done.
   202  	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
   203  	// because setup will take care of those when bootstrapBuildTools returns.
   204  
   205  	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
   206  	os.Setenv("GOROOT", goroot_bootstrap)
   207  
   208  	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
   209  	os.Setenv("GOPATH", workspace)
   210  
   211  	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
   212  	os.Setenv("GOBIN", "")
   213  
   214  	os.Setenv("GOOS", "")
   215  	os.Setenv("GOHOSTOS", "")
   216  	os.Setenv("GOARCH", "")
   217  	os.Setenv("GOHOSTARCH", "")
   218  
   219  	// Run Go bootstrap to build binaries.
   220  	// Use the math_big_pure_go build tag to disable the assembly in math/big
   221  	// which may contain unsupported instructions.
   222  	// Use the purego build tag to disable other assembly code,
   223  	// such as in cmd/internal/notsha256.
   224  	cmd := []string{
   225  		pathf("%s/bin/go", goroot_bootstrap),
   226  		"install",
   227  		"-tags=math_big_pure_go compiler_bootstrap purego",
   228  	}
   229  	if vflag > 0 {
   230  		cmd = append(cmd, "-v")
   231  	}
   232  	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
   233  		cmd = append(cmd, "-toolexec="+tool)
   234  	}
   235  	cmd = append(cmd, "bootstrap/cmd/...")
   236  	run(base, ShowOutput|CheckExit, cmd...)
   237  
   238  	// Copy binaries into tool binary directory.
   239  	for _, name := range bootstrapDirs {
   240  		if !strings.HasPrefix(name, "cmd/") {
   241  			continue
   242  		}
   243  		name = name[len("cmd/"):]
   244  		if !strings.Contains(name, "/") {
   245  			copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
   246  		}
   247  	}
   248  
   249  	if vflag > 0 {
   250  		xprintf("\n")
   251  	}
   252  }
   253  
   254  var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
   255  
   256  // isUnneededSSARewriteFile reports whether srcFile is a
   257  // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
   258  // architecture that isn't for the given GOARCH.
   259  //
   260  // When unneeded is true archCaps is the rewrite base filename without
   261  // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
   262  func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
   263  	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
   264  		return "", false
   265  	}
   266  	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
   267  	if fileArch == "" {
   268  		return "", false
   269  	}
   270  	b := fileArch[0]
   271  	if b == '_' || ('a' <= b && b <= 'z') {
   272  		return "", false
   273  	}
   274  	archCaps = fileArch
   275  	fileArch = strings.ToLower(fileArch)
   276  	fileArch = strings.TrimSuffix(fileArch, "splitload")
   277  	fileArch = strings.TrimSuffix(fileArch, "latelower")
   278  	if fileArch == goArch {
   279  		return "", false
   280  	}
   281  	if fileArch == strings.TrimSuffix(goArch, "le") {
   282  		return "", false
   283  	}
   284  	return archCaps, true
   285  }
   286  
   287  func bootstrapRewriteFile(srcFile string) string {
   288  	// During bootstrap, generate dummy rewrite files for
   289  	// irrelevant architectures. We only need to build a bootstrap
   290  	// binary that works for the current gohostarch.
   291  	// This saves 6+ seconds of bootstrap.
   292  	if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
   293  		return fmt.Sprintf(`%spackage ssa
   294  
   295  func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
   296  func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
   297  `, generatedHeader, archCaps, archCaps)
   298  	}
   299  
   300  	return bootstrapFixImports(srcFile)
   301  }
   302  
   303  func bootstrapFixImports(srcFile string) string {
   304  	text := readfile(srcFile)
   305  	if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) {
   306  		text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}")
   307  	}
   308  	lines := strings.SplitAfter(text, "\n")
   309  	inBlock := false
   310  	for i, line := range lines {
   311  		if strings.HasPrefix(line, "import (") {
   312  			inBlock = true
   313  			continue
   314  		}
   315  		if inBlock && strings.HasPrefix(line, ")") {
   316  			inBlock = false
   317  			continue
   318  		}
   319  		if strings.HasPrefix(line, `import `) || inBlock {
   320  			line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
   321  			for _, dir := range bootstrapDirs {
   322  				if strings.HasPrefix(dir, "cmd/") {
   323  					continue
   324  				}
   325  				line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
   326  			}
   327  			lines[i] = line
   328  		}
   329  	}
   330  
   331  	lines[0] = generatedHeader + "// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
   332  
   333  	return strings.Join(lines, "")
   334  }
   335
View as plain text