vendor.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modcmd
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"errors"
    11  	"fmt"
    12  	"go/build"
    13  	"io"
    14  	"io/fs"
    15  	"os"
    16  	"path"
    17  	"path/filepath"
    18  	"sort"
    19  	"strings"
    20  
    21  	"cmd/go/internal/base"
    22  	"cmd/go/internal/cfg"
    23  	"cmd/go/internal/fsys"
    24  	"cmd/go/internal/gover"
    25  	"cmd/go/internal/imports"
    26  	"cmd/go/internal/load"
    27  	"cmd/go/internal/modload"
    28  	"cmd/go/internal/str"
    29  
    30  	"golang.org/x/mod/module"
    31  )
    32  
    33  var cmdVendor = &base.Command{
    34  	UsageLine: "go mod vendor [-e] [-v] [-o outdir]",
    35  	Short:     "make vendored copy of dependencies",
    36  	Long: `
    37  Vendor resets the main module's vendor directory to include all packages
    38  needed to build and test all the main module's packages.
    39  It does not include test code for vendored packages.
    40  
    41  The -v flag causes vendor to print the names of vendored
    42  modules and packages to standard error.
    43  
    44  The -e flag causes vendor to attempt to proceed despite errors
    45  encountered while loading packages.
    46  
    47  The -o flag causes vendor to create the vendor directory at the given
    48  path instead of "vendor". The go command can only use a vendor directory
    49  named "vendor" within the module root directory, so this flag is
    50  primarily useful for other tools.
    51  
    52  See https://go.dev/ref/mod#go-mod-vendor for more about 'go mod vendor'.
    53  	`,
    54  	Run: runVendor,
    55  }
    56  
    57  var (
    58  	vendorE bool   // if true, report errors but proceed anyway
    59  	vendorO string // if set, overrides the default output directory
    60  )
    61  
    62  func init() {
    63  	cmdVendor.Flag.BoolVar(&cfg.BuildV, "v", false, "print the names of packages as they are processed")
    64  	cmdVendor.Flag.BoolVar(&vendorE, "e", false, "report errors but proceed anyway")
    65  	cmdVendor.Flag.StringVar(&vendorO, "o", "", "the output `directory` to write vendor modules to")
    66  	base.AddChdirFlag(&cmdVendor.Flag)
    67  	base.AddModCommonFlags(&cmdVendor.Flag)
    68  }
    69  
    70  func runVendor(ctx context.Context, cmd *base.Command, args []string) {
    71  	moduleLoader := modload.NewLoader()
    72  	moduleLoader.InitWorkfile()
    73  	if modload.WorkFilePath(moduleLoader) != "" {
    74  		base.Fatalf("go: 'go mod vendor' cannot be run in workspace mode. Run 'go work vendor' to vendor the workspace or set 'GOWORK=off' to exit workspace mode.")
    75  	}
    76  	RunVendor(moduleLoader, ctx, vendorE, vendorO, args)
    77  }
    78  
    79  func RunVendor(ld *modload.Loader, ctx context.Context, vendorE bool, vendorO string, args []string) {
    80  	if len(args) != 0 {
    81  		base.Fatalf("go: 'go mod vendor' accepts no arguments")
    82  	}
    83  	ld.ForceUseModules = true
    84  	ld.RootMode = modload.NeedRoot
    85  
    86  	loadOpts := modload.PackageOpts{
    87  		Tags:                     imports.AnyTags(),
    88  		VendorModulesInGOROOTSrc: true,
    89  		ResolveMissingImports:    true,
    90  		UseVendorAll:             true,
    91  		AllowErrors:              vendorE,
    92  		SilenceMissingStdImports: true,
    93  	}
    94  	_, pkgs := modload.LoadPackages(ld, ctx, loadOpts, "all")
    95  
    96  	var vdir string
    97  	switch {
    98  	case filepath.IsAbs(vendorO):
    99  		vdir = vendorO
   100  	case vendorO != "":
   101  		vdir = filepath.Join(base.Cwd(), vendorO)
   102  	default:
   103  		vdir = filepath.Join(modload.VendorDir(ld))
   104  	}
   105  	if err := os.RemoveAll(vdir); err != nil {
   106  		base.Fatal(err)
   107  	}
   108  
   109  	modpkgs := make(map[module.Version][]string)
   110  	for _, pkg := range pkgs {
   111  		m := ld.PackageModule(pkg)
   112  		if m.Path == "" || ld.MainModules.Contains(m.Path) {
   113  			continue
   114  		}
   115  		modpkgs[m] = append(modpkgs[m], pkg)
   116  	}
   117  	checkPathCollisions(modpkgs)
   118  
   119  	includeAllReplacements := false
   120  	includeGoVersions := false
   121  	isExplicit := map[module.Version]bool{}
   122  	gv := ld.MainModules.GoVersion(ld)
   123  	if gover.Compare(gv, "1.14") >= 0 && (ld.FindGoWork(base.Cwd()) != "" || modload.ModFile(ld).Go != nil) {
   124  		// If the Go version is at least 1.14, annotate all explicit 'require' and
   125  		// 'replace' targets found in the go.mod file so that we can perform a
   126  		// stronger consistency check when -mod=vendor is set.
   127  		for _, m := range ld.MainModules.Versions() {
   128  			if modFile := ld.MainModules.ModFile(m); modFile != nil {
   129  				for _, r := range modFile.Require {
   130  					isExplicit[r.Mod] = true
   131  				}
   132  			}
   133  		}
   134  		includeAllReplacements = true
   135  	}
   136  	if gover.Compare(gv, "1.17") >= 0 {
   137  		// If the Go version is at least 1.17, annotate all modules with their
   138  		// 'go' version directives.
   139  		includeGoVersions = true
   140  	}
   141  
   142  	var vendorMods []module.Version
   143  	for m := range isExplicit {
   144  		vendorMods = append(vendorMods, m)
   145  	}
   146  	for m := range modpkgs {
   147  		if !isExplicit[m] {
   148  			vendorMods = append(vendorMods, m)
   149  		}
   150  	}
   151  	gover.ModSort(vendorMods)
   152  
   153  	var (
   154  		buf bytes.Buffer
   155  		w   io.Writer = &buf
   156  	)
   157  	if cfg.BuildV {
   158  		w = io.MultiWriter(&buf, os.Stderr)
   159  	}
   160  
   161  	if ld.MainModules.WorkFile() != nil {
   162  		fmt.Fprintf(w, "## workspace\n")
   163  	}
   164  
   165  	replacementWritten := make(map[module.Version]bool)
   166  	for _, m := range vendorMods {
   167  		replacement := modload.Replacement(ld, m)
   168  		line := moduleLine(m, replacement)
   169  		replacementWritten[m] = true
   170  		io.WriteString(w, line)
   171  
   172  		goVersion := ""
   173  		if includeGoVersions {
   174  			goVersion = modload.ModuleInfo(ld, ctx, m.Path).GoVersion
   175  		}
   176  		switch {
   177  		case isExplicit[m] && goVersion != "":
   178  			fmt.Fprintf(w, "## explicit; go %s\n", goVersion)
   179  		case isExplicit[m]:
   180  			io.WriteString(w, "## explicit\n")
   181  		case goVersion != "":
   182  			fmt.Fprintf(w, "## go %s\n", goVersion)
   183  		}
   184  
   185  		pkgs := modpkgs[m]
   186  		sort.Strings(pkgs)
   187  		for _, pkg := range pkgs {
   188  			fmt.Fprintf(w, "%s\n", pkg)
   189  			vendorPkg(ld, vdir, pkg)
   190  		}
   191  	}
   192  
   193  	if includeAllReplacements {
   194  		// Record unused and wildcard replacements at the end of the modules.txt file:
   195  		// without access to the complete build list, the consumer of the vendor
   196  		// directory can't otherwise determine that those replacements had no effect.
   197  		for _, m := range ld.MainModules.Versions() {
   198  			if workFile := ld.MainModules.WorkFile(); workFile != nil {
   199  				for _, r := range workFile.Replace {
   200  					if replacementWritten[r.Old] {
   201  						// We already recorded this replacement.
   202  						continue
   203  					}
   204  					replacementWritten[r.Old] = true
   205  
   206  					line := moduleLine(r.Old, r.New)
   207  					buf.WriteString(line)
   208  					if cfg.BuildV {
   209  						os.Stderr.WriteString(line)
   210  					}
   211  				}
   212  			}
   213  			if modFile := ld.MainModules.ModFile(m); modFile != nil {
   214  				for _, r := range modFile.Replace {
   215  					if replacementWritten[r.Old] {
   216  						// We already recorded this replacement.
   217  						continue
   218  					}
   219  					replacementWritten[r.Old] = true
   220  					rNew := modload.Replacement(ld, r.Old)
   221  					if rNew == (module.Version{}) {
   222  						// There is no replacement. Don't try to write it.
   223  						continue
   224  					}
   225  
   226  					line := moduleLine(r.Old, rNew)
   227  					buf.WriteString(line)
   228  					if cfg.BuildV {
   229  						os.Stderr.WriteString(line)
   230  					}
   231  				}
   232  			}
   233  		}
   234  	}
   235  
   236  	if buf.Len() == 0 {
   237  		fmt.Fprintf(os.Stderr, "go: no dependencies to vendor\n")
   238  		return
   239  	}
   240  
   241  	if err := os.MkdirAll(vdir, 0777); err != nil {
   242  		base.Fatal(err)
   243  	}
   244  
   245  	if err := os.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil {
   246  		base.Fatal(err)
   247  	}
   248  }
   249  
   250  func moduleLine(m, r module.Version) string {
   251  	b := new(strings.Builder)
   252  	b.WriteString("# ")
   253  	b.WriteString(m.Path)
   254  	if m.Version != "" {
   255  		b.WriteString(" ")
   256  		b.WriteString(m.Version)
   257  	}
   258  	if r.Path != "" {
   259  		if str.HasFilePathPrefix(filepath.Clean(r.Path), "vendor") {
   260  			base.Fatalf("go: replacement path %s inside vendor directory", r.Path)
   261  		}
   262  		b.WriteString(" => ")
   263  		b.WriteString(r.Path)
   264  		if r.Version != "" {
   265  			b.WriteString(" ")
   266  			b.WriteString(r.Version)
   267  		}
   268  	}
   269  	b.WriteString("\n")
   270  	return b.String()
   271  }
   272  
   273  func vendorPkg(s *modload.Loader, vdir, pkg string) {
   274  	src, realPath, _ := modload.Lookup(s, "", false, pkg)
   275  	if src == "" {
   276  		base.Errorf("internal error: no pkg for %s\n", pkg)
   277  		return
   278  	}
   279  	if realPath != pkg {
   280  		// TODO(#26904): Revisit whether this behavior still makes sense.
   281  		// This should actually be impossible today, because the import map is the
   282  		// identity function for packages outside of the standard library.
   283  		//
   284  		// Part of the purpose of the vendor directory is to allow the packages in
   285  		// the module to continue to build in GOPATH mode, and GOPATH-mode users
   286  		// won't know about replacement aliasing. How important is it to maintain
   287  		// compatibility?
   288  		fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg)
   289  	}
   290  
   291  	copiedFiles := make(map[string]bool)
   292  	dst := filepath.Join(vdir, pkg)
   293  	matcher := func(dir string, info fs.DirEntry) bool {
   294  		goVersion := s.MainModules.GoVersion(s)
   295  		return matchPotentialSourceFile(dir, info, goVersion)
   296  	}
   297  	copyDir(dst, src, matcher, copiedFiles)
   298  	if m := s.PackageModule(realPath); m.Path != "" {
   299  		copyMetadata(m.Path, realPath, dst, src, copiedFiles)
   300  	}
   301  
   302  	ctx := build.Default
   303  	ctx.UseAllFiles = true
   304  	bp, err := ctx.ImportDir(src, build.IgnoreVendor)
   305  	// Because UseAllFiles is set on the build.Context, it's possible ta get
   306  	// a MultiplePackageError on an otherwise valid package: the package could
   307  	// have different names for GOOS=windows and GOOS=mac for example. On the
   308  	// other hand if there's a NoGoError, the package might have source files
   309  	// specifying "//go:build ignore" those packages should be skipped because
   310  	// embeds from ignored files can't be used.
   311  	// TODO(#42504): Find a better way to avoid errors from ImportDir. We'll
   312  	// need to figure this out when we switch to PackagesAndErrors as per the
   313  	// TODO above.
   314  	var multiplePackageError *build.MultiplePackageError
   315  	var noGoError *build.NoGoError
   316  	if err != nil {
   317  		if errors.As(err, &noGoError) {
   318  			return // No source files in this package are built. Skip embeds in ignored files.
   319  		} else if !errors.As(err, &multiplePackageError) { // multiplePackageErrors are OK, but others are not.
   320  			base.Fatalf("internal error: failed to find embedded files of %s: %v\n", pkg, err)
   321  		}
   322  	}
   323  	var embedPatterns []string
   324  	if gover.Compare(s.MainModules.GoVersion(s), "1.22") >= 0 {
   325  		embedPatterns = bp.EmbedPatterns
   326  	} else {
   327  		// Maintain the behavior of https://github.com/golang/go/issues/63473
   328  		// so that we continue to agree with older versions of the go command
   329  		// about the contents of vendor directories in existing modules
   330  		embedPatterns = str.StringList(bp.EmbedPatterns, bp.TestEmbedPatterns, bp.XTestEmbedPatterns)
   331  	}
   332  	embeds, err := load.ResolveEmbed(bp.Dir, embedPatterns)
   333  	if err != nil {
   334  		format := "go: resolving embeds in %s: %v\n"
   335  		if vendorE {
   336  			fmt.Fprintf(os.Stderr, format, pkg, err)
   337  		} else {
   338  			base.Errorf(format, pkg, err)
   339  		}
   340  		return
   341  	}
   342  	for _, embed := range embeds {
   343  		embedDst := filepath.Join(dst, embed)
   344  		if copiedFiles[embedDst] {
   345  			continue
   346  		}
   347  
   348  		// Copy the file as is done by copyDir below.
   349  		err := func() error {
   350  			r, err := os.Open(filepath.Join(src, embed))
   351  			if err != nil {
   352  				return err
   353  			}
   354  			if err := os.MkdirAll(filepath.Dir(embedDst), 0777); err != nil {
   355  				return err
   356  			}
   357  			w, err := os.Create(embedDst)
   358  			if err != nil {
   359  				return err
   360  			}
   361  			if _, err := io.Copy(w, r); err != nil {
   362  				return err
   363  			}
   364  			r.Close()
   365  			return w.Close()
   366  		}()
   367  		if err != nil {
   368  			if vendorE {
   369  				fmt.Fprintf(os.Stderr, "go: %v\n", err)
   370  			} else {
   371  				base.Error(err)
   372  			}
   373  		}
   374  	}
   375  }
   376  
   377  type metakey struct {
   378  	modPath string
   379  	dst     string
   380  }
   381  
   382  var copiedMetadata = make(map[metakey]bool)
   383  
   384  // copyMetadata copies metadata files from parents of src to parents of dst,
   385  // stopping after processing the src parent for modPath.
   386  func copyMetadata(modPath, pkg, dst, src string, copiedFiles map[string]bool) {
   387  	for parent := 0; ; parent++ {
   388  		if copiedMetadata[metakey{modPath, dst}] {
   389  			break
   390  		}
   391  		copiedMetadata[metakey{modPath, dst}] = true
   392  		if parent > 0 {
   393  			copyDir(dst, src, matchMetadata, copiedFiles)
   394  		}
   395  		if modPath == pkg {
   396  			break
   397  		}
   398  		pkg = path.Dir(pkg)
   399  		dst = filepath.Dir(dst)
   400  		src = filepath.Dir(src)
   401  	}
   402  }
   403  
   404  // metaPrefixes is the list of metadata file prefixes.
   405  // Vendoring copies metadata files from parents of copied directories.
   406  // Note that this list could be arbitrarily extended, and it is longer
   407  // in other tools (such as godep or dep). By using this limited set of
   408  // prefixes and also insisting on capitalized file names, we are trying
   409  // to nudge people toward more agreement on the naming
   410  // and also trying to avoid false positives.
   411  var metaPrefixes = []string{
   412  	"AUTHORS",
   413  	"CONTRIBUTORS",
   414  	"COPYLEFT",
   415  	"COPYING",
   416  	"COPYRIGHT",
   417  	"LEGAL",
   418  	"LICENSE",
   419  	"NOTICE",
   420  	"PATENTS",
   421  }
   422  
   423  // matchMetadata reports whether info is a metadata file.
   424  func matchMetadata(dir string, info fs.DirEntry) bool {
   425  	name := info.Name()
   426  	for _, p := range metaPrefixes {
   427  		if strings.HasPrefix(name, p) {
   428  			return true
   429  		}
   430  	}
   431  	return false
   432  }
   433  
   434  // matchPotentialSourceFile reports whether info may be relevant to a build operation.
   435  func matchPotentialSourceFile(dir string, info fs.DirEntry, goVersion string) bool {
   436  	if strings.HasSuffix(info.Name(), "_test.go") {
   437  		return false
   438  	}
   439  	if info.Name() == "go.mod" || info.Name() == "go.sum" {
   440  		if gover.Compare(goVersion, "1.17") >= 0 {
   441  			// As of Go 1.17, we strip go.mod and go.sum files from dependency modules.
   442  			// Otherwise, 'go' commands invoked within the vendor subtree may misidentify
   443  			// an arbitrary directory within the vendor tree as a module root.
   444  			// (See https://golang.org/issue/42970.)
   445  			return false
   446  		}
   447  	}
   448  	if strings.HasSuffix(info.Name(), ".go") {
   449  		f, err := fsys.Open(filepath.Join(dir, info.Name()))
   450  		if err != nil {
   451  			base.Fatal(err)
   452  		}
   453  		defer f.Close()
   454  
   455  		content, err := imports.ReadImports(f, false, nil)
   456  		if err == nil && !imports.ShouldBuild(content, imports.AnyTags()) {
   457  			// The file is explicitly tagged "ignore", so it can't affect the build.
   458  			// Leave it out.
   459  			return false
   460  		}
   461  		return true
   462  	}
   463  
   464  	// We don't know anything about this file, so optimistically assume that it is
   465  	// needed.
   466  	return true
   467  }
   468  
   469  // copyDir copies all regular files satisfying match(info) from src to dst.
   470  func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool, copiedFiles map[string]bool) {
   471  	files, err := os.ReadDir(src)
   472  	if err != nil {
   473  		base.Fatal(err)
   474  	}
   475  	if err := os.MkdirAll(dst, 0777); err != nil {
   476  		base.Fatal(err)
   477  	}
   478  	for _, file := range files {
   479  		if file.IsDir() || !file.Type().IsRegular() || !match(src, file) {
   480  			continue
   481  		}
   482  		copiedFiles[file.Name()] = true
   483  		r, err := os.Open(filepath.Join(src, file.Name()))
   484  		if err != nil {
   485  			base.Fatal(err)
   486  		}
   487  		dstPath := filepath.Join(dst, file.Name())
   488  		copiedFiles[dstPath] = true
   489  		w, err := os.Create(dstPath)
   490  		if err != nil {
   491  			base.Fatal(err)
   492  		}
   493  		if _, err := io.Copy(w, r); err != nil {
   494  			base.Fatal(err)
   495  		}
   496  		r.Close()
   497  		if err := w.Close(); err != nil {
   498  			base.Fatal(err)
   499  		}
   500  	}
   501  }
   502  
   503  // checkPathCollisions will fail if case-insensitive collisions are present.
   504  // The reason why we do this check in go mod vendor is to keep consistency
   505  // with go build. If modifying, consider changing load() in
   506  // src/cmd/go/internal/load/pkg.go
   507  func checkPathCollisions(modpkgs map[module.Version][]string) {
   508  	foldPath := make(map[string]string, len(modpkgs))
   509  	for m := range modpkgs {
   510  		fold := str.ToFold(m.Path)
   511  		if other := foldPath[fold]; other == "" {
   512  			foldPath[fold] = m.Path
   513  		} else if other != m.Path {
   514  			base.Fatalf("go.mod: case-insensitive import collision: %q and %q", m.Path, other)
   515  		}
   516  	}
   517  }
   518
View as plain text