vendor.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modcmd
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"errors"
    11  	"fmt"
    12  	"go/build"
    13  	"io"
    14  	"io/fs"
    15  	"os"
    16  	"path"
    17  	"path/filepath"
    18  	"sort"
    19  	"strings"
    20  
    21  	"cmd/go/internal/base"
    22  	"cmd/go/internal/cfg"
    23  	"cmd/go/internal/fsys"
    24  	"cmd/go/internal/gover"
    25  	"cmd/go/internal/imports"
    26  	"cmd/go/internal/load"
    27  	"cmd/go/internal/modload"
    28  	"cmd/go/internal/str"
    29  
    30  	"golang.org/x/mod/module"
    31  )
    32  
    33  var cmdVendor = &base.Command{
    34  	UsageLine: "go mod vendor [-e] [-v] [-o outdir]",
    35  	Short:     "make vendored copy of dependencies",
    36  	Long: `
    37  Vendor resets the main module's vendor directory to include all packages
    38  needed to build and test all the main module's packages.
    39  It does not include test code for vendored packages.
    40  
    41  The -v flag causes vendor to print the names of vendored
    42  modules and packages to standard error.
    43  
    44  The -e flag causes vendor to attempt to proceed despite errors
    45  encountered while loading packages.
    46  
    47  The -o flag causes vendor to create the vendor directory at the given
    48  path instead of "vendor". The go command can only use a vendor directory
    49  named "vendor" within the module root directory, so this flag is
    50  primarily useful for other tools.
    51  
    52  See https://golang.org/ref/mod#go-mod-vendor for more about 'go mod vendor'.
    53  	`,
    54  	Run: runVendor,
    55  }
    56  
    57  var vendorE bool   // if true, report errors but proceed anyway
    58  var vendorO string // if set, overrides the default output directory
    59  
    60  func init() {
    61  	cmdVendor.Flag.BoolVar(&cfg.BuildV, "v", false, "")
    62  	cmdVendor.Flag.BoolVar(&vendorE, "e", false, "")
    63  	cmdVendor.Flag.StringVar(&vendorO, "o", "", "")
    64  	base.AddChdirFlag(&cmdVendor.Flag)
    65  	base.AddModCommonFlags(&cmdVendor.Flag)
    66  }
    67  
    68  func runVendor(ctx context.Context, cmd *base.Command, args []string) {
    69  	moduleLoaderState := modload.NewState()
    70  	moduleLoaderState.InitWorkfile()
    71  	if modload.WorkFilePath(moduleLoaderState) != "" {
    72  		base.Fatalf("go: 'go mod vendor' cannot be run in workspace mode. Run 'go work vendor' to vendor the workspace or set 'GOWORK=off' to exit workspace mode.")
    73  	}
    74  	RunVendor(moduleLoaderState, ctx, vendorE, vendorO, args)
    75  }
    76  
    77  func RunVendor(loaderstate *modload.State, ctx context.Context, vendorE bool, vendorO string, args []string) {
    78  	if len(args) != 0 {
    79  		base.Fatalf("go: 'go mod vendor' accepts no arguments")
    80  	}
    81  	loaderstate.ForceUseModules = true
    82  	loaderstate.RootMode = modload.NeedRoot
    83  
    84  	loadOpts := modload.PackageOpts{
    85  		Tags:                     imports.AnyTags(),
    86  		VendorModulesInGOROOTSrc: true,
    87  		ResolveMissingImports:    true,
    88  		UseVendorAll:             true,
    89  		AllowErrors:              vendorE,
    90  		SilenceMissingStdImports: true,
    91  	}
    92  	_, pkgs := modload.LoadPackages(loaderstate, ctx, loadOpts, "all")
    93  
    94  	var vdir string
    95  	switch {
    96  	case filepath.IsAbs(vendorO):
    97  		vdir = vendorO
    98  	case vendorO != "":
    99  		vdir = filepath.Join(base.Cwd(), vendorO)
   100  	default:
   101  		vdir = filepath.Join(modload.VendorDir(loaderstate))
   102  	}
   103  	if err := os.RemoveAll(vdir); err != nil {
   104  		base.Fatal(err)
   105  	}
   106  
   107  	modpkgs := make(map[module.Version][]string)
   108  	for _, pkg := range pkgs {
   109  		m := modload.PackageModule(pkg)
   110  		if m.Path == "" || loaderstate.MainModules.Contains(m.Path) {
   111  			continue
   112  		}
   113  		modpkgs[m] = append(modpkgs[m], pkg)
   114  	}
   115  	checkPathCollisions(modpkgs)
   116  
   117  	includeAllReplacements := false
   118  	includeGoVersions := false
   119  	isExplicit := map[module.Version]bool{}
   120  	gv := loaderstate.MainModules.GoVersion(loaderstate)
   121  	if gover.Compare(gv, "1.14") >= 0 && (loaderstate.FindGoWork(base.Cwd()) != "" || modload.ModFile(loaderstate).Go != nil) {
   122  		// If the Go version is at least 1.14, annotate all explicit 'require' and
   123  		// 'replace' targets found in the go.mod file so that we can perform a
   124  		// stronger consistency check when -mod=vendor is set.
   125  		for _, m := range loaderstate.MainModules.Versions() {
   126  			if modFile := loaderstate.MainModules.ModFile(m); modFile != nil {
   127  				for _, r := range modFile.Require {
   128  					isExplicit[r.Mod] = true
   129  				}
   130  			}
   131  
   132  		}
   133  		includeAllReplacements = true
   134  	}
   135  	if gover.Compare(gv, "1.17") >= 0 {
   136  		// If the Go version is at least 1.17, annotate all modules with their
   137  		// 'go' version directives.
   138  		includeGoVersions = true
   139  	}
   140  
   141  	var vendorMods []module.Version
   142  	for m := range isExplicit {
   143  		vendorMods = append(vendorMods, m)
   144  	}
   145  	for m := range modpkgs {
   146  		if !isExplicit[m] {
   147  			vendorMods = append(vendorMods, m)
   148  		}
   149  	}
   150  	gover.ModSort(vendorMods)
   151  
   152  	var (
   153  		buf bytes.Buffer
   154  		w   io.Writer = &buf
   155  	)
   156  	if cfg.BuildV {
   157  		w = io.MultiWriter(&buf, os.Stderr)
   158  	}
   159  
   160  	if loaderstate.MainModules.WorkFile() != nil {
   161  		fmt.Fprintf(w, "## workspace\n")
   162  	}
   163  
   164  	replacementWritten := make(map[module.Version]bool)
   165  	for _, m := range vendorMods {
   166  		replacement := modload.Replacement(loaderstate, m)
   167  		line := moduleLine(m, replacement)
   168  		replacementWritten[m] = true
   169  		io.WriteString(w, line)
   170  
   171  		goVersion := ""
   172  		if includeGoVersions {
   173  			goVersion = modload.ModuleInfo(loaderstate, ctx, m.Path).GoVersion
   174  		}
   175  		switch {
   176  		case isExplicit[m] && goVersion != "":
   177  			fmt.Fprintf(w, "## explicit; go %s\n", goVersion)
   178  		case isExplicit[m]:
   179  			io.WriteString(w, "## explicit\n")
   180  		case goVersion != "":
   181  			fmt.Fprintf(w, "## go %s\n", goVersion)
   182  		}
   183  
   184  		pkgs := modpkgs[m]
   185  		sort.Strings(pkgs)
   186  		for _, pkg := range pkgs {
   187  			fmt.Fprintf(w, "%s\n", pkg)
   188  			vendorPkg(loaderstate, vdir, pkg)
   189  		}
   190  	}
   191  
   192  	if includeAllReplacements {
   193  		// Record unused and wildcard replacements at the end of the modules.txt file:
   194  		// without access to the complete build list, the consumer of the vendor
   195  		// directory can't otherwise determine that those replacements had no effect.
   196  		for _, m := range loaderstate.MainModules.Versions() {
   197  			if workFile := loaderstate.MainModules.WorkFile(); workFile != nil {
   198  				for _, r := range workFile.Replace {
   199  					if replacementWritten[r.Old] {
   200  						// We already recorded this replacement.
   201  						continue
   202  					}
   203  					replacementWritten[r.Old] = true
   204  
   205  					line := moduleLine(r.Old, r.New)
   206  					buf.WriteString(line)
   207  					if cfg.BuildV {
   208  						os.Stderr.WriteString(line)
   209  					}
   210  				}
   211  			}
   212  			if modFile := loaderstate.MainModules.ModFile(m); modFile != nil {
   213  				for _, r := range modFile.Replace {
   214  					if replacementWritten[r.Old] {
   215  						// We already recorded this replacement.
   216  						continue
   217  					}
   218  					replacementWritten[r.Old] = true
   219  					rNew := modload.Replacement(loaderstate, r.Old)
   220  					if rNew == (module.Version{}) {
   221  						// There is no replacement. Don't try to write it.
   222  						continue
   223  					}
   224  
   225  					line := moduleLine(r.Old, rNew)
   226  					buf.WriteString(line)
   227  					if cfg.BuildV {
   228  						os.Stderr.WriteString(line)
   229  					}
   230  				}
   231  			}
   232  		}
   233  	}
   234  
   235  	if buf.Len() == 0 {
   236  		fmt.Fprintf(os.Stderr, "go: no dependencies to vendor\n")
   237  		return
   238  	}
   239  
   240  	if err := os.MkdirAll(vdir, 0777); err != nil {
   241  		base.Fatal(err)
   242  	}
   243  
   244  	if err := os.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil {
   245  		base.Fatal(err)
   246  	}
   247  }
   248  
   249  func moduleLine(m, r module.Version) string {
   250  	b := new(strings.Builder)
   251  	b.WriteString("# ")
   252  	b.WriteString(m.Path)
   253  	if m.Version != "" {
   254  		b.WriteString(" ")
   255  		b.WriteString(m.Version)
   256  	}
   257  	if r.Path != "" {
   258  		if str.HasFilePathPrefix(filepath.Clean(r.Path), "vendor") {
   259  			base.Fatalf("go: replacement path %s inside vendor directory", r.Path)
   260  		}
   261  		b.WriteString(" => ")
   262  		b.WriteString(r.Path)
   263  		if r.Version != "" {
   264  			b.WriteString(" ")
   265  			b.WriteString(r.Version)
   266  		}
   267  	}
   268  	b.WriteString("\n")
   269  	return b.String()
   270  }
   271  
   272  func vendorPkg(s *modload.State, vdir, pkg string) {
   273  	src, realPath, _ := modload.Lookup(s, "", false, pkg)
   274  	if src == "" {
   275  		base.Errorf("internal error: no pkg for %s\n", pkg)
   276  		return
   277  	}
   278  	if realPath != pkg {
   279  		// TODO(#26904): Revisit whether this behavior still makes sense.
   280  		// This should actually be impossible today, because the import map is the
   281  		// identity function for packages outside of the standard library.
   282  		//
   283  		// Part of the purpose of the vendor directory is to allow the packages in
   284  		// the module to continue to build in GOPATH mode, and GOPATH-mode users
   285  		// won't know about replacement aliasing. How important is it to maintain
   286  		// compatibility?
   287  		fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg)
   288  	}
   289  
   290  	copiedFiles := make(map[string]bool)
   291  	dst := filepath.Join(vdir, pkg)
   292  	matcher := func(dir string, info fs.DirEntry) bool {
   293  		goVersion := s.MainModules.GoVersion(s)
   294  		return matchPotentialSourceFile(dir, info, goVersion)
   295  	}
   296  	copyDir(dst, src, matcher, copiedFiles)
   297  	if m := modload.PackageModule(realPath); m.Path != "" {
   298  		copyMetadata(m.Path, realPath, dst, src, copiedFiles)
   299  	}
   300  
   301  	ctx := build.Default
   302  	ctx.UseAllFiles = true
   303  	bp, err := ctx.ImportDir(src, build.IgnoreVendor)
   304  	// Because UseAllFiles is set on the build.Context, it's possible ta get
   305  	// a MultiplePackageError on an otherwise valid package: the package could
   306  	// have different names for GOOS=windows and GOOS=mac for example. On the
   307  	// other hand if there's a NoGoError, the package might have source files
   308  	// specifying "//go:build ignore" those packages should be skipped because
   309  	// embeds from ignored files can't be used.
   310  	// TODO(#42504): Find a better way to avoid errors from ImportDir. We'll
   311  	// need to figure this out when we switch to PackagesAndErrors as per the
   312  	// TODO above.
   313  	var multiplePackageError *build.MultiplePackageError
   314  	var noGoError *build.NoGoError
   315  	if err != nil {
   316  		if errors.As(err, &noGoError) {
   317  			return // No source files in this package are built. Skip embeds in ignored files.
   318  		} else if !errors.As(err, &multiplePackageError) { // multiplePackageErrors are OK, but others are not.
   319  			base.Fatalf("internal error: failed to find embedded files of %s: %v\n", pkg, err)
   320  		}
   321  	}
   322  	var embedPatterns []string
   323  	if gover.Compare(s.MainModules.GoVersion(s), "1.22") >= 0 {
   324  		embedPatterns = bp.EmbedPatterns
   325  	} else {
   326  		// Maintain the behavior of https://github.com/golang/go/issues/63473
   327  		// so that we continue to agree with older versions of the go command
   328  		// about the contents of vendor directories in existing modules
   329  		embedPatterns = str.StringList(bp.EmbedPatterns, bp.TestEmbedPatterns, bp.XTestEmbedPatterns)
   330  	}
   331  	embeds, err := load.ResolveEmbed(bp.Dir, embedPatterns)
   332  	if err != nil {
   333  		format := "go: resolving embeds in %s: %v\n"
   334  		if vendorE {
   335  			fmt.Fprintf(os.Stderr, format, pkg, err)
   336  		} else {
   337  			base.Errorf(format, pkg, err)
   338  		}
   339  		return
   340  	}
   341  	for _, embed := range embeds {
   342  		embedDst := filepath.Join(dst, embed)
   343  		if copiedFiles[embedDst] {
   344  			continue
   345  		}
   346  
   347  		// Copy the file as is done by copyDir below.
   348  		err := func() error {
   349  			r, err := os.Open(filepath.Join(src, embed))
   350  			if err != nil {
   351  				return err
   352  			}
   353  			if err := os.MkdirAll(filepath.Dir(embedDst), 0777); err != nil {
   354  				return err
   355  			}
   356  			w, err := os.Create(embedDst)
   357  			if err != nil {
   358  				return err
   359  			}
   360  			if _, err := io.Copy(w, r); err != nil {
   361  				return err
   362  			}
   363  			r.Close()
   364  			return w.Close()
   365  		}()
   366  		if err != nil {
   367  			if vendorE {
   368  				fmt.Fprintf(os.Stderr, "go: %v\n", err)
   369  			} else {
   370  				base.Error(err)
   371  			}
   372  		}
   373  	}
   374  }
   375  
   376  type metakey struct {
   377  	modPath string
   378  	dst     string
   379  }
   380  
   381  var copiedMetadata = make(map[metakey]bool)
   382  
   383  // copyMetadata copies metadata files from parents of src to parents of dst,
   384  // stopping after processing the src parent for modPath.
   385  func copyMetadata(modPath, pkg, dst, src string, copiedFiles map[string]bool) {
   386  	for parent := 0; ; parent++ {
   387  		if copiedMetadata[metakey{modPath, dst}] {
   388  			break
   389  		}
   390  		copiedMetadata[metakey{modPath, dst}] = true
   391  		if parent > 0 {
   392  			copyDir(dst, src, matchMetadata, copiedFiles)
   393  		}
   394  		if modPath == pkg {
   395  			break
   396  		}
   397  		pkg = path.Dir(pkg)
   398  		dst = filepath.Dir(dst)
   399  		src = filepath.Dir(src)
   400  	}
   401  }
   402  
   403  // metaPrefixes is the list of metadata file prefixes.
   404  // Vendoring copies metadata files from parents of copied directories.
   405  // Note that this list could be arbitrarily extended, and it is longer
   406  // in other tools (such as godep or dep). By using this limited set of
   407  // prefixes and also insisting on capitalized file names, we are trying
   408  // to nudge people toward more agreement on the naming
   409  // and also trying to avoid false positives.
   410  var metaPrefixes = []string{
   411  	"AUTHORS",
   412  	"CONTRIBUTORS",
   413  	"COPYLEFT",
   414  	"COPYING",
   415  	"COPYRIGHT",
   416  	"LEGAL",
   417  	"LICENSE",
   418  	"NOTICE",
   419  	"PATENTS",
   420  }
   421  
   422  // matchMetadata reports whether info is a metadata file.
   423  func matchMetadata(dir string, info fs.DirEntry) bool {
   424  	name := info.Name()
   425  	for _, p := range metaPrefixes {
   426  		if strings.HasPrefix(name, p) {
   427  			return true
   428  		}
   429  	}
   430  	return false
   431  }
   432  
   433  // matchPotentialSourceFile reports whether info may be relevant to a build operation.
   434  func matchPotentialSourceFile(dir string, info fs.DirEntry, goVersion string) bool {
   435  	if strings.HasSuffix(info.Name(), "_test.go") {
   436  		return false
   437  	}
   438  	if info.Name() == "go.mod" || info.Name() == "go.sum" {
   439  		if gover.Compare(goVersion, "1.17") >= 0 {
   440  			// As of Go 1.17, we strip go.mod and go.sum files from dependency modules.
   441  			// Otherwise, 'go' commands invoked within the vendor subtree may misidentify
   442  			// an arbitrary directory within the vendor tree as a module root.
   443  			// (See https://golang.org/issue/42970.)
   444  			return false
   445  		}
   446  	}
   447  	if strings.HasSuffix(info.Name(), ".go") {
   448  		f, err := fsys.Open(filepath.Join(dir, info.Name()))
   449  		if err != nil {
   450  			base.Fatal(err)
   451  		}
   452  		defer f.Close()
   453  
   454  		content, err := imports.ReadImports(f, false, nil)
   455  		if err == nil && !imports.ShouldBuild(content, imports.AnyTags()) {
   456  			// The file is explicitly tagged "ignore", so it can't affect the build.
   457  			// Leave it out.
   458  			return false
   459  		}
   460  		return true
   461  	}
   462  
   463  	// We don't know anything about this file, so optimistically assume that it is
   464  	// needed.
   465  	return true
   466  }
   467  
   468  // copyDir copies all regular files satisfying match(info) from src to dst.
   469  func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool, copiedFiles map[string]bool) {
   470  	files, err := os.ReadDir(src)
   471  	if err != nil {
   472  		base.Fatal(err)
   473  	}
   474  	if err := os.MkdirAll(dst, 0777); err != nil {
   475  		base.Fatal(err)
   476  	}
   477  	for _, file := range files {
   478  		if file.IsDir() || !file.Type().IsRegular() || !match(src, file) {
   479  			continue
   480  		}
   481  		copiedFiles[file.Name()] = true
   482  		r, err := os.Open(filepath.Join(src, file.Name()))
   483  		if err != nil {
   484  			base.Fatal(err)
   485  		}
   486  		dstPath := filepath.Join(dst, file.Name())
   487  		copiedFiles[dstPath] = true
   488  		w, err := os.Create(dstPath)
   489  		if err != nil {
   490  			base.Fatal(err)
   491  		}
   492  		if _, err := io.Copy(w, r); err != nil {
   493  			base.Fatal(err)
   494  		}
   495  		r.Close()
   496  		if err := w.Close(); err != nil {
   497  			base.Fatal(err)
   498  		}
   499  	}
   500  }
   501  
   502  // checkPathCollisions will fail if case-insensitive collisions are present.
   503  // The reason why we do this check in go mod vendor is to keep consistency
   504  // with go build. If modifying, consider changing load() in
   505  // src/cmd/go/internal/load/pkg.go
   506  func checkPathCollisions(modpkgs map[module.Version][]string) {
   507  	var foldPath = make(map[string]string, len(modpkgs))
   508  	for m := range modpkgs {
   509  		fold := str.ToFold(m.Path)
   510  		if other := foldPath[fold]; other == "" {
   511  			foldPath[fold] = m.Path
   512  		} else if other != m.Path {
   513  			base.Fatalf("go.mod: case-insensitive import collision: %q and %q", m.Path, other)
   514  		}
   515  	}
   516  }
   517
View as plain text