Source file src/cmd/go/internal/modfetch/fetch.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"archive/zip"
     9  	"bytes"
    10  	"context"
    11  	"crypto/sha256"
    12  	"encoding/base64"
    13  	"errors"
    14  	"fmt"
    15  	"io"
    16  	"io/fs"
    17  	"os"
    18  	"path/filepath"
    19  	"sort"
    20  	"strings"
    21  	"sync"
    22  
    23  	"cmd/go/internal/base"
    24  	"cmd/go/internal/cfg"
    25  	"cmd/go/internal/fsys"
    26  	"cmd/go/internal/gover"
    27  	"cmd/go/internal/lockedfile"
    28  	"cmd/go/internal/par"
    29  	"cmd/go/internal/robustio"
    30  	"cmd/go/internal/str"
    31  	"cmd/go/internal/trace"
    32  
    33  	"golang.org/x/mod/module"
    34  	"golang.org/x/mod/sumdb/dirhash"
    35  	modzip "golang.org/x/mod/zip"
    36  )
    37  
    38  var downloadCache par.ErrCache[module.Version, string] // version → directory
    39  
    40  var ErrToolchain = errors.New("internal error: invalid operation on toolchain module")
    41  
    42  // Download downloads the specific module version to the
    43  // local download cache and returns the name of the directory
    44  // corresponding to the root of the module's file tree.
    45  func Download(ctx context.Context, mod module.Version) (dir string, err error) {
    46  	if gover.IsToolchain(mod.Path) {
    47  		return "", ErrToolchain
    48  	}
    49  	if err := checkCacheDir(ctx); err != nil {
    50  		base.Fatal(err)
    51  	}
    52  
    53  	// The par.Cache here avoids duplicate work.
    54  	return downloadCache.Do(mod, func() (string, error) {
    55  		dir, err := download(ctx, mod)
    56  		if err != nil {
    57  			return "", err
    58  		}
    59  		checkMod(ctx, mod)
    60  
    61  		// If go.mod exists (not an old legacy module), check version is not too new.
    62  		if data, err := os.ReadFile(filepath.Join(dir, "go.mod")); err == nil {
    63  			goVersion := gover.GoModLookup(data, "go")
    64  			if gover.Compare(goVersion, gover.Local()) > 0 {
    65  				return "", &gover.TooNewError{What: mod.String(), GoVersion: goVersion}
    66  			}
    67  		} else if !errors.Is(err, fs.ErrNotExist) {
    68  			return "", err
    69  		}
    70  
    71  		return dir, nil
    72  	})
    73  }
    74  
    75  func download(ctx context.Context, mod module.Version) (dir string, err error) {
    76  	ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String())
    77  	defer span.Done()
    78  
    79  	dir, err = DownloadDir(ctx, mod)
    80  	if err == nil {
    81  		// The directory has already been completely extracted (no .partial file exists).
    82  		return dir, nil
    83  	} else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
    84  		return "", err
    85  	}
    86  
    87  	// To avoid cluttering the cache with extraneous files,
    88  	// DownloadZip uses the same lockfile as Download.
    89  	// Invoke DownloadZip before locking the file.
    90  	zipfile, err := DownloadZip(ctx, mod)
    91  	if err != nil {
    92  		return "", err
    93  	}
    94  
    95  	unlock, err := lockVersion(ctx, mod)
    96  	if err != nil {
    97  		return "", err
    98  	}
    99  	defer unlock()
   100  
   101  	ctx, span = trace.StartSpan(ctx, "unzip "+zipfile)
   102  	defer span.Done()
   103  
   104  	// Check whether the directory was populated while we were waiting on the lock.
   105  	_, dirErr := DownloadDir(ctx, mod)
   106  	if dirErr == nil {
   107  		return dir, nil
   108  	}
   109  	_, dirExists := dirErr.(*DownloadDirPartialError)
   110  
   111  	// Clean up any remaining temporary directories created by old versions
   112  	// (before 1.16), as well as partially extracted directories (indicated by
   113  	// DownloadDirPartialError, usually because of a .partial file). This is only
   114  	// safe to do because the lock file ensures that their writers are no longer
   115  	// active.
   116  	parentDir := filepath.Dir(dir)
   117  	tmpPrefix := filepath.Base(dir) + ".tmp-"
   118  	if old, err := filepath.Glob(filepath.Join(str.QuoteGlob(parentDir), str.QuoteGlob(tmpPrefix)+"*")); err == nil {
   119  		for _, path := range old {
   120  			RemoveAll(path) // best effort
   121  		}
   122  	}
   123  	if dirExists {
   124  		if err := RemoveAll(dir); err != nil {
   125  			return "", err
   126  		}
   127  	}
   128  
   129  	partialPath, err := CachePath(ctx, mod, "partial")
   130  	if err != nil {
   131  		return "", err
   132  	}
   133  
   134  	// Extract the module zip directory at its final location.
   135  	//
   136  	// To prevent other processes from reading the directory if we crash,
   137  	// create a .partial file before extracting the directory, and delete
   138  	// the .partial file afterward (all while holding the lock).
   139  	//
   140  	// Before Go 1.16, we extracted to a temporary directory with a random name
   141  	// then renamed it into place with os.Rename. On Windows, this failed with
   142  	// ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner)
   143  	// opened files in the temporary directory.
   144  	//
   145  	// Go 1.14.2 and higher respect .partial files. Older versions may use
   146  	// partially extracted directories. 'go mod verify' can detect this,
   147  	// and 'go clean -modcache' can fix it.
   148  	if err := os.MkdirAll(parentDir, 0777); err != nil {
   149  		return "", err
   150  	}
   151  	if err := os.WriteFile(partialPath, nil, 0666); err != nil {
   152  		return "", err
   153  	}
   154  	if err := modzip.Unzip(dir, mod, zipfile); err != nil {
   155  		fmt.Fprintf(os.Stderr, "-> %s\n", err)
   156  		if rmErr := RemoveAll(dir); rmErr == nil {
   157  			os.Remove(partialPath)
   158  		}
   159  		return "", err
   160  	}
   161  	if err := os.Remove(partialPath); err != nil {
   162  		return "", err
   163  	}
   164  
   165  	if !cfg.ModCacheRW {
   166  		makeDirsReadOnly(dir)
   167  	}
   168  	return dir, nil
   169  }
   170  
   171  var downloadZipCache par.ErrCache[module.Version, string]
   172  
   173  // DownloadZip downloads the specific module version to the
   174  // local zip cache and returns the name of the zip file.
   175  func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) {
   176  	// The par.Cache here avoids duplicate work.
   177  	return downloadZipCache.Do(mod, func() (string, error) {
   178  		zipfile, err := CachePath(ctx, mod, "zip")
   179  		if err != nil {
   180  			return "", err
   181  		}
   182  		ziphashfile := zipfile + "hash"
   183  
   184  		// Return without locking if the zip and ziphash files exist.
   185  		if _, err := os.Stat(zipfile); err == nil {
   186  			if _, err := os.Stat(ziphashfile); err == nil {
   187  				return zipfile, nil
   188  			}
   189  		}
   190  
   191  		// The zip or ziphash file does not exist. Acquire the lock and create them.
   192  		if cfg.CmdName != "mod download" {
   193  			vers := mod.Version
   194  			if mod.Path == "golang.org/toolchain" {
   195  				// Shorten v0.0.1-go1.13.1.darwin-amd64 to go1.13.1.darwin-amd64
   196  				_, vers, _ = strings.Cut(vers, "-")
   197  				if i := strings.LastIndex(vers, "."); i >= 0 {
   198  					goos, goarch, _ := strings.Cut(vers[i+1:], "-")
   199  					vers = vers[:i] + " (" + goos + "/" + goarch + ")"
   200  				}
   201  				fmt.Fprintf(os.Stderr, "go: downloading %s\n", vers)
   202  			} else {
   203  				fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, vers)
   204  			}
   205  		}
   206  		unlock, err := lockVersion(ctx, mod)
   207  		if err != nil {
   208  			return "", err
   209  		}
   210  		defer unlock()
   211  
   212  		if err := downloadZip(ctx, mod, zipfile); err != nil {
   213  			return "", err
   214  		}
   215  		return zipfile, nil
   216  	})
   217  }
   218  
   219  func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) {
   220  	ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile)
   221  	defer span.Done()
   222  
   223  	// Double-check that the zipfile was not created while we were waiting for
   224  	// the lock in DownloadZip.
   225  	ziphashfile := zipfile + "hash"
   226  	var zipExists, ziphashExists bool
   227  	if _, err := os.Stat(zipfile); err == nil {
   228  		zipExists = true
   229  	}
   230  	if _, err := os.Stat(ziphashfile); err == nil {
   231  		ziphashExists = true
   232  	}
   233  	if zipExists && ziphashExists {
   234  		return nil
   235  	}
   236  
   237  	// Create parent directories.
   238  	if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
   239  		return err
   240  	}
   241  
   242  	// Clean up any remaining tempfiles from previous runs.
   243  	// This is only safe to do because the lock file ensures that their
   244  	// writers are no longer active.
   245  	tmpPattern := filepath.Base(zipfile) + "*.tmp"
   246  	if old, err := filepath.Glob(filepath.Join(str.QuoteGlob(filepath.Dir(zipfile)), tmpPattern)); err == nil {
   247  		for _, path := range old {
   248  			os.Remove(path) // best effort
   249  		}
   250  	}
   251  
   252  	// If the zip file exists, the ziphash file must have been deleted
   253  	// or lost after a file system crash. Re-hash the zip without downloading.
   254  	if zipExists {
   255  		return hashZip(mod, zipfile, ziphashfile)
   256  	}
   257  
   258  	// From here to the os.Rename call below is functionally almost equivalent to
   259  	// renameio.WriteToFile, with one key difference: we want to validate the
   260  	// contents of the file (by hashing it) before we commit it. Because the file
   261  	// is zip-compressed, we need an actual file — or at least an io.ReaderAt — to
   262  	// validate it: we can't just tee the stream as we write it.
   263  	f, err := tempFile(ctx, filepath.Dir(zipfile), filepath.Base(zipfile), 0666)
   264  	if err != nil {
   265  		return err
   266  	}
   267  	defer func() {
   268  		if err != nil {
   269  			f.Close()
   270  			os.Remove(f.Name())
   271  		}
   272  	}()
   273  
   274  	var unrecoverableErr error
   275  	err = TryProxies(func(proxy string) error {
   276  		if unrecoverableErr != nil {
   277  			return unrecoverableErr
   278  		}
   279  		repo := Lookup(ctx, proxy, mod.Path)
   280  		err := repo.Zip(ctx, f, mod.Version)
   281  		if err != nil {
   282  			// Zip may have partially written to f before failing.
   283  			// (Perhaps the server crashed while sending the file?)
   284  			// Since we allow fallback on error in some cases, we need to fix up the
   285  			// file to be empty again for the next attempt.
   286  			if _, err := f.Seek(0, io.SeekStart); err != nil {
   287  				unrecoverableErr = err
   288  				return err
   289  			}
   290  			if err := f.Truncate(0); err != nil {
   291  				unrecoverableErr = err
   292  				return err
   293  			}
   294  		}
   295  		return err
   296  	})
   297  	if err != nil {
   298  		return err
   299  	}
   300  
   301  	// Double-check that the paths within the zip file are well-formed.
   302  	//
   303  	// TODO(bcmills): There is a similar check within the Unzip function. Can we eliminate one?
   304  	fi, err := f.Stat()
   305  	if err != nil {
   306  		return err
   307  	}
   308  	z, err := zip.NewReader(f, fi.Size())
   309  	if err != nil {
   310  		return err
   311  	}
   312  	prefix := mod.Path + "@" + mod.Version + "/"
   313  	for _, f := range z.File {
   314  		if !strings.HasPrefix(f.Name, prefix) {
   315  			return fmt.Errorf("zip for %s has unexpected file %s", prefix[:len(prefix)-1], f.Name)
   316  		}
   317  	}
   318  
   319  	if err := f.Close(); err != nil {
   320  		return err
   321  	}
   322  
   323  	// Hash the zip file and check the sum before renaming to the final location.
   324  	if err := hashZip(mod, f.Name(), ziphashfile); err != nil {
   325  		return err
   326  	}
   327  	if err := os.Rename(f.Name(), zipfile); err != nil {
   328  		return err
   329  	}
   330  
   331  	// TODO(bcmills): Should we make the .zip and .ziphash files read-only to discourage tampering?
   332  
   333  	return nil
   334  }
   335  
   336  // hashZip reads the zip file opened in f, then writes the hash to ziphashfile,
   337  // overwriting that file if it exists.
   338  //
   339  // If the hash does not match go.sum (or the sumdb if enabled), hashZip returns
   340  // an error and does not write ziphashfile.
   341  func hashZip(mod module.Version, zipfile, ziphashfile string) (err error) {
   342  	hash, err := dirhash.HashZip(zipfile, dirhash.DefaultHash)
   343  	if err != nil {
   344  		return err
   345  	}
   346  	if err := checkModSum(mod, hash); err != nil {
   347  		return err
   348  	}
   349  	hf, err := lockedfile.Create(ziphashfile)
   350  	if err != nil {
   351  		return err
   352  	}
   353  	defer func() {
   354  		if closeErr := hf.Close(); err == nil && closeErr != nil {
   355  			err = closeErr
   356  		}
   357  	}()
   358  	if err := hf.Truncate(int64(len(hash))); err != nil {
   359  		return err
   360  	}
   361  	if _, err := hf.WriteAt([]byte(hash), 0); err != nil {
   362  		return err
   363  	}
   364  	return nil
   365  }
   366  
   367  // makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
   368  // and its transitive contents.
   369  func makeDirsReadOnly(dir string) {
   370  	type pathMode struct {
   371  		path string
   372  		mode fs.FileMode
   373  	}
   374  	var dirs []pathMode // in lexical order
   375  	filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
   376  		if err == nil && d.IsDir() {
   377  			info, err := d.Info()
   378  			if err == nil && info.Mode()&0222 != 0 {
   379  				dirs = append(dirs, pathMode{path, info.Mode()})
   380  			}
   381  		}
   382  		return nil
   383  	})
   384  
   385  	// Run over list backward to chmod children before parents.
   386  	for i := len(dirs) - 1; i >= 0; i-- {
   387  		os.Chmod(dirs[i].path, dirs[i].mode&^0222)
   388  	}
   389  }
   390  
   391  // RemoveAll removes a directory written by Download or Unzip, first applying
   392  // any permission changes needed to do so.
   393  func RemoveAll(dir string) error {
   394  	// Module cache has 0555 directories; make them writable in order to remove content.
   395  	filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error {
   396  		if err != nil {
   397  			return nil // ignore errors walking in file system
   398  		}
   399  		if info.IsDir() {
   400  			os.Chmod(path, 0777)
   401  		}
   402  		return nil
   403  	})
   404  	return robustio.RemoveAll(dir)
   405  }
   406  
   407  var GoSumFile string             // path to go.sum; set by package modload
   408  var WorkspaceGoSumFiles []string // path to module go.sums in workspace; set by package modload
   409  
   410  type modSum struct {
   411  	mod module.Version
   412  	sum string
   413  }
   414  
   415  var goSum struct {
   416  	mu        sync.Mutex
   417  	m         map[module.Version][]string            // content of go.sum file
   418  	w         map[string]map[module.Version][]string // sum file in workspace -> content of that sum file
   419  	status    map[modSum]modSumStatus                // state of sums in m
   420  	overwrite bool                                   // if true, overwrite go.sum without incorporating its contents
   421  	enabled   bool                                   // whether to use go.sum at all
   422  }
   423  
   424  type modSumStatus struct {
   425  	used, dirty bool
   426  }
   427  
   428  // Reset resets globals in the modfetch package, so previous loads don't affect
   429  // contents of go.sum files.
   430  func Reset() {
   431  	GoSumFile = ""
   432  	WorkspaceGoSumFiles = nil
   433  
   434  	// Uses of lookupCache and downloadCache both can call checkModSum,
   435  	// which in turn sets the used bit on goSum.status for modules.
   436  	// Reset them so used can be computed properly.
   437  	lookupCache = par.Cache[lookupCacheKey, Repo]{}
   438  	downloadCache = par.ErrCache[module.Version, string]{}
   439  
   440  	// Clear all fields on goSum. It will be initialized later
   441  	goSum.mu.Lock()
   442  	goSum.m = nil
   443  	goSum.w = nil
   444  	goSum.status = nil
   445  	goSum.overwrite = false
   446  	goSum.enabled = false
   447  	goSum.mu.Unlock()
   448  }
   449  
   450  // initGoSum initializes the go.sum data.
   451  // The boolean it returns reports whether the
   452  // use of go.sum is now enabled.
   453  // The goSum lock must be held.
   454  func initGoSum() (bool, error) {
   455  	if GoSumFile == "" {
   456  		return false, nil
   457  	}
   458  	if goSum.m != nil {
   459  		return true, nil
   460  	}
   461  
   462  	goSum.m = make(map[module.Version][]string)
   463  	goSum.status = make(map[modSum]modSumStatus)
   464  	goSum.w = make(map[string]map[module.Version][]string)
   465  
   466  	for _, f := range WorkspaceGoSumFiles {
   467  		goSum.w[f] = make(map[module.Version][]string)
   468  		_, err := readGoSumFile(goSum.w[f], f)
   469  		if err != nil {
   470  			return false, err
   471  		}
   472  	}
   473  
   474  	enabled, err := readGoSumFile(goSum.m, GoSumFile)
   475  	goSum.enabled = enabled
   476  	return enabled, err
   477  }
   478  
   479  func readGoSumFile(dst map[module.Version][]string, file string) (bool, error) {
   480  	var (
   481  		data []byte
   482  		err  error
   483  	)
   484  	if actualSumFile, ok := fsys.OverlayPath(file); ok {
   485  		// Don't lock go.sum if it's part of the overlay.
   486  		// On Plan 9, locking requires chmod, and we don't want to modify any file
   487  		// in the overlay. See #44700.
   488  		data, err = os.ReadFile(actualSumFile)
   489  	} else {
   490  		data, err = lockedfile.Read(file)
   491  	}
   492  	if err != nil && !os.IsNotExist(err) {
   493  		return false, err
   494  	}
   495  	readGoSum(dst, file, data)
   496  
   497  	return true, nil
   498  }
   499  
   500  // emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod.
   501  // A bug caused us to write these into go.sum files for non-modules.
   502  // We detect and remove them.
   503  const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY="
   504  
   505  // readGoSum parses data, which is the content of file,
   506  // and adds it to goSum.m. The goSum lock must be held.
   507  func readGoSum(dst map[module.Version][]string, file string, data []byte) {
   508  	lineno := 0
   509  	for len(data) > 0 {
   510  		var line []byte
   511  		lineno++
   512  		i := bytes.IndexByte(data, '\n')
   513  		if i < 0 {
   514  			line, data = data, nil
   515  		} else {
   516  			line, data = data[:i], data[i+1:]
   517  		}
   518  		f := strings.Fields(string(line))
   519  		if len(f) == 0 {
   520  			// blank line; skip it
   521  			continue
   522  		}
   523  		if len(f) != 3 {
   524  			if cfg.CmdName == "mod tidy" {
   525  				// ignore malformed line so that go mod tidy can fix go.sum
   526  				continue
   527  			} else {
   528  				base.Fatalf("malformed go.sum:\n%s:%d: wrong number of fields %v\n", file, lineno, len(f))
   529  			}
   530  		}
   531  		if f[2] == emptyGoModHash {
   532  			// Old bug; drop it.
   533  			continue
   534  		}
   535  		mod := module.Version{Path: f[0], Version: f[1]}
   536  		dst[mod] = append(dst[mod], f[2])
   537  	}
   538  }
   539  
   540  // HaveSum returns true if the go.sum file contains an entry for mod.
   541  // The entry's hash must be generated with a known hash algorithm.
   542  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   543  // .mod and .zip files.
   544  func HaveSum(mod module.Version) bool {
   545  	goSum.mu.Lock()
   546  	defer goSum.mu.Unlock()
   547  	inited, err := initGoSum()
   548  	if err != nil || !inited {
   549  		return false
   550  	}
   551  	for _, goSums := range goSum.w {
   552  		for _, h := range goSums[mod] {
   553  			if !strings.HasPrefix(h, "h1:") {
   554  				continue
   555  			}
   556  			if !goSum.status[modSum{mod, h}].dirty {
   557  				return true
   558  			}
   559  		}
   560  	}
   561  	for _, h := range goSum.m[mod] {
   562  		if !strings.HasPrefix(h, "h1:") {
   563  			continue
   564  		}
   565  		if !goSum.status[modSum{mod, h}].dirty {
   566  			return true
   567  		}
   568  	}
   569  	return false
   570  }
   571  
   572  // RecordedSum returns the sum if the go.sum file contains an entry for mod.
   573  // The boolean reports true if an entry was found or
   574  // false if no entry found or two conflicting sums are found.
   575  // The entry's hash must be generated with a known hash algorithm.
   576  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   577  // .mod and .zip files.
   578  func RecordedSum(mod module.Version) (sum string, ok bool) {
   579  	goSum.mu.Lock()
   580  	defer goSum.mu.Unlock()
   581  	inited, err := initGoSum()
   582  	foundSum := ""
   583  	if err != nil || !inited {
   584  		return "", false
   585  	}
   586  	for _, goSums := range goSum.w {
   587  		for _, h := range goSums[mod] {
   588  			if !strings.HasPrefix(h, "h1:") {
   589  				continue
   590  			}
   591  			if !goSum.status[modSum{mod, h}].dirty {
   592  				if foundSum != "" && foundSum != h { // conflicting sums exist
   593  					return "", false
   594  				}
   595  				foundSum = h
   596  			}
   597  		}
   598  	}
   599  	for _, h := range goSum.m[mod] {
   600  		if !strings.HasPrefix(h, "h1:") {
   601  			continue
   602  		}
   603  		if !goSum.status[modSum{mod, h}].dirty {
   604  			if foundSum != "" && foundSum != h { // conflicting sums exist
   605  				return "", false
   606  			}
   607  			foundSum = h
   608  		}
   609  	}
   610  	return foundSum, true
   611  }
   612  
   613  // checkMod checks the given module's checksum and Go version.
   614  func checkMod(ctx context.Context, mod module.Version) {
   615  	// Do the file I/O before acquiring the go.sum lock.
   616  	ziphash, err := CachePath(ctx, mod, "ziphash")
   617  	if err != nil {
   618  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   619  	}
   620  	data, err := lockedfile.Read(ziphash)
   621  	if err != nil {
   622  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   623  	}
   624  	data = bytes.TrimSpace(data)
   625  	if !isValidSum(data) {
   626  		// Recreate ziphash file from zip file and use that to check the mod sum.
   627  		zip, err := CachePath(ctx, mod, "zip")
   628  		if err != nil {
   629  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   630  		}
   631  		err = hashZip(mod, zip, ziphash)
   632  		if err != nil {
   633  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   634  		}
   635  		return
   636  	}
   637  	h := string(data)
   638  	if !strings.HasPrefix(h, "h1:") {
   639  		base.Fatalf("verifying %v", module.VersionError(mod, fmt.Errorf("unexpected ziphash: %q", h)))
   640  	}
   641  
   642  	if err := checkModSum(mod, h); err != nil {
   643  		base.Fatalf("%s", err)
   644  	}
   645  }
   646  
   647  // goModSum returns the checksum for the go.mod contents.
   648  func goModSum(data []byte) (string, error) {
   649  	return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
   650  		return io.NopCloser(bytes.NewReader(data)), nil
   651  	})
   652  }
   653  
   654  // checkGoMod checks the given module's go.mod checksum;
   655  // data is the go.mod content.
   656  func checkGoMod(path, version string, data []byte) error {
   657  	h, err := goModSum(data)
   658  	if err != nil {
   659  		return &module.ModuleError{Path: path, Version: version, Err: fmt.Errorf("verifying go.mod: %v", err)}
   660  	}
   661  
   662  	return checkModSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
   663  }
   664  
   665  // checkModSum checks that the recorded checksum for mod is h.
   666  //
   667  // mod.Version may have the additional suffix "/go.mod" to request the checksum
   668  // for the module's go.mod file only.
   669  func checkModSum(mod module.Version, h string) error {
   670  	// We lock goSum when manipulating it,
   671  	// but we arrange to release the lock when calling checkSumDB,
   672  	// so that parallel calls to checkModHash can execute parallel calls
   673  	// to checkSumDB.
   674  
   675  	// Check whether mod+h is listed in go.sum already. If so, we're done.
   676  	goSum.mu.Lock()
   677  	inited, err := initGoSum()
   678  	if err != nil {
   679  		goSum.mu.Unlock()
   680  		return err
   681  	}
   682  	done := inited && haveModSumLocked(mod, h)
   683  	if inited {
   684  		st := goSum.status[modSum{mod, h}]
   685  		st.used = true
   686  		goSum.status[modSum{mod, h}] = st
   687  	}
   688  	goSum.mu.Unlock()
   689  
   690  	if done {
   691  		return nil
   692  	}
   693  
   694  	// Not listed, so we want to add them.
   695  	// Consult checksum database if appropriate.
   696  	if useSumDB(mod) {
   697  		// Calls base.Fatalf if mismatch detected.
   698  		if err := checkSumDB(mod, h); err != nil {
   699  			return err
   700  		}
   701  	}
   702  
   703  	// Add mod+h to go.sum, if it hasn't appeared already.
   704  	if inited {
   705  		goSum.mu.Lock()
   706  		addModSumLocked(mod, h)
   707  		st := goSum.status[modSum{mod, h}]
   708  		st.dirty = true
   709  		goSum.status[modSum{mod, h}] = st
   710  		goSum.mu.Unlock()
   711  	}
   712  	return nil
   713  }
   714  
   715  // haveModSumLocked reports whether the pair mod,h is already listed in go.sum.
   716  // If it finds a conflicting pair instead, it calls base.Fatalf.
   717  // goSum.mu must be locked.
   718  func haveModSumLocked(mod module.Version, h string) bool {
   719  	sumFileName := "go.sum"
   720  	if strings.HasSuffix(GoSumFile, "go.work.sum") {
   721  		sumFileName = "go.work.sum"
   722  	}
   723  	for _, vh := range goSum.m[mod] {
   724  		if h == vh {
   725  			return true
   726  		}
   727  		if strings.HasPrefix(vh, "h1:") {
   728  			base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, sumFileName, vh)
   729  		}
   730  	}
   731  	// Also check workspace sums.
   732  	foundMatch := false
   733  	// Check sums from all files in case there are conflicts between
   734  	// the files.
   735  	for goSumFile, goSums := range goSum.w {
   736  		for _, vh := range goSums[mod] {
   737  			if h == vh {
   738  				foundMatch = true
   739  			} else if strings.HasPrefix(vh, "h1:") {
   740  				base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, goSumFile, vh)
   741  			}
   742  		}
   743  	}
   744  	return foundMatch
   745  }
   746  
   747  // addModSumLocked adds the pair mod,h to go.sum.
   748  // goSum.mu must be locked.
   749  func addModSumLocked(mod module.Version, h string) {
   750  	if haveModSumLocked(mod, h) {
   751  		return
   752  	}
   753  	if len(goSum.m[mod]) > 0 {
   754  		fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h)
   755  	}
   756  	goSum.m[mod] = append(goSum.m[mod], h)
   757  }
   758  
   759  // checkSumDB checks the mod, h pair against the Go checksum database.
   760  // It calls base.Fatalf if the hash is to be rejected.
   761  func checkSumDB(mod module.Version, h string) error {
   762  	modWithoutSuffix := mod
   763  	noun := "module"
   764  	if before, found := strings.CutSuffix(mod.Version, "/go.mod"); found {
   765  		noun = "go.mod"
   766  		modWithoutSuffix.Version = before
   767  	}
   768  
   769  	db, lines, err := lookupSumDB(mod)
   770  	if err != nil {
   771  		return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: %v", noun, err))
   772  	}
   773  
   774  	have := mod.Path + " " + mod.Version + " " + h
   775  	prefix := mod.Path + " " + mod.Version + " h1:"
   776  	for _, line := range lines {
   777  		if line == have {
   778  			return nil
   779  		}
   780  		if strings.HasPrefix(line, prefix) {
   781  			return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, noun, h, db, line[len(prefix)-len("h1:"):]))
   782  		}
   783  	}
   784  	return nil
   785  }
   786  
   787  // Sum returns the checksum for the downloaded copy of the given module,
   788  // if present in the download cache.
   789  func Sum(ctx context.Context, mod module.Version) string {
   790  	if cfg.GOMODCACHE == "" {
   791  		// Do not use current directory.
   792  		return ""
   793  	}
   794  
   795  	ziphash, err := CachePath(ctx, mod, "ziphash")
   796  	if err != nil {
   797  		return ""
   798  	}
   799  	data, err := lockedfile.Read(ziphash)
   800  	if err != nil {
   801  		return ""
   802  	}
   803  	data = bytes.TrimSpace(data)
   804  	if !isValidSum(data) {
   805  		return ""
   806  	}
   807  	return string(data)
   808  }
   809  
   810  // isValidSum returns true if data is the valid contents of a zip hash file.
   811  // Certain critical files are written to disk by first truncating
   812  // then writing the actual bytes, so that if the write fails
   813  // the corrupt file should contain at least one of the null
   814  // bytes written by the truncate operation.
   815  func isValidSum(data []byte) bool {
   816  	if bytes.IndexByte(data, '\000') >= 0 {
   817  		return false
   818  	}
   819  
   820  	if len(data) != len("h1:")+base64.StdEncoding.EncodedLen(sha256.Size) {
   821  		return false
   822  	}
   823  
   824  	return true
   825  }
   826  
   827  var ErrGoSumDirty = errors.New("updates to go.sum needed, disabled by -mod=readonly")
   828  
   829  // WriteGoSum writes the go.sum file if it needs to be updated.
   830  //
   831  // keep is used to check whether a newly added sum should be saved in go.sum.
   832  // It should have entries for both module content sums and go.mod sums
   833  // (version ends with "/go.mod"). Existing sums will be preserved unless they
   834  // have been marked for deletion with TrimGoSum.
   835  func WriteGoSum(ctx context.Context, keep map[module.Version]bool, readonly bool) error {
   836  	goSum.mu.Lock()
   837  	defer goSum.mu.Unlock()
   838  
   839  	// If we haven't read the go.sum file yet, don't bother writing it.
   840  	if !goSum.enabled {
   841  		return nil
   842  	}
   843  
   844  	// Check whether we need to add sums for which keep[m] is true or remove
   845  	// unused sums marked with TrimGoSum. If there are no changes to make,
   846  	// just return without opening go.sum.
   847  	dirty := false
   848  Outer:
   849  	for m, hs := range goSum.m {
   850  		for _, h := range hs {
   851  			st := goSum.status[modSum{m, h}]
   852  			if st.dirty && (!st.used || keep[m]) {
   853  				dirty = true
   854  				break Outer
   855  			}
   856  		}
   857  	}
   858  	if !dirty {
   859  		return nil
   860  	}
   861  	if readonly {
   862  		return ErrGoSumDirty
   863  	}
   864  	if _, ok := fsys.OverlayPath(GoSumFile); ok {
   865  		base.Fatalf("go: updates to go.sum needed, but go.sum is part of the overlay specified with -overlay")
   866  	}
   867  
   868  	// Make a best-effort attempt to acquire the side lock, only to exclude
   869  	// previous versions of the 'go' command from making simultaneous edits.
   870  	if unlock, err := SideLock(ctx); err == nil {
   871  		defer unlock()
   872  	}
   873  
   874  	err := lockedfile.Transform(GoSumFile, func(data []byte) ([]byte, error) {
   875  		if !goSum.overwrite {
   876  			// Incorporate any sums added by other processes in the meantime.
   877  			// Add only the sums that we actually checked: the user may have edited or
   878  			// truncated the file to remove erroneous hashes, and we shouldn't restore
   879  			// them without good reason.
   880  			goSum.m = make(map[module.Version][]string, len(goSum.m))
   881  			readGoSum(goSum.m, GoSumFile, data)
   882  			for ms, st := range goSum.status {
   883  				if st.used && !sumInWorkspaceModulesLocked(ms.mod) {
   884  					addModSumLocked(ms.mod, ms.sum)
   885  				}
   886  			}
   887  		}
   888  
   889  		var mods []module.Version
   890  		for m := range goSum.m {
   891  			mods = append(mods, m)
   892  		}
   893  		module.Sort(mods)
   894  
   895  		var buf bytes.Buffer
   896  		for _, m := range mods {
   897  			list := goSum.m[m]
   898  			sort.Strings(list)
   899  			str.Uniq(&list)
   900  			for _, h := range list {
   901  				st := goSum.status[modSum{m, h}]
   902  				if (!st.dirty || (st.used && keep[m])) && !sumInWorkspaceModulesLocked(m) {
   903  					fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
   904  				}
   905  			}
   906  		}
   907  		return buf.Bytes(), nil
   908  	})
   909  
   910  	if err != nil {
   911  		return fmt.Errorf("updating go.sum: %w", err)
   912  	}
   913  
   914  	goSum.status = make(map[modSum]modSumStatus)
   915  	goSum.overwrite = false
   916  	return nil
   917  }
   918  
   919  func sumInWorkspaceModulesLocked(m module.Version) bool {
   920  	for _, goSums := range goSum.w {
   921  		if _, ok := goSums[m]; ok {
   922  			return true
   923  		}
   924  	}
   925  	return false
   926  }
   927  
   928  // TrimGoSum trims go.sum to contain only the modules needed for reproducible
   929  // builds.
   930  //
   931  // keep is used to check whether a sum should be retained in go.mod. It should
   932  // have entries for both module content sums and go.mod sums (version ends
   933  // with "/go.mod").
   934  func TrimGoSum(keep map[module.Version]bool) {
   935  	goSum.mu.Lock()
   936  	defer goSum.mu.Unlock()
   937  	inited, err := initGoSum()
   938  	if err != nil {
   939  		base.Fatalf("%s", err)
   940  	}
   941  	if !inited {
   942  		return
   943  	}
   944  
   945  	for m, hs := range goSum.m {
   946  		if !keep[m] {
   947  			for _, h := range hs {
   948  				goSum.status[modSum{m, h}] = modSumStatus{used: false, dirty: true}
   949  			}
   950  			goSum.overwrite = true
   951  		}
   952  	}
   953  }
   954  
   955  const goSumMismatch = `
   956  
   957  SECURITY ERROR
   958  This download does NOT match an earlier download recorded in go.sum.
   959  The bits may have been replaced on the origin server, or an attacker may
   960  have intercepted the download attempt.
   961  
   962  For more information, see 'go help module-auth'.
   963  `
   964  
   965  const sumdbMismatch = `
   966  
   967  SECURITY ERROR
   968  This download does NOT match the one reported by the checksum server.
   969  The bits may have been replaced on the origin server, or an attacker may
   970  have intercepted the download attempt.
   971  
   972  For more information, see 'go help module-auth'.
   973  `
   974  
   975  const hashVersionMismatch = `
   976  
   977  SECURITY WARNING
   978  This download is listed in go.sum, but using an unknown hash algorithm.
   979  The download cannot be verified.
   980  
   981  For more information, see 'go help module-auth'.
   982  
   983  `
   984  
   985  var HelpModuleAuth = &base.Command{
   986  	UsageLine: "module-auth",
   987  	Short:     "module authentication using go.sum",
   988  	Long: `
   989  When the go command downloads a module zip file or go.mod file into the
   990  module cache, it computes a cryptographic hash and compares it with a known
   991  value to verify the file hasn't changed since it was first downloaded. Known
   992  hashes are stored in a file in the module root directory named go.sum. Hashes
   993  may also be downloaded from the checksum database depending on the values of
   994  GOSUMDB, GOPRIVATE, and GONOSUMDB.
   995  
   996  For details, see https://golang.org/ref/mod#authenticating.
   997  `,
   998  }
   999  
  1000  var HelpPrivate = &base.Command{
  1001  	UsageLine: "private",
  1002  	Short:     "configuration for downloading non-public code",
  1003  	Long: `
  1004  The go command defaults to downloading modules from the public Go module
  1005  mirror at proxy.golang.org. It also defaults to validating downloaded modules,
  1006  regardless of source, against the public Go checksum database at sum.golang.org.
  1007  These defaults work well for publicly available source code.
  1008  
  1009  The GOPRIVATE environment variable controls which modules the go command
  1010  considers to be private (not available publicly) and should therefore not use
  1011  the proxy or checksum database. The variable is a comma-separated list of
  1012  glob patterns (in the syntax of Go's path.Match) of module path prefixes.
  1013  For example,
  1014  
  1015  	GOPRIVATE=*.corp.example.com,rsc.io/private
  1016  
  1017  causes the go command to treat as private any module with a path prefix
  1018  matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private,
  1019  and rsc.io/private/quux.
  1020  
  1021  For fine-grained control over module download and validation, the GONOPROXY
  1022  and GONOSUMDB environment variables accept the same kind of glob list
  1023  and override GOPRIVATE for the specific decision of whether to use the proxy
  1024  and checksum database, respectively.
  1025  
  1026  For example, if a company ran a module proxy serving private modules,
  1027  users would configure go using:
  1028  
  1029  	GOPRIVATE=*.corp.example.com
  1030  	GOPROXY=proxy.example.com
  1031  	GONOPROXY=none
  1032  
  1033  The GOPRIVATE variable is also used to define the "public" and "private"
  1034  patterns for the GOVCS variable; see 'go help vcs'. For that usage,
  1035  GOPRIVATE applies even in GOPATH mode. In that case, it matches import paths
  1036  instead of module paths.
  1037  
  1038  The 'go env -w' command (see 'go help env') can be used to set these variables
  1039  for future go command invocations.
  1040  
  1041  For more details, see https://golang.org/ref/mod#private-modules.
  1042  `,
  1043  }
  1044  

View as plain text