Source file src/cmd/go/internal/modfetch/fetch.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"archive/zip"
     9  	"bytes"
    10  	"context"
    11  	"crypto/sha256"
    12  	"encoding/base64"
    13  	"errors"
    14  	"fmt"
    15  	"io"
    16  	"io/fs"
    17  	"os"
    18  	"path/filepath"
    19  	"sort"
    20  	"strings"
    21  	"sync"
    22  
    23  	"cmd/go/internal/base"
    24  	"cmd/go/internal/cfg"
    25  	"cmd/go/internal/fsys"
    26  	"cmd/go/internal/gover"
    27  	"cmd/go/internal/lockedfile"
    28  	"cmd/go/internal/str"
    29  	"cmd/go/internal/trace"
    30  	"cmd/internal/par"
    31  	"cmd/internal/robustio"
    32  
    33  	"golang.org/x/mod/module"
    34  	"golang.org/x/mod/sumdb/dirhash"
    35  	modzip "golang.org/x/mod/zip"
    36  )
    37  
    38  // The downloadCache is used to cache the operation of downloading a module to disk
    39  // (if it's not already downloaded) and getting the directory it was downloaded to.
    40  // It is important that downloadCache must not be accessed by any of the exported
    41  // functions of this package after they return, because it can be modified by the
    42  // non-thread-safe SetState function
    43  var downloadCache = new(par.ErrCache[module.Version, string]) // version → directory;
    44  
    45  var ErrToolchain = errors.New("internal error: invalid operation on toolchain module")
    46  
    47  // Download downloads the specific module version to the
    48  // local download cache and returns the name of the directory
    49  // corresponding to the root of the module's file tree.
    50  func Download(ctx context.Context, mod module.Version) (dir string, err error) {
    51  	if gover.IsToolchain(mod.Path) {
    52  		return "", ErrToolchain
    53  	}
    54  	if err := checkCacheDir(ctx); err != nil {
    55  		base.Fatal(err)
    56  	}
    57  
    58  	// The par.Cache here avoids duplicate work.
    59  	return downloadCache.Do(mod, func() (string, error) {
    60  		dir, err := download(ctx, mod)
    61  		if err != nil {
    62  			return "", err
    63  		}
    64  		checkMod(ctx, mod)
    65  
    66  		// If go.mod exists (not an old legacy module), check version is not too new.
    67  		if data, err := os.ReadFile(filepath.Join(dir, "go.mod")); err == nil {
    68  			goVersion := gover.GoModLookup(data, "go")
    69  			if gover.Compare(goVersion, gover.Local()) > 0 {
    70  				return "", &gover.TooNewError{What: mod.String(), GoVersion: goVersion}
    71  			}
    72  		} else if !errors.Is(err, fs.ErrNotExist) {
    73  			return "", err
    74  		}
    75  
    76  		return dir, nil
    77  	})
    78  }
    79  
    80  // Unzip is like Download but is given the explicit zip file to use,
    81  // rather than downloading it. This is used for the GOFIPS140 zip files,
    82  // which ship in the Go distribution itself.
    83  func Unzip(ctx context.Context, mod module.Version, zipfile string) (dir string, err error) {
    84  	if err := checkCacheDir(ctx); err != nil {
    85  		base.Fatal(err)
    86  	}
    87  
    88  	return downloadCache.Do(mod, func() (string, error) {
    89  		ctx, span := trace.StartSpan(ctx, "modfetch.Unzip "+mod.String())
    90  		defer span.Done()
    91  
    92  		dir, err = DownloadDir(ctx, mod)
    93  		if err == nil {
    94  			// The directory has already been completely extracted (no .partial file exists).
    95  			return dir, nil
    96  		} else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
    97  			return "", err
    98  		}
    99  
   100  		return unzip(ctx, mod, zipfile)
   101  	})
   102  }
   103  
   104  func download(ctx context.Context, mod module.Version) (dir string, err error) {
   105  	ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String())
   106  	defer span.Done()
   107  
   108  	dir, err = DownloadDir(ctx, mod)
   109  	if err == nil {
   110  		// The directory has already been completely extracted (no .partial file exists).
   111  		return dir, nil
   112  	} else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
   113  		return "", err
   114  	}
   115  
   116  	// To avoid cluttering the cache with extraneous files,
   117  	// DownloadZip uses the same lockfile as Download.
   118  	// Invoke DownloadZip before locking the file.
   119  	zipfile, err := DownloadZip(ctx, mod)
   120  	if err != nil {
   121  		return "", err
   122  	}
   123  
   124  	return unzip(ctx, mod, zipfile)
   125  }
   126  
   127  func unzip(ctx context.Context, mod module.Version, zipfile string) (dir string, err error) {
   128  	unlock, err := lockVersion(ctx, mod)
   129  	if err != nil {
   130  		return "", err
   131  	}
   132  	defer unlock()
   133  
   134  	ctx, span := trace.StartSpan(ctx, "unzip "+zipfile)
   135  	defer span.Done()
   136  
   137  	// Check whether the directory was populated while we were waiting on the lock.
   138  	dir, dirErr := DownloadDir(ctx, mod)
   139  	if dirErr == nil {
   140  		return dir, nil
   141  	}
   142  	_, dirExists := dirErr.(*DownloadDirPartialError)
   143  
   144  	// Clean up any remaining temporary directories created by old versions
   145  	// (before 1.16), as well as partially extracted directories (indicated by
   146  	// DownloadDirPartialError, usually because of a .partial file). This is only
   147  	// safe to do because the lock file ensures that their writers are no longer
   148  	// active.
   149  	parentDir := filepath.Dir(dir)
   150  	tmpPrefix := filepath.Base(dir) + ".tmp-"
   151  	if old, err := filepath.Glob(filepath.Join(str.QuoteGlob(parentDir), str.QuoteGlob(tmpPrefix)+"*")); err == nil {
   152  		for _, path := range old {
   153  			RemoveAll(path) // best effort
   154  		}
   155  	}
   156  	if dirExists {
   157  		if err := RemoveAll(dir); err != nil {
   158  			return "", err
   159  		}
   160  	}
   161  
   162  	partialPath, err := CachePath(ctx, mod, "partial")
   163  	if err != nil {
   164  		return "", err
   165  	}
   166  
   167  	// Extract the module zip directory at its final location.
   168  	//
   169  	// To prevent other processes from reading the directory if we crash,
   170  	// create a .partial file before extracting the directory, and delete
   171  	// the .partial file afterward (all while holding the lock).
   172  	//
   173  	// Before Go 1.16, we extracted to a temporary directory with a random name
   174  	// then renamed it into place with os.Rename. On Windows, this failed with
   175  	// ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner)
   176  	// opened files in the temporary directory.
   177  	//
   178  	// Go 1.14.2 and higher respect .partial files. Older versions may use
   179  	// partially extracted directories. 'go mod verify' can detect this,
   180  	// and 'go clean -modcache' can fix it.
   181  	if err := os.MkdirAll(parentDir, 0777); err != nil {
   182  		return "", err
   183  	}
   184  	if err := os.WriteFile(partialPath, nil, 0666); err != nil {
   185  		return "", err
   186  	}
   187  	if err := modzip.Unzip(dir, mod, zipfile); err != nil {
   188  		fmt.Fprintf(os.Stderr, "-> %s\n", err)
   189  		if rmErr := RemoveAll(dir); rmErr == nil {
   190  			os.Remove(partialPath)
   191  		}
   192  		return "", err
   193  	}
   194  	if err := os.Remove(partialPath); err != nil {
   195  		return "", err
   196  	}
   197  
   198  	if !cfg.ModCacheRW {
   199  		makeDirsReadOnly(dir)
   200  	}
   201  	return dir, nil
   202  }
   203  
   204  var downloadZipCache par.ErrCache[module.Version, string]
   205  
   206  // DownloadZip downloads the specific module version to the
   207  // local zip cache and returns the name of the zip file.
   208  func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) {
   209  	// The par.Cache here avoids duplicate work.
   210  	return downloadZipCache.Do(mod, func() (string, error) {
   211  		zipfile, err := CachePath(ctx, mod, "zip")
   212  		if err != nil {
   213  			return "", err
   214  		}
   215  		ziphashfile := zipfile + "hash"
   216  
   217  		// Return without locking if the zip and ziphash files exist.
   218  		if _, err := os.Stat(zipfile); err == nil {
   219  			if _, err := os.Stat(ziphashfile); err == nil {
   220  				return zipfile, nil
   221  			}
   222  		}
   223  
   224  		// The zip or ziphash file does not exist. Acquire the lock and create them.
   225  		if cfg.CmdName != "mod download" {
   226  			vers := mod.Version
   227  			if mod.Path == "golang.org/toolchain" {
   228  				// Shorten v0.0.1-go1.13.1.darwin-amd64 to go1.13.1.darwin-amd64
   229  				_, vers, _ = strings.Cut(vers, "-")
   230  				if i := strings.LastIndex(vers, "."); i >= 0 {
   231  					goos, goarch, _ := strings.Cut(vers[i+1:], "-")
   232  					vers = vers[:i] + " (" + goos + "/" + goarch + ")"
   233  				}
   234  				fmt.Fprintf(os.Stderr, "go: downloading %s\n", vers)
   235  			} else {
   236  				fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, vers)
   237  			}
   238  		}
   239  		unlock, err := lockVersion(ctx, mod)
   240  		if err != nil {
   241  			return "", err
   242  		}
   243  		defer unlock()
   244  
   245  		if err := downloadZip(ctx, mod, zipfile); err != nil {
   246  			return "", err
   247  		}
   248  		return zipfile, nil
   249  	})
   250  }
   251  
   252  func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) {
   253  	ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile)
   254  	defer span.Done()
   255  
   256  	// Double-check that the zipfile was not created while we were waiting for
   257  	// the lock in DownloadZip.
   258  	ziphashfile := zipfile + "hash"
   259  	var zipExists, ziphashExists bool
   260  	if _, err := os.Stat(zipfile); err == nil {
   261  		zipExists = true
   262  	}
   263  	if _, err := os.Stat(ziphashfile); err == nil {
   264  		ziphashExists = true
   265  	}
   266  	if zipExists && ziphashExists {
   267  		return nil
   268  	}
   269  
   270  	// Create parent directories.
   271  	if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
   272  		return err
   273  	}
   274  
   275  	// Clean up any remaining tempfiles from previous runs.
   276  	// This is only safe to do because the lock file ensures that their
   277  	// writers are no longer active.
   278  	tmpPattern := filepath.Base(zipfile) + "*.tmp"
   279  	if old, err := filepath.Glob(filepath.Join(str.QuoteGlob(filepath.Dir(zipfile)), tmpPattern)); err == nil {
   280  		for _, path := range old {
   281  			os.Remove(path) // best effort
   282  		}
   283  	}
   284  
   285  	// If the zip file exists, the ziphash file must have been deleted
   286  	// or lost after a file system crash. Re-hash the zip without downloading.
   287  	if zipExists {
   288  		return hashZip(mod, zipfile, ziphashfile)
   289  	}
   290  
   291  	// From here to the os.Rename call below is functionally almost equivalent to
   292  	// renameio.WriteToFile, with one key difference: we want to validate the
   293  	// contents of the file (by hashing it) before we commit it. Because the file
   294  	// is zip-compressed, we need an actual file — or at least an io.ReaderAt — to
   295  	// validate it: we can't just tee the stream as we write it.
   296  	f, err := tempFile(ctx, filepath.Dir(zipfile), filepath.Base(zipfile), 0666)
   297  	if err != nil {
   298  		return err
   299  	}
   300  	defer func() {
   301  		if err != nil {
   302  			f.Close()
   303  			os.Remove(f.Name())
   304  		}
   305  	}()
   306  
   307  	var unrecoverableErr error
   308  	err = TryProxies(func(proxy string) error {
   309  		if unrecoverableErr != nil {
   310  			return unrecoverableErr
   311  		}
   312  		repo := Lookup(ctx, proxy, mod.Path)
   313  		err := repo.Zip(ctx, f, mod.Version)
   314  		if err != nil {
   315  			// Zip may have partially written to f before failing.
   316  			// (Perhaps the server crashed while sending the file?)
   317  			// Since we allow fallback on error in some cases, we need to fix up the
   318  			// file to be empty again for the next attempt.
   319  			if _, err := f.Seek(0, io.SeekStart); err != nil {
   320  				unrecoverableErr = err
   321  				return err
   322  			}
   323  			if err := f.Truncate(0); err != nil {
   324  				unrecoverableErr = err
   325  				return err
   326  			}
   327  		}
   328  		return err
   329  	})
   330  	if err != nil {
   331  		return err
   332  	}
   333  
   334  	// Double-check that the paths within the zip file are well-formed.
   335  	//
   336  	// TODO(bcmills): There is a similar check within the Unzip function. Can we eliminate one?
   337  	fi, err := f.Stat()
   338  	if err != nil {
   339  		return err
   340  	}
   341  	z, err := zip.NewReader(f, fi.Size())
   342  	if err != nil {
   343  		return err
   344  	}
   345  	prefix := mod.Path + "@" + mod.Version + "/"
   346  	for _, f := range z.File {
   347  		if !strings.HasPrefix(f.Name, prefix) {
   348  			return fmt.Errorf("zip for %s has unexpected file %s", prefix[:len(prefix)-1], f.Name)
   349  		}
   350  	}
   351  
   352  	if err := f.Close(); err != nil {
   353  		return err
   354  	}
   355  
   356  	// Hash the zip file and check the sum before renaming to the final location.
   357  	if err := hashZip(mod, f.Name(), ziphashfile); err != nil {
   358  		return err
   359  	}
   360  	if err := os.Rename(f.Name(), zipfile); err != nil {
   361  		return err
   362  	}
   363  
   364  	// TODO(bcmills): Should we make the .zip and .ziphash files read-only to discourage tampering?
   365  
   366  	return nil
   367  }
   368  
   369  // hashZip reads the zip file opened in f, then writes the hash to ziphashfile,
   370  // overwriting that file if it exists.
   371  //
   372  // If the hash does not match go.sum (or the sumdb if enabled), hashZip returns
   373  // an error and does not write ziphashfile.
   374  func hashZip(mod module.Version, zipfile, ziphashfile string) (err error) {
   375  	hash, err := dirhash.HashZip(zipfile, dirhash.DefaultHash)
   376  	if err != nil {
   377  		return err
   378  	}
   379  	if err := checkModSum(mod, hash); err != nil {
   380  		return err
   381  	}
   382  	hf, err := lockedfile.Create(ziphashfile)
   383  	if err != nil {
   384  		return err
   385  	}
   386  	defer func() {
   387  		if closeErr := hf.Close(); err == nil && closeErr != nil {
   388  			err = closeErr
   389  		}
   390  	}()
   391  	if err := hf.Truncate(int64(len(hash))); err != nil {
   392  		return err
   393  	}
   394  	if _, err := hf.WriteAt([]byte(hash), 0); err != nil {
   395  		return err
   396  	}
   397  	return nil
   398  }
   399  
   400  // makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
   401  // and its transitive contents.
   402  func makeDirsReadOnly(dir string) {
   403  	type pathMode struct {
   404  		path string
   405  		mode fs.FileMode
   406  	}
   407  	var dirs []pathMode // in lexical order
   408  	filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
   409  		if err == nil && d.IsDir() {
   410  			info, err := d.Info()
   411  			if err == nil && info.Mode()&0222 != 0 {
   412  				dirs = append(dirs, pathMode{path, info.Mode()})
   413  			}
   414  		}
   415  		return nil
   416  	})
   417  
   418  	// Run over list backward to chmod children before parents.
   419  	for i := len(dirs) - 1; i >= 0; i-- {
   420  		os.Chmod(dirs[i].path, dirs[i].mode&^0222)
   421  	}
   422  }
   423  
   424  // RemoveAll removes a directory written by Download or Unzip, first applying
   425  // any permission changes needed to do so.
   426  func RemoveAll(dir string) error {
   427  	// Module cache has 0555 directories; make them writable in order to remove content.
   428  	filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error {
   429  		if err != nil {
   430  			return nil // ignore errors walking in file system
   431  		}
   432  		if info.IsDir() {
   433  			os.Chmod(path, 0777)
   434  		}
   435  		return nil
   436  	})
   437  	return robustio.RemoveAll(dir)
   438  }
   439  
   440  // The GoSumFile, WorkspaceGoSumFiles, and goSum are global state that must not be
   441  // accessed by any of the exported functions of this package after they return, because
   442  // they can be modified by the non-thread-safe SetState function.
   443  
   444  var GoSumFile string             // path to go.sum; set by package modload
   445  var WorkspaceGoSumFiles []string // path to module go.sums in workspace; set by package modload
   446  
   447  type modSum struct {
   448  	mod module.Version
   449  	sum string
   450  }
   451  
   452  var goSum struct {
   453  	mu sync.Mutex
   454  	sumState
   455  }
   456  
   457  type sumState struct {
   458  	m         map[module.Version][]string            // content of go.sum file
   459  	w         map[string]map[module.Version][]string // sum file in workspace -> content of that sum file
   460  	status    map[modSum]modSumStatus                // state of sums in m
   461  	overwrite bool                                   // if true, overwrite go.sum without incorporating its contents
   462  	enabled   bool                                   // whether to use go.sum at all
   463  }
   464  
   465  type modSumStatus struct {
   466  	used, dirty bool
   467  }
   468  
   469  // State holds a snapshot of the global state of the modfetch package.
   470  type State struct {
   471  	goSumFile           string
   472  	workspaceGoSumFiles []string
   473  	lookupCache         *par.Cache[lookupCacheKey, Repo]
   474  	downloadCache       *par.ErrCache[module.Version, string]
   475  	sumState            sumState
   476  }
   477  
   478  // Reset resets globals in the modfetch package, so previous loads don't affect
   479  // contents of go.sum files.
   480  func Reset() {
   481  	SetState(State{})
   482  }
   483  
   484  // SetState sets the global state of the modfetch package to the newState, and returns the previous
   485  // global state. newState should have been returned by SetState, or be an empty State.
   486  // There should be no concurrent calls to any of the exported functions of this package with
   487  // a call to SetState because it will modify the global state in a non-thread-safe way.
   488  func SetState(newState State) (oldState State) {
   489  	if newState.lookupCache == nil {
   490  		newState.lookupCache = new(par.Cache[lookupCacheKey, Repo])
   491  	}
   492  	if newState.downloadCache == nil {
   493  		newState.downloadCache = new(par.ErrCache[module.Version, string])
   494  	}
   495  
   496  	goSum.mu.Lock()
   497  	defer goSum.mu.Unlock()
   498  
   499  	oldState = State{
   500  		goSumFile:           GoSumFile,
   501  		workspaceGoSumFiles: WorkspaceGoSumFiles,
   502  		lookupCache:         lookupCache,
   503  		downloadCache:       downloadCache,
   504  		sumState:            goSum.sumState,
   505  	}
   506  
   507  	GoSumFile = newState.goSumFile
   508  	WorkspaceGoSumFiles = newState.workspaceGoSumFiles
   509  	// Uses of lookupCache and downloadCache both can call checkModSum,
   510  	// which in turn sets the used bit on goSum.status for modules.
   511  	// Set (or reset) them so used can be computed properly.
   512  	lookupCache = newState.lookupCache
   513  	downloadCache = newState.downloadCache
   514  	// Set, or reset all fields on goSum. If being reset to empty, it will be initialized later.
   515  	goSum.sumState = newState.sumState
   516  
   517  	return oldState
   518  }
   519  
   520  // initGoSum initializes the go.sum data.
   521  // The boolean it returns reports whether the
   522  // use of go.sum is now enabled.
   523  // The goSum lock must be held.
   524  func initGoSum() (bool, error) {
   525  	if GoSumFile == "" {
   526  		return false, nil
   527  	}
   528  	if goSum.m != nil {
   529  		return true, nil
   530  	}
   531  
   532  	goSum.m = make(map[module.Version][]string)
   533  	goSum.status = make(map[modSum]modSumStatus)
   534  	goSum.w = make(map[string]map[module.Version][]string)
   535  
   536  	for _, f := range WorkspaceGoSumFiles {
   537  		goSum.w[f] = make(map[module.Version][]string)
   538  		_, err := readGoSumFile(goSum.w[f], f)
   539  		if err != nil {
   540  			return false, err
   541  		}
   542  	}
   543  
   544  	enabled, err := readGoSumFile(goSum.m, GoSumFile)
   545  	goSum.enabled = enabled
   546  	return enabled, err
   547  }
   548  
   549  func readGoSumFile(dst map[module.Version][]string, file string) (bool, error) {
   550  	var (
   551  		data []byte
   552  		err  error
   553  	)
   554  	if fsys.Replaced(file) {
   555  		// Don't lock go.sum if it's part of the overlay.
   556  		// On Plan 9, locking requires chmod, and we don't want to modify any file
   557  		// in the overlay. See #44700.
   558  		data, err = os.ReadFile(fsys.Actual(file))
   559  	} else {
   560  		data, err = lockedfile.Read(file)
   561  	}
   562  	if err != nil && !os.IsNotExist(err) {
   563  		return false, err
   564  	}
   565  	readGoSum(dst, file, data)
   566  
   567  	return true, nil
   568  }
   569  
   570  // emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod.
   571  // A bug caused us to write these into go.sum files for non-modules.
   572  // We detect and remove them.
   573  const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY="
   574  
   575  // readGoSum parses data, which is the content of file,
   576  // and adds it to goSum.m. The goSum lock must be held.
   577  func readGoSum(dst map[module.Version][]string, file string, data []byte) {
   578  	lineno := 0
   579  	for len(data) > 0 {
   580  		var line []byte
   581  		lineno++
   582  		i := bytes.IndexByte(data, '\n')
   583  		if i < 0 {
   584  			line, data = data, nil
   585  		} else {
   586  			line, data = data[:i], data[i+1:]
   587  		}
   588  		f := strings.Fields(string(line))
   589  		if len(f) == 0 {
   590  			// blank line; skip it
   591  			continue
   592  		}
   593  		if len(f) != 3 {
   594  			if cfg.CmdName == "mod tidy" {
   595  				// ignore malformed line so that go mod tidy can fix go.sum
   596  				continue
   597  			} else {
   598  				base.Fatalf("malformed go.sum:\n%s:%d: wrong number of fields %v\n", file, lineno, len(f))
   599  			}
   600  		}
   601  		if f[2] == emptyGoModHash {
   602  			// Old bug; drop it.
   603  			continue
   604  		}
   605  		mod := module.Version{Path: f[0], Version: f[1]}
   606  		dst[mod] = append(dst[mod], f[2])
   607  	}
   608  }
   609  
   610  // HaveSum returns true if the go.sum file contains an entry for mod.
   611  // The entry's hash must be generated with a known hash algorithm.
   612  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   613  // .mod and .zip files.
   614  func HaveSum(mod module.Version) bool {
   615  	goSum.mu.Lock()
   616  	defer goSum.mu.Unlock()
   617  	inited, err := initGoSum()
   618  	if err != nil || !inited {
   619  		return false
   620  	}
   621  	for _, goSums := range goSum.w {
   622  		for _, h := range goSums[mod] {
   623  			if !strings.HasPrefix(h, "h1:") {
   624  				continue
   625  			}
   626  			if !goSum.status[modSum{mod, h}].dirty {
   627  				return true
   628  			}
   629  		}
   630  	}
   631  	for _, h := range goSum.m[mod] {
   632  		if !strings.HasPrefix(h, "h1:") {
   633  			continue
   634  		}
   635  		if !goSum.status[modSum{mod, h}].dirty {
   636  			return true
   637  		}
   638  	}
   639  	return false
   640  }
   641  
   642  // RecordedSum returns the sum if the go.sum file contains an entry for mod.
   643  // The boolean reports true if an entry was found or
   644  // false if no entry found or two conflicting sums are found.
   645  // The entry's hash must be generated with a known hash algorithm.
   646  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   647  // .mod and .zip files.
   648  func RecordedSum(mod module.Version) (sum string, ok bool) {
   649  	goSum.mu.Lock()
   650  	defer goSum.mu.Unlock()
   651  	inited, err := initGoSum()
   652  	foundSum := ""
   653  	if err != nil || !inited {
   654  		return "", false
   655  	}
   656  	for _, goSums := range goSum.w {
   657  		for _, h := range goSums[mod] {
   658  			if !strings.HasPrefix(h, "h1:") {
   659  				continue
   660  			}
   661  			if !goSum.status[modSum{mod, h}].dirty {
   662  				if foundSum != "" && foundSum != h { // conflicting sums exist
   663  					return "", false
   664  				}
   665  				foundSum = h
   666  			}
   667  		}
   668  	}
   669  	for _, h := range goSum.m[mod] {
   670  		if !strings.HasPrefix(h, "h1:") {
   671  			continue
   672  		}
   673  		if !goSum.status[modSum{mod, h}].dirty {
   674  			if foundSum != "" && foundSum != h { // conflicting sums exist
   675  				return "", false
   676  			}
   677  			foundSum = h
   678  		}
   679  	}
   680  	return foundSum, true
   681  }
   682  
   683  // checkMod checks the given module's checksum and Go version.
   684  func checkMod(ctx context.Context, mod module.Version) {
   685  	// Do the file I/O before acquiring the go.sum lock.
   686  	ziphash, err := CachePath(ctx, mod, "ziphash")
   687  	if err != nil {
   688  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   689  	}
   690  	data, err := lockedfile.Read(ziphash)
   691  	if err != nil {
   692  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   693  	}
   694  	data = bytes.TrimSpace(data)
   695  	if !isValidSum(data) {
   696  		// Recreate ziphash file from zip file and use that to check the mod sum.
   697  		zip, err := CachePath(ctx, mod, "zip")
   698  		if err != nil {
   699  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   700  		}
   701  		err = hashZip(mod, zip, ziphash)
   702  		if err != nil {
   703  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   704  		}
   705  		return
   706  	}
   707  	h := string(data)
   708  	if !strings.HasPrefix(h, "h1:") {
   709  		base.Fatalf("verifying %v", module.VersionError(mod, fmt.Errorf("unexpected ziphash: %q", h)))
   710  	}
   711  
   712  	if err := checkModSum(mod, h); err != nil {
   713  		base.Fatalf("%s", err)
   714  	}
   715  }
   716  
   717  // goModSum returns the checksum for the go.mod contents.
   718  func goModSum(data []byte) (string, error) {
   719  	return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
   720  		return io.NopCloser(bytes.NewReader(data)), nil
   721  	})
   722  }
   723  
   724  // checkGoMod checks the given module's go.mod checksum;
   725  // data is the go.mod content.
   726  func checkGoMod(path, version string, data []byte) error {
   727  	h, err := goModSum(data)
   728  	if err != nil {
   729  		return &module.ModuleError{Path: path, Version: version, Err: fmt.Errorf("verifying go.mod: %v", err)}
   730  	}
   731  
   732  	return checkModSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
   733  }
   734  
   735  // checkModSum checks that the recorded checksum for mod is h.
   736  //
   737  // mod.Version may have the additional suffix "/go.mod" to request the checksum
   738  // for the module's go.mod file only.
   739  func checkModSum(mod module.Version, h string) error {
   740  	// We lock goSum when manipulating it,
   741  	// but we arrange to release the lock when calling checkSumDB,
   742  	// so that parallel calls to checkModHash can execute parallel calls
   743  	// to checkSumDB.
   744  
   745  	// Check whether mod+h is listed in go.sum already. If so, we're done.
   746  	goSum.mu.Lock()
   747  	inited, err := initGoSum()
   748  	if err != nil {
   749  		goSum.mu.Unlock()
   750  		return err
   751  	}
   752  	done := inited && haveModSumLocked(mod, h)
   753  	if inited {
   754  		st := goSum.status[modSum{mod, h}]
   755  		st.used = true
   756  		goSum.status[modSum{mod, h}] = st
   757  	}
   758  	goSum.mu.Unlock()
   759  
   760  	if done {
   761  		return nil
   762  	}
   763  
   764  	// Not listed, so we want to add them.
   765  	// Consult checksum database if appropriate.
   766  	if useSumDB(mod) {
   767  		// Calls base.Fatalf if mismatch detected.
   768  		if err := checkSumDB(mod, h); err != nil {
   769  			return err
   770  		}
   771  	}
   772  
   773  	// Add mod+h to go.sum, if it hasn't appeared already.
   774  	if inited {
   775  		goSum.mu.Lock()
   776  		addModSumLocked(mod, h)
   777  		st := goSum.status[modSum{mod, h}]
   778  		st.dirty = true
   779  		goSum.status[modSum{mod, h}] = st
   780  		goSum.mu.Unlock()
   781  	}
   782  	return nil
   783  }
   784  
   785  // haveModSumLocked reports whether the pair mod,h is already listed in go.sum.
   786  // If it finds a conflicting pair instead, it calls base.Fatalf.
   787  // goSum.mu must be locked.
   788  func haveModSumLocked(mod module.Version, h string) bool {
   789  	sumFileName := "go.sum"
   790  	if strings.HasSuffix(GoSumFile, "go.work.sum") {
   791  		sumFileName = "go.work.sum"
   792  	}
   793  	for _, vh := range goSum.m[mod] {
   794  		if h == vh {
   795  			return true
   796  		}
   797  		if strings.HasPrefix(vh, "h1:") {
   798  			base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, sumFileName, vh)
   799  		}
   800  	}
   801  	// Also check workspace sums.
   802  	foundMatch := false
   803  	// Check sums from all files in case there are conflicts between
   804  	// the files.
   805  	for goSumFile, goSums := range goSum.w {
   806  		for _, vh := range goSums[mod] {
   807  			if h == vh {
   808  				foundMatch = true
   809  			} else if strings.HasPrefix(vh, "h1:") {
   810  				base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, goSumFile, vh)
   811  			}
   812  		}
   813  	}
   814  	return foundMatch
   815  }
   816  
   817  // addModSumLocked adds the pair mod,h to go.sum.
   818  // goSum.mu must be locked.
   819  func addModSumLocked(mod module.Version, h string) {
   820  	if haveModSumLocked(mod, h) {
   821  		return
   822  	}
   823  	if len(goSum.m[mod]) > 0 {
   824  		fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h)
   825  	}
   826  	goSum.m[mod] = append(goSum.m[mod], h)
   827  }
   828  
   829  // checkSumDB checks the mod, h pair against the Go checksum database.
   830  // It calls base.Fatalf if the hash is to be rejected.
   831  func checkSumDB(mod module.Version, h string) error {
   832  	modWithoutSuffix := mod
   833  	noun := "module"
   834  	if before, found := strings.CutSuffix(mod.Version, "/go.mod"); found {
   835  		noun = "go.mod"
   836  		modWithoutSuffix.Version = before
   837  	}
   838  
   839  	db, lines, err := lookupSumDB(mod)
   840  	if err != nil {
   841  		return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: %v", noun, err))
   842  	}
   843  
   844  	have := mod.Path + " " + mod.Version + " " + h
   845  	prefix := mod.Path + " " + mod.Version + " h1:"
   846  	for _, line := range lines {
   847  		if line == have {
   848  			return nil
   849  		}
   850  		if strings.HasPrefix(line, prefix) {
   851  			return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, noun, h, db, line[len(prefix)-len("h1:"):]))
   852  		}
   853  	}
   854  	return nil
   855  }
   856  
   857  // Sum returns the checksum for the downloaded copy of the given module,
   858  // if present in the download cache.
   859  func Sum(ctx context.Context, mod module.Version) string {
   860  	if cfg.GOMODCACHE == "" {
   861  		// Do not use current directory.
   862  		return ""
   863  	}
   864  
   865  	ziphash, err := CachePath(ctx, mod, "ziphash")
   866  	if err != nil {
   867  		return ""
   868  	}
   869  	data, err := lockedfile.Read(ziphash)
   870  	if err != nil {
   871  		return ""
   872  	}
   873  	data = bytes.TrimSpace(data)
   874  	if !isValidSum(data) {
   875  		return ""
   876  	}
   877  	return string(data)
   878  }
   879  
   880  // isValidSum returns true if data is the valid contents of a zip hash file.
   881  // Certain critical files are written to disk by first truncating
   882  // then writing the actual bytes, so that if the write fails
   883  // the corrupt file should contain at least one of the null
   884  // bytes written by the truncate operation.
   885  func isValidSum(data []byte) bool {
   886  	if bytes.IndexByte(data, '\000') >= 0 {
   887  		return false
   888  	}
   889  
   890  	if len(data) != len("h1:")+base64.StdEncoding.EncodedLen(sha256.Size) {
   891  		return false
   892  	}
   893  
   894  	return true
   895  }
   896  
   897  var ErrGoSumDirty = errors.New("updates to go.sum needed, disabled by -mod=readonly")
   898  
   899  // WriteGoSum writes the go.sum file if it needs to be updated.
   900  //
   901  // keep is used to check whether a newly added sum should be saved in go.sum.
   902  // It should have entries for both module content sums and go.mod sums
   903  // (version ends with "/go.mod"). Existing sums will be preserved unless they
   904  // have been marked for deletion with TrimGoSum.
   905  func WriteGoSum(ctx context.Context, keep map[module.Version]bool, readonly bool) error {
   906  	goSum.mu.Lock()
   907  	defer goSum.mu.Unlock()
   908  
   909  	// If we haven't read the go.sum file yet, don't bother writing it.
   910  	if !goSum.enabled {
   911  		return nil
   912  	}
   913  
   914  	// Check whether we need to add sums for which keep[m] is true or remove
   915  	// unused sums marked with TrimGoSum. If there are no changes to make,
   916  	// just return without opening go.sum.
   917  	dirty := false
   918  Outer:
   919  	for m, hs := range goSum.m {
   920  		for _, h := range hs {
   921  			st := goSum.status[modSum{m, h}]
   922  			if st.dirty && (!st.used || keep[m]) {
   923  				dirty = true
   924  				break Outer
   925  			}
   926  		}
   927  	}
   928  	if !dirty {
   929  		return nil
   930  	}
   931  	if readonly {
   932  		return ErrGoSumDirty
   933  	}
   934  	if fsys.Replaced(GoSumFile) {
   935  		base.Fatalf("go: updates to go.sum needed, but go.sum is part of the overlay specified with -overlay")
   936  	}
   937  
   938  	// Make a best-effort attempt to acquire the side lock, only to exclude
   939  	// previous versions of the 'go' command from making simultaneous edits.
   940  	if unlock, err := SideLock(ctx); err == nil {
   941  		defer unlock()
   942  	}
   943  
   944  	err := lockedfile.Transform(GoSumFile, func(data []byte) ([]byte, error) {
   945  		tidyGoSum := tidyGoSum(data, keep)
   946  		return tidyGoSum, nil
   947  	})
   948  
   949  	if err != nil {
   950  		return fmt.Errorf("updating go.sum: %w", err)
   951  	}
   952  
   953  	goSum.status = make(map[modSum]modSumStatus)
   954  	goSum.overwrite = false
   955  	return nil
   956  }
   957  
   958  // TidyGoSum returns a tidy version of the go.sum file.
   959  // A missing go.sum file is treated as if empty.
   960  func TidyGoSum(keep map[module.Version]bool) (before, after []byte) {
   961  	goSum.mu.Lock()
   962  	defer goSum.mu.Unlock()
   963  	before, err := lockedfile.Read(GoSumFile)
   964  	if err != nil && !errors.Is(err, fs.ErrNotExist) {
   965  		base.Fatalf("reading go.sum: %v", err)
   966  	}
   967  	after = tidyGoSum(before, keep)
   968  	return before, after
   969  }
   970  
   971  // tidyGoSum returns a tidy version of the go.sum file.
   972  // The goSum lock must be held.
   973  func tidyGoSum(data []byte, keep map[module.Version]bool) []byte {
   974  	if !goSum.overwrite {
   975  		// Incorporate any sums added by other processes in the meantime.
   976  		// Add only the sums that we actually checked: the user may have edited or
   977  		// truncated the file to remove erroneous hashes, and we shouldn't restore
   978  		// them without good reason.
   979  		goSum.m = make(map[module.Version][]string, len(goSum.m))
   980  		readGoSum(goSum.m, GoSumFile, data)
   981  		for ms, st := range goSum.status {
   982  			if st.used && !sumInWorkspaceModulesLocked(ms.mod) {
   983  				addModSumLocked(ms.mod, ms.sum)
   984  			}
   985  		}
   986  	}
   987  
   988  	mods := make([]module.Version, 0, len(goSum.m))
   989  	for m := range goSum.m {
   990  		mods = append(mods, m)
   991  	}
   992  	module.Sort(mods)
   993  
   994  	var buf bytes.Buffer
   995  	for _, m := range mods {
   996  		list := goSum.m[m]
   997  		sort.Strings(list)
   998  		str.Uniq(&list)
   999  		for _, h := range list {
  1000  			st := goSum.status[modSum{m, h}]
  1001  			if (!st.dirty || (st.used && keep[m])) && !sumInWorkspaceModulesLocked(m) {
  1002  				fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
  1003  			}
  1004  		}
  1005  	}
  1006  	return buf.Bytes()
  1007  }
  1008  
  1009  func sumInWorkspaceModulesLocked(m module.Version) bool {
  1010  	for _, goSums := range goSum.w {
  1011  		if _, ok := goSums[m]; ok {
  1012  			return true
  1013  		}
  1014  	}
  1015  	return false
  1016  }
  1017  
  1018  // TrimGoSum trims go.sum to contain only the modules needed for reproducible
  1019  // builds.
  1020  //
  1021  // keep is used to check whether a sum should be retained in go.mod. It should
  1022  // have entries for both module content sums and go.mod sums (version ends
  1023  // with "/go.mod").
  1024  func TrimGoSum(keep map[module.Version]bool) {
  1025  	goSum.mu.Lock()
  1026  	defer goSum.mu.Unlock()
  1027  	inited, err := initGoSum()
  1028  	if err != nil {
  1029  		base.Fatalf("%s", err)
  1030  	}
  1031  	if !inited {
  1032  		return
  1033  	}
  1034  
  1035  	for m, hs := range goSum.m {
  1036  		if !keep[m] {
  1037  			for _, h := range hs {
  1038  				goSum.status[modSum{m, h}] = modSumStatus{used: false, dirty: true}
  1039  			}
  1040  			goSum.overwrite = true
  1041  		}
  1042  	}
  1043  }
  1044  
  1045  const goSumMismatch = `
  1046  
  1047  SECURITY ERROR
  1048  This download does NOT match an earlier download recorded in go.sum.
  1049  The bits may have been replaced on the origin server, or an attacker may
  1050  have intercepted the download attempt.
  1051  
  1052  For more information, see 'go help module-auth'.
  1053  `
  1054  
  1055  const sumdbMismatch = `
  1056  
  1057  SECURITY ERROR
  1058  This download does NOT match the one reported by the checksum server.
  1059  The bits may have been replaced on the origin server, or an attacker may
  1060  have intercepted the download attempt.
  1061  
  1062  For more information, see 'go help module-auth'.
  1063  `
  1064  
  1065  const hashVersionMismatch = `
  1066  
  1067  SECURITY WARNING
  1068  This download is listed in go.sum, but using an unknown hash algorithm.
  1069  The download cannot be verified.
  1070  
  1071  For more information, see 'go help module-auth'.
  1072  
  1073  `
  1074  
  1075  var HelpModuleAuth = &base.Command{
  1076  	UsageLine: "module-auth",
  1077  	Short:     "module authentication using go.sum",
  1078  	Long: `
  1079  When the go command downloads a module zip file or go.mod file into the
  1080  module cache, it computes a cryptographic hash and compares it with a known
  1081  value to verify the file hasn't changed since it was first downloaded. Known
  1082  hashes are stored in a file in the module root directory named go.sum. Hashes
  1083  may also be downloaded from the checksum database depending on the values of
  1084  GOSUMDB, GOPRIVATE, and GONOSUMDB.
  1085  
  1086  For details, see https://golang.org/ref/mod#authenticating.
  1087  `,
  1088  }
  1089  
  1090  var HelpPrivate = &base.Command{
  1091  	UsageLine: "private",
  1092  	Short:     "configuration for downloading non-public code",
  1093  	Long: `
  1094  The go command defaults to downloading modules from the public Go module
  1095  mirror at proxy.golang.org. It also defaults to validating downloaded modules,
  1096  regardless of source, against the public Go checksum database at sum.golang.org.
  1097  These defaults work well for publicly available source code.
  1098  
  1099  The GOPRIVATE environment variable controls which modules the go command
  1100  considers to be private (not available publicly) and should therefore not use
  1101  the proxy or checksum database. The variable is a comma-separated list of
  1102  glob patterns (in the syntax of Go's path.Match) of module path prefixes.
  1103  For example,
  1104  
  1105  	GOPRIVATE=*.corp.example.com,rsc.io/private
  1106  
  1107  causes the go command to treat as private any module with a path prefix
  1108  matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private,
  1109  and rsc.io/private/quux.
  1110  
  1111  For fine-grained control over module download and validation, the GONOPROXY
  1112  and GONOSUMDB environment variables accept the same kind of glob list
  1113  and override GOPRIVATE for the specific decision of whether to use the proxy
  1114  and checksum database, respectively.
  1115  
  1116  For example, if a company ran a module proxy serving private modules,
  1117  users would configure go using:
  1118  
  1119  	GOPRIVATE=*.corp.example.com
  1120  	GOPROXY=proxy.example.com
  1121  	GONOPROXY=none
  1122  
  1123  The GOPRIVATE variable is also used to define the "public" and "private"
  1124  patterns for the GOVCS variable; see 'go help vcs'. For that usage,
  1125  GOPRIVATE applies even in GOPATH mode. In that case, it matches import paths
  1126  instead of module paths.
  1127  
  1128  The 'go env -w' command (see 'go help env') can be used to set these variables
  1129  for future go command invocations.
  1130  
  1131  For more details, see https://golang.org/ref/mod#private-modules.
  1132  `,
  1133  }
  1134  

View as plain text