fetch.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"archive/zip"
     9  	"bytes"
    10  	"context"
    11  	"crypto/sha256"
    12  	"encoding/base64"
    13  	"errors"
    14  	"fmt"
    15  	"io"
    16  	"io/fs"
    17  	"os"
    18  	"path/filepath"
    19  	"sort"
    20  	"strings"
    21  	"sync"
    22  
    23  	"cmd/go/internal/base"
    24  	"cmd/go/internal/cfg"
    25  	"cmd/go/internal/fsys"
    26  	"cmd/go/internal/gover"
    27  	"cmd/go/internal/lockedfile"
    28  	"cmd/go/internal/str"
    29  	"cmd/go/internal/trace"
    30  	"cmd/internal/par"
    31  	"cmd/internal/robustio"
    32  
    33  	"golang.org/x/mod/module"
    34  	"golang.org/x/mod/sumdb/dirhash"
    35  	modzip "golang.org/x/mod/zip"
    36  )
    37  
    38  var ErrToolchain = errors.New("internal error: invalid operation on toolchain module")
    39  
    40  // Download downloads the specific module version to the
    41  // local download cache and returns the name of the directory
    42  // corresponding to the root of the module's file tree.
    43  func (f *Fetcher) Download(ctx context.Context, mod module.Version) (dir string, err error) {
    44  	if gover.IsToolchain(mod.Path) {
    45  		return "", ErrToolchain
    46  	}
    47  	if err := checkCacheDir(ctx); err != nil {
    48  		base.Fatal(err)
    49  	}
    50  
    51  	// The par.Cache here avoids duplicate work.
    52  	return f.downloadCache.Do(mod, func() (string, error) {
    53  		dir, err := f.download(ctx, mod)
    54  		if err != nil {
    55  			return "", err
    56  		}
    57  		f.checkMod(ctx, mod)
    58  
    59  		// If go.mod exists (not an old legacy module), check version is not too new.
    60  		if data, err := os.ReadFile(filepath.Join(dir, "go.mod")); err == nil {
    61  			goVersion := gover.GoModLookup(data, "go")
    62  			if gover.Compare(goVersion, gover.Local()) > 0 {
    63  				return "", &gover.TooNewError{What: mod.String(), GoVersion: goVersion}
    64  			}
    65  		} else if !errors.Is(err, fs.ErrNotExist) {
    66  			return "", err
    67  		}
    68  
    69  		return dir, nil
    70  	})
    71  }
    72  
    73  // Unzip is like Download but is given the explicit zip file to use,
    74  // rather than downloading it. This is used for the GOFIPS140 zip files,
    75  // which ship in the Go distribution itself.
    76  func (f *Fetcher) Unzip(ctx context.Context, mod module.Version, zipfile string) (dir string, err error) {
    77  	if err := checkCacheDir(ctx); err != nil {
    78  		base.Fatal(err)
    79  	}
    80  
    81  	return f.downloadCache.Do(mod, func() (string, error) {
    82  		ctx, span := trace.StartSpan(ctx, "modfetch.Unzip "+mod.String())
    83  		defer span.Done()
    84  
    85  		dir, err = DownloadDir(ctx, mod)
    86  		if err == nil {
    87  			// The directory has already been completely extracted (no .partial file exists).
    88  			return dir, nil
    89  		} else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
    90  			return "", err
    91  		}
    92  
    93  		return unzip(ctx, mod, zipfile)
    94  	})
    95  }
    96  
    97  func (f *Fetcher) download(ctx context.Context, mod module.Version) (dir string, err error) {
    98  	ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String())
    99  	defer span.Done()
   100  
   101  	dir, err = DownloadDir(ctx, mod)
   102  	if err == nil {
   103  		// The directory has already been completely extracted (no .partial file exists).
   104  		return dir, nil
   105  	} else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
   106  		return "", err
   107  	}
   108  
   109  	// To avoid cluttering the cache with extraneous files,
   110  	// DownloadZip uses the same lockfile as Download.
   111  	// Invoke DownloadZip before locking the file.
   112  	zipfile, err := f.DownloadZip(ctx, mod)
   113  	if err != nil {
   114  		return "", err
   115  	}
   116  
   117  	return unzip(ctx, mod, zipfile)
   118  }
   119  
   120  func unzip(ctx context.Context, mod module.Version, zipfile string) (dir string, err error) {
   121  	unlock, err := lockVersion(ctx, mod)
   122  	if err != nil {
   123  		return "", err
   124  	}
   125  	defer unlock()
   126  
   127  	ctx, span := trace.StartSpan(ctx, "unzip "+zipfile)
   128  	defer span.Done()
   129  
   130  	// Check whether the directory was populated while we were waiting on the lock.
   131  	dir, dirErr := DownloadDir(ctx, mod)
   132  	if dirErr == nil {
   133  		return dir, nil
   134  	}
   135  	_, dirExists := dirErr.(*DownloadDirPartialError)
   136  
   137  	// Clean up any remaining temporary directories created by old versions
   138  	// (before 1.16), as well as partially extracted directories (indicated by
   139  	// DownloadDirPartialError, usually because of a .partial file). This is only
   140  	// safe to do because the lock file ensures that their writers are no longer
   141  	// active.
   142  	parentDir := filepath.Dir(dir)
   143  	tmpPrefix := filepath.Base(dir) + ".tmp-"
   144  	if old, err := filepath.Glob(filepath.Join(str.QuoteGlob(parentDir), str.QuoteGlob(tmpPrefix)+"*")); err == nil {
   145  		for _, path := range old {
   146  			RemoveAll(path) // best effort
   147  		}
   148  	}
   149  	if dirExists {
   150  		if err := RemoveAll(dir); err != nil {
   151  			return "", err
   152  		}
   153  	}
   154  
   155  	partialPath, err := CachePath(ctx, mod, "partial")
   156  	if err != nil {
   157  		return "", err
   158  	}
   159  
   160  	// Extract the module zip directory at its final location.
   161  	//
   162  	// To prevent other processes from reading the directory if we crash,
   163  	// create a .partial file before extracting the directory, and delete
   164  	// the .partial file afterward (all while holding the lock).
   165  	//
   166  	// Before Go 1.16, we extracted to a temporary directory with a random name
   167  	// then renamed it into place with os.Rename. On Windows, this failed with
   168  	// ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner)
   169  	// opened files in the temporary directory.
   170  	//
   171  	// Go 1.14.2 and higher respect .partial files. Older versions may use
   172  	// partially extracted directories. 'go mod verify' can detect this,
   173  	// and 'go clean -modcache' can fix it.
   174  	if err := os.MkdirAll(parentDir, 0o777); err != nil {
   175  		return "", err
   176  	}
   177  	if err := os.WriteFile(partialPath, nil, 0o666); err != nil {
   178  		return "", err
   179  	}
   180  	if err := modzip.Unzip(dir, mod, zipfile); err != nil {
   181  		fmt.Fprintf(os.Stderr, "-> %s\n", err)
   182  		if rmErr := RemoveAll(dir); rmErr == nil {
   183  			os.Remove(partialPath)
   184  		}
   185  		return "", err
   186  	}
   187  	if err := os.Remove(partialPath); err != nil {
   188  		return "", err
   189  	}
   190  
   191  	if !cfg.ModCacheRW {
   192  		makeDirsReadOnly(dir)
   193  	}
   194  	return dir, nil
   195  }
   196  
   197  var downloadZipCache par.ErrCache[module.Version, string]
   198  
   199  // DownloadZip downloads the specific module version to the
   200  // local zip cache and returns the name of the zip file.
   201  func (f *Fetcher) DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) {
   202  	// The par.Cache here avoids duplicate work.
   203  	return downloadZipCache.Do(mod, func() (string, error) {
   204  		zipfile, err := CachePath(ctx, mod, "zip")
   205  		if err != nil {
   206  			return "", err
   207  		}
   208  		ziphashfile := zipfile + "hash"
   209  
   210  		// Return early if the zip and ziphash files exist.
   211  		if _, err := os.Stat(zipfile); err == nil {
   212  			if _, err := os.Stat(ziphashfile); err == nil {
   213  				if !HaveSum(f, mod) {
   214  					f.checkMod(ctx, mod)
   215  				}
   216  				return zipfile, nil
   217  			}
   218  		}
   219  
   220  		// The zip or ziphash file does not exist. Acquire the lock and create them.
   221  		if cfg.CmdName != "mod download" {
   222  			vers := mod.Version
   223  			if mod.Path == "golang.org/toolchain" {
   224  				// Shorten v0.0.1-go1.13.1.darwin-amd64 to go1.13.1.darwin-amd64
   225  				_, vers, _ = strings.Cut(vers, "-")
   226  				if i := strings.LastIndex(vers, "."); i >= 0 {
   227  					goos, goarch, _ := strings.Cut(vers[i+1:], "-")
   228  					vers = vers[:i] + " (" + goos + "/" + goarch + ")"
   229  				}
   230  				fmt.Fprintf(os.Stderr, "go: downloading %s\n", vers)
   231  			} else {
   232  				fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, vers)
   233  			}
   234  		}
   235  		unlock, err := lockVersion(ctx, mod)
   236  		if err != nil {
   237  			return "", err
   238  		}
   239  		defer unlock()
   240  
   241  		if err := f.downloadZip(ctx, mod, zipfile); err != nil {
   242  			return "", err
   243  		}
   244  		return zipfile, nil
   245  	})
   246  }
   247  
   248  func (f *Fetcher) downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) {
   249  	ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile)
   250  	defer span.Done()
   251  
   252  	// Double-check that the zipfile was not created while we were waiting for
   253  	// the lock in DownloadZip.
   254  	ziphashfile := zipfile + "hash"
   255  	var zipExists, ziphashExists bool
   256  	if _, err := os.Stat(zipfile); err == nil {
   257  		zipExists = true
   258  	}
   259  	if _, err := os.Stat(ziphashfile); err == nil {
   260  		ziphashExists = true
   261  	}
   262  	if zipExists && ziphashExists {
   263  		return nil
   264  	}
   265  
   266  	// Create parent directories.
   267  	if err := os.MkdirAll(filepath.Dir(zipfile), 0o777); err != nil {
   268  		return err
   269  	}
   270  
   271  	// Clean up any remaining tempfiles from previous runs.
   272  	// This is only safe to do because the lock file ensures that their
   273  	// writers are no longer active.
   274  	tmpPattern := filepath.Base(zipfile) + "*.tmp"
   275  	if old, err := filepath.Glob(filepath.Join(str.QuoteGlob(filepath.Dir(zipfile)), tmpPattern)); err == nil {
   276  		for _, path := range old {
   277  			os.Remove(path) // best effort
   278  		}
   279  	}
   280  
   281  	// If the zip file exists, the ziphash file must have been deleted
   282  	// or lost after a file system crash. Re-hash the zip without downloading.
   283  	if zipExists {
   284  		return hashZip(f, mod, zipfile, ziphashfile)
   285  	}
   286  
   287  	// From here to the os.Rename call below is functionally almost equivalent to
   288  	// renameio.WriteToFile, with one key difference: we want to validate the
   289  	// contents of the file (by hashing it) before we commit it. Because the file
   290  	// is zip-compressed, we need an actual file — or at least an io.ReaderAt — to
   291  	// validate it: we can't just tee the stream as we write it.
   292  	file, err := tempFile(ctx, filepath.Dir(zipfile), filepath.Base(zipfile), 0o666)
   293  	if err != nil {
   294  		return err
   295  	}
   296  	defer func() {
   297  		if err != nil {
   298  			file.Close()
   299  			os.Remove(file.Name())
   300  		}
   301  	}()
   302  
   303  	var unrecoverableErr error
   304  	err = TryProxies(func(proxy string) error {
   305  		if unrecoverableErr != nil {
   306  			return unrecoverableErr
   307  		}
   308  		repo := f.Lookup(ctx, proxy, mod.Path)
   309  		err := repo.Zip(ctx, file, mod.Version)
   310  		if err != nil {
   311  			// Zip may have partially written to f before failing.
   312  			// (Perhaps the server crashed while sending the file?)
   313  			// Since we allow fallback on error in some cases, we need to fix up the
   314  			// file to be empty again for the next attempt.
   315  			if _, err := file.Seek(0, io.SeekStart); err != nil {
   316  				unrecoverableErr = err
   317  				return err
   318  			}
   319  			if err := file.Truncate(0); err != nil {
   320  				unrecoverableErr = err
   321  				return err
   322  			}
   323  		}
   324  		return err
   325  	})
   326  	if err != nil {
   327  		return err
   328  	}
   329  
   330  	// Double-check that the paths within the zip file are well-formed.
   331  	//
   332  	// TODO(bcmills): There is a similar check within the Unzip function. Can we eliminate one?
   333  	fi, err := file.Stat()
   334  	if err != nil {
   335  		return err
   336  	}
   337  	z, err := zip.NewReader(file, fi.Size())
   338  	if err != nil {
   339  		return err
   340  	}
   341  	prefix := mod.Path + "@" + mod.Version + "/"
   342  	for _, zf := range z.File {
   343  		if !strings.HasPrefix(zf.Name, prefix) {
   344  			return fmt.Errorf("zip for %s has unexpected file %s", prefix[:len(prefix)-1], zf.Name)
   345  		}
   346  	}
   347  
   348  	if err := file.Close(); err != nil {
   349  		return err
   350  	}
   351  
   352  	// Hash the zip file and check the sum before renaming to the final location.
   353  	if err := hashZip(f, mod, file.Name(), ziphashfile); err != nil {
   354  		return err
   355  	}
   356  	if err := os.Rename(file.Name(), zipfile); err != nil {
   357  		return err
   358  	}
   359  
   360  	// TODO(bcmills): Should we make the .zip and .ziphash files read-only to discourage tampering?
   361  
   362  	return nil
   363  }
   364  
   365  // hashZip reads the zip file opened in f, then writes the hash to ziphashfile,
   366  // overwriting that file if it exists.
   367  //
   368  // If the hash does not match go.sum (or the sumdb if enabled), hashZip returns
   369  // an error and does not write ziphashfile.
   370  func hashZip(f *Fetcher, mod module.Version, zipfile, ziphashfile string) (err error) {
   371  	hash, err := dirhash.HashZip(zipfile, dirhash.DefaultHash)
   372  	if err != nil {
   373  		return err
   374  	}
   375  	if err := checkModSum(f, mod, hash); err != nil {
   376  		return err
   377  	}
   378  	hf, err := lockedfile.Create(ziphashfile)
   379  	if err != nil {
   380  		return err
   381  	}
   382  	defer func() {
   383  		if closeErr := hf.Close(); err == nil && closeErr != nil {
   384  			err = closeErr
   385  		}
   386  	}()
   387  	if err := hf.Truncate(int64(len(hash))); err != nil {
   388  		return err
   389  	}
   390  	if _, err := hf.WriteAt([]byte(hash), 0); err != nil {
   391  		return err
   392  	}
   393  	return nil
   394  }
   395  
   396  // makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
   397  // and its transitive contents.
   398  func makeDirsReadOnly(dir string) {
   399  	type pathMode struct {
   400  		path string
   401  		mode fs.FileMode
   402  	}
   403  	var dirs []pathMode // in lexical order
   404  	filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
   405  		if err == nil && d.IsDir() {
   406  			info, err := d.Info()
   407  			if err == nil && info.Mode()&0o222 != 0 {
   408  				dirs = append(dirs, pathMode{path, info.Mode()})
   409  			}
   410  		}
   411  		return nil
   412  	})
   413  
   414  	// Run over list backward to chmod children before parents.
   415  	for i := len(dirs) - 1; i >= 0; i-- {
   416  		os.Chmod(dirs[i].path, dirs[i].mode&^0o222)
   417  	}
   418  }
   419  
   420  // RemoveAll removes a directory written by Download or Unzip, first applying
   421  // any permission changes needed to do so.
   422  func RemoveAll(dir string) error {
   423  	// Module cache has 0555 directories; make them writable in order to remove content.
   424  	filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error {
   425  		if err != nil {
   426  			return nil // ignore errors walking in file system
   427  		}
   428  		if info.IsDir() {
   429  			os.Chmod(path, 0o777)
   430  		}
   431  		return nil
   432  	})
   433  	return robustio.RemoveAll(dir)
   434  }
   435  
   436  // The GoSumFile, WorkspaceGoSumFiles, and goSum are global state that must not be
   437  // accessed by any of the exported functions of this package after they return, because
   438  // they can be modified by the non-thread-safe SetState function.
   439  
   440  type modSum struct {
   441  	mod module.Version
   442  	sum string
   443  }
   444  
   445  type sumState struct {
   446  	m         map[module.Version][]string            // content of go.sum file
   447  	w         map[string]map[module.Version][]string // sum file in workspace -> content of that sum file
   448  	status    map[modSum]modSumStatus                // state of sums in m
   449  	overwrite bool                                   // if true, overwrite go.sum without incorporating its contents
   450  	enabled   bool                                   // whether to use go.sum at all
   451  }
   452  
   453  type modSumStatus struct {
   454  	used, dirty bool
   455  }
   456  
   457  // Fetcher holds a snapshot of the global state of the modfetch package.
   458  type Fetcher struct {
   459  	// path to go.sum; set by package modload
   460  	goSumFile string
   461  	// path to module go.sums in workspace; set by package modload
   462  	workspaceGoSumFiles []string
   463  	// The Lookup cache is used cache the work done by Lookup.
   464  	// It is important that the global functions of this package that access it do not
   465  	// do so after they return.
   466  	lookupCache *par.Cache[lookupCacheKey, Repo]
   467  	// The downloadCache is used to cache the operation of downloading a module to disk
   468  	// (if it's not already downloaded) and getting the directory it was downloaded to.
   469  	// It is important that downloadCache must not be accessed by any of the exported
   470  	// functions of this package after they return, because it can be modified by the
   471  	// non-thread-safe SetState function.
   472  	downloadCache *par.ErrCache[module.Version, string] // version → directory;
   473  
   474  	mu       sync.Mutex
   475  	sumState sumState
   476  }
   477  
   478  func NewFetcher() *Fetcher {
   479  	f := new(Fetcher)
   480  	f.lookupCache = new(par.Cache[lookupCacheKey, Repo])
   481  	f.downloadCache = new(par.ErrCache[module.Version, string])
   482  	return f
   483  }
   484  
   485  func (f *Fetcher) GoSumFile() string {
   486  	return f.goSumFile
   487  }
   488  
   489  func (f *Fetcher) SetGoSumFile(str string) {
   490  	f.goSumFile = str
   491  }
   492  
   493  func (f *Fetcher) AddWorkspaceGoSumFile(file string) {
   494  	f.workspaceGoSumFiles = append(f.workspaceGoSumFiles, file)
   495  }
   496  
   497  // ReloadWorkspaceGoSumFiles reloads the go.sum files for workspace modules.
   498  func (f *Fetcher) ReloadWorkspaceGoSumFiles() error {
   499  	f.mu.Lock()
   500  	defer f.mu.Unlock()
   501  	if _, err := f.initGoSum(); err != nil {
   502  		return err
   503  	}
   504  
   505  	w := make(map[string]map[module.Version][]string, len(f.workspaceGoSumFiles))
   506  	for _, file := range f.workspaceGoSumFiles {
   507  		w[file] = make(map[module.Version][]string)
   508  		if _, err := readGoSumFile(w[file], file); err != nil {
   509  			return err
   510  		}
   511  	}
   512  	f.sumState.w = w
   513  	return nil
   514  }
   515  
   516  // Reset resets globals in the modfetch package, so previous loads don't affect
   517  // contents of go.sum files.
   518  func (f *Fetcher) Reset() {
   519  	f.SetState(NewFetcher())
   520  }
   521  
   522  // SetState sets the global state of the modfetch package to the newState, and returns the previous
   523  // global state. newState should have been returned by SetState, or be an empty State.
   524  // There should be no concurrent calls to any of the exported functions of this package with
   525  // a call to SetState because it will modify the global state in a non-thread-safe way.
   526  func (f *Fetcher) SetState(newState *Fetcher) (oldState *Fetcher) {
   527  	if newState.lookupCache == nil {
   528  		newState.lookupCache = new(par.Cache[lookupCacheKey, Repo])
   529  	}
   530  	if newState.downloadCache == nil {
   531  		newState.downloadCache = new(par.ErrCache[module.Version, string])
   532  	}
   533  
   534  	f.mu.Lock()
   535  	defer f.mu.Unlock()
   536  
   537  	oldState = &Fetcher{
   538  		goSumFile:           f.goSumFile,
   539  		workspaceGoSumFiles: f.workspaceGoSumFiles,
   540  		lookupCache:         f.lookupCache,
   541  		downloadCache:       f.downloadCache,
   542  		sumState:            f.sumState,
   543  	}
   544  
   545  	f.SetGoSumFile(newState.goSumFile)
   546  	f.workspaceGoSumFiles = newState.workspaceGoSumFiles
   547  	// Uses of lookupCache and downloadCache both can call checkModSum,
   548  	// which in turn sets the used bit on goSum.status for modules.
   549  	// Set (or reset) them so used can be computed properly.
   550  	f.lookupCache = newState.lookupCache
   551  	f.downloadCache = newState.downloadCache
   552  	// Set, or reset all fields on goSum. If being reset to empty, it will be initialized later.
   553  	f.sumState = newState.sumState
   554  
   555  	return oldState
   556  }
   557  
   558  // initGoSum initializes the go.sum data.
   559  // The boolean it returns reports whether the
   560  // use of go.sum is now enabled.
   561  // The goSum lock must be held.
   562  func (f *Fetcher) initGoSum() (bool, error) {
   563  	if f.goSumFile == "" {
   564  		return false, nil
   565  	}
   566  	if f.sumState.m != nil {
   567  		return true, nil
   568  	}
   569  
   570  	f.sumState.m = make(map[module.Version][]string)
   571  	f.sumState.status = make(map[modSum]modSumStatus)
   572  	f.sumState.w = make(map[string]map[module.Version][]string)
   573  
   574  	for _, fn := range f.workspaceGoSumFiles {
   575  		f.sumState.w[fn] = make(map[module.Version][]string)
   576  		_, err := readGoSumFile(f.sumState.w[fn], fn)
   577  		if err != nil {
   578  			return false, err
   579  		}
   580  	}
   581  
   582  	enabled, err := readGoSumFile(f.sumState.m, f.goSumFile)
   583  	f.sumState.enabled = enabled
   584  	return enabled, err
   585  }
   586  
   587  func readGoSumFile(dst map[module.Version][]string, file string) (bool, error) {
   588  	var (
   589  		data []byte
   590  		err  error
   591  	)
   592  	if fsys.Replaced(file) {
   593  		// Don't lock go.sum if it's part of the overlay.
   594  		// On Plan 9, locking requires chmod, and we don't want to modify any file
   595  		// in the overlay. See #44700.
   596  		data, err = os.ReadFile(fsys.Actual(file))
   597  	} else {
   598  		data, err = lockedfile.Read(file)
   599  	}
   600  	if err != nil && !os.IsNotExist(err) {
   601  		return false, err
   602  	}
   603  	readGoSum(dst, file, data)
   604  
   605  	return true, nil
   606  }
   607  
   608  // emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod.
   609  // A bug caused us to write these into go.sum files for non-modules.
   610  // We detect and remove them.
   611  const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY="
   612  
   613  // readGoSum parses data, which is the content of file,
   614  // and adds it to goSum.m. The goSum lock must be held.
   615  func readGoSum(dst map[module.Version][]string, file string, data []byte) {
   616  	lineno := 0
   617  	for len(data) > 0 {
   618  		var line []byte
   619  		lineno++
   620  		i := bytes.IndexByte(data, '\n')
   621  		if i < 0 {
   622  			line, data = data, nil
   623  		} else {
   624  			line, data = data[:i], data[i+1:]
   625  		}
   626  		f := strings.Fields(string(line))
   627  		if len(f) == 0 {
   628  			// blank line; skip it
   629  			continue
   630  		}
   631  		if len(f) != 3 {
   632  			if cfg.CmdName == "mod tidy" {
   633  				// ignore malformed line so that go mod tidy can fix go.sum
   634  				continue
   635  			} else {
   636  				base.Fatalf("malformed go.sum:\n%s:%d: wrong number of fields %v\n", file, lineno, len(f))
   637  			}
   638  		}
   639  		if f[2] == emptyGoModHash {
   640  			// Old bug; drop it.
   641  			continue
   642  		}
   643  		mod := module.Version{Path: f[0], Version: f[1]}
   644  		dst[mod] = append(dst[mod], f[2])
   645  	}
   646  }
   647  
   648  // HaveSum returns true if the go.sum file contains an entry for mod.
   649  // The entry's hash must be generated with a known hash algorithm.
   650  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   651  // .mod and .zip files.
   652  func HaveSum(f *Fetcher, mod module.Version) bool {
   653  	f.mu.Lock()
   654  	defer f.mu.Unlock()
   655  	inited, err := f.initGoSum()
   656  	if err != nil || !inited {
   657  		return false
   658  	}
   659  	for _, goSums := range f.sumState.w {
   660  		for _, h := range goSums[mod] {
   661  			if !strings.HasPrefix(h, "h1:") {
   662  				continue
   663  			}
   664  			if !f.sumState.status[modSum{mod, h}].dirty {
   665  				return true
   666  			}
   667  		}
   668  	}
   669  	for _, h := range f.sumState.m[mod] {
   670  		if !strings.HasPrefix(h, "h1:") {
   671  			continue
   672  		}
   673  		if !f.sumState.status[modSum{mod, h}].dirty {
   674  			return true
   675  		}
   676  	}
   677  	return false
   678  }
   679  
   680  // RecordedSum returns the sum if the go.sum file contains an entry for mod.
   681  // The boolean reports true if an entry was found or
   682  // false if no entry found or two conflicting sums are found.
   683  // The entry's hash must be generated with a known hash algorithm.
   684  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   685  // .mod and .zip files.
   686  func (f *Fetcher) RecordedSum(mod module.Version) (sum string, ok bool) {
   687  	f.mu.Lock()
   688  	defer f.mu.Unlock()
   689  	inited, err := f.initGoSum()
   690  	foundSum := ""
   691  	if err != nil || !inited {
   692  		return "", false
   693  	}
   694  	for _, goSums := range f.sumState.w {
   695  		for _, h := range goSums[mod] {
   696  			if !strings.HasPrefix(h, "h1:") {
   697  				continue
   698  			}
   699  			if !f.sumState.status[modSum{mod, h}].dirty {
   700  				if foundSum != "" && foundSum != h { // conflicting sums exist
   701  					return "", false
   702  				}
   703  				foundSum = h
   704  			}
   705  		}
   706  	}
   707  	for _, h := range f.sumState.m[mod] {
   708  		if !strings.HasPrefix(h, "h1:") {
   709  			continue
   710  		}
   711  		if !f.sumState.status[modSum{mod, h}].dirty {
   712  			if foundSum != "" && foundSum != h { // conflicting sums exist
   713  				return "", false
   714  			}
   715  			foundSum = h
   716  		}
   717  	}
   718  	return foundSum, true
   719  }
   720  
   721  // checkMod checks the given module's checksum and Go version.
   722  func (f *Fetcher) checkMod(ctx context.Context, mod module.Version) {
   723  	// Do the file I/O before acquiring the go.sum lock.
   724  	ziphash, err := CachePath(ctx, mod, "ziphash")
   725  	if err != nil {
   726  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   727  	}
   728  	data, err := lockedfile.Read(ziphash)
   729  	if err != nil {
   730  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   731  	}
   732  	data = bytes.TrimSpace(data)
   733  	if !isValidSum(data) {
   734  		// Recreate ziphash file from zip file and use that to check the mod sum.
   735  		zip, err := CachePath(ctx, mod, "zip")
   736  		if err != nil {
   737  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   738  		}
   739  		err = hashZip(f, mod, zip, ziphash)
   740  		if err != nil {
   741  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   742  		}
   743  		return
   744  	}
   745  	h := string(data)
   746  	if !strings.HasPrefix(h, "h1:") {
   747  		base.Fatalf("verifying %v", module.VersionError(mod, fmt.Errorf("unexpected ziphash: %q", h)))
   748  	}
   749  
   750  	if err := checkModSum(f, mod, h); err != nil {
   751  		base.Fatalf("%s", err)
   752  	}
   753  }
   754  
   755  // goModSum returns the checksum for the go.mod contents.
   756  func goModSum(data []byte) (string, error) {
   757  	return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
   758  		return io.NopCloser(bytes.NewReader(data)), nil
   759  	})
   760  }
   761  
   762  // checkGoMod checks the given module's go.mod checksum;
   763  // data is the go.mod content.
   764  func checkGoMod(f *Fetcher, path, version string, data []byte) error {
   765  	h, err := goModSum(data)
   766  	if err != nil {
   767  		return &module.ModuleError{Path: path, Version: version, Err: fmt.Errorf("verifying go.mod: %v", err)}
   768  	}
   769  
   770  	return checkModSum(f, module.Version{Path: path, Version: version + "/go.mod"}, h)
   771  }
   772  
   773  // checkModSum checks that the recorded checksum for mod is h.
   774  //
   775  // mod.Version may have the additional suffix "/go.mod" to request the checksum
   776  // for the module's go.mod file only.
   777  func checkModSum(f *Fetcher, mod module.Version, h string) error {
   778  	// We lock goSum when manipulating it,
   779  	// but we arrange to release the lock when calling checkSumDB,
   780  	// so that parallel calls to checkModHash can execute parallel calls
   781  	// to checkSumDB.
   782  
   783  	// Check whether mod+h is listed in go.sum already. If so, we're done.
   784  	f.mu.Lock()
   785  	inited, err := f.initGoSum()
   786  	if err != nil {
   787  		f.mu.Unlock()
   788  		return err
   789  	}
   790  	done := inited && haveModSumLocked(f, mod, h)
   791  	if inited {
   792  		st := f.sumState.status[modSum{mod, h}]
   793  		st.used = true
   794  		f.sumState.status[modSum{mod, h}] = st
   795  	}
   796  	f.mu.Unlock()
   797  
   798  	if done {
   799  		return nil
   800  	}
   801  
   802  	// Not listed, so we want to add them.
   803  	// Consult checksum database if appropriate.
   804  	if useSumDB(mod) {
   805  		// Calls base.Fatalf if mismatch detected.
   806  		if err := checkSumDB(mod, h); err != nil {
   807  			return err
   808  		}
   809  	}
   810  
   811  	// Add mod+h to go.sum, if it hasn't appeared already.
   812  	if inited {
   813  		f.mu.Lock()
   814  		addModSumLocked(f, mod, h)
   815  		st := f.sumState.status[modSum{mod, h}]
   816  		st.dirty = true
   817  		f.sumState.status[modSum{mod, h}] = st
   818  		f.mu.Unlock()
   819  	}
   820  	return nil
   821  }
   822  
   823  // haveModSumLocked reports whether the pair mod,h is already listed in go.sum.
   824  // If it finds a conflicting pair instead, it calls base.Fatalf.
   825  // goSum.mu must be locked.
   826  func haveModSumLocked(f *Fetcher, mod module.Version, h string) bool {
   827  	sumFileName := "go.sum"
   828  	if strings.HasSuffix(f.goSumFile, "go.work.sum") {
   829  		sumFileName = "go.work.sum"
   830  	}
   831  	for _, vh := range f.sumState.m[mod] {
   832  		if h == vh {
   833  			return true
   834  		}
   835  		if strings.HasPrefix(vh, "h1:") {
   836  			base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, sumFileName, vh)
   837  		}
   838  	}
   839  	// Also check workspace sums.
   840  	foundMatch := false
   841  	// Check sums from all files in case there are conflicts between
   842  	// the files.
   843  	for goSumFile, goSums := range f.sumState.w {
   844  		for _, vh := range goSums[mod] {
   845  			if h == vh {
   846  				foundMatch = true
   847  			} else if strings.HasPrefix(vh, "h1:") {
   848  				base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, goSumFile, vh)
   849  			}
   850  		}
   851  	}
   852  	return foundMatch
   853  }
   854  
   855  // addModSumLocked adds the pair mod,h to go.sum.
   856  // goSum.mu must be locked.
   857  func addModSumLocked(f *Fetcher, mod module.Version, h string) {
   858  	if haveModSumLocked(f, mod, h) {
   859  		return
   860  	}
   861  	if len(f.sumState.m[mod]) > 0 {
   862  		fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(f.sumState.m[mod], ", "), h)
   863  	}
   864  	f.sumState.m[mod] = append(f.sumState.m[mod], h)
   865  }
   866  
   867  // checkSumDB checks the mod, h pair against the Go checksum database.
   868  // It calls base.Fatalf if the hash is to be rejected.
   869  func checkSumDB(mod module.Version, h string) error {
   870  	modWithoutSuffix := mod
   871  	noun := "module"
   872  	if before, found := strings.CutSuffix(mod.Version, "/go.mod"); found {
   873  		noun = "go.mod"
   874  		modWithoutSuffix.Version = before
   875  	}
   876  
   877  	db, lines, err := lookupSumDB(mod)
   878  	if err != nil {
   879  		return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: %v", noun, err))
   880  	}
   881  
   882  	have := mod.Path + " " + mod.Version + " " + h
   883  	prefix := mod.Path + " " + mod.Version + " h1:"
   884  	for _, line := range lines {
   885  		if line == have {
   886  			return nil
   887  		}
   888  		if strings.HasPrefix(line, prefix) {
   889  			return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, noun, h, db, line[len(prefix)-len("h1:"):]))
   890  		}
   891  	}
   892  	return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: checksum missing from sumdb response"+sumdbAbsent, noun))
   893  }
   894  
   895  // Sum returns the checksum for the downloaded copy of the given module,
   896  // if present in the download cache.
   897  func Sum(ctx context.Context, mod module.Version) string {
   898  	if cfg.GOMODCACHE == "" {
   899  		// Do not use current directory.
   900  		return ""
   901  	}
   902  
   903  	ziphash, err := CachePath(ctx, mod, "ziphash")
   904  	if err != nil {
   905  		return ""
   906  	}
   907  	data, err := lockedfile.Read(ziphash)
   908  	if err != nil {
   909  		return ""
   910  	}
   911  	data = bytes.TrimSpace(data)
   912  	if !isValidSum(data) {
   913  		return ""
   914  	}
   915  	return string(data)
   916  }
   917  
   918  // isValidSum returns true if data is the valid contents of a zip hash file.
   919  // Certain critical files are written to disk by first truncating
   920  // then writing the actual bytes, so that if the write fails
   921  // the corrupt file should contain at least one of the null
   922  // bytes written by the truncate operation.
   923  func isValidSum(data []byte) bool {
   924  	if bytes.IndexByte(data, '\000') >= 0 {
   925  		return false
   926  	}
   927  
   928  	if len(data) != len("h1:")+base64.StdEncoding.EncodedLen(sha256.Size) {
   929  		return false
   930  	}
   931  
   932  	return true
   933  }
   934  
   935  var ErrGoSumDirty = errors.New("updates to go.sum needed, disabled by -mod=readonly")
   936  
   937  // WriteGoSum writes the go.sum file if it needs to be updated.
   938  //
   939  // keep is used to check whether a newly added sum should be saved in go.sum.
   940  // It should have entries for both module content sums and go.mod sums
   941  // (version ends with "/go.mod"). Existing sums will be preserved unless they
   942  // have been marked for deletion with TrimGoSum.
   943  func (f *Fetcher) WriteGoSum(ctx context.Context, keep map[module.Version]bool, readonly bool) error {
   944  	f.mu.Lock()
   945  	defer f.mu.Unlock()
   946  
   947  	// If we haven't read the go.sum file yet, don't bother writing it.
   948  	if !f.sumState.enabled {
   949  		return nil
   950  	}
   951  
   952  	// Check whether we need to add sums for which keep[m] is true or remove
   953  	// unused sums marked with TrimGoSum. If there are no changes to make,
   954  	// just return without opening go.sum.
   955  	dirty := false
   956  Outer:
   957  	for m, hs := range f.sumState.m {
   958  		for _, h := range hs {
   959  			st := f.sumState.status[modSum{m, h}]
   960  			if st.dirty && (!st.used || keep[m]) {
   961  				dirty = true
   962  				break Outer
   963  			}
   964  		}
   965  	}
   966  	if !dirty {
   967  		return nil
   968  	}
   969  	if readonly {
   970  		return ErrGoSumDirty
   971  	}
   972  	if fsys.Replaced(f.goSumFile) {
   973  		base.Fatalf("go: updates to go.sum needed, but go.sum is part of the overlay specified with -overlay")
   974  	}
   975  
   976  	// Make a best-effort attempt to acquire the side lock, only to exclude
   977  	// previous versions of the 'go' command from making simultaneous edits.
   978  	if unlock, err := SideLock(ctx); err == nil {
   979  		defer unlock()
   980  	}
   981  
   982  	err := lockedfile.Transform(f.goSumFile, func(data []byte) ([]byte, error) {
   983  		tidyGoSum := tidyGoSum(f, data, keep)
   984  		return tidyGoSum, nil
   985  	})
   986  	if err != nil {
   987  		return fmt.Errorf("updating go.sum: %w", err)
   988  	}
   989  
   990  	f.sumState.status = make(map[modSum]modSumStatus)
   991  	f.sumState.overwrite = false
   992  	return nil
   993  }
   994  
   995  // TidyGoSum returns a tidy version of the go.sum file.
   996  // A missing go.sum file is treated as if empty.
   997  func (f *Fetcher) TidyGoSum(keep map[module.Version]bool) (before, after []byte) {
   998  	f.mu.Lock()
   999  	defer f.mu.Unlock()
  1000  	before, err := lockedfile.Read(f.goSumFile)
  1001  	if err != nil && !errors.Is(err, fs.ErrNotExist) {
  1002  		base.Fatalf("reading go.sum: %v", err)
  1003  	}
  1004  	after = tidyGoSum(f, before, keep)
  1005  	return before, after
  1006  }
  1007  
  1008  // tidyGoSum returns a tidy version of the go.sum file.
  1009  // The goSum lock must be held.
  1010  func tidyGoSum(f *Fetcher, data []byte, keep map[module.Version]bool) []byte {
  1011  	if !f.sumState.overwrite {
  1012  		// Incorporate any sums added by other processes in the meantime.
  1013  		// Add only the sums that we actually checked: the user may have edited or
  1014  		// truncated the file to remove erroneous hashes, and we shouldn't restore
  1015  		// them without good reason.
  1016  		f.sumState.m = make(map[module.Version][]string, len(f.sumState.m))
  1017  		readGoSum(f.sumState.m, f.goSumFile, data)
  1018  		for ms, st := range f.sumState.status {
  1019  			if st.used && !sumInWorkspaceModulesLocked(f, ms.mod) {
  1020  				addModSumLocked(f, ms.mod, ms.sum)
  1021  			}
  1022  		}
  1023  	}
  1024  
  1025  	mods := make([]module.Version, 0, len(f.sumState.m))
  1026  	for m := range f.sumState.m {
  1027  		mods = append(mods, m)
  1028  	}
  1029  	module.Sort(mods)
  1030  
  1031  	var buf bytes.Buffer
  1032  	for _, m := range mods {
  1033  		list := f.sumState.m[m]
  1034  		sort.Strings(list)
  1035  		str.Uniq(&list)
  1036  		for _, h := range list {
  1037  			st := f.sumState.status[modSum{m, h}]
  1038  			if (!st.dirty || (st.used && keep[m])) && !sumInWorkspaceModulesLocked(f, m) {
  1039  				fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
  1040  			}
  1041  		}
  1042  	}
  1043  	return buf.Bytes()
  1044  }
  1045  
  1046  func sumInWorkspaceModulesLocked(f *Fetcher, m module.Version) bool {
  1047  	for _, goSums := range f.sumState.w {
  1048  		if _, ok := goSums[m]; ok {
  1049  			return true
  1050  		}
  1051  	}
  1052  	return false
  1053  }
  1054  
  1055  // TrimGoSum trims go.sum to contain only the modules needed for reproducible
  1056  // builds.
  1057  //
  1058  // keep is used to check whether a sum should be retained in go.mod. It should
  1059  // have entries for both module content sums and go.mod sums (version ends
  1060  // with "/go.mod").
  1061  func (f *Fetcher) TrimGoSum(keep map[module.Version]bool) {
  1062  	f.mu.Lock()
  1063  	defer f.mu.Unlock()
  1064  	inited, err := f.initGoSum()
  1065  	if err != nil {
  1066  		base.Fatalf("%s", err)
  1067  	}
  1068  	if !inited {
  1069  		return
  1070  	}
  1071  
  1072  	for m, hs := range f.sumState.m {
  1073  		if !keep[m] {
  1074  			for _, h := range hs {
  1075  				f.sumState.status[modSum{m, h}] = modSumStatus{used: false, dirty: true}
  1076  			}
  1077  			f.sumState.overwrite = true
  1078  		}
  1079  	}
  1080  }
  1081  
  1082  const goSumMismatch = `
  1083  
  1084  SECURITY ERROR
  1085  This download does NOT match an earlier download recorded in go.sum.
  1086  The bits may have been replaced on the origin server, or an attacker may
  1087  have intercepted the download attempt.
  1088  
  1089  For more information, see 'go help module-auth'.
  1090  `
  1091  
  1092  const sumdbMismatch = `
  1093  
  1094  SECURITY ERROR
  1095  This download does NOT match the one reported by the checksum server.
  1096  The bits may have been replaced on the origin server, or an attacker may
  1097  have intercepted the download attempt.
  1098  
  1099  For more information, see 'go help module-auth'.
  1100  `
  1101  
  1102  const sumdbAbsent = `
  1103  
  1104  SECURITY ERROR
  1105  This download does NOT match one reported by the checksum server.
  1106  The checksum server has provided checksums, but the checksums do
  1107  not contain an entry for the download.
  1108  The checksum server may be malfunctioning, or an attacker may have
  1109  intercepted the checksum request.
  1110  The download cannot be verified.
  1111  
  1112  For more information, see 'go help module-auth'.
  1113  `
  1114  
  1115  const hashVersionMismatch = `
  1116  
  1117  SECURITY WARNING
  1118  This download is listed in go.sum, but using an unknown hash algorithm.
  1119  The download cannot be verified.
  1120  
  1121  For more information, see 'go help module-auth'.
  1122  
  1123  `
  1124  
  1125  var HelpModuleAuth = &base.Command{
  1126  	UsageLine: "module-auth",
  1127  	Short:     "module authentication using go.sum",
  1128  	Long: `
  1129  When the go command downloads a module zip file or go.mod file into the
  1130  module cache, it computes a cryptographic hash and compares it with a known
  1131  value to verify the file hasn't changed since it was first downloaded. Known
  1132  hashes are stored in a file in the module root directory named go.sum. Hashes
  1133  may also be downloaded from the checksum database depending on the values of
  1134  GOSUMDB, GOPRIVATE, and GONOSUMDB.
  1135  
  1136  For details, see https://go.dev/ref/mod#authenticating.
  1137  `,
  1138  }
  1139  
  1140  var HelpPrivate = &base.Command{
  1141  	UsageLine: "private",
  1142  	Short:     "configuration for downloading non-public code",
  1143  	Long: `
  1144  The go command defaults to downloading modules from the public Go module
  1145  mirror at proxy.golang.org. It also defaults to validating downloaded modules,
  1146  regardless of source, against the public Go checksum database at sum.golang.org.
  1147  These defaults work well for publicly available source code.
  1148  
  1149  The GOPRIVATE environment variable controls which modules the go command
  1150  considers to be private (not available publicly) and should therefore not use
  1151  the proxy or checksum database. The variable is a comma-separated list of
  1152  glob patterns (in the syntax of Go's path.Match) of module path prefixes.
  1153  For example,
  1154  
  1155  	GOPRIVATE=*.corp.example.com,rsc.io/private
  1156  
  1157  causes the go command to treat as private any module with a path prefix
  1158  matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private,
  1159  and rsc.io/private/quux.
  1160  
  1161  For fine-grained control over module download and validation, the GONOPROXY
  1162  and GONOSUMDB environment variables accept the same kind of glob list
  1163  and override GOPRIVATE for the specific decision of whether to use the proxy
  1164  and checksum database, respectively.
  1165  
  1166  For example, if a company ran a module proxy serving private modules,
  1167  users would configure go using:
  1168  
  1169  	GOPRIVATE=*.corp.example.com
  1170  	GOPROXY=proxy.example.com
  1171  	GONOPROXY=none
  1172  
  1173  The GOPRIVATE variable is also used to define the "public" and "private"
  1174  patterns for the GOVCS variable; see 'go help vcs'. For that usage,
  1175  GOPRIVATE applies even in GOPATH mode. In that case, it matches import paths
  1176  instead of module paths.
  1177  
  1178  The 'go env -w' command (see 'go help env') can be used to set these variables
  1179  for future go command invocations.
  1180  
  1181  For more details, see https://go.dev/ref/mod#private-modules.
  1182  `,
  1183  }
  1184
View as plain text