cache.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"encoding/json"
    11  	"errors"
    12  	"fmt"
    13  	"io"
    14  	"io/fs"
    15  	"math/rand"
    16  	"os"
    17  	"path/filepath"
    18  	"strconv"
    19  	"strings"
    20  	"sync"
    21  
    22  	"cmd/go/internal/base"
    23  	"cmd/go/internal/cfg"
    24  	"cmd/go/internal/gover"
    25  	"cmd/go/internal/lockedfile"
    26  	"cmd/go/internal/modfetch/codehost"
    27  	"cmd/internal/par"
    28  	"cmd/internal/robustio"
    29  	"cmd/internal/telemetry/counter"
    30  
    31  	"golang.org/x/mod/module"
    32  	"golang.org/x/mod/semver"
    33  )
    34  
    35  func cacheDir(ctx context.Context, path string) (string, error) {
    36  	if err := checkCacheDir(ctx); err != nil {
    37  		return "", err
    38  	}
    39  	enc, err := module.EscapePath(path)
    40  	if err != nil {
    41  		return "", err
    42  	}
    43  	return filepath.Join(cfg.GOMODCACHE, "cache/download", enc, "/@v"), nil
    44  }
    45  
    46  func CachePath(ctx context.Context, m module.Version, suffix string) (string, error) {
    47  	if gover.IsToolchain(m.Path) {
    48  		return "", ErrToolchain
    49  	}
    50  	dir, err := cacheDir(ctx, m.Path)
    51  	if err != nil {
    52  		return "", err
    53  	}
    54  	if !gover.ModIsValid(m.Path, m.Version) {
    55  		return "", fmt.Errorf("non-semver module version %q", m.Version)
    56  	}
    57  	if module.CanonicalVersion(m.Version) != m.Version {
    58  		return "", fmt.Errorf("non-canonical module version %q", m.Version)
    59  	}
    60  	encVer, err := module.EscapeVersion(m.Version)
    61  	if err != nil {
    62  		return "", err
    63  	}
    64  	return filepath.Join(dir, encVer+"."+suffix), nil
    65  }
    66  
    67  // DownloadDir returns the directory to which m should have been downloaded.
    68  // An error will be returned if the module path or version cannot be escaped.
    69  // An error satisfying errors.Is(err, fs.ErrNotExist) will be returned
    70  // along with the directory if the directory does not exist or if the directory
    71  // is not completely populated.
    72  func DownloadDir(ctx context.Context, m module.Version) (string, error) {
    73  	if gover.IsToolchain(m.Path) {
    74  		return "", ErrToolchain
    75  	}
    76  	if err := checkCacheDir(ctx); err != nil {
    77  		return "", err
    78  	}
    79  	enc, err := module.EscapePath(m.Path)
    80  	if err != nil {
    81  		return "", err
    82  	}
    83  	if !gover.ModIsValid(m.Path, m.Version) {
    84  		return "", fmt.Errorf("non-semver module version %q", m.Version)
    85  	}
    86  	if module.CanonicalVersion(m.Version) != m.Version {
    87  		return "", fmt.Errorf("non-canonical module version %q", m.Version)
    88  	}
    89  	encVer, err := module.EscapeVersion(m.Version)
    90  	if err != nil {
    91  		return "", err
    92  	}
    93  
    94  	// Check whether the directory itself exists.
    95  	dir := filepath.Join(cfg.GOMODCACHE, enc+"@"+encVer)
    96  	if fi, err := os.Stat(dir); os.IsNotExist(err) {
    97  		return dir, err
    98  	} else if err != nil {
    99  		return dir, &DownloadDirPartialError{dir, err}
   100  	} else if !fi.IsDir() {
   101  		return dir, &DownloadDirPartialError{dir, errors.New("not a directory")}
   102  	}
   103  
   104  	// Check if a .partial file exists. This is created at the beginning of
   105  	// a download and removed after the zip is extracted.
   106  	partialPath, err := CachePath(ctx, m, "partial")
   107  	if err != nil {
   108  		return dir, err
   109  	}
   110  	if _, err := os.Stat(partialPath); err == nil {
   111  		return dir, &DownloadDirPartialError{dir, errors.New("not completely extracted")}
   112  	} else if !os.IsNotExist(err) {
   113  		return dir, err
   114  	}
   115  
   116  	// Special case: ziphash is not required for the golang.org/fips140 module,
   117  	// because it is unpacked from a file in GOROOT, not downloaded.
   118  	// We've already checked that it's not a partial unpacking, so we're happy.
   119  	if m.Path == "golang.org/fips140" {
   120  		return dir, nil
   121  	}
   122  
   123  	// Check if a .ziphash file exists. It should be created before the
   124  	// zip is extracted, but if it was deleted (by another program?), we need
   125  	// to re-calculate it. Note that checkMod will repopulate the ziphash
   126  	// file if it doesn't exist, but if the module is excluded by checks
   127  	// through GONOSUMDB or GOPRIVATE, that check and repopulation won't happen.
   128  	ziphashPath, err := CachePath(ctx, m, "ziphash")
   129  	if err != nil {
   130  		return dir, err
   131  	}
   132  	if _, err := os.Stat(ziphashPath); os.IsNotExist(err) {
   133  		return dir, &DownloadDirPartialError{dir, errors.New("ziphash file is missing")}
   134  	} else if err != nil {
   135  		return dir, err
   136  	}
   137  	return dir, nil
   138  }
   139  
   140  // DownloadDirPartialError is returned by DownloadDir if a module directory
   141  // exists but was not completely populated.
   142  //
   143  // DownloadDirPartialError is equivalent to fs.ErrNotExist.
   144  type DownloadDirPartialError struct {
   145  	Dir string
   146  	Err error
   147  }
   148  
   149  func (e *DownloadDirPartialError) Error() string     { return fmt.Sprintf("%s: %v", e.Dir, e.Err) }
   150  func (e *DownloadDirPartialError) Is(err error) bool { return err == fs.ErrNotExist }
   151  
   152  // lockVersion locks a file within the module cache that guards the downloading
   153  // and extraction of the zipfile for the given module version.
   154  func lockVersion(ctx context.Context, mod module.Version) (unlock func(), err error) {
   155  	path, err := CachePath(ctx, mod, "lock")
   156  	if err != nil {
   157  		return nil, err
   158  	}
   159  	if err := os.MkdirAll(filepath.Dir(path), 0o777); err != nil {
   160  		return nil, err
   161  	}
   162  	return lockedfile.MutexAt(path).Lock()
   163  }
   164  
   165  // SideLock locks a file within the module cache that previously guarded
   166  // edits to files outside the cache, such as go.sum and go.mod files in the
   167  // user's working directory.
   168  // If err is nil, the caller MUST eventually call the unlock function.
   169  func SideLock(ctx context.Context) (unlock func(), err error) {
   170  	if err := checkCacheDir(ctx); err != nil {
   171  		return nil, err
   172  	}
   173  
   174  	path := filepath.Join(cfg.GOMODCACHE, "cache", "lock")
   175  	if err := os.MkdirAll(filepath.Dir(path), 0o777); err != nil {
   176  		return nil, fmt.Errorf("failed to create cache directory: %w", err)
   177  	}
   178  
   179  	return lockedfile.MutexAt(path).Lock()
   180  }
   181  
   182  // A cachingRepo is a cache around an underlying Repo,
   183  // avoiding redundant calls to ModulePath, Versions, Stat, Latest, and GoMod (but not CheckReuse or Zip).
   184  // It is also safe for simultaneous use by multiple goroutines
   185  // (so that it can be returned from Lookup multiple times).
   186  // It serializes calls to the underlying Repo.
   187  type cachingRepo struct {
   188  	path          string
   189  	versionsCache par.ErrCache[string, *Versions]
   190  	statCache     par.ErrCache[string, *RevInfo]
   191  	latestCache   par.ErrCache[struct{}, *RevInfo]
   192  	gomodCache    par.ErrCache[string, []byte]
   193  
   194  	once     sync.Once
   195  	initRepo func(context.Context) (Repo, error)
   196  	r        Repo
   197  	fetcher  *Fetcher
   198  }
   199  
   200  func newCachingRepo(ctx context.Context, fetcher *Fetcher, path string, initRepo func(context.Context) (Repo, error)) *cachingRepo {
   201  	return &cachingRepo{
   202  		path:     path,
   203  		initRepo: initRepo,
   204  		fetcher:  fetcher,
   205  	}
   206  }
   207  
   208  func (r *cachingRepo) repo(ctx context.Context) Repo {
   209  	r.once.Do(func() {
   210  		var err error
   211  		r.r, err = r.initRepo(ctx)
   212  		if err != nil {
   213  			r.r = errRepo{r.path, err}
   214  		}
   215  	})
   216  	return r.r
   217  }
   218  
   219  func (r *cachingRepo) CheckReuse(ctx context.Context, old *codehost.Origin) error {
   220  	return r.repo(ctx).CheckReuse(ctx, old)
   221  }
   222  
   223  func (r *cachingRepo) ModulePath() string {
   224  	return r.path
   225  }
   226  
   227  func (r *cachingRepo) Versions(ctx context.Context, prefix string) (*Versions, error) {
   228  	v, err := r.versionsCache.Do(prefix, func() (*Versions, error) {
   229  		return r.repo(ctx).Versions(ctx, prefix)
   230  	})
   231  	if err != nil {
   232  		return nil, err
   233  	}
   234  	return &Versions{
   235  		Origin: v.Origin,
   236  		List:   append([]string(nil), v.List...),
   237  	}, nil
   238  }
   239  
   240  type cachedInfo struct {
   241  	info *RevInfo
   242  	err  error
   243  }
   244  
   245  func (r *cachingRepo) Stat(ctx context.Context, rev string) (*RevInfo, error) {
   246  	if gover.IsToolchain(r.path) {
   247  		// Skip disk cache; the underlying golang.org/toolchain repo is cached instead.
   248  		return r.repo(ctx).Stat(ctx, rev)
   249  	}
   250  	info, err := r.statCache.Do(rev, func() (*RevInfo, error) {
   251  		file, info, err := readDiskStat(ctx, r.path, rev)
   252  		if err == nil {
   253  			return info, err
   254  		}
   255  
   256  		info, err = r.repo(ctx).Stat(ctx, rev)
   257  		if err == nil {
   258  			// If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78,
   259  			// then save the information under the proper version, for future use.
   260  			if info.Version != rev {
   261  				file, _ = CachePath(ctx, module.Version{Path: r.path, Version: info.Version}, "info")
   262  				r.statCache.Do(info.Version, func() (*RevInfo, error) {
   263  					return info, nil
   264  				})
   265  			}
   266  
   267  			if err := writeDiskStat(ctx, file, info); err != nil {
   268  				fmt.Fprintf(os.Stderr, "go: writing stat cache: %v\n", err)
   269  			}
   270  		}
   271  		return info, err
   272  	})
   273  	if info != nil {
   274  		copy := *info
   275  		info = &copy
   276  	}
   277  	return info, err
   278  }
   279  
   280  func (r *cachingRepo) Latest(ctx context.Context) (*RevInfo, error) {
   281  	if gover.IsToolchain(r.path) {
   282  		// Skip disk cache; the underlying golang.org/toolchain repo is cached instead.
   283  		return r.repo(ctx).Latest(ctx)
   284  	}
   285  	info, err := r.latestCache.Do(struct{}{}, func() (*RevInfo, error) {
   286  		info, err := r.repo(ctx).Latest(ctx)
   287  
   288  		// Save info for likely future Stat call.
   289  		if err == nil {
   290  			r.statCache.Do(info.Version, func() (*RevInfo, error) {
   291  				return info, nil
   292  			})
   293  			if file, _, err := readDiskStat(ctx, r.path, info.Version); err != nil {
   294  				writeDiskStat(ctx, file, info)
   295  			}
   296  		}
   297  
   298  		return info, err
   299  	})
   300  	if info != nil {
   301  		copy := *info
   302  		info = &copy
   303  	}
   304  	return info, err
   305  }
   306  
   307  func (r *cachingRepo) GoMod(ctx context.Context, version string) ([]byte, error) {
   308  	if gover.IsToolchain(r.path) {
   309  		// Skip disk cache; the underlying golang.org/toolchain repo is cached instead.
   310  		return r.repo(ctx).GoMod(ctx, version)
   311  	}
   312  	text, err := r.gomodCache.Do(version, func() ([]byte, error) {
   313  		file, text, err := r.fetcher.readDiskGoMod(ctx, r.path, version)
   314  		if err == nil {
   315  			// Note: readDiskGoMod already called checkGoMod.
   316  			return text, nil
   317  		}
   318  
   319  		text, err = r.repo(ctx).GoMod(ctx, version)
   320  		if err == nil {
   321  			if err := checkGoMod(r.fetcher, r.path, version, text); err != nil {
   322  				return text, err
   323  			}
   324  			if err := writeDiskGoMod(ctx, file, text); err != nil {
   325  				fmt.Fprintf(os.Stderr, "go: writing go.mod cache: %v\n", err)
   326  			}
   327  		}
   328  		return text, err
   329  	})
   330  	if err != nil {
   331  		return nil, err
   332  	}
   333  	return append([]byte(nil), text...), nil
   334  }
   335  
   336  func (r *cachingRepo) Zip(ctx context.Context, dst io.Writer, version string) error {
   337  	if gover.IsToolchain(r.path) {
   338  		return ErrToolchain
   339  	}
   340  	return r.repo(ctx).Zip(ctx, dst, version)
   341  }
   342  
   343  // InfoFile is like Lookup(ctx, path).Stat(version) but also returns the name of the file
   344  // containing the cached information.
   345  func (f *Fetcher) InfoFile(ctx context.Context, path, version string) (*RevInfo, string, error) {
   346  	if !gover.ModIsValid(path, version) {
   347  		return nil, "", fmt.Errorf("invalid version %q", version)
   348  	}
   349  
   350  	if file, info, err := readDiskStat(ctx, path, version); err == nil {
   351  		return info, file, nil
   352  	}
   353  
   354  	var info *RevInfo
   355  	var err2info map[error]*RevInfo
   356  	err := TryProxies(func(proxy string) error {
   357  		i, err := f.Lookup(ctx, proxy, path).Stat(ctx, version)
   358  		if err == nil {
   359  			info = i
   360  		} else {
   361  			if err2info == nil {
   362  				err2info = make(map[error]*RevInfo)
   363  			}
   364  			err2info[err] = info
   365  		}
   366  		return err
   367  	})
   368  	if err != nil {
   369  		return err2info[err], "", err
   370  	}
   371  
   372  	// Stat should have populated the disk cache for us.
   373  	file, err := CachePath(ctx, module.Version{Path: path, Version: version}, "info")
   374  	if err != nil {
   375  		return nil, "", err
   376  	}
   377  	return info, file, nil
   378  }
   379  
   380  // GoMod is like Lookup(ctx, path).GoMod(rev) but avoids the
   381  // repository path resolution in Lookup if the result is
   382  // already cached on local disk.
   383  func (f *Fetcher) GoMod(ctx context.Context, path, rev string) ([]byte, error) {
   384  	// Convert commit hash to pseudo-version
   385  	// to increase cache hit rate.
   386  	if !gover.ModIsValid(path, rev) {
   387  		if _, info, err := readDiskStat(ctx, path, rev); err == nil {
   388  			rev = info.Version
   389  		} else {
   390  			if errors.Is(err, statCacheErr) {
   391  				return nil, err
   392  			}
   393  			err := TryProxies(func(proxy string) error {
   394  				info, err := f.Lookup(ctx, proxy, path).Stat(ctx, rev)
   395  				if err == nil {
   396  					rev = info.Version
   397  				}
   398  				return err
   399  			})
   400  			if err != nil {
   401  				return nil, err
   402  			}
   403  		}
   404  	}
   405  
   406  	_, data, err := f.readDiskGoMod(ctx, path, rev)
   407  	if err == nil {
   408  		return data, nil
   409  	}
   410  
   411  	err = TryProxies(func(proxy string) (err error) {
   412  		data, err = f.Lookup(ctx, proxy, path).GoMod(ctx, rev)
   413  		return err
   414  	})
   415  	return data, err
   416  }
   417  
   418  // GoModFile is like GoMod but returns the name of the file containing
   419  // the cached information.
   420  func (f *Fetcher) GoModFile(ctx context.Context, path, version string) (string, error) {
   421  	if !gover.ModIsValid(path, version) {
   422  		return "", fmt.Errorf("invalid version %q", version)
   423  	}
   424  	if _, err := f.GoMod(ctx, path, version); err != nil {
   425  		return "", err
   426  	}
   427  	// GoMod should have populated the disk cache for us.
   428  	file, err := CachePath(ctx, module.Version{Path: path, Version: version}, "mod")
   429  	if err != nil {
   430  		return "", err
   431  	}
   432  	return file, nil
   433  }
   434  
   435  // GoModSum returns the go.sum entry for the module version's go.mod file.
   436  // (That is, it returns the entry listed in go.sum as "path version/go.mod".)
   437  func (f *Fetcher) GoModSum(ctx context.Context, path, version string) (string, error) {
   438  	if !gover.ModIsValid(path, version) {
   439  		return "", fmt.Errorf("invalid version %q", version)
   440  	}
   441  	data, err := f.GoMod(ctx, path, version)
   442  	if err != nil {
   443  		return "", err
   444  	}
   445  	sum, err := goModSum(data)
   446  	if err != nil {
   447  		return "", err
   448  	}
   449  	return sum, nil
   450  }
   451  
   452  var errNotCached = fmt.Errorf("not in cache")
   453  
   454  // readDiskStat reads a cached stat result from disk,
   455  // returning the name of the cache file and the result.
   456  // If the read fails, the caller can use
   457  // writeDiskStat(file, info) to write a new cache entry.
   458  func readDiskStat(ctx context.Context, path, rev string) (file string, info *RevInfo, err error) {
   459  	if gover.IsToolchain(path) {
   460  		return "", nil, errNotCached
   461  	}
   462  	file, data, err := readDiskCache(ctx, path, rev, "info")
   463  	if err != nil {
   464  		// If the cache already contains a pseudo-version with the given hash, we
   465  		// would previously return that pseudo-version without checking upstream.
   466  		// However, that produced an unfortunate side-effect: if the author added a
   467  		// tag to the repository, 'go get' would not pick up the effect of that new
   468  		// tag on the existing commits, and 'go' commands that referred to those
   469  		// commits would use the previous name instead of the new one.
   470  		//
   471  		// That's especially problematic if the original pseudo-version starts with
   472  		// v0.0.0-, as was the case for all pseudo-versions during vgo development,
   473  		// since a v0.0.0- pseudo-version has lower precedence than pretty much any
   474  		// tagged version.
   475  		//
   476  		// In practice, we're only looking up by hash during initial conversion of a
   477  		// legacy config and during an explicit 'go get', and a little extra latency
   478  		// for those operations seems worth the benefit of picking up more accurate
   479  		// versions.
   480  		//
   481  		// Fall back to this resolution scheme only if the GOPROXY setting prohibits
   482  		// us from resolving upstream tags.
   483  		if cfg.GOPROXY == "off" {
   484  			if file, info, err := readDiskStatByHash(ctx, path, rev); err == nil {
   485  				return file, info, nil
   486  			}
   487  		}
   488  		return file, nil, err
   489  	}
   490  	info = new(RevInfo)
   491  	if err := json.Unmarshal(data, info); err != nil {
   492  		return file, nil, errNotCached
   493  	}
   494  	// The disk might have stale .info files that have Name and Short fields set.
   495  	// We want to canonicalize to .info files with those fields omitted.
   496  	// Remarshal and update the cache file if needed.
   497  	data2, err := json.Marshal(info)
   498  	if err == nil && !bytes.Equal(data2, data) {
   499  		writeDiskCache(ctx, file, data)
   500  	}
   501  	return file, info, nil
   502  }
   503  
   504  // readDiskStatByHash is a fallback for readDiskStat for the case
   505  // where rev is a commit hash instead of a proper semantic version.
   506  // In that case, we look for a cached pseudo-version that matches
   507  // the commit hash. If we find one, we use it.
   508  // This matters most for converting legacy package management
   509  // configs, when we are often looking up commits by full hash.
   510  // Without this check we'd be doing network I/O to the remote repo
   511  // just to find out about a commit we already know about
   512  // (and have cached under its pseudo-version).
   513  func readDiskStatByHash(ctx context.Context, path, rev string) (file string, info *RevInfo, err error) {
   514  	if gover.IsToolchain(path) {
   515  		return "", nil, errNotCached
   516  	}
   517  	if cfg.GOMODCACHE == "" {
   518  		// Do not download to current directory.
   519  		return "", nil, errNotCached
   520  	}
   521  
   522  	if !codehost.AllHex(rev) || len(rev) < 12 {
   523  		return "", nil, errNotCached
   524  	}
   525  	rev = rev[:12]
   526  	cdir, err := cacheDir(ctx, path)
   527  	if err != nil {
   528  		return "", nil, errNotCached
   529  	}
   530  	dir, err := os.Open(cdir)
   531  	if err != nil {
   532  		return "", nil, errNotCached
   533  	}
   534  	names, err := dir.Readdirnames(-1)
   535  	dir.Close()
   536  	if err != nil {
   537  		return "", nil, errNotCached
   538  	}
   539  
   540  	// A given commit hash may map to more than one pseudo-version,
   541  	// depending on which tags are present on the repository.
   542  	// Take the highest such version.
   543  	var maxVersion string
   544  	suffix := "-" + rev + ".info"
   545  	err = errNotCached
   546  	for _, name := range names {
   547  		if strings.HasSuffix(name, suffix) {
   548  			v := strings.TrimSuffix(name, ".info")
   549  			if module.IsPseudoVersion(v) && semver.Compare(v, maxVersion) > 0 {
   550  				maxVersion = v
   551  				file, info, err = readDiskStat(ctx, path, strings.TrimSuffix(name, ".info"))
   552  			}
   553  		}
   554  	}
   555  	return file, info, err
   556  }
   557  
   558  // oldVgoPrefix is the prefix in the old auto-generated cached go.mod files.
   559  // We stopped trying to auto-generate the go.mod files. Now we use a trivial
   560  // go.mod with only a module line, and we've dropped the version prefix
   561  // entirely. If we see a version prefix, that means we're looking at an old copy
   562  // and should ignore it.
   563  var oldVgoPrefix = []byte("//vgo 0.0.")
   564  
   565  // readDiskGoMod reads a cached go.mod file from disk,
   566  // returning the name of the cache file and the result.
   567  // If the read fails, the caller can use
   568  // writeDiskGoMod(file, data) to write a new cache entry.
   569  func (f *Fetcher) readDiskGoMod(ctx context.Context, path, rev string) (file string, data []byte, err error) {
   570  	if gover.IsToolchain(path) {
   571  		return "", nil, errNotCached
   572  	}
   573  	file, data, err = readDiskCache(ctx, path, rev, "mod")
   574  
   575  	// If the file has an old auto-conversion prefix, pretend it's not there.
   576  	if bytes.HasPrefix(data, oldVgoPrefix) {
   577  		err = errNotCached
   578  		data = nil
   579  	}
   580  
   581  	if err == nil {
   582  		if err := checkGoMod(f, path, rev, data); err != nil {
   583  			return "", nil, err
   584  		}
   585  	}
   586  
   587  	return file, data, err
   588  }
   589  
   590  // readDiskCache is the generic "read from a cache file" implementation.
   591  // It takes the revision and an identifying suffix for the kind of data being cached.
   592  // It returns the name of the cache file and the content of the file.
   593  // If the read fails, the caller can use
   594  // writeDiskCache(file, data) to write a new cache entry.
   595  func readDiskCache(ctx context.Context, path, rev, suffix string) (file string, data []byte, err error) {
   596  	if gover.IsToolchain(path) {
   597  		return "", nil, errNotCached
   598  	}
   599  	file, err = CachePath(ctx, module.Version{Path: path, Version: rev}, suffix)
   600  	if err != nil {
   601  		return "", nil, errNotCached
   602  	}
   603  	data, err = robustio.ReadFile(file)
   604  	if err != nil {
   605  		return file, nil, errNotCached
   606  	}
   607  	return file, data, nil
   608  }
   609  
   610  // writeDiskStat writes a stat result cache entry.
   611  // The file name must have been returned by a previous call to readDiskStat.
   612  func writeDiskStat(ctx context.Context, file string, info *RevInfo) error {
   613  	if file == "" {
   614  		return nil
   615  	}
   616  
   617  	if info.Origin != nil {
   618  		// Clean the origin information, which might have too many
   619  		// validation criteria, for example if we are saving the result of
   620  		// m@master as m@pseudo-version.
   621  		clean := *info
   622  		info = &clean
   623  		o := *info.Origin
   624  		info.Origin = &o
   625  
   626  		// Tags and RepoSum never matter if you are starting with a semver version,
   627  		// as we would be when finding this cache entry.
   628  		o.TagSum = ""
   629  		o.TagPrefix = ""
   630  		o.RepoSum = ""
   631  		// Ref doesn't matter if you have a pseudoversion.
   632  		if module.IsPseudoVersion(info.Version) {
   633  			o.Ref = ""
   634  		}
   635  	}
   636  
   637  	js, err := json.Marshal(info)
   638  	if err != nil {
   639  		return err
   640  	}
   641  	return writeDiskCache(ctx, file, js)
   642  }
   643  
   644  // writeDiskGoMod writes a go.mod cache entry.
   645  // The file name must have been returned by a previous call to readDiskGoMod.
   646  func writeDiskGoMod(ctx context.Context, file string, text []byte) error {
   647  	return writeDiskCache(ctx, file, text)
   648  }
   649  
   650  // writeDiskCache is the generic "write to a cache file" implementation.
   651  // The file must have been returned by a previous call to readDiskCache.
   652  func writeDiskCache(ctx context.Context, file string, data []byte) error {
   653  	if file == "" {
   654  		return nil
   655  	}
   656  	// Make sure directory for file exists.
   657  	if err := os.MkdirAll(filepath.Dir(file), 0o777); err != nil {
   658  		return err
   659  	}
   660  
   661  	// Write the file to a temporary location, and then rename it to its final
   662  	// path to reduce the likelihood of a corrupt file existing at that final path.
   663  	f, err := tempFile(ctx, filepath.Dir(file), filepath.Base(file), 0o666)
   664  	if err != nil {
   665  		return err
   666  	}
   667  	defer func() {
   668  		// Only call os.Remove on f.Name() if we failed to rename it: otherwise,
   669  		// some other process may have created a new file with the same name after
   670  		// the rename completed.
   671  		if err != nil {
   672  			f.Close()
   673  			os.Remove(f.Name())
   674  		}
   675  	}()
   676  
   677  	if _, err := f.Write(data); err != nil {
   678  		return err
   679  	}
   680  	if err := f.Close(); err != nil {
   681  		return err
   682  	}
   683  	if err := robustio.Rename(f.Name(), file); err != nil {
   684  		return err
   685  	}
   686  
   687  	if strings.HasSuffix(file, ".mod") {
   688  		rewriteVersionList(ctx, filepath.Dir(file))
   689  	}
   690  	return nil
   691  }
   692  
   693  // tempFile creates a new temporary file with given permission bits.
   694  func tempFile(ctx context.Context, dir, prefix string, perm fs.FileMode) (f *os.File, err error) {
   695  	for i := 0; i < 10000; i++ {
   696  		name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+".tmp")
   697  		f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm)
   698  		if os.IsExist(err) {
   699  			if ctx.Err() != nil {
   700  				return nil, ctx.Err()
   701  			}
   702  			continue
   703  		}
   704  		break
   705  	}
   706  	return
   707  }
   708  
   709  // rewriteVersionList rewrites the version list in dir
   710  // after a new *.mod file has been written.
   711  func rewriteVersionList(ctx context.Context, dir string) (err error) {
   712  	if filepath.Base(dir) != "@v" {
   713  		base.Fatalf("go: internal error: misuse of rewriteVersionList")
   714  	}
   715  
   716  	listFile := filepath.Join(dir, "list")
   717  
   718  	// Lock listfile when writing to it to try to avoid corruption to the file.
   719  	// Under rare circumstances, for instance, if the system loses power in the
   720  	// middle of a write it is possible for corrupt data to be written. This is
   721  	// not a problem for the go command itself, but may be an issue if the
   722  	// cache is being served by a GOPROXY HTTP server. This will be corrected
   723  	// the next time a new version of the module is fetched and the file is rewritten.
   724  	// TODO(matloob): golang.org/issue/43313 covers adding a go mod verify
   725  	// command that removes module versions that fail checksums. It should also
   726  	// remove list files that are detected to be corrupt.
   727  	f, err := lockedfile.Edit(listFile)
   728  	if err != nil {
   729  		return err
   730  	}
   731  	defer func() {
   732  		if cerr := f.Close(); cerr != nil && err == nil {
   733  			err = cerr
   734  		}
   735  	}()
   736  	infos, err := os.ReadDir(dir)
   737  	if err != nil {
   738  		return err
   739  	}
   740  	var list []string
   741  	for _, info := range infos {
   742  		// We look for *.mod files on the theory that if we can't supply
   743  		// the .mod file then there's no point in listing that version,
   744  		// since it's unusable. (We can have *.info without *.mod.)
   745  		// We don't require *.zip files on the theory that for code only
   746  		// involved in module graph construction, many *.zip files
   747  		// will never be requested.
   748  		name := info.Name()
   749  		if v, found := strings.CutSuffix(name, ".mod"); found {
   750  			if v != "" && module.CanonicalVersion(v) == v {
   751  				list = append(list, v)
   752  			}
   753  		}
   754  	}
   755  	semver.Sort(list)
   756  
   757  	var buf bytes.Buffer
   758  	for _, v := range list {
   759  		buf.WriteString(v)
   760  		buf.WriteString("\n")
   761  	}
   762  	if fi, err := f.Stat(); err == nil && int(fi.Size()) == buf.Len() {
   763  		old := make([]byte, buf.Len()+1)
   764  		if n, err := f.ReadAt(old, 0); err == io.EOF && n == buf.Len() && bytes.Equal(buf.Bytes(), old) {
   765  			return nil // No edit needed.
   766  		}
   767  	}
   768  	// Remove existing contents, so that when we truncate to the actual size it will zero-fill,
   769  	// and we will be able to detect (some) incomplete writes as files containing trailing NUL bytes.
   770  	if err := f.Truncate(0); err != nil {
   771  		return err
   772  	}
   773  	// Reserve the final size and zero-fill.
   774  	if err := f.Truncate(int64(buf.Len())); err != nil {
   775  		return err
   776  	}
   777  	// Write the actual contents. If this fails partway through,
   778  	// the remainder of the file should remain as zeroes.
   779  	if _, err := f.Write(buf.Bytes()); err != nil {
   780  		f.Truncate(0)
   781  		return err
   782  	}
   783  
   784  	return nil
   785  }
   786  
   787  var (
   788  	statCacheOnce sync.Once
   789  	statCacheErr  error
   790  
   791  	counterErrorsGOMODCACHEEntryRelative = counter.New("go/errors:gomodcache-entry-relative")
   792  )
   793  
   794  // checkCacheDir checks if the directory specified by GOMODCACHE exists. An
   795  // error is returned if it does not.
   796  func checkCacheDir(ctx context.Context) error {
   797  	if cfg.GOMODCACHE == "" {
   798  		// modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE
   799  		// is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen.
   800  		return fmt.Errorf("module cache not found: neither GOMODCACHE nor GOPATH is set")
   801  	}
   802  	if !filepath.IsAbs(cfg.GOMODCACHE) {
   803  		counterErrorsGOMODCACHEEntryRelative.Inc()
   804  		return fmt.Errorf("GOMODCACHE entry is relative; must be absolute path: %q.\n", cfg.GOMODCACHE)
   805  	}
   806  
   807  	// os.Stat is slow on Windows, so we only call it once to prevent unnecessary
   808  	// I/O every time this function is called.
   809  	statCacheOnce.Do(func() {
   810  		fi, err := os.Stat(cfg.GOMODCACHE)
   811  		if err != nil {
   812  			if !os.IsNotExist(err) {
   813  				statCacheErr = fmt.Errorf("could not create module cache: %w", err)
   814  				return
   815  			}
   816  			if err := os.MkdirAll(cfg.GOMODCACHE, 0o777); err != nil {
   817  				statCacheErr = fmt.Errorf("could not create module cache: %w", err)
   818  				return
   819  			}
   820  			return
   821  		}
   822  		if !fi.IsDir() {
   823  			statCacheErr = fmt.Errorf("could not create module cache: %q is not a directory", cfg.GOMODCACHE)
   824  			return
   825  		}
   826  	})
   827  	return statCacheErr
   828  }
   829
View as plain text