Source file src/cmd/go/internal/modload/load.go
1 // Copyright 2018 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package modload 6 7 // This file contains the module-mode package loader, as well as some accessory 8 // functions pertaining to the package import graph. 9 // 10 // There are two exported entry points into package loading — LoadPackages and 11 // ImportFromFiles — both implemented in terms of loadFromRoots, which itself 12 // manipulates an instance of the loader struct. 13 // 14 // Although most of the loading state is maintained in the loader struct, 15 // one key piece - the build list - is a global, so that it can be modified 16 // separate from the loading operation, such as during "go get" 17 // upgrades/downgrades or in "go mod" operations. 18 // TODO(#40775): It might be nice to make the loader take and return 19 // a buildList rather than hard-coding use of the global. 20 // 21 // Loading is an iterative process. On each iteration, we try to load the 22 // requested packages and their transitive imports, then try to resolve modules 23 // for any imported packages that are still missing. 24 // 25 // The first step of each iteration identifies a set of “root” packages. 26 // Normally the root packages are exactly those matching the named pattern 27 // arguments. However, for the "all" meta-pattern, the final set of packages is 28 // computed from the package import graph, and therefore cannot be an initial 29 // input to loading that graph. Instead, the root packages for the "all" pattern 30 // are those contained in the main module, and allPatternIsRoot parameter to the 31 // loader instructs it to dynamically expand those roots to the full "all" 32 // pattern as loading progresses. 33 // 34 // The pkgInAll flag on each loadPkg instance tracks whether that 35 // package is known to match the "all" meta-pattern. 36 // A package matches the "all" pattern if: 37 // - it is in the main module, or 38 // - it is imported by any test in the main module, or 39 // - it is imported by another package in "all", or 40 // - the main module specifies a go version ≤ 1.15, and the package is imported 41 // by a *test of* another package in "all". 42 // 43 // When graph pruning is in effect, we want to spot-check the graph-pruning 44 // invariants — which depend on which packages are known to be in "all" — even 45 // when we are only loading individual packages, so we set the pkgInAll flag 46 // regardless of the whether the "all" pattern is a root. 47 // (This is necessary to maintain the “import invariant” described in 48 // https://golang.org/design/36460-lazy-module-loading.) 49 // 50 // Because "go mod vendor" prunes out the tests of vendored packages, the 51 // behavior of the "all" pattern with -mod=vendor in Go 1.11–1.15 is the same 52 // as the "all" pattern (regardless of the -mod flag) in 1.16+. 53 // The loader uses the GoVersion parameter to determine whether the "all" 54 // pattern should close over tests (as in Go 1.11–1.15) or stop at only those 55 // packages transitively imported by the packages and tests in the main module 56 // ("all" in Go 1.16+ and "go mod vendor" in Go 1.11+). 57 // 58 // Note that it is possible for a loaded package NOT to be in "all" even when we 59 // are loading the "all" pattern. For example, packages that are transitive 60 // dependencies of other roots named on the command line must be loaded, but are 61 // not in "all". (The mod_notall test illustrates this behavior.) 62 // Similarly, if the LoadTests flag is set but the "all" pattern does not close 63 // over test dependencies, then when we load the test of a package that is in 64 // "all" but outside the main module, the dependencies of that test will not 65 // necessarily themselves be in "all". (That configuration does not arise in Go 66 // 1.11–1.15, but it will be possible in Go 1.16+.) 67 // 68 // Loading proceeds from the roots, using a parallel work-queue with a limit on 69 // the amount of active work (to avoid saturating disks, CPU cores, and/or 70 // network connections). Each package is added to the queue the first time it is 71 // imported by another package. When we have finished identifying the imports of 72 // a package, we add the test for that package if it is needed. A test may be 73 // needed if: 74 // - the package matches a root pattern and tests of the roots were requested, or 75 // - the package is in the main module and the "all" pattern is requested 76 // (because the "all" pattern includes the dependencies of tests in the main 77 // module), or 78 // - the package is in "all" and the definition of "all" we are using includes 79 // dependencies of tests (as is the case in Go ≤1.15). 80 // 81 // After all available packages have been loaded, we examine the results to 82 // identify any requested or imported packages that are still missing, and if 83 // so, which modules we could add to the module graph in order to make the 84 // missing packages available. We add those to the module graph and iterate, 85 // until either all packages resolve successfully or we cannot identify any 86 // module that would resolve any remaining missing package. 87 // 88 // If the main module is “tidy” (that is, if "go mod tidy" is a no-op for it) 89 // and all requested packages are in "all", then loading completes in a single 90 // iteration. 91 // TODO(bcmills): We should also be able to load in a single iteration if the 92 // requested packages all come from modules that are themselves tidy, regardless 93 // of whether those packages are in "all". Today, that requires two iterations 94 // if those packages are not found in existing dependencies of the main module. 95 96 import ( 97 "context" 98 "errors" 99 "fmt" 100 "go/build" 101 "io/fs" 102 "maps" 103 "os" 104 "path" 105 pathpkg "path" 106 "path/filepath" 107 "runtime" 108 "slices" 109 "sort" 110 "strings" 111 "sync" 112 "sync/atomic" 113 114 "cmd/go/internal/base" 115 "cmd/go/internal/cfg" 116 "cmd/go/internal/fsys" 117 "cmd/go/internal/gover" 118 "cmd/go/internal/imports" 119 "cmd/go/internal/modfetch" 120 "cmd/go/internal/modindex" 121 "cmd/go/internal/mvs" 122 "cmd/go/internal/par" 123 "cmd/go/internal/search" 124 "cmd/go/internal/str" 125 126 "golang.org/x/mod/module" 127 ) 128 129 // loaded is the most recently-used package loader. 130 // It holds details about individual packages. 131 // 132 // This variable should only be accessed directly in top-level exported 133 // functions. All other functions that require or produce a *loader should pass 134 // or return it as an explicit parameter. 135 var loaded *loader 136 137 // PackageOpts control the behavior of the LoadPackages function. 138 type PackageOpts struct { 139 // TidyGoVersion is the Go version to which the go.mod file should be updated 140 // after packages have been loaded. 141 // 142 // An empty TidyGoVersion means to use the Go version already specified in the 143 // main module's go.mod file, or the latest Go version if there is no main 144 // module. 145 TidyGoVersion string 146 147 // Tags are the build tags in effect (as interpreted by the 148 // cmd/go/internal/imports package). 149 // If nil, treated as equivalent to imports.Tags(). 150 Tags map[string]bool 151 152 // Tidy, if true, requests that the build list and go.sum file be reduced to 153 // the minimal dependencies needed to reproducibly reload the requested 154 // packages. 155 Tidy bool 156 157 // TidyCompatibleVersion is the oldest Go version that must be able to 158 // reproducibly reload the requested packages. 159 // 160 // If empty, the compatible version is the Go version immediately prior to the 161 // 'go' version listed in the go.mod file. 162 TidyCompatibleVersion string 163 164 // VendorModulesInGOROOTSrc indicates that if we are within a module in 165 // GOROOT/src, packages in the module's vendor directory should be resolved as 166 // actual module dependencies (instead of standard-library packages). 167 VendorModulesInGOROOTSrc bool 168 169 // ResolveMissingImports indicates that we should attempt to add module 170 // dependencies as needed to resolve imports of packages that are not found. 171 // 172 // For commands that support the -mod flag, resolving imports may still fail 173 // if the flag is set to "readonly" (the default) or "vendor". 174 ResolveMissingImports bool 175 176 // AssumeRootsImported indicates that the transitive dependencies of the root 177 // packages should be treated as if those roots will be imported by the main 178 // module. 179 AssumeRootsImported bool 180 181 // AllowPackage, if non-nil, is called after identifying the module providing 182 // each package. If AllowPackage returns a non-nil error, that error is set 183 // for the package, and the imports and test of that package will not be 184 // loaded. 185 // 186 // AllowPackage may be invoked concurrently by multiple goroutines, 187 // and may be invoked multiple times for a given package path. 188 AllowPackage func(ctx context.Context, path string, mod module.Version) error 189 190 // LoadTests loads the test dependencies of each package matching a requested 191 // pattern. If ResolveMissingImports is also true, test dependencies will be 192 // resolved if missing. 193 LoadTests bool 194 195 // UseVendorAll causes the "all" package pattern to be interpreted as if 196 // running "go mod vendor" (or building with "-mod=vendor"). 197 // 198 // This is a no-op for modules that declare 'go 1.16' or higher, for which this 199 // is the default (and only) interpretation of the "all" pattern in module mode. 200 UseVendorAll bool 201 202 // AllowErrors indicates that LoadPackages should not terminate the process if 203 // an error occurs. 204 AllowErrors bool 205 206 // SilencePackageErrors indicates that LoadPackages should not print errors 207 // that occur while matching or loading packages, and should not terminate the 208 // process if such an error occurs. 209 // 210 // Errors encountered in the module graph will still be reported. 211 // 212 // The caller may retrieve the silenced package errors using the Lookup 213 // function, and matching errors are still populated in the Errs field of the 214 // associated search.Match.) 215 SilencePackageErrors bool 216 217 // SilenceMissingStdImports indicates that LoadPackages should not print 218 // errors or terminate the process if an imported package is missing, and the 219 // import path looks like it might be in the standard library (perhaps in a 220 // future version). 221 SilenceMissingStdImports bool 222 223 // SilenceNoGoErrors indicates that LoadPackages should not print 224 // imports.ErrNoGo errors. 225 // This allows the caller to invoke LoadPackages (and report other errors) 226 // without knowing whether the requested packages exist for the given tags. 227 // 228 // Note that if a requested package does not exist *at all*, it will fail 229 // during module resolution and the error will not be suppressed. 230 SilenceNoGoErrors bool 231 232 // SilenceUnmatchedWarnings suppresses the warnings normally emitted for 233 // patterns that did not match any packages. 234 SilenceUnmatchedWarnings bool 235 236 // Resolve the query against this module. 237 MainModule module.Version 238 239 // If Switcher is non-nil, then LoadPackages passes all encountered errors 240 // to Switcher.Error and tries Switcher.Switch before base.ExitIfErrors. 241 Switcher gover.Switcher 242 } 243 244 // LoadPackages identifies the set of packages matching the given patterns and 245 // loads the packages in the import graph rooted at that set. 246 func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (matches []*search.Match, loadedPackages []string) { 247 if opts.Tags == nil { 248 opts.Tags = imports.Tags() 249 } 250 251 patterns = search.CleanPatterns(patterns) 252 matches = make([]*search.Match, 0, len(patterns)) 253 allPatternIsRoot := false 254 for _, pattern := range patterns { 255 matches = append(matches, search.NewMatch(pattern)) 256 if pattern == "all" { 257 allPatternIsRoot = true 258 } 259 } 260 261 updateMatches := func(rs *Requirements, ld *loader) { 262 for _, m := range matches { 263 switch { 264 case m.IsLocal(): 265 // Evaluate list of file system directories on first iteration. 266 if m.Dirs == nil { 267 matchModRoots := modRoots 268 if opts.MainModule != (module.Version{}) { 269 matchModRoots = []string{MainModules.ModRoot(opts.MainModule)} 270 } 271 matchLocalDirs(ctx, matchModRoots, m, rs) 272 } 273 274 // Make a copy of the directory list and translate to import paths. 275 // Note that whether a directory corresponds to an import path 276 // changes as the build list is updated, and a directory can change 277 // from not being in the build list to being in it and back as 278 // the exact version of a particular module increases during 279 // the loader iterations. 280 m.Pkgs = m.Pkgs[:0] 281 for _, dir := range m.Dirs { 282 pkg, err := resolveLocalPackage(ctx, dir, rs) 283 if err != nil { 284 if !m.IsLiteral() && (err == errPkgIsBuiltin || err == errPkgIsGorootSrc) { 285 continue // Don't include "builtin" or GOROOT/src in wildcard patterns. 286 } 287 288 // If we're outside of a module, ensure that the failure mode 289 // indicates that. 290 if !HasModRoot() { 291 die() 292 } 293 294 if ld != nil { 295 m.AddError(err) 296 } 297 continue 298 } 299 m.Pkgs = append(m.Pkgs, pkg) 300 } 301 302 case m.IsLiteral(): 303 m.Pkgs = []string{m.Pattern()} 304 305 case strings.Contains(m.Pattern(), "..."): 306 m.Errs = m.Errs[:0] 307 mg, err := rs.Graph(ctx) 308 if err != nil { 309 // The module graph is (or may be) incomplete — perhaps we failed to 310 // load the requirements of some module. This is an error in matching 311 // the patterns to packages, because we may be missing some packages 312 // or we may erroneously match packages in the wrong versions of 313 // modules. However, for cases like 'go list -e', the error should not 314 // necessarily prevent us from loading the packages we could find. 315 m.Errs = append(m.Errs, err) 316 } 317 matchPackages(ctx, m, opts.Tags, includeStd, mg.BuildList()) 318 319 case m.Pattern() == "all": 320 if ld == nil { 321 // The initial roots are the packages in the main module. 322 // loadFromRoots will expand that to "all". 323 m.Errs = m.Errs[:0] 324 matchModules := MainModules.Versions() 325 if opts.MainModule != (module.Version{}) { 326 matchModules = []module.Version{opts.MainModule} 327 } 328 matchPackages(ctx, m, opts.Tags, omitStd, matchModules) 329 } else { 330 // Starting with the packages in the main module, 331 // enumerate the full list of "all". 332 m.Pkgs = ld.computePatternAll() 333 } 334 335 case m.Pattern() == "std" || m.Pattern() == "cmd": 336 if m.Pkgs == nil { 337 m.MatchPackages() // Locate the packages within GOROOT/src. 338 } 339 340 default: 341 panic(fmt.Sprintf("internal error: modload missing case for pattern %s", m.Pattern())) 342 } 343 } 344 } 345 346 initialRS, err := loadModFile(ctx, &opts) 347 if err != nil { 348 base.Fatal(err) 349 } 350 351 ld := loadFromRoots(ctx, loaderParams{ 352 PackageOpts: opts, 353 requirements: initialRS, 354 355 allPatternIsRoot: allPatternIsRoot, 356 357 listRoots: func(rs *Requirements) (roots []string) { 358 updateMatches(rs, nil) 359 for _, m := range matches { 360 roots = append(roots, m.Pkgs...) 361 } 362 return roots 363 }, 364 }) 365 366 // One last pass to finalize wildcards. 367 updateMatches(ld.requirements, ld) 368 369 // List errors in matching patterns (such as directory permission 370 // errors for wildcard patterns). 371 if !ld.SilencePackageErrors { 372 for _, match := range matches { 373 for _, err := range match.Errs { 374 ld.error(err) 375 } 376 } 377 } 378 ld.exitIfErrors(ctx) 379 380 if !opts.SilenceUnmatchedWarnings { 381 search.WarnUnmatched(matches) 382 } 383 384 if opts.Tidy { 385 if cfg.BuildV { 386 mg, _ := ld.requirements.Graph(ctx) 387 for _, m := range initialRS.rootModules { 388 var unused bool 389 if ld.requirements.pruning == unpruned { 390 // m is unused if it was dropped from the module graph entirely. If it 391 // was only demoted from direct to indirect, it may still be in use via 392 // a transitive import. 393 unused = mg.Selected(m.Path) == "none" 394 } else { 395 // m is unused if it was dropped from the roots. If it is still present 396 // as a transitive dependency, that transitive dependency is not needed 397 // by any package or test in the main module. 398 _, ok := ld.requirements.rootSelected(m.Path) 399 unused = !ok 400 } 401 if unused { 402 fmt.Fprintf(os.Stderr, "unused %s\n", m.Path) 403 } 404 } 405 } 406 407 keep := keepSums(ctx, ld, ld.requirements, loadedZipSumsOnly) 408 compatVersion := ld.TidyCompatibleVersion 409 goVersion := ld.requirements.GoVersion() 410 if compatVersion == "" { 411 if gover.Compare(goVersion, gover.GoStrictVersion) < 0 { 412 compatVersion = gover.Prev(goVersion) 413 } else { 414 // Starting at GoStrictVersion, we no longer maintain compatibility with 415 // versions older than what is listed in the go.mod file. 416 compatVersion = goVersion 417 } 418 } 419 if gover.Compare(compatVersion, goVersion) > 0 { 420 // Each version of the Go toolchain knows how to interpret go.mod and 421 // go.sum files produced by all previous versions, so a compatibility 422 // version higher than the go.mod version adds nothing. 423 compatVersion = goVersion 424 } 425 if compatPruning := pruningForGoVersion(compatVersion); compatPruning != ld.requirements.pruning { 426 compatRS := newRequirements(compatPruning, ld.requirements.rootModules, ld.requirements.direct) 427 ld.checkTidyCompatibility(ctx, compatRS, compatVersion) 428 429 for m := range keepSums(ctx, ld, compatRS, loadedZipSumsOnly) { 430 keep[m] = true 431 } 432 } 433 434 if !ExplicitWriteGoMod { 435 modfetch.TrimGoSum(keep) 436 437 // commitRequirements below will also call WriteGoSum, but the "keep" map 438 // we have here could be strictly larger: commitRequirements only commits 439 // loaded.requirements, but here we may have also loaded (and want to 440 // preserve checksums for) additional entities from compatRS, which are 441 // only needed for compatibility with ld.TidyCompatibleVersion. 442 if err := modfetch.WriteGoSum(ctx, keep, mustHaveCompleteRequirements()); err != nil { 443 base.Fatal(err) 444 } 445 } 446 } 447 448 // Success! Update go.mod and go.sum (if needed) and return the results. 449 // We'll skip updating if ExplicitWriteGoMod is true (the caller has opted 450 // to call WriteGoMod itself) or if ResolveMissingImports is false (the 451 // command wants to examine the package graph as-is). 452 loaded = ld 453 requirements = loaded.requirements 454 455 for _, pkg := range ld.pkgs { 456 if !pkg.isTest() { 457 loadedPackages = append(loadedPackages, pkg.path) 458 } 459 } 460 sort.Strings(loadedPackages) 461 462 if !ExplicitWriteGoMod && opts.ResolveMissingImports { 463 if err := commitRequirements(ctx, WriteOpts{}); err != nil { 464 base.Fatal(err) 465 } 466 } 467 468 return matches, loadedPackages 469 } 470 471 // matchLocalDirs is like m.MatchDirs, but tries to avoid scanning directories 472 // outside of the standard library and active modules. 473 func matchLocalDirs(ctx context.Context, modRoots []string, m *search.Match, rs *Requirements) { 474 if !m.IsLocal() { 475 panic(fmt.Sprintf("internal error: resolveLocalDirs on non-local pattern %s", m.Pattern())) 476 } 477 478 if i := strings.Index(m.Pattern(), "..."); i >= 0 { 479 // The pattern is local, but it is a wildcard. Its packages will 480 // only resolve to paths if they are inside of the standard 481 // library, the main module, or some dependency of the main 482 // module. Verify that before we walk the filesystem: a filesystem 483 // walk in a directory like /var or /etc can be very expensive! 484 dir := filepath.Dir(filepath.Clean(m.Pattern()[:i+3])) 485 absDir := dir 486 if !filepath.IsAbs(dir) { 487 absDir = filepath.Join(base.Cwd(), dir) 488 } 489 490 modRoot := findModuleRoot(absDir) 491 found := false 492 for _, mainModuleRoot := range modRoots { 493 if mainModuleRoot == modRoot { 494 found = true 495 break 496 } 497 } 498 if !found && search.InDir(absDir, cfg.GOROOTsrc) == "" && pathInModuleCache(ctx, absDir, rs) == "" { 499 m.Dirs = []string{} 500 scope := "main module or its selected dependencies" 501 if inWorkspaceMode() { 502 scope = "modules listed in go.work or their selected dependencies" 503 } 504 m.AddError(fmt.Errorf("directory prefix %s does not contain %s", base.ShortPath(absDir), scope)) 505 return 506 } 507 } 508 509 m.MatchDirs(modRoots) 510 } 511 512 // resolveLocalPackage resolves a filesystem path to a package path. 513 func resolveLocalPackage(ctx context.Context, dir string, rs *Requirements) (string, error) { 514 var absDir string 515 if filepath.IsAbs(dir) { 516 absDir = filepath.Clean(dir) 517 } else { 518 absDir = filepath.Join(base.Cwd(), dir) 519 } 520 521 bp, err := cfg.BuildContext.ImportDir(absDir, 0) 522 if err != nil && (bp == nil || len(bp.IgnoredGoFiles) == 0) { 523 // golang.org/issue/32917: We should resolve a relative path to a 524 // package path only if the relative path actually contains the code 525 // for that package. 526 // 527 // If the named directory does not exist or contains no Go files, 528 // the package does not exist. 529 // Other errors may affect package loading, but not resolution. 530 if _, err := fsys.Stat(absDir); err != nil { 531 if os.IsNotExist(err) { 532 // Canonicalize OS-specific errors to errDirectoryNotFound so that error 533 // messages will be easier for users to search for. 534 return "", &fs.PathError{Op: "stat", Path: absDir, Err: errDirectoryNotFound} 535 } 536 return "", err 537 } 538 if _, noGo := err.(*build.NoGoError); noGo { 539 // A directory that does not contain any Go source files — even ignored 540 // ones! — is not a Go package, and we can't resolve it to a package 541 // path because that path could plausibly be provided by some other 542 // module. 543 // 544 // Any other error indicates that the package “exists” (at least in the 545 // sense that it cannot exist in any other module), but has some other 546 // problem (such as a syntax error). 547 return "", err 548 } 549 } 550 551 for _, mod := range MainModules.Versions() { 552 modRoot := MainModules.ModRoot(mod) 553 if modRoot != "" && absDir == modRoot { 554 if absDir == cfg.GOROOTsrc { 555 return "", errPkgIsGorootSrc 556 } 557 return MainModules.PathPrefix(mod), nil 558 } 559 } 560 561 // Note: The checks for @ here are just to avoid misinterpreting 562 // the module cache directories (formerly GOPATH/src/mod/foo@v1.5.2/bar). 563 // It's not strictly necessary but helpful to keep the checks. 564 var pkgNotFoundErr error 565 pkgNotFoundLongestPrefix := "" 566 for _, mainModule := range MainModules.Versions() { 567 modRoot := MainModules.ModRoot(mainModule) 568 if modRoot != "" && str.HasFilePathPrefix(absDir, modRoot) && !strings.Contains(absDir[len(modRoot):], "@") { 569 suffix := filepath.ToSlash(str.TrimFilePathPrefix(absDir, modRoot)) 570 if pkg, found := strings.CutPrefix(suffix, "vendor/"); found { 571 if cfg.BuildMod != "vendor" { 572 return "", fmt.Errorf("without -mod=vendor, directory %s has no package path", absDir) 573 } 574 575 readVendorList(VendorDir()) 576 if _, ok := vendorPkgModule[pkg]; !ok { 577 return "", fmt.Errorf("directory %s is not a package listed in vendor/modules.txt", absDir) 578 } 579 return pkg, nil 580 } 581 582 mainModulePrefix := MainModules.PathPrefix(mainModule) 583 if mainModulePrefix == "" { 584 pkg := suffix 585 if pkg == "builtin" { 586 // "builtin" is a pseudo-package with a real source file. 587 // It's not included in "std", so it shouldn't resolve from "." 588 // within module "std" either. 589 return "", errPkgIsBuiltin 590 } 591 return pkg, nil 592 } 593 594 pkg := pathpkg.Join(mainModulePrefix, suffix) 595 if _, ok, err := dirInModule(pkg, mainModulePrefix, modRoot, true); err != nil { 596 return "", err 597 } else if !ok { 598 // This main module could contain the directory but doesn't. Other main 599 // modules might contain the directory, so wait till we finish the loop 600 // to see if another main module contains directory. But if not, 601 // return an error. 602 if len(mainModulePrefix) > len(pkgNotFoundLongestPrefix) { 603 pkgNotFoundLongestPrefix = mainModulePrefix 604 pkgNotFoundErr = &PackageNotInModuleError{MainModules: []module.Version{mainModule}, Pattern: pkg} 605 } 606 continue 607 } 608 return pkg, nil 609 } 610 } 611 if pkgNotFoundErr != nil { 612 return "", pkgNotFoundErr 613 } 614 615 if sub := search.InDir(absDir, cfg.GOROOTsrc); sub != "" && sub != "." && !strings.Contains(sub, "@") { 616 pkg := filepath.ToSlash(sub) 617 if pkg == "builtin" { 618 return "", errPkgIsBuiltin 619 } 620 return pkg, nil 621 } 622 623 pkg := pathInModuleCache(ctx, absDir, rs) 624 if pkg == "" { 625 dirstr := fmt.Sprintf("directory %s", base.ShortPath(absDir)) 626 if dirstr == "directory ." { 627 dirstr = "current directory" 628 } 629 if inWorkspaceMode() { 630 if mr := findModuleRoot(absDir); mr != "" { 631 return "", fmt.Errorf("%s is contained in a module that is not one of the workspace modules listed in go.work. You can add the module to the workspace using:\n\tgo work use %s", dirstr, base.ShortPath(mr)) 632 } 633 return "", fmt.Errorf("%s outside modules listed in go.work or their selected dependencies", dirstr) 634 } 635 return "", fmt.Errorf("%s outside main module or its selected dependencies", dirstr) 636 } 637 return pkg, nil 638 } 639 640 var ( 641 errDirectoryNotFound = errors.New("directory not found") 642 errPkgIsGorootSrc = errors.New("GOROOT/src is not an importable package") 643 errPkgIsBuiltin = errors.New(`"builtin" is a pseudo-package, not an importable package`) 644 ) 645 646 // pathInModuleCache returns the import path of the directory dir, 647 // if dir is in the module cache copy of a module in our build list. 648 func pathInModuleCache(ctx context.Context, dir string, rs *Requirements) string { 649 tryMod := func(m module.Version) (string, bool) { 650 if gover.IsToolchain(m.Path) { 651 return "", false 652 } 653 var root string 654 var err error 655 if repl := Replacement(m); repl.Path != "" && repl.Version == "" { 656 root = repl.Path 657 if !filepath.IsAbs(root) { 658 root = filepath.Join(replaceRelativeTo(), root) 659 } 660 } else if repl.Path != "" { 661 root, err = modfetch.DownloadDir(ctx, repl) 662 } else { 663 root, err = modfetch.DownloadDir(ctx, m) 664 } 665 if err != nil { 666 return "", false 667 } 668 669 sub := search.InDir(dir, root) 670 if sub == "" { 671 return "", false 672 } 673 sub = filepath.ToSlash(sub) 674 if strings.Contains(sub, "/vendor/") || strings.HasPrefix(sub, "vendor/") || strings.Contains(sub, "@") { 675 return "", false 676 } 677 678 return path.Join(m.Path, filepath.ToSlash(sub)), true 679 } 680 681 if rs.pruning == pruned { 682 for _, m := range rs.rootModules { 683 if v, _ := rs.rootSelected(m.Path); v != m.Version { 684 continue // m is a root, but we have a higher root for the same path. 685 } 686 if importPath, ok := tryMod(m); ok { 687 // checkMultiplePaths ensures that a module can be used for at most one 688 // requirement, so this must be it. 689 return importPath 690 } 691 } 692 } 693 694 // None of the roots contained dir, or the graph is unpruned (so we don't want 695 // to distinguish between roots and transitive dependencies). Either way, 696 // check the full graph to see if the directory is a non-root dependency. 697 // 698 // If the roots are not consistent with the full module graph, the selected 699 // versions of root modules may differ from what we already checked above. 700 // Re-check those paths too. 701 702 mg, _ := rs.Graph(ctx) 703 var importPath string 704 for _, m := range mg.BuildList() { 705 var found bool 706 importPath, found = tryMod(m) 707 if found { 708 break 709 } 710 } 711 return importPath 712 } 713 714 // ImportFromFiles adds modules to the build list as needed 715 // to satisfy the imports in the named Go source files. 716 // 717 // Errors in missing dependencies are silenced. 718 // 719 // TODO(bcmills): Silencing errors seems off. Take a closer look at this and 720 // figure out what the error-reporting actually ought to be. 721 func ImportFromFiles(ctx context.Context, gofiles []string) { 722 rs := LoadModFile(ctx) 723 724 tags := imports.Tags() 725 imports, testImports, err := imports.ScanFiles(gofiles, tags) 726 if err != nil { 727 base.Fatal(err) 728 } 729 730 loaded = loadFromRoots(ctx, loaderParams{ 731 PackageOpts: PackageOpts{ 732 Tags: tags, 733 ResolveMissingImports: true, 734 SilencePackageErrors: true, 735 }, 736 requirements: rs, 737 listRoots: func(*Requirements) (roots []string) { 738 roots = append(roots, imports...) 739 roots = append(roots, testImports...) 740 return roots 741 }, 742 }) 743 requirements = loaded.requirements 744 745 if !ExplicitWriteGoMod { 746 if err := commitRequirements(ctx, WriteOpts{}); err != nil { 747 base.Fatal(err) 748 } 749 } 750 } 751 752 // DirImportPath returns the effective import path for dir, 753 // provided it is within a main module, or else returns ".". 754 func (mms *MainModuleSet) DirImportPath(ctx context.Context, dir string) (path string, m module.Version) { 755 if !HasModRoot() { 756 return ".", module.Version{} 757 } 758 LoadModFile(ctx) // Sets targetPrefix. 759 760 if !filepath.IsAbs(dir) { 761 dir = filepath.Join(base.Cwd(), dir) 762 } else { 763 dir = filepath.Clean(dir) 764 } 765 766 var longestPrefix string 767 var longestPrefixPath string 768 var longestPrefixVersion module.Version 769 for _, v := range mms.Versions() { 770 modRoot := mms.ModRoot(v) 771 if dir == modRoot { 772 return mms.PathPrefix(v), v 773 } 774 if str.HasFilePathPrefix(dir, modRoot) { 775 pathPrefix := MainModules.PathPrefix(v) 776 if pathPrefix > longestPrefix { 777 longestPrefix = pathPrefix 778 longestPrefixVersion = v 779 suffix := filepath.ToSlash(str.TrimFilePathPrefix(dir, modRoot)) 780 if strings.HasPrefix(suffix, "vendor/") { 781 longestPrefixPath = suffix[len("vendor/"):] 782 continue 783 } 784 longestPrefixPath = pathpkg.Join(mms.PathPrefix(v), suffix) 785 } 786 } 787 } 788 if len(longestPrefix) > 0 { 789 return longestPrefixPath, longestPrefixVersion 790 } 791 792 return ".", module.Version{} 793 } 794 795 // PackageModule returns the module providing the package named by the import path. 796 func PackageModule(path string) module.Version { 797 pkg, ok := loaded.pkgCache.Get(path) 798 if !ok { 799 return module.Version{} 800 } 801 return pkg.mod 802 } 803 804 // Lookup returns the source directory, import path, and any loading error for 805 // the package at path as imported from the package in parentDir. 806 // Lookup requires that one of the Load functions in this package has already 807 // been called. 808 func Lookup(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) { 809 if path == "" { 810 panic("Lookup called with empty package path") 811 } 812 813 if parentIsStd { 814 path = loaded.stdVendor(parentPath, path) 815 } 816 pkg, ok := loaded.pkgCache.Get(path) 817 if !ok { 818 // The loader should have found all the relevant paths. 819 // There are a few exceptions, though: 820 // - during go list without -test, the p.Resolve calls to process p.TestImports and p.XTestImports 821 // end up here to canonicalize the import paths. 822 // - during any load, non-loaded packages like "unsafe" end up here. 823 // - during any load, build-injected dependencies like "runtime/cgo" end up here. 824 // - because we ignore appengine/* in the module loader, 825 // the dependencies of any actual appengine/* library end up here. 826 dir := findStandardImportPath(path) 827 if dir != "" { 828 return dir, path, nil 829 } 830 return "", "", errMissing 831 } 832 return pkg.dir, pkg.path, pkg.err 833 } 834 835 // A loader manages the process of loading information about 836 // the required packages for a particular build, 837 // checking that the packages are available in the module set, 838 // and updating the module set if needed. 839 type loader struct { 840 loaderParams 841 842 // allClosesOverTests indicates whether the "all" pattern includes 843 // dependencies of tests outside the main module (as in Go 1.11–1.15). 844 // (Otherwise — as in Go 1.16+ — the "all" pattern includes only the packages 845 // transitively *imported by* the packages and tests in the main module.) 846 allClosesOverTests bool 847 848 // skipImportModFiles indicates whether we may skip loading go.mod files 849 // for imported packages (as in 'go mod tidy' in Go 1.17–1.20). 850 skipImportModFiles bool 851 852 work *par.Queue 853 854 // reset on each iteration 855 roots []*loadPkg 856 pkgCache *par.Cache[string, *loadPkg] 857 pkgs []*loadPkg // transitive closure of loaded packages and tests; populated in buildStacks 858 } 859 860 // loaderParams configure the packages loaded by, and the properties reported 861 // by, a loader instance. 862 type loaderParams struct { 863 PackageOpts 864 requirements *Requirements 865 866 allPatternIsRoot bool // Is the "all" pattern an additional root? 867 868 listRoots func(rs *Requirements) []string 869 } 870 871 func (ld *loader) reset() { 872 select { 873 case <-ld.work.Idle(): 874 default: 875 panic("loader.reset when not idle") 876 } 877 878 ld.roots = nil 879 ld.pkgCache = new(par.Cache[string, *loadPkg]) 880 ld.pkgs = nil 881 } 882 883 // error reports an error via either os.Stderr or base.Error, 884 // according to whether ld.AllowErrors is set. 885 func (ld *loader) error(err error) { 886 if ld.AllowErrors { 887 fmt.Fprintf(os.Stderr, "go: %v\n", err) 888 } else if ld.Switcher != nil { 889 ld.Switcher.Error(err) 890 } else { 891 base.Error(err) 892 } 893 } 894 895 // switchIfErrors switches toolchains if a switch is needed. 896 func (ld *loader) switchIfErrors(ctx context.Context) { 897 if ld.Switcher != nil { 898 ld.Switcher.Switch(ctx) 899 } 900 } 901 902 // exitIfErrors switches toolchains if a switch is needed 903 // or else exits if any errors have been reported. 904 func (ld *loader) exitIfErrors(ctx context.Context) { 905 ld.switchIfErrors(ctx) 906 base.ExitIfErrors() 907 } 908 909 // goVersion reports the Go version that should be used for the loader's 910 // requirements: ld.TidyGoVersion if set, or ld.requirements.GoVersion() 911 // otherwise. 912 func (ld *loader) goVersion() string { 913 if ld.TidyGoVersion != "" { 914 return ld.TidyGoVersion 915 } 916 return ld.requirements.GoVersion() 917 } 918 919 // A loadPkg records information about a single loaded package. 920 type loadPkg struct { 921 // Populated at construction time: 922 path string // import path 923 testOf *loadPkg 924 925 // Populated at construction time and updated by (*loader).applyPkgFlags: 926 flags atomicLoadPkgFlags 927 928 // Populated by (*loader).load: 929 mod module.Version // module providing package 930 dir string // directory containing source code 931 err error // error loading package 932 imports []*loadPkg // packages imported by this one 933 testImports []string // test-only imports, saved for use by pkg.test. 934 inStd bool 935 altMods []module.Version // modules that could have contained the package but did not 936 937 // Populated by (*loader).pkgTest: 938 testOnce sync.Once 939 test *loadPkg 940 941 // Populated by postprocessing in (*loader).buildStacks: 942 stack *loadPkg // package importing this one in minimal import stack for this pkg 943 } 944 945 // loadPkgFlags is a set of flags tracking metadata about a package. 946 type loadPkgFlags int8 947 948 const ( 949 // pkgInAll indicates that the package is in the "all" package pattern, 950 // regardless of whether we are loading the "all" package pattern. 951 // 952 // When the pkgInAll flag and pkgImportsLoaded flags are both set, the caller 953 // who set the last of those flags must propagate the pkgInAll marking to all 954 // of the imports of the marked package. 955 // 956 // A test is marked with pkgInAll if that test would promote the packages it 957 // imports to be in "all" (such as when the test is itself within the main 958 // module, or when ld.allClosesOverTests is true). 959 pkgInAll loadPkgFlags = 1 << iota 960 961 // pkgIsRoot indicates that the package matches one of the root package 962 // patterns requested by the caller. 963 // 964 // If LoadTests is set, then when pkgIsRoot and pkgImportsLoaded are both set, 965 // the caller who set the last of those flags must populate a test for the 966 // package (in the pkg.test field). 967 // 968 // If the "all" pattern is included as a root, then non-test packages in "all" 969 // are also roots (and must be marked pkgIsRoot). 970 pkgIsRoot 971 972 // pkgFromRoot indicates that the package is in the transitive closure of 973 // imports starting at the roots. (Note that every package marked as pkgIsRoot 974 // is also trivially marked pkgFromRoot.) 975 pkgFromRoot 976 977 // pkgImportsLoaded indicates that the imports and testImports fields of a 978 // loadPkg have been populated. 979 pkgImportsLoaded 980 ) 981 982 // has reports whether all of the flags in cond are set in f. 983 func (f loadPkgFlags) has(cond loadPkgFlags) bool { 984 return f&cond == cond 985 } 986 987 // An atomicLoadPkgFlags stores a loadPkgFlags for which individual flags can be 988 // added atomically. 989 type atomicLoadPkgFlags struct { 990 bits atomic.Int32 991 } 992 993 // update sets the given flags in af (in addition to any flags already set). 994 // 995 // update returns the previous flag state so that the caller may determine which 996 // flags were newly-set. 997 func (af *atomicLoadPkgFlags) update(flags loadPkgFlags) (old loadPkgFlags) { 998 for { 999 old := af.bits.Load() 1000 new := old | int32(flags) 1001 if new == old || af.bits.CompareAndSwap(old, new) { 1002 return loadPkgFlags(old) 1003 } 1004 } 1005 } 1006 1007 // has reports whether all of the flags in cond are set in af. 1008 func (af *atomicLoadPkgFlags) has(cond loadPkgFlags) bool { 1009 return loadPkgFlags(af.bits.Load())&cond == cond 1010 } 1011 1012 // isTest reports whether pkg is a test of another package. 1013 func (pkg *loadPkg) isTest() bool { 1014 return pkg.testOf != nil 1015 } 1016 1017 // fromExternalModule reports whether pkg was loaded from a module other than 1018 // the main module. 1019 func (pkg *loadPkg) fromExternalModule() bool { 1020 if pkg.mod.Path == "" { 1021 return false // loaded from the standard library, not a module 1022 } 1023 return !MainModules.Contains(pkg.mod.Path) 1024 } 1025 1026 var errMissing = errors.New("cannot find package") 1027 1028 // loadFromRoots attempts to load the build graph needed to process a set of 1029 // root packages and their dependencies. 1030 // 1031 // The set of root packages is returned by the params.listRoots function, and 1032 // expanded to the full set of packages by tracing imports (and possibly tests) 1033 // as needed. 1034 func loadFromRoots(ctx context.Context, params loaderParams) *loader { 1035 ld := &loader{ 1036 loaderParams: params, 1037 work: par.NewQueue(runtime.GOMAXPROCS(0)), 1038 } 1039 1040 if ld.requirements.pruning == unpruned { 1041 // If the module graph does not support pruning, we assume that we will need 1042 // the full module graph in order to load package dependencies. 1043 // 1044 // This might not be strictly necessary, but it matches the historical 1045 // behavior of the 'go' command and keeps the go.mod file more consistent in 1046 // case of erroneous hand-edits — which are less likely to be detected by 1047 // spot-checks in modules that do not maintain the expanded go.mod 1048 // requirements needed for graph pruning. 1049 var err error 1050 ld.requirements, _, err = expandGraph(ctx, ld.requirements) 1051 if err != nil { 1052 ld.error(err) 1053 } 1054 } 1055 ld.exitIfErrors(ctx) 1056 1057 updateGoVersion := func() { 1058 goVersion := ld.goVersion() 1059 1060 if ld.requirements.pruning != workspace { 1061 var err error 1062 ld.requirements, err = convertPruning(ctx, ld.requirements, pruningForGoVersion(goVersion)) 1063 if err != nil { 1064 ld.error(err) 1065 ld.exitIfErrors(ctx) 1066 } 1067 } 1068 1069 // If the module's Go version omits go.sum entries for go.mod files for test 1070 // dependencies of external packages, avoid loading those files in the first 1071 // place. 1072 ld.skipImportModFiles = ld.Tidy && gover.Compare(goVersion, gover.TidyGoModSumVersion) < 0 1073 1074 // If the module's go version explicitly predates the change in "all" for 1075 // graph pruning, continue to use the older interpretation. 1076 ld.allClosesOverTests = gover.Compare(goVersion, gover.NarrowAllVersion) < 0 && !ld.UseVendorAll 1077 } 1078 1079 for { 1080 ld.reset() 1081 updateGoVersion() 1082 1083 // Load the root packages and their imports. 1084 // Note: the returned roots can change on each iteration, 1085 // since the expansion of package patterns depends on the 1086 // build list we're using. 1087 rootPkgs := ld.listRoots(ld.requirements) 1088 1089 if ld.requirements.pruning == pruned && cfg.BuildMod == "mod" { 1090 // Before we start loading transitive imports of packages, locate all of 1091 // the root packages and promote their containing modules to root modules 1092 // dependencies. If their go.mod files are tidy (the common case) and the 1093 // set of root packages does not change then we can select the correct 1094 // versions of all transitive imports on the first try and complete 1095 // loading in a single iteration. 1096 changedBuildList := ld.preloadRootModules(ctx, rootPkgs) 1097 if changedBuildList { 1098 // The build list has changed, so the set of root packages may have also 1099 // changed. Start over to pick up the changes. (Preloading roots is much 1100 // cheaper than loading the full import graph, so we would rather pay 1101 // for an extra iteration of preloading than potentially end up 1102 // discarding the result of a full iteration of loading.) 1103 continue 1104 } 1105 } 1106 1107 inRoots := map[*loadPkg]bool{} 1108 for _, path := range rootPkgs { 1109 root := ld.pkg(ctx, path, pkgIsRoot) 1110 if !inRoots[root] { 1111 ld.roots = append(ld.roots, root) 1112 inRoots[root] = true 1113 } 1114 } 1115 1116 // ld.pkg adds imported packages to the work queue and calls applyPkgFlags, 1117 // which adds tests (and test dependencies) as needed. 1118 // 1119 // When all of the work in the queue has completed, we'll know that the 1120 // transitive closure of dependencies has been loaded. 1121 <-ld.work.Idle() 1122 1123 ld.buildStacks() 1124 1125 changed, err := ld.updateRequirements(ctx) 1126 if err != nil { 1127 ld.error(err) 1128 break 1129 } 1130 if changed { 1131 // Don't resolve missing imports until the module graph has stabilized. 1132 // If the roots are still changing, they may turn out to specify a 1133 // requirement on the missing package(s), and we would rather use a 1134 // version specified by a new root than add a new dependency on an 1135 // unrelated version. 1136 continue 1137 } 1138 1139 if !ld.ResolveMissingImports || (!HasModRoot() && !allowMissingModuleImports) { 1140 // We've loaded as much as we can without resolving missing imports. 1141 break 1142 } 1143 1144 modAddedBy, err := ld.resolveMissingImports(ctx) 1145 if err != nil { 1146 ld.error(err) 1147 break 1148 } 1149 if len(modAddedBy) == 0 { 1150 // The roots are stable, and we've resolved all of the missing packages 1151 // that we can. 1152 break 1153 } 1154 1155 toAdd := make([]module.Version, 0, len(modAddedBy)) 1156 for m := range modAddedBy { 1157 toAdd = append(toAdd, m) 1158 } 1159 gover.ModSort(toAdd) // to make errors deterministic 1160 1161 // We ran updateRequirements before resolving missing imports and it didn't 1162 // make any changes, so we know that the requirement graph is already 1163 // consistent with ld.pkgs: we don't need to pass ld.pkgs to updateRoots 1164 // again. (That would waste time looking for changes that we have already 1165 // applied.) 1166 var noPkgs []*loadPkg 1167 // We also know that we're going to call updateRequirements again next 1168 // iteration so we don't need to also update it here. (That would waste time 1169 // computing a "direct" map that we'll have to recompute later anyway.) 1170 direct := ld.requirements.direct 1171 rs, err := updateRoots(ctx, direct, ld.requirements, noPkgs, toAdd, ld.AssumeRootsImported) 1172 if err != nil { 1173 // If an error was found in a newly added module, report the package 1174 // import stack instead of the module requirement stack. Packages 1175 // are more descriptive. 1176 if err, ok := err.(*mvs.BuildListError); ok { 1177 if pkg := modAddedBy[err.Module()]; pkg != nil { 1178 ld.error(fmt.Errorf("%s: %w", pkg.stackText(), err.Err)) 1179 break 1180 } 1181 } 1182 ld.error(err) 1183 break 1184 } 1185 if slices.Equal(rs.rootModules, ld.requirements.rootModules) { 1186 // Something is deeply wrong. resolveMissingImports gave us a non-empty 1187 // set of modules to add to the graph, but adding those modules had no 1188 // effect — either they were already in the graph, or updateRoots did not 1189 // add them as requested. 1190 panic(fmt.Sprintf("internal error: adding %v to module graph had no effect on root requirements (%v)", toAdd, rs.rootModules)) 1191 } 1192 ld.requirements = rs 1193 } 1194 ld.exitIfErrors(ctx) 1195 1196 // Tidy the build list, if applicable, before we report errors. 1197 // (The process of tidying may remove errors from irrelevant dependencies.) 1198 if ld.Tidy { 1199 rs, err := tidyRoots(ctx, ld.requirements, ld.pkgs) 1200 if err != nil { 1201 ld.error(err) 1202 } else { 1203 if ld.TidyGoVersion != "" { 1204 // Attempt to switch to the requested Go version. We have been using its 1205 // pruning and semantics all along, but there may have been — and may 1206 // still be — requirements on higher versions in the graph. 1207 tidy := overrideRoots(ctx, rs, []module.Version{{Path: "go", Version: ld.TidyGoVersion}}) 1208 mg, err := tidy.Graph(ctx) 1209 if err != nil { 1210 ld.error(err) 1211 } 1212 if v := mg.Selected("go"); v == ld.TidyGoVersion { 1213 rs = tidy 1214 } else { 1215 conflict := Conflict{ 1216 Path: mg.g.FindPath(func(m module.Version) bool { 1217 return m.Path == "go" && m.Version == v 1218 })[1:], 1219 Constraint: module.Version{Path: "go", Version: ld.TidyGoVersion}, 1220 } 1221 msg := conflict.Summary() 1222 if cfg.BuildV { 1223 msg = conflict.String() 1224 } 1225 ld.error(errors.New(msg)) 1226 } 1227 } 1228 1229 if ld.requirements.pruning == pruned { 1230 // We continuously add tidy roots to ld.requirements during loading, so 1231 // at this point the tidy roots (other than possibly the "go" version 1232 // edited above) should be a subset of the roots of ld.requirements, 1233 // ensuring that no new dependencies are brought inside the 1234 // graph-pruning horizon. 1235 // If that is not the case, there is a bug in the loading loop above. 1236 for _, m := range rs.rootModules { 1237 if m.Path == "go" && ld.TidyGoVersion != "" { 1238 continue 1239 } 1240 if v, ok := ld.requirements.rootSelected(m.Path); !ok || v != m.Version { 1241 ld.error(fmt.Errorf("internal error: a requirement on %v is needed but was not added during package loading (selected %s)", m, v)) 1242 } 1243 } 1244 } 1245 1246 ld.requirements = rs 1247 } 1248 1249 ld.exitIfErrors(ctx) 1250 } 1251 1252 // Report errors, if any. 1253 for _, pkg := range ld.pkgs { 1254 if pkg.err == nil { 1255 continue 1256 } 1257 1258 // Add importer information to checksum errors. 1259 if sumErr := (*ImportMissingSumError)(nil); errors.As(pkg.err, &sumErr) { 1260 if importer := pkg.stack; importer != nil { 1261 sumErr.importer = importer.path 1262 sumErr.importerVersion = importer.mod.Version 1263 sumErr.importerIsTest = importer.testOf != nil 1264 } 1265 } 1266 1267 if stdErr := (*ImportMissingError)(nil); errors.As(pkg.err, &stdErr) && stdErr.isStd { 1268 // Add importer go version information to import errors of standard 1269 // library packages arising from newer releases. 1270 if importer := pkg.stack; importer != nil { 1271 if v, ok := rawGoVersion.Load(importer.mod); ok && gover.Compare(gover.Local(), v.(string)) < 0 { 1272 stdErr.importerGoVersion = v.(string) 1273 } 1274 } 1275 if ld.SilenceMissingStdImports { 1276 continue 1277 } 1278 } 1279 if ld.SilencePackageErrors { 1280 continue 1281 } 1282 if ld.SilenceNoGoErrors && errors.Is(pkg.err, imports.ErrNoGo) { 1283 continue 1284 } 1285 1286 ld.error(fmt.Errorf("%s: %w", pkg.stackText(), pkg.err)) 1287 } 1288 1289 ld.checkMultiplePaths() 1290 return ld 1291 } 1292 1293 // updateRequirements ensures that ld.requirements is consistent with the 1294 // information gained from ld.pkgs. 1295 // 1296 // In particular: 1297 // 1298 // - Modules that provide packages directly imported from the main module are 1299 // marked as direct, and are promoted to explicit roots. If a needed root 1300 // cannot be promoted due to -mod=readonly or -mod=vendor, the importing 1301 // package is marked with an error. 1302 // 1303 // - If ld scanned the "all" pattern independent of build constraints, it is 1304 // guaranteed to have seen every direct import. Module dependencies that did 1305 // not provide any directly-imported package are then marked as indirect. 1306 // 1307 // - Root dependencies are updated to their selected versions. 1308 // 1309 // The "changed" return value reports whether the update changed the selected 1310 // version of any module that either provided a loaded package or may now 1311 // provide a package that was previously unresolved. 1312 func (ld *loader) updateRequirements(ctx context.Context) (changed bool, err error) { 1313 rs := ld.requirements 1314 1315 // direct contains the set of modules believed to provide packages directly 1316 // imported by the main module. 1317 var direct map[string]bool 1318 1319 // If we didn't scan all of the imports from the main module, or didn't use 1320 // imports.AnyTags, then we didn't necessarily load every package that 1321 // contributes “direct” imports — so we can't safely mark existing direct 1322 // dependencies in ld.requirements as indirect-only. Propagate them as direct. 1323 loadedDirect := ld.allPatternIsRoot && maps.Equal(ld.Tags, imports.AnyTags()) 1324 if loadedDirect { 1325 direct = make(map[string]bool) 1326 } else { 1327 // TODO(bcmills): It seems like a shame to allocate and copy a map here when 1328 // it will only rarely actually vary from rs.direct. Measure this cost and 1329 // maybe avoid the copy. 1330 direct = make(map[string]bool, len(rs.direct)) 1331 for mPath := range rs.direct { 1332 direct[mPath] = true 1333 } 1334 } 1335 1336 var maxTooNew *gover.TooNewError 1337 for _, pkg := range ld.pkgs { 1338 if pkg.err != nil { 1339 if tooNew := (*gover.TooNewError)(nil); errors.As(pkg.err, &tooNew) { 1340 if maxTooNew == nil || gover.Compare(tooNew.GoVersion, maxTooNew.GoVersion) > 0 { 1341 maxTooNew = tooNew 1342 } 1343 } 1344 } 1345 if pkg.mod.Version != "" || !MainModules.Contains(pkg.mod.Path) { 1346 continue 1347 } 1348 1349 for _, dep := range pkg.imports { 1350 if !dep.fromExternalModule() { 1351 continue 1352 } 1353 1354 if inWorkspaceMode() { 1355 // In workspace mode / workspace pruning mode, the roots are the main modules 1356 // rather than the main module's direct dependencies. The check below on the selected 1357 // roots does not apply. 1358 if cfg.BuildMod == "vendor" { 1359 // In workspace vendor mode, we don't need to load the requirements of the workspace 1360 // modules' dependencies so the check below doesn't work. But that's okay, because 1361 // checking whether modules are required directly for the purposes of pruning is 1362 // less important in vendor mode: if we were able to load the package, we have 1363 // everything we need to build the package, and dependencies' tests are pruned out 1364 // of the vendor directory anyway. 1365 continue 1366 } 1367 if mg, err := rs.Graph(ctx); err != nil { 1368 return false, err 1369 } else if _, ok := mg.RequiredBy(dep.mod); !ok { 1370 // dep.mod is not an explicit dependency, but needs to be. 1371 // See comment on error returned below. 1372 pkg.err = &DirectImportFromImplicitDependencyError{ 1373 ImporterPath: pkg.path, 1374 ImportedPath: dep.path, 1375 Module: dep.mod, 1376 } 1377 } 1378 } else if pkg.err == nil && cfg.BuildMod != "mod" { 1379 if v, ok := rs.rootSelected(dep.mod.Path); !ok || v != dep.mod.Version { 1380 // dep.mod is not an explicit dependency, but needs to be. 1381 // Because we are not in "mod" mode, we will not be able to update it. 1382 // Instead, mark the importing package with an error. 1383 // 1384 // TODO(#41688): The resulting error message fails to include the file 1385 // position of the import statement (because that information is not 1386 // tracked by the module loader). Figure out how to plumb the import 1387 // position through. 1388 pkg.err = &DirectImportFromImplicitDependencyError{ 1389 ImporterPath: pkg.path, 1390 ImportedPath: dep.path, 1391 Module: dep.mod, 1392 } 1393 // cfg.BuildMod does not allow us to change dep.mod to be a direct 1394 // dependency, so don't mark it as such. 1395 continue 1396 } 1397 } 1398 1399 // dep is a package directly imported by a package or test in the main 1400 // module and loaded from some other module (not the standard library). 1401 // Mark its module as a direct dependency. 1402 direct[dep.mod.Path] = true 1403 } 1404 } 1405 if maxTooNew != nil { 1406 return false, maxTooNew 1407 } 1408 1409 var addRoots []module.Version 1410 if ld.Tidy { 1411 // When we are tidying a module with a pruned dependency graph, we may need 1412 // to add roots to preserve the versions of indirect, test-only dependencies 1413 // that are upgraded above or otherwise missing from the go.mod files of 1414 // direct dependencies. (For example, the direct dependency might be a very 1415 // stable codebase that predates modules and thus lacks a go.mod file, or 1416 // the author of the direct dependency may have forgotten to commit a change 1417 // to the go.mod file, or may have made an erroneous hand-edit that causes 1418 // it to be untidy.) 1419 // 1420 // Promoting an indirect dependency to a root adds the next layer of its 1421 // dependencies to the module graph, which may increase the selected 1422 // versions of other modules from which we have already loaded packages. 1423 // So after we promote an indirect dependency to a root, we need to reload 1424 // packages, which means another iteration of loading. 1425 // 1426 // As an extra wrinkle, the upgrades due to promoting a root can cause 1427 // previously-resolved packages to become unresolved. For example, the 1428 // module providing an unstable package might be upgraded to a version 1429 // that no longer contains that package. If we then resolve the missing 1430 // package, we might add yet another root that upgrades away some other 1431 // dependency. (The tests in mod_tidy_convergence*.txt illustrate some 1432 // particularly worrisome cases.) 1433 // 1434 // To ensure that this process of promoting, adding, and upgrading roots 1435 // eventually terminates, during iteration we only ever add modules to the 1436 // root set — we only remove irrelevant roots at the very end of 1437 // iteration, after we have already added every root that we plan to need 1438 // in the (eventual) tidy root set. 1439 // 1440 // Since we do not remove any roots during iteration, even if they no 1441 // longer provide any imported packages, the selected versions of the 1442 // roots can only increase and the set of roots can only expand. The set 1443 // of extant root paths is finite and the set of versions of each path is 1444 // finite, so the iteration *must* reach a stable fixed-point. 1445 tidy, err := tidyRoots(ctx, rs, ld.pkgs) 1446 if err != nil { 1447 return false, err 1448 } 1449 addRoots = tidy.rootModules 1450 } 1451 1452 rs, err = updateRoots(ctx, direct, rs, ld.pkgs, addRoots, ld.AssumeRootsImported) 1453 if err != nil { 1454 // We don't actually know what even the root requirements are supposed to be, 1455 // so we can't proceed with loading. Return the error to the caller 1456 return false, err 1457 } 1458 1459 if rs.GoVersion() != ld.requirements.GoVersion() { 1460 // A change in the selected Go version may or may not affect the set of 1461 // loaded packages, but in some cases it can change the meaning of the "all" 1462 // pattern, the level of pruning in the module graph, and even the set of 1463 // packages present in the standard library. If it has changed, it's best to 1464 // reload packages once more to be sure everything is stable. 1465 changed = true 1466 } else if rs != ld.requirements && !slices.Equal(rs.rootModules, ld.requirements.rootModules) { 1467 // The roots of the module graph have changed in some way (not just the 1468 // "direct" markings). Check whether the changes affected any of the loaded 1469 // packages. 1470 mg, err := rs.Graph(ctx) 1471 if err != nil { 1472 return false, err 1473 } 1474 for _, pkg := range ld.pkgs { 1475 if pkg.fromExternalModule() && mg.Selected(pkg.mod.Path) != pkg.mod.Version { 1476 changed = true 1477 break 1478 } 1479 if pkg.err != nil { 1480 // Promoting a module to a root may resolve an import that was 1481 // previously missing (by pulling in a previously-prune dependency that 1482 // provides it) or ambiguous (by promoting exactly one of the 1483 // alternatives to a root and ignoring the second-level alternatives) or 1484 // otherwise errored out (by upgrading from a version that cannot be 1485 // fetched to one that can be). 1486 // 1487 // Instead of enumerating all of the possible errors, we'll just check 1488 // whether importFromModules returns nil for the package. 1489 // False-positives are ok: if we have a false-positive here, we'll do an 1490 // extra iteration of package loading this time, but we'll still 1491 // converge when the root set stops changing. 1492 // 1493 // In some sense, we can think of this as ‘upgraded the module providing 1494 // pkg.path from "none" to a version higher than "none"’. 1495 if _, _, _, _, err = importFromModules(ctx, pkg.path, rs, nil, ld.skipImportModFiles); err == nil { 1496 changed = true 1497 break 1498 } 1499 } 1500 } 1501 } 1502 1503 ld.requirements = rs 1504 return changed, nil 1505 } 1506 1507 // resolveMissingImports returns a set of modules that could be added as 1508 // dependencies in order to resolve missing packages from pkgs. 1509 // 1510 // The newly-resolved packages are added to the addedModuleFor map, and 1511 // resolveMissingImports returns a map from each new module version to 1512 // the first missing package that module would resolve. 1513 func (ld *loader) resolveMissingImports(ctx context.Context) (modAddedBy map[module.Version]*loadPkg, err error) { 1514 type pkgMod struct { 1515 pkg *loadPkg 1516 mod *module.Version 1517 } 1518 var pkgMods []pkgMod 1519 for _, pkg := range ld.pkgs { 1520 if pkg.err == nil { 1521 continue 1522 } 1523 if pkg.isTest() { 1524 // If we are missing a test, we are also missing its non-test version, and 1525 // we should only add the missing import once. 1526 continue 1527 } 1528 if !errors.As(pkg.err, new(*ImportMissingError)) { 1529 // Leave other errors for Import or load.Packages to report. 1530 continue 1531 } 1532 1533 pkg := pkg 1534 var mod module.Version 1535 ld.work.Add(func() { 1536 var err error 1537 mod, err = queryImport(ctx, pkg.path, ld.requirements) 1538 if err != nil { 1539 var ime *ImportMissingError 1540 if errors.As(err, &ime) { 1541 for curstack := pkg.stack; curstack != nil; curstack = curstack.stack { 1542 if MainModules.Contains(curstack.mod.Path) { 1543 ime.ImportingMainModule = curstack.mod 1544 break 1545 } 1546 } 1547 } 1548 // pkg.err was already non-nil, so we can reasonably attribute the error 1549 // for pkg to either the original error or the one returned by 1550 // queryImport. The existing error indicates only that we couldn't find 1551 // the package, whereas the query error also explains why we didn't fix 1552 // the problem — so we prefer the latter. 1553 pkg.err = err 1554 } 1555 1556 // err is nil, but we intentionally leave pkg.err non-nil and pkg.mod 1557 // unset: we still haven't satisfied other invariants of a 1558 // successfully-loaded package, such as scanning and loading the imports 1559 // of that package. If we succeed in resolving the new dependency graph, 1560 // the caller can reload pkg and update the error at that point. 1561 // 1562 // Even then, the package might not be loaded from the version we've 1563 // identified here. The module may be upgraded by some other dependency, 1564 // or by a transitive dependency of mod itself, or — less likely — the 1565 // package may be rejected by an AllowPackage hook or rendered ambiguous 1566 // by some other newly-added or newly-upgraded dependency. 1567 }) 1568 1569 pkgMods = append(pkgMods, pkgMod{pkg: pkg, mod: &mod}) 1570 } 1571 <-ld.work.Idle() 1572 1573 modAddedBy = map[module.Version]*loadPkg{} 1574 1575 var ( 1576 maxTooNew *gover.TooNewError 1577 maxTooNewPkg *loadPkg 1578 ) 1579 for _, pm := range pkgMods { 1580 if tooNew := (*gover.TooNewError)(nil); errors.As(pm.pkg.err, &tooNew) { 1581 if maxTooNew == nil || gover.Compare(tooNew.GoVersion, maxTooNew.GoVersion) > 0 { 1582 maxTooNew = tooNew 1583 maxTooNewPkg = pm.pkg 1584 } 1585 } 1586 } 1587 if maxTooNew != nil { 1588 fmt.Fprintf(os.Stderr, "go: toolchain upgrade needed to resolve %s\n", maxTooNewPkg.path) 1589 return nil, maxTooNew 1590 } 1591 1592 for _, pm := range pkgMods { 1593 pkg, mod := pm.pkg, *pm.mod 1594 if mod.Path == "" { 1595 continue 1596 } 1597 1598 fmt.Fprintf(os.Stderr, "go: found %s in %s %s\n", pkg.path, mod.Path, mod.Version) 1599 if modAddedBy[mod] == nil { 1600 modAddedBy[mod] = pkg 1601 } 1602 } 1603 1604 return modAddedBy, nil 1605 } 1606 1607 // pkg locates the *loadPkg for path, creating and queuing it for loading if 1608 // needed, and updates its state to reflect the given flags. 1609 // 1610 // The imports of the returned *loadPkg will be loaded asynchronously in the 1611 // ld.work queue, and its test (if requested) will also be populated once 1612 // imports have been resolved. When ld.work goes idle, all transitive imports of 1613 // the requested package (and its test, if requested) will have been loaded. 1614 func (ld *loader) pkg(ctx context.Context, path string, flags loadPkgFlags) *loadPkg { 1615 if flags.has(pkgImportsLoaded) { 1616 panic("internal error: (*loader).pkg called with pkgImportsLoaded flag set") 1617 } 1618 1619 pkg := ld.pkgCache.Do(path, func() *loadPkg { 1620 pkg := &loadPkg{ 1621 path: path, 1622 } 1623 ld.applyPkgFlags(ctx, pkg, flags) 1624 1625 ld.work.Add(func() { ld.load(ctx, pkg) }) 1626 return pkg 1627 }) 1628 1629 ld.applyPkgFlags(ctx, pkg, flags) 1630 return pkg 1631 } 1632 1633 // applyPkgFlags updates pkg.flags to set the given flags and propagate the 1634 // (transitive) effects of those flags, possibly loading or enqueueing further 1635 // packages as a result. 1636 func (ld *loader) applyPkgFlags(ctx context.Context, pkg *loadPkg, flags loadPkgFlags) { 1637 if flags == 0 { 1638 return 1639 } 1640 1641 if flags.has(pkgInAll) && ld.allPatternIsRoot && !pkg.isTest() { 1642 // This package matches a root pattern by virtue of being in "all". 1643 flags |= pkgIsRoot 1644 } 1645 if flags.has(pkgIsRoot) { 1646 flags |= pkgFromRoot 1647 } 1648 1649 old := pkg.flags.update(flags) 1650 new := old | flags 1651 if new == old || !new.has(pkgImportsLoaded) { 1652 // We either didn't change the state of pkg, or we don't know anything about 1653 // its dependencies yet. Either way, we can't usefully load its test or 1654 // update its dependencies. 1655 return 1656 } 1657 1658 if !pkg.isTest() { 1659 // Check whether we should add (or update the flags for) a test for pkg. 1660 // ld.pkgTest is idempotent and extra invocations are inexpensive, 1661 // so it's ok if we call it more than is strictly necessary. 1662 wantTest := false 1663 switch { 1664 case ld.allPatternIsRoot && MainModules.Contains(pkg.mod.Path): 1665 // We are loading the "all" pattern, which includes packages imported by 1666 // tests in the main module. This package is in the main module, so we 1667 // need to identify the imports of its test even if LoadTests is not set. 1668 // 1669 // (We will filter out the extra tests explicitly in computePatternAll.) 1670 wantTest = true 1671 1672 case ld.allPatternIsRoot && ld.allClosesOverTests && new.has(pkgInAll): 1673 // This variant of the "all" pattern includes imports of tests of every 1674 // package that is itself in "all", and pkg is in "all", so its test is 1675 // also in "all" (as above). 1676 wantTest = true 1677 1678 case ld.LoadTests && new.has(pkgIsRoot): 1679 // LoadTest explicitly requests tests of “the root packages”. 1680 wantTest = true 1681 } 1682 1683 if wantTest { 1684 var testFlags loadPkgFlags 1685 if MainModules.Contains(pkg.mod.Path) || (ld.allClosesOverTests && new.has(pkgInAll)) { 1686 // Tests of packages in the main module are in "all", in the sense that 1687 // they cause the packages they import to also be in "all". So are tests 1688 // of packages in "all" if "all" closes over test dependencies. 1689 testFlags |= pkgInAll 1690 } 1691 ld.pkgTest(ctx, pkg, testFlags) 1692 } 1693 } 1694 1695 if new.has(pkgInAll) && !old.has(pkgInAll|pkgImportsLoaded) { 1696 // We have just marked pkg with pkgInAll, or we have just loaded its 1697 // imports, or both. Now is the time to propagate pkgInAll to the imports. 1698 for _, dep := range pkg.imports { 1699 ld.applyPkgFlags(ctx, dep, pkgInAll) 1700 } 1701 } 1702 1703 if new.has(pkgFromRoot) && !old.has(pkgFromRoot|pkgImportsLoaded) { 1704 for _, dep := range pkg.imports { 1705 ld.applyPkgFlags(ctx, dep, pkgFromRoot) 1706 } 1707 } 1708 } 1709 1710 // preloadRootModules loads the module requirements needed to identify the 1711 // selected version of each module providing a package in rootPkgs, 1712 // adding new root modules to the module graph if needed. 1713 func (ld *loader) preloadRootModules(ctx context.Context, rootPkgs []string) (changedBuildList bool) { 1714 needc := make(chan map[module.Version]bool, 1) 1715 needc <- map[module.Version]bool{} 1716 for _, path := range rootPkgs { 1717 path := path 1718 ld.work.Add(func() { 1719 // First, try to identify the module containing the package using only roots. 1720 // 1721 // If the main module is tidy and the package is in "all" — or if we're 1722 // lucky — we can identify all of its imports without actually loading the 1723 // full module graph. 1724 m, _, _, _, err := importFromModules(ctx, path, ld.requirements, nil, ld.skipImportModFiles) 1725 if err != nil { 1726 var missing *ImportMissingError 1727 if errors.As(err, &missing) && ld.ResolveMissingImports { 1728 // This package isn't provided by any selected module. 1729 // If we can find it, it will be a new root dependency. 1730 m, err = queryImport(ctx, path, ld.requirements) 1731 } 1732 if err != nil { 1733 // We couldn't identify the root module containing this package. 1734 // Leave it unresolved; we will report it during loading. 1735 return 1736 } 1737 } 1738 if m.Path == "" { 1739 // The package is in std or cmd. We don't need to change the root set. 1740 return 1741 } 1742 1743 v, ok := ld.requirements.rootSelected(m.Path) 1744 if !ok || v != m.Version { 1745 // We found the requested package in m, but m is not a root, so 1746 // loadModGraph will not load its requirements. We need to promote the 1747 // module to a root to ensure that any other packages this package 1748 // imports are resolved from correct dependency versions. 1749 // 1750 // (This is the “argument invariant” from 1751 // https://golang.org/design/36460-lazy-module-loading.) 1752 need := <-needc 1753 need[m] = true 1754 needc <- need 1755 } 1756 }) 1757 } 1758 <-ld.work.Idle() 1759 1760 need := <-needc 1761 if len(need) == 0 { 1762 return false // No roots to add. 1763 } 1764 1765 toAdd := make([]module.Version, 0, len(need)) 1766 for m := range need { 1767 toAdd = append(toAdd, m) 1768 } 1769 gover.ModSort(toAdd) 1770 1771 rs, err := updateRoots(ctx, ld.requirements.direct, ld.requirements, nil, toAdd, ld.AssumeRootsImported) 1772 if err != nil { 1773 // We are missing some root dependency, and for some reason we can't load 1774 // enough of the module dependency graph to add the missing root. Package 1775 // loading is doomed to fail, so fail quickly. 1776 ld.error(err) 1777 ld.exitIfErrors(ctx) 1778 return false 1779 } 1780 if slices.Equal(rs.rootModules, ld.requirements.rootModules) { 1781 // Something is deeply wrong. resolveMissingImports gave us a non-empty 1782 // set of modules to add to the graph, but adding those modules had no 1783 // effect — either they were already in the graph, or updateRoots did not 1784 // add them as requested. 1785 panic(fmt.Sprintf("internal error: adding %v to module graph had no effect on root requirements (%v)", toAdd, rs.rootModules)) 1786 } 1787 1788 ld.requirements = rs 1789 return true 1790 } 1791 1792 // load loads an individual package. 1793 func (ld *loader) load(ctx context.Context, pkg *loadPkg) { 1794 var mg *ModuleGraph 1795 if ld.requirements.pruning == unpruned { 1796 var err error 1797 mg, err = ld.requirements.Graph(ctx) 1798 if err != nil { 1799 // We already checked the error from Graph in loadFromRoots and/or 1800 // updateRequirements, so we ignored the error on purpose and we should 1801 // keep trying to push past it. 1802 // 1803 // However, because mg may be incomplete (and thus may select inaccurate 1804 // versions), we shouldn't use it to load packages. Instead, we pass a nil 1805 // *ModuleGraph, which will cause mg to first try loading from only the 1806 // main module and root dependencies. 1807 mg = nil 1808 } 1809 } 1810 1811 var modroot string 1812 pkg.mod, modroot, pkg.dir, pkg.altMods, pkg.err = importFromModules(ctx, pkg.path, ld.requirements, mg, ld.skipImportModFiles) 1813 if pkg.dir == "" { 1814 return 1815 } 1816 if MainModules.Contains(pkg.mod.Path) { 1817 // Go ahead and mark pkg as in "all". This provides the invariant that a 1818 // package that is *only* imported by other packages in "all" is always 1819 // marked as such before loading its imports. 1820 // 1821 // We don't actually rely on that invariant at the moment, but it may 1822 // improve efficiency somewhat and makes the behavior a bit easier to reason 1823 // about (by reducing churn on the flag bits of dependencies), and costs 1824 // essentially nothing (these atomic flag ops are essentially free compared 1825 // to scanning source code for imports). 1826 ld.applyPkgFlags(ctx, pkg, pkgInAll) 1827 } 1828 if ld.AllowPackage != nil { 1829 if err := ld.AllowPackage(ctx, pkg.path, pkg.mod); err != nil { 1830 pkg.err = err 1831 } 1832 } 1833 1834 pkg.inStd = (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "") 1835 1836 var imports, testImports []string 1837 1838 if cfg.BuildContext.Compiler == "gccgo" && pkg.inStd { 1839 // We can't scan standard packages for gccgo. 1840 } else { 1841 var err error 1842 imports, testImports, err = scanDir(modroot, pkg.dir, ld.Tags) 1843 if err != nil { 1844 pkg.err = err 1845 return 1846 } 1847 } 1848 1849 pkg.imports = make([]*loadPkg, 0, len(imports)) 1850 var importFlags loadPkgFlags 1851 if pkg.flags.has(pkgInAll) { 1852 importFlags = pkgInAll 1853 } 1854 for _, path := range imports { 1855 if pkg.inStd { 1856 // Imports from packages in "std" and "cmd" should resolve using 1857 // GOROOT/src/vendor even when "std" is not the main module. 1858 path = ld.stdVendor(pkg.path, path) 1859 } 1860 pkg.imports = append(pkg.imports, ld.pkg(ctx, path, importFlags)) 1861 } 1862 pkg.testImports = testImports 1863 1864 ld.applyPkgFlags(ctx, pkg, pkgImportsLoaded) 1865 } 1866 1867 // pkgTest locates the test of pkg, creating it if needed, and updates its state 1868 // to reflect the given flags. 1869 // 1870 // pkgTest requires that the imports of pkg have already been loaded (flagged 1871 // with pkgImportsLoaded). 1872 func (ld *loader) pkgTest(ctx context.Context, pkg *loadPkg, testFlags loadPkgFlags) *loadPkg { 1873 if pkg.isTest() { 1874 panic("pkgTest called on a test package") 1875 } 1876 1877 createdTest := false 1878 pkg.testOnce.Do(func() { 1879 pkg.test = &loadPkg{ 1880 path: pkg.path, 1881 testOf: pkg, 1882 mod: pkg.mod, 1883 dir: pkg.dir, 1884 err: pkg.err, 1885 inStd: pkg.inStd, 1886 } 1887 ld.applyPkgFlags(ctx, pkg.test, testFlags) 1888 createdTest = true 1889 }) 1890 1891 test := pkg.test 1892 if createdTest { 1893 test.imports = make([]*loadPkg, 0, len(pkg.testImports)) 1894 var importFlags loadPkgFlags 1895 if test.flags.has(pkgInAll) { 1896 importFlags = pkgInAll 1897 } 1898 for _, path := range pkg.testImports { 1899 if pkg.inStd { 1900 path = ld.stdVendor(test.path, path) 1901 } 1902 test.imports = append(test.imports, ld.pkg(ctx, path, importFlags)) 1903 } 1904 pkg.testImports = nil 1905 ld.applyPkgFlags(ctx, test, pkgImportsLoaded) 1906 } else { 1907 ld.applyPkgFlags(ctx, test, testFlags) 1908 } 1909 1910 return test 1911 } 1912 1913 // stdVendor returns the canonical import path for the package with the given 1914 // path when imported from the standard-library package at parentPath. 1915 func (ld *loader) stdVendor(parentPath, path string) string { 1916 if search.IsStandardImportPath(path) { 1917 return path 1918 } 1919 1920 if str.HasPathPrefix(parentPath, "cmd") { 1921 if !ld.VendorModulesInGOROOTSrc || !MainModules.Contains("cmd") { 1922 vendorPath := pathpkg.Join("cmd", "vendor", path) 1923 1924 if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { 1925 return vendorPath 1926 } 1927 } 1928 } else if !ld.VendorModulesInGOROOTSrc || !MainModules.Contains("std") || str.HasPathPrefix(parentPath, "vendor") { 1929 // If we are outside of the 'std' module, resolve imports from within 'std' 1930 // to the vendor directory. 1931 // 1932 // Do the same for importers beginning with the prefix 'vendor/' even if we 1933 // are *inside* of the 'std' module: the 'vendor/' packages that resolve 1934 // globally from GOROOT/src/vendor (and are listed as part of 'go list std') 1935 // are distinct from the real module dependencies, and cannot import 1936 // internal packages from the real module. 1937 // 1938 // (Note that although the 'vendor/' packages match the 'std' *package* 1939 // pattern, they are not part of the std *module*, and do not affect 1940 // 'go mod tidy' and similar module commands when working within std.) 1941 vendorPath := pathpkg.Join("vendor", path) 1942 if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { 1943 return vendorPath 1944 } 1945 } 1946 1947 // Not vendored: resolve from modules. 1948 return path 1949 } 1950 1951 // computePatternAll returns the list of packages matching pattern "all", 1952 // starting with a list of the import paths for the packages in the main module. 1953 func (ld *loader) computePatternAll() (all []string) { 1954 for _, pkg := range ld.pkgs { 1955 if pkg.flags.has(pkgInAll) && !pkg.isTest() { 1956 all = append(all, pkg.path) 1957 } 1958 } 1959 sort.Strings(all) 1960 return all 1961 } 1962 1963 // checkMultiplePaths verifies that a given module path is used as itself 1964 // or as a replacement for another module, but not both at the same time. 1965 // 1966 // (See https://golang.org/issue/26607 and https://golang.org/issue/34650.) 1967 func (ld *loader) checkMultiplePaths() { 1968 mods := ld.requirements.rootModules 1969 if cached := ld.requirements.graph.Load(); cached != nil { 1970 if mg := cached.mg; mg != nil { 1971 mods = mg.BuildList() 1972 } 1973 } 1974 1975 firstPath := map[module.Version]string{} 1976 for _, mod := range mods { 1977 src := resolveReplacement(mod) 1978 if prev, ok := firstPath[src]; !ok { 1979 firstPath[src] = mod.Path 1980 } else if prev != mod.Path { 1981 ld.error(fmt.Errorf("%s@%s used for two different module paths (%s and %s)", src.Path, src.Version, prev, mod.Path)) 1982 } 1983 } 1984 } 1985 1986 // checkTidyCompatibility emits an error if any package would be loaded from a 1987 // different module under rs than under ld.requirements. 1988 func (ld *loader) checkTidyCompatibility(ctx context.Context, rs *Requirements, compatVersion string) { 1989 goVersion := rs.GoVersion() 1990 suggestUpgrade := false 1991 suggestEFlag := false 1992 suggestFixes := func() { 1993 if ld.AllowErrors { 1994 // The user is explicitly ignoring these errors, so don't bother them with 1995 // other options. 1996 return 1997 } 1998 1999 // We print directly to os.Stderr because this information is advice about 2000 // how to fix errors, not actually an error itself. 2001 // (The actual errors should have been logged already.) 2002 2003 fmt.Fprintln(os.Stderr) 2004 2005 goFlag := "" 2006 if goVersion != MainModules.GoVersion() { 2007 goFlag = " -go=" + goVersion 2008 } 2009 2010 compatFlag := "" 2011 if compatVersion != gover.Prev(goVersion) { 2012 compatFlag = " -compat=" + compatVersion 2013 } 2014 if suggestUpgrade { 2015 eDesc := "" 2016 eFlag := "" 2017 if suggestEFlag { 2018 eDesc = ", leaving some packages unresolved" 2019 eFlag = " -e" 2020 } 2021 fmt.Fprintf(os.Stderr, "To upgrade to the versions selected by go %s%s:\n\tgo mod tidy%s -go=%s && go mod tidy%s -go=%s%s\n", compatVersion, eDesc, eFlag, compatVersion, eFlag, goVersion, compatFlag) 2022 } else if suggestEFlag { 2023 // If some packages are missing but no package is upgraded, then we 2024 // shouldn't suggest upgrading to the Go 1.16 versions explicitly — that 2025 // wouldn't actually fix anything for Go 1.16 users, and *would* break 2026 // something for Go 1.17 users. 2027 fmt.Fprintf(os.Stderr, "To proceed despite packages unresolved in go %s:\n\tgo mod tidy -e%s%s\n", compatVersion, goFlag, compatFlag) 2028 } 2029 2030 fmt.Fprintf(os.Stderr, "If reproducibility with go %s is not needed:\n\tgo mod tidy%s -compat=%s\n", compatVersion, goFlag, goVersion) 2031 2032 // TODO(#46141): Populate the linked wiki page. 2033 fmt.Fprintf(os.Stderr, "For other options, see:\n\thttps://golang.org/doc/modules/pruning\n") 2034 } 2035 2036 mg, err := rs.Graph(ctx) 2037 if err != nil { 2038 ld.error(fmt.Errorf("error loading go %s module graph: %w", compatVersion, err)) 2039 ld.switchIfErrors(ctx) 2040 suggestFixes() 2041 ld.exitIfErrors(ctx) 2042 return 2043 } 2044 2045 // Re-resolve packages in parallel. 2046 // 2047 // We re-resolve each package — rather than just checking versions — to ensure 2048 // that we have fetched module source code (and, importantly, checksums for 2049 // that source code) for all modules that are necessary to ensure that imports 2050 // are unambiguous. That also produces clearer diagnostics, since we can say 2051 // exactly what happened to the package if it became ambiguous or disappeared 2052 // entirely. 2053 // 2054 // We re-resolve the packages in parallel because this process involves disk 2055 // I/O to check for package sources, and because the process of checking for 2056 // ambiguous imports may require us to download additional modules that are 2057 // otherwise pruned out in Go 1.17 — we don't want to block progress on other 2058 // packages while we wait for a single new download. 2059 type mismatch struct { 2060 mod module.Version 2061 err error 2062 } 2063 mismatchMu := make(chan map[*loadPkg]mismatch, 1) 2064 mismatchMu <- map[*loadPkg]mismatch{} 2065 for _, pkg := range ld.pkgs { 2066 if pkg.mod.Path == "" && pkg.err == nil { 2067 // This package is from the standard library (which does not vary based on 2068 // the module graph). 2069 continue 2070 } 2071 2072 pkg := pkg 2073 ld.work.Add(func() { 2074 mod, _, _, _, err := importFromModules(ctx, pkg.path, rs, mg, ld.skipImportModFiles) 2075 if mod != pkg.mod { 2076 mismatches := <-mismatchMu 2077 mismatches[pkg] = mismatch{mod: mod, err: err} 2078 mismatchMu <- mismatches 2079 } 2080 }) 2081 } 2082 <-ld.work.Idle() 2083 2084 mismatches := <-mismatchMu 2085 if len(mismatches) == 0 { 2086 // Since we're running as part of 'go mod tidy', the roots of the module 2087 // graph should contain only modules that are relevant to some package in 2088 // the package graph. We checked every package in the package graph and 2089 // didn't find any mismatches, so that must mean that all of the roots of 2090 // the module graph are also consistent. 2091 // 2092 // If we're wrong, Go 1.16 in -mod=readonly mode will error out with 2093 // "updates to go.mod needed", which would be very confusing. So instead, 2094 // we'll double-check that our reasoning above actually holds — if it 2095 // doesn't, we'll emit an internal error and hopefully the user will report 2096 // it as a bug. 2097 for _, m := range ld.requirements.rootModules { 2098 if v := mg.Selected(m.Path); v != m.Version { 2099 fmt.Fprintln(os.Stderr) 2100 base.Fatalf("go: internal error: failed to diagnose selected-version mismatch for module %s: go %s selects %s, but go %s selects %s\n\tPlease report this at https://golang.org/issue.", m.Path, goVersion, m.Version, compatVersion, v) 2101 } 2102 } 2103 return 2104 } 2105 2106 // Iterate over the packages (instead of the mismatches map) to emit errors in 2107 // deterministic order. 2108 for _, pkg := range ld.pkgs { 2109 mismatch, ok := mismatches[pkg] 2110 if !ok { 2111 continue 2112 } 2113 2114 if pkg.isTest() { 2115 // We already did (or will) report an error for the package itself, 2116 // so don't report a duplicate (and more verbose) error for its test. 2117 if _, ok := mismatches[pkg.testOf]; !ok { 2118 base.Fatalf("go: internal error: mismatch recorded for test %s, but not its non-test package", pkg.path) 2119 } 2120 continue 2121 } 2122 2123 switch { 2124 case mismatch.err != nil: 2125 // pkg resolved successfully, but errors out using the requirements in rs. 2126 // 2127 // This could occur because the import is provided by a single root (and 2128 // is thus unambiguous in a main module with a pruned module graph) and 2129 // also one or more transitive dependencies (and is ambiguous with an 2130 // unpruned graph). 2131 // 2132 // It could also occur because some transitive dependency upgrades the 2133 // module that previously provided the package to a version that no 2134 // longer does, or to a version for which the module source code (but 2135 // not the go.mod file in isolation) has a checksum error. 2136 if missing := (*ImportMissingError)(nil); errors.As(mismatch.err, &missing) { 2137 selected := module.Version{ 2138 Path: pkg.mod.Path, 2139 Version: mg.Selected(pkg.mod.Path), 2140 } 2141 ld.error(fmt.Errorf("%s loaded from %v,\n\tbut go %s would fail to locate it in %s", pkg.stackText(), pkg.mod, compatVersion, selected)) 2142 } else { 2143 if ambiguous := (*AmbiguousImportError)(nil); errors.As(mismatch.err, &ambiguous) { 2144 // TODO: Is this check needed? 2145 } 2146 ld.error(fmt.Errorf("%s loaded from %v,\n\tbut go %s would fail to locate it:\n\t%v", pkg.stackText(), pkg.mod, compatVersion, mismatch.err)) 2147 } 2148 2149 suggestEFlag = true 2150 2151 // Even if we press ahead with the '-e' flag, the older version will 2152 // error out in readonly mode if it thinks the go.mod file contains 2153 // any *explicit* dependency that is not at its selected version, 2154 // even if that dependency is not relevant to any package being loaded. 2155 // 2156 // We check for that condition here. If all of the roots are consistent 2157 // the '-e' flag suffices, but otherwise we need to suggest an upgrade. 2158 if !suggestUpgrade { 2159 for _, m := range ld.requirements.rootModules { 2160 if v := mg.Selected(m.Path); v != m.Version { 2161 suggestUpgrade = true 2162 break 2163 } 2164 } 2165 } 2166 2167 case pkg.err != nil: 2168 // pkg had an error in with a pruned module graph (presumably suppressed 2169 // with the -e flag), but the error went away using an unpruned graph. 2170 // 2171 // This is possible, if, say, the import is unresolved in the pruned graph 2172 // (because the "latest" version of each candidate module either is 2173 // unavailable or does not contain the package), but is resolved in the 2174 // unpruned graph due to a newer-than-latest dependency that is normally 2175 // pruned out. 2176 // 2177 // This could also occur if the source code for the module providing the 2178 // package in the pruned graph has a checksum error, but the unpruned 2179 // graph upgrades that module to a version with a correct checksum. 2180 // 2181 // pkg.err should have already been logged elsewhere — along with a 2182 // stack trace — so log only the import path and non-error info here. 2183 suggestUpgrade = true 2184 ld.error(fmt.Errorf("%s failed to load from any module,\n\tbut go %s would load it from %v", pkg.path, compatVersion, mismatch.mod)) 2185 2186 case pkg.mod != mismatch.mod: 2187 // The package is loaded successfully by both Go versions, but from a 2188 // different module in each. This could lead to subtle (and perhaps even 2189 // unnoticed!) variations in behavior between builds with different 2190 // toolchains. 2191 suggestUpgrade = true 2192 ld.error(fmt.Errorf("%s loaded from %v,\n\tbut go %s would select %v\n", pkg.stackText(), pkg.mod, compatVersion, mismatch.mod.Version)) 2193 2194 default: 2195 base.Fatalf("go: internal error: mismatch recorded for package %s, but no differences found", pkg.path) 2196 } 2197 } 2198 2199 ld.switchIfErrors(ctx) 2200 suggestFixes() 2201 ld.exitIfErrors(ctx) 2202 } 2203 2204 // scanDir is like imports.ScanDir but elides known magic imports from the list, 2205 // so that we do not go looking for packages that don't really exist. 2206 // 2207 // The standard magic import is "C", for cgo. 2208 // 2209 // The only other known magic imports are appengine and appengine/*. 2210 // These are so old that they predate "go get" and did not use URL-like paths. 2211 // Most code today now uses google.golang.org/appengine instead, 2212 // but not all code has been so updated. When we mostly ignore build tags 2213 // during "go vendor", we look into "// +build appengine" files and 2214 // may see these legacy imports. We drop them so that the module 2215 // search does not look for modules to try to satisfy them. 2216 func scanDir(modroot string, dir string, tags map[string]bool) (imports_, testImports []string, err error) { 2217 if ip, mierr := modindex.GetPackage(modroot, dir); mierr == nil { 2218 imports_, testImports, err = ip.ScanDir(tags) 2219 goto Happy 2220 } else if !errors.Is(mierr, modindex.ErrNotIndexed) { 2221 return nil, nil, mierr 2222 } 2223 2224 imports_, testImports, err = imports.ScanDir(dir, tags) 2225 Happy: 2226 2227 filter := func(x []string) []string { 2228 w := 0 2229 for _, pkg := range x { 2230 if pkg != "C" && pkg != "appengine" && !strings.HasPrefix(pkg, "appengine/") && 2231 pkg != "appengine_internal" && !strings.HasPrefix(pkg, "appengine_internal/") { 2232 x[w] = pkg 2233 w++ 2234 } 2235 } 2236 return x[:w] 2237 } 2238 2239 return filter(imports_), filter(testImports), err 2240 } 2241 2242 // buildStacks computes minimal import stacks for each package, 2243 // for use in error messages. When it completes, packages that 2244 // are part of the original root set have pkg.stack == nil, 2245 // and other packages have pkg.stack pointing at the next 2246 // package up the import stack in their minimal chain. 2247 // As a side effect, buildStacks also constructs ld.pkgs, 2248 // the list of all packages loaded. 2249 func (ld *loader) buildStacks() { 2250 if len(ld.pkgs) > 0 { 2251 panic("buildStacks") 2252 } 2253 for _, pkg := range ld.roots { 2254 pkg.stack = pkg // sentinel to avoid processing in next loop 2255 ld.pkgs = append(ld.pkgs, pkg) 2256 } 2257 for i := 0; i < len(ld.pkgs); i++ { // not range: appending to ld.pkgs in loop 2258 pkg := ld.pkgs[i] 2259 for _, next := range pkg.imports { 2260 if next.stack == nil { 2261 next.stack = pkg 2262 ld.pkgs = append(ld.pkgs, next) 2263 } 2264 } 2265 if next := pkg.test; next != nil && next.stack == nil { 2266 next.stack = pkg 2267 ld.pkgs = append(ld.pkgs, next) 2268 } 2269 } 2270 for _, pkg := range ld.roots { 2271 pkg.stack = nil 2272 } 2273 } 2274 2275 // stackText builds the import stack text to use when 2276 // reporting an error in pkg. It has the general form 2277 // 2278 // root imports 2279 // other imports 2280 // other2 tested by 2281 // other2.test imports 2282 // pkg 2283 func (pkg *loadPkg) stackText() string { 2284 var stack []*loadPkg 2285 for p := pkg; p != nil; p = p.stack { 2286 stack = append(stack, p) 2287 } 2288 2289 var buf strings.Builder 2290 for i := len(stack) - 1; i >= 0; i-- { 2291 p := stack[i] 2292 fmt.Fprint(&buf, p.path) 2293 if p.testOf != nil { 2294 fmt.Fprint(&buf, ".test") 2295 } 2296 if i > 0 { 2297 if stack[i-1].testOf == p { 2298 fmt.Fprint(&buf, " tested by\n\t") 2299 } else { 2300 fmt.Fprint(&buf, " imports\n\t") 2301 } 2302 } 2303 } 2304 return buf.String() 2305 } 2306 2307 // why returns the text to use in "go mod why" output about the given package. 2308 // It is less ornate than the stackText but contains the same information. 2309 func (pkg *loadPkg) why() string { 2310 var buf strings.Builder 2311 var stack []*loadPkg 2312 for p := pkg; p != nil; p = p.stack { 2313 stack = append(stack, p) 2314 } 2315 2316 for i := len(stack) - 1; i >= 0; i-- { 2317 p := stack[i] 2318 if p.testOf != nil { 2319 fmt.Fprintf(&buf, "%s.test\n", p.testOf.path) 2320 } else { 2321 fmt.Fprintf(&buf, "%s\n", p.path) 2322 } 2323 } 2324 return buf.String() 2325 } 2326 2327 // Why returns the "go mod why" output stanza for the given package, 2328 // without the leading # comment. 2329 // The package graph must have been loaded already, usually by LoadPackages. 2330 // If there is no reason for the package to be in the current build, 2331 // Why returns an empty string. 2332 func Why(path string) string { 2333 pkg, ok := loaded.pkgCache.Get(path) 2334 if !ok { 2335 return "" 2336 } 2337 return pkg.why() 2338 } 2339 2340 // WhyDepth returns the number of steps in the Why listing. 2341 // If there is no reason for the package to be in the current build, 2342 // WhyDepth returns 0. 2343 func WhyDepth(path string) int { 2344 n := 0 2345 pkg, _ := loaded.pkgCache.Get(path) 2346 for p := pkg; p != nil; p = p.stack { 2347 n++ 2348 } 2349 return n 2350 } 2351