script_test.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Script-driven tests.
     6  // See testdata/script/README for an overview.
     7  
     8  //go:generate go test cmd/go -v -run=TestScript/README --fixreadme
     9  
    10  package main_test
    11  
    12  import (
    13  	"bufio"
    14  	"bytes"
    15  	"context"
    16  	_ "embed"
    17  	"flag"
    18  	"internal/testenv"
    19  	"internal/txtar"
    20  	"net/url"
    21  	"os"
    22  	"path/filepath"
    23  	"runtime"
    24  	"strings"
    25  	"sync"
    26  	"testing"
    27  	"time"
    28  
    29  	"cmd/go/internal/cfg"
    30  	"cmd/go/internal/gover"
    31  	"cmd/go/internal/script"
    32  	"cmd/go/internal/script/scripttest"
    33  	"cmd/go/internal/vcweb/vcstest"
    34  
    35  	"golang.org/x/telemetry/counter/countertest"
    36  )
    37  
    38  var testSum = flag.String("testsum", "", `may be tidy, listm, or listall. If set, TestScript generates a go.sum file at the beginning of each test and updates test files if they pass.`)
    39  
    40  // TestScript runs the tests in testdata/script/*.txt.
    41  func TestScript(t *testing.T) {
    42  	testenv.MustHaveGoBuild(t)
    43  	testenv.SkipIfShortAndSlow(t)
    44  
    45  	srv, err := vcstest.NewServer()
    46  	if err != nil {
    47  		t.Fatal(err)
    48  	}
    49  	t.Cleanup(func() {
    50  		if err := srv.Close(); err != nil {
    51  			t.Fatal(err)
    52  		}
    53  	})
    54  	certFile, err := srv.WriteCertificateFile()
    55  	if err != nil {
    56  		t.Fatal(err)
    57  	}
    58  
    59  	StartProxy()
    60  
    61  	var (
    62  		ctx         = context.Background()
    63  		gracePeriod = 100 * time.Millisecond
    64  	)
    65  	if deadline, ok := t.Deadline(); ok {
    66  		timeout := time.Until(deadline)
    67  
    68  		// If time allows, increase the termination grace period to 5% of the
    69  		// remaining time.
    70  		if gp := timeout / 20; gp > gracePeriod {
    71  			gracePeriod = gp
    72  		}
    73  
    74  		// When we run commands that execute subprocesses, we want to reserve two
    75  		// grace periods to clean up. We will send the first termination signal when
    76  		// the context expires, then wait one grace period for the process to
    77  		// produce whatever useful output it can (such as a stack trace). After the
    78  		// first grace period expires, we'll escalate to os.Kill, leaving the second
    79  		// grace period for the test function to record its output before the test
    80  		// process itself terminates.
    81  		timeout -= 2 * gracePeriod
    82  
    83  		var cancel context.CancelFunc
    84  		ctx, cancel = context.WithTimeout(ctx, timeout)
    85  		t.Cleanup(cancel)
    86  	}
    87  
    88  	env, err := scriptEnv(srv, certFile)
    89  	if err != nil {
    90  		t.Fatal(err)
    91  	}
    92  	engine := &script.Engine{
    93  		Conds: scriptConditions(),
    94  		Cmds:  scriptCommands(quitSignal(), gracePeriod),
    95  		Quiet: !testing.Verbose(),
    96  	}
    97  
    98  	t.Run("README", func(t *testing.T) {
    99  		checkScriptReadme(t, engine, env)
   100  	})
   101  
   102  	files, err := filepath.Glob("testdata/script/*.txt")
   103  	if err != nil {
   104  		t.Fatal(err)
   105  	}
   106  	for _, file := range files {
   107  		file := file
   108  		name := strings.TrimSuffix(filepath.Base(file), ".txt")
   109  		t.Run(name, func(t *testing.T) {
   110  			t.Parallel()
   111  			StartProxy()
   112  
   113  			workdir, err := os.MkdirTemp(testTmpDir, name)
   114  			if err != nil {
   115  				t.Fatal(err)
   116  			}
   117  			if !*testWork {
   118  				defer removeAll(workdir)
   119  			}
   120  
   121  			s, err := script.NewState(tbContext(ctx, t), workdir, env)
   122  			if err != nil {
   123  				t.Fatal(err)
   124  			}
   125  
   126  			// Unpack archive.
   127  			a, err := txtar.ParseFile(file)
   128  			if err != nil {
   129  				t.Fatal(err)
   130  			}
   131  			telemetryDir := initScriptDirs(t, s)
   132  			if err := s.ExtractFiles(a); err != nil {
   133  				t.Fatal(err)
   134  			}
   135  
   136  			t.Log(time.Now().UTC().Format(time.RFC3339))
   137  			work, _ := s.LookupEnv("WORK")
   138  			t.Logf("$WORK=%s", work)
   139  
   140  			// With -testsum, if a go.mod file is present in the test's initial
   141  			// working directory, run 'go mod tidy'.
   142  			if *testSum != "" {
   143  				if updateSum(t, engine, s, a) {
   144  					defer func() {
   145  						if t.Failed() {
   146  							return
   147  						}
   148  						data := txtar.Format(a)
   149  						if err := os.WriteFile(file, data, 0666); err != nil {
   150  							t.Errorf("rewriting test file: %v", err)
   151  						}
   152  					}()
   153  				}
   154  			}
   155  
   156  			// Note: Do not use filepath.Base(file) here:
   157  			// editors that can jump to file:line references in the output
   158  			// will work better seeing the full path relative to cmd/go
   159  			// (where the "go test" command is usually run).
   160  			scripttest.Run(t, engine, s, file, bytes.NewReader(a.Comment))
   161  			checkCounters(t, telemetryDir)
   162  		})
   163  	}
   164  }
   165  
   166  // testingTBKey is the Context key for a testing.TB.
   167  type testingTBKey struct{}
   168  
   169  // tbContext returns a Context derived from ctx and associated with t.
   170  func tbContext(ctx context.Context, t testing.TB) context.Context {
   171  	return context.WithValue(ctx, testingTBKey{}, t)
   172  }
   173  
   174  // tbFromContext returns the testing.TB associated with ctx, if any.
   175  func tbFromContext(ctx context.Context) (testing.TB, bool) {
   176  	t := ctx.Value(testingTBKey{})
   177  	if t == nil {
   178  		return nil, false
   179  	}
   180  	return t.(testing.TB), true
   181  }
   182  
   183  // initScriptDirs creates the initial directory structure in s for unpacking a
   184  // cmd/go script.
   185  func initScriptDirs(t testing.TB, s *script.State) (telemetryDir string) {
   186  	must := func(err error) {
   187  		if err != nil {
   188  			t.Helper()
   189  			t.Fatal(err)
   190  		}
   191  	}
   192  
   193  	work := s.Getwd()
   194  	must(s.Setenv("WORK", work))
   195  
   196  	telemetryDir = filepath.Join(work, "telemetry")
   197  	must(os.MkdirAll(telemetryDir, 0777))
   198  	must(s.Setenv("TEST_TELEMETRY_DIR", filepath.Join(work, "telemetry")))
   199  
   200  	must(os.MkdirAll(filepath.Join(work, "tmp"), 0777))
   201  	must(s.Setenv(tempEnvName(), filepath.Join(work, "tmp")))
   202  
   203  	gopath := filepath.Join(work, "gopath")
   204  	must(s.Setenv("GOPATH", gopath))
   205  	gopathSrc := filepath.Join(gopath, "src")
   206  	must(os.MkdirAll(gopathSrc, 0777))
   207  	must(s.Chdir(gopathSrc))
   208  	return telemetryDir
   209  }
   210  
   211  func scriptEnv(srv *vcstest.Server, srvCertFile string) ([]string, error) {
   212  	httpURL, err := url.Parse(srv.HTTP.URL)
   213  	if err != nil {
   214  		return nil, err
   215  	}
   216  	httpsURL, err := url.Parse(srv.HTTPS.URL)
   217  	if err != nil {
   218  		return nil, err
   219  	}
   220  	env := []string{
   221  		pathEnvName() + "=" + testBin + string(filepath.ListSeparator) + os.Getenv(pathEnvName()),
   222  		homeEnvName() + "=/no-home",
   223  		"CCACHE_DISABLE=1", // ccache breaks with non-existent HOME
   224  		"GOARCH=" + runtime.GOARCH,
   225  		"TESTGO_GOHOSTARCH=" + goHostArch,
   226  		"GOCACHE=" + testGOCACHE,
   227  		"GOCOVERDIR=" + os.Getenv("GOCOVERDIR"),
   228  		"GODEBUG=" + os.Getenv("GODEBUG"),
   229  		"GOEXE=" + cfg.ExeSuffix,
   230  		"GOEXPERIMENT=" + os.Getenv("GOEXPERIMENT"),
   231  		"GOOS=" + runtime.GOOS,
   232  		"TESTGO_GOHOSTOS=" + goHostOS,
   233  		"GOPROXY=" + proxyURL,
   234  		"GOPRIVATE=",
   235  		"GOROOT=" + testGOROOT,
   236  		"GOTRACEBACK=system",
   237  		"TESTGONETWORK=panic", // allow only local connections by default; the [net] condition resets this
   238  		"TESTGO_GOROOT=" + testGOROOT,
   239  		"TESTGO_EXE=" + testGo,
   240  		"TESTGO_VCSTEST_HOST=" + httpURL.Host,
   241  		"TESTGO_VCSTEST_TLS_HOST=" + httpsURL.Host,
   242  		"TESTGO_VCSTEST_CERT=" + srvCertFile,
   243  		"TESTGONETWORK=panic", // cleared by the [net] condition
   244  		"GOSUMDB=" + testSumDBVerifierKey,
   245  		"GONOPROXY=",
   246  		"GONOSUMDB=",
   247  		"GOVCS=*:all",
   248  		"devnull=" + os.DevNull,
   249  		"goversion=" + gover.Local(),
   250  		"CMDGO_TEST_RUN_MAIN=true",
   251  		"HGRCPATH=",
   252  		"GOTOOLCHAIN=auto",
   253  		"newline=\n",
   254  	}
   255  
   256  	if testenv.Builder() != "" || os.Getenv("GIT_TRACE_CURL") == "1" {
   257  		// To help diagnose https://go.dev/issue/52545,
   258  		// enable tracing for Git HTTPS requests.
   259  		env = append(env,
   260  			"GIT_TRACE_CURL=1",
   261  			"GIT_TRACE_CURL_NO_DATA=1",
   262  			"GIT_REDACT_COOKIES=o,SSO,GSSO_Uberproxy")
   263  	}
   264  	if testing.Short() {
   265  		// VCS commands are always somewhat slow: they either require access to external hosts,
   266  		// or they require our intercepted vcs-test.golang.org to regenerate the repository.
   267  		// Require all tests that use VCS commands to be skipped in short mode.
   268  		env = append(env, "TESTGOVCS=panic")
   269  	}
   270  
   271  	if os.Getenv("CGO_ENABLED") != "" || runtime.GOOS != goHostOS || runtime.GOARCH != goHostArch {
   272  		// If the actual CGO_ENABLED might not match the cmd/go default, set it
   273  		// explicitly in the environment. Otherwise, leave it unset so that we also
   274  		// cover the default behaviors.
   275  		env = append(env, "CGO_ENABLED="+cgoEnabled)
   276  	}
   277  
   278  	for _, key := range extraEnvKeys {
   279  		if val, ok := os.LookupEnv(key); ok {
   280  			env = append(env, key+"="+val)
   281  		}
   282  	}
   283  
   284  	return env, nil
   285  }
   286  
   287  var extraEnvKeys = []string{
   288  	"SYSTEMROOT",         // must be preserved on Windows to find DLLs; golang.org/issue/25210
   289  	"WINDIR",             // must be preserved on Windows to be able to run PowerShell command; golang.org/issue/30711
   290  	"LD_LIBRARY_PATH",    // must be preserved on Unix systems to find shared libraries
   291  	"LIBRARY_PATH",       // allow override of non-standard static library paths
   292  	"C_INCLUDE_PATH",     // allow override non-standard include paths
   293  	"CC",                 // don't lose user settings when invoking cgo
   294  	"GO_TESTING_GOTOOLS", // for gccgo testing
   295  	"GCCGO",              // for gccgo testing
   296  	"GCCGOTOOLDIR",       // for gccgo testing
   297  }
   298  
   299  // updateSum runs 'go mod tidy', 'go list -mod=mod -m all', or
   300  // 'go list -mod=mod all' in the test's current directory if a file named
   301  // "go.mod" is present after the archive has been extracted. updateSum modifies
   302  // archive and returns true if go.mod or go.sum were changed.
   303  func updateSum(t testing.TB, e *script.Engine, s *script.State, archive *txtar.Archive) (rewrite bool) {
   304  	gomodIdx, gosumIdx := -1, -1
   305  	for i := range archive.Files {
   306  		switch archive.Files[i].Name {
   307  		case "go.mod":
   308  			gomodIdx = i
   309  		case "go.sum":
   310  			gosumIdx = i
   311  		}
   312  	}
   313  	if gomodIdx < 0 {
   314  		return false
   315  	}
   316  
   317  	var cmd string
   318  	switch *testSum {
   319  	case "tidy":
   320  		cmd = "go mod tidy"
   321  	case "listm":
   322  		cmd = "go list -m -mod=mod all"
   323  	case "listall":
   324  		cmd = "go list -mod=mod all"
   325  	default:
   326  		t.Fatalf(`unknown value for -testsum %q; may be "tidy", "listm", or "listall"`, *testSum)
   327  	}
   328  
   329  	log := new(strings.Builder)
   330  	err := e.Execute(s, "updateSum", bufio.NewReader(strings.NewReader(cmd)), log)
   331  	if log.Len() > 0 {
   332  		t.Logf("%s", log)
   333  	}
   334  	if err != nil {
   335  		t.Fatal(err)
   336  	}
   337  
   338  	newGomodData, err := os.ReadFile(s.Path("go.mod"))
   339  	if err != nil {
   340  		t.Fatalf("reading go.mod after -testsum: %v", err)
   341  	}
   342  	if !bytes.Equal(newGomodData, archive.Files[gomodIdx].Data) {
   343  		archive.Files[gomodIdx].Data = newGomodData
   344  		rewrite = true
   345  	}
   346  
   347  	newGosumData, err := os.ReadFile(s.Path("go.sum"))
   348  	if err != nil && !os.IsNotExist(err) {
   349  		t.Fatalf("reading go.sum after -testsum: %v", err)
   350  	}
   351  	switch {
   352  	case os.IsNotExist(err) && gosumIdx >= 0:
   353  		// go.sum was deleted.
   354  		rewrite = true
   355  		archive.Files = append(archive.Files[:gosumIdx], archive.Files[gosumIdx+1:]...)
   356  	case err == nil && gosumIdx < 0:
   357  		// go.sum was created.
   358  		rewrite = true
   359  		gosumIdx = gomodIdx + 1
   360  		archive.Files = append(archive.Files, txtar.File{})
   361  		copy(archive.Files[gosumIdx+1:], archive.Files[gosumIdx:])
   362  		archive.Files[gosumIdx] = txtar.File{Name: "go.sum", Data: newGosumData}
   363  	case err == nil && gosumIdx >= 0 && !bytes.Equal(newGosumData, archive.Files[gosumIdx].Data):
   364  		// go.sum was changed.
   365  		rewrite = true
   366  		archive.Files[gosumIdx].Data = newGosumData
   367  	}
   368  	return rewrite
   369  }
   370  
   371  func readCounters(t *testing.T, telemetryDir string) map[string]uint64 {
   372  	localDir := filepath.Join(telemetryDir, "local")
   373  	dirents, err := os.ReadDir(localDir)
   374  	if err != nil {
   375  		if os.IsNotExist(err) {
   376  			return nil // The Go command didn't ever run so the local dir wasn't created
   377  		}
   378  		t.Fatalf("reading telemetry local dir: %v", err)
   379  	}
   380  	totals := map[string]uint64{}
   381  	for _, dirent := range dirents {
   382  		if dirent.IsDir() || !strings.HasSuffix(dirent.Name(), ".count") {
   383  			// not a counter file
   384  			continue
   385  		}
   386  		counters, _, err := countertest.ReadFile(filepath.Join(localDir, dirent.Name()))
   387  		if err != nil {
   388  			t.Fatalf("reading counter file: %v", err)
   389  		}
   390  		for k, v := range counters {
   391  			totals[k] += v
   392  		}
   393  	}
   394  
   395  	return totals
   396  }
   397  
   398  //go:embed testdata/counters.txt
   399  var countersTxt string
   400  
   401  var (
   402  	allowedCountersOnce sync.Once
   403  	allowedCounters     = map[string]bool{} // Set of allowed counters.
   404  )
   405  
   406  func checkCounters(t *testing.T, telemetryDir string) {
   407  	allowedCountersOnce.Do(func() {
   408  		for _, counter := range strings.Fields(countersTxt) {
   409  			allowedCounters[counter] = true
   410  		}
   411  	})
   412  	counters := readCounters(t, telemetryDir)
   413  	if _, ok := scriptGoInvoked.Load(testing.TB(t)); ok {
   414  		if !disabledOnPlatform && len(counters) == 0 {
   415  			t.Fatal("go was invoked but no counters were incremented")
   416  		}
   417  	}
   418  	for name := range counters {
   419  		if !allowedCounters[name] {
   420  			t.Fatalf("incremented counter %q is not in testdata/counters.txt. "+
   421  				"Please update counters_test.go to produce an entry for it.", name)
   422  		}
   423  	}
   424  }
   425  
   426  // Copied from https://go.googlesource.com/telemetry/+/5f08a0cbff3f/internal/telemetry/mode.go#122
   427  // TODO(go.dev/issues/66205): replace this with the public API once it becomes available.
   428  //
   429  // disabledOnPlatform indicates whether telemetry is disabled
   430  // due to bugs in the current platform.
   431  const disabledOnPlatform = false ||
   432  	// The following platforms could potentially be supported in the future:
   433  	runtime.GOOS == "openbsd" || // #60614
   434  	runtime.GOOS == "solaris" || // #60968 #60970
   435  	runtime.GOOS == "android" || // #60967
   436  	runtime.GOOS == "illumos" || // #65544
   437  	// These platforms fundamentally can't be supported:
   438  	runtime.GOOS == "js" || // #60971
   439  	runtime.GOOS == "wasip1" || // #60971
   440  	runtime.GOOS == "plan9" // https://github.com/golang/go/issues/57540#issuecomment-1470766639
   441
View as plain text