script_test.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Script-driven tests.
     6  // See testdata/script/README for an overview.
     7  
     8  //go:generate go test cmd/go -v -run=TestScript/README --fixreadme
     9  
    10  package main_test
    11  
    12  import (
    13  	"bufio"
    14  	"bytes"
    15  	"context"
    16  	_ "embed"
    17  	"flag"
    18  	"internal/testenv"
    19  	"internal/txtar"
    20  	"net/url"
    21  	"os"
    22  	"path/filepath"
    23  	"runtime"
    24  	"strings"
    25  	"testing"
    26  	"time"
    27  
    28  	"cmd/go/internal/cfg"
    29  	"cmd/go/internal/gover"
    30  	"cmd/go/internal/vcweb/vcstest"
    31  	"cmd/internal/script"
    32  	"cmd/internal/script/scripttest"
    33  
    34  	"golang.org/x/telemetry/counter/countertest"
    35  )
    36  
    37  var testSum = flag.String("testsum", "", `may be tidy, listm, or listall. If set, TestScript generates a go.sum file at the beginning of each test and updates test files if they pass.`)
    38  
    39  // TestScript runs the tests in testdata/script/*.txt.
    40  func TestScript(t *testing.T) {
    41  	t.Parallel()
    42  
    43  	testenv.MustHaveGoBuild(t)
    44  	testenv.SkipIfShortAndSlow(t)
    45  
    46  	if testing.Short() && runtime.GOOS == "plan9" {
    47  		t.Skipf("skipping test in -short mode on %s", runtime.GOOS)
    48  	}
    49  
    50  	srv, err := vcstest.NewServer()
    51  	if err != nil {
    52  		t.Fatal(err)
    53  	}
    54  	t.Cleanup(func() {
    55  		if err := srv.Close(); err != nil {
    56  			t.Fatal(err)
    57  		}
    58  	})
    59  	certFile, err := srv.WriteCertificateFile()
    60  	if err != nil {
    61  		t.Fatal(err)
    62  	}
    63  
    64  	StartProxy()
    65  
    66  	var (
    67  		ctx         = context.Background()
    68  		gracePeriod = 100 * time.Millisecond
    69  	)
    70  	if deadline, ok := t.Deadline(); ok {
    71  		timeout := time.Until(deadline)
    72  
    73  		// If time allows, increase the termination grace period to 5% of the
    74  		// remaining time.
    75  		if gp := timeout / 20; gp > gracePeriod {
    76  			gracePeriod = gp
    77  		}
    78  
    79  		// When we run commands that execute subprocesses, we want to reserve two
    80  		// grace periods to clean up. We will send the first termination signal when
    81  		// the context expires, then wait one grace period for the process to
    82  		// produce whatever useful output it can (such as a stack trace). After the
    83  		// first grace period expires, we'll escalate to os.Kill, leaving the second
    84  		// grace period for the test function to record its output before the test
    85  		// process itself terminates.
    86  		timeout -= 2 * gracePeriod
    87  
    88  		var cancel context.CancelFunc
    89  		ctx, cancel = context.WithTimeout(ctx, timeout)
    90  		t.Cleanup(cancel)
    91  	}
    92  
    93  	env, err := scriptEnv(srv, certFile)
    94  	if err != nil {
    95  		t.Fatal(err)
    96  	}
    97  	engine := &script.Engine{
    98  		Conds: scriptConditions(t),
    99  		Cmds:  scriptCommands(quitSignal(), gracePeriod),
   100  		Quiet: !testing.Verbose(),
   101  	}
   102  
   103  	t.Run("README", func(t *testing.T) {
   104  		t.Parallel()
   105  		checkScriptReadme(t, engine, env)
   106  	})
   107  
   108  	files, err := filepath.Glob("testdata/script/*.txt")
   109  	if err != nil {
   110  		t.Fatal(err)
   111  	}
   112  	for _, file := range files {
   113  		name := strings.TrimSuffix(filepath.Base(file), ".txt")
   114  		t.Run(name, func(t *testing.T) {
   115  			t.Parallel()
   116  			StartProxy()
   117  
   118  			workdir, err := os.MkdirTemp(testTmpDir, name)
   119  			if err != nil {
   120  				t.Fatal(err)
   121  			}
   122  			if !*testWork {
   123  				defer removeAll(workdir)
   124  			}
   125  
   126  			s, err := script.NewState(tbContext(ctx, t), workdir, env)
   127  			if err != nil {
   128  				t.Fatal(err)
   129  			}
   130  
   131  			// Unpack archive.
   132  			a, err := txtar.ParseFile(file)
   133  			if err != nil {
   134  				t.Fatal(err)
   135  			}
   136  			telemetryDir := initScriptDirs(t, s)
   137  			if err := s.ExtractFiles(a); err != nil {
   138  				t.Fatal(err)
   139  			}
   140  
   141  			t.Log(time.Now().UTC().Format(time.RFC3339))
   142  			work, _ := s.LookupEnv("WORK")
   143  			t.Logf("$WORK=%s", work)
   144  
   145  			// With -testsum, if a go.mod file is present in the test's initial
   146  			// working directory, run 'go mod tidy'.
   147  			if *testSum != "" {
   148  				if updateSum(t, engine, s, a) {
   149  					defer func() {
   150  						if t.Failed() {
   151  							return
   152  						}
   153  						data := txtar.Format(a)
   154  						if err := os.WriteFile(file, data, 0666); err != nil {
   155  							t.Errorf("rewriting test file: %v", err)
   156  						}
   157  					}()
   158  				}
   159  			}
   160  
   161  			// Note: Do not use filepath.Base(file) here:
   162  			// editors that can jump to file:line references in the output
   163  			// will work better seeing the full path relative to cmd/go
   164  			// (where the "go test" command is usually run).
   165  			scripttest.Run(t, engine, s, file, bytes.NewReader(a.Comment))
   166  			checkCounters(t, telemetryDir)
   167  		})
   168  	}
   169  }
   170  
   171  // testingTBKey is the Context key for a testing.TB.
   172  type testingTBKey struct{}
   173  
   174  // tbContext returns a Context derived from ctx and associated with t.
   175  func tbContext(ctx context.Context, t testing.TB) context.Context {
   176  	return context.WithValue(ctx, testingTBKey{}, t)
   177  }
   178  
   179  // tbFromContext returns the testing.TB associated with ctx, if any.
   180  func tbFromContext(ctx context.Context) (testing.TB, bool) {
   181  	t := ctx.Value(testingTBKey{})
   182  	if t == nil {
   183  		return nil, false
   184  	}
   185  	return t.(testing.TB), true
   186  }
   187  
   188  // initScriptDirs creates the initial directory structure in s for unpacking a
   189  // cmd/go script.
   190  func initScriptDirs(t testing.TB, s *script.State) (telemetryDir string) {
   191  	must := func(err error) {
   192  		if err != nil {
   193  			t.Helper()
   194  			t.Fatal(err)
   195  		}
   196  	}
   197  
   198  	work := s.Getwd()
   199  	must(s.Setenv("WORK", work))
   200  
   201  	telemetryDir = filepath.Join(work, "telemetry")
   202  	must(os.MkdirAll(telemetryDir, 0777))
   203  	must(s.Setenv("TEST_TELEMETRY_DIR", filepath.Join(work, "telemetry")))
   204  
   205  	must(os.MkdirAll(filepath.Join(work, "tmp"), 0777))
   206  	must(s.Setenv(tempEnvName(), filepath.Join(work, "tmp")))
   207  
   208  	gopath := filepath.Join(work, "gopath")
   209  	must(s.Setenv("GOPATH", gopath))
   210  	gopathSrc := filepath.Join(gopath, "src")
   211  	must(os.MkdirAll(gopathSrc, 0777))
   212  	must(s.Chdir(gopathSrc))
   213  	return telemetryDir
   214  }
   215  
   216  func scriptEnv(srv *vcstest.Server, srvCertFile string) ([]string, error) {
   217  	httpURL, err := url.Parse(srv.HTTP.URL)
   218  	if err != nil {
   219  		return nil, err
   220  	}
   221  	httpsURL, err := url.Parse(srv.HTTPS.URL)
   222  	if err != nil {
   223  		return nil, err
   224  	}
   225  	env := []string{
   226  		pathEnvName() + "=" + testBin + string(filepath.ListSeparator) + os.Getenv(pathEnvName()),
   227  		homeEnvName() + "=/no-home",
   228  		"CCACHE_DISABLE=1", // ccache breaks with non-existent HOME
   229  		"GOARCH=" + runtime.GOARCH,
   230  		"TESTGO_GOHOSTARCH=" + goHostArch,
   231  		"GOCACHE=" + testGOCACHE,
   232  		"GOCOVERDIR=" + os.Getenv("GOCOVERDIR"),
   233  		"GODEBUG=" + os.Getenv("GODEBUG"),
   234  		"GOEXE=" + cfg.ExeSuffix,
   235  		"GOEXPERIMENT=" + os.Getenv("GOEXPERIMENT"),
   236  		"GOOS=" + runtime.GOOS,
   237  		"TESTGO_GOHOSTOS=" + goHostOS,
   238  		"GOPROXY=" + proxyURL,
   239  		"GOPRIVATE=",
   240  		"GOROOT=" + testGOROOT,
   241  		"GOTRACEBACK=system",
   242  		"TESTGONETWORK=panic", // allow only local connections by default; the [net] condition resets this
   243  		"TESTGO_GOROOT=" + testGOROOT,
   244  		"TESTGO_EXE=" + testGo,
   245  		"TESTGO_VCSTEST_HOST=" + httpURL.Host,
   246  		"TESTGO_VCSTEST_TLS_HOST=" + httpsURL.Host,
   247  		"TESTGO_VCSTEST_CERT=" + srvCertFile,
   248  		"TESTGONETWORK=panic", // cleared by the [net] condition
   249  		"GOSUMDB=" + testSumDBVerifierKey,
   250  		"GONOPROXY=",
   251  		"GONOSUMDB=",
   252  		"GOVCS=*:all",
   253  		"devnull=" + os.DevNull,
   254  		"goversion=" + gover.Local(),
   255  		"CMDGO_TEST_RUN_MAIN=true",
   256  		"HGRCPATH=",
   257  		"GOTOOLCHAIN=auto",
   258  		"newline=\n",
   259  	}
   260  
   261  	if testenv.Builder() != "" || os.Getenv("GIT_TRACE_CURL") == "1" {
   262  		// To help diagnose https://go.dev/issue/52545,
   263  		// enable tracing for Git HTTPS requests.
   264  		env = append(env,
   265  			"GIT_TRACE_CURL=1",
   266  			"GIT_TRACE_CURL_NO_DATA=1",
   267  			"GIT_REDACT_COOKIES=o,SSO,GSSO_Uberproxy")
   268  	}
   269  	if testing.Short() {
   270  		// VCS commands are always somewhat slow: they either require access to external hosts,
   271  		// or they require our intercepted vcs-test.golang.org to regenerate the repository.
   272  		// Require all tests that use VCS commands which require remote lookups to be skipped in
   273  		// short mode.
   274  		env = append(env, "TESTGOVCSREMOTE=panic")
   275  	}
   276  	if os.Getenv("CGO_ENABLED") != "" || runtime.GOOS != goHostOS || runtime.GOARCH != goHostArch {
   277  		// If the actual CGO_ENABLED might not match the cmd/go default, set it
   278  		// explicitly in the environment. Otherwise, leave it unset so that we also
   279  		// cover the default behaviors.
   280  		env = append(env, "CGO_ENABLED="+cgoEnabled)
   281  	}
   282  
   283  	for _, key := range extraEnvKeys {
   284  		if val, ok := os.LookupEnv(key); ok {
   285  			env = append(env, key+"="+val)
   286  		}
   287  	}
   288  
   289  	return env, nil
   290  }
   291  
   292  var extraEnvKeys = []string{
   293  	"SYSTEMROOT",         // must be preserved on Windows to find DLLs; golang.org/issue/25210
   294  	"WINDIR",             // must be preserved on Windows to be able to run PowerShell command; golang.org/issue/30711
   295  	"LD_LIBRARY_PATH",    // must be preserved on Unix systems to find shared libraries
   296  	"LIBRARY_PATH",       // allow override of non-standard static library paths
   297  	"C_INCLUDE_PATH",     // allow override non-standard include paths
   298  	"CC",                 // don't lose user settings when invoking cgo
   299  	"GO_TESTING_GOTOOLS", // for gccgo testing
   300  	"GCCGO",              // for gccgo testing
   301  	"GCCGOTOOLDIR",       // for gccgo testing
   302  }
   303  
   304  // updateSum runs 'go mod tidy', 'go list -mod=mod -m all', or
   305  // 'go list -mod=mod all' in the test's current directory if a file named
   306  // "go.mod" is present after the archive has been extracted. updateSum modifies
   307  // archive and returns true if go.mod or go.sum were changed.
   308  func updateSum(t testing.TB, e *script.Engine, s *script.State, archive *txtar.Archive) (rewrite bool) {
   309  	gomodIdx, gosumIdx := -1, -1
   310  	for i := range archive.Files {
   311  		switch archive.Files[i].Name {
   312  		case "go.mod":
   313  			gomodIdx = i
   314  		case "go.sum":
   315  			gosumIdx = i
   316  		}
   317  	}
   318  	if gomodIdx < 0 {
   319  		return false
   320  	}
   321  
   322  	var cmd string
   323  	switch *testSum {
   324  	case "tidy":
   325  		cmd = "go mod tidy"
   326  	case "listm":
   327  		cmd = "go list -m -mod=mod all"
   328  	case "listall":
   329  		cmd = "go list -mod=mod all"
   330  	default:
   331  		t.Fatalf(`unknown value for -testsum %q; may be "tidy", "listm", or "listall"`, *testSum)
   332  	}
   333  
   334  	log := new(strings.Builder)
   335  	err := e.Execute(s, "updateSum", bufio.NewReader(strings.NewReader(cmd)), log)
   336  	if log.Len() > 0 {
   337  		t.Logf("%s", log)
   338  	}
   339  	if err != nil {
   340  		t.Fatal(err)
   341  	}
   342  
   343  	newGomodData, err := os.ReadFile(s.Path("go.mod"))
   344  	if err != nil {
   345  		t.Fatalf("reading go.mod after -testsum: %v", err)
   346  	}
   347  	if !bytes.Equal(newGomodData, archive.Files[gomodIdx].Data) {
   348  		archive.Files[gomodIdx].Data = newGomodData
   349  		rewrite = true
   350  	}
   351  
   352  	newGosumData, err := os.ReadFile(s.Path("go.sum"))
   353  	if err != nil && !os.IsNotExist(err) {
   354  		t.Fatalf("reading go.sum after -testsum: %v", err)
   355  	}
   356  	switch {
   357  	case os.IsNotExist(err) && gosumIdx >= 0:
   358  		// go.sum was deleted.
   359  		rewrite = true
   360  		archive.Files = append(archive.Files[:gosumIdx], archive.Files[gosumIdx+1:]...)
   361  	case err == nil && gosumIdx < 0:
   362  		// go.sum was created.
   363  		rewrite = true
   364  		gosumIdx = gomodIdx + 1
   365  		archive.Files = append(archive.Files, txtar.File{})
   366  		copy(archive.Files[gosumIdx+1:], archive.Files[gosumIdx:])
   367  		archive.Files[gosumIdx] = txtar.File{Name: "go.sum", Data: newGosumData}
   368  	case err == nil && gosumIdx >= 0 && !bytes.Equal(newGosumData, archive.Files[gosumIdx].Data):
   369  		// go.sum was changed.
   370  		rewrite = true
   371  		archive.Files[gosumIdx].Data = newGosumData
   372  	}
   373  	return rewrite
   374  }
   375  
   376  func readCounters(t *testing.T, telemetryDir string) map[string]uint64 {
   377  	localDir := filepath.Join(telemetryDir, "local")
   378  	dirents, err := os.ReadDir(localDir)
   379  	if err != nil {
   380  		if os.IsNotExist(err) {
   381  			return nil // The Go command didn't ever run so the local dir wasn't created
   382  		}
   383  		t.Fatalf("reading telemetry local dir: %v", err)
   384  	}
   385  	totals := map[string]uint64{}
   386  	for _, dirent := range dirents {
   387  		if dirent.IsDir() || !strings.HasSuffix(dirent.Name(), ".count") {
   388  			// not a counter file
   389  			continue
   390  		}
   391  		counters, _, err := countertest.ReadFile(filepath.Join(localDir, dirent.Name()))
   392  		if err != nil {
   393  			t.Fatalf("reading counter file: %v", err)
   394  		}
   395  		for k, v := range counters {
   396  			totals[k] += v
   397  		}
   398  	}
   399  
   400  	return totals
   401  }
   402  
   403  func checkCounters(t *testing.T, telemetryDir string) {
   404  	counters := readCounters(t, telemetryDir)
   405  	if _, ok := scriptGoInvoked.Load(testing.TB(t)); ok {
   406  		if !disabledOnPlatform && len(counters) == 0 {
   407  			t.Fatal("go was invoked but no counters were incremented")
   408  		}
   409  	}
   410  }
   411  
   412  // Copied from https://go.googlesource.com/telemetry/+/5f08a0cbff3f/internal/telemetry/mode.go#122
   413  // TODO(go.dev/issues/66205): replace this with the public API once it becomes available.
   414  //
   415  // disabledOnPlatform indicates whether telemetry is disabled
   416  // due to bugs in the current platform.
   417  const disabledOnPlatform = false ||
   418  	// The following platforms could potentially be supported in the future:
   419  	runtime.GOOS == "openbsd" || // #60614
   420  	runtime.GOOS == "solaris" || // #60968 #60970
   421  	runtime.GOOS == "android" || // #60967
   422  	runtime.GOOS == "illumos" || // #65544
   423  	// These platforms fundamentally can't be supported:
   424  	runtime.GOOS == "js" || // #60971
   425  	runtime.GOOS == "wasip1" || // #60971
   426  	runtime.GOOS == "plan9" // https://github.com/golang/go/issues/57540#issuecomment-1470766639
   427
View as plain text