Source file src/runtime/crash_test.go

     1  // Copyright 2012 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime_test
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"errors"
    11  	"flag"
    12  	"fmt"
    13  	"internal/asan"
    14  	"internal/msan"
    15  	"internal/profile"
    16  	"internal/race"
    17  	"internal/testenv"
    18  	traceparse "internal/trace"
    19  	"io"
    20  	"log"
    21  	"os"
    22  	"os/exec"
    23  	"path/filepath"
    24  	"regexp"
    25  	"runtime"
    26  	"runtime/trace"
    27  	"strings"
    28  	"sync"
    29  	"testing"
    30  	"time"
    31  )
    32  
    33  var toRemove []string
    34  
    35  const entrypointVar = "RUNTIME_TEST_ENTRYPOINT"
    36  
    37  func TestMain(m *testing.M) {
    38  	switch entrypoint := os.Getenv(entrypointVar); entrypoint {
    39  	case "panic":
    40  		crashViaPanic()
    41  		panic("unreachable")
    42  	case "trap":
    43  		crashViaTrap()
    44  		panic("unreachable")
    45  	default:
    46  		log.Fatalf("invalid %s: %q", entrypointVar, entrypoint)
    47  	case "":
    48  		// fall through to normal behavior
    49  	}
    50  
    51  	_, coreErrBefore := os.Stat("core")
    52  
    53  	status := m.Run()
    54  	for _, file := range toRemove {
    55  		os.RemoveAll(file)
    56  	}
    57  
    58  	_, coreErrAfter := os.Stat("core")
    59  	if coreErrBefore != nil && coreErrAfter == nil {
    60  		fmt.Fprintln(os.Stderr, "runtime.test: some test left a core file behind")
    61  		if status == 0 {
    62  			status = 1
    63  		}
    64  	}
    65  
    66  	os.Exit(status)
    67  }
    68  
    69  var testprog struct {
    70  	sync.Mutex
    71  	dir    string
    72  	target map[string]*buildexe
    73  }
    74  
    75  type buildexe struct {
    76  	once sync.Once
    77  	exe  string
    78  	err  error
    79  }
    80  
    81  func runTestProg(t *testing.T, binary, name string, env ...string) string {
    82  	if *flagQuick {
    83  		t.Skip("-quick")
    84  	}
    85  
    86  	testenv.MustHaveGoBuild(t)
    87  	t.Helper()
    88  
    89  	exe, err := buildTestProg(t, binary)
    90  	if err != nil {
    91  		t.Fatal(err)
    92  	}
    93  
    94  	return runBuiltTestProg(t, exe, name, env...)
    95  }
    96  
    97  func runBuiltTestProg(t *testing.T, exe, name string, env ...string) string {
    98  	t.Helper()
    99  
   100  	out, _ := runBuiltTestProgErr(t, exe, name, env...)
   101  	return out
   102  }
   103  
   104  func runBuiltTestProgErr(t *testing.T, exe, name string, env ...string) (string, error) {
   105  	t.Helper()
   106  
   107  	if *flagQuick {
   108  		t.Skip("-quick")
   109  	}
   110  
   111  	start := time.Now()
   112  
   113  	cmd := testenv.CleanCmdEnv(testenv.Command(t, exe, name))
   114  	cmd.Env = append(cmd.Env, env...)
   115  	if testing.Short() {
   116  		cmd.Env = append(cmd.Env, "RUNTIME_TEST_SHORT=1")
   117  	}
   118  	out, err := cmd.CombinedOutput()
   119  	if err == nil {
   120  		t.Logf("%v (%v): ok", cmd, time.Since(start))
   121  	} else {
   122  		if _, ok := err.(*exec.ExitError); ok {
   123  			t.Logf("%v: %v", cmd, err)
   124  		} else if errors.Is(err, exec.ErrWaitDelay) {
   125  			// Report the WaitDelay to help with triage. If it shows a small value like 100ms,
   126  			// it might not be enough for slow builders. See watchflakes issue #76685.
   127  			t.Fatalf("%v: %v: output pipes not closed after waiting %v", cmd, err, cmd.WaitDelay)
   128  		} else {
   129  			t.Fatalf("%v failed to start: %v", cmd, err)
   130  		}
   131  	}
   132  	return string(out), err
   133  }
   134  
   135  var serializeBuild = make(chan bool, 2)
   136  
   137  func buildTestProg(t *testing.T, binary string, flags ...string) (string, error) {
   138  	if *flagQuick {
   139  		t.Skip("-quick")
   140  	}
   141  	testenv.MustHaveGoBuild(t)
   142  
   143  	testprog.Lock()
   144  	if testprog.dir == "" {
   145  		dir, err := os.MkdirTemp("", "go-build")
   146  		if err != nil {
   147  			t.Fatalf("failed to create temp directory: %v", err)
   148  		}
   149  		testprog.dir = dir
   150  		toRemove = append(toRemove, dir)
   151  	}
   152  
   153  	if testprog.target == nil {
   154  		testprog.target = make(map[string]*buildexe)
   155  	}
   156  	name := binary
   157  	if len(flags) > 0 {
   158  		name += "_" + strings.Join(flags, "_")
   159  	}
   160  	target, ok := testprog.target[name]
   161  	if !ok {
   162  		target = &buildexe{}
   163  		testprog.target[name] = target
   164  	}
   165  
   166  	dir := testprog.dir
   167  
   168  	// Unlock testprog while actually building, so that other
   169  	// tests can look up executables that were already built.
   170  	testprog.Unlock()
   171  
   172  	target.once.Do(func() {
   173  		// Only do two "go build"'s at a time,
   174  		// to keep load from getting too high.
   175  		serializeBuild <- true
   176  		defer func() { <-serializeBuild }()
   177  
   178  		// Don't get confused if testenv.GoToolPath calls t.Skip.
   179  		target.err = errors.New("building test called t.Skip")
   180  
   181  		if asan.Enabled {
   182  			flags = append(flags, "-asan")
   183  		}
   184  		if msan.Enabled {
   185  			flags = append(flags, "-msan")
   186  		}
   187  		if race.Enabled {
   188  			flags = append(flags, "-race")
   189  		}
   190  
   191  		exe := filepath.Join(dir, name+".exe")
   192  
   193  		start := time.Now()
   194  		cmd := exec.Command(testenv.GoToolPath(t), append([]string{"build", "-o", exe}, flags...)...)
   195  		t.Logf("running %v", cmd)
   196  		cmd.Dir = "testdata/" + binary
   197  		cmd = testenv.CleanCmdEnv(cmd)
   198  
   199  		out, err := cmd.CombinedOutput()
   200  		if err != nil {
   201  			target.err = fmt.Errorf("building %s %v: %v\n%s", binary, flags, err, out)
   202  		} else {
   203  			t.Logf("built %v in %v", name, time.Since(start))
   204  			target.exe = exe
   205  			target.err = nil
   206  		}
   207  	})
   208  
   209  	return target.exe, target.err
   210  }
   211  
   212  func TestVDSO(t *testing.T) {
   213  	t.Parallel()
   214  	output := runTestProg(t, "testprog", "SignalInVDSO")
   215  	want := "success\n"
   216  	if output != want {
   217  		t.Fatalf("output:\n%s\n\nwanted:\n%s", output, want)
   218  	}
   219  }
   220  
   221  func testCrashHandler(t *testing.T, cgo bool) {
   222  	type crashTest struct {
   223  		Cgo bool
   224  	}
   225  	var output string
   226  	if cgo {
   227  		if runtime.GOOS == "freebsd" && race.Enabled {
   228  			t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.")
   229  		}
   230  		output = runTestProg(t, "testprogcgo", "Crash")
   231  	} else {
   232  		output = runTestProg(t, "testprog", "Crash")
   233  	}
   234  	want := "main: recovered done\nnew-thread: recovered done\nsecond-new-thread: recovered done\nmain-again: recovered done\n"
   235  	if output != want {
   236  		t.Fatalf("output:\n%s\n\nwanted:\n%s", output, want)
   237  	}
   238  }
   239  
   240  func TestCrashHandler(t *testing.T) {
   241  	testCrashHandler(t, false)
   242  }
   243  
   244  var deadlockBuildTypes = testenv.SpecialBuildTypes{
   245  	// External linking brings in cgo, causing deadlock detection not working.
   246  	Cgo:  false,
   247  	Asan: asan.Enabled,
   248  	Msan: msan.Enabled,
   249  	Race: race.Enabled,
   250  }
   251  
   252  func testDeadlock(t *testing.T, name string) {
   253  	// External linking brings in cgo, causing deadlock detection not working.
   254  	testenv.MustInternalLink(t, deadlockBuildTypes)
   255  
   256  	output := runTestProg(t, "testprog", name)
   257  	want := "fatal error: all goroutines are asleep - deadlock!\n"
   258  	if !strings.HasPrefix(output, want) {
   259  		t.Fatalf("output does not start with %q:\n%s", want, output)
   260  	}
   261  }
   262  
   263  func TestSimpleDeadlock(t *testing.T) {
   264  	testDeadlock(t, "SimpleDeadlock")
   265  }
   266  
   267  func TestInitDeadlock(t *testing.T) {
   268  	testDeadlock(t, "InitDeadlock")
   269  }
   270  
   271  func TestLockedDeadlock(t *testing.T) {
   272  	testDeadlock(t, "LockedDeadlock")
   273  }
   274  
   275  func TestLockedDeadlock2(t *testing.T) {
   276  	testDeadlock(t, "LockedDeadlock2")
   277  }
   278  
   279  func TestGoexitDeadlock(t *testing.T) {
   280  	// External linking brings in cgo, causing deadlock detection not working.
   281  	testenv.MustInternalLink(t, deadlockBuildTypes)
   282  
   283  	output := runTestProg(t, "testprog", "GoexitDeadlock")
   284  	want := "no goroutines (main called runtime.Goexit) - deadlock!"
   285  	if !strings.Contains(output, want) {
   286  		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
   287  	}
   288  }
   289  
   290  func TestStackOverflow(t *testing.T) {
   291  	output := runTestProg(t, "testprog", "StackOverflow")
   292  	want := []string{
   293  		"runtime: goroutine stack exceeds 1474560-byte limit\n",
   294  		"fatal error: stack overflow",
   295  		// information about the current SP and stack bounds
   296  		"runtime: sp=",
   297  		"stack=[",
   298  	}
   299  	if !strings.HasPrefix(output, want[0]) {
   300  		t.Errorf("output does not start with %q", want[0])
   301  	}
   302  	for _, s := range want[1:] {
   303  		if !strings.Contains(output, s) {
   304  			t.Errorf("output does not contain %q", s)
   305  		}
   306  	}
   307  	if t.Failed() {
   308  		t.Logf("output:\n%s", output)
   309  	}
   310  }
   311  
   312  func TestThreadExhaustion(t *testing.T) {
   313  	output := runTestProg(t, "testprog", "ThreadExhaustion")
   314  	want := "runtime: program exceeds 10-thread limit\nfatal error: thread exhaustion"
   315  	if !strings.HasPrefix(output, want) {
   316  		t.Fatalf("output does not start with %q:\n%s", want, output)
   317  	}
   318  }
   319  
   320  func TestRecursivePanic(t *testing.T) {
   321  	output := runTestProg(t, "testprog", "RecursivePanic")
   322  	want := `wrap: bad
   323  panic: again
   324  
   325  `
   326  	if !strings.HasPrefix(output, want) {
   327  		t.Fatalf("output does not start with %q:\n%s", want, output)
   328  	}
   329  
   330  }
   331  
   332  func TestRecursivePanic2(t *testing.T) {
   333  	output := runTestProg(t, "testprog", "RecursivePanic2")
   334  	want := `first panic
   335  second panic
   336  panic: third panic
   337  
   338  `
   339  	if !strings.HasPrefix(output, want) {
   340  		t.Fatalf("output does not start with %q:\n%s", want, output)
   341  	}
   342  
   343  }
   344  
   345  func TestRecursivePanic3(t *testing.T) {
   346  	output := runTestProg(t, "testprog", "RecursivePanic3")
   347  	want := `panic: first panic
   348  
   349  `
   350  	if !strings.HasPrefix(output, want) {
   351  		t.Fatalf("output does not start with %q:\n%s", want, output)
   352  	}
   353  
   354  }
   355  
   356  func TestRecursivePanic4(t *testing.T) {
   357  	output := runTestProg(t, "testprog", "RecursivePanic4")
   358  	want := `panic: first panic [recovered]
   359  	panic: second panic
   360  `
   361  	if !strings.HasPrefix(output, want) {
   362  		t.Fatalf("output does not start with %q:\n%s", want, output)
   363  	}
   364  
   365  }
   366  
   367  func TestRecursivePanic5(t *testing.T) {
   368  	output := runTestProg(t, "testprog", "RecursivePanic5")
   369  	want := `first panic
   370  second panic
   371  panic: third panic
   372  `
   373  	if !strings.HasPrefix(output, want) {
   374  		t.Fatalf("output does not start with %q:\n%s", want, output)
   375  	}
   376  
   377  }
   378  
   379  func TestRepanickedPanic(t *testing.T) {
   380  	output := runTestProg(t, "testprog", "RepanickedPanic")
   381  	want := `panic: message [recovered, repanicked]
   382  `
   383  	if !strings.HasPrefix(output, want) {
   384  		t.Fatalf("output does not start with %q:\n%s", want, output)
   385  	}
   386  }
   387  
   388  func TestRepanickedMiddlePanic(t *testing.T) {
   389  	output := runTestProg(t, "testprog", "RepanickedMiddlePanic")
   390  	want := `panic: inner [recovered]
   391  	panic: middle [recovered, repanicked]
   392  	panic: outer
   393  `
   394  	if !strings.HasPrefix(output, want) {
   395  		t.Fatalf("output does not start with %q:\n%s", want, output)
   396  	}
   397  }
   398  
   399  func TestRepanickedPanicSandwich(t *testing.T) {
   400  	output := runTestProg(t, "testprog", "RepanickedPanicSandwich")
   401  	want := `panic: outer [recovered]
   402  	panic: inner [recovered]
   403  	panic: outer
   404  `
   405  	if !strings.HasPrefix(output, want) {
   406  		t.Fatalf("output does not start with %q:\n%s", want, output)
   407  	}
   408  }
   409  
   410  func TestDoublePanicWithSameValue(t *testing.T) {
   411  	output := runTestProg(t, "testprog", "DoublePanicWithSameValue")
   412  	want := `panic: message
   413  `
   414  	if !strings.HasPrefix(output, want) {
   415  		t.Fatalf("output does not start with %q:\n%s", want, output)
   416  	}
   417  }
   418  
   419  func TestGoexitCrash(t *testing.T) {
   420  	// External linking brings in cgo, causing deadlock detection not working.
   421  	testenv.MustInternalLink(t, deadlockBuildTypes)
   422  
   423  	output := runTestProg(t, "testprog", "GoexitExit")
   424  	want := "no goroutines (main called runtime.Goexit) - deadlock!"
   425  	if !strings.Contains(output, want) {
   426  		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
   427  	}
   428  }
   429  
   430  func TestGoexitDefer(t *testing.T) {
   431  	c := make(chan struct{})
   432  	go func() {
   433  		defer func() {
   434  			r := recover()
   435  			if r != nil {
   436  				t.Errorf("non-nil recover during Goexit")
   437  			}
   438  			c <- struct{}{}
   439  		}()
   440  		runtime.Goexit()
   441  	}()
   442  	// Note: if the defer fails to run, we will get a deadlock here
   443  	<-c
   444  }
   445  
   446  func TestGoNil(t *testing.T) {
   447  	output := runTestProg(t, "testprog", "GoNil")
   448  	want := "go of nil func value"
   449  	if !strings.Contains(output, want) {
   450  		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
   451  	}
   452  }
   453  
   454  func TestMainGoroutineID(t *testing.T) {
   455  	output := runTestProg(t, "testprog", "MainGoroutineID")
   456  	want := "panic: test\n\ngoroutine 1 [running]:\n"
   457  	if !strings.HasPrefix(output, want) {
   458  		t.Fatalf("output does not start with %q:\n%s", want, output)
   459  	}
   460  }
   461  
   462  func TestNoHelperGoroutines(t *testing.T) {
   463  	output := runTestProg(t, "testprog", "NoHelperGoroutines")
   464  	matches := regexp.MustCompile(`goroutine [0-9]+ \[`).FindAllStringSubmatch(output, -1)
   465  	if len(matches) != 1 || matches[0][0] != "goroutine 1 [" {
   466  		t.Fatalf("want to see only goroutine 1, see:\n%s", output)
   467  	}
   468  }
   469  
   470  func TestBreakpoint(t *testing.T) {
   471  	output := runTestProg(t, "testprog", "Breakpoint")
   472  	// If runtime.Breakpoint() is inlined, then the stack trace prints
   473  	// "runtime.Breakpoint(...)" instead of "runtime.Breakpoint()".
   474  	want := "runtime.Breakpoint("
   475  	if !strings.Contains(output, want) {
   476  		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
   477  	}
   478  }
   479  
   480  func TestGoexitInPanic(t *testing.T) {
   481  	// External linking brings in cgo, causing deadlock detection not working.
   482  	testenv.MustInternalLink(t, deadlockBuildTypes)
   483  
   484  	// see issue 8774: this code used to trigger an infinite recursion
   485  	output := runTestProg(t, "testprog", "GoexitInPanic")
   486  	want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"
   487  	if !strings.HasPrefix(output, want) {
   488  		t.Fatalf("output does not start with %q:\n%s", want, output)
   489  	}
   490  }
   491  
   492  // Issue 14965: Runtime panics should be of type runtime.Error
   493  func TestRuntimePanicWithRuntimeError(t *testing.T) {
   494  	testCases := [...]func(){
   495  		0: func() {
   496  			var m map[uint64]bool
   497  			m[1234] = true
   498  		},
   499  		1: func() {
   500  			ch := make(chan struct{})
   501  			close(ch)
   502  			close(ch)
   503  		},
   504  		2: func() {
   505  			var ch = make(chan struct{})
   506  			close(ch)
   507  			ch <- struct{}{}
   508  		},
   509  		3: func() {
   510  			var s = make([]int, 2)
   511  			_ = s[2]
   512  		},
   513  		4: func() {
   514  			n := -1
   515  			_ = make(chan bool, n)
   516  		},
   517  		5: func() {
   518  			close((chan bool)(nil))
   519  		},
   520  	}
   521  
   522  	for i, fn := range testCases {
   523  		got := panicValue(fn)
   524  		if _, ok := got.(runtime.Error); !ok {
   525  			t.Errorf("test #%d: recovered value %v(type %T) does not implement runtime.Error", i, got, got)
   526  		}
   527  	}
   528  }
   529  
   530  func panicValue(fn func()) (recovered any) {
   531  	defer func() {
   532  		recovered = recover()
   533  	}()
   534  	fn()
   535  	return
   536  }
   537  
   538  func TestPanicAfterGoexit(t *testing.T) {
   539  	// an uncaught panic should still work after goexit
   540  	output := runTestProg(t, "testprog", "PanicAfterGoexit")
   541  	want := "panic: hello"
   542  	if !strings.HasPrefix(output, want) {
   543  		t.Fatalf("output does not start with %q:\n%s", want, output)
   544  	}
   545  }
   546  
   547  func TestRecoveredPanicAfterGoexit(t *testing.T) {
   548  	// External linking brings in cgo, causing deadlock detection not working.
   549  	testenv.MustInternalLink(t, deadlockBuildTypes)
   550  
   551  	output := runTestProg(t, "testprog", "RecoveredPanicAfterGoexit")
   552  	want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"
   553  	if !strings.HasPrefix(output, want) {
   554  		t.Fatalf("output does not start with %q:\n%s", want, output)
   555  	}
   556  }
   557  
   558  func TestRecoverBeforePanicAfterGoexit(t *testing.T) {
   559  	// External linking brings in cgo, causing deadlock detection not working.
   560  	testenv.MustInternalLink(t, deadlockBuildTypes)
   561  
   562  	t.Parallel()
   563  	output := runTestProg(t, "testprog", "RecoverBeforePanicAfterGoexit")
   564  	want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"
   565  	if !strings.HasPrefix(output, want) {
   566  		t.Fatalf("output does not start with %q:\n%s", want, output)
   567  	}
   568  }
   569  
   570  func TestRecoverBeforePanicAfterGoexit2(t *testing.T) {
   571  	// External linking brings in cgo, causing deadlock detection not working.
   572  	testenv.MustInternalLink(t, deadlockBuildTypes)
   573  
   574  	t.Parallel()
   575  	output := runTestProg(t, "testprog", "RecoverBeforePanicAfterGoexit2")
   576  	want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"
   577  	if !strings.HasPrefix(output, want) {
   578  		t.Fatalf("output does not start with %q:\n%s", want, output)
   579  	}
   580  }
   581  
   582  func TestNetpollDeadlock(t *testing.T) {
   583  	t.Parallel()
   584  	output := runTestProg(t, "testprognet", "NetpollDeadlock")
   585  	want := "done\n"
   586  	if !strings.HasSuffix(output, want) {
   587  		t.Fatalf("output does not start with %q:\n%s", want, output)
   588  	}
   589  }
   590  
   591  func TestPanicTraceback(t *testing.T) {
   592  	t.Parallel()
   593  	output := runTestProg(t, "testprog", "PanicTraceback")
   594  	want := "panic: hello\n\tpanic: panic pt2\n\tpanic: panic pt1\n"
   595  	if !strings.HasPrefix(output, want) {
   596  		t.Fatalf("output does not start with %q:\n%s", want, output)
   597  	}
   598  
   599  	// Check functions in the traceback.
   600  	fns := []string{"main.pt1.func1", "panic", "main.pt2.func1", "panic", "main.pt2", "main.pt1"}
   601  	for _, fn := range fns {
   602  		re := regexp.MustCompile(`(?m)^` + regexp.QuoteMeta(fn) + `\(.*\n`)
   603  		idx := re.FindStringIndex(output)
   604  		if idx == nil {
   605  			t.Fatalf("expected %q function in traceback:\n%s", fn, output)
   606  		}
   607  		output = output[idx[1]:]
   608  	}
   609  }
   610  
   611  func testPanicDeadlock(t *testing.T, name string, want string) {
   612  	// test issue 14432
   613  	output := runTestProg(t, "testprog", name)
   614  	if !strings.HasPrefix(output, want) {
   615  		t.Fatalf("output does not start with %q:\n%s", want, output)
   616  	}
   617  }
   618  
   619  func TestPanicDeadlockGosched(t *testing.T) {
   620  	testPanicDeadlock(t, "GoschedInPanic", "panic: errorThatGosched\n\n")
   621  }
   622  
   623  func TestPanicDeadlockSyscall(t *testing.T) {
   624  	testPanicDeadlock(t, "SyscallInPanic", "1\n2\npanic: 3\n\n")
   625  }
   626  
   627  func TestPanicLoop(t *testing.T) {
   628  	output := runTestProg(t, "testprog", "PanicLoop")
   629  	if want := "panic while printing panic value"; !strings.Contains(output, want) {
   630  		t.Errorf("output does not contain %q:\n%s", want, output)
   631  	}
   632  }
   633  
   634  func TestMemPprof(t *testing.T) {
   635  	testenv.MustHaveGoRun(t)
   636  
   637  	exe, err := buildTestProg(t, "testprog")
   638  	if err != nil {
   639  		t.Fatal(err)
   640  	}
   641  
   642  	got, err := testenv.CleanCmdEnv(exec.Command(exe, "MemProf")).CombinedOutput()
   643  	if err != nil {
   644  		t.Fatalf("testprog failed: %s, output:\n%s", err, got)
   645  	}
   646  	fn := strings.TrimSpace(string(got))
   647  	defer os.Remove(fn)
   648  
   649  	for try := 0; try < 2; try++ {
   650  		cmd := testenv.CleanCmdEnv(exec.Command(testenv.GoToolPath(t), "tool", "pprof", "-alloc_space", "-top"))
   651  		// Check that pprof works both with and without explicit executable on command line.
   652  		if try == 0 {
   653  			cmd.Args = append(cmd.Args, exe, fn)
   654  		} else {
   655  			cmd.Args = append(cmd.Args, fn)
   656  		}
   657  		found := false
   658  		for i, e := range cmd.Env {
   659  			if strings.HasPrefix(e, "PPROF_TMPDIR=") {
   660  				cmd.Env[i] = "PPROF_TMPDIR=" + os.TempDir()
   661  				found = true
   662  				break
   663  			}
   664  		}
   665  		if !found {
   666  			cmd.Env = append(cmd.Env, "PPROF_TMPDIR="+os.TempDir())
   667  		}
   668  
   669  		top, err := cmd.CombinedOutput()
   670  		t.Logf("%s:\n%s", cmd.Args, top)
   671  		if err != nil {
   672  			t.Error(err)
   673  		} else if !bytes.Contains(top, []byte("MemProf")) {
   674  			t.Error("missing MemProf in pprof output")
   675  		}
   676  	}
   677  }
   678  
   679  var concurrentMapTest = flag.Bool("run_concurrent_map_tests", false, "also run flaky concurrent map tests")
   680  
   681  func TestConcurrentMapWrites(t *testing.T) {
   682  	if !*concurrentMapTest {
   683  		t.Skip("skipping without -run_concurrent_map_tests")
   684  	}
   685  	if race.Enabled {
   686  		t.Skip("skipping test: -race will catch the race, this test is for the built-in race detection")
   687  	}
   688  	testenv.MustHaveGoRun(t)
   689  	output := runTestProg(t, "testprog", "concurrentMapWrites")
   690  	want := "fatal error: concurrent map writes\n"
   691  	if !strings.HasPrefix(output, want) {
   692  		t.Fatalf("output does not start with %q:\n%s", want, output)
   693  	}
   694  }
   695  func TestConcurrentMapReadWrite(t *testing.T) {
   696  	if !*concurrentMapTest {
   697  		t.Skip("skipping without -run_concurrent_map_tests")
   698  	}
   699  	if race.Enabled {
   700  		t.Skip("skipping test: -race will catch the race, this test is for the built-in race detection")
   701  	}
   702  	testenv.MustHaveGoRun(t)
   703  	output := runTestProg(t, "testprog", "concurrentMapReadWrite")
   704  	want := "fatal error: concurrent map read and map write\n"
   705  	if !strings.HasPrefix(output, want) {
   706  		t.Fatalf("output does not start with %q:\n%s", want, output)
   707  	}
   708  }
   709  func TestConcurrentMapIterateWrite(t *testing.T) {
   710  	if !*concurrentMapTest {
   711  		t.Skip("skipping without -run_concurrent_map_tests")
   712  	}
   713  	if race.Enabled {
   714  		t.Skip("skipping test: -race will catch the race, this test is for the built-in race detection")
   715  	}
   716  	testenv.MustHaveGoRun(t)
   717  	output := runTestProg(t, "testprog", "concurrentMapIterateWrite")
   718  	want := "fatal error: concurrent map iteration and map write\n"
   719  	if !strings.HasPrefix(output, want) {
   720  		t.Fatalf("output does not start with %q:\n%s", want, output)
   721  	}
   722  }
   723  
   724  func TestConcurrentMapWritesIssue69447(t *testing.T) {
   725  	testenv.MustHaveGoRun(t)
   726  	if race.Enabled {
   727  		t.Skip("skipping test: -race will catch the race, this test is for the built-in race detection")
   728  	}
   729  	exe, err := buildTestProg(t, "testprog")
   730  	if err != nil {
   731  		t.Fatal(err)
   732  	}
   733  	for i := 0; i < 200; i++ {
   734  		output := runBuiltTestProg(t, exe, "concurrentMapWrites")
   735  		if output == "" {
   736  			// If we didn't detect an error, that's ok.
   737  			// This case makes this test not flaky like
   738  			// the other ones above.
   739  			// (More correctly, this case makes this test flaky
   740  			// in the other direction, in that it might not
   741  			// detect a problem even if there is one.)
   742  			continue
   743  		}
   744  		want := "fatal error: concurrent map writes\n"
   745  		if !strings.HasPrefix(output, want) {
   746  			t.Fatalf("output does not start with %q:\n%s", want, output)
   747  		}
   748  	}
   749  }
   750  
   751  type point struct {
   752  	x, y *int
   753  }
   754  
   755  func (p *point) negate() {
   756  	*p.x = *p.x * -1
   757  	*p.y = *p.y * -1
   758  }
   759  
   760  // Test for issue #10152.
   761  func TestPanicInlined(t *testing.T) {
   762  	defer func() {
   763  		r := recover()
   764  		if r == nil {
   765  			t.Fatalf("recover failed")
   766  		}
   767  		buf := make([]byte, 2048)
   768  		n := runtime.Stack(buf, false)
   769  		buf = buf[:n]
   770  		if !bytes.Contains(buf, []byte("(*point).negate(")) {
   771  			t.Fatalf("expecting stack trace to contain call to (*point).negate()")
   772  		}
   773  	}()
   774  
   775  	pt := new(point)
   776  	pt.negate()
   777  }
   778  
   779  // Test for issues #3934 and #20018.
   780  // We want to delay exiting until a panic print is complete.
   781  func TestPanicRace(t *testing.T) {
   782  	testenv.MustHaveGoRun(t)
   783  
   784  	exe, err := buildTestProg(t, "testprog")
   785  	if err != nil {
   786  		t.Fatal(err)
   787  	}
   788  
   789  	// The test is intentionally racy, and in my testing does not
   790  	// produce the expected output about 0.05% of the time.
   791  	// So run the program in a loop and only fail the test if we
   792  	// get the wrong output ten times in a row.
   793  	const tries = 10
   794  retry:
   795  	for i := 0; i < tries; i++ {
   796  		got, err := testenv.CleanCmdEnv(exec.Command(exe, "PanicRace")).CombinedOutput()
   797  		if err == nil {
   798  			t.Logf("try %d: program exited successfully, should have failed", i+1)
   799  			continue
   800  		}
   801  
   802  		if i > 0 {
   803  			t.Logf("try %d:\n", i+1)
   804  		}
   805  		t.Logf("%s\n", got)
   806  
   807  		wants := []string{
   808  			"panic: crash",
   809  			"PanicRace",
   810  			"created by ",
   811  		}
   812  		for _, want := range wants {
   813  			if !bytes.Contains(got, []byte(want)) {
   814  				t.Logf("did not find expected string %q", want)
   815  				continue retry
   816  			}
   817  		}
   818  
   819  		// Test generated expected output.
   820  		return
   821  	}
   822  	t.Errorf("test ran %d times without producing expected output", tries)
   823  }
   824  
   825  func TestBadTraceback(t *testing.T) {
   826  	if asan.Enabled || msan.Enabled || race.Enabled {
   827  		t.Skip("skipped test: checkptr mode catches the corruption")
   828  	}
   829  	output := runTestProg(t, "testprog", "BadTraceback")
   830  	for _, want := range []string{
   831  		"unexpected return pc",
   832  		"called from 0xbad",
   833  		"00000bad",    // Smashed LR in hex dump
   834  		"<main.badLR", // Symbolization in hex dump (badLR1 or badLR2)
   835  	} {
   836  		if !strings.Contains(output, want) {
   837  			t.Errorf("output does not contain %q:\n%s", want, output)
   838  		}
   839  	}
   840  }
   841  
   842  func TestTimePprof(t *testing.T) {
   843  	// This test is unreliable on any system in which nanotime
   844  	// calls into libc.
   845  	switch runtime.GOOS {
   846  	case "aix", "darwin", "illumos", "openbsd", "solaris":
   847  		t.Skipf("skipping on %s because nanotime calls libc", runtime.GOOS)
   848  	}
   849  	if race.Enabled || asan.Enabled || msan.Enabled {
   850  		t.Skip("skipping on sanitizers because the sanitizer runtime is external code")
   851  	}
   852  
   853  	// Pass GOTRACEBACK for issue #41120 to try to get more
   854  	// information on timeout.
   855  	fn := runTestProg(t, "testprog", "TimeProf", "GOTRACEBACK=crash")
   856  	fn = strings.TrimSpace(fn)
   857  	defer os.Remove(fn)
   858  
   859  	cmd := testenv.CleanCmdEnv(exec.Command(testenv.GoToolPath(t), "tool", "pprof", "-top", "-nodecount=1", fn))
   860  	cmd.Env = append(cmd.Env, "PPROF_TMPDIR="+os.TempDir())
   861  	top, err := cmd.CombinedOutput()
   862  	t.Logf("%s", top)
   863  	if err != nil {
   864  		t.Error(err)
   865  	} else if bytes.Contains(top, []byte("ExternalCode")) {
   866  		t.Error("profiler refers to ExternalCode")
   867  	}
   868  }
   869  
   870  // Test that runtime.abort does so.
   871  func TestAbort(t *testing.T) {
   872  	// Pass GOTRACEBACK to ensure we get runtime frames.
   873  	output := runTestProg(t, "testprog", "Abort", "GOTRACEBACK=system")
   874  	if want := "runtime.abort"; !strings.Contains(output, want) {
   875  		t.Errorf("output does not contain %q:\n%s", want, output)
   876  	}
   877  	if strings.Contains(output, "BAD") {
   878  		t.Errorf("output contains BAD:\n%s", output)
   879  	}
   880  	// Check that it's a signal traceback.
   881  	want := "PC="
   882  	// For systems that use a breakpoint, check specifically for that.
   883  	switch runtime.GOARCH {
   884  	case "386", "amd64":
   885  		switch runtime.GOOS {
   886  		case "plan9":
   887  			want = "sys: breakpoint"
   888  		case "windows":
   889  			want = "Exception 0x80000003"
   890  		default:
   891  			want = "SIGTRAP"
   892  		}
   893  	}
   894  	if !strings.Contains(output, want) {
   895  		t.Errorf("output does not contain %q:\n%s", want, output)
   896  	}
   897  }
   898  
   899  // For TestRuntimePanic: test a panic in the runtime package without
   900  // involving the testing harness.
   901  func init() {
   902  	if os.Getenv("GO_TEST_RUNTIME_PANIC") == "1" {
   903  		defer func() {
   904  			if r := recover(); r != nil {
   905  				// We expect to crash, so exit 0
   906  				// to indicate failure.
   907  				os.Exit(0)
   908  			}
   909  		}()
   910  		runtime.PanicForTesting(nil, 1)
   911  		// We expect to crash, so exit 0 to indicate failure.
   912  		os.Exit(0)
   913  	}
   914  	if os.Getenv("GO_TEST_RUNTIME_NPE_READMEMSTATS") == "1" {
   915  		runtime.ReadMemStats(nil)
   916  		os.Exit(0)
   917  	}
   918  	if os.Getenv("GO_TEST_RUNTIME_NPE_FUNCMETHOD") == "1" {
   919  		var f *runtime.Func
   920  		_ = f.Entry()
   921  		os.Exit(0)
   922  	}
   923  
   924  }
   925  
   926  func TestRuntimePanic(t *testing.T) {
   927  	cmd := testenv.CleanCmdEnv(exec.Command(testenv.Executable(t), "-test.run=^TestRuntimePanic$"))
   928  	cmd.Env = append(cmd.Env, "GO_TEST_RUNTIME_PANIC=1")
   929  	out, err := cmd.CombinedOutput()
   930  	t.Logf("%s", out)
   931  	if err == nil {
   932  		t.Error("child process did not fail")
   933  	} else if want := "runtime.unexportedPanicForTesting"; !bytes.Contains(out, []byte(want)) {
   934  		t.Errorf("output did not contain expected string %q", want)
   935  	}
   936  }
   937  
   938  func TestTracebackRuntimeFunction(t *testing.T) {
   939  	cmd := testenv.CleanCmdEnv(exec.Command(testenv.Executable(t), "-test.run=^TestTracebackRuntimeFunction$"))
   940  	cmd.Env = append(cmd.Env, "GO_TEST_RUNTIME_NPE_READMEMSTATS=1")
   941  	out, err := cmd.CombinedOutput()
   942  	t.Logf("%s", out)
   943  	if err == nil {
   944  		t.Error("child process did not fail")
   945  	} else if want := "runtime.ReadMemStats"; !bytes.Contains(out, []byte(want)) {
   946  		t.Errorf("output did not contain expected string %q", want)
   947  	}
   948  }
   949  
   950  func TestTracebackRuntimeMethod(t *testing.T) {
   951  	cmd := testenv.CleanCmdEnv(exec.Command(testenv.Executable(t), "-test.run=^TestTracebackRuntimeMethod$"))
   952  	cmd.Env = append(cmd.Env, "GO_TEST_RUNTIME_NPE_FUNCMETHOD=1")
   953  	out, err := cmd.CombinedOutput()
   954  	t.Logf("%s", out)
   955  	if err == nil {
   956  		t.Error("child process did not fail")
   957  	} else if want := "runtime.(*Func).Entry"; !bytes.Contains(out, []byte(want)) {
   958  		t.Errorf("output did not contain expected string %q", want)
   959  	}
   960  }
   961  
   962  // Test that g0 stack overflows are handled gracefully.
   963  func TestG0StackOverflow(t *testing.T) {
   964  	if runtime.GOOS == "ios" {
   965  		testenv.SkipFlaky(t, 62671)
   966  	}
   967  
   968  	if os.Getenv("TEST_G0_STACK_OVERFLOW") != "1" {
   969  		cmd := testenv.CleanCmdEnv(testenv.Command(t, testenv.Executable(t), "-test.run=^TestG0StackOverflow$", "-test.v"))
   970  		cmd.Env = append(cmd.Env, "TEST_G0_STACK_OVERFLOW=1")
   971  		out, err := cmd.CombinedOutput()
   972  		t.Logf("output:\n%s", out)
   973  		// Don't check err since it's expected to crash.
   974  		if n := strings.Count(string(out), "morestack on g0\n"); n != 1 {
   975  			t.Fatalf("%s\n(exit status %v)", out, err)
   976  		}
   977  		if runtime.CrashStackImplemented {
   978  			// check for a stack trace
   979  			want := "runtime.stackOverflow"
   980  			if n := strings.Count(string(out), want); n < 5 {
   981  				t.Errorf("output does not contain %q at least 5 times:\n%s", want, out)
   982  			}
   983  			return // it's not a signal-style traceback
   984  		}
   985  		// Check that it's a signal-style traceback.
   986  		if runtime.GOOS != "windows" {
   987  			if want := "PC="; !strings.Contains(string(out), want) {
   988  				t.Errorf("output does not contain %q:\n%s", want, out)
   989  			}
   990  		}
   991  		return
   992  	}
   993  
   994  	runtime.G0StackOverflow()
   995  }
   996  
   997  // For TestCrashWhileTracing: test a panic without involving the testing
   998  // harness, as we rely on stdout only containing trace output.
   999  func init() {
  1000  	if os.Getenv("TEST_CRASH_WHILE_TRACING") == "1" {
  1001  		trace.Start(os.Stdout)
  1002  		trace.Log(context.Background(), "xyzzy-cat", "xyzzy-msg")
  1003  		panic("yzzyx")
  1004  	}
  1005  }
  1006  
  1007  func TestCrashWhileTracing(t *testing.T) {
  1008  	testenv.MustHaveExec(t)
  1009  
  1010  	cmd := testenv.CleanCmdEnv(testenv.Command(t, testenv.Executable(t)))
  1011  	cmd.Env = append(cmd.Env, "TEST_CRASH_WHILE_TRACING=1")
  1012  	stdOut, err := cmd.StdoutPipe()
  1013  	var errOut bytes.Buffer
  1014  	cmd.Stderr = &errOut
  1015  
  1016  	if err := cmd.Start(); err != nil {
  1017  		t.Fatalf("could not start subprocess: %v", err)
  1018  	}
  1019  	r, err := traceparse.NewReader(stdOut)
  1020  	if err != nil {
  1021  		t.Fatalf("could not create trace.NewReader: %v", err)
  1022  	}
  1023  	var seen bool
  1024  	nSync := 0
  1025  	i := 1
  1026  loop:
  1027  	for ; ; i++ {
  1028  		ev, err := r.ReadEvent()
  1029  		if err != nil {
  1030  			// We may have a broken tail to the trace -- that's OK.
  1031  			// We'll make sure we saw at least one complete generation.
  1032  			if err != io.EOF {
  1033  				t.Logf("error at event %d: %v", i, err)
  1034  			}
  1035  			break loop
  1036  		}
  1037  		switch ev.Kind() {
  1038  		case traceparse.EventSync:
  1039  			nSync = ev.Sync().N
  1040  		case traceparse.EventLog:
  1041  			v := ev.Log()
  1042  			if v.Category == "xyzzy-cat" && v.Message == "xyzzy-msg" {
  1043  				// Should we already stop reading here? More events may come, but
  1044  				// we're not guaranteeing a fully unbroken trace until the last
  1045  				// byte...
  1046  				seen = true
  1047  			}
  1048  		}
  1049  	}
  1050  	if err := cmd.Wait(); err == nil {
  1051  		t.Error("the process should have panicked")
  1052  	}
  1053  	if nSync <= 1 {
  1054  		t.Errorf("expected at least one full generation to have been emitted before the trace was considered broken")
  1055  	}
  1056  	if !seen {
  1057  		t.Errorf("expected one matching log event matching, but none of the %d received trace events match", i)
  1058  	}
  1059  	t.Logf("stderr output:\n%s", errOut.String())
  1060  	needle := "yzzyx\n"
  1061  	if n := strings.Count(errOut.String(), needle); n != 1 {
  1062  		t.Fatalf("did not find expected panic message %q\n(exit status %v)", needle, err)
  1063  	}
  1064  }
  1065  
  1066  // Test that panic message is not clobbered.
  1067  // See issue 30150.
  1068  func TestDoublePanic(t *testing.T) {
  1069  	output := runTestProg(t, "testprog", "DoublePanic", "GODEBUG=clobberfree=1")
  1070  	wants := []string{"panic: XXX", "panic: YYY"}
  1071  	for _, want := range wants {
  1072  		if !strings.Contains(output, want) {
  1073  			t.Errorf("output:\n%s\n\nwant output containing: %s", output, want)
  1074  		}
  1075  	}
  1076  }
  1077  
  1078  // Test that panic while panicking discards error message
  1079  // See issue 52257
  1080  func TestPanicWhilePanicking(t *testing.T) {
  1081  	tests := []struct {
  1082  		Want string
  1083  		Func string
  1084  	}{
  1085  		{
  1086  			"panic while printing panic value: important multi-line\n\terror message",
  1087  			"ErrorPanic",
  1088  		},
  1089  		{
  1090  			"panic while printing panic value: important multi-line\n\tstringer message",
  1091  			"StringerPanic",
  1092  		},
  1093  		{
  1094  			"panic while printing panic value: type",
  1095  			"DoubleErrorPanic",
  1096  		},
  1097  		{
  1098  			"panic while printing panic value: type",
  1099  			"DoubleStringerPanic",
  1100  		},
  1101  		{
  1102  			"panic while printing panic value: type",
  1103  			"CircularPanic",
  1104  		},
  1105  		{
  1106  			"important multi-line\n\tstring message",
  1107  			"StringPanic",
  1108  		},
  1109  		{
  1110  			"nil",
  1111  			"NilPanic",
  1112  		},
  1113  	}
  1114  	for _, x := range tests {
  1115  		output := runTestProg(t, "testprog", x.Func)
  1116  		if !strings.Contains(output, x.Want) {
  1117  			t.Errorf("output does not contain %q:\n%s", x.Want, output)
  1118  		}
  1119  	}
  1120  }
  1121  
  1122  func TestPanicOnUnsafeSlice(t *testing.T) {
  1123  	output := runTestProg(t, "testprog", "panicOnNilAndEleSizeIsZero")
  1124  	// Note: This is normally a panic, but is a throw when checkptr is
  1125  	// enabled.
  1126  	want := "unsafe.Slice: ptr is nil and len is not zero"
  1127  	if !strings.Contains(output, want) {
  1128  		t.Errorf("output does not contain %q:\n%s", want, output)
  1129  	}
  1130  }
  1131  
  1132  func TestNetpollWaiters(t *testing.T) {
  1133  	t.Parallel()
  1134  	output := runTestProg(t, "testprognet", "NetpollWaiters")
  1135  	want := "OK\n"
  1136  	if output != want {
  1137  		t.Fatalf("output is not %q\n%s", want, output)
  1138  	}
  1139  }
  1140  
  1141  func TestFinalizerOrCleanupDeadlock(t *testing.T) {
  1142  	t.Parallel()
  1143  
  1144  	for _, useCleanup := range []bool{false, true} {
  1145  		progName := "Finalizer"
  1146  		want := "runtime.runFinalizers"
  1147  		if useCleanup {
  1148  			progName = "Cleanup"
  1149  			want = "runtime.runCleanups"
  1150  		}
  1151  		t.Run(progName, func(t *testing.T) {
  1152  			// The runtime.runFinalizers/runtime.runCleanups frame should appear in panics, even if
  1153  			// runtime frames are normally hidden (GOTRACEBACK=all).
  1154  			t.Run("Panic", func(t *testing.T) {
  1155  				t.Parallel()
  1156  				output := runTestProg(t, "testprog", progName+"Deadlock", "GOTRACEBACK=all", "GO_TEST_FINALIZER_DEADLOCK=panic")
  1157  				want := want + "()"
  1158  				if !strings.Contains(output, want) {
  1159  					t.Errorf("output does not contain %q:\n%s", want, output)
  1160  				}
  1161  			})
  1162  
  1163  			// The runtime.runFinalizers/runtime.Cleanups frame should appear in runtime.Stack,
  1164  			// even though runtime frames are normally hidden.
  1165  			t.Run("Stack", func(t *testing.T) {
  1166  				t.Parallel()
  1167  				output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=stack")
  1168  				want := want + "()"
  1169  				if !strings.Contains(output, want) {
  1170  					t.Errorf("output does not contain %q:\n%s", want, output)
  1171  				}
  1172  			})
  1173  
  1174  			// The runtime.runFinalizers/runtime.Cleanups frame should appear in goroutine
  1175  			// profiles.
  1176  			t.Run("PprofProto", func(t *testing.T) {
  1177  				t.Parallel()
  1178  				output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_proto")
  1179  
  1180  				p, err := profile.Parse(strings.NewReader(output))
  1181  				if err != nil {
  1182  					// Logging the binary proto data is not very nice, but it might
  1183  					// be a text error message instead.
  1184  					t.Logf("Output: %s", output)
  1185  					t.Fatalf("Error parsing proto output: %v", err)
  1186  				}
  1187  				for _, s := range p.Sample {
  1188  					for _, loc := range s.Location {
  1189  						for _, line := range loc.Line {
  1190  							if line.Function.Name == want {
  1191  								// Done!
  1192  								return
  1193  							}
  1194  						}
  1195  					}
  1196  				}
  1197  				t.Errorf("Profile does not contain %q:\n%s", want, p)
  1198  			})
  1199  
  1200  			// The runtime.runFinalizers/runtime.runCleanups frame should appear in goroutine
  1201  			// profiles (debug=1).
  1202  			t.Run("PprofDebug1", func(t *testing.T) {
  1203  				t.Parallel()
  1204  				output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_debug1")
  1205  				want := want + "+"
  1206  				if !strings.Contains(output, want) {
  1207  					t.Errorf("output does not contain %q:\n%s", want, output)
  1208  				}
  1209  			})
  1210  
  1211  			// The runtime.runFinalizers/runtime.runCleanups frame should appear in goroutine
  1212  			// profiles (debug=2).
  1213  			t.Run("PprofDebug2", func(t *testing.T) {
  1214  				t.Parallel()
  1215  				output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_debug2")
  1216  				want := want + "()"
  1217  				if !strings.Contains(output, want) {
  1218  					t.Errorf("output does not contain %q:\n%s", want, output)
  1219  				}
  1220  			})
  1221  		})
  1222  	}
  1223  }
  1224  
  1225  func TestSynctestCondSignalFromNoBubble(t *testing.T) {
  1226  	for _, test := range []string{
  1227  		"SynctestCond/signal/no_bubble",
  1228  		"SynctestCond/broadcast/no_bubble",
  1229  		"SynctestCond/signal/other_bubble",
  1230  		"SynctestCond/broadcast/other_bubble",
  1231  	} {
  1232  		t.Run(test, func(t *testing.T) {
  1233  			output := runTestProg(t, "testprog", test)
  1234  			want := "fatal error: semaphore wake of synctest goroutine from outside bubble"
  1235  			if !strings.Contains(output, want) {
  1236  				t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
  1237  			}
  1238  		})
  1239  	}
  1240  }
  1241  

View as plain text