Source file src/runtime/traceruntime.go

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Runtime -> tracer API.
     6  
     7  package runtime
     8  
     9  import (
    10  	"internal/runtime/atomic"
    11  	"internal/trace/tracev2"
    12  	_ "unsafe" // for go:linkname
    13  )
    14  
    15  // gTraceState is per-G state for the tracer.
    16  type gTraceState struct {
    17  	traceSchedResourceState
    18  }
    19  
    20  // reset resets the gTraceState for a new goroutine.
    21  func (s *gTraceState) reset() {
    22  	s.seq = [2]uint64{}
    23  	// N.B. s.statusTraced is managed and cleared separately.
    24  }
    25  
    26  // mTraceState is per-M state for the tracer.
    27  type mTraceState struct {
    28  	seqlock       atomic.Uintptr                       // seqlock indicating that this M is writing to a trace buffer.
    29  	buf           [2][tracev2.NumExperiments]*traceBuf // Per-M traceBuf for writing. Indexed by trace.gen%2.
    30  	link          *m                                   // Snapshot of alllink or freelink.
    31  	reentered     uint32                               // Whether we've reentered tracing from within tracing.
    32  	entryGen      uintptr                              // The generation value on first entry.
    33  	oldthrowsplit bool                                 // gp.throwsplit upon calling traceLocker.writer. For debugging.
    34  }
    35  
    36  // pTraceState is per-P state for the tracer.
    37  type pTraceState struct {
    38  	traceSchedResourceState
    39  
    40  	// mSyscallID is the ID of the M this was bound to before entering a syscall.
    41  	mSyscallID int64
    42  
    43  	// maySweep indicates the sweep events should be traced.
    44  	// This is used to defer the sweep start event until a span
    45  	// has actually been swept.
    46  	maySweep bool
    47  
    48  	// inSweep indicates that at least one sweep event has been traced.
    49  	inSweep bool
    50  
    51  	// swept and reclaimed track the number of bytes swept and reclaimed
    52  	// by sweeping in the current sweep loop (while maySweep was true).
    53  	swept, reclaimed uintptr
    54  }
    55  
    56  // traceLockInit initializes global trace locks.
    57  func traceLockInit() {
    58  	// Sharing a lock rank here is fine because they should never be accessed
    59  	// together. If they are, we want to find out immediately.
    60  	lockInit(&trace.stringTab[0].lock, lockRankTraceStrings)
    61  	lockInit(&trace.stringTab[0].tab.mem.lock, lockRankTraceStrings)
    62  	lockInit(&trace.stringTab[1].lock, lockRankTraceStrings)
    63  	lockInit(&trace.stringTab[1].tab.mem.lock, lockRankTraceStrings)
    64  	lockInit(&trace.stackTab[0].tab.mem.lock, lockRankTraceStackTab)
    65  	lockInit(&trace.stackTab[1].tab.mem.lock, lockRankTraceStackTab)
    66  	lockInit(&trace.typeTab[0].tab.mem.lock, lockRankTraceTypeTab)
    67  	lockInit(&trace.typeTab[1].tab.mem.lock, lockRankTraceTypeTab)
    68  	lockInit(&trace.lock, lockRankTrace)
    69  }
    70  
    71  // lockRankMayTraceFlush records the lock ranking effects of a
    72  // potential call to traceFlush.
    73  //
    74  // nosplit because traceAcquire is nosplit.
    75  //
    76  //go:nosplit
    77  func lockRankMayTraceFlush() {
    78  	lockWithRankMayAcquire(&trace.lock, getLockRank(&trace.lock))
    79  }
    80  
    81  // traceBlockReason is an enumeration of reasons a goroutine might block.
    82  // This is the interface the rest of the runtime uses to tell the
    83  // tracer why a goroutine blocked. The tracer then propagates this information
    84  // into the trace however it sees fit.
    85  //
    86  // Note that traceBlockReasons should not be compared, since reasons that are
    87  // distinct by name may *not* be distinct by value.
    88  type traceBlockReason uint8
    89  
    90  const (
    91  	traceBlockGeneric traceBlockReason = iota
    92  	traceBlockForever
    93  	traceBlockNet
    94  	traceBlockSelect
    95  	traceBlockCondWait
    96  	traceBlockSync
    97  	traceBlockChanSend
    98  	traceBlockChanRecv
    99  	traceBlockGCMarkAssist
   100  	traceBlockGCSweep
   101  	traceBlockSystemGoroutine
   102  	traceBlockPreempted
   103  	traceBlockDebugCall
   104  	traceBlockUntilGCEnds
   105  	traceBlockSleep
   106  	traceBlockGCWeakToStrongWait
   107  	traceBlockSynctest
   108  )
   109  
   110  var traceBlockReasonStrings = [...]string{
   111  	traceBlockGeneric:            "unspecified",
   112  	traceBlockForever:            "forever",
   113  	traceBlockNet:                "network",
   114  	traceBlockSelect:             "select",
   115  	traceBlockCondWait:           "sync.(*Cond).Wait",
   116  	traceBlockSync:               "sync",
   117  	traceBlockChanSend:           "chan send",
   118  	traceBlockChanRecv:           "chan receive",
   119  	traceBlockGCMarkAssist:       "GC mark assist wait for work",
   120  	traceBlockGCSweep:            "GC background sweeper wait",
   121  	traceBlockSystemGoroutine:    "system goroutine wait",
   122  	traceBlockPreempted:          "preempted",
   123  	traceBlockDebugCall:          "wait for debug call",
   124  	traceBlockUntilGCEnds:        "wait until GC ends",
   125  	traceBlockSleep:              "sleep",
   126  	traceBlockGCWeakToStrongWait: "GC weak to strong wait",
   127  	traceBlockSynctest:           "synctest",
   128  }
   129  
   130  // traceGoStopReason is an enumeration of reasons a goroutine might yield.
   131  //
   132  // Note that traceGoStopReasons should not be compared, since reasons that are
   133  // distinct by name may *not* be distinct by value.
   134  type traceGoStopReason uint8
   135  
   136  const (
   137  	traceGoStopGeneric traceGoStopReason = iota
   138  	traceGoStopGoSched
   139  	traceGoStopPreempted
   140  )
   141  
   142  var traceGoStopReasonStrings = [...]string{
   143  	traceGoStopGeneric:   "unspecified",
   144  	traceGoStopGoSched:   "runtime.Gosched",
   145  	traceGoStopPreempted: "preempted",
   146  }
   147  
   148  // traceEnabled returns true if the trace is currently enabled.
   149  //
   150  //go:nosplit
   151  func traceEnabled() bool {
   152  	return trace.enabled
   153  }
   154  
   155  // traceAllocFreeEnabled returns true if the trace is currently enabled
   156  // and alloc/free events are also enabled.
   157  //
   158  //go:nosplit
   159  func traceAllocFreeEnabled() bool {
   160  	return trace.enabledWithAllocFree
   161  }
   162  
   163  // traceShuttingDown returns true if the trace is currently shutting down.
   164  func traceShuttingDown() bool {
   165  	return trace.shutdown.Load()
   166  }
   167  
   168  // traceLocker represents an M writing trace events. While a traceLocker value
   169  // is valid, the tracer observes all operations on the G/M/P or trace events being
   170  // written as happening atomically.
   171  type traceLocker struct {
   172  	mp  *m
   173  	gen uintptr
   174  }
   175  
   176  // debugTraceReentrancy checks if the trace is reentrant.
   177  //
   178  // This is optional because throwing in a function makes it instantly
   179  // not inlineable, and we want traceAcquire to be inlineable for
   180  // low overhead when the trace is disabled.
   181  const debugTraceReentrancy = false
   182  
   183  // traceAcquire prepares this M for writing one or more trace events.
   184  //
   185  // nosplit because it's called on the syscall path when stack movement is forbidden.
   186  //
   187  //go:nosplit
   188  func traceAcquire() traceLocker {
   189  	if !traceEnabled() {
   190  		return traceLocker{}
   191  	}
   192  	return traceAcquireEnabled()
   193  }
   194  
   195  // traceAcquireEnabled is the traceEnabled path for traceAcquire. It's explicitly
   196  // broken out to make traceAcquire inlineable to keep the overhead of the tracer
   197  // when it's disabled low.
   198  //
   199  // nosplit because it's called by traceAcquire, which is nosplit.
   200  //
   201  //go:nosplit
   202  func traceAcquireEnabled() traceLocker {
   203  	// Any time we acquire a traceLocker, we may flush a trace buffer. But
   204  	// buffer flushes are rare. Record the lock edge even if it doesn't happen
   205  	// this time.
   206  	lockRankMayTraceFlush()
   207  
   208  	// Prevent preemption.
   209  	mp := acquirem()
   210  
   211  	// Check if we're already tracing. It's safe to be reentrant in general,
   212  	// because this function (and the invariants of traceLocker.writer) ensure
   213  	// that it is.
   214  	if mp.trace.seqlock.Load()%2 == 1 {
   215  		mp.trace.reentered++
   216  		return traceLocker{mp, mp.trace.entryGen}
   217  	}
   218  
   219  	// Acquire the trace seqlock. This prevents traceAdvance from moving forward
   220  	// until all Ms are observed to be outside of their seqlock critical section.
   221  	//
   222  	// Note: The seqlock is mutated here and also in traceCPUSample. If you update
   223  	// usage of the seqlock here, make sure to also look at what traceCPUSample is
   224  	// doing.
   225  	seq := mp.trace.seqlock.Add(1)
   226  	if debugTraceReentrancy && seq%2 != 1 {
   227  		throw("bad use of trace.seqlock")
   228  	}
   229  
   230  	// N.B. This load of gen appears redundant with the one in traceEnabled.
   231  	// However, it's very important that the gen we use for writing to the trace
   232  	// is acquired under a traceLocker so traceAdvance can make sure no stale
   233  	// gen values are being used.
   234  	//
   235  	// Because we're doing this load again, it also means that the trace
   236  	// might end up being disabled when we load it. In that case we need to undo
   237  	// what we did and bail.
   238  	gen := trace.gen.Load()
   239  	if gen == 0 {
   240  		mp.trace.seqlock.Add(1)
   241  		releasem(mp)
   242  		return traceLocker{}
   243  	}
   244  	mp.trace.entryGen = gen
   245  	return traceLocker{mp, gen}
   246  }
   247  
   248  // ok returns true if the traceLocker is valid (i.e. tracing is enabled).
   249  //
   250  // nosplit because it's called on the syscall path when stack movement is forbidden.
   251  //
   252  //go:nosplit
   253  func (tl traceLocker) ok() bool {
   254  	return tl.gen != 0
   255  }
   256  
   257  // traceRelease indicates that this M is done writing trace events.
   258  //
   259  // nosplit because it's called on the syscall path when stack movement is forbidden.
   260  //
   261  //go:nosplit
   262  func traceRelease(tl traceLocker) {
   263  	if tl.mp.trace.reentered > 0 {
   264  		tl.mp.trace.reentered--
   265  	} else {
   266  		seq := tl.mp.trace.seqlock.Add(1)
   267  		if debugTraceReentrancy && seq%2 != 0 {
   268  			print("runtime: seq=", seq, "\n")
   269  			throw("bad use of trace.seqlock")
   270  		}
   271  	}
   272  	releasem(tl.mp)
   273  }
   274  
   275  // traceExitingSyscall marks a goroutine as exiting the syscall slow path.
   276  //
   277  // Must be paired with a traceExitedSyscall call.
   278  func traceExitingSyscall() {
   279  	trace.exitingSyscall.Add(1)
   280  }
   281  
   282  // traceExitedSyscall marks a goroutine as having exited the syscall slow path.
   283  func traceExitedSyscall() {
   284  	trace.exitingSyscall.Add(-1)
   285  }
   286  
   287  // Gomaxprocs emits a ProcsChange event.
   288  func (tl traceLocker) Gomaxprocs(procs int32) {
   289  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvProcsChange, traceArg(procs), tl.stack(1))
   290  }
   291  
   292  // ProcStart traces a ProcStart event.
   293  //
   294  // Must be called with a valid P.
   295  func (tl traceLocker) ProcStart() {
   296  	pp := tl.mp.p.ptr()
   297  	// Procs are typically started within the scheduler when there is no user goroutine. If there is a user goroutine,
   298  	// it must be in _Gsyscall because the only time a goroutine is allowed to have its Proc moved around from under it
   299  	// is during a syscall.
   300  	tl.eventWriter(tracev2.GoSyscall, tracev2.ProcIdle).event(tracev2.EvProcStart, traceArg(pp.id), pp.trace.nextSeq(tl.gen))
   301  }
   302  
   303  // ProcStop traces a ProcStop event.
   304  func (tl traceLocker) ProcStop(pp *p) {
   305  	// The only time a goroutine is allowed to have its Proc moved around
   306  	// from under it is during a syscall.
   307  	tl.eventWriter(tracev2.GoSyscall, tracev2.ProcRunning).event(tracev2.EvProcStop)
   308  }
   309  
   310  // GCActive traces a GCActive event.
   311  //
   312  // Must be emitted by an actively running goroutine on an active P. This restriction can be changed
   313  // easily and only depends on where it's currently called.
   314  func (tl traceLocker) GCActive() {
   315  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCActive, traceArg(trace.seqGC))
   316  	// N.B. Only one GC can be running at a time, so this is naturally
   317  	// serialized by the caller.
   318  	trace.seqGC++
   319  }
   320  
   321  // GCStart traces a GCBegin event.
   322  //
   323  // Must be emitted by an actively running goroutine on an active P. This restriction can be changed
   324  // easily and only depends on where it's currently called.
   325  func (tl traceLocker) GCStart() {
   326  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCBegin, traceArg(trace.seqGC), tl.stack(3))
   327  	// N.B. Only one GC can be running at a time, so this is naturally
   328  	// serialized by the caller.
   329  	trace.seqGC++
   330  }
   331  
   332  // GCDone traces a GCEnd event.
   333  //
   334  // Must be emitted by an actively running goroutine on an active P. This restriction can be changed
   335  // easily and only depends on where it's currently called.
   336  func (tl traceLocker) GCDone() {
   337  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCEnd, traceArg(trace.seqGC))
   338  	// N.B. Only one GC can be running at a time, so this is naturally
   339  	// serialized by the caller.
   340  	trace.seqGC++
   341  }
   342  
   343  // STWStart traces a STWBegin event.
   344  func (tl traceLocker) STWStart(reason stwReason) {
   345  	// Although the current P may be in _Pgcstop here, we model the P as running during the STW. This deviates from the
   346  	// runtime's state tracking, but it's more accurate and doesn't result in any loss of information.
   347  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvSTWBegin, tl.string(reason.String()), tl.stack(2))
   348  }
   349  
   350  // STWDone traces a STWEnd event.
   351  func (tl traceLocker) STWDone() {
   352  	// Although the current P may be in _Pgcstop here, we model the P as running during the STW. This deviates from the
   353  	// runtime's state tracking, but it's more accurate and doesn't result in any loss of information.
   354  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvSTWEnd)
   355  }
   356  
   357  // GCSweepStart prepares to trace a sweep loop. This does not
   358  // emit any events until traceGCSweepSpan is called.
   359  //
   360  // GCSweepStart must be paired with traceGCSweepDone and there
   361  // must be no preemption points between these two calls.
   362  //
   363  // Must be called with a valid P.
   364  func (tl traceLocker) GCSweepStart() {
   365  	// Delay the actual GCSweepBegin event until the first span
   366  	// sweep. If we don't sweep anything, don't emit any events.
   367  	pp := tl.mp.p.ptr()
   368  	if pp.trace.maySweep {
   369  		throw("double traceGCSweepStart")
   370  	}
   371  	pp.trace.maySweep, pp.trace.swept, pp.trace.reclaimed = true, 0, 0
   372  }
   373  
   374  // GCSweepSpan traces the sweep of a single span. If this is
   375  // the first span swept since traceGCSweepStart was called, this
   376  // will emit a GCSweepBegin event.
   377  //
   378  // This may be called outside a traceGCSweepStart/traceGCSweepDone
   379  // pair; however, it will not emit any trace events in this case.
   380  //
   381  // Must be called with a valid P.
   382  func (tl traceLocker) GCSweepSpan(bytesSwept uintptr) {
   383  	pp := tl.mp.p.ptr()
   384  	if pp.trace.maySweep {
   385  		if pp.trace.swept == 0 {
   386  			tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCSweepBegin, tl.stack(1))
   387  			pp.trace.inSweep = true
   388  		}
   389  		pp.trace.swept += bytesSwept
   390  	}
   391  }
   392  
   393  // GCSweepDone finishes tracing a sweep loop. If any memory was
   394  // swept (i.e. traceGCSweepSpan emitted an event) then this will emit
   395  // a GCSweepEnd event.
   396  //
   397  // Must be called with a valid P.
   398  func (tl traceLocker) GCSweepDone() {
   399  	pp := tl.mp.p.ptr()
   400  	if !pp.trace.maySweep {
   401  		throw("missing traceGCSweepStart")
   402  	}
   403  	if pp.trace.inSweep {
   404  		tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCSweepEnd, traceArg(pp.trace.swept), traceArg(pp.trace.reclaimed))
   405  		pp.trace.inSweep = false
   406  	}
   407  	pp.trace.maySweep = false
   408  }
   409  
   410  // GCMarkAssistStart emits a MarkAssistBegin event.
   411  func (tl traceLocker) GCMarkAssistStart() {
   412  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCMarkAssistBegin, tl.stack(1))
   413  }
   414  
   415  // GCMarkAssistDone emits a MarkAssistEnd event.
   416  func (tl traceLocker) GCMarkAssistDone() {
   417  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCMarkAssistEnd)
   418  }
   419  
   420  // GoCreate emits a GoCreate event.
   421  func (tl traceLocker) GoCreate(newg *g, pc uintptr, blocked bool) {
   422  	newg.trace.setStatusTraced(tl.gen)
   423  	ev := tracev2.EvGoCreate
   424  	if blocked {
   425  		ev = tracev2.EvGoCreateBlocked
   426  	}
   427  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(ev, traceArg(newg.goid), tl.startPC(pc), tl.stack(2))
   428  }
   429  
   430  // GoStart emits a GoStart event.
   431  //
   432  // Must be called with a valid P.
   433  func (tl traceLocker) GoStart() {
   434  	gp := getg().m.curg
   435  	pp := gp.m.p
   436  	w := tl.eventWriter(tracev2.GoRunnable, tracev2.ProcRunning)
   437  	w.event(tracev2.EvGoStart, traceArg(gp.goid), gp.trace.nextSeq(tl.gen))
   438  	if pp.ptr().gcMarkWorkerMode != gcMarkWorkerNotWorker {
   439  		w.event(tracev2.EvGoLabel, trace.markWorkerLabels[tl.gen%2][pp.ptr().gcMarkWorkerMode])
   440  	}
   441  }
   442  
   443  // GoEnd emits a GoDestroy event.
   444  //
   445  // TODO(mknyszek): Rename this to GoDestroy.
   446  func (tl traceLocker) GoEnd() {
   447  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoDestroy)
   448  }
   449  
   450  // GoSched emits a GoStop event with a GoSched reason.
   451  func (tl traceLocker) GoSched() {
   452  	tl.GoStop(traceGoStopGoSched)
   453  }
   454  
   455  // GoPreempt emits a GoStop event with a GoPreempted reason.
   456  func (tl traceLocker) GoPreempt() {
   457  	tl.GoStop(traceGoStopPreempted)
   458  }
   459  
   460  // GoStop emits a GoStop event with the provided reason.
   461  func (tl traceLocker) GoStop(reason traceGoStopReason) {
   462  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoStop, trace.goStopReasons[tl.gen%2][reason], tl.stack(0))
   463  }
   464  
   465  // GoPark emits a GoBlock event with the provided reason.
   466  //
   467  // TODO(mknyszek): Replace traceBlockReason with waitReason. It's silly
   468  // that we have both, and waitReason is way more descriptive.
   469  func (tl traceLocker) GoPark(reason traceBlockReason, skip int) {
   470  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoBlock, trace.goBlockReasons[tl.gen%2][reason], tl.stack(skip))
   471  }
   472  
   473  // GoUnpark emits a GoUnblock event.
   474  func (tl traceLocker) GoUnpark(gp *g, skip int) {
   475  	// Emit a GoWaiting status if necessary for the unblocked goroutine.
   476  	tl.emitUnblockStatus(gp, tl.gen)
   477  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoUnblock, traceArg(gp.goid), gp.trace.nextSeq(tl.gen), tl.stack(skip))
   478  }
   479  
   480  // GoSwitch emits a GoSwitch event. If destroy is true, the calling goroutine
   481  // is simultaneously being destroyed.
   482  func (tl traceLocker) GoSwitch(nextg *g, destroy bool) {
   483  	// Emit a GoWaiting status if necessary for the unblocked goroutine.
   484  	tl.emitUnblockStatus(nextg, tl.gen)
   485  	w := tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning)
   486  	ev := tracev2.EvGoSwitch
   487  	if destroy {
   488  		ev = tracev2.EvGoSwitchDestroy
   489  	}
   490  	w.event(ev, traceArg(nextg.goid), nextg.trace.nextSeq(tl.gen))
   491  }
   492  
   493  // emitUnblockStatus emits a GoStatus GoWaiting event for a goroutine about to be
   494  // unblocked to the trace writer.
   495  func (tl traceLocker) emitUnblockStatus(gp *g, gen uintptr) {
   496  	if !gp.trace.statusWasTraced(gen) && gp.trace.acquireStatus(gen) {
   497  		// TODO(go.dev/issue/65634): Although it would be nice to add a stack trace here of gp,
   498  		// we cannot safely do so. gp is in _Gwaiting and so we don't have ownership of its stack.
   499  		// We can fix this by acquiring the goroutine's scan bit.
   500  		tl.writer().writeGoStatus(gp.goid, -1, tracev2.GoWaiting, gp.inMarkAssist, 0).end()
   501  	}
   502  }
   503  
   504  // GoSysCall emits a GoSyscallBegin event.
   505  //
   506  // Must be called with a valid P.
   507  func (tl traceLocker) GoSysCall() {
   508  	// Scribble down the M that the P is currently attached to.
   509  	pp := tl.mp.p.ptr()
   510  	pp.trace.mSyscallID = int64(tl.mp.procid)
   511  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoSyscallBegin, pp.trace.nextSeq(tl.gen), tl.stack(1))
   512  }
   513  
   514  // GoSysExit emits a GoSyscallEnd event, possibly along with a GoSyscallBlocked event
   515  // if lostP is true.
   516  //
   517  // lostP must be true in all cases that a goroutine loses its P during a syscall.
   518  // This means it's not sufficient to check if it has no P. In particular, it needs to be
   519  // true in the following cases:
   520  // - The goroutine lost its P, it ran some other code, and then got it back. It's now running with that P.
   521  // - The goroutine lost its P and was unable to reacquire it, and is now running without a P.
   522  // - The goroutine lost its P and acquired a different one, and is now running with that P.
   523  func (tl traceLocker) GoSysExit(lostP bool) {
   524  	ev := tracev2.EvGoSyscallEnd
   525  	procStatus := tracev2.ProcSyscall // Procs implicitly enter tracev2.ProcSyscall on GoSyscallBegin.
   526  	if lostP {
   527  		ev = tracev2.EvGoSyscallEndBlocked
   528  		procStatus = tracev2.ProcRunning // If a G has a P when emitting this event, it reacquired a P and is indeed running.
   529  	} else {
   530  		tl.mp.p.ptr().trace.mSyscallID = -1
   531  	}
   532  	tl.eventWriter(tracev2.GoSyscall, procStatus).event(ev)
   533  }
   534  
   535  // ProcSteal indicates that our current M stole a P from another M.
   536  //
   537  // The caller must have ownership of pp.
   538  func (tl traceLocker) ProcSteal(pp *p) {
   539  	// Grab the M ID we stole from.
   540  	mStolenFrom := pp.trace.mSyscallID
   541  	pp.trace.mSyscallID = -1
   542  
   543  	// Emit the status of the P we're stealing. We may be just about to do this when creating the event
   544  	// writer but it's not guaranteed, even if we're stealing from a syscall. Although it might seem like
   545  	// from a syscall context we're always stealing a P for ourselves, we may have not wired it up yet (so
   546  	// it wouldn't be visible to eventWriter) or we may not even intend to wire it up to ourselves
   547  	// at all and plan to hand it back to the runtime.
   548  	if !pp.trace.statusWasTraced(tl.gen) && pp.trace.acquireStatus(tl.gen) {
   549  		// Careful: don't use the event writer. We never want status or in-progress events
   550  		// to trigger more in-progress events.
   551  		tl.writer().writeProcStatus(uint64(pp.id), tracev2.ProcSyscallAbandoned, pp.trace.inSweep).end()
   552  	}
   553  
   554  	// The status of the proc and goroutine, if we need to emit one here, is not evident from the
   555  	// context of just emitting this event alone. There are two cases. Either we're trying to steal
   556  	// the P just to get its attention (e.g. STW or sysmon retake) or we're trying to steal a P for
   557  	// ourselves specifically to keep running. The two contexts look different, but can be summarized
   558  	// fairly succinctly. In the former, we're a regular running goroutine and proc, if we have either.
   559  	// In the latter, we're a goroutine in a syscall.
   560  	goStatus := tracev2.GoRunning
   561  	procStatus := tracev2.ProcRunning
   562  	if tl.mp.curg != nil && tl.mp.curg.syscallsp != 0 {
   563  		goStatus = tracev2.GoSyscall
   564  		procStatus = tracev2.ProcSyscallAbandoned
   565  	}
   566  	tl.eventWriter(goStatus, procStatus).event(tracev2.EvProcSteal, traceArg(pp.id), pp.trace.nextSeq(tl.gen), traceArg(mStolenFrom))
   567  }
   568  
   569  // HeapAlloc emits a HeapAlloc event.
   570  func (tl traceLocker) HeapAlloc(live uint64) {
   571  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvHeapAlloc, traceArg(live))
   572  }
   573  
   574  // HeapGoal reads the current heap goal and emits a HeapGoal event.
   575  func (tl traceLocker) HeapGoal() {
   576  	heapGoal := gcController.heapGoal()
   577  	// The heapGoal calculations will result in strange numbers if the GC if off. See go.dev/issue/63864.
   578  	// Check gcPercent before using the heapGoal in the trace.
   579  	if heapGoal == ^uint64(0) || gcController.gcPercent.Load() < 0 {
   580  		// Heap-based triggering is disabled.
   581  		heapGoal = 0
   582  	}
   583  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvHeapGoal, traceArg(heapGoal))
   584  }
   585  
   586  // GoCreateSyscall indicates that a goroutine has transitioned from dead to GoSyscall.
   587  //
   588  // Unlike GoCreate, the caller must be running on gp.
   589  //
   590  // This occurs when C code calls into Go. On pthread platforms it occurs only when
   591  // a C thread calls into Go code for the first time.
   592  func (tl traceLocker) GoCreateSyscall(gp *g) {
   593  	// N.B. We should never trace a status for this goroutine (which we're currently running on),
   594  	// since we want this to appear like goroutine creation.
   595  	gp.trace.setStatusTraced(tl.gen)
   596  
   597  	// We might have a P left over on the thread from the last cgo callback,
   598  	// but in a syscall context, it is NOT ours. Act as if we do not have a P,
   599  	// and don't record a status.
   600  	tl.rawEventWriter().event(tracev2.EvGoCreateSyscall, traceArg(gp.goid))
   601  }
   602  
   603  // GoDestroySyscall indicates that a goroutine has transitioned from GoSyscall to dead.
   604  //
   605  // This occurs when Go code returns back to C. On pthread platforms it occurs only when
   606  // the C thread is destroyed.
   607  func (tl traceLocker) GoDestroySyscall() {
   608  	// Write the status for the goroutine if necessary.
   609  	if gp := tl.mp.curg; gp != nil && !gp.trace.statusWasTraced(tl.gen) && gp.trace.acquireStatus(tl.gen) {
   610  		tl.writer().writeGoStatus(gp.goid, int64(tl.mp.procid), tracev2.GoSyscall, false, 0 /* no stack */).end()
   611  	}
   612  
   613  	// We might have a P left over on the thread from the last cgo callback,
   614  	// but in a syscall context, it is NOT ours. Act as if we do not have a P,
   615  	// and don't record a status.
   616  	tl.rawEventWriter().event(tracev2.EvGoDestroySyscall)
   617  }
   618  
   619  // To access runtime functions from runtime/trace.
   620  // See runtime/trace/annotation.go
   621  
   622  // trace_userTaskCreate emits a UserTaskCreate event.
   623  //
   624  //go:linkname trace_userTaskCreate runtime/trace.userTaskCreate
   625  func trace_userTaskCreate(id, parentID uint64, taskType string) {
   626  	tl := traceAcquire()
   627  	if !tl.ok() {
   628  		// Need to do this check because the caller won't have it.
   629  		return
   630  	}
   631  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvUserTaskBegin, traceArg(id), traceArg(parentID), tl.string(taskType), tl.stack(3))
   632  	traceRelease(tl)
   633  }
   634  
   635  // trace_userTaskEnd emits a UserTaskEnd event.
   636  //
   637  //go:linkname trace_userTaskEnd runtime/trace.userTaskEnd
   638  func trace_userTaskEnd(id uint64) {
   639  	tl := traceAcquire()
   640  	if !tl.ok() {
   641  		// Need to do this check because the caller won't have it.
   642  		return
   643  	}
   644  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvUserTaskEnd, traceArg(id), tl.stack(2))
   645  	traceRelease(tl)
   646  }
   647  
   648  // trace_userRegion emits a UserRegionBegin or UserRegionEnd event,
   649  // depending on mode (0 == Begin, 1 == End).
   650  //
   651  // TODO(mknyszek): Just make this two functions.
   652  //
   653  //go:linkname trace_userRegion runtime/trace.userRegion
   654  func trace_userRegion(id, mode uint64, name string) {
   655  	tl := traceAcquire()
   656  	if !tl.ok() {
   657  		// Need to do this check because the caller won't have it.
   658  		return
   659  	}
   660  	var ev tracev2.EventType
   661  	switch mode {
   662  	case 0:
   663  		ev = tracev2.EvUserRegionBegin
   664  	case 1:
   665  		ev = tracev2.EvUserRegionEnd
   666  	default:
   667  		return
   668  	}
   669  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(ev, traceArg(id), tl.string(name), tl.stack(3))
   670  	traceRelease(tl)
   671  }
   672  
   673  // trace_userLog emits a UserRegionBegin or UserRegionEnd event.
   674  //
   675  //go:linkname trace_userLog runtime/trace.userLog
   676  func trace_userLog(id uint64, category, message string) {
   677  	tl := traceAcquire()
   678  	if !tl.ok() {
   679  		// Need to do this check because the caller won't have it.
   680  		return
   681  	}
   682  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvUserLog, traceArg(id), tl.string(category), tl.uniqueString(message), tl.stack(3))
   683  	traceRelease(tl)
   684  }
   685  
   686  // traceThreadDestroy is called when a thread is removed from
   687  // sched.freem.
   688  //
   689  // mp must not be able to emit trace events anymore.
   690  //
   691  // sched.lock must be held to synchronize with traceAdvance.
   692  func traceThreadDestroy(mp *m) {
   693  	assertLockHeld(&sched.lock)
   694  
   695  	// Flush all outstanding buffers to maintain the invariant
   696  	// that an M only has active buffers while on sched.freem
   697  	// or allm.
   698  	//
   699  	// Perform a traceAcquire/traceRelease on behalf of mp to
   700  	// synchronize with the tracer trying to flush our buffer
   701  	// as well.
   702  	seq := mp.trace.seqlock.Add(1)
   703  	if debugTraceReentrancy && seq%2 != 1 {
   704  		throw("bad use of trace.seqlock")
   705  	}
   706  	systemstack(func() {
   707  		lock(&trace.lock)
   708  		for i := range mp.trace.buf {
   709  			for exp, buf := range mp.trace.buf[i] {
   710  				if buf != nil {
   711  					// N.B. traceBufFlush accepts a generation, but it
   712  					// really just cares about gen%2.
   713  					traceBufFlush(buf, uintptr(i))
   714  					mp.trace.buf[i][exp] = nil
   715  				}
   716  			}
   717  		}
   718  		unlock(&trace.lock)
   719  	})
   720  	seq1 := mp.trace.seqlock.Add(1)
   721  	if seq1 != seq+1 {
   722  		print("runtime: seq1=", seq1, "\n")
   723  		throw("bad use of trace.seqlock")
   724  	}
   725  }
   726  

View as plain text