Source file src/runtime/runtime2.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"internal/chacha8rand"
    10  	"internal/goarch"
    11  	"internal/runtime/atomic"
    12  	"internal/runtime/sys"
    13  	"unsafe"
    14  )
    15  
    16  // defined constants
    17  const (
    18  	// G status
    19  	//
    20  	// Beyond indicating the general state of a G, the G status
    21  	// acts like a lock on the goroutine's stack (and hence its
    22  	// ability to execute user code).
    23  	//
    24  	// If you add to this list, add to the list
    25  	// of "okay during garbage collection" status
    26  	// in mgcmark.go too.
    27  	//
    28  	// TODO(austin): The _Gscan bit could be much lighter-weight.
    29  	// For example, we could choose not to run _Gscanrunnable
    30  	// goroutines found in the run queue, rather than CAS-looping
    31  	// until they become _Grunnable. And transitions like
    32  	// _Gscanwaiting -> _Gscanrunnable are actually okay because
    33  	// they don't affect stack ownership.
    34  
    35  	// _Gidle means this goroutine was just allocated and has not
    36  	// yet been initialized.
    37  	_Gidle = iota // 0
    38  
    39  	// _Grunnable means this goroutine is on a run queue. It is
    40  	// not currently executing user code. The stack is not owned.
    41  	_Grunnable // 1
    42  
    43  	// _Grunning means this goroutine may execute user code. The
    44  	// stack is owned by this goroutine. It is not on a run queue.
    45  	// It is assigned an M (g.m is valid) and it usually has a P
    46  	// (g.m.p is valid), but there are small windows of time where
    47  	// it might not, namely upon entering and exiting _Gsyscall.
    48  	_Grunning // 2
    49  
    50  	// _Gsyscall means this goroutine is executing a system call.
    51  	// It is not executing user code. The stack is owned by this
    52  	// goroutine. It is not on a run queue. It is assigned an M.
    53  	// It may have a P attached, but it does not own it. Code
    54  	// executing in this state must not touch g.m.p.
    55  	_Gsyscall // 3
    56  
    57  	// _Gwaiting means this goroutine is blocked in the runtime.
    58  	// It is not executing user code. It is not on a run queue,
    59  	// but should be recorded somewhere (e.g., a channel wait
    60  	// queue) so it can be ready()d when necessary. The stack is
    61  	// not owned *except* that a channel operation may read or
    62  	// write parts of the stack under the appropriate channel
    63  	// lock. Otherwise, it is not safe to access the stack after a
    64  	// goroutine enters _Gwaiting (e.g., it may get moved).
    65  	_Gwaiting // 4
    66  
    67  	// _Gmoribund_unused is currently unused, but hardcoded in gdb
    68  	// scripts.
    69  	_Gmoribund_unused // 5
    70  
    71  	// _Gdead means this goroutine is currently unused. It may be
    72  	// just exited, on a free list, or just being initialized. It
    73  	// is not executing user code. It may or may not have a stack
    74  	// allocated. The G and its stack (if any) are owned by the M
    75  	// that is exiting the G or that obtained the G from the free
    76  	// list.
    77  	_Gdead // 6
    78  
    79  	// _Genqueue_unused is currently unused.
    80  	_Genqueue_unused // 7
    81  
    82  	// _Gcopystack means this goroutine's stack is being moved. It
    83  	// is not executing user code and is not on a run queue. The
    84  	// stack is owned by the goroutine that put it in _Gcopystack.
    85  	_Gcopystack // 8
    86  
    87  	// _Gpreempted means this goroutine stopped itself for a
    88  	// suspendG preemption. It is like _Gwaiting, but nothing is
    89  	// yet responsible for ready()ing it. Some suspendG must CAS
    90  	// the status to _Gwaiting to take responsibility for
    91  	// ready()ing this G.
    92  	_Gpreempted // 9
    93  
    94  	// _Gleaked represents a leaked goroutine caught by the GC.
    95  	_Gleaked // 10
    96  
    97  	// _Gdeadextra is a _Gdead goroutine that's attached to an extra M
    98  	// used for cgo callbacks.
    99  	_Gdeadextra // 11
   100  
   101  	// _Gscan combined with one of the above states other than
   102  	// _Grunning indicates that GC is scanning the stack. The
   103  	// goroutine is not executing user code and the stack is owned
   104  	// by the goroutine that set the _Gscan bit.
   105  	//
   106  	// _Gscanrunning is different: it is used to briefly block
   107  	// state transitions while GC signals the G to scan its own
   108  	// stack. This is otherwise like _Grunning.
   109  	//
   110  	// atomicstatus&~Gscan gives the state the goroutine will
   111  	// return to when the scan completes.
   112  	_Gscan          = 0x1000
   113  	_Gscanrunnable  = _Gscan + _Grunnable  // 0x1001
   114  	_Gscanrunning   = _Gscan + _Grunning   // 0x1002
   115  	_Gscansyscall   = _Gscan + _Gsyscall   // 0x1003
   116  	_Gscanwaiting   = _Gscan + _Gwaiting   // 0x1004
   117  	_Gscanpreempted = _Gscan + _Gpreempted // 0x1009
   118  	_Gscanleaked    = _Gscan + _Gleaked    // 0x100a
   119  	_Gscandeadextra = _Gscan + _Gdeadextra // 0x100b
   120  )
   121  
   122  const (
   123  	// P status
   124  
   125  	// _Pidle means a P is not being used to run user code or the
   126  	// scheduler. Typically, it's on the idle P list and available
   127  	// to the scheduler, but it may just be transitioning between
   128  	// other states.
   129  	//
   130  	// The P is owned by the idle list or by whatever is
   131  	// transitioning its state. Its run queue is empty.
   132  	_Pidle = iota
   133  
   134  	// _Prunning means a P is owned by an M and is being used to
   135  	// run user code or the scheduler. Only the M that owns this P
   136  	// is allowed to change the P's status from _Prunning. The M
   137  	// may transition the P to _Pidle (if it has no more work to
   138  	// do), or _Pgcstop (to halt for the GC). The M may also hand
   139  	// ownership of the P off directly to another M (for example,
   140  	// to schedule a locked G).
   141  	_Prunning
   142  
   143  	// _Psyscall_unused is a now-defunct state for a P. A P is
   144  	// identified as "in a system call" by looking at the goroutine's
   145  	// state.
   146  	_Psyscall_unused
   147  
   148  	// _Pgcstop means a P is halted for STW and owned by the M
   149  	// that stopped the world. The M that stopped the world
   150  	// continues to use its P, even in _Pgcstop. Transitioning
   151  	// from _Prunning to _Pgcstop causes an M to release its P and
   152  	// park.
   153  	//
   154  	// The P retains its run queue and startTheWorld will restart
   155  	// the scheduler on Ps with non-empty run queues.
   156  	_Pgcstop
   157  
   158  	// _Pdead means a P is no longer used (GOMAXPROCS shrank). We
   159  	// reuse Ps if GOMAXPROCS increases. A dead P is mostly
   160  	// stripped of its resources, though a few things remain
   161  	// (e.g., trace buffers).
   162  	_Pdead
   163  )
   164  
   165  // Mutual exclusion locks.  In the uncontended case,
   166  // as fast as spin locks (just a few user-level instructions),
   167  // but on the contention path they sleep in the kernel.
   168  // A zeroed Mutex is unlocked (no need to initialize each lock).
   169  // Initialization is helpful for static lock ranking, but not required.
   170  type mutex struct {
   171  	// Empty struct if lock ranking is disabled, otherwise includes the lock rank
   172  	lockRankStruct
   173  	// Futex-based impl treats it as uint32 key,
   174  	// while sema-based impl as M* waitm.
   175  	// Used to be a union, but unions break precise GC.
   176  	key uintptr
   177  }
   178  
   179  type funcval struct {
   180  	fn uintptr
   181  	// variable-size, fn-specific data here
   182  }
   183  
   184  type iface struct {
   185  	tab  *itab
   186  	data unsafe.Pointer
   187  }
   188  
   189  type eface struct {
   190  	_type *_type
   191  	data  unsafe.Pointer
   192  }
   193  
   194  func efaceOf(ep *any) *eface {
   195  	return (*eface)(unsafe.Pointer(ep))
   196  }
   197  
   198  // The guintptr, muintptr, and puintptr are all used to bypass write barriers.
   199  // It is particularly important to avoid write barriers when the current P has
   200  // been released, because the GC thinks the world is stopped, and an
   201  // unexpected write barrier would not be synchronized with the GC,
   202  // which can lead to a half-executed write barrier that has marked the object
   203  // but not queued it. If the GC skips the object and completes before the
   204  // queuing can occur, it will incorrectly free the object.
   205  //
   206  // We tried using special assignment functions invoked only when not
   207  // holding a running P, but then some updates to a particular memory
   208  // word went through write barriers and some did not. This breaks the
   209  // write barrier shadow checking mode, and it is also scary: better to have
   210  // a word that is completely ignored by the GC than to have one for which
   211  // only a few updates are ignored.
   212  //
   213  // Gs and Ps are always reachable via true pointers in the
   214  // allgs and allp lists or (during allocation before they reach those lists)
   215  // from stack variables.
   216  //
   217  // Ms are always reachable via true pointers either from allm or
   218  // freem. Unlike Gs and Ps we do free Ms, so it's important that
   219  // nothing ever hold an muintptr across a safe point.
   220  
   221  // A guintptr holds a goroutine pointer, but typed as a uintptr
   222  // to bypass write barriers. It is used in the Gobuf goroutine state
   223  // and in scheduling lists that are manipulated without a P.
   224  //
   225  // The Gobuf.g goroutine pointer is almost always updated by assembly code.
   226  // In one of the few places it is updated by Go code - func save - it must be
   227  // treated as a uintptr to avoid a write barrier being emitted at a bad time.
   228  // Instead of figuring out how to emit the write barriers missing in the
   229  // assembly manipulation, we change the type of the field to uintptr,
   230  // so that it does not require write barriers at all.
   231  //
   232  // Goroutine structs are published in the allg list and never freed.
   233  // That will keep the goroutine structs from being collected.
   234  // There is never a time that Gobuf.g's contain the only references
   235  // to a goroutine: the publishing of the goroutine in allg comes first.
   236  // Goroutine pointers are also kept in non-GC-visible places like TLS,
   237  // so I can't see them ever moving. If we did want to start moving data
   238  // in the GC, we'd need to allocate the goroutine structs from an
   239  // alternate arena. Using guintptr doesn't make that problem any worse.
   240  // Note that pollDesc.rg, pollDesc.wg also store g in uintptr form,
   241  // so they would need to be updated too if g's start moving.
   242  type guintptr uintptr
   243  
   244  //go:nosplit
   245  func (gp guintptr) ptr() *g { return (*g)(unsafe.Pointer(gp)) }
   246  
   247  //go:nosplit
   248  func (gp *guintptr) set(g *g) { *gp = guintptr(unsafe.Pointer(g)) }
   249  
   250  //go:nosplit
   251  func (gp *guintptr) cas(old, new guintptr) bool {
   252  	return atomic.Casuintptr((*uintptr)(unsafe.Pointer(gp)), uintptr(old), uintptr(new))
   253  }
   254  
   255  //go:nosplit
   256  func (gp *g) guintptr() guintptr {
   257  	return guintptr(unsafe.Pointer(gp))
   258  }
   259  
   260  // setGNoWB performs *gp = new without a write barrier.
   261  // For times when it's impractical to use a guintptr.
   262  //
   263  //go:nosplit
   264  //go:nowritebarrier
   265  func setGNoWB(gp **g, new *g) {
   266  	(*guintptr)(unsafe.Pointer(gp)).set(new)
   267  }
   268  
   269  type puintptr uintptr
   270  
   271  //go:nosplit
   272  func (pp puintptr) ptr() *p { return (*p)(unsafe.Pointer(pp)) }
   273  
   274  //go:nosplit
   275  func (pp *puintptr) set(p *p) { *pp = puintptr(unsafe.Pointer(p)) }
   276  
   277  // muintptr is a *m that is not tracked by the garbage collector.
   278  //
   279  // Because we do free Ms, there are some additional constrains on
   280  // muintptrs:
   281  //
   282  //  1. Never hold an muintptr locally across a safe point.
   283  //
   284  //  2. Any muintptr in the heap must be owned by the M itself so it can
   285  //     ensure it is not in use when the last true *m is released.
   286  type muintptr uintptr
   287  
   288  //go:nosplit
   289  func (mp muintptr) ptr() *m { return (*m)(unsafe.Pointer(mp)) }
   290  
   291  //go:nosplit
   292  func (mp *muintptr) set(m *m) { *mp = muintptr(unsafe.Pointer(m)) }
   293  
   294  // setMNoWB performs *mp = new without a write barrier.
   295  // For times when it's impractical to use an muintptr.
   296  //
   297  //go:nosplit
   298  //go:nowritebarrier
   299  func setMNoWB(mp **m, new *m) {
   300  	(*muintptr)(unsafe.Pointer(mp)).set(new)
   301  }
   302  
   303  type gobuf struct {
   304  	// ctxt is unusual with respect to GC: it may be a
   305  	// heap-allocated funcval, so GC needs to track it, but it
   306  	// needs to be set and cleared from assembly, where it's
   307  	// difficult to have write barriers. However, ctxt is really a
   308  	// saved, live register, and we only ever exchange it between
   309  	// the real register and the gobuf. Hence, we treat it as a
   310  	// root during stack scanning, which means assembly that saves
   311  	// and restores it doesn't need write barriers. It's still
   312  	// typed as a pointer so that any other writes from Go get
   313  	// write barriers.
   314  	sp   uintptr
   315  	pc   uintptr
   316  	g    guintptr
   317  	ctxt unsafe.Pointer
   318  	lr   uintptr // link register; on wasm32, PC_F paired with pc
   319  	bp   uintptr // for framepointer-enabled architectures
   320  }
   321  
   322  // maybeTraceablePtr is a special pointer that is conditionally trackable
   323  // by the GC. It consists of an address as a uintptr (vu) and a pointer
   324  // to a data element (vp).
   325  //
   326  // maybeTraceablePtr values can be in one of three states:
   327  // 1. Unset: vu == 0 && vp == nil
   328  // 2. Untracked: vu != 0 && vp == nil
   329  // 3. Tracked: vu != 0 && vp != nil
   330  //
   331  // Do not set fields manually. Use methods instead.
   332  // Extend this type with additional methods if needed.
   333  type maybeTraceablePtr struct {
   334  	vp unsafe.Pointer // For liveness only.
   335  	vu uintptr        // Source of truth.
   336  }
   337  
   338  // untrack unsets the pointer but preserves the address.
   339  // This is used to hide the pointer from the GC.
   340  //
   341  //go:nosplit
   342  func (p *maybeTraceablePtr) setUntraceable() {
   343  	p.vp = nil
   344  }
   345  
   346  // setTraceable resets the pointer to the stored address.
   347  // This is used to make the pointer visible to the GC.
   348  //
   349  //go:nosplit
   350  func (p *maybeTraceablePtr) setTraceable() {
   351  	p.vp = unsafe.Pointer(p.vu)
   352  }
   353  
   354  // set sets the pointer to the data element and updates the address.
   355  //
   356  //go:nosplit
   357  func (p *maybeTraceablePtr) set(v unsafe.Pointer) {
   358  	p.vp = v
   359  	p.vu = uintptr(v)
   360  }
   361  
   362  // get retrieves the pointer to the data element.
   363  //
   364  //go:nosplit
   365  func (p *maybeTraceablePtr) get() unsafe.Pointer {
   366  	return unsafe.Pointer(p.vu)
   367  }
   368  
   369  // uintptr returns the uintptr address of the pointer.
   370  //
   371  //go:nosplit
   372  func (p *maybeTraceablePtr) uintptr() uintptr {
   373  	return p.vu
   374  }
   375  
   376  // maybeTraceableChan extends conditionally trackable pointers (maybeTraceablePtr)
   377  // to track hchan pointers.
   378  //
   379  // Do not set fields manually. Use methods instead.
   380  type maybeTraceableChan struct {
   381  	maybeTraceablePtr
   382  }
   383  
   384  //go:nosplit
   385  func (p *maybeTraceableChan) set(c *hchan) {
   386  	p.maybeTraceablePtr.set(unsafe.Pointer(c))
   387  }
   388  
   389  //go:nosplit
   390  func (p *maybeTraceableChan) get() *hchan {
   391  	return (*hchan)(p.maybeTraceablePtr.get())
   392  }
   393  
   394  // waitingGType describes which type of synchronization primitive a waitingG is
   395  // used with.
   396  type waitingGType uint8
   397  
   398  const (
   399  	waitingGTypeNone waitingGType = iota
   400  
   401  	// Channel operations, including select.
   402  	waitingGTypeChan
   403  
   404  	// semaphores (semacquire, which transitively includes sync.Mutex/RWMutex).
   405  	waitingGTypeSema
   406  
   407  	// sync.Cond, via runtime.notifyList.
   408  	waitingGTypeCond
   409  )
   410  
   411  // waitingG represents a g on a wait list of a synchronization primitive:
   412  // either a channel, semaphore (including sync.Mutex) or notifyList
   413  // (sync.Cond).
   414  //
   415  // waitingG is necessary because the g ↔ synchronization object relation
   416  // is many-to-many. A g can be on many wait lists, so there may be
   417  // many waitingGs for one g; and many gs may be waiting on the same
   418  // synchronization object, so there may be many waitingGs for one object.
   419  //
   420  // waitingGs are allocated from a special pool. Use acquireWaitingG and
   421  // releaseWaitingG to allocate and free them.
   422  type waitingG struct {
   423  	// Which type of waitingG is this. Immutable after allocation.
   424  	typ waitingGType
   425  
   426  	// For channels, the following fields are protected by the
   427  	// hchan.lock of the channel this waitingG is blocking on. shrinkstack
   428  	// depends on this for waitingGs involved in channel ops.
   429  	//
   430  	// For semaphores, all fields are only accessed when holding a semaRoot
   431  	// lock.
   432  
   433  	g *g
   434  
   435  	next *waitingG
   436  	prev *waitingG
   437  
   438  	elem maybeTraceablePtr // data element (may point to stack)
   439  
   440  	// The following fields are never accessed concurrently.
   441  	// For channels, waitlink is only accessed by g.
   442  
   443  	acquiretime int64
   444  	releasetime int64
   445  	ticket      uint32
   446  
   447  	// isSelect indicates g is participating in a select, so
   448  	// g.selectDone must be CAS'd to win the wake-up race.
   449  	isSelect bool
   450  
   451  	// success indicates whether communication over channel c
   452  	// succeeded. It is true if the goroutine was awoken because a
   453  	// value was delivered over channel c, and false if awoken
   454  	// because c was closed.
   455  	success bool
   456  
   457  	// waiters is a count of semaRoot waiting list other than head of list,
   458  	// clamped to a uint16 to fit in unused space.
   459  	// Only meaningful at the head of the list.
   460  	// (If we wanted to be overly clever, we could store a high 16 bits
   461  	// in the second entry in the list.)
   462  	waiters uint16
   463  
   464  	parent   *waitingG          // semaRoot binary tree
   465  	waitlink *waitingG          // g.waiting list or semaRoot
   466  	waittail *waitingG          // semaRoot
   467  	c        maybeTraceableChan // channel
   468  }
   469  
   470  type libcall struct {
   471  	fn   uintptr
   472  	n    uintptr // number of parameters
   473  	args uintptr // parameters
   474  	r1   uintptr // return values
   475  	r2   uintptr
   476  	err  uintptr // error number
   477  }
   478  
   479  // Stack describes a Go execution stack.
   480  // The bounds of the stack are exactly [lo, hi),
   481  // with no implicit data structures on either side.
   482  type stack struct {
   483  	lo uintptr
   484  	hi uintptr
   485  }
   486  
   487  // A stackOff is a position on a goroutine's stack, recorded as its distance
   488  // below the top of the stack (gp.stack.hi).
   489  //
   490  // Stack copying preserves the distance from every stack address to the top of
   491  // the stack, so unlike an address, a stackOff remains valid across stack copies
   492  // without any adjustment. And unlike a pointer, it does not keep stack objects
   493  // alive during stack scanning.
   494  type stackOff uintptr
   495  
   496  // toStackOff returns the stackOff of addr, which must be on gp's stack
   497  // (or exactly gp.stack.hi, which can be a frame boundary). It throws if addr
   498  // isn't, which catches addresses taken on the wrong goroutine or left stale by
   499  // a stack move.
   500  //
   501  // This must be nosplit: callers pass absolute stack addresses, which a stack
   502  // move in this function's prologue would leave stale.
   503  //
   504  //go:nosplit
   505  func (gp *g) toStackOff(addr uintptr) stackOff {
   506  	if addr < gp.stack.lo || addr > gp.stack.hi {
   507  		badStackOff(gp, addr)
   508  	}
   509  	return stackOff(gp.stack.hi - addr)
   510  }
   511  
   512  // badStackOff reports a toStackOff of an address that is not on gp's stack.
   513  // It is out of line to keep toStackOff inlinable.
   514  //
   515  //go:noinline
   516  func badStackOff(gp *g, addr uintptr) {
   517  	print("runtime: address ", hex(addr), " not on goroutine ", gp.goid, " stack [", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n")
   518  	throw("toStackOff: address not on stack")
   519  }
   520  
   521  // fromStackOff returns the current address of off on gp's stack. The result is
   522  // only valid until gp's stack next moves.
   523  //
   524  // This must be nosplit for the same reason as toStackOff: callers may have
   525  // loaded other absolute stack addresses to compare against the result.
   526  //
   527  //go:nosplit
   528  func (gp *g) fromStackOff(off stackOff) uintptr {
   529  	return gp.stack.hi - uintptr(off)
   530  }
   531  
   532  // heldLockInfo gives info on a held lock and the rank of that lock
   533  type heldLockInfo struct {
   534  	lockAddr uintptr
   535  	rank     lockRank
   536  }
   537  
   538  type g struct {
   539  	// Stack parameters.
   540  	// stack describes the actual stack memory: [stack.lo, stack.hi).
   541  	// stackguard0 is the stack pointer compared in the Go stack growth prologue.
   542  	// It is stack.lo+StackGuard normally, but can be StackPreempt to trigger a preemption.
   543  	// stackguard1 is the stack pointer compared in the //go:systemstack stack growth prologue.
   544  	// It is stack.lo+StackGuard on g0 and gsignal stacks.
   545  	// It is ~0 on other goroutine stacks, to trigger a call to morestackc (and crash).
   546  	stack       stack   // offset known to runtime/cgo
   547  	stackguard0 uintptr // offset known to cmd/internal/obj/*
   548  	stackguard1 uintptr // offset known to cmd/internal/obj/*
   549  
   550  	_panic    *_panic // innermost panic
   551  	_defer    *_defer // innermost defer
   552  	m         *m      // current m
   553  	sched     gobuf
   554  	syscallsp uintptr // if status==Gsyscall, syscallsp = sched.sp to use during gc
   555  	syscallpc uintptr // if status==Gsyscall, syscallpc = sched.pc to use during gc
   556  	syscallbp uintptr // if status==Gsyscall, syscallbp = sched.bp to use in fpTraceback
   557  	stktopsp  uintptr // expected sp at top of stack, to check in traceback
   558  	// param is a generic pointer parameter field used to pass
   559  	// values in particular contexts where other storage for the
   560  	// parameter would be difficult to find. It is currently used
   561  	// in four ways:
   562  	// 1. When a channel operation wakes up a blocked goroutine, it sets param to
   563  	//    point to the waitingG of the completed blocking operation.
   564  	// 2. By gcAssistAlloc1 to signal back to its caller that the goroutine completed
   565  	//    the GC cycle. It is unsafe to do so in any other way, because the goroutine's
   566  	//    stack may have moved in the meantime.
   567  	// 3. By debugCallWrap to pass parameters to a new goroutine because allocating a
   568  	//    closure in the runtime is forbidden.
   569  	// 4. When a panic is recovered and control returns to the respective frame,
   570  	//    param may point to a savedOpenDeferState.
   571  	param        unsafe.Pointer
   572  	atomicstatus atomic.Uint32
   573  	wasmCont     wasmContPerG // GOWASM=stackswitch only; fills the padding before goid
   574  	goid         uint64
   575  	schedlink    guintptr
   576  	waitsince    int64      // approx time when the g become blocked
   577  	waitreason   waitReason // if status==Gwaiting
   578  
   579  	preempt       bool // preemption signal, duplicates stackguard0 = stackpreempt
   580  	preemptStop   bool // transition to _Gpreempted on preemption; otherwise, just deschedule
   581  	preemptShrink bool // shrink stack at synchronous safe point
   582  
   583  	// asyncSafePoint is set if g is stopped at an asynchronous
   584  	// safe point. This means there are frames on the stack
   585  	// without precise pointer information.
   586  	asyncSafePoint bool
   587  
   588  	paniconfault bool // panic (instead of crash) on unexpected fault address
   589  	gcscandone   bool // g has scanned stack; protected by _Gscan bit in status
   590  	throwsplit   bool // must not split stack
   591  	// activeStackChans indicates that there are unlocked channels
   592  	// pointing into this goroutine's stack. If true, stack
   593  	// copying needs to acquire channel locks to protect these
   594  	// areas of the stack.
   595  	activeStackChans bool
   596  	// parkingOnChan indicates that the goroutine is about to
   597  	// park on a chansend or chanrecv. Used to signal an unsafe point
   598  	// for stack shrinking.
   599  	parkingOnChan atomic.Bool
   600  	// inMarkAssist indicates whether the goroutine is in mark assist.
   601  	// Used by the execution tracer.
   602  	inMarkAssist bool
   603  	coroexit     bool // argument to coroswitch_m
   604  
   605  	raceignore      int8  // ignore race detection events
   606  	nocgocallback   bool  // whether disable callback from C
   607  	tracking        bool  // whether we're tracking this G for sched latency statistics
   608  	trackingSeq     uint8 // used to decide whether to track this G
   609  	trackingStamp   int64 // timestamp of when the G last started being tracked
   610  	runnableTime    int64 // the amount of time spent runnable, cleared when running, only used when tracking
   611  	lockedm         muintptr
   612  	fipsIndicator   uint8
   613  	fipsOnlyBypass  bool
   614  	ditWanted       bool // set if g wants to be executed with DIT enabled
   615  	syncSafePoint   bool // set if g is stopped at a synchronous safe point.
   616  	runningCleanups atomic.Bool
   617  
   618  	// waiting references the synchronization primitives that g is
   619  	// currently or soon waiting on.
   620  	//
   621  	// For waitingGTypeChan: waiting.elem refers to the src/dst location
   622  	// for the channel operation. waitingG.waitlink forms a linked list of
   623  	// channels the G is waiting on (that have a valid elem pointer), in
   624  	// lock order.
   625  	//
   626  	// For waitingGTypeSema: waiting.elem refers to the semaphore the G is
   627  	// waiting on.
   628  	//
   629  	// For waitingGTypeCond: waiting.elem refers to the notifyList the G is
   630  	// waiting on.
   631  	//
   632  	// Note that waiting need not be set atomically with parking. Thus the
   633  	// G may be preempted with waiting set.
   634  	waiting *waitingG
   635  
   636  	sig        uint32
   637  	secret     int32 // current nesting of runtime/secret.Do calls.
   638  	writebuf   []byte
   639  	sigcode0   uintptr
   640  	sigcode1   uintptr
   641  	sigpc      uintptr
   642  	parentGoid uint64          // goid of goroutine that created this goroutine
   643  	gopc       uintptr         // pc of go statement that created this goroutine
   644  	ancestors  *[]ancestorInfo // ancestor information goroutine(s) that created this goroutine (only used if debug.tracebackancestors)
   645  	startpc    uintptr         // pc of goroutine function
   646  	racectx    uintptr
   647  	cgoCtxt    []uintptr      // cgo traceback context
   648  	labels     unsafe.Pointer // profiler labels
   649  	timer      *timer         // cached timer for time.Sleep
   650  	sleepWhen  int64          // when to sleep until
   651  	selectDone atomic.Uint32  // are we participating in a select and did someone win the race?
   652  
   653  	// goroutineProfiled indicates the status of this goroutine's stack for the
   654  	// current in-progress goroutine profile
   655  	goroutineProfiled goroutineProfileStateHolder
   656  
   657  	coroarg *coro // argument during coroutine transfers
   658  	bubble  *synctestBubble
   659  
   660  	// xRegs stores the extended register state if this G has been
   661  	// asynchronously preempted.
   662  	xRegs xRegPerG
   663  
   664  	// valgrind is the Valgrind state of this G's stack. It is empty
   665  	// unless the runtime is built with Valgrind support (see valgrind.go).
   666  	valgrind valgrindPerG
   667  
   668  	// Per-G tracer state.
   669  	trace gTraceState
   670  
   671  	// Per-G GC state
   672  
   673  	// gcAssistBytes is this G's GC assist credit in terms of
   674  	// bytes allocated. If this is positive, then the G has credit
   675  	// to allocate gcAssistBytes bytes without assisting. If this
   676  	// is negative, then the G must correct this by performing
   677  	// scan work. We track this in bytes to make it fast to update
   678  	// and check for debt in the malloc hot path. The assist ratio
   679  	// determines how this corresponds to scan work debt.
   680  	gcAssistBytes int64
   681  }
   682  
   683  // gTrackingPeriod is the number of transitions out of _Grunning between
   684  // latency tracking runs.
   685  const gTrackingPeriod = 8
   686  
   687  const (
   688  	// tlsSlots is the number of pointer-sized slots reserved for TLS on some platforms,
   689  	// like Windows.
   690  	tlsSlots = 6
   691  	tlsSize  = tlsSlots * goarch.PtrSize
   692  )
   693  
   694  // Values for m.freeWait.
   695  const (
   696  	freeMStack = 0 // M done, free stack and reference.
   697  	freeMRef   = 1 // M done, free reference.
   698  	freeMWait  = 2 // M still in use.
   699  )
   700  
   701  type m struct {
   702  	g0      *g     // goroutine with scheduling stack
   703  	morebuf gobuf  // gobuf arg to morestack
   704  	divmod  uint32 // div/mod denominator for arm - known to liblink (cmd/internal/obj/arm/obj5.go)
   705  
   706  	// Fields whose offsets are not known to debuggers.
   707  
   708  	procid     uint64            // for debuggers, but offset not hard-coded
   709  	gsignal    *g                // signal-handling g
   710  	goSigStack gsignalStack      // Go-allocated signal handling stack
   711  	sigmask    sigset            // storage for saved signal mask
   712  	tls        [tlsSlots]uintptr // thread-local storage (for x86 extern register)
   713  	mstartfn   func()
   714  	curg       *g       // current running goroutine
   715  	caughtsig  guintptr // goroutine running during fatal signal
   716  
   717  	// Indicates whether we've received a signal while
   718  	// running in secret mode.
   719  	signalSecret bool
   720  
   721  	// p is the currently attached P for executing Go code, nil if not executing user Go code.
   722  	//
   723  	// A non-nil p implies exclusive ownership of the P, unless curg is in _Gsyscall.
   724  	// In _Gsyscall the scheduler may mutate this instead. The point of synchronization
   725  	// is the _Gscan bit on curg's status. The scheduler must arrange to prevent curg
   726  	// from transitioning out of _Gsyscall if it intends to mutate p.
   727  	p puintptr
   728  
   729  	nextp           puintptr // The next P to install before executing. Implies exclusive ownership of this P.
   730  	oldp            puintptr // The P that was attached before executing a syscall.
   731  	id              int64
   732  	mallocing       int32
   733  	throwing        throwType
   734  	preemptoff      string // if != "", keep curg running on this m
   735  	locks           int32
   736  	dying           int32
   737  	profilehz       int32
   738  	spinning        bool // m is out of work and is actively looking for work
   739  	blocked         bool // m is blocked on a note
   740  	newSigstack     bool // minit on C thread called sigaltstack
   741  	printlock       int8
   742  	incgo           bool          // m is executing a cgo call
   743  	isextra         bool          // m is an extra m
   744  	isExtraInC      bool          // m is an extra m that does not have any Go frames
   745  	isExtraInSig    bool          // m is an extra m in a signal handler
   746  	freeWait        atomic.Uint32 // Whether it is safe to free g0 and delete m (one of freeMRef, freeMStack, freeMWait)
   747  	needextram      bool
   748  	g0StackAccurate bool // whether the g0 stack has accurate bounds
   749  	traceback       uint8
   750  	allpSnapshot    []*p          // Snapshot of allp for use after dropping P in findRunnable, nil otherwise.
   751  	ncgocall        uint64        // number of cgo calls in total
   752  	ncgo            int32         // number of cgo calls currently in progress
   753  	cgoCallersUse   atomic.Uint32 // if non-zero, cgoCallers in use temporarily
   754  	cgoCallers      *cgoCallers   // cgo traceback if crashing in cgo call
   755  	park            note
   756  	alllink         *m // on allm
   757  	schedlink       muintptr
   758  	idleNode        listNodeManual
   759  	lockedg         guintptr
   760  	createstack     [32]uintptr // stack that created this thread, it's used for StackRecord.Stack0, so it must align with it.
   761  	lockedExt       uint32      // tracking for external LockOSThread
   762  	lockedInt       uint32      // tracking for internal lockOSThread
   763  	mWaitList       mWaitList   // list of runtime lock waiters
   764  	ditEnabled      bool        // set if DIT is currently enabled on this M
   765  	syncIOWorker    bool        // dedicated Windows synchronous pipe I/O worker; immutable after startup
   766  
   767  	mLockProfile mLockProfile // fields relating to runtime.lock contention
   768  	profStack    []uintptr    // used for memory/block/mutex stack traces
   769  
   770  	// wait* are used to carry arguments from gopark into park_m, because
   771  	// there's no stack to put them on. That is their sole purpose.
   772  	waitunlockf          func(*g, unsafe.Pointer) bool
   773  	waitlock             unsafe.Pointer
   774  	waitTraceSkip        int
   775  	waitTraceBlockReason traceBlockReason
   776  
   777  	syscalltick uint32
   778  	freelink    *m // on sched.freem
   779  	trace       mTraceState
   780  
   781  	// These are here to avoid using the G stack so the stack can move during the call.
   782  	libcallpc  uintptr // for cpu profiler
   783  	libcallsp  uintptr
   784  	libcallg   guintptr
   785  	winsyscall winlibcall // stores syscall parameters on windows
   786  
   787  	vdsoSP uintptr // SP for traceback while in VDSO call (0 if not in call)
   788  	vdsoPC uintptr // PC for traceback while in VDSO call
   789  
   790  	// preemptGen counts the number of completed preemption
   791  	// signals. This is used to detect when a preemption is
   792  	// requested, but fails.
   793  	preemptGen atomic.Uint32
   794  
   795  	// Whether this is a pending preemption signal on this M.
   796  	signalPending atomic.Uint32
   797  
   798  	// pcvalue lookup cache
   799  	pcvalueCache pcvalueCache
   800  
   801  	dlogPerM
   802  
   803  	mOS
   804  
   805  	chacha8     chacha8rand.State
   806  	cheaprand   uint32
   807  	cheaprand64 uint64
   808  
   809  	// Up to 10 locks held by this m, maintained by the lock ranking code.
   810  	locksHeldLen int
   811  	locksHeld    [10]heldLockInfo
   812  
   813  	// self points this M until mexit clears it to return nil.
   814  	self mWeakPointer
   815  }
   816  
   817  const mRedZoneSize = (16 << 3) * asanenabledBit // redZoneSize(2048)
   818  
   819  type mPadded struct {
   820  	m
   821  
   822  	// Size the runtime.m structure so it fits in the 2048-byte size class, and
   823  	// not in the next-smallest (1792-byte) size class. That leaves the 11 low
   824  	// bits of muintptr values available for flags, as required by
   825  	// lock_spinbit.go.
   826  	_ [(1 - goarch.IsWasm) * (2048 - mallocHeaderSize - mRedZoneSize - unsafe.Sizeof(m{}))]byte
   827  }
   828  
   829  // mWeakPointer is a "weak" pointer to an M. A weak pointer for each M is
   830  // available as m.self. Users may copy mWeakPointer arbitrarily, and get will
   831  // return the M if it is still live, or nil after mexit.
   832  //
   833  // The zero value is treated as a nil pointer.
   834  //
   835  // Note that get may race with M exit. A successful get will keep the m object
   836  // alive, but the M itself may be exited and thus not actually usable.
   837  type mWeakPointer struct {
   838  	m *atomic.Pointer[m]
   839  }
   840  
   841  func newMWeakPointer(mp *m) mWeakPointer {
   842  	w := mWeakPointer{m: new(atomic.Pointer[m])}
   843  	w.m.Store(mp)
   844  	return w
   845  }
   846  
   847  func (w mWeakPointer) get() *m {
   848  	if w.m == nil {
   849  		return nil
   850  	}
   851  	return w.m.Load()
   852  }
   853  
   854  // clear sets the weak pointer to nil. It cannot be used on zero value
   855  // mWeakPointers.
   856  func (w mWeakPointer) clear() {
   857  	w.m.Store(nil)
   858  }
   859  
   860  type p struct {
   861  	id          int32
   862  	status      uint32 // one of pidle/prunning/...
   863  	link        puintptr
   864  	schedtick   uint32     // incremented on every scheduler call
   865  	syscalltick uint32     // incremented on every system call
   866  	sysmontick  sysmontick // last tick observed by sysmon
   867  	m           muintptr   // back-link to associated m (nil if idle)
   868  	mcache      *mcache
   869  	pcache      pageCache
   870  	raceprocctx uintptr
   871  
   872  	// oldm is the previous m this p ran on.
   873  	//
   874  	// We are not associated with this m, so we have no control over its
   875  	// lifecycle. This value is an m.self object which points to the m
   876  	// until the m exits.
   877  	//
   878  	// Note that this m may be idle, running, or exiting. It should only be
   879  	// used with mgetSpecific, which will take ownership of the m only if
   880  	// it is idle.
   881  	oldm mWeakPointer
   882  
   883  	deferpool    []*_defer // pool of available defer structs (see panic.go)
   884  	deferpoolbuf [32]*_defer
   885  
   886  	// Cache of goroutine ids, amortizes accesses to runtime·sched.goidgen.
   887  	goidcache    uint64
   888  	goidcacheend uint64
   889  
   890  	// Queue of runnable goroutines. Accessed without lock.
   891  	runqhead uint32
   892  	runqtail uint32
   893  	runq     [256]guintptr
   894  	// runnext, if non-nil, is a runnable G that was ready'd by
   895  	// the current G and should be run next instead of what's in
   896  	// runq if there's time remaining in the running G's time
   897  	// slice. It will inherit the time left in the current time
   898  	// slice. If a set of goroutines is locked in a
   899  	// communicate-and-wait pattern, this schedules that set as a
   900  	// unit and eliminates the (potentially large) scheduling
   901  	// latency that otherwise arises from adding the ready'd
   902  	// goroutines to the end of the run queue.
   903  	//
   904  	// Note that while other P's may atomically CAS this to zero,
   905  	// only the owner P can CAS it to a valid G.
   906  	runnext guintptr
   907  
   908  	// Available G's (status == Gdead)
   909  	gFree gList
   910  
   911  	waitingGCache []*waitingG
   912  	waitingGBuf   [128]*waitingG
   913  
   914  	// Cache of mspan objects from the heap.
   915  	mspancache struct {
   916  		// We need an explicit length here because this field is used
   917  		// in allocation codepaths where write barriers are not allowed,
   918  		// and eliminating the write barrier/keeping it eliminated from
   919  		// slice updates is tricky, more so than just managing the length
   920  		// ourselves.
   921  		len int
   922  		buf [128]*mspan
   923  	}
   924  
   925  	// Cache of a single pinner object to reduce allocations from repeated
   926  	// pinner creation.
   927  	pinnerCache *pinner
   928  
   929  	// Cache of one unused pin counter. Accessed by this P with preemption
   930  	// disabled, or during STW. Returned to the allocator when the P is destroyed.
   931  	pinCounterCache *specialPinCounter
   932  
   933  	trace pTraceState
   934  
   935  	palloc persistentAlloc // per-P to avoid mutex
   936  
   937  	// Per-P GC state
   938  	gcAssistTime         int64        // Nanoseconds in assistAlloc
   939  	gcFractionalMarkTime atomic.Int64 // Nanoseconds in fractional mark worker
   940  
   941  	// limiterEvent tracks events for the GC CPU limiter.
   942  	limiterEvent limiterEvent
   943  
   944  	// gcMarkWorkerMode is the mode for the next mark worker to run in.
   945  	// That is, this is used to communicate with the worker goroutine
   946  	// selected for immediate execution by
   947  	// gcController.findRunnableGCWorker. When scheduling other goroutines,
   948  	// this field must be set to gcMarkWorkerNotWorker.
   949  	gcMarkWorkerMode gcMarkWorkerMode
   950  	// gcMarkWorkerStartTime is the nanotime() at which the most recent
   951  	// mark worker started.
   952  	gcMarkWorkerStartTime int64
   953  
   954  	// nextGCMarkWorker is the next mark worker to run. This may be set
   955  	// during start-the-world to assign a worker to this P. The P runs this
   956  	// worker on the next call to gcController.findRunnableGCWorker. If the
   957  	// P runs something else or stops, it must release this worker via
   958  	// gcController.releaseNextGCMarkWorker.
   959  	//
   960  	// See comment in gcBgMarkWorker about the lifetime of
   961  	// gcBgMarkWorkerNode.
   962  	//
   963  	// Only accessed by this P or during STW.
   964  	nextGCMarkWorker *gcBgMarkWorkerNode
   965  
   966  	// gcw is this P's GC work buffer cache. The work buffer is
   967  	// filled by write barriers, drained by mutator assists, and
   968  	// disposed on certain GC state transitions.
   969  	gcw gcWork
   970  
   971  	// wbBuf is this P's GC write barrier buffer.
   972  	//
   973  	// TODO: Consider caching this in the running G.
   974  	wbBuf wbBuf
   975  
   976  	runSafePointFn uint32 // if 1, run sched.safePointFn at next safe point
   977  
   978  	// statsSeq is a counter indicating whether this P is currently
   979  	// writing any stats. Its value is even when not, odd when it is.
   980  	statsSeq atomic.Uint32
   981  
   982  	// Timer heap.
   983  	timers timers
   984  
   985  	// Cleanups.
   986  	cleanups       *cleanupBlock
   987  	cleanupsQueued uint64 // monotonic count of cleanups queued by this P
   988  
   989  	// maxStackScanDelta accumulates the amount of stack space held by
   990  	// live goroutines (i.e. those eligible for stack scanning).
   991  	// Flushed to gcController.maxStackScan once maxStackScanSlack
   992  	// or -maxStackScanSlack is reached.
   993  	maxStackScanDelta int64
   994  
   995  	// gc-time statistics about current goroutines
   996  	// Note that this differs from maxStackScan in that this
   997  	// accumulates the actual stack observed to be used at GC time (hi - sp),
   998  	// not an instantaneous measure of the total stack size that might need
   999  	// to be scanned (hi - lo).
  1000  	scannedStackSize uint64 // stack size of goroutines scanned by this P
  1001  	scannedStacks    uint64 // number of goroutines scanned by this P
  1002  
  1003  	// preempt is set to indicate that this P should be enter the
  1004  	// scheduler ASAP (regardless of what G is running on it).
  1005  	preempt bool
  1006  
  1007  	// gcStopTime is the nanotime timestamp that this P last entered _Pgcstop.
  1008  	gcStopTime int64
  1009  
  1010  	// goroutinesCreated is the total count of goroutines created by this P.
  1011  	goroutinesCreated uint64
  1012  
  1013  	// xRegs is the per-P extended register state used by asynchronous
  1014  	// preemption. This is an empty struct on platforms that don't use extended
  1015  	// register state.
  1016  	xRegs xRegPerP
  1017  
  1018  	// Padding is no longer needed. False sharing is now not a worry because p is large enough
  1019  	// that its size class is an integer multiple of the cache line size (for any of our architectures).
  1020  }
  1021  
  1022  type schedt struct {
  1023  	goidgen    atomic.Uint64
  1024  	lastpoll   atomic.Int64 // time of last network poll, 0 if currently polling
  1025  	pollUntil  atomic.Int64 // time to which current poll is sleeping
  1026  	pollingNet atomic.Int32 // 1 if some P doing non-blocking network poll
  1027  
  1028  	lock mutex
  1029  
  1030  	// When increasing nmidle, nmidlelocked, nmsys, or nmfreed, be
  1031  	// sure to call checkdead().
  1032  
  1033  	midle        listHeadManual // idle m's waiting for work
  1034  	nmidle       int32          // number of idle m's waiting for work
  1035  	nmidlelocked int32          // number of locked m's waiting for work
  1036  	mnext        int64          // number of m's that have been created and next M ID
  1037  	maxmcount    int32          // maximum number of m's allowed (or die)
  1038  	nmsys        int32          // number of system m's not counted for deadlock
  1039  	nmfreed      int64          // cumulative number of freed m's
  1040  
  1041  	ngsys        atomic.Int32 // number of system goroutines
  1042  	nGsyscallNoP atomic.Int32 // number of goroutines in syscalls without a P but whose M is not isExtraInC
  1043  
  1044  	pidle        puintptr // idle p's
  1045  	npidle       atomic.Int32
  1046  	nmspinning   atomic.Int32  // See "Worker thread parking/unparking" comment in proc.go.
  1047  	needspinning atomic.Uint32 // See "Delicate dance" comment in proc.go. Boolean. Must hold sched.lock to set to 1.
  1048  
  1049  	// Global runnable queue.
  1050  	runq gQueue
  1051  
  1052  	// disable controls selective disabling of the scheduler.
  1053  	//
  1054  	// Use schedEnableUser to control this.
  1055  	//
  1056  	// disable is protected by sched.lock.
  1057  	disable struct {
  1058  		// user disables scheduling of user goroutines.
  1059  		user     bool
  1060  		runnable gQueue // pending runnable Gs
  1061  	}
  1062  
  1063  	// Global cache of dead G's.
  1064  	gFree struct {
  1065  		lock    mutex
  1066  		stack   gList // Gs with stacks
  1067  		noStack gList // Gs without stacks
  1068  	}
  1069  
  1070  	// Central cache of waitingG structs.
  1071  	waitingGLock  mutex
  1072  	waitingGCache *waitingG
  1073  
  1074  	// Central pool of available defer structs.
  1075  	deferlock mutex
  1076  	deferpool *_defer
  1077  
  1078  	// freem is the list of m's waiting to be freed when their
  1079  	// m.exited is set. Linked through m.freelink.
  1080  	freem *m
  1081  
  1082  	// gcwaiting indicates that the gc is waiting to run. When runqgrab delays
  1083  	// before stealing runnext, platforms with futex sleep on this address.
  1084  	gcwaiting  gcWaitingWord
  1085  	stopwait   int32
  1086  	stopnote   note
  1087  	sysmonwait atomic.Bool
  1088  	sysmonnote note
  1089  
  1090  	// safePointFn should be called on each P at the next GC
  1091  	// safepoint if p.runSafePointFn is set.
  1092  	safePointFn   func(*p)
  1093  	safePointWait int32
  1094  	safePointNote note
  1095  
  1096  	profilehz int32 // cpu profiling rate
  1097  
  1098  	procresizetime int64 // nanotime() of last change to gomaxprocs
  1099  	totaltime      int64 // ∫gomaxprocs dt up to procresizetime
  1100  
  1101  	customGOMAXPROCS bool // GOMAXPROCS was manually set from the environment or runtime.GOMAXPROCS
  1102  
  1103  	// sysmonlock protects sysmon's actions on the runtime.
  1104  	//
  1105  	// Acquire and hold this mutex to block sysmon from interacting
  1106  	// with the rest of the runtime.
  1107  	sysmonlock mutex
  1108  
  1109  	// timeToRun is a distribution of scheduling latencies, defined
  1110  	// as the sum of time a G spends in the _Grunnable state before
  1111  	// it transitions to _Grunning.
  1112  	timeToRun timeHistogram
  1113  
  1114  	// idleTime is the total CPU time Ps have "spent" idle.
  1115  	//
  1116  	// Reset on each GC cycle.
  1117  	idleTime atomic.Int64
  1118  
  1119  	// totalMutexWaitTime is the sum of time goroutines have spent in _Gwaiting
  1120  	// with a waitreason of the form waitReasonSync{RW,}Mutex{R,}Lock.
  1121  	totalMutexWaitTime atomic.Int64
  1122  
  1123  	// stwStoppingTimeGC/Other are distributions of stop-the-world stopping
  1124  	// latencies, defined as the time taken by stopTheWorldWithSema to get
  1125  	// all Ps to stop. stwStoppingTimeGC covers all GC-related STWs,
  1126  	// stwStoppingTimeOther covers the others.
  1127  	stwStoppingTimeGC    timeHistogram
  1128  	stwStoppingTimeOther timeHistogram
  1129  
  1130  	// stwTotalTimeGC/Other are distributions of stop-the-world total
  1131  	// latencies, defined as the total time from stopTheWorldWithSema to
  1132  	// startTheWorldWithSema. This is a superset of
  1133  	// stwStoppingTimeGC/Other. stwTotalTimeGC covers all GC-related STWs,
  1134  	// stwTotalTimeOther covers the others.
  1135  	stwTotalTimeGC    timeHistogram
  1136  	stwTotalTimeOther timeHistogram
  1137  
  1138  	// totalRuntimeLockWaitTime (plus the value of lockWaitTime on each M in
  1139  	// allm) is the sum of time goroutines have spent in _Grunnable and with an
  1140  	// M, but waiting for locks within the runtime. This field stores the value
  1141  	// for Ms that have exited.
  1142  	totalRuntimeLockWaitTime atomic.Int64
  1143  
  1144  	// goroutinesCreated (plus the value of goroutinesCreated on each P in allp)
  1145  	// is the sum of all goroutines created by the program.
  1146  	goroutinesCreated atomic.Uint64
  1147  }
  1148  
  1149  // gcWaitingWord is an atomic boolean, whose address is also available as a
  1150  // *uint32 for futexsleep.
  1151  type gcWaitingWord struct {
  1152  	val uint32
  1153  }
  1154  
  1155  //go:nosplit
  1156  func (p *gcWaitingWord) Store(v bool) {
  1157  	var val uint32
  1158  	if v {
  1159  		val = 1
  1160  	}
  1161  	atomic.Store(&p.val, val)
  1162  }
  1163  
  1164  //go:nosplit
  1165  func (p *gcWaitingWord) Load() bool {
  1166  	return atomic.Load(&p.val) != 0
  1167  }
  1168  
  1169  //go:nosplit
  1170  func (p *gcWaitingWord) Addr() *uint32 {
  1171  	return &p.val
  1172  }
  1173  
  1174  // Values for the flags field of a sigTabT.
  1175  const (
  1176  	_SigNotify   = 1 << iota // let signal.Notify have signal, even if from kernel
  1177  	_SigKill                 // if signal.Notify doesn't take it, exit quietly
  1178  	_SigThrow                // if signal.Notify doesn't take it, exit loudly
  1179  	_SigPanic                // if the signal is from the kernel, panic
  1180  	_SigDefault              // if the signal isn't explicitly requested, don't monitor it
  1181  	_SigGoExit               // cause all runtime procs to exit (only used on Plan 9).
  1182  	_SigSetStack             // Don't explicitly install handler, but add SA_ONSTACK to existing libc handler
  1183  	_SigUnblock              // always unblock; see blockableSig
  1184  	_SigIgn                  // _SIG_DFL action is to ignore the signal
  1185  )
  1186  
  1187  // Layout of in-memory per-function information prepared by linker
  1188  // See https://golang.org/s/go12symtab.
  1189  // Keep in sync with linker (../cmd/link/internal/ld/pcln.go:/pclntab)
  1190  // and with package debug/gosym and with symtab.go in package runtime.
  1191  type _func struct {
  1192  	sys.NotInHeap // Only in static data
  1193  
  1194  	entryOff uint32 // start pc, as offset from moduledata.text
  1195  	nameOff  int32  // function name, as index into moduledata.funcnametab.
  1196  
  1197  	args        int32  // in/out args size
  1198  	deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any.
  1199  
  1200  	pcsp      uint32
  1201  	pcfile    uint32
  1202  	pcln      uint32
  1203  	npcdata   uint32
  1204  	cuOffset  uint32     // runtime.cutab offset of this function's CU
  1205  	startLine int32      // line number of start of function (func keyword/TEXT directive)
  1206  	funcID    abi.FuncID // set for certain special runtime functions
  1207  	flag      abi.FuncFlag
  1208  	_         [1]byte // pad
  1209  	nfuncdata uint8   // must be last, must end on a uint32-aligned boundary
  1210  
  1211  	// The end of the struct is followed immediately by two variable-length
  1212  	// arrays that reference the pcdata and funcdata locations for this
  1213  	// function.
  1214  
  1215  	// pcdata contains the offset into moduledata.pctab for the start of
  1216  	// that index's table. e.g.,
  1217  	// &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of
  1218  	// the unsafe point table.
  1219  	//
  1220  	// An offset of 0 indicates that there is no table.
  1221  	//
  1222  	// pcdata [npcdata]uint32
  1223  
  1224  	// funcdata contains the offset past moduledata.gofunc which contains a
  1225  	// pointer to that index's funcdata. e.g.,
  1226  	// *(moduledata.gofunc +  _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is
  1227  	// the argument pointer map.
  1228  	//
  1229  	// An offset of ^uint32(0) indicates that there is no entry.
  1230  	//
  1231  	// funcdata [nfuncdata]uint32
  1232  }
  1233  
  1234  // Pseudo-Func that is returned for PCs that occur in inlined code.
  1235  // A *Func can be either a *_func or a *funcinl, and they are distinguished
  1236  // by the first uintptr.
  1237  //
  1238  // TODO(austin): Can we merge this with inlinedCall?
  1239  type funcinl struct {
  1240  	ones      uint32  // set to ^0 to distinguish from _func
  1241  	entry     uintptr // entry of the real (the "outermost") frame
  1242  	name      string
  1243  	file      string
  1244  	line      int32
  1245  	startLine int32
  1246  }
  1247  
  1248  type itab = abi.ITab
  1249  
  1250  // Lock-free stack node.
  1251  // Also known to export_test.go.
  1252  type lfnode struct {
  1253  	next    uint64
  1254  	pushcnt uintptr
  1255  }
  1256  
  1257  type forcegcstate struct {
  1258  	lock mutex
  1259  	g    *g
  1260  	idle atomic.Bool
  1261  }
  1262  
  1263  // A _defer holds an entry on the list of deferred calls.
  1264  // If you add a field here, add code to clear it in deferProcStack.
  1265  // This struct must match the code in cmd/compile/internal/ssagen/ssa.go:deferstruct
  1266  // and cmd/compile/internal/ssagen/ssa.go:(*state).call.
  1267  // Some defers will be allocated on the stack and some on the heap.
  1268  // All defers are logically part of the stack, so write barriers to
  1269  // initialize them are not required. All defers must be manually scanned,
  1270  // and for heap defers, marked.
  1271  type _defer struct {
  1272  	heap      bool
  1273  	rangefunc bool    // true for rangefunc list
  1274  	sp        uintptr // sp at time of defer
  1275  	pc        uintptr // pc at time of defer
  1276  	fn        func()  // can be nil for open-coded defers
  1277  	link      *_defer // next defer on G; can point to either heap or stack!
  1278  
  1279  	// If rangefunc is true, *head is the head of the atomic linked list
  1280  	// during a range-over-func execution.
  1281  	head *atomic.Pointer[_defer]
  1282  }
  1283  
  1284  // A _panic holds information about an active panic.
  1285  //
  1286  // A _panic value must only ever live on the stack.
  1287  //
  1288  // The link, deferBitsPtr, and slotsPtr fields are stack pointers, but
  1289  // don't need special handling during stack growth: because they are
  1290  // pointer-typed and _panic values only live on the stack, regular stack
  1291  // pointer adjustment takes care of them.
  1292  //
  1293  // startSP, sp, and fp record frame boundaries. These are stackOffs rather
  1294  // than pointers because a frame boundary is often also the address of a
  1295  // stack object (for example, on amd64, a caller's SP is the address of the
  1296  // stack arguments it passes to its callee), and a pointer would keep that
  1297  // object alive after it is dead. See go.dev/issue/82394 and
  1298  // go.dev/issue/82917.
  1299  type _panic struct {
  1300  	arg  any     // argument to panic
  1301  	link *_panic // link to earlier panic
  1302  
  1303  	// startPC and startSP track where _panic.start was called.
  1304  	// (These are the SP and PC of the gopanic frame itself.)
  1305  	startPC uintptr
  1306  	startSP stackOff
  1307  
  1308  	// The current stack frame that we're running deferred calls for.
  1309  	pc uintptr
  1310  	sp stackOff
  1311  	fp stackOff
  1312  
  1313  	// retpc stores the PC where the panic should jump back to, if the
  1314  	// function last returned by _panic.nextDefer() recovers the panic.
  1315  	retpc uintptr
  1316  
  1317  	// Extra state for handling open-coded defers.
  1318  	deferBitsPtr *uint8
  1319  	slotsPtr     unsafe.Pointer
  1320  
  1321  	recoverAtToken *recoverAtToken // identify this panic to gorecoverat
  1322  
  1323  	recovered   bool // whether this panic has been recovered
  1324  	repanicked  bool // whether this panic repanicked
  1325  	goexit      bool
  1326  	deferreturn bool
  1327  }
  1328  
  1329  // savedOpenDeferState tracks the extra state from _panic that's
  1330  // necessary for deferreturn to pick up where gopanic left off,
  1331  // without needing to unwind the stack.
  1332  type savedOpenDeferState struct {
  1333  	retpc           uintptr
  1334  	deferBitsOffset uintptr
  1335  	slotsOffset     uintptr
  1336  }
  1337  
  1338  // ancestorInfo records details of where a goroutine was started.
  1339  type ancestorInfo struct {
  1340  	pcs  []uintptr // pcs from the stack of this goroutine
  1341  	goid uint64    // goroutine id of this goroutine; original goroutine possibly dead
  1342  	gopc uintptr   // pc of go statement that created this goroutine
  1343  }
  1344  
  1345  // A waitReason explains why a goroutine has been stopped.
  1346  // See gopark. Do not re-use waitReasons, add new ones.
  1347  type waitReason uint8
  1348  
  1349  const (
  1350  	waitReasonZero                  waitReason = iota // ""
  1351  	waitReasonGCAssistMarking                         // "GC assist marking"
  1352  	waitReasonIOWait                                  // "IO wait"
  1353  	waitReasonDumpingHeap                             // "dumping heap"
  1354  	waitReasonGarbageCollection                       // "garbage collection"
  1355  	waitReasonGarbageCollectionScan                   // "garbage collection scan"
  1356  	waitReasonPanicWait                               // "panicwait"
  1357  	waitReasonGCAssistWait                            // "GC assist wait"
  1358  	waitReasonGCSweepWait                             // "GC sweep wait"
  1359  	waitReasonGCScavengeWait                          // "GC scavenge wait"
  1360  	waitReasonFinalizerWait                           // "finalizer wait"
  1361  	waitReasonForceGCIdle                             // "force gc (idle)"
  1362  	waitReasonUpdateGOMAXPROCSIdle                    // "GOMAXPROCS updater (idle)"
  1363  	waitReasonSemacquire                              // "semacquire"
  1364  	waitReasonSleep                                   // "sleep"
  1365  	waitReasonChanReceiveNilChan                      // "chan receive (nil chan)"
  1366  	waitReasonChanSendNilChan                         // "chan send (nil chan)"
  1367  	waitReasonSelectNoCases                           // "select (no cases)"
  1368  	waitReasonSelect                                  // "select"
  1369  	waitReasonChanReceive                             // "chan receive"
  1370  	waitReasonChanSend                                // "chan send"
  1371  	waitReasonSyncCondWait                            // "sync.Cond.Wait"
  1372  	waitReasonSyncMutexLock                           // "sync.Mutex.Lock"
  1373  	waitReasonSyncRWMutexRLock                        // "sync.RWMutex.RLock"
  1374  	waitReasonSyncRWMutexLock                         // "sync.RWMutex.Lock"
  1375  	waitReasonSyncWaitGroupWait                       // "sync.WaitGroup.Wait"
  1376  	waitReasonTraceReaderBlocked                      // "trace reader (blocked)"
  1377  	waitReasonWaitForGCCycle                          // "wait for GC cycle"
  1378  	waitReasonGCWorkerIdle                            // "GC worker (idle)"
  1379  	waitReasonGCWorkerActive                          // "GC worker (active)"
  1380  	waitReasonPreempted                               // "preempted"
  1381  	waitReasonDebugCall                               // "debug call"
  1382  	waitReasonGCMarkTermination                       // "GC mark termination"
  1383  	waitReasonStoppingTheWorld                        // "stopping the world"
  1384  	waitReasonFlushProcCaches                         // "flushing proc caches"
  1385  	waitReasonTraceGoroutineStatus                    // "trace goroutine status"
  1386  	waitReasonTraceProcStatus                         // "trace proc status"
  1387  	waitReasonPageTraceFlush                          // "page trace flush"
  1388  	waitReasonCoroutine                               // "coroutine"
  1389  	waitReasonGCWeakToStrongWait                      // "GC weak to strong wait"
  1390  	waitReasonSynctestRun                             // "synctest.Run"
  1391  	waitReasonSynctestWait                            // "synctest.Wait"
  1392  	waitReasonSynctestChanReceive                     // "chan receive (durable)"
  1393  	waitReasonSynctestChanSend                        // "chan send (durable)"
  1394  	waitReasonSynctestSelect                          // "select (durable)"
  1395  	waitReasonSynctestWaitGroupWait                   // "sync.WaitGroup.Wait (durable)"
  1396  	waitReasonCleanupWait                             // "cleanup wait"
  1397  )
  1398  
  1399  var waitReasonStrings = [...]string{
  1400  	waitReasonZero:                  "",
  1401  	waitReasonGCAssistMarking:       "GC assist marking",
  1402  	waitReasonIOWait:                "IO wait",
  1403  	waitReasonChanReceiveNilChan:    "chan receive (nil chan)",
  1404  	waitReasonChanSendNilChan:       "chan send (nil chan)",
  1405  	waitReasonDumpingHeap:           "dumping heap",
  1406  	waitReasonGarbageCollection:     "garbage collection",
  1407  	waitReasonGarbageCollectionScan: "garbage collection scan",
  1408  	waitReasonPanicWait:             "panicwait",
  1409  	waitReasonSelect:                "select",
  1410  	waitReasonSelectNoCases:         "select (no cases)",
  1411  	waitReasonGCAssistWait:          "GC assist wait",
  1412  	waitReasonGCSweepWait:           "GC sweep wait",
  1413  	waitReasonGCScavengeWait:        "GC scavenge wait",
  1414  	waitReasonChanReceive:           "chan receive",
  1415  	waitReasonChanSend:              "chan send",
  1416  	waitReasonFinalizerWait:         "finalizer wait",
  1417  	waitReasonForceGCIdle:           "force gc (idle)",
  1418  	waitReasonUpdateGOMAXPROCSIdle:  "GOMAXPROCS updater (idle)",
  1419  	waitReasonSemacquire:            "semacquire",
  1420  	waitReasonSleep:                 "sleep",
  1421  	waitReasonSyncCondWait:          "sync.Cond.Wait",
  1422  	waitReasonSyncMutexLock:         "sync.Mutex.Lock",
  1423  	waitReasonSyncRWMutexRLock:      "sync.RWMutex.RLock",
  1424  	waitReasonSyncRWMutexLock:       "sync.RWMutex.Lock",
  1425  	waitReasonSyncWaitGroupWait:     "sync.WaitGroup.Wait",
  1426  	waitReasonTraceReaderBlocked:    "trace reader (blocked)",
  1427  	waitReasonWaitForGCCycle:        "wait for GC cycle",
  1428  	waitReasonGCWorkerIdle:          "GC worker (idle)",
  1429  	waitReasonGCWorkerActive:        "GC worker (active)",
  1430  	waitReasonPreempted:             "preempted",
  1431  	waitReasonDebugCall:             "debug call",
  1432  	waitReasonGCMarkTermination:     "GC mark termination",
  1433  	waitReasonStoppingTheWorld:      "stopping the world",
  1434  	waitReasonFlushProcCaches:       "flushing proc caches",
  1435  	waitReasonTraceGoroutineStatus:  "trace goroutine status",
  1436  	waitReasonTraceProcStatus:       "trace proc status",
  1437  	waitReasonPageTraceFlush:        "page trace flush",
  1438  	waitReasonCoroutine:             "coroutine",
  1439  	waitReasonGCWeakToStrongWait:    "GC weak to strong wait",
  1440  	waitReasonSynctestRun:           "synctest.Run",
  1441  	waitReasonSynctestWait:          "synctest.Wait",
  1442  	waitReasonSynctestChanReceive:   "chan receive (durable)",
  1443  	waitReasonSynctestChanSend:      "chan send (durable)",
  1444  	waitReasonSynctestSelect:        "select (durable)",
  1445  	waitReasonSynctestWaitGroupWait: "sync.WaitGroup.Wait (durable)",
  1446  	waitReasonCleanupWait:           "cleanup wait",
  1447  }
  1448  
  1449  func (w waitReason) String() string {
  1450  	if w < 0 || w >= waitReason(len(waitReasonStrings)) {
  1451  		return "unknown wait reason"
  1452  	}
  1453  	return waitReasonStrings[w]
  1454  }
  1455  
  1456  // isMutexWait returns true if the goroutine is blocked because of
  1457  // sync.Mutex.Lock or sync.RWMutex.[R]Lock.
  1458  //
  1459  //go:nosplit
  1460  func (w waitReason) isMutexWait() bool {
  1461  	return w == waitReasonSyncMutexLock ||
  1462  		w == waitReasonSyncRWMutexRLock ||
  1463  		w == waitReasonSyncRWMutexLock
  1464  }
  1465  
  1466  // isSyncWait returns true if the goroutine is blocked because of
  1467  // sync library primitive operations.
  1468  //
  1469  //go:nosplit
  1470  func (w waitReason) isSyncWait() bool {
  1471  	return waitReasonSyncCondWait <= w && w <= waitReasonSyncWaitGroupWait
  1472  }
  1473  
  1474  // isChanWait is true if the goroutine is blocked because of non-nil
  1475  // channel operations or a select statement with at least one case.
  1476  //
  1477  //go:nosplit
  1478  func (w waitReason) isChanWait() bool {
  1479  	return w == waitReasonSelect ||
  1480  		w == waitReasonChanReceive ||
  1481  		w == waitReasonChanSend
  1482  }
  1483  
  1484  func (w waitReason) isWaitingForSuspendG() bool {
  1485  	return isWaitingForSuspendG[w]
  1486  }
  1487  
  1488  // isWaitingForSuspendG indicates that a goroutine is only entering _Gwaiting and
  1489  // setting a waitReason because it needs to be able to let the suspendG
  1490  // (used by the GC and the execution tracer) take ownership of its stack.
  1491  // The G is always actually executing on the system stack in these cases.
  1492  //
  1493  // TODO(mknyszek): Consider replacing this with a new dedicated G status.
  1494  var isWaitingForSuspendG = [len(waitReasonStrings)]bool{
  1495  	waitReasonStoppingTheWorld:      true,
  1496  	waitReasonGCMarkTermination:     true,
  1497  	waitReasonGarbageCollection:     true,
  1498  	waitReasonGarbageCollectionScan: true,
  1499  	waitReasonTraceGoroutineStatus:  true,
  1500  	waitReasonTraceProcStatus:       true,
  1501  	waitReasonPageTraceFlush:        true,
  1502  	waitReasonGCAssistMarking:       true,
  1503  	waitReasonGCWorkerActive:        true,
  1504  	waitReasonFlushProcCaches:       true,
  1505  }
  1506  
  1507  func (w waitReason) isIdleInSynctest() bool {
  1508  	return isIdleInSynctest[w]
  1509  }
  1510  
  1511  // isIdleInSynctest indicates that a goroutine is considered idle by synctest.Wait.
  1512  var isIdleInSynctest = [len(waitReasonStrings)]bool{
  1513  	waitReasonChanReceiveNilChan:    true,
  1514  	waitReasonChanSendNilChan:       true,
  1515  	waitReasonSelectNoCases:         true,
  1516  	waitReasonSleep:                 true,
  1517  	waitReasonSyncCondWait:          true,
  1518  	waitReasonSynctestWaitGroupWait: true,
  1519  	waitReasonCoroutine:             true,
  1520  	waitReasonSynctestRun:           true,
  1521  	waitReasonSynctestWait:          true,
  1522  	waitReasonSynctestChanReceive:   true,
  1523  	waitReasonSynctestChanSend:      true,
  1524  	waitReasonSynctestSelect:        true,
  1525  }
  1526  
  1527  var (
  1528  	// Linked-list of all Ms. Written under sched.lock, read atomically.
  1529  	allm *m
  1530  
  1531  	gomaxprocs    int32
  1532  	numCPUStartup int32
  1533  	forcegc       forcegcstate
  1534  	sched         schedt
  1535  	newprocs      int32
  1536  )
  1537  
  1538  var (
  1539  	// allpLock protects P-less reads and size changes of allp, idlepMask,
  1540  	// and timerpMask, and all writes to allp.
  1541  	allpLock mutex
  1542  
  1543  	// len(allp) == gomaxprocs; may change at safe points, otherwise
  1544  	// immutable.
  1545  	allp []*p
  1546  
  1547  	// Bitmask of Ps in _Pidle list, one bit per P. Reads and writes must
  1548  	// be atomic. Length may change at safe points.
  1549  	//
  1550  	// Each P must update only its own bit. In order to maintain
  1551  	// consistency, a P going idle must set the idle mask simultaneously with
  1552  	// updates to the idle P list under the sched.lock, otherwise a racing
  1553  	// pidleget may clear the mask before pidleput sets the mask,
  1554  	// corrupting the bitmap.
  1555  	//
  1556  	// N.B., procresize takes ownership of all Ps in stopTheWorldWithSema.
  1557  	idlepMask pMask
  1558  
  1559  	// Bitmask of Ps that may have a timer, one bit per P. Reads and writes
  1560  	// must be atomic. Length may change at safe points.
  1561  	//
  1562  	// Ideally, the timer mask would be kept immediately consistent on any timer
  1563  	// operations. Unfortunately, updating a shared global data structure in the
  1564  	// timer hot path adds too much overhead in applications frequently switching
  1565  	// between no timers and some timers.
  1566  	//
  1567  	// As a compromise, the timer mask is updated only on pidleget / pidleput. A
  1568  	// running P (returned by pidleget) may add a timer at any time, so its mask
  1569  	// must be set. An idle P (passed to pidleput) cannot add new timers while
  1570  	// idle, so if it has no timers at that time, its mask may be cleared.
  1571  	//
  1572  	// Thus, we get the following effects on timer-stealing in findRunnable:
  1573  	//
  1574  	//   - Idle Ps with no timers when they go idle are never checked in findRunnable
  1575  	//     (for work- or timer-stealing; this is the ideal case).
  1576  	//   - Running Ps must always be checked.
  1577  	//   - Idle Ps whose timers are stolen must continue to be checked until they run
  1578  	//     again, even after timer expiration.
  1579  	//
  1580  	// When the P starts running again, the mask should be set, as a timer may be
  1581  	// added at any time.
  1582  	//
  1583  	// TODO(prattmic): Additional targeted updates may improve the above cases.
  1584  	// e.g., updating the mask when stealing a timer.
  1585  	timerpMask pMask
  1586  )
  1587  
  1588  // goarmsoftfp is used by runtime/cgo assembly.
  1589  //
  1590  //go:linkname goarmsoftfp
  1591  
  1592  var (
  1593  	// Pool of GC parked background workers. Entries are type
  1594  	// *gcBgMarkWorkerNode.
  1595  	gcBgMarkWorkerPool lfstack
  1596  
  1597  	// Total number of gcBgMarkWorker goroutines. Protected by worldsema.
  1598  	gcBgMarkWorkerCount int32
  1599  
  1600  	// Information about what cpu features are available.
  1601  	// Packages outside the runtime should not use these
  1602  	// as they are not an external api.
  1603  	// Set on startup in asm_{386,amd64}.s
  1604  	processorVersionInfo uint32
  1605  	isIntel              bool
  1606  )
  1607  
  1608  // set by cmd/link on arm systems
  1609  // accessed using linkname by internal/runtime/atomic.
  1610  //
  1611  // goarm should be an internal detail,
  1612  // but widely used packages access it using linkname.
  1613  // Notable members of the hall of shame include:
  1614  //   - github.com/creativeprojects/go-selfupdate
  1615  //
  1616  // Do not remove or change the type signature.
  1617  // See go.dev/issue/67401.
  1618  //
  1619  //go:linkname goarm
  1620  var (
  1621  	goarm       uint8
  1622  	goarmsoftfp uint8
  1623  )
  1624  
  1625  // Set by the linker so the runtime can determine the buildmode.
  1626  var (
  1627  	islibrary bool // -buildmode=c-shared
  1628  	isarchive bool // -buildmode=c-archive
  1629  )
  1630  
  1631  // Must agree with internal/buildcfg.FramePointerEnabled.
  1632  const framepointer_enabled = GOARCH == "amd64" || GOARCH == "arm64"
  1633  
  1634  // getcallerfp returns the frame pointer of the caller of the caller
  1635  // of this function.
  1636  //
  1637  //go:nosplit
  1638  //go:noinline
  1639  func getcallerfp() uintptr {
  1640  	fp := getfp() // This frame's FP.
  1641  	if fp != 0 {
  1642  		fp = *(*uintptr)(unsafe.Pointer(fp)) // The caller's FP.
  1643  		fp = *(*uintptr)(unsafe.Pointer(fp)) // The caller's caller's FP.
  1644  	}
  1645  	return fp
  1646  }
  1647  

View as plain text