Source file src/runtime/runtime2.go
1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "internal/chacha8rand" 10 "internal/goarch" 11 "internal/runtime/atomic" 12 "internal/runtime/sys" 13 "unsafe" 14 ) 15 16 // defined constants 17 const ( 18 // G status 19 // 20 // Beyond indicating the general state of a G, the G status 21 // acts like a lock on the goroutine's stack (and hence its 22 // ability to execute user code). 23 // 24 // If you add to this list, add to the list 25 // of "okay during garbage collection" status 26 // in mgcmark.go too. 27 // 28 // TODO(austin): The _Gscan bit could be much lighter-weight. 29 // For example, we could choose not to run _Gscanrunnable 30 // goroutines found in the run queue, rather than CAS-looping 31 // until they become _Grunnable. And transitions like 32 // _Gscanwaiting -> _Gscanrunnable are actually okay because 33 // they don't affect stack ownership. 34 35 // _Gidle means this goroutine was just allocated and has not 36 // yet been initialized. 37 _Gidle = iota // 0 38 39 // _Grunnable means this goroutine is on a run queue. It is 40 // not currently executing user code. The stack is not owned. 41 _Grunnable // 1 42 43 // _Grunning means this goroutine may execute user code. The 44 // stack is owned by this goroutine. It is not on a run queue. 45 // It is assigned an M (g.m is valid) and it usually has a P 46 // (g.m.p is valid), but there are small windows of time where 47 // it might not, namely upon entering and exiting _Gsyscall. 48 _Grunning // 2 49 50 // _Gsyscall means this goroutine is executing a system call. 51 // It is not executing user code. The stack is owned by this 52 // goroutine. It is not on a run queue. It is assigned an M. 53 // It may have a P attached, but it does not own it. Code 54 // executing in this state must not touch g.m.p. 55 _Gsyscall // 3 56 57 // _Gwaiting means this goroutine is blocked in the runtime. 58 // It is not executing user code. It is not on a run queue, 59 // but should be recorded somewhere (e.g., a channel wait 60 // queue) so it can be ready()d when necessary. The stack is 61 // not owned *except* that a channel operation may read or 62 // write parts of the stack under the appropriate channel 63 // lock. Otherwise, it is not safe to access the stack after a 64 // goroutine enters _Gwaiting (e.g., it may get moved). 65 _Gwaiting // 4 66 67 // _Gmoribund_unused is currently unused, but hardcoded in gdb 68 // scripts. 69 _Gmoribund_unused // 5 70 71 // _Gdead means this goroutine is currently unused. It may be 72 // just exited, on a free list, or just being initialized. It 73 // is not executing user code. It may or may not have a stack 74 // allocated. The G and its stack (if any) are owned by the M 75 // that is exiting the G or that obtained the G from the free 76 // list. 77 _Gdead // 6 78 79 // _Genqueue_unused is currently unused. 80 _Genqueue_unused // 7 81 82 // _Gcopystack means this goroutine's stack is being moved. It 83 // is not executing user code and is not on a run queue. The 84 // stack is owned by the goroutine that put it in _Gcopystack. 85 _Gcopystack // 8 86 87 // _Gpreempted means this goroutine stopped itself for a 88 // suspendG preemption. It is like _Gwaiting, but nothing is 89 // yet responsible for ready()ing it. Some suspendG must CAS 90 // the status to _Gwaiting to take responsibility for 91 // ready()ing this G. 92 _Gpreempted // 9 93 94 // _Gleaked represents a leaked goroutine caught by the GC. 95 _Gleaked // 10 96 97 // _Gdeadextra is a _Gdead goroutine that's attached to an extra M 98 // used for cgo callbacks. 99 _Gdeadextra // 11 100 101 // _Gscan combined with one of the above states other than 102 // _Grunning indicates that GC is scanning the stack. The 103 // goroutine is not executing user code and the stack is owned 104 // by the goroutine that set the _Gscan bit. 105 // 106 // _Gscanrunning is different: it is used to briefly block 107 // state transitions while GC signals the G to scan its own 108 // stack. This is otherwise like _Grunning. 109 // 110 // atomicstatus&~Gscan gives the state the goroutine will 111 // return to when the scan completes. 112 _Gscan = 0x1000 113 _Gscanrunnable = _Gscan + _Grunnable // 0x1001 114 _Gscanrunning = _Gscan + _Grunning // 0x1002 115 _Gscansyscall = _Gscan + _Gsyscall // 0x1003 116 _Gscanwaiting = _Gscan + _Gwaiting // 0x1004 117 _Gscanpreempted = _Gscan + _Gpreempted // 0x1009 118 _Gscanleaked = _Gscan + _Gleaked // 0x100a 119 _Gscandeadextra = _Gscan + _Gdeadextra // 0x100b 120 ) 121 122 const ( 123 // P status 124 125 // _Pidle means a P is not being used to run user code or the 126 // scheduler. Typically, it's on the idle P list and available 127 // to the scheduler, but it may just be transitioning between 128 // other states. 129 // 130 // The P is owned by the idle list or by whatever is 131 // transitioning its state. Its run queue is empty. 132 _Pidle = iota 133 134 // _Prunning means a P is owned by an M and is being used to 135 // run user code or the scheduler. Only the M that owns this P 136 // is allowed to change the P's status from _Prunning. The M 137 // may transition the P to _Pidle (if it has no more work to 138 // do), or _Pgcstop (to halt for the GC). The M may also hand 139 // ownership of the P off directly to another M (for example, 140 // to schedule a locked G). 141 _Prunning 142 143 // _Psyscall_unused is a now-defunct state for a P. A P is 144 // identified as "in a system call" by looking at the goroutine's 145 // state. 146 _Psyscall_unused 147 148 // _Pgcstop means a P is halted for STW and owned by the M 149 // that stopped the world. The M that stopped the world 150 // continues to use its P, even in _Pgcstop. Transitioning 151 // from _Prunning to _Pgcstop causes an M to release its P and 152 // park. 153 // 154 // The P retains its run queue and startTheWorld will restart 155 // the scheduler on Ps with non-empty run queues. 156 _Pgcstop 157 158 // _Pdead means a P is no longer used (GOMAXPROCS shrank). We 159 // reuse Ps if GOMAXPROCS increases. A dead P is mostly 160 // stripped of its resources, though a few things remain 161 // (e.g., trace buffers). 162 _Pdead 163 ) 164 165 // Mutual exclusion locks. In the uncontended case, 166 // as fast as spin locks (just a few user-level instructions), 167 // but on the contention path they sleep in the kernel. 168 // A zeroed Mutex is unlocked (no need to initialize each lock). 169 // Initialization is helpful for static lock ranking, but not required. 170 type mutex struct { 171 // Empty struct if lock ranking is disabled, otherwise includes the lock rank 172 lockRankStruct 173 // Futex-based impl treats it as uint32 key, 174 // while sema-based impl as M* waitm. 175 // Used to be a union, but unions break precise GC. 176 key uintptr 177 } 178 179 type funcval struct { 180 fn uintptr 181 // variable-size, fn-specific data here 182 } 183 184 type iface struct { 185 tab *itab 186 data unsafe.Pointer 187 } 188 189 type eface struct { 190 _type *_type 191 data unsafe.Pointer 192 } 193 194 func efaceOf(ep *any) *eface { 195 return (*eface)(unsafe.Pointer(ep)) 196 } 197 198 // The guintptr, muintptr, and puintptr are all used to bypass write barriers. 199 // It is particularly important to avoid write barriers when the current P has 200 // been released, because the GC thinks the world is stopped, and an 201 // unexpected write barrier would not be synchronized with the GC, 202 // which can lead to a half-executed write barrier that has marked the object 203 // but not queued it. If the GC skips the object and completes before the 204 // queuing can occur, it will incorrectly free the object. 205 // 206 // We tried using special assignment functions invoked only when not 207 // holding a running P, but then some updates to a particular memory 208 // word went through write barriers and some did not. This breaks the 209 // write barrier shadow checking mode, and it is also scary: better to have 210 // a word that is completely ignored by the GC than to have one for which 211 // only a few updates are ignored. 212 // 213 // Gs and Ps are always reachable via true pointers in the 214 // allgs and allp lists or (during allocation before they reach those lists) 215 // from stack variables. 216 // 217 // Ms are always reachable via true pointers either from allm or 218 // freem. Unlike Gs and Ps we do free Ms, so it's important that 219 // nothing ever hold an muintptr across a safe point. 220 221 // A guintptr holds a goroutine pointer, but typed as a uintptr 222 // to bypass write barriers. It is used in the Gobuf goroutine state 223 // and in scheduling lists that are manipulated without a P. 224 // 225 // The Gobuf.g goroutine pointer is almost always updated by assembly code. 226 // In one of the few places it is updated by Go code - func save - it must be 227 // treated as a uintptr to avoid a write barrier being emitted at a bad time. 228 // Instead of figuring out how to emit the write barriers missing in the 229 // assembly manipulation, we change the type of the field to uintptr, 230 // so that it does not require write barriers at all. 231 // 232 // Goroutine structs are published in the allg list and never freed. 233 // That will keep the goroutine structs from being collected. 234 // There is never a time that Gobuf.g's contain the only references 235 // to a goroutine: the publishing of the goroutine in allg comes first. 236 // Goroutine pointers are also kept in non-GC-visible places like TLS, 237 // so I can't see them ever moving. If we did want to start moving data 238 // in the GC, we'd need to allocate the goroutine structs from an 239 // alternate arena. Using guintptr doesn't make that problem any worse. 240 // Note that pollDesc.rg, pollDesc.wg also store g in uintptr form, 241 // so they would need to be updated too if g's start moving. 242 type guintptr uintptr 243 244 //go:nosplit 245 func (gp guintptr) ptr() *g { return (*g)(unsafe.Pointer(gp)) } 246 247 //go:nosplit 248 func (gp *guintptr) set(g *g) { *gp = guintptr(unsafe.Pointer(g)) } 249 250 //go:nosplit 251 func (gp *guintptr) cas(old, new guintptr) bool { 252 return atomic.Casuintptr((*uintptr)(unsafe.Pointer(gp)), uintptr(old), uintptr(new)) 253 } 254 255 //go:nosplit 256 func (gp *g) guintptr() guintptr { 257 return guintptr(unsafe.Pointer(gp)) 258 } 259 260 // setGNoWB performs *gp = new without a write barrier. 261 // For times when it's impractical to use a guintptr. 262 // 263 //go:nosplit 264 //go:nowritebarrier 265 func setGNoWB(gp **g, new *g) { 266 (*guintptr)(unsafe.Pointer(gp)).set(new) 267 } 268 269 type puintptr uintptr 270 271 //go:nosplit 272 func (pp puintptr) ptr() *p { return (*p)(unsafe.Pointer(pp)) } 273 274 //go:nosplit 275 func (pp *puintptr) set(p *p) { *pp = puintptr(unsafe.Pointer(p)) } 276 277 // muintptr is a *m that is not tracked by the garbage collector. 278 // 279 // Because we do free Ms, there are some additional constrains on 280 // muintptrs: 281 // 282 // 1. Never hold an muintptr locally across a safe point. 283 // 284 // 2. Any muintptr in the heap must be owned by the M itself so it can 285 // ensure it is not in use when the last true *m is released. 286 type muintptr uintptr 287 288 //go:nosplit 289 func (mp muintptr) ptr() *m { return (*m)(unsafe.Pointer(mp)) } 290 291 //go:nosplit 292 func (mp *muintptr) set(m *m) { *mp = muintptr(unsafe.Pointer(m)) } 293 294 // setMNoWB performs *mp = new without a write barrier. 295 // For times when it's impractical to use an muintptr. 296 // 297 //go:nosplit 298 //go:nowritebarrier 299 func setMNoWB(mp **m, new *m) { 300 (*muintptr)(unsafe.Pointer(mp)).set(new) 301 } 302 303 type gobuf struct { 304 // ctxt is unusual with respect to GC: it may be a 305 // heap-allocated funcval, so GC needs to track it, but it 306 // needs to be set and cleared from assembly, where it's 307 // difficult to have write barriers. However, ctxt is really a 308 // saved, live register, and we only ever exchange it between 309 // the real register and the gobuf. Hence, we treat it as a 310 // root during stack scanning, which means assembly that saves 311 // and restores it doesn't need write barriers. It's still 312 // typed as a pointer so that any other writes from Go get 313 // write barriers. 314 sp uintptr 315 pc uintptr 316 g guintptr 317 ctxt unsafe.Pointer 318 lr uintptr // link register; on wasm32, PC_F paired with pc 319 bp uintptr // for framepointer-enabled architectures 320 } 321 322 // maybeTraceablePtr is a special pointer that is conditionally trackable 323 // by the GC. It consists of an address as a uintptr (vu) and a pointer 324 // to a data element (vp). 325 // 326 // maybeTraceablePtr values can be in one of three states: 327 // 1. Unset: vu == 0 && vp == nil 328 // 2. Untracked: vu != 0 && vp == nil 329 // 3. Tracked: vu != 0 && vp != nil 330 // 331 // Do not set fields manually. Use methods instead. 332 // Extend this type with additional methods if needed. 333 type maybeTraceablePtr struct { 334 vp unsafe.Pointer // For liveness only. 335 vu uintptr // Source of truth. 336 } 337 338 // untrack unsets the pointer but preserves the address. 339 // This is used to hide the pointer from the GC. 340 // 341 //go:nosplit 342 func (p *maybeTraceablePtr) setUntraceable() { 343 p.vp = nil 344 } 345 346 // setTraceable resets the pointer to the stored address. 347 // This is used to make the pointer visible to the GC. 348 // 349 //go:nosplit 350 func (p *maybeTraceablePtr) setTraceable() { 351 p.vp = unsafe.Pointer(p.vu) 352 } 353 354 // set sets the pointer to the data element and updates the address. 355 // 356 //go:nosplit 357 func (p *maybeTraceablePtr) set(v unsafe.Pointer) { 358 p.vp = v 359 p.vu = uintptr(v) 360 } 361 362 // get retrieves the pointer to the data element. 363 // 364 //go:nosplit 365 func (p *maybeTraceablePtr) get() unsafe.Pointer { 366 return unsafe.Pointer(p.vu) 367 } 368 369 // uintptr returns the uintptr address of the pointer. 370 // 371 //go:nosplit 372 func (p *maybeTraceablePtr) uintptr() uintptr { 373 return p.vu 374 } 375 376 // maybeTraceableChan extends conditionally trackable pointers (maybeTraceablePtr) 377 // to track hchan pointers. 378 // 379 // Do not set fields manually. Use methods instead. 380 type maybeTraceableChan struct { 381 maybeTraceablePtr 382 } 383 384 //go:nosplit 385 func (p *maybeTraceableChan) set(c *hchan) { 386 p.maybeTraceablePtr.set(unsafe.Pointer(c)) 387 } 388 389 //go:nosplit 390 func (p *maybeTraceableChan) get() *hchan { 391 return (*hchan)(p.maybeTraceablePtr.get()) 392 } 393 394 // waitingGType describes which type of synchronization primitive a waitingG is 395 // used with. 396 type waitingGType uint8 397 398 const ( 399 waitingGTypeNone waitingGType = iota 400 401 // Channel operations, including select. 402 waitingGTypeChan 403 404 // semaphores (semacquire, which transitively includes sync.Mutex/RWMutex). 405 waitingGTypeSema 406 407 // sync.Cond, via runtime.notifyList. 408 waitingGTypeCond 409 ) 410 411 // waitingG represents a g on a wait list of a synchronization primitive: 412 // either a channel, semaphore (including sync.Mutex) or notifyList 413 // (sync.Cond). 414 // 415 // waitingG is necessary because the g ↔ synchronization object relation 416 // is many-to-many. A g can be on many wait lists, so there may be 417 // many waitingGs for one g; and many gs may be waiting on the same 418 // synchronization object, so there may be many waitingGs for one object. 419 // 420 // waitingGs are allocated from a special pool. Use acquireWaitingG and 421 // releaseWaitingG to allocate and free them. 422 type waitingG struct { 423 // Which type of waitingG is this. Immutable after allocation. 424 typ waitingGType 425 426 // For channels, the following fields are protected by the 427 // hchan.lock of the channel this waitingG is blocking on. shrinkstack 428 // depends on this for waitingGs involved in channel ops. 429 // 430 // For semaphores, all fields are only accessed when holding a semaRoot 431 // lock. 432 433 g *g 434 435 next *waitingG 436 prev *waitingG 437 438 elem maybeTraceablePtr // data element (may point to stack) 439 440 // The following fields are never accessed concurrently. 441 // For channels, waitlink is only accessed by g. 442 443 acquiretime int64 444 releasetime int64 445 ticket uint32 446 447 // isSelect indicates g is participating in a select, so 448 // g.selectDone must be CAS'd to win the wake-up race. 449 isSelect bool 450 451 // success indicates whether communication over channel c 452 // succeeded. It is true if the goroutine was awoken because a 453 // value was delivered over channel c, and false if awoken 454 // because c was closed. 455 success bool 456 457 // waiters is a count of semaRoot waiting list other than head of list, 458 // clamped to a uint16 to fit in unused space. 459 // Only meaningful at the head of the list. 460 // (If we wanted to be overly clever, we could store a high 16 bits 461 // in the second entry in the list.) 462 waiters uint16 463 464 parent *waitingG // semaRoot binary tree 465 waitlink *waitingG // g.waiting list or semaRoot 466 waittail *waitingG // semaRoot 467 c maybeTraceableChan // channel 468 } 469 470 type libcall struct { 471 fn uintptr 472 n uintptr // number of parameters 473 args uintptr // parameters 474 r1 uintptr // return values 475 r2 uintptr 476 err uintptr // error number 477 } 478 479 // Stack describes a Go execution stack. 480 // The bounds of the stack are exactly [lo, hi), 481 // with no implicit data structures on either side. 482 type stack struct { 483 lo uintptr 484 hi uintptr 485 } 486 487 // A stackOff is a position on a goroutine's stack, recorded as its distance 488 // below the top of the stack (gp.stack.hi). 489 // 490 // Stack copying preserves the distance from every stack address to the top of 491 // the stack, so unlike an address, a stackOff remains valid across stack copies 492 // without any adjustment. And unlike a pointer, it does not keep stack objects 493 // alive during stack scanning. 494 type stackOff uintptr 495 496 // toStackOff returns the stackOff of addr, which must be on gp's stack 497 // (or exactly gp.stack.hi, which can be a frame boundary). It throws if addr 498 // isn't, which catches addresses taken on the wrong goroutine or left stale by 499 // a stack move. 500 // 501 // This must be nosplit: callers pass absolute stack addresses, which a stack 502 // move in this function's prologue would leave stale. 503 // 504 //go:nosplit 505 func (gp *g) toStackOff(addr uintptr) stackOff { 506 if addr < gp.stack.lo || addr > gp.stack.hi { 507 badStackOff(gp, addr) 508 } 509 return stackOff(gp.stack.hi - addr) 510 } 511 512 // badStackOff reports a toStackOff of an address that is not on gp's stack. 513 // It is out of line to keep toStackOff inlinable. 514 // 515 //go:noinline 516 func badStackOff(gp *g, addr uintptr) { 517 print("runtime: address ", hex(addr), " not on goroutine ", gp.goid, " stack [", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n") 518 throw("toStackOff: address not on stack") 519 } 520 521 // fromStackOff returns the current address of off on gp's stack. The result is 522 // only valid until gp's stack next moves. 523 // 524 // This must be nosplit for the same reason as toStackOff: callers may have 525 // loaded other absolute stack addresses to compare against the result. 526 // 527 //go:nosplit 528 func (gp *g) fromStackOff(off stackOff) uintptr { 529 return gp.stack.hi - uintptr(off) 530 } 531 532 // heldLockInfo gives info on a held lock and the rank of that lock 533 type heldLockInfo struct { 534 lockAddr uintptr 535 rank lockRank 536 } 537 538 type g struct { 539 // Stack parameters. 540 // stack describes the actual stack memory: [stack.lo, stack.hi). 541 // stackguard0 is the stack pointer compared in the Go stack growth prologue. 542 // It is stack.lo+StackGuard normally, but can be StackPreempt to trigger a preemption. 543 // stackguard1 is the stack pointer compared in the //go:systemstack stack growth prologue. 544 // It is stack.lo+StackGuard on g0 and gsignal stacks. 545 // It is ~0 on other goroutine stacks, to trigger a call to morestackc (and crash). 546 stack stack // offset known to runtime/cgo 547 stackguard0 uintptr // offset known to cmd/internal/obj/* 548 stackguard1 uintptr // offset known to cmd/internal/obj/* 549 550 _panic *_panic // innermost panic 551 _defer *_defer // innermost defer 552 m *m // current m 553 sched gobuf 554 syscallsp uintptr // if status==Gsyscall, syscallsp = sched.sp to use during gc 555 syscallpc uintptr // if status==Gsyscall, syscallpc = sched.pc to use during gc 556 syscallbp uintptr // if status==Gsyscall, syscallbp = sched.bp to use in fpTraceback 557 stktopsp uintptr // expected sp at top of stack, to check in traceback 558 // param is a generic pointer parameter field used to pass 559 // values in particular contexts where other storage for the 560 // parameter would be difficult to find. It is currently used 561 // in four ways: 562 // 1. When a channel operation wakes up a blocked goroutine, it sets param to 563 // point to the waitingG of the completed blocking operation. 564 // 2. By gcAssistAlloc1 to signal back to its caller that the goroutine completed 565 // the GC cycle. It is unsafe to do so in any other way, because the goroutine's 566 // stack may have moved in the meantime. 567 // 3. By debugCallWrap to pass parameters to a new goroutine because allocating a 568 // closure in the runtime is forbidden. 569 // 4. When a panic is recovered and control returns to the respective frame, 570 // param may point to a savedOpenDeferState. 571 param unsafe.Pointer 572 atomicstatus atomic.Uint32 573 wasmCont wasmContPerG // GOWASM=stackswitch only; fills the padding before goid 574 goid uint64 575 schedlink guintptr 576 waitsince int64 // approx time when the g become blocked 577 waitreason waitReason // if status==Gwaiting 578 579 preempt bool // preemption signal, duplicates stackguard0 = stackpreempt 580 preemptStop bool // transition to _Gpreempted on preemption; otherwise, just deschedule 581 preemptShrink bool // shrink stack at synchronous safe point 582 583 // asyncSafePoint is set if g is stopped at an asynchronous 584 // safe point. This means there are frames on the stack 585 // without precise pointer information. 586 asyncSafePoint bool 587 588 paniconfault bool // panic (instead of crash) on unexpected fault address 589 gcscandone bool // g has scanned stack; protected by _Gscan bit in status 590 throwsplit bool // must not split stack 591 // activeStackChans indicates that there are unlocked channels 592 // pointing into this goroutine's stack. If true, stack 593 // copying needs to acquire channel locks to protect these 594 // areas of the stack. 595 activeStackChans bool 596 // parkingOnChan indicates that the goroutine is about to 597 // park on a chansend or chanrecv. Used to signal an unsafe point 598 // for stack shrinking. 599 parkingOnChan atomic.Bool 600 // inMarkAssist indicates whether the goroutine is in mark assist. 601 // Used by the execution tracer. 602 inMarkAssist bool 603 coroexit bool // argument to coroswitch_m 604 605 raceignore int8 // ignore race detection events 606 nocgocallback bool // whether disable callback from C 607 tracking bool // whether we're tracking this G for sched latency statistics 608 trackingSeq uint8 // used to decide whether to track this G 609 trackingStamp int64 // timestamp of when the G last started being tracked 610 runnableTime int64 // the amount of time spent runnable, cleared when running, only used when tracking 611 lockedm muintptr 612 fipsIndicator uint8 613 fipsOnlyBypass bool 614 ditWanted bool // set if g wants to be executed with DIT enabled 615 syncSafePoint bool // set if g is stopped at a synchronous safe point. 616 runningCleanups atomic.Bool 617 618 // waiting references the synchronization primitives that g is 619 // currently or soon waiting on. 620 // 621 // For waitingGTypeChan: waiting.elem refers to the src/dst location 622 // for the channel operation. waitingG.waitlink forms a linked list of 623 // channels the G is waiting on (that have a valid elem pointer), in 624 // lock order. 625 // 626 // For waitingGTypeSema: waiting.elem refers to the semaphore the G is 627 // waiting on. 628 // 629 // For waitingGTypeCond: waiting.elem refers to the notifyList the G is 630 // waiting on. 631 // 632 // Note that waiting need not be set atomically with parking. Thus the 633 // G may be preempted with waiting set. 634 waiting *waitingG 635 636 sig uint32 637 secret int32 // current nesting of runtime/secret.Do calls. 638 writebuf []byte 639 sigcode0 uintptr 640 sigcode1 uintptr 641 sigpc uintptr 642 parentGoid uint64 // goid of goroutine that created this goroutine 643 gopc uintptr // pc of go statement that created this goroutine 644 ancestors *[]ancestorInfo // ancestor information goroutine(s) that created this goroutine (only used if debug.tracebackancestors) 645 startpc uintptr // pc of goroutine function 646 racectx uintptr 647 cgoCtxt []uintptr // cgo traceback context 648 labels unsafe.Pointer // profiler labels 649 timer *timer // cached timer for time.Sleep 650 sleepWhen int64 // when to sleep until 651 selectDone atomic.Uint32 // are we participating in a select and did someone win the race? 652 653 // goroutineProfiled indicates the status of this goroutine's stack for the 654 // current in-progress goroutine profile 655 goroutineProfiled goroutineProfileStateHolder 656 657 coroarg *coro // argument during coroutine transfers 658 bubble *synctestBubble 659 660 // xRegs stores the extended register state if this G has been 661 // asynchronously preempted. 662 xRegs xRegPerG 663 664 // valgrind is the Valgrind state of this G's stack. It is empty 665 // unless the runtime is built with Valgrind support (see valgrind.go). 666 valgrind valgrindPerG 667 668 // Per-G tracer state. 669 trace gTraceState 670 671 // Per-G GC state 672 673 // gcAssistBytes is this G's GC assist credit in terms of 674 // bytes allocated. If this is positive, then the G has credit 675 // to allocate gcAssistBytes bytes without assisting. If this 676 // is negative, then the G must correct this by performing 677 // scan work. We track this in bytes to make it fast to update 678 // and check for debt in the malloc hot path. The assist ratio 679 // determines how this corresponds to scan work debt. 680 gcAssistBytes int64 681 } 682 683 // gTrackingPeriod is the number of transitions out of _Grunning between 684 // latency tracking runs. 685 const gTrackingPeriod = 8 686 687 const ( 688 // tlsSlots is the number of pointer-sized slots reserved for TLS on some platforms, 689 // like Windows. 690 tlsSlots = 6 691 tlsSize = tlsSlots * goarch.PtrSize 692 ) 693 694 // Values for m.freeWait. 695 const ( 696 freeMStack = 0 // M done, free stack and reference. 697 freeMRef = 1 // M done, free reference. 698 freeMWait = 2 // M still in use. 699 ) 700 701 type m struct { 702 g0 *g // goroutine with scheduling stack 703 morebuf gobuf // gobuf arg to morestack 704 divmod uint32 // div/mod denominator for arm - known to liblink (cmd/internal/obj/arm/obj5.go) 705 706 // Fields whose offsets are not known to debuggers. 707 708 procid uint64 // for debuggers, but offset not hard-coded 709 gsignal *g // signal-handling g 710 goSigStack gsignalStack // Go-allocated signal handling stack 711 sigmask sigset // storage for saved signal mask 712 tls [tlsSlots]uintptr // thread-local storage (for x86 extern register) 713 mstartfn func() 714 curg *g // current running goroutine 715 caughtsig guintptr // goroutine running during fatal signal 716 717 // Indicates whether we've received a signal while 718 // running in secret mode. 719 signalSecret bool 720 721 // p is the currently attached P for executing Go code, nil if not executing user Go code. 722 // 723 // A non-nil p implies exclusive ownership of the P, unless curg is in _Gsyscall. 724 // In _Gsyscall the scheduler may mutate this instead. The point of synchronization 725 // is the _Gscan bit on curg's status. The scheduler must arrange to prevent curg 726 // from transitioning out of _Gsyscall if it intends to mutate p. 727 p puintptr 728 729 nextp puintptr // The next P to install before executing. Implies exclusive ownership of this P. 730 oldp puintptr // The P that was attached before executing a syscall. 731 id int64 732 mallocing int32 733 throwing throwType 734 preemptoff string // if != "", keep curg running on this m 735 locks int32 736 dying int32 737 profilehz int32 738 spinning bool // m is out of work and is actively looking for work 739 blocked bool // m is blocked on a note 740 newSigstack bool // minit on C thread called sigaltstack 741 printlock int8 742 incgo bool // m is executing a cgo call 743 isextra bool // m is an extra m 744 isExtraInC bool // m is an extra m that does not have any Go frames 745 isExtraInSig bool // m is an extra m in a signal handler 746 freeWait atomic.Uint32 // Whether it is safe to free g0 and delete m (one of freeMRef, freeMStack, freeMWait) 747 needextram bool 748 g0StackAccurate bool // whether the g0 stack has accurate bounds 749 traceback uint8 750 allpSnapshot []*p // Snapshot of allp for use after dropping P in findRunnable, nil otherwise. 751 ncgocall uint64 // number of cgo calls in total 752 ncgo int32 // number of cgo calls currently in progress 753 cgoCallersUse atomic.Uint32 // if non-zero, cgoCallers in use temporarily 754 cgoCallers *cgoCallers // cgo traceback if crashing in cgo call 755 park note 756 alllink *m // on allm 757 schedlink muintptr 758 idleNode listNodeManual 759 lockedg guintptr 760 createstack [32]uintptr // stack that created this thread, it's used for StackRecord.Stack0, so it must align with it. 761 lockedExt uint32 // tracking for external LockOSThread 762 lockedInt uint32 // tracking for internal lockOSThread 763 mWaitList mWaitList // list of runtime lock waiters 764 ditEnabled bool // set if DIT is currently enabled on this M 765 syncIOWorker bool // dedicated Windows synchronous pipe I/O worker; immutable after startup 766 767 mLockProfile mLockProfile // fields relating to runtime.lock contention 768 profStack []uintptr // used for memory/block/mutex stack traces 769 770 // wait* are used to carry arguments from gopark into park_m, because 771 // there's no stack to put them on. That is their sole purpose. 772 waitunlockf func(*g, unsafe.Pointer) bool 773 waitlock unsafe.Pointer 774 waitTraceSkip int 775 waitTraceBlockReason traceBlockReason 776 777 syscalltick uint32 778 freelink *m // on sched.freem 779 trace mTraceState 780 781 // These are here to avoid using the G stack so the stack can move during the call. 782 libcallpc uintptr // for cpu profiler 783 libcallsp uintptr 784 libcallg guintptr 785 winsyscall winlibcall // stores syscall parameters on windows 786 787 vdsoSP uintptr // SP for traceback while in VDSO call (0 if not in call) 788 vdsoPC uintptr // PC for traceback while in VDSO call 789 790 // preemptGen counts the number of completed preemption 791 // signals. This is used to detect when a preemption is 792 // requested, but fails. 793 preemptGen atomic.Uint32 794 795 // Whether this is a pending preemption signal on this M. 796 signalPending atomic.Uint32 797 798 // pcvalue lookup cache 799 pcvalueCache pcvalueCache 800 801 dlogPerM 802 803 mOS 804 805 chacha8 chacha8rand.State 806 cheaprand uint32 807 cheaprand64 uint64 808 809 // Up to 10 locks held by this m, maintained by the lock ranking code. 810 locksHeldLen int 811 locksHeld [10]heldLockInfo 812 813 // self points this M until mexit clears it to return nil. 814 self mWeakPointer 815 } 816 817 const mRedZoneSize = (16 << 3) * asanenabledBit // redZoneSize(2048) 818 819 type mPadded struct { 820 m 821 822 // Size the runtime.m structure so it fits in the 2048-byte size class, and 823 // not in the next-smallest (1792-byte) size class. That leaves the 11 low 824 // bits of muintptr values available for flags, as required by 825 // lock_spinbit.go. 826 _ [(1 - goarch.IsWasm) * (2048 - mallocHeaderSize - mRedZoneSize - unsafe.Sizeof(m{}))]byte 827 } 828 829 // mWeakPointer is a "weak" pointer to an M. A weak pointer for each M is 830 // available as m.self. Users may copy mWeakPointer arbitrarily, and get will 831 // return the M if it is still live, or nil after mexit. 832 // 833 // The zero value is treated as a nil pointer. 834 // 835 // Note that get may race with M exit. A successful get will keep the m object 836 // alive, but the M itself may be exited and thus not actually usable. 837 type mWeakPointer struct { 838 m *atomic.Pointer[m] 839 } 840 841 func newMWeakPointer(mp *m) mWeakPointer { 842 w := mWeakPointer{m: new(atomic.Pointer[m])} 843 w.m.Store(mp) 844 return w 845 } 846 847 func (w mWeakPointer) get() *m { 848 if w.m == nil { 849 return nil 850 } 851 return w.m.Load() 852 } 853 854 // clear sets the weak pointer to nil. It cannot be used on zero value 855 // mWeakPointers. 856 func (w mWeakPointer) clear() { 857 w.m.Store(nil) 858 } 859 860 type p struct { 861 id int32 862 status uint32 // one of pidle/prunning/... 863 link puintptr 864 schedtick uint32 // incremented on every scheduler call 865 syscalltick uint32 // incremented on every system call 866 sysmontick sysmontick // last tick observed by sysmon 867 m muintptr // back-link to associated m (nil if idle) 868 mcache *mcache 869 pcache pageCache 870 raceprocctx uintptr 871 872 // oldm is the previous m this p ran on. 873 // 874 // We are not associated with this m, so we have no control over its 875 // lifecycle. This value is an m.self object which points to the m 876 // until the m exits. 877 // 878 // Note that this m may be idle, running, or exiting. It should only be 879 // used with mgetSpecific, which will take ownership of the m only if 880 // it is idle. 881 oldm mWeakPointer 882 883 deferpool []*_defer // pool of available defer structs (see panic.go) 884 deferpoolbuf [32]*_defer 885 886 // Cache of goroutine ids, amortizes accesses to runtime·sched.goidgen. 887 goidcache uint64 888 goidcacheend uint64 889 890 // Queue of runnable goroutines. Accessed without lock. 891 runqhead uint32 892 runqtail uint32 893 runq [256]guintptr 894 // runnext, if non-nil, is a runnable G that was ready'd by 895 // the current G and should be run next instead of what's in 896 // runq if there's time remaining in the running G's time 897 // slice. It will inherit the time left in the current time 898 // slice. If a set of goroutines is locked in a 899 // communicate-and-wait pattern, this schedules that set as a 900 // unit and eliminates the (potentially large) scheduling 901 // latency that otherwise arises from adding the ready'd 902 // goroutines to the end of the run queue. 903 // 904 // Note that while other P's may atomically CAS this to zero, 905 // only the owner P can CAS it to a valid G. 906 runnext guintptr 907 908 // Available G's (status == Gdead) 909 gFree gList 910 911 waitingGCache []*waitingG 912 waitingGBuf [128]*waitingG 913 914 // Cache of mspan objects from the heap. 915 mspancache struct { 916 // We need an explicit length here because this field is used 917 // in allocation codepaths where write barriers are not allowed, 918 // and eliminating the write barrier/keeping it eliminated from 919 // slice updates is tricky, more so than just managing the length 920 // ourselves. 921 len int 922 buf [128]*mspan 923 } 924 925 // Cache of a single pinner object to reduce allocations from repeated 926 // pinner creation. 927 pinnerCache *pinner 928 929 // Cache of one unused pin counter. Accessed by this P with preemption 930 // disabled, or during STW. Returned to the allocator when the P is destroyed. 931 pinCounterCache *specialPinCounter 932 933 trace pTraceState 934 935 palloc persistentAlloc // per-P to avoid mutex 936 937 // Per-P GC state 938 gcAssistTime int64 // Nanoseconds in assistAlloc 939 gcFractionalMarkTime atomic.Int64 // Nanoseconds in fractional mark worker 940 941 // limiterEvent tracks events for the GC CPU limiter. 942 limiterEvent limiterEvent 943 944 // gcMarkWorkerMode is the mode for the next mark worker to run in. 945 // That is, this is used to communicate with the worker goroutine 946 // selected for immediate execution by 947 // gcController.findRunnableGCWorker. When scheduling other goroutines, 948 // this field must be set to gcMarkWorkerNotWorker. 949 gcMarkWorkerMode gcMarkWorkerMode 950 // gcMarkWorkerStartTime is the nanotime() at which the most recent 951 // mark worker started. 952 gcMarkWorkerStartTime int64 953 954 // nextGCMarkWorker is the next mark worker to run. This may be set 955 // during start-the-world to assign a worker to this P. The P runs this 956 // worker on the next call to gcController.findRunnableGCWorker. If the 957 // P runs something else or stops, it must release this worker via 958 // gcController.releaseNextGCMarkWorker. 959 // 960 // See comment in gcBgMarkWorker about the lifetime of 961 // gcBgMarkWorkerNode. 962 // 963 // Only accessed by this P or during STW. 964 nextGCMarkWorker *gcBgMarkWorkerNode 965 966 // gcw is this P's GC work buffer cache. The work buffer is 967 // filled by write barriers, drained by mutator assists, and 968 // disposed on certain GC state transitions. 969 gcw gcWork 970 971 // wbBuf is this P's GC write barrier buffer. 972 // 973 // TODO: Consider caching this in the running G. 974 wbBuf wbBuf 975 976 runSafePointFn uint32 // if 1, run sched.safePointFn at next safe point 977 978 // statsSeq is a counter indicating whether this P is currently 979 // writing any stats. Its value is even when not, odd when it is. 980 statsSeq atomic.Uint32 981 982 // Timer heap. 983 timers timers 984 985 // Cleanups. 986 cleanups *cleanupBlock 987 cleanupsQueued uint64 // monotonic count of cleanups queued by this P 988 989 // maxStackScanDelta accumulates the amount of stack space held by 990 // live goroutines (i.e. those eligible for stack scanning). 991 // Flushed to gcController.maxStackScan once maxStackScanSlack 992 // or -maxStackScanSlack is reached. 993 maxStackScanDelta int64 994 995 // gc-time statistics about current goroutines 996 // Note that this differs from maxStackScan in that this 997 // accumulates the actual stack observed to be used at GC time (hi - sp), 998 // not an instantaneous measure of the total stack size that might need 999 // to be scanned (hi - lo). 1000 scannedStackSize uint64 // stack size of goroutines scanned by this P 1001 scannedStacks uint64 // number of goroutines scanned by this P 1002 1003 // preempt is set to indicate that this P should be enter the 1004 // scheduler ASAP (regardless of what G is running on it). 1005 preempt bool 1006 1007 // gcStopTime is the nanotime timestamp that this P last entered _Pgcstop. 1008 gcStopTime int64 1009 1010 // goroutinesCreated is the total count of goroutines created by this P. 1011 goroutinesCreated uint64 1012 1013 // xRegs is the per-P extended register state used by asynchronous 1014 // preemption. This is an empty struct on platforms that don't use extended 1015 // register state. 1016 xRegs xRegPerP 1017 1018 // Padding is no longer needed. False sharing is now not a worry because p is large enough 1019 // that its size class is an integer multiple of the cache line size (for any of our architectures). 1020 } 1021 1022 type schedt struct { 1023 goidgen atomic.Uint64 1024 lastpoll atomic.Int64 // time of last network poll, 0 if currently polling 1025 pollUntil atomic.Int64 // time to which current poll is sleeping 1026 pollingNet atomic.Int32 // 1 if some P doing non-blocking network poll 1027 1028 lock mutex 1029 1030 // When increasing nmidle, nmidlelocked, nmsys, or nmfreed, be 1031 // sure to call checkdead(). 1032 1033 midle listHeadManual // idle m's waiting for work 1034 nmidle int32 // number of idle m's waiting for work 1035 nmidlelocked int32 // number of locked m's waiting for work 1036 mnext int64 // number of m's that have been created and next M ID 1037 maxmcount int32 // maximum number of m's allowed (or die) 1038 nmsys int32 // number of system m's not counted for deadlock 1039 nmfreed int64 // cumulative number of freed m's 1040 1041 ngsys atomic.Int32 // number of system goroutines 1042 nGsyscallNoP atomic.Int32 // number of goroutines in syscalls without a P but whose M is not isExtraInC 1043 1044 pidle puintptr // idle p's 1045 npidle atomic.Int32 1046 nmspinning atomic.Int32 // See "Worker thread parking/unparking" comment in proc.go. 1047 needspinning atomic.Uint32 // See "Delicate dance" comment in proc.go. Boolean. Must hold sched.lock to set to 1. 1048 1049 // Global runnable queue. 1050 runq gQueue 1051 1052 // disable controls selective disabling of the scheduler. 1053 // 1054 // Use schedEnableUser to control this. 1055 // 1056 // disable is protected by sched.lock. 1057 disable struct { 1058 // user disables scheduling of user goroutines. 1059 user bool 1060 runnable gQueue // pending runnable Gs 1061 } 1062 1063 // Global cache of dead G's. 1064 gFree struct { 1065 lock mutex 1066 stack gList // Gs with stacks 1067 noStack gList // Gs without stacks 1068 } 1069 1070 // Central cache of waitingG structs. 1071 waitingGLock mutex 1072 waitingGCache *waitingG 1073 1074 // Central pool of available defer structs. 1075 deferlock mutex 1076 deferpool *_defer 1077 1078 // freem is the list of m's waiting to be freed when their 1079 // m.exited is set. Linked through m.freelink. 1080 freem *m 1081 1082 // gcwaiting indicates that the gc is waiting to run. When runqgrab delays 1083 // before stealing runnext, platforms with futex sleep on this address. 1084 gcwaiting gcWaitingWord 1085 stopwait int32 1086 stopnote note 1087 sysmonwait atomic.Bool 1088 sysmonnote note 1089 1090 // safePointFn should be called on each P at the next GC 1091 // safepoint if p.runSafePointFn is set. 1092 safePointFn func(*p) 1093 safePointWait int32 1094 safePointNote note 1095 1096 profilehz int32 // cpu profiling rate 1097 1098 procresizetime int64 // nanotime() of last change to gomaxprocs 1099 totaltime int64 // ∫gomaxprocs dt up to procresizetime 1100 1101 customGOMAXPROCS bool // GOMAXPROCS was manually set from the environment or runtime.GOMAXPROCS 1102 1103 // sysmonlock protects sysmon's actions on the runtime. 1104 // 1105 // Acquire and hold this mutex to block sysmon from interacting 1106 // with the rest of the runtime. 1107 sysmonlock mutex 1108 1109 // timeToRun is a distribution of scheduling latencies, defined 1110 // as the sum of time a G spends in the _Grunnable state before 1111 // it transitions to _Grunning. 1112 timeToRun timeHistogram 1113 1114 // idleTime is the total CPU time Ps have "spent" idle. 1115 // 1116 // Reset on each GC cycle. 1117 idleTime atomic.Int64 1118 1119 // totalMutexWaitTime is the sum of time goroutines have spent in _Gwaiting 1120 // with a waitreason of the form waitReasonSync{RW,}Mutex{R,}Lock. 1121 totalMutexWaitTime atomic.Int64 1122 1123 // stwStoppingTimeGC/Other are distributions of stop-the-world stopping 1124 // latencies, defined as the time taken by stopTheWorldWithSema to get 1125 // all Ps to stop. stwStoppingTimeGC covers all GC-related STWs, 1126 // stwStoppingTimeOther covers the others. 1127 stwStoppingTimeGC timeHistogram 1128 stwStoppingTimeOther timeHistogram 1129 1130 // stwTotalTimeGC/Other are distributions of stop-the-world total 1131 // latencies, defined as the total time from stopTheWorldWithSema to 1132 // startTheWorldWithSema. This is a superset of 1133 // stwStoppingTimeGC/Other. stwTotalTimeGC covers all GC-related STWs, 1134 // stwTotalTimeOther covers the others. 1135 stwTotalTimeGC timeHistogram 1136 stwTotalTimeOther timeHistogram 1137 1138 // totalRuntimeLockWaitTime (plus the value of lockWaitTime on each M in 1139 // allm) is the sum of time goroutines have spent in _Grunnable and with an 1140 // M, but waiting for locks within the runtime. This field stores the value 1141 // for Ms that have exited. 1142 totalRuntimeLockWaitTime atomic.Int64 1143 1144 // goroutinesCreated (plus the value of goroutinesCreated on each P in allp) 1145 // is the sum of all goroutines created by the program. 1146 goroutinesCreated atomic.Uint64 1147 } 1148 1149 // gcWaitingWord is an atomic boolean, whose address is also available as a 1150 // *uint32 for futexsleep. 1151 type gcWaitingWord struct { 1152 val uint32 1153 } 1154 1155 //go:nosplit 1156 func (p *gcWaitingWord) Store(v bool) { 1157 var val uint32 1158 if v { 1159 val = 1 1160 } 1161 atomic.Store(&p.val, val) 1162 } 1163 1164 //go:nosplit 1165 func (p *gcWaitingWord) Load() bool { 1166 return atomic.Load(&p.val) != 0 1167 } 1168 1169 //go:nosplit 1170 func (p *gcWaitingWord) Addr() *uint32 { 1171 return &p.val 1172 } 1173 1174 // Values for the flags field of a sigTabT. 1175 const ( 1176 _SigNotify = 1 << iota // let signal.Notify have signal, even if from kernel 1177 _SigKill // if signal.Notify doesn't take it, exit quietly 1178 _SigThrow // if signal.Notify doesn't take it, exit loudly 1179 _SigPanic // if the signal is from the kernel, panic 1180 _SigDefault // if the signal isn't explicitly requested, don't monitor it 1181 _SigGoExit // cause all runtime procs to exit (only used on Plan 9). 1182 _SigSetStack // Don't explicitly install handler, but add SA_ONSTACK to existing libc handler 1183 _SigUnblock // always unblock; see blockableSig 1184 _SigIgn // _SIG_DFL action is to ignore the signal 1185 ) 1186 1187 // Layout of in-memory per-function information prepared by linker 1188 // See https://golang.org/s/go12symtab. 1189 // Keep in sync with linker (../cmd/link/internal/ld/pcln.go:/pclntab) 1190 // and with package debug/gosym and with symtab.go in package runtime. 1191 type _func struct { 1192 sys.NotInHeap // Only in static data 1193 1194 entryOff uint32 // start pc, as offset from moduledata.text 1195 nameOff int32 // function name, as index into moduledata.funcnametab. 1196 1197 args int32 // in/out args size 1198 deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any. 1199 1200 pcsp uint32 1201 pcfile uint32 1202 pcln uint32 1203 npcdata uint32 1204 cuOffset uint32 // runtime.cutab offset of this function's CU 1205 startLine int32 // line number of start of function (func keyword/TEXT directive) 1206 funcID abi.FuncID // set for certain special runtime functions 1207 flag abi.FuncFlag 1208 _ [1]byte // pad 1209 nfuncdata uint8 // must be last, must end on a uint32-aligned boundary 1210 1211 // The end of the struct is followed immediately by two variable-length 1212 // arrays that reference the pcdata and funcdata locations for this 1213 // function. 1214 1215 // pcdata contains the offset into moduledata.pctab for the start of 1216 // that index's table. e.g., 1217 // &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of 1218 // the unsafe point table. 1219 // 1220 // An offset of 0 indicates that there is no table. 1221 // 1222 // pcdata [npcdata]uint32 1223 1224 // funcdata contains the offset past moduledata.gofunc which contains a 1225 // pointer to that index's funcdata. e.g., 1226 // *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is 1227 // the argument pointer map. 1228 // 1229 // An offset of ^uint32(0) indicates that there is no entry. 1230 // 1231 // funcdata [nfuncdata]uint32 1232 } 1233 1234 // Pseudo-Func that is returned for PCs that occur in inlined code. 1235 // A *Func can be either a *_func or a *funcinl, and they are distinguished 1236 // by the first uintptr. 1237 // 1238 // TODO(austin): Can we merge this with inlinedCall? 1239 type funcinl struct { 1240 ones uint32 // set to ^0 to distinguish from _func 1241 entry uintptr // entry of the real (the "outermost") frame 1242 name string 1243 file string 1244 line int32 1245 startLine int32 1246 } 1247 1248 type itab = abi.ITab 1249 1250 // Lock-free stack node. 1251 // Also known to export_test.go. 1252 type lfnode struct { 1253 next uint64 1254 pushcnt uintptr 1255 } 1256 1257 type forcegcstate struct { 1258 lock mutex 1259 g *g 1260 idle atomic.Bool 1261 } 1262 1263 // A _defer holds an entry on the list of deferred calls. 1264 // If you add a field here, add code to clear it in deferProcStack. 1265 // This struct must match the code in cmd/compile/internal/ssagen/ssa.go:deferstruct 1266 // and cmd/compile/internal/ssagen/ssa.go:(*state).call. 1267 // Some defers will be allocated on the stack and some on the heap. 1268 // All defers are logically part of the stack, so write barriers to 1269 // initialize them are not required. All defers must be manually scanned, 1270 // and for heap defers, marked. 1271 type _defer struct { 1272 heap bool 1273 rangefunc bool // true for rangefunc list 1274 sp uintptr // sp at time of defer 1275 pc uintptr // pc at time of defer 1276 fn func() // can be nil for open-coded defers 1277 link *_defer // next defer on G; can point to either heap or stack! 1278 1279 // If rangefunc is true, *head is the head of the atomic linked list 1280 // during a range-over-func execution. 1281 head *atomic.Pointer[_defer] 1282 } 1283 1284 // A _panic holds information about an active panic. 1285 // 1286 // A _panic value must only ever live on the stack. 1287 // 1288 // The link, deferBitsPtr, and slotsPtr fields are stack pointers, but 1289 // don't need special handling during stack growth: because they are 1290 // pointer-typed and _panic values only live on the stack, regular stack 1291 // pointer adjustment takes care of them. 1292 // 1293 // startSP, sp, and fp record frame boundaries. These are stackOffs rather 1294 // than pointers because a frame boundary is often also the address of a 1295 // stack object (for example, on amd64, a caller's SP is the address of the 1296 // stack arguments it passes to its callee), and a pointer would keep that 1297 // object alive after it is dead. See go.dev/issue/82394 and 1298 // go.dev/issue/82917. 1299 type _panic struct { 1300 arg any // argument to panic 1301 link *_panic // link to earlier panic 1302 1303 // startPC and startSP track where _panic.start was called. 1304 // (These are the SP and PC of the gopanic frame itself.) 1305 startPC uintptr 1306 startSP stackOff 1307 1308 // The current stack frame that we're running deferred calls for. 1309 pc uintptr 1310 sp stackOff 1311 fp stackOff 1312 1313 // retpc stores the PC where the panic should jump back to, if the 1314 // function last returned by _panic.nextDefer() recovers the panic. 1315 retpc uintptr 1316 1317 // Extra state for handling open-coded defers. 1318 deferBitsPtr *uint8 1319 slotsPtr unsafe.Pointer 1320 1321 recoverAtToken *recoverAtToken // identify this panic to gorecoverat 1322 1323 recovered bool // whether this panic has been recovered 1324 repanicked bool // whether this panic repanicked 1325 goexit bool 1326 deferreturn bool 1327 } 1328 1329 // savedOpenDeferState tracks the extra state from _panic that's 1330 // necessary for deferreturn to pick up where gopanic left off, 1331 // without needing to unwind the stack. 1332 type savedOpenDeferState struct { 1333 retpc uintptr 1334 deferBitsOffset uintptr 1335 slotsOffset uintptr 1336 } 1337 1338 // ancestorInfo records details of where a goroutine was started. 1339 type ancestorInfo struct { 1340 pcs []uintptr // pcs from the stack of this goroutine 1341 goid uint64 // goroutine id of this goroutine; original goroutine possibly dead 1342 gopc uintptr // pc of go statement that created this goroutine 1343 } 1344 1345 // A waitReason explains why a goroutine has been stopped. 1346 // See gopark. Do not re-use waitReasons, add new ones. 1347 type waitReason uint8 1348 1349 const ( 1350 waitReasonZero waitReason = iota // "" 1351 waitReasonGCAssistMarking // "GC assist marking" 1352 waitReasonIOWait // "IO wait" 1353 waitReasonDumpingHeap // "dumping heap" 1354 waitReasonGarbageCollection // "garbage collection" 1355 waitReasonGarbageCollectionScan // "garbage collection scan" 1356 waitReasonPanicWait // "panicwait" 1357 waitReasonGCAssistWait // "GC assist wait" 1358 waitReasonGCSweepWait // "GC sweep wait" 1359 waitReasonGCScavengeWait // "GC scavenge wait" 1360 waitReasonFinalizerWait // "finalizer wait" 1361 waitReasonForceGCIdle // "force gc (idle)" 1362 waitReasonUpdateGOMAXPROCSIdle // "GOMAXPROCS updater (idle)" 1363 waitReasonSemacquire // "semacquire" 1364 waitReasonSleep // "sleep" 1365 waitReasonChanReceiveNilChan // "chan receive (nil chan)" 1366 waitReasonChanSendNilChan // "chan send (nil chan)" 1367 waitReasonSelectNoCases // "select (no cases)" 1368 waitReasonSelect // "select" 1369 waitReasonChanReceive // "chan receive" 1370 waitReasonChanSend // "chan send" 1371 waitReasonSyncCondWait // "sync.Cond.Wait" 1372 waitReasonSyncMutexLock // "sync.Mutex.Lock" 1373 waitReasonSyncRWMutexRLock // "sync.RWMutex.RLock" 1374 waitReasonSyncRWMutexLock // "sync.RWMutex.Lock" 1375 waitReasonSyncWaitGroupWait // "sync.WaitGroup.Wait" 1376 waitReasonTraceReaderBlocked // "trace reader (blocked)" 1377 waitReasonWaitForGCCycle // "wait for GC cycle" 1378 waitReasonGCWorkerIdle // "GC worker (idle)" 1379 waitReasonGCWorkerActive // "GC worker (active)" 1380 waitReasonPreempted // "preempted" 1381 waitReasonDebugCall // "debug call" 1382 waitReasonGCMarkTermination // "GC mark termination" 1383 waitReasonStoppingTheWorld // "stopping the world" 1384 waitReasonFlushProcCaches // "flushing proc caches" 1385 waitReasonTraceGoroutineStatus // "trace goroutine status" 1386 waitReasonTraceProcStatus // "trace proc status" 1387 waitReasonPageTraceFlush // "page trace flush" 1388 waitReasonCoroutine // "coroutine" 1389 waitReasonGCWeakToStrongWait // "GC weak to strong wait" 1390 waitReasonSynctestRun // "synctest.Run" 1391 waitReasonSynctestWait // "synctest.Wait" 1392 waitReasonSynctestChanReceive // "chan receive (durable)" 1393 waitReasonSynctestChanSend // "chan send (durable)" 1394 waitReasonSynctestSelect // "select (durable)" 1395 waitReasonSynctestWaitGroupWait // "sync.WaitGroup.Wait (durable)" 1396 waitReasonCleanupWait // "cleanup wait" 1397 ) 1398 1399 var waitReasonStrings = [...]string{ 1400 waitReasonZero: "", 1401 waitReasonGCAssistMarking: "GC assist marking", 1402 waitReasonIOWait: "IO wait", 1403 waitReasonChanReceiveNilChan: "chan receive (nil chan)", 1404 waitReasonChanSendNilChan: "chan send (nil chan)", 1405 waitReasonDumpingHeap: "dumping heap", 1406 waitReasonGarbageCollection: "garbage collection", 1407 waitReasonGarbageCollectionScan: "garbage collection scan", 1408 waitReasonPanicWait: "panicwait", 1409 waitReasonSelect: "select", 1410 waitReasonSelectNoCases: "select (no cases)", 1411 waitReasonGCAssistWait: "GC assist wait", 1412 waitReasonGCSweepWait: "GC sweep wait", 1413 waitReasonGCScavengeWait: "GC scavenge wait", 1414 waitReasonChanReceive: "chan receive", 1415 waitReasonChanSend: "chan send", 1416 waitReasonFinalizerWait: "finalizer wait", 1417 waitReasonForceGCIdle: "force gc (idle)", 1418 waitReasonUpdateGOMAXPROCSIdle: "GOMAXPROCS updater (idle)", 1419 waitReasonSemacquire: "semacquire", 1420 waitReasonSleep: "sleep", 1421 waitReasonSyncCondWait: "sync.Cond.Wait", 1422 waitReasonSyncMutexLock: "sync.Mutex.Lock", 1423 waitReasonSyncRWMutexRLock: "sync.RWMutex.RLock", 1424 waitReasonSyncRWMutexLock: "sync.RWMutex.Lock", 1425 waitReasonSyncWaitGroupWait: "sync.WaitGroup.Wait", 1426 waitReasonTraceReaderBlocked: "trace reader (blocked)", 1427 waitReasonWaitForGCCycle: "wait for GC cycle", 1428 waitReasonGCWorkerIdle: "GC worker (idle)", 1429 waitReasonGCWorkerActive: "GC worker (active)", 1430 waitReasonPreempted: "preempted", 1431 waitReasonDebugCall: "debug call", 1432 waitReasonGCMarkTermination: "GC mark termination", 1433 waitReasonStoppingTheWorld: "stopping the world", 1434 waitReasonFlushProcCaches: "flushing proc caches", 1435 waitReasonTraceGoroutineStatus: "trace goroutine status", 1436 waitReasonTraceProcStatus: "trace proc status", 1437 waitReasonPageTraceFlush: "page trace flush", 1438 waitReasonCoroutine: "coroutine", 1439 waitReasonGCWeakToStrongWait: "GC weak to strong wait", 1440 waitReasonSynctestRun: "synctest.Run", 1441 waitReasonSynctestWait: "synctest.Wait", 1442 waitReasonSynctestChanReceive: "chan receive (durable)", 1443 waitReasonSynctestChanSend: "chan send (durable)", 1444 waitReasonSynctestSelect: "select (durable)", 1445 waitReasonSynctestWaitGroupWait: "sync.WaitGroup.Wait (durable)", 1446 waitReasonCleanupWait: "cleanup wait", 1447 } 1448 1449 func (w waitReason) String() string { 1450 if w < 0 || w >= waitReason(len(waitReasonStrings)) { 1451 return "unknown wait reason" 1452 } 1453 return waitReasonStrings[w] 1454 } 1455 1456 // isMutexWait returns true if the goroutine is blocked because of 1457 // sync.Mutex.Lock or sync.RWMutex.[R]Lock. 1458 // 1459 //go:nosplit 1460 func (w waitReason) isMutexWait() bool { 1461 return w == waitReasonSyncMutexLock || 1462 w == waitReasonSyncRWMutexRLock || 1463 w == waitReasonSyncRWMutexLock 1464 } 1465 1466 // isSyncWait returns true if the goroutine is blocked because of 1467 // sync library primitive operations. 1468 // 1469 //go:nosplit 1470 func (w waitReason) isSyncWait() bool { 1471 return waitReasonSyncCondWait <= w && w <= waitReasonSyncWaitGroupWait 1472 } 1473 1474 // isChanWait is true if the goroutine is blocked because of non-nil 1475 // channel operations or a select statement with at least one case. 1476 // 1477 //go:nosplit 1478 func (w waitReason) isChanWait() bool { 1479 return w == waitReasonSelect || 1480 w == waitReasonChanReceive || 1481 w == waitReasonChanSend 1482 } 1483 1484 func (w waitReason) isWaitingForSuspendG() bool { 1485 return isWaitingForSuspendG[w] 1486 } 1487 1488 // isWaitingForSuspendG indicates that a goroutine is only entering _Gwaiting and 1489 // setting a waitReason because it needs to be able to let the suspendG 1490 // (used by the GC and the execution tracer) take ownership of its stack. 1491 // The G is always actually executing on the system stack in these cases. 1492 // 1493 // TODO(mknyszek): Consider replacing this with a new dedicated G status. 1494 var isWaitingForSuspendG = [len(waitReasonStrings)]bool{ 1495 waitReasonStoppingTheWorld: true, 1496 waitReasonGCMarkTermination: true, 1497 waitReasonGarbageCollection: true, 1498 waitReasonGarbageCollectionScan: true, 1499 waitReasonTraceGoroutineStatus: true, 1500 waitReasonTraceProcStatus: true, 1501 waitReasonPageTraceFlush: true, 1502 waitReasonGCAssistMarking: true, 1503 waitReasonGCWorkerActive: true, 1504 waitReasonFlushProcCaches: true, 1505 } 1506 1507 func (w waitReason) isIdleInSynctest() bool { 1508 return isIdleInSynctest[w] 1509 } 1510 1511 // isIdleInSynctest indicates that a goroutine is considered idle by synctest.Wait. 1512 var isIdleInSynctest = [len(waitReasonStrings)]bool{ 1513 waitReasonChanReceiveNilChan: true, 1514 waitReasonChanSendNilChan: true, 1515 waitReasonSelectNoCases: true, 1516 waitReasonSleep: true, 1517 waitReasonSyncCondWait: true, 1518 waitReasonSynctestWaitGroupWait: true, 1519 waitReasonCoroutine: true, 1520 waitReasonSynctestRun: true, 1521 waitReasonSynctestWait: true, 1522 waitReasonSynctestChanReceive: true, 1523 waitReasonSynctestChanSend: true, 1524 waitReasonSynctestSelect: true, 1525 } 1526 1527 var ( 1528 // Linked-list of all Ms. Written under sched.lock, read atomically. 1529 allm *m 1530 1531 gomaxprocs int32 1532 numCPUStartup int32 1533 forcegc forcegcstate 1534 sched schedt 1535 newprocs int32 1536 ) 1537 1538 var ( 1539 // allpLock protects P-less reads and size changes of allp, idlepMask, 1540 // and timerpMask, and all writes to allp. 1541 allpLock mutex 1542 1543 // len(allp) == gomaxprocs; may change at safe points, otherwise 1544 // immutable. 1545 allp []*p 1546 1547 // Bitmask of Ps in _Pidle list, one bit per P. Reads and writes must 1548 // be atomic. Length may change at safe points. 1549 // 1550 // Each P must update only its own bit. In order to maintain 1551 // consistency, a P going idle must set the idle mask simultaneously with 1552 // updates to the idle P list under the sched.lock, otherwise a racing 1553 // pidleget may clear the mask before pidleput sets the mask, 1554 // corrupting the bitmap. 1555 // 1556 // N.B., procresize takes ownership of all Ps in stopTheWorldWithSema. 1557 idlepMask pMask 1558 1559 // Bitmask of Ps that may have a timer, one bit per P. Reads and writes 1560 // must be atomic. Length may change at safe points. 1561 // 1562 // Ideally, the timer mask would be kept immediately consistent on any timer 1563 // operations. Unfortunately, updating a shared global data structure in the 1564 // timer hot path adds too much overhead in applications frequently switching 1565 // between no timers and some timers. 1566 // 1567 // As a compromise, the timer mask is updated only on pidleget / pidleput. A 1568 // running P (returned by pidleget) may add a timer at any time, so its mask 1569 // must be set. An idle P (passed to pidleput) cannot add new timers while 1570 // idle, so if it has no timers at that time, its mask may be cleared. 1571 // 1572 // Thus, we get the following effects on timer-stealing in findRunnable: 1573 // 1574 // - Idle Ps with no timers when they go idle are never checked in findRunnable 1575 // (for work- or timer-stealing; this is the ideal case). 1576 // - Running Ps must always be checked. 1577 // - Idle Ps whose timers are stolen must continue to be checked until they run 1578 // again, even after timer expiration. 1579 // 1580 // When the P starts running again, the mask should be set, as a timer may be 1581 // added at any time. 1582 // 1583 // TODO(prattmic): Additional targeted updates may improve the above cases. 1584 // e.g., updating the mask when stealing a timer. 1585 timerpMask pMask 1586 ) 1587 1588 // goarmsoftfp is used by runtime/cgo assembly. 1589 // 1590 //go:linkname goarmsoftfp 1591 1592 var ( 1593 // Pool of GC parked background workers. Entries are type 1594 // *gcBgMarkWorkerNode. 1595 gcBgMarkWorkerPool lfstack 1596 1597 // Total number of gcBgMarkWorker goroutines. Protected by worldsema. 1598 gcBgMarkWorkerCount int32 1599 1600 // Information about what cpu features are available. 1601 // Packages outside the runtime should not use these 1602 // as they are not an external api. 1603 // Set on startup in asm_{386,amd64}.s 1604 processorVersionInfo uint32 1605 isIntel bool 1606 ) 1607 1608 // set by cmd/link on arm systems 1609 // accessed using linkname by internal/runtime/atomic. 1610 // 1611 // goarm should be an internal detail, 1612 // but widely used packages access it using linkname. 1613 // Notable members of the hall of shame include: 1614 // - github.com/creativeprojects/go-selfupdate 1615 // 1616 // Do not remove or change the type signature. 1617 // See go.dev/issue/67401. 1618 // 1619 //go:linkname goarm 1620 var ( 1621 goarm uint8 1622 goarmsoftfp uint8 1623 ) 1624 1625 // Set by the linker so the runtime can determine the buildmode. 1626 var ( 1627 islibrary bool // -buildmode=c-shared 1628 isarchive bool // -buildmode=c-archive 1629 ) 1630 1631 // Must agree with internal/buildcfg.FramePointerEnabled. 1632 const framepointer_enabled = GOARCH == "amd64" || GOARCH == "arm64" 1633 1634 // getcallerfp returns the frame pointer of the caller of the caller 1635 // of this function. 1636 // 1637 //go:nosplit 1638 //go:noinline 1639 func getcallerfp() uintptr { 1640 fp := getfp() // This frame's FP. 1641 if fp != 0 { 1642 fp = *(*uintptr)(unsafe.Pointer(fp)) // The caller's FP. 1643 fp = *(*uintptr)(unsafe.Pointer(fp)) // The caller's caller's FP. 1644 } 1645 return fp 1646 } 1647