Source file src/runtime/mstats.go
1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Memory statistics 6 7 package runtime 8 9 import ( 10 "internal/runtime/atomic" 11 "internal/runtime/gc" 12 "unsafe" 13 ) 14 15 type mstats struct { 16 // Statistics about malloc heap. 17 heapStats consistentHeapStats 18 19 // Statistics about stacks. 20 stacks_sys sysMemStat // only counts newosproc0 stack in mstats; differs from MemStats.StackSys 21 22 // Statistics about allocation of low-level fixed-size structures. 23 mspan_sys sysMemStat 24 mcache_sys sysMemStat 25 buckhash_sys sysMemStat // profiling bucket hash table 26 27 // Statistics about GC overhead. 28 gcMiscSys sysMemStat // updated atomically or during STW 29 30 // Miscellaneous statistics. 31 other_sys sysMemStat // updated atomically or during STW 32 33 // Statistics about the garbage collector. 34 35 // Protected by mheap or worldsema during GC. 36 last_gc_unix uint64 // last gc (in unix time) 37 pause_total_ns uint64 38 pause_ns [256]uint64 // circular buffer of recent gc pause lengths 39 pause_end [256]uint64 // circular buffer of recent gc end times (nanoseconds since 1970) 40 numgc uint32 41 numforcedgc uint32 // number of user-forced GCs 42 gc_cpu_fraction float64 // fraction of CPU time used by GC 43 44 last_gc_nanotime uint64 // last gc (monotonic time) 45 lastHeapInUse uint64 // heapInUse at mark termination of the previous GC 46 47 lastScanStats [gc.NumSizeClasses]sizeClassScanStats 48 49 enablegc bool 50 } 51 52 var memstats mstats 53 54 // A MemStats records statistics about the memory allocator. 55 type MemStats struct { 56 // General statistics. 57 58 // Alloc is bytes of allocated heap objects. 59 // 60 // This is the same as HeapAlloc (see below). 61 Alloc uint64 62 63 // TotalAlloc is cumulative bytes allocated for heap objects. 64 // 65 // TotalAlloc increases as heap objects are allocated, but 66 // unlike Alloc and HeapAlloc, it does not decrease when 67 // objects are freed. 68 TotalAlloc uint64 69 70 // Sys is the total bytes of memory obtained from the OS. 71 // 72 // Sys is the sum of the XSys fields below. Sys measures the 73 // virtual address space reserved by the Go runtime for the 74 // heap, stacks, and other internal data structures. It's 75 // likely that not all of the virtual address space is backed 76 // by physical memory at any given moment, though in general 77 // it all was at some point. 78 Sys uint64 79 80 // Lookups is the number of pointer lookups performed by the 81 // runtime. 82 // 83 // This is primarily useful for debugging runtime internals. 84 Lookups uint64 85 86 // Mallocs is the cumulative count of heap objects allocated. 87 // The number of live objects is Mallocs - Frees. 88 Mallocs uint64 89 90 // Frees is the cumulative count of heap objects freed. 91 Frees uint64 92 93 // Heap memory statistics. 94 // 95 // Interpreting the heap statistics requires some knowledge of 96 // how Go organizes memory. Go divides the virtual address 97 // space of the heap into "spans", which are contiguous 98 // regions of memory 8K or larger. A span may be in one of 99 // three states: 100 // 101 // An "idle" span contains no objects or other data. The 102 // physical memory backing an idle span can be released back 103 // to the OS (but the virtual address space never is), or it 104 // can be converted into an "in use" or "stack" span. 105 // 106 // An "in use" span contains at least one heap object and may 107 // have free space available to allocate more heap objects. 108 // 109 // A "stack" span is used for goroutine stacks. Stack spans 110 // are not considered part of the heap. A span can change 111 // between heap and stack memory; it is never used for both 112 // simultaneously. 113 114 // HeapAlloc is bytes of allocated heap objects. 115 // 116 // "Allocated" heap objects include all reachable objects, as 117 // well as unreachable objects that the garbage collector has 118 // not yet freed. Specifically, HeapAlloc increases as heap 119 // objects are allocated and decreases as the heap is swept 120 // and unreachable objects are freed. Sweeping occurs 121 // incrementally between GC cycles, so these two processes 122 // occur simultaneously, and as a result HeapAlloc tends to 123 // change smoothly (in contrast with the sawtooth that is 124 // typical of stop-the-world garbage collectors). 125 HeapAlloc uint64 126 127 // HeapSys is bytes of heap memory obtained from the OS. 128 // 129 // HeapSys measures the amount of virtual address space 130 // reserved for the heap. This includes virtual address space 131 // that has been reserved but not yet used, which consumes no 132 // physical memory, but tends to be small, as well as virtual 133 // address space for which the physical memory has been 134 // returned to the OS after it became unused (see HeapReleased 135 // for a measure of the latter). 136 // 137 // HeapSys estimates the largest size the heap has had. 138 HeapSys uint64 139 140 // HeapIdle is bytes in idle (unused) spans. 141 // 142 // Idle spans have no objects in them. These spans could be 143 // (and may already have been) returned to the OS, or they can 144 // be reused for heap allocations, or they can be reused as 145 // stack memory. 146 // 147 // HeapIdle minus HeapReleased estimates the amount of memory 148 // that could be returned to the OS, but is being retained by 149 // the runtime so it can grow the heap without requesting more 150 // memory from the OS. If this difference is significantly 151 // larger than the heap size, it indicates there was a recent 152 // transient spike in live heap size. 153 HeapIdle uint64 154 155 // HeapInuse is bytes in in-use spans. 156 // 157 // In-use spans have at least one object in them. These spans 158 // can only be used for other objects of roughly the same 159 // size. 160 // 161 // HeapInuse minus HeapAlloc estimates the amount of memory 162 // that has been dedicated to particular size classes, but is 163 // not currently being used. This is an upper bound on 164 // fragmentation, but in general this memory can be reused 165 // efficiently. 166 HeapInuse uint64 167 168 // HeapReleased is bytes of physical memory returned to the OS. 169 // 170 // This counts heap memory from idle spans that was returned 171 // to the OS and has not yet been reacquired for the heap. 172 HeapReleased uint64 173 174 // HeapObjects is the number of allocated heap objects. 175 // 176 // Like HeapAlloc, this increases as objects are allocated and 177 // decreases as the heap is swept and unreachable objects are 178 // freed. 179 HeapObjects uint64 180 181 // Stack memory statistics. 182 // 183 // Stacks are not considered part of the heap, but the runtime 184 // can reuse a span of heap memory for stack memory, and 185 // vice-versa. 186 187 // StackInuse is bytes in stack spans. 188 // 189 // In-use stack spans have at least one stack in them. These 190 // spans can only be used for other stacks of the same size. 191 // 192 // There is no StackIdle because unused stack spans are 193 // returned to the heap (and hence counted toward HeapIdle). 194 StackInuse uint64 195 196 // StackSys is bytes of stack memory obtained from the OS. 197 // 198 // StackSys is StackInuse, plus any memory obtained directly 199 // from the OS for OS thread stacks. 200 // 201 // In non-cgo programs this metric is currently equal to StackInuse 202 // (but this should not be relied upon, and the value may change in 203 // the future). 204 // 205 // In cgo programs this metric includes OS thread stacks allocated 206 // directly from the OS. Currently, this only accounts for one stack in 207 // c-shared and c-archive build modes and other sources of stacks from 208 // the OS (notably, any allocated by C code) are not currently measured. 209 // Note this too may change in the future. 210 StackSys uint64 211 212 // Off-heap memory statistics. 213 // 214 // The following statistics measure runtime-internal 215 // structures that are not allocated from heap memory (usually 216 // because they are part of implementing the heap). Unlike 217 // heap or stack memory, any memory allocated to these 218 // structures is dedicated to these structures. 219 // 220 // These are primarily useful for debugging runtime memory 221 // overheads. 222 223 // MSpanInuse is bytes of allocated mspan structures. 224 MSpanInuse uint64 225 226 // MSpanSys is bytes of memory obtained from the OS for mspan 227 // structures. 228 MSpanSys uint64 229 230 // MCacheInuse is bytes of allocated mcache structures. 231 MCacheInuse uint64 232 233 // MCacheSys is bytes of memory obtained from the OS for 234 // mcache structures. 235 MCacheSys uint64 236 237 // BuckHashSys is bytes of memory in profiling bucket hash tables. 238 BuckHashSys uint64 239 240 // GCSys is bytes of memory in garbage collection metadata. 241 GCSys uint64 242 243 // OtherSys is bytes of memory in miscellaneous off-heap 244 // runtime allocations. 245 OtherSys uint64 246 247 // Garbage collector statistics. 248 249 // NextGC is the target heap size of the next GC cycle. 250 // 251 // The garbage collector's goal is to keep HeapAlloc ≤ NextGC. 252 // At the end of each GC cycle, the target for the next cycle 253 // is computed based on the amount of reachable data and the 254 // value of GOGC. 255 NextGC uint64 256 257 // LastGC is the time the last garbage collection finished, as 258 // nanoseconds since 1970 (the UNIX epoch). 259 LastGC uint64 260 261 // PauseTotalNs is the cumulative nanoseconds in GC 262 // stop-the-world pauses since the program started. 263 // 264 // During a stop-the-world pause, all goroutines are paused 265 // and only the garbage collector can run. 266 PauseTotalNs uint64 267 268 // PauseNs is a circular buffer of recent GC stop-the-world 269 // pause times in nanoseconds. 270 // 271 // The most recent pause is at PauseNs[(NumGC+255)%256]. In 272 // general, PauseNs[N%256] records the time paused in the most 273 // recent N%256th GC cycle. There may be multiple pauses per 274 // GC cycle; this is the sum of all pauses during a cycle. 275 PauseNs [256]uint64 276 277 // PauseEnd is a circular buffer of recent GC pause end times, 278 // as nanoseconds since 1970 (the UNIX epoch). 279 // 280 // This buffer is filled the same way as PauseNs. There may be 281 // multiple pauses per GC cycle; this records the end of the 282 // last pause in a cycle. 283 PauseEnd [256]uint64 284 285 // NumGC is the number of completed GC cycles. 286 NumGC uint32 287 288 // NumForcedGC is the number of GC cycles that were forced by 289 // the application calling the GC function. 290 NumForcedGC uint32 291 292 // GCCPUFraction is the fraction of this program's available 293 // CPU time used by the GC since the program started. 294 // 295 // GCCPUFraction is expressed as a number between 0 and 1, 296 // where 0 means GC has consumed none of this program's CPU. A 297 // program's available CPU time is defined as the integral of 298 // GOMAXPROCS since the program started. That is, if 299 // GOMAXPROCS is 2 and a program has been running for 10 300 // seconds, its "available CPU" is 20 seconds. GCCPUFraction 301 // does not include CPU time used for write barrier activity. 302 // 303 // This is the same as the fraction of CPU reported by 304 // GODEBUG=gctrace=1. 305 GCCPUFraction float64 306 307 // EnableGC indicates that GC is enabled. It is always true, 308 // even if GOGC=off. 309 EnableGC bool 310 311 // DebugGC is currently unused. 312 DebugGC bool 313 314 // BySize reports per-size class allocation statistics. 315 // 316 // BySize[N] gives statistics for allocations of size S where 317 // BySize[N-1].Size < S ≤ BySize[N].Size. 318 // 319 // This does not report allocations larger than BySize[60].Size. 320 BySize [61]struct { 321 // Size is the maximum byte size of an object in this 322 // size class. 323 Size uint32 324 325 // Mallocs is the cumulative count of heap objects 326 // allocated in this size class. The cumulative bytes 327 // of allocation is Size*Mallocs. The number of live 328 // objects in this size class is Mallocs - Frees. 329 Mallocs uint64 330 331 // Frees is the cumulative count of heap objects freed 332 // in this size class. 333 Frees uint64 334 } 335 } 336 337 func init() { 338 if offset := unsafe.Offsetof(memstats.heapStats); offset%8 != 0 { 339 println(offset) 340 throw("memstats.heapStats not aligned to 8 bytes") 341 } 342 // Ensure the size of heapStatsDelta causes adjacent fields/slots (e.g. 343 // [3]heapStatsDelta) to be 8-byte aligned. 344 if size := unsafe.Sizeof(heapStatsDelta{}); size%8 != 0 { 345 println(size) 346 throw("heapStatsDelta not a multiple of 8 bytes in size") 347 } 348 } 349 350 // ReadMemStats populates m with memory allocator statistics. 351 // 352 // The returned memory allocator statistics are up to date as of the 353 // call to ReadMemStats. This is in contrast with a heap profile, 354 // which is a snapshot as of the most recently completed garbage 355 // collection cycle. 356 func ReadMemStats(m *MemStats) { 357 _ = m.Alloc // nil check test before we switch stacks, see issue 61158 358 stw := stopTheWorld(stwReadMemStats) 359 360 systemstack(func() { 361 readmemstats_m(m) 362 }) 363 364 startTheWorld(stw) 365 } 366 367 // doubleCheckReadMemStats controls a double-check mode for ReadMemStats that 368 // ensures consistency between the values that ReadMemStats is using and the 369 // runtime-internal stats. 370 var doubleCheckReadMemStats = false 371 372 // readmemstats_m populates stats for internal runtime values. 373 // 374 // The world must be stopped. 375 func readmemstats_m(stats *MemStats) { 376 assertWorldStopped() 377 378 // Flush mcaches to mcentral before doing anything else. 379 // 380 // Flushing to the mcentral may in general cause stats to 381 // change as mcentral data structures are manipulated. 382 systemstack(flushallmcaches) 383 384 // Calculate memory allocator stats. 385 // During program execution we only count number of frees and amount of freed memory. 386 // Current number of alive objects in the heap and amount of alive heap memory 387 // are calculated by scanning all spans. 388 // Total number of mallocs is calculated as number of frees plus number of alive objects. 389 // Similarly, total amount of allocated memory is calculated as amount of freed memory 390 // plus amount of alive heap memory. 391 392 // Collect consistent stats, which are the source-of-truth in some cases. 393 var consStats heapStatsDelta 394 memstats.heapStats.unsafeRead(&consStats) 395 396 // Collect large allocation stats. 397 totalAlloc := consStats.largeAlloc 398 nMalloc := consStats.largeAllocCount 399 totalFree := consStats.largeFree 400 nFree := consStats.largeFreeCount 401 402 // Collect per-sizeclass stats. 403 var bySize [gc.NumSizeClasses]struct { 404 Size uint32 405 Mallocs uint64 406 Frees uint64 407 } 408 for i := range bySize { 409 bySize[i].Size = uint32(gc.SizeClassToSize[i]) 410 411 // Malloc stats. 412 a := consStats.smallAllocCount[i] 413 totalAlloc += a * uint64(gc.SizeClassToSize[i]) 414 nMalloc += a 415 bySize[i].Mallocs = a 416 417 // Free stats. 418 f := consStats.smallFreeCount[i] 419 totalFree += f * uint64(gc.SizeClassToSize[i]) 420 nFree += f 421 bySize[i].Frees = f 422 } 423 424 // Account for tiny allocations. 425 // For historical reasons, MemStats includes tiny allocations 426 // in both the total free and total alloc count. This double-counts 427 // memory in some sense because their tiny allocation block is also 428 // counted. Tracking the lifetime of individual tiny allocations is 429 // currently not done because it would be too expensive. 430 nFree += consStats.tinyAllocCount 431 nMalloc += consStats.tinyAllocCount 432 433 // Calculate derived stats. 434 435 stackInUse := uint64(consStats.inStacks) 436 gcWorkBufInUse := uint64(consStats.inWorkBufs) 437 438 totalMapped := gcController.heapInUse.load() + gcController.heapFree.load() + gcController.heapReleased.load() + 439 memstats.stacks_sys.load() + memstats.mspan_sys.load() + memstats.mcache_sys.load() + 440 memstats.buckhash_sys.load() + memstats.gcMiscSys.load() + memstats.other_sys.load() + 441 stackInUse + gcWorkBufInUse 442 443 heapGoal := gcController.heapGoal() 444 445 if doubleCheckReadMemStats { 446 // Only check this if we're debugging. It would be bad to crash an application 447 // just because the debugging stats are wrong. We mostly rely on tests to catch 448 // these issues, and we enable the double check mode for tests. 449 // 450 // The world is stopped, so the consistent stats (after aggregation) 451 // should be identical to some combination of memstats. In particular: 452 // 453 // * memstats.heapInUse == inHeap 454 // * memstats.heapReleased == released 455 // * memstats.heapInUse + memstats.heapFree == committed - inStacks - inWorkBufs 456 // * memstats.totalAlloc == totalAlloc 457 // * memstats.totalFree == totalFree 458 // 459 // Check if that's actually true. 460 // 461 // Prevent sysmon and the tracer from skewing the stats since they can 462 // act without synchronizing with a STW. See #64401. 463 lock(&sched.sysmonlock) 464 lock(&trace.lock) 465 if gcController.heapInUse.load() != uint64(consStats.inHeap) { 466 print("runtime: heapInUse=", gcController.heapInUse.load(), "\n") 467 print("runtime: consistent value=", consStats.inHeap, "\n") 468 throw("heapInUse and consistent stats are not equal") 469 } 470 if gcController.heapReleased.load() != uint64(consStats.released) { 471 print("runtime: heapReleased=", gcController.heapReleased.load(), "\n") 472 print("runtime: consistent value=", consStats.released, "\n") 473 throw("heapReleased and consistent stats are not equal") 474 } 475 heapRetained := gcController.heapInUse.load() + gcController.heapFree.load() 476 consRetained := uint64(consStats.committed - consStats.inStacks - consStats.inWorkBufs) 477 if heapRetained != consRetained { 478 print("runtime: global value=", heapRetained, "\n") 479 print("runtime: consistent value=", consRetained, "\n") 480 throw("measures of the retained heap are not equal") 481 } 482 if gcController.totalAlloc.Load() != totalAlloc { 483 print("runtime: totalAlloc=", gcController.totalAlloc.Load(), "\n") 484 print("runtime: consistent value=", totalAlloc, "\n") 485 throw("totalAlloc and consistent stats are not equal") 486 } 487 if gcController.totalFree.Load() != totalFree { 488 print("runtime: totalFree=", gcController.totalFree.Load(), "\n") 489 print("runtime: consistent value=", totalFree, "\n") 490 throw("totalFree and consistent stats are not equal") 491 } 492 // Also check that mappedReady lines up with totalMapped - released. 493 // This isn't really the same type of "make sure consistent stats line up" situation, 494 // but this is an opportune time to check. 495 if gcController.mappedReady.Load() != totalMapped-uint64(consStats.released) { 496 print("runtime: mappedReady=", gcController.mappedReady.Load(), "\n") 497 print("runtime: totalMapped=", totalMapped, "\n") 498 print("runtime: released=", uint64(consStats.released), "\n") 499 print("runtime: totalMapped-released=", totalMapped-uint64(consStats.released), "\n") 500 throw("mappedReady and other memstats are not equal") 501 } 502 unlock(&trace.lock) 503 unlock(&sched.sysmonlock) 504 } 505 506 // We've calculated all the values we need. Now, populate stats. 507 508 stats.Alloc = totalAlloc - totalFree 509 stats.TotalAlloc = totalAlloc 510 stats.Sys = totalMapped 511 stats.Mallocs = nMalloc 512 stats.Frees = nFree 513 stats.HeapAlloc = totalAlloc - totalFree 514 stats.HeapSys = gcController.heapInUse.load() + gcController.heapFree.load() + gcController.heapReleased.load() 515 // By definition, HeapIdle is memory that was mapped 516 // for the heap but is not currently used to hold heap 517 // objects. It also specifically is memory that can be 518 // used for other purposes, like stacks, but this memory 519 // is subtracted out of HeapSys before it makes that 520 // transition. Put another way: 521 // 522 // HeapSys = bytes allocated from the OS for the heap - bytes ultimately used for non-heap purposes 523 // HeapIdle = bytes allocated from the OS for the heap - bytes ultimately used for any purpose 524 // 525 // or 526 // 527 // HeapSys = sys - stacks_inuse - gcWorkBufInUse 528 // HeapIdle = sys - stacks_inuse - gcWorkBufInUse - heapInUse 529 // 530 // => HeapIdle = HeapSys - heapInUse = heapFree + heapReleased 531 stats.HeapIdle = gcController.heapFree.load() + gcController.heapReleased.load() 532 stats.HeapInuse = gcController.heapInUse.load() 533 stats.HeapReleased = gcController.heapReleased.load() 534 stats.HeapObjects = nMalloc - nFree 535 stats.StackInuse = stackInUse 536 // memstats.stacks_sys is only memory mapped directly for OS stacks. 537 // Add in heap-allocated stack memory for user consumption. 538 stats.StackSys = stackInUse + memstats.stacks_sys.load() 539 stats.MSpanInuse = uint64(mheap_.spanalloc.inuse) 540 stats.MSpanSys = memstats.mspan_sys.load() 541 stats.MCacheInuse = uint64(mheap_.cachealloc.inuse) 542 stats.MCacheSys = memstats.mcache_sys.load() 543 stats.BuckHashSys = memstats.buckhash_sys.load() 544 // MemStats defines GCSys as an aggregate of all memory related 545 // to the memory management system, but we track this memory 546 // at a more granular level in the runtime. 547 stats.GCSys = memstats.gcMiscSys.load() + gcWorkBufInUse 548 stats.OtherSys = memstats.other_sys.load() 549 stats.NextGC = heapGoal 550 stats.LastGC = memstats.last_gc_unix 551 stats.PauseTotalNs = memstats.pause_total_ns 552 stats.PauseNs = memstats.pause_ns 553 stats.PauseEnd = memstats.pause_end 554 stats.NumGC = memstats.numgc 555 stats.NumForcedGC = memstats.numforcedgc 556 stats.GCCPUFraction = memstats.gc_cpu_fraction 557 stats.EnableGC = true 558 559 // stats.BySize and bySize might not match in length. 560 // That's OK, stats.BySize cannot change due to backwards 561 // compatibility issues. copy will copy the minimum amount 562 // of values between the two of them. 563 copy(stats.BySize[:], bySize[:]) 564 } 565 566 //go:linkname readGCStats runtime/debug.readGCStats 567 func readGCStats(pauses *[]uint64) { 568 systemstack(func() { 569 readGCStats_m(pauses) 570 }) 571 } 572 573 // readGCStats_m must be called on the system stack because it acquires the heap 574 // lock. See mheap for details. 575 // 576 //go:systemstack 577 func readGCStats_m(pauses *[]uint64) { 578 p := *pauses 579 // Calling code in runtime/debug should make the slice large enough. 580 if cap(p) < len(memstats.pause_ns)+3 { 581 throw("short slice passed to readGCStats") 582 } 583 584 // Pass back: pauses, pause ends, last gc (absolute time), number of gc, total pause ns. 585 lock(&mheap_.lock) 586 587 n := memstats.numgc 588 if n > uint32(len(memstats.pause_ns)) { 589 n = uint32(len(memstats.pause_ns)) 590 } 591 592 // The pause buffer is circular. The most recent pause is at 593 // pause_ns[(numgc-1)%len(pause_ns)], and then backward 594 // from there to go back farther in time. We deliver the times 595 // most recent first (in p[0]). 596 p = p[:cap(p)] 597 for i := uint32(0); i < n; i++ { 598 j := (memstats.numgc - 1 - i) % uint32(len(memstats.pause_ns)) 599 p[i] = memstats.pause_ns[j] 600 p[n+i] = memstats.pause_end[j] 601 } 602 603 p[n+n] = memstats.last_gc_unix 604 p[n+n+1] = uint64(memstats.numgc) 605 p[n+n+2] = memstats.pause_total_ns 606 unlock(&mheap_.lock) 607 *pauses = p[:n+n+3] 608 } 609 610 // flushmcache flushes the mcache of allp[i]. 611 // 612 // The world must be stopped. 613 // 614 //go:nowritebarrier 615 func flushmcache(i int) { 616 assertWorldStopped() 617 618 p := allp[i] 619 c := p.mcache 620 if c == nil { 621 return 622 } 623 c.releaseAll() 624 stackcache_clear(c) 625 } 626 627 // flushallmcaches flushes the mcaches of all Ps. 628 // 629 // The world must be stopped. 630 // 631 //go:nowritebarrier 632 func flushallmcaches() { 633 assertWorldStopped() 634 635 for i := 0; i < int(gomaxprocs); i++ { 636 flushmcache(i) 637 } 638 } 639 640 // sysMemStat represents a global system statistic that is managed atomically. 641 // 642 // This type must structurally be a uint64 so that mstats aligns with MemStats. 643 type sysMemStat uint64 644 645 // load atomically reads the value of the stat. 646 // 647 // Must be nosplit as it is called in runtime initialization, e.g. newosproc0. 648 // 649 //go:nosplit 650 func (s *sysMemStat) load() uint64 { 651 return atomic.Load64((*uint64)(s)) 652 } 653 654 // add atomically adds the sysMemStat by n. 655 // 656 // Must be nosplit as it is called in runtime initialization, e.g. newosproc0. 657 // 658 //go:nosplit 659 func (s *sysMemStat) add(n int64) { 660 val := atomic.Xadd64((*uint64)(s), n) 661 if (n > 0 && int64(val) < n) || (n < 0 && int64(val)+n < n) { 662 print("runtime: val=", val, " n=", n, "\n") 663 throw("sysMemStat overflow") 664 } 665 } 666 667 // heapStatsDelta contains deltas of various runtime memory statistics 668 // that need to be updated together in order for them to be kept 669 // consistent with one another. 670 type heapStatsDelta struct { 671 // Memory stats. 672 committed int64 // byte delta of memory committed 673 released int64 // byte delta of released memory generated 674 inHeap int64 // byte delta of memory placed in the heap 675 inStacks int64 // byte delta of memory reserved for stacks 676 inWorkBufs int64 // byte delta of memory reserved for work bufs 677 678 // Allocator stats. 679 // 680 // These are all uint64 because they're cumulative, and could quickly wrap 681 // around otherwise. 682 tinyAllocCount uint64 // number of tiny allocations 683 largeAlloc uint64 // bytes allocated for large objects 684 largeAllocCount uint64 // number of large object allocations 685 smallAllocCount [gc.NumSizeClasses]uint64 // number of allocs for small objects 686 largeFree uint64 // bytes freed for large objects (>maxSmallSize) 687 largeFreeCount uint64 // number of frees for large objects (>maxSmallSize) 688 smallFreeCount [gc.NumSizeClasses]uint64 // number of frees for small objects (<=maxSmallSize) 689 690 // NOTE: This struct must be a multiple of 8 bytes in size because it 691 // is stored in an array. If it's not, atomic accesses to the above 692 // fields may be unaligned and fail on 32-bit platforms. 693 } 694 695 // merge adds in the deltas from b into a. 696 func (a *heapStatsDelta) merge(b *heapStatsDelta) { 697 a.committed += b.committed 698 a.released += b.released 699 a.inHeap += b.inHeap 700 a.inStacks += b.inStacks 701 a.inWorkBufs += b.inWorkBufs 702 703 a.tinyAllocCount += b.tinyAllocCount 704 a.largeAlloc += b.largeAlloc 705 a.largeAllocCount += b.largeAllocCount 706 for i := range b.smallAllocCount { 707 a.smallAllocCount[i] += b.smallAllocCount[i] 708 } 709 a.largeFree += b.largeFree 710 a.largeFreeCount += b.largeFreeCount 711 for i := range b.smallFreeCount { 712 a.smallFreeCount[i] += b.smallFreeCount[i] 713 } 714 } 715 716 // consistentHeapStats represents a set of various memory statistics 717 // whose updates must be viewed completely to get a consistent 718 // state of the world. 719 // 720 // To write updates to memory stats use the acquire and release 721 // methods. To obtain a consistent global snapshot of these statistics, 722 // use read. 723 type consistentHeapStats struct { 724 // stats is a ring buffer of heapStatsDelta values. 725 // Writers always atomically update the delta at index gen. 726 // 727 // Readers operate by rotating gen (0 -> 1 -> 2 -> 0 -> ...) 728 // and synchronizing with writers by observing each P's 729 // statsSeq field. If the reader observes a P not writing, 730 // it can be sure that it will pick up the new gen value the 731 // next time it writes. 732 // 733 // The reader then takes responsibility by clearing space 734 // in the ring buffer for the next reader to rotate gen to 735 // that space (i.e. it merges in values from index (gen-2) mod 3 736 // to index (gen-1) mod 3, then clears the former). 737 // 738 // Note that this means only one reader can be reading at a time. 739 // There is no way for readers to synchronize. 740 // 741 // This process is why we need a ring buffer of size 3 instead 742 // of 2: one is for the writers, one contains the most recent 743 // data, and the last one is clear so writers can begin writing 744 // to it the moment gen is updated. 745 stats [3]heapStatsDelta 746 747 // gen represents the current index into which writers 748 // are writing, and can take on the value of 0, 1, or 2. 749 gen atomic.Uint32 750 751 // noPLock is intended to provide mutual exclusion for updating 752 // stats when no P is available. It does not block other writers 753 // with a P, only other writers without a P and the reader. Because 754 // stats are usually updated when a P is available, contention on 755 // this lock should be minimal. 756 noPLock mutex 757 } 758 759 // acquire returns a heapStatsDelta to be updated. In effect, 760 // it acquires the shard for writing. release must be called 761 // as soon as the relevant deltas are updated. 762 // 763 // The returned heapStatsDelta must be updated atomically. 764 // 765 // The caller's P must not change between acquire and 766 // release. This also means that the caller should not 767 // acquire a P or release its P in between. A P also must 768 // not acquire a given consistentHeapStats if it hasn't 769 // yet released it. 770 // 771 // nosplit because a stack growth in this function could 772 // lead to a stack allocation that could reenter the 773 // function. 774 // 775 //go:nosplit 776 func (m *consistentHeapStats) acquire() *heapStatsDelta { 777 if pp := getg().m.p.ptr(); pp != nil { 778 seq := pp.statsSeq.Add(1) 779 if seq%2 == 0 { 780 // Should have been incremented to odd. 781 print("runtime: seq=", seq, "\n") 782 throw("bad sequence number") 783 } 784 } else { 785 lock(&m.noPLock) 786 } 787 gen := m.gen.Load() % 3 788 return &m.stats[gen] 789 } 790 791 // release indicates that the writer is done modifying 792 // the delta. The value returned by the corresponding 793 // acquire must no longer be accessed or modified after 794 // release is called. 795 // 796 // The caller's P must not change between acquire and 797 // release. This also means that the caller should not 798 // acquire a P or release its P in between. 799 // 800 // nosplit because a stack growth in this function could 801 // lead to a stack allocation that causes another acquire 802 // before this operation has completed. 803 // 804 //go:nosplit 805 func (m *consistentHeapStats) release() { 806 if pp := getg().m.p.ptr(); pp != nil { 807 seq := pp.statsSeq.Add(1) 808 if seq%2 != 0 { 809 // Should have been incremented to even. 810 print("runtime: seq=", seq, "\n") 811 throw("bad sequence number") 812 } 813 } else { 814 unlock(&m.noPLock) 815 } 816 } 817 818 // unsafeRead aggregates the delta for this shard into out. 819 // 820 // Unsafe because it does so without any synchronization. The 821 // world must be stopped. 822 func (m *consistentHeapStats) unsafeRead(out *heapStatsDelta) { 823 assertWorldStopped() 824 825 for i := range m.stats { 826 out.merge(&m.stats[i]) 827 } 828 } 829 830 // unsafeClear clears the shard. 831 // 832 // Unsafe because the world must be stopped and values should 833 // be donated elsewhere before clearing. 834 func (m *consistentHeapStats) unsafeClear() { 835 assertWorldStopped() 836 837 clear(m.stats[:]) 838 } 839 840 // read takes a globally consistent snapshot of m 841 // and puts the aggregated value in out. Even though out is a 842 // heapStatsDelta, the resulting values should be complete and 843 // valid statistic values. 844 // 845 // Not safe to call concurrently. The world must be stopped 846 // or metricsSema must be held. 847 func (m *consistentHeapStats) read(out *heapStatsDelta) { 848 // Getting preempted after this point is not safe because 849 // we read allp. We need to make sure a STW can't happen 850 // so it doesn't change out from under us. 851 mp := acquirem() 852 853 // Get the current generation. We can be confident that this 854 // will not change since read is serialized and is the only 855 // one that modifies currGen. 856 currGen := m.gen.Load() 857 prevGen := currGen - 1 858 if currGen == 0 { 859 prevGen = 2 860 } 861 862 // Prevent writers without a P from writing while we update gen. 863 lock(&m.noPLock) 864 865 // Rotate gen, effectively taking a snapshot of the state of 866 // these statistics at the point of the exchange by moving 867 // writers to the next set of deltas. 868 // 869 // This exchange is safe to do because we won't race 870 // with anyone else trying to update this value. 871 m.gen.Swap((currGen + 1) % 3) 872 873 // Allow P-less writers to continue. They'll be writing to the 874 // next generation now. 875 unlock(&m.noPLock) 876 877 for _, p := range allp { 878 // Spin until there are no more writers. 879 for p.statsSeq.Load()%2 != 0 { 880 } 881 } 882 883 // At this point we've observed that each sequence 884 // number is even, so any future writers will observe 885 // the new gen value. That means it's safe to read from 886 // the other deltas in the stats buffer. 887 888 // Perform our responsibilities and free up 889 // stats[prevGen] for the next time we want to take 890 // a snapshot. 891 m.stats[currGen].merge(&m.stats[prevGen]) 892 m.stats[prevGen] = heapStatsDelta{} 893 894 // Finally, copy out the complete delta. 895 *out = m.stats[currGen] 896 897 releasem(mp) 898 } 899 900 type cpuStats struct { 901 // All fields are CPU time in nanoseconds computed by comparing 902 // calls of nanotime. This means they're all overestimates, because 903 // they don't accurately compute on-CPU time (so some of the time 904 // could be spent scheduled away by the OS). 905 906 GCAssistTime int64 // GC assists 907 GCDedicatedTime int64 // GC dedicated mark workers + pauses 908 GCIdleTime int64 // GC idle mark workers 909 GCPauseTime int64 // GC pauses (all GOMAXPROCS, even if just 1 is running) 910 GCTotalTime int64 911 912 ScavengeAssistTime int64 // background scavenger 913 ScavengeBgTime int64 // scavenge assists 914 ScavengeTotalTime int64 915 916 IdleTime int64 // Time Ps spent in _Pidle. 917 UserTime int64 // Time Ps spent in _Prunning or _Psyscall that's not any of the above. 918 919 TotalTime int64 // GOMAXPROCS * (monotonic wall clock time elapsed) 920 } 921 922 // accumulateGCPauseTime add dt*stwProcs to the GC CPU pause time stats. dt should be 923 // the actual time spent paused, for orthogonality. maxProcs should be GOMAXPROCS, 924 // not work.stwprocs, since this number must be comparable to a total time computed 925 // from GOMAXPROCS. 926 func (s *cpuStats) accumulateGCPauseTime(dt int64, maxProcs int32) { 927 cpu := dt * int64(maxProcs) 928 s.GCPauseTime += cpu 929 s.GCTotalTime += cpu 930 } 931 932 // accumulate takes a cpuStats and adds in the current state of all GC CPU 933 // counters. 934 // 935 // gcMarkPhase indicates that we're in the mark phase and that certain counter 936 // values should be used. 937 func (s *cpuStats) accumulate(now int64, gcMarkPhase bool) { 938 // N.B. Mark termination and sweep termination pauses are 939 // accumulated in work.cpuStats at the end of their respective pauses. 940 var ( 941 markAssistCpu int64 942 markDedicatedCpu int64 943 markFractionalCpu int64 944 markIdleCpu int64 945 ) 946 if gcMarkPhase { 947 // N.B. These stats may have stale values if the GC is not 948 // currently in the mark phase. 949 markAssistCpu = gcController.assistTime.Load() 950 markDedicatedCpu = gcController.dedicatedMarkTime.Load() 951 markFractionalCpu = gcController.fractionalMarkTime.Load() 952 markIdleCpu = gcController.idleMarkTime.Load() 953 } 954 955 // The rest of the stats below are either derived from the above or 956 // are reset on each mark termination. 957 958 scavAssistCpu := scavenge.assistTime.Load() 959 scavBgCpu := scavenge.backgroundTime.Load() 960 961 // Update cumulative GC CPU stats. 962 s.GCAssistTime += markAssistCpu 963 s.GCDedicatedTime += markDedicatedCpu + markFractionalCpu 964 s.GCIdleTime += markIdleCpu 965 s.GCTotalTime += markAssistCpu + markDedicatedCpu + markFractionalCpu + markIdleCpu 966 967 // Update cumulative scavenge CPU stats. 968 s.ScavengeAssistTime += scavAssistCpu 969 s.ScavengeBgTime += scavBgCpu 970 s.ScavengeTotalTime += scavAssistCpu + scavBgCpu 971 972 // Update total CPU. 973 s.TotalTime = sched.totaltime + (now-sched.procresizetime)*int64(gomaxprocs) 974 s.IdleTime += sched.idleTime.Load() 975 976 // Compute userTime. We compute this indirectly as everything that's not the above. 977 // 978 // Since time spent in _Pgcstop is covered by gcPauseTime, and time spent in _Pidle 979 // is covered by idleTime, what we're left with is time spent in _Prunning and _Psyscall, 980 // the latter of which is fine because the P will either go idle or get used for something 981 // else via sysmon. Meanwhile if we subtract GC time from whatever's left, we get non-GC 982 // _Prunning time. Note that this still leaves time spent in sweeping and in the scheduler, 983 // but that's fine. The overwhelming majority of this time will be actual user time. 984 s.UserTime = s.TotalTime - (s.GCTotalTime + s.ScavengeTotalTime + s.IdleTime) 985 } 986