Source file src/runtime/mgcpacer.go

     1  // Copyright 2021 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/cpu"
     9  	"internal/goexperiment"
    10  	"internal/runtime/atomic"
    11  	_ "unsafe" // for go:linkname
    12  )
    13  
    14  const (
    15  	// gcGoalUtilization is the goal CPU utilization for
    16  	// marking as a fraction of GOMAXPROCS.
    17  	//
    18  	// Increasing the goal utilization will shorten GC cycles as the GC
    19  	// has more resources behind it, lessening costs from the write barrier,
    20  	// but comes at the cost of increasing mutator latency.
    21  	gcGoalUtilization = gcBackgroundUtilization
    22  
    23  	// gcBackgroundUtilization is the fixed CPU utilization for background
    24  	// marking. It must be <= gcGoalUtilization. The difference between
    25  	// gcGoalUtilization and gcBackgroundUtilization will be made up by
    26  	// mark assists. The scheduler will aim to use within 50% of this
    27  	// goal.
    28  	//
    29  	// As a general rule, there's little reason to set gcBackgroundUtilization
    30  	// < gcGoalUtilization. One reason might be in mostly idle applications,
    31  	// where goroutines are unlikely to assist at all, so the actual
    32  	// utilization will be lower than the goal. But this is moot point
    33  	// because the idle mark workers already soak up idle CPU resources.
    34  	// These two values are still kept separate however because they are
    35  	// distinct conceptually, and in previous iterations of the pacer the
    36  	// distinction was more important.
    37  	gcBackgroundUtilization = 0.25
    38  
    39  	// gcCreditSlack is the amount of scan work credit that can
    40  	// accumulate locally before updating gcController.heapScanWork and,
    41  	// optionally, gcController.bgScanCredit. Lower values give a more
    42  	// accurate assist ratio and make it more likely that assists will
    43  	// successfully steal background credit. Higher values reduce memory
    44  	// contention.
    45  	gcCreditSlack = 2000
    46  
    47  	// gcAssistTimeSlack is the nanoseconds of mutator assist time that
    48  	// can accumulate on a P before updating gcController.assistTime.
    49  	gcAssistTimeSlack = 5000
    50  
    51  	// gcOverAssistWork determines how many extra units of scan work a GC
    52  	// assist does when an assist happens. This amortizes the cost of an
    53  	// assist by pre-paying for this many bytes of future allocations.
    54  	gcOverAssistWork = 64 << 10
    55  
    56  	// defaultHeapMinimum is the value of heapMinimum for GOGC==100.
    57  	defaultHeapMinimum = (goexperiment.HeapMinimum512KiBInt)*(512<<10) +
    58  		(1-goexperiment.HeapMinimum512KiBInt)*(4<<20)
    59  
    60  	// maxStackScanSlack is the bytes of stack space allocated or freed
    61  	// that can accumulate on a P before updating gcController.stackSize.
    62  	maxStackScanSlack = 8 << 10
    63  
    64  	// memoryLimitMinHeapGoalHeadroom is the minimum amount of headroom the
    65  	// pacer gives to the heap goal when operating in the memory-limited regime.
    66  	// That is, it'll reduce the heap goal by this many extra bytes off of the
    67  	// base calculation, at minimum.
    68  	memoryLimitMinHeapGoalHeadroom = 1 << 20
    69  
    70  	// memoryLimitHeapGoalHeadroomPercent is how headroom the memory-limit-based
    71  	// heap goal should have as a percent of the maximum possible heap goal allowed
    72  	// to maintain the memory limit.
    73  	memoryLimitHeapGoalHeadroomPercent = 3
    74  )
    75  
    76  // gcController implements the GC pacing controller that determines
    77  // when to trigger concurrent garbage collection and how much marking
    78  // work to do in mutator assists and background marking.
    79  //
    80  // It calculates the ratio between the allocation rate (in terms of CPU
    81  // time) and the GC scan throughput to determine the heap size at which to
    82  // trigger a GC cycle such that no GC assists are required to finish on time.
    83  // This algorithm thus optimizes GC CPU utilization to the dedicated background
    84  // mark utilization of 25% of GOMAXPROCS by minimizing GC assists.
    85  // GOMAXPROCS. The high-level design of this algorithm is documented
    86  // at https://github.com/golang/proposal/blob/master/design/44167-gc-pacer-redesign.md.
    87  // See https://golang.org/s/go15gcpacing for additional historical context.
    88  var gcController gcControllerState
    89  
    90  type gcControllerState struct {
    91  	// Initialized from GOGC. GOGC=off means no GC.
    92  	gcPercent atomic.Int32
    93  
    94  	// memoryLimit is the soft memory limit in bytes.
    95  	//
    96  	// Initialized from GOMEMLIMIT. GOMEMLIMIT=off is equivalent to MaxInt64
    97  	// which means no soft memory limit in practice.
    98  	//
    99  	// This is an int64 instead of a uint64 to more easily maintain parity with
   100  	// the SetMemoryLimit API, which sets a maximum at MaxInt64. This value
   101  	// should never be negative.
   102  	memoryLimit atomic.Int64
   103  
   104  	// heapMinimum is the minimum heap size at which to trigger GC.
   105  	// For small heaps, this overrides the usual GOGC*live set rule.
   106  	//
   107  	// When there is a very small live set but a lot of allocation, simply
   108  	// collecting when the heap reaches GOGC*live results in many GC
   109  	// cycles and high total per-GC overhead. This minimum amortizes this
   110  	// per-GC overhead while keeping the heap reasonably small.
   111  	//
   112  	// During initialization this is set to 4MB*GOGC/100. In the case of
   113  	// GOGC==0, this will set heapMinimum to 0, resulting in constant
   114  	// collection even when the heap size is small, which is useful for
   115  	// debugging.
   116  	heapMinimum uint64
   117  
   118  	// runway is the amount of runway in heap bytes allocated by the
   119  	// application that we want to give the GC once it starts.
   120  	//
   121  	// This is computed from consMark during mark termination.
   122  	runway atomic.Uint64
   123  
   124  	// consMark is the estimated per-CPU consMark ratio for the application.
   125  	//
   126  	// It represents the ratio between the application's allocation
   127  	// rate, as bytes allocated per CPU-time, and the GC's scan rate,
   128  	// as bytes scanned per CPU-time.
   129  	// The units of this ratio are (B / cpu-ns) / (B / cpu-ns).
   130  	//
   131  	// At a high level, this value is computed as the bytes of memory
   132  	// allocated (cons) per unit of scan work completed (mark) in a GC
   133  	// cycle, divided by the CPU time spent on each activity.
   134  	//
   135  	// Updated at the end of each GC cycle, in endCycle.
   136  	consMark float64
   137  
   138  	// lastConsMark is the computed cons/mark value for the previous 4 GC
   139  	// cycles. Note that this is *not* the last value of consMark, but the
   140  	// measured cons/mark value in endCycle.
   141  	lastConsMark [4]float64
   142  
   143  	// gcPercentHeapGoal is the goal heapLive for when next GC ends derived
   144  	// from gcPercent.
   145  	//
   146  	// Set to ^uint64(0) if gcPercent is disabled.
   147  	gcPercentHeapGoal atomic.Uint64
   148  
   149  	// sweepDistMinTrigger is the minimum trigger to ensure a minimum
   150  	// sweep distance.
   151  	//
   152  	// This bound is also special because it applies to both the trigger
   153  	// *and* the goal (all other trigger bounds must be based *on* the goal).
   154  	//
   155  	// It is computed ahead of time, at commit time. The theory is that,
   156  	// absent a sudden change to a parameter like gcPercent, the trigger
   157  	// will be chosen to always give the sweeper enough headroom. However,
   158  	// such a change might dramatically and suddenly move up the trigger,
   159  	// in which case we need to ensure the sweeper still has enough headroom.
   160  	sweepDistMinTrigger atomic.Uint64
   161  
   162  	// triggered is the point at which the current GC cycle actually triggered.
   163  	// Only valid during the mark phase of a GC cycle, otherwise set to ^uint64(0).
   164  	//
   165  	// Updated while the world is stopped.
   166  	triggered uint64
   167  
   168  	// lastHeapGoal is the value of heapGoal at the moment the last GC
   169  	// ended. Note that this is distinct from the last value heapGoal had,
   170  	// because it could change if e.g. gcPercent changes.
   171  	//
   172  	// Read and written with the world stopped or with mheap_.lock held.
   173  	lastHeapGoal uint64
   174  
   175  	// heapLive is the number of bytes considered live by the GC.
   176  	// That is: retained by the most recent GC plus allocated
   177  	// since then. heapLive ≤ memstats.totalAlloc-memstats.totalFree, since
   178  	// heapAlloc includes unmarked objects that have not yet been swept (and
   179  	// hence goes up as we allocate and down as we sweep) while heapLive
   180  	// excludes these objects (and hence only goes up between GCs).
   181  	//
   182  	// To reduce contention, this is updated only when obtaining a span
   183  	// from an mcentral and at this point it counts all of the unallocated
   184  	// slots in that span (which will be allocated before that mcache
   185  	// obtains another span from that mcentral). Hence, it slightly
   186  	// overestimates the "true" live heap size. It's better to overestimate
   187  	// than to underestimate because 1) this triggers the GC earlier than
   188  	// necessary rather than potentially too late and 2) this leads to a
   189  	// conservative GC rate rather than a GC rate that is potentially too
   190  	// low.
   191  	//
   192  	// Whenever this is updated, call traceHeapAlloc() and
   193  	// this gcControllerState's revise() method.
   194  	heapLive atomic.Uint64
   195  
   196  	// heapScan is the number of bytes of "scannable" heap. This is the
   197  	// live heap (as counted by heapLive), but omitting no-scan objects and
   198  	// no-scan tails of objects.
   199  	//
   200  	// This value is fixed at the start of a GC cycle. It represents the
   201  	// maximum scannable heap.
   202  	heapScan atomic.Uint64
   203  
   204  	// lastHeapScan is the number of bytes of heap that were scanned
   205  	// last GC cycle. It is the same as heapMarked, but only
   206  	// includes the "scannable" parts of objects.
   207  	//
   208  	// Updated when the world is stopped.
   209  	lastHeapScan uint64
   210  
   211  	// lastStackScan is the number of bytes of stack that were scanned
   212  	// last GC cycle.
   213  	lastStackScan atomic.Uint64
   214  
   215  	// maxStackScan is the amount of allocated goroutine stack space in
   216  	// use by goroutines.
   217  	//
   218  	// This number tracks allocated goroutine stack space rather than used
   219  	// goroutine stack space (i.e. what is actually scanned) because used
   220  	// goroutine stack space is much harder to measure cheaply. By using
   221  	// allocated space, we make an overestimate; this is OK, it's better
   222  	// to conservatively overcount than undercount.
   223  	maxStackScan atomic.Uint64
   224  
   225  	// globalsScan is the total amount of global variable space
   226  	// that is scannable.
   227  	globalsScan atomic.Uint64
   228  
   229  	// heapMarked is the number of bytes marked by the previous
   230  	// GC. After mark termination, heapLive == heapMarked, but
   231  	// unlike heapLive, heapMarked does not change until the
   232  	// next mark termination.
   233  	heapMarked uint64
   234  
   235  	// heapScanWork is the total heap scan work performed this cycle.
   236  	// stackScanWork is the total stack scan work performed this cycle.
   237  	// globalsScanWork is the total globals scan work performed this cycle.
   238  	//
   239  	// These are updated atomically during the cycle. Updates occur in
   240  	// bounded batches, since they are both written and read
   241  	// throughout the cycle. At the end of the cycle, heapScanWork is how
   242  	// much of the retained heap is scannable.
   243  	//
   244  	// Currently these are measured in bytes. For most uses, this is an
   245  	// opaque unit of work, but for estimation the definition is important.
   246  	//
   247  	// Note that stackScanWork includes only stack space scanned, not all
   248  	// of the allocated stack.
   249  	heapScanWork    atomic.Int64
   250  	stackScanWork   atomic.Int64
   251  	globalsScanWork atomic.Int64
   252  
   253  	// bgScanCredit is the scan work credit accumulated by the concurrent
   254  	// background scan. This credit is accumulated by the background scan
   255  	// and stolen by mutator assists.  Updates occur in bounded batches,
   256  	// since it is both written and read throughout the cycle.
   257  	bgScanCredit atomic.Int64
   258  
   259  	// assistTime is the nanoseconds spent in mutator assists
   260  	// during this cycle. This is updated atomically, and must also
   261  	// be updated atomically even during a STW, because it is read
   262  	// by sysmon. Updates occur in bounded batches, since it is both
   263  	// written and read throughout the cycle.
   264  	assistTime atomic.Int64
   265  
   266  	// dedicatedMarkTime is the nanoseconds spent in dedicated mark workers
   267  	// during this cycle. This is updated at the end of the concurrent mark
   268  	// phase.
   269  	dedicatedMarkTime atomic.Int64
   270  
   271  	// fractionalMarkTime is the nanoseconds spent in the fractional mark
   272  	// worker during this cycle. This is updated throughout the cycle and
   273  	// will be up-to-date if the fractional mark worker is not currently
   274  	// running.
   275  	fractionalMarkTime atomic.Int64
   276  
   277  	// idleMarkTime is the nanoseconds spent in idle marking during this
   278  	// cycle. This is updated throughout the cycle.
   279  	idleMarkTime atomic.Int64
   280  
   281  	// markStartTime is the absolute start time in nanoseconds
   282  	// that assists and background mark workers started.
   283  	markStartTime int64
   284  
   285  	// dedicatedMarkWorkersNeeded is the number of dedicated mark workers
   286  	// that need to be started. This is computed at the beginning of each
   287  	// cycle and decremented as dedicated mark workers get started.
   288  	dedicatedMarkWorkersNeeded atomic.Int64
   289  
   290  	// idleMarkWorkers is two packed int32 values in a single uint64.
   291  	// These two values are always updated simultaneously.
   292  	//
   293  	// The bottom int32 is the current number of idle mark workers executing.
   294  	//
   295  	// The top int32 is the maximum number of idle mark workers allowed to
   296  	// execute concurrently. Normally, this number is just gomaxprocs. However,
   297  	// during periodic GC cycles it is set to 0 because the system is idle
   298  	// anyway; there's no need to go full blast on all of GOMAXPROCS.
   299  	//
   300  	// The maximum number of idle mark workers is used to prevent new workers
   301  	// from starting, but it is not a hard maximum. It is possible (but
   302  	// exceedingly rare) for the current number of idle mark workers to
   303  	// transiently exceed the maximum. This could happen if the maximum changes
   304  	// just after a GC ends, and an M with no P.
   305  	//
   306  	// Note that if we have no dedicated mark workers, we set this value to
   307  	// 1 in this case we only have fractional GC workers which aren't scheduled
   308  	// strictly enough to ensure GC progress. As a result, idle-priority mark
   309  	// workers are vital to GC progress in these situations.
   310  	//
   311  	// For example, consider a situation in which goroutines block on the GC
   312  	// (such as via runtime.GOMAXPROCS) and only fractional mark workers are
   313  	// scheduled (e.g. GOMAXPROCS=1). Without idle-priority mark workers, the
   314  	// last running M might skip scheduling a fractional mark worker if its
   315  	// utilization goal is met, such that once it goes to sleep (because there's
   316  	// nothing to do), there will be nothing else to spin up a new M for the
   317  	// fractional worker in the future, stalling GC progress and causing a
   318  	// deadlock. However, idle-priority workers will *always* run when there is
   319  	// nothing left to do, ensuring the GC makes progress.
   320  	//
   321  	// See github.com/golang/go/issues/44163 for more details.
   322  	idleMarkWorkers atomic.Uint64
   323  
   324  	// assistWorkPerByte is the ratio of scan work to allocated
   325  	// bytes that should be performed by mutator assists. This is
   326  	// computed at the beginning of each cycle and updated every
   327  	// time heapScan is updated.
   328  	assistWorkPerByte atomic.Float64
   329  
   330  	// assistBytesPerWork is 1/assistWorkPerByte.
   331  	//
   332  	// Note that because this is read and written independently
   333  	// from assistWorkPerByte users may notice a skew between
   334  	// the two values, and such a state should be safe.
   335  	assistBytesPerWork atomic.Float64
   336  
   337  	// fractionalUtilizationGoal is the fraction of wall clock
   338  	// time that should be spent in the fractional mark worker on
   339  	// each P that isn't running a dedicated worker.
   340  	//
   341  	// For example, if the utilization goal is 25% and there are
   342  	// no dedicated workers, this will be 0.25. If the goal is
   343  	// 25%, there is one dedicated worker, and GOMAXPROCS is 5,
   344  	// this will be 0.05 to make up the missing 5%.
   345  	//
   346  	// If this is zero, no fractional workers are needed.
   347  	fractionalUtilizationGoal float64
   348  
   349  	// These memory stats are effectively duplicates of fields from
   350  	// memstats.heapStats but are updated atomically or with the world
   351  	// stopped and don't provide the same consistency guarantees.
   352  	//
   353  	// Because the runtime is responsible for managing a memory limit, it's
   354  	// useful to couple these stats more tightly to the gcController, which
   355  	// is intimately connected to how that memory limit is maintained.
   356  	heapInUse    sysMemStat    // bytes in mSpanInUse spans
   357  	heapReleased sysMemStat    // bytes released to the OS
   358  	heapFree     sysMemStat    // bytes not in any span, but not released to the OS
   359  	totalAlloc   atomic.Uint64 // total bytes allocated
   360  	totalFree    atomic.Uint64 // total bytes freed
   361  	mappedReady  atomic.Uint64 // total virtual memory in the Ready state (see mem.go).
   362  
   363  	// test indicates that this is a test-only copy of gcControllerState.
   364  	test bool
   365  
   366  	_ cpu.CacheLinePad
   367  }
   368  
   369  func (c *gcControllerState) init(gcPercent int32, memoryLimit int64) {
   370  	c.heapMinimum = defaultHeapMinimum
   371  	c.triggered = ^uint64(0)
   372  	c.setGCPercent(gcPercent)
   373  	c.setMemoryLimit(memoryLimit)
   374  	c.commit(true) // No sweep phase in the first GC cycle.
   375  	// N.B. Don't bother calling traceHeapGoal. Tracing is never enabled at
   376  	// initialization time.
   377  	// N.B. No need to call revise; there's no GC enabled during
   378  	// initialization.
   379  }
   380  
   381  // startCycle resets the GC controller's state and computes estimates
   382  // for a new GC cycle. The caller must hold worldsema and the world
   383  // must be stopped.
   384  func (c *gcControllerState) startCycle(markStartTime int64, procs int, trigger gcTrigger) {
   385  	c.heapScanWork.Store(0)
   386  	c.stackScanWork.Store(0)
   387  	c.globalsScanWork.Store(0)
   388  	c.bgScanCredit.Store(0)
   389  	c.assistTime.Store(0)
   390  	c.dedicatedMarkTime.Store(0)
   391  	c.fractionalMarkTime.Store(0)
   392  	c.idleMarkTime.Store(0)
   393  	c.markStartTime = markStartTime
   394  	c.triggered = c.heapLive.Load()
   395  
   396  	// Compute the background mark utilization goal. In general,
   397  	// this may not come out exactly. We round the number of
   398  	// dedicated workers so that the utilization is closest to
   399  	// 25%. For small GOMAXPROCS, this would introduce too much
   400  	// error, so we add fractional workers in that case.
   401  	totalUtilizationGoal := float64(procs) * gcBackgroundUtilization
   402  	dedicatedMarkWorkersNeeded := int64(totalUtilizationGoal + 0.5)
   403  	utilError := float64(dedicatedMarkWorkersNeeded)/totalUtilizationGoal - 1
   404  	const maxUtilError = 0.3
   405  	if utilError < -maxUtilError || utilError > maxUtilError {
   406  		// Rounding put us more than 30% off our goal. With
   407  		// gcBackgroundUtilization of 25%, this happens for
   408  		// GOMAXPROCS<=3 or GOMAXPROCS=6. Enable fractional
   409  		// workers to compensate.
   410  		if float64(dedicatedMarkWorkersNeeded) > totalUtilizationGoal {
   411  			// Too many dedicated workers.
   412  			dedicatedMarkWorkersNeeded--
   413  		}
   414  		c.fractionalUtilizationGoal = (totalUtilizationGoal - float64(dedicatedMarkWorkersNeeded)) / float64(procs)
   415  	} else {
   416  		c.fractionalUtilizationGoal = 0
   417  	}
   418  
   419  	// In STW mode, we just want dedicated workers.
   420  	if debug.gcstoptheworld > 0 {
   421  		dedicatedMarkWorkersNeeded = int64(procs)
   422  		c.fractionalUtilizationGoal = 0
   423  	}
   424  
   425  	// Clear per-P state
   426  	for _, p := range allp {
   427  		p.gcAssistTime = 0
   428  		p.gcFractionalMarkTime = 0
   429  	}
   430  
   431  	if trigger.kind == gcTriggerTime {
   432  		// During a periodic GC cycle, reduce the number of idle mark workers
   433  		// required. However, we need at least one dedicated mark worker or
   434  		// idle GC worker to ensure GC progress in some scenarios (see comment
   435  		// on maxIdleMarkWorkers).
   436  		if dedicatedMarkWorkersNeeded > 0 {
   437  			c.setMaxIdleMarkWorkers(0)
   438  		} else {
   439  			// TODO(mknyszek): The fundamental reason why we need this is because
   440  			// we can't count on the fractional mark worker to get scheduled.
   441  			// Fix that by ensuring it gets scheduled according to its quota even
   442  			// if the rest of the application is idle.
   443  			c.setMaxIdleMarkWorkers(1)
   444  		}
   445  	} else {
   446  		// N.B. gomaxprocs and dedicatedMarkWorkersNeeded are guaranteed not to
   447  		// change during a GC cycle.
   448  		c.setMaxIdleMarkWorkers(int32(procs) - int32(dedicatedMarkWorkersNeeded))
   449  	}
   450  
   451  	// Compute initial values for controls that are updated
   452  	// throughout the cycle.
   453  	c.dedicatedMarkWorkersNeeded.Store(dedicatedMarkWorkersNeeded)
   454  	c.revise()
   455  
   456  	if debug.gcpacertrace > 0 {
   457  		heapGoal := c.heapGoal()
   458  		assistRatio := c.assistWorkPerByte.Load()
   459  		print("pacer: assist ratio=", assistRatio,
   460  			" (scan ", gcController.heapScan.Load()>>20, " MB in ",
   461  			work.initialHeapLive>>20, "->",
   462  			heapGoal>>20, " MB)",
   463  			" workers=", dedicatedMarkWorkersNeeded,
   464  			"+", c.fractionalUtilizationGoal, "\n")
   465  	}
   466  }
   467  
   468  // revise updates the assist ratio during the GC cycle to account for
   469  // improved estimates. This should be called whenever gcController.heapScan,
   470  // gcController.heapLive, or if any inputs to gcController.heapGoal are
   471  // updated. It is safe to call concurrently, but it may race with other
   472  // calls to revise.
   473  //
   474  // The result of this race is that the two assist ratio values may not line
   475  // up or may be stale. In practice this is OK because the assist ratio
   476  // moves slowly throughout a GC cycle, and the assist ratio is a best-effort
   477  // heuristic anyway. Furthermore, no part of the heuristic depends on
   478  // the two assist ratio values being exact reciprocals of one another, since
   479  // the two values are used to convert values from different sources.
   480  //
   481  // The worst case result of this raciness is that we may miss a larger shift
   482  // in the ratio (say, if we decide to pace more aggressively against the
   483  // hard heap goal) but even this "hard goal" is best-effort (see #40460).
   484  // The dedicated GC should ensure we don't exceed the hard goal by too much
   485  // in the rare case we do exceed it.
   486  //
   487  // It should only be called when gcBlackenEnabled != 0 (because this
   488  // is when assists are enabled and the necessary statistics are
   489  // available).
   490  func (c *gcControllerState) revise() {
   491  	gcPercent := c.gcPercent.Load()
   492  	if gcPercent < 0 {
   493  		// If GC is disabled but we're running a forced GC,
   494  		// act like GOGC is huge for the below calculations.
   495  		gcPercent = 100000
   496  	}
   497  	live := c.heapLive.Load()
   498  	scan := c.heapScan.Load()
   499  	work := c.heapScanWork.Load() + c.stackScanWork.Load() + c.globalsScanWork.Load()
   500  
   501  	// Assume we're under the soft goal. Pace GC to complete at
   502  	// heapGoal assuming the heap is in steady-state.
   503  	heapGoal := int64(c.heapGoal())
   504  
   505  	// The expected scan work is computed as the amount of bytes scanned last
   506  	// GC cycle (both heap and stack), plus our estimate of globals work for this cycle.
   507  	scanWorkExpected := int64(c.lastHeapScan + c.lastStackScan.Load() + c.globalsScan.Load())
   508  
   509  	// maxScanWork is a worst-case estimate of the amount of scan work that
   510  	// needs to be performed in this GC cycle. Specifically, it represents
   511  	// the case where *all* scannable memory turns out to be live, and
   512  	// *all* allocated stack space is scannable.
   513  	maxStackScan := c.maxStackScan.Load()
   514  	maxScanWork := int64(scan + maxStackScan + c.globalsScan.Load())
   515  	if work > scanWorkExpected {
   516  		// We've already done more scan work than expected. Because our expectation
   517  		// is based on a steady-state scannable heap size, we assume this means our
   518  		// heap is growing. Compute a new heap goal that takes our existing runway
   519  		// computed for scanWorkExpected and extrapolates it to maxScanWork, the worst-case
   520  		// scan work. This keeps our assist ratio stable if the heap continues to grow.
   521  		//
   522  		// The effect of this mechanism is that assists stay flat in the face of heap
   523  		// growths. It's OK to use more memory this cycle to scan all the live heap,
   524  		// because the next GC cycle is inevitably going to use *at least* that much
   525  		// memory anyway.
   526  		extHeapGoal := int64(float64(heapGoal-int64(c.triggered))/float64(scanWorkExpected)*float64(maxScanWork)) + int64(c.triggered)
   527  		scanWorkExpected = maxScanWork
   528  
   529  		// hardGoal is a hard limit on the amount that we're willing to push back the
   530  		// heap goal, and that's twice the heap goal (i.e. if GOGC=100 and the heap and/or
   531  		// stacks and/or globals grow to twice their size, this limits the current GC cycle's
   532  		// growth to 4x the original live heap's size).
   533  		//
   534  		// This maintains the invariant that we use no more memory than the next GC cycle
   535  		// will anyway.
   536  		hardGoal := int64((1.0 + float64(gcPercent)/100.0) * float64(heapGoal))
   537  		if extHeapGoal > hardGoal {
   538  			extHeapGoal = hardGoal
   539  		}
   540  		heapGoal = extHeapGoal
   541  	}
   542  	if int64(live) > heapGoal {
   543  		// We're already past our heap goal, even the extrapolated one.
   544  		// Leave ourselves some extra runway, so in the worst case we
   545  		// finish by that point.
   546  		const maxOvershoot = 1.1
   547  		heapGoal = int64(float64(heapGoal) * maxOvershoot)
   548  
   549  		// Compute the upper bound on the scan work remaining.
   550  		scanWorkExpected = maxScanWork
   551  	}
   552  
   553  	// Compute the remaining scan work estimate.
   554  	//
   555  	// Note that we currently count allocations during GC as both
   556  	// scannable heap (heapScan) and scan work completed
   557  	// (scanWork), so allocation will change this difference
   558  	// slowly in the soft regime and not at all in the hard
   559  	// regime.
   560  	scanWorkRemaining := scanWorkExpected - work
   561  	if scanWorkRemaining < 1000 {
   562  		// We set a somewhat arbitrary lower bound on
   563  		// remaining scan work since if we aim a little high,
   564  		// we can miss by a little.
   565  		//
   566  		// We *do* need to enforce that this is at least 1,
   567  		// since marking is racy and double-scanning objects
   568  		// may legitimately make the remaining scan work
   569  		// negative, even in the hard goal regime.
   570  		scanWorkRemaining = 1000
   571  	}
   572  
   573  	// Compute the heap distance remaining.
   574  	heapRemaining := heapGoal - int64(live)
   575  	if heapRemaining <= 0 {
   576  		// This shouldn't happen, but if it does, avoid
   577  		// dividing by zero or setting the assist negative.
   578  		heapRemaining = 1
   579  	}
   580  
   581  	// Compute the mutator assist ratio so by the time the mutator
   582  	// allocates the remaining heap bytes up to heapGoal, it will
   583  	// have done (or stolen) the remaining amount of scan work.
   584  	// Note that the assist ratio values are updated atomically
   585  	// but not together. This means there may be some degree of
   586  	// skew between the two values. This is generally OK as the
   587  	// values shift relatively slowly over the course of a GC
   588  	// cycle.
   589  	assistWorkPerByte := float64(scanWorkRemaining) / float64(heapRemaining)
   590  	assistBytesPerWork := float64(heapRemaining) / float64(scanWorkRemaining)
   591  	c.assistWorkPerByte.Store(assistWorkPerByte)
   592  	c.assistBytesPerWork.Store(assistBytesPerWork)
   593  }
   594  
   595  // endCycle computes the consMark estimate for the next cycle.
   596  // userForced indicates whether the current GC cycle was forced
   597  // by the application.
   598  func (c *gcControllerState) endCycle(now int64, procs int, userForced bool) {
   599  	// Record last heap goal for the scavenger.
   600  	// We'll be updating the heap goal soon.
   601  	gcController.lastHeapGoal = c.heapGoal()
   602  
   603  	// Compute the duration of time for which assists were turned on.
   604  	assistDuration := now - c.markStartTime
   605  
   606  	// Assume background mark hit its utilization goal.
   607  	utilization := gcBackgroundUtilization
   608  	// Add assist utilization; avoid divide by zero.
   609  	if assistDuration > 0 {
   610  		utilization += float64(c.assistTime.Load()) / float64(assistDuration*int64(procs))
   611  	}
   612  
   613  	if c.heapLive.Load() <= c.triggered {
   614  		// Shouldn't happen, but let's be very safe about this in case the
   615  		// GC is somehow extremely short.
   616  		//
   617  		// In this case though, the only reasonable value for c.heapLive-c.triggered
   618  		// would be 0, which isn't really all that useful, i.e. the GC was so short
   619  		// that it didn't matter.
   620  		//
   621  		// Ignore this case and don't update anything.
   622  		return
   623  	}
   624  	idleUtilization := 0.0
   625  	if assistDuration > 0 {
   626  		idleUtilization = float64(c.idleMarkTime.Load()) / float64(assistDuration*int64(procs))
   627  	}
   628  	// Determine the cons/mark ratio.
   629  	//
   630  	// The units we want for the numerator and denominator are both B / cpu-ns.
   631  	// We get this by taking the bytes allocated or scanned, and divide by the amount of
   632  	// CPU time it took for those operations. For allocations, that CPU time is
   633  	//
   634  	//    assistDuration * procs * (1 - utilization)
   635  	//
   636  	// Where utilization includes just background GC workers and assists. It does *not*
   637  	// include idle GC work time, because in theory the mutator is free to take that at
   638  	// any point.
   639  	//
   640  	// For scanning, that CPU time is
   641  	//
   642  	//    assistDuration * procs * (utilization + idleUtilization)
   643  	//
   644  	// In this case, we *include* idle utilization, because that is additional CPU time that
   645  	// the GC had available to it.
   646  	//
   647  	// In effect, idle GC time is sort of double-counted here, but it's very weird compared
   648  	// to other kinds of GC work, because of how fluid it is. Namely, because the mutator is
   649  	// *always* free to take it.
   650  	//
   651  	// So this calculation is really:
   652  	//     (heapLive-trigger) / (assistDuration * procs * (1-utilization)) /
   653  	//         (scanWork) / (assistDuration * procs * (utilization+idleUtilization))
   654  	//
   655  	// Note that because we only care about the ratio, assistDuration and procs cancel out.
   656  	scanWork := c.heapScanWork.Load() + c.stackScanWork.Load() + c.globalsScanWork.Load()
   657  	currentConsMark := (float64(c.heapLive.Load()-c.triggered) * (utilization + idleUtilization)) /
   658  		(float64(scanWork) * (1 - utilization))
   659  
   660  	// Update our cons/mark estimate. This is the maximum of the value we just computed and the last
   661  	// 4 cons/mark values we measured. The reason we take the maximum here is to bias a noisy
   662  	// cons/mark measurement toward fewer assists at the expense of additional GC cycles (starting
   663  	// earlier).
   664  	oldConsMark := c.consMark
   665  	c.consMark = currentConsMark
   666  	for i := range c.lastConsMark {
   667  		if c.lastConsMark[i] > c.consMark {
   668  			c.consMark = c.lastConsMark[i]
   669  		}
   670  	}
   671  	copy(c.lastConsMark[:], c.lastConsMark[1:])
   672  	c.lastConsMark[len(c.lastConsMark)-1] = currentConsMark
   673  
   674  	if debug.gcpacertrace > 0 {
   675  		printlock()
   676  		goal := gcGoalUtilization * 100
   677  		print("pacer: ", int(utilization*100), "% CPU (", int(goal), " exp.) for ")
   678  		print(c.heapScanWork.Load(), "+", c.stackScanWork.Load(), "+", c.globalsScanWork.Load(), " B work (", c.lastHeapScan+c.lastStackScan.Load()+c.globalsScan.Load(), " B exp.) ")
   679  		live := c.heapLive.Load()
   680  		print("in ", c.triggered, " B -> ", live, " B (∆goal ", int64(live)-int64(c.lastHeapGoal), ", cons/mark ", oldConsMark, ")")
   681  		println()
   682  		printunlock()
   683  	}
   684  }
   685  
   686  // enlistWorker encourages another dedicated mark worker to start on
   687  // another P if there are spare worker slots. It is used by putfull
   688  // when more work is made available.
   689  //
   690  //go:nowritebarrier
   691  func (c *gcControllerState) enlistWorker() {
   692  	// If there are idle Ps, wake one so it will run an idle worker.
   693  	// NOTE: This is suspected of causing deadlocks. See golang.org/issue/19112.
   694  	//
   695  	//	if sched.npidle.Load() != 0 && sched.nmspinning.Load() == 0 {
   696  	//		wakep()
   697  	//		return
   698  	//	}
   699  
   700  	// There are no idle Ps. If we need more dedicated workers,
   701  	// try to preempt a running P so it will switch to a worker.
   702  	if c.dedicatedMarkWorkersNeeded.Load() <= 0 {
   703  		return
   704  	}
   705  	// Pick a random other P to preempt.
   706  	if gomaxprocs <= 1 {
   707  		return
   708  	}
   709  	gp := getg()
   710  	if gp == nil || gp.m == nil || gp.m.p == 0 {
   711  		return
   712  	}
   713  	myID := gp.m.p.ptr().id
   714  	for tries := 0; tries < 5; tries++ {
   715  		id := int32(cheaprandn(uint32(gomaxprocs - 1)))
   716  		if id >= myID {
   717  			id++
   718  		}
   719  		p := allp[id]
   720  		if p.status != _Prunning {
   721  			continue
   722  		}
   723  		if preemptone(p) {
   724  			return
   725  		}
   726  	}
   727  }
   728  
   729  // findRunnableGCWorker returns a background mark worker for pp if it
   730  // should be run. This must only be called when gcBlackenEnabled != 0.
   731  func (c *gcControllerState) findRunnableGCWorker(pp *p, now int64) (*g, int64) {
   732  	if gcBlackenEnabled == 0 {
   733  		throw("gcControllerState.findRunnable: blackening not enabled")
   734  	}
   735  
   736  	// Since we have the current time, check if the GC CPU limiter
   737  	// hasn't had an update in a while. This check is necessary in
   738  	// case the limiter is on but hasn't been checked in a while and
   739  	// so may have left sufficient headroom to turn off again.
   740  	if now == 0 {
   741  		now = nanotime()
   742  	}
   743  	if gcCPULimiter.needUpdate(now) {
   744  		gcCPULimiter.update(now)
   745  	}
   746  
   747  	if !gcMarkWorkAvailable(pp) {
   748  		// No work to be done right now. This can happen at
   749  		// the end of the mark phase when there are still
   750  		// assists tapering off. Don't bother running a worker
   751  		// now because it'll just return immediately.
   752  		return nil, now
   753  	}
   754  
   755  	// Grab a worker before we commit to running below.
   756  	node := (*gcBgMarkWorkerNode)(gcBgMarkWorkerPool.pop())
   757  	if node == nil {
   758  		// There is at least one worker per P, so normally there are
   759  		// enough workers to run on all Ps, if necessary. However, once
   760  		// a worker enters gcMarkDone it may park without rejoining the
   761  		// pool, thus freeing a P with no corresponding worker.
   762  		// gcMarkDone never depends on another worker doing work, so it
   763  		// is safe to simply do nothing here.
   764  		//
   765  		// If gcMarkDone bails out without completing the mark phase,
   766  		// it will always do so with queued global work. Thus, that P
   767  		// will be immediately eligible to re-run the worker G it was
   768  		// just using, ensuring work can complete.
   769  		return nil, now
   770  	}
   771  
   772  	decIfPositive := func(val *atomic.Int64) bool {
   773  		for {
   774  			v := val.Load()
   775  			if v <= 0 {
   776  				return false
   777  			}
   778  
   779  			if val.CompareAndSwap(v, v-1) {
   780  				return true
   781  			}
   782  		}
   783  	}
   784  
   785  	if decIfPositive(&c.dedicatedMarkWorkersNeeded) {
   786  		// This P is now dedicated to marking until the end of
   787  		// the concurrent mark phase.
   788  		pp.gcMarkWorkerMode = gcMarkWorkerDedicatedMode
   789  	} else if c.fractionalUtilizationGoal == 0 {
   790  		// No need for fractional workers.
   791  		gcBgMarkWorkerPool.push(&node.node)
   792  		return nil, now
   793  	} else {
   794  		// Is this P behind on the fractional utilization
   795  		// goal?
   796  		//
   797  		// This should be kept in sync with pollFractionalWorkerExit.
   798  		delta := now - c.markStartTime
   799  		if delta > 0 && float64(pp.gcFractionalMarkTime)/float64(delta) > c.fractionalUtilizationGoal {
   800  			// Nope. No need to run a fractional worker.
   801  			gcBgMarkWorkerPool.push(&node.node)
   802  			return nil, now
   803  		}
   804  		// Run a fractional worker.
   805  		pp.gcMarkWorkerMode = gcMarkWorkerFractionalMode
   806  	}
   807  
   808  	// Run the background mark worker.
   809  	gp := node.gp.ptr()
   810  	trace := traceAcquire()
   811  	casgstatus(gp, _Gwaiting, _Grunnable)
   812  	if trace.ok() {
   813  		trace.GoUnpark(gp, 0)
   814  		traceRelease(trace)
   815  	}
   816  	return gp, now
   817  }
   818  
   819  // resetLive sets up the controller state for the next mark phase after the end
   820  // of the previous one. Must be called after endCycle and before commit, before
   821  // the world is started.
   822  //
   823  // The world must be stopped.
   824  func (c *gcControllerState) resetLive(bytesMarked uint64) {
   825  	c.heapMarked = bytesMarked
   826  	c.heapLive.Store(bytesMarked)
   827  	c.heapScan.Store(uint64(c.heapScanWork.Load()))
   828  	c.lastHeapScan = uint64(c.heapScanWork.Load())
   829  	c.lastStackScan.Store(uint64(c.stackScanWork.Load()))
   830  	c.triggered = ^uint64(0) // Reset triggered.
   831  
   832  	// heapLive was updated, so emit a trace event.
   833  	trace := traceAcquire()
   834  	if trace.ok() {
   835  		trace.HeapAlloc(bytesMarked)
   836  		traceRelease(trace)
   837  	}
   838  }
   839  
   840  // markWorkerStop must be called whenever a mark worker stops executing.
   841  //
   842  // It updates mark work accounting in the controller by a duration of
   843  // work in nanoseconds and other bookkeeping.
   844  //
   845  // Safe to execute at any time.
   846  func (c *gcControllerState) markWorkerStop(mode gcMarkWorkerMode, duration int64) {
   847  	switch mode {
   848  	case gcMarkWorkerDedicatedMode:
   849  		c.dedicatedMarkTime.Add(duration)
   850  		c.dedicatedMarkWorkersNeeded.Add(1)
   851  	case gcMarkWorkerFractionalMode:
   852  		c.fractionalMarkTime.Add(duration)
   853  	case gcMarkWorkerIdleMode:
   854  		c.idleMarkTime.Add(duration)
   855  		c.removeIdleMarkWorker()
   856  	default:
   857  		throw("markWorkerStop: unknown mark worker mode")
   858  	}
   859  }
   860  
   861  func (c *gcControllerState) update(dHeapLive, dHeapScan int64) {
   862  	if dHeapLive != 0 {
   863  		trace := traceAcquire()
   864  		live := gcController.heapLive.Add(dHeapLive)
   865  		if trace.ok() {
   866  			// gcController.heapLive changed.
   867  			trace.HeapAlloc(live)
   868  			traceRelease(trace)
   869  		}
   870  	}
   871  	if gcBlackenEnabled == 0 {
   872  		// Update heapScan when we're not in a current GC. It is fixed
   873  		// at the beginning of a cycle.
   874  		if dHeapScan != 0 {
   875  			gcController.heapScan.Add(dHeapScan)
   876  		}
   877  	} else {
   878  		// gcController.heapLive changed.
   879  		c.revise()
   880  	}
   881  }
   882  
   883  func (c *gcControllerState) addScannableStack(pp *p, amount int64) {
   884  	if pp == nil {
   885  		c.maxStackScan.Add(amount)
   886  		return
   887  	}
   888  	pp.maxStackScanDelta += amount
   889  	if pp.maxStackScanDelta >= maxStackScanSlack || pp.maxStackScanDelta <= -maxStackScanSlack {
   890  		c.maxStackScan.Add(pp.maxStackScanDelta)
   891  		pp.maxStackScanDelta = 0
   892  	}
   893  }
   894  
   895  func (c *gcControllerState) addGlobals(amount int64) {
   896  	c.globalsScan.Add(amount)
   897  }
   898  
   899  // heapGoal returns the current heap goal.
   900  func (c *gcControllerState) heapGoal() uint64 {
   901  	goal, _ := c.heapGoalInternal()
   902  	return goal
   903  }
   904  
   905  // heapGoalInternal is the implementation of heapGoal which returns additional
   906  // information that is necessary for computing the trigger.
   907  //
   908  // The returned minTrigger is always <= goal.
   909  func (c *gcControllerState) heapGoalInternal() (goal, minTrigger uint64) {
   910  	// Start with the goal calculated for gcPercent.
   911  	goal = c.gcPercentHeapGoal.Load()
   912  
   913  	// Check if the memory-limit-based goal is smaller, and if so, pick that.
   914  	if newGoal := c.memoryLimitHeapGoal(); newGoal < goal {
   915  		goal = newGoal
   916  	} else {
   917  		// We're not limited by the memory limit goal, so perform a series of
   918  		// adjustments that might move the goal forward in a variety of circumstances.
   919  
   920  		sweepDistTrigger := c.sweepDistMinTrigger.Load()
   921  		if sweepDistTrigger > goal {
   922  			// Set the goal to maintain a minimum sweep distance since
   923  			// the last call to commit. Note that we never want to do this
   924  			// if we're in the memory limit regime, because it could push
   925  			// the goal up.
   926  			goal = sweepDistTrigger
   927  		}
   928  		// Since we ignore the sweep distance trigger in the memory
   929  		// limit regime, we need to ensure we don't propagate it to
   930  		// the trigger, because it could cause a violation of the
   931  		// invariant that the trigger < goal.
   932  		minTrigger = sweepDistTrigger
   933  
   934  		// Ensure that the heap goal is at least a little larger than
   935  		// the point at which we triggered. This may not be the case if GC
   936  		// start is delayed or if the allocation that pushed gcController.heapLive
   937  		// over trigger is large or if the trigger is really close to
   938  		// GOGC. Assist is proportional to this distance, so enforce a
   939  		// minimum distance, even if it means going over the GOGC goal
   940  		// by a tiny bit.
   941  		//
   942  		// Ignore this if we're in the memory limit regime: we'd prefer to
   943  		// have the GC respond hard about how close we are to the goal than to
   944  		// push the goal back in such a manner that it could cause us to exceed
   945  		// the memory limit.
   946  		const minRunway = 64 << 10
   947  		if c.triggered != ^uint64(0) && goal < c.triggered+minRunway {
   948  			goal = c.triggered + minRunway
   949  		}
   950  	}
   951  	return
   952  }
   953  
   954  // memoryLimitHeapGoal returns a heap goal derived from memoryLimit.
   955  func (c *gcControllerState) memoryLimitHeapGoal() uint64 {
   956  	// Start by pulling out some values we'll need. Be careful about overflow.
   957  	var heapFree, heapAlloc, mappedReady uint64
   958  	for {
   959  		heapFree = c.heapFree.load()                         // Free and unscavenged memory.
   960  		heapAlloc = c.totalAlloc.Load() - c.totalFree.Load() // Heap object bytes in use.
   961  		mappedReady = c.mappedReady.Load()                   // Total unreleased mapped memory.
   962  		if heapFree+heapAlloc <= mappedReady {
   963  			break
   964  		}
   965  		// It is impossible for total unreleased mapped memory to exceed heap memory, but
   966  		// because these stats are updated independently, we may observe a partial update
   967  		// including only some values. Thus, we appear to break the invariant. However,
   968  		// this condition is necessarily transient, so just try again. In the case of a
   969  		// persistent accounting error, we'll deadlock here.
   970  	}
   971  
   972  	// Below we compute a goal from memoryLimit. There are a few things to be aware of.
   973  	// Firstly, the memoryLimit does not easily compare to the heap goal: the former
   974  	// is total mapped memory by the runtime that hasn't been released, while the latter is
   975  	// only heap object memory. Intuitively, the way we convert from one to the other is to
   976  	// subtract everything from memoryLimit that both contributes to the memory limit (so,
   977  	// ignore scavenged memory) and doesn't contain heap objects. This isn't quite what
   978  	// lines up with reality, but it's a good starting point.
   979  	//
   980  	// In practice this computation looks like the following:
   981  	//
   982  	//    goal := memoryLimit - ((mappedReady - heapFree - heapAlloc) + max(mappedReady - memoryLimit, 0))
   983  	//                    ^1                                    ^2
   984  	//    goal -= goal / 100 * memoryLimitHeapGoalHeadroomPercent
   985  	//    ^3
   986  	//
   987  	// Let's break this down.
   988  	//
   989  	// The first term (marker 1) is everything that contributes to the memory limit and isn't
   990  	// or couldn't become heap objects. It represents, broadly speaking, non-heap overheads.
   991  	// One oddity you may have noticed is that we also subtract out heapFree, i.e. unscavenged
   992  	// memory that may contain heap objects in the future.
   993  	//
   994  	// Let's take a step back. In an ideal world, this term would look something like just
   995  	// the heap goal. That is, we "reserve" enough space for the heap to grow to the heap
   996  	// goal, and subtract out everything else. This is of course impossible; the definition
   997  	// is circular! However, this impossible definition contains a key insight: the amount
   998  	// we're *going* to use matters just as much as whatever we're currently using.
   999  	//
  1000  	// Consider if the heap shrinks to 1/10th its size, leaving behind lots of free and
  1001  	// unscavenged memory. mappedReady - heapAlloc will be quite large, because of that free
  1002  	// and unscavenged memory, pushing the goal down significantly.
  1003  	//
  1004  	// heapFree is also safe to exclude from the memory limit because in the steady-state, it's
  1005  	// just a pool of memory for future heap allocations, and making new allocations from heapFree
  1006  	// memory doesn't increase overall memory use. In transient states, the scavenger and the
  1007  	// allocator actively manage the pool of heapFree memory to maintain the memory limit.
  1008  	//
  1009  	// The second term (marker 2) is the amount of memory we've exceeded the limit by, and is
  1010  	// intended to help recover from such a situation. By pushing the heap goal down, we also
  1011  	// push the trigger down, triggering and finishing a GC sooner in order to make room for
  1012  	// other memory sources. Note that since we're effectively reducing the heap goal by X bytes,
  1013  	// we're actually giving more than X bytes of headroom back, because the heap goal is in
  1014  	// terms of heap objects, but it takes more than X bytes (e.g. due to fragmentation) to store
  1015  	// X bytes worth of objects.
  1016  	//
  1017  	// The final adjustment (marker 3) reduces the maximum possible memory limit heap goal by
  1018  	// memoryLimitHeapGoalPercent. As the name implies, this is to provide additional headroom in
  1019  	// the face of pacing inaccuracies, and also to leave a buffer of unscavenged memory so the
  1020  	// allocator isn't constantly scavenging. The reduction amount also has a fixed minimum
  1021  	// (memoryLimitMinHeapGoalHeadroom, not pictured) because the aforementioned pacing inaccuracies
  1022  	// disproportionately affect small heaps: as heaps get smaller, the pacer's inputs get fuzzier.
  1023  	// Shorter GC cycles and less GC work means noisy external factors like the OS scheduler have a
  1024  	// greater impact.
  1025  
  1026  	memoryLimit := uint64(c.memoryLimit.Load())
  1027  
  1028  	// Compute term 1.
  1029  	nonHeapMemory := mappedReady - heapFree - heapAlloc
  1030  
  1031  	// Compute term 2.
  1032  	var overage uint64
  1033  	if mappedReady > memoryLimit {
  1034  		overage = mappedReady - memoryLimit
  1035  	}
  1036  
  1037  	if nonHeapMemory+overage >= memoryLimit {
  1038  		// We're at a point where non-heap memory exceeds the memory limit on its own.
  1039  		// There's honestly not much we can do here but just trigger GCs continuously
  1040  		// and let the CPU limiter reign that in. Something has to give at this point.
  1041  		// Set it to heapMarked, the lowest possible goal.
  1042  		return c.heapMarked
  1043  	}
  1044  
  1045  	// Compute the goal.
  1046  	goal := memoryLimit - (nonHeapMemory + overage)
  1047  
  1048  	// Apply some headroom to the goal to account for pacing inaccuracies and to reduce
  1049  	// the impact of scavenging at allocation time in response to a high allocation rate
  1050  	// when GOGC=off. See issue #57069. Also, be careful about small limits.
  1051  	headroom := goal / 100 * memoryLimitHeapGoalHeadroomPercent
  1052  	if headroom < memoryLimitMinHeapGoalHeadroom {
  1053  		// Set a fixed minimum to deal with the particularly large effect pacing inaccuracies
  1054  		// have for smaller heaps.
  1055  		headroom = memoryLimitMinHeapGoalHeadroom
  1056  	}
  1057  	if goal < headroom || goal-headroom < headroom {
  1058  		goal = headroom
  1059  	} else {
  1060  		goal = goal - headroom
  1061  	}
  1062  	// Don't let us go below the live heap. A heap goal below the live heap doesn't make sense.
  1063  	if goal < c.heapMarked {
  1064  		goal = c.heapMarked
  1065  	}
  1066  	return goal
  1067  }
  1068  
  1069  const (
  1070  	// These constants determine the bounds on the GC trigger as a fraction
  1071  	// of heap bytes allocated between the start of a GC (heapLive == heapMarked)
  1072  	// and the end of a GC (heapLive == heapGoal).
  1073  	//
  1074  	// The constants are obscured in this way for efficiency. The denominator
  1075  	// of the fraction is always a power-of-two for a quick division, so that
  1076  	// the numerator is a single constant integer multiplication.
  1077  	triggerRatioDen = 64
  1078  
  1079  	// The minimum trigger constant was chosen empirically: given a sufficiently
  1080  	// fast/scalable allocator with 48 Ps that could drive the trigger ratio
  1081  	// to <0.05, this constant causes applications to retain the same peak
  1082  	// RSS compared to not having this allocator.
  1083  	minTriggerRatioNum = 45 // ~0.7
  1084  
  1085  	// The maximum trigger constant is chosen somewhat arbitrarily, but the
  1086  	// current constant has served us well over the years.
  1087  	maxTriggerRatioNum = 61 // ~0.95
  1088  )
  1089  
  1090  // trigger returns the current point at which a GC should trigger along with
  1091  // the heap goal.
  1092  //
  1093  // The returned value may be compared against heapLive to determine whether
  1094  // the GC should trigger. Thus, the GC trigger condition should be (but may
  1095  // not be, in the case of small movements for efficiency) checked whenever
  1096  // the heap goal may change.
  1097  func (c *gcControllerState) trigger() (uint64, uint64) {
  1098  	goal, minTrigger := c.heapGoalInternal()
  1099  
  1100  	// Invariant: the trigger must always be less than the heap goal.
  1101  	//
  1102  	// Note that the memory limit sets a hard maximum on our heap goal,
  1103  	// but the live heap may grow beyond it.
  1104  
  1105  	if c.heapMarked >= goal {
  1106  		// The goal should never be smaller than heapMarked, but let's be
  1107  		// defensive about it. The only reasonable trigger here is one that
  1108  		// causes a continuous GC cycle at heapMarked, but respect the goal
  1109  		// if it came out as smaller than that.
  1110  		return goal, goal
  1111  	}
  1112  
  1113  	// Below this point, c.heapMarked < goal.
  1114  
  1115  	// heapMarked is our absolute minimum, and it's possible the trigger
  1116  	// bound we get from heapGoalinternal is less than that.
  1117  	if minTrigger < c.heapMarked {
  1118  		minTrigger = c.heapMarked
  1119  	}
  1120  
  1121  	// If we let the trigger go too low, then if the application
  1122  	// is allocating very rapidly we might end up in a situation
  1123  	// where we're allocating black during a nearly always-on GC.
  1124  	// The result of this is a growing heap and ultimately an
  1125  	// increase in RSS. By capping us at a point >0, we're essentially
  1126  	// saying that we're OK using more CPU during the GC to prevent
  1127  	// this growth in RSS.
  1128  	triggerLowerBound := ((goal-c.heapMarked)/triggerRatioDen)*minTriggerRatioNum + c.heapMarked
  1129  	if minTrigger < triggerLowerBound {
  1130  		minTrigger = triggerLowerBound
  1131  	}
  1132  
  1133  	// For small heaps, set the max trigger point at maxTriggerRatio of the way
  1134  	// from the live heap to the heap goal. This ensures we always have *some*
  1135  	// headroom when the GC actually starts. For larger heaps, set the max trigger
  1136  	// point at the goal, minus the minimum heap size.
  1137  	//
  1138  	// This choice follows from the fact that the minimum heap size is chosen
  1139  	// to reflect the costs of a GC with no work to do. With a large heap but
  1140  	// very little scan work to perform, this gives us exactly as much runway
  1141  	// as we would need, in the worst case.
  1142  	maxTrigger := ((goal-c.heapMarked)/triggerRatioDen)*maxTriggerRatioNum + c.heapMarked
  1143  	if goal > defaultHeapMinimum && goal-defaultHeapMinimum > maxTrigger {
  1144  		maxTrigger = goal - defaultHeapMinimum
  1145  	}
  1146  	maxTrigger = max(maxTrigger, minTrigger)
  1147  
  1148  	// Compute the trigger from our bounds and the runway stored by commit.
  1149  	var trigger uint64
  1150  	runway := c.runway.Load()
  1151  	if runway > goal {
  1152  		trigger = minTrigger
  1153  	} else {
  1154  		trigger = goal - runway
  1155  	}
  1156  	trigger = max(trigger, minTrigger)
  1157  	trigger = min(trigger, maxTrigger)
  1158  	if trigger > goal {
  1159  		print("trigger=", trigger, " heapGoal=", goal, "\n")
  1160  		print("minTrigger=", minTrigger, " maxTrigger=", maxTrigger, "\n")
  1161  		throw("produced a trigger greater than the heap goal")
  1162  	}
  1163  	return trigger, goal
  1164  }
  1165  
  1166  // commit recomputes all pacing parameters needed to derive the
  1167  // trigger and the heap goal. Namely, the gcPercent-based heap goal,
  1168  // and the amount of runway we want to give the GC this cycle.
  1169  //
  1170  // This can be called any time. If GC is the in the middle of a
  1171  // concurrent phase, it will adjust the pacing of that phase.
  1172  //
  1173  // isSweepDone should be the result of calling isSweepDone(),
  1174  // unless we're testing or we know we're executing during a GC cycle.
  1175  //
  1176  // This depends on gcPercent, gcController.heapMarked, and
  1177  // gcController.heapLive. These must be up to date.
  1178  //
  1179  // Callers must call gcControllerState.revise after calling this
  1180  // function if the GC is enabled.
  1181  //
  1182  // mheap_.lock must be held or the world must be stopped.
  1183  func (c *gcControllerState) commit(isSweepDone bool) {
  1184  	if !c.test {
  1185  		assertWorldStoppedOrLockHeld(&mheap_.lock)
  1186  	}
  1187  
  1188  	if isSweepDone {
  1189  		// The sweep is done, so there aren't any restrictions on the trigger
  1190  		// we need to think about.
  1191  		c.sweepDistMinTrigger.Store(0)
  1192  	} else {
  1193  		// Concurrent sweep happens in the heap growth
  1194  		// from gcController.heapLive to trigger. Make sure we
  1195  		// give the sweeper some runway if it doesn't have enough.
  1196  		c.sweepDistMinTrigger.Store(c.heapLive.Load() + sweepMinHeapDistance)
  1197  	}
  1198  
  1199  	// Compute the next GC goal, which is when the allocated heap
  1200  	// has grown by GOGC/100 over where it started the last cycle,
  1201  	// plus additional runway for non-heap sources of GC work.
  1202  	gcPercentHeapGoal := ^uint64(0)
  1203  	if gcPercent := c.gcPercent.Load(); gcPercent >= 0 {
  1204  		gcPercentHeapGoal = c.heapMarked + (c.heapMarked+c.lastStackScan.Load()+c.globalsScan.Load())*uint64(gcPercent)/100
  1205  	}
  1206  	// Apply the minimum heap size here. It's defined in terms of gcPercent
  1207  	// and is only updated by functions that call commit.
  1208  	if gcPercentHeapGoal < c.heapMinimum {
  1209  		gcPercentHeapGoal = c.heapMinimum
  1210  	}
  1211  	c.gcPercentHeapGoal.Store(gcPercentHeapGoal)
  1212  
  1213  	// Compute the amount of runway we want the GC to have by using our
  1214  	// estimate of the cons/mark ratio.
  1215  	//
  1216  	// The idea is to take our expected scan work, and multiply it by
  1217  	// the cons/mark ratio to determine how long it'll take to complete
  1218  	// that scan work in terms of bytes allocated. This gives us our GC's
  1219  	// runway.
  1220  	//
  1221  	// However, the cons/mark ratio is a ratio of rates per CPU-second, but
  1222  	// here we care about the relative rates for some division of CPU
  1223  	// resources among the mutator and the GC.
  1224  	//
  1225  	// To summarize, we have B / cpu-ns, and we want B / ns. We get that
  1226  	// by multiplying by our desired division of CPU resources. We choose
  1227  	// to express CPU resources as GOMAPROCS*fraction. Note that because
  1228  	// we're working with a ratio here, we can omit the number of CPU cores,
  1229  	// because they'll appear in the numerator and denominator and cancel out.
  1230  	// As a result, this is basically just "weighing" the cons/mark ratio by
  1231  	// our desired division of resources.
  1232  	//
  1233  	// Furthermore, by setting the runway so that CPU resources are divided
  1234  	// this way, assuming that the cons/mark ratio is correct, we make that
  1235  	// division a reality.
  1236  	c.runway.Store(uint64((c.consMark * (1 - gcGoalUtilization) / (gcGoalUtilization)) * float64(c.lastHeapScan+c.lastStackScan.Load()+c.globalsScan.Load())))
  1237  }
  1238  
  1239  // setGCPercent updates gcPercent. commit must be called after.
  1240  // Returns the old value of gcPercent.
  1241  //
  1242  // The world must be stopped, or mheap_.lock must be held.
  1243  func (c *gcControllerState) setGCPercent(in int32) int32 {
  1244  	if !c.test {
  1245  		assertWorldStoppedOrLockHeld(&mheap_.lock)
  1246  	}
  1247  
  1248  	out := c.gcPercent.Load()
  1249  	if in < 0 {
  1250  		in = -1
  1251  	}
  1252  	c.heapMinimum = defaultHeapMinimum * uint64(in) / 100
  1253  	c.gcPercent.Store(in)
  1254  
  1255  	return out
  1256  }
  1257  
  1258  //go:linkname setGCPercent runtime/debug.setGCPercent
  1259  func setGCPercent(in int32) (out int32) {
  1260  	// Run on the system stack since we grab the heap lock.
  1261  	systemstack(func() {
  1262  		lock(&mheap_.lock)
  1263  		out = gcController.setGCPercent(in)
  1264  		gcControllerCommit()
  1265  		unlock(&mheap_.lock)
  1266  	})
  1267  
  1268  	// If we just disabled GC, wait for any concurrent GC mark to
  1269  	// finish so we always return with no GC running.
  1270  	if in < 0 {
  1271  		gcWaitOnMark(work.cycles.Load())
  1272  	}
  1273  
  1274  	return out
  1275  }
  1276  
  1277  func readGOGC() int32 {
  1278  	p := gogetenv("GOGC")
  1279  	if p == "off" {
  1280  		return -1
  1281  	}
  1282  	if n, ok := atoi32(p); ok {
  1283  		return n
  1284  	}
  1285  	return 100
  1286  }
  1287  
  1288  // setMemoryLimit updates memoryLimit. commit must be called after
  1289  // Returns the old value of memoryLimit.
  1290  //
  1291  // The world must be stopped, or mheap_.lock must be held.
  1292  func (c *gcControllerState) setMemoryLimit(in int64) int64 {
  1293  	if !c.test {
  1294  		assertWorldStoppedOrLockHeld(&mheap_.lock)
  1295  	}
  1296  
  1297  	out := c.memoryLimit.Load()
  1298  	if in >= 0 {
  1299  		c.memoryLimit.Store(in)
  1300  	}
  1301  
  1302  	return out
  1303  }
  1304  
  1305  //go:linkname setMemoryLimit runtime/debug.setMemoryLimit
  1306  func setMemoryLimit(in int64) (out int64) {
  1307  	// Run on the system stack since we grab the heap lock.
  1308  	systemstack(func() {
  1309  		lock(&mheap_.lock)
  1310  		out = gcController.setMemoryLimit(in)
  1311  		if in < 0 || out == in {
  1312  			// If we're just checking the value or not changing
  1313  			// it, there's no point in doing the rest.
  1314  			unlock(&mheap_.lock)
  1315  			return
  1316  		}
  1317  		gcControllerCommit()
  1318  		unlock(&mheap_.lock)
  1319  	})
  1320  	return out
  1321  }
  1322  
  1323  func readGOMEMLIMIT() int64 {
  1324  	p := gogetenv("GOMEMLIMIT")
  1325  	if p == "" || p == "off" {
  1326  		return maxInt64
  1327  	}
  1328  	n, ok := parseByteCount(p)
  1329  	if !ok {
  1330  		print("GOMEMLIMIT=", p, "\n")
  1331  		throw("malformed GOMEMLIMIT; see `go doc runtime/debug.SetMemoryLimit`")
  1332  	}
  1333  	return n
  1334  }
  1335  
  1336  // addIdleMarkWorker attempts to add a new idle mark worker.
  1337  //
  1338  // If this returns true, the caller must become an idle mark worker unless
  1339  // there's no background mark worker goroutines in the pool. This case is
  1340  // harmless because there are already background mark workers running.
  1341  // If this returns false, the caller must NOT become an idle mark worker.
  1342  //
  1343  // nosplit because it may be called without a P.
  1344  //
  1345  //go:nosplit
  1346  func (c *gcControllerState) addIdleMarkWorker() bool {
  1347  	for {
  1348  		old := c.idleMarkWorkers.Load()
  1349  		n, max := int32(old&uint64(^uint32(0))), int32(old>>32)
  1350  		if n >= max {
  1351  			// See the comment on idleMarkWorkers for why
  1352  			// n > max is tolerated.
  1353  			return false
  1354  		}
  1355  		if n < 0 {
  1356  			print("n=", n, " max=", max, "\n")
  1357  			throw("negative idle mark workers")
  1358  		}
  1359  		new := uint64(uint32(n+1)) | (uint64(max) << 32)
  1360  		if c.idleMarkWorkers.CompareAndSwap(old, new) {
  1361  			return true
  1362  		}
  1363  	}
  1364  }
  1365  
  1366  // needIdleMarkWorker is a hint as to whether another idle mark worker is needed.
  1367  //
  1368  // The caller must still call addIdleMarkWorker to become one. This is mainly
  1369  // useful for a quick check before an expensive operation.
  1370  //
  1371  // nosplit because it may be called without a P.
  1372  //
  1373  //go:nosplit
  1374  func (c *gcControllerState) needIdleMarkWorker() bool {
  1375  	p := c.idleMarkWorkers.Load()
  1376  	n, max := int32(p&uint64(^uint32(0))), int32(p>>32)
  1377  	return n < max
  1378  }
  1379  
  1380  // removeIdleMarkWorker must be called when a new idle mark worker stops executing.
  1381  func (c *gcControllerState) removeIdleMarkWorker() {
  1382  	for {
  1383  		old := c.idleMarkWorkers.Load()
  1384  		n, max := int32(old&uint64(^uint32(0))), int32(old>>32)
  1385  		if n-1 < 0 {
  1386  			print("n=", n, " max=", max, "\n")
  1387  			throw("negative idle mark workers")
  1388  		}
  1389  		new := uint64(uint32(n-1)) | (uint64(max) << 32)
  1390  		if c.idleMarkWorkers.CompareAndSwap(old, new) {
  1391  			return
  1392  		}
  1393  	}
  1394  }
  1395  
  1396  // setMaxIdleMarkWorkers sets the maximum number of idle mark workers allowed.
  1397  //
  1398  // This method is optimistic in that it does not wait for the number of
  1399  // idle mark workers to reduce to max before returning; it assumes the workers
  1400  // will deschedule themselves.
  1401  func (c *gcControllerState) setMaxIdleMarkWorkers(max int32) {
  1402  	for {
  1403  		old := c.idleMarkWorkers.Load()
  1404  		n := int32(old & uint64(^uint32(0)))
  1405  		if n < 0 {
  1406  			print("n=", n, " max=", max, "\n")
  1407  			throw("negative idle mark workers")
  1408  		}
  1409  		new := uint64(uint32(n)) | (uint64(max) << 32)
  1410  		if c.idleMarkWorkers.CompareAndSwap(old, new) {
  1411  			return
  1412  		}
  1413  	}
  1414  }
  1415  
  1416  // gcControllerCommit is gcController.commit, but passes arguments from live
  1417  // (non-test) data. It also updates any consumers of the GC pacing, such as
  1418  // sweep pacing and the background scavenger.
  1419  //
  1420  // Calls gcController.commit.
  1421  //
  1422  // The heap lock must be held, so this must be executed on the system stack.
  1423  //
  1424  //go:systemstack
  1425  func gcControllerCommit() {
  1426  	assertWorldStoppedOrLockHeld(&mheap_.lock)
  1427  
  1428  	gcController.commit(isSweepDone())
  1429  
  1430  	// Update mark pacing.
  1431  	if gcphase != _GCoff {
  1432  		gcController.revise()
  1433  	}
  1434  
  1435  	// TODO(mknyszek): This isn't really accurate any longer because the heap
  1436  	// goal is computed dynamically. Still useful to snapshot, but not as useful.
  1437  	trace := traceAcquire()
  1438  	if trace.ok() {
  1439  		trace.HeapGoal()
  1440  		traceRelease(trace)
  1441  	}
  1442  
  1443  	trigger, heapGoal := gcController.trigger()
  1444  	gcPaceSweeper(trigger)
  1445  	gcPaceScavenger(gcController.memoryLimit.Load(), heapGoal, gcController.lastHeapGoal)
  1446  }
  1447  

View as plain text