Source file src/runtime/mwbbuf.go

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // This implements the write barrier buffer. The write barrier itself
     6  // is gcWriteBarrier and is implemented in assembly.
     7  //
     8  // See mbarrier.go for algorithmic details on the write barrier. This
     9  // file deals only with the buffer.
    10  //
    11  // The write barrier has a fast path and a slow path. The fast path
    12  // simply enqueues to a per-P write barrier buffer. It's written in
    13  // assembly and doesn't clobber any general purpose registers, so it
    14  // doesn't have the usual overheads of a Go call.
    15  //
    16  // When the buffer fills up, the write barrier invokes the slow path
    17  // (wbBufFlush) to flush the buffer to the GC work queues. In this
    18  // path, since the compiler didn't spill registers, we spill *all*
    19  // registers and disallow any GC safe points that could observe the
    20  // stack frame (since we don't know the types of the spilled
    21  // registers).
    22  
    23  package runtime
    24  
    25  import (
    26  	"internal/goarch"
    27  	"internal/runtime/atomic"
    28  	"unsafe"
    29  )
    30  
    31  // testSmallBuf forces a small write barrier buffer to stress write
    32  // barrier flushing.
    33  const testSmallBuf = false
    34  
    35  // wbBuf is a per-P buffer of pointers queued by the write barrier.
    36  // This buffer is flushed to the GC workbufs when it fills up and on
    37  // various GC transitions.
    38  //
    39  // This is closely related to a "sequential store buffer" (SSB),
    40  // except that SSBs are usually used for maintaining remembered sets,
    41  // while this is used for marking.
    42  type wbBuf struct {
    43  	// next points to the next slot in buf. It must not be a
    44  	// pointer type because it can point past the end of buf and
    45  	// must be updated without write barriers.
    46  	//
    47  	// This is a pointer rather than an index to optimize the
    48  	// write barrier assembly.
    49  	next uintptr
    50  
    51  	// end points to just past the end of buf. It must not be a
    52  	// pointer type because it points past the end of buf and must
    53  	// be updated without write barriers.
    54  	end uintptr
    55  
    56  	// buf stores a series of pointers to execute write barriers on.
    57  	buf [wbBufEntries]uintptr
    58  }
    59  
    60  const (
    61  	// wbBufEntries is the maximum number of pointers that can be
    62  	// stored in the write barrier buffer.
    63  	//
    64  	// This trades latency for throughput amortization. Higher
    65  	// values amortize flushing overhead more, but increase the
    66  	// latency of flushing. Higher values also increase the cache
    67  	// footprint of the buffer.
    68  	//
    69  	// TODO: What is the latency cost of this? Tune this value.
    70  	wbBufEntries = 512
    71  
    72  	// Maximum number of entries that we need to ask from the
    73  	// buffer in a single call.
    74  	wbMaxEntriesPerCall = 8
    75  )
    76  
    77  // reset empties b by resetting its next and end pointers.
    78  func (b *wbBuf) reset() {
    79  	start := uintptr(unsafe.Pointer(&b.buf[0]))
    80  	b.next = start
    81  	if testSmallBuf {
    82  		// For testing, make the buffer smaller but more than
    83  		// 1 write barrier's worth, so it tests both the
    84  		// immediate flush and delayed flush cases.
    85  		b.end = uintptr(unsafe.Pointer(&b.buf[wbMaxEntriesPerCall+1]))
    86  	} else {
    87  		b.end = start + uintptr(len(b.buf))*unsafe.Sizeof(b.buf[0])
    88  	}
    89  
    90  	if (b.end-b.next)%unsafe.Sizeof(b.buf[0]) != 0 {
    91  		throw("bad write barrier buffer bounds")
    92  	}
    93  }
    94  
    95  // discard resets b's next pointer, but not its end pointer.
    96  //
    97  // This must be nosplit because it's called by wbBufFlush.
    98  //
    99  //go:nosplit
   100  func (b *wbBuf) discard() {
   101  	b.next = uintptr(unsafe.Pointer(&b.buf[0]))
   102  }
   103  
   104  // empty reports whether b contains no pointers.
   105  func (b *wbBuf) empty() bool {
   106  	return b.next == uintptr(unsafe.Pointer(&b.buf[0]))
   107  }
   108  
   109  // getX returns space in the write barrier buffer to store X pointers.
   110  // getX will flush the buffer if necessary. Callers should use this as:
   111  //
   112  //	buf := &getg().m.p.ptr().wbBuf
   113  //	p := buf.get2()
   114  //	p[0], p[1] = old, new
   115  //	... actual memory write ...
   116  //
   117  // The caller must ensure there are no preemption points during the
   118  // above sequence. There must be no preemption points while buf is in
   119  // use because it is a per-P resource. There must be no preemption
   120  // points between the buffer put and the write to memory because this
   121  // could allow a GC phase change, which could result in missed write
   122  // barriers.
   123  //
   124  // getX must be nowritebarrierrec to because write barriers here would
   125  // corrupt the write barrier buffer. It (and everything it calls, if
   126  // it called anything) has to be nosplit to avoid scheduling on to a
   127  // different P and a different buffer.
   128  //
   129  //go:nowritebarrierrec
   130  //go:nosplit
   131  func (b *wbBuf) get1() *[1]uintptr {
   132  	if b.next+goarch.PtrSize > b.end {
   133  		wbBufFlush()
   134  	}
   135  	p := (*[1]uintptr)(unsafe.Pointer(b.next))
   136  	b.next += goarch.PtrSize
   137  	return p
   138  }
   139  
   140  //go:nowritebarrierrec
   141  //go:nosplit
   142  func (b *wbBuf) get2() *[2]uintptr {
   143  	if b.next+2*goarch.PtrSize > b.end {
   144  		wbBufFlush()
   145  	}
   146  	p := (*[2]uintptr)(unsafe.Pointer(b.next))
   147  	b.next += 2 * goarch.PtrSize
   148  	return p
   149  }
   150  
   151  // wbBufFlush flushes the current P's write barrier buffer to the GC
   152  // workbufs.
   153  //
   154  // This must not have write barriers because it is part of the write
   155  // barrier implementation.
   156  //
   157  // This and everything it calls must be nosplit because 1) the stack
   158  // contains untyped slots from gcWriteBarrier and 2) there must not be
   159  // a GC safe point between the write barrier test in the caller and
   160  // flushing the buffer.
   161  //
   162  // TODO: A "go:nosplitrec" annotation would be perfect for this.
   163  //
   164  //go:nowritebarrierrec
   165  //go:nosplit
   166  func wbBufFlush() {
   167  	// Note: Every possible return from this function must reset
   168  	// the buffer's next pointer to prevent buffer overflow.
   169  
   170  	if getg().m.dying > 0 {
   171  		// We're going down. Not much point in write barriers
   172  		// and this way we can allow write barriers in the
   173  		// panic path.
   174  		getg().m.p.ptr().wbBuf.discard()
   175  		return
   176  	}
   177  
   178  	// Switch to the system stack so we don't have to worry about
   179  	// safe points.
   180  	systemstack(func() {
   181  		wbBufFlush1(getg().m.p.ptr())
   182  	})
   183  }
   184  
   185  // wbBufFlush1 flushes p's write barrier buffer to the GC work queue.
   186  //
   187  // This must not have write barriers because it is part of the write
   188  // barrier implementation, so this may lead to infinite loops or
   189  // buffer corruption.
   190  //
   191  // This must be non-preemptible because it uses the P's workbuf.
   192  //
   193  //go:nowritebarrierrec
   194  //go:systemstack
   195  func wbBufFlush1(pp *p) {
   196  	// Get the buffered pointers.
   197  	start := uintptr(unsafe.Pointer(&pp.wbBuf.buf[0]))
   198  	n := (pp.wbBuf.next - start) / unsafe.Sizeof(pp.wbBuf.buf[0])
   199  	ptrs := pp.wbBuf.buf[:n]
   200  
   201  	// Poison the buffer to make extra sure nothing is enqueued
   202  	// while we're processing the buffer.
   203  	pp.wbBuf.next = 0
   204  
   205  	if useCheckmark {
   206  		// Slow path for checkmark mode.
   207  		for _, ptr := range ptrs {
   208  			shade(ptr)
   209  		}
   210  		pp.wbBuf.reset()
   211  		return
   212  	}
   213  
   214  	// Mark all of the pointers in the buffer and record only the
   215  	// pointers we greyed. We use the buffer itself to temporarily
   216  	// record greyed pointers.
   217  	//
   218  	// TODO: Should scanobject/scanblock just stuff pointers into
   219  	// the wbBuf? Then this would become the sole greying path.
   220  	//
   221  	// TODO: We could avoid shading any of the "new" pointers in
   222  	// the buffer if the stack has been shaded, or even avoid
   223  	// putting them in the buffer at all (which would double its
   224  	// capacity). This is slightly complicated with the buffer; we
   225  	// could track whether any un-shaded goroutine has used the
   226  	// buffer, or just track globally whether there are any
   227  	// un-shaded stacks and flush after each stack scan.
   228  	gcw := &pp.gcw
   229  	pos := 0
   230  	for _, ptr := range ptrs {
   231  		if ptr < minLegalPointer {
   232  			// nil pointers are very common, especially
   233  			// for the "old" values. Filter out these and
   234  			// other "obvious" non-heap pointers ASAP.
   235  			//
   236  			// TODO: Should we filter out nils in the fast
   237  			// path to reduce the rate of flushes?
   238  			continue
   239  		}
   240  		obj, span, objIndex := findObject(ptr, 0, 0)
   241  		if obj == 0 {
   242  			continue
   243  		}
   244  		// TODO: Consider making two passes where the first
   245  		// just prefetches the mark bits.
   246  		mbits := span.markBitsForIndex(objIndex)
   247  		if mbits.isMarked() {
   248  			continue
   249  		}
   250  		mbits.setMarked()
   251  
   252  		// Mark span.
   253  		arena, pageIdx, pageMask := pageIndexOf(span.base())
   254  		if arena.pageMarks[pageIdx]&pageMask == 0 {
   255  			atomic.Or8(&arena.pageMarks[pageIdx], pageMask)
   256  		}
   257  
   258  		if span.spanclass.noscan() {
   259  			gcw.bytesMarked += uint64(span.elemsize)
   260  			continue
   261  		}
   262  		ptrs[pos] = obj
   263  		pos++
   264  	}
   265  
   266  	// Enqueue the greyed objects.
   267  	gcw.putBatch(ptrs[:pos])
   268  
   269  	pp.wbBuf.reset()
   270  }
   271  

View as plain text