pinner.go

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"internal/runtime/atomic"
    10  	"unsafe"
    11  )
    12  
    13  // A Pinner is a set of Go objects each pinned to a fixed location in memory. The
    14  // [Pinner.Pin] method pins one object, while [Pinner.Unpin] unpins all pinned
    15  // objects.
    16  //
    17  // The purpose of a Pinner is two-fold.
    18  // First, it allows C code to safely use Go pointers that have not been passed
    19  // explicitly to the C code via a cgo call.
    20  // For example, for safely interacting with a pointer stored inside of a struct
    21  // whose pointer is passed to a C function.
    22  // Second, it allows C memory to safely retain that Go pointer even after the
    23  // cgo call returns, provided the object remains pinned.
    24  //
    25  // A Pinner arranges for its objects to be automatically unpinned some time after
    26  // it becomes unreachable, so its referents will not leak. However, this means the
    27  // Pinner itself must be kept alive across a cgo call, or as long as C retains a
    28  // reference to the pinned Go pointers.
    29  //
    30  // Reusing a Pinner is safe, and in fact encouraged, to avoid the cost of
    31  // initializing new Pinners on first use.
    32  //
    33  // The zero value of Pinner is ready to use.
    34  type Pinner struct {
    35  	*pinner
    36  }
    37  
    38  // Pin pins a Go object, preventing it from being moved or freed by the garbage
    39  // collector until the [Pinner.Unpin] method has been called.
    40  //
    41  // A pointer to a pinned object can be directly stored in C memory or can be
    42  // contained in Go memory passed to C functions. If the pinned object itself
    43  // contains pointers to Go objects, these objects must be pinned separately if they
    44  // are going to be accessed from C code.
    45  //
    46  // The argument must be a pointer of any type or an [unsafe.Pointer].
    47  //
    48  // It's safe to call Pin on non-Go pointers, in which case Pin will do nothing.
    49  func (p *Pinner) Pin(pointer any) {
    50  	if p.pinner == nil {
    51  		// Check the pinner cache first.
    52  		mp := acquirem()
    53  		if pp := mp.p.ptr(); pp != nil {
    54  			p.pinner = pp.pinnerCache
    55  			pp.pinnerCache = nil
    56  		}
    57  		releasem(mp)
    58  
    59  		if p.pinner == nil {
    60  			// Didn't get anything from the pinner cache.
    61  			p.pinner = new(pinner)
    62  			p.refs = p.refStore[:0]
    63  
    64  			// We set this finalizer once and never clear it. Thus, if the
    65  			// pinner gets cached, we'll reuse it, along with its finalizer.
    66  			// This lets us avoid the relatively expensive SetFinalizer call
    67  			// when reusing from the cache. The finalizer however has to be
    68  			// resilient to an empty pinner being finalized, which is done
    69  			// by checking p.refs' length.
    70  			SetFinalizer(p.pinner, func(i *pinner) {
    71  				if len(i.refs) != 0 {
    72  					i.unpin() // only required to make the test idempotent
    73  					pinnerLeakPanic()
    74  				}
    75  			})
    76  		}
    77  	}
    78  	ptr := pinnerGetPtr(&pointer)
    79  	if setPinned(ptr, true) {
    80  		p.refs = append(p.refs, ptr)
    81  	}
    82  }
    83  
    84  // Unpin unpins all pinned objects of the [Pinner].
    85  // It's safe and encouraged to reuse a Pinner after calling Unpin.
    86  func (p *Pinner) Unpin() {
    87  	p.pinner.unpin()
    88  
    89  	mp := acquirem()
    90  	if pp := mp.p.ptr(); pp != nil && pp.pinnerCache == nil {
    91  		// Put the pinner back in the cache, but only if the
    92  		// cache is empty. If application code is reusing Pinners
    93  		// on its own, we want to leave the backing store in place
    94  		// so reuse is more efficient.
    95  		pp.pinnerCache = p.pinner
    96  		p.pinner = nil
    97  	}
    98  	releasem(mp)
    99  }
   100  
   101  const (
   102  	pinnerSize         = 64
   103  	pinnerRefStoreSize = (pinnerSize - unsafe.Sizeof([]unsafe.Pointer{})) / unsafe.Sizeof(unsafe.Pointer(nil))
   104  )
   105  
   106  type pinner struct {
   107  	refs     []unsafe.Pointer
   108  	refStore [pinnerRefStoreSize]unsafe.Pointer
   109  }
   110  
   111  func (p *pinner) unpin() {
   112  	if p == nil || p.refs == nil {
   113  		return
   114  	}
   115  	for i := range p.refs {
   116  		setPinned(p.refs[i], false)
   117  	}
   118  	// The following two lines make all pointers to references
   119  	// in p.refs unreachable, either by deleting them or dropping
   120  	// p.refs' backing store (if it was not backed by refStore).
   121  	p.refStore = [pinnerRefStoreSize]unsafe.Pointer{}
   122  	p.refs = p.refStore[:0]
   123  }
   124  
   125  func pinnerGetPtr(i *any) unsafe.Pointer {
   126  	e := efaceOf(i)
   127  	etyp := e._type
   128  	if etyp == nil {
   129  		panic(errorString("runtime.Pinner: argument is nil"))
   130  	}
   131  	if kind := etyp.Kind(); kind != abi.Pointer && kind != abi.UnsafePointer {
   132  		panic(errorString("runtime.Pinner: argument is not a pointer: " + toRType(etyp).string()))
   133  	}
   134  	if inUserArenaChunk(uintptr(e.data)) {
   135  		// Arena-allocated objects are not eligible for pinning.
   136  		panic(errorString("runtime.Pinner: object was allocated into an arena"))
   137  	}
   138  	return e.data
   139  }
   140  
   141  // isPinned checks if a Go pointer is pinned.
   142  // nosplit, because it's called from nosplit code in cgocheck.
   143  //
   144  //go:nosplit
   145  func isPinned(ptr unsafe.Pointer) bool {
   146  	span := spanOfHeap(uintptr(ptr))
   147  	if span == nil {
   148  		// this code is only called for Go pointer, so this must be a
   149  		// linker-allocated global object.
   150  		return true
   151  	}
   152  	pinnerBits := span.getPinnerBits()
   153  	// these pinnerBits might get unlinked by a concurrently running sweep, but
   154  	// that's OK because gcBits don't get cleared until the following GC cycle
   155  	// (nextMarkBitArenaEpoch)
   156  	if pinnerBits == nil {
   157  		return false
   158  	}
   159  	objIndex := span.objIndex(uintptr(ptr))
   160  	pinState := pinnerBits.ofObject(objIndex)
   161  	KeepAlive(ptr) // make sure ptr is alive until we are done so the span can't be freed
   162  	return pinState.isPinned()
   163  }
   164  
   165  // setPinned marks or unmarks a Go pointer as pinned, when the ptr is a Go pointer.
   166  // It will be ignored while trying to pin a non-Go pointer.
   167  // It will panic while trying to unpin a non-Go pointer,
   168  // which should not happen in normal usage.
   169  func setPinned(ptr unsafe.Pointer, pin bool) bool {
   170  	span := spanOfHeap(uintptr(ptr))
   171  	if span == nil {
   172  		if !pin {
   173  			panic(errorString("tried to unpin non-Go pointer"))
   174  		}
   175  		// This is a linker-allocated, zero size object or other object,
   176  		// nothing to do, silently ignore it.
   177  		return false
   178  	}
   179  
   180  	// ensure that the span is swept, b/c sweeping accesses the specials list
   181  	// w/o locks.
   182  	mp := acquirem()
   183  	span.ensureSwept()
   184  	KeepAlive(ptr) // make sure ptr is still alive after span is swept
   185  
   186  	objIndex := span.objIndex(uintptr(ptr))
   187  
   188  	lock(&span.speciallock) // guard against concurrent calls of setPinned on same span
   189  
   190  	pinnerBits := span.getPinnerBits()
   191  	if pinnerBits == nil {
   192  		pinnerBits = span.newPinnerBits()
   193  		span.setPinnerBits(pinnerBits)
   194  	}
   195  	pinState := pinnerBits.ofObject(objIndex)
   196  	if pin {
   197  		if pinState.isPinned() {
   198  			// multiple pins on same object, set multipin bit
   199  			pinState.setMultiPinned(true)
   200  			// and increase the pin counter
   201  			// TODO(mknyszek): investigate if systemstack is necessary here
   202  			systemstack(func() {
   203  				offset := objIndex * span.elemsize
   204  				span.incPinCounter(offset)
   205  			})
   206  		} else {
   207  			// set pin bit
   208  			pinState.setPinned(true)
   209  		}
   210  	} else {
   211  		// unpin
   212  		if pinState.isPinned() {
   213  			if pinState.isMultiPinned() {
   214  				var exists bool
   215  				// TODO(mknyszek): investigate if systemstack is necessary here
   216  				systemstack(func() {
   217  					offset := objIndex * span.elemsize
   218  					exists = span.decPinCounter(offset)
   219  				})
   220  				if !exists {
   221  					// counter is 0, clear multipin bit
   222  					pinState.setMultiPinned(false)
   223  				}
   224  			} else {
   225  				// no multipins recorded. unpin object.
   226  				pinState.setPinned(false)
   227  			}
   228  		} else {
   229  			// unpinning unpinned object, bail out
   230  			throw("runtime.Pinner: object already unpinned")
   231  		}
   232  	}
   233  	unlock(&span.speciallock)
   234  	releasem(mp)
   235  	return true
   236  }
   237  
   238  type pinState struct {
   239  	bytep   *uint8
   240  	byteVal uint8
   241  	mask    uint8
   242  }
   243  
   244  // nosplit, because it's called by isPinned, which is nosplit
   245  //
   246  //go:nosplit
   247  func (v *pinState) isPinned() bool {
   248  	return (v.byteVal & v.mask) != 0
   249  }
   250  
   251  func (v *pinState) isMultiPinned() bool {
   252  	return (v.byteVal & (v.mask << 1)) != 0
   253  }
   254  
   255  func (v *pinState) setPinned(val bool) {
   256  	v.set(val, false)
   257  }
   258  
   259  func (v *pinState) setMultiPinned(val bool) {
   260  	v.set(val, true)
   261  }
   262  
   263  // set sets the pin bit of the pinState to val. If multipin is true, it
   264  // sets/unsets the multipin bit instead.
   265  func (v *pinState) set(val bool, multipin bool) {
   266  	mask := v.mask
   267  	if multipin {
   268  		mask <<= 1
   269  	}
   270  	if val {
   271  		atomic.Or8(v.bytep, mask)
   272  	} else {
   273  		atomic.And8(v.bytep, ^mask)
   274  	}
   275  }
   276  
   277  // pinnerBits is the same type as gcBits but has different methods.
   278  type pinnerBits gcBits
   279  
   280  // ofObject returns the pinState of the n'th object.
   281  // nosplit, because it's called by isPinned, which is nosplit
   282  //
   283  //go:nosplit
   284  func (p *pinnerBits) ofObject(n uintptr) pinState {
   285  	bytep, mask := (*gcBits)(p).bitp(n * 2)
   286  	byteVal := atomic.Load8(bytep)
   287  	return pinState{bytep, byteVal, mask}
   288  }
   289  
   290  func (s *mspan) pinnerBitSize() uintptr {
   291  	return divRoundUp(uintptr(s.nelems)*2, 8)
   292  }
   293  
   294  // newPinnerBits returns a pointer to 8 byte aligned bytes to be used for this
   295  // span's pinner bits. newPinnerBits is used to mark objects that are pinned.
   296  // They are copied when the span is swept.
   297  func (s *mspan) newPinnerBits() *pinnerBits {
   298  	return (*pinnerBits)(newMarkBits(uintptr(s.nelems) * 2))
   299  }
   300  
   301  // nosplit, because it's called by isPinned, which is nosplit
   302  //
   303  //go:nosplit
   304  func (s *mspan) getPinnerBits() *pinnerBits {
   305  	return (*pinnerBits)(atomic.Loadp(unsafe.Pointer(&s.pinnerBits)))
   306  }
   307  
   308  func (s *mspan) setPinnerBits(p *pinnerBits) {
   309  	atomicstorep(unsafe.Pointer(&s.pinnerBits), unsafe.Pointer(p))
   310  }
   311  
   312  // refreshPinnerBits replaces pinnerBits with a fresh copy in the arenas for the
   313  // next GC cycle. If it does not contain any pinned objects, pinnerBits of the
   314  // span is set to nil.
   315  func (s *mspan) refreshPinnerBits() {
   316  	p := s.getPinnerBits()
   317  	if p == nil {
   318  		return
   319  	}
   320  
   321  	hasPins := false
   322  	bytes := alignUp(s.pinnerBitSize(), 8)
   323  
   324  	// Iterate over each 8-byte chunk and check for pins. Note that
   325  	// newPinnerBits guarantees that pinnerBits will be 8-byte aligned, so we
   326  	// don't have to worry about edge cases, irrelevant bits will simply be
   327  	// zero.
   328  	for _, x := range unsafe.Slice((*uint64)(unsafe.Pointer(&p.x)), bytes/8) {
   329  		if x != 0 {
   330  			hasPins = true
   331  			break
   332  		}
   333  	}
   334  
   335  	if hasPins {
   336  		newPinnerBits := s.newPinnerBits()
   337  		memmove(unsafe.Pointer(&newPinnerBits.x), unsafe.Pointer(&p.x), bytes)
   338  		s.setPinnerBits(newPinnerBits)
   339  	} else {
   340  		s.setPinnerBits(nil)
   341  	}
   342  }
   343  
   344  // incPinCounter is only called for multiple pins of the same object and records
   345  // the _additional_ pins.
   346  func (span *mspan) incPinCounter(offset uintptr) {
   347  	var rec *specialPinCounter
   348  	ref, exists := span.specialFindSplicePoint(offset, _KindSpecialPinCounter)
   349  	if !exists {
   350  		lock(&mheap_.speciallock)
   351  		rec = (*specialPinCounter)(mheap_.specialPinCounterAlloc.alloc())
   352  		unlock(&mheap_.speciallock)
   353  		// splice in record, fill in offset.
   354  		rec.special.offset = offset
   355  		rec.special.kind = _KindSpecialPinCounter
   356  		rec.special.next = *ref
   357  		*ref = (*special)(unsafe.Pointer(rec))
   358  		spanHasSpecials(span)
   359  	} else {
   360  		rec = (*specialPinCounter)(unsafe.Pointer(*ref))
   361  	}
   362  	rec.counter++
   363  }
   364  
   365  // decPinCounter decreases the counter. If the counter reaches 0, the counter
   366  // special is deleted and false is returned. Otherwise true is returned.
   367  func (span *mspan) decPinCounter(offset uintptr) bool {
   368  	ref, exists := span.specialFindSplicePoint(offset, _KindSpecialPinCounter)
   369  	if !exists {
   370  		throw("runtime.Pinner: decreased non-existing pin counter")
   371  	}
   372  	counter := (*specialPinCounter)(unsafe.Pointer(*ref))
   373  	counter.counter--
   374  	if counter.counter == 0 {
   375  		*ref = counter.special.next
   376  		if span.specials == nil {
   377  			spanHasNoSpecials(span)
   378  		}
   379  		lock(&mheap_.speciallock)
   380  		mheap_.specialPinCounterAlloc.free(unsafe.Pointer(counter))
   381  		unlock(&mheap_.speciallock)
   382  		return false
   383  	}
   384  	return true
   385  }
   386  
   387  // only for tests
   388  func pinnerGetPinCounter(addr unsafe.Pointer) *uintptr {
   389  	_, span, objIndex := findObject(uintptr(addr), 0, 0)
   390  	offset := objIndex * span.elemsize
   391  	t, exists := span.specialFindSplicePoint(offset, _KindSpecialPinCounter)
   392  	if !exists {
   393  		return nil
   394  	}
   395  	counter := (*specialPinCounter)(unsafe.Pointer(*t))
   396  	return &counter.counter
   397  }
   398  
   399  // to be able to test that the GC panics when a pinned pointer is leaking, this
   400  // panic function is a variable, that can be overwritten by a test.
   401  var pinnerLeakPanic = func() {
   402  	panic(errorString("runtime.Pinner: found leaking pinned pointer; forgot to call Unpin()?"))
   403  }
   404
View as plain text