Source file src/runtime/cgo/cgo_bindings.go

     1  // Copyright 2026 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build goexperiment.runtimecgopurego && !msan && !tsan
     6  
     7  package cgo
     8  
     9  import (
    10  	"internal/cpu"
    11  	"internal/goos"
    12  	"internal/runtime/atomic"
    13  	"unsafe"
    14  )
    15  
    16  const offsetX86HasAVX = unsafe.Offsetof(cpu.X86.HasAVX)
    17  
    18  //go:cgo_export_static _cgo_wait_runtime_init_done
    19  //go:cgo_export_dynamic _cgo_wait_runtime_init_done
    20  //go:cgo_export_static _cgo_release_context
    21  //go:cgo_export_dynamic _cgo_release_context
    22  
    23  var pthread_key_created uintptr
    24  
    25  //go:linkname _cgo_pthread_key_created _cgo_pthread_key_created
    26  var _cgo_pthread_key_created = &pthread_key_created
    27  
    28  // x_crosscall2_ptr and _crosscall2_ptr are vestigial on Go-backed runtime/cgo
    29  // targets (nothing reads x_crosscall2_ptr), but runtime.main still calls
    30  // runtime.set_crosscall2, which writes through _crosscall2_ptr and is kept for
    31  // external linkname compatibility (for example, github.com/ebitengine/purego).
    32  var x_crosscall2_ptr uintptr
    33  
    34  //go:linkname _crosscall2_ptr _crosscall2_ptr
    35  var _crosscall2_ptr = &x_crosscall2_ptr
    36  
    37  //go:linknamestd runtime_asmcgocall runtime.asmcgocall
    38  //go:noescape
    39  func runtime_asmcgocall(fn, arg unsafe.Pointer) int32
    40  
    41  //go:linknamestd runtime_asmcgocall_no_g runtime.asmcgocall_no_g
    42  //go:noescape
    43  func runtime_asmcgocall_no_g(fn, arg unsafe.Pointer)
    44  
    45  //go:linkname runtime_cgocall runtime.cgocall
    46  //go:noescape
    47  func runtime_cgocall(fn, arg unsafe.Pointer) int32
    48  
    49  // cgoCallFrame holds the target function, up to 6 integer/pointer arguments,
    50  // and the return value for cgoCallTrampoline.
    51  //
    52  // The helpers below fill in a cgoCallFrame field by field rather than with
    53  // a composite literal. When compiled with -N, a composite literal is built
    54  // in a separate stack temporary and then copied, which would make the
    55  // frames of the nosplit helpers large enough to push nosplit chains such
    56  // as x_cgo_init's over the nosplit stack limit.
    57  type cgoCallFrame struct {
    58  	fn                     uintptr
    59  	a0, a1, a2, a3, a4, a5 uintptr
    60  	r1                     uintptr
    61  }
    62  
    63  // asmcgocall calls a C function pointer fn with up to 6 integer/pointer
    64  // arguments on the system stack via runtime.asmcgocall without entering
    65  // the scheduler.
    66  //
    67  //go:nosplit
    68  //go:norace
    69  //go:nocheckptr
    70  func asmcgocall(fn, a0, a1, a2, a3, a4, a5 uintptr) uintptr {
    71  	var f cgoCallFrame
    72  	f.fn, f.a0, f.a1, f.a2, f.a3, f.a4, f.a5 = fn, a0, a1, a2, a3, a4, a5
    73  	runtime_asmcgocall(cgoCallTrampolinePC(), unsafe.Pointer(&f))
    74  	return f.r1
    75  }
    76  
    77  // asmcgocallNoG calls a C function pointer fn with up to 6 integer/pointer
    78  // arguments on the current stack via runtime.asmcgocall_no_g without touching
    79  // g or TLS. It is used before runtime initialization (for example, libpreinit,
    80  // libInit, or x_cgo_init) or on a C/signal stack where switching to g0 is
    81  // unsafe.
    82  //
    83  //go:nosplit
    84  //go:norace
    85  //go:nocheckptr
    86  func asmcgocallNoG(fn, a0, a1, a2, a3, a4, a5 uintptr) uintptr {
    87  	var f cgoCallFrame
    88  	f.fn, f.a0, f.a1, f.a2, f.a3, f.a4, f.a5 = fn, a0, a1, a2, a3, a4, a5
    89  	runtime_asmcgocall_no_g(cgoCallTrampolinePC(), unsafe.Pointer(&f))
    90  	return f.r1
    91  }
    92  
    93  //go:nosplit
    94  //go:norace
    95  func asmcgocallMode(noG bool, fn, a0, a1, a2, a3, a4, a5 uintptr) uintptr {
    96  	if noG {
    97  		return asmcgocallNoG(fn, a0, a1, a2, a3, a4, a5)
    98  	}
    99  	return asmcgocall(fn, a0, a1, a2, a3, a4, a5)
   100  }
   101  
   102  // cgocall calls a C function pointer fn with up to 6 integer/pointer
   103  // arguments via runtime.cgocall, entering and exiting the scheduler.
   104  //
   105  //go:nosplit
   106  //go:norace
   107  //go:nocheckptr
   108  func cgocall(fn, a0, a1, a2, a3, a4, a5 uintptr) uintptr {
   109  	var f cgoCallFrame
   110  	f.fn, f.a0, f.a1, f.a2, f.a3, f.a4, f.a5 = fn, a0, a1, a2, a3, a4, a5
   111  	runtime_cgocall(cgoCallTrampolinePC(), unsafe.Pointer(&f))
   112  	return f.r1
   113  }
   114  
   115  // cgocallErr is like cgocall and also returns the int32 result from
   116  // runtime.cgocall (on Windows, GetLastError captured before exitsyscall).
   117  //
   118  //go:norace
   119  //go:nocheckptr
   120  func cgocallErr(fn, a0, a1, a2, a3, a4, a5 uintptr) (uintptr, uint32) {
   121  	var f cgoCallFrame
   122  	f.fn, f.a0, f.a1, f.a2, f.a3, f.a4, f.a5 = fn, a0, a1, a2, a3, a4, a5
   123  	errno := runtime_cgocall(cgoCallTrampolinePC(), unsafe.Pointer(&f))
   124  	return f.r1, uint32(errno)
   125  }
   126  
   127  //go:nosplit
   128  //go:norace
   129  func tsanAcquire() {}
   130  
   131  //go:nosplit
   132  //go:norace
   133  func tsanAcquireNoG() {}
   134  
   135  //go:nosplit
   136  //go:norace
   137  func tsanAcquireMode(noG bool) {}
   138  
   139  //go:nosplit
   140  //go:norace
   141  func tsanRelease() {}
   142  
   143  //go:nosplit
   144  //go:norace
   145  func tsanReleaseNoG() {}
   146  
   147  //go:nosplit
   148  //go:norace
   149  func tsanReleaseMode(noG bool) {}
   150  
   151  //go:nosplit
   152  //go:norace
   153  func msanUnpoison(addr unsafe.Pointer, size uintptr) {}
   154  
   155  //go:nosplit
   156  //go:norace
   157  func msanUnpoisonNoG(addr unsafe.Pointer, size uintptr) {}
   158  
   159  //go:nosplit
   160  //go:norace
   161  func msanUnpoisonMode(noG bool, addr unsafe.Pointer, size uintptr) {}
   162  
   163  type cgoContextArg struct {
   164  	context uintptr
   165  }
   166  
   167  var (
   168  	runtime_init_done    uint32
   169  	cgo_context_function uintptr
   170  )
   171  
   172  //go:nosplit
   173  //go:norace
   174  //go:nocheckptr
   175  func cgoWaitRuntimeInit() uintptr {
   176  	if atomic.Load(&runtime_init_done) != 2 {
   177  		for atomic.Load(&runtime_init_done) == 0 {
   178  			cgoYieldWaitRuntimeInit()
   179  		}
   180  		atomic.Store(&runtime_init_done, 2)
   181  	}
   182  	// TODO(iant): For the case of a new C thread calling into Go, such
   183  	// as when using -buildmode=c-archive, we know that Go runtime
   184  	// initialization is complete but we do not know that all Go init
   185  	// functions have been run. We should not fetch cgo_context_function
   186  	// until they have been, because that is where a call to
   187  	// SetCgoTraceback is likely to occur. We are going to wait for Go
   188  	// initialization to be complete anyhow, later, by waiting for
   189  	// main_init_done to be closed in cgocallbackg1. We should wait here
   190  	// instead. See also issue #15943.
   191  	return atomic.Loaduintptr(&cgo_context_function)
   192  }
   193  
   194  //go:nosplit
   195  //go:norace
   196  //go:nocheckptr
   197  func cgoWaitRuntimeInitDone() uintptr {
   198  	if pfn := cgoWaitRuntimeInit(); pfn != 0 {
   199  		var arg cgoContextArg
   200  		msanUnpoisonNoG(unsafe.Pointer(&arg), unsafe.Sizeof(arg))
   201  		fn := pfn
   202  		if goos.IsAix != 0 {
   203  			// On AIX, cgoCallTrampoline expects the address of a
   204  			// function descriptor pointer (see asm_unix_ppc64x.s).
   205  			fn = uintptr(unsafe.Pointer(&pfn))
   206  		}
   207  		asmcgocallNoG(fn, uintptr(unsafe.Pointer(&arg)), 0, 0, 0, 0, 0)
   208  		return arg.context
   209  	}
   210  	return 0
   211  }
   212  
   213  //go:linknamestd _cgoNotifyRuntimeInitDone runtime.cgoNotifyRuntimeInitDone
   214  var _cgoNotifyRuntimeInitDone = cgoNotifyRuntimeInitDone
   215  
   216  //go:norace
   217  //go:nocheckptr
   218  func cgoNotifyRuntimeInitDone() {
   219  	cgoNotifyRuntimeInitDoneOS()
   220  	atomic.Store(&runtime_init_done, 1)
   221  }
   222  
   223  //go:nosplit
   224  //go:norace
   225  //go:nocheckptr
   226  func cgoReleaseContext(ctxt uintptr) {
   227  	if ctxt == 0 {
   228  		return
   229  	}
   230  	if pfn := atomic.Loaduintptr(&cgo_context_function); pfn != 0 {
   231  		arg := cgoContextArg{context: ctxt}
   232  		msanUnpoisonNoG(unsafe.Pointer(&arg), unsafe.Sizeof(arg))
   233  		fn := pfn
   234  		if goos.IsAix != 0 {
   235  			// See cgoWaitRuntimeInitDone.
   236  			fn = uintptr(unsafe.Pointer(&pfn))
   237  		}
   238  		asmcgocallNoG(fn, uintptr(unsafe.Pointer(&arg)), 0, 0, 0, 0, 0)
   239  	}
   240  }
   241  
   242  //go:linknamestd _cgoSetContextFunction runtime.cgoSetContextFunction
   243  var _cgoSetContextFunction = cgoSetContextFunction
   244  
   245  //go:norace
   246  func cgoSetContextFunction(context uintptr) {
   247  	atomic.Storeuintptr(&cgo_context_function, context)
   248  }
   249  

View as plain text