Source file src/runtime/cgo/cgo_unix.go

     1  // Copyright 2026 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build goexperiment.runtimecgopurego && !msan && !tsan && unix
     6  
     7  package cgo
     8  
     9  import (
    10  	"internal/goos"
    11  	"internal/runtime/atomic"
    12  	"internal/runtime/sys"
    13  	"unsafe"
    14  )
    15  
    16  //go:linkname _cgo_libc_setenv _cgo_libc_setenv
    17  var _cgo_libc_setenv byte
    18  var libc_setenv = unsafe.Pointer(&_cgo_libc_setenv)
    19  
    20  //go:linkname _cgo_libc_unsetenv _cgo_libc_unsetenv
    21  var _cgo_libc_unsetenv byte
    22  var libc_unsetenv = unsafe.Pointer(&_cgo_libc_unsetenv)
    23  
    24  //go:linkname _cgo_libc_pthread_create _cgo_libc_pthread_create
    25  var _cgo_libc_pthread_create byte
    26  var libc_pthread_create = unsafe.Pointer(&_cgo_libc_pthread_create)
    27  
    28  //go:linkname _cgo_libc_pthread_attr_init _cgo_libc_pthread_attr_init
    29  var _cgo_libc_pthread_attr_init byte
    30  var libc_pthread_attr_init = unsafe.Pointer(&_cgo_libc_pthread_attr_init)
    31  
    32  //go:linkname _cgo_libc_pthread_attr_setdetachstate _cgo_libc_pthread_attr_setdetachstate
    33  var _cgo_libc_pthread_attr_setdetachstate byte
    34  var libc_pthread_attr_setdetachstate = unsafe.Pointer(&_cgo_libc_pthread_attr_setdetachstate)
    35  
    36  //go:linkname _cgo_libc_pthread_attr_getstacksize _cgo_libc_pthread_attr_getstacksize
    37  var _cgo_libc_pthread_attr_getstacksize byte
    38  var libc_pthread_attr_getstacksize = unsafe.Pointer(&_cgo_libc_pthread_attr_getstacksize)
    39  
    40  //go:linkname _cgo_libc_pthread_attr_setstacksize _cgo_libc_pthread_attr_setstacksize
    41  var _cgo_libc_pthread_attr_setstacksize byte
    42  var libc_pthread_attr_setstacksize = unsafe.Pointer(&_cgo_libc_pthread_attr_setstacksize)
    43  
    44  //go:linkname _cgo_libc_pthread_attr_getstack _cgo_libc_pthread_attr_getstack
    45  var _cgo_libc_pthread_attr_getstack byte
    46  var libc_pthread_attr_getstack = unsafe.Pointer(&_cgo_libc_pthread_attr_getstack)
    47  
    48  //go:linkname _cgo_libc_pthread_attr_destroy _cgo_libc_pthread_attr_destroy
    49  var _cgo_libc_pthread_attr_destroy byte
    50  var libc_pthread_attr_destroy = unsafe.Pointer(&_cgo_libc_pthread_attr_destroy)
    51  
    52  //go:linkname _cgo_libc_pthread_key_create _cgo_libc_pthread_key_create
    53  var _cgo_libc_pthread_key_create byte
    54  var libc_pthread_key_create = unsafe.Pointer(&_cgo_libc_pthread_key_create)
    55  
    56  //go:linkname _cgo_libc_pthread_setspecific _cgo_libc_pthread_setspecific
    57  var _cgo_libc_pthread_setspecific byte
    58  var libc_pthread_setspecific = unsafe.Pointer(&_cgo_libc_pthread_setspecific)
    59  
    60  //go:linkname _cgo_libc_pthread_self _cgo_libc_pthread_self
    61  var _cgo_libc_pthread_self byte
    62  var libc_pthread_self = unsafe.Pointer(&_cgo_libc_pthread_self)
    63  
    64  //go:linkname _cgo_libc_pthread_get_stackaddr_np _cgo_libc_pthread_get_stackaddr_np
    65  var _cgo_libc_pthread_get_stackaddr_np byte
    66  var libc_pthread_get_stackaddr_np = unsafe.Pointer(&_cgo_libc_pthread_get_stackaddr_np)
    67  
    68  //go:linkname _cgo_libc_pthread_get_stacksize_np _cgo_libc_pthread_get_stacksize_np
    69  var _cgo_libc_pthread_get_stacksize_np byte
    70  var libc_pthread_get_stacksize_np = unsafe.Pointer(&_cgo_libc_pthread_get_stacksize_np)
    71  
    72  //go:linkname _cgo_libc_pthread_getattr_np _cgo_libc_pthread_getattr_np
    73  var _cgo_libc_pthread_getattr_np byte
    74  var libc_pthread_getattr_np = unsafe.Pointer(&_cgo_libc_pthread_getattr_np)
    75  
    76  //go:linkname _cgo_libc_pthread_attr_get_np _cgo_libc_pthread_attr_get_np
    77  var _cgo_libc_pthread_attr_get_np byte
    78  var libc_pthread_attr_get_np = unsafe.Pointer(&_cgo_libc_pthread_attr_get_np)
    79  
    80  //go:linkname _cgo_libc_pthread_sigmask _cgo_libc_pthread_sigmask
    81  var _cgo_libc_pthread_sigmask byte
    82  var libc_pthread_sigmask = unsafe.Pointer(&_cgo_libc_pthread_sigmask)
    83  
    84  //go:linkname _cgo_libc_sigfillset _cgo_libc_sigfillset
    85  var _cgo_libc_sigfillset byte
    86  var libc_sigfillset = unsafe.Pointer(&_cgo_libc_sigfillset)
    87  
    88  //go:linkname _cgo_libc_sigaltstack _cgo_libc_sigaltstack
    89  var _cgo_libc_sigaltstack byte
    90  var libc_sigaltstack = unsafe.Pointer(&_cgo_libc_sigaltstack)
    91  
    92  // libc_dlopen, libc_dlsym, and libc_dlerror are imported for
    93  // Open only when the cgobindings experiment is enabled (see library_*.go),
    94  // except on Android, where runtime/cgo initialization uses them.
    95  
    96  //go:linkname _cgo_libc_dlopen _cgo_libc_dlopen
    97  var _cgo_libc_dlopen byte
    98  var libc_dlopen = unsafe.Pointer(&_cgo_libc_dlopen)
    99  
   100  //go:linkname _cgo_libc_dlsym _cgo_libc_dlsym
   101  var _cgo_libc_dlsym byte
   102  var libc_dlsym = unsafe.Pointer(&_cgo_libc_dlsym)
   103  
   104  //go:linkname _cgo_libc_dlerror _cgo_libc_dlerror
   105  var _cgo_libc_dlerror byte
   106  var libc_dlerror = unsafe.Pointer(&_cgo_libc_dlerror)
   107  
   108  //go:linknamestd runtime_setg runtime.setg
   109  func runtime_setg(gg uintptr)
   110  
   111  //go:linknamestd runtime_mstart runtime.mstart
   112  func runtime_mstart()
   113  
   114  //go:linknamestd runtime_cgoUsleep runtime.cgoUsleep
   115  func runtime_cgoUsleep(usec uint32)
   116  
   117  //go:linknamestd runtime_writeErrStr runtime.writeErrStr
   118  //go:noescape
   119  func runtime_writeErrStr(s string)
   120  
   121  //go:linkname runtime_abort runtime.abort
   122  func runtime_abort()
   123  
   124  //go:linknamestd _cgoSetenv runtime.cgoSetenv
   125  var _cgoSetenv = cgoSetenv
   126  
   127  //go:norace
   128  func cgoSetenv(k, v unsafe.Pointer) {
   129  	tsanAcquire()
   130  	asmcgocall(uintptr(libc_setenv), uintptr(k), uintptr(v), 1, 0, 0, 0)
   131  	tsanRelease()
   132  }
   133  
   134  //go:linknamestd _cgoUnsetenv runtime.cgoUnsetenv
   135  var _cgoUnsetenv = cgoUnsetenv
   136  
   137  //go:norace
   138  func cgoUnsetenv(k unsafe.Pointer) {
   139  	tsanAcquire()
   140  	asmcgocall(uintptr(libc_unsetenv), uintptr(k), 0, 0, 0, 0, 0)
   141  	tsanRelease()
   142  }
   143  
   144  //go:linknamestd _cgoGetStackBound runtime.cgoGetStackBound
   145  var _cgoGetStackBound = cgoGetStackBound
   146  
   147  // g matches the first few fields that we need from the runtime's g struct.
   148  type g struct {
   149  	_       sys.NotInHeap
   150  	stacklo uintptr
   151  	stackhi uintptr
   152  }
   153  
   154  // pthreadAttr corresponds to pthread_attr_t.
   155  type pthreadAttr struct {
   156  	data [9]uintptr
   157  }
   158  
   159  type sigset struct {
   160  	mask [128 / unsafe.Sizeof(uintptr(0))]uintptr
   161  }
   162  
   163  var (
   164  	pthread_g  uintptr
   165  	fullSigset sigset
   166  )
   167  
   168  //go:nosplit
   169  //go:norace
   170  //go:nocheckptr
   171  func cgoThreadStackSize(attr *pthreadAttr) uintptr {
   172  	if goos.IsDarwin|goos.IsIos != 0 {
   173  		// Copy stack size from parent thread instead of using the
   174  		// non-main thread default stack size.
   175  		self := asmcgocall(uintptr(libc_pthread_self), 0, 0, 0, 0, 0, 0)
   176  		size := asmcgocall(uintptr(libc_pthread_get_stacksize_np), self, 0, 0, 0, 0, 0)
   177  		asmcgocall(uintptr(libc_pthread_attr_setstacksize), uintptr(unsafe.Pointer(attr)), size, 0, 0, 0, 0)
   178  		return size
   179  	}
   180  	var size uintptr
   181  	asmcgocall(uintptr(libc_pthread_attr_getstacksize), uintptr(unsafe.Pointer(attr)), uintptr(unsafe.Pointer(&size)), 0, 0, 0, 0)
   182  	if goos.IsSolaris|goos.IsIllumos != 0 && size == 0 {
   183  		// Solaris can report 0 stack size; fix it.
   184  		size = 2 << 20
   185  		if int32(asmcgocall(uintptr(libc_pthread_attr_setstacksize), uintptr(unsafe.Pointer(attr)), size, 0, 0, 0, 0)) != 0 {
   186  			runtime_writeErrStr("runtime/cgo: pthread_attr_setstacksize failed\n")
   187  			runtime_abort()
   188  		}
   189  	}
   190  	return size
   191  }
   192  
   193  // cgoGetStackBound returns the current OS thread's stack bounds [lo, hi).
   194  // It must be called while pinned to the M (or on a system/C stack) so
   195  // pthread_self refers to a fixed OS thread.
   196  //
   197  //go:nosplit
   198  //go:norace
   199  //go:nocheckptr
   200  func cgoGetStackBound() (uintptr, uintptr) {
   201  	switch {
   202  	case goos.IsDarwin|goos.IsIos != 0:
   203  		self := asmcgocallNoG(uintptr(libc_pthread_self), 0, 0, 0, 0, 0, 0)
   204  		high := asmcgocallNoG(uintptr(libc_pthread_get_stackaddr_np), self, 0, 0, 0, 0, 0)
   205  		size := asmcgocallNoG(uintptr(libc_pthread_get_stacksize_np), self, 0, 0, 0, 0, 0)
   206  		return high - size, high
   207  
   208  	case goos.IsLinux|goos.IsAndroid|goos.IsSolaris|goos.IsIllumos != 0:
   209  		var attr pthreadAttr
   210  		var addr, size uintptr
   211  		// Use if/else rather than overwriting a default so that
   212  		// -gcflags=-N does not keep a reference to the function
   213  		// missing on the other OS.
   214  		var getattr unsafe.Pointer
   215  		if goos.IsIllumos != 0 {
   216  			getattr = libc_pthread_attr_get_np
   217  		} else {
   218  			getattr = libc_pthread_getattr_np
   219  		}
   220  		// Needed before pthread_getattr_np, too, since before glibc 2.32
   221  		// it did not call pthread_attr_init in all cases (see #65625).
   222  		asmcgocallNoG(uintptr(libc_pthread_attr_init), uintptr(unsafe.Pointer(&attr)), 0, 0, 0, 0, 0)
   223  		self := asmcgocallNoG(uintptr(libc_pthread_self), 0, 0, 0, 0, 0, 0)
   224  		if int32(asmcgocallNoG(uintptr(getattr), self, uintptr(unsafe.Pointer(&attr)), 0, 0, 0, 0)) == 0 {
   225  			asmcgocallNoG(uintptr(libc_pthread_attr_getstack), uintptr(unsafe.Pointer(&attr)), uintptr(unsafe.Pointer(&addr)), uintptr(unsafe.Pointer(&size)), 0, 0, 0)
   226  		}
   227  		asmcgocallNoG(uintptr(libc_pthread_attr_destroy), uintptr(unsafe.Pointer(&attr)), 0, 0, 0, 0, 0)
   228  		return addr, addr + size
   229  
   230  	default:
   231  		return 0, 0
   232  	}
   233  }
   234  
   235  //go:nosplit
   236  //go:norace
   237  func cgoYieldWaitRuntimeInit() {
   238  	runtime_cgoUsleep(100) // 100us
   239  }
   240  
   241  //go:norace
   242  //go:nocheckptr
   243  func cgoNotifyRuntimeInitDoneOS() {
   244  	var r, keyVal uintptr
   245  	if goos.IsDarwin|goos.IsIos != 0 {
   246  		var key uintptr
   247  		tsanAcquire()
   248  		r = asmcgocall(uintptr(libc_pthread_key_create),
   249  			uintptr(unsafe.Pointer(&key)),
   250  			pthreadKeyDestructorPC(), 0, 0, 0, 0)
   251  		tsanRelease()
   252  		keyVal = key
   253  	} else {
   254  		var key uint32
   255  		tsanAcquire()
   256  		r = asmcgocall(uintptr(libc_pthread_key_create),
   257  			uintptr(unsafe.Pointer(&key)),
   258  			pthreadKeyDestructorPC(), 0, 0, 0, 0)
   259  		tsanRelease()
   260  		keyVal = uintptr(key)
   261  	}
   262  	if int32(r) == 0 {
   263  		atomic.Storeuintptr(&pthread_g, keyVal)
   264  		atomic.Storeuintptr(&pthread_key_created, 1)
   265  	}
   266  }
   267  
   268  //go:linknamestd _cgoBindM runtime.cgoBindM
   269  var _cgoBindM = cgoBindM
   270  
   271  //go:nosplit
   272  //go:norace
   273  func cgoBindM(g uintptr) {
   274  	asmcgocall(uintptr(libc_pthread_setspecific), atomic.Loaduintptr(&pthread_g), g, 0, 0, 0, 0)
   275  }
   276  
   277  //go:nosplit
   278  //go:norace
   279  //go:nocheckptr
   280  func cgoInit(g *g, _ uintptr, tlsg, tlsbase uintptr) {
   281  	if tlsg != 0 {
   282  		cgoInitTLS(tlsg, tlsbase)
   283  	}
   284  	asmcgocallNoG(uintptr(libc_sigfillset), uintptr(unsafe.Pointer(&fullSigset)), 0, 0, 0, 0, 0)
   285  	lo, _ := cgoGetStackBound()
   286  	g.stacklo = lo
   287  	if g.stacklo >= g.stackhi {
   288  		cgoFatal("runtime/cgo: bad stack bounds\n\x00")
   289  	}
   290  }
   291  
   292  //go:linknamestd _cgoThreadStart runtime.cgoThreadStart
   293  var _cgoThreadStart = cgoThreadStart
   294  
   295  //go:nosplit
   296  //go:norace
   297  //go:nocheckptr
   298  func cgoBlockSigs(oset *sigset) {
   299  	asmcgocall(uintptr(libc_pthread_sigmask), _SIG_SETMASK, uintptr(unsafe.Pointer(&fullSigset)), uintptr(unsafe.Pointer(oset)), 0, 0, 0)
   300  }
   301  
   302  //go:nosplit
   303  //go:norace
   304  //go:nocheckptr
   305  func cgoRestoreSigs(oset *sigset) {
   306  	asmcgocall(uintptr(libc_pthread_sigmask), _SIG_SETMASK, uintptr(unsafe.Pointer(oset)), 0, 0, 0, 0)
   307  }
   308  
   309  //go:nosplit
   310  //go:norace
   311  //go:nocheckptr
   312  func cgoInitThreadAttr(attr *pthreadAttr) int32 {
   313  	asmcgocall(uintptr(libc_pthread_attr_init), uintptr(unsafe.Pointer(attr)), 0, 0, 0, 0, 0)
   314  	return int32(asmcgocall(uintptr(libc_pthread_attr_setdetachstate), uintptr(unsafe.Pointer(attr)), _PTHREAD_CREATE_DETACHED, 0, 0, 0, 0))
   315  }
   316  
   317  //go:nosplit
   318  //go:norace
   319  //go:nocheckptr
   320  func cgoDestroyThreadAttr(attr *pthreadAttr) {
   321  	asmcgocall(uintptr(libc_pthread_attr_destroy), uintptr(unsafe.Pointer(attr)), 0, 0, 0, 0, 0)
   322  }
   323  
   324  //go:nosplit
   325  //go:norace
   326  //go:nocheckptr
   327  func cgoPthreadCreate(thread *uintptr, attr *pthreadAttr, pfn, arg uintptr) int32 {
   328  	return int32(asmcgocall(uintptr(libc_pthread_create),
   329  		uintptr(unsafe.Pointer(thread)),
   330  		uintptr(unsafe.Pointer(attr)),
   331  		pfn, arg, 0, 0))
   332  }
   333  
   334  // cgoPthreadCreateNoG is like cgoPthreadCreate, but for callers without a g.
   335  //
   336  //go:nosplit
   337  //go:norace
   338  //go:nocheckptr
   339  func cgoPthreadCreateNoG(thread *uintptr, attr *pthreadAttr, pfn, arg uintptr) int32 {
   340  	return int32(asmcgocallNoG(uintptr(libc_pthread_create),
   341  		uintptr(unsafe.Pointer(thread)),
   342  		uintptr(unsafe.Pointer(attr)),
   343  		pfn, arg, 0, 0))
   344  }
   345  
   346  // cgoThreadStart is called by runtime.newm1 on the system stack.
   347  //
   348  //go:norace
   349  //go:nocheckptr
   350  func cgoThreadStart(gp *g) (int32, string) {
   351  	var attr pthreadAttr
   352  	var oset sigset
   353  	var thread uintptr
   354  
   355  	cgoBlockSigs(&oset)
   356  	if err := cgoInitThreadAttr(&attr); err != 0 {
   357  		cgoDestroyThreadAttr(&attr)
   358  		cgoRestoreSigs(&oset)
   359  		return err, "pthread_attr_setdetachstate failed"
   360  	}
   361  	gp.stackhi = cgoThreadStackSize(&attr)
   362  
   363  	err := cgoTryPthreadCreate(&thread, &attr, threadentryPC(), uintptr(unsafe.Pointer(gp)))
   364  	cgoDestroyThreadAttr(&attr)
   365  	cgoRestoreSigs(&oset)
   366  
   367  	if err != 0 {
   368  		return err, "pthread_create failed"
   369  	}
   370  	return 0, ""
   371  }
   372  
   373  //go:nosplit
   374  //go:norace
   375  //go:nocheckptr
   376  func threadentry(v unsafe.Pointer) {
   377  	if goos.IsNetbsd != 0 {
   378  		// On NetBSD, a new thread inherits the signal stack of the
   379  		// creating thread. Remove that signal stack before mstart so
   380  		// minit can install the new thread's own signal stack.
   381  		// Do this before setting g: the g0 stack bounds are not
   382  		// set up yet, so we cannot switch stacks with asmcgocall.
   383  		const _SS_DISABLE = 4
   384  		ss := struct {
   385  			sp    uintptr
   386  			size  uintptr
   387  			flags int32
   388  		}{flags: _SS_DISABLE}
   389  		msanUnpoisonNoG(unsafe.Pointer(&ss), unsafe.Sizeof(ss))
   390  		asmcgocallNoG(uintptr(libc_sigaltstack), uintptr(unsafe.Pointer(&ss)), 0, 0, 0, 0, 0)
   391  	}
   392  	runtime_setg(uintptr(v))
   393  	runtime_mstart()
   394  }
   395  
   396  //go:linknamestd _cgoSysThreadCreate runtime.cgoSysThreadCreate
   397  var _cgoSysThreadCreate = cgoSysThreadCreate
   398  
   399  //go:nosplit
   400  //go:norace
   401  //go:nocheckptr
   402  func cgoSysThreadCreate(fn unsafe.Pointer) int32 {
   403  	var attr pthreadAttr
   404  	var p uintptr
   405  
   406  	asmcgocallNoG(uintptr(libc_pthread_attr_init), uintptr(unsafe.Pointer(&attr)), 0, 0, 0, 0, 0)
   407  	if err := int32(asmcgocallNoG(uintptr(libc_pthread_attr_setdetachstate), uintptr(unsafe.Pointer(&attr)), _PTHREAD_CREATE_DETACHED, 0, 0, 0, 0)); err != 0 {
   408  		asmcgocallNoG(uintptr(libc_pthread_attr_destroy), uintptr(unsafe.Pointer(&attr)), 0, 0, 0, 0, 0)
   409  		return err
   410  	}
   411  	err := cgoTryPthreadCreateNoG(&p, &attr, uintptr(fn), 0)
   412  	asmcgocallNoG(uintptr(libc_pthread_attr_destroy), uintptr(unsafe.Pointer(&attr)), 0, 0, 0, 0, 0)
   413  	return err
   414  }
   415  
   416  //go:nosplit
   417  //go:norace
   418  //go:nocheckptr
   419  func cgoTryPthreadCreate(thread *uintptr, attr *pthreadAttr, pfn, arg uintptr) int32 {
   420  	for tries := uint32(0); tries < 20; tries++ {
   421  		err := cgoPthreadCreate(thread, attr, pfn, arg)
   422  		if err == 0 {
   423  			return 0
   424  		}
   425  		if err != _EAGAIN {
   426  			return err
   427  		}
   428  		runtime_cgoUsleep((tries + 1) * 1000) // Milliseconds.
   429  	}
   430  	return _EAGAIN
   431  }
   432  
   433  // cgoTryPthreadCreateNoG is like cgoTryPthreadCreate, but for callers
   434  // without a g. It is separate, rather than taking a flag, so that those
   435  // callers do not statically reach asmcgocall: the linker's nosplit check
   436  // would count it, as dead branches remain with -gcflags=-N.
   437  //
   438  //go:nosplit
   439  //go:norace
   440  //go:nocheckptr
   441  func cgoTryPthreadCreateNoG(thread *uintptr, attr *pthreadAttr, pfn, arg uintptr) int32 {
   442  	for tries := uint32(0); tries < 20; tries++ {
   443  		err := cgoPthreadCreateNoG(thread, attr, pfn, arg)
   444  		if err == 0 {
   445  			return 0
   446  		}
   447  		if err != _EAGAIN {
   448  			return err
   449  		}
   450  		runtime_cgoUsleep((tries + 1) * 1000) // Milliseconds.
   451  	}
   452  	return _EAGAIN
   453  }
   454  

View as plain text