Source file src/runtime/stubs.go
1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "unsafe" 10 ) 11 12 // Should be a built-in for unsafe.Pointer? 13 // 14 // add should be an internal detail, 15 // but widely used packages access it using linkname. 16 // Notable members of the hall of shame include: 17 // - fortio.org/log 18 // 19 // Do not remove or change the type signature. 20 // See go.dev/issue/67401. 21 // 22 //go:linkname add 23 //go:nosplit 24 func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { 25 return unsafe.Pointer(uintptr(p) + x) 26 } 27 28 // getg returns the pointer to the current g. 29 // The compiler rewrites calls to this function into instructions 30 // that fetch the g directly (from TLS or from the dedicated register). 31 func getg() *g 32 33 // mcall switches from the g to the g0 stack and invokes fn(g), 34 // where g is the goroutine that made the call. 35 // mcall saves g's current PC/SP in g->sched so that it can be restored later. 36 // It is up to fn to arrange for that later execution, typically by recording 37 // g in a data structure, causing something to call ready(g) later. 38 // mcall returns to the original goroutine g later, when g has been rescheduled. 39 // fn must not return at all; typically it ends by calling schedule, to let the m 40 // run other goroutines. 41 // 42 // mcall can only be called from g stacks (not g0, not gsignal). 43 // 44 // This must NOT be go:noescape: if fn is a stack-allocated closure, 45 // fn puts g on a run queue, and g executes before fn returns, the 46 // closure will be invalidated while it is still executing. 47 func mcall(fn func(*g)) 48 49 // systemstack runs fn on a system stack. 50 // If systemstack is called from the per-OS-thread (g0) stack, or 51 // if systemstack is called from the signal handling (gsignal) stack, 52 // systemstack calls fn directly and returns. 53 // Otherwise, systemstack is being called from the limited stack 54 // of an ordinary goroutine. In this case, systemstack switches 55 // to the per-OS-thread stack, calls fn, and switches back. 56 // It is common to use a func literal as the argument, in order 57 // to share inputs and outputs with the code around the call 58 // to system stack: 59 // 60 // ... set up y ... 61 // systemstack(func() { 62 // x = bigcall(y) 63 // }) 64 // ... use x ... 65 // 66 //go:noescape 67 func systemstack(fn func()) 68 69 //go:nosplit 70 //go:nowritebarrierrec 71 func badsystemstack() { 72 writeErrStr("fatal: systemstack called from unexpected goroutine") 73 } 74 75 // memclrNoHeapPointers clears n bytes starting at ptr. 76 // 77 // Usually you should use typedmemclr. memclrNoHeapPointers should be 78 // used only when the caller knows that *ptr contains no heap pointers 79 // because either: 80 // 81 // *ptr is initialized memory and its type is pointer-free, or 82 // 83 // *ptr is uninitialized memory (e.g., memory that's being reused 84 // for a new allocation) and hence contains only "junk". 85 // 86 // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n 87 // is a multiple of the pointer size, then any pointer-aligned, 88 // pointer-sized portion is cleared atomically. Despite the function 89 // name, this is necessary because this function is the underlying 90 // implementation of typedmemclr and memclrHasPointers. See the doc of 91 // memmove for more details. 92 // 93 // The (CPU-specific) implementations of this function are in memclr_*.s. 94 // 95 // memclrNoHeapPointers should be an internal detail, 96 // but widely used packages access it using linkname. 97 // Notable members of the hall of shame include: 98 // - github.com/bytedance/sonic 99 // - github.com/chenzhuoyu/iasm 100 // - github.com/dgraph-io/ristretto 101 // - github.com/outcaste-io/ristretto 102 // 103 // Do not remove or change the type signature. 104 // See go.dev/issue/67401. 105 // 106 //go:linkname memclrNoHeapPointers 107 //go:noescape 108 func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) 109 110 //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers 111 func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { 112 memclrNoHeapPointers(ptr, n) 113 } 114 115 // memmove copies n bytes from "from" to "to". 116 // 117 // memmove ensures that any pointer in "from" is written to "to" with 118 // an indivisible write, so that racy reads cannot observe a 119 // half-written pointer. This is necessary to prevent the garbage 120 // collector from observing invalid pointers, and differs from memmove 121 // in unmanaged languages. However, memmove is only required to do 122 // this if "from" and "to" may contain pointers, which can only be the 123 // case if "from", "to", and "n" are all be word-aligned. 124 // 125 // Implementations are in memmove_*.s. 126 // 127 // Outside assembly calls memmove. 128 // 129 // memmove should be an internal detail, 130 // but widely used packages access it using linkname. 131 // Notable members of the hall of shame include: 132 // - github.com/bytedance/sonic 133 // - github.com/cloudwego/dynamicgo 134 // - github.com/ebitengine/purego 135 // - github.com/tetratelabs/wazero 136 // - github.com/ugorji/go/codec 137 // - gvisor.dev/gvisor 138 // - github.com/sagernet/gvisor 139 // 140 // Do not remove or change the type signature. 141 // See go.dev/issue/67401. 142 // 143 //go:linkname memmove 144 //go:noescape 145 func memmove(to, from unsafe.Pointer, n uintptr) 146 147 //go:linkname reflect_memmove reflect.memmove 148 func reflect_memmove(to, from unsafe.Pointer, n uintptr) { 149 memmove(to, from, n) 150 } 151 152 // exported value for testing 153 const hashLoad = float32(loadFactorNum) / float32(loadFactorDen) 154 155 // in internal/bytealg/equal_*.s 156 // 157 // memequal should be an internal detail, 158 // but widely used packages access it using linkname. 159 // Notable members of the hall of shame include: 160 // - github.com/bytedance/sonic 161 // 162 // Do not remove or change the type signature. 163 // See go.dev/issue/67401. 164 // 165 //go:linkname memequal 166 //go:noescape 167 func memequal(a, b unsafe.Pointer, size uintptr) bool 168 169 // noescape hides a pointer from escape analysis. noescape is 170 // the identity function but escape analysis doesn't think the 171 // output depends on the input. noescape is inlined and currently 172 // compiles down to zero instructions. 173 // USE CAREFULLY! 174 // 175 // noescape should be an internal detail, 176 // but widely used packages access it using linkname. 177 // Notable members of the hall of shame include: 178 // - github.com/bytedance/gopkg 179 // - github.com/ebitengine/purego 180 // - github.com/hamba/avro/v2 181 // - github.com/puzpuzpuz/xsync/v3 182 // - github.com/songzhibin97/gkit 183 // 184 // Do not remove or change the type signature. 185 // See go.dev/issue/67401. 186 // 187 //go:linkname noescape 188 //go:nosplit 189 func noescape(p unsafe.Pointer) unsafe.Pointer { 190 x := uintptr(p) 191 return unsafe.Pointer(x ^ 0) 192 } 193 194 // noEscapePtr hides a pointer from escape analysis. See noescape. 195 // USE CAREFULLY! 196 // 197 //go:nosplit 198 func noEscapePtr[T any](p *T) *T { 199 x := uintptr(unsafe.Pointer(p)) 200 return (*T)(unsafe.Pointer(x ^ 0)) 201 } 202 203 // Not all cgocallback frames are actually cgocallback, 204 // so not all have these arguments. Mark them uintptr so that the GC 205 // does not misinterpret memory when the arguments are not present. 206 // cgocallback is not called from Go, only from crosscall2 in runtime/cgo. 207 // This in turn calls cgocallbackg, which is where we'll find 208 // pointer-declared arguments. 209 // 210 // When fn is nil (frame is saved g), call dropm instead, 211 // this is used when the C thread is exiting. 212 // 213 // cgocallback should be an internal detail, 214 // but widely used packages access it using linkname. 215 // Notable members of the hall of shame include: 216 // - github.com/ebitengine/purego 217 // 218 //go:linkname cgocallback 219 func cgocallback(fn, frame, ctxt uintptr) 220 221 func gogo(buf *gobuf) 222 223 func asminit() 224 func setg(gg *g) 225 func breakpoint() 226 227 // reflectcall calls fn with arguments described by stackArgs, stackArgsSize, 228 // frameSize, and regArgs. 229 // 230 // Arguments passed on the stack and space for return values passed on the stack 231 // must be laid out at the space pointed to by stackArgs (with total length 232 // stackArgsSize) according to the ABI. 233 // 234 // stackRetOffset must be some value <= stackArgsSize that indicates the 235 // offset within stackArgs where the return value space begins. 236 // 237 // frameSize is the total size of the argument frame at stackArgs and must 238 // therefore be >= stackArgsSize. It must include additional space for spilling 239 // register arguments for stack growth and preemption. 240 // 241 // TODO(mknyszek): Once we don't need the additional spill space, remove frameSize, 242 // since frameSize will be redundant with stackArgsSize. 243 // 244 // Arguments passed in registers must be laid out in regArgs according to the ABI. 245 // regArgs will hold any return values passed in registers after the call. 246 // 247 // reflectcall copies stack arguments from stackArgs to the goroutine stack, and 248 // then copies back stackArgsSize-stackRetOffset bytes back to the return space 249 // in stackArgs once fn has completed. It also "unspills" argument registers from 250 // regArgs before calling fn, and spills them back into regArgs immediately 251 // following the call to fn. If there are results being returned on the stack, 252 // the caller should pass the argument frame type as stackArgsType so that 253 // reflectcall can execute appropriate write barriers during the copy. 254 // 255 // reflectcall expects regArgs.ReturnIsPtr to be populated indicating which 256 // registers on the return path will contain Go pointers. It will then store 257 // these pointers in regArgs.Ptrs such that they are visible to the GC. 258 // 259 // Package reflect passes a frame type. In package runtime, there is only 260 // one call that copies results back, in callbackWrap in syscall_windows.go, and it 261 // does NOT pass a frame type, meaning there are no write barriers invoked. See that 262 // call site for justification. 263 // 264 // Package reflect accesses this symbol through a linkname. 265 // 266 // Arguments passed through to reflectcall do not escape. The type is used 267 // only in a very limited callee of reflectcall, the stackArgs are copied, and 268 // regArgs is only used in the reflectcall frame. 269 // 270 //go:linknamestd reflectcall 271 //go:noescape 272 func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 273 274 // procyield should be an internal detail, 275 // but widely used packages access it using linkname. 276 // Notable members of the hall of shame include: 277 // - github.com/sagernet/sing-tun 278 // - github.com/slackhq/nebula 279 // - golang.zx2c4.com/wireguard 280 // 281 // Do not remove or change the type signature. 282 // See go.dev/issue/67401. 283 // 284 //go:linkname procyield 285 //go:nosplit 286 func procyield(cycles uint32) { 287 if cycles == 0 { 288 return 289 } 290 procyieldAsm(cycles) 291 } 292 293 // procyieldAsm is the assembly implementation of procyield. 294 func procyieldAsm(cycles uint32) 295 296 type neverCallThisFunction struct{} 297 298 // goexit is the return stub at the top of every goroutine call stack. 299 // Each goroutine stack is constructed as if goexit called the 300 // goroutine's entry point function, so that when the entry point 301 // function returns, it will return to goexit, which will call goexit1 302 // to perform the actual exit. 303 // 304 // This function must never be called directly. Call goexit1 instead. 305 // gentraceback assumes that goexit terminates the stack. A direct 306 // call on the stack will cause gentraceback to stop walking the stack 307 // prematurely and if there is leftover state it may panic. 308 func goexit(neverCallThisFunction) 309 310 // publicationBarrier performs a store/store barrier (a "publication" 311 // or "export" barrier). Some form of synchronization is required 312 // between initializing an object and making that object accessible to 313 // another processor. Without synchronization, the initialization 314 // writes and the "publication" write may be reordered, allowing the 315 // other processor to follow the pointer and observe an uninitialized 316 // object. In general, higher-level synchronization should be used, 317 // such as locking or an atomic pointer write. publicationBarrier is 318 // for when those aren't an option, such as in the implementation of 319 // the memory manager. 320 // 321 // There's no corresponding barrier for the read side because the read 322 // side naturally has a data dependency order. All architectures that 323 // Go supports or seems likely to ever support automatically enforce 324 // data dependency ordering. 325 func publicationBarrier() 326 327 //go:noescape 328 func asmcgocall(fn, arg unsafe.Pointer) int32 329 330 func morestack() 331 332 // morestack_noctxt should be an internal detail, 333 // but widely used packages access it using linkname. 334 // Notable members of the hall of shame include: 335 // - github.com/bytedance/sonic 336 // 337 // Do not remove or change the type signature. 338 // See go.dev/issues/67401. 339 // See go.dev/issues/71672. 340 // 341 //go:linkname morestack_noctxt 342 func morestack_noctxt() 343 344 func rt0_go() 345 func rt0_lib_go() 346 347 // in asm_*.s 348 // not called directly; definitions here supply type information for traceback. 349 // These must have the same signature (arg pointer map) as reflectcall. 350 func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 351 func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 352 func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 353 func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 354 func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 355 func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 356 func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 357 func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 358 func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 359 func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 360 func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 361 func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 362 func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 363 func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 364 func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 365 func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 366 func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 367 func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 368 func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 369 func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 370 func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 371 func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 372 func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 373 func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 374 func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 375 func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 376 func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 377 378 func systemstack_switch() 379 380 // alignUp rounds n up to a multiple of a. a must be a power of 2. 381 // 382 //go:nosplit 383 func alignUp(n, a uintptr) uintptr { 384 return (n + a - 1) &^ (a - 1) 385 } 386 387 // alignDown rounds n down to a multiple of a. a must be a power of 2. 388 // 389 //go:nosplit 390 func alignDown(n, a uintptr) uintptr { 391 return n &^ (a - 1) 392 } 393 394 // divRoundUp returns ceil(n / a). 395 // 396 //go:nosplit 397 func divRoundUp(n, a uintptr) uintptr { 398 // a is generally a power of two. This will get inlined and 399 // the compiler will optimize the division. 400 return (n + a - 1) / a 401 } 402 403 func memequal_varlen(a, b unsafe.Pointer) bool 404 405 // bool2int returns 0 if x is false or 1 if x is true. 406 func bool2int(x bool) int { 407 // Avoid branches. In the SSA compiler, this compiles to 408 // exactly what you would want it to. 409 return int(*(*uint8)(unsafe.Pointer(&x))) 410 } 411 412 // abort crashes the runtime in situations where even throw might not 413 // work. In general it should do something a debugger will recognize 414 // (e.g., an INT3 on x86). A crash in abort is recognized by the 415 // signal handler, which will attempt to tear down the runtime 416 // immediately. 417 // 418 // Also called from runtime test. 419 // 420 //go:linkname abort 421 func abort() 422 423 // Called from compiled code; declared for vet; do NOT call from Go. 424 func gcWriteBarrier1() 425 426 // gcWriteBarrier2 should be an internal detail, 427 // but widely used packages access it using linkname. 428 // Notable members of the hall of shame include: 429 // - github.com/bytedance/sonic 430 // 431 // Do not remove or change the type signature. 432 // See go.dev/issue/67401. 433 // 434 //go:linkname gcWriteBarrier2 435 func gcWriteBarrier2() 436 437 func gcWriteBarrier3() 438 func gcWriteBarrier4() 439 func gcWriteBarrier5() 440 func gcWriteBarrier6() 441 func gcWriteBarrier7() 442 func gcWriteBarrier8() 443 func duffzero() 444 func duffcopy() 445 446 // Called from linker-generated .initarray; declared for go vet; do NOT call from Go. 447 func addmoduledata() 448 449 // Injected by the signal handler for panicking signals. 450 // Initializes any registers that have fixed meaning at calls but 451 // are scratch in bodies and calls sigpanic. 452 // On many platforms it just jumps to sigpanic. 453 func sigpanic0() 454 455 // intArgRegs is used by the various register assignment 456 // algorithm implementations in the runtime. These include:. 457 // - Finalizers (mfinal.go) 458 // - Windows callbacks (syscall_windows.go) 459 // 460 // Both are stripped-down versions of the algorithm since they 461 // only have to deal with a subset of cases (finalizers only 462 // take a pointer or interface argument, Go Windows callbacks 463 // don't support floating point). 464 // 465 // It should be modified with care and are generally only 466 // modified when testing this package. 467 // 468 // It should never be set higher than its internal/abi 469 // constant counterparts, because the system relies on a 470 // structure that is at least large enough to hold the 471 // registers the system supports. 472 // 473 // Protected by finlock. 474 var intArgRegs = abi.IntArgRegs 475