Source file src/runtime/stubs.go
1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "unsafe" 10 ) 11 12 // Should be a built-in for unsafe.Pointer? 13 // 14 // add should be an internal detail, 15 // but widely used packages access it using linkname. 16 // Notable members of the hall of shame include: 17 // - fortio.org/log 18 // 19 // Do not remove or change the type signature. 20 // See go.dev/issue/67401. 21 // 22 //go:linkname add 23 //go:nosplit 24 func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { 25 return unsafe.Pointer(uintptr(p) + x) 26 } 27 28 // getg returns the pointer to the current g. 29 // The compiler rewrites calls to this function into instructions 30 // that fetch the g directly (from TLS or from the dedicated register). 31 func getg() *g 32 33 // mcall switches from the g to the g0 stack and invokes fn(g), 34 // where g is the goroutine that made the call. 35 // mcall saves g's current PC/SP in g->sched so that it can be restored later. 36 // It is up to fn to arrange for that later execution, typically by recording 37 // g in a data structure, causing something to call ready(g) later. 38 // mcall returns to the original goroutine g later, when g has been rescheduled. 39 // fn must not return at all; typically it ends by calling schedule, to let the m 40 // run other goroutines. 41 // 42 // mcall can only be called from g stacks (not g0, not gsignal). 43 // 44 // This must NOT be go:noescape: if fn is a stack-allocated closure, 45 // fn puts g on a run queue, and g executes before fn returns, the 46 // closure will be invalidated while it is still executing. 47 func mcall(fn func(*g)) 48 49 // systemstack runs fn on a system stack. 50 // If systemstack is called from the per-OS-thread (g0) stack, or 51 // if systemstack is called from the signal handling (gsignal) stack, 52 // systemstack calls fn directly and returns. 53 // Otherwise, systemstack is being called from the limited stack 54 // of an ordinary goroutine. In this case, systemstack switches 55 // to the per-OS-thread stack, calls fn, and switches back. 56 // It is common to use a func literal as the argument, in order 57 // to share inputs and outputs with the code around the call 58 // to system stack: 59 // 60 // ... set up y ... 61 // systemstack(func() { 62 // x = bigcall(y) 63 // }) 64 // ... use x ... 65 // 66 //go:noescape 67 func systemstack(fn func()) 68 69 //go:nosplit 70 //go:nowritebarrierrec 71 func badsystemstack() { 72 writeErrStr("fatal: systemstack called from unexpected goroutine") 73 } 74 75 // memclrNoHeapPointers clears n bytes starting at ptr. 76 // 77 // Usually you should use typedmemclr. memclrNoHeapPointers should be 78 // used only when the caller knows that *ptr contains no heap pointers 79 // because either: 80 // 81 // *ptr is initialized memory and its type is pointer-free, or 82 // 83 // *ptr is uninitialized memory (e.g., memory that's being reused 84 // for a new allocation) and hence contains only "junk". 85 // 86 // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n 87 // is a multiple of the pointer size, then any pointer-aligned, 88 // pointer-sized portion is cleared atomically. Despite the function 89 // name, this is necessary because this function is the underlying 90 // implementation of typedmemclr and memclrHasPointers. See the doc of 91 // memmove for more details. 92 // 93 // The (CPU-specific) implementations of this function are in memclr_*.s. 94 // 95 // memclrNoHeapPointers should be an internal detail, 96 // but widely used packages access it using linkname. 97 // Notable members of the hall of shame include: 98 // - github.com/bytedance/sonic 99 // - github.com/chenzhuoyu/iasm 100 // - github.com/dgraph-io/ristretto 101 // - github.com/outcaste-io/ristretto 102 // 103 // Do not remove or change the type signature. 104 // See go.dev/issue/67401. 105 // 106 //go:linkname memclrNoHeapPointers 107 //go:noescape 108 func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) 109 110 //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers 111 func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { 112 memclrNoHeapPointers(ptr, n) 113 } 114 115 // memmove copies n bytes from "from" to "to". 116 // 117 // memmove ensures that any pointer in "from" is written to "to" with 118 // an indivisible write, so that racy reads cannot observe a 119 // half-written pointer. This is necessary to prevent the garbage 120 // collector from observing invalid pointers, and differs from memmove 121 // in unmanaged languages. However, memmove is only required to do 122 // this if "from" and "to" may contain pointers, which can only be the 123 // case if "from", "to", and "n" are all be word-aligned. 124 // 125 // Implementations are in memmove_*.s. 126 // 127 // Outside assembly calls memmove. 128 // 129 // memmove should be an internal detail, 130 // but widely used packages access it using linkname. 131 // Notable members of the hall of shame include: 132 // - github.com/bytedance/sonic 133 // - github.com/cloudwego/dynamicgo 134 // - github.com/ebitengine/purego 135 // - github.com/tetratelabs/wazero 136 // - github.com/ugorji/go/codec 137 // - gvisor.dev/gvisor 138 // - github.com/sagernet/gvisor 139 // 140 // Do not remove or change the type signature. 141 // See go.dev/issue/67401. 142 // 143 //go:linkname memmove 144 //go:noescape 145 func memmove(to, from unsafe.Pointer, n uintptr) 146 147 //go:linkname reflect_memmove reflect.memmove 148 func reflect_memmove(to, from unsafe.Pointer, n uintptr) { 149 memmove(to, from, n) 150 } 151 152 // exported value for testing 153 const hashLoad = float32(loadFactorNum) / float32(loadFactorDen) 154 155 // in internal/bytealg/equal_*.s 156 // 157 // memequal should be an internal detail, 158 // but widely used packages access it using linkname. 159 // Notable members of the hall of shame include: 160 // - github.com/bytedance/sonic 161 // 162 // Do not remove or change the type signature. 163 // See go.dev/issue/67401. 164 // 165 //go:linkname memequal 166 //go:noescape 167 func memequal(a, b unsafe.Pointer, size uintptr) bool 168 169 // noescape hides a pointer from escape analysis. noescape is 170 // the identity function but escape analysis doesn't think the 171 // output depends on the input. noescape is inlined and currently 172 // compiles down to zero instructions. 173 // USE CAREFULLY! 174 // 175 // noescape should be an internal detail, 176 // but widely used packages access it using linkname. 177 // Notable members of the hall of shame include: 178 // - github.com/bytedance/gopkg 179 // - github.com/ebitengine/purego 180 // - github.com/hamba/avro/v2 181 // - github.com/puzpuzpuz/xsync/v3 182 // - github.com/songzhibin97/gkit 183 // 184 // Do not remove or change the type signature. 185 // See go.dev/issue/67401. 186 // 187 //go:linkname noescape 188 //go:nosplit 189 func noescape(p unsafe.Pointer) unsafe.Pointer { 190 x := uintptr(p) 191 return unsafe.Pointer(x ^ 0) 192 } 193 194 // noEscapePtr hides a pointer from escape analysis. See noescape. 195 // USE CAREFULLY! 196 // 197 //go:nosplit 198 func noEscapePtr[T any](p *T) *T { 199 x := uintptr(unsafe.Pointer(p)) 200 return (*T)(unsafe.Pointer(x ^ 0)) 201 } 202 203 // Not all cgocallback frames are actually cgocallback, 204 // so not all have these arguments. Mark them uintptr so that the GC 205 // does not misinterpret memory when the arguments are not present. 206 // cgocallback is not called from Go, only from crosscall2. 207 // This in turn calls cgocallbackg, which is where we'll find 208 // pointer-declared arguments. 209 // 210 // When fn is nil (frame is saved g), call dropm instead, 211 // this is used when the C thread is exiting. 212 func cgocallback(fn, frame, ctxt uintptr) 213 214 func gogo(buf *gobuf) 215 216 func asminit() 217 func setg(gg *g) 218 func breakpoint() 219 220 // reflectcall calls fn with arguments described by stackArgs, stackArgsSize, 221 // frameSize, and regArgs. 222 // 223 // Arguments passed on the stack and space for return values passed on the stack 224 // must be laid out at the space pointed to by stackArgs (with total length 225 // stackArgsSize) according to the ABI. 226 // 227 // stackRetOffset must be some value <= stackArgsSize that indicates the 228 // offset within stackArgs where the return value space begins. 229 // 230 // frameSize is the total size of the argument frame at stackArgs and must 231 // therefore be >= stackArgsSize. It must include additional space for spilling 232 // register arguments for stack growth and preemption. 233 // 234 // TODO(mknyszek): Once we don't need the additional spill space, remove frameSize, 235 // since frameSize will be redundant with stackArgsSize. 236 // 237 // Arguments passed in registers must be laid out in regArgs according to the ABI. 238 // regArgs will hold any return values passed in registers after the call. 239 // 240 // reflectcall copies stack arguments from stackArgs to the goroutine stack, and 241 // then copies back stackArgsSize-stackRetOffset bytes back to the return space 242 // in stackArgs once fn has completed. It also "unspills" argument registers from 243 // regArgs before calling fn, and spills them back into regArgs immediately 244 // following the call to fn. If there are results being returned on the stack, 245 // the caller should pass the argument frame type as stackArgsType so that 246 // reflectcall can execute appropriate write barriers during the copy. 247 // 248 // reflectcall expects regArgs.ReturnIsPtr to be populated indicating which 249 // registers on the return path will contain Go pointers. It will then store 250 // these pointers in regArgs.Ptrs such that they are visible to the GC. 251 // 252 // Package reflect passes a frame type. In package runtime, there is only 253 // one call that copies results back, in callbackWrap in syscall_windows.go, and it 254 // does NOT pass a frame type, meaning there are no write barriers invoked. See that 255 // call site for justification. 256 // 257 // Package reflect accesses this symbol through a linkname. 258 // 259 // Arguments passed through to reflectcall do not escape. The type is used 260 // only in a very limited callee of reflectcall, the stackArgs are copied, and 261 // regArgs is only used in the reflectcall frame. 262 // 263 //go:noescape 264 func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 265 266 // procyield should be an internal detail, 267 // but widely used packages access it using linkname. 268 // Notable members of the hall of shame include: 269 // - github.com/sagernet/sing-tun 270 // - github.com/slackhq/nebula 271 // - golang.zx2c4.com/wireguard 272 // 273 // Do not remove or change the type signature. 274 // See go.dev/issue/67401. 275 // 276 //go:linkname procyield 277 //go:nosplit 278 func procyield(cycles uint32) { 279 if cycles == 0 { 280 return 281 } 282 procyieldAsm(cycles) 283 } 284 285 // procyieldAsm is the assembly implementation of procyield. 286 func procyieldAsm(cycles uint32) 287 288 type neverCallThisFunction struct{} 289 290 // goexit is the return stub at the top of every goroutine call stack. 291 // Each goroutine stack is constructed as if goexit called the 292 // goroutine's entry point function, so that when the entry point 293 // function returns, it will return to goexit, which will call goexit1 294 // to perform the actual exit. 295 // 296 // This function must never be called directly. Call goexit1 instead. 297 // gentraceback assumes that goexit terminates the stack. A direct 298 // call on the stack will cause gentraceback to stop walking the stack 299 // prematurely and if there is leftover state it may panic. 300 func goexit(neverCallThisFunction) 301 302 // publicationBarrier performs a store/store barrier (a "publication" 303 // or "export" barrier). Some form of synchronization is required 304 // between initializing an object and making that object accessible to 305 // another processor. Without synchronization, the initialization 306 // writes and the "publication" write may be reordered, allowing the 307 // other processor to follow the pointer and observe an uninitialized 308 // object. In general, higher-level synchronization should be used, 309 // such as locking or an atomic pointer write. publicationBarrier is 310 // for when those aren't an option, such as in the implementation of 311 // the memory manager. 312 // 313 // There's no corresponding barrier for the read side because the read 314 // side naturally has a data dependency order. All architectures that 315 // Go supports or seems likely to ever support automatically enforce 316 // data dependency ordering. 317 func publicationBarrier() 318 319 //go:noescape 320 func asmcgocall(fn, arg unsafe.Pointer) int32 321 322 func morestack() 323 324 // morestack_noctxt should be an internal detail, 325 // but widely used packages access it using linkname. 326 // Notable members of the hall of shame include: 327 // - github.com/bytedance/sonic 328 // 329 // Do not remove or change the type signature. 330 // See go.dev/issues/67401. 331 // See go.dev/issues/71672. 332 // 333 //go:linkname morestack_noctxt 334 func morestack_noctxt() 335 336 func rt0_go() 337 338 // in asm_*.s 339 // not called directly; definitions here supply type information for traceback. 340 // These must have the same signature (arg pointer map) as reflectcall. 341 func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 342 func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 343 func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 344 func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 345 func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 346 func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 347 func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 348 func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 349 func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 350 func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 351 func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 352 func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 353 func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 354 func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 355 func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 356 func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 357 func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 358 func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 359 func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 360 func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 361 func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 362 func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 363 func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 364 func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 365 func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 366 func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 367 func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 368 369 func systemstack_switch() 370 371 // alignUp rounds n up to a multiple of a. a must be a power of 2. 372 // 373 //go:nosplit 374 func alignUp(n, a uintptr) uintptr { 375 return (n + a - 1) &^ (a - 1) 376 } 377 378 // alignDown rounds n down to a multiple of a. a must be a power of 2. 379 // 380 //go:nosplit 381 func alignDown(n, a uintptr) uintptr { 382 return n &^ (a - 1) 383 } 384 385 // divRoundUp returns ceil(n / a). 386 // 387 //go:nosplit 388 func divRoundUp(n, a uintptr) uintptr { 389 // a is generally a power of two. This will get inlined and 390 // the compiler will optimize the division. 391 return (n + a - 1) / a 392 } 393 394 // checkASM reports whether assembly runtime checks have passed. 395 func checkASM() bool 396 397 func memequal_varlen(a, b unsafe.Pointer) bool 398 399 // bool2int returns 0 if x is false or 1 if x is true. 400 func bool2int(x bool) int { 401 // Avoid branches. In the SSA compiler, this compiles to 402 // exactly what you would want it to. 403 return int(*(*uint8)(unsafe.Pointer(&x))) 404 } 405 406 // abort crashes the runtime in situations where even throw might not 407 // work. In general it should do something a debugger will recognize 408 // (e.g., an INT3 on x86). A crash in abort is recognized by the 409 // signal handler, which will attempt to tear down the runtime 410 // immediately. 411 func abort() 412 413 // Called from compiled code; declared for vet; do NOT call from Go. 414 func gcWriteBarrier1() 415 416 // gcWriteBarrier2 should be an internal detail, 417 // but widely used packages access it using linkname. 418 // Notable members of the hall of shame include: 419 // - github.com/bytedance/sonic 420 // 421 // Do not remove or change the type signature. 422 // See go.dev/issue/67401. 423 // 424 //go:linkname gcWriteBarrier2 425 func gcWriteBarrier2() 426 427 func gcWriteBarrier3() 428 func gcWriteBarrier4() 429 func gcWriteBarrier5() 430 func gcWriteBarrier6() 431 func gcWriteBarrier7() 432 func gcWriteBarrier8() 433 func duffzero() 434 func duffcopy() 435 436 // Called from linker-generated .initarray; declared for go vet; do NOT call from Go. 437 func addmoduledata() 438 439 // Injected by the signal handler for panicking signals. 440 // Initializes any registers that have fixed meaning at calls but 441 // are scratch in bodies and calls sigpanic. 442 // On many platforms it just jumps to sigpanic. 443 func sigpanic0() 444 445 // intArgRegs is used by the various register assignment 446 // algorithm implementations in the runtime. These include:. 447 // - Finalizers (mfinal.go) 448 // - Windows callbacks (syscall_windows.go) 449 // 450 // Both are stripped-down versions of the algorithm since they 451 // only have to deal with a subset of cases (finalizers only 452 // take a pointer or interface argument, Go Windows callbacks 453 // don't support floating point). 454 // 455 // It should be modified with care and are generally only 456 // modified when testing this package. 457 // 458 // It should never be set higher than its internal/abi 459 // constant counterparts, because the system relies on a 460 // structure that is at least large enough to hold the 461 // registers the system supports. 462 // 463 // Protected by finlock. 464 var intArgRegs = abi.IntArgRegs 465