Text file
src/runtime/asm_amd64.s
1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "funcdata.h"
8 #include "textflag.h"
9 #include "cgo/abi_amd64.h"
10
11 // _rt0_amd64 is common startup code for most amd64 systems when using
12 // internal linking. This is the entry point for the program from the
13 // kernel for an ordinary -buildmode=exe program. The stack holds the
14 // number of arguments and the C-style argv.
15 TEXT _rt0_amd64(SB),NOSPLIT,$-8
16 MOVQ 0(SP), DI // argc
17 LEAQ 8(SP), SI // argv
18 JMP runtime·rt0_go(SB)
19
20 // main is common startup code for most amd64 systems when using
21 // external linking. The C startup code will call the symbol "main"
22 // passing argc and argv in the usual C ABI registers DI and SI.
23 TEXT main(SB),NOSPLIT,$-8
24 JMP runtime·rt0_go(SB)
25
26 // _rt0_amd64_lib is common startup code for most amd64 systems when
27 // using -buildmode=c-archive or -buildmode=c-shared. The linker will
28 // arrange to invoke this function as a global constructor (for
29 // c-archive) or when the shared library is loaded (for c-shared).
30 // We expect argc and argv to be passed in the usual C ABI registers
31 // DI and SI.
32 TEXT _rt0_amd64_lib(SB),NOSPLIT|NOFRAME,$0
33 // Transition from C ABI to Go ABI.
34 PUSH_REGS_HOST_TO_ABI0()
35
36 MOVQ DI, _rt0_amd64_lib_argc<>(SB)
37 MOVQ SI, _rt0_amd64_lib_argv<>(SB)
38
39 CALL runtime·libInit(SB)
40
41 POP_REGS_HOST_TO_ABI0()
42 RET
43
44 // rt0_lib_go initializes the Go runtime.
45 // This is started in a separate thread by _rt0_amd64_lib.
46 TEXT runtime·rt0_lib_go<ABIInternal>(SB),NOSPLIT,$0
47 MOVQ _rt0_amd64_lib_argc<>(SB), DI
48 MOVQ _rt0_amd64_lib_argv<>(SB), SI
49 JMP runtime·rt0_go(SB)
50
51 DATA _rt0_amd64_lib_argc<>(SB)/8, $0
52 GLOBL _rt0_amd64_lib_argc<>(SB),NOPTR, $8
53 DATA _rt0_amd64_lib_argv<>(SB)/8, $0
54 GLOBL _rt0_amd64_lib_argv<>(SB),NOPTR, $8
55
56 #ifdef GOAMD64_v2
57 DATA bad_cpu_msg<>+0x00(SB)/84, $"This program can only be run on AMD64 processors with v2 microarchitecture support.\n"
58 #endif
59
60 #ifdef GOAMD64_v3
61 DATA bad_cpu_msg<>+0x00(SB)/84, $"This program can only be run on AMD64 processors with v3 microarchitecture support.\n"
62 #endif
63
64 #ifdef GOAMD64_v4
65 DATA bad_cpu_msg<>+0x00(SB)/84, $"This program can only be run on AMD64 processors with v4 microarchitecture support.\n"
66 #endif
67
68 GLOBL bad_cpu_msg<>(SB), RODATA, $84
69
70 // Define a list of AMD64 microarchitecture level features
71 // https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels
72
73 // SSE3 SSSE3 CMPXCHNG16 SSE4.1 SSE4.2 POPCNT
74 #define V2_FEATURES_CX (1 << 0 | 1 << 9 | 1 << 13 | 1 << 19 | 1 << 20 | 1 << 23)
75 // LAHF/SAHF
76 #define V2_EXT_FEATURES_CX (1 << 0)
77 // FMA MOVBE OSXSAVE AVX F16C
78 #define V3_FEATURES_CX (V2_FEATURES_CX | 1 << 12 | 1 << 22 | 1 << 27 | 1 << 28 | 1 << 29)
79 // ABM (FOR LZNCT)
80 #define V3_EXT_FEATURES_CX (V2_EXT_FEATURES_CX | 1 << 5)
81 // BMI1 AVX2 BMI2
82 #define V3_EXT_FEATURES_BX (1 << 3 | 1 << 5 | 1 << 8)
83 // XMM YMM
84 #define V3_OS_SUPPORT_AX (1 << 1 | 1 << 2)
85
86 #define V4_FEATURES_CX V3_FEATURES_CX
87
88 #define V4_EXT_FEATURES_CX V3_EXT_FEATURES_CX
89 // AVX512F AVX512DQ AVX512CD AVX512BW AVX512VL
90 #define V4_EXT_FEATURES_BX (V3_EXT_FEATURES_BX | 1 << 16 | 1 << 17 | 1 << 28 | 1 << 30 | 1 << 31)
91 // OPMASK ZMM
92 #define V4_OS_SUPPORT_AX (V3_OS_SUPPORT_AX | 1 << 5 | (1 << 6 | 1 << 7))
93
94 #ifdef GOAMD64_v2
95 #define NEED_MAX_CPUID 0x80000001
96 #define NEED_FEATURES_CX V2_FEATURES_CX
97 #define NEED_EXT_FEATURES_CX V2_EXT_FEATURES_CX
98 #endif
99
100 #ifdef GOAMD64_v3
101 #define NEED_MAX_CPUID 0x80000001
102 #define NEED_FEATURES_CX V3_FEATURES_CX
103 #define NEED_EXT_FEATURES_CX V3_EXT_FEATURES_CX
104 #define NEED_EXT_FEATURES_BX V3_EXT_FEATURES_BX
105 #define NEED_OS_SUPPORT_AX V3_OS_SUPPORT_AX
106 #endif
107
108 #ifdef GOAMD64_v4
109 #define NEED_MAX_CPUID 0x80000001
110 #define NEED_FEATURES_CX V4_FEATURES_CX
111 #define NEED_EXT_FEATURES_CX V4_EXT_FEATURES_CX
112 #define NEED_EXT_FEATURES_BX V4_EXT_FEATURES_BX
113
114 // Darwin requires a different approach to check AVX512 support, see CL 285572.
115 #ifdef GOOS_darwin
116 #define NEED_OS_SUPPORT_AX V3_OS_SUPPORT_AX
117 // These values are from:
118 // https://github.com/apple/darwin-xnu/blob/xnu-4570.1.46/osfmk/i386/cpu_capabilities.h
119 #define commpage64_base_address 0x00007fffffe00000
120 #define commpage64_cpu_capabilities64 (commpage64_base_address+0x010)
121 #define commpage64_version (commpage64_base_address+0x01E)
122 #define AVX512F 0x0000004000000000
123 #define AVX512CD 0x0000008000000000
124 #define AVX512DQ 0x0000010000000000
125 #define AVX512BW 0x0000020000000000
126 #define AVX512VL 0x0000100000000000
127 #define NEED_DARWIN_SUPPORT (AVX512F | AVX512DQ | AVX512CD | AVX512BW | AVX512VL)
128 #else
129 #define NEED_OS_SUPPORT_AX V4_OS_SUPPORT_AX
130 #endif
131
132 #endif
133
134 TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0
135 // copy arguments forward on an even stack
136 MOVQ DI, AX // argc
137 MOVQ SI, BX // argv
138 SUBQ $(5*8), SP // 3args 2auto
139 ANDQ $~15, SP
140 MOVQ AX, 24(SP)
141 MOVQ BX, 32(SP)
142
143 // This is typically the entry point for Go programs.
144 // Call stack unwinding must not proceed past this frame.
145 // Set the frame pointer register to 0 so that frame pointer-based unwinders
146 // (which don't use debug info for performance reasons)
147 // won't attempt to unwind past this function.
148 // See go.dev/issue/63630
149 MOVQ $0, BP
150
151 // create istack out of the given (operating system) stack.
152 // _cgo_init may update stackguard.
153 MOVQ $runtime·g0(SB), DI
154 LEAQ (-64*1024)(SP), BX
155 MOVQ BX, g_stackguard0(DI)
156 MOVQ BX, g_stackguard1(DI)
157 MOVQ BX, (g_stack+stack_lo)(DI)
158 MOVQ SP, (g_stack+stack_hi)(DI)
159
160 // find out information about the processor we're on
161 MOVL $0, AX
162 CPUID
163 CMPL AX, $0
164 JE nocpuinfo
165
166 CMPL BX, $0x756E6547 // "Genu"
167 JNE notintel
168 CMPL DX, $0x49656E69 // "ineI"
169 JNE notintel
170 CMPL CX, $0x6C65746E // "ntel"
171 JNE notintel
172 MOVB $1, runtime·isIntel(SB)
173
174 notintel:
175 // Load EAX=1 cpuid flags
176 MOVL $1, AX
177 CPUID
178 MOVL AX, runtime·processorVersionInfo(SB)
179
180 nocpuinfo:
181 // if there is an _cgo_init, call it.
182 MOVQ _cgo_init(SB), AX
183 TESTQ AX, AX
184 JZ needtls
185 // arg 1: g0, already in DI
186 MOVQ $setg_gcc<>(SB), SI // arg 2: setg_gcc
187 MOVQ $0, DX // arg 3, 4: not used when using platform's TLS
188 MOVQ $0, CX
189 #ifdef GOOS_android
190 MOVQ $runtime·tls_g(SB), DX // arg 3: &tls_g
191 // arg 4: TLS base, stored in slot 0 (Android's TLS_SLOT_SELF).
192 // Compensate for tls_g (+16).
193 MOVQ -16(TLS), CX
194 #endif
195 #ifdef GOOS_windows
196 // Adjust for the Win64 calling convention.
197 MOVQ CX, R9 // arg 4
198 MOVQ DX, R8 // arg 3
199 MOVQ SI, DX // arg 2
200 MOVQ DI, CX // arg 1
201 #endif
202 CALL AX
203
204 // update stackguard after _cgo_init
205 MOVQ $runtime·g0(SB), CX
206 MOVQ (g_stack+stack_lo)(CX), AX
207 ADDQ $const_stackGuard, AX
208 MOVQ AX, g_stackguard0(CX)
209 MOVQ AX, g_stackguard1(CX)
210
211 JMP ok
212 needtls:
213 #ifdef GOOS_plan9
214 // skip TLS setup on Plan 9
215 JMP ok
216 #endif
217 #ifdef GOOS_solaris
218 // skip TLS setup on Solaris
219 JMP ok
220 #endif
221 #ifdef GOOS_illumos
222 // skip TLS setup on illumos
223 JMP ok
224 #endif
225 #ifdef GOOS_darwin
226 // skip TLS setup on Darwin
227 JMP ok
228 #endif
229 #ifdef GOOS_openbsd
230 // skip TLS setup on OpenBSD
231 JMP ok
232 #endif
233 #ifdef GOOS_windows
234 // The Windows loader has already set up TLS.
235 JMP ok
236 #else
237 LEAQ runtime·m0+m_tls(SB), DI
238 CALL runtime·settls(SB)
239
240 // store through it, to make sure it works
241 get_tls(BX)
242 MOVQ $0x123, g(BX)
243 MOVQ runtime·m0+m_tls(SB), AX
244 CMPQ AX, $0x123
245 JEQ 2(PC)
246 CALL runtime·abort(SB)
247 #endif
248 ok:
249 // set the per-goroutine and per-mach "registers"
250 get_tls(BX)
251 LEAQ runtime·g0(SB), CX
252 MOVQ CX, g(BX)
253 LEAQ runtime·m0(SB), AX
254
255 // save m->g0 = g0
256 MOVQ CX, m_g0(AX)
257 // save m0 to g0->m
258 MOVQ AX, g_m(CX)
259
260 CLD // convention is D is always left cleared
261
262 // Check GOAMD64 requirements
263 // We need to do this after setting up TLS, so that
264 // we can report an error if there is a failure. See issue 49586.
265 #ifdef NEED_FEATURES_CX
266 MOVL $0, AX
267 CPUID
268 CMPL AX, $0
269 JE bad_cpu
270 MOVL $1, AX
271 CPUID
272 ANDL $NEED_FEATURES_CX, CX
273 CMPL CX, $NEED_FEATURES_CX
274 JNE bad_cpu
275 #endif
276
277 #ifdef NEED_MAX_CPUID
278 MOVL $0x80000000, AX
279 CPUID
280 CMPL AX, $NEED_MAX_CPUID
281 JL bad_cpu
282 #endif
283
284 #ifdef NEED_EXT_FEATURES_BX
285 MOVL $7, AX
286 MOVL $0, CX
287 CPUID
288 ANDL $NEED_EXT_FEATURES_BX, BX
289 CMPL BX, $NEED_EXT_FEATURES_BX
290 JNE bad_cpu
291 #endif
292
293 #ifdef NEED_EXT_FEATURES_CX
294 MOVL $0x80000001, AX
295 CPUID
296 ANDL $NEED_EXT_FEATURES_CX, CX
297 CMPL CX, $NEED_EXT_FEATURES_CX
298 JNE bad_cpu
299 #endif
300
301 #ifdef NEED_OS_SUPPORT_AX
302 XORL CX, CX
303 XGETBV
304 ANDL $NEED_OS_SUPPORT_AX, AX
305 CMPL AX, $NEED_OS_SUPPORT_AX
306 JNE bad_cpu
307 #endif
308
309 #ifdef NEED_DARWIN_SUPPORT
310 MOVQ $commpage64_version, BX
311 CMPW (BX), $13 // cpu_capabilities64 undefined in versions < 13
312 JL bad_cpu
313 MOVQ $commpage64_cpu_capabilities64, BX
314 MOVQ (BX), BX
315 MOVQ $NEED_DARWIN_SUPPORT, CX
316 ANDQ CX, BX
317 CMPQ BX, CX
318 JNE bad_cpu
319 #endif
320
321 CALL runtime·check(SB)
322
323 MOVL 24(SP), AX // copy argc
324 MOVL AX, 0(SP)
325 MOVQ 32(SP), AX // copy argv
326 MOVQ AX, 8(SP)
327 CALL runtime·args(SB)
328 CALL runtime·osinit(SB)
329 CALL runtime·schedinit(SB)
330
331 // create a new goroutine to start program
332 MOVQ $runtime·mainPC(SB), AX // entry
333 PUSHQ AX
334 CALL runtime·newproc(SB)
335 POPQ AX
336
337 // start this M
338 CALL runtime·mstart(SB)
339
340 CALL runtime·abort(SB) // mstart should never return
341 RET
342
343 bad_cpu: // show that the program requires a certain microarchitecture level.
344 MOVQ $2, 0(SP)
345 MOVQ $bad_cpu_msg<>(SB), AX
346 MOVQ AX, 8(SP)
347 MOVQ $84, 16(SP)
348 CALL runtime·write(SB)
349 MOVQ $1, 0(SP)
350 CALL runtime·exit(SB)
351 CALL runtime·abort(SB)
352 RET
353
354 // Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are
355 // intended to be called by debuggers.
356 MOVQ $runtime·debugPinnerV1<ABIInternal>(SB), AX
357 MOVQ $runtime·debugCallV2<ABIInternal>(SB), AX
358 RET
359
360 // mainPC is a function value for runtime.main, to be passed to newproc.
361 // The reference to runtime.main is made via ABIInternal, since the
362 // actual function (not the ABI0 wrapper) is needed by newproc.
363 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
364 GLOBL runtime·mainPC(SB),RODATA,$8
365
366 TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
367 BYTE $0xcc
368 RET
369
370 TEXT runtime·asminit(SB),NOSPLIT,$0-0
371 // No per-thread init.
372 RET
373
374 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME|NOFRAME,$0
375 // This is the root frame of new Go-created OS threads.
376 // Call stack unwinding must not proceed past this frame.
377 // Set the frame pointer register to 0 so that frame pointer-based unwinders
378 // (which don't use debug info for performance reasons)
379 // won't attempt to unwind past this function.
380 // See go.dev/issue/63630
381 MOVD $0, BP
382 CALL runtime·mstart0(SB)
383 RET // not reached
384
385 /*
386 * go-routine
387 */
388
389 // func gogo(buf *gobuf)
390 // restore state from Gobuf; longjmp
391 TEXT runtime·gogo(SB), NOSPLIT, $0-8
392 MOVQ buf+0(FP), BX // gobuf
393 MOVQ gobuf_g(BX), DX
394 MOVQ 0(DX), CX // make sure g != nil
395 JMP gogo<>(SB)
396
397 TEXT gogo<>(SB), NOSPLIT, $0
398 get_tls(CX)
399 MOVQ DX, g(CX)
400 MOVQ DX, R14 // set the g register
401 MOVQ gobuf_sp(BX), SP // restore SP
402 MOVQ gobuf_ctxt(BX), DX
403 MOVQ gobuf_bp(BX), BP
404 MOVQ $0, gobuf_sp(BX) // clear to help garbage collector
405 MOVQ $0, gobuf_ctxt(BX)
406 MOVQ $0, gobuf_bp(BX)
407 MOVQ gobuf_pc(BX), BX
408 JMP BX
409
410 // func mcall(fn func(*g))
411 // Switch to m->g0's stack, call fn(g).
412 // Fn must never return. It should gogo(&g->sched)
413 // to keep running g.
414 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT, $0-8
415 #ifdef GOEXPERIMENT_runtimesecret
416 CMPL g_secret(R14), $0
417 JEQ nosecret
418 CALL ·secretEraseRegistersMcall(SB)
419 nosecret:
420 #endif
421
422 MOVQ AX, DX // DX = fn
423
424 // Save state in g->sched. The caller's SP and PC are restored by gogo to
425 // resume execution in the caller's frame (implicit return). The caller's BP
426 // is also restored to support frame pointer unwinding.
427 MOVQ SP, BX // hide (SP) reads from vet
428 MOVQ 8(BX), BX // caller's PC
429 MOVQ BX, (g_sched+gobuf_pc)(R14)
430 LEAQ fn+0(FP), BX // caller's SP
431 MOVQ BX, (g_sched+gobuf_sp)(R14)
432 // Get the caller's frame pointer by dereferencing BP. Storing BP as it is
433 // can cause a frame pointer cycle, see CL 476235.
434 MOVQ (BP), BX // caller's BP
435 MOVQ BX, (g_sched+gobuf_bp)(R14)
436
437 // switch to m->g0 & its stack, call fn
438 MOVQ g_m(R14), BX
439 MOVQ m_g0(BX), SI // SI = g.m.g0
440 CMPQ SI, R14 // if g == m->g0 call badmcall
441 JNE goodm
442 JMP runtime·badmcall(SB)
443 goodm:
444 MOVQ R14, AX // AX (and arg 0) = g
445 MOVQ SI, R14 // g = g.m.g0
446 get_tls(CX) // Set G in TLS
447 MOVQ R14, g(CX)
448 MOVQ (g_sched+gobuf_sp)(R14), SP // sp = g0.sched.sp
449 MOVQ $0, BP // clear frame pointer, as caller may execute on another M
450 PUSHQ AX // open up space for fn's arg spill slot
451 MOVQ 0(DX), R12
452 CALL R12 // fn(g)
453 // The Windows native stack unwinder incorrectly classifies the next instruction
454 // as part of the function epilogue, producing a wrong call stack.
455 // Add a NOP to work around this issue. See go.dev/issue/67007.
456 BYTE $0x90
457 POPQ AX
458 JMP runtime·badmcall2(SB)
459 RET
460
461 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
462 // of the G stack. We need to distinguish the routine that
463 // lives at the bottom of the G stack from the one that lives
464 // at the top of the system stack because the one at the top of
465 // the system stack terminates the stack walk (see topofstack()).
466 // The frame layout needs to match systemstack
467 // so that it can pretend to be systemstack_switch.
468 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
469 // Align for consistency with offset used in gosave_systemstack_switch
470 PCALIGN $8
471 UNDEF
472 // Make sure this function is not leaf,
473 // so the frame is saved.
474 CALL runtime·abort(SB)
475 RET
476
477 // func systemstack(fn func())
478 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
479 #ifdef GOEXPERIMENT_runtimesecret
480 // If in secret mode, erase registers on transition
481 // from G stack to M stack,
482 get_tls(CX)
483 MOVQ g(CX), AX
484 CMPL g_secret(AX), $0
485 JEQ nosecret
486 CALL ·secretEraseRegisters(SB)
487 nosecret:
488 #endif
489
490 MOVQ fn+0(FP), DI // DI = fn
491 get_tls(CX)
492 MOVQ g(CX), AX // AX = g
493 MOVQ g_m(AX), BX // BX = m
494
495 CMPQ AX, m_gsignal(BX)
496 JEQ noswitch
497
498 MOVQ m_g0(BX), DX // DX = g0
499 CMPQ AX, DX
500 JEQ noswitch
501
502 CMPQ AX, m_curg(BX)
503 JNE bad
504
505 // Switch stacks.
506 // The original frame pointer is stored in BP,
507 // which is useful for stack unwinding.
508 // Save our state in g->sched. Pretend to
509 // be systemstack_switch if the G stack is scanned.
510 CALL gosave_systemstack_switch<>(SB)
511
512 // switch to g0
513 MOVQ DX, g(CX)
514 MOVQ DX, R14 // set the g register
515 MOVQ (g_sched+gobuf_sp)(DX), SP
516
517 // call target function
518 MOVQ DI, DX
519 MOVQ 0(DI), DI
520 CALL DI
521
522 // switch back to g
523 get_tls(CX)
524 MOVQ g(CX), AX
525 MOVQ g_m(AX), BX
526 MOVQ m_curg(BX), AX
527 MOVQ AX, g(CX)
528 MOVQ (g_sched+gobuf_sp)(AX), SP
529 MOVQ (g_sched+gobuf_bp)(AX), BP
530 MOVQ $0, (g_sched+gobuf_sp)(AX)
531 MOVQ $0, (g_sched+gobuf_bp)(AX)
532 RET
533
534 noswitch:
535 // already on m stack; tail call the function
536 // Using a tail call here cleans up tracebacks since we won't stop
537 // at an intermediate systemstack.
538 MOVQ DI, DX
539 MOVQ 0(DI), DI
540 // The function epilogue is not called on a tail call.
541 // Pop BP from the stack to simulate it.
542 POPQ BP
543 JMP DI
544
545 bad:
546 // Bad: g is not gsignal, not g0, not curg. What is it?
547 MOVQ $runtime·badsystemstack(SB), AX
548 CALL AX
549 INT $3
550
551 // func switchToCrashStack0(fn func())
552 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
553 MOVQ g_m(R14), BX // curm
554
555 // set g to gcrash
556 LEAQ runtime·gcrash(SB), R14 // g = &gcrash
557 MOVQ BX, g_m(R14) // g.m = curm
558 MOVQ R14, m_g0(BX) // curm.g0 = g
559 get_tls(CX)
560 MOVQ R14, g(CX)
561
562 // switch to crashstack
563 MOVQ (g_stack+stack_hi)(R14), BX
564 SUBQ $(4*8), BX
565 MOVQ BX, SP
566
567 // call target function
568 MOVQ AX, DX
569 MOVQ 0(AX), AX
570 CALL AX
571
572 // should never return
573 CALL runtime·abort(SB)
574 UNDEF
575
576 /*
577 * support for morestack
578 */
579
580 // Called during function prolog when more stack is needed.
581 //
582 // The traceback routines see morestack on a g0 as being
583 // the top of a stack (for example, morestack calling newstack
584 // calling the scheduler calling newm calling gc), so we must
585 // record an argument size. For that purpose, it has no arguments.
586 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
587 // Cannot grow scheduler stack (m->g0).
588 get_tls(CX)
589 MOVQ g(CX), DI // DI = g
590 MOVQ g_m(DI), BX // BX = m
591
592 // Set g->sched to context in f.
593 MOVQ 0(SP), AX // f's PC
594 MOVQ AX, (g_sched+gobuf_pc)(DI)
595 LEAQ 8(SP), AX // f's SP
596 MOVQ AX, (g_sched+gobuf_sp)(DI)
597 MOVQ BP, (g_sched+gobuf_bp)(DI)
598 MOVQ DX, (g_sched+gobuf_ctxt)(DI)
599
600 MOVQ m_g0(BX), SI // SI = m.g0
601 CMPQ DI, SI
602 JNE 3(PC)
603 CALL runtime·badmorestackg0(SB)
604 CALL runtime·abort(SB)
605
606 // Cannot grow signal stack (m->gsignal).
607 MOVQ m_gsignal(BX), SI
608 CMPQ DI, SI
609 JNE 3(PC)
610 CALL runtime·badmorestackgsignal(SB)
611 CALL runtime·abort(SB)
612
613 // Called from f.
614 // Set m->morebuf to f's caller.
615 NOP SP // tell vet SP changed - stop checking offsets
616 MOVQ 8(SP), AX // f's caller's PC
617 MOVQ AX, (m_morebuf+gobuf_pc)(BX)
618 LEAQ 16(SP), AX // f's caller's SP
619 MOVQ AX, (m_morebuf+gobuf_sp)(BX)
620 MOVQ DI, (m_morebuf+gobuf_g)(BX)
621
622 // If in secret mode, erase registers on transition
623 // from G stack to M stack,
624 #ifdef GOEXPERIMENT_runtimesecret
625 CMPL g_secret(DI), $0
626 JEQ nosecret
627 CALL ·secretEraseRegisters(SB)
628 get_tls(CX)
629 MOVQ g(CX), DI // DI = g
630 MOVQ g_m(DI), BX // BX = m
631 nosecret:
632 #endif
633
634 // Call newstack on m->g0's stack.
635 MOVQ m_g0(BX), BX
636 MOVQ BX, g(CX)
637 MOVQ (g_sched+gobuf_sp)(BX), SP
638 MOVQ $0, BP // clear frame pointer, as caller may execute on another M
639 CALL runtime·newstack(SB)
640 CALL runtime·abort(SB) // crash if newstack returns
641 RET
642
643 // morestack but not preserving ctxt.
644 TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
645 MOVL $0, DX
646 JMP runtime·morestack(SB)
647
648 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R12.
649 TEXT ·spillArgs(SB),NOSPLIT,$0-0
650 MOVQ AX, 0(R12)
651 MOVQ BX, 8(R12)
652 MOVQ CX, 16(R12)
653 MOVQ DI, 24(R12)
654 MOVQ SI, 32(R12)
655 MOVQ R8, 40(R12)
656 MOVQ R9, 48(R12)
657 MOVQ R10, 56(R12)
658 MOVQ R11, 64(R12)
659 MOVQ X0, 72(R12)
660 MOVQ X1, 80(R12)
661 MOVQ X2, 88(R12)
662 MOVQ X3, 96(R12)
663 MOVQ X4, 104(R12)
664 MOVQ X5, 112(R12)
665 MOVQ X6, 120(R12)
666 MOVQ X7, 128(R12)
667 MOVQ X8, 136(R12)
668 MOVQ X9, 144(R12)
669 MOVQ X10, 152(R12)
670 MOVQ X11, 160(R12)
671 MOVQ X12, 168(R12)
672 MOVQ X13, 176(R12)
673 MOVQ X14, 184(R12)
674 RET
675
676 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R12.
677 TEXT ·unspillArgs(SB),NOSPLIT,$0-0
678 MOVQ 0(R12), AX
679 MOVQ 8(R12), BX
680 MOVQ 16(R12), CX
681 MOVQ 24(R12), DI
682 MOVQ 32(R12), SI
683 MOVQ 40(R12), R8
684 MOVQ 48(R12), R9
685 MOVQ 56(R12), R10
686 MOVQ 64(R12), R11
687 MOVQ 72(R12), X0
688 MOVQ 80(R12), X1
689 MOVQ 88(R12), X2
690 MOVQ 96(R12), X3
691 MOVQ 104(R12), X4
692 MOVQ 112(R12), X5
693 MOVQ 120(R12), X6
694 MOVQ 128(R12), X7
695 MOVQ 136(R12), X8
696 MOVQ 144(R12), X9
697 MOVQ 152(R12), X10
698 MOVQ 160(R12), X11
699 MOVQ 168(R12), X12
700 MOVQ 176(R12), X13
701 MOVQ 184(R12), X14
702 RET
703
704 // reflectcall: call a function with the given argument list
705 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
706 // we don't have variable-sized frames, so we use a small number
707 // of constant-sized-frame functions to encode a few bits of size in the pc.
708 // Caution: ugly multiline assembly macros in your future!
709
710 #define DISPATCH(NAME,MAXSIZE) \
711 CMPQ CX, $MAXSIZE; \
712 JA 3(PC); \
713 MOVQ $NAME(SB), AX; \
714 JMP AX
715 // Note: can't just "JMP NAME(SB)" - bad inlining results.
716
717 TEXT ·reflectcall(SB), NOSPLIT, $0-48
718 MOVLQZX frameSize+32(FP), CX
719 DISPATCH(runtime·call16, 16)
720 DISPATCH(runtime·call32, 32)
721 DISPATCH(runtime·call64, 64)
722 DISPATCH(runtime·call128, 128)
723 DISPATCH(runtime·call256, 256)
724 DISPATCH(runtime·call512, 512)
725 DISPATCH(runtime·call1024, 1024)
726 DISPATCH(runtime·call2048, 2048)
727 DISPATCH(runtime·call4096, 4096)
728 DISPATCH(runtime·call8192, 8192)
729 DISPATCH(runtime·call16384, 16384)
730 DISPATCH(runtime·call32768, 32768)
731 DISPATCH(runtime·call65536, 65536)
732 DISPATCH(runtime·call131072, 131072)
733 DISPATCH(runtime·call262144, 262144)
734 DISPATCH(runtime·call524288, 524288)
735 DISPATCH(runtime·call1048576, 1048576)
736 DISPATCH(runtime·call2097152, 2097152)
737 DISPATCH(runtime·call4194304, 4194304)
738 DISPATCH(runtime·call8388608, 8388608)
739 DISPATCH(runtime·call16777216, 16777216)
740 DISPATCH(runtime·call33554432, 33554432)
741 DISPATCH(runtime·call67108864, 67108864)
742 DISPATCH(runtime·call134217728, 134217728)
743 DISPATCH(runtime·call268435456, 268435456)
744 DISPATCH(runtime·call536870912, 536870912)
745 DISPATCH(runtime·call1073741824, 1073741824)
746 MOVQ $runtime·badreflectcall(SB), AX
747 JMP AX
748
749 #define CALLFN(NAME,MAXSIZE) \
750 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
751 NO_LOCAL_POINTERS; \
752 /* copy arguments to stack */ \
753 MOVQ stackArgs+16(FP), SI; \
754 MOVLQZX stackArgsSize+24(FP), CX; \
755 MOVQ SP, DI; \
756 REP;MOVSB; \
757 /* set up argument registers */ \
758 MOVQ regArgs+40(FP), R12; \
759 CALL ·unspillArgs(SB); \
760 /* call function */ \
761 MOVQ f+8(FP), DX; \
762 PCDATA $PCDATA_StackMapIndex, $0; \
763 MOVQ (DX), R12; \
764 CALL R12; \
765 /* copy register return values back */ \
766 MOVQ regArgs+40(FP), R12; \
767 CALL ·spillArgs(SB); \
768 MOVLQZX stackArgsSize+24(FP), CX; \
769 MOVLQZX stackRetOffset+28(FP), BX; \
770 MOVQ stackArgs+16(FP), DI; \
771 MOVQ stackArgsType+0(FP), DX; \
772 MOVQ SP, SI; \
773 ADDQ BX, DI; \
774 ADDQ BX, SI; \
775 SUBQ BX, CX; \
776 CALL callRet<>(SB); \
777 RET
778
779 // callRet copies return values back at the end of call*. This is a
780 // separate function so it can allocate stack space for the arguments
781 // to reflectcallmove. It does not follow the Go ABI; it expects its
782 // arguments in registers.
783 TEXT callRet<>(SB), NOSPLIT, $40-0
784 NO_LOCAL_POINTERS
785 MOVQ DX, 0(SP)
786 MOVQ DI, 8(SP)
787 MOVQ SI, 16(SP)
788 MOVQ CX, 24(SP)
789 MOVQ R12, 32(SP)
790 CALL runtime·reflectcallmove(SB)
791 RET
792
793 CALLFN(·call16, 16)
794 CALLFN(·call32, 32)
795 CALLFN(·call64, 64)
796 CALLFN(·call128, 128)
797 CALLFN(·call256, 256)
798 CALLFN(·call512, 512)
799 CALLFN(·call1024, 1024)
800 CALLFN(·call2048, 2048)
801 CALLFN(·call4096, 4096)
802 CALLFN(·call8192, 8192)
803 CALLFN(·call16384, 16384)
804 CALLFN(·call32768, 32768)
805 CALLFN(·call65536, 65536)
806 CALLFN(·call131072, 131072)
807 CALLFN(·call262144, 262144)
808 CALLFN(·call524288, 524288)
809 CALLFN(·call1048576, 1048576)
810 CALLFN(·call2097152, 2097152)
811 CALLFN(·call4194304, 4194304)
812 CALLFN(·call8388608, 8388608)
813 CALLFN(·call16777216, 16777216)
814 CALLFN(·call33554432, 33554432)
815 CALLFN(·call67108864, 67108864)
816 CALLFN(·call134217728, 134217728)
817 CALLFN(·call268435456, 268435456)
818 CALLFN(·call536870912, 536870912)
819 CALLFN(·call1073741824, 1073741824)
820
821 TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0
822 MOVL cycles+0(FP), AX
823 TESTL AX, AX
824 JZ done
825 again:
826 PAUSE
827 SUBL $1, AX
828 JNZ again
829 done:
830 RET
831
832
833 TEXT ·publicationBarrier<ABIInternal>(SB),NOSPLIT,$0-0
834 // Stores are already ordered on x86, so this is just a
835 // compile barrier.
836 RET
837
838 // Save state of caller into g->sched,
839 // but using fake PC from systemstack_switch.
840 // Must only be called from functions with frame pointer
841 // and without locals ($0) or else unwinding from
842 // systemstack_switch is incorrect.
843 // Smashes R9.
844 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
845 // Take systemstack_switch PC and add 8 bytes to skip
846 // the prologue. Keep 8 bytes offset consistent with
847 // PCALIGN $8 in systemstack_swtich, pointing start of
848 // UNDEF instruction beyond prologue.
849 MOVQ $runtime·systemstack_switch+8(SB), R9
850 MOVQ R9, (g_sched+gobuf_pc)(R14)
851 LEAQ 8(SP), R9
852 MOVQ R9, (g_sched+gobuf_sp)(R14)
853 MOVQ BP, (g_sched+gobuf_bp)(R14)
854 // Assert ctxt is zero. See func save.
855 MOVQ (g_sched+gobuf_ctxt)(R14), R9
856 TESTQ R9, R9
857 JZ 2(PC)
858 CALL runtime·abort(SB)
859 RET
860
861 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
862 // Call fn(arg) aligned appropriately for the gcc ABI.
863 // Called on a system stack, and there may be no g yet (during needm).
864 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$32-16
865 MOVQ fn+0(FP), AX
866 MOVQ arg+8(FP), BX
867 MOVQ SP, DX
868 ANDQ $~15, SP // alignment
869 MOVQ DX, 8(SP)
870 MOVQ BX, DI // DI = first argument in AMD64 ABI
871 MOVQ BX, CX // CX = first argument in Win64
872 CALL AX
873 MOVQ 8(SP), DX
874 MOVQ DX, SP
875 RET
876
877 // asmcgocall_landingpad calls AX with BX as argument.
878 // Must be called on the system stack.
879 TEXT ·asmcgocall_landingpad(SB),NOSPLIT,$0-0
880 #ifdef GOOS_windows
881 // Make sure we have enough room for 4 stack-backed fast-call
882 // registers as per Windows amd64 calling convention.
883 ADJSP $32
884 // On Windows, asmcgocall_landingpad acts as landing pad for exceptions
885 // thrown in the cgo call. Exceptions that reach this function will be
886 // handled by runtime.sehtramp thanks to the SEH metadata added
887 // by the compiler.
888 // Note that runtime.sehtramp can't be attached directly to asmcgocall
889 // because its initial stack pointer can be outside the system stack bounds,
890 // and Windows stops the stack unwinding without calling the exception handler
891 // when it reaches that point.
892 MOVQ BX, CX // CX = first argument in Win64
893 CALL AX
894 // The exception handler is not called if the next instruction is part of
895 // the epilogue, which includes the RET instruction, so we need to add a NOP here.
896 BYTE $0x90
897 ADJSP $-32
898 RET
899 #endif
900 // Tail call AX on non-Windows, as the extra stack frame is not needed.
901 MOVQ BX, DI // DI = first argument in AMD64 ABI
902 JMP AX
903
904 // func asmcgocall(fn, arg unsafe.Pointer) int32
905 // Call fn(arg) on the scheduler stack,
906 // aligned appropriately for the gcc ABI.
907 // See cgocall.go for more details.
908 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
909 // Figure out if we need to switch to m->g0 stack.
910 // We get called to create new OS threads too, and those
911 // come in on the m->g0 stack already. Or we might already
912 // be on the m->gsignal stack.
913 get_tls(CX)
914 MOVQ g(CX), DI
915 CMPQ DI, $0
916 JEQ nosave
917 MOVQ g_m(DI), R8
918 MOVQ m_gsignal(R8), SI
919 CMPQ DI, SI
920 JEQ nosave
921 MOVQ m_g0(R8), SI
922 CMPQ DI, SI
923 JEQ nosave
924
925 // Running on a user G
926 // Figure out if we're running secret code and clear the registers
927 // so that the C code we're about to call doesn't spill confidential
928 // information into memory
929 #ifdef GOEXPERIMENT_runtimesecret
930 CMPL g_secret(DI), $0
931 JEQ nosecret
932 CALL ·secretEraseRegisters(SB)
933
934 get_tls(CX)
935 MOVQ g(CX), DI
936 MOVQ g_m(DI), R8
937 MOVQ m_g0(R8), SI
938
939 nosecret:
940 #endif
941 MOVQ fn+0(FP), AX
942 MOVQ arg+8(FP), BX
943 MOVQ SP, DX
944
945 // Switch to system stack.
946 // The original frame pointer is stored in BP,
947 // which is useful for stack unwinding.
948 CALL gosave_systemstack_switch<>(SB)
949 MOVQ SI, g(CX)
950 MOVQ (g_sched+gobuf_sp)(SI), SP
951
952 // Now on a scheduling stack (a pthread-created stack).
953 SUBQ $16, SP
954 ANDQ $~15, SP // alignment for gcc ABI
955 MOVQ DI, 8(SP) // save g
956 MOVQ (g_stack+stack_hi)(DI), DI
957 SUBQ DX, DI
958 MOVQ DI, 0(SP) // save depth in stack (can't just save SP, as stack might be copied during a callback)
959 CALL runtime·asmcgocall_landingpad(SB)
960
961 // Restore registers, g, stack pointer.
962 get_tls(CX)
963 MOVQ 8(SP), DI
964 MOVQ (g_stack+stack_hi)(DI), SI
965 SUBQ 0(SP), SI
966 MOVQ DI, g(CX)
967 MOVQ SI, SP
968
969 MOVL AX, ret+16(FP)
970 RET
971
972 nosave:
973 // Running on a system stack, perhaps even without a g.
974 // Having no g can happen during thread creation or thread teardown
975 // (see needm/dropm on Solaris, for example).
976 // This code is like the above sequence but without saving/restoring g
977 // and without worrying about the stack moving out from under us
978 // (because we're on a system stack, not a goroutine stack).
979 MOVQ fn+0(FP), AX
980 MOVQ arg+8(FP), BX
981 MOVQ SP, DX
982
983 SUBQ $16, SP
984 ANDQ $~15, SP
985 MOVQ $0, 8(SP) // where above code stores g, in case someone looks during debugging
986 MOVQ DX, 0(SP) // save original stack pointer
987 CALL runtime·asmcgocall_landingpad(SB)
988 MOVQ 0(SP), SI // restore original stack pointer
989 MOVQ SI, SP
990 MOVL AX, ret+16(FP)
991 RET
992 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
993 // See cgocall.go for more details.
994 TEXT ·cgocallback(SB),NOSPLIT,$24-24
995 NO_LOCAL_POINTERS
996
997 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
998 // It is used to dropm while thread is exiting.
999 MOVQ fn+0(FP), AX
1000 CMPQ AX, $0
1001 JNE loadg
1002 // Restore the g from frame.
1003 get_tls(CX)
1004 MOVQ frame+8(FP), BX
1005 MOVQ BX, g(CX)
1006 JMP dropm
1007
1008 loadg:
1009 // If g is nil, Go did not create the current thread,
1010 // or if this thread never called into Go on pthread platforms.
1011 // Call needm to obtain one m for temporary use.
1012 // In this case, we're running on the thread stack, so there's
1013 // lots of space, but the linker doesn't know. Hide the call from
1014 // the linker analysis by using an indirect call through AX.
1015 get_tls(CX)
1016 MOVQ g(CX), BX
1017 CMPQ BX, $0
1018 JEQ needm
1019 MOVQ g_m(BX), BX
1020 MOVQ BX, savedm-8(SP) // saved copy of oldm
1021 JMP havem
1022 needm:
1023 // On some platforms (Windows) we cannot call needm through
1024 // an ABI wrapper because g is nil, and the ABI wrapper will
1025 // try to restore the G register (R14) from TLS.
1026 // Clear X15 because Go expects it and we're not calling
1027 // through a wrapper, but otherwise avoid setting the G
1028 // register in the wrapper and call needm directly. It
1029 // takes no arguments and doesn't return any values so
1030 // there's no need to handle that. Clear R14 so that there's
1031 // a bad value in there, in case needm tries to use it.
1032 XORPS X15, X15
1033 #ifndef GOAMD64_v3
1034 #ifndef GOAMD64_v4
1035 CMPB internal∕cpu·X86+const_offsetX86HasAVX(SB), $1
1036 JNE 2(PC)
1037 #endif
1038 #endif
1039 VXORPS X15, X15, X15
1040 XORQ R14, R14
1041 MOVQ $runtime·needAndBindM<ABIInternal>(SB), AX
1042 CALL AX
1043 MOVQ $0, savedm-8(SP)
1044 get_tls(CX)
1045 MOVQ g(CX), BX
1046 MOVQ g_m(BX), BX
1047
1048 // Set m->sched.sp = SP, so that if a panic happens
1049 // during the function we are about to execute, it will
1050 // have a valid SP to run on the g0 stack.
1051 // The next few lines (after the havem label)
1052 // will save this SP onto the stack and then write
1053 // the same SP back to m->sched.sp. That seems redundant,
1054 // but if an unrecovered panic happens, unwindm will
1055 // restore the g->sched.sp from the stack location
1056 // and then systemstack will try to use it. If we don't set it here,
1057 // that restored SP will be uninitialized (typically 0) and
1058 // will not be usable.
1059 MOVQ m_g0(BX), SI
1060 MOVQ SP, (g_sched+gobuf_sp)(SI)
1061
1062 havem:
1063 // Now there's a valid m, and we're running on its m->g0.
1064 // Save current m->g0->sched.sp on stack and then set it to SP.
1065 // Save current sp in m->g0->sched.sp in preparation for
1066 // switch back to m->curg stack.
1067 // NOTE: unwindm knows that the saved g->sched.sp is at 0(SP).
1068 MOVQ m_g0(BX), SI
1069 MOVQ (g_sched+gobuf_sp)(SI), AX
1070 MOVQ AX, 0(SP)
1071 MOVQ SP, (g_sched+gobuf_sp)(SI)
1072
1073 // Switch to m->curg stack and call runtime.cgocallbackg.
1074 // Because we are taking over the execution of m->curg
1075 // but *not* resuming what had been running, we need to
1076 // save that information (m->curg->sched) so we can restore it.
1077 // We can restore m->curg->sched.sp easily, because calling
1078 // runtime.cgocallbackg leaves SP unchanged upon return.
1079 // To save m->curg->sched.pc, we push it onto the curg stack and
1080 // open a frame the same size as cgocallback's g0 frame.
1081 // Once we switch to the curg stack, the pushed PC will appear
1082 // to be the return PC of cgocallback, so that the traceback
1083 // will seamlessly trace back into the earlier calls.
1084 MOVQ m_curg(BX), SI
1085 MOVQ SI, g(CX)
1086 MOVQ SI, R14 // set the g register, as required by ABIInternal.
1087 XORPS X15, X15 // clear X15, as required by ABIInternal.
1088 MOVQ (g_sched+gobuf_sp)(SI), DI // prepare stack as DI
1089 MOVQ (g_sched+gobuf_pc)(SI), BX
1090 MOVQ BX, -8(DI) // "push" return PC on the g stack
1091 // Gather our arguments into registers.
1092 MOVQ fn+0(FP), AX
1093 MOVQ frame+8(FP), BX
1094 MOVQ ctxt+16(FP), CX
1095 // Compute the size of the frame, including the return PC and
1096 // saved frame pointer
1097 LEAQ fn+0(FP), R8
1098 SUBQ SP, R8 // R8 is our actual frame size
1099 SUBQ R8, DI // Allocate the same frame size on the g stack
1100 MOVQ DI, SP
1101
1102 MOVQ $runtime·cgocallbackg<ABIInternal>(SB), DX
1103 CALL DX // indirect call to bypass nosplit check. We're on a different stack now.
1104
1105 // Compute the size of the frame again. FP and SP have
1106 // completely different values here than they did above,
1107 // but only their difference matters.
1108 LEAQ fn+0(FP), AX
1109 SUBQ SP, AX
1110
1111 // Restore g->sched (== m->curg->sched) from saved values.
1112 get_tls(CX)
1113 MOVQ g(CX), SI
1114 MOVQ SP, DI
1115 ADDQ AX, DI
1116 MOVQ -8(DI), BX
1117 MOVQ BX, (g_sched+gobuf_pc)(SI)
1118 MOVQ DI, (g_sched+gobuf_sp)(SI)
1119
1120 // Switch back to m->g0's stack and restore m->g0->sched.sp.
1121 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
1122 // so we do not have to restore it.)
1123 MOVQ g(CX), BX
1124 MOVQ g_m(BX), BX
1125 MOVQ m_g0(BX), SI
1126 MOVQ SI, g(CX)
1127 MOVQ (g_sched+gobuf_sp)(SI), SP
1128 MOVQ 0(SP), AX
1129 MOVQ AX, (g_sched+gobuf_sp)(SI)
1130
1131 // If the m on entry was nil, we called needm above to borrow an m,
1132 // 1. for the duration of the call on non-pthread platforms,
1133 // 2. or the duration of the C thread alive on pthread platforms.
1134 // If the m on entry wasn't nil,
1135 // 1. the thread might be a Go thread,
1136 // 2. or it wasn't the first call from a C thread on pthread platforms,
1137 // since then we skip dropm to reuse the m in the first call.
1138 MOVQ savedm-8(SP), BX
1139 CMPQ BX, $0
1140 JNE done
1141
1142 // Skip dropm to reuse it in the next call, when a pthread key has been created.
1143 MOVQ _cgo_pthread_key_created(SB), AX
1144 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
1145 CMPQ AX, $0
1146 JEQ dropm
1147 CMPQ (AX), $0
1148 JNE done
1149
1150 dropm:
1151 MOVQ $runtime·dropm(SB), AX
1152 CALL AX
1153 #ifdef GOOS_windows
1154 // Clear g in case the next
1155 // thread that comes into Go tries to reuse that space
1156 // but uses the same M.
1157 get_tls(CX)
1158 MOVQ $0, g(CX)
1159 #endif
1160 done:
1161
1162 // Done!
1163 RET
1164
1165 // func setg(gg *g)
1166 // set g. for use by needm.
1167 TEXT runtime·setg(SB), NOSPLIT, $0-8
1168 MOVQ gg+0(FP), BX
1169 get_tls(CX)
1170 MOVQ BX, g(CX)
1171 RET
1172
1173 // void setg_gcc(G*); set g called from gcc.
1174 TEXT setg_gcc<>(SB),NOSPLIT,$0
1175 get_tls(AX)
1176 MOVQ DI, g(AX)
1177 MOVQ DI, R14 // set the g register
1178 RET
1179
1180 TEXT runtime·abort(SB),NOSPLIT,$0-0
1181 INT $3
1182 loop:
1183 JMP loop
1184
1185 // check that SP is in range [g->stack.lo, g->stack.hi)
1186 TEXT runtime·stackcheck(SB), NOSPLIT|NOFRAME, $0-0
1187 get_tls(CX)
1188 MOVQ g(CX), AX
1189 CMPQ (g_stack+stack_hi)(AX), SP
1190 JHI 2(PC)
1191 CALL runtime·abort(SB)
1192 CMPQ SP, (g_stack+stack_lo)(AX)
1193 JHI 2(PC)
1194 CALL runtime·abort(SB)
1195 RET
1196
1197 // func cputicks() int64
1198 TEXT runtime·cputicks(SB),NOSPLIT,$0-0
1199 CMPB internal∕cpu·X86+const_offsetX86HasRDTSCP(SB), $1
1200 JNE fences
1201 // Instruction stream serializing RDTSCP is supported.
1202 // RDTSCP is supported by Intel Nehalem (2008) and
1203 // AMD K8 Rev. F (2006) and newer.
1204 RDTSCP
1205 done:
1206 SHLQ $32, DX
1207 ADDQ DX, AX
1208 MOVQ AX, ret+0(FP)
1209 RET
1210 fences:
1211 // MFENCE is instruction stream serializing and flushes the
1212 // store buffers on AMD. The serialization semantics of LFENCE on AMD
1213 // are dependent on MSR C001_1029 and CPU generation.
1214 // LFENCE on Intel does wait for all previous instructions to have executed.
1215 // Intel recommends MFENCE;LFENCE in its manuals before RDTSC to have all
1216 // previous instructions executed and all previous loads and stores to globally visible.
1217 // Using MFENCE;LFENCE here aligns the serializing properties without
1218 // runtime detection of CPU manufacturer.
1219 MFENCE
1220 LFENCE
1221 RDTSC
1222 JMP done
1223
1224 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
1225 // Must obey the gcc calling convention.
1226 TEXT _cgo_topofstack(SB),NOSPLIT,$0
1227 get_tls(CX)
1228 MOVQ g(CX), AX
1229 MOVQ g_m(AX), AX
1230 MOVQ m_curg(AX), AX
1231 MOVQ (g_stack+stack_hi)(AX), AX
1232 RET
1233
1234 // The top-most function running on a goroutine
1235 // returns to goexit+PCQuantum.
1236 TEXT runtime·goexit(SB),NOSPLIT|TOPFRAME|NOFRAME,$0-0
1237 BYTE $0x90 // NOP
1238 CALL runtime·goexit1(SB) // does not return
1239 // traceback from goexit1 must hit code range of goexit
1240 BYTE $0x90 // NOP
1241
1242 // This is called from .init_array and follows the platform, not Go, ABI.
1243 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
1244 PUSHQ R15 // The access to global variables below implicitly uses R15, which is callee-save
1245 MOVQ runtime·lastmoduledatap(SB), AX
1246 MOVQ DI, moduledata_next(AX)
1247 MOVQ DI, runtime·lastmoduledatap(SB)
1248 POPQ R15
1249 RET
1250
1251 // Initialize special registers then jump to sigpanic.
1252 // This function is injected from the signal handler for panicking
1253 // signals. It is quite painful to set X15 in the signal context,
1254 // so we do it here.
1255 TEXT ·sigpanic0(SB),NOSPLIT,$0-0
1256 get_tls(R14)
1257 MOVQ g(R14), R14
1258 XORPS X15, X15
1259 #ifndef GOAMD64_v3
1260 #ifndef GOAMD64_v4
1261 CMPB internal∕cpu·X86+const_offsetX86HasAVX(SB), $1
1262 JNE 2(PC)
1263 #endif
1264 #endif
1265 VXORPS X15, X15, X15
1266 JMP ·sigpanic<ABIInternal>(SB)
1267
1268 // gcWriteBarrier informs the GC about heap pointer writes.
1269 //
1270 // gcWriteBarrier returns space in a write barrier buffer which
1271 // should be filled in by the caller.
1272 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
1273 // number of bytes of buffer needed in R11, and returns a pointer
1274 // to the buffer space in R11.
1275 // It clobbers FLAGS. It does not clobber any general-purpose registers,
1276 // but may clobber others (e.g., SSE registers).
1277 // Typical use would be, when doing *(CX+88) = AX
1278 // CMPL $0, runtime.writeBarrier(SB)
1279 // JEQ dowrite
1280 // CALL runtime.gcBatchBarrier2(SB)
1281 // MOVQ AX, (R11)
1282 // MOVQ 88(CX), DX
1283 // MOVQ DX, 8(R11)
1284 // dowrite:
1285 // MOVQ AX, 88(CX)
1286 TEXT gcWriteBarrier<>(SB),NOSPLIT,$112
1287 // Save the registers clobbered by the fast path. This is slightly
1288 // faster than having the caller spill these.
1289 MOVQ R12, 96(SP)
1290 MOVQ R13, 104(SP)
1291 retry:
1292 // TODO: Consider passing g.m.p in as an argument so they can be shared
1293 // across a sequence of write barriers.
1294 MOVQ g_m(R14), R13
1295 MOVQ m_p(R13), R13
1296 // Get current buffer write position.
1297 MOVQ (p_wbBuf+wbBuf_next)(R13), R12 // original next position
1298 ADDQ R11, R12 // new next position
1299 // Is the buffer full?
1300 CMPQ R12, (p_wbBuf+wbBuf_end)(R13)
1301 JA flush
1302 // Commit to the larger buffer.
1303 MOVQ R12, (p_wbBuf+wbBuf_next)(R13)
1304 // Make return value (the original next position)
1305 SUBQ R11, R12
1306 MOVQ R12, R11
1307 // Restore registers.
1308 MOVQ 96(SP), R12
1309 MOVQ 104(SP), R13
1310 RET
1311
1312 flush:
1313 // Save all general purpose registers since these could be
1314 // clobbered by wbBufFlush and were not saved by the caller.
1315 // It is possible for wbBufFlush to clobber other registers
1316 // (e.g., SSE registers), but the compiler takes care of saving
1317 // those in the caller if necessary. This strikes a balance
1318 // with registers that are likely to be used.
1319 //
1320 // We don't have type information for these, but all code under
1321 // here is NOSPLIT, so nothing will observe these.
1322 //
1323 // TODO: We could strike a different balance; e.g., saving X0
1324 // and not saving GP registers that are less likely to be used.
1325 MOVQ DI, 0(SP)
1326 MOVQ AX, 8(SP)
1327 MOVQ BX, 16(SP)
1328 MOVQ CX, 24(SP)
1329 MOVQ DX, 32(SP)
1330 // DI already saved
1331 MOVQ SI, 40(SP)
1332 MOVQ BP, 48(SP)
1333 MOVQ R8, 56(SP)
1334 MOVQ R9, 64(SP)
1335 MOVQ R10, 72(SP)
1336 MOVQ R11, 80(SP)
1337 // R12 already saved
1338 // R13 already saved
1339 // R14 is g
1340 MOVQ R15, 88(SP)
1341
1342 CALL runtime·wbBufFlush(SB)
1343
1344 MOVQ 0(SP), DI
1345 MOVQ 8(SP), AX
1346 MOVQ 16(SP), BX
1347 MOVQ 24(SP), CX
1348 MOVQ 32(SP), DX
1349 MOVQ 40(SP), SI
1350 MOVQ 48(SP), BP
1351 MOVQ 56(SP), R8
1352 MOVQ 64(SP), R9
1353 MOVQ 72(SP), R10
1354 MOVQ 80(SP), R11
1355 MOVQ 88(SP), R15
1356 JMP retry
1357
1358 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1359 MOVL $8, R11
1360 JMP gcWriteBarrier<>(SB)
1361 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1362 MOVL $16, R11
1363 JMP gcWriteBarrier<>(SB)
1364 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1365 MOVL $24, R11
1366 JMP gcWriteBarrier<>(SB)
1367 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1368 MOVL $32, R11
1369 JMP gcWriteBarrier<>(SB)
1370 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1371 MOVL $40, R11
1372 JMP gcWriteBarrier<>(SB)
1373 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1374 MOVL $48, R11
1375 JMP gcWriteBarrier<>(SB)
1376 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1377 MOVL $56, R11
1378 JMP gcWriteBarrier<>(SB)
1379 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1380 MOVL $64, R11
1381 JMP gcWriteBarrier<>(SB)
1382
1383 DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1384 GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1385
1386 // debugCallV2 is the entry point for debugger-injected function
1387 // calls on running goroutines. It informs the runtime that a
1388 // debug call has been injected and creates a call frame for the
1389 // debugger to fill in.
1390 //
1391 // To inject a function call, a debugger should:
1392 // 1. Check that the goroutine is in state _Grunning and that
1393 // there are at least 256 bytes free on the stack.
1394 // 2. Push the current PC on the stack (updating SP).
1395 // 3. Write the desired argument frame size at SP-16 (using the SP
1396 // after step 2).
1397 // 4. Save all machine registers (including flags and XMM registers)
1398 // so they can be restored later by the debugger.
1399 // 5. Set the PC to debugCallV2 and resume execution.
1400 //
1401 // If the goroutine is in state _Grunnable, then it's not generally
1402 // safe to inject a call because it may return out via other runtime
1403 // operations. Instead, the debugger should unwind the stack to find
1404 // the return to non-runtime code, add a temporary breakpoint there,
1405 // and inject the call once that breakpoint is hit.
1406 //
1407 // If the goroutine is in any other state, it's not safe to inject a call.
1408 //
1409 // This function communicates back to the debugger by setting R12 and
1410 // invoking INT3 to raise a breakpoint signal. See the comments in the
1411 // implementation for the protocol the debugger is expected to
1412 // follow. InjectDebugCall in the runtime tests demonstrates this protocol.
1413 //
1414 // The debugger must ensure that any pointers passed to the function
1415 // obey escape analysis requirements. Specifically, it must not pass
1416 // a stack pointer to an escaping argument. debugCallV2 cannot check
1417 // this invariant.
1418 //
1419 // This is ABIInternal because Go code injects its PC directly into new
1420 // goroutine stacks.
1421 TEXT runtime·debugCallV2<ABIInternal>(SB),NOSPLIT,$152-0
1422 // Save all registers that may contain pointers so they can be
1423 // conservatively scanned.
1424 //
1425 // We can't do anything that might clobber any of these
1426 // registers before this.
1427 MOVQ R15, r15-(14*8+8)(SP)
1428 MOVQ R14, r14-(13*8+8)(SP)
1429 MOVQ R13, r13-(12*8+8)(SP)
1430 MOVQ R12, r12-(11*8+8)(SP)
1431 MOVQ R11, r11-(10*8+8)(SP)
1432 MOVQ R10, r10-(9*8+8)(SP)
1433 MOVQ R9, r9-(8*8+8)(SP)
1434 MOVQ R8, r8-(7*8+8)(SP)
1435 MOVQ DI, di-(6*8+8)(SP)
1436 MOVQ SI, si-(5*8+8)(SP)
1437 MOVQ BP, bp-(4*8+8)(SP)
1438 MOVQ BX, bx-(3*8+8)(SP)
1439 MOVQ DX, dx-(2*8+8)(SP)
1440 // Save the frame size before we clobber it. Either of the last
1441 // saves could clobber this depending on whether there's a saved BP.
1442 MOVQ frameSize-24(FP), DX // aka -16(RSP) before prologue
1443 MOVQ CX, cx-(1*8+8)(SP)
1444 MOVQ AX, ax-(0*8+8)(SP)
1445
1446 // Save the argument frame size.
1447 MOVQ DX, frameSize-128(SP)
1448
1449 // Perform a safe-point check.
1450 MOVQ retpc-8(FP), AX // Caller's PC
1451 MOVQ AX, 0(SP)
1452 CALL runtime·debugCallCheck(SB)
1453 MOVQ 8(SP), AX
1454 TESTQ AX, AX
1455 JZ good
1456 // The safety check failed. Put the reason string at the top
1457 // of the stack.
1458 MOVQ AX, 0(SP)
1459 MOVQ 16(SP), AX
1460 MOVQ AX, 8(SP)
1461 // Set R12 to 8 and invoke INT3. The debugger should get the
1462 // reason a call can't be injected from the top of the stack
1463 // and resume execution.
1464 MOVQ $8, R12
1465 BYTE $0xcc
1466 JMP restore
1467
1468 good:
1469 // Registers are saved and it's safe to make a call.
1470 // Open up a call frame, moving the stack if necessary.
1471 //
1472 // Once the frame is allocated, this will set R12 to 0 and
1473 // invoke INT3. The debugger should write the argument
1474 // frame for the call at SP, set up argument registers, push
1475 // the trapping PC on the stack, set the PC to the function to
1476 // call, set RDX to point to the closure (if a closure call),
1477 // and resume execution.
1478 //
1479 // If the function returns, this will set R12 to 1 and invoke
1480 // INT3. The debugger can then inspect any return value saved
1481 // on the stack at SP and in registers and resume execution again.
1482 //
1483 // If the function panics, this will set R12 to 2 and invoke INT3.
1484 // The interface{} value of the panic will be at SP. The debugger
1485 // can inspect the panic value and resume execution again.
1486 #define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1487 CMPQ AX, $MAXSIZE; \
1488 JA 5(PC); \
1489 MOVQ $NAME(SB), AX; \
1490 MOVQ AX, 0(SP); \
1491 CALL runtime·debugCallWrap(SB); \
1492 JMP restore
1493
1494 MOVQ frameSize-128(SP), AX
1495 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1496 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1497 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1498 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1499 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1500 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1501 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1502 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1503 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1504 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1505 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1506 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1507 // The frame size is too large. Report the error.
1508 MOVQ $debugCallFrameTooLarge<>(SB), AX
1509 MOVQ AX, 0(SP)
1510 MOVQ $20, 8(SP) // length of debugCallFrameTooLarge string
1511 MOVQ $8, R12
1512 BYTE $0xcc
1513 JMP restore
1514
1515 restore:
1516 // Calls and failures resume here.
1517 //
1518 // Set R12 to 16 and invoke INT3. The debugger should restore
1519 // all registers except RIP and RSP and resume execution.
1520 MOVQ $16, R12
1521 BYTE $0xcc
1522 // We must not modify flags after this point.
1523
1524 // Restore pointer-containing registers, which may have been
1525 // modified from the debugger's copy by stack copying.
1526 MOVQ ax-(0*8+8)(SP), AX
1527 MOVQ cx-(1*8+8)(SP), CX
1528 MOVQ dx-(2*8+8)(SP), DX
1529 MOVQ bx-(3*8+8)(SP), BX
1530 MOVQ bp-(4*8+8)(SP), BP
1531 MOVQ si-(5*8+8)(SP), SI
1532 MOVQ di-(6*8+8)(SP), DI
1533 MOVQ r8-(7*8+8)(SP), R8
1534 MOVQ r9-(8*8+8)(SP), R9
1535 MOVQ r10-(9*8+8)(SP), R10
1536 MOVQ r11-(10*8+8)(SP), R11
1537 MOVQ r12-(11*8+8)(SP), R12
1538 MOVQ r13-(12*8+8)(SP), R13
1539 MOVQ r14-(13*8+8)(SP), R14
1540 MOVQ r15-(14*8+8)(SP), R15
1541
1542 RET
1543
1544 // runtime.debugCallCheck assumes that functions defined with the
1545 // DEBUG_CALL_FN macro are safe points to inject calls.
1546 #define DEBUG_CALL_FN(NAME,MAXSIZE) \
1547 TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1548 NO_LOCAL_POINTERS; \
1549 MOVQ $0, R12; \
1550 BYTE $0xcc; \
1551 MOVQ $1, R12; \
1552 BYTE $0xcc; \
1553 RET
1554 DEBUG_CALL_FN(debugCall32<>, 32)
1555 DEBUG_CALL_FN(debugCall64<>, 64)
1556 DEBUG_CALL_FN(debugCall128<>, 128)
1557 DEBUG_CALL_FN(debugCall256<>, 256)
1558 DEBUG_CALL_FN(debugCall512<>, 512)
1559 DEBUG_CALL_FN(debugCall1024<>, 1024)
1560 DEBUG_CALL_FN(debugCall2048<>, 2048)
1561 DEBUG_CALL_FN(debugCall4096<>, 4096)
1562 DEBUG_CALL_FN(debugCall8192<>, 8192)
1563 DEBUG_CALL_FN(debugCall16384<>, 16384)
1564 DEBUG_CALL_FN(debugCall32768<>, 32768)
1565 DEBUG_CALL_FN(debugCall65536<>, 65536)
1566
1567 // func debugCallPanicked(val interface{})
1568 TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16
1569 // Copy the panic value to the top of stack.
1570 MOVQ val_type+0(FP), AX
1571 MOVQ AX, 0(SP)
1572 MOVQ val_data+8(FP), AX
1573 MOVQ AX, 8(SP)
1574 MOVQ $2, R12
1575 BYTE $0xcc
1576 RET
1577
1578 TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-0
1579 NO_LOCAL_POINTERS
1580 // Save all 14 int registers that could have an index in them.
1581 // They may be pointers, but if they are they are dead.
1582 MOVQ AX, 16(SP)
1583 MOVQ CX, 24(SP)
1584 MOVQ DX, 32(SP)
1585 MOVQ BX, 40(SP)
1586 // skip SP @ 48(SP)
1587 MOVQ BP, 56(SP)
1588 MOVQ SI, 64(SP)
1589 MOVQ DI, 72(SP)
1590 MOVQ R8, 80(SP)
1591 MOVQ R9, 88(SP)
1592 MOVQ R10, 96(SP)
1593 MOVQ R11, 104(SP)
1594 MOVQ R12, 112(SP)
1595 MOVQ R13, 120(SP)
1596 // skip R14 @ 128(SP) (aka G)
1597 MOVQ R15, 136(SP)
1598
1599 MOVQ SP, AX // hide SP read from vet
1600 MOVQ 152(AX), AX // PC immediately after call to panicBounds
1601 LEAQ 16(SP), BX
1602 CALL runtime·panicBounds64<ABIInternal>(SB)
1603 RET
1604
1605 #ifdef GOOS_android
1606 // Use the free TLS_SLOT_APP slot #2 on Android Q.
1607 // Earlier androids are set up in gcc_android.c.
1608 DATA runtime·tls_g+0(SB)/8, $16
1609 GLOBL runtime·tls_g+0(SB), NOPTR, $8
1610 #endif
1611 #ifdef GOOS_windows
1612 DATA runtime·tls_g+0(SB)/8, $0
1613 GLOBL runtime·tls_g+0(SB), NOPTR, $8
1614 #endif
1615
1616 // The compiler and assembler's -spectre=ret mode rewrites
1617 // all indirect CALL AX / JMP AX instructions to be
1618 // CALL retpolineAX / JMP retpolineAX.
1619 // See https://support.google.com/faqs/answer/7625886.
1620 #define RETPOLINE(reg) \
1621 /* CALL setup */ BYTE $0xE8; BYTE $(2+2); BYTE $0; BYTE $0; BYTE $0; \
1622 /* nospec: */ \
1623 /* PAUSE */ BYTE $0xF3; BYTE $0x90; \
1624 /* JMP nospec */ BYTE $0xEB; BYTE $-(2+2); \
1625 /* setup: */ \
1626 /* MOVQ AX, 0(SP) */ BYTE $0x48|((reg&8)>>1); BYTE $0x89; \
1627 BYTE $0x04|((reg&7)<<3); BYTE $0x24; \
1628 /* RET */ BYTE $0xC3
1629
1630 TEXT runtime·retpolineAX(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(0)
1631 TEXT runtime·retpolineCX(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(1)
1632 TEXT runtime·retpolineDX(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(2)
1633 TEXT runtime·retpolineBX(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(3)
1634 /* SP is 4, can't happen / magic encodings */
1635 TEXT runtime·retpolineBP(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(5)
1636 TEXT runtime·retpolineSI(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(6)
1637 TEXT runtime·retpolineDI(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(7)
1638 TEXT runtime·retpolineR8(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(8)
1639 TEXT runtime·retpolineR9(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(9)
1640 TEXT runtime·retpolineR10(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(10)
1641 TEXT runtime·retpolineR11(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(11)
1642 TEXT runtime·retpolineR12(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(12)
1643 TEXT runtime·retpolineR13(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(13)
1644 TEXT runtime·retpolineR14(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(14)
1645 TEXT runtime·retpolineR15(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(15)
1646
1647 TEXT ·getfp<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1648 MOVQ BP, AX
1649 RET
1650
View as plain text