Text file
src/runtime/asm_arm64.s
1 // Copyright 2015 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "tls_arm64.h"
8 #include "funcdata.h"
9 #include "textflag.h"
10 #include "cgo/abi_arm64.h"
11
12 // _rt0_arm64 is common startup code for most arm64 systems when using
13 // internal linking. This is the entry point for the program from the
14 // kernel for an ordinary -buildmode=exe program. The stack holds the
15 // number of arguments and the C-style argv.
16 TEXT _rt0_arm64(SB),NOSPLIT,$0
17 MOVD 0(RSP), R0 // argc
18 ADD $8, RSP, R1 // argv
19 JMP runtime·rt0_go(SB)
20
21 // main is common startup code for most amd64 systems when using
22 // external linking. The C startup code will call the symbol "main"
23 // passing argc and argv in the usual C ABI registers R0 and R1.
24 TEXT main(SB),NOSPLIT,$0
25 JMP runtime·rt0_go(SB)
26
27 // _rt0_arm64_lib is common startup code for most arm64 systems when
28 // using -buildmode=c-archive or -buildmode=c-shared. The linker will
29 // arrange to invoke this function as a global constructor (for
30 // c-archive) or when the shared library is loaded (for c-shared).
31 // We expect argc and argv to be passed in the usual C ABI registers
32 // R0 and R1.
33 TEXT _rt0_arm64_lib(SB),NOSPLIT,$184
34 // Preserve callee-save registers.
35 SAVE_R19_TO_R28(24)
36 SAVE_F8_TO_F15(104)
37
38 // Initialize g as null in case of using g later e.g. sigaction in cgo_sigaction.go
39 MOVD ZR, g
40
41 MOVD R0, _rt0_arm64_lib_argc<>(SB)
42 MOVD R1, _rt0_arm64_lib_argv<>(SB)
43
44 MOVD $runtime·libInit(SB), R4
45 BL (R4)
46
47 // Restore callee-save registers.
48 RESTORE_R19_TO_R28(24)
49 RESTORE_F8_TO_F15(104)
50 RET
51
52 TEXT runtime·rt0_lib_go<ABIInternal>(SB),NOSPLIT,$0
53 MOVD _rt0_arm64_lib_argc<>(SB), R0
54 MOVD _rt0_arm64_lib_argv<>(SB), R1
55 MOVD $runtime·rt0_go(SB),R4
56 B (R4)
57
58 DATA _rt0_arm64_lib_argc<>(SB)/8, $0
59 GLOBL _rt0_arm64_lib_argc<>(SB),NOPTR, $8
60 DATA _rt0_arm64_lib_argv<>(SB)/8, $0
61 GLOBL _rt0_arm64_lib_argv<>(SB),NOPTR, $8
62
63 #ifdef GOARM64_LSE
64 DATA no_lse_msg<>+0x00(SB)/64, $"This program can only run on ARM64 processors with LSE support.\n"
65 GLOBL no_lse_msg<>(SB), RODATA, $64
66 #endif
67
68 // We know for sure that Linux and FreeBSD allow to read instruction set
69 // attribute registers (while some others OSes, like OpenBSD and Darwin,
70 // are not). Let's be conservative and allow code reading such registers
71 // only when we sure this won't lead to sigill.
72 #ifdef GOOS_linux
73 #define ISA_REGS_READABLE
74 #endif
75 #ifdef GOOS_freebsd
76 #define ISA_REGS_READABLE
77 #endif
78
79 #ifdef GOARM64_LSE
80 #ifdef ISA_REGS_READABLE
81 #define CHECK_GOARM64_LSE
82 #endif
83 #endif
84
85 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
86 // SP = stack; R0 = argc; R1 = argv
87
88 SUB $32, RSP
89 MOVW R0, 8(RSP) // argc
90 MOVD R1, 16(RSP) // argv
91
92 // This is typically the entry point for Go programs.
93 // Call stack unwinding must not proceed past this frame.
94 // Set the frame pointer register to 0 so that frame pointer-based unwinders
95 // (which don't use debug info for performance reasons)
96 // won't attempt to unwind past this function.
97 // See go.dev/issue/63630
98 MOVD $0, R29
99
100 #ifdef TLS_darwin
101 // Initialize TLS.
102 MOVD ZR, g // clear g, make sure it's not junk.
103 SUB $32, RSP
104 MRS_TPIDR_R0
105 AND $~7, R0
106 MOVD R0, 16(RSP) // arg2: TLS base
107 MOVD $runtime·tls_g(SB), R2
108 MOVD R2, 8(RSP) // arg1: &tlsg
109 BL ·tlsinit(SB)
110 ADD $32, RSP
111 #endif
112
113 // create istack out of the given (operating system) stack.
114 // _cgo_init may update stackguard.
115 MOVD $runtime·g0(SB), g
116 MOVD RSP, R7
117 MOVD $(-64*1024)(R7), R0
118 MOVD R0, g_stackguard0(g)
119 MOVD R0, g_stackguard1(g)
120 MOVD R0, (g_stack+stack_lo)(g)
121 MOVD R7, (g_stack+stack_hi)(g)
122
123 // if there is a _cgo_init, call it using the gcc ABI.
124 MOVD _cgo_init(SB), R12
125 CBZ R12, nocgo
126
127 #ifdef GOOS_android
128 MRS_TPIDR_R0 // load TLS base pointer
129 MOVD R0, R3 // arg 3: TLS base pointer
130 MOVD $runtime·tls_g(SB), R2 // arg 2: &tls_g
131 #else
132 MOVD $0, R2 // arg 2: not used when using platform's TLS
133 #endif
134 MOVD $setg_gcc<>(SB), R1 // arg 1: setg
135 MOVD g, R0 // arg 0: G
136 SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved.
137 BL (R12)
138 ADD $16, RSP
139
140 nocgo:
141 BL runtime·save_g(SB)
142 // update stackguard after _cgo_init
143 MOVD (g_stack+stack_lo)(g), R0
144 ADD $const_stackGuard, R0
145 MOVD R0, g_stackguard0(g)
146 MOVD R0, g_stackguard1(g)
147
148 // set the per-goroutine and per-mach "registers"
149 MOVD $runtime·m0(SB), R0
150
151 // save m->g0 = g0
152 MOVD g, m_g0(R0)
153 // save m0 to g0->m
154 MOVD R0, g_m(g)
155
156 BL runtime·check(SB)
157
158 // Check that CPU we use for execution supports instructions targeted during compile-time.
159 #ifdef CHECK_GOARM64_LSE
160 // Read the ID_AA64ISAR0_EL1 register
161 MRS ID_AA64ISAR0_EL1, R0
162
163 // Extract the LSE field (bits [23:20])
164 LSR $20, R0, R0
165 AND $0xf, R0, R0
166
167 // LSE support is indicated by a non-zero value
168 CBZ R0, no_lse
169 #endif
170
171 MOVW 8(RSP), R0 // copy argc
172 MOVW R0, -8(RSP)
173 MOVD 16(RSP), R0 // copy argv
174 MOVD R0, 0(RSP)
175 BL runtime·args(SB)
176 BL runtime·osinit(SB)
177 BL runtime·schedinit(SB)
178
179 // create a new goroutine to start program
180 MOVD $runtime·mainPC(SB), R0 // entry
181 SUB $16, RSP
182 MOVD R0, 8(RSP) // arg
183 MOVD $0, 0(RSP) // dummy LR
184 BL runtime·newproc(SB)
185 ADD $16, RSP
186
187 // start this M
188 BL runtime·mstart(SB)
189 UNDEF
190
191 #ifdef CHECK_GOARM64_LSE
192 no_lse:
193 MOVD $1, R0 // stderr
194 MOVD R0, 8(RSP)
195 MOVD $no_lse_msg<>(SB), R1 // message address
196 MOVD R1, 16(RSP)
197 MOVD $64, R2 // message length
198 MOVD R2, 24(RSP)
199 CALL runtime·write(SB)
200 CALL runtime·exit(SB)
201 CALL runtime·abort(SB)
202 RET
203 #endif
204
205 // Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are
206 // intended to be called by debuggers.
207 MOVD $runtime·debugPinnerV1<ABIInternal>(SB), R0
208 MOVD $runtime·debugCallV2<ABIInternal>(SB), R0
209
210 MOVD $0, R0
211 MOVD R0, (R0) // boom
212 UNDEF
213
214 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
215 GLOBL runtime·mainPC(SB),RODATA,$8
216
217 // Windows ARM64 needs an immediate 0xf000 argument.
218 // See go.dev/issues/53837.
219 #define BREAK \
220 #ifdef GOOS_windows \
221 BRK $0xf000 \
222 #else \
223 BRK \
224 #endif \
225
226
227 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
228 BREAK
229 RET
230
231 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
232 RET
233
234 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
235 // This is the root frame of new Go-created OS threads.
236 // Call stack unwinding must not proceed past this frame.
237 // Set the frame pointer register to 0 so that frame pointer-based unwinders
238 // (which don't use debug info for performance reasons)
239 // won't attempt to unwind past this function.
240 // See go.dev/issue/63630
241 MOVD $0, R29
242 BL runtime·mstart0(SB)
243 RET // not reached
244
245 /*
246 * go-routine
247 */
248
249 // void gogo(Gobuf*)
250 // restore state from Gobuf; longjmp
251 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
252 MOVD buf+0(FP), R5
253 MOVD gobuf_g(R5), R6
254 MOVD 0(R6), R4 // make sure g != nil
255 B gogo<>(SB)
256
257 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
258 MOVD R6, g
259 BL runtime·save_g(SB)
260
261 MOVD gobuf_sp(R5), R0
262 MOVD R0, RSP
263 MOVD gobuf_bp(R5), R29
264 MOVD gobuf_lr(R5), LR
265 MOVD gobuf_ctxt(R5), R26
266 MOVD $0, gobuf_sp(R5)
267 MOVD $0, gobuf_bp(R5)
268 MOVD $0, gobuf_lr(R5)
269 MOVD $0, gobuf_ctxt(R5)
270 CMP ZR, ZR // set condition codes for == test, needed by stack split
271 MOVD gobuf_pc(R5), R6
272 B (R6)
273
274 // void mcall(fn func(*g))
275 // Switch to m->g0's stack, call fn(g).
276 // Fn must never return. It should gogo(&g->sched)
277 // to keep running g.
278 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
279 #ifdef GOEXPERIMENT_runtimesecret
280 MOVW g_secret(g), R26
281 CBZ R26, nosecret
282 // Use R26 as a secondary link register
283 // We purposefully don't erase it in secretEraseRegistersMcall
284 MOVD LR, R26
285 BL runtime·secretEraseRegistersMcall(SB)
286 MOVD R26, LR
287
288 nosecret:
289 #endif
290 MOVD R0, R26 // context
291
292 // Save caller state in g->sched
293 MOVD RSP, R0
294 MOVD R0, (g_sched+gobuf_sp)(g)
295 MOVD R29, (g_sched+gobuf_bp)(g)
296 MOVD LR, (g_sched+gobuf_pc)(g)
297 MOVD $0, (g_sched+gobuf_lr)(g)
298
299 // Switch to m->g0 & its stack, call fn.
300 MOVD g, R3
301 MOVD g_m(g), R8
302 MOVD m_g0(R8), g
303 BL runtime·save_g(SB)
304 CMP g, R3
305 BNE 2(PC)
306 B runtime·badmcall(SB)
307
308 MOVD (g_sched+gobuf_sp)(g), R0
309 MOVD R0, RSP // sp = m->g0->sched.sp
310 MOVD $0, R29 // clear frame pointer, as caller may execute on another M
311 MOVD R3, R0 // arg = g
312 MOVD $0, -16(RSP) // dummy LR
313 SUB $16, RSP
314 MOVD 0(R26), R4 // code pointer
315 BL (R4)
316 B runtime·badmcall2(SB)
317
318 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
319 // of the G stack. We need to distinguish the routine that
320 // lives at the bottom of the G stack from the one that lives
321 // at the top of the system stack because the one at the top of
322 // the system stack terminates the stack walk (see topofstack()).
323 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
324 UNDEF
325 BL (LR) // make sure this function is not leaf
326 RET
327
328 // func systemstack(fn func())
329 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
330 #ifdef GOEXPERIMENT_runtimesecret
331 MOVW g_secret(g), R3
332 CBZ R3, nosecret
333 BL ·secretEraseRegisters(SB)
334 nosecret:
335 #endif
336 MOVD fn+0(FP), R3 // R3 = fn
337 MOVD R3, R26 // context
338 MOVD g_m(g), R4 // R4 = m
339
340 MOVD m_gsignal(R4), R5 // R5 = gsignal
341 CMP g, R5
342 BEQ noswitch
343
344 MOVD m_g0(R4), R5 // R5 = g0
345 CMP g, R5
346 BEQ noswitch
347
348 MOVD m_curg(R4), R6
349 CMP g, R6
350 BEQ switch
351
352 // Bad: g is not gsignal, not g0, not curg. What is it?
353 // Hide call from linker nosplit analysis.
354 MOVD $runtime·badsystemstack(SB), R3
355 BL (R3)
356 B runtime·abort(SB)
357
358 switch:
359 // Switch stacks.
360 // The original frame pointer is stored in R29,
361 // which is useful for stack unwinding.
362 // Save our state in g->sched. Pretend to
363 // be systemstack_switch if the G stack is scanned.
364 BL gosave_systemstack_switch<>(SB)
365
366 // switch to g0
367 MOVD R5, g
368 BL runtime·save_g(SB)
369 MOVD (g_sched+gobuf_sp)(g), R3
370 MOVD R3, RSP
371
372 // call target function
373 MOVD 0(R26), R3 // code pointer
374 BL (R3)
375
376 // switch back to g
377 MOVD g_m(g), R3
378 MOVD m_curg(R3), g
379 BL runtime·save_g(SB)
380 MOVD (g_sched+gobuf_sp)(g), R0
381 MOVD R0, RSP
382 MOVD (g_sched+gobuf_bp)(g), R29
383 MOVD $0, (g_sched+gobuf_sp)(g)
384 MOVD $0, (g_sched+gobuf_bp)(g)
385 RET
386
387 noswitch:
388 // already on m stack, just call directly
389 // Using a tail call here cleans up tracebacks since we won't stop
390 // at an intermediate systemstack.
391 MOVD 0(R26), R3 // code pointer
392 MOVD.P 16(RSP), R30 // restore LR
393 SUB $8, RSP, R29 // restore FP
394 B (R3)
395
396 // func switchToCrashStack0(fn func())
397 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
398 MOVD R0, R26 // context register
399 MOVD g_m(g), R1 // curm
400
401 // set g to gcrash
402 MOVD $runtime·gcrash(SB), g // g = &gcrash
403 BL runtime·save_g(SB) // clobbers R0
404 MOVD R1, g_m(g) // g.m = curm
405 MOVD g, m_g0(R1) // curm.g0 = g
406
407 // switch to crashstack
408 MOVD (g_stack+stack_hi)(g), R1
409 SUB $(4*8), R1
410 MOVD R1, RSP
411
412 // call target function
413 MOVD 0(R26), R0
414 CALL (R0)
415
416 // should never return
417 CALL runtime·abort(SB)
418 UNDEF
419
420 /*
421 * support for morestack
422 */
423
424 // Called during function prolog when more stack is needed.
425 // Caller has already loaded:
426 // R3 prolog's LR (R30)
427 //
428 // The traceback routines see morestack on a g0 as being
429 // the top of a stack (for example, morestack calling newstack
430 // calling the scheduler calling newm calling gc), so we must
431 // record an argument size. For that purpose, it has no arguments.
432 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
433 // Cannot grow scheduler stack (m->g0).
434 MOVD g_m(g), R8
435 MOVD m_g0(R8), R4
436
437 // Called from f.
438 // Set g->sched to context in f
439 MOVD RSP, R0
440 MOVD R0, (g_sched+gobuf_sp)(g)
441 MOVD R29, (g_sched+gobuf_bp)(g)
442 MOVD LR, (g_sched+gobuf_pc)(g)
443 MOVD R3, (g_sched+gobuf_lr)(g)
444 MOVD R26, (g_sched+gobuf_ctxt)(g)
445
446 CMP g, R4
447 BNE 3(PC)
448 BL runtime·badmorestackg0(SB)
449 B runtime·abort(SB)
450
451 // Cannot grow signal stack (m->gsignal).
452 MOVD m_gsignal(R8), R4
453 CMP g, R4
454 BNE 3(PC)
455 BL runtime·badmorestackgsignal(SB)
456 B runtime·abort(SB)
457
458 // Called from f.
459 // Set m->morebuf to f's callers.
460 MOVD R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC
461 MOVD RSP, R0
462 MOVD R0, (m_morebuf+gobuf_sp)(R8) // f's caller's RSP
463 MOVD g, (m_morebuf+gobuf_g)(R8)
464
465 // If in secret mode, erase registers on transition
466 // from G stack to M stack,
467 #ifdef GOEXPERIMENT_runtimesecret
468 MOVW g_secret(g), R4
469 CBZ R4, nosecret
470 BL ·secretEraseRegisters(SB)
471 MOVD g_m(g), R8
472 nosecret:
473 #endif
474
475 // Call newstack on m->g0's stack.
476 MOVD m_g0(R8), g
477 BL runtime·save_g(SB)
478 MOVD (g_sched+gobuf_sp)(g), R0
479 MOVD R0, RSP
480 MOVD $0, R29 // clear frame pointer, as caller may execute on another M
481 MOVD.W $0, -16(RSP) // create a call frame on g0 (saved LR; keep 16-aligned)
482 BL runtime·newstack(SB)
483
484 // Not reached, but make sure the return PC from the call to newstack
485 // is still in this function, and not the beginning of the next.
486 UNDEF
487
488 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
489 // Force SPWRITE. This function doesn't actually write SP,
490 // but it is called with a special calling convention where
491 // the caller doesn't save LR on stack but passes it as a
492 // register (R3), and the unwinder currently doesn't understand.
493 // Make it SPWRITE to stop unwinding. (See issue 54332)
494 MOVD RSP, RSP
495
496 MOVW $0, R26
497 B runtime·morestack(SB)
498
499 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
500 TEXT ·spillArgs(SB),NOSPLIT,$0-0
501 STP (R0, R1), (0*8)(R20)
502 STP (R2, R3), (2*8)(R20)
503 STP (R4, R5), (4*8)(R20)
504 STP (R6, R7), (6*8)(R20)
505 STP (R8, R9), (8*8)(R20)
506 STP (R10, R11), (10*8)(R20)
507 STP (R12, R13), (12*8)(R20)
508 STP (R14, R15), (14*8)(R20)
509 FSTPD (F0, F1), (16*8)(R20)
510 FSTPD (F2, F3), (18*8)(R20)
511 FSTPD (F4, F5), (20*8)(R20)
512 FSTPD (F6, F7), (22*8)(R20)
513 FSTPD (F8, F9), (24*8)(R20)
514 FSTPD (F10, F11), (26*8)(R20)
515 FSTPD (F12, F13), (28*8)(R20)
516 FSTPD (F14, F15), (30*8)(R20)
517 RET
518
519 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
520 TEXT ·unspillArgs(SB),NOSPLIT,$0-0
521 LDP (0*8)(R20), (R0, R1)
522 LDP (2*8)(R20), (R2, R3)
523 LDP (4*8)(R20), (R4, R5)
524 LDP (6*8)(R20), (R6, R7)
525 LDP (8*8)(R20), (R8, R9)
526 LDP (10*8)(R20), (R10, R11)
527 LDP (12*8)(R20), (R12, R13)
528 LDP (14*8)(R20), (R14, R15)
529 FLDPD (16*8)(R20), (F0, F1)
530 FLDPD (18*8)(R20), (F2, F3)
531 FLDPD (20*8)(R20), (F4, F5)
532 FLDPD (22*8)(R20), (F6, F7)
533 FLDPD (24*8)(R20), (F8, F9)
534 FLDPD (26*8)(R20), (F10, F11)
535 FLDPD (28*8)(R20), (F12, F13)
536 FLDPD (30*8)(R20), (F14, F15)
537 RET
538
539 // reflectcall: call a function with the given argument list
540 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
541 // we don't have variable-sized frames, so we use a small number
542 // of constant-sized-frame functions to encode a few bits of size in the pc.
543 // Caution: ugly multiline assembly macros in your future!
544
545 #define DISPATCH(NAME,MAXSIZE) \
546 MOVD $MAXSIZE, R27; \
547 CMP R27, R16; \
548 BGT 3(PC); \
549 MOVD $NAME(SB), R27; \
550 B (R27)
551 // Note: can't just "B NAME(SB)" - bad inlining results.
552
553 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
554 MOVWU frameSize+32(FP), R16
555 DISPATCH(runtime·call16, 16)
556 DISPATCH(runtime·call32, 32)
557 DISPATCH(runtime·call64, 64)
558 DISPATCH(runtime·call128, 128)
559 DISPATCH(runtime·call256, 256)
560 DISPATCH(runtime·call512, 512)
561 DISPATCH(runtime·call1024, 1024)
562 DISPATCH(runtime·call2048, 2048)
563 DISPATCH(runtime·call4096, 4096)
564 DISPATCH(runtime·call8192, 8192)
565 DISPATCH(runtime·call16384, 16384)
566 DISPATCH(runtime·call32768, 32768)
567 DISPATCH(runtime·call65536, 65536)
568 DISPATCH(runtime·call131072, 131072)
569 DISPATCH(runtime·call262144, 262144)
570 DISPATCH(runtime·call524288, 524288)
571 DISPATCH(runtime·call1048576, 1048576)
572 DISPATCH(runtime·call2097152, 2097152)
573 DISPATCH(runtime·call4194304, 4194304)
574 DISPATCH(runtime·call8388608, 8388608)
575 DISPATCH(runtime·call16777216, 16777216)
576 DISPATCH(runtime·call33554432, 33554432)
577 DISPATCH(runtime·call67108864, 67108864)
578 DISPATCH(runtime·call134217728, 134217728)
579 DISPATCH(runtime·call268435456, 268435456)
580 DISPATCH(runtime·call536870912, 536870912)
581 DISPATCH(runtime·call1073741824, 1073741824)
582 MOVD $runtime·badreflectcall(SB), R0
583 B (R0)
584
585 #define CALLFN(NAME,MAXSIZE) \
586 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
587 NO_LOCAL_POINTERS; \
588 /* copy arguments to stack */ \
589 MOVD stackArgs+16(FP), R3; \
590 MOVWU stackArgsSize+24(FP), R4; \
591 ADD $8, RSP, R5; \
592 BIC $0xf, R4, R6; \
593 CBZ R6, 6(PC); \
594 /* if R6=(argsize&~15) != 0 */ \
595 ADD R6, R5, R6; \
596 /* copy 16 bytes a time */ \
597 LDP.P 16(R3), (R7, R8); \
598 STP.P (R7, R8), 16(R5); \
599 CMP R5, R6; \
600 BNE -3(PC); \
601 AND $0xf, R4, R6; \
602 CBZ R6, 6(PC); \
603 /* if R6=(argsize&15) != 0 */ \
604 ADD R6, R5, R6; \
605 /* copy 1 byte a time for the rest */ \
606 MOVBU.P 1(R3), R7; \
607 MOVBU.P R7, 1(R5); \
608 CMP R5, R6; \
609 BNE -3(PC); \
610 /* set up argument registers */ \
611 MOVD regArgs+40(FP), R20; \
612 CALL ·unspillArgs(SB); \
613 /* call function */ \
614 MOVD f+8(FP), R26; \
615 MOVD (R26), R20; \
616 PCDATA $PCDATA_StackMapIndex, $0; \
617 BL (R20); \
618 /* copy return values back */ \
619 MOVD regArgs+40(FP), R20; \
620 CALL ·spillArgs(SB); \
621 MOVD stackArgsType+0(FP), R7; \
622 MOVD stackArgs+16(FP), R3; \
623 MOVWU stackArgsSize+24(FP), R4; \
624 MOVWU stackRetOffset+28(FP), R6; \
625 ADD $8, RSP, R5; \
626 ADD R6, R5; \
627 ADD R6, R3; \
628 SUB R6, R4; \
629 BL callRet<>(SB); \
630 RET
631
632 // callRet copies return values back at the end of call*. This is a
633 // separate function so it can allocate stack space for the arguments
634 // to reflectcallmove. It does not follow the Go ABI; it expects its
635 // arguments in registers.
636 TEXT callRet<>(SB), NOSPLIT, $48-0
637 NO_LOCAL_POINTERS
638 STP (R7, R3), 8(RSP)
639 STP (R5, R4), 24(RSP)
640 MOVD R20, 40(RSP)
641 BL runtime·reflectcallmove(SB)
642 RET
643
644 CALLFN(·call16, 16)
645 CALLFN(·call32, 32)
646 CALLFN(·call64, 64)
647 CALLFN(·call128, 128)
648 CALLFN(·call256, 256)
649 CALLFN(·call512, 512)
650 CALLFN(·call1024, 1024)
651 CALLFN(·call2048, 2048)
652 CALLFN(·call4096, 4096)
653 CALLFN(·call8192, 8192)
654 CALLFN(·call16384, 16384)
655 CALLFN(·call32768, 32768)
656 CALLFN(·call65536, 65536)
657 CALLFN(·call131072, 131072)
658 CALLFN(·call262144, 262144)
659 CALLFN(·call524288, 524288)
660 CALLFN(·call1048576, 1048576)
661 CALLFN(·call2097152, 2097152)
662 CALLFN(·call4194304, 4194304)
663 CALLFN(·call8388608, 8388608)
664 CALLFN(·call16777216, 16777216)
665 CALLFN(·call33554432, 33554432)
666 CALLFN(·call67108864, 67108864)
667 CALLFN(·call134217728, 134217728)
668 CALLFN(·call268435456, 268435456)
669 CALLFN(·call536870912, 536870912)
670 CALLFN(·call1073741824, 1073741824)
671
672 // The Arm architecture provides a user space accessible counter-timer which
673 // is incremented at a fixed but machine-specific rate. Software can (spin)
674 // wait until the counter-timer reaches some desired value.
675 //
676 // Armv8.7-A introduced the WFET (FEAT_WFxT) instruction, which allows the
677 // processor to enter a low power state for a set time, or until an event is
678 // received.
679 //
680 // However, WFET is not used here because it is only available on newer hardware,
681 // and we aim to maintain compatibility with older Armv8-A platforms that do not
682 // support this feature.
683 //
684 // As a fallback, we can instead use the ISB instruction to decrease processor
685 // activity and thus power consumption between checks of the counter-timer.
686 // Note that we do not depend on the latency of the ISB instruction which is
687 // implementation specific. Actual delay comes from comparing against a fresh
688 // read of the counter-timer value.
689 //
690 // Read more in this Arm blog post:
691 // https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/multi-threaded-applications-arm
692
693 TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0
694 MOVWU cycles+0(FP), R0
695 CBZ R0, done
696 //Prevent speculation of subsequent counter/timer reads and memory accesses.
697 ISB $15
698 // If the delay is very short, just return.
699 // Hardcode 18ns as the first ISB delay.
700 CMP $18, R0
701 BLS done
702 // Adjust for overhead of initial ISB.
703 SUB $18, R0, R0
704 // Convert the delay from nanoseconds to counter/timer ticks.
705 // Read the counter/timer frequency.
706 // delay_ticks = (delay * CNTFRQ_EL0) / 1e9
707 // With the below simplifications and adjustments,
708 // we are usually within 2% of the correct value:
709 // delay_ticks = (delay + delay / 16) * CNTFRQ_EL0 >> 30
710 MRS CNTFRQ_EL0, R1
711 ADD R0>>4, R0, R0
712 MUL R1, R0, R0
713 LSR $30, R0, R0
714 CBZ R0, done
715 // start = current counter/timer value
716 MRS CNTVCT_EL0, R2
717 delay:
718 // Delay using ISB for all ticks.
719 ISB $15
720 // Substract and compare to handle counter roll-over.
721 // counter_read() - start < delay_ticks
722 MRS CNTVCT_EL0, R1
723 SUB R2, R1, R1
724 CMP R0, R1
725 BCC delay
726 done:
727 RET
728
729 // Save state of caller into g->sched,
730 // but using fake PC from systemstack_switch.
731 // Must only be called from functions with no locals ($0)
732 // or else unwinding from systemstack_switch is incorrect.
733 // Smashes R0.
734 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
735 MOVD $runtime·systemstack_switch(SB), R0
736 ADD $8, R0 // get past prologue
737 MOVD R0, (g_sched+gobuf_pc)(g)
738 MOVD RSP, R0
739 MOVD R0, (g_sched+gobuf_sp)(g)
740 MOVD R29, (g_sched+gobuf_bp)(g)
741 MOVD $0, (g_sched+gobuf_lr)(g)
742 // Assert ctxt is zero. See func save.
743 MOVD (g_sched+gobuf_ctxt)(g), R0
744 CBZ R0, 2(PC)
745 CALL runtime·abort(SB)
746 RET
747
748 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
749 // Call fn(arg) aligned appropriately for the gcc ABI.
750 // Called on a system stack, and there may be no g yet (during needm).
751 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
752 MOVD fn+0(FP), R1
753 MOVD arg+8(FP), R0
754 SUB $16, RSP // skip over saved frame pointer below RSP
755 BL (R1)
756 ADD $16, RSP // skip over saved frame pointer below RSP
757 RET
758
759 // func asmcgocall(fn, arg unsafe.Pointer) int32
760 // Call fn(arg) on the scheduler stack,
761 // aligned appropriately for the gcc ABI.
762 // See cgocall.go for more details.
763 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
764 CBZ g, nosave
765
766 // Figure out if we need to switch to m->g0 stack.
767 // We get called to create new OS threads too, and those
768 // come in on the m->g0 stack already. Or we might already
769 // be on the m->gsignal stack.
770 MOVD g_m(g), R8
771 MOVD m_gsignal(R8), R3
772 CMP R3, g
773 BEQ nosave
774 MOVD m_g0(R8), R3
775 CMP R3, g
776 BEQ nosave
777
778 // running on a user stack. Figure out if we're running
779 // secret code and clear our registers if so.
780 #ifdef GOEXPERIMENT_runtimesecret
781 MOVW g_secret(g), R5
782 CBZ R5, nosecret
783 BL ·secretEraseRegisters(SB)
784 // restore g0 back into R3
785 MOVD g_m(g), R3
786 MOVD m_g0(R3), R3
787
788 nosecret:
789 #endif
790 MOVD fn+0(FP), R1
791 MOVD arg+8(FP), R0
792 MOVD RSP, R2
793 MOVD g, R4
794
795 // Switch to system stack.
796 MOVD R0, R9 // gosave_systemstack_switch<> and save_g might clobber R0
797 BL gosave_systemstack_switch<>(SB)
798 MOVD R3, g
799 BL runtime·save_g(SB)
800 MOVD (g_sched+gobuf_sp)(g), R0
801 MOVD R0, RSP
802 MOVD (g_sched+gobuf_bp)(g), R29
803 MOVD R9, R0
804
805 // Now on a scheduling stack (a pthread-created stack).
806 // Save room for two of our pointers /*, plus 32 bytes of callee
807 // save area that lives on the caller stack. */
808 MOVD RSP, R13
809 SUB $16, R13
810 MOVD R13, RSP
811 MOVD R4, 0(RSP) // save old g on stack
812 MOVD (g_stack+stack_hi)(R4), R4
813 SUB R2, R4
814 MOVD R4, 8(RSP) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
815 BL (R1)
816 MOVD R0, R9
817
818 // Restore g, stack pointer. R0 is errno, so don't touch it
819 MOVD 0(RSP), g
820 BL runtime·save_g(SB)
821 MOVD (g_stack+stack_hi)(g), R5
822 MOVD 8(RSP), R6
823 SUB R6, R5
824 MOVD R9, R0
825 MOVD R5, RSP
826
827 MOVW R0, ret+16(FP)
828 RET
829
830 nosave:
831 // Running on a system stack, perhaps even without a g.
832 // Having no g can happen during thread creation or thread teardown
833 // (see needm/dropm on Solaris, for example).
834 // This code is like the above sequence but without saving/restoring g
835 // and without worrying about the stack moving out from under us
836 // (because we're on a system stack, not a goroutine stack).
837 MOVD fn+0(FP), R1
838 MOVD arg+8(FP), R0
839 MOVD RSP, R2
840 MOVD R2, R13
841 SUB $16, R13
842 MOVD R13, RSP
843 MOVD $0, R4
844 MOVD R4, 0(RSP) // Where above code stores g, in case someone looks during debugging.
845 MOVD R2, 8(RSP) // Save original stack pointer.
846 BL (R1)
847 // Restore stack pointer.
848 MOVD 8(RSP), R2
849 MOVD R2, RSP
850 MOVD R0, ret+16(FP)
851 RET
852
853 // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
854 // See cgocall.go for more details.
855 TEXT ·cgocallback(SB),NOSPLIT,$24-24
856 NO_LOCAL_POINTERS
857
858 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
859 // It is used to dropm while thread is exiting.
860 MOVD fn+0(FP), R1
861 CBNZ R1, loadg
862 // Restore the g from frame.
863 MOVD frame+8(FP), g
864 B dropm
865
866 loadg:
867 // Load g from thread-local storage.
868 BL runtime·load_g(SB)
869
870 // If g is nil, Go did not create the current thread,
871 // or if this thread never called into Go on pthread platforms.
872 // Call needm to obtain one for temporary use.
873 // In this case, we're running on the thread stack, so there's
874 // lots of space, but the linker doesn't know. Hide the call from
875 // the linker analysis by using an indirect call.
876 CBZ g, needm
877
878 MOVD g_m(g), R8
879 MOVD R8, savedm-8(SP)
880 B havem
881
882 needm:
883 MOVD g, savedm-8(SP) // g is zero, so is m.
884 MOVD $runtime·needAndBindM(SB), R0
885 BL (R0)
886
887 // Set m->g0->sched.sp = SP, so that if a panic happens
888 // during the function we are about to execute, it will
889 // have a valid SP to run on the g0 stack.
890 // The next few lines (after the havem label)
891 // will save this SP onto the stack and then write
892 // the same SP back to m->sched.sp. That seems redundant,
893 // but if an unrecovered panic happens, unwindm will
894 // restore the g->sched.sp from the stack location
895 // and then systemstack will try to use it. If we don't set it here,
896 // that restored SP will be uninitialized (typically 0) and
897 // will not be usable.
898 MOVD g_m(g), R8
899 MOVD m_g0(R8), R3
900 MOVD RSP, R0
901 MOVD R0, (g_sched+gobuf_sp)(R3)
902 MOVD R29, (g_sched+gobuf_bp)(R3)
903
904 havem:
905 // Now there's a valid m, and we're running on its m->g0.
906 // Save current m->g0->sched.sp on stack and then set it to SP.
907 // Save current sp in m->g0->sched.sp in preparation for
908 // switch back to m->curg stack.
909 // NOTE: unwindm knows that the saved g->sched.sp is at 16(RSP) aka savedsp-16(SP).
910 // Beware that the frame size is actually 32+16.
911 MOVD m_g0(R8), R3
912 MOVD (g_sched+gobuf_sp)(R3), R4
913 MOVD R4, savedsp-16(SP)
914 MOVD RSP, R0
915 MOVD R0, (g_sched+gobuf_sp)(R3)
916
917 // Switch to m->curg stack and call runtime.cgocallbackg.
918 // Because we are taking over the execution of m->curg
919 // but *not* resuming what had been running, we need to
920 // save that information (m->curg->sched) so we can restore it.
921 // We can restore m->curg->sched.sp easily, because calling
922 // runtime.cgocallbackg leaves SP unchanged upon return.
923 // To save m->curg->sched.pc, we push it onto the curg stack and
924 // open a frame the same size as cgocallback's g0 frame.
925 // Once we switch to the curg stack, the pushed PC will appear
926 // to be the return PC of cgocallback, so that the traceback
927 // will seamlessly trace back into the earlier calls.
928 MOVD m_curg(R8), g
929 BL runtime·save_g(SB)
930 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
931 MOVD (g_sched+gobuf_pc)(g), R5
932 MOVD R5, -48(R4)
933 MOVD (g_sched+gobuf_bp)(g), R6
934 MOVD R6, -56(R4)
935
936 // Gather our arguments into registers.
937 MOVD fn+0(FP), R0
938 MOVD frame+8(FP), R1
939 MOVD ctxt+16(FP), R2
940
941 SUB $48, R4 // Allocate the same frame size on the g stack
942 MOVD R4, RSP // switch stack
943 MOVD $runtime·cgocallbackg<ABIInternal>(SB), R11
944 CALL (R11) // indirect call to bypass nosplit check. We're on a different stack now.
945
946 // Restore g->sched (== m->curg->sched) from saved values.
947 MOVD 0(RSP), R5
948 MOVD R5, (g_sched+gobuf_pc)(g)
949 MOVD -8(RSP), R6
950 MOVD R6, (g_sched+gobuf_bp)(g)
951 MOVD RSP, R4
952 ADD $48, R4, R4
953 MOVD R4, (g_sched+gobuf_sp)(g)
954
955 // Switch back to m->g0's stack and restore m->g0->sched.sp.
956 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
957 // so we do not have to restore it.)
958 MOVD g_m(g), R8
959 MOVD m_g0(R8), g
960 BL runtime·save_g(SB)
961 MOVD (g_sched+gobuf_sp)(g), R0
962 MOVD R0, RSP
963 MOVD savedsp-16(SP), R4
964 MOVD R4, (g_sched+gobuf_sp)(g)
965
966 // If the m on entry was nil, we called needm above to borrow an m,
967 // 1. for the duration of the call on non-pthread platforms,
968 // 2. or the duration of the C thread alive on pthread platforms.
969 // If the m on entry wasn't nil,
970 // 1. the thread might be a Go thread,
971 // 2. or it wasn't the first call from a C thread on pthread platforms,
972 // since then we skip dropm to reuse the m in the first call.
973 MOVD savedm-8(SP), R6
974 CBNZ R6, droppedm
975
976 // Skip dropm to reuse it in the next call, when a pthread key has been created.
977 MOVD _cgo_pthread_key_created(SB), R6
978 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
979 CBZ R6, dropm
980 MOVD (R6), R6
981 CBNZ R6, droppedm
982
983 dropm:
984 MOVD $runtime·dropm(SB), R0
985 BL (R0)
986 droppedm:
987
988 // Done!
989 RET
990
991 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
992 // Must obey the gcc calling convention.
993 TEXT _cgo_topofstack(SB),NOSPLIT,$24
994 // g (R28) and REGTMP (R27) might be clobbered by load_g. They
995 // are callee-save in the gcc calling convention, so save them.
996 MOVD R27, savedR27-8(SP)
997 MOVD g, saveG-16(SP)
998
999 BL runtime·load_g(SB)
1000 MOVD g_m(g), R0
1001 MOVD m_curg(R0), R0
1002 MOVD (g_stack+stack_hi)(R0), R0
1003
1004 MOVD saveG-16(SP), g
1005 MOVD savedR28-8(SP), R27
1006 RET
1007
1008 // void setg(G*); set g. for use by needm.
1009 TEXT runtime·setg(SB), NOSPLIT, $0-8
1010 MOVD gg+0(FP), g
1011 // This only happens if iscgo, so jump straight to save_g
1012 BL runtime·save_g(SB)
1013 RET
1014
1015 // void setg_gcc(G*); set g called from gcc
1016 TEXT setg_gcc<>(SB),NOSPLIT,$8
1017 MOVD R0, g
1018 MOVD R27, savedR27-8(SP)
1019 BL runtime·save_g(SB)
1020 MOVD savedR27-8(SP), R27
1021 RET
1022
1023 TEXT runtime·emptyfunc(SB),0,$0-0
1024 RET
1025
1026 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
1027 MOVD ZR, R0
1028 MOVD (R0), R0
1029 UNDEF
1030
1031 // The top-most function running on a goroutine
1032 // returns to goexit+PCQuantum.
1033 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
1034 MOVD R0, R0 // NOP
1035 BL runtime·goexit1(SB) // does not return
1036
1037 // This is called from .init_array and follows the platform, not Go, ABI.
1038 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
1039 SUB $0x10, RSP
1040 MOVD R27, 8(RSP) // The access to global variables below implicitly uses R27, which is callee-save
1041 MOVD runtime·lastmoduledatap(SB), R1
1042 MOVD R0, moduledata_next(R1)
1043 MOVD R0, runtime·lastmoduledatap(SB)
1044 MOVD 8(RSP), R27
1045 ADD $0x10, RSP
1046 RET
1047
1048 // gcWriteBarrier informs the GC about heap pointer writes.
1049 //
1050 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
1051 // number of bytes of buffer needed in R25, and returns a pointer
1052 // to the buffer space in R25.
1053 // It clobbers condition codes.
1054 // It does not clobber any general-purpose registers except R27,
1055 // but may clobber others (e.g., floating point registers)
1056 // The act of CALLing gcWriteBarrier will clobber R30 (LR).
1057 TEXT gcWriteBarrier<>(SB),NOSPLIT,$200
1058 // Save the registers clobbered by the fast path.
1059 STP (R0, R1), 184(RSP)
1060 retry:
1061 MOVD g_m(g), R0
1062 MOVD m_p(R0), R0
1063 MOVD (p_wbBuf+wbBuf_next)(R0), R1
1064 MOVD (p_wbBuf+wbBuf_end)(R0), R27
1065 // Increment wbBuf.next position.
1066 ADD R25, R1
1067 // Is the buffer full?
1068 CMP R27, R1
1069 BHI flush
1070 // Commit to the larger buffer.
1071 MOVD R1, (p_wbBuf+wbBuf_next)(R0)
1072 // Make return value (the original next position)
1073 SUB R25, R1, R25
1074 // Restore registers.
1075 LDP 184(RSP), (R0, R1)
1076 RET
1077
1078 flush:
1079 // Save all general purpose registers since these could be
1080 // clobbered by wbBufFlush and were not saved by the caller.
1081 // R0 and R1 already saved
1082 STP (R2, R3), 1*8(RSP)
1083 STP (R4, R5), 3*8(RSP)
1084 STP (R6, R7), 5*8(RSP)
1085 STP (R8, R9), 7*8(RSP)
1086 STP (R10, R11), 9*8(RSP)
1087 STP (R12, R13), 11*8(RSP)
1088 STP (R14, R15), 13*8(RSP)
1089 // R16, R17 may be clobbered by linker trampoline
1090 // R18 is unused.
1091 STP (R19, R20), 15*8(RSP)
1092 STP (R21, R22), 17*8(RSP)
1093 STP (R23, R24), 19*8(RSP)
1094 STP (R25, R26), 21*8(RSP)
1095 // R27 is temp register.
1096 // R28 is g.
1097 // R29 is frame pointer (unused).
1098 // R30 is LR, which was saved by the prologue.
1099 // R31 is SP.
1100
1101 CALL runtime·wbBufFlush(SB)
1102 LDP 1*8(RSP), (R2, R3)
1103 LDP 3*8(RSP), (R4, R5)
1104 LDP 5*8(RSP), (R6, R7)
1105 LDP 7*8(RSP), (R8, R9)
1106 LDP 9*8(RSP), (R10, R11)
1107 LDP 11*8(RSP), (R12, R13)
1108 LDP 13*8(RSP), (R14, R15)
1109 LDP 15*8(RSP), (R19, R20)
1110 LDP 17*8(RSP), (R21, R22)
1111 LDP 19*8(RSP), (R23, R24)
1112 LDP 21*8(RSP), (R25, R26)
1113 JMP retry
1114
1115 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
1116 MOVD $8, R25
1117 JMP gcWriteBarrier<>(SB)
1118 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
1119 MOVD $16, R25
1120 JMP gcWriteBarrier<>(SB)
1121 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
1122 MOVD $24, R25
1123 JMP gcWriteBarrier<>(SB)
1124 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
1125 MOVD $32, R25
1126 JMP gcWriteBarrier<>(SB)
1127 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
1128 MOVD $40, R25
1129 JMP gcWriteBarrier<>(SB)
1130 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
1131 MOVD $48, R25
1132 JMP gcWriteBarrier<>(SB)
1133 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
1134 MOVD $56, R25
1135 JMP gcWriteBarrier<>(SB)
1136 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
1137 MOVD $64, R25
1138 JMP gcWriteBarrier<>(SB)
1139
1140 DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1141 GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1142
1143 // debugCallV2 is the entry point for debugger-injected function
1144 // calls on running goroutines. It informs the runtime that a
1145 // debug call has been injected and creates a call frame for the
1146 // debugger to fill in.
1147 //
1148 // To inject a function call, a debugger should:
1149 // 1. Check that the goroutine is in state _Grunning and that
1150 // there are at least 288 bytes free on the stack.
1151 // 2. Set SP as SP-16.
1152 // 3. Store the current LR in (SP) (using the SP after step 2).
1153 // 4. Store the current PC in the LR register.
1154 // 5. Write the desired argument frame size at SP-16
1155 // 6. Save all machine registers (including flags and fpsimd registers)
1156 // so they can be restored later by the debugger.
1157 // 7. Set the PC to debugCallV2 and resume execution.
1158 //
1159 // If the goroutine is in state _Grunnable, then it's not generally
1160 // safe to inject a call because it may return out via other runtime
1161 // operations. Instead, the debugger should unwind the stack to find
1162 // the return to non-runtime code, add a temporary breakpoint there,
1163 // and inject the call once that breakpoint is hit.
1164 //
1165 // If the goroutine is in any other state, it's not safe to inject a call.
1166 //
1167 // This function communicates back to the debugger by setting R20 and
1168 // invoking BRK to raise a breakpoint signal. Note that the signal PC of
1169 // the signal triggered by the BRK instruction is the PC where the signal
1170 // is trapped, not the next PC, so to resume execution, the debugger needs
1171 // to set the signal PC to PC+4. See the comments in the implementation for
1172 // the protocol the debugger is expected to follow. InjectDebugCall in the
1173 // runtime tests demonstrates this protocol.
1174 //
1175 // The debugger must ensure that any pointers passed to the function
1176 // obey escape analysis requirements. Specifically, it must not pass
1177 // a stack pointer to an escaping argument. debugCallV2 cannot check
1178 // this invariant.
1179 //
1180 // This is ABIInternal because Go code injects its PC directly into new
1181 // goroutine stacks.
1182 TEXT runtime·debugCallV2<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
1183 STP (R29, R30), -280(RSP)
1184 SUB $272, RSP, RSP
1185 SUB $8, RSP, R29
1186 // Save all registers that may contain pointers so they can be
1187 // conservatively scanned.
1188 //
1189 // We can't do anything that might clobber any of these
1190 // registers before this.
1191 STP (R27, g), (30*8)(RSP)
1192 STP (R25, R26), (28*8)(RSP)
1193 STP (R23, R24), (26*8)(RSP)
1194 STP (R21, R22), (24*8)(RSP)
1195 STP (R19, R20), (22*8)(RSP)
1196 STP (R16, R17), (20*8)(RSP)
1197 STP (R14, R15), (18*8)(RSP)
1198 STP (R12, R13), (16*8)(RSP)
1199 STP (R10, R11), (14*8)(RSP)
1200 STP (R8, R9), (12*8)(RSP)
1201 STP (R6, R7), (10*8)(RSP)
1202 STP (R4, R5), (8*8)(RSP)
1203 STP (R2, R3), (6*8)(RSP)
1204 STP (R0, R1), (4*8)(RSP)
1205
1206 // Perform a safe-point check.
1207 MOVD R30, 8(RSP) // Caller's PC
1208 CALL runtime·debugCallCheck(SB)
1209 MOVD 16(RSP), R0
1210 CBZ R0, good
1211
1212 // The safety check failed. Put the reason string at the top
1213 // of the stack.
1214 MOVD R0, 8(RSP)
1215 MOVD 24(RSP), R0
1216 MOVD R0, 16(RSP)
1217
1218 // Set R20 to 8 and invoke BRK. The debugger should get the
1219 // reason a call can't be injected from SP+8 and resume execution.
1220 MOVD $8, R20
1221 BREAK
1222 JMP restore
1223
1224 good:
1225 // Registers are saved and it's safe to make a call.
1226 // Open up a call frame, moving the stack if necessary.
1227 //
1228 // Once the frame is allocated, this will set R20 to 0 and
1229 // invoke BRK. The debugger should write the argument
1230 // frame for the call at SP+8, set up argument registers,
1231 // set the LR as the signal PC + 4, set the PC to the function
1232 // to call, set R26 to point to the closure (if a closure call),
1233 // and resume execution.
1234 //
1235 // If the function returns, this will set R20 to 1 and invoke
1236 // BRK. The debugger can then inspect any return value saved
1237 // on the stack at SP+8 and in registers. To resume execution,
1238 // the debugger should restore the LR from (SP).
1239 //
1240 // If the function panics, this will set R20 to 2 and invoke BRK.
1241 // The interface{} value of the panic will be at SP+8. The debugger
1242 // can inspect the panic value and resume execution again.
1243 #define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1244 CMP $MAXSIZE, R0; \
1245 BGT 5(PC); \
1246 MOVD $NAME(SB), R0; \
1247 MOVD R0, 8(RSP); \
1248 CALL runtime·debugCallWrap(SB); \
1249 JMP restore
1250
1251 MOVD 256(RSP), R0 // the argument frame size
1252 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1253 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1254 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1255 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1256 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1257 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1258 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1259 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1260 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1261 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1262 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1263 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1264 // The frame size is too large. Report the error.
1265 MOVD $debugCallFrameTooLarge<>(SB), R0
1266 MOVD R0, 8(RSP)
1267 MOVD $20, R0
1268 MOVD R0, 16(RSP) // length of debugCallFrameTooLarge string
1269 MOVD $8, R20
1270 BREAK
1271 JMP restore
1272
1273 restore:
1274 // Calls and failures resume here.
1275 //
1276 // Set R20 to 16 and invoke BRK. The debugger should restore
1277 // all registers except for PC and RSP and resume execution.
1278 MOVD $16, R20
1279 BREAK
1280 // We must not modify flags after this point.
1281
1282 // Restore pointer-containing registers, which may have been
1283 // modified from the debugger's copy by stack copying.
1284 LDP (30*8)(RSP), (R27, g)
1285 LDP (28*8)(RSP), (R25, R26)
1286 LDP (26*8)(RSP), (R23, R24)
1287 LDP (24*8)(RSP), (R21, R22)
1288 LDP (22*8)(RSP), (R19, R20)
1289 LDP (20*8)(RSP), (R16, R17)
1290 LDP (18*8)(RSP), (R14, R15)
1291 LDP (16*8)(RSP), (R12, R13)
1292 LDP (14*8)(RSP), (R10, R11)
1293 LDP (12*8)(RSP), (R8, R9)
1294 LDP (10*8)(RSP), (R6, R7)
1295 LDP (8*8)(RSP), (R4, R5)
1296 LDP (6*8)(RSP), (R2, R3)
1297 LDP (4*8)(RSP), (R0, R1)
1298
1299 LDP -8(RSP), (R29, R27)
1300 ADD $288, RSP, RSP // Add 16 more bytes, see saveSigContext
1301 MOVD -16(RSP), R30 // restore old lr
1302 JMP (R27)
1303
1304 // runtime.debugCallCheck assumes that functions defined with the
1305 // DEBUG_CALL_FN macro are safe points to inject calls.
1306 #define DEBUG_CALL_FN(NAME,MAXSIZE) \
1307 TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1308 NO_LOCAL_POINTERS; \
1309 MOVD $0, R20; \
1310 BREAK; \
1311 MOVD $1, R20; \
1312 BREAK; \
1313 RET
1314 DEBUG_CALL_FN(debugCall32<>, 32)
1315 DEBUG_CALL_FN(debugCall64<>, 64)
1316 DEBUG_CALL_FN(debugCall128<>, 128)
1317 DEBUG_CALL_FN(debugCall256<>, 256)
1318 DEBUG_CALL_FN(debugCall512<>, 512)
1319 DEBUG_CALL_FN(debugCall1024<>, 1024)
1320 DEBUG_CALL_FN(debugCall2048<>, 2048)
1321 DEBUG_CALL_FN(debugCall4096<>, 4096)
1322 DEBUG_CALL_FN(debugCall8192<>, 8192)
1323 DEBUG_CALL_FN(debugCall16384<>, 16384)
1324 DEBUG_CALL_FN(debugCall32768<>, 32768)
1325 DEBUG_CALL_FN(debugCall65536<>, 65536)
1326
1327 // func debugCallPanicked(val interface{})
1328 TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16
1329 // Copy the panic value to the top of stack at SP+8.
1330 MOVD val_type+0(FP), R0
1331 MOVD R0, 8(RSP)
1332 MOVD val_data+8(FP), R0
1333 MOVD R0, 16(RSP)
1334 MOVD $2, R20
1335 BREAK
1336 RET
1337
1338 TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-0
1339 NO_LOCAL_POINTERS
1340 // Save all 16 int registers that could have an index in them.
1341 // They may be pointers, but if they are they are dead.
1342 STP (R0, R1), 24(RSP)
1343 STP (R2, R3), 40(RSP)
1344 STP (R4, R5), 56(RSP)
1345 STP (R6, R7), 72(RSP)
1346 STP (R8, R9), 88(RSP)
1347 STP (R10, R11), 104(RSP)
1348 STP (R12, R13), 120(RSP)
1349 STP (R14, R15), 136(RSP)
1350 MOVD LR, R0 // PC immediately after call to panicBounds
1351 ADD $24, RSP, R1 // pointer to save area
1352 CALL runtime·panicBounds64<ABIInternal>(SB)
1353 RET
1354
1355 TEXT ·getfp<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1356 MOVD R29, R0
1357 RET
1358
View as plain text