Text file
src/runtime/asm_arm64.s
1 // Copyright 2015 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "tls_arm64.h"
8 #include "funcdata.h"
9 #include "textflag.h"
10 #include "cgo/abi_arm64.h"
11
12 // _rt0_arm64 is common startup code for most arm64 systems when using
13 // internal linking. This is the entry point for the program from the
14 // kernel for an ordinary -buildmode=exe program. The stack holds the
15 // number of arguments and the C-style argv.
16 TEXT _rt0_arm64(SB),NOSPLIT,$0
17 MOVD 0(RSP), R0 // argc
18 ADD $8, RSP, R1 // argv
19 JMP runtime·rt0_go(SB)
20
21 // main is common startup code for most amd64 systems when using
22 // external linking. The C startup code will call the symbol "main"
23 // passing argc and argv in the usual C ABI registers R0 and R1.
24 TEXT main(SB),NOSPLIT,$0
25 JMP runtime·rt0_go(SB)
26
27 // _rt0_arm64_lib is common startup code for most arm64 systems when
28 // using -buildmode=c-archive or -buildmode=c-shared. The linker will
29 // arrange to invoke this function as a global constructor (for
30 // c-archive) or when the shared library is loaded (for c-shared).
31 // We expect argc and argv to be passed in the usual C ABI registers
32 // R0 and R1.
33 TEXT _rt0_arm64_lib(SB),NOSPLIT,$184
34 // Preserve callee-save registers.
35 SAVE_R19_TO_R28(24)
36 SAVE_F8_TO_F15(104)
37
38 // Initialize g as null in case of using g later e.g. sigaction in cgo_sigaction.go
39 MOVD ZR, g
40
41 MOVD R0, _rt0_arm64_lib_argc<>(SB)
42 MOVD R1, _rt0_arm64_lib_argv<>(SB)
43
44 MOVD $runtime·libInit(SB), R4
45 BL (R4)
46
47 // Restore callee-save registers.
48 RESTORE_R19_TO_R28(24)
49 RESTORE_F8_TO_F15(104)
50 RET
51
52 TEXT runtime·rt0_lib_go<ABIInternal>(SB),NOSPLIT,$0
53 MOVD _rt0_arm64_lib_argc<>(SB), R0
54 MOVD _rt0_arm64_lib_argv<>(SB), R1
55 MOVD $runtime·rt0_go(SB),R4
56 B (R4)
57
58 DATA _rt0_arm64_lib_argc<>(SB)/8, $0
59 GLOBL _rt0_arm64_lib_argc<>(SB),NOPTR, $8
60 DATA _rt0_arm64_lib_argv<>(SB)/8, $0
61 GLOBL _rt0_arm64_lib_argv<>(SB),NOPTR, $8
62
63 #ifdef GOARM64_LSE
64 DATA no_lse_msg<>+0x00(SB)/64, $"This program can only run on ARM64 processors with LSE support.\n"
65 GLOBL no_lse_msg<>(SB), RODATA, $64
66 #endif
67
68 // We know for sure that Linux and FreeBSD allow to read instruction set
69 // attribute registers (while some others OSes, like OpenBSD and Darwin,
70 // are not). Let's be conservative and allow code reading such registers
71 // only when we sure this won't lead to sigill.
72 #ifdef GOOS_linux
73 #define ISA_REGS_READABLE
74 #endif
75 #ifdef GOOS_freebsd
76 #define ISA_REGS_READABLE
77 #endif
78
79 #ifdef GOARM64_LSE
80 #ifdef ISA_REGS_READABLE
81 #define CHECK_GOARM64_LSE
82 #endif
83 #endif
84
85 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
86 // SP = stack; R0 = argc; R1 = argv
87
88 SUB $32, RSP
89 MOVW R0, 8(RSP) // argc
90 MOVD R1, 16(RSP) // argv
91
92 // This is typically the entry point for Go programs.
93 // Call stack unwinding must not proceed past this frame.
94 // Set the frame pointer register to 0 so that frame pointer-based unwinders
95 // (which don't use debug info for performance reasons)
96 // won't attempt to unwind past this function.
97 // See go.dev/issue/63630
98 MOVD $0, R29
99
100 #ifdef TLS_darwin
101 // Initialize TLS.
102 MOVD ZR, g // clear g, make sure it's not junk.
103 SUB $32, RSP
104 MRS_TPIDR_R0
105 AND $~7, R0
106 MOVD R0, 16(RSP) // arg2: TLS base
107 MOVD $runtime·tls_g(SB), R2
108 MOVD R2, 8(RSP) // arg1: &tlsg
109 BL ·tlsinit(SB)
110 ADD $32, RSP
111 #endif
112
113 // create istack out of the given (operating system) stack.
114 // _cgo_init may update stackguard.
115 MOVD $runtime·g0(SB), g
116 MOVD RSP, R7
117 MOVD $(-64*1024)(R7), R0
118 MOVD R0, g_stackguard0(g)
119 MOVD R0, g_stackguard1(g)
120 MOVD R0, (g_stack+stack_lo)(g)
121 MOVD R7, (g_stack+stack_hi)(g)
122
123 // if there is a _cgo_init, call it using the gcc ABI.
124 MOVD _cgo_init(SB), R12
125 CBZ R12, nocgo
126
127 #ifdef GOOS_android
128 MRS_TPIDR_R0 // load TLS base pointer
129 MOVD R0, R3 // arg 3: TLS base pointer
130 MOVD $runtime·tls_g(SB), R2 // arg 2: &tls_g
131 #else
132 MOVD $0, R2 // arg 2: not used when using platform's TLS
133 #endif
134 MOVD $setg_gcc<>(SB), R1 // arg 1: setg
135 MOVD g, R0 // arg 0: G
136 SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved.
137 BL (R12)
138 ADD $16, RSP
139
140 nocgo:
141 BL runtime·save_g(SB)
142 // update stackguard after _cgo_init
143 MOVD (g_stack+stack_lo)(g), R0
144 ADD $const_stackGuard, R0
145 MOVD R0, g_stackguard0(g)
146 MOVD R0, g_stackguard1(g)
147
148 // set the per-goroutine and per-mach "registers"
149 MOVD $runtime·m0(SB), R0
150
151 // save m->g0 = g0
152 MOVD g, m_g0(R0)
153 // save m0 to g0->m
154 MOVD R0, g_m(g)
155
156 BL runtime·check(SB)
157
158 #ifdef GOOS_windows
159 BL runtime·wintls(SB)
160 #endif
161
162 // Check that CPU we use for execution supports instructions targeted during compile-time.
163 #ifdef CHECK_GOARM64_LSE
164 // Read the ID_AA64ISAR0_EL1 register
165 MRS ID_AA64ISAR0_EL1, R0
166
167 // Extract the LSE field (bits [23:20])
168 LSR $20, R0, R0
169 AND $0xf, R0, R0
170
171 // LSE support is indicated by a non-zero value
172 CBZ R0, no_lse
173 #endif
174
175 MOVW 8(RSP), R0 // copy argc
176 MOVW R0, -8(RSP)
177 MOVD 16(RSP), R0 // copy argv
178 MOVD R0, 0(RSP)
179 BL runtime·args(SB)
180 BL runtime·osinit(SB)
181 BL runtime·schedinit(SB)
182
183 // create a new goroutine to start program
184 MOVD $runtime·mainPC(SB), R0 // entry
185 SUB $16, RSP
186 MOVD R0, 8(RSP) // arg
187 MOVD $0, 0(RSP) // dummy LR
188 BL runtime·newproc(SB)
189 ADD $16, RSP
190
191 // start this M
192 BL runtime·mstart(SB)
193 UNDEF
194
195 #ifdef CHECK_GOARM64_LSE
196 no_lse:
197 MOVD $1, R0 // stderr
198 MOVD R0, 8(RSP)
199 MOVD $no_lse_msg<>(SB), R1 // message address
200 MOVD R1, 16(RSP)
201 MOVD $64, R2 // message length
202 MOVD R2, 24(RSP)
203 CALL runtime·write(SB)
204 CALL runtime·exit(SB)
205 CALL runtime·abort(SB)
206 RET
207 #endif
208
209 // Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are
210 // intended to be called by debuggers.
211 MOVD $runtime·debugPinnerV1<ABIInternal>(SB), R0
212 MOVD $runtime·debugCallV2<ABIInternal>(SB), R0
213
214 MOVD $0, R0
215 MOVD R0, (R0) // boom
216 UNDEF
217
218 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
219 GLOBL runtime·mainPC(SB),RODATA,$8
220
221 // Windows ARM64 needs an immediate 0xf000 argument.
222 // See go.dev/issues/53837.
223 #define BREAK \
224 #ifdef GOOS_windows \
225 BRK $0xf000 \
226 #else \
227 BRK \
228 #endif \
229
230
231 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
232 BREAK
233 RET
234
235 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
236 RET
237
238 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
239 // This is the root frame of new Go-created OS threads.
240 // Call stack unwinding must not proceed past this frame.
241 // Set the frame pointer register to 0 so that frame pointer-based unwinders
242 // (which don't use debug info for performance reasons)
243 // won't attempt to unwind past this function.
244 // See go.dev/issue/63630
245 MOVD $0, R29
246 BL runtime·mstart0(SB)
247 RET // not reached
248
249 /*
250 * go-routine
251 */
252
253 // void gogo(Gobuf*)
254 // restore state from Gobuf; longjmp
255 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
256 MOVD buf+0(FP), R5
257 MOVD gobuf_g(R5), R6
258 MOVD 0(R6), R4 // make sure g != nil
259 B gogo<>(SB)
260
261 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
262 MOVD R6, g
263 BL runtime·save_g(SB)
264
265 MOVD gobuf_sp(R5), R0
266 MOVD R0, RSP
267 MOVD gobuf_bp(R5), R29
268 MOVD gobuf_lr(R5), LR
269 MOVD gobuf_ctxt(R5), R26
270 MOVD $0, gobuf_sp(R5)
271 MOVD $0, gobuf_bp(R5)
272 MOVD $0, gobuf_lr(R5)
273 MOVD $0, gobuf_ctxt(R5)
274 CMP ZR, ZR // set condition codes for == test, needed by stack split
275 MOVD gobuf_pc(R5), R6
276 B (R6)
277
278 // void mcall(fn func(*g))
279 // Switch to m->g0's stack, call fn(g).
280 // Fn must never return. It should gogo(&g->sched)
281 // to keep running g.
282 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
283 #ifdef GOEXPERIMENT_runtimesecret
284 MOVW g_secret(g), R26
285 CBZ R26, nosecret
286 // Use R26 as a secondary link register
287 // We purposefully don't erase it in secretEraseRegistersMcall
288 MOVD LR, R26
289 BL runtime·secretEraseRegistersMcall(SB)
290 MOVD R26, LR
291
292 nosecret:
293 #endif
294 MOVD R0, R26 // context
295
296 // Save caller state in g->sched
297 MOVD RSP, R0
298 MOVD R0, (g_sched+gobuf_sp)(g)
299 MOVD R29, (g_sched+gobuf_bp)(g)
300 MOVD LR, (g_sched+gobuf_pc)(g)
301 MOVD $0, (g_sched+gobuf_lr)(g)
302
303 // Switch to m->g0 & its stack, call fn.
304 MOVD g, R3
305 MOVD g_m(g), R8
306 MOVD m_g0(R8), g
307 BL runtime·save_g(SB)
308 CMP g, R3
309 BNE 2(PC)
310 B runtime·badmcall(SB)
311
312 MOVD (g_sched+gobuf_sp)(g), R0
313 MOVD R0, RSP // sp = m->g0->sched.sp
314 MOVD $0, R29 // clear frame pointer, as caller may execute on another M
315 MOVD R3, R0 // arg = g
316 MOVD $0, -16(RSP) // dummy LR
317 SUB $16, RSP
318 MOVD 0(R26), R4 // code pointer
319 BL (R4)
320 B runtime·badmcall2(SB)
321
322 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
323 // of the G stack. We need to distinguish the routine that
324 // lives at the bottom of the G stack from the one that lives
325 // at the top of the system stack because the one at the top of
326 // the system stack terminates the stack walk (see topofstack()).
327 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
328 UNDEF
329 BL (LR) // make sure this function is not leaf
330 RET
331
332 // func systemstack(fn func())
333 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
334 #ifdef GOEXPERIMENT_runtimesecret
335 MOVW g_secret(g), R3
336 CBZ R3, nosecret
337 BL ·secretEraseRegisters(SB)
338
339 nosecret:
340 #endif
341 MOVD fn+0(FP), R3 // R3 = fn
342 MOVD R3, R26 // context
343 MOVD g_m(g), R4 // R4 = m
344
345 MOVD m_gsignal(R4), R5 // R5 = gsignal
346 CMP g, R5
347 BEQ noswitch
348
349 MOVD m_g0(R4), R5 // R5 = g0
350 CMP g, R5
351 BEQ noswitch
352
353 MOVD m_curg(R4), R6
354 CMP g, R6
355 BEQ switch
356
357 // Bad: g is not gsignal, not g0, not curg. What is it?
358 // Hide call from linker nosplit analysis.
359 MOVD $runtime·badsystemstack(SB), R3
360 BL (R3)
361 B runtime·abort(SB)
362
363 switch:
364 // Switch stacks.
365 // The original frame pointer is stored in R29,
366 // which is useful for stack unwinding.
367 // Save our state in g->sched. Pretend to
368 // be systemstack_switch if the G stack is scanned.
369 BL gosave_systemstack_switch<>(SB)
370
371 // switch to g0
372 MOVD R5, g
373 BL runtime·save_g(SB)
374 MOVD (g_sched+gobuf_sp)(g), R3
375 MOVD R3, RSP
376
377 // call target function
378 MOVD 0(R26), R3 // code pointer
379 BL (R3)
380
381 // switch back to g
382 MOVD g_m(g), R3
383 MOVD m_curg(R3), g
384 BL runtime·save_g(SB)
385 MOVD (g_sched+gobuf_sp)(g), R0
386 MOVD R0, RSP
387 MOVD (g_sched+gobuf_bp)(g), R29
388 MOVD $0, (g_sched+gobuf_sp)(g)
389 MOVD $0, (g_sched+gobuf_bp)(g)
390 RET
391
392 noswitch:
393 // already on m stack, just call directly
394 // Using a tail call here cleans up tracebacks since we won't stop
395 // at an intermediate systemstack.
396 MOVD 0(R26), R3 // code pointer
397 MOVD.P 16(RSP), R30 // restore LR
398 SUB $8, RSP, R29 // restore FP
399 B (R3)
400
401 // func switchToCrashStack0(fn func())
402 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
403 MOVD R0, R26 // context register
404 MOVD g_m(g), R1 // curm
405
406 // set g to gcrash
407 MOVD $runtime·gcrash(SB), g // g = &gcrash
408 BL runtime·save_g(SB) // clobbers R0
409 MOVD R1, g_m(g) // g.m = curm
410 MOVD g, m_g0(R1) // curm.g0 = g
411
412 // switch to crashstack
413 MOVD (g_stack+stack_hi)(g), R1
414 SUB $(4*8), R1
415 MOVD R1, RSP
416
417 // call target function
418 MOVD 0(R26), R0
419 CALL (R0)
420
421 // should never return
422 CALL runtime·abort(SB)
423 UNDEF
424
425 /*
426 * support for morestack
427 */
428
429 // Called during function prolog when more stack is needed.
430 // Caller has already loaded:
431 // R3 prolog's LR (R30)
432 //
433 // The traceback routines see morestack on a g0 as being
434 // the top of a stack (for example, morestack calling newstack
435 // calling the scheduler calling newm calling gc), so we must
436 // record an argument size. For that purpose, it has no arguments.
437 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
438 // Cannot grow scheduler stack (m->g0).
439 MOVD g_m(g), R8
440 MOVD m_g0(R8), R4
441
442 // Called from f.
443 // Set g->sched to context in f
444 MOVD RSP, R0
445 MOVD R0, (g_sched+gobuf_sp)(g)
446 MOVD R29, (g_sched+gobuf_bp)(g)
447 MOVD LR, (g_sched+gobuf_pc)(g)
448 MOVD R3, (g_sched+gobuf_lr)(g)
449 MOVD R26, (g_sched+gobuf_ctxt)(g)
450
451 CMP g, R4
452 BNE 3(PC)
453 BL runtime·badmorestackg0(SB)
454 B runtime·abort(SB)
455
456 // Cannot grow signal stack (m->gsignal).
457 MOVD m_gsignal(R8), R4
458 CMP g, R4
459 BNE 3(PC)
460 BL runtime·badmorestackgsignal(SB)
461 B runtime·abort(SB)
462
463 // Called from f.
464 // Set m->morebuf to f's callers.
465 MOVD R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC
466 MOVD RSP, R0
467 MOVD R0, (m_morebuf+gobuf_sp)(R8) // f's caller's RSP
468 MOVD g, (m_morebuf+gobuf_g)(R8)
469
470 // If in secret mode, erase registers on transition
471 // from G stack to M stack,
472 #ifdef GOEXPERIMENT_runtimesecret
473 MOVW g_secret(g), R4
474 CBZ R4, nosecret
475 BL ·secretEraseRegisters(SB)
476 MOVD g_m(g), R8
477 nosecret:
478 #endif
479
480 // Call newstack on m->g0's stack.
481 MOVD m_g0(R8), g
482 BL runtime·save_g(SB)
483 MOVD (g_sched+gobuf_sp)(g), R0
484 MOVD R0, RSP
485 MOVD $0, R29 // clear frame pointer, as caller may execute on another M
486 MOVD.W $0, -16(RSP) // create a call frame on g0 (saved LR; keep 16-aligned)
487 BL runtime·newstack(SB)
488
489 // Not reached, but make sure the return PC from the call to newstack
490 // is still in this function, and not the beginning of the next.
491 UNDEF
492
493 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
494 // Force SPWRITE. This function doesn't actually write SP,
495 // but it is called with a special calling convention where
496 // the caller doesn't save LR on stack but passes it as a
497 // register (R3), and the unwinder currently doesn't understand.
498 // Make it SPWRITE to stop unwinding. (See issue 54332)
499 MOVD RSP, RSP
500
501 MOVW $0, R26
502 B runtime·morestack(SB)
503
504 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
505 TEXT ·spillArgs(SB),NOSPLIT,$0-0
506 STP (R0, R1), (0*8)(R20)
507 STP (R2, R3), (2*8)(R20)
508 STP (R4, R5), (4*8)(R20)
509 STP (R6, R7), (6*8)(R20)
510 STP (R8, R9), (8*8)(R20)
511 STP (R10, R11), (10*8)(R20)
512 STP (R12, R13), (12*8)(R20)
513 STP (R14, R15), (14*8)(R20)
514 FSTPD (F0, F1), (16*8)(R20)
515 FSTPD (F2, F3), (18*8)(R20)
516 FSTPD (F4, F5), (20*8)(R20)
517 FSTPD (F6, F7), (22*8)(R20)
518 FSTPD (F8, F9), (24*8)(R20)
519 FSTPD (F10, F11), (26*8)(R20)
520 FSTPD (F12, F13), (28*8)(R20)
521 FSTPD (F14, F15), (30*8)(R20)
522 RET
523
524 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
525 TEXT ·unspillArgs(SB),NOSPLIT,$0-0
526 LDP (0*8)(R20), (R0, R1)
527 LDP (2*8)(R20), (R2, R3)
528 LDP (4*8)(R20), (R4, R5)
529 LDP (6*8)(R20), (R6, R7)
530 LDP (8*8)(R20), (R8, R9)
531 LDP (10*8)(R20), (R10, R11)
532 LDP (12*8)(R20), (R12, R13)
533 LDP (14*8)(R20), (R14, R15)
534 FLDPD (16*8)(R20), (F0, F1)
535 FLDPD (18*8)(R20), (F2, F3)
536 FLDPD (20*8)(R20), (F4, F5)
537 FLDPD (22*8)(R20), (F6, F7)
538 FLDPD (24*8)(R20), (F8, F9)
539 FLDPD (26*8)(R20), (F10, F11)
540 FLDPD (28*8)(R20), (F12, F13)
541 FLDPD (30*8)(R20), (F14, F15)
542 RET
543
544 // reflectcall: call a function with the given argument list
545 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
546 // we don't have variable-sized frames, so we use a small number
547 // of constant-sized-frame functions to encode a few bits of size in the pc.
548 // Caution: ugly multiline assembly macros in your future!
549
550 #define DISPATCH(NAME,MAXSIZE) \
551 MOVD $MAXSIZE, R27; \
552 CMP R27, R16; \
553 BGT 3(PC); \
554 MOVD $NAME(SB), R27; \
555 B (R27)
556 // Note: can't just "B NAME(SB)" - bad inlining results.
557
558 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
559 MOVWU frameSize+32(FP), R16
560 DISPATCH(runtime·call16, 16)
561 DISPATCH(runtime·call32, 32)
562 DISPATCH(runtime·call64, 64)
563 DISPATCH(runtime·call128, 128)
564 DISPATCH(runtime·call256, 256)
565 DISPATCH(runtime·call512, 512)
566 DISPATCH(runtime·call1024, 1024)
567 DISPATCH(runtime·call2048, 2048)
568 DISPATCH(runtime·call4096, 4096)
569 DISPATCH(runtime·call8192, 8192)
570 DISPATCH(runtime·call16384, 16384)
571 DISPATCH(runtime·call32768, 32768)
572 DISPATCH(runtime·call65536, 65536)
573 DISPATCH(runtime·call131072, 131072)
574 DISPATCH(runtime·call262144, 262144)
575 DISPATCH(runtime·call524288, 524288)
576 DISPATCH(runtime·call1048576, 1048576)
577 DISPATCH(runtime·call2097152, 2097152)
578 DISPATCH(runtime·call4194304, 4194304)
579 DISPATCH(runtime·call8388608, 8388608)
580 DISPATCH(runtime·call16777216, 16777216)
581 DISPATCH(runtime·call33554432, 33554432)
582 DISPATCH(runtime·call67108864, 67108864)
583 DISPATCH(runtime·call134217728, 134217728)
584 DISPATCH(runtime·call268435456, 268435456)
585 DISPATCH(runtime·call536870912, 536870912)
586 DISPATCH(runtime·call1073741824, 1073741824)
587 MOVD $runtime·badreflectcall(SB), R0
588 B (R0)
589
590 #define CALLFN(NAME,MAXSIZE) \
591 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
592 NO_LOCAL_POINTERS; \
593 /* copy arguments to stack */ \
594 MOVD stackArgs+16(FP), R3; \
595 MOVWU stackArgsSize+24(FP), R4; \
596 ADD $8, RSP, R5; \
597 BIC $0xf, R4, R6; \
598 CBZ R6, 6(PC); \
599 /* if R6=(argsize&~15) != 0 */ \
600 ADD R6, R5, R6; \
601 /* copy 16 bytes a time */ \
602 LDP.P 16(R3), (R7, R8); \
603 STP.P (R7, R8), 16(R5); \
604 CMP R5, R6; \
605 BNE -3(PC); \
606 AND $0xf, R4, R6; \
607 CBZ R6, 6(PC); \
608 /* if R6=(argsize&15) != 0 */ \
609 ADD R6, R5, R6; \
610 /* copy 1 byte a time for the rest */ \
611 MOVBU.P 1(R3), R7; \
612 MOVBU.P R7, 1(R5); \
613 CMP R5, R6; \
614 BNE -3(PC); \
615 /* set up argument registers */ \
616 MOVD regArgs+40(FP), R20; \
617 CALL ·unspillArgs(SB); \
618 /* call function */ \
619 MOVD f+8(FP), R26; \
620 MOVD (R26), R20; \
621 PCDATA $PCDATA_StackMapIndex, $0; \
622 BL (R20); \
623 /* copy return values back */ \
624 MOVD regArgs+40(FP), R20; \
625 CALL ·spillArgs(SB); \
626 MOVD stackArgsType+0(FP), R7; \
627 MOVD stackArgs+16(FP), R3; \
628 MOVWU stackArgsSize+24(FP), R4; \
629 MOVWU stackRetOffset+28(FP), R6; \
630 ADD $8, RSP, R5; \
631 ADD R6, R5; \
632 ADD R6, R3; \
633 SUB R6, R4; \
634 BL callRet<>(SB); \
635 RET
636
637 // callRet copies return values back at the end of call*. This is a
638 // separate function so it can allocate stack space for the arguments
639 // to reflectcallmove. It does not follow the Go ABI; it expects its
640 // arguments in registers.
641 TEXT callRet<>(SB), NOSPLIT, $48-0
642 NO_LOCAL_POINTERS
643 STP (R7, R3), 8(RSP)
644 STP (R5, R4), 24(RSP)
645 MOVD R20, 40(RSP)
646 BL runtime·reflectcallmove(SB)
647 RET
648
649 CALLFN(·call16, 16)
650 CALLFN(·call32, 32)
651 CALLFN(·call64, 64)
652 CALLFN(·call128, 128)
653 CALLFN(·call256, 256)
654 CALLFN(·call512, 512)
655 CALLFN(·call1024, 1024)
656 CALLFN(·call2048, 2048)
657 CALLFN(·call4096, 4096)
658 CALLFN(·call8192, 8192)
659 CALLFN(·call16384, 16384)
660 CALLFN(·call32768, 32768)
661 CALLFN(·call65536, 65536)
662 CALLFN(·call131072, 131072)
663 CALLFN(·call262144, 262144)
664 CALLFN(·call524288, 524288)
665 CALLFN(·call1048576, 1048576)
666 CALLFN(·call2097152, 2097152)
667 CALLFN(·call4194304, 4194304)
668 CALLFN(·call8388608, 8388608)
669 CALLFN(·call16777216, 16777216)
670 CALLFN(·call33554432, 33554432)
671 CALLFN(·call67108864, 67108864)
672 CALLFN(·call134217728, 134217728)
673 CALLFN(·call268435456, 268435456)
674 CALLFN(·call536870912, 536870912)
675 CALLFN(·call1073741824, 1073741824)
676
677 // The Arm architecture provides a user space accessible counter-timer which
678 // is incremented at a fixed but machine-specific rate. Software can (spin)
679 // wait until the counter-timer reaches some desired value.
680 //
681 // Armv8.7-A introduced the WFET (FEAT_WFxT) instruction, which allows the
682 // processor to enter a low power state for a set time, or until an event is
683 // received.
684 //
685 // However, WFET is not used here because it is only available on newer hardware,
686 // and we aim to maintain compatibility with older Armv8-A platforms that do not
687 // support this feature.
688 //
689 // As a fallback, we can instead use the ISB instruction to decrease processor
690 // activity and thus power consumption between checks of the counter-timer.
691 // Note that we do not depend on the latency of the ISB instruction which is
692 // implementation specific. Actual delay comes from comparing against a fresh
693 // read of the counter-timer value.
694 //
695 // Read more in this Arm blog post:
696 // https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/multi-threaded-applications-arm
697
698 TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0
699 MOVWU cycles+0(FP), R0
700 CBZ R0, done
701 //Prevent speculation of subsequent counter/timer reads and memory accesses.
702 ISB $15
703 // If the delay is very short, just return.
704 // Hardcode 18ns as the first ISB delay.
705 CMP $18, R0
706 BLS done
707 // Adjust for overhead of initial ISB.
708 SUB $18, R0, R0
709 // Convert the delay from nanoseconds to counter/timer ticks.
710 // Read the counter/timer frequency.
711 // delay_ticks = (delay * CNTFRQ_EL0) / 1e9
712 // With the below simplifications and adjustments,
713 // we are usually within 2% of the correct value:
714 // delay_ticks = (delay + delay / 16) * CNTFRQ_EL0 >> 30
715 MRS CNTFRQ_EL0, R1
716 ADD R0>>4, R0, R0
717 MUL R1, R0, R0
718 LSR $30, R0, R0
719 CBZ R0, done
720 // start = current counter/timer value
721 MRS CNTVCT_EL0, R2
722 delay:
723 // Delay using ISB for all ticks.
724 ISB $15
725 // Substract and compare to handle counter roll-over.
726 // counter_read() - start < delay_ticks
727 MRS CNTVCT_EL0, R1
728 SUB R2, R1, R1
729 CMP R0, R1
730 BCC delay
731 done:
732 RET
733
734 // Save state of caller into g->sched,
735 // but using fake PC from systemstack_switch.
736 // Must only be called from functions with no locals ($0)
737 // or else unwinding from systemstack_switch is incorrect.
738 // Smashes R0.
739 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
740 MOVD $runtime·systemstack_switch(SB), R0
741 ADD $8, R0 // get past prologue
742 MOVD R0, (g_sched+gobuf_pc)(g)
743 MOVD RSP, R0
744 MOVD R0, (g_sched+gobuf_sp)(g)
745 MOVD R29, (g_sched+gobuf_bp)(g)
746 MOVD $0, (g_sched+gobuf_lr)(g)
747 // Assert ctxt is zero. See func save.
748 MOVD (g_sched+gobuf_ctxt)(g), R0
749 CBZ R0, 2(PC)
750 CALL runtime·abort(SB)
751 RET
752
753 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
754 // Call fn(arg) aligned appropriately for the gcc ABI.
755 // Called on a system stack, and there may be no g yet (during needm).
756 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
757 MOVD fn+0(FP), R1
758 MOVD arg+8(FP), R0
759 SUB $16, RSP // skip over saved frame pointer below RSP
760 BL (R1)
761 ADD $16, RSP // skip over saved frame pointer below RSP
762 RET
763
764 // func asmcgocall(fn, arg unsafe.Pointer) int32
765 // Call fn(arg) on the scheduler stack,
766 // aligned appropriately for the gcc ABI.
767 // See cgocall.go for more details.
768 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
769 CBZ g, nosave
770
771 // Figure out if we need to switch to m->g0 stack.
772 // We get called to create new OS threads too, and those
773 // come in on the m->g0 stack already. Or we might already
774 // be on the m->gsignal stack.
775 MOVD g_m(g), R8
776 MOVD m_gsignal(R8), R3
777 CMP R3, g
778 BEQ nosave
779 MOVD m_g0(R8), R3
780 CMP R3, g
781 BEQ nosave
782
783 // running on a user stack. Figure out if we're running
784 // secret code and clear our registers if so.
785 #ifdef GOEXPERIMENT_runtimesecret
786 MOVW g_secret(g), R5
787 CBZ R5, nosecret
788 BL ·secretEraseRegisters(SB)
789 // restore g0 back into R3
790 MOVD g_m(g), R3
791 MOVD m_g0(R3), R3
792
793 nosecret:
794 #endif
795 MOVD fn+0(FP), R1
796 MOVD arg+8(FP), R0
797 MOVD RSP, R2
798 MOVD g, R4
799
800 // Switch to system stack.
801 MOVD R0, R9 // gosave_systemstack_switch<> and save_g might clobber R0
802 BL gosave_systemstack_switch<>(SB)
803 MOVD R3, g
804 BL runtime·save_g(SB)
805 MOVD (g_sched+gobuf_sp)(g), R0
806 MOVD R0, RSP
807 MOVD (g_sched+gobuf_bp)(g), R29
808 MOVD R9, R0
809
810 // Now on a scheduling stack (a pthread-created stack).
811 // Save room for two of our pointers /*, plus 32 bytes of callee
812 // save area that lives on the caller stack. */
813 MOVD RSP, R13
814 SUB $16, R13
815 MOVD R13, RSP
816 MOVD R4, 0(RSP) // save old g on stack
817 MOVD (g_stack+stack_hi)(R4), R4
818 SUB R2, R4
819 MOVD R4, 8(RSP) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
820 BL (R1)
821 MOVD R0, R9
822
823 // Restore g, stack pointer. R0 is errno, so don't touch it
824 MOVD 0(RSP), g
825 BL runtime·save_g(SB)
826 MOVD (g_stack+stack_hi)(g), R5
827 MOVD 8(RSP), R6
828 SUB R6, R5
829 MOVD R9, R0
830 MOVD R5, RSP
831
832 MOVW R0, ret+16(FP)
833 RET
834
835 nosave:
836 // Running on a system stack, perhaps even without a g.
837 // Having no g can happen during thread creation or thread teardown
838 // (see needm/dropm on Solaris, for example).
839 // This code is like the above sequence but without saving/restoring g
840 // and without worrying about the stack moving out from under us
841 // (because we're on a system stack, not a goroutine stack).
842 MOVD fn+0(FP), R1
843 MOVD arg+8(FP), R0
844 MOVD RSP, R2
845 MOVD R2, R13
846 SUB $16, R13
847 MOVD R13, RSP
848 MOVD $0, R4
849 MOVD R4, 0(RSP) // Where above code stores g, in case someone looks during debugging.
850 MOVD R2, 8(RSP) // Save original stack pointer.
851 BL (R1)
852 // Restore stack pointer.
853 MOVD 8(RSP), R2
854 MOVD R2, RSP
855 MOVD R0, ret+16(FP)
856 RET
857
858 // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
859 // See cgocall.go for more details.
860 TEXT ·cgocallback(SB),NOSPLIT,$24-24
861 NO_LOCAL_POINTERS
862
863 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
864 // It is used to dropm while thread is exiting.
865 MOVD fn+0(FP), R1
866 CBNZ R1, loadg
867 // Restore the g from frame.
868 MOVD frame+8(FP), g
869 B dropm
870
871 loadg:
872 // Load g from thread-local storage.
873 BL runtime·load_g(SB)
874
875 // If g is nil, Go did not create the current thread,
876 // or if this thread never called into Go on pthread platforms.
877 // Call needm to obtain one for temporary use.
878 // In this case, we're running on the thread stack, so there's
879 // lots of space, but the linker doesn't know. Hide the call from
880 // the linker analysis by using an indirect call.
881 CBZ g, needm
882
883 MOVD g_m(g), R8
884 MOVD R8, savedm-8(SP)
885 B havem
886
887 needm:
888 MOVD g, savedm-8(SP) // g is zero, so is m.
889 MOVD $runtime·needAndBindM(SB), R0
890 BL (R0)
891
892 // Set m->g0->sched.sp = SP, so that if a panic happens
893 // during the function we are about to execute, it will
894 // have a valid SP to run on the g0 stack.
895 // The next few lines (after the havem label)
896 // will save this SP onto the stack and then write
897 // the same SP back to m->sched.sp. That seems redundant,
898 // but if an unrecovered panic happens, unwindm will
899 // restore the g->sched.sp from the stack location
900 // and then systemstack will try to use it. If we don't set it here,
901 // that restored SP will be uninitialized (typically 0) and
902 // will not be usable.
903 MOVD g_m(g), R8
904 MOVD m_g0(R8), R3
905 MOVD RSP, R0
906 MOVD R0, (g_sched+gobuf_sp)(R3)
907 MOVD R29, (g_sched+gobuf_bp)(R3)
908
909 havem:
910 // Now there's a valid m, and we're running on its m->g0.
911 // Save current m->g0->sched.sp on stack and then set it to SP.
912 // Save current sp in m->g0->sched.sp in preparation for
913 // switch back to m->curg stack.
914 // NOTE: unwindm knows that the saved g->sched.sp is at 16(RSP) aka savedsp-16(SP).
915 // Beware that the frame size is actually 32+16.
916 MOVD m_g0(R8), R3
917 MOVD (g_sched+gobuf_sp)(R3), R4
918 MOVD R4, savedsp-16(SP)
919 MOVD RSP, R0
920 MOVD R0, (g_sched+gobuf_sp)(R3)
921
922 // Switch to m->curg stack and call runtime.cgocallbackg.
923 // Because we are taking over the execution of m->curg
924 // but *not* resuming what had been running, we need to
925 // save that information (m->curg->sched) so we can restore it.
926 // We can restore m->curg->sched.sp easily, because calling
927 // runtime.cgocallbackg leaves SP unchanged upon return.
928 // To save m->curg->sched.pc, we push it onto the curg stack and
929 // open a frame the same size as cgocallback's g0 frame.
930 // Once we switch to the curg stack, the pushed PC will appear
931 // to be the return PC of cgocallback, so that the traceback
932 // will seamlessly trace back into the earlier calls.
933 MOVD m_curg(R8), g
934 BL runtime·save_g(SB)
935 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
936 MOVD (g_sched+gobuf_pc)(g), R5
937 MOVD R5, -48(R4)
938 MOVD (g_sched+gobuf_bp)(g), R6
939 MOVD R6, -56(R4)
940
941 // Gather our arguments into registers.
942 MOVD fn+0(FP), R0
943 MOVD frame+8(FP), R1
944 MOVD ctxt+16(FP), R2
945
946 SUB $48, R4 // Allocate the same frame size on the g stack
947 MOVD R4, RSP // switch stack
948 MOVD $runtime·cgocallbackg<ABIInternal>(SB), R11
949 CALL (R11) // indirect call to bypass nosplit check. We're on a different stack now.
950
951 // Restore g->sched (== m->curg->sched) from saved values.
952 MOVD 0(RSP), R5
953 MOVD R5, (g_sched+gobuf_pc)(g)
954 MOVD -8(RSP), R6
955 MOVD R6, (g_sched+gobuf_bp)(g)
956 MOVD RSP, R4
957 ADD $48, R4, R4
958 MOVD R4, (g_sched+gobuf_sp)(g)
959
960 // Switch back to m->g0's stack and restore m->g0->sched.sp.
961 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
962 // so we do not have to restore it.)
963 MOVD g_m(g), R8
964 MOVD m_g0(R8), g
965 BL runtime·save_g(SB)
966 MOVD (g_sched+gobuf_sp)(g), R0
967 MOVD R0, RSP
968 MOVD savedsp-16(SP), R4
969 MOVD R4, (g_sched+gobuf_sp)(g)
970
971 // If the m on entry was nil, we called needm above to borrow an m,
972 // 1. for the duration of the call on non-pthread platforms,
973 // 2. or the duration of the C thread alive on pthread platforms.
974 // If the m on entry wasn't nil,
975 // 1. the thread might be a Go thread,
976 // 2. or it wasn't the first call from a C thread on pthread platforms,
977 // since then we skip dropm to reuse the m in the first call.
978 MOVD savedm-8(SP), R6
979 CBNZ R6, droppedm
980
981 // Skip dropm to reuse it in the next call, when a pthread key has been created.
982 MOVD _cgo_pthread_key_created(SB), R6
983 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
984 CBZ R6, dropm
985 MOVD (R6), R6
986 CBNZ R6, droppedm
987
988 dropm:
989 MOVD $runtime·dropm(SB), R0
990 BL (R0)
991 droppedm:
992
993 // Done!
994 RET
995
996 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
997 // Must obey the gcc calling convention.
998 TEXT _cgo_topofstack(SB),NOSPLIT,$24
999 // g (R28) and REGTMP (R27) might be clobbered by load_g. They
1000 // are callee-save in the gcc calling convention, so save them.
1001 MOVD R27, savedR27-8(SP)
1002 MOVD g, saveG-16(SP)
1003
1004 BL runtime·load_g(SB)
1005 MOVD g_m(g), R0
1006 MOVD m_curg(R0), R0
1007 MOVD (g_stack+stack_hi)(R0), R0
1008
1009 MOVD saveG-16(SP), g
1010 MOVD savedR28-8(SP), R27
1011 RET
1012
1013 // void setg(G*); set g. for use by needm.
1014 TEXT runtime·setg(SB), NOSPLIT, $0-8
1015 MOVD gg+0(FP), g
1016 // This only happens if iscgo, so jump straight to save_g
1017 BL runtime·save_g(SB)
1018 RET
1019
1020 // void setg_gcc(G*); set g called from gcc
1021 TEXT setg_gcc<>(SB),NOSPLIT,$8
1022 MOVD R0, g
1023 MOVD R27, savedR27-8(SP)
1024 BL runtime·save_g(SB)
1025 MOVD savedR27-8(SP), R27
1026 RET
1027
1028 TEXT runtime·emptyfunc(SB),0,$0-0
1029 RET
1030
1031 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
1032 MOVD ZR, R0
1033 MOVD (R0), R0
1034 UNDEF
1035
1036 // The top-most function running on a goroutine
1037 // returns to goexit+PCQuantum.
1038 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
1039 MOVD R0, R0 // NOP
1040 BL runtime·goexit1(SB) // does not return
1041
1042 // This is called from .init_array and follows the platform, not Go, ABI.
1043 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
1044 SUB $0x10, RSP
1045 MOVD R27, 8(RSP) // The access to global variables below implicitly uses R27, which is callee-save
1046 MOVD runtime·lastmoduledatap(SB), R1
1047 MOVD R0, moduledata_next(R1)
1048 MOVD R0, runtime·lastmoduledatap(SB)
1049 MOVD 8(RSP), R27
1050 ADD $0x10, RSP
1051 RET
1052
1053 // gcWriteBarrier informs the GC about heap pointer writes.
1054 //
1055 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
1056 // number of bytes of buffer needed in R25, and returns a pointer
1057 // to the buffer space in R25.
1058 // It clobbers condition codes.
1059 // It does not clobber any general-purpose registers except R27,
1060 // but may clobber others (e.g., floating point registers)
1061 // The act of CALLing gcWriteBarrier will clobber R30 (LR).
1062 TEXT gcWriteBarrier<>(SB),NOSPLIT,$200
1063 // Save the registers clobbered by the fast path.
1064 STP (R0, R1), 184(RSP)
1065 retry:
1066 MOVD g_m(g), R0
1067 MOVD m_p(R0), R0
1068 MOVD (p_wbBuf+wbBuf_next)(R0), R1
1069 MOVD (p_wbBuf+wbBuf_end)(R0), R27
1070 // Increment wbBuf.next position.
1071 ADD R25, R1
1072 // Is the buffer full?
1073 CMP R27, R1
1074 BHI flush
1075 // Commit to the larger buffer.
1076 MOVD R1, (p_wbBuf+wbBuf_next)(R0)
1077 // Make return value (the original next position)
1078 SUB R25, R1, R25
1079 // Restore registers.
1080 LDP 184(RSP), (R0, R1)
1081 RET
1082
1083 flush:
1084 // Save all general purpose registers since these could be
1085 // clobbered by wbBufFlush and were not saved by the caller.
1086 // R0 and R1 already saved
1087 STP (R2, R3), 1*8(RSP)
1088 STP (R4, R5), 3*8(RSP)
1089 STP (R6, R7), 5*8(RSP)
1090 STP (R8, R9), 7*8(RSP)
1091 STP (R10, R11), 9*8(RSP)
1092 STP (R12, R13), 11*8(RSP)
1093 STP (R14, R15), 13*8(RSP)
1094 // R16, R17 may be clobbered by linker trampoline
1095 // R18 is unused.
1096 STP (R19, R20), 15*8(RSP)
1097 STP (R21, R22), 17*8(RSP)
1098 STP (R23, R24), 19*8(RSP)
1099 STP (R25, R26), 21*8(RSP)
1100 // R27 is temp register.
1101 // R28 is g.
1102 // R29 is frame pointer (unused).
1103 // R30 is LR, which was saved by the prologue.
1104 // R31 is SP.
1105
1106 CALL runtime·wbBufFlush(SB)
1107 LDP 1*8(RSP), (R2, R3)
1108 LDP 3*8(RSP), (R4, R5)
1109 LDP 5*8(RSP), (R6, R7)
1110 LDP 7*8(RSP), (R8, R9)
1111 LDP 9*8(RSP), (R10, R11)
1112 LDP 11*8(RSP), (R12, R13)
1113 LDP 13*8(RSP), (R14, R15)
1114 LDP 15*8(RSP), (R19, R20)
1115 LDP 17*8(RSP), (R21, R22)
1116 LDP 19*8(RSP), (R23, R24)
1117 LDP 21*8(RSP), (R25, R26)
1118 JMP retry
1119
1120 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
1121 MOVD $8, R25
1122 JMP gcWriteBarrier<>(SB)
1123 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
1124 MOVD $16, R25
1125 JMP gcWriteBarrier<>(SB)
1126 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
1127 MOVD $24, R25
1128 JMP gcWriteBarrier<>(SB)
1129 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
1130 MOVD $32, R25
1131 JMP gcWriteBarrier<>(SB)
1132 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
1133 MOVD $40, R25
1134 JMP gcWriteBarrier<>(SB)
1135 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
1136 MOVD $48, R25
1137 JMP gcWriteBarrier<>(SB)
1138 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
1139 MOVD $56, R25
1140 JMP gcWriteBarrier<>(SB)
1141 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
1142 MOVD $64, R25
1143 JMP gcWriteBarrier<>(SB)
1144
1145 DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1146 GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1147
1148 // debugCallV2 is the entry point for debugger-injected function
1149 // calls on running goroutines. It informs the runtime that a
1150 // debug call has been injected and creates a call frame for the
1151 // debugger to fill in.
1152 //
1153 // To inject a function call, a debugger should:
1154 // 1. Check that the goroutine is in state _Grunning and that
1155 // there are at least 288 bytes free on the stack.
1156 // 2. Set SP as SP-16.
1157 // 3. Store the current LR in (SP) (using the SP after step 2).
1158 // 4. Store the current PC in the LR register.
1159 // 5. Write the desired argument frame size at SP-16
1160 // 6. Save all machine registers (including flags and fpsimd registers)
1161 // so they can be restored later by the debugger.
1162 // 7. Set the PC to debugCallV2 and resume execution.
1163 //
1164 // If the goroutine is in state _Grunnable, then it's not generally
1165 // safe to inject a call because it may return out via other runtime
1166 // operations. Instead, the debugger should unwind the stack to find
1167 // the return to non-runtime code, add a temporary breakpoint there,
1168 // and inject the call once that breakpoint is hit.
1169 //
1170 // If the goroutine is in any other state, it's not safe to inject a call.
1171 //
1172 // This function communicates back to the debugger by setting R20 and
1173 // invoking BRK to raise a breakpoint signal. Note that the signal PC of
1174 // the signal triggered by the BRK instruction is the PC where the signal
1175 // is trapped, not the next PC, so to resume execution, the debugger needs
1176 // to set the signal PC to PC+4. See the comments in the implementation for
1177 // the protocol the debugger is expected to follow. InjectDebugCall in the
1178 // runtime tests demonstrates this protocol.
1179 //
1180 // The debugger must ensure that any pointers passed to the function
1181 // obey escape analysis requirements. Specifically, it must not pass
1182 // a stack pointer to an escaping argument. debugCallV2 cannot check
1183 // this invariant.
1184 //
1185 // This is ABIInternal because Go code injects its PC directly into new
1186 // goroutine stacks.
1187 TEXT runtime·debugCallV2<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
1188 STP (R29, R30), -280(RSP)
1189 SUB $272, RSP, RSP
1190 SUB $8, RSP, R29
1191 // Save all registers that may contain pointers so they can be
1192 // conservatively scanned.
1193 //
1194 // We can't do anything that might clobber any of these
1195 // registers before this.
1196 STP (R27, g), (30*8)(RSP)
1197 STP (R25, R26), (28*8)(RSP)
1198 STP (R23, R24), (26*8)(RSP)
1199 STP (R21, R22), (24*8)(RSP)
1200 STP (R19, R20), (22*8)(RSP)
1201 STP (R16, R17), (20*8)(RSP)
1202 STP (R14, R15), (18*8)(RSP)
1203 STP (R12, R13), (16*8)(RSP)
1204 STP (R10, R11), (14*8)(RSP)
1205 STP (R8, R9), (12*8)(RSP)
1206 STP (R6, R7), (10*8)(RSP)
1207 STP (R4, R5), (8*8)(RSP)
1208 STP (R2, R3), (6*8)(RSP)
1209 STP (R0, R1), (4*8)(RSP)
1210
1211 // Perform a safe-point check.
1212 MOVD R30, 8(RSP) // Caller's PC
1213 CALL runtime·debugCallCheck(SB)
1214 MOVD 16(RSP), R0
1215 CBZ R0, good
1216
1217 // The safety check failed. Put the reason string at the top
1218 // of the stack.
1219 MOVD R0, 8(RSP)
1220 MOVD 24(RSP), R0
1221 MOVD R0, 16(RSP)
1222
1223 // Set R20 to 8 and invoke BRK. The debugger should get the
1224 // reason a call can't be injected from SP+8 and resume execution.
1225 MOVD $8, R20
1226 BREAK
1227 JMP restore
1228
1229 good:
1230 // Registers are saved and it's safe to make a call.
1231 // Open up a call frame, moving the stack if necessary.
1232 //
1233 // Once the frame is allocated, this will set R20 to 0 and
1234 // invoke BRK. The debugger should write the argument
1235 // frame for the call at SP+8, set up argument registers,
1236 // set the LR as the signal PC + 4, set the PC to the function
1237 // to call, set R26 to point to the closure (if a closure call),
1238 // and resume execution.
1239 //
1240 // If the function returns, this will set R20 to 1 and invoke
1241 // BRK. The debugger can then inspect any return value saved
1242 // on the stack at SP+8 and in registers. To resume execution,
1243 // the debugger should restore the LR from (SP).
1244 //
1245 // If the function panics, this will set R20 to 2 and invoke BRK.
1246 // The interface{} value of the panic will be at SP+8. The debugger
1247 // can inspect the panic value and resume execution again.
1248 #define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1249 CMP $MAXSIZE, R0; \
1250 BGT 5(PC); \
1251 MOVD $NAME(SB), R0; \
1252 MOVD R0, 8(RSP); \
1253 CALL runtime·debugCallWrap(SB); \
1254 JMP restore
1255
1256 MOVD 256(RSP), R0 // the argument frame size
1257 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1258 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1259 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1260 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1261 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1262 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1263 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1264 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1265 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1266 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1267 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1268 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1269 // The frame size is too large. Report the error.
1270 MOVD $debugCallFrameTooLarge<>(SB), R0
1271 MOVD R0, 8(RSP)
1272 MOVD $20, R0
1273 MOVD R0, 16(RSP) // length of debugCallFrameTooLarge string
1274 MOVD $8, R20
1275 BREAK
1276 JMP restore
1277
1278 restore:
1279 // Calls and failures resume here.
1280 //
1281 // Set R20 to 16 and invoke BRK. The debugger should restore
1282 // all registers except for PC and RSP and resume execution.
1283 MOVD $16, R20
1284 BREAK
1285 // We must not modify flags after this point.
1286
1287 // Restore pointer-containing registers, which may have been
1288 // modified from the debugger's copy by stack copying.
1289 LDP (30*8)(RSP), (R27, g)
1290 LDP (28*8)(RSP), (R25, R26)
1291 LDP (26*8)(RSP), (R23, R24)
1292 LDP (24*8)(RSP), (R21, R22)
1293 LDP (22*8)(RSP), (R19, R20)
1294 LDP (20*8)(RSP), (R16, R17)
1295 LDP (18*8)(RSP), (R14, R15)
1296 LDP (16*8)(RSP), (R12, R13)
1297 LDP (14*8)(RSP), (R10, R11)
1298 LDP (12*8)(RSP), (R8, R9)
1299 LDP (10*8)(RSP), (R6, R7)
1300 LDP (8*8)(RSP), (R4, R5)
1301 LDP (6*8)(RSP), (R2, R3)
1302 LDP (4*8)(RSP), (R0, R1)
1303
1304 LDP -8(RSP), (R29, R27)
1305 ADD $288, RSP, RSP // Add 16 more bytes, see saveSigContext
1306 MOVD -16(RSP), R30 // restore old lr
1307 JMP (R27)
1308
1309 // runtime.debugCallCheck assumes that functions defined with the
1310 // DEBUG_CALL_FN macro are safe points to inject calls.
1311 #define DEBUG_CALL_FN(NAME,MAXSIZE) \
1312 TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1313 NO_LOCAL_POINTERS; \
1314 MOVD $0, R20; \
1315 BREAK; \
1316 MOVD $1, R20; \
1317 BREAK; \
1318 RET
1319 DEBUG_CALL_FN(debugCall32<>, 32)
1320 DEBUG_CALL_FN(debugCall64<>, 64)
1321 DEBUG_CALL_FN(debugCall128<>, 128)
1322 DEBUG_CALL_FN(debugCall256<>, 256)
1323 DEBUG_CALL_FN(debugCall512<>, 512)
1324 DEBUG_CALL_FN(debugCall1024<>, 1024)
1325 DEBUG_CALL_FN(debugCall2048<>, 2048)
1326 DEBUG_CALL_FN(debugCall4096<>, 4096)
1327 DEBUG_CALL_FN(debugCall8192<>, 8192)
1328 DEBUG_CALL_FN(debugCall16384<>, 16384)
1329 DEBUG_CALL_FN(debugCall32768<>, 32768)
1330 DEBUG_CALL_FN(debugCall65536<>, 65536)
1331
1332 // func debugCallPanicked(val interface{})
1333 TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16
1334 // Copy the panic value to the top of stack at SP+8.
1335 MOVD val_type+0(FP), R0
1336 MOVD R0, 8(RSP)
1337 MOVD val_data+8(FP), R0
1338 MOVD R0, 16(RSP)
1339 MOVD $2, R20
1340 BREAK
1341 RET
1342
1343 TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-0
1344 NO_LOCAL_POINTERS
1345 // Save all 16 int registers that could have an index in them.
1346 // They may be pointers, but if they are they are dead.
1347 STP (R0, R1), 24(RSP)
1348 STP (R2, R3), 40(RSP)
1349 STP (R4, R5), 56(RSP)
1350 STP (R6, R7), 72(RSP)
1351 STP (R8, R9), 88(RSP)
1352 STP (R10, R11), 104(RSP)
1353 STP (R12, R13), 120(RSP)
1354 STP (R14, R15), 136(RSP)
1355 MOVD LR, R0 // PC immediately after call to panicBounds
1356 ADD $24, RSP, R1 // pointer to save area
1357 CALL runtime·panicBounds64<ABIInternal>(SB)
1358 RET
1359
1360 TEXT ·getfp<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1361 MOVD R29, R0
1362 RET
1363
View as plain text