Text file
src/runtime/asm_ppc64x.s
1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build ppc64 || ppc64le
6
7 #include "go_asm.h"
8 #include "go_tls.h"
9 #include "funcdata.h"
10 #include "textflag.h"
11 #include "asm_ppc64x.h"
12
13 #ifdef GOOS_aix
14 #define cgoCalleeStackSize 48
15 #else
16 #define cgoCalleeStackSize 32
17 #endif
18
19 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
20 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer
21
22 // initialize essential registers
23 BL runtime·reginit(SB)
24
25 SUB $(FIXED_FRAME+16), R1
26 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame
27 MOVW R3, FIXED_FRAME+0(R1) // argc
28 MOVD R4, FIXED_FRAME+8(R1) // argv
29
30 // create istack out of the given (operating system) stack.
31 // _cgo_init may update stackguard.
32 MOVD $runtime·g0(SB), g
33 BL runtime·save_g(SB)
34 MOVD $(-64*1024), R31
35 ADD R31, R1, R3
36 MOVD R3, g_stackguard0(g)
37 MOVD R3, g_stackguard1(g)
38 MOVD R3, (g_stack+stack_lo)(g)
39 MOVD R1, (g_stack+stack_hi)(g)
40
41 // If there is a _cgo_init, call it using the gcc ABI.
42 MOVD _cgo_init(SB), R12
43 CMP R0, R12
44 BEQ nocgo
45
46 #ifdef GO_PPC64X_HAS_FUNCDESC
47 // Load the real entry address from the first slot of the function descriptor.
48 MOVD 8(R12), R2
49 MOVD (R12), R12
50 #endif
51 MOVD R12, CTR // r12 = "global function entry point"
52 MOVD R13, R5 // arg 2: TLS base pointer
53 MOVD $setg_gcc<>(SB), R4 // arg 1: setg
54 MOVD g, R3 // arg 0: G
55 // C functions expect 32 (48 for AIX) bytes of space on caller
56 // stack frame and a 16-byte aligned R1
57 MOVD R1, R14 // save current stack
58 SUB $cgoCalleeStackSize, R1 // reserve the callee area
59 RLDCR $0, R1, $~15, R1 // 16-byte align
60 BL (CTR) // may clobber R0, R3-R12
61 MOVD R14, R1 // restore stack
62 #ifndef GOOS_aix
63 MOVD 24(R1), R2
64 #endif
65 XOR R0, R0 // fix R0
66
67 nocgo:
68 // update stackguard after _cgo_init
69 MOVD (g_stack+stack_lo)(g), R3
70 ADD $const_stackGuard, R3
71 MOVD R3, g_stackguard0(g)
72 MOVD R3, g_stackguard1(g)
73
74 // set the per-goroutine and per-mach "registers"
75 MOVD $runtime·m0(SB), R3
76
77 // save m->g0 = g0
78 MOVD g, m_g0(R3)
79 // save m0 to g0->m
80 MOVD R3, g_m(g)
81
82 BL runtime·check(SB)
83
84 // args are already prepared
85 BL runtime·args(SB)
86 BL runtime·osinit(SB)
87 BL runtime·schedinit(SB)
88
89 // create a new goroutine to start program
90 MOVD $runtime·mainPC(SB), R3 // entry
91 MOVDU R3, -8(R1)
92 MOVDU R0, -8(R1)
93 MOVDU R0, -8(R1)
94 MOVDU R0, -8(R1)
95 MOVDU R0, -8(R1)
96 BL runtime·newproc(SB)
97 ADD $(8+FIXED_FRAME), R1
98
99 // start this M
100 BL runtime·mstart(SB)
101 // Prevent dead-code elimination of debugCallV2, which is
102 // intended to be called by debuggers.
103 #ifdef GOARCH_ppc64le
104 MOVD $runtime·debugCallV2<ABIInternal>(SB), R31
105 #endif
106 MOVD R0, 0(R0)
107 RET
108
109 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
110 GLOBL runtime·mainPC(SB),RODATA,$8
111
112 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
113 TW $31, R0, R0
114 RET
115
116 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
117 RET
118
119 // Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall_ppc64
120 TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0
121 // crosscall_ppc64 and crosscall2 need to reginit, but can't
122 // get at the 'runtime.reginit' symbol.
123 BR runtime·reginit(SB)
124
125 TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0
126 // set R0 to zero, it's expected by the toolchain
127 XOR R0, R0
128 RET
129
130 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
131 BL runtime·mstart0(SB)
132 RET // not reached
133
134 /*
135 * go-routine
136 */
137
138 // void gogo(Gobuf*)
139 // restore state from Gobuf; longjmp
140 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
141 MOVD buf+0(FP), R5
142 MOVD gobuf_g(R5), R6
143 MOVD 0(R6), R4 // make sure g != nil
144 BR gogo<>(SB)
145
146 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
147 MOVD R6, g
148 BL runtime·save_g(SB)
149
150 MOVD gobuf_sp(R5), R1
151 MOVD gobuf_lr(R5), R31
152 #ifndef GOOS_aix
153 MOVD 24(R1), R2 // restore R2
154 #endif
155 MOVD R31, LR
156 MOVD gobuf_ret(R5), R3
157 MOVD gobuf_ctxt(R5), R11
158 MOVD R0, gobuf_sp(R5)
159 MOVD R0, gobuf_ret(R5)
160 MOVD R0, gobuf_lr(R5)
161 MOVD R0, gobuf_ctxt(R5)
162 CMP R0, R0 // set condition codes for == test, needed by stack split
163 MOVD gobuf_pc(R5), R12
164 MOVD R12, CTR
165 BR (CTR)
166
167 // void mcall(fn func(*g))
168 // Switch to m->g0's stack, call fn(g).
169 // Fn must never return. It should gogo(&g->sched)
170 // to keep running g.
171 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
172 // Save caller state in g->sched
173 // R11 should be safe across save_g??
174 MOVD R3, R11
175 MOVD R1, (g_sched+gobuf_sp)(g)
176 MOVD LR, R31
177 MOVD R31, (g_sched+gobuf_pc)(g)
178 MOVD R0, (g_sched+gobuf_lr)(g)
179
180 // Switch to m->g0 & its stack, call fn.
181 MOVD g, R3
182 MOVD g_m(g), R8
183 MOVD m_g0(R8), g
184 BL runtime·save_g(SB)
185 CMP g, R3
186 BNE 2(PC)
187 BR runtime·badmcall(SB)
188 MOVD 0(R11), R12 // code pointer
189 MOVD R12, CTR
190 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp
191 // Don't need to do anything special for regabiargs here
192 // R3 is g; stack is set anyway
193 MOVDU R3, -8(R1)
194 MOVDU R0, -8(R1)
195 MOVDU R0, -8(R1)
196 MOVDU R0, -8(R1)
197 MOVDU R0, -8(R1)
198 BL (CTR)
199 MOVD 24(R1), R2
200 BR runtime·badmcall2(SB)
201
202 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
203 // of the G stack. We need to distinguish the routine that
204 // lives at the bottom of the G stack from the one that lives
205 // at the top of the system stack because the one at the top of
206 // the system stack terminates the stack walk (see topofstack()).
207 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
208 // We have several undefs here so that 16 bytes past
209 // $runtime·systemstack_switch lies within them whether or not the
210 // instructions that derive r2 from r12 are there.
211 UNDEF
212 UNDEF
213 UNDEF
214 BL (LR) // make sure this function is not leaf
215 RET
216
217 // func systemstack(fn func())
218 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
219 MOVD fn+0(FP), R3 // R3 = fn
220 MOVD R3, R11 // context
221 MOVD g_m(g), R4 // R4 = m
222
223 MOVD m_gsignal(R4), R5 // R5 = gsignal
224 CMP g, R5
225 BEQ noswitch
226
227 MOVD m_g0(R4), R5 // R5 = g0
228 CMP g, R5
229 BEQ noswitch
230
231 MOVD m_curg(R4), R6
232 CMP g, R6
233 BEQ switch
234
235 // Bad: g is not gsignal, not g0, not curg. What is it?
236 // Hide call from linker nosplit analysis.
237 MOVD $runtime·badsystemstack(SB), R12
238 MOVD R12, CTR
239 BL (CTR)
240 BL runtime·abort(SB)
241
242 switch:
243 // save our state in g->sched. Pretend to
244 // be systemstack_switch if the G stack is scanned.
245 BL gosave_systemstack_switch<>(SB)
246
247 // switch to g0
248 MOVD R5, g
249 BL runtime·save_g(SB)
250 MOVD (g_sched+gobuf_sp)(g), R1
251
252 // call target function
253 MOVD 0(R11), R12 // code pointer
254 MOVD R12, CTR
255 BL (CTR)
256
257 // restore TOC pointer. It seems unlikely that we will use systemstack
258 // to call a function defined in another module, but the results of
259 // doing so would be so confusing that it's worth doing this.
260 MOVD g_m(g), R3
261 MOVD m_curg(R3), g
262 MOVD (g_sched+gobuf_sp)(g), R3
263 #ifndef GOOS_aix
264 MOVD 24(R3), R2
265 #endif
266 // switch back to g
267 MOVD g_m(g), R3
268 MOVD m_curg(R3), g
269 BL runtime·save_g(SB)
270 MOVD (g_sched+gobuf_sp)(g), R1
271 MOVD R0, (g_sched+gobuf_sp)(g)
272 RET
273
274 noswitch:
275 // already on m stack, just call directly
276 // On other arches we do a tail call here, but it appears to be
277 // impossible to tail call a function pointer in shared mode on
278 // ppc64 because the caller is responsible for restoring the TOC.
279 MOVD 0(R11), R12 // code pointer
280 MOVD R12, CTR
281 BL (CTR)
282 #ifndef GOOS_aix
283 MOVD 24(R1), R2
284 #endif
285 RET
286
287 // func switchToCrashStack0(fn func())
288 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
289 MOVD R3, R11 // context register
290 MOVD g_m(g), R3 // curm
291
292 // set g to gcrash
293 MOVD $runtime·gcrash(SB), g // g = &gcrash
294 CALL runtime·save_g(SB) // clobbers R31
295 MOVD R3, g_m(g) // g.m = curm
296 MOVD g, m_g0(R3) // curm.g0 = g
297
298 // switch to crashstack
299 MOVD (g_stack+stack_hi)(g), R3
300 SUB $(4*8), R3
301 MOVD R3, R1
302
303 // call target function
304 MOVD 0(R11), R12 // code pointer
305 MOVD R12, CTR
306 BL (CTR)
307
308 // should never return
309 CALL runtime·abort(SB)
310 UNDEF
311
312 /*
313 * support for morestack
314 */
315
316 // Called during function prolog when more stack is needed.
317 // Caller has already loaded:
318 // R3: framesize, R4: argsize, R5: LR
319 //
320 // The traceback routines see morestack on a g0 as being
321 // the top of a stack (for example, morestack calling newstack
322 // calling the scheduler calling newm calling gc), so we must
323 // record an argument size. For that purpose, it has no arguments.
324 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
325 // Called from f.
326 // Set g->sched to context in f.
327 MOVD R1, (g_sched+gobuf_sp)(g)
328 MOVD LR, R8
329 MOVD R8, (g_sched+gobuf_pc)(g)
330 MOVD R5, (g_sched+gobuf_lr)(g)
331 MOVD R11, (g_sched+gobuf_ctxt)(g)
332
333 // Cannot grow scheduler stack (m->g0).
334 MOVD g_m(g), R7
335 MOVD m_g0(R7), R8
336 CMP g, R8
337 BNE 3(PC)
338 BL runtime·badmorestackg0(SB)
339 BL runtime·abort(SB)
340
341 // Cannot grow signal stack (m->gsignal).
342 MOVD m_gsignal(R7), R8
343 CMP g, R8
344 BNE 3(PC)
345 BL runtime·badmorestackgsignal(SB)
346 BL runtime·abort(SB)
347
348 // Called from f.
349 // Set m->morebuf to f's caller.
350 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
351 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
352 MOVD g, (m_morebuf+gobuf_g)(R7)
353
354 // Call newstack on m->g0's stack.
355 MOVD m_g0(R7), g
356 BL runtime·save_g(SB)
357 MOVD (g_sched+gobuf_sp)(g), R1
358 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0
359 BL runtime·newstack(SB)
360
361 // Not reached, but make sure the return PC from the call to newstack
362 // is still in this function, and not the beginning of the next.
363 UNDEF
364
365 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
366 // Force SPWRITE. This function doesn't actually write SP,
367 // but it is called with a special calling convention where
368 // the caller doesn't save LR on stack but passes it as a
369 // register (R5), and the unwinder currently doesn't understand.
370 // Make it SPWRITE to stop unwinding. (See issue 54332)
371 // Use OR R0, R1 instead of MOVD R1, R1 as the MOVD instruction
372 // has a special affect on Power8,9,10 by lowering the thread
373 // priority and causing a slowdown in execution time
374
375 OR R0, R1
376 MOVD R0, R11
377 BR runtime·morestack(SB)
378
379 // reflectcall: call a function with the given argument list
380 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
381 // we don't have variable-sized frames, so we use a small number
382 // of constant-sized-frame functions to encode a few bits of size in the pc.
383 // Caution: ugly multiline assembly macros in your future!
384
385 #define DISPATCH(NAME,MAXSIZE) \
386 MOVD $MAXSIZE, R31; \
387 CMP R3, R31; \
388 BGT 4(PC); \
389 MOVD $NAME(SB), R12; \
390 MOVD R12, CTR; \
391 BR (CTR)
392 // Note: can't just "BR NAME(SB)" - bad inlining results.
393
394 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
395 MOVWZ frameSize+32(FP), R3
396 DISPATCH(runtime·call16, 16)
397 DISPATCH(runtime·call32, 32)
398 DISPATCH(runtime·call64, 64)
399 DISPATCH(runtime·call128, 128)
400 DISPATCH(runtime·call256, 256)
401 DISPATCH(runtime·call512, 512)
402 DISPATCH(runtime·call1024, 1024)
403 DISPATCH(runtime·call2048, 2048)
404 DISPATCH(runtime·call4096, 4096)
405 DISPATCH(runtime·call8192, 8192)
406 DISPATCH(runtime·call16384, 16384)
407 DISPATCH(runtime·call32768, 32768)
408 DISPATCH(runtime·call65536, 65536)
409 DISPATCH(runtime·call131072, 131072)
410 DISPATCH(runtime·call262144, 262144)
411 DISPATCH(runtime·call524288, 524288)
412 DISPATCH(runtime·call1048576, 1048576)
413 DISPATCH(runtime·call2097152, 2097152)
414 DISPATCH(runtime·call4194304, 4194304)
415 DISPATCH(runtime·call8388608, 8388608)
416 DISPATCH(runtime·call16777216, 16777216)
417 DISPATCH(runtime·call33554432, 33554432)
418 DISPATCH(runtime·call67108864, 67108864)
419 DISPATCH(runtime·call134217728, 134217728)
420 DISPATCH(runtime·call268435456, 268435456)
421 DISPATCH(runtime·call536870912, 536870912)
422 DISPATCH(runtime·call1073741824, 1073741824)
423 MOVD $runtime·badreflectcall(SB), R12
424 MOVD R12, CTR
425 BR (CTR)
426
427 #define CALLFN(NAME,MAXSIZE) \
428 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
429 NO_LOCAL_POINTERS; \
430 /* copy arguments to stack */ \
431 MOVD stackArgs+16(FP), R3; \
432 MOVWZ stackArgsSize+24(FP), R4; \
433 MOVD R1, R5; \
434 CMP R4, $8; \
435 BLT tailsetup; \
436 /* copy 8 at a time if possible */ \
437 ADD $(FIXED_FRAME-8), R5; \
438 SUB $8, R3; \
439 top: \
440 MOVDU 8(R3), R7; \
441 MOVDU R7, 8(R5); \
442 SUB $8, R4; \
443 CMP R4, $8; \
444 BGE top; \
445 /* handle remaining bytes */ \
446 CMP $0, R4; \
447 BEQ callfn; \
448 ADD $7, R3; \
449 ADD $7, R5; \
450 BR tail; \
451 tailsetup: \
452 CMP $0, R4; \
453 BEQ callfn; \
454 ADD $(FIXED_FRAME-1), R5; \
455 SUB $1, R3; \
456 tail: \
457 MOVBU 1(R3), R6; \
458 MOVBU R6, 1(R5); \
459 SUB $1, R4; \
460 CMP $0, R4; \
461 BGT tail; \
462 callfn: \
463 /* call function */ \
464 MOVD f+8(FP), R11; \
465 #ifdef GOOS_aix \
466 /* AIX won't trigger a SIGSEGV if R11 = nil */ \
467 /* So it manually triggers it */ \
468 CMP R0, R11 \
469 BNE 2(PC) \
470 MOVD R0, 0(R0) \
471 #endif \
472 MOVD regArgs+40(FP), R20; \
473 BL runtime·unspillArgs(SB); \
474 MOVD (R11), R12; \
475 MOVD R12, CTR; \
476 PCDATA $PCDATA_StackMapIndex, $0; \
477 BL (CTR); \
478 #ifndef GOOS_aix \
479 MOVD 24(R1), R2; \
480 #endif \
481 /* copy return values back */ \
482 MOVD regArgs+40(FP), R20; \
483 BL runtime·spillArgs(SB); \
484 MOVD stackArgsType+0(FP), R7; \
485 MOVD stackArgs+16(FP), R3; \
486 MOVWZ stackArgsSize+24(FP), R4; \
487 MOVWZ stackRetOffset+28(FP), R6; \
488 ADD $FIXED_FRAME, R1, R5; \
489 ADD R6, R5; \
490 ADD R6, R3; \
491 SUB R6, R4; \
492 BL callRet<>(SB); \
493 RET
494
495 // callRet copies return values back at the end of call*. This is a
496 // separate function so it can allocate stack space for the arguments
497 // to reflectcallmove. It does not follow the Go ABI; it expects its
498 // arguments in registers.
499 TEXT callRet<>(SB), NOSPLIT, $40-0
500 NO_LOCAL_POINTERS
501 MOVD R7, FIXED_FRAME+0(R1)
502 MOVD R3, FIXED_FRAME+8(R1)
503 MOVD R5, FIXED_FRAME+16(R1)
504 MOVD R4, FIXED_FRAME+24(R1)
505 MOVD R20, FIXED_FRAME+32(R1)
506 BL runtime·reflectcallmove(SB)
507 RET
508
509 CALLFN(·call16, 16)
510 CALLFN(·call32, 32)
511 CALLFN(·call64, 64)
512 CALLFN(·call128, 128)
513 CALLFN(·call256, 256)
514 CALLFN(·call512, 512)
515 CALLFN(·call1024, 1024)
516 CALLFN(·call2048, 2048)
517 CALLFN(·call4096, 4096)
518 CALLFN(·call8192, 8192)
519 CALLFN(·call16384, 16384)
520 CALLFN(·call32768, 32768)
521 CALLFN(·call65536, 65536)
522 CALLFN(·call131072, 131072)
523 CALLFN(·call262144, 262144)
524 CALLFN(·call524288, 524288)
525 CALLFN(·call1048576, 1048576)
526 CALLFN(·call2097152, 2097152)
527 CALLFN(·call4194304, 4194304)
528 CALLFN(·call8388608, 8388608)
529 CALLFN(·call16777216, 16777216)
530 CALLFN(·call33554432, 33554432)
531 CALLFN(·call67108864, 67108864)
532 CALLFN(·call134217728, 134217728)
533 CALLFN(·call268435456, 268435456)
534 CALLFN(·call536870912, 536870912)
535 CALLFN(·call1073741824, 1073741824)
536
537 TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0-4
538 MOVW cycles+0(FP), R7
539 // POWER does not have a pause/yield instruction equivalent.
540 // Instead, we can lower the program priority by setting the
541 // Program Priority Register prior to the wait loop and set it
542 // back to default afterwards. On Linux, the default priority is
543 // medium-low. For details, see page 837 of the ISA 3.0.
544 OR R1, R1, R1 // Set PPR priority to low
545 again:
546 SUB $1, R7
547 CMP $0, R7
548 BNE again
549 OR R6, R6, R6 // Set PPR priority back to medium-low
550 RET
551
552 // Save state of caller into g->sched,
553 // but using fake PC from systemstack_switch.
554 // Must only be called from functions with no locals ($0)
555 // or else unwinding from systemstack_switch is incorrect.
556 // Smashes R31.
557 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
558 MOVD $runtime·systemstack_switch(SB), R31
559 ADD $16, R31 // get past prologue (including r2-setting instructions when they're there)
560 MOVD R31, (g_sched+gobuf_pc)(g)
561 MOVD R1, (g_sched+gobuf_sp)(g)
562 MOVD R0, (g_sched+gobuf_lr)(g)
563 MOVD R0, (g_sched+gobuf_ret)(g)
564 // Assert ctxt is zero. See func save.
565 MOVD (g_sched+gobuf_ctxt)(g), R31
566 CMP R0, R31
567 BEQ 2(PC)
568 BL runtime·abort(SB)
569 RET
570
571 #ifdef GOOS_aix
572 #define asmcgocallSaveOffset cgoCalleeStackSize + 8
573 #else
574 #define asmcgocallSaveOffset cgoCalleeStackSize
575 #endif
576
577 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
578 // Call fn(arg) aligned appropriately for the gcc ABI.
579 // Called on a system stack, and there may be no g yet (during needm).
580 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
581 MOVD fn+0(FP), R3
582 MOVD arg+8(FP), R4
583
584 MOVD R1, R15
585 SUB $(asmcgocallSaveOffset+8), R1
586 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
587 MOVD R15, asmcgocallSaveOffset(R1)
588
589 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
590
591 // This is a "global call", so put the global entry point in r12
592 MOVD R3, R12
593
594 #ifdef GO_PPC64X_HAS_FUNCDESC
595 // Load the real entry address from the first slot of the function descriptor.
596 MOVD 8(R12), R2
597 MOVD (R12), R12
598 #endif
599 MOVD R12, CTR
600 MOVD R4, R3 // arg in r3
601 BL (CTR)
602
603 // C code can clobber R0, so set it back to 0. F27-F31 are
604 // callee save, so we don't need to recover those.
605 XOR R0, R0
606
607 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
608 #ifndef GOOS_aix
609 MOVD 24(R1), R2
610 #endif
611
612 RET
613
614 // func asmcgocall(fn, arg unsafe.Pointer) int32
615 // Call fn(arg) on the scheduler stack,
616 // aligned appropriately for the gcc ABI.
617 // See cgocall.go for more details.
618 TEXT ·asmcgocall<ABIInternal>(SB),NOSPLIT,$0-20
619 // R3 = fn
620 // R4 = arg
621
622 MOVD R1, R7 // save original stack pointer
623 CMP $0, g
624 BEQ nosave
625 MOVD g, R5
626
627 // Figure out if we need to switch to m->g0 stack.
628 // We get called to create new OS threads too, and those
629 // come in on the m->g0 stack already. Or we might already
630 // be on the m->gsignal stack.
631 MOVD g_m(g), R8
632 MOVD m_gsignal(R8), R6
633 CMP R6, g
634 BEQ nosave
635 MOVD m_g0(R8), R6
636 CMP R6, g
637 BEQ nosave
638
639 BL gosave_systemstack_switch<>(SB)
640 MOVD R6, g
641 BL runtime·save_g(SB)
642 MOVD (g_sched+gobuf_sp)(g), R1
643
644 // Now on a scheduling stack (a pthread-created stack).
645 #ifdef GOOS_aix
646 // Create a fake LR to improve backtrace.
647 MOVD $runtime·asmcgocall(SB), R6
648 MOVD R6, 16(R1)
649 // AIX also saves one argument on the stack.
650 SUB $8, R1
651 #endif
652 // Save room for two of our pointers, plus the callee
653 // save area that lives on the caller stack.
654 // Do arithmetics in R10 to hide from the assembler
655 // counting it as SP delta, which is irrelevant as we are
656 // on the system stack.
657 SUB $(asmcgocallSaveOffset+16), R1, R10
658 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
659 MOVD R5, (asmcgocallSaveOffset+8)(R1) // save old g on stack
660 MOVD (g_stack+stack_hi)(R5), R5
661 SUB R7, R5
662 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
663 #ifdef GOOS_aix
664 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb
665 #else
666 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
667 #endif
668 // This is a "global call", so put the global entry point in r12
669 MOVD R3, R12
670
671 #ifdef GO_PPC64X_HAS_FUNCDESC
672 // Load the real entry address from the first slot of the function descriptor.
673 MOVD 8(R12), R2
674 MOVD (R12), R12
675 #endif
676 MOVD R12, CTR
677 MOVD R4, R3 // arg in r3
678 BL (CTR)
679
680 // Reinitialise zero value register.
681 XOR R0, R0
682
683 // Restore g, stack pointer, toc pointer.
684 // R3 is errno, so don't touch it
685 MOVD (asmcgocallSaveOffset+8)(R1), g
686 MOVD (g_stack+stack_hi)(g), R5
687 MOVD asmcgocallSaveOffset(R1), R6
688 SUB R6, R5
689 #ifndef GOOS_aix
690 MOVD 24(R5), R2
691 #endif
692 MOVD R5, R1
693 BL runtime·save_g(SB)
694
695 // ret = R3
696 RET
697
698 nosave:
699 // Running on a system stack, perhaps even without a g.
700 // Having no g can happen during thread creation or thread teardown.
701 // This code is like the above sequence but without saving/restoring g
702 // and without worrying about the stack moving out from under us
703 // (because we're on a system stack, not a goroutine stack).
704 // The above code could be used directly if already on a system stack,
705 // but then the only path through this code would be a rare case.
706 // Using this code for all "already on system stack" calls exercises it more,
707 // which should help keep it correct.
708
709 SUB $(asmcgocallSaveOffset+8), R1, R10
710 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
711 MOVD R7, asmcgocallSaveOffset(R1) // Save original stack pointer.
712
713 MOVD R3, R12 // fn
714 #ifdef GO_PPC64X_HAS_FUNCDESC
715 // Load the real entry address from the first slot of the function descriptor.
716 MOVD 8(R12), R2
717 MOVD (R12), R12
718 #endif
719 MOVD R12, CTR
720 MOVD R4, R3 // arg
721 BL (CTR)
722
723 // Reinitialise zero value register.
724 XOR R0, R0
725
726 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
727 #ifndef GOOS_aix
728 MOVD 24(R1), R2
729 #endif
730 // ret = R3
731 RET
732
733 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
734 // See cgocall.go for more details.
735 TEXT ·cgocallback(SB),NOSPLIT,$24-24
736 NO_LOCAL_POINTERS
737
738 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
739 // It is used to dropm while thread is exiting.
740 MOVD fn+0(FP), R5
741 CMP R5, $0
742 BNE loadg
743 // Restore the g from frame.
744 MOVD frame+8(FP), g
745 BR dropm
746
747 loadg:
748 // Load m and g from thread-local storage.
749 #ifndef GOOS_openbsd
750 MOVBZ runtime·iscgo(SB), R3
751 CMP R3, $0
752 BEQ nocgo
753 #endif
754 BL runtime·load_g(SB)
755 nocgo:
756
757 // If g is nil, Go did not create the current thread,
758 // or if this thread never called into Go on pthread platforms.
759 // Call needm to obtain one for temporary use.
760 // In this case, we're running on the thread stack, so there's
761 // lots of space, but the linker doesn't know. Hide the call from
762 // the linker analysis by using an indirect call.
763 CMP g, $0
764 BEQ needm
765
766 MOVD g_m(g), R8
767 MOVD R8, savedm-8(SP)
768 BR havem
769
770 needm:
771 MOVD g, savedm-8(SP) // g is zero, so is m.
772 MOVD $runtime·needAndBindM(SB), R12
773 MOVD R12, CTR
774 BL (CTR)
775
776 // Set m->sched.sp = SP, so that if a panic happens
777 // during the function we are about to execute, it will
778 // have a valid SP to run on the g0 stack.
779 // The next few lines (after the havem label)
780 // will save this SP onto the stack and then write
781 // the same SP back to m->sched.sp. That seems redundant,
782 // but if an unrecovered panic happens, unwindm will
783 // restore the g->sched.sp from the stack location
784 // and then systemstack will try to use it. If we don't set it here,
785 // that restored SP will be uninitialized (typically 0) and
786 // will not be usable.
787 MOVD g_m(g), R8
788 MOVD m_g0(R8), R3
789 MOVD R1, (g_sched+gobuf_sp)(R3)
790
791 havem:
792 // Now there's a valid m, and we're running on its m->g0.
793 // Save current m->g0->sched.sp on stack and then set it to SP.
794 // Save current sp in m->g0->sched.sp in preparation for
795 // switch back to m->curg stack.
796 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
797 MOVD m_g0(R8), R3
798 MOVD (g_sched+gobuf_sp)(R3), R4
799 MOVD R4, savedsp-24(SP) // must match frame size
800 MOVD R1, (g_sched+gobuf_sp)(R3)
801
802 // Switch to m->curg stack and call runtime.cgocallbackg.
803 // Because we are taking over the execution of m->curg
804 // but *not* resuming what had been running, we need to
805 // save that information (m->curg->sched) so we can restore it.
806 // We can restore m->curg->sched.sp easily, because calling
807 // runtime.cgocallbackg leaves SP unchanged upon return.
808 // To save m->curg->sched.pc, we push it onto the curg stack and
809 // open a frame the same size as cgocallback's g0 frame.
810 // Once we switch to the curg stack, the pushed PC will appear
811 // to be the return PC of cgocallback, so that the traceback
812 // will seamlessly trace back into the earlier calls.
813 MOVD m_curg(R8), g
814 BL runtime·save_g(SB)
815 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
816 MOVD (g_sched+gobuf_pc)(g), R5
817 MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size
818 // Gather our arguments into registers.
819 MOVD fn+0(FP), R5
820 MOVD frame+8(FP), R6
821 MOVD ctxt+16(FP), R7
822 MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size
823 MOVD R5, FIXED_FRAME+0(R1)
824 MOVD R6, FIXED_FRAME+8(R1)
825 MOVD R7, FIXED_FRAME+16(R1)
826
827 MOVD $runtime·cgocallbackg(SB), R12
828 MOVD R12, CTR
829 CALL (CTR) // indirect call to bypass nosplit check. We're on a different stack now.
830
831 // Restore g->sched (== m->curg->sched) from saved values.
832 MOVD 0(R1), R5
833 MOVD R5, (g_sched+gobuf_pc)(g)
834 MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size
835 MOVD R4, (g_sched+gobuf_sp)(g)
836
837 // Switch back to m->g0's stack and restore m->g0->sched.sp.
838 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
839 // so we do not have to restore it.)
840 MOVD g_m(g), R8
841 MOVD m_g0(R8), g
842 BL runtime·save_g(SB)
843 MOVD (g_sched+gobuf_sp)(g), R1
844 MOVD savedsp-24(SP), R4 // must match frame size
845 MOVD R4, (g_sched+gobuf_sp)(g)
846
847 // If the m on entry was nil, we called needm above to borrow an m,
848 // 1. for the duration of the call on non-pthread platforms,
849 // 2. or the duration of the C thread alive on pthread platforms.
850 // If the m on entry wasn't nil,
851 // 1. the thread might be a Go thread,
852 // 2. or it wasn't the first call from a C thread on pthread platforms,
853 // since then we skip dropm to reuse the m in the first call.
854 MOVD savedm-8(SP), R6
855 CMP R6, $0
856 BNE droppedm
857
858 // Skip dropm to reuse it in the next call, when a pthread key has been created.
859 MOVD _cgo_pthread_key_created(SB), R6
860 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
861 CMP R6, $0
862 BEQ dropm
863 MOVD (R6), R6
864 CMP R6, $0
865 BNE droppedm
866
867 dropm:
868 MOVD $runtime·dropm(SB), R12
869 MOVD R12, CTR
870 BL (CTR)
871 droppedm:
872
873 // Done!
874 RET
875
876 // void setg(G*); set g. for use by needm.
877 TEXT runtime·setg(SB), NOSPLIT, $0-8
878 MOVD gg+0(FP), g
879 // This only happens if iscgo, so jump straight to save_g
880 BL runtime·save_g(SB)
881 RET
882
883 #ifdef GO_PPC64X_HAS_FUNCDESC
884 DEFINE_PPC64X_FUNCDESC(setg_gcc<>, _setg_gcc<>)
885 TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
886 #else
887 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
888 #endif
889 // The standard prologue clobbers R31, which is callee-save in
890 // the C ABI, so we have to use $-8-0 and save LR ourselves.
891 MOVD LR, R4
892 // Also save g and R31, since they're callee-save in C ABI
893 MOVD R31, R5
894 MOVD g, R6
895
896 MOVD R3, g
897 BL runtime·save_g(SB)
898
899 MOVD R6, g
900 MOVD R5, R31
901 MOVD R4, LR
902 RET
903
904 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
905 MOVW (R0), R0
906 UNDEF
907
908 #define TBR 268
909
910 // int64 runtime·cputicks(void)
911 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
912 MOVD SPR(TBR), R3
913 MOVD R3, ret+0(FP)
914 RET
915
916 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
917 TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
918 MOVD R3, 0(R20)
919 MOVD R4, 8(R20)
920 MOVD R5, 16(R20)
921 MOVD R6, 24(R20)
922 MOVD R7, 32(R20)
923 MOVD R8, 40(R20)
924 MOVD R9, 48(R20)
925 MOVD R10, 56(R20)
926 MOVD R14, 64(R20)
927 MOVD R15, 72(R20)
928 MOVD R16, 80(R20)
929 MOVD R17, 88(R20)
930 FMOVD F1, 96(R20)
931 FMOVD F2, 104(R20)
932 FMOVD F3, 112(R20)
933 FMOVD F4, 120(R20)
934 FMOVD F5, 128(R20)
935 FMOVD F6, 136(R20)
936 FMOVD F7, 144(R20)
937 FMOVD F8, 152(R20)
938 FMOVD F9, 160(R20)
939 FMOVD F10, 168(R20)
940 FMOVD F11, 176(R20)
941 FMOVD F12, 184(R20)
942 RET
943
944 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
945 TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
946 MOVD 0(R20), R3
947 MOVD 8(R20), R4
948 MOVD 16(R20), R5
949 MOVD 24(R20), R6
950 MOVD 32(R20), R7
951 MOVD 40(R20), R8
952 MOVD 48(R20), R9
953 MOVD 56(R20), R10
954 MOVD 64(R20), R14
955 MOVD 72(R20), R15
956 MOVD 80(R20), R16
957 MOVD 88(R20), R17
958 FMOVD 96(R20), F1
959 FMOVD 104(R20), F2
960 FMOVD 112(R20), F3
961 FMOVD 120(R20), F4
962 FMOVD 128(R20), F5
963 FMOVD 136(R20), F6
964 FMOVD 144(R20), F7
965 FMOVD 152(R20), F8
966 FMOVD 160(R20), F9
967 FMOVD 168(R20), F10
968 FMOVD 176(R20), F11
969 FMOVD 184(R20), F12
970 RET
971
972 // AES hashing not implemented for ppc64
973 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
974 JMP runtime·memhashFallback<ABIInternal>(SB)
975 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
976 JMP runtime·strhashFallback<ABIInternal>(SB)
977 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
978 JMP runtime·memhash32Fallback<ABIInternal>(SB)
979 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
980 JMP runtime·memhash64Fallback<ABIInternal>(SB)
981
982 TEXT runtime·return0(SB), NOSPLIT, $0
983 MOVW $0, R3
984 RET
985
986 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
987 // Must obey the gcc calling convention.
988 #ifdef GOOS_aix
989 // On AIX, _cgo_topofstack is defined in runtime/cgo, because it must
990 // be a longcall in order to prevent trampolines from ld.
991 TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
992 #else
993 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
994 #endif
995 // g (R30) and R31 are callee-save in the C ABI, so save them
996 MOVD g, R4
997 MOVD R31, R5
998 MOVD LR, R6
999
1000 BL runtime·load_g(SB) // clobbers g (R30), R31
1001 MOVD g_m(g), R3
1002 MOVD m_curg(R3), R3
1003 MOVD (g_stack+stack_hi)(R3), R3
1004
1005 MOVD R4, g
1006 MOVD R5, R31
1007 MOVD R6, LR
1008 RET
1009
1010 // The top-most function running on a goroutine
1011 // returns to goexit+PCQuantum.
1012 //
1013 // When dynamically linking Go, it can be returned to from a function
1014 // implemented in a different module and so needs to reload the TOC pointer
1015 // from the stack (although this function declares that it does not set up x-a
1016 // frame, newproc1 does in fact allocate one for goexit and saves the TOC
1017 // pointer in the correct place).
1018 // goexit+_PCQuantum is halfway through the usual global entry point prologue
1019 // that derives r2 from r12 which is a bit silly, but not harmful.
1020 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
1021 MOVD 24(R1), R2
1022 BL runtime·goexit1(SB) // does not return
1023 // traceback from goexit1 must hit code range of goexit
1024 MOVD R0, R0 // NOP
1025
1026 // prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the
1027 // module containing runtime) to the frame that goexit will execute in when
1028 // the goroutine exits. It's implemented in assembly mainly because that's the
1029 // easiest way to get access to R2.
1030 TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8
1031 MOVD sp+0(FP), R3
1032 MOVD R2, 24(R3)
1033 RET
1034
1035 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
1036 ADD $-8, R1
1037 MOVD R31, 0(R1)
1038 MOVD runtime·lastmoduledatap(SB), R4
1039 MOVD R3, moduledata_next(R4)
1040 MOVD R3, runtime·lastmoduledatap(SB)
1041 MOVD 0(R1), R31
1042 ADD $8, R1
1043 RET
1044
1045 TEXT ·checkASM(SB),NOSPLIT,$0-1
1046 MOVW $1, R3
1047 MOVB R3, ret+0(FP)
1048 RET
1049
1050 // gcWriteBarrier informs the GC about heap pointer writes.
1051 //
1052 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
1053 // number of bytes of buffer needed in R29, and returns a pointer
1054 // to the buffer space in R29.
1055 // It clobbers condition codes.
1056 // It does not clobber R0 through R17 (except special registers),
1057 // but may clobber any other register, *including* R31.
1058 TEXT gcWriteBarrier<>(SB),NOSPLIT,$120
1059 // The standard prologue clobbers R31.
1060 // We use R18, R19, and R31 as scratch registers.
1061 retry:
1062 MOVD g_m(g), R18
1063 MOVD m_p(R18), R18
1064 MOVD (p_wbBuf+wbBuf_next)(R18), R19
1065 MOVD (p_wbBuf+wbBuf_end)(R18), R31
1066 // Increment wbBuf.next position.
1067 ADD R29, R19
1068 // Is the buffer full?
1069 CMPU R31, R19
1070 BLT flush
1071 // Commit to the larger buffer.
1072 MOVD R19, (p_wbBuf+wbBuf_next)(R18)
1073 // Make return value (the original next position)
1074 SUB R29, R19, R29
1075 RET
1076
1077 flush:
1078 // Save registers R0 through R15 since these were not saved by the caller.
1079 // We don't save all registers on ppc64 because it takes too much space.
1080 MOVD R20, (FIXED_FRAME+0)(R1)
1081 MOVD R21, (FIXED_FRAME+8)(R1)
1082 // R0 is always 0, so no need to spill.
1083 // R1 is SP.
1084 // R2 is SB.
1085 MOVD R3, (FIXED_FRAME+16)(R1)
1086 MOVD R4, (FIXED_FRAME+24)(R1)
1087 MOVD R5, (FIXED_FRAME+32)(R1)
1088 MOVD R6, (FIXED_FRAME+40)(R1)
1089 MOVD R7, (FIXED_FRAME+48)(R1)
1090 MOVD R8, (FIXED_FRAME+56)(R1)
1091 MOVD R9, (FIXED_FRAME+64)(R1)
1092 MOVD R10, (FIXED_FRAME+72)(R1)
1093 // R11, R12 may be clobbered by external-linker-inserted trampoline
1094 // R13 is REGTLS
1095 MOVD R14, (FIXED_FRAME+80)(R1)
1096 MOVD R15, (FIXED_FRAME+88)(R1)
1097 MOVD R16, (FIXED_FRAME+96)(R1)
1098 MOVD R17, (FIXED_FRAME+104)(R1)
1099 MOVD R29, (FIXED_FRAME+112)(R1)
1100
1101 CALL runtime·wbBufFlush(SB)
1102
1103 MOVD (FIXED_FRAME+0)(R1), R20
1104 MOVD (FIXED_FRAME+8)(R1), R21
1105 MOVD (FIXED_FRAME+16)(R1), R3
1106 MOVD (FIXED_FRAME+24)(R1), R4
1107 MOVD (FIXED_FRAME+32)(R1), R5
1108 MOVD (FIXED_FRAME+40)(R1), R6
1109 MOVD (FIXED_FRAME+48)(R1), R7
1110 MOVD (FIXED_FRAME+56)(R1), R8
1111 MOVD (FIXED_FRAME+64)(R1), R9
1112 MOVD (FIXED_FRAME+72)(R1), R10
1113 MOVD (FIXED_FRAME+80)(R1), R14
1114 MOVD (FIXED_FRAME+88)(R1), R15
1115 MOVD (FIXED_FRAME+96)(R1), R16
1116 MOVD (FIXED_FRAME+104)(R1), R17
1117 MOVD (FIXED_FRAME+112)(R1), R29
1118 JMP retry
1119
1120 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
1121 MOVD $8, R29
1122 JMP gcWriteBarrier<>(SB)
1123 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
1124 MOVD $16, R29
1125 JMP gcWriteBarrier<>(SB)
1126 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
1127 MOVD $24, R29
1128 JMP gcWriteBarrier<>(SB)
1129 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
1130 MOVD $32, R29
1131 JMP gcWriteBarrier<>(SB)
1132 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
1133 MOVD $40, R29
1134 JMP gcWriteBarrier<>(SB)
1135 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
1136 MOVD $48, R29
1137 JMP gcWriteBarrier<>(SB)
1138 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
1139 MOVD $56, R29
1140 JMP gcWriteBarrier<>(SB)
1141 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
1142 MOVD $64, R29
1143 JMP gcWriteBarrier<>(SB)
1144
1145 DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1146 GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1147
1148 // debugCallV2 is the entry point for debugger-injected function
1149 // calls on running goroutines. It informs the runtime that a
1150 // debug call has been injected and creates a call frame for the
1151 // debugger to fill in.
1152 //
1153 // To inject a function call, a debugger should:
1154 // 1. Check that the goroutine is in state _Grunning and that
1155 // there are at least 320 bytes free on the stack.
1156 // 2. Set SP as SP-32.
1157 // 3. Store the current LR in (SP) (using the SP after step 2).
1158 // 4. Store the current PC in the LR register.
1159 // 5. Write the desired argument frame size at SP-32
1160 // 6. Save all machine registers (including flags and floating point registers)
1161 // so they can be restored later by the debugger.
1162 // 7. Set the PC to debugCallV2 and resume execution.
1163 //
1164 // If the goroutine is in state _Grunnable, then it's not generally
1165 // safe to inject a call because it may return out via other runtime
1166 // operations. Instead, the debugger should unwind the stack to find
1167 // the return to non-runtime code, add a temporary breakpoint there,
1168 // and inject the call once that breakpoint is hit.
1169 //
1170 // If the goroutine is in any other state, it's not safe to inject a call.
1171 //
1172 // This function communicates back to the debugger by setting R20 and
1173 // invoking TW to raise a breakpoint signal. Note that the signal PC of
1174 // the signal triggered by the TW instruction is the PC where the signal
1175 // is trapped, not the next PC, so to resume execution, the debugger needs
1176 // to set the signal PC to PC+4. See the comments in the implementation for
1177 // the protocol the debugger is expected to follow. InjectDebugCall in the
1178 // runtime tests demonstrates this protocol.
1179 // The debugger must ensure that any pointers passed to the function
1180 // obey escape analysis requirements. Specifically, it must not pass
1181 // a stack pointer to an escaping argument. debugCallV2 cannot check
1182 // this invariant.
1183 //
1184 // This is ABIInternal because Go code injects its PC directly into new
1185 // goroutine stacks.
1186 #ifdef GOARCH_ppc64le
1187 TEXT runtime·debugCallV2<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-0
1188 // save scratch register R31 first
1189 MOVD R31, -184(R1)
1190 MOVD 0(R1), R31
1191 // save caller LR
1192 MOVD R31, -304(R1)
1193 MOVD -32(R1), R31
1194 // save argument frame size
1195 MOVD R31, -192(R1)
1196 MOVD LR, R31
1197 MOVD R31, -320(R1)
1198 ADD $-320, R1
1199 // save all registers that can contain pointers
1200 // and the CR register
1201 MOVW CR, R31
1202 MOVD R31, 8(R1)
1203 MOVD R2, 24(R1)
1204 MOVD R3, 56(R1)
1205 MOVD R4, 64(R1)
1206 MOVD R5, 72(R1)
1207 MOVD R6, 80(R1)
1208 MOVD R7, 88(R1)
1209 MOVD R8, 96(R1)
1210 MOVD R9, 104(R1)
1211 MOVD R10, 112(R1)
1212 MOVD R11, 120(R1)
1213 MOVD R12, 144(R1)
1214 MOVD R13, 152(R1)
1215 MOVD R14, 160(R1)
1216 MOVD R15, 168(R1)
1217 MOVD R16, 176(R1)
1218 MOVD R17, 184(R1)
1219 MOVD R18, 192(R1)
1220 MOVD R19, 200(R1)
1221 MOVD R20, 208(R1)
1222 MOVD R21, 216(R1)
1223 MOVD R22, 224(R1)
1224 MOVD R23, 232(R1)
1225 MOVD R24, 240(R1)
1226 MOVD R25, 248(R1)
1227 MOVD R26, 256(R1)
1228 MOVD R27, 264(R1)
1229 MOVD R28, 272(R1)
1230 MOVD R29, 280(R1)
1231 MOVD g, 288(R1)
1232 MOVD LR, R31
1233 MOVD R31, 32(R1)
1234 CALL runtime·debugCallCheck(SB)
1235 MOVD 40(R1), R22
1236 XOR R0, R0
1237 CMP R22, R0
1238 BEQ good
1239 MOVD 48(R1), R22
1240 MOVD $8, R20
1241 TW $31, R0, R0
1242
1243 BR restore
1244
1245 good:
1246 #define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1247 MOVD $MAXSIZE, R23; \
1248 CMP R26, R23; \
1249 BGT 5(PC); \
1250 MOVD $NAME(SB), R26; \
1251 MOVD R26, 32(R1); \
1252 CALL runtime·debugCallWrap(SB); \
1253 BR restore
1254
1255 // the argument frame size
1256 MOVD 128(R1), R26
1257
1258 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1259 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1260 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1261 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1262 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1263 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1264 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1265 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1266 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1267 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1268 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1269 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1270 // The frame size is too large. Report the error.
1271 MOVD $debugCallFrameTooLarge<>(SB), R22
1272 MOVD R22, 32(R1)
1273 MOVD $20, R22
1274 // length of debugCallFrameTooLarge string
1275 MOVD R22, 40(R1)
1276 MOVD $8, R20
1277 TW $31, R0, R0
1278 BR restore
1279 restore:
1280 MOVD $16, R20
1281 TW $31, R0, R0
1282 // restore all registers that can contain
1283 // pointers including CR
1284 MOVD 8(R1), R31
1285 MOVW R31, CR
1286 MOVD 24(R1), R2
1287 MOVD 56(R1), R3
1288 MOVD 64(R1), R4
1289 MOVD 72(R1), R5
1290 MOVD 80(R1), R6
1291 MOVD 88(R1), R7
1292 MOVD 96(R1), R8
1293 MOVD 104(R1), R9
1294 MOVD 112(R1), R10
1295 MOVD 120(R1), R11
1296 MOVD 144(R1), R12
1297 MOVD 152(R1), R13
1298 MOVD 160(R1), R14
1299 MOVD 168(R1), R15
1300 MOVD 176(R1), R16
1301 MOVD 184(R1), R17
1302 MOVD 192(R1), R18
1303 MOVD 200(R1), R19
1304 MOVD 208(R1), R20
1305 MOVD 216(R1), R21
1306 MOVD 224(R1), R22
1307 MOVD 232(R1), R23
1308 MOVD 240(R1), R24
1309 MOVD 248(R1), R25
1310 MOVD 256(R1), R26
1311 MOVD 264(R1), R27
1312 MOVD 272(R1), R28
1313 MOVD 280(R1), R29
1314 MOVD 288(R1), g
1315 MOVD 16(R1), R31
1316 // restore old LR
1317 MOVD R31, LR
1318 // restore caller PC
1319 MOVD 0(R1), CTR
1320 MOVD 136(R1), R31
1321 // Add 32 bytes more to compensate for SP change in saveSigContext
1322 ADD $352, R1
1323 JMP (CTR)
1324 #endif
1325 #define DEBUG_CALL_FN(NAME,MAXSIZE) \
1326 TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1327 NO_LOCAL_POINTERS; \
1328 MOVD $0, R20; \
1329 TW $31, R0, R0 \
1330 MOVD $1, R20; \
1331 TW $31, R0, R0 \
1332 RET
1333 DEBUG_CALL_FN(debugCall32<>, 32)
1334 DEBUG_CALL_FN(debugCall64<>, 64)
1335 DEBUG_CALL_FN(debugCall128<>, 128)
1336 DEBUG_CALL_FN(debugCall256<>, 256)
1337 DEBUG_CALL_FN(debugCall512<>, 512)
1338 DEBUG_CALL_FN(debugCall1024<>, 1024)
1339 DEBUG_CALL_FN(debugCall2048<>, 2048)
1340 DEBUG_CALL_FN(debugCall4096<>, 4096)
1341 DEBUG_CALL_FN(debugCall8192<>, 8192)
1342 DEBUG_CALL_FN(debugCall16384<>, 16384)
1343 DEBUG_CALL_FN(debugCall32768<>, 32768)
1344 DEBUG_CALL_FN(debugCall65536<>, 65536)
1345
1346 #ifdef GOARCH_ppc64le
1347 // func debugCallPanicked(val interface{})
1348 TEXT runtime·debugCallPanicked(SB),NOSPLIT,$32-16
1349 // Copy the panic value to the top of stack at SP+32.
1350 MOVD val_type+0(FP), R31
1351 MOVD R31, 32(R1)
1352 MOVD val_data+8(FP), R31
1353 MOVD R31, 40(R1)
1354 MOVD $2, R20
1355 TW $31, R0, R0
1356 RET
1357 #endif
1358 // Note: these functions use a special calling convention to save generated code space.
1359 // Arguments are passed in registers, but the space for those arguments are allocated
1360 // in the caller's stack frame. These stubs write the args into that stack space and
1361 // then tail call to the corresponding runtime handler.
1362 // The tail call makes these stubs disappear in backtraces.
1363 TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
1364 JMP runtime·goPanicIndex<ABIInternal>(SB)
1365 TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
1366 JMP runtime·goPanicIndexU<ABIInternal>(SB)
1367 TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
1368 MOVD R4, R3
1369 MOVD R5, R4
1370 JMP runtime·goPanicSliceAlen<ABIInternal>(SB)
1371 TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
1372 MOVD R4, R3
1373 MOVD R5, R4
1374 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB)
1375 TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
1376 MOVD R4, R3
1377 MOVD R5, R4
1378 JMP runtime·goPanicSliceAcap<ABIInternal>(SB)
1379 TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
1380 MOVD R4, R3
1381 MOVD R5, R4
1382 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB)
1383 TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
1384 JMP runtime·goPanicSliceB<ABIInternal>(SB)
1385 TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
1386 JMP runtime·goPanicSliceBU<ABIInternal>(SB)
1387 TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
1388 MOVD R5, R3
1389 MOVD R6, R4
1390 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB)
1391 TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
1392 MOVD R5, R3
1393 MOVD R6, R4
1394 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB)
1395 TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
1396 MOVD R5, R3
1397 MOVD R6, R4
1398 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB)
1399 TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
1400 MOVD R5, R3
1401 MOVD R6, R4
1402 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB)
1403 TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
1404 MOVD R4, R3
1405 MOVD R5, R4
1406 JMP runtime·goPanicSlice3B<ABIInternal>(SB)
1407 TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
1408 MOVD R4, R3
1409 MOVD R5, R4
1410 JMP runtime·goPanicSlice3BU<ABIInternal>(SB)
1411 TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
1412 JMP runtime·goPanicSlice3C<ABIInternal>(SB)
1413 TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
1414 JMP runtime·goPanicSlice3CU<ABIInternal>(SB)
1415 TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
1416 MOVD R5, R3
1417 MOVD R6, R4
1418 JMP runtime·goPanicSliceConvert<ABIInternal>(SB)
1419
1420 // These functions are used when internal linking cgo with external
1421 // objects compiled with the -Os on gcc. They reduce prologue/epilogue
1422 // size by deferring preservation of callee-save registers to a shared
1423 // function. These are defined in PPC64 ELFv2 2.3.3 (but also present
1424 // in ELFv1)
1425 //
1426 // These appear unused, but the linker will redirect calls to functions
1427 // like _savegpr0_14 or _restgpr1_14 to runtime.elf_savegpr0 or
1428 // runtime.elf_restgpr1 with an appropriate offset based on the number
1429 // register operations required when linking external objects which
1430 // make these calls. For GPR/FPR saves, the minimum register value is
1431 // 14, for VR it is 20.
1432 //
1433 // These are only used when linking such cgo code internally. Note, R12
1434 // and R0 may be used in different ways than regular ELF compliant
1435 // functions.
1436 TEXT runtime·elf_savegpr0(SB),NOSPLIT|NOFRAME,$0
1437 // R0 holds the LR of the caller's caller, R1 holds save location
1438 MOVD R14, -144(R1)
1439 MOVD R15, -136(R1)
1440 MOVD R16, -128(R1)
1441 MOVD R17, -120(R1)
1442 MOVD R18, -112(R1)
1443 MOVD R19, -104(R1)
1444 MOVD R20, -96(R1)
1445 MOVD R21, -88(R1)
1446 MOVD R22, -80(R1)
1447 MOVD R23, -72(R1)
1448 MOVD R24, -64(R1)
1449 MOVD R25, -56(R1)
1450 MOVD R26, -48(R1)
1451 MOVD R27, -40(R1)
1452 MOVD R28, -32(R1)
1453 MOVD R29, -24(R1)
1454 MOVD g, -16(R1)
1455 MOVD R31, -8(R1)
1456 MOVD R0, 16(R1)
1457 RET
1458 TEXT runtime·elf_restgpr0(SB),NOSPLIT|NOFRAME,$0
1459 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1460 MOVD -144(R1), R14
1461 MOVD -136(R1), R15
1462 MOVD -128(R1), R16
1463 MOVD -120(R1), R17
1464 MOVD -112(R1), R18
1465 MOVD -104(R1), R19
1466 MOVD -96(R1), R20
1467 MOVD -88(R1), R21
1468 MOVD -80(R1), R22
1469 MOVD -72(R1), R23
1470 MOVD -64(R1), R24
1471 MOVD -56(R1), R25
1472 MOVD -48(R1), R26
1473 MOVD -40(R1), R27
1474 MOVD -32(R1), R28
1475 MOVD -24(R1), R29
1476 MOVD -16(R1), g
1477 MOVD -8(R1), R31
1478 MOVD 16(R1), R0 // Load and return to saved LR
1479 MOVD R0, LR
1480 RET
1481 TEXT runtime·elf_savegpr1(SB),NOSPLIT|NOFRAME,$0
1482 // R12 holds the save location
1483 MOVD R14, -144(R12)
1484 MOVD R15, -136(R12)
1485 MOVD R16, -128(R12)
1486 MOVD R17, -120(R12)
1487 MOVD R18, -112(R12)
1488 MOVD R19, -104(R12)
1489 MOVD R20, -96(R12)
1490 MOVD R21, -88(R12)
1491 MOVD R22, -80(R12)
1492 MOVD R23, -72(R12)
1493 MOVD R24, -64(R12)
1494 MOVD R25, -56(R12)
1495 MOVD R26, -48(R12)
1496 MOVD R27, -40(R12)
1497 MOVD R28, -32(R12)
1498 MOVD R29, -24(R12)
1499 MOVD g, -16(R12)
1500 MOVD R31, -8(R12)
1501 RET
1502 TEXT runtime·elf_restgpr1(SB),NOSPLIT|NOFRAME,$0
1503 // R12 holds the save location
1504 MOVD -144(R12), R14
1505 MOVD -136(R12), R15
1506 MOVD -128(R12), R16
1507 MOVD -120(R12), R17
1508 MOVD -112(R12), R18
1509 MOVD -104(R12), R19
1510 MOVD -96(R12), R20
1511 MOVD -88(R12), R21
1512 MOVD -80(R12), R22
1513 MOVD -72(R12), R23
1514 MOVD -64(R12), R24
1515 MOVD -56(R12), R25
1516 MOVD -48(R12), R26
1517 MOVD -40(R12), R27
1518 MOVD -32(R12), R28
1519 MOVD -24(R12), R29
1520 MOVD -16(R12), g
1521 MOVD -8(R12), R31
1522 RET
1523 TEXT runtime·elf_savefpr(SB),NOSPLIT|NOFRAME,$0
1524 // R0 holds the LR of the caller's caller, R1 holds save location
1525 FMOVD F14, -144(R1)
1526 FMOVD F15, -136(R1)
1527 FMOVD F16, -128(R1)
1528 FMOVD F17, -120(R1)
1529 FMOVD F18, -112(R1)
1530 FMOVD F19, -104(R1)
1531 FMOVD F20, -96(R1)
1532 FMOVD F21, -88(R1)
1533 FMOVD F22, -80(R1)
1534 FMOVD F23, -72(R1)
1535 FMOVD F24, -64(R1)
1536 FMOVD F25, -56(R1)
1537 FMOVD F26, -48(R1)
1538 FMOVD F27, -40(R1)
1539 FMOVD F28, -32(R1)
1540 FMOVD F29, -24(R1)
1541 FMOVD F30, -16(R1)
1542 FMOVD F31, -8(R1)
1543 MOVD R0, 16(R1)
1544 RET
1545 TEXT runtime·elf_restfpr(SB),NOSPLIT|NOFRAME,$0
1546 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1547 FMOVD -144(R1), F14
1548 FMOVD -136(R1), F15
1549 FMOVD -128(R1), F16
1550 FMOVD -120(R1), F17
1551 FMOVD -112(R1), F18
1552 FMOVD -104(R1), F19
1553 FMOVD -96(R1), F20
1554 FMOVD -88(R1), F21
1555 FMOVD -80(R1), F22
1556 FMOVD -72(R1), F23
1557 FMOVD -64(R1), F24
1558 FMOVD -56(R1), F25
1559 FMOVD -48(R1), F26
1560 FMOVD -40(R1), F27
1561 FMOVD -32(R1), F28
1562 FMOVD -24(R1), F29
1563 FMOVD -16(R1), F30
1564 FMOVD -8(R1), F31
1565 MOVD 16(R1), R0 // Load and return to saved LR
1566 MOVD R0, LR
1567 RET
1568 TEXT runtime·elf_savevr(SB),NOSPLIT|NOFRAME,$0
1569 // R0 holds the save location, R12 is clobbered
1570 MOVD $-192, R12
1571 STVX V20, (R0+R12)
1572 MOVD $-176, R12
1573 STVX V21, (R0+R12)
1574 MOVD $-160, R12
1575 STVX V22, (R0+R12)
1576 MOVD $-144, R12
1577 STVX V23, (R0+R12)
1578 MOVD $-128, R12
1579 STVX V24, (R0+R12)
1580 MOVD $-112, R12
1581 STVX V25, (R0+R12)
1582 MOVD $-96, R12
1583 STVX V26, (R0+R12)
1584 MOVD $-80, R12
1585 STVX V27, (R0+R12)
1586 MOVD $-64, R12
1587 STVX V28, (R0+R12)
1588 MOVD $-48, R12
1589 STVX V29, (R0+R12)
1590 MOVD $-32, R12
1591 STVX V30, (R0+R12)
1592 MOVD $-16, R12
1593 STVX V31, (R0+R12)
1594 RET
1595 TEXT runtime·elf_restvr(SB),NOSPLIT|NOFRAME,$0
1596 // R0 holds the save location, R12 is clobbered
1597 MOVD $-192, R12
1598 LVX (R0+R12), V20
1599 MOVD $-176, R12
1600 LVX (R0+R12), V21
1601 MOVD $-160, R12
1602 LVX (R0+R12), V22
1603 MOVD $-144, R12
1604 LVX (R0+R12), V23
1605 MOVD $-128, R12
1606 LVX (R0+R12), V24
1607 MOVD $-112, R12
1608 LVX (R0+R12), V25
1609 MOVD $-96, R12
1610 LVX (R0+R12), V26
1611 MOVD $-80, R12
1612 LVX (R0+R12), V27
1613 MOVD $-64, R12
1614 LVX (R0+R12), V28
1615 MOVD $-48, R12
1616 LVX (R0+R12), V29
1617 MOVD $-32, R12
1618 LVX (R0+R12), V30
1619 MOVD $-16, R12
1620 LVX (R0+R12), V31
1621 RET
1622
View as plain text