Text file
src/runtime/asm_ppc64x.s
1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build ppc64 || ppc64le
6
7 #include "go_asm.h"
8 #include "go_tls.h"
9 #include "funcdata.h"
10 #include "textflag.h"
11 #include "asm_ppc64x.h"
12 #include "cgo/abi_ppc64x.h"
13
14 // This is called using the host ABI. argc and argv arguments
15 // should be in R3 and R4 respectively.
16 TEXT _rt0_ppc64x_lib(SB),NOSPLIT|NOFRAME,$0
17 // Start with standard C stack frame layout and linkage, allocate
18 // 16 bytes of argument space, save callee-save regs, and set R0 to $0.
19 // Allocate an extra 16 bytes to account for the larger fixed frame size
20 // of aix/elfv1 (48 vs 32) to ensure 16 bytes of parameter save space.
21 STACK_AND_SAVE_HOST_TO_GO_ABI(32)
22 // The above will not preserve R2 (TOC). Save it in case Go is
23 // compiled without a TOC pointer (e.g -buildmode=default).
24 MOVD R2, 24(R1)
25
26 MOVD R3, _rt0_ppc64x_lib_argc<>(SB)
27 MOVD R4, _rt0_ppc64x_lib_argv<>(SB)
28
29 // Synchronous initialization.
30 MOVD $runtime·reginit(SB), R12
31 MOVD R12, CTR
32 BL (CTR)
33
34 // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go
35 MOVD R0, g
36
37 #ifdef GOOS_aix
38 // See runtime/cgo/gcc_aix_ppc64.c
39 MOVBZ runtime·isarchive(SB), R3 // Check buildmode = c-archive
40 CMP $0, R3
41 BEQ skipInit
42 #endif
43
44 MOVD $runtime·libInit(SB), R12
45 MOVD R12, CTR
46 BL (CTR)
47
48 skipInit:
49 // Restore and return to ELFv2 caller.
50 UNSTACK_AND_RESTORE_GO_TO_HOST_ABI(32)
51 RET
52
53 TEXT runtime·rt0_lib_go<ABIInternal>(SB),NOSPLIT,$0
54 MOVD _rt0_ppc64x_lib_argc<>(SB), R3
55 MOVD _rt0_ppc64x_lib_argv<>(SB), R4
56 MOVD $runtime·rt0_go(SB), R12
57 MOVD R12, CTR
58 BR (CTR)
59
60 DATA _rt0_ppc64x_lib_argc<>(SB)/8, $0
61 GLOBL _rt0_ppc64x_lib_argc<>(SB),NOPTR, $8
62 DATA _rt0_ppc64x_lib_argv<>(SB)/8, $0
63 GLOBL _rt0_ppc64x_lib_argv<>(SB),NOPTR, $8
64
65
66 #ifdef GOOS_aix
67 #define cgoCalleeStackSize 48
68 #else
69 #define cgoCalleeStackSize 32
70 #endif
71
72 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
73 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer
74
75 // initialize essential registers
76 BL runtime·reginit(SB)
77
78 SUB $(FIXED_FRAME+16), R1
79 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame
80 MOVW R3, FIXED_FRAME+0(R1) // argc
81 MOVD R4, FIXED_FRAME+8(R1) // argv
82
83 // create istack out of the given (operating system) stack.
84 // _cgo_init may update stackguard.
85 MOVD $runtime·g0(SB), g
86 BL runtime·save_g(SB)
87 MOVD $(-64*1024), R31
88 ADD R31, R1, R3
89 MOVD R3, g_stackguard0(g)
90 MOVD R3, g_stackguard1(g)
91 MOVD R3, (g_stack+stack_lo)(g)
92 MOVD R1, (g_stack+stack_hi)(g)
93
94 // If there is a _cgo_init, call it using the gcc ABI.
95 MOVD _cgo_init(SB), R12
96 CMP R12, $0
97 BEQ nocgo
98
99 #ifdef GO_PPC64X_HAS_FUNCDESC
100 // Load the real entry address from the first slot of the function descriptor.
101 MOVD 8(R12), R2
102 MOVD (R12), R12
103 #endif
104 MOVD R12, CTR // r12 = "global function entry point"
105 MOVD R13, R5 // arg 2: TLS base pointer
106 MOVD $setg_gcc<>(SB), R4 // arg 1: setg
107 MOVD g, R3 // arg 0: G
108 // C functions expect 32 (48 for AIX) bytes of space on caller
109 // stack frame and a 16-byte aligned R1
110 MOVD R1, R14 // save current stack
111 SUB $cgoCalleeStackSize, R1 // reserve the callee area
112 RLDCR $0, R1, $~15, R1 // 16-byte align
113 BL (CTR) // may clobber R0, R3-R12
114 MOVD R14, R1 // restore stack
115 #ifndef GOOS_aix
116 MOVD 24(R1), R2
117 #endif
118 XOR R0, R0 // fix R0
119
120 nocgo:
121 // update stackguard after _cgo_init
122 MOVD (g_stack+stack_lo)(g), R3
123 ADD $const_stackGuard, R3
124 MOVD R3, g_stackguard0(g)
125 MOVD R3, g_stackguard1(g)
126
127 // set the per-goroutine and per-mach "registers"
128 MOVD $runtime·m0(SB), R3
129
130 // save m->g0 = g0
131 MOVD g, m_g0(R3)
132 // save m0 to g0->m
133 MOVD R3, g_m(g)
134
135 BL runtime·check(SB)
136
137 // args are already prepared
138 BL runtime·args(SB)
139 BL runtime·osinit(SB)
140 BL runtime·schedinit(SB)
141
142 // create a new goroutine to start program
143 MOVD $runtime·mainPC(SB), R3 // entry
144 MOVDU R3, -8(R1)
145 MOVDU R0, -8(R1)
146 MOVDU R0, -8(R1)
147 MOVDU R0, -8(R1)
148 MOVDU R0, -8(R1)
149 BL runtime·newproc(SB)
150 ADD $(8+FIXED_FRAME), R1
151
152 // start this M
153 BL runtime·mstart(SB)
154 // Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are
155 // intended to be called by debuggers.
156 #ifdef GOARCH_ppc64le
157 MOVD $runtime·debugPinnerV1<ABIInternal>(SB), R31
158 MOVD $runtime·debugCallV2<ABIInternal>(SB), R31
159 #endif
160 MOVD R0, 0(R0)
161 RET
162
163 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
164 GLOBL runtime·mainPC(SB),RODATA,$8
165
166 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
167 TW $31, R0, R0
168 RET
169
170 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
171 RET
172
173 // Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall1
174 TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0
175 // crosscall1 and crosscall2 need to reginit, but can't
176 // get at the 'runtime.reginit' symbol.
177 BR runtime·reginit(SB)
178
179 TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0
180 // set R0 to zero, it's expected by the toolchain
181 XOR R0, R0
182 RET
183
184 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
185 BL runtime·mstart0(SB)
186 RET // not reached
187
188 /*
189 * go-routine
190 */
191
192 // void gogo(Gobuf*)
193 // restore state from Gobuf; longjmp
194 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
195 MOVD buf+0(FP), R5
196 MOVD gobuf_g(R5), R6
197 MOVD 0(R6), R4 // make sure g != nil
198 BR gogo<>(SB)
199
200 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
201 MOVD R6, g
202 BL runtime·save_g(SB)
203
204 MOVD gobuf_sp(R5), R1
205 MOVD gobuf_lr(R5), R31
206 #ifndef GOOS_aix
207 MOVD 24(R1), R2 // restore R2
208 #endif
209 MOVD R31, LR
210 MOVD gobuf_ctxt(R5), R11
211 MOVD R0, gobuf_sp(R5)
212 MOVD R0, gobuf_lr(R5)
213 MOVD R0, gobuf_ctxt(R5)
214 CMP R0, R0 // set condition codes for == test, needed by stack split
215 MOVD gobuf_pc(R5), R12
216 MOVD R12, CTR
217 BR (CTR)
218
219 // void mcall(fn func(*g))
220 // Switch to m->g0's stack, call fn(g).
221 // Fn must never return. It should gogo(&g->sched)
222 // to keep running g.
223 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
224 // Save caller state in g->sched
225 // R11 should be safe across save_g??
226 MOVD R3, R11
227 MOVD R1, (g_sched+gobuf_sp)(g)
228 MOVD LR, R31
229 MOVD R31, (g_sched+gobuf_pc)(g)
230 MOVD R0, (g_sched+gobuf_lr)(g)
231
232 // Switch to m->g0 & its stack, call fn.
233 MOVD g, R3
234 MOVD g_m(g), R8
235 MOVD m_g0(R8), g
236 BL runtime·save_g(SB)
237 CMP g, R3
238 BNE 2(PC)
239 BR runtime·badmcall(SB)
240 MOVD 0(R11), R12 // code pointer
241 MOVD R12, CTR
242 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp
243 // Don't need to do anything special for regabiargs here
244 // R3 is g; stack is set anyway
245 MOVDU R3, -8(R1)
246 MOVDU R0, -8(R1)
247 MOVDU R0, -8(R1)
248 MOVDU R0, -8(R1)
249 MOVDU R0, -8(R1)
250 BL (CTR)
251 MOVD 24(R1), R2
252 BR runtime·badmcall2(SB)
253
254 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
255 // of the G stack. We need to distinguish the routine that
256 // lives at the bottom of the G stack from the one that lives
257 // at the top of the system stack because the one at the top of
258 // the system stack terminates the stack walk (see topofstack()).
259 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
260 // We have several undefs here so that 16 bytes past
261 // $runtime·systemstack_switch lies within them whether or not the
262 // instructions that derive r2 from r12 are there.
263 UNDEF
264 UNDEF
265 UNDEF
266 BL (LR) // make sure this function is not leaf
267 RET
268
269 // func systemstack(fn func())
270 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
271 MOVD fn+0(FP), R3 // R3 = fn
272 MOVD R3, R11 // context
273 MOVD g_m(g), R4 // R4 = m
274
275 MOVD m_gsignal(R4), R5 // R5 = gsignal
276 CMP g, R5
277 BEQ noswitch
278
279 MOVD m_g0(R4), R5 // R5 = g0
280 CMP g, R5
281 BEQ noswitch
282
283 MOVD m_curg(R4), R6
284 CMP g, R6
285 BEQ switch
286
287 // Bad: g is not gsignal, not g0, not curg. What is it?
288 // Hide call from linker nosplit analysis.
289 MOVD $runtime·badsystemstack(SB), R12
290 MOVD R12, CTR
291 BL (CTR)
292 BL runtime·abort(SB)
293
294 switch:
295 // save our state in g->sched. Pretend to
296 // be systemstack_switch if the G stack is scanned.
297 BL gosave_systemstack_switch<>(SB)
298
299 // switch to g0
300 MOVD R5, g
301 BL runtime·save_g(SB)
302 MOVD (g_sched+gobuf_sp)(g), R1
303
304 // call target function
305 MOVD 0(R11), R12 // code pointer
306 MOVD R12, CTR
307 BL (CTR)
308
309 // restore TOC pointer. It seems unlikely that we will use systemstack
310 // to call a function defined in another module, but the results of
311 // doing so would be so confusing that it's worth doing this.
312 MOVD g_m(g), R3
313 MOVD m_curg(R3), g
314 MOVD (g_sched+gobuf_sp)(g), R3
315 #ifndef GOOS_aix
316 MOVD 24(R3), R2
317 #endif
318 // switch back to g
319 MOVD g_m(g), R3
320 MOVD m_curg(R3), g
321 BL runtime·save_g(SB)
322 MOVD (g_sched+gobuf_sp)(g), R1
323 MOVD R0, (g_sched+gobuf_sp)(g)
324 RET
325
326 noswitch:
327 // already on m stack, just call directly
328 // On other arches we do a tail call here, but it appears to be
329 // impossible to tail call a function pointer in shared mode on
330 // ppc64 because the caller is responsible for restoring the TOC.
331 MOVD 0(R11), R12 // code pointer
332 MOVD R12, CTR
333 BL (CTR)
334 #ifndef GOOS_aix
335 MOVD 24(R1), R2
336 #endif
337 RET
338
339 // func switchToCrashStack0(fn func())
340 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
341 MOVD R3, R11 // context register
342 MOVD g_m(g), R3 // curm
343
344 // set g to gcrash
345 MOVD $runtime·gcrash(SB), g // g = &gcrash
346 CALL runtime·save_g(SB) // clobbers R31
347 MOVD R3, g_m(g) // g.m = curm
348 MOVD g, m_g0(R3) // curm.g0 = g
349
350 // switch to crashstack
351 MOVD (g_stack+stack_hi)(g), R3
352 SUB $(4*8), R3
353 MOVD R3, R1
354
355 // call target function
356 MOVD 0(R11), R12 // code pointer
357 MOVD R12, CTR
358 BL (CTR)
359
360 // should never return
361 CALL runtime·abort(SB)
362 UNDEF
363
364 /*
365 * support for morestack
366 */
367
368 // Called during function prolog when more stack is needed.
369 // Caller has already loaded:
370 // R3: framesize, R4: argsize, R5: LR
371 //
372 // The traceback routines see morestack on a g0 as being
373 // the top of a stack (for example, morestack calling newstack
374 // calling the scheduler calling newm calling gc), so we must
375 // record an argument size. For that purpose, it has no arguments.
376 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
377 // Called from f.
378 // Set g->sched to context in f.
379 MOVD R1, (g_sched+gobuf_sp)(g)
380 MOVD LR, R8
381 MOVD R8, (g_sched+gobuf_pc)(g)
382 MOVD R5, (g_sched+gobuf_lr)(g)
383 MOVD R11, (g_sched+gobuf_ctxt)(g)
384
385 // Cannot grow scheduler stack (m->g0).
386 MOVD g_m(g), R7
387 MOVD m_g0(R7), R8
388 CMP g, R8
389 BNE 3(PC)
390 BL runtime·badmorestackg0(SB)
391 BL runtime·abort(SB)
392
393 // Cannot grow signal stack (m->gsignal).
394 MOVD m_gsignal(R7), R8
395 CMP g, R8
396 BNE 3(PC)
397 BL runtime·badmorestackgsignal(SB)
398 BL runtime·abort(SB)
399
400 // Called from f.
401 // Set m->morebuf to f's caller.
402 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
403 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
404 MOVD g, (m_morebuf+gobuf_g)(R7)
405
406 // Call newstack on m->g0's stack.
407 MOVD m_g0(R7), g
408 BL runtime·save_g(SB)
409 MOVD (g_sched+gobuf_sp)(g), R1
410 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0
411 BL runtime·newstack(SB)
412
413 // Not reached, but make sure the return PC from the call to newstack
414 // is still in this function, and not the beginning of the next.
415 UNDEF
416
417 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
418 // Force SPWRITE. This function doesn't actually write SP,
419 // but it is called with a special calling convention where
420 // the caller doesn't save LR on stack but passes it as a
421 // register (R5), and the unwinder currently doesn't understand.
422 // Make it SPWRITE to stop unwinding. (See issue 54332)
423 // Use OR R0, R1 instead of MOVD R1, R1 as the MOVD instruction
424 // has a special affect on Power8,9,10 by lowering the thread
425 // priority and causing a slowdown in execution time
426
427 OR R0, R1
428 MOVD R0, R11
429 BR runtime·morestack(SB)
430
431 // reflectcall: call a function with the given argument list
432 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
433 // we don't have variable-sized frames, so we use a small number
434 // of constant-sized-frame functions to encode a few bits of size in the pc.
435 // Caution: ugly multiline assembly macros in your future!
436
437 #define DISPATCH(NAME,MAXSIZE) \
438 MOVD $MAXSIZE, R31; \
439 CMP R3, R31; \
440 BGT 4(PC); \
441 MOVD $NAME(SB), R12; \
442 MOVD R12, CTR; \
443 BR (CTR)
444 // Note: can't just "BR NAME(SB)" - bad inlining results.
445
446 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
447 MOVWZ frameSize+32(FP), R3
448 DISPATCH(runtime·call16, 16)
449 DISPATCH(runtime·call32, 32)
450 DISPATCH(runtime·call64, 64)
451 DISPATCH(runtime·call128, 128)
452 DISPATCH(runtime·call256, 256)
453 DISPATCH(runtime·call512, 512)
454 DISPATCH(runtime·call1024, 1024)
455 DISPATCH(runtime·call2048, 2048)
456 DISPATCH(runtime·call4096, 4096)
457 DISPATCH(runtime·call8192, 8192)
458 DISPATCH(runtime·call16384, 16384)
459 DISPATCH(runtime·call32768, 32768)
460 DISPATCH(runtime·call65536, 65536)
461 DISPATCH(runtime·call131072, 131072)
462 DISPATCH(runtime·call262144, 262144)
463 DISPATCH(runtime·call524288, 524288)
464 DISPATCH(runtime·call1048576, 1048576)
465 DISPATCH(runtime·call2097152, 2097152)
466 DISPATCH(runtime·call4194304, 4194304)
467 DISPATCH(runtime·call8388608, 8388608)
468 DISPATCH(runtime·call16777216, 16777216)
469 DISPATCH(runtime·call33554432, 33554432)
470 DISPATCH(runtime·call67108864, 67108864)
471 DISPATCH(runtime·call134217728, 134217728)
472 DISPATCH(runtime·call268435456, 268435456)
473 DISPATCH(runtime·call536870912, 536870912)
474 DISPATCH(runtime·call1073741824, 1073741824)
475 MOVD $runtime·badreflectcall(SB), R12
476 MOVD R12, CTR
477 BR (CTR)
478
479 #define CALLFN(NAME,MAXSIZE) \
480 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
481 NO_LOCAL_POINTERS; \
482 /* copy arguments to stack */ \
483 MOVD stackArgs+16(FP), R3; \
484 MOVWZ stackArgsSize+24(FP), R4; \
485 MOVD R1, R5; \
486 CMP R4, $8; \
487 BLT tailsetup; \
488 /* copy 8 at a time if possible */ \
489 ADD $(FIXED_FRAME-8), R5; \
490 SUB $8, R3; \
491 top: \
492 MOVDU 8(R3), R7; \
493 MOVDU R7, 8(R5); \
494 SUB $8, R4; \
495 CMP R4, $8; \
496 BGE top; \
497 /* handle remaining bytes */ \
498 CMP $0, R4; \
499 BEQ callfn; \
500 ADD $7, R3; \
501 ADD $7, R5; \
502 BR tail; \
503 tailsetup: \
504 CMP $0, R4; \
505 BEQ callfn; \
506 ADD $(FIXED_FRAME-1), R5; \
507 SUB $1, R3; \
508 tail: \
509 MOVBU 1(R3), R6; \
510 MOVBU R6, 1(R5); \
511 SUB $1, R4; \
512 CMP $0, R4; \
513 BGT tail; \
514 callfn: \
515 /* call function */ \
516 MOVD f+8(FP), R11; \
517 #ifdef GOOS_aix \
518 /* AIX won't trigger a SIGSEGV if R11 = nil */ \
519 /* So it manually triggers it */ \
520 CMP R11, $0 \
521 BNE 2(PC) \
522 MOVD R0, 0(R0) \
523 #endif \
524 MOVD regArgs+40(FP), R20; \
525 BL runtime·unspillArgs(SB); \
526 MOVD (R11), R12; \
527 MOVD R12, CTR; \
528 PCDATA $PCDATA_StackMapIndex, $0; \
529 BL (CTR); \
530 #ifndef GOOS_aix \
531 MOVD 24(R1), R2; \
532 #endif \
533 /* copy return values back */ \
534 MOVD regArgs+40(FP), R20; \
535 BL runtime·spillArgs(SB); \
536 MOVD stackArgsType+0(FP), R7; \
537 MOVD stackArgs+16(FP), R3; \
538 MOVWZ stackArgsSize+24(FP), R4; \
539 MOVWZ stackRetOffset+28(FP), R6; \
540 ADD $FIXED_FRAME, R1, R5; \
541 ADD R6, R5; \
542 ADD R6, R3; \
543 SUB R6, R4; \
544 BL callRet<>(SB); \
545 RET
546
547 // callRet copies return values back at the end of call*. This is a
548 // separate function so it can allocate stack space for the arguments
549 // to reflectcallmove. It does not follow the Go ABI; it expects its
550 // arguments in registers.
551 TEXT callRet<>(SB), NOSPLIT, $40-0
552 NO_LOCAL_POINTERS
553 MOVD R7, FIXED_FRAME+0(R1)
554 MOVD R3, FIXED_FRAME+8(R1)
555 MOVD R5, FIXED_FRAME+16(R1)
556 MOVD R4, FIXED_FRAME+24(R1)
557 MOVD R20, FIXED_FRAME+32(R1)
558 BL runtime·reflectcallmove(SB)
559 RET
560
561 CALLFN(·call16, 16)
562 CALLFN(·call32, 32)
563 CALLFN(·call64, 64)
564 CALLFN(·call128, 128)
565 CALLFN(·call256, 256)
566 CALLFN(·call512, 512)
567 CALLFN(·call1024, 1024)
568 CALLFN(·call2048, 2048)
569 CALLFN(·call4096, 4096)
570 CALLFN(·call8192, 8192)
571 CALLFN(·call16384, 16384)
572 CALLFN(·call32768, 32768)
573 CALLFN(·call65536, 65536)
574 CALLFN(·call131072, 131072)
575 CALLFN(·call262144, 262144)
576 CALLFN(·call524288, 524288)
577 CALLFN(·call1048576, 1048576)
578 CALLFN(·call2097152, 2097152)
579 CALLFN(·call4194304, 4194304)
580 CALLFN(·call8388608, 8388608)
581 CALLFN(·call16777216, 16777216)
582 CALLFN(·call33554432, 33554432)
583 CALLFN(·call67108864, 67108864)
584 CALLFN(·call134217728, 134217728)
585 CALLFN(·call268435456, 268435456)
586 CALLFN(·call536870912, 536870912)
587 CALLFN(·call1073741824, 1073741824)
588
589 TEXT runtime·procyieldAsm(SB),NOSPLIT|NOFRAME,$0-4
590 MOVW cycles+0(FP), R7
591 CMP $0, R7
592 BEQ done
593 // POWER does not have a pause/yield instruction equivalent.
594 // Instead, we can lower the program priority by setting the
595 // Program Priority Register prior to the wait loop and set it
596 // back to default afterwards. On Linux, the default priority is
597 // medium-low. For details, see page 837 of the ISA 3.0.
598 OR R1, R1, R1 // Set PPR priority to low
599 again:
600 SUB $1, R7
601 CMP $0, R7
602 BNE again
603 OR R6, R6, R6 // Set PPR priority back to medium-low
604 done:
605 RET
606
607 // Save state of caller into g->sched,
608 // but using fake PC from systemstack_switch.
609 // Must only be called from functions with no locals ($0)
610 // or else unwinding from systemstack_switch is incorrect.
611 // Smashes R31.
612 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
613 MOVD $runtime·systemstack_switch(SB), R31
614 ADD $16, R31 // get past prologue (including r2-setting instructions when they're there)
615 MOVD R31, (g_sched+gobuf_pc)(g)
616 MOVD R1, (g_sched+gobuf_sp)(g)
617 MOVD R0, (g_sched+gobuf_lr)(g)
618 // Assert ctxt is zero. See func save.
619 MOVD (g_sched+gobuf_ctxt)(g), R31
620 CMP R31, $0
621 BEQ 2(PC)
622 BL runtime·abort(SB)
623 RET
624
625 #ifdef GOOS_aix
626 #define asmcgocallSaveOffset cgoCalleeStackSize + 8
627 #else
628 #define asmcgocallSaveOffset cgoCalleeStackSize
629 #endif
630
631 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
632 // Call fn(arg) aligned appropriately for the gcc ABI.
633 // Called on a system stack, and there may be no g yet (during needm).
634 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
635 MOVD fn+0(FP), R3
636 MOVD arg+8(FP), R4
637
638 MOVD R1, R15
639 SUB $(asmcgocallSaveOffset+8), R1
640 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
641 MOVD R15, asmcgocallSaveOffset(R1)
642
643 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
644
645 // This is a "global call", so put the global entry point in r12
646 MOVD R3, R12
647
648 #ifdef GO_PPC64X_HAS_FUNCDESC
649 // Load the real entry address from the first slot of the function descriptor.
650 MOVD 8(R12), R2
651 MOVD (R12), R12
652 #endif
653 MOVD R12, CTR
654 MOVD R4, R3 // arg in r3
655 BL (CTR)
656
657 // C code can clobber R0, so set it back to 0. F27-F31 are
658 // callee save, so we don't need to recover those.
659 XOR R0, R0
660
661 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
662 #ifndef GOOS_aix
663 MOVD 24(R1), R2
664 #endif
665
666 RET
667
668 // func asmcgocall(fn, arg unsafe.Pointer) int32
669 // Call fn(arg) on the scheduler stack,
670 // aligned appropriately for the gcc ABI.
671 // See cgocall.go for more details.
672 TEXT ·asmcgocall<ABIInternal>(SB),NOSPLIT,$0-20
673 // R3 = fn
674 // R4 = arg
675
676 MOVD R1, R7 // save original stack pointer
677 CMP $0, g
678 BEQ nosave
679 MOVD g, R5
680
681 // Figure out if we need to switch to m->g0 stack.
682 // We get called to create new OS threads too, and those
683 // come in on the m->g0 stack already. Or we might already
684 // be on the m->gsignal stack.
685 MOVD g_m(g), R8
686 MOVD m_gsignal(R8), R6
687 CMP R6, g
688 BEQ nosave
689 MOVD m_g0(R8), R6
690 CMP R6, g
691 BEQ nosave
692
693 BL gosave_systemstack_switch<>(SB)
694 MOVD R6, g
695 BL runtime·save_g(SB)
696 MOVD (g_sched+gobuf_sp)(g), R1
697
698 // Now on a scheduling stack (a pthread-created stack).
699 #ifdef GOOS_aix
700 // Create a fake LR to improve backtrace.
701 MOVD $runtime·asmcgocall(SB), R6
702 MOVD R6, 16(R1)
703 // AIX also saves one argument on the stack.
704 SUB $8, R1
705 #endif
706 // Save room for two of our pointers, plus the callee
707 // save area that lives on the caller stack.
708 // Do arithmetics in R10 to hide from the assembler
709 // counting it as SP delta, which is irrelevant as we are
710 // on the system stack.
711 SUB $(asmcgocallSaveOffset+16), R1, R10
712 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
713 MOVD R5, (asmcgocallSaveOffset+8)(R1) // save old g on stack
714 MOVD (g_stack+stack_hi)(R5), R5
715 SUB R7, R5
716 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
717 #ifdef GOOS_aix
718 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb
719 #else
720 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
721 #endif
722 // This is a "global call", so put the global entry point in r12
723 MOVD R3, R12
724
725 #ifdef GO_PPC64X_HAS_FUNCDESC
726 // Load the real entry address from the first slot of the function descriptor.
727 MOVD 8(R12), R2
728 MOVD (R12), R12
729 #endif
730 MOVD R12, CTR
731 MOVD R4, R3 // arg in r3
732 BL (CTR)
733
734 // Reinitialise zero value register.
735 XOR R0, R0
736
737 // Restore g, stack pointer, toc pointer.
738 // R3 is errno, so don't touch it
739 MOVD (asmcgocallSaveOffset+8)(R1), g
740 MOVD (g_stack+stack_hi)(g), R5
741 MOVD asmcgocallSaveOffset(R1), R6
742 SUB R6, R5
743 #ifndef GOOS_aix
744 MOVD 24(R5), R2
745 #endif
746 MOVD R5, R1
747 BL runtime·save_g(SB)
748
749 // ret = R3
750 RET
751
752 nosave:
753 // Running on a system stack, perhaps even without a g.
754 // Having no g can happen during thread creation or thread teardown.
755 // This code is like the above sequence but without saving/restoring g
756 // and without worrying about the stack moving out from under us
757 // (because we're on a system stack, not a goroutine stack).
758
759 SUB $(asmcgocallSaveOffset+8), R1, R10
760 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
761 MOVD R7, asmcgocallSaveOffset(R1) // Save original stack pointer.
762
763 MOVD R3, R12 // fn
764 #ifdef GO_PPC64X_HAS_FUNCDESC
765 // Load the real entry address from the first slot of the function descriptor.
766 MOVD 8(R12), R2
767 MOVD (R12), R12
768 #endif
769 MOVD R12, CTR
770 MOVD R4, R3 // arg
771 BL (CTR)
772
773 // Reinitialise zero value register.
774 XOR R0, R0
775
776 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
777 #ifndef GOOS_aix
778 MOVD 24(R1), R2
779 #endif
780 // ret = R3
781 RET
782
783 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
784 // See cgocall.go for more details.
785 TEXT ·cgocallback(SB),NOSPLIT,$24-24
786 NO_LOCAL_POINTERS
787
788 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
789 // It is used to dropm while thread is exiting.
790 MOVD fn+0(FP), R5
791 CMP R5, $0
792 BNE loadg
793 // Restore the g from frame.
794 MOVD frame+8(FP), g
795 BR dropm
796
797 loadg:
798 // Load m and g from thread-local storage.
799 #ifndef GOOS_openbsd
800 MOVBZ runtime·iscgo(SB), R3
801 CMP R3, $0
802 BEQ nocgo
803 #endif
804 BL runtime·load_g(SB)
805 nocgo:
806
807 // If g is nil, Go did not create the current thread,
808 // or if this thread never called into Go on pthread platforms.
809 // Call needm to obtain one for temporary use.
810 // In this case, we're running on the thread stack, so there's
811 // lots of space, but the linker doesn't know. Hide the call from
812 // the linker analysis by using an indirect call.
813 CMP g, $0
814 BEQ needm
815
816 MOVD g_m(g), R8
817 MOVD R8, savedm-8(SP)
818 BR havem
819
820 needm:
821 MOVD g, savedm-8(SP) // g is zero, so is m.
822 MOVD $runtime·needAndBindM(SB), R12
823 MOVD R12, CTR
824 BL (CTR)
825
826 // Set m->sched.sp = SP, so that if a panic happens
827 // during the function we are about to execute, it will
828 // have a valid SP to run on the g0 stack.
829 // The next few lines (after the havem label)
830 // will save this SP onto the stack and then write
831 // the same SP back to m->sched.sp. That seems redundant,
832 // but if an unrecovered panic happens, unwindm will
833 // restore the g->sched.sp from the stack location
834 // and then systemstack will try to use it. If we don't set it here,
835 // that restored SP will be uninitialized (typically 0) and
836 // will not be usable.
837 MOVD g_m(g), R8
838 MOVD m_g0(R8), R3
839 MOVD R1, (g_sched+gobuf_sp)(R3)
840
841 havem:
842 // Now there's a valid m, and we're running on its m->g0.
843 // Save current m->g0->sched.sp on stack and then set it to SP.
844 // Save current sp in m->g0->sched.sp in preparation for
845 // switch back to m->curg stack.
846 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
847 MOVD m_g0(R8), R3
848 MOVD (g_sched+gobuf_sp)(R3), R4
849 MOVD R4, savedsp-24(SP) // must match frame size
850 MOVD R1, (g_sched+gobuf_sp)(R3)
851
852 // Switch to m->curg stack and call runtime.cgocallbackg.
853 // Because we are taking over the execution of m->curg
854 // but *not* resuming what had been running, we need to
855 // save that information (m->curg->sched) so we can restore it.
856 // We can restore m->curg->sched.sp easily, because calling
857 // runtime.cgocallbackg leaves SP unchanged upon return.
858 // To save m->curg->sched.pc, we push it onto the curg stack and
859 // open a frame the same size as cgocallback's g0 frame.
860 // Once we switch to the curg stack, the pushed PC will appear
861 // to be the return PC of cgocallback, so that the traceback
862 // will seamlessly trace back into the earlier calls.
863 MOVD m_curg(R8), g
864 BL runtime·save_g(SB)
865 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
866 MOVD (g_sched+gobuf_pc)(g), R5
867 MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size
868 // Gather our arguments into registers.
869 MOVD fn+0(FP), R5
870 MOVD frame+8(FP), R6
871 MOVD ctxt+16(FP), R7
872 MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size
873 MOVD R5, FIXED_FRAME+0(R1)
874 MOVD R6, FIXED_FRAME+8(R1)
875 MOVD R7, FIXED_FRAME+16(R1)
876
877 MOVD $runtime·cgocallbackg(SB), R12
878 MOVD R12, CTR
879 CALL (CTR) // indirect call to bypass nosplit check. We're on a different stack now.
880
881 // Restore g->sched (== m->curg->sched) from saved values.
882 MOVD 0(R1), R5
883 MOVD R5, (g_sched+gobuf_pc)(g)
884 MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size
885 MOVD R4, (g_sched+gobuf_sp)(g)
886
887 // Switch back to m->g0's stack and restore m->g0->sched.sp.
888 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
889 // so we do not have to restore it.)
890 MOVD g_m(g), R8
891 MOVD m_g0(R8), g
892 BL runtime·save_g(SB)
893 MOVD (g_sched+gobuf_sp)(g), R1
894 MOVD savedsp-24(SP), R4 // must match frame size
895 MOVD R4, (g_sched+gobuf_sp)(g)
896
897 // If the m on entry was nil, we called needm above to borrow an m,
898 // 1. for the duration of the call on non-pthread platforms,
899 // 2. or the duration of the C thread alive on pthread platforms.
900 // If the m on entry wasn't nil,
901 // 1. the thread might be a Go thread,
902 // 2. or it wasn't the first call from a C thread on pthread platforms,
903 // since then we skip dropm to reuse the m in the first call.
904 MOVD savedm-8(SP), R6
905 CMP R6, $0
906 BNE droppedm
907
908 // Skip dropm to reuse it in the next call, when a pthread key has been created.
909 MOVD _cgo_pthread_key_created(SB), R6
910 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
911 CMP R6, $0
912 BEQ dropm
913 MOVD (R6), R6
914 CMP R6, $0
915 BNE droppedm
916
917 dropm:
918 MOVD $runtime·dropm(SB), R12
919 MOVD R12, CTR
920 BL (CTR)
921 droppedm:
922
923 // Done!
924 RET
925
926 // void setg(G*); set g. for use by needm.
927 TEXT runtime·setg(SB), NOSPLIT, $0-8
928 MOVD gg+0(FP), g
929 // This only happens if iscgo, so jump straight to save_g
930 BL runtime·save_g(SB)
931 RET
932
933 #ifdef GO_PPC64X_HAS_FUNCDESC
934 DEFINE_PPC64X_FUNCDESC(setg_gcc<>, _setg_gcc<>)
935 TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
936 #else
937 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
938 #endif
939 // The standard prologue clobbers R31, which is callee-save in
940 // the C ABI, so we have to use $-8-0 and save LR ourselves.
941 MOVD LR, R4
942 // Also save g and R31, since they're callee-save in C ABI
943 MOVD R31, R5
944 MOVD g, R6
945
946 MOVD R3, g
947 BL runtime·save_g(SB)
948
949 MOVD R6, g
950 MOVD R5, R31
951 MOVD R4, LR
952 RET
953
954 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
955 MOVW (R0), R0
956 UNDEF
957
958 #define TBR 268
959
960 // int64 runtime·cputicks(void)
961 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
962 MOVD SPR(TBR), R3
963 MOVD R3, ret+0(FP)
964 RET
965
966 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
967 TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
968 MOVD R3, 0(R20)
969 MOVD R4, 8(R20)
970 MOVD R5, 16(R20)
971 MOVD R6, 24(R20)
972 MOVD R7, 32(R20)
973 MOVD R8, 40(R20)
974 MOVD R9, 48(R20)
975 MOVD R10, 56(R20)
976 MOVD R14, 64(R20)
977 MOVD R15, 72(R20)
978 MOVD R16, 80(R20)
979 MOVD R17, 88(R20)
980 FMOVD F1, 96(R20)
981 FMOVD F2, 104(R20)
982 FMOVD F3, 112(R20)
983 FMOVD F4, 120(R20)
984 FMOVD F5, 128(R20)
985 FMOVD F6, 136(R20)
986 FMOVD F7, 144(R20)
987 FMOVD F8, 152(R20)
988 FMOVD F9, 160(R20)
989 FMOVD F10, 168(R20)
990 FMOVD F11, 176(R20)
991 FMOVD F12, 184(R20)
992 RET
993
994 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
995 TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
996 MOVD 0(R20), R3
997 MOVD 8(R20), R4
998 MOVD 16(R20), R5
999 MOVD 24(R20), R6
1000 MOVD 32(R20), R7
1001 MOVD 40(R20), R8
1002 MOVD 48(R20), R9
1003 MOVD 56(R20), R10
1004 MOVD 64(R20), R14
1005 MOVD 72(R20), R15
1006 MOVD 80(R20), R16
1007 MOVD 88(R20), R17
1008 FMOVD 96(R20), F1
1009 FMOVD 104(R20), F2
1010 FMOVD 112(R20), F3
1011 FMOVD 120(R20), F4
1012 FMOVD 128(R20), F5
1013 FMOVD 136(R20), F6
1014 FMOVD 144(R20), F7
1015 FMOVD 152(R20), F8
1016 FMOVD 160(R20), F9
1017 FMOVD 168(R20), F10
1018 FMOVD 176(R20), F11
1019 FMOVD 184(R20), F12
1020 RET
1021
1022 // AES hashing not implemented for ppc64
1023 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
1024 JMP runtime·memhashFallback<ABIInternal>(SB)
1025 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1026 JMP runtime·strhashFallback<ABIInternal>(SB)
1027 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1028 JMP runtime·memhash32Fallback<ABIInternal>(SB)
1029 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1030 JMP runtime·memhash64Fallback<ABIInternal>(SB)
1031
1032 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
1033 // Must obey the gcc calling convention.
1034 #ifdef GOOS_aix
1035 // On AIX, _cgo_topofstack is defined in runtime/cgo, because it must
1036 // be a longcall in order to prevent trampolines from ld.
1037 TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
1038 #else
1039 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
1040 #endif
1041 // g (R30) and R31 are callee-save in the C ABI, so save them
1042 MOVD g, R4
1043 MOVD R31, R5
1044 MOVD LR, R6
1045
1046 BL runtime·load_g(SB) // clobbers g (R30), R31
1047 MOVD g_m(g), R3
1048 MOVD m_curg(R3), R3
1049 MOVD (g_stack+stack_hi)(R3), R3
1050
1051 MOVD R4, g
1052 MOVD R5, R31
1053 MOVD R6, LR
1054 RET
1055
1056 // The top-most function running on a goroutine
1057 // returns to goexit+PCQuantum.
1058 //
1059 // When dynamically linking Go, it can be returned to from a function
1060 // implemented in a different module and so needs to reload the TOC pointer
1061 // from the stack (although this function declares that it does not set up x-a
1062 // frame, newproc1 does in fact allocate one for goexit and saves the TOC
1063 // pointer in the correct place).
1064 // goexit+_PCQuantum is halfway through the usual global entry point prologue
1065 // that derives r2 from r12 which is a bit silly, but not harmful.
1066 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
1067 MOVD 24(R1), R2
1068 BL runtime·goexit1(SB) // does not return
1069 // traceback from goexit1 must hit code range of goexit
1070 MOVD R0, R0 // NOP
1071
1072 // prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the
1073 // module containing runtime) to the frame that goexit will execute in when
1074 // the goroutine exits. It's implemented in assembly mainly because that's the
1075 // easiest way to get access to R2.
1076 TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8
1077 MOVD sp+0(FP), R3
1078 MOVD R2, 24(R3)
1079 RET
1080
1081 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
1082 ADD $-8, R1
1083 MOVD R31, 0(R1)
1084 MOVD runtime·lastmoduledatap(SB), R4
1085 MOVD R3, moduledata_next(R4)
1086 MOVD R3, runtime·lastmoduledatap(SB)
1087 MOVD 0(R1), R31
1088 ADD $8, R1
1089 RET
1090
1091 TEXT ·checkASM(SB),NOSPLIT,$0-1
1092 MOVW $1, R3
1093 MOVB R3, ret+0(FP)
1094 RET
1095
1096 // gcWriteBarrier informs the GC about heap pointer writes.
1097 //
1098 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
1099 // number of bytes of buffer needed in R29, and returns a pointer
1100 // to the buffer space in R29.
1101 // It clobbers condition codes.
1102 // It does not clobber R0 through R17 (except special registers),
1103 // but may clobber any other register, *including* R31.
1104 TEXT gcWriteBarrier<>(SB),NOSPLIT,$120
1105 // The standard prologue clobbers R31.
1106 // We use R18, R19, and R31 as scratch registers.
1107 retry:
1108 MOVD g_m(g), R18
1109 MOVD m_p(R18), R18
1110 MOVD (p_wbBuf+wbBuf_next)(R18), R19
1111 MOVD (p_wbBuf+wbBuf_end)(R18), R31
1112 // Increment wbBuf.next position.
1113 ADD R29, R19
1114 // Is the buffer full?
1115 CMPU R31, R19
1116 BLT flush
1117 // Commit to the larger buffer.
1118 MOVD R19, (p_wbBuf+wbBuf_next)(R18)
1119 // Make return value (the original next position)
1120 SUB R29, R19, R29
1121 RET
1122
1123 flush:
1124 // Save registers R0 through R15 since these were not saved by the caller.
1125 // We don't save all registers on ppc64 because it takes too much space.
1126 MOVD R20, (FIXED_FRAME+0)(R1)
1127 MOVD R21, (FIXED_FRAME+8)(R1)
1128 // R0 is always 0, so no need to spill.
1129 // R1 is SP.
1130 // R2 is SB.
1131 MOVD R3, (FIXED_FRAME+16)(R1)
1132 MOVD R4, (FIXED_FRAME+24)(R1)
1133 MOVD R5, (FIXED_FRAME+32)(R1)
1134 MOVD R6, (FIXED_FRAME+40)(R1)
1135 MOVD R7, (FIXED_FRAME+48)(R1)
1136 MOVD R8, (FIXED_FRAME+56)(R1)
1137 MOVD R9, (FIXED_FRAME+64)(R1)
1138 MOVD R10, (FIXED_FRAME+72)(R1)
1139 // R11, R12 may be clobbered by external-linker-inserted trampoline
1140 // R13 is REGTLS
1141 MOVD R14, (FIXED_FRAME+80)(R1)
1142 MOVD R15, (FIXED_FRAME+88)(R1)
1143 MOVD R16, (FIXED_FRAME+96)(R1)
1144 MOVD R17, (FIXED_FRAME+104)(R1)
1145 MOVD R29, (FIXED_FRAME+112)(R1)
1146
1147 CALL runtime·wbBufFlush(SB)
1148
1149 MOVD (FIXED_FRAME+0)(R1), R20
1150 MOVD (FIXED_FRAME+8)(R1), R21
1151 MOVD (FIXED_FRAME+16)(R1), R3
1152 MOVD (FIXED_FRAME+24)(R1), R4
1153 MOVD (FIXED_FRAME+32)(R1), R5
1154 MOVD (FIXED_FRAME+40)(R1), R6
1155 MOVD (FIXED_FRAME+48)(R1), R7
1156 MOVD (FIXED_FRAME+56)(R1), R8
1157 MOVD (FIXED_FRAME+64)(R1), R9
1158 MOVD (FIXED_FRAME+72)(R1), R10
1159 MOVD (FIXED_FRAME+80)(R1), R14
1160 MOVD (FIXED_FRAME+88)(R1), R15
1161 MOVD (FIXED_FRAME+96)(R1), R16
1162 MOVD (FIXED_FRAME+104)(R1), R17
1163 MOVD (FIXED_FRAME+112)(R1), R29
1164 JMP retry
1165
1166 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
1167 MOVD $8, R29
1168 JMP gcWriteBarrier<>(SB)
1169 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
1170 MOVD $16, R29
1171 JMP gcWriteBarrier<>(SB)
1172 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
1173 MOVD $24, R29
1174 JMP gcWriteBarrier<>(SB)
1175 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
1176 MOVD $32, R29
1177 JMP gcWriteBarrier<>(SB)
1178 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
1179 MOVD $40, R29
1180 JMP gcWriteBarrier<>(SB)
1181 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
1182 MOVD $48, R29
1183 JMP gcWriteBarrier<>(SB)
1184 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
1185 MOVD $56, R29
1186 JMP gcWriteBarrier<>(SB)
1187 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
1188 MOVD $64, R29
1189 JMP gcWriteBarrier<>(SB)
1190
1191 DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1192 GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1193
1194 // debugCallV2 is the entry point for debugger-injected function
1195 // calls on running goroutines. It informs the runtime that a
1196 // debug call has been injected and creates a call frame for the
1197 // debugger to fill in.
1198 //
1199 // To inject a function call, a debugger should:
1200 // 1. Check that the goroutine is in state _Grunning and that
1201 // there are at least 320 bytes free on the stack.
1202 // 2. Set SP as SP-32.
1203 // 3. Store the current LR in (SP) (using the SP after step 2).
1204 // 4. Store the current PC in the LR register.
1205 // 5. Write the desired argument frame size at SP-32
1206 // 6. Save all machine registers (including flags and floating point registers)
1207 // so they can be restored later by the debugger.
1208 // 7. Set the PC to debugCallV2 and resume execution.
1209 //
1210 // If the goroutine is in state _Grunnable, then it's not generally
1211 // safe to inject a call because it may return out via other runtime
1212 // operations. Instead, the debugger should unwind the stack to find
1213 // the return to non-runtime code, add a temporary breakpoint there,
1214 // and inject the call once that breakpoint is hit.
1215 //
1216 // If the goroutine is in any other state, it's not safe to inject a call.
1217 //
1218 // This function communicates back to the debugger by setting R20 and
1219 // invoking TW to raise a breakpoint signal. Note that the signal PC of
1220 // the signal triggered by the TW instruction is the PC where the signal
1221 // is trapped, not the next PC, so to resume execution, the debugger needs
1222 // to set the signal PC to PC+4. See the comments in the implementation for
1223 // the protocol the debugger is expected to follow. InjectDebugCall in the
1224 // runtime tests demonstrates this protocol.
1225 // The debugger must ensure that any pointers passed to the function
1226 // obey escape analysis requirements. Specifically, it must not pass
1227 // a stack pointer to an escaping argument. debugCallV2 cannot check
1228 // this invariant.
1229 //
1230 // This is ABIInternal because Go code injects its PC directly into new
1231 // goroutine stacks.
1232 #ifdef GOARCH_ppc64le
1233 TEXT runtime·debugCallV2<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-0
1234 // save scratch register R31 first
1235 MOVD R31, -184(R1)
1236 MOVD 0(R1), R31
1237 // save caller LR
1238 MOVD R31, -304(R1)
1239 MOVD -32(R1), R31
1240 // save argument frame size
1241 MOVD R31, -192(R1)
1242 MOVD LR, R31
1243 MOVD R31, -320(R1)
1244 ADD $-320, R1
1245 // save all registers that can contain pointers
1246 // and the CR register
1247 MOVW CR, R31
1248 MOVD R31, 8(R1)
1249 MOVD R2, 24(R1)
1250 MOVD R3, 56(R1)
1251 MOVD R4, 64(R1)
1252 MOVD R5, 72(R1)
1253 MOVD R6, 80(R1)
1254 MOVD R7, 88(R1)
1255 MOVD R8, 96(R1)
1256 MOVD R9, 104(R1)
1257 MOVD R10, 112(R1)
1258 MOVD R11, 120(R1)
1259 MOVD R12, 144(R1)
1260 MOVD R13, 152(R1)
1261 MOVD R14, 160(R1)
1262 MOVD R15, 168(R1)
1263 MOVD R16, 176(R1)
1264 MOVD R17, 184(R1)
1265 MOVD R18, 192(R1)
1266 MOVD R19, 200(R1)
1267 MOVD R20, 208(R1)
1268 MOVD R21, 216(R1)
1269 MOVD R22, 224(R1)
1270 MOVD R23, 232(R1)
1271 MOVD R24, 240(R1)
1272 MOVD R25, 248(R1)
1273 MOVD R26, 256(R1)
1274 MOVD R27, 264(R1)
1275 MOVD R28, 272(R1)
1276 MOVD R29, 280(R1)
1277 MOVD g, 288(R1)
1278 MOVD LR, R31
1279 MOVD R31, 32(R1)
1280 CALL runtime·debugCallCheck(SB)
1281 MOVD 40(R1), R22
1282 XOR R0, R0
1283 CMP R22, $0
1284 BEQ good
1285 MOVD 48(R1), R22
1286 MOVD $8, R20
1287 TW $31, R0, R0
1288
1289 BR restore
1290
1291 good:
1292 #define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1293 MOVD $MAXSIZE, R23; \
1294 CMP R26, R23; \
1295 BGT 5(PC); \
1296 MOVD $NAME(SB), R26; \
1297 MOVD R26, 32(R1); \
1298 CALL runtime·debugCallWrap(SB); \
1299 BR restore
1300
1301 // the argument frame size
1302 MOVD 128(R1), R26
1303
1304 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1305 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1306 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1307 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1308 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1309 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1310 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1311 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1312 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1313 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1314 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1315 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1316 // The frame size is too large. Report the error.
1317 MOVD $debugCallFrameTooLarge<>(SB), R22
1318 MOVD R22, 32(R1)
1319 MOVD $20, R22
1320 // length of debugCallFrameTooLarge string
1321 MOVD R22, 40(R1)
1322 MOVD $8, R20
1323 TW $31, R0, R0
1324 BR restore
1325 restore:
1326 MOVD $16, R20
1327 TW $31, R0, R0
1328 // restore all registers that can contain
1329 // pointers including CR
1330 MOVD 8(R1), R31
1331 MOVW R31, CR
1332 MOVD 24(R1), R2
1333 MOVD 56(R1), R3
1334 MOVD 64(R1), R4
1335 MOVD 72(R1), R5
1336 MOVD 80(R1), R6
1337 MOVD 88(R1), R7
1338 MOVD 96(R1), R8
1339 MOVD 104(R1), R9
1340 MOVD 112(R1), R10
1341 MOVD 120(R1), R11
1342 MOVD 144(R1), R12
1343 MOVD 152(R1), R13
1344 MOVD 160(R1), R14
1345 MOVD 168(R1), R15
1346 MOVD 176(R1), R16
1347 MOVD 184(R1), R17
1348 MOVD 192(R1), R18
1349 MOVD 200(R1), R19
1350 MOVD 208(R1), R20
1351 MOVD 216(R1), R21
1352 MOVD 224(R1), R22
1353 MOVD 232(R1), R23
1354 MOVD 240(R1), R24
1355 MOVD 248(R1), R25
1356 MOVD 256(R1), R26
1357 MOVD 264(R1), R27
1358 MOVD 272(R1), R28
1359 MOVD 280(R1), R29
1360 MOVD 288(R1), g
1361 MOVD 16(R1), R31
1362 // restore old LR
1363 MOVD R31, LR
1364 // restore caller PC
1365 MOVD 0(R1), CTR
1366 MOVD 136(R1), R31
1367 // Add 32 bytes more to compensate for SP change in saveSigContext
1368 ADD $352, R1
1369 JMP (CTR)
1370 #endif
1371 #define DEBUG_CALL_FN(NAME,MAXSIZE) \
1372 TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1373 NO_LOCAL_POINTERS; \
1374 MOVD $0, R20; \
1375 TW $31, R0, R0 \
1376 MOVD $1, R20; \
1377 TW $31, R0, R0 \
1378 RET
1379 DEBUG_CALL_FN(debugCall32<>, 32)
1380 DEBUG_CALL_FN(debugCall64<>, 64)
1381 DEBUG_CALL_FN(debugCall128<>, 128)
1382 DEBUG_CALL_FN(debugCall256<>, 256)
1383 DEBUG_CALL_FN(debugCall512<>, 512)
1384 DEBUG_CALL_FN(debugCall1024<>, 1024)
1385 DEBUG_CALL_FN(debugCall2048<>, 2048)
1386 DEBUG_CALL_FN(debugCall4096<>, 4096)
1387 DEBUG_CALL_FN(debugCall8192<>, 8192)
1388 DEBUG_CALL_FN(debugCall16384<>, 16384)
1389 DEBUG_CALL_FN(debugCall32768<>, 32768)
1390 DEBUG_CALL_FN(debugCall65536<>, 65536)
1391
1392 #ifdef GOARCH_ppc64le
1393 // func debugCallPanicked(val interface{})
1394 TEXT runtime·debugCallPanicked(SB),NOSPLIT,$32-16
1395 // Copy the panic value to the top of stack at SP+32.
1396 MOVD val_type+0(FP), R31
1397 MOVD R31, 32(R1)
1398 MOVD val_data+8(FP), R31
1399 MOVD R31, 40(R1)
1400 MOVD $2, R20
1401 TW $31, R0, R0
1402 RET
1403 #endif
1404
1405 TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$88-0
1406 // Note: frame size is 16 bytes larger than necessary
1407 // in order to pacify vet. Vet doesn't understand ppc64
1408 // layout properly.
1409 NO_LOCAL_POINTERS
1410 // Save all 7 int registers that could have an index in them.
1411 // They may be pointers, but if so they are dead.
1412 // Skip R0 aka ZERO, R1 aka SP, R2 aka SB
1413 MOVD R3, 48(R1)
1414 MOVD R4, 56(R1)
1415 MOVD R5, 64(R1)
1416 MOVD R6, 72(R1)
1417 MOVD R7, 80(R1)
1418 MOVD R8, 88(R1)
1419 MOVD R9, 96(R1)
1420 // Note: we only save 7 registers to keep under nosplit stack limit
1421 // Also, R11 is clobbered in dynamic linking situations
1422
1423 MOVD LR, R3 // PC immediately after call to panicBounds
1424 ADD $48, R1, R4 // pointer to save area
1425 CALL runtime·panicBounds64<ABIInternal>(SB)
1426 RET
1427
1428 // These functions are used when internal linking cgo with external
1429 // objects compiled with the -Os on gcc. They reduce prologue/epilogue
1430 // size by deferring preservation of callee-save registers to a shared
1431 // function. These are defined in PPC64 ELFv2 2.3.3 (but also present
1432 // in ELFv1)
1433 //
1434 // These appear unused, but the linker will redirect calls to functions
1435 // like _savegpr0_14 or _restgpr1_14 to runtime.elf_savegpr0 or
1436 // runtime.elf_restgpr1 with an appropriate offset based on the number
1437 // register operations required when linking external objects which
1438 // make these calls. For GPR/FPR saves, the minimum register value is
1439 // 14, for VR it is 20.
1440 //
1441 // These are only used when linking such cgo code internally. Note, R12
1442 // and R0 may be used in different ways than regular ELF compliant
1443 // functions.
1444 TEXT runtime·elf_savegpr0(SB),NOSPLIT|NOFRAME,$0
1445 // R0 holds the LR of the caller's caller, R1 holds save location
1446 MOVD R14, -144(R1)
1447 MOVD R15, -136(R1)
1448 MOVD R16, -128(R1)
1449 MOVD R17, -120(R1)
1450 MOVD R18, -112(R1)
1451 MOVD R19, -104(R1)
1452 MOVD R20, -96(R1)
1453 MOVD R21, -88(R1)
1454 MOVD R22, -80(R1)
1455 MOVD R23, -72(R1)
1456 MOVD R24, -64(R1)
1457 MOVD R25, -56(R1)
1458 MOVD R26, -48(R1)
1459 MOVD R27, -40(R1)
1460 MOVD R28, -32(R1)
1461 MOVD R29, -24(R1)
1462 MOVD g, -16(R1)
1463 MOVD R31, -8(R1)
1464 MOVD R0, 16(R1)
1465 RET
1466 TEXT runtime·elf_restgpr0(SB),NOSPLIT|NOFRAME,$0
1467 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1468 MOVD -144(R1), R14
1469 MOVD -136(R1), R15
1470 MOVD -128(R1), R16
1471 MOVD -120(R1), R17
1472 MOVD -112(R1), R18
1473 MOVD -104(R1), R19
1474 MOVD -96(R1), R20
1475 MOVD -88(R1), R21
1476 MOVD -80(R1), R22
1477 MOVD -72(R1), R23
1478 MOVD -64(R1), R24
1479 MOVD -56(R1), R25
1480 MOVD -48(R1), R26
1481 MOVD -40(R1), R27
1482 MOVD -32(R1), R28
1483 MOVD -24(R1), R29
1484 MOVD -16(R1), g
1485 MOVD -8(R1), R31
1486 MOVD 16(R1), R0 // Load and return to saved LR
1487 MOVD R0, LR
1488 RET
1489 TEXT runtime·elf_savegpr1(SB),NOSPLIT|NOFRAME,$0
1490 // R12 holds the save location
1491 MOVD R14, -144(R12)
1492 MOVD R15, -136(R12)
1493 MOVD R16, -128(R12)
1494 MOVD R17, -120(R12)
1495 MOVD R18, -112(R12)
1496 MOVD R19, -104(R12)
1497 MOVD R20, -96(R12)
1498 MOVD R21, -88(R12)
1499 MOVD R22, -80(R12)
1500 MOVD R23, -72(R12)
1501 MOVD R24, -64(R12)
1502 MOVD R25, -56(R12)
1503 MOVD R26, -48(R12)
1504 MOVD R27, -40(R12)
1505 MOVD R28, -32(R12)
1506 MOVD R29, -24(R12)
1507 MOVD g, -16(R12)
1508 MOVD R31, -8(R12)
1509 RET
1510 TEXT runtime·elf_restgpr1(SB),NOSPLIT|NOFRAME,$0
1511 // R12 holds the save location
1512 MOVD -144(R12), R14
1513 MOVD -136(R12), R15
1514 MOVD -128(R12), R16
1515 MOVD -120(R12), R17
1516 MOVD -112(R12), R18
1517 MOVD -104(R12), R19
1518 MOVD -96(R12), R20
1519 MOVD -88(R12), R21
1520 MOVD -80(R12), R22
1521 MOVD -72(R12), R23
1522 MOVD -64(R12), R24
1523 MOVD -56(R12), R25
1524 MOVD -48(R12), R26
1525 MOVD -40(R12), R27
1526 MOVD -32(R12), R28
1527 MOVD -24(R12), R29
1528 MOVD -16(R12), g
1529 MOVD -8(R12), R31
1530 RET
1531 TEXT runtime·elf_savefpr(SB),NOSPLIT|NOFRAME,$0
1532 // R0 holds the LR of the caller's caller, R1 holds save location
1533 FMOVD F14, -144(R1)
1534 FMOVD F15, -136(R1)
1535 FMOVD F16, -128(R1)
1536 FMOVD F17, -120(R1)
1537 FMOVD F18, -112(R1)
1538 FMOVD F19, -104(R1)
1539 FMOVD F20, -96(R1)
1540 FMOVD F21, -88(R1)
1541 FMOVD F22, -80(R1)
1542 FMOVD F23, -72(R1)
1543 FMOVD F24, -64(R1)
1544 FMOVD F25, -56(R1)
1545 FMOVD F26, -48(R1)
1546 FMOVD F27, -40(R1)
1547 FMOVD F28, -32(R1)
1548 FMOVD F29, -24(R1)
1549 FMOVD F30, -16(R1)
1550 FMOVD F31, -8(R1)
1551 MOVD R0, 16(R1)
1552 RET
1553 TEXT runtime·elf_restfpr(SB),NOSPLIT|NOFRAME,$0
1554 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1555 FMOVD -144(R1), F14
1556 FMOVD -136(R1), F15
1557 FMOVD -128(R1), F16
1558 FMOVD -120(R1), F17
1559 FMOVD -112(R1), F18
1560 FMOVD -104(R1), F19
1561 FMOVD -96(R1), F20
1562 FMOVD -88(R1), F21
1563 FMOVD -80(R1), F22
1564 FMOVD -72(R1), F23
1565 FMOVD -64(R1), F24
1566 FMOVD -56(R1), F25
1567 FMOVD -48(R1), F26
1568 FMOVD -40(R1), F27
1569 FMOVD -32(R1), F28
1570 FMOVD -24(R1), F29
1571 FMOVD -16(R1), F30
1572 FMOVD -8(R1), F31
1573 MOVD 16(R1), R0 // Load and return to saved LR
1574 MOVD R0, LR
1575 RET
1576 TEXT runtime·elf_savevr(SB),NOSPLIT|NOFRAME,$0
1577 // R0 holds the save location, R12 is clobbered
1578 MOVD $-192, R12
1579 STVX V20, (R0+R12)
1580 MOVD $-176, R12
1581 STVX V21, (R0+R12)
1582 MOVD $-160, R12
1583 STVX V22, (R0+R12)
1584 MOVD $-144, R12
1585 STVX V23, (R0+R12)
1586 MOVD $-128, R12
1587 STVX V24, (R0+R12)
1588 MOVD $-112, R12
1589 STVX V25, (R0+R12)
1590 MOVD $-96, R12
1591 STVX V26, (R0+R12)
1592 MOVD $-80, R12
1593 STVX V27, (R0+R12)
1594 MOVD $-64, R12
1595 STVX V28, (R0+R12)
1596 MOVD $-48, R12
1597 STVX V29, (R0+R12)
1598 MOVD $-32, R12
1599 STVX V30, (R0+R12)
1600 MOVD $-16, R12
1601 STVX V31, (R0+R12)
1602 RET
1603 TEXT runtime·elf_restvr(SB),NOSPLIT|NOFRAME,$0
1604 // R0 holds the save location, R12 is clobbered
1605 MOVD $-192, R12
1606 LVX (R0+R12), V20
1607 MOVD $-176, R12
1608 LVX (R0+R12), V21
1609 MOVD $-160, R12
1610 LVX (R0+R12), V22
1611 MOVD $-144, R12
1612 LVX (R0+R12), V23
1613 MOVD $-128, R12
1614 LVX (R0+R12), V24
1615 MOVD $-112, R12
1616 LVX (R0+R12), V25
1617 MOVD $-96, R12
1618 LVX (R0+R12), V26
1619 MOVD $-80, R12
1620 LVX (R0+R12), V27
1621 MOVD $-64, R12
1622 LVX (R0+R12), V28
1623 MOVD $-48, R12
1624 LVX (R0+R12), V29
1625 MOVD $-32, R12
1626 LVX (R0+R12), V30
1627 MOVD $-16, R12
1628 LVX (R0+R12), V31
1629 RET
1630
View as plain text