Text file
src/runtime/asm_ppc64x.s
1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build ppc64 || ppc64le
6
7 #include "go_asm.h"
8 #include "go_tls.h"
9 #include "funcdata.h"
10 #include "textflag.h"
11 #include "asm_ppc64x.h"
12 #include "cgo/abi_ppc64x.h"
13
14
15 TEXT _rt0_ppc64x_lib(SB),NOSPLIT|NOFRAME,$0
16 // This is called with ELFv2 calling conventions. Convert to Go.
17 // Allocate argument storage for call to newosproc0.
18 STACK_AND_SAVE_HOST_TO_GO_ABI(16)
19
20 MOVD R3, _rt0_ppc64x_lib_argc<>(SB)
21 MOVD R4, _rt0_ppc64x_lib_argv<>(SB)
22
23 // Synchronous initialization.
24 MOVD $runtime·reginit(SB), R12
25 MOVD R12, CTR
26 BL (CTR)
27 MOVD $runtime·libpreinit(SB), R12
28 MOVD R12, CTR
29 BL (CTR)
30
31 // Create a new thread to do the runtime initialization and return.
32 MOVD _cgo_sys_thread_create(SB), R12
33 CMP $0, R12
34 BEQ nocgo
35 MOVD $_rt0_ppc64x_lib_go(SB), R3
36 MOVD $0, R4
37 MOVD R12, CTR
38 BL (CTR)
39 BR done
40
41 nocgo:
42 MOVD $0x800000, R12 // stacksize = 8192KB
43 MOVD R12, 8+FIXED_FRAME(R1)
44 MOVD $_rt0_ppc64x_lib_go(SB), R12
45 MOVD R12, 16+FIXED_FRAME(R1)
46 MOVD $runtime·newosproc0(SB),R12
47 MOVD R12, CTR
48 BL (CTR)
49
50 done:
51 // Restore and return to ELFv2 caller.
52 UNSTACK_AND_RESTORE_GO_TO_HOST_ABI(16)
53 RET
54
55 #ifdef GO_PPC64X_HAS_FUNCDESC
56 DEFINE_PPC64X_FUNCDESC(_rt0_ppc64x_lib_go, __rt0_ppc64x_lib_go)
57 TEXT __rt0_ppc64x_lib_go(SB),NOSPLIT,$0
58 #else
59 TEXT _rt0_ppc64x_lib_go(SB),NOSPLIT,$0
60 #endif
61 MOVD _rt0_ppc64x_lib_argc<>(SB), R3
62 MOVD _rt0_ppc64x_lib_argv<>(SB), R4
63 MOVD $runtime·rt0_go(SB), R12
64 MOVD R12, CTR
65 BR (CTR)
66
67 DATA _rt0_ppc64x_lib_argc<>(SB)/8, $0
68 GLOBL _rt0_ppc64x_lib_argc<>(SB),NOPTR, $8
69 DATA _rt0_ppc64x_lib_argv<>(SB)/8, $0
70 GLOBL _rt0_ppc64x_lib_argv<>(SB),NOPTR, $8
71
72
73 #ifdef GOOS_aix
74 #define cgoCalleeStackSize 48
75 #else
76 #define cgoCalleeStackSize 32
77 #endif
78
79 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
80 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer
81
82 // initialize essential registers
83 BL runtime·reginit(SB)
84
85 SUB $(FIXED_FRAME+16), R1
86 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame
87 MOVW R3, FIXED_FRAME+0(R1) // argc
88 MOVD R4, FIXED_FRAME+8(R1) // argv
89
90 // create istack out of the given (operating system) stack.
91 // _cgo_init may update stackguard.
92 MOVD $runtime·g0(SB), g
93 BL runtime·save_g(SB)
94 MOVD $(-64*1024), R31
95 ADD R31, R1, R3
96 MOVD R3, g_stackguard0(g)
97 MOVD R3, g_stackguard1(g)
98 MOVD R3, (g_stack+stack_lo)(g)
99 MOVD R1, (g_stack+stack_hi)(g)
100
101 // If there is a _cgo_init, call it using the gcc ABI.
102 MOVD _cgo_init(SB), R12
103 CMP R12, $0
104 BEQ nocgo
105
106 #ifdef GO_PPC64X_HAS_FUNCDESC
107 // Load the real entry address from the first slot of the function descriptor.
108 MOVD 8(R12), R2
109 MOVD (R12), R12
110 #endif
111 MOVD R12, CTR // r12 = "global function entry point"
112 MOVD R13, R5 // arg 2: TLS base pointer
113 MOVD $setg_gcc<>(SB), R4 // arg 1: setg
114 MOVD g, R3 // arg 0: G
115 // C functions expect 32 (48 for AIX) bytes of space on caller
116 // stack frame and a 16-byte aligned R1
117 MOVD R1, R14 // save current stack
118 SUB $cgoCalleeStackSize, R1 // reserve the callee area
119 RLDCR $0, R1, $~15, R1 // 16-byte align
120 BL (CTR) // may clobber R0, R3-R12
121 MOVD R14, R1 // restore stack
122 #ifndef GOOS_aix
123 MOVD 24(R1), R2
124 #endif
125 XOR R0, R0 // fix R0
126
127 nocgo:
128 // update stackguard after _cgo_init
129 MOVD (g_stack+stack_lo)(g), R3
130 ADD $const_stackGuard, R3
131 MOVD R3, g_stackguard0(g)
132 MOVD R3, g_stackguard1(g)
133
134 // set the per-goroutine and per-mach "registers"
135 MOVD $runtime·m0(SB), R3
136
137 // save m->g0 = g0
138 MOVD g, m_g0(R3)
139 // save m0 to g0->m
140 MOVD R3, g_m(g)
141
142 BL runtime·check(SB)
143
144 // args are already prepared
145 BL runtime·args(SB)
146 BL runtime·osinit(SB)
147 BL runtime·schedinit(SB)
148
149 // create a new goroutine to start program
150 MOVD $runtime·mainPC(SB), R3 // entry
151 MOVDU R3, -8(R1)
152 MOVDU R0, -8(R1)
153 MOVDU R0, -8(R1)
154 MOVDU R0, -8(R1)
155 MOVDU R0, -8(R1)
156 BL runtime·newproc(SB)
157 ADD $(8+FIXED_FRAME), R1
158
159 // start this M
160 BL runtime·mstart(SB)
161 // Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are
162 // intended to be called by debuggers.
163 #ifdef GOARCH_ppc64le
164 MOVD $runtime·debugPinnerV1<ABIInternal>(SB), R31
165 MOVD $runtime·debugCallV2<ABIInternal>(SB), R31
166 #endif
167 MOVD R0, 0(R0)
168 RET
169
170 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
171 GLOBL runtime·mainPC(SB),RODATA,$8
172
173 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
174 TW $31, R0, R0
175 RET
176
177 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
178 RET
179
180 // Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall_ppc64
181 TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0
182 // crosscall_ppc64 and crosscall2 need to reginit, but can't
183 // get at the 'runtime.reginit' symbol.
184 BR runtime·reginit(SB)
185
186 TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0
187 // set R0 to zero, it's expected by the toolchain
188 XOR R0, R0
189 RET
190
191 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
192 BL runtime·mstart0(SB)
193 RET // not reached
194
195 /*
196 * go-routine
197 */
198
199 // void gogo(Gobuf*)
200 // restore state from Gobuf; longjmp
201 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
202 MOVD buf+0(FP), R5
203 MOVD gobuf_g(R5), R6
204 MOVD 0(R6), R4 // make sure g != nil
205 BR gogo<>(SB)
206
207 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
208 MOVD R6, g
209 BL runtime·save_g(SB)
210
211 MOVD gobuf_sp(R5), R1
212 MOVD gobuf_lr(R5), R31
213 #ifndef GOOS_aix
214 MOVD 24(R1), R2 // restore R2
215 #endif
216 MOVD R31, LR
217 MOVD gobuf_ctxt(R5), R11
218 MOVD R0, gobuf_sp(R5)
219 MOVD R0, gobuf_lr(R5)
220 MOVD R0, gobuf_ctxt(R5)
221 CMP R0, R0 // set condition codes for == test, needed by stack split
222 MOVD gobuf_pc(R5), R12
223 MOVD R12, CTR
224 BR (CTR)
225
226 // void mcall(fn func(*g))
227 // Switch to m->g0's stack, call fn(g).
228 // Fn must never return. It should gogo(&g->sched)
229 // to keep running g.
230 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
231 // Save caller state in g->sched
232 // R11 should be safe across save_g??
233 MOVD R3, R11
234 MOVD R1, (g_sched+gobuf_sp)(g)
235 MOVD LR, R31
236 MOVD R31, (g_sched+gobuf_pc)(g)
237 MOVD R0, (g_sched+gobuf_lr)(g)
238
239 // Switch to m->g0 & its stack, call fn.
240 MOVD g, R3
241 MOVD g_m(g), R8
242 MOVD m_g0(R8), g
243 BL runtime·save_g(SB)
244 CMP g, R3
245 BNE 2(PC)
246 BR runtime·badmcall(SB)
247 MOVD 0(R11), R12 // code pointer
248 MOVD R12, CTR
249 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp
250 // Don't need to do anything special for regabiargs here
251 // R3 is g; stack is set anyway
252 MOVDU R3, -8(R1)
253 MOVDU R0, -8(R1)
254 MOVDU R0, -8(R1)
255 MOVDU R0, -8(R1)
256 MOVDU R0, -8(R1)
257 BL (CTR)
258 MOVD 24(R1), R2
259 BR runtime·badmcall2(SB)
260
261 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
262 // of the G stack. We need to distinguish the routine that
263 // lives at the bottom of the G stack from the one that lives
264 // at the top of the system stack because the one at the top of
265 // the system stack terminates the stack walk (see topofstack()).
266 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
267 // We have several undefs here so that 16 bytes past
268 // $runtime·systemstack_switch lies within them whether or not the
269 // instructions that derive r2 from r12 are there.
270 UNDEF
271 UNDEF
272 UNDEF
273 BL (LR) // make sure this function is not leaf
274 RET
275
276 // func systemstack(fn func())
277 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
278 MOVD fn+0(FP), R3 // R3 = fn
279 MOVD R3, R11 // context
280 MOVD g_m(g), R4 // R4 = m
281
282 MOVD m_gsignal(R4), R5 // R5 = gsignal
283 CMP g, R5
284 BEQ noswitch
285
286 MOVD m_g0(R4), R5 // R5 = g0
287 CMP g, R5
288 BEQ noswitch
289
290 MOVD m_curg(R4), R6
291 CMP g, R6
292 BEQ switch
293
294 // Bad: g is not gsignal, not g0, not curg. What is it?
295 // Hide call from linker nosplit analysis.
296 MOVD $runtime·badsystemstack(SB), R12
297 MOVD R12, CTR
298 BL (CTR)
299 BL runtime·abort(SB)
300
301 switch:
302 // save our state in g->sched. Pretend to
303 // be systemstack_switch if the G stack is scanned.
304 BL gosave_systemstack_switch<>(SB)
305
306 // switch to g0
307 MOVD R5, g
308 BL runtime·save_g(SB)
309 MOVD (g_sched+gobuf_sp)(g), R1
310
311 // call target function
312 MOVD 0(R11), R12 // code pointer
313 MOVD R12, CTR
314 BL (CTR)
315
316 // restore TOC pointer. It seems unlikely that we will use systemstack
317 // to call a function defined in another module, but the results of
318 // doing so would be so confusing that it's worth doing this.
319 MOVD g_m(g), R3
320 MOVD m_curg(R3), g
321 MOVD (g_sched+gobuf_sp)(g), R3
322 #ifndef GOOS_aix
323 MOVD 24(R3), R2
324 #endif
325 // switch back to g
326 MOVD g_m(g), R3
327 MOVD m_curg(R3), g
328 BL runtime·save_g(SB)
329 MOVD (g_sched+gobuf_sp)(g), R1
330 MOVD R0, (g_sched+gobuf_sp)(g)
331 RET
332
333 noswitch:
334 // already on m stack, just call directly
335 // On other arches we do a tail call here, but it appears to be
336 // impossible to tail call a function pointer in shared mode on
337 // ppc64 because the caller is responsible for restoring the TOC.
338 MOVD 0(R11), R12 // code pointer
339 MOVD R12, CTR
340 BL (CTR)
341 #ifndef GOOS_aix
342 MOVD 24(R1), R2
343 #endif
344 RET
345
346 // func switchToCrashStack0(fn func())
347 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
348 MOVD R3, R11 // context register
349 MOVD g_m(g), R3 // curm
350
351 // set g to gcrash
352 MOVD $runtime·gcrash(SB), g // g = &gcrash
353 CALL runtime·save_g(SB) // clobbers R31
354 MOVD R3, g_m(g) // g.m = curm
355 MOVD g, m_g0(R3) // curm.g0 = g
356
357 // switch to crashstack
358 MOVD (g_stack+stack_hi)(g), R3
359 SUB $(4*8), R3
360 MOVD R3, R1
361
362 // call target function
363 MOVD 0(R11), R12 // code pointer
364 MOVD R12, CTR
365 BL (CTR)
366
367 // should never return
368 CALL runtime·abort(SB)
369 UNDEF
370
371 /*
372 * support for morestack
373 */
374
375 // Called during function prolog when more stack is needed.
376 // Caller has already loaded:
377 // R3: framesize, R4: argsize, R5: LR
378 //
379 // The traceback routines see morestack on a g0 as being
380 // the top of a stack (for example, morestack calling newstack
381 // calling the scheduler calling newm calling gc), so we must
382 // record an argument size. For that purpose, it has no arguments.
383 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
384 // Called from f.
385 // Set g->sched to context in f.
386 MOVD R1, (g_sched+gobuf_sp)(g)
387 MOVD LR, R8
388 MOVD R8, (g_sched+gobuf_pc)(g)
389 MOVD R5, (g_sched+gobuf_lr)(g)
390 MOVD R11, (g_sched+gobuf_ctxt)(g)
391
392 // Cannot grow scheduler stack (m->g0).
393 MOVD g_m(g), R7
394 MOVD m_g0(R7), R8
395 CMP g, R8
396 BNE 3(PC)
397 BL runtime·badmorestackg0(SB)
398 BL runtime·abort(SB)
399
400 // Cannot grow signal stack (m->gsignal).
401 MOVD m_gsignal(R7), R8
402 CMP g, R8
403 BNE 3(PC)
404 BL runtime·badmorestackgsignal(SB)
405 BL runtime·abort(SB)
406
407 // Called from f.
408 // Set m->morebuf to f's caller.
409 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
410 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
411 MOVD g, (m_morebuf+gobuf_g)(R7)
412
413 // Call newstack on m->g0's stack.
414 MOVD m_g0(R7), g
415 BL runtime·save_g(SB)
416 MOVD (g_sched+gobuf_sp)(g), R1
417 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0
418 BL runtime·newstack(SB)
419
420 // Not reached, but make sure the return PC from the call to newstack
421 // is still in this function, and not the beginning of the next.
422 UNDEF
423
424 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
425 // Force SPWRITE. This function doesn't actually write SP,
426 // but it is called with a special calling convention where
427 // the caller doesn't save LR on stack but passes it as a
428 // register (R5), and the unwinder currently doesn't understand.
429 // Make it SPWRITE to stop unwinding. (See issue 54332)
430 // Use OR R0, R1 instead of MOVD R1, R1 as the MOVD instruction
431 // has a special affect on Power8,9,10 by lowering the thread
432 // priority and causing a slowdown in execution time
433
434 OR R0, R1
435 MOVD R0, R11
436 BR runtime·morestack(SB)
437
438 // reflectcall: call a function with the given argument list
439 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
440 // we don't have variable-sized frames, so we use a small number
441 // of constant-sized-frame functions to encode a few bits of size in the pc.
442 // Caution: ugly multiline assembly macros in your future!
443
444 #define DISPATCH(NAME,MAXSIZE) \
445 MOVD $MAXSIZE, R31; \
446 CMP R3, R31; \
447 BGT 4(PC); \
448 MOVD $NAME(SB), R12; \
449 MOVD R12, CTR; \
450 BR (CTR)
451 // Note: can't just "BR NAME(SB)" - bad inlining results.
452
453 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
454 MOVWZ frameSize+32(FP), R3
455 DISPATCH(runtime·call16, 16)
456 DISPATCH(runtime·call32, 32)
457 DISPATCH(runtime·call64, 64)
458 DISPATCH(runtime·call128, 128)
459 DISPATCH(runtime·call256, 256)
460 DISPATCH(runtime·call512, 512)
461 DISPATCH(runtime·call1024, 1024)
462 DISPATCH(runtime·call2048, 2048)
463 DISPATCH(runtime·call4096, 4096)
464 DISPATCH(runtime·call8192, 8192)
465 DISPATCH(runtime·call16384, 16384)
466 DISPATCH(runtime·call32768, 32768)
467 DISPATCH(runtime·call65536, 65536)
468 DISPATCH(runtime·call131072, 131072)
469 DISPATCH(runtime·call262144, 262144)
470 DISPATCH(runtime·call524288, 524288)
471 DISPATCH(runtime·call1048576, 1048576)
472 DISPATCH(runtime·call2097152, 2097152)
473 DISPATCH(runtime·call4194304, 4194304)
474 DISPATCH(runtime·call8388608, 8388608)
475 DISPATCH(runtime·call16777216, 16777216)
476 DISPATCH(runtime·call33554432, 33554432)
477 DISPATCH(runtime·call67108864, 67108864)
478 DISPATCH(runtime·call134217728, 134217728)
479 DISPATCH(runtime·call268435456, 268435456)
480 DISPATCH(runtime·call536870912, 536870912)
481 DISPATCH(runtime·call1073741824, 1073741824)
482 MOVD $runtime·badreflectcall(SB), R12
483 MOVD R12, CTR
484 BR (CTR)
485
486 #define CALLFN(NAME,MAXSIZE) \
487 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
488 NO_LOCAL_POINTERS; \
489 /* copy arguments to stack */ \
490 MOVD stackArgs+16(FP), R3; \
491 MOVWZ stackArgsSize+24(FP), R4; \
492 MOVD R1, R5; \
493 CMP R4, $8; \
494 BLT tailsetup; \
495 /* copy 8 at a time if possible */ \
496 ADD $(FIXED_FRAME-8), R5; \
497 SUB $8, R3; \
498 top: \
499 MOVDU 8(R3), R7; \
500 MOVDU R7, 8(R5); \
501 SUB $8, R4; \
502 CMP R4, $8; \
503 BGE top; \
504 /* handle remaining bytes */ \
505 CMP $0, R4; \
506 BEQ callfn; \
507 ADD $7, R3; \
508 ADD $7, R5; \
509 BR tail; \
510 tailsetup: \
511 CMP $0, R4; \
512 BEQ callfn; \
513 ADD $(FIXED_FRAME-1), R5; \
514 SUB $1, R3; \
515 tail: \
516 MOVBU 1(R3), R6; \
517 MOVBU R6, 1(R5); \
518 SUB $1, R4; \
519 CMP $0, R4; \
520 BGT tail; \
521 callfn: \
522 /* call function */ \
523 MOVD f+8(FP), R11; \
524 #ifdef GOOS_aix \
525 /* AIX won't trigger a SIGSEGV if R11 = nil */ \
526 /* So it manually triggers it */ \
527 CMP R11, $0 \
528 BNE 2(PC) \
529 MOVD R0, 0(R0) \
530 #endif \
531 MOVD regArgs+40(FP), R20; \
532 BL runtime·unspillArgs(SB); \
533 MOVD (R11), R12; \
534 MOVD R12, CTR; \
535 PCDATA $PCDATA_StackMapIndex, $0; \
536 BL (CTR); \
537 #ifndef GOOS_aix \
538 MOVD 24(R1), R2; \
539 #endif \
540 /* copy return values back */ \
541 MOVD regArgs+40(FP), R20; \
542 BL runtime·spillArgs(SB); \
543 MOVD stackArgsType+0(FP), R7; \
544 MOVD stackArgs+16(FP), R3; \
545 MOVWZ stackArgsSize+24(FP), R4; \
546 MOVWZ stackRetOffset+28(FP), R6; \
547 ADD $FIXED_FRAME, R1, R5; \
548 ADD R6, R5; \
549 ADD R6, R3; \
550 SUB R6, R4; \
551 BL callRet<>(SB); \
552 RET
553
554 // callRet copies return values back at the end of call*. This is a
555 // separate function so it can allocate stack space for the arguments
556 // to reflectcallmove. It does not follow the Go ABI; it expects its
557 // arguments in registers.
558 TEXT callRet<>(SB), NOSPLIT, $40-0
559 NO_LOCAL_POINTERS
560 MOVD R7, FIXED_FRAME+0(R1)
561 MOVD R3, FIXED_FRAME+8(R1)
562 MOVD R5, FIXED_FRAME+16(R1)
563 MOVD R4, FIXED_FRAME+24(R1)
564 MOVD R20, FIXED_FRAME+32(R1)
565 BL runtime·reflectcallmove(SB)
566 RET
567
568 CALLFN(·call16, 16)
569 CALLFN(·call32, 32)
570 CALLFN(·call64, 64)
571 CALLFN(·call128, 128)
572 CALLFN(·call256, 256)
573 CALLFN(·call512, 512)
574 CALLFN(·call1024, 1024)
575 CALLFN(·call2048, 2048)
576 CALLFN(·call4096, 4096)
577 CALLFN(·call8192, 8192)
578 CALLFN(·call16384, 16384)
579 CALLFN(·call32768, 32768)
580 CALLFN(·call65536, 65536)
581 CALLFN(·call131072, 131072)
582 CALLFN(·call262144, 262144)
583 CALLFN(·call524288, 524288)
584 CALLFN(·call1048576, 1048576)
585 CALLFN(·call2097152, 2097152)
586 CALLFN(·call4194304, 4194304)
587 CALLFN(·call8388608, 8388608)
588 CALLFN(·call16777216, 16777216)
589 CALLFN(·call33554432, 33554432)
590 CALLFN(·call67108864, 67108864)
591 CALLFN(·call134217728, 134217728)
592 CALLFN(·call268435456, 268435456)
593 CALLFN(·call536870912, 536870912)
594 CALLFN(·call1073741824, 1073741824)
595
596 TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0-4
597 MOVW cycles+0(FP), R7
598 // POWER does not have a pause/yield instruction equivalent.
599 // Instead, we can lower the program priority by setting the
600 // Program Priority Register prior to the wait loop and set it
601 // back to default afterwards. On Linux, the default priority is
602 // medium-low. For details, see page 837 of the ISA 3.0.
603 OR R1, R1, R1 // Set PPR priority to low
604 again:
605 SUB $1, R7
606 CMP $0, R7
607 BNE again
608 OR R6, R6, R6 // Set PPR priority back to medium-low
609 RET
610
611 // Save state of caller into g->sched,
612 // but using fake PC from systemstack_switch.
613 // Must only be called from functions with no locals ($0)
614 // or else unwinding from systemstack_switch is incorrect.
615 // Smashes R31.
616 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
617 MOVD $runtime·systemstack_switch(SB), R31
618 ADD $16, R31 // get past prologue (including r2-setting instructions when they're there)
619 MOVD R31, (g_sched+gobuf_pc)(g)
620 MOVD R1, (g_sched+gobuf_sp)(g)
621 MOVD R0, (g_sched+gobuf_lr)(g)
622 // Assert ctxt is zero. See func save.
623 MOVD (g_sched+gobuf_ctxt)(g), R31
624 CMP R31, $0
625 BEQ 2(PC)
626 BL runtime·abort(SB)
627 RET
628
629 #ifdef GOOS_aix
630 #define asmcgocallSaveOffset cgoCalleeStackSize + 8
631 #else
632 #define asmcgocallSaveOffset cgoCalleeStackSize
633 #endif
634
635 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
636 // Call fn(arg) aligned appropriately for the gcc ABI.
637 // Called on a system stack, and there may be no g yet (during needm).
638 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
639 MOVD fn+0(FP), R3
640 MOVD arg+8(FP), R4
641
642 MOVD R1, R15
643 SUB $(asmcgocallSaveOffset+8), R1
644 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
645 MOVD R15, asmcgocallSaveOffset(R1)
646
647 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
648
649 // This is a "global call", so put the global entry point in r12
650 MOVD R3, R12
651
652 #ifdef GO_PPC64X_HAS_FUNCDESC
653 // Load the real entry address from the first slot of the function descriptor.
654 MOVD 8(R12), R2
655 MOVD (R12), R12
656 #endif
657 MOVD R12, CTR
658 MOVD R4, R3 // arg in r3
659 BL (CTR)
660
661 // C code can clobber R0, so set it back to 0. F27-F31 are
662 // callee save, so we don't need to recover those.
663 XOR R0, R0
664
665 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
666 #ifndef GOOS_aix
667 MOVD 24(R1), R2
668 #endif
669
670 RET
671
672 // func asmcgocall(fn, arg unsafe.Pointer) int32
673 // Call fn(arg) on the scheduler stack,
674 // aligned appropriately for the gcc ABI.
675 // See cgocall.go for more details.
676 TEXT ·asmcgocall<ABIInternal>(SB),NOSPLIT,$0-20
677 // R3 = fn
678 // R4 = arg
679
680 MOVD R1, R7 // save original stack pointer
681 CMP $0, g
682 BEQ nosave
683 MOVD g, R5
684
685 // Figure out if we need to switch to m->g0 stack.
686 // We get called to create new OS threads too, and those
687 // come in on the m->g0 stack already. Or we might already
688 // be on the m->gsignal stack.
689 MOVD g_m(g), R8
690 MOVD m_gsignal(R8), R6
691 CMP R6, g
692 BEQ nosave
693 MOVD m_g0(R8), R6
694 CMP R6, g
695 BEQ nosave
696
697 BL gosave_systemstack_switch<>(SB)
698 MOVD R6, g
699 BL runtime·save_g(SB)
700 MOVD (g_sched+gobuf_sp)(g), R1
701
702 // Now on a scheduling stack (a pthread-created stack).
703 #ifdef GOOS_aix
704 // Create a fake LR to improve backtrace.
705 MOVD $runtime·asmcgocall(SB), R6
706 MOVD R6, 16(R1)
707 // AIX also saves one argument on the stack.
708 SUB $8, R1
709 #endif
710 // Save room for two of our pointers, plus the callee
711 // save area that lives on the caller stack.
712 // Do arithmetics in R10 to hide from the assembler
713 // counting it as SP delta, which is irrelevant as we are
714 // on the system stack.
715 SUB $(asmcgocallSaveOffset+16), R1, R10
716 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
717 MOVD R5, (asmcgocallSaveOffset+8)(R1) // save old g on stack
718 MOVD (g_stack+stack_hi)(R5), R5
719 SUB R7, R5
720 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
721 #ifdef GOOS_aix
722 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb
723 #else
724 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
725 #endif
726 // This is a "global call", so put the global entry point in r12
727 MOVD R3, R12
728
729 #ifdef GO_PPC64X_HAS_FUNCDESC
730 // Load the real entry address from the first slot of the function descriptor.
731 MOVD 8(R12), R2
732 MOVD (R12), R12
733 #endif
734 MOVD R12, CTR
735 MOVD R4, R3 // arg in r3
736 BL (CTR)
737
738 // Reinitialise zero value register.
739 XOR R0, R0
740
741 // Restore g, stack pointer, toc pointer.
742 // R3 is errno, so don't touch it
743 MOVD (asmcgocallSaveOffset+8)(R1), g
744 MOVD (g_stack+stack_hi)(g), R5
745 MOVD asmcgocallSaveOffset(R1), R6
746 SUB R6, R5
747 #ifndef GOOS_aix
748 MOVD 24(R5), R2
749 #endif
750 MOVD R5, R1
751 BL runtime·save_g(SB)
752
753 // ret = R3
754 RET
755
756 nosave:
757 // Running on a system stack, perhaps even without a g.
758 // Having no g can happen during thread creation or thread teardown.
759 // This code is like the above sequence but without saving/restoring g
760 // and without worrying about the stack moving out from under us
761 // (because we're on a system stack, not a goroutine stack).
762 // The above code could be used directly if already on a system stack,
763 // but then the only path through this code would be a rare case.
764 // Using this code for all "already on system stack" calls exercises it more,
765 // which should help keep it correct.
766
767 SUB $(asmcgocallSaveOffset+8), R1, R10
768 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
769 MOVD R7, asmcgocallSaveOffset(R1) // Save original stack pointer.
770
771 MOVD R3, R12 // fn
772 #ifdef GO_PPC64X_HAS_FUNCDESC
773 // Load the real entry address from the first slot of the function descriptor.
774 MOVD 8(R12), R2
775 MOVD (R12), R12
776 #endif
777 MOVD R12, CTR
778 MOVD R4, R3 // arg
779 BL (CTR)
780
781 // Reinitialise zero value register.
782 XOR R0, R0
783
784 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
785 #ifndef GOOS_aix
786 MOVD 24(R1), R2
787 #endif
788 // ret = R3
789 RET
790
791 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
792 // See cgocall.go for more details.
793 TEXT ·cgocallback(SB),NOSPLIT,$24-24
794 NO_LOCAL_POINTERS
795
796 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
797 // It is used to dropm while thread is exiting.
798 MOVD fn+0(FP), R5
799 CMP R5, $0
800 BNE loadg
801 // Restore the g from frame.
802 MOVD frame+8(FP), g
803 BR dropm
804
805 loadg:
806 // Load m and g from thread-local storage.
807 #ifndef GOOS_openbsd
808 MOVBZ runtime·iscgo(SB), R3
809 CMP R3, $0
810 BEQ nocgo
811 #endif
812 BL runtime·load_g(SB)
813 nocgo:
814
815 // If g is nil, Go did not create the current thread,
816 // or if this thread never called into Go on pthread platforms.
817 // Call needm to obtain one for temporary use.
818 // In this case, we're running on the thread stack, so there's
819 // lots of space, but the linker doesn't know. Hide the call from
820 // the linker analysis by using an indirect call.
821 CMP g, $0
822 BEQ needm
823
824 MOVD g_m(g), R8
825 MOVD R8, savedm-8(SP)
826 BR havem
827
828 needm:
829 MOVD g, savedm-8(SP) // g is zero, so is m.
830 MOVD $runtime·needAndBindM(SB), R12
831 MOVD R12, CTR
832 BL (CTR)
833
834 // Set m->sched.sp = SP, so that if a panic happens
835 // during the function we are about to execute, it will
836 // have a valid SP to run on the g0 stack.
837 // The next few lines (after the havem label)
838 // will save this SP onto the stack and then write
839 // the same SP back to m->sched.sp. That seems redundant,
840 // but if an unrecovered panic happens, unwindm will
841 // restore the g->sched.sp from the stack location
842 // and then systemstack will try to use it. If we don't set it here,
843 // that restored SP will be uninitialized (typically 0) and
844 // will not be usable.
845 MOVD g_m(g), R8
846 MOVD m_g0(R8), R3
847 MOVD R1, (g_sched+gobuf_sp)(R3)
848
849 havem:
850 // Now there's a valid m, and we're running on its m->g0.
851 // Save current m->g0->sched.sp on stack and then set it to SP.
852 // Save current sp in m->g0->sched.sp in preparation for
853 // switch back to m->curg stack.
854 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
855 MOVD m_g0(R8), R3
856 MOVD (g_sched+gobuf_sp)(R3), R4
857 MOVD R4, savedsp-24(SP) // must match frame size
858 MOVD R1, (g_sched+gobuf_sp)(R3)
859
860 // Switch to m->curg stack and call runtime.cgocallbackg.
861 // Because we are taking over the execution of m->curg
862 // but *not* resuming what had been running, we need to
863 // save that information (m->curg->sched) so we can restore it.
864 // We can restore m->curg->sched.sp easily, because calling
865 // runtime.cgocallbackg leaves SP unchanged upon return.
866 // To save m->curg->sched.pc, we push it onto the curg stack and
867 // open a frame the same size as cgocallback's g0 frame.
868 // Once we switch to the curg stack, the pushed PC will appear
869 // to be the return PC of cgocallback, so that the traceback
870 // will seamlessly trace back into the earlier calls.
871 MOVD m_curg(R8), g
872 BL runtime·save_g(SB)
873 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
874 MOVD (g_sched+gobuf_pc)(g), R5
875 MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size
876 // Gather our arguments into registers.
877 MOVD fn+0(FP), R5
878 MOVD frame+8(FP), R6
879 MOVD ctxt+16(FP), R7
880 MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size
881 MOVD R5, FIXED_FRAME+0(R1)
882 MOVD R6, FIXED_FRAME+8(R1)
883 MOVD R7, FIXED_FRAME+16(R1)
884
885 MOVD $runtime·cgocallbackg(SB), R12
886 MOVD R12, CTR
887 CALL (CTR) // indirect call to bypass nosplit check. We're on a different stack now.
888
889 // Restore g->sched (== m->curg->sched) from saved values.
890 MOVD 0(R1), R5
891 MOVD R5, (g_sched+gobuf_pc)(g)
892 MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size
893 MOVD R4, (g_sched+gobuf_sp)(g)
894
895 // Switch back to m->g0's stack and restore m->g0->sched.sp.
896 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
897 // so we do not have to restore it.)
898 MOVD g_m(g), R8
899 MOVD m_g0(R8), g
900 BL runtime·save_g(SB)
901 MOVD (g_sched+gobuf_sp)(g), R1
902 MOVD savedsp-24(SP), R4 // must match frame size
903 MOVD R4, (g_sched+gobuf_sp)(g)
904
905 // If the m on entry was nil, we called needm above to borrow an m,
906 // 1. for the duration of the call on non-pthread platforms,
907 // 2. or the duration of the C thread alive on pthread platforms.
908 // If the m on entry wasn't nil,
909 // 1. the thread might be a Go thread,
910 // 2. or it wasn't the first call from a C thread on pthread platforms,
911 // since then we skip dropm to reuse the m in the first call.
912 MOVD savedm-8(SP), R6
913 CMP R6, $0
914 BNE droppedm
915
916 // Skip dropm to reuse it in the next call, when a pthread key has been created.
917 MOVD _cgo_pthread_key_created(SB), R6
918 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
919 CMP R6, $0
920 BEQ dropm
921 MOVD (R6), R6
922 CMP R6, $0
923 BNE droppedm
924
925 dropm:
926 MOVD $runtime·dropm(SB), R12
927 MOVD R12, CTR
928 BL (CTR)
929 droppedm:
930
931 // Done!
932 RET
933
934 // void setg(G*); set g. for use by needm.
935 TEXT runtime·setg(SB), NOSPLIT, $0-8
936 MOVD gg+0(FP), g
937 // This only happens if iscgo, so jump straight to save_g
938 BL runtime·save_g(SB)
939 RET
940
941 #ifdef GO_PPC64X_HAS_FUNCDESC
942 DEFINE_PPC64X_FUNCDESC(setg_gcc<>, _setg_gcc<>)
943 TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
944 #else
945 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
946 #endif
947 // The standard prologue clobbers R31, which is callee-save in
948 // the C ABI, so we have to use $-8-0 and save LR ourselves.
949 MOVD LR, R4
950 // Also save g and R31, since they're callee-save in C ABI
951 MOVD R31, R5
952 MOVD g, R6
953
954 MOVD R3, g
955 BL runtime·save_g(SB)
956
957 MOVD R6, g
958 MOVD R5, R31
959 MOVD R4, LR
960 RET
961
962 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
963 MOVW (R0), R0
964 UNDEF
965
966 #define TBR 268
967
968 // int64 runtime·cputicks(void)
969 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
970 MOVD SPR(TBR), R3
971 MOVD R3, ret+0(FP)
972 RET
973
974 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
975 TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
976 MOVD R3, 0(R20)
977 MOVD R4, 8(R20)
978 MOVD R5, 16(R20)
979 MOVD R6, 24(R20)
980 MOVD R7, 32(R20)
981 MOVD R8, 40(R20)
982 MOVD R9, 48(R20)
983 MOVD R10, 56(R20)
984 MOVD R14, 64(R20)
985 MOVD R15, 72(R20)
986 MOVD R16, 80(R20)
987 MOVD R17, 88(R20)
988 FMOVD F1, 96(R20)
989 FMOVD F2, 104(R20)
990 FMOVD F3, 112(R20)
991 FMOVD F4, 120(R20)
992 FMOVD F5, 128(R20)
993 FMOVD F6, 136(R20)
994 FMOVD F7, 144(R20)
995 FMOVD F8, 152(R20)
996 FMOVD F9, 160(R20)
997 FMOVD F10, 168(R20)
998 FMOVD F11, 176(R20)
999 FMOVD F12, 184(R20)
1000 RET
1001
1002 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
1003 TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
1004 MOVD 0(R20), R3
1005 MOVD 8(R20), R4
1006 MOVD 16(R20), R5
1007 MOVD 24(R20), R6
1008 MOVD 32(R20), R7
1009 MOVD 40(R20), R8
1010 MOVD 48(R20), R9
1011 MOVD 56(R20), R10
1012 MOVD 64(R20), R14
1013 MOVD 72(R20), R15
1014 MOVD 80(R20), R16
1015 MOVD 88(R20), R17
1016 FMOVD 96(R20), F1
1017 FMOVD 104(R20), F2
1018 FMOVD 112(R20), F3
1019 FMOVD 120(R20), F4
1020 FMOVD 128(R20), F5
1021 FMOVD 136(R20), F6
1022 FMOVD 144(R20), F7
1023 FMOVD 152(R20), F8
1024 FMOVD 160(R20), F9
1025 FMOVD 168(R20), F10
1026 FMOVD 176(R20), F11
1027 FMOVD 184(R20), F12
1028 RET
1029
1030 // AES hashing not implemented for ppc64
1031 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
1032 JMP runtime·memhashFallback<ABIInternal>(SB)
1033 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1034 JMP runtime·strhashFallback<ABIInternal>(SB)
1035 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1036 JMP runtime·memhash32Fallback<ABIInternal>(SB)
1037 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1038 JMP runtime·memhash64Fallback<ABIInternal>(SB)
1039
1040 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
1041 // Must obey the gcc calling convention.
1042 #ifdef GOOS_aix
1043 // On AIX, _cgo_topofstack is defined in runtime/cgo, because it must
1044 // be a longcall in order to prevent trampolines from ld.
1045 TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
1046 #else
1047 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
1048 #endif
1049 // g (R30) and R31 are callee-save in the C ABI, so save them
1050 MOVD g, R4
1051 MOVD R31, R5
1052 MOVD LR, R6
1053
1054 BL runtime·load_g(SB) // clobbers g (R30), R31
1055 MOVD g_m(g), R3
1056 MOVD m_curg(R3), R3
1057 MOVD (g_stack+stack_hi)(R3), R3
1058
1059 MOVD R4, g
1060 MOVD R5, R31
1061 MOVD R6, LR
1062 RET
1063
1064 // The top-most function running on a goroutine
1065 // returns to goexit+PCQuantum.
1066 //
1067 // When dynamically linking Go, it can be returned to from a function
1068 // implemented in a different module and so needs to reload the TOC pointer
1069 // from the stack (although this function declares that it does not set up x-a
1070 // frame, newproc1 does in fact allocate one for goexit and saves the TOC
1071 // pointer in the correct place).
1072 // goexit+_PCQuantum is halfway through the usual global entry point prologue
1073 // that derives r2 from r12 which is a bit silly, but not harmful.
1074 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
1075 MOVD 24(R1), R2
1076 BL runtime·goexit1(SB) // does not return
1077 // traceback from goexit1 must hit code range of goexit
1078 MOVD R0, R0 // NOP
1079
1080 // prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the
1081 // module containing runtime) to the frame that goexit will execute in when
1082 // the goroutine exits. It's implemented in assembly mainly because that's the
1083 // easiest way to get access to R2.
1084 TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8
1085 MOVD sp+0(FP), R3
1086 MOVD R2, 24(R3)
1087 RET
1088
1089 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
1090 ADD $-8, R1
1091 MOVD R31, 0(R1)
1092 MOVD runtime·lastmoduledatap(SB), R4
1093 MOVD R3, moduledata_next(R4)
1094 MOVD R3, runtime·lastmoduledatap(SB)
1095 MOVD 0(R1), R31
1096 ADD $8, R1
1097 RET
1098
1099 TEXT ·checkASM(SB),NOSPLIT,$0-1
1100 MOVW $1, R3
1101 MOVB R3, ret+0(FP)
1102 RET
1103
1104 // gcWriteBarrier informs the GC about heap pointer writes.
1105 //
1106 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
1107 // number of bytes of buffer needed in R29, and returns a pointer
1108 // to the buffer space in R29.
1109 // It clobbers condition codes.
1110 // It does not clobber R0 through R17 (except special registers),
1111 // but may clobber any other register, *including* R31.
1112 TEXT gcWriteBarrier<>(SB),NOSPLIT,$120
1113 // The standard prologue clobbers R31.
1114 // We use R18, R19, and R31 as scratch registers.
1115 retry:
1116 MOVD g_m(g), R18
1117 MOVD m_p(R18), R18
1118 MOVD (p_wbBuf+wbBuf_next)(R18), R19
1119 MOVD (p_wbBuf+wbBuf_end)(R18), R31
1120 // Increment wbBuf.next position.
1121 ADD R29, R19
1122 // Is the buffer full?
1123 CMPU R31, R19
1124 BLT flush
1125 // Commit to the larger buffer.
1126 MOVD R19, (p_wbBuf+wbBuf_next)(R18)
1127 // Make return value (the original next position)
1128 SUB R29, R19, R29
1129 RET
1130
1131 flush:
1132 // Save registers R0 through R15 since these were not saved by the caller.
1133 // We don't save all registers on ppc64 because it takes too much space.
1134 MOVD R20, (FIXED_FRAME+0)(R1)
1135 MOVD R21, (FIXED_FRAME+8)(R1)
1136 // R0 is always 0, so no need to spill.
1137 // R1 is SP.
1138 // R2 is SB.
1139 MOVD R3, (FIXED_FRAME+16)(R1)
1140 MOVD R4, (FIXED_FRAME+24)(R1)
1141 MOVD R5, (FIXED_FRAME+32)(R1)
1142 MOVD R6, (FIXED_FRAME+40)(R1)
1143 MOVD R7, (FIXED_FRAME+48)(R1)
1144 MOVD R8, (FIXED_FRAME+56)(R1)
1145 MOVD R9, (FIXED_FRAME+64)(R1)
1146 MOVD R10, (FIXED_FRAME+72)(R1)
1147 // R11, R12 may be clobbered by external-linker-inserted trampoline
1148 // R13 is REGTLS
1149 MOVD R14, (FIXED_FRAME+80)(R1)
1150 MOVD R15, (FIXED_FRAME+88)(R1)
1151 MOVD R16, (FIXED_FRAME+96)(R1)
1152 MOVD R17, (FIXED_FRAME+104)(R1)
1153 MOVD R29, (FIXED_FRAME+112)(R1)
1154
1155 CALL runtime·wbBufFlush(SB)
1156
1157 MOVD (FIXED_FRAME+0)(R1), R20
1158 MOVD (FIXED_FRAME+8)(R1), R21
1159 MOVD (FIXED_FRAME+16)(R1), R3
1160 MOVD (FIXED_FRAME+24)(R1), R4
1161 MOVD (FIXED_FRAME+32)(R1), R5
1162 MOVD (FIXED_FRAME+40)(R1), R6
1163 MOVD (FIXED_FRAME+48)(R1), R7
1164 MOVD (FIXED_FRAME+56)(R1), R8
1165 MOVD (FIXED_FRAME+64)(R1), R9
1166 MOVD (FIXED_FRAME+72)(R1), R10
1167 MOVD (FIXED_FRAME+80)(R1), R14
1168 MOVD (FIXED_FRAME+88)(R1), R15
1169 MOVD (FIXED_FRAME+96)(R1), R16
1170 MOVD (FIXED_FRAME+104)(R1), R17
1171 MOVD (FIXED_FRAME+112)(R1), R29
1172 JMP retry
1173
1174 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
1175 MOVD $8, R29
1176 JMP gcWriteBarrier<>(SB)
1177 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
1178 MOVD $16, R29
1179 JMP gcWriteBarrier<>(SB)
1180 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
1181 MOVD $24, R29
1182 JMP gcWriteBarrier<>(SB)
1183 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
1184 MOVD $32, R29
1185 JMP gcWriteBarrier<>(SB)
1186 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
1187 MOVD $40, R29
1188 JMP gcWriteBarrier<>(SB)
1189 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
1190 MOVD $48, R29
1191 JMP gcWriteBarrier<>(SB)
1192 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
1193 MOVD $56, R29
1194 JMP gcWriteBarrier<>(SB)
1195 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
1196 MOVD $64, R29
1197 JMP gcWriteBarrier<>(SB)
1198
1199 DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1200 GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1201
1202 // debugCallV2 is the entry point for debugger-injected function
1203 // calls on running goroutines. It informs the runtime that a
1204 // debug call has been injected and creates a call frame for the
1205 // debugger to fill in.
1206 //
1207 // To inject a function call, a debugger should:
1208 // 1. Check that the goroutine is in state _Grunning and that
1209 // there are at least 320 bytes free on the stack.
1210 // 2. Set SP as SP-32.
1211 // 3. Store the current LR in (SP) (using the SP after step 2).
1212 // 4. Store the current PC in the LR register.
1213 // 5. Write the desired argument frame size at SP-32
1214 // 6. Save all machine registers (including flags and floating point registers)
1215 // so they can be restored later by the debugger.
1216 // 7. Set the PC to debugCallV2 and resume execution.
1217 //
1218 // If the goroutine is in state _Grunnable, then it's not generally
1219 // safe to inject a call because it may return out via other runtime
1220 // operations. Instead, the debugger should unwind the stack to find
1221 // the return to non-runtime code, add a temporary breakpoint there,
1222 // and inject the call once that breakpoint is hit.
1223 //
1224 // If the goroutine is in any other state, it's not safe to inject a call.
1225 //
1226 // This function communicates back to the debugger by setting R20 and
1227 // invoking TW to raise a breakpoint signal. Note that the signal PC of
1228 // the signal triggered by the TW instruction is the PC where the signal
1229 // is trapped, not the next PC, so to resume execution, the debugger needs
1230 // to set the signal PC to PC+4. See the comments in the implementation for
1231 // the protocol the debugger is expected to follow. InjectDebugCall in the
1232 // runtime tests demonstrates this protocol.
1233 // The debugger must ensure that any pointers passed to the function
1234 // obey escape analysis requirements. Specifically, it must not pass
1235 // a stack pointer to an escaping argument. debugCallV2 cannot check
1236 // this invariant.
1237 //
1238 // This is ABIInternal because Go code injects its PC directly into new
1239 // goroutine stacks.
1240 #ifdef GOARCH_ppc64le
1241 TEXT runtime·debugCallV2<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-0
1242 // save scratch register R31 first
1243 MOVD R31, -184(R1)
1244 MOVD 0(R1), R31
1245 // save caller LR
1246 MOVD R31, -304(R1)
1247 MOVD -32(R1), R31
1248 // save argument frame size
1249 MOVD R31, -192(R1)
1250 MOVD LR, R31
1251 MOVD R31, -320(R1)
1252 ADD $-320, R1
1253 // save all registers that can contain pointers
1254 // and the CR register
1255 MOVW CR, R31
1256 MOVD R31, 8(R1)
1257 MOVD R2, 24(R1)
1258 MOVD R3, 56(R1)
1259 MOVD R4, 64(R1)
1260 MOVD R5, 72(R1)
1261 MOVD R6, 80(R1)
1262 MOVD R7, 88(R1)
1263 MOVD R8, 96(R1)
1264 MOVD R9, 104(R1)
1265 MOVD R10, 112(R1)
1266 MOVD R11, 120(R1)
1267 MOVD R12, 144(R1)
1268 MOVD R13, 152(R1)
1269 MOVD R14, 160(R1)
1270 MOVD R15, 168(R1)
1271 MOVD R16, 176(R1)
1272 MOVD R17, 184(R1)
1273 MOVD R18, 192(R1)
1274 MOVD R19, 200(R1)
1275 MOVD R20, 208(R1)
1276 MOVD R21, 216(R1)
1277 MOVD R22, 224(R1)
1278 MOVD R23, 232(R1)
1279 MOVD R24, 240(R1)
1280 MOVD R25, 248(R1)
1281 MOVD R26, 256(R1)
1282 MOVD R27, 264(R1)
1283 MOVD R28, 272(R1)
1284 MOVD R29, 280(R1)
1285 MOVD g, 288(R1)
1286 MOVD LR, R31
1287 MOVD R31, 32(R1)
1288 CALL runtime·debugCallCheck(SB)
1289 MOVD 40(R1), R22
1290 XOR R0, R0
1291 CMP R22, $0
1292 BEQ good
1293 MOVD 48(R1), R22
1294 MOVD $8, R20
1295 TW $31, R0, R0
1296
1297 BR restore
1298
1299 good:
1300 #define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1301 MOVD $MAXSIZE, R23; \
1302 CMP R26, R23; \
1303 BGT 5(PC); \
1304 MOVD $NAME(SB), R26; \
1305 MOVD R26, 32(R1); \
1306 CALL runtime·debugCallWrap(SB); \
1307 BR restore
1308
1309 // the argument frame size
1310 MOVD 128(R1), R26
1311
1312 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1313 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1314 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1315 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1316 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1317 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1318 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1319 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1320 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1321 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1322 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1323 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1324 // The frame size is too large. Report the error.
1325 MOVD $debugCallFrameTooLarge<>(SB), R22
1326 MOVD R22, 32(R1)
1327 MOVD $20, R22
1328 // length of debugCallFrameTooLarge string
1329 MOVD R22, 40(R1)
1330 MOVD $8, R20
1331 TW $31, R0, R0
1332 BR restore
1333 restore:
1334 MOVD $16, R20
1335 TW $31, R0, R0
1336 // restore all registers that can contain
1337 // pointers including CR
1338 MOVD 8(R1), R31
1339 MOVW R31, CR
1340 MOVD 24(R1), R2
1341 MOVD 56(R1), R3
1342 MOVD 64(R1), R4
1343 MOVD 72(R1), R5
1344 MOVD 80(R1), R6
1345 MOVD 88(R1), R7
1346 MOVD 96(R1), R8
1347 MOVD 104(R1), R9
1348 MOVD 112(R1), R10
1349 MOVD 120(R1), R11
1350 MOVD 144(R1), R12
1351 MOVD 152(R1), R13
1352 MOVD 160(R1), R14
1353 MOVD 168(R1), R15
1354 MOVD 176(R1), R16
1355 MOVD 184(R1), R17
1356 MOVD 192(R1), R18
1357 MOVD 200(R1), R19
1358 MOVD 208(R1), R20
1359 MOVD 216(R1), R21
1360 MOVD 224(R1), R22
1361 MOVD 232(R1), R23
1362 MOVD 240(R1), R24
1363 MOVD 248(R1), R25
1364 MOVD 256(R1), R26
1365 MOVD 264(R1), R27
1366 MOVD 272(R1), R28
1367 MOVD 280(R1), R29
1368 MOVD 288(R1), g
1369 MOVD 16(R1), R31
1370 // restore old LR
1371 MOVD R31, LR
1372 // restore caller PC
1373 MOVD 0(R1), CTR
1374 MOVD 136(R1), R31
1375 // Add 32 bytes more to compensate for SP change in saveSigContext
1376 ADD $352, R1
1377 JMP (CTR)
1378 #endif
1379 #define DEBUG_CALL_FN(NAME,MAXSIZE) \
1380 TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1381 NO_LOCAL_POINTERS; \
1382 MOVD $0, R20; \
1383 TW $31, R0, R0 \
1384 MOVD $1, R20; \
1385 TW $31, R0, R0 \
1386 RET
1387 DEBUG_CALL_FN(debugCall32<>, 32)
1388 DEBUG_CALL_FN(debugCall64<>, 64)
1389 DEBUG_CALL_FN(debugCall128<>, 128)
1390 DEBUG_CALL_FN(debugCall256<>, 256)
1391 DEBUG_CALL_FN(debugCall512<>, 512)
1392 DEBUG_CALL_FN(debugCall1024<>, 1024)
1393 DEBUG_CALL_FN(debugCall2048<>, 2048)
1394 DEBUG_CALL_FN(debugCall4096<>, 4096)
1395 DEBUG_CALL_FN(debugCall8192<>, 8192)
1396 DEBUG_CALL_FN(debugCall16384<>, 16384)
1397 DEBUG_CALL_FN(debugCall32768<>, 32768)
1398 DEBUG_CALL_FN(debugCall65536<>, 65536)
1399
1400 #ifdef GOARCH_ppc64le
1401 // func debugCallPanicked(val interface{})
1402 TEXT runtime·debugCallPanicked(SB),NOSPLIT,$32-16
1403 // Copy the panic value to the top of stack at SP+32.
1404 MOVD val_type+0(FP), R31
1405 MOVD R31, 32(R1)
1406 MOVD val_data+8(FP), R31
1407 MOVD R31, 40(R1)
1408 MOVD $2, R20
1409 TW $31, R0, R0
1410 RET
1411 #endif
1412
1413 TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$88-0
1414 // Note: frame size is 16 bytes larger than necessary
1415 // in order to pacify vet. Vet doesn't understand ppc64
1416 // layout properly.
1417 NO_LOCAL_POINTERS
1418 // Save all 7 int registers that could have an index in them.
1419 // They may be pointers, but if so they are dead.
1420 // Skip R0 aka ZERO, R1 aka SP, R2 aka SB
1421 MOVD R3, 48(R1)
1422 MOVD R4, 56(R1)
1423 MOVD R5, 64(R1)
1424 MOVD R6, 72(R1)
1425 MOVD R7, 80(R1)
1426 MOVD R8, 88(R1)
1427 MOVD R9, 96(R1)
1428 // Note: we only save 7 registers to keep under nosplit stack limit
1429 // Also, R11 is clobbered in dynamic linking situations
1430
1431 MOVD LR, R3 // PC immediately after call to panicBounds
1432 ADD $48, R1, R4 // pointer to save area
1433 CALL runtime·panicBounds64<ABIInternal>(SB)
1434 RET
1435
1436 // These functions are used when internal linking cgo with external
1437 // objects compiled with the -Os on gcc. They reduce prologue/epilogue
1438 // size by deferring preservation of callee-save registers to a shared
1439 // function. These are defined in PPC64 ELFv2 2.3.3 (but also present
1440 // in ELFv1)
1441 //
1442 // These appear unused, but the linker will redirect calls to functions
1443 // like _savegpr0_14 or _restgpr1_14 to runtime.elf_savegpr0 or
1444 // runtime.elf_restgpr1 with an appropriate offset based on the number
1445 // register operations required when linking external objects which
1446 // make these calls. For GPR/FPR saves, the minimum register value is
1447 // 14, for VR it is 20.
1448 //
1449 // These are only used when linking such cgo code internally. Note, R12
1450 // and R0 may be used in different ways than regular ELF compliant
1451 // functions.
1452 TEXT runtime·elf_savegpr0(SB),NOSPLIT|NOFRAME,$0
1453 // R0 holds the LR of the caller's caller, R1 holds save location
1454 MOVD R14, -144(R1)
1455 MOVD R15, -136(R1)
1456 MOVD R16, -128(R1)
1457 MOVD R17, -120(R1)
1458 MOVD R18, -112(R1)
1459 MOVD R19, -104(R1)
1460 MOVD R20, -96(R1)
1461 MOVD R21, -88(R1)
1462 MOVD R22, -80(R1)
1463 MOVD R23, -72(R1)
1464 MOVD R24, -64(R1)
1465 MOVD R25, -56(R1)
1466 MOVD R26, -48(R1)
1467 MOVD R27, -40(R1)
1468 MOVD R28, -32(R1)
1469 MOVD R29, -24(R1)
1470 MOVD g, -16(R1)
1471 MOVD R31, -8(R1)
1472 MOVD R0, 16(R1)
1473 RET
1474 TEXT runtime·elf_restgpr0(SB),NOSPLIT|NOFRAME,$0
1475 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1476 MOVD -144(R1), R14
1477 MOVD -136(R1), R15
1478 MOVD -128(R1), R16
1479 MOVD -120(R1), R17
1480 MOVD -112(R1), R18
1481 MOVD -104(R1), R19
1482 MOVD -96(R1), R20
1483 MOVD -88(R1), R21
1484 MOVD -80(R1), R22
1485 MOVD -72(R1), R23
1486 MOVD -64(R1), R24
1487 MOVD -56(R1), R25
1488 MOVD -48(R1), R26
1489 MOVD -40(R1), R27
1490 MOVD -32(R1), R28
1491 MOVD -24(R1), R29
1492 MOVD -16(R1), g
1493 MOVD -8(R1), R31
1494 MOVD 16(R1), R0 // Load and return to saved LR
1495 MOVD R0, LR
1496 RET
1497 TEXT runtime·elf_savegpr1(SB),NOSPLIT|NOFRAME,$0
1498 // R12 holds the save location
1499 MOVD R14, -144(R12)
1500 MOVD R15, -136(R12)
1501 MOVD R16, -128(R12)
1502 MOVD R17, -120(R12)
1503 MOVD R18, -112(R12)
1504 MOVD R19, -104(R12)
1505 MOVD R20, -96(R12)
1506 MOVD R21, -88(R12)
1507 MOVD R22, -80(R12)
1508 MOVD R23, -72(R12)
1509 MOVD R24, -64(R12)
1510 MOVD R25, -56(R12)
1511 MOVD R26, -48(R12)
1512 MOVD R27, -40(R12)
1513 MOVD R28, -32(R12)
1514 MOVD R29, -24(R12)
1515 MOVD g, -16(R12)
1516 MOVD R31, -8(R12)
1517 RET
1518 TEXT runtime·elf_restgpr1(SB),NOSPLIT|NOFRAME,$0
1519 // R12 holds the save location
1520 MOVD -144(R12), R14
1521 MOVD -136(R12), R15
1522 MOVD -128(R12), R16
1523 MOVD -120(R12), R17
1524 MOVD -112(R12), R18
1525 MOVD -104(R12), R19
1526 MOVD -96(R12), R20
1527 MOVD -88(R12), R21
1528 MOVD -80(R12), R22
1529 MOVD -72(R12), R23
1530 MOVD -64(R12), R24
1531 MOVD -56(R12), R25
1532 MOVD -48(R12), R26
1533 MOVD -40(R12), R27
1534 MOVD -32(R12), R28
1535 MOVD -24(R12), R29
1536 MOVD -16(R12), g
1537 MOVD -8(R12), R31
1538 RET
1539 TEXT runtime·elf_savefpr(SB),NOSPLIT|NOFRAME,$0
1540 // R0 holds the LR of the caller's caller, R1 holds save location
1541 FMOVD F14, -144(R1)
1542 FMOVD F15, -136(R1)
1543 FMOVD F16, -128(R1)
1544 FMOVD F17, -120(R1)
1545 FMOVD F18, -112(R1)
1546 FMOVD F19, -104(R1)
1547 FMOVD F20, -96(R1)
1548 FMOVD F21, -88(R1)
1549 FMOVD F22, -80(R1)
1550 FMOVD F23, -72(R1)
1551 FMOVD F24, -64(R1)
1552 FMOVD F25, -56(R1)
1553 FMOVD F26, -48(R1)
1554 FMOVD F27, -40(R1)
1555 FMOVD F28, -32(R1)
1556 FMOVD F29, -24(R1)
1557 FMOVD F30, -16(R1)
1558 FMOVD F31, -8(R1)
1559 MOVD R0, 16(R1)
1560 RET
1561 TEXT runtime·elf_restfpr(SB),NOSPLIT|NOFRAME,$0
1562 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1563 FMOVD -144(R1), F14
1564 FMOVD -136(R1), F15
1565 FMOVD -128(R1), F16
1566 FMOVD -120(R1), F17
1567 FMOVD -112(R1), F18
1568 FMOVD -104(R1), F19
1569 FMOVD -96(R1), F20
1570 FMOVD -88(R1), F21
1571 FMOVD -80(R1), F22
1572 FMOVD -72(R1), F23
1573 FMOVD -64(R1), F24
1574 FMOVD -56(R1), F25
1575 FMOVD -48(R1), F26
1576 FMOVD -40(R1), F27
1577 FMOVD -32(R1), F28
1578 FMOVD -24(R1), F29
1579 FMOVD -16(R1), F30
1580 FMOVD -8(R1), F31
1581 MOVD 16(R1), R0 // Load and return to saved LR
1582 MOVD R0, LR
1583 RET
1584 TEXT runtime·elf_savevr(SB),NOSPLIT|NOFRAME,$0
1585 // R0 holds the save location, R12 is clobbered
1586 MOVD $-192, R12
1587 STVX V20, (R0+R12)
1588 MOVD $-176, R12
1589 STVX V21, (R0+R12)
1590 MOVD $-160, R12
1591 STVX V22, (R0+R12)
1592 MOVD $-144, R12
1593 STVX V23, (R0+R12)
1594 MOVD $-128, R12
1595 STVX V24, (R0+R12)
1596 MOVD $-112, R12
1597 STVX V25, (R0+R12)
1598 MOVD $-96, R12
1599 STVX V26, (R0+R12)
1600 MOVD $-80, R12
1601 STVX V27, (R0+R12)
1602 MOVD $-64, R12
1603 STVX V28, (R0+R12)
1604 MOVD $-48, R12
1605 STVX V29, (R0+R12)
1606 MOVD $-32, R12
1607 STVX V30, (R0+R12)
1608 MOVD $-16, R12
1609 STVX V31, (R0+R12)
1610 RET
1611 TEXT runtime·elf_restvr(SB),NOSPLIT|NOFRAME,$0
1612 // R0 holds the save location, R12 is clobbered
1613 MOVD $-192, R12
1614 LVX (R0+R12), V20
1615 MOVD $-176, R12
1616 LVX (R0+R12), V21
1617 MOVD $-160, R12
1618 LVX (R0+R12), V22
1619 MOVD $-144, R12
1620 LVX (R0+R12), V23
1621 MOVD $-128, R12
1622 LVX (R0+R12), V24
1623 MOVD $-112, R12
1624 LVX (R0+R12), V25
1625 MOVD $-96, R12
1626 LVX (R0+R12), V26
1627 MOVD $-80, R12
1628 LVX (R0+R12), V27
1629 MOVD $-64, R12
1630 LVX (R0+R12), V28
1631 MOVD $-48, R12
1632 LVX (R0+R12), V29
1633 MOVD $-32, R12
1634 LVX (R0+R12), V30
1635 MOVD $-16, R12
1636 LVX (R0+R12), V31
1637 RET
1638
View as plain text