Text file
src/runtime/asm_ppc64x.s
1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build ppc64 || ppc64le
6
7 #include "go_asm.h"
8 #include "go_tls.h"
9 #include "funcdata.h"
10 #include "textflag.h"
11 #include "asm_ppc64x.h"
12 #include "cgo/abi_ppc64x.h"
13
14 // This is called using the host ABI. argc and argv arguments
15 // should be in R3 and R4 respectively.
16 TEXT _rt0_ppc64x_lib(SB),NOSPLIT|NOFRAME,$0
17 // Start with standard C stack frame layout and linkage, allocate
18 // 16 bytes of argument space, save callee-save regs, and set R0 to $0.
19 // Allocate an extra 16 bytes to account for the larger fixed frame size
20 // of aix/elfv1 (48 vs 32) to ensure 16 bytes of parameter save space.
21 STACK_AND_SAVE_HOST_TO_GO_ABI(32)
22 // The above will not preserve R2 (TOC). Save it in case Go is
23 // compiled without a TOC pointer (e.g -buildmode=default).
24 MOVD R2, 24(R1)
25
26 MOVD R3, _rt0_ppc64x_lib_argc<>(SB)
27 MOVD R4, _rt0_ppc64x_lib_argv<>(SB)
28
29 // Synchronous initialization.
30 MOVD $runtime·reginit(SB), R12
31 MOVD R12, CTR
32 BL (CTR)
33 MOVD $runtime·libpreinit(SB), R12
34 MOVD R12, CTR
35 BL (CTR)
36
37 #ifdef GOOS_aix
38 // See runtime/cgo/gcc_aix_ppc64.c
39 MOVBZ runtime·isarchive(SB), R3 // Check buildmode = c-archive
40 CMP $0, R3
41 BEQ done
42 #endif
43
44 // Create a new thread to do the runtime initialization and return.
45 // _cgo_sys_thread_create is a C function.
46 MOVD _cgo_sys_thread_create(SB), R12
47 CMP $0, R12
48 BEQ nocgo
49 MOVD $_rt0_ppc64x_lib_go(SB), R3
50 MOVD $0, R4
51 #ifdef GO_PPC64X_HAS_FUNCDESC
52 // Load the real entry address from the first slot of the function descriptor.
53 MOVD 8(R12), R2
54 MOVD (R12), R12
55 #endif
56 MOVD R12, CTR
57 BL (CTR)
58 MOVD 24(R1), R2 // Restore the old frame, and R2.
59 BR done
60
61 nocgo:
62 MOVD $0x800000, R12 // stacksize = 8192KB
63 MOVD R12, 8+FIXED_FRAME(R1)
64 MOVD $_rt0_ppc64x_lib_go(SB), R12
65 MOVD R12, 16+FIXED_FRAME(R1)
66 MOVD $runtime·newosproc0(SB),R12
67 MOVD R12, CTR
68 BL (CTR)
69
70 done:
71 UNSTACK_AND_RESTORE_GO_TO_HOST_ABI(32)
72 RET
73
74 #ifdef GO_PPC64X_HAS_FUNCDESC
75 DEFINE_PPC64X_FUNCDESC(_rt0_ppc64x_lib_go, __rt0_ppc64x_lib_go)
76 TEXT __rt0_ppc64x_lib_go(SB),NOSPLIT,$0
77 #else
78 TEXT _rt0_ppc64x_lib_go(SB),NOSPLIT,$0
79 #endif
80 MOVD _rt0_ppc64x_lib_argc<>(SB), R3
81 MOVD _rt0_ppc64x_lib_argv<>(SB), R4
82 MOVD $runtime·rt0_go(SB), R12
83 MOVD R12, CTR
84 BR (CTR)
85
86 DATA _rt0_ppc64x_lib_argc<>(SB)/8, $0
87 GLOBL _rt0_ppc64x_lib_argc<>(SB),NOPTR, $8
88 DATA _rt0_ppc64x_lib_argv<>(SB)/8, $0
89 GLOBL _rt0_ppc64x_lib_argv<>(SB),NOPTR, $8
90
91
92 #ifdef GOOS_aix
93 #define cgoCalleeStackSize 48
94 #else
95 #define cgoCalleeStackSize 32
96 #endif
97
98 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
99 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer
100
101 // initialize essential registers
102 BL runtime·reginit(SB)
103
104 SUB $(FIXED_FRAME+16), R1
105 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame
106 MOVW R3, FIXED_FRAME+0(R1) // argc
107 MOVD R4, FIXED_FRAME+8(R1) // argv
108
109 // create istack out of the given (operating system) stack.
110 // _cgo_init may update stackguard.
111 MOVD $runtime·g0(SB), g
112 BL runtime·save_g(SB)
113 MOVD $(-64*1024), R31
114 ADD R31, R1, R3
115 MOVD R3, g_stackguard0(g)
116 MOVD R3, g_stackguard1(g)
117 MOVD R3, (g_stack+stack_lo)(g)
118 MOVD R1, (g_stack+stack_hi)(g)
119
120 // If there is a _cgo_init, call it using the gcc ABI.
121 MOVD _cgo_init(SB), R12
122 CMP R12, $0
123 BEQ nocgo
124
125 #ifdef GO_PPC64X_HAS_FUNCDESC
126 // Load the real entry address from the first slot of the function descriptor.
127 MOVD 8(R12), R2
128 MOVD (R12), R12
129 #endif
130 MOVD R12, CTR // r12 = "global function entry point"
131 MOVD R13, R5 // arg 2: TLS base pointer
132 MOVD $setg_gcc<>(SB), R4 // arg 1: setg
133 MOVD g, R3 // arg 0: G
134 // C functions expect 32 (48 for AIX) bytes of space on caller
135 // stack frame and a 16-byte aligned R1
136 MOVD R1, R14 // save current stack
137 SUB $cgoCalleeStackSize, R1 // reserve the callee area
138 RLDCR $0, R1, $~15, R1 // 16-byte align
139 BL (CTR) // may clobber R0, R3-R12
140 MOVD R14, R1 // restore stack
141 #ifndef GOOS_aix
142 MOVD 24(R1), R2
143 #endif
144 XOR R0, R0 // fix R0
145
146 nocgo:
147 // update stackguard after _cgo_init
148 MOVD (g_stack+stack_lo)(g), R3
149 ADD $const_stackGuard, R3
150 MOVD R3, g_stackguard0(g)
151 MOVD R3, g_stackguard1(g)
152
153 // set the per-goroutine and per-mach "registers"
154 MOVD $runtime·m0(SB), R3
155
156 // save m->g0 = g0
157 MOVD g, m_g0(R3)
158 // save m0 to g0->m
159 MOVD R3, g_m(g)
160
161 BL runtime·check(SB)
162
163 // args are already prepared
164 BL runtime·args(SB)
165 BL runtime·osinit(SB)
166 BL runtime·schedinit(SB)
167
168 // create a new goroutine to start program
169 MOVD $runtime·mainPC(SB), R3 // entry
170 MOVDU R3, -8(R1)
171 MOVDU R0, -8(R1)
172 MOVDU R0, -8(R1)
173 MOVDU R0, -8(R1)
174 MOVDU R0, -8(R1)
175 BL runtime·newproc(SB)
176 ADD $(8+FIXED_FRAME), R1
177
178 // start this M
179 BL runtime·mstart(SB)
180 // Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are
181 // intended to be called by debuggers.
182 #ifdef GOARCH_ppc64le
183 MOVD $runtime·debugPinnerV1<ABIInternal>(SB), R31
184 MOVD $runtime·debugCallV2<ABIInternal>(SB), R31
185 #endif
186 MOVD R0, 0(R0)
187 RET
188
189 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
190 GLOBL runtime·mainPC(SB),RODATA,$8
191
192 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
193 TW $31, R0, R0
194 RET
195
196 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
197 RET
198
199 // Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall_ppc64
200 TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0
201 // crosscall_ppc64 and crosscall2 need to reginit, but can't
202 // get at the 'runtime.reginit' symbol.
203 BR runtime·reginit(SB)
204
205 TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0
206 // set R0 to zero, it's expected by the toolchain
207 XOR R0, R0
208 RET
209
210 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
211 BL runtime·mstart0(SB)
212 RET // not reached
213
214 /*
215 * go-routine
216 */
217
218 // void gogo(Gobuf*)
219 // restore state from Gobuf; longjmp
220 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
221 MOVD buf+0(FP), R5
222 MOVD gobuf_g(R5), R6
223 MOVD 0(R6), R4 // make sure g != nil
224 BR gogo<>(SB)
225
226 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
227 MOVD R6, g
228 BL runtime·save_g(SB)
229
230 MOVD gobuf_sp(R5), R1
231 MOVD gobuf_lr(R5), R31
232 #ifndef GOOS_aix
233 MOVD 24(R1), R2 // restore R2
234 #endif
235 MOVD R31, LR
236 MOVD gobuf_ctxt(R5), R11
237 MOVD R0, gobuf_sp(R5)
238 MOVD R0, gobuf_lr(R5)
239 MOVD R0, gobuf_ctxt(R5)
240 CMP R0, R0 // set condition codes for == test, needed by stack split
241 MOVD gobuf_pc(R5), R12
242 MOVD R12, CTR
243 BR (CTR)
244
245 // void mcall(fn func(*g))
246 // Switch to m->g0's stack, call fn(g).
247 // Fn must never return. It should gogo(&g->sched)
248 // to keep running g.
249 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
250 // Save caller state in g->sched
251 // R11 should be safe across save_g??
252 MOVD R3, R11
253 MOVD R1, (g_sched+gobuf_sp)(g)
254 MOVD LR, R31
255 MOVD R31, (g_sched+gobuf_pc)(g)
256 MOVD R0, (g_sched+gobuf_lr)(g)
257
258 // Switch to m->g0 & its stack, call fn.
259 MOVD g, R3
260 MOVD g_m(g), R8
261 MOVD m_g0(R8), g
262 BL runtime·save_g(SB)
263 CMP g, R3
264 BNE 2(PC)
265 BR runtime·badmcall(SB)
266 MOVD 0(R11), R12 // code pointer
267 MOVD R12, CTR
268 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp
269 // Don't need to do anything special for regabiargs here
270 // R3 is g; stack is set anyway
271 MOVDU R3, -8(R1)
272 MOVDU R0, -8(R1)
273 MOVDU R0, -8(R1)
274 MOVDU R0, -8(R1)
275 MOVDU R0, -8(R1)
276 BL (CTR)
277 MOVD 24(R1), R2
278 BR runtime·badmcall2(SB)
279
280 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
281 // of the G stack. We need to distinguish the routine that
282 // lives at the bottom of the G stack from the one that lives
283 // at the top of the system stack because the one at the top of
284 // the system stack terminates the stack walk (see topofstack()).
285 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
286 // We have several undefs here so that 16 bytes past
287 // $runtime·systemstack_switch lies within them whether or not the
288 // instructions that derive r2 from r12 are there.
289 UNDEF
290 UNDEF
291 UNDEF
292 BL (LR) // make sure this function is not leaf
293 RET
294
295 // func systemstack(fn func())
296 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
297 MOVD fn+0(FP), R3 // R3 = fn
298 MOVD R3, R11 // context
299 MOVD g_m(g), R4 // R4 = m
300
301 MOVD m_gsignal(R4), R5 // R5 = gsignal
302 CMP g, R5
303 BEQ noswitch
304
305 MOVD m_g0(R4), R5 // R5 = g0
306 CMP g, R5
307 BEQ noswitch
308
309 MOVD m_curg(R4), R6
310 CMP g, R6
311 BEQ switch
312
313 // Bad: g is not gsignal, not g0, not curg. What is it?
314 // Hide call from linker nosplit analysis.
315 MOVD $runtime·badsystemstack(SB), R12
316 MOVD R12, CTR
317 BL (CTR)
318 BL runtime·abort(SB)
319
320 switch:
321 // save our state in g->sched. Pretend to
322 // be systemstack_switch if the G stack is scanned.
323 BL gosave_systemstack_switch<>(SB)
324
325 // switch to g0
326 MOVD R5, g
327 BL runtime·save_g(SB)
328 MOVD (g_sched+gobuf_sp)(g), R1
329
330 // call target function
331 MOVD 0(R11), R12 // code pointer
332 MOVD R12, CTR
333 BL (CTR)
334
335 // restore TOC pointer. It seems unlikely that we will use systemstack
336 // to call a function defined in another module, but the results of
337 // doing so would be so confusing that it's worth doing this.
338 MOVD g_m(g), R3
339 MOVD m_curg(R3), g
340 MOVD (g_sched+gobuf_sp)(g), R3
341 #ifndef GOOS_aix
342 MOVD 24(R3), R2
343 #endif
344 // switch back to g
345 MOVD g_m(g), R3
346 MOVD m_curg(R3), g
347 BL runtime·save_g(SB)
348 MOVD (g_sched+gobuf_sp)(g), R1
349 MOVD R0, (g_sched+gobuf_sp)(g)
350 RET
351
352 noswitch:
353 // already on m stack, just call directly
354 // On other arches we do a tail call here, but it appears to be
355 // impossible to tail call a function pointer in shared mode on
356 // ppc64 because the caller is responsible for restoring the TOC.
357 MOVD 0(R11), R12 // code pointer
358 MOVD R12, CTR
359 BL (CTR)
360 #ifndef GOOS_aix
361 MOVD 24(R1), R2
362 #endif
363 RET
364
365 // func switchToCrashStack0(fn func())
366 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
367 MOVD R3, R11 // context register
368 MOVD g_m(g), R3 // curm
369
370 // set g to gcrash
371 MOVD $runtime·gcrash(SB), g // g = &gcrash
372 CALL runtime·save_g(SB) // clobbers R31
373 MOVD R3, g_m(g) // g.m = curm
374 MOVD g, m_g0(R3) // curm.g0 = g
375
376 // switch to crashstack
377 MOVD (g_stack+stack_hi)(g), R3
378 SUB $(4*8), R3
379 MOVD R3, R1
380
381 // call target function
382 MOVD 0(R11), R12 // code pointer
383 MOVD R12, CTR
384 BL (CTR)
385
386 // should never return
387 CALL runtime·abort(SB)
388 UNDEF
389
390 /*
391 * support for morestack
392 */
393
394 // Called during function prolog when more stack is needed.
395 // Caller has already loaded:
396 // R3: framesize, R4: argsize, R5: LR
397 //
398 // The traceback routines see morestack on a g0 as being
399 // the top of a stack (for example, morestack calling newstack
400 // calling the scheduler calling newm calling gc), so we must
401 // record an argument size. For that purpose, it has no arguments.
402 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
403 // Called from f.
404 // Set g->sched to context in f.
405 MOVD R1, (g_sched+gobuf_sp)(g)
406 MOVD LR, R8
407 MOVD R8, (g_sched+gobuf_pc)(g)
408 MOVD R5, (g_sched+gobuf_lr)(g)
409 MOVD R11, (g_sched+gobuf_ctxt)(g)
410
411 // Cannot grow scheduler stack (m->g0).
412 MOVD g_m(g), R7
413 MOVD m_g0(R7), R8
414 CMP g, R8
415 BNE 3(PC)
416 BL runtime·badmorestackg0(SB)
417 BL runtime·abort(SB)
418
419 // Cannot grow signal stack (m->gsignal).
420 MOVD m_gsignal(R7), R8
421 CMP g, R8
422 BNE 3(PC)
423 BL runtime·badmorestackgsignal(SB)
424 BL runtime·abort(SB)
425
426 // Called from f.
427 // Set m->morebuf to f's caller.
428 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
429 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
430 MOVD g, (m_morebuf+gobuf_g)(R7)
431
432 // Call newstack on m->g0's stack.
433 MOVD m_g0(R7), g
434 BL runtime·save_g(SB)
435 MOVD (g_sched+gobuf_sp)(g), R1
436 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0
437 BL runtime·newstack(SB)
438
439 // Not reached, but make sure the return PC from the call to newstack
440 // is still in this function, and not the beginning of the next.
441 UNDEF
442
443 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
444 // Force SPWRITE. This function doesn't actually write SP,
445 // but it is called with a special calling convention where
446 // the caller doesn't save LR on stack but passes it as a
447 // register (R5), and the unwinder currently doesn't understand.
448 // Make it SPWRITE to stop unwinding. (See issue 54332)
449 // Use OR R0, R1 instead of MOVD R1, R1 as the MOVD instruction
450 // has a special affect on Power8,9,10 by lowering the thread
451 // priority and causing a slowdown in execution time
452
453 OR R0, R1
454 MOVD R0, R11
455 BR runtime·morestack(SB)
456
457 // reflectcall: call a function with the given argument list
458 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
459 // we don't have variable-sized frames, so we use a small number
460 // of constant-sized-frame functions to encode a few bits of size in the pc.
461 // Caution: ugly multiline assembly macros in your future!
462
463 #define DISPATCH(NAME,MAXSIZE) \
464 MOVD $MAXSIZE, R31; \
465 CMP R3, R31; \
466 BGT 4(PC); \
467 MOVD $NAME(SB), R12; \
468 MOVD R12, CTR; \
469 BR (CTR)
470 // Note: can't just "BR NAME(SB)" - bad inlining results.
471
472 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
473 MOVWZ frameSize+32(FP), R3
474 DISPATCH(runtime·call16, 16)
475 DISPATCH(runtime·call32, 32)
476 DISPATCH(runtime·call64, 64)
477 DISPATCH(runtime·call128, 128)
478 DISPATCH(runtime·call256, 256)
479 DISPATCH(runtime·call512, 512)
480 DISPATCH(runtime·call1024, 1024)
481 DISPATCH(runtime·call2048, 2048)
482 DISPATCH(runtime·call4096, 4096)
483 DISPATCH(runtime·call8192, 8192)
484 DISPATCH(runtime·call16384, 16384)
485 DISPATCH(runtime·call32768, 32768)
486 DISPATCH(runtime·call65536, 65536)
487 DISPATCH(runtime·call131072, 131072)
488 DISPATCH(runtime·call262144, 262144)
489 DISPATCH(runtime·call524288, 524288)
490 DISPATCH(runtime·call1048576, 1048576)
491 DISPATCH(runtime·call2097152, 2097152)
492 DISPATCH(runtime·call4194304, 4194304)
493 DISPATCH(runtime·call8388608, 8388608)
494 DISPATCH(runtime·call16777216, 16777216)
495 DISPATCH(runtime·call33554432, 33554432)
496 DISPATCH(runtime·call67108864, 67108864)
497 DISPATCH(runtime·call134217728, 134217728)
498 DISPATCH(runtime·call268435456, 268435456)
499 DISPATCH(runtime·call536870912, 536870912)
500 DISPATCH(runtime·call1073741824, 1073741824)
501 MOVD $runtime·badreflectcall(SB), R12
502 MOVD R12, CTR
503 BR (CTR)
504
505 #define CALLFN(NAME,MAXSIZE) \
506 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
507 NO_LOCAL_POINTERS; \
508 /* copy arguments to stack */ \
509 MOVD stackArgs+16(FP), R3; \
510 MOVWZ stackArgsSize+24(FP), R4; \
511 MOVD R1, R5; \
512 CMP R4, $8; \
513 BLT tailsetup; \
514 /* copy 8 at a time if possible */ \
515 ADD $(FIXED_FRAME-8), R5; \
516 SUB $8, R3; \
517 top: \
518 MOVDU 8(R3), R7; \
519 MOVDU R7, 8(R5); \
520 SUB $8, R4; \
521 CMP R4, $8; \
522 BGE top; \
523 /* handle remaining bytes */ \
524 CMP $0, R4; \
525 BEQ callfn; \
526 ADD $7, R3; \
527 ADD $7, R5; \
528 BR tail; \
529 tailsetup: \
530 CMP $0, R4; \
531 BEQ callfn; \
532 ADD $(FIXED_FRAME-1), R5; \
533 SUB $1, R3; \
534 tail: \
535 MOVBU 1(R3), R6; \
536 MOVBU R6, 1(R5); \
537 SUB $1, R4; \
538 CMP $0, R4; \
539 BGT tail; \
540 callfn: \
541 /* call function */ \
542 MOVD f+8(FP), R11; \
543 #ifdef GOOS_aix \
544 /* AIX won't trigger a SIGSEGV if R11 = nil */ \
545 /* So it manually triggers it */ \
546 CMP R11, $0 \
547 BNE 2(PC) \
548 MOVD R0, 0(R0) \
549 #endif \
550 MOVD regArgs+40(FP), R20; \
551 BL runtime·unspillArgs(SB); \
552 MOVD (R11), R12; \
553 MOVD R12, CTR; \
554 PCDATA $PCDATA_StackMapIndex, $0; \
555 BL (CTR); \
556 #ifndef GOOS_aix \
557 MOVD 24(R1), R2; \
558 #endif \
559 /* copy return values back */ \
560 MOVD regArgs+40(FP), R20; \
561 BL runtime·spillArgs(SB); \
562 MOVD stackArgsType+0(FP), R7; \
563 MOVD stackArgs+16(FP), R3; \
564 MOVWZ stackArgsSize+24(FP), R4; \
565 MOVWZ stackRetOffset+28(FP), R6; \
566 ADD $FIXED_FRAME, R1, R5; \
567 ADD R6, R5; \
568 ADD R6, R3; \
569 SUB R6, R4; \
570 BL callRet<>(SB); \
571 RET
572
573 // callRet copies return values back at the end of call*. This is a
574 // separate function so it can allocate stack space for the arguments
575 // to reflectcallmove. It does not follow the Go ABI; it expects its
576 // arguments in registers.
577 TEXT callRet<>(SB), NOSPLIT, $40-0
578 NO_LOCAL_POINTERS
579 MOVD R7, FIXED_FRAME+0(R1)
580 MOVD R3, FIXED_FRAME+8(R1)
581 MOVD R5, FIXED_FRAME+16(R1)
582 MOVD R4, FIXED_FRAME+24(R1)
583 MOVD R20, FIXED_FRAME+32(R1)
584 BL runtime·reflectcallmove(SB)
585 RET
586
587 CALLFN(·call16, 16)
588 CALLFN(·call32, 32)
589 CALLFN(·call64, 64)
590 CALLFN(·call128, 128)
591 CALLFN(·call256, 256)
592 CALLFN(·call512, 512)
593 CALLFN(·call1024, 1024)
594 CALLFN(·call2048, 2048)
595 CALLFN(·call4096, 4096)
596 CALLFN(·call8192, 8192)
597 CALLFN(·call16384, 16384)
598 CALLFN(·call32768, 32768)
599 CALLFN(·call65536, 65536)
600 CALLFN(·call131072, 131072)
601 CALLFN(·call262144, 262144)
602 CALLFN(·call524288, 524288)
603 CALLFN(·call1048576, 1048576)
604 CALLFN(·call2097152, 2097152)
605 CALLFN(·call4194304, 4194304)
606 CALLFN(·call8388608, 8388608)
607 CALLFN(·call16777216, 16777216)
608 CALLFN(·call33554432, 33554432)
609 CALLFN(·call67108864, 67108864)
610 CALLFN(·call134217728, 134217728)
611 CALLFN(·call268435456, 268435456)
612 CALLFN(·call536870912, 536870912)
613 CALLFN(·call1073741824, 1073741824)
614
615 TEXT runtime·procyieldAsm(SB),NOSPLIT|NOFRAME,$0-4
616 MOVW cycles+0(FP), R7
617 CMP $0, R7
618 BEQ done
619 // POWER does not have a pause/yield instruction equivalent.
620 // Instead, we can lower the program priority by setting the
621 // Program Priority Register prior to the wait loop and set it
622 // back to default afterwards. On Linux, the default priority is
623 // medium-low. For details, see page 837 of the ISA 3.0.
624 OR R1, R1, R1 // Set PPR priority to low
625 again:
626 SUB $1, R7
627 CMP $0, R7
628 BNE again
629 OR R6, R6, R6 // Set PPR priority back to medium-low
630 done:
631 RET
632
633 // Save state of caller into g->sched,
634 // but using fake PC from systemstack_switch.
635 // Must only be called from functions with no locals ($0)
636 // or else unwinding from systemstack_switch is incorrect.
637 // Smashes R31.
638 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
639 MOVD $runtime·systemstack_switch(SB), R31
640 ADD $16, R31 // get past prologue (including r2-setting instructions when they're there)
641 MOVD R31, (g_sched+gobuf_pc)(g)
642 MOVD R1, (g_sched+gobuf_sp)(g)
643 MOVD R0, (g_sched+gobuf_lr)(g)
644 // Assert ctxt is zero. See func save.
645 MOVD (g_sched+gobuf_ctxt)(g), R31
646 CMP R31, $0
647 BEQ 2(PC)
648 BL runtime·abort(SB)
649 RET
650
651 #ifdef GOOS_aix
652 #define asmcgocallSaveOffset cgoCalleeStackSize + 8
653 #else
654 #define asmcgocallSaveOffset cgoCalleeStackSize
655 #endif
656
657 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
658 // Call fn(arg) aligned appropriately for the gcc ABI.
659 // Called on a system stack, and there may be no g yet (during needm).
660 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
661 MOVD fn+0(FP), R3
662 MOVD arg+8(FP), R4
663
664 MOVD R1, R15
665 SUB $(asmcgocallSaveOffset+8), R1
666 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
667 MOVD R15, asmcgocallSaveOffset(R1)
668
669 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
670
671 // This is a "global call", so put the global entry point in r12
672 MOVD R3, R12
673
674 #ifdef GO_PPC64X_HAS_FUNCDESC
675 // Load the real entry address from the first slot of the function descriptor.
676 MOVD 8(R12), R2
677 MOVD (R12), R12
678 #endif
679 MOVD R12, CTR
680 MOVD R4, R3 // arg in r3
681 BL (CTR)
682
683 // C code can clobber R0, so set it back to 0. F27-F31 are
684 // callee save, so we don't need to recover those.
685 XOR R0, R0
686
687 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
688 #ifndef GOOS_aix
689 MOVD 24(R1), R2
690 #endif
691
692 RET
693
694 // func asmcgocall(fn, arg unsafe.Pointer) int32
695 // Call fn(arg) on the scheduler stack,
696 // aligned appropriately for the gcc ABI.
697 // See cgocall.go for more details.
698 TEXT ·asmcgocall<ABIInternal>(SB),NOSPLIT,$0-20
699 // R3 = fn
700 // R4 = arg
701
702 MOVD R1, R7 // save original stack pointer
703 CMP $0, g
704 BEQ nosave
705 MOVD g, R5
706
707 // Figure out if we need to switch to m->g0 stack.
708 // We get called to create new OS threads too, and those
709 // come in on the m->g0 stack already. Or we might already
710 // be on the m->gsignal stack.
711 MOVD g_m(g), R8
712 MOVD m_gsignal(R8), R6
713 CMP R6, g
714 BEQ nosave
715 MOVD m_g0(R8), R6
716 CMP R6, g
717 BEQ nosave
718
719 BL gosave_systemstack_switch<>(SB)
720 MOVD R6, g
721 BL runtime·save_g(SB)
722 MOVD (g_sched+gobuf_sp)(g), R1
723
724 // Now on a scheduling stack (a pthread-created stack).
725 #ifdef GOOS_aix
726 // Create a fake LR to improve backtrace.
727 MOVD $runtime·asmcgocall(SB), R6
728 MOVD R6, 16(R1)
729 // AIX also saves one argument on the stack.
730 SUB $8, R1
731 #endif
732 // Save room for two of our pointers, plus the callee
733 // save area that lives on the caller stack.
734 // Do arithmetics in R10 to hide from the assembler
735 // counting it as SP delta, which is irrelevant as we are
736 // on the system stack.
737 SUB $(asmcgocallSaveOffset+16), R1, R10
738 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
739 MOVD R5, (asmcgocallSaveOffset+8)(R1) // save old g on stack
740 MOVD (g_stack+stack_hi)(R5), R5
741 SUB R7, R5
742 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
743 #ifdef GOOS_aix
744 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb
745 #else
746 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
747 #endif
748 // This is a "global call", so put the global entry point in r12
749 MOVD R3, R12
750
751 #ifdef GO_PPC64X_HAS_FUNCDESC
752 // Load the real entry address from the first slot of the function descriptor.
753 MOVD 8(R12), R2
754 MOVD (R12), R12
755 #endif
756 MOVD R12, CTR
757 MOVD R4, R3 // arg in r3
758 BL (CTR)
759
760 // Reinitialise zero value register.
761 XOR R0, R0
762
763 // Restore g, stack pointer, toc pointer.
764 // R3 is errno, so don't touch it
765 MOVD (asmcgocallSaveOffset+8)(R1), g
766 MOVD (g_stack+stack_hi)(g), R5
767 MOVD asmcgocallSaveOffset(R1), R6
768 SUB R6, R5
769 #ifndef GOOS_aix
770 MOVD 24(R5), R2
771 #endif
772 MOVD R5, R1
773 BL runtime·save_g(SB)
774
775 // ret = R3
776 RET
777
778 nosave:
779 // Running on a system stack, perhaps even without a g.
780 // Having no g can happen during thread creation or thread teardown.
781 // This code is like the above sequence but without saving/restoring g
782 // and without worrying about the stack moving out from under us
783 // (because we're on a system stack, not a goroutine stack).
784 // The above code could be used directly if already on a system stack,
785 // but then the only path through this code would be a rare case.
786 // Using this code for all "already on system stack" calls exercises it more,
787 // which should help keep it correct.
788
789 SUB $(asmcgocallSaveOffset+8), R1, R10
790 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
791 MOVD R7, asmcgocallSaveOffset(R1) // Save original stack pointer.
792
793 MOVD R3, R12 // fn
794 #ifdef GO_PPC64X_HAS_FUNCDESC
795 // Load the real entry address from the first slot of the function descriptor.
796 MOVD 8(R12), R2
797 MOVD (R12), R12
798 #endif
799 MOVD R12, CTR
800 MOVD R4, R3 // arg
801 BL (CTR)
802
803 // Reinitialise zero value register.
804 XOR R0, R0
805
806 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
807 #ifndef GOOS_aix
808 MOVD 24(R1), R2
809 #endif
810 // ret = R3
811 RET
812
813 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
814 // See cgocall.go for more details.
815 TEXT ·cgocallback(SB),NOSPLIT,$24-24
816 NO_LOCAL_POINTERS
817
818 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
819 // It is used to dropm while thread is exiting.
820 MOVD fn+0(FP), R5
821 CMP R5, $0
822 BNE loadg
823 // Restore the g from frame.
824 MOVD frame+8(FP), g
825 BR dropm
826
827 loadg:
828 // Load m and g from thread-local storage.
829 #ifndef GOOS_openbsd
830 MOVBZ runtime·iscgo(SB), R3
831 CMP R3, $0
832 BEQ nocgo
833 #endif
834 BL runtime·load_g(SB)
835 nocgo:
836
837 // If g is nil, Go did not create the current thread,
838 // or if this thread never called into Go on pthread platforms.
839 // Call needm to obtain one for temporary use.
840 // In this case, we're running on the thread stack, so there's
841 // lots of space, but the linker doesn't know. Hide the call from
842 // the linker analysis by using an indirect call.
843 CMP g, $0
844 BEQ needm
845
846 MOVD g_m(g), R8
847 MOVD R8, savedm-8(SP)
848 BR havem
849
850 needm:
851 MOVD g, savedm-8(SP) // g is zero, so is m.
852 MOVD $runtime·needAndBindM(SB), R12
853 MOVD R12, CTR
854 BL (CTR)
855
856 // Set m->sched.sp = SP, so that if a panic happens
857 // during the function we are about to execute, it will
858 // have a valid SP to run on the g0 stack.
859 // The next few lines (after the havem label)
860 // will save this SP onto the stack and then write
861 // the same SP back to m->sched.sp. That seems redundant,
862 // but if an unrecovered panic happens, unwindm will
863 // restore the g->sched.sp from the stack location
864 // and then systemstack will try to use it. If we don't set it here,
865 // that restored SP will be uninitialized (typically 0) and
866 // will not be usable.
867 MOVD g_m(g), R8
868 MOVD m_g0(R8), R3
869 MOVD R1, (g_sched+gobuf_sp)(R3)
870
871 havem:
872 // Now there's a valid m, and we're running on its m->g0.
873 // Save current m->g0->sched.sp on stack and then set it to SP.
874 // Save current sp in m->g0->sched.sp in preparation for
875 // switch back to m->curg stack.
876 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
877 MOVD m_g0(R8), R3
878 MOVD (g_sched+gobuf_sp)(R3), R4
879 MOVD R4, savedsp-24(SP) // must match frame size
880 MOVD R1, (g_sched+gobuf_sp)(R3)
881
882 // Switch to m->curg stack and call runtime.cgocallbackg.
883 // Because we are taking over the execution of m->curg
884 // but *not* resuming what had been running, we need to
885 // save that information (m->curg->sched) so we can restore it.
886 // We can restore m->curg->sched.sp easily, because calling
887 // runtime.cgocallbackg leaves SP unchanged upon return.
888 // To save m->curg->sched.pc, we push it onto the curg stack and
889 // open a frame the same size as cgocallback's g0 frame.
890 // Once we switch to the curg stack, the pushed PC will appear
891 // to be the return PC of cgocallback, so that the traceback
892 // will seamlessly trace back into the earlier calls.
893 MOVD m_curg(R8), g
894 BL runtime·save_g(SB)
895 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
896 MOVD (g_sched+gobuf_pc)(g), R5
897 MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size
898 // Gather our arguments into registers.
899 MOVD fn+0(FP), R5
900 MOVD frame+8(FP), R6
901 MOVD ctxt+16(FP), R7
902 MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size
903 MOVD R5, FIXED_FRAME+0(R1)
904 MOVD R6, FIXED_FRAME+8(R1)
905 MOVD R7, FIXED_FRAME+16(R1)
906
907 MOVD $runtime·cgocallbackg(SB), R12
908 MOVD R12, CTR
909 CALL (CTR) // indirect call to bypass nosplit check. We're on a different stack now.
910
911 // Restore g->sched (== m->curg->sched) from saved values.
912 MOVD 0(R1), R5
913 MOVD R5, (g_sched+gobuf_pc)(g)
914 MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size
915 MOVD R4, (g_sched+gobuf_sp)(g)
916
917 // Switch back to m->g0's stack and restore m->g0->sched.sp.
918 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
919 // so we do not have to restore it.)
920 MOVD g_m(g), R8
921 MOVD m_g0(R8), g
922 BL runtime·save_g(SB)
923 MOVD (g_sched+gobuf_sp)(g), R1
924 MOVD savedsp-24(SP), R4 // must match frame size
925 MOVD R4, (g_sched+gobuf_sp)(g)
926
927 // If the m on entry was nil, we called needm above to borrow an m,
928 // 1. for the duration of the call on non-pthread platforms,
929 // 2. or the duration of the C thread alive on pthread platforms.
930 // If the m on entry wasn't nil,
931 // 1. the thread might be a Go thread,
932 // 2. or it wasn't the first call from a C thread on pthread platforms,
933 // since then we skip dropm to reuse the m in the first call.
934 MOVD savedm-8(SP), R6
935 CMP R6, $0
936 BNE droppedm
937
938 // Skip dropm to reuse it in the next call, when a pthread key has been created.
939 MOVD _cgo_pthread_key_created(SB), R6
940 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
941 CMP R6, $0
942 BEQ dropm
943 MOVD (R6), R6
944 CMP R6, $0
945 BNE droppedm
946
947 dropm:
948 MOVD $runtime·dropm(SB), R12
949 MOVD R12, CTR
950 BL (CTR)
951 droppedm:
952
953 // Done!
954 RET
955
956 // void setg(G*); set g. for use by needm.
957 TEXT runtime·setg(SB), NOSPLIT, $0-8
958 MOVD gg+0(FP), g
959 // This only happens if iscgo, so jump straight to save_g
960 BL runtime·save_g(SB)
961 RET
962
963 #ifdef GO_PPC64X_HAS_FUNCDESC
964 DEFINE_PPC64X_FUNCDESC(setg_gcc<>, _setg_gcc<>)
965 TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
966 #else
967 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
968 #endif
969 // The standard prologue clobbers R31, which is callee-save in
970 // the C ABI, so we have to use $-8-0 and save LR ourselves.
971 MOVD LR, R4
972 // Also save g and R31, since they're callee-save in C ABI
973 MOVD R31, R5
974 MOVD g, R6
975
976 MOVD R3, g
977 BL runtime·save_g(SB)
978
979 MOVD R6, g
980 MOVD R5, R31
981 MOVD R4, LR
982 RET
983
984 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
985 MOVW (R0), R0
986 UNDEF
987
988 #define TBR 268
989
990 // int64 runtime·cputicks(void)
991 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
992 MOVD SPR(TBR), R3
993 MOVD R3, ret+0(FP)
994 RET
995
996 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
997 TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
998 MOVD R3, 0(R20)
999 MOVD R4, 8(R20)
1000 MOVD R5, 16(R20)
1001 MOVD R6, 24(R20)
1002 MOVD R7, 32(R20)
1003 MOVD R8, 40(R20)
1004 MOVD R9, 48(R20)
1005 MOVD R10, 56(R20)
1006 MOVD R14, 64(R20)
1007 MOVD R15, 72(R20)
1008 MOVD R16, 80(R20)
1009 MOVD R17, 88(R20)
1010 FMOVD F1, 96(R20)
1011 FMOVD F2, 104(R20)
1012 FMOVD F3, 112(R20)
1013 FMOVD F4, 120(R20)
1014 FMOVD F5, 128(R20)
1015 FMOVD F6, 136(R20)
1016 FMOVD F7, 144(R20)
1017 FMOVD F8, 152(R20)
1018 FMOVD F9, 160(R20)
1019 FMOVD F10, 168(R20)
1020 FMOVD F11, 176(R20)
1021 FMOVD F12, 184(R20)
1022 RET
1023
1024 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
1025 TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
1026 MOVD 0(R20), R3
1027 MOVD 8(R20), R4
1028 MOVD 16(R20), R5
1029 MOVD 24(R20), R6
1030 MOVD 32(R20), R7
1031 MOVD 40(R20), R8
1032 MOVD 48(R20), R9
1033 MOVD 56(R20), R10
1034 MOVD 64(R20), R14
1035 MOVD 72(R20), R15
1036 MOVD 80(R20), R16
1037 MOVD 88(R20), R17
1038 FMOVD 96(R20), F1
1039 FMOVD 104(R20), F2
1040 FMOVD 112(R20), F3
1041 FMOVD 120(R20), F4
1042 FMOVD 128(R20), F5
1043 FMOVD 136(R20), F6
1044 FMOVD 144(R20), F7
1045 FMOVD 152(R20), F8
1046 FMOVD 160(R20), F9
1047 FMOVD 168(R20), F10
1048 FMOVD 176(R20), F11
1049 FMOVD 184(R20), F12
1050 RET
1051
1052 // AES hashing not implemented for ppc64
1053 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
1054 JMP runtime·memhashFallback<ABIInternal>(SB)
1055 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1056 JMP runtime·strhashFallback<ABIInternal>(SB)
1057 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1058 JMP runtime·memhash32Fallback<ABIInternal>(SB)
1059 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1060 JMP runtime·memhash64Fallback<ABIInternal>(SB)
1061
1062 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
1063 // Must obey the gcc calling convention.
1064 #ifdef GOOS_aix
1065 // On AIX, _cgo_topofstack is defined in runtime/cgo, because it must
1066 // be a longcall in order to prevent trampolines from ld.
1067 TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
1068 #else
1069 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
1070 #endif
1071 // g (R30) and R31 are callee-save in the C ABI, so save them
1072 MOVD g, R4
1073 MOVD R31, R5
1074 MOVD LR, R6
1075
1076 BL runtime·load_g(SB) // clobbers g (R30), R31
1077 MOVD g_m(g), R3
1078 MOVD m_curg(R3), R3
1079 MOVD (g_stack+stack_hi)(R3), R3
1080
1081 MOVD R4, g
1082 MOVD R5, R31
1083 MOVD R6, LR
1084 RET
1085
1086 // The top-most function running on a goroutine
1087 // returns to goexit+PCQuantum.
1088 //
1089 // When dynamically linking Go, it can be returned to from a function
1090 // implemented in a different module and so needs to reload the TOC pointer
1091 // from the stack (although this function declares that it does not set up x-a
1092 // frame, newproc1 does in fact allocate one for goexit and saves the TOC
1093 // pointer in the correct place).
1094 // goexit+_PCQuantum is halfway through the usual global entry point prologue
1095 // that derives r2 from r12 which is a bit silly, but not harmful.
1096 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
1097 MOVD 24(R1), R2
1098 BL runtime·goexit1(SB) // does not return
1099 // traceback from goexit1 must hit code range of goexit
1100 MOVD R0, R0 // NOP
1101
1102 // prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the
1103 // module containing runtime) to the frame that goexit will execute in when
1104 // the goroutine exits. It's implemented in assembly mainly because that's the
1105 // easiest way to get access to R2.
1106 TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8
1107 MOVD sp+0(FP), R3
1108 MOVD R2, 24(R3)
1109 RET
1110
1111 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
1112 ADD $-8, R1
1113 MOVD R31, 0(R1)
1114 MOVD runtime·lastmoduledatap(SB), R4
1115 MOVD R3, moduledata_next(R4)
1116 MOVD R3, runtime·lastmoduledatap(SB)
1117 MOVD 0(R1), R31
1118 ADD $8, R1
1119 RET
1120
1121 TEXT ·checkASM(SB),NOSPLIT,$0-1
1122 MOVW $1, R3
1123 MOVB R3, ret+0(FP)
1124 RET
1125
1126 // gcWriteBarrier informs the GC about heap pointer writes.
1127 //
1128 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
1129 // number of bytes of buffer needed in R29, and returns a pointer
1130 // to the buffer space in R29.
1131 // It clobbers condition codes.
1132 // It does not clobber R0 through R17 (except special registers),
1133 // but may clobber any other register, *including* R31.
1134 TEXT gcWriteBarrier<>(SB),NOSPLIT,$120
1135 // The standard prologue clobbers R31.
1136 // We use R18, R19, and R31 as scratch registers.
1137 retry:
1138 MOVD g_m(g), R18
1139 MOVD m_p(R18), R18
1140 MOVD (p_wbBuf+wbBuf_next)(R18), R19
1141 MOVD (p_wbBuf+wbBuf_end)(R18), R31
1142 // Increment wbBuf.next position.
1143 ADD R29, R19
1144 // Is the buffer full?
1145 CMPU R31, R19
1146 BLT flush
1147 // Commit to the larger buffer.
1148 MOVD R19, (p_wbBuf+wbBuf_next)(R18)
1149 // Make return value (the original next position)
1150 SUB R29, R19, R29
1151 RET
1152
1153 flush:
1154 // Save registers R0 through R15 since these were not saved by the caller.
1155 // We don't save all registers on ppc64 because it takes too much space.
1156 MOVD R20, (FIXED_FRAME+0)(R1)
1157 MOVD R21, (FIXED_FRAME+8)(R1)
1158 // R0 is always 0, so no need to spill.
1159 // R1 is SP.
1160 // R2 is SB.
1161 MOVD R3, (FIXED_FRAME+16)(R1)
1162 MOVD R4, (FIXED_FRAME+24)(R1)
1163 MOVD R5, (FIXED_FRAME+32)(R1)
1164 MOVD R6, (FIXED_FRAME+40)(R1)
1165 MOVD R7, (FIXED_FRAME+48)(R1)
1166 MOVD R8, (FIXED_FRAME+56)(R1)
1167 MOVD R9, (FIXED_FRAME+64)(R1)
1168 MOVD R10, (FIXED_FRAME+72)(R1)
1169 // R11, R12 may be clobbered by external-linker-inserted trampoline
1170 // R13 is REGTLS
1171 MOVD R14, (FIXED_FRAME+80)(R1)
1172 MOVD R15, (FIXED_FRAME+88)(R1)
1173 MOVD R16, (FIXED_FRAME+96)(R1)
1174 MOVD R17, (FIXED_FRAME+104)(R1)
1175 MOVD R29, (FIXED_FRAME+112)(R1)
1176
1177 CALL runtime·wbBufFlush(SB)
1178
1179 MOVD (FIXED_FRAME+0)(R1), R20
1180 MOVD (FIXED_FRAME+8)(R1), R21
1181 MOVD (FIXED_FRAME+16)(R1), R3
1182 MOVD (FIXED_FRAME+24)(R1), R4
1183 MOVD (FIXED_FRAME+32)(R1), R5
1184 MOVD (FIXED_FRAME+40)(R1), R6
1185 MOVD (FIXED_FRAME+48)(R1), R7
1186 MOVD (FIXED_FRAME+56)(R1), R8
1187 MOVD (FIXED_FRAME+64)(R1), R9
1188 MOVD (FIXED_FRAME+72)(R1), R10
1189 MOVD (FIXED_FRAME+80)(R1), R14
1190 MOVD (FIXED_FRAME+88)(R1), R15
1191 MOVD (FIXED_FRAME+96)(R1), R16
1192 MOVD (FIXED_FRAME+104)(R1), R17
1193 MOVD (FIXED_FRAME+112)(R1), R29
1194 JMP retry
1195
1196 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
1197 MOVD $8, R29
1198 JMP gcWriteBarrier<>(SB)
1199 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
1200 MOVD $16, R29
1201 JMP gcWriteBarrier<>(SB)
1202 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
1203 MOVD $24, R29
1204 JMP gcWriteBarrier<>(SB)
1205 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
1206 MOVD $32, R29
1207 JMP gcWriteBarrier<>(SB)
1208 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
1209 MOVD $40, R29
1210 JMP gcWriteBarrier<>(SB)
1211 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
1212 MOVD $48, R29
1213 JMP gcWriteBarrier<>(SB)
1214 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
1215 MOVD $56, R29
1216 JMP gcWriteBarrier<>(SB)
1217 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
1218 MOVD $64, R29
1219 JMP gcWriteBarrier<>(SB)
1220
1221 DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1222 GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1223
1224 // debugCallV2 is the entry point for debugger-injected function
1225 // calls on running goroutines. It informs the runtime that a
1226 // debug call has been injected and creates a call frame for the
1227 // debugger to fill in.
1228 //
1229 // To inject a function call, a debugger should:
1230 // 1. Check that the goroutine is in state _Grunning and that
1231 // there are at least 320 bytes free on the stack.
1232 // 2. Set SP as SP-32.
1233 // 3. Store the current LR in (SP) (using the SP after step 2).
1234 // 4. Store the current PC in the LR register.
1235 // 5. Write the desired argument frame size at SP-32
1236 // 6. Save all machine registers (including flags and floating point registers)
1237 // so they can be restored later by the debugger.
1238 // 7. Set the PC to debugCallV2 and resume execution.
1239 //
1240 // If the goroutine is in state _Grunnable, then it's not generally
1241 // safe to inject a call because it may return out via other runtime
1242 // operations. Instead, the debugger should unwind the stack to find
1243 // the return to non-runtime code, add a temporary breakpoint there,
1244 // and inject the call once that breakpoint is hit.
1245 //
1246 // If the goroutine is in any other state, it's not safe to inject a call.
1247 //
1248 // This function communicates back to the debugger by setting R20 and
1249 // invoking TW to raise a breakpoint signal. Note that the signal PC of
1250 // the signal triggered by the TW instruction is the PC where the signal
1251 // is trapped, not the next PC, so to resume execution, the debugger needs
1252 // to set the signal PC to PC+4. See the comments in the implementation for
1253 // the protocol the debugger is expected to follow. InjectDebugCall in the
1254 // runtime tests demonstrates this protocol.
1255 // The debugger must ensure that any pointers passed to the function
1256 // obey escape analysis requirements. Specifically, it must not pass
1257 // a stack pointer to an escaping argument. debugCallV2 cannot check
1258 // this invariant.
1259 //
1260 // This is ABIInternal because Go code injects its PC directly into new
1261 // goroutine stacks.
1262 #ifdef GOARCH_ppc64le
1263 TEXT runtime·debugCallV2<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-0
1264 // save scratch register R31 first
1265 MOVD R31, -184(R1)
1266 MOVD 0(R1), R31
1267 // save caller LR
1268 MOVD R31, -304(R1)
1269 MOVD -32(R1), R31
1270 // save argument frame size
1271 MOVD R31, -192(R1)
1272 MOVD LR, R31
1273 MOVD R31, -320(R1)
1274 ADD $-320, R1
1275 // save all registers that can contain pointers
1276 // and the CR register
1277 MOVW CR, R31
1278 MOVD R31, 8(R1)
1279 MOVD R2, 24(R1)
1280 MOVD R3, 56(R1)
1281 MOVD R4, 64(R1)
1282 MOVD R5, 72(R1)
1283 MOVD R6, 80(R1)
1284 MOVD R7, 88(R1)
1285 MOVD R8, 96(R1)
1286 MOVD R9, 104(R1)
1287 MOVD R10, 112(R1)
1288 MOVD R11, 120(R1)
1289 MOVD R12, 144(R1)
1290 MOVD R13, 152(R1)
1291 MOVD R14, 160(R1)
1292 MOVD R15, 168(R1)
1293 MOVD R16, 176(R1)
1294 MOVD R17, 184(R1)
1295 MOVD R18, 192(R1)
1296 MOVD R19, 200(R1)
1297 MOVD R20, 208(R1)
1298 MOVD R21, 216(R1)
1299 MOVD R22, 224(R1)
1300 MOVD R23, 232(R1)
1301 MOVD R24, 240(R1)
1302 MOVD R25, 248(R1)
1303 MOVD R26, 256(R1)
1304 MOVD R27, 264(R1)
1305 MOVD R28, 272(R1)
1306 MOVD R29, 280(R1)
1307 MOVD g, 288(R1)
1308 MOVD LR, R31
1309 MOVD R31, 32(R1)
1310 CALL runtime·debugCallCheck(SB)
1311 MOVD 40(R1), R22
1312 XOR R0, R0
1313 CMP R22, $0
1314 BEQ good
1315 MOVD 48(R1), R22
1316 MOVD $8, R20
1317 TW $31, R0, R0
1318
1319 BR restore
1320
1321 good:
1322 #define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1323 MOVD $MAXSIZE, R23; \
1324 CMP R26, R23; \
1325 BGT 5(PC); \
1326 MOVD $NAME(SB), R26; \
1327 MOVD R26, 32(R1); \
1328 CALL runtime·debugCallWrap(SB); \
1329 BR restore
1330
1331 // the argument frame size
1332 MOVD 128(R1), R26
1333
1334 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1335 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1336 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1337 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1338 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1339 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1340 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1341 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1342 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1343 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1344 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1345 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1346 // The frame size is too large. Report the error.
1347 MOVD $debugCallFrameTooLarge<>(SB), R22
1348 MOVD R22, 32(R1)
1349 MOVD $20, R22
1350 // length of debugCallFrameTooLarge string
1351 MOVD R22, 40(R1)
1352 MOVD $8, R20
1353 TW $31, R0, R0
1354 BR restore
1355 restore:
1356 MOVD $16, R20
1357 TW $31, R0, R0
1358 // restore all registers that can contain
1359 // pointers including CR
1360 MOVD 8(R1), R31
1361 MOVW R31, CR
1362 MOVD 24(R1), R2
1363 MOVD 56(R1), R3
1364 MOVD 64(R1), R4
1365 MOVD 72(R1), R5
1366 MOVD 80(R1), R6
1367 MOVD 88(R1), R7
1368 MOVD 96(R1), R8
1369 MOVD 104(R1), R9
1370 MOVD 112(R1), R10
1371 MOVD 120(R1), R11
1372 MOVD 144(R1), R12
1373 MOVD 152(R1), R13
1374 MOVD 160(R1), R14
1375 MOVD 168(R1), R15
1376 MOVD 176(R1), R16
1377 MOVD 184(R1), R17
1378 MOVD 192(R1), R18
1379 MOVD 200(R1), R19
1380 MOVD 208(R1), R20
1381 MOVD 216(R1), R21
1382 MOVD 224(R1), R22
1383 MOVD 232(R1), R23
1384 MOVD 240(R1), R24
1385 MOVD 248(R1), R25
1386 MOVD 256(R1), R26
1387 MOVD 264(R1), R27
1388 MOVD 272(R1), R28
1389 MOVD 280(R1), R29
1390 MOVD 288(R1), g
1391 MOVD 16(R1), R31
1392 // restore old LR
1393 MOVD R31, LR
1394 // restore caller PC
1395 MOVD 0(R1), CTR
1396 MOVD 136(R1), R31
1397 // Add 32 bytes more to compensate for SP change in saveSigContext
1398 ADD $352, R1
1399 JMP (CTR)
1400 #endif
1401 #define DEBUG_CALL_FN(NAME,MAXSIZE) \
1402 TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1403 NO_LOCAL_POINTERS; \
1404 MOVD $0, R20; \
1405 TW $31, R0, R0 \
1406 MOVD $1, R20; \
1407 TW $31, R0, R0 \
1408 RET
1409 DEBUG_CALL_FN(debugCall32<>, 32)
1410 DEBUG_CALL_FN(debugCall64<>, 64)
1411 DEBUG_CALL_FN(debugCall128<>, 128)
1412 DEBUG_CALL_FN(debugCall256<>, 256)
1413 DEBUG_CALL_FN(debugCall512<>, 512)
1414 DEBUG_CALL_FN(debugCall1024<>, 1024)
1415 DEBUG_CALL_FN(debugCall2048<>, 2048)
1416 DEBUG_CALL_FN(debugCall4096<>, 4096)
1417 DEBUG_CALL_FN(debugCall8192<>, 8192)
1418 DEBUG_CALL_FN(debugCall16384<>, 16384)
1419 DEBUG_CALL_FN(debugCall32768<>, 32768)
1420 DEBUG_CALL_FN(debugCall65536<>, 65536)
1421
1422 #ifdef GOARCH_ppc64le
1423 // func debugCallPanicked(val interface{})
1424 TEXT runtime·debugCallPanicked(SB),NOSPLIT,$32-16
1425 // Copy the panic value to the top of stack at SP+32.
1426 MOVD val_type+0(FP), R31
1427 MOVD R31, 32(R1)
1428 MOVD val_data+8(FP), R31
1429 MOVD R31, 40(R1)
1430 MOVD $2, R20
1431 TW $31, R0, R0
1432 RET
1433 #endif
1434
1435 TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$88-0
1436 // Note: frame size is 16 bytes larger than necessary
1437 // in order to pacify vet. Vet doesn't understand ppc64
1438 // layout properly.
1439 NO_LOCAL_POINTERS
1440 // Save all 7 int registers that could have an index in them.
1441 // They may be pointers, but if so they are dead.
1442 // Skip R0 aka ZERO, R1 aka SP, R2 aka SB
1443 MOVD R3, 48(R1)
1444 MOVD R4, 56(R1)
1445 MOVD R5, 64(R1)
1446 MOVD R6, 72(R1)
1447 MOVD R7, 80(R1)
1448 MOVD R8, 88(R1)
1449 MOVD R9, 96(R1)
1450 // Note: we only save 7 registers to keep under nosplit stack limit
1451 // Also, R11 is clobbered in dynamic linking situations
1452
1453 MOVD LR, R3 // PC immediately after call to panicBounds
1454 ADD $48, R1, R4 // pointer to save area
1455 CALL runtime·panicBounds64<ABIInternal>(SB)
1456 RET
1457
1458 // These functions are used when internal linking cgo with external
1459 // objects compiled with the -Os on gcc. They reduce prologue/epilogue
1460 // size by deferring preservation of callee-save registers to a shared
1461 // function. These are defined in PPC64 ELFv2 2.3.3 (but also present
1462 // in ELFv1)
1463 //
1464 // These appear unused, but the linker will redirect calls to functions
1465 // like _savegpr0_14 or _restgpr1_14 to runtime.elf_savegpr0 or
1466 // runtime.elf_restgpr1 with an appropriate offset based on the number
1467 // register operations required when linking external objects which
1468 // make these calls. For GPR/FPR saves, the minimum register value is
1469 // 14, for VR it is 20.
1470 //
1471 // These are only used when linking such cgo code internally. Note, R12
1472 // and R0 may be used in different ways than regular ELF compliant
1473 // functions.
1474 TEXT runtime·elf_savegpr0(SB),NOSPLIT|NOFRAME,$0
1475 // R0 holds the LR of the caller's caller, R1 holds save location
1476 MOVD R14, -144(R1)
1477 MOVD R15, -136(R1)
1478 MOVD R16, -128(R1)
1479 MOVD R17, -120(R1)
1480 MOVD R18, -112(R1)
1481 MOVD R19, -104(R1)
1482 MOVD R20, -96(R1)
1483 MOVD R21, -88(R1)
1484 MOVD R22, -80(R1)
1485 MOVD R23, -72(R1)
1486 MOVD R24, -64(R1)
1487 MOVD R25, -56(R1)
1488 MOVD R26, -48(R1)
1489 MOVD R27, -40(R1)
1490 MOVD R28, -32(R1)
1491 MOVD R29, -24(R1)
1492 MOVD g, -16(R1)
1493 MOVD R31, -8(R1)
1494 MOVD R0, 16(R1)
1495 RET
1496 TEXT runtime·elf_restgpr0(SB),NOSPLIT|NOFRAME,$0
1497 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1498 MOVD -144(R1), R14
1499 MOVD -136(R1), R15
1500 MOVD -128(R1), R16
1501 MOVD -120(R1), R17
1502 MOVD -112(R1), R18
1503 MOVD -104(R1), R19
1504 MOVD -96(R1), R20
1505 MOVD -88(R1), R21
1506 MOVD -80(R1), R22
1507 MOVD -72(R1), R23
1508 MOVD -64(R1), R24
1509 MOVD -56(R1), R25
1510 MOVD -48(R1), R26
1511 MOVD -40(R1), R27
1512 MOVD -32(R1), R28
1513 MOVD -24(R1), R29
1514 MOVD -16(R1), g
1515 MOVD -8(R1), R31
1516 MOVD 16(R1), R0 // Load and return to saved LR
1517 MOVD R0, LR
1518 RET
1519 TEXT runtime·elf_savegpr1(SB),NOSPLIT|NOFRAME,$0
1520 // R12 holds the save location
1521 MOVD R14, -144(R12)
1522 MOVD R15, -136(R12)
1523 MOVD R16, -128(R12)
1524 MOVD R17, -120(R12)
1525 MOVD R18, -112(R12)
1526 MOVD R19, -104(R12)
1527 MOVD R20, -96(R12)
1528 MOVD R21, -88(R12)
1529 MOVD R22, -80(R12)
1530 MOVD R23, -72(R12)
1531 MOVD R24, -64(R12)
1532 MOVD R25, -56(R12)
1533 MOVD R26, -48(R12)
1534 MOVD R27, -40(R12)
1535 MOVD R28, -32(R12)
1536 MOVD R29, -24(R12)
1537 MOVD g, -16(R12)
1538 MOVD R31, -8(R12)
1539 RET
1540 TEXT runtime·elf_restgpr1(SB),NOSPLIT|NOFRAME,$0
1541 // R12 holds the save location
1542 MOVD -144(R12), R14
1543 MOVD -136(R12), R15
1544 MOVD -128(R12), R16
1545 MOVD -120(R12), R17
1546 MOVD -112(R12), R18
1547 MOVD -104(R12), R19
1548 MOVD -96(R12), R20
1549 MOVD -88(R12), R21
1550 MOVD -80(R12), R22
1551 MOVD -72(R12), R23
1552 MOVD -64(R12), R24
1553 MOVD -56(R12), R25
1554 MOVD -48(R12), R26
1555 MOVD -40(R12), R27
1556 MOVD -32(R12), R28
1557 MOVD -24(R12), R29
1558 MOVD -16(R12), g
1559 MOVD -8(R12), R31
1560 RET
1561 TEXT runtime·elf_savefpr(SB),NOSPLIT|NOFRAME,$0
1562 // R0 holds the LR of the caller's caller, R1 holds save location
1563 FMOVD F14, -144(R1)
1564 FMOVD F15, -136(R1)
1565 FMOVD F16, -128(R1)
1566 FMOVD F17, -120(R1)
1567 FMOVD F18, -112(R1)
1568 FMOVD F19, -104(R1)
1569 FMOVD F20, -96(R1)
1570 FMOVD F21, -88(R1)
1571 FMOVD F22, -80(R1)
1572 FMOVD F23, -72(R1)
1573 FMOVD F24, -64(R1)
1574 FMOVD F25, -56(R1)
1575 FMOVD F26, -48(R1)
1576 FMOVD F27, -40(R1)
1577 FMOVD F28, -32(R1)
1578 FMOVD F29, -24(R1)
1579 FMOVD F30, -16(R1)
1580 FMOVD F31, -8(R1)
1581 MOVD R0, 16(R1)
1582 RET
1583 TEXT runtime·elf_restfpr(SB),NOSPLIT|NOFRAME,$0
1584 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1585 FMOVD -144(R1), F14
1586 FMOVD -136(R1), F15
1587 FMOVD -128(R1), F16
1588 FMOVD -120(R1), F17
1589 FMOVD -112(R1), F18
1590 FMOVD -104(R1), F19
1591 FMOVD -96(R1), F20
1592 FMOVD -88(R1), F21
1593 FMOVD -80(R1), F22
1594 FMOVD -72(R1), F23
1595 FMOVD -64(R1), F24
1596 FMOVD -56(R1), F25
1597 FMOVD -48(R1), F26
1598 FMOVD -40(R1), F27
1599 FMOVD -32(R1), F28
1600 FMOVD -24(R1), F29
1601 FMOVD -16(R1), F30
1602 FMOVD -8(R1), F31
1603 MOVD 16(R1), R0 // Load and return to saved LR
1604 MOVD R0, LR
1605 RET
1606 TEXT runtime·elf_savevr(SB),NOSPLIT|NOFRAME,$0
1607 // R0 holds the save location, R12 is clobbered
1608 MOVD $-192, R12
1609 STVX V20, (R0+R12)
1610 MOVD $-176, R12
1611 STVX V21, (R0+R12)
1612 MOVD $-160, R12
1613 STVX V22, (R0+R12)
1614 MOVD $-144, R12
1615 STVX V23, (R0+R12)
1616 MOVD $-128, R12
1617 STVX V24, (R0+R12)
1618 MOVD $-112, R12
1619 STVX V25, (R0+R12)
1620 MOVD $-96, R12
1621 STVX V26, (R0+R12)
1622 MOVD $-80, R12
1623 STVX V27, (R0+R12)
1624 MOVD $-64, R12
1625 STVX V28, (R0+R12)
1626 MOVD $-48, R12
1627 STVX V29, (R0+R12)
1628 MOVD $-32, R12
1629 STVX V30, (R0+R12)
1630 MOVD $-16, R12
1631 STVX V31, (R0+R12)
1632 RET
1633 TEXT runtime·elf_restvr(SB),NOSPLIT|NOFRAME,$0
1634 // R0 holds the save location, R12 is clobbered
1635 MOVD $-192, R12
1636 LVX (R0+R12), V20
1637 MOVD $-176, R12
1638 LVX (R0+R12), V21
1639 MOVD $-160, R12
1640 LVX (R0+R12), V22
1641 MOVD $-144, R12
1642 LVX (R0+R12), V23
1643 MOVD $-128, R12
1644 LVX (R0+R12), V24
1645 MOVD $-112, R12
1646 LVX (R0+R12), V25
1647 MOVD $-96, R12
1648 LVX (R0+R12), V26
1649 MOVD $-80, R12
1650 LVX (R0+R12), V27
1651 MOVD $-64, R12
1652 LVX (R0+R12), V28
1653 MOVD $-48, R12
1654 LVX (R0+R12), V29
1655 MOVD $-32, R12
1656 LVX (R0+R12), V30
1657 MOVD $-16, R12
1658 LVX (R0+R12), V31
1659 RET
1660
View as plain text