Text file
src/runtime/asm_s390x.s
1 // Copyright 2016 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "funcdata.h"
8 #include "textflag.h"
9
10 // _rt0_s390x_lib is common startup code for s390x systems when
11 // using -buildmode=c-archive or -buildmode=c-shared. The linker will
12 // arrange to invoke this function as a global constructor (for
13 // c-archive) or when the shared library is loaded (for c-shared).
14 // We expect argc and argv to be passed in the usual C ABI registers
15 // R2 and R3.
16 TEXT _rt0_s390x_lib(SB), NOSPLIT|NOFRAME, $0
17 STMG R6, R15, 48(R15)
18 MOVD R2, _rt0_s390x_lib_argc<>(SB)
19 MOVD R3, _rt0_s390x_lib_argv<>(SB)
20
21 // Save R6-R15 in the register save area of the calling function.
22 STMG R6, R15, 48(R15)
23
24 // Allocate 80 bytes on the stack.
25 MOVD $-80(R15), R15
26
27 // Save F8-F15 in our stack frame.
28 FMOVD F8, 16(R15)
29 FMOVD F9, 24(R15)
30 FMOVD F10, 32(R15)
31 FMOVD F11, 40(R15)
32 FMOVD F12, 48(R15)
33 FMOVD F13, 56(R15)
34 FMOVD F14, 64(R15)
35 FMOVD F15, 72(R15)
36
37 // Synchronous initialization.
38 MOVD $runtime·libpreinit(SB), R1
39 BL R1
40
41 // Create a new thread to finish Go runtime initialization.
42 MOVD _cgo_sys_thread_create(SB), R1
43 CMP R1, $0
44 BEQ nocgo
45 MOVD $_rt0_s390x_lib_go(SB), R2
46 MOVD $0, R3
47 BL R1
48 BR restore
49
50 nocgo:
51 MOVD $0x800000, R1 // stacksize
52 MOVD R1, 0(R15)
53 MOVD $_rt0_s390x_lib_go(SB), R1
54 MOVD R1, 8(R15) // fn
55 MOVD $runtime·newosproc(SB), R1
56 BL R1
57
58 restore:
59 // Restore F8-F15 from our stack frame.
60 FMOVD 16(R15), F8
61 FMOVD 24(R15), F9
62 FMOVD 32(R15), F10
63 FMOVD 40(R15), F11
64 FMOVD 48(R15), F12
65 FMOVD 56(R15), F13
66 FMOVD 64(R15), F14
67 FMOVD 72(R15), F15
68 MOVD $80(R15), R15
69
70 // Restore R6-R15.
71 LMG 48(R15), R6, R15
72 RET
73
74 // _rt0_s390x_lib_go initializes the Go runtime.
75 // This is started in a separate thread by _rt0_s390x_lib.
76 TEXT _rt0_s390x_lib_go(SB), NOSPLIT|NOFRAME, $0
77 MOVD _rt0_s390x_lib_argc<>(SB), R2
78 MOVD _rt0_s390x_lib_argv<>(SB), R3
79 MOVD $runtime·rt0_go(SB), R1
80 BR R1
81
82 DATA _rt0_s390x_lib_argc<>(SB)/8, $0
83 GLOBL _rt0_s390x_lib_argc<>(SB), NOPTR, $8
84 DATA _rt0_s90x_lib_argv<>(SB)/8, $0
85 GLOBL _rt0_s390x_lib_argv<>(SB), NOPTR, $8
86
87 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
88 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
89 // C TLS base pointer in AR0:AR1
90
91 // initialize essential registers
92 XOR R0, R0
93
94 SUB $24, R15
95 MOVW R2, 8(R15) // argc
96 MOVD R3, 16(R15) // argv
97
98 // create istack out of the given (operating system) stack.
99 // _cgo_init may update stackguard.
100 MOVD $runtime·g0(SB), g
101 MOVD R15, R11
102 SUB $(64*1024), R11
103 MOVD R11, g_stackguard0(g)
104 MOVD R11, g_stackguard1(g)
105 MOVD R11, (g_stack+stack_lo)(g)
106 MOVD R15, (g_stack+stack_hi)(g)
107
108 // if there is a _cgo_init, call it using the gcc ABI.
109 MOVD _cgo_init(SB), R11
110 CMPBEQ R11, $0, nocgo
111 MOVW AR0, R4 // (AR0 << 32 | AR1) is the TLS base pointer; MOVD is translated to EAR
112 SLD $32, R4, R4
113 MOVW AR1, R4 // arg 2: TLS base pointer
114 MOVD $setg_gcc<>(SB), R3 // arg 1: setg
115 MOVD g, R2 // arg 0: G
116 // C functions expect 160 bytes of space on caller stack frame
117 // and an 8-byte aligned stack pointer
118 MOVD R15, R9 // save current stack (R9 is preserved in the Linux ABI)
119 SUB $160, R15 // reserve 160 bytes
120 MOVD $~7, R6
121 AND R6, R15 // 8-byte align
122 BL R11 // this call clobbers volatile registers according to Linux ABI (R0-R5, R14)
123 MOVD R9, R15 // restore stack
124 XOR R0, R0 // zero R0
125
126 nocgo:
127 // update stackguard after _cgo_init
128 MOVD (g_stack+stack_lo)(g), R2
129 ADD $const_stackGuard, R2
130 MOVD R2, g_stackguard0(g)
131 MOVD R2, g_stackguard1(g)
132
133 // set the per-goroutine and per-mach "registers"
134 MOVD $runtime·m0(SB), R2
135
136 // save m->g0 = g0
137 MOVD g, m_g0(R2)
138 // save m0 to g0->m
139 MOVD R2, g_m(g)
140
141 BL runtime·check(SB)
142
143 // argc/argv are already prepared on stack
144 BL runtime·args(SB)
145 BL runtime·checkS390xCPU(SB)
146 BL runtime·osinit(SB)
147 BL runtime·schedinit(SB)
148
149 // create a new goroutine to start program
150 MOVD $runtime·mainPC(SB), R2 // entry
151 SUB $16, R15
152 MOVD R2, 8(R15)
153 MOVD $0, 0(R15)
154 BL runtime·newproc(SB)
155 ADD $16, R15
156
157 // start this M
158 BL runtime·mstart(SB)
159
160 MOVD $0, 1(R0)
161 RET
162
163 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB)
164 GLOBL runtime·mainPC(SB),RODATA,$8
165
166 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
167 BRRK
168 RET
169
170 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
171 RET
172
173 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
174 CALL runtime·mstart0(SB)
175 RET // not reached
176
177 /*
178 * go-routine
179 */
180
181 // void gogo(Gobuf*)
182 // restore state from Gobuf; longjmp
183 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
184 MOVD buf+0(FP), R5
185 MOVD gobuf_g(R5), R6
186 MOVD 0(R6), R7 // make sure g != nil
187 BR gogo<>(SB)
188
189 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
190 MOVD R6, g
191 BL runtime·save_g(SB)
192
193 MOVD 0(g), R4
194 MOVD gobuf_sp(R5), R15
195 MOVD gobuf_lr(R5), LR
196 MOVD gobuf_ret(R5), R3
197 MOVD gobuf_ctxt(R5), R12
198 MOVD $0, gobuf_sp(R5)
199 MOVD $0, gobuf_ret(R5)
200 MOVD $0, gobuf_lr(R5)
201 MOVD $0, gobuf_ctxt(R5)
202 CMP R0, R0 // set condition codes for == test, needed by stack split
203 MOVD gobuf_pc(R5), R6
204 BR (R6)
205
206 // void mcall(fn func(*g))
207 // Switch to m->g0's stack, call fn(g).
208 // Fn must never return. It should gogo(&g->sched)
209 // to keep running g.
210 TEXT runtime·mcall(SB), NOSPLIT, $-8-8
211 // Save caller state in g->sched
212 MOVD R15, (g_sched+gobuf_sp)(g)
213 MOVD LR, (g_sched+gobuf_pc)(g)
214 MOVD $0, (g_sched+gobuf_lr)(g)
215
216 // Switch to m->g0 & its stack, call fn.
217 MOVD g, R3
218 MOVD g_m(g), R8
219 MOVD m_g0(R8), g
220 BL runtime·save_g(SB)
221 CMP g, R3
222 BNE 2(PC)
223 BR runtime·badmcall(SB)
224 MOVD fn+0(FP), R12 // context
225 MOVD 0(R12), R4 // code pointer
226 MOVD (g_sched+gobuf_sp)(g), R15 // sp = m->g0->sched.sp
227 SUB $16, R15
228 MOVD R3, 8(R15)
229 MOVD $0, 0(R15)
230 BL (R4)
231 BR runtime·badmcall2(SB)
232
233 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
234 // of the G stack. We need to distinguish the routine that
235 // lives at the bottom of the G stack from the one that lives
236 // at the top of the system stack because the one at the top of
237 // the system stack terminates the stack walk (see topofstack()).
238 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
239 UNDEF
240 BL (LR) // make sure this function is not leaf
241 RET
242
243 // func systemstack(fn func())
244 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
245 MOVD fn+0(FP), R3 // R3 = fn
246 MOVD R3, R12 // context
247 MOVD g_m(g), R4 // R4 = m
248
249 MOVD m_gsignal(R4), R5 // R5 = gsignal
250 CMPBEQ g, R5, noswitch
251
252 MOVD m_g0(R4), R5 // R5 = g0
253 CMPBEQ g, R5, noswitch
254
255 MOVD m_curg(R4), R6
256 CMPBEQ g, R6, switch
257
258 // Bad: g is not gsignal, not g0, not curg. What is it?
259 // Hide call from linker nosplit analysis.
260 MOVD $runtime·badsystemstack(SB), R3
261 BL (R3)
262 BL runtime·abort(SB)
263
264 switch:
265 // save our state in g->sched. Pretend to
266 // be systemstack_switch if the G stack is scanned.
267 BL gosave_systemstack_switch<>(SB)
268
269 // switch to g0
270 MOVD R5, g
271 BL runtime·save_g(SB)
272 MOVD (g_sched+gobuf_sp)(g), R15
273
274 // call target function
275 MOVD 0(R12), R3 // code pointer
276 BL (R3)
277
278 // switch back to g
279 MOVD g_m(g), R3
280 MOVD m_curg(R3), g
281 BL runtime·save_g(SB)
282 MOVD (g_sched+gobuf_sp)(g), R15
283 MOVD $0, (g_sched+gobuf_sp)(g)
284 RET
285
286 noswitch:
287 // already on m stack, just call directly
288 // Using a tail call here cleans up tracebacks since we won't stop
289 // at an intermediate systemstack.
290 MOVD 0(R12), R3 // code pointer
291 MOVD 0(R15), LR // restore LR
292 ADD $8, R15
293 BR (R3)
294
295 // func switchToCrashStack0(fn func())
296 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
297 MOVD fn+0(FP), R12 // context
298 MOVD g_m(g), R4 // curm
299
300 // set g to gcrash
301 MOVD $runtime·gcrash(SB), g // g = &gcrash
302 BL runtime·save_g(SB)
303 MOVD R4, g_m(g) // g.m = curm
304 MOVD g, m_g0(R4) // curm.g0 = g
305
306 // switch to crashstack
307 MOVD (g_stack+stack_hi)(g), R4
308 ADD $(-4*8), R4, R15
309
310 // call target function
311 MOVD 0(R12), R3 // code pointer
312 BL (R3)
313
314 // should never return
315 BL runtime·abort(SB)
316 UNDEF
317
318 /*
319 * support for morestack
320 */
321
322 // Called during function prolog when more stack is needed.
323 // Caller has already loaded:
324 // R3: framesize, R4: argsize, R5: LR
325 //
326 // The traceback routines see morestack on a g0 as being
327 // the top of a stack (for example, morestack calling newstack
328 // calling the scheduler calling newm calling gc), so we must
329 // record an argument size. For that purpose, it has no arguments.
330 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
331 // Called from f.
332 // Set g->sched to context in f.
333 MOVD R15, (g_sched+gobuf_sp)(g)
334 MOVD LR, R8
335 MOVD R8, (g_sched+gobuf_pc)(g)
336 MOVD R5, (g_sched+gobuf_lr)(g)
337 MOVD R12, (g_sched+gobuf_ctxt)(g)
338
339 // Cannot grow scheduler stack (m->g0).
340 MOVD g_m(g), R7
341 MOVD m_g0(R7), R8
342 CMPBNE g, R8, 3(PC)
343 BL runtime·badmorestackg0(SB)
344 BL runtime·abort(SB)
345
346 // Cannot grow signal stack (m->gsignal).
347 MOVD m_gsignal(R7), R8
348 CMP g, R8
349 BNE 3(PC)
350 BL runtime·badmorestackgsignal(SB)
351 BL runtime·abort(SB)
352
353 // Called from f.
354 // Set m->morebuf to f's caller.
355 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
356 MOVD R15, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
357 MOVD g, (m_morebuf+gobuf_g)(R7)
358
359 // Call newstack on m->g0's stack.
360 MOVD m_g0(R7), g
361 BL runtime·save_g(SB)
362 MOVD (g_sched+gobuf_sp)(g), R15
363 // Create a stack frame on g0 to call newstack.
364 MOVD $0, -8(R15) // Zero saved LR in frame
365 SUB $8, R15
366 BL runtime·newstack(SB)
367
368 // Not reached, but make sure the return PC from the call to newstack
369 // is still in this function, and not the beginning of the next.
370 UNDEF
371
372 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
373 // Force SPWRITE. This function doesn't actually write SP,
374 // but it is called with a special calling convention where
375 // the caller doesn't save LR on stack but passes it as a
376 // register (R5), and the unwinder currently doesn't understand.
377 // Make it SPWRITE to stop unwinding. (See issue 54332)
378 MOVD R15, R15
379
380 MOVD $0, R12
381 BR runtime·morestack(SB)
382
383 // reflectcall: call a function with the given argument list
384 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
385 // we don't have variable-sized frames, so we use a small number
386 // of constant-sized-frame functions to encode a few bits of size in the pc.
387 // Caution: ugly multiline assembly macros in your future!
388
389 #define DISPATCH(NAME,MAXSIZE) \
390 MOVD $MAXSIZE, R4; \
391 CMP R3, R4; \
392 BGT 3(PC); \
393 MOVD $NAME(SB), R5; \
394 BR (R5)
395 // Note: can't just "BR NAME(SB)" - bad inlining results.
396
397 TEXT ·reflectcall(SB), NOSPLIT, $-8-48
398 MOVWZ frameSize+32(FP), R3
399 DISPATCH(runtime·call16, 16)
400 DISPATCH(runtime·call32, 32)
401 DISPATCH(runtime·call64, 64)
402 DISPATCH(runtime·call128, 128)
403 DISPATCH(runtime·call256, 256)
404 DISPATCH(runtime·call512, 512)
405 DISPATCH(runtime·call1024, 1024)
406 DISPATCH(runtime·call2048, 2048)
407 DISPATCH(runtime·call4096, 4096)
408 DISPATCH(runtime·call8192, 8192)
409 DISPATCH(runtime·call16384, 16384)
410 DISPATCH(runtime·call32768, 32768)
411 DISPATCH(runtime·call65536, 65536)
412 DISPATCH(runtime·call131072, 131072)
413 DISPATCH(runtime·call262144, 262144)
414 DISPATCH(runtime·call524288, 524288)
415 DISPATCH(runtime·call1048576, 1048576)
416 DISPATCH(runtime·call2097152, 2097152)
417 DISPATCH(runtime·call4194304, 4194304)
418 DISPATCH(runtime·call8388608, 8388608)
419 DISPATCH(runtime·call16777216, 16777216)
420 DISPATCH(runtime·call33554432, 33554432)
421 DISPATCH(runtime·call67108864, 67108864)
422 DISPATCH(runtime·call134217728, 134217728)
423 DISPATCH(runtime·call268435456, 268435456)
424 DISPATCH(runtime·call536870912, 536870912)
425 DISPATCH(runtime·call1073741824, 1073741824)
426 MOVD $runtime·badreflectcall(SB), R5
427 BR (R5)
428
429 #define CALLFN(NAME,MAXSIZE) \
430 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
431 NO_LOCAL_POINTERS; \
432 /* copy arguments to stack */ \
433 MOVD stackArgs+16(FP), R4; \
434 MOVWZ stackArgsSize+24(FP), R5; \
435 MOVD $stack-MAXSIZE(SP), R6; \
436 loopArgs: /* copy 256 bytes at a time */ \
437 CMP R5, $256; \
438 BLT tailArgs; \
439 SUB $256, R5; \
440 MVC $256, 0(R4), 0(R6); \
441 MOVD $256(R4), R4; \
442 MOVD $256(R6), R6; \
443 BR loopArgs; \
444 tailArgs: /* copy remaining bytes */ \
445 CMP R5, $0; \
446 BEQ callFunction; \
447 SUB $1, R5; \
448 EXRL $callfnMVC<>(SB), R5; \
449 callFunction: \
450 MOVD f+8(FP), R12; \
451 MOVD (R12), R8; \
452 PCDATA $PCDATA_StackMapIndex, $0; \
453 BL (R8); \
454 /* copy return values back */ \
455 MOVD stackArgsType+0(FP), R7; \
456 MOVD stackArgs+16(FP), R6; \
457 MOVWZ stackArgsSize+24(FP), R5; \
458 MOVD $stack-MAXSIZE(SP), R4; \
459 MOVWZ stackRetOffset+28(FP), R1; \
460 ADD R1, R4; \
461 ADD R1, R6; \
462 SUB R1, R5; \
463 BL callRet<>(SB); \
464 RET
465
466 // callRet copies return values back at the end of call*. This is a
467 // separate function so it can allocate stack space for the arguments
468 // to reflectcallmove. It does not follow the Go ABI; it expects its
469 // arguments in registers.
470 TEXT callRet<>(SB), NOSPLIT, $40-0
471 MOVD R7, 8(R15)
472 MOVD R6, 16(R15)
473 MOVD R4, 24(R15)
474 MOVD R5, 32(R15)
475 MOVD $0, 40(R15)
476 BL runtime·reflectcallmove(SB)
477 RET
478
479 CALLFN(·call16, 16)
480 CALLFN(·call32, 32)
481 CALLFN(·call64, 64)
482 CALLFN(·call128, 128)
483 CALLFN(·call256, 256)
484 CALLFN(·call512, 512)
485 CALLFN(·call1024, 1024)
486 CALLFN(·call2048, 2048)
487 CALLFN(·call4096, 4096)
488 CALLFN(·call8192, 8192)
489 CALLFN(·call16384, 16384)
490 CALLFN(·call32768, 32768)
491 CALLFN(·call65536, 65536)
492 CALLFN(·call131072, 131072)
493 CALLFN(·call262144, 262144)
494 CALLFN(·call524288, 524288)
495 CALLFN(·call1048576, 1048576)
496 CALLFN(·call2097152, 2097152)
497 CALLFN(·call4194304, 4194304)
498 CALLFN(·call8388608, 8388608)
499 CALLFN(·call16777216, 16777216)
500 CALLFN(·call33554432, 33554432)
501 CALLFN(·call67108864, 67108864)
502 CALLFN(·call134217728, 134217728)
503 CALLFN(·call268435456, 268435456)
504 CALLFN(·call536870912, 536870912)
505 CALLFN(·call1073741824, 1073741824)
506
507 // Not a function: target for EXRL (execute relative long) instruction.
508 TEXT callfnMVC<>(SB),NOSPLIT|NOFRAME,$0-0
509 MVC $1, 0(R4), 0(R6)
510
511 TEXT runtime·procyield(SB),NOSPLIT,$0-0
512 RET
513
514 // Save state of caller into g->sched,
515 // but using fake PC from systemstack_switch.
516 // Must only be called from functions with no locals ($0)
517 // or else unwinding from systemstack_switch is incorrect.
518 // Smashes R1.
519 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
520 MOVD $runtime·systemstack_switch(SB), R1
521 ADD $16, R1 // get past prologue
522 MOVD R1, (g_sched+gobuf_pc)(g)
523 MOVD R15, (g_sched+gobuf_sp)(g)
524 MOVD $0, (g_sched+gobuf_lr)(g)
525 MOVD $0, (g_sched+gobuf_ret)(g)
526 // Assert ctxt is zero. See func save.
527 MOVD (g_sched+gobuf_ctxt)(g), R1
528 CMPBEQ R1, $0, 2(PC)
529 BL runtime·abort(SB)
530 RET
531
532 // func asmcgocall(fn, arg unsafe.Pointer) int32
533 // Call fn(arg) on the scheduler stack,
534 // aligned appropriately for the gcc ABI.
535 // See cgocall.go for more details.
536 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
537 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
538 // C TLS base pointer in AR0:AR1
539 MOVD fn+0(FP), R3
540 MOVD arg+8(FP), R4
541
542 MOVD R15, R2 // save original stack pointer
543 MOVD g, R5
544
545 // Figure out if we need to switch to m->g0 stack.
546 // We get called to create new OS threads too, and those
547 // come in on the m->g0 stack already. Or we might already
548 // be on the m->gsignal stack.
549 MOVD g_m(g), R6
550 MOVD m_gsignal(R6), R7
551 CMPBEQ R7, g, g0
552 MOVD m_g0(R6), R7
553 CMPBEQ R7, g, g0
554 BL gosave_systemstack_switch<>(SB)
555 MOVD R7, g
556 BL runtime·save_g(SB)
557 MOVD (g_sched+gobuf_sp)(g), R15
558
559 // Now on a scheduling stack (a pthread-created stack).
560 g0:
561 // Save room for two of our pointers, plus 160 bytes of callee
562 // save area that lives on the caller stack.
563 SUB $176, R15
564 MOVD $~7, R6
565 AND R6, R15 // 8-byte alignment for gcc ABI
566 MOVD R5, 168(R15) // save old g on stack
567 MOVD (g_stack+stack_hi)(R5), R5
568 SUB R2, R5
569 MOVD R5, 160(R15) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
570 MOVD $0, 0(R15) // clear back chain pointer (TODO can we give it real back trace information?)
571 MOVD R4, R2 // arg in R2
572 BL R3 // can clobber: R0-R5, R14, F0-F3, F5, F7-F15
573
574 XOR R0, R0 // set R0 back to 0.
575 // Restore g, stack pointer.
576 MOVD 168(R15), g
577 BL runtime·save_g(SB)
578 MOVD (g_stack+stack_hi)(g), R5
579 MOVD 160(R15), R6
580 SUB R6, R5
581 MOVD R5, R15
582
583 MOVW R2, ret+16(FP)
584 RET
585
586 // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
587 // See cgocall.go for more details.
588 TEXT ·cgocallback(SB),NOSPLIT,$24-24
589 NO_LOCAL_POINTERS
590
591 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
592 // It is used to dropm while thread is exiting.
593 MOVD fn+0(FP), R1
594 CMPBNE R1, $0, loadg
595 // Restore the g from frame.
596 MOVD frame+8(FP), g
597 BR dropm
598
599 loadg:
600 // Load m and g from thread-local storage.
601 MOVB runtime·iscgo(SB), R3
602 CMPBEQ R3, $0, nocgo
603 BL runtime·load_g(SB)
604
605 nocgo:
606 // If g is nil, Go did not create the current thread,
607 // or if this thread never called into Go on pthread platforms.
608 // Call needm to obtain one for temporary use.
609 // In this case, we're running on the thread stack, so there's
610 // lots of space, but the linker doesn't know. Hide the call from
611 // the linker analysis by using an indirect call.
612 CMPBEQ g, $0, needm
613
614 MOVD g_m(g), R8
615 MOVD R8, savedm-8(SP)
616 BR havem
617
618 needm:
619 MOVD g, savedm-8(SP) // g is zero, so is m.
620 MOVD $runtime·needAndBindM(SB), R3
621 BL (R3)
622
623 // Set m->sched.sp = SP, so that if a panic happens
624 // during the function we are about to execute, it will
625 // have a valid SP to run on the g0 stack.
626 // The next few lines (after the havem label)
627 // will save this SP onto the stack and then write
628 // the same SP back to m->sched.sp. That seems redundant,
629 // but if an unrecovered panic happens, unwindm will
630 // restore the g->sched.sp from the stack location
631 // and then systemstack will try to use it. If we don't set it here,
632 // that restored SP will be uninitialized (typically 0) and
633 // will not be usable.
634 MOVD g_m(g), R8
635 MOVD m_g0(R8), R3
636 MOVD R15, (g_sched+gobuf_sp)(R3)
637
638 havem:
639 // Now there's a valid m, and we're running on its m->g0.
640 // Save current m->g0->sched.sp on stack and then set it to SP.
641 // Save current sp in m->g0->sched.sp in preparation for
642 // switch back to m->curg stack.
643 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
644 MOVD m_g0(R8), R3
645 MOVD (g_sched+gobuf_sp)(R3), R4
646 MOVD R4, savedsp-24(SP) // must match frame size
647 MOVD R15, (g_sched+gobuf_sp)(R3)
648
649 // Switch to m->curg stack and call runtime.cgocallbackg.
650 // Because we are taking over the execution of m->curg
651 // but *not* resuming what had been running, we need to
652 // save that information (m->curg->sched) so we can restore it.
653 // We can restore m->curg->sched.sp easily, because calling
654 // runtime.cgocallbackg leaves SP unchanged upon return.
655 // To save m->curg->sched.pc, we push it onto the curg stack and
656 // open a frame the same size as cgocallback's g0 frame.
657 // Once we switch to the curg stack, the pushed PC will appear
658 // to be the return PC of cgocallback, so that the traceback
659 // will seamlessly trace back into the earlier calls.
660 MOVD m_curg(R8), g
661 BL runtime·save_g(SB)
662 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
663 MOVD (g_sched+gobuf_pc)(g), R5
664 MOVD R5, -(24+8)(R4) // "saved LR"; must match frame size
665 // Gather our arguments into registers.
666 MOVD fn+0(FP), R1
667 MOVD frame+8(FP), R2
668 MOVD ctxt+16(FP), R3
669 MOVD $-(24+8)(R4), R15 // switch stack; must match frame size
670 MOVD R1, 8(R15)
671 MOVD R2, 16(R15)
672 MOVD R3, 24(R15)
673 BL runtime·cgocallbackg(SB)
674
675 // Restore g->sched (== m->curg->sched) from saved values.
676 MOVD 0(R15), R5
677 MOVD R5, (g_sched+gobuf_pc)(g)
678 MOVD $(24+8)(R15), R4 // must match frame size
679 MOVD R4, (g_sched+gobuf_sp)(g)
680
681 // Switch back to m->g0's stack and restore m->g0->sched.sp.
682 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
683 // so we do not have to restore it.)
684 MOVD g_m(g), R8
685 MOVD m_g0(R8), g
686 BL runtime·save_g(SB)
687 MOVD (g_sched+gobuf_sp)(g), R15
688 MOVD savedsp-24(SP), R4 // must match frame size
689 MOVD R4, (g_sched+gobuf_sp)(g)
690
691 // If the m on entry was nil, we called needm above to borrow an m,
692 // 1. for the duration of the call on non-pthread platforms,
693 // 2. or the duration of the C thread alive on pthread platforms.
694 // If the m on entry wasn't nil,
695 // 1. the thread might be a Go thread,
696 // 2. or it wasn't the first call from a C thread on pthread platforms,
697 // since then we skip dropm to reuse the m in the first call.
698 MOVD savedm-8(SP), R6
699 CMPBNE R6, $0, droppedm
700
701 // Skip dropm to reuse it in the next call, when a pthread key has been created.
702 MOVD _cgo_pthread_key_created(SB), R6
703 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
704 CMPBEQ R6, $0, dropm
705 MOVD (R6), R6
706 CMPBNE R6, $0, droppedm
707
708 dropm:
709 MOVD $runtime·dropm(SB), R3
710 BL (R3)
711 droppedm:
712
713 // Done!
714 RET
715
716 // void setg(G*); set g. for use by needm.
717 TEXT runtime·setg(SB), NOSPLIT, $0-8
718 MOVD gg+0(FP), g
719 // This only happens if iscgo, so jump straight to save_g
720 BL runtime·save_g(SB)
721 RET
722
723 // void setg_gcc(G*); set g in C TLS.
724 // Must obey the gcc calling convention.
725 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
726 // The standard prologue clobbers LR (R14), which is callee-save in
727 // the C ABI, so we have to use NOFRAME and save LR ourselves.
728 MOVD LR, R1
729 // Also save g, R10, and R11 since they're callee-save in C ABI
730 MOVD R10, R3
731 MOVD g, R4
732 MOVD R11, R5
733
734 MOVD R2, g
735 BL runtime·save_g(SB)
736
737 MOVD R5, R11
738 MOVD R4, g
739 MOVD R3, R10
740 MOVD R1, LR
741 RET
742
743 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
744 MOVW (R0), R0
745 UNDEF
746
747 // int64 runtime·cputicks(void)
748 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
749 // The TOD clock on s390 counts from the year 1900 in ~250ps intervals.
750 // This means that since about 1972 the msb has been set, making the
751 // result of a call to STORE CLOCK (stck) a negative number.
752 // We clear the msb to make it positive.
753 STCK ret+0(FP) // serialises before and after call
754 MOVD ret+0(FP), R3 // R3 will wrap to 0 in the year 2043
755 SLD $1, R3
756 SRD $1, R3
757 MOVD R3, ret+0(FP)
758 RET
759
760 // AES hashing not implemented for s390x
761 TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32
762 JMP runtime·memhashFallback(SB)
763 TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24
764 JMP runtime·strhashFallback(SB)
765 TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24
766 JMP runtime·memhash32Fallback(SB)
767 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24
768 JMP runtime·memhash64Fallback(SB)
769
770 TEXT runtime·return0(SB), NOSPLIT, $0
771 MOVW $0, R3
772 RET
773
774 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
775 // Must obey the gcc calling convention.
776 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
777 // g (R13), R10, R11 and LR (R14) are callee-save in the C ABI, so save them
778 MOVD g, R1
779 MOVD R10, R3
780 MOVD LR, R4
781 MOVD R11, R5
782
783 BL runtime·load_g(SB) // clobbers g (R13), R10, R11
784 MOVD g_m(g), R2
785 MOVD m_curg(R2), R2
786 MOVD (g_stack+stack_hi)(R2), R2
787
788 MOVD R1, g
789 MOVD R3, R10
790 MOVD R4, LR
791 MOVD R5, R11
792 RET
793
794 // The top-most function running on a goroutine
795 // returns to goexit+PCQuantum.
796 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
797 BYTE $0x07; BYTE $0x00; // 2-byte nop
798 BL runtime·goexit1(SB) // does not return
799 // traceback from goexit1 must hit code range of goexit
800 BYTE $0x07; BYTE $0x00; // 2-byte nop
801
802 TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
803 // Stores are already ordered on s390x, so this is just a
804 // compile barrier.
805 RET
806
807 // This is called from .init_array and follows the platform, not Go, ABI.
808 // We are overly conservative. We could only save the registers we use.
809 // However, since this function is only called once per loaded module
810 // performance is unimportant.
811 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
812 // Save R6-R15 in the register save area of the calling function.
813 // Don't bother saving F8-F15 as we aren't doing any calls.
814 STMG R6, R15, 48(R15)
815
816 // append the argument (passed in R2, as per the ELF ABI) to the
817 // moduledata linked list.
818 MOVD runtime·lastmoduledatap(SB), R1
819 MOVD R2, moduledata_next(R1)
820 MOVD R2, runtime·lastmoduledatap(SB)
821
822 // Restore R6-R15.
823 LMG 48(R15), R6, R15
824 RET
825
826 TEXT ·checkASM(SB),NOSPLIT,$0-1
827 MOVB $1, ret+0(FP)
828 RET
829
830 // gcWriteBarrier informs the GC about heap pointer writes.
831 //
832 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
833 // number of bytes of buffer needed in R9, and returns a pointer
834 // to the buffer space in R9.
835 // It clobbers R10 (the temp register) and R1 (used by PLT stub).
836 // It does not clobber any other general-purpose registers,
837 // but may clobber others (e.g., floating point registers).
838 TEXT gcWriteBarrier<>(SB),NOSPLIT,$96
839 // Save the registers clobbered by the fast path.
840 MOVD R4, 96(R15)
841 retry:
842 MOVD g_m(g), R1
843 MOVD m_p(R1), R1
844 // Increment wbBuf.next position.
845 MOVD R9, R4
846 ADD (p_wbBuf+wbBuf_next)(R1), R4
847 // Is the buffer full?
848 MOVD (p_wbBuf+wbBuf_end)(R1), R10
849 CMPUBGT R4, R10, flush
850 // Commit to the larger buffer.
851 MOVD R4, (p_wbBuf+wbBuf_next)(R1)
852 // Make return value (the original next position)
853 SUB R9, R4, R9
854 // Restore registers.
855 MOVD 96(R15), R4
856 RET
857
858 flush:
859 // Save all general purpose registers since these could be
860 // clobbered by wbBufFlush and were not saved by the caller.
861 STMG R2, R3, 8(R15)
862 MOVD R0, 24(R15)
863 // R1 already saved.
864 // R4 already saved.
865 STMG R5, R12, 32(R15) // save R5 - R12
866 // R13 is g.
867 // R14 is LR.
868 // R15 is SP.
869
870 CALL runtime·wbBufFlush(SB)
871
872 LMG 8(R15), R2, R3 // restore R2 - R3
873 MOVD 24(R15), R0 // restore R0
874 LMG 32(R15), R5, R12 // restore R5 - R12
875 JMP retry
876
877 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
878 MOVD $8, R9
879 JMP gcWriteBarrier<>(SB)
880 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
881 MOVD $16, R9
882 JMP gcWriteBarrier<>(SB)
883 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
884 MOVD $24, R9
885 JMP gcWriteBarrier<>(SB)
886 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
887 MOVD $32, R9
888 JMP gcWriteBarrier<>(SB)
889 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
890 MOVD $40, R9
891 JMP gcWriteBarrier<>(SB)
892 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
893 MOVD $48, R9
894 JMP gcWriteBarrier<>(SB)
895 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
896 MOVD $56, R9
897 JMP gcWriteBarrier<>(SB)
898 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
899 MOVD $64, R9
900 JMP gcWriteBarrier<>(SB)
901
902 // Note: these functions use a special calling convention to save generated code space.
903 // Arguments are passed in registers, but the space for those arguments are allocated
904 // in the caller's stack frame. These stubs write the args into that stack space and
905 // then tail call to the corresponding runtime handler.
906 // The tail call makes these stubs disappear in backtraces.
907 TEXT runtime·panicIndex(SB),NOSPLIT,$0-16
908 MOVD R0, x+0(FP)
909 MOVD R1, y+8(FP)
910 JMP runtime·goPanicIndex(SB)
911 TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16
912 MOVD R0, x+0(FP)
913 MOVD R1, y+8(FP)
914 JMP runtime·goPanicIndexU(SB)
915 TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16
916 MOVD R1, x+0(FP)
917 MOVD R2, y+8(FP)
918 JMP runtime·goPanicSliceAlen(SB)
919 TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16
920 MOVD R1, x+0(FP)
921 MOVD R2, y+8(FP)
922 JMP runtime·goPanicSliceAlenU(SB)
923 TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16
924 MOVD R1, x+0(FP)
925 MOVD R2, y+8(FP)
926 JMP runtime·goPanicSliceAcap(SB)
927 TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16
928 MOVD R1, x+0(FP)
929 MOVD R2, y+8(FP)
930 JMP runtime·goPanicSliceAcapU(SB)
931 TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16
932 MOVD R0, x+0(FP)
933 MOVD R1, y+8(FP)
934 JMP runtime·goPanicSliceB(SB)
935 TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16
936 MOVD R0, x+0(FP)
937 MOVD R1, y+8(FP)
938 JMP runtime·goPanicSliceBU(SB)
939 TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16
940 MOVD R2, x+0(FP)
941 MOVD R3, y+8(FP)
942 JMP runtime·goPanicSlice3Alen(SB)
943 TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16
944 MOVD R2, x+0(FP)
945 MOVD R3, y+8(FP)
946 JMP runtime·goPanicSlice3AlenU(SB)
947 TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16
948 MOVD R2, x+0(FP)
949 MOVD R3, y+8(FP)
950 JMP runtime·goPanicSlice3Acap(SB)
951 TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16
952 MOVD R2, x+0(FP)
953 MOVD R3, y+8(FP)
954 JMP runtime·goPanicSlice3AcapU(SB)
955 TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16
956 MOVD R1, x+0(FP)
957 MOVD R2, y+8(FP)
958 JMP runtime·goPanicSlice3B(SB)
959 TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16
960 MOVD R1, x+0(FP)
961 MOVD R2, y+8(FP)
962 JMP runtime·goPanicSlice3BU(SB)
963 TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16
964 MOVD R0, x+0(FP)
965 MOVD R1, y+8(FP)
966 JMP runtime·goPanicSlice3C(SB)
967 TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16
968 MOVD R0, x+0(FP)
969 MOVD R1, y+8(FP)
970 JMP runtime·goPanicSlice3CU(SB)
971 TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16
972 MOVD R2, x+0(FP)
973 MOVD R3, y+8(FP)
974 JMP runtime·goPanicSliceConvert(SB)
975
View as plain text