Text file
src/runtime/asm_loong64.s
1 // Copyright 2022 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "funcdata.h"
8 #include "textflag.h"
9
10 #define REGCTXT R29
11
12 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
13 // R3 = stack; R4 = argc; R5 = argv
14
15 ADDV $-24, R3
16 MOVW R4, 8(R3) // argc
17 MOVV R5, 16(R3) // argv
18
19 // create istack out of the given (operating system) stack.
20 // _cgo_init may update stackguard.
21 MOVV $runtime·g0(SB), g
22 MOVV $(-64*1024), R30
23 ADDV R30, R3, R19
24 MOVV R19, g_stackguard0(g)
25 MOVV R19, g_stackguard1(g)
26 MOVV R19, (g_stack+stack_lo)(g)
27 MOVV R3, (g_stack+stack_hi)(g)
28
29 // if there is a _cgo_init, call it using the gcc ABI.
30 MOVV _cgo_init(SB), R25
31 BEQ R25, nocgo
32
33 MOVV R0, R7 // arg 3: not used
34 MOVV R0, R6 // arg 2: not used
35 MOVV $setg_gcc<>(SB), R5 // arg 1: setg
36 MOVV g, R4 // arg 0: G
37 JAL (R25)
38
39 nocgo:
40 // update stackguard after _cgo_init
41 MOVV (g_stack+stack_lo)(g), R19
42 ADDV $const_stackGuard, R19
43 MOVV R19, g_stackguard0(g)
44 MOVV R19, g_stackguard1(g)
45
46 // set the per-goroutine and per-mach "registers"
47 MOVV $runtime·m0(SB), R19
48
49 // save m->g0 = g0
50 MOVV g, m_g0(R19)
51 // save m0 to g0->m
52 MOVV R19, g_m(g)
53
54 JAL runtime·check(SB)
55
56 // args are already prepared
57 JAL runtime·args(SB)
58 JAL runtime·osinit(SB)
59 JAL runtime·schedinit(SB)
60
61 // create a new goroutine to start program
62 MOVV $runtime·mainPC(SB), R19 // entry
63 ADDV $-16, R3
64 MOVV R19, 8(R3)
65 MOVV R0, 0(R3)
66 JAL runtime·newproc(SB)
67 ADDV $16, R3
68
69 // start this M
70 JAL runtime·mstart(SB)
71
72 MOVV R0, 1(R0)
73 RET
74
75 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
76 GLOBL runtime·mainPC(SB),RODATA,$8
77
78 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
79 BREAK
80 RET
81
82 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
83 RET
84
85 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
86 JAL runtime·mstart0(SB)
87 RET // not reached
88
89 // func cputicks() int64
90 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
91 RDTIMED R0, R4
92 MOVV R4, ret+0(FP)
93 RET
94
95 /*
96 * go-routine
97 */
98
99 // void gogo(Gobuf*)
100 // restore state from Gobuf; longjmp
101 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
102 MOVV buf+0(FP), R4
103 MOVV gobuf_g(R4), R5
104 MOVV 0(R5), R0 // make sure g != nil
105 JMP gogo<>(SB)
106
107 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
108 MOVV R5, g
109 JAL runtime·save_g(SB)
110
111 MOVV gobuf_sp(R4), R3
112 MOVV gobuf_lr(R4), R1
113 MOVV gobuf_ret(R4), R19
114 MOVV gobuf_ctxt(R4), REGCTXT
115 MOVV R0, gobuf_sp(R4)
116 MOVV R0, gobuf_ret(R4)
117 MOVV R0, gobuf_lr(R4)
118 MOVV R0, gobuf_ctxt(R4)
119 MOVV gobuf_pc(R4), R6
120 JMP (R6)
121
122 // void mcall(fn func(*g))
123 // Switch to m->g0's stack, call fn(g).
124 // Fn must never return. It should gogo(&g->sched)
125 // to keep running g.
126 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
127 #ifdef GOEXPERIMENT_regabiargs
128 MOVV R4, REGCTXT
129 #else
130 MOVV fn+0(FP), REGCTXT
131 #endif
132
133 // Save caller state in g->sched
134 MOVV R3, (g_sched+gobuf_sp)(g)
135 MOVV R1, (g_sched+gobuf_pc)(g)
136 MOVV R0, (g_sched+gobuf_lr)(g)
137
138 // Switch to m->g0 & its stack, call fn.
139 MOVV g, R4 // arg = g
140 MOVV g_m(g), R20
141 MOVV m_g0(R20), g
142 JAL runtime·save_g(SB)
143 BNE g, R4, 2(PC)
144 JMP runtime·badmcall(SB)
145 MOVV 0(REGCTXT), R20 // code pointer
146 MOVV (g_sched+gobuf_sp)(g), R3 // sp = m->g0->sched.sp
147 ADDV $-16, R3
148 MOVV R4, 8(R3)
149 MOVV R0, 0(R3)
150 JAL (R20)
151 JMP runtime·badmcall2(SB)
152
153 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
154 // of the G stack. We need to distinguish the routine that
155 // lives at the bottom of the G stack from the one that lives
156 // at the top of the system stack because the one at the top of
157 // the system stack terminates the stack walk (see topofstack()).
158 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
159 UNDEF
160 JAL (R1) // make sure this function is not leaf
161 RET
162
163 // func systemstack(fn func())
164 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
165 MOVV fn+0(FP), R19 // R19 = fn
166 MOVV R19, REGCTXT // context
167 MOVV g_m(g), R4 // R4 = m
168
169 MOVV m_gsignal(R4), R5 // R5 = gsignal
170 BEQ g, R5, noswitch
171
172 MOVV m_g0(R4), R5 // R5 = g0
173 BEQ g, R5, noswitch
174
175 MOVV m_curg(R4), R6
176 BEQ g, R6, switch
177
178 // Bad: g is not gsignal, not g0, not curg. What is it?
179 // Hide call from linker nosplit analysis.
180 MOVV $runtime·badsystemstack(SB), R7
181 JAL (R7)
182 JAL runtime·abort(SB)
183
184 switch:
185 // save our state in g->sched. Pretend to
186 // be systemstack_switch if the G stack is scanned.
187 JAL gosave_systemstack_switch<>(SB)
188
189 // switch to g0
190 MOVV R5, g
191 JAL runtime·save_g(SB)
192 MOVV (g_sched+gobuf_sp)(g), R19
193 MOVV R19, R3
194
195 // call target function
196 MOVV 0(REGCTXT), R6 // code pointer
197 JAL (R6)
198
199 // switch back to g
200 MOVV g_m(g), R4
201 MOVV m_curg(R4), g
202 JAL runtime·save_g(SB)
203 MOVV (g_sched+gobuf_sp)(g), R3
204 MOVV R0, (g_sched+gobuf_sp)(g)
205 RET
206
207 noswitch:
208 // already on m stack, just call directly
209 // Using a tail call here cleans up tracebacks since we won't stop
210 // at an intermediate systemstack.
211 MOVV 0(REGCTXT), R4 // code pointer
212 MOVV 0(R3), R1 // restore LR
213 ADDV $8, R3
214 JMP (R4)
215
216 // func switchToCrashStack0(fn func())
217 TEXT runtime·switchToCrashStack0(SB), NOSPLIT, $0-8
218 MOVV fn+0(FP), REGCTXT // context register
219 MOVV g_m(g), R4 // curm
220
221 // set g to gcrash
222 MOVV $runtime·gcrash(SB), g // g = &gcrash
223 JAL runtime·save_g(SB)
224 MOVV R4, g_m(g) // g.m = curm
225 MOVV g, m_g0(R4) // curm.g0 = g
226
227 // switch to crashstack
228 MOVV (g_stack+stack_hi)(g), R4
229 ADDV $(-4*8), R4, R3
230
231 // call target function
232 MOVV 0(REGCTXT), R6
233 JAL (R6)
234
235 // should never return
236 JAL runtime·abort(SB)
237 UNDEF
238
239 /*
240 * support for morestack
241 */
242
243 // Called during function prolog when more stack is needed.
244 // Caller has already loaded:
245 // loong64: R31: LR
246 //
247 // The traceback routines see morestack on a g0 as being
248 // the top of a stack (for example, morestack calling newstack
249 // calling the scheduler calling newm calling gc), so we must
250 // record an argument size. For that purpose, it has no arguments.
251 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
252 // Called from f.
253 // Set g->sched to context in f.
254 MOVV R3, (g_sched+gobuf_sp)(g)
255 MOVV R1, (g_sched+gobuf_pc)(g)
256 MOVV R31, (g_sched+gobuf_lr)(g)
257 MOVV REGCTXT, (g_sched+gobuf_ctxt)(g)
258
259 // Cannot grow scheduler stack (m->g0).
260 MOVV g_m(g), R7
261 MOVV m_g0(R7), R8
262 BNE g, R8, 3(PC)
263 JAL runtime·badmorestackg0(SB)
264 JAL runtime·abort(SB)
265
266 // Cannot grow signal stack (m->gsignal).
267 MOVV m_gsignal(R7), R8
268 BNE g, R8, 3(PC)
269 JAL runtime·badmorestackgsignal(SB)
270 JAL runtime·abort(SB)
271
272 // Called from f.
273 // Set m->morebuf to f's caller.
274 MOVV R31, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
275 MOVV R3, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
276 MOVV g, (m_morebuf+gobuf_g)(R7)
277
278 // Call newstack on m->g0's stack.
279 MOVV m_g0(R7), g
280 JAL runtime·save_g(SB)
281 MOVV (g_sched+gobuf_sp)(g), R3
282 // Create a stack frame on g0 to call newstack.
283 MOVV R0, -8(R3) // Zero saved LR in frame
284 ADDV $-8, R3
285 JAL runtime·newstack(SB)
286
287 // Not reached, but make sure the return PC from the call to newstack
288 // is still in this function, and not the beginning of the next.
289 UNDEF
290
291 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
292 // Force SPWRITE. This function doesn't actually write SP,
293 // but it is called with a special calling convention where
294 // the caller doesn't save LR on stack but passes it as a
295 // register (R5), and the unwinder currently doesn't understand.
296 // Make it SPWRITE to stop unwinding. (See issue 54332)
297 MOVV R3, R3
298
299 MOVV R0, REGCTXT
300 JMP runtime·morestack(SB)
301
302 // reflectcall: call a function with the given argument list
303 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
304 // we don't have variable-sized frames, so we use a small number
305 // of constant-sized-frame functions to encode a few bits of size in the pc.
306 // Caution: ugly multiline assembly macros in your future!
307
308 #define DISPATCH(NAME,MAXSIZE) \
309 MOVV $MAXSIZE, R30; \
310 SGTU R19, R30, R30; \
311 BNE R30, 3(PC); \
312 MOVV $NAME(SB), R4; \
313 JMP (R4)
314 // Note: can't just "BR NAME(SB)" - bad inlining results.
315
316 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
317 MOVWU frameSize+32(FP), R19
318 DISPATCH(runtime·call32, 32)
319 DISPATCH(runtime·call64, 64)
320 DISPATCH(runtime·call128, 128)
321 DISPATCH(runtime·call256, 256)
322 DISPATCH(runtime·call512, 512)
323 DISPATCH(runtime·call1024, 1024)
324 DISPATCH(runtime·call2048, 2048)
325 DISPATCH(runtime·call4096, 4096)
326 DISPATCH(runtime·call8192, 8192)
327 DISPATCH(runtime·call16384, 16384)
328 DISPATCH(runtime·call32768, 32768)
329 DISPATCH(runtime·call65536, 65536)
330 DISPATCH(runtime·call131072, 131072)
331 DISPATCH(runtime·call262144, 262144)
332 DISPATCH(runtime·call524288, 524288)
333 DISPATCH(runtime·call1048576, 1048576)
334 DISPATCH(runtime·call2097152, 2097152)
335 DISPATCH(runtime·call4194304, 4194304)
336 DISPATCH(runtime·call8388608, 8388608)
337 DISPATCH(runtime·call16777216, 16777216)
338 DISPATCH(runtime·call33554432, 33554432)
339 DISPATCH(runtime·call67108864, 67108864)
340 DISPATCH(runtime·call134217728, 134217728)
341 DISPATCH(runtime·call268435456, 268435456)
342 DISPATCH(runtime·call536870912, 536870912)
343 DISPATCH(runtime·call1073741824, 1073741824)
344 MOVV $runtime·badreflectcall(SB), R4
345 JMP (R4)
346
347 #define CALLFN(NAME,MAXSIZE) \
348 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
349 NO_LOCAL_POINTERS; \
350 /* copy arguments to stack */ \
351 MOVV arg+16(FP), R4; \
352 MOVWU argsize+24(FP), R5; \
353 MOVV R3, R12; \
354 ADDV $8, R12; \
355 ADDV R12, R5; \
356 BEQ R12, R5, 6(PC); \
357 MOVBU (R4), R6; \
358 ADDV $1, R4; \
359 MOVBU R6, (R12); \
360 ADDV $1, R12; \
361 JMP -5(PC); \
362 /* set up argument registers */ \
363 MOVV regArgs+40(FP), R25; \
364 JAL ·unspillArgs(SB); \
365 /* call function */ \
366 MOVV f+8(FP), REGCTXT; \
367 MOVV (REGCTXT), R25; \
368 PCDATA $PCDATA_StackMapIndex, $0; \
369 JAL (R25); \
370 /* copy return values back */ \
371 MOVV regArgs+40(FP), R25; \
372 JAL ·spillArgs(SB); \
373 MOVV argtype+0(FP), R7; \
374 MOVV arg+16(FP), R4; \
375 MOVWU n+24(FP), R5; \
376 MOVWU retoffset+28(FP), R6; \
377 ADDV $8, R3, R12; \
378 ADDV R6, R12; \
379 ADDV R6, R4; \
380 SUBVU R6, R5; \
381 JAL callRet<>(SB); \
382 RET
383
384 // callRet copies return values back at the end of call*. This is a
385 // separate function so it can allocate stack space for the arguments
386 // to reflectcallmove. It does not follow the Go ABI; it expects its
387 // arguments in registers.
388 TEXT callRet<>(SB), NOSPLIT, $40-0
389 NO_LOCAL_POINTERS
390 MOVV R7, 8(R3)
391 MOVV R4, 16(R3)
392 MOVV R12, 24(R3)
393 MOVV R5, 32(R3)
394 MOVV R25, 40(R3)
395 JAL runtime·reflectcallmove(SB)
396 RET
397
398 CALLFN(·call16, 16)
399 CALLFN(·call32, 32)
400 CALLFN(·call64, 64)
401 CALLFN(·call128, 128)
402 CALLFN(·call256, 256)
403 CALLFN(·call512, 512)
404 CALLFN(·call1024, 1024)
405 CALLFN(·call2048, 2048)
406 CALLFN(·call4096, 4096)
407 CALLFN(·call8192, 8192)
408 CALLFN(·call16384, 16384)
409 CALLFN(·call32768, 32768)
410 CALLFN(·call65536, 65536)
411 CALLFN(·call131072, 131072)
412 CALLFN(·call262144, 262144)
413 CALLFN(·call524288, 524288)
414 CALLFN(·call1048576, 1048576)
415 CALLFN(·call2097152, 2097152)
416 CALLFN(·call4194304, 4194304)
417 CALLFN(·call8388608, 8388608)
418 CALLFN(·call16777216, 16777216)
419 CALLFN(·call33554432, 33554432)
420 CALLFN(·call67108864, 67108864)
421 CALLFN(·call134217728, 134217728)
422 CALLFN(·call268435456, 268435456)
423 CALLFN(·call536870912, 536870912)
424 CALLFN(·call1073741824, 1073741824)
425
426 TEXT runtime·procyield(SB),NOSPLIT,$0-0
427 RET
428
429 // Save state of caller into g->sched.
430 // but using fake PC from systemstack_switch.
431 // Must only be called from functions with no locals ($0)
432 // or else unwinding from systemstack_switch is incorrect.
433 // Smashes R19.
434 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
435 MOVV $runtime·systemstack_switch(SB), R19
436 ADDV $8, R19
437 MOVV R19, (g_sched+gobuf_pc)(g)
438 MOVV R3, (g_sched+gobuf_sp)(g)
439 MOVV R0, (g_sched+gobuf_lr)(g)
440 MOVV R0, (g_sched+gobuf_ret)(g)
441 // Assert ctxt is zero. See func save.
442 MOVV (g_sched+gobuf_ctxt)(g), R19
443 BEQ R19, 2(PC)
444 JAL runtime·abort(SB)
445 RET
446
447 // func asmcgocall(fn, arg unsafe.Pointer) int32
448 // Call fn(arg) on the scheduler stack,
449 // aligned appropriately for the gcc ABI.
450 // See cgocall.go for more details.
451 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
452 MOVV fn+0(FP), R25
453 MOVV arg+8(FP), R4
454
455 MOVV R3, R12 // save original stack pointer
456 MOVV g, R13
457
458 // Figure out if we need to switch to m->g0 stack.
459 // We get called to create new OS threads too, and those
460 // come in on the m->g0 stack already.
461 MOVV g_m(g), R5
462 MOVV m_gsignal(R5), R6
463 BEQ R6, g, g0
464 MOVV m_g0(R5), R6
465 BEQ R6, g, g0
466
467 JAL gosave_systemstack_switch<>(SB)
468 MOVV R6, g
469 JAL runtime·save_g(SB)
470 MOVV (g_sched+gobuf_sp)(g), R3
471
472 // Now on a scheduling stack (a pthread-created stack).
473 g0:
474 // Save room for two of our pointers.
475 ADDV $-16, R3
476 MOVV R13, 0(R3) // save old g on stack
477 MOVV (g_stack+stack_hi)(R13), R13
478 SUBVU R12, R13
479 MOVV R13, 8(R3) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
480 JAL (R25)
481
482 // Restore g, stack pointer. R4 is return value.
483 MOVV 0(R3), g
484 JAL runtime·save_g(SB)
485 MOVV (g_stack+stack_hi)(g), R5
486 MOVV 8(R3), R6
487 SUBVU R6, R5
488 MOVV R5, R3
489
490 MOVW R4, ret+16(FP)
491 RET
492
493 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
494 // See cgocall.go for more details.
495 TEXT ·cgocallback(SB),NOSPLIT,$24-24
496 NO_LOCAL_POINTERS
497
498 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
499 // It is used to dropm while thread is exiting.
500 MOVV fn+0(FP), R5
501 BNE R5, loadg
502 // Restore the g from frame.
503 MOVV frame+8(FP), g
504 JMP dropm
505
506 loadg:
507 // Load m and g from thread-local storage.
508 MOVB runtime·iscgo(SB), R19
509 BEQ R19, nocgo
510 JAL runtime·load_g(SB)
511 nocgo:
512
513 // If g is nil, Go did not create the current thread,
514 // or if this thread never called into Go on pthread platforms.
515 // Call needm to obtain one for temporary use.
516 // In this case, we're running on the thread stack, so there's
517 // lots of space, but the linker doesn't know. Hide the call from
518 // the linker analysis by using an indirect call.
519 BEQ g, needm
520
521 MOVV g_m(g), R12
522 MOVV R12, savedm-8(SP)
523 JMP havem
524
525 needm:
526 MOVV g, savedm-8(SP) // g is zero, so is m.
527 MOVV $runtime·needAndBindM(SB), R4
528 JAL (R4)
529
530 // Set m->sched.sp = SP, so that if a panic happens
531 // during the function we are about to execute, it will
532 // have a valid SP to run on the g0 stack.
533 // The next few lines (after the havem label)
534 // will save this SP onto the stack and then write
535 // the same SP back to m->sched.sp. That seems redundant,
536 // but if an unrecovered panic happens, unwindm will
537 // restore the g->sched.sp from the stack location
538 // and then systemstack will try to use it. If we don't set it here,
539 // that restored SP will be uninitialized (typically 0) and
540 // will not be usable.
541 MOVV g_m(g), R12
542 MOVV m_g0(R12), R19
543 MOVV R3, (g_sched+gobuf_sp)(R19)
544
545 havem:
546 // Now there's a valid m, and we're running on its m->g0.
547 // Save current m->g0->sched.sp on stack and then set it to SP.
548 // Save current sp in m->g0->sched.sp in preparation for
549 // switch back to m->curg stack.
550 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R29) aka savedsp-16(SP).
551 MOVV m_g0(R12), R19
552 MOVV (g_sched+gobuf_sp)(R19), R13
553 MOVV R13, savedsp-24(SP) // must match frame size
554 MOVV R3, (g_sched+gobuf_sp)(R19)
555
556 // Switch to m->curg stack and call runtime.cgocallbackg.
557 // Because we are taking over the execution of m->curg
558 // but *not* resuming what had been running, we need to
559 // save that information (m->curg->sched) so we can restore it.
560 // We can restore m->curg->sched.sp easily, because calling
561 // runtime.cgocallbackg leaves SP unchanged upon return.
562 // To save m->curg->sched.pc, we push it onto the stack.
563 // This has the added benefit that it looks to the traceback
564 // routine like cgocallbackg is going to return to that
565 // PC (because the frame we allocate below has the same
566 // size as cgocallback_gofunc's frame declared above)
567 // so that the traceback will seamlessly trace back into
568 // the earlier calls.
569 MOVV m_curg(R12), g
570 JAL runtime·save_g(SB)
571 MOVV (g_sched+gobuf_sp)(g), R13 // prepare stack as R13
572 MOVV (g_sched+gobuf_pc)(g), R4
573 MOVV R4, -(24+8)(R13) // "saved LR"; must match frame size
574 MOVV fn+0(FP), R5
575 MOVV frame+8(FP), R6
576 MOVV ctxt+16(FP), R7
577 MOVV $-(24+8)(R13), R3
578 MOVV R5, 8(R3)
579 MOVV R6, 16(R3)
580 MOVV R7, 24(R3)
581 JAL runtime·cgocallbackg(SB)
582
583 // Restore g->sched (== m->curg->sched) from saved values.
584 MOVV 0(R3), R4
585 MOVV R4, (g_sched+gobuf_pc)(g)
586 MOVV $(24+8)(R3), R13 // must match frame size
587 MOVV R13, (g_sched+gobuf_sp)(g)
588
589 // Switch back to m->g0's stack and restore m->g0->sched.sp.
590 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
591 // so we do not have to restore it.)
592 MOVV g_m(g), R12
593 MOVV m_g0(R12), g
594 JAL runtime·save_g(SB)
595 MOVV (g_sched+gobuf_sp)(g), R3
596 MOVV savedsp-24(SP), R13 // must match frame size
597 MOVV R13, (g_sched+gobuf_sp)(g)
598
599 // If the m on entry was nil, we called needm above to borrow an m,
600 // 1. for the duration of the call on non-pthread platforms,
601 // 2. or the duration of the C thread alive on pthread platforms.
602 // If the m on entry wasn't nil,
603 // 1. the thread might be a Go thread,
604 // 2. or it wasn't the first call from a C thread on pthread platforms,
605 // since then we skip dropm to resue the m in the first call.
606 MOVV savedm-8(SP), R12
607 BNE R12, droppedm
608
609 // Skip dropm to reuse it in the next call, when a pthread key has been created.
610 MOVV _cgo_pthread_key_created(SB), R12
611 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
612 BEQ R12, dropm
613 MOVV (R12), R12
614 BNE R12, droppedm
615
616 dropm:
617 MOVV $runtime·dropm(SB), R4
618 JAL (R4)
619 droppedm:
620
621 // Done!
622 RET
623
624 // void setg(G*); set g. for use by needm.
625 TEXT runtime·setg(SB), NOSPLIT, $0-8
626 MOVV gg+0(FP), g
627 // This only happens if iscgo, so jump straight to save_g
628 JAL runtime·save_g(SB)
629 RET
630
631 // void setg_gcc(G*); set g called from gcc with g in R19
632 TEXT setg_gcc<>(SB),NOSPLIT,$0-0
633 MOVV R19, g
634 JAL runtime·save_g(SB)
635 RET
636
637 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
638 MOVW (R0), R0
639 UNDEF
640
641 // AES hashing not implemented for loong64
642 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
643 JMP runtime·memhashFallback<ABIInternal>(SB)
644 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
645 JMP runtime·strhashFallback<ABIInternal>(SB)
646 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
647 JMP runtime·memhash32Fallback<ABIInternal>(SB)
648 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
649 JMP runtime·memhash64Fallback<ABIInternal>(SB)
650
651 TEXT runtime·return0(SB), NOSPLIT, $0
652 MOVW $0, R19
653 RET
654
655 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
656 // Must obey the gcc calling convention.
657 TEXT _cgo_topofstack(SB),NOSPLIT,$16
658 // g (R22) and REGTMP (R30) might be clobbered by load_g. They
659 // are callee-save in the gcc calling convention, so save them.
660 MOVV R30, savedREGTMP-16(SP)
661 MOVV g, savedG-8(SP)
662
663 JAL runtime·load_g(SB)
664 MOVV g_m(g), R19
665 MOVV m_curg(R19), R19
666 MOVV (g_stack+stack_hi)(R19), R4 // return value in R4
667
668 MOVV savedG-8(SP), g
669 MOVV savedREGTMP-16(SP), R30
670 RET
671
672 // The top-most function running on a goroutine
673 // returns to goexit+PCQuantum.
674 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
675 NOOP
676 JAL runtime·goexit1(SB) // does not return
677 // traceback from goexit1 must hit code range of goexit
678 NOOP
679
680 // This is called from .init_array and follows the platform, not Go, ABI.
681 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
682 ADDV $-0x10, R3
683 MOVV R30, 8(R3) // The access to global variables below implicitly uses R30, which is callee-save
684 MOVV runtime·lastmoduledatap(SB), R12
685 MOVV R4, moduledata_next(R12)
686 MOVV R4, runtime·lastmoduledatap(SB)
687 MOVV 8(R3), R30
688 ADDV $0x10, R3
689 RET
690
691 TEXT ·checkASM(SB),NOSPLIT,$0-1
692 MOVW $1, R19
693 MOVB R19, ret+0(FP)
694 RET
695
696 #ifdef GOEXPERIMENT_regabiargs
697 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R25.
698 TEXT ·spillArgs(SB),NOSPLIT,$0-0
699 MOVV R4, (0*8)(R25)
700 MOVV R5, (1*8)(R25)
701 MOVV R6, (2*8)(R25)
702 MOVV R7, (3*8)(R25)
703 MOVV R8, (4*8)(R25)
704 MOVV R9, (5*8)(R25)
705 MOVV R10, (6*8)(R25)
706 MOVV R11, (7*8)(R25)
707 MOVV R12, (8*8)(R25)
708 MOVV R13, (9*8)(R25)
709 MOVV R14, (10*8)(R25)
710 MOVV R15, (11*8)(R25)
711 MOVV R16, (12*8)(R25)
712 MOVV R17, (13*8)(R25)
713 MOVV R18, (14*8)(R25)
714 MOVV R19, (15*8)(R25)
715 MOVD F0, (16*8)(R25)
716 MOVD F1, (17*8)(R25)
717 MOVD F2, (18*8)(R25)
718 MOVD F3, (19*8)(R25)
719 MOVD F4, (20*8)(R25)
720 MOVD F5, (21*8)(R25)
721 MOVD F6, (22*8)(R25)
722 MOVD F7, (23*8)(R25)
723 MOVD F8, (24*8)(R25)
724 MOVD F9, (25*8)(R25)
725 MOVD F10, (26*8)(R25)
726 MOVD F11, (27*8)(R25)
727 MOVD F12, (28*8)(R25)
728 MOVD F13, (29*8)(R25)
729 MOVD F14, (30*8)(R25)
730 MOVD F15, (31*8)(R25)
731 RET
732
733 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R25.
734 TEXT ·unspillArgs(SB),NOSPLIT,$0-0
735 MOVV (0*8)(R25), R4
736 MOVV (1*8)(R25), R5
737 MOVV (2*8)(R25), R6
738 MOVV (3*8)(R25), R7
739 MOVV (4*8)(R25), R8
740 MOVV (5*8)(R25), R9
741 MOVV (6*8)(R25), R10
742 MOVV (7*8)(R25), R11
743 MOVV (8*8)(R25), R12
744 MOVV (9*8)(R25), R13
745 MOVV (10*8)(R25), R14
746 MOVV (11*8)(R25), R15
747 MOVV (12*8)(R25), R16
748 MOVV (13*8)(R25), R17
749 MOVV (14*8)(R25), R18
750 MOVV (15*8)(R25), R19
751 MOVD (16*8)(R25), F0
752 MOVD (17*8)(R25), F1
753 MOVD (18*8)(R25), F2
754 MOVD (19*8)(R25), F3
755 MOVD (20*8)(R25), F4
756 MOVD (21*8)(R25), F5
757 MOVD (22*8)(R25), F6
758 MOVD (23*8)(R25), F7
759 MOVD (24*8)(R25), F8
760 MOVD (25*8)(R25), F9
761 MOVD (26*8)(R25), F10
762 MOVD (27*8)(R25), F11
763 MOVD (28*8)(R25), F12
764 MOVD (29*8)(R25), F13
765 MOVD (30*8)(R25), F14
766 MOVD (31*8)(R25), F15
767 RET
768 #else
769 TEXT ·spillArgs(SB),NOSPLIT,$0-0
770 RET
771
772 TEXT ·unspillArgs(SB),NOSPLIT,$0-0
773 RET
774 #endif
775
776 // gcWriteBarrier informs the GC about heap pointer writes.
777 //
778 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
779 // number of bytes of buffer needed in R29, and returns a pointer
780 // to the buffer space in R29.
781 // It clobbers R30 (the linker temp register).
782 // The act of CALLing gcWriteBarrier will clobber R1 (LR).
783 // It does not clobber any other general-purpose registers,
784 // but may clobber others (e.g., floating point registers).
785 TEXT gcWriteBarrier<>(SB),NOSPLIT,$216
786 // Save the registers clobbered by the fast path.
787 MOVV R19, 208(R3)
788 MOVV R13, 216(R3)
789 retry:
790 MOVV g_m(g), R19
791 MOVV m_p(R19), R19
792 MOVV (p_wbBuf+wbBuf_next)(R19), R13
793 MOVV (p_wbBuf+wbBuf_end)(R19), R30 // R30 is linker temp register
794 // Increment wbBuf.next position.
795 ADDV R29, R13
796 // Is the buffer full?
797 BLTU R30, R13, flush
798 // Commit to the larger buffer.
799 MOVV R13, (p_wbBuf+wbBuf_next)(R19)
800 // Make return value (the original next position)
801 SUBV R29, R13, R29
802 // Restore registers.
803 MOVV 208(R3), R19
804 MOVV 216(R3), R13
805 RET
806
807 flush:
808 // Save all general purpose registers since these could be
809 // clobbered by wbBufFlush and were not saved by the caller.
810 MOVV R27, 8(R3)
811 MOVV R28, 16(R3)
812 // R1 is LR, which was saved by the prologue.
813 MOVV R2, 24(R3)
814 // R3 is SP.
815 MOVV R4, 32(R3)
816 MOVV R5, 40(R3)
817 MOVV R6, 48(R3)
818 MOVV R7, 56(R3)
819 MOVV R8, 64(R3)
820 MOVV R9, 72(R3)
821 MOVV R10, 80(R3)
822 MOVV R11, 88(R3)
823 MOVV R12, 96(R3)
824 // R13 already saved
825 MOVV R14, 104(R3)
826 MOVV R15, 112(R3)
827 MOVV R16, 120(R3)
828 MOVV R17, 128(R3)
829 MOVV R18, 136(R3)
830 // R19 already saved
831 MOVV R20, 144(R3)
832 MOVV R21, 152(R3)
833 // R22 is g.
834 MOVV R23, 160(R3)
835 MOVV R24, 168(R3)
836 MOVV R25, 176(R3)
837 MOVV R26, 184(R3)
838 // R27 already saved
839 // R28 already saved.
840 MOVV R29, 192(R3)
841 // R30 is tmp register.
842 MOVV R31, 200(R3)
843
844 CALL runtime·wbBufFlush(SB)
845
846 MOVV 8(R3), R27
847 MOVV 16(R3), R28
848 MOVV 24(R3), R2
849 MOVV 32(R3), R4
850 MOVV 40(R3), R5
851 MOVV 48(R3), R6
852 MOVV 56(R3), R7
853 MOVV 64(R3), R8
854 MOVV 72(R3), R9
855 MOVV 80(R3), R10
856 MOVV 88(R3), R11
857 MOVV 96(R3), R12
858 MOVV 104(R3), R14
859 MOVV 112(R3), R15
860 MOVV 120(R3), R16
861 MOVV 128(R3), R17
862 MOVV 136(R3), R18
863 MOVV 144(R3), R20
864 MOVV 152(R3), R21
865 MOVV 160(R3), R23
866 MOVV 168(R3), R24
867 MOVV 176(R3), R25
868 MOVV 184(R3), R26
869 MOVV 192(R3), R29
870 MOVV 200(R3), R31
871 JMP retry
872
873 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
874 MOVV $8, R29
875 JMP gcWriteBarrier<>(SB)
876 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
877 MOVV $16, R29
878 JMP gcWriteBarrier<>(SB)
879 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
880 MOVV $24, R29
881 JMP gcWriteBarrier<>(SB)
882 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
883 MOVV $32, R29
884 JMP gcWriteBarrier<>(SB)
885 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
886 MOVV $40, R29
887 JMP gcWriteBarrier<>(SB)
888 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
889 MOVV $48, R29
890 JMP gcWriteBarrier<>(SB)
891 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
892 MOVV $56, R29
893 JMP gcWriteBarrier<>(SB)
894 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
895 MOVV $64, R29
896 JMP gcWriteBarrier<>(SB)
897
898 // Note: these functions use a special calling convention to save generated code space.
899 // Arguments are passed in registers, but the space for those arguments are allocated
900 // in the caller's stack frame. These stubs write the args into that stack space and
901 // then tail call to the corresponding runtime handler.
902 // The tail call makes these stubs disappear in backtraces.
903 TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
904 #ifdef GOEXPERIMENT_regabiargs
905 MOVV R20, R4
906 MOVV R21, R5
907 #else
908 MOVV R20, x+0(FP)
909 MOVV R21, y+8(FP)
910 #endif
911 JMP runtime·goPanicIndex<ABIInternal>(SB)
912 TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
913 #ifdef GOEXPERIMENT_regabiargs
914 MOVV R20, R4
915 MOVV R21, R5
916 #else
917 MOVV R20, x+0(FP)
918 MOVV R21, y+8(FP)
919 #endif
920 JMP runtime·goPanicIndexU<ABIInternal>(SB)
921 TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
922 #ifdef GOEXPERIMENT_regabiargs
923 MOVV R21, R4
924 MOVV R23, R5
925 #else
926 MOVV R21, x+0(FP)
927 MOVV R23, y+8(FP)
928 #endif
929 JMP runtime·goPanicSliceAlen<ABIInternal>(SB)
930 TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
931 #ifdef GOEXPERIMENT_regabiargs
932 MOVV R21, R4
933 MOVV R23, R5
934 #else
935 MOVV R21, x+0(FP)
936 MOVV R23, y+8(FP)
937 #endif
938 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB)
939 TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
940 #ifdef GOEXPERIMENT_regabiargs
941 MOVV R21, R4
942 MOVV R23, R5
943 #else
944 MOVV R21, x+0(FP)
945 MOVV R23, y+8(FP)
946 #endif
947 JMP runtime·goPanicSliceAcap<ABIInternal>(SB)
948 TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
949 #ifdef GOEXPERIMENT_regabiargs
950 MOVV R21, R4
951 MOVV R23, R5
952 #else
953 MOVV R21, x+0(FP)
954 MOVV R23, y+8(FP)
955 #endif
956 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB)
957 TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
958 #ifdef GOEXPERIMENT_regabiargs
959 MOVV R20, R4
960 MOVV R21, R5
961 #else
962 MOVV R20, x+0(FP)
963 MOVV R21, y+8(FP)
964 #endif
965 JMP runtime·goPanicSliceB<ABIInternal>(SB)
966 TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
967 #ifdef GOEXPERIMENT_regabiargs
968 MOVV R20, R4
969 MOVV R21, R5
970 #else
971 MOVV R20, x+0(FP)
972 MOVV R21, y+8(FP)
973 #endif
974 JMP runtime·goPanicSliceBU<ABIInternal>(SB)
975 TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
976 #ifdef GOEXPERIMENT_regabiargs
977 MOVV R23, R4
978 MOVV R24, R5
979 #else
980 MOVV R23, x+0(FP)
981 MOVV R24, y+8(FP)
982 #endif
983 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB)
984 TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
985 #ifdef GOEXPERIMENT_regabiargs
986 MOVV R23, R4
987 MOVV R24, R5
988 #else
989 MOVV R23, x+0(FP)
990 MOVV R24, y+8(FP)
991 #endif
992 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB)
993 TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
994 #ifdef GOEXPERIMENT_regabiargs
995 MOVV R23, R4
996 MOVV R24, R5
997 #else
998 MOVV R23, x+0(FP)
999 MOVV R24, y+8(FP)
1000 #endif
1001 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB)
1002 TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
1003 #ifdef GOEXPERIMENT_regabiargs
1004 MOVV R23, R4
1005 MOVV R24, R5
1006 #else
1007 MOVV R23, x+0(FP)
1008 MOVV R24, y+8(FP)
1009 #endif
1010 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB)
1011 TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
1012 #ifdef GOEXPERIMENT_regabiargs
1013 MOVV R21, R4
1014 MOVV R23, R5
1015 #else
1016 MOVV R21, x+0(FP)
1017 MOVV R23, y+8(FP)
1018 #endif
1019 JMP runtime·goPanicSlice3B<ABIInternal>(SB)
1020 TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
1021 #ifdef GOEXPERIMENT_regabiargs
1022 MOVV R21, R4
1023 MOVV R23, R5
1024 #else
1025 MOVV R21, x+0(FP)
1026 MOVV R23, y+8(FP)
1027 #endif
1028 JMP runtime·goPanicSlice3BU<ABIInternal>(SB)
1029 TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
1030 #ifdef GOEXPERIMENT_regabiargs
1031 MOVV R20, R4
1032 MOVV R21, R5
1033 #else
1034 MOVV R20, x+0(FP)
1035 MOVV R21, y+8(FP)
1036 #endif
1037 JMP runtime·goPanicSlice3C<ABIInternal>(SB)
1038 TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
1039 #ifdef GOEXPERIMENT_regabiargs
1040 MOVV R20, R4
1041 MOVV R21, R5
1042 #else
1043 MOVV R20, x+0(FP)
1044 MOVV R21, y+8(FP)
1045 #endif
1046 JMP runtime·goPanicSlice3CU<ABIInternal>(SB)
1047 TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
1048 #ifdef GOEXPERIMENT_regabiargs
1049 MOVV R23, R4
1050 MOVV R24, R5
1051 #else
1052 MOVV R23, x+0(FP)
1053 MOVV R24, y+8(FP)
1054 #endif
1055 JMP runtime·goPanicSliceConvert<ABIInternal>(SB)
1056
View as plain text