Text file src/runtime/race_s390x.s

     1  // Copyright 2021 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build race
     6  
     7  #include "go_asm.h"
     8  #include "funcdata.h"
     9  #include "textflag.h"
    10  
    11  // The following thunks allow calling the gcc-compiled race runtime directly
    12  // from Go code without going all the way through cgo.
    13  // First, it's much faster (up to 50% speedup for real Go programs).
    14  // Second, it eliminates race-related special cases from cgocall and scheduler.
    15  // Third, in long-term it will allow to remove cyclic runtime/race dependency on cmd/go.
    16  
    17  // A brief recap of the s390x C calling convention.
    18  // Arguments are passed in R2...R6, the rest is on stack.
    19  // Callee-saved registers are: R6...R13, R15.
    20  // Temporary registers are: R0...R5, R14.
    21  
    22  // When calling racecalladdr, R1 is the call target address.
    23  
    24  // The race ctx, ThreadState *thr below, is passed in R2 and loaded in racecalladdr.
    25  
    26  // func runtime·raceread(addr uintptr)
    27  // Called from instrumented code.
    28  TEXT	runtime·raceread(SB), NOSPLIT, $0-8
    29  	// void __tsan_read(ThreadState *thr, void *addr, void *pc);
    30  	MOVD	$__tsan_read(SB), R1
    31  	MOVD	addr+0(FP), R3
    32  	MOVD	R14, R4
    33  	JMP	racecalladdr<>(SB)
    34  
    35  // func runtime·RaceRead(addr uintptr)
    36  TEXT	runtime·RaceRead(SB), NOSPLIT, $0-8
    37  	// This needs to be a tail call, because raceread reads caller pc.
    38  	JMP	runtime·raceread(SB)
    39  
    40  // func runtime·racereadpc(void *addr, void *callpc, void *pc)
    41  TEXT	runtime·racereadpc(SB), NOSPLIT, $0-24
    42  	// void __tsan_read_pc(ThreadState *thr, void *addr, void *callpc, void *pc);
    43  	MOVD	$__tsan_read_pc(SB), R1
    44  	LMG	addr+0(FP), R3, R5
    45  	JMP	racecalladdr<>(SB)
    46  
    47  // func runtime·racewrite(addr uintptr)
    48  // Called from instrumented code.
    49  TEXT	runtime·racewrite(SB), NOSPLIT, $0-8
    50  	// void __tsan_write(ThreadState *thr, void *addr, void *pc);
    51  	MOVD	$__tsan_write(SB), R1
    52  	MOVD	addr+0(FP), R3
    53  	MOVD	R14, R4
    54  	JMP	racecalladdr<>(SB)
    55  
    56  // func runtime·RaceWrite(addr uintptr)
    57  TEXT	runtime·RaceWrite(SB), NOSPLIT, $0-8
    58  	// This needs to be a tail call, because racewrite reads caller pc.
    59  	JMP	runtime·racewrite(SB)
    60  
    61  // func runtime·racewritepc(void *addr, void *callpc, void *pc)
    62  TEXT	runtime·racewritepc(SB), NOSPLIT, $0-24
    63  	// void __tsan_write_pc(ThreadState *thr, void *addr, void *callpc, void *pc);
    64  	MOVD	$__tsan_write_pc(SB), R1
    65  	LMG	addr+0(FP), R3, R5
    66  	JMP	racecalladdr<>(SB)
    67  
    68  // func runtime·racereadrange(addr, size uintptr)
    69  // Called from instrumented code.
    70  TEXT	runtime·racereadrange(SB), NOSPLIT, $0-16
    71  	// void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, void *pc);
    72  	MOVD	$__tsan_read_range(SB), R1
    73  	LMG	addr+0(FP), R3, R4
    74  	MOVD	R14, R5
    75  	JMP	racecalladdr<>(SB)
    76  
    77  // func runtime·RaceReadRange(addr, size uintptr)
    78  TEXT	runtime·RaceReadRange(SB), NOSPLIT, $0-16
    79  	// This needs to be a tail call, because racereadrange reads caller pc.
    80  	JMP	runtime·racereadrange(SB)
    81  
    82  // func runtime·racereadrangepc1(void *addr, uintptr sz, void *pc)
    83  TEXT	runtime·racereadrangepc1(SB), NOSPLIT, $0-24
    84  	// void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, void *pc);
    85  	MOVD	$__tsan_read_range(SB), R1
    86  	LMG	addr+0(FP), R3, R5
    87  	// pc is an interceptor address, but TSan expects it to point to the
    88  	// middle of an interceptor (see LLVM's SCOPED_INTERCEPTOR_RAW).
    89  	ADD	$2, R5
    90  	JMP	racecalladdr<>(SB)
    91  
    92  // func runtime·racewriterange(addr, size uintptr)
    93  // Called from instrumented code.
    94  TEXT	runtime·racewriterange(SB), NOSPLIT, $0-16
    95  	// void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, void *pc);
    96  	MOVD	$__tsan_write_range(SB), R1
    97  	LMG	addr+0(FP), R3, R4
    98  	MOVD	R14, R5
    99  	JMP	racecalladdr<>(SB)
   100  
   101  // func runtime·RaceWriteRange(addr, size uintptr)
   102  TEXT	runtime·RaceWriteRange(SB), NOSPLIT, $0-16
   103  	// This needs to be a tail call, because racewriterange reads caller pc.
   104  	JMP	runtime·racewriterange(SB)
   105  
   106  // func runtime·racewriterangepc1(void *addr, uintptr sz, void *pc)
   107  TEXT	runtime·racewriterangepc1(SB), NOSPLIT, $0-24
   108  	// void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, void *pc);
   109  	MOVD	$__tsan_write_range(SB), R1
   110  	LMG	addr+0(FP), R3, R5
   111  	// pc is an interceptor address, but TSan expects it to point to the
   112  	// middle of an interceptor (see LLVM's SCOPED_INTERCEPTOR_RAW).
   113  	ADD	$2, R5
   114  	JMP	racecalladdr<>(SB)
   115  
   116  // If R3 is out of range, do nothing. Otherwise, setup goroutine context and
   117  // invoke racecall. Other arguments are already set.
   118  TEXT	racecalladdr<>(SB), NOSPLIT, $0-0
   119  	MOVD	runtime·racearenastart(SB), R0
   120  	CMPUBLT	R3, R0, data			// Before racearena start?
   121  	MOVD	runtime·racearenaend(SB), R0
   122  	CMPUBLT	R3, R0, call			// Before racearena end?
   123  data:
   124  	MOVD	runtime·racedatastart(SB), R0
   125  	CMPUBLT	R3, R0, ret			// Before racedata start?
   126  	MOVD	runtime·racedataend(SB), R0
   127  	CMPUBGE	R3, R0, ret			// At or after racedata end?
   128  call:
   129  	MOVD	g_racectx(g), R2
   130  	JMP	racecall<>(SB)
   131  ret:
   132  	RET
   133  
   134  // func runtime·racefuncenter(pc uintptr)
   135  // Called from instrumented code.
   136  TEXT	runtime·racefuncenter(SB), NOSPLIT, $0-8
   137  	MOVD	callpc+0(FP), R3
   138  	JMP	racefuncenter<>(SB)
   139  
   140  // Common code for racefuncenter
   141  // R3 = caller's return address
   142  TEXT	racefuncenter<>(SB), NOSPLIT, $0-0
   143  	// void __tsan_func_enter(ThreadState *thr, void *pc);
   144  	MOVD	$__tsan_func_enter(SB), R1
   145  	MOVD	g_racectx(g), R2
   146  	BL	racecall<>(SB)
   147  	RET
   148  
   149  // func runtime·racefuncexit()
   150  // Called from instrumented code.
   151  TEXT	runtime·racefuncexit(SB), NOSPLIT, $0-0
   152  	// void __tsan_func_exit(ThreadState *thr);
   153  	MOVD	$__tsan_func_exit(SB), R1
   154  	MOVD	g_racectx(g), R2
   155  	JMP	racecall<>(SB)
   156  
   157  // Atomic operations for sync/atomic package.
   158  
   159  // Load
   160  
   161  TEXT	sync∕atomic·LoadInt32(SB), NOSPLIT, $0-12
   162  	GO_ARGS
   163  	MOVD	$__tsan_go_atomic32_load(SB), R1
   164  	BL	racecallatomic<>(SB)
   165  	RET
   166  
   167  TEXT	sync∕atomic·LoadInt64(SB), NOSPLIT, $0-16
   168  	GO_ARGS
   169  	MOVD	$__tsan_go_atomic64_load(SB), R1
   170  	BL	racecallatomic<>(SB)
   171  	RET
   172  
   173  TEXT	sync∕atomic·LoadUint32(SB), NOSPLIT, $0-12
   174  	GO_ARGS
   175  	JMP	sync∕atomic·LoadInt32(SB)
   176  
   177  TEXT	sync∕atomic·LoadUint64(SB), NOSPLIT, $0-16
   178  	GO_ARGS
   179  	JMP	sync∕atomic·LoadInt64(SB)
   180  
   181  TEXT	sync∕atomic·LoadUintptr(SB), NOSPLIT, $0-16
   182  	GO_ARGS
   183  	JMP	sync∕atomic·LoadInt64(SB)
   184  
   185  TEXT	sync∕atomic·LoadPointer(SB), NOSPLIT, $0-16
   186  	GO_ARGS
   187  	JMP	sync∕atomic·LoadInt64(SB)
   188  
   189  // Store
   190  
   191  TEXT	sync∕atomic·StoreInt32(SB), NOSPLIT, $0-12
   192  	GO_ARGS
   193  	MOVD	$__tsan_go_atomic32_store(SB), R1
   194  	BL	racecallatomic<>(SB)
   195  	RET
   196  
   197  TEXT	sync∕atomic·StoreInt64(SB), NOSPLIT, $0-16
   198  	GO_ARGS
   199  	MOVD	$__tsan_go_atomic64_store(SB), R1
   200  	BL	racecallatomic<>(SB)
   201  	RET
   202  
   203  TEXT	sync∕atomic·StoreUint32(SB), NOSPLIT, $0-12
   204  	GO_ARGS
   205  	JMP	sync∕atomic·StoreInt32(SB)
   206  
   207  TEXT	sync∕atomic·StoreUint64(SB), NOSPLIT, $0-16
   208  	GO_ARGS
   209  	JMP	sync∕atomic·StoreInt64(SB)
   210  
   211  TEXT	sync∕atomic·StoreUintptr(SB), NOSPLIT, $0-16
   212  	GO_ARGS
   213  	JMP	sync∕atomic·StoreInt64(SB)
   214  
   215  // Swap
   216  
   217  TEXT	sync∕atomic·SwapInt32(SB), NOSPLIT, $0-20
   218  	GO_ARGS
   219  	MOVD	$__tsan_go_atomic32_exchange(SB), R1
   220  	BL	racecallatomic<>(SB)
   221  	RET
   222  
   223  TEXT	sync∕atomic·SwapInt64(SB), NOSPLIT, $0-24
   224  	GO_ARGS
   225  	MOVD	$__tsan_go_atomic64_exchange(SB), R1
   226  	BL	racecallatomic<>(SB)
   227  	RET
   228  
   229  TEXT	sync∕atomic·SwapUint32(SB), NOSPLIT, $0-20
   230  	GO_ARGS
   231  	JMP	sync∕atomic·SwapInt32(SB)
   232  
   233  TEXT	sync∕atomic·SwapUint64(SB), NOSPLIT, $0-24
   234  	GO_ARGS
   235  	JMP	sync∕atomic·SwapInt64(SB)
   236  
   237  TEXT	sync∕atomic·SwapUintptr(SB), NOSPLIT, $0-24
   238  	GO_ARGS
   239  	JMP	sync∕atomic·SwapInt64(SB)
   240  
   241  // Add
   242  
   243  TEXT	sync∕atomic·AddInt32(SB), NOSPLIT, $0-20
   244  	GO_ARGS
   245  	MOVD	$__tsan_go_atomic32_fetch_add(SB), R1
   246  	BL	racecallatomic<>(SB)
   247  	// TSan performed fetch_add, but Go needs add_fetch.
   248  	MOVW	add+8(FP), R0
   249  	MOVW	ret+16(FP), R1
   250  	ADD	R0, R1, R0
   251  	MOVW	R0, ret+16(FP)
   252  	RET
   253  
   254  TEXT	sync∕atomic·AddInt64(SB), NOSPLIT, $0-24
   255  	GO_ARGS
   256  	MOVD	$__tsan_go_atomic64_fetch_add(SB), R1
   257  	BL	racecallatomic<>(SB)
   258  	// TSan performed fetch_add, but Go needs add_fetch.
   259  	MOVD	add+8(FP), R0
   260  	MOVD	ret+16(FP), R1
   261  	ADD	R0, R1, R0
   262  	MOVD	R0, ret+16(FP)
   263  	RET
   264  
   265  TEXT	sync∕atomic·AddUint32(SB), NOSPLIT, $0-20
   266  	GO_ARGS
   267  	JMP	sync∕atomic·AddInt32(SB)
   268  
   269  TEXT	sync∕atomic·AddUint64(SB), NOSPLIT, $0-24
   270  	GO_ARGS
   271  	JMP	sync∕atomic·AddInt64(SB)
   272  
   273  TEXT	sync∕atomic·AddUintptr(SB), NOSPLIT, $0-24
   274  	GO_ARGS
   275  	JMP	sync∕atomic·AddInt64(SB)
   276  
   277  // And
   278  TEXT	sync∕atomic·AndInt32(SB), NOSPLIT, $0-20
   279  	GO_ARGS
   280  	MOVD	$__tsan_go_atomic32_fetch_and(SB), R1
   281  	BL	racecallatomic<>(SB)
   282  	RET
   283  
   284  TEXT	sync∕atomic·AndInt64(SB), NOSPLIT, $0-24
   285  	GO_ARGS
   286  	MOVD	$__tsan_go_atomic64_fetch_and(SB), R1
   287  	BL	racecallatomic<>(SB)
   288  	RET
   289  
   290  TEXT	sync∕atomic·AndUint32(SB), NOSPLIT, $0-20
   291  	GO_ARGS
   292  	JMP	sync∕atomic·AndInt32(SB)
   293  
   294  TEXT	sync∕atomic·AndUint64(SB), NOSPLIT, $0-24
   295  	GO_ARGS
   296  	JMP	sync∕atomic·AndInt64(SB)
   297  
   298  TEXT	sync∕atomic·AndUintptr(SB), NOSPLIT, $0-24
   299  	GO_ARGS
   300  	JMP	sync∕atomic·AndInt64(SB)
   301  
   302  // Or
   303  TEXT	sync∕atomic·OrInt32(SB), NOSPLIT, $0-20
   304  	GO_ARGS
   305  	MOVD	$__tsan_go_atomic32_fetch_or(SB), R1
   306  	BL	racecallatomic<>(SB)
   307  	RET
   308  
   309  TEXT	sync∕atomic·OrInt64(SB), NOSPLIT, $0-24
   310  	GO_ARGS
   311  	MOVD	$__tsan_go_atomic64_fetch_or(SB), R1
   312  	BL	racecallatomic<>(SB)
   313  	RET
   314  
   315  TEXT	sync∕atomic·OrUint32(SB), NOSPLIT, $0-20
   316  	GO_ARGS
   317  	JMP	sync∕atomic·OrInt32(SB)
   318  
   319  TEXT	sync∕atomic·OrUint64(SB), NOSPLIT, $0-24
   320  	GO_ARGS
   321  	JMP	sync∕atomic·OrInt64(SB)
   322  
   323  TEXT	sync∕atomic·OrUintptr(SB), NOSPLIT, $0-24
   324  	GO_ARGS
   325  	JMP	sync∕atomic·OrInt64(SB)
   326  
   327  // CompareAndSwap
   328  
   329  TEXT	sync∕atomic·CompareAndSwapInt32(SB), NOSPLIT, $0-17
   330  	GO_ARGS
   331  	MOVD	$__tsan_go_atomic32_compare_exchange(SB), R1
   332  	BL	racecallatomic<>(SB)
   333  	RET
   334  
   335  TEXT	sync∕atomic·CompareAndSwapInt64(SB), NOSPLIT, $0-25
   336  	GO_ARGS
   337  	MOVD	$__tsan_go_atomic64_compare_exchange(SB), R1
   338  	BL	racecallatomic<>(SB)
   339  	RET
   340  
   341  TEXT	sync∕atomic·CompareAndSwapUint32(SB), NOSPLIT, $0-17
   342  	GO_ARGS
   343  	JMP	sync∕atomic·CompareAndSwapInt32(SB)
   344  
   345  TEXT	sync∕atomic·CompareAndSwapUint64(SB), NOSPLIT, $0-25
   346  	GO_ARGS
   347  	JMP	sync∕atomic·CompareAndSwapInt64(SB)
   348  
   349  TEXT	sync∕atomic·CompareAndSwapUintptr(SB), NOSPLIT, $0-25
   350  	GO_ARGS
   351  	JMP	sync∕atomic·CompareAndSwapInt64(SB)
   352  
   353  // Common code for atomic operations. Calls R1.
   354  TEXT	racecallatomic<>(SB), NOSPLIT, $0
   355  	MOVD	24(R15), R5			// Address (arg1, after 2xBL).
   356  	// If we pass an invalid pointer to the TSan runtime, it will cause a
   357  	// "fatal error: unknown caller pc". So trigger a SEGV here instead.
   358  	MOVB	(R5), R0
   359  	MOVD	runtime·racearenastart(SB), R0
   360  	CMPUBLT	R5, R0, racecallatomic_data	// Before racearena start?
   361  	MOVD	runtime·racearenaend(SB), R0
   362  	CMPUBLT	R5, R0, racecallatomic_ok	// Before racearena end?
   363  racecallatomic_data:
   364  	MOVD	runtime·racedatastart(SB), R0
   365  	CMPUBLT	R5, R0, racecallatomic_ignore	// Before racedata start?
   366  	MOVD	runtime·racedataend(SB), R0
   367  	CMPUBGE	R5, R0,	racecallatomic_ignore	// At or after racearena end?
   368  racecallatomic_ok:
   369  	MOVD	g_racectx(g), R2		// ThreadState *.
   370  	MOVD	8(R15), R3			// Caller PC.
   371  	MOVD	R14, R4				// PC.
   372  	ADD	$24, R15, R5			// Arguments.
   373  	// Tail call fails to restore R15, so use a normal one.
   374  	BL	racecall<>(SB)
   375  	RET
   376  racecallatomic_ignore:
   377  	// Call __tsan_go_ignore_sync_begin to ignore synchronization during
   378  	// the atomic op. An attempt to synchronize on the address would cause
   379  	// a crash.
   380  	MOVD	R1, R6				// Save target function.
   381  	MOVD	R14, R7				// Save PC.
   382  	MOVD	$__tsan_go_ignore_sync_begin(SB), R1
   383  	MOVD	g_racectx(g), R2		// ThreadState *.
   384  	BL	racecall<>(SB)
   385  	MOVD	R6, R1				// Restore target function.
   386  	MOVD	g_racectx(g), R2		// ThreadState *.
   387  	MOVD	8(R15), R3			// Caller PC.
   388  	MOVD	R7, R4				// PC.
   389  	ADD	$24, R15, R5			// Arguments.
   390  	BL	racecall<>(SB)
   391  	MOVD	$__tsan_go_ignore_sync_end(SB), R1
   392  	MOVD	g_racectx(g), R2		// ThreadState *.
   393  	BL	racecall<>(SB)
   394  	RET
   395  
   396  // func runtime·racecall(void(*f)(...), ...)
   397  // Calls C function f from race runtime and passes up to 4 arguments to it.
   398  // The arguments are never heap-object-preserving pointers, so we pretend there
   399  // are no arguments.
   400  TEXT	runtime·racecall(SB), NOSPLIT, $0-0
   401  	MOVD	fn+0(FP), R1
   402  	MOVD	arg0+8(FP), R2
   403  	MOVD	arg1+16(FP), R3
   404  	MOVD	arg2+24(FP), R4
   405  	MOVD	arg3+32(FP), R5
   406  	JMP	racecall<>(SB)
   407  
   408  // Switches SP to g0 stack and calls R1. Arguments are already set.
   409  TEXT	racecall<>(SB), NOSPLIT, $0-0
   410  	BL	runtime·save_g(SB)		// Save g for callbacks.
   411  	MOVD	R15, R7				// Save SP.
   412  	MOVD	g_m(g), R8			// R8 = thread.
   413  	MOVD	m_g0(R8), R8			// R8 = g0.
   414  	CMPBEQ	R8, g, call			// Already on g0?
   415  	MOVD	(g_sched+gobuf_sp)(R8), R15	// Switch SP to g0.
   416  call:	SUB	$160, R15			// Allocate C frame.
   417  	BL	R1				// Call C code.
   418  	MOVD	R7, R15				// Restore SP.
   419  	RET					// Return to Go.
   420  
   421  // C->Go callback thunk that allows to call runtime·racesymbolize from C
   422  // code. racecall has only switched SP, finish g->g0 switch by setting correct
   423  // g. R2 contains command code, R3 contains command-specific context. See
   424  // racecallback for command codes.
   425  TEXT	runtime·racecallbackthunk(SB), NOSPLIT|NOFRAME, $0
   426  	STMG	R6, R15, 48(R15)		// Save non-volatile regs.
   427  	BL	runtime·load_g(SB)		// Saved by racecall.
   428  	CMPBNE	R2, $0, rest			// raceGetProcCmd?
   429  	MOVD	g_m(g), R2			// R2 = thread.
   430  	MOVD	m_p(R2), R2			// R2 = processor.
   431  	MVC	$8, p_raceprocctx(R2), (R3)	// *R3 = ThreadState *.
   432  	LMG	48(R15), R6, R15		// Restore non-volatile regs.
   433  	BR	R14				// Return to C.
   434  rest:	MOVD	g_m(g), R4			// R4 = current thread.
   435  	MOVD	m_g0(R4), g			// Switch to g0.
   436  	SUB	$24, R15			// Allocate Go argument slots.
   437  	STMG	R2, R3, 8(R15)			// Fill Go frame.
   438  	BL	runtime·racecallback(SB)	// Call Go code.
   439  	LMG	72(R15), R6, R15		// Restore non-volatile regs.
   440  	BR	R14				// Return to C.
   441  

View as plain text