race.go

     1  // Copyright 2012 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build race
     6  
     7  package runtime
     8  
     9  import (
    10  	"internal/abi"
    11  	"unsafe"
    12  )
    13  
    14  // Public race detection API, present iff build with -race.
    15  
    16  func RaceRead(addr unsafe.Pointer)
    17  
    18  //go:linkname race_Read internal/race.Read
    19  //go:nosplit
    20  func race_Read(addr unsafe.Pointer) {
    21  	RaceRead(addr)
    22  }
    23  
    24  func RaceWrite(addr unsafe.Pointer)
    25  
    26  //go:linkname race_Write internal/race.Write
    27  //go:nosplit
    28  func race_Write(addr unsafe.Pointer) {
    29  	RaceWrite(addr)
    30  }
    31  
    32  func RaceReadRange(addr unsafe.Pointer, len int)
    33  
    34  //go:linkname race_ReadRange internal/race.ReadRange
    35  //go:nosplit
    36  func race_ReadRange(addr unsafe.Pointer, len int) {
    37  	RaceReadRange(addr, len)
    38  }
    39  
    40  func RaceWriteRange(addr unsafe.Pointer, len int)
    41  
    42  //go:linkname race_WriteRange internal/race.WriteRange
    43  //go:nosplit
    44  func race_WriteRange(addr unsafe.Pointer, len int) {
    45  	RaceWriteRange(addr, len)
    46  }
    47  
    48  func RaceErrors() int {
    49  	var n uint64
    50  	racecall(&__tsan_report_count, uintptr(unsafe.Pointer(&n)), 0, 0, 0)
    51  	return int(n)
    52  }
    53  
    54  //go:linkname race_Errors internal/race.Errors
    55  //go:nosplit
    56  func race_Errors() int {
    57  	return RaceErrors()
    58  }
    59  
    60  // RaceAcquire/RaceRelease/RaceReleaseMerge establish happens-before relations
    61  // between goroutines. These inform the race detector about actual synchronization
    62  // that it can't see for some reason (e.g. synchronization within RaceDisable/RaceEnable
    63  // sections of code).
    64  // RaceAcquire establishes a happens-before relation with the preceding
    65  // RaceReleaseMerge on addr up to and including the last RaceRelease on addr.
    66  // In terms of the C memory model (C11 §5.1.2.4, §7.17.3),
    67  // RaceAcquire is equivalent to atomic_load(memory_order_acquire).
    68  //
    69  //go:nosplit
    70  func RaceAcquire(addr unsafe.Pointer) {
    71  	raceacquire(addr)
    72  }
    73  
    74  //go:linkname race_Acquire internal/race.Acquire
    75  //go:nosplit
    76  func race_Acquire(addr unsafe.Pointer) {
    77  	RaceAcquire(addr)
    78  }
    79  
    80  // RaceRelease performs a release operation on addr that
    81  // can synchronize with a later RaceAcquire on addr.
    82  //
    83  // In terms of the C memory model, RaceRelease is equivalent to
    84  // atomic_store(memory_order_release).
    85  //
    86  //go:nosplit
    87  func RaceRelease(addr unsafe.Pointer) {
    88  	racerelease(addr)
    89  }
    90  
    91  //go:linkname race_Release internal/race.Release
    92  //go:nosplit
    93  func race_Release(addr unsafe.Pointer) {
    94  	RaceRelease(addr)
    95  }
    96  
    97  // RaceReleaseMerge is like RaceRelease, but also establishes a happens-before
    98  // relation with the preceding RaceRelease or RaceReleaseMerge on addr.
    99  //
   100  // In terms of the C memory model, RaceReleaseMerge is equivalent to
   101  // atomic_exchange(memory_order_release).
   102  //
   103  //go:nosplit
   104  func RaceReleaseMerge(addr unsafe.Pointer) {
   105  	racereleasemerge(addr)
   106  }
   107  
   108  //go:linkname race_ReleaseMerge internal/race.ReleaseMerge
   109  //go:nosplit
   110  func race_ReleaseMerge(addr unsafe.Pointer) {
   111  	RaceReleaseMerge(addr)
   112  }
   113  
   114  // RaceDisable disables handling of race synchronization events in the current goroutine.
   115  // Handling is re-enabled with RaceEnable. RaceDisable/RaceEnable can be nested.
   116  // Non-synchronization events (memory accesses, function entry/exit) still affect
   117  // the race detector.
   118  //
   119  //go:nosplit
   120  func RaceDisable() {
   121  	gp := getg()
   122  	if gp.raceignore == 0 {
   123  		racecall(&__tsan_go_ignore_sync_begin, gp.racectx, 0, 0, 0)
   124  	}
   125  	gp.raceignore++
   126  }
   127  
   128  //go:linkname race_Disable internal/race.Disable
   129  //go:nosplit
   130  func race_Disable() {
   131  	RaceDisable()
   132  }
   133  
   134  // RaceEnable re-enables handling of race events in the current goroutine.
   135  //
   136  //go:nosplit
   137  func RaceEnable() {
   138  	gp := getg()
   139  	gp.raceignore--
   140  	if gp.raceignore == 0 {
   141  		racecall(&__tsan_go_ignore_sync_end, gp.racectx, 0, 0, 0)
   142  	}
   143  }
   144  
   145  //go:linkname race_Enable internal/race.Enable
   146  //go:nosplit
   147  func race_Enable() {
   148  	RaceEnable()
   149  }
   150  
   151  // Private interface for the runtime.
   152  
   153  const raceenabled = true
   154  
   155  // For all functions accepting callerpc and pc,
   156  // callerpc is a return PC of the function that calls this function,
   157  // pc is start PC of the function that calls this function.
   158  func raceReadObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) {
   159  	kind := t.Kind_ & abi.KindMask
   160  	if kind == abi.Array || kind == abi.Struct {
   161  		// for composite objects we have to read every address
   162  		// because a write might happen to any subobject.
   163  		racereadrangepc(addr, t.Size_, callerpc, pc)
   164  	} else {
   165  		// for non-composite objects we can read just the start
   166  		// address, as any write must write the first byte.
   167  		racereadpc(addr, callerpc, pc)
   168  	}
   169  }
   170  
   171  //go:linkname race_ReadObjectPC internal/race.ReadObjectPC
   172  func race_ReadObjectPC(t *abi.Type, addr unsafe.Pointer, callerpc, pc uintptr) {
   173  	raceReadObjectPC(t, addr, callerpc, pc)
   174  }
   175  
   176  func raceWriteObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) {
   177  	kind := t.Kind_ & abi.KindMask
   178  	if kind == abi.Array || kind == abi.Struct {
   179  		// for composite objects we have to write every address
   180  		// because a write might happen to any subobject.
   181  		racewriterangepc(addr, t.Size_, callerpc, pc)
   182  	} else {
   183  		// for non-composite objects we can write just the start
   184  		// address, as any write must write the first byte.
   185  		racewritepc(addr, callerpc, pc)
   186  	}
   187  }
   188  
   189  //go:linkname race_WriteObjectPC internal/race.WriteObjectPC
   190  func race_WriteObjectPC(t *abi.Type, addr unsafe.Pointer, callerpc, pc uintptr) {
   191  	raceWriteObjectPC(t, addr, callerpc, pc)
   192  }
   193  
   194  //go:noescape
   195  func racereadpc(addr unsafe.Pointer, callpc, pc uintptr)
   196  
   197  //go:noescape
   198  func racewritepc(addr unsafe.Pointer, callpc, pc uintptr)
   199  
   200  //go:linkname race_ReadPC internal/race.ReadPC
   201  func race_ReadPC(addr unsafe.Pointer, callerpc, pc uintptr) {
   202  	racereadpc(addr, callerpc, pc)
   203  }
   204  
   205  //go:linkname race_WritePC internal/race.WritePC
   206  func race_WritePC(addr unsafe.Pointer, callerpc, pc uintptr) {
   207  	racewritepc(addr, callerpc, pc)
   208  }
   209  
   210  type symbolizeCodeContext struct {
   211  	pc   uintptr
   212  	fn   *byte
   213  	file *byte
   214  	line uintptr
   215  	off  uintptr
   216  	res  uintptr
   217  }
   218  
   219  var qq = [...]byte{'?', '?', 0}
   220  var dash = [...]byte{'-', 0}
   221  
   222  const (
   223  	raceGetProcCmd = iota
   224  	raceSymbolizeCodeCmd
   225  	raceSymbolizeDataCmd
   226  )
   227  
   228  // Callback from C into Go, runs on g0.
   229  func racecallback(cmd uintptr, ctx unsafe.Pointer) {
   230  	switch cmd {
   231  	case raceGetProcCmd:
   232  		throw("should have been handled by racecallbackthunk")
   233  	case raceSymbolizeCodeCmd:
   234  		raceSymbolizeCode((*symbolizeCodeContext)(ctx))
   235  	case raceSymbolizeDataCmd:
   236  		raceSymbolizeData((*symbolizeDataContext)(ctx))
   237  	default:
   238  		throw("unknown command")
   239  	}
   240  }
   241  
   242  // raceSymbolizeCode reads ctx.pc and populates the rest of *ctx with
   243  // information about the code at that pc.
   244  //
   245  // The race detector has already subtracted 1 from pcs, so they point to the last
   246  // byte of call instructions (including calls to runtime.racewrite and friends).
   247  //
   248  // If the incoming pc is part of an inlined function, *ctx is populated
   249  // with information about the inlined function, and on return ctx.pc is set
   250  // to a pc in the logically containing function. (The race detector should call this
   251  // function again with that pc.)
   252  //
   253  // If the incoming pc is not part of an inlined function, the return pc is unchanged.
   254  func raceSymbolizeCode(ctx *symbolizeCodeContext) {
   255  	pc := ctx.pc
   256  	fi := findfunc(pc)
   257  	if fi.valid() {
   258  		u, uf := newInlineUnwinder(fi, pc)
   259  		for ; uf.valid(); uf = u.next(uf) {
   260  			sf := u.srcFunc(uf)
   261  			if sf.funcID == abi.FuncIDWrapper && u.isInlined(uf) {
   262  				// Ignore wrappers, unless we're at the outermost frame of u.
   263  				// A non-inlined wrapper frame always means we have a physical
   264  				// frame consisting entirely of wrappers, in which case we'll
   265  				// take an outermost wrapper over nothing.
   266  				continue
   267  			}
   268  
   269  			name := sf.name()
   270  			file, line := u.fileLine(uf)
   271  			if line == 0 {
   272  				// Failure to symbolize
   273  				continue
   274  			}
   275  			ctx.fn = &bytes(name)[0] // assume NUL-terminated
   276  			ctx.line = uintptr(line)
   277  			ctx.file = &bytes(file)[0] // assume NUL-terminated
   278  			ctx.off = pc - fi.entry()
   279  			ctx.res = 1
   280  			if u.isInlined(uf) {
   281  				// Set ctx.pc to the "caller" so the race detector calls this again
   282  				// to further unwind.
   283  				uf = u.next(uf)
   284  				ctx.pc = uf.pc
   285  			}
   286  			return
   287  		}
   288  	}
   289  	ctx.fn = &qq[0]
   290  	ctx.file = &dash[0]
   291  	ctx.line = 0
   292  	ctx.off = ctx.pc
   293  	ctx.res = 1
   294  }
   295  
   296  type symbolizeDataContext struct {
   297  	addr  uintptr
   298  	heap  uintptr
   299  	start uintptr
   300  	size  uintptr
   301  	name  *byte
   302  	file  *byte
   303  	line  uintptr
   304  	res   uintptr
   305  }
   306  
   307  func raceSymbolizeData(ctx *symbolizeDataContext) {
   308  	if base, span, _ := findObject(ctx.addr, 0, 0); base != 0 {
   309  		// TODO: Does this need to handle malloc headers?
   310  		ctx.heap = 1
   311  		ctx.start = base
   312  		ctx.size = span.elemsize
   313  		ctx.res = 1
   314  	}
   315  }
   316  
   317  // Race runtime functions called via runtime·racecall.
   318  //
   319  //go:linkname __tsan_init __tsan_init
   320  var __tsan_init byte
   321  
   322  //go:linkname __tsan_fini __tsan_fini
   323  var __tsan_fini byte
   324  
   325  //go:linkname __tsan_proc_create __tsan_proc_create
   326  var __tsan_proc_create byte
   327  
   328  //go:linkname __tsan_proc_destroy __tsan_proc_destroy
   329  var __tsan_proc_destroy byte
   330  
   331  //go:linkname __tsan_map_shadow __tsan_map_shadow
   332  var __tsan_map_shadow byte
   333  
   334  //go:linkname __tsan_finalizer_goroutine __tsan_finalizer_goroutine
   335  var __tsan_finalizer_goroutine byte
   336  
   337  //go:linkname __tsan_go_start __tsan_go_start
   338  var __tsan_go_start byte
   339  
   340  //go:linkname __tsan_go_end __tsan_go_end
   341  var __tsan_go_end byte
   342  
   343  //go:linkname __tsan_malloc __tsan_malloc
   344  var __tsan_malloc byte
   345  
   346  //go:linkname __tsan_free __tsan_free
   347  var __tsan_free byte
   348  
   349  //go:linkname __tsan_acquire __tsan_acquire
   350  var __tsan_acquire byte
   351  
   352  //go:linkname __tsan_release __tsan_release
   353  var __tsan_release byte
   354  
   355  //go:linkname __tsan_release_acquire __tsan_release_acquire
   356  var __tsan_release_acquire byte
   357  
   358  //go:linkname __tsan_release_merge __tsan_release_merge
   359  var __tsan_release_merge byte
   360  
   361  //go:linkname __tsan_go_ignore_sync_begin __tsan_go_ignore_sync_begin
   362  var __tsan_go_ignore_sync_begin byte
   363  
   364  //go:linkname __tsan_go_ignore_sync_end __tsan_go_ignore_sync_end
   365  var __tsan_go_ignore_sync_end byte
   366  
   367  //go:linkname __tsan_report_count __tsan_report_count
   368  var __tsan_report_count byte
   369  
   370  // Mimic what cmd/cgo would do.
   371  //
   372  //go:cgo_import_static __tsan_init
   373  //go:cgo_import_static __tsan_fini
   374  //go:cgo_import_static __tsan_proc_create
   375  //go:cgo_import_static __tsan_proc_destroy
   376  //go:cgo_import_static __tsan_map_shadow
   377  //go:cgo_import_static __tsan_finalizer_goroutine
   378  //go:cgo_import_static __tsan_go_start
   379  //go:cgo_import_static __tsan_go_end
   380  //go:cgo_import_static __tsan_malloc
   381  //go:cgo_import_static __tsan_free
   382  //go:cgo_import_static __tsan_acquire
   383  //go:cgo_import_static __tsan_release
   384  //go:cgo_import_static __tsan_release_acquire
   385  //go:cgo_import_static __tsan_release_merge
   386  //go:cgo_import_static __tsan_go_ignore_sync_begin
   387  //go:cgo_import_static __tsan_go_ignore_sync_end
   388  //go:cgo_import_static __tsan_report_count
   389  
   390  // These are called from race_amd64.s.
   391  //
   392  //go:cgo_import_static __tsan_read
   393  //go:cgo_import_static __tsan_read_pc
   394  //go:cgo_import_static __tsan_read_range
   395  //go:cgo_import_static __tsan_write
   396  //go:cgo_import_static __tsan_write_pc
   397  //go:cgo_import_static __tsan_write_range
   398  //go:cgo_import_static __tsan_func_enter
   399  //go:cgo_import_static __tsan_func_exit
   400  
   401  //go:cgo_import_static __tsan_go_atomic32_load
   402  //go:cgo_import_static __tsan_go_atomic64_load
   403  //go:cgo_import_static __tsan_go_atomic32_store
   404  //go:cgo_import_static __tsan_go_atomic64_store
   405  //go:cgo_import_static __tsan_go_atomic32_exchange
   406  //go:cgo_import_static __tsan_go_atomic64_exchange
   407  //go:cgo_import_static __tsan_go_atomic32_fetch_add
   408  //go:cgo_import_static __tsan_go_atomic64_fetch_add
   409  //go:cgo_import_static __tsan_go_atomic32_fetch_and
   410  //go:cgo_import_static __tsan_go_atomic64_fetch_and
   411  //go:cgo_import_static __tsan_go_atomic32_fetch_or
   412  //go:cgo_import_static __tsan_go_atomic64_fetch_or
   413  //go:cgo_import_static __tsan_go_atomic32_compare_exchange
   414  //go:cgo_import_static __tsan_go_atomic64_compare_exchange
   415  
   416  // start/end of global data (data+bss).
   417  var racedatastart uintptr
   418  var racedataend uintptr
   419  
   420  // start/end of heap for race_amd64.s
   421  var racearenastart uintptr
   422  var racearenaend uintptr
   423  
   424  func racefuncenter(callpc uintptr)
   425  func racefuncenterfp(fp uintptr)
   426  func racefuncexit()
   427  func raceread(addr uintptr)
   428  func racewrite(addr uintptr)
   429  func racereadrange(addr, size uintptr)
   430  func racewriterange(addr, size uintptr)
   431  func racereadrangepc1(addr, size, pc uintptr)
   432  func racewriterangepc1(addr, size, pc uintptr)
   433  func racecallbackthunk(uintptr)
   434  
   435  // racecall allows calling an arbitrary function fn from C race runtime
   436  // with up to 4 uintptr arguments.
   437  func racecall(fn *byte, arg0, arg1, arg2, arg3 uintptr)
   438  
   439  // checks if the address has shadow (i.e. heap or data/bss).
   440  //
   441  //go:nosplit
   442  func isvalidaddr(addr unsafe.Pointer) bool {
   443  	return racearenastart <= uintptr(addr) && uintptr(addr) < racearenaend ||
   444  		racedatastart <= uintptr(addr) && uintptr(addr) < racedataend
   445  }
   446  
   447  //go:nosplit
   448  func raceinit() (gctx, pctx uintptr) {
   449  	lockInit(&raceFiniLock, lockRankRaceFini)
   450  
   451  	// On most machines, cgo is required to initialize libc, which is used by race runtime.
   452  	if !iscgo && GOOS != "darwin" {
   453  		throw("raceinit: race build must use cgo")
   454  	}
   455  
   456  	racecall(&__tsan_init, uintptr(unsafe.Pointer(&gctx)), uintptr(unsafe.Pointer(&pctx)), abi.FuncPCABI0(racecallbackthunk), 0)
   457  
   458  	start := ^uintptr(0)
   459  	end := uintptr(0)
   460  	if start > firstmoduledata.noptrdata {
   461  		start = firstmoduledata.noptrdata
   462  	}
   463  	if start > firstmoduledata.data {
   464  		start = firstmoduledata.data
   465  	}
   466  	if start > firstmoduledata.noptrbss {
   467  		start = firstmoduledata.noptrbss
   468  	}
   469  	if start > firstmoduledata.bss {
   470  		start = firstmoduledata.bss
   471  	}
   472  	if end < firstmoduledata.enoptrdata {
   473  		end = firstmoduledata.enoptrdata
   474  	}
   475  	if end < firstmoduledata.edata {
   476  		end = firstmoduledata.edata
   477  	}
   478  	if end < firstmoduledata.enoptrbss {
   479  		end = firstmoduledata.enoptrbss
   480  	}
   481  	if end < firstmoduledata.ebss {
   482  		end = firstmoduledata.ebss
   483  	}
   484  	// Use exact bounds for boundary check in racecalladdr. See issue 73483.
   485  	racedatastart = start
   486  	racedataend = end
   487  	// Round data segment to page boundaries for race detector (TODO: still needed?)
   488  	start = alignDown(start, _PageSize)
   489  	end = alignUp(end, _PageSize)
   490  	racecall(&__tsan_map_shadow, start, end-start, 0, 0)
   491  
   492  	return
   493  }
   494  
   495  //go:nosplit
   496  func racefini() {
   497  	// racefini() can only be called once to avoid races.
   498  	// This eventually (via __tsan_fini) calls C.exit which has
   499  	// undefined behavior if called more than once. If the lock is
   500  	// already held it's assumed that the first caller exits the program
   501  	// so other calls can hang forever without an issue.
   502  	lock(&raceFiniLock)
   503  
   504  	// __tsan_fini will run C atexit functions and C++ destructors,
   505  	// which can theoretically call back into Go.
   506  	// Tell the scheduler we entering external code.
   507  	entersyscall()
   508  
   509  	// We're entering external code that may call ExitProcess on
   510  	// Windows.
   511  	osPreemptExtEnter(getg().m)
   512  
   513  	racecall(&__tsan_fini, 0, 0, 0, 0)
   514  }
   515  
   516  //go:nosplit
   517  func raceproccreate() uintptr {
   518  	var ctx uintptr
   519  	racecall(&__tsan_proc_create, uintptr(unsafe.Pointer(&ctx)), 0, 0, 0)
   520  	return ctx
   521  }
   522  
   523  //go:nosplit
   524  func raceprocdestroy(ctx uintptr) {
   525  	racecall(&__tsan_proc_destroy, ctx, 0, 0, 0)
   526  }
   527  
   528  //go:nosplit
   529  func racemapshadow(addr unsafe.Pointer, size uintptr) {
   530  	if racearenastart == 0 {
   531  		racearenastart = uintptr(addr)
   532  	}
   533  	if racearenaend < uintptr(addr)+size {
   534  		racearenaend = uintptr(addr) + size
   535  	}
   536  	racecall(&__tsan_map_shadow, uintptr(addr), size, 0, 0)
   537  }
   538  
   539  //go:nosplit
   540  func racemalloc(p unsafe.Pointer, sz uintptr) {
   541  	racecall(&__tsan_malloc, 0, 0, uintptr(p), sz)
   542  }
   543  
   544  //go:nosplit
   545  func racefree(p unsafe.Pointer, sz uintptr) {
   546  	racecall(&__tsan_free, uintptr(p), sz, 0, 0)
   547  }
   548  
   549  //go:nosplit
   550  func racegostart(pc uintptr) uintptr {
   551  	gp := getg()
   552  	var spawng *g
   553  	if gp.m.curg != nil {
   554  		spawng = gp.m.curg
   555  	} else {
   556  		spawng = gp
   557  	}
   558  
   559  	var racectx uintptr
   560  	racecall(&__tsan_go_start, spawng.racectx, uintptr(unsafe.Pointer(&racectx)), pc, 0)
   561  	return racectx
   562  }
   563  
   564  //go:nosplit
   565  func racegoend() {
   566  	racecall(&__tsan_go_end, getg().racectx, 0, 0, 0)
   567  }
   568  
   569  //go:nosplit
   570  func racectxstart(pc, spawnctx uintptr) uintptr {
   571  	var racectx uintptr
   572  	racecall(&__tsan_go_start, spawnctx, uintptr(unsafe.Pointer(&racectx)), pc, 0)
   573  	return racectx
   574  }
   575  
   576  //go:nosplit
   577  func racectxend(racectx uintptr) {
   578  	racecall(&__tsan_go_end, racectx, 0, 0, 0)
   579  }
   580  
   581  //go:nosplit
   582  func racewriterangepc(addr unsafe.Pointer, sz, callpc, pc uintptr) {
   583  	gp := getg()
   584  	if gp != gp.m.curg {
   585  		// The call is coming from manual instrumentation of Go code running on g0/gsignal.
   586  		// Not interesting.
   587  		return
   588  	}
   589  	if callpc != 0 {
   590  		racefuncenter(callpc)
   591  	}
   592  	racewriterangepc1(uintptr(addr), sz, pc)
   593  	if callpc != 0 {
   594  		racefuncexit()
   595  	}
   596  }
   597  
   598  //go:nosplit
   599  func racereadrangepc(addr unsafe.Pointer, sz, callpc, pc uintptr) {
   600  	gp := getg()
   601  	if gp != gp.m.curg {
   602  		// The call is coming from manual instrumentation of Go code running on g0/gsignal.
   603  		// Not interesting.
   604  		return
   605  	}
   606  	if callpc != 0 {
   607  		racefuncenter(callpc)
   608  	}
   609  	racereadrangepc1(uintptr(addr), sz, pc)
   610  	if callpc != 0 {
   611  		racefuncexit()
   612  	}
   613  }
   614  
   615  //go:nosplit
   616  func raceacquire(addr unsafe.Pointer) {
   617  	raceacquireg(getg(), addr)
   618  }
   619  
   620  //go:nosplit
   621  func raceacquireg(gp *g, addr unsafe.Pointer) {
   622  	if getg().raceignore != 0 || !isvalidaddr(addr) {
   623  		return
   624  	}
   625  	racecall(&__tsan_acquire, gp.racectx, uintptr(addr), 0, 0)
   626  }
   627  
   628  //go:nosplit
   629  func raceacquirectx(racectx uintptr, addr unsafe.Pointer) {
   630  	if !isvalidaddr(addr) {
   631  		return
   632  	}
   633  	racecall(&__tsan_acquire, racectx, uintptr(addr), 0, 0)
   634  }
   635  
   636  //go:nosplit
   637  func racerelease(addr unsafe.Pointer) {
   638  	racereleaseg(getg(), addr)
   639  }
   640  
   641  //go:nosplit
   642  func racereleaseg(gp *g, addr unsafe.Pointer) {
   643  	if getg().raceignore != 0 || !isvalidaddr(addr) {
   644  		return
   645  	}
   646  	racecall(&__tsan_release, gp.racectx, uintptr(addr), 0, 0)
   647  }
   648  
   649  //go:nosplit
   650  func racereleaseacquire(addr unsafe.Pointer) {
   651  	racereleaseacquireg(getg(), addr)
   652  }
   653  
   654  //go:nosplit
   655  func racereleaseacquireg(gp *g, addr unsafe.Pointer) {
   656  	if getg().raceignore != 0 || !isvalidaddr(addr) {
   657  		return
   658  	}
   659  	racecall(&__tsan_release_acquire, gp.racectx, uintptr(addr), 0, 0)
   660  }
   661  
   662  //go:nosplit
   663  func racereleasemerge(addr unsafe.Pointer) {
   664  	racereleasemergeg(getg(), addr)
   665  }
   666  
   667  //go:nosplit
   668  func racereleasemergeg(gp *g, addr unsafe.Pointer) {
   669  	if getg().raceignore != 0 || !isvalidaddr(addr) {
   670  		return
   671  	}
   672  	racecall(&__tsan_release_merge, gp.racectx, uintptr(addr), 0, 0)
   673  }
   674  
   675  //go:nosplit
   676  func racefingo() {
   677  	racecall(&__tsan_finalizer_goroutine, getg().racectx, 0, 0, 0)
   678  }
   679  
   680  // The declarations below generate ABI wrappers for functions
   681  // implemented in assembly in this package but declared in another
   682  // package.
   683  
   684  //go:linkname abigen_sync_atomic_LoadInt32 sync/atomic.LoadInt32
   685  func abigen_sync_atomic_LoadInt32(addr *int32) (val int32)
   686  
   687  //go:linkname abigen_sync_atomic_LoadInt64 sync/atomic.LoadInt64
   688  func abigen_sync_atomic_LoadInt64(addr *int64) (val int64)
   689  
   690  //go:linkname abigen_sync_atomic_LoadUint32 sync/atomic.LoadUint32
   691  func abigen_sync_atomic_LoadUint32(addr *uint32) (val uint32)
   692  
   693  //go:linkname abigen_sync_atomic_LoadUint64 sync/atomic.LoadUint64
   694  func abigen_sync_atomic_LoadUint64(addr *uint64) (val uint64)
   695  
   696  //go:linkname abigen_sync_atomic_LoadUintptr sync/atomic.LoadUintptr
   697  func abigen_sync_atomic_LoadUintptr(addr *uintptr) (val uintptr)
   698  
   699  //go:linkname abigen_sync_atomic_LoadPointer sync/atomic.LoadPointer
   700  func abigen_sync_atomic_LoadPointer(addr *unsafe.Pointer) (val unsafe.Pointer)
   701  
   702  //go:linkname abigen_sync_atomic_StoreInt32 sync/atomic.StoreInt32
   703  func abigen_sync_atomic_StoreInt32(addr *int32, val int32)
   704  
   705  //go:linkname abigen_sync_atomic_StoreInt64 sync/atomic.StoreInt64
   706  func abigen_sync_atomic_StoreInt64(addr *int64, val int64)
   707  
   708  //go:linkname abigen_sync_atomic_StoreUint32 sync/atomic.StoreUint32
   709  func abigen_sync_atomic_StoreUint32(addr *uint32, val uint32)
   710  
   711  //go:linkname abigen_sync_atomic_StoreUint64 sync/atomic.StoreUint64
   712  func abigen_sync_atomic_StoreUint64(addr *uint64, val uint64)
   713  
   714  //go:linkname abigen_sync_atomic_SwapInt32 sync/atomic.SwapInt32
   715  func abigen_sync_atomic_SwapInt32(addr *int32, new int32) (old int32)
   716  
   717  //go:linkname abigen_sync_atomic_SwapInt64 sync/atomic.SwapInt64
   718  func abigen_sync_atomic_SwapInt64(addr *int64, new int64) (old int64)
   719  
   720  //go:linkname abigen_sync_atomic_SwapUint32 sync/atomic.SwapUint32
   721  func abigen_sync_atomic_SwapUint32(addr *uint32, new uint32) (old uint32)
   722  
   723  //go:linkname abigen_sync_atomic_SwapUint64 sync/atomic.SwapUint64
   724  func abigen_sync_atomic_SwapUint64(addr *uint64, new uint64) (old uint64)
   725  
   726  //go:linkname abigen_sync_atomic_AddInt32 sync/atomic.AddInt32
   727  func abigen_sync_atomic_AddInt32(addr *int32, delta int32) (new int32)
   728  
   729  //go:linkname abigen_sync_atomic_AddUint32 sync/atomic.AddUint32
   730  func abigen_sync_atomic_AddUint32(addr *uint32, delta uint32) (new uint32)
   731  
   732  //go:linkname abigen_sync_atomic_AddInt64 sync/atomic.AddInt64
   733  func abigen_sync_atomic_AddInt64(addr *int64, delta int64) (new int64)
   734  
   735  //go:linkname abigen_sync_atomic_AddUint64 sync/atomic.AddUint64
   736  func abigen_sync_atomic_AddUint64(addr *uint64, delta uint64) (new uint64)
   737  
   738  //go:linkname abigen_sync_atomic_AddUintptr sync/atomic.AddUintptr
   739  func abigen_sync_atomic_AddUintptr(addr *uintptr, delta uintptr) (new uintptr)
   740  
   741  //go:linkname abigen_sync_atomic_AndInt32 sync/atomic.AndInt32
   742  func abigen_sync_atomic_AndInt32(addr *int32, mask int32) (old int32)
   743  
   744  //go:linkname abigen_sync_atomic_AndUint32 sync/atomic.AndUint32
   745  func abigen_sync_atomic_AndUint32(addr *uint32, mask uint32) (old uint32)
   746  
   747  //go:linkname abigen_sync_atomic_AndInt64 sync/atomic.AndInt64
   748  func abigen_sync_atomic_AndInt64(addr *int64, mask int64) (old int64)
   749  
   750  //go:linkname abigen_sync_atomic_AndUint64 sync/atomic.AndUint64
   751  func abigen_sync_atomic_AndUint64(addr *uint64, mask uint64) (old uint64)
   752  
   753  //go:linkname abigen_sync_atomic_AndUintptr sync/atomic.AndUintptr
   754  func abigen_sync_atomic_AndUintptr(addr *uintptr, mask uintptr) (old uintptr)
   755  
   756  //go:linkname abigen_sync_atomic_OrInt32 sync/atomic.OrInt32
   757  func abigen_sync_atomic_OrInt32(addr *int32, mask int32) (old int32)
   758  
   759  //go:linkname abigen_sync_atomic_OrUint32 sync/atomic.OrUint32
   760  func abigen_sync_atomic_OrUint32(addr *uint32, mask uint32) (old uint32)
   761  
   762  //go:linkname abigen_sync_atomic_OrInt64 sync/atomic.OrInt64
   763  func abigen_sync_atomic_OrInt64(addr *int64, mask int64) (old int64)
   764  
   765  //go:linkname abigen_sync_atomic_OrUint64 sync/atomic.OrUint64
   766  func abigen_sync_atomic_OrUint64(addr *uint64, mask uint64) (old uint64)
   767  
   768  //go:linkname abigen_sync_atomic_OrUintptr sync/atomic.OrUintptr
   769  func abigen_sync_atomic_OrUintptr(addr *uintptr, mask uintptr) (old uintptr)
   770  
   771  //go:linkname abigen_sync_atomic_CompareAndSwapInt32 sync/atomic.CompareAndSwapInt32
   772  func abigen_sync_atomic_CompareAndSwapInt32(addr *int32, old, new int32) (swapped bool)
   773  
   774  //go:linkname abigen_sync_atomic_CompareAndSwapInt64 sync/atomic.CompareAndSwapInt64
   775  func abigen_sync_atomic_CompareAndSwapInt64(addr *int64, old, new int64) (swapped bool)
   776  
   777  //go:linkname abigen_sync_atomic_CompareAndSwapUint32 sync/atomic.CompareAndSwapUint32
   778  func abigen_sync_atomic_CompareAndSwapUint32(addr *uint32, old, new uint32) (swapped bool)
   779  
   780  //go:linkname abigen_sync_atomic_CompareAndSwapUint64 sync/atomic.CompareAndSwapUint64
   781  func abigen_sync_atomic_CompareAndSwapUint64(addr *uint64, old, new uint64) (swapped bool)
   782
View as plain text