writebarrier.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssacompile
     6  
     7  import (
     8  	"internal/buildcfg"
     9  
    10  	"cmd/compile/internal/reflectdata"
    11  	"cmd/compile/internal/ssa"
    12  	"cmd/compile/internal/ssa/block"
    13  	"cmd/compile/internal/ssa/ssaop"
    14  	"cmd/compile/internal/types"
    15  	"cmd/internal/obj"
    16  	"cmd/internal/objabi"
    17  	"cmd/internal/src"
    18  )
    19  
    20  // mightBeHeapPointer reports whether v might point to the heap.
    21  // v must have pointer type.
    22  func mightBeHeapPointer(v *ssa.Value) bool {
    23  	if IsGlobalAddr(v) {
    24  		return false
    25  	}
    26  	return true
    27  }
    28  
    29  // mightContainHeapPointer reports whether the data currently at addresses
    30  // [ptr,ptr+size) might contain heap pointers. "currently" means at memory state mem.
    31  // zeroes contains ZeroRegion data to help make that decision (see computeZeroMap).
    32  func mightContainHeapPointer(ptr *ssa.Value, size int64, mem *ssa.Value, zeroes map[ssa.ID]ssa.ZeroRegion) bool {
    33  	if IsReadOnlyGlobalAddr(ptr) {
    34  		// The read-only globals section cannot contain any heap pointers.
    35  		return false
    36  	}
    37  
    38  	// See if we can prove that the queried memory is all zero.
    39  
    40  	// Find base pointer and offset. Hopefully, the base is the result of a new(T).
    41  	var off int64
    42  	for ptr.Op == ssaop.OpOffPtr {
    43  		off += ptr.AuxInt
    44  		ptr = ptr.Args[0]
    45  	}
    46  
    47  	ptrSize := ptr.Block.Func.Config.PtrSize
    48  	if off%ptrSize != 0 {
    49  		return true // see issue 61187
    50  	}
    51  	if size%ptrSize != 0 {
    52  		ptr.Fatalf("unaligned pointer write")
    53  	}
    54  	if off < 0 || off+size > 64*ptrSize {
    55  		// memory range goes off end of tracked offsets
    56  		return true
    57  	}
    58  	z := zeroes[mem.ID]
    59  	if ptr != z.Base {
    60  		// This isn't the object we know about at this memory state.
    61  		return true
    62  	}
    63  	// Mask of bits we're asking about
    64  	m := (uint64(1)<<(size/ptrSize) - 1) << (off / ptrSize)
    65  
    66  	if z.Mask&m == m {
    67  		// All locations are known to be zero, so no heap pointers.
    68  		return false
    69  	}
    70  	return true
    71  }
    72  
    73  // needwb reports whether we need write barrier for store op v.
    74  // v must be Store/Move/Zero.
    75  // zeroes provides known zero information (keyed by ID of memory-type values).
    76  func needwb(v *ssa.Value, zeroes map[ssa.ID]ssa.ZeroRegion) bool {
    77  	t, ok := v.Aux.(*types.Type)
    78  	if !ok {
    79  		v.Fatalf("store aux is not a type: %s", v.LongString())
    80  	}
    81  	if !t.HasPointers() {
    82  		return false
    83  	}
    84  	dst := v.Args[0]
    85  	if ssa.IsStackAddr(dst) {
    86  		return false // writes into the stack don't need write barrier
    87  	}
    88  	// If we're writing to a place that might have heap pointers, we need
    89  	// the write barrier.
    90  	if mightContainHeapPointer(dst, t.Size(), v.MemoryArg(), zeroes) {
    91  		return true
    92  	}
    93  	// Lastly, check if the values we're writing might be heap pointers.
    94  	// If they aren't, we don't need a write barrier.
    95  	switch v.Op {
    96  	case ssaop.OpStore:
    97  		if !mightBeHeapPointer(v.Args[1]) {
    98  			return false
    99  		}
   100  	case ssaop.OpZero:
   101  		return false // nil is not a heap pointer
   102  	case ssaop.OpMove:
   103  		if !mightContainHeapPointer(v.Args[1], t.Size(), v.Args[2], zeroes) {
   104  			return false
   105  		}
   106  	default:
   107  		v.Fatalf("store op unknown: %s", v.LongString())
   108  	}
   109  	return true
   110  }
   111  
   112  // needWBsrc reports whether GC needs to see v when it is the source of a store.
   113  func needWBsrc(v *ssa.Value) bool {
   114  	return !IsGlobalAddr(v)
   115  }
   116  
   117  // needWBdst reports whether GC needs to see what used to be in *ptr when ptr is
   118  // the target of a pointer store.
   119  func needWBdst(ptr, mem *ssa.Value, zeroes map[ssa.ID]ssa.ZeroRegion) bool {
   120  	// Detect storing to zeroed memory.
   121  	var off int64
   122  	for ptr.Op == ssaop.OpOffPtr {
   123  		off += ptr.AuxInt
   124  		ptr = ptr.Args[0]
   125  	}
   126  	ptrSize := ptr.Block.Func.Config.PtrSize
   127  	if off%ptrSize != 0 {
   128  		return true // see issue 61187
   129  	}
   130  	if off < 0 || off >= 64*ptrSize {
   131  		// write goes off end of tracked offsets
   132  		return true
   133  	}
   134  	z := zeroes[mem.ID]
   135  	if ptr != z.Base {
   136  		return true
   137  	}
   138  	// If destination is known to be zeroed, we don't need the write barrier
   139  	// to record the old value in *ptr.
   140  	return z.Mask>>uint(off/ptrSize)&1 == 0
   141  }
   142  
   143  // writebarrier pass inserts write barriers for store ops (Store, Move, Zero)
   144  // when necessary (the condition above). It rewrites store ops to branches
   145  // and runtime calls, like
   146  //
   147  //	if writeBarrier.enabled {
   148  //		buf := gcWriteBarrier2()	// Not a regular Go call
   149  //		buf[0] = val
   150  //		buf[1] = *ptr
   151  //	}
   152  //	*ptr = val
   153  //
   154  // A sequence of WB stores for many pointer fields of a single type will
   155  // be emitted together, with a single branch.
   156  func writebarrier(f *ssa.Func) {
   157  	if !f.Fe.UseWriteBarrier() {
   158  		return
   159  	}
   160  
   161  	// Number of write buffer entries we can request at once.
   162  	// Must match runtime/mwbbuf.go:wbMaxEntriesPerCall.
   163  	// It must also match the number of instances of runtime.gcWriteBarrier{X}.
   164  	const maxEntries = 8
   165  
   166  	var sb, sp, wbaddr, const0 *ssa.Value
   167  	var cgoCheckPtrWrite, cgoCheckMemmove *obj.LSym
   168  	var wbZero, wbMove *obj.LSym
   169  	var stores, after []*ssa.Value
   170  	var sset, sset2 *ssa.SparseSet
   171  	var storeNumber []int32
   172  
   173  	// Compute map from a value to the SelectN [1] value that uses it.
   174  	select1 := f.Cache.AllocValueSlice(f.NumValues())
   175  	defer func() { f.Cache.FreeValueSlice(select1) }()
   176  	for _, b := range f.Blocks {
   177  		for _, v := range b.Values {
   178  			if v.Op != ssaop.OpSelectN {
   179  				continue
   180  			}
   181  			if v.AuxInt != 1 {
   182  				continue
   183  			}
   184  			select1[v.Args[0].ID] = v
   185  		}
   186  	}
   187  
   188  	zeroes := f.ComputeZeroMap(select1)
   189  	for _, b := range f.Blocks { // range loop is safe since the blocks we added contain no stores to expand
   190  		// first, identify all the stores that need to insert a write barrier.
   191  		// mark them with WB ops temporarily. record presence of WB ops.
   192  		nWBops := 0 // count of temporarily created WB ops remaining to be rewritten in the current block
   193  		for _, v := range b.Values {
   194  			switch v.Op {
   195  			case ssaop.OpStore, ssaop.OpMove, ssaop.OpZero:
   196  				if needwb(v, zeroes) {
   197  					switch v.Op {
   198  					case ssaop.OpStore:
   199  						v.Op = ssaop.OpStoreWB
   200  					case ssaop.OpMove:
   201  						v.Op = ssaop.OpMoveWB
   202  					case ssaop.OpZero:
   203  						v.Op = ssaop.OpZeroWB
   204  					}
   205  					nWBops++
   206  				}
   207  			}
   208  		}
   209  		if nWBops == 0 {
   210  			continue
   211  		}
   212  
   213  		if wbaddr == nil {
   214  			// lazily initialize global values for write barrier test and calls
   215  			// find SB and SP values in entry block
   216  			initpos := f.Entry.Pos
   217  			sp, sb = f.SpSb()
   218  			wbsym := f.Fe.Syslook("writeBarrier")
   219  			wbaddr = f.Entry.NewValue1A(initpos, ssaop.OpAddr, f.Config.Types.UInt32Ptr, wbsym, sb)
   220  			wbZero = f.Fe.Syslook("wbZero")
   221  			wbMove = f.Fe.Syslook("wbMove")
   222  			if buildcfg.Experiment.CgoCheck2 {
   223  				cgoCheckPtrWrite = f.Fe.Syslook("cgoCheckPtrWrite")
   224  				cgoCheckMemmove = f.Fe.Syslook("cgoCheckMemmove")
   225  			}
   226  			const0 = f.ConstInt32(f.Config.Types.UInt32, 0)
   227  
   228  			// allocate auxiliary data structures for computing store order
   229  			sset = f.NewSparseSet(f.NumValues())
   230  			defer f.RetSparseSet(sset)
   231  			sset2 = f.NewSparseSet(f.NumValues())
   232  			defer f.RetSparseSet(sset2)
   233  			storeNumber = f.Cache.AllocInt32Slice(f.NumValues())
   234  			defer f.Cache.FreeInt32Slice(storeNumber)
   235  		}
   236  
   237  		// order values in store order
   238  		b.Values = storeOrder(b.Values, sset, storeNumber)
   239  	again:
   240  		// find the start and end of the last contiguous WB store sequence.
   241  		// a branch will be inserted there. values after it will be moved
   242  		// to a new block.
   243  		var last *ssa.Value
   244  		var start, end int
   245  		var nonPtrStores int
   246  		values := b.Values
   247  		hasMove := false
   248  	FindSeq:
   249  		for i := len(values) - 1; i >= 0; i-- {
   250  			w := values[i]
   251  			switch w.Op {
   252  			case ssaop.OpStoreWB, ssaop.OpMoveWB, ssaop.OpZeroWB:
   253  				start = i
   254  				if last == nil {
   255  					last = w
   256  					end = i + 1
   257  				}
   258  				nonPtrStores = 0
   259  				if w.Op == ssaop.OpMoveWB {
   260  					hasMove = true
   261  				}
   262  			case ssaop.OpVarDef, ssaop.OpVarLive:
   263  				continue
   264  			case ssaop.OpStore:
   265  				if last == nil {
   266  					continue
   267  				}
   268  				nonPtrStores++
   269  				if nonPtrStores > 2 {
   270  					break FindSeq
   271  				}
   272  				if hasMove {
   273  					// We need to ensure that this store happens
   274  					// before we issue a wbMove, as the wbMove might
   275  					// use the result of this store as its source.
   276  					// Even though this store is not write-barrier
   277  					// eligible, it might nevertheless be the store
   278  					// of a pointer to the stack, which is then the
   279  					// source of the move.
   280  					// See issue 71228.
   281  					break FindSeq
   282  				}
   283  			default:
   284  				if last == nil {
   285  					continue
   286  				}
   287  				break FindSeq
   288  			}
   289  		}
   290  		stores = append(stores[:0], b.Values[start:end]...) // copy to avoid aliasing
   291  		after = append(after[:0], b.Values[end:]...)
   292  		b.Values = b.Values[:start]
   293  
   294  		// find the memory before the WB stores
   295  		mem := stores[0].MemoryArg()
   296  		pos := stores[0].Pos
   297  
   298  		// If there is a nil check before the WB store, duplicate it to
   299  		// the two branches, where the store and the WB load occur. So
   300  		// they are more likely be removed by late nilcheck removal (which
   301  		// is block-local).
   302  		var nilcheck, nilcheckThen, nilcheckEnd *ssa.Value
   303  		if a := stores[0].Args[0]; a.Op == ssaop.OpNilCheck && a.Args[1] == mem {
   304  			nilcheck = a
   305  		}
   306  
   307  		// If the source of a MoveWB is volatile (will be clobbered by a
   308  		// function call), we need to copy it to a temporary location, as
   309  		// marshaling the args of wbMove might clobber the value we're
   310  		// trying to move.
   311  		// Look for volatile source, copy it to temporary before we check
   312  		// the write barrier flag.
   313  		// It is unlikely to have more than one of them. Just do a linear
   314  		// search instead of using a map.
   315  		// See issue 15854.
   316  		type volatileCopy struct {
   317  			src *ssa.Value // address of original volatile value
   318  			tmp *ssa.Value // address of temporary we've copied the volatile value into
   319  		}
   320  		var volatiles []volatileCopy
   321  	copyLoop:
   322  		for _, w := range stores {
   323  			if w.Op == ssaop.OpMoveWB {
   324  				val := w.Args[1]
   325  				if ssa.IsVolatile(val) {
   326  					for _, c := range volatiles {
   327  						if val == c.src {
   328  							continue copyLoop // already copied
   329  						}
   330  					}
   331  
   332  					t := val.Type.Elem()
   333  					tmp := f.NewLocal(w.Pos, t)
   334  					mem = b.NewValue1A(w.Pos, ssaop.OpVarDef, types.TypeMem, tmp, mem)
   335  					tmpaddr := b.NewValue2A(w.Pos, ssaop.OpLocalAddr, t.PtrTo(), tmp, sp, mem)
   336  					siz := t.Size()
   337  					mem = b.NewValue3I(w.Pos, ssaop.OpMove, types.TypeMem, siz, tmpaddr, val, mem)
   338  					mem.Aux = t
   339  					volatiles = append(volatiles, volatileCopy{val, tmpaddr})
   340  				}
   341  			}
   342  		}
   343  
   344  		// Build branch point.
   345  		bThen := f.NewBlock(block.BlockPlain)
   346  		bEnd := f.NewBlock(b.Kind)
   347  		bThen.Pos = pos
   348  		bEnd.Pos = b.Pos
   349  		b.Pos = pos
   350  
   351  		// Set up control flow for end block.
   352  		bEnd.CopyControls(b)
   353  		bEnd.Likely = b.Likely
   354  		for _, e := range b.Succs {
   355  			bEnd.Succs = append(bEnd.Succs, e)
   356  			e.B.Preds[e.I].B = bEnd
   357  		}
   358  
   359  		// set up control flow for write barrier test
   360  		// load word, test word, avoiding partial register write from load byte.
   361  		cfgtypes := &f.Config.Types
   362  		flag := b.NewValue2(pos, ssaop.OpLoad, cfgtypes.UInt32, wbaddr, mem)
   363  		flag = b.NewValue2(pos, ssaop.OpNeq32, cfgtypes.Bool, flag, const0)
   364  		b.Kind = block.BlockIf
   365  		b.SetControl(flag)
   366  		b.Likely = ssa.BranchUnlikely
   367  		b.Succs = b.Succs[:0]
   368  		b.AddEdgeTo(bThen)
   369  		b.AddEdgeTo(bEnd)
   370  		bThen.AddEdgeTo(bEnd)
   371  
   372  		// For each write barrier store, append write barrier code to bThen.
   373  		memThen := mem
   374  
   375  		if nilcheck != nil {
   376  			nilcheckThen = bThen.NewValue2(nilcheck.Pos, ssaop.OpNilCheck, nilcheck.Type, nilcheck.Args[0], memThen)
   377  		}
   378  
   379  		// Note: we can issue the write barrier code in any order. In particular,
   380  		// it doesn't matter if they are in a different order *even if* they end
   381  		// up referring to overlapping memory regions. For instance if an OpStore
   382  		// stores to a location that is later read by an OpMove. In all cases
   383  		// any pointers we must get into the write barrier buffer still make it,
   384  		// possibly in a different order and possibly a different (but definitely
   385  		// more than 0) number of times.
   386  		// In light of that, we process all the OpStoreWBs first. This minimizes
   387  		// the amount of spill/restore code we need around the Zero/Move calls.
   388  
   389  		// srcs contains the value IDs of pointer values we've put in the write barrier buffer.
   390  		srcs := sset
   391  		srcs.Clear()
   392  		// dsts contains the value IDs of locations which we've read a pointer out of
   393  		// and put the result in the write barrier buffer.
   394  		dsts := sset2
   395  		dsts.Clear()
   396  
   397  		// Buffer up entries that we need to put in the write barrier buffer.
   398  		type write struct {
   399  			ptr *ssa.Value // value to put in write barrier buffer
   400  			pos src.XPos   // location to use for the write
   401  		}
   402  		var writeStore [maxEntries]write
   403  		writes := writeStore[:0]
   404  
   405  		flush := func() {
   406  			if len(writes) == 0 {
   407  				return
   408  			}
   409  			// Issue a call to get a write barrier buffer.
   410  			t := types.NewTuple(types.Types[types.TUINTPTR].PtrTo(), types.TypeMem)
   411  			call := bThen.NewValue1I(pos, ssaop.OpWB, t, int64(len(writes)), memThen)
   412  			curPtr := bThen.NewValue1(pos, ssaop.OpSelect0, types.Types[types.TUINTPTR].PtrTo(), call)
   413  			memThen = bThen.NewValue1(pos, ssaop.OpSelect1, types.TypeMem, call)
   414  			// Write each pending pointer to a slot in the buffer.
   415  			for i, write := range writes {
   416  				wbuf := bThen.NewValue1I(write.pos, ssaop.OpOffPtr, types.Types[types.TUINTPTR].PtrTo(), int64(i)*f.Config.PtrSize, curPtr)
   417  				memThen = bThen.NewValue3A(write.pos, ssaop.OpStore, types.TypeMem, types.Types[types.TUINTPTR], wbuf, write.ptr, memThen)
   418  			}
   419  			writes = writes[:0]
   420  		}
   421  		addEntry := func(pos src.XPos, ptr *ssa.Value) {
   422  			writes = append(writes, write{ptr: ptr, pos: pos})
   423  			if len(writes) == maxEntries {
   424  				flush()
   425  			}
   426  		}
   427  
   428  		// Find all the pointers we need to write to the buffer.
   429  		for _, w := range stores {
   430  			if w.Op != ssaop.OpStoreWB {
   431  				continue
   432  			}
   433  			pos := w.Pos
   434  			ptr := w.Args[0]
   435  			val := w.Args[1]
   436  			if !srcs.Contains(val.ID) && needWBsrc(val) {
   437  				srcs.Add(val.ID)
   438  				addEntry(pos, val)
   439  			}
   440  			if !dsts.Contains(ptr.ID) && needWBdst(ptr, w.Args[2], zeroes) {
   441  				dsts.Add(ptr.ID)
   442  				// Load old value from store target.
   443  				// Note: This turns bad pointer writes into bad
   444  				// pointer reads, which could be confusing. We could avoid
   445  				// reading from obviously bad pointers, which would
   446  				// take care of the vast majority of these. We could
   447  				// patch this up in the signal handler, or use XCHG to
   448  				// combine the read and the write.
   449  				if ptr == nilcheck {
   450  					ptr = nilcheckThen
   451  				}
   452  				oldVal := bThen.NewValue2(pos, ssaop.OpLoad, types.Types[types.TUINTPTR], ptr, memThen)
   453  				// Save old value to write buffer.
   454  				addEntry(pos, oldVal)
   455  			}
   456  			f.Fe.Func().SetWBPos(pos)
   457  			nWBops--
   458  		}
   459  		flush()
   460  
   461  		// Now do the rare cases, Zeros and Moves.
   462  		for _, w := range stores {
   463  			pos := w.Pos
   464  			dst := w.Args[0]
   465  			if dst == nilcheck {
   466  				dst = nilcheckThen
   467  			}
   468  			switch w.Op {
   469  			case ssaop.OpZeroWB:
   470  				typ := reflectdata.TypeLinksym(w.Aux.(*types.Type))
   471  				// zeroWB(&typ, dst)
   472  				taddr := b.NewValue1A(pos, ssaop.OpAddr, b.Func.Config.Types.Uintptr, typ, sb)
   473  				memThen = wbcall(pos, bThen, wbZero, sp, memThen, taddr, dst)
   474  				f.Fe.Func().SetWBPos(pos)
   475  				nWBops--
   476  			case ssaop.OpMoveWB:
   477  				src := w.Args[1]
   478  				if ssa.IsVolatile(src) {
   479  					for _, c := range volatiles {
   480  						if src == c.src {
   481  							src = c.tmp
   482  							break
   483  						}
   484  					}
   485  				}
   486  				typ := reflectdata.TypeLinksym(w.Aux.(*types.Type))
   487  				// moveWB(&typ, dst, src)
   488  				taddr := b.NewValue1A(pos, ssaop.OpAddr, b.Func.Config.Types.Uintptr, typ, sb)
   489  				memThen = wbcall(pos, bThen, wbMove, sp, memThen, taddr, dst, src)
   490  				f.Fe.Func().SetWBPos(pos)
   491  				nWBops--
   492  			}
   493  		}
   494  
   495  		// merge memory
   496  		mem = bEnd.NewValue2(pos, ssaop.OpPhi, types.TypeMem, mem, memThen)
   497  
   498  		if nilcheck != nil {
   499  			nilcheckEnd = bEnd.NewValue2(nilcheck.Pos, ssaop.OpNilCheck, nilcheck.Type, nilcheck.Args[0], mem)
   500  		}
   501  
   502  		// Do raw stores after merge point.
   503  		for _, w := range stores {
   504  			pos := w.Pos
   505  			dst := w.Args[0]
   506  			if dst == nilcheck {
   507  				dst = nilcheckEnd
   508  			}
   509  			switch w.Op {
   510  			case ssaop.OpStoreWB:
   511  				val := w.Args[1]
   512  				if buildcfg.Experiment.CgoCheck2 {
   513  					// Issue cgo checking code.
   514  					mem = wbcall(pos, bEnd, cgoCheckPtrWrite, sp, mem, dst, val)
   515  				}
   516  				mem = bEnd.NewValue3A(pos, ssaop.OpStore, types.TypeMem, w.Aux, dst, val, mem)
   517  			case ssaop.OpZeroWB:
   518  				mem = bEnd.NewValue2I(pos, ssaop.OpZero, types.TypeMem, w.AuxInt, dst, mem)
   519  				mem.Aux = w.Aux
   520  			case ssaop.OpMoveWB:
   521  				src := w.Args[1]
   522  				if ssa.IsVolatile(src) {
   523  					for _, c := range volatiles {
   524  						if src == c.src {
   525  							src = c.tmp
   526  							break
   527  						}
   528  					}
   529  				}
   530  				if buildcfg.Experiment.CgoCheck2 {
   531  					// Issue cgo checking code.
   532  					typ := reflectdata.TypeLinksym(w.Aux.(*types.Type))
   533  					taddr := b.NewValue1A(pos, ssaop.OpAddr, b.Func.Config.Types.Uintptr, typ, sb)
   534  					mem = wbcall(pos, bEnd, cgoCheckMemmove, sp, mem, taddr, dst, src)
   535  				}
   536  				mem = bEnd.NewValue3I(pos, ssaop.OpMove, types.TypeMem, w.AuxInt, dst, src, mem)
   537  				mem.Aux = w.Aux
   538  			case ssaop.OpVarDef, ssaop.OpVarLive:
   539  				mem = bEnd.NewValue1A(pos, w.Op, types.TypeMem, w.Aux, mem)
   540  			case ssaop.OpStore:
   541  				val := w.Args[1]
   542  				mem = bEnd.NewValue3A(pos, ssaop.OpStore, types.TypeMem, w.Aux, dst, val, mem)
   543  			}
   544  		}
   545  
   546  		// The last store becomes the WBend marker. This marker is used by the liveness
   547  		// pass to determine what parts of the code are preemption-unsafe.
   548  		// All subsequent memory operations use this memory, so we have to sacrifice the
   549  		// previous last memory op to become this new value.
   550  		bEnd.Values = append(bEnd.Values, last)
   551  		last.Block = bEnd
   552  		last.Reset(ssaop.OpWBend)
   553  		last.Pos = last.Pos.WithNotStmt()
   554  		last.Type = types.TypeMem
   555  		last.AddArg(mem)
   556  
   557  		// Free all the old stores, except last which became the WBend marker.
   558  		for _, w := range stores {
   559  			if w != last {
   560  				w.ResetArgs()
   561  			}
   562  		}
   563  		for _, w := range stores {
   564  			if w != last {
   565  				f.FreeValue(w)
   566  			}
   567  		}
   568  		if nilcheck != nil && nilcheck.Uses == 0 {
   569  			nilcheck.Reset(ssaop.OpInvalid)
   570  		}
   571  
   572  		// put values after the store sequence into the end block
   573  		bEnd.Values = append(bEnd.Values, after...)
   574  		for _, w := range after {
   575  			w.Block = bEnd
   576  		}
   577  
   578  		// if we have more stores in this block, do this block again
   579  		if nWBops > 0 {
   580  			goto again
   581  		}
   582  	}
   583  }
   584  
   585  // wbcall emits write barrier runtime call in b, returns memory.
   586  func wbcall(pos src.XPos, b *ssa.Block, fn *obj.LSym, sp, mem *ssa.Value, args ...*ssa.Value) *ssa.Value {
   587  	config := b.Func.Config
   588  	typ := config.Types.Uintptr // type of all argument values
   589  	nargs := len(args)
   590  
   591  	// TODO (register args) this is a bit of a hack.
   592  	inRegs := b.Func.ABIDefault == b.Func.ABI1 && len(config.IntParamRegs) >= 3
   593  
   594  	if !inRegs {
   595  		// Store arguments to the appropriate stack slot.
   596  		off := config.Ctxt.Arch.FixedFrameSize
   597  		for _, arg := range args {
   598  			stkaddr := b.NewValue1I(pos, ssaop.OpOffPtr, typ.PtrTo(), off, sp)
   599  			mem = b.NewValue3A(pos, ssaop.OpStore, types.TypeMem, typ, stkaddr, arg, mem)
   600  			off += typ.Size()
   601  		}
   602  		args = args[:0]
   603  	}
   604  
   605  	args = append(args, mem)
   606  
   607  	// issue call
   608  	argTypes := make([]*types.Type, nargs, 3) // at most 3 args; allows stack allocation
   609  	for i := 0; i < nargs; i++ {
   610  		argTypes[i] = typ
   611  	}
   612  	call := b.NewValue0A(pos, ssaop.OpStaticCall, types.TypeResultMem, ssa.StaticAuxCall(fn, b.Func.ABIDefault.ABIAnalyzeTypes(argTypes, nil)))
   613  	call.AddArgs(args...)
   614  	call.AuxInt = int64(nargs) * typ.Size()
   615  	return b.NewValue1I(pos, ssaop.OpSelectN, types.TypeMem, 0, call)
   616  }
   617  
   618  // IsGlobalAddr reports whether v is known to be an address of a global (or nil).
   619  func IsGlobalAddr(v *ssa.Value) bool {
   620  	for v.Op == ssaop.OpOffPtr || v.Op == ssaop.OpAddPtr || v.Op == ssaop.OpPtrIndex || v.Op == ssaop.OpCopy {
   621  		v = v.Args[0]
   622  	}
   623  	if v.Op == ssaop.OpAddr && v.Args[0].Op == ssaop.OpSB {
   624  		return true // address of a global
   625  	}
   626  	if v.Op == ssaop.OpConstNil {
   627  		return true
   628  	}
   629  	if v.Op == ssaop.OpLoad && IsReadOnlyGlobalAddr(v.Args[0]) {
   630  		return true // loading from a read-only global - the resulting address can't be a heap address.
   631  	}
   632  	return false
   633  }
   634  
   635  // IsReadOnlyGlobalAddr reports whether v is known to be an address of a read-only global.
   636  func IsReadOnlyGlobalAddr(v *ssa.Value) bool {
   637  	if v.Op == ssaop.OpConstNil {
   638  		// Nil pointers are read only. See issue 33438.
   639  		return true
   640  	}
   641  	if v.Op == ssaop.OpAddr && v.Aux != nil && v.Aux.(*obj.LSym).Type == objabi.SRODATA {
   642  		return true
   643  	}
   644  	return false
   645  }
   646
View as plain text