iface.go

     1  // Copyright 2014 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"internal/goarch"
    10  	"internal/runtime/atomic"
    11  	"internal/runtime/sys"
    12  	"unsafe"
    13  )
    14  
    15  const itabInitSize = 512
    16  
    17  var (
    18  	itabLock      mutex                               // lock for accessing itab table
    19  	itabTable     = &itabTableInit                    // pointer to current table
    20  	itabTableInit = itabTableType{size: itabInitSize} // starter table
    21  )
    22  
    23  // Note: change the formula in the mallocgc call in itabAdd if you change these fields.
    24  type itabTableType struct {
    25  	size    uintptr             // length of entries array. Always a power of 2.
    26  	count   uintptr             // current number of filled entries.
    27  	entries [itabInitSize]*itab // really [size] large
    28  }
    29  
    30  func itabHashFunc(inter *interfacetype, typ *_type) uintptr {
    31  	// compiler has provided some good hash codes for us.
    32  	return uintptr(inter.Type.Hash ^ typ.Hash)
    33  }
    34  
    35  // getitab should be an internal detail,
    36  // but widely used packages access it using linkname.
    37  // Notable members of the hall of shame include:
    38  //   - github.com/bytedance/sonic
    39  //
    40  // Do not remove or change the type signature.
    41  // See go.dev/issue/67401.
    42  //
    43  //go:linkname getitab
    44  func getitab(inter *interfacetype, typ *_type, canfail bool) *itab {
    45  	if len(inter.Methods) == 0 {
    46  		throw("internal error - misuse of itab")
    47  	}
    48  
    49  	// easy case
    50  	if typ.TFlag&abi.TFlagUncommon == 0 {
    51  		if canfail {
    52  			return nil
    53  		}
    54  		name := toRType(&inter.Type).nameOff(inter.Methods[0].Name)
    55  		panic(&TypeAssertionError{nil, typ, &inter.Type, name.Name()})
    56  	}
    57  
    58  	var m *itab
    59  
    60  	// First, look in the existing table to see if we can find the itab we need.
    61  	// This is by far the most common case, so do it without locks.
    62  	// Use atomic to ensure we see any previous writes done by the thread
    63  	// that updates the itabTable field (with atomic.Storep in itabAdd).
    64  	t := (*itabTableType)(atomic.Loadp(unsafe.Pointer(&itabTable)))
    65  	if m = t.find(inter, typ); m != nil {
    66  		goto finish
    67  	}
    68  
    69  	// Not found.  Grab the lock and try again.
    70  	lock(&itabLock)
    71  	if m = itabTable.find(inter, typ); m != nil {
    72  		unlock(&itabLock)
    73  		goto finish
    74  	}
    75  
    76  	// Entry doesn't exist yet. Make a new entry & add it.
    77  	m = (*itab)(persistentalloc(unsafe.Sizeof(itab{})+uintptr(len(inter.Methods)-1)*goarch.PtrSize, 0, &memstats.other_sys))
    78  	m.Inter = inter
    79  	m.Type = typ
    80  	// The hash is used in type switches. However, compiler statically generates itab's
    81  	// for all interface/type pairs used in switches (which are added to itabTable
    82  	// in itabsinit). The dynamically-generated itab's never participate in type switches,
    83  	// and thus the hash is irrelevant.
    84  	// Note: m.Hash is _not_ the hash used for the runtime itabTable hash table.
    85  	m.Hash = 0
    86  	itabInit(m, true)
    87  	itabAdd(m)
    88  	unlock(&itabLock)
    89  finish:
    90  	if m.Fun[0] != 0 {
    91  		return m
    92  	}
    93  	if canfail {
    94  		return nil
    95  	}
    96  	// this can only happen if the conversion
    97  	// was already done once using the , ok form
    98  	// and we have a cached negative result.
    99  	// The cached result doesn't record which
   100  	// interface function was missing, so initialize
   101  	// the itab again to get the missing function name.
   102  	panic(&TypeAssertionError{concrete: typ, asserted: &inter.Type, missingMethod: itabInit(m, false)})
   103  }
   104  
   105  // find finds the given interface/type pair in t.
   106  // Returns nil if the given interface/type pair isn't present.
   107  func (t *itabTableType) find(inter *interfacetype, typ *_type) *itab {
   108  	// Implemented using quadratic probing.
   109  	// Probe sequence is h(i) = h0 + i*(i+1)/2 mod 2^k.
   110  	// We're guaranteed to hit all table entries using this probe sequence.
   111  	mask := t.size - 1
   112  	h := itabHashFunc(inter, typ) & mask
   113  	for i := uintptr(1); ; i++ {
   114  		p := (**itab)(add(unsafe.Pointer(&t.entries), h*goarch.PtrSize))
   115  		// Use atomic read here so if we see m != nil, we also see
   116  		// the initializations of the fields of m.
   117  		// m := *p
   118  		m := (*itab)(atomic.Loadp(unsafe.Pointer(p)))
   119  		if m == nil {
   120  			return nil
   121  		}
   122  		if m.Inter == inter && m.Type == typ {
   123  			return m
   124  		}
   125  		h += i
   126  		h &= mask
   127  	}
   128  }
   129  
   130  // itabAdd adds the given itab to the itab hash table.
   131  // itabLock must be held.
   132  func itabAdd(m *itab) {
   133  	// Bugs can lead to calling this while mallocing is set,
   134  	// typically because this is called while panicking.
   135  	// Crash reliably, rather than only when we need to grow
   136  	// the hash table.
   137  	if getg().m.mallocing != 0 {
   138  		throw("malloc deadlock")
   139  	}
   140  
   141  	t := itabTable
   142  	if t.count >= 3*(t.size/4) { // 75% load factor
   143  		// Grow hash table.
   144  		// t2 = new(itabTableType) + some additional entries
   145  		// We lie and tell malloc we want pointer-free memory because
   146  		// all the pointed-to values are not in the heap.
   147  		t2 := (*itabTableType)(mallocgc((2+2*t.size)*goarch.PtrSize, nil, true))
   148  		t2.size = t.size * 2
   149  
   150  		// Copy over entries.
   151  		// Note: while copying, other threads may look for an itab and
   152  		// fail to find it. That's ok, they will then try to get the itab lock
   153  		// and as a consequence wait until this copying is complete.
   154  		iterate_itabs(t2.add)
   155  		if t2.count != t.count {
   156  			throw("mismatched count during itab table copy")
   157  		}
   158  		// Publish new hash table. Use an atomic write: see comment in getitab.
   159  		atomicstorep(unsafe.Pointer(&itabTable), unsafe.Pointer(t2))
   160  		// Adopt the new table as our own.
   161  		t = itabTable
   162  		// Note: the old table can be GC'ed here.
   163  	}
   164  	t.add(m)
   165  }
   166  
   167  // add adds the given itab to itab table t.
   168  // itabLock must be held.
   169  func (t *itabTableType) add(m *itab) {
   170  	// See comment in find about the probe sequence.
   171  	// Insert new itab in the first empty spot in the probe sequence.
   172  	mask := t.size - 1
   173  	h := itabHashFunc(m.Inter, m.Type) & mask
   174  	for i := uintptr(1); ; i++ {
   175  		p := (**itab)(add(unsafe.Pointer(&t.entries), h*goarch.PtrSize))
   176  		m2 := *p
   177  		if m2 != nil && m2.Inter == m.Inter && m2.Type == m.Type {
   178  			// A plugin has its own copy of the itabs that the main program
   179  			// also has. Type switches and type assertions compare against
   180  			// the itab that is already in the table, so don't add a second
   181  			// itab for the same interface/type pair.
   182  			return
   183  		}
   184  		if m2 == nil {
   185  			// Use atomic write here so if a reader sees m, it also
   186  			// sees the correctly initialized fields of m.
   187  			// NoWB is ok because m is not in heap memory.
   188  			// *p = m
   189  			atomic.StorepNoWB(unsafe.Pointer(p), unsafe.Pointer(m))
   190  			t.count++
   191  			return
   192  		}
   193  		h += i
   194  		h &= mask
   195  	}
   196  }
   197  
   198  // itabInit fills in the m.Fun array with all the code pointers for
   199  // the m.Inter/m.Type pair. If the type does not implement the interface,
   200  // it sets m.Fun[0] to 0 and returns the name of an interface function that is missing.
   201  // If !firstTime, itabInit will not write anything to m.Fun (see issue 65962).
   202  // It is ok to call this multiple times on the same m, even concurrently
   203  // (although it will only be called once with firstTime==true).
   204  //
   205  // itabInit stores only code pointers, so it must not emit a write barrier;
   206  // //go:nowritebarrier enforces that (see the methods store below).
   207  //
   208  //go:nowritebarrier
   209  func itabInit(m *itab, firstTime bool) string {
   210  	inter := m.Inter
   211  	typ := m.Type
   212  	x := typ.Uncommon()
   213  
   214  	// both inter and typ have method sorted by name,
   215  	// and interface names are unique,
   216  	// so can iterate over both in lock step;
   217  	// the loop is O(ni+nt) not O(ni*nt).
   218  	ni := len(inter.Methods)
   219  	nt := int(x.Mcount)
   220  	xmhdr := unsafe.Slice((*abi.Method)(add(unsafe.Pointer(x), uintptr(x.Moff))), nt)
   221  	j := 0
   222  	// These are code pointers, not heap pointers. Use a uintptr slice so the
   223  	// store emits no write barrier: on wasm a code pointer can look like a
   224  	// heap pointer and make the GC crash (see issue 80472).
   225  	methods := unsafe.Slice(&m.Fun[0], ni)
   226  	var fun0 unsafe.Pointer
   227  imethods:
   228  	for k := 0; k < ni; k++ {
   229  		i := &inter.Methods[k]
   230  		itype := toRType(&inter.Type).typeOff(i.Typ)
   231  		name := toRType(&inter.Type).nameOff(i.Name)
   232  		iname := name.Name()
   233  		ipkg := pkgPath(name)
   234  		if ipkg == "" {
   235  			ipkg = inter.PkgPath.Name()
   236  		}
   237  		for ; j < nt; j++ {
   238  			t := &xmhdr[j]
   239  			rtyp := toRType(typ)
   240  			tname := rtyp.nameOff(t.Name)
   241  			if rtyp.typeOff(t.Mtyp) == itype && tname.Name() == iname {
   242  				pkgPath := pkgPath(tname)
   243  				if pkgPath == "" {
   244  					pkgPath = rtyp.nameOff(x.PkgPath).Name()
   245  				}
   246  				if tname.IsExported() || pkgPath == ipkg {
   247  					ifn := rtyp.textOff(t.Ifn)
   248  					if k == 0 {
   249  						fun0 = ifn // we'll set m.Fun[0] at the end
   250  					} else if firstTime {
   251  						methods[k] = uintptr(ifn)
   252  					}
   253  					continue imethods
   254  				}
   255  			}
   256  		}
   257  		// didn't find method
   258  		// Leaves m.Fun[0] set to 0.
   259  		return iname
   260  	}
   261  	if firstTime {
   262  		m.Fun[0] = uintptr(fun0)
   263  	}
   264  	return ""
   265  }
   266  
   267  func itabsinit() {
   268  	lockInit(&itabLock, lockRankItab)
   269  	lock(&itabLock)
   270  	for _, md := range activeModules() {
   271  		addModuleItabs(md)
   272  	}
   273  	unlock(&itabLock)
   274  }
   275  
   276  // addModuleItabs adds the pre-compiled itabs from md to the itab hash table.
   277  // This is an optimization to let us skip creating itabs we already have.
   278  func addModuleItabs(md *moduledata) {
   279  	p := md.types + md.itaboffset
   280  	end := p + md.itabsize
   281  	for p < end {
   282  		itab := (*itab)(unsafe.Pointer(p))
   283  		itabAdd(itab)
   284  		p += uintptr(itab.Size())
   285  	}
   286  }
   287  
   288  // panicdottypeE is called when doing an e.(T) conversion and the conversion fails.
   289  // have = the dynamic type we have.
   290  // want = the static type we're trying to convert to.
   291  // iface = the static type we're converting from.
   292  func panicdottypeE(have, want, iface *_type) {
   293  	panic(&TypeAssertionError{iface, have, want, ""})
   294  }
   295  
   296  // panicdottypeI is called when doing an i.(T) conversion and the conversion fails.
   297  // Same args as panicdottypeE, but "have" is the dynamic itab we have.
   298  func panicdottypeI(have *itab, want, iface *_type) {
   299  	var t *_type
   300  	if have != nil {
   301  		t = have.Type
   302  	}
   303  	panicdottypeE(t, want, iface)
   304  }
   305  
   306  // panicnildottype is called when doing an i.(T) conversion and the interface i is nil.
   307  // want = the static type we're trying to convert to.
   308  func panicnildottype(want *_type) {
   309  	panic(&TypeAssertionError{nil, nil, want, ""})
   310  	// TODO: Add the static type we're converting from as well.
   311  	// It might generate a better error message.
   312  	// Just to match other nil conversion errors, we don't for now.
   313  }
   314  
   315  // The specialized convTx routines need a type descriptor to use when calling mallocgc.
   316  // We don't need the type to be exact, just to have the correct size, alignment, and pointer-ness.
   317  // However, when debugging, it'd be nice to have some indication in mallocgc where the types came from,
   318  // so we use named types here.
   319  // We then construct interface values of these types,
   320  // and then extract the type word to use as needed.
   321  type (
   322  	uint16InterfacePtr uint16
   323  	uint32InterfacePtr uint32
   324  	uint64InterfacePtr uint64
   325  	stringInterfacePtr string
   326  	sliceInterfacePtr  []byte
   327  )
   328  
   329  var (
   330  	uint16Eface any = uint16InterfacePtr(0)
   331  	uint32Eface any = uint32InterfacePtr(0)
   332  	uint64Eface any = uint64InterfacePtr(0)
   333  	stringEface any = stringInterfacePtr("")
   334  	sliceEface  any = sliceInterfacePtr(nil)
   335  
   336  	uint16Type *_type = efaceOf(&uint16Eface)._type
   337  	uint32Type *_type = efaceOf(&uint32Eface)._type
   338  	uint64Type *_type = efaceOf(&uint64Eface)._type
   339  	stringType *_type = efaceOf(&stringEface)._type
   340  	sliceType  *_type = efaceOf(&sliceEface)._type
   341  )
   342  
   343  // The conv and assert functions below do very similar things.
   344  // The convXXX functions are guaranteed by the compiler to succeed.
   345  // The assertXXX functions may fail (either panicking or returning false,
   346  // depending on whether they are 1-result or 2-result).
   347  // The convXXX functions succeed on a nil input, whereas the assertXXX
   348  // functions fail on a nil input.
   349  
   350  // convT converts a value of type t, which is pointed to by v, to a pointer that can
   351  // be used as the second word of an interface value.
   352  func convT(t *_type, v unsafe.Pointer) unsafe.Pointer {
   353  	if raceenabled {
   354  		raceReadObjectPC(t, v, sys.GetCallerPC(), abi.FuncPCABIInternal(convT))
   355  	}
   356  	if msanenabled {
   357  		msanread(v, t.Size_)
   358  	}
   359  	if asanenabled {
   360  		asanread(v, t.Size_)
   361  	}
   362  	x := mallocgc(t.Size_, t, true)
   363  	typedmemmove(t, x, v)
   364  	return x
   365  }
   366  func convTnoptr(t *_type, v unsafe.Pointer) unsafe.Pointer {
   367  	// TODO: maybe take size instead of type?
   368  	if raceenabled {
   369  		raceReadObjectPC(t, v, sys.GetCallerPC(), abi.FuncPCABIInternal(convTnoptr))
   370  	}
   371  	if msanenabled {
   372  		msanread(v, t.Size_)
   373  	}
   374  	if asanenabled {
   375  		asanread(v, t.Size_)
   376  	}
   377  
   378  	x := mallocgc(t.Size_, t, false)
   379  	memmove(x, v, t.Size_)
   380  	return x
   381  }
   382  
   383  func convT16(val uint16) (x unsafe.Pointer) {
   384  	if val < uint16(len(staticuint64s)) {
   385  		x = unsafe.Pointer(&staticuint64s[val])
   386  		if goarch.BigEndian {
   387  			x = add(x, 6)
   388  		}
   389  	} else {
   390  		x = mallocgc(2, uint16Type, false)
   391  		*(*uint16)(x) = val
   392  	}
   393  	return
   394  }
   395  
   396  func convT32(val uint32) (x unsafe.Pointer) {
   397  	if val < uint32(len(staticuint64s)) {
   398  		x = unsafe.Pointer(&staticuint64s[val])
   399  		if goarch.BigEndian {
   400  			x = add(x, 4)
   401  		}
   402  	} else {
   403  		x = mallocgc(4, uint32Type, false)
   404  		*(*uint32)(x) = val
   405  	}
   406  	return
   407  }
   408  
   409  // convT64 should be an internal detail,
   410  // but widely used packages access it using linkname.
   411  // Notable members of the hall of shame include:
   412  //   - github.com/bytedance/sonic
   413  //
   414  // Do not remove or change the type signature.
   415  // See go.dev/issue/67401.
   416  //
   417  //go:linkname convT64
   418  func convT64(val uint64) (x unsafe.Pointer) {
   419  	if val < uint64(len(staticuint64s)) {
   420  		x = unsafe.Pointer(&staticuint64s[val])
   421  	} else {
   422  		x = mallocgc(8, uint64Type, false)
   423  		*(*uint64)(x) = val
   424  	}
   425  	return
   426  }
   427  
   428  // convTstring should be an internal detail,
   429  // but widely used packages access it using linkname.
   430  // Notable members of the hall of shame include:
   431  //   - github.com/bytedance/sonic
   432  //
   433  // Do not remove or change the type signature.
   434  // See go.dev/issue/67401.
   435  //
   436  //go:linkname convTstring
   437  func convTstring(val string) (x unsafe.Pointer) {
   438  	if val == "" {
   439  		x = unsafe.Pointer(&zeroVal[0])
   440  	} else {
   441  		x = mallocgc(unsafe.Sizeof(val), stringType, true)
   442  		*(*string)(x) = val
   443  	}
   444  	return
   445  }
   446  
   447  // convTslice should be an internal detail,
   448  // but widely used packages access it using linkname.
   449  // Notable members of the hall of shame include:
   450  //   - github.com/bytedance/sonic
   451  //
   452  // Do not remove or change the type signature.
   453  // See go.dev/issue/67401.
   454  //
   455  //go:linkname convTslice
   456  func convTslice(val []byte) (x unsafe.Pointer) {
   457  	// Note: this must work for any element type, not just byte.
   458  	if (*slice)(unsafe.Pointer(&val)).array == nil {
   459  		x = unsafe.Pointer(&zeroVal[0])
   460  	} else {
   461  		x = mallocgc(unsafe.Sizeof(val), sliceType, true)
   462  		*(*[]byte)(x) = val
   463  	}
   464  	return
   465  }
   466  
   467  func assertE2I(inter *interfacetype, t *_type) *itab {
   468  	if t == nil {
   469  		// explicit conversions require non-nil interface value.
   470  		panic(&TypeAssertionError{nil, nil, &inter.Type, ""})
   471  	}
   472  	return getitab(inter, t, false)
   473  }
   474  
   475  func assertE2I2(inter *interfacetype, t *_type) *itab {
   476  	if t == nil {
   477  		return nil
   478  	}
   479  	return getitab(inter, t, true)
   480  }
   481  
   482  // typeAssert builds an itab for the concrete type t and the
   483  // interface type s.Inter. If the conversion is not possible it
   484  // panics if s.CanFail is false and returns nil if s.CanFail is true.
   485  func typeAssert(s *abi.TypeAssert, t *_type) *itab {
   486  	var tab *itab
   487  	if t == nil {
   488  		if !s.CanFail {
   489  			panic(&TypeAssertionError{nil, nil, &s.Inter.Type, ""})
   490  		}
   491  	} else {
   492  		tab = getitab(s.Inter, t, s.CanFail)
   493  	}
   494  
   495  	if !abi.UseInterfaceSwitchCache(goarch.ArchFamily) {
   496  		return tab
   497  	}
   498  
   499  	// Maybe update the cache, so the next time the generated code
   500  	// doesn't need to call into the runtime.
   501  	if cheaprand()&1023 != 0 {
   502  		// Only bother updating the cache ~1 in 1000 times.
   503  		return tab
   504  	}
   505  	// Load the current cache.
   506  	oldC := (*abi.TypeAssertCache)(atomic.Loadp(unsafe.Pointer(&s.Cache)))
   507  
   508  	if cheaprand()&uint32(oldC.Mask) != 0 {
   509  		// As cache gets larger, choose to update it less often
   510  		// so we can amortize the cost of building a new cache.
   511  		return tab
   512  	}
   513  
   514  	// Make a new cache.
   515  	newC := buildTypeAssertCache(oldC, t, tab)
   516  
   517  	// Update cache. Use compare-and-swap so if multiple threads
   518  	// are fighting to update the cache, at least one of their
   519  	// updates will stick.
   520  	atomic_casPointer((*unsafe.Pointer)(unsafe.Pointer(&s.Cache)), unsafe.Pointer(oldC), unsafe.Pointer(newC))
   521  
   522  	return tab
   523  }
   524  
   525  func buildTypeAssertCache(oldC *abi.TypeAssertCache, typ *_type, tab *itab) *abi.TypeAssertCache {
   526  	oldEntries := unsafe.Slice(&oldC.Entries[0], oldC.Mask+1)
   527  
   528  	// Count the number of entries we need.
   529  	n := 1
   530  	for _, e := range oldEntries {
   531  		if e.Typ != 0 {
   532  			n++
   533  		}
   534  	}
   535  
   536  	// Figure out how big a table we need.
   537  	// We need at least one more slot than the number of entries
   538  	// so that we are guaranteed an empty slot (for termination).
   539  	newN := n * 2                         // make it at most 50% full
   540  	newN = 1 << sys.Len64(uint64(newN-1)) // round up to a power of 2
   541  
   542  	// Allocate the new table.
   543  	newSize := unsafe.Sizeof(abi.TypeAssertCache{}) + uintptr(newN-1)*unsafe.Sizeof(abi.TypeAssertCacheEntry{})
   544  	newC := (*abi.TypeAssertCache)(mallocgc(newSize, nil, true))
   545  	newC.Mask = uintptr(newN - 1)
   546  	newEntries := unsafe.Slice(&newC.Entries[0], newN)
   547  
   548  	// Fill the new table.
   549  	addEntry := func(typ *_type, tab *itab) {
   550  		h := int(typ.Hash) & (newN - 1)
   551  		for {
   552  			if newEntries[h].Typ == 0 {
   553  				newEntries[h].Typ = uintptr(unsafe.Pointer(typ))
   554  				newEntries[h].Itab = uintptr(unsafe.Pointer(tab))
   555  				return
   556  			}
   557  			h = (h + 1) & (newN - 1)
   558  		}
   559  	}
   560  	for _, e := range oldEntries {
   561  		if e.Typ != 0 {
   562  			addEntry((*_type)(unsafe.Pointer(e.Typ)), (*itab)(unsafe.Pointer(e.Itab)))
   563  		}
   564  	}
   565  	addEntry(typ, tab)
   566  
   567  	return newC
   568  }
   569  
   570  // Empty type assert cache. Contains one entry with a nil Typ (which
   571  // causes a cache lookup to fail immediately.)
   572  var emptyTypeAssertCache = abi.TypeAssertCache{Mask: 0}
   573  
   574  // interfaceSwitch compares t against the list of cases in s.
   575  // If t matches case i, interfaceSwitch returns the case index i and
   576  // an itab for the pair <t, s.Cases[i]>.
   577  // If there is no match, return N,nil, where N is the number
   578  // of cases.
   579  func interfaceSwitch(s *abi.InterfaceSwitch, t *_type) (int, *itab) {
   580  	cases := unsafe.Slice(&s.Cases[0], s.NCases)
   581  
   582  	// Results if we don't find a match.
   583  	case_ := len(cases)
   584  	var tab *itab
   585  
   586  	// Look through each case in order.
   587  	for i, c := range cases {
   588  		tab = getitab(c, t, true)
   589  		if tab != nil {
   590  			case_ = i
   591  			break
   592  		}
   593  	}
   594  
   595  	if !abi.UseInterfaceSwitchCache(goarch.ArchFamily) {
   596  		return case_, tab
   597  	}
   598  
   599  	// Maybe update the cache, so the next time the generated code
   600  	// doesn't need to call into the runtime.
   601  	if cheaprand()&1023 != 0 {
   602  		// Only bother updating the cache ~1 in 1000 times.
   603  		// This ensures we don't waste memory on switches, or
   604  		// switch arguments, that only happen a few times.
   605  		return case_, tab
   606  	}
   607  	// Load the current cache.
   608  	oldC := (*abi.InterfaceSwitchCache)(atomic.Loadp(unsafe.Pointer(&s.Cache)))
   609  
   610  	if cheaprand()&uint32(oldC.Mask) != 0 {
   611  		// As cache gets larger, choose to update it less often
   612  		// so we can amortize the cost of building a new cache
   613  		// (that cost is linear in oldc.Mask).
   614  		return case_, tab
   615  	}
   616  
   617  	// Make a new cache.
   618  	newC := buildInterfaceSwitchCache(oldC, t, case_, tab)
   619  
   620  	// Update cache. Use compare-and-swap so if multiple threads
   621  	// are fighting to update the cache, at least one of their
   622  	// updates will stick.
   623  	atomic_casPointer((*unsafe.Pointer)(unsafe.Pointer(&s.Cache)), unsafe.Pointer(oldC), unsafe.Pointer(newC))
   624  
   625  	return case_, tab
   626  }
   627  
   628  // buildInterfaceSwitchCache constructs an interface switch cache
   629  // containing all the entries from oldC plus the new entry
   630  // (typ,case_,tab).
   631  func buildInterfaceSwitchCache(oldC *abi.InterfaceSwitchCache, typ *_type, case_ int, tab *itab) *abi.InterfaceSwitchCache {
   632  	oldEntries := unsafe.Slice(&oldC.Entries[0], oldC.Mask+1)
   633  
   634  	// Count the number of entries we need.
   635  	n := 1
   636  	for _, e := range oldEntries {
   637  		if e.Typ != 0 {
   638  			n++
   639  		}
   640  	}
   641  
   642  	// Figure out how big a table we need.
   643  	// We need at least one more slot than the number of entries
   644  	// so that we are guaranteed an empty slot (for termination).
   645  	newN := n * 2                         // make it at most 50% full
   646  	newN = 1 << sys.Len64(uint64(newN-1)) // round up to a power of 2
   647  
   648  	// Allocate the new table.
   649  	newSize := unsafe.Sizeof(abi.InterfaceSwitchCache{}) + uintptr(newN-1)*unsafe.Sizeof(abi.InterfaceSwitchCacheEntry{})
   650  	newC := (*abi.InterfaceSwitchCache)(mallocgc(newSize, nil, true))
   651  	newC.Mask = uintptr(newN - 1)
   652  	newEntries := unsafe.Slice(&newC.Entries[0], newN)
   653  
   654  	// Fill the new table.
   655  	addEntry := func(typ *_type, case_ int, tab *itab) {
   656  		h := int(typ.Hash) & (newN - 1)
   657  		for {
   658  			if newEntries[h].Typ == 0 {
   659  				newEntries[h].Typ = uintptr(unsafe.Pointer(typ))
   660  				newEntries[h].Case = case_
   661  				newEntries[h].Itab = uintptr(unsafe.Pointer(tab))
   662  				return
   663  			}
   664  			h = (h + 1) & (newN - 1)
   665  		}
   666  	}
   667  	for _, e := range oldEntries {
   668  		if e.Typ != 0 {
   669  			addEntry((*_type)(unsafe.Pointer(e.Typ)), e.Case, (*itab)(unsafe.Pointer(e.Itab)))
   670  		}
   671  	}
   672  	addEntry(typ, case_, tab)
   673  
   674  	return newC
   675  }
   676  
   677  // Empty interface switch cache. Contains one entry with a nil Typ (which
   678  // causes a cache lookup to fail immediately.)
   679  var emptyInterfaceSwitchCache = abi.InterfaceSwitchCache{Mask: 0}
   680  
   681  // reflect_ifaceE2I is for package reflect,
   682  // but widely used packages access it using linkname.
   683  // Notable members of the hall of shame include:
   684  //   - gitee.com/quant1x/gox
   685  //   - github.com/modern-go/reflect2
   686  //   - github.com/v2pro/plz
   687  //
   688  // Do not remove or change the type signature.
   689  //
   690  //go:linkname reflect_ifaceE2I reflect.ifaceE2I
   691  func reflect_ifaceE2I(inter *interfacetype, e eface, dst *iface) {
   692  	*dst = iface{assertE2I(inter, e._type), e.data}
   693  }
   694  
   695  //go:linkname reflectlite_ifaceE2I internal/reflectlite.ifaceE2I
   696  func reflectlite_ifaceE2I(inter *interfacetype, e eface, dst *iface) {
   697  	*dst = iface{assertE2I(inter, e._type), e.data}
   698  }
   699  
   700  func iterate_itabs(fn func(*itab)) {
   701  	// Note: only runs during stop the world or with itabLock held,
   702  	// so no other locks/atomics needed.
   703  	t := itabTable
   704  	for i := uintptr(0); i < t.size; i++ {
   705  		m := *(**itab)(add(unsafe.Pointer(&t.entries), i*goarch.PtrSize))
   706  		if m != nil {
   707  			fn(m)
   708  		}
   709  	}
   710  }
   711  
   712  // staticuint64s is used to avoid allocating in convTx for small integer values.
   713  // staticuint64s[0] == 0, staticuint64s[1] == 1, and so forth.
   714  // It is defined in assembler code so that it is read-only.
   715  var staticuint64s [256]uint64
   716  
   717  // getStaticuint64s is called by the reflect package to get a pointer
   718  // to the read-only array.
   719  //
   720  //go:linkname getStaticuint64s
   721  func getStaticuint64s() *[256]uint64 {
   722  	return &staticuint64s
   723  }
   724  
   725  // The linker redirects a reference of a method that it determined
   726  // unreachable to a reference to this function, so it will throw if
   727  // ever called.
   728  func unreachableMethod() {
   729  	throw("unreachable method called. linker bug?")
   730  }
   731
View as plain text