Source file
src/runtime/iface.go
1
2
3
4
5 package runtime
6
7 import (
8 "internal/abi"
9 "internal/goarch"
10 "internal/runtime/atomic"
11 "runtime/internal/sys"
12 "unsafe"
13 )
14
15 const itabInitSize = 512
16
17 var (
18 itabLock mutex
19 itabTable = &itabTableInit
20 itabTableInit = itabTableType{size: itabInitSize}
21 )
22
23
24 type itabTableType struct {
25 size uintptr
26 count uintptr
27 entries [itabInitSize]*itab
28 }
29
30 func itabHashFunc(inter *interfacetype, typ *_type) uintptr {
31
32 return uintptr(inter.Type.Hash ^ typ.Hash)
33 }
34
35 func getitab(inter *interfacetype, typ *_type, canfail bool) *itab {
36 if len(inter.Methods) == 0 {
37 throw("internal error - misuse of itab")
38 }
39
40
41 if typ.TFlag&abi.TFlagUncommon == 0 {
42 if canfail {
43 return nil
44 }
45 name := toRType(&inter.Type).nameOff(inter.Methods[0].Name)
46 panic(&TypeAssertionError{nil, typ, &inter.Type, name.Name()})
47 }
48
49 var m *itab
50
51
52
53
54
55 t := (*itabTableType)(atomic.Loadp(unsafe.Pointer(&itabTable)))
56 if m = t.find(inter, typ); m != nil {
57 goto finish
58 }
59
60
61 lock(&itabLock)
62 if m = itabTable.find(inter, typ); m != nil {
63 unlock(&itabLock)
64 goto finish
65 }
66
67
68 m = (*itab)(persistentalloc(unsafe.Sizeof(itab{})+uintptr(len(inter.Methods)-1)*goarch.PtrSize, 0, &memstats.other_sys))
69 m.Inter = inter
70 m.Type = typ
71
72
73
74
75
76 m.Hash = 0
77 itabInit(m, true)
78 itabAdd(m)
79 unlock(&itabLock)
80 finish:
81 if m.Fun[0] != 0 {
82 return m
83 }
84 if canfail {
85 return nil
86 }
87
88
89
90
91
92
93 panic(&TypeAssertionError{concrete: typ, asserted: &inter.Type, missingMethod: itabInit(m, false)})
94 }
95
96
97
98 func (t *itabTableType) find(inter *interfacetype, typ *_type) *itab {
99
100
101
102 mask := t.size - 1
103 h := itabHashFunc(inter, typ) & mask
104 for i := uintptr(1); ; i++ {
105 p := (**itab)(add(unsafe.Pointer(&t.entries), h*goarch.PtrSize))
106
107
108
109 m := (*itab)(atomic.Loadp(unsafe.Pointer(p)))
110 if m == nil {
111 return nil
112 }
113 if m.Inter == inter && m.Type == typ {
114 return m
115 }
116 h += i
117 h &= mask
118 }
119 }
120
121
122
123 func itabAdd(m *itab) {
124
125
126
127
128 if getg().m.mallocing != 0 {
129 throw("malloc deadlock")
130 }
131
132 t := itabTable
133 if t.count >= 3*(t.size/4) {
134
135
136
137
138 t2 := (*itabTableType)(mallocgc((2+2*t.size)*goarch.PtrSize, nil, true))
139 t2.size = t.size * 2
140
141
142
143
144
145 iterate_itabs(t2.add)
146 if t2.count != t.count {
147 throw("mismatched count during itab table copy")
148 }
149
150 atomicstorep(unsafe.Pointer(&itabTable), unsafe.Pointer(t2))
151
152 t = itabTable
153
154 }
155 t.add(m)
156 }
157
158
159
160 func (t *itabTableType) add(m *itab) {
161
162
163 mask := t.size - 1
164 h := itabHashFunc(m.Inter, m.Type) & mask
165 for i := uintptr(1); ; i++ {
166 p := (**itab)(add(unsafe.Pointer(&t.entries), h*goarch.PtrSize))
167 m2 := *p
168 if m2 == m {
169
170
171
172
173 return
174 }
175 if m2 == nil {
176
177
178
179
180 atomic.StorepNoWB(unsafe.Pointer(p), unsafe.Pointer(m))
181 t.count++
182 return
183 }
184 h += i
185 h &= mask
186 }
187 }
188
189
190
191
192
193
194
195 func itabInit(m *itab, firstTime bool) string {
196 inter := m.Inter
197 typ := m.Type
198 x := typ.Uncommon()
199
200
201
202
203
204 ni := len(inter.Methods)
205 nt := int(x.Mcount)
206 xmhdr := (*[1 << 16]abi.Method)(add(unsafe.Pointer(x), uintptr(x.Moff)))[:nt:nt]
207 j := 0
208 methods := (*[1 << 16]unsafe.Pointer)(unsafe.Pointer(&m.Fun[0]))[:ni:ni]
209 var fun0 unsafe.Pointer
210 imethods:
211 for k := 0; k < ni; k++ {
212 i := &inter.Methods[k]
213 itype := toRType(&inter.Type).typeOff(i.Typ)
214 name := toRType(&inter.Type).nameOff(i.Name)
215 iname := name.Name()
216 ipkg := pkgPath(name)
217 if ipkg == "" {
218 ipkg = inter.PkgPath.Name()
219 }
220 for ; j < nt; j++ {
221 t := &xmhdr[j]
222 rtyp := toRType(typ)
223 tname := rtyp.nameOff(t.Name)
224 if rtyp.typeOff(t.Mtyp) == itype && tname.Name() == iname {
225 pkgPath := pkgPath(tname)
226 if pkgPath == "" {
227 pkgPath = rtyp.nameOff(x.PkgPath).Name()
228 }
229 if tname.IsExported() || pkgPath == ipkg {
230 ifn := rtyp.textOff(t.Ifn)
231 if k == 0 {
232 fun0 = ifn
233 } else if firstTime {
234 methods[k] = ifn
235 }
236 continue imethods
237 }
238 }
239 }
240
241
242 return iname
243 }
244 if firstTime {
245 m.Fun[0] = uintptr(fun0)
246 }
247 return ""
248 }
249
250 func itabsinit() {
251 lockInit(&itabLock, lockRankItab)
252 lock(&itabLock)
253 for _, md := range activeModules() {
254 for _, i := range md.itablinks {
255 itabAdd(i)
256 }
257 }
258 unlock(&itabLock)
259 }
260
261
262
263
264
265 func panicdottypeE(have, want, iface *_type) {
266 panic(&TypeAssertionError{iface, have, want, ""})
267 }
268
269
270
271 func panicdottypeI(have *itab, want, iface *_type) {
272 var t *_type
273 if have != nil {
274 t = have.Type
275 }
276 panicdottypeE(t, want, iface)
277 }
278
279
280
281 func panicnildottype(want *_type) {
282 panic(&TypeAssertionError{nil, nil, want, ""})
283
284
285
286 }
287
288
289
290
291
292
293
294 type (
295 uint16InterfacePtr uint16
296 uint32InterfacePtr uint32
297 uint64InterfacePtr uint64
298 stringInterfacePtr string
299 sliceInterfacePtr []byte
300 )
301
302 var (
303 uint16Eface any = uint16InterfacePtr(0)
304 uint32Eface any = uint32InterfacePtr(0)
305 uint64Eface any = uint64InterfacePtr(0)
306 stringEface any = stringInterfacePtr("")
307 sliceEface any = sliceInterfacePtr(nil)
308
309 uint16Type *_type = efaceOf(&uint16Eface)._type
310 uint32Type *_type = efaceOf(&uint32Eface)._type
311 uint64Type *_type = efaceOf(&uint64Eface)._type
312 stringType *_type = efaceOf(&stringEface)._type
313 sliceType *_type = efaceOf(&sliceEface)._type
314 )
315
316
317
318
319
320
321
322
323
324
325 func convT(t *_type, v unsafe.Pointer) unsafe.Pointer {
326 if raceenabled {
327 raceReadObjectPC(t, v, getcallerpc(), abi.FuncPCABIInternal(convT))
328 }
329 if msanenabled {
330 msanread(v, t.Size_)
331 }
332 if asanenabled {
333 asanread(v, t.Size_)
334 }
335 x := mallocgc(t.Size_, t, true)
336 typedmemmove(t, x, v)
337 return x
338 }
339 func convTnoptr(t *_type, v unsafe.Pointer) unsafe.Pointer {
340
341 if raceenabled {
342 raceReadObjectPC(t, v, getcallerpc(), abi.FuncPCABIInternal(convTnoptr))
343 }
344 if msanenabled {
345 msanread(v, t.Size_)
346 }
347 if asanenabled {
348 asanread(v, t.Size_)
349 }
350
351 x := mallocgc(t.Size_, t, false)
352 memmove(x, v, t.Size_)
353 return x
354 }
355
356 func convT16(val uint16) (x unsafe.Pointer) {
357 if val < uint16(len(staticuint64s)) {
358 x = unsafe.Pointer(&staticuint64s[val])
359 if goarch.BigEndian {
360 x = add(x, 6)
361 }
362 } else {
363 x = mallocgc(2, uint16Type, false)
364 *(*uint16)(x) = val
365 }
366 return
367 }
368
369 func convT32(val uint32) (x unsafe.Pointer) {
370 if val < uint32(len(staticuint64s)) {
371 x = unsafe.Pointer(&staticuint64s[val])
372 if goarch.BigEndian {
373 x = add(x, 4)
374 }
375 } else {
376 x = mallocgc(4, uint32Type, false)
377 *(*uint32)(x) = val
378 }
379 return
380 }
381
382 func convT64(val uint64) (x unsafe.Pointer) {
383 if val < uint64(len(staticuint64s)) {
384 x = unsafe.Pointer(&staticuint64s[val])
385 } else {
386 x = mallocgc(8, uint64Type, false)
387 *(*uint64)(x) = val
388 }
389 return
390 }
391
392 func convTstring(val string) (x unsafe.Pointer) {
393 if val == "" {
394 x = unsafe.Pointer(&zeroVal[0])
395 } else {
396 x = mallocgc(unsafe.Sizeof(val), stringType, true)
397 *(*string)(x) = val
398 }
399 return
400 }
401
402 func convTslice(val []byte) (x unsafe.Pointer) {
403
404 if (*slice)(unsafe.Pointer(&val)).array == nil {
405 x = unsafe.Pointer(&zeroVal[0])
406 } else {
407 x = mallocgc(unsafe.Sizeof(val), sliceType, true)
408 *(*[]byte)(x) = val
409 }
410 return
411 }
412
413 func assertE2I(inter *interfacetype, t *_type) *itab {
414 if t == nil {
415
416 panic(&TypeAssertionError{nil, nil, &inter.Type, ""})
417 }
418 return getitab(inter, t, false)
419 }
420
421 func assertE2I2(inter *interfacetype, t *_type) *itab {
422 if t == nil {
423 return nil
424 }
425 return getitab(inter, t, true)
426 }
427
428
429
430
431 func typeAssert(s *abi.TypeAssert, t *_type) *itab {
432 var tab *itab
433 if t == nil {
434 if !s.CanFail {
435 panic(&TypeAssertionError{nil, nil, &s.Inter.Type, ""})
436 }
437 } else {
438 tab = getitab(s.Inter, t, s.CanFail)
439 }
440
441 if !abi.UseInterfaceSwitchCache(GOARCH) {
442 return tab
443 }
444
445
446
447 if cheaprand()&1023 != 0 {
448
449 return tab
450 }
451
452 oldC := (*abi.TypeAssertCache)(atomic.Loadp(unsafe.Pointer(&s.Cache)))
453
454 if cheaprand()&uint32(oldC.Mask) != 0 {
455
456
457 return tab
458 }
459
460
461 newC := buildTypeAssertCache(oldC, t, tab)
462
463
464
465
466 atomic_casPointer((*unsafe.Pointer)(unsafe.Pointer(&s.Cache)), unsafe.Pointer(oldC), unsafe.Pointer(newC))
467
468 return tab
469 }
470
471 func buildTypeAssertCache(oldC *abi.TypeAssertCache, typ *_type, tab *itab) *abi.TypeAssertCache {
472 oldEntries := unsafe.Slice(&oldC.Entries[0], oldC.Mask+1)
473
474
475 n := 1
476 for _, e := range oldEntries {
477 if e.Typ != 0 {
478 n++
479 }
480 }
481
482
483
484
485 newN := n * 2
486 newN = 1 << sys.Len64(uint64(newN-1))
487
488
489 newSize := unsafe.Sizeof(abi.TypeAssertCache{}) + uintptr(newN-1)*unsafe.Sizeof(abi.TypeAssertCacheEntry{})
490 newC := (*abi.TypeAssertCache)(mallocgc(newSize, nil, true))
491 newC.Mask = uintptr(newN - 1)
492 newEntries := unsafe.Slice(&newC.Entries[0], newN)
493
494
495 addEntry := func(typ *_type, tab *itab) {
496 h := int(typ.Hash) & (newN - 1)
497 for {
498 if newEntries[h].Typ == 0 {
499 newEntries[h].Typ = uintptr(unsafe.Pointer(typ))
500 newEntries[h].Itab = uintptr(unsafe.Pointer(tab))
501 return
502 }
503 h = (h + 1) & (newN - 1)
504 }
505 }
506 for _, e := range oldEntries {
507 if e.Typ != 0 {
508 addEntry((*_type)(unsafe.Pointer(e.Typ)), (*itab)(unsafe.Pointer(e.Itab)))
509 }
510 }
511 addEntry(typ, tab)
512
513 return newC
514 }
515
516
517
518 var emptyTypeAssertCache = abi.TypeAssertCache{Mask: 0}
519
520
521
522
523
524
525 func interfaceSwitch(s *abi.InterfaceSwitch, t *_type) (int, *itab) {
526 cases := unsafe.Slice(&s.Cases[0], s.NCases)
527
528
529 case_ := len(cases)
530 var tab *itab
531
532
533 for i, c := range cases {
534 tab = getitab(c, t, true)
535 if tab != nil {
536 case_ = i
537 break
538 }
539 }
540
541 if !abi.UseInterfaceSwitchCache(GOARCH) {
542 return case_, tab
543 }
544
545
546
547 if cheaprand()&1023 != 0 {
548
549
550
551 return case_, tab
552 }
553
554 oldC := (*abi.InterfaceSwitchCache)(atomic.Loadp(unsafe.Pointer(&s.Cache)))
555
556 if cheaprand()&uint32(oldC.Mask) != 0 {
557
558
559
560 return case_, tab
561 }
562
563
564 newC := buildInterfaceSwitchCache(oldC, t, case_, tab)
565
566
567
568
569 atomic_casPointer((*unsafe.Pointer)(unsafe.Pointer(&s.Cache)), unsafe.Pointer(oldC), unsafe.Pointer(newC))
570
571 return case_, tab
572 }
573
574
575
576
577 func buildInterfaceSwitchCache(oldC *abi.InterfaceSwitchCache, typ *_type, case_ int, tab *itab) *abi.InterfaceSwitchCache {
578 oldEntries := unsafe.Slice(&oldC.Entries[0], oldC.Mask+1)
579
580
581 n := 1
582 for _, e := range oldEntries {
583 if e.Typ != 0 {
584 n++
585 }
586 }
587
588
589
590
591 newN := n * 2
592 newN = 1 << sys.Len64(uint64(newN-1))
593
594
595 newSize := unsafe.Sizeof(abi.InterfaceSwitchCache{}) + uintptr(newN-1)*unsafe.Sizeof(abi.InterfaceSwitchCacheEntry{})
596 newC := (*abi.InterfaceSwitchCache)(mallocgc(newSize, nil, true))
597 newC.Mask = uintptr(newN - 1)
598 newEntries := unsafe.Slice(&newC.Entries[0], newN)
599
600
601 addEntry := func(typ *_type, case_ int, tab *itab) {
602 h := int(typ.Hash) & (newN - 1)
603 for {
604 if newEntries[h].Typ == 0 {
605 newEntries[h].Typ = uintptr(unsafe.Pointer(typ))
606 newEntries[h].Case = case_
607 newEntries[h].Itab = uintptr(unsafe.Pointer(tab))
608 return
609 }
610 h = (h + 1) & (newN - 1)
611 }
612 }
613 for _, e := range oldEntries {
614 if e.Typ != 0 {
615 addEntry((*_type)(unsafe.Pointer(e.Typ)), e.Case, (*itab)(unsafe.Pointer(e.Itab)))
616 }
617 }
618 addEntry(typ, case_, tab)
619
620 return newC
621 }
622
623
624
625 var emptyInterfaceSwitchCache = abi.InterfaceSwitchCache{Mask: 0}
626
627
628 func reflect_ifaceE2I(inter *interfacetype, e eface, dst *iface) {
629 *dst = iface{assertE2I(inter, e._type), e.data}
630 }
631
632
633 func reflectlite_ifaceE2I(inter *interfacetype, e eface, dst *iface) {
634 *dst = iface{assertE2I(inter, e._type), e.data}
635 }
636
637 func iterate_itabs(fn func(*itab)) {
638
639
640 t := itabTable
641 for i := uintptr(0); i < t.size; i++ {
642 m := *(**itab)(add(unsafe.Pointer(&t.entries), i*goarch.PtrSize))
643 if m != nil {
644 fn(m)
645 }
646 }
647 }
648
649
650 var staticuint64s = [...]uint64{
651 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
652 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
653 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
654 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
655 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
656 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
657 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
658 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
659 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
660 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,
661 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
662 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
663 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
664 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
665 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
666 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f,
667 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
668 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
669 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
670 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,
671 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
672 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
673 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
674 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,
675 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
676 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
677 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
678 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
679 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
680 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
681 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
682 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff,
683 }
684
685
686
687
688 func unreachableMethod() {
689 throw("unreachable method called. linker bug?")
690 }
691
View as plain text