Source file
src/runtime/mgcmark.go
1
2
3
4
5
6
7 package runtime
8
9 import (
10 "internal/abi"
11 "internal/goarch"
12 "internal/goexperiment"
13 "internal/runtime/atomic"
14 "runtime/internal/sys"
15 "unsafe"
16 )
17
18 const (
19 fixedRootFinalizers = iota
20 fixedRootFreeGStacks
21 fixedRootCount
22
23
24
25 rootBlockBytes = 256 << 10
26
27
28
29
30
31
32
33
34 maxObletBytes = 128 << 10
35
36
37
38
39
40
41
42 drainCheckThreshold = 100000
43
44
45
46
47
48
49
50
51
52 pagesPerSpanRoot = 512
53 )
54
55
56
57
58
59 func gcMarkRootPrepare() {
60 assertWorldStopped()
61
62
63 nBlocks := func(bytes uintptr) int {
64 return int(divRoundUp(bytes, rootBlockBytes))
65 }
66
67 work.nDataRoots = 0
68 work.nBSSRoots = 0
69
70
71 for _, datap := range activeModules() {
72 nDataRoots := nBlocks(datap.edata - datap.data)
73 if nDataRoots > work.nDataRoots {
74 work.nDataRoots = nDataRoots
75 }
76
77 nBSSRoots := nBlocks(datap.ebss - datap.bss)
78 if nBSSRoots > work.nBSSRoots {
79 work.nBSSRoots = nBSSRoots
80 }
81 }
82
83
84
85
86
87
88
89
90
91
92
93
94
95 mheap_.markArenas = mheap_.allArenas[:len(mheap_.allArenas):len(mheap_.allArenas)]
96 work.nSpanRoots = len(mheap_.markArenas) * (pagesPerArena / pagesPerSpanRoot)
97
98
99
100
101
102
103
104 work.stackRoots = allGsSnapshot()
105 work.nStackRoots = len(work.stackRoots)
106
107 work.markrootNext = 0
108 work.markrootJobs = uint32(fixedRootCount + work.nDataRoots + work.nBSSRoots + work.nSpanRoots + work.nStackRoots)
109
110
111 work.baseData = uint32(fixedRootCount)
112 work.baseBSS = work.baseData + uint32(work.nDataRoots)
113 work.baseSpans = work.baseBSS + uint32(work.nBSSRoots)
114 work.baseStacks = work.baseSpans + uint32(work.nSpanRoots)
115 work.baseEnd = work.baseStacks + uint32(work.nStackRoots)
116 }
117
118
119
120 func gcMarkRootCheck() {
121 if work.markrootNext < work.markrootJobs {
122 print(work.markrootNext, " of ", work.markrootJobs, " markroot jobs done\n")
123 throw("left over markroot jobs")
124 }
125
126
127
128
129
130
131 i := 0
132 forEachGRace(func(gp *g) {
133 if i >= work.nStackRoots {
134 return
135 }
136
137 if !gp.gcscandone {
138 println("gp", gp, "goid", gp.goid,
139 "status", readgstatus(gp),
140 "gcscandone", gp.gcscandone)
141 throw("scan missed a g")
142 }
143
144 i++
145 })
146 }
147
148
149 var oneptrmask = [...]uint8{1}
150
151
152
153
154
155
156
157
158
159
160
161
162 func markroot(gcw *gcWork, i uint32, flushBgCredit bool) int64 {
163
164 var workDone int64
165 var workCounter *atomic.Int64
166 switch {
167 case work.baseData <= i && i < work.baseBSS:
168 workCounter = &gcController.globalsScanWork
169 for _, datap := range activeModules() {
170 workDone += markrootBlock(datap.data, datap.edata-datap.data, datap.gcdatamask.bytedata, gcw, int(i-work.baseData))
171 }
172
173 case work.baseBSS <= i && i < work.baseSpans:
174 workCounter = &gcController.globalsScanWork
175 for _, datap := range activeModules() {
176 workDone += markrootBlock(datap.bss, datap.ebss-datap.bss, datap.gcbssmask.bytedata, gcw, int(i-work.baseBSS))
177 }
178
179 case i == fixedRootFinalizers:
180 for fb := allfin; fb != nil; fb = fb.alllink {
181 cnt := uintptr(atomic.Load(&fb.cnt))
182 scanblock(uintptr(unsafe.Pointer(&fb.fin[0])), cnt*unsafe.Sizeof(fb.fin[0]), &finptrmask[0], gcw, nil)
183 }
184
185 case i == fixedRootFreeGStacks:
186
187
188 systemstack(markrootFreeGStacks)
189
190 case work.baseSpans <= i && i < work.baseStacks:
191
192 markrootSpans(gcw, int(i-work.baseSpans))
193
194 default:
195
196 workCounter = &gcController.stackScanWork
197 if i < work.baseStacks || work.baseEnd <= i {
198 printlock()
199 print("runtime: markroot index ", i, " not in stack roots range [", work.baseStacks, ", ", work.baseEnd, ")\n")
200 throw("markroot: bad index")
201 }
202 gp := work.stackRoots[i-work.baseStacks]
203
204
205
206 status := readgstatus(gp)
207 if (status == _Gwaiting || status == _Gsyscall) && gp.waitsince == 0 {
208 gp.waitsince = work.tstart
209 }
210
211
212
213 systemstack(func() {
214
215
216
217
218 userG := getg().m.curg
219 selfScan := gp == userG && readgstatus(userG) == _Grunning
220 if selfScan {
221 casGToWaiting(userG, _Grunning, waitReasonGarbageCollectionScan)
222 }
223
224
225
226
227
228
229
230
231 stopped := suspendG(gp)
232 if stopped.dead {
233 gp.gcscandone = true
234 return
235 }
236 if gp.gcscandone {
237 throw("g already scanned")
238 }
239 workDone += scanstack(gp, gcw)
240 gp.gcscandone = true
241 resumeG(stopped)
242
243 if selfScan {
244 casgstatus(userG, _Gwaiting, _Grunning)
245 }
246 })
247 }
248 if workCounter != nil && workDone != 0 {
249 workCounter.Add(workDone)
250 if flushBgCredit {
251 gcFlushBgCredit(workDone)
252 }
253 }
254 return workDone
255 }
256
257
258
259
260
261
262
263 func markrootBlock(b0, n0 uintptr, ptrmask0 *uint8, gcw *gcWork, shard int) int64 {
264 if rootBlockBytes%(8*goarch.PtrSize) != 0 {
265
266 throw("rootBlockBytes must be a multiple of 8*ptrSize")
267 }
268
269
270
271
272 off := uintptr(shard) * rootBlockBytes
273 if off >= n0 {
274 return 0
275 }
276 b := b0 + off
277 ptrmask := (*uint8)(add(unsafe.Pointer(ptrmask0), uintptr(shard)*(rootBlockBytes/(8*goarch.PtrSize))))
278 n := uintptr(rootBlockBytes)
279 if off+n > n0 {
280 n = n0 - off
281 }
282
283
284 scanblock(b, n, ptrmask, gcw, nil)
285 return int64(n)
286 }
287
288
289
290
291
292 func markrootFreeGStacks() {
293
294 lock(&sched.gFree.lock)
295 list := sched.gFree.stack
296 sched.gFree.stack = gList{}
297 unlock(&sched.gFree.lock)
298 if list.empty() {
299 return
300 }
301
302
303 q := gQueue{list.head, list.head}
304 for gp := list.head.ptr(); gp != nil; gp = gp.schedlink.ptr() {
305 stackfree(gp.stack)
306 gp.stack.lo = 0
307 gp.stack.hi = 0
308
309
310 q.tail.set(gp)
311 }
312
313
314 lock(&sched.gFree.lock)
315 sched.gFree.noStack.pushAll(q)
316 unlock(&sched.gFree.lock)
317 }
318
319
320
321
322 func markrootSpans(gcw *gcWork, shard int) {
323
324
325
326
327
328
329
330
331
332 sg := mheap_.sweepgen
333
334
335 ai := mheap_.markArenas[shard/(pagesPerArena/pagesPerSpanRoot)]
336 ha := mheap_.arenas[ai.l1()][ai.l2()]
337 arenaPage := uint(uintptr(shard) * pagesPerSpanRoot % pagesPerArena)
338
339
340 specialsbits := ha.pageSpecials[arenaPage/8:]
341 specialsbits = specialsbits[:pagesPerSpanRoot/8]
342 for i := range specialsbits {
343
344 specials := atomic.Load8(&specialsbits[i])
345 if specials == 0 {
346 continue
347 }
348 for j := uint(0); j < 8; j++ {
349 if specials&(1<<j) == 0 {
350 continue
351 }
352
353
354
355
356
357
358 s := ha.spans[arenaPage+uint(i)*8+j]
359
360
361
362 if state := s.state.get(); state != mSpanInUse {
363 print("s.state = ", state, "\n")
364 throw("non in-use span found with specials bit set")
365 }
366
367 if !useCheckmark && !(s.sweepgen == sg || s.sweepgen == sg+3) {
368
369 print("sweep ", s.sweepgen, " ", sg, "\n")
370 throw("gc: unswept span")
371 }
372
373
374
375 lock(&s.speciallock)
376 for sp := s.specials; sp != nil; sp = sp.next {
377 if sp.kind != _KindSpecialFinalizer {
378 continue
379 }
380
381
382 spf := (*specialfinalizer)(unsafe.Pointer(sp))
383
384 p := s.base() + uintptr(spf.special.offset)/s.elemsize*s.elemsize
385
386
387
388
389 if !s.spanclass.noscan() {
390 scanobject(p, gcw)
391 }
392
393
394 scanblock(uintptr(unsafe.Pointer(&spf.fn)), goarch.PtrSize, &oneptrmask[0], gcw, nil)
395 }
396 unlock(&s.speciallock)
397 }
398 }
399 }
400
401
402
403
404
405 func gcAssistAlloc(gp *g) {
406
407
408 if getg() == gp.m.g0 {
409 return
410 }
411 if mp := getg().m; mp.locks > 0 || mp.preemptoff != "" {
412 return
413 }
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433 enteredMarkAssistForTracing := false
434 retry:
435 if gcCPULimiter.limiting() {
436
437
438 if enteredMarkAssistForTracing {
439 trace := traceAcquire()
440 if trace.ok() {
441 trace.GCMarkAssistDone()
442
443
444
445
446
447
448
449 gp.inMarkAssist = false
450 traceRelease(trace)
451 } else {
452
453
454
455 gp.inMarkAssist = false
456 }
457 }
458 return
459 }
460
461
462
463
464 assistWorkPerByte := gcController.assistWorkPerByte.Load()
465 assistBytesPerWork := gcController.assistBytesPerWork.Load()
466 debtBytes := -gp.gcAssistBytes
467 scanWork := int64(assistWorkPerByte * float64(debtBytes))
468 if scanWork < gcOverAssistWork {
469 scanWork = gcOverAssistWork
470 debtBytes = int64(assistBytesPerWork * float64(scanWork))
471 }
472
473
474
475
476
477
478
479 bgScanCredit := gcController.bgScanCredit.Load()
480 stolen := int64(0)
481 if bgScanCredit > 0 {
482 if bgScanCredit < scanWork {
483 stolen = bgScanCredit
484 gp.gcAssistBytes += 1 + int64(assistBytesPerWork*float64(stolen))
485 } else {
486 stolen = scanWork
487 gp.gcAssistBytes += debtBytes
488 }
489 gcController.bgScanCredit.Add(-stolen)
490
491 scanWork -= stolen
492
493 if scanWork == 0 {
494
495
496 if enteredMarkAssistForTracing {
497 trace := traceAcquire()
498 if trace.ok() {
499 trace.GCMarkAssistDone()
500
501
502
503
504
505
506
507 gp.inMarkAssist = false
508 traceRelease(trace)
509 } else {
510
511
512
513 gp.inMarkAssist = false
514 }
515 }
516 return
517 }
518 }
519 if !enteredMarkAssistForTracing {
520 trace := traceAcquire()
521 if trace.ok() {
522 if !goexperiment.ExecTracer2 {
523
524
525
526
527
528
529
530
531
532 enteredMarkAssistForTracing = true
533 }
534 trace.GCMarkAssistStart()
535
536
537 gp.inMarkAssist = true
538 traceRelease(trace)
539 } else {
540 gp.inMarkAssist = true
541 }
542 if goexperiment.ExecTracer2 {
543
544
545
546
547
548 enteredMarkAssistForTracing = true
549 }
550 }
551
552
553 systemstack(func() {
554 gcAssistAlloc1(gp, scanWork)
555
556
557 })
558
559 completed := gp.param != nil
560 gp.param = nil
561 if completed {
562 gcMarkDone()
563 }
564
565 if gp.gcAssistBytes < 0 {
566
567
568
569
570
571
572
573 if gp.preempt {
574 Gosched()
575 goto retry
576 }
577
578
579
580
581
582
583
584
585
586
587 if !gcParkAssist() {
588 goto retry
589 }
590
591
592
593 }
594 if enteredMarkAssistForTracing {
595 trace := traceAcquire()
596 if trace.ok() {
597 trace.GCMarkAssistDone()
598
599
600
601
602
603
604
605 gp.inMarkAssist = false
606 traceRelease(trace)
607 } else {
608
609
610
611 gp.inMarkAssist = false
612 }
613 }
614 }
615
616
617
618
619
620
621
622
623
624
625
626 func gcAssistAlloc1(gp *g, scanWork int64) {
627
628
629 gp.param = nil
630
631 if atomic.Load(&gcBlackenEnabled) == 0 {
632
633
634
635
636
637
638
639 gp.gcAssistBytes = 0
640 return
641 }
642
643
644
645
646
647
648 startTime := nanotime()
649 trackLimiterEvent := gp.m.p.ptr().limiterEvent.start(limiterEventMarkAssist, startTime)
650
651 decnwait := atomic.Xadd(&work.nwait, -1)
652 if decnwait == work.nproc {
653 println("runtime: work.nwait =", decnwait, "work.nproc=", work.nproc)
654 throw("nwait > work.nprocs")
655 }
656
657
658 casGToWaiting(gp, _Grunning, waitReasonGCAssistMarking)
659
660
661
662 gcw := &getg().m.p.ptr().gcw
663 workDone := gcDrainN(gcw, scanWork)
664
665 casgstatus(gp, _Gwaiting, _Grunning)
666
667
668
669
670
671
672
673 assistBytesPerWork := gcController.assistBytesPerWork.Load()
674 gp.gcAssistBytes += 1 + int64(assistBytesPerWork*float64(workDone))
675
676
677
678 incnwait := atomic.Xadd(&work.nwait, +1)
679 if incnwait > work.nproc {
680 println("runtime: work.nwait=", incnwait,
681 "work.nproc=", work.nproc)
682 throw("work.nwait > work.nproc")
683 }
684
685 if incnwait == work.nproc && !gcMarkWorkAvailable(nil) {
686
687
688
689
690 gp.param = unsafe.Pointer(gp)
691 }
692 now := nanotime()
693 duration := now - startTime
694 pp := gp.m.p.ptr()
695 pp.gcAssistTime += duration
696 if trackLimiterEvent {
697 pp.limiterEvent.stop(limiterEventMarkAssist, now)
698 }
699 if pp.gcAssistTime > gcAssistTimeSlack {
700 gcController.assistTime.Add(pp.gcAssistTime)
701 gcCPULimiter.update(now)
702 pp.gcAssistTime = 0
703 }
704 }
705
706
707
708
709 func gcWakeAllAssists() {
710 lock(&work.assistQueue.lock)
711 list := work.assistQueue.q.popList()
712 injectglist(&list)
713 unlock(&work.assistQueue.lock)
714 }
715
716
717
718
719
720 func gcParkAssist() bool {
721 lock(&work.assistQueue.lock)
722
723
724
725 if atomic.Load(&gcBlackenEnabled) == 0 {
726 unlock(&work.assistQueue.lock)
727 return true
728 }
729
730 gp := getg()
731 oldList := work.assistQueue.q
732 work.assistQueue.q.pushBack(gp)
733
734
735
736
737
738 if gcController.bgScanCredit.Load() > 0 {
739 work.assistQueue.q = oldList
740 if oldList.tail != 0 {
741 oldList.tail.ptr().schedlink.set(nil)
742 }
743 unlock(&work.assistQueue.lock)
744 return false
745 }
746
747 goparkunlock(&work.assistQueue.lock, waitReasonGCAssistWait, traceBlockGCMarkAssist, 2)
748 return true
749 }
750
751
752
753
754
755
756
757
758
759
760
761 func gcFlushBgCredit(scanWork int64) {
762 if work.assistQueue.q.empty() {
763
764
765
766
767 gcController.bgScanCredit.Add(scanWork)
768 return
769 }
770
771 assistBytesPerWork := gcController.assistBytesPerWork.Load()
772 scanBytes := int64(float64(scanWork) * assistBytesPerWork)
773
774 lock(&work.assistQueue.lock)
775 for !work.assistQueue.q.empty() && scanBytes > 0 {
776 gp := work.assistQueue.q.pop()
777
778
779 if scanBytes+gp.gcAssistBytes >= 0 {
780
781 scanBytes += gp.gcAssistBytes
782 gp.gcAssistBytes = 0
783
784
785
786
787
788
789 ready(gp, 0, false)
790 } else {
791
792 gp.gcAssistBytes += scanBytes
793 scanBytes = 0
794
795
796
797
798 work.assistQueue.q.pushBack(gp)
799 break
800 }
801 }
802
803 if scanBytes > 0 {
804
805 assistWorkPerByte := gcController.assistWorkPerByte.Load()
806 scanWork = int64(float64(scanBytes) * assistWorkPerByte)
807 gcController.bgScanCredit.Add(scanWork)
808 }
809 unlock(&work.assistQueue.lock)
810 }
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829 func scanstack(gp *g, gcw *gcWork) int64 {
830 if readgstatus(gp)&_Gscan == 0 {
831 print("runtime:scanstack: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", hex(readgstatus(gp)), "\n")
832 throw("scanstack - bad status")
833 }
834
835 switch readgstatus(gp) &^ _Gscan {
836 default:
837 print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
838 throw("mark - bad status")
839 case _Gdead:
840 return 0
841 case _Grunning:
842 print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
843 throw("scanstack: goroutine not stopped")
844 case _Grunnable, _Gsyscall, _Gwaiting:
845
846 }
847
848 if gp == getg() {
849 throw("can't scan our own stack")
850 }
851
852
853
854
855 var sp uintptr
856 if gp.syscallsp != 0 {
857 sp = gp.syscallsp
858 } else {
859 sp = gp.sched.sp
860 }
861 scannedSize := gp.stack.hi - sp
862
863
864
865 p := getg().m.p.ptr()
866 p.scannedStackSize += uint64(scannedSize)
867 p.scannedStacks++
868
869 if isShrinkStackSafe(gp) {
870
871 shrinkstack(gp)
872 } else {
873
874 gp.preemptShrink = true
875 }
876
877 var state stackScanState
878 state.stack = gp.stack
879
880 if stackTraceDebug {
881 println("stack trace goroutine", gp.goid)
882 }
883
884 if debugScanConservative && gp.asyncSafePoint {
885 print("scanning async preempted goroutine ", gp.goid, " stack [", hex(gp.stack.lo), ",", hex(gp.stack.hi), ")\n")
886 }
887
888
889
890
891 if gp.sched.ctxt != nil {
892 scanblock(uintptr(unsafe.Pointer(&gp.sched.ctxt)), goarch.PtrSize, &oneptrmask[0], gcw, &state)
893 }
894
895
896 var u unwinder
897 for u.init(gp, 0); u.valid(); u.next() {
898 scanframeworker(&u.frame, &state, gcw)
899 }
900
901
902
903
904
905 for d := gp._defer; d != nil; d = d.link {
906 if d.fn != nil {
907
908
909 scanblock(uintptr(unsafe.Pointer(&d.fn)), goarch.PtrSize, &oneptrmask[0], gcw, &state)
910 }
911 if d.link != nil {
912
913
914 scanblock(uintptr(unsafe.Pointer(&d.link)), goarch.PtrSize, &oneptrmask[0], gcw, &state)
915 }
916
917
918
919 if d.heap {
920 scanblock(uintptr(unsafe.Pointer(&d)), goarch.PtrSize, &oneptrmask[0], gcw, &state)
921 }
922 }
923 if gp._panic != nil {
924
925 state.putPtr(uintptr(unsafe.Pointer(gp._panic)), false)
926 }
927
928
929
930
931
932
933 state.buildIndex()
934 for {
935 p, conservative := state.getPtr()
936 if p == 0 {
937 break
938 }
939 obj := state.findObject(p)
940 if obj == nil {
941 continue
942 }
943 r := obj.r
944 if r == nil {
945
946 continue
947 }
948 obj.setRecord(nil)
949 if stackTraceDebug {
950 printlock()
951 print(" live stkobj at", hex(state.stack.lo+uintptr(obj.off)), "of size", obj.size)
952 if conservative {
953 print(" (conservative)")
954 }
955 println()
956 printunlock()
957 }
958 gcdata := r.gcdata()
959 var s *mspan
960 if r.useGCProg() {
961
962
963
964
965
966
967
968
969
970 s = materializeGCProg(r.ptrdata(), gcdata)
971 gcdata = (*byte)(unsafe.Pointer(s.startAddr))
972 }
973
974 b := state.stack.lo + uintptr(obj.off)
975 if conservative {
976 scanConservative(b, r.ptrdata(), gcdata, gcw, &state)
977 } else {
978 scanblock(b, r.ptrdata(), gcdata, gcw, &state)
979 }
980
981 if s != nil {
982 dematerializeGCProg(s)
983 }
984 }
985
986
987
988 for state.head != nil {
989 x := state.head
990 state.head = x.next
991 if stackTraceDebug {
992 for i := 0; i < x.nobj; i++ {
993 obj := &x.obj[i]
994 if obj.r == nil {
995 continue
996 }
997 println(" dead stkobj at", hex(gp.stack.lo+uintptr(obj.off)), "of size", obj.r.size)
998
999 }
1000 }
1001 x.nobj = 0
1002 putempty((*workbuf)(unsafe.Pointer(x)))
1003 }
1004 if state.buf != nil || state.cbuf != nil || state.freeBuf != nil {
1005 throw("remaining pointer buffers")
1006 }
1007 return int64(scannedSize)
1008 }
1009
1010
1011
1012
1013 func scanframeworker(frame *stkframe, state *stackScanState, gcw *gcWork) {
1014 if _DebugGC > 1 && frame.continpc != 0 {
1015 print("scanframe ", funcname(frame.fn), "\n")
1016 }
1017
1018 isAsyncPreempt := frame.fn.valid() && frame.fn.funcID == abi.FuncID_asyncPreempt
1019 isDebugCall := frame.fn.valid() && frame.fn.funcID == abi.FuncID_debugCallV2
1020 if state.conservative || isAsyncPreempt || isDebugCall {
1021 if debugScanConservative {
1022 println("conservatively scanning function", funcname(frame.fn), "at PC", hex(frame.continpc))
1023 }
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033 if frame.varp != 0 {
1034 size := frame.varp - frame.sp
1035 if size > 0 {
1036 scanConservative(frame.sp, size, nil, gcw, state)
1037 }
1038 }
1039
1040
1041 if n := frame.argBytes(); n != 0 {
1042
1043
1044 scanConservative(frame.argp, n, nil, gcw, state)
1045 }
1046
1047 if isAsyncPreempt || isDebugCall {
1048
1049
1050
1051
1052 state.conservative = true
1053 } else {
1054
1055
1056
1057 state.conservative = false
1058 }
1059 return
1060 }
1061
1062 locals, args, objs := frame.getStackMap(false)
1063
1064
1065 if locals.n > 0 {
1066 size := uintptr(locals.n) * goarch.PtrSize
1067 scanblock(frame.varp-size, size, locals.bytedata, gcw, state)
1068 }
1069
1070
1071 if args.n > 0 {
1072 scanblock(frame.argp, uintptr(args.n)*goarch.PtrSize, args.bytedata, gcw, state)
1073 }
1074
1075
1076 if frame.varp != 0 {
1077
1078
1079
1080 for i := range objs {
1081 obj := &objs[i]
1082 off := obj.off
1083 base := frame.varp
1084 if off >= 0 {
1085 base = frame.argp
1086 }
1087 ptr := base + uintptr(off)
1088 if ptr < frame.sp {
1089
1090 continue
1091 }
1092 if stackTraceDebug {
1093 println("stkobj at", hex(ptr), "of size", obj.size)
1094 }
1095 state.addObject(ptr, obj)
1096 }
1097 }
1098 }
1099
1100 type gcDrainFlags int
1101
1102 const (
1103 gcDrainUntilPreempt gcDrainFlags = 1 << iota
1104 gcDrainFlushBgCredit
1105 gcDrainIdle
1106 gcDrainFractional
1107 )
1108
1109
1110
1111 func gcDrainMarkWorkerIdle(gcw *gcWork) {
1112 gcDrain(gcw, gcDrainIdle|gcDrainUntilPreempt|gcDrainFlushBgCredit)
1113 }
1114
1115
1116
1117 func gcDrainMarkWorkerDedicated(gcw *gcWork, untilPreempt bool) {
1118 flags := gcDrainFlushBgCredit
1119 if untilPreempt {
1120 flags |= gcDrainUntilPreempt
1121 }
1122 gcDrain(gcw, flags)
1123 }
1124
1125
1126
1127 func gcDrainMarkWorkerFractional(gcw *gcWork) {
1128 gcDrain(gcw, gcDrainFractional|gcDrainUntilPreempt|gcDrainFlushBgCredit)
1129 }
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161 func gcDrain(gcw *gcWork, flags gcDrainFlags) {
1162 if !writeBarrier.enabled {
1163 throw("gcDrain phase incorrect")
1164 }
1165
1166
1167
1168 gp := getg().m.curg
1169 pp := gp.m.p.ptr()
1170 preemptible := flags&gcDrainUntilPreempt != 0
1171 flushBgCredit := flags&gcDrainFlushBgCredit != 0
1172 idle := flags&gcDrainIdle != 0
1173
1174 initScanWork := gcw.heapScanWork
1175
1176
1177
1178 checkWork := int64(1<<63 - 1)
1179 var check func() bool
1180 if flags&(gcDrainIdle|gcDrainFractional) != 0 {
1181 checkWork = initScanWork + drainCheckThreshold
1182 if idle {
1183 check = pollWork
1184 } else if flags&gcDrainFractional != 0 {
1185 check = pollFractionalWorkerExit
1186 }
1187 }
1188
1189
1190 if work.markrootNext < work.markrootJobs {
1191
1192
1193 for !(gp.preempt && (preemptible || sched.gcwaiting.Load() || pp.runSafePointFn != 0)) {
1194 job := atomic.Xadd(&work.markrootNext, +1) - 1
1195 if job >= work.markrootJobs {
1196 break
1197 }
1198 markroot(gcw, job, flushBgCredit)
1199 if check != nil && check() {
1200 goto done
1201 }
1202 }
1203 }
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215 for !(gp.preempt && (preemptible || sched.gcwaiting.Load() || pp.runSafePointFn != 0)) {
1216
1217
1218
1219
1220
1221 if work.full == 0 {
1222 gcw.balance()
1223 }
1224
1225 b := gcw.tryGetFast()
1226 if b == 0 {
1227 b = gcw.tryGet()
1228 if b == 0 {
1229
1230
1231
1232 wbBufFlush()
1233 b = gcw.tryGet()
1234 }
1235 }
1236 if b == 0 {
1237
1238 break
1239 }
1240 scanobject(b, gcw)
1241
1242
1243
1244
1245 if gcw.heapScanWork >= gcCreditSlack {
1246 gcController.heapScanWork.Add(gcw.heapScanWork)
1247 if flushBgCredit {
1248 gcFlushBgCredit(gcw.heapScanWork - initScanWork)
1249 initScanWork = 0
1250 }
1251 checkWork -= gcw.heapScanWork
1252 gcw.heapScanWork = 0
1253
1254 if checkWork <= 0 {
1255 checkWork += drainCheckThreshold
1256 if check != nil && check() {
1257 break
1258 }
1259 }
1260 }
1261 }
1262
1263 done:
1264
1265 if gcw.heapScanWork > 0 {
1266 gcController.heapScanWork.Add(gcw.heapScanWork)
1267 if flushBgCredit {
1268 gcFlushBgCredit(gcw.heapScanWork - initScanWork)
1269 }
1270 gcw.heapScanWork = 0
1271 }
1272 }
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287 func gcDrainN(gcw *gcWork, scanWork int64) int64 {
1288 if !writeBarrier.enabled {
1289 throw("gcDrainN phase incorrect")
1290 }
1291
1292
1293
1294 workFlushed := -gcw.heapScanWork
1295
1296
1297
1298 gp := getg().m.curg
1299 for !gp.preempt && !gcCPULimiter.limiting() && workFlushed+gcw.heapScanWork < scanWork {
1300
1301 if work.full == 0 {
1302 gcw.balance()
1303 }
1304
1305 b := gcw.tryGetFast()
1306 if b == 0 {
1307 b = gcw.tryGet()
1308 if b == 0 {
1309
1310
1311 wbBufFlush()
1312 b = gcw.tryGet()
1313 }
1314 }
1315
1316 if b == 0 {
1317
1318 if work.markrootNext < work.markrootJobs {
1319 job := atomic.Xadd(&work.markrootNext, +1) - 1
1320 if job < work.markrootJobs {
1321 workFlushed += markroot(gcw, job, false)
1322 continue
1323 }
1324 }
1325
1326 break
1327 }
1328
1329 scanobject(b, gcw)
1330
1331
1332 if gcw.heapScanWork >= gcCreditSlack {
1333 gcController.heapScanWork.Add(gcw.heapScanWork)
1334 workFlushed += gcw.heapScanWork
1335 gcw.heapScanWork = 0
1336 }
1337 }
1338
1339
1340
1341
1342
1343 return workFlushed + gcw.heapScanWork
1344 }
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355 func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork, stk *stackScanState) {
1356
1357
1358
1359 b := b0
1360 n := n0
1361
1362 for i := uintptr(0); i < n; {
1363
1364 bits := uint32(*addb(ptrmask, i/(goarch.PtrSize*8)))
1365 if bits == 0 {
1366 i += goarch.PtrSize * 8
1367 continue
1368 }
1369 for j := 0; j < 8 && i < n; j++ {
1370 if bits&1 != 0 {
1371
1372 p := *(*uintptr)(unsafe.Pointer(b + i))
1373 if p != 0 {
1374 if obj, span, objIndex := findObject(p, b, i); obj != 0 {
1375 greyobject(obj, b, i, span, gcw, objIndex)
1376 } else if stk != nil && p >= stk.stack.lo && p < stk.stack.hi {
1377 stk.putPtr(p, false)
1378 }
1379 }
1380 }
1381 bits >>= 1
1382 i += goarch.PtrSize
1383 }
1384 }
1385 }
1386
1387
1388
1389
1390
1391
1392
1393 func scanobject(b uintptr, gcw *gcWork) {
1394
1395
1396
1397
1398 sys.Prefetch(b)
1399
1400
1401
1402
1403
1404
1405 s := spanOfUnchecked(b)
1406 n := s.elemsize
1407 if n == 0 {
1408 throw("scanobject n == 0")
1409 }
1410 if s.spanclass.noscan() {
1411
1412
1413 throw("scanobject of a noscan object")
1414 }
1415
1416 var tp typePointers
1417 if n > maxObletBytes {
1418
1419
1420 if b == s.base() {
1421
1422
1423
1424
1425
1426 for oblet := b + maxObletBytes; oblet < s.base()+s.elemsize; oblet += maxObletBytes {
1427 if !gcw.putFast(oblet) {
1428 gcw.put(oblet)
1429 }
1430 }
1431 }
1432
1433
1434
1435
1436 n = s.base() + s.elemsize - b
1437 n = min(n, maxObletBytes)
1438 if goexperiment.AllocHeaders {
1439 tp = s.typePointersOfUnchecked(s.base())
1440 tp = tp.fastForward(b-tp.addr, b+n)
1441 }
1442 } else {
1443 if goexperiment.AllocHeaders {
1444 tp = s.typePointersOfUnchecked(b)
1445 }
1446 }
1447
1448 var hbits heapBits
1449 if !goexperiment.AllocHeaders {
1450 hbits = heapBitsForAddr(b, n)
1451 }
1452 var scanSize uintptr
1453 for {
1454 var addr uintptr
1455 if goexperiment.AllocHeaders {
1456 if tp, addr = tp.nextFast(); addr == 0 {
1457 if tp, addr = tp.next(b + n); addr == 0 {
1458 break
1459 }
1460 }
1461 } else {
1462 if hbits, addr = hbits.nextFast(); addr == 0 {
1463 if hbits, addr = hbits.next(); addr == 0 {
1464 break
1465 }
1466 }
1467 }
1468
1469
1470
1471
1472 scanSize = addr - b + goarch.PtrSize
1473
1474
1475
1476 obj := *(*uintptr)(unsafe.Pointer(addr))
1477
1478
1479
1480 if obj != 0 && obj-b >= n {
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490 if obj, span, objIndex := findObject(obj, b, addr-b); obj != 0 {
1491 greyobject(obj, b, addr-b, span, gcw, objIndex)
1492 }
1493 }
1494 }
1495 gcw.bytesMarked += uint64(n)
1496 gcw.heapScanWork += int64(scanSize)
1497 }
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507 func scanConservative(b, n uintptr, ptrmask *uint8, gcw *gcWork, state *stackScanState) {
1508 if debugScanConservative {
1509 printlock()
1510 print("conservatively scanning [", hex(b), ",", hex(b+n), ")\n")
1511 hexdumpWords(b, b+n, func(p uintptr) byte {
1512 if ptrmask != nil {
1513 word := (p - b) / goarch.PtrSize
1514 bits := *addb(ptrmask, word/8)
1515 if (bits>>(word%8))&1 == 0 {
1516 return '$'
1517 }
1518 }
1519
1520 val := *(*uintptr)(unsafe.Pointer(p))
1521 if state != nil && state.stack.lo <= val && val < state.stack.hi {
1522 return '@'
1523 }
1524
1525 span := spanOfHeap(val)
1526 if span == nil {
1527 return ' '
1528 }
1529 idx := span.objIndex(val)
1530 if span.isFree(idx) {
1531 return ' '
1532 }
1533 return '*'
1534 })
1535 printunlock()
1536 }
1537
1538 for i := uintptr(0); i < n; i += goarch.PtrSize {
1539 if ptrmask != nil {
1540 word := i / goarch.PtrSize
1541 bits := *addb(ptrmask, word/8)
1542 if bits == 0 {
1543
1544
1545
1546
1547
1548
1549 if i%(goarch.PtrSize*8) != 0 {
1550 throw("misaligned mask")
1551 }
1552 i += goarch.PtrSize*8 - goarch.PtrSize
1553 continue
1554 }
1555 if (bits>>(word%8))&1 == 0 {
1556 continue
1557 }
1558 }
1559
1560 val := *(*uintptr)(unsafe.Pointer(b + i))
1561
1562
1563 if state != nil && state.stack.lo <= val && val < state.stack.hi {
1564
1565
1566
1567
1568
1569
1570
1571
1572 state.putPtr(val, true)
1573 continue
1574 }
1575
1576
1577 span := spanOfHeap(val)
1578 if span == nil {
1579 continue
1580 }
1581
1582
1583 idx := span.objIndex(val)
1584 if span.isFree(idx) {
1585 continue
1586 }
1587
1588
1589 obj := span.base() + idx*span.elemsize
1590 greyobject(obj, b, i, span, gcw, idx)
1591 }
1592 }
1593
1594
1595
1596
1597
1598
1599 func shade(b uintptr) {
1600 if obj, span, objIndex := findObject(b, 0, 0); obj != 0 {
1601 gcw := &getg().m.p.ptr().gcw
1602 greyobject(obj, 0, 0, span, gcw, objIndex)
1603 }
1604 }
1605
1606
1607
1608
1609
1610
1611
1612
1613 func greyobject(obj, base, off uintptr, span *mspan, gcw *gcWork, objIndex uintptr) {
1614
1615 if obj&(goarch.PtrSize-1) != 0 {
1616 throw("greyobject: obj not pointer-aligned")
1617 }
1618 mbits := span.markBitsForIndex(objIndex)
1619
1620 if useCheckmark {
1621 if setCheckmark(obj, base, off, mbits) {
1622
1623 return
1624 }
1625 } else {
1626 if debug.gccheckmark > 0 && span.isFree(objIndex) {
1627 print("runtime: marking free object ", hex(obj), " found at *(", hex(base), "+", hex(off), ")\n")
1628 gcDumpObject("base", base, off)
1629 gcDumpObject("obj", obj, ^uintptr(0))
1630 getg().m.traceback = 2
1631 throw("marking free object")
1632 }
1633
1634
1635 if mbits.isMarked() {
1636 return
1637 }
1638 mbits.setMarked()
1639
1640
1641 arena, pageIdx, pageMask := pageIndexOf(span.base())
1642 if arena.pageMarks[pageIdx]&pageMask == 0 {
1643 atomic.Or8(&arena.pageMarks[pageIdx], pageMask)
1644 }
1645
1646
1647
1648 if span.spanclass.noscan() {
1649 gcw.bytesMarked += uint64(span.elemsize)
1650 return
1651 }
1652 }
1653
1654
1655
1656
1657
1658 sys.Prefetch(obj)
1659
1660 if !gcw.putFast(obj) {
1661 gcw.put(obj)
1662 }
1663 }
1664
1665
1666
1667 func gcDumpObject(label string, obj, off uintptr) {
1668 s := spanOf(obj)
1669 print(label, "=", hex(obj))
1670 if s == nil {
1671 print(" s=nil\n")
1672 return
1673 }
1674 print(" s.base()=", hex(s.base()), " s.limit=", hex(s.limit), " s.spanclass=", s.spanclass, " s.elemsize=", s.elemsize, " s.state=")
1675 if state := s.state.get(); 0 <= state && int(state) < len(mSpanStateNames) {
1676 print(mSpanStateNames[state], "\n")
1677 } else {
1678 print("unknown(", state, ")\n")
1679 }
1680
1681 skipped := false
1682 size := s.elemsize
1683 if s.state.get() == mSpanManual && size == 0 {
1684
1685
1686
1687 size = off + goarch.PtrSize
1688 }
1689 for i := uintptr(0); i < size; i += goarch.PtrSize {
1690
1691
1692
1693 if !(i < 128*goarch.PtrSize || off-16*goarch.PtrSize < i && i < off+16*goarch.PtrSize) {
1694 skipped = true
1695 continue
1696 }
1697 if skipped {
1698 print(" ...\n")
1699 skipped = false
1700 }
1701 print(" *(", label, "+", i, ") = ", hex(*(*uintptr)(unsafe.Pointer(obj + i))))
1702 if i == off {
1703 print(" <==")
1704 }
1705 print("\n")
1706 }
1707 if skipped {
1708 print(" ...\n")
1709 }
1710 }
1711
1712
1713
1714
1715
1716
1717
1718
1719 func gcmarknewobject(span *mspan, obj uintptr) {
1720 if useCheckmark {
1721 throw("gcmarknewobject called while doing checkmark")
1722 }
1723
1724
1725 objIndex := span.objIndex(obj)
1726 span.markBitsForIndex(objIndex).setMarked()
1727
1728
1729 arena, pageIdx, pageMask := pageIndexOf(span.base())
1730 if arena.pageMarks[pageIdx]&pageMask == 0 {
1731 atomic.Or8(&arena.pageMarks[pageIdx], pageMask)
1732 }
1733
1734 gcw := &getg().m.p.ptr().gcw
1735 gcw.bytesMarked += uint64(span.elemsize)
1736 }
1737
1738
1739
1740
1741 func gcMarkTinyAllocs() {
1742 assertWorldStopped()
1743
1744 for _, p := range allp {
1745 c := p.mcache
1746 if c == nil || c.tiny == 0 {
1747 continue
1748 }
1749 _, span, objIndex := findObject(c.tiny, 0, 0)
1750 gcw := &p.gcw
1751 greyobject(c.tiny, 0, 0, span, gcw, objIndex)
1752 }
1753 }
1754
View as plain text