Source file
src/runtime/mprof.go
1
2
3
4
5
6
7
8 package runtime
9
10 import (
11 "internal/abi"
12 "internal/runtime/atomic"
13 "runtime/internal/sys"
14 "unsafe"
15 )
16
17
18 var (
19
20 profInsertLock mutex
21
22 profBlockLock mutex
23
24 profMemActiveLock mutex
25
26
27 profMemFutureLock [len(memRecord{}.future)]mutex
28 )
29
30
31
32
33 const (
34
35 memProfile bucketType = 1 + iota
36 blockProfile
37 mutexProfile
38
39
40 buckHashSize = 179999
41
42
43
44
45
46 maxStack = 32
47 )
48
49 type bucketType int
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64 type bucket struct {
65 _ sys.NotInHeap
66 next *bucket
67 allnext *bucket
68 typ bucketType
69 hash uintptr
70 size uintptr
71 nstk uintptr
72 }
73
74
75
76 type memRecord struct {
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121 active memRecordCycle
122
123
124
125
126
127
128
129
130
131
132
133 future [3]memRecordCycle
134 }
135
136
137 type memRecordCycle struct {
138 allocs, frees uintptr
139 alloc_bytes, free_bytes uintptr
140 }
141
142
143 func (a *memRecordCycle) add(b *memRecordCycle) {
144 a.allocs += b.allocs
145 a.frees += b.frees
146 a.alloc_bytes += b.alloc_bytes
147 a.free_bytes += b.free_bytes
148 }
149
150
151
152 type blockRecord struct {
153 count float64
154 cycles int64
155 }
156
157 var (
158 mbuckets atomic.UnsafePointer
159 bbuckets atomic.UnsafePointer
160 xbuckets atomic.UnsafePointer
161 buckhash atomic.UnsafePointer
162
163 mProfCycle mProfCycleHolder
164 )
165
166 type buckhashArray [buckHashSize]atomic.UnsafePointer
167
168 const mProfCycleWrap = uint32(len(memRecord{}.future)) * (2 << 24)
169
170
171
172
173
174 type mProfCycleHolder struct {
175 value atomic.Uint32
176 }
177
178
179 func (c *mProfCycleHolder) read() (cycle uint32) {
180 v := c.value.Load()
181 cycle = v >> 1
182 return cycle
183 }
184
185
186
187 func (c *mProfCycleHolder) setFlushed() (cycle uint32, alreadyFlushed bool) {
188 for {
189 prev := c.value.Load()
190 cycle = prev >> 1
191 alreadyFlushed = (prev & 0x1) != 0
192 next := prev | 0x1
193 if c.value.CompareAndSwap(prev, next) {
194 return cycle, alreadyFlushed
195 }
196 }
197 }
198
199
200
201 func (c *mProfCycleHolder) increment() {
202
203
204
205 for {
206 prev := c.value.Load()
207 cycle := prev >> 1
208 cycle = (cycle + 1) % mProfCycleWrap
209 next := cycle << 1
210 if c.value.CompareAndSwap(prev, next) {
211 break
212 }
213 }
214 }
215
216
217 func newBucket(typ bucketType, nstk int) *bucket {
218 size := unsafe.Sizeof(bucket{}) + uintptr(nstk)*unsafe.Sizeof(uintptr(0))
219 switch typ {
220 default:
221 throw("invalid profile bucket type")
222 case memProfile:
223 size += unsafe.Sizeof(memRecord{})
224 case blockProfile, mutexProfile:
225 size += unsafe.Sizeof(blockRecord{})
226 }
227
228 b := (*bucket)(persistentalloc(size, 0, &memstats.buckhash_sys))
229 b.typ = typ
230 b.nstk = uintptr(nstk)
231 return b
232 }
233
234
235 func (b *bucket) stk() []uintptr {
236 stk := (*[maxStack]uintptr)(add(unsafe.Pointer(b), unsafe.Sizeof(*b)))
237 if b.nstk > maxStack {
238
239 throw("bad profile stack count")
240 }
241 return stk[:b.nstk:b.nstk]
242 }
243
244
245 func (b *bucket) mp() *memRecord {
246 if b.typ != memProfile {
247 throw("bad use of bucket.mp")
248 }
249 data := add(unsafe.Pointer(b), unsafe.Sizeof(*b)+b.nstk*unsafe.Sizeof(uintptr(0)))
250 return (*memRecord)(data)
251 }
252
253
254 func (b *bucket) bp() *blockRecord {
255 if b.typ != blockProfile && b.typ != mutexProfile {
256 throw("bad use of bucket.bp")
257 }
258 data := add(unsafe.Pointer(b), unsafe.Sizeof(*b)+b.nstk*unsafe.Sizeof(uintptr(0)))
259 return (*blockRecord)(data)
260 }
261
262
263 func stkbucket(typ bucketType, size uintptr, stk []uintptr, alloc bool) *bucket {
264 bh := (*buckhashArray)(buckhash.Load())
265 if bh == nil {
266 lock(&profInsertLock)
267
268 bh = (*buckhashArray)(buckhash.Load())
269 if bh == nil {
270 bh = (*buckhashArray)(sysAlloc(unsafe.Sizeof(buckhashArray{}), &memstats.buckhash_sys))
271 if bh == nil {
272 throw("runtime: cannot allocate memory")
273 }
274 buckhash.StoreNoWB(unsafe.Pointer(bh))
275 }
276 unlock(&profInsertLock)
277 }
278
279
280 var h uintptr
281 for _, pc := range stk {
282 h += pc
283 h += h << 10
284 h ^= h >> 6
285 }
286
287 h += size
288 h += h << 10
289 h ^= h >> 6
290
291 h += h << 3
292 h ^= h >> 11
293
294 i := int(h % buckHashSize)
295
296 for b := (*bucket)(bh[i].Load()); b != nil; b = b.next {
297 if b.typ == typ && b.hash == h && b.size == size && eqslice(b.stk(), stk) {
298 return b
299 }
300 }
301
302 if !alloc {
303 return nil
304 }
305
306 lock(&profInsertLock)
307
308 for b := (*bucket)(bh[i].Load()); b != nil; b = b.next {
309 if b.typ == typ && b.hash == h && b.size == size && eqslice(b.stk(), stk) {
310 unlock(&profInsertLock)
311 return b
312 }
313 }
314
315
316 b := newBucket(typ, len(stk))
317 copy(b.stk(), stk)
318 b.hash = h
319 b.size = size
320
321 var allnext *atomic.UnsafePointer
322 if typ == memProfile {
323 allnext = &mbuckets
324 } else if typ == mutexProfile {
325 allnext = &xbuckets
326 } else {
327 allnext = &bbuckets
328 }
329
330 b.next = (*bucket)(bh[i].Load())
331 b.allnext = (*bucket)(allnext.Load())
332
333 bh[i].StoreNoWB(unsafe.Pointer(b))
334 allnext.StoreNoWB(unsafe.Pointer(b))
335
336 unlock(&profInsertLock)
337 return b
338 }
339
340 func eqslice(x, y []uintptr) bool {
341 if len(x) != len(y) {
342 return false
343 }
344 for i, xi := range x {
345 if xi != y[i] {
346 return false
347 }
348 }
349 return true
350 }
351
352
353
354
355
356
357
358
359
360 func mProf_NextCycle() {
361 mProfCycle.increment()
362 }
363
364
365
366
367
368
369
370
371 func mProf_Flush() {
372 cycle, alreadyFlushed := mProfCycle.setFlushed()
373 if alreadyFlushed {
374 return
375 }
376
377 index := cycle % uint32(len(memRecord{}.future))
378 lock(&profMemActiveLock)
379 lock(&profMemFutureLock[index])
380 mProf_FlushLocked(index)
381 unlock(&profMemFutureLock[index])
382 unlock(&profMemActiveLock)
383 }
384
385
386
387
388
389 func mProf_FlushLocked(index uint32) {
390 assertLockHeld(&profMemActiveLock)
391 assertLockHeld(&profMemFutureLock[index])
392 head := (*bucket)(mbuckets.Load())
393 for b := head; b != nil; b = b.allnext {
394 mp := b.mp()
395
396
397
398 mpc := &mp.future[index]
399 mp.active.add(mpc)
400 *mpc = memRecordCycle{}
401 }
402 }
403
404
405
406
407
408 func mProf_PostSweep() {
409
410
411
412
413
414 cycle := mProfCycle.read() + 1
415
416 index := cycle % uint32(len(memRecord{}.future))
417 lock(&profMemActiveLock)
418 lock(&profMemFutureLock[index])
419 mProf_FlushLocked(index)
420 unlock(&profMemFutureLock[index])
421 unlock(&profMemActiveLock)
422 }
423
424
425 func mProf_Malloc(p unsafe.Pointer, size uintptr) {
426 var stk [maxStack]uintptr
427 nstk := callers(4, stk[:])
428
429 index := (mProfCycle.read() + 2) % uint32(len(memRecord{}.future))
430
431 b := stkbucket(memProfile, size, stk[:nstk], true)
432 mp := b.mp()
433 mpc := &mp.future[index]
434
435 lock(&profMemFutureLock[index])
436 mpc.allocs++
437 mpc.alloc_bytes += size
438 unlock(&profMemFutureLock[index])
439
440
441
442
443
444 systemstack(func() {
445 setprofilebucket(p, b)
446 })
447 }
448
449
450 func mProf_Free(b *bucket, size uintptr) {
451 index := (mProfCycle.read() + 1) % uint32(len(memRecord{}.future))
452
453 mp := b.mp()
454 mpc := &mp.future[index]
455
456 lock(&profMemFutureLock[index])
457 mpc.frees++
458 mpc.free_bytes += size
459 unlock(&profMemFutureLock[index])
460 }
461
462 var blockprofilerate uint64
463
464
465
466
467
468
469
470 func SetBlockProfileRate(rate int) {
471 var r int64
472 if rate <= 0 {
473 r = 0
474 } else if rate == 1 {
475 r = 1
476 } else {
477
478 r = int64(float64(rate) * float64(ticksPerSecond()) / (1000 * 1000 * 1000))
479 if r == 0 {
480 r = 1
481 }
482 }
483
484 atomic.Store64(&blockprofilerate, uint64(r))
485 }
486
487 func blockevent(cycles int64, skip int) {
488 if cycles <= 0 {
489 cycles = 1
490 }
491
492 rate := int64(atomic.Load64(&blockprofilerate))
493 if blocksampled(cycles, rate) {
494 saveblockevent(cycles, rate, skip+1, blockProfile)
495 }
496 }
497
498
499
500 func blocksampled(cycles, rate int64) bool {
501 if rate <= 0 || (rate > cycles && cheaprand64()%rate > cycles) {
502 return false
503 }
504 return true
505 }
506
507 func saveblockevent(cycles, rate int64, skip int, which bucketType) {
508 gp := getg()
509 var nstk int
510 var stk [maxStack]uintptr
511 if gp.m.curg == nil || gp.m.curg == gp {
512 nstk = callers(skip, stk[:])
513 } else {
514 nstk = gcallers(gp.m.curg, skip, stk[:])
515 }
516
517 saveBlockEventStack(cycles, rate, stk[:nstk], which)
518 }
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578 type lockTimer struct {
579 lock *mutex
580 timeRate int64
581 timeStart int64
582 tickStart int64
583 }
584
585 func (lt *lockTimer) begin() {
586 rate := int64(atomic.Load64(&mutexprofilerate))
587
588 lt.timeRate = gTrackingPeriod
589 if rate != 0 && rate < lt.timeRate {
590 lt.timeRate = rate
591 }
592 if int64(cheaprand())%lt.timeRate == 0 {
593 lt.timeStart = nanotime()
594 }
595
596 if rate > 0 && int64(cheaprand())%rate == 0 {
597 lt.tickStart = cputicks()
598 }
599 }
600
601 func (lt *lockTimer) end() {
602 gp := getg()
603
604 if lt.timeStart != 0 {
605 nowTime := nanotime()
606 gp.m.mLockProfile.waitTime.Add((nowTime - lt.timeStart) * lt.timeRate)
607 }
608
609 if lt.tickStart != 0 {
610 nowTick := cputicks()
611 gp.m.mLockProfile.recordLock(nowTick-lt.tickStart, lt.lock)
612 }
613 }
614
615 type mLockProfile struct {
616 waitTime atomic.Int64
617 stack [maxStack]uintptr
618 pending uintptr
619 cycles int64
620 cyclesLost int64
621 disabled bool
622 }
623
624 func (prof *mLockProfile) recordLock(cycles int64, l *mutex) {
625 if cycles <= 0 {
626 return
627 }
628
629 if prof.disabled {
630
631
632
633 prof.cyclesLost += cycles
634 return
635 }
636
637 if uintptr(unsafe.Pointer(l)) == prof.pending {
638
639
640 prof.cycles += cycles
641 return
642 }
643
644 if prev := prof.cycles; prev > 0 {
645
646
647
648 prevScore := uint64(cheaprand64()) % uint64(prev)
649 thisScore := uint64(cheaprand64()) % uint64(cycles)
650 if prevScore > thisScore {
651 prof.cyclesLost += cycles
652 return
653 } else {
654 prof.cyclesLost += prev
655 }
656 }
657
658
659
660
661 prof.pending = uintptr(unsafe.Pointer(l))
662 prof.cycles = cycles
663 }
664
665
666
667
668 func (prof *mLockProfile) recordUnlock(l *mutex) {
669 if uintptr(unsafe.Pointer(l)) == prof.pending {
670 prof.captureStack()
671 }
672 if gp := getg(); gp.m.locks == 1 && gp.m.mLockProfile.cycles != 0 {
673 prof.store()
674 }
675 }
676
677 func (prof *mLockProfile) captureStack() {
678 skip := 3
679 if staticLockRanking {
680
681
682
683
684
685
686
687
688
689 skip += 1
690 }
691 prof.pending = 0
692
693 if debug.runtimeContentionStacks.Load() == 0 {
694 prof.stack[0] = abi.FuncPCABIInternal(_LostContendedRuntimeLock) + sys.PCQuantum
695 prof.stack[1] = 0
696 return
697 }
698
699 var nstk int
700 gp := getg()
701 sp := getcallersp()
702 pc := getcallerpc()
703 systemstack(func() {
704 var u unwinder
705 u.initAt(pc, sp, 0, gp, unwindSilentErrors|unwindJumpStack)
706 nstk = tracebackPCs(&u, skip, prof.stack[:])
707 })
708 if nstk < len(prof.stack) {
709 prof.stack[nstk] = 0
710 }
711 }
712
713 func (prof *mLockProfile) store() {
714
715
716
717
718 mp := acquirem()
719 prof.disabled = true
720
721 nstk := maxStack
722 for i := 0; i < nstk; i++ {
723 if pc := prof.stack[i]; pc == 0 {
724 nstk = i
725 break
726 }
727 }
728
729 cycles, lost := prof.cycles, prof.cyclesLost
730 prof.cycles, prof.cyclesLost = 0, 0
731
732 rate := int64(atomic.Load64(&mutexprofilerate))
733 saveBlockEventStack(cycles, rate, prof.stack[:nstk], mutexProfile)
734 if lost > 0 {
735 lostStk := [...]uintptr{
736 abi.FuncPCABIInternal(_LostContendedRuntimeLock) + sys.PCQuantum,
737 }
738 saveBlockEventStack(lost, rate, lostStk[:], mutexProfile)
739 }
740
741 prof.disabled = false
742 releasem(mp)
743 }
744
745 func saveBlockEventStack(cycles, rate int64, stk []uintptr, which bucketType) {
746 b := stkbucket(which, 0, stk, true)
747 bp := b.bp()
748
749 lock(&profBlockLock)
750
751
752
753
754
755 if which == blockProfile && cycles < rate {
756
757 bp.count += float64(rate) / float64(cycles)
758 bp.cycles += rate
759 } else if which == mutexProfile {
760 bp.count += float64(rate)
761 bp.cycles += rate * cycles
762 } else {
763 bp.count++
764 bp.cycles += cycles
765 }
766 unlock(&profBlockLock)
767 }
768
769 var mutexprofilerate uint64
770
771
772
773
774
775
776
777
778 func SetMutexProfileFraction(rate int) int {
779 if rate < 0 {
780 return int(mutexprofilerate)
781 }
782 old := mutexprofilerate
783 atomic.Store64(&mutexprofilerate, uint64(rate))
784 return int(old)
785 }
786
787
788 func mutexevent(cycles int64, skip int) {
789 if cycles < 0 {
790 cycles = 0
791 }
792 rate := int64(atomic.Load64(&mutexprofilerate))
793 if rate > 0 && cheaprand64()%rate == 0 {
794 saveblockevent(cycles, rate, skip+1, mutexProfile)
795 }
796 }
797
798
799
800
801 type StackRecord struct {
802 Stack0 [32]uintptr
803 }
804
805
806
807 func (r *StackRecord) Stack() []uintptr {
808 for i, v := range r.Stack0 {
809 if v == 0 {
810 return r.Stack0[0:i]
811 }
812 }
813 return r.Stack0[0:]
814 }
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830 var MemProfileRate int = 512 * 1024
831
832
833
834
835 var disableMemoryProfiling bool
836
837
838
839 type MemProfileRecord struct {
840 AllocBytes, FreeBytes int64
841 AllocObjects, FreeObjects int64
842 Stack0 [32]uintptr
843 }
844
845
846 func (r *MemProfileRecord) InUseBytes() int64 { return r.AllocBytes - r.FreeBytes }
847
848
849 func (r *MemProfileRecord) InUseObjects() int64 {
850 return r.AllocObjects - r.FreeObjects
851 }
852
853
854
855 func (r *MemProfileRecord) Stack() []uintptr {
856 for i, v := range r.Stack0 {
857 if v == 0 {
858 return r.Stack0[0:i]
859 }
860 }
861 return r.Stack0[0:]
862 }
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885 func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) {
886 cycle := mProfCycle.read()
887
888
889
890 index := cycle % uint32(len(memRecord{}.future))
891 lock(&profMemActiveLock)
892 lock(&profMemFutureLock[index])
893 mProf_FlushLocked(index)
894 unlock(&profMemFutureLock[index])
895 clear := true
896 head := (*bucket)(mbuckets.Load())
897 for b := head; b != nil; b = b.allnext {
898 mp := b.mp()
899 if inuseZero || mp.active.alloc_bytes != mp.active.free_bytes {
900 n++
901 }
902 if mp.active.allocs != 0 || mp.active.frees != 0 {
903 clear = false
904 }
905 }
906 if clear {
907
908
909
910
911 n = 0
912 for b := head; b != nil; b = b.allnext {
913 mp := b.mp()
914 for c := range mp.future {
915 lock(&profMemFutureLock[c])
916 mp.active.add(&mp.future[c])
917 mp.future[c] = memRecordCycle{}
918 unlock(&profMemFutureLock[c])
919 }
920 if inuseZero || mp.active.alloc_bytes != mp.active.free_bytes {
921 n++
922 }
923 }
924 }
925 if n <= len(p) {
926 ok = true
927 idx := 0
928 for b := head; b != nil; b = b.allnext {
929 mp := b.mp()
930 if inuseZero || mp.active.alloc_bytes != mp.active.free_bytes {
931 record(&p[idx], b)
932 idx++
933 }
934 }
935 }
936 unlock(&profMemActiveLock)
937 return
938 }
939
940
941 func record(r *MemProfileRecord, b *bucket) {
942 mp := b.mp()
943 r.AllocBytes = int64(mp.active.alloc_bytes)
944 r.FreeBytes = int64(mp.active.free_bytes)
945 r.AllocObjects = int64(mp.active.allocs)
946 r.FreeObjects = int64(mp.active.frees)
947 if raceenabled {
948 racewriterangepc(unsafe.Pointer(&r.Stack0[0]), unsafe.Sizeof(r.Stack0), getcallerpc(), abi.FuncPCABIInternal(MemProfile))
949 }
950 if msanenabled {
951 msanwrite(unsafe.Pointer(&r.Stack0[0]), unsafe.Sizeof(r.Stack0))
952 }
953 if asanenabled {
954 asanwrite(unsafe.Pointer(&r.Stack0[0]), unsafe.Sizeof(r.Stack0))
955 }
956 copy(r.Stack0[:], b.stk())
957 clear(r.Stack0[b.nstk:])
958 }
959
960 func iterate_memprof(fn func(*bucket, uintptr, *uintptr, uintptr, uintptr, uintptr)) {
961 lock(&profMemActiveLock)
962 head := (*bucket)(mbuckets.Load())
963 for b := head; b != nil; b = b.allnext {
964 mp := b.mp()
965 fn(b, b.nstk, &b.stk()[0], b.size, mp.active.allocs, mp.active.frees)
966 }
967 unlock(&profMemActiveLock)
968 }
969
970
971
972 type BlockProfileRecord struct {
973 Count int64
974 Cycles int64
975 StackRecord
976 }
977
978
979
980
981
982
983
984
985 func BlockProfile(p []BlockProfileRecord) (n int, ok bool) {
986 lock(&profBlockLock)
987 head := (*bucket)(bbuckets.Load())
988 for b := head; b != nil; b = b.allnext {
989 n++
990 }
991 if n <= len(p) {
992 ok = true
993 for b := head; b != nil; b = b.allnext {
994 bp := b.bp()
995 r := &p[0]
996 r.Count = int64(bp.count)
997
998
999 if r.Count == 0 {
1000 r.Count = 1
1001 }
1002 r.Cycles = bp.cycles
1003 if raceenabled {
1004 racewriterangepc(unsafe.Pointer(&r.Stack0[0]), unsafe.Sizeof(r.Stack0), getcallerpc(), abi.FuncPCABIInternal(BlockProfile))
1005 }
1006 if msanenabled {
1007 msanwrite(unsafe.Pointer(&r.Stack0[0]), unsafe.Sizeof(r.Stack0))
1008 }
1009 if asanenabled {
1010 asanwrite(unsafe.Pointer(&r.Stack0[0]), unsafe.Sizeof(r.Stack0))
1011 }
1012 i := copy(r.Stack0[:], b.stk())
1013 clear(r.Stack0[i:])
1014 p = p[1:]
1015 }
1016 }
1017 unlock(&profBlockLock)
1018 return
1019 }
1020
1021
1022
1023
1024
1025
1026
1027 func MutexProfile(p []BlockProfileRecord) (n int, ok bool) {
1028 lock(&profBlockLock)
1029 head := (*bucket)(xbuckets.Load())
1030 for b := head; b != nil; b = b.allnext {
1031 n++
1032 }
1033 if n <= len(p) {
1034 ok = true
1035 for b := head; b != nil; b = b.allnext {
1036 bp := b.bp()
1037 r := &p[0]
1038 r.Count = int64(bp.count)
1039 r.Cycles = bp.cycles
1040 i := copy(r.Stack0[:], b.stk())
1041 clear(r.Stack0[i:])
1042 p = p[1:]
1043 }
1044 }
1045 unlock(&profBlockLock)
1046 return
1047 }
1048
1049
1050
1051
1052
1053
1054
1055 func ThreadCreateProfile(p []StackRecord) (n int, ok bool) {
1056 first := (*m)(atomic.Loadp(unsafe.Pointer(&allm)))
1057 for mp := first; mp != nil; mp = mp.alllink {
1058 n++
1059 }
1060 if n <= len(p) {
1061 ok = true
1062 i := 0
1063 for mp := first; mp != nil; mp = mp.alllink {
1064 p[i].Stack0 = mp.createstack
1065 i++
1066 }
1067 }
1068 return
1069 }
1070
1071
1072 func runtime_goroutineProfileWithLabels(p []StackRecord, labels []unsafe.Pointer) (n int, ok bool) {
1073 return goroutineProfileWithLabels(p, labels)
1074 }
1075
1076
1077 func goroutineProfileWithLabels(p []StackRecord, labels []unsafe.Pointer) (n int, ok bool) {
1078 if labels != nil && len(labels) != len(p) {
1079 labels = nil
1080 }
1081
1082 return goroutineProfileWithLabelsConcurrent(p, labels)
1083 }
1084
1085 var goroutineProfile = struct {
1086 sema uint32
1087 active bool
1088 offset atomic.Int64
1089 records []StackRecord
1090 labels []unsafe.Pointer
1091 }{
1092 sema: 1,
1093 }
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106 type goroutineProfileState uint32
1107
1108 const (
1109 goroutineProfileAbsent goroutineProfileState = iota
1110 goroutineProfileInProgress
1111 goroutineProfileSatisfied
1112 )
1113
1114 type goroutineProfileStateHolder atomic.Uint32
1115
1116 func (p *goroutineProfileStateHolder) Load() goroutineProfileState {
1117 return goroutineProfileState((*atomic.Uint32)(p).Load())
1118 }
1119
1120 func (p *goroutineProfileStateHolder) Store(value goroutineProfileState) {
1121 (*atomic.Uint32)(p).Store(uint32(value))
1122 }
1123
1124 func (p *goroutineProfileStateHolder) CompareAndSwap(old, new goroutineProfileState) bool {
1125 return (*atomic.Uint32)(p).CompareAndSwap(uint32(old), uint32(new))
1126 }
1127
1128 func goroutineProfileWithLabelsConcurrent(p []StackRecord, labels []unsafe.Pointer) (n int, ok bool) {
1129 if len(p) == 0 {
1130
1131
1132
1133
1134 return int(gcount()), false
1135 }
1136
1137 semacquire(&goroutineProfile.sema)
1138
1139 ourg := getg()
1140
1141 stw := stopTheWorld(stwGoroutineProfile)
1142
1143
1144
1145
1146
1147
1148
1149 n = int(gcount())
1150 if fingStatus.Load()&fingRunningFinalizer != 0 {
1151 n++
1152 }
1153
1154 if n > len(p) {
1155
1156
1157
1158 startTheWorld(stw)
1159 semrelease(&goroutineProfile.sema)
1160 return n, false
1161 }
1162
1163
1164 sp := getcallersp()
1165 pc := getcallerpc()
1166 systemstack(func() {
1167 saveg(pc, sp, ourg, &p[0])
1168 })
1169 if labels != nil {
1170 labels[0] = ourg.labels
1171 }
1172 ourg.goroutineProfiled.Store(goroutineProfileSatisfied)
1173 goroutineProfile.offset.Store(1)
1174
1175
1176
1177
1178
1179
1180 goroutineProfile.active = true
1181 goroutineProfile.records = p
1182 goroutineProfile.labels = labels
1183
1184
1185
1186 if fing != nil {
1187 fing.goroutineProfiled.Store(goroutineProfileSatisfied)
1188 if readgstatus(fing) != _Gdead && !isSystemGoroutine(fing, false) {
1189 doRecordGoroutineProfile(fing)
1190 }
1191 }
1192 startTheWorld(stw)
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205 forEachGRace(func(gp1 *g) {
1206 tryRecordGoroutineProfile(gp1, Gosched)
1207 })
1208
1209 stw = stopTheWorld(stwGoroutineProfileCleanup)
1210 endOffset := goroutineProfile.offset.Swap(0)
1211 goroutineProfile.active = false
1212 goroutineProfile.records = nil
1213 goroutineProfile.labels = nil
1214 startTheWorld(stw)
1215
1216
1217
1218 forEachGRace(func(gp1 *g) {
1219 gp1.goroutineProfiled.Store(goroutineProfileAbsent)
1220 })
1221
1222 if raceenabled {
1223 raceacquire(unsafe.Pointer(&labelSync))
1224 }
1225
1226 if n != int(endOffset) {
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236 }
1237
1238 semrelease(&goroutineProfile.sema)
1239 return n, true
1240 }
1241
1242
1243
1244
1245
1246 func tryRecordGoroutineProfileWB(gp1 *g) {
1247 if getg().m.p.ptr() == nil {
1248 throw("no P available, write barriers are forbidden")
1249 }
1250 tryRecordGoroutineProfile(gp1, osyield)
1251 }
1252
1253
1254
1255
1256 func tryRecordGoroutineProfile(gp1 *g, yield func()) {
1257 if readgstatus(gp1) == _Gdead {
1258
1259
1260
1261
1262 return
1263 }
1264 if isSystemGoroutine(gp1, true) {
1265
1266
1267 return
1268 }
1269
1270 for {
1271 prev := gp1.goroutineProfiled.Load()
1272 if prev == goroutineProfileSatisfied {
1273
1274
1275 break
1276 }
1277 if prev == goroutineProfileInProgress {
1278
1279
1280 yield()
1281 continue
1282 }
1283
1284
1285
1286
1287
1288
1289 mp := acquirem()
1290 if gp1.goroutineProfiled.CompareAndSwap(goroutineProfileAbsent, goroutineProfileInProgress) {
1291 doRecordGoroutineProfile(gp1)
1292 gp1.goroutineProfiled.Store(goroutineProfileSatisfied)
1293 }
1294 releasem(mp)
1295 }
1296 }
1297
1298
1299
1300
1301
1302
1303
1304
1305 func doRecordGoroutineProfile(gp1 *g) {
1306 if readgstatus(gp1) == _Grunning {
1307 print("doRecordGoroutineProfile gp1=", gp1.goid, "\n")
1308 throw("cannot read stack of running goroutine")
1309 }
1310
1311 offset := int(goroutineProfile.offset.Add(1)) - 1
1312
1313 if offset >= len(goroutineProfile.records) {
1314
1315
1316
1317 return
1318 }
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328 systemstack(func() { saveg(^uintptr(0), ^uintptr(0), gp1, &goroutineProfile.records[offset]) })
1329
1330 if goroutineProfile.labels != nil {
1331 goroutineProfile.labels[offset] = gp1.labels
1332 }
1333 }
1334
1335 func goroutineProfileWithLabelsSync(p []StackRecord, labels []unsafe.Pointer) (n int, ok bool) {
1336 gp := getg()
1337
1338 isOK := func(gp1 *g) bool {
1339
1340
1341 return gp1 != gp && readgstatus(gp1) != _Gdead && !isSystemGoroutine(gp1, false)
1342 }
1343
1344 stw := stopTheWorld(stwGoroutineProfile)
1345
1346
1347 n = 1
1348 forEachGRace(func(gp1 *g) {
1349 if isOK(gp1) {
1350 n++
1351 }
1352 })
1353
1354 if n <= len(p) {
1355 ok = true
1356 r, lbl := p, labels
1357
1358
1359 sp := getcallersp()
1360 pc := getcallerpc()
1361 systemstack(func() {
1362 saveg(pc, sp, gp, &r[0])
1363 })
1364 r = r[1:]
1365
1366
1367 if labels != nil {
1368 lbl[0] = gp.labels
1369 lbl = lbl[1:]
1370 }
1371
1372
1373 forEachGRace(func(gp1 *g) {
1374 if !isOK(gp1) {
1375 return
1376 }
1377
1378 if len(r) == 0 {
1379
1380
1381 return
1382 }
1383
1384
1385
1386
1387 systemstack(func() { saveg(^uintptr(0), ^uintptr(0), gp1, &r[0]) })
1388 if labels != nil {
1389 lbl[0] = gp1.labels
1390 lbl = lbl[1:]
1391 }
1392 r = r[1:]
1393 })
1394 }
1395
1396 if raceenabled {
1397 raceacquire(unsafe.Pointer(&labelSync))
1398 }
1399
1400 startTheWorld(stw)
1401 return n, ok
1402 }
1403
1404
1405
1406
1407
1408
1409
1410 func GoroutineProfile(p []StackRecord) (n int, ok bool) {
1411
1412 return goroutineProfileWithLabels(p, nil)
1413 }
1414
1415 func saveg(pc, sp uintptr, gp *g, r *StackRecord) {
1416 var u unwinder
1417 u.initAt(pc, sp, 0, gp, unwindSilentErrors)
1418 n := tracebackPCs(&u, 0, r.Stack0[:])
1419 if n < len(r.Stack0) {
1420 r.Stack0[n] = 0
1421 }
1422 }
1423
1424
1425
1426
1427
1428 func Stack(buf []byte, all bool) int {
1429 var stw worldStop
1430 if all {
1431 stw = stopTheWorld(stwAllGoroutinesStack)
1432 }
1433
1434 n := 0
1435 if len(buf) > 0 {
1436 gp := getg()
1437 sp := getcallersp()
1438 pc := getcallerpc()
1439 systemstack(func() {
1440 g0 := getg()
1441
1442
1443
1444 g0.m.traceback = 1
1445 g0.writebuf = buf[0:0:len(buf)]
1446 goroutineheader(gp)
1447 traceback(pc, sp, 0, gp)
1448 if all {
1449 tracebackothers(gp)
1450 }
1451 g0.m.traceback = 0
1452 n = len(g0.writebuf)
1453 g0.writebuf = nil
1454 })
1455 }
1456
1457 if all {
1458 startTheWorld(stw)
1459 }
1460 return n
1461 }
1462
1463
1464
1465 var tracelock mutex
1466
1467 func tracealloc(p unsafe.Pointer, size uintptr, typ *_type) {
1468 lock(&tracelock)
1469 gp := getg()
1470 gp.m.traceback = 2
1471 if typ == nil {
1472 print("tracealloc(", p, ", ", hex(size), ")\n")
1473 } else {
1474 print("tracealloc(", p, ", ", hex(size), ", ", toRType(typ).string(), ")\n")
1475 }
1476 if gp.m.curg == nil || gp == gp.m.curg {
1477 goroutineheader(gp)
1478 pc := getcallerpc()
1479 sp := getcallersp()
1480 systemstack(func() {
1481 traceback(pc, sp, 0, gp)
1482 })
1483 } else {
1484 goroutineheader(gp.m.curg)
1485 traceback(^uintptr(0), ^uintptr(0), 0, gp.m.curg)
1486 }
1487 print("\n")
1488 gp.m.traceback = 0
1489 unlock(&tracelock)
1490 }
1491
1492 func tracefree(p unsafe.Pointer, size uintptr) {
1493 lock(&tracelock)
1494 gp := getg()
1495 gp.m.traceback = 2
1496 print("tracefree(", p, ", ", hex(size), ")\n")
1497 goroutineheader(gp)
1498 pc := getcallerpc()
1499 sp := getcallersp()
1500 systemstack(func() {
1501 traceback(pc, sp, 0, gp)
1502 })
1503 print("\n")
1504 gp.m.traceback = 0
1505 unlock(&tracelock)
1506 }
1507
1508 func tracegc() {
1509 lock(&tracelock)
1510 gp := getg()
1511 gp.m.traceback = 2
1512 print("tracegc()\n")
1513
1514 tracebackothers(gp)
1515 print("end tracegc\n")
1516 print("\n")
1517 gp.m.traceback = 0
1518 unlock(&tracelock)
1519 }
1520
View as plain text