tracebuf.go

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Trace buffer management.
     6  
     7  package runtime
     8  
     9  import (
    10  	"runtime/internal/sys"
    11  	"unsafe"
    12  )
    13  
    14  // Maximum number of bytes required to encode uint64 in base-128.
    15  const traceBytesPerNumber = 10
    16  
    17  // traceWriter is the interface for writing all trace data.
    18  //
    19  // This type is passed around as a value, and all of its methods return
    20  // a new traceWriter. This allows for chaining together calls in a fluent-style
    21  // API. This is partly stylistic, and very slightly for performance, since
    22  // the compiler can destructure this value and pass it between calls as
    23  // just regular arguments. However, this style is not load-bearing, and
    24  // we can change it if it's deemed too error-prone.
    25  type traceWriter struct {
    26  	traceLocker
    27  	*traceBuf
    28  }
    29  
    30  // write returns an a traceWriter that writes into the current M's stream.
    31  func (tl traceLocker) writer() traceWriter {
    32  	return traceWriter{traceLocker: tl, traceBuf: tl.mp.trace.buf[tl.gen%2]}
    33  }
    34  
    35  // unsafeTraceWriter produces a traceWriter that doesn't lock the trace.
    36  //
    37  // It should only be used in contexts where either:
    38  // - Another traceLocker is held.
    39  // - trace.gen is prevented from advancing.
    40  //
    41  // buf may be nil.
    42  func unsafeTraceWriter(gen uintptr, buf *traceBuf) traceWriter {
    43  	return traceWriter{traceLocker: traceLocker{gen: gen}, traceBuf: buf}
    44  }
    45  
    46  // end writes the buffer back into the m.
    47  func (w traceWriter) end() {
    48  	if w.mp == nil {
    49  		// Tolerate a nil mp. It makes code that creates traceWriters directly
    50  		// less error-prone.
    51  		return
    52  	}
    53  	w.mp.trace.buf[w.gen%2] = w.traceBuf
    54  }
    55  
    56  // ensure makes sure that at least maxSize bytes are available to write.
    57  //
    58  // Returns whether the buffer was flushed.
    59  func (w traceWriter) ensure(maxSize int) (traceWriter, bool) {
    60  	refill := w.traceBuf == nil || !w.available(maxSize)
    61  	if refill {
    62  		w = w.refill()
    63  	}
    64  	return w, refill
    65  }
    66  
    67  // flush puts w.traceBuf on the queue of full buffers.
    68  func (w traceWriter) flush() traceWriter {
    69  	systemstack(func() {
    70  		lock(&trace.lock)
    71  		if w.traceBuf != nil {
    72  			traceBufFlush(w.traceBuf, w.gen)
    73  		}
    74  		unlock(&trace.lock)
    75  	})
    76  	w.traceBuf = nil
    77  	return w
    78  }
    79  
    80  // refill puts w.traceBuf on the queue of full buffers and refresh's w's buffer.
    81  func (w traceWriter) refill() traceWriter {
    82  	systemstack(func() {
    83  		lock(&trace.lock)
    84  		if w.traceBuf != nil {
    85  			traceBufFlush(w.traceBuf, w.gen)
    86  		}
    87  		if trace.empty != nil {
    88  			w.traceBuf = trace.empty
    89  			trace.empty = w.traceBuf.link
    90  			unlock(&trace.lock)
    91  		} else {
    92  			unlock(&trace.lock)
    93  			w.traceBuf = (*traceBuf)(sysAlloc(unsafe.Sizeof(traceBuf{}), &memstats.other_sys))
    94  			if w.traceBuf == nil {
    95  				throw("trace: out of memory")
    96  			}
    97  		}
    98  	})
    99  	// Initialize the buffer.
   100  	ts := traceClockNow()
   101  	if ts <= w.traceBuf.lastTime {
   102  		ts = w.traceBuf.lastTime + 1
   103  	}
   104  	w.traceBuf.lastTime = ts
   105  	w.traceBuf.link = nil
   106  	w.traceBuf.pos = 0
   107  
   108  	// Tolerate a nil mp.
   109  	mID := ^uint64(0)
   110  	if w.mp != nil {
   111  		mID = uint64(w.mp.procid)
   112  	}
   113  
   114  	// Write the buffer's header.
   115  	w.byte(byte(traceEvEventBatch))
   116  	w.varint(uint64(w.gen))
   117  	w.varint(uint64(mID))
   118  	w.varint(uint64(ts))
   119  	w.traceBuf.lenPos = w.varintReserve()
   120  	return w
   121  }
   122  
   123  // traceBufQueue is a FIFO of traceBufs.
   124  type traceBufQueue struct {
   125  	head, tail *traceBuf
   126  }
   127  
   128  // push queues buf into queue of buffers.
   129  func (q *traceBufQueue) push(buf *traceBuf) {
   130  	buf.link = nil
   131  	if q.head == nil {
   132  		q.head = buf
   133  	} else {
   134  		q.tail.link = buf
   135  	}
   136  	q.tail = buf
   137  }
   138  
   139  // pop dequeues from the queue of buffers.
   140  func (q *traceBufQueue) pop() *traceBuf {
   141  	buf := q.head
   142  	if buf == nil {
   143  		return nil
   144  	}
   145  	q.head = buf.link
   146  	if q.head == nil {
   147  		q.tail = nil
   148  	}
   149  	buf.link = nil
   150  	return buf
   151  }
   152  
   153  func (q *traceBufQueue) empty() bool {
   154  	return q.head == nil
   155  }
   156  
   157  // traceBufHeader is per-P tracing buffer.
   158  type traceBufHeader struct {
   159  	link     *traceBuf // in trace.empty/full
   160  	lastTime traceTime // when we wrote the last event
   161  	pos      int       // next write offset in arr
   162  	lenPos   int       // position of batch length value
   163  }
   164  
   165  // traceBuf is per-M tracing buffer.
   166  //
   167  // TODO(mknyszek): Rename traceBuf to traceBatch, since they map 1:1 with event batches.
   168  type traceBuf struct {
   169  	_ sys.NotInHeap
   170  	traceBufHeader
   171  	arr [64<<10 - unsafe.Sizeof(traceBufHeader{})]byte // underlying buffer for traceBufHeader.buf
   172  }
   173  
   174  // byte appends v to buf.
   175  func (buf *traceBuf) byte(v byte) {
   176  	buf.arr[buf.pos] = v
   177  	buf.pos++
   178  }
   179  
   180  // varint appends v to buf in little-endian-base-128 encoding.
   181  func (buf *traceBuf) varint(v uint64) {
   182  	pos := buf.pos
   183  	arr := buf.arr[pos : pos+traceBytesPerNumber]
   184  	for i := range arr {
   185  		if v < 0x80 {
   186  			pos += i + 1
   187  			arr[i] = byte(v)
   188  			break
   189  		}
   190  		arr[i] = 0x80 | byte(v)
   191  		v >>= 7
   192  	}
   193  	buf.pos = pos
   194  }
   195  
   196  // varintReserve reserves enough space in buf to hold any varint.
   197  //
   198  // Space reserved this way can be filled in with the varintAt method.
   199  func (buf *traceBuf) varintReserve() int {
   200  	p := buf.pos
   201  	buf.pos += traceBytesPerNumber
   202  	return p
   203  }
   204  
   205  // stringData appends s's data directly to buf.
   206  func (buf *traceBuf) stringData(s string) {
   207  	buf.pos += copy(buf.arr[buf.pos:], s)
   208  }
   209  
   210  func (buf *traceBuf) available(size int) bool {
   211  	return len(buf.arr)-buf.pos >= size
   212  }
   213  
   214  // varintAt writes varint v at byte position pos in buf. This always
   215  // consumes traceBytesPerNumber bytes. This is intended for when the caller
   216  // needs to reserve space for a varint but can't populate it until later.
   217  // Use varintReserve to reserve this space.
   218  func (buf *traceBuf) varintAt(pos int, v uint64) {
   219  	for i := 0; i < traceBytesPerNumber; i++ {
   220  		if i < traceBytesPerNumber-1 {
   221  			buf.arr[pos] = 0x80 | byte(v)
   222  		} else {
   223  			buf.arr[pos] = byte(v)
   224  		}
   225  		v >>= 7
   226  		pos++
   227  	}
   228  	if v != 0 {
   229  		throw("v could not fit in traceBytesPerNumber")
   230  	}
   231  }
   232  
   233  // traceBufFlush flushes a trace buffer.
   234  //
   235  // Must run on the system stack because trace.lock must be held.
   236  //
   237  //go:systemstack
   238  func traceBufFlush(buf *traceBuf, gen uintptr) {
   239  	assertLockHeld(&trace.lock)
   240  
   241  	// Write out the non-header length of the batch in the header.
   242  	//
   243  	// Note: the length of the header is not included to make it easier
   244  	// to calculate this value when deserializing and reserializing the
   245  	// trace. Varints can have additional padding of zero bits that is
   246  	// quite difficult to preserve, and if we include the header we
   247  	// force serializers to do more work. Nothing else actually needs
   248  	// padding.
   249  	buf.varintAt(buf.lenPos, uint64(buf.pos-(buf.lenPos+traceBytesPerNumber)))
   250  	trace.full[gen%2].push(buf)
   251  
   252  	// Notify the scheduler that there's work available and that the trace
   253  	// reader should be scheduled.
   254  	if !trace.workAvailable.Load() {
   255  		trace.workAvailable.Store(true)
   256  	}
   257  }
   258
View as plain text