Source file src/runtime/symtab.go

     1  // Copyright 2014 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"internal/goarch"
    10  	"internal/runtime/atomic"
    11  	"runtime/internal/sys"
    12  	"unsafe"
    13  )
    14  
    15  // Frames may be used to get function/file/line information for a
    16  // slice of PC values returned by [Callers].
    17  type Frames struct {
    18  	// callers is a slice of PCs that have not yet been expanded to frames.
    19  	callers []uintptr
    20  
    21  	// nextPC is a next PC to expand ahead of processing callers.
    22  	nextPC uintptr
    23  
    24  	// frames is a slice of Frames that have yet to be returned.
    25  	frames     []Frame
    26  	frameStore [2]Frame
    27  }
    28  
    29  // Frame is the information returned by [Frames] for each call frame.
    30  type Frame struct {
    31  	// PC is the program counter for the location in this frame.
    32  	// For a frame that calls another frame, this will be the
    33  	// program counter of a call instruction. Because of inlining,
    34  	// multiple frames may have the same PC value, but different
    35  	// symbolic information.
    36  	PC uintptr
    37  
    38  	// Func is the Func value of this call frame. This may be nil
    39  	// for non-Go code or fully inlined functions.
    40  	Func *Func
    41  
    42  	// Function is the package path-qualified function name of
    43  	// this call frame. If non-empty, this string uniquely
    44  	// identifies a single function in the program.
    45  	// This may be the empty string if not known.
    46  	// If Func is not nil then Function == Func.Name().
    47  	Function string
    48  
    49  	// File and Line are the file name and line number of the
    50  	// location in this frame. For non-leaf frames, this will be
    51  	// the location of a call. These may be the empty string and
    52  	// zero, respectively, if not known.
    53  	File string
    54  	Line int
    55  
    56  	// startLine is the line number of the beginning of the function in
    57  	// this frame. Specifically, it is the line number of the func keyword
    58  	// for Go functions. Note that //line directives can change the
    59  	// filename and/or line number arbitrarily within a function, meaning
    60  	// that the Line - startLine offset is not always meaningful.
    61  	//
    62  	// This may be zero if not known.
    63  	startLine int
    64  
    65  	// Entry point program counter for the function; may be zero
    66  	// if not known. If Func is not nil then Entry ==
    67  	// Func.Entry().
    68  	Entry uintptr
    69  
    70  	// The runtime's internal view of the function. This field
    71  	// is set (funcInfo.valid() returns true) only for Go functions,
    72  	// not for C functions.
    73  	funcInfo funcInfo
    74  }
    75  
    76  // CallersFrames takes a slice of PC values returned by [Callers] and
    77  // prepares to return function/file/line information.
    78  // Do not change the slice until you are done with the [Frames].
    79  func CallersFrames(callers []uintptr) *Frames {
    80  	f := &Frames{callers: callers}
    81  	f.frames = f.frameStore[:0]
    82  	return f
    83  }
    84  
    85  // Next returns a [Frame] representing the next call frame in the slice
    86  // of PC values. If it has already returned all call frames, Next
    87  // returns a zero [Frame].
    88  //
    89  // The more result indicates whether the next call to Next will return
    90  // a valid [Frame]. It does not necessarily indicate whether this call
    91  // returned one.
    92  //
    93  // See the [Frames] example for idiomatic usage.
    94  func (ci *Frames) Next() (frame Frame, more bool) {
    95  	for len(ci.frames) < 2 {
    96  		// Find the next frame.
    97  		// We need to look for 2 frames so we know what
    98  		// to return for the "more" result.
    99  		if len(ci.callers) == 0 {
   100  			break
   101  		}
   102  		var pc uintptr
   103  		if ci.nextPC != 0 {
   104  			pc, ci.nextPC = ci.nextPC, 0
   105  		} else {
   106  			pc, ci.callers = ci.callers[0], ci.callers[1:]
   107  		}
   108  		funcInfo := findfunc(pc)
   109  		if !funcInfo.valid() {
   110  			if cgoSymbolizer != nil {
   111  				// Pre-expand cgo frames. We could do this
   112  				// incrementally, too, but there's no way to
   113  				// avoid allocation in this case anyway.
   114  				ci.frames = append(ci.frames, expandCgoFrames(pc)...)
   115  			}
   116  			continue
   117  		}
   118  		f := funcInfo._Func()
   119  		entry := f.Entry()
   120  		if pc > entry {
   121  			// We store the pc of the start of the instruction following
   122  			// the instruction in question (the call or the inline mark).
   123  			// This is done for historical reasons, and to make FuncForPC
   124  			// work correctly for entries in the result of runtime.Callers.
   125  			pc--
   126  		}
   127  		// It's important that interpret pc non-strictly as cgoTraceback may
   128  		// have added bogus PCs with a valid funcInfo but invalid PCDATA.
   129  		u, uf := newInlineUnwinder(funcInfo, pc)
   130  		sf := u.srcFunc(uf)
   131  		if u.isInlined(uf) {
   132  			// Note: entry is not modified. It always refers to a real frame, not an inlined one.
   133  			// File/line from funcline1 below are already correct.
   134  			f = nil
   135  
   136  			// When CallersFrame is invoked using the PC list returned by Callers,
   137  			// the PC list includes virtual PCs corresponding to each outer frame
   138  			// around an innermost real inlined PC.
   139  			// We also want to support code passing in a PC list extracted from a
   140  			// stack trace, and there only the real PCs are printed, not the virtual ones.
   141  			// So check to see if the implied virtual PC for this PC (obtained from the
   142  			// unwinder itself) is the next PC in ci.callers. If not, insert it.
   143  			// The +1 here correspond to the pc-- above: the output of Callers
   144  			// and therefore the input to CallersFrames is return PCs from the stack;
   145  			// The pc-- backs up into the CALL instruction (not the first byte of the CALL
   146  			// instruction, but good enough to find it nonetheless).
   147  			// There are no cycles in implied virtual PCs (some number of frames were
   148  			// inlined, but that number is finite), so this unpacking cannot cause an infinite loop.
   149  			for unext := u.next(uf); unext.valid() && len(ci.callers) > 0 && ci.callers[0] != unext.pc+1; unext = u.next(unext) {
   150  				snext := u.srcFunc(unext)
   151  				if snext.funcID == abi.FuncIDWrapper && elideWrapperCalling(sf.funcID) {
   152  					// Skip, because tracebackPCs (inside runtime.Callers) would too.
   153  					continue
   154  				}
   155  				ci.nextPC = unext.pc + 1
   156  				break
   157  			}
   158  		}
   159  		ci.frames = append(ci.frames, Frame{
   160  			PC:        pc,
   161  			Func:      f,
   162  			Function:  funcNameForPrint(sf.name()),
   163  			Entry:     entry,
   164  			startLine: int(sf.startLine),
   165  			funcInfo:  funcInfo,
   166  			// Note: File,Line set below
   167  		})
   168  	}
   169  
   170  	// Pop one frame from the frame list. Keep the rest.
   171  	// Avoid allocation in the common case, which is 1 or 2 frames.
   172  	switch len(ci.frames) {
   173  	case 0: // In the rare case when there are no frames at all, we return Frame{}.
   174  		return
   175  	case 1:
   176  		frame = ci.frames[0]
   177  		ci.frames = ci.frameStore[:0]
   178  	case 2:
   179  		frame = ci.frames[0]
   180  		ci.frameStore[0] = ci.frames[1]
   181  		ci.frames = ci.frameStore[:1]
   182  	default:
   183  		frame = ci.frames[0]
   184  		ci.frames = ci.frames[1:]
   185  	}
   186  	more = len(ci.frames) > 0
   187  	if frame.funcInfo.valid() {
   188  		// Compute file/line just before we need to return it,
   189  		// as it can be expensive. This avoids computing file/line
   190  		// for the Frame we find but don't return. See issue 32093.
   191  		file, line := funcline1(frame.funcInfo, frame.PC, false)
   192  		frame.File, frame.Line = file, int(line)
   193  	}
   194  	return
   195  }
   196  
   197  // runtime_FrameStartLine returns the start line of the function in a Frame.
   198  //
   199  //go:linkname runtime_FrameStartLine runtime/pprof.runtime_FrameStartLine
   200  func runtime_FrameStartLine(f *Frame) int {
   201  	return f.startLine
   202  }
   203  
   204  // runtime_FrameSymbolName returns the full symbol name of the function in a Frame.
   205  // For generic functions this differs from f.Function in that this doesn't replace
   206  // the shape name to "...".
   207  //
   208  //go:linkname runtime_FrameSymbolName runtime/pprof.runtime_FrameSymbolName
   209  func runtime_FrameSymbolName(f *Frame) string {
   210  	if !f.funcInfo.valid() {
   211  		return f.Function
   212  	}
   213  	u, uf := newInlineUnwinder(f.funcInfo, f.PC)
   214  	sf := u.srcFunc(uf)
   215  	return sf.name()
   216  }
   217  
   218  // runtime_expandFinalInlineFrame expands the final pc in stk to include all
   219  // "callers" if pc is inline.
   220  //
   221  //go:linkname runtime_expandFinalInlineFrame runtime/pprof.runtime_expandFinalInlineFrame
   222  func runtime_expandFinalInlineFrame(stk []uintptr) []uintptr {
   223  	// TODO: It would be more efficient to report only physical PCs to pprof and
   224  	// just expand the whole stack.
   225  	if len(stk) == 0 {
   226  		return stk
   227  	}
   228  	pc := stk[len(stk)-1]
   229  	tracepc := pc - 1
   230  
   231  	f := findfunc(tracepc)
   232  	if !f.valid() {
   233  		// Not a Go function.
   234  		return stk
   235  	}
   236  
   237  	u, uf := newInlineUnwinder(f, tracepc)
   238  	if !u.isInlined(uf) {
   239  		// Nothing inline at tracepc.
   240  		return stk
   241  	}
   242  
   243  	// Treat the previous func as normal. We haven't actually checked, but
   244  	// since this pc was included in the stack, we know it shouldn't be
   245  	// elided.
   246  	calleeID := abi.FuncIDNormal
   247  
   248  	// Remove pc from stk; we'll re-add it below.
   249  	stk = stk[:len(stk)-1]
   250  
   251  	for ; uf.valid(); uf = u.next(uf) {
   252  		funcID := u.srcFunc(uf).funcID
   253  		if funcID == abi.FuncIDWrapper && elideWrapperCalling(calleeID) {
   254  			// ignore wrappers
   255  		} else {
   256  			stk = append(stk, uf.pc+1)
   257  		}
   258  		calleeID = funcID
   259  	}
   260  
   261  	return stk
   262  }
   263  
   264  // expandCgoFrames expands frame information for pc, known to be
   265  // a non-Go function, using the cgoSymbolizer hook. expandCgoFrames
   266  // returns nil if pc could not be expanded.
   267  func expandCgoFrames(pc uintptr) []Frame {
   268  	arg := cgoSymbolizerArg{pc: pc}
   269  	callCgoSymbolizer(&arg)
   270  
   271  	if arg.file == nil && arg.funcName == nil {
   272  		// No useful information from symbolizer.
   273  		return nil
   274  	}
   275  
   276  	var frames []Frame
   277  	for {
   278  		frames = append(frames, Frame{
   279  			PC:       pc,
   280  			Func:     nil,
   281  			Function: gostring(arg.funcName),
   282  			File:     gostring(arg.file),
   283  			Line:     int(arg.lineno),
   284  			Entry:    arg.entry,
   285  			// funcInfo is zero, which implies !funcInfo.valid().
   286  			// That ensures that we use the File/Line info given here.
   287  		})
   288  		if arg.more == 0 {
   289  			break
   290  		}
   291  		callCgoSymbolizer(&arg)
   292  	}
   293  
   294  	// No more frames for this PC. Tell the symbolizer we are done.
   295  	// We don't try to maintain a single cgoSymbolizerArg for the
   296  	// whole use of Frames, because there would be no good way to tell
   297  	// the symbolizer when we are done.
   298  	arg.pc = 0
   299  	callCgoSymbolizer(&arg)
   300  
   301  	return frames
   302  }
   303  
   304  // NOTE: Func does not expose the actual unexported fields, because we return *Func
   305  // values to users, and we want to keep them from being able to overwrite the data
   306  // with (say) *f = Func{}.
   307  // All code operating on a *Func must call raw() to get the *_func
   308  // or funcInfo() to get the funcInfo instead.
   309  
   310  // A Func represents a Go function in the running binary.
   311  type Func struct {
   312  	opaque struct{} // unexported field to disallow conversions
   313  }
   314  
   315  func (f *Func) raw() *_func {
   316  	return (*_func)(unsafe.Pointer(f))
   317  }
   318  
   319  func (f *Func) funcInfo() funcInfo {
   320  	return f.raw().funcInfo()
   321  }
   322  
   323  func (f *_func) funcInfo() funcInfo {
   324  	// Find the module containing fn. fn is located in the pclntable.
   325  	// The unsafe.Pointer to uintptr conversions and arithmetic
   326  	// are safe because we are working with module addresses.
   327  	ptr := uintptr(unsafe.Pointer(f))
   328  	var mod *moduledata
   329  	for datap := &firstmoduledata; datap != nil; datap = datap.next {
   330  		if len(datap.pclntable) == 0 {
   331  			continue
   332  		}
   333  		base := uintptr(unsafe.Pointer(&datap.pclntable[0]))
   334  		if base <= ptr && ptr < base+uintptr(len(datap.pclntable)) {
   335  			mod = datap
   336  			break
   337  		}
   338  	}
   339  	return funcInfo{f, mod}
   340  }
   341  
   342  // pcHeader holds data used by the pclntab lookups.
   343  type pcHeader struct {
   344  	magic          uint32  // 0xFFFFFFF1
   345  	pad1, pad2     uint8   // 0,0
   346  	minLC          uint8   // min instruction size
   347  	ptrSize        uint8   // size of a ptr in bytes
   348  	nfunc          int     // number of functions in the module
   349  	nfiles         uint    // number of entries in the file tab
   350  	textStart      uintptr // base for function entry PC offsets in this module, equal to moduledata.text
   351  	funcnameOffset uintptr // offset to the funcnametab variable from pcHeader
   352  	cuOffset       uintptr // offset to the cutab variable from pcHeader
   353  	filetabOffset  uintptr // offset to the filetab variable from pcHeader
   354  	pctabOffset    uintptr // offset to the pctab variable from pcHeader
   355  	pclnOffset     uintptr // offset to the pclntab variable from pcHeader
   356  }
   357  
   358  // moduledata records information about the layout of the executable
   359  // image. It is written by the linker. Any changes here must be
   360  // matched changes to the code in cmd/link/internal/ld/symtab.go:symtab.
   361  // moduledata is stored in statically allocated non-pointer memory;
   362  // none of the pointers here are visible to the garbage collector.
   363  type moduledata struct {
   364  	sys.NotInHeap // Only in static data
   365  
   366  	pcHeader     *pcHeader
   367  	funcnametab  []byte
   368  	cutab        []uint32
   369  	filetab      []byte
   370  	pctab        []byte
   371  	pclntable    []byte
   372  	ftab         []functab
   373  	findfunctab  uintptr
   374  	minpc, maxpc uintptr
   375  
   376  	text, etext           uintptr
   377  	noptrdata, enoptrdata uintptr
   378  	data, edata           uintptr
   379  	bss, ebss             uintptr
   380  	noptrbss, enoptrbss   uintptr
   381  	covctrs, ecovctrs     uintptr
   382  	end, gcdata, gcbss    uintptr
   383  	types, etypes         uintptr
   384  	rodata                uintptr
   385  	gofunc                uintptr // go.func.*
   386  
   387  	textsectmap []textsect
   388  	typelinks   []int32 // offsets from types
   389  	itablinks   []*itab
   390  
   391  	ptab []ptabEntry
   392  
   393  	pluginpath string
   394  	pkghashes  []modulehash
   395  
   396  	// This slice records the initializing tasks that need to be
   397  	// done to start up the program. It is built by the linker.
   398  	inittasks []*initTask
   399  
   400  	modulename   string
   401  	modulehashes []modulehash
   402  
   403  	hasmain uint8 // 1 if module contains the main function, 0 otherwise
   404  
   405  	gcdatamask, gcbssmask bitvector
   406  
   407  	typemap map[typeOff]*_type // offset to *_rtype in previous module
   408  
   409  	bad bool // module failed to load and should be ignored
   410  
   411  	next *moduledata
   412  }
   413  
   414  // A modulehash is used to compare the ABI of a new module or a
   415  // package in a new module with the loaded program.
   416  //
   417  // For each shared library a module links against, the linker creates an entry in the
   418  // moduledata.modulehashes slice containing the name of the module, the abi hash seen
   419  // at link time and a pointer to the runtime abi hash. These are checked in
   420  // moduledataverify1 below.
   421  //
   422  // For each loaded plugin, the pkghashes slice has a modulehash of the
   423  // newly loaded package that can be used to check the plugin's version of
   424  // a package against any previously loaded version of the package.
   425  // This is done in plugin.lastmoduleinit.
   426  type modulehash struct {
   427  	modulename   string
   428  	linktimehash string
   429  	runtimehash  *string
   430  }
   431  
   432  // pinnedTypemaps are the map[typeOff]*_type from the moduledata objects.
   433  //
   434  // These typemap objects are allocated at run time on the heap, but the
   435  // only direct reference to them is in the moduledata, created by the
   436  // linker and marked SNOPTRDATA so it is ignored by the GC.
   437  //
   438  // To make sure the map isn't collected, we keep a second reference here.
   439  var pinnedTypemaps []map[typeOff]*_type
   440  
   441  var firstmoduledata moduledata  // linker symbol
   442  var lastmoduledatap *moduledata // linker symbol
   443  var modulesSlice *[]*moduledata // see activeModules
   444  
   445  // activeModules returns a slice of active modules.
   446  //
   447  // A module is active once its gcdatamask and gcbssmask have been
   448  // assembled and it is usable by the GC.
   449  //
   450  // This is nosplit/nowritebarrier because it is called by the
   451  // cgo pointer checking code.
   452  //
   453  //go:nosplit
   454  //go:nowritebarrier
   455  func activeModules() []*moduledata {
   456  	p := (*[]*moduledata)(atomic.Loadp(unsafe.Pointer(&modulesSlice)))
   457  	if p == nil {
   458  		return nil
   459  	}
   460  	return *p
   461  }
   462  
   463  // modulesinit creates the active modules slice out of all loaded modules.
   464  //
   465  // When a module is first loaded by the dynamic linker, an .init_array
   466  // function (written by cmd/link) is invoked to call addmoduledata,
   467  // appending to the module to the linked list that starts with
   468  // firstmoduledata.
   469  //
   470  // There are two times this can happen in the lifecycle of a Go
   471  // program. First, if compiled with -linkshared, a number of modules
   472  // built with -buildmode=shared can be loaded at program initialization.
   473  // Second, a Go program can load a module while running that was built
   474  // with -buildmode=plugin.
   475  //
   476  // After loading, this function is called which initializes the
   477  // moduledata so it is usable by the GC and creates a new activeModules
   478  // list.
   479  //
   480  // Only one goroutine may call modulesinit at a time.
   481  func modulesinit() {
   482  	modules := new([]*moduledata)
   483  	for md := &firstmoduledata; md != nil; md = md.next {
   484  		if md.bad {
   485  			continue
   486  		}
   487  		*modules = append(*modules, md)
   488  		if md.gcdatamask == (bitvector{}) {
   489  			scanDataSize := md.edata - md.data
   490  			md.gcdatamask = progToPointerMask((*byte)(unsafe.Pointer(md.gcdata)), scanDataSize)
   491  			scanBSSSize := md.ebss - md.bss
   492  			md.gcbssmask = progToPointerMask((*byte)(unsafe.Pointer(md.gcbss)), scanBSSSize)
   493  			gcController.addGlobals(int64(scanDataSize + scanBSSSize))
   494  		}
   495  	}
   496  
   497  	// Modules appear in the moduledata linked list in the order they are
   498  	// loaded by the dynamic loader, with one exception: the
   499  	// firstmoduledata itself the module that contains the runtime. This
   500  	// is not always the first module (when using -buildmode=shared, it
   501  	// is typically libstd.so, the second module). The order matters for
   502  	// typelinksinit, so we swap the first module with whatever module
   503  	// contains the main function.
   504  	//
   505  	// See Issue #18729.
   506  	for i, md := range *modules {
   507  		if md.hasmain != 0 {
   508  			(*modules)[0] = md
   509  			(*modules)[i] = &firstmoduledata
   510  			break
   511  		}
   512  	}
   513  
   514  	atomicstorep(unsafe.Pointer(&modulesSlice), unsafe.Pointer(modules))
   515  }
   516  
   517  type functab struct {
   518  	entryoff uint32 // relative to runtime.text
   519  	funcoff  uint32
   520  }
   521  
   522  // Mapping information for secondary text sections
   523  
   524  type textsect struct {
   525  	vaddr    uintptr // prelinked section vaddr
   526  	end      uintptr // vaddr + section length
   527  	baseaddr uintptr // relocated section address
   528  }
   529  
   530  // findfuncbucket is an array of these structures.
   531  // Each bucket represents 4096 bytes of the text segment.
   532  // Each subbucket represents 256 bytes of the text segment.
   533  // To find a function given a pc, locate the bucket and subbucket for
   534  // that pc. Add together the idx and subbucket value to obtain a
   535  // function index. Then scan the functab array starting at that
   536  // index to find the target function.
   537  // This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead.
   538  type findfuncbucket struct {
   539  	idx        uint32
   540  	subbuckets [16]byte
   541  }
   542  
   543  func moduledataverify() {
   544  	for datap := &firstmoduledata; datap != nil; datap = datap.next {
   545  		moduledataverify1(datap)
   546  	}
   547  }
   548  
   549  const debugPcln = false
   550  
   551  func moduledataverify1(datap *moduledata) {
   552  	// Check that the pclntab's format is valid.
   553  	hdr := datap.pcHeader
   554  	if hdr.magic != 0xfffffff1 || hdr.pad1 != 0 || hdr.pad2 != 0 ||
   555  		hdr.minLC != sys.PCQuantum || hdr.ptrSize != goarch.PtrSize || hdr.textStart != datap.text {
   556  		println("runtime: pcHeader: magic=", hex(hdr.magic), "pad1=", hdr.pad1, "pad2=", hdr.pad2,
   557  			"minLC=", hdr.minLC, "ptrSize=", hdr.ptrSize, "pcHeader.textStart=", hex(hdr.textStart),
   558  			"text=", hex(datap.text), "pluginpath=", datap.pluginpath)
   559  		throw("invalid function symbol table")
   560  	}
   561  
   562  	// ftab is lookup table for function by program counter.
   563  	nftab := len(datap.ftab) - 1
   564  	for i := 0; i < nftab; i++ {
   565  		// NOTE: ftab[nftab].entry is legal; it is the address beyond the final function.
   566  		if datap.ftab[i].entryoff > datap.ftab[i+1].entryoff {
   567  			f1 := funcInfo{(*_func)(unsafe.Pointer(&datap.pclntable[datap.ftab[i].funcoff])), datap}
   568  			f2 := funcInfo{(*_func)(unsafe.Pointer(&datap.pclntable[datap.ftab[i+1].funcoff])), datap}
   569  			f2name := "end"
   570  			if i+1 < nftab {
   571  				f2name = funcname(f2)
   572  			}
   573  			println("function symbol table not sorted by PC offset:", hex(datap.ftab[i].entryoff), funcname(f1), ">", hex(datap.ftab[i+1].entryoff), f2name, ", plugin:", datap.pluginpath)
   574  			for j := 0; j <= i; j++ {
   575  				println("\t", hex(datap.ftab[j].entryoff), funcname(funcInfo{(*_func)(unsafe.Pointer(&datap.pclntable[datap.ftab[j].funcoff])), datap}))
   576  			}
   577  			if GOOS == "aix" && isarchive {
   578  				println("-Wl,-bnoobjreorder is mandatory on aix/ppc64 with c-archive")
   579  			}
   580  			throw("invalid runtime symbol table")
   581  		}
   582  	}
   583  
   584  	min := datap.textAddr(datap.ftab[0].entryoff)
   585  	max := datap.textAddr(datap.ftab[nftab].entryoff)
   586  	if datap.minpc != min || datap.maxpc != max {
   587  		println("minpc=", hex(datap.minpc), "min=", hex(min), "maxpc=", hex(datap.maxpc), "max=", hex(max))
   588  		throw("minpc or maxpc invalid")
   589  	}
   590  
   591  	for _, modulehash := range datap.modulehashes {
   592  		if modulehash.linktimehash != *modulehash.runtimehash {
   593  			println("abi mismatch detected between", datap.modulename, "and", modulehash.modulename)
   594  			throw("abi mismatch")
   595  		}
   596  	}
   597  }
   598  
   599  // textAddr returns md.text + off, with special handling for multiple text sections.
   600  // off is a (virtual) offset computed at internal linking time,
   601  // before the external linker adjusts the sections' base addresses.
   602  //
   603  // The text, or instruction stream is generated as one large buffer.
   604  // The off (offset) for a function is its offset within this buffer.
   605  // If the total text size gets too large, there can be issues on platforms like ppc64
   606  // if the target of calls are too far for the call instruction.
   607  // To resolve the large text issue, the text is split into multiple text sections
   608  // to allow the linker to generate long calls when necessary.
   609  // When this happens, the vaddr for each text section is set to its offset within the text.
   610  // Each function's offset is compared against the section vaddrs and ends to determine the containing section.
   611  // Then the section relative offset is added to the section's
   612  // relocated baseaddr to compute the function address.
   613  //
   614  // It is nosplit because it is part of the findfunc implementation.
   615  //
   616  //go:nosplit
   617  func (md *moduledata) textAddr(off32 uint32) uintptr {
   618  	off := uintptr(off32)
   619  	res := md.text + off
   620  	if len(md.textsectmap) > 1 {
   621  		for i, sect := range md.textsectmap {
   622  			// For the last section, include the end address (etext), as it is included in the functab.
   623  			if off >= sect.vaddr && off < sect.end || (i == len(md.textsectmap)-1 && off == sect.end) {
   624  				res = sect.baseaddr + off - sect.vaddr
   625  				break
   626  			}
   627  		}
   628  		if res > md.etext && GOARCH != "wasm" { // on wasm, functions do not live in the same address space as the linear memory
   629  			println("runtime: textAddr", hex(res), "out of range", hex(md.text), "-", hex(md.etext))
   630  			throw("runtime: text offset out of range")
   631  		}
   632  	}
   633  	return res
   634  }
   635  
   636  // textOff is the opposite of textAddr. It converts a PC to a (virtual) offset
   637  // to md.text, and returns if the PC is in any Go text section.
   638  //
   639  // It is nosplit because it is part of the findfunc implementation.
   640  //
   641  //go:nosplit
   642  func (md *moduledata) textOff(pc uintptr) (uint32, bool) {
   643  	res := uint32(pc - md.text)
   644  	if len(md.textsectmap) > 1 {
   645  		for i, sect := range md.textsectmap {
   646  			if sect.baseaddr > pc {
   647  				// pc is not in any section.
   648  				return 0, false
   649  			}
   650  			end := sect.baseaddr + (sect.end - sect.vaddr)
   651  			// For the last section, include the end address (etext), as it is included in the functab.
   652  			if i == len(md.textsectmap)-1 {
   653  				end++
   654  			}
   655  			if pc < end {
   656  				res = uint32(pc - sect.baseaddr + sect.vaddr)
   657  				break
   658  			}
   659  		}
   660  	}
   661  	return res, true
   662  }
   663  
   664  // funcName returns the string at nameOff in the function name table.
   665  func (md *moduledata) funcName(nameOff int32) string {
   666  	if nameOff == 0 {
   667  		return ""
   668  	}
   669  	return gostringnocopy(&md.funcnametab[nameOff])
   670  }
   671  
   672  // FuncForPC returns a *[Func] describing the function that contains the
   673  // given program counter address, or else nil.
   674  //
   675  // If pc represents multiple functions because of inlining, it returns
   676  // the *Func describing the innermost function, but with an entry of
   677  // the outermost function.
   678  func FuncForPC(pc uintptr) *Func {
   679  	f := findfunc(pc)
   680  	if !f.valid() {
   681  		return nil
   682  	}
   683  	// This must interpret PC non-strictly so bad PCs (those between functions) don't crash the runtime.
   684  	// We just report the preceding function in that situation. See issue 29735.
   685  	// TODO: Perhaps we should report no function at all in that case.
   686  	// The runtime currently doesn't have function end info, alas.
   687  	u, uf := newInlineUnwinder(f, pc)
   688  	if !u.isInlined(uf) {
   689  		return f._Func()
   690  	}
   691  	sf := u.srcFunc(uf)
   692  	file, line := u.fileLine(uf)
   693  	fi := &funcinl{
   694  		ones:      ^uint32(0),
   695  		entry:     f.entry(), // entry of the real (the outermost) function.
   696  		name:      sf.name(),
   697  		file:      file,
   698  		line:      int32(line),
   699  		startLine: sf.startLine,
   700  	}
   701  	return (*Func)(unsafe.Pointer(fi))
   702  }
   703  
   704  // Name returns the name of the function.
   705  func (f *Func) Name() string {
   706  	if f == nil {
   707  		return ""
   708  	}
   709  	fn := f.raw()
   710  	if fn.isInlined() { // inlined version
   711  		fi := (*funcinl)(unsafe.Pointer(fn))
   712  		return funcNameForPrint(fi.name)
   713  	}
   714  	return funcNameForPrint(funcname(f.funcInfo()))
   715  }
   716  
   717  // Entry returns the entry address of the function.
   718  func (f *Func) Entry() uintptr {
   719  	fn := f.raw()
   720  	if fn.isInlined() { // inlined version
   721  		fi := (*funcinl)(unsafe.Pointer(fn))
   722  		return fi.entry
   723  	}
   724  	return fn.funcInfo().entry()
   725  }
   726  
   727  // FileLine returns the file name and line number of the
   728  // source code corresponding to the program counter pc.
   729  // The result will not be accurate if pc is not a program
   730  // counter within f.
   731  func (f *Func) FileLine(pc uintptr) (file string, line int) {
   732  	fn := f.raw()
   733  	if fn.isInlined() { // inlined version
   734  		fi := (*funcinl)(unsafe.Pointer(fn))
   735  		return fi.file, int(fi.line)
   736  	}
   737  	// Pass strict=false here, because anyone can call this function,
   738  	// and they might just be wrong about targetpc belonging to f.
   739  	file, line32 := funcline1(f.funcInfo(), pc, false)
   740  	return file, int(line32)
   741  }
   742  
   743  // startLine returns the starting line number of the function. i.e., the line
   744  // number of the func keyword.
   745  func (f *Func) startLine() int32 {
   746  	fn := f.raw()
   747  	if fn.isInlined() { // inlined version
   748  		fi := (*funcinl)(unsafe.Pointer(fn))
   749  		return fi.startLine
   750  	}
   751  	return fn.funcInfo().startLine
   752  }
   753  
   754  // findmoduledatap looks up the moduledata for a PC.
   755  //
   756  // It is nosplit because it's part of the isgoexception
   757  // implementation.
   758  //
   759  //go:nosplit
   760  func findmoduledatap(pc uintptr) *moduledata {
   761  	for datap := &firstmoduledata; datap != nil; datap = datap.next {
   762  		if datap.minpc <= pc && pc < datap.maxpc {
   763  			return datap
   764  		}
   765  	}
   766  	return nil
   767  }
   768  
   769  type funcInfo struct {
   770  	*_func
   771  	datap *moduledata
   772  }
   773  
   774  func (f funcInfo) valid() bool {
   775  	return f._func != nil
   776  }
   777  
   778  func (f funcInfo) _Func() *Func {
   779  	return (*Func)(unsafe.Pointer(f._func))
   780  }
   781  
   782  // isInlined reports whether f should be re-interpreted as a *funcinl.
   783  func (f *_func) isInlined() bool {
   784  	return f.entryOff == ^uint32(0) // see comment for funcinl.ones
   785  }
   786  
   787  // entry returns the entry PC for f.
   788  func (f funcInfo) entry() uintptr {
   789  	return f.datap.textAddr(f.entryOff)
   790  }
   791  
   792  // findfunc looks up function metadata for a PC.
   793  //
   794  // It is nosplit because it's part of the isgoexception
   795  // implementation.
   796  //
   797  //go:nosplit
   798  func findfunc(pc uintptr) funcInfo {
   799  	datap := findmoduledatap(pc)
   800  	if datap == nil {
   801  		return funcInfo{}
   802  	}
   803  	const nsub = uintptr(len(findfuncbucket{}.subbuckets))
   804  
   805  	pcOff, ok := datap.textOff(pc)
   806  	if !ok {
   807  		return funcInfo{}
   808  	}
   809  
   810  	x := uintptr(pcOff) + datap.text - datap.minpc // TODO: are datap.text and datap.minpc always equal?
   811  	b := x / abi.FuncTabBucketSize
   812  	i := x % abi.FuncTabBucketSize / (abi.FuncTabBucketSize / nsub)
   813  
   814  	ffb := (*findfuncbucket)(add(unsafe.Pointer(datap.findfunctab), b*unsafe.Sizeof(findfuncbucket{})))
   815  	idx := ffb.idx + uint32(ffb.subbuckets[i])
   816  
   817  	// Find the ftab entry.
   818  	for datap.ftab[idx+1].entryoff <= pcOff {
   819  		idx++
   820  	}
   821  
   822  	funcoff := datap.ftab[idx].funcoff
   823  	return funcInfo{(*_func)(unsafe.Pointer(&datap.pclntable[funcoff])), datap}
   824  }
   825  
   826  // A srcFunc represents a logical function in the source code. This may
   827  // correspond to an actual symbol in the binary text, or it may correspond to a
   828  // source function that has been inlined.
   829  type srcFunc struct {
   830  	datap     *moduledata
   831  	nameOff   int32
   832  	startLine int32
   833  	funcID    abi.FuncID
   834  }
   835  
   836  func (f funcInfo) srcFunc() srcFunc {
   837  	if !f.valid() {
   838  		return srcFunc{}
   839  	}
   840  	return srcFunc{f.datap, f.nameOff, f.startLine, f.funcID}
   841  }
   842  
   843  func (s srcFunc) name() string {
   844  	if s.datap == nil {
   845  		return ""
   846  	}
   847  	return s.datap.funcName(s.nameOff)
   848  }
   849  
   850  type pcvalueCache struct {
   851  	entries [2][8]pcvalueCacheEnt
   852  	inUse   int
   853  }
   854  
   855  type pcvalueCacheEnt struct {
   856  	// targetpc and off together are the key of this cache entry.
   857  	targetpc uintptr
   858  	off      uint32
   859  
   860  	val   int32   // The value of this entry.
   861  	valPC uintptr // The PC at which val starts
   862  }
   863  
   864  // pcvalueCacheKey returns the outermost index in a pcvalueCache to use for targetpc.
   865  // It must be very cheap to calculate.
   866  // For now, align to goarch.PtrSize and reduce mod the number of entries.
   867  // In practice, this appears to be fairly randomly and evenly distributed.
   868  func pcvalueCacheKey(targetpc uintptr) uintptr {
   869  	return (targetpc / goarch.PtrSize) % uintptr(len(pcvalueCache{}.entries))
   870  }
   871  
   872  // Returns the PCData value, and the PC where this value starts.
   873  func pcvalue(f funcInfo, off uint32, targetpc uintptr, strict bool) (int32, uintptr) {
   874  	// If true, when we get a cache hit, still look up the data and make sure it
   875  	// matches the cached contents.
   876  	const debugCheckCache = false
   877  
   878  	if off == 0 {
   879  		return -1, 0
   880  	}
   881  
   882  	// Check the cache. This speeds up walks of deep stacks, which
   883  	// tend to have the same recursive functions over and over,
   884  	// or repetitive stacks between goroutines.
   885  	var checkVal int32
   886  	var checkPC uintptr
   887  	ck := pcvalueCacheKey(targetpc)
   888  	{
   889  		mp := acquirem()
   890  		cache := &mp.pcvalueCache
   891  		// The cache can be used by the signal handler on this M. Avoid
   892  		// re-entrant use of the cache. The signal handler can also write inUse,
   893  		// but will always restore its value, so we can use a regular increment
   894  		// even if we get signaled in the middle of it.
   895  		cache.inUse++
   896  		if cache.inUse == 1 {
   897  			for i := range cache.entries[ck] {
   898  				// We check off first because we're more
   899  				// likely to have multiple entries with
   900  				// different offsets for the same targetpc
   901  				// than the other way around, so we'll usually
   902  				// fail in the first clause.
   903  				ent := &cache.entries[ck][i]
   904  				if ent.off == off && ent.targetpc == targetpc {
   905  					val, pc := ent.val, ent.valPC
   906  					if debugCheckCache {
   907  						checkVal, checkPC = ent.val, ent.valPC
   908  						break
   909  					} else {
   910  						cache.inUse--
   911  						releasem(mp)
   912  						return val, pc
   913  					}
   914  				}
   915  			}
   916  		} else if debugCheckCache && (cache.inUse < 1 || cache.inUse > 2) {
   917  			// Catch accounting errors or deeply reentrant use. In principle
   918  			// "inUse" should never exceed 2.
   919  			throw("cache.inUse out of range")
   920  		}
   921  		cache.inUse--
   922  		releasem(mp)
   923  	}
   924  
   925  	if !f.valid() {
   926  		if strict && panicking.Load() == 0 {
   927  			println("runtime: no module data for", hex(f.entry()))
   928  			throw("no module data")
   929  		}
   930  		return -1, 0
   931  	}
   932  	datap := f.datap
   933  	p := datap.pctab[off:]
   934  	pc := f.entry()
   935  	prevpc := pc
   936  	val := int32(-1)
   937  	for {
   938  		var ok bool
   939  		p, ok = step(p, &pc, &val, pc == f.entry())
   940  		if !ok {
   941  			break
   942  		}
   943  		if targetpc < pc {
   944  			// Replace a random entry in the cache. Random
   945  			// replacement prevents a performance cliff if
   946  			// a recursive stack's cycle is slightly
   947  			// larger than the cache.
   948  			// Put the new element at the beginning,
   949  			// since it is the most likely to be newly used.
   950  			if debugCheckCache && checkPC != 0 {
   951  				if checkVal != val || checkPC != prevpc {
   952  					print("runtime: table value ", val, "@", prevpc, " != cache value ", checkVal, "@", checkPC, " at PC ", targetpc, " off ", off, "\n")
   953  					throw("bad pcvalue cache")
   954  				}
   955  			} else {
   956  				mp := acquirem()
   957  				cache := &mp.pcvalueCache
   958  				cache.inUse++
   959  				if cache.inUse == 1 {
   960  					e := &cache.entries[ck]
   961  					ci := cheaprandn(uint32(len(cache.entries[ck])))
   962  					e[ci] = e[0]
   963  					e[0] = pcvalueCacheEnt{
   964  						targetpc: targetpc,
   965  						off:      off,
   966  						val:      val,
   967  						valPC:    prevpc,
   968  					}
   969  				}
   970  				cache.inUse--
   971  				releasem(mp)
   972  			}
   973  
   974  			return val, prevpc
   975  		}
   976  		prevpc = pc
   977  	}
   978  
   979  	// If there was a table, it should have covered all program counters.
   980  	// If not, something is wrong.
   981  	if panicking.Load() != 0 || !strict {
   982  		return -1, 0
   983  	}
   984  
   985  	print("runtime: invalid pc-encoded table f=", funcname(f), " pc=", hex(pc), " targetpc=", hex(targetpc), " tab=", p, "\n")
   986  
   987  	p = datap.pctab[off:]
   988  	pc = f.entry()
   989  	val = -1
   990  	for {
   991  		var ok bool
   992  		p, ok = step(p, &pc, &val, pc == f.entry())
   993  		if !ok {
   994  			break
   995  		}
   996  		print("\tvalue=", val, " until pc=", hex(pc), "\n")
   997  	}
   998  
   999  	throw("invalid runtime symbol table")
  1000  	return -1, 0
  1001  }
  1002  
  1003  func funcname(f funcInfo) string {
  1004  	if !f.valid() {
  1005  		return ""
  1006  	}
  1007  	return f.datap.funcName(f.nameOff)
  1008  }
  1009  
  1010  func funcpkgpath(f funcInfo) string {
  1011  	name := funcNameForPrint(funcname(f))
  1012  	i := len(name) - 1
  1013  	for ; i > 0; i-- {
  1014  		if name[i] == '/' {
  1015  			break
  1016  		}
  1017  	}
  1018  	for ; i < len(name); i++ {
  1019  		if name[i] == '.' {
  1020  			break
  1021  		}
  1022  	}
  1023  	return name[:i]
  1024  }
  1025  
  1026  func funcfile(f funcInfo, fileno int32) string {
  1027  	datap := f.datap
  1028  	if !f.valid() {
  1029  		return "?"
  1030  	}
  1031  	// Make sure the cu index and file offset are valid
  1032  	if fileoff := datap.cutab[f.cuOffset+uint32(fileno)]; fileoff != ^uint32(0) {
  1033  		return gostringnocopy(&datap.filetab[fileoff])
  1034  	}
  1035  	// pcln section is corrupt.
  1036  	return "?"
  1037  }
  1038  
  1039  func funcline1(f funcInfo, targetpc uintptr, strict bool) (file string, line int32) {
  1040  	datap := f.datap
  1041  	if !f.valid() {
  1042  		return "?", 0
  1043  	}
  1044  	fileno, _ := pcvalue(f, f.pcfile, targetpc, strict)
  1045  	line, _ = pcvalue(f, f.pcln, targetpc, strict)
  1046  	if fileno == -1 || line == -1 || int(fileno) >= len(datap.filetab) {
  1047  		// print("looking for ", hex(targetpc), " in ", funcname(f), " got file=", fileno, " line=", lineno, "\n")
  1048  		return "?", 0
  1049  	}
  1050  	file = funcfile(f, fileno)
  1051  	return
  1052  }
  1053  
  1054  func funcline(f funcInfo, targetpc uintptr) (file string, line int32) {
  1055  	return funcline1(f, targetpc, true)
  1056  }
  1057  
  1058  func funcspdelta(f funcInfo, targetpc uintptr) int32 {
  1059  	x, _ := pcvalue(f, f.pcsp, targetpc, true)
  1060  	if debugPcln && x&(goarch.PtrSize-1) != 0 {
  1061  		print("invalid spdelta ", funcname(f), " ", hex(f.entry()), " ", hex(targetpc), " ", hex(f.pcsp), " ", x, "\n")
  1062  		throw("bad spdelta")
  1063  	}
  1064  	return x
  1065  }
  1066  
  1067  // funcMaxSPDelta returns the maximum spdelta at any point in f.
  1068  func funcMaxSPDelta(f funcInfo) int32 {
  1069  	datap := f.datap
  1070  	p := datap.pctab[f.pcsp:]
  1071  	pc := f.entry()
  1072  	val := int32(-1)
  1073  	most := int32(0)
  1074  	for {
  1075  		var ok bool
  1076  		p, ok = step(p, &pc, &val, pc == f.entry())
  1077  		if !ok {
  1078  			return most
  1079  		}
  1080  		most = max(most, val)
  1081  	}
  1082  }
  1083  
  1084  func pcdatastart(f funcInfo, table uint32) uint32 {
  1085  	return *(*uint32)(add(unsafe.Pointer(&f.nfuncdata), unsafe.Sizeof(f.nfuncdata)+uintptr(table)*4))
  1086  }
  1087  
  1088  func pcdatavalue(f funcInfo, table uint32, targetpc uintptr) int32 {
  1089  	if table >= f.npcdata {
  1090  		return -1
  1091  	}
  1092  	r, _ := pcvalue(f, pcdatastart(f, table), targetpc, true)
  1093  	return r
  1094  }
  1095  
  1096  func pcdatavalue1(f funcInfo, table uint32, targetpc uintptr, strict bool) int32 {
  1097  	if table >= f.npcdata {
  1098  		return -1
  1099  	}
  1100  	r, _ := pcvalue(f, pcdatastart(f, table), targetpc, strict)
  1101  	return r
  1102  }
  1103  
  1104  // Like pcdatavalue, but also return the start PC of this PCData value.
  1105  func pcdatavalue2(f funcInfo, table uint32, targetpc uintptr) (int32, uintptr) {
  1106  	if table >= f.npcdata {
  1107  		return -1, 0
  1108  	}
  1109  	return pcvalue(f, pcdatastart(f, table), targetpc, true)
  1110  }
  1111  
  1112  // funcdata returns a pointer to the ith funcdata for f.
  1113  // funcdata should be kept in sync with cmd/link:writeFuncs.
  1114  func funcdata(f funcInfo, i uint8) unsafe.Pointer {
  1115  	if i < 0 || i >= f.nfuncdata {
  1116  		return nil
  1117  	}
  1118  	base := f.datap.gofunc // load gofunc address early so that we calculate during cache misses
  1119  	p := uintptr(unsafe.Pointer(&f.nfuncdata)) + unsafe.Sizeof(f.nfuncdata) + uintptr(f.npcdata)*4 + uintptr(i)*4
  1120  	off := *(*uint32)(unsafe.Pointer(p))
  1121  	// Return off == ^uint32(0) ? 0 : f.datap.gofunc + uintptr(off), but without branches.
  1122  	// The compiler calculates mask on most architectures using conditional assignment.
  1123  	var mask uintptr
  1124  	if off == ^uint32(0) {
  1125  		mask = 1
  1126  	}
  1127  	mask--
  1128  	raw := base + uintptr(off)
  1129  	return unsafe.Pointer(raw & mask)
  1130  }
  1131  
  1132  // step advances to the next pc, value pair in the encoded table.
  1133  func step(p []byte, pc *uintptr, val *int32, first bool) (newp []byte, ok bool) {
  1134  	// For both uvdelta and pcdelta, the common case (~70%)
  1135  	// is that they are a single byte. If so, avoid calling readvarint.
  1136  	uvdelta := uint32(p[0])
  1137  	if uvdelta == 0 && !first {
  1138  		return nil, false
  1139  	}
  1140  	n := uint32(1)
  1141  	if uvdelta&0x80 != 0 {
  1142  		n, uvdelta = readvarint(p)
  1143  	}
  1144  	*val += int32(-(uvdelta & 1) ^ (uvdelta >> 1))
  1145  	p = p[n:]
  1146  
  1147  	pcdelta := uint32(p[0])
  1148  	n = 1
  1149  	if pcdelta&0x80 != 0 {
  1150  		n, pcdelta = readvarint(p)
  1151  	}
  1152  	p = p[n:]
  1153  	*pc += uintptr(pcdelta * sys.PCQuantum)
  1154  	return p, true
  1155  }
  1156  
  1157  // readvarint reads a varint from p.
  1158  func readvarint(p []byte) (read uint32, val uint32) {
  1159  	var v, shift, n uint32
  1160  	for {
  1161  		b := p[n]
  1162  		n++
  1163  		v |= uint32(b&0x7F) << (shift & 31)
  1164  		if b&0x80 == 0 {
  1165  			break
  1166  		}
  1167  		shift += 7
  1168  	}
  1169  	return n, v
  1170  }
  1171  
  1172  type stackmap struct {
  1173  	n        int32   // number of bitmaps
  1174  	nbit     int32   // number of bits in each bitmap
  1175  	bytedata [1]byte // bitmaps, each starting on a byte boundary
  1176  }
  1177  
  1178  //go:nowritebarrier
  1179  func stackmapdata(stkmap *stackmap, n int32) bitvector {
  1180  	// Check this invariant only when stackDebug is on at all.
  1181  	// The invariant is already checked by many of stackmapdata's callers,
  1182  	// and disabling it by default allows stackmapdata to be inlined.
  1183  	if stackDebug > 0 && (n < 0 || n >= stkmap.n) {
  1184  		throw("stackmapdata: index out of range")
  1185  	}
  1186  	return bitvector{stkmap.nbit, addb(&stkmap.bytedata[0], uintptr(n*((stkmap.nbit+7)>>3)))}
  1187  }
  1188  

View as plain text