exportdata.go

     1  // Copyright 2024 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package exportdata implements common utilities for finding
     6  // and reading gc-generated object files.
     7  package exportdata
     8  
     9  // This file should be kept in sync with src/cmd/compile/internal/gc/obj.go .
    10  
    11  import (
    12  	"bufio"
    13  	"bytes"
    14  	"errors"
    15  	"fmt"
    16  	"go/build"
    17  	"internal/saferio"
    18  	"io"
    19  	"os"
    20  	"os/exec"
    21  	"path/filepath"
    22  	"strings"
    23  	"sync"
    24  )
    25  
    26  // ReadUnified reads the contents of the unified export data from a reader r
    27  // that contains the contents of a GC-created archive file.
    28  //
    29  // On success, the reader will be positioned after the end-of-section marker "\n$$\n".
    30  //
    31  // Supported GC-created archive files have 4 layers of nesting:
    32  //   - An archive file containing a package definition file.
    33  //   - The package definition file contains headers followed by a data section.
    34  //     Headers are lines (≤ 4kb) that do not start with "$$".
    35  //   - The data section starts with "$$B\n" followed by export data followed
    36  //     by an end of section marker "\n$$\n". (The section start "$$\n" is no
    37  //     longer supported.)
    38  //   - The export data starts with a format byte ('u') followed by the <data> in
    39  //     the given format. (See ReadExportDataHeader for older formats.)
    40  //
    41  // Putting this together, the bytes in a GC-created archive files are expected
    42  // to look like the following.
    43  // See cmd/internal/archive for more details on ar file headers.
    44  //
    45  // | <!arch>\n             | ar file signature
    46  // | __.PKGDEF...size...\n | ar header for __.PKGDEF including size.
    47  // | go object <...>\n     | objabi header
    48  // | <optional headers>\n  | other headers such as build id
    49  // | $$B\n                 | binary format marker
    50  // | u<data>\n             | unified export <data>
    51  // | $$\n                  | end-of-section marker
    52  // | [optional padding]    | padding byte (0x0A) if size is odd
    53  // | [ar file header]      | other ar files
    54  // | [ar file data]        |
    55  func ReadUnified(r *bufio.Reader) (data []byte, err error) {
    56  	// We historically guaranteed headers at the default buffer size (4096) work.
    57  	// This ensures we can use ReadSlice throughout.
    58  	const minBufferSize = 4096
    59  	r = bufio.NewReaderSize(r, minBufferSize)
    60  
    61  	size, err := FindPackageDefinition(r)
    62  	if err != nil {
    63  		return
    64  	}
    65  	n := size
    66  
    67  	objapi, headers, err := ReadObjectHeaders(r)
    68  	if err != nil {
    69  		return
    70  	}
    71  	n -= len(objapi)
    72  	for _, h := range headers {
    73  		n -= len(h)
    74  	}
    75  
    76  	hdrlen, err := ReadExportDataHeader(r)
    77  	if err != nil {
    78  		return
    79  	}
    80  	n -= hdrlen
    81  
    82  	// size also includes the end of section marker. Remove that many bytes from the end.
    83  	const marker = "\n$$\n"
    84  	n -= len(marker)
    85  
    86  	if n < 0 {
    87  		err = fmt.Errorf("invalid size (%d) in the archive file: %d bytes remain without section headers (recompile package)", size, n)
    88  	}
    89  
    90  	// Read n bytes from buf.
    91  	data, err = saferio.ReadData(r, uint64(n))
    92  	if err != nil {
    93  		return
    94  	}
    95  
    96  	// Check for marker at the end.
    97  	var suffix [len(marker)]byte
    98  	_, err = io.ReadFull(r, suffix[:])
    99  	if err != nil {
   100  		return
   101  	}
   102  	if s := string(suffix[:]); s != marker {
   103  		err = fmt.Errorf("read %q instead of end-of-section marker (%q)", s, marker)
   104  		return
   105  	}
   106  
   107  	return
   108  }
   109  
   110  // FindPackageDefinition positions the reader r at the beginning of a package
   111  // definition file ("__.PKGDEF") within a GC-created archive by reading
   112  // from it, and returns the size of the package definition file in the archive.
   113  //
   114  // The reader must be positioned at the start of the archive file before calling
   115  // this function, and "__.PKGDEF" is assumed to be the first file in the archive.
   116  //
   117  // See cmd/internal/archive for details on the archive format.
   118  func FindPackageDefinition(r *bufio.Reader) (size int, err error) {
   119  	// Uses ReadSlice to limit risk of malformed inputs.
   120  
   121  	// Read first line to make sure this is an object file.
   122  	line, err := r.ReadSlice('\n')
   123  	if err != nil {
   124  		err = fmt.Errorf("can't find export data (%v)", err)
   125  		return
   126  	}
   127  
   128  	// Is the first line an archive file signature?
   129  	if string(line) != "!<arch>\n" {
   130  		err = fmt.Errorf("not the start of an archive file (%q)", line)
   131  		return
   132  	}
   133  
   134  	// package export block should be first
   135  	size = readArchiveHeader(r, "__.PKGDEF")
   136  	if size <= 0 {
   137  		err = fmt.Errorf("not a package file")
   138  		return
   139  	}
   140  
   141  	return
   142  }
   143  
   144  // ReadObjectHeaders reads object headers from the reader. Object headers are
   145  // lines that do not start with an end-of-section marker "$$". The first header
   146  // is the objabi header. On success, the reader will be positioned at the beginning
   147  // of the end-of-section marker.
   148  //
   149  // It returns an error if any header does not fit in r.Size() bytes.
   150  func ReadObjectHeaders(r *bufio.Reader) (objapi string, headers []string, err error) {
   151  	// line is a temporary buffer for headers.
   152  	// Use bounded reads (ReadSlice, Peek) to limit risk of malformed inputs.
   153  	var line []byte
   154  
   155  	// objapi header should be the first line
   156  	if line, err = r.ReadSlice('\n'); err != nil {
   157  		err = fmt.Errorf("can't find export data (%v)", err)
   158  		return
   159  	}
   160  	objapi = string(line)
   161  
   162  	// objapi header begins with "go object ".
   163  	if !strings.HasPrefix(objapi, "go object ") {
   164  		err = fmt.Errorf("not a go object file: %s", objapi)
   165  		return
   166  	}
   167  
   168  	// process remaining object header lines
   169  	for {
   170  		// check for an end of section marker "$$"
   171  		line, err = r.Peek(2)
   172  		if err != nil {
   173  			return
   174  		}
   175  		if string(line) == "$$" {
   176  			return // stop
   177  		}
   178  
   179  		// read next header
   180  		line, err = r.ReadSlice('\n')
   181  		if err != nil {
   182  			return
   183  		}
   184  		headers = append(headers, string(line))
   185  	}
   186  }
   187  
   188  // ReadExportDataHeader reads the export data header and format from r.
   189  // It returns the number of bytes read, or an error if the format is no longer
   190  // supported or it failed to read.
   191  //
   192  // The only currently supported format is binary export data in the
   193  // unified export format.
   194  func ReadExportDataHeader(r *bufio.Reader) (n int, err error) {
   195  	// Read export data header.
   196  	line, err := r.ReadSlice('\n')
   197  	if err != nil {
   198  		return
   199  	}
   200  
   201  	hdr := string(line)
   202  	switch hdr {
   203  	case "$$\n":
   204  		err = fmt.Errorf("old textual export format no longer supported (recompile package)")
   205  		return
   206  
   207  	case "$$B\n":
   208  		var format byte
   209  		format, err = r.ReadByte()
   210  		if err != nil {
   211  			return
   212  		}
   213  		// The unified export format starts with a 'u'.
   214  		switch format {
   215  		case 'u':
   216  		default:
   217  			// Older no longer supported export formats include:
   218  			// indexed export format which started with an 'i'; and
   219  			// the older binary export format which started with a 'c',
   220  			// 'd', or 'v' (from "version").
   221  			err = fmt.Errorf("binary export format %q is no longer supported (recompile package)", format)
   222  			return
   223  		}
   224  
   225  	default:
   226  		err = fmt.Errorf("unknown export data header: %q", hdr)
   227  		return
   228  	}
   229  
   230  	n = len(hdr) + 1 // + 1 is for 'u'
   231  	return
   232  }
   233  
   234  // FindPkg returns the filename and unique package id for an import
   235  // path based on package information provided by build.Import (using
   236  // the build.Default build.Context). A relative srcDir is interpreted
   237  // relative to the current working directory.
   238  func FindPkg(path, srcDir string) (filename, id string, err error) {
   239  	if path == "" {
   240  		return "", "", errors.New("path is empty")
   241  	}
   242  
   243  	var noext string
   244  	switch {
   245  	default:
   246  		// "x" -> "$GOPATH/pkg/$GOOS_$GOARCH/x.ext", "x"
   247  		// Don't require the source files to be present.
   248  		if abs, err := filepath.Abs(srcDir); err == nil { // see issue 14282
   249  			srcDir = abs
   250  		}
   251  		var bp *build.Package
   252  		bp, err = build.Import(path, srcDir, build.FindOnly|build.AllowBinary)
   253  		if bp.PkgObj == "" {
   254  			if bp.Goroot && bp.Dir != "" {
   255  				filename, err = lookupGorootExport(bp.Dir)
   256  				if err == nil {
   257  					_, err = os.Stat(filename)
   258  				}
   259  				if err == nil {
   260  					return filename, bp.ImportPath, nil
   261  				}
   262  			}
   263  			goto notfound
   264  		} else {
   265  			noext = strings.TrimSuffix(bp.PkgObj, ".a")
   266  		}
   267  		id = bp.ImportPath
   268  
   269  	case build.IsLocalImport(path):
   270  		// "./x" -> "/this/directory/x.ext", "/this/directory/x"
   271  		noext = filepath.Join(srcDir, path)
   272  		id = noext
   273  
   274  	case filepath.IsAbs(path):
   275  		// for completeness only - go/build.Import
   276  		// does not support absolute imports
   277  		// "/x" -> "/x.ext", "/x"
   278  		noext = path
   279  		id = path
   280  	}
   281  
   282  	if false { // for debugging
   283  		if path != id {
   284  			fmt.Printf("%s -> %s\n", path, id)
   285  		}
   286  	}
   287  
   288  	// try extensions
   289  	for _, ext := range pkgExts {
   290  		filename = noext + ext
   291  		f, statErr := os.Stat(filename)
   292  		if statErr == nil && !f.IsDir() {
   293  			return filename, id, nil
   294  		}
   295  		if err == nil {
   296  			err = statErr
   297  		}
   298  	}
   299  
   300  notfound:
   301  	if err == nil {
   302  		return "", path, fmt.Errorf("can't find import: %q", path)
   303  	}
   304  	return "", path, fmt.Errorf("can't find import: %q: %w", path, err)
   305  }
   306  
   307  var pkgExts = [...]string{".a", ".o"} // a file from the build cache will have no extension
   308  
   309  var exportMap sync.Map // package dir → func() (string, error)
   310  
   311  // lookupGorootExport returns the location of the export data
   312  // (normally found in the build cache, but located in GOROOT/pkg
   313  // in prior Go releases) for the package located in pkgDir.
   314  //
   315  // (We use the package's directory instead of its import path
   316  // mainly to simplify handling of the packages in src/vendor
   317  // and cmd/vendor.)
   318  func lookupGorootExport(pkgDir string) (string, error) {
   319  	f, ok := exportMap.Load(pkgDir)
   320  	if !ok {
   321  		var (
   322  			listOnce   sync.Once
   323  			exportPath string
   324  			err        error
   325  		)
   326  		f, _ = exportMap.LoadOrStore(pkgDir, func() (string, error) {
   327  			listOnce.Do(func() {
   328  				cmd := exec.Command(filepath.Join(build.Default.GOROOT, "bin", "go"), "list", "-export", "-f", "{{.Export}}", pkgDir)
   329  				cmd.Dir = build.Default.GOROOT
   330  				cmd.Env = append(os.Environ(), "PWD="+cmd.Dir, "GOROOT="+build.Default.GOROOT)
   331  				var output []byte
   332  				output, err = cmd.Output()
   333  				if err != nil {
   334  					if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
   335  						err = errors.New(string(ee.Stderr))
   336  					}
   337  					return
   338  				}
   339  
   340  				exports := strings.Split(string(bytes.TrimSpace(output)), "\n")
   341  				if len(exports) != 1 {
   342  					err = fmt.Errorf("go list reported %d exports; expected 1", len(exports))
   343  					return
   344  				}
   345  
   346  				exportPath = exports[0]
   347  			})
   348  
   349  			return exportPath, err
   350  		})
   351  	}
   352  
   353  	return f.(func() (string, error))()
   354  }
   355
View as plain text