exportdata.go

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // This file should be kept in sync with $GOROOT/src/internal/exportdata/exportdata.go.
     6  // This file also additionally implements FindExportData for gcexportdata.NewReader.
     7  
     8  package gcimporter
     9  
    10  import (
    11  	"bufio"
    12  	"bytes"
    13  	"errors"
    14  	"fmt"
    15  	"go/build"
    16  	"io"
    17  	"os"
    18  	"os/exec"
    19  	"path/filepath"
    20  	"strings"
    21  	"sync"
    22  )
    23  
    24  // FindExportData positions the reader r at the beginning of the
    25  // export data section of an underlying cmd/compile created archive
    26  // file by reading from it. The reader must be positioned at the
    27  // start of the file before calling this function.
    28  // This returns the length of the export data in bytes.
    29  //
    30  // This function is needed by [gcexportdata.Read], which must
    31  // accept inputs produced by the last two releases of cmd/compile,
    32  // plus tip.
    33  func FindExportData(r *bufio.Reader) (size int64, err error) {
    34  	arsize, err := FindPackageDefinition(r)
    35  	if err != nil {
    36  		return
    37  	}
    38  	size = int64(arsize)
    39  
    40  	objapi, headers, err := ReadObjectHeaders(r)
    41  	if err != nil {
    42  		return
    43  	}
    44  	size -= int64(len(objapi))
    45  	for _, h := range headers {
    46  		size -= int64(len(h))
    47  	}
    48  
    49  	// Check for the binary export data section header "$$B\n".
    50  	// TODO(taking): Unify with ReadExportDataHeader so that it stops at the 'u' instead of reading
    51  	line, err := r.ReadSlice('\n')
    52  	if err != nil {
    53  		return
    54  	}
    55  	hdr := string(line)
    56  	if hdr != "$$B\n" {
    57  		err = fmt.Errorf("unknown export data header: %q", hdr)
    58  		return
    59  	}
    60  	size -= int64(len(hdr))
    61  
    62  	// For files with a binary export data header "$$B\n",
    63  	// these are always terminated by an end-of-section marker "\n$$\n".
    64  	// So the last bytes must always be this constant.
    65  	//
    66  	// The end-of-section marker is not a part of the export data itself.
    67  	// Do not include these in size.
    68  	//
    69  	// It would be nice to have sanity check that the final bytes after
    70  	// the export data are indeed the end-of-section marker. The split
    71  	// of gcexportdata.NewReader and gcexportdata.Read make checking this
    72  	// ugly so gcimporter gives up enforcing this. The compiler and go/types
    73  	// importer do enforce this, which seems good enough.
    74  	const endofsection = "\n$$\n"
    75  	size -= int64(len(endofsection))
    76  
    77  	if size < 0 {
    78  		err = fmt.Errorf("invalid size (%d) in the archive file: %d bytes remain without section headers (recompile package)", arsize, size)
    79  		return
    80  	}
    81  
    82  	return
    83  }
    84  
    85  // ReadUnified reads the contents of the unified export data from a reader r
    86  // that contains the contents of a GC-created archive file.
    87  //
    88  // On success, the reader will be positioned after the end-of-section marker "\n$$\n".
    89  //
    90  // Supported GC-created archive files have 4 layers of nesting:
    91  //   - An archive file containing a package definition file.
    92  //   - The package definition file contains headers followed by a data section.
    93  //     Headers are lines (≤ 4kb) that do not start with "$$".
    94  //   - The data section starts with "$$B\n" followed by export data followed
    95  //     by an end of section marker "\n$$\n". (The section start "$$\n" is no
    96  //     longer supported.)
    97  //   - The export data starts with a format byte ('u') followed by the <data> in
    98  //     the given format. (See ReadExportDataHeader for older formats.)
    99  //
   100  // Putting this together, the bytes in a GC-created archive files are expected
   101  // to look like the following.
   102  // See cmd/internal/archive for more details on ar file headers.
   103  //
   104  // | <!arch>\n             | ar file signature
   105  // | __.PKGDEF...size...\n | ar header for __.PKGDEF including size.
   106  // | go object <...>\n     | objabi header
   107  // | <optional headers>\n  | other headers such as build id
   108  // | $$B\n                 | binary format marker
   109  // | u<data>\n             | unified export <data>
   110  // | $$\n                  | end-of-section marker
   111  // | [optional padding]    | padding byte (0x0A) if size is odd
   112  // | [ar file header]      | other ar files
   113  // | [ar file data]        |
   114  func ReadUnified(r *bufio.Reader) (data []byte, err error) {
   115  	// We historically guaranteed headers at the default buffer size (4096) work.
   116  	// This ensures we can use ReadSlice throughout.
   117  	const minBufferSize = 4096
   118  	r = bufio.NewReaderSize(r, minBufferSize)
   119  
   120  	size, err := FindPackageDefinition(r)
   121  	if err != nil {
   122  		return
   123  	}
   124  	n := size
   125  
   126  	objapi, headers, err := ReadObjectHeaders(r)
   127  	if err != nil {
   128  		return
   129  	}
   130  	n -= len(objapi)
   131  	for _, h := range headers {
   132  		n -= len(h)
   133  	}
   134  
   135  	hdrlen, err := ReadExportDataHeader(r)
   136  	if err != nil {
   137  		return
   138  	}
   139  	n -= hdrlen
   140  
   141  	// size also includes the end of section marker. Remove that many bytes from the end.
   142  	const marker = "\n$$\n"
   143  	n -= len(marker)
   144  
   145  	if n < 0 {
   146  		err = fmt.Errorf("invalid size (%d) in the archive file: %d bytes remain without section headers (recompile package)", size, n)
   147  		return
   148  	}
   149  
   150  	// Read n bytes from buf.
   151  	data = make([]byte, n)
   152  	_, err = io.ReadFull(r, data)
   153  	if err != nil {
   154  		return
   155  	}
   156  
   157  	// Check for marker at the end.
   158  	var suffix [len(marker)]byte
   159  	_, err = io.ReadFull(r, suffix[:])
   160  	if err != nil {
   161  		return
   162  	}
   163  	if s := string(suffix[:]); s != marker {
   164  		err = fmt.Errorf("read %q instead of end-of-section marker (%q)", s, marker)
   165  		return
   166  	}
   167  
   168  	return
   169  }
   170  
   171  // FindPackageDefinition positions the reader r at the beginning of a package
   172  // definition file ("__.PKGDEF") within a GC-created archive by reading
   173  // from it, and returns the size of the package definition file in the archive.
   174  //
   175  // The reader must be positioned at the start of the archive file before calling
   176  // this function, and "__.PKGDEF" is assumed to be the first file in the archive.
   177  //
   178  // See cmd/internal/archive for details on the archive format.
   179  func FindPackageDefinition(r *bufio.Reader) (size int, err error) {
   180  	// Uses ReadSlice to limit risk of malformed inputs.
   181  
   182  	// Read first line to make sure this is an object file.
   183  	line, err := r.ReadSlice('\n')
   184  	if err != nil {
   185  		err = fmt.Errorf("can't find export data (%v)", err)
   186  		return
   187  	}
   188  
   189  	// Is the first line an archive file signature?
   190  	if string(line) != "!<arch>\n" {
   191  		err = fmt.Errorf("not the start of an archive file (%q)", line)
   192  		return
   193  	}
   194  
   195  	// package export block should be first
   196  	size = readArchiveHeader(r, "__.PKGDEF")
   197  	if size <= 0 {
   198  		err = fmt.Errorf("not a package file")
   199  		return
   200  	}
   201  
   202  	return
   203  }
   204  
   205  // ReadObjectHeaders reads object headers from the reader. Object headers are
   206  // lines that do not start with an end-of-section marker "$$". The first header
   207  // is the objabi header. On success, the reader will be positioned at the beginning
   208  // of the end-of-section marker.
   209  //
   210  // It returns an error if any header does not fit in r.Size() bytes.
   211  func ReadObjectHeaders(r *bufio.Reader) (objapi string, headers []string, err error) {
   212  	// line is a temporary buffer for headers.
   213  	// Use bounded reads (ReadSlice, Peek) to limit risk of malformed inputs.
   214  	var line []byte
   215  
   216  	// objapi header should be the first line
   217  	if line, err = r.ReadSlice('\n'); err != nil {
   218  		err = fmt.Errorf("can't find export data (%v)", err)
   219  		return
   220  	}
   221  	objapi = string(line)
   222  
   223  	// objapi header begins with "go object ".
   224  	if !strings.HasPrefix(objapi, "go object ") {
   225  		err = fmt.Errorf("not a go object file: %s", objapi)
   226  		return
   227  	}
   228  
   229  	// process remaining object header lines
   230  	for {
   231  		// check for an end of section marker "$$"
   232  		line, err = r.Peek(2)
   233  		if err != nil {
   234  			return
   235  		}
   236  		if string(line) == "$$" {
   237  			return // stop
   238  		}
   239  
   240  		// read next header
   241  		line, err = r.ReadSlice('\n')
   242  		if err != nil {
   243  			return
   244  		}
   245  		headers = append(headers, string(line))
   246  	}
   247  }
   248  
   249  // ReadExportDataHeader reads the export data header and format from r.
   250  // It returns the number of bytes read, or an error if the format is no longer
   251  // supported or it failed to read.
   252  //
   253  // The only currently supported format is binary export data in the
   254  // unified export format.
   255  func ReadExportDataHeader(r *bufio.Reader) (n int, err error) {
   256  	// Read export data header.
   257  	line, err := r.ReadSlice('\n')
   258  	if err != nil {
   259  		return
   260  	}
   261  
   262  	hdr := string(line)
   263  	switch hdr {
   264  	case "$$\n":
   265  		err = fmt.Errorf("old textual export format no longer supported (recompile package)")
   266  		return
   267  
   268  	case "$$B\n":
   269  		var format byte
   270  		format, err = r.ReadByte()
   271  		if err != nil {
   272  			return
   273  		}
   274  		// The unified export format starts with a 'u'.
   275  		switch format {
   276  		case 'u':
   277  		default:
   278  			// Older no longer supported export formats include:
   279  			// indexed export format which started with an 'i'; and
   280  			// the older binary export format which started with a 'c',
   281  			// 'd', or 'v' (from "version").
   282  			err = fmt.Errorf("binary export format %q is no longer supported (recompile package)", format)
   283  			return
   284  		}
   285  
   286  	default:
   287  		err = fmt.Errorf("unknown export data header: %q", hdr)
   288  		return
   289  	}
   290  
   291  	n = len(hdr) + 1 // + 1 is for 'u'
   292  	return
   293  }
   294  
   295  // FindPkg returns the filename and unique package id for an import
   296  // path based on package information provided by build.Import (using
   297  // the build.Default build.Context). A relative srcDir is interpreted
   298  // relative to the current working directory.
   299  //
   300  // FindPkg is only used in tests within x/tools.
   301  func FindPkg(path, srcDir string) (filename, id string, err error) {
   302  	// TODO(taking): Move internal/exportdata.FindPkg into its own file,
   303  	// and then this copy into a _test package.
   304  	if path == "" {
   305  		return "", "", errors.New("path is empty")
   306  	}
   307  
   308  	var noext string
   309  	switch {
   310  	default:
   311  		// "x" -> "$GOPATH/pkg/$GOOS_$GOARCH/x.ext", "x"
   312  		// Don't require the source files to be present.
   313  		if abs, err := filepath.Abs(srcDir); err == nil { // see issue 14282
   314  			srcDir = abs
   315  		}
   316  		var bp *build.Package
   317  		bp, err = build.Import(path, srcDir, build.FindOnly|build.AllowBinary)
   318  		if bp.PkgObj == "" {
   319  			if bp.Goroot && bp.Dir != "" {
   320  				filename, err = lookupGorootExport(bp.Dir)
   321  				if err == nil {
   322  					_, err = os.Stat(filename)
   323  				}
   324  				if err == nil {
   325  					return filename, bp.ImportPath, nil
   326  				}
   327  			}
   328  			goto notfound
   329  		} else {
   330  			noext = strings.TrimSuffix(bp.PkgObj, ".a")
   331  		}
   332  		id = bp.ImportPath
   333  
   334  	case build.IsLocalImport(path):
   335  		// "./x" -> "/this/directory/x.ext", "/this/directory/x"
   336  		noext = filepath.Join(srcDir, path)
   337  		id = noext
   338  
   339  	case filepath.IsAbs(path):
   340  		// for completeness only - go/build.Import
   341  		// does not support absolute imports
   342  		// "/x" -> "/x.ext", "/x"
   343  		noext = path
   344  		id = path
   345  	}
   346  
   347  	if false { // for debugging
   348  		if path != id {
   349  			fmt.Printf("%s -> %s\n", path, id)
   350  		}
   351  	}
   352  
   353  	// try extensions
   354  	for _, ext := range pkgExts {
   355  		filename = noext + ext
   356  		f, statErr := os.Stat(filename)
   357  		if statErr == nil && !f.IsDir() {
   358  			return filename, id, nil
   359  		}
   360  		if err == nil {
   361  			err = statErr
   362  		}
   363  	}
   364  
   365  notfound:
   366  	if err == nil {
   367  		return "", path, fmt.Errorf("can't find import: %q", path)
   368  	}
   369  	return "", path, fmt.Errorf("can't find import: %q: %w", path, err)
   370  }
   371  
   372  var pkgExts = [...]string{".a", ".o"} // a file from the build cache will have no extension
   373  
   374  var exportMap sync.Map // package dir → func() (string, error)
   375  
   376  // lookupGorootExport returns the location of the export data
   377  // (normally found in the build cache, but located in GOROOT/pkg
   378  // in prior Go releases) for the package located in pkgDir.
   379  //
   380  // (We use the package's directory instead of its import path
   381  // mainly to simplify handling of the packages in src/vendor
   382  // and cmd/vendor.)
   383  //
   384  // lookupGorootExport is only used in tests within x/tools.
   385  func lookupGorootExport(pkgDir string) (string, error) {
   386  	f, ok := exportMap.Load(pkgDir)
   387  	if !ok {
   388  		var (
   389  			listOnce   sync.Once
   390  			exportPath string
   391  			err        error
   392  		)
   393  		f, _ = exportMap.LoadOrStore(pkgDir, func() (string, error) {
   394  			listOnce.Do(func() {
   395  				cmd := exec.Command(filepath.Join(build.Default.GOROOT, "bin", "go"), "list", "-export", "-f", "{{.Export}}", pkgDir)
   396  				cmd.Dir = build.Default.GOROOT
   397  				cmd.Env = append(os.Environ(), "PWD="+cmd.Dir, "GOROOT="+build.Default.GOROOT)
   398  				var output []byte
   399  				output, err = cmd.Output()
   400  				if err != nil {
   401  					if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
   402  						err = errors.New(string(ee.Stderr))
   403  					}
   404  					return
   405  				}
   406  
   407  				exports := strings.Split(string(bytes.TrimSpace(output)), "\n")
   408  				if len(exports) != 1 {
   409  					err = fmt.Errorf("go list reported %d exports; expected 1", len(exports))
   410  					return
   411  				}
   412  
   413  				exportPath = exports[0]
   414  			})
   415  
   416  			return exportPath, err
   417  		})
   418  	}
   419  
   420  	return f.(func() (string, error))()
   421  }
   422
View as plain text