relnote.go

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package relnote supports working with release notes.
     6  //
     7  // Its main feature is the ability to merge Markdown fragments into a single
     8  // document. (See [Merge].)
     9  //
    10  // This package has minimal imports, so that it can be vendored into the
    11  // main go repo.
    12  package relnote
    13  
    14  import (
    15  	"bufio"
    16  	"bytes"
    17  	"errors"
    18  	"fmt"
    19  	"io"
    20  	"io/fs"
    21  	"path"
    22  	"regexp"
    23  	"slices"
    24  	"strconv"
    25  	"strings"
    26  
    27  	md "rsc.io/markdown"
    28  )
    29  
    30  // NewParser returns a properly configured Markdown parser.
    31  func NewParser() *md.Parser {
    32  	var p md.Parser
    33  	p.HeadingIDs = true
    34  	return &p
    35  }
    36  
    37  // CheckFragment reports problems in a release-note fragment.
    38  func CheckFragment(data string) error {
    39  	doc := NewParser().Parse(data)
    40  	// Check that the content of the document contains either a TODO or at least one sentence.
    41  	txt := ""
    42  	if len(doc.Blocks) > 0 {
    43  		txt = text(doc)
    44  	}
    45  	if !strings.Contains(txt, "TODO") && !strings.ContainsAny(txt, ".?!") {
    46  		return errors.New("File must contain a complete sentence or a TODO.")
    47  	}
    48  	return nil
    49  }
    50  
    51  // text returns all the text in a block, without any formatting.
    52  func text(b md.Block) string {
    53  	switch b := b.(type) {
    54  	case *md.Document:
    55  		return blocksText(b.Blocks)
    56  	case *md.Heading:
    57  		return text(b.Text)
    58  	case *md.Text:
    59  		return inlineText(b.Inline)
    60  	case *md.CodeBlock:
    61  		return strings.Join(b.Text, "\n")
    62  	case *md.HTMLBlock:
    63  		return strings.Join(b.Text, "\n")
    64  	case *md.List:
    65  		return blocksText(b.Items)
    66  	case *md.Item:
    67  		return blocksText(b.Blocks)
    68  	case *md.Empty:
    69  		return ""
    70  	case *md.Paragraph:
    71  		return text(b.Text)
    72  	case *md.Quote:
    73  		return blocksText(b.Blocks)
    74  	case *md.ThematicBreak:
    75  		return "---"
    76  	default:
    77  		panic(fmt.Sprintf("unknown block type %T", b))
    78  	}
    79  }
    80  
    81  // blocksText returns all the text in a slice of block nodes.
    82  func blocksText(bs []md.Block) string {
    83  	var d strings.Builder
    84  	for _, b := range bs {
    85  		io.WriteString(&d, text(b))
    86  		fmt.Fprintln(&d)
    87  	}
    88  	return d.String()
    89  }
    90  
    91  // inlineText returns all the next in a slice of inline nodes.
    92  func inlineText(ins []md.Inline) string {
    93  	var buf bytes.Buffer
    94  	for _, in := range ins {
    95  		in.PrintText(&buf)
    96  	}
    97  	return buf.String()
    98  }
    99  
   100  // Merge combines the markdown documents (files ending in ".md") in the tree rooted
   101  // at fs into a single document.
   102  // The blocks of the documents are concatenated in lexicographic order by filename.
   103  // Heading with no content are removed.
   104  // The link keys must be unique, and are combined into a single map.
   105  //
   106  // Files in the "minor changes" directory (the unique directory matching the glob
   107  // "*stdlib/*minor") are named after the package to which they refer, and will have
   108  // the package heading inserted automatically and links to other standard library
   109  // symbols expanded automatically. For example, if a file *stdlib/minor/bytes/f.md
   110  // contains the text
   111  //
   112  //	[Reader] implements [io.Reader].
   113  //
   114  // then that will become
   115  //
   116  //	[Reader](/pkg/bytes#Reader) implements [io.Reader](/pkg/io#Reader).
   117  func Merge(fsys fs.FS) (*md.Document, error) {
   118  	filenames, err := sortedMarkdownFilenames(fsys)
   119  	if err != nil {
   120  		return nil, err
   121  	}
   122  	doc := &md.Document{Links: map[string]*md.Link{}}
   123  	var prevPkg string // previous stdlib package, if any
   124  	for _, filename := range filenames {
   125  		newdoc, err := parseMarkdownFile(fsys, filename)
   126  		if err != nil {
   127  			return nil, err
   128  		}
   129  		if len(newdoc.Blocks) == 0 {
   130  			continue
   131  		}
   132  		pkg := stdlibPackage(filename)
   133  		// Autolink Go symbols.
   134  		addSymbolLinks(newdoc, pkg)
   135  		if len(doc.Blocks) > 0 {
   136  			// If this is the first file of a new stdlib package under the "Minor changes
   137  			// to the library" section, insert a heading for the package.
   138  			if pkg != "" && pkg != prevPkg {
   139  				h := stdlibPackageHeading(pkg, lastBlock(doc).Pos().EndLine)
   140  				doc.Blocks = append(doc.Blocks, h)
   141  			}
   142  			prevPkg = pkg
   143  			// Put a blank line between the current and new blocks, so that the end
   144  			// of a file acts as a blank line.
   145  			lastLine := lastBlock(doc).Pos().EndLine
   146  			delta := lastLine + 2 - newdoc.Blocks[0].Pos().StartLine
   147  			if pkg != "" {
   148  				// For stdlib minor changes, include the file name as a comment, as
   149  				// it often contains the issue number which is helpful.
   150  				comment := &md.HTMLBlock{
   151  					Position: md.Position{StartLine: lastLine + 2, EndLine: lastLine + 2},
   152  					Text:     []string{"<!-- " + filename + " -->"},
   153  				}
   154  				delta++
   155  				doc.Blocks = append(doc.Blocks, comment)
   156  			}
   157  			for _, b := range newdoc.Blocks {
   158  				addLines(b, delta)
   159  			}
   160  		}
   161  		// Append non-empty blocks to the result document.
   162  		for _, b := range newdoc.Blocks {
   163  			if _, ok := b.(*md.Empty); !ok {
   164  				doc.Blocks = append(doc.Blocks, b)
   165  			}
   166  		}
   167  		// Merge link references.
   168  		for key, link := range newdoc.Links {
   169  			if doc.Links[key] != nil {
   170  				return nil, fmt.Errorf("duplicate link reference %q; second in %s", key, filename)
   171  			}
   172  			doc.Links[key] = link
   173  		}
   174  	}
   175  	// Remove headings with empty contents.
   176  	doc.Blocks = removeEmptySections(doc.Blocks)
   177  	if len(doc.Blocks) > 0 && len(doc.Links) > 0 {
   178  		// Add a blank line to separate the links.
   179  		lastPos := lastBlock(doc).Pos()
   180  		lastPos.StartLine += 2
   181  		lastPos.EndLine += 2
   182  		doc.Blocks = append(doc.Blocks, &md.Empty{Position: lastPos})
   183  	}
   184  	return doc, nil
   185  }
   186  
   187  // stdlibPackage returns the standard library package for the given filename.
   188  // If the filename does not represent a package, it returns the empty string.
   189  // A filename represents package P if it is in a directory matching the glob
   190  // "*stdlib/*minor/P".
   191  func stdlibPackage(filename string) string {
   192  	dir, rest, _ := strings.Cut(filename, "/")
   193  	if !strings.HasSuffix(dir, "stdlib") {
   194  		return ""
   195  	}
   196  	dir, rest, _ = strings.Cut(rest, "/")
   197  	if !strings.HasSuffix(dir, "minor") {
   198  		return ""
   199  	}
   200  	pkg := path.Dir(rest)
   201  	if pkg == "." {
   202  		return ""
   203  	}
   204  	return pkg
   205  }
   206  
   207  func stdlibPackageHeading(pkg string, lastLine int) *md.Heading {
   208  	line := lastLine + 2
   209  	pos := md.Position{StartLine: line, EndLine: line}
   210  	return &md.Heading{
   211  		Position: pos,
   212  		Level:    4,
   213  		Text: &md.Text{
   214  			Position: pos,
   215  			Inline: []md.Inline{
   216  				&md.Link{
   217  					Inner: []md.Inline{&md.Code{Text: pkg}},
   218  					URL:   "/pkg/" + pkg + "/",
   219  				},
   220  			},
   221  		},
   222  	}
   223  }
   224  
   225  // removeEmptySections removes headings with no content. A heading has no content
   226  // if there are no blocks between it and the next heading at the same level, or the
   227  // end of the document.
   228  func removeEmptySections(bs []md.Block) []md.Block {
   229  	res := bs[:0]
   230  	delta := 0 // number of lines by which to adjust positions
   231  
   232  	// Remove preceding headings at same or higher level; they are empty.
   233  	rem := func(level int) {
   234  		for len(res) > 0 {
   235  			last := res[len(res)-1]
   236  			if lh, ok := last.(*md.Heading); ok && lh.Level >= level {
   237  				res = res[:len(res)-1]
   238  				// Adjust subsequent block positions by the size of this block
   239  				// plus 1 for the blank line between headings.
   240  				delta += lh.EndLine - lh.StartLine + 2
   241  			} else {
   242  				break
   243  			}
   244  		}
   245  	}
   246  
   247  	for _, b := range bs {
   248  		if h, ok := b.(*md.Heading); ok {
   249  			rem(h.Level)
   250  		}
   251  		addLines(b, -delta)
   252  		res = append(res, b)
   253  	}
   254  	// Remove empty headings at the end of the document.
   255  	rem(1)
   256  	return res
   257  }
   258  
   259  func sortedMarkdownFilenames(fsys fs.FS) ([]string, error) {
   260  	var filenames []string
   261  	err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
   262  		if err != nil {
   263  			return err
   264  		}
   265  		if !d.IsDir() && strings.HasSuffix(path, ".md") {
   266  			filenames = append(filenames, path)
   267  		}
   268  		return nil
   269  	})
   270  	if err != nil {
   271  		return nil, err
   272  	}
   273  	// '.' comes before '/', which comes before alphanumeric characters.
   274  	// So just sorting the list will put a filename like "net.md" before
   275  	// the directory "net". That is what we want.
   276  	slices.Sort(filenames)
   277  	return filenames, nil
   278  }
   279  
   280  // lastBlock returns the last block in the document.
   281  // It panics if the document has no blocks.
   282  func lastBlock(doc *md.Document) md.Block {
   283  	return doc.Blocks[len(doc.Blocks)-1]
   284  }
   285  
   286  // addLines adds n lines to the position of b.
   287  // n can be negative.
   288  func addLines(b md.Block, n int) {
   289  	pos := position(b)
   290  	pos.StartLine += n
   291  	pos.EndLine += n
   292  }
   293  
   294  func position(b md.Block) *md.Position {
   295  	switch b := b.(type) {
   296  	case *md.Heading:
   297  		return &b.Position
   298  	case *md.Text:
   299  		return &b.Position
   300  	case *md.CodeBlock:
   301  		return &b.Position
   302  	case *md.HTMLBlock:
   303  		return &b.Position
   304  	case *md.List:
   305  		return &b.Position
   306  	case *md.Item:
   307  		return &b.Position
   308  	case *md.Empty:
   309  		return &b.Position
   310  	case *md.Paragraph:
   311  		return &b.Position
   312  	case *md.Quote:
   313  		return &b.Position
   314  	case *md.ThematicBreak:
   315  		return &b.Position
   316  	default:
   317  		panic(fmt.Sprintf("unknown block type %T", b))
   318  	}
   319  }
   320  
   321  func parseMarkdownFile(fsys fs.FS, path string) (*md.Document, error) {
   322  	f, err := fsys.Open(path)
   323  	if err != nil {
   324  		return nil, err
   325  	}
   326  	defer f.Close()
   327  	data, err := io.ReadAll(f)
   328  	if err != nil {
   329  		return nil, err
   330  	}
   331  	in := string(data)
   332  	doc := NewParser().Parse(in)
   333  	return doc, nil
   334  }
   335  
   336  // An APIFeature is a symbol mentioned in an API file,
   337  // like the ones in the main go repo in the api directory.
   338  type APIFeature struct {
   339  	Package string // package that the feature is in
   340  	Build   string // build that the symbol is relevant for (e.g. GOOS, GOARCH)
   341  	Feature string // everything about the feature other than the package
   342  	Issue   int    // the issue that introduced the feature, or 0 if none
   343  }
   344  
   345  // This regexp has four capturing groups: package, build, feature and issue.
   346  var apiFileLineRegexp = regexp.MustCompile(`^pkg ([^ \t]+)[ \t]*(\([^)]+\))?, ([^#]*)(#\d+)?$`)
   347  
   348  // parseAPIFile parses a file in the api format and returns a list of the file's features.
   349  // A feature is represented by a single line that looks like
   350  //
   351  //	pkg PKG (BUILD) FEATURE #ISSUE
   352  //
   353  // where the BUILD and ISSUE may be absent.
   354  func parseAPIFile(fsys fs.FS, filename string) ([]APIFeature, error) {
   355  	f, err := fsys.Open(filename)
   356  	if err != nil {
   357  		return nil, err
   358  	}
   359  	defer f.Close()
   360  	var features []APIFeature
   361  	scan := bufio.NewScanner(f)
   362  	for scan.Scan() {
   363  		line := strings.TrimSpace(scan.Text())
   364  		if line == "" || line[0] == '#' {
   365  			continue
   366  		}
   367  		matches := apiFileLineRegexp.FindStringSubmatch(line)
   368  		if len(matches) == 0 {
   369  			return nil, fmt.Errorf("%s: malformed line %q", filename, line)
   370  		}
   371  		if len(matches) != 5 {
   372  			return nil, fmt.Errorf("wrong number of matches for line %q", line)
   373  		}
   374  		f := APIFeature{
   375  			Package: matches[1],
   376  			Build:   matches[2],
   377  			Feature: strings.TrimSpace(matches[3]),
   378  		}
   379  		if issue := matches[4]; issue != "" {
   380  			var err error
   381  			f.Issue, err = strconv.Atoi(issue[1:]) // skip leading '#'
   382  			if err != nil {
   383  				return nil, err
   384  			}
   385  		}
   386  		features = append(features, f)
   387  	}
   388  	if scan.Err() != nil {
   389  		return nil, scan.Err()
   390  	}
   391  	return features, nil
   392  }
   393  
   394  // GroupAPIFeaturesByFile returns a map of the given features keyed by
   395  // the doc filename that they are associated with.
   396  // A feature with package P and issue N should be documented in the file
   397  // "P/N.md".
   398  func GroupAPIFeaturesByFile(fs []APIFeature) (map[string][]APIFeature, error) {
   399  	m := map[string][]APIFeature{}
   400  	for _, f := range fs {
   401  		if f.Issue == 0 {
   402  			return nil, fmt.Errorf("%+v: zero issue", f)
   403  		}
   404  		filename := fmt.Sprintf("%s/%d.md", f.Package, f.Issue)
   405  		m[filename] = append(m[filename], f)
   406  	}
   407  	return m, nil
   408  }
   409  
   410  // CheckAPIFile reads the api file at filename in apiFS, and checks the corresponding
   411  // release-note files under docFS. It checks that the files exist and that they have
   412  // some minimal content (see [CheckFragment]).
   413  // The docRoot argument is the path from the repo or project root to the root of docFS.
   414  // It is used only for error messages.
   415  func CheckAPIFile(apiFS fs.FS, filename string, docFS fs.FS, docRoot string) error {
   416  	features, err := parseAPIFile(apiFS, filename)
   417  	if err != nil {
   418  		return err
   419  	}
   420  	byFile, err := GroupAPIFeaturesByFile(features)
   421  	if err != nil {
   422  		return err
   423  	}
   424  	var filenames []string
   425  	for fn := range byFile {
   426  		filenames = append(filenames, fn)
   427  	}
   428  	slices.Sort(filenames)
   429  	mcDir, err := minorChangesDir(docFS)
   430  	if err != nil {
   431  		return err
   432  	}
   433  	var errs []error
   434  	for _, fn := range filenames {
   435  		// Use path.Join for consistency with io/fs pathnames.
   436  		fn = path.Join(mcDir, fn)
   437  		// TODO(jba): check that the file mentions each feature?
   438  		if err := checkFragmentFile(docFS, fn); err != nil {
   439  			errs = append(errs, fmt.Errorf("%s: %v\nSee doc/README.md for more information.", path.Join(docRoot, fn), err))
   440  		}
   441  	}
   442  	return errors.Join(errs...)
   443  }
   444  
   445  // minorChangesDir returns the unique directory in docFS that corresponds to the
   446  // "Minor changes to the standard library" section of the release notes.
   447  func minorChangesDir(docFS fs.FS) (string, error) {
   448  	dirs, err := fs.Glob(docFS, "*stdlib/*minor")
   449  	if err != nil {
   450  		return "", err
   451  	}
   452  	var bad string
   453  	if len(dirs) == 0 {
   454  		bad = "No"
   455  	} else if len(dirs) > 1 {
   456  		bad = "More than one"
   457  	}
   458  	if bad != "" {
   459  		return "", fmt.Errorf("%s directory matches *stdlib/*minor.\nThis shouldn't happen; please file a bug at https://go.dev/issues/new.",
   460  			bad)
   461  	}
   462  	return dirs[0], nil
   463  }
   464  
   465  func checkFragmentFile(fsys fs.FS, filename string) error {
   466  	f, err := fsys.Open(filename)
   467  	if err != nil {
   468  		if errors.Is(err, fs.ErrNotExist) {
   469  			err = errors.New("File does not exist. Every API change must have a corresponding release note file.")
   470  		}
   471  		return err
   472  	}
   473  	defer f.Close()
   474  	data, err := io.ReadAll(f)
   475  	if err != nil {
   476  		return err
   477  	}
   478  	return CheckFragment(string(data))
   479  }
   480
View as plain text