quote.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:generate go run makeisprint.go -output isprint.go
     6  
     7  package strconv
     8  
     9  import (
    10  	"unicode/utf8"
    11  )
    12  
    13  const (
    14  	lowerhex = "0123456789abcdef"
    15  	upperhex = "0123456789ABCDEF"
    16  )
    17  
    18  // contains reports whether the string contains the byte c.
    19  func contains(s string, c byte) bool {
    20  	return index(s, c) != -1
    21  }
    22  
    23  func quoteWith(s string, quote byte, ASCIIonly, graphicOnly bool) string {
    24  	return string(appendQuotedWith(make([]byte, 0, 3*len(s)/2), s, quote, ASCIIonly, graphicOnly))
    25  }
    26  
    27  func quoteRuneWith(r rune, quote byte, ASCIIonly, graphicOnly bool) string {
    28  	return string(appendQuotedRuneWith(nil, r, quote, ASCIIonly, graphicOnly))
    29  }
    30  
    31  func appendQuotedWith(buf []byte, s string, quote byte, ASCIIonly, graphicOnly bool) []byte {
    32  	// Often called with big strings, so preallocate. If there's quoting,
    33  	// this is conservative but still helps a lot.
    34  	if cap(buf)-len(buf) < len(s) {
    35  		nBuf := make([]byte, len(buf), len(buf)+1+len(s)+1)
    36  		copy(nBuf, buf)
    37  		buf = nBuf
    38  	}
    39  	buf = append(buf, quote)
    40  	for r, width := rune(0), 0; len(s) > 0; s = s[width:] {
    41  		r, width = utf8.DecodeRuneInString(s)
    42  		if width == 1 && r == utf8.RuneError {
    43  			buf = append(buf, `\x`...)
    44  			buf = append(buf, lowerhex[s[0]>>4])
    45  			buf = append(buf, lowerhex[s[0]&0xF])
    46  			continue
    47  		}
    48  		buf = appendEscapedRune(buf, r, quote, ASCIIonly, graphicOnly)
    49  	}
    50  	buf = append(buf, quote)
    51  	return buf
    52  }
    53  
    54  func appendQuotedRuneWith(buf []byte, r rune, quote byte, ASCIIonly, graphicOnly bool) []byte {
    55  	buf = append(buf, quote)
    56  	if !utf8.ValidRune(r) {
    57  		r = utf8.RuneError
    58  	}
    59  	buf = appendEscapedRune(buf, r, quote, ASCIIonly, graphicOnly)
    60  	buf = append(buf, quote)
    61  	return buf
    62  }
    63  
    64  func appendEscapedRune(buf []byte, r rune, quote byte, ASCIIonly, graphicOnly bool) []byte {
    65  	if r == rune(quote) || r == '\\' { // always backslashed
    66  		buf = append(buf, '\\')
    67  		buf = append(buf, byte(r))
    68  		return buf
    69  	}
    70  	if ASCIIonly {
    71  		if r < utf8.RuneSelf && IsPrint(r) {
    72  			buf = append(buf, byte(r))
    73  			return buf
    74  		}
    75  	} else if IsPrint(r) || graphicOnly && isInGraphicList(r) {
    76  		return utf8.AppendRune(buf, r)
    77  	}
    78  	switch r {
    79  	case '\a':
    80  		buf = append(buf, `\a`...)
    81  	case '\b':
    82  		buf = append(buf, `\b`...)
    83  	case '\f':
    84  		buf = append(buf, `\f`...)
    85  	case '\n':
    86  		buf = append(buf, `\n`...)
    87  	case '\r':
    88  		buf = append(buf, `\r`...)
    89  	case '\t':
    90  		buf = append(buf, `\t`...)
    91  	case '\v':
    92  		buf = append(buf, `\v`...)
    93  	default:
    94  		switch {
    95  		case r < ' ' || r == 0x7f:
    96  			buf = append(buf, `\x`...)
    97  			buf = append(buf, lowerhex[byte(r)>>4])
    98  			buf = append(buf, lowerhex[byte(r)&0xF])
    99  		case !utf8.ValidRune(r):
   100  			r = 0xFFFD
   101  			fallthrough
   102  		case r < 0x10000:
   103  			buf = append(buf, `\u`...)
   104  			for s := 12; s >= 0; s -= 4 {
   105  				buf = append(buf, lowerhex[r>>uint(s)&0xF])
   106  			}
   107  		default:
   108  			buf = append(buf, `\U`...)
   109  			for s := 28; s >= 0; s -= 4 {
   110  				buf = append(buf, lowerhex[r>>uint(s)&0xF])
   111  			}
   112  		}
   113  	}
   114  	return buf
   115  }
   116  
   117  // Quote returns a double-quoted Go string literal representing s. The
   118  // returned string uses Go escape sequences (\t, \n, \xFF, \u0100) for
   119  // control characters and non-printable characters as defined by
   120  // [IsPrint].
   121  func Quote(s string) string {
   122  	return quoteWith(s, '"', false, false)
   123  }
   124  
   125  // AppendQuote appends a double-quoted Go string literal representing s,
   126  // as generated by [Quote], to dst and returns the extended buffer.
   127  func AppendQuote(dst []byte, s string) []byte {
   128  	return appendQuotedWith(dst, s, '"', false, false)
   129  }
   130  
   131  // QuoteToASCII returns a double-quoted Go string literal representing s.
   132  // The returned string uses Go escape sequences (\t, \n, \xFF, \u0100) for
   133  // non-ASCII characters and non-printable characters as defined by [IsPrint].
   134  func QuoteToASCII(s string) string {
   135  	return quoteWith(s, '"', true, false)
   136  }
   137  
   138  // AppendQuoteToASCII appends a double-quoted Go string literal representing s,
   139  // as generated by [QuoteToASCII], to dst and returns the extended buffer.
   140  func AppendQuoteToASCII(dst []byte, s string) []byte {
   141  	return appendQuotedWith(dst, s, '"', true, false)
   142  }
   143  
   144  // QuoteToGraphic returns a double-quoted Go string literal representing s.
   145  // The returned string leaves Unicode graphic characters, as defined by
   146  // [IsGraphic], unchanged and uses Go escape sequences (\t, \n, \xFF, \u0100)
   147  // for non-graphic characters.
   148  func QuoteToGraphic(s string) string {
   149  	return quoteWith(s, '"', false, true)
   150  }
   151  
   152  // AppendQuoteToGraphic appends a double-quoted Go string literal representing s,
   153  // as generated by [QuoteToGraphic], to dst and returns the extended buffer.
   154  func AppendQuoteToGraphic(dst []byte, s string) []byte {
   155  	return appendQuotedWith(dst, s, '"', false, true)
   156  }
   157  
   158  // QuoteRune returns a single-quoted Go character literal representing the
   159  // rune. The returned string uses Go escape sequences (\t, \n, \xFF, \u0100)
   160  // for control characters and non-printable characters as defined by [IsPrint].
   161  // If r is not a valid Unicode code point, it is interpreted as the Unicode
   162  // replacement character U+FFFD.
   163  func QuoteRune(r rune) string {
   164  	return quoteRuneWith(r, '\'', false, false)
   165  }
   166  
   167  // AppendQuoteRune appends a single-quoted Go character literal representing the rune,
   168  // as generated by [QuoteRune], to dst and returns the extended buffer.
   169  func AppendQuoteRune(dst []byte, r rune) []byte {
   170  	return appendQuotedRuneWith(dst, r, '\'', false, false)
   171  }
   172  
   173  // QuoteRuneToASCII returns a single-quoted Go character literal representing
   174  // the rune. The returned string uses Go escape sequences (\t, \n, \xFF,
   175  // \u0100) for non-ASCII characters and non-printable characters as defined
   176  // by [IsPrint].
   177  // If r is not a valid Unicode code point, it is interpreted as the Unicode
   178  // replacement character U+FFFD.
   179  func QuoteRuneToASCII(r rune) string {
   180  	return quoteRuneWith(r, '\'', true, false)
   181  }
   182  
   183  // AppendQuoteRuneToASCII appends a single-quoted Go character literal representing the rune,
   184  // as generated by [QuoteRuneToASCII], to dst and returns the extended buffer.
   185  func AppendQuoteRuneToASCII(dst []byte, r rune) []byte {
   186  	return appendQuotedRuneWith(dst, r, '\'', true, false)
   187  }
   188  
   189  // QuoteRuneToGraphic returns a single-quoted Go character literal representing
   190  // the rune. If the rune is not a Unicode graphic character,
   191  // as defined by [IsGraphic], the returned string will use a Go escape sequence
   192  // (\t, \n, \xFF, \u0100).
   193  // If r is not a valid Unicode code point, it is interpreted as the Unicode
   194  // replacement character U+FFFD.
   195  func QuoteRuneToGraphic(r rune) string {
   196  	return quoteRuneWith(r, '\'', false, true)
   197  }
   198  
   199  // AppendQuoteRuneToGraphic appends a single-quoted Go character literal representing the rune,
   200  // as generated by [QuoteRuneToGraphic], to dst and returns the extended buffer.
   201  func AppendQuoteRuneToGraphic(dst []byte, r rune) []byte {
   202  	return appendQuotedRuneWith(dst, r, '\'', false, true)
   203  }
   204  
   205  // CanBackquote reports whether the string s can be represented
   206  // unchanged as a single-line backquoted string without control
   207  // characters other than tab.
   208  func CanBackquote(s string) bool {
   209  	for len(s) > 0 {
   210  		r, wid := utf8.DecodeRuneInString(s)
   211  		s = s[wid:]
   212  		if wid > 1 {
   213  			if r == '\ufeff' {
   214  				return false // BOMs are invisible and should not be quoted.
   215  			}
   216  			continue // All other multibyte runes are correctly encoded and assumed printable.
   217  		}
   218  		if r == utf8.RuneError {
   219  			return false
   220  		}
   221  		if (r < ' ' && r != '\t') || r == '`' || r == '\u007F' {
   222  			return false
   223  		}
   224  	}
   225  	return true
   226  }
   227  
   228  func unhex(b byte) (v rune, ok bool) {
   229  	c := rune(b)
   230  	switch {
   231  	case '0' <= c && c <= '9':
   232  		return c - '0', true
   233  	case 'a' <= c && c <= 'f':
   234  		return c - 'a' + 10, true
   235  	case 'A' <= c && c <= 'F':
   236  		return c - 'A' + 10, true
   237  	}
   238  	return
   239  }
   240  
   241  // UnquoteChar decodes the first character or byte in the escaped string
   242  // or character literal represented by the string s.
   243  // It returns four values:
   244  //
   245  //  1. value, the decoded Unicode code point or byte value;
   246  //  2. multibyte, a boolean indicating whether the decoded character requires a multibyte UTF-8 representation;
   247  //  3. tail, the remainder of the string after the character; and
   248  //  4. an error that will be nil if the character is syntactically valid.
   249  //
   250  // The second argument, quote, specifies the type of literal being parsed
   251  // and therefore which escaped quote character is permitted.
   252  // If set to a single quote, it permits the sequence \' and disallows unescaped '.
   253  // If set to a double quote, it permits \" and disallows unescaped ".
   254  // If set to zero, it does not permit either escape and allows both quote characters to appear unescaped.
   255  func UnquoteChar(s string, quote byte) (value rune, multibyte bool, tail string, err error) {
   256  	// easy cases
   257  	if len(s) == 0 {
   258  		err = ErrSyntax
   259  		return
   260  	}
   261  	switch c := s[0]; {
   262  	case c == quote && (quote == '\'' || quote == '"'):
   263  		err = ErrSyntax
   264  		return
   265  	case c >= utf8.RuneSelf:
   266  		r, size := utf8.DecodeRuneInString(s)
   267  		return r, true, s[size:], nil
   268  	case c != '\\':
   269  		return rune(s[0]), false, s[1:], nil
   270  	}
   271  
   272  	// hard case: c is backslash
   273  	if len(s) <= 1 {
   274  		err = ErrSyntax
   275  		return
   276  	}
   277  	c := s[1]
   278  	s = s[2:]
   279  
   280  	switch c {
   281  	case 'a':
   282  		value = '\a'
   283  	case 'b':
   284  		value = '\b'
   285  	case 'f':
   286  		value = '\f'
   287  	case 'n':
   288  		value = '\n'
   289  	case 'r':
   290  		value = '\r'
   291  	case 't':
   292  		value = '\t'
   293  	case 'v':
   294  		value = '\v'
   295  	case 'x', 'u', 'U':
   296  		n := 0
   297  		switch c {
   298  		case 'x':
   299  			n = 2
   300  		case 'u':
   301  			n = 4
   302  		case 'U':
   303  			n = 8
   304  		}
   305  		var v rune
   306  		if len(s) < n {
   307  			err = ErrSyntax
   308  			return
   309  		}
   310  		for j := 0; j < n; j++ {
   311  			x, ok := unhex(s[j])
   312  			if !ok {
   313  				err = ErrSyntax
   314  				return
   315  			}
   316  			v = v<<4 | x
   317  		}
   318  		s = s[n:]
   319  		if c == 'x' {
   320  			// byte value in a string literal (possibly not UTF-8);
   321  			// or code point in a rune literal.
   322  			value = v
   323  			break
   324  		}
   325  		if !utf8.ValidRune(v) {
   326  			err = ErrSyntax
   327  			return
   328  		}
   329  		value = v
   330  		multibyte = true
   331  	case '0', '1', '2', '3', '4', '5', '6', '7':
   332  		v := rune(c) - '0'
   333  		if len(s) < 2 {
   334  			err = ErrSyntax
   335  			return
   336  		}
   337  		for j := 0; j < 2; j++ { // one digit already; two more
   338  			x := rune(s[j]) - '0'
   339  			if x < 0 || x > 7 {
   340  				err = ErrSyntax
   341  				return
   342  			}
   343  			v = (v << 3) | x
   344  		}
   345  		s = s[2:]
   346  		if v > 255 {
   347  			err = ErrSyntax
   348  			return
   349  		}
   350  		value = v
   351  	case '\\':
   352  		value = '\\'
   353  	case '\'', '"':
   354  		if c != quote {
   355  			err = ErrSyntax
   356  			return
   357  		}
   358  		value = rune(c)
   359  	default:
   360  		err = ErrSyntax
   361  		return
   362  	}
   363  	tail = s
   364  	return
   365  }
   366  
   367  // QuotedPrefix returns the quoted string (as understood by [Unquote]) at the prefix of s.
   368  // If s does not start with a valid quoted string, QuotedPrefix returns an error.
   369  func QuotedPrefix(s string) (string, error) {
   370  	out, _, err := unquote(s, false)
   371  	return out, err
   372  }
   373  
   374  // Unquote interprets s as a single-quoted, double-quoted,
   375  // or backquoted Go string literal, returning the string value
   376  // that s quotes.  (If s is single-quoted, it would be a Go
   377  // character literal; Unquote returns the corresponding
   378  // one-character string. For an empty character literal
   379  // Unquote returns the empty string.)
   380  func Unquote(s string) (string, error) {
   381  	out, rem, err := unquote(s, true)
   382  	if len(rem) > 0 {
   383  		return "", ErrSyntax
   384  	}
   385  	return out, err
   386  }
   387  
   388  // unquote parses a quoted string at the start of the input,
   389  // returning the parsed prefix, the remaining suffix, and any parse errors.
   390  // If unescape is true, the parsed prefix is unescaped,
   391  // otherwise the input prefix is provided verbatim.
   392  func unquote(in string, unescape bool) (out, rem string, err error) {
   393  	// Determine the quote form and optimistically find the terminating quote.
   394  	if len(in) < 2 {
   395  		return "", in, ErrSyntax
   396  	}
   397  	quote := in[0]
   398  	end := index(in[1:], quote)
   399  	if end < 0 {
   400  		return "", in, ErrSyntax
   401  	}
   402  	end += 2 // position after terminating quote; may be wrong if escape sequences are present
   403  
   404  	switch quote {
   405  	case '`':
   406  		switch {
   407  		case !unescape:
   408  			out = in[:end] // include quotes
   409  		case !contains(in[:end], '\r'):
   410  			out = in[len("`") : end-len("`")] // exclude quotes
   411  		default:
   412  			// Carriage return characters ('\r') inside raw string literals
   413  			// are discarded from the raw string value.
   414  			buf := make([]byte, 0, end-len("`")-len("\r")-len("`"))
   415  			for i := len("`"); i < end-len("`"); i++ {
   416  				if in[i] != '\r' {
   417  					buf = append(buf, in[i])
   418  				}
   419  			}
   420  			out = string(buf)
   421  		}
   422  		// NOTE: Prior implementations did not verify that raw strings consist
   423  		// of valid UTF-8 characters and we continue to not verify it as such.
   424  		// The Go specification does not explicitly require valid UTF-8,
   425  		// but only mention that it is implicitly valid for Go source code
   426  		// (which must be valid UTF-8).
   427  		return out, in[end:], nil
   428  	case '"', '\'':
   429  		// Handle quoted strings without any escape sequences.
   430  		if !contains(in[:end], '\\') && !contains(in[:end], '\n') {
   431  			var valid bool
   432  			switch quote {
   433  			case '"':
   434  				valid = utf8.ValidString(in[len(`"`) : end-len(`"`)])
   435  			case '\'':
   436  				r, n := utf8.DecodeRuneInString(in[len("'") : end-len("'")])
   437  				valid = len("'")+n+len("'") == end && (r != utf8.RuneError || n != 1)
   438  			}
   439  			if valid {
   440  				out = in[:end]
   441  				if unescape {
   442  					out = out[1 : end-1] // exclude quotes
   443  				}
   444  				return out, in[end:], nil
   445  			}
   446  		}
   447  
   448  		// Handle quoted strings with escape sequences.
   449  		var buf []byte
   450  		in0 := in
   451  		in = in[1:] // skip starting quote
   452  		if unescape {
   453  			buf = make([]byte, 0, 3*end/2) // try to avoid more allocations
   454  		}
   455  		for len(in) > 0 && in[0] != quote {
   456  			// Process the next character,
   457  			// rejecting any unescaped newline characters which are invalid.
   458  			r, multibyte, rem, err := UnquoteChar(in, quote)
   459  			if in[0] == '\n' || err != nil {
   460  				return "", in0, ErrSyntax
   461  			}
   462  			in = rem
   463  
   464  			// Append the character if unescaping the input.
   465  			if unescape {
   466  				// As a rune literal, a hex or octal escape denotes
   467  				// a code point ('\xff' = U+00FF), so it must be
   468  				// UTF-8 encoded like any other rune.
   469  				// Within a string literal, such escapes
   470  				// denote individual bytes.
   471  				if r < utf8.RuneSelf ||
   472  					!multibyte && quote == '"' {
   473  					buf = append(buf, byte(r))
   474  				} else {
   475  					buf = utf8.AppendRune(buf, r)
   476  				}
   477  			}
   478  
   479  			// Single quoted strings must be a single character.
   480  			if quote == '\'' {
   481  				break
   482  			}
   483  		}
   484  
   485  		// Verify that the string ends with a terminating quote.
   486  		if !(len(in) > 0 && in[0] == quote) {
   487  			return "", in0, ErrSyntax
   488  		}
   489  		in = in[1:] // skip terminating quote
   490  
   491  		if unescape {
   492  			return string(buf), in, nil
   493  		}
   494  		return in0[:len(in0)-len(in)], in, nil
   495  	default:
   496  		return "", in, ErrSyntax
   497  	}
   498  }
   499  
   500  // bsearch is semantically the same as [slices.BinarySearch] (without NaN checks)
   501  // We copied this function because we can not import "slices" here.
   502  func bsearch[S ~[]E, E ~uint16 | ~uint32](s S, v E) (int, bool) {
   503  	n := len(s)
   504  	i, j := 0, n
   505  	for i < j {
   506  		h := i + (j-i)>>1
   507  		if s[h] < v {
   508  			i = h + 1
   509  		} else {
   510  			j = h
   511  		}
   512  	}
   513  	return i, i < n && s[i] == v
   514  }
   515  
   516  // TODO: IsPrint is a local implementation of unicode.IsPrint, verified by the tests
   517  // to give the same answer. It allows this package not to depend on unicode,
   518  // and therefore not pull in all the Unicode tables. If the linker were better
   519  // at tossing unused tables, we could get rid of this implementation.
   520  // That would be nice.
   521  
   522  // IsPrint reports whether the rune is defined as printable by Go, with
   523  // the same definition as [unicode.IsPrint]: letters, numbers, punctuation,
   524  // symbols and ASCII space.
   525  func IsPrint(r rune) bool {
   526  	// Fast check for Latin-1
   527  	if r <= 0xFF {
   528  		if 0x20 <= r && r <= 0x7E {
   529  			// All the ASCII is printable from space through DEL-1.
   530  			return true
   531  		}
   532  		if 0xA1 <= r && r <= 0xFF {
   533  			// Similarly for ¡ through ÿ...
   534  			return r != 0xAD // ...except for the bizarre soft hyphen.
   535  		}
   536  		return false
   537  	}
   538  
   539  	// Same algorithm, either on uint16 or uint32 value.
   540  	// First, find first i such that isPrint[i] >= x.
   541  	// This is the index of either the start or end of a pair that might span x.
   542  	// The start is even (isPrint[i&^1]) and the end is odd (isPrint[i|1]).
   543  	// If we find x in a range, make sure x is not in isNotPrint list.
   544  
   545  	if 0 <= r && r < 1<<16 {
   546  		rr, isPrint, isNotPrint := uint16(r), isPrint16, isNotPrint16
   547  		i, _ := bsearch(isPrint, rr)
   548  		if i >= len(isPrint) || rr < isPrint[i&^1] || isPrint[i|1] < rr {
   549  			return false
   550  		}
   551  		_, found := bsearch(isNotPrint, rr)
   552  		return !found
   553  	}
   554  
   555  	rr, isPrint, isNotPrint := uint32(r), isPrint32, isNotPrint32
   556  	i, _ := bsearch(isPrint, rr)
   557  	if i >= len(isPrint) || rr < isPrint[i&^1] || isPrint[i|1] < rr {
   558  		return false
   559  	}
   560  	if r >= 0x20000 {
   561  		return true
   562  	}
   563  	r -= 0x10000
   564  	_, found := bsearch(isNotPrint, uint16(r))
   565  	return !found
   566  }
   567  
   568  // IsGraphic reports whether the rune is defined as a Graphic by Unicode. Such
   569  // characters include letters, marks, numbers, punctuation, symbols, and
   570  // spaces, from categories L, M, N, P, S, and Zs.
   571  func IsGraphic(r rune) bool {
   572  	if IsPrint(r) {
   573  		return true
   574  	}
   575  	return isInGraphicList(r)
   576  }
   577  
   578  // isInGraphicList reports whether the rune is in the isGraphic list. This separation
   579  // from IsGraphic allows quoteWith to avoid two calls to IsPrint.
   580  // Should be called only if IsPrint fails.
   581  func isInGraphicList(r rune) bool {
   582  	// We know r must fit in 16 bits - see makeisprint.go.
   583  	if r > 0xFFFF {
   584  		return false
   585  	}
   586  	_, found := bsearch(isGraphic, uint16(r))
   587  	return found
   588  }
   589
View as plain text