Source file src/encoding/json/v2_diff_test.go

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build goexperiment.jsonv2
     6  
     7  package json_test
     8  
     9  import (
    10  	"errors"
    11  	"path"
    12  	"reflect"
    13  	"strings"
    14  	"testing"
    15  	"time"
    16  
    17  	jsonv1 "encoding/json"
    18  	"encoding/json/jsontext"
    19  	jsonv2 "encoding/json/v2"
    20  )
    21  
    22  // NOTE: This file serves as a list of semantic differences between v1 and v2.
    23  // Each test explains how v1 behaves, how v2 behaves, and
    24  // a rationale for why the behavior was changed.
    25  
    26  var jsonPackages = []struct {
    27  	Version   string
    28  	Marshal   func(any) ([]byte, error)
    29  	Unmarshal func([]byte, any) error
    30  }{
    31  	{"v1", jsonv1.Marshal, jsonv1.Unmarshal},
    32  	{"v2",
    33  		func(in any) ([]byte, error) { return jsonv2.Marshal(in) },
    34  		func(in []byte, out any) error { return jsonv2.Unmarshal(in, out) }},
    35  }
    36  
    37  // In v1, unmarshal matches struct fields using a case-insensitive match.
    38  // In v2, unmarshal matches struct fields using a case-sensitive match.
    39  //
    40  // Case-insensitive matching is a surprising default and
    41  // incurs significant performance cost when unmarshaling unknown fields.
    42  // In v2, we can opt into v1-like behavior with the `case:ignore` tag option.
    43  // The case-insensitive matching performed by v2 is looser than that of v1
    44  // where it also ignores dashes and underscores.
    45  // This allows v2 to match fields regardless of whether the name is in
    46  // snake_case, camelCase, or kebab-case.
    47  //
    48  // Related issue:
    49  //
    50  //	https://go.dev/issue/14750
    51  func TestCaseSensitivity(t *testing.T) {
    52  	type Fields struct {
    53  		FieldA bool
    54  		FieldB bool `json:"fooBar"`
    55  		FieldC bool `json:"fizzBuzz,case:ignore"` // `case:ignore` is used by v2 to explicitly enable case-insensitive matching
    56  	}
    57  
    58  	for _, json := range jsonPackages {
    59  		t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) {
    60  			// This is a mapping from Go field names to JSON member names to
    61  			// whether the JSON member name would match the Go field name.
    62  			type goName = string
    63  			type jsonName = string
    64  			onlyV1 := json.Version == "v1"
    65  			onlyV2 := json.Version == "v2"
    66  			allMatches := map[goName]map[jsonName]bool{
    67  				"FieldA": {
    68  					"FieldA": true,   // exact match
    69  					"fielda": onlyV1, // v1 is case-insensitive by default
    70  					"fieldA": onlyV1, // v1 is case-insensitive by default
    71  					"FIELDA": onlyV1, // v1 is case-insensitive by default
    72  					"FieldB": false,
    73  					"FieldC": false,
    74  				},
    75  				"FieldB": {
    76  					"fooBar":   true,   // exact match for explicitly specified JSON name
    77  					"FooBar":   onlyV1, // v1 is case-insensitive even if an explicit JSON name is provided
    78  					"foobar":   onlyV1, // v1 is case-insensitive even if an explicit JSON name is provided
    79  					"FOOBAR":   onlyV1, // v1 is case-insensitive even if an explicit JSON name is provided
    80  					"fizzBuzz": false,
    81  					"FieldA":   false,
    82  					"FieldB":   false, // explicit JSON name means that the Go field name is not used for matching
    83  					"FieldC":   false,
    84  				},
    85  				"FieldC": {
    86  					"fizzBuzz":  true,   // exact match for explicitly specified JSON name
    87  					"fizzbuzz":  true,   // v2 is case-insensitive due to `case:ignore` tag
    88  					"FIZZBUZZ":  true,   // v2 is case-insensitive due to `case:ignore` tag
    89  					"fizz_buzz": onlyV2, // case-insensitivity in v2 ignores dashes and underscores
    90  					"fizz-buzz": onlyV2, // case-insensitivity in v2 ignores dashes and underscores
    91  					"fooBar":    false,
    92  					"FieldA":    false,
    93  					"FieldC":    false, // explicit JSON name means that the Go field name is not used for matching
    94  					"FieldB":    false,
    95  				},
    96  			}
    97  
    98  			for goFieldName, matches := range allMatches {
    99  				for jsonMemberName, wantMatch := range matches {
   100  					in := `{"` + jsonMemberName + `":true}`
   101  					var s Fields
   102  					if err := json.Unmarshal([]byte(in), &s); err != nil {
   103  						t.Fatalf("json.Unmarshal error: %v", err)
   104  					}
   105  					gotMatch := reflect.ValueOf(s).FieldByName(goFieldName).Bool()
   106  					if gotMatch != wantMatch {
   107  						t.Fatalf("%T.%s = %v, want %v", s, goFieldName, gotMatch, wantMatch)
   108  					}
   109  				}
   110  			}
   111  		})
   112  	}
   113  }
   114  
   115  // In v1, the "omitempty" option specifies that a struct field is omitted
   116  // when marshaling if it is an empty Go value, which is defined as
   117  // false, 0, a nil pointer, a nil interface value, and
   118  // any empty array, slice, map, or string.
   119  //
   120  // In v2, the "omitempty" option specifies that a struct field is omitted
   121  // when marshaling if it is an empty JSON value, which is defined as
   122  // a JSON null or empty JSON string, object, or array.
   123  //
   124  // In v2, we also provide the "omitzero" option which specifies that a field
   125  // is omitted if it is the zero Go value or if it implements an "IsZero() bool"
   126  // method that reports true. Together, "omitzero" and "omitempty" can cover
   127  // all the prior use cases of the v1 definition of "omitempty".
   128  // Note that "omitempty" is defined in terms of the Go type system in v1,
   129  // but now defined in terms of the JSON type system in v2.
   130  //
   131  // Related issues:
   132  //
   133  //	https://go.dev/issue/11939
   134  //	https://go.dev/issue/22480
   135  //	https://go.dev/issue/29310
   136  //	https://go.dev/issue/32675
   137  //	https://go.dev/issue/45669
   138  //	https://go.dev/issue/45787
   139  //	https://go.dev/issue/50480
   140  //	https://go.dev/issue/52803
   141  func TestOmitEmptyOption(t *testing.T) {
   142  	type Struct struct {
   143  		Foo string  `json:",omitempty"`
   144  		Bar []int   `json:",omitempty"`
   145  		Baz *Struct `json:",omitempty"`
   146  	}
   147  	type Types struct {
   148  		Bool       bool              `json:",omitempty"`
   149  		StringA    string            `json:",omitempty"`
   150  		StringB    string            `json:",omitempty"`
   151  		BytesA     []byte            `json:",omitempty"`
   152  		BytesB     []byte            `json:",omitempty"`
   153  		BytesC     []byte            `json:",omitempty"`
   154  		Int        int               `json:",omitempty"`
   155  		MapA       map[string]string `json:",omitempty"`
   156  		MapB       map[string]string `json:",omitempty"`
   157  		MapC       map[string]string `json:",omitempty"`
   158  		StructA    Struct            `json:",omitempty"`
   159  		StructB    Struct            `json:",omitempty"`
   160  		StructC    Struct            `json:",omitempty"`
   161  		SliceA     []string          `json:",omitempty"`
   162  		SliceB     []string          `json:",omitempty"`
   163  		SliceC     []string          `json:",omitempty"`
   164  		Array      [1]string         `json:",omitempty"`
   165  		PointerA   *string           `json:",omitempty"`
   166  		PointerB   *string           `json:",omitempty"`
   167  		PointerC   *string           `json:",omitempty"`
   168  		InterfaceA any               `json:",omitempty"`
   169  		InterfaceB any               `json:",omitempty"`
   170  		InterfaceC any               `json:",omitempty"`
   171  		InterfaceD any               `json:",omitempty"`
   172  	}
   173  
   174  	something := "something"
   175  	for _, json := range jsonPackages {
   176  		t.Run(path.Join("Marshal", json.Version), func(t *testing.T) {
   177  			in := Types{
   178  				Bool:       false,
   179  				StringA:    "",
   180  				StringB:    something,
   181  				BytesA:     nil,
   182  				BytesB:     []byte{},
   183  				BytesC:     []byte(something),
   184  				Int:        0,
   185  				MapA:       nil,
   186  				MapB:       map[string]string{},
   187  				MapC:       map[string]string{something: something},
   188  				StructA:    Struct{},
   189  				StructB:    Struct{Bar: []int{}, Baz: new(Struct)},
   190  				StructC:    Struct{Foo: something},
   191  				SliceA:     nil,
   192  				SliceB:     []string{},
   193  				SliceC:     []string{something},
   194  				Array:      [1]string{something},
   195  				PointerA:   nil,
   196  				PointerB:   new(string),
   197  				PointerC:   &something,
   198  				InterfaceA: nil,
   199  				InterfaceB: (*string)(nil),
   200  				InterfaceC: new(string),
   201  				InterfaceD: &something,
   202  			}
   203  			b, err := json.Marshal(in)
   204  			if err != nil {
   205  				t.Fatalf("json.Marshal error: %v", err)
   206  			}
   207  			var out map[string]any
   208  			if err := json.Unmarshal(b, &out); err != nil {
   209  				t.Fatalf("json.Unmarshal error: %v", err)
   210  			}
   211  
   212  			onlyV1 := json.Version == "v1"
   213  			onlyV2 := json.Version == "v2"
   214  			wantPresent := map[string]bool{
   215  				"Bool":       onlyV2, // false is an empty Go bool, but is NOT an empty JSON value
   216  				"StringA":    false,
   217  				"StringB":    true,
   218  				"BytesA":     false,
   219  				"BytesB":     false,
   220  				"BytesC":     true,
   221  				"Int":        onlyV2, // 0 is an empty Go integer, but NOT an empty JSON value
   222  				"MapA":       false,
   223  				"MapB":       false,
   224  				"MapC":       true,
   225  				"StructA":    onlyV1, // Struct{} is NOT an empty Go value, but {} is an empty JSON value
   226  				"StructB":    onlyV1, // Struct{...} is NOT an empty Go value, but {} is an empty JSON value
   227  				"StructC":    true,
   228  				"SliceA":     false,
   229  				"SliceB":     false,
   230  				"SliceC":     true,
   231  				"Array":      true,
   232  				"PointerA":   false,
   233  				"PointerB":   onlyV1, // new(string) is NOT a nil Go pointer, but "" is an empty JSON value
   234  				"PointerC":   true,
   235  				"InterfaceA": false,
   236  				"InterfaceB": onlyV1, // (*string)(nil) is NOT a nil Go interface, but null is an empty JSON value
   237  				"InterfaceC": onlyV1, // new(string) is NOT a nil Go interface, but "" is an empty JSON value
   238  				"InterfaceD": true,
   239  			}
   240  			for field, want := range wantPresent {
   241  				_, got := out[field]
   242  				if got != want {
   243  					t.Fatalf("%T.%s = %v, want %v", in, field, got, want)
   244  				}
   245  			}
   246  		})
   247  	}
   248  }
   249  
   250  func addr[T any](v T) *T {
   251  	return &v
   252  }
   253  
   254  // In v1, the "string" option specifies that Go strings, bools, and numeric
   255  // values are encoded within a JSON string when marshaling and
   256  // are unmarshaled from its native representation escaped within a JSON string.
   257  // The "string" option is not applied recursively, and so does not affect
   258  // strings, bools, and numeric values within a Go slice or map, but
   259  // does have special handling to affect the underlying value within a pointer.
   260  // If the "string" option is present on an unsupported type, it is simply ignored.
   261  // When unmarshaling, the "string" option permits decoding from a JSON null
   262  // escaped within a JSON string in some inconsistent cases.
   263  //
   264  // In v2, the "string" option specifies that only numeric values are encoded as
   265  // a JSON number within a JSON string when marshaling and are unmarshaled
   266  // from either a JSON number or a JSON string containing a JSON number.
   267  // The "string" option is still not applied recursively, and so does not affect
   268  // within a Go slice or map, but it retains special handling to affect the
   269  // underlying value within a pointer.
   270  // If the "string" option is present on an unsupported type, a runtime error is
   271  // reported.
   272  // There is no support for escaped JSON nulls within a JSON string.
   273  //
   274  // The main utility for stringifying JSON numbers is because JSON parsers
   275  // often represents numbers as IEEE 754 floating-point numbers.
   276  // This results in a loss of precision representing 64-bit integer values.
   277  // Consequently, many JSON-based APIs actually requires that such values
   278  // be encoded within a JSON string. Since the main utility of stringification
   279  // is for numeric values, v2 limits the effect of the "string" option
   280  // to just numeric Go types. According to all code known by the Go module proxy,
   281  // there are close to zero usages of the "string" option on a Go string or bool.
   282  //
   283  // The ability to decode from a JSON null wrapped within a JSON string
   284  // is removed in v2 because this behavior was surprising and inconsistent in v1.
   285  //
   286  // Related issues:
   287  //
   288  //	https://go.dev/issue/15624
   289  //	https://go.dev/issue/20651
   290  //	https://go.dev/issue/22177
   291  //	https://go.dev/issue/32055
   292  //	https://go.dev/issue/32117
   293  //	https://go.dev/issue/50997
   294  //	https://go.dev/issue/79065
   295  func TestStringOption(t *testing.T) {
   296  	type AllTypes struct {
   297  		String     string              `json:",string"`
   298  		Bool       bool                `json:",string"`
   299  		Int        int                 `json:",string"`
   300  		Float      float64             `json:",string"`
   301  		Map        map[string]int      `json:",string"`
   302  		Struct     struct{ Field int } `json:",string"`
   303  		Slice      []int               `json:",string"`
   304  		Array      [1]int              `json:",string"`
   305  		PointerA   *int                `json:",string"`
   306  		PointerB   *int                `json:",string"`
   307  		PointerC   **int               `json:",string"`
   308  		InterfaceA any                 `json:",string"`
   309  		InterfaceB any                 `json:",string"`
   310  	}
   311  
   312  	type V2Types struct {
   313  		Int      int     `json:",string"`
   314  		Float    float64 `json:",string"`
   315  		PointerA *int    `json:",string"`
   316  	}
   317  
   318  	for _, json := range jsonPackages {
   319  		t.Run(path.Join("Marshal", json.Version), func(t *testing.T) {
   320  			in := AllTypes{
   321  				String:     "string",
   322  				Bool:       true,
   323  				Int:        1,
   324  				Float:      1,
   325  				Map:        map[string]int{"Name": 1},
   326  				Struct:     struct{ Field int }{1},
   327  				Slice:      []int{1},
   328  				Array:      [1]int{1},
   329  				PointerA:   nil,
   330  				PointerB:   addr(1),
   331  				PointerC:   addr(addr(1)),
   332  				InterfaceA: nil,
   333  				InterfaceB: 1,
   334  			}
   335  			quote := func(s string) string {
   336  				b, _ := jsontext.AppendQuote(nil, s)
   337  				return string(b)
   338  			}
   339  			quoteOnlyV1 := func(s string) string {
   340  				if json.Version == "v1" {
   341  					s = quote(s)
   342  				}
   343  				return s
   344  			}
   345  			quoteOnlyV2 := func(s string) string {
   346  				if json.Version == "v2" {
   347  					s = quote(s)
   348  				}
   349  				return s
   350  			}
   351  			want := strings.Join([]string{
   352  				`{`,
   353  				`"String":` + quoteOnlyV1(`"string"`) + `,`, // in v1, Go strings are also stringified
   354  				`"Bool":` + quoteOnlyV1("true") + `,`,       // in v1, Go bools are also stringified
   355  				`"Int":` + quote("1") + `,`,
   356  				`"Float":` + quote("1") + `,`,
   357  				`"Map":{"Name":1},`,     // no recursive stringification
   358  				`"Struct":{"Field":1},`, // no recursive stringification
   359  				`"Slice":[1],`,          // no recursive stringification
   360  				`"Array":[1],`,          // no recursive stringification
   361  				`"PointerA":null,`,
   362  				`"PointerB":` + quote("1") + `,`,       // numbers are stringified after a single pointer indirection
   363  				`"PointerC":` + quoteOnlyV2("1") + `,`, // in v2, numbers are stringified through all pointer indirections
   364  				`"InterfaceA":null,`,
   365  				`"InterfaceB":` + quoteOnlyV2("1"), // in v2, numbers are stringified through all interface indirections
   366  				`}`}, "")
   367  			var got []byte
   368  			var err error
   369  			if json.Version == "v2" {
   370  				// Suppress type errors in v2,
   371  				// so we can compare the effects regardless of type errors.
   372  				got, err = jsonv2.Marshal(in, jsonv1.ReportErrorsWithLegacySemantics(true))
   373  			} else {
   374  				got, err = json.Marshal(in)
   375  			}
   376  			if err != nil {
   377  				t.Fatalf("json.Marshal error: %v", err)
   378  			}
   379  			if string(got) != want {
   380  				t.Fatalf("json.Marshal = %s, want %s", got, want)
   381  			}
   382  		})
   383  	}
   384  
   385  	for _, json := range jsonPackages {
   386  		t.Run(path.Join("Unmarshal/Null", json.Version), func(t *testing.T) {
   387  			var got AllTypes
   388  			err := json.Unmarshal([]byte(`{
   389  				"Bool":     "null",
   390  				"Int":      "null",
   391  				"PointerA": "null"
   392  			}`), &got)
   393  			switch {
   394  			case json.Version == "v1" && err != nil:
   395  				t.Fatalf("json.Unmarshal error: %v", err)
   396  			case json.Version == "v2" && err == nil:
   397  				t.Fatal("json.Unmarshal error is nil, want non-nil")
   398  			case !reflect.DeepEqual(got, AllTypes{}):
   399  				t.Fatalf("json.Unmarshal = %+v, want %+v", got, AllTypes{})
   400  			}
   401  		})
   402  
   403  		t.Run(path.Join("Unmarshal/Bool", json.Version), func(t *testing.T) {
   404  			var got AllTypes
   405  			want := map[string]AllTypes{
   406  				"v1": {Bool: true},
   407  				"v2": {Bool: false},
   408  			}[json.Version]
   409  			err := json.Unmarshal([]byte(`{"Bool": "true"}`), &got)
   410  			switch {
   411  			case json.Version == "v1" && err != nil:
   412  				t.Fatalf("json.Unmarshal error: %v", err)
   413  			case json.Version == "v2" && err == nil:
   414  				t.Fatal("json.Unmarshal error is nil, want non-nil")
   415  			case !reflect.DeepEqual(got, want):
   416  				t.Fatalf("json.Unmarshal = %v, want %v", got, want)
   417  			}
   418  		})
   419  
   420  		t.Run(path.Join("Unmarshal/Shallow", json.Version), func(t *testing.T) {
   421  			var got V2Types
   422  			want := V2Types{Int: 1, PointerA: addr(1)}
   423  			err := json.Unmarshal([]byte(`{
   424  				"Int":      "1",
   425  				"PointerA": "1"
   426  			}`), &got)
   427  			switch {
   428  			case err != nil:
   429  				t.Fatalf("json.Unmarshal error: %v", err)
   430  			case !reflect.DeepEqual(got, want):
   431  				t.Fatalf("json.Unmarshal =\n%+v, want\n%+v", got, want)
   432  			}
   433  		})
   434  	}
   435  }
   436  
   437  // In v1, nil slices and maps are marshaled as a JSON null.
   438  // In v2, nil slices and maps are marshaled as an empty JSON object or array.
   439  //
   440  // Users of v2 can opt into the v1 behavior by setting the
   441  // [jsonv2.FormatNilSliceAsNull] and [jsonv2.FormatNilMapAsNull] options.
   442  //
   443  // JSON is a language-agnostic data interchange format.
   444  // The fact that maps and slices are nil-able in Go is a semantic detail of the
   445  // Go language. We should avoid leaking such details to the JSON representation.
   446  // When JSON implementations leak language-specific details,
   447  // it complicates transition to/from languages with different type systems.
   448  //
   449  // Furthermore, consider two related Go types: string and []byte.
   450  // It's an asymmetric oddity of v1 that zero values of string and []byte marshal
   451  // as an empty JSON string for the former, while the latter as a JSON null.
   452  // The non-zero values of those types always marshal as JSON strings.
   453  //
   454  // Related issues:
   455  //
   456  //	https://go.dev/issue/27589
   457  //	https://go.dev/issue/37711
   458  func TestNilSlicesAndMaps(t *testing.T) {
   459  	type Composites struct {
   460  		B []byte            // always encoded in v2 as a JSON string
   461  		S []string          // always encoded in v2 as a JSON array
   462  		M map[string]string // always encoded in v2 as a JSON object
   463  	}
   464  
   465  	for _, json := range jsonPackages {
   466  		t.Run(path.Join("Marshal", json.Version), func(t *testing.T) {
   467  			in := []Composites{
   468  				{B: []byte(nil), S: []string(nil), M: map[string]string(nil)},
   469  				{B: []byte{}, S: []string{}, M: map[string]string{}},
   470  			}
   471  			want := map[string]string{
   472  				"v1": `[{"B":null,"S":null,"M":null},{"B":"","S":[],"M":{}}]`,
   473  				"v2": `[{"B":"","S":[],"M":{}},{"B":"","S":[],"M":{}}]`, // v2 emits nil slices and maps as empty JSON objects and arrays
   474  			}[json.Version]
   475  			got, err := json.Marshal(in)
   476  			if err != nil {
   477  				t.Fatalf("json.Marshal error: %v", err)
   478  			}
   479  			if string(got) != want {
   480  				t.Fatalf("json.Marshal = %s, want %s", got, want)
   481  			}
   482  		})
   483  	}
   484  }
   485  
   486  // In v1, unmarshaling into a Go array permits JSON arrays with any length.
   487  // In v2, unmarshaling into a Go array requires that the JSON array
   488  // have the exact same number of elements as the Go array.
   489  //
   490  // Go arrays are often used because the exact length has significant meaning.
   491  // Ignoring this detail seems like a mistake. Also, the v1 behavior leads to
   492  // silent data loss when excess JSON array elements are discarded.
   493  func TestArrays(t *testing.T) {
   494  	for _, json := range jsonPackages {
   495  		t.Run(path.Join("Unmarshal/TooFew", json.Version), func(t *testing.T) {
   496  			var got [2]int
   497  			err := json.Unmarshal([]byte(`[1]`), &got)
   498  			switch {
   499  			case got != [2]int{1, 0}:
   500  				t.Fatalf(`json.Unmarshal = %v, want [1 0]`, got)
   501  			case json.Version == "v1" && err != nil:
   502  				t.Fatalf("json.Unmarshal error: %v", err)
   503  			case json.Version == "v2" && err == nil:
   504  				t.Fatal("json.Unmarshal error is nil, want non-nil")
   505  			}
   506  		})
   507  	}
   508  
   509  	for _, json := range jsonPackages {
   510  		t.Run(path.Join("Unmarshal/TooMany", json.Version), func(t *testing.T) {
   511  			var got [2]int
   512  			err := json.Unmarshal([]byte(`[1,2,3]`), &got)
   513  			switch {
   514  			case got != [2]int{1, 2}:
   515  				t.Fatalf(`json.Unmarshal = %v, want [1 2]`, got)
   516  			case json.Version == "v1" && err != nil:
   517  				t.Fatalf("json.Unmarshal error: %v", err)
   518  			case json.Version == "v2" && err == nil:
   519  				t.Fatal("json.Unmarshal error is nil, want non-nil")
   520  			}
   521  		})
   522  	}
   523  }
   524  
   525  // In v1, byte arrays are treated as arrays of unsigned integers.
   526  // In v2, byte arrays are treated as binary values (similar to []byte).
   527  // This is to make the behavior of [N]byte and []byte more consistent.
   528  //
   529  // Users of v2 can opt into the v1 behavior by setting the
   530  // [jsonv1.FormatByteArrayAsArray] option.
   531  func TestByteArrays(t *testing.T) {
   532  	for _, json := range jsonPackages {
   533  		t.Run(path.Join("Marshal", json.Version), func(t *testing.T) {
   534  			in := [4]byte{1, 2, 3, 4}
   535  			got, err := json.Marshal(in)
   536  			if err != nil {
   537  				t.Fatalf("json.Marshal error: %v", err)
   538  			}
   539  			want := map[string]string{
   540  				"v1": `[1,2,3,4]`,
   541  				"v2": `"AQIDBA=="`,
   542  			}[json.Version]
   543  			if string(got) != want {
   544  				t.Fatalf("json.Marshal = %s, want %s", got, want)
   545  			}
   546  		})
   547  	}
   548  
   549  	for _, json := range jsonPackages {
   550  		t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) {
   551  			in := map[string]string{
   552  				"v1": `[1,2,3,4]`,
   553  				"v2": `"AQIDBA=="`,
   554  			}[json.Version]
   555  			var got [4]byte
   556  			err := json.Unmarshal([]byte(in), &got)
   557  			switch {
   558  			case err != nil:
   559  				t.Fatalf("json.Unmarshal error: %v", err)
   560  			case got != [4]byte{1, 2, 3, 4}:
   561  				t.Fatalf("json.Unmarshal = %v, want [1 2 3 4]", got)
   562  			}
   563  		})
   564  	}
   565  }
   566  
   567  // CallCheck implements json.{Marshaler,Unmarshaler} on a pointer receiver.
   568  type CallCheck string
   569  
   570  // MarshalJSON always returns a JSON string with the literal "CALLED".
   571  func (*CallCheck) MarshalJSON() ([]byte, error) {
   572  	return []byte(`"CALLED"`), nil
   573  }
   574  
   575  // UnmarshalJSON always stores a string with the literal "CALLED".
   576  func (v *CallCheck) UnmarshalJSON([]byte) error {
   577  	*v = `CALLED`
   578  	return nil
   579  }
   580  
   581  // In v1, the implementation is inconsistent about whether it calls
   582  // MarshalJSON and UnmarshalJSON methods declared on pointer receivers
   583  // when it has an unaddressable value (per reflect.Value.CanAddr) on hand.
   584  // When marshaling, it never boxes the value on the heap to make it addressable,
   585  // while it sometimes boxes values (e.g., for map entries) when unmarshaling.
   586  //
   587  // In v2, the implementation always calls MarshalJSON and UnmarshalJSON methods
   588  // by boxing the value on the heap if necessary.
   589  //
   590  // The v1 behavior is surprising at best and buggy at worst.
   591  // Unfortunately, it cannot be changed without breaking existing usages.
   592  //
   593  // Related issues:
   594  //
   595  //	https://go.dev/issue/27722
   596  //	https://go.dev/issue/33993
   597  //	https://go.dev/issue/42508
   598  func TestPointerReceiver(t *testing.T) {
   599  	type Values struct {
   600  		S []CallCheck
   601  		A [1]CallCheck
   602  		M map[string]CallCheck
   603  		V CallCheck
   604  		I any
   605  	}
   606  
   607  	for _, json := range jsonPackages {
   608  		t.Run(path.Join("Marshal", json.Version), func(t *testing.T) {
   609  			var cc CallCheck
   610  			in := Values{
   611  				S: []CallCheck{cc},
   612  				A: [1]CallCheck{cc},             // MarshalJSON not called on v1
   613  				M: map[string]CallCheck{"": cc}, // MarshalJSON not called on v1
   614  				V: cc,                           // MarshalJSON not called on v1
   615  				I: cc,                           // MarshalJSON not called on v1
   616  			}
   617  			want := map[string]string{
   618  				"v1": `{"S":["CALLED"],"A":[""],"M":{"":""},"V":"","I":""}`,
   619  				"v2": `{"S":["CALLED"],"A":["CALLED"],"M":{"":"CALLED"},"V":"CALLED","I":"CALLED"}`,
   620  			}[json.Version]
   621  			got, err := json.Marshal(in)
   622  			if err != nil {
   623  				t.Fatalf("json.Marshal error: %v", err)
   624  			}
   625  			if string(got) != want {
   626  				t.Fatalf("json.Marshal = %s, want %s", got, want)
   627  			}
   628  		})
   629  	}
   630  
   631  	for _, json := range jsonPackages {
   632  		t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) {
   633  			in := `{"S":[""],"A":[""],"M":{"":""},"V":"","I":""}`
   634  			called := CallCheck("CALLED") // resulting state if UnmarshalJSON is called
   635  			want := map[string]Values{
   636  				"v1": {
   637  					S: []CallCheck{called},
   638  					A: [1]CallCheck{called},
   639  					M: map[string]CallCheck{"": called},
   640  					V: called,
   641  					I: "", // UnmarshalJSON not called on v1; replaced with Go string
   642  				},
   643  				"v2": {
   644  					S: []CallCheck{called},
   645  					A: [1]CallCheck{called},
   646  					M: map[string]CallCheck{"": called},
   647  					V: called,
   648  					I: called,
   649  				},
   650  			}[json.Version]
   651  			got := Values{
   652  				A: [1]CallCheck{CallCheck("")},
   653  				S: []CallCheck{CallCheck("")},
   654  				M: map[string]CallCheck{"": CallCheck("")},
   655  				V: CallCheck(""),
   656  				I: CallCheck(""),
   657  			}
   658  			if err := json.Unmarshal([]byte(in), &got); err != nil {
   659  				t.Fatalf("json.Unmarshal error: %v", err)
   660  			}
   661  			if !reflect.DeepEqual(got, want) {
   662  				t.Fatalf("json.Unmarshal = %v, want %v", got, want)
   663  			}
   664  		})
   665  	}
   666  }
   667  
   668  // In v1, maps are marshaled in a deterministic order.
   669  // In v2, maps are marshaled in a non-deterministic order.
   670  //
   671  // The reason for the change is that v2 prioritizes performance and
   672  // the guarantee that marshaling operates primarily in a streaming manner.
   673  //
   674  // The v2 API provides jsontext.Value.Canonicalize if stability is needed:
   675  //
   676  //	(*jsontext.Value)(&b).Canonicalize()
   677  //
   678  // Related issue:
   679  //
   680  //	https://go.dev/issue/7872
   681  //	https://go.dev/issue/33714
   682  func TestMapDeterminism(t *testing.T) {
   683  	const iterations = 10
   684  	in := map[int]int{0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8, 9: 9}
   685  
   686  	for _, json := range jsonPackages {
   687  		t.Run(path.Join("Marshal", json.Version), func(t *testing.T) {
   688  			outs := make(map[string]bool)
   689  			for range iterations {
   690  				b, err := json.Marshal(in)
   691  				if err != nil {
   692  					t.Fatalf("json.Marshal error: %v", err)
   693  				}
   694  				outs[string(b)] = true
   695  			}
   696  			switch {
   697  			case json.Version == "v1" && len(outs) != 1:
   698  				t.Fatalf("json.Marshal encoded to %d unique forms, expected 1", len(outs))
   699  			case json.Version == "v2" && len(outs) == 1:
   700  				t.Logf("json.Marshal encoded to 1 unique form by chance; are you feeling lucky?")
   701  			}
   702  		})
   703  	}
   704  }
   705  
   706  // In v1, JSON string encoding escapes special characters related to HTML.
   707  // In v2, JSON string encoding uses a normalized representation (per RFC 8785).
   708  //
   709  // Users of v2 can opt into the v1 behavior by setting EscapeForHTML and EscapeForJS.
   710  //
   711  // Escaping HTML-specific characters in a JSON library is a layering violation.
   712  // It presumes that JSON is always used with HTML and ignores other
   713  // similar classes of injection attacks (e.g., SQL injection).
   714  // Users of JSON with HTML should either manually ensure that embedded JSON is
   715  // properly escaped or be relying on a module like "github.com/google/safehtml"
   716  // to handle safe interoperability of JSON and HTML.
   717  func TestEscapeHTML(t *testing.T) {
   718  	for _, json := range jsonPackages {
   719  		t.Run(path.Join("Marshal", json.Version), func(t *testing.T) {
   720  			const in = `<script> console.log("Hello, world!"); </script>`
   721  			got, err := json.Marshal(in)
   722  			if err != nil {
   723  				t.Fatalf("json.Marshal error: %v", err)
   724  			}
   725  			want := map[string]string{
   726  				"v1": `"\u003cscript\u003e console.log(\"Hello, world!\"); \u003c/script\u003e"`,
   727  				"v2": `"<script> console.log(\"Hello, world!\"); </script>"`,
   728  			}[json.Version]
   729  			if string(got) != want {
   730  				t.Fatalf("json.Marshal = %s, want %s", got, want)
   731  			}
   732  		})
   733  	}
   734  }
   735  
   736  // In v1, JSON serialization silently ignored invalid UTF-8 by
   737  // replacing such bytes with the Unicode replacement character.
   738  // In v2, JSON serialization reports an error if invalid UTF-8 is encountered.
   739  //
   740  // Users of v2 can opt into the v1 behavior by setting [AllowInvalidUTF8].
   741  //
   742  // Silently allowing invalid UTF-8 causes data corruption that can be difficult
   743  // to detect until it is too late. Once it has been discovered, strict UTF-8
   744  // behavior sometimes cannot be enabled since other logic may be depending
   745  // on the current behavior due to Hyrum's Law.
   746  //
   747  // Tim Bray, the author of RFC 8259 recommends that implementations should
   748  // go beyond RFC 8259 and instead target compliance with RFC 7493,
   749  // which makes strict decisions about behavior left undefined in RFC 8259.
   750  // In particular, RFC 7493 rejects the presence of invalid UTF-8.
   751  // See https://www.tbray.org/ongoing/When/201x/2017/12/14/RFC-8259-STD-90
   752  func TestInvalidUTF8(t *testing.T) {
   753  	for _, json := range jsonPackages {
   754  		t.Run(path.Join("Marshal", json.Version), func(t *testing.T) {
   755  			got, err := json.Marshal("\xff")
   756  			switch {
   757  			case json.Version == "v1" && err != nil:
   758  				t.Fatalf("json.Marshal error: %v", err)
   759  			case json.Version == "v1" && string(got) != "\"\ufffd\"":
   760  				t.Fatalf(`json.Marshal = %s, want %q`, got, "\ufffd")
   761  			case json.Version == "v2" && err == nil:
   762  				t.Fatal("json.Marshal error is nil, want non-nil")
   763  			}
   764  		})
   765  	}
   766  
   767  	for _, json := range jsonPackages {
   768  		t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) {
   769  			const in = "\"\xff\""
   770  			var got string
   771  			err := json.Unmarshal([]byte(in), &got)
   772  			switch {
   773  			case json.Version == "v1" && err != nil:
   774  				t.Fatalf("json.Unmarshal error: %v", err)
   775  			case json.Version == "v1" && got != "\ufffd":
   776  				t.Fatalf(`json.Unmarshal = %q, want "\ufffd"`, got)
   777  			case json.Version == "v2" && err == nil:
   778  				t.Fatal("json.Unmarshal error is nil, want non-nil")
   779  			}
   780  		})
   781  	}
   782  }
   783  
   784  // In v1, duplicate JSON object names are permitted by default where
   785  // they follow the inconsistent and difficult-to-explain merge semantics of v1.
   786  // In v2, duplicate JSON object names are rejected by default where
   787  // they follow the merge semantics of v2 based on RFC 7396.
   788  //
   789  // Users of v2 can opt into the v1 behavior by setting [AllowDuplicateNames].
   790  //
   791  // Per RFC 8259, the handling of duplicate names is left as undefined behavior.
   792  // Rejecting such inputs is within the realm of valid behavior.
   793  // Tim Bray, the author of RFC 8259 recommends that implementations should
   794  // go beyond RFC 8259 and instead target compliance with RFC 7493,
   795  // which makes strict decisions about behavior left undefined in RFC 8259.
   796  // In particular, RFC 7493 rejects the presence of duplicate object names.
   797  // See https://www.tbray.org/ongoing/When/201x/2017/12/14/RFC-8259-STD-90
   798  //
   799  // The lack of duplicate name rejection has correctness implications where
   800  // roundtrip unmarshal/marshal do not result in semantically equivalent JSON.
   801  // This is surprising behavior for users when they accidentally
   802  // send JSON objects with duplicate names.
   803  //
   804  // The lack of duplicate name rejection may have security implications since it
   805  // becomes difficult for a security tool to validate the semantic meaning of a
   806  // JSON object since meaning is undefined in the presence of duplicate names.
   807  // See https://labs.bishopfox.com/tech-blog/an-exploration-of-json-interoperability-vulnerabilities
   808  //
   809  // Related issue:
   810  //
   811  //	https://go.dev/issue/48298
   812  func TestDuplicateNames(t *testing.T) {
   813  	for _, json := range jsonPackages {
   814  		t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) {
   815  			const in = `{"Name":1,"Name":2}`
   816  			var got struct{ Name int }
   817  			err := json.Unmarshal([]byte(in), &got)
   818  			switch {
   819  			case json.Version == "v1" && err != nil:
   820  				t.Fatalf("json.Unmarshal error: %v", err)
   821  			case json.Version == "v1" && got != struct{ Name int }{2}:
   822  				t.Fatalf(`json.Unmarshal = %v, want {2}`, got)
   823  			case json.Version == "v2" && err == nil:
   824  				t.Fatal("json.Unmarshal error is nil, want non-nil")
   825  			}
   826  		})
   827  	}
   828  }
   829  
   830  // In v1, unmarshaling a JSON null into a non-empty value was inconsistent
   831  // in that sometimes it would be ignored and other times clear the value.
   832  // In v2, unmarshaling a JSON null into a non-empty value would consistently
   833  // always clear the value regardless of the value's type.
   834  //
   835  // The purpose of this change is to have consistent behavior with how JSON nulls
   836  // are handled during Unmarshal. This semantic detail has no effect
   837  // when Unmarshaling into a empty value.
   838  //
   839  // Related issues:
   840  //
   841  //	https://go.dev/issue/22177
   842  //	https://go.dev/issue/33835
   843  func TestMergeNull(t *testing.T) {
   844  	type Types struct {
   845  		Bool      bool
   846  		String    string
   847  		Bytes     []byte
   848  		Int       int
   849  		Map       map[string]string
   850  		Struct    struct{ Field string }
   851  		Slice     []string
   852  		Array     [1]string
   853  		Pointer   *string
   854  		Interface any
   855  	}
   856  
   857  	for _, json := range jsonPackages {
   858  		t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) {
   859  			// Start with a non-empty value where all fields are populated.
   860  			in := Types{
   861  				Bool:      true,
   862  				String:    "old",
   863  				Bytes:     []byte("old"),
   864  				Int:       1234,
   865  				Map:       map[string]string{"old": "old"},
   866  				Struct:    struct{ Field string }{"old"},
   867  				Slice:     []string{"old"},
   868  				Array:     [1]string{"old"},
   869  				Pointer:   new(string),
   870  				Interface: "old",
   871  			}
   872  
   873  			// Unmarshal a JSON null into every field.
   874  			if err := json.Unmarshal([]byte(`{
   875  				"Bool":      null,
   876  				"String":    null,
   877  				"Bytes":     null,
   878  				"Int":       null,
   879  				"Map":       null,
   880  				"Struct":    null,
   881  				"Slice":     null,
   882  				"Array":     null,
   883  				"Pointer":   null,
   884  				"Interface": null
   885  			}`), &in); err != nil {
   886  				t.Fatalf("json.Unmarshal error: %v", err)
   887  			}
   888  
   889  			want := map[string]Types{
   890  				"v1": {
   891  					Bool:   true,
   892  					String: "old",
   893  					Int:    1234,
   894  					Struct: struct{ Field string }{"old"},
   895  					Array:  [1]string{"old"},
   896  				},
   897  				"v2": {}, // all fields are zeroed
   898  			}[json.Version]
   899  			if !reflect.DeepEqual(in, want) {
   900  				t.Fatalf("json.Unmarshal = %+v, want %+v", in, want)
   901  			}
   902  		})
   903  	}
   904  }
   905  
   906  // In v1, merge semantics are inconsistent and difficult to explain.
   907  // In v2, merge semantics replaces the destination value for anything
   908  // other than a JSON object, and recursively merges JSON objects.
   909  //
   910  // Merge semantics in v1 are inconsistent and difficult to explain
   911  // largely because the behavior came about organically, rather than
   912  // having a principled approach to how the semantics should operate.
   913  // In v2, merging follows behavior based on RFC 7396.
   914  //
   915  // Related issues:
   916  //
   917  //	https://go.dev/issue/21092
   918  //	https://go.dev/issue/26946
   919  //	https://go.dev/issue/27172
   920  //	https://go.dev/issue/30701
   921  //	https://go.dev/issue/31924
   922  //	https://go.dev/issue/43664
   923  func TestMergeComposite(t *testing.T) {
   924  	type Tuple struct{ Old, New bool }
   925  	type Composites struct {
   926  		Slice            []Tuple
   927  		Array            [1]Tuple
   928  		Map              map[string]Tuple
   929  		MapPointer       map[string]*Tuple
   930  		Struct           struct{ Tuple Tuple }
   931  		StructPointer    *struct{ Tuple Tuple }
   932  		Interface        any
   933  		InterfacePointer any
   934  	}
   935  
   936  	for _, json := range jsonPackages {
   937  		t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) {
   938  			// Start with a non-empty value where all fields are populated.
   939  			in := Composites{
   940  				Slice:            []Tuple{{Old: true}, {Old: true}}[:1],
   941  				Array:            [1]Tuple{{Old: true}},
   942  				Map:              map[string]Tuple{"Tuple": {Old: true}},
   943  				MapPointer:       map[string]*Tuple{"Tuple": {Old: true}},
   944  				Struct:           struct{ Tuple Tuple }{Tuple{Old: true}},
   945  				StructPointer:    &struct{ Tuple Tuple }{Tuple{Old: true}},
   946  				Interface:        Tuple{Old: true},
   947  				InterfacePointer: &Tuple{Old: true},
   948  			}
   949  
   950  			// Unmarshal into every pre-populated field.
   951  			if err := json.Unmarshal([]byte(`{
   952  				"Slice":            [{"New":true}, {"New":true}],
   953  				"Array":            [{"New":true}],
   954  				"Map":              {"Tuple": {"New":true}},
   955  				"MapPointer":       {"Tuple": {"New":true}},
   956  				"Struct":           {"Tuple": {"New":true}},
   957  				"StructPointer":    {"Tuple": {"New":true}},
   958  				"Interface":        {"New":true},
   959  				"InterfacePointer": {"New":true}
   960  			}`), &in); err != nil {
   961  				t.Fatalf("json.Unmarshal error: %v", err)
   962  			}
   963  
   964  			merged := Tuple{Old: true, New: true}
   965  			replaced := Tuple{Old: false, New: true}
   966  			want := map[string]Composites{
   967  				"v1": {
   968  					Slice:            []Tuple{merged, merged},               // merged
   969  					Array:            [1]Tuple{merged},                      // merged
   970  					Map:              map[string]Tuple{"Tuple": replaced},   // replaced
   971  					MapPointer:       map[string]*Tuple{"Tuple": &replaced}, // replaced
   972  					Struct:           struct{ Tuple Tuple }{merged},         // merged (same as v2)
   973  					StructPointer:    &struct{ Tuple Tuple }{merged},        // merged (same as v2)
   974  					Interface:        map[string]any{"New": true},           // replaced
   975  					InterfacePointer: &merged,                               // merged (same as v2)
   976  				},
   977  				"v2": {
   978  					Slice:            []Tuple{replaced, replaced},         // replaced
   979  					Array:            [1]Tuple{replaced},                  // replaced
   980  					Map:              map[string]Tuple{"Tuple": merged},   // merged
   981  					MapPointer:       map[string]*Tuple{"Tuple": &merged}, // merged
   982  					Struct:           struct{ Tuple Tuple }{merged},       // merged (same as v1)
   983  					StructPointer:    &struct{ Tuple Tuple }{merged},      // merged (same as v1)
   984  					Interface:        merged,                              // merged
   985  					InterfacePointer: &merged,                             // merged (same as v1)
   986  				},
   987  			}[json.Version]
   988  			if !reflect.DeepEqual(in, want) {
   989  				t.Fatalf("json.Unmarshal = %+v, want %+v", in, want)
   990  			}
   991  		})
   992  	}
   993  }
   994  
   995  // In v1, there was no special support for time.Duration,
   996  // which resulted in that type simply being treated as a signed integer.
   997  // In v2, there is now first-class support for time.Duration, where the type is
   998  // formatted and parsed using time.Duration.String and time.ParseDuration.
   999  //
  1000  // Users of v2 can opt into the v1 behavior by setting the
  1001  // [jsonv1.FormatDurationAsNano] option.
  1002  //
  1003  // Related issue:
  1004  //
  1005  //	https://go.dev/issue/10275
  1006  func TestTimeDurations(t *testing.T) {
  1007  	t.SkipNow() // TODO(https://go.dev/issue/71631): The default representation of time.Duration is still undecided.
  1008  	for _, json := range jsonPackages {
  1009  		t.Run(path.Join("Marshal", json.Version), func(t *testing.T) {
  1010  			got, err := json.Marshal(time.Minute)
  1011  			switch {
  1012  			case err != nil:
  1013  				t.Fatalf("json.Marshal error: %v", err)
  1014  			case json.Version == "v1" && string(got) != "60000000000":
  1015  				t.Fatalf("json.Marshal = %s, want 60000000000", got)
  1016  			case json.Version == "v2" && string(got) != `"1m0s"`:
  1017  				t.Fatalf(`json.Marshal = %s, want "1m0s"`, got)
  1018  			}
  1019  		})
  1020  	}
  1021  
  1022  	for _, json := range jsonPackages {
  1023  		t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) {
  1024  			in := map[string]string{
  1025  				"v1": "60000000000",
  1026  				"v2": `"1m0s"`,
  1027  			}[json.Version]
  1028  			var got time.Duration
  1029  			err := json.Unmarshal([]byte(in), &got)
  1030  			switch {
  1031  			case err != nil:
  1032  				t.Fatalf("json.Unmarshal error: %v", err)
  1033  			case got != time.Minute:
  1034  				t.Fatalf("json.Unmarshal = %v, want 1m0s", got)
  1035  			}
  1036  		})
  1037  	}
  1038  }
  1039  
  1040  // In v1, non-empty structs without any JSON serializable fields are permitted.
  1041  // In v2, non-empty structs without any JSON serializable fields are rejected.
  1042  //
  1043  // The purpose of this change is to avoid a common pitfall for new users
  1044  // where they expect JSON serialization to handle unexported fields.
  1045  // However, this does not work since Go reflection does not
  1046  // provide the package the ability to mutate such fields.
  1047  // Rejecting unserializable structs in v2 is intended to be a clear signal
  1048  // that the type is not supposed to be serialized.
  1049  func TestEmptyStructs(t *testing.T) {
  1050  	never := func(string) bool { return false }
  1051  	onlyV2 := func(v string) bool { return v == "v2" }
  1052  	values := []struct {
  1053  		in        any
  1054  		wantError func(string) bool
  1055  	}{
  1056  		// It is okay to marshal a truly empty struct in v1 and v2.
  1057  		{in: addr(struct{}{}), wantError: never},
  1058  		// In v1, a non-empty struct without exported fields
  1059  		// is equivalent to an empty struct, but is rejected in v2.
  1060  		// Note that errors.errorString type has only unexported fields.
  1061  		{in: errors.New("error"), wantError: onlyV2},
  1062  		// A mix of exported and unexported fields is permitted.
  1063  		{in: addr(struct{ Exported, unexported int }{}), wantError: never},
  1064  	}
  1065  
  1066  	for _, json := range jsonPackages {
  1067  		t.Run("Marshal", func(t *testing.T) {
  1068  			for _, value := range values {
  1069  				wantError := value.wantError(json.Version)
  1070  				_, err := json.Marshal(value.in)
  1071  				switch {
  1072  				case (err == nil) && wantError:
  1073  					t.Fatalf("json.Marshal error is nil, want non-nil")
  1074  				case (err != nil) && !wantError:
  1075  					t.Fatalf("json.Marshal error: %v", err)
  1076  				}
  1077  			}
  1078  		})
  1079  	}
  1080  
  1081  	for _, json := range jsonPackages {
  1082  		t.Run("Unmarshal", func(t *testing.T) {
  1083  			for _, value := range values {
  1084  				wantError := value.wantError(json.Version)
  1085  				out := reflect.New(reflect.TypeOf(value.in).Elem()).Interface()
  1086  				err := json.Unmarshal([]byte("{}"), out)
  1087  				switch {
  1088  				case (err == nil) && wantError:
  1089  					t.Fatalf("json.Unmarshal error is nil, want non-nil")
  1090  				case (err != nil) && !wantError:
  1091  					t.Fatalf("json.Unmarshal error: %v", err)
  1092  				}
  1093  			}
  1094  		})
  1095  	}
  1096  }
  1097  

View as plain text