1
2
3
4
5
6
7 package jsonwire
8
9 import (
10 "bufio"
11 "bytes"
12 "compress/gzip"
13 "crypto/sha256"
14 "encoding/binary"
15 "encoding/hex"
16 "flag"
17 "math"
18 "net/http"
19 "reflect"
20 "strconv"
21 "strings"
22 "testing"
23 "time"
24
25 "encoding/json/internal/jsonflags"
26 )
27
28 func TestAppendQuote(t *testing.T) {
29 tests := []struct {
30 in string
31 flags jsonflags.Bools
32 want string
33 wantErr error
34 wantErrUTF8 error
35 }{
36 {"", 0, `""`, nil, nil},
37 {"hello", 0, `"hello"`, nil, nil},
38 {"\x00", 0, `"\u0000"`, nil, nil},
39 {"\x1f", 0, `"\u001f"`, nil, nil},
40 {"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", 0, `"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"`, nil, nil},
41 {" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f", 0, "\" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f\"", nil, nil},
42 {" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f", jsonflags.EscapeForHTML, "\" !#$%\\u0026'()*+,-./0123456789:;\\u003c=\\u003e?@[]^_`{|}~\x7f\"", nil, nil},
43 {" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f", jsonflags.EscapeForJS, "\" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f\"", nil, nil},
44 {"\u2027\u2028\u2029\u2030", 0, "\"\u2027\u2028\u2029\u2030\"", nil, nil},
45 {"\u2027\u2028\u2029\u2030", jsonflags.EscapeForHTML, "\"\u2027\u2028\u2029\u2030\"", nil, nil},
46 {"\u2027\u2028\u2029\u2030", jsonflags.EscapeForJS, "\"\u2027\\u2028\\u2029\u2030\"", nil, nil},
47 {"x\x80\ufffd", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8},
48 {"x\xff\ufffd", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8},
49 {"x\xc0", 0, "\"x\ufffd\"", nil, ErrInvalidUTF8},
50 {"x\xc0\x80", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8},
51 {"x\xe0", 0, "\"x\ufffd\"", nil, ErrInvalidUTF8},
52 {"x\xe0\x80", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8},
53 {"x\xe0\x80\x80", 0, "\"x\ufffd\ufffd\ufffd\"", nil, ErrInvalidUTF8},
54 {"x\xf0", 0, "\"x\ufffd\"", nil, ErrInvalidUTF8},
55 {"x\xf0\x80", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8},
56 {"x\xf0\x80\x80", 0, "\"x\ufffd\ufffd\ufffd\"", nil, ErrInvalidUTF8},
57 {"x\xf0\x80\x80\x80", 0, "\"x\ufffd\ufffd\ufffd\ufffd\"", nil, ErrInvalidUTF8},
58 {"x\xed\xba\xad", 0, "\"x\ufffd\ufffd\ufffd\"", nil, ErrInvalidUTF8},
59 {"\"\\/\b\f\n\r\t", 0, `"\"\\/\b\f\n\r\t"`, nil, nil},
60 {"٩(-̮̮̃-̃)۶ ٩(●̮̮̃•̃)۶ ٩(͡๏̯͡๏)۶ ٩(-̮̮̃•̃).", 0, `"٩(-̮̮̃-̃)۶ ٩(●̮̮̃•̃)۶ ٩(͡๏̯͡๏)۶ ٩(-̮̮̃•̃)."`, nil, nil},
61 {"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602", 0, "\"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602\"", nil, nil},
62 {"\u0000\u001f\u0020\u0022\u0026\u003c\u003e\u005c\u007f\u0080\u2028\u2029\ufffd\U0001f602", 0, "\"\\u0000\\u001f\u0020\\\"\u0026\u003c\u003e\\\\\u007f\u0080\u2028\u2029\ufffd\U0001f602\"", nil, nil},
63 }
64
65 for _, tt := range tests {
66 t.Run("", func(t *testing.T) {
67 var flags jsonflags.Flags
68 flags.Set(tt.flags | 1)
69
70 flags.Set(jsonflags.AllowInvalidUTF8 | 1)
71 got, gotErr := AppendQuote(nil, tt.in, &flags)
72 if string(got) != tt.want || !reflect.DeepEqual(gotErr, tt.wantErr) {
73 t.Errorf("AppendQuote(nil, %q, ...) = (%s, %v), want (%s, %v)", tt.in, got, gotErr, tt.want, tt.wantErr)
74 }
75 flags.Set(jsonflags.AllowInvalidUTF8 | 0)
76 switch got, gotErr := AppendQuote(nil, tt.in, &flags); {
77 case tt.wantErrUTF8 == nil && (string(got) != tt.want || !reflect.DeepEqual(gotErr, tt.wantErr)):
78 t.Errorf("AppendQuote(nil, %q, ...) = (%s, %v), want (%s, %v)", tt.in, got, gotErr, tt.want, tt.wantErr)
79 case tt.wantErrUTF8 != nil && (!strings.HasPrefix(tt.want, string(got)) || !reflect.DeepEqual(gotErr, tt.wantErrUTF8)):
80 t.Errorf("AppendQuote(nil, %q, ...) = (%s, %v), want (%s, %v)", tt.in, got, gotErr, tt.want, tt.wantErrUTF8)
81 }
82 })
83 }
84 }
85
86 func TestAppendNumber(t *testing.T) {
87 tests := []struct {
88 in float64
89 want32 string
90 want64 string
91 }{
92 {math.E, "2.7182817", "2.718281828459045"},
93 {math.Pi, "3.1415927", "3.141592653589793"},
94 {math.SmallestNonzeroFloat32, "1e-45", "1.401298464324817e-45"},
95 {math.SmallestNonzeroFloat64, "0", "5e-324"},
96 {math.MaxFloat32, "3.4028235e+38", "3.4028234663852886e+38"},
97 {math.MaxFloat64, "", "1.7976931348623157e+308"},
98 {0.1111111111111111, "0.11111111", "0.1111111111111111"},
99 {0.2222222222222222, "0.22222222", "0.2222222222222222"},
100 {0.3333333333333333, "0.33333334", "0.3333333333333333"},
101 {0.4444444444444444, "0.44444445", "0.4444444444444444"},
102 {0.5555555555555555, "0.5555556", "0.5555555555555555"},
103 {0.6666666666666666, "0.6666667", "0.6666666666666666"},
104 {0.7777777777777777, "0.7777778", "0.7777777777777777"},
105 {0.8888888888888888, "0.8888889", "0.8888888888888888"},
106 {0.9999999999999999, "1", "0.9999999999999999"},
107
108
109
110 {math.Float64frombits(0x0000000000000000), "0", "0"},
111 {math.Float64frombits(0x8000000000000000), "-0", "-0"},
112 {math.Float64frombits(0x0000000000000001), "0", "5e-324"},
113 {math.Float64frombits(0x8000000000000001), "-0", "-5e-324"},
114 {math.Float64frombits(0x7fefffffffffffff), "", "1.7976931348623157e+308"},
115 {math.Float64frombits(0xffefffffffffffff), "", "-1.7976931348623157e+308"},
116 {math.Float64frombits(0x4340000000000000), "9007199000000000", "9007199254740992"},
117 {math.Float64frombits(0xc340000000000000), "-9007199000000000", "-9007199254740992"},
118 {math.Float64frombits(0x4430000000000000), "295147900000000000000", "295147905179352830000"},
119 {math.Float64frombits(0x44b52d02c7e14af5), "1e+23", "9.999999999999997e+22"},
120 {math.Float64frombits(0x44b52d02c7e14af6), "1e+23", "1e+23"},
121 {math.Float64frombits(0x44b52d02c7e14af7), "1e+23", "1.0000000000000001e+23"},
122 {math.Float64frombits(0x444b1ae4d6e2ef4e), "1e+21", "999999999999999700000"},
123 {math.Float64frombits(0x444b1ae4d6e2ef4f), "1e+21", "999999999999999900000"},
124 {math.Float64frombits(0x444b1ae4d6e2ef50), "1e+21", "1e+21"},
125 {math.Float64frombits(0x3eb0c6f7a0b5ed8c), "0.000001", "9.999999999999997e-7"},
126 {math.Float64frombits(0x3eb0c6f7a0b5ed8d), "0.000001", "0.000001"},
127 {math.Float64frombits(0x41b3de4355555553), "333333340", "333333333.3333332"},
128 {math.Float64frombits(0x41b3de4355555554), "333333340", "333333333.33333325"},
129 {math.Float64frombits(0x41b3de4355555555), "333333340", "333333333.3333333"},
130 {math.Float64frombits(0x41b3de4355555556), "333333340", "333333333.3333334"},
131 {math.Float64frombits(0x41b3de4355555557), "333333340", "333333333.33333343"},
132 {math.Float64frombits(0xbecbf647612f3696), "-0.0000033333333", "-0.0000033333333333333333"},
133 {math.Float64frombits(0x43143ff3c1cb0959), "1424953900000000", "1424953923781206.2"},
134
135
136
137 {float64(math.Float32frombits(0x65a96815)), "9.999999e+22", "9.999998877476383e+22"},
138 {float64(math.Float32frombits(0x65a96816)), "1e+23", "9.999999778196308e+22"},
139 {float64(math.Float32frombits(0x65a96817)), "1.0000001e+23", "1.0000000678916234e+23"},
140 {float64(math.Float32frombits(0x6258d725)), "999999900000000000000", "999999879303389000000"},
141 {float64(math.Float32frombits(0x6258d726)), "999999950000000000000", "999999949672133200000"},
142 {float64(math.Float32frombits(0x6258d727)), "1e+21", "1.0000000200408773e+21"},
143 {float64(math.Float32frombits(0x6258d728)), "1.0000001e+21", "1.0000000904096215e+21"},
144 {float64(math.Float32frombits(0x358637bc)), "9.999999e-7", "9.99999883788405e-7"},
145 {float64(math.Float32frombits(0x358637bd)), "0.000001", "9.999999974752427e-7"},
146 {float64(math.Float32frombits(0x358637be)), "0.0000010000001", "0.0000010000001111620804"},
147 }
148
149 for _, tt := range tests {
150 t.Run("", func(t *testing.T) {
151 if got32 := string(AppendFloat(nil, tt.in, 32)); got32 != tt.want32 && tt.want32 != "" {
152 t.Errorf("AppendFloat(nil, %v, 32) = %v, want %v", tt.in, got32, tt.want32)
153 }
154 if got64 := string(AppendFloat(nil, tt.in, 64)); got64 != tt.want64 && tt.want64 != "" {
155 t.Errorf("AppendFloat(nil, %v, 64) = %v, want %v", tt.in, got64, tt.want64)
156 }
157 })
158 }
159 }
160
161
162
163
164
165
166 var testCanonicalNumberLines = flag.Float64("canonical-number-lines", 1e4, "specify the number of lines to check from the canonical numbers testdata")
167
168
169
170
171 func TestCanonicalNumber(t *testing.T) {
172 const testfileURL = "https://github.com/cyberphone/json-canonicalization/releases/download/es6testfile/es6testfile100m.txt.gz"
173 hashes := map[float64]string{
174 1e3: "be18b62b6f69cdab33a7e0dae0d9cfa869fda80ddc712221570f9f40a5878687",
175 1e4: "b9f7a8e75ef22a835685a52ccba7f7d6bdc99e34b010992cbc5864cd12be6892",
176 1e5: "22776e6d4b49fa294a0d0f349268e5c28808fe7e0cb2bcbe28f63894e494d4c7",
177 1e6: "49415fee2c56c77864931bd3624faad425c3c577d6d74e89a83bc725506dad16",
178 1e7: "b9f8a44a91d46813b21b9602e72f112613c91408db0b8341fb94603d9db135e0",
179 1e8: "0f7dda6b0837dde083c5d6b896f7d62340c8a2415b0c7121d83145e08a755272",
180 }
181 wantHash := hashes[*testCanonicalNumberLines]
182 if wantHash == "" {
183 t.Fatalf("canonical-number-lines must be one of the following values: 1e3, 1e4, 1e5, 1e6, 1e7, 1e8")
184 }
185 numLines := int(*testCanonicalNumberLines)
186
187
188
189 generator := func() func() float64 {
190 static := [...]uint64{
191 0x0000000000000000, 0x8000000000000000, 0x0000000000000001, 0x8000000000000001,
192 0xc46696695dbd1cc3, 0xc43211ede4974a35, 0xc3fce97ca0f21056, 0xc3c7213080c1a6ac,
193 0xc39280f39a348556, 0xc35d9b1f5d20d557, 0xc327af4c4a80aaac, 0xc2f2f2a36ecd5556,
194 0xc2be51057e155558, 0xc28840d131aaaaac, 0xc253670dc1555557, 0xc21f0b4935555557,
195 0xc1e8d5d42aaaaaac, 0xc1b3de4355555556, 0xc17fca0555555556, 0xc1496e6aaaaaaaab,
196 0xc114585555555555, 0xc0e046aaaaaaaaab, 0xc0aa0aaaaaaaaaaa, 0xc074d55555555555,
197 0xc040aaaaaaaaaaab, 0xc00aaaaaaaaaaaab, 0xbfd5555555555555, 0xbfa1111111111111,
198 0xbf6b4e81b4e81b4f, 0xbf35d867c3ece2a5, 0xbf0179ec9cbd821e, 0xbecbf647612f3696,
199 0xbe965e9f80f29212, 0xbe61e54c672874db, 0xbe2ca213d840baf8, 0xbdf6e80fe033c8c6,
200 0xbdc2533fe68fd3d2, 0xbd8d51ffd74c861c, 0xbd5774ccac3d3817, 0xbd22c3d6f030f9ac,
201 0xbcee0624b3818f79, 0xbcb804ea293472c7, 0xbc833721ba905bd3, 0xbc4ebe9c5db3c61e,
202 0xbc18987d17c304e5, 0xbbe3ad30dfcf371d, 0xbbaf7b816618582f, 0xbb792f9ab81379bf,
203 0xbb442615600f9499, 0xbb101e77800c76e1, 0xbad9ca58cce0be35, 0xbaa4a1e0a3e6fe90,
204 0xba708180831f320d, 0xba3a68cd9e985016, 0x446696695dbd1cc3, 0x443211ede4974a35,
205 0x43fce97ca0f21056, 0x43c7213080c1a6ac, 0x439280f39a348556, 0x435d9b1f5d20d557,
206 0x4327af4c4a80aaac, 0x42f2f2a36ecd5556, 0x42be51057e155558, 0x428840d131aaaaac,
207 0x4253670dc1555557, 0x421f0b4935555557, 0x41e8d5d42aaaaaac, 0x41b3de4355555556,
208 0x417fca0555555556, 0x41496e6aaaaaaaab, 0x4114585555555555, 0x40e046aaaaaaaaab,
209 0x40aa0aaaaaaaaaaa, 0x4074d55555555555, 0x4040aaaaaaaaaaab, 0x400aaaaaaaaaaaab,
210 0x3fd5555555555555, 0x3fa1111111111111, 0x3f6b4e81b4e81b4f, 0x3f35d867c3ece2a5,
211 0x3f0179ec9cbd821e, 0x3ecbf647612f3696, 0x3e965e9f80f29212, 0x3e61e54c672874db,
212 0x3e2ca213d840baf8, 0x3df6e80fe033c8c6, 0x3dc2533fe68fd3d2, 0x3d8d51ffd74c861c,
213 0x3d5774ccac3d3817, 0x3d22c3d6f030f9ac, 0x3cee0624b3818f79, 0x3cb804ea293472c7,
214 0x3c833721ba905bd3, 0x3c4ebe9c5db3c61e, 0x3c18987d17c304e5, 0x3be3ad30dfcf371d,
215 0x3baf7b816618582f, 0x3b792f9ab81379bf, 0x3b442615600f9499, 0x3b101e77800c76e1,
216 0x3ad9ca58cce0be35, 0x3aa4a1e0a3e6fe90, 0x3a708180831f320d, 0x3a3a68cd9e985016,
217 0x4024000000000000, 0x4014000000000000, 0x3fe0000000000000, 0x3fa999999999999a,
218 0x3f747ae147ae147b, 0x3f40624dd2f1a9fc, 0x3f0a36e2eb1c432d, 0x3ed4f8b588e368f1,
219 0x3ea0c6f7a0b5ed8d, 0x3e6ad7f29abcaf48, 0x3e35798ee2308c3a, 0x3ed539223589fa95,
220 0x3ed4ff26cd5a7781, 0x3ed4f95a762283ff, 0x3ed4f8c60703520c, 0x3ed4f8b72f19cd0d,
221 0x3ed4f8b5b31c0c8d, 0x3ed4f8b58d1c461a, 0x3ed4f8b5894f7f0e, 0x3ed4f8b588ee37f3,
222 0x3ed4f8b588e47da4, 0x3ed4f8b588e3849c, 0x3ed4f8b588e36bb5, 0x3ed4f8b588e36937,
223 0x3ed4f8b588e368f8, 0x3ed4f8b588e368f1, 0x3ff0000000000000, 0xbff0000000000000,
224 0xbfeffffffffffffa, 0xbfeffffffffffffb, 0x3feffffffffffffa, 0x3feffffffffffffb,
225 0x3feffffffffffffc, 0x3feffffffffffffe, 0xbfefffffffffffff, 0xbfefffffffffffff,
226 0x3fefffffffffffff, 0x3fefffffffffffff, 0x3fd3333333333332, 0x3fd3333333333333,
227 0x3fd3333333333334, 0x0010000000000000, 0x000ffffffffffffd, 0x000fffffffffffff,
228 0x7fefffffffffffff, 0xffefffffffffffff, 0x4340000000000000, 0xc340000000000000,
229 0x4430000000000000, 0x44b52d02c7e14af5, 0x44b52d02c7e14af6, 0x44b52d02c7e14af7,
230 0x444b1ae4d6e2ef4e, 0x444b1ae4d6e2ef4f, 0x444b1ae4d6e2ef50, 0x3eb0c6f7a0b5ed8c,
231 0x3eb0c6f7a0b5ed8d, 0x41b3de4355555553, 0x41b3de4355555554, 0x41b3de4355555555,
232 0x41b3de4355555556, 0x41b3de4355555557, 0xbecbf647612f3696, 0x43143ff3c1cb0959,
233 }
234 var state struct {
235 idx int
236 data []byte
237 block [sha256.Size]byte
238 }
239 return func() float64 {
240 const numSerial = 2000
241 var f float64
242 switch {
243 case state.idx < len(static):
244 f = math.Float64frombits(static[state.idx])
245 case state.idx < len(static)+numSerial:
246 f = math.Float64frombits(0x0010000000000000 + uint64(state.idx-len(static)))
247 default:
248 for f == 0 || math.IsNaN(f) || math.IsInf(f, 0) {
249 if len(state.data) == 0 {
250 state.block = sha256.Sum256(state.block[:])
251 state.data = state.block[:]
252 }
253 f = math.Float64frombits(binary.LittleEndian.Uint64(state.data))
254 state.data = state.data[8:]
255 }
256 }
257 state.idx++
258 return f
259 }
260 }
261
262
263
264
265 for _, checkGolden := range []bool{false, true} {
266 var br *bufio.Reader
267 if checkGolden {
268 resp, err := http.Get(testfileURL)
269 if err != nil {
270 t.Fatalf("http.Get error: %v", err)
271 }
272 defer resp.Body.Close()
273
274 zr, err := gzip.NewReader(resp.Body)
275 if err != nil {
276 t.Fatalf("gzip.NewReader error: %v", err)
277 }
278
279 br = bufio.NewReader(zr)
280 }
281
282
283 appendNumberJCS := func(b []byte, f float64) []byte {
284 if math.Signbit(f) && f == 0 {
285 return append(b, '0')
286 }
287 return AppendFloat(b, f, 64)
288 }
289
290 var gotLine []byte
291 next := generator()
292 hash := sha256.New()
293 start := time.Now()
294 lastPrint := start
295 for n := 1; n <= numLines; n++ {
296
297 f := next()
298 gotLine = gotLine[:0]
299 gotLine = strconv.AppendUint(gotLine, math.Float64bits(f), 16)
300 gotLine = append(gotLine, ',')
301 gotLine = appendNumberJCS(gotLine, f)
302 gotLine = append(gotLine, '\n')
303 hash.Write(gotLine)
304
305
306 if checkGolden {
307 wantLine, err := br.ReadBytes('\n')
308 if err != nil {
309 t.Fatalf("bufio.Reader.ReadBytes error: %v", err)
310 }
311 if !bytes.Equal(gotLine, wantLine) {
312 t.Errorf("mismatch on line %d:\n\tgot %v\n\twant %v",
313 n, strings.TrimSpace(string(gotLine)), strings.TrimSpace(string(wantLine)))
314 }
315 }
316
317
318 if now := time.Now(); now.Sub(lastPrint) > time.Second || n == numLines {
319 remaining := float64(now.Sub(start)) * float64(numLines-n) / float64(n)
320 t.Logf("%0.3f%% (%v remaining)",
321 100.0*float64(n)/float64(numLines),
322 time.Duration(remaining).Round(time.Second))
323 lastPrint = now
324 }
325 }
326
327 gotHash := hex.EncodeToString(hash.Sum(nil))
328 if gotHash == wantHash {
329 return
330 }
331 }
332 }
333
View as plain text