mleku/next.orly.dev
1
0
Fork 1
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

implement filter codec

Browse Source
This commit is contained in:
mleku
2025-08-26 18:26:34 +01:00
parent c958a7d9ed
commit 1ba2bb0a9b
47 changed files with 687 additions and 19133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
pkg/encoders/json/internal/jsonflags/flags_test.go
View File

@@ -1,75 +0,0 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build goexperiment.jsonv2
package jsonflags
import "testing"
func TestFlags(t *testing.T) {
type Check struct{ want Flags }
type Join struct{ in Flags }
type Set struct{ in Bools }
type Clear struct{ in Bools }
type Get struct {
in Bools
want bool
wantOk bool
}
calls := []any{
Get{in: AllowDuplicateNames, want: false, wantOk: false},
Set{in: AllowDuplicateNames | 0},
Get{in: AllowDuplicateNames, want: false, wantOk: true},
Set{in: AllowDuplicateNames | 1},
Get{in: AllowDuplicateNames, want: true, wantOk: true},
Check{want: Flags{Presence: uint64(AllowDuplicateNames), Values: uint64(AllowDuplicateNames)}},
Get{in: AllowInvalidUTF8, want: false, wantOk: false},
Set{in: AllowInvalidUTF8 | 1},
Get{in: AllowInvalidUTF8, want: true, wantOk: true},
Set{in: AllowInvalidUTF8 | 0},
Get{in: AllowInvalidUTF8, want: false, wantOk: true},
Get{in: AllowDuplicateNames, want: true, wantOk: true},
Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames)}},
Set{in: AllowDuplicateNames | AllowInvalidUTF8 | 0},
Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(0)}},
Set{in: AllowDuplicateNames | AllowInvalidUTF8 | 0},
Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(0)}},
Set{in: AllowDuplicateNames | AllowInvalidUTF8 | 1},
Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames | AllowInvalidUTF8)}},
Join{in: Flags{Presence: 0, Values: 0}},
Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames | AllowInvalidUTF8)}},
Join{in: Flags{Presence: uint64(Multiline | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames)}},
Check{want: Flags{Presence: uint64(Multiline | AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames)}},
Clear{in: AllowDuplicateNames | AllowInvalidUTF8},
Check{want: Flags{Presence: uint64(Multiline), Values: uint64(0)}},
Set{in: AllowInvalidUTF8 | Deterministic | ReportErrorsWithLegacySemantics | 1},
Set{in: Multiline | StringifyNumbers | 0},
Check{want: Flags{Presence: uint64(AllowInvalidUTF8 | Deterministic | ReportErrorsWithLegacySemantics | Multiline | StringifyNumbers), Values: uint64(AllowInvalidUTF8 | Deterministic | ReportErrorsWithLegacySemantics)}},
Clear{in: ^AllCoderFlags},
Check{want: Flags{Presence: uint64(AllowInvalidUTF8 | Multiline), Values: uint64(AllowInvalidUTF8)}},
}
var fs Flags
for i, call := range calls {
switch call := call.(type) {
case Join:
fs.Join(call.in)
case Set:
fs.Set(call.in)
case Clear:
fs.Clear(call.in)
case Get:
got := fs.Get(call.in)
gotOk := fs.Has(call.in)
if got != call.want || gotOk != call.wantOk {
t.Fatalf("%d: GetOk = (%v, %v), want (%v, %v)", i, got, gotOk, call.want, call.wantOk)
}
case Check:
if fs != call.want {
t.Fatalf("%d: got %x, want %x", i, fs, call.want)
}
}
}
}

233
pkg/encoders/json/internal/jsonopts/options_test.go
View File

@@ -1,233 +0,0 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build goexperiment.jsonv2
package jsonopts_test
import (
"reflect"
"testing"
"encoding/json/internal/jsonflags"
. "encoding/json/internal/jsonopts"
"encoding/json/jsontext"
"encoding/json/v2"
)
func makeFlags(f ...jsonflags.Bools) (fs jsonflags.Flags) {
for _, f := range f {
fs.Set(f)
}
return fs
}
func TestJoin(t *testing.T) {
tests := []struct {
in Options
excludeCoders bool
want *Struct
}{{
in: jsonflags.AllowInvalidUTF8 | 1,
want: &Struct{Flags: makeFlags(jsonflags.AllowInvalidUTF8 | 1)},
}, {
in: jsonflags.Multiline | 0,
want: &Struct{
Flags: makeFlags(jsonflags.AllowInvalidUTF8|1, jsonflags.Multiline|0)},
}, {
in: Indent("\t"), // implicitly sets Multiline=true
want: &Struct{
Flags: makeFlags(jsonflags.AllowInvalidUTF8 | jsonflags.Multiline | jsonflags.Indent | 1),
CoderValues: CoderValues{Indent: "\t"},
},
}, {
in: &Struct{
Flags: makeFlags(jsonflags.Multiline|jsonflags.EscapeForJS|0, jsonflags.AllowInvalidUTF8|1),
},
want: &Struct{
Flags: makeFlags(jsonflags.AllowInvalidUTF8|jsonflags.Indent|1, jsonflags.Multiline|jsonflags.EscapeForJS|0),
CoderValues: CoderValues{Indent: "\t"},
},
}, {
in: &DefaultOptionsV1,
want: func() *Struct {
v1 := DefaultOptionsV1
v1.Flags.Set(jsonflags.Indent | 1)
v1.Flags.Set(jsonflags.Multiline | 0)
v1.Indent = "\t"
return &v1
}(), // v1 fully replaces before (except for whitespace related flags)
}, {
in: &DefaultOptionsV2,
want: func() *Struct {
v2 := DefaultOptionsV2
v2.Flags.Set(jsonflags.Indent | 1)
v2.Flags.Set(jsonflags.Multiline | 0)
v2.Indent = "\t"
return &v2
}(), // v2 fully replaces before (except for whitespace related flags)
}, {
in: jsonflags.Deterministic | jsonflags.AllowInvalidUTF8 | 1, excludeCoders: true,
want: func() *Struct {
v2 := DefaultOptionsV2
v2.Flags.Set(jsonflags.Deterministic | 1)
v2.Flags.Set(jsonflags.Indent | 1)
v2.Flags.Set(jsonflags.Multiline | 0)
v2.Indent = "\t"
return &v2
}(),
}, {
in: jsontext.WithIndentPrefix(" "), excludeCoders: true,
want: func() *Struct {
v2 := DefaultOptionsV2
v2.Flags.Set(jsonflags.Deterministic | 1)
v2.Flags.Set(jsonflags.Indent | 1)
v2.Flags.Set(jsonflags.Multiline | 0)
v2.Indent = "\t"
return &v2
}(),
}, {
in: jsontext.WithIndentPrefix(" "), excludeCoders: false,
want: func() *Struct {
v2 := DefaultOptionsV2
v2.Flags.Set(jsonflags.Deterministic | 1)
v2.Flags.Set(jsonflags.Indent | 1)
v2.Flags.Set(jsonflags.IndentPrefix | 1)
v2.Flags.Set(jsonflags.Multiline | 1)
v2.Indent = "\t"
v2.IndentPrefix = " "
return &v2
}(),
}, {
in: &Struct{
Flags: jsonflags.Flags{
Presence: uint64(jsonflags.Deterministic | jsonflags.Indent | jsonflags.IndentPrefix),
Values: uint64(jsonflags.Indent | jsonflags.IndentPrefix),
},
CoderValues: CoderValues{Indent: " ", IndentPrefix: " "},
},
excludeCoders: true,
want: func() *Struct {
v2 := DefaultOptionsV2
v2.Flags.Set(jsonflags.Indent | 1)
v2.Flags.Set(jsonflags.IndentPrefix | 1)
v2.Flags.Set(jsonflags.Multiline | 1)
v2.Indent = "\t"
v2.IndentPrefix = " "
return &v2
}(),
}, {
in: &Struct{
Flags: jsonflags.Flags{
Presence: uint64(jsonflags.Deterministic | jsonflags.Indent | jsonflags.IndentPrefix),
Values: uint64(jsonflags.Indent | jsonflags.IndentPrefix),
},
CoderValues: CoderValues{Indent: " ", IndentPrefix: " "},
},
excludeCoders: false,
want: func() *Struct {
v2 := DefaultOptionsV2
v2.Flags.Set(jsonflags.Indent | 1)
v2.Flags.Set(jsonflags.IndentPrefix | 1)
v2.Flags.Set(jsonflags.Multiline | 1)
v2.Indent = " "
v2.IndentPrefix = " "
return &v2
}(),
}}
got := new(Struct)
for i, tt := range tests {
if tt.excludeCoders {
got.JoinWithoutCoderOptions(tt.in)
} else {
got.Join(tt.in)
}
if !reflect.DeepEqual(got, tt.want) {
t.Fatalf("%d: Join:\n\tgot: %+v\n\twant: %+v", i, got, tt.want)
}
}
}
func TestGet(t *testing.T) {
opts := &Struct{
Flags: makeFlags(jsonflags.Indent|jsonflags.Deterministic|jsonflags.Marshalers|1, jsonflags.Multiline|0),
CoderValues: CoderValues{Indent: "\t"},
ArshalValues: ArshalValues{Marshalers: new(json.Marshalers)},
}
if v, ok := json.GetOption(nil, jsontext.AllowDuplicateNames); v || ok {
t.Errorf("GetOption(..., AllowDuplicateNames) = (%v, %v), want (false, false)", v, ok)
}
if v, ok := json.GetOption(jsonflags.AllowInvalidUTF8|0, jsontext.AllowDuplicateNames); v || ok {
t.Errorf("GetOption(..., AllowDuplicateNames) = (%v, %v), want (false, false)", v, ok)
}
if v, ok := json.GetOption(jsonflags.AllowDuplicateNames|0, jsontext.AllowDuplicateNames); v || !ok {
t.Errorf("GetOption(..., AllowDuplicateNames) = (%v, %v), want (false, true)", v, ok)
}
if v, ok := json.GetOption(jsonflags.AllowDuplicateNames|1, jsontext.AllowDuplicateNames); !v || !ok {
t.Errorf("GetOption(..., AllowDuplicateNames) = (%v, %v), want (true, true)", v, ok)
}
if v, ok := json.GetOption(Indent(""), jsontext.AllowDuplicateNames); v || ok {
t.Errorf("GetOption(..., AllowDuplicateNames) = (%v, %v), want (false, false)", v, ok)
}
if v, ok := json.GetOption(Indent(" "), jsontext.WithIndent); v != " " || !ok {
t.Errorf(`GetOption(..., WithIndent) = (%q, %v), want (" ", true)`, v, ok)
}
if v, ok := json.GetOption(jsonflags.AllowDuplicateNames|1, jsontext.WithIndent); v != "" || ok {
t.Errorf(`GetOption(..., WithIndent) = (%q, %v), want ("", false)`, v, ok)
}
if v, ok := json.GetOption(opts, jsontext.AllowDuplicateNames); v || ok {
t.Errorf("GetOption(..., AllowDuplicateNames) = (%v, %v), want (false, false)", v, ok)
}
if v, ok := json.GetOption(opts, json.Deterministic); !v || !ok {
t.Errorf("GetOption(..., Deterministic) = (%v, %v), want (true, true)", v, ok)
}
if v, ok := json.GetOption(opts, jsontext.Multiline); v || !ok {
t.Errorf("GetOption(..., Multiline) = (%v, %v), want (false, true)", v, ok)
}
if v, ok := json.GetOption(opts, jsontext.AllowInvalidUTF8); v || ok {
t.Errorf("GetOption(..., AllowInvalidUTF8) = (%v, %v), want (false, false)", v, ok)
}
if v, ok := json.GetOption(opts, jsontext.WithIndent); v != "\t" || !ok {
t.Errorf(`GetOption(..., WithIndent) = (%q, %v), want ("\t", true)`, v, ok)
}
if v, ok := json.GetOption(opts, jsontext.WithIndentPrefix); v != "" || ok {
t.Errorf(`GetOption(..., WithIndentPrefix) = (%q, %v), want ("", false)`, v, ok)
}
if v, ok := json.GetOption(opts, json.WithMarshalers); v == nil || !ok {
t.Errorf(`GetOption(..., WithMarshalers) = (%v, %v), want (non-nil, true)`, v, ok)
}
if v, ok := json.GetOption(opts, json.WithUnmarshalers); v != nil || ok {
t.Errorf(`GetOption(..., WithUnmarshalers) = (%v, %v), want (nil, false)`, v, ok)
}
}
var sink struct {
Bool bool
String string
Marshalers *json.Marshalers
}
func BenchmarkGetBool(b *testing.B) {
b.ReportAllocs()
opts := json.DefaultOptionsV2()
for range b.N {
sink.Bool, sink.Bool = json.GetOption(opts, jsontext.AllowDuplicateNames)
}
}
func BenchmarkGetIndent(b *testing.B) {
b.ReportAllocs()
opts := json.DefaultOptionsV2()
for range b.N {
sink.String, sink.Bool = json.GetOption(opts, jsontext.WithIndent)
}
}
func BenchmarkGetMarshalers(b *testing.B) {
b.ReportAllocs()
opts := json.JoinOptions(json.DefaultOptionsV2(), json.WithMarshalers(nil))
for range b.N {
sink.Marshalers, sink.Bool = json.GetOption(opts, json.WithMarshalers)
}
}

443
pkg/encoders/json/internal/jsonwire/decode_test.go
View File

@@ -1,443 +0,0 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build goexperiment.jsonv2
package jsonwire
import (
"errors"
"io"
"math"
"reflect"
"strings"
"testing"
)
func TestConsumeWhitespace(t *testing.T) {
tests := []struct {
in string
want int
}{
{"", 0},
{"a", 0},
{" a", 1},
{" a ", 1},
{" \n\r\ta", 4},
{" \n\r\t \n\r\t \n\r\t \n\r\t", 16},
{"\u00a0", 0}, // non-breaking space is not JSON whitespace
}
for _, tt := range tests {
t.Run("", func(t *testing.T) {
if got := ConsumeWhitespace([]byte(tt.in)); got != tt.want {
t.Errorf("ConsumeWhitespace(%q) = %v, want %v", tt.in, got, tt.want)
}
})
}
}
func TestConsumeLiteral(t *testing.T) {
tests := []struct {
literal string
in string
want int
wantErr error
}{
{"null", "", 0, io.ErrUnexpectedEOF},
{"null", "n", 1, io.ErrUnexpectedEOF},
{"null", "nu", 2, io.ErrUnexpectedEOF},
{"null", "nul", 3, io.ErrUnexpectedEOF},
{"null", "null", 4, nil},
{"null", "nullx", 4, nil},
{"null", "x", 0, NewInvalidCharacterError("x", "in literal null (expecting 'n')")},
{"null", "nuxx", 2, NewInvalidCharacterError("x", "in literal null (expecting 'l')")},
{"false", "", 0, io.ErrUnexpectedEOF},
{"false", "f", 1, io.ErrUnexpectedEOF},
{"false", "fa", 2, io.ErrUnexpectedEOF},
{"false", "fal", 3, io.ErrUnexpectedEOF},
{"false", "fals", 4, io.ErrUnexpectedEOF},
{"false", "false", 5, nil},
{"false", "falsex", 5, nil},
{"false", "x", 0, NewInvalidCharacterError("x", "in literal false (expecting 'f')")},
{"false", "falsx", 4, NewInvalidCharacterError("x", "in literal false (expecting 'e')")},
{"true", "", 0, io.ErrUnexpectedEOF},
{"true", "t", 1, io.ErrUnexpectedEOF},
{"true", "tr", 2, io.ErrUnexpectedEOF},
{"true", "tru", 3, io.ErrUnexpectedEOF},
{"true", "true", 4, nil},
{"true", "truex", 4, nil},
{"true", "x", 0, NewInvalidCharacterError("x", "in literal true (expecting 't')")},
{"true", "trux", 3, NewInvalidCharacterError("x", "in literal true (expecting 'e')")},
}
for _, tt := range tests {
t.Run("", func(t *testing.T) {
var got int
switch tt.literal {
case "null":
got = ConsumeNull([]byte(tt.in))
case "false":
got = ConsumeFalse([]byte(tt.in))
case "true":
got = ConsumeTrue([]byte(tt.in))
default:
t.Errorf("invalid literal: %v", tt.literal)
}
switch {
case tt.wantErr == nil && got != tt.want:
t.Errorf("Consume%v(%q) = %v, want %v", strings.Title(tt.literal), tt.in, got, tt.want)
case tt.wantErr != nil && got != 0:
t.Errorf("Consume%v(%q) = %v, want %v", strings.Title(tt.literal), tt.in, got, 0)
}
got, gotErr := ConsumeLiteral([]byte(tt.in), tt.literal)
if got != tt.want || !reflect.DeepEqual(gotErr, tt.wantErr) {
t.Errorf("ConsumeLiteral(%q, %q) = (%v, %v), want (%v, %v)", tt.in, tt.literal, got, gotErr, tt.want, tt.wantErr)
}
})
}
}
func TestConsumeString(t *testing.T) {
var errPrev = errors.New("same as previous error")
tests := []struct {
in string
simple bool
want int
wantUTF8 int // consumed bytes if validateUTF8 is specified
wantFlags ValueFlags
wantUnquote string
wantErr error
wantErrUTF8 error // error if validateUTF8 is specified
wantErrUnquote error
}{
{``, false, 0, 0, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"`, false, 1, 1, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev},
{`""`, true, 2, 2, 0, "", nil, nil, nil},
{`""x`, true, 2, 2, 0, "", nil, nil, NewInvalidCharacterError("x", "after string value")},
{` ""x`, false, 0, 0, 0, "", NewInvalidCharacterError(" ", "at start of string (expecting '\"')"), errPrev, errPrev},
{`"hello`, false, 6, 6, 0, "hello", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"hello"`, true, 7, 7, 0, "hello", nil, nil, nil},
{"\"\x00\"", false, 1, 1, stringNonVerbatim | stringNonCanonical, "", NewInvalidCharacterError("\x00", "in string (expecting non-control character)"), errPrev, errPrev},
{`"\u0000"`, false, 8, 8, stringNonVerbatim, "\x00", nil, nil, nil},
{"\"\x1f\"", false, 1, 1, stringNonVerbatim | stringNonCanonical, "", NewInvalidCharacterError("\x1f", "in string (expecting non-control character)"), errPrev, errPrev},
{`"\u001f"`, false, 8, 8, stringNonVerbatim, "\x1f", nil, nil, nil},
{`"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"`, true, 54, 54, 0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", nil, nil, nil},
{"\" !#$%'()*+,-./0123456789:;=?@[]^_`{|}~\x7f\"", true, 41, 41, 0, " !#$%'()*+,-./0123456789:;=?@[]^_`{|}~\x7f", nil, nil, nil},
{`"&"`, false, 3, 3, 0, "&", nil, nil, nil},
{`"<"`, false, 3, 3, 0, "<", nil, nil, nil},
{`">"`, false, 3, 3, 0, ">", nil, nil, nil},
{"\"x\x80\"", false, 4, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd", nil, ErrInvalidUTF8, errPrev},
{"\"x\xff\"", false, 4, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd", nil, ErrInvalidUTF8, errPrev},
{"\"x\xc0", false, 3, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd", io.ErrUnexpectedEOF, ErrInvalidUTF8, io.ErrUnexpectedEOF},
{"\"x\xc0\x80\"", false, 5, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd\ufffd", nil, ErrInvalidUTF8, errPrev},
{"\"x\xe0", false, 2, 2, 0, "x", io.ErrUnexpectedEOF, errPrev, errPrev},
{"\"x\xe0\x80", false, 4, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd\ufffd", io.ErrUnexpectedEOF, ErrInvalidUTF8, io.ErrUnexpectedEOF},
{"\"x\xe0\x80\x80\"", false, 6, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd\ufffd\ufffd", nil, ErrInvalidUTF8, errPrev},
{"\"x\xf0", false, 2, 2, 0, "x", io.ErrUnexpectedEOF, errPrev, errPrev},
{"\"x\xf0\x80", false, 4, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd\ufffd", io.ErrUnexpectedEOF, ErrInvalidUTF8, io.ErrUnexpectedEOF},
{"\"x\xf0\x80\x80", false, 5, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd\ufffd\ufffd", io.ErrUnexpectedEOF, ErrInvalidUTF8, io.ErrUnexpectedEOF},
{"\"x\xf0\x80\x80\x80\"", false, 7, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd\ufffd\ufffd\ufffd", nil, ErrInvalidUTF8, errPrev},
{"\"x\xed\xba\xad\"", false, 6, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd\ufffd\ufffd", nil, ErrInvalidUTF8, errPrev},
{"\"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602\"", false, 25, 25, 0, "\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602", nil, nil, nil},
{`"¢"`[:2], false, 1, 1, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"¢"`[:3], false, 3, 3, 0, "¢", io.ErrUnexpectedEOF, errPrev, errPrev}, // missing terminating quote
{`"¢"`[:4], false, 4, 4, 0, "¢", nil, nil, nil},
{`"€"`[:2], false, 1, 1, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"€"`[:3], false, 1, 1, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"€"`[:4], false, 4, 4, 0, "€", io.ErrUnexpectedEOF, errPrev, errPrev}, // missing terminating quote
{`"€"`[:5], false, 5, 5, 0, "€", nil, nil, nil},
{`"𐍈"`[:2], false, 1, 1, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"𐍈"`[:3], false, 1, 1, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"𐍈"`[:4], false, 1, 1, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"𐍈"`[:5], false, 5, 5, 0, "𐍈", io.ErrUnexpectedEOF, errPrev, errPrev}, // missing terminating quote
{`"𐍈"`[:6], false, 6, 6, 0, "𐍈", nil, nil, nil},
{`"x\`, false, 2, 2, stringNonVerbatim, "x", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"x\"`, false, 4, 4, stringNonVerbatim, "x\"", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"x\x"`, false, 2, 2, stringNonVerbatim | stringNonCanonical, "x", NewInvalidEscapeSequenceError(`\x`), errPrev, errPrev},
{`"\"\\\b\f\n\r\t"`, false, 16, 16, stringNonVerbatim, "\"\\\b\f\n\r\t", nil, nil, nil},
{`"/"`, true, 3, 3, 0, "/", nil, nil, nil},
{`"\/"`, false, 4, 4, stringNonVerbatim | stringNonCanonical, "/", nil, nil, nil},
{`"\u002f"`, false, 8, 8, stringNonVerbatim | stringNonCanonical, "/", nil, nil, nil},
{`"\u`, false, 1, 1, stringNonVerbatim, "", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"\uf`, false, 1, 1, stringNonVerbatim, "", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"\uff`, false, 1, 1, stringNonVerbatim, "", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"\ufff`, false, 1, 1, stringNonVerbatim, "", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"\ufffd`, false, 7, 7, stringNonVerbatim | stringNonCanonical, "\ufffd", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"\ufffd"`, false, 8, 8, stringNonVerbatim | stringNonCanonical, "\ufffd", nil, nil, nil},
{`"\uABCD"`, false, 8, 8, stringNonVerbatim | stringNonCanonical, "\uabcd", nil, nil, nil},
{`"\uefX0"`, false, 1, 1, stringNonVerbatim | stringNonCanonical, "", NewInvalidEscapeSequenceError(`\uefX0`), errPrev, errPrev},
{`"\uDEAD`, false, 7, 1, stringNonVerbatim | stringNonCanonical, "\ufffd", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"\uDEAD"`, false, 8, 1, stringNonVerbatim | stringNonCanonical, "\ufffd", nil, NewInvalidEscapeSequenceError(`\uDEAD"`), errPrev},
{`"\uDEAD______"`, false, 14, 1, stringNonVerbatim | stringNonCanonical, "\ufffd______", nil, NewInvalidEscapeSequenceError(`\uDEAD______`), errPrev},
{`"\uDEAD\uXXXX"`, false, 7, 1, stringNonVerbatim | stringNonCanonical, "\ufffd", NewInvalidEscapeSequenceError(`\uXXXX`), NewInvalidEscapeSequenceError(`\uDEAD\uXXXX`), NewInvalidEscapeSequenceError(`\uXXXX`)},
{`"\uDEAD\uBEEF"`, false, 14, 1, stringNonVerbatim | stringNonCanonical, "\ufffd\ubeef", nil, NewInvalidEscapeSequenceError(`\uDEAD\uBEEF`), errPrev},
{`"\uD800\udea`, false, 7, 1, stringNonVerbatim | stringNonCanonical, "\ufffd", io.ErrUnexpectedEOF, errPrev, errPrev},
{`"\uD800\udb`, false, 7, 1, stringNonVerbatim | stringNonCanonical, "\ufffd", io.ErrUnexpectedEOF, NewInvalidEscapeSequenceError(`\uD800\udb`), io.ErrUnexpectedEOF},
{`"\uD800\udead"`, false, 14, 14, stringNonVerbatim | stringNonCanonical, "\U000102ad", nil, nil, nil},
{`"\u0022\u005c\u002f\u0008\u000c\u000a\u000d\u0009"`, false, 50, 50, stringNonVerbatim | stringNonCanonical, "\"\\/\b\f\n\r\t", nil, nil, nil},
{`"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\ud83d\ude02"`, false, 56, 56, stringNonVerbatim | stringNonCanonical, "\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602", nil, nil, nil},
}
for _, tt := range tests {
t.Run("", func(t *testing.T) {
if tt.wantErrUTF8 == errPrev {
tt.wantErrUTF8 = tt.wantErr
}
if tt.wantErrUnquote == errPrev {
tt.wantErrUnquote = tt.wantErrUTF8
}
switch got := ConsumeSimpleString([]byte(tt.in)); {
case tt.simple && got != tt.want:
t.Errorf("consumeSimpleString(%q) = %v, want %v", tt.in, got, tt.want)
case !tt.simple && got != 0:
t.Errorf("consumeSimpleString(%q) = %v, want %v", tt.in, got, 0)
}
var gotFlags ValueFlags
got, gotErr := ConsumeString(&gotFlags, []byte(tt.in), false)
if gotFlags != tt.wantFlags {
t.Errorf("consumeString(%q, false) flags = %v, want %v", tt.in, gotFlags, tt.wantFlags)
}
if got != tt.want || !reflect.DeepEqual(gotErr, tt.wantErr) {
t.Errorf("consumeString(%q, false) = (%v, %v), want (%v, %v)", tt.in, got, gotErr, tt.want, tt.wantErr)
}
got, gotErr = ConsumeString(&gotFlags, []byte(tt.in), true)
if got != tt.wantUTF8 || !reflect.DeepEqual(gotErr, tt.wantErrUTF8) {
t.Errorf("consumeString(%q, false) = (%v, %v), want (%v, %v)", tt.in, got, gotErr, tt.wantUTF8, tt.wantErrUTF8)
}
gotUnquote, gotErr := AppendUnquote(nil, tt.in)
if string(gotUnquote) != tt.wantUnquote || !reflect.DeepEqual(gotErr, tt.wantErrUnquote) {
t.Errorf("AppendUnquote(nil, %q) = (%q, %v), want (%q, %v)", tt.in[:got], gotUnquote, gotErr, tt.wantUnquote, tt.wantErrUnquote)
}
})
}
}
func TestConsumeNumber(t *testing.T) {
tests := []struct {
in string
simple bool
want int
wantErr error
}{
{"", false, 0, io.ErrUnexpectedEOF},
{`"NaN"`, false, 0, NewInvalidCharacterError("\"", "in number (expecting digit)")},
{`"Infinity"`, false, 0, NewInvalidCharacterError("\"", "in number (expecting digit)")},
{`"-Infinity"`, false, 0, NewInvalidCharacterError("\"", "in number (expecting digit)")},
{".0", false, 0, NewInvalidCharacterError(".", "in number (expecting digit)")},
{"0", true, 1, nil},
{"-0", false, 2, nil},
{"+0", false, 0, NewInvalidCharacterError("+", "in number (expecting digit)")},
{"1", true, 1, nil},
{"-1", false, 2, nil},
{"00", true, 1, nil},
{"-00", false, 2, nil},
{"01", true, 1, nil},
{"-01", false, 2, nil},
{"0i", true, 1, nil},
{"-0i", false, 2, nil},
{"0f", true, 1, nil},
{"-0f", false, 2, nil},
{"9876543210", true, 10, nil},
{"-9876543210", false, 11, nil},
{"9876543210x", true, 10, nil},
{"-9876543210x", false, 11, nil},
{" 9876543210", true, 0, NewInvalidCharacterError(" ", "in number (expecting digit)")},
{"- 9876543210", false, 1, NewInvalidCharacterError(" ", "in number (expecting digit)")},
{strings.Repeat("9876543210", 1000), true, 10000, nil},
{"-" + strings.Repeat("9876543210", 1000), false, 1 + 10000, nil},
{"0.", false, 1, io.ErrUnexpectedEOF},
{"-0.", false, 2, io.ErrUnexpectedEOF},
{"0e", false, 1, io.ErrUnexpectedEOF},
{"-0e", false, 2, io.ErrUnexpectedEOF},
{"0E", false, 1, io.ErrUnexpectedEOF},
{"-0E", false, 2, io.ErrUnexpectedEOF},
{"0.0", false, 3, nil},
{"-0.0", false, 4, nil},
{"0e0", false, 3, nil},
{"-0e0", false, 4, nil},
{"0E0", false, 3, nil},
{"-0E0", false, 4, nil},
{"0.0123456789", false, 12, nil},
{"-0.0123456789", false, 13, nil},
{"1.f", false, 2, NewInvalidCharacterError("f", "in number (expecting digit)")},
{"-1.f", false, 3, NewInvalidCharacterError("f", "in number (expecting digit)")},
{"1.e", false, 2, NewInvalidCharacterError("e", "in number (expecting digit)")},
{"-1.e", false, 3, NewInvalidCharacterError("e", "in number (expecting digit)")},
{"1e0", false, 3, nil},
{"-1e0", false, 4, nil},
{"1E0", false, 3, nil},
{"-1E0", false, 4, nil},
{"1Ex", false, 2, NewInvalidCharacterError("x", "in number (expecting digit)")},
{"-1Ex", false, 3, NewInvalidCharacterError("x", "in number (expecting digit)")},
{"1e-0", false, 4, nil},
{"-1e-0", false, 5, nil},
{"1e+0", false, 4, nil},
{"-1e+0", false, 5, nil},
{"1E-0", false, 4, nil},
{"-1E-0", false, 5, nil},
{"1E+0", false, 4, nil},
{"-1E+0", false, 5, nil},
{"1E+00500", false, 8, nil},
{"-1E+00500", false, 9, nil},
{"1E+00500x", false, 8, nil},
{"-1E+00500x", false, 9, nil},
{"9876543210.0123456789e+01234589x", false, 31, nil},
{"-9876543210.0123456789e+01234589x", false, 32, nil},
{"1_000_000", true, 1, nil},
{"0x12ef", true, 1, nil},
{"0x1p-2", true, 1, nil},
}
for _, tt := range tests {
t.Run("", func(t *testing.T) {
switch got := ConsumeSimpleNumber([]byte(tt.in)); {
case tt.simple && got != tt.want:
t.Errorf("ConsumeSimpleNumber(%q) = %v, want %v", tt.in, got, tt.want)
case !tt.simple && got != 0:
t.Errorf("ConsumeSimpleNumber(%q) = %v, want %v", tt.in, got, 0)
}
got, gotErr := ConsumeNumber([]byte(tt.in))
if got != tt.want || !reflect.DeepEqual(gotErr, tt.wantErr) {
t.Errorf("ConsumeNumber(%q) = (%v, %v), want (%v, %v)", tt.in, got, gotErr, tt.want, tt.wantErr)
}
})
}
}
func TestParseHexUint16(t *testing.T) {
tests := []struct {
in string
want uint16
wantOk bool
}{
{"", 0, false},
{"a", 0, false},
{"ab", 0, false},
{"abc", 0, false},
{"abcd", 0xabcd, true},
{"abcde", 0, false},
{"9eA1", 0x9ea1, true},
{"gggg", 0, false},
{"0000", 0x0000, true},
{"1234", 0x1234, true},
}
for _, tt := range tests {
t.Run("", func(t *testing.T) {
got, gotOk := parseHexUint16([]byte(tt.in))
if got != tt.want || gotOk != tt.wantOk {
t.Errorf("parseHexUint16(%q) = (0x%04x, %v), want (0x%04x, %v)", tt.in, got, gotOk, tt.want, tt.wantOk)
}
})
}
}
func TestParseUint(t *testing.T) {
tests := []struct {
in string
want uint64
wantOk bool
}{
{"", 0, false},
{"0", 0, true},
{"1", 1, true},
{"-1", 0, false},
{"1f", 0, false},
{"00", 0, false},
{"01", 0, false},
{"10", 10, true},
{"10.9", 0, false},
{" 10", 0, false},
{"10 ", 0, false},
{"123456789", 123456789, true},
{"123456789d", 0, false},
{"18446744073709551614", math.MaxUint64 - 1, true},
{"18446744073709551615", math.MaxUint64, true},
{"18446744073709551616", math.MaxUint64, false},
{"18446744073709551620", math.MaxUint64, false},
{"18446744073709551700", math.MaxUint64, false},
{"18446744073709552000", math.MaxUint64, false},
{"18446744073709560000", math.MaxUint64, false},
{"18446744073709600000", math.MaxUint64, false},
{"18446744073710000000", math.MaxUint64, false},
{"18446744073800000000", math.MaxUint64, false},
{"18446744074000000000", math.MaxUint64, false},
{"18446744080000000000", math.MaxUint64, false},
{"18446744100000000000", math.MaxUint64, false},
{"18446745000000000000", math.MaxUint64, false},
{"18446750000000000000", math.MaxUint64, false},
{"18446800000000000000", math.MaxUint64, false},
{"18447000000000000000", math.MaxUint64, false},
{"18450000000000000000", math.MaxUint64, false},
{"18500000000000000000", math.MaxUint64, false},
{"19000000000000000000", math.MaxUint64, false},
{"19999999999999999999", math.MaxUint64, false},
{"20000000000000000000", math.MaxUint64, false},
{"100000000000000000000", math.MaxUint64, false},
{"99999999999999999999999999999999", math.MaxUint64, false},
{"99999999999999999999999999999999f", 0, false},
}
for _, tt := range tests {
t.Run("", func(t *testing.T) {
got, gotOk := ParseUint([]byte(tt.in))
if got != tt.want || gotOk != tt.wantOk {
t.Errorf("ParseUint(%q) = (%v, %v), want (%v, %v)", tt.in, got, gotOk, tt.want, tt.wantOk)
}
})
}
}
func TestParseFloat(t *testing.T) {
tests := []struct {
in string
want32 float64
want64 float64
wantOk bool
}{
{"0", 0, 0, true},
{"-1", -1, -1, true},
{"1", 1, 1, true},
{"-16777215", -16777215, -16777215, true}, // -(1<<24 - 1)
{"16777215", 16777215, 16777215, true}, // +(1<<24 - 1)
{"-16777216", -16777216, -16777216, true}, // -(1<<24)
{"16777216", 16777216, 16777216, true}, // +(1<<24)
{"-16777217", -16777216, -16777217, true}, // -(1<<24 + 1)
{"16777217", 16777216, 16777217, true}, // +(1<<24 + 1)
{"-9007199254740991", -9007199254740992, -9007199254740991, true}, // -(1<<53 - 1)
{"9007199254740991", 9007199254740992, 9007199254740991, true}, // +(1<<53 - 1)
{"-9007199254740992", -9007199254740992, -9007199254740992, true}, // -(1<<53)
{"9007199254740992", 9007199254740992, 9007199254740992, true}, // +(1<<53)
{"-9007199254740993", -9007199254740992, -9007199254740992, true}, // -(1<<53 + 1)
{"9007199254740993", 9007199254740992, 9007199254740992, true}, // +(1<<53 + 1)
{"-1e1000", -math.MaxFloat32, -math.MaxFloat64, false},
{"1e1000", +math.MaxFloat32, +math.MaxFloat64, false},
}
for _, tt := range tests {
t.Run("", func(t *testing.T) {
got32, gotOk32 := ParseFloat([]byte(tt.in), 32)
if got32 != tt.want32 || gotOk32 != tt.wantOk {
t.Errorf("ParseFloat(%q, 32) = (%v, %v), want (%v, %v)", tt.in, got32, gotOk32, tt.want32, tt.wantOk)
}
got64, gotOk64 := ParseFloat([]byte(tt.in), 64)
if got64 != tt.want64 || gotOk64 != tt.wantOk {
t.Errorf("ParseFloat(%q, 64) = (%v, %v), want (%v, %v)", tt.in, got64, gotOk64, tt.want64, tt.wantOk)
}
})
}
}

332
pkg/encoders/json/internal/jsonwire/encode_test.go
View File

@@ -1,332 +0,0 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build goexperiment.jsonv2
package jsonwire
import (
"bufio"
"bytes"
"compress/gzip"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"flag"
"math"
"net/http"
"reflect"
"strconv"
"strings"
"testing"
"time"
"encoding/json/internal/jsonflags"
)
func TestAppendQuote(t *testing.T) {
tests := []struct {
in string
flags jsonflags.Bools
want string
wantErr error
wantErrUTF8 error
}{
{"", 0, `""`, nil, nil},
{"hello", 0, `"hello"`, nil, nil},
{"\x00", 0, `"\u0000"`, nil, nil},
{"\x1f", 0, `"\u001f"`, nil, nil},
{"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", 0, `"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"`, nil, nil},
{" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f", 0, "\" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f\"", nil, nil},
{" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f", jsonflags.EscapeForHTML, "\" !#$%\\u0026'()*+,-./0123456789:;\\u003c=\\u003e?@[]^_`{|}~\x7f\"", nil, nil},
{" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f", jsonflags.EscapeForJS, "\" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f\"", nil, nil},
{"\u2027\u2028\u2029\u2030", 0, "\"\u2027\u2028\u2029\u2030\"", nil, nil},
{"\u2027\u2028\u2029\u2030", jsonflags.EscapeForHTML, "\"\u2027\u2028\u2029\u2030\"", nil, nil},
{"\u2027\u2028\u2029\u2030", jsonflags.EscapeForJS, "\"\u2027\\u2028\\u2029\u2030\"", nil, nil},
{"x\x80\ufffd", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8},
{"x\xff\ufffd", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8},
{"x\xc0", 0, "\"x\ufffd\"", nil, ErrInvalidUTF8},
{"x\xc0\x80", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8},
{"x\xe0", 0, "\"x\ufffd\"", nil, ErrInvalidUTF8},
{"x\xe0\x80", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8},
{"x\xe0\x80\x80", 0, "\"x\ufffd\ufffd\ufffd\"", nil, ErrInvalidUTF8},
{"x\xf0", 0, "\"x\ufffd\"", nil, ErrInvalidUTF8},
{"x\xf0\x80", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8},
{"x\xf0\x80\x80", 0, "\"x\ufffd\ufffd\ufffd\"", nil, ErrInvalidUTF8},
{"x\xf0\x80\x80\x80", 0, "\"x\ufffd\ufffd\ufffd\ufffd\"", nil, ErrInvalidUTF8},
{"x\xed\xba\xad", 0, "\"x\ufffd\ufffd\ufffd\"", nil, ErrInvalidUTF8},
{"\"\\/\b\f\n\r\t", 0, `"\"\\/\b\f\n\r\t"`, nil, nil},
{"٩(-̮̮̃-̃)۶ ٩(●̮̮̃•̃)۶ ٩(͡๏̯͡๏)۶ ٩(-̮̮̃•̃).", 0, `"٩(-̮̮̃-̃)۶ ٩(●̮̮̃•̃)۶ ٩(͡๏̯͡๏)۶ ٩(-̮̮̃•̃)."`, nil, nil},
{"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602", 0, "\"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602\"", nil, nil},
{"\u0000\u001f\u0020\u0022\u0026\u003c\u003e\u005c\u007f\u0080\u2028\u2029\ufffd\U0001f602", 0, "\"\\u0000\\u001f\u0020\\\"\u0026\u003c\u003e\\\\\u007f\u0080\u2028\u2029\ufffd\U0001f602\"", nil, nil},
}
for _, tt := range tests {
t.Run("", func(t *testing.T) {
var flags jsonflags.Flags
flags.Set(tt.flags | 1)
flags.Set(jsonflags.AllowInvalidUTF8 | 1)
got, gotErr := AppendQuote(nil, tt.in, &flags)
if string(got) != tt.want || !reflect.DeepEqual(gotErr, tt.wantErr) {
t.Errorf("AppendQuote(nil, %q, ...) = (%s, %v), want (%s, %v)", tt.in, got, gotErr, tt.want, tt.wantErr)
}
flags.Set(jsonflags.AllowInvalidUTF8 | 0)
switch got, gotErr := AppendQuote(nil, tt.in, &flags); {
case tt.wantErrUTF8 == nil && (string(got) != tt.want || !reflect.DeepEqual(gotErr, tt.wantErr)):
t.Errorf("AppendQuote(nil, %q, ...) = (%s, %v), want (%s, %v)", tt.in, got, gotErr, tt.want, tt.wantErr)
case tt.wantErrUTF8 != nil && (!strings.HasPrefix(tt.want, string(got)) || !reflect.DeepEqual(gotErr, tt.wantErrUTF8)):
t.Errorf("AppendQuote(nil, %q, ...) = (%s, %v), want (%s, %v)", tt.in, got, gotErr, tt.want, tt.wantErrUTF8)
}
})
}
}
func TestAppendNumber(t *testing.T) {
tests := []struct {
in float64
want32 string
want64 string
}{
{math.E, "2.7182817", "2.718281828459045"},
{math.Pi, "3.1415927", "3.141592653589793"},
{math.SmallestNonzeroFloat32, "1e-45", "1.401298464324817e-45"},
{math.SmallestNonzeroFloat64, "0", "5e-324"},
{math.MaxFloat32, "3.4028235e+38", "3.4028234663852886e+38"},
{math.MaxFloat64, "", "1.7976931348623157e+308"},
{0.1111111111111111, "0.11111111", "0.1111111111111111"},
{0.2222222222222222, "0.22222222", "0.2222222222222222"},
{0.3333333333333333, "0.33333334", "0.3333333333333333"},
{0.4444444444444444, "0.44444445", "0.4444444444444444"},
{0.5555555555555555, "0.5555556", "0.5555555555555555"},
{0.6666666666666666, "0.6666667", "0.6666666666666666"},
{0.7777777777777777, "0.7777778", "0.7777777777777777"},
{0.8888888888888888, "0.8888889", "0.8888888888888888"},
{0.9999999999999999, "1", "0.9999999999999999"},
// The following entries are from RFC 8785, appendix B
// which are designed to ensure repeatable formatting of 64-bit floats.
{math.Float64frombits(0x0000000000000000), "0", "0"},
{math.Float64frombits(0x8000000000000000), "-0", "-0"}, // differs from RFC 8785
{math.Float64frombits(0x0000000000000001), "0", "5e-324"},
{math.Float64frombits(0x8000000000000001), "-0", "-5e-324"},
{math.Float64frombits(0x7fefffffffffffff), "", "1.7976931348623157e+308"},
{math.Float64frombits(0xffefffffffffffff), "", "-1.7976931348623157e+308"},
{math.Float64frombits(0x4340000000000000), "9007199000000000", "9007199254740992"},
{math.Float64frombits(0xc340000000000000), "-9007199000000000", "-9007199254740992"},
{math.Float64frombits(0x4430000000000000), "295147900000000000000", "295147905179352830000"},
{math.Float64frombits(0x44b52d02c7e14af5), "1e+23", "9.999999999999997e+22"},
{math.Float64frombits(0x44b52d02c7e14af6), "1e+23", "1e+23"},
{math.Float64frombits(0x44b52d02c7e14af7), "1e+23", "1.0000000000000001e+23"},
{math.Float64frombits(0x444b1ae4d6e2ef4e), "1e+21", "999999999999999700000"},
{math.Float64frombits(0x444b1ae4d6e2ef4f), "1e+21", "999999999999999900000"},
{math.Float64frombits(0x444b1ae4d6e2ef50), "1e+21", "1e+21"},
{math.Float64frombits(0x3eb0c6f7a0b5ed8c), "0.000001", "9.999999999999997e-7"},
{math.Float64frombits(0x3eb0c6f7a0b5ed8d), "0.000001", "0.000001"},
{math.Float64frombits(0x41b3de4355555553), "333333340", "333333333.3333332"},
{math.Float64frombits(0x41b3de4355555554), "333333340", "333333333.33333325"},
{math.Float64frombits(0x41b3de4355555555), "333333340", "333333333.3333333"},
{math.Float64frombits(0x41b3de4355555556), "333333340", "333333333.3333334"},
{math.Float64frombits(0x41b3de4355555557), "333333340", "333333333.33333343"},
{math.Float64frombits(0xbecbf647612f3696), "-0.0000033333333", "-0.0000033333333333333333"},
{math.Float64frombits(0x43143ff3c1cb0959), "1424953900000000", "1424953923781206.2"},
// The following are select entries from RFC 8785, appendix B,
// but modified for equivalent 32-bit behavior.
{float64(math.Float32frombits(0x65a96815)), "9.999999e+22", "9.999998877476383e+22"},
{float64(math.Float32frombits(0x65a96816)), "1e+23", "9.999999778196308e+22"},
{float64(math.Float32frombits(0x65a96817)), "1.0000001e+23", "1.0000000678916234e+23"},
{float64(math.Float32frombits(0x6258d725)), "999999900000000000000", "999999879303389000000"},
{float64(math.Float32frombits(0x6258d726)), "999999950000000000000", "999999949672133200000"},
{float64(math.Float32frombits(0x6258d727)), "1e+21", "1.0000000200408773e+21"},
{float64(math.Float32frombits(0x6258d728)), "1.0000001e+21", "1.0000000904096215e+21"},
{float64(math.Float32frombits(0x358637bc)), "9.999999e-7", "9.99999883788405e-7"},
{float64(math.Float32frombits(0x358637bd)), "0.000001", "9.999999974752427e-7"},
{float64(math.Float32frombits(0x358637be)), "0.0000010000001", "0.0000010000001111620804"},
}
for _, tt := range tests {
t.Run("", func(t *testing.T) {
if got32 := string(AppendFloat(nil, tt.in, 32)); got32 != tt.want32 && tt.want32 != "" {
t.Errorf("AppendFloat(nil, %v, 32) = %v, want %v", tt.in, got32, tt.want32)
}
if got64 := string(AppendFloat(nil, tt.in, 64)); got64 != tt.want64 && tt.want64 != "" {
t.Errorf("AppendFloat(nil, %v, 64) = %v, want %v", tt.in, got64, tt.want64)
}
})
}
}
// The default of 1e4 lines was chosen since it is sufficiently large to include
// test numbers from all three categories (i.e., static, series, and random).
// Yet, it is sufficiently low to execute quickly relative to other tests.
//
// Processing 1e8 lines takes a minute and processes about 4GiB worth of text.
var testCanonicalNumberLines = flag.Float64("canonical-number-lines", 1e4, "specify the number of lines to check from the canonical numbers testdata")
// TestCanonicalNumber verifies that appendNumber complies with RFC 8785
// according to the testdata provided by the reference implementation.
// See https://github.com/cyberphone/json-canonicalization/tree/master/testdata#es6-numbers.
func TestCanonicalNumber(t *testing.T) {
const testfileURL = "https://github.com/cyberphone/json-canonicalization/releases/download/es6testfile/es6testfile100m.txt.gz"
hashes := map[float64]string{
1e3: "be18b62b6f69cdab33a7e0dae0d9cfa869fda80ddc712221570f9f40a5878687",
1e4: "b9f7a8e75ef22a835685a52ccba7f7d6bdc99e34b010992cbc5864cd12be6892",
1e5: "22776e6d4b49fa294a0d0f349268e5c28808fe7e0cb2bcbe28f63894e494d4c7",
1e6: "49415fee2c56c77864931bd3624faad425c3c577d6d74e89a83bc725506dad16",
1e7: "b9f8a44a91d46813b21b9602e72f112613c91408db0b8341fb94603d9db135e0",
1e8: "0f7dda6b0837dde083c5d6b896f7d62340c8a2415b0c7121d83145e08a755272",
}
wantHash := hashes[*testCanonicalNumberLines]
if wantHash == "" {
t.Fatalf("canonical-number-lines must be one of the following values: 1e3, 1e4, 1e5, 1e6, 1e7, 1e8")
}
numLines := int(*testCanonicalNumberLines)
// generator returns a function that generates the next float64 to format.
// This implements the algorithm specified in the reference implementation.
generator := func() func() float64 {
static := [...]uint64{
0x0000000000000000, 0x8000000000000000, 0x0000000000000001, 0x8000000000000001,
0xc46696695dbd1cc3, 0xc43211ede4974a35, 0xc3fce97ca0f21056, 0xc3c7213080c1a6ac,
0xc39280f39a348556, 0xc35d9b1f5d20d557, 0xc327af4c4a80aaac, 0xc2f2f2a36ecd5556,
0xc2be51057e155558, 0xc28840d131aaaaac, 0xc253670dc1555557, 0xc21f0b4935555557,
0xc1e8d5d42aaaaaac, 0xc1b3de4355555556, 0xc17fca0555555556, 0xc1496e6aaaaaaaab,
0xc114585555555555, 0xc0e046aaaaaaaaab, 0xc0aa0aaaaaaaaaaa, 0xc074d55555555555,
0xc040aaaaaaaaaaab, 0xc00aaaaaaaaaaaab, 0xbfd5555555555555, 0xbfa1111111111111,
0xbf6b4e81b4e81b4f, 0xbf35d867c3ece2a5, 0xbf0179ec9cbd821e, 0xbecbf647612f3696,
0xbe965e9f80f29212, 0xbe61e54c672874db, 0xbe2ca213d840baf8, 0xbdf6e80fe033c8c6,
0xbdc2533fe68fd3d2, 0xbd8d51ffd74c861c, 0xbd5774ccac3d3817, 0xbd22c3d6f030f9ac,
0xbcee0624b3818f79, 0xbcb804ea293472c7, 0xbc833721ba905bd3, 0xbc4ebe9c5db3c61e,
0xbc18987d17c304e5, 0xbbe3ad30dfcf371d, 0xbbaf7b816618582f, 0xbb792f9ab81379bf,
0xbb442615600f9499, 0xbb101e77800c76e1, 0xbad9ca58cce0be35, 0xbaa4a1e0a3e6fe90,
0xba708180831f320d, 0xba3a68cd9e985016, 0x446696695dbd1cc3, 0x443211ede4974a35,
0x43fce97ca0f21056, 0x43c7213080c1a6ac, 0x439280f39a348556, 0x435d9b1f5d20d557,
0x4327af4c4a80aaac, 0x42f2f2a36ecd5556, 0x42be51057e155558, 0x428840d131aaaaac,
0x4253670dc1555557, 0x421f0b4935555557, 0x41e8d5d42aaaaaac, 0x41b3de4355555556,
0x417fca0555555556, 0x41496e6aaaaaaaab, 0x4114585555555555, 0x40e046aaaaaaaaab,
0x40aa0aaaaaaaaaaa, 0x4074d55555555555, 0x4040aaaaaaaaaaab, 0x400aaaaaaaaaaaab,
0x3fd5555555555555, 0x3fa1111111111111, 0x3f6b4e81b4e81b4f, 0x3f35d867c3ece2a5,
0x3f0179ec9cbd821e, 0x3ecbf647612f3696, 0x3e965e9f80f29212, 0x3e61e54c672874db,
0x3e2ca213d840baf8, 0x3df6e80fe033c8c6, 0x3dc2533fe68fd3d2, 0x3d8d51ffd74c861c,
0x3d5774ccac3d3817, 0x3d22c3d6f030f9ac, 0x3cee0624b3818f79, 0x3cb804ea293472c7,
0x3c833721ba905bd3, 0x3c4ebe9c5db3c61e, 0x3c18987d17c304e5, 0x3be3ad30dfcf371d,
0x3baf7b816618582f, 0x3b792f9ab81379bf, 0x3b442615600f9499, 0x3b101e77800c76e1,
0x3ad9ca58cce0be35, 0x3aa4a1e0a3e6fe90, 0x3a708180831f320d, 0x3a3a68cd9e985016,
0x4024000000000000, 0x4014000000000000, 0x3fe0000000000000, 0x3fa999999999999a,
0x3f747ae147ae147b, 0x3f40624dd2f1a9fc, 0x3f0a36e2eb1c432d, 0x3ed4f8b588e368f1,
0x3ea0c6f7a0b5ed8d, 0x3e6ad7f29abcaf48, 0x3e35798ee2308c3a, 0x3ed539223589fa95,
0x3ed4ff26cd5a7781, 0x3ed4f95a762283ff, 0x3ed4f8c60703520c, 0x3ed4f8b72f19cd0d,
0x3ed4f8b5b31c0c8d, 0x3ed4f8b58d1c461a, 0x3ed4f8b5894f7f0e, 0x3ed4f8b588ee37f3,
0x3ed4f8b588e47da4, 0x3ed4f8b588e3849c, 0x3ed4f8b588e36bb5, 0x3ed4f8b588e36937,
0x3ed4f8b588e368f8, 0x3ed4f8b588e368f1, 0x3ff0000000000000, 0xbff0000000000000,
0xbfeffffffffffffa, 0xbfeffffffffffffb, 0x3feffffffffffffa, 0x3feffffffffffffb,
0x3feffffffffffffc, 0x3feffffffffffffe, 0xbfefffffffffffff, 0xbfefffffffffffff,
0x3fefffffffffffff, 0x3fefffffffffffff, 0x3fd3333333333332, 0x3fd3333333333333,
0x3fd3333333333334, 0x0010000000000000, 0x000ffffffffffffd, 0x000fffffffffffff,
0x7fefffffffffffff, 0xffefffffffffffff, 0x4340000000000000, 0xc340000000000000,
0x4430000000000000, 0x44b52d02c7e14af5, 0x44b52d02c7e14af6, 0x44b52d02c7e14af7,
0x444b1ae4d6e2ef4e, 0x444b1ae4d6e2ef4f, 0x444b1ae4d6e2ef50, 0x3eb0c6f7a0b5ed8c,
0x3eb0c6f7a0b5ed8d, 0x41b3de4355555553, 0x41b3de4355555554, 0x41b3de4355555555,
0x41b3de4355555556, 0x41b3de4355555557, 0xbecbf647612f3696, 0x43143ff3c1cb0959,
}
var state struct {
idx int
data []byte
block [sha256.Size]byte
}
return func() float64 {
const numSerial = 2000
var f float64
switch {
case state.idx < len(static):
f = math.Float64frombits(static[state.idx])
case state.idx < len(static)+numSerial:
f = math.Float64frombits(0x0010000000000000 + uint64(state.idx-len(static)))
default:
for f == 0 || math.IsNaN(f) || math.IsInf(f, 0) {
if len(state.data) == 0 {
state.block = sha256.Sum256(state.block[:])
state.data = state.block[:]
}
f = math.Float64frombits(binary.LittleEndian.Uint64(state.data))
state.data = state.data[8:]
}
}
state.idx++
return f
}
}
// Pass through the test twice. In the first pass we only hash the output,
// while in the second pass we check every line against the golden testdata.
// If the hashes match in the first pass, then we skip the second pass.
for _, checkGolden := range []bool{false, true} {
var br *bufio.Reader // for line-by-line reading of es6testfile100m.txt
if checkGolden {
resp, err := http.Get(testfileURL)
if err != nil {
t.Fatalf("http.Get error: %v", err)
}
defer resp.Body.Close()
zr, err := gzip.NewReader(resp.Body)
if err != nil {
t.Fatalf("gzip.NewReader error: %v", err)
}
br = bufio.NewReader(zr)
}
// appendNumberJCS differs from appendNumber only for -0.
appendNumberJCS := func(b []byte, f float64) []byte {
if math.Signbit(f) && f == 0 {
return append(b, '0')
}
return AppendFloat(b, f, 64)
}
var gotLine []byte
next := generator()
hash := sha256.New()
start := time.Now()
lastPrint := start
for n := 1; n <= numLines; n++ {
// Generate the formatted line for this number.
f := next()
gotLine = gotLine[:0] // reset from previous usage
gotLine = strconv.AppendUint(gotLine, math.Float64bits(f), 16)
gotLine = append(gotLine, ',')
gotLine = appendNumberJCS(gotLine, f)
gotLine = append(gotLine, '\n')
hash.Write(gotLine)
// Check that the formatted line matches.
if checkGolden {
wantLine, err := br.ReadBytes('\n')
if err != nil {
t.Fatalf("bufio.Reader.ReadBytes error: %v", err)
}
if !bytes.Equal(gotLine, wantLine) {
t.Errorf("mismatch on line %d:\n\tgot %v\n\twant %v",
n, strings.TrimSpace(string(gotLine)), strings.TrimSpace(string(wantLine)))
}
}
// Print progress.
if now := time.Now(); now.Sub(lastPrint) > time.Second || n == numLines {
remaining := float64(now.Sub(start)) * float64(numLines-n) / float64(n)
t.Logf("%0.3f%% (%v remaining)",
100.0*float64(n)/float64(numLines),
time.Duration(remaining).Round(time.Second))
lastPrint = now
}
}
gotHash := hex.EncodeToString(hash.Sum(nil))
if gotHash == wantHash {
return // hashes match, no need to check golden testdata
}
}
}

98
pkg/encoders/json/internal/jsonwire/wire_test.go
View File

@@ -1,98 +0,0 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build goexperiment.jsonv2
package jsonwire
import (
"cmp"
"slices"
"testing"
"unicode/utf16"
"unicode/utf8"
)
func TestQuoteRune(t *testing.T) {
tests := []struct{ in, want string }{
{"x", `'x'`},
{"\n", `'\n'`},
{"'", `'\''`},
{"\xff", `'\xff'`},
{"💩", `'💩'`},
{"💩"[:1], `'\xf0'`},
{"\uffff", `'\uffff'`},
{"\U00101234", `'\U00101234'`},
}
for _, tt := range tests {
got := QuoteRune([]byte(tt.in))
if got != tt.want {
t.Errorf("quoteRune(%q) = %s, want %s", tt.in, got, tt.want)
}
}
}
var compareUTF16Testdata = []string{"", "\r", "1", "f\xfe", "f\xfe\xff", "f\xff", "\u0080", "\u00f6", "\u20ac", "\U0001f600", "\ufb33"}
func TestCompareUTF16(t *testing.T) {
for i, si := range compareUTF16Testdata {
for j, sj := range compareUTF16Testdata {
got := CompareUTF16([]byte(si), []byte(sj))
want := cmp.Compare(i, j)
if got != want {
t.Errorf("CompareUTF16(%q, %q) = %v, want %v", si, sj, got, want)
}
}
}
}
func FuzzCompareUTF16(f *testing.F) {
for _, td1 := range compareUTF16Testdata {
for _, td2 := range compareUTF16Testdata {
f.Add([]byte(td1), []byte(td2))
}
}
// CompareUTF16Simple is identical to CompareUTF16,
// but relies on naively converting a string to a []uint16 codepoints.
// It is easy to verify as correct, but is slow.
CompareUTF16Simple := func(x, y []byte) int {
ux := utf16.Encode([]rune(string(x)))
uy := utf16.Encode([]rune(string(y)))
return slices.Compare(ux, uy)
}
f.Fuzz(func(t *testing.T, s1, s2 []byte) {
// Compare the optimized and simplified implementations.
got := CompareUTF16(s1, s2)
want := CompareUTF16Simple(s1, s2)
if got != want && utf8.Valid(s1) && utf8.Valid(s2) {
t.Errorf("CompareUTF16(%q, %q) = %v, want %v", s1, s2, got, want)
}
})
}
func TestTruncatePointer(t *testing.T) {
tests := []struct{ in, want string }{
{"hello", "hello"},
{"/a/b/c", "/a/b/c"},
{"/a/b/c/d/e/f/g", "/a/b/…/f/g"},
{"supercalifragilisticexpialidocious", "super…cious"},
{"/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious", "/supe…/…cious"},
{"/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious", "/supe…/…/…cious"},
{"/a/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious", "/a/…/…cious"},
{"/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious/b", "/supe…/…/b"},
{"/fizz/buzz/bazz", "/fizz/…/bazz"},
{"/fizz/buzz/bazz/razz", "/fizz/…/razz"},
{"/////////////////////////////", "/////…/////"},
{"/🎄❤️✨/🎁✅😊/🎅🔥⭐", "/🎄…/…/…⭐"},
}
for _, tt := range tests {
got := TruncatePointer(tt.in, 10)
if got != tt.want {
t.Errorf("TruncatePointer(%q) = %q, want %q", tt.in, got, tt.want)
}
}
}