From b2d0aeb3c78510d11e106e8e9a8a59e38fa96d55 Mon Sep 17 00:00:00 2001 From: Steve van Loben Sels Date: Wed, 2 Nov 2022 09:38:54 -0700 Subject: [PATCH 01/10] Added Position field to the Tokenizer (#128) The Position gives a means to index into the Tokenizer's underlying byte slice. This enables use cases where the caller is planning on making edits to the JSON document but wants to leverage the copy func to optimize data movement and/or to copy remaining bytes if the caller wants to exit the tokenizing loop early. --- json/token.go | 18 +++++++++++++++++- json/token_test.go | 33 +++++++++++++++++++++------------ 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/json/token.go b/json/token.go index b9f46ae..b82f49b 100644 --- a/json/token.go +++ b/json/token.go @@ -31,7 +31,6 @@ import ( // ... // } // } -// type Tokenizer struct { // When the tokenizer is positioned on a json delimiter this field is not // zero. In this case the possible values are '{', '}', '[', ']', ':', and @@ -44,6 +43,17 @@ type Tokenizer struct { // null, true, false, numbers, or quoted strings. Value RawValue + // Position is the Tokenizer's current index into the underlying byte slice. + // Since the Tokenizer has already been advanced by calling Next, this + // position will be the first index of the next token. The position of + // the current Value can be calculated by subtracting len(token.value). + // Accordingly, slicing the underlying bytes like: + // + // b[token.Position-len(token.Value):token.Position] + // + // will yield the current Value. + Position int + // When the tokenizer has encountered invalid content this field is not nil. Err error @@ -92,6 +102,7 @@ func (t *Tokenizer) Reset(b []byte) { // However, it does not compile down to an invocation of duff-copy. t.Delim = 0 t.Value = nil + t.Position = 0 t.Err = nil t.Depth = 0 t.Index = 0 @@ -128,6 +139,7 @@ skipLoop: if i > 0 { t.json = t.json[i:] + t.Position += i } if len(t.json) == 0 { @@ -135,6 +147,8 @@ skipLoop: return false } + lenBefore := len(t.json) + var kind Kind switch t.json[0] { case '"': @@ -165,6 +179,8 @@ skipLoop: t.Value, t.json, t.Err = t.json[:1], t.json[1:], syntaxError(t.json, "expected token but found '%c'", t.json[0]) } + t.Position += lenBefore - len(t.json) + t.Depth = t.depth() t.Index = t.index() t.flags = t.flags.withKind(kind) diff --git a/json/token_test.go b/json/token_test.go index 2805de3..f5dbf65 100644 --- a/json/token_test.go +++ b/json/token_test.go @@ -1,6 +1,7 @@ package json import ( + "bytes" "reflect" "testing" ) @@ -40,22 +41,30 @@ func value(v string, depth, index int) token { } } -func tokenize(b []byte) (tokens []token) { - t := NewTokenizer(b) +func tokenize(t *testing.T, b []byte) (tokens []token) { + tok := NewTokenizer(b) + + for tok.Next() { + start, end := tok.Position-len(tok.Value), tok.Position + if end > len(b) { + t.Fatalf("token position too far [%d:%d], len(b) is %d", start, end, len(b)) + } + if !bytes.Equal(b[start:end], tok.Value) { + t.Fatalf("token position is wrong [%d:%d]", start, end) + } - for t.Next() { tokens = append(tokens, token{ - delim: t.Delim, - value: t.Value, - err: t.Err, - depth: t.Depth, - index: t.Index, - isKey: t.IsKey, + delim: tok.Delim, + value: tok.Value, + err: tok.Err, + depth: tok.Depth, + index: tok.Index, + isKey: tok.IsKey, }) } - if t.Err != nil { - panic(t.Err) + if tok.Err != nil { + t.Fatal(tok.Err) } return @@ -174,7 +183,7 @@ func TestTokenizer(t *testing.T) { for _, test := range tests { t.Run(string(test.input), func(t *testing.T) { - tokens := tokenize(test.input) + tokens := tokenize(t, test.input) if !reflect.DeepEqual(tokens, test.tokens) { t.Error("tokens mismatch") From 3b49d71a5eb29f09671609a0e9d3eb48037a17a4 Mon Sep 17 00:00:00 2001 From: Steve van Loben Sels Date: Thu, 10 Nov 2022 14:19:01 -0800 Subject: [PATCH 02/10] Revert #128, update docs and add tests to Tokenizer.Remaining() (#129) --- json/token.go | 25 ++++++++----------------- json/token_test.go | 3 ++- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/json/token.go b/json/token.go index b82f49b..161b7fa 100644 --- a/json/token.go +++ b/json/token.go @@ -43,17 +43,6 @@ type Tokenizer struct { // null, true, false, numbers, or quoted strings. Value RawValue - // Position is the Tokenizer's current index into the underlying byte slice. - // Since the Tokenizer has already been advanced by calling Next, this - // position will be the first index of the next token. The position of - // the current Value can be calculated by subtracting len(token.value). - // Accordingly, slicing the underlying bytes like: - // - // b[token.Position-len(token.Value):token.Position] - // - // will yield the current Value. - Position int - // When the tokenizer has encountered invalid content this field is not nil. Err error @@ -102,7 +91,6 @@ func (t *Tokenizer) Reset(b []byte) { // However, it does not compile down to an invocation of duff-copy. t.Delim = 0 t.Value = nil - t.Position = 0 t.Err = nil t.Depth = 0 t.Index = 0 @@ -139,7 +127,6 @@ skipLoop: if i > 0 { t.json = t.json[i:] - t.Position += i } if len(t.json) == 0 { @@ -147,8 +134,6 @@ skipLoop: return false } - lenBefore := len(t.json) - var kind Kind switch t.json[0] { case '"': @@ -179,8 +164,6 @@ skipLoop: t.Value, t.json, t.Err = t.json[:1], t.json[1:], syntaxError(t.json, "expected token but found '%c'", t.json[0]) } - t.Position += lenBefore - len(t.json) - t.Depth = t.depth() t.Index = t.index() t.flags = t.flags.withKind(kind) @@ -319,6 +302,14 @@ func (t *Tokenizer) String() []byte { } // Remaining returns the number of bytes left to parse. +// +// The position of the tokenizer's current Value within the original byte slice +// can be calculated like so: +// +// end := len(b) - tok.Remaining() +// start := end - len(tok.Value) +// +// And slicing b[start:end] will yield the tokenizer's current Value. func (t *Tokenizer) Remaining() int { return len(t.json) } diff --git a/json/token_test.go b/json/token_test.go index f5dbf65..cc05365 100644 --- a/json/token_test.go +++ b/json/token_test.go @@ -45,7 +45,8 @@ func tokenize(t *testing.T, b []byte) (tokens []token) { tok := NewTokenizer(b) for tok.Next() { - start, end := tok.Position-len(tok.Value), tok.Position + end := len(b) - tok.Remaining() + start := end - len(tok.Value) if end > len(b) { t.Fatalf("token position too far [%d:%d], len(b) is %d", start, end, len(b)) } From 6dfc1b0e15041ef27898e847c2005974968828d3 Mon Sep 17 00:00:00 2001 From: Kevin Gillette Date: Mon, 27 Nov 2023 11:28:27 -0700 Subject: [PATCH 03/10] fix encoding RawMessage that contains leading space (#136) * fix encoding RawMessage that contains leading space. * add json/bugs/issue136 --- Makefile | 2 +- json/bugs/issue11/main.go | 3 --- json/bugs/issue136/main_test.go | 23 +++++++++++++++++++++++ json/bugs/issue18/main.go | 4 ---- json/bugs/issue84/main.go | 3 --- json/encode.go | 1 + json/parse.go | 1 - 7 files changed, 25 insertions(+), 12 deletions(-) delete mode 100644 json/bugs/issue11/main.go create mode 100644 json/bugs/issue136/main_test.go delete mode 100644 json/bugs/issue18/main.go delete mode 100644 json/bugs/issue84/main.go diff --git a/Makefile b/Makefile index 9e540d7..ffb1396 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ test-json: go test -cover -race ./json test-json-bugs: - go test -cover -race ./json/bugs/... + go test -race ./json/bugs/... test-json-1.17: go test -cover -race -tags go1.17 ./json diff --git a/json/bugs/issue11/main.go b/json/bugs/issue11/main.go deleted file mode 100644 index 38dd16d..0000000 --- a/json/bugs/issue11/main.go +++ /dev/null @@ -1,3 +0,0 @@ -package main - -func main() {} diff --git a/json/bugs/issue136/main_test.go b/json/bugs/issue136/main_test.go new file mode 100644 index 0000000..044af10 --- /dev/null +++ b/json/bugs/issue136/main_test.go @@ -0,0 +1,23 @@ +package main + +import ( + "bytes" + "testing" + + "github.com/segmentio/encoding/json" +) + +func TestIssue136(t *testing.T) { + input := json.RawMessage(` null`) + + got, err := json.Marshal(input) + if err != nil { + t.Fatal(err) + } + + want := bytes.TrimSpace(input) + + if !bytes.Equal(got, want) { + t.Fatalf("Marshal(%q) = %q, want %q", input, got, want) + } +} diff --git a/json/bugs/issue18/main.go b/json/bugs/issue18/main.go deleted file mode 100644 index da29a2c..0000000 --- a/json/bugs/issue18/main.go +++ /dev/null @@ -1,4 +0,0 @@ -package main - -func main() { -} diff --git a/json/bugs/issue84/main.go b/json/bugs/issue84/main.go deleted file mode 100644 index 38dd16d..0000000 --- a/json/bugs/issue84/main.go +++ /dev/null @@ -1,3 +0,0 @@ -package main - -func main() {} diff --git a/json/encode.go b/json/encode.go index acb3b67..4173ddd 100644 --- a/json/encode.go +++ b/json/encode.go @@ -858,6 +858,7 @@ func (e encoder) encodeRawMessage(b []byte, p unsafe.Pointer) ([]byte, error) { s = v } else { var err error + v = skipSpaces(v) // don't assume that a RawMessage starts with a token. d := decoder{} s, _, _, err = d.parseValue(v) if err != nil { diff --git a/json/parse.go b/json/parse.go index 3e65621..49f63aa 100644 --- a/json/parse.go +++ b/json/parse.go @@ -709,7 +709,6 @@ func (d decoder) parseValue(b []byte) ([]byte, []byte, Kind, error) { case '{': v, b, k, err = d.parseObject(b) case '[': - k = Array v, b, k, err = d.parseArray(b) case '"': v, b, k, err = d.parseString(b) From 3055897c1c5b74bc4c4ec5f702c22ed90020f410 Mon Sep 17 00:00:00 2001 From: Kevin Gillette Date: Mon, 4 Dec 2023 15:00:43 -0700 Subject: [PATCH 04/10] json: configurable numeric decoding (#137) * json: stylistic improvements, better code reuse Initial benchmark results show this change to be approximately performance-neutral. * bump tested Go versions to just 1.20 and 1.21 * json: add ParseFlags values UseInt64, UseUint64, UseBigInt * json: use atomic.Pointer --- .github/workflows/benchmark.yml | 88 ++++++++++---------- .github/workflows/test.yml | 29 ++++--- Makefile | 7 +- go.mod | 4 +- json/codec.go | 31 ++++--- json/decode.go | 112 ++++++++++++++++++++----- json/golang_bench_test.go | 6 +- json/int_test.go | 3 +- json/json.go | 13 +++ json/json_test.go | 142 +++++++++++++++++++++++++++++++- proto/fixtures/generate/main.go | 3 +- proto/proto_test.go | 4 +- thrift/decode.go | 3 +- 13 files changed, 334 insertions(+), 111 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 3d2383c..805d1c4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,61 +1,63 @@ +--- name: Benchmark -on: -- pull_request +"on": + - pull_request jobs: benchmark: strategy: matrix: ref: - - master - - ${{ github.sha }} + - master + - ${{ github.sha }} runs-on: ubuntu-latest steps: - - name: Steup Go - uses: actions/setup-go@v2 - with: - go-version: 1.17 - - - name: Checkout - uses: actions/checkout@v2 - with: - ref: ${{ matrix.ref }} - - - name: Run Benchmarks - run: go test -v -run '^$' -bench '(Marshal|Unmarshal)$/codeResponse' -benchmem -benchtime 3s -cpu 1 -count 5 ./json | tee bench.txt - - - name: Upload Benchmarks - uses: actions/upload-artifact@v2 - with: - name: ${{ matrix.ref }} - path: bench.txt + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: "1.21" + + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ matrix.ref }} + + - name: Run Benchmarks + # Without 6 iterations, benchstat will claim statistical insignificance. + run: go test -v -run '^$' -bench '(Marshal|Unmarshal)$/codeResponse' -benchmem -benchtime 3s -cpu 1 -count 6 ./json | tee bench.txt + + - name: Upload Benchmarks + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.ref }} + path: bench.txt benchstat: needs: [benchmark] runs-on: ubuntu-latest steps: - - name: Steup Go - uses: actions/setup-go@v2 - with: - go-version: 1.17 - - - name: Setup Benchstat - run: go install golang.org/x/perf/cmd/benchstat@latest - - - name: Download Benchmark Results - uses: actions/download-artifact@v2 - with: - path: . - - - name: Run Benchstat - run: benchstat ./master/bench.txt ./${{ github.sha }}/bench.txt | tee benchstat.txt - - - name: Upload Benchstat Results - uses: actions/upload-artifact@v2 - with: - name: benchstat - path: benchstat.txt + - name: Steup Go + uses: actions/setup-go@v2 + with: + go-version: "1.21" + + - name: Setup Benchstat + run: go install golang.org/x/perf/cmd/benchstat@latest + + - name: Download Benchmark Results + uses: actions/download-artifact@v2 + with: + path: . + + - name: Run Benchstat + run: benchstat ./master/bench.txt ./${{ github.sha }}/bench.txt | tee benchstat.txt + + - name: Upload Benchstat Results + uses: actions/upload-artifact@v2 + with: + name: benchstat + path: benchstat.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 16acc54..7e4bc41 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,30 +1,29 @@ +--- name: Test -on: -- pull_request +"on": + - pull_request jobs: test: strategy: matrix: go: - - 1.14 - - 1.15 - - 1.16 - - 1.17 + - "1.20" + - "1.21" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v2 - - name: Setup Go ${{ matrix.go }} - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} + - name: Setup Go ${{ matrix.go }} + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} - - name: Download Dependencies - run: go mod download + - name: Download Dependencies + run: go mod download - - name: Run Tests - run: make test + - name: Run Tests + run: make test diff --git a/Makefile b/Makefile index ffb1396..4640625 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: test bench-simple clean update-golang-test fuzz fuzz-json -golang.version ?= 1.15.2 +golang.version ?= 1.21 golang.tmp.root := /tmp/golang$(golang.version) golang.tmp.json.root := $(golang.tmp.root)/go-go$(golang.version)/src/encoding/json golang.test.files := $(wildcard json/golang_*_test.go) @@ -10,7 +10,7 @@ go-fuzz-build := ${GOPATH}/bin/go-fuzz-build go-fuzz-corpus := ${GOPATH}/src/github.com/dvyukov/go-fuzz-corpus go-fuzz-dep := ${GOPATH}/src/github.com/dvyukov/go-fuzz/go-fuzz-dep -test: test-ascii test-json test-json-bugs test-json-1.17 test-proto test-iso8601 test-thrift test-purego +test: test-ascii test-json test-json-bugs test-proto test-iso8601 test-thrift test-purego test-ascii: go test -cover -race ./ascii @@ -21,9 +21,6 @@ test-json: test-json-bugs: go test -race ./json/bugs/... -test-json-1.17: - go test -cover -race -tags go1.17 ./json - test-proto: go test -cover -race ./proto diff --git a/go.mod b/go.mod index 5f43ab4..0096cd2 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/segmentio/encoding -go 1.14 +go 1.18 require github.com/segmentio/asm v1.1.3 + +require golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 // indirect diff --git a/json/codec.go b/json/codec.go index 908c3f6..bd3c1d4 100644 --- a/json/codec.go +++ b/json/codec.go @@ -4,6 +4,7 @@ import ( "encoding" "encoding/json" "fmt" + "math/big" "reflect" "sort" "strconv" @@ -49,19 +50,21 @@ type decodeFunc func(decoder, []byte, unsafe.Pointer) ([]byte, error) type emptyFunc func(unsafe.Pointer) bool type sortFunc func([]reflect.Value) -var ( - // Eventually consistent cache mapping go types to dynamically generated - // codecs. - // - // Note: using a uintptr as key instead of reflect.Type shaved ~15ns off of - // the ~30ns Marhsal/Unmarshal functions which were dominated by the map - // lookup time for simple types like bool, int, etc.. - cache unsafe.Pointer // map[unsafe.Pointer]codec -) +// Eventually consistent cache mapping go types to dynamically generated +// codecs. +// +// Note: using a uintptr as key instead of reflect.Type shaved ~15ns off of +// the ~30ns Marhsal/Unmarshal functions which were dominated by the map +// lookup time for simple types like bool, int, etc.. +var cache atomic.Pointer[map[unsafe.Pointer]codec] func cacheLoad() map[unsafe.Pointer]codec { - p := atomic.LoadPointer(&cache) - return *(*map[unsafe.Pointer]codec)(unsafe.Pointer(&p)) + p := cache.Load() + if p == nil { + return nil + } + + return *p } func cacheStore(typ reflect.Type, cod codec, oldCodecs map[unsafe.Pointer]codec) { @@ -72,7 +75,7 @@ func cacheStore(typ reflect.Type, cod codec, oldCodecs map[unsafe.Pointer]codec) newCodecs[t] = c } - atomic.StorePointer(&cache, *(*unsafe.Pointer)(unsafe.Pointer(&newCodecs))) + cache.Store(&newCodecs) } func typeid(t reflect.Type) unsafe.Pointer { @@ -838,6 +841,7 @@ func constructInlineValueEncodeFunc(encode encodeFunc) encodeFunc { // compiles down to zero instructions. // USE CAREFULLY! // This was copied from the runtime; see issues 23382 and 7921. +// //go:nosplit func noescape(p unsafe.Pointer) unsafe.Pointer { x := uintptr(p) @@ -1078,6 +1082,7 @@ var ( float32Type = reflect.TypeOf(float32(0)) float64Type = reflect.TypeOf(float64(0)) + bigIntType = reflect.TypeOf(new(big.Int)) numberType = reflect.TypeOf(json.Number("")) stringType = reflect.TypeOf("") stringsType = reflect.TypeOf([]string(nil)) @@ -1104,6 +1109,8 @@ var ( jsonUnmarshalerType = reflect.TypeOf((*Unmarshaler)(nil)).Elem() textMarshalerType = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem() textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() + + bigIntDecoder = constructJSONUnmarshalerDecodeFunc(bigIntType, false) ) // ============================================================================= diff --git a/json/decode.go b/json/decode.go index b1723c2..9792af0 100644 --- a/json/decode.go +++ b/json/decode.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "math" + "math/big" "reflect" "strconv" "time" @@ -16,6 +17,10 @@ import ( "github.com/segmentio/encoding/iso8601" ) +func (d decoder) anyFlagsSet(flags ParseFlags) bool { + return d.flags&flags != 0 +} + func (d decoder) decodeNull(b []byte, p unsafe.Pointer) ([]byte, error) { if hasNullPrefix(b) { return b[4:], nil @@ -738,10 +743,12 @@ func (d decoder) decodeMapStringInterface(b []byte, p unsafe.Pointer) ([]byte, e m = make(map[string]interface{}, 64) } - var err error - var key string - var val interface{} - var input = b + var ( + input = b + key string + val any + err error + ) b = b[1:] for { @@ -1276,6 +1283,7 @@ func (d decoder) decodeInterface(b []byte, p unsafe.Pointer) ([]byte, error) { if err == nil { *(*interface{})(p) = val } + return b, err } @@ -1286,19 +1294,15 @@ func (d decoder) decodeInterface(b []byte, p unsafe.Pointer) ([]byte, error) { switch k.Class() { case Object: - m := make(map[string]interface{}) - v, err = d.decodeMapStringInterface(v, unsafe.Pointer(&m)) - val = m + v, err = decodeInto[map[string]any](&val, v, d, decoder.decodeMapStringInterface) case Array: - a := make([]interface{}, 0, 10) - v, err = d.decodeSlice(v, unsafe.Pointer(&a), unsafe.Sizeof(a[0]), sliceInterfaceType, decoder.decodeInterface) - val = a + size := alignedSize(interfaceType) + fn := constructSliceDecodeFunc(size, sliceInterfaceType, decoder.decodeInterface) + v, err = decodeInto[[]any](&val, v, d, fn) case String: - s := "" - v, err = d.decodeString(v, unsafe.Pointer(&s)) - val = s + v, err = decodeInto[string](&val, v, d, decoder.decodeString) case Null: v, val = nil, nil @@ -1307,15 +1311,7 @@ func (d decoder) decodeInterface(b []byte, p unsafe.Pointer) ([]byte, error) { v, val = nil, k == True case Num: - if (d.flags & UseNumber) != 0 { - n := Number("") - v, err = d.decodeNumber(v, unsafe.Pointer(&n)) - val = n - } else { - f := 0.0 - v, err = d.decodeFloat64(v, unsafe.Pointer(&f)) - val = f - } + v, err = d.decodeDynamicNumber(v, unsafe.Pointer(&val)) default: return b, syntaxError(v, "expected token but found '%c'", v[0]) @@ -1333,6 +1329,68 @@ func (d decoder) decodeInterface(b []byte, p unsafe.Pointer) ([]byte, error) { return b, nil } +func (d decoder) decodeDynamicNumber(b []byte, p unsafe.Pointer) ([]byte, error) { + kind := Float + var err error + + // Only pre-parse for numeric kind if a conditional decode + // has been requested. + if d.anyFlagsSet(UseBigInt | UseInt64 | UseUint64) { + _, _, kind, err = d.parseNumber(b) + if err != nil { + return b, err + } + } + + var rem []byte + anyPtr := (*any)(p) + + // Mutually exclusive integer handling cases. + switch { + // If requested, attempt decode of positive integers as uint64. + case kind == Uint && d.anyFlagsSet(UseUint64): + rem, err = decodeInto[uint64](anyPtr, b, d, decoder.decodeUint64) + if err == nil { + return rem, err + } + + // If uint64 decode was not requested but int64 decode was requested, + // then attempt decode of positive integers as int64. + case kind == Uint && d.anyFlagsSet(UseInt64): + fallthrough + + // If int64 decode was requested, + // attempt decode of negative integers as int64. + case kind == Int && d.anyFlagsSet(UseInt64): + rem, err = decodeInto[int64](anyPtr, b, d, decoder.decodeInt64) + if err == nil { + return rem, err + } + } + + // Fallback numeric handling cases: + // these cannot be combined into the above switch, + // since these cases also handle overflow + // from the above cases, if decode was already attempted. + switch { + // If *big.Int decode was requested, handle that case for any integer. + case kind == Uint && d.anyFlagsSet(UseBigInt): + fallthrough + case kind == Int && d.anyFlagsSet(UseBigInt): + rem, err = decodeInto[*big.Int](anyPtr, b, d, bigIntDecoder) + + // If json.Number decode was requested, handle that for any number. + case d.anyFlagsSet(UseNumber): + rem, err = decodeInto[Number](anyPtr, b, d, decoder.decodeNumber) + + // Fall back to float64 decode when no special decoding has been requested. + default: + rem, err = decodeInto[float64](anyPtr, b, d, decoder.decodeFloat64) + } + + return rem, err +} + func (d decoder) decodeMaybeEmptyInterface(b []byte, p unsafe.Pointer, t reflect.Type) ([]byte, error) { if hasNullPrefix(b) { *(*interface{})(p) = nil @@ -1460,3 +1518,13 @@ func (d decoder) inputError(b []byte, t reflect.Type) ([]byte, error) { } return skipSpaces(r), unmarshalTypeError(b, t) } + +func decodeInto[T any](dest *any, b []byte, d decoder, fn decodeFunc) ([]byte, error) { + var v T + rem, err := fn(d, b, unsafe.Pointer(&v)) + if err == nil { + *dest = v + } + + return rem, err +} diff --git a/json/golang_bench_test.go b/json/golang_bench_test.go index 07cc378..04c9176 100644 --- a/json/golang_bench_test.go +++ b/json/golang_bench_test.go @@ -14,7 +14,7 @@ import ( "bytes" "compress/gzip" "fmt" - "io/ioutil" + "io" "os" "reflect" "runtime" @@ -51,7 +51,7 @@ func codeInit() { if err != nil { panic(err) } - data, err := ioutil.ReadAll(gz) + data, err := io.ReadAll(gz) if err != nil { panic(err) } @@ -88,7 +88,7 @@ func BenchmarkCodeEncoder(b *testing.B) { b.StartTimer() } b.RunParallel(func(pb *testing.PB) { - enc := NewEncoder(ioutil.Discard) + enc := NewEncoder(io.Discard) for pb.Next() { if err := enc.Encode(&codeStruct); err != nil { b.Fatal("Encode:", err) diff --git a/json/int_test.go b/json/int_test.go index 4611375..cca5af1 100644 --- a/json/int_test.go +++ b/json/int_test.go @@ -13,8 +13,7 @@ func TestAppendInt(t *testing.T) { ints = append(ints, int64(u-1), int64(u), int64(u+1), -int64(u)) } - var std [20]byte - var our [20]byte + var std, our [20]byte for _, i := range ints { expected := strconv.AppendInt(std[:], i, 10) diff --git a/json/json.go b/json/json.go index 47f3ba1..d5f6f9d 100644 --- a/json/json.go +++ b/json/json.go @@ -128,6 +128,19 @@ const ( // mode. DontMatchCaseInsensitiveStructFields + // Decode integers into *big.Int. + // Takes precedence over UseNumber for integers. + UseBigInt + + // Decode in-range integers to int64. + // Takes precedence over UseNumber and UseBigInt for in-range integers. + UseInt64 + + // Decode in-range positive integers to uint64. + // Takes precedence over UseNumber, UseBigInt, and UseInt64 + // for positive, in-range integers. + UseUint64 + // ZeroCopy is a parsing flag that combines all the copy optimizations // available in the package. // diff --git a/json/json_test.go b/json/json_test.go index 6840b85..fb77868 100644 --- a/json/json_test.go +++ b/json/json_test.go @@ -9,8 +9,8 @@ import ( "flag" "fmt" "io" - "io/ioutil" "math" + "math/big" "os" "path/filepath" "reflect" @@ -86,6 +86,13 @@ type tree struct { Right *tree } +var ( + // bigPos128 and bigNeg128 are 1<<128 and -1<<128 + // certainly neither is representable using a uint64/int64. + bigPos128 = new(big.Int).Lsh(big.NewInt(1), 128) + bigNeg128 = new(big.Int).Neg(bigPos128) +) + var testValues = [...]interface{}{ // constants nil, @@ -127,6 +134,9 @@ var testValues = [...]interface{}{ float64(math.SmallestNonzeroFloat64), float64(math.MaxFloat64), + bigPos128, + bigNeg128, + // number Number("0"), Number("1234567890"), @@ -402,7 +412,7 @@ func TestCodec(t *testing.T) { t.Logf("found: %#v", x2) } - if b, err := ioutil.ReadAll(dec.Buffered()); err != nil { + if b, err := io.ReadAll(dec.Buffered()); err != nil { t.Error(err) } else if len(b) != 0 { t.Errorf("leftover trailing bytes in the decoder: %q", b) @@ -485,6 +495,134 @@ func TestCodecDuration(t *testing.T) { } } +var numericParseTests = [...]struct { + name string + input string + flags ParseFlags + want any +}{ + { + name: "zero_flags_default", + input: `0`, + flags: 0, + want: float64(0), + }, + { + name: "zero_flags_int_uint_bigint_number", + input: `0`, + flags: UseInt64 | UseUint64 | UseBigInt | UseNumber, + want: uint64(0), + }, + { + name: "zero_flags_int_bigint_number", + input: `0`, + flags: UseInt64 | UseBigInt | UseNumber, + want: int64(0), + }, + { + name: "zero_flags_bigint_number", + input: `0`, + flags: UseBigInt | UseNumber, + want: big.NewInt(0), + }, + { + name: "zero_flags_number", + input: `0`, + flags: UseNumber, + want: json.Number(`0`), + }, + { + name: "max_uint64_flags_default", + input: fmt.Sprint(uint64(math.MaxUint64)), + flags: 0, + want: float64(math.MaxUint64), + }, + { + name: "max_uint64_flags_int_uint_bigint_number", + input: fmt.Sprint(uint64(math.MaxUint64)), + flags: UseInt64 | UseUint64 | UseBigInt | UseNumber, + want: uint64(math.MaxUint64), + }, + { + name: "min_int64_flags_uint_int_bigint_number", + input: fmt.Sprint(int64(math.MinInt64)), + flags: UseInt64 | UseBigInt | UseNumber, + want: int64(math.MinInt64), + }, + { + name: "max_uint64_flags_int_bigint_number", + input: fmt.Sprint(uint64(math.MaxUint64)), + flags: UseInt64 | UseBigInt | UseNumber, + want: new(big.Int).SetUint64(math.MaxUint64), + }, + { + name: "overflow_uint64_flags_uint_int_bigint_number", + input: bigPos128.String(), + flags: UseUint64 | UseInt64 | UseBigInt | UseNumber, + want: bigPos128, + }, + { + name: "underflow_uint64_flags_uint_int_bigint_number", + input: bigNeg128.String(), + flags: UseUint64 | UseInt64 | UseBigInt | UseNumber, + want: bigNeg128, + }, + { + name: "overflow_uint64_flags_uint_int_number", + input: bigPos128.String(), + flags: UseUint64 | UseInt64 | UseNumber, + want: json.Number(bigPos128.String()), + }, + { + name: "underflow_uint64_flags_uint_int_number", + input: bigNeg128.String(), + flags: UseUint64 | UseInt64 | UseNumber, + want: json.Number(bigNeg128.String()), + }, + { + name: "overflow_uint64_flags_uint_int", + input: bigPos128.String(), + flags: UseUint64 | UseInt64, + want: float64(1 << 128), + }, + { + name: "underflow_uint64_flags_uint_int", + input: bigNeg128.String(), + flags: UseUint64 | UseInt64, + want: float64(-1 << 128), + }, +} + +func TestParse_numeric(t *testing.T) { + t.Parallel() + + for _, test := range numericParseTests { + test := test + + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + var got any + + rem, err := Parse([]byte(test.input), &got, test.flags) + if err != nil { + format := "Parse(%#q, ..., %#b) = %q [error], want nil" + t.Errorf(format, test.input, test.flags, err) + } + + if len(rem) != 0 { + format := "Parse(%#q, ..., %#b) = %#q, want zero length" + t.Errorf(format, test.input, test.flags, rem) + } + + if !reflect.DeepEqual(got, test.want) { + format := "Parse(%#q, %#b) -> %T(%#[3]v), want %T(%#[4]v)" + t.Errorf(format, test.input, test.flags, got, test.want) + } + }) + } +} + func newValue(model interface{}) reflect.Value { if model == nil { return reflect.New(reflect.TypeOf(&model).Elem()) diff --git a/proto/fixtures/generate/main.go b/proto/fixtures/generate/main.go index 3016afd..0614934 100644 --- a/proto/fixtures/generate/main.go +++ b/proto/fixtures/generate/main.go @@ -1,7 +1,6 @@ package main import ( - "io/ioutil" "os" "github.com/golang/protobuf/proto" @@ -28,6 +27,6 @@ func main() { for _, test := range tests { b, _ := proto.Marshal(&test.value) - ioutil.WriteFile("protobuf/"+test.name, b, 0644) + os.WriteFile("protobuf/"+test.name, b, 0644) } } diff --git a/proto/proto_test.go b/proto/proto_test.go index fd7573a..d978aaf 100644 --- a/proto/proto_test.go +++ b/proto/proto_test.go @@ -3,8 +3,8 @@ package proto import ( "encoding/binary" "fmt" - "io/ioutil" "math" + "os" "reflect" "testing" ) @@ -366,7 +366,7 @@ func TestMarshalUnmarshal(t *testing.T) { } func loadProtobuf(t *testing.T, fileName string) RawMessage { - b, err := ioutil.ReadFile("fixtures/protobuf/" + fileName) + b, err := os.ReadFile("fixtures/protobuf/" + fileName) if err != nil { t.Fatal(err) } diff --git a/thrift/decode.go b/thrift/decode.go index 341d10a..4cc394b 100644 --- a/thrift/decode.go +++ b/thrift/decode.go @@ -5,7 +5,6 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "reflect" "sync/atomic" ) @@ -656,7 +655,7 @@ func skipBinary(r Reader) error { case *bufio.Reader: _, err = x.Discard(int(n)) default: - _, err = io.CopyN(ioutil.Discard, x, int64(n)) + _, err = io.CopyN(io.Discard, x, int64(n)) } return dontExpectEOF(err) } From 93030d39bf47faefc6e56ab8de8876ddcea83267 Mon Sep 17 00:00:00 2001 From: Kevin Burke Date: Tue, 3 Sep 2024 16:30:33 -0700 Subject: [PATCH 05/10] .github: fix security vuln --- .github/workflows/benchmark.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 805d1c4..ce38da0 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -30,7 +30,7 @@ jobs: run: go test -v -run '^$' -bench '(Marshal|Unmarshal)$/codeResponse' -benchmem -benchtime 3s -cpu 1 -count 6 ./json | tee bench.txt - name: Upload Benchmarks - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: ${{ matrix.ref }} path: bench.txt @@ -49,7 +49,7 @@ jobs: run: go install golang.org/x/perf/cmd/benchstat@latest - name: Download Benchmark Results - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: path: . @@ -57,7 +57,7 @@ jobs: run: benchstat ./master/bench.txt ./${{ github.sha }}/bench.txt | tee benchstat.txt - name: Upload Benchstat Results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: benchstat path: benchstat.txt From a8de6b894791a3f1fdc1be2cf9e9a5f9fdcefbbd Mon Sep 17 00:00:00 2001 From: Jeremy Larkin Date: Thu, 21 Nov 2024 09:11:13 -0800 Subject: [PATCH 06/10] Add protobuf rewrite rule overrides (#144) * allow rewrite rule overrides and add bitwise-or rewriter * finish docs * remove repeated, these apply a little odd in the rewriter --- proto/rewrite.go | 150 ++++++++++++++++++++++++++++++++++++++++-- proto/rewrite_test.go | 64 ++++++++++++++++++ 2 files changed, 207 insertions(+), 7 deletions(-) diff --git a/proto/rewrite.go b/proto/rewrite.go index 90957a1..ef7c0f5 100644 --- a/proto/rewrite.go +++ b/proto/rewrite.go @@ -139,17 +139,27 @@ func (f fieldset) index(i int) (int, int) { // ParseRewriteTemplate constructs a Rewriter for a protobuf type using the // given json template to describe the rewrite rules. // -// The json template contains a representation of the -func ParseRewriteTemplate(typ Type, jsonTemplate []byte) (Rewriter, error) { +// The json template contains a representation of the message that is used as the +// source values to overwrite in the protobuf targeted by the resulting rewriter. +// +// The rules are an optional set of RewriterRules that can provide alternative +// Rewriters from the default used for the field type. These rules are given the +// json.RawMessage bytes from the template, and they are expected to create a +// Rewriter to be applied against the target protobuf. +func ParseRewriteTemplate(typ Type, jsonTemplate []byte, rules ...RewriterRules) (Rewriter, error) { switch typ.Kind() { case Struct: - return parseRewriteTemplateStruct(typ, 0, jsonTemplate) + return parseRewriteTemplateStruct(typ, 0, jsonTemplate, rules...) default: return nil, fmt.Errorf("cannot construct a rewrite template from a non-struct type %s", typ.Name()) } } -func parseRewriteTemplate(t Type, f FieldNumber, j json.RawMessage) (Rewriter, error) { +func parseRewriteTemplate(t Type, f FieldNumber, j json.RawMessage, rule any) (Rewriter, error) { + if rwer, ok := rule.(Rewriterer); ok { + return rwer.Rewriter(t, f, j) + } + switch t.Kind() { case Bool: return parseRewriteTemplateBool(t, f, j) @@ -184,7 +194,11 @@ func parseRewriteTemplate(t Type, f FieldNumber, j json.RawMessage) (Rewriter, e case Map: return parseRewriteTemplateMap(t, f, j) case Struct: - return parseRewriteTemplateStruct(t, f, j) + sub, n, ok := [1]RewriterRules{}, 0, false + if sub[0], ok = rule.(RewriterRules); ok { + n = 1 + } + return parseRewriteTemplateStruct(t, f, j, sub[:n]...) default: return nil, fmt.Errorf("cannot construct a rewriter from type %s", t.Name()) } @@ -376,7 +390,7 @@ func parseRewriteTemplateMap(t Type, f FieldNumber, j json.RawMessage) (Rewriter return MultiRewriter(rewriters...), nil } -func parseRewriteTemplateStruct(t Type, f FieldNumber, j json.RawMessage) (Rewriter, error) { +func parseRewriteTemplateStruct(t Type, f FieldNumber, j json.RawMessage, rules ...RewriterRules) (Rewriter, error) { template := map[string]json.RawMessage{} if err := json.Unmarshal(j, &template); err != nil { @@ -408,10 +422,18 @@ func parseRewriteTemplateStruct(t Type, f FieldNumber, j json.RawMessage) (Rewri fields = []json.RawMessage{v} } + var rule any + for i := range rules { + if r, ok := rules[i][f.Name]; ok { + rule = r + break + } + } + rewriters = rewriters[:0] for _, v := range fields { - rw, err := parseRewriteTemplate(f.Type, f.Number, v) + rw, err := parseRewriteTemplate(f.Type, f.Number, v, rule) if err != nil { return nil, fmt.Errorf("%s: %w", k, err) } @@ -462,3 +484,117 @@ func (f *embddedRewriter) Rewrite(out, in []byte) ([]byte, error) { copy(out[prefix:], b[:tagAndLen]) return out, nil } + +// RewriterRules defines a set of rules for overriding the Rewriter used for any +// particular field. These maps may be nested for defining rules for struct members. +// +// For example: +// +// rules := proto.RewriterRules { +// "flags": proto.BitOr[uint64]{}, +// "nested": proto.RewriterRules { +// "name": myCustomRewriter, +// }, +// } +type RewriterRules map[string]any + +// Rewriterer is the interface for producing a Rewriter for a given Type, FieldNumber +// and json.RawMessage. The JSON value is the JSON-encoded payload that should be +// decoded to produce the appropriate Rewriter. Implementations of the Rewriterer +// interface are added to the RewriterRules to specify the rules for performing +// custom rewrite logic. +type Rewriterer interface { + Rewriter(Type, FieldNumber, json.RawMessage) (Rewriter, error) +} + +// BitOr implments the Rewriterer interface for providing a bitwise-or rewrite +// logic for integers rather than replacing them. Instances of this type are +// zero-size, carrying only the generic type for creating the appropriate +// Rewriter when requested. +// +// Adding these to a RewriterRules looks like: +// +// rules := proto.RewriterRules { +// "flags": proto.BitOr[uint64]{}, +// } +// +// When used as a rule when rewriting from a template, the BitOr expects a JSON- +// encoded integer passed into the Rewriter method. This parsed integer is then +// used to perform a bitwise-or against the protobuf message that is being rewritten. +// +// The above example can then be used like: +// +// template := []byte(`{"flags": 8}`) // n |= 0b1000 +// rw, err := proto.ParseRewriteTemplate(typ, template, rules) +type BitOr[T integer] struct{} + +// integer is the contraint used by the BitOr Rewriterer and the bitOrRW Rewriter. +// Because these perform bitwise-or operations, the types must be integer-like. +type integer interface { + ~int | ~int32 | ~int64 | ~uint | ~uint32 | ~uint64 +} + +// Rewriter implements the Rewriterer interface. The JSON value provided to this +// method comes from the template used for rewriting. The returned Rewriter will use +// this JSON-encoded integer to perform a bitwise-or against the protobuf message +// that is being rewritten. +func (BitOr[T]) Rewriter(t Type, f FieldNumber, j json.RawMessage) (Rewriter, error) { + var v T + err := json.Unmarshal(j, &v) + if err != nil { + return nil, err + } + return BitOrRewriter(t, f, v) +} + +// BitOrRewriter creates a bitwise-or Rewriter for a given field type and number. +// The mask is the value or'ed with values in the target protobuf. +func BitOrRewriter[T integer](t Type, f FieldNumber, mask T) (Rewriter, error) { + switch t.Kind() { + case Int32, Int64, Sint32, Sint64, Uint32, Uint64, Fix32, Fix64, Sfix32, Sfix64: + default: + return nil, fmt.Errorf("cannot construct a rewriter from type %s", t.Name()) + } + return bitOrRW[T]{mask: mask, t: t, f: f}, nil +} + +// bitOrRW is the Rewriter returned by the BitOr Rewriter method. +type bitOrRW[T integer] struct { + mask T + t Type + f FieldNumber +} + +// Rewrite implements the Rewriter interface performing a bitwise-or between the +// template value and the input value. +func (r bitOrRW[T]) Rewrite(out, in []byte) ([]byte, error) { + var v T + if err := Unmarshal(in, &v); err != nil { + return nil, err + } + + v |= r.mask + + switch r.t.Kind() { + case Int32: + return r.f.Int32(int32(v)).Rewrite(out, in) + case Int64: + return r.f.Int64(int64(v)).Rewrite(out, in) + case Sint32: + return r.f.Uint32(encodeZigZag32(int32(v))).Rewrite(out, in) + case Sint64: + return r.f.Uint64(encodeZigZag64(int64(v))).Rewrite(out, in) + case Uint32, Uint64: + return r.f.Uint64(uint64(v)).Rewrite(out, in) + case Fix32: + return r.f.Fixed32(uint32(v)).Rewrite(out, in) + case Fix64: + return r.f.Fixed64(uint64(v)).Rewrite(out, in) + case Sfix32: + return r.f.Fixed32(encodeZigZag32(int32(v))).Rewrite(out, in) + case Sfix64: + return r.f.Fixed64(encodeZigZag64(int64(v))).Rewrite(out, in) + } + + panic("unreachable") // Kind is validated when creating instances +} diff --git a/proto/rewrite_test.go b/proto/rewrite_test.go index 49960f8..5ff906d 100644 --- a/proto/rewrite_test.go +++ b/proto/rewrite_test.go @@ -291,6 +291,70 @@ func TestParseRewriteTemplate(t *testing.T) { } } +func TestParseRewriteRules(t *testing.T) { + type submessage struct { + Flags uint64 `protobuf:"varint,1,opt,name=flags,proto3"` + } + + type message struct { + Flags uint64 `protobuf:"varint,2,opt,name=flags,proto3"` + Subfield *submessage `protobuf:"bytes,99,opt,name=subfield,proto3"` + } + + original := &message{ + Flags: 0b00000001, + Subfield: &submessage{ + Flags: 0b00000010, + }, + } + + expected := &message{ + Flags: 0b00000001 | 16, + Subfield: &submessage{ + Flags: 0b00000010 | 32, + }, + } + + rules := RewriterRules{ + "flags": BitOr[uint64]{}, + "subfield": RewriterRules{ + "flags": BitOr[uint64]{}, + }, + } + + rw, err := ParseRewriteTemplate(TypeOf(reflect.TypeOf(original)), []byte(`{ + "flags": 16, + "subfield": { + "flags": 32 + } +}`), rules) + + if err != nil { + t.Fatal(err) + } + + b1, err := Marshal(original) + if err != nil { + t.Fatal(err) + } + + b2, err := rw.Rewrite(nil, b1) + if err != nil { + t.Fatal(err) + } + + found := &message{} + if err := Unmarshal(b2, &found); err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(expected, found) { + t.Error("messages mismatch after rewrite") + t.Logf("want:\n%+v", expected) + t.Logf("got:\n%+v", found) + } +} + func BenchmarkRewrite(b *testing.B) { type message struct { A int From a54af38dc519c77f48db0d3f483a208da286cf88 Mon Sep 17 00:00:00 2001 From: Kevin Gillette Date: Mon, 9 Jun 2025 14:28:19 -0600 Subject: [PATCH 07/10] json: encode \b and \f like the Go 1.22 stdlib does (#145) * go.mod: upgrade to 1.23 (oldest supported by Go project) * json: encode `\b` and `\f` like the Go 1.22 stdlib does https://go-review.googlesource.com/c/go/+/521675 --- .github/workflows/benchmark.yml | 4 ++-- .github/workflows/test.yml | 4 ++-- go.mod | 2 +- json/encode.go | 25 ++----------------------- json/golang_encode_test.go | 4 ++-- json/string.go | 19 +++++++++++++++++++ 6 files changed, 28 insertions(+), 30 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ce38da0..1bd2c95 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -18,7 +18,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: "1.21" + go-version: "1.24" - name: Checkout uses: actions/checkout@v2 @@ -43,7 +43,7 @@ jobs: - name: Steup Go uses: actions/setup-go@v2 with: - go-version: "1.21" + go-version: "1.24" - name: Setup Benchstat run: go install golang.org/x/perf/cmd/benchstat@latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7e4bc41..25fac3f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,8 +9,8 @@ jobs: strategy: matrix: go: - - "1.20" - - "1.21" + - "1.23" + - "1.24" runs-on: ubuntu-latest diff --git a/go.mod b/go.mod index 0096cd2..28ffff2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/segmentio/encoding -go 1.18 +go 1.23 require github.com/segmentio/asm v1.1.3 diff --git a/json/encode.go b/json/encode.go index 4173ddd..1b6379b 100644 --- a/json/encode.go +++ b/json/encode.go @@ -158,30 +158,9 @@ func (e encoder) encodeString(b []byte, p unsafe.Pointer) ([]byte, error) { } switch c { - case '\\', '"': + case '\\', '"', '\b', '\f', '\n', '\r', '\t': b = append(b, s[i:j]...) - b = append(b, '\\', c) - i = j + 1 - j = j + 1 - continue - - case '\n': - b = append(b, s[i:j]...) - b = append(b, '\\', 'n') - i = j + 1 - j = j + 1 - continue - - case '\r': - b = append(b, s[i:j]...) - b = append(b, '\\', 'r') - i = j + 1 - j = j + 1 - continue - - case '\t': - b = append(b, s[i:j]...) - b = append(b, '\\', 't') + b = append(b, '\\', escapeByteRepr(c)) i = j + 1 j = j + 1 continue diff --git a/json/golang_encode_test.go b/json/golang_encode_test.go index b31b6a4..5c465dc 100644 --- a/json/golang_encode_test.go +++ b/json/golang_encode_test.go @@ -685,11 +685,11 @@ var encodeStringTests = []struct { {"\x05", `"\u0005"`}, {"\x06", `"\u0006"`}, {"\x07", `"\u0007"`}, - {"\x08", `"\u0008"`}, + {"\x08", `"\b"`}, {"\x09", `"\t"`}, {"\x0a", `"\n"`}, {"\x0b", `"\u000b"`}, - {"\x0c", `"\u000c"`}, + {"\x0c", `"\f"`}, {"\x0d", `"\r"`}, {"\x0e", `"\u000e"`}, {"\x0f", `"\u000f"`}, diff --git a/json/string.go b/json/string.go index dba5c5d..a9a972b 100644 --- a/json/string.go +++ b/json/string.go @@ -40,6 +40,25 @@ func escapeIndex(s string, escapeHTML bool) int { return -1 } +func escapeByteRepr(b byte) byte { + switch b { + case '\\', '"': + return b + case '\b': + return 'b' + case '\f': + return 'f' + case '\n': + return 'n' + case '\r': + return 'r' + case '\t': + return 't' + } + + return 0 +} + // below return a mask that can be used to determine if any of the bytes // in `n` are below `b`. If a byte's MSB is set in the mask then that byte was // below `b`. The result is only valid if `b`, and each byte in `n`, is below From 7a0606c75807b35f13b92dcbb6bf7d098631865c Mon Sep 17 00:00:00 2001 From: Kevin Gillette Date: Mon, 9 Jun 2025 15:06:38 -0600 Subject: [PATCH 08/10] lint fixes (#146) --- ascii/ascii_test.go | 4 +- internal/runtime_reflect/map.go | 1 + iso8601/parse_test.go | 46 ++-- iso8601/valid.go | 2 +- iso8601/valid_test.go | 8 +- json/bugs/issue11/main_test.go | 2 +- json/codec.go | 30 +-- json/decode.go | 38 ++-- json/encode.go | 34 +-- json/fuzz/fuzz.go | 21 +- json/golang_bench_test.go | 22 +- json/golang_decode_test.go | 103 ++++----- json/golang_encode_test.go | 373 ++++++++++++++++---------------- json/golang_example_test.go | 6 +- json/golang_number_test.go | 8 +- json/golang_scanner_test.go | 21 +- json/golang_shim_test.go | 10 +- json/golang_tagkey_test.go | 6 +- json/int_test.go | 6 +- json/json.go | 21 +- json/json_test.go | 141 ++++++------ json/parse.go | 18 +- json/parse_test.go | 33 +-- json/string_test.go | 2 +- json/token.go | 8 +- json/token_test.go | 2 +- proto/bytes_test.go | 2 +- proto/decode.go | 4 +- proto/decode_test.go | 9 +- proto/encode_test.go | 12 +- proto/error.go | 4 +- proto/fixtures/fixtures.pb.go | 4 +- proto/fixtures/generate/main.go | 4 +- proto/message.go | 2 +- proto/message_test.go | 2 +- proto/proto.go | 12 +- proto/proto_test.go | 22 +- proto/reflect.go | 2 +- proto/reflect_test.go | 2 +- proto/rewrite.go | 2 +- proto/rewrite_test.go | 3 +- proto/slice.go | 4 +- proto/struct.go | 6 +- thrift/debug.go | 4 +- thrift/decode.go | 20 +- thrift/encode.go | 16 +- thrift/protocol_test.go | 30 +-- thrift/struct.go | 2 +- thrift/thrift_test.go | 52 ++--- 49 files changed, 596 insertions(+), 590 deletions(-) diff --git a/ascii/ascii_test.go b/ascii/ascii_test.go index 396ff3b..4512113 100644 --- a/ascii/ascii_test.go +++ b/ascii/ascii_test.go @@ -97,7 +97,7 @@ func BenchmarkValidPrint(b *testing.B) { func benchmarkValidationFunction(b *testing.B, function func(string) bool) { for _, test := range testStrings { b.Run(limit(test), func(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { _ = function(test) } b.SetBytes(int64(len(test))) @@ -233,7 +233,7 @@ func BenchmarkEqualFold(b *testing.B) { b.Run(limit(test), func(b *testing.B) { other := test + "_" // not the same pointer - for i := 0; i < b.N; i++ { + for range b.N { _ = EqualFoldString(test, other[:len(test)]) // same length } diff --git a/internal/runtime_reflect/map.go b/internal/runtime_reflect/map.go index b682acc..5864563 100644 --- a/internal/runtime_reflect/map.go +++ b/internal/runtime_reflect/map.go @@ -79,6 +79,7 @@ func makemap(t unsafe.Pointer, cap int) unsafe.Pointer // m escapes into the return value, but the caller of mapiterinit // doesn't let the return value escape. +// //go:noescape //go:linkname mapiterinit runtime.mapiterinit func mapiterinit(t unsafe.Pointer, m unsafe.Pointer, it *hiter) diff --git a/iso8601/parse_test.go b/iso8601/parse_test.go index 1094d2a..ad335f7 100644 --- a/iso8601/parse_test.go +++ b/iso8601/parse_test.go @@ -78,9 +78,9 @@ func TestParse(t *testing.T) { } // Check ~4M YYYY-MM-DD dates in 20 byte form. - for year := 0; year <= 9999; year++ { - for month := 0; month <= 13; month++ { - for day := 0; day <= 32; day++ { + for year := range 10000 { + for month := range 14 { + for day := range 33 { input := fmt.Sprintf("%04d-%02d-%02dT12:34:56Z", year, month, day) expect, expectErr := time.Parse(time.RFC3339Nano, input) actual, actualErr := Parse(input) @@ -94,9 +94,9 @@ func TestParse(t *testing.T) { } // Check ~4M YYYY-MM-DD dates in 24 byte form. - for year := 0; year <= 9999; year++ { - for month := 0; month <= 13; month++ { - for day := 0; day <= 32; day++ { + for year := range 10000 { + for month := range 14 { + for day := range 33 { input := fmt.Sprintf("%04d-%02d-%02dT12:34:56.789Z", year, month, day) expect, expectErr := time.Parse(time.RFC3339Nano, input) actual, actualErr := Parse(input) @@ -110,9 +110,9 @@ func TestParse(t *testing.T) { } // Check ~4M YYYY-MM-DD dates in 30 byte form. - for year := 0; year <= 9999; year++ { - for month := 0; month <= 13; month++ { - for day := 0; day <= 32; day++ { + for year := range 10000 { + for month := range 14 { + for day := range 33 { input := fmt.Sprintf("%04d-%02d-%02dT12:34:56.123456789Z", year, month, day) expect, expectErr := time.Parse(time.RFC3339Nano, input) actual, actualErr := Parse(input) @@ -126,9 +126,9 @@ func TestParse(t *testing.T) { } // Check all ~1M HH:MM:SS times in 20 byte form. - for hour := 0; hour < 100; hour++ { - for minute := 0; minute < 100; minute++ { - for second := 0; second < 100; second++ { + for hour := range 100 { + for minute := range 100 { + for second := range 100 { input := fmt.Sprintf("2000-01-01T%02d:%02d:%02dZ", hour, minute, second) expect, expectErr := time.Parse(time.RFC3339Nano, input) actual, actualErr := Parse(input) @@ -142,9 +142,9 @@ func TestParse(t *testing.T) { } // Check ~1M HH:MM:SS.MMM times in 24 byte form. - for hour := 0; hour < 100; hour++ { - for minute := 0; minute < 100; minute++ { - for second := 0; second < 100; second++ { + for hour := range 100 { + for minute := range 100 { + for second := range 100 { input := fmt.Sprintf("2000-01-01T%02d:%02d:%02d.123Z", hour, minute, second) expect, expectErr := time.Parse(time.RFC3339Nano, input) actual, actualErr := Parse(input) @@ -158,9 +158,9 @@ func TestParse(t *testing.T) { } // Check ~1M HH:MM:SS.MMM times in 30 byte form. - for hour := 0; hour < 100; hour++ { - for minute := 0; minute < 100; minute++ { - for second := 0; second < 100; second++ { + for hour := range 100 { + for minute := range 100 { + for second := range 100 { input := fmt.Sprintf("2000-01-01T%02d:%02d:%02d.123456789Z", hour, minute, second) expect, expectErr := time.Parse(time.RFC3339Nano, input) actual, actualErr := Parse(input) @@ -405,31 +405,31 @@ func TestParseInvalid(t *testing.T) { } func BenchmarkParse(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { Parse("2006-01-02T15:04:05Z") } } func BenchmarkParseMilliseconds(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { Parse("2006-01-02T15:04:05.123Z") } } func BenchmarkParseMicroseconds(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { Parse("2006-01-02T15:04:05.123456Z") } } func BenchmarkParseNanoseconds(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { Parse("2006-01-02T15:04:05.123456789Z") } } func BenchmarkParseInvalid(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { Parse("2006-01-02T15:04:05.XZ") } } diff --git a/iso8601/valid.go b/iso8601/valid.go index 2c4edec..187b4ef 100644 --- a/iso8601/valid.go +++ b/iso8601/valid.go @@ -1,7 +1,7 @@ package iso8601 // ValidFlags is a bitset type used to configure the behavior of the Valid -//function. +// function. type ValidFlags int const ( diff --git a/iso8601/valid_test.go b/iso8601/valid_test.go index acfbab1..3879205 100644 --- a/iso8601/valid_test.go +++ b/iso8601/valid_test.go @@ -68,7 +68,7 @@ func BenchmarkValidate(b *testing.B) { } func benchmarkValidateSuccess(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { if !Valid("2018-01-01T23:42:59.123456789Z", Flexible) { b.Fatal("not valid") } @@ -76,7 +76,7 @@ func benchmarkValidateSuccess(b *testing.B) { } func benchmarkValidateFailure(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { if Valid("2018-01-01T23:42:59 oops!", Flexible) { b.Fatal("valid but should not") } @@ -89,7 +89,7 @@ func BenchmarkTimeParse(b *testing.B) { } func benchmarkTimeParseSuccess(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { if _, err := time.Parse(time.RFC3339Nano, "2018-01-01T23:42:59.123456789Z"); err != nil { b.Fatal("not valid") } @@ -97,7 +97,7 @@ func benchmarkTimeParseSuccess(b *testing.B) { } func benchmarkTimeParseFailure(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { if _, err := time.Parse(time.RFC3339Nano, "2018-01-01T23:42:59 oops!"); err == nil { b.Fatal("valid but should not") } diff --git a/json/bugs/issue11/main_test.go b/json/bugs/issue11/main_test.go index eeae147..2541186 100644 --- a/json/bugs/issue11/main_test.go +++ b/json/bugs/issue11/main_test.go @@ -9,7 +9,7 @@ import ( ) func TestIssue11(t *testing.T) { - m := map[string]map[string]interface{}{ + m := map[string]map[string]any{ "outerkey": { "innerkey": "innervalue", }, diff --git a/json/codec.go b/json/codec.go index bd3c1d4..32c078f 100644 --- a/json/codec.go +++ b/json/codec.go @@ -44,11 +44,15 @@ type decoder struct { flags ParseFlags } -type encodeFunc func(encoder, []byte, unsafe.Pointer) ([]byte, error) -type decodeFunc func(decoder, []byte, unsafe.Pointer) ([]byte, error) +type ( + encodeFunc func(encoder, []byte, unsafe.Pointer) ([]byte, error) + decodeFunc func(decoder, []byte, unsafe.Pointer) ([]byte, error) +) -type emptyFunc func(unsafe.Pointer) bool -type sortFunc func([]reflect.Value) +type ( + emptyFunc func(unsafe.Pointer) bool + sortFunc func([]reflect.Value) +) // Eventually consistent cache mapping go types to dynamically generated // codecs. @@ -558,7 +562,7 @@ func appendStructFields(fields []structField, t reflect.Type, offset uintptr, se names := make(map[string]struct{}) embedded := make([]embeddedField, 0, 10) - for i, n := 0, t.NumField(); i < n; i++ { + for i := range t.NumField() { f := t.Field(i) var ( @@ -706,7 +710,7 @@ func appendStructFields(fields []structField, t reflect.Type, offset uintptr, se for _, embfield := range embedded { subfield := *embfield.subfield - if ambiguousNames[subfield.name] > 1 && !(subfield.tag && ambiguousTags[subfield.name] == 1) { + if ambiguousNames[subfield.name] > 1 && (!subfield.tag || ambiguousTags[subfield.name] != 1) { continue // ambiguous embedded field } @@ -1001,14 +1005,14 @@ var syntaxErrorMsgOffset = ^uintptr(0) func init() { t := reflect.TypeOf(SyntaxError{}) - for i, n := 0, t.NumField(); i < n; i++ { + for i := range t.NumField() { if f := t.Field(i); f.Type.Kind() == reflect.String { syntaxErrorMsgOffset = f.Offset } } } -func syntaxError(b []byte, msg string, args ...interface{}) error { +func syntaxError(b []byte, msg string, args ...any) error { e := new(SyntaxError) i := syntaxErrorMsgOffset if i != ^uintptr(0) { @@ -1096,15 +1100,15 @@ var ( timePtrType = reflect.PtrTo(timeType) rawMessagePtrType = reflect.PtrTo(rawMessageType) - sliceInterfaceType = reflect.TypeOf(([]interface{})(nil)) - sliceStringType = reflect.TypeOf(([]interface{})(nil)) - mapStringInterfaceType = reflect.TypeOf((map[string]interface{})(nil)) + sliceInterfaceType = reflect.TypeOf(([]any)(nil)) + sliceStringType = reflect.TypeOf(([]any)(nil)) + mapStringInterfaceType = reflect.TypeOf((map[string]any)(nil)) mapStringRawMessageType = reflect.TypeOf((map[string]RawMessage)(nil)) mapStringStringType = reflect.TypeOf((map[string]string)(nil)) mapStringStringSliceType = reflect.TypeOf((map[string][]string)(nil)) mapStringBoolType = reflect.TypeOf((map[string]bool)(nil)) - interfaceType = reflect.TypeOf((*interface{})(nil)).Elem() + interfaceType = reflect.TypeOf((*any)(nil)).Elem() jsonMarshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() jsonUnmarshalerType = reflect.TypeOf((*Unmarshaler)(nil)).Elem() textMarshalerType = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem() @@ -1203,7 +1207,7 @@ func fmtFrac(buf []byte, v uint64, prec int) (nw int, nv uint64) { // Omit trailing zeros up to and including decimal point. w := len(buf) print := false - for i := 0; i < prec; i++ { + for range prec { digit := v % 10 print = print || digit != 0 if print { diff --git a/json/decode.go b/json/decode.go index 9792af0..46922d5 100644 --- a/json/decode.go +++ b/json/decode.go @@ -514,7 +514,7 @@ func (d decoder) decodeArray(b []byte, p unsafe.Pointer, n int, size uintptr, t b = b[1:] var err error - for i := 0; i < n; i++ { + for i := range n { b = skipSpaces(b) if i != 0 { @@ -564,10 +564,8 @@ func (d decoder) decodeArray(b []byte, p unsafe.Pointer, n int, size uintptr, t } } -var ( - // This is a placeholder used to consturct non-nil empty slices. - empty struct{} -) +// This is a placeholder used to consturct non-nil empty slices. +var empty struct{} func (d decoder) decodeSlice(b []byte, p unsafe.Pointer, size uintptr, t reflect.Type, decode decodeFunc) ([]byte, error) { if hasNullPrefix(b) { @@ -737,10 +735,10 @@ func (d decoder) decodeMapStringInterface(b []byte, p unsafe.Pointer) ([]byte, e } i := 0 - m := *(*map[string]interface{})(p) + m := *(*map[string]any)(p) if m == nil { - m = make(map[string]interface{}, 64) + m = make(map[string]any, 64) } var ( @@ -829,7 +827,7 @@ func (d decoder) decodeMapStringRawMessage(b []byte, p unsafe.Pointer) ([]byte, var err error var key string var val RawMessage - var input = b + input := b b = b[1:] for { @@ -910,7 +908,7 @@ func (d decoder) decodeMapStringString(b []byte, p unsafe.Pointer) ([]byte, erro var err error var key string var val string - var input = b + input := b b = b[1:] for { @@ -991,8 +989,8 @@ func (d decoder) decodeMapStringStringSlice(b []byte, p unsafe.Pointer) ([]byte, var err error var key string var buf []string - var input = b - var stringSize = unsafe.Sizeof("") + input := b + stringSize := unsafe.Sizeof("") b = b[1:] for { @@ -1076,7 +1074,7 @@ func (d decoder) decodeMapStringBool(b []byte, p unsafe.Pointer) ([]byte, error) var err error var key string var val bool - var input = b + input := b b = b[1:] for { @@ -1153,7 +1151,7 @@ func (d decoder) decodeStruct(b []byte, p unsafe.Pointer, st *structType) ([]byt // memory buffer used to convert short field names to lowercase var buf [64]byte var key []byte - var input = b + input := b b = b[1:] for { @@ -1265,8 +1263,8 @@ func (d decoder) decodePointer(b []byte, p unsafe.Pointer, t reflect.Type, decod } func (d decoder) decodeInterface(b []byte, p unsafe.Pointer) ([]byte, error) { - val := *(*interface{})(p) - *(*interface{})(p) = nil + val := *(*any)(p) + *(*any)(p) = nil if t := reflect.TypeOf(val); t != nil && t.Kind() == reflect.Ptr { if v := reflect.ValueOf(val); v.IsNil() || t.Elem().Kind() != reflect.Ptr { @@ -1274,14 +1272,14 @@ func (d decoder) decodeInterface(b []byte, p unsafe.Pointer) ([]byte, error) { // `null`, and the encoding/json package always nils the destination // interface value in this case. if hasNullPrefix(b) { - *(*interface{})(p) = nil + *(*any)(p) = nil return b[4:], nil } } b, err := Parse(b, val, d.flags) if err == nil { - *(*interface{})(p) = val + *(*any)(p) = val } return b, err @@ -1325,7 +1323,7 @@ func (d decoder) decodeInterface(b []byte, p unsafe.Pointer) ([]byte, error) { return b, syntaxError(v, "unexpected trailing trailing tokens after json value") } - *(*interface{})(p) = val + *(*any)(p) = val return b, nil } @@ -1393,7 +1391,7 @@ func (d decoder) decodeDynamicNumber(b []byte, p unsafe.Pointer) ([]byte, error) func (d decoder) decodeMaybeEmptyInterface(b []byte, p unsafe.Pointer, t reflect.Type) ([]byte, error) { if hasNullPrefix(b) { - *(*interface{})(p) = nil + *(*any)(p) = nil return b[4:], nil } @@ -1402,7 +1400,7 @@ func (d decoder) decodeMaybeEmptyInterface(b []byte, p unsafe.Pointer, t reflect return Parse(b, e.Interface(), d.flags) } } else if t.NumMethod() == 0 { // empty interface - return Parse(b, (*interface{})(p), d.flags) + return Parse(b, (*any)(p), d.flags) } return d.decodeUnmarshalTypeError(b, p, t) diff --git a/json/encode.go b/json/encode.go index 1b6379b..2a6da07 100644 --- a/json/encode.go +++ b/json/encode.go @@ -278,11 +278,11 @@ func (e encoder) encodeTime(b []byte, p unsafe.Pointer) ([]byte, error) { } func (e encoder) encodeArray(b []byte, p unsafe.Pointer, n int, size uintptr, t reflect.Type, encode encodeFunc) ([]byte, error) { - var start = len(b) + start := len(b) var err error b = append(b, '[') - for i := 0; i < n; i++ { + for i := range n { if i != 0 { b = append(b, ',') } @@ -316,7 +316,7 @@ func (e encoder) encodeMap(b []byte, p unsafe.Pointer, t reflect.Type, encodeKey sortKeys(keys) } - var start = len(b) + start := len(b) var err error b = append(b, '{') @@ -344,7 +344,7 @@ func (e encoder) encodeMap(b []byte, p unsafe.Pointer, t reflect.Type, encodeKey type element struct { key string - val interface{} + val any raw RawMessage } @@ -357,11 +357,11 @@ func (m *mapslice) Less(i, j int) bool { return m.elements[i].key < m.elements[j func (m *mapslice) Swap(i, j int) { m.elements[i], m.elements[j] = m.elements[j], m.elements[i] } var mapslicePool = sync.Pool{ - New: func() interface{} { return new(mapslice) }, + New: func() any { return new(mapslice) }, } func (e encoder) encodeMapStringInterface(b []byte, p unsafe.Pointer) ([]byte, error) { - m := *(*map[string]interface{})(p) + m := *(*map[string]any)(p) if m == nil { return append(b, "null"...), nil } @@ -373,7 +373,7 @@ func (e encoder) encodeMapStringInterface(b []byte, p unsafe.Pointer) ([]byte, e if len(m) != 0 { var err error - var i = 0 + i := 0 for k, v := range m { if i != 0 { @@ -405,7 +405,7 @@ func (e encoder) encodeMapStringInterface(b []byte, p unsafe.Pointer) ([]byte, e } sort.Sort(s) - var start = len(b) + start := len(b) var err error b = append(b, '{') @@ -451,7 +451,7 @@ func (e encoder) encodeMapStringRawMessage(b []byte, p unsafe.Pointer) ([]byte, if len(m) != 0 { var err error - var i = 0 + i := 0 for k, v := range m { if i != 0 { @@ -484,7 +484,7 @@ func (e encoder) encodeMapStringRawMessage(b []byte, p unsafe.Pointer) ([]byte, } sort.Sort(s) - var start = len(b) + start := len(b) var err error b = append(b, '{') @@ -529,7 +529,7 @@ func (e encoder) encodeMapStringString(b []byte, p unsafe.Pointer) ([]byte, erro b = append(b, '{') if len(m) != 0 { - var i = 0 + i := 0 for k, v := range m { if i != 0 { @@ -589,7 +589,7 @@ func (e encoder) encodeMapStringStringSlice(b []byte, p unsafe.Pointer) ([]byte, return append(b, "null"...), nil } - var stringSize = unsafe.Sizeof("") + stringSize := unsafe.Sizeof("") if (e.flags & SortMapKeys) == 0 { // Optimized code path when the program does not need the map keys to be @@ -598,7 +598,7 @@ func (e encoder) encodeMapStringStringSlice(b []byte, p unsafe.Pointer) ([]byte, if len(m) != 0 { var err error - var i = 0 + i := 0 for k, v := range m { if i != 0 { @@ -631,7 +631,7 @@ func (e encoder) encodeMapStringStringSlice(b []byte, p unsafe.Pointer) ([]byte, } sort.Sort(s) - var start = len(b) + start := len(b) var err error b = append(b, '{') @@ -676,7 +676,7 @@ func (e encoder) encodeMapStringBool(b []byte, p unsafe.Pointer) ([]byte, error) b = append(b, '{') if len(m) != 0 { - var i = 0 + i := 0 for k, v := range m { if i != 0 { @@ -736,7 +736,7 @@ func (e encoder) encodeMapStringBool(b []byte, p unsafe.Pointer) ([]byte, error) } func (e encoder) encodeStruct(b []byte, p unsafe.Pointer, st *structType) ([]byte, error) { - var start = len(b) + start := len(b) var err error var k string var n int @@ -813,7 +813,7 @@ func (e encoder) encodePointer(b []byte, p unsafe.Pointer, t reflect.Type, encod } func (e encoder) encodeInterface(b []byte, p unsafe.Pointer) ([]byte, error) { - return Append(b, *(*interface{})(p), e.flags) + return Append(b, *(*any)(p), e.flags) } func (e encoder) encodeMaybeEmptyInterface(b []byte, p unsafe.Pointer, t reflect.Type) ([]byte, error) { diff --git a/json/fuzz/fuzz.go b/json/fuzz/fuzz.go index 667ca44..3f1bf6b 100644 --- a/json/fuzz/fuzz.go +++ b/json/fuzz/fuzz.go @@ -1,3 +1,4 @@ +//go:build ignore // +build ignore // Copyright 2015 go-fuzz project authors. All rights reserved. @@ -15,7 +16,7 @@ import ( "github.com/segmentio/encoding/json" ) -func fixS(v interface{}) { +func fixS(v any) { if s, ok := v.(*S); ok { if len(s.P) == 0 { s.P = []byte(`""`) @@ -25,12 +26,12 @@ func fixS(v interface{}) { func Fuzz(data []byte) int { score := 0 - for _, ctor := range []func() interface{}{ - func() interface{} { return nil }, - func() interface{} { return new([]interface{}) }, - func() interface{} { m := map[string]string{}; return &m }, - func() interface{} { m := map[string]interface{}{}; return &m }, - func() interface{} { return new(S) }, + for _, ctor := range []func() any{ + func() any { return nil }, + func() any { return new([]any) }, + func() any { m := map[string]string{}; return &m }, + func() any { m := map[string]any{}; return &m }, + func() any { return new(S) }, } { // Note: we modified the test to verify that we behavior like the // standard encoding/json package, whether it's right or wrong. @@ -102,10 +103,10 @@ type S struct { D bool E uint8 F []byte - G interface{} - H map[string]interface{} + G any + H map[string]any I map[string]string - J []interface{} + J []any K []string L S1 M *S1 diff --git a/json/golang_bench_test.go b/json/golang_bench_test.go index 04c9176..201352f 100644 --- a/json/golang_bench_test.go +++ b/json/golang_bench_test.go @@ -38,8 +38,10 @@ type codeNode struct { MeanT int64 `json:"mean_t"` } -var codeJSON []byte -var codeStruct codeResponse +var ( + codeJSON []byte + codeStruct codeResponse +) func codeInit() { f, err := os.Open("testdata/code.json.gz") @@ -68,7 +70,7 @@ func codeInit() { if !bytes.Equal(data, codeJSON) { println("different lengths", len(data), len(codeJSON)) - for i := 0; i < len(data) && i < len(codeJSON); i++ { + for i := range min(len(data), len(codeJSON)) { if data[i] != codeJSON[i] { println("re-marshal: changed at byte", i) println("orig: ", string(codeJSON[i-10:i+10])) @@ -125,7 +127,7 @@ func benchMarshalBytes(n int) func(*testing.B) { bytes.Repeat(sample, (n/len(sample))+1)[:n], } return func(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { if _, err := Marshal(v); err != nil { b.Fatal("Marshal:", err) } @@ -177,7 +179,7 @@ func BenchmarkUnicodeDecoder(b *testing.B) { dec := NewDecoder(r) var out string b.ResetTimer() - for i := 0; i < b.N; i++ { + for range b.N { if err := dec.Decode(&out); err != nil { b.Fatal("Decode:", err) } @@ -191,13 +193,15 @@ func BenchmarkDecoderStream(b *testing.B) { var buf bytes.Buffer dec := NewDecoder(&buf) buf.WriteString(`"` + strings.Repeat("x", 1000000) + `"` + "\n\n\n") - var x interface{} + var x any if err := dec.Decode(&x); err != nil { b.Fatal("Decode:", err) } ones := strings.Repeat(" 1\n", 300000) + "\n\n\n" b.StartTimer() - for i := 0; i < b.N; i++ { + for i := range b.N { + // XXX: making use of the index variable + // is probably a misuse of b.N loops. if i%300000 == 0 { buf.WriteString(ones) } @@ -338,10 +342,10 @@ func BenchmarkTypeFieldsCache(b *testing.B) { ts := types[:nt] b.Run(fmt.Sprintf("MissTypes%d", nt), func(b *testing.B) { nc := runtime.GOMAXPROCS(0) - for i := 0; i < b.N; i++ { + for range b.N { clearCache() var wg sync.WaitGroup - for j := 0; j < nc; j++ { + for j := range nc { wg.Add(1) go func(j int) { for _, t := range ts[(j*len(ts))/nc : ((j+1)*len(ts))/nc] { diff --git a/json/golang_decode_test.go b/json/golang_decode_test.go index 0cc7af4..5e419a2 100644 --- a/json/golang_decode_test.go +++ b/json/golang_decode_test.go @@ -31,7 +31,7 @@ type U struct { } type V struct { - F1 interface{} + F1 any F2 int32 F3 Number F4 *VOuter @@ -62,18 +62,18 @@ func (*SS) UnmarshalJSON(data []byte) error { // ifaceNumAsFloat64/ifaceNumAsNumber are used to test unmarshaling with and // without UseNumber -var ifaceNumAsFloat64 = map[string]interface{}{ +var ifaceNumAsFloat64 = map[string]any{ "k1": float64(1), "k2": "s", - "k3": []interface{}{float64(1), float64(2.0), float64(3e-3)}, - "k4": map[string]interface{}{"kk1": "s", "kk2": float64(2)}, + "k3": []any{float64(1), float64(2.0), float64(3e-3)}, + "k4": map[string]any{"kk1": "s", "kk2": float64(2)}, } -var ifaceNumAsNumber = map[string]interface{}{ +var ifaceNumAsNumber = map[string]any{ "k1": Number("1"), "k2": "s", - "k3": []interface{}{Number("1"), Number("2.0"), Number("3e-3")}, - "k4": map[string]interface{}{"kk1": "s", "kk2": Number("2")}, + "k3": []any{Number("1"), Number("2.0"), Number("3e-3")}, + "k4": map[string]any{"kk1": "s", "kk2": Number("2")}, } type tx struct { @@ -270,9 +270,9 @@ type Ambig struct { } type XYZ struct { - X interface{} - Y interface{} - Z interface{} + X any + Y any + Z any } type unexportedWithMethods struct{} @@ -400,8 +400,8 @@ type mapStringToStringData struct { type unmarshalTest struct { in string - ptr interface{} - out interface{} + ptr any + out any err error useNumber bool golden bool @@ -420,23 +420,24 @@ var unmarshalTests = []unmarshalTest{ {in: `-5`, ptr: new(int16), out: int16(-5)}, {in: `2`, ptr: new(Number), out: Number("2"), useNumber: true}, {in: `2`, ptr: new(Number), out: Number("2")}, - {in: `2`, ptr: new(interface{}), out: float64(2.0)}, - {in: `2`, ptr: new(interface{}), out: Number("2"), useNumber: true}, + {in: `2`, ptr: new(any), out: float64(2.0)}, + {in: `2`, ptr: new(any), out: Number("2"), useNumber: true}, {in: `"a\u1234"`, ptr: new(string), out: "a\u1234"}, {in: `"http:\/\/"`, ptr: new(string), out: "http://"}, {in: `"g-clef: \uD834\uDD1E"`, ptr: new(string), out: "g-clef: \U0001D11E"}, //TODO //{in: `"invalid: \uD834x\uDD1E"`, ptr: new(string), out: "invalid: \uFFFDx\uFFFD"}, - {in: "null", ptr: new(interface{}), out: nil}, - {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &UnmarshalTypeError{"array", reflect.TypeOf(""), 7, "T", "X"}}, - {in: `{"X": 23}`, ptr: new(T), out: T{}, err: &UnmarshalTypeError{"number", reflect.TypeOf(""), 8, "T", "X"}}, {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, + {in: "null", ptr: new(any), out: nil}, + {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &UnmarshalTypeError{Value: "array", Type: reflect.TypeOf(""), Offset: 7, Struct: "T", Field: "X"}}, + {in: `{"X": 23}`, ptr: new(T), out: T{}, err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 8, Struct: "T", Field: "X"}}, + {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, {in: `{"x": 1}`, ptr: new(tx), err: fmt.Errorf("json: unknown field \"x\""), disallowUnknownFields: true}, - {in: `{"S": 23}`, ptr: new(W), out: W{}, err: &UnmarshalTypeError{"number", reflect.TypeOf(SS("")), 0, "W", "S"}}, + {in: `{"S": 23}`, ptr: new(W), out: W{}, err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(SS("")), Offset: 0, Struct: "W", Field: "S"}}, {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: Number("3")}}, {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: Number("1"), F2: int32(2), F3: Number("3")}, useNumber: true}, - {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsFloat64}, - {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsNumber, useNumber: true}, + {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(any), out: ifaceNumAsFloat64}, + {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(any), out: ifaceNumAsNumber, useNumber: true}, // raw values with whitespace {in: "\n true ", ptr: new(bool), out: true}, @@ -478,10 +479,10 @@ var unmarshalTests = []unmarshalTest{ {in: `[1, 2, 3]`, ptr: new(MustNotUnmarshalJSON), err: errors.New("MustNotUnmarshalJSON was used")}, // empty array to interface test - {in: `[]`, ptr: new([]interface{}), out: []interface{}{}}, - {in: `null`, ptr: new([]interface{}), out: []interface{}(nil)}, - {in: `{"T":[]}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": []interface{}{}}}, - {in: `{"T":null}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": interface{}(nil)}}, + {in: `[]`, ptr: new([]any), out: []any{}}, + {in: `null`, ptr: new([]any), out: []any(nil)}, + {in: `{"T":[]}`, ptr: new(map[string]any), out: map[string]any{"T": []any{}}}, + {in: `{"T":null}`, ptr: new(map[string]any), out: map[string]any{"T": any(nil)}}, // composite tests {in: allValueIndent, ptr: new(All), out: allValue}, @@ -1131,7 +1132,7 @@ func TestUnmarshal(t *testing.T) { func TestUnmarshalMarshal(t *testing.T) { initBig() - var v interface{} + var v any if err := Unmarshal(jsonBig, &v); err != nil { t.Fatalf("Unmarshal: %v", err) } @@ -1203,7 +1204,7 @@ type Xint struct { func TestUnmarshalInterface(t *testing.T) { var xint Xint - var i interface{} = &xint + var i any = &xint if err := Unmarshal([]byte(`{"X":1}`), &i); err != nil { t.Fatalf("Unmarshal: %v", err) } @@ -1265,7 +1266,7 @@ func TestErrorMessageFromMisusedString(t *testing.T) { } func noSpace(c rune) rune { - if isSpace(byte(c)) { //only used for ascii + if isSpace(byte(c)) { // only used for ascii return -1 } return c @@ -1334,8 +1335,8 @@ type All struct { PSmall *Small PPSmall **Small - Interface interface{} - PInterface *interface{} + Interface any + PInterface *any unexported int } @@ -1669,9 +1670,9 @@ func intpp(x *int) **int { } var interfaceSetTests = []struct { - pre interface{} + pre any json string - post interface{} + post any }{ {"foo", `"bar"`, "bar"}, {"foo", `2`, 2.0}, @@ -1690,7 +1691,7 @@ var interfaceSetTests = []struct { func TestInterfaceSet(t *testing.T) { for _, tt := range interfaceSetTests { - b := struct{ X interface{} }{tt.pre} + b := struct{ X any }{tt.pre} blob := `{"X":` + tt.json + `}` if err := Unmarshal([]byte(blob), &b); err != nil { t.Errorf("Unmarshal %#q: %v", blob, err) @@ -1720,7 +1721,7 @@ type NullTest struct { PBool *bool Map map[string]string Slice []string - Interface interface{} + Interface any PRaw *RawMessage PTime *time.Time @@ -1755,7 +1756,7 @@ type NullTestStrings struct { PBool *bool `json:",string"` Map map[string]string `json:",string"` Slice []string `json:",string"` - Interface interface{} `json:",string"` + Interface any `json:",string"` PRaw *RawMessage `json:",string"` PTime *time.Time `json:",string"` @@ -1976,7 +1977,7 @@ func TestSliceOfCustomByte(t *testing.T) { } var decodeTypeErrorTests = []struct { - dest interface{} + dest any src string }{ {new(string), `{"user": "name"}`}, // issue 4628. @@ -2009,7 +2010,7 @@ var unmarshalSyntaxTests = []string{ } func TestUnmarshalSyntax(t *testing.T) { - var x interface{} + var x any for _, src := range unmarshalSyntaxTests { err := Unmarshal([]byte(src), &x) if _, ok := err.(*SyntaxError); !ok { @@ -2022,8 +2023,8 @@ func TestUnmarshalSyntax(t *testing.T) { // Issue 4660 type unexportedFields struct { Name string - m map[string]interface{} `json:"-"` - m2 map[string]interface{} `json:"abcd"` + m map[string]any `json:"-"` + m2 map[string]any `json:"abcd"` s []int `json:"-"` } @@ -2074,7 +2075,7 @@ func TestUnmarshalJSONLiteralError(t *testing.T) { // Issue 3717 func TestSkipArrayObjects(t *testing.T) { json := `[{}]` - var dest [0]interface{} + var dest [0]any err := Unmarshal([]byte(json), &dest) if err != nil { @@ -2085,13 +2086,13 @@ func TestSkipArrayObjects(t *testing.T) { // Test semantics of pre-filled struct fields and pre-filled map fields. // Issue 4900. func TestPrefilled(t *testing.T) { - ptrToMap := func(m map[string]interface{}) *map[string]interface{} { return &m } + ptrToMap := func(m map[string]any) *map[string]any { return &m } // Values here change, cannot reuse table across runs. - var prefillTests = []struct { + prefillTests := []struct { in string - ptr interface{} - out interface{} + ptr any + out any }{ { in: `{"X": 1, "Y": 2}`, @@ -2100,8 +2101,8 @@ func TestPrefilled(t *testing.T) { }, { in: `{"X": 1, "Y": 2}`, - ptr: ptrToMap(map[string]interface{}{"X": float32(3), "Y": int16(4), "Z": 1.5}), - out: ptrToMap(map[string]interface{}{"X": float64(1), "Y": float64(2), "Z": 1.5}), + ptr: ptrToMap(map[string]any{"X": float32(3), "Y": int16(4), "Z": 1.5}), + out: ptrToMap(map[string]any{"X": float64(1), "Y": float64(2), "Z": 1.5}), }, } @@ -2118,7 +2119,7 @@ func TestPrefilled(t *testing.T) { } var invalidUnmarshalTests = []struct { - v interface{} + v any want string }{ {nil, "json: Unmarshal(nil)"}, @@ -2141,7 +2142,7 @@ func TestInvalidUnmarshal(t *testing.T) { } var invalidUnmarshalTextTests = []struct { - v interface{} + v any want string }{ {nil, "json: Unmarshal(nil)"}, @@ -2173,7 +2174,7 @@ func TestInvalidStringOption(t *testing.T) { M map[string]string `json:",string"` S []string `json:",string"` A [1]string `json:",string"` - I interface{} `json:",string"` + I any `json:",string"` P *int `json:",string"` }{M: make(map[string]string), S: make([]string, 0), I: num, P: &num} @@ -2244,8 +2245,8 @@ func TestUnmarshalEmbeddedUnexported(t *testing.T) { tests := []struct { in string - ptr interface{} - out interface{} + ptr any + out any err error }{{ // Error since we cannot set S1.embed1, but still able to set S1.R. @@ -2336,7 +2337,7 @@ func TestUnmarshalPanic(t *testing.T) { // The decoder used to hang if decoding into an interface pointing to its own address. // See golang.org/issues/31740. func TestUnmarshalRecursivePointer(t *testing.T) { - var v interface{} + var v any v = &v data := []byte(`{"a": "b"}`) diff --git a/json/golang_encode_test.go b/json/golang_encode_test.go index 5c465dc..5e334a6 100644 --- a/json/golang_encode_test.go +++ b/json/golang_encode_test.go @@ -27,8 +27,8 @@ type Optionals struct { Slr []string `json:"slr,random"` Slo []string `json:"slo,omitempty"` - Mr map[string]interface{} `json:"mr"` - Mo map[string]interface{} `json:",omitempty"` + Mr map[string]any `json:"mr"` + Mo map[string]any `json:",omitempty"` Fr float64 `json:"fr"` Fo float64 `json:"fo,omitempty"` @@ -58,8 +58,8 @@ var optionalsExpected = `{ func TestOmitEmpty(t *testing.T) { var o Optionals o.Sw = "something" - o.Mr = map[string]interface{}{} - o.Mo = map[string]interface{}{} + o.Mr = map[string]any{} + o.Mo = map[string]any{} got, err := MarshalIndent(&o, "", " ") if err != nil { @@ -110,9 +110,11 @@ func TestStringTag(t *testing.T) { } // byte slices are special even if they're renamed types. -type renamedByte byte -type renamedByteSlice []byte -type renamedRenamedByteSlice []renamedByte +type ( + renamedByte byte + renamedByteSlice []byte + renamedRenamedByteSlice []renamedByte +) func TestEncodeRenamedByteSlice(t *testing.T) { s := renamedByteSlice("abc") @@ -134,7 +136,7 @@ func TestEncodeRenamedByteSlice(t *testing.T) { } } -var unsupportedValues = []interface{}{ +var unsupportedValues = []any{ math.NaN(), math.Inf(-1), math.Inf(1), @@ -191,7 +193,7 @@ func (ValText) MarshalText() ([]byte, error) { } func TestRefValMarshal(t *testing.T) { - var s = struct { + s := struct { R0 Ref R1 *Ref R2 RefText @@ -258,160 +260,169 @@ func TestMarshalerEscaping(t *testing.T) { func TestAnonymousFields(t *testing.T) { tests := []struct { - label string // Test name - makeInput func() interface{} // Function to create input value - want string // Expected JSON output - }{{ - // Both S1 and S2 have a field named X. From the perspective of S, - // it is ambiguous which one X refers to. - // This should not serialize either field. - label: "AmbiguousField", - makeInput: func() interface{} { - type ( - S1 struct{ x, X int } - S2 struct{ x, X int } - S struct { - S1 - S2 - } - ) - return S{S1{1, 2}, S2{3, 4}} + label string // Test name + makeInput func() any // Function to create input value + want string // Expected JSON output + }{ + { + // Both S1 and S2 have a field named X. From the perspective of S, + // it is ambiguous which one X refers to. + // This should not serialize either field. + label: "AmbiguousField", + makeInput: func() any { + type ( + S1 struct{ x, X int } + S2 struct{ x, X int } + S struct { + S1 + S2 + } + ) + return S{S1{1, 2}, S2{3, 4}} + }, + want: `{}`, }, - want: `{}`, - }, { - label: "DominantField", - // Both S1 and S2 have a field named X, but since S has an X field as - // well, it takes precedence over S1.X and S2.X. - makeInput: func() interface{} { - type ( - S1 struct{ x, X int } - S2 struct{ x, X int } - S struct { - S1 - S2 - x, X int - } - ) - return S{S1{1, 2}, S2{3, 4}, 5, 6} + { + label: "DominantField", + // Both S1 and S2 have a field named X, but since S has an X field as + // well, it takes precedence over S1.X and S2.X. + makeInput: func() any { + type ( + S1 struct{ x, X int } + S2 struct{ x, X int } + S struct { + S1 + S2 + x, X int + } + ) + return S{S1{1, 2}, S2{3, 4}, 5, 6} + }, + want: `{"X":6}`, }, - want: `{"X":6}`, - }, { - // Unexported embedded field of non-struct type should not be serialized. - label: "UnexportedEmbeddedInt", - makeInput: func() interface{} { - type ( - myInt int - S struct{ myInt } - ) - return S{5} + { + // Unexported embedded field of non-struct type should not be serialized. + label: "UnexportedEmbeddedInt", + makeInput: func() any { + type ( + myInt int + S struct{ myInt } + ) + return S{5} + }, + want: `{}`, }, - want: `{}`, - }, { - // Exported embedded field of non-struct type should be serialized. - label: "ExportedEmbeddedInt", - makeInput: func() interface{} { - type ( - MyInt int - S struct{ MyInt } - ) - return S{5} + { + // Exported embedded field of non-struct type should be serialized. + label: "ExportedEmbeddedInt", + makeInput: func() any { + type ( + MyInt int + S struct{ MyInt } + ) + return S{5} + }, + want: `{"MyInt":5}`, }, - want: `{"MyInt":5}`, - }, { - // Unexported embedded field of pointer to non-struct type - // should not be serialized. - label: "UnexportedEmbeddedIntPointer", - makeInput: func() interface{} { - type ( - myInt int - S struct{ *myInt } - ) - s := S{new(myInt)} - *s.myInt = 5 - return s + { + // Unexported embedded field of pointer to non-struct type + // should not be serialized. + label: "UnexportedEmbeddedIntPointer", + makeInput: func() any { + type ( + myInt int + S struct{ *myInt } + ) + s := S{new(myInt)} + *s.myInt = 5 + return s + }, + want: `{}`, }, - want: `{}`, - }, { - // Exported embedded field of pointer to non-struct type - // should be serialized. - label: "ExportedEmbeddedIntPointer", - makeInput: func() interface{} { - type ( - MyInt int - S struct{ *MyInt } - ) - s := S{new(MyInt)} - *s.MyInt = 5 - return s + { + // Exported embedded field of pointer to non-struct type + // should be serialized. + label: "ExportedEmbeddedIntPointer", + makeInput: func() any { + type ( + MyInt int + S struct{ *MyInt } + ) + s := S{new(MyInt)} + *s.MyInt = 5 + return s + }, + want: `{"MyInt":5}`, }, - want: `{"MyInt":5}`, - }, { - // Exported fields of embedded structs should have their - // exported fields be serialized regardless of whether the struct types - // themselves are exported. - label: "EmbeddedStruct", - makeInput: func() interface{} { - type ( - s1 struct{ x, X int } - S2 struct{ y, Y int } - S struct { - s1 - S2 - } - ) - return S{s1{1, 2}, S2{3, 4}} + { + // Exported fields of embedded structs should have their + // exported fields be serialized regardless of whether the struct types + // themselves are exported. + label: "EmbeddedStruct", + makeInput: func() any { + type ( + s1 struct{ x, X int } + S2 struct{ y, Y int } + S struct { + s1 + S2 + } + ) + return S{s1{1, 2}, S2{3, 4}} + }, + want: `{"X":2,"Y":4}`, }, - want: `{"X":2,"Y":4}`, - }, { - // Exported fields of pointers to embedded structs should have their - // exported fields be serialized regardless of whether the struct types - // themselves are exported. - label: "EmbeddedStructPointer", - makeInput: func() interface{} { - type ( - s1 struct{ x, X int } - S2 struct{ y, Y int } - S struct { - *s1 - *S2 - } - ) - return S{&s1{1, 2}, &S2{3, 4}} + { + // Exported fields of pointers to embedded structs should have their + // exported fields be serialized regardless of whether the struct types + // themselves are exported. + label: "EmbeddedStructPointer", + makeInput: func() any { + type ( + s1 struct{ x, X int } + S2 struct{ y, Y int } + S struct { + *s1 + *S2 + } + ) + return S{&s1{1, 2}, &S2{3, 4}} + }, + want: `{"X":2,"Y":4}`, }, - want: `{"X":2,"Y":4}`, - }, { - // Exported fields on embedded unexported structs at multiple levels - // of nesting should still be serialized. - label: "NestedStructAndInts", - makeInput: func() interface{} { - type ( - MyInt1 int - MyInt2 int - myInt int - s2 struct { - MyInt2 - myInt - } - s1 struct { - MyInt1 - myInt - s2 - } - S struct { - s1 - myInt - } - ) - return S{s1{1, 2, s2{3, 4}}, 6} + { + // Exported fields on embedded unexported structs at multiple levels + // of nesting should still be serialized. + label: "NestedStructAndInts", + makeInput: func() any { + type ( + MyInt1 int + MyInt2 int + myInt int + s2 struct { + MyInt2 + myInt + } + s1 struct { + MyInt1 + myInt + s2 + } + S struct { + s1 + myInt + } + ) + return S{s1{1, 2, s2{3, 4}}, 6} + }, + want: `{"MyInt1":1,"MyInt2":3}`, }, - want: `{"MyInt1":1,"MyInt2":3}`, - }, { // If an anonymous struct pointer field is nil, we should ignore // the embedded fields behind it. Not properly doing so may // result in the wrong output or reflect panics. label: "EmbeddedFieldBehindNilPointer", - makeInput: func() interface{} { + makeInput: func() any { type ( S2 struct{ Field string } S struct{ *S2 } @@ -469,19 +480,19 @@ func (nm *nilMarshaler) MarshalJSON() ([]byte, error) { // Issue 16042. func TestNilMarshal(t *testing.T) { testCases := []struct { - v interface{} + v any want string }{ {v: nil, want: `null`}, {v: new(float64), want: `0`}, - {v: []interface{}(nil), want: `null`}, + {v: []any(nil), want: `null`}, {v: []string(nil), want: `null`}, {v: map[string]string(nil), want: `null`}, {v: []byte(nil), want: `null`}, {v: struct{ M string }{"gopher"}, want: `{"M":"gopher"}`}, {v: struct{ M Marshaler }{}, want: `{"M":null}`}, {v: struct{ M Marshaler }{(*nilMarshaler)(nil)}, want: `{"M":"0zenil0"}`}, - {v: struct{ M interface{} }{(*nilMarshaler)(nil)}, want: `{"M":null}`}, + {v: struct{ M any }{(*nilMarshaler)(nil)}, want: `{"M":null}`}, } for _, tt := range testCases { @@ -741,7 +752,7 @@ type textint int func (i textint) MarshalText() ([]byte, error) { return tenc(`TI:%d`, i) } -func tenc(format string, a ...interface{}) ([]byte, error) { +func tenc(format string, a ...any) ([]byte, error) { var buf bytes.Buffer fmt.Fprintf(&buf, format, a...) return buf.Bytes(), nil @@ -750,7 +761,7 @@ func tenc(format string, a ...interface{}) ([]byte, error) { // Issue 13783 func TestEncodeBytekind(t *testing.T) { testdata := []struct { - data interface{} + data any want string }{ {byte(7), "7"}, @@ -823,7 +834,7 @@ func TestMarshalFloat(t *testing.T) { t.Parallel() nfail := 0 test := func(f float64, bits int) { - vf := interface{}(f) + vf := any(f) if bits == 32 { f = float64(float32(f)) // round vf = float32(f) @@ -867,7 +878,7 @@ func TestMarshalFloat(t *testing.T) { smaller = math.Inf(-1) ) - var digits = "1.2345678901234567890123" + digits := "1.2345678901234567890123" for i := len(digits); i >= 2; i-- { if testing.Short() && i < len(digits)-4 { break @@ -919,25 +930,25 @@ func TestMarshalRawMessageValue(t *testing.T) { ) tests := []struct { - in interface{} + in any want string ok bool }{ // Test with nil RawMessage. {rawNil, "null", true}, {&rawNil, "null", true}, - {[]interface{}{rawNil}, "[null]", true}, - {&[]interface{}{rawNil}, "[null]", true}, - {[]interface{}{&rawNil}, "[null]", true}, - {&[]interface{}{&rawNil}, "[null]", true}, + {[]any{rawNil}, "[null]", true}, + {&[]any{rawNil}, "[null]", true}, + {[]any{&rawNil}, "[null]", true}, + {&[]any{&rawNil}, "[null]", true}, {struct{ M RawMessage }{rawNil}, `{"M":null}`, true}, {&struct{ M RawMessage }{rawNil}, `{"M":null}`, true}, {struct{ M *RawMessage }{&rawNil}, `{"M":null}`, true}, {&struct{ M *RawMessage }{&rawNil}, `{"M":null}`, true}, - {map[string]interface{}{"M": rawNil}, `{"M":null}`, true}, - {&map[string]interface{}{"M": rawNil}, `{"M":null}`, true}, - {map[string]interface{}{"M": &rawNil}, `{"M":null}`, true}, - {&map[string]interface{}{"M": &rawNil}, `{"M":null}`, true}, + {map[string]any{"M": rawNil}, `{"M":null}`, true}, + {&map[string]any{"M": rawNil}, `{"M":null}`, true}, + {map[string]any{"M": &rawNil}, `{"M":null}`, true}, + {&map[string]any{"M": &rawNil}, `{"M":null}`, true}, {T1{rawNil}, "{}", true}, {T2{&rawNil}, `{"M":null}`, true}, {&T1{rawNil}, "{}", true}, @@ -946,18 +957,18 @@ func TestMarshalRawMessageValue(t *testing.T) { // Test with empty, but non-nil, RawMessage. {rawEmpty, "", false}, {&rawEmpty, "", false}, - {[]interface{}{rawEmpty}, "", false}, - {&[]interface{}{rawEmpty}, "", false}, - {[]interface{}{&rawEmpty}, "", false}, - {&[]interface{}{&rawEmpty}, "", false}, + {[]any{rawEmpty}, "", false}, + {&[]any{rawEmpty}, "", false}, + {[]any{&rawEmpty}, "", false}, + {&[]any{&rawEmpty}, "", false}, {struct{ X RawMessage }{rawEmpty}, "", false}, {&struct{ X RawMessage }{rawEmpty}, "", false}, {struct{ X *RawMessage }{&rawEmpty}, "", false}, {&struct{ X *RawMessage }{&rawEmpty}, "", false}, - {map[string]interface{}{"nil": rawEmpty}, "", false}, - {&map[string]interface{}{"nil": rawEmpty}, "", false}, - {map[string]interface{}{"nil": &rawEmpty}, "", false}, - {&map[string]interface{}{"nil": &rawEmpty}, "", false}, + {map[string]any{"nil": rawEmpty}, "", false}, + {&map[string]any{"nil": rawEmpty}, "", false}, + {map[string]any{"nil": &rawEmpty}, "", false}, + {&map[string]any{"nil": &rawEmpty}, "", false}, {T1{rawEmpty}, "{}", true}, {T2{&rawEmpty}, "", false}, {&T1{rawEmpty}, "{}", true}, @@ -970,18 +981,18 @@ func TestMarshalRawMessageValue(t *testing.T) { // See https://golang.org/issues/14493#issuecomment-255857318 {rawText, `"foo"`, true}, // Issue6458 {&rawText, `"foo"`, true}, - {[]interface{}{rawText}, `["foo"]`, true}, // Issue6458 - {&[]interface{}{rawText}, `["foo"]`, true}, // Issue6458 - {[]interface{}{&rawText}, `["foo"]`, true}, - {&[]interface{}{&rawText}, `["foo"]`, true}, + {[]any{rawText}, `["foo"]`, true}, // Issue6458 + {&[]any{rawText}, `["foo"]`, true}, // Issue6458 + {[]any{&rawText}, `["foo"]`, true}, + {&[]any{&rawText}, `["foo"]`, true}, {struct{ M RawMessage }{rawText}, `{"M":"foo"}`, true}, // Issue6458 {&struct{ M RawMessage }{rawText}, `{"M":"foo"}`, true}, {struct{ M *RawMessage }{&rawText}, `{"M":"foo"}`, true}, {&struct{ M *RawMessage }{&rawText}, `{"M":"foo"}`, true}, - {map[string]interface{}{"M": rawText}, `{"M":"foo"}`, true}, // Issue6458 - {&map[string]interface{}{"M": rawText}, `{"M":"foo"}`, true}, // Issue6458 - {map[string]interface{}{"M": &rawText}, `{"M":"foo"}`, true}, - {&map[string]interface{}{"M": &rawText}, `{"M":"foo"}`, true}, + {map[string]any{"M": rawText}, `{"M":"foo"}`, true}, // Issue6458 + {&map[string]any{"M": rawText}, `{"M":"foo"}`, true}, // Issue6458 + {map[string]any{"M": &rawText}, `{"M":"foo"}`, true}, + {&map[string]any{"M": &rawText}, `{"M":"foo"}`, true}, {T1{rawText}, `{"M":"foo"}`, true}, // Issue6458 {T2{&rawText}, `{"M":"foo"}`, true}, {&T1{rawText}, `{"M":"foo"}`, true}, diff --git a/json/golang_example_test.go b/json/golang_example_test.go index 2088c34..627965e 100644 --- a/json/golang_example_test.go +++ b/json/golang_example_test.go @@ -35,7 +35,7 @@ func ExampleMarshal() { } func ExampleUnmarshal() { - var jsonBlob = []byte(`[ + jsonBlob := []byte(`[ {"Name": "Platypus", "Order": "Monotremata"}, {"Name": "Quoll", "Order": "Dasyuromorphia"} ]`) @@ -189,7 +189,7 @@ func ExampleRawMessage_unmarshal() { Cr int8 } - var j = []byte(`[ + j := []byte(`[ {"Space": "YCbCr", "Point": {"Y": 255, "Cb": 0, "Cr": -10}}, {"Space": "RGB", "Point": {"R": 98, "G": 218, "B": 255}} ]`) @@ -200,7 +200,7 @@ func ExampleRawMessage_unmarshal() { } for _, c := range colors { - var dst interface{} + var dst any switch c.Space { case "RGB": dst = new(RGB) diff --git a/json/golang_number_test.go b/json/golang_number_test.go index f89ac8b..14b0f74 100644 --- a/json/golang_number_test.go +++ b/json/golang_number_test.go @@ -11,7 +11,7 @@ import ( func TestNumberIsValid(t *testing.T) { // From: https://stackoverflow.com/a/13340826 - var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`) + jsonNumberRegexp := regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`) validTests := []string{ "0", @@ -115,15 +115,15 @@ func TestNumberIsValid(t *testing.T) { func BenchmarkNumberIsValid(b *testing.B) { s := "-61657.61667E+61673" - for i := 0; i < b.N; i++ { + for range b.N { isValidNumber(s) } } func BenchmarkNumberIsValidRegexp(b *testing.B) { - var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`) + jsonNumberRegexp := regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`) s := "-61657.61667E+61673" - for i := 0; i < b.N; i++ { + for range b.N { jsonNumberRegexp.MatchString(s) } } diff --git a/json/golang_scanner_test.go b/json/golang_scanner_test.go index a26bace..3cda7a2 100644 --- a/json/golang_scanner_test.go +++ b/json/golang_scanner_test.go @@ -199,10 +199,7 @@ func TestIndentErrors(t *testing.T) { func diff(t *testing.T, a, b []byte) { for i := 0; ; i++ { if i >= len(a) || i >= len(b) || a[i] != b[i] { - j := i - 10 - if j < 0 { - j = 0 - } + j := max(0, i-10) t.Errorf("diverge at %d: «%s» vs «%s»", i, trim(a[j:]), trim(b[j:])) return } @@ -232,7 +229,7 @@ func initBig() { jsonBig = b } -func genValue(n int) interface{} { +func genValue(n int) any { if n > 1 { switch rand.Intn(2) { case 0: @@ -265,7 +262,7 @@ func genString(stddev float64) string { return string(c) } -func genArray(n int) []interface{} { +func genArray(n int) []any { f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2))) if f > n { f = n @@ -273,23 +270,21 @@ func genArray(n int) []interface{} { if f < 1 { f = 1 } - x := make([]interface{}, f) + x := make([]any, f) for i := range x { x[i] = genValue(((i+1)*n)/f - (i*n)/f) } return x } -func genMap(n int) map[string]interface{} { +func genMap(n int) map[string]any { f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2))) - if f > n { - f = n - } + f = min(f, n) if n > 0 && f == 0 { f = 1 } - x := make(map[string]interface{}) - for i := 0; i < f; i++ { + x := make(map[string]any) + for i := range f { x[genString(10)] = genValue(((i+1)*n)/f - (i*n)/f) } return x diff --git a/json/golang_shim_test.go b/json/golang_shim_test.go index b615e36..5a19b7f 100644 --- a/json/golang_shim_test.go +++ b/json/golang_shim_test.go @@ -15,8 +15,7 @@ var fieldCache = sync.Map{} func cachedTypeFields(reflect.Type) {} // Fake test env for golang_bench_test.go -type testenvShim struct { -} +type testenvShim struct{} func (ts testenvShim) Builder() string { return "" @@ -25,8 +24,7 @@ func (ts testenvShim) Builder() string { var testenv testenvShim // Fake scanner for golang_decode_test.go -type scanner struct { -} +type scanner struct{} func checkValid(in []byte, scan *scanner) error { return nil @@ -53,7 +51,7 @@ func isValidNumber(n string) bool { return true } -func assertErrorPresence(t *testing.T, expected error, actual error, prefixes ...interface{}) { +func assertErrorPresence(t *testing.T, expected error, actual error, prefixes ...any) { if expected != nil && actual == nil { errorWithPrefixes(t, prefixes, "expected error, but did not get an error") } else if expected == nil && actual != nil { @@ -61,7 +59,7 @@ func assertErrorPresence(t *testing.T, expected error, actual error, prefixes .. } } -func errorWithPrefixes(t *testing.T, prefixes []interface{}, format string, elements ...interface{}) { +func errorWithPrefixes(t *testing.T, prefixes []any, format string, elements ...any) { fullFormat := format allElements := append(prefixes, elements...) diff --git a/json/golang_tagkey_test.go b/json/golang_tagkey_test.go index 17d4c03..fd47efa 100644 --- a/json/golang_tagkey_test.go +++ b/json/golang_tagkey_test.go @@ -73,7 +73,7 @@ type unicodeTag struct { } var structTagObjectKeyTests = []struct { - raw interface{} + raw any value string key string }{ @@ -101,12 +101,12 @@ func TestStructTagObjectKey(t *testing.T) { if err != nil { t.Fatalf("Marshal(%#q) failed: %v", tt.raw, err) } - var f interface{} + var f any err = Unmarshal(b, &f) if err != nil { t.Fatalf("Unmarshal(%#q) failed: %v", b, err) } - for i, v := range f.(map[string]interface{}) { + for i, v := range f.(map[string]any) { switch i { case tt.key: if s, ok := v.(string); !ok || s != tt.value { diff --git a/json/int_test.go b/json/int_test.go index cca5af1..0c6cf34 100644 --- a/json/int_test.go +++ b/json/int_test.go @@ -8,7 +8,7 @@ import ( func TestAppendInt(t *testing.T) { var ints []int64 - for i := 0; i < 64; i++ { + for i := range 64 { u := uint64(1) << i ints = append(ints, int64(u-1), int64(u), int64(u+1), -int64(u)) } @@ -27,7 +27,7 @@ func TestAppendInt(t *testing.T) { func benchStd(b *testing.B, n int64) { var buf [20]byte b.ResetTimer() - for i := 0; i < b.N; i++ { + for range b.N { strconv.AppendInt(buf[:0], n, 10) } } @@ -35,7 +35,7 @@ func benchStd(b *testing.B, n int64) { func benchNew(b *testing.B, n int64) { var buf [20]byte b.ResetTimer() - for i := 0; i < b.N; i++ { + for range b.N { appendInt(buf[:0], n) } } diff --git a/json/json.go b/json/json.go index d5f6f9d..11ec69c 100644 --- a/json/json.go +++ b/json/json.go @@ -193,7 +193,7 @@ func (k Kind) Class() Kind { return Kind(1 << uint(bits.Len(uint(k))-1)) } // Append acts like Marshal but appends the json representation to b instead of // always reallocating a new slice. -func Append(b []byte, x interface{}, flags AppendFlags) ([]byte, error) { +func Append(b []byte, x any, flags AppendFlags) ([]byte, error) { if x == nil { // Special case for nil values because it makes the rest of the code // simpler to assume that it won't be seeing nil pointers. @@ -269,9 +269,9 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { } // Marshal is documented at https://golang.org/pkg/encoding/json/#Marshal -func Marshal(x interface{}) ([]byte, error) { +func Marshal(x any) ([]byte, error) { var err error - var buf = encoderBufferPool.Get().(*encoderBuffer) + buf := encoderBufferPool.Get().(*encoderBuffer) if buf.data, err = Append(buf.data[:0], x, EscapeHTML|SortMapKeys); err != nil { return nil, err @@ -284,7 +284,7 @@ func Marshal(x interface{}) ([]byte, error) { } // MarshalIndent is documented at https://golang.org/pkg/encoding/json/#MarshalIndent -func MarshalIndent(x interface{}, prefix, indent string) ([]byte, error) { +func MarshalIndent(x any, prefix, indent string) ([]byte, error) { b, err := Marshal(x) if err == nil { @@ -299,7 +299,7 @@ func MarshalIndent(x interface{}, prefix, indent string) ([]byte, error) { } // Unmarshal is documented at https://golang.org/pkg/encoding/json/#Unmarshal -func Unmarshal(b []byte, x interface{}) error { +func Unmarshal(b []byte, x any) error { r, err := Parse(b, x, 0) if len(r) != 0 { if _, ok := err.(*SyntaxError); !ok { @@ -314,7 +314,7 @@ func Unmarshal(b []byte, x interface{}) error { // Parse behaves like Unmarshal but the caller can pass a set of flags to // configure the parsing behavior. -func Parse(b []byte, x interface{}, flags ParseFlags) ([]byte, error) { +func Parse(b []byte, x any, flags ParseFlags) ([]byte, error) { t := reflect.TypeOf(x) p := (*iface)(unsafe.Pointer(&x)).ptr @@ -373,7 +373,7 @@ func (dec *Decoder) Buffered() io.Reader { } // Decode is documented at https://golang.org/pkg/encoding/json/#Decoder.Decode -func (dec *Decoder) Decode(v interface{}) error { +func (dec *Decoder) Decode(v any) error { raw, err := dec.readValue() if err != nil { return err @@ -503,16 +503,15 @@ func NewEncoder(w io.Writer) *Encoder { } // Encode is documented at https://golang.org/pkg/encoding/json/#Encoder.Encode -func (enc *Encoder) Encode(v interface{}) error { +func (enc *Encoder) Encode(v any) error { if enc.err != nil { return enc.err } var err error - var buf = encoderBufferPool.Get().(*encoderBuffer) + buf := encoderBufferPool.Get().(*encoderBuffer) buf.data, err = Append(buf.data[:0], v, enc.flags) - if err != nil { encoderBufferPool.Put(buf) return err @@ -589,7 +588,7 @@ func (enc *Encoder) SetAppendNewline(on bool) { } var encoderBufferPool = sync.Pool{ - New: func() interface{} { return &encoderBuffer{data: make([]byte, 0, 4096)} }, + New: func() any { return &encoderBuffer{data: make([]byte, 0, 4096)} }, } type encoderBuffer struct{ data []byte } diff --git a/json/json_test.go b/json/json_test.go index fb77868..b40e000 100644 --- a/json/json_test.go +++ b/json/json_test.go @@ -31,8 +31,8 @@ type testSyntaxError struct { func (e *testSyntaxError) Error() string { return e.msg } var ( - marshal func([]byte, interface{}) ([]byte, error) - unmarshal func([]byte, interface{}) error + marshal func([]byte, any) ([]byte, error) + unmarshal func([]byte, any) error escapeHTML bool ) @@ -48,7 +48,7 @@ func TestMain(m *testing.M) { enc := json.NewEncoder(buf) enc.SetEscapeHTML(escapeHTML) - marshal = func(b []byte, v interface{}) ([]byte, error) { + marshal = func(b []byte, v any) ([]byte, error) { buf.data = b err := enc.Encode(v) return buf.data, err @@ -62,11 +62,11 @@ func TestMain(m *testing.M) { flags |= EscapeHTML } - marshal = func(b []byte, v interface{}) ([]byte, error) { + marshal = func(b []byte, v any) ([]byte, error) { return Append(b, v, flags) } - unmarshal = func(b []byte, v interface{}) error { + unmarshal = func(b []byte, v any) error { _, err := Parse(b, v, ZeroCopy) return err } @@ -93,7 +93,7 @@ var ( bigNeg128 = new(big.Int).Neg(bigPos128) ) -var testValues = [...]interface{}{ +var testValues = [...]any{ // constants nil, false, @@ -189,7 +189,7 @@ var testValues = [...]interface{}{ makeSlice(250), makeSlice(1020), []string{"A", "B", "C"}, - []interface{}{nil, true, false, 0.5, "Hello World!"}, + []any{nil, true, false, 0.5, "Hello World!"}, // map makeMapStringBool(0), @@ -227,7 +227,7 @@ var testValues = [...]interface{}{ S string }{42, time.Date(2016, 12, 20, 0, 20, 1, 0, time.UTC), "Hello World!"}, // These types are interesting because they fit in a pointer so the compiler - // puts their value directly into the pointer field of the interface{} that + // puts their value directly into the pointer field of the any that // is passed to Marshal. struct{ X *int }{}, struct{ X *int }{new(int)}, @@ -237,12 +237,12 @@ var testValues = [...]interface{}{ struct{ X, Y *int }{}, struct{ X, Y *int }{new(int), new(int)}, struct { - A string `json:"name"` - B string `json:"-"` - C string `json:",omitempty"` - D map[string]interface{} `json:",string"` + A string `json:"name"` + B string `json:"-"` + C string `json:",omitempty"` + D map[string]any `json:",string"` e string - }{A: "Luke", D: map[string]interface{}{"answer": float64(42)}}, + }{A: "Luke", D: map[string]any{"answer": float64(42)}}, struct{ point }{point{1, 2}}, tree{ Value: "T", @@ -272,7 +272,7 @@ var testValues = [...]interface{}{ loadTestdata(filepath.Join(runtime.GOROOT(), "src/encoding/json/testdata/code.json.gz")), } -var durationTestValues = []interface{}{ +var durationTestValues = []any{ // duration time.Nanosecond, time.Microsecond, @@ -295,7 +295,7 @@ func makeSlice(n int) []int { func makeMapStringBool(n int) map[string]bool { m := make(map[string]bool, n) - for i := 0; i != n; i++ { + for i := range n { m[strconv.Itoa(i)] = true } return m @@ -303,7 +303,7 @@ func makeMapStringBool(n int) map[string]bool { func makeMapStringString(n int) map[string]string { m := make(map[string]string, n) - for i := 0; i != n; i++ { + for i := range n { m[strconv.Itoa(i)] = fmt.Sprintf("%d Hello, world!", i) } return m @@ -311,21 +311,21 @@ func makeMapStringString(n int) map[string]string { func makeMapStringStringSlice(n int) map[string][]string { m := make(map[string][]string, n) - for i := 0; i != n; i++ { + for i := range n { m[strconv.Itoa(i)] = []string{strconv.Itoa(i), "Hello,", "world!"} } return m } -func makeMapStringInterface(n int) map[string]interface{} { - m := make(map[string]interface{}, n) - for i := 0; i != n; i++ { +func makeMapStringInterface(n int) map[string]any { + m := make(map[string]any, n) + for i := range n { m[strconv.Itoa(i)] = nil } return m } -func testName(v interface{}) string { +func testName(v any) string { return fmt.Sprintf("%T", v) } @@ -344,7 +344,7 @@ type codeNode2 struct { MeanT int64 `json:"mean_t"` } -func loadTestdata(path string) interface{} { +func loadTestdata(path string) any { f, err := os.Open(path) if err != nil { return err.Error() @@ -623,7 +623,7 @@ func TestParse_numeric(t *testing.T) { } } -func newValue(model interface{}) reflect.Value { +func newValue(model any) reflect.Value { if model == nil { return reflect.New(reflect.TypeOf(&model).Elem()) } @@ -639,7 +639,7 @@ func BenchmarkMarshal(b *testing.B) { return } - for i := 0; i != b.N; i++ { + for range b.N { j, _ = marshal(j[:0], v) } @@ -663,7 +663,7 @@ func BenchmarkUnmarshal(b *testing.B) { j, _ := json.Marshal(x) x = newValue(v).Interface() - for i := 0; i != b.N; i++ { + for range b.N { unmarshal(j, x) } @@ -764,7 +764,6 @@ func TestDecodeLines(t *testing.T) { reader io.Reader expectCount int }{ - // simple { @@ -948,7 +947,7 @@ func TestDontMatchCaseIncensitiveStructFields(t *testing.T) { func TestMarshalFuzzBugs(t *testing.T) { tests := []struct { - value interface{} + value any output string }{ { // html sequences are escaped even in RawMessage @@ -984,27 +983,27 @@ func TestMarshalFuzzBugs(t *testing.T) { func TestUnmarshalFuzzBugs(t *testing.T) { tests := []struct { input string - value interface{} + value any }{ { // non-UTF8 sequences must be converted to the utf8.RuneError character. input: "[\"00000\xef\"]", - value: []interface{}{"00000�"}, + value: []any{"00000�"}, }, { // UTF16 surrogate followed by null character input: "[\"\\ud800\\u0000\"]", - value: []interface{}{"�\x00"}, + value: []any{"�\x00"}, }, { // UTF16 surrogate followed by ascii character input: "[\"\\uDF00\\u000e\"]", - value: []interface{}{"�\x0e"}, + value: []any{"�\x0e"}, }, { // UTF16 surrogate followed by unicode character input: "[[\"\\uDF00\\u0800\"]]", - value: []interface{}{[]interface{}{"�ࠀ"}}, + value: []any{[]any{"�ࠀ"}}, }, { // invalid UTF16 surrogate sequenced followed by a valid UTF16 surrogate sequence input: "[\"\\udf00\\udb00\\udf00\"]", - value: []interface{}{"�\U000d0300"}, + value: []any{"�\U000d0300"}, }, { // decode single-element slice into []byte field input: "{\"f\":[0],\"0\":[0]}", @@ -1051,7 +1050,7 @@ func TestUnmarshalFuzzBugs(t *testing.T) { }, { // random ASCII character input: "}", - value: []interface{}{}, + value: []any{}, }, { // random byte after valid JSON, decoded to a nil type input: "0\x93", @@ -1062,23 +1061,23 @@ func TestUnmarshalFuzzBugs(t *testing.T) { }, { // random byte after valid JSON, decoded to a slice type input: "0\x93", - value: []interface{}{}, + value: []any{}, }, { // decode integer into slice input: "0", - value: []interface{}{}, + value: []any{}, }, { // decode integer with trailing space into slice input: "0\t", - value: []interface{}{}, + value: []any{}, }, { // decode integer with leading random bytes into slice input: "\b0", - value: []interface{}{}, + value: []any{}, }, { // decode string into slice followed by number input: "\"\"0", - value: []interface{}{}, + value: []any{}, }, { // decode what looks like an object followed by a number into a string input: "{0", @@ -1101,15 +1100,15 @@ func TestUnmarshalFuzzBugs(t *testing.T) { }, { // decode what looks like an array followed by a number into a slice input: "[9E600", - value: []interface{}{}, + value: []any{}, }, { // decode a number which is too large to fit in a float64 input: "[1e900]", - value: []interface{}{}, + value: []any{}, }, { // many nested arrays openings input: "[[[[[[", - value: []interface{}{}, + value: []any{}, }, { // decode a map with value type mismatch and missing closing character input: "{\"\":0", @@ -1143,7 +1142,7 @@ func TestUnmarshalFuzzBugs(t *testing.T) { }, { // decode object with null key into map input: "{null:0}", - value: map[string]interface{}{}, + value: map[string]any{}, }, { // decode unquoted integer into struct field with string tag input: "{\"S\":0}", @@ -1312,8 +1311,8 @@ func TestUnmarshalFuzzBugs(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { - var ptr1 interface{} - var ptr2 interface{} + var ptr1 any + var ptr2 any if test.value != nil { ptr1 = reflect.New(reflect.TypeOf(test.value)).Interface() @@ -1351,9 +1350,9 @@ func BenchmarkEasyjsonUnmarshalSmallStruct(b *testing.B) { UserMentions []*string `json:"user_mentions"` } - var json = []byte(`{"hashtags":[{"indices":[5, 10],"text":"some-text"}],"urls":[],"user_mentions":[]}`) + json := []byte(`{"hashtags":[{"indices":[5, 10],"text":"some-text"}],"urls":[],"user_mentions":[]}`) - for i := 0; i < b.N; i++ { + for range b.N { var value Entities if err := Unmarshal(json, &value); err != nil { b.Fatal(err) @@ -1437,7 +1436,7 @@ func TestGithubIssue13(t *testing.T) { func TestGithubIssue15(t *testing.T) { // https://github.com/segmentio/encoding/issues/15 tests := []struct { - m interface{} + m any s string }{ { @@ -1496,15 +1495,17 @@ func (*intPtrB) MarshalText() ([]byte, error) { return []byte("B"), nil } -type structA struct{ I intPtrA } -type structB struct{ I intPtrB } -type structC struct{ M Marshaler } -type structD struct{ M encoding.TextMarshaler } +type ( + structA struct{ I intPtrA } + structB struct{ I intPtrB } + structC struct{ M Marshaler } + structD struct{ M encoding.TextMarshaler } +) func TestGithubIssue16(t *testing.T) { // https://github.com/segmentio/encoding/issues/16 tests := []struct { - value interface{} + value any output string }{ {value: sliceA(nil), output: `"A"`}, @@ -1648,9 +1649,9 @@ func TestGithubIssue23(t *testing.T) { b, _ := Marshal(C{ C: map[string]B{ - "1": B{ + "1": { B: map[string]A{ - "2": A{ + "2": { A: map[string]string{"3": "!"}, }, }, @@ -1673,10 +1674,10 @@ func TestGithubIssue23(t *testing.T) { } func TestGithubIssue26(t *testing.T) { - type interfaceType interface{} + type interfaceType any var value interfaceType - var data = []byte(`{}`) + data := []byte(`{}`) if err := Unmarshal(data, &value); err != nil { t.Error(err) @@ -1693,7 +1694,6 @@ func TestGithubIssue28(t *testing.T) { } else if string(b) != `{"err":{}}` { t.Error(string(b)) } - } func TestGithubIssue41(t *testing.T) { @@ -1716,7 +1716,6 @@ func TestGithubIssue41(t *testing.T) { "expected: ", expectedString, ) } - } func TestGithubIssue44(t *testing.T) { @@ -1778,7 +1777,7 @@ func TestSetTrustRawMessage(t *testing.T) { b := buf.Bytes() exp := []byte(`{"k":"value"}`) exp = append(exp, '\n') - if bytes.Compare(exp, b) != 0 { + if !bytes.Equal(exp, b) { t.Error( "unexpected encoding:", "expected", exp, @@ -1798,7 +1797,7 @@ func TestSetTrustRawMessage(t *testing.T) { b = buf.Bytes() exp = []byte(`{"k":bad"value}`) exp = append(exp, '\n') - if bytes.Compare(exp, b) != 0 { + if !bytes.Equal(exp, b) { t.Error( "unexpected encoding:", "expected", exp, @@ -1820,7 +1819,7 @@ func TestSetAppendNewline(t *testing.T) { b := buf.Bytes() exp := []byte(`"value"`) exp = append(exp, '\n') - if bytes.Compare(exp, b) != 0 { + if !bytes.Equal(exp, b) { t.Error( "unexpected encoding:", "expected", exp, @@ -1836,7 +1835,7 @@ func TestSetAppendNewline(t *testing.T) { } b = buf.Bytes() exp = []byte(`"value"`) - if bytes.Compare(exp, b) != 0 { + if !bytes.Equal(exp, b) { t.Error( "unexpected encoding:", "expected", exp, @@ -1862,7 +1861,7 @@ func TestAppendEscape(t *testing.T) { t.Run("basic", func(t *testing.T) { b := AppendEscape([]byte{}, `value`, AppendFlags(0)) exp := []byte(`"value"`) - if bytes.Compare(exp, b) != 0 { + if !bytes.Equal(exp, b) { t.Error( "unexpected encoding:", "expected", exp, @@ -1874,7 +1873,7 @@ func TestAppendEscape(t *testing.T) { t.Run("escaped", func(t *testing.T) { b := AppendEscape([]byte{}, `"escaped" `, EscapeHTML) exp := []byte(`"\"escaped\"\t\u003cvalue\u003e"`) - if bytes.Compare(exp, b) != 0 { + if !bytes.Equal(exp, b) { t.Error( "unexpected encoding:", "expected", exp, @@ -1891,7 +1890,7 @@ func TestAppendEscape(t *testing.T) { b = AppendEscape(b, `"escaped" `, EscapeHTML) b = append(b, '}') exp := []byte(`{"key":"\"escaped\"\t\u003cvalue\u003e"}`) - if bytes.Compare(exp, b) != 0 { + if !bytes.Equal(exp, b) { t.Error( "unexpected encoding:", "expected", exp, @@ -1918,7 +1917,7 @@ func TestAppendUnescape(t *testing.T) { t.Run("basic", func(t *testing.T) { out := AppendUnescape([]byte{}, []byte(`"value"`), ParseFlags(0)) exp := []byte("value") - if bytes.Compare(exp, out) != 0 { + if !bytes.Equal(exp, out) { t.Error( "unexpected decoding:", "expected", exp, @@ -1930,7 +1929,7 @@ func TestAppendUnescape(t *testing.T) { t.Run("escaped", func(t *testing.T) { b := AppendUnescape([]byte{}, []byte(`"\"escaped\"\t\u003cvalue\u003e"`), ParseFlags(0)) exp := []byte(`"escaped" `) - if bytes.Compare(exp, b) != 0 { + if !bytes.Equal(exp, b) { t.Error( "unexpected encoding:", "expected", exp, @@ -1945,7 +1944,7 @@ func TestAppendUnescape(t *testing.T) { b = AppendUnescape(b, []byte(`"\"escaped\"\t\u003cvalue\u003e"`), ParseFlags(0)) b = append(b, '}') exp := []byte(`{"key":"escaped" }`) - if bytes.Compare(exp, b) != 0 { + if !bytes.Equal(exp, b) { t.Error( "unexpected encoding:", "expected", string(exp), @@ -1958,7 +1957,7 @@ func TestAppendUnescape(t *testing.T) { func BenchmarkUnescape(b *testing.B) { s := []byte(`"\"escaped\"\t\u003cvalue\u003e"`) out := []byte{} - for i := 0; i < b.N; i++ { + for range b.N { out = Unescape(s) } @@ -1969,7 +1968,7 @@ func BenchmarkUnmarshalField(b *testing.B) { s := []byte(`"\"escaped\"\t\u003cvalue\u003e"`) var v string - for i := 0; i < b.N; i++ { + for range b.N { json.Unmarshal(s, &v) } diff --git a/json/parse.go b/json/parse.go index 49f63aa..949e7f3 100644 --- a/json/parse.go +++ b/json/parse.go @@ -106,7 +106,7 @@ func (d decoder) parseInt(b []byte, t reflect.Type) (int64, []byte, error) { } for _, c := range b[1:] { - if !(c >= '0' && c <= '9') { + if c < '0' || c > '9' { if count == 0 { b, err := d.inputError(b, t) return 0, b, err @@ -340,7 +340,7 @@ func (d decoder) parseNumber(b []byte) (v, r []byte, kind Kind, err error) { decimalStart := i for i < len(b) { - if c := b[i]; !('0' <= c && c <= '9') { + if c := b[i]; '0' > c || c > '9' { if i == decimalStart { r, err = b[i:], syntaxError(b, "expected digit but found '%c'", c) return @@ -375,7 +375,7 @@ func (d decoder) parseNumber(b []byte) (v, r []byte, kind Kind, err error) { exponentStart := i for i < len(b) { - if c := b[i]; !('0' <= c && c <= '9') { + if c := b[i]; '0' > c || c > '9' { if i == exponentStart { err = syntaxError(b, "expected digit but found '%c'", c) return @@ -588,9 +588,9 @@ func (d decoder) parseObject(b []byte) ([]byte, []byte, Kind, error) { } var err error - var a = b - var n = len(b) - var i = 0 + a := b + n := len(b) + i := 0 b = b[1:] for { @@ -654,9 +654,9 @@ func (d decoder) parseArray(b []byte) ([]byte, []byte, Kind, error) { } var err error - var a = b - var n = len(b) - var i = 0 + a := b + n := len(b) + i := 0 b = b[1:] for { diff --git a/json/parse_test.go b/json/parse_test.go index f835f6f..cb6d773 100644 --- a/json/parse_test.go +++ b/json/parse_test.go @@ -73,7 +73,6 @@ func TestParseStringUnquote(t *testing.T) { for _, test := range tests { t.Run(test.in, func(t *testing.T) { out, ext, _, err := d.parseStringUnquote([]byte(test.in), nil) - if err != nil { t.Errorf("%s => %s", test.in, err) return @@ -137,7 +136,7 @@ func BenchmarkParseString(b *testing.B) { s := []byte(`"__segment_internal"`) d := decoder{} - for i := 0; i != b.N; i++ { + for range b.N { d.parseString(s) } } @@ -145,7 +144,7 @@ func BenchmarkParseString(b *testing.B) { func BenchmarkToLower(b *testing.B) { s := []byte("someFieldWithALongName") - for i := 0; i != b.N; i++ { + for range b.N { bytes.ToLower(s) } } @@ -154,44 +153,46 @@ func BenchmarkAppendToLower(b *testing.B) { a := []byte(nil) s := []byte("someFieldWithALongName") - for i := 0; i != b.N; i++ { + for range b.N { a = appendToLower(a[:0], s) } } -var benchmarkHasPrefixString = []byte("some random string") -var benchmarkHasPrefixResult = false +var ( + benchmarkHasPrefixString = []byte("some random string") + benchmarkHasPrefixResult = false +) func BenchmarkHasPrefix(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { benchmarkHasPrefixResult = hasPrefix(benchmarkHasPrefixString, "null") } } func BenchmarkHasNullPrefix(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { benchmarkHasPrefixResult = hasNullPrefix(benchmarkHasPrefixString) } } func BenchmarkHasTruePrefix(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { benchmarkHasPrefixResult = hasTruePrefix(benchmarkHasPrefixString) } } func BenchmarkHasFalsePrefix(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { benchmarkHasPrefixResult = hasFalsePrefix(benchmarkHasPrefixString) } } func BenchmarkParseStringEscapeNone(b *testing.B) { - var j = []byte(`"` + strings.Repeat(`a`, 1000) + `"`) + j := []byte(`"` + strings.Repeat(`a`, 1000) + `"`) var s string b.SetBytes(int64(len(j))) - for i := 0; i < b.N; i++ { + for range b.N { if err := Unmarshal(j, &s); err != nil { b.Fatal(err) } @@ -200,11 +201,11 @@ func BenchmarkParseStringEscapeNone(b *testing.B) { } func BenchmarkParseStringEscapeOne(b *testing.B) { - var j = []byte(`"` + strings.Repeat(`a`, 998) + `\n"`) + j := []byte(`"` + strings.Repeat(`a`, 998) + `\n"`) var s string b.SetBytes(int64(len(j))) - for i := 0; i < b.N; i++ { + for range b.N { if err := Unmarshal(j, &s); err != nil { b.Fatal(err) } @@ -213,11 +214,11 @@ func BenchmarkParseStringEscapeOne(b *testing.B) { } func BenchmarkParseStringEscapeAll(b *testing.B) { - var j = []byte(`"` + strings.Repeat(`\`, 1000) + `"`) + j := []byte(`"` + strings.Repeat(`\`, 1000) + `"`) var s string b.SetBytes(int64(len(j))) - for i := 0; i < b.N; i++ { + for range b.N { if err := Unmarshal(j, &s); err != nil { b.Fatal(err) } diff --git a/json/string_test.go b/json/string_test.go index 84ac70b..8c2c614 100644 --- a/json/string_test.go +++ b/json/string_test.go @@ -31,7 +31,7 @@ func BenchmarkEscapeIndex7EscapeHTML(b *testing.B) { func benchmarkEscapeIndex(b *testing.B, s string, escapeHTML bool) { b.ResetTimer() - for i := 0; i < b.N; i++ { + for range b.N { escapeIndex(s, escapeHTML) } b.SetBytes(int64(len(s))) diff --git a/json/token.go b/json/token.go index 161b7fa..ddcd05d 100644 --- a/json/token.go +++ b/json/token.go @@ -306,8 +306,8 @@ func (t *Tokenizer) String() []byte { // The position of the tokenizer's current Value within the original byte slice // can be calculated like so: // -// end := len(b) - tok.Remaining() -// start := end - len(tok.Value) +// end := len(b) - tok.Remaining() +// start := end - len(tok.Value) // // And slicing b[start:end] will yield the tokenizer's current Value. func (t *Tokenizer) Remaining() int { @@ -423,6 +423,4 @@ func releaseStack(s *stack) { stackPool.Put(s) } -var ( - stackPool sync.Pool // *stack -) +var stackPool sync.Pool // *stack diff --git a/json/token_test.go b/json/token_test.go index cc05365..2fddea7 100644 --- a/json/token_test.go +++ b/json/token_test.go @@ -304,7 +304,7 @@ func BenchmarkTokenizer(b *testing.B) { function: func(b *testing.B, json []byte) { t := NewTokenizer(nil) - for i := 0; i < b.N; i++ { + for range b.N { t.Reset(json) for t.Next() { diff --git a/proto/bytes_test.go b/proto/bytes_test.go index b4df00f..2faa28f 100644 --- a/proto/bytes_test.go +++ b/proto/bytes_test.go @@ -19,7 +19,7 @@ func BenchmarkIsZeroBytes64K(b *testing.B) { } func benchmarkIsZeroBytes(b *testing.B, slice []byte) { - for i := 0; i < b.N; i++ { + for range b.N { isZeroBytes(slice) } } diff --git a/proto/decode.go b/proto/decode.go index 5c2bd02..aeab0bb 100644 --- a/proto/decode.go +++ b/proto/decode.go @@ -27,9 +27,7 @@ func decodeZigZag32(v uint32) int32 { type decodeFunc = func([]byte, unsafe.Pointer, flags) (int, error) -var ( - errVarintOverflow = errors.New("varint overflowed 64 bits integer") -) +var errVarintOverflow = errors.New("varint overflowed 64 bits integer") func decodeVarint(b []byte) (uint64, int, error) { if len(b) != 0 && b[0] < 0x80 { diff --git a/proto/decode_test.go b/proto/decode_test.go index 6c7e3b6..384bfe3 100644 --- a/proto/decode_test.go +++ b/proto/decode_test.go @@ -70,7 +70,7 @@ func BenchmarkDecodeTag(b *testing.B) { c := [8]byte{} n, _ := encodeTag(c[:], 1, varint) - for i := 0; i < b.N; i++ { + for range b.N { decodeTag(c[:n]) } } @@ -89,7 +89,7 @@ func BenchmarkDecodeMessage(b *testing.B) { msg := message{} b.SetBytes(int64(len(data))) - for i := 0; i < b.N; i++ { + for range b.N { if err := Unmarshal(data, &msg); err != nil { b.Fatal(err) } @@ -115,7 +115,7 @@ func BenchmarkDecodeMap(b *testing.B) { msg := message{} b.SetBytes(int64(len(data))) - for i := 0; i < b.N; i++ { + for range b.N { if err := Unmarshal(data, &msg); err != nil { b.Fatal(err) } @@ -137,11 +137,10 @@ func BenchmarkDecodeSlice(b *testing.B) { msg := message{} b.SetBytes(int64(len(data))) - for i := 0; i < b.N; i++ { + for range b.N { if err := Unmarshal(data, &msg); err != nil { b.Fatal(err) } msg = message{} } - } diff --git a/proto/encode_test.go b/proto/encode_test.go index c7d9704..225f904 100644 --- a/proto/encode_test.go +++ b/proto/encode_test.go @@ -37,7 +37,7 @@ func TestMarshalToShortBuffer(t *testing.T) { func BenchmarkEncodeVarintShort(b *testing.B) { c := [10]byte{} - for i := 0; i < b.N; i++ { + for range b.N { encodeVarint(c[:], 0) } } @@ -45,7 +45,7 @@ func BenchmarkEncodeVarintShort(b *testing.B) { func BenchmarkEncodeVarintLong(b *testing.B) { c := [10]byte{} - for i := 0; i < b.N; i++ { + for range b.N { encodeVarint(c[:], math.MaxUint64) } } @@ -53,7 +53,7 @@ func BenchmarkEncodeVarintLong(b *testing.B) { func BenchmarkEncodeTag(b *testing.B) { c := [8]byte{} - for i := 0; i < b.N; i++ { + for range b.N { encodeTag(c[:], 1, varint) } } @@ -74,7 +74,7 @@ func BenchmarkEncodeMessage(b *testing.B) { data := buf[:size] b.SetBytes(int64(size)) - for i := 0; i < b.N; i++ { + for range b.N { if _, err := MarshalTo(data, msg); err != nil { b.Fatal(err) } @@ -95,7 +95,7 @@ func BenchmarkEncodeMap(b *testing.B) { data := buf[:size] b.SetBytes(int64(size)) - for i := 0; i < b.N; i++ { + for range b.N { if _, err := MarshalTo(data, msg); err != nil { b.Fatal(err) } @@ -114,7 +114,7 @@ func BenchmarkEncodeSlice(b *testing.B) { data := buf[:size] b.SetBytes(int64(size)) - for i := 0; i < b.N; i++ { + for range b.N { if _, err := MarshalTo(data, &msg); err != nil { b.Fatal(err) } diff --git a/proto/error.go b/proto/error.go index 968c9a7..6e57716 100644 --- a/proto/error.go +++ b/proto/error.go @@ -5,9 +5,7 @@ import ( "fmt" ) -var ( - ErrWireTypeUnknown = errors.New("unknown wire type") -) +var ErrWireTypeUnknown = errors.New("unknown wire type") type UnmarshalFieldError struct { FieldNumer int diff --git a/proto/fixtures/fixtures.pb.go b/proto/fixtures/fixtures.pb.go index 2c48570..67e4943 100644 --- a/proto/fixtures/fixtures.pb.go +++ b/proto/fixtures/fixtures.pb.go @@ -202,7 +202,7 @@ func file_fixtures_proto_rawDescGZIP() []byte { } var file_fixtures_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_fixtures_proto_goTypes = []interface{}{ +var file_fixtures_proto_goTypes = []any{ (*Message)(nil), // 0: fixtures.Message } var file_fixtures_proto_depIdxs = []int32{ @@ -219,7 +219,7 @@ func file_fixtures_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_fixtures_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_fixtures_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*Message); i { case 0: return &v.state diff --git a/proto/fixtures/generate/main.go b/proto/fixtures/generate/main.go index 0614934..d89f5d9 100644 --- a/proto/fixtures/generate/main.go +++ b/proto/fixtures/generate/main.go @@ -8,7 +8,7 @@ import ( ) func main() { - os.Mkdir("protobuf", 0755) + os.Mkdir("protobuf", 0o755) tests := []struct { name string @@ -27,6 +27,6 @@ func main() { for _, test := range tests { b, _ := proto.Marshal(&test.value) - os.WriteFile("protobuf/"+test.name, b, 0644) + os.WriteFile("protobuf/"+test.name, b, 0o644) } } diff --git a/proto/message.go b/proto/message.go index 618300a..130185f 100644 --- a/proto/message.go +++ b/proto/message.go @@ -106,7 +106,7 @@ func (f FieldNumber) Bytes(v []byte) RawMessage { } // Value constructs a RawMessage for field number f from v. -func (f FieldNumber) Value(v interface{}) RawMessage { +func (f FieldNumber) Value(v any) RawMessage { switch x := v.(type) { case bool: return f.Bool(x) diff --git a/proto/message_test.go b/proto/message_test.go index 5805a3d..4e996e5 100644 --- a/proto/message_test.go +++ b/proto/message_test.go @@ -174,7 +174,7 @@ func BenchmarkScan(b *testing.B) { }, }) - for i := 0; i < b.N; i++ { + for range b.N { Scan(m, func(f FieldNumber, t WireType, v RawValue) (bool, error) { switch f { case 1, 2, 3: diff --git a/proto/proto.go b/proto/proto.go index e4353fd..d881d79 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -7,13 +7,13 @@ import ( "unsafe" ) -func Size(v interface{}) int { +func Size(v any) int { t, p := inspect(v) c := cachedCodecOf(t) return c.size(p, inline|toplevel) } -func Marshal(v interface{}) ([]byte, error) { +func Marshal(v any) ([]byte, error) { t, p := inspect(v) c := cachedCodecOf(t) b := make([]byte, c.size(p, inline|toplevel)) @@ -24,7 +24,7 @@ func Marshal(v interface{}) ([]byte, error) { return b, nil } -func MarshalTo(b []byte, v interface{}) (int, error) { +func MarshalTo(b []byte, v any) (int, error) { t, p := inspect(v) c := cachedCodecOf(t) n, err := c.encode(b, p, inline|toplevel) @@ -34,7 +34,7 @@ func MarshalTo(b []byte, v interface{}) (int, error) { return n, err } -func Unmarshal(b []byte, v interface{}) error { +func Unmarshal(b []byte, v any) error { if len(b) == 0 { // An empty input is a valid protobuf message with all fields set to the // zero-value. @@ -100,11 +100,11 @@ type iface struct { ptr unsafe.Pointer } -func inspect(v interface{}) (reflect.Type, unsafe.Pointer) { +func inspect(v any) (reflect.Type, unsafe.Pointer) { return reflect.TypeOf(v), pointer(v) } -func pointer(v interface{}) unsafe.Pointer { +func pointer(v any) unsafe.Pointer { return (*iface)(unsafe.Pointer(&v)).ptr } diff --git a/proto/proto_test.go b/proto/proto_test.go index d978aaf..4a97557 100644 --- a/proto/proto_test.go +++ b/proto/proto_test.go @@ -116,7 +116,7 @@ type messageWithCustomField struct { func TestMarshalUnmarshal(t *testing.T) { intVal := 42 - values := []interface{}{ + values := []any{ // bool true, false, @@ -279,20 +279,20 @@ func TestMarshalUnmarshal(t *testing.T) { M1: map[int]int{0: 1}, M2: map[string]string{"": "A"}, M3: map[string]message{ - "m0": message{}, - "m1": message{A: 42}, - "m3": message{S: submessage{X: "X", Y: "Y"}}, + "m0": {}, + "m1": {A: 42}, + "m3": {S: submessage{X: "X", Y: "Y"}}, }, M4: map[string]*message{ - "m0": &message{}, - "m1": &message{A: 42}, - "m3": &message{S: submessage{X: "X", Y: "Y"}}, + "m0": {}, + "m1": {A: 42}, + "m3": {S: submessage{X: "X", Y: "Y"}}, }, M5: map[key]uint{ - key{Hi: 0, Lo: 0}: 0, - key{Hi: 1, Lo: 0}: 1, - key{Hi: 0, Lo: 1}: 2, - key{Hi: math.MaxUint64, Lo: math.MaxUint64}: 3, + {Hi: 0, Lo: 0}: 0, + {Hi: 1, Lo: 0}: 1, + {Hi: 0, Lo: 1}: 2, + {Hi: math.MaxUint64, Lo: math.MaxUint64}: 3, }, }, diff --git a/proto/reflect.go b/proto/reflect.go index 3ba2302..b948d0b 100644 --- a/proto/reflect.go +++ b/proto/reflect.go @@ -366,7 +366,7 @@ func structTypeOf(t reflect.Type, seen map[reflect.Type]Type) *structType { fieldNumber := FieldNumber(0) taggedFields := FieldNumber(0) - for i, n := 0, t.NumField(); i < n; i++ { + for i := range t.NumField() { f := t.Field(i) if f.PkgPath != "" { diff --git a/proto/reflect_test.go b/proto/reflect_test.go index a7502a9..56c14b3 100644 --- a/proto/reflect_test.go +++ b/proto/reflect_test.go @@ -12,7 +12,7 @@ type RecursiveMessage struct { func TestTypeOf(t *testing.T) { tests := []struct { - value interface{} + value any proto string }{ // primitive types diff --git a/proto/rewrite.go b/proto/rewrite.go index ef7c0f5..1e44730 100644 --- a/proto/rewrite.go +++ b/proto/rewrite.go @@ -399,7 +399,7 @@ func parseRewriteTemplateStruct(t Type, f FieldNumber, j json.RawMessage, rules fieldsByName := map[string]Field{} - for i, n := 0, t.NumField(); i < n; i++ { + for i := range t.NumField() { f := t.Field(i) fieldsByName[f.Name] = f } diff --git a/proto/rewrite_test.go b/proto/rewrite_test.go index 5ff906d..e7675d2 100644 --- a/proto/rewrite_test.go +++ b/proto/rewrite_test.go @@ -328,7 +328,6 @@ func TestParseRewriteRules(t *testing.T) { "flags": 32 } }`), rules) - if err != nil { t.Fatal(err) } @@ -378,7 +377,7 @@ func BenchmarkRewrite(b *testing.B) { out := make([]byte, 0, 2*cap(p)) - for i := 0; i < b.N; i++ { + for range b.N { out, err = rw.Rewrite(out[:0], p) } } diff --git a/proto/slice.go b/proto/slice.go index 4778a7c..fa9ee9f 100644 --- a/proto/slice.go +++ b/proto/slice.go @@ -40,7 +40,7 @@ func sliceSizeFuncOf(t reflect.Type, r *repeatedField) sizeFunc { n := 0 if v := (*Slice)(p); v != nil { - for i := 0; i < v.Len(); i++ { + for i := range v.Len() { elem := v.Index(i, elemSize) size := r.codec.size(elem, wantzero) n += tagSize + size @@ -63,7 +63,7 @@ func sliceEncodeFuncOf(t reflect.Type, r *repeatedField) encodeFunc { offset := 0 if s := (*Slice)(p); s != nil { - for i := 0; i < s.Len(); i++ { + for i := range s.Len() { elem := s.Index(i, elemSize) size := r.codec.size(elem, wantzero) diff --git a/proto/struct.go b/proto/struct.go index a845b1b..5d739d9 100644 --- a/proto/struct.go +++ b/proto/struct.go @@ -57,7 +57,7 @@ func structCodecOf(t reflect.Type, seen map[reflect.Type]*codec) *codec { number := fieldNumber(1) fields := make([]structField, 0, numField) - for i := 0; i < numField; i++ { + for i := range numField { f := t.Field(i) if f.PkgPath != "" { @@ -164,7 +164,7 @@ func baseTypeOf(t reflect.Type) reflect.Type { } func structSizeFuncOf(t reflect.Type, fields []structField) sizeFunc { - var inlined = inlined(t) + inlined := inlined(t) var unique, repeated []*structField for i := range fields { @@ -212,7 +212,7 @@ func structSizeFuncOf(t reflect.Type, fields []structField) sizeFunc { } func structEncodeFuncOf(t reflect.Type, fields []structField) encodeFunc { - var inlined = inlined(t) + inlined := inlined(t) var unique, repeated []*structField for i := range fields { diff --git a/thrift/debug.go b/thrift/debug.go index 3bf76d7..25fe7da 100644 --- a/thrift/debug.go +++ b/thrift/debug.go @@ -24,7 +24,7 @@ type debugReader struct { l *log.Logger } -func (d *debugReader) log(method string, res interface{}, err error) { +func (d *debugReader) log(method string, res any, err error) { if err != nil { d.l.Printf("(%T).%s() → ERROR: %v", d.r, method, err) } else { @@ -129,7 +129,7 @@ type debugWriter struct { l *log.Logger } -func (d *debugWriter) log(method string, arg interface{}, err error) { +func (d *debugWriter) log(method string, arg any, err error) { if err != nil { d.l.Printf("(%T).%s(%#v) → ERROR: %v", d.w, method, arg, err) } else { diff --git a/thrift/decode.go b/thrift/decode.go index 4cc394b..6f9ae03 100644 --- a/thrift/decode.go +++ b/thrift/decode.go @@ -19,7 +19,7 @@ import ( // to Unmarshal, allowing the function to reuse objects referenced by pointer // fields of struct values. When reusing objects, the application is responsible // for resetting the state of v before calling Unmarshal again. -func Unmarshal(p Protocol, b []byte, v interface{}) error { +func Unmarshal(p Protocol, b []byte, v any) error { br := bytes.NewReader(b) pr := p.NewReader(br) @@ -43,7 +43,7 @@ func NewDecoder(r Reader) *Decoder { return &Decoder{r: r, f: decoderFlags(r)} } -func (d *Decoder) Decode(v interface{}) error { +func (d *Decoder) Decode(v any) error { t := reflect.TypeOf(v) p := reflect.ValueOf(v) @@ -55,7 +55,7 @@ func (d *Decoder) Decode(v interface{}) error { p = p.Elem() cache, _ := decoderCache.Load().(map[typeID]decodeFunc) - decode, _ := cache[makeTypeID(t)] + decode := cache[makeTypeID(t)] if decode == nil { decode = decodeFuncOf(t, make(decodeFuncCache)) @@ -237,7 +237,7 @@ func decodeFuncSliceOf(t reflect.Type, seen decodeFuncCache) decodeFunc { v.Set(reflect.MakeSlice(t, int(l.Size), int(l.Size))) flags = flags.only(decodeFlags) - for i := 0; i < int(l.Size); i++ { + for i := range int(l.Size) { if err := dec(r, v.Index(i), flags); err != nil { return with(dontExpectEOF(err), &decodeErrorList{cause: l, index: i}) } @@ -292,7 +292,7 @@ func decodeFuncMapOf(t reflect.Type, seen decodeFuncCache) decodeFunc { tmpElem := reflect.New(elem).Elem() flags = flags.only(decodeFlags) - for i := 0; i < int(m.Size); i++ { + for i := range int(m.Size) { if err := decodeKey(r, tmpKey, flags); err != nil { return with(dontExpectEOF(err), &decodeErrorMap{cause: m, index: i}) } @@ -345,7 +345,7 @@ func decodeFuncMapAsSetOf(t reflect.Type, seen decodeFuncCache) decodeFunc { tmp := reflect.New(key).Elem() flags = flags.only(decodeFlags) - for i := 0; i < int(s.Size); i++ { + for i := range int(s.Size) { if err := dec(r, tmp, flags); err != nil { return with(dontExpectEOF(err), &decodeErrorSet{cause: s, index: i}) } @@ -385,7 +385,7 @@ func (dec *structDecoder) decode(r Reader, v reflect.Value, flags flags) error { seen[i/64] |= 1 << (i % 64) // TODO: implement type conversions? - if f.Type != field.typ && !(f.Type == TRUE && field.typ == BOOL) { + if f.Type != field.typ && (f.Type != TRUE || field.typ != BOOL) { if flags.have(strict) { return &TypeMismatch{item: "field value", Expect: field.typ, Found: f.Type} } @@ -542,7 +542,7 @@ func readList(r Reader, f func(Reader, Type) error) error { return err } - for i := 0; i < int(l.Size); i++ { + for i := range int(l.Size) { if err := f(r, l.Type); err != nil { return with(dontExpectEOF(err), &decodeErrorList{cause: l, index: i}) } @@ -557,7 +557,7 @@ func readSet(r Reader, f func(Reader, Type) error) error { return err } - for i := 0; i < int(s.Size); i++ { + for i := range int(s.Size) { if err := f(r, s.Type); err != nil { return with(dontExpectEOF(err), &decodeErrorSet{cause: s, index: i}) } @@ -572,7 +572,7 @@ func readMap(r Reader, f func(Reader, Type, Type) error) error { return err } - for i := 0; i < int(m.Size); i++ { + for i := range int(m.Size) { if err := f(r, m.Key, m.Value); err != nil { return with(dontExpectEOF(err), &decodeErrorMap{cause: m, index: i}) } diff --git a/thrift/encode.go b/thrift/encode.go index bd2c3a9..8fb351d 100644 --- a/thrift/encode.go +++ b/thrift/encode.go @@ -13,7 +13,7 @@ import ( // protocol p. // // The function panics if v cannot be converted to a thrift representation. -func Marshal(p Protocol, v interface{}) ([]byte, error) { +func Marshal(p Protocol, v any) ([]byte, error) { buf := new(bytes.Buffer) enc := NewEncoder(p.NewWriter(buf)) err := enc.Encode(v) @@ -29,10 +29,10 @@ func NewEncoder(w Writer) *Encoder { return &Encoder{w: w, f: encoderFlags(w)} } -func (e *Encoder) Encode(v interface{}) error { +func (e *Encoder) Encode(v any) error { t := reflect.TypeOf(v) cache, _ := encoderCache.Load().(map[typeID]encodeFunc) - encode, _ := cache[makeTypeID(t)] + encode := cache[makeTypeID(t)] if encode == nil { encode = encodeFuncOf(t, make(encodeFuncCache)) @@ -154,7 +154,7 @@ func encodeFuncSliceOf(t reflect.Type, seen encodeFuncCache) encodeFunc { return err } - for i := 0; i < n; i++ { + for i := range n { if err := enc(w, v.Index(i), flags); err != nil { return err } @@ -193,7 +193,8 @@ func encodeFuncMapOf(t reflect.Type, seen encodeFuncCache) encodeFunc { return nil } - for i, iter := 0, v.MapRange(); iter.Next(); i++ { + iter := v.MapRange() + for iter.Next() { if err := encodeKey(w, iter.Key(), flags); err != nil { return err } @@ -228,7 +229,8 @@ func encodeFuncMapAsSetOf(t reflect.Type, seen encodeFuncCache) encodeFunc { return nil } - for i, iter := 0, v.MapRange(); iter.Next(); i++ { + iter := v.MapRange() + for iter.Next() { if err := enc(w, iter.Key(), flags); err != nil { return err } @@ -295,7 +297,7 @@ encodeFields: } skipValue := coalesceBoolFields && field.Type == BOOL - if skipValue && isTrue(x) == true { + if skipValue && isTrue(x) { field.Type = TRUE } diff --git a/thrift/protocol_test.go b/thrift/protocol_test.go index 0bdd38e..8aac085 100644 --- a/thrift/protocol_test.go +++ b/thrift/protocol_test.go @@ -11,57 +11,57 @@ import ( var protocolReadWriteTests = [...]struct { scenario string - read interface{} - write interface{} - values []interface{} + read any + write any + values []any }{ { scenario: "bool", read: thrift.Reader.ReadBool, write: thrift.Writer.WriteBool, - values: []interface{}{false, true}, + values: []any{false, true}, }, { scenario: "int8", read: thrift.Reader.ReadInt8, write: thrift.Writer.WriteInt8, - values: []interface{}{int8(0), int8(1), int8(-1)}, + values: []any{int8(0), int8(1), int8(-1)}, }, { scenario: "int16", read: thrift.Reader.ReadInt16, write: thrift.Writer.WriteInt16, - values: []interface{}{int16(0), int16(1), int16(-1)}, + values: []any{int16(0), int16(1), int16(-1)}, }, { scenario: "int32", read: thrift.Reader.ReadInt32, write: thrift.Writer.WriteInt32, - values: []interface{}{int32(0), int32(1), int32(-1)}, + values: []any{int32(0), int32(1), int32(-1)}, }, { scenario: "int64", read: thrift.Reader.ReadInt64, write: thrift.Writer.WriteInt64, - values: []interface{}{int64(0), int64(1), int64(-1)}, + values: []any{int64(0), int64(1), int64(-1)}, }, { scenario: "float64", read: thrift.Reader.ReadFloat64, write: thrift.Writer.WriteFloat64, - values: []interface{}{float64(0), float64(1), float64(-1)}, + values: []any{float64(0), float64(1), float64(-1)}, }, { scenario: "bytes", read: thrift.Reader.ReadBytes, write: thrift.Writer.WriteBytes, - values: []interface{}{ + values: []any{ []byte(""), []byte("A"), []byte("1234567890"), @@ -73,7 +73,7 @@ var protocolReadWriteTests = [...]struct { scenario: "string", read: thrift.Reader.ReadString, write: thrift.Writer.WriteString, - values: []interface{}{ + values: []any{ "", "A", "1234567890", @@ -85,7 +85,7 @@ var protocolReadWriteTests = [...]struct { scenario: "message", read: thrift.Reader.ReadMessage, write: thrift.Writer.WriteMessage, - values: []interface{}{ + values: []any{ thrift.Message{}, thrift.Message{Type: thrift.Call, Name: "Hello", SeqID: 10}, thrift.Message{Type: thrift.Reply, Name: "World", SeqID: 11}, @@ -98,7 +98,7 @@ var protocolReadWriteTests = [...]struct { scenario: "field", read: thrift.Reader.ReadField, write: thrift.Writer.WriteField, - values: []interface{}{ + values: []any{ thrift.Field{ID: 101, Type: thrift.TRUE}, thrift.Field{ID: 102, Type: thrift.FALSE}, thrift.Field{ID: 103, Type: thrift.I8}, @@ -119,7 +119,7 @@ var protocolReadWriteTests = [...]struct { scenario: "list", read: thrift.Reader.ReadList, write: thrift.Writer.WriteList, - values: []interface{}{ + values: []any{ thrift.List{}, thrift.List{Size: 0, Type: thrift.BOOL}, thrift.List{Size: 1, Type: thrift.I8}, @@ -131,7 +131,7 @@ var protocolReadWriteTests = [...]struct { scenario: "map", read: thrift.Reader.ReadMap, write: thrift.Writer.WriteMap, - values: []interface{}{ + values: []any{ thrift.Map{}, thrift.Map{Size: 1, Key: thrift.BINARY, Value: thrift.MAP}, thrift.Map{Size: 1000, Key: thrift.BINARY, Value: thrift.LIST}, diff --git a/thrift/struct.go b/thrift/struct.go index aa556f3..43710dd 100644 --- a/thrift/struct.go +++ b/thrift/struct.go @@ -50,7 +50,7 @@ type structField struct { } func forEachStructField(t reflect.Type, index []int, do func(structField)) { - for i, n := 0, t.NumField(); i < n; i++ { + for i := range t.NumField() { f := t.Field(i) if f.PkgPath != "" && !f.Anonymous { // unexported diff --git a/thrift/thrift_test.go b/thrift/thrift_test.go index 04c8efd..f90026e 100644 --- a/thrift/thrift_test.go +++ b/thrift/thrift_test.go @@ -12,16 +12,16 @@ import ( var marshalTestValues = [...]struct { scenario string - values []interface{} + values []any }{ { scenario: "bool", - values: []interface{}{false, true}, + values: []any{false, true}, }, { scenario: "int", - values: []interface{}{ + values: []any{ int(0), int(-1), int(1), @@ -30,7 +30,7 @@ var marshalTestValues = [...]struct { { scenario: "int8", - values: []interface{}{ + values: []any{ int8(0), int8(-1), int8(1), @@ -41,7 +41,7 @@ var marshalTestValues = [...]struct { { scenario: "int16", - values: []interface{}{ + values: []any{ int16(0), int16(-1), int16(1), @@ -52,7 +52,7 @@ var marshalTestValues = [...]struct { { scenario: "int32", - values: []interface{}{ + values: []any{ int32(0), int32(-1), int32(1), @@ -63,7 +63,7 @@ var marshalTestValues = [...]struct { { scenario: "int64", - values: []interface{}{ + values: []any{ int64(0), int64(-1), int64(1), @@ -74,7 +74,7 @@ var marshalTestValues = [...]struct { { scenario: "string", - values: []interface{}{ + values: []any{ "", "A", "1234567890", @@ -84,7 +84,7 @@ var marshalTestValues = [...]struct { { scenario: "[]byte", - values: []interface{}{ + values: []any{ []byte(""), []byte("A"), []byte("1234567890"), @@ -94,7 +94,7 @@ var marshalTestValues = [...]struct { { scenario: "[]string", - values: []interface{}{ + values: []any{ []string{}, []string{"A"}, []string{"hello", "world", "!!!"}, @@ -104,7 +104,7 @@ var marshalTestValues = [...]struct { { scenario: "map[string]int", - values: []interface{}{ + values: []any{ map[string]int{}, map[string]int{"A": 1}, map[string]int{"hello": 1, "world": 2, "answer": 42}, @@ -113,7 +113,7 @@ var marshalTestValues = [...]struct { { scenario: "map[int64]struct{}", - values: []interface{}{ + values: []any{ map[int64]struct{}{}, map[int64]struct{}{0: {}, 1: {}, 2: {}}, }, @@ -121,7 +121,7 @@ var marshalTestValues = [...]struct { { scenario: "[]map[string]struct{}", - values: []interface{}{ + values: []any{ []map[string]struct{}{}, []map[string]struct{}{{}, {"A": {}, "B": {}, "C": {}}}, }, @@ -129,12 +129,12 @@ var marshalTestValues = [...]struct { { scenario: "struct{}", - values: []interface{}{struct{}{}}, + values: []any{struct{}{}}, }, { scenario: "Point2D", - values: []interface{}{ + values: []any{ Point2D{}, Point2D{X: 1}, Point2D{Y: 2}, @@ -144,7 +144,7 @@ var marshalTestValues = [...]struct { { scenario: "RecursiveStruct", - values: []interface{}{ + values: []any{ RecursiveStruct{}, RecursiveStruct{Value: "hello"}, RecursiveStruct{Value: "hello", Next: &RecursiveStruct{}}, @@ -154,7 +154,7 @@ var marshalTestValues = [...]struct { { scenario: "StructWithEnum", - values: []interface{}{ + values: []any{ StructWithEnum{}, StructWithEnum{Enum: 1}, StructWithEnum{Enum: 2}, @@ -163,7 +163,7 @@ var marshalTestValues = [...]struct { { scenario: "StructWithPointToPointerToBool", - values: []interface{}{ + values: []any{ StructWithPointerToPointerToBool{ Test: newBoolPtr(true), }, @@ -172,7 +172,7 @@ var marshalTestValues = [...]struct { { scenario: "StructWithEmbeddedStrutPointerWithPointerToPointer", - values: []interface{}{ + values: []any{ StructWithEmbeddedStrutPointerWithPointerToPointer{ StructWithPointerToPointerToBool: &StructWithPointerToPointerToBool{ Test: newBoolPtr(true), @@ -183,7 +183,7 @@ var marshalTestValues = [...]struct { { scenario: "Union", - values: []interface{}{ + values: []any{ Union{}, Union{A: true, F: newBool(true)}, Union{B: 42, F: newInt(42)}, @@ -216,10 +216,10 @@ type StructWithEmbeddedStrutPointerWithPointerToPointer struct { } type Union struct { - A bool `thrift:"1"` - B int `thrift:"2"` - C string `thrift:"3"` - F interface{} `thrift:",union"` + A bool `thrift:"1"` + B int `thrift:"2"` + C string `thrift:"3"` + F any `thrift:",union"` } func newBool(b bool) *bool { return &b } @@ -286,7 +286,7 @@ func benchmarkMarshal(b *testing.B, p thrift.Protocol) { }, } - for i := 0; i < b.N; i++ { + for range b.N { buf.Reset() enc.Encode(val) } @@ -323,7 +323,7 @@ func benchmarkUnmarshal(b *testing.B, p thrift.Protocol) { dec := thrift.NewDecoder(p.NewReader(rb)) val := &BenchmarkDecodeType{} - for i := 0; i < b.N; i++ { + for range b.N { rb.Reset(buf) dec.Decode(val) } From 7b9ca4e0d1bc3f570ac39c2009df846e18ce1606 Mon Sep 17 00:00:00 2001 From: Dominic Barnes Date: Fri, 18 Jul 2025 09:10:32 -0700 Subject: [PATCH 09/10] perf(json): address performance penalty found in v0.4.0+ (#149) * perf(json): revert decodeInto helper changes for object/array/string * per(json): add benchmarks for decoding object and array with mixed members --- json/decode.go | 14 +++++++++----- json/golang_bench_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/json/decode.go b/json/decode.go index 46922d5..b44dde6 100644 --- a/json/decode.go +++ b/json/decode.go @@ -1292,15 +1292,19 @@ func (d decoder) decodeInterface(b []byte, p unsafe.Pointer) ([]byte, error) { switch k.Class() { case Object: - v, err = decodeInto[map[string]any](&val, v, d, decoder.decodeMapStringInterface) + m := make(map[string]interface{}) + v, err = d.decodeMapStringInterface(v, unsafe.Pointer(&m)) + val = m case Array: - size := alignedSize(interfaceType) - fn := constructSliceDecodeFunc(size, sliceInterfaceType, decoder.decodeInterface) - v, err = decodeInto[[]any](&val, v, d, fn) + a := make([]interface{}, 0, 10) + v, err = d.decodeSlice(v, unsafe.Pointer(&a), unsafe.Sizeof(a[0]), sliceInterfaceType, decoder.decodeInterface) + val = a case String: - v, err = decodeInto[string](&val, v, d, decoder.decodeString) + s := "" + v, err = d.decodeString(v, unsafe.Pointer(&s)) + val = s case Null: v, val = nil, nil diff --git a/json/golang_bench_test.go b/json/golang_bench_test.go index 201352f..3cf7e5a 100644 --- a/json/golang_bench_test.go +++ b/json/golang_bench_test.go @@ -261,6 +261,32 @@ func BenchmarkUnmarshalString(b *testing.B) { }) } +func BenchmarkUnmarshalObjectMixed(b *testing.B) { + b.ReportAllocs() + data := []byte(`{"string":"hello world","time":"2025-07-17T18:40:04.338Z","bool":true,"integer":42,"decimal":3.14,"null":null,"object":{"hello":"world"},"array":[1,2,3]}`) + b.RunParallel(func(pb *testing.PB) { + var m map[string]any + for pb.Next() { + if err := Unmarshal(data, &m); err != nil { + b.Fatal("Unmarshal:", err) + } + } + }) +} + +func BenchmarkUnmarshalArrayMixed(b *testing.B) { + b.ReportAllocs() + data := []byte(`["hello world","2025-07-17T18:40:04.338Z",true,42,3.14,null,{"hello":"world"},[1,2,3]]`) + b.RunParallel(func(pb *testing.PB) { + var a []any + for pb.Next() { + if err := Unmarshal(data, &a); err != nil { + b.Fatal("Unmarshal:", err) + } + } + }) +} + func BenchmarkUnmarshalFloat64(b *testing.B) { b.ReportAllocs() data := []byte(`3.14`) From fd406855de30c54110d23eace25478ab9c6fa2cc Mon Sep 17 00:00:00 2001 From: Kevin Gillette Date: Tue, 29 Jul 2025 00:02:30 -0600 Subject: [PATCH 10/10] lint fixes (#151) * json: use reflect.PointerTo (avoid deprecated naming) * json: use maps.Copy * lint: remove unused consts/fields * exclude stdlib copied tests from linting * json: partial lint fixes --- .github/workflows/benchmark.yml | 2 +- .golangci.yml | 7 +++++++ benchmarks/Makefile | 2 +- json/codec.go | 21 +++++++++------------ json/decode.go | 4 ++-- json/json.go | 4 ++-- json/json_test.go | 11 +++-------- json/parse.go | 5 ----- 8 files changed, 25 insertions(+), 31 deletions(-) create mode 100644 .golangci.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 1bd2c95..1ec1dfb 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - - name: Steup Go + - name: Setup Go uses: actions/setup-go@v2 with: go-version: "1.24" diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..01db160 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,7 @@ +version: "2" +linters: + exclusions: + rules: + # Tests copied from the stdlib are not meant to be linted. + - path: 'golang_(.+_)?test\.go' + source: "^" # regex diff --git a/benchmarks/Makefile b/benchmarks/Makefile index e14cb3e..8b36507 100644 --- a/benchmarks/Makefile +++ b/benchmarks/Makefile @@ -74,7 +74,7 @@ benchstat := ${GOPATH}/bin/benchstat all: $(benchstat): - go install golang.org/x/perf/cmd/benchstat + go install golang.org/x/perf/cmd/benchstat@latest $(benchmark.cmd.dir)/message.pb.go: $(benchmark.cmd.dir)/message.proto @protoc -I. \ diff --git a/json/codec.go b/json/codec.go index 32c078f..77fe264 100644 --- a/json/codec.go +++ b/json/codec.go @@ -4,6 +4,7 @@ import ( "encoding" "encoding/json" "fmt" + "maps" "math/big" "reflect" "sort" @@ -73,12 +74,9 @@ func cacheLoad() map[unsafe.Pointer]codec { func cacheStore(typ reflect.Type, cod codec, oldCodecs map[unsafe.Pointer]codec) { newCodecs := make(map[unsafe.Pointer]codec, len(oldCodecs)+1) + maps.Copy(newCodecs, oldCodecs) newCodecs[typeid(typ)] = cod - for t, c := range oldCodecs { - newCodecs[t] = c - } - cache.Store(&newCodecs) } @@ -205,7 +203,7 @@ func constructCodec(t reflect.Type, seen map[reflect.Type]*structType, canAddr b c = constructUnsupportedTypeCodec(t) } - p := reflect.PtrTo(t) + p := reflect.PointerTo(t) if canAddr { switch { @@ -291,7 +289,7 @@ func constructSliceCodec(t reflect.Type, seen map[reflect.Type]*structType) code // Go 1.7+ behavior: slices of byte types (and aliases) may override the // default encoding and decoding behaviors by implementing marshaler and // unmarshaler interfaces. - p := reflect.PtrTo(e) + p := reflect.PointerTo(e) c := codec{} switch { @@ -391,7 +389,7 @@ func constructMapCodec(t reflect.Type, seen map[reflect.Type]*structType) codec kc := codec{} vc := constructCodec(v, seen, false) - if k.Implements(textMarshalerType) || reflect.PtrTo(k).Implements(textUnmarshalerType) { + if k.Implements(textMarshalerType) || reflect.PointerTo(k).Implements(textUnmarshalerType) { kc.encode = constructTextMarshalerEncodeFunc(k, false) kc.decode = constructTextUnmarshalerDecodeFunc(k, true) @@ -972,7 +970,6 @@ type structType struct { ficaseIndex map[string]*structField keyset []byte typ reflect.Type - inlined bool } type structField struct { @@ -1095,10 +1092,10 @@ var ( timeType = reflect.TypeOf(time.Time{}) rawMessageType = reflect.TypeOf(RawMessage(nil)) - numberPtrType = reflect.PtrTo(numberType) - durationPtrType = reflect.PtrTo(durationType) - timePtrType = reflect.PtrTo(timeType) - rawMessagePtrType = reflect.PtrTo(rawMessageType) + numberPtrType = reflect.PointerTo(numberType) + durationPtrType = reflect.PointerTo(durationType) + timePtrType = reflect.PointerTo(timeType) + rawMessagePtrType = reflect.PointerTo(rawMessageType) sliceInterfaceType = reflect.TypeOf(([]any)(nil)) sliceStringType = reflect.TypeOf(([]any)(nil)) diff --git a/json/decode.go b/json/decode.go index b44dde6..c87f01e 100644 --- a/json/decode.go +++ b/json/decode.go @@ -1410,7 +1410,7 @@ func (d decoder) decodeMaybeEmptyInterface(b []byte, p unsafe.Pointer, t reflect return d.decodeUnmarshalTypeError(b, p, t) } -func (d decoder) decodeUnmarshalTypeError(b []byte, p unsafe.Pointer, t reflect.Type) ([]byte, error) { +func (d decoder) decodeUnmarshalTypeError(b []byte, _ unsafe.Pointer, t reflect.Type) ([]byte, error) { v, b, _, err := d.parseValue(b) if err != nil { return b, err @@ -1500,7 +1500,7 @@ func (d decoder) decodeTextUnmarshaler(b []byte, p unsafe.Pointer, t reflect.Typ value = "array" } - return b, &UnmarshalTypeError{Value: value, Type: reflect.PtrTo(t)} + return b, &UnmarshalTypeError{Value: value, Type: reflect.PointerTo(t)} } func (d decoder) prependField(key, field string) string { diff --git a/json/json.go b/json/json.go index 11ec69c..028fd1f 100644 --- a/json/json.go +++ b/json/json.go @@ -15,7 +15,7 @@ import ( type Delim = json.Delim // InvalidUTF8Error is documented at https://golang.org/pkg/encoding/json/#InvalidUTF8Error -type InvalidUTF8Error = json.InvalidUTF8Error +type InvalidUTF8Error = json.InvalidUTF8Error //nolint:staticcheck // compat. // InvalidUnmarshalError is documented at https://golang.org/pkg/encoding/json/#InvalidUnmarshalError type InvalidUnmarshalError = json.InvalidUnmarshalError @@ -39,7 +39,7 @@ type SyntaxError = json.SyntaxError type Token = json.Token // UnmarshalFieldError is documented at https://golang.org/pkg/encoding/json/#UnmarshalFieldError -type UnmarshalFieldError = json.UnmarshalFieldError +type UnmarshalFieldError = json.UnmarshalFieldError //nolint:staticcheck // compat. // UnmarshalTypeError is documented at https://golang.org/pkg/encoding/json/#UnmarshalTypeError type UnmarshalTypeError = json.UnmarshalTypeError diff --git a/json/json_test.go b/json/json_test.go index b40e000..8256be2 100644 --- a/json/json_test.go +++ b/json/json_test.go @@ -240,7 +240,7 @@ var testValues = [...]any{ A string `json:"name"` B string `json:"-"` C string `json:",omitempty"` - D map[string]any `json:",string"` + D map[string]any `json:",string"` //nolint:staticcheck // intentional e string }{A: "Luke", D: map[string]any{"answer": float64(42)}}, struct{ point }{point{1, 2}}, @@ -880,12 +880,11 @@ func TestDecodeLines(t *testing.T) { t.Run(test.desc, func(t *testing.T) { d := NewDecoder(test.reader) var count int - var err error for { var o obj - err = d.Decode(&o) + err := d.Decode(&o) if err != nil { - if err == io.EOF { + if errors.Is(err, io.EOF) { break } @@ -904,10 +903,6 @@ func TestDecodeLines(t *testing.T) { count++ } - if err != nil && err != io.EOF { - t.Error(err) - } - if count != test.expectCount { t.Errorf("expected %d objects, got %d", test.expectCount, count) } diff --git a/json/parse.go b/json/parse.go index 949e7f3..d0ee221 100644 --- a/json/parse.go +++ b/json/parse.go @@ -21,11 +21,6 @@ const ( cr = '\r' ) -const ( - escape = '\\' - quote = '"' -) - func internalParseFlags(b []byte) (flags ParseFlags) { // Don't consider surrounding whitespace b = skipSpaces(b)