diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..02dd0e61 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,34 @@ +name: test + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + push: + branches: + - 'master' + - 'main' + tags: + - 'v*' + pull_request: + +permissions: + contents: read + +jobs: + test: + strategy: + matrix: + go-version: [1.18.x, 1.19.x, 1.20.x, 1.21.x] + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + steps: + - uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - uses: actions/checkout@v3 + - name: test + run: make ci diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 00000000..3eff0426 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,39 @@ +name: validate + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + push: + branches: + - 'master' + - 'main' + tags: + - 'v*' + pull_request: + +permissions: + contents: read + +jobs: + linters: + strategy: + matrix: + go-version: [1.21.x] + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + steps: + - uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - uses: actions/checkout@v3 + - name: prepare generated code + run: make prepare + - name: lint + uses: golangci/golangci-lint-action@v3 + with: + version: v1.55.1 + args: --print-resources-usage --timeout=10m --verbose diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..4c44c5fa --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,3 @@ +linters: + disable: + - errcheck diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 8b6e7931..00000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -dist: bionic - -language: go - -arch: - - amd64 - - ppc64le - - arm64 -go: - - 1.15.x - - 1.16.x - - tip - -jobs: - allow_failures: - - go: 'tip' - include: - - arch: amd64 - env: GIMME_ARCH=386 - -script: "make travis" - diff --git a/Makefile b/Makefile index 4c551fdc..dfcee7d2 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ SHELL := /bin/bash BIN = $(GOBIN)/msgp -.PHONY: clean wipe install get-deps bench all +.PHONY: clean wipe install get-deps bench all ci prepare $(BIN): */*.go @go install ./... @@ -43,12 +43,20 @@ get-deps: all: install $(GGEN) $(MGEN) -# travis CI enters here -travis: - arch - if [ `arch` == 'x86_64' ]; then sudo apt update; sudo apt install build-essential; wget https://github.com/tinygo-org/tinygo/releases/download/v0.18.0/tinygo_0.18.0_amd64.deb; sudo dpkg -i tinygo_0.18.0_amd64.deb; export PATH=$PATH:/usr/local/tinygo/bin; fi - go get -d -t ./... - go build -o "$${GOPATH%%:*}/bin/msgp" . +# Prepare generated code to be used for linting and testing in CI +prepare: + go install . go generate ./msgp go generate ./_generated + +# CI enters here +ci: prepare + arch + if [ `arch` == 'x86_64' ]; then \ + sudo apt-get -y -q update; \ + sudo apt-get -y -q install build-essential; \ + wget -q https://github.com/tinygo-org/tinygo/releases/download/v0.30.0/tinygo_0.30.0_amd64.deb; \ + sudo dpkg -i tinygo_0.30.0_amd64.deb; \ + export PATH=$$PATH:/usr/local/tinygo/bin; \ + fi go test -v ./... ./_generated diff --git a/README.md b/README.md index 8ba4e708..cbb2f950 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ -MessagePack Code Generator [![Build Status](https://travis-ci.org/desertbit/msgp.svg?branch=master)](https://travis-ci.org/desertbit/msgp) + +This is a code generation tool and serialization library for [MessagePack](http://msgpack.org). You can read more about MessagePack [in the wiki](http://github.com/desertbit/msgp/wiki), or at [msgpack.org](http://msgpack.org). ======= +MessagePack Code Generator +======= + +[![Go Reference](https://pkg.go.dev/badge/github.com/desertbit/msgp.svg)](https://pkg.go.dev/github.com/desertbit/msgp) +[![test](https://github.com/desertbit/msgp/actions/workflows/test.yml/badge.svg)](https://github.com/desertbit/msgp/actions/workflows/test.yml) +[![validate](https://github.com/desertbit/msgp/actions/workflows/validate.yml/badge.svg)](https://github.com/desertbit/msgp/actions/workflows/validate.yml) This is a code generation tool and serialization library for [MessagePack](http://msgpack.org). You can read more about MessagePack [in the wiki](http://github.com/desertbit/msgp/wiki), or at [msgpack.org](http://msgpack.org). @@ -7,7 +14,7 @@ This is a code generation tool and serialization library for [MessagePack](http: - Use Go as your schema language - Performance -- [JSON interop](http://godoc.org/github.com/desertbit/msgp/msgp#CopyToJSON) +- [JSON interop](https://pkg.go.dev/github.com/desertbit/msgp/msgp#CopyToJSON) - [User-defined extensions](http://github.com/desertbit/msgp/wiki/Using-Extensions) - Type safety - Encoding flexibility diff --git a/_generated/allownil.go b/_generated/allownil.go new file mode 100644 index 00000000..aeb3e4a1 --- /dev/null +++ b/_generated/allownil.go @@ -0,0 +1,230 @@ +package _generated + +import "time" + +//go:generate msgp + +type AllowNil0 struct { + ABool bool `msg:"abool,allownil"` + AInt int `msg:"aint,allownil"` + AInt8 int8 `msg:"aint8,allownil"` + AInt16 int16 `msg:"aint16,allownil"` + AInt32 int32 `msg:"aint32,allownil"` + AInt64 int64 `msg:"aint64,allownil"` + AUint uint `msg:"auint,allownil"` + AUint8 uint8 `msg:"auint8,allownil"` + AUint16 uint16 `msg:"auint16,allownil"` + AUint32 uint32 `msg:"auint32,allownil"` + AUint64 uint64 `msg:"auint64,allownil"` + AFloat32 float32 `msg:"afloat32,allownil"` + AFloat64 float64 `msg:"afloat64,allownil"` + AComplex64 complex64 `msg:"acomplex64,allownil"` + AComplex128 complex128 `msg:"acomplex128,allownil"` + + ANamedBool bool `msg:"anamedbool,allownil"` + ANamedInt int `msg:"anamedint,allownil"` + ANamedFloat64 float64 `msg:"anamedfloat64,allownil"` + + AMapStrStr map[string]string `msg:"amapstrstr,allownil"` + + APtrNamedStr *NamedString `msg:"aptrnamedstr,allownil"` + + AString string `msg:"astring,allownil"` + ANamedString string `msg:"anamedstring,allownil"` + AByteSlice []byte `msg:"abyteslice,allownil"` + + ASliceString []string `msg:"aslicestring,allownil"` + ASliceNamedString []NamedString `msg:"aslicenamedstring,allownil"` + + ANamedStruct NamedStructAN `msg:"anamedstruct,allownil"` + APtrNamedStruct *NamedStructAN `msg:"aptrnamedstruct,allownil"` + + AUnnamedStruct struct { + A string `msg:"a,allownil"` + } `msg:"aunnamedstruct,allownil"` // allownil not supported on unnamed struct + + EmbeddableStructAN `msg:",flatten,allownil"` // embed flat + + EmbeddableStruct2AN `msg:"embeddablestruct2,allownil"` // embed non-flat + + AArrayInt [5]int `msg:"aarrayint,allownil"` // not supported + + ATime time.Time `msg:"atime,allownil"` +} + +type AllowNil1 struct { + ABool []bool `msg:"abool,allownil"` + AInt []int `msg:"aint,allownil"` + AInt8 []int8 `msg:"aint8,allownil"` + AInt16 []int16 `msg:"aint16,allownil"` + AInt32 []int32 `msg:"aint32,allownil"` + AInt64 []int64 `msg:"aint64,allownil"` + AUint []uint `msg:"auint,allownil"` + AUint8 []uint8 `msg:"auint8,allownil"` + AUint16 []uint16 `msg:"auint16,allownil"` + AUint32 []uint32 `msg:"auint32,allownil"` + AUint64 []uint64 `msg:"auint64,allownil"` + AFloat32 []float32 `msg:"afloat32,allownil"` + AFloat64 []float64 `msg:"afloat64,allownil"` + AComplex64 []complex64 `msg:"acomplex64,allownil"` + AComplex128 []complex128 `msg:"acomplex128,allownil"` + + ANamedBool []bool `msg:"anamedbool,allownil"` + ANamedInt []int `msg:"anamedint,allownil"` + ANamedFloat64 []float64 `msg:"anamedfloat64,allownil"` + + AMapStrStr map[string]string `msg:"amapstrstr,allownil"` + + APtrNamedStr *NamedString `msg:"aptrnamedstr,allownil"` + + AString []string `msg:"astring,allownil"` + ANamedString []string `msg:"anamedstring,allownil"` + AByteSlice []byte `msg:"abyteslice,allownil"` + + ASliceString []string `msg:"aslicestring,allownil"` + ASliceNamedString []NamedString `msg:"aslicenamedstring,allownil"` + + ANamedStruct NamedStructAN `msg:"anamedstruct,allownil"` + APtrNamedStruct *NamedStructAN `msg:"aptrnamedstruct,allownil"` + + AUnnamedStruct struct { + A []string `msg:"a,allownil"` + } `msg:"aunnamedstruct,allownil"` + + *EmbeddableStructAN `msg:",flatten,allownil"` // embed flat + + *EmbeddableStruct2AN `msg:"embeddablestruct2,allownil"` // embed non-flat + + AArrayInt [5]int `msg:"aarrayint,allownil"` // not supported + + ATime *time.Time `msg:"atime,allownil"` +} + +type EmbeddableStructAN struct { + SomeEmbed []string `msg:"someembed,allownil"` +} + +type EmbeddableStruct2AN struct { + SomeEmbed2 []string `msg:"someembed2,allownil"` +} + +type NamedStructAN struct { + A []string `msg:"a,allownil"` + B []string `msg:"b,allownil"` +} + +type AllowNilHalfFull struct { + Field00 []string `msg:"field00,allownil"` + Field01 []string `msg:"field01"` + Field02 []string `msg:"field02,allownil"` + Field03 []string `msg:"field03"` +} + +type AllowNilLotsOFields struct { + Field00 []string `msg:"field00,allownil"` + Field01 []string `msg:"field01,allownil"` + Field02 []string `msg:"field02,allownil"` + Field03 []string `msg:"field03,allownil"` + Field04 []string `msg:"field04,allownil"` + Field05 []string `msg:"field05,allownil"` + Field06 []string `msg:"field06,allownil"` + Field07 []string `msg:"field07,allownil"` + Field08 []string `msg:"field08,allownil"` + Field09 []string `msg:"field09,allownil"` + Field10 []string `msg:"field10,allownil"` + Field11 []string `msg:"field11,allownil"` + Field12 []string `msg:"field12,allownil"` + Field13 []string `msg:"field13,allownil"` + Field14 []string `msg:"field14,allownil"` + Field15 []string `msg:"field15,allownil"` + Field16 []string `msg:"field16,allownil"` + Field17 []string `msg:"field17,allownil"` + Field18 []string `msg:"field18,allownil"` + Field19 []string `msg:"field19,allownil"` + Field20 []string `msg:"field20,allownil"` + Field21 []string `msg:"field21,allownil"` + Field22 []string `msg:"field22,allownil"` + Field23 []string `msg:"field23,allownil"` + Field24 []string `msg:"field24,allownil"` + Field25 []string `msg:"field25,allownil"` + Field26 []string `msg:"field26,allownil"` + Field27 []string `msg:"field27,allownil"` + Field28 []string `msg:"field28,allownil"` + Field29 []string `msg:"field29,allownil"` + Field30 []string `msg:"field30,allownil"` + Field31 []string `msg:"field31,allownil"` + Field32 []string `msg:"field32,allownil"` + Field33 []string `msg:"field33,allownil"` + Field34 []string `msg:"field34,allownil"` + Field35 []string `msg:"field35,allownil"` + Field36 []string `msg:"field36,allownil"` + Field37 []string `msg:"field37,allownil"` + Field38 []string `msg:"field38,allownil"` + Field39 []string `msg:"field39,allownil"` + Field40 []string `msg:"field40,allownil"` + Field41 []string `msg:"field41,allownil"` + Field42 []string `msg:"field42,allownil"` + Field43 []string `msg:"field43,allownil"` + Field44 []string `msg:"field44,allownil"` + Field45 []string `msg:"field45,allownil"` + Field46 []string `msg:"field46,allownil"` + Field47 []string `msg:"field47,allownil"` + Field48 []string `msg:"field48,allownil"` + Field49 []string `msg:"field49,allownil"` + Field50 []string `msg:"field50,allownil"` + Field51 []string `msg:"field51,allownil"` + Field52 []string `msg:"field52,allownil"` + Field53 []string `msg:"field53,allownil"` + Field54 []string `msg:"field54,allownil"` + Field55 []string `msg:"field55,allownil"` + Field56 []string `msg:"field56,allownil"` + Field57 []string `msg:"field57,allownil"` + Field58 []string `msg:"field58,allownil"` + Field59 []string `msg:"field59,allownil"` + Field60 []string `msg:"field60,allownil"` + Field61 []string `msg:"field61,allownil"` + Field62 []string `msg:"field62,allownil"` + Field63 []string `msg:"field63,allownil"` + Field64 []string `msg:"field64,allownil"` + Field65 []string `msg:"field65,allownil"` + Field66 []string `msg:"field66,allownil"` + Field67 []string `msg:"field67,allownil"` + Field68 []string `msg:"field68,allownil"` + Field69 []string `msg:"field69,allownil"` +} + +type AllowNil10 struct { + Field00 []string `msg:"field00,allownil"` + Field01 []string `msg:"field01,allownil"` + Field02 []string `msg:"field02,allownil"` + Field03 []string `msg:"field03,allownil"` + Field04 []string `msg:"field04,allownil"` + Field05 []string `msg:"field05,allownil"` + Field06 []string `msg:"field06,allownil"` + Field07 []string `msg:"field07,allownil"` + Field08 []string `msg:"field08,allownil"` + Field09 []string `msg:"field09,allownil"` +} + +type NotAllowNil10 struct { + Field00 []string `msg:"field00"` + Field01 []string `msg:"field01"` + Field02 []string `msg:"field02"` + Field03 []string `msg:"field03"` + Field04 []string `msg:"field04"` + Field05 []string `msg:"field05"` + Field06 []string `msg:"field06"` + Field07 []string `msg:"field07"` + Field08 []string `msg:"field08"` + Field09 []string `msg:"field09"` +} + +type AllowNilOmitEmpty struct { + Field00 []string `msg:"field00,allownil,omitempty"` + Field01 []string `msg:"field01,allownil"` +} + +type AllowNilOmitEmpty2 struct { + Field00 []string `msg:"field00,allownil,omitempty"` + Field01 []string `msg:"field01,allownil,omitempty"` +} diff --git a/_generated/allownil_test.go b/_generated/allownil_test.go new file mode 100644 index 00000000..e67cea87 --- /dev/null +++ b/_generated/allownil_test.go @@ -0,0 +1,57 @@ +package _generated + +import ( + "bytes" + "reflect" + "testing" + + "github.com/desertbit/msgp/msgp" +) + +func TestAllownil(t *testing.T) { + tt := &NamedStructAN{ + A: []string{}, + B: nil, + } + var buf bytes.Buffer + + err := msgp.Encode(&buf, tt) + if err != nil { + t.Fatal(err) + } + in := buf.Bytes() + + for _, tnew := range []*NamedStructAN{{}, {A: []string{}}, {B: []string{}}} { + err = msgp.Decode(bytes.NewBuffer(in), tnew) + if err != nil { + t.Error(err) + } + + if !reflect.DeepEqual(tt, tnew) { + t.Logf("in: %#v", tt) + t.Logf("out: %#v", tnew) + t.Fatal("objects not equal") + } + } + + in, err = tt.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + for _, tanother := range []*NamedStructAN{{}, {A: []string{}}, {B: []string{}}} { + var left []byte + left, err = tanother.UnmarshalMsg(in) + if err != nil { + t.Error(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left", len(left)) + } + + if !reflect.DeepEqual(tt, tanother) { + t.Logf("in: %#v", tt) + t.Logf("out: %#v", tanother) + t.Fatal("objects not equal") + } + } +} diff --git a/_generated/def.go b/_generated/def.go index 8c43da1b..6e5a88d9 100644 --- a/_generated/def.go +++ b/_generated/def.go @@ -46,27 +46,31 @@ type TestType struct { ValueA string `msg:"value_a"` ValueB []byte `msg:"value_b"` } `msg:"object"` - Child *TestType `msg:"child"` - Time time.Time `msg:"time"` - Any interface{} `msg:"any"` - Appended msgp.Raw `msg:"appended"` - Num msgp.Number `msg:"num"` - Byte byte - Rune rune - RunePtr *rune - RunePtrPtr **rune - RuneSlice []rune - Slice1 []string - Slice2 []string - SlicePtr *[]string + Child *TestType `msg:"child"` + Time time.Time `msg:"time"` + Any interface{} `msg:"any"` + Appended msgp.Raw `msg:"appended"` + Num msgp.Number `msg:"num"` + Byte byte + Rune rune + RunePtr *rune + RunePtrPtr **rune + RuneSlice []rune + Slice1 []string + Slice2 []string + SlicePtr *[]string + MapStringEmpty map[string]struct{} + MapStringEmpty2 map[string]EmptyStruct } //msgp:tuple Object type Object struct { - ObjectNo string `msg:"objno"` - Slice1 []string `msg:"slice1"` - Slice2 []string `msg:"slice2"` - MapMap map[string]map[string]string + ObjectNo string `msg:"objno"` + Slice1 []string `msg:"slice1"` + Slice2 []string `msg:"slice2"` + MapMap map[string]map[string]string + MapStringEmpty map[string]struct{} + MapStringEmpty2 map[string]EmptyStruct } //msgp:tuple TestBench @@ -88,15 +92,19 @@ type TestFast struct { } // Test nested aliases -type FastAlias TestFast -type AliasContainer struct { - Fast FastAlias -} +type ( + FastAlias TestFast + AliasContainer struct { + Fast FastAlias + } +) // Test dependency resolution -type IntA int -type IntB IntA -type IntC IntB +type ( + IntA int + IntB IntA + IntC IntB +) type TestHidden struct { A string @@ -124,8 +132,10 @@ type Things struct { //msgp:shim SpecialID as:[]byte using:toBytes/fromBytes -type SpecialID string -type TestObj struct{ ID1, ID2 SpecialID } +type ( + SpecialID string + TestObj struct{ ID1, ID2 SpecialID } +) func toBytes(id SpecialID) []byte { return []byte(string(id)) } func fromBytes(id []byte) SpecialID { return SpecialID(string(id)) } @@ -204,8 +214,10 @@ type FileHandle struct { Name string `msg:"name"` } -type CustomInt int -type CustomBytes []byte +type ( + CustomInt int + CustomBytes []byte +) type Wrapper struct { Tree *Tree @@ -243,7 +255,7 @@ type ArrayConstants struct { ConstantUint32 [ConstantUint32]string ConstantUint64 [ConstantUint64]string ConstantHex [0x16]string - ConstantOctal [07]string + ConstantOctal [0o7]string } // Ensure non-msg struct tags work: @@ -258,8 +270,11 @@ type NonMsgStructTags struct { B string `json:"b"` C []string `json:"c"` VeryNested []struct { - A []string `json:"a"` - B []string `msg:"bbbb" xml:"-"` + A []string `json:"a"` + B []string `msg:"bbbb" xml:"-"` + C map[string]struct{} `msg:"cccc"` } } } + +type EmptyStruct struct{} diff --git a/_generated/gen_test.go b/_generated/gen_test.go index bc9573bd..6b64a9db 100644 --- a/_generated/gen_test.go +++ b/_generated/gen_test.go @@ -68,6 +68,15 @@ func (a *TestType) Equal(b *TestType) bool { if !bytes.Equal(aa, ab) { return false } + if len(a.MapStringEmpty) == 0 && len(b.MapStringEmpty) == 0 { + a.MapStringEmpty = nil + b.MapStringEmpty = nil + } + if len(a.MapStringEmpty2) == 0 && len(b.MapStringEmpty2) == 0 { + a.MapStringEmpty2 = nil + b.MapStringEmpty2 = nil + } + a.Time, b.Time = time.Time{}, time.Time{} aa, ab = nil, nil ok := reflect.DeepEqual(a, b) @@ -77,12 +86,11 @@ func (a *TestType) Equal(b *TestType) bool { } // This covers the following cases: -// - Recursive types -// - Non-builtin identifiers (and recursive types) -// - time.Time -// - map[string]string -// - anonymous structs -// +// - Recursive types +// - Non-builtin identifiers (and recursive types) +// - time.Time +// - map[string]string +// - anonymous structs func Test1EncodeDecode(t *testing.T) { f := 32.00 tt := &TestType{ @@ -98,9 +106,11 @@ func Test1EncodeDecode(t *testing.T) { ValueA: "here's the first inner value", ValueB: []byte("here's the second inner value"), }, - Child: nil, - Time: time.Now(), - Appended: msgp.Raw([]byte{}), // 'nil' + Child: nil, + Time: time.Now(), + Appended: msgp.Raw([]byte{}), // 'nil' + MapStringEmpty: map[string]struct{}{"Key": {}, "Key2": {}}, + MapStringEmpty2: map[string]EmptyStruct{"Key3": {}, "Key4": {}}, } var buf bytes.Buffer @@ -118,8 +128,8 @@ func Test1EncodeDecode(t *testing.T) { } if !tt.Equal(tnew) { - t.Logf("in: %v", tt) - t.Logf("out: %v", tnew) + t.Logf("in: %#v", tt) + t.Logf("out: %#v", tnew) t.Fatal("objects not equal") } diff --git a/_generated/issue94.go b/_generated/issue94.go index 4384d5d0..696ca4cc 100644 --- a/_generated/issue94.go +++ b/_generated/issue94.go @@ -6,16 +6,6 @@ import ( //go:generate msgp -// Issue 94: shims were not propogated recursively, -// which caused shims that weren't at the top level -// to be silently ignored. -// -// The following line will generate an error after -// the code is generated if the generated code doesn't -// have the right identifier in it. - -//go:generate ./search.sh $GOFILE timetostr - //msgp:shim time.Time as:string using:timetostr/strtotime type T struct { T time.Time diff --git a/_generated/issue94_test.go b/_generated/issue94_test.go new file mode 100644 index 00000000..38fec306 --- /dev/null +++ b/_generated/issue94_test.go @@ -0,0 +1,25 @@ +package _generated + +import ( + "bytes" + "os" + "testing" +) + +// Issue 94: shims were not propogated recursively, +// which caused shims that weren't at the top level +// to be silently ignored. +// +// The following line will generate an error after +// the code is generated if the generated code doesn't +// have the right identifier in it. +func TestIssue94(t *testing.T) { + b, err := os.ReadFile("issue94_gen.go") + if err != nil { + t.Fatal(err) + } + const want = "timetostr" + if !bytes.Contains(b, []byte(want)) { + t.Errorf("generated code did not contain %q", want) + } +} diff --git a/_generated/omitempty.go b/_generated/omitempty.go index 71f29b91..1701547f 100644 --- a/_generated/omitempty.go +++ b/_generated/omitempty.go @@ -52,10 +52,12 @@ type OmitEmpty0 struct { ATime time.Time `msg:"atime,omitempty"` } -type NamedBool bool -type NamedInt int -type NamedFloat64 float64 -type NamedString string +type ( + NamedBool bool + NamedInt int + NamedFloat64 float64 + NamedString string +) type EmbeddableStruct struct { SomeEmbed string `msg:"someembed,omitempty"` diff --git a/_generated/omitempty_test.go b/_generated/omitempty_test.go index 8d17e77d..453529cc 100644 --- a/_generated/omitempty_test.go +++ b/_generated/omitempty_test.go @@ -29,7 +29,6 @@ func mustEncodeToJSON(o msgp.Encodable) string { } func TestOmitEmpty0(t *testing.T) { - var s string var oe0a OmitEmpty0 @@ -92,12 +91,10 @@ func TestOmitEmpty0(t *testing.T) { if oe0c.AInt64 != oe0d.AInt64 { t.Fail() } - } // TestOmitEmptyHalfFull tests mixed omitempty and not func TestOmitEmptyHalfFull(t *testing.T) { - var s string var oeA OmitEmptyHalfFull @@ -124,7 +121,6 @@ func TestOmitEmptyHalfFull(t *testing.T) { // TestOmitEmptyLotsOFields tests the case of > 64 fields (triggers the bitmask needing to be an array instead of a single value) func TestOmitEmptyLotsOFields(t *testing.T) { - var s string var oeLotsA OmitEmptyLotsOFields @@ -147,11 +143,9 @@ func TestOmitEmptyLotsOFields(t *testing.T) { if s != `{"field64":"val64"}` { t.Errorf("wrong result: %s", s) } - } func BenchmarkOmitEmpty10AllEmpty(b *testing.B) { - en := msgp.NewWriter(ioutil.Discard) var s OmitEmpty10 @@ -163,11 +157,9 @@ func BenchmarkOmitEmpty10AllEmpty(b *testing.B) { b.Fatal(err) } } - } func BenchmarkOmitEmpty10AllFull(b *testing.B) { - en := msgp.NewWriter(ioutil.Discard) var s OmitEmpty10 s.Field00 = "this is the value of field00" @@ -189,11 +181,9 @@ func BenchmarkOmitEmpty10AllFull(b *testing.B) { b.Fatal(err) } } - } func BenchmarkNotOmitEmpty10AllEmpty(b *testing.B) { - en := msgp.NewWriter(ioutil.Discard) var s NotOmitEmpty10 @@ -208,7 +198,6 @@ func BenchmarkNotOmitEmpty10AllEmpty(b *testing.B) { } func BenchmarkNotOmitEmpty10AllFull(b *testing.B) { - en := msgp.NewWriter(ioutil.Discard) var s NotOmitEmpty10 s.Field00 = "this is the value of field00" diff --git a/_generated/omitzero.go b/_generated/omitzero.go new file mode 100644 index 00000000..1732ffb7 --- /dev/null +++ b/_generated/omitzero.go @@ -0,0 +1,86 @@ +package _generated + +import "time" + +//go:generate msgp + +// check some specific cases for omitzero + +type OmitZero0 struct { + AStruct OmitZeroA `msg:"astruct,omitempty"` // leave this one omitempty + BStruct OmitZeroA `msg:"bstruct,omitzero"` // and compare to this + AStructPtr *OmitZeroA `msg:"astructptr,omitempty"` // a pointer case omitempty + BStructPtr *OmitZeroA `msg:"bstructptr,omitzero"` // a pointer case omitzero + AExt OmitZeroExt `msg:"aext,omitzero"` // external type case + AExtPtr *OmitZeroExtPtr `msg:"aextptr,omitzero"` // external type pointer case + + // more + APtrNamedStr *NamedStringOZ `msg:"aptrnamedstr,omitzero"` + ANamedStruct NamedStructOZ `msg:"anamedstruct,omitzero"` + APtrNamedStruct *NamedStructOZ `msg:"aptrnamedstruct,omitzero"` + EmbeddableStruct `msg:",flatten,omitzero"` // embed flat + EmbeddableStructOZ `msg:"embeddablestruct2,omitzero"` // embed non-flat + ATime time.Time `msg:"atime,omitzero"` + + OmitZeroTuple OmitZeroTuple `msg:"ozt"` // the inside of a tuple should ignore both omitempty and omitzero +} + +type OmitZeroA struct { + A string `msg:"a,omitempty"` + B NamedStringOZ `msg:"b,omitzero"` + C NamedStringOZ `msg:"c,omitzero"` +} + +func (o *OmitZeroA) IsZero() bool { + if o == nil { + return true + } + return *o == (OmitZeroA{}) +} + +type NamedStructOZ struct { + A string `msg:"a,omitempty"` + B string `msg:"b,omitempty"` +} + +func (ns *NamedStructOZ) IsZero() bool { + if ns == nil { + return true + } + return *ns == (NamedStructOZ{}) +} + +type NamedStringOZ string + +func (ns *NamedStringOZ) IsZero() bool { + if ns == nil { + return true + } + return *ns == "" +} + +type EmbeddableStructOZ struct { + SomeEmbed string `msg:"someembed2,omitempty"` +} + +func (es EmbeddableStructOZ) IsZero() bool { return es == (EmbeddableStructOZ{}) } + +type EmbeddableStructOZ2 struct { + SomeEmbed2 string `msg:"someembed2,omitempty"` +} + +func (es EmbeddableStructOZ2) IsZero() bool { return es == (EmbeddableStructOZ2{}) } + +//msgp:tuple OmitZeroTuple + +// OmitZeroTuple is flagged for tuple output, it should ignore all omitempty and omitzero functionality +// since it's fundamentally incompatible. +type OmitZeroTuple struct { + FieldA string `msg:"fielda,omitempty"` + FieldB NamedStringOZ `msg:"fieldb,omitzero"` + FieldC NamedStringOZ `msg:"fieldc,omitzero"` +} + +type OmitZero1 struct { + T1 OmitZeroTuple `msg:"t1"` +} diff --git a/_generated/omitzero_ext.go b/_generated/omitzero_ext.go new file mode 100644 index 00000000..1fe2d8eb --- /dev/null +++ b/_generated/omitzero_ext.go @@ -0,0 +1,114 @@ +package _generated + +import ( + "github.com/desertbit/msgp/msgp" +) + +// this has "external" types that will show up +// as generic IDENT during code generation + +type OmitZeroExt struct { + a int // custom type +} + +// IsZero will return true if a is not positive +func (o OmitZeroExt) IsZero() bool { return o.a <= 0 } + +// EncodeMsg implements msgp.Encodable +func (o OmitZeroExt) EncodeMsg(en *msgp.Writer) (err error) { + if o.a > 0 { + return en.WriteInt(o.a) + } + return en.WriteNil() +} + +// DecodeMsg implements msgp.Decodable +func (o *OmitZeroExt) DecodeMsg(dc *msgp.Reader) (err error) { + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + return + } + o.a = 0 + return + } + o.a, err = dc.ReadInt() + return err +} + +// MarshalMsg implements msgp.Marshaler +func (o OmitZeroExt) MarshalMsg(b []byte) (ret []byte, err error) { + ret = msgp.Require(b, o.Msgsize()) + if o.a > 0 { + return msgp.AppendInt(ret, o.a), nil + } + return msgp.AppendNil(ret), nil +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (o *OmitZeroExt) UnmarshalMsg(bts []byte) (ret []byte, err error) { + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + return bts, err + } + o.a, bts, err = msgp.ReadIntBytes(bts) + return bts, err +} + +// Msgsize implements msgp.Msgsizer +func (o OmitZeroExt) Msgsize() (s int) { + return msgp.IntSize +} + +type OmitZeroExtPtr struct { + a int // custom type +} + +// IsZero will return true if a is nil or not positive +func (o *OmitZeroExtPtr) IsZero() bool { return o == nil || o.a <= 0 } + +// EncodeMsg implements msgp.Encodable +func (o *OmitZeroExtPtr) EncodeMsg(en *msgp.Writer) (err error) { + if o.a > 0 { + return en.WriteInt(o.a) + } + return en.WriteNil() +} + +// DecodeMsg implements msgp.Decodable +func (o *OmitZeroExtPtr) DecodeMsg(dc *msgp.Reader) (err error) { + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + return + } + o.a = 0 + return + } + o.a, err = dc.ReadInt() + return err +} + +// MarshalMsg implements msgp.Marshaler +func (o *OmitZeroExtPtr) MarshalMsg(b []byte) (ret []byte, err error) { + ret = msgp.Require(b, o.Msgsize()) + if o.a > 0 { + return msgp.AppendInt(ret, o.a), nil + } + return msgp.AppendNil(ret), nil +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (o *OmitZeroExtPtr) UnmarshalMsg(bts []byte) (ret []byte, err error) { + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + return bts, err + } + o.a, bts, err = msgp.ReadIntBytes(bts) + return bts, err +} + +// Msgsize implements msgp.Msgsizer +func (o *OmitZeroExtPtr) Msgsize() (s int) { + return msgp.IntSize +} diff --git a/_generated/omitzero_test.go b/_generated/omitzero_test.go new file mode 100644 index 00000000..749f6ba5 --- /dev/null +++ b/_generated/omitzero_test.go @@ -0,0 +1,77 @@ +package _generated + +import ( + "bytes" + "testing" +) + +func TestOmitZero(t *testing.T) { + + t.Run("OmitZeroExt_not_empty", func(t *testing.T) { + + z := OmitZero0{AExt: OmitZeroExt{a: 1}} + b, err := z.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(b, []byte("aext")) { + t.Errorf("expected to find aext in bytes %X", b) + } + z = OmitZero0{} + _, err = z.UnmarshalMsg(b) + if err != nil { + t.Fatal(err) + } + if z.AExt.a != 1 { + t.Errorf("z.AExt.a expected 1 but got %d", z.AExt.a) + } + + }) + + t.Run("OmitZeroExt_negative", func(t *testing.T) { + + z := OmitZero0{AExt: OmitZeroExt{a: -1}} // negative value should act as empty, via IsEmpty() call + b, err := z.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(b, []byte("aext")) { + t.Errorf("expected to not find aext in bytes %X", b) + } + z = OmitZero0{} + _, err = z.UnmarshalMsg(b) + if err != nil { + t.Fatal(err) + } + if z.AExt.a != 0 { + t.Errorf("z.AExt.a expected 0 but got %d", z.AExt.a) + } + + }) + + t.Run("OmitZeroTuple", func(t *testing.T) { + + // make sure tuple encoding isn't affected by omitempty or omitzero + + z := OmitZero0{OmitZeroTuple: OmitZeroTuple{FieldA: "", FieldB: "", FieldC: "fcval"}} + b, err := z.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + // verify the exact binary encoding, that the values follow each other without field names + if !bytes.Contains(b, []byte{0xA0, 0xA0, 0xA5, 'f', 'c', 'v', 'a', 'l'}) { + t.Errorf("failed to find expected bytes in %X", b) + } + z = OmitZero0{} + _, err = z.UnmarshalMsg(b) + if err != nil { + t.Fatal(err) + } + if z.OmitZeroTuple.FieldA != "" || + z.OmitZeroTuple.FieldB != "" || + z.OmitZeroTuple.FieldC != "fcval" { + t.Errorf("z.OmitZeroTuple unexpected value: %#v", z.OmitZeroTuple) + } + + }) +} diff --git a/_generated/search.sh b/_generated/search.sh deleted file mode 100755 index aa6d6477..00000000 --- a/_generated/search.sh +++ /dev/null @@ -1,12 +0,0 @@ -#! /bin/sh - -FILE=$(echo $1 | sed s/.go/_gen.go/) -echo "searching" $FILE "for" $2 -grep -q $2 $FILE -if [ $? -eq 0 ] -then - echo "OK" -else - echo "whoops!" - exit 1 -fi diff --git a/gen/decode.go b/gen/decode.go index 10f3fd8a..1a1d639b 100644 --- a/gen/decode.go +++ b/gen/decode.go @@ -63,7 +63,6 @@ func (d *decodeGen) gStruct(s *Struct) { } else { d.structAsMap(s) } - return } func (d *decodeGen) assignAndCheck(name string, typ string) { @@ -85,9 +84,21 @@ func (d *decodeGen) structAsTuple(s *Struct) { if !d.p.ok() { return } + fieldElem := s.Fields[i].FieldElem + anField := s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() + if anField { + d.p.print("\nif dc.IsNil() {") + d.p.print("\nerr = dc.ReadNil()") + d.p.wrapErrCheck(d.ctx.ArgsStr()) + d.p.printf("\n%s = nil\n} else {", s.Fields[i].FieldElem.Varname()) + } + SetIsAllowNil(fieldElem, anField) d.ctx.PushString(s.Fields[i].FieldName) - next(d, s.Fields[i].FieldElem) + next(d, fieldElem) d.ctx.Pop() + if anField { + d.p.printf("\n}") // close if statement + } } } @@ -103,11 +114,23 @@ func (d *decodeGen) structAsMap(s *Struct) { for i := range s.Fields { d.ctx.PushString(s.Fields[i].FieldName) d.p.printf("\ncase \"%s\":", s.Fields[i].FieldTag) - next(d, s.Fields[i].FieldElem) + fieldElem := s.Fields[i].FieldElem + anField := s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() + if anField { + d.p.print("\nif dc.IsNil() {") + d.p.print("\nerr = dc.ReadNil()") + d.p.wrapErrCheck(d.ctx.ArgsStr()) + d.p.printf("\n%s = nil\n} else {", fieldElem.Varname()) + } + SetIsAllowNil(fieldElem, anField) + next(d, fieldElem) d.ctx.Pop() if !d.p.ok() { return } + if anField { + d.p.printf("\n}") // close if statement + } } d.p.print("\ndefault:\nerr = dc.Skip()") d.p.wrapErrCheck(d.ctx.ArgsStr()) @@ -136,7 +159,8 @@ func (d *decodeGen) gBase(b *BaseElem) { switch b.Value { case Bytes: if b.Convert { - d.p.printf("\n%s, err = dc.ReadBytes([]byte(%s))", tmp, vname) + lowered := b.ToBase() + "(" + vname + ")" + d.p.printf("\n%s, err = dc.ReadBytes(%s)", tmp, lowered) } else { d.p.printf("\n%s, err = dc.ReadBytes(%s)", vname, vname) } @@ -177,6 +201,7 @@ func (d *decodeGen) gMap(m *Map) { // for element in map, read string/value // pair and assign + d.needsField() d.p.printf("\nfor %s > 0 {\n%s--", sz, sz) d.p.declare(m.Keyidx, "string") d.p.declare(m.Validx, m.Value.TypeName()) @@ -195,7 +220,11 @@ func (d *decodeGen) gSlice(s *Slice) { sz := randIdent() d.p.declare(sz, u32) d.assignAndCheck(sz, arrayHeader) - d.p.resizeSlice(sz, s) + if s.isAllowNil { + d.p.resizeSliceNoNil(sz, s) + } else { + d.p.resizeSlice(sz, s) + } d.p.rangeBlock(d.ctx, s.Index, s.Varname(), d, s.Els) } diff --git a/gen/elem.go b/gen/elem.go index 50f07e79..b8b821ab 100644 --- a/gen/elem.go +++ b/gen/elem.go @@ -88,9 +88,10 @@ const ( Int32 Int64 Bool - Intf // interface{} - Time // time.Time - Ext // extension + Intf // interface{} + Time // time.Time + Duration // time.Duration + Ext // extension IDENT // IDENT means an unrecognized identifier ) @@ -119,6 +120,7 @@ var primitives = map[string]Primitive{ "bool": Bool, "interface{}": Intf, "time.Time": Time, + "time.Duration": Duration, "msgp.Extension": Ext, } @@ -126,8 +128,8 @@ var primitives = map[string]Primitive{ // that satisfy all of the // interfaces. var builtins = map[string]struct{}{ - "msgp.Raw": struct{}{}, - "msgp.Number": struct{}{}, + "msgp.Raw": {}, + "msgp.Number": {}, } // common data/methods for every Elem @@ -137,6 +139,7 @@ func (c *common) SetVarname(s string) { c.vname = s } func (c *common) Varname() string { return c.vname } func (c *common) Alias(typ string) { c.alias = typ } func (c *common) hidden() {} +func (c *common) AllowNil() bool { return false } func IsPrintable(e Elem) bool { if be, ok := e.(*BaseElem); ok && !be.Printable() { @@ -183,11 +186,17 @@ type Elem interface { // Returns "" if zero/empty not supported for this Elem. ZeroExpr() string - // IfZeroExpr returns the expression to compare to zero/empty - // for this type. It is meant to be used in an if statement + // AllowNil will return true for types that can be nil but doesn't automatically check. + // This is true for slices and maps. + AllowNil() bool + + // IfZeroExpr returns the expression to compare to an empty value + // for this type, per the rules of the `omitempty` feature. + // It is meant to be used in an if statement // and may include the simple statement form followed by // semicolon and then the expression. // Returns "" if zero/empty not supported for this Elem. + // Note that this is NOT used by the `omitzero` feature. IfZeroExpr() string hidden() @@ -251,9 +260,10 @@ func (a *Array) IfZeroExpr() string { return "" } // Map is a map[string]Elem type Map struct { common - Keyidx string // key variable name - Validx string // value variable name - Value Elem // value element + Keyidx string // key variable name + Validx string // value variable name + Value Elem // value element + isAllowNil bool } func (m *Map) SetVarname(s string) { @@ -292,10 +302,17 @@ func (m *Map) ZeroExpr() string { return "nil" } // IfZeroExpr returns the expression to compare to zero/empty. func (m *Map) IfZeroExpr() string { return m.Varname() + " == nil" } +// AllowNil is true for maps. +func (m *Map) AllowNil() bool { return true } + +// SetIsAllowNil sets whether the map is allowed to be nil. +func (m *Map) SetIsAllowNil(b bool) { m.isAllowNil = b } + type Slice struct { common - Index string - Els Elem // The type of each element + Index string + isAllowNil bool + Els Elem // The type of each element } func (s *Slice) SetVarname(a string) { @@ -333,6 +350,22 @@ func (s *Slice) ZeroExpr() string { return "nil" } // IfZeroExpr returns the expression to compare to zero/empty. func (s *Slice) IfZeroExpr() string { return s.Varname() + " == nil" } +// AllowNil is true for slices. +func (s *Slice) AllowNil() bool { return true } + +// SetIsAllowNil sets whether the slice is allowed to be nil. +func (s *Slice) SetIsAllowNil(b bool) { s.isAllowNil = b } + +// SetIsAllowNil will set whether the element is allowed to be nil. +func SetIsAllowNil(e Elem, b bool) { + type i interface { + SetIsAllowNil(b bool) + } + if x, ok := e.(i); ok { + x.SetIsAllowNil(b) + } +} + type Ptr struct { common Value Elem @@ -517,6 +550,8 @@ func (s *BaseElem) Alias(typ string) { } } +func (s *BaseElem) AllowNil() bool { return s.Value == Bytes } + func (s *BaseElem) SetVarname(a string) { // extensions whose parents // are not pointers need to @@ -562,11 +597,14 @@ func (s *BaseElem) FromBase() string { // BaseName returns the string form of the // base type (e.g. Float64, Ident, etc) func (s *BaseElem) BaseName() string { - // time is a special case; + // time.Time and time.Duration are special cases; // we strip the package prefix if s.Value == Time { return "Time" } + if s.Value == Duration { + return "Duration" + } return s.Value.String() } @@ -583,6 +621,8 @@ func (s *BaseElem) BaseType() string { return "[]byte" case Time: return "time.Time" + case Duration: + return "time.Duration" case Ext: return "msgp.Extension" @@ -626,7 +666,6 @@ func (s *BaseElem) Resolved() bool { // ZeroExpr returns the zero/empty expression or empty string if not supported. func (s *BaseElem) ZeroExpr() string { - switch s.Value { case Bytes: return "nil" @@ -646,7 +685,8 @@ func (s *BaseElem) ZeroExpr() string { Int8, Int16, Int32, - Int64: + Int64, + Duration: return "0" case Bool: return "false" @@ -710,6 +750,8 @@ func (k Primitive) String() string { return "Intf" case Time: return "time.Time" + case Duration: + return "time.Duration" case Ext: return "Extension" case IDENT: @@ -733,7 +775,6 @@ func writeStructFields(s []StructField, name string) { // ArrayHeader implementation in this library using uint32. On the Go side, we // can declare array lengths as any constant integer width, which breaks when // attempting a direct comparison to an array header's uint32. -// func coerceArraySize(asz string) string { return fmt.Sprintf("uint32(%s)", asz) } diff --git a/gen/encode.go b/gen/encode.go index 6ec14916..b9997f8f 100644 --- a/gen/encode.go +++ b/gen/encode.go @@ -78,7 +78,6 @@ func (e *encodeGen) gStruct(s *Struct) { } else { e.structmap(s) } - return } func (e *encodeGen) tuple(s *Struct) { @@ -93,9 +92,20 @@ func (e *encodeGen) tuple(s *Struct) { if !e.p.ok() { return } + fieldElem := s.Fields[i].FieldElem + anField := s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() + if anField { + e.p.printf("\nif %s { // allownil: if nil", fieldElem.IfZeroExpr()) + e.p.printf("\nerr = en.WriteNil(); if err != nil { return; }") + e.p.printf("\n} else {") + } + SetIsAllowNil(fieldElem, anField) e.ctx.PushString(s.Fields[i].FieldName) next(e, s.Fields[i].FieldElem) e.ctx.Pop() + if anField { + e.p.print("\n}") // close if statement + } } } @@ -111,7 +121,6 @@ func (e *encodeGen) appendraw(bts []byte) { } func (e *encodeGen) structmap(s *Struct) { - oeIdentPrefix := randIdent() var data []byte @@ -122,14 +131,16 @@ func (e *encodeGen) structmap(s *Struct) { } omitempty := s.AnyHasTagPart("omitempty") + omitzero := s.AnyHasTagPart("omitzero") var fieldNVar string - if omitempty { + if omitempty || omitzero { fieldNVar = oeIdentPrefix + "Len" - e.p.printf("\n// omitempty: check for empty values") + e.p.printf("\n// check for omitted fields") e.p.printf("\n%s := uint32(%d)", fieldNVar, nfields) e.p.printf("\n%s", bm.typeDecl()) + e.p.printf("\n_ = %s", bm.varname) for i, sf := range s.Fields { if !e.p.ok() { return @@ -139,6 +150,11 @@ func (e *encodeGen) structmap(s *Struct) { e.p.printf("\n%s--", fieldNVar) e.p.printf("\n%s", bm.setStmt(i)) e.p.printf("\n}") + } else if sf.HasTagPart("omitzero") { + e.p.printf("\nif %s.IsZero() {", sf.FieldElem.Varname()) + e.p.printf("\n%s--", fieldNVar) + e.p.printf("\n%s", bm.setStmt(i)) + e.p.printf("\n}") } } @@ -156,11 +172,12 @@ func (e *encodeGen) structmap(s *Struct) { } else { - // non-omitempty version + // non-omit version data = msgp.AppendMapHeader(nil, uint32(nfields)) e.p.printf("\n// map header, size %d", nfields) e.Fuse(data) if len(s.Fields) == 0 { + e.p.printf("\n_ = %s", s.vname) e.fuseHook() } @@ -171,22 +188,32 @@ func (e *encodeGen) structmap(s *Struct) { return } - // if field is omitempty, wrap with if statement based on the emptymask - oeField := s.Fields[i].HasTagPart("omitempty") && s.Fields[i].FieldElem.IfZeroExpr() != "" + // if field is omitempty or omitzero, wrap with if statement based on the emptymask + oeField := (omitempty || omitzero) && + ((s.Fields[i].HasTagPart("omitempty") && s.Fields[i].FieldElem.IfZeroExpr() != "") || + s.Fields[i].HasTagPart("omitzero")) if oeField { - e.p.printf("\nif %s == 0 { // if not empty", bm.readExpr(i)) + e.p.printf("\nif %s == 0 { // if not omitted", bm.readExpr(i)) } data = msgp.AppendString(nil, s.Fields[i].FieldTag) e.p.printf("\n// write %q", s.Fields[i].FieldTag) e.Fuse(data) e.fuseHook() + fieldElem := s.Fields[i].FieldElem + anField := !oeField && s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() + if anField { + e.p.printf("\nif %s { // allownil: if nil", s.Fields[i].FieldElem.IfZeroExpr()) + e.p.printf("\nerr = en.WriteNil(); if err != nil { return; }") + e.p.printf("\n} else {") + } + SetIsAllowNil(fieldElem, anField) e.ctx.PushString(s.Fields[i].FieldName) next(e, s.Fields[i].FieldElem) e.ctx.Pop() - if oeField { + if oeField || anField { e.p.print("\n}") // close if statement } diff --git a/gen/marshal.go b/gen/marshal.go index 979ec725..90d09ec8 100644 --- a/gen/marshal.go +++ b/gen/marshal.go @@ -84,7 +84,6 @@ func (m *marshalGen) gStruct(s *Struct) { } else { m.mapstruct(s) } - return } func (m *marshalGen) tuple(s *Struct) { @@ -99,14 +98,24 @@ func (m *marshalGen) tuple(s *Struct) { if !m.p.ok() { return } + fieldElem := s.Fields[i].FieldElem + anField := s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() + if anField { + m.p.printf("\nif %s { // allownil: if nil", fieldElem.IfZeroExpr()) + m.p.printf("\no = msgp.AppendNil(o)") + m.p.printf("\n} else {") + } m.ctx.PushString(s.Fields[i].FieldName) - next(m, s.Fields[i].FieldElem) + SetIsAllowNil(fieldElem, anField) + next(m, fieldElem) m.ctx.Pop() + if anField { + m.p.printf("\n}") // close if statement + } } } func (m *marshalGen) mapstruct(s *Struct) { - oeIdentPrefix := randIdent() var data []byte @@ -117,14 +126,16 @@ func (m *marshalGen) mapstruct(s *Struct) { } omitempty := s.AnyHasTagPart("omitempty") + omitzero := s.AnyHasTagPart("omitzero") var fieldNVar string - if omitempty { + if omitempty || omitzero { fieldNVar = oeIdentPrefix + "Len" - m.p.printf("\n// omitempty: check for empty values") + m.p.printf("\n// check for omitted fields") m.p.printf("\n%s := uint32(%d)", fieldNVar, nfields) m.p.printf("\n%s", bm.typeDecl()) + m.p.printf("\n_ = %s", bm.varname) for i, sf := range s.Fields { if !m.p.ok() { return @@ -134,6 +145,11 @@ func (m *marshalGen) mapstruct(s *Struct) { m.p.printf("\n%s--", fieldNVar) m.p.printf("\n%s", bm.setStmt(i)) m.p.printf("\n}") + } else if sf.HasTagPart("omitzero") { + m.p.printf("\nif %s.IsZero() {", sf.FieldElem.Varname()) + m.p.printf("\n%s--", fieldNVar) + m.p.printf("\n%s", bm.setStmt(i)) + m.p.printf("\n}") } } @@ -156,6 +172,7 @@ func (m *marshalGen) mapstruct(s *Struct) { m.p.printf("\n// map header, size %d", len(s.Fields)) m.Fuse(data) if len(s.Fields) == 0 { + m.p.printf("\n_ = %s", s.vname) m.fuseHook() } @@ -166,10 +183,12 @@ func (m *marshalGen) mapstruct(s *Struct) { return } - // if field is omitempty, wrap with if statement based on the emptymask - oeField := s.Fields[i].HasTagPart("omitempty") && s.Fields[i].FieldElem.IfZeroExpr() != "" + // if field is omitempty or omitzero, wrap with if statement based on the emptymask + oeField := (omitempty || omitzero) && + ((s.Fields[i].HasTagPart("omitempty") && s.Fields[i].FieldElem.IfZeroExpr() != "") || + s.Fields[i].HasTagPart("omitzero")) if oeField { - m.p.printf("\nif %s == 0 { // if not empty", bm.readExpr(i)) + m.p.printf("\nif %s == 0 { // if not omitted", bm.readExpr(i)) } data = msgp.AppendString(nil, s.Fields[i].FieldTag) @@ -178,11 +197,19 @@ func (m *marshalGen) mapstruct(s *Struct) { m.Fuse(data) m.fuseHook() + fieldElem := s.Fields[i].FieldElem + anField := !oeField && s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() + if anField { + m.p.printf("\nif %s { // allownil: if nil", fieldElem.IfZeroExpr()) + m.p.printf("\no = msgp.AppendNil(o)") + m.p.printf("\n} else {") + } m.ctx.PushString(s.Fields[i].FieldName) - next(m, s.Fields[i].FieldElem) + SetIsAllowNil(fieldElem, anField) + next(m, fieldElem) m.ctx.Pop() - if oeField { + if oeField || anField { m.p.printf("\n}") // close if statement } diff --git a/gen/spec.go b/gen/spec.go index f3d5818b..bd57743c 100644 --- a/gen/spec.go +++ b/gen/spec.go @@ -62,25 +62,6 @@ func (m Method) String() string { } } -func strtoMeth(s string) Method { - switch s { - case "encode": - return Encode - case "decode": - return Decode - case "marshal": - return Marshal - case "unmarshal": - return Unmarshal - case "size": - return Size - case "test": - return Test - default: - return 0 - } -} - const ( Decode Method = 1 << iota // msgp.Decodable Encode // msgp.Encodable @@ -337,12 +318,12 @@ func (p *printer) declare(name string, typ string) { // does: // -// if m == nil { -// m = make(type, size) -// } else if len(m) > 0 { -// for key := range m { delete(m, key) } -// } +// if m == nil { +// m = make(type, size) +// } else if len(m) > 0 { // +// for key := range m { delete(m, key) } +// } func (p *printer) resizeMap(size string, m *Map) { vn := m.Varname() if !p.ok() { @@ -379,6 +360,13 @@ func (p *printer) resizeSlice(size string, s *Slice) { p.printf("\nif cap(%[1]s) >= int(%[2]s) { %[1]s = (%[1]s)[:%[2]s] } else { %[1]s = make(%[3]s, %[2]s) }", s.Varname(), size, s.TypeName()) } +// resizeSliceNoNil will resize a slice and will not allow nil slices. +func (p *printer) resizeSliceNoNil(size string, s *Slice) { + p.printf("\nif %[1]s != nil && cap(%[1]s) >= int(%[2]s) {", s.Varname(), size) + p.printf("\n%[1]s = (%[1]s)[:%[2]s]", s.Varname(), size) + p.printf("\n} else { %[1]s = make(%[3]s, %[2]s) }", s.Varname(), size, s.TypeName()) +} + func (p *printer) arrayCheck(want string, got string) { p.printf("\nif %[1]s != %[2]s { err = msgp.ArrayError{Wanted: %[2]s, Got: %[1]s}; return }", got, want) } @@ -387,10 +375,9 @@ func (p *printer) closeblock() { p.print("\n}") } // does: // -// for idx := range iter { -// {{generate inner}} -// } -// +// for idx := range iter { +// {{generate inner}} +// } func (p *printer) rangeBlock(ctx *Context, idx string, iter string, t traversal, inner Elem) { ctx.PushVar(idx) p.printf("\n for %s := range %s {", idx, iter) @@ -463,7 +450,6 @@ func (b *bmask) typeDecl() string { // typeName returns the type, e.g. "uint8" or "[2]uint64" func (b *bmask) typeName() string { - if b.bitlen <= 8 { return "uint8" } @@ -483,7 +469,6 @@ func (b *bmask) typeName() string { // readExpr returns the expression to read from a position in the bitmask. // Compare ==0 for false or !=0 for true. func (b *bmask) readExpr(bitoffset int) string { - if bitoffset < 0 || bitoffset >= b.bitlen { panic(fmt.Errorf("bitoffset %d out of range for bitlen %d", bitoffset, b.bitlen)) } @@ -500,12 +485,10 @@ func (b *bmask) readExpr(bitoffset int) string { buf.WriteByte(')') return buf.String() - } // setStmt returns the statement to set the specified bit in the bitmask. func (b *bmask) setStmt(bitoffset int) string { - var buf bytes.Buffer buf.Grow(len(b.varname) + 16) buf.WriteString(b.varname) @@ -515,5 +498,4 @@ func (b *bmask) setStmt(bitoffset int) string { fmt.Fprintf(&buf, " |= 0x%X", (uint64(1) << (uint64(bitoffset) % 64))) return buf.String() - } diff --git a/gen/testgen.go b/gen/testgen.go index a80151bc..0ed81245 100644 --- a/gen/testgen.go +++ b/gen/testgen.go @@ -178,5 +178,4 @@ func BenchmarkDecode{{.TypeName}}(b *testing.B) { } `)) - } diff --git a/gen/unmarshal.go b/gen/unmarshal.go index 99c0226b..c179208f 100644 --- a/gen/unmarshal.go +++ b/gen/unmarshal.go @@ -71,11 +71,9 @@ func (u *unmarshalGen) gStruct(s *Struct) { } else { u.mapstruct(s) } - return } func (u *unmarshalGen) tuple(s *Struct) { - // open block sz := randIdent() u.p.declare(sz, u32) @@ -86,8 +84,17 @@ func (u *unmarshalGen) tuple(s *Struct) { return } u.ctx.PushString(s.Fields[i].FieldName) - next(u, s.Fields[i].FieldElem) + fieldElem := s.Fields[i].FieldElem + anField := s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() + if anField { + u.p.printf("\nif msgp.IsNil(bts) {\nbts = bts[1:]\n%s = nil\n} else {", fieldElem.Varname()) + } + SetIsAllowNil(fieldElem, anField) + next(u, fieldElem) u.ctx.Pop() + if anField { + u.p.printf("\n}") + } } } @@ -107,8 +114,18 @@ func (u *unmarshalGen) mapstruct(s *Struct) { } u.p.printf("\ncase \"%s\":", s.Fields[i].FieldTag) u.ctx.PushString(s.Fields[i].FieldName) - next(u, s.Fields[i].FieldElem) + + fieldElem := s.Fields[i].FieldElem + anField := s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() + if anField { + u.p.printf("\nif msgp.IsNil(bts) {\nbts = bts[1:]\n%s = nil\n} else {", fieldElem.Varname()) + } + SetIsAllowNil(fieldElem, anField) + next(u, fieldElem) u.ctx.Pop() + if anField { + u.p.printf("\n}") + } } u.p.print("\ndefault:\nbts, err = msgp.Skip(bts)") u.p.wrapErrCheck(u.ctx.ArgsStr()) @@ -180,7 +197,11 @@ func (u *unmarshalGen) gSlice(s *Slice) { sz := randIdent() u.p.declare(sz, u32) u.assignAndCheck(sz, arrayHeader) - u.p.resizeSlice(sz, s) + if s.isAllowNil { + u.p.resizeSliceNoNil(sz, s) + } else { + u.p.resizeSlice(sz, s) + } u.p.rangeBlock(u.ctx, s.Index, s.Varname(), u, s.Els) } @@ -195,6 +216,10 @@ func (u *unmarshalGen) gMap(m *Map) { // allocate or clear map u.p.resizeMap(sz, m) + // We likely need a field. + // Add now to not be inside for scope. + u.needsField() + // loop and get key,value u.p.printf("\nfor %s > 0 {", sz) u.p.printf("\nvar %s string; var %s %s; %s--", m.Keyidx, m.Validx, m.Value.TypeName(), sz) diff --git a/go.mod b/go.mod index 7789dbc1..0bb76f0e 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,13 @@ module github.com/desertbit/msgp -go 1.14 +go 1.18 require ( - github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9 - golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9 + github.com/philhofer/fwd v1.1.2 + golang.org/x/tools v0.14.0 +) + +require ( + golang.org/x/mod v0.13.0 // indirect + golang.org/x/sys v0.13.0 // indirect ) diff --git a/go.sum b/go.sum index cd9a8a68..1cff9254 100644 --- a/go.sum +++ b/go.sum @@ -1,26 +1,8 @@ -github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9 h1:6ob53CVz+ja2i7easAStApZJlh7sxyq3Cm7g1Di6iqA= -github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9 h1:sEvmEcJVKBNUvgCUClbUQeHOAa9U0I2Ce1BooMvVCY4= -golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= +github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= diff --git a/issue185_test.go b/issue185_test.go index 8b06423f..57406dfa 100644 --- a/issue185_test.go +++ b/issue185_test.go @@ -25,9 +25,8 @@ const debugTemp = false // // structs are currently processed alphabetically by msgp. this test relies on // that property. -// func TestIssue185Idents(t *testing.T) { - var identCases = []struct { + identCases := []struct { tpl *template.Template expectedChanged []string }{ @@ -90,7 +89,7 @@ type issue185TplData struct { } func TestIssue185Overlap(t *testing.T) { - var overlapCases = []struct { + overlapCases := []struct { tpl *template.Template data issue185TplData }{ @@ -225,7 +224,7 @@ func extractVars(file string) (extractedVars, error) { } func goGenerateTpl(cwd, tfile string, tpl *template.Template, tplData interface{}) error { - outf, err := os.OpenFile(tfile, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) + outf, err := os.OpenFile(tfile, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o600) if err != nil { return err } diff --git a/main.go b/main.go index 8fc75935..a0ee5043 100644 --- a/main.go +++ b/main.go @@ -6,20 +6,19 @@ // To use it, include the following directive in a // go source file with types requiring source generation: // -// //go:generate msgp +// //go:generate msgp // // The go generate tool should set the proper environment variables for // the generator to execute without any command-line flags. However, the // following options are supported, if you need them: // -// -o = output file name (default is {input}_gen.go) -// -file = input file name (or directory; default is $GOFILE, which is set by the `go generate` command) -// -io = satisfy the `msgp.Decodable` and `msgp.Encodable` interfaces (default is true) -// -marshal = satisfy the `msgp.Marshaler` and `msgp.Unmarshaler` interfaces (default is true) -// -tests = generate tests and benchmarks (default is true) +// -o = output file name (default is {input}_gen.go) +// -file = input file name (or directory; default is $GOFILE, which is set by the `go generate` command) +// -io = satisfy the `msgp.Decodable` and `msgp.Encodable` interfaces (default is true) +// -marshal = satisfy the `msgp.Marshaler` and `msgp.Unmarshaler` interfaces (default is true) +// -tests = generate tests and benchmarks (default is true) // // For more information, please read README.md, and the wiki at github.com/desertbit/msgp -// package main import ( @@ -98,7 +97,6 @@ func main() { // Run writes all methods using the associated file or path, e.g. // // err := msgp.Run("path/to/myfile.go", gen.Size|gen.Marshal|gen.Unmarshal|gen.Test, false) -// func Run(gofile string, mode gen.Method, unexported bool) error { if mode&^gen.Test == 0 { return nil diff --git a/msgp/advise_linux.go b/msgp/advise_linux.go index 7b6049f4..d2a66857 100644 --- a/msgp/advise_linux.go +++ b/msgp/advise_linux.go @@ -1,3 +1,4 @@ +//go:build linux && !appengine && !tinygo // +build linux,!appengine,!tinygo package msgp diff --git a/msgp/advise_other.go b/msgp/advise_other.go index 15f2b1e1..07f524af 100644 --- a/msgp/advise_other.go +++ b/msgp/advise_other.go @@ -1,4 +1,5 @@ -// +build !linux,!tinygo appengine +//go:build (!linux && !tinygo && !windows) || appengine +// +build !linux,!tinygo,!windows appengine package msgp diff --git a/msgp/defs.go b/msgp/defs.go index 07c1da61..a35dd071 100644 --- a/msgp/defs.go +++ b/msgp/defs.go @@ -5,16 +5,19 @@ // generator implement the Marshaler/Unmarshaler and Encodable/Decodable interfaces. // // This package defines four "families" of functions: -// - AppendXxxx() appends an object to a []byte in MessagePack encoding. -// - ReadXxxxBytes() reads an object from a []byte and returns the remaining bytes. -// - (*Writer).WriteXxxx() writes an object to the buffered *Writer type. -// - (*Reader).ReadXxxx() reads an object from a buffered *Reader type. +// - AppendXxxx() appends an object to a []byte in MessagePack encoding. +// - ReadXxxxBytes() reads an object from a []byte and returns the remaining bytes. +// - (*Writer).WriteXxxx() writes an object to the buffered *Writer type. +// - (*Reader).ReadXxxx() reads an object from a buffered *Reader type. // // Once a type has satisfied the `Encodable` and `Decodable` interfaces, // it can be written and read from arbitrary `io.Writer`s and `io.Reader`s using -// msgp.Encode(io.Writer, msgp.Encodable) +// +// msgp.Encode(io.Writer, msgp.Encodable) +// // and -// msgp.Decode(io.Reader, msgp.Decodable) +// +// msgp.Decode(io.Reader, msgp.Decodable) // // There are also methods for converting MessagePack to JSON without // an explicit de-serialization step. @@ -23,11 +26,13 @@ // the wiki at http://github.com/desertbit/msgp package msgp -const last4 = 0x0f -const first4 = 0xf0 -const last5 = 0x1f -const first3 = 0xe0 -const last7 = 0x7f +const ( + last4 = 0x0f + first4 = 0xf0 + last5 = 0x1f + first3 = 0xe0 + last7 = 0x7f +) func isfixint(b byte) bool { return b>>7 == 0 diff --git a/msgp/edit_test.go b/msgp/edit_test.go index e33b4e1b..8f1cd621 100644 --- a/msgp/edit_test.go +++ b/msgp/edit_test.go @@ -81,7 +81,6 @@ func TestLocate(t *testing.T) { if len(field) != 0 { t.Fatalf("wanted a zero-length returned slice") } - } func TestReplace(t *testing.T) { diff --git a/msgp/elsize.go b/msgp/elsize.go index abc4b925..a05b0b21 100644 --- a/msgp/elsize.go +++ b/msgp/elsize.go @@ -1,7 +1,6 @@ package msgp func calcBytespec(v byte) bytespec { - // single byte values switch v { @@ -97,7 +96,6 @@ func calcBytespec(v byte) bytespec { // everything else is covered above return bytespec{} - } func getType(v byte) Type { @@ -120,11 +118,11 @@ type varmode int8 const ( constsize varmode = 0 // constant size (size bytes + uint8(varmode) objects) - extra8 = -1 // has uint8(p[1]) extra bytes - extra16 = -2 // has be16(p[1:]) extra bytes - extra32 = -3 // has be32(p[1:]) extra bytes - map16v = -4 // use map16 - map32v = -5 // use map32 - array16v = -6 // use array16 - array32v = -7 // use array32 + extra8 varmode = -1 // has uint8(p[1]) extra bytes + extra16 varmode = -2 // has be16(p[1:]) extra bytes + extra32 varmode = -3 // has be32(p[1:]) extra bytes + map16v varmode = -4 // use map16 + map32v varmode = -5 // use map32 + array16v varmode = -6 // use array16 + array32v varmode = -7 // use array32 ) diff --git a/msgp/elsize_default.go b/msgp/elsize_default.go index e1893a74..e7e8b547 100644 --- a/msgp/elsize_default.go +++ b/msgp/elsize_default.go @@ -1,3 +1,4 @@ +//go:build !tinygo // +build !tinygo package msgp @@ -6,7 +7,6 @@ package msgp // plus type information. gives us // constant-time type information // for traversing composite objects. -// var sizes [256]bytespec func init() { diff --git a/msgp/elsize_test.go b/msgp/elsize_test.go index e258fdeb..b300ee51 100644 --- a/msgp/elsize_test.go +++ b/msgp/elsize_test.go @@ -3,7 +3,6 @@ package msgp import "testing" func TestBytespec(t *testing.T) { - // verify that bytespec refactor for TinyGo behaves the same as the old code // previous sizes array setup verbatim: @@ -13,7 +12,7 @@ func TestBytespec(t *testing.T) { // constant-time type information // for traversing composite objects. // - var sizes = [256]bytespec{ + sizes := [256]bytespec{ mnil: {size: 1, extra: constsize, typ: NilType}, mfalse: {size: 1, extra: constsize, typ: BoolType}, mtrue: {size: 1, extra: constsize, typ: BoolType}, @@ -47,7 +46,6 @@ func TestBytespec(t *testing.T) { mmap32: {size: 5, extra: map32v, typ: MapType}, } - //func init() { // set up fixed fields // fixint @@ -75,11 +73,9 @@ func TestBytespec(t *testing.T) { for i := mfixarray; i < 0xa0; i++ { sizes[i] = bytespec{size: 1, extra: varmode(rfixarray(i)), typ: ArrayType} } - //} // compare all values to calcBytespec for i := 0; i < 256; i++ { - sizeb := sizes[byte(i)] cb := calcBytespec(byte(i)) if sizeb != cb { @@ -90,7 +86,5 @@ func TestBytespec(t *testing.T) { if i != 0xC1 && sizeb.size == 0 { t.Errorf("unexpected zero size for index 0x%x", i) } - } - } diff --git a/msgp/elsize_tinygo.go b/msgp/elsize_tinygo.go index 341640b2..041f4ad6 100644 --- a/msgp/elsize_tinygo.go +++ b/msgp/elsize_tinygo.go @@ -1,3 +1,4 @@ +//go:build tinygo // +build tinygo package msgp diff --git a/msgp/errors.go b/msgp/errors.go index fb1e109e..4f19359a 100644 --- a/msgp/errors.go +++ b/msgp/errors.go @@ -69,7 +69,6 @@ func Resumable(e error) bool { // // ErrShortBytes is not wrapped with any context due to backward compatibility // issues with the public API. -// func WrapError(err error, ctx ...interface{}) error { switch e := err.(type) { case errShort: @@ -310,7 +309,6 @@ func (e *ErrUnsupportedType) withContext(ctx string) error { // (unicode tables, needed for IsPrint(), are big). // It lives in errors.go just so we can test it in errors_test.go func simpleQuoteStr(s string) string { - const ( lowerhex = "0123456789abcdef" ) diff --git a/msgp/errors_default.go b/msgp/errors_default.go index d3a278d4..e45c00a8 100644 --- a/msgp/errors_default.go +++ b/msgp/errors_default.go @@ -1,3 +1,4 @@ +//go:build !tinygo // +build !tinygo package msgp diff --git a/msgp/errors_test.go b/msgp/errors_test.go index d78c2306..086b2f2e 100644 --- a/msgp/errors_test.go +++ b/msgp/errors_test.go @@ -90,7 +90,6 @@ func TestCauseShortByte(t *testing.T) { } func TestUnwrap(t *testing.T) { - // check errors that get wrapped for idx, err := range []error{ errors.New("test"), @@ -124,14 +123,11 @@ func TestUnwrap(t *testing.T) { if errors.Unwrap(cerr) != nil { t.Fatal() } - }) } - } func TestSimpleQuoteStr(t *testing.T) { - // check some cases for simpleQuoteStr type tcase struct { in string @@ -181,5 +177,4 @@ func TestSimpleQuoteStr(t *testing.T) { } }) } - } diff --git a/msgp/errors_tinygo.go b/msgp/errors_tinygo.go index a1d4a643..8691cd38 100644 --- a/msgp/errors_tinygo.go +++ b/msgp/errors_tinygo.go @@ -1,3 +1,4 @@ +//go:build tinygo // +build tinygo package msgp @@ -24,7 +25,6 @@ type stringer interface { } func ifToStr(i interface{}) string { - switch v := i.(type) { case stringer: return v.String() @@ -35,7 +35,6 @@ func ifToStr(i interface{}) string { default: return reflect.ValueOf(i).String() } - } func quoteStr(s string) string { diff --git a/msgp/extension.go b/msgp/extension.go index d6ca0f92..5f762473 100644 --- a/msgp/extension.go +++ b/msgp/extension.go @@ -31,7 +31,7 @@ var extensionReg = make(map[int8]func() Extension) // // For example, if you wanted to register a user-defined struct: // -// msgp.RegisterExtension(10, func() msgp.Extension { &MyExtension{} }) +// msgp.RegisterExtension(10, func() msgp.Extension { &MyExtension{} }) // // RegisterExtension will panic if you call it multiple times // with the same 'typ' argument, or if you use a reserved @@ -121,11 +121,8 @@ func (r *RawExtension) UnmarshalBinary(b []byte) error { return nil } -// WriteExtension writes an extension type to the writer -func (mw *Writer) WriteExtension(e Extension) error { - l := e.Len() - var err error - switch l { +func (mw *Writer) writeExtensionHeader(length int, extType int8) error { + switch length { case 0: o, err := mw.require(3) if err != nil { @@ -133,75 +130,88 @@ func (mw *Writer) WriteExtension(e Extension) error { } mw.buf[o] = mext8 mw.buf[o+1] = 0 - mw.buf[o+2] = byte(e.ExtensionType()) + mw.buf[o+2] = byte(extType) case 1: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext1 - mw.buf[o+1] = byte(e.ExtensionType()) + mw.buf[o+1] = byte(extType) case 2: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext2 - mw.buf[o+1] = byte(e.ExtensionType()) + mw.buf[o+1] = byte(extType) case 4: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext4 - mw.buf[o+1] = byte(e.ExtensionType()) + mw.buf[o+1] = byte(extType) case 8: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext8 - mw.buf[o+1] = byte(e.ExtensionType()) + mw.buf[o+1] = byte(extType) case 16: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext16 - mw.buf[o+1] = byte(e.ExtensionType()) + mw.buf[o+1] = byte(extType) default: switch { - case l < math.MaxUint8: + case length < math.MaxUint8: o, err := mw.require(3) if err != nil { return err } mw.buf[o] = mext8 - mw.buf[o+1] = byte(uint8(l)) - mw.buf[o+2] = byte(e.ExtensionType()) - case l < math.MaxUint16: + mw.buf[o+1] = byte(uint8(length)) + mw.buf[o+2] = byte(extType) + case length < math.MaxUint16: o, err := mw.require(4) if err != nil { return err } mw.buf[o] = mext16 - big.PutUint16(mw.buf[o+1:], uint16(l)) - mw.buf[o+3] = byte(e.ExtensionType()) + big.PutUint16(mw.buf[o+1:], uint16(length)) + mw.buf[o+3] = byte(extType) default: o, err := mw.require(6) if err != nil { return err } mw.buf[o] = mext32 - big.PutUint32(mw.buf[o+1:], uint32(l)) - mw.buf[o+5] = byte(e.ExtensionType()) + big.PutUint32(mw.buf[o+1:], uint32(length)) + mw.buf[o+5] = byte(extType) } } + + return nil +} + +// WriteExtension writes an extension type to the writer +func (mw *Writer) WriteExtension(e Extension) error { + length := e.Len() + + err := mw.writeExtensionHeader(length, e.ExtensionType()) + if err != nil { + return err + } + // we can only write directly to the // buffer if we're sure that it // fits the object - if l <= mw.bufsize() { - o, err := mw.require(l) + if length <= mw.bufsize() { + o, err := mw.require(length) if err != nil { return err } @@ -214,36 +224,50 @@ func (mw *Writer) WriteExtension(e Extension) error { if err != nil { return err } - buf := make([]byte, l) + buf := make([]byte, length) err = e.MarshalBinaryTo(buf) if err != nil { return err } mw.buf = buf - mw.wloc = l + mw.wloc = length + return nil +} + +// WriteExtensionRaw writes an extension type to the writer +func (mw *Writer) WriteExtensionRaw(extType int8, payload []byte) error { + if err := mw.writeExtensionHeader(len(payload), extType); err != nil { + return err + } + + // instead of using mw.Write(), we'll copy the data through the internal + // buffer, otherwise the payload would be moved to the heap + // (meaning we can use stack-allocated buffers with zero allocations) + for len(payload) > 0 { + chunkSize := mw.avail() + if chunkSize == 0 { + if err := mw.flush(); err != nil { + return err + } + chunkSize = mw.avail() + } + if chunkSize > len(payload) { + chunkSize = len(payload) + } + + mw.wloc += copy(mw.buf[mw.wloc:], payload[:chunkSize]) + payload = payload[chunkSize:] + } + return nil } // peek at the extension type, assuming the next // kind to be read is Extension func (m *Reader) peekExtensionType() (int8, error) { - p, err := m.R.Peek(2) - if err != nil { - return 0, err - } - spec := getBytespec(p[0]) - if spec.typ != ExtensionType { - return 0, badPrefix(ExtensionType, p[0]) - } - if spec.extra == constsize { - return int8(p[1]), nil - } - size := spec.size - p, err = m.R.Peek(int(size)) - if err != nil { - return 0, err - } - return int8(p[size-1]), nil + _, _, extType, err := m.peekExtensionHeader() + + return extType, err } // peekExtension peeks at the extension encoding type @@ -268,93 +292,40 @@ func peekExtension(b []byte) (int8, error) { return int8(b[size-1]), nil } -// ReadExtension reads the next object from the reader -// as an extension. ReadExtension will fail if the next -// object in the stream is not an extension, or if -// e.Type() is not the same as the wire type. -func (m *Reader) ReadExtension(e Extension) (err error) { +func (m *Reader) peekExtensionHeader() (offset int, length int, extType int8, err error) { var p []byte p, err = m.R.Peek(2) if err != nil { return } + + offset = 2 + lead := p[0] - var read int - var off int switch lead { case mfixext1: - if int8(p[1]) != e.ExtensionType() { - err = errExt(int8(p[1]), e.ExtensionType()) - return - } - p, err = m.R.Peek(3) - if err != nil { - return - } - err = e.UnmarshalBinary(p[2:]) - if err == nil { - _, err = m.R.Skip(3) - } + extType = int8(p[1]) + length = 1 return case mfixext2: - if int8(p[1]) != e.ExtensionType() { - err = errExt(int8(p[1]), e.ExtensionType()) - return - } - p, err = m.R.Peek(4) - if err != nil { - return - } - err = e.UnmarshalBinary(p[2:]) - if err == nil { - _, err = m.R.Skip(4) - } + extType = int8(p[1]) + length = 2 return case mfixext4: - if int8(p[1]) != e.ExtensionType() { - err = errExt(int8(p[1]), e.ExtensionType()) - return - } - p, err = m.R.Peek(6) - if err != nil { - return - } - err = e.UnmarshalBinary(p[2:]) - if err == nil { - _, err = m.R.Skip(6) - } + extType = int8(p[1]) + length = 4 return case mfixext8: - if int8(p[1]) != e.ExtensionType() { - err = errExt(int8(p[1]), e.ExtensionType()) - return - } - p, err = m.R.Peek(10) - if err != nil { - return - } - err = e.UnmarshalBinary(p[2:]) - if err == nil { - _, err = m.R.Skip(10) - } + extType = int8(p[1]) + length = 8 return case mfixext16: - if int8(p[1]) != e.ExtensionType() { - err = errExt(int8(p[1]), e.ExtensionType()) - return - } - p, err = m.R.Peek(18) - if err != nil { - return - } - err = e.UnmarshalBinary(p[2:]) - if err == nil { - _, err = m.R.Skip(18) - } + extType = int8(p[1]) + length = 16 return case mext8: @@ -362,51 +333,77 @@ func (m *Reader) ReadExtension(e Extension) (err error) { if err != nil { return } - if int8(p[2]) != e.ExtensionType() { - err = errExt(int8(p[2]), e.ExtensionType()) - return - } - read = int(uint8(p[1])) - off = 3 + offset = 3 + extType = int8(p[2]) + length = int(uint8(p[1])) case mext16: p, err = m.R.Peek(4) if err != nil { return } - if int8(p[3]) != e.ExtensionType() { - err = errExt(int8(p[3]), e.ExtensionType()) - return - } - read = int(big.Uint16(p[1:])) - off = 4 + offset = 4 + extType = int8(p[3]) + length = int(big.Uint16(p[1:])) case mext32: p, err = m.R.Peek(6) if err != nil { return } - if int8(p[5]) != e.ExtensionType() { - err = errExt(int8(p[5]), e.ExtensionType()) - return - } - read = int(big.Uint32(p[1:])) - off = 6 + offset = 6 + extType = int8(p[5]) + length = int(big.Uint32(p[1:])) default: err = badPrefix(ExtensionType, lead) return } - p, err = m.R.Peek(read + off) + return +} + +// ReadExtension reads the next object from the reader +// as an extension. ReadExtension will fail if the next +// object in the stream is not an extension, or if +// e.Type() is not the same as the wire type. +func (m *Reader) ReadExtension(e Extension) error { + offset, length, extType, err := m.peekExtensionHeader() if err != nil { - return + return err + } + + if expectedType := e.ExtensionType(); extType != expectedType { + return errExt(extType, expectedType) } - err = e.UnmarshalBinary(p[off:]) + + p, err := m.R.Peek(offset + length) + if err != nil { + return err + } + err = e.UnmarshalBinary(p[offset:]) if err == nil { - _, err = m.R.Skip(read + off) + // consume the peeked bytes + _, err = m.R.Skip(offset + length) } - return + return err +} + +// ReadExtensionRaw reads the next object from the reader +// as an extension. The returned slice is only +// valid until the next *Reader method call. +func (m *Reader) ReadExtensionRaw() (int8, []byte, error) { + offset, length, extType, err := m.peekExtensionHeader() + if err != nil { + return 0, nil, err + } + + payload, err := m.R.Next(offset + length) + if err != nil { + return 0, nil, err + } + + return extType, payload[offset:], nil } // AppendExtension appends a MessagePack extension to the provided slice diff --git a/msgp/extension_test.go b/msgp/extension_test.go index 5bf4273d..9daa2d62 100644 --- a/msgp/extension_test.go +++ b/msgp/extension_test.go @@ -18,19 +18,104 @@ func randomExt() RawExtension { func TestReadWriteExtension(t *testing.T) { rand.Seed(time.Now().Unix()) + var buf bytes.Buffer en := NewWriter(&buf) dc := NewReader(&buf) - for i := 0; i < 25; i++ { + t.Run("interface", func(t *testing.T) { + for i := 0; i < 25; i++ { + buf.Reset() + e := randomExt() + en.WriteExtension(&e) + en.Flush() + err := dc.ReadExtension(&e) + if err != nil { + t.Errorf("error with extension (length %d): %s", len(buf.Bytes()), err) + } + } + }) + + t.Run("raw", func(t *testing.T) { + for i := 0; i < 25; i++ { + buf.Reset() + e := randomExt() + en.WriteExtensionRaw(e.Type, e.Data) + en.Flush() + typ, payload, err := dc.ReadExtensionRaw() + if err != nil { + t.Errorf("error with extension (length %d): %s", len(buf.Bytes()), err) + } + if typ != e.Type || !bytes.Equal(payload, e.Data) { + t.Errorf("extension mismatch: %d %x != %d %x", typ, payload, e.Type, e.Data) + } + } + }) +} + +func TestReadWriteLargeExtensionRaw(t *testing.T) { + var buf bytes.Buffer + en := NewWriter(&buf) + dc := NewReader(&buf) + + largeExt := RawExtension{Type: int8(rand.Int()), Data: RandBytes(int(tuint32))} + + err := en.WriteExtensionRaw(largeExt.Type, largeExt.Data) + if err != nil { + t.Errorf("error with large extension write: %s", err) + } + // write a nil as a marker + err = en.WriteNil() + if err != nil { + t.Errorf("error with large extension write: %s", err) + } + en.Flush() + + typ, payload, err := dc.ReadExtensionRaw() + if err != nil { + t.Errorf("error with large extension read: %s", err) + } + if typ != largeExt.Type || !bytes.Equal(payload, largeExt.Data) { + t.Errorf("large extension mismatch: %d %x != %d %x", typ, payload, largeExt.Type, largeExt.Data) + } + err = dc.ReadNil() + if err != nil { + t.Errorf("error with large extension read: %s", err) + } +} + +func TestExtensionRawStackBuffer(t *testing.T) { + var buf bytes.Buffer + en := NewWriter(&buf) + dc := NewReader(&buf) + + const bufSize = 128 + e := &RawExtension{Type: int8(rand.Int()), Data: RandBytes(bufSize)} + + allocs := testing.AllocsPerRun(100, func() { buf.Reset() - e := randomExt() - en.WriteExtension(&e) + + var staticBuf [bufSize]byte + slc := e.Data[:rand.Intn(bufSize)] + copy(staticBuf[:], slc) + + err := en.WriteExtensionRaw(e.Type, staticBuf[:len(slc)]) + if err != nil { + t.Errorf("error writing extension: %s", err) + } en.Flush() - err := dc.ReadExtension(&e) + + typ, payload, err := dc.ReadExtensionRaw() if err != nil { - t.Errorf("error with extension (length %d): %s", len(buf.Bytes()), err) + t.Errorf("error reading extension: %s", err) } + if typ != e.Type || !bytes.Equal(payload, slc) { + t.Errorf("extension mismatch: %d %x != %d %x", typ, payload, e.Type, slc) + } + }) + + if allocs != 0 { + t.Errorf("using stack allocated buffer with WriteExtensionRaw caused %f allocations", allocs) } } @@ -72,3 +157,55 @@ func TestAppendAndWriteCompatibility(t *testing.T) { } } } + +func BenchmarkExtensionReadWrite(b *testing.B) { + var buf bytes.Buffer + en := NewWriter(&buf) + dc := NewReader(&buf) + + b.Run("interface", func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + buf.Reset() + + e := randomExt() + err := en.WriteExtension(&e) + if err != nil { + b.Errorf("error writing extension: %s", err) + } + en.Flush() + + err = dc.ReadExtension(&e) + if err != nil { + b.Errorf("error reading extension: %s", err) + } + } + }) + + b.Run("raw", func(b *testing.B) { + // this should have zero allocations + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + buf.Reset() + + e := randomExt() + err := en.WriteExtensionRaw(e.Type, e.Data) + if err != nil { + b.Errorf("error writing extension: %s", err) + } + en.Flush() + + typ, payload, err := dc.ReadExtensionRaw() + if err != nil { + b.Errorf("error reading extension: %s", err) + } + if typ != e.Type || !bytes.Equal(payload, e.Data) { + b.Errorf("extension mismatch: %d %x != %d %x", typ, payload, e.Type, e.Data) + } + + buf.Reset() + } + }) +} diff --git a/msgp/file.go b/msgp/file.go index 54586ba9..a6d91ede 100644 --- a/msgp/file.go +++ b/msgp/file.go @@ -1,5 +1,7 @@ -// +build linux darwin dragonfly freebsd netbsd openbsd -// +build !appengine,!tinygo +//go:build (linux || darwin || dragonfly || freebsd || illumos || netbsd || openbsd) && !appengine && !tinygo +// +build linux darwin dragonfly freebsd illumos netbsd openbsd +// +build !appengine +// +build !tinygo package msgp @@ -20,7 +22,6 @@ import ( // is only efficient for large files; small // files are best read and written using // the ordinary streaming interfaces. -// func ReadFile(dst Unmarshaler, file *os.File) error { stat, err := file.Stat() if err != nil { diff --git a/msgp/file_port.go b/msgp/file_port.go index 3e33b1f6..2bbb3ad1 100644 --- a/msgp/file_port.go +++ b/msgp/file_port.go @@ -1,3 +1,4 @@ +//go:build windows || appengine || tinygo // +build windows appengine tinygo package msgp diff --git a/msgp/file_test.go b/msgp/file_test.go index fb58c6cb..5f580901 100644 --- a/msgp/file_test.go +++ b/msgp/file_test.go @@ -1,10 +1,12 @@ -// +build linux darwin dragonfly freebsd netbsd openbsd +//go:build linux || darwin || dragonfly || freebsd || illumos || netbsd || openbsd +// +build linux darwin dragonfly freebsd illumos netbsd openbsd package msgp_test import ( "bytes" "crypto/rand" + "io" prand "math/rand" "os" "testing" @@ -49,7 +51,7 @@ func TestReadWriteFile(t *testing.T) { } var out rawBytes - f.Seek(0, os.SEEK_SET) + f.Seek(0, io.SeekStart) err = msgp.ReadFile(&out, f) if err != nil { t.Fatal(err) @@ -60,16 +62,17 @@ func TestReadWriteFile(t *testing.T) { } } -var blobstrings = []string{"", "a string", "a longer string here!"} -var blobfloats = []float64{0.0, -1.0, 1.0, 3.1415926535} -var blobints = []int64{0, 1, -1, 80000, 1 << 30} -var blobbytes = [][]byte{[]byte{}, []byte("hello"), []byte("{\"is_json\":true,\"is_compact\":\"unable to determine\"}")} +var ( + blobstrings = []string{"", "a string", "a longer string here!"} + blobfloats = []float64{0.0, -1.0, 1.0, 3.1415926535} + blobints = []int64{0, 1, -1, 80000, 1 << 30} + blobbytes = [][]byte{{}, []byte("hello"), []byte(`{"is_json":true,"is_compact":"unable to determine"}`)} +) func BenchmarkWriteReadFile(b *testing.B) { - // let's not run out of disk space... if b.N > 10000000 { - b.N = 10000000 + b.N = 10000000 //nolint:staticcheck // ignoring "SA3001: should not assign to b.N (staticcheck)" as this should not usually happen. } fname := "bench-tmpfile" diff --git a/msgp/json_bytes.go b/msgp/json_bytes.go index 438caf53..e6162d0a 100644 --- a/msgp/json_bytes.go +++ b/msgp/json_bytes.go @@ -12,7 +12,6 @@ import ( var unfuns [_maxtype]func(jsWriter, []byte, []byte) ([]byte, []byte, error) func init() { - // NOTE(pmh): this is best expressed as a jump table, // but gc doesn't do that yet. revisit post-go1.5. unfuns = [_maxtype]func(jsWriter, []byte, []byte) ([]byte, []byte, error){ @@ -223,27 +222,6 @@ func rwUintBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) return msg, scratch, err } -func rwFloatBytes(w jsWriter, msg []byte, f64 bool, scratch []byte) ([]byte, []byte, error) { - var f float64 - var err error - var sz int - if f64 { - sz = 64 - f, msg, err = ReadFloat64Bytes(msg) - } else { - sz = 32 - var v float32 - v, msg, err = ReadFloat32Bytes(msg) - f = float64(v) - } - if err != nil { - return msg, scratch, err - } - scratch = strconv.AppendFloat(scratch, f, 'f', -1, sz) - _, err = w.Write(scratch) - return msg, scratch, err -} - func rwFloat32Bytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { var f float32 var err error diff --git a/msgp/json_bytes_test.go b/msgp/json_bytes_test.go index 726974ab..4688c77a 100644 --- a/msgp/json_bytes_test.go +++ b/msgp/json_bytes_test.go @@ -72,7 +72,6 @@ func TestUnmarshalJSON(t *testing.T) { } else { t.Error(`can't type-assert "c" to map[string]interface{}`) } - } t.Logf("JSON: %s", js.Bytes()) diff --git a/msgp/number.go b/msgp/number.go index ad07ef99..edfe328b 100644 --- a/msgp/number.go +++ b/msgp/number.go @@ -29,7 +29,6 @@ type Number struct { // AsInt sets the number to an int64. func (n *Number) AsInt(i int64) { - // we always store int(0) // as {0, InvalidType} in // order to preserve diff --git a/msgp/number_test.go b/msgp/number_test.go index 640e30ba..f2e2f7ff 100644 --- a/msgp/number_test.go +++ b/msgp/number_test.go @@ -6,7 +6,6 @@ import ( ) func TestNumber(t *testing.T) { - n := Number{} if n.Type() != IntType { @@ -93,5 +92,4 @@ func TestNumber(t *testing.T) { if !bytes.Equal(dat, buf.Bytes()) { t.Errorf("encode: expected output %#v; got %#v", dat, buf.Bytes()) } - } diff --git a/msgp/purego.go b/msgp/purego.go index c828f7ec..2cd35c3e 100644 --- a/msgp/purego.go +++ b/msgp/purego.go @@ -1,3 +1,4 @@ +//go:build purego || appengine // +build purego appengine package msgp diff --git a/msgp/read.go b/msgp/read.go index e2b63c2b..e6d72f17 100644 --- a/msgp/read.go +++ b/msgp/read.go @@ -36,6 +36,7 @@ const ( IntType UintType NilType + DurationType ExtensionType // pseudo-types provided @@ -394,7 +395,7 @@ func (m *Reader) ReadMapKey(scratch []byte) ([]byte, error) { return out, nil } -// MapKeyPtr returns a []byte pointing to the contents +// ReadMapKeyPtr returns a []byte pointing to the contents // of a valid map key. The key cannot be empty, and it // must be shorter than the total buffer size of the // *Reader. Additionally, the returned slice is only @@ -559,6 +560,12 @@ func (m *Reader) ReadBool() (b bool, err error) { return } +// ReadDuration reads a time.Duration from the reader +func (m *Reader) ReadDuration() (d time.Duration, err error) { + i, err := m.ReadInt64() + return time.Duration(i), err +} + // ReadInt64 reads an int64 from the reader func (m *Reader) ReadInt64() (i int64, err error) { var p []byte @@ -1302,6 +1309,10 @@ func (m *Reader) ReadIntf() (i interface{}, err error) { i, err = m.ReadTime() return + case DurationType: + i, err = m.ReadDuration() + return + case ExtensionType: var t int8 t, err = m.peekExtensionType() diff --git a/msgp/read_bytes.go b/msgp/read_bytes.go index 2e575454..a204ac4b 100644 --- a/msgp/read_bytes.go +++ b/msgp/read_bytes.go @@ -12,7 +12,7 @@ var big = binary.BigEndian // NextType returns the type of the next // object in the slice. If the length // of the input is zero, it returns -// InvalidType. +// [InvalidType]. func NextType(b []byte) Type { if len(b) == 0 { return InvalidType @@ -55,7 +55,7 @@ func IsNil(b []byte) bool { // data without interpreting its contents. type Raw []byte -// MarshalMsg implements msgp.Marshaler. +// MarshalMsg implements [Marshaler]. // It appends the raw contents of 'raw' // to the provided byte slice. If 'raw' // is 0 bytes, 'nil' will be appended instead. @@ -69,7 +69,7 @@ func (r Raw) MarshalMsg(b []byte) ([]byte, error) { return o, nil } -// UnmarshalMsg implements msgp.Unmarshaler. +// UnmarshalMsg implements [Unmarshaler]. // It sets the contents of *Raw to be the next // object in the provided byte slice. func (r *Raw) UnmarshalMsg(b []byte) ([]byte, error) { @@ -91,7 +91,7 @@ func (r *Raw) UnmarshalMsg(b []byte) ([]byte, error) { return out, nil } -// EncodeMsg implements msgp.Encodable. +// EncodeMsg implements [Encodable]. // It writes the raw bytes to the writer. // If r is empty, it writes 'nil' instead. func (r Raw) EncodeMsg(w *Writer) error { @@ -102,7 +102,7 @@ func (r Raw) EncodeMsg(w *Writer) error { return err } -// DecodeMsg implements msgp.Decodable. +// DecodeMsg implements [Decodable]. // It sets the value of *Raw to be the // next object on the wire. func (r *Raw) DecodeMsg(f *Reader) error { @@ -114,7 +114,7 @@ func (r *Raw) DecodeMsg(f *Reader) error { return err } -// Msgsize implements msgp.Sizer +// Msgsize implements [Sizer]. func (r Raw) Msgsize() int { l := len(r) if l == 0 { @@ -144,7 +144,7 @@ func appendNext(f *Reader, d *[]byte) error { return nil } -// MarshalJSON implements json.Marshaler +// MarshalJSON implements [json.Marshaler]. func (r *Raw) MarshalJSON() ([]byte, error) { var buf bytes.Buffer _, err := UnmarshalAsJSON(&buf, []byte(*r)) @@ -153,9 +153,11 @@ func (r *Raw) MarshalJSON() ([]byte, error) { // ReadMapHeaderBytes reads a map header size // from 'b' and returns the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a map) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a map) func ReadMapHeaderBytes(b []byte) (sz uint32, o []byte, err error) { l := len(b) if l < 1 { @@ -197,9 +199,11 @@ func ReadMapHeaderBytes(b []byte) (sz uint32, o []byte, err error) { // ReadMapKeyZC attempts to read a map key // from 'b' and returns the key bytes and the remaining bytes +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a str or bin) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a str or bin) func ReadMapKeyZC(b []byte) ([]byte, []byte, error) { o, x, err := ReadStringZC(b) if err != nil { @@ -214,9 +218,11 @@ func ReadMapKeyZC(b []byte) ([]byte, []byte, error) { // ReadArrayHeaderBytes attempts to read // the array header size off of 'b' and return // the size and remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not an array) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not an array) func ReadArrayHeaderBytes(b []byte) (sz uint32, o []byte, err error) { if len(b) < 1 { return 0, nil, ErrShortBytes @@ -255,9 +261,11 @@ func ReadArrayHeaderBytes(b []byte) (sz uint32, o []byte, err error) { // ReadBytesHeader reads the 'bin' header size // off of 'b' and returns the size and remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a bin object) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a bin object) func ReadBytesHeader(b []byte) (sz uint32, o []byte, err error) { if len(b) < 1 { return 0, nil, ErrShortBytes @@ -295,10 +303,12 @@ func ReadBytesHeader(b []byte) (sz uint32, o []byte, err error) { // ReadNilBytes tries to read a "nil" byte // off of 'b' and return the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a 'nil') -// - InvalidPrefixError +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a 'nil') +// - [InvalidPrefixError] func ReadNilBytes(b []byte) ([]byte, error) { if len(b) < 1 { return nil, ErrShortBytes @@ -311,9 +321,11 @@ func ReadNilBytes(b []byte) ([]byte, error) { // ReadFloat64Bytes tries to read a float64 // from 'b' and return the value and the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a float64) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a float64) func ReadFloat64Bytes(b []byte) (f float64, o []byte, err error) { if len(b) < 9 { if len(b) >= 5 && b[0] == mfloat32 { @@ -344,9 +356,11 @@ func ReadFloat64Bytes(b []byte) (f float64, o []byte, err error) { // ReadFloat32Bytes tries to read a float64 // from 'b' and return the value and the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a float32) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a float32) func ReadFloat32Bytes(b []byte) (f float32, o []byte, err error) { if len(b) < 5 { err = ErrShortBytes @@ -365,9 +379,11 @@ func ReadFloat32Bytes(b []byte) (f float32, o []byte, err error) { // ReadBoolBytes tries to read a float64 // from 'b' and return the value and the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a bool) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a bool) func ReadBoolBytes(b []byte) (bool, []byte, error) { if len(b) < 1 { return false, b, ErrShortBytes @@ -382,11 +398,25 @@ func ReadBoolBytes(b []byte) (bool, []byte, error) { } } +// ReadDurationBytes tries to read a time.Duration +// from 'b' and return the value and the remaining bytes. +// +// Possible errors: +// +// - [ErrShortBytes] (too few bytes) +// - TypeError (not a int) +func ReadDurationBytes(b []byte) (d time.Duration, o []byte, err error) { + i, o, err := ReadInt64Bytes(b) + return time.Duration(i), o, err +} + // ReadInt64Bytes tries to read an int64 // from 'b' and return the value and the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError (not a int) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a int) func ReadInt64Bytes(b []byte) (i int64, o []byte, err error) { l := len(b) if l < 1 { @@ -491,10 +521,12 @@ func ReadInt64Bytes(b []byte) (i int64, o []byte, err error) { // ReadInt32Bytes tries to read an int32 // from 'b' and return the value and the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a int) -// - IntOverflow{} (value doesn't fit in int32) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a int) +// - [IntOverflow] (value doesn't fit in int32) func ReadInt32Bytes(b []byte) (int32, []byte, error) { i, o, err := ReadInt64Bytes(b) if i > math.MaxInt32 || i < math.MinInt32 { @@ -505,10 +537,12 @@ func ReadInt32Bytes(b []byte) (int32, []byte, error) { // ReadInt16Bytes tries to read an int16 // from 'b' and return the value and the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a int) -// - IntOverflow{} (value doesn't fit in int16) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a int) +// - [IntOverflow] (value doesn't fit in int16) func ReadInt16Bytes(b []byte) (int16, []byte, error) { i, o, err := ReadInt64Bytes(b) if i > math.MaxInt16 || i < math.MinInt16 { @@ -519,10 +553,12 @@ func ReadInt16Bytes(b []byte) (int16, []byte, error) { // ReadInt8Bytes tries to read an int16 // from 'b' and return the value and the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a int) -// - IntOverflow{} (value doesn't fit in int8) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a int) +// - [IntOverflow] (value doesn't fit in int8) func ReadInt8Bytes(b []byte) (int8, []byte, error) { i, o, err := ReadInt64Bytes(b) if i > math.MaxInt8 || i < math.MinInt8 { @@ -533,10 +569,12 @@ func ReadInt8Bytes(b []byte) (int8, []byte, error) { // ReadIntBytes tries to read an int // from 'b' and return the value and the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a int) -// - IntOverflow{} (value doesn't fit in int; 32-bit platforms only) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a int) +// - [IntOverflow] (value doesn't fit in int; 32-bit platforms only) func ReadIntBytes(b []byte) (int, []byte, error) { if smallint { i, b, err := ReadInt32Bytes(b) @@ -548,9 +586,11 @@ func ReadIntBytes(b []byte) (int, []byte, error) { // ReadUint64Bytes tries to read a uint64 // from 'b' and return the value and the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a uint) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a uint) func ReadUint64Bytes(b []byte) (u uint64, o []byte, err error) { l := len(b) if l < 1 { @@ -669,10 +709,12 @@ func ReadUint64Bytes(b []byte) (u uint64, o []byte, err error) { // ReadUint32Bytes tries to read a uint32 // from 'b' and return the value and the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a uint) -// - UintOverflow{} (value too large for uint32) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a uint) +// - [UintOverflow] (value too large for uint32) func ReadUint32Bytes(b []byte) (uint32, []byte, error) { v, o, err := ReadUint64Bytes(b) if v > math.MaxUint32 { @@ -683,10 +725,12 @@ func ReadUint32Bytes(b []byte) (uint32, []byte, error) { // ReadUint16Bytes tries to read a uint16 // from 'b' and return the value and the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a uint) -// - UintOverflow{} (value too large for uint16) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a uint) +// - [UintOverflow] (value too large for uint16) func ReadUint16Bytes(b []byte) (uint16, []byte, error) { v, o, err := ReadUint64Bytes(b) if v > math.MaxUint16 { @@ -697,10 +741,12 @@ func ReadUint16Bytes(b []byte) (uint16, []byte, error) { // ReadUint8Bytes tries to read a uint8 // from 'b' and return the value and the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a uint) -// - UintOverflow{} (value too large for uint8) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a uint) +// - [UintOverflow] (value too large for uint8) func ReadUint8Bytes(b []byte) (uint8, []byte, error) { v, o, err := ReadUint64Bytes(b) if v > math.MaxUint8 { @@ -711,10 +757,12 @@ func ReadUint8Bytes(b []byte) (uint8, []byte, error) { // ReadUintBytes tries to read a uint // from 'b' and return the value and the remaining bytes. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a uint) -// - UintOverflow{} (value too large for uint; 32-bit platforms only) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a uint) +// - [UintOverflow] (value too large for uint; 32-bit platforms only) func ReadUintBytes(b []byte) (uint, []byte, error) { if smallint { u, b, err := ReadUint32Bytes(b) @@ -732,9 +780,11 @@ func ReadByteBytes(b []byte) (byte, []byte, error) { // ReadBytesBytes reads a 'bin' object // from 'b' and returns its vaue and // the remaining bytes in 'b'. +// // Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a 'bin' object) +// +// - [ErrShortBytes] (too few bytes) +// - [TypeError] (not a 'bin' object) func ReadBytesBytes(b []byte, scratch []byte) (v []byte, o []byte, err error) { return readBytesBytes(b, scratch, false) } @@ -803,9 +853,11 @@ func readBytesBytes(b []byte, scratch []byte, zc bool) (v []byte, o []byte, err // ReadBytesZC extracts the messagepack-encoded // binary field without copying. The returned []byte // points to the same memory as the input slice. +// // Possible errors: -// - ErrShortBytes (b not long enough) -// - TypeError{} (object not 'bin') +// +// - [ErrShortBytes] (b not long enough) +// - [TypeError] (object not 'bin') func ReadBytesZC(b []byte) (v []byte, o []byte, err error) { return readBytesBytes(b, nil, true) } @@ -863,9 +915,11 @@ func ReadExactBytes(b []byte, into []byte) (o []byte, err error) { // ReadStringZC reads a messagepack string field // without copying. The returned []byte points // to the same memory as the input slice. +// // Possible errors: -// - ErrShortBytes (b not long enough) -// - TypeError{} (object not 'str') +// +// - [ErrShortBytes] (b not long enough) +// - [TypeError] (object not 'str') func ReadStringZC(b []byte) (v []byte, o []byte, err error) { l := len(b) if l < 1 { @@ -923,10 +977,12 @@ func ReadStringZC(b []byte) (v []byte, o []byte, err error) { // ReadStringBytes reads a 'str' object // from 'b' and returns its value and the // remaining bytes in 'b'. +// // Possible errors: -// - ErrShortBytes (b not long enough) -// - TypeError{} (not 'str' type) -// - InvalidPrefixError +// +// - [ErrShortBytes] (b not long enough) +// - [TypeError] (not 'str' type) +// - [InvalidPrefixError] func ReadStringBytes(b []byte) (string, []byte, error) { v, o, err := ReadStringZC(b) return string(v), o, err @@ -936,11 +992,13 @@ func ReadStringBytes(b []byte) (string, []byte, error) { // into a slice of bytes. 'v' is the value of // the 'str' object, which may reside in memory // pointed to by 'scratch.' 'o' is the remaining bytes -// in 'b.'' +// in 'b'. +// // Possible errors: -// - ErrShortBytes (b not long enough) -// - TypeError{} (not 'str' type) -// - InvalidPrefixError (unknown type marker) +// +// - [ErrShortBytes] (b not long enough) +// - [TypeError] (not 'str' type) +// - [InvalidPrefixError] (unknown type marker) func ReadStringAsBytes(b []byte, scratch []byte) (v []byte, o []byte, err error) { var tmp []byte tmp, o, err = ReadStringZC(b) @@ -951,11 +1009,13 @@ func ReadStringAsBytes(b []byte, scratch []byte) (v []byte, o []byte, err error) // ReadComplex128Bytes reads a complex128 // extension object from 'b' and returns the // remaining bytes. +// // Possible errors: -// - ErrShortBytes (not enough bytes in 'b') -// - TypeError{} (object not a complex128) -// - InvalidPrefixError -// - ExtensionTypeError{} (object an extension of the correct size, but not a complex128) +// +// - [ErrShortBytes] (not enough bytes in 'b') +// - [TypeError] (object not a complex128) +// - [InvalidPrefixError] +// - [ExtensionTypeError] (object an extension of the correct size, but not a complex128) func ReadComplex128Bytes(b []byte) (c complex128, o []byte, err error) { if len(b) < 18 { err = ErrShortBytes @@ -978,10 +1038,12 @@ func ReadComplex128Bytes(b []byte) (c complex128, o []byte, err error) { // ReadComplex64Bytes reads a complex64 // extension object from 'b' and returns the // remaining bytes. +// // Possible errors: -// - ErrShortBytes (not enough bytes in 'b') -// - TypeError{} (object not a complex64) -// - ExtensionTypeError{} (object an extension of the correct size, but not a complex64) +// +// - [ErrShortBytes] (not enough bytes in 'b') +// - [TypeError] (object not a complex64) +// - [ExtensionTypeError] (object an extension of the correct size, but not a complex64) func ReadComplex64Bytes(b []byte) (c complex64, o []byte, err error) { if len(b) < 10 { err = ErrShortBytes @@ -1004,10 +1066,12 @@ func ReadComplex64Bytes(b []byte) (c complex64, o []byte, err error) { // ReadTimeBytes reads a time.Time // extension object from 'b' and returns the // remaining bytes. +// // Possible errors: -// - ErrShortBytes (not enough bytes in 'b') -// - TypeError{} (object not a complex64) -// - ExtensionTypeError{} (object an extension of the correct size, but not a time.Time) +// +// - [ErrShortBytes] (not enough bytes in 'b') +// - [TypeError] (object not a complex64) +// - [ExtensionTypeError] (object an extension of the correct size, but not a time.Time) func ReadTimeBytes(b []byte) (t time.Time, o []byte, err error) { if len(b) < 15 { err = ErrShortBytes @@ -1176,9 +1240,11 @@ func ReadIntfBytes(b []byte) (i interface{}, o []byte, err error) { // returns the remaining bytes. If the object // is a map or array, all of its elements // will be skipped. -// Possible Errors: -// - ErrShortBytes (not enough bytes in b) -// - InvalidPrefixError (bad encoding) +// +// Possible errors: +// +// - [ErrShortBytes] (not enough bytes in b) +// - [InvalidPrefixError] (bad encoding) func Skip(b []byte) ([]byte, error) { sz, asz, err := getSize(b) if err != nil { diff --git a/msgp/read_bytes_test.go b/msgp/read_bytes_test.go index e1dde6b9..724a061a 100644 --- a/msgp/read_bytes_test.go +++ b/msgp/read_bytes_test.go @@ -238,7 +238,6 @@ func TestReadBoolBytes(t *testing.T) { en.WriteBool(v) en.Flush() out, left, err := ReadBoolBytes(buf.Bytes()) - if err != nil { t.Errorf("test case %d: %s", i, err) } @@ -271,8 +270,10 @@ func TestReadInt64Bytes(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) - ints := []int64{-100000, -5000, -5, 0, 8, 240, int64(tuint16), int64(tuint32), int64(tuint64), - -5, -30, 0, 1, 127, 300, 40921, 34908219} + ints := []int64{ + -100000, -5000, -5, 0, 8, 240, int64(tuint16), int64(tuint32), int64(tuint64), + -5, -30, 0, 1, 127, 300, 40921, 34908219, + } uints := []uint64{0, 8, 240, uint64(tuint16), uint64(tuint32), uint64(tuint64)} @@ -481,7 +482,7 @@ func TestReadBytesBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) - tests := [][]byte{[]byte{}, []byte("some bytes"), []byte("some more bytes")} + tests := [][]byte{{}, []byte("some bytes"), []byte("some more bytes")} var scratch []byte for i, v := range tests { @@ -505,7 +506,7 @@ func TestReadZCBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) - tests := [][]byte{[]byte{}, []byte("some bytes"), []byte("some more bytes")} + tests := [][]byte{{}, []byte("some bytes"), []byte("some more bytes")} for i, v := range tests { buf.Reset() @@ -721,7 +722,6 @@ func TestReadIntfBytes(t *testing.T) { t.Errorf("ReadIntf(): %v in; %v out", v, out) } } - } func BenchmarkSkipBytes(b *testing.B) { diff --git a/msgp/read_test.go b/msgp/read_test.go index ea9257f4..86099c85 100644 --- a/msgp/read_test.go +++ b/msgp/read_test.go @@ -24,12 +24,13 @@ func TestReadIntf(t *testing.T) { // always read out as int64, and // unsigned integers as uint64 - var testCases = []interface{}{ + testCases := []interface{}{ float64(128.032), float32(9082.092), int64(-40), uint64(9082981), time.Now(), + 48*time.Hour + 3*time.Minute + 2*time.Second + 3*time.Nanosecond, "hello!", []byte("hello!"), map[string]interface{}{ @@ -66,11 +67,16 @@ func TestReadIntf(t *testing.T) { if !tm.Equal(v.(time.Time)) { t.Errorf("%v != %v", ts, v) } + } else if intd, ok := ts.(time.Duration); ok { + /* for time.Duration, cast before comparing */ + outtd := time.Duration(v.(int64)) + if intd != outtd { + t.Errorf("%v in; %v out", intd, outtd) + } } else if !reflect.DeepEqual(v, ts) { t.Errorf("%v in; %v out", ts, v) } } - } func TestReadMapHeader(t *testing.T) { @@ -357,10 +363,7 @@ func TestReadIntOverflows(t *testing.T) { case UintOverflow: bits = err.FailedBitsize } - if bits == failBits { - return true - } - return false + return bits == failBits } belowZeroErr := func(err error, failBits int) bool { @@ -853,7 +856,6 @@ func TestSkip(t *testing.T) { t.Errorf("expected %q; got %q", io.EOF, err) t.Errorf("returned type %q", tp) } - } func BenchmarkSkip(b *testing.B) { diff --git a/msgp/size.go b/msgp/size.go index ce2f8b16..e3a613b2 100644 --- a/msgp/size.go +++ b/msgp/size.go @@ -25,9 +25,10 @@ const ( Complex64Size = 10 Complex128Size = 18 - TimeSize = 15 - BoolSize = 1 - NilSize = 1 + DurationSize = Int64Size + TimeSize = 15 + BoolSize = 1 + NilSize = 1 MapHeaderSize = 5 ArrayHeaderSize = 5 diff --git a/msgp/unsafe.go b/msgp/unsafe.go index d9fb3535..06e8d843 100644 --- a/msgp/unsafe.go +++ b/msgp/unsafe.go @@ -1,3 +1,4 @@ +//go:build !purego && !appengine // +build !purego,!appengine package msgp diff --git a/msgp/write.go b/msgp/write.go index f6ad9a05..ec2f6f52 100644 --- a/msgp/write.go +++ b/msgp/write.go @@ -356,6 +356,11 @@ func (mw *Writer) WriteFloat32(f float32) error { return mw.prefix32(mfloat32, math.Float32bits(f)) } +// WriteDuration writes a time.Duration to the writer +func (mw *Writer) WriteDuration(d time.Duration) error { + return mw.WriteInt64(int64(d)) +} + // WriteInt64 writes an int64 to the writer func (mw *Writer) WriteInt64(i int64) error { if i >= 0 { @@ -621,12 +626,12 @@ func (mw *Writer) WriteTime(t time.Time) error { // WriteIntf writes the concrete type of 'v'. // WriteIntf will error if 'v' is not one of the following: -// - A bool, float, string, []byte, int, uint, or complex -// - A map of supported types (with string keys) -// - An array or slice of supported types -// - A pointer to a supported type -// - A type that satisfies the msgp.Encodable interface -// - A type that satisfies the msgp.Extension interface +// - A bool, float, string, []byte, int, uint, or complex +// - A map of supported types (with string keys) +// - An array or slice of supported types +// - A pointer to a supported type +// - A type that satisfies the msgp.Encodable interface +// - A type that satisfies the msgp.Extension interface func (mw *Writer) WriteIntf(v interface{}) error { if v == nil { return mw.WriteNil() @@ -682,6 +687,8 @@ func (mw *Writer) WriteIntf(v interface{}) error { return mw.WriteMapStrIntf(v) case time.Time: return mw.WriteTime(v) + case time.Duration: + return mw.WriteDuration(v) } val := reflect.ValueOf(v) @@ -746,60 +753,6 @@ func (mw *Writer) writeSlice(v reflect.Value) (err error) { return } -func (mw *Writer) writeStruct(v reflect.Value) error { - if enc, ok := v.Interface().(Encodable); ok { - return enc.EncodeMsg(mw) - } - return errors.New("msgp: unsupported type: " + v.Type().String()) -} - -func (mw *Writer) writeVal(v reflect.Value) error { - if !isSupported(v.Kind()) { - return errors.New("msgp: msgp/enc: type not supported: " + v.Type().String()) - } - - // shortcut for nil values - if v.IsNil() { - return mw.WriteNil() - } - switch v.Kind() { - case reflect.Bool: - return mw.WriteBool(v.Bool()) - - case reflect.Float32, reflect.Float64: - return mw.WriteFloat64(v.Float()) - - case reflect.Complex64, reflect.Complex128: - return mw.WriteComplex128(v.Complex()) - - case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8: - return mw.WriteInt64(v.Int()) - - case reflect.Interface, reflect.Ptr: - if v.IsNil() { - mw.WriteNil() - } - return mw.writeVal(v.Elem()) - - case reflect.Map: - return mw.writeMap(v) - - case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint8: - return mw.WriteUint64(v.Uint()) - - case reflect.String: - return mw.WriteString(v.String()) - - case reflect.Slice, reflect.Array: - return mw.writeSlice(v) - - case reflect.Struct: - return mw.writeStruct(v) - - } - return errors.New("msgp: msgp/enc: type not supported: " + v.Type().String()) -} - // is the reflect.Kind encodable? func isSupported(k reflect.Kind) bool { switch k { diff --git a/msgp/write_bytes.go b/msgp/write_bytes.go index 93d6d764..8199ac28 100644 --- a/msgp/write_bytes.go +++ b/msgp/write_bytes.go @@ -1,6 +1,7 @@ package msgp import ( + "errors" "math" "reflect" "time" @@ -73,6 +74,11 @@ func AppendFloat32(b []byte, f float32) []byte { return o } +// AppendDuration appends a time.Duration to the slice +func AppendDuration(b []byte, d time.Duration) []byte { + return AppendInt64(b, int64(d)) +} + // AppendInt64 appends an int64 to the slice func AppendInt64(b []byte, i int64) []byte { if i >= 0 { @@ -335,13 +341,13 @@ func AppendMapStrIntf(b []byte, m map[string]interface{}) ([]byte, error) { // AppendIntf appends the concrete type of 'i' to the // provided []byte. 'i' must be one of the following: -// - 'nil' -// - A bool, float, string, []byte, int, uint, or complex -// - A map[string]interface{} or map[string]string -// - A []T, where T is another supported type -// - A *T, where T is another supported type -// - A type that satisfieds the msgp.Marshaler interface -// - A type that satisfies the msgp.Extension interface +// - 'nil' +// - A bool, float, string, []byte, int, uint, or complex +// - A map[string]T where T is another supported type +// - A []T, where T is another supported type +// - A *T, where T is another supported type +// - A type that satisfies the msgp.Marshaler interface +// - A type that satisfies the msgp.Extension interface func AppendIntf(b []byte, i interface{}) ([]byte, error) { if i == nil { return AppendNil(b), nil @@ -409,6 +415,21 @@ func AppendIntf(b []byte, i interface{}) ([]byte, error) { var err error v := reflect.ValueOf(i) switch v.Kind() { + case reflect.Map: + if v.Type().Key().Kind() != reflect.String { + return b, errors.New("msgp: map keys must be strings") + } + ks := v.MapKeys() + b = AppendMapHeader(b, uint32(len(ks))) + for _, key := range ks { + val := v.MapIndex(key) + b = AppendString(b, key.String()) + b, err = AppendIntf(b, val.Interface()) + if err != nil { + return nil, err + } + } + return b, nil case reflect.Array, reflect.Slice: l := v.Len() b = AppendArrayHeader(b, uint32(l)) diff --git a/msgp/write_bytes_test.go b/msgp/write_bytes_test.go index 69866fe1..93e13ff6 100644 --- a/msgp/write_bytes_test.go +++ b/msgp/write_bytes_test.go @@ -3,6 +3,8 @@ package msgp import ( "bytes" "math" + "reflect" + "strings" "testing" "time" ) @@ -348,3 +350,134 @@ func BenchmarkAppendTime(b *testing.B) { AppendTime(buf[0:0], t) } } + +// TestEncodeDecode does a back-and-forth test of encoding and decoding and compare the value with a given output. +func TestEncodeDecode(t *testing.T) { + for _, tc := range []struct { + name string + input interface{} + output interface{} + encodeError string + }{ + { + name: "nil", + input: nil, + }, + { + name: "bool", + input: true, + }, + { + name: "int", + input: int64(42), + }, + { + name: "float", + input: 3.14159, + }, + { + name: "string", + input: "hello", + }, + { + name: "bytes", + input: []byte("hello"), + }, + { + name: "array-empty", + input: []interface{}{}, + }, + { + name: "array", + input: []interface{}{int64(1), int64(2), int64(3)}, + }, + { + name: "map-empty", + input: map[string]interface{}{}, + }, + { + name: "map", + input: map[string]interface{}{"a": int64(1), "b": int64(2)}, + }, + { + name: "map-interface", + input: map[string]interface{}{"a": int64(1), "b": "2"}, + }, + { + name: "map-string", + input: map[string]string{"a": "1", "b": "2"}, + output: map[string]interface{}{"a": "1", "b": "2"}, + }, + { + name: "map-array", + input: map[string][]int64{"a": {1, 2}, "b": {3}}, + output: map[string]interface{}{"a": []interface{}{int64(1), int64(2)}, "b": []interface{}{int64(3)}}, + }, + { + name: "map-map", + input: map[string]map[string]int64{"a": {"a": 1, "b": 2}, "b": {"c": 3}}, + output: map[string]interface{}{"a": map[string]interface{}{"a": int64(1), "b": int64(2)}, "b": map[string]interface{}{"c": int64(3)}}, + }, + { + name: "array-map", + input: []interface{}{map[string]interface{}{"a": int64(1), "b": "2"}, map[string]int64{"c": 3}}, + output: []interface{}{map[string]interface{}{"a": int64(1), "b": "2"}, map[string]interface{}{"c": int64(3)}}, + }, + { + name: "array-array", + input: []interface{}{[]int64{1, 2}, []interface{}{int64(3)}}, + output: []interface{}{[]interface{}{int64(1), int64(2)}, []interface{}{int64(3)}}, + }, + { + name: "array-array-map", + input: []interface{}{[]interface{}{int64(1), int64(2)}, map[string]interface{}{"c": int64(3)}}, + }, + { + name: "map-array-map", + input: map[string]interface{}{"a": []interface{}{int64(1), int64(2)}, "b": map[string]interface{}{"c": int64(3)}}, + }, + { + name: "map-invalid-keys", + input: map[interface{}]interface{}{int64(1): int64(2)}, + encodeError: "msgp: map keys must be strings", + }, + { + name: "map-nested-invalid-keys", + input: map[string]interface{}{"a": map[int64]string{1: "2"}}, + encodeError: "msgp: map keys must be strings", + }, + { + name: "invalid-type", + input: struct{}{}, + encodeError: "msgp: type \"struct {}\" not supported", + }, + } { + t.Run(tc.name, func(t *testing.T) { + // If no output is given, use the input as output + if tc.output == nil { + tc.output = tc.input + } + + buf, err := AppendIntf(nil, tc.input) + if tc.encodeError != "" { + if err == nil || !strings.Contains(err.Error(), tc.encodeError) { + t.Fatalf("expected encode error '%s' but got '%s'", tc.encodeError, err) + } + return + } + + if tc.encodeError == "" && err != nil { + t.Fatalf("expected no encode error but got '%s'", err.Error()) + } + + out, _, _ := ReadIntfBytes(buf) + if err != nil { + t.Fatalf("expected no decode error but got '%s'", err.Error()) + } + + if !reflect.DeepEqual(tc.output, out) { + t.Fatalf("expected '%v' but got '%v'", tc.input, out) + } + }) + } +} diff --git a/msgp/write_test.go b/msgp/write_test.go index c5e97fe2..1b32a2e9 100644 --- a/msgp/write_test.go +++ b/msgp/write_test.go @@ -34,8 +34,10 @@ func TestWriteMapHeader(t *testing.T) { {0, []byte{mfixmap}}, {1, []byte{mfixmap | byte(1)}}, {100, []byte{mmap16, byte(uint16(100) >> 8), byte(uint16(100))}}, - {tuint32, - []byte{mmap32, + { + tuint32, + []byte{ + mmap32, byte(tuint32 >> 24), byte(tuint32 >> 16), byte(tuint32 >> 8), @@ -214,6 +216,39 @@ func TestWriteFloat64(t *testing.T) { } } +func TestReadWriterDuration(t *testing.T) { + var buf bytes.Buffer + wr := NewWriter(&buf) + + for i := 0; i < 10000; i++ { + buf.Reset() + dur := time.Duration(rand.Int63()) + err := wr.WriteDuration(dur) + if err != nil { + t.Errorf("Error with %v: %s", dur, err) + } + err = wr.Flush() + if err != nil { + t.Fatal(err) + } + + bts := buf.Bytes() + + if bts[0] != mint64 { + t.Errorf("Leading byte was %x and not %x", bts[0], mint64) + } + + wr := NewReader(&buf) + d, err := wr.ReadDuration() + if err != nil { + t.Errorf("Error reading duration: %v", err) + } + if d != dur { + t.Errorf("Got duration %v, want %v", d, dur) + } + } +} + func BenchmarkWriteFloat64(b *testing.B) { f := rand.Float64() wr := NewWriter(Nowhere) diff --git a/parse/getast.go b/parse/getast.go index 5eba89d9..b3e1f3ee 100644 --- a/parse/getast.go +++ b/parse/getast.go @@ -126,10 +126,10 @@ func (f *FileSet) applyDirectives() { // into just one level of indirection. // In other words, if we have: // -// type A uint64 -// type B A -// type C B -// type D C +// type A uint64 +// type B A +// type C B +// type D C // // ... then we want to end up // figuring out that D is just a uint64. @@ -164,7 +164,6 @@ func (f *FileSet) resolve(ls linkset) { // process takes the contents of f.Specs and // uses them to populate f.Identities func (f *FileSet) process() { - deferred := make(linkset) parse: for name, def := range f.Specs { @@ -268,23 +267,18 @@ func (f *FileSet) PrintTo(p *gen.Printer) error { // getTypeSpecs extracts all of the *ast.TypeSpecs in the file // into fs.Identities, but does not set the actual element func (fs *FileSet) getTypeSpecs(f *ast.File) { - // collect all imports... fs.Imports = append(fs.Imports, f.Imports...) // check all declarations... for i := range f.Decls { - // for GenDecls... if g, ok := f.Decls[i].(*ast.GenDecl); ok { - // and check the specs... for _, s := range g.Specs { - // for ast.TypeSpecs.... if ts, ok := s.(*ast.TypeSpec); ok { switch ts.Type.(type) { - // this is the list of parse-able // type specs case *ast.StructType, @@ -293,7 +287,6 @@ func (fs *FileSet) getTypeSpecs(f *ast.File) { *ast.MapType, *ast.Ident: fs.Specs[ts.Name.Name] = ts.Type - } } } @@ -432,9 +425,9 @@ func (fs *FileSet) getFieldsFromEmbeddedStruct(f ast.Expr) []gen.StructField { // // so, for a struct like // -// type A struct { -// io.Writer -// } +// type A struct { +// io.Writer +// } // // we want "Writer" func embedded(f ast.Expr) string { diff --git a/printer/print.go b/printer/print.go index 37b36e9b..e1792dea 100644 --- a/printer/print.go +++ b/printer/print.go @@ -52,7 +52,7 @@ func format(file string, data []byte) error { if err != nil { return err } - return ioutil.WriteFile(file, out, 0600) + return ioutil.WriteFile(file, out, 0o600) } func goformat(file string, data []byte) <-chan error { diff --git a/tinygotest/testdata/roundtrip/main.go b/tinygotest/testdata/roundtrip/main.go index 2fb417c5..6a941509 100644 --- a/tinygotest/testdata/roundtrip/main.go +++ b/tinygotest/testdata/roundtrip/main.go @@ -65,7 +65,6 @@ func (e *Example) Setup() { e.Omitted = "nope" e.OmitEmptyString = "here" - } func (e *Example) Eq(e2 *Example) bool { @@ -143,7 +142,6 @@ func (e *Example) Eq(e2 *Example) bool { var buf [256]byte func main() { - var e Example e.Setup() @@ -192,5 +190,4 @@ func main() { if bytes.Compare(wbuf.Bytes(), b1) != 0 { panic("writer and marshal produced different results") } - } diff --git a/tinygotest/testdata/simple_bytes_append/main.go b/tinygotest/testdata/simple_bytes_append/main.go index 98bfb890..9064bbda 100644 --- a/tinygotest/testdata/simple_bytes_append/main.go +++ b/tinygotest/testdata/simple_bytes_append/main.go @@ -10,7 +10,6 @@ type Example struct { var buf [64]byte func main() { - e := Example{ I: 1, S: "2", @@ -28,5 +27,4 @@ func main() { print(b[i], " ") } println() - } diff --git a/tinygotest/testdata/simple_bytes_read/main.go b/tinygotest/testdata/simple_bytes_read/main.go index 4d3314ce..c9195d8d 100644 --- a/tinygotest/testdata/simple_bytes_read/main.go +++ b/tinygotest/testdata/simple_bytes_read/main.go @@ -8,7 +8,6 @@ type Example struct { } func main() { - b := []byte{130, 161, 105, 1, 161, 115, 161, 50} e := Example{} @@ -53,5 +52,4 @@ func main() { } println("done") - } diff --git a/tinygotest/testdata/simple_marshal/main.go b/tinygotest/testdata/simple_marshal/main.go index dc03a2d4..83535fba 100644 --- a/tinygotest/testdata/simple_marshal/main.go +++ b/tinygotest/testdata/simple_marshal/main.go @@ -8,7 +8,6 @@ type Example struct { } func main() { - e := Example{ I: 1, S: "2", @@ -23,5 +22,4 @@ func main() { print(b[i], " ") } println() - } diff --git a/tinygotest/testdata/simple_roundtrip/main.go b/tinygotest/testdata/simple_roundtrip/main.go index e1f291dd..f9961928 100644 --- a/tinygotest/testdata/simple_roundtrip/main.go +++ b/tinygotest/testdata/simple_roundtrip/main.go @@ -11,7 +11,6 @@ type Example struct { } func main() { - e := Example{ I: 1, S: "2", @@ -35,5 +34,4 @@ func main() { } println("done, len(b):", len(b)) - } diff --git a/tinygotest/testdata/simple_unmarshal/main.go b/tinygotest/testdata/simple_unmarshal/main.go index f7104d11..c948248f 100644 --- a/tinygotest/testdata/simple_unmarshal/main.go +++ b/tinygotest/testdata/simple_unmarshal/main.go @@ -8,7 +8,6 @@ type Example struct { } func main() { - b := []byte{130, 161, 105, 1, 161, 115, 161, 50} var e2 Example @@ -25,5 +24,4 @@ func main() { } println("done, len(b):", len(b)) - } diff --git a/tinygotest/tinygo_test.go b/tinygotest/tinygo_test.go index 24966df5..2659abed 100644 --- a/tinygotest/tinygo_test.go +++ b/tinygotest/tinygo_test.go @@ -1,4 +1,4 @@ -// +build amd64 darwin +//go:build amd64 || darwin package tinygotest @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" ) @@ -68,7 +69,6 @@ func TestSimpleRoundtripBuild(t *testing.T) { if sz > 20000 { t.Errorf("arduino-nano33.bin is larger than expected: %d", sz) } - } // simple_marshal is just the MarshalMsg part @@ -148,6 +148,9 @@ var buildOnlyTargets = []string{ func run(t *testing.T, dir, exe string, args ...string) { t.Helper() + if runtime.GOOS == "windows" { + exe += ".exe" + } cmd := exec.Command("./" + exe) wd, err := os.Getwd() if err != nil { @@ -183,7 +186,6 @@ func goGenerate(t *testing.T, dir string) { } func tinygoBuild(t *testing.T, dir string, targets ...string) { - t.Helper() wd, err := os.Getwd() @@ -203,12 +205,19 @@ func tinygoBuild(t *testing.T, dir string, targets ...string) { if tgt == "wasm" { ext = ".wasm" } + dst := tgt + ext + if tgt == "" { + dst = "nativebin" + if runtime.GOOS == "windows" { + dst += ".exe" + } + } var args []string if tgt == "" { // empty target means the native platform - args = []string{"build", "-o=nativebin", "."} + args = []string{"build", "-o=" + dst, "."} } else { - args = []string{"build", "-target=" + tgt, "-o=" + tgt + ext, "."} + args = []string{"build", "-target=" + tgt, "-o=" + dst, "."} } t.Logf("%s: tinygo %v", dir, args) @@ -219,10 +228,13 @@ func tinygoBuild(t *testing.T, dir string, targets ...string) { t.Logf("%s: tinygo build %v; output: %s", dir, args, b) } if err != nil { + // See https://github.com/tinygo-org/tinygo/issues/3977 + if strings.Contains(string(b), "could not find wasm-opt") { + t.Skipf("skipping wasm test because wasm-opt is not installed") + } t.Fatal(err) } } - } var spacePad = strings.Repeat(" ", 64)