diff --git a/go.mod b/go.mod index 154b356a..112ca680 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/tinylib/msgp go 1.20 require ( - github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 + github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c golang.org/x/tools v0.22.0 ) diff --git a/go.sum b/go.sum index 6f52deeb..f09ebae4 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 h1:jYi87L8j62qkXzaYHAQAhEapgukhenIMZRBKTNRLHJ4= -github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY= +github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= diff --git a/msgp/circular.go b/msgp/circular.go index a0434c7e..5d2408e5 100644 --- a/msgp/circular.go +++ b/msgp/circular.go @@ -16,6 +16,10 @@ type EndlessReader struct { // NewEndlessReader returns a new endless reader func NewEndlessReader(b []byte, tb timer) *EndlessReader { + // Double until we reach 4K. + for len(b) < 4<<10 { + b = append(b, b...) + } return &EndlessReader{tb: tb, data: b, offset: 0} } @@ -24,16 +28,14 @@ func NewEndlessReader(b []byte, tb timer) *EndlessReader { // fills the supplied slice while the benchmark // timer is stopped. func (c *EndlessReader) Read(p []byte) (int, error) { - c.tb.StopTimer() var n int l := len(p) m := len(c.data) + nn := copy(p[n:], c.data[c.offset:]) + n += nn for n < l { - nn := copy(p[n:], c.data[c.offset:]) - n += nn - c.offset += nn - c.offset %= m + n += copy(p[n:], c.data[:]) } - c.tb.StartTimer() + c.offset = (c.offset + l) % m return n, nil } diff --git a/msgp/json.go b/msgp/json.go index fe570373..dbd70cda 100644 --- a/msgp/json.go +++ b/msgp/json.go @@ -346,14 +346,12 @@ func rwExtension(dst jsWriter, src *Reader) (n int, err error) { } func rwString(dst jsWriter, src *Reader) (n int, err error) { - var p []byte - p, err = src.R.Peek(1) + lead, err := src.R.PeekByte() if err != nil { return } - lead := p[0] var read int - + var p []byte if isfixstr(lead) { read = int(rfixstr(lead)) src.R.Skip(1) diff --git a/msgp/read.go b/msgp/read.go index 0215a5b9..31628782 100644 --- a/msgp/read.go +++ b/msgp/read.go @@ -236,13 +236,13 @@ func (m *Reader) BufferSize() int { return m.R.BufferSize() } // NextType returns the next object type to be decoded. func (m *Reader) NextType() (Type, error) { - p, err := m.R.Peek(1) + next, err := m.R.PeekByte() if err != nil { return InvalidType, err } - t := getType(p[0]) + t := getType(next) if t == InvalidType { - return t, InvalidPrefixError(p[0]) + return t, InvalidPrefixError(next) } if t == ExtensionType { v, err := m.peekExtensionType() @@ -264,8 +264,8 @@ func (m *Reader) NextType() (Type, error) { // IsNil returns whether or not // the next byte is a null messagepack byte func (m *Reader) IsNil() bool { - p, err := m.R.Peek(1) - return err == nil && p[0] == mnil + p, err := m.R.PeekByte() + return err == nil && p == mnil } // getNextSize returns the size of the next object on the wire. @@ -276,11 +276,10 @@ func (m *Reader) IsNil() bool { // use uintptr b/c it's guaranteed to be large enough // to hold whatever we can fit in memory. func getNextSize(r *fwd.Reader) (uintptr, uintptr, error) { - b, err := r.Peek(1) + lead, err := r.PeekByte() if err != nil { return 0, 0, err } - lead := b[0] spec := getBytespec(lead) size, mode := spec.size, spec.extra if size == 0 { @@ -289,7 +288,7 @@ func getNextSize(r *fwd.Reader) (uintptr, uintptr, error) { if mode >= 0 { return uintptr(size), uintptr(mode), nil } - b, err = r.Peek(int(size)) + b, err := r.Peek(int(size)) if err != nil { return 0, 0, err } @@ -373,11 +372,10 @@ func (m *Reader) Skip() error { func (m *Reader) ReadMapHeader() (sz uint32, err error) { var p []byte var lead byte - p, err = m.R.Peek(1) + lead, err = m.R.PeekByte() if err != nil { return } - lead = p[0] if isfixmap(lead) { sz = uint32(rfixmap(lead)) _, err = m.R.Skip(1) @@ -427,12 +425,12 @@ func (m *Reader) ReadMapKey(scratch []byte) ([]byte, error) { // method; writing into the returned slice may // corrupt future reads. func (m *Reader) ReadMapKeyPtr() ([]byte, error) { - p, err := m.R.Peek(1) + lead, err := m.R.PeekByte() if err != nil { return nil, err } - lead := p[0] var read int + var p []byte if isfixstr(lead) { read = int(rfixstr(lead)) m.R.Skip(1) @@ -471,18 +469,16 @@ fill: // array header and returns the size of the array // and the number of bytes read. func (m *Reader) ReadArrayHeader() (sz uint32, err error) { - var lead byte - var p []byte - p, err = m.R.Peek(1) + lead, err := m.R.PeekByte() if err != nil { return } - lead = p[0] if isfixarray(lead) { sz = uint32(rfixarray(lead)) _, err = m.R.Skip(1) return } + var p []byte switch lead { case marray16: p, err = m.R.Next(3) @@ -508,12 +504,12 @@ func (m *Reader) ReadArrayHeader() (sz uint32, err error) { // ReadNil reads a 'nil' MessagePack byte from the reader func (m *Reader) ReadNil() error { - p, err := m.R.Peek(1) + p, err := m.R.PeekByte() if err != nil { return err } - if p[0] != mnil { - return badPrefix(NilType, p[0]) + if p != mnil { + return badPrefix(NilType, p) } _, err = m.R.Skip(1) return err @@ -566,17 +562,17 @@ func (m *Reader) ReadFloat32() (f float32, err error) { // ReadBool reads a bool from the reader func (m *Reader) ReadBool() (b bool, err error) { - var p []byte - p, err = m.R.Peek(1) + var p byte + p, err = m.R.PeekByte() if err != nil { return } - switch p[0] { + switch p { case mtrue: b = true case mfalse: default: - err = badPrefix(BoolType, p[0]) + err = badPrefix(BoolType, p) return } _, err = m.R.Skip(1) @@ -592,12 +588,10 @@ func (m *Reader) ReadDuration() (d time.Duration, err error) { // ReadInt64 reads an int64 from the reader func (m *Reader) ReadInt64() (i int64, err error) { var p []byte - var lead byte - p, err = m.R.Peek(1) + lead, err := m.R.PeekByte() if err != nil { return } - lead = p[0] if isfixint(lead) { i = int64(rfixint(lead)) @@ -738,12 +732,10 @@ func (m *Reader) ReadInt() (i int, err error) { // ReadUint64 reads a uint64 from the reader func (m *Reader) ReadUint64() (u uint64, err error) { var p []byte - var lead byte - p, err = m.R.Peek(1) + lead, err := m.R.PeekByte() if err != nil { return } - lead = p[0] if isfixint(lead) { u = uint64(rfixint(lead)) _, err = m.R.Skip(1) @@ -958,11 +950,11 @@ func (m *Reader) ReadBytes(scratch []byte) (b []byte, err error) { // way. func (m *Reader) ReadBytesHeader() (sz uint32, err error) { var p []byte - p, err = m.R.Peek(1) + lead, err := m.R.PeekByte() if err != nil { return } - switch p[0] { + switch lead { case mbin8: p, err = m.R.Next(2) if err != nil { @@ -1036,12 +1028,10 @@ func (m *Reader) ReadExactBytes(into []byte) error { // if it is non-nil. func (m *Reader) ReadStringAsBytes(scratch []byte) (b []byte, err error) { var p []byte - var lead byte - p, err = m.R.Peek(1) + lead, err := m.R.PeekByte() if err != nil { return } - lead = p[0] var read int64 if isfixstr(lead) { @@ -1088,17 +1078,16 @@ fill: // for dealing with the next 'sz' bytes from // the reader in an application-specific manner. func (m *Reader) ReadStringHeader() (sz uint32, err error) { - var p []byte - p, err = m.R.Peek(1) + lead, err := m.R.PeekByte() if err != nil { return } - lead := p[0] if isfixstr(lead) { sz = uint32(rfixstr(lead)) m.R.Skip(1) return } + var p []byte switch lead { case mstr8: p, err = m.R.Next(2) @@ -1129,15 +1118,13 @@ func (m *Reader) ReadStringHeader() (sz uint32, err error) { // ReadString reads a utf-8 string from the reader func (m *Reader) ReadString() (s string, err error) { - var p []byte - var lead byte var read int64 - p, err = m.R.Peek(1) + lead, err := m.R.PeekByte() if err != nil { return } - lead = p[0] + var p []byte if isfixstr(lead) { read = int64(rfixstr(lead)) m.R.Skip(1) diff --git a/msgp/read_bytes.go b/msgp/read_bytes.go index 948faf1d..05ca3bc7 100644 --- a/msgp/read_bytes.go +++ b/msgp/read_bytes.go @@ -166,29 +166,30 @@ func ReadMapHeaderBytes(b []byte) (sz uint32, o []byte, err error) { } lead := b[0] + b = b[1:] if isfixmap(lead) { sz = uint32(rfixmap(lead)) - o = b[1:] + o = b return } switch lead { case mmap16: - if l < 3 { + if len(b) < 2 { err = ErrShortBytes return } - sz = uint32(big.Uint16(b[1:])) - o = b[3:] + sz = uint32(big.Uint16(b)) + o = b[2:] return case mmap32: - if l < 5 { + if len(b) < 4 { err = ErrShortBytes return } - sz = big.Uint32(b[1:]) - o = b[5:] + sz = big.Uint32(b) + o = b[4:] return default: @@ -228,29 +229,30 @@ func ReadArrayHeaderBytes(b []byte) (sz uint32, o []byte, err error) { return 0, nil, ErrShortBytes } lead := b[0] + b = b[1:] if isfixarray(lead) { sz = uint32(rfixarray(lead)) - o = b[1:] + o = b return } switch lead { case marray16: - if len(b) < 3 { + if len(b) < 2 { err = ErrShortBytes return } - sz = uint32(big.Uint16(b[1:])) - o = b[3:] + sz = uint32(big.Uint16(b)) + o = b[2:] return case marray32: - if len(b) < 5 { + if len(b) < 4 { err = ErrShortBytes return } - sz = big.Uint32(b[1:]) - o = b[5:] + sz = big.Uint32(b) + o = b[4:] return default: @@ -418,99 +420,99 @@ func ReadDurationBytes(b []byte) (d time.Duration, o []byte, err error) { // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a int) func ReadInt64Bytes(b []byte) (i int64, o []byte, err error) { - l := len(b) - if l < 1 { + if len(b) < 1 { return 0, nil, ErrShortBytes } lead := b[0] + b = b[1:] if isfixint(lead) { i = int64(rfixint(lead)) - o = b[1:] + o = b return } if isnfixint(lead) { i = int64(rnfixint(lead)) - o = b[1:] + o = b return } switch lead { case mint8: - if l < 2 { + if len(b) < 1 { err = ErrShortBytes return } - i = int64(getMint8(b)) - o = b[2:] + i = int64(int8(b[0])) + o = b[1:] return case muint8: - if l < 2 { + if len(b) < 1 { err = ErrShortBytes return } - i = int64(getMuint8(b)) - o = b[2:] + i = int64(b[0]) + o = b[1:] return case mint16: - if l < 3 { + if len(b) < 2 { err = ErrShortBytes return } - i = int64(getMint16(b)) - o = b[3:] + i = int64(int16(big.Uint16(b))) + o = b[2:] return case muint16: - if l < 3 { + if len(b) < 2 { err = ErrShortBytes return } - i = int64(getMuint16(b)) - o = b[3:] + i = int64(big.Uint16(b)) + o = b[2:] return case mint32: - if l < 5 { + if len(b) < 4 { err = ErrShortBytes return } - i = int64(getMint32(b)) - o = b[5:] + i = int64(int32(big.Uint32(b))) + o = b[4:] return case muint32: - if l < 5 { + if len(b) < 4 { err = ErrShortBytes return } - i = int64(getMuint32(b)) - o = b[5:] + i = int64(big.Uint32(b)) + o = b[4:] return case mint64: - if l < 9 { + if len(b) < 8 { err = ErrShortBytes return } - i = int64(getMint64(b)) - o = b[9:] + i = int64(big.Uint64(b)) + o = b[8:] return case muint64: - if l < 9 { + if len(b) < 8 { err = ErrShortBytes return } - u := getMuint64(b) + u := big.Uint64(b) if u > math.MaxInt64 { err = UintOverflow{Value: u, FailedBitsize: 64} return } i = int64(u) - o = b[9:] + o = b[8:] return default: @@ -592,109 +594,109 @@ func ReadIntBytes(b []byte) (int, []byte, error) { // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a uint) func ReadUint64Bytes(b []byte) (u uint64, o []byte, err error) { - l := len(b) - if l < 1 { + if len(b) < 1 { return 0, nil, ErrShortBytes } lead := b[0] + b = b[1:] if isfixint(lead) { u = uint64(rfixint(lead)) - o = b[1:] + o = b return } switch lead { case mint8: - if l < 2 { + if len(b) < 1 { err = ErrShortBytes return } - v := int64(getMint8(b)) + v := int64(int8(b[0])) if v < 0 { err = UintBelowZero{Value: v} return } u = uint64(v) - o = b[2:] + o = b[1:] return case muint8: - if l < 2 { + if len(b) < 1 { err = ErrShortBytes return } - u = uint64(getMuint8(b)) - o = b[2:] + u = uint64(b[0]) + o = b[1:] return case mint16: - if l < 3 { + if len(b) < 2 { err = ErrShortBytes return } - v := int64(getMint16(b)) + v := int64((int16(b[0]) << 8) | int16(b[1])) if v < 0 { err = UintBelowZero{Value: v} return } u = uint64(v) - o = b[3:] + o = b[2:] return case muint16: - if l < 3 { + if len(b) < 2 { err = ErrShortBytes return } - u = uint64(getMuint16(b)) - o = b[3:] + u = uint64(big.Uint16(b)) + o = b[2:] return case mint32: - if l < 5 { + if len(b) < 4 { err = ErrShortBytes return } - v := int64(getMint32(b)) + v := int64(int32(big.Uint32(b))) if v < 0 { err = UintBelowZero{Value: v} return } u = uint64(v) - o = b[5:] + o = b[4:] return case muint32: - if l < 5 { + if len(b) < 4 { err = ErrShortBytes return } - u = uint64(getMuint32(b)) - o = b[5:] + u = uint64(big.Uint32(b)) + o = b[4:] return case mint64: - if l < 9 { + if len(b) < 8 { err = ErrShortBytes return } - v := int64(getMint64(b)) + v := int64(big.Uint64(b)) if v < 0 { err = UintBelowZero{Value: v} return } u = uint64(v) - o = b[9:] + o = b[8:] return case muint64: - if l < 9 { + if len(b) < 8 { err = ErrShortBytes return } - u = getMuint64(b) - o = b[9:] + u = big.Uint64(b) + o = b[8:] return default: @@ -796,32 +798,33 @@ func readBytesBytes(b []byte, scratch []byte, zc bool) (v []byte, o []byte, err } lead := b[0] + b = b[1:] var read int switch lead { case mbin8: - if l < 2 { + if len(b) < 1 { err = ErrShortBytes return } - read = int(b[1]) - b = b[2:] + read = int(b[0]) + b = b[1:] case mbin16: - if l < 3 { + if len(b) < 2 { err = ErrShortBytes return } - read = int(big.Uint16(b[1:])) - b = b[3:] + read = int(big.Uint16(b)) + b = b[2:] case mbin32: - if l < 5 { + if len(b) < 4 { err = ErrShortBytes return } - read = int(big.Uint32(b[1:])) - b = b[5:] + read = int(big.Uint32(b)) + b = b[4:] default: err = badPrefix(BinType, lead) @@ -863,40 +866,39 @@ func ReadBytesZC(b []byte) (v []byte, o []byte, err error) { } func ReadExactBytes(b []byte, into []byte) (o []byte, err error) { - l := len(b) - if l < 1 { + if len(b) < 1 { err = ErrShortBytes return } lead := b[0] var read uint32 - var skip int + b = b[1:] switch lead { case mbin8: - if l < 2 { + if len(b) < 1 { err = ErrShortBytes return } - read = uint32(b[1]) - skip = 2 + read = uint32(b[0]) + b = b[1:] case mbin16: - if l < 3 { + if len(b) < 2 { err = ErrShortBytes return } - read = uint32(big.Uint16(b[1:])) - skip = 3 + read = uint32(big.Uint16(b)) + b = b[2:] case mbin32: - if l < 5 { + if len(b) < 4 { err = ErrShortBytes return } - read = uint32(big.Uint32(b[1:])) - skip = 5 + read = big.Uint32(b) + b = b[4:] default: err = badPrefix(BinType, lead) @@ -908,7 +910,7 @@ func ReadExactBytes(b []byte, into []byte) (o []byte, err error) { return } - o = b[skip+copy(into, b[skip:]):] + o = b[copy(into, b):] return } @@ -921,42 +923,41 @@ func ReadExactBytes(b []byte, into []byte) (o []byte, err error) { // - [ErrShortBytes] (b not long enough) // - [TypeError] (object not 'str') func ReadStringZC(b []byte) (v []byte, o []byte, err error) { - l := len(b) - if l < 1 { + if len(b) < 1 { return nil, nil, ErrShortBytes } lead := b[0] var read int + b = b[1:] if isfixstr(lead) { read = int(rfixstr(lead)) - b = b[1:] } else { switch lead { case mstr8: - if l < 2 { + if len(b) < 1 { err = ErrShortBytes return } - read = int(b[1]) - b = b[2:] + read = int(b[0]) + b = b[1:] case mstr16: - if l < 3 { + if len(b) < 2 { err = ErrShortBytes return } - read = int(big.Uint16(b[1:])) - b = b[3:] + read = int(big.Uint16(b)) + b = b[2:] case mstr32: - if l < 5 { + if len(b) < 4 { err = ErrShortBytes return } - read = int(big.Uint32(b[1:])) - b = b[5:] + read = int(big.Uint32(b)) + b = b[4:] default: err = TypeError{Method: StrType, Encoded: getType(lead)} diff --git a/msgp/read_bytes_test.go b/msgp/read_bytes_test.go index 724a061a..d9ee6c9f 100644 --- a/msgp/read_bytes_test.go +++ b/msgp/read_bytes_test.go @@ -472,7 +472,7 @@ func TestReadIntBytesOverflows(t *testing.T) { _, _, err = ReadUint8Bytes(buf.Bytes()) } if !v.errCheck(err, v.failBits) { - t.Fatal(err) + t.Fatal(err, v.rdBits, v.failBits) } }) } diff --git a/msgp/write_test.go b/msgp/write_test.go index 1b32a2e9..11542e05 100644 --- a/msgp/write_test.go +++ b/msgp/write_test.go @@ -20,8 +20,9 @@ var ( func RandBytes(sz int) []byte { out := make([]byte, sz) + rng := rand.New(rand.NewSource(time.Now().UnixNano())) for i := range out { - out[i] = byte(rand.Int63n(math.MaxInt64) % 256) + out[i] = byte(rng.Uint32()) } return out }