Added SubSecondPrecision flag in fluentd logger

Signed-off-by: dungeonmaster18 <umesh4257@gmail.com>

daemon/logger/fluentd/fluentd.go

@@ -48,11 +48,12 @@ const (
 	defaultRetryWait  = 1000
 	defaultMaxRetries = math.MaxInt32
 
-	addressKey      = "fluentd-address"
-	bufferLimitKey  = "fluentd-buffer-limit"
-	retryWaitKey    = "fluentd-retry-wait"
-	maxRetriesKey   = "fluentd-max-retries"
-	asyncConnectKey = "fluentd-async-connect"
+	addressKey            = "fluentd-address"
+	bufferLimitKey        = "fluentd-buffer-limit"
+	retryWaitKey          = "fluentd-retry-wait"
+	maxRetriesKey         = "fluentd-max-retries"
+	asyncConnectKey       = "fluentd-async-connect"
+	subSecondPrecisionKey = "fluentd-sub-second-precision"
 )
 
 func init() {
@@ -117,15 +118,23 @@ func New(info logger.Info) (logger.Logger, error) {
 		}
 	}
 
+	subSecondPrecision := false
+	if info.Config[subSecondPrecisionKey] != "" {
+		if subSecondPrecision, err = strconv.ParseBool(info.Config[subSecondPrecisionKey]); err != nil {
+			return nil, err
+		}
+	}
+
 	fluentConfig := fluent.Config{
-		FluentPort:       loc.port,
-		FluentHost:       loc.host,
-		FluentNetwork:    loc.protocol,
-		FluentSocketPath: loc.path,
-		BufferLimit:      bufferLimit,
-		RetryWait:        retryWait,
-		MaxRetry:         maxRetries,
-		AsyncConnect:     asyncConnect,
+		FluentPort:         loc.port,
+		FluentHost:         loc.host,
+		FluentNetwork:      loc.protocol,
+		FluentSocketPath:   loc.path,
+		BufferLimit:        bufferLimit,
+		RetryWait:          retryWait,
+		MaxRetry:           maxRetries,
+		AsyncConnect:       asyncConnect,
+		SubSecondPrecision: subSecondPrecision,
 	}
 
 	logrus.WithField("container", info.ContainerID).WithField("config", fluentConfig).
@@ -183,6 +192,7 @@ func ValidateLogOpt(cfg map[string]string) error {
 		case retryWaitKey:
 		case maxRetriesKey:
 		case asyncConnectKey:
+		case subSecondPrecisionKey:
 			// Accepted
 		default:
 			return fmt.Errorf("unknown log opt '%s' for fluentd log driver", key)

vendor.conf

@@ -79,10 +79,10 @@ github.com/golang/protobuf 7a211bcf3bce0e3f1d74f9894916e6f116ae83b4
 # gelf logging driver deps
 github.com/Graylog2/go-gelf v2
 
-github.com/fluent/fluent-logger-golang v1.2.1
+github.com/fluent/fluent-logger-golang v1.3.0
 # fluent-logger-golang deps
 github.com/philhofer/fwd 98c11a7a6ec829d672b03833c3d69a7fae1ca972
-github.com/tinylib/msgp 75ee40d2601edf122ef667e2a07d600d4c44490c
+github.com/tinylib/msgp 3b556c64540842d4f82967be066a7f7fffc3adad
 
 # fsnotify
 github.com/fsnotify/fsnotify v1.4.2

vendor/github.com/fluent/fluent-logger-golang/README.md

@@ -21,7 +21,7 @@ import "github.com/fluent/fluent-logger-golang/fluent"
 
 GoDoc: http://godoc.org/github.com/fluent/fluent-logger-golang/fluent
 
-##Example
+## Example
 
 ```go
 package main
@@ -44,14 +44,14 @@ func main() {
     "hoge": "hoge",
   }
   error := logger.Post(tag, data)
-  // error := logger.Post(tag, time.Time.Now(), data)
+  // error := logger.PostWithTime(tag, time.Now(), data)
   if error != nil {
     panic(error)
   }
 }
 ```
 
-`data` must be a value like `map[string]literal`, `map[string]interface{}` or `struct`. Logger refers tags `msg` or `codec` of each fields of structs.
+`data` must be a value like `map[string]literal`, `map[string]interface{}`, `struct` or [`msgp.Marshaler`](http://godoc.org/github.com/tinylib/msgp/msgp#Marshaler). Logger refers tags `msg` or `codec` of each fields of structs.
 
 ## Setting config values
 
@@ -59,6 +59,11 @@ func main() {
 f := fluent.New(fluent.Config{FluentPort: 80, FluentHost: "example.com"})
 ```
 
+### WriteTimeout
+
+Sets the timeout for Write call of logger.Post.
+Since the default is zero value, Write will not time out.
+
 ## Tests
 ```
 go test

vendor/github.com/fluent/fluent-logger-golang/fluent/fluent.go

@@ -4,13 +4,14 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"math"
 	"net"
 	"reflect"
 	"strconv"
 	"sync"
 	"time"
+
+	"github.com/tinylib/msgp/msgp"
 )
 
 const (
@@ -19,10 +20,14 @@ const (
 	defaultSocketPath             = ""
 	defaultPort                   = 24224
 	defaultTimeout                = 3 * time.Second
+	defaultWriteTimeout           = time.Duration(0) // Write() will not time out
 	defaultBufferLimit            = 8 * 1024 * 1024
 	defaultRetryWait              = 500
 	defaultMaxRetry               = 13
 	defaultReconnectWaitIncreRate = 1.5
+	// Default sub-second precision value to false since it is only compatible
+	// with fluentd versions v0.14 and above.
+	defaultSubSecondPrecision = false
 )
 
 type Config struct {
@@ -31,12 +36,17 @@ type Config struct {
 	FluentNetwork    string        `json:"fluent_network"`
 	FluentSocketPath string        `json:"fluent_socket_path"`
 	Timeout          time.Duration `json:"timeout"`
+	WriteTimeout     time.Duration `json:"write_timeout"`
 	BufferLimit      int           `json:"buffer_limit"`
 	RetryWait        int           `json:"retry_wait"`
 	MaxRetry         int           `json:"max_retry"`
 	TagPrefix        string        `json:"tag_prefix"`
 	AsyncConnect     bool          `json:"async_connect"`
 	MarshalAsJSON    bool          `json:"marshal_as_json"`
+
+	// Sub-second precision timestamps are only possible for those using fluentd
+	// v0.14+ and serializing their messages with msgpack.
+	SubSecondPrecision bool `json:"sub_second_precision"`
 }
 
 type Fluent struct {
@@ -46,7 +56,7 @@ type Fluent struct {
 	pending []byte
 
 	muconn       sync.Mutex
-	conn         io.WriteCloser
+	conn         net.Conn
 	reconnecting bool
 }
 
@@ -67,6 +77,9 @@ func New(config Config) (f *Fluent, err error) {
 	if config.Timeout == 0 {
 		config.Timeout = defaultTimeout
 	}
+	if config.WriteTimeout == 0 {
+		config.WriteTimeout = defaultWriteTimeout
+	}
 	if config.BufferLimit == 0 {
 		config.BufferLimit = defaultBufferLimit
 	}
@@ -90,9 +103,6 @@ func New(config Config) (f *Fluent, err error) {
 //
 // Examples:
 //
-//  // send string
-//  f.Post("tag_name", "data")
-//
 //  // send map[string]
 //  mapStringData := map[string]string{
 //  	"foo":  "bar",
@@ -124,6 +134,10 @@ func (f *Fluent) PostWithTime(tag string, tm time.Time, message interface{}) err
 		tag = f.TagPrefix + "." + tag
 	}
 
+	if m, ok := message.(msgp.Marshaler); ok {
+		return f.EncodeAndPostData(tag, tm, m)
+	}
+
 	msg := reflect.ValueOf(message)
 	msgtype := msg.Type()
 
@@ -203,6 +217,9 @@ func (f *Fluent) EncodeData(tag string, tm time.Time, message interface{}) (data
 		msg := Message{Tag: tag, Time: timeUnix, Record: message}
 		chunk := &MessageChunk{message: msg}
 		data, err = json.Marshal(chunk)
+	} else if f.Config.SubSecondPrecision {
+		msg := &MessageExt{Tag: tag, Time: EventTime(tm), Record: message}
+		data, err = msg.MarshalMsg(nil)
 	} else {
 		msg := &Message{Tag: tag, Time: timeUnix, Record: message}
 		data, err = msg.MarshalMsg(nil)
@@ -297,6 +314,12 @@ func (f *Fluent) send() error {
 
 	var err error
 	if len(f.pending) > 0 {
+		t := f.Config.WriteTimeout
+		if time.Duration(0) < t {
+			f.conn.SetWriteDeadline(time.Now().Add(t))
+		} else {
+			f.conn.SetWriteDeadline(time.Time{})
+		}
 		_, err = f.conn.Write(f.pending)
 		if err != nil {
 			f.conn.Close()

vendor/github.com/fluent/fluent-logger-golang/fluent/proto.go

@@ -2,6 +2,12 @@
 
 package fluent
 
+import (
+	"time"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
 //msgp:tuple Entry
 type Entry struct {
 	Time   int64       `msg:"time"`
@@ -22,3 +28,69 @@ type Message struct {
 	Record interface{} `msg:"record"`
 	Option interface{} `msg:"option"`
 }
+
+//msgp:tuple MessageExt
+type MessageExt struct {
+	Tag    string      `msg:"tag"`
+	Time   EventTime   `msg:"time,extension"`
+	Record interface{} `msg:"record"`
+	Option interface{} `msg:"option"`
+}
+
+// EventTime is an extension to the serialized time value. It builds in support
+// for sub-second (nanosecond) precision in serialized timestamps.
+//
+// You can find the full specification for the msgpack message payload here:
+// https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1.
+//
+// You can find more information on msgpack extension types here:
+// https://github.com/tinylib/msgp/wiki/Using-Extensions.
+type EventTime time.Time
+
+const (
+	extensionType = 0
+	length        = 8
+)
+
+func init() {
+	msgp.RegisterExtension(extensionType, func() msgp.Extension { return new(EventTime) })
+}
+
+func (t *EventTime) ExtensionType() int8 { return extensionType }
+
+func (t *EventTime) Len() int { return length }
+
+func (t *EventTime) MarshalBinaryTo(b []byte) error {
+	// Unwrap to Golang time
+	goTime := time.Time(*t)
+
+	// There's no support for timezones in fluentd's protocol for EventTime.
+	// Convert to UTC.
+	utc := goTime.UTC()
+
+	// Warning! Converting seconds to an int32 is a lossy operation. This code
+	// will hit the "Year 2038" problem.
+	sec := int32(utc.Unix())
+	nsec := utc.Nanosecond()
+
+	// Fill the buffer with 4 bytes for the second component of the timestamp.
+	b[0] = byte(sec >> 24)
+	b[1] = byte(sec >> 16)
+	b[2] = byte(sec >> 8)
+	b[3] = byte(sec)
+
+	// Fill the buffer with 4 bytes for the nanosecond component of the
+	// timestamp.
+	b[4] = byte(nsec >> 24)
+	b[5] = byte(nsec >> 16)
+	b[6] = byte(nsec >> 8)
+	b[7] = byte(nsec)
+
+	return nil
+}
+
+// UnmarshalBinary is not implemented since decoding messages is not supported
+// by this library.
+func (t *EventTime) UnmarshalBinary(b []byte) error {
+	return nil
+}

vendor/github.com/fluent/fluent-logger-golang/fluent/proto_gen.go

@@ -10,13 +10,13 @@ import (
 
 // DecodeMsg implements msgp.Decodable
 func (z *Entry) DecodeMsg(dc *msgp.Reader) (err error) {
-	var ssz uint32
-	ssz, err = dc.ReadArrayHeader()
+	var zxvk uint32
+	zxvk, err = dc.ReadArrayHeader()
 	if err != nil {
 		return
 	}
-	if ssz != 2 {
-		err = msgp.ArrayError{Wanted: 2, Got: ssz}
+	if zxvk != 2 {
+		err = msgp.ArrayError{Wanted: 2, Got: zxvk}
 		return
 	}
 	z.Time, err = dc.ReadInt64()
@@ -32,9 +32,10 @@ func (z *Entry) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z Entry) EncodeMsg(en *msgp.Writer) (err error) {
-	err = en.WriteArrayHeader(2)
+	// array header, size 2
+	err = en.Append(0x92)
 	if err != nil {
-		return
+		return err
 	}
 	err = en.WriteInt64(z.Time)
 	if err != nil {
@@ -50,7 +51,8 @@ func (z Entry) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z Entry) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	o = msgp.AppendArrayHeader(o, 2)
+	// array header, size 2
+	o = append(o, 0x92)
 	o = msgp.AppendInt64(o, z.Time)
 	o, err = msgp.AppendIntf(o, z.Record)
 	if err != nil {
@@ -61,16 +63,14 @@ func (z Entry) MarshalMsg(b []byte) (o []byte, err error) {
 
 // UnmarshalMsg implements msgp.Unmarshaler
 func (z *Entry) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	{
-		var ssz uint32
-		ssz, bts, err = msgp.ReadArrayHeaderBytes(bts)
-		if err != nil {
-			return
-		}
-		if ssz != 2 {
-			err = msgp.ArrayError{Wanted: 2, Got: ssz}
-			return
-		}
+	var zbzg uint32
+	zbzg, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		return
+	}
+	if zbzg != 2 {
+		err = msgp.ArrayError{Wanted: 2, Got: zbzg}
+		return
 	}
 	z.Time, bts, err = msgp.ReadInt64Bytes(bts)
 	if err != nil {
@@ -84,51 +84,52 @@ func (z *Entry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z Entry) Msgsize() (s int) {
-	s = msgp.ArrayHeaderSize + msgp.Int64Size + msgp.GuessSize(z.Record)
+	s = 1 + msgp.Int64Size + msgp.GuessSize(z.Record)
 	return
 }
 
 // DecodeMsg implements msgp.Decodable
 func (z *Forward) DecodeMsg(dc *msgp.Reader) (err error) {
-	var ssz uint32
-	ssz, err = dc.ReadArrayHeader()
+	var zcmr uint32
+	zcmr, err = dc.ReadArrayHeader()
 	if err != nil {
 		return
 	}
-	if ssz != 3 {
-		err = msgp.ArrayError{Wanted: 3, Got: ssz}
+	if zcmr != 3 {
+		err = msgp.ArrayError{Wanted: 3, Got: zcmr}
 		return
 	}
 	z.Tag, err = dc.ReadString()
 	if err != nil {
 		return
 	}
-	var xsz uint32
-	xsz, err = dc.ReadArrayHeader()
+	var zajw uint32
+	zajw, err = dc.ReadArrayHeader()
 	if err != nil {
 		return
 	}
-	if cap(z.Entries) >= int(xsz) {
-		z.Entries = z.Entries[:xsz]
+	if cap(z.Entries) >= int(zajw) {
+		z.Entries = (z.Entries)[:zajw]
 	} else {
-		z.Entries = make([]Entry, xsz)
+		z.Entries = make([]Entry, zajw)
 	}
-	for xvk := range z.Entries {
-		var ssz uint32
-		ssz, err = dc.ReadArrayHeader()
+	for zbai := range z.Entries {
+		var zwht uint32
+		zwht, err = dc.ReadArrayHeader()
 		if err != nil {
 			return
 		}
-		if ssz != 2 {
-			err = msgp.ArrayError{Wanted: 2, Got: ssz}
+		if zwht != 2 {
+			err = msgp.ArrayError{Wanted: 2, Got: zwht}
 			return
 		}
-		z.Entries[xvk].Time, err = dc.ReadInt64()
+		z.Entries[zbai].Time, err = dc.ReadInt64()
 		if err != nil {
 			return
 		}
-		z.Entries[xvk].Record, err = dc.ReadIntf()
+		z.Entries[zbai].Record, err = dc.ReadIntf()
 		if err != nil {
 			return
 		}
@@ -142,9 +143,10 @@ func (z *Forward) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *Forward) EncodeMsg(en *msgp.Writer) (err error) {
-	err = en.WriteArrayHeader(3)
+	// array header, size 3
+	err = en.Append(0x93)
 	if err != nil {
-		return
+		return err
 	}
 	err = en.WriteString(z.Tag)
 	if err != nil {
@@ -154,16 +156,17 @@ func (z *Forward) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	for xvk := range z.Entries {
-		err = en.WriteArrayHeader(2)
+	for zbai := range z.Entries {
+		// array header, size 2
+		err = en.Append(0x92)
 		if err != nil {
-			return
+			return err
 		}
-		err = en.WriteInt64(z.Entries[xvk].Time)
+		err = en.WriteInt64(z.Entries[zbai].Time)
 		if err != nil {
 			return
 		}
-		err = en.WriteIntf(z.Entries[xvk].Record)
+		err = en.WriteIntf(z.Entries[zbai].Record)
 		if err != nil {
 			return
 		}
@@ -178,13 +181,15 @@ func (z *Forward) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *Forward) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	o = msgp.AppendArrayHeader(o, 3)
+	// array header, size 3
+	o = append(o, 0x93)
 	o = msgp.AppendString(o, z.Tag)
 	o = msgp.AppendArrayHeader(o, uint32(len(z.Entries)))
-	for xvk := range z.Entries {
-		o = msgp.AppendArrayHeader(o, 2)
-		o = msgp.AppendInt64(o, z.Entries[xvk].Time)
-		o, err = msgp.AppendIntf(o, z.Entries[xvk].Record)
+	for zbai := range z.Entries {
+		// array header, size 2
+		o = append(o, 0x92)
+		o = msgp.AppendInt64(o, z.Entries[zbai].Time)
+		o, err = msgp.AppendIntf(o, z.Entries[zbai].Record)
 		if err != nil {
 			return
 		}
@@ -198,48 +203,44 @@ func (z *Forward) MarshalMsg(b []byte) (o []byte, err error) {
 
 // UnmarshalMsg implements msgp.Unmarshaler
 func (z *Forward) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	{
-		var ssz uint32
-		ssz, bts, err = msgp.ReadArrayHeaderBytes(bts)
-		if err != nil {
-			return
-		}
-		if ssz != 3 {
-			err = msgp.ArrayError{Wanted: 3, Got: ssz}
-			return
-		}
+	var zhct uint32
+	zhct, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		return
+	}
+	if zhct != 3 {
+		err = msgp.ArrayError{Wanted: 3, Got: zhct}
+		return
 	}
 	z.Tag, bts, err = msgp.ReadStringBytes(bts)
 	if err != nil {
 		return
 	}
-	var xsz uint32
-	xsz, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	var zcua uint32
+	zcua, bts, err = msgp.ReadArrayHeaderBytes(bts)
 	if err != nil {
 		return
 	}
-	if cap(z.Entries) >= int(xsz) {
-		z.Entries = z.Entries[:xsz]
+	if cap(z.Entries) >= int(zcua) {
+		z.Entries = (z.Entries)[:zcua]
 	} else {
-		z.Entries = make([]Entry, xsz)
-	}
-	for xvk := range z.Entries {
-		{
-			var ssz uint32
-			ssz, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				return
-			}
-			if ssz != 2 {
-				err = msgp.ArrayError{Wanted: 2, Got: ssz}
-				return
-			}
+		z.Entries = make([]Entry, zcua)
+	}
+	for zbai := range z.Entries {
+		var zxhx uint32
+		zxhx, bts, err = msgp.ReadArrayHeaderBytes(bts)
+		if err != nil {
+			return
 		}
-		z.Entries[xvk].Time, bts, err = msgp.ReadInt64Bytes(bts)
+		if zxhx != 2 {
+			err = msgp.ArrayError{Wanted: 2, Got: zxhx}
+			return
+		}
+		z.Entries[zbai].Time, bts, err = msgp.ReadInt64Bytes(bts)
 		if err != nil {
 			return
 		}
-		z.Entries[xvk].Record, bts, err = msgp.ReadIntfBytes(bts)
+		z.Entries[zbai].Record, bts, err = msgp.ReadIntfBytes(bts)
 		if err != nil {
 			return
 		}
@@ -252,10 +253,11 @@ func (z *Forward) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *Forward) Msgsize() (s int) {
-	s = msgp.ArrayHeaderSize + msgp.StringPrefixSize + len(z.Tag) + msgp.ArrayHeaderSize
-	for xvk := range z.Entries {
-		s += msgp.ArrayHeaderSize + msgp.Int64Size + msgp.GuessSize(z.Entries[xvk].Record)
+	s = 1 + msgp.StringPrefixSize + len(z.Tag) + msgp.ArrayHeaderSize
+	for zbai := range z.Entries {
+		s += 1 + msgp.Int64Size + msgp.GuessSize(z.Entries[zbai].Record)
 	}
 	s += msgp.GuessSize(z.Option)
 	return
@@ -263,13 +265,13 @@ func (z *Forward) Msgsize() (s int) {
 
 // DecodeMsg implements msgp.Decodable
 func (z *Message) DecodeMsg(dc *msgp.Reader) (err error) {
-	var ssz uint32
-	ssz, err = dc.ReadArrayHeader()
+	var zlqf uint32
+	zlqf, err = dc.ReadArrayHeader()
 	if err != nil {
 		return
 	}
-	if ssz != 4 {
-		err = msgp.ArrayError{Wanted: 4, Got: ssz}
+	if zlqf != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zlqf}
 		return
 	}
 	z.Tag, err = dc.ReadString()
@@ -293,9 +295,10 @@ func (z *Message) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *Message) EncodeMsg(en *msgp.Writer) (err error) {
-	err = en.WriteArrayHeader(4)
+	// array header, size 4
+	err = en.Append(0x94)
 	if err != nil {
-		return
+		return err
 	}
 	err = en.WriteString(z.Tag)
 	if err != nil {
@@ -319,7 +322,8 @@ func (z *Message) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *Message) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	o = msgp.AppendArrayHeader(o, 4)
+	// array header, size 4
+	o = append(o, 0x94)
 	o = msgp.AppendString(o, z.Tag)
 	o = msgp.AppendInt64(o, z.Time)
 	o, err = msgp.AppendIntf(o, z.Record)
@@ -335,16 +339,14 @@ func (z *Message) MarshalMsg(b []byte) (o []byte, err error) {
 
 // UnmarshalMsg implements msgp.Unmarshaler
 func (z *Message) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	{
-		var ssz uint32
-		ssz, bts, err = msgp.ReadArrayHeaderBytes(bts)
-		if err != nil {
-			return
-		}
-		if ssz != 4 {
-			err = msgp.ArrayError{Wanted: 4, Got: ssz}
-			return
-		}
+	var zdaf uint32
+	zdaf, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		return
+	}
+	if zdaf != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zdaf}
+		return
 	}
 	z.Tag, bts, err = msgp.ReadStringBytes(bts)
 	if err != nil {
@@ -366,7 +368,122 @@ func (z *Message) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *Message) Msgsize() (s int) {
-	s = msgp.ArrayHeaderSize + msgp.StringPrefixSize + len(z.Tag) + msgp.Int64Size + msgp.GuessSize(z.Record) + msgp.GuessSize(z.Option)
+	s = 1 + msgp.StringPrefixSize + len(z.Tag) + msgp.Int64Size + msgp.GuessSize(z.Record) + msgp.GuessSize(z.Option)
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *MessageExt) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zpks uint32
+	zpks, err = dc.ReadArrayHeader()
+	if err != nil {
+		return
+	}
+	if zpks != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zpks}
+		return
+	}
+	z.Tag, err = dc.ReadString()
+	if err != nil {
+		return
+	}
+	err = dc.ReadExtension(&z.Time)
+	if err != nil {
+		return
+	}
+	z.Record, err = dc.ReadIntf()
+	if err != nil {
+		return
+	}
+	z.Option, err = dc.ReadIntf()
+	if err != nil {
+		return
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *MessageExt) EncodeMsg(en *msgp.Writer) (err error) {
+	// array header, size 4
+	err = en.Append(0x94)
+	if err != nil {
+		return err
+	}
+	err = en.WriteString(z.Tag)
+	if err != nil {
+		return
+	}
+	err = en.WriteExtension(&z.Time)
+	if err != nil {
+		return
+	}
+	err = en.WriteIntf(z.Record)
+	if err != nil {
+		return
+	}
+	err = en.WriteIntf(z.Option)
+	if err != nil {
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *MessageExt) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// array header, size 4
+	o = append(o, 0x94)
+	o = msgp.AppendString(o, z.Tag)
+	o, err = msgp.AppendExtension(o, &z.Time)
+	if err != nil {
+		return
+	}
+	o, err = msgp.AppendIntf(o, z.Record)
+	if err != nil {
+		return
+	}
+	o, err = msgp.AppendIntf(o, z.Option)
+	if err != nil {
+		return
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *MessageExt) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zjfb uint32
+	zjfb, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		return
+	}
+	if zjfb != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zjfb}
+		return
+	}
+	z.Tag, bts, err = msgp.ReadStringBytes(bts)
+	if err != nil {
+		return
+	}
+	bts, err = msgp.ReadExtensionBytes(bts, &z.Time)
+	if err != nil {
+		return
+	}
+	z.Record, bts, err = msgp.ReadIntfBytes(bts)
+	if err != nil {
+		return
+	}
+	z.Option, bts, err = msgp.ReadIntfBytes(bts)
+	if err != nil {
+		return
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *MessageExt) Msgsize() (s int) {
+	s = 1 + msgp.StringPrefixSize + len(z.Tag) + msgp.ExtensionPrefixSize + z.Time.Len() + msgp.GuessSize(z.Record) + msgp.GuessSize(z.Option)
 	return
 }

vendor/github.com/fluent/fluent-logger-golang/fluent/test_message.go

@@ -0,0 +1,7 @@
+package fluent
+
+//go:generate msgp
+type TestMessage struct {
+	Foo  string `msg:"foo" json:"foo,omitempty"`
+	Hoge string `msg:"hoge" json:"hoge,omitempty"`
+}

vendor/github.com/fluent/fluent-logger-golang/fluent/test_message_gen.go

@@ -0,0 +1,125 @@
+package fluent
+
+// NOTE: THIS FILE WAS PRODUCED BY THE
+// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
+// DO NOT EDIT
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *TestMessage) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zxvk uint32
+	zxvk, err = dc.ReadMapHeader()
+	if err != nil {
+		return
+	}
+	for zxvk > 0 {
+		zxvk--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "foo":
+			z.Foo, err = dc.ReadString()
+			if err != nil {
+				return
+			}
+		case "hoge":
+			z.Hoge, err = dc.ReadString()
+			if err != nil {
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z TestMessage) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "foo"
+	err = en.Append(0x82, 0xa3, 0x66, 0x6f, 0x6f)
+	if err != nil {
+		return err
+	}
+	err = en.WriteString(z.Foo)
+	if err != nil {
+		return
+	}
+	// write "hoge"
+	err = en.Append(0xa4, 0x68, 0x6f, 0x67, 0x65)
+	if err != nil {
+		return err
+	}
+	err = en.WriteString(z.Hoge)
+	if err != nil {
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z TestMessage) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "foo"
+	o = append(o, 0x82, 0xa3, 0x66, 0x6f, 0x6f)
+	o = msgp.AppendString(o, z.Foo)
+	// string "hoge"
+	o = append(o, 0xa4, 0x68, 0x6f, 0x67, 0x65)
+	o = msgp.AppendString(o, z.Hoge)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *TestMessage) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zbzg uint32
+	zbzg, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		return
+	}
+	for zbzg > 0 {
+		zbzg--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "foo":
+			z.Foo, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				return
+			}
+		case "hoge":
+			z.Hoge, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z TestMessage) Msgsize() (s int) {
+	s = 1 + 4 + msgp.StringPrefixSize + len(z.Foo) + 5 + msgp.StringPrefixSize + len(z.Hoge)
+	return
+}

vendor/github.com/fluent/fluent-logger-golang/fluent/version.go

@@ -1,3 +1,3 @@
 package fluent
 
-const Version = "1.2.1"
+const Version = "1.3.0"

vendor/github.com/tinylib/msgp/README.md

@@ -1,15 +1,12 @@
 MessagePack Code Generator [![Build Status](https://travis-ci.org/tinylib/msgp.svg?branch=master)](https://travis-ci.org/tinylib/msgp)
 =======
 
-[![forthebadge](http://forthebadge.com/badges/uses-badges.svg)](http://forthebadge.com)
-[![forthebadge](http://forthebadge.com/badges/ages-12.svg)](http://forthebadge.com)
-
-This is a code generation tool and serialization library for [MessagePack](http://msgpack.org). It is targeted at the `go generate` [tool](http://tip.golang.org/cmd/go/#hdr-Generate_Go_files_by_processing_source). You can read more about MessagePack [in the wiki](http://github.com/tinylib/msgp/wiki), or at [msgpack.org](http://msgpack.org).
+This is a code generation tool and serialization library for [MessagePack](http://msgpack.org). You can read more about MessagePack [in the wiki](http://github.com/tinylib/msgp/wiki), or at [msgpack.org](http://msgpack.org).
 
 ### Why?
 
 - Use Go as your schema language
-- Speeeeeed (400MB/s on modern hardware)
+- Performance
 - [JSON interop](http://godoc.org/github.com/tinylib/msgp/msgp#CopyToJSON)
 - [User-defined extensions](http://github.com/tinylib/msgp/wiki/Using-Extensions)
 - Type safety
@@ -17,8 +14,6 @@ This is a code generation tool and serialization library for [MessagePack](http:
 
 ### Quickstart
 
-Note: you need at least go 1.3 to compile this package, and at least go 1.4 to use `go generate`.
-
 In a source file, include the following directive:
 
 ```go
@@ -45,7 +40,7 @@ type Person struct {
 
 By default, the code generator will satisfy `msgp.Sizer`, `msgp.Encodable`, `msgp.Decodable`, 
 `msgp.Marshaler`, and `msgp.Unmarshaler`. Carefully-designed applications can use these methods to do
-marshalling/unmarshalling with zero allocations.
+marshalling/unmarshalling with zero heap allocations.
 
 While `msgp.Marshaler` and `msgp.Unmarshaler` are quite similar to the standard library's
 `json.Marshaler` and `json.Unmarshaler`, `msgp.Encodable` and `msgp.Decodable` are useful for 
@@ -62,6 +57,7 @@ of `*bufio.Writer` and `*bufio.Reader`, respectively.)
  - Generation of both `[]byte`-oriented and `io.Reader/io.Writer`-oriented methods
  - Support for arbitrary type system extensions
  - [Preprocessor directives](http://github.com/tinylib/msgp/wiki/Preprocessor-Directives)
+ - File-based dependency model means fast codegen regardless of source tree size.
 
 Consider the following:
 ```go
@@ -84,21 +80,23 @@ the data "type" (`int8`) and the raw binary. You [can see a worked example in th
 
 ### Status
 
-Alpha. I _will_ break stuff. There is an open milestone for Beta stability (targeted for January.) Only the `/msgp` sub-directory will have a stability guarantee.
+Mostly stable, in that no breaking changes have been made to the `/msgp` library in more than a year. Newer versions
+of the code may generate different code than older versions for performance reasons. I (@philhofer) am aware of a
+number of stability-critical commercial applications that use this code with good results. But, caveat emptor.
 
 You can read more about how `msgp` maps MessagePack types onto Go types [in the wiki](http://github.com/tinylib/msgp/wiki).
 
 Here some of the known limitations/restrictions:
 
- - Identifiers from outside the processed source file are assumed (optimistically) to satisfy the generator's interfaces. If this isn't the case, your code will fail to compile.
- - Like most serializers, `chan` and `func` fields are ignored, as well as non-exported fields.
- - Encoding of `interface{}` is limited to built-ins or types that have explicit encoding methods.
- - _Maps must have `string` keys._ This is intentional (as it preserves JSON interop.) Although non-string map keys are not forbidden by the MessagePack standard, many serializers impose this restriction. (It also means *any* well-formed `struct` can be de-serialized into a `map[string]interface{}`.) The only exception to this rule is that the deserializers will allow you to read map keys encoded as `bin` types, due to the fact that some legacy encodings permitted this. (However, those values will still be cast to Go `string`s, and they will be converted to `str` types when re-encoded. It is the responsibility of the user to ensure that map keys are UTF-8 safe in this case.) The same rules hold true for JSON translation.
+- Identifiers from outside the processed source file are assumed (optimistically) to satisfy the generator's interfaces. If this isn't the case, your code will fail to compile.
+- Like most serializers, `chan` and `func` fields are ignored, as well as non-exported fields.
+- Encoding of `interface{}` is limited to built-ins or types that have explicit encoding methods.
+- _Maps must have `string` keys._ This is intentional (as it preserves JSON interop.) Although non-string map keys are not forbidden by the MessagePack standard, many serializers impose this restriction. (It also means *any* well-formed `struct` can be de-serialized into a `map[string]interface{}`.) The only exception to this rule is that the deserializers will allow you to read map keys encoded as `bin` types, due to the fact that some legacy encodings permitted this. (However, those values will still be cast to Go `string`s, and they will be converted to `str` types when re-encoded. It is the responsibility of the user to ensure that map keys are UTF-8 safe in this case.) The same rules hold true for JSON translation.
 
 If the output compiles, then there's a pretty good chance things are fine. (Plus, we generate tests for you.) *Please, please, please* file an issue if you think the generator is writing broken code.
 
 ### Performance
 
-If you like benchmarks, see [here.](https://github.com/alecthomas/go_serialization_benchmarks)
+If you like benchmarks, see [here](http://bravenewgeek.com/so-you-wanna-go-fast/) and [here](https://github.com/alecthomas/go_serialization_benchmarks).
 
-As one might expect, the generated methods that deal with `[]byte` are faster, but the `io.Reader/Writer` methods are generally more memory-efficient for large (> 2KB) objects.
+As one might expect, the generated methods that deal with `[]byte` are faster for small objects, but the `io.Reader/Writer` methods are generally more memory-efficient (and, at some point, faster) for large (> 2KB) objects.

vendor/github.com/tinylib/msgp/msgp/advise_linux.go

@@ -0,0 +1,24 @@
+// +build linux,!appengine
+
+package msgp
+
+import (
+	"os"
+	"syscall"
+)
+
+func adviseRead(mem []byte) {
+	syscall.Madvise(mem, syscall.MADV_SEQUENTIAL|syscall.MADV_WILLNEED)
+}
+
+func adviseWrite(mem []byte) {
+	syscall.Madvise(mem, syscall.MADV_SEQUENTIAL)
+}
+
+func fallocate(f *os.File, sz int64) error {
+	err := syscall.Fallocate(int(f.Fd()), 0, 0, sz)
+	if err == syscall.ENOTSUP {
+		return f.Truncate(sz)
+	}
+	return err
+}

vendor/github.com/tinylib/msgp/msgp/advise_other.go

@@ -0,0 +1,17 @@
+// +build !linux appengine
+
+package msgp
+
+import (
+	"os"
+)
+
+// TODO: darwin, BSD support
+
+func adviseRead(mem []byte) {}
+
+func adviseWrite(mem []byte) {}
+
+func fallocate(f *os.File, sz int64) error {
+	return f.Truncate(sz)
+}

vendor/github.com/tinylib/msgp/msgp/appengine.go

@@ -0,0 +1,15 @@
+// +build appengine
+
+package msgp
+
+// let's just assume appengine
+// uses 64-bit hardware...
+const smallint = false
+
+func UnsafeString(b []byte) string {
+	return string(b)
+}
+
+func UnsafeBytes(s string) []byte {
+	return []byte(s)
+}

vendor/github.com/tinylib/msgp/msgp/circular.go

@@ -1,20 +1,21 @@
 package msgp
 
-import (
-	"testing"
-)
+type timer interface {
+	StartTimer()
+	StopTimer()
+}
 
 // EndlessReader is an io.Reader
 // that loops over the same data
 // endlessly. It is used for benchmarking.
 type EndlessReader struct {
-	tb     *testing.B
+	tb     timer
 	data   []byte
 	offset int
 }
 
 // NewEndlessReader returns a new endless reader
-func NewEndlessReader(b []byte, tb *testing.B) *EndlessReader {
+func NewEndlessReader(b []byte, tb timer) *EndlessReader {
 	return &EndlessReader{tb: tb, data: b, offset: 0}
 }
 

vendor/github.com/tinylib/msgp/msgp/extension.go

@@ -226,7 +226,7 @@ func (mw *Writer) WriteExtension(e Extension) error {
 // peek at the extension type, assuming the next
 // kind to be read is Extension
 func (m *Reader) peekExtensionType() (int8, error) {
-	p, err := m.r.Peek(2)
+	p, err := m.R.Peek(2)
 	if err != nil {
 		return 0, err
 	}
@@ -238,7 +238,7 @@ func (m *Reader) peekExtensionType() (int8, error) {
 		return int8(p[1]), nil
 	}
 	size := spec.size
-	p, err = m.r.Peek(int(size))
+	p, err = m.R.Peek(int(size))
 	if err != nil {
 		return 0, err
 	}
@@ -273,7 +273,7 @@ func peekExtension(b []byte) (int8, error) {
 // e.Type() is not the same as the wire type.
 func (m *Reader) ReadExtension(e Extension) (err error) {
 	var p []byte
-	p, err = m.r.Peek(2)
+	p, err = m.R.Peek(2)
 	if err != nil {
 		return
 	}
@@ -286,13 +286,13 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
 			err = errExt(int8(p[1]), e.ExtensionType())
 			return
 		}
-		p, err = m.r.Peek(3)
+		p, err = m.R.Peek(3)
 		if err != nil {
 			return
 		}
 		err = e.UnmarshalBinary(p[2:])
 		if err == nil {
-			_, err = m.r.Skip(3)
+			_, err = m.R.Skip(3)
 		}
 		return
 
@@ -301,13 +301,13 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
 			err = errExt(int8(p[1]), e.ExtensionType())
 			return
 		}
-		p, err = m.r.Peek(4)
+		p, err = m.R.Peek(4)
 		if err != nil {
 			return
 		}
 		err = e.UnmarshalBinary(p[2:])
 		if err == nil {
-			_, err = m.r.Skip(4)
+			_, err = m.R.Skip(4)
 		}
 		return
 
@@ -316,13 +316,13 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
 			err = errExt(int8(p[1]), e.ExtensionType())
 			return
 		}
-		p, err = m.r.Peek(6)
+		p, err = m.R.Peek(6)
 		if err != nil {
 			return
 		}
 		err = e.UnmarshalBinary(p[2:])
 		if err == nil {
-			_, err = m.r.Skip(6)
+			_, err = m.R.Skip(6)
 		}
 		return
 
@@ -331,13 +331,13 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
 			err = errExt(int8(p[1]), e.ExtensionType())
 			return
 		}
-		p, err = m.r.Peek(10)
+		p, err = m.R.Peek(10)
 		if err != nil {
 			return
 		}
 		err = e.UnmarshalBinary(p[2:])
 		if err == nil {
-			_, err = m.r.Skip(10)
+			_, err = m.R.Skip(10)
 		}
 		return
 
@@ -346,18 +346,18 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
 			err = errExt(int8(p[1]), e.ExtensionType())
 			return
 		}
-		p, err = m.r.Peek(18)
+		p, err = m.R.Peek(18)
 		if err != nil {
 			return
 		}
 		err = e.UnmarshalBinary(p[2:])
 		if err == nil {
-			_, err = m.r.Skip(18)
+			_, err = m.R.Skip(18)
 		}
 		return
 
 	case mext8:
-		p, err = m.r.Peek(3)
+		p, err = m.R.Peek(3)
 		if err != nil {
 			return
 		}
@@ -369,7 +369,7 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
 		off = 3
 
 	case mext16:
-		p, err = m.r.Peek(4)
+		p, err = m.R.Peek(4)
 		if err != nil {
 			return
 		}
@@ -381,7 +381,7 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
 		off = 4
 
 	case mext32:
-		p, err = m.r.Peek(6)
+		p, err = m.R.Peek(6)
 		if err != nil {
 			return
 		}
@@ -397,13 +397,13 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
 		return
 	}
 
-	p, err = m.r.Peek(read + off)
+	p, err = m.R.Peek(read + off)
 	if err != nil {
 		return
 	}
 	err = e.UnmarshalBinary(p[off:])
 	if err == nil {
-		_, err = m.r.Skip(read + off)
+		_, err = m.R.Skip(read + off)
 	}
 	return
 }

vendor/github.com/tinylib/msgp/msgp/file.go

@@ -0,0 +1,92 @@
+// +build linux darwin dragonfly freebsd netbsd openbsd
+// +build !appengine
+
+package msgp
+
+import (
+	"os"
+	"syscall"
+)
+
+// ReadFile reads a file into 'dst' using
+// a read-only memory mapping. Consequently,
+// the file must be mmap-able, and the
+// Unmarshaler should never write to
+// the source memory. (Methods generated
+// by the msgp tool obey that constraint, but
+// user-defined implementations may not.)
+//
+// Reading and writing through file mappings
+// is only efficient for large files; small
+// files are best read and written using
+// the ordinary streaming interfaces.
+//
+func ReadFile(dst Unmarshaler, file *os.File) error {
+	stat, err := file.Stat()
+	if err != nil {
+		return err
+	}
+	data, err := syscall.Mmap(int(file.Fd()), 0, int(stat.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
+	if err != nil {
+		return err
+	}
+	adviseRead(data)
+	_, err = dst.UnmarshalMsg(data)
+	uerr := syscall.Munmap(data)
+	if err == nil {
+		err = uerr
+	}
+	return err
+}
+
+// MarshalSizer is the combination
+// of the Marshaler and Sizer
+// interfaces.
+type MarshalSizer interface {
+	Marshaler
+	Sizer
+}
+
+// WriteFile writes a file from 'src' using
+// memory mapping. It overwrites the entire
+// contents of the previous file.
+// The mapping size is calculated
+// using the `Msgsize()` method
+// of 'src', so it must produce a result
+// equal to or greater than the actual encoded
+// size of the object. Otherwise,
+// a fault (SIGBUS) will occur.
+//
+// Reading and writing through file mappings
+// is only efficient for large files; small
+// files are best read and written using
+// the ordinary streaming interfaces.
+//
+// NOTE: The performance of this call
+// is highly OS- and filesystem-dependent.
+// Users should take care to test that this
+// performs as expected in a production environment.
+// (Linux users should run a kernel and filesystem
+// that support fallocate(2) for the best results.)
+func WriteFile(src MarshalSizer, file *os.File) error {
+	sz := src.Msgsize()
+	err := fallocate(file, int64(sz))
+	if err != nil {
+		return err
+	}
+	data, err := syscall.Mmap(int(file.Fd()), 0, sz, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
+	if err != nil {
+		return err
+	}
+	adviseWrite(data)
+	chunk := data[:0]
+	chunk, err = src.MarshalMsg(chunk)
+	if err != nil {
+		return err
+	}
+	uerr := syscall.Munmap(data)
+	if uerr != nil {
+		return uerr
+	}
+	return file.Truncate(int64(len(chunk)))
+}

vendor/github.com/tinylib/msgp/msgp/file_port.go

@@ -0,0 +1,47 @@
+// +build windows appengine
+
+package msgp
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+// MarshalSizer is the combination
+// of the Marshaler and Sizer
+// interfaces.
+type MarshalSizer interface {
+	Marshaler
+	Sizer
+}
+
+func ReadFile(dst Unmarshaler, file *os.File) error {
+	if u, ok := dst.(Decodable); ok {
+		return u.DecodeMsg(NewReader(file))
+	}
+
+	data, err := ioutil.ReadAll(file)
+	if err != nil {
+		return err
+	}
+	_, err = dst.UnmarshalMsg(data)
+	return err
+}
+
+func WriteFile(src MarshalSizer, file *os.File) error {
+	if e, ok := src.(Encodable); ok {
+		w := NewWriter(file)
+		err := e.EncodeMsg(w)
+		if err == nil {
+			err = w.Flush()
+		}
+		return err
+	}
+
+	raw, err := src.MarshalMsg(nil)
+	if err != nil {
+		return err
+	}
+	_, err = file.Write(raw)
+	return err
+}

vendor/github.com/tinylib/msgp/msgp/json.go

@@ -66,7 +66,7 @@ func (r *Reader) WriteToJSON(w io.Writer) (n int64, err error) {
 	if jsw, ok := w.(jsWriter); ok {
 		j = jsw
 	} else {
-		bf = bufio.NewWriterSize(w, 512)
+		bf = bufio.NewWriter(w)
 		j = bf
 	}
 	var nn int
@@ -333,7 +333,7 @@ func rwExtension(dst jsWriter, src *Reader) (n int, err error) {
 
 func rwString(dst jsWriter, src *Reader) (n int, err error) {
 	var p []byte
-	p, err = src.r.Peek(1)
+	p, err = src.R.Peek(1)
 	if err != nil {
 		return
 	}
@@ -342,25 +342,25 @@ func rwString(dst jsWriter, src *Reader) (n int, err error) {
 
 	if isfixstr(lead) {
 		read = int(rfixstr(lead))
-		src.r.Skip(1)
+		src.R.Skip(1)
 		goto write
 	}
 
 	switch lead {
 	case mstr8:
-		p, err = src.r.Next(2)
+		p, err = src.R.Next(2)
 		if err != nil {
 			return
 		}
 		read = int(uint8(p[1]))
 	case mstr16:
-		p, err = src.r.Next(3)
+		p, err = src.R.Next(3)
 		if err != nil {
 			return
 		}
 		read = int(big.Uint16(p[1:]))
 	case mstr32:
-		p, err = src.r.Next(5)
+		p, err = src.R.Next(5)
 		if err != nil {
 			return
 		}
@@ -370,7 +370,7 @@ func rwString(dst jsWriter, src *Reader) (n int, err error) {
 		return
 	}
 write:
-	p, err = src.r.Next(read)
+	p, err = src.R.Next(read)
 	if err != nil {
 		return
 	}

vendor/github.com/tinylib/msgp/msgp/number.go

@@ -1,11 +1,105 @@
 package msgp
 
 import (
+	"math"
 	"strconv"
 )
 
 // The portable parts of the Number implementation
 
+// Number can be
+// an int64, uint64, float32,
+// or float64 internally.
+// It can decode itself
+// from any of the native
+// messagepack number types.
+// The zero-value of Number
+// is Int(0). Using the equality
+// operator with Number compares
+// both the type and the value
+// of the number.
+type Number struct {
+	// internally, this
+	// is just a tagged union.
+	// the raw bits of the number
+	// are stored the same way regardless.
+	bits uint64
+	typ  Type
+}
+
+// AsInt sets the number to an int64.
+func (n *Number) AsInt(i int64) {
+
+	// we always store int(0)
+	// as {0, InvalidType} in
+	// order to preserve
+	// the behavior of the == operator
+	if i == 0 {
+		n.typ = InvalidType
+		n.bits = 0
+		return
+	}
+
+	n.typ = IntType
+	n.bits = uint64(i)
+}
+
+// AsUint sets the number to a uint64.
+func (n *Number) AsUint(u uint64) {
+	n.typ = UintType
+	n.bits = u
+}
+
+// AsFloat32 sets the value of the number
+// to a float32.
+func (n *Number) AsFloat32(f float32) {
+	n.typ = Float32Type
+	n.bits = uint64(math.Float32bits(f))
+}
+
+// AsFloat64 sets the value of the
+// number to a float64.
+func (n *Number) AsFloat64(f float64) {
+	n.typ = Float64Type
+	n.bits = math.Float64bits(f)
+}
+
+// Int casts the number as an int64, and
+// returns whether or not that was the
+// underlying type.
+func (n *Number) Int() (int64, bool) {
+	return int64(n.bits), n.typ == IntType || n.typ == InvalidType
+}
+
+// Uint casts the number as a uint64, and returns
+// whether or not that was the underlying type.
+func (n *Number) Uint() (uint64, bool) {
+	return n.bits, n.typ == UintType
+}
+
+// Float casts the number to a float64, and
+// returns whether or not that was the underlying
+// type (either a float64 or a float32).
+func (n *Number) Float() (float64, bool) {
+	switch n.typ {
+	case Float32Type:
+		return float64(math.Float32frombits(uint32(n.bits))), true
+	case Float64Type:
+		return math.Float64frombits(n.bits), true
+	default:
+		return 0.0, false
+	}
+}
+
+// Type will return one of:
+// Float64Type, Float32Type, UintType, or IntType.
+func (n *Number) Type() Type {
+	if n.typ == InvalidType {
+		return IntType
+	}
+	return n.typ
+}
+
 // DecodeMsg implements msgp.Decodable
 func (n *Number) DecodeMsg(r *Reader) error {
 	typ, err := r.NextType()
@@ -83,6 +177,38 @@ func (n *Number) UnmarshalMsg(b []byte) ([]byte, error) {
 	}
 }
 
+// MarshalMsg implements msgp.Marshaler
+func (n *Number) MarshalMsg(b []byte) ([]byte, error) {
+	switch n.typ {
+	case IntType:
+		return AppendInt64(b, int64(n.bits)), nil
+	case UintType:
+		return AppendUint64(b, uint64(n.bits)), nil
+	case Float64Type:
+		return AppendFloat64(b, math.Float64frombits(n.bits)), nil
+	case Float32Type:
+		return AppendFloat32(b, math.Float32frombits(uint32(n.bits))), nil
+	default:
+		return AppendInt64(b, 0), nil
+	}
+}
+
+// EncodeMsg implements msgp.Encodable
+func (n *Number) EncodeMsg(w *Writer) error {
+	switch n.typ {
+	case IntType:
+		return w.WriteInt64(int64(n.bits))
+	case UintType:
+		return w.WriteUint64(n.bits)
+	case Float64Type:
+		return w.WriteFloat64(math.Float64frombits(n.bits))
+	case Float32Type:
+		return w.WriteFloat32(math.Float32frombits(uint32(n.bits)))
+	default:
+		return w.WriteInt64(0)
+	}
+}
+
 // Msgsize implements msgp.Sizer
 func (n *Number) Msgsize() int {
 	switch n.typ {
@@ -121,6 +247,7 @@ func (n *Number) MarshalJSON() ([]byte, error) {
 	}
 }
 
+// String implements fmt.Stringer
 func (n *Number) String() string {
 	switch n.typ {
 	case InvalidType:

vendor/github.com/tinylib/msgp/msgp/number_appengine.go

@@ -1,101 +0,0 @@
-// +build appengine
-
-package msgp
-
-// let's just assume appengine
-// uses 64-bit hardware...
-const smallint = false
-
-func UnsafeString(b []byte) string {
-	return string(b)
-}
-
-func UnsafeBytes(s string) []byte {
-	return []byte(s)
-}
-
-type Number struct {
-	ibits uint64  // zero or bits
-	fbits float64 // zero or bits
-	typ   Type    // zero or type
-}
-
-func (n *Number) AsFloat64(f float64) {
-	n.typ = Float64Type
-	n.fbits = f
-	n.ibits = 0
-}
-
-func (n *Number) AsFloat32(f float32) {
-	n.typ = Float32Type
-	n.fbits = float64(f)
-	n.ibits = 0
-}
-
-func (n *Number) AsInt(i int64) {
-	n.fbits = 0
-	if i == 0 {
-		n.typ = InvalidType
-		n.ibits = 0
-		return
-	}
-	n.ibits = uint64(i)
-	n.typ = IntType
-}
-
-func (n *Number) AsUint(u uint64) {
-	n.ibits = u
-	n.fbits = 0
-	n.typ = UintType
-}
-
-func (n *Number) Float() (float64, bool) {
-	return n.fbits, n.typ == Float64Type || n.typ == Float32Type
-}
-
-func (n *Number) Int() (int64, bool) {
-	return int64(n.ibits), n.typ == IntType
-}
-
-func (n *Number) Uint() (uint64, bool) {
-	return n.ibits, n.typ == UintType
-}
-
-func (n *Number) Type() Type {
-	if n.typ == InvalidType {
-		return IntType
-	}
-	return n.typ
-}
-
-func (n *Number) MarshalMsg(o []byte) ([]byte, error) {
-	switch n.typ {
-	case InvalidType:
-		return AppendInt64(o, 0), nil
-	case IntType:
-		return AppendInt64(o, int64(n.ibits)), nil
-	case UintType:
-		return AppendUint64(o, n.ibits), nil
-	case Float32Type:
-		return AppendFloat32(o, float32(n.fbits)), nil
-	case Float64Type:
-		return AppendFloat64(o, n.fbits), nil
-	}
-	panic("unreachable code!")
-}
-
-func (n *Number) EncodeMsg(w *Writer) error {
-	switch n.typ {
-	case InvalidType:
-		return w.WriteInt64(0)
-	case IntType:
-		return w.WriteInt64(int64(n.ibits))
-	case UintType:
-		return w.WriteUint64(n.ibits)
-	case Float32Type:
-		return w.WriteFloat32(float32(n.fbits))
-	case Float64Type:
-		return w.WriteFloat64(n.fbits)
-	}
-	panic("unreachable code!")
-}

vendor/github.com/tinylib/msgp/msgp/number_unsafe.go

@@ -1,159 +0,0 @@
-// +build !appengine
-
-package msgp
-
-import (
-	"reflect"
-	"unsafe"
-)
-
-const (
-	// spec says int and uint are always
-	// the same size, but that int/uint
-	// size may not be machine word size
-	smallint = unsafe.Sizeof(int(0)) == 4
-)
-
-// UnsafeString returns the byte slice as a volatile string
-// THIS SHOULD ONLY BE USED BY THE CODE GENERATOR.
-// THIS IS EVIL CODE.
-// YOU HAVE BEEN WARNED.
-func UnsafeString(b []byte) string {
-	return *(*string)(unsafe.Pointer(&reflect.StringHeader{Data: uintptr(unsafe.Pointer(&b[0])), Len: len(b)}))
-}
-
-// UnsafeBytes returns the string as a byte slice
-// THIS SHOULD ONLY BE USED BY THE CODE GENERATOR.
-// THIS IS EVIL CODE.
-// YOU HAVE BEEN WARNED.
-func UnsafeBytes(s string) []byte {
-	return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
-		Len:  len(s),
-		Cap:  len(s),
-		Data: (*(*reflect.StringHeader)(unsafe.Pointer(&s))).Data,
-	}))
-}
-
-// Number can be
-// an int64, uint64, float32,
-// or float64 internally.
-// It can decode itself
-// from any of the native
-// messagepack number types.
-// The zero-value of Number
-// is Int(0). Using the equality
-// operator with Number compares
-// both the type and the value
-// of the number.
-type Number struct {
-	// internally, this
-	// is just a tagged union.
-	// the raw bits of the number
-	// are stored the same way regardless.
-	bits uint64
-	typ  Type
-}
-
-// AsFloat64 sets the number to
-// a float64.
-func (n *Number) AsFloat64(f float64) {
-	n.typ = Float64Type
-	n.bits = *(*uint64)(unsafe.Pointer(&f))
-}
-
-// AsInt sets the number to an int64.
-func (n *Number) AsInt(i int64) {
-
-	// we always store int(0)
-	// as {0, InvalidType} in
-	// order to preserve
-	// the behavior of the == operator
-	if i == 0 {
-		n.typ = InvalidType
-		n.bits = 0
-		return
-	}
-
-	n.typ = IntType
-	n.bits = uint64(i)
-}
-
-// AsUint sets the number to a uint64.
-func (n *Number) AsUint(u uint64) {
-	n.typ = UintType
-	n.bits = u
-}
-
-// AsFloat32 sets the number to a float32.
-func (n *Number) AsFloat32(f float32) {
-	n.typ = Float32Type
-	g := float64(f)
-	n.bits = *(*uint64)(unsafe.Pointer(&g))
-}
-
-// Type will return one of:
-// Float64Type, Float32Type, UintType, or IntType.
-func (n *Number) Type() Type {
-	if n.typ == InvalidType {
-		return IntType
-	}
-	return n.typ
-}
-
-// Float casts the number of the float,
-// and returns whether or not that was
-// the underlying type. (This is legal
-// for both float32 and float64 types.)
-func (n *Number) Float() (float64, bool) {
-	return *(*float64)(unsafe.Pointer(&n.bits)), n.typ == Float64Type || n.typ == Float32Type
-}
-
-// Int casts the number as an int64, and
-// returns whether or not that was the
-// underlying type.
-func (n *Number) Int() (int64, bool) {
-	return int64(n.bits), n.typ == IntType || n.typ == InvalidType
-}
-
-// Uint casts the number as a uint64, and returns
-// whether or not that was the underlying type.
-func (n *Number) Uint() (uint64, bool) {
-	return n.bits, n.typ == UintType
-}
-
-// EncodeMsg implements msgp.Encodable
-func (n *Number) EncodeMsg(w *Writer) error {
-	switch n.typ {
-	case InvalidType:
-		return w.WriteInt(0)
-	case IntType:
-		return w.WriteInt64(int64(n.bits))
-	case UintType:
-		return w.WriteUint64(n.bits)
-	case Float64Type:
-		return w.WriteFloat64(*(*float64)(unsafe.Pointer(&n.bits)))
-	case Float32Type:
-		return w.WriteFloat32(float32(*(*float64)(unsafe.Pointer(&n.bits))))
-	default:
-		// this should never ever happen
-		panic("(*Number).typ is invalid")
-	}
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (n *Number) MarshalMsg(b []byte) ([]byte, error) {
-	switch n.typ {
-	case InvalidType:
-		return AppendInt(b, 0), nil
-	case IntType:
-		return AppendInt64(b, int64(n.bits)), nil
-	case UintType:
-		return AppendUint64(b, n.bits), nil
-	case Float64Type:
-		return AppendFloat64(b, *(*float64)(unsafe.Pointer(&n.bits))), nil
-	case Float32Type:
-		return AppendFloat32(b, float32(*(*float64)(unsafe.Pointer(&n.bits)))), nil
-	default:
-		panic("(*Number).typ is invalid")
-	}
-}

vendor/github.com/tinylib/msgp/msgp/read.go

@@ -1,11 +1,12 @@
 package msgp
 
 import (
-	"github.com/philhofer/fwd"
 	"io"
 	"math"
 	"sync"
 	"time"
+
+	"github.com/philhofer/fwd"
 )
 
 // where we keep old *Readers
@@ -111,10 +112,10 @@ func Decode(r io.Reader, d Decodable) error {
 // reader will be buffered.
 func NewReader(r io.Reader) *Reader {
 	p := readerPool.Get().(*Reader)
-	if p.r == nil {
-		p.r = fwd.NewReader(r)
+	if p.R == nil {
+		p.R = fwd.NewReader(r)
 	} else {
-		p.r.Reset(r)
+		p.R.Reset(r)
 	}
 	return p
 }
@@ -122,39 +123,96 @@ func NewReader(r io.Reader) *Reader {
 // NewReaderSize returns a *Reader with a buffer of the given size.
 // (This is vastly preferable to passing the decoder a reader that is already buffered.)
 func NewReaderSize(r io.Reader, sz int) *Reader {
-	return &Reader{r: fwd.NewReaderSize(r, sz)}
+	return &Reader{R: fwd.NewReaderSize(r, sz)}
 }
 
 // Reader wraps an io.Reader and provides
 // methods to read MessagePack-encoded values
 // from it. Readers are buffered.
 type Reader struct {
-	r       *fwd.Reader
+	// R is the buffered reader
+	// that the Reader uses
+	// to decode MessagePack.
+	// The Reader itself
+	// is stateless; all the
+	// buffering is done
+	// within R.
+	R       *fwd.Reader
 	scratch []byte
 }
 
 // Read implements `io.Reader`
 func (m *Reader) Read(p []byte) (int, error) {
-	return m.r.Read(p)
+	return m.R.Read(p)
+}
+
+// CopyNext reads the next object from m without decoding it and writes it to w.
+// It avoids unnecessary copies internally.
+func (m *Reader) CopyNext(w io.Writer) (int64, error) {
+	sz, o, err := getNextSize(m.R)
+	if err != nil {
+		return 0, err
+	}
+
+	var n int64
+	// Opportunistic optimization: if we can fit the whole thing in the m.R
+	// buffer, then just get a pointer to that, and pass it to w.Write,
+	// avoiding an allocation.
+	if int(sz) <= m.R.BufferSize() {
+		var nn int
+		var buf []byte
+		buf, err = m.R.Next(int(sz))
+		if err != nil {
+			if err == io.ErrUnexpectedEOF {
+				err = ErrShortBytes
+			}
+			return 0, err
+		}
+		nn, err = w.Write(buf)
+		n += int64(nn)
+	} else {
+		// Fall back to io.CopyN.
+		// May avoid allocating if w is a ReaderFrom (e.g. bytes.Buffer)
+		n, err = io.CopyN(w, m.R, int64(sz))
+		if err == io.ErrUnexpectedEOF {
+			err = ErrShortBytes
+		}
+	}
+	if err != nil {
+		return n, err
+	} else if n < int64(sz) {
+		return n, io.ErrShortWrite
+	}
+
+	// for maps and slices, read elements
+	for x := uintptr(0); x < o; x++ {
+		var n2 int64
+		n2, err = m.CopyNext(w)
+		if err != nil {
+			return n, err
+		}
+		n += n2
+	}
+	return n, nil
 }
 
 // ReadFull implements `io.ReadFull`
 func (m *Reader) ReadFull(p []byte) (int, error) {
-	return m.r.ReadFull(p)
+	return m.R.ReadFull(p)
 }
 
 // Reset resets the underlying reader.
-func (m *Reader) Reset(r io.Reader) { m.r.Reset(r) }
+func (m *Reader) Reset(r io.Reader) { m.R.Reset(r) }
 
 // Buffered returns the number of bytes currently in the read buffer.
-func (m *Reader) Buffered() int { return m.r.Buffered() }
+func (m *Reader) Buffered() int { return m.R.Buffered() }
 
 // BufferSize returns the capacity of the read buffer.
-func (m *Reader) BufferSize() int { return m.r.BufferSize() }
+func (m *Reader) BufferSize() int { return m.R.BufferSize() }
 
 // NextType returns the next object type to be decoded.
 func (m *Reader) NextType() (Type, error) {
-	p, err := m.r.Peek(1)
+	p, err := m.R.Peek(1)
 	if err != nil {
 		return InvalidType, err
 	}
@@ -182,12 +240,14 @@ func (m *Reader) NextType() (Type, error) {
 // IsNil returns whether or not
 // the next byte is a null messagepack byte
 func (m *Reader) IsNil() bool {
-	p, err := m.r.Peek(1)
+	p, err := m.R.Peek(1)
 	return err == nil && p[0] == mnil
 }
 
+// getNextSize returns the size of the next object on the wire.
 // returns (obj size, obj elements, error)
 // only maps and arrays have non-zero obj elements
+// for maps and arrays, obj size does not include elements
 //
 // use uintptr b/c it's guaranteed to be large enough
 // to hold whatever we can fit in memory.
@@ -243,8 +303,8 @@ func (m *Reader) Skip() error {
 	// we can use the faster
 	// method if we have enough
 	// buffered data
-	if m.r.Buffered() >= 5 {
-		p, err = m.r.Peek(5)
+	if m.R.Buffered() >= 5 {
+		p, err = m.R.Peek(5)
 		if err != nil {
 			return err
 		}
@@ -253,7 +313,7 @@ func (m *Reader) Skip() error {
 			return err
 		}
 	} else {
-		v, o, err = getNextSize(m.r)
+		v, o, err = getNextSize(m.R)
 		if err != nil {
 			return err
 		}
@@ -261,7 +321,7 @@ func (m *Reader) Skip() error {
 
 	// 'v' is always non-zero
 	// if err == nil
-	_, err = m.r.Skip(int(v))
+	_, err = m.R.Skip(int(v))
 	if err != nil {
 		return err
 	}
@@ -284,26 +344,26 @@ func (m *Reader) Skip() error {
 func (m *Reader) ReadMapHeader() (sz uint32, err error) {
 	var p []byte
 	var lead byte
-	p, err = m.r.Peek(1)
+	p, err = m.R.Peek(1)
 	if err != nil {
 		return
 	}
 	lead = p[0]
 	if isfixmap(lead) {
 		sz = uint32(rfixmap(lead))
-		_, err = m.r.Skip(1)
+		_, err = m.R.Skip(1)
 		return
 	}
 	switch lead {
 	case mmap16:
-		p, err = m.r.Next(3)
+		p, err = m.R.Next(3)
 		if err != nil {
 			return
 		}
 		sz = uint32(big.Uint16(p[1:]))
 		return
 	case mmap32:
-		p, err = m.r.Next(5)
+		p, err = m.R.Next(5)
 		if err != nil {
 			return
 		}
@@ -338,7 +398,7 @@ func (m *Reader) ReadMapKey(scratch []byte) ([]byte, error) {
 // method; writing into the returned slice may
 // corrupt future reads.
 func (m *Reader) ReadMapKeyPtr() ([]byte, error) {
-	p, err := m.r.Peek(1)
+	p, err := m.R.Peek(1)
 	if err != nil {
 		return nil, err
 	}
@@ -346,24 +406,24 @@ func (m *Reader) ReadMapKeyPtr() ([]byte, error) {
 	var read int
 	if isfixstr(lead) {
 		read = int(rfixstr(lead))
-		m.r.Skip(1)
+		m.R.Skip(1)
 		goto fill
 	}
 	switch lead {
 	case mstr8, mbin8:
-		p, err = m.r.Next(2)
+		p, err = m.R.Next(2)
 		if err != nil {
 			return nil, err
 		}
 		read = int(p[1])
 	case mstr16, mbin16:
-		p, err = m.r.Next(3)
+		p, err = m.R.Next(3)
 		if err != nil {
 			return nil, err
 		}
 		read = int(big.Uint16(p[1:]))
 	case mstr32, mbin32:
-		p, err = m.r.Next(5)
+		p, err = m.R.Next(5)
 		if err != nil {
 			return nil, err
 		}
@@ -375,7 +435,7 @@ fill:
 	if read == 0 {
 		return nil, ErrShortBytes
 	}
-	return m.r.Next(read)
+	return m.R.Next(read)
 }
 
 // ReadArrayHeader reads the next object as an
@@ -384,19 +444,19 @@ fill:
 func (m *Reader) ReadArrayHeader() (sz uint32, err error) {
 	var lead byte
 	var p []byte
-	p, err = m.r.Peek(1)
+	p, err = m.R.Peek(1)
 	if err != nil {
 		return
 	}
 	lead = p[0]
 	if isfixarray(lead) {
 		sz = uint32(rfixarray(lead))
-		_, err = m.r.Skip(1)
+		_, err = m.R.Skip(1)
 		return
 	}
 	switch lead {
 	case marray16:
-		p, err = m.r.Next(3)
+		p, err = m.R.Next(3)
 		if err != nil {
 			return
 		}
@@ -404,7 +464,7 @@ func (m *Reader) ReadArrayHeader() (sz uint32, err error) {
 		return
 
 	case marray32:
-		p, err = m.r.Next(5)
+		p, err = m.R.Next(5)
 		if err != nil {
 			return
 		}
@@ -419,14 +479,14 @@ func (m *Reader) ReadArrayHeader() (sz uint32, err error) {
 
 // ReadNil reads a 'nil' MessagePack byte from the reader
 func (m *Reader) ReadNil() error {
-	p, err := m.r.Peek(1)
+	p, err := m.R.Peek(1)
 	if err != nil {
 		return err
 	}
 	if p[0] != mnil {
 		return badPrefix(NilType, p[0])
 	}
-	_, err = m.r.Skip(1)
+	_, err = m.R.Skip(1)
 	return err
 }
 
@@ -435,7 +495,7 @@ func (m *Reader) ReadNil() error {
 // it will be up-cast to a float64.)
 func (m *Reader) ReadFloat64() (f float64, err error) {
 	var p []byte
-	p, err = m.r.Peek(9)
+	p, err = m.R.Peek(9)
 	if err != nil {
 		// we'll allow a coversion from float32 to float64,
 		// since we don't lose any precision
@@ -455,14 +515,14 @@ func (m *Reader) ReadFloat64() (f float64, err error) {
 		return
 	}
 	f = math.Float64frombits(getMuint64(p))
-	_, err = m.r.Skip(9)
+	_, err = m.R.Skip(9)
 	return
 }
 
 // ReadFloat32 reads a float32 from the reader
 func (m *Reader) ReadFloat32() (f float32, err error) {
 	var p []byte
-	p, err = m.r.Peek(5)
+	p, err = m.R.Peek(5)
 	if err != nil {
 		return
 	}
@@ -471,14 +531,14 @@ func (m *Reader) ReadFloat32() (f float32, err error) {
 		return
 	}
 	f = math.Float32frombits(getMuint32(p))
-	_, err = m.r.Skip(5)
+	_, err = m.R.Skip(5)
 	return
 }
 
 // ReadBool reads a bool from the reader
 func (m *Reader) ReadBool() (b bool, err error) {
 	var p []byte
-	p, err = m.r.Peek(1)
+	p, err = m.R.Peek(1)
 	if err != nil {
 		return
 	}
@@ -490,7 +550,7 @@ func (m *Reader) ReadBool() (b bool, err error) {
 		err = badPrefix(BoolType, p[0])
 		return
 	}
-	_, err = m.r.Skip(1)
+	_, err = m.R.Skip(1)
 	return
 }
 
@@ -498,7 +558,7 @@ func (m *Reader) ReadBool() (b bool, err error) {
 func (m *Reader) ReadInt64() (i int64, err error) {
 	var p []byte
 	var lead byte
-	p, err = m.r.Peek(1)
+	p, err = m.R.Peek(1)
 	if err != nil {
 		return
 	}
@@ -506,17 +566,17 @@ func (m *Reader) ReadInt64() (i int64, err error) {
 
 	if isfixint(lead) {
 		i = int64(rfixint(lead))
-		_, err = m.r.Skip(1)
+		_, err = m.R.Skip(1)
 		return
 	} else if isnfixint(lead) {
 		i = int64(rnfixint(lead))
-		_, err = m.r.Skip(1)
+		_, err = m.R.Skip(1)
 		return
 	}
 
 	switch lead {
 	case mint8:
-		p, err = m.r.Next(2)
+		p, err = m.R.Next(2)
 		if err != nil {
 			return
 		}
@@ -524,7 +584,7 @@ func (m *Reader) ReadInt64() (i int64, err error) {
 		return
 
 	case mint16:
-		p, err = m.r.Next(3)
+		p, err = m.R.Next(3)
 		if err != nil {
 			return
 		}
@@ -532,7 +592,7 @@ func (m *Reader) ReadInt64() (i int64, err error) {
 		return
 
 	case mint32:
-		p, err = m.r.Next(5)
+		p, err = m.R.Next(5)
 		if err != nil {
 			return
 		}
@@ -540,7 +600,7 @@ func (m *Reader) ReadInt64() (i int64, err error) {
 		return
 
 	case mint64:
-		p, err = m.r.Next(9)
+		p, err = m.R.Next(9)
 		if err != nil {
 			return
 		}
@@ -607,19 +667,19 @@ func (m *Reader) ReadInt() (i int, err error) {
 func (m *Reader) ReadUint64() (u uint64, err error) {
 	var p []byte
 	var lead byte
-	p, err = m.r.Peek(1)
+	p, err = m.R.Peek(1)
 	if err != nil {
 		return
 	}
 	lead = p[0]
 	if isfixint(lead) {
 		u = uint64(rfixint(lead))
-		_, err = m.r.Skip(1)
+		_, err = m.R.Skip(1)
 		return
 	}
 	switch lead {
 	case muint8:
-		p, err = m.r.Next(2)
+		p, err = m.R.Next(2)
 		if err != nil {
 			return
 		}
@@ -627,7 +687,7 @@ func (m *Reader) ReadUint64() (u uint64, err error) {
 		return
 
 	case muint16:
-		p, err = m.r.Next(3)
+		p, err = m.R.Next(3)
 		if err != nil {
 			return
 		}
@@ -635,7 +695,7 @@ func (m *Reader) ReadUint64() (u uint64, err error) {
 		return
 
 	case muint32:
-		p, err = m.r.Next(5)
+		p, err = m.R.Next(5)
 		if err != nil {
 			return
 		}
@@ -643,7 +703,7 @@ func (m *Reader) ReadUint64() (u uint64, err error) {
 		return
 
 	case muint64:
-		p, err = m.r.Next(9)
+		p, err = m.R.Next(9)
 		if err != nil {
 			return
 		}
@@ -707,6 +767,10 @@ func (m *Reader) ReadUint() (u uint, err error) {
 	return
 }
 
+// ReadByte is analogous to ReadUint8.
+//
+// NOTE: this is *not* an implementation
+// of io.ByteReader.
 func (m *Reader) ReadByte() (b byte, err error) {
 	var in uint64
 	in, err = m.ReadUint64()
@@ -724,7 +788,7 @@ func (m *Reader) ReadByte() (b byte, err error) {
 func (m *Reader) ReadBytes(scratch []byte) (b []byte, err error) {
 	var p []byte
 	var lead byte
-	p, err = m.r.Peek(2)
+	p, err = m.R.Peek(2)
 	if err != nil {
 		return
 	}
@@ -733,15 +797,15 @@ func (m *Reader) ReadBytes(scratch []byte) (b []byte, err error) {
 	switch lead {
 	case mbin8:
 		read = int64(p[1])
-		m.r.Skip(2)
+		m.R.Skip(2)
 	case mbin16:
-		p, err = m.r.Next(3)
+		p, err = m.R.Next(3)
 		if err != nil {
 			return
 		}
 		read = int64(big.Uint16(p[1:]))
 	case mbin32:
-		p, err = m.r.Next(5)
+		p, err = m.R.Next(5)
 		if err != nil {
 			return
 		}
@@ -755,16 +819,55 @@ func (m *Reader) ReadBytes(scratch []byte) (b []byte, err error) {
 	} else {
 		b = scratch[0:read]
 	}
-	_, err = m.r.ReadFull(b)
+	_, err = m.R.ReadFull(b)
 	return
 }
 
+// ReadBytesHeader reads the size header
+// of a MessagePack 'bin' object. The user
+// is responsible for dealing with the next
+// 'sz' bytes from the reader in an application-specific
+// way.
+func (m *Reader) ReadBytesHeader() (sz uint32, err error) {
+	var p []byte
+	p, err = m.R.Peek(1)
+	if err != nil {
+		return
+	}
+	switch p[0] {
+	case mbin8:
+		p, err = m.R.Next(2)
+		if err != nil {
+			return
+		}
+		sz = uint32(p[1])
+		return
+	case mbin16:
+		p, err = m.R.Next(3)
+		if err != nil {
+			return
+		}
+		sz = uint32(big.Uint16(p[1:]))
+		return
+	case mbin32:
+		p, err = m.R.Next(5)
+		if err != nil {
+			return
+		}
+		sz = uint32(big.Uint32(p[1:]))
+		return
+	default:
+		err = badPrefix(BinType, p[0])
+		return
+	}
+}
+
 // ReadExactBytes reads a MessagePack 'bin'-encoded
 // object off of the wire into the provided slice. An
 // ArrayError will be returned if the object is not
 // exactly the length of the input slice.
 func (m *Reader) ReadExactBytes(into []byte) error {
-	p, err := m.r.Peek(2)
+	p, err := m.R.Peek(2)
 	if err != nil {
 		return err
 	}
@@ -776,14 +879,14 @@ func (m *Reader) ReadExactBytes(into []byte) error {
 		read = int64(p[1])
 		skip = 2
 	case mbin16:
-		p, err = m.r.Peek(3)
+		p, err = m.R.Peek(3)
 		if err != nil {
 			return err
 		}
 		read = int64(big.Uint16(p[1:]))
 		skip = 3
 	case mbin32:
-		p, err = m.r.Peek(5)
+		p, err = m.R.Peek(5)
 		if err != nil {
 			return err
 		}
@@ -795,8 +898,8 @@ func (m *Reader) ReadExactBytes(into []byte) error {
 	if read != int64(len(into)) {
 		return ArrayError{Wanted: uint32(len(into)), Got: uint32(read)}
 	}
-	m.r.Skip(skip)
-	_, err = m.r.ReadFull(into)
+	m.R.Skip(skip)
+	_, err = m.R.ReadFull(into)
 	return err
 }
 
@@ -806,7 +909,7 @@ func (m *Reader) ReadExactBytes(into []byte) error {
 func (m *Reader) ReadStringAsBytes(scratch []byte) (b []byte, err error) {
 	var p []byte
 	var lead byte
-	p, err = m.r.Peek(1)
+	p, err = m.R.Peek(1)
 	if err != nil {
 		return
 	}
@@ -815,25 +918,25 @@ func (m *Reader) ReadStringAsBytes(scratch []byte) (b []byte, err error) {
 
 	if isfixstr(lead) {
 		read = int64(rfixstr(lead))
-		m.r.Skip(1)
+		m.R.Skip(1)
 		goto fill
 	}
 
 	switch lead {
 	case mstr8:
-		p, err = m.r.Next(2)
+		p, err = m.R.Next(2)
 		if err != nil {
 			return
 		}
 		read = int64(uint8(p[1]))
 	case mstr16:
-		p, err = m.r.Next(3)
+		p, err = m.R.Next(3)
 		if err != nil {
 			return
 		}
 		read = int64(big.Uint16(p[1:]))
 	case mstr32:
-		p, err = m.r.Next(5)
+		p, err = m.R.Next(5)
 		if err != nil {
 			return
 		}
@@ -848,16 +951,60 @@ fill:
 	} else {
 		b = scratch[0:read]
 	}
-	_, err = m.r.ReadFull(b)
+	_, err = m.R.ReadFull(b)
 	return
 }
 
+// ReadStringHeader reads a string header
+// off of the wire. The user is then responsible
+// for dealing with the next 'sz' bytes from
+// the reader in an application-specific manner.
+func (m *Reader) ReadStringHeader() (sz uint32, err error) {
+	var p []byte
+	p, err = m.R.Peek(1)
+	if err != nil {
+		return
+	}
+	lead := p[0]
+	if isfixstr(lead) {
+		sz = uint32(rfixstr(lead))
+		m.R.Skip(1)
+		return
+	}
+	switch lead {
+	case mstr8:
+		p, err = m.R.Next(2)
+		if err != nil {
+			return
+		}
+		sz = uint32(p[1])
+		return
+	case mstr16:
+		p, err = m.R.Next(3)
+		if err != nil {
+			return
+		}
+		sz = uint32(big.Uint16(p[1:]))
+		return
+	case mstr32:
+		p, err = m.R.Next(5)
+		if err != nil {
+			return
+		}
+		sz = big.Uint32(p[1:])
+		return
+	default:
+		err = badPrefix(StrType, lead)
+		return
+	}
+}
+
 // ReadString reads a utf-8 string from the reader
 func (m *Reader) ReadString() (s string, err error) {
 	var p []byte
 	var lead byte
 	var read int64
-	p, err = m.r.Peek(1)
+	p, err = m.R.Peek(1)
 	if err != nil {
 		return
 	}
@@ -865,25 +1012,25 @@ func (m *Reader) ReadString() (s string, err error) {
 
 	if isfixstr(lead) {
 		read = int64(rfixstr(lead))
-		m.r.Skip(1)
+		m.R.Skip(1)
 		goto fill
 	}
 
 	switch lead {
 	case mstr8:
-		p, err = m.r.Next(2)
+		p, err = m.R.Next(2)
 		if err != nil {
 			return
 		}
 		read = int64(uint8(p[1]))
 	case mstr16:
-		p, err = m.r.Next(3)
+		p, err = m.R.Next(3)
 		if err != nil {
 			return
 		}
 		read = int64(big.Uint16(p[1:]))
 	case mstr32:
-		p, err = m.r.Next(5)
+		p, err = m.R.Next(5)
 		if err != nil {
 			return
 		}
@@ -915,7 +1062,7 @@ fill:
 	// thus escape analysis *must* conclude that
 	// 'out' escapes.
 	out := make([]byte, read)
-	_, err = m.r.ReadFull(out)
+	_, err = m.R.ReadFull(out)
 	if err != nil {
 		return
 	}
@@ -926,7 +1073,7 @@ fill:
 // ReadComplex64 reads a complex64 from the reader
 func (m *Reader) ReadComplex64() (f complex64, err error) {
 	var p []byte
-	p, err = m.r.Peek(10)
+	p, err = m.R.Peek(10)
 	if err != nil {
 		return
 	}
@@ -940,14 +1087,14 @@ func (m *Reader) ReadComplex64() (f complex64, err error) {
 	}
 	f = complex(math.Float32frombits(big.Uint32(p[2:])),
 		math.Float32frombits(big.Uint32(p[6:])))
-	_, err = m.r.Skip(10)
+	_, err = m.R.Skip(10)
 	return
 }
 
 // ReadComplex128 reads a complex128 from the reader
 func (m *Reader) ReadComplex128() (f complex128, err error) {
 	var p []byte
-	p, err = m.r.Peek(18)
+	p, err = m.R.Peek(18)
 	if err != nil {
 		return
 	}
@@ -961,7 +1108,7 @@ func (m *Reader) ReadComplex128() (f complex128, err error) {
 	}
 	f = complex(math.Float64frombits(big.Uint64(p[2:])),
 		math.Float64frombits(big.Uint64(p[10:])))
-	_, err = m.r.Skip(18)
+	_, err = m.R.Skip(18)
 	return
 }
 
@@ -996,7 +1143,7 @@ func (m *Reader) ReadMapStrIntf(mp map[string]interface{}) (err error) {
 // The returned time's location will be set to time.Local.
 func (m *Reader) ReadTime() (t time.Time, err error) {
 	var p []byte
-	p, err = m.r.Peek(15)
+	p, err = m.R.Peek(15)
 	if err != nil {
 		return
 	}
@@ -1010,7 +1157,7 @@ func (m *Reader) ReadTime() (t time.Time, err error) {
 	}
 	sec, nsec := getUnix(p[3:])
 	t = time.Unix(sec, int64(nsec)).Local()
-	_, err = m.r.Skip(15)
+	_, err = m.R.Skip(15)
 	return
 }
 

vendor/github.com/tinylib/msgp/msgp/read_bytes.go

@@ -117,13 +117,13 @@ func (r Raw) Msgsize() int {
 }
 
 func appendNext(f *Reader, d *[]byte) error {
-	amt, o, err := getNextSize(f.r)
+	amt, o, err := getNextSize(f.R)
 	if err != nil {
 		return err
 	}
 	var i int
 	*d, i = ensure(*d, int(amt))
-	_, err = f.r.ReadFull((*d)[i:])
+	_, err = f.R.ReadFull((*d)[i:])
 	if err != nil {
 		return err
 	}
@@ -576,7 +576,7 @@ func ReadUintBytes(b []byte) (uint, []byte, error) {
 	return uint(u), b, err
 }
 
-// ReadByteBytes is analagous to ReadUint8Bytes
+// ReadByteBytes is analogous to ReadUint8Bytes
 func ReadByteBytes(b []byte) (byte, []byte, error) {
 	return ReadUint8Bytes(b)
 }
@@ -784,6 +784,22 @@ func ReadStringBytes(b []byte) (string, []byte, error) {
 	return string(v), o, err
 }
 
+// ReadStringAsBytes reads a 'str' object
+// into a slice of bytes. 'v' is the value of
+// the 'str' object, which may reside in memory
+// pointed to by 'scratch.' 'o' is the remaining bytes
+// in 'b.''
+// Possible errors:
+// - ErrShortBytes (b not long enough)
+// - TypeError{} (not 'str' type)
+// - InvalidPrefixError (unknown type marker)
+func ReadStringAsBytes(b []byte, scratch []byte) (v []byte, o []byte, err error) {
+	var tmp []byte
+	tmp, o, err = ReadStringZC(b)
+	v = append(scratch[:0], tmp...)
+	return
+}
+
 // ReadComplex128Bytes reads a complex128
 // extension object from 'b' and returns the
 // remaining bytes.
@@ -922,14 +938,14 @@ func ReadIntfBytes(b []byte) (i interface{}, o []byte, err error) {
 
 	case ArrayType:
 		var sz uint32
-		sz, b, err = ReadArrayHeaderBytes(b)
+		sz, o, err = ReadArrayHeaderBytes(b)
 		if err != nil {
 			return
 		}
 		j := make([]interface{}, int(sz))
 		i = j
 		for d := range j {
-			j[d], b, err = ReadIntfBytes(b)
+			j[d], o, err = ReadIntfBytes(o)
 			if err != nil {
 				return
 			}

vendor/github.com/tinylib/msgp/msgp/unsafe.go

@@ -0,0 +1,41 @@
+// +build !appengine
+
+package msgp
+
+import (
+	"reflect"
+	"unsafe"
+)
+
+// NOTE:
+// all of the definition in this file
+// should be repeated in appengine.go,
+// but without using unsafe
+
+const (
+	// spec says int and uint are always
+	// the same size, but that int/uint
+	// size may not be machine word size
+	smallint = unsafe.Sizeof(int(0)) == 4
+)
+
+// UnsafeString returns the byte slice as a volatile string
+// THIS SHOULD ONLY BE USED BY THE CODE GENERATOR.
+// THIS IS EVIL CODE.
+// YOU HAVE BEEN WARNED.
+func UnsafeString(b []byte) string {
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	return *(*string)(unsafe.Pointer(&reflect.StringHeader{Data: sh.Data, Len: sh.Len}))
+}
+
+// UnsafeBytes returns the string as a byte slice
+// THIS SHOULD ONLY BE USED BY THE CODE GENERATOR.
+// THIS IS EVIL CODE.
+// YOU HAVE BEEN WARNED.
+func UnsafeBytes(s string) []byte {
+	return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
+		Len:  len(s),
+		Cap:  len(s),
+		Data: (*(*reflect.StringHeader)(unsafe.Pointer(&s))).Data,
+	}))
+}

vendor/github.com/tinylib/msgp/msgp/write.go

@@ -10,13 +10,6 @@ import (
 	"time"
 )
 
-func abs(i int64) int64 {
-	if i < 0 {
-		return -i
-	}
-	return i
-}
-
 // Sizer is an interface implemented
 // by types that can estimate their
 // size when MessagePack encoded.
@@ -59,15 +52,26 @@ func pushWriter(wr *Writer) {
 // it will cause undefined behavior.
 func freeW(w *Writer) { pushWriter(w) }
 
-// Require ensures that cap(old)-len(old) >= extra
+// Require ensures that cap(old)-len(old) >= extra.
 func Require(old []byte, extra int) []byte {
-	if cap(old)-len(old) >= extra {
+	l := len(old)
+	c := cap(old)
+	r := l + extra
+	if c >= r {
 		return old
-	}
-	if len(old) == 0 {
+	} else if l == 0 {
 		return make([]byte, 0, extra)
 	}
-	n := make([]byte, len(old), cap(old)-len(old)+extra)
+	// the new size is the greater
+	// of double the old capacity
+	// and the sum of the old length
+	// and the number of new bytes
+	// necessary.
+	c <<= 1
+	if c < r {
+		c = r
+	}
+	n := make([]byte, l, c)
 	copy(n, old)
 	return n
 }
@@ -184,6 +188,17 @@ func (mw *Writer) require(n int) (int, error) {
 	return wl, nil
 }
 
+func (mw *Writer) Append(b ...byte) error {
+	if mw.avail() < len(b) {
+		err := mw.flush()
+		if err != nil {
+			return err
+		}
+	}
+	mw.wloc += copy(mw.buf[mw.wloc:], b)
+	return nil
+}
+
 // push one byte onto the buffer
 //
 // NOTE: this is a hot code path
@@ -289,9 +304,9 @@ func (mw *Writer) Reset(w io.Writer) {
 // size to the writer
 func (mw *Writer) WriteMapHeader(sz uint32) error {
 	switch {
-	case sz < 16:
+	case sz <= 15:
 		return mw.push(wfixmap(uint8(sz)))
-	case sz < math.MaxUint16:
+	case sz <= math.MaxUint16:
 		return mw.prefix16(mmap16, uint16(sz))
 	default:
 		return mw.prefix32(mmap32, sz)
@@ -302,9 +317,9 @@ func (mw *Writer) WriteMapHeader(sz uint32) error {
 // given size to the writer
 func (mw *Writer) WriteArrayHeader(sz uint32) error {
 	switch {
-	case sz < 16:
+	case sz <= 15:
 		return mw.push(wfixarray(uint8(sz)))
-	case sz < math.MaxUint16:
+	case sz <= math.MaxUint16:
 		return mw.prefix16(marray16, uint16(sz))
 	default:
 		return mw.prefix32(marray32, sz)
@@ -328,17 +343,26 @@ func (mw *Writer) WriteFloat32(f float32) error {
 
 // WriteInt64 writes an int64 to the writer
 func (mw *Writer) WriteInt64(i int64) error {
-	a := abs(i)
+	if i >= 0 {
+		switch {
+		case i <= math.MaxInt8:
+			return mw.push(wfixint(uint8(i)))
+		case i <= math.MaxInt16:
+			return mw.prefix16(mint16, uint16(i))
+		case i <= math.MaxInt32:
+			return mw.prefix32(mint32, uint32(i))
+		default:
+			return mw.prefix64(mint64, uint64(i))
+		}
+	}
 	switch {
-	case i < 0 && i > -32:
+	case i >= -32:
 		return mw.push(wnfixint(int8(i)))
-	case i >= 0 && i < 128:
-		return mw.push(wfixint(uint8(i)))
-	case a < math.MaxInt8:
+	case i >= math.MinInt8:
 		return mw.prefix8(mint8, uint8(i))
-	case a < math.MaxInt16:
+	case i >= math.MinInt16:
 		return mw.prefix16(mint16, uint16(i))
-	case a < math.MaxInt32:
+	case i >= math.MinInt32:
 		return mw.prefix32(mint32, uint32(i))
 	default:
 		return mw.prefix64(mint64, uint64(i))
@@ -360,20 +384,20 @@ func (mw *Writer) WriteInt(i int) error { return mw.WriteInt64(int64(i)) }
 // WriteUint64 writes a uint64 to the writer
 func (mw *Writer) WriteUint64(u uint64) error {
 	switch {
-	case u < (1 << 7):
+	case u <= (1<<7)-1:
 		return mw.push(wfixint(uint8(u)))
-	case u < math.MaxUint8:
+	case u <= math.MaxUint8:
 		return mw.prefix8(muint8, uint8(u))
-	case u < math.MaxUint16:
+	case u <= math.MaxUint16:
 		return mw.prefix16(muint16, uint16(u))
-	case u < math.MaxUint32:
+	case u <= math.MaxUint32:
 		return mw.prefix32(muint32, uint32(u))
 	default:
 		return mw.prefix64(muint64, u)
 	}
 }
 
-// WriteByte is analagous to WriteUint8
+// WriteByte is analogous to WriteUint8
 func (mw *Writer) WriteByte(u byte) error { return mw.WriteUint8(uint8(u)) }
 
 // WriteUint8 writes a uint8 to the writer
@@ -393,9 +417,9 @@ func (mw *Writer) WriteBytes(b []byte) error {
 	sz := uint32(len(b))
 	var err error
 	switch {
-	case sz < math.MaxUint8:
+	case sz <= math.MaxUint8:
 		err = mw.prefix8(mbin8, uint8(sz))
-	case sz < math.MaxUint16:
+	case sz <= math.MaxUint16:
 		err = mw.prefix16(mbin16, uint16(sz))
 	default:
 		err = mw.prefix32(mbin32, sz)
@@ -407,6 +431,20 @@ func (mw *Writer) WriteBytes(b []byte) error {
 	return err
 }
 
+// WriteBytesHeader writes just the size header
+// of a MessagePack 'bin' object. The user is responsible
+// for then writing 'sz' more bytes into the stream.
+func (mw *Writer) WriteBytesHeader(sz uint32) error {
+	switch {
+	case sz <= math.MaxUint8:
+		return mw.prefix8(mbin8, uint8(sz))
+	case sz <= math.MaxUint16:
+		return mw.prefix16(mbin16, uint16(sz))
+	default:
+		return mw.prefix32(mbin32, sz)
+	}
+}
+
 // WriteBool writes a bool to the writer
 func (mw *Writer) WriteBool(b bool) error {
 	if b {
@@ -421,11 +459,11 @@ func (mw *Writer) WriteString(s string) error {
 	sz := uint32(len(s))
 	var err error
 	switch {
-	case sz < 32:
+	case sz <= 31:
 		err = mw.push(wfixstr(uint8(sz)))
-	case sz < math.MaxUint8:
+	case sz <= math.MaxUint8:
 		err = mw.prefix8(mstr8, uint8(sz))
-	case sz < math.MaxUint16:
+	case sz <= math.MaxUint16:
 		err = mw.prefix16(mstr16, uint16(sz))
 	default:
 		err = mw.prefix32(mstr32, sz)
@@ -436,6 +474,45 @@ func (mw *Writer) WriteString(s string) error {
 	return mw.writeString(s)
 }
 
+// WriteStringHeader writes just the string size
+// header of a MessagePack 'str' object. The user
+// is responsible for writing 'sz' more valid UTF-8
+// bytes to the stream.
+func (mw *Writer) WriteStringHeader(sz uint32) error {
+	switch {
+	case sz <= 31:
+		return mw.push(wfixstr(uint8(sz)))
+	case sz <= math.MaxUint8:
+		return mw.prefix8(mstr8, uint8(sz))
+	case sz <= math.MaxUint16:
+		return mw.prefix16(mstr16, uint16(sz))
+	default:
+		return mw.prefix32(mstr32, sz)
+	}
+}
+
+// WriteStringFromBytes writes a 'str' object
+// from a []byte.
+func (mw *Writer) WriteStringFromBytes(str []byte) error {
+	sz := uint32(len(str))
+	var err error
+	switch {
+	case sz <= 31:
+		err = mw.push(wfixstr(uint8(sz)))
+	case sz <= math.MaxUint8:
+		err = mw.prefix8(mstr8, uint8(sz))
+	case sz <= math.MaxUint16:
+		err = mw.prefix16(mstr16, uint16(sz))
+	default:
+		err = mw.prefix32(mstr32, sz)
+	}
+	if err != nil {
+		return err
+	}
+	_, err = mw.Write(str)
+	return err
+}
+
 // WriteComplex64 writes a complex64 to the writer
 func (mw *Writer) WriteComplex64(f complex64) error {
 	o, err := mw.require(10)
@@ -509,7 +586,7 @@ func (mw *Writer) WriteMapStrIntf(mp map[string]interface{}) (err error) {
 // elapsed since "zero" Unix time, followed by 4 bytes
 // for a big-endian 32-bit signed integer denoting
 // the nanosecond offset of the time. This encoding
-// is intended to ease portability accross languages.
+// is intended to ease portability across languages.
 // (Note that this is *not* the standard time.Time
 // binary encoding, because its implementation relies
 // heavily on the internal representation used by the
@@ -612,7 +689,7 @@ func (mw *Writer) WriteIntf(v interface{}) error {
 }
 
 func (mw *Writer) writeMap(v reflect.Value) (err error) {
-	if v.Elem().Kind() != reflect.String {
+	if v.Type().Key().Kind() != reflect.String {
 		return errors.New("msgp: map keys must be strings")
 	}
 	ks := v.MapKeys()

vendor/github.com/tinylib/msgp/msgp/write_bytes.go

@@ -22,10 +22,10 @@ func ensure(b []byte, sz int) ([]byte, int) {
 // given size to the slice
 func AppendMapHeader(b []byte, sz uint32) []byte {
 	switch {
-	case sz < 16:
+	case sz <= 15:
 		return append(b, wfixmap(uint8(sz)))
 
-	case sz < math.MaxUint16:
+	case sz <= math.MaxUint16:
 		o, n := ensure(b, 3)
 		prefixu16(o[n:], mmap16, uint16(sz))
 		return o
@@ -41,10 +41,10 @@ func AppendMapHeader(b []byte, sz uint32) []byte {
 // the given size to the slice
 func AppendArrayHeader(b []byte, sz uint32) []byte {
 	switch {
-	case sz < 16:
+	case sz <= 15:
 		return append(b, wfixarray(uint8(sz)))
 
-	case sz < math.MaxUint16:
+	case sz <= math.MaxUint16:
 		o, n := ensure(b, 3)
 		prefixu16(o[n:], marray16, uint16(sz))
 		return o
@@ -75,29 +75,39 @@ func AppendFloat32(b []byte, f float32) []byte {
 
 // AppendInt64 appends an int64 to the slice
 func AppendInt64(b []byte, i int64) []byte {
-	a := abs(i)
+	if i >= 0 {
+		switch {
+		case i <= math.MaxInt8:
+			return append(b, wfixint(uint8(i)))
+		case i <= math.MaxInt16:
+			o, n := ensure(b, 3)
+			putMint16(o[n:], int16(i))
+			return o
+		case i <= math.MaxInt32:
+			o, n := ensure(b, 5)
+			putMint32(o[n:], int32(i))
+			return o
+		default:
+			o, n := ensure(b, 9)
+			putMint64(o[n:], i)
+			return o
+		}
+	}
 	switch {
-	case i < 0 && i > -32:
+	case i >= -32:
 		return append(b, wnfixint(int8(i)))
-
-	case i >= 0 && i < 128:
-		return append(b, wfixint(uint8(i)))
-
-	case a < math.MaxInt8:
+	case i >= math.MinInt8:
 		o, n := ensure(b, 2)
 		putMint8(o[n:], int8(i))
 		return o
-
-	case a < math.MaxInt16:
+	case i >= math.MinInt16:
 		o, n := ensure(b, 3)
 		putMint16(o[n:], int16(i))
 		return o
-
-	case a < math.MaxInt32:
+	case i >= math.MinInt32:
 		o, n := ensure(b, 5)
 		putMint32(o[n:], int32(i))
 		return o
-
 	default:
 		o, n := ensure(b, 9)
 		putMint64(o[n:], i)
@@ -120,20 +130,20 @@ func AppendInt32(b []byte, i int32) []byte { return AppendInt64(b, int64(i)) }
 // AppendUint64 appends a uint64 to the slice
 func AppendUint64(b []byte, u uint64) []byte {
 	switch {
-	case u < (1 << 7):
+	case u <= (1<<7)-1:
 		return append(b, wfixint(uint8(u)))
 
-	case u < math.MaxUint8:
+	case u <= math.MaxUint8:
 		o, n := ensure(b, 2)
 		putMuint8(o[n:], uint8(u))
 		return o
 
-	case u < math.MaxUint16:
+	case u <= math.MaxUint16:
 		o, n := ensure(b, 3)
 		putMuint16(o[n:], uint16(u))
 		return o
 
-	case u < math.MaxUint32:
+	case u <= math.MaxUint32:
 		o, n := ensure(b, 5)
 		putMuint32(o[n:], uint32(u))
 		return o
@@ -152,7 +162,7 @@ func AppendUint(b []byte, u uint) []byte { return AppendUint64(b, uint64(u)) }
 // AppendUint8 appends a uint8 to the slice
 func AppendUint8(b []byte, u uint8) []byte { return AppendUint64(b, uint64(u)) }
 
-// AppendByte is analagous to AppendUint8
+// AppendByte is analogous to AppendUint8
 func AppendByte(b []byte, u byte) []byte { return AppendUint8(b, uint8(u)) }
 
 // AppendUint16 appends a uint16 to the slice
@@ -167,11 +177,11 @@ func AppendBytes(b []byte, bts []byte) []byte {
 	var o []byte
 	var n int
 	switch {
-	case sz < math.MaxUint8:
+	case sz <= math.MaxUint8:
 		o, n = ensure(b, 2+sz)
 		prefixu8(o[n:], mbin8, uint8(sz))
 		n += 2
-	case sz < math.MaxUint16:
+	case sz <= math.MaxUint16:
 		o, n = ensure(b, 3+sz)
 		prefixu16(o[n:], mbin16, uint16(sz))
 		n += 3
@@ -197,15 +207,15 @@ func AppendString(b []byte, s string) []byte {
 	var n int
 	var o []byte
 	switch {
-	case sz < 32:
+	case sz <= 31:
 		o, n = ensure(b, 1+sz)
 		o[n] = wfixstr(uint8(sz))
 		n++
-	case sz < math.MaxUint8:
+	case sz <= math.MaxUint8:
 		o, n = ensure(b, 2+sz)
 		prefixu8(o[n:], mstr8, uint8(sz))
 		n += 2
-	case sz < math.MaxUint16:
+	case sz <= math.MaxUint16:
 		o, n = ensure(b, 3+sz)
 		prefixu16(o[n:], mstr16, uint16(sz))
 		n += 3
@@ -217,6 +227,33 @@ func AppendString(b []byte, s string) []byte {
 	return o[:n+copy(o[n:], s)]
 }
 
+// AppendStringFromBytes appends a []byte
+// as a MessagePack 'str' to the slice 'b.'
+func AppendStringFromBytes(b []byte, str []byte) []byte {
+	sz := len(str)
+	var n int
+	var o []byte
+	switch {
+	case sz <= 31:
+		o, n = ensure(b, 1+sz)
+		o[n] = wfixstr(uint8(sz))
+		n++
+	case sz <= math.MaxUint8:
+		o, n = ensure(b, 2+sz)
+		prefixu8(o[n:], mstr8, uint8(sz))
+		n += 2
+	case sz <= math.MaxUint16:
+		o, n = ensure(b, 3+sz)
+		prefixu16(o[n:], mstr16, uint16(sz))
+		n += 3
+	default:
+		o, n = ensure(b, 5+sz)
+		prefixu32(o[n:], mstr32, uint32(sz))
+		n += 5
+	}
+	return o[:n+copy(o[n:], str)]
+}
+
 // AppendComplex64 appends a complex64 to the slice as a MessagePack extension
 func AppendComplex64(b []byte, c complex64) []byte {
 	o, n := ensure(b, Complex64Size)
@@ -362,7 +399,12 @@ func AppendIntf(b []byte, i interface{}) ([]byte, error) {
 			}
 		}
 		return b, nil
-
+	case reflect.Ptr:
+		if v.IsNil() {
+			return AppendNil(b), err
+		}
+		b, err = AppendIntf(b, v.Elem().Interface())
+		return b, err
 	default:
 		return b, &ErrUnsupportedType{T: v.Type()}
 	}