123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589 |
- // Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
- // Use of this source code is governed by a BSD-style license found in the LICENSE file.
- package codec
- // Contains code shared by both encode and decode.
- import (
- "encoding/binary"
- "fmt"
- "math"
- "reflect"
- "sort"
- "strings"
- "sync"
- "time"
- "unicode"
- "unicode/utf8"
- )
- const (
- structTagName = "codec"
- // Support
- // encoding.BinaryMarshaler: MarshalBinary() (data []byte, err error)
- // encoding.BinaryUnmarshaler: UnmarshalBinary(data []byte) error
- // This constant flag will enable or disable it.
- supportBinaryMarshal = true
- // Each Encoder or Decoder uses a cache of functions based on conditionals,
- // so that the conditionals are not run every time.
- //
- // Either a map or a slice is used to keep track of the functions.
- // The map is more natural, but has a higher cost than a slice/array.
- // This flag (useMapForCodecCache) controls which is used.
- useMapForCodecCache = false
- // For some common container types, we can short-circuit an elaborate
- // reflection dance and call encode/decode directly.
- // The currently supported types are:
- // - slices of strings, or id's (int64,uint64) or interfaces.
- // - maps of str->str, str->intf, id(int64,uint64)->intf, intf->intf
- shortCircuitReflectToFastPath = true
- // for debugging, set this to false, to catch panic traces.
- // Note that this will always cause rpc tests to fail, since they need io.EOF sent via panic.
- recoverPanicToErr = true
- )
- type charEncoding uint8
- const (
- c_RAW charEncoding = iota
- c_UTF8
- c_UTF16LE
- c_UTF16BE
- c_UTF32LE
- c_UTF32BE
- )
- // valueType is the stream type
- type valueType uint8
- const (
- valueTypeUnset valueType = iota
- valueTypeNil
- valueTypeInt
- valueTypeUint
- valueTypeFloat
- valueTypeBool
- valueTypeString
- valueTypeSymbol
- valueTypeBytes
- valueTypeMap
- valueTypeArray
- valueTypeTimestamp
- valueTypeExt
- valueTypeInvalid = 0xff
- )
- var (
- bigen = binary.BigEndian
- structInfoFieldName = "_struct"
- cachedTypeInfo = make(map[uintptr]*typeInfo, 4)
- cachedTypeInfoMutex sync.RWMutex
- intfSliceTyp = reflect.TypeOf([]interface{}(nil))
- intfTyp = intfSliceTyp.Elem()
- strSliceTyp = reflect.TypeOf([]string(nil))
- boolSliceTyp = reflect.TypeOf([]bool(nil))
- uintSliceTyp = reflect.TypeOf([]uint(nil))
- uint8SliceTyp = reflect.TypeOf([]uint8(nil))
- uint16SliceTyp = reflect.TypeOf([]uint16(nil))
- uint32SliceTyp = reflect.TypeOf([]uint32(nil))
- uint64SliceTyp = reflect.TypeOf([]uint64(nil))
- intSliceTyp = reflect.TypeOf([]int(nil))
- int8SliceTyp = reflect.TypeOf([]int8(nil))
- int16SliceTyp = reflect.TypeOf([]int16(nil))
- int32SliceTyp = reflect.TypeOf([]int32(nil))
- int64SliceTyp = reflect.TypeOf([]int64(nil))
- float32SliceTyp = reflect.TypeOf([]float32(nil))
- float64SliceTyp = reflect.TypeOf([]float64(nil))
- mapIntfIntfTyp = reflect.TypeOf(map[interface{}]interface{}(nil))
- mapStrIntfTyp = reflect.TypeOf(map[string]interface{}(nil))
- mapStrStrTyp = reflect.TypeOf(map[string]string(nil))
- mapIntIntfTyp = reflect.TypeOf(map[int]interface{}(nil))
- mapInt64IntfTyp = reflect.TypeOf(map[int64]interface{}(nil))
- mapUintIntfTyp = reflect.TypeOf(map[uint]interface{}(nil))
- mapUint64IntfTyp = reflect.TypeOf(map[uint64]interface{}(nil))
- stringTyp = reflect.TypeOf("")
- timeTyp = reflect.TypeOf(time.Time{})
- rawExtTyp = reflect.TypeOf(RawExt{})
- mapBySliceTyp = reflect.TypeOf((*MapBySlice)(nil)).Elem()
- binaryMarshalerTyp = reflect.TypeOf((*binaryMarshaler)(nil)).Elem()
- binaryUnmarshalerTyp = reflect.TypeOf((*binaryUnmarshaler)(nil)).Elem()
- rawExtTypId = reflect.ValueOf(rawExtTyp).Pointer()
- intfTypId = reflect.ValueOf(intfTyp).Pointer()
- timeTypId = reflect.ValueOf(timeTyp).Pointer()
- intfSliceTypId = reflect.ValueOf(intfSliceTyp).Pointer()
- strSliceTypId = reflect.ValueOf(strSliceTyp).Pointer()
- boolSliceTypId = reflect.ValueOf(boolSliceTyp).Pointer()
- uintSliceTypId = reflect.ValueOf(uintSliceTyp).Pointer()
- uint8SliceTypId = reflect.ValueOf(uint8SliceTyp).Pointer()
- uint16SliceTypId = reflect.ValueOf(uint16SliceTyp).Pointer()
- uint32SliceTypId = reflect.ValueOf(uint32SliceTyp).Pointer()
- uint64SliceTypId = reflect.ValueOf(uint64SliceTyp).Pointer()
- intSliceTypId = reflect.ValueOf(intSliceTyp).Pointer()
- int8SliceTypId = reflect.ValueOf(int8SliceTyp).Pointer()
- int16SliceTypId = reflect.ValueOf(int16SliceTyp).Pointer()
- int32SliceTypId = reflect.ValueOf(int32SliceTyp).Pointer()
- int64SliceTypId = reflect.ValueOf(int64SliceTyp).Pointer()
- float32SliceTypId = reflect.ValueOf(float32SliceTyp).Pointer()
- float64SliceTypId = reflect.ValueOf(float64SliceTyp).Pointer()
- mapStrStrTypId = reflect.ValueOf(mapStrStrTyp).Pointer()
- mapIntfIntfTypId = reflect.ValueOf(mapIntfIntfTyp).Pointer()
- mapStrIntfTypId = reflect.ValueOf(mapStrIntfTyp).Pointer()
- mapIntIntfTypId = reflect.ValueOf(mapIntIntfTyp).Pointer()
- mapInt64IntfTypId = reflect.ValueOf(mapInt64IntfTyp).Pointer()
- mapUintIntfTypId = reflect.ValueOf(mapUintIntfTyp).Pointer()
- mapUint64IntfTypId = reflect.ValueOf(mapUint64IntfTyp).Pointer()
- // Id = reflect.ValueOf().Pointer()
- // mapBySliceTypId = reflect.ValueOf(mapBySliceTyp).Pointer()
- binaryMarshalerTypId = reflect.ValueOf(binaryMarshalerTyp).Pointer()
- binaryUnmarshalerTypId = reflect.ValueOf(binaryUnmarshalerTyp).Pointer()
- intBitsize uint8 = uint8(reflect.TypeOf(int(0)).Bits())
- uintBitsize uint8 = uint8(reflect.TypeOf(uint(0)).Bits())
- bsAll0x00 = []byte{0, 0, 0, 0, 0, 0, 0, 0}
- bsAll0xff = []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
- )
- type binaryUnmarshaler interface {
- UnmarshalBinary(data []byte) error
- }
- type binaryMarshaler interface {
- MarshalBinary() (data []byte, err error)
- }
- // MapBySlice represents a slice which should be encoded as a map in the stream.
- // The slice contains a sequence of key-value pairs.
- type MapBySlice interface {
- MapBySlice()
- }
- // WARNING: DO NOT USE DIRECTLY. EXPORTED FOR GODOC BENEFIT. WILL BE REMOVED.
- //
- // BasicHandle encapsulates the common options and extension functions.
- type BasicHandle struct {
- extHandle
- EncodeOptions
- DecodeOptions
- }
- // Handle is the interface for a specific encoding format.
- //
- // Typically, a Handle is pre-configured before first time use,
- // and not modified while in use. Such a pre-configured Handle
- // is safe for concurrent access.
- type Handle interface {
- writeExt() bool
- getBasicHandle() *BasicHandle
- newEncDriver(w encWriter) encDriver
- newDecDriver(r decReader) decDriver
- }
- // RawExt represents raw unprocessed extension data.
- type RawExt struct {
- Tag byte
- Data []byte
- }
- type extTypeTagFn struct {
- rtid uintptr
- rt reflect.Type
- tag byte
- encFn func(reflect.Value) ([]byte, error)
- decFn func(reflect.Value, []byte) error
- }
- type extHandle []*extTypeTagFn
- // AddExt registers an encode and decode function for a reflect.Type.
- // Note that the type must be a named type, and specifically not
- // a pointer or Interface. An error is returned if that is not honored.
- //
- // To Deregister an ext, call AddExt with 0 tag, nil encfn and nil decfn.
- func (o *extHandle) AddExt(
- rt reflect.Type,
- tag byte,
- encfn func(reflect.Value) ([]byte, error),
- decfn func(reflect.Value, []byte) error,
- ) (err error) {
- // o is a pointer, because we may need to initialize it
- if rt.PkgPath() == "" || rt.Kind() == reflect.Interface {
- err = fmt.Errorf("codec.Handle.AddExt: Takes named type, especially not a pointer or interface: %T",
- reflect.Zero(rt).Interface())
- return
- }
- // o cannot be nil, since it is always embedded in a Handle.
- // if nil, let it panic.
- // if o == nil {
- // err = errors.New("codec.Handle.AddExt: extHandle cannot be a nil pointer.")
- // return
- // }
- rtid := reflect.ValueOf(rt).Pointer()
- for _, v := range *o {
- if v.rtid == rtid {
- v.tag, v.encFn, v.decFn = tag, encfn, decfn
- return
- }
- }
- *o = append(*o, &extTypeTagFn{rtid, rt, tag, encfn, decfn})
- return
- }
- func (o extHandle) getExt(rtid uintptr) *extTypeTagFn {
- for _, v := range o {
- if v.rtid == rtid {
- return v
- }
- }
- return nil
- }
- func (o extHandle) getExtForTag(tag byte) *extTypeTagFn {
- for _, v := range o {
- if v.tag == tag {
- return v
- }
- }
- return nil
- }
- func (o extHandle) getDecodeExtForTag(tag byte) (
- rv reflect.Value, fn func(reflect.Value, []byte) error) {
- if x := o.getExtForTag(tag); x != nil {
- // ext is only registered for base
- rv = reflect.New(x.rt).Elem()
- fn = x.decFn
- }
- return
- }
- func (o extHandle) getDecodeExt(rtid uintptr) (tag byte, fn func(reflect.Value, []byte) error) {
- if x := o.getExt(rtid); x != nil {
- tag = x.tag
- fn = x.decFn
- }
- return
- }
- func (o extHandle) getEncodeExt(rtid uintptr) (tag byte, fn func(reflect.Value) ([]byte, error)) {
- if x := o.getExt(rtid); x != nil {
- tag = x.tag
- fn = x.encFn
- }
- return
- }
- type structFieldInfo struct {
- encName string // encode name
- // only one of 'i' or 'is' can be set. If 'i' is -1, then 'is' has been set.
- is []int // (recursive/embedded) field index in struct
- i int16 // field index in struct
- omitEmpty bool
- toArray bool // if field is _struct, is the toArray set?
- // tag string // tag
- // name string // field name
- // encNameBs []byte // encoded name as byte stream
- // ikind int // kind of the field as an int i.e. int(reflect.Kind)
- }
- func parseStructFieldInfo(fname string, stag string) *structFieldInfo {
- if fname == "" {
- panic("parseStructFieldInfo: No Field Name")
- }
- si := structFieldInfo{
- // name: fname,
- encName: fname,
- // tag: stag,
- }
- if stag != "" {
- for i, s := range strings.Split(stag, ",") {
- if i == 0 {
- if s != "" {
- si.encName = s
- }
- } else {
- switch s {
- case "omitempty":
- si.omitEmpty = true
- case "toarray":
- si.toArray = true
- }
- }
- }
- }
- // si.encNameBs = []byte(si.encName)
- return &si
- }
- type sfiSortedByEncName []*structFieldInfo
- func (p sfiSortedByEncName) Len() int {
- return len(p)
- }
- func (p sfiSortedByEncName) Less(i, j int) bool {
- return p[i].encName < p[j].encName
- }
- func (p sfiSortedByEncName) Swap(i, j int) {
- p[i], p[j] = p[j], p[i]
- }
- // typeInfo keeps information about each type referenced in the encode/decode sequence.
- //
- // During an encode/decode sequence, we work as below:
- // - If base is a built in type, en/decode base value
- // - If base is registered as an extension, en/decode base value
- // - If type is binary(M/Unm)arshaler, call Binary(M/Unm)arshal method
- // - Else decode appropriately based on the reflect.Kind
- type typeInfo struct {
- sfi []*structFieldInfo // sorted. Used when enc/dec struct to map.
- sfip []*structFieldInfo // unsorted. Used when enc/dec struct to array.
- rt reflect.Type
- rtid uintptr
- // baseId gives pointer to the base reflect.Type, after deferencing
- // the pointers. E.g. base type of ***time.Time is time.Time.
- base reflect.Type
- baseId uintptr
- baseIndir int8 // number of indirections to get to base
- mbs bool // base type (T or *T) is a MapBySlice
- m bool // base type (T or *T) is a binaryMarshaler
- unm bool // base type (T or *T) is a binaryUnmarshaler
- mIndir int8 // number of indirections to get to binaryMarshaler type
- unmIndir int8 // number of indirections to get to binaryUnmarshaler type
- toArray bool // whether this (struct) type should be encoded as an array
- }
- func (ti *typeInfo) indexForEncName(name string) int {
- //tisfi := ti.sfi
- const binarySearchThreshold = 16
- if sfilen := len(ti.sfi); sfilen < binarySearchThreshold {
- // linear search. faster than binary search in my testing up to 16-field structs.
- for i, si := range ti.sfi {
- if si.encName == name {
- return i
- }
- }
- } else {
- // binary search. adapted from sort/search.go.
- h, i, j := 0, 0, sfilen
- for i < j {
- h = i + (j-i)/2
- if ti.sfi[h].encName < name {
- i = h + 1
- } else {
- j = h
- }
- }
- if i < sfilen && ti.sfi[i].encName == name {
- return i
- }
- }
- return -1
- }
- func getTypeInfo(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
- var ok bool
- cachedTypeInfoMutex.RLock()
- pti, ok = cachedTypeInfo[rtid]
- cachedTypeInfoMutex.RUnlock()
- if ok {
- return
- }
- cachedTypeInfoMutex.Lock()
- defer cachedTypeInfoMutex.Unlock()
- if pti, ok = cachedTypeInfo[rtid]; ok {
- return
- }
- ti := typeInfo{rt: rt, rtid: rtid}
- pti = &ti
- var indir int8
- if ok, indir = implementsIntf(rt, binaryMarshalerTyp); ok {
- ti.m, ti.mIndir = true, indir
- }
- if ok, indir = implementsIntf(rt, binaryUnmarshalerTyp); ok {
- ti.unm, ti.unmIndir = true, indir
- }
- if ok, _ = implementsIntf(rt, mapBySliceTyp); ok {
- ti.mbs = true
- }
- pt := rt
- var ptIndir int8
- // for ; pt.Kind() == reflect.Ptr; pt, ptIndir = pt.Elem(), ptIndir+1 { }
- for pt.Kind() == reflect.Ptr {
- pt = pt.Elem()
- ptIndir++
- }
- if ptIndir == 0 {
- ti.base = rt
- ti.baseId = rtid
- } else {
- ti.base = pt
- ti.baseId = reflect.ValueOf(pt).Pointer()
- ti.baseIndir = ptIndir
- }
- if rt.Kind() == reflect.Struct {
- var siInfo *structFieldInfo
- if f, ok := rt.FieldByName(structInfoFieldName); ok {
- siInfo = parseStructFieldInfo(structInfoFieldName, f.Tag.Get(structTagName))
- ti.toArray = siInfo.toArray
- }
- sfip := make([]*structFieldInfo, 0, rt.NumField())
- rgetTypeInfo(rt, nil, make(map[string]bool), &sfip, siInfo)
- // // try to put all si close together
- // const tryToPutAllStructFieldInfoTogether = true
- // if tryToPutAllStructFieldInfoTogether {
- // sfip2 := make([]structFieldInfo, len(sfip))
- // for i, si := range sfip {
- // sfip2[i] = *si
- // }
- // for i := range sfip {
- // sfip[i] = &sfip2[i]
- // }
- // }
- ti.sfip = make([]*structFieldInfo, len(sfip))
- ti.sfi = make([]*structFieldInfo, len(sfip))
- copy(ti.sfip, sfip)
- sort.Sort(sfiSortedByEncName(sfip))
- copy(ti.sfi, sfip)
- }
- // sfi = sfip
- cachedTypeInfo[rtid] = pti
- return
- }
- func rgetTypeInfo(rt reflect.Type, indexstack []int, fnameToHastag map[string]bool,
- sfi *[]*structFieldInfo, siInfo *structFieldInfo,
- ) {
- // for rt.Kind() == reflect.Ptr {
- // // indexstack = append(indexstack, 0)
- // rt = rt.Elem()
- // }
- for j := 0; j < rt.NumField(); j++ {
- f := rt.Field(j)
- stag := f.Tag.Get(structTagName)
- if stag == "-" {
- continue
- }
- if r1, _ := utf8.DecodeRuneInString(f.Name); r1 == utf8.RuneError || !unicode.IsUpper(r1) {
- continue
- }
- // if anonymous and there is no struct tag and its a struct (or pointer to struct), inline it.
- if f.Anonymous && stag == "" {
- ft := f.Type
- for ft.Kind() == reflect.Ptr {
- ft = ft.Elem()
- }
- if ft.Kind() == reflect.Struct {
- indexstack2 := append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
- rgetTypeInfo(ft, indexstack2, fnameToHastag, sfi, siInfo)
- continue
- }
- }
- // do not let fields with same name in embedded structs override field at higher level.
- // this must be done after anonymous check, to allow anonymous field
- // still include their child fields
- if _, ok := fnameToHastag[f.Name]; ok {
- continue
- }
- si := parseStructFieldInfo(f.Name, stag)
- // si.ikind = int(f.Type.Kind())
- if len(indexstack) == 0 {
- si.i = int16(j)
- } else {
- si.i = -1
- si.is = append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
- }
- if siInfo != nil {
- if siInfo.omitEmpty {
- si.omitEmpty = true
- }
- }
- *sfi = append(*sfi, si)
- fnameToHastag[f.Name] = stag != ""
- }
- }
- func panicToErr(err *error) {
- if recoverPanicToErr {
- if x := recover(); x != nil {
- //debug.PrintStack()
- panicValToErr(x, err)
- }
- }
- }
- func doPanic(tag string, format string, params ...interface{}) {
- params2 := make([]interface{}, len(params)+1)
- params2[0] = tag
- copy(params2[1:], params)
- panic(fmt.Errorf("%s: "+format, params2...))
- }
- func checkOverflowFloat32(f float64, doCheck bool) {
- if !doCheck {
- return
- }
- // check overflow (logic adapted from std pkg reflect/value.go OverflowFloat()
- f2 := f
- if f2 < 0 {
- f2 = -f
- }
- if math.MaxFloat32 < f2 && f2 <= math.MaxFloat64 {
- decErr("Overflow float32 value: %v", f2)
- }
- }
- func checkOverflow(ui uint64, i int64, bitsize uint8) {
- // check overflow (logic adapted from std pkg reflect/value.go OverflowUint()
- if bitsize == 0 {
- return
- }
- if i != 0 {
- if trunc := (i << (64 - bitsize)) >> (64 - bitsize); i != trunc {
- decErr("Overflow int value: %v", i)
- }
- }
- if ui != 0 {
- if trunc := (ui << (64 - bitsize)) >> (64 - bitsize); ui != trunc {
- decErr("Overflow uint value: %v", ui)
- }
- }
- }
|