| // Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved. |
| // Use of this source code is governed by a BSD-style license found in the LICENSE file. |
| |
| package codec |
| |
| // Contains code shared by both encode and decode. |
| |
| import ( |
| "encoding/binary" |
| "fmt" |
| "math" |
| "reflect" |
| "sort" |
| "strings" |
| "sync" |
| "time" |
| "unicode" |
| "unicode/utf8" |
| ) |
| |
| const ( |
| structTagName = "codec" |
| |
| // Support |
| // encoding.BinaryMarshaler: MarshalBinary() (data []byte, err error) |
| // encoding.BinaryUnmarshaler: UnmarshalBinary(data []byte) error |
| // This constant flag will enable or disable it. |
| supportBinaryMarshal = true |
| |
| // Each Encoder or Decoder uses a cache of functions based on conditionals, |
| // so that the conditionals are not run every time. |
| // |
| // Either a map or a slice is used to keep track of the functions. |
| // The map is more natural, but has a higher cost than a slice/array. |
| // This flag (useMapForCodecCache) controls which is used. |
| useMapForCodecCache = false |
| |
| // For some common container types, we can short-circuit an elaborate |
| // reflection dance and call encode/decode directly. |
| // The currently supported types are: |
| // - slices of strings, or id's (int64,uint64) or interfaces. |
| // - maps of str->str, str->intf, id(int64,uint64)->intf, intf->intf |
| shortCircuitReflectToFastPath = true |
| |
| // for debugging, set this to false, to catch panic traces. |
| // Note that this will always cause rpc tests to fail, since they need io.EOF sent via panic. |
| recoverPanicToErr = true |
| ) |
| |
| type charEncoding uint8 |
| |
| const ( |
| c_RAW charEncoding = iota |
| c_UTF8 |
| c_UTF16LE |
| c_UTF16BE |
| c_UTF32LE |
| c_UTF32BE |
| ) |
| |
| // valueType is the stream type |
| type valueType uint8 |
| |
| const ( |
| valueTypeUnset valueType = iota |
| valueTypeNil |
| valueTypeInt |
| valueTypeUint |
| valueTypeFloat |
| valueTypeBool |
| valueTypeString |
| valueTypeSymbol |
| valueTypeBytes |
| valueTypeMap |
| valueTypeArray |
| valueTypeTimestamp |
| valueTypeExt |
| |
| valueTypeInvalid = 0xff |
| ) |
| |
| var ( |
| bigen = binary.BigEndian |
| structInfoFieldName = "_struct" |
| |
| cachedTypeInfo = make(map[uintptr]*typeInfo, 4) |
| cachedTypeInfoMutex sync.RWMutex |
| |
| intfSliceTyp = reflect.TypeOf([]interface{}(nil)) |
| intfTyp = intfSliceTyp.Elem() |
| |
| strSliceTyp = reflect.TypeOf([]string(nil)) |
| boolSliceTyp = reflect.TypeOf([]bool(nil)) |
| uintSliceTyp = reflect.TypeOf([]uint(nil)) |
| uint8SliceTyp = reflect.TypeOf([]uint8(nil)) |
| uint16SliceTyp = reflect.TypeOf([]uint16(nil)) |
| uint32SliceTyp = reflect.TypeOf([]uint32(nil)) |
| uint64SliceTyp = reflect.TypeOf([]uint64(nil)) |
| intSliceTyp = reflect.TypeOf([]int(nil)) |
| int8SliceTyp = reflect.TypeOf([]int8(nil)) |
| int16SliceTyp = reflect.TypeOf([]int16(nil)) |
| int32SliceTyp = reflect.TypeOf([]int32(nil)) |
| int64SliceTyp = reflect.TypeOf([]int64(nil)) |
| float32SliceTyp = reflect.TypeOf([]float32(nil)) |
| float64SliceTyp = reflect.TypeOf([]float64(nil)) |
| |
| mapIntfIntfTyp = reflect.TypeOf(map[interface{}]interface{}(nil)) |
| mapStrIntfTyp = reflect.TypeOf(map[string]interface{}(nil)) |
| mapStrStrTyp = reflect.TypeOf(map[string]string(nil)) |
| |
| mapIntIntfTyp = reflect.TypeOf(map[int]interface{}(nil)) |
| mapInt64IntfTyp = reflect.TypeOf(map[int64]interface{}(nil)) |
| mapUintIntfTyp = reflect.TypeOf(map[uint]interface{}(nil)) |
| mapUint64IntfTyp = reflect.TypeOf(map[uint64]interface{}(nil)) |
| |
| stringTyp = reflect.TypeOf("") |
| timeTyp = reflect.TypeOf(time.Time{}) |
| rawExtTyp = reflect.TypeOf(RawExt{}) |
| |
| mapBySliceTyp = reflect.TypeOf((*MapBySlice)(nil)).Elem() |
| binaryMarshalerTyp = reflect.TypeOf((*binaryMarshaler)(nil)).Elem() |
| binaryUnmarshalerTyp = reflect.TypeOf((*binaryUnmarshaler)(nil)).Elem() |
| |
| rawExtTypId = reflect.ValueOf(rawExtTyp).Pointer() |
| intfTypId = reflect.ValueOf(intfTyp).Pointer() |
| timeTypId = reflect.ValueOf(timeTyp).Pointer() |
| |
| intfSliceTypId = reflect.ValueOf(intfSliceTyp).Pointer() |
| strSliceTypId = reflect.ValueOf(strSliceTyp).Pointer() |
| |
| boolSliceTypId = reflect.ValueOf(boolSliceTyp).Pointer() |
| uintSliceTypId = reflect.ValueOf(uintSliceTyp).Pointer() |
| uint8SliceTypId = reflect.ValueOf(uint8SliceTyp).Pointer() |
| uint16SliceTypId = reflect.ValueOf(uint16SliceTyp).Pointer() |
| uint32SliceTypId = reflect.ValueOf(uint32SliceTyp).Pointer() |
| uint64SliceTypId = reflect.ValueOf(uint64SliceTyp).Pointer() |
| intSliceTypId = reflect.ValueOf(intSliceTyp).Pointer() |
| int8SliceTypId = reflect.ValueOf(int8SliceTyp).Pointer() |
| int16SliceTypId = reflect.ValueOf(int16SliceTyp).Pointer() |
| int32SliceTypId = reflect.ValueOf(int32SliceTyp).Pointer() |
| int64SliceTypId = reflect.ValueOf(int64SliceTyp).Pointer() |
| float32SliceTypId = reflect.ValueOf(float32SliceTyp).Pointer() |
| float64SliceTypId = reflect.ValueOf(float64SliceTyp).Pointer() |
| |
| mapStrStrTypId = reflect.ValueOf(mapStrStrTyp).Pointer() |
| mapIntfIntfTypId = reflect.ValueOf(mapIntfIntfTyp).Pointer() |
| mapStrIntfTypId = reflect.ValueOf(mapStrIntfTyp).Pointer() |
| mapIntIntfTypId = reflect.ValueOf(mapIntIntfTyp).Pointer() |
| mapInt64IntfTypId = reflect.ValueOf(mapInt64IntfTyp).Pointer() |
| mapUintIntfTypId = reflect.ValueOf(mapUintIntfTyp).Pointer() |
| mapUint64IntfTypId = reflect.ValueOf(mapUint64IntfTyp).Pointer() |
| // Id = reflect.ValueOf().Pointer() |
| // mapBySliceTypId = reflect.ValueOf(mapBySliceTyp).Pointer() |
| |
| binaryMarshalerTypId = reflect.ValueOf(binaryMarshalerTyp).Pointer() |
| binaryUnmarshalerTypId = reflect.ValueOf(binaryUnmarshalerTyp).Pointer() |
| |
| intBitsize uint8 = uint8(reflect.TypeOf(int(0)).Bits()) |
| uintBitsize uint8 = uint8(reflect.TypeOf(uint(0)).Bits()) |
| |
| bsAll0x00 = []byte{0, 0, 0, 0, 0, 0, 0, 0} |
| bsAll0xff = []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} |
| ) |
| |
| type binaryUnmarshaler interface { |
| UnmarshalBinary(data []byte) error |
| } |
| |
| type binaryMarshaler interface { |
| MarshalBinary() (data []byte, err error) |
| } |
| |
| // MapBySlice represents a slice which should be encoded as a map in the stream. |
| // The slice contains a sequence of key-value pairs. |
| type MapBySlice interface { |
| MapBySlice() |
| } |
| |
| // WARNING: DO NOT USE DIRECTLY. EXPORTED FOR GODOC BENEFIT. WILL BE REMOVED. |
| // |
| // BasicHandle encapsulates the common options and extension functions. |
| type BasicHandle struct { |
| extHandle |
| EncodeOptions |
| DecodeOptions |
| } |
| |
| // Handle is the interface for a specific encoding format. |
| // |
| // Typically, a Handle is pre-configured before first time use, |
| // and not modified while in use. Such a pre-configured Handle |
| // is safe for concurrent access. |
| type Handle interface { |
| writeExt() bool |
| getBasicHandle() *BasicHandle |
| newEncDriver(w encWriter) encDriver |
| newDecDriver(r decReader) decDriver |
| } |
| |
| // RawExt represents raw unprocessed extension data. |
| type RawExt struct { |
| Tag byte |
| Data []byte |
| } |
| |
| type extTypeTagFn struct { |
| rtid uintptr |
| rt reflect.Type |
| tag byte |
| encFn func(reflect.Value) ([]byte, error) |
| decFn func(reflect.Value, []byte) error |
| } |
| |
| type extHandle []*extTypeTagFn |
| |
| // AddExt registers an encode and decode function for a reflect.Type. |
| // Note that the type must be a named type, and specifically not |
| // a pointer or Interface. An error is returned if that is not honored. |
| // |
| // To Deregister an ext, call AddExt with 0 tag, nil encfn and nil decfn. |
| func (o *extHandle) AddExt( |
| rt reflect.Type, |
| tag byte, |
| encfn func(reflect.Value) ([]byte, error), |
| decfn func(reflect.Value, []byte) error, |
| ) (err error) { |
| // o is a pointer, because we may need to initialize it |
| if rt.PkgPath() == "" || rt.Kind() == reflect.Interface { |
| err = fmt.Errorf("codec.Handle.AddExt: Takes named type, especially not a pointer or interface: %T", |
| reflect.Zero(rt).Interface()) |
| return |
| } |
| |
| // o cannot be nil, since it is always embedded in a Handle. |
| // if nil, let it panic. |
| // if o == nil { |
| // err = errors.New("codec.Handle.AddExt: extHandle cannot be a nil pointer.") |
| // return |
| // } |
| |
| rtid := reflect.ValueOf(rt).Pointer() |
| for _, v := range *o { |
| if v.rtid == rtid { |
| v.tag, v.encFn, v.decFn = tag, encfn, decfn |
| return |
| } |
| } |
| |
| *o = append(*o, &extTypeTagFn{rtid, rt, tag, encfn, decfn}) |
| return |
| } |
| |
| func (o extHandle) getExt(rtid uintptr) *extTypeTagFn { |
| for _, v := range o { |
| if v.rtid == rtid { |
| return v |
| } |
| } |
| return nil |
| } |
| |
| func (o extHandle) getExtForTag(tag byte) *extTypeTagFn { |
| for _, v := range o { |
| if v.tag == tag { |
| return v |
| } |
| } |
| return nil |
| } |
| |
| func (o extHandle) getDecodeExtForTag(tag byte) ( |
| rv reflect.Value, fn func(reflect.Value, []byte) error) { |
| if x := o.getExtForTag(tag); x != nil { |
| // ext is only registered for base |
| rv = reflect.New(x.rt).Elem() |
| fn = x.decFn |
| } |
| return |
| } |
| |
| func (o extHandle) getDecodeExt(rtid uintptr) (tag byte, fn func(reflect.Value, []byte) error) { |
| if x := o.getExt(rtid); x != nil { |
| tag = x.tag |
| fn = x.decFn |
| } |
| return |
| } |
| |
| func (o extHandle) getEncodeExt(rtid uintptr) (tag byte, fn func(reflect.Value) ([]byte, error)) { |
| if x := o.getExt(rtid); x != nil { |
| tag = x.tag |
| fn = x.encFn |
| } |
| return |
| } |
| |
| type structFieldInfo struct { |
| encName string // encode name |
| |
| // only one of 'i' or 'is' can be set. If 'i' is -1, then 'is' has been set. |
| |
| is []int // (recursive/embedded) field index in struct |
| i int16 // field index in struct |
| omitEmpty bool |
| toArray bool // if field is _struct, is the toArray set? |
| |
| // tag string // tag |
| // name string // field name |
| // encNameBs []byte // encoded name as byte stream |
| // ikind int // kind of the field as an int i.e. int(reflect.Kind) |
| } |
| |
| func parseStructFieldInfo(fname string, stag string) *structFieldInfo { |
| if fname == "" { |
| panic("parseStructFieldInfo: No Field Name") |
| } |
| si := structFieldInfo{ |
| // name: fname, |
| encName: fname, |
| // tag: stag, |
| } |
| |
| if stag != "" { |
| for i, s := range strings.Split(stag, ",") { |
| if i == 0 { |
| if s != "" { |
| si.encName = s |
| } |
| } else { |
| switch s { |
| case "omitempty": |
| si.omitEmpty = true |
| case "toarray": |
| si.toArray = true |
| } |
| } |
| } |
| } |
| // si.encNameBs = []byte(si.encName) |
| return &si |
| } |
| |
| type sfiSortedByEncName []*structFieldInfo |
| |
| func (p sfiSortedByEncName) Len() int { |
| return len(p) |
| } |
| |
| func (p sfiSortedByEncName) Less(i, j int) bool { |
| return p[i].encName < p[j].encName |
| } |
| |
| func (p sfiSortedByEncName) Swap(i, j int) { |
| p[i], p[j] = p[j], p[i] |
| } |
| |
| // typeInfo keeps information about each type referenced in the encode/decode sequence. |
| // |
| // During an encode/decode sequence, we work as below: |
| // - If base is a built in type, en/decode base value |
| // - If base is registered as an extension, en/decode base value |
| // - If type is binary(M/Unm)arshaler, call Binary(M/Unm)arshal method |
| // - Else decode appropriately based on the reflect.Kind |
| type typeInfo struct { |
| sfi []*structFieldInfo // sorted. Used when enc/dec struct to map. |
| sfip []*structFieldInfo // unsorted. Used when enc/dec struct to array. |
| |
| rt reflect.Type |
| rtid uintptr |
| |
| // baseId gives pointer to the base reflect.Type, after deferencing |
| // the pointers. E.g. base type of ***time.Time is time.Time. |
| base reflect.Type |
| baseId uintptr |
| baseIndir int8 // number of indirections to get to base |
| |
| mbs bool // base type (T or *T) is a MapBySlice |
| |
| m bool // base type (T or *T) is a binaryMarshaler |
| unm bool // base type (T or *T) is a binaryUnmarshaler |
| mIndir int8 // number of indirections to get to binaryMarshaler type |
| unmIndir int8 // number of indirections to get to binaryUnmarshaler type |
| toArray bool // whether this (struct) type should be encoded as an array |
| } |
| |
| func (ti *typeInfo) indexForEncName(name string) int { |
| //tisfi := ti.sfi |
| const binarySearchThreshold = 16 |
| if sfilen := len(ti.sfi); sfilen < binarySearchThreshold { |
| // linear search. faster than binary search in my testing up to 16-field structs. |
| for i, si := range ti.sfi { |
| if si.encName == name { |
| return i |
| } |
| } |
| } else { |
| // binary search. adapted from sort/search.go. |
| h, i, j := 0, 0, sfilen |
| for i < j { |
| h = i + (j-i)/2 |
| if ti.sfi[h].encName < name { |
| i = h + 1 |
| } else { |
| j = h |
| } |
| } |
| if i < sfilen && ti.sfi[i].encName == name { |
| return i |
| } |
| } |
| return -1 |
| } |
| |
| func getTypeInfo(rtid uintptr, rt reflect.Type) (pti *typeInfo) { |
| var ok bool |
| cachedTypeInfoMutex.RLock() |
| pti, ok = cachedTypeInfo[rtid] |
| cachedTypeInfoMutex.RUnlock() |
| if ok { |
| return |
| } |
| |
| cachedTypeInfoMutex.Lock() |
| defer cachedTypeInfoMutex.Unlock() |
| if pti, ok = cachedTypeInfo[rtid]; ok { |
| return |
| } |
| |
| ti := typeInfo{rt: rt, rtid: rtid} |
| pti = &ti |
| |
| var indir int8 |
| if ok, indir = implementsIntf(rt, binaryMarshalerTyp); ok { |
| ti.m, ti.mIndir = true, indir |
| } |
| if ok, indir = implementsIntf(rt, binaryUnmarshalerTyp); ok { |
| ti.unm, ti.unmIndir = true, indir |
| } |
| if ok, _ = implementsIntf(rt, mapBySliceTyp); ok { |
| ti.mbs = true |
| } |
| |
| pt := rt |
| var ptIndir int8 |
| // for ; pt.Kind() == reflect.Ptr; pt, ptIndir = pt.Elem(), ptIndir+1 { } |
| for pt.Kind() == reflect.Ptr { |
| pt = pt.Elem() |
| ptIndir++ |
| } |
| if ptIndir == 0 { |
| ti.base = rt |
| ti.baseId = rtid |
| } else { |
| ti.base = pt |
| ti.baseId = reflect.ValueOf(pt).Pointer() |
| ti.baseIndir = ptIndir |
| } |
| |
| if rt.Kind() == reflect.Struct { |
| var siInfo *structFieldInfo |
| if f, ok := rt.FieldByName(structInfoFieldName); ok { |
| siInfo = parseStructFieldInfo(structInfoFieldName, f.Tag.Get(structTagName)) |
| ti.toArray = siInfo.toArray |
| } |
| sfip := make([]*structFieldInfo, 0, rt.NumField()) |
| rgetTypeInfo(rt, nil, make(map[string]bool), &sfip, siInfo) |
| |
| // // try to put all si close together |
| // const tryToPutAllStructFieldInfoTogether = true |
| // if tryToPutAllStructFieldInfoTogether { |
| // sfip2 := make([]structFieldInfo, len(sfip)) |
| // for i, si := range sfip { |
| // sfip2[i] = *si |
| // } |
| // for i := range sfip { |
| // sfip[i] = &sfip2[i] |
| // } |
| // } |
| |
| ti.sfip = make([]*structFieldInfo, len(sfip)) |
| ti.sfi = make([]*structFieldInfo, len(sfip)) |
| copy(ti.sfip, sfip) |
| sort.Sort(sfiSortedByEncName(sfip)) |
| copy(ti.sfi, sfip) |
| } |
| // sfi = sfip |
| cachedTypeInfo[rtid] = pti |
| return |
| } |
| |
| func rgetTypeInfo(rt reflect.Type, indexstack []int, fnameToHastag map[string]bool, |
| sfi *[]*structFieldInfo, siInfo *structFieldInfo, |
| ) { |
| // for rt.Kind() == reflect.Ptr { |
| // // indexstack = append(indexstack, 0) |
| // rt = rt.Elem() |
| // } |
| for j := 0; j < rt.NumField(); j++ { |
| f := rt.Field(j) |
| stag := f.Tag.Get(structTagName) |
| if stag == "-" { |
| continue |
| } |
| if r1, _ := utf8.DecodeRuneInString(f.Name); r1 == utf8.RuneError || !unicode.IsUpper(r1) { |
| continue |
| } |
| // if anonymous and there is no struct tag and its a struct (or pointer to struct), inline it. |
| if f.Anonymous && stag == "" { |
| ft := f.Type |
| for ft.Kind() == reflect.Ptr { |
| ft = ft.Elem() |
| } |
| if ft.Kind() == reflect.Struct { |
| indexstack2 := append(append(make([]int, 0, len(indexstack)+4), indexstack...), j) |
| rgetTypeInfo(ft, indexstack2, fnameToHastag, sfi, siInfo) |
| continue |
| } |
| } |
| // do not let fields with same name in embedded structs override field at higher level. |
| // this must be done after anonymous check, to allow anonymous field |
| // still include their child fields |
| if _, ok := fnameToHastag[f.Name]; ok { |
| continue |
| } |
| si := parseStructFieldInfo(f.Name, stag) |
| // si.ikind = int(f.Type.Kind()) |
| if len(indexstack) == 0 { |
| si.i = int16(j) |
| } else { |
| si.i = -1 |
| si.is = append(append(make([]int, 0, len(indexstack)+4), indexstack...), j) |
| } |
| |
| if siInfo != nil { |
| if siInfo.omitEmpty { |
| si.omitEmpty = true |
| } |
| } |
| *sfi = append(*sfi, si) |
| fnameToHastag[f.Name] = stag != "" |
| } |
| } |
| |
| func panicToErr(err *error) { |
| if recoverPanicToErr { |
| if x := recover(); x != nil { |
| //debug.PrintStack() |
| panicValToErr(x, err) |
| } |
| } |
| } |
| |
| func doPanic(tag string, format string, params ...interface{}) { |
| params2 := make([]interface{}, len(params)+1) |
| params2[0] = tag |
| copy(params2[1:], params) |
| panic(fmt.Errorf("%s: "+format, params2...)) |
| } |
| |
| func checkOverflowFloat32(f float64, doCheck bool) { |
| if !doCheck { |
| return |
| } |
| // check overflow (logic adapted from std pkg reflect/value.go OverflowFloat() |
| f2 := f |
| if f2 < 0 { |
| f2 = -f |
| } |
| if math.MaxFloat32 < f2 && f2 <= math.MaxFloat64 { |
| decErr("Overflow float32 value: %v", f2) |
| } |
| } |
| |
| func checkOverflow(ui uint64, i int64, bitsize uint8) { |
| // check overflow (logic adapted from std pkg reflect/value.go OverflowUint() |
| if bitsize == 0 { |
| return |
| } |
| if i != 0 { |
| if trunc := (i << (64 - bitsize)) >> (64 - bitsize); i != trunc { |
| decErr("Overflow int value: %v", i) |
| } |
| } |
| if ui != 0 { |
| if trunc := (ui << (64 - bitsize)) >> (64 - bitsize); ui != trunc { |
| decErr("Overflow uint value: %v", ui) |
| } |
| } |
| } |
