datastore/prop.go

// Copyright 2011 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package datastore

import (
	"fmt"
	"reflect"
	"strings"
	"sync"
	"unicode"
)

// Entities with more than this many indexed properties will not be saved.
const maxIndexedProperties = 20000

// []byte fields more than 1 megabyte long will not be loaded or saved.
const maxBlobLen = 1 << 20

// Property is a name/value pair plus some metadata. A datastore entity's
// contents are loaded and saved as a sequence of Properties. An entity can
// have multiple Properties with the same name, provided that p.Multiple is
// true on all of that entity's Properties with that name.
type Property struct {
	// Name is the property name.
	Name string
	// Value is the property value. The valid types are:
	//	- int64
	//	- bool
	//	- string
	//	- float64
	//	- ByteString
	//	- *Key
	//	- time.Time
	//	- appengine.BlobKey
	//	- appengine.GeoPoint
	//	- []byte (up to 1 megabyte in length)
	//	- *Entity (representing a nested struct)
	// This set is smaller than the set of valid struct field types that the
	// datastore can load and save. A Property Value cannot be a slice (apart
	// from []byte); use multiple Properties instead. Also, a Value's type
	// must be explicitly on the list above; it is not sufficient for the
	// underlying type to be on that list. For example, a Value of "type
	// myInt64 int64" is invalid. Smaller-width integers and floats are also
	// invalid. Again, this is more restrictive than the set of valid struct
	// field types.
	//
	// A Value will have an opaque type when loading entities from an index,
	// such as via a projection query. Load entities into a struct instead
	// of a PropertyLoadSaver when using a projection query.
	//
	// A Value may also be the nil interface value; this is equivalent to
	// Python's None but not directly representable by a Go struct. Loading
	// a nil-valued property into a struct will set that field to the zero
	// value.
	Value interface{}
	// NoIndex is whether the datastore cannot index this property.
	NoIndex bool
	// Multiple is whether the entity can have multiple properties with
	// the same name. Even if a particular instance only has one property with
	// a certain name, Multiple should be true if a struct would best represent
	// it as a field of type []T instead of type T.
	Multiple bool
}

// An Entity is the value type for a nested struct.
// This type is only used for a Property's Value.
type Entity struct {
	Key        *Key
	Properties []Property
}

// ByteString is a short byte slice (up to 1500 bytes) that can be indexed.
type ByteString []byte

// PropertyLoadSaver can be converted from and to a slice of Properties.
type PropertyLoadSaver interface {
	Load([]Property) error
	Save() ([]Property, error)
}

// PropertyList converts a []Property to implement PropertyLoadSaver.
type PropertyList []Property

var (
	typeOfPropertyLoadSaver = reflect.TypeOf((*PropertyLoadSaver)(nil)).Elem()
	typeOfPropertyList      = reflect.TypeOf(PropertyList(nil))
)

// Load loads all of the provided properties into l.
// It does not first reset *l to an empty slice.
func (l *PropertyList) Load(p []Property) error {
	*l = append(*l, p...)
	return nil
}

// Save saves all of l's properties as a slice or Properties.
func (l *PropertyList) Save() ([]Property, error) {
	return *l, nil
}

// validPropertyName returns whether name consists of one or more valid Go
// identifiers joined by ".".
func validPropertyName(name string) bool {
	if name == "" {
		return false
	}
	for _, s := range strings.Split(name, ".") {
		if s == "" {
			return false
		}
		first := true
		for _, c := range s {
			if first {
				first = false
				if c != '_' && !unicode.IsLetter(c) {
					return false
				}
			} else {
				if c != '_' && !unicode.IsLetter(c) && !unicode.IsDigit(c) {
					return false
				}
			}
		}
	}
	return true
}

// structCodec describes how to convert a struct to and from a sequence of
// properties.
type structCodec struct {
	// fields gives the field codec for the structTag with the given name.
	fields map[string]fieldCodec
	// hasSlice is whether a struct or any of its nested or embedded structs
	// has a slice-typed field (other than []byte).
	hasSlice bool
	// keyField is the index of a *Key field with structTag __key__.
	// This field is not relevant for the top level struct, only for
	// nested structs.
	keyField int
	// complete is whether the structCodec is complete. An incomplete
	// structCodec may be encountered when walking a recursive struct.
	complete bool
}

// fieldCodec is a struct field's index and, if that struct field's type is
// itself a struct, that substruct's structCodec.
type fieldCodec struct {
	// path is the index path to the field
	path    []int
	noIndex bool
	// omitEmpty indicates that the field should be omitted on save
	// if empty.
	omitEmpty bool
	// structCodec is the codec fot the struct field at index 'path',
	// or nil if the field is not a struct.
	structCodec *structCodec
}

// structCodecs collects the structCodecs that have already been calculated.
var (
	structCodecsMutex sync.Mutex
	structCodecs      = make(map[reflect.Type]*structCodec)
)

// getStructCodec returns the structCodec for the given struct type.
func getStructCodec(t reflect.Type) (*structCodec, error) {
	structCodecsMutex.Lock()
	defer structCodecsMutex.Unlock()
	return getStructCodecLocked(t)
}

// getStructCodecLocked implements getStructCodec. The structCodecsMutex must
// be held when calling this function.
func getStructCodecLocked(t reflect.Type) (ret *structCodec, retErr error) {
	c, ok := structCodecs[t]
	if ok {
		return c, nil
	}
	c = &structCodec{
		fields: make(map[string]fieldCodec),
		// We initialize keyField to -1 so that the zero-value is not
		// misinterpreted as index 0.
		keyField: -1,
	}

	// Add c to the structCodecs map before we are sure it is good. If t is
	// a recursive type, it needs to find the incomplete entry for itself in
	// the map.
	structCodecs[t] = c
	defer func() {
		if retErr != nil {
			delete(structCodecs, t)
		}
	}()

	for i := 0; i < t.NumField(); i++ {
		f := t.Field(i)
		// Skip unexported fields.
		// Note that if f is an anonymous, unexported struct field,
		// we will promote its fields.
		if f.PkgPath != "" && !f.Anonymous {
			continue
		}

		tags := strings.Split(f.Tag.Get("datastore"), ",")
		name := tags[0]
		opts := make(map[string]bool)
		for _, t := range tags[1:] {
			opts[t] = true
		}
		switch {
		case name == "":
			if !f.Anonymous {
				name = f.Name
			}
		case name == "-":
			continue
		case name == "__key__":
			if f.Type != typeOfKeyPtr {
				return nil, fmt.Errorf("datastore: __key__ field on struct %v is not a *datastore.Key", t)
			}
			c.keyField = i
		case !validPropertyName(name):
			return nil, fmt.Errorf("datastore: struct tag has invalid property name: %q", name)
		}

		substructType, fIsSlice := reflect.Type(nil), false
		switch f.Type.Kind() {
		case reflect.Struct:
			substructType = f.Type
		case reflect.Slice:
			if f.Type.Elem().Kind() == reflect.Struct {
				substructType = f.Type.Elem()
			}
			fIsSlice = f.Type != typeOfByteSlice
			c.hasSlice = c.hasSlice || fIsSlice
		}

		var sub *structCodec
		if substructType != nil && substructType != typeOfTime && substructType != typeOfGeoPoint {
			var err error
			sub, err = getStructCodecLocked(substructType)
			if err != nil {
				return nil, err
			}
			if !sub.complete {
				return nil, fmt.Errorf("datastore: recursive struct: field %q", f.Name)
			}
			if fIsSlice && sub.hasSlice {
				return nil, fmt.Errorf(
					"datastore: flattening nested structs leads to a slice of slices: field %q", f.Name)
			}
			c.hasSlice = c.hasSlice || sub.hasSlice
			// If f is an anonymous struct field, we promote the substruct's fields up to this level
			// in the linked list of struct codecs.
			if f.Anonymous {
				for subname, subfield := range sub.fields {
					if name != "" {
						subname = name + "." + subname
					}
					if _, ok := c.fields[subname]; ok {
						return nil, fmt.Errorf("datastore: struct tag has repeated property name: %q", subname)
					}
					c.fields[subname] = fieldCodec{
						path:        append([]int{i}, subfield.path...),
						noIndex:     subfield.noIndex || opts["noindex"],
						omitEmpty:   subfield.omitEmpty,
						structCodec: subfield.structCodec,
					}
				}
				continue
			}
		}

		if _, ok := c.fields[name]; ok {
			return nil, fmt.Errorf("datastore: struct tag has repeated property name: %q", name)
		}
		c.fields[name] = fieldCodec{
			path:        []int{i},
			noIndex:     opts["noindex"],
			omitEmpty:   opts["omitempty"],
			structCodec: sub,
		}
	}
	c.complete = true
	return c, nil
}

// structPLS adapts a struct to be a PropertyLoadSaver.
type structPLS struct {
	v     reflect.Value
	codec *structCodec
}

// newStructPLS returns a structPLS, which implements the
// PropertyLoadSaver interface, for the struct pointer p.
func newStructPLS(p interface{}) (*structPLS, error) {
	v := reflect.ValueOf(p)
	if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct {
		return nil, ErrInvalidEntityType
	}
	v = v.Elem()
	codec, err := getStructCodec(v.Type())
	if err != nil {
		return nil, err
	}
	return &structPLS{v, codec}, nil
}

// LoadStruct loads the properties from p to dst.
// dst must be a struct pointer.
func LoadStruct(dst interface{}, p []Property) error {
	x, err := newStructPLS(dst)
	if err != nil {
		return err
	}
	return x.Load(p)
}

// SaveStruct returns the properties from src as a slice of Properties.
// src must be a struct pointer.
func SaveStruct(src interface{}) ([]Property, error) {
	x, err := newStructPLS(src)
	if err != nil {
		return nil, err
	}
	return x.Save()
}