bootstrap/bpdoc/bpdoc.go

package bpdoc

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/doc"
	"go/parser"
	"go/token"
	"io/ioutil"
	"reflect"
	"sort"
	"strconv"
	"strings"
	"sync"
	"text/template"

	"github.com/google/blueprint"
	"github.com/google/blueprint/proptools"
)

type Context struct {
	pkgFiles map[string][]string // Map of package name to source files, provided by constructor

	mutex sync.Mutex
	pkgs  map[string]*doc.Package    // Map of package name to parsed Go AST, protected by mutex
	ps    map[string]*PropertyStruct // Map of type name to property struct, protected by mutex
}

func NewContext(pkgFiles map[string][]string) *Context {
	return &Context{
		pkgFiles: pkgFiles,
		pkgs:     make(map[string]*doc.Package),
		ps:       make(map[string]*PropertyStruct),
	}
}

// Return the PropertyStruct associated with a property struct type.  The type should be in the
// format <package path>.<type name>
func (c *Context) PropertyStruct(pkgPath, name string, defaults reflect.Value) (*PropertyStruct, error) {
	ps := c.getPropertyStruct(pkgPath, name)

	if ps == nil {
		pkg, err := c.pkg(pkgPath)
		if err != nil {
			return nil, err
		}

		for _, t := range pkg.Types {
			if t.Name == name {
				ps, err = newPropertyStruct(t)
				if err != nil {
					return nil, err
				}
				ps = c.putPropertyStruct(pkgPath, name, ps)
			}
		}
	}

	if ps == nil {
		return nil, fmt.Errorf("package %q type %q not found", pkgPath, name)
	}

	ps = ps.Clone()
	ps.SetDefaults(defaults)

	return ps, nil
}

func (c *Context) getPropertyStruct(pkgPath, name string) *PropertyStruct {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	name = pkgPath + "." + name

	return c.ps[name]
}

func (c *Context) putPropertyStruct(pkgPath, name string, ps *PropertyStruct) *PropertyStruct {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	name = pkgPath + "." + name

	if c.ps[name] != nil {
		return c.ps[name]
	} else {
		c.ps[name] = ps
		return ps
	}
}

type PropertyStruct struct {
	Name       string
	Text       string
	Properties []Property
}

type Property struct {
	Name       string
	OtherNames []string
	Type       string
	Tag        reflect.StructTag
	Text       string
	OtherTexts []string
	Properties []Property
	Default    string
}

func (ps *PropertyStruct) Clone() *PropertyStruct {
	ret := *ps
	ret.Properties = append([]Property(nil), ret.Properties...)
	for i, prop := range ret.Properties {
		ret.Properties[i] = prop.Clone()
	}

	return &ret
}

func (p *Property) Clone() Property {
	ret := *p
	ret.Properties = append([]Property(nil), ret.Properties...)
	for i, prop := range ret.Properties {
		ret.Properties[i] = prop.Clone()
	}

	return ret
}

func (p *Property) Equal(other Property) bool {
	return p.Name == other.Name && p.Type == other.Type && p.Tag == other.Tag &&
		p.Text == other.Text && p.Default == other.Default &&
		stringArrayEqual(p.OtherNames, other.OtherNames) &&
		stringArrayEqual(p.OtherTexts, other.OtherTexts) &&
		p.SameSubProperties(other)
}

func (ps *PropertyStruct) SetDefaults(defaults reflect.Value) {
	setDefaults(ps.Properties, defaults)
}

func setDefaults(properties []Property, defaults reflect.Value) {
	for i := range properties {
		prop := &properties[i]
		fieldName := proptools.FieldNameForProperty(prop.Name)
		f := defaults.FieldByName(fieldName)
		if (f == reflect.Value{}) {
			panic(fmt.Errorf("property %q does not exist in %q", fieldName, defaults.Type()))
		}

		if reflect.DeepEqual(f.Interface(), reflect.Zero(f.Type()).Interface()) {
			continue
		}

		if f.Type().Kind() == reflect.Interface {
			f = f.Elem()
		}

		if f.Type().Kind() == reflect.Ptr {
			f = f.Elem()
		}

		if f.Type().Kind() == reflect.Struct {
			setDefaults(prop.Properties, f)
		} else {
			prop.Default = fmt.Sprintf("%v", f.Interface())
		}
	}
}

func stringArrayEqual(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}

	for i := range a {
		if a[i] != b[i] {
			return false
		}
	}

	return true
}

func (p *Property) SameSubProperties(other Property) bool {
	if len(p.Properties) != len(other.Properties) {
		return false
	}

	for i := range p.Properties {
		if !p.Properties[i].Equal(other.Properties[i]) {
			return false
		}
	}

	return true
}

func (ps *PropertyStruct) GetByName(name string) *Property {
	return getByName(name, "", &ps.Properties)
}

func getByName(name string, prefix string, props *[]Property) *Property {
	for i := range *props {
		if prefix+(*props)[i].Name == name {
			return &(*props)[i]
		} else if strings.HasPrefix(name, prefix+(*props)[i].Name+".") {
			return getByName(name, prefix+(*props)[i].Name+".", &(*props)[i].Properties)
		}
	}
	return nil
}

func (p *Property) Nest(nested *PropertyStruct) {
	//p.Name += "(" + nested.Name + ")"
	//p.Text += "(" + nested.Text + ")"
	p.Properties = append(p.Properties, nested.Properties...)
}

func newPropertyStruct(t *doc.Type) (*PropertyStruct, error) {
	typeSpec := t.Decl.Specs[0].(*ast.TypeSpec)
	ps := PropertyStruct{
		Name: t.Name,
		Text: t.Doc,
	}

	structType, ok := typeSpec.Type.(*ast.StructType)
	if !ok {
		return nil, fmt.Errorf("type of %q is not a struct", t.Name)
	}

	var err error
	ps.Properties, err = structProperties(structType)
	if err != nil {
		return nil, err
	}

	return &ps, nil
}

func structProperties(structType *ast.StructType) (props []Property, err error) {
	for _, f := range structType.Fields.List {
		names := f.Names
		if names == nil {
			// Anonymous fields have no name, use the type as the name
			// TODO: hide the name and make the properties show up in the embedding struct
			if t, ok := f.Type.(*ast.Ident); ok {
				names = append(names, t)
			}
		}
		for _, n := range names {
			var name, typ, tag, text string
			var innerProps []Property
			if n != nil {
				name = proptools.PropertyNameForField(n.Name)
			}
			if f.Doc != nil {
				text = f.Doc.Text()
			}
			if f.Tag != nil {
				tag, err = strconv.Unquote(f.Tag.Value)
				if err != nil {
					return nil, err
				}
			}

			t := f.Type
			if star, ok := t.(*ast.StarExpr); ok {
				t = star.X
			}
			switch a := t.(type) {
			case *ast.ArrayType:
				typ = "list of strings"
			case *ast.InterfaceType:
				typ = "interface"
			case *ast.Ident:
				typ = a.Name
			case *ast.StructType:
				innerProps, err = structProperties(a)
				if err != nil {
					return nil, err
				}
			default:
				typ = fmt.Sprintf("%T", f.Type)
			}

			props = append(props, Property{
				Name:       name,
				Type:       typ,
				Tag:        reflect.StructTag(tag),
				Text:       text,
				Properties: innerProps,
			})
		}
	}

	return props, nil
}

func (ps *PropertyStruct) ExcludeByTag(key, value string) {
	filterPropsByTag(&ps.Properties, key, value, true)
}

func (ps *PropertyStruct) IncludeByTag(key, value string) {
	filterPropsByTag(&ps.Properties, key, value, false)
}

func filterPropsByTag(props *[]Property, key, value string, exclude bool) {
	// Create a slice that shares the storage of props but has 0 length.  Appending up to
	// len(props) times to this slice will overwrite the original slice contents
	filtered := (*props)[:0]
	for _, x := range *props {
		tag := x.Tag.Get(key)
		for _, entry := range strings.Split(tag, ",") {
			if (entry == value) == !exclude {
				filtered = append(filtered, x)
			}
		}
	}

	*props = filtered
}

// Package AST generation and storage
func (c *Context) pkg(pkgPath string) (*doc.Package, error) {
	pkg := c.getPackage(pkgPath)
	if pkg == nil {
		if files, ok := c.pkgFiles[pkgPath]; ok {
			var err error
			pkgAST, err := NewPackageAST(files)
			if err != nil {
				return nil, err
			}
			pkg = doc.New(pkgAST, pkgPath, doc.AllDecls)
			pkg = c.putPackage(pkgPath, pkg)
		} else {
			return nil, fmt.Errorf("unknown package %q", pkgPath)
		}
	}
	return pkg, nil
}

func (c *Context) getPackage(pkgPath string) *doc.Package {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	return c.pkgs[pkgPath]
}

func (c *Context) putPackage(pkgPath string, pkg *doc.Package) *doc.Package {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	if c.pkgs[pkgPath] != nil {
		return c.pkgs[pkgPath]
	} else {
		c.pkgs[pkgPath] = pkg
		return pkg
	}
}

func NewPackageAST(files []string) (*ast.Package, error) {
	asts := make(map[string]*ast.File)

	fset := token.NewFileSet()
	for _, file := range files {
		ast, err := parser.ParseFile(fset, file, nil, parser.ParseComments)
		if err != nil {
			return nil, err
		}
		asts[file] = ast
	}

	pkg, _ := ast.NewPackage(fset, asts, nil, nil)
	return pkg, nil
}

func Write(filename string, pkgFiles map[string][]string,
	moduleTypePropertyStructs map[string][]interface{}) error {

	c := NewContext(pkgFiles)

	var moduleTypeList []*moduleType
	for moduleType, propertyStructs := range moduleTypePropertyStructs {
		mt, err := getModuleType(c, moduleType, propertyStructs)
		if err != nil {
			return err
		}
		removeEmptyPropertyStructs(mt)
		collapseDuplicatePropertyStructs(mt)
		collapseNestedPropertyStructs(mt)
		combineDuplicateProperties(mt)
		moduleTypeList = append(moduleTypeList, mt)
	}

	sort.Sort(moduleTypeByName(moduleTypeList))

	buf := &bytes.Buffer{}

	unique := 0

	tmpl, err := template.New("file").Funcs(map[string]interface{}{
		"unique": func() int {
			unique++
			return unique
		}}).Parse(fileTemplate)
	if err != nil {
		return err
	}

	err = tmpl.Execute(buf, moduleTypeList)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(filename, buf.Bytes(), 0666)
	if err != nil {
		return err
	}

	return nil
}

func getModuleType(c *Context, moduleTypeName string,
	propertyStructs []interface{}) (*moduleType, error) {
	mt := &moduleType{
		Name: moduleTypeName,
		//Text: c.ModuleTypeDocs(moduleType),
	}

	for _, s := range propertyStructs {
		v := reflect.ValueOf(s).Elem()
		t := v.Type()

		// Ignore property structs with unexported or unnamed types
		if t.PkgPath() == "" {
			continue
		}
		ps, err := c.PropertyStruct(t.PkgPath(), t.Name(), v)
		if err != nil {
			return nil, err
		}
		ps.ExcludeByTag("blueprint", "mutated")

		for nestedName, nestedValue := range nestedPropertyStructs(v) {
			nestedType := nestedValue.Type()

			// Ignore property structs with unexported or unnamed types
			if nestedType.PkgPath() == "" {
				continue
			}
			nested, err := c.PropertyStruct(nestedType.PkgPath(), nestedType.Name(), nestedValue)
			if err != nil {
				return nil, err
			}
			nested.ExcludeByTag("blueprint", "mutated")
			nestPoint := ps.GetByName(nestedName)
			if nestPoint == nil {
				return nil, fmt.Errorf("nesting point %q not found", nestedName)
			}

			key, value, err := blueprint.HasFilter(nestPoint.Tag)
			if err != nil {
				return nil, err
			}
			if key != "" {
				nested.IncludeByTag(key, value)
			}

			nestPoint.Nest(nested)
		}
		mt.PropertyStructs = append(mt.PropertyStructs, ps)
	}

	return mt, nil
}

func nestedPropertyStructs(s reflect.Value) map[string]reflect.Value {
	ret := make(map[string]reflect.Value)
	var walk func(structValue reflect.Value, prefix string)
	walk = func(structValue reflect.Value, prefix string) {
		typ := structValue.Type()
		for i := 0; i < structValue.NumField(); i++ {
			field := typ.Field(i)
			if field.PkgPath != "" {
				// The field is not exported so just skip it.
				continue
			}

			fieldValue := structValue.Field(i)

			switch fieldValue.Kind() {
			case reflect.Bool, reflect.String, reflect.Slice, reflect.Int, reflect.Uint:
				// Nothing
			case reflect.Struct:
				walk(fieldValue, prefix+proptools.PropertyNameForField(field.Name)+".")
			case reflect.Ptr, reflect.Interface:
				if !fieldValue.IsNil() {
					// We leave the pointer intact and zero out the struct that's
					// pointed to.
					elem := fieldValue.Elem()
					if fieldValue.Kind() == reflect.Interface {
						if elem.Kind() != reflect.Ptr {
							panic(fmt.Errorf("can't get type of field %q: interface "+
								"refers to a non-pointer", field.Name))
						}
						elem = elem.Elem()
					}
					if elem.Kind() == reflect.Struct {
						nestPoint := prefix + proptools.PropertyNameForField(field.Name)
						ret[nestPoint] = elem
						walk(elem, nestPoint+".")
					}
				}
			default:
				panic(fmt.Errorf("unexpected kind for property struct field %q: %s",
					field.Name, fieldValue.Kind()))
			}
		}

	}

	walk(s, "")
	return ret
}

// Remove any property structs that have no exported fields
func removeEmptyPropertyStructs(mt *moduleType) {
	for i := 0; i < len(mt.PropertyStructs); i++ {
		if len(mt.PropertyStructs[i].Properties) == 0 {
			mt.PropertyStructs = append(mt.PropertyStructs[:i], mt.PropertyStructs[i+1:]...)
			i--
		}
	}
}

// Squashes duplicates of the same property struct into single entries
func collapseDuplicatePropertyStructs(mt *moduleType) {
	var collapsed []*PropertyStruct

propertyStructLoop:
	for _, from := range mt.PropertyStructs {
		for _, to := range collapsed {
			if from.Name == to.Name {
				collapseDuplicateProperties(&to.Properties, &from.Properties)
				continue propertyStructLoop
			}
		}
		collapsed = append(collapsed, from)
	}
	mt.PropertyStructs = collapsed
}

func collapseDuplicateProperties(to, from *[]Property) {
propertyLoop:
	for _, f := range *from {
		for i := range *to {
			t := &(*to)[i]
			if f.Name == t.Name {
				collapseDuplicateProperties(&t.Properties, &f.Properties)
				continue propertyLoop
			}
		}
		*to = append(*to, f)
	}
}

// Find all property structs that only contain structs, and move their children up one with
// a prefixed name
func collapseNestedPropertyStructs(mt *moduleType) {
	for _, ps := range mt.PropertyStructs {
		collapseNestedProperties(&ps.Properties)
	}
}

func collapseNestedProperties(p *[]Property) {
	var n []Property

	for _, parent := range *p {
		var containsProperty bool
		for j := range parent.Properties {
			child := &parent.Properties[j]
			if len(child.Properties) > 0 {
				collapseNestedProperties(&child.Properties)
			} else {
				containsProperty = true
			}
		}
		if containsProperty || len(parent.Properties) == 0 {
			n = append(n, parent)
		} else {
			for j := range parent.Properties {
				child := parent.Properties[j]
				child.Name = parent.Name + "." + child.Name
				n = append(n, child)
			}
		}
	}
	*p = n
}

func combineDuplicateProperties(mt *moduleType) {
	for _, ps := range mt.PropertyStructs {
		combineDuplicateSubProperties(&ps.Properties)
	}
}

func combineDuplicateSubProperties(p *[]Property) {
	var n []Property
propertyLoop:
	for _, child := range *p {
		if len(child.Properties) > 0 {
			combineDuplicateSubProperties(&child.Properties)
			for i := range n {
				s := &n[i]
				if s.SameSubProperties(child) {
					s.OtherNames = append(s.OtherNames, child.Name)
					s.OtherTexts = append(s.OtherTexts, child.Text)
					continue propertyLoop
				}
			}
		}
		n = append(n, child)
	}

	*p = n
}

type moduleTypeByName []*moduleType

func (l moduleTypeByName) Len() int           { return len(l) }
func (l moduleTypeByName) Less(i, j int) bool { return l[i].Name < l[j].Name }
func (l moduleTypeByName) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }

type moduleType struct {
	Name            string
	Text            string
	PropertyStructs []*PropertyStruct
}

var (
	fileTemplate = `
<html>
<head>
<title>Build Docs</title>
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css">
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.4/jquery.min.js"></script>
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
</head>
<body>
<h1>Build Docs</h1>
<div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
  {{range .}}
    {{ $collapseIndex := unique }}
    <div class="panel panel-default">
      <div class="panel-heading" role="tab" id="heading{{$collapseIndex}}">
        <h2 class="panel-title">
          <a class="collapsed" role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse{{$collapseIndex}}" aria-expanded="false" aria-controls="collapse{{$collapseIndex}}">
             {{.Name}}
          </a>
        </h2>
      </div>
    </div>
    <div id="collapse{{$collapseIndex}}" class="panel-collapse collapse" role="tabpanel" aria-labelledby="heading{{$collapseIndex}}">
      <div class="panel-body">
        <p>{{.Text}}</p>
        {{range .PropertyStructs}}
          <p>{{.Text}}</p>
          {{template "properties" .Properties}}
        {{end}}
      </div>
    </div>
  {{end}}
</div>
</body>
</html>

{{define "properties"}}
  <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
    {{range .}}
      {{$collapseIndex := unique}}
      {{if .Properties}}
        <div class="panel panel-default">
          <div class="panel-heading" role="tab" id="heading{{$collapseIndex}}">
            <h4 class="panel-title">
              <a class="collapsed" role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse{{$collapseIndex}}" aria-expanded="false" aria-controls="collapse{{$collapseIndex}}">
                 {{.Name}}{{range .OtherNames}}, {{.}}{{end}}
              </a>
            </h4>
          </div>
        </div>
        <div id="collapse{{$collapseIndex}}" class="panel-collapse collapse" role="tabpanel" aria-labelledby="heading{{$collapseIndex}}">
          <div class="panel-body">
            <p>{{.Text}}</p>
            {{range .OtherTexts}}<p>{{.}}</p>{{end}}
            {{template "properties" .Properties}}
          </div>
        </div>
      {{else}}
        <div>
          <h4>{{.Name}}{{range .OtherNames}}, {{.}}{{end}}</h4>
          <p>{{.Text}}</p>
          {{range .OtherTexts}}<p>{{.}}</p>{{end}}
          <p><i>Type: {{.Type}}</i></p>
          {{if .Default}}<p><i>Default: {{.Default}}</i></p>{{end}}
        </div>
      {{end}}
    {{end}}
  </div>
{{end}}
`
)