dashboard/app/entities.go - third_party/syzkaller - Git at Google

 // Copyright 2017 syzkaller project authors. All rights reserved.
 // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

 package dash

 import (
 	"fmt"
 	"regexp"
 	"strconv"
 	"time"

 	"github.com/google/syzkaller/dashboard/dashapi"
 	"github.com/google/syzkaller/pkg/hash"
 	"golang.org/x/net/context"
 	"google.golang.org/appengine/datastore"
 )

 // This file contains definitions of entities stored in datastore.

 const (
 	maxTextLen   = 200
 	MaxStringLen = 1024

 	maxCrashes = 40
 )

 type Manager struct {
 	Namespace      string
 	Name           string
 	Link           string
 	CurrentBuild   string
 	FailedBuildBug string
 	LastAlive      time.Time
 	CurrentUpTime  time.Duration
 }

 // ManagerStats holds per-day manager runtime stats.
 // Has Manager as parent entity. Keyed by Date.
 type ManagerStats struct {
 	Date             int // YYYYMMDD
 	MaxCorpus        int64
 	MaxCover         int64
 	TotalFuzzingTime time.Duration
 	TotalCrashes     int64
 	TotalExecs       int64
 }

 type Build struct {
 	Namespace       string
 	Manager         string
 	ID              string // unique ID generated by syz-ci
 	Type            BuildType
 	Time            time.Time
 	OS              string
 	Arch            string
 	VMArch          string
 	SyzkallerCommit string
 	CompilerID      string
 	KernelRepo      string
 	KernelBranch    string
 	KernelCommit    string
 	KernelConfig    int64 // reference to KernelConfig text entity
 }

 type Bug struct {
 	Namespace  string
 	Seq        int64 // sequences of the bug with the same title
 	Title      string
 	Status     int
 	DupOf      string
 	NumCrashes int64
 	NumRepro   int64
 	ReproLevel dashapi.ReproLevel
 	HasReport  bool
 	FirstTime  time.Time
 	LastTime   time.Time
 	Closed     time.Time
 	Reporting  []BugReporting
 	Commits    []string
 	PatchedOn  []string
 }

 type BugReporting struct {
 	Name       string // refers to Reporting.Name
 	ID         string // unique ID per BUG/BugReporting used in commucation with external systems
 	ExtID      string // arbitrary reporting ID that is passed back in dashapi.BugReport
 	Link       string
 	CC         string // additional emails added to CC list (|-delimited list)
 	ReproLevel dashapi.ReproLevel
 	Reported   time.Time
 	Closed     time.Time
 }

 type Crash struct {
 	Manager     string
 	BuildID     string
 	Time        time.Time
 	Maintainers []string `datastore:",noindex"`
 	Log         int64    // reference to CrashLog text entity
 	Report      int64    // reference to CrashReport text entity
 	ReproOpts   []byte   `datastore:",noindex"`
 	ReproSyz    int64    // reference to ReproSyz text entity
 	ReproC      int64    // reference to ReproC text entity
 	ReportLen   int
 }

 // ReportingState holds dynamic info associated with reporting.
 type ReportingState struct {
 	Entries []ReportingStateEntry
 }

 type ReportingStateEntry struct {
 	Namespace string
 	Name      string
 	// Current reporting quota consumption.
 	Sent int
 	Date int // YYYYMMDD
 }

 // Job represent a single patch testing job for syz-ci.
 // Later we may want to extend this to other types of jobs (hense the generic name):
 //   - test of a committed fix
 //   - reproduce crash
 //   - test that crash still happens on HEAD
 //   - crash bisect
 // Job has Bug as parent entity.
 type Job struct {
 	Created   time.Time
 	User      string
 	Reporting string
 	ExtID     string // email Message-ID
 	Link      string // web link for the job (e.g. email in the group)
 	Namespace string
 	Manager   string
 	BugTitle  string
 	CrashID   int64

 	// Provided by user:
 	KernelRepo   string
 	KernelBranch string
 	Patch        int64 // reference to Patch text entity

 	Attempts int // number of times we tried to execute this job
 	Started  time.Time
 	Finished time.Time // if set, job is finished

 	// Result of execution:
 	CrashTitle  string // if empty, we did not hit crash during testing
 	CrashLog    int64  // reference to CrashLog text entity
 	CrashReport int64  // reference to CrashReport text entity
 	BuildID     string
 	Error       int64 // reference to Error text entity, if set job failed

 	Reported bool // have we reported result back to user?
 }

 // Text holds text blobs (crash logs, reports, reproducers, etc).
 type Text struct {
 	Namespace string
 	Text      []byte `datastore:",noindex"` // gzip-compressed text
 }

 const (
 	BugStatusOpen = iota
 )

 const (
 	BugStatusFixed = 1000 + iota
 	BugStatusInvalid
 	BugStatusDup
 )

 const (
 	ReproLevelNone = dashapi.ReproLevelNone
 	ReproLevelSyz  = dashapi.ReproLevelSyz
 	ReproLevelC    = dashapi.ReproLevelC
 )

 type BuildType int

 const (
 	BuildNormal BuildType = iota
 	BuildFailed
 	BuildJob
 )

 // updateManager does transactional compare-and-swap on the manager and its current stats.
 func updateManager(c context.Context, ns, name string, fn func(mgr *Manager, stats *ManagerStats)) error {
 	date := timeDate(timeNow(c))
 	tx := func(c context.Context) error {
 		mgr := new(Manager)
 		mgrKey := datastore.NewKey(c, "Manager", fmt.Sprintf("%v-%v", ns, name), 0, nil)
 		if err := datastore.Get(c, mgrKey, mgr); err != nil {
 			if err != datastore.ErrNoSuchEntity {
 				return fmt.Errorf("failed to get manager %v/%v: %v", ns, name, err)
 			}
 			mgr = &Manager{
 				Namespace: ns,
 				Name:      name,
 			}
 		}
 		stats := new(ManagerStats)
 		statsKey := datastore.NewKey(c, "ManagerStats", "", int64(date), mgrKey)
 		if err := datastore.Get(c, statsKey, stats); err != nil {
 			if err != datastore.ErrNoSuchEntity {
 				return fmt.Errorf("failed to get stats %v/%v/%v: %v", ns, name, date, err)
 			}
 			stats = &ManagerStats{
 				Date: date,
 			}
 		}

 		fn(mgr, stats)

 		if _, err := datastore.Put(c, mgrKey, mgr); err != nil {
 			return fmt.Errorf("failed to put manager: %v", err)
 		}
 		if _, err := datastore.Put(c, statsKey, stats); err != nil {
 			return fmt.Errorf("failed to put manager stats: %v", err)
 		}
 		return nil
 	}
 	return datastore.RunInTransaction(c, tx, &datastore.TransactionOptions{Attempts: 10})
 }

 func loadAllManagers(c context.Context) ([]*Manager, []*datastore.Key, error) {
 	var managers []*Manager
 	keys, err := datastore.NewQuery("Manager").
 		GetAll(c, &managers)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to query managers: %v", err)
 	}
 	var result []*Manager
 	var resultKeys []*datastore.Key

 	for i, mgr := range managers {
 		if _, ok := config.Namespaces[mgr.Namespace].DecommissionedManagers[mgr.Name]; ok {
 			continue
 		}
 		result = append(result, mgr)
 		resultKeys = append(resultKeys, keys[i])
 	}
 	return result, resultKeys, nil
 }

 func buildKey(c context.Context, ns, id string) *datastore.Key {
 	if ns == "" {
 		panic("requesting build key outside of namespace")
 	}
 	h := hash.String([]byte(fmt.Sprintf("%v-%v", ns, id)))
 	return datastore.NewKey(c, "Build", h, 0, nil)
 }

 func loadBuild(c context.Context, ns, id string) (*Build, error) {
 	build := new(Build)
 	if err := datastore.Get(c, buildKey(c, ns, id), build); err != nil {
 		if err == datastore.ErrNoSuchEntity {
 			return nil, fmt.Errorf("unknown build %v/%v", ns, id)
 		}
 		return nil, fmt.Errorf("failed to get build %v/%v: %v", ns, id, err)
 	}
 	return build, nil
 }

 func (bug *Bug) displayTitle() string {
 	if bug.Seq == 0 {
 		return bug.Title
 	}
 	return fmt.Sprintf("%v (%v)", bug.Title, bug.Seq+1)
 }

 var displayTitleRe = regexp.MustCompile("^(.*) \\(([0-9]+)\\)$")

 func splitDisplayTitle(display string) (string, int64, error) {
 	match := displayTitleRe.FindStringSubmatchIndex(display)
 	if match == nil {
 		return display, 0, nil
 	}
 	title := display[match[2]:match[3]]
 	seqStr := display[match[4]:match[5]]
 	seq, err := strconv.ParseInt(seqStr, 10, 64)
 	if err != nil {
 		return "", 0, fmt.Errorf("failed to parse bug title: %v", err)
 	}
 	if seq <= 0 || seq > 1e6 {
 		return "", 0, fmt.Errorf("failed to parse bug title: seq=%v", seq)
 	}
 	return title, seq - 1, nil
 }

 func canonicalBug(c context.Context, bug *Bug) (*Bug, error) {
 	for {
 		if bug.Status != BugStatusDup {
 			return bug, nil
 		}
 		canon := new(Bug)
 		bugKey := datastore.NewKey(c, "Bug", bug.DupOf, 0, nil)
 		if err := datastore.Get(c, bugKey, canon); err != nil {
 			return nil, fmt.Errorf("failed to get dup bug %q for %q: %v",
 				bug.DupOf, bugKeyHash(bug.Namespace, bug.Title, bug.Seq), err)
 		}
 		bug = canon
 	}
 }

 func bugKeyHash(ns, title string, seq int64) string {
 	return hash.String([]byte(fmt.Sprintf("%v-%v-%v-%v", config.Namespaces[ns].Key, ns, title, seq)))
 }

 func bugReportingHash(bugHash, reporting string) string {
 	return hash.String([]byte(fmt.Sprintf("%v-%v", bugHash, reporting)))
 }

 func textLink(tag string, id int64) string {
 	if id == 0 {
 		return ""
 	}
 	return fmt.Sprintf("/text?tag=%v&id=%v", tag, id)
 }

 // timeDate returns t's date as a single int YYYYMMDD.
 func timeDate(t time.Time) int {
 	year, month, day := t.Date()
 	return year*10000 + int(month)*100 + day
 }
	// Copyright 2017 syzkaller project authors. All rights reserved.
	// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

	package dash

	import (
	"fmt"
	"regexp"
	"strconv"
	"time"

	"github.com/google/syzkaller/dashboard/dashapi"
	"github.com/google/syzkaller/pkg/hash"
	"golang.org/x/net/context"
	"google.golang.org/appengine/datastore"
	)

	// This file contains definitions of entities stored in datastore.

	const (
	maxTextLen = 200
	MaxStringLen = 1024

	maxCrashes = 40
	)

	type Manager struct {
	Namespace string
	Name string
	Link string
	CurrentBuild string
	FailedBuildBug string
	LastAlive time.Time
	CurrentUpTime time.Duration
	}

	// ManagerStats holds per-day manager runtime stats.
	// Has Manager as parent entity. Keyed by Date.
	type ManagerStats struct {
	Date int // YYYYMMDD
	MaxCorpus int64
	MaxCover int64
	TotalFuzzingTime time.Duration
	TotalCrashes int64
	TotalExecs int64
	}

	type Build struct {
	Namespace string
	Manager string
	ID string // unique ID generated by syz-ci
	Type BuildType
	Time time.Time
	OS string
	Arch string
	VMArch string
	SyzkallerCommit string
	CompilerID string
	KernelRepo string
	KernelBranch string
	KernelCommit string
	KernelConfig int64 // reference to KernelConfig text entity
	}

	type Bug struct {
	Namespace string
	Seq int64 // sequences of the bug with the same title
	Title string
	Status int
	DupOf string
	NumCrashes int64
	NumRepro int64
	ReproLevel dashapi.ReproLevel
	HasReport bool
	FirstTime time.Time
	LastTime time.Time
	Closed time.Time
	Reporting []BugReporting
	Commits []string
	PatchedOn []string
	}

	type BugReporting struct {
	Name string // refers to Reporting.Name
	ID string // unique ID per BUG/BugReporting used in commucation with external systems
	ExtID string // arbitrary reporting ID that is passed back in dashapi.BugReport
	Link string
	CC string // additional emails added to CC list (\|-delimited list)
	ReproLevel dashapi.ReproLevel
	Reported time.Time
	Closed time.Time
	}

	type Crash struct {
	Manager string
	BuildID string
	Time time.Time
	Maintainers []string `datastore:",noindex"`
	Log int64 // reference to CrashLog text entity
	Report int64 // reference to CrashReport text entity
	ReproOpts []byte `datastore:",noindex"`
	ReproSyz int64 // reference to ReproSyz text entity
	ReproC int64 // reference to ReproC text entity
	ReportLen int
	}

	// ReportingState holds dynamic info associated with reporting.
	type ReportingState struct {
	Entries []ReportingStateEntry
	}

	type ReportingStateEntry struct {
	Namespace string
	Name string
	// Current reporting quota consumption.
	Sent int
	Date int // YYYYMMDD
	}

	// Job represent a single patch testing job for syz-ci.
	// Later we may want to extend this to other types of jobs (hense the generic name):
	// - test of a committed fix
	// - reproduce crash
	// - test that crash still happens on HEAD
	// - crash bisect
	// Job has Bug as parent entity.
	type Job struct {
	Created time.Time
	User string
	Reporting string
	ExtID string // email Message-ID
	Link string // web link for the job (e.g. email in the group)
	Namespace string
	Manager string
	BugTitle string
	CrashID int64

	// Provided by user:
	KernelRepo string
	KernelBranch string
	Patch int64 // reference to Patch text entity

	Attempts int // number of times we tried to execute this job
	Started time.Time
	Finished time.Time // if set, job is finished

	// Result of execution:
	CrashTitle string // if empty, we did not hit crash during testing
	CrashLog int64 // reference to CrashLog text entity
	CrashReport int64 // reference to CrashReport text entity
	BuildID string
	Error int64 // reference to Error text entity, if set job failed

	Reported bool // have we reported result back to user?
	}

	// Text holds text blobs (crash logs, reports, reproducers, etc).
	type Text struct {
	Namespace string
	Text []byte `datastore:",noindex"` // gzip-compressed text
	}

	const (
	BugStatusOpen = iota
	)

	const (
	BugStatusFixed = 1000 + iota
	BugStatusInvalid
	BugStatusDup
	)

	const (
	ReproLevelNone = dashapi.ReproLevelNone
	ReproLevelSyz = dashapi.ReproLevelSyz
	ReproLevelC = dashapi.ReproLevelC
	)

	type BuildType int

	const (
	BuildNormal BuildType = iota
	BuildFailed
	BuildJob
	)

	// updateManager does transactional compare-and-swap on the manager and its current stats.
	func updateManager(c context.Context, ns, name string, fn func(mgr Manager, stats ManagerStats)) error {
	date := timeDate(timeNow(c))
	tx := func(c context.Context) error {
	mgr := new(Manager)
	mgrKey := datastore.NewKey(c, "Manager", fmt.Sprintf("%v-%v", ns, name), 0, nil)
	if err := datastore.Get(c, mgrKey, mgr); err != nil {
	if err != datastore.ErrNoSuchEntity {
	return fmt.Errorf("failed to get manager %v/%v: %v", ns, name, err)
	}
	mgr = &Manager{
	Namespace: ns,
	Name: name,
	}
	}
	stats := new(ManagerStats)
	statsKey := datastore.NewKey(c, "ManagerStats", "", int64(date), mgrKey)
	if err := datastore.Get(c, statsKey, stats); err != nil {
	if err != datastore.ErrNoSuchEntity {
	return fmt.Errorf("failed to get stats %v/%v/%v: %v", ns, name, date, err)
	}
	stats = &ManagerStats{
	Date: date,
	}
	}

	fn(mgr, stats)

	if _, err := datastore.Put(c, mgrKey, mgr); err != nil {
	return fmt.Errorf("failed to put manager: %v", err)
	}
	if _, err := datastore.Put(c, statsKey, stats); err != nil {
	return fmt.Errorf("failed to put manager stats: %v", err)
	}
	return nil
	}
	return datastore.RunInTransaction(c, tx, &datastore.TransactionOptions{Attempts: 10})
	}

	func loadAllManagers(c context.Context) ([]Manager, []datastore.Key, error) {
	var managers []*Manager
	keys, err := datastore.NewQuery("Manager").
	GetAll(c, &managers)
	if err != nil {
	return nil, nil, fmt.Errorf("failed to query managers: %v", err)
	}
	var result []*Manager
	var resultKeys []*datastore.Key

	for i, mgr := range managers {
	if _, ok := config.Namespaces[mgr.Namespace].DecommissionedManagers[mgr.Name]; ok {
	continue
	}
	result = append(result, mgr)
	resultKeys = append(resultKeys, keys[i])
	}
	return result, resultKeys, nil
	}

	func buildKey(c context.Context, ns, id string) *datastore.Key {
	if ns == "" {
	panic("requesting build key outside of namespace")
	}
	h := hash.String([]byte(fmt.Sprintf("%v-%v", ns, id)))
	return datastore.NewKey(c, "Build", h, 0, nil)
	}

	func loadBuild(c context.Context, ns, id string) (*Build, error) {
	build := new(Build)
	if err := datastore.Get(c, buildKey(c, ns, id), build); err != nil {
	if err == datastore.ErrNoSuchEntity {
	return nil, fmt.Errorf("unknown build %v/%v", ns, id)
	}
	return nil, fmt.Errorf("failed to get build %v/%v: %v", ns, id, err)
	}
	return build, nil
	}

	func (bug *Bug) displayTitle() string {
	if bug.Seq == 0 {
	return bug.Title
	}
	return fmt.Sprintf("%v (%v)", bug.Title, bug.Seq+1)
	}

	var displayTitleRe = regexp.MustCompile("^(.*) \\(([0-9]+)\\)$")

	func splitDisplayTitle(display string) (string, int64, error) {
	match := displayTitleRe.FindStringSubmatchIndex(display)
	if match == nil {
	return display, 0, nil
	}
	title := display[match[2]:match[3]]
	seqStr := display[match[4]:match[5]]
	seq, err := strconv.ParseInt(seqStr, 10, 64)
	if err != nil {
	return "", 0, fmt.Errorf("failed to parse bug title: %v", err)
	}
	if seq <= 0 \|\| seq > 1e6 {
	return "", 0, fmt.Errorf("failed to parse bug title: seq=%v", seq)
	}
	return title, seq - 1, nil
	}

	func canonicalBug(c context.Context, bug Bug) (Bug, error) {
	for {
	if bug.Status != BugStatusDup {
	return bug, nil
	}
	canon := new(Bug)
	bugKey := datastore.NewKey(c, "Bug", bug.DupOf, 0, nil)
	if err := datastore.Get(c, bugKey, canon); err != nil {
	return nil, fmt.Errorf("failed to get dup bug %q for %q: %v",
	bug.DupOf, bugKeyHash(bug.Namespace, bug.Title, bug.Seq), err)
	}
	bug = canon
	}
	}

	func bugKeyHash(ns, title string, seq int64) string {
	return hash.String([]byte(fmt.Sprintf("%v-%v-%v-%v", config.Namespaces[ns].Key, ns, title, seq)))
	}

	func bugReportingHash(bugHash, reporting string) string {
	return hash.String([]byte(fmt.Sprintf("%v-%v", bugHash, reporting)))
	}

	func textLink(tag string, id int64) string {
	if id == 0 {
	return ""
	}
	return fmt.Sprintf("/text?tag=%v&id=%v", tag, id)
	}

	// timeDate returns t's date as a single int YYYYMMDD.
	func timeDate(t time.Time) int {
	year, month, day := t.Date()
	return year10000 + int(month)100 + day
	}