[attributes] Adds "attributes" XML attributes to manifest
This patch adds "attributes" attribute to projects and packages, which
will help to group different projects and packages and can be used in
git attributes in the future. When "attributes" is explictly set for a
project or package, it will not be fetched by default. These projects
can be fetched by setting optional attributes using
'jiri init -fetch-optional=att1,attr2'.
Bug: VC-154
Test: CQ
Change-Id: I40e085cd2efef0d89c82fb5254c5908ddcf3fabc
diff --git a/cmd/jiri/fetch_pkgs.go b/cmd/jiri/fetch_pkgs.go
index 72882c4..0610f21 100644
--- a/cmd/jiri/fetch_pkgs.go
+++ b/cmd/jiri/fetch_pkgs.go
@@ -53,6 +53,9 @@
if err != nil {
return err
}
+ if err := project.FilterOptionalProjectsPackages(jirix, jirix.FetchingAttrs, nil, pkgs); err != nil {
+ return err
+ }
if len(pkgs) > 0 {
return project.FetchPackages(jirix, projs, pkgs, fetchPkgsFlags.fetchPkgsTimeout)
}
diff --git a/cmd/jiri/init.go b/cmd/jiri/init.go
index 1bc5e7e..8b6a27f 100644
--- a/cmd/jiri/init.go
+++ b/cmd/jiri/init.go
@@ -45,6 +45,11 @@
enableLockfileFlag string
lockfileNameFlag string
prebuiltJSON string
+ optionalAttrs string
+)
+
+const (
+ optionalAttrsNotSet = "[ATTRIBUTES_NOT_SET]"
)
func init() {
@@ -58,6 +63,9 @@
cmdInit.Flags.StringVar(&enableLockfileFlag, "enable-lockfile", "", "Enable lockfile enforcement")
cmdInit.Flags.StringVar(&lockfileNameFlag, "lockfile-name", "", "Set up filename of lockfile")
cmdInit.Flags.StringVar(&prebuiltJSON, "prebuilt-json", "", "Set up filename for prebuilt json file")
+ // Empty string is not used as default value for optionalAttrs as we
+ // use empty string to clear existing saved attributes.
+ cmdInit.Flags.StringVar(&optionalAttrs, "fetch-optional", optionalAttrsNotSet, "Set up attributes of optional projects and packages that should be fetched by jiri.")
}
func runInit(env *cmdline.Env, args []string) error {
@@ -145,6 +153,10 @@
}
}
+ if optionalAttrs != optionalAttrsNotSet {
+ config.FetchingAttrs = optionalAttrs
+ }
+
if ssoCookieFlag != "" {
config.SsoCookiePath = ssoCookieFlag
}
diff --git a/cmd/jiri/run_hooks.go b/cmd/jiri/run_hooks.go
index 3b93bf9..0bcc108 100644
--- a/cmd/jiri/run_hooks.go
+++ b/cmd/jiri/run_hooks.go
@@ -59,6 +59,9 @@
if err = project.RunHooks(jirix, hooks, runHooksFlags.hookTimeout); err != nil {
return err
}
+ if err := project.FilterOptionalProjectsPackages(jirix, jirix.FetchingAttrs, nil, pkgs); err != nil {
+ return err
+ }
// Get packages if the fetchPackages is true
if runHooksFlags.fetchPackages && len(pkgs) > 0 {
// Extend timeout for packages to be 5 times the timeout of a single hook.
diff --git a/jiritest/fake.go b/jiritest/fake.go
index e914845..76f2935 100644
--- a/jiritest/fake.go
+++ b/jiritest/fake.go
@@ -35,9 +35,8 @@
// closure must be run to cleanup temporary directories and restore the original
// environment; typically it is run as a defer function.
func NewFakeJiriRoot(t *testing.T) (*FakeJiriRoot, func()) {
+ // lockfiles are disabled in tests by defaults
jirix, cleanup := NewX(t)
- // Disable lockfile in tests
- jirix.LockfileEnabled = false
fake := &FakeJiriRoot{
X: jirix,
Projects: map[string]string{},
@@ -121,6 +120,19 @@
return nil
}
+// AddPackage adds the given package to a remote manifest.
+func (fake FakeJiriRoot) AddPackage(pkg project.Package) error {
+ manifest, err := fake.ReadRemoteManifest()
+ if err != nil {
+ return err
+ }
+ manifest.Packages = append(manifest.Packages, pkg)
+ if err := fake.WriteRemoteManifest(manifest); err != nil {
+ return err
+ }
+ return nil
+}
+
// DisableRemoteManifestPush disables pushes to the remote manifest
// repository.
func (fake FakeJiriRoot) DisableRemoteManifestPush() error {
diff --git a/jiritest/x.go b/jiritest/x.go
index e848bfc..6709ea0 100644
--- a/jiritest/x.go
+++ b/jiritest/x.go
@@ -36,5 +36,5 @@
t.Fatalf("RemoveAll(%q) failed: %v", root, err)
}
}
- return &jiri.X{Context: ctx, Root: root, Jobs: jiri.DefaultJobs, Color: color, Logger: logger, Attempts: 1}, cleanup
+ return &jiri.X{Context: ctx, Root: root, Jobs: jiri.DefaultJobs, Color: color, Logger: logger, Attempts: 1, LockfileEnabled: false}, cleanup
}
diff --git a/project/loader.go b/project/loader.go
index d1b0b99..8f7897d 100644
--- a/project/loader.go
+++ b/project/loader.go
@@ -9,6 +9,7 @@
"io/ioutil"
"os"
"path/filepath"
+ "reflect"
"strings"
"fuchsia.googlesource.com/jiri"
@@ -324,6 +325,12 @@
return err
}
+ if jirix.UsingSnapshot {
+ // using attributes defined in snapshot file instead of
+ // using predefined ones in jiri init.
+ jirix.FetchingAttrs = m.Attributes
+ }
+
// Process remote imports.
for _, remote := range m.Imports {
nextRoot := filepath.Join(root, remote.Root)
@@ -377,6 +384,9 @@
// Collect projects.
for _, project := range m.Projects {
+ // normalize project attributes
+ project.ComputedAttributes = newAttributes(project.Attributes)
+ project.Attributes = project.ComputedAttributes.String()
// Make paths absolute by prepending <root>.
project.absolutizePaths(filepath.Join(jirix.Root, root))
@@ -401,7 +411,7 @@
}
}
- if dup, ok := ld.Projects[key]; ok && dup != project {
+ if dup, ok := ld.Projects[key]; ok && !reflect.DeepEqual(dup, project) {
// TODO(toddw): Tell the user the other conflicting file.
return fmt.Errorf("duplicate project %q found in %q", key, shortFileName(jirix.Root, repoPath, file, ref))
}
@@ -442,6 +452,9 @@
}
for _, pkg := range m.Packages {
+ // normalize package attributes.
+ pkg.ComputedAttributes = newAttributes(pkg.Attributes)
+ pkg.Attributes = pkg.ComputedAttributes.String()
key := pkg.Key()
ld.Packages[key] = pkg
}
diff --git a/project/manifest.go b/project/manifest.go
index 0933df0..afd3d9c 100644
--- a/project/manifest.go
+++ b/project/manifest.go
@@ -36,6 +36,7 @@
// Manifest represents a setting used for updating the universe.
type Manifest struct {
Version string `xml:"version,attr,omitempty"`
+ Attributes string `xml:"attributes,attr,omitempty"`
Imports []Import `xml:"imports>import"`
LocalImports []LocalImport `xml:"imports>localimport"`
Projects []Project `xml:"projects>project"`
@@ -116,6 +117,7 @@
x.Hooks = append([]Hook(nil), m.Hooks...)
x.Packages = append([]Package(nil), m.Packages...)
x.Version = m.Version
+ x.Attributes = m.Attributes
return x
}
@@ -427,14 +429,39 @@
// Package struct represents the <package> tag in manifest files.
type Package struct {
- Name string `xml:"name,attr"`
- Version string `xml:"version,attr"`
- Path string `xml:"path,attr,omitempty"`
- Internal bool `xml:"internal,attr,omitempty"`
- Platforms string `xml:"platforms,attr,omitempty"`
- Flag string `xml:"flag,attr,omitempty"`
+ // Name represents the remote cipd path of the package.
+ Name string `xml:"name,attr"`
+
+ // Version represents the version tag of the cipd package.
+ Version string `xml:"version,attr"`
+
+ // Path stores the local path of fetched cipd package.
+ Path string `xml:"path,attr,omitempty"`
+
+ // Internal marks if this package require special permission
+ // for access
+ Internal bool `xml:"internal,attr,omitempty"`
+
+ // Platforms defines the available platforms for this cipd package.
+ Platforms string `xml:"platforms,attr,omitempty"`
+
+ // Flag defines the content that should be written to a file when
+ // this package is successfully fetched.
+ Flag string `xml:"flag,attr,omitempty"`
+
+ // Attributes store the the list attributes for this package.
+ // When it starts with "+", a computed default attributes will
+ // be appended.
+ Attributes string `xml:"attributes,attr,omitempty"`
+
+ // Instances store the known instance ids for this package.
+ // It is mainly used by snapshot file.
Instances []PackageInstance `xml:"instance"`
XMLName struct{} `xml:"package"`
+
+ // ComputedAttributes stores computed attributes object
+ // which is easiler to perform matching and comparing.
+ ComputedAttributes attributes `xml:"-"`
}
type PackageKey string
diff --git a/project/project.go b/project/project.go
index e08680e..e4b1194 100644
--- a/project/project.go
+++ b/project/project.go
@@ -72,6 +72,11 @@
// this project.
GitHooks string `xml:"githooks,attr,omitempty"`
+ // Attributes is a list of attributes for a project seperated by comma that
+ // will be helpful to group projects with similar purposes together. By
+ // default, jiri will use the directory name as attribute.
+ Attributes string `xml:"attributes,attr,omitempty"`
+
XMLName struct{} `xml:"project"`
// This is used to store computed key. This is useful when remote and
@@ -80,6 +85,10 @@
// This stores the local configuration file for the project
LocalConfig LocalConfig `xml:"-"`
+
+ // ComputedAttributes stores computed attributes object
+ // which is easiler to perform matching and comparing.
+ ComputedAttributes attributes `xml:"-"`
}
// ProjectsByPath implements the Sort interface. It sorts Projects by
@@ -242,6 +251,58 @@
}
}
+type attributes map[string]bool
+
+// newAttributes will create a new attributes object
+// which is used in Project and Package objects.
+func newAttributes(attrs string) attributes {
+ retMap := make(attributes)
+ if strings.HasPrefix(attrs, "+") {
+ attrs = attrs[1:]
+ }
+ for _, v := range strings.Split(attrs, ",") {
+ key := strings.TrimSpace(v)
+ if key != "" {
+ retMap[key] = true
+ }
+ }
+ return retMap
+}
+
+func (m attributes) IsEmpty() bool {
+ return len(m) == 0
+}
+
+func (m attributes) Add(other attributes) {
+ for k := range other {
+ if _, ok := m[k]; !ok {
+ m[k] = true
+ }
+ }
+}
+
+func (m attributes) Match(other attributes) bool {
+ for k := range other {
+ if _, ok := m[k]; ok {
+ return true
+ }
+ }
+ return false
+}
+
+func (m attributes) String() string {
+ var buf bytes.Buffer
+ first := true
+ for k := range m {
+ if !first {
+ buf.WriteString(",")
+ }
+ buf.WriteString(k)
+ first = false
+ }
+ return buf.String()
+}
+
// ProjectLock describes locked version information for a jiri managed project.
type ProjectLock struct {
Remote string `json:"repository_url"`
@@ -527,8 +588,12 @@
jirix.TimerPush("create snapshot")
defer jirix.TimerPop()
- // Create a new Manifest with a Jiri version pinned to each snapshot
- manifest := Manifest{Version: ManifestVersion}
+ // Create a new Manifest with a Jiri version and current attributes
+ // pinned to each snapshot
+ manifest := Manifest{
+ Version: ManifestVersion,
+ Attributes: jirix.FetchingAttrs,
+ }
// Add all local projects to manifest.
localProjects, err := LocalProjects(jirix, FullScan)
@@ -810,7 +875,7 @@
addProject := func(projects Projects) error {
for _, project := range projects {
if existingProject, ok := allProjects[project.Key()]; ok {
- if existingProject != project {
+ if !reflect.DeepEqual(existingProject, project) {
return fmt.Errorf("project: %v conflicts with project: %v", existingProject, project)
}
continue
@@ -1678,10 +1743,47 @@
return nil
}
+// FilterOptionalProjectsPackages removes projects and packages in place if the Optional field is true and
+// attributes in attrs does not match the Attributes field. Currently "match" means the intersection of
+// both attributes is not empty.
+func FilterOptionalProjectsPackages(jirix *jiri.X, attrs string, projects Projects, pkgs Packages) error {
+ allowedAttrs := newAttributes(attrs)
+
+ for k, v := range projects {
+ if !v.ComputedAttributes.IsEmpty() {
+ if v.ComputedAttributes == nil {
+ return fmt.Errorf("project %+v should have valid ComputedAttributes, but it is nil", v)
+ }
+ if !allowedAttrs.Match(v.ComputedAttributes) {
+ jirix.Logger.Debugf("project %q is filtered (%s:%s)", v.Name, v.ComputedAttributes, allowedAttrs)
+ delete(projects, k)
+ }
+ }
+ }
+
+ for k, v := range pkgs {
+ if !v.ComputedAttributes.IsEmpty() {
+ if v.ComputedAttributes == nil {
+ return fmt.Errorf("package %+v should have valid ComputedAttributes, but it is nil", v)
+ }
+ if !allowedAttrs.Match(v.ComputedAttributes) {
+ jirix.Logger.Debugf("package %q is filtered (%s:%s)", v.Name, v.ComputedAttributes, allowedAttrs)
+ delete(pkgs, k)
+ }
+ }
+ }
+ return nil
+}
+
func updateProjects(jirix *jiri.X, localProjects, remoteProjects Projects, hooks Hooks, pkgs Packages, gc bool, runHookTimeout, fetchTimeout uint, rebaseTracked, rebaseUntracked, rebaseAll, snapshot, shouldRunHooks, shouldFetchPkgs bool) error {
jirix.TimerPush("update projects")
defer jirix.TimerPop()
+ // filter optional projects
+ if err := FilterOptionalProjectsPackages(jirix, jirix.FetchingAttrs, remoteProjects, pkgs); err != nil {
+ return err
+ }
+
if err := updateCache(jirix, remoteProjects); err != nil {
return err
}
diff --git a/project/project_test.go b/project/project_test.go
index 246b541..156eaaa 100644
--- a/project/project_test.go
+++ b/project/project_test.go
@@ -2396,3 +2396,116 @@
}
}
}
+
+func TestOptionalProjectsAndPackages(t *testing.T) {
+ fake, cleanup := jiritest.NewFakeJiriRoot(t)
+ defer cleanup()
+
+ // Set up projects and packages with explict attributes
+ numProjects := 3
+ numOptionalProjects := 2
+ localProjects := []project.Project{}
+
+ createRemoteProj := func(i int, attributes string) {
+ name := projectName(i)
+ path := fmt.Sprintf("path-%d", i)
+ if err := fake.CreateRemoteProject(name); err != nil {
+ t.Errorf("failed to create remote project due to error: %v", err)
+ }
+ p := project.Project{
+ Name: name,
+ Path: filepath.Join(fake.X.Root, path),
+ Remote: fake.Projects[name],
+ Attributes: attributes,
+ }
+ localProjects = append(localProjects, p)
+ if err := fake.AddProject(p); err != nil {
+ t.Errorf("failed to add a project to manifest due to error: %v", err)
+ }
+ }
+
+ for i := 0; i < numProjects; i++ {
+ createRemoteProj(i, "")
+ }
+
+ for i := numProjects; i < numProjects+numOptionalProjects; i++ {
+ createRemoteProj(i, "optional,debug")
+ }
+
+ // Create initial commit in each repo.
+ for _, remoteProjectDir := range fake.Projects {
+ writeReadme(t, fake.X, remoteProjectDir, "initial readme")
+ }
+ pkg0 := project.Package{
+ Name: "gn/gn/${platform}",
+ Path: "path-pkg0",
+ Version: "git_revision:bdb0fd02324b120cacde634a9235405061c8ea06",
+ Attributes: "debug,testing",
+ }
+ pkg1 := project.Package{
+ Name: "fuchsia/tools/jiri/${platform}",
+ Path: "path-pkg1",
+ Version: "git_revision:05715c8fbbdb952ab38e50533a1b653445e74b40",
+ Attributes: "",
+ }
+
+ fake.AddPackage(pkg0)
+ fake.AddPackage(pkg1)
+
+ pathExists := func(projPath string) bool {
+ if _, err := os.Stat(projPath); err != nil {
+ if os.IsNotExist(err) {
+ return false
+ }
+ t.Errorf("failed to access path due to error: %v", err)
+ }
+ return true
+ }
+ assertExist := func(localPath string) {
+ if !pathExists(localPath) {
+ t.Errorf("expecting path %q exists, but it does not", localPath)
+ }
+ }
+ assertNotExist := func(localPath string) {
+ if pathExists(localPath) {
+ t.Errorf("expecting path %q does not exist, but it does", localPath)
+ }
+ }
+
+ // Try default mode
+ fake.X.FetchingAttrs = ""
+ fake.UpdateUniverse(true)
+ // The optional projects should not be fetched
+ for i := 0; i < numProjects; i++ {
+ assertExist(localProjects[i].Path)
+ }
+ for i := numProjects; i < numOptionalProjects+numProjects; i++ {
+ assertNotExist(localProjects[i].Path)
+ }
+ assertNotExist(filepath.Join(fake.X.Root, pkg0.Path))
+ assertExist(filepath.Join(fake.X.Root, pkg1.Path))
+
+ // Try setting attributes to "optional, testing"
+ fake.X.FetchingAttrs = "optional, testing"
+ fake.UpdateUniverse(true)
+ for i := 0; i < numProjects; i++ {
+ assertExist(localProjects[i].Path)
+ }
+ for i := numProjects; i < numOptionalProjects+numProjects; i++ {
+ assertExist(localProjects[i].Path)
+ }
+ assertExist(filepath.Join(fake.X.Root, pkg0.Path))
+ assertExist(filepath.Join(fake.X.Root, pkg1.Path))
+
+ // Reset optional attributes
+ fake.X.FetchingAttrs = "nonexist"
+ fake.UpdateUniverse(true)
+ for i := 0; i < numProjects; i++ {
+ assertExist(localProjects[i].Path)
+ }
+ for i := numProjects; i < numOptionalProjects+numProjects; i++ {
+ assertNotExist(localProjects[i].Path)
+ }
+ assertNotExist(filepath.Join(fake.X.Root, pkg0.Path))
+ assertExist(filepath.Join(fake.X.Root, pkg1.Path))
+}
diff --git a/x.go b/x.go
index 372710a..34b3b0a 100644
--- a/x.go
+++ b/x.go
@@ -53,6 +53,7 @@
LockfileEnabled bool `xml:"lockfile>enabled,omitempty"`
LockfileName string `xml:"lockfile>name,omitempty"`
PrebuiltJSON string `xml:"prebuilt>JSON,omitempty"`
+ FetchingAttrs string `xml:"fetchingAttrs,omitempty"`
AnalyticsOptIn string `xml:"analytics>optin,omitempty"`
AnalyticsUserId string `xml:"analytics>userId,omitempty"`
// version user has opted-in to
@@ -108,6 +109,7 @@
LockfileName string
SsoCookiePath string
PrebuiltJSON string
+ FetchingAttrs string
UsingSnapshot bool
IgnoreLockConflicts bool
Color color.Color
@@ -247,6 +249,7 @@
x.LockfileEnabled = x.config.LockfileEnabled
x.LockfileName = x.config.LockfileName
x.PrebuiltJSON = x.config.PrebuiltJSON
+ x.FetchingAttrs = x.config.FetchingAttrs
if x.LockfileName == "" {
x.LockfileName = "jiri.lock"
}
