[project] Add repeatable -package-to-skip flag
The -package-to-skip flag is a repeatable flag of package names which,
if fed into the update, run-hooks, or fetch-packages subcommands, will
direct Jiri to skip fetching the specified packages.
Test: Ran `jiri update` with the new flag against
infra/roller-tests/integration.git, confirmed that the specified
package was skipped.
Bug: 100001
Change-Id: Ie568a71218c2c20599cb93768ac06f1f844e6758
Reviewed-on: https://fuchsia-review.googlesource.com/c/jiri/+/678659
Reviewed-by: Nathan Mulcahey <nmulcahey@google.com>
Commit-Queue: Anthony Fandrianto <atyfto@google.com>
diff --git a/cmd/jiri/fetch_pkgs.go b/cmd/jiri/fetch_pkgs.go
index e9e0af9..842ebcc 100644
--- a/cmd/jiri/fetch_pkgs.go
+++ b/cmd/jiri/fetch_pkgs.go
@@ -15,6 +15,7 @@
fetchPkgsTimeout uint
attempts uint
skipLocalProjects bool
+ packagesToSkip arrayFlag
}
var cmdFetchPkgs = &cmdline.Command{
@@ -32,6 +33,7 @@
cmdFetchPkgs.Flags.UintVar(&fetchPkgsFlags.fetchPkgsTimeout, "fetch-packages-timeout", project.DefaultPackageTimeout, "Timeout in minutes for fetching prebuilt packages using cipd.")
cmdFetchPkgs.Flags.UintVar(&fetchPkgsFlags.attempts, "attempts", 1, "Number of attempts before failing.")
cmdFetchPkgs.Flags.BoolVar(&fetchPkgsFlags.skipLocalProjects, "skip-local-projects", false, "Skip checking local project state.")
+ cmdFetchPkgs.Flags.Var(&runHooksFlags.packagesToSkip, "package-to-skip", "Skip fetching this package. Repeatable.")
}
func runFetchPkgs(jirix *jiri.X, args []string) (err error) {
@@ -60,6 +62,7 @@
if err := project.FilterOptionalProjectsPackages(jirix, jirix.FetchingAttrs, nil, pkgs); err != nil {
return err
}
+ project.FilterPackagesByName(jirix, pkgs, fetchPkgsFlags.packagesToSkip)
if len(pkgs) > 0 {
return project.FetchPackages(jirix, pkgs, fetchPkgsFlags.fetchPkgsTimeout)
}
diff --git a/cmd/jiri/run_hooks.go b/cmd/jiri/run_hooks.go
index f699d6c..c7cad9d 100644
--- a/cmd/jiri/run_hooks.go
+++ b/cmd/jiri/run_hooks.go
@@ -11,10 +11,11 @@
)
var runHooksFlags struct {
- localManifest bool
- hookTimeout uint
- attempts uint
- fetchPackages bool
+ localManifest bool
+ hookTimeout uint
+ attempts uint
+ fetchPackages bool
+ packagesToSkip arrayFlag
}
var cmdRunHooks = &cmdline.Command{
@@ -32,6 +33,7 @@
cmdRunHooks.Flags.UintVar(&runHooksFlags.hookTimeout, "hook-timeout", project.DefaultHookTimeout, "Timeout in minutes for running the hooks operation.")
cmdRunHooks.Flags.UintVar(&runHooksFlags.attempts, "attempts", 1, "Number of attempts before failing.")
cmdRunHooks.Flags.BoolVar(&runHooksFlags.fetchPackages, "fetch-packages", true, "Use fetching packages using jiri.")
+ cmdRunHooks.Flags.Var(&runHooksFlags.packagesToSkip, "package-to-skip", "Skip fetching this package. Repeatable.")
}
func runHooks(jirix *jiri.X, args []string) (err error) {
@@ -61,6 +63,7 @@
if err := project.FilterOptionalProjectsPackages(jirix, jirix.FetchingAttrs, nil, pkgs); err != nil {
return err
}
+ project.FilterPackagesByName(jirix, pkgs, runHooksFlags.packagesToSkip)
// Get packages if the fetchPackages is true
if runHooksFlags.fetchPackages && len(pkgs) > 0 {
// Extend timeout for packages to be 5 times the timeout of a single hook.
diff --git a/cmd/jiri/snapshot_test.go b/cmd/jiri/snapshot_test.go
index dc7a6f0..e030350 100644
--- a/cmd/jiri/snapshot_test.go
+++ b/cmd/jiri/snapshot_test.go
@@ -84,7 +84,7 @@
for i := 0; i < numProjects; i++ {
writeReadme(t, fake.X, fake.Projects[remoteProjectName(i)], "revision 1")
}
- if err := project.UpdateUniverse(fake.X, true, false, false, false, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout); err != nil {
+ if err := project.UpdateUniverse(fake.X, true, false, false, false, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil); err != nil {
t.Fatalf("%v", err)
}
@@ -111,7 +111,7 @@
}
snapshotFile := tmpfile.Name()
- if err := project.CheckoutSnapshot(fake.X, snapshotFile, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout); err != nil {
+ if err := project.CheckoutSnapshot(fake.X, snapshotFile, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil); err != nil {
t.Fatalf("%s", err)
}
for i := range remoteProjects {
diff --git a/cmd/jiri/source_manifest_test.go b/cmd/jiri/source_manifest_test.go
index 0159a67..2d1c0ae 100644
--- a/cmd/jiri/source_manifest_test.go
+++ b/cmd/jiri/source_manifest_test.go
@@ -53,7 +53,7 @@
for i := 0; i < numProjects; i++ {
writeReadme(t, fake.X, fake.Projects[remoteProjectName(i)], fmt.Sprintf("proj %d", i))
}
- if err := project.UpdateUniverse(fake.X, true, false, false, false, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout); err != nil {
+ if err := project.UpdateUniverse(fake.X, true, false, false, false, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil); err != nil {
t.Fatalf("%s", err)
}
diff --git a/cmd/jiri/update.go b/cmd/jiri/update.go
index b992196..9b64670 100644
--- a/cmd/jiri/update.go
+++ b/cmd/jiri/update.go
@@ -30,6 +30,7 @@
runHooksFlag bool
fetchPkgsFlag bool
overrideOptionalFlag bool
+ packagesToSkipFlag arrayFlag
)
const (
@@ -52,6 +53,7 @@
cmdUpdate.Flags.BoolVar(&runHooksFlag, "run-hooks", true, "Run hooks after updating sources.")
cmdUpdate.Flags.BoolVar(&fetchPkgsFlag, "fetch-packages", true, "Use cipd to fetch packages.")
cmdUpdate.Flags.BoolVar(&overrideOptionalFlag, "override-optional", false, "Override existing optional attributes in the snapshot file with current jiri settings")
+ cmdUpdate.Flags.Var(&packagesToSkipFlag, "package-to-skip", "Skip fetching this package. Repeatable.")
}
// cmdUpdate represents the "jiri update" command.
@@ -95,7 +97,7 @@
if len(args) > 0 {
jirix.OverrideOptional = overrideOptionalFlag
- if err := project.CheckoutSnapshot(jirix, args[0], gcFlag, runHooksFlag, fetchPkgsFlag, hookTimeoutFlag, fetchPkgsTimeoutFlag); err != nil {
+ if err := project.CheckoutSnapshot(jirix, args[0], gcFlag, runHooksFlag, fetchPkgsFlag, hookTimeoutFlag, fetchPkgsTimeoutFlag, packagesToSkipFlag); err != nil {
return err
}
} else {
@@ -109,7 +111,7 @@
}
err := project.UpdateUniverse(jirix, gcFlag, localManifestFlag,
- rebaseTrackedFlag, rebaseUntrackedFlag, rebaseAllFlag, runHooksFlag, fetchPkgsFlag, hookTimeoutFlag, fetchPkgsTimeoutFlag)
+ rebaseTrackedFlag, rebaseUntrackedFlag, rebaseAllFlag, runHooksFlag, fetchPkgsFlag, hookTimeoutFlag, fetchPkgsTimeoutFlag, packagesToSkipFlag)
if err2 := project.WriteUpdateHistorySnapshot(jirix, nil, nil, localManifestFlag); err2 != nil {
if err != nil {
return fmt.Errorf("while updating: %s, while writing history: %s", err, err2)
diff --git a/jiritest/fake.go b/jiritest/fake.go
index 8b43926..16af0ad 100644
--- a/jiritest/fake.go
+++ b/jiritest/fake.go
@@ -203,7 +203,7 @@
// UpdateUniverse synchronizes the content of the Vanadium fake based
// on the content of the remote manifest.
func (fake FakeJiriRoot) UpdateUniverse(gc bool) error {
- if err := project.UpdateUniverse(fake.X, gc, false, false, false, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout); err != nil {
+ if err := project.UpdateUniverse(fake.X, gc, false, false, false, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil); err != nil {
return err
}
return nil
diff --git a/project/project.go b/project/project.go
index 4ae1539..8c72f30 100644
--- a/project/project.go
+++ b/project/project.go
@@ -823,7 +823,7 @@
// CheckoutSnapshot updates project state to the state specified in the given
// snapshot file. Note that the snapshot file must not contain remote imports.
-func CheckoutSnapshot(jirix *jiri.X, snapshot string, gc, runHooks, fetchPkgs bool, runHookTimeout, fetchTimeout uint) error {
+func CheckoutSnapshot(jirix *jiri.X, snapshot string, gc, runHooks, fetchPkgs bool, runHookTimeout, fetchTimeout uint, pkgsToSkip []string) error {
jirix.UsingSnapshot = true
// Find all local projects.
scanMode := FastScan
@@ -838,7 +838,7 @@
if err != nil {
return err
}
- if err := updateProjects(jirix, localProjects, remoteProjects, hooks, pkgs, gc, runHookTimeout, fetchTimeout, false /*rebaseTracked*/, false /*rebaseUntracked*/, false /*rebaseAll*/, true /*snapshot*/, runHooks, fetchPkgs); err != nil {
+ if err := updateProjects(jirix, localProjects, remoteProjects, hooks, pkgs, gc, runHookTimeout, fetchTimeout, false /*rebaseTracked*/, false /*rebaseUntracked*/, false /*rebaseAll*/, true /*snapshot*/, runHooks, fetchPkgs, pkgsToSkip); err != nil {
return err
}
return WriteUpdateHistorySnapshot(jirix, hooks, pkgs, false)
@@ -1412,7 +1412,7 @@
// counterparts identified in the manifest. Optionally, the 'gc' flag can be
// used to indicate that local projects that no longer exist remotely should be
// removed.
-func UpdateUniverse(jirix *jiri.X, gc, localManifest, rebaseTracked, rebaseUntracked, rebaseAll, runHooks, fetchPkgs bool, runHookTimeout, fetchTimeout uint) (e error) {
+func UpdateUniverse(jirix *jiri.X, gc, localManifest, rebaseTracked, rebaseUntracked, rebaseAll, runHooks, fetchPkgs bool, runHookTimeout, fetchTimeout uint, pkgsToSkip []string) (e error) {
jirix.Logger.Infof("Updating all projects")
updateFn := func(scanMode ScanMode) error {
jirix.TimerPush(fmt.Sprintf("update universe: %s", scanMode))
@@ -1432,7 +1432,7 @@
}
// Actually update the projects.
- return updateProjects(jirix, localProjects, remoteProjects, hooks, pkgs, gc, runHookTimeout, fetchTimeout, rebaseTracked, rebaseUntracked, rebaseAll, false /*snapshot*/, runHooks, fetchPkgs)
+ return updateProjects(jirix, localProjects, remoteProjects, hooks, pkgs, gc, runHookTimeout, fetchTimeout, rebaseTracked, rebaseUntracked, rebaseAll, false /*snapshot*/, runHooks, fetchPkgs, pkgsToSkip)
}
// Specifying gc should always force a full filesystem scan.
@@ -2332,6 +2332,24 @@
return nil
}
+// FilterPackagesByName removes packages in place given a list of CIPD package names.
+func FilterPackagesByName(jirix *jiri.X, pkgs Packages, pkgsToSkip []string) {
+ if len(pkgsToSkip) == 0 {
+ return
+ }
+ jirix.TimerPush("filter packages by name")
+ defer jirix.TimerPop()
+ pkgsSet := make(map[string]bool)
+ for _, p := range pkgsToSkip {
+ pkgsSet[p] = true
+ }
+ for k, v := range pkgs {
+ if _, ok := pkgsSet[v.Name]; ok {
+ delete(pkgs, k)
+ }
+ }
+}
+
// FilterOptionalProjectsPackages removes projects and packages in place if the Optional field is true and
// attributes in attrs does not match the Attributes field. Currently "match" means the intersection of
// both attributes is not empty.
@@ -2364,7 +2382,7 @@
return nil
}
-func updateProjects(jirix *jiri.X, localProjects, remoteProjects Projects, hooks Hooks, pkgs Packages, gc bool, runHookTimeout, fetchTimeout uint, rebaseTracked, rebaseUntracked, rebaseAll, snapshot, shouldRunHooks, shouldFetchPkgs bool) error {
+func updateProjects(jirix *jiri.X, localProjects, remoteProjects Projects, hooks Hooks, pkgs Packages, gc bool, runHookTimeout, fetchTimeout uint, rebaseTracked, rebaseUntracked, rebaseAll, snapshot, shouldRunHooks, shouldFetchPkgs bool, pkgsToSkip []string) error {
jirix.TimerPush("update projects")
defer jirix.TimerPop()
@@ -2383,6 +2401,7 @@
if err := FilterOptionalProjectsPackages(jirix, jirix.FetchingAttrs, remoteProjects, pkgs); err != nil {
return err
}
+ FilterPackagesByName(jirix, pkgs, pkgsToSkip)
if err := updateCache(jirix, remoteProjects); err != nil {
return err
diff --git a/project/project_test.go b/project/project_test.go
index 16d1e04..6ed5278 100644
--- a/project/project_test.go
+++ b/project/project_test.go
@@ -360,7 +360,7 @@
writeFile(t, fake.X, fake.Projects[localProjects[1].Name], "file1", "file1")
gitRemote := gitutil.New(fake.X, gitutil.RootDirOpt(fake.Projects[localProjects[1].Name]))
remoteRev, _ := gitRemote.CurrentRevision()
- if err := project.UpdateUniverse(fake.X, false, false, false, false, true /*rebase-all*/, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout); err != nil {
+ if err := project.UpdateUniverse(fake.X, false, false, false, false, true /*rebase-all*/, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil); err != nil {
t.Fatal(err)
}
projects, err := project.LocalProjects(fake.X, project.FastScan)
@@ -400,7 +400,7 @@
writeFile(t, fake.X, fake.Projects[localProjects[1].Name], "file1", "file1")
remoteRev, _ := gitRemote.CurrentRevision()
- if err := project.UpdateUniverse(fake.X, false, false, false, false, true /*rebase-all*/, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout); err != nil {
+ if err := project.UpdateUniverse(fake.X, false, false, false, false, true /*rebase-all*/, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil); err != nil {
t.Fatal(err)
}
projects, err := project.LocalProjects(fake.X, project.FastScan)
@@ -770,7 +770,7 @@
if err := manifest.ToFile(fake.X, filepath.Join(fake.X.Root, jiritest.ManifestProjectPath, jiritest.ManifestFileName)); err != nil {
t.Fatal(err)
}
- if err := project.UpdateUniverse(fake.X, false, true /* localManifest */, false, false, false, false, false, project.DefaultHookTimeout, project.DefaultPackageTimeout); err != nil {
+ if err := project.UpdateUniverse(fake.X, false, true /* localManifest */, false, false, false, false, false, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil); err != nil {
t.Fatal(err)
}
@@ -852,7 +852,7 @@
// Add new commit to last project
writeFile(t, fake.X, fake.Projects[lastProject.Name], "file1", "file1")
- if err := project.UpdateUniverse(fake.X, false, true /* localManifest */, false, false, false, false, false, project.DefaultHookTimeout, project.DefaultPackageTimeout); err != nil {
+ if err := project.UpdateUniverse(fake.X, false, true /* localManifest */, false, false, false, false, false, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil); err != nil {
t.Fatal(err)
}
// check last project revision
@@ -1001,7 +1001,7 @@
if err != nil {
t.Fatal(err)
}
- if err := project.UpdateUniverse(fake.X, false, false, true /*rebaseTracked*/, false, false, false /*run-hooks*/, false /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout); err != nil {
+ if err := project.UpdateUniverse(fake.X, false, false, true /*rebaseTracked*/, false, false, false /*run-hooks*/, false /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil); err != nil {
t.Fatal(err)
}
}
@@ -1659,7 +1659,7 @@
t.Fatal(err)
}
- if err := project.UpdateUniverse(fake.X, false, false, true /*rebaseTracked*/, false, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout); err != nil {
+ if err := project.UpdateUniverse(fake.X, false, false, true /*rebaseTracked*/, false, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil); err != nil {
t.Fatal(err)
}
@@ -1851,9 +1851,9 @@
}))
defer server.Close()
- project.CheckoutSnapshot(fake.X, server.URL, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout)
+ project.CheckoutSnapshot(fake.X, server.URL, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil)
} else {
- project.CheckoutSnapshot(fake.X, snapshotFile, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout)
+ project.CheckoutSnapshot(fake.X, snapshotFile, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil)
}
sort.Sort(project.ProjectsByPath(localProjects))
for i, localProject := range localProjects {
@@ -1904,7 +1904,7 @@
}
}
- if err := project.UpdateUniverse(fake.X, false, false, false, false, rebaseAll, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout); err != nil {
+ if err := project.UpdateUniverse(fake.X, false, false, false, false, rebaseAll, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil); err != nil {
t.Fatal(err)
}
@@ -2007,7 +2007,7 @@
}
// The update should complain about the cycle.
- err := project.UpdateUniverse(jirix, false, false, false, false, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout)
+ err := project.UpdateUniverse(jirix, false, false, false, false, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil)
if got, want := fmt.Sprint(err), "import cycle detected in local manifest files"; !strings.Contains(got, want) {
t.Errorf("got error %v, want substr %v", got, want)
}
@@ -2058,7 +2058,7 @@
commitFile(t, fake.X, remote2, fileB, "commit B")
// The update should complain about the cycle.
- err := project.UpdateUniverse(fake.X, false, false, false, false, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout)
+ err := project.UpdateUniverse(fake.X, false, false, false, false, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil)
if got, want := fmt.Sprint(err), "import cycle detected in remote manifest imports"; !strings.Contains(got, want) {
t.Errorf("got error %v, want substr %v", got, want)
}
@@ -2128,7 +2128,7 @@
commitFile(t, fake.X, remote1, fileD, "commit D")
// The update should complain about the cycle.
- err := project.UpdateUniverse(fake.X, false, false, false, false, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout)
+ err := project.UpdateUniverse(fake.X, false, false, false, false, false, true /*run-hooks*/, true /*run-packages*/, project.DefaultHookTimeout, project.DefaultPackageTimeout, nil)
if got, want := fmt.Sprint(err), "import cycle detected"; !strings.Contains(got, want) {
t.Errorf("got error %v, want substr %v", got, want)
}
@@ -2846,3 +2846,33 @@
t.Errorf("expecting %v first level nodes, but got %v", 3, len(root.Children))
}
}
+
+func TestFilterPackagesByName(t *testing.T) {
+ jirix, cleanup := xtest.NewX(t)
+ defer cleanup()
+ p := []project.Package{
+ {Name: "test0", Version: "version"},
+ {Name: "test1", Version: "version"},
+ {Name: "test2", Version: "version"},
+ {Name: "test3", Version: "version"},
+ }
+ e := []project.Package{
+ {Name: "test0", Version: "version"},
+ {Name: "test3", Version: "version"},
+ }
+ pkgsToSkip := []string{"test1", "test2"}
+
+ pkgs := make(project.Packages)
+ expected := make(project.Packages)
+ for _, v := range p {
+ pkgs[v.Key()] = v
+ }
+ for _, v := range e {
+ expected[v.Key()] = v
+ }
+
+ project.FilterPackagesByName(jirix, pkgs, pkgsToSkip)
+ if !reflect.DeepEqual(pkgs, expected) {
+ t.Errorf("filter packages got %v, want %v", pkgs, expected)
+ }
+}
