[partial] Update partial cache logic for git 2.24.
This makes use of multiple promisor remotes, and improves cache checks
for the presence of a revision.
It also relaxes some cache checks, we were previously ignoring tag
presense in the cache, this restricts references to ignore to only HEAD.
Bug: 34965
Change-Id: I949b53b81c6b83bcba00c85d3e55243df66ebbc4
diff --git a/gitutil/git.go b/gitutil/git.go
index 07729a7..a65612e 100644
--- a/gitutil/git.go
+++ b/gitutil/git.go
@@ -6,7 +6,6 @@
import (
"bytes"
- "encoding/hex"
"fmt"
"io"
"os"
@@ -137,6 +136,28 @@
return g.run("remote", "add", name, path)
}
+// AddOrReplacePartialRemote adds a new partial remote with given name and path.
+// If the name already exists, it replaces the named remote with new path.
+func (g *Git) AddOrReplacePartialRemote(name, path string) error {
+ configStr := fmt.Sprintf("remote.%s.url", name)
+ if err := g.Config(configStr, path); err != nil {
+ return err
+ }
+ configStr = fmt.Sprintf("remote.%s.partialCloneFilter", name)
+ if err := g.Config(configStr, "blob:none"); err != nil {
+ return err
+ }
+ configStr = fmt.Sprintf("remote.%s.promisor", name)
+ if err := g.Config(configStr, "true"); err != nil {
+ return err
+ }
+ configStr = fmt.Sprintf("remote.%s.fetch", name)
+ if err := g.Config(configStr, "+refs/heads/*:refs/remotes/origin/*"); err != nil {
+ return err
+ }
+ return nil
+}
+
// AddOrReplaceRemote adds a new remote with given name and path. If the name
// already exists, it replaces the named remote with new path.
func (g *Git) AddOrReplaceRemote(name, path string) error {
@@ -217,8 +238,8 @@
return branches, nil
}
-// IsRevAvailable runs cat-file on a commit hash is available locally.
-func (g *Git) IsRevAvailable(rev string) bool {
+// IsRevAvailable checks if a commit hash is available locally.
+func (g *Git) IsRevAvailable(jirix *jiri.X, rev string) bool {
// TODO: (haowei@)(11517) We are having issues with corrupted
// cache data on mac builders. Return a non-nil error
// to force the mac builders fetch from remote to avoid
@@ -226,11 +247,26 @@
if runtime.GOOS == "darwin" {
return false
}
- // test if rev is a legit sha1 hash string
- if _, err := hex.DecodeString(rev); len(rev) != 40 || err != nil {
+ // If it wants HEAD, always fetch.
+ if rev == "HEAD" {
return false
}
-
+ // Ensure the revision is present.
+ if jirix.Partial {
+ currentRevision, err := g.CurrentRevision()
+ if err != nil {
+ jirix.Logger.Errorf("could not get current revision\n")
+ return false
+ }
+ expectedRevision, err := g.CurrentRevisionForRef(rev)
+ if err != nil {
+ jirix.Logger.Errorf("could not get revision\n")
+ return false
+ }
+ if currentRevision != expectedRevision {
+ return false
+ }
+ }
if err := g.run("cat-file", "-e", rev); err != nil {
return false
}
@@ -656,6 +692,7 @@
updateShallow := false
depth := 0
fetchTag := ""
+ updateHeadOk := false
for _, opt := range opts {
switch typedOpt := opt.(type) {
case TagsOpt:
@@ -670,6 +707,8 @@
updateShallow = bool(typedOpt)
case FetchTagOpt:
fetchTag = string(typedOpt)
+ case UpdateHeadOkOpt:
+ updateHeadOk = bool(typedOpt)
}
}
args := []string{}
@@ -689,6 +728,9 @@
if all {
args = append(args, "--all")
}
+ if updateHeadOk {
+ args = append(args, "--update-head-ok")
+ }
if remote != "" {
args = append(args, remote)
}
diff --git a/gitutil/options.go b/gitutil/options.go
index 902a494..9462573 100644
--- a/gitutil/options.go
+++ b/gitutil/options.go
@@ -126,3 +126,7 @@
type RebaseMerges bool
func (RebaseMerges) rebaseOpt() {}
+
+type UpdateHeadOkOpt bool
+
+func (UpdateHeadOkOpt) fetchOpt() {}
diff --git a/project/loader.go b/project/loader.go
index b40a954..9938954 100644
--- a/project/loader.go
+++ b/project/loader.go
@@ -324,7 +324,6 @@
return fmtError(err)
}
remoteUrl := rewriteRemote(jirix, p.Remote)
- r := remoteUrl
task := jirix.Logger.AddTaskMsg("Creating manifest: %s", remote.Name)
defer task.Done()
if cacheDirPath != "" {
@@ -335,21 +334,25 @@
if err := updateOrCreateCache(jirix, cacheDirPath, remoteUrl, remote.RemoteBranch, remote.Revision, 0); err != nil {
return err
}
- r = cacheDirPath
}
opts := []gitutil.CloneOpt{gitutil.ReferenceOpt(cacheDirPath), gitutil.NoCheckoutOpt(true)}
if jirix.Partial {
opts = append(opts, gitutil.OmitBlobsOpt(true))
}
- if err := clone(jirix, r, path, opts...); err != nil {
+ if err := clone(jirix, remoteUrl, path, opts...); err != nil {
return err
}
- scm := gitutil.New(jirix, gitutil.RootDirOpt(path))
- defer func() {
- if err := scm.AddOrReplaceRemote("origin", remoteUrl); err != nil {
- jirix.Logger.Errorf("failed to set remote back to %v for project %+v", remoteUrl, p)
+ if jirix.Partial && cacheDirPath != "" {
+ // Set Cache Remote
+ scm := gitutil.New(jirix, gitutil.RootDirOpt(p.Path))
+ if err := scm.Config("extensions.partialClone", "origin"); err != nil {
+ return err
}
- }()
+ if err := scm.AddOrReplacePartialRemote("cache", cacheDirPath); err != nil {
+ return err
+ }
+ }
+
p.Revision = remote.Revision
p.RemoteBranch = remote.RemoteBranch
if err := checkoutHeadRevision(jirix, p, false); err != nil {
diff --git a/project/operations.go b/project/operations.go
index 69284b8..26b71d7 100644
--- a/project/operations.go
+++ b/project/operations.go
@@ -7,6 +7,7 @@
import (
"fmt"
"io"
+ "io/ioutil"
"os"
"path"
"path/filepath"
@@ -88,31 +89,54 @@
func (op createOperation) checkoutProject(jirix *jiri.X, cache string) error {
var err error
remote := rewriteRemote(jirix, op.project.Remote)
+ scm := gitutil.New(jirix, gitutil.RootDirOpt(op.project.Path))
// Hack to make fuchsia.git happen
if op.destination == jirix.Root {
- scm := gitutil.New(jirix, gitutil.RootDirOpt(op.project.Path))
if err = scm.Init(op.destination); err != nil {
return err
}
if err = scm.AddOrReplaceRemote("origin", remote); err != nil {
return err
}
- // We must specify a refspec here in order for patch to be able to set
- // upstream to 'origin/master'.
- if op.project.HistoryDepth > 0 && cache != "" {
- if err = scm.FetchRefspec(cache, "+refs/heads/*:refs/remotes/origin/*", gitutil.DepthOpt(op.project.HistoryDepth)); err != nil {
+ // This appears to be set to 0 via some quirk of `git init`.
+ if err := scm.Config("core.repositoryformatversion", "1"); err != nil {
+ return err
+ }
+ if jirix.Partial {
+ if err := scm.Config("extensions.partialClone", "origin"); err != nil {
return err
}
- } else if cache != "" {
- if err = scm.FetchRefspec(cache, "+refs/heads/*:refs/remotes/origin/*"); err != nil {
- return err
- }
- } else {
- if err = scm.FetchRefspec(remote, "+refs/heads/*:refs/remotes/origin/*"); err != nil {
+ if err := scm.AddOrReplacePartialRemote("origin", remote); err != nil {
return err
}
}
+ // We must specify a refspec here in order for patch to be able to set
+ // upstream to 'origin/master'.
+ if err := scm.Config("remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"); err != nil {
+ return err
+ }
+ if cache != "" {
+ objPath := "objects"
+ if jirix.Partial {
+ objPath = ".git/objects"
+ }
+ if err := ioutil.WriteFile(filepath.Join(op.destination, ".git/objects/info/alternates"), []byte(filepath.Join(cache, objPath) + "\n"), 0644); err != nil {
+ return err
+ }
+ }
+ if err = fetchAll(jirix, op.project); err != nil {
+ return err
+ }
} else {
+ r := remote
+ if cache != "" {
+ r = cache
+ defer func() {
+ if err := scm.AddOrReplaceRemote("origin", remote); err != nil {
+ jirix.Logger.Errorf("failed to set remote back to %v for project %+v", remote, op.project)
+ }
+ }()
+ }
opts := []gitutil.CloneOpt{gitutil.NoCheckoutOpt(true)}
if op.project.HistoryDepth > 0 {
opts = append(opts, gitutil.DepthOpt(op.project.HistoryDepth))
@@ -123,18 +147,18 @@
if jirix.Partial {
opts = append(opts, gitutil.OmitBlobsOpt(true))
}
- if cache != "" {
- if err = clone(jirix, cache, op.destination, opts...); err != nil {
- return err
- }
- scm := gitutil.New(jirix, gitutil.RootDirOpt(op.project.Path))
- if err = scm.AddOrReplaceRemote("origin", remote); err != nil {
- return err
- }
- } else {
- if err = clone(jirix, remote, op.destination, opts...); err != nil {
- return err
- }
+ if err = clone(jirix, r, op.destination, opts...); err != nil {
+ return err
+ }
+ }
+
+ if jirix.Partial && cache != "" {
+ // Set Cache Remote
+ if err := scm.Config("extensions.partialClone", "origin"); err != nil {
+ return err
+ }
+ if err := scm.AddOrReplacePartialRemote("cache", cache); err != nil {
+ return err
}
}
@@ -149,13 +173,6 @@
if err := writeMetadata(jirix, op.project, op.project.Path); err != nil {
return err
}
- scm := gitutil.New(jirix, gitutil.RootDirOpt(op.project.Path))
-
- // Reset remote to point to correct location so that shared cache does not cause problem.
- if err := scm.SetRemoteUrl("origin", remote); err != nil {
- return err
- }
-
// Delete inital branch(es)
if branches, _, err := scm.GetBranches(); err != nil {
jirix.Logger.Warningf("not able to get branches for newly created project %s(%s)\n\n", op.project.Name, op.project.Path)
diff --git a/project/project.go b/project/project.go
index 45076bc..145e9fc 100644
--- a/project/project.go
+++ b/project/project.go
@@ -1967,27 +1967,34 @@
jirix.Logger.Warningf("set remote.origin.fetch failed under git cache directory %q due to error: %v", dir, err)
return errCacheCorruption
}
- // Cache already present, update it
- // TODO : update this after implementing FetchAll using g
- if scm.IsRevAvailable(revision) {
- jirix.Logger.Infof("%s(%s) cache up-to-date; skipping\n", remote, dir)
- return nil
+ if jirix.Partial {
+ if err := scm.AddOrReplacePartialRemote("origin", remote); err != nil {
+ return err
+ }
}
msg := fmt.Sprintf("Updating cache: %q", dir)
task := jirix.Logger.AddTaskMsg(msg)
defer task.Done()
t := jirix.Logger.TrackTime(msg)
defer t.Done()
+ // Cache already present, update it
+ // TODO : update this after implementing FetchAll using g
+ if scm.IsRevAvailable(jirix, revision) {
+ jirix.Logger.Infof("%s(%s) cache up-to-date; skipping\n", remote, dir)
+ return nil
+ }
// We need to explicitly specify the ref for fetch to update in case
// the cache was created with a previous version and uses "refs/*"
if err := retry.Function(jirix, func() error {
- git := gitutil.New(jirix, gitutil.RootDirOpt(dir))
- if err := git.FetchRefspec("origin", refspec,
- gitutil.DepthOpt(depth), gitutil.PruneOpt(true), gitutil.UpdateShallowOpt(true)); err != nil {
+ // Use --update-head-ok here to force fetch to update the current branch.
+ // This is used in the case of a partial clone having a working tree
+ // checked out in the cache.
+ if err := scm.FetchRefspec("origin", refspec,
+ gitutil.DepthOpt(depth), gitutil.PruneOpt(true), gitutil.UpdateShallowOpt(true), gitutil.UpdateHeadOkOpt(true)); err != nil {
return err
}
if jirix.Partial {
- if err := git.CheckoutBranch(revision, gitutil.DetachOpt(true), gitutil.ForceOpt(true)); err != nil {
+ if err := scm.CheckoutBranch(revision, gitutil.DetachOpt(true), gitutil.ForceOpt(true)); err != nil {
return err
}
}
@@ -2036,17 +2043,19 @@
defer t.Done()
// Try use clone.bundle to speed up the initialization of git cache.
os.MkdirAll(dir, 0755)
- if err := createCacheThroughBundle(); err != nil {
- jirix.Logger.Debugf("create git cache for %q through clone.bundle failed due to error: %v", remote, err)
- os.RemoveAll(dir)
- } else {
- jirix.Logger.Debugf("git cache for %q created through clone.bundle", remote)
- return nil
+ if !jirix.Partial {
+ if err := createCacheThroughBundle(); err != nil {
+ jirix.Logger.Debugf("create git cache for %q through clone.bundle failed due to error: %v", remote, err)
+ os.RemoveAll(dir)
+ } else {
+ jirix.Logger.Debugf("git cache for %q created through clone.bundle", remote)
+ return nil
+ }
}
opts := []gitutil.CloneOpt{gitutil.DepthOpt(depth)}
if jirix.Partial {
- opts = append(opts, gitutil.OmitBlobsOpt(true))
+ opts = append(opts, gitutil.NoCheckoutOpt(true), gitutil.OmitBlobsOpt(true))
} else {
opts = append(opts, gitutil.BareOpt(true))
}
@@ -2065,6 +2074,9 @@
if err := git.Config("remote.origin.fetch", refspec); err != nil {
return err
}
+ if err := git.Config("uploadpack.allowFilter", "true"); err != nil {
+ return err
+ }
return nil
}
