[JSON] Write JSON file for packages
This patch add a new feature to jiri to generate JSON file for packages.
If there is any internal package and jiri fetched these packages
successfully, jiri will write "internal_access = true" in generated JSON
file. If jiri failed to fetch internal packages, "internal_access =
false" will be written. If there is no internal packages, jiri will not
generate the JSON file. This patch also add a feature to jiri to set up
default paths to packages which do not explicitly define one in
manifest.
Test: CQ
Change-Id: I8d3ded3e8afdcd5aa229860869ba5bd8a6a0f63f
diff --git a/cmd/jiri/init.go b/cmd/jiri/init.go
index 13d56c9..1bc5e7e 100644
--- a/cmd/jiri/init.go
+++ b/cmd/jiri/init.go
@@ -44,6 +44,7 @@
keepGitHooks string
enableLockfileFlag string
lockfileNameFlag string
+ prebuiltJSON string
)
func init() {
@@ -56,6 +57,7 @@
cmdInit.Flags.StringVar(&keepGitHooks, "keep-git-hooks", "", "Whether to keep current git hooks in '.git/hooks' when doing 'jiri update'. Takes true/false.")
cmdInit.Flags.StringVar(&enableLockfileFlag, "enable-lockfile", "", "Enable lockfile enforcement")
cmdInit.Flags.StringVar(&lockfileNameFlag, "lockfile-name", "", "Set up filename of lockfile")
+ cmdInit.Flags.StringVar(&prebuiltJSON, "prebuilt-json", "", "Set up filename for prebuilt json file")
}
func runInit(env *cmdline.Env, args []string) error {
@@ -151,6 +153,10 @@
config.LockfileName = lockfileNameFlag
}
+ if prebuiltJSON != "" {
+ config.PrebuiltJSON = prebuiltJSON
+ }
+
if enableLockfileFlag != "" {
if val, err := strconv.ParseBool(enableLockfileFlag); err != nil {
return fmt.Errorf("'enableLockfileFlag' flag should be true or false")
diff --git a/project/manifest.go b/project/manifest.go
index 5c66729..0918ab1 100644
--- a/project/manifest.go
+++ b/project/manifest.go
@@ -7,6 +7,7 @@
import (
"bytes"
"context"
+ "encoding/json"
"encoding/xml"
"errors"
"fmt"
@@ -16,6 +17,7 @@
"net/url"
"os"
"os/exec"
+ "path"
"path/filepath"
"sort"
"strings"
@@ -441,6 +443,35 @@
return PackageKey(p.Path + KeySeparator + p.Name)
}
+// FilterACL returns a new Packages map without any inaccessible packages.
+func (p *Packages) FilterACL(jirix *jiri.X) (Packages, bool, error) {
+ // Perform ACL checks on internal projects
+ pkgACLMap := make(map[string]bool)
+ pkgVersionMap := make(map[string]string)
+ hasInternal := false
+ for _, pkg := range *p {
+ pkg.Name = strings.TrimRight(pkg.Name, "/")
+ if pkg.Internal {
+ hasInternal = true
+ pkgACLMap[pkg.Name] = false
+ pkgVersionMap[pkg.Name] = pkg.Version
+ }
+ }
+ if len(pkgACLMap) != 0 {
+ if err := cipd.CheckPackageACL(jirix, pkgACLMap, pkgVersionMap); err != nil {
+ return nil, false, err
+ }
+ }
+ retPkgs := make(Packages)
+ for _, pkg := range *p {
+ if val, ok := pkgACLMap[pkg.Name]; ok && !val {
+ continue
+ }
+ retPkgs[pkg.Key()] = pkg
+ }
+ return retPkgs, hasInternal, nil
+}
+
type PackageInstance struct {
Name string `xml:"name,attr"`
ID string `xml:"id,attr"`
@@ -461,6 +492,35 @@
return nil
}
+// GetPath returns the relative path that Package p should be
+// downloaded to.
+func (p *Package) GetPath() (string, error) {
+ if p.Path == "" {
+ cipdPath := p.Name
+ // Replace template with current platform information.
+ // If failed, skip filling in default path.
+ if cipd.MustExpand(cipdPath) {
+ expanded, err := cipd.Expand(cipdPath, []cipd.Platform{cipd.CipdPlatform})
+ if err != nil {
+ return "", err
+ }
+ if len(expanded) > 0 {
+ cipdPath = expanded[0]
+ }
+ }
+ if !cipd.MustExpand(cipdPath) {
+ base := path.Base(cipdPath)
+ if _, err := cipd.NewPlatform(base); err == nil {
+ // base is the name for a platform
+ base = filepath.Join(path.Base(path.Dir(cipdPath)), base)
+ }
+ return filepath.Join("prebuilt", base), nil
+ }
+ return "prebuilt", nil
+ }
+ return p.Path, nil
+}
+
// GetPlatforms returns the platforms information of
// this Package struct.
func (p *Package) GetPlatforms() ([]cipd.Platform, error) {
@@ -605,6 +665,11 @@
jirix.TimerPush("resove instance id for cipd packages")
defer jirix.TimerPop()
+ pkgs, _, err := pkgs.FilterACL(jirix)
+ if err != nil {
+ return nil, err
+ }
+
ensureFilePath, err := generateEnsureFile(jirix, pkgs, false)
if err != nil {
return nil, err
@@ -642,7 +707,12 @@
jirix.TimerPush("fetch cipd packages")
defer jirix.TimerPop()
- ensureFilePath, err := generateEnsureFile(jirix, pkgs, !jirix.LockfileEnabled || jirix.UsingSnapshot)
+ pkgsWAccess, hasInternalPkgs, err := pkgs.FilterACL(jirix)
+ if err != nil {
+ return err
+ }
+
+ ensureFilePath, err := generateEnsureFile(jirix, pkgsWAccess, !jirix.LockfileEnabled || jirix.UsingSnapshot)
if err != nil {
return err
}
@@ -660,9 +730,42 @@
return err
}
+ if hasInternalPkgs {
+ if err := writePackageJSON(jirix, len(pkgs) == len(pkgsWAccess)); err != nil {
+ return err
+ }
+ }
+
+ if len(pkgs) > len(pkgsWAccess) {
+ cipdLoggedIn, err := cipd.CheckLoggedIn(jirix)
+ if err != nil {
+ return err
+ }
+ if !cipdLoggedIn {
+ jirix.Logger.Warningf("Some packages are skipped by cipd due to lack of access, you might want to run \"cipd auth-login\" and try again")
+ }
+ }
return nil
}
+// GenerateJSON generates a json file which contains fetched
+// packages.
+func writePackageJSON(jirix *jiri.X, access bool) error {
+ var internalAccess struct {
+ Access bool `json:"internal_access"`
+ }
+ internalAccess.Access = access
+ jsonData, err := json.MarshalIndent(&internalAccess, "", " ")
+ if err != nil {
+ return err
+ }
+ if jirix.PrebuiltJSON == "" {
+ // Skip json file creation if PrebuiltJSON is not set.
+ return nil
+ }
+ return ioutil.WriteFile(filepath.Join(jirix.RootMetaDir(), jirix.PrebuiltJSON), jsonData, 0644)
+}
+
func generateEnsureFile(jirix *jiri.X, pkgs Packages, ignoreCryptoCheck bool) (string, error) {
ensureFile, err := ioutil.TempFile("", "jiri*.ensure")
if err != nil {
@@ -703,28 +806,8 @@
ensureFileBuf.WriteString("$ParanoidMode CheckPresence\n")
ensureFileBuf.WriteString("\n")
- // Perform ACL checks on internal projects
- pkgACLMap := make(map[string]bool)
- pkgVersionMap := make(map[string]string)
for _, pkg := range pkgs {
- pkg.Name = strings.TrimRight(pkg.Name, "/")
- if pkg.Internal {
- pkgACLMap[pkg.Name] = false
- pkgVersionMap[pkg.Name] = pkg.Version
- }
- }
- if len(pkgACLMap) != 0 {
- if err := cipd.CheckPackageACL(jirix, pkgACLMap, pkgVersionMap); err != nil {
- return "", err
- }
- }
- hasSkippedPkgs := false
- for _, pkg := range pkgs {
- if val, ok := pkgACLMap[pkg.Name]; ok && !val {
- hasSkippedPkgs = true
- continue
- }
cipdDecl, err := pkg.cipdDecl(jirix.UsingSnapshot)
if err != nil {
return "", err
@@ -740,22 +823,17 @@
if err := ensureFile.Sync(); err != nil {
return "", err
}
- if hasSkippedPkgs {
- cipdLoggedIn, err := cipd.CheckLoggedIn(jirix)
- if err != nil {
- return "", err
- }
- if !cipdLoggedIn {
- jirix.Logger.Warningf("Some packages are skipped by cipd due to lack of access, you might want to run \"cipd auth-login\" and try again")
- }
- }
+
return ensureFilePath, nil
}
func (p *Package) cipdDecl(usingSnapshot bool) (string, error) {
var buf bytes.Buffer
// Write "@Subdir" line to cipd declaration
- subdir := p.Path
+ subdir, err := p.GetPath()
+ if err != nil {
+ return "", err
+ }
tmpl, err := template.New("pack").Parse(subdir)
if err != nil {
return "", fmt.Errorf("parsing package path %q failed", subdir)
diff --git a/project/project_test.go b/project/project_test.go
index 09ffb6e..0d7b9bf 100644
--- a/project/project_test.go
+++ b/project/project_test.go
@@ -19,6 +19,7 @@
"testing"
"fuchsia.googlesource.com/jiri"
+ "fuchsia.googlesource.com/jiri/cipd"
"fuchsia.googlesource.com/jiri/gitutil"
"fuchsia.googlesource.com/jiri/jiritest"
"fuchsia.googlesource.com/jiri/project"
@@ -2265,3 +2266,31 @@
}
}
+
+func TestGetPath(t *testing.T) {
+ testPkgs := []project.Package{
+ project.Package{Name: "test0", Version: "version", Path: "A/test0"},
+ project.Package{Name: "test1/${platform}", Version: "version", Path: ""},
+ project.Package{Name: "test2/${os}-${arch}", Version: "version", Path: ""},
+ project.Package{Name: "test3/${platform=linux-armv6l}", Version: "version", Path: ""},
+ }
+ testResults := []string{
+ "A/test0",
+ "prebuilt/test1/",
+ "prebuilt/test2/",
+ "prebuilt",
+ }
+
+ for i, v := range testPkgs {
+ defaultPath, err := v.GetPath()
+ if err != nil {
+ t.Errorf("TestGetPath failed due to error: %v", err)
+ }
+ if strings.HasSuffix(testResults[i], "/") {
+ testResults[i] += cipd.CipdPlatform.String()
+ }
+ if testResults[i] != defaultPath {
+ t.Errorf("expecting %q, got %q", testResults[i], defaultPath)
+ }
+ }
+}
diff --git a/x.go b/x.go
index 7c98262..dadd7a7 100644
--- a/x.go
+++ b/x.go
@@ -51,6 +51,7 @@
SsoCookiePath string `xml:"SsoCookiePath,omitempty"`
LockfileEnabled bool `xml:"lockfile>enabled,omitempty"`
LockfileName string `xml:"lockfile>name,omitempty"`
+ PrebuiltJSON string `xml:"prebuilt>JSON,omitempty"`
AnalyticsOptIn string `xml:"analytics>optin,omitempty"`
AnalyticsUserId string `xml:"analytics>userId,omitempty"`
// version user has opted-in to
@@ -105,6 +106,7 @@
LockfileEnabled bool
LockfileName string
SsoCookiePath string
+ PrebuiltJSON string
UsingSnapshot bool
IgnoreLockConflicts bool
Color color.Color
@@ -237,9 +239,13 @@
x.SsoCookiePath = x.config.SsoCookiePath
x.LockfileEnabled = x.config.LockfileEnabled
x.LockfileName = x.config.LockfileName
+ x.PrebuiltJSON = x.config.PrebuiltJSON
if x.LockfileName == "" {
x.LockfileName = "jiri.lock"
}
+ if x.PrebuiltJSON == "" {
+ x.PrebuiltJSON = "prebuilt.json"
+ }
}
x.Cache, err = findCache(root, x.config)
if x.config != nil {
