[cipd] add platforms attribute for packages
This patch adds platforms attribute for <package> tags in manifests.
Currently jiri does not generate locks for linux-arm64 packages
as not all packages are targeted for linux-arm64. The newly added
'platforms' allows user to specify the exact platforms the packages
are targeted, which make it possible for jiri to generate correct
locks for additional platforms including linux-arm64.
Change-Id: Icb0bc5c615c329f10f688f55a1c209e92105cbf1
diff --git a/cipd/cipd.go b/cipd/cipd.go
index cb2e839..af076f2 100644
--- a/cipd/cipd.go
+++ b/cipd/cipd.go
@@ -503,6 +503,15 @@
Arch string
}
+// NewPlatform parses a platform string into Platform struct.
+func NewPlatform(s string) (Platform, error) {
+ fields := strings.Split(s, "-")
+ if len(fields) != 2 {
+ return Platform{"", ""}, fmt.Errorf("illegal platform %q", s)
+ }
+ return Platform{fields[0], fields[1]}, nil
+}
+
// String generates a string represents the Platform in "OS"-"Arch" form.
func (p Platform) String() string {
return p.OS + "-" + p.Arch
@@ -587,7 +596,7 @@
func Expand(cipdPath string, platforms []Platform) ([]string, error) {
output := make([]string, 0)
//expanders := make([]Expander, 0)
- if !templateRE.MatchString(cipdPath) {
+ if !MustExpand(cipdPath) {
output = append(output, cipdPath)
return output, nil
}
@@ -605,6 +614,61 @@
return output, nil
}
+// Decl method expands a cipdPath that contains ${platform}, ${os}, ${arch}
+// with information in platforms. Unlike the Expand method which
+// returns a list of expanded cipd paths, the Decl method only returns a
+// single path containing all platforms. For example, if platforms contain
+// "linux-amd64" and "linux-arm64", ${platform} will be replaced to
+// ${platform=linux-amd64,linux-arm64}. This is a workaround for a limitation
+// in 'cipd ensure-file-resolve' which requires the header of '.ensure' file
+// to contain all available platforms. But in some cases, a package may miss
+// a particular platform, which will cause a crash on this cipd command. By
+// explicitly list all supporting platforms in the cipdPath, we can avoid
+// crashing cipd.
+func Decl(cipdPath string, platforms []Platform) (string, error) {
+ if !MustExpand(cipdPath) || len(platforms) == 0 {
+ return cipdPath, nil
+ }
+
+ osMap := make(map[string]bool)
+ platMap := make(map[string]bool)
+ archMap := make(map[string]bool)
+
+ replacedOS := "${os="
+ replacedArch := "${arch="
+ replacedPlat := "${platform="
+
+ for _, plat := range platforms {
+ if _, ok := osMap[plat.OS]; !ok {
+ replacedOS += plat.OS + ","
+ osMap[plat.OS] = true
+ }
+ if _, ok := archMap[plat.Arch]; !ok {
+ replacedArch += plat.Arch + ","
+ archMap[plat.Arch] = true
+ }
+ if _, ok := platMap[plat.String()]; !ok {
+ replacedPlat += plat.String() + ","
+ platMap[plat.String()] = true
+ }
+ }
+ replacedOS = replacedOS[:len(replacedOS)-1] + "}"
+ replacedArch = replacedArch[:len(replacedArch)-1] + "}"
+ replacedPlat = replacedPlat[:len(replacedPlat)-1] + "}"
+
+ cipdPath = strings.Replace(cipdPath, "${os}", replacedOS, -1)
+ cipdPath = strings.Replace(cipdPath, "${arch}", replacedArch, -1)
+ cipdPath = strings.Replace(cipdPath, "${platform}", replacedPlat, -1)
+ return cipdPath, nil
+}
+
+// MustExpand checks if template usages such as "${platform}" exist
+// in cipdPath. If they exist, this function will return true. Otherwise
+// it returns false.
+func MustExpand(cipdPath string) bool {
+ return templateRE.MatchString(cipdPath)
+}
+
// DefaultPlatforms returns a slice of Platform objects that are currently
// validated by jiri.
func DefaultPlatforms() []Platform {
diff --git a/cipd/cipd_test.go b/cipd/cipd_test.go
index e96cce6..25a8061 100644
--- a/cipd/cipd_test.go
+++ b/cipd/cipd_test.go
@@ -294,3 +294,43 @@
}
}
}
+
+func TestMustExpand(t *testing.T) {
+ tests := map[string]bool{
+ "fuchsia/clang/${platform}": true,
+ "fuchsia/clang/${os}-${arch}": true,
+ "fuchsia/clang/${os=linux}-${arch=amd64}": true,
+ "fuchsia/clang/linux-amd64": false,
+ }
+ for k, v := range tests {
+ if MustExpand(k) != v {
+ t.Errorf("MustExpand failed on package %q, expecting %v got %v", k, v, MustExpand(k))
+ }
+ }
+}
+
+func TestDecl(t *testing.T) {
+ platforms := []Platform{
+ Platform{"linux", "amd64"},
+ Platform{"linux", "arm64"},
+ Platform{"mac", "amd64"},
+ }
+
+ tests := map[string]string{
+ "fuchsia/clang/${platform}": "fuchsia/clang/${platform=linux-amd64,linux-arm64,mac-amd64}",
+ "fuchsia/clang/${os}-${arch}": "fuchsia/clang/${os=linux,mac}-${arch=amd64,arm64}",
+ "fuchsia/clang/${os=linux}-${arch}": "fuchsia/clang/${os=linux}-${arch=amd64,arm64}",
+ "fuchsia/clang/${os=linux}-${arch=amd64}": "fuchsia/clang/${os=linux}-${arch=amd64}",
+ "fuchsia/clang/linux-amd64": "fuchsia/clang/linux-amd64",
+ }
+
+ for k, v := range tests {
+ cipdPath, err := Decl(k, platforms)
+ if err != nil {
+ t.Errorf("Decl failed on cipdPath %q due to error: %v", k, err)
+ }
+ if cipdPath != v {
+ t.Errorf("test on %q failed: expecting %q, got %q", k, v, cipdPath)
+ }
+ }
+}
diff --git a/project/manifest.go b/project/manifest.go
index b1a9b2e..5191459 100644
--- a/project/manifest.go
+++ b/project/manifest.go
@@ -428,6 +428,7 @@
Version string `xml:"version,attr"`
Path string `xml:"path,attr,omitempty"`
Internal bool `xml:"internal,attr,omitempty"`
+ Platforms string `xml:"platforms,attr,omitempty"`
Instances []PackageInstance `xml:"instance"`
XMLName struct{} `xml:"package"`
}
@@ -436,8 +437,8 @@
type Packages map[PackageKey]Package
-func (pkg Package) Key() PackageKey {
- return PackageKey(pkg.Name)
+func (p Package) Key() PackageKey {
+ return PackageKey(p.Name)
}
type PackageInstance struct {
@@ -446,6 +447,41 @@
XMLName struct{} `xml:"instance"`
}
+// FillDefaults function fills default platforms information into
+// Package struct if it is not defined and path is using template.
+func (p *Package) FillDefaults() error {
+ if cipd.MustExpand(p.Name) && p.Platforms == "" {
+ for _, v := range cipd.DefaultPlatforms() {
+ p.Platforms += v.String() + ","
+ }
+ if p.Platforms[len(p.Platforms)-1] == ',' {
+ p.Platforms = p.Platforms[:len(p.Platforms)-1]
+ }
+ }
+ return nil
+}
+
+// GetPlatforms returns the platforms information of
+// this Package struct.
+func (p *Package) GetPlatforms() ([]cipd.Platform, error) {
+ if err := p.FillDefaults(); err != nil {
+ return nil, err
+ }
+ retList := make([]cipd.Platform, 0)
+ platStrs := strings.Split(p.Platforms, ",")
+ for _, platStr := range platStrs {
+ if platStr == "" {
+ continue
+ }
+ plat, err := cipd.NewPlatform(platStr)
+ if err != nil {
+ return nil, err
+ }
+ retList = append(retList, plat)
+ }
+ return retList, nil
+}
+
// LoadManifest loads the manifest, starting with the .jiri_manifest file,
// resolving remote and local imports. Returns the projects specified by
// the manifest.
@@ -637,7 +673,25 @@
// to avoid hardcoding platform names in Jiri
var ensureFileBuf bytes.Buffer
if !ignoreCryptoCheck {
+ // Collect platforms used by this project
+ allPlats := make(map[string]cipd.Platform)
+ // CIPD ensure-file-resolve requires $VerifiedPlatform to be present
+ // even if the package name is not using ${platform} template.
+ // Put DefaultPlatforms into header to walkaround this issue.
for _, plat := range cipd.DefaultPlatforms() {
+ allPlats[plat.String()] = plat
+ }
+ for _, pkg := range pkgs {
+ plats, err := pkg.GetPlatforms()
+ if err != nil {
+ return "", err
+ }
+ for _, plat := range plats {
+ allPlats[plat.String()] = plat
+ }
+ }
+
+ for _, plat := range allPlats {
ensureFileBuf.WriteString(fmt.Sprintf("$VerifiedPlatform %s\n", plat))
}
versionFileName := ensureFilePath[:len(ensureFilePath)-len(".ensure")] + ".version"
@@ -695,10 +749,10 @@
return ensureFilePath, nil
}
-func (pkg *Package) cipdDecl() (string, error) {
+func (p *Package) cipdDecl() (string, error) {
var buf bytes.Buffer
// Write "@Subdir" line to cipd declaration
- subdir := pkg.Path
+ subdir := p.Path
tmpl, err := template.New("pack").Parse(subdir)
if err != nil {
return "", fmt.Errorf("parsing package path %q failed", subdir)
@@ -708,7 +762,15 @@
subdir = subdirBuf.String()
buf.WriteString(fmt.Sprintf("@Subdir %s\n", subdir))
// Write package version line to cipd declaration
- buf.WriteString(fmt.Sprintf("%s %s\n", pkg.Name, pkg.Version))
+ plats, err := p.GetPlatforms()
+ if err != nil {
+ return "", err
+ }
+ cipdPath, err := cipd.Decl(p.Name, plats)
+ if err != nil {
+ return "", err
+ }
+ buf.WriteString(fmt.Sprintf("%s %s\n", cipdPath, p.Version))
return buf.String(), nil
}
