[sdk][merge] Start a Rust version of the SDK merging tool.
This change introduces the foundations for the tool, which can read and
write tarballs but does not perform any meaningful merging operation.
Actual functionality will be added in follow-up changes.
Bug: DX-1056
Change-Id: Ic90ef75055cdaddf4b8c3d0f7eaa15ac9a5a270e
diff --git a/build/sdk/tools/merge/BUILD.gn b/build/sdk/tools/merge/BUILD.gn
new file mode 100644
index 0000000..8cf60a4
--- /dev/null
+++ b/build/sdk/tools/merge/BUILD.gn
@@ -0,0 +1,22 @@
+# Copyright 2019 The Fuchsia Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/rust/rustc_binary.gni")
+
+rustc_binary("merge_sdk") {
+ name = "merge_sdk"
+
+ with_unit_tests = true
+
+ edition = "2018"
+
+ deps = [
+ "//build/sdk/meta:rust",
+ "//third_party/rust_crates:failure",
+ "//third_party/rust_crates:flate2",
+ "//third_party/rust_crates:serde_json",
+ "//third_party/rust_crates:structopt",
+ "//third_party/rust_crates:tar",
+ ]
+}
diff --git a/build/sdk/tools/merge/src/app.rs b/build/sdk/tools/merge/src/app.rs
new file mode 100644
index 0000000..64bfe6e
--- /dev/null
+++ b/build/sdk/tools/merge/src/app.rs
@@ -0,0 +1,21 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+use failure::Fail;
+
+/// The various types of errors raised by this tool.
+#[derive(Debug, Fail)]
+pub enum Error {
+ #[fail(display = "could not find file in archive: {}", name)]
+ ArchiveFileNotFound {
+ name: String,
+ },
+ #[fail(display = "path already maps to a file: {}", path)]
+ PathAlreadyExists {
+ path: String,
+ },
+}
+
+/// Common result types for methods in this crate.
+pub type Result<T> = std::result::Result<T, failure::Error>;
diff --git a/build/sdk/tools/merge/src/flags.rs b/build/sdk/tools/merge/src/flags.rs
new file mode 100644
index 0000000..422e03f
--- /dev/null
+++ b/build/sdk/tools/merge/src/flags.rs
@@ -0,0 +1,17 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub struct Flags {
+ #[structopt(short = "b", long = "base-archive")]
+ pub base: String,
+
+ #[structopt(short = "c", long = "complement-archive")]
+ pub complement: String,
+
+ #[structopt(short = "o", long = "output-archive")]
+ pub output: String,
+}
diff --git a/build/sdk/tools/merge/src/main.rs b/build/sdk/tools/merge/src/main.rs
new file mode 100644
index 0000000..0ec2a13
--- /dev/null
+++ b/build/sdk/tools/merge/src/main.rs
@@ -0,0 +1,123 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+use std::collections::HashSet;
+use std::iter::{FromIterator, Iterator};
+
+use structopt::StructOpt;
+
+use sdk_metadata::{JsonObject, Manifest, Part};
+
+mod app;
+mod flags;
+mod tarball;
+
+use crate::app::Result;
+use crate::tarball::{OutputTarball, SourceTarball};
+
+const MANIFEST_PATH: &str = "meta/manifest.json";
+
+fn merge_manifests(_base: &Manifest, _complement: &Manifest) -> Result<Manifest> {
+ let result: Manifest = Default::default();
+ // TODO(DX-1056): perform some meaningful merging operation here.
+ result.validate()?;
+ Ok(result)
+}
+
+fn merge_common_part(part: &Part, _base: &SourceTarball, _complement: &SourceTarball, _output: &OutputTarball) -> Result<()> {
+ // TODO(DX-1056): implement me.
+ println!(" - {}", part);
+ Ok(())
+}
+
+fn copy_part_as_is(part: &Part, _source: &SourceTarball, _output: &OutputTarball) -> Result<()> {
+ // TODO(DX-1056): implement me.
+ println!(" - {}", part);
+ Ok(())
+}
+
+fn main() -> Result<()> {
+ let flags = flags::Flags::from_args();
+
+ let mut base = SourceTarball::new(flags.base)?;
+ let mut complement = SourceTarball::new(flags.complement)?;
+ let mut output = OutputTarball::new();
+
+ let base_manifest: Manifest = base.get_metadata(MANIFEST_PATH)?;
+ let complement_manifest: Manifest = complement.get_metadata(MANIFEST_PATH)?;
+
+ let base_parts: HashSet<Part> = HashSet::from_iter(base_manifest.parts.iter().cloned());
+ let complement_parts: HashSet<Part> = HashSet::from_iter(complement_manifest.parts.iter().cloned());
+
+ println!("Common parts");
+ for part in base_parts.intersection(&complement_parts) {
+ merge_common_part(&part, &base, &complement, &output)?;
+ }
+
+ println!("Unique base parts");
+ for part in base_parts.difference(&complement_parts) {
+ copy_part_as_is(&part, &base, &output)?;
+ }
+
+ println!("Unique complement parts");
+ for part in complement_parts.difference(&base_parts) {
+ copy_part_as_is(&part, &complement, &output)?;
+ }
+
+ let merged_manifest = merge_manifests(&base_manifest, &complement_manifest)?;
+ merged_manifest.validate()?;
+
+ output.write("meta/manifest.json".to_string(), merged_manifest.to_string()?)?;
+ output.export(flags.output)?;
+
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use serde_json::{json, from_value};
+ use serde_json::value::Value;
+
+ use sdk_metadata::Manifest;
+
+ use super::*;
+
+ macro_rules! test_merge {
+ (
+ name = $name:ident,
+ base = $base:expr,
+ complement = $complement:expr,
+ success = $success:expr,
+ ) => {
+ #[test]
+ fn $name() {
+ merge_test($base, $complement, $success);
+ }
+ }
+ }
+
+ fn merge_test(base: Value, complement: Value, success: bool) {
+ let base_manifest: Manifest = from_value(base).unwrap();
+ let complement_manifest: Manifest = from_value(complement).unwrap();
+ let merged_manifest = merge_manifests(&base_manifest, &complement_manifest);
+ assert_eq!(merged_manifest.is_ok(), success);
+ }
+
+ test_merge!(
+ name = test_just_an_experiment,
+ base = json!({
+ "arch": { "host": "foobarblah", "target": ["x64"] },
+ "parts": [],
+ "id": "bleh",
+ "schema_version": "1",
+ }),
+ complement = json!({
+ "arch": { "host": "foobarblah", "target": ["x64"] },
+ "parts": [],
+ "id": "bleh",
+ "schema_version": "1",
+ }),
+ success = true,
+ );
+}
diff --git a/build/sdk/tools/merge/src/tarball.rs b/build/sdk/tools/merge/src/tarball.rs
new file mode 100644
index 0000000..c76a6cd
--- /dev/null
+++ b/build/sdk/tools/merge/src/tarball.rs
@@ -0,0 +1,115 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file
+
+use std::collections::HashMap;
+use std::convert::TryFrom;
+use std::fs::File;
+use std::io::Read;
+use std::path::Path;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use flate2::Compression;
+use flate2::write::GzEncoder;
+use flate2::read::GzDecoder;
+use tar::{Archive, Builder, EntryType, Header};
+
+use sdk_metadata::JsonObject;
+
+use crate::app::{Error, Result};
+
+type SdkArchive = Archive<GzDecoder<File>>;
+
+/// A tarball that can be read from.
+pub struct SourceTarball {
+ archive: SdkArchive,
+}
+
+impl SourceTarball {
+ /// Creates a new tarball at the given path.
+ pub fn new(path: String) -> Result<SourceTarball> {
+ let tar_gz = File::open(path)?;
+ let tar = GzDecoder::new(tar_gz)?;
+ Ok(SourceTarball { archive: Archive::new(tar) })
+ }
+
+ /// Reads a file from the tarball.
+ fn get_file<'a>(&'a mut self, path: &str) -> Result<impl Read + 'a> {
+ let archive = &mut self.archive;
+ let entries = archive.entries()?;
+ Ok(entries
+ .filter_map(|entry| { entry.ok() })
+ .find(|entry| {
+ if let Ok(entry_path) = entry.path() {
+ return entry_path.to_str() == Some(path)
+ }
+ false
+ })
+ .ok_or(Error::ArchiveFileNotFound { name: path.to_string() })?)
+ }
+
+ /// Reads a metadata object from the tarball.
+ pub fn get_metadata<T: JsonObject>(&mut self, path: &str) -> Result<T> {
+ T::new(self.get_file(path)?)
+ }
+}
+
+/// The types of content that can be written to a tarball.
+enum TarballContent {
+ /// Plain string content, exported as a read-only file.
+ Plain(String),
+}
+
+/// A tarball that can be written into.
+pub struct OutputTarball {
+ contents: HashMap<String, TarballContent>,
+}
+
+impl OutputTarball {
+ /// Creates a new tarball.
+ pub fn new() -> OutputTarball {
+ OutputTarball { contents: HashMap::new() }
+ }
+
+ /// Writes the given content to the given path in the tarball.
+ ///
+ /// It is an error to write to the same path twice.
+ pub fn write(&mut self, path: String, content: String) -> Result<()> {
+ if let Some(_) = self.contents.insert(path.clone(), TarballContent::Plain(content)) {
+ return Err(Error::PathAlreadyExists { path })?;
+ }
+ Ok(())
+ }
+
+ /// Creates the tarball at the given path.
+ ///
+ /// This method will obliterate any file that already exists at that path.
+ pub fn export(&self, path: String) -> Result<()> {
+ let output_path = Path::new(&path);
+ if output_path.exists() {
+ std::fs::remove_file(output_path)?;
+ }
+ let tar = File::create(&path)?;
+ let tar_gz = GzEncoder::new(tar, Compression::default());
+ let mut builder = Builder::new(tar_gz);
+ for (file_path, content) in &self.contents {
+ match content {
+ TarballContent::Plain(s) => {
+ let bytes = s.as_bytes();
+ let mut header = Header::new_gnu();
+ header.set_path(file_path)?;
+ header.set_size(u64::try_from(bytes.len())?);
+ header.set_entry_type(EntryType::Regular);
+ // Make the file readable.
+ header.set_mode(0o444);
+ // Add a timestamp.
+ let start = SystemTime::now();
+ let epoch = start.duration_since(UNIX_EPOCH)?.as_secs();
+ header.set_mtime(epoch);
+ builder.append_data(&mut header, file_path, bytes)?;
+ },
+ }
+ }
+ Ok(builder.finish()?)
+ }
+}
