| //! Build a dist manifest, hash and sign everything. |
| //! This gets called by `promote-release` |
| //! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release) |
| //! via `x.py dist hash-and-sign`; the cmdline arguments are set up |
| //! by rustbuild (in `src/bootstrap/dist.rs`). |
| |
| mod manifest; |
| mod versions; |
| |
| use crate::manifest::{Component, FileHash, Manifest, Package, Rename, Target}; |
| use crate::versions::{PkgType, Versions}; |
| use rayon::prelude::*; |
| use sha2::Digest; |
| use std::collections::{BTreeMap, HashMap, HashSet}; |
| use std::env; |
| use std::error::Error; |
| use std::fs::{self, File}; |
| use std::io::{self, BufReader, Read, Write}; |
| use std::path::{Path, PathBuf}; |
| use std::process::{Command, Stdio}; |
| use std::sync::Mutex; |
| use std::time::Instant; |
| |
| static HOSTS: &[&str] = &[ |
| "aarch64-apple-darwin", |
| "aarch64-pc-windows-msvc", |
| "aarch64-unknown-linux-gnu", |
| "aarch64-unknown-linux-musl", |
| "arm-unknown-linux-gnueabi", |
| "arm-unknown-linux-gnueabihf", |
| "armv7-unknown-linux-gnueabihf", |
| "i686-apple-darwin", |
| "i686-pc-windows-gnu", |
| "i686-pc-windows-msvc", |
| "i686-unknown-linux-gnu", |
| "mips-unknown-linux-gnu", |
| "mips64-unknown-linux-gnuabi64", |
| "mips64el-unknown-linux-gnuabi64", |
| "mipsel-unknown-linux-gnu", |
| "mipsisa32r6-unknown-linux-gnu", |
| "mipsisa32r6el-unknown-linux-gnu", |
| "mipsisa64r6-unknown-linux-gnuabi64", |
| "mipsisa64r6el-unknown-linux-gnuabi64", |
| "powerpc-unknown-linux-gnu", |
| "powerpc64-unknown-linux-gnu", |
| "powerpc64le-unknown-linux-gnu", |
| "riscv64gc-unknown-linux-gnu", |
| "s390x-unknown-linux-gnu", |
| "x86_64-apple-darwin", |
| "x86_64-pc-windows-gnu", |
| "x86_64-pc-windows-msvc", |
| "x86_64-unknown-freebsd", |
| "x86_64-unknown-illumos", |
| "x86_64-unknown-linux-gnu", |
| "x86_64-unknown-linux-musl", |
| "x86_64-unknown-netbsd", |
| ]; |
| |
| static TARGETS: &[&str] = &[ |
| "aarch64-apple-darwin", |
| "aarch64-apple-ios", |
| "aarch64-fuchsia", |
| "aarch64-linux-android", |
| "aarch64-pc-windows-msvc", |
| "aarch64-unknown-cloudabi", |
| "aarch64-unknown-hermit", |
| "aarch64-unknown-linux-gnu", |
| "aarch64-unknown-linux-musl", |
| "aarch64-unknown-none", |
| "aarch64-unknown-none-softfloat", |
| "aarch64-unknown-redox", |
| "arm-linux-androideabi", |
| "arm-unknown-linux-gnueabi", |
| "arm-unknown-linux-gnueabihf", |
| "arm-unknown-linux-musleabi", |
| "arm-unknown-linux-musleabihf", |
| "armv5te-unknown-linux-gnueabi", |
| "armv5te-unknown-linux-musleabi", |
| "armv7-apple-ios", |
| "armv7-linux-androideabi", |
| "thumbv7neon-linux-androideabi", |
| "armv7-unknown-linux-gnueabi", |
| "armv7-unknown-linux-gnueabihf", |
| "armv7a-none-eabi", |
| "thumbv7neon-unknown-linux-gnueabihf", |
| "armv7-unknown-linux-musleabi", |
| "armv7-unknown-linux-musleabihf", |
| "armebv7r-none-eabi", |
| "armebv7r-none-eabihf", |
| "armv7r-none-eabi", |
| "armv7r-none-eabihf", |
| "armv7s-apple-ios", |
| "asmjs-unknown-emscripten", |
| "i386-apple-ios", |
| "i586-pc-windows-msvc", |
| "i586-unknown-linux-gnu", |
| "i586-unknown-linux-musl", |
| "i686-apple-darwin", |
| "i686-linux-android", |
| "i686-pc-windows-gnu", |
| "i686-pc-windows-msvc", |
| "i686-unknown-freebsd", |
| "i686-unknown-linux-gnu", |
| "i686-unknown-linux-musl", |
| "mips-unknown-linux-gnu", |
| "mips-unknown-linux-musl", |
| "mips64-unknown-linux-gnuabi64", |
| "mips64-unknown-linux-muslabi64", |
| "mips64el-unknown-linux-gnuabi64", |
| "mips64el-unknown-linux-muslabi64", |
| "mipsisa32r6-unknown-linux-gnu", |
| "mipsisa32r6el-unknown-linux-gnu", |
| "mipsisa64r6-unknown-linux-gnuabi64", |
| "mipsisa64r6el-unknown-linux-gnuabi64", |
| "mipsel-unknown-linux-gnu", |
| "mipsel-unknown-linux-musl", |
| "nvptx64-nvidia-cuda", |
| "powerpc-unknown-linux-gnu", |
| "powerpc64-unknown-linux-gnu", |
| "powerpc64le-unknown-linux-gnu", |
| "riscv32i-unknown-none-elf", |
| "riscv32imc-unknown-none-elf", |
| "riscv32imac-unknown-none-elf", |
| "riscv32gc-unknown-linux-gnu", |
| "riscv64imac-unknown-none-elf", |
| "riscv64gc-unknown-none-elf", |
| "riscv64gc-unknown-linux-gnu", |
| "s390x-unknown-linux-gnu", |
| "sparc64-unknown-linux-gnu", |
| "sparcv9-sun-solaris", |
| "thumbv6m-none-eabi", |
| "thumbv7em-none-eabi", |
| "thumbv7em-none-eabihf", |
| "thumbv7m-none-eabi", |
| "thumbv8m.base-none-eabi", |
| "thumbv8m.main-none-eabi", |
| "thumbv8m.main-none-eabihf", |
| "wasm32-unknown-emscripten", |
| "wasm32-unknown-unknown", |
| "wasm32-wasi", |
| "x86_64-apple-darwin", |
| "x86_64-apple-ios", |
| "x86_64-fortanix-unknown-sgx", |
| "x86_64-fuchsia", |
| "x86_64-linux-android", |
| "x86_64-pc-windows-gnu", |
| "x86_64-pc-windows-msvc", |
| "x86_64-rumprun-netbsd", |
| "x86_64-sun-solaris", |
| "x86_64-pc-solaris", |
| "x86_64-unknown-cloudabi", |
| "x86_64-unknown-freebsd", |
| "x86_64-unknown-illumos", |
| "x86_64-unknown-linux-gnu", |
| "x86_64-unknown-linux-gnux32", |
| "x86_64-unknown-linux-musl", |
| "x86_64-unknown-netbsd", |
| "x86_64-unknown-redox", |
| "x86_64-unknown-hermit", |
| ]; |
| |
| static DOCS_TARGETS: &[&str] = &[ |
| "i686-apple-darwin", |
| "i686-pc-windows-gnu", |
| "i686-pc-windows-msvc", |
| "i686-unknown-linux-gnu", |
| "x86_64-apple-darwin", |
| "x86_64-pc-windows-gnu", |
| "x86_64-pc-windows-msvc", |
| "x86_64-unknown-linux-gnu", |
| "x86_64-unknown-linux-musl", |
| ]; |
| |
| static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"]; |
| |
| static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview", "rust-analyzer-preview"]; |
| |
| macro_rules! t { |
| ($e:expr) => { |
| match $e { |
| Ok(e) => e, |
| Err(e) => panic!("{} failed with {}", stringify!($e), e), |
| } |
| }; |
| } |
| |
| struct Builder { |
| versions: Versions, |
| |
| input: PathBuf, |
| output: PathBuf, |
| s3_address: String, |
| date: String, |
| |
| legacy: bool, |
| legacy_gpg_passphrase: String, |
| } |
| |
| fn main() { |
| // Up until Rust 1.48 the release process relied on build-manifest to create the SHA256 |
| // checksums of released files and to sign the tarballs. That was moved over to promote-release |
| // in time for the branching of Rust 1.48, but the old release process still had to work the |
| // old way. |
| // |
| // When running build-manifest through the old ./x.py dist hash-and-sign the environment |
| // variable will be set, enabling the legacy behavior of generating the .sha256 files and |
| // signing the tarballs. |
| // |
| // Once the old release process is fully decommissioned, the environment variable, all the |
| // related code in this tool and ./x.py dist hash-and-sign can be removed. |
| let legacy = env::var_os("BUILD_MANIFEST_LEGACY").is_some(); |
| |
| let num_threads = if legacy { |
| // Avoid overloading the old server in legacy mode. |
| 1 |
| } else if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") { |
| num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS") |
| } else { |
| num_cpus::get() |
| }; |
| rayon::ThreadPoolBuilder::new() |
| .num_threads(num_threads) |
| .build_global() |
| .expect("failed to initialize Rayon"); |
| |
| let mut args = env::args().skip(1); |
| let input = PathBuf::from(args.next().unwrap()); |
| let output = PathBuf::from(args.next().unwrap()); |
| let date = args.next().unwrap(); |
| let s3_address = args.next().unwrap(); |
| let channel = args.next().unwrap(); |
| |
| // Do not ask for a passphrase while manually testing |
| let mut passphrase = String::new(); |
| if legacy { |
| // `x.py` passes the passphrase via stdin. |
| t!(io::stdin().read_to_string(&mut passphrase)); |
| } |
| |
| Builder { |
| versions: Versions::new(&channel, &input).unwrap(), |
| |
| input, |
| output, |
| s3_address, |
| date, |
| |
| legacy, |
| legacy_gpg_passphrase: passphrase, |
| } |
| .build(); |
| } |
| |
| impl Builder { |
| fn build(&mut self) { |
| self.check_toolstate(); |
| if self.legacy { |
| self.digest_and_sign(); |
| } |
| let manifest = self.build_manifest(); |
| |
| self.write_channel_files(self.versions.channel(), &manifest); |
| if self.versions.channel() == "stable" { |
| // channel-rust-1.XX.YY.toml |
| let rust_version = self.versions.rustc_version(); |
| self.write_channel_files(rust_version, &manifest); |
| |
| // channel-rust-1.XX.toml |
| let major_minor = rust_version.split('.').take(2).collect::<Vec<_>>().join("."); |
| self.write_channel_files(&major_minor, &manifest); |
| } |
| } |
| |
| /// If a tool does not pass its tests, don't ship it. |
| /// Right now, we do this only for Miri. |
| fn check_toolstate(&mut self) { |
| let toolstates: Option<HashMap<String, String>> = |
| File::open(self.input.join("toolstates-linux.json")) |
| .ok() |
| .and_then(|f| serde_json::from_reader(&f).ok()); |
| let toolstates = toolstates.unwrap_or_else(|| { |
| println!( |
| "WARNING: `toolstates-linux.json` missing/malformed; \ |
| assuming all tools failed" |
| ); |
| HashMap::default() // Use empty map if anything went wrong. |
| }); |
| // Mark some tools as missing based on toolstate. |
| if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") { |
| println!("Miri tests are not passing, removing component"); |
| self.versions.disable_version(&PkgType::Miri); |
| } |
| } |
| |
| /// Hash all files, compute their signatures, and collect the hashes in `self.digests`. |
| fn digest_and_sign(&mut self) { |
| for file in t!(self.input.read_dir()).map(|e| t!(e).path()) { |
| file.file_name().unwrap().to_str().unwrap(); |
| self.hash(&file); |
| self.sign(&file); |
| } |
| } |
| |
| fn build_manifest(&mut self) -> Manifest { |
| let mut manifest = Manifest { |
| manifest_version: "2".to_string(), |
| date: self.date.to_string(), |
| pkg: BTreeMap::new(), |
| renames: BTreeMap::new(), |
| profiles: BTreeMap::new(), |
| }; |
| self.add_packages_to(&mut manifest); |
| self.add_profiles_to(&mut manifest); |
| self.add_renames_to(&mut manifest); |
| manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest)); |
| |
| self.fill_missing_hashes(&mut manifest); |
| |
| manifest |
| } |
| |
| fn add_packages_to(&mut self, manifest: &mut Manifest) { |
| let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets); |
| package("rustc", HOSTS); |
| package("rustc-dev", HOSTS); |
| package("rustc-docs", HOSTS); |
| package("cargo", HOSTS); |
| package("rust-mingw", MINGW); |
| package("rust-std", TARGETS); |
| package("rust-docs", DOCS_TARGETS); |
| package("rust-src", &["*"]); |
| package("rls-preview", HOSTS); |
| package("rust-analyzer-preview", HOSTS); |
| package("clippy-preview", HOSTS); |
| package("miri-preview", HOSTS); |
| package("rustfmt-preview", HOSTS); |
| package("rust-analysis", TARGETS); |
| package("llvm-tools-preview", TARGETS); |
| } |
| |
| fn add_profiles_to(&mut self, manifest: &mut Manifest) { |
| let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs); |
| profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]); |
| profile( |
| "default", |
| &[ |
| "rustc", |
| "cargo", |
| "rust-std", |
| "rust-mingw", |
| "rust-docs", |
| "rustfmt-preview", |
| "clippy-preview", |
| ], |
| ); |
| profile( |
| "complete", |
| &[ |
| "rustc", |
| "cargo", |
| "rust-std", |
| "rust-mingw", |
| "rust-docs", |
| "rustfmt-preview", |
| "clippy-preview", |
| "rls-preview", |
| "rust-analyzer-preview", |
| "rust-src", |
| "llvm-tools-preview", |
| "rust-analysis", |
| "miri-preview", |
| ], |
| ); |
| |
| // The compiler libraries are not stable for end users, and they're also huge, so we only |
| // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible |
| // for users to install the additional component manually, if needed. |
| if self.versions.channel() == "nightly" { |
| self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]); |
| self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]); |
| } |
| } |
| |
| fn add_renames_to(&self, manifest: &mut Manifest) { |
| let mut rename = |from: &str, to: &str| { |
| manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() }) |
| }; |
| rename("rls", "rls-preview"); |
| rename("rustfmt", "rustfmt-preview"); |
| rename("clippy", "clippy-preview"); |
| rename("miri", "miri-preview"); |
| } |
| |
| fn rust_package(&mut self, manifest: &Manifest) -> Package { |
| let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball"); |
| let mut pkg = Package { |
| version: version_info.version.expect("missing Rust version"), |
| git_commit_hash: version_info.git_commit, |
| target: BTreeMap::new(), |
| }; |
| for host in HOSTS { |
| if let Some(target) = self.target_host_combination(host, &manifest) { |
| pkg.target.insert(host.to_string(), target); |
| } else { |
| pkg.target.insert(host.to_string(), Target::unavailable()); |
| continue; |
| } |
| } |
| pkg |
| } |
| |
| fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> { |
| let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap(); |
| |
| let mut target = Target::from_compressed_tar(self, &filename); |
| if !target.available { |
| return None; |
| } |
| |
| let mut components = Vec::new(); |
| let mut extensions = Vec::new(); |
| |
| let host_component = |pkg| Component::from_str(pkg, host); |
| |
| // rustc/rust-std/cargo/docs are all required, |
| // and so is rust-mingw if it's available for the target. |
| components.extend(vec![ |
| host_component("rustc"), |
| host_component("rust-std"), |
| host_component("cargo"), |
| host_component("rust-docs"), |
| ]); |
| if host.contains("pc-windows-gnu") { |
| components.push(host_component("rust-mingw")); |
| } |
| |
| // Tools are always present in the manifest, |
| // but might be marked as unavailable if they weren't built. |
| extensions.extend(vec![ |
| host_component("clippy-preview"), |
| host_component("miri-preview"), |
| host_component("rls-preview"), |
| host_component("rust-analyzer-preview"), |
| host_component("rustfmt-preview"), |
| host_component("llvm-tools-preview"), |
| host_component("rust-analysis"), |
| ]); |
| |
| extensions.extend( |
| TARGETS |
| .iter() |
| .filter(|&&target| target != host) |
| .map(|target| Component::from_str("rust-std", target)), |
| ); |
| extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-dev", target))); |
| extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-docs", target))); |
| extensions.push(Component::from_str("rust-src", "*")); |
| |
| // If the components/extensions don't actually exist for this |
| // particular host/target combination then nix it entirely from our |
| // lists. |
| let has_component = |c: &Component| { |
| if c.target == "*" { |
| return true; |
| } |
| let pkg = match manifest.pkg.get(&c.pkg) { |
| Some(p) => p, |
| None => return false, |
| }; |
| pkg.target.get(&c.target).is_some() |
| }; |
| extensions.retain(&has_component); |
| components.retain(&has_component); |
| |
| target.components = Some(components); |
| target.extensions = Some(extensions); |
| Some(target) |
| } |
| |
| fn profile( |
| &mut self, |
| profile_name: &str, |
| dst: &mut BTreeMap<String, Vec<String>>, |
| pkgs: &[&str], |
| ) { |
| dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect()); |
| } |
| |
| fn extend_profile( |
| &mut self, |
| profile_name: &str, |
| dst: &mut BTreeMap<String, Vec<String>>, |
| pkgs: &[&str], |
| ) { |
| dst.get_mut(profile_name) |
| .expect("existing profile") |
| .extend(pkgs.iter().map(|s| (*s).to_owned())); |
| } |
| |
| fn package(&mut self, pkgname: &str, dst: &mut BTreeMap<String, Package>, targets: &[&str]) { |
| let version_info = self |
| .versions |
| .version(&PkgType::from_component(pkgname)) |
| .expect("failed to load package version"); |
| let mut is_present = version_info.present; |
| |
| // Never ship nightly-only components for other trains. |
| if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) { |
| is_present = false; // Pretend the component is entirely missing. |
| } |
| |
| let targets = targets |
| .iter() |
| .map(|name| { |
| let target = if is_present { |
| let filename = self |
| .versions |
| .tarball_name(&PkgType::from_component(pkgname), name) |
| .unwrap(); |
| |
| Target::from_compressed_tar(self, &filename) |
| } else { |
| // If the component is not present for this build add it anyway but mark it as |
| // unavailable -- this way rustup won't allow upgrades without --force |
| Target::unavailable() |
| }; |
| (name.to_string(), target) |
| }) |
| .collect(); |
| |
| dst.insert( |
| pkgname.to_string(), |
| Package { |
| version: version_info.version.unwrap_or_default(), |
| git_commit_hash: version_info.git_commit, |
| target: targets, |
| }, |
| ); |
| } |
| |
| fn url(&self, path: &Path) -> String { |
| let file_name = path.file_name().unwrap().to_str().unwrap(); |
| format!("{}/{}/{}", self.s3_address, self.date, file_name) |
| } |
| |
| fn hash(&self, path: &Path) -> String { |
| let sha = t!(Command::new("shasum") |
| .arg("-a") |
| .arg("256") |
| .arg(path.file_name().unwrap()) |
| .current_dir(path.parent().unwrap()) |
| .output()); |
| assert!(sha.status.success()); |
| |
| let filename = path.file_name().unwrap().to_str().unwrap(); |
| let sha256 = self.output.join(format!("{}.sha256", filename)); |
| t!(fs::write(&sha256, &sha.stdout)); |
| |
| let stdout = String::from_utf8_lossy(&sha.stdout); |
| stdout.split_whitespace().next().unwrap().to_string() |
| } |
| |
| fn sign(&self, path: &Path) { |
| if !self.legacy { |
| return; |
| } |
| |
| let filename = path.file_name().unwrap().to_str().unwrap(); |
| let asc = self.output.join(format!("{}.asc", filename)); |
| println!("signing: {:?}", path); |
| let mut cmd = Command::new("gpg"); |
| cmd.arg("--pinentry-mode=loopback") |
| .arg("--no-tty") |
| .arg("--yes") |
| .arg("--batch") |
| .arg("--passphrase-fd") |
| .arg("0") |
| .arg("--personal-digest-preferences") |
| .arg("SHA512") |
| .arg("--armor") |
| .arg("--output") |
| .arg(&asc) |
| .arg("--detach-sign") |
| .arg(path) |
| .stdin(Stdio::piped()); |
| let mut child = t!(cmd.spawn()); |
| t!(child.stdin.take().unwrap().write_all(self.legacy_gpg_passphrase.as_bytes())); |
| assert!(t!(child.wait()).success()); |
| } |
| |
| fn fill_missing_hashes(&self, manifest: &mut Manifest) { |
| // First collect all files that need hashes |
| let mut need_hashes = HashSet::new(); |
| crate::manifest::visit_file_hashes(manifest, |file_hash| { |
| if let FileHash::Missing(path) = file_hash { |
| need_hashes.insert(path.clone()); |
| } |
| }); |
| |
| let collected = Mutex::new(HashMap::new()); |
| let collection_start = Instant::now(); |
| println!( |
| "collecting hashes for {} tarballs across {} threads", |
| need_hashes.len(), |
| rayon::current_num_threads().min(need_hashes.len()), |
| ); |
| need_hashes.par_iter().for_each(|path| match fetch_hash(path) { |
| Ok(hash) => { |
| collected.lock().unwrap().insert(path, hash); |
| } |
| Err(err) => eprintln!("error while fetching the hash for {}: {}", path.display(), err), |
| }); |
| let collected = collected.into_inner().unwrap(); |
| println!("collected {} hashes in {:.2?}", collected.len(), collection_start.elapsed()); |
| |
| crate::manifest::visit_file_hashes(manifest, |file_hash| { |
| if let FileHash::Missing(path) = file_hash { |
| match collected.get(path) { |
| Some(hash) => *file_hash = FileHash::Present(hash.clone()), |
| None => panic!("missing hash for file {}", path.display()), |
| } |
| } |
| }) |
| } |
| |
| fn write_channel_files(&self, channel_name: &str, manifest: &Manifest) { |
| self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml"); |
| self.write(&manifest.date, channel_name, "-date.txt"); |
| self.write( |
| manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(), |
| channel_name, |
| "-git-commit-hash.txt", |
| ); |
| } |
| |
| fn write(&self, contents: &str, channel_name: &str, suffix: &str) { |
| let dst = self.output.join(format!("channel-rust-{}{}", channel_name, suffix)); |
| t!(fs::write(&dst, contents)); |
| if self.legacy { |
| self.hash(&dst); |
| self.sign(&dst); |
| } |
| } |
| } |
| |
| fn fetch_hash(path: &Path) -> Result<String, Box<dyn Error>> { |
| let mut file = BufReader::new(File::open(path)?); |
| let mut sha256 = sha2::Sha256::default(); |
| std::io::copy(&mut file, &mut sha256)?; |
| Ok(hex::encode(sha256.finalize())) |
| } |
