From 68b6addb8388733abc2da74fd133264ae7bc9a7f Mon Sep 17 00:00:00 2001 From: Wim Looman Date: Tue, 10 Nov 2020 22:38:58 +0100 Subject: [PATCH 1/2] :ferrisSweat: --- Cargo.lock | 41 ++++++++-- Cargo.toml | 1 + crates/metadata/lib.rs | 48 ++++++----- src/docbuilder/rustwide_builder.rs | 127 +++++++++++++++++++++++++++-- 4 files changed, 181 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd65ffba9..17ecc9618 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -239,6 +239,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "716960a18f978640f25101b5cbf1c6f6b0d3192fab36a2d98ca96f0ecbe41010" +[[package]] +name = "cargo_metadata" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5a5f7b42f606b7f23674f6f4d877628350682bc40687d3fae65679a58d55345" +dependencies = [ + "semver 0.11.0", + "serde", + "serde_json", +] + [[package]] name = "cast" version = "0.2.3" @@ -670,6 +681,7 @@ dependencies = [ "backtrace", "badge", "base64 0.12.1", + "cargo_metadata", "chrono", "comrak", "crates-index", @@ -708,7 +720,7 @@ dependencies = [ "sass-rs", "schemamama", "schemamama_postgres", - "semver", + "semver 0.9.0", "serde", "serde_json", "slug", @@ -2856,7 +2868,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver", + "semver 0.9.0", ] [[package]] @@ -3055,7 +3067,17 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser", + "semver-parser 0.7.0", + "serde", +] + +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser 0.10.1", "serde", ] @@ -3065,6 +3087,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "semver-parser" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ef146c2ad5e5f4b037cd6ce2ebb775401729b19a82040c1beac9d36c7d1428" +dependencies = [ + "pest", +] + [[package]] name = "serde" version = "1.0.110" @@ -3097,9 +3128,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.53" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993948e75b189211a9b31a7528f950c6adc21f9720b6438ff80a7fa2f864cea2" +checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" dependencies = [ "itoa", "ryu", diff --git a/Cargo.toml b/Cargo.toml index 6bee0222e..1228c0f7d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,6 +85,7 @@ notify = "4.0.15" # Date and Time utilities chrono = { version = "0.4.11", features = ["serde"] } time = "0.1" # TODO: Remove once `iron` is removed +cargo_metadata = "0.12.0" [dependencies.postgres] version = "0.17" diff --git a/crates/metadata/lib.rs b/crates/metadata/lib.rs index 8ceeee651..50ff26ca6 100644 --- a/crates/metadata/lib.rs +++ b/crates/metadata/lib.rs @@ -216,6 +216,31 @@ impl Metadata { } } + /// Return the cargo specific arguments requested by the metadata + pub fn just_cargo_args(&self) -> Vec { + let mut cargo_args = vec![]; + + if let Some(features) = &self.features { + cargo_args.push("--features".into()); + cargo_args.push(features.join(" ")); + } + + if self.all_features { + cargo_args.push("--all-features".into()); + } + + if self.no_default_features { + cargo_args.push("--no-default-features".into()); + } + + cargo_args + } + + /// Return the rustc specific arguments requested by the metadata + pub fn just_rustc_args(&self) -> Vec { + self.rustc_args.clone() + } + /// Return the arguments that should be passed to `cargo`. /// /// This will always include `rustdoc --lib --`. @@ -231,28 +256,7 @@ impl Metadata { pub fn cargo_args(&self, additional_args: &[String], rustdoc_args: &[String]) -> Vec { let mut cargo_args: Vec = vec!["rustdoc".into(), "--lib".into()]; - if let Some(features) = &self.features { - cargo_args.push("--features".into()); - cargo_args.push(features.join(" ")); - } - - if self.all_features { - cargo_args.push("--all-features".into()); - } - - if self.no_default_features { - cargo_args.push("--no-default-features".into()); - } - - // Pass `RUSTFLAGS` using `cargo --config`, which handles whitespace correctly. - if !self.rustc_args.is_empty() { - cargo_args.push("-Z".into()); - cargo_args.push("unstable-options".into()); - cargo_args.push("--config".into()); - let rustflags = - toml::to_string(&self.rustc_args).expect("serializing a string should never fail"); - cargo_args.push(format!("build.rustflags={}", rustflags)); - } + cargo_args.extend(self.just_cargo_args()); cargo_args.extend(additional_args.iter().map(|s| s.to_owned())); cargo_args.push("--".into()); diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index ce53f771e..c35a12549 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -287,6 +287,16 @@ impl RustwideBuilder { self.build_package(&package.name, &package.version, PackageKind::Local(path)) } + fn ensure_target(&self, target: &str) -> Result<()> { + // If the explicit target is not a tier one target, we need to install it. + if !docsrs_metadata::DEFAULT_TARGETS.contains(&target) { + // This is a no-op if the target is already installed. + self.toolchain.add_target(&self.workspace, target)?; + } + + Ok(()) + } + pub fn build_package( &mut self, name: &str, @@ -537,9 +547,18 @@ impl RustwideBuilder { storage.set_max_size(limits.max_log_size()); let successful = logging::capture(&storage, || { - self.prepare_command(build, target, metadata, limits, rustdoc_flags) - .and_then(|command| command.run().map_err(failure::Error::from)) - .is_ok() + (|| { + self.ensure_target(target)?; + let extra_flags = self + .run_check(build, target, metadata, limits) + .context("Running check")?; + rustdoc_flags.extend(extra_flags); + self.prepare_command(build, target, metadata, limits, rustdoc_flags)? + .run()?; + Result::Ok(()) + })() + .map_err(|err| log::error!("failure during build: {:?}", err)) + .is_ok() }); let doc_coverage = if successful { self.get_coverage(target, build, metadata, limits)? @@ -573,6 +592,102 @@ impl RustwideBuilder { }) } + /// Runs `cargo check` for the build, returning a list of extra rustdoc flags gathered during it + fn run_check( + &self, + build: &Build, + target: &str, + docsrs_metadata: &Metadata, + limits: &Limits, + ) -> Result> { + let mut command = build + .cargo() + .log_output(false) + .timeout(Some(limits.timeout())) + .no_output_timeout(None) + .args(&["metadata", "--format-version=1"]) + .args(&docsrs_metadata.just_cargo_args()); + + for (key, val) in docsrs_metadata.environment_variables() { + command = command.env(key, val); + } + + let output = command.run_capture()?; + let metadata: cargo_metadata::Metadata = serde_json::from_str( + &output + .stdout_lines() + .iter() + .map(|s| &**s) + .collect::(), + )?; + + let mut rustc_args = docsrs_metadata.just_rustc_args(); + rustc_args.push("--cap-lints=warn".into()); + + let mut command = build + .cargo() + .timeout(Some(limits.timeout())) + .no_output_timeout(None) + .args(&["check"]) + .args(&docsrs_metadata.just_cargo_args()) + .args(&[ + "--message-format=json", + "-Zunstable-options", + &format!("--config=build.rustflags={}", toml::to_string(&rustc_args)?), + ]); + + for (key, val) in docsrs_metadata.environment_variables() { + command = command.env(key, val); + } + + if let Some(cpu_limit) = self.config.build_cpu_limit { + command = command.args(&["--jobs", &cpu_limit.to_string()]); + } + + if target != HOST_TARGET { + command = command.args(&["--target", target]); + } + + let mut flags = Vec::new(); + + command + .process_lines(&mut |line, action| { + let _ = (|| { + // mix of stderr and stdout, if parsing fails assume it's stderr that we don't care + // about + if let Ok(msg) = serde_json::from_str(line) { + action.remove_line(); // not important to see these in logs + if let cargo_metadata::Message::CompilerArtifact(msg) = msg { + let package = metadata + .packages + .iter() + .find(|p| p.id == msg.package_id) + .ok_or_else(|| { + failure::err_msg("artifact for non-existent package") + })?; + // TODO: Filter out dependencies that can't contribute to docs (build-deps, + // proc-macros) + for filename in &msg.filenames { + flags.push(format!( + "--extern-html-root-url={}=https://docs.rs/{}/{}", + filename.to_str().ok_or_else(|| failure::err_msg( + "non-utf-8 artifact name" + ))?, + package.name, + package.version + )); + } + } + } + Result::Ok(()) + })() + .map_err(|e| log::error!("Ignoring error during check: {:?}", e)); + }) + .run()?; + + Ok(flags) + } + fn prepare_command<'ws, 'pl>( &self, build: &'ws Build, @@ -581,12 +696,6 @@ impl RustwideBuilder { limits: &Limits, mut rustdoc_flags_extras: Vec, ) -> Result> { - // If the explicit target is not a tier one target, we need to install it. - if !docsrs_metadata::DEFAULT_TARGETS.contains(&target) { - // This is a no-op if the target is already installed. - self.toolchain.add_target(&self.workspace, target)?; - } - // Add docs.rs specific arguments let mut cargo_args = Vec::new(); if let Some(cpu_limit) = self.config.build_cpu_limit { From 2a5ae92ac38a1820d9069528cb693d0af309226d Mon Sep 17 00:00:00 2001 From: Wim Looman Date: Mon, 16 Nov 2020 16:13:00 +0100 Subject: [PATCH 2/2] wip --- crates/metadata/lib.rs | 9 +++++++++ src/docbuilder/limits.rs | 2 +- src/docbuilder/rustwide_builder.rs | 3 +-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/metadata/lib.rs b/crates/metadata/lib.rs index 50ff26ca6..990bd3860 100644 --- a/crates/metadata/lib.rs +++ b/crates/metadata/lib.rs @@ -258,6 +258,15 @@ impl Metadata { cargo_args.extend(self.just_cargo_args()); + if !self.rustc_args.is_empty() { + cargo_args.push("-Z".into()); + cargo_args.push("unstable-options".into()); + cargo_args.push("--config".into()); + let rustflags = + toml::to_string(&self.rustc_args).expect("serializing a string should never fail"); + cargo_args.push(format!("build.rustflags={}", rustflags)); + } + cargo_args.extend(additional_args.iter().map(|s| s.to_owned())); cargo_args.push("--".into()); cargo_args.extend_from_slice(&self.rustdoc_args); diff --git a/src/docbuilder/limits.rs b/src/docbuilder/limits.rs index 6155567a1..8727b32b9 100644 --- a/src/docbuilder/limits.rs +++ b/src/docbuilder/limits.rs @@ -19,7 +19,7 @@ impl Default for Limits { timeout: Duration::from_secs(15 * 60), // 15 minutes targets: 10, networking: false, - max_log_size: 100 * 1024, // 100 KB + max_log_size: 10000 * 1024, // 100 KB } } } diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index c35a12549..dd4e9b35d 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -621,8 +621,7 @@ impl RustwideBuilder { .collect::(), )?; - let mut rustc_args = docsrs_metadata.just_rustc_args(); - rustc_args.push("--cap-lints=warn".into()); + let rustc_args = docsrs_metadata.just_rustc_args(); let mut command = build .cargo()