diff --git a/CURATORS.md b/CURATORS.md index 2a6ce088..23f84211 100644 --- a/CURATORS.md +++ b/CURATORS.md @@ -512,6 +512,24 @@ sections, or that are of the form `- package < 0 # $version`. * Please make sure to separate bounds issues from compilation failures/test run failures, as we cannot verify that a package builds or that tests pass without running the build! +#### Diffing snapshots / Inspecting changes + +To diff existing snapshots, or to evaluate changes before they end up +in a snapshot you can run: + +``` +commenter diff-snapshot +``` + +Existing snapshots can be retrieved from https://github.com/commercialhaskell/stackage-snapshots. Preliminary snapshots can be generated by running relevant parts of `automated/build.sh`, at the time of writing: + +``` +TARGET=nightly-2021-01-14 \ # the date doesn't matter + curator update && \ + curator constraints --target $TARGET && \ + curator snapshot-incomplete --target $TARGET && \ + curator snapshot +``` ## Adding new curators diff --git a/etc/commenter/Cargo.lock b/etc/commenter/Cargo.lock index c69a4517..85dbdfe9 100644 --- a/etc/commenter/Cargo.lock +++ b/etc/commenter/Cargo.lock @@ -31,6 +31,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + [[package]] name = "bitflags" version = "1.3.2" @@ -58,9 +64,17 @@ version = "0.2.0" dependencies = [ "lazy-regex", "regex", + "serde", + "serde_yaml", "structopt", ] +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + [[package]] name = "heck" version = "0.3.3" @@ -80,10 +94,20 @@ dependencies = [ ] [[package]] -name = "lazy-regex" -version = "2.2.1" +name = "indexmap" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17d198f91272f6e788a5c0bd5d741cf778da4e5bc761ec67b32d5d3b0db34a54" +checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "lazy-regex" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "919a16773ebf2de27e95fc58460110932e55bb0780e23aa51fa5a6b59c9e2b3d" dependencies = [ "lazy-regex-proc_macros", "once_cell", @@ -92,9 +116,9 @@ dependencies = [ [[package]] name = "lazy-regex-proc_macros" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c12938b1b92cf5be22940527e15b79fd0c7e706e34bc70816f6a72b3484f84e" +checksum = "5fbe6bf0a04af51c07976625d5007e75ed9b8b955befc21c77b3947733496e36" dependencies = [ "proc-macro2", "quote", @@ -110,21 +134,27 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.108" +version = "0.2.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8521a1b57e76b1ec69af7599e75e38e7b7fad6610f037db8c79b127201b5d119" +checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" + +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" [[package]] name = "memchr" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" [[package]] name = "once_cell" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" [[package]] name = "proc-macro-error" @@ -152,18 +182,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.27" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" +checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" dependencies = [ "unicode-xid", ] [[package]] name = "quote" -version = "1.0.9" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +checksum = "47aa80447ce4daf1717500037052af176af5d38cc3e571d9ec1c7353fc10c87d" dependencies = [ "proc-macro2", ] @@ -185,6 +215,44 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "ryu" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" + +[[package]] +name = "serde" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97565067517b60e2d1ea8b268e59ce036de907ac523ad83a0475da04e818989a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed201699328568d8d08208fdd080e3ff594e6c422e438b6705905da01005d537" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_yaml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a521f2940385c165a24ee286aa8599633d162077a54bdcae2a6fd5a7bfa7a0" +dependencies = [ + "indexmap", + "ryu", + "serde", + "yaml-rust", +] + [[package]] name = "strsim" version = "0.8.0" @@ -217,9 +285,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.73" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7" +checksum = "a684ac3dcd8913827e18cd09a68384ee66c1de24157e3c556c9ab16d85695fb7" dependencies = [ "proc-macro2", "quote", @@ -261,9 +329,9 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version_check" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "winapi" @@ -286,3 +354,12 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] diff --git a/etc/commenter/Cargo.toml b/etc/commenter/Cargo.toml index adb15eab..4c57ad43 100644 --- a/etc/commenter/Cargo.toml +++ b/etc/commenter/Cargo.toml @@ -9,4 +9,6 @@ edition = "2018" [dependencies] lazy-regex = "2.2.1" regex = "1.5.4" +serde = { version = "1.0.133", features = ["derive"] } +serde_yaml = "0.8.23" structopt = "0.3.25" diff --git a/etc/commenter/src/lib.rs b/etc/commenter/src/lib.rs index 994f310a..1d02896c 100644 --- a/etc/commenter/src/lib.rs +++ b/etc/commenter/src/lib.rs @@ -1,10 +1,13 @@ use std::collections::{BTreeMap, BTreeSet}; +use std::error::Error; +use std::fmt; use std::fs::File; use std::io::{BufRead, BufReader, LineWriter, Lines, Write}; use std::path::Path; use std::process::Command; use lazy_regex::regex; +use serde::{Deserialize, Deserializer}; pub fn clear() { handle(true, |loc, _lines| match loc { @@ -86,11 +89,9 @@ pub fn outdated() { let latest = latest_version(&package); if version.version() != latest { println!( - "{} mismatch, {}: {}, hackage: {}", - package, - version.tag(), - version.version(), - latest + "{package} mismatch, {tag}: {version}, hackage: {latest}", + tag = version.tag(), + version = version.version(), ); } } @@ -111,18 +112,17 @@ pub fn outdated() { let dependents = dependents .into_iter() .take(max) - .map(|(p, v)| format!("{}-{}", p, v)) + .map(|(p, v)| format!("{p}-{v}")) .collect::>() .join(", "); let dependents = if dependents_stripped > 0 { - format!("{} and {} more", dependents, dependents_stripped) + format!("{dependents} and {dependents_stripped} more") } else { dependents }; println!( - "{} mismatch, snapshot: {}, hackage: {}, dependents: {}", - package, version, latest, dependents, + "{package} mismatch, snapshot: {version}, hackage: {latest}, dependents: {dependents}" ); } } @@ -295,3 +295,124 @@ where let file = File::open(filename).unwrap(); BufReader::new(file).lines() } + +#[derive(Deserialize)] +struct SnapshotYaml { + // flags: BTreeMap>, + // publish_time + packages: Vec, + // hidden + // resolver +} + +#[derive(Deserialize)] +struct SnapshotPackage { + hackage: PackageWithVersionAndSha, + // pantry-tree +} + +#[derive(PartialOrd, Ord, PartialEq, Eq, Clone)] +struct PackageName(String); + +impl fmt::Display for PackageName { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +#[derive(Clone, PartialOrd, Ord, PartialEq, Eq)] +struct Version(String); + +impl fmt::Display for Version { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +// zstd-0.1.3.0@sha256:4c0a372251068eb6086b8c3a0a9f347488f08b570a7705844ffeb2c720c97223,3723 +struct PackageWithVersionAndSha { + name: PackageName, + version: Version, +} + +impl<'de> serde::Deserialize<'de> for PackageWithVersionAndSha { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s: String = String::deserialize(deserializer)?; + let r = regex!(r#"^(.+?)-([.\d]+)@sha256:[\da-z]+,\d+$"#); + if let Some(caps) = r.captures(&s) { + let name = PackageName(caps.get(1).unwrap().as_str().to_owned()); + let version = Version(caps.get(2).unwrap().as_str().to_owned()); + Ok(Self { name, version }) + } else { + Err(serde::de::Error::invalid_value( + serde::de::Unexpected::Other(&s), + &"Invalid PackageVersionWithSha", + )) + } + } +} + +fn yaml_from_file>(path: P) -> Result> +where + A: for<'de> Deserialize<'de>, +{ + let file = File::open(path)?; + let reader = BufReader::new(file); + let u = serde_yaml::from_reader(reader)?; + Ok(u) +} + +struct Snapshot { + packages: BTreeMap>, +} + +#[derive(Clone, Copy)] +enum Diff { + Left(A), + Right(A), + Both(A, A), +} + +fn to_diff(a: SnapshotYaml, b: SnapshotYaml) -> Snapshot { + let mut packages = BTreeMap::new(); + for s in a.packages { + let package = s.hackage; + packages.insert(package.name, Diff::Left(package.version)); + } + for s in b.packages { + let package = s.hackage; + let name = package.name; + let version = package.version; + if let Some(a) = packages.remove(&name) { + match a { + Diff::Left(a) => { + if a == version { + packages.remove(&name); + } else { + packages.insert(name, Diff::Both(a, version)); + } + } + _ => unreachable!(), + } + } else { + packages.insert(name, Diff::Right(version)); + } + } + + Snapshot { packages } +} + +pub fn diff_snapshot(a: String, b: String) { + let diff = to_diff(yaml_from_file(a).unwrap(), yaml_from_file(b).unwrap()); + for (name, diff) in diff.packages { + let s = match diff { + Diff::Left(a) => format!("- {name}-{a}"), + Diff::Right(b) => format!("+ {name}-{b}"), + Diff::Both(a, b) => format!("~ {name}-{a} -> {b}"), + }; + println!("{s}"); + } +} diff --git a/etc/commenter/src/main.rs b/etc/commenter/src/main.rs index 07c1662f..25db2bbf 100644 --- a/etc/commenter/src/main.rs +++ b/etc/commenter/src/main.rs @@ -22,25 +22,19 @@ enum Opt { Clear, Add, Outdated, + DiffSnapshot { a: String, b: String }, } fn main() { let opt = Opt::from_args(); match opt { - Opt::Clear => clear(), + Opt::Clear => commenter::clear(), Opt::Add => add(), - Opt::Outdated => outdated(), + Opt::Outdated => commenter::outdated(), + Opt::DiffSnapshot { a, b } => commenter::diff_snapshot(a, b), } } -fn clear() { - commenter::clear(); -} - -fn outdated() { - commenter::outdated(); -} - fn add() { let mut lib_exes: H = Default::default(); let mut tests: H = Default::default(); @@ -64,7 +58,7 @@ fn add() { } else if line == "curator: Snapshot dependency graph contains errors:" { process_line = true; } else if !process_line { - println!("[INFO] {}", line); + println!("[INFO] {line}"); } else if let Some(cap) = package.captures(&line) { let root = last_header.clone().unwrap(); let package = cap.name("package").unwrap().as_str(); @@ -100,7 +94,7 @@ fn add() { for (header, packages) in lib_exes { for (package, version, component) in packages { let s = printer(" ", &package, true, &version, &component, &header); - println!("{}", s); + println!("{s}"); auto_lib_exes.push(s); } } @@ -111,7 +105,7 @@ fn add() { for (header, packages) in tests { for (package, version, component) in packages { let s = printer(" ", &package, false, &version, &component, &header); - println!("{}", s); + println!("{s}"); auto_tests.push(s); } } @@ -122,7 +116,7 @@ fn add() { for (header, packages) in benches { for (package, version, component) in packages { let s = printer(" ", &package, false, &version, &component, &header); - println!("{}", s); + println!("{s}"); auto_benches.push(s); } } @@ -148,11 +142,6 @@ fn printer( let lt0 = if lt0 { " < 0" } else { "" }; format!( "{indent}- {package}{lt0} # tried {package}-{version}, but its *{component}* {cause}", - indent = indent, - package = package, - lt0 = lt0, - version = version, - component = component, cause = match header { Header::Versioned { package, version } => format!( "does not support: {package}-{version}",