Merge remote-tracking branch 'origin/master' into ghc-9.2

CURATORS.md

@@ -206,12 +206,13 @@ If a new package fails to build because of missing system libraries we often ask
 
 ### Upgrading GHC version
 The Dockerfile contains information on which GHC versions should be used. You
-can modify it and push it to Github to trigger a DockerHub build. The nightly
+can modify it and push it to Github to trigger a build. The nightly
 branch is used for nightlies. For LTSes, we use the ltsX branch, where X is the
 major version number (e.g., lts3 for lts-3.\*).
 
-Note that when starting a new LTS major release, you'll need to modify Docker
+Update the `Win32` version in `build-constraints.yaml` under "GHC upper bounds".
-Hub to create a new Docker tag for the relevant branch name.
+
+Note that when starting a new LTS major release, you'll need to modify `.github/workflows/image.yml` to add a new lts branch.
 
 Update `GHCVER` in `Dockerfile`. (This env var automatically gets passed to `debian-bootstrap.sh`.)
 
@@ -241,7 +242,7 @@ docker rm $(docker ps -a -q)
 docker rmi $(docker images -q)
 ```
 
-but `docker pull commercialhaskell/stackage:nightly` can also be run instead just to update the nightly image say.
+but `docker pull ghcr.io/commercialhaskell/stackage/build:nightly` can also be run instead just to update the nightly image say.
 
 For a new GHC version you should also delete the ~~cache~~ .stack-work snapshot install directories on the stackage-build server to
 ~~force all packages to be rebuilt~~ clear up some space. See: [issue#746](https://github.com/commercialhaskell/stackage/issues/746). Eg:
@@ -513,6 +514,24 @@ sections, or that are of the form `- package < 0 # $version`.
 
 * Please make sure to separate bounds issues from compilation failures/test run failures, as we cannot verify that a package builds or that tests pass without running the build!
 
+#### Diffing snapshots / Inspecting changes
+
+To diff existing snapshots, or to evaluate changes before they end up
+in a snapshot you can run:
+
+```
+commenter diff-snapshot <old-snapshot.yaml> <new-snapshot.yaml>
+```
+
+Existing snapshots can be retrieved from https://github.com/commercialhaskell/stackage-snapshots. Preliminary snapshots can be generated by running relevant parts of `automated/build.sh`, at the time of writing:
+
+```
+TARGET=nightly-2021-01-14 \ # the date doesn't matter
+  curator update && \
+  curator constraints --target $TARGET && \
+  curator snapshot-incomplete --target $TARGET && \
+  curator snapshot
+```
 
 ## Adding new curators
 
@@ -555,8 +574,7 @@ Every 3-6 months, we make a new major release of LTS. The procedure we follow fo
    relaxed upper bounds. There will likely be some hard decisions to be made
    regarding relaxing a bound versus keeping more packages. All of these changes
    occur on master and affect nightly.
-4. Once the estimated date hits, push a new `ltsXX` branch and trigger Docker
+4. Once the estimated date hits, push a new `ltsXX` and wait for the docker image build.
-   Hub to build a Docker image for the new release.
 5. Run the build procedure for the new LTS release.
 6. After the LTS build completes, more aggressively prune upper bounds from
    `build-constraints.yaml`.

Dockerfile

@@ -2,7 +2,7 @@ FROM fpco/pid1:18.04
 
 ENV HOME /home/stackage
 ENV LANG en_US.UTF-8
-ENV GHCVER 9.0.1
+ENV GHCVER 9.0.2
 
 # NOTE: also update debian-bootstrap.sh when cuda version changes
 ENV PATH /home/stackage/.stack/programs/x86_64-linux/ghc-$GHCVER/bin:/usr/local/cuda-10.0/bin:/usr/sbin:/usr/bin:/sbin:/bin

etc/commenter/Cargo.lock

@@ -31,6 +31,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "autocfg"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -58,9 +64,17 @@ version = "0.2.0"
 dependencies = [
  "lazy-regex",
  "regex",
+ "serde",
+ "serde_yaml",
  "structopt",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+
 [[package]]
 name = "heck"
 version = "0.3.3"
@@ -80,10 +94,20 @@ dependencies = [
 ]
 
 [[package]]
-name = "lazy-regex"
+name = "indexmap"
-version = "2.2.1"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17d198f91272f6e788a5c0bd5d741cf778da4e5bc761ec67b32d5d3b0db34a54"
+checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+]
+
+[[package]]
+name = "lazy-regex"
+version = "2.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "919a16773ebf2de27e95fc58460110932e55bb0780e23aa51fa5a6b59c9e2b3d"
 dependencies = [
  "lazy-regex-proc_macros",
  "once_cell",
@@ -92,9 +116,9 @@ dependencies = [
 
 [[package]]
 name = "lazy-regex-proc_macros"
-version = "2.2.1"
+version = "2.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c12938b1b92cf5be22940527e15b79fd0c7e706e34bc70816f6a72b3484f84e"
+checksum = "5fbe6bf0a04af51c07976625d5007e75ed9b8b955befc21c77b3947733496e36"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -110,21 +134,27 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.108"
+version = "0.2.112"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8521a1b57e76b1ec69af7599e75e38e7b7fad6610f037db8c79b127201b5d119"
+checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125"
+
+[[package]]
+name = "linked-hash-map"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
 
 [[package]]
 name = "memchr"
-version = "2.4.0"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
+checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
 
 [[package]]
 name = "once_cell"
-version = "1.8.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56"
+checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5"
 
 [[package]]
 name = "proc-macro-error"
@@ -152,18 +182,18 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.27"
+version = "1.0.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
+checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029"
 dependencies = [
  "unicode-xid",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.9"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
+checksum = "47aa80447ce4daf1717500037052af176af5d38cc3e571d9ec1c7353fc10c87d"
 dependencies = [
  "proc-macro2",
 ]
@@ -185,6 +215,44 @@ version = "0.6.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
 
+[[package]]
+name = "ryu"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
+
+[[package]]
+name = "serde"
+version = "1.0.133"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97565067517b60e2d1ea8b268e59ce036de907ac523ad83a0475da04e818989a"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.133"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed201699328568d8d08208fdd080e3ff594e6c422e438b6705905da01005d537"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_yaml"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a521f2940385c165a24ee286aa8599633d162077a54bdcae2a6fd5a7bfa7a0"
+dependencies = [
+ "indexmap",
+ "ryu",
+ "serde",
+ "yaml-rust",
+]
+
 [[package]]
 name = "strsim"
 version = "0.8.0"
@@ -217,9 +285,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.73"
+version = "1.0.85"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7"
+checksum = "a684ac3dcd8913827e18cd09a68384ee66c1de24157e3c556c9ab16d85695fb7"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -261,9 +329,9 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
 
 [[package]]
 name = "version_check"
-version = "0.9.3"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "winapi"
@@ -286,3 +354,12 @@ name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "yaml-rust"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
+dependencies = [
+ "linked-hash-map",
+]

etc/commenter/Cargo.toml

@@ -9,4 +9,6 @@ edition = "2018"
 [dependencies]
 lazy-regex = "2.2.1"
 regex = "1.5.4"
+serde = { version = "1.0.133", features = ["derive"] }
+serde_yaml = "0.8.23"
 structopt = "0.3.25"

etc/commenter/src/lib.rs

@@ -1,10 +1,13 @@
 use std::collections::{BTreeMap, BTreeSet};
+use std::error::Error;
+use std::fmt;
 use std::fs::File;
 use std::io::{BufRead, BufReader, LineWriter, Lines, Write};
 use std::path::Path;
 use std::process::Command;
 
 use lazy_regex::regex;
+use serde::{Deserialize, Deserializer};
 
 pub fn clear() {
     handle(true, |loc, _lines| match loc {
@@ -41,20 +44,20 @@ impl VersionTag {
 
     fn version(&self) -> &str {
         match self {
-            VersionTag::Manual(s) => &s,
+            VersionTag::Manual(s) => s,
-            VersionTag::Auto(s) => &s,
+            VersionTag::Auto(s) => s,
         }
     }
 }
 
 pub fn outdated() {
     let mut all: Vec<String> = vec![];
-    let disabled = handle(false, |_loc, lines| {
+    let versioned = handle(false, |_loc, lines| {
         all.extend(lines);
         vec![]
     });
     let mut map: BTreeMap<String, VersionTag> = BTreeMap::new();
-    for DisabledPackage { package, version } in disabled {
+    for VersionedPackage { package, version } in versioned {
         map.insert(package, VersionTag::Manual(version));
     }
     let mut support: BTreeMap<(String, String), BTreeSet<(String, String)>> = BTreeMap::new();
@@ -86,11 +89,9 @@ pub fn outdated() {
         let latest = latest_version(&package);
         if version.version() != latest {
             println!(
-                "{} mismatch, {}: {}, hackage: {}",
+                "{package} mismatch, {tag}: {version}, hackage: {latest}",
-                package,
+                tag = version.tag(),
-                version.tag(),
+                version = version.version(),
-                version.version(),
-                latest
             );
         }
     }
@@ -107,22 +108,21 @@ pub fn outdated() {
         let latest = latest_version(&package);
         if version != latest {
             let max = 3;
-            let dependents_stripped = dependents.len().checked_sub(max).unwrap_or(0);
+            let dependents_stripped = dependents.len().saturating_sub(max);
             let dependents = dependents
                 .into_iter()
                 .take(max)
-                .map(|(p, v)| format!("{}-{}", p, v))
+                .map(|(p, v)| format!("{p}-{v}"))
                 .collect::<Vec<String>>()
                 .join(", ");
             let dependents = if dependents_stripped > 0 {
-                format!("{} and {} more", dependents, dependents_stripped)
+                format!("{dependents} and {dependents_stripped} more")
             } else {
                 dependents
             };
 
             println!(
-                "{} mismatch, snapshot: {}, hackage: {}, dependents: {}",
+                "{package} mismatch, snapshot: {version}, hackage: {latest}, dependents: {dependents}"
-                package, version, latest, dependents,
             );
         }
     }
@@ -139,9 +139,11 @@ fn is_boot(package: &str) -> bool {
         "filepath",
         "deepseq",
         "ghc",
+        "ghc-bignum",
         "ghc-boot",
         "ghc-boot-th",
         "ghc-prim",
+        "ghc-lib-parser", // not a boot lib, but tied to the GHC version.
         "integer-gmp",
         "process",
         "stm",
@@ -175,34 +177,38 @@ enum State {
     Done,
 }
 
-struct DisabledPackage {
+struct VersionedPackage {
     package: String,
     version: String,
 }
 
-fn parse_disabled_package(s: &str) -> Option<DisabledPackage> {
+fn parse_versioned_package(s: &str) -> Option<VersionedPackage> {
     if let Some(caps) = regex!(r#"- *([^ ]+) < *0 *# *([\d.]+)"#).captures(s) {
         let package = caps.get(1).unwrap().as_str().to_owned();
         let version = caps.get(2).unwrap().as_str().to_owned();
-        Some(DisabledPackage { package, version })
+        Some(VersionedPackage { package, version })
+    } else if let Some(caps) = regex!(r#"- *([^ ]+) *# *([\d.]+)"#).captures(s) {
+        let package = caps.get(1).unwrap().as_str().to_owned();
+        let version = caps.get(2).unwrap().as_str().to_owned();
+        Some(VersionedPackage { package, version })
     } else {
         None
     }
 }
 
-fn handle<F>(write: bool, mut f: F) -> Vec<DisabledPackage>
+fn handle<F>(write: bool, mut f: F) -> Vec<VersionedPackage>
 where
     F: FnMut(Location, Vec<String>) -> Vec<String>,
 {
     let path = "build-constraints.yaml";
     let mut new_lines: Vec<String> = vec![];
-    let mut disabled_packages: Vec<DisabledPackage> = vec![];
+    let mut versioned_packages: Vec<VersionedPackage> = vec![];
 
     let mut state = State::LookingForLibBounds;
     let mut buf = vec![];
     for line in read_lines(path).map(|s| s.unwrap()) {
-        if let Some(disabled_package) = parse_disabled_package(&line) {
+        if let Some(versioned_package) = parse_versioned_package(&line) {
-            disabled_packages.push(disabled_package);
+            versioned_packages.push(versioned_package);
         }
 
         match state {
@@ -273,7 +279,7 @@ where
         file.flush().unwrap();
     }
 
-    disabled_packages
+    versioned_packages
 }
 
 enum Location {
@@ -289,3 +295,124 @@ where
     let file = File::open(filename).unwrap();
     BufReader::new(file).lines()
 }
+
+#[derive(Deserialize)]
+struct SnapshotYaml {
+    // flags: BTreeMap<PackageName, BTreeMap<PackageFlag, bool>>,
+    // publish_time
+    packages: Vec<SnapshotPackage>,
+    // hidden
+    // resolver
+}
+
+#[derive(Deserialize)]
+struct SnapshotPackage {
+    hackage: PackageWithVersionAndSha,
+    // pantry-tree
+}
+
+#[derive(PartialOrd, Ord, PartialEq, Eq, Clone)]
+struct PackageName(String);
+
+impl fmt::Display for PackageName {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+#[derive(Clone, PartialOrd, Ord, PartialEq, Eq)]
+struct Version(String);
+
+impl fmt::Display for Version {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+// zstd-0.1.3.0@sha256:4c0a372251068eb6086b8c3a0a9f347488f08b570a7705844ffeb2c720c97223,3723
+struct PackageWithVersionAndSha {
+    name: PackageName,
+    version: Version,
+}
+
+impl<'de> serde::Deserialize<'de> for PackageWithVersionAndSha {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s: String = String::deserialize(deserializer)?;
+        let r = regex!(r#"^(.+?)-([.\d]+)@sha256:[\da-z]+,\d+$"#);
+        if let Some(caps) = r.captures(&s) {
+            let name = PackageName(caps.get(1).unwrap().as_str().to_owned());
+            let version = Version(caps.get(2).unwrap().as_str().to_owned());
+            Ok(Self { name, version })
+        } else {
+            Err(serde::de::Error::invalid_value(
+                serde::de::Unexpected::Other(&s),
+                &"Invalid PackageVersionWithSha",
+            ))
+        }
+    }
+}
+
+fn yaml_from_file<A, P: AsRef<Path>>(path: P) -> Result<A, Box<dyn Error>>
+where
+    A: for<'de> Deserialize<'de>,
+{
+    let file = File::open(path)?;
+    let reader = BufReader::new(file);
+    let u = serde_yaml::from_reader(reader)?;
+    Ok(u)
+}
+
+struct Snapshot {
+    packages: BTreeMap<PackageName, Diff<Version>>,
+}
+
+#[derive(Clone, Copy)]
+enum Diff<A> {
+    Left(A),
+    Right(A),
+    Both(A, A),
+}
+
+fn to_diff(a: SnapshotYaml, b: SnapshotYaml) -> Snapshot {
+    let mut packages = BTreeMap::new();
+    for s in a.packages {
+        let package = s.hackage;
+        packages.insert(package.name, Diff::Left(package.version));
+    }
+    for s in b.packages {
+        let package = s.hackage;
+        let name = package.name;
+        let version = package.version;
+        if let Some(a) = packages.remove(&name) {
+            match a {
+                Diff::Left(a) => {
+                    if a == version {
+                        packages.remove(&name);
+                    } else {
+                        packages.insert(name, Diff::Both(a, version));
+                    }
+                }
+                _ => unreachable!(),
+            }
+        } else {
+            packages.insert(name, Diff::Right(version));
+        }
+    }
+
+    Snapshot { packages }
+}
+
+pub fn diff_snapshot(a: String, b: String) {
+    let diff = to_diff(yaml_from_file(a).unwrap(), yaml_from_file(b).unwrap());
+    for (name, diff) in diff.packages {
+        let s = match diff {
+            Diff::Left(a) => format!("- {name}-{a}"),
+            Diff::Right(b) => format!("+ {name}-{b}"),
+            Diff::Both(a, b) => format!("~ {name}-{a} -> {b}"),
+        };
+        println!("{s}");
+    }
+}

etc/commenter/src/main.rs

@@ -22,25 +22,19 @@ enum Opt {
     Clear,
     Add,
     Outdated,
+    DiffSnapshot { a: String, b: String },
 }
 
 fn main() {
     let opt = Opt::from_args();
     match opt {
-        Opt::Clear => clear(),
+        Opt::Clear => commenter::clear(),
         Opt::Add => add(),
-        Opt::Outdated => outdated(),
+        Opt::Outdated => commenter::outdated(),
+        Opt::DiffSnapshot { a, b } => commenter::diff_snapshot(a, b),
     }
 }
 
-fn clear() {
-    commenter::clear();
-}
-
-fn outdated() {
-    commenter::outdated();
-}
-
 fn add() {
     let mut lib_exes: H = Default::default();
     let mut tests: H = Default::default();
@@ -64,7 +58,7 @@ fn add() {
         } else if line == "curator: Snapshot dependency graph contains errors:" {
             process_line = true;
         } else if !process_line {
-            println!("[INFO] {}", line);
+            println!("[INFO] {line}");
         } else if let Some(cap) = package.captures(&line) {
             let root = last_header.clone().unwrap();
             let package = cap.name("package").unwrap().as_str();
@@ -100,7 +94,7 @@ fn add() {
     for (header, packages) in lib_exes {
         for (package, version, component) in packages {
             let s = printer("        ", &package, true, &version, &component, &header);
-            println!("{}", s);
+            println!("{s}");
             auto_lib_exes.push(s);
         }
     }
@@ -111,7 +105,7 @@ fn add() {
     for (header, packages) in tests {
         for (package, version, component) in packages {
             let s = printer("    ", &package, false, &version, &component, &header);
-            println!("{}", s);
+            println!("{s}");
             auto_tests.push(s);
         }
     }
@@ -122,7 +116,7 @@ fn add() {
     for (header, packages) in benches {
         for (package, version, component) in packages {
             let s = printer("    ", &package, false, &version, &component, &header);
-            println!("{}", s);
+            println!("{s}");
             auto_benches.push(s);
         }
     }
@@ -148,11 +142,6 @@ fn printer(
     let lt0 = if lt0 { " < 0" } else { "" };
     format!(
         "{indent}- {package}{lt0} # tried {package}-{version}, but its *{component}* {cause}",
-        indent = indent,
-        package = package,
-        lt0 = lt0,
-        version = version,
-        component = component,
         cause = match header {
             Header::Versioned { package, version } => format!(
                 "does not support: {package}-{version}",