From 7f6003b4593dc3b5d52936d95aa1ad7c323b0341 Mon Sep 17 00:00:00 2001 From: Adam Bergmark Date: Mon, 21 Jun 2021 14:46:45 +0200 Subject: [PATCH] Add `commenter` script to help manage bounds issues --- CURATORS.md | 50 ++++++++++++++ etc/commenter/.gitignore | 3 + etc/commenter/Cargo.lock | 40 ++++++++++++ etc/commenter/Cargo.toml | 10 +++ etc/commenter/README.md | 1 + etc/commenter/commenter | 2 + etc/commenter/src/main.rs | 133 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 239 insertions(+) create mode 100644 etc/commenter/.gitignore create mode 100644 etc/commenter/Cargo.lock create mode 100644 etc/commenter/Cargo.toml create mode 100644 etc/commenter/README.md create mode 100755 etc/commenter/commenter create mode 100644 etc/commenter/src/main.rs diff --git a/CURATORS.md b/CURATORS.md index fa3bf75a..f449d1f9 100644 --- a/CURATORS.md +++ b/CURATORS.md @@ -406,6 +406,56 @@ errors for builds, tests and benchmarks. [tell-me-when-its-released]: https://github.com/commercialhaskell/stackage/blob/master/CURATORS.md#waiting-for-new-releases +### Large scale enabling/disabling of packages + +`etc/commenter` is a binary that semi-automates the translation of +`./check` errors into lines that can be copy pasted into +`build-constraints.yaml`. It can only handle bounds issues, +compilation issues still need to be analyzed manually. + +Example usage: After disabling a few packages you get this curator output: + +``` +ConfigFile (GHC 9 bounds issues, @maintainer) (not present) depended on by: +- [ ] xdg-desktop-entry-0.1.1.1 (-any). @maintainer. Used by: library + + +pipes-misc (GHC 9 bounds issues, @maintainer) (not present) depended on by: +- [ ] pipes-fluid-0.6.0.1 (>=0.5). @handles. Used by: test-suite + + +testing-feat (GHC 9 bounds issues, Grandfathered dependencies) (not present) depended on by: +- [ ] dual-tree-0.2.3.0 (-any). Grandfathered dependencies. @handles. Used by: test-suite +``` + +Copy this into `etc/commenter/comments.txt` and run `./commenter` (currently requires Rust) + +You will get this output: +``` +LIBS + EXES + + - xdg-desktop-entry < 0 # tried xdg-desktop-entry-0.1.1.1, but its *library* requires the disabled package: ConfigFile + +TESTS + + - dual-tree # tried dual-tree-0.2.3.0, but its *test-suite* requires the disabled package: testing-feat + - pipes-fluid # tried pipes-fluid-0.6.0.1, but its *test-suite* requires the disabled package: pipes-misc +``` + +The lines under LIBS+EXES should be pasted in the shared section (currently called "GHC 9 bounds issues"). + +TESTS have a similar section under `skipped-tests`, and BENCHMARKS under `skipped-benchmarks`. + +#### Re-enabling + +We can periodically remove all packages under the bounds sections and then re-run the disabling flow above until we get a clean plan. This way we will automatically pick up packages that have been fixed. + +#### Notes + +* Please keep these lists sorted as the diffs will be much cleaner when we re-enable packages and re-run this flow +* Please make sure to separate bounds issues from compilation failures/test run failures, as we cannot verify that a package builds or that tests pass without running the build! + + ## Adding new curators 1. Add public ssh key to `~/.ssh/authorized_keys` on build server diff --git a/etc/commenter/.gitignore b/etc/commenter/.gitignore new file mode 100644 index 00000000..72b58cc3 --- /dev/null +++ b/etc/commenter/.gitignore @@ -0,0 +1,3 @@ +/target +/comments.txt +/out.txt diff --git a/etc/commenter/Cargo.lock b/etc/commenter/Cargo.lock new file mode 100644 index 00000000..697797d2 --- /dev/null +++ b/etc/commenter/Cargo.lock @@ -0,0 +1,40 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "commenter" +version = "0.1.0" +dependencies = [ + "regex", +] + +[[package]] +name = "memchr" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" + +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" diff --git a/etc/commenter/Cargo.toml b/etc/commenter/Cargo.toml new file mode 100644 index 00000000..12ef031f --- /dev/null +++ b/etc/commenter/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "commenter" +version = "0.1.0" +authors = ["Adam Bergmark "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +regex = "1.5.4" diff --git a/etc/commenter/README.md b/etc/commenter/README.md new file mode 100644 index 00000000..6e505cc8 --- /dev/null +++ b/etc/commenter/README.md @@ -0,0 +1 @@ +Helps automate mass-disabling of packages in Stackage's build-constraint.yaml. diff --git a/etc/commenter/commenter b/etc/commenter/commenter new file mode 100755 index 00000000..2380d69d --- /dev/null +++ b/etc/commenter/commenter @@ -0,0 +1,2 @@ +#!/bin/bash +cargo run -- $@ diff --git a/etc/commenter/src/main.rs b/etc/commenter/src/main.rs new file mode 100644 index 00000000..66ff23c9 --- /dev/null +++ b/etc/commenter/src/main.rs @@ -0,0 +1,133 @@ +use std::collections::HashMap; +use std::fs::File; +use std::io::{self, BufRead}; +use std::path::Path; + +use regex::Regex; + +type H = HashMap>; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +enum Header { + Versioned { package: String, version: String }, + Missing { package: String }, +} + +fn main() { + let mut lib_exes: H = Default::default(); + let mut tests: H = Default::default(); + let mut benches: H = Default::default(); + let mut last_header: Option
= None; + let empty = Regex::new(r#"^\s*$"#).unwrap(); + let header_versioned = + Regex::new(r#"^(?P[a-zA-z]([a-zA-z0-9.-]*?))-(?P(\d+(\.\d+)*)).+?is out of bounds for:$"#).unwrap(); + let header_missing = + Regex::new(r#"^(?P[a-zA-z]([a-zA-z0-9.-]*)).+?depended on by:$"#).unwrap(); + let package = + Regex::new(r#"^- \[ \] (?P[a-zA-z]([a-zA-z0-9.-]*?))-(?P(\d+(\.\d+)*)).+?Used by: (?P.+)$"#) + .unwrap(); + + if let Ok(lines) = read_lines("./comments.txt") { + for line in lines { + if let Ok(line) = line { + if empty.captures(&line).is_some() { + } else if let Some(cap) = package.captures(&line) { + let root = last_header.clone().unwrap(); + let package = cap.name("package").unwrap().as_str(); + let version = cap.name("version").unwrap().as_str(); + let component = cap.name("component").unwrap().as_str(); + match component { + "library" | "executable" => { + insert(&mut lib_exes, root, package, version, component) + } + "benchmark" => insert(&mut benches, root, package, version, "benchmarks"), + "test-suite" => insert(&mut tests, root, package, version, component), + _ => panic!("Bad component: {}", component), + } + } else if let Some(cap) = header_versioned.captures(&line) { + let package = cap.name("package").unwrap().as_str().to_owned(); + let version = cap.name("version").unwrap().as_str().to_owned(); + last_header = Some(Header::Versioned { package, version }); + } else if let Some(cap) = header_missing.captures(&line) { + let package = cap.name("package").unwrap().as_str().to_owned(); + last_header = Some(Header::Missing { package }); + } else { + panic!("Unhandled: {:?}", line); + } + } + } + } + + if !lib_exes.is_empty() { + println!("\nLIBS + EXES\n"); + } + for (header, packages) in lib_exes { + for (package, version, component) in packages { + printer(" ", &package, true, &version, &component, &header); + } + } + + if !tests.is_empty() { + println!("\nTESTS\n"); + } + for (header, packages) in tests { + for (package, version, component) in packages { + printer(" ", &package, false, &version, &component, &header); + } + } + + if !benches.is_empty() { + println!("\nBENCHMARKS\n"); + } + for (header, packages) in benches { + for (package, version, component) in packages { + printer(" ", &package, false, &version, &component, &header); + } + } +} + +fn printer( + indent: &str, + package: &str, + lt0: bool, + version: &str, + component: &str, + header: &Header, +) -> () { + let lt0 = if lt0 { " < 0" } else { "" }; + println!( + "{indent}- {package}{lt0} # tried {package}-{version}, but its *{component}* {cause}", + indent = indent, + package = package, + lt0 = lt0, + version = version, + component = component, + cause = match header { + Header::Versioned { package, version } => format!( + "does not support: {package}-{version}", + package = package, + version = version + ), + Header::Missing { package } => format!( + "requires the disabled package: {package}", + package = package + ), + }, + ); +} + +fn insert(h: &mut H, header: Header, package: &str, version: &str, component: &str) { + (*h.entry(header).or_insert_with(|| vec![])).push(( + package.to_owned(), + version.to_owned(), + component.to_owned(), + )); +} + +fn read_lines

(filename: P) -> io::Result>> +where + P: AsRef, +{ + let file = File::open(filename)?; + Ok(io::BufReader::new(file).lines()) +}