From 953281e263a51d77434d557b3f52490c9ffeff21 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sun, 9 Feb 2020 14:08:59 +0200 Subject: [PATCH] More cleanups to build script --- automated/.gitignore | 13 ++--- automated/{build-next.sh => build.sh} | 7 ++- automated/check-cabal.sh | 72 ------------------------ automated/cron.sh | 24 -------- automated/new-stackage-format/convert.sh | 35 ------------ automated/nix-build/README.md | 5 -- automated/nix-build/build.sh | 12 ---- automated/nix-build/fetch-nixpkgs.nix | 23 -------- automated/nix-build/nixpkgs-src.json | 6 -- automated/nix-build/snapshot-fixed.nix | 41 -------------- automated/run-nightly.sh | 5 +- 11 files changed, 10 insertions(+), 233 deletions(-) rename automated/{build-next.sh => build.sh} (96%) delete mode 100755 automated/check-cabal.sh delete mode 100755 automated/cron.sh delete mode 100755 automated/new-stackage-format/convert.sh delete mode 100644 automated/nix-build/README.md delete mode 100644 automated/nix-build/build.sh delete mode 100644 automated/nix-build/fetch-nixpkgs.nix delete mode 100644 automated/nix-build/nixpkgs-src.json delete mode 100644 automated/nix-build/snapshot-fixed.nix diff --git a/automated/.gitignore b/automated/.gitignore index 9c4d36e6..c43ceb23 100644 --- a/automated/.gitignore +++ b/automated/.gitignore @@ -1,12 +1,9 @@ -/auth-token /bin/ /gitconfig /hackage-creds -/ssh-nightly/ -/ssh-lts/ -/nighlty/ -/lts-*/ -/stackage-curator/ -/stackage-update/ +/ssh/ +/work/ /aws.sh -/crondir/ +/pantry/ +/stack/ +/dot-stackage/ diff --git a/automated/build-next.sh b/automated/build.sh similarity index 96% rename from automated/build-next.sh rename to automated/build.sh index 27adc744..b077babf 100755 --- a/automated/build-next.sh +++ b/automated/build.sh @@ -12,17 +12,18 @@ source aws.sh SHORTNAME=$(echo $TARGET | cut -d- -f 1) if [ $SHORTNAME = "lts" ] then - TAG=$(echo $TARGET | sed 's@^lts-\([0-9]*\)\.[0-9]*@lts\1@') + TAG=$(echo $TARGET | sed 's@^lts-\([0-9]*\)\.[0-9]*@lts-\1@') + WORKDIR=$ROOT/work/$(echo $TARGET | sed 's@^lts-\([0-9]*\)\.[0-9]*@lts\1@') else TAG=$SHORTNAME + WORKDIR=$ROOT/work/$TAG fi IMAGE=commercialhaskell/stackage:$TAG -PANTRY_DIR=$ROOT/pantry +PANTRY_DIR=$ROOT/stack/pantry STACK_DIR=$ROOT/stack DOT_STACKAGE_DIR=$ROOT/dot-stackage -WORKDIR=$ROOT/$TAG/work # ssh key is used for committing snapshots (and their constraints) to Github SSH_DIR=$ROOT/ssh USERID=$(id -u) diff --git a/automated/check-cabal.sh b/automated/check-cabal.sh deleted file mode 100755 index c920f98a..00000000 --- a/automated/check-cabal.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -# This script is intended for testing Cabal HEAD against a Stackage snapshot. - -set -eux - -ROOT=$(cd $(dirname $0) ; pwd) -TARGET=$1 - -# For nightly-YYYY-MM-DD, tag should be nightly -# For lts-X.Y, tag should be ltsX -SHORTNAME=$(echo $TARGET | cut -d- -f 1) -if [ $SHORTNAME = "lts" ] -then - TAG=$(echo $TARGET | sed 's@^lts-\([0-9]*\)\.[0-9]*@lts\1@') - PLAN_URL=https://raw.githubusercontent.com/fpco/lts-haskell/master/$TARGET.yaml -else - TAG=$SHORTNAME - PLAN_URL=https://raw.githubusercontent.com/fpco/stackage-nightly/master/$TARGET.yaml -fi - -IMAGE=snoyberg/stackage:$TAG - -PLAN_FILE=$TARGET-plan.yaml -DOCMAP_FILE=$TARGET-docmap.yaml -BUNDLE_FILE=$TARGET.bundle= - -CABAL_DIR=$ROOT/cabal -STACK_DIR=$ROOT/stack -GHC_DIR=$ROOT/ghc -DOT_STACKAGE_DIR=$ROOT/dot-stackage -WORKDIR=$ROOT/$TAG/work -EXTRA_BIN_DIR=$ROOT/extra-bin - -mkdir -p \ - "$CABAL_DIR" \ - "$STACK_DIR" \ - "$GHC_DIR" \ - "$DOT_STACKAGE_DIR" \ - "$WORKDIR" \ - "$EXTRA_BIN_DIR" - -curl "$PLAN_URL" > $WORKDIR/$PLAN_FILE - -mkdir -p $ROOT/bin -BINDIR=$(cd $ROOT/bin ; pwd) -( -cd $BINDIR -rm -f stackage-curator stackage-curator.bz2 -wget https://s3.amazonaws.com/stackage-travis/stackage-curator/stackage-curator.bz2 -bunzip2 stackage-curator.bz2 -chmod +x stackage-curator -) - -ARGS_COMMON="--rm -v $WORKDIR:/home/stackage/work -w /home/stackage/work -v $BINDIR/stackage-curator:/usr/bin/stackage-curator:ro -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v $EXTRA_BIN_DIR:/home/stackage/bin:ro" -ARGS_PREBUILD="$ARGS_COMMON -u $USER -v $CABAL_DIR:/home/stackage/.cabal -v $STACK_DIR:/home/stackage/.stack -v $GHC_DIR:/home/stackage/.ghc -v $DOT_STACKAGE_DIR:/home/stackage/.stackage" -ARGS_BUILD="$ARGS_COMMON -v $CABAL_DIR:/home/stackage/.cabal:ro -v $STACK_DIR:/home/stackage/.stack:ro -v $GHC_DIR:/home/stackage/.ghc:ro" - -# Get latest stack -curl -L https://www.stackage.org/stack/linux-x86_64 | tar xz --wildcards --strip-components=1 -C $EXTRA_BIN_DIR '*/stack' - -# Do all of the pre-build actions: -# -# * Update the package index -# * Fetch all needed tarballs (the build step does not have write access to the tarball directory) -# * Do a single unpack to create the package index cache (again due to directory perms) -docker run $ARGS_PREBUILD $IMAGE /bin/bash -c "/home/stackage/bin/stack update && stackage-curator fetch --plan-file $PLAN_FILE && cd /tmp && /home/stackage/bin/stack unpack random" - -# Now do the actual build. We need to first set the owner of the home directory -# correctly, so we run the command as root, change owner, and then use sudo to -# switch back to the current user -docker run $ARGS_BUILD $IMAGE /bin/bash -c "chown $USER /home/stackage && sudo -E -u $USER env \"PATH=\$PATH:/home/stackage/bin\" stackage-curator make-bundle --plan-file $PLAN_FILE --docmap-file $DOCMAP_FILE --bundle-file $BUNDLE_FILE --target $TARGET --cabal-from-head" diff --git a/automated/cron.sh b/automated/cron.sh deleted file mode 100755 index 2d804ad8..00000000 --- a/automated/cron.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -cd "$(dirname "${BASH_SOURCE[0]}")" - -CRONDIR=$(pwd)/crondir -mkdir -p $CRONDIR -source aws.sh - -IMAGE=fpco/stackage-server-prod:latest -docker pull $IMAGE - -stack update - -date -echo "Running stackage-server-cron..." -docker run --rm \ - -v $CRONDIR:/home/ubuntu \ - -v $HOME/.stack/indices:/home/ubuntu/.stack/indices:ro \ - --workdir /home/ubuntu \ - -p 17834:17834 \ - $IMAGE \ - bash -c "useradd $(whoami) -u $(id -u); sudo -u $(whoami) env HOME=/home/ubuntu AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY bash -c '/usr/local/bin/stackage-server-cron 2>&1 | tee -a /home/ubuntu/stackage-server-cron.log'" diff --git a/automated/new-stackage-format/convert.sh b/automated/new-stackage-format/convert.sh deleted file mode 100755 index 092b293a..00000000 --- a/automated/new-stackage-format/convert.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -if [[ ! -f convert ]] -then - curl https://s3.amazonaws.com/www.snoyman.com/convert-old-stackage-c9c4d06b31cb7aafedd23aa316b8008c45e4d4dd.bz2 > convert.bz2 - chmod +x convert.bz2 - bunzip2 convert.bz2 -fi - -cd $(dirname ${BASH_SOURCE[0]}) - -for d in lts-haskell stackage-nightly stackage-snapshots -do - if [[ ! -d "$d" ]] - then - git clone https://github.com/commercialhaskell/$d - else - (cd "$d" && git pull || echo "Git pull failed, ignoring") - fi -done - -( -export HOME=$(pwd) -./convert -) - -cd stackage-snapshots -git add lts nightly -git diff-index --quiet HEAD && echo No changes && exit 0 -git config user.name "Stackage build server" -git config user.email "michael@snoyman.com" -git commit -m "More conversions $(date)" -GIT_SSH_COMMAND='ssh -i ../../ssh-lts/id_rsa' git push git@github.com:commercialhaskell/stackage-snapshots master diff --git a/automated/nix-build/README.md b/automated/nix-build/README.md deleted file mode 100644 index 56901b26..00000000 --- a/automated/nix-build/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Building Stackage with Nix - -The script `build.sh` is supposed to be run in a directory where `curator unpack` -was run - it assumes existence of `unpack-dir` directory with a Stackage -snapshot `stack.yaml` and unpacked packages in it. diff --git a/automated/nix-build/build.sh b/automated/nix-build/build.sh deleted file mode 100644 index 62918fe5..00000000 --- a/automated/nix-build/build.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -eu +x - -NIX_VERSION="2.0.4" -curl https://nixos.org/releases/nix/nix-$NIX_VERSION/install | sh - -# no code to install stack2nix yet - -stack2nix --cabal2nix-args="--no-hpack" unpack-dir > snapshot.nix - -nix build -f snapshot-fixed.nix diff --git a/automated/nix-build/fetch-nixpkgs.nix b/automated/nix-build/fetch-nixpkgs.nix deleted file mode 100644 index 29cac91f..00000000 --- a/automated/nix-build/fetch-nixpkgs.nix +++ /dev/null @@ -1,23 +0,0 @@ -let - spec = builtins.fromJSON (builtins.readFile ./nixpkgs-src.json); - src = import { - url = "https://github.com/${spec.owner}/${spec.repo}/archive/${spec.rev}.tar.gz"; - inherit (spec) sha256; - }; - nixcfg = import ; -in builtins.derivation { - system = builtins.currentSystem; - name = "${src.name}-unpacked"; - builder = builtins.storePath nixcfg.shell; - inherit src; - args = [ - (builtins.toFile "builder" '' - $coreutils/mkdir $out - cd $out - $gzip -d < $src | $tar -x --strip-components=1 - '') - ]; - coreutils = builtins.storePath nixcfg.coreutils; - tar = builtins.storePath nixcfg.tar; - gzip = builtins.storePath nixcfg.gzip; -} diff --git a/automated/nix-build/nixpkgs-src.json b/automated/nix-build/nixpkgs-src.json deleted file mode 100644 index 00ed9d5d..00000000 --- a/automated/nix-build/nixpkgs-src.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "c2950341d038995bf46a7b72db961bb3d3e9ac12", - "sha256": "0ghirjm6xvsihci3fladrn4nrnh2yr529v7c406hmw0yp25kv0i4" -} diff --git a/automated/nix-build/snapshot-fixed.nix b/automated/nix-build/snapshot-fixed.nix deleted file mode 100644 index bac92a89..00000000 --- a/automated/nix-build/snapshot-fixed.nix +++ /dev/null @@ -1,41 +0,0 @@ -{ pkgs ? import (import ./fetch-nixpkgs.nix) {} -, compiler ? pkgs.haskell.packages.ghc863 -}: -let - snapshot = import ./snapshot.nix { pkgs = pkgs; compiler = compiler; }; -in - snapshot.override (with pkgs.haskell.lib; { - overrides = self: super: { - # disabling packages from non-hackage-packages.nix - futhark = null; - multi-ghc-travis = null; - vaultenv = null; - # native deps - check-email = super.check-email.override { resolv = null; }; - flac = super.flac.override { FLAC = pkgs.flac; }; - freenect = super.freenect.override { freenect_sync = null; libfreenect = null; }; - haskell-gi = super.haskell-gi.override { gobject-introspection = pkgs.gnome3.gobjectIntrospection; }; - HDBC-mysql = super.HDBC-mysql.override { mysqlclient = pkgs.mysql;}; - libffi = super.libffi.override { ffi = null; }; - # fixes - blas-ffi = overrideCabal super.blas-ffi ( - old: { libraryPkgconfigDepends = old.libraryPkgconfigDepends ++ [ pkgs.openblasCompat ]; } - ); - odbc = overrideCabal super.odbc ( - old: { librarySystemDepends = old.librarySystemDepends ++ [ pkgs.freetds ]; } - ); - html-entities = null; # https://github.com/nikita-volkov/html-entities/issues/8 - category = null; # conflict in Prelude.hs - constraint = null; # because of category - lame = null; # fails with 'Total ticks: 451489' - proto-lens-protobuf-types = overrideCabal super.proto-lens-protobuf-types ( - old: { libraryToolDepends = old.libraryToolDepends ++ [ pkgs.protobuf ]; } - ); - # libraft = null; # xxx/postgres/entries.sql: openFile: does not exist (No such file or directory) in libraft-0.2.0.0 - xmlbf-xeno = null; # because of html-entities - xmonad = null; # xmonad requires extra patches for Nix - xmonad-contrib = null; - xmonad-extras = null; - hlibgit2 = disableHardening super.hlibgit2 [ "format" ]; - }; - }) diff --git a/automated/run-nightly.sh b/automated/run-nightly.sh index 7b7ca37a..9f21bd98 100755 --- a/automated/run-nightly.sh +++ b/automated/run-nightly.sh @@ -3,12 +3,9 @@ cd "$(dirname "${BASH_SOURCE[0]}")" while true; do - ./build-next.sh nightly-$(date -u +%F) + ./build.sh nightly-$(date -u +%F) date - #./new-stackage-format/convert.sh > /dev/null 2> /dev/null - #date - sleep 30m echo done