From fa54517d15bba2d6b5bb5d1673e4b41cf3cf2e8c Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Thu, 2 Sep 2021 14:23:13 +0200 Subject: [PATCH] chore(nix): build docker container --- flake.nix | 190 +++++------------------------- nix/docker-demo/default.nix | 102 ++++++++++++++++ nix/uniworx/backend.nix | 74 ++++++++++++ nix/uniworx/default.nix | 6 + nix/uniworx/frontend.nix | 25 ++++ nix/uniworx/node-dependencies.nix | 15 +++ nix/uniworx/well-known.nix | 23 ++++ 7 files changed, 277 insertions(+), 158 deletions(-) create mode 100644 nix/docker-demo/default.nix create mode 100644 nix/uniworx/backend.nix create mode 100644 nix/uniworx/default.nix create mode 100644 nix/uniworx/frontend.nix create mode 100644 nix/uniworx/node-dependencies.nix create mode 100644 nix/uniworx/well-known.nix diff --git a/flake.nix b/flake.nix index a05be898a..719bc4e4c 100644 --- a/flake.nix +++ b/flake.nix @@ -71,176 +71,50 @@ outputs = inputs@{ self, nixpkgs, flake-utils, haskell-nix, ... }: flake-utils.lib.eachSystem ["x86_64-linux"] (system: - let pkgs = import nixpkgs { + let frontendSource = pkgs.lib.sourceByRegex ./. [ + "^(assets|frontend)(/.*)?$" + "^config(/(favicon\.json|robots\.txt))?$" + "^(webpack|postcss)\.config\.js$" + "^(package|jsconfig)\.json$" + "^\.babelrc$" + ]; + backendSource = pkgs.lib.sourceByRegex ./. [ + "^(package|stack-flake)\.yaml$" + "^stack\.yaml\.lock$" + "^(assets|app|hlint|load|messages|models|src|templates|test|testdata|wflint)(/.*)?$" + "^config(/(archive-types|mimetypes|personalised-sheet-files-collate|settings\.yml|submission-blacklist|test-settings\.yml|video-types|wordlist\.txt))?$" + "^routes$" + ]; + + pkgs = import nixpkgs { inherit system overlays; config.allowUnfree = true; }; - overlays = - let - frontendSource = pkgs.lib.sourceByRegex ./. [ - "^(assets|frontend)(/.*)?$" - "^config(/(favicon\.json|robots\.txt))?$" - "^(webpack|postcss)\.config\.js$" - "^(package|jsconfig)\.json$" - "^\.babelrc$" - ]; - backendSource = pkgs.lib.sourceByRegex ./. [ - "^(package|stack-flake)\.yaml$" - "^stack\.yaml\.lock$" - "^(assets|app|hlint|load|messages|models|src|templates|test|testdata|wflint)(/.*)?$" - "^config(/(archive-types|mimetypes|personalised-sheet-files-collate|settings\.yml|submission-blacklist|test-settings\.yml|video-types|wordlist\.txt))?$" - "^routes$" - ]; - in [ - (import ./nix/maildev) - haskell-nix.overlay - (final: prev: { - uniworx-node-dependencies = (prev.callPackage ./nix/frontend {}).nodeDependencies.override (oldArgs: { - dependencies = - let - srcOverrides = { - "tail.datetime" = inputs.tail-DateTime; - "@fortawesome/fontawesome-pro" = prev.fetchurl { - url = "https://npm.fontawesome.com/@fortawesome/fontawesome-pro/-/5.14.0/fontawesome-pro-5.14.0.tgz"; - curlOpts = "-H @${prev.writeText "headers.txt" "Authorization: Bearer ${builtins.readFile inputs.fontawesome-token}"}"; - hash = "sha256-jGvPrTKKL0rCWRZUEnJEmrOdHyQYs3M5709B1hjmFw4="; - }; - }; - in map (dep: dep // { src = srcOverrides."${dep.packageName}" or dep.src; }) oldArgs.dependencies; - }); - }) - (final: prev: { - uniworx-well-known = prev.stdenv.mkDerivation { - name = "uniworx-well-known"; - src = frontendSource; - - phases = ["unpackPhase" "buildPhase" "installPhase" "fixupPhase"]; - - buildPhase = '' - ln -s ${prev.uniworx-node-dependencies}/lib/node_modules ./node_modules - export PATH="${prev.uniworx-node-dependencies}/bin:${prev.exiftool}/bin:$PATH" - webpack --progress - ''; - - installPhase = '' - mkdir -p $out - cp -r --reflink=auto well-known $out/.nix-well-known - ''; - - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - outputHash = "tDaffdAT5EGPKdDJ2ovo9XSGdV48W3Efqe+iBmakh6g="; - }; - }) - (final: prev: { - uniworx-frontend = prev.stdenv.mkDerivation { - name = "uniworx-frontend"; - srcs = [frontendSource prev.uniworx-well-known]; - sourceRoot = "source"; - - phases = ["unpackPhase" "buildPhase" "installPhase"]; - - postUnpack = '' - cp -pr --reflink=auto uniworx-well-known/. $sourceRoot - ''; - - buildPhase = '' - ln -s ${prev.uniworx-node-dependencies}/lib/node_modules ./node_modules - export PATH="${prev.uniworx-node-dependencies}/bin:$PATH" - webpack --progress - ''; - - installPhase = '' - mkdir -p $out $out/config - cp -r --reflink=auto well-known static $out - cp -r --reflink=auto config/webpack.yml $out/config - ''; - }; - }) - (final: prev: { - uniworx = final.haskell-nix.stackProject { - src = prev.stdenv.mkDerivation { - name = "uniworx-src"; - src = backendSource; - - phases = ["unpackPhase" "patchPhase" "installPhase"]; - - patchPhase = '' - substitute stack-flake.yaml stack.yaml \ - ${prev.lib.concatMapStringsSep " \\\n" (pkgName: "--replace @${pkgName}@ ${inputs."${pkgName}"}") haskellInputs} - ''; - - installPhase = '' - mkdir -p $out - cp -pr --reflink=auto ./. $out - ''; - }; - compiler-nix-name = "ghc8104"; - # stack-sha256 = "1n7z294ldv2rjkfj1vs3kqmnbp34m2scrmyrp5kwmga9vp86fd9z"; - modules = [ - { - packages = { - encoding.src = inputs.encoding; - memcached-binary.src = inputs.memcached-binary; - conduit-resumablesink.src = inputs.conduit-resumablesink; - HaskellNet-SSL.src = inputs.HaskellNet-SSL; - ldap-client.src = inputs.ldap-client; - serversession.src = "${inputs.serversession}/serversession"; - serversession-backend-acid-state.src = "${inputs.serversession}/serversession-backend-acid-state"; - xss-sanitize.src = inputs.xss-sanitize; - colonnade.src = "${inputs.colonnade}/colonnade"; - minio-hs.src = inputs.minio-hs; - cryptoids-class.src = "${inputs.cryptoids}/cryptoids-class"; - cryptoids-types.src = "${inputs.cryptoids}/cryptoids-types"; - cryptoids.src = "${inputs.cryptoids}/cryptoids"; - filepath-crypto.src = "${inputs.cryptoids}/filepath-crypto"; - uuid-crypto.src = "${inputs.cryptoids}/uuid-crypto"; - zip-stream.src = inputs.zip-stream; - yesod.src = "${inputs.yesod}/yesod"; - yesod-core.src = "${inputs.yesod}/yesod-core"; - yesod-static.src = "${inputs.yesod}/yesod-static"; - yesod-persistent.src = "${inputs.yesod}/yesod-persistent"; - yesod-form.src = "${inputs.yesod}/yesod-form"; - yesod-auth.src = "${inputs.yesod}/yesod-auth"; - yesod-test.src = "${inputs.yesod}/yesod-test"; - cryptonite.src = inputs.cryptonite; - esqueleto.src = inputs.esqueleto; - }; - } - { - packages.uniworx = { - postUnpack = '' - cp -pr --reflink=auto ${prev.uniworx-frontend}/. $sourceRoot - chmod a+w -R $sourceRoot - ''; - preBuild = '' - export TZDIR=${final.tzdata}/share/zoneinfo - ''; - components.library.build-tools = with final.pkgs; [ llvm_9 ]; - components.exes.uniworx.build-tools = with final.pkgs; [ llvm_9 ]; - components.exes.uniworxdb.build-tools = with final.pkgs; [ llvm_9 ]; - components.exes.uniworxload.build-tools = with final.pkgs; [ llvm_9 ]; - components.exes.uniworx-wflint.build-tools = with final.pkgs; [ llvm_9 ]; - components.tests.yesod.build-tools = with final.pkgs; [ llvm_9 ]; - components.tests.hlint.build-tools = with final.pkgs; [ llvm_9 ]; - }; - } - ]; - }; - }) + overlays = [ + (import ./nix/maildev) + haskell-nix.overlay + (import ./nix/uniworx { inherit inputs frontendSource backendSource; }) + (import ./nix/docker-demo { inherit self; }) ]; - haskellInputs = ["encoding" "memcached-binary" "conduit-resumablesink" "HaskellNet-SSL" "ldap-client" "serversession" "xss-sanitize" "colonnade" "minio-hs" "cryptoids" "zip-stream" "yesod" "cryptonite" "esqueleto"]; - haskellFlake = pkgs.uniworx.flake {}; + + pushUniworxDemoDocker = pkgs.writeScriptBin "push-uniworx-demo-docker" '' + #!${pkgs.zsh}/bin/zsh -xe + + target=''${1-docker://registry.gitlab.com/fradrive/fradrive/uniworx-demo} + [[ -n "''${1}" ]] && shift + ${pkgs.skopeo}/bin/skopeo ''${@} --insecure-policy copy docker-archive://${pkgs.uniworxDemoDocker} ''${target} + ''; in { packages = haskellFlake.packages // { inherit (pkgs) uniworx-node-dependencies uniworx-well-known uniworx-frontend; inherit (pkgs.uniworx.stack-nix.passthru) calculateMaterializedSha; }; - inherit (haskellFlake) checks apps; + apps = haskellFlake.apps // { push-uniworx-demo-docker = flake-utils.lib.mkApp { drv = pushUniworxDemoDocker; }; }; + inherit (haskellFlake) checks; devShell = import ./shell.nix { inherit pkgs; }; legacyPackages = pkgs; - defaultPackage = haskellFlake.packages."uniworx:exe:uniworx"; + defaultPackage = pkgs.uniworxDemoDocker; } ); } diff --git a/nix/docker-demo/default.nix b/nix/docker-demo/default.nix new file mode 100644 index 000000000..77bb59c5f --- /dev/null +++ b/nix/docker-demo/default.nix @@ -0,0 +1,102 @@ +{ self }: final: prev: { + uniworxDemoDocker = prev.dockerTools.buildImage { + name = "uniworx-demo"; + tag = self.rev or null; + created = + let + fromDate = builtins.readFile (prev.runCommand "date" { nativeBuildInputs = with final; [ coreutils ]; } '' + printf '%s' $(date -Is -d '@${toString self.lastModified}') > $out + ''); + in if self ? lastModified then fromDate else "1970-01-01T00:00:01Z"; + + contents = with final; [ + uniworx.uniworx.components.exes.uniworx + prev.dockerTools.binSh postgresql_12 + memcached + ]; + + runAsRoot = '' + #!${final.stdenv.shell} + + ${prev.dockerTools.shadowSetup} + + mkdir -p /var/lib + + groupadd -r postgres + useradd -r -g postgres -d /var/lib/postgres -M postgres + install -d -g postgres -o postgres -m 0750 /var/lib/postgres + + groupadd -r memcached + useradd -r -g memcached -d /var/lib/memcached -M memcached + install -d -g memcached -o memcached -m 0750 /var/lib/memcached + + groupadd -r uniworx + useradd -r -g uniworx -d /var/lib/uniworx -M uniworx + install -d -g uniworx -o uniworx -m 0750 /var/lib/uniworx + gpasswd -a uniworx postgres + + mkdir -p /var/log + install -d -g postgres -o postgres -m 0755 /var/log/postgres + install -d -g memcached -o memcached -m 0755 /var/log/memcached + install -d -g uniworx -o uniworx -m 0755 /var/log/uniworx + + mkdir -p /run + install -d -g postgres -o postgres -m 0755 /run/postgres + ''; + + config = + let + entrypoint = prev.writeScriptBin "uniworx-entrypoint" '' + #!${final.zsh}/bin/zsh -xe + export PATH=${final.su}/bin:${final.findutils}/bin:${final.coreutils}/bin:/bin + + cTime=$(date -Is) + + pgDir=/var/lib/postgres + pgSockDir=/run/postgres + pgLogFile=/var/log/postgres/''${cTime}.log + export PGHOST=''${pgSockDir} + export PGLOG=''${pgLogFile} + + pgNew= + if [[ -n "$(find ''${pgDir} -maxdepth 0 -type d -empty 2>/dev/null)" ]]; then + pgNew=1 + fi + + [[ -z "''${pgNew}" ]] || su postgres -c "initdb --no-locale --encoding=UTF8 -D ''${pgDir}" + su postgres -c "pg_ctl start -D ''${pgDir} -l ''${pgLogFile} -w -o '-k ''${pgSockDir} -c listen_addresses= -c hba_file=${postgresHba} -c unix_socket_permissions=0777 -c max_connections=9990 -c shared_preload_libraries=pg_stat_statements -c auto_explain.log_min_duration=100ms'" + [[ -z "''${pgNew}" ]] || psql -f ${postgresSchema} postgres postgres + + su memcached -c "cd /var/lib/memcached; memcached -p 11212" &>/var/log/memcached/''${cTime}.log & + export SESSION_MEMCACHED_HOST=localhost + export SESSION_MEMCACHED_PORT=11212 + + export LOGDEST=/var/log/uniworx/''${cTime}.log + exec -- su uniworx -c "cd /var/lib/uniworx; uniworx ${uniworxConfig}" + ''; + postgresSchema = prev.writeText "schema.sql" '' + CREATE USER uniworx WITH SUPERUSER; + CREATE DATABASE uniworx; + GRANT ALL ON DATABASE uniworx TO uniworx; + ''; + + postgresHba = prev.writeText "hba_file" '' + local all all trust + ''; + uniworxConfig = prev.writeText "uni2work.yml" '' + port: 8080 + approot: "_env:APPROOT:http://localhost:8080" + ''; + in { + Cmd = [ "${entrypoint}/bin/uniworx-entrypoint" ]; + ExposedPorts = { + "8080/tcp" = {}; + }; + Volumes = { + "/var/lib/postgres" = {}; + "/var/lib/uniworx" = {}; + "/var/log" = {}; + }; + }; + }; +} diff --git a/nix/uniworx/backend.nix b/nix/uniworx/backend.nix new file mode 100644 index 000000000..f77c199a0 --- /dev/null +++ b/nix/uniworx/backend.nix @@ -0,0 +1,74 @@ +{ inputs, backendSource, ... }: final: prev: +let + haskellInputs = ["encoding" "memcached-binary" "conduit-resumablesink" "HaskellNet-SSL" "ldap-client" "serversession" "xss-sanitize" "colonnade" "minio-hs" "cryptoids" "zip-stream" "yesod" "cryptonite" "esqueleto"]; +in { + uniworx = final.haskell-nix.stackProject { + src = prev.stdenv.mkDerivation { + name = "uniworx-src"; + src = backendSource; + + phases = ["unpackPhase" "patchPhase" "installPhase"]; + + patchPhase = '' + substitute stack-flake.yaml stack.yaml \ + ${prev.lib.concatMapStringsSep " \\\n" (pkgName: "--replace @${pkgName}@ ${inputs."${pkgName}"}") haskellInputs} + ''; + + installPhase = '' + mkdir -p $out + cp -pr --reflink=auto ./. $out + ''; + }; + compiler-nix-name = "ghc8104"; + # stack-sha256 = "1n7z294ldv2rjkfj1vs3kqmnbp34m2scrmyrp5kwmga9vp86fd9z"; + modules = [ + { + packages = { + encoding.src = inputs.encoding; + memcached-binary.src = inputs.memcached-binary; + conduit-resumablesink.src = inputs.conduit-resumablesink; + HaskellNet-SSL.src = inputs.HaskellNet-SSL; + ldap-client.src = inputs.ldap-client; + serversession.src = "${inputs.serversession}/serversession"; + serversession-backend-acid-state.src = "${inputs.serversession}/serversession-backend-acid-state"; + xss-sanitize.src = inputs.xss-sanitize; + colonnade.src = "${inputs.colonnade}/colonnade"; + minio-hs.src = inputs.minio-hs; + cryptoids-class.src = "${inputs.cryptoids}/cryptoids-class"; + cryptoids-types.src = "${inputs.cryptoids}/cryptoids-types"; + cryptoids.src = "${inputs.cryptoids}/cryptoids"; + filepath-crypto.src = "${inputs.cryptoids}/filepath-crypto"; + uuid-crypto.src = "${inputs.cryptoids}/uuid-crypto"; + zip-stream.src = inputs.zip-stream; + yesod.src = "${inputs.yesod}/yesod"; + yesod-core.src = "${inputs.yesod}/yesod-core"; + yesod-static.src = "${inputs.yesod}/yesod-static"; + yesod-persistent.src = "${inputs.yesod}/yesod-persistent"; + yesod-form.src = "${inputs.yesod}/yesod-form"; + yesod-auth.src = "${inputs.yesod}/yesod-auth"; + yesod-test.src = "${inputs.yesod}/yesod-test"; + cryptonite.src = inputs.cryptonite; + esqueleto.src = inputs.esqueleto; + }; + } + { + packages.uniworx = { + postUnpack = '' + cp -pr --reflink=auto ${prev.uniworx-frontend}/. $sourceRoot + chmod a+w -R $sourceRoot + ''; + preBuild = '' + export TZDIR=${final.tzdata}/share/zoneinfo + ''; + components.library.build-tools = with final.pkgs; [ llvm_9 ]; + components.exes.uniworx.build-tools = with final.pkgs; [ llvm_9 ]; + components.exes.uniworxdb.build-tools = with final.pkgs; [ llvm_9 ]; + components.exes.uniworxload.build-tools = with final.pkgs; [ llvm_9 ]; + components.exes.uniworx-wflint.build-tools = with final.pkgs; [ llvm_9 ]; + components.tests.yesod.build-tools = with final.pkgs; [ llvm_9 ]; + components.tests.hlint.build-tools = with final.pkgs; [ llvm_9 ]; + }; + } + ]; + }; +} diff --git a/nix/uniworx/default.nix b/nix/uniworx/default.nix new file mode 100644 index 000000000..c353be222 --- /dev/null +++ b/nix/uniworx/default.nix @@ -0,0 +1,6 @@ +{ inputs, frontendSource, backendSource }: final: prev: prev.lib.composeManyExtensions [ + (import ./node-dependencies.nix { inherit inputs; }) + (import ./well-known.nix { inherit frontendSource; }) + (import ./frontend.nix { inherit frontendSource; }) + (import ./backend.nix { inherit backendSource inputs; }) +] final prev diff --git a/nix/uniworx/frontend.nix b/nix/uniworx/frontend.nix new file mode 100644 index 000000000..bbffa391e --- /dev/null +++ b/nix/uniworx/frontend.nix @@ -0,0 +1,25 @@ +{ frontendSource, ... }: final: prev: { + uniworx-frontend = prev.stdenv.mkDerivation { + name = "uniworx-frontend"; + srcs = [frontendSource prev.uniworx-well-known]; + sourceRoot = "source"; + + phases = ["unpackPhase" "buildPhase" "installPhase"]; + + postUnpack = '' + cp -pr --reflink=auto uniworx-well-known/. $sourceRoot + ''; + + buildPhase = '' + ln -s ${prev.uniworx-node-dependencies}/lib/node_modules ./node_modules + export PATH="${prev.uniworx-node-dependencies}/bin:$PATH" + webpack --progress + ''; + + installPhase = '' + mkdir -p $out $out/config + cp -r --reflink=auto well-known static $out + cp -r --reflink=auto config/webpack.yml $out/config + ''; + }; +} diff --git a/nix/uniworx/node-dependencies.nix b/nix/uniworx/node-dependencies.nix new file mode 100644 index 000000000..7893f47b1 --- /dev/null +++ b/nix/uniworx/node-dependencies.nix @@ -0,0 +1,15 @@ +{ inputs, ... }: final: prev: { + uniworx-node-dependencies = (prev.callPackage ../frontend {}).nodeDependencies.override (oldArgs: { + dependencies = + let + srcOverrides = { + "tail.datetime" = inputs.tail-DateTime; + "@fortawesome/fontawesome-pro" = prev.fetchurl { + url = "https://npm.fontawesome.com/@fortawesome/fontawesome-pro/-/5.14.0/fontawesome-pro-5.14.0.tgz"; + curlOpts = "-H @${prev.writeText "headers.txt" "Authorization: Bearer ${builtins.readFile inputs.fontawesome-token}"}"; + hash = "sha256-jGvPrTKKL0rCWRZUEnJEmrOdHyQYs3M5709B1hjmFw4="; + }; + }; + in map (dep: dep // { src = srcOverrides."${dep.packageName}" or dep.src; }) oldArgs.dependencies; + }); +} diff --git a/nix/uniworx/well-known.nix b/nix/uniworx/well-known.nix new file mode 100644 index 000000000..615269a9e --- /dev/null +++ b/nix/uniworx/well-known.nix @@ -0,0 +1,23 @@ +{ frontendSource, ... }: final: prev: { + uniworx-well-known = prev.stdenv.mkDerivation { + name = "uniworx-well-known"; + src = frontendSource; + + phases = ["unpackPhase" "buildPhase" "installPhase" "fixupPhase"]; + + buildPhase = '' + ln -s ${prev.uniworx-node-dependencies}/lib/node_modules ./node_modules + export PATH="${prev.uniworx-node-dependencies}/bin:${prev.exiftool}/bin:$PATH" + webpack --progress + ''; + + installPhase = '' + mkdir -p $out + cp -r --reflink=auto well-known $out/.nix-well-known + ''; + + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = "tDaffdAT5EGPKdDJ2ovo9XSGdV48W3Efqe+iBmakh6g="; + }; +}