From 96e2c4d1a66fe5fd3b4d724e845372f067192926 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Fri, 27 Aug 2021 20:52:45 +0200 Subject: [PATCH 01/52] chore(nix): build uniworx via flake --- .gitignore | 1 + flake.lock | 564 +- flake.nix | 252 +- nix/frontend/default.nix | 17 + nix/frontend/node-env.nix | 567 ++ nix/frontend/node-packages.nix | 16604 +++++++++++++++++++++++++++++++ package.json | 2 +- shell.nix | 1 - stack-flake.yaml | 80 + webpack.config.js | 38 +- 10 files changed, 18085 insertions(+), 41 deletions(-) create mode 100644 nix/frontend/default.nix create mode 100644 nix/frontend/node-env.nix create mode 100644 nix/frontend/node-packages.nix create mode 100644 stack-flake.yaml diff --git a/.gitignore b/.gitignore index 39a52e97b..9fff5192f 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ tunnel.log /static /well-known /.well-known-cache +/.nix-well-known /**/tmp-* /testdata/bigAlloc_*.csv /sessions diff --git a/flake.lock b/flake.lock index f5c74561b..9d1b7a81e 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,190 @@ { "nodes": { + "HTTP": { + "flake": false, + "locked": { + "lastModified": 1451647621, + "narHash": "sha256-oHIyw3x0iKBexEo49YeUDV1k74ZtyYKGR2gNJXXRxts=", + "owner": "phadej", + "repo": "HTTP", + "rev": "9bc0996d412fef1787449d841277ef663ad9a915", + "type": "github" + }, + "original": { + "owner": "phadej", + "repo": "HTTP", + "type": "github" + } + }, + "HaskellNet-SSL": { + "flake": false, + "locked": { + "lastModified": 1582382981, + "narHash": "sha256-fjw+bKaGrGw9uTUd6fWZCIz24uuSTA1VIeijZ+zSq/M=", + "ref": "uni2work", + "rev": "40393c938111ac78232dc2c7eec5edb4a22d03e8", + "revCount": 62, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/HaskellNet-SSL.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/HaskellNet-SSL.git" + } + }, + "cabal-32": { + "flake": false, + "locked": { + "lastModified": 1603716527, + "narHash": "sha256-sDbrmur9Zfp4mPKohCD8IDZfXJ0Tjxpmr2R+kg5PpSY=", + "owner": "haskell", + "repo": "cabal", + "rev": "94aaa8e4720081f9c75497e2735b90f6a819b08e", + "type": "github" + }, + "original": { + "owner": "haskell", + "ref": "3.2", + "repo": "cabal", + "type": "github" + } + }, + "cabal-34": { + "flake": false, + "locked": { + "lastModified": 1622475795, + "narHash": "sha256-chwTL304Cav+7p38d9mcb+egABWmxo2Aq+xgVBgEb/U=", + "owner": "haskell", + "repo": "cabal", + "rev": "b086c1995cdd616fc8d91f46a21e905cc50a1049", + "type": "github" + }, + "original": { + "owner": "haskell", + "ref": "3.4", + "repo": "cabal", + "type": "github" + } + }, + "cardano-shell": { + "flake": false, + "locked": { + "lastModified": 1608537748, + "narHash": "sha256-PulY1GfiMgKVnBci3ex4ptk2UNYMXqGjJOxcPy2KYT4=", + "owner": "input-output-hk", + "repo": "cardano-shell", + "rev": "9392c75087cb9a3d453998f4230930dea3a95725", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "repo": "cardano-shell", + "type": "github" + } + }, + "colonnade": { + "flake": false, + "locked": { + "lastModified": 1592144408, + "narHash": "sha256-bQSOjbn56ZEjzqkC/oqZ331FdxF7tbhJndtXeRqWcq8=", + "ref": "uni2work", + "rev": "f8170266ab25b533576e96715bedffc5aa4f19fa", + "revCount": 153, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/colonnade.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/colonnade.git" + } + }, + "conduit-resumablesink": { + "flake": false, + "locked": { + "lastModified": 1533667275, + "narHash": "sha256-+TR0tYRk1WFQRrFs6oO80jdlfY7OTyB7th7Hi/tDQMw=", + "ref": "uni2work", + "rev": "cbea6159c2975d42f948525e03e12fc390da53c5", + "revCount": 10, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/conduit-resumablesink.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/conduit-resumablesink.git" + } + }, + "cryptoids": { + "flake": false, + "locked": { + "lastModified": 1602592869, + "narHash": "sha256-uhbIaVFXtdWmeDJl0ZDyJnE2Ul3bV324KkaK+ix6USA=", + "ref": "uni2work", + "rev": "130b0dcbf2b09ccdf387b50262f1efbbbf1819e3", + "revCount": 44, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/cryptoids.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/cryptoids.git" + } + }, + "cryptonite": { + "flake": false, + "locked": { + "lastModified": 1624444174, + "narHash": "sha256-sDMA4ej1NIModAt7PQvcgIknI3KwfzcAp9YQUSe4CWw=", + "ref": "uni2work", + "rev": "71a630edaf5f22c464e24fac8d9d310f4055ea1f", + "revCount": 1202, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/cryptonite.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/cryptonite.git" + } + }, + "encoding": { + "flake": false, + "locked": { + "lastModified": 1582400874, + "narHash": "sha256-ukQw4tvjtvqR8HEPgPTSmvtTc6WeXEs3IfCLLNy+YJc=", + "ref": "uni2work", + "rev": "22fc3bb14841d8d50997aa47f1be3852e666f787", + "revCount": 162, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/encoding.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/encoding.git" + } + }, + "esqueleto": { + "flake": false, + "locked": { + "lastModified": 1625584503, + "narHash": "sha256-kgNcNKODv4xkmS6+MKDAVDpbL/IQ5Imucst0uV4HzYU=", + "ref": "uni2work", + "rev": "b9987d94af9d7403eded8ca75ad761eb7fc06e4c", + "revCount": 697, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/esqueleto.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/esqueleto.git" + } + }, "flake-utils": { "locked": { "lastModified": 1623875721, @@ -11,31 +196,396 @@ }, "original": { "owner": "numtide", - "ref": "master", "repo": "flake-utils", "type": "github" } }, + "fontawesome-token": { + "flake": false, + "locked": { + "narHash": "sha256-XABvCxD/7zXRyyR01dr5IvK+8A0VoibiVUS5ZAU+f+o=", + "path": "/etc/fontawesome-token", + "type": "path" + }, + "original": { + "path": "/etc/fontawesome-token", + "type": "path" + } + }, + "ghc-8.6.5-iohk": { + "flake": false, + "locked": { + "lastModified": 1600920045, + "narHash": "sha256-DO6kxJz248djebZLpSzTGD6s8WRpNI9BTwUeOf5RwY8=", + "owner": "input-output-hk", + "repo": "ghc", + "rev": "95713a6ecce4551240da7c96b6176f980af75cae", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "ref": "release/8.6.5-iohk", + "repo": "ghc", + "type": "github" + } + }, + "hackage": { + "flake": false, + "locked": { + "lastModified": 1629940355, + "narHash": "sha256-o9/U8R/JtyHIcxhMLaWYP+D/52B6LH/ikCyNZ7+mymI=", + "owner": "input-output-hk", + "repo": "hackage.nix", + "rev": "429deb2a137084c011310bad92f4cecf244f2fc2", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "repo": "hackage.nix", + "type": "github" + } + }, + "haskell-nix": { + "inputs": { + "HTTP": "HTTP", + "cabal-32": "cabal-32", + "cabal-34": "cabal-34", + "cardano-shell": "cardano-shell", + "flake-utils": "flake-utils", + "ghc-8.6.5-iohk": "ghc-8.6.5-iohk", + "hackage": "hackage", + "hpc-coveralls": "hpc-coveralls", + "nix-tools": "nix-tools", + "nixpkgs": "nixpkgs", + "nixpkgs-2003": "nixpkgs-2003", + "nixpkgs-2009": "nixpkgs-2009", + "nixpkgs-2105": "nixpkgs-2105", + "nixpkgs-unstable": "nixpkgs-unstable", + "old-ghc-nix": "old-ghc-nix", + "stackage": "stackage" + }, + "locked": { + "lastModified": 1629940530, + "narHash": "sha256-cf/bshYWloj+rJSmOcFokN6Rs1lpFkiECKpykN3JBH8=", + "owner": "input-output-hk", + "repo": "haskell.nix", + "rev": "5fcd4faf98fc8ca8287e2c7bc1fff71dfd340f1f", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "repo": "haskell.nix", + "type": "github" + } + }, + "hpc-coveralls": { + "flake": false, + "locked": { + "lastModified": 1607498076, + "narHash": "sha256-8uqsEtivphgZWYeUo5RDUhp6bO9j2vaaProQxHBltQk=", + "owner": "sevanspowell", + "repo": "hpc-coveralls", + "rev": "14df0f7d229f4cd2e79f8eabb1a740097fdfa430", + "type": "github" + }, + "original": { + "owner": "sevanspowell", + "repo": "hpc-coveralls", + "type": "github" + } + }, + "ldap-client": { + "flake": false, + "locked": { + "lastModified": 1582401733, + "narHash": "sha256-StLj8BnQqbl8fnE+xWlSOScVRGnl19cv8d1CBZ29O0k=", + "ref": "uni2work", + "rev": "01afaf599ba6f8a9d804c269e91d3190b249d3f0", + "revCount": 61, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/ldap-client.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/ldap-client.git" + } + }, + "memcached-binary": { + "flake": false, + "locked": { + "lastModified": 1582403725, + "narHash": "sha256-40BNhNNYC/B4u16dKwBrkk5hmhncq805YC4y1aGFRqs=", + "ref": "uni2work", + "rev": "b7071df50bad3a251a544b984e4bf98fa09b8fae", + "revCount": 28, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/memcached-binary.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/memcached-binary.git" + } + }, + "minio-hs": { + "flake": false, + "locked": { + "lastModified": 1597069863, + "narHash": "sha256-JmMajaLT4+zt+w2koDkaloFL8ugmrQBlcYKj+78qn9M=", + "ref": "uni2work", + "rev": "42103ab247057c04c8ce7a83d9d4c160713a3df1", + "revCount": 197, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/minio-hs.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/minio-hs.git" + } + }, + "nix-tools": { + "flake": false, + "locked": { + "lastModified": 1626997434, + "narHash": "sha256-1judQmP298ao6cGUNxcGhcAXHOnA9qSLvWk/ZtoUL7w=", + "owner": "input-output-hk", + "repo": "nix-tools", + "rev": "c8c5e6a6fbb12a73598d1a434984a36e880ce3cf", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "repo": "nix-tools", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1624788075, - "narHash": "sha256-xzO2aL5gGejNvey2jKGnbnFXbo99pdytlY5FF/IhvAE=", + "lastModified": 1624291665, + "narHash": "sha256-kNkaoa3dai9WOi7fsPklCCWZ8hRAkXx0ZUhpYKShyUk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "4ba70da807359ed01d662763a96c7b442762e5ef", + "rev": "3c6f3f84af60a8ed5b8a79cf3026b7630fcdefb8", "type": "github" }, "original": { "owner": "NixOS", - "ref": "master", + "ref": "nixpkgs-20.09-darwin", "repo": "nixpkgs", "type": "github" } }, + "nixpkgs-2003": { + "locked": { + "lastModified": 1620055814, + "narHash": "sha256-8LEHoYSJiL901bTMVatq+rf8y7QtWuZhwwpKE2fyaRY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "1db42b7fe3878f3f5f7a4f2dc210772fd080e205", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-20.03-darwin", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-2009": { + "locked": { + "lastModified": 1624271064, + "narHash": "sha256-qns/uRW7MR2EfVf6VEeLgCsCp7pIOjDeR44JzTF09MA=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "46d1c3f28ca991601a53e9a14fdd53fcd3dd8416", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-20.09-darwin", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-2105": { + "locked": { + "lastModified": 1624291665, + "narHash": "sha256-kNkaoa3dai9WOi7fsPklCCWZ8hRAkXx0ZUhpYKShyUk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "3c6f3f84af60a8ed5b8a79cf3026b7630fcdefb8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-21.05-darwin", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1628785280, + "narHash": "sha256-2B5eMrEr6O8ff2aQNeVxTB+9WrGE80OB4+oM6T7fOcc=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "6525bbc06a39f26750ad8ee0d40000ddfdc24acb", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "old-ghc-nix": { + "flake": false, + "locked": { + "lastModified": 1621819714, + "narHash": "sha256-EJCnYQSWk7FRLwS0lZgTWIiQ6pcvDX1VuD6LGD4Uwzs=", + "owner": "angerman", + "repo": "old-ghc-nix", + "rev": "f089a6f090cdb35fcf95f865fc6a31ba6b3ac4eb", + "type": "github" + }, + "original": { + "owner": "angerman", + "ref": "master2", + "repo": "old-ghc-nix", + "type": "github" + } + }, "root": { "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" + "HaskellNet-SSL": "HaskellNet-SSL", + "colonnade": "colonnade", + "conduit-resumablesink": "conduit-resumablesink", + "cryptoids": "cryptoids", + "cryptonite": "cryptonite", + "encoding": "encoding", + "esqueleto": "esqueleto", + "flake-utils": [ + "haskell-nix", + "flake-utils" + ], + "fontawesome-token": "fontawesome-token", + "haskell-nix": "haskell-nix", + "ldap-client": "ldap-client", + "memcached-binary": "memcached-binary", + "minio-hs": "minio-hs", + "nixpkgs": [ + "haskell-nix", + "nixpkgs-unstable" + ], + "serversession": "serversession", + "tail-DateTime": "tail-DateTime", + "xss-sanitize": "xss-sanitize", + "yesod": "yesod", + "zip-stream": "zip-stream" + } + }, + "serversession": { + "flake": false, + "locked": { + "lastModified": 1624794082, + "narHash": "sha256-mixhAsgOq0iiAhOcZfFSyl8CwS7utaKeZBz/Lix16hk=", + "ref": "uni2work", + "rev": "b9d76def10da1260c7f6aa82bda32111f37a952b", + "revCount": 174, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/serversession.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/serversession.git" + } + }, + "stackage": { + "flake": false, + "locked": { + "lastModified": 1629940472, + "narHash": "sha256-LFrNC25OpNoS6dGG5om+UGP8YdYjp01Qm6cenbemaVg=", + "owner": "input-output-hk", + "repo": "stackage.nix", + "rev": "c50dd0527babf85818cc67a7099e532e453752db", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "repo": "stackage.nix", + "type": "github" + } + }, + "tail-DateTime": { + "flake": false, + "locked": { + "lastModified": 1576249828, + "narHash": "sha256-90okKfi5Nf5xw+IFsPXzdykdboLMbOYHmUVyNZS65Go=", + "ref": "uni2work", + "rev": "ef1237569ff22667acb64a9dfd64682ee55817eb", + "revCount": 132, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/tail.DateTime.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/tail.DateTime.git" + } + }, + "xss-sanitize": { + "flake": false, + "locked": { + "lastModified": 1602593438, + "narHash": "sha256-2jAHm79x8f+ygGHq9gipF3h8+eMT7eCxVMOGMc/brKE=", + "ref": "uni2work", + "rev": "dc928c3a456074b8777603bea20e81937321777f", + "revCount": 114, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/xss-sanitize.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/xss-sanitize.git" + } + }, + "yesod": { + "flake": false, + "locked": { + "lastModified": 1625061191, + "narHash": "sha256-K0X2MwUStChml1DlJ7t4yBMDwrMe6j/780nJtSy9Hss=", + "ref": "uni2work", + "rev": "a59f63e0336ee61f7a90b8778e9147305d3127bb", + "revCount": 5053, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/yesod.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/yesod.git" + } + }, + "zip-stream": { + "flake": false, + "locked": { + "lastModified": 1599734754, + "narHash": "sha256-gqIlgj2dxVkqEMafuVL+dqZPoV+WLpOJwz1NhsKnH64=", + "ref": "uni2work", + "rev": "843683d024f767de236f74d24a3348f69181a720", + "revCount": 39, + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/zip-stream.git" + }, + "original": { + "ref": "uni2work", + "type": "git", + "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/zip-stream.git" } } }, diff --git a/flake.nix b/flake.nix index bd6be4594..a05be898a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,32 +1,246 @@ { - inputs = { - nixpkgs = { - type = "github"; - owner = "NixOS"; - repo = "nixpkgs"; - ref = "master"; - }; - flake-utils = { - type = "github"; - owner = "numtide"; - repo = "flake-utils"; - ref = "master"; - }; + inputs.haskell-nix.url = "github:input-output-hk/haskell.nix"; + inputs.nixpkgs.follows = "haskell-nix/nixpkgs-unstable"; + inputs.flake-utils.follows = "haskell-nix/flake-utils"; + + inputs.encoding = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/encoding.git?ref=uni2work"; + flake = false; + }; + inputs.memcached-binary = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/memcached-binary.git?ref=uni2work"; + flake = false; + }; + inputs.conduit-resumablesink = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/conduit-resumablesink.git?ref=uni2work"; + flake = false; + }; + inputs.HaskellNet-SSL = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/HaskellNet-SSL.git?ref=uni2work"; + flake = false; + }; + inputs.ldap-client = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/ldap-client.git?ref=uni2work"; + flake = false; + }; + inputs.serversession = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/serversession.git?ref=uni2work"; + flake = false; + }; + inputs.xss-sanitize = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/xss-sanitize.git?ref=uni2work"; + flake = false; + }; + inputs.colonnade = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/colonnade.git?ref=uni2work"; + flake = false; + }; + inputs.minio-hs = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/minio-hs.git?ref=uni2work"; + flake = false; + }; + inputs.cryptoids = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/cryptoids.git?ref=uni2work"; + flake = false; + }; + inputs.zip-stream = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/zip-stream.git?ref=uni2work"; + flake = false; + }; + inputs.yesod = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/yesod.git?ref=uni2work"; + flake = false; + }; + inputs.cryptonite = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/cryptonite.git?ref=uni2work"; + flake = false; + }; + inputs.esqueleto = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/esqueleto.git?ref=uni2work"; + flake = false; }; - outputs = { self, nixpkgs, flake-utils, ... }: flake-utils.lib.eachDefaultSystem + inputs.tail-DateTime = { + url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/tail.DateTime.git?ref=uni2work"; + flake = false; + }; + inputs.fontawesome-token = { + url = "path:/etc/fontawesome-token"; + flake = false; + }; + + outputs = inputs@{ self, nixpkgs, flake-utils, haskell-nix, ... }: flake-utils.lib.eachSystem ["x86_64-linux"] (system: let pkgs = import nixpkgs { - inherit system; + inherit system overlays; config.allowUnfree = true; }; - overlay = import ./nix/maildev; + overlays = + let + frontendSource = pkgs.lib.sourceByRegex ./. [ + "^(assets|frontend)(/.*)?$" + "^config(/(favicon\.json|robots\.txt))?$" + "^(webpack|postcss)\.config\.js$" + "^(package|jsconfig)\.json$" + "^\.babelrc$" + ]; + backendSource = pkgs.lib.sourceByRegex ./. [ + "^(package|stack-flake)\.yaml$" + "^stack\.yaml\.lock$" + "^(assets|app|hlint|load|messages|models|src|templates|test|testdata|wflint)(/.*)?$" + "^config(/(archive-types|mimetypes|personalised-sheet-files-collate|settings\.yml|submission-blacklist|test-settings\.yml|video-types|wordlist\.txt))?$" + "^routes$" + ]; + in [ + (import ./nix/maildev) + haskell-nix.overlay + (final: prev: { + uniworx-node-dependencies = (prev.callPackage ./nix/frontend {}).nodeDependencies.override (oldArgs: { + dependencies = + let + srcOverrides = { + "tail.datetime" = inputs.tail-DateTime; + "@fortawesome/fontawesome-pro" = prev.fetchurl { + url = "https://npm.fontawesome.com/@fortawesome/fontawesome-pro/-/5.14.0/fontawesome-pro-5.14.0.tgz"; + curlOpts = "-H @${prev.writeText "headers.txt" "Authorization: Bearer ${builtins.readFile inputs.fontawesome-token}"}"; + hash = "sha256-jGvPrTKKL0rCWRZUEnJEmrOdHyQYs3M5709B1hjmFw4="; + }; + }; + in map (dep: dep // { src = srcOverrides."${dep.packageName}" or dep.src; }) oldArgs.dependencies; + }); + }) + (final: prev: { + uniworx-well-known = prev.stdenv.mkDerivation { + name = "uniworx-well-known"; + src = frontendSource; - inherit (pkgs.lib) recursiveUpdate; + phases = ["unpackPhase" "buildPhase" "installPhase" "fixupPhase"]; + + buildPhase = '' + ln -s ${prev.uniworx-node-dependencies}/lib/node_modules ./node_modules + export PATH="${prev.uniworx-node-dependencies}/bin:${prev.exiftool}/bin:$PATH" + webpack --progress + ''; + + installPhase = '' + mkdir -p $out + cp -r --reflink=auto well-known $out/.nix-well-known + ''; + + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = "tDaffdAT5EGPKdDJ2ovo9XSGdV48W3Efqe+iBmakh6g="; + }; + }) + (final: prev: { + uniworx-frontend = prev.stdenv.mkDerivation { + name = "uniworx-frontend"; + srcs = [frontendSource prev.uniworx-well-known]; + sourceRoot = "source"; + + phases = ["unpackPhase" "buildPhase" "installPhase"]; + + postUnpack = '' + cp -pr --reflink=auto uniworx-well-known/. $sourceRoot + ''; + + buildPhase = '' + ln -s ${prev.uniworx-node-dependencies}/lib/node_modules ./node_modules + export PATH="${prev.uniworx-node-dependencies}/bin:$PATH" + webpack --progress + ''; + + installPhase = '' + mkdir -p $out $out/config + cp -r --reflink=auto well-known static $out + cp -r --reflink=auto config/webpack.yml $out/config + ''; + }; + }) + (final: prev: { + uniworx = final.haskell-nix.stackProject { + src = prev.stdenv.mkDerivation { + name = "uniworx-src"; + src = backendSource; + + phases = ["unpackPhase" "patchPhase" "installPhase"]; + + patchPhase = '' + substitute stack-flake.yaml stack.yaml \ + ${prev.lib.concatMapStringsSep " \\\n" (pkgName: "--replace @${pkgName}@ ${inputs."${pkgName}"}") haskellInputs} + ''; + + installPhase = '' + mkdir -p $out + cp -pr --reflink=auto ./. $out + ''; + }; + compiler-nix-name = "ghc8104"; + # stack-sha256 = "1n7z294ldv2rjkfj1vs3kqmnbp34m2scrmyrp5kwmga9vp86fd9z"; + modules = [ + { + packages = { + encoding.src = inputs.encoding; + memcached-binary.src = inputs.memcached-binary; + conduit-resumablesink.src = inputs.conduit-resumablesink; + HaskellNet-SSL.src = inputs.HaskellNet-SSL; + ldap-client.src = inputs.ldap-client; + serversession.src = "${inputs.serversession}/serversession"; + serversession-backend-acid-state.src = "${inputs.serversession}/serversession-backend-acid-state"; + xss-sanitize.src = inputs.xss-sanitize; + colonnade.src = "${inputs.colonnade}/colonnade"; + minio-hs.src = inputs.minio-hs; + cryptoids-class.src = "${inputs.cryptoids}/cryptoids-class"; + cryptoids-types.src = "${inputs.cryptoids}/cryptoids-types"; + cryptoids.src = "${inputs.cryptoids}/cryptoids"; + filepath-crypto.src = "${inputs.cryptoids}/filepath-crypto"; + uuid-crypto.src = "${inputs.cryptoids}/uuid-crypto"; + zip-stream.src = inputs.zip-stream; + yesod.src = "${inputs.yesod}/yesod"; + yesod-core.src = "${inputs.yesod}/yesod-core"; + yesod-static.src = "${inputs.yesod}/yesod-static"; + yesod-persistent.src = "${inputs.yesod}/yesod-persistent"; + yesod-form.src = "${inputs.yesod}/yesod-form"; + yesod-auth.src = "${inputs.yesod}/yesod-auth"; + yesod-test.src = "${inputs.yesod}/yesod-test"; + cryptonite.src = inputs.cryptonite; + esqueleto.src = inputs.esqueleto; + }; + } + { + packages.uniworx = { + postUnpack = '' + cp -pr --reflink=auto ${prev.uniworx-frontend}/. $sourceRoot + chmod a+w -R $sourceRoot + ''; + preBuild = '' + export TZDIR=${final.tzdata}/share/zoneinfo + ''; + components.library.build-tools = with final.pkgs; [ llvm_9 ]; + components.exes.uniworx.build-tools = with final.pkgs; [ llvm_9 ]; + components.exes.uniworxdb.build-tools = with final.pkgs; [ llvm_9 ]; + components.exes.uniworxload.build-tools = with final.pkgs; [ llvm_9 ]; + components.exes.uniworx-wflint.build-tools = with final.pkgs; [ llvm_9 ]; + components.tests.yesod.build-tools = with final.pkgs; [ llvm_9 ]; + components.tests.hlint.build-tools = with final.pkgs; [ llvm_9 ]; + }; + } + ]; + }; + }) + ]; + haskellInputs = ["encoding" "memcached-binary" "conduit-resumablesink" "HaskellNet-SSL" "ldap-client" "serversession" "xss-sanitize" "colonnade" "minio-hs" "cryptoids" "zip-stream" "yesod" "cryptonite" "esqueleto"]; + + haskellFlake = pkgs.uniworx.flake {}; in { - devShell = import ./shell.nix { pkgs = self.legacyPackages.${system}; }; + packages = haskellFlake.packages // { inherit (pkgs) uniworx-node-dependencies uniworx-well-known uniworx-frontend; inherit (pkgs.uniworx.stack-nix.passthru) calculateMaterializedSha; }; + inherit (haskellFlake) checks apps; + + devShell = import ./shell.nix { inherit pkgs; }; - legacyPackages = recursiveUpdate (overlay self.legacyPackages.${system} pkgs) pkgs; + legacyPackages = pkgs; + + defaultPackage = haskellFlake.packages."uniworx:exe:uniworx"; } ); } diff --git a/nix/frontend/default.nix b/nix/frontend/default.nix new file mode 100644 index 000000000..f929727d5 --- /dev/null +++ b/nix/frontend/default.nix @@ -0,0 +1,17 @@ +# This file has been generated by node2nix 1.9.0. Do not edit! + +{pkgs ? import { + inherit system; + }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-14_x"}: + +let + nodeEnv = import ./node-env.nix { + inherit (pkgs) stdenv lib python2 runCommand writeTextFile; + inherit pkgs nodejs; + libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null; + }; +in +import ./node-packages.nix { + inherit (pkgs) fetchurl nix-gitignore stdenv lib fetchgit; + inherit nodeEnv; +} diff --git a/nix/frontend/node-env.nix b/nix/frontend/node-env.nix new file mode 100644 index 000000000..c2b723195 --- /dev/null +++ b/nix/frontend/node-env.nix @@ -0,0 +1,567 @@ +# This file originates from node2nix + +{lib, stdenv, nodejs, python2, pkgs, libtool, runCommand, writeTextFile}: + +let + # Workaround to cope with utillinux in Nixpkgs 20.09 and util-linux in Nixpkgs master + utillinux = if pkgs ? utillinux then pkgs.utillinux else pkgs.util-linux; + + python = if nodejs ? python then nodejs.python else python2; + + # Create a tar wrapper that filters all the 'Ignoring unknown extended header keyword' noise + tarWrapper = runCommand "tarWrapper" {} '' + mkdir -p $out/bin + + cat > $out/bin/tar <> $out/nix-support/hydra-build-products + ''; + }; + + includeDependencies = {dependencies}: + lib.optionalString (dependencies != []) + (lib.concatMapStrings (dependency: + '' + # Bundle the dependencies of the package + mkdir -p node_modules + cd node_modules + + # Only include dependencies if they don't exist. They may also be bundled in the package. + if [ ! -e "${dependency.name}" ] + then + ${composePackage dependency} + fi + + cd .. + '' + ) dependencies); + + # Recursively composes the dependencies of a package + composePackage = { name, packageName, src, dependencies ? [], ... }@args: + builtins.addErrorContext "while evaluating node package '${packageName}'" '' + DIR=$(pwd) + cd $TMPDIR + + unpackFile ${src} + + # Make the base dir in which the target dependency resides first + mkdir -p "$(dirname "$DIR/${packageName}")" + + if [ -f "${src}" ] + then + # Figure out what directory has been unpacked + packageDir="$(find . -maxdepth 1 -type d | tail -1)" + + # Restore write permissions to make building work + find "$packageDir" -type d -exec chmod u+x {} \; + chmod -R u+w "$packageDir" + + # Move the extracted tarball into the output folder + mv "$packageDir" "$DIR/${packageName}" + elif [ -d "${src}" ] + then + # Get a stripped name (without hash) of the source directory. + # On old nixpkgs it's already set internally. + if [ -z "$strippedName" ] + then + strippedName="$(stripHash ${src})" + fi + + # Restore write permissions to make building work + chmod -R u+w "$strippedName" + + # Move the extracted directory into the output folder + mv "$strippedName" "$DIR/${packageName}" + fi + + # Unset the stripped name to not confuse the next unpack step + unset strippedName + + # Include the dependencies of the package + cd "$DIR/${packageName}" + ${includeDependencies { inherit dependencies; }} + cd .. + ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + ''; + + pinpointDependencies = {dependencies, production}: + let + pinpointDependenciesFromPackageJSON = writeTextFile { + name = "pinpointDependencies.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + function resolveDependencyVersion(location, name) { + if(location == process.env['NIX_STORE']) { + return null; + } else { + var dependencyPackageJSON = path.join(location, "node_modules", name, "package.json"); + + if(fs.existsSync(dependencyPackageJSON)) { + var dependencyPackageObj = JSON.parse(fs.readFileSync(dependencyPackageJSON)); + + if(dependencyPackageObj.name == name) { + return dependencyPackageObj.version; + } + } else { + return resolveDependencyVersion(path.resolve(location, ".."), name); + } + } + } + + function replaceDependencies(dependencies) { + if(typeof dependencies == "object" && dependencies !== null) { + for(var dependency in dependencies) { + var resolvedVersion = resolveDependencyVersion(process.cwd(), dependency); + + if(resolvedVersion === null) { + process.stderr.write("WARNING: cannot pinpoint dependency: "+dependency+", context: "+process.cwd()+"\n"); + } else { + dependencies[dependency] = resolvedVersion; + } + } + } + } + + /* Read the package.json configuration */ + var packageObj = JSON.parse(fs.readFileSync('./package.json')); + + /* Pinpoint all dependencies */ + replaceDependencies(packageObj.dependencies); + if(process.argv[2] == "development") { + replaceDependencies(packageObj.devDependencies); + } + replaceDependencies(packageObj.optionalDependencies); + + /* Write the fixed package.json file */ + fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2)); + ''; + }; + in + '' + node ${pinpointDependenciesFromPackageJSON} ${if production then "production" else "development"} + + ${lib.optionalString (dependencies != []) + '' + if [ -d node_modules ] + then + cd node_modules + ${lib.concatMapStrings (dependency: pinpointDependenciesOfPackage dependency) dependencies} + cd .. + fi + ''} + ''; + + # Recursively traverses all dependencies of a package and pinpoints all + # dependencies in the package.json file to the versions that are actually + # being used. + + pinpointDependenciesOfPackage = { packageName, dependencies ? [], production ? true, ... }@args: + '' + if [ -d "${packageName}" ] + then + cd "${packageName}" + ${pinpointDependencies { inherit dependencies production; }} + cd .. + ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + fi + ''; + + # Extract the Node.js source code which is used to compile packages with + # native bindings + nodeSources = runCommand "node-sources" {} '' + tar --no-same-owner --no-same-permissions -xf ${nodejs.src} + mv node-* $out + ''; + + # Script that adds _integrity fields to all package.json files to prevent NPM from consulting the cache (that is empty) + addIntegrityFieldsScript = writeTextFile { + name = "addintegrityfields.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + function augmentDependencies(baseDir, dependencies) { + for(var dependencyName in dependencies) { + var dependency = dependencies[dependencyName]; + + // Open package.json and augment metadata fields + var packageJSONDir = path.join(baseDir, "node_modules", dependencyName); + var packageJSONPath = path.join(packageJSONDir, "package.json"); + + if(fs.existsSync(packageJSONPath)) { // Only augment packages that exist. Sometimes we may have production installs in which development dependencies can be ignored + console.log("Adding metadata fields to: "+packageJSONPath); + var packageObj = JSON.parse(fs.readFileSync(packageJSONPath)); + + if(dependency.integrity) { + packageObj["_integrity"] = dependency.integrity; + } else { + packageObj["_integrity"] = "sha1-000000000000000000000000000="; // When no _integrity string has been provided (e.g. by Git dependencies), add a dummy one. It does not seem to harm and it bypasses downloads. + } + + if(dependency.resolved) { + packageObj["_resolved"] = dependency.resolved; // Adopt the resolved property if one has been provided + } else { + packageObj["_resolved"] = dependency.version; // Set the resolved version to the version identifier. This prevents NPM from cloning Git repositories. + } + + if(dependency.from !== undefined) { // Adopt from property if one has been provided + packageObj["_from"] = dependency.from; + } + + fs.writeFileSync(packageJSONPath, JSON.stringify(packageObj, null, 2)); + } + + // Augment transitive dependencies + if(dependency.dependencies !== undefined) { + augmentDependencies(packageJSONDir, dependency.dependencies); + } + } + } + + if(fs.existsSync("./package-lock.json")) { + var packageLock = JSON.parse(fs.readFileSync("./package-lock.json")); + + if(![1, 2].includes(packageLock.lockfileVersion)) { + process.stderr.write("Sorry, I only understand lock file versions 1 and 2!\n"); + process.exit(1); + } + + if(packageLock.dependencies !== undefined) { + augmentDependencies(".", packageLock.dependencies); + } + } + ''; + }; + + # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes + reconstructPackageLock = writeTextFile { + name = "addintegrityfields.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + var packageObj = JSON.parse(fs.readFileSync("package.json")); + + var lockObj = { + name: packageObj.name, + version: packageObj.version, + lockfileVersion: 1, + requires: true, + dependencies: {} + }; + + function augmentPackageJSON(filePath, dependencies) { + var packageJSON = path.join(filePath, "package.json"); + if(fs.existsSync(packageJSON)) { + var packageObj = JSON.parse(fs.readFileSync(packageJSON)); + dependencies[packageObj.name] = { + version: packageObj.version, + integrity: "sha1-000000000000000000000000000=", + dependencies: {} + }; + processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies); + } + } + + function processDependencies(dir, dependencies) { + if(fs.existsSync(dir)) { + var files = fs.readdirSync(dir); + + files.forEach(function(entry) { + var filePath = path.join(dir, entry); + var stats = fs.statSync(filePath); + + if(stats.isDirectory()) { + if(entry.substr(0, 1) == "@") { + // When we encounter a namespace folder, augment all packages belonging to the scope + var pkgFiles = fs.readdirSync(filePath); + + pkgFiles.forEach(function(entry) { + if(stats.isDirectory()) { + var pkgFilePath = path.join(filePath, entry); + augmentPackageJSON(pkgFilePath, dependencies); + } + }); + } else { + augmentPackageJSON(filePath, dependencies); + } + } + }); + } + } + + processDependencies("node_modules", lockObj.dependencies); + + fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2)); + ''; + }; + + prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}: + let + forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com"; + in + '' + # Pinpoint the versions of all dependencies to the ones that are actually being used + echo "pinpointing versions of dependencies..." + source $pinpointDependenciesScriptPath + + # Patch the shebangs of the bundled modules to prevent them from + # calling executables outside the Nix store as much as possible + patchShebangs . + + # Deploy the Node.js package by running npm install. Since the + # dependencies have been provided already by ourselves, it should not + # attempt to install them again, which is good, because we want to make + # it Nix's responsibility. If it needs to install any dependencies + # anyway (e.g. because the dependency parameters are + # incomplete/incorrect), it fails. + # + # The other responsibilities of NPM are kept -- version checks, build + # steps, postprocessing etc. + + export HOME=$TMPDIR + cd "${packageName}" + runHook preRebuild + + ${lib.optionalString bypassCache '' + ${lib.optionalString reconstructLock '' + if [ -f package-lock.json ] + then + echo "WARNING: Reconstruct lock option enabled, but a lock file already exists!" + echo "This will most likely result in version mismatches! We will remove the lock file and regenerate it!" + rm package-lock.json + else + echo "No package-lock.json file found, reconstructing..." + fi + + node ${reconstructPackageLock} + ''} + + node ${addIntegrityFieldsScript} + ''} + + npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild + + if [ "''${dontNpmInstall-}" != "1" ] + then + # NPM tries to download packages even when they already exist if npm-shrinkwrap is used. + rm -f npm-shrinkwrap.json + + npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} install + fi + ''; + + # Builds and composes an NPM package including all its dependencies + buildNodePackage = + { name + , packageName + , version + , dependencies ? [] + , buildInputs ? [] + , production ? true + , npmFlags ? "" + , dontNpmInstall ? false + , bypassCache ? false + , reconstructLock ? false + , preRebuild ? "" + , dontStrip ? true + , unpackPhase ? "true" + , buildPhase ? "true" + , ... }@args: + + let + extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "preRebuild" "unpackPhase" "buildPhase" ]; + in + stdenv.mkDerivation ({ + name = "node_${name}-${version}"; + buildInputs = [ tarWrapper python nodejs ] + ++ lib.optional (stdenv.isLinux) utillinux + ++ lib.optional (stdenv.isDarwin) libtool + ++ buildInputs; + + inherit nodejs; + + inherit dontStrip; # Stripping may fail a build for some package deployments + inherit dontNpmInstall preRebuild unpackPhase buildPhase; + + compositionScript = composePackage args; + pinpointDependenciesScript = pinpointDependenciesOfPackage args; + + passAsFile = [ "compositionScript" "pinpointDependenciesScript" ]; + + installPhase = '' + # Create and enter a root node_modules/ folder + mkdir -p $out/lib/node_modules + cd $out/lib/node_modules + + # Compose the package and all its dependencies + source $compositionScriptPath + + ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }} + + # Create symlink to the deployed executable folder, if applicable + if [ -d "$out/lib/node_modules/.bin" ] + then + ln -s $out/lib/node_modules/.bin $out/bin + fi + + # Create symlinks to the deployed manual page folders, if applicable + if [ -d "$out/lib/node_modules/${packageName}/man" ] + then + mkdir -p $out/share + for dir in "$out/lib/node_modules/${packageName}/man/"* + do + mkdir -p $out/share/man/$(basename "$dir") + for page in "$dir"/* + do + ln -s $page $out/share/man/$(basename "$dir") + done + done + fi + + # Run post install hook, if provided + runHook postInstall + ''; + } // extraArgs); + + # Builds a node environment (a node_modules folder and a set of binaries) + buildNodeDependencies = + { name + , packageName + , version + , src + , dependencies ? [] + , buildInputs ? [] + , production ? true + , npmFlags ? "" + , dontNpmInstall ? false + , bypassCache ? false + , reconstructLock ? false + , dontStrip ? true + , unpackPhase ? "true" + , buildPhase ? "true" + , ... }@args: + + let + extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" ]; + in + stdenv.mkDerivation ({ + name = "node-dependencies-${name}-${version}"; + + buildInputs = [ tarWrapper python nodejs ] + ++ lib.optional (stdenv.isLinux) utillinux + ++ lib.optional (stdenv.isDarwin) libtool + ++ buildInputs; + + inherit dontStrip; # Stripping may fail a build for some package deployments + inherit dontNpmInstall unpackPhase buildPhase; + + includeScript = includeDependencies { inherit dependencies; }; + pinpointDependenciesScript = pinpointDependenciesOfPackage args; + + passAsFile = [ "includeScript" "pinpointDependenciesScript" ]; + + installPhase = '' + mkdir -p $out/${packageName} + cd $out/${packageName} + + source $includeScriptPath + + # Create fake package.json to make the npm commands work properly + cp ${src}/package.json . + chmod 644 package.json + ${lib.optionalString bypassCache '' + if [ -f ${src}/package-lock.json ] + then + cp ${src}/package-lock.json . + fi + ''} + + # Go to the parent folder to make sure that all packages are pinpointed + cd .. + ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + + ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }} + + # Expose the executables that were installed + cd .. + ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + + mv ${packageName} lib + ln -s $out/lib/node_modules/.bin $out/bin + ''; + } // extraArgs); + + # Builds a development shell + buildNodeShell = + { name + , packageName + , version + , src + , dependencies ? [] + , buildInputs ? [] + , production ? true + , npmFlags ? "" + , dontNpmInstall ? false + , bypassCache ? false + , reconstructLock ? false + , dontStrip ? true + , unpackPhase ? "true" + , buildPhase ? "true" + , ... }@args: + + let + nodeDependencies = buildNodeDependencies args; + in + stdenv.mkDerivation { + name = "node-shell-${name}-${version}"; + + buildInputs = [ python nodejs ] ++ lib.optional (stdenv.isLinux) utillinux ++ buildInputs; + buildCommand = '' + mkdir -p $out/bin + cat > $out/bin/shell < vObj.relevance.automated_update).slice(-1)[0].version; - } catch(e) { - console.error(e); + if (!fs.existsSync(faviconNixDirectory)) { + try { + const faviconApiChangelog = await request({ + method: 'GET', + uri: 'https://realfavicongenerator.net/api/versions', + headers: { + 'Accept': '*/*' + }, + json: true + }); + faviconApiVersion = faviconApiChangelog.filter(vObj => vObj.relevance.automated_update).slice(-1)[0].version; + } catch(e) { + console.error(e); + } } return { @@ -224,7 +227,16 @@ async function webpackConfig() { const cacheDirectory = path.resolve(__dirname, '.well-known-cache', `${cacheDigest}-${lang}`); - if (fs.existsSync(cacheDirectory) && (!faviconApiVersion || faviconApiVersion === cachedVersion)) { + if (fs.existsSync(faviconNixDirectory)) { + console.log("Using favicons generated by nix"); + return [ + new CopyPlugin({ + patterns: [ + { from: path.resolve(faviconNixDirectory, lang), to: path.resolve(__dirname, 'well-known', lang) } + ] + }) + ]; + } else if (fs.existsSync(cacheDirectory) && (!faviconApiVersion || faviconApiVersion === cachedVersion)) { console.log(`Using cached well-known from ${cacheDirectory} for ${lang}`); return [ new CopyPlugin({ From fa54517d15bba2d6b5bb5d1673e4b41cf3cf2e8c Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Thu, 2 Sep 2021 14:23:13 +0200 Subject: [PATCH 02/52] chore(nix): build docker container --- flake.nix | 190 +++++------------------------- nix/docker-demo/default.nix | 102 ++++++++++++++++ nix/uniworx/backend.nix | 74 ++++++++++++ nix/uniworx/default.nix | 6 + nix/uniworx/frontend.nix | 25 ++++ nix/uniworx/node-dependencies.nix | 15 +++ nix/uniworx/well-known.nix | 23 ++++ 7 files changed, 277 insertions(+), 158 deletions(-) create mode 100644 nix/docker-demo/default.nix create mode 100644 nix/uniworx/backend.nix create mode 100644 nix/uniworx/default.nix create mode 100644 nix/uniworx/frontend.nix create mode 100644 nix/uniworx/node-dependencies.nix create mode 100644 nix/uniworx/well-known.nix diff --git a/flake.nix b/flake.nix index a05be898a..719bc4e4c 100644 --- a/flake.nix +++ b/flake.nix @@ -71,176 +71,50 @@ outputs = inputs@{ self, nixpkgs, flake-utils, haskell-nix, ... }: flake-utils.lib.eachSystem ["x86_64-linux"] (system: - let pkgs = import nixpkgs { + let frontendSource = pkgs.lib.sourceByRegex ./. [ + "^(assets|frontend)(/.*)?$" + "^config(/(favicon\.json|robots\.txt))?$" + "^(webpack|postcss)\.config\.js$" + "^(package|jsconfig)\.json$" + "^\.babelrc$" + ]; + backendSource = pkgs.lib.sourceByRegex ./. [ + "^(package|stack-flake)\.yaml$" + "^stack\.yaml\.lock$" + "^(assets|app|hlint|load|messages|models|src|templates|test|testdata|wflint)(/.*)?$" + "^config(/(archive-types|mimetypes|personalised-sheet-files-collate|settings\.yml|submission-blacklist|test-settings\.yml|video-types|wordlist\.txt))?$" + "^routes$" + ]; + + pkgs = import nixpkgs { inherit system overlays; config.allowUnfree = true; }; - overlays = - let - frontendSource = pkgs.lib.sourceByRegex ./. [ - "^(assets|frontend)(/.*)?$" - "^config(/(favicon\.json|robots\.txt))?$" - "^(webpack|postcss)\.config\.js$" - "^(package|jsconfig)\.json$" - "^\.babelrc$" - ]; - backendSource = pkgs.lib.sourceByRegex ./. [ - "^(package|stack-flake)\.yaml$" - "^stack\.yaml\.lock$" - "^(assets|app|hlint|load|messages|models|src|templates|test|testdata|wflint)(/.*)?$" - "^config(/(archive-types|mimetypes|personalised-sheet-files-collate|settings\.yml|submission-blacklist|test-settings\.yml|video-types|wordlist\.txt))?$" - "^routes$" - ]; - in [ - (import ./nix/maildev) - haskell-nix.overlay - (final: prev: { - uniworx-node-dependencies = (prev.callPackage ./nix/frontend {}).nodeDependencies.override (oldArgs: { - dependencies = - let - srcOverrides = { - "tail.datetime" = inputs.tail-DateTime; - "@fortawesome/fontawesome-pro" = prev.fetchurl { - url = "https://npm.fontawesome.com/@fortawesome/fontawesome-pro/-/5.14.0/fontawesome-pro-5.14.0.tgz"; - curlOpts = "-H @${prev.writeText "headers.txt" "Authorization: Bearer ${builtins.readFile inputs.fontawesome-token}"}"; - hash = "sha256-jGvPrTKKL0rCWRZUEnJEmrOdHyQYs3M5709B1hjmFw4="; - }; - }; - in map (dep: dep // { src = srcOverrides."${dep.packageName}" or dep.src; }) oldArgs.dependencies; - }); - }) - (final: prev: { - uniworx-well-known = prev.stdenv.mkDerivation { - name = "uniworx-well-known"; - src = frontendSource; - - phases = ["unpackPhase" "buildPhase" "installPhase" "fixupPhase"]; - - buildPhase = '' - ln -s ${prev.uniworx-node-dependencies}/lib/node_modules ./node_modules - export PATH="${prev.uniworx-node-dependencies}/bin:${prev.exiftool}/bin:$PATH" - webpack --progress - ''; - - installPhase = '' - mkdir -p $out - cp -r --reflink=auto well-known $out/.nix-well-known - ''; - - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - outputHash = "tDaffdAT5EGPKdDJ2ovo9XSGdV48W3Efqe+iBmakh6g="; - }; - }) - (final: prev: { - uniworx-frontend = prev.stdenv.mkDerivation { - name = "uniworx-frontend"; - srcs = [frontendSource prev.uniworx-well-known]; - sourceRoot = "source"; - - phases = ["unpackPhase" "buildPhase" "installPhase"]; - - postUnpack = '' - cp -pr --reflink=auto uniworx-well-known/. $sourceRoot - ''; - - buildPhase = '' - ln -s ${prev.uniworx-node-dependencies}/lib/node_modules ./node_modules - export PATH="${prev.uniworx-node-dependencies}/bin:$PATH" - webpack --progress - ''; - - installPhase = '' - mkdir -p $out $out/config - cp -r --reflink=auto well-known static $out - cp -r --reflink=auto config/webpack.yml $out/config - ''; - }; - }) - (final: prev: { - uniworx = final.haskell-nix.stackProject { - src = prev.stdenv.mkDerivation { - name = "uniworx-src"; - src = backendSource; - - phases = ["unpackPhase" "patchPhase" "installPhase"]; - - patchPhase = '' - substitute stack-flake.yaml stack.yaml \ - ${prev.lib.concatMapStringsSep " \\\n" (pkgName: "--replace @${pkgName}@ ${inputs."${pkgName}"}") haskellInputs} - ''; - - installPhase = '' - mkdir -p $out - cp -pr --reflink=auto ./. $out - ''; - }; - compiler-nix-name = "ghc8104"; - # stack-sha256 = "1n7z294ldv2rjkfj1vs3kqmnbp34m2scrmyrp5kwmga9vp86fd9z"; - modules = [ - { - packages = { - encoding.src = inputs.encoding; - memcached-binary.src = inputs.memcached-binary; - conduit-resumablesink.src = inputs.conduit-resumablesink; - HaskellNet-SSL.src = inputs.HaskellNet-SSL; - ldap-client.src = inputs.ldap-client; - serversession.src = "${inputs.serversession}/serversession"; - serversession-backend-acid-state.src = "${inputs.serversession}/serversession-backend-acid-state"; - xss-sanitize.src = inputs.xss-sanitize; - colonnade.src = "${inputs.colonnade}/colonnade"; - minio-hs.src = inputs.minio-hs; - cryptoids-class.src = "${inputs.cryptoids}/cryptoids-class"; - cryptoids-types.src = "${inputs.cryptoids}/cryptoids-types"; - cryptoids.src = "${inputs.cryptoids}/cryptoids"; - filepath-crypto.src = "${inputs.cryptoids}/filepath-crypto"; - uuid-crypto.src = "${inputs.cryptoids}/uuid-crypto"; - zip-stream.src = inputs.zip-stream; - yesod.src = "${inputs.yesod}/yesod"; - yesod-core.src = "${inputs.yesod}/yesod-core"; - yesod-static.src = "${inputs.yesod}/yesod-static"; - yesod-persistent.src = "${inputs.yesod}/yesod-persistent"; - yesod-form.src = "${inputs.yesod}/yesod-form"; - yesod-auth.src = "${inputs.yesod}/yesod-auth"; - yesod-test.src = "${inputs.yesod}/yesod-test"; - cryptonite.src = inputs.cryptonite; - esqueleto.src = inputs.esqueleto; - }; - } - { - packages.uniworx = { - postUnpack = '' - cp -pr --reflink=auto ${prev.uniworx-frontend}/. $sourceRoot - chmod a+w -R $sourceRoot - ''; - preBuild = '' - export TZDIR=${final.tzdata}/share/zoneinfo - ''; - components.library.build-tools = with final.pkgs; [ llvm_9 ]; - components.exes.uniworx.build-tools = with final.pkgs; [ llvm_9 ]; - components.exes.uniworxdb.build-tools = with final.pkgs; [ llvm_9 ]; - components.exes.uniworxload.build-tools = with final.pkgs; [ llvm_9 ]; - components.exes.uniworx-wflint.build-tools = with final.pkgs; [ llvm_9 ]; - components.tests.yesod.build-tools = with final.pkgs; [ llvm_9 ]; - components.tests.hlint.build-tools = with final.pkgs; [ llvm_9 ]; - }; - } - ]; - }; - }) + overlays = [ + (import ./nix/maildev) + haskell-nix.overlay + (import ./nix/uniworx { inherit inputs frontendSource backendSource; }) + (import ./nix/docker-demo { inherit self; }) ]; - haskellInputs = ["encoding" "memcached-binary" "conduit-resumablesink" "HaskellNet-SSL" "ldap-client" "serversession" "xss-sanitize" "colonnade" "minio-hs" "cryptoids" "zip-stream" "yesod" "cryptonite" "esqueleto"]; - haskellFlake = pkgs.uniworx.flake {}; + + pushUniworxDemoDocker = pkgs.writeScriptBin "push-uniworx-demo-docker" '' + #!${pkgs.zsh}/bin/zsh -xe + + target=''${1-docker://registry.gitlab.com/fradrive/fradrive/uniworx-demo} + [[ -n "''${1}" ]] && shift + ${pkgs.skopeo}/bin/skopeo ''${@} --insecure-policy copy docker-archive://${pkgs.uniworxDemoDocker} ''${target} + ''; in { packages = haskellFlake.packages // { inherit (pkgs) uniworx-node-dependencies uniworx-well-known uniworx-frontend; inherit (pkgs.uniworx.stack-nix.passthru) calculateMaterializedSha; }; - inherit (haskellFlake) checks apps; + apps = haskellFlake.apps // { push-uniworx-demo-docker = flake-utils.lib.mkApp { drv = pushUniworxDemoDocker; }; }; + inherit (haskellFlake) checks; devShell = import ./shell.nix { inherit pkgs; }; legacyPackages = pkgs; - defaultPackage = haskellFlake.packages."uniworx:exe:uniworx"; + defaultPackage = pkgs.uniworxDemoDocker; } ); } diff --git a/nix/docker-demo/default.nix b/nix/docker-demo/default.nix new file mode 100644 index 000000000..77bb59c5f --- /dev/null +++ b/nix/docker-demo/default.nix @@ -0,0 +1,102 @@ +{ self }: final: prev: { + uniworxDemoDocker = prev.dockerTools.buildImage { + name = "uniworx-demo"; + tag = self.rev or null; + created = + let + fromDate = builtins.readFile (prev.runCommand "date" { nativeBuildInputs = with final; [ coreutils ]; } '' + printf '%s' $(date -Is -d '@${toString self.lastModified}') > $out + ''); + in if self ? lastModified then fromDate else "1970-01-01T00:00:01Z"; + + contents = with final; [ + uniworx.uniworx.components.exes.uniworx + prev.dockerTools.binSh postgresql_12 + memcached + ]; + + runAsRoot = '' + #!${final.stdenv.shell} + + ${prev.dockerTools.shadowSetup} + + mkdir -p /var/lib + + groupadd -r postgres + useradd -r -g postgres -d /var/lib/postgres -M postgres + install -d -g postgres -o postgres -m 0750 /var/lib/postgres + + groupadd -r memcached + useradd -r -g memcached -d /var/lib/memcached -M memcached + install -d -g memcached -o memcached -m 0750 /var/lib/memcached + + groupadd -r uniworx + useradd -r -g uniworx -d /var/lib/uniworx -M uniworx + install -d -g uniworx -o uniworx -m 0750 /var/lib/uniworx + gpasswd -a uniworx postgres + + mkdir -p /var/log + install -d -g postgres -o postgres -m 0755 /var/log/postgres + install -d -g memcached -o memcached -m 0755 /var/log/memcached + install -d -g uniworx -o uniworx -m 0755 /var/log/uniworx + + mkdir -p /run + install -d -g postgres -o postgres -m 0755 /run/postgres + ''; + + config = + let + entrypoint = prev.writeScriptBin "uniworx-entrypoint" '' + #!${final.zsh}/bin/zsh -xe + export PATH=${final.su}/bin:${final.findutils}/bin:${final.coreutils}/bin:/bin + + cTime=$(date -Is) + + pgDir=/var/lib/postgres + pgSockDir=/run/postgres + pgLogFile=/var/log/postgres/''${cTime}.log + export PGHOST=''${pgSockDir} + export PGLOG=''${pgLogFile} + + pgNew= + if [[ -n "$(find ''${pgDir} -maxdepth 0 -type d -empty 2>/dev/null)" ]]; then + pgNew=1 + fi + + [[ -z "''${pgNew}" ]] || su postgres -c "initdb --no-locale --encoding=UTF8 -D ''${pgDir}" + su postgres -c "pg_ctl start -D ''${pgDir} -l ''${pgLogFile} -w -o '-k ''${pgSockDir} -c listen_addresses= -c hba_file=${postgresHba} -c unix_socket_permissions=0777 -c max_connections=9990 -c shared_preload_libraries=pg_stat_statements -c auto_explain.log_min_duration=100ms'" + [[ -z "''${pgNew}" ]] || psql -f ${postgresSchema} postgres postgres + + su memcached -c "cd /var/lib/memcached; memcached -p 11212" &>/var/log/memcached/''${cTime}.log & + export SESSION_MEMCACHED_HOST=localhost + export SESSION_MEMCACHED_PORT=11212 + + export LOGDEST=/var/log/uniworx/''${cTime}.log + exec -- su uniworx -c "cd /var/lib/uniworx; uniworx ${uniworxConfig}" + ''; + postgresSchema = prev.writeText "schema.sql" '' + CREATE USER uniworx WITH SUPERUSER; + CREATE DATABASE uniworx; + GRANT ALL ON DATABASE uniworx TO uniworx; + ''; + + postgresHba = prev.writeText "hba_file" '' + local all all trust + ''; + uniworxConfig = prev.writeText "uni2work.yml" '' + port: 8080 + approot: "_env:APPROOT:http://localhost:8080" + ''; + in { + Cmd = [ "${entrypoint}/bin/uniworx-entrypoint" ]; + ExposedPorts = { + "8080/tcp" = {}; + }; + Volumes = { + "/var/lib/postgres" = {}; + "/var/lib/uniworx" = {}; + "/var/log" = {}; + }; + }; + }; +} diff --git a/nix/uniworx/backend.nix b/nix/uniworx/backend.nix new file mode 100644 index 000000000..f77c199a0 --- /dev/null +++ b/nix/uniworx/backend.nix @@ -0,0 +1,74 @@ +{ inputs, backendSource, ... }: final: prev: +let + haskellInputs = ["encoding" "memcached-binary" "conduit-resumablesink" "HaskellNet-SSL" "ldap-client" "serversession" "xss-sanitize" "colonnade" "minio-hs" "cryptoids" "zip-stream" "yesod" "cryptonite" "esqueleto"]; +in { + uniworx = final.haskell-nix.stackProject { + src = prev.stdenv.mkDerivation { + name = "uniworx-src"; + src = backendSource; + + phases = ["unpackPhase" "patchPhase" "installPhase"]; + + patchPhase = '' + substitute stack-flake.yaml stack.yaml \ + ${prev.lib.concatMapStringsSep " \\\n" (pkgName: "--replace @${pkgName}@ ${inputs."${pkgName}"}") haskellInputs} + ''; + + installPhase = '' + mkdir -p $out + cp -pr --reflink=auto ./. $out + ''; + }; + compiler-nix-name = "ghc8104"; + # stack-sha256 = "1n7z294ldv2rjkfj1vs3kqmnbp34m2scrmyrp5kwmga9vp86fd9z"; + modules = [ + { + packages = { + encoding.src = inputs.encoding; + memcached-binary.src = inputs.memcached-binary; + conduit-resumablesink.src = inputs.conduit-resumablesink; + HaskellNet-SSL.src = inputs.HaskellNet-SSL; + ldap-client.src = inputs.ldap-client; + serversession.src = "${inputs.serversession}/serversession"; + serversession-backend-acid-state.src = "${inputs.serversession}/serversession-backend-acid-state"; + xss-sanitize.src = inputs.xss-sanitize; + colonnade.src = "${inputs.colonnade}/colonnade"; + minio-hs.src = inputs.minio-hs; + cryptoids-class.src = "${inputs.cryptoids}/cryptoids-class"; + cryptoids-types.src = "${inputs.cryptoids}/cryptoids-types"; + cryptoids.src = "${inputs.cryptoids}/cryptoids"; + filepath-crypto.src = "${inputs.cryptoids}/filepath-crypto"; + uuid-crypto.src = "${inputs.cryptoids}/uuid-crypto"; + zip-stream.src = inputs.zip-stream; + yesod.src = "${inputs.yesod}/yesod"; + yesod-core.src = "${inputs.yesod}/yesod-core"; + yesod-static.src = "${inputs.yesod}/yesod-static"; + yesod-persistent.src = "${inputs.yesod}/yesod-persistent"; + yesod-form.src = "${inputs.yesod}/yesod-form"; + yesod-auth.src = "${inputs.yesod}/yesod-auth"; + yesod-test.src = "${inputs.yesod}/yesod-test"; + cryptonite.src = inputs.cryptonite; + esqueleto.src = inputs.esqueleto; + }; + } + { + packages.uniworx = { + postUnpack = '' + cp -pr --reflink=auto ${prev.uniworx-frontend}/. $sourceRoot + chmod a+w -R $sourceRoot + ''; + preBuild = '' + export TZDIR=${final.tzdata}/share/zoneinfo + ''; + components.library.build-tools = with final.pkgs; [ llvm_9 ]; + components.exes.uniworx.build-tools = with final.pkgs; [ llvm_9 ]; + components.exes.uniworxdb.build-tools = with final.pkgs; [ llvm_9 ]; + components.exes.uniworxload.build-tools = with final.pkgs; [ llvm_9 ]; + components.exes.uniworx-wflint.build-tools = with final.pkgs; [ llvm_9 ]; + components.tests.yesod.build-tools = with final.pkgs; [ llvm_9 ]; + components.tests.hlint.build-tools = with final.pkgs; [ llvm_9 ]; + }; + } + ]; + }; +} diff --git a/nix/uniworx/default.nix b/nix/uniworx/default.nix new file mode 100644 index 000000000..c353be222 --- /dev/null +++ b/nix/uniworx/default.nix @@ -0,0 +1,6 @@ +{ inputs, frontendSource, backendSource }: final: prev: prev.lib.composeManyExtensions [ + (import ./node-dependencies.nix { inherit inputs; }) + (import ./well-known.nix { inherit frontendSource; }) + (import ./frontend.nix { inherit frontendSource; }) + (import ./backend.nix { inherit backendSource inputs; }) +] final prev diff --git a/nix/uniworx/frontend.nix b/nix/uniworx/frontend.nix new file mode 100644 index 000000000..bbffa391e --- /dev/null +++ b/nix/uniworx/frontend.nix @@ -0,0 +1,25 @@ +{ frontendSource, ... }: final: prev: { + uniworx-frontend = prev.stdenv.mkDerivation { + name = "uniworx-frontend"; + srcs = [frontendSource prev.uniworx-well-known]; + sourceRoot = "source"; + + phases = ["unpackPhase" "buildPhase" "installPhase"]; + + postUnpack = '' + cp -pr --reflink=auto uniworx-well-known/. $sourceRoot + ''; + + buildPhase = '' + ln -s ${prev.uniworx-node-dependencies}/lib/node_modules ./node_modules + export PATH="${prev.uniworx-node-dependencies}/bin:$PATH" + webpack --progress + ''; + + installPhase = '' + mkdir -p $out $out/config + cp -r --reflink=auto well-known static $out + cp -r --reflink=auto config/webpack.yml $out/config + ''; + }; +} diff --git a/nix/uniworx/node-dependencies.nix b/nix/uniworx/node-dependencies.nix new file mode 100644 index 000000000..7893f47b1 --- /dev/null +++ b/nix/uniworx/node-dependencies.nix @@ -0,0 +1,15 @@ +{ inputs, ... }: final: prev: { + uniworx-node-dependencies = (prev.callPackage ../frontend {}).nodeDependencies.override (oldArgs: { + dependencies = + let + srcOverrides = { + "tail.datetime" = inputs.tail-DateTime; + "@fortawesome/fontawesome-pro" = prev.fetchurl { + url = "https://npm.fontawesome.com/@fortawesome/fontawesome-pro/-/5.14.0/fontawesome-pro-5.14.0.tgz"; + curlOpts = "-H @${prev.writeText "headers.txt" "Authorization: Bearer ${builtins.readFile inputs.fontawesome-token}"}"; + hash = "sha256-jGvPrTKKL0rCWRZUEnJEmrOdHyQYs3M5709B1hjmFw4="; + }; + }; + in map (dep: dep // { src = srcOverrides."${dep.packageName}" or dep.src; }) oldArgs.dependencies; + }); +} diff --git a/nix/uniworx/well-known.nix b/nix/uniworx/well-known.nix new file mode 100644 index 000000000..615269a9e --- /dev/null +++ b/nix/uniworx/well-known.nix @@ -0,0 +1,23 @@ +{ frontendSource, ... }: final: prev: { + uniworx-well-known = prev.stdenv.mkDerivation { + name = "uniworx-well-known"; + src = frontendSource; + + phases = ["unpackPhase" "buildPhase" "installPhase" "fixupPhase"]; + + buildPhase = '' + ln -s ${prev.uniworx-node-dependencies}/lib/node_modules ./node_modules + export PATH="${prev.uniworx-node-dependencies}/bin:${prev.exiftool}/bin:$PATH" + webpack --progress + ''; + + installPhase = '' + mkdir -p $out + cp -r --reflink=auto well-known $out/.nix-well-known + ''; + + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = "tDaffdAT5EGPKdDJ2ovo9XSGdV48W3Efqe+iBmakh6g="; + }; +} From b7b5a24be66bb8a4acca7b92d4601969a7a00bf0 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Mon, 13 Sep 2021 09:13:28 +0200 Subject: [PATCH 03/52] chore(docker): versioning --- flake.nix | 2 +- nix/docker-demo/default.nix | 17 +++++++++++++++-- nix/docker-demo/version.toml | 3 +++ 3 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 nix/docker-demo/version.toml diff --git a/flake.nix b/flake.nix index 719bc4e4c..19b133a8f 100644 --- a/flake.nix +++ b/flake.nix @@ -101,7 +101,7 @@ pushUniworxDemoDocker = pkgs.writeScriptBin "push-uniworx-demo-docker" '' #!${pkgs.zsh}/bin/zsh -xe - target=''${1-docker://registry.gitlab.com/fradrive/fradrive/uniworx-demo} + target=''${1-docker://registry.gitlab.com/fradrive/fradrive/uniworx-demo:${pkgs.uniworxDemoDocker.imageTag}} [[ -n "''${1}" ]] && shift ${pkgs.skopeo}/bin/skopeo ''${@} --insecure-policy copy docker-archive://${pkgs.uniworxDemoDocker} ''${target} ''; diff --git a/nix/docker-demo/default.nix b/nix/docker-demo/default.nix index 77bb59c5f..aaaa4224b 100644 --- a/nix/docker-demo/default.nix +++ b/nix/docker-demo/default.nix @@ -1,7 +1,11 @@ { self }: final: prev: { uniworxDemoDocker = prev.dockerTools.buildImage { name = "uniworx-demo"; - tag = self.rev or null; + tag = + let + inherit (builtins.fromTOML (prev.lib.readFile ./version.toml)) semver; + hash = prev.lib.substring 0 8 (self.rev or prev.lib.removePrefix "sha256-" self.narHash); + in "${semver}+${hash}"; created = let fromDate = builtins.readFile (prev.runCommand "date" { nativeBuildInputs = with final; [ coreutils ]; } '' @@ -72,7 +76,16 @@ export SESSION_MEMCACHED_PORT=11212 export LOGDEST=/var/log/uniworx/''${cTime}.log - exec -- su uniworx -c "cd /var/lib/uniworx; uniworx ${uniworxConfig}" + typeset -a configs + configs=() + configDir=''${CONFIG_DIR-/cfg} + if [[ -d "''${configDir}" ]]; then + while IFS= read -d $'\0' cfg; do + configs+=("''${(q)cfg}") + done < <(find "''${configDir}" \( -name '*.yml' -o -name '*.yaml' \) -print0 | sort -rz) + fi + configs+=('${uniworxConfig}') + exec -- su uniworx -c "cd /var/lib/uniworx; uniworx ''${configs}" ''; postgresSchema = prev.writeText "schema.sql" '' CREATE USER uniworx WITH SUPERUSER; diff --git a/nix/docker-demo/version.toml b/nix/docker-demo/version.toml new file mode 100644 index 000000000..6b0106c9b --- /dev/null +++ b/nix/docker-demo/version.toml @@ -0,0 +1,3 @@ +# TODO: Bump version (https://semver.org) on every semantic change +# docker-demo should always use major version zero +semver = "0.2.0" \ No newline at end of file From bbe49cb2ff5d8f519e3773f35c6d6400c5d62966 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Mon, 13 Sep 2021 09:23:15 +0200 Subject: [PATCH 04/52] chore(docker): fix versioning --- flake.nix | 2 +- nix/docker-demo/default.nix | 5 +++-- nix/docker-demo/version.json | 3 +++ nix/docker-demo/version.toml | 3 --- 4 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 nix/docker-demo/version.json delete mode 100644 nix/docker-demo/version.toml diff --git a/flake.nix b/flake.nix index 19b133a8f..73816ffc9 100644 --- a/flake.nix +++ b/flake.nix @@ -106,7 +106,7 @@ ${pkgs.skopeo}/bin/skopeo ''${@} --insecure-policy copy docker-archive://${pkgs.uniworxDemoDocker} ''${target} ''; in { - packages = haskellFlake.packages // { inherit (pkgs) uniworx-node-dependencies uniworx-well-known uniworx-frontend; inherit (pkgs.uniworx.stack-nix.passthru) calculateMaterializedSha; }; + packages = haskellFlake.packages // { inherit (pkgs) uniworx-node-dependencies uniworx-well-known uniworx-frontend uniworxDemoDocker; inherit (pkgs.uniworx.stack-nix.passthru) calculateMaterializedSha; }; apps = haskellFlake.apps // { push-uniworx-demo-docker = flake-utils.lib.mkApp { drv = pushUniworxDemoDocker; }; }; inherit (haskellFlake) checks; diff --git a/nix/docker-demo/default.nix b/nix/docker-demo/default.nix index aaaa4224b..e67e5c54e 100644 --- a/nix/docker-demo/default.nix +++ b/nix/docker-demo/default.nix @@ -3,8 +3,9 @@ name = "uniworx-demo"; tag = let - inherit (builtins.fromTOML (prev.lib.readFile ./version.toml)) semver; - hash = prev.lib.substring 0 8 (self.rev or prev.lib.removePrefix "sha256-" self.narHash); + versionJSON = builtins.fromJSON (prev.lib.readFile ./version.json); + semver = versionJSON.version; + hash = prev.lib.substring 0 8 (self.rev or (prev.lib.removePrefix "sha256-" self.narHash)); in "${semver}+${hash}"; created = let diff --git a/nix/docker-demo/version.json b/nix/docker-demo/version.json new file mode 100644 index 000000000..10e64e00d --- /dev/null +++ b/nix/docker-demo/version.json @@ -0,0 +1,3 @@ +{ + "version": "0.2.0" +} diff --git a/nix/docker-demo/version.toml b/nix/docker-demo/version.toml deleted file mode 100644 index 6b0106c9b..000000000 --- a/nix/docker-demo/version.toml +++ /dev/null @@ -1,3 +0,0 @@ -# TODO: Bump version (https://semver.org) on every semantic change -# docker-demo should always use major version zero -semver = "0.2.0" \ No newline at end of file From 1c9fe6e0cb531017fc059bb95a4b9672daccb2fa Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Mon, 13 Sep 2021 10:40:52 +0200 Subject: [PATCH 05/52] chore(docker): remove semver metadata --- nix/docker-demo/default.nix | 7 +------ shell.nix | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/nix/docker-demo/default.nix b/nix/docker-demo/default.nix index e67e5c54e..317d7205d 100644 --- a/nix/docker-demo/default.nix +++ b/nix/docker-demo/default.nix @@ -1,12 +1,7 @@ { self }: final: prev: { uniworxDemoDocker = prev.dockerTools.buildImage { name = "uniworx-demo"; - tag = - let - versionJSON = builtins.fromJSON (prev.lib.readFile ./version.json); - semver = versionJSON.version; - hash = prev.lib.substring 0 8 (self.rev or (prev.lib.removePrefix "sha256-" self.narHash)); - in "${semver}+${hash}"; + tag = (builtins.fromJSON (prev.lib.readFile ./version.json)).version; created = let fromDate = builtins.readFile (prev.runCommand "date" { nativeBuildInputs = with final; [ coreutils ]; } '' diff --git a/shell.nix b/shell.nix index 2ab614f6a..0d5fe3975 100644 --- a/shell.nix +++ b/shell.nix @@ -260,5 +260,5 @@ let ''; in pkgs.mkShell { name = "uni2work"; - nativeBuildInputs = [develop inDevelop killallUni2work diffRunning] ++ (with pkgs; [ nodejs-14_x postgresql_12 openldap google-chrome exiftool memcached minio minio-client gup ]) ++ (with pkgs.haskellPackages; [ stack yesod-bin hlint cabal-install weeder profiteur ]); + nativeBuildInputs = [develop inDevelop killallUni2work diffRunning] ++ (with pkgs; [ nodejs-14_x postgresql_12 openldap google-chrome exiftool memcached minio minio-client gup skopeo ]) ++ (with pkgs.haskellPackages; [ stack yesod-bin hlint cabal-install weeder profiteur ]); } From fd89f34746c8be908ab6885fadf8cc934f157c3e Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Mon, 13 Sep 2021 17:09:06 +0200 Subject: [PATCH 06/52] chore(docker): build production container --- .versionrc.js | 4 + flake.nix | 30 ++- nix/develop.nix | 212 ++++++++++++++++++ nix/docker-demo/default.nix | 111 --------- nix/docker/default.nix | 127 +++++++++++ .../version.json => docker/demo-version.json} | 0 nix/docker/version.json | 3 + nix/uniworx/backend.nix | 19 +- nix/uniworx/frontend.nix | 47 +++- nix/uniworx/node-dependencies.nix | 2 +- nix/uniworx/well-known.nix | 6 +- shell.nix | 198 +--------------- 12 files changed, 426 insertions(+), 333 deletions(-) create mode 100644 nix/develop.nix delete mode 100644 nix/docker-demo/default.nix create mode 100644 nix/docker/default.nix rename nix/{docker-demo/version.json => docker/demo-version.json} (100%) create mode 100644 nix/docker/version.json diff --git a/.versionrc.js b/.versionrc.js index 50f38f817..4a65f1245 100644 --- a/.versionrc.js +++ b/.versionrc.js @@ -18,6 +18,10 @@ module.exports = { { filename: 'package.yaml', updater: standardVersionUpdaterYaml + }, + { + filename: 'nix/docker/version.json', + type: 'json' } ], commitUrlFormat: 'https://gitlab2.rz.ifi.lmu.de/uni2work/uni2work/commit/{{hash}}', diff --git a/flake.nix b/flake.nix index 73816ffc9..c0e07cded 100644 --- a/flake.nix +++ b/flake.nix @@ -75,11 +75,12 @@ "^(assets|frontend)(/.*)?$" "^config(/(favicon\.json|robots\.txt))?$" "^(webpack|postcss)\.config\.js$" - "^(package|jsconfig)\.json$" + "^karma\.conf\.js$" + "^(package|jsconfig|\.eslintrc)\.json$" "^\.babelrc$" ]; backendSource = pkgs.lib.sourceByRegex ./. [ - "^(package|stack-flake)\.yaml$" + "^(\.hlint|package|stack-flake)\.yaml$" "^stack\.yaml\.lock$" "^(assets|app|hlint|load|messages|models|src|templates|test|testdata|wflint)(/.*)?$" "^config(/(archive-types|mimetypes|personalised-sheet-files-collate|settings\.yml|submission-blacklist|test-settings\.yml|video-types|wordlist\.txt))?$" @@ -94,27 +95,36 @@ (import ./nix/maildev) haskell-nix.overlay (import ./nix/uniworx { inherit inputs frontendSource backendSource; }) - (import ./nix/docker-demo { inherit self; }) + (import ./nix/docker { inherit self; }) ]; haskellFlake = pkgs.uniworx.flake {}; - pushUniworxDemoDocker = pkgs.writeScriptBin "push-uniworx-demo-docker" '' + mkPushUniworxDocker = dockerImage: pkgs.writeScriptBin "push-${dockerImage.name}" '' #!${pkgs.zsh}/bin/zsh -xe - target=''${1-docker://registry.gitlab.com/fradrive/fradrive/uniworx-demo:${pkgs.uniworxDemoDocker.imageTag}} + target=''${1-docker://registry.gitlab.com/fradrive/fradrive/uniworx-demo:${dockerImage.imageTag}} [[ -n "''${1}" ]] && shift - ${pkgs.skopeo}/bin/skopeo ''${@} --insecure-policy copy docker-archive://${pkgs.uniworxDemoDocker} ''${target} + ${pkgs.skopeo}/bin/skopeo ''${@} --insecure-policy copy docker-archive://${dockerImage} ''${target} ''; in { - packages = haskellFlake.packages // { inherit (pkgs) uniworx-node-dependencies uniworx-well-known uniworx-frontend uniworxDemoDocker; inherit (pkgs.uniworx.stack-nix.passthru) calculateMaterializedSha; }; - apps = haskellFlake.apps // { push-uniworx-demo-docker = flake-utils.lib.mkApp { drv = pushUniworxDemoDocker; }; }; - inherit (haskellFlake) checks; + packages = haskellFlake.packages // { + inherit (pkgs) uniworxNodeDependencies uniworxWellKnown uniworxFrontend uniworxDemoDocker uniworxDocker; + }; + apps = haskellFlake.apps // { + pushUniworxDemoDocker = flake-utils.lib.mkApp { drv = mkPushUniworxDocker pkgs.uniworxDemoDocker; }; + pushUniworxDocker = flake-utils.lib.mkApp { drv = mkPushUniworxDocker pkgs.uniworxDocker; }; + calculateMaterializedSha = flake-utils.lib.mkApp { drv = pkgs.uniworx.stack-nix.passthru.calculateMaterializedSha; exePath = ""; }; + }; + checks = haskellFlake.checks // { + uniworxFrontend = pkgs.uniworxFrontend.check; + }; devShell = import ./shell.nix { inherit pkgs; }; legacyPackages = pkgs; - defaultPackage = pkgs.uniworxDemoDocker; + defaultPackage = self.packages.${system}."uniworx:exe:uniworx"; + defaultApp = self.apps.${system}."uniworx:exe:uniworx"; } ); } diff --git a/nix/develop.nix b/nix/develop.nix new file mode 100644 index 000000000..03a89f0ef --- /dev/null +++ b/nix/develop.nix @@ -0,0 +1,212 @@ +{ pkgs +, doPortOffset ? true +, doDevelopEnv ? true +}: + +with pkgs.lib; + +let + withDevelop = action: '' + #!${pkgs.zsh}/bin/zsh -e + + ${optionalString doDevelopEnv '' + basePath=$(pwd) + exec 4<>''${basePath}/.develop.env + + flockRes= + set +e + ${pkgs.util-linux}/bin/flock -en 4; flockRes=$? + set -e + if [[ ''${flockRes} -ne 0 ]]; then + echo "Could not take exclusive lock; is another develop running?" >&2 + exit ''${flockRes} + fi + ''} + + cleanup() { + set +e -x + type cleanup_postgres &>/dev/null && cleanup_postgres + type cleanup_widget_memcached &>/dev/null && cleanup_widget_memcached + type cleanup_session_memcached &>/dev/null && cleanup_session_memcached + type cleanup_cache_memcached &>/dev/null && cleanup_cache_memcached + type cleanup_minio &>/dev/null && cleanup_minio + type cleanup_maildev &>/dev/null && cleanup_maildev + + ${optionalString doDevelopEnv '' + [ -f "''${basePath}/.develop.env" ] && rm -vf "''${basePath}/.develop.env" + ''} + set +x + } + + trap cleanup EXIT + + export PORT_OFFSET=${if doPortOffset then "$(((16#$(echo \"fradrive $(whoami)\" | sha256sum | head -c 16)) % 1000))" else "0"} + + if [[ -z "$PGHOST" ]]; then + set -xe + + pgDir=$(mktemp -d --tmpdir=''${XDG_RUNTIME_DIR} postgresql.XXXXXX) + pgSockDir=$(mktemp -d --tmpdir=''${XDG_RUNTIME_DIR} postgresql.sock.XXXXXX) + pgLogFile=$(mktemp --tmpdir=''${XDG_RUNTIME_DIR} postgresql.XXXXXX.log) + initdb --no-locale -D ''${pgDir} + pg_ctl start -D ''${pgDir} -l ''${pgLogFile} -w -o "-k ''${pgSockDir} -c listen_addresses=''' -c hba_file='${postgresHba}' -c unix_socket_permissions=0700 -c max_connections=9990 -c shared_preload_libraries=pg_stat_statements -c auto_explain.log_min_duration=100ms" + psql -h ''${pgSockDir} -f ${postgresSchema} postgres + printf "Postgres logfile is %s\nPostgres socket directory is %s\n" ''${pgLogFile} ''${pgSockDir} + + export PGHOST=''${pgSockDir} + export PGLOG=''${pgLogFile} + + cleanup_postgres() { + set +e -x + pg_ctl stop -D ''${pgDir} + rm -rvf ''${pgDir} ''${pgSockDir} ''${pgLogFile} + set +x + } + + set +xe + fi + + if [[ -z "$WIDGET_MEMCACHED_HOST" ]]; then + set -xe + + memcached -l localhost -p $(($PORT_OFFSET + 11211)) &>/dev/null & + widget_memcached_pid=$! + + export WIDGET_MEMCACHED_HOST=localhost + export WIDGET_MEMCACHED_PORT=$(($PORT_OFFSET + 11211)) + + cleanup_widget_memcached() { + [[ -n "$widget_memcached_pid" ]] && kill $widget_memcached_pid + } + + set +xe + fi + + if [[ -z "$SESSION_MEMCACHED_HOST" ]]; then + set -xe + + memcached -l localhost -p $(($PORT_OFFSET + 11212)) &>/dev/null & + session_memcached_pid=$! + + export SESSION_MEMCACHED_HOST=localhost + export SESSION_MEMCACHED_PORT=$(($PORT_OFFSET + 11212)) + + cleanup_session_memcached() { + [[ -n "$session_memcached_pid" ]] && kill $session_memcached_pid + } + + set +xe + fi + + if [[ -z "$MEMCACHED_HOST" ]]; then + set -xe + + memcached -l localhost -p $(($PORT_OFFSET + 11213)) &>/dev/null & + memcached_pid=$! + + export MEMCACHED_HOST=localhost + export MEMCACHED_PORT=$(($PORT_OFFSET + 11212)) + + cleanup_session_memcached() { + [[ -n "$memcached_pid" ]] && kill $memcached_pid + } + + set +xe + fi + + if [[ -z "$UPLOAD_S3_HOST" ]]; then + set -xe + + cleanup_minio() { + [[ -n "$minio_pid" ]] && kill $minio_pid + [[ -n "''${MINIO_DIR}" ]] && rm -rvf ''${MINIO_DIR} + [[ -n "''${MINIO_LOGFILE}" ]] && rm -rvf ''${MINIO_LOGFILE} + } + + export MINIO_DIR=$(mktemp -d --tmpdir=''${XDG_RUNTIME_DIR} minio.XXXXXX) + export MINIO_LOGFILE=$(mktemp --tmpdir=''${XDG_RUNTIME_DIR} minio.XXXXXX.log) + export MINIO_ACCESS_KEY=$(${pkgs.pwgen}/bin/pwgen -s 16 1) + export MINIO_SECRET_KEY=$(${pkgs.pwgen}/bin/pwgen -s 32 1) + + minio server --address localhost:$(($PORT_OFFSET + 9000)) ''${MINIO_DIR} &>''${MINIO_LOGFILE} & + minio_pid=$! + + export UPLOAD_S3_HOST=localhost + export UPLOAD_S3_PORT=$(($PORT_OFFSET + 9000)) + export UPLOAD_S3_SSL=false + export UPLOAD_S3_KEY_ID=''${MINIO_ACCESS_KEY} + export UPLOAD_S3_KEY=''${MINIO_SECRET_KEY} + + sleep 1 + + set +xe + fi + + ${optionalString (pkgs.nodePackages ? "maildev") '' + if [[ -z "$SMTPHOST" ]]; then + set -xe + + cleanup_maildev() { + [[ -n "$maildev_pid" ]] && kill $maildev_pid + } + + TMPDIR=''${XDG_RUNTIME_DIR} ${pkgs.nodePackages.maildev}/bin/maildev --smtp $(($PORT_OFFSET + 1025)) --web $(($PORT_OFFSET + 8080)) --ip localhost --web-ip localhost &>/dev/null & + maildev_pid=$! + + export SMTPHOST=localhost + export SMTPPORT=$(($PORT_OFFSET + 1025)) + export SMTPSSL=none + + set +xe + fi + ''} + + ${optionalString doDevelopEnv '' + set -xe + + cat >&4 < $out - ''); - in if self ? lastModified then fromDate else "1970-01-01T00:00:01Z"; - - contents = with final; [ - uniworx.uniworx.components.exes.uniworx - prev.dockerTools.binSh postgresql_12 - memcached - ]; - - runAsRoot = '' - #!${final.stdenv.shell} - - ${prev.dockerTools.shadowSetup} - - mkdir -p /var/lib - - groupadd -r postgres - useradd -r -g postgres -d /var/lib/postgres -M postgres - install -d -g postgres -o postgres -m 0750 /var/lib/postgres - - groupadd -r memcached - useradd -r -g memcached -d /var/lib/memcached -M memcached - install -d -g memcached -o memcached -m 0750 /var/lib/memcached - - groupadd -r uniworx - useradd -r -g uniworx -d /var/lib/uniworx -M uniworx - install -d -g uniworx -o uniworx -m 0750 /var/lib/uniworx - gpasswd -a uniworx postgres - - mkdir -p /var/log - install -d -g postgres -o postgres -m 0755 /var/log/postgres - install -d -g memcached -o memcached -m 0755 /var/log/memcached - install -d -g uniworx -o uniworx -m 0755 /var/log/uniworx - - mkdir -p /run - install -d -g postgres -o postgres -m 0755 /run/postgres - ''; - - config = - let - entrypoint = prev.writeScriptBin "uniworx-entrypoint" '' - #!${final.zsh}/bin/zsh -xe - export PATH=${final.su}/bin:${final.findutils}/bin:${final.coreutils}/bin:/bin - - cTime=$(date -Is) - - pgDir=/var/lib/postgres - pgSockDir=/run/postgres - pgLogFile=/var/log/postgres/''${cTime}.log - export PGHOST=''${pgSockDir} - export PGLOG=''${pgLogFile} - - pgNew= - if [[ -n "$(find ''${pgDir} -maxdepth 0 -type d -empty 2>/dev/null)" ]]; then - pgNew=1 - fi - - [[ -z "''${pgNew}" ]] || su postgres -c "initdb --no-locale --encoding=UTF8 -D ''${pgDir}" - su postgres -c "pg_ctl start -D ''${pgDir} -l ''${pgLogFile} -w -o '-k ''${pgSockDir} -c listen_addresses= -c hba_file=${postgresHba} -c unix_socket_permissions=0777 -c max_connections=9990 -c shared_preload_libraries=pg_stat_statements -c auto_explain.log_min_duration=100ms'" - [[ -z "''${pgNew}" ]] || psql -f ${postgresSchema} postgres postgres - - su memcached -c "cd /var/lib/memcached; memcached -p 11212" &>/var/log/memcached/''${cTime}.log & - export SESSION_MEMCACHED_HOST=localhost - export SESSION_MEMCACHED_PORT=11212 - - export LOGDEST=/var/log/uniworx/''${cTime}.log - typeset -a configs - configs=() - configDir=''${CONFIG_DIR-/cfg} - if [[ -d "''${configDir}" ]]; then - while IFS= read -d $'\0' cfg; do - configs+=("''${(q)cfg}") - done < <(find "''${configDir}" \( -name '*.yml' -o -name '*.yaml' \) -print0 | sort -rz) - fi - configs+=('${uniworxConfig}') - exec -- su uniworx -c "cd /var/lib/uniworx; uniworx ''${configs}" - ''; - postgresSchema = prev.writeText "schema.sql" '' - CREATE USER uniworx WITH SUPERUSER; - CREATE DATABASE uniworx; - GRANT ALL ON DATABASE uniworx TO uniworx; - ''; - - postgresHba = prev.writeText "hba_file" '' - local all all trust - ''; - uniworxConfig = prev.writeText "uni2work.yml" '' - port: 8080 - approot: "_env:APPROOT:http://localhost:8080" - ''; - in { - Cmd = [ "${entrypoint}/bin/uniworx-entrypoint" ]; - ExposedPorts = { - "8080/tcp" = {}; - }; - Volumes = { - "/var/lib/postgres" = {}; - "/var/lib/uniworx" = {}; - "/var/log" = {}; - }; - }; - }; -} diff --git a/nix/docker/default.nix b/nix/docker/default.nix new file mode 100644 index 000000000..558dadfde --- /dev/null +++ b/nix/docker/default.nix @@ -0,0 +1,127 @@ +{ self }: final: prev: + +with prev.lib; + +let + mkUniworxDocker = { isDemo }: prev.dockerTools.buildImage { + name = "uniworx${optionalString isDemo "-demo"}"; + tag = + let + versionFile = if isDemo then ./demo-version.json else ./version.json; + in (builtins.fromJSON (prev.lib.readFile versionFile)).version; + created = + let + fromDate = builtins.readFile (prev.runCommand "date" { nativeBuildInputs = with final; [ coreutils ]; } '' + printf '%s' $(date -Is -d '@${toString self.lastModified}') > $out + ''); + in if self ? lastModified then fromDate else "1970-01-01T00:00:01Z"; + + contents = with final; [ + uniworx.uniworx.components.exes.uniworx + prev.dockerTools.binSh + ] ++ optionals isDemo [ postgresql_12 memcached ]; + + runAsRoot = '' + #!${final.stdenv.shell} + + ${prev.dockerTools.shadowSetup} + + mkdir -p /var/lib + + groupadd -r uniworx + useradd -r -g uniworx -d /var/lib/uniworx -M uniworx + install -d -g uniworx -o uniworx -m 0750 /var/lib/uniworx + + mkdir -p /var/log + install -d -g uniworx -o uniworx -m 0755 /var/log/uniworx + + ${optionalString isDemo '' + groupadd -r postgres + useradd -r -g postgres -d /var/lib/postgres -M postgres + install -d -g postgres -o postgres -m 0750 /var/lib/postgres + + groupadd -r memcached + useradd -r -g memcached -d /var/lib/memcached -M memcached + install -d -g memcached -o memcached -m 0750 /var/lib/memcached + + gpasswd -a uniworx postgres + + install -d -g postgres -o postgres -m 0755 /var/log/postgres + install -d -g memcached -o memcached -m 0755 /var/log/memcached + + mkdir -p /run + install -d -g postgres -o postgres -m 0755 /run/postgres + ''} + ''; + + config = + let + entrypoint = prev.writeScriptBin "uniworx-entrypoint" '' + #!${final.zsh}/bin/zsh -xe + export PATH=${final.su}/bin:${final.findutils}/bin:${final.coreutils}/bin:/bin + + cTime=$(date -Is) + + ${optionalString isDemo '' + pgDir=/var/lib/postgres + pgSockDir=/run/postgres + pgLogFile=/var/log/postgres/''${cTime}.log + export PGHOST=''${pgSockDir} + export PGLOG=''${pgLogFile} + + pgNew= + if [[ -n "$(find ''${pgDir} -maxdepth 0 -type d -empty 2>/dev/null)" ]]; then + pgNew=1 + fi + + [[ -z "''${pgNew}" ]] || su postgres -c "initdb --no-locale --encoding=UTF8 -D ''${pgDir}" + su postgres -c "pg_ctl start -D ''${pgDir} -l ''${pgLogFile} -w -o '-k ''${pgSockDir} -c listen_addresses= -c hba_file=${postgresHba} -c unix_socket_permissions=0777 -c max_connections=9990 -c shared_preload_libraries=pg_stat_statements -c auto_explain.log_min_duration=100ms'" + [[ -z "''${pgNew}" ]] || psql -f ${postgresSchema} postgres postgres + + su memcached -c "cd /var/lib/memcached; memcached -p 11212" &>/var/log/memcached/''${cTime}.log & + export SESSION_MEMCACHED_HOST=localhost + export SESSION_MEMCACHED_PORT=11212 + ''} + + export LOGDEST=/var/log/uniworx/''${cTime}.log + typeset -a configs + configs=() + configDir=''${CONFIG_DIR-/cfg} + if [[ -d "''${configDir}" ]]; then + while IFS= read -d $'\0' cfg; do + configs+=("''${(q)cfg}") + done < <(find "''${configDir}" \( -name '*.yml' -o -name '*.yaml' \) -print0 | sort -rz) + fi + configs+=('${uniworxConfig}') + exec -- su uniworx -c "cd /var/lib/uniworx; uniworx ''${configs}" + ''; + postgresSchema = prev.writeText "schema.sql" '' + CREATE USER uniworx WITH SUPERUSER; + CREATE DATABASE uniworx; + GRANT ALL ON DATABASE uniworx TO uniworx; + ''; + + postgresHba = prev.writeText "hba_file" '' + local all all trust + ''; + uniworxConfig = prev.writeText "uni2work.yml" '' + port: 8080 + approot: "_env:APPROOT:http://localhost:8080" + ''; + in { + Cmd = [ "${entrypoint}/bin/uniworx-entrypoint" ]; + ExposedPorts = { + "8080/tcp" = {}; + }; + Volumes = { + "/var/lib/uniworx" = {}; + "/var/log" = {}; + } // optionalAttrs isDemo { + "/var/lib/postgres" = {}; + }; + }; + }; +in mapAttrs (_name: mkUniworxDocker) { + uniworxDemoDocker = { isDemo = true; }; + uniworxDocker = { isDemo = false; }; +} diff --git a/nix/docker-demo/version.json b/nix/docker/demo-version.json similarity index 100% rename from nix/docker-demo/version.json rename to nix/docker/demo-version.json diff --git a/nix/docker/version.json b/nix/docker/version.json new file mode 100644 index 000000000..74e9a96b4 --- /dev/null +++ b/nix/docker/version.json @@ -0,0 +1,3 @@ +{ + "version": "25.21.0" +} diff --git a/nix/uniworx/backend.nix b/nix/uniworx/backend.nix index f77c199a0..2a31434b7 100644 --- a/nix/uniworx/backend.nix +++ b/nix/uniworx/backend.nix @@ -1,4 +1,7 @@ { inputs, backendSource, ... }: final: prev: + +with prev.lib; + let haskellInputs = ["encoding" "memcached-binary" "conduit-resumablesink" "HaskellNet-SSL" "ldap-client" "serversession" "xss-sanitize" "colonnade" "minio-hs" "cryptoids" "zip-stream" "yesod" "cryptonite" "esqueleto"]; in { @@ -11,7 +14,7 @@ in { patchPhase = '' substitute stack-flake.yaml stack.yaml \ - ${prev.lib.concatMapStringsSep " \\\n" (pkgName: "--replace @${pkgName}@ ${inputs."${pkgName}"}") haskellInputs} + ${concatMapStringsSep " \\\n" (pkgName: "--replace @${pkgName}@ ${inputs."${pkgName}"}") haskellInputs} ''; installPhase = '' @@ -20,7 +23,7 @@ in { ''; }; compiler-nix-name = "ghc8104"; - # stack-sha256 = "1n7z294ldv2rjkfj1vs3kqmnbp34m2scrmyrp5kwmga9vp86fd9z"; + # stack-sha256 = "1n7z294ldv2rjkfj1vs3kqmnbp34m2scrmyrp5kwmga9vp86fd9z"; # produces errors gregor does not understand :( modules = [ { packages = { @@ -54,7 +57,7 @@ in { { packages.uniworx = { postUnpack = '' - cp -pr --reflink=auto ${prev.uniworx-frontend}/. $sourceRoot + ${final.xorg.lndir}/bin/lndir -silent ${prev.uniworxFrontend} $sourceRoot chmod a+w -R $sourceRoot ''; preBuild = '' @@ -65,8 +68,14 @@ in { components.exes.uniworxdb.build-tools = with final.pkgs; [ llvm_9 ]; components.exes.uniworxload.build-tools = with final.pkgs; [ llvm_9 ]; components.exes.uniworx-wflint.build-tools = with final.pkgs; [ llvm_9 ]; - components.tests.yesod.build-tools = with final.pkgs; [ llvm_9 ]; - components.tests.hlint.build-tools = with final.pkgs; [ llvm_9 ]; + components.tests.yesod = { + build-tools = with final.pkgs; [ llvm_9 final.uniworx.hsPkgs.hspec-discover ]; + testWrapper = + let + testWrapper = prev.writeScript "test-wrapper" (import ../develop.nix { inherit pkgs; doDevelopEnv = false; } "$@"); + in singleton (toString testWrapper); + }; + components.tests.hlint.build-tools = with final.pkgs; [ llvm_9 final.uniworx.hsPkgs.hlint-test ]; }; } ]; diff --git a/nix/uniworx/frontend.nix b/nix/uniworx/frontend.nix index bbffa391e..dff9a92dc 100644 --- a/nix/uniworx/frontend.nix +++ b/nix/uniworx/frontend.nix @@ -1,19 +1,43 @@ -{ frontendSource, ... }: final: prev: { - uniworx-frontend = prev.stdenv.mkDerivation { +{ frontendSource, ... }: final: prev: +let + setupNodeDeps = '' + ln -s ${final.uniworxNodeDependencies}/lib/node_modules ./node_modules + export PATH="${final.uniworxNodeDependencies}/bin:$PATH" + ''; +in { + uniworxFrontend = prev.stdenv.mkDerivation { name = "uniworx-frontend"; - srcs = [frontendSource prev.uniworx-well-known]; + srcs = [frontendSource prev.uniworxWellKnown]; sourceRoot = "source"; - phases = ["unpackPhase" "buildPhase" "installPhase"]; + phases = ["unpackPhase" "checkPhase" "buildPhase" "installPhase"]; postUnpack = '' - cp -pr --reflink=auto uniworx-well-known/. $sourceRoot + ${final.xorg.lndir}/bin/lndir -silent ../uniworx-well-known $sourceRoot ''; + preBuild = setupNodeDeps; buildPhase = '' - ln -s ${prev.uniworx-node-dependencies}/lib/node_modules ./node_modules - export PATH="${prev.uniworx-node-dependencies}/bin:$PATH" + runHook preBuild + webpack --progress + + runHook postBuild + ''; + + preCheck = '' + ${setupNodeDeps} + export FONTCONFIG_FILE="${final.fontconfig.out}/etc/fonts/fonts.conf" + export FONTCONFIG_PATH="${final.fontconfig.out}/etc/fonts/" + export CHROME_BIN="${final.chromium}/bin/chromium-browser" + ''; + checkPhase = '' + runHook preCheck + + eslint frontend/src + karma start --conf karma.conf.js + + runHook postCheck ''; installPhase = '' @@ -21,5 +45,14 @@ cp -r --reflink=auto well-known static $out cp -r --reflink=auto config/webpack.yml $out/config ''; + + passthru.check = final.uniworxFrontend.overrideAttrs (oldAttrs: { + name = "${oldAttrs.name}-check"; + phases = ["unpackPhase" "buildPhase"]; + buildPhase = '' + mkdir $out + ( ${oldAttrs.checkPhase} ) | tee $out/test-stdout + ''; + }); }; } diff --git a/nix/uniworx/node-dependencies.nix b/nix/uniworx/node-dependencies.nix index 7893f47b1..fe7f45d65 100644 --- a/nix/uniworx/node-dependencies.nix +++ b/nix/uniworx/node-dependencies.nix @@ -1,5 +1,5 @@ { inputs, ... }: final: prev: { - uniworx-node-dependencies = (prev.callPackage ../frontend {}).nodeDependencies.override (oldArgs: { + uniworxNodeDependencies = (prev.callPackage ../frontend {}).nodeDependencies.override (oldArgs: { dependencies = let srcOverrides = { diff --git a/nix/uniworx/well-known.nix b/nix/uniworx/well-known.nix index 615269a9e..9e0ab278f 100644 --- a/nix/uniworx/well-known.nix +++ b/nix/uniworx/well-known.nix @@ -1,13 +1,13 @@ { frontendSource, ... }: final: prev: { - uniworx-well-known = prev.stdenv.mkDerivation { + uniworxWellKnown = prev.stdenv.mkDerivation { name = "uniworx-well-known"; src = frontendSource; phases = ["unpackPhase" "buildPhase" "installPhase" "fixupPhase"]; buildPhase = '' - ln -s ${prev.uniworx-node-dependencies}/lib/node_modules ./node_modules - export PATH="${prev.uniworx-node-dependencies}/bin:${prev.exiftool}/bin:$PATH" + ln -s ${final.uniworxNodeDependencies}/lib/node_modules ./node_modules + export PATH="${final.uniworxNodeDependencies}/bin:${prev.exiftool}/bin:$PATH" webpack --progress ''; diff --git a/shell.nix b/shell.nix index 0d5fe3975..d420eec75 100644 --- a/shell.nix +++ b/shell.nix @@ -5,201 +5,7 @@ let haskellPackages = pkgs.haskellPackages; - postgresSchema = pkgs.writeText "schema.sql" '' - CREATE USER uniworx WITH SUPERUSER; - CREATE DATABASE uniworx_test; - GRANT ALL ON DATABASE uniworx_test TO uniworx; - CREATE DATABASE uniworx; - GRANT ALL ON DATABASE uniworx TO uniworx; - ''; - - postgresHba = pkgs.writeText "hba_file" '' - local all all trust - ''; - - develop = pkgs.writeScriptBin "develop" '' - #!${pkgs.zsh}/bin/zsh -e - - basePath=$(pwd) - exec 4<>''${basePath}/.develop.env - - flockRes= - set +e - ${pkgs.util-linux}/bin/flock -en 4; flockRes=$? - set -e - if [[ ''${flockRes} -ne 0 ]]; then - echo "Could not take exclusive lock; is another develop running?" >&2 - exit ''${flockRes} - fi - - cleanup() { - set +e -x - type cleanup_postgres &>/dev/null && cleanup_postgres - type cleanup_widget_memcached &>/dev/null && cleanup_widget_memcached - type cleanup_session_memcached &>/dev/null && cleanup_session_memcached - type cleanup_cache_memcached &>/dev/null && cleanup_cache_memcached - type cleanup_minio &>/dev/null && cleanup_minio - type cleanup_maildev &>/dev/null && cleanup_maildev - - [ -f "''${basePath}/.develop.env" ] && rm -vf "''${basePath}/.develop.env" - set +x - } - - trap cleanup EXIT - - export PORT_OFFSET=$(((16#$(echo "fradrive $(whoami)" | sha256sum | head -c 16)) % 1000)) - - if [[ -z "$PGHOST" ]]; then - set -xe - - pgDir=$(mktemp -d --tmpdir=''${XDG_RUNTIME_DIR} postgresql.XXXXXX) - pgSockDir=$(mktemp -d --tmpdir=''${XDG_RUNTIME_DIR} postgresql.sock.XXXXXX) - pgLogFile=$(mktemp --tmpdir=''${XDG_RUNTIME_DIR} postgresql.XXXXXX.log) - initdb --no-locale -D ''${pgDir} - pg_ctl start -D ''${pgDir} -l ''${pgLogFile} -w -o "-k ''${pgSockDir} -c listen_addresses=''' -c hba_file='${postgresHba}' -c unix_socket_permissions=0700 -c max_connections=9990 -c shared_preload_libraries=pg_stat_statements -c auto_explain.log_min_duration=100ms" - psql -h ''${pgSockDir} -f ${postgresSchema} postgres - printf "Postgres logfile is %s\nPostgres socket directory is %s\n" ''${pgLogFile} ''${pgSockDir} - - export PGHOST=''${pgSockDir} - export PGLOG=''${pgLogFile} - - cleanup_postgres() { - set +e -x - pg_ctl stop -D ''${pgDir} - rm -rvf ''${pgDir} ''${pgSockDir} ''${pgLogFile} - set +x - } - - set +xe - fi - - if [[ -z "$WIDGET_MEMCACHED_HOST" ]]; then - set -xe - - memcached -l localhost -p $(($PORT_OFFSET + 11211)) &>/dev/null & - widget_memcached_pid=$! - - export WIDGET_MEMCACHED_HOST=localhost - export WIDGET_MEMCACHED_PORT=$(($PORT_OFFSET + 11211)) - - cleanup_widget_memcached() { - [[ -n "$widget_memcached_pid" ]] && kill $widget_memcached_pid - } - - set +xe - fi - - if [[ -z "$SESSION_MEMCACHED_HOST" ]]; then - set -xe - - memcached -l localhost -p $(($PORT_OFFSET + 11212)) &>/dev/null & - session_memcached_pid=$! - - export SESSION_MEMCACHED_HOST=localhost - export SESSION_MEMCACHED_PORT=$(($PORT_OFFSET + 11212)) - - cleanup_session_memcached() { - [[ -n "$session_memcached_pid" ]] && kill $session_memcached_pid - } - - set +xe - fi - - if [[ -z "$MEMCACHED_HOST" ]]; then - set -xe - - memcached -l localhost -p $(($PORT_OFFSET + 11213)) &>/dev/null & - memcached_pid=$! - - export MEMCACHED_HOST=localhost - export MEMCACHED_PORT=$(($PORT_OFFSET + 11212)) - - cleanup_session_memcached() { - [[ -n "$memcached_pid" ]] && kill $memcached_pid - } - - set +xe - fi - - if [[ -z "$UPLOAD_S3_HOST" ]]; then - set -xe - - cleanup_minio() { - [[ -n "$minio_pid" ]] && kill $minio_pid - [[ -n "''${MINIO_DIR}" ]] && rm -rvf ''${MINIO_DIR} - [[ -n "''${MINIO_LOGFILE}" ]] && rm -rvf ''${MINIO_LOGFILE} - } - - export MINIO_DIR=$(mktemp -d --tmpdir=''${XDG_RUNTIME_DIR} minio.XXXXXX) - export MINIO_LOGFILE=$(mktemp --tmpdir=''${XDG_RUNTIME_DIR} minio.XXXXXX.log) - export MINIO_ACCESS_KEY=$(${pkgs.pwgen}/bin/pwgen -s 16 1) - export MINIO_SECRET_KEY=$(${pkgs.pwgen}/bin/pwgen -s 32 1) - - minio server --address localhost:$(($PORT_OFFSET + 9000)) ''${MINIO_DIR} &>''${MINIO_LOGFILE} & - minio_pid=$! - - export UPLOAD_S3_HOST=localhost - export UPLOAD_S3_PORT=$(($PORT_OFFSET + 9000)) - export UPLOAD_S3_SSL=false - export UPLOAD_S3_KEY_ID=''${MINIO_ACCESS_KEY} - export UPLOAD_S3_KEY=''${MINIO_SECRET_KEY} - - sleep 1 - - set +xe - fi - - ${optionalString (pkgs.nodePackages ? "maildev") '' - if [[ -z "$SMTPHOST" ]]; then - set -xe - - cleanup_maildev() { - [[ -n "$maildev_pid" ]] && kill $maildev_pid - } - - TMPDIR=''${XDG_RUNTIME_DIR} ${pkgs.nodePackages.maildev}/bin/maildev --smtp $(($PORT_OFFSET + 1025)) --web $(($PORT_OFFSET + 8080)) --ip localhost --web-ip localhost &>/dev/null & - maildev_pid=$! - - export SMTPHOST=localhost - export SMTPPORT=$(($PORT_OFFSET + 1025)) - export SMTPSSL=none - - set +xe - fi - ''} - - set -xe - - cat >&4 < Date: Tue, 14 Sep 2021 12:09:49 +0200 Subject: [PATCH 07/52] chore(nix): backend tests --- nix/develop.nix | 7 ++++--- nix/uniworx/backend.nix | 10 ++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/nix/develop.nix b/nix/develop.nix index 03a89f0ef..e4e29cf0a 100644 --- a/nix/develop.nix +++ b/nix/develop.nix @@ -1,9 +1,10 @@ { pkgs +, prev ? pkgs , doPortOffset ? true , doDevelopEnv ? true }: -with pkgs.lib; +with prev.lib; let withDevelop = action: '' @@ -198,7 +199,7 @@ let ${action} ''; - postgresSchema = pkgs.writeText "schema.sql" '' + postgresSchema = prev.writeText "schema.sql" '' CREATE USER uniworx WITH SUPERUSER; CREATE DATABASE uniworx_test; GRANT ALL ON DATABASE uniworx_test TO uniworx; @@ -206,7 +207,7 @@ let GRANT ALL ON DATABASE uniworx TO uniworx; ''; - postgresHba = pkgs.writeText "hba_file" '' + postgresHba = prev.writeText "hba_file" '' local all all trust ''; in withDevelop diff --git a/nix/uniworx/backend.nix b/nix/uniworx/backend.nix index 2a31434b7..fc1ceb525 100644 --- a/nix/uniworx/backend.nix +++ b/nix/uniworx/backend.nix @@ -72,8 +72,14 @@ in { build-tools = with final.pkgs; [ llvm_9 final.uniworx.hsPkgs.hspec-discover ]; testWrapper = let - testWrapper = prev.writeScript "test-wrapper" (import ../develop.nix { inherit pkgs; doDevelopEnv = false; } "$@"); - in singleton (toString testWrapper); + testWrapper = prev.writeScript "test-wrapper" (import ../develop.nix { inherit prev; pkgs = final; doDevelopEnv = false; } "$@"); + testWrapperWrapped = prev.runCommand "test-wrapper" { buildInputs = [final.makeWrapper]; } '' + makeWrapper ${testWrapper} $out \ + --prefix PATH : ${final.postgresql_12}/bin \ + --prefix PATH : ${final.minio}/bin \ + --prefix PATH : ${final.memcached}/bin + ''; + in singleton (toString testWrapperWrapped); }; components.tests.hlint.build-tools = with final.pkgs; [ llvm_9 final.uniworx.hsPkgs.hlint-test ]; }; From 8a0f7ffaab17f032a196e72e93b91c26eb8e6e07 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Tue, 14 Sep 2021 12:11:08 +0200 Subject: [PATCH 08/52] chore(gitlab-ci): nix based build, nodeDependencies for now --- .gitlab-ci.yml | 480 +------------------------------------------- .gitlab-ci/nix.conf | 1 + 2 files changed, 8 insertions(+), 473 deletions(-) create mode 100644 .gitlab-ci/nix.conf diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 176f7d6ed..60c723369 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,488 +6,22 @@ workflow: default: image: - name: fpco/stack-build:lts-17.15 - -variables: - STACK_ROOT: "${CI_PROJECT_DIR}/.stack" - CHROME_BIN: "/usr/bin/chromium-browser" - POSTGRES_DB: uniworx_test - POSTGRES_USER: uniworx - POSTGRES_PASSWORD: uniworx - MINIO_ACCESS_KEY: gOel7KvadwNKgjjy - MINIO_SECRET_KEY: ugO5pkEla7F0JW9MdPwLi4MWLT5ZbqAL - UPLOAD_S3_HOST: localhost - UPLOAD_S3_PORT: 9000 - UPLOAD_S3_KEY_ID: gOel7KvadwNKgjjy - UPLOAD_S3_KEY: ugO5pkEla7F0JW9MdPwLi4MWLT5ZbqAL - N_PREFIX: "${HOME}/.n" - PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/uni2work" + name: nixos/nix:2.3.12 stages: - setup - - frontend:build - - yesod:build - - lint - - test - - prepare release - - upload packages - - release -# - deploy - -npm install: - cache: - - &npm-cache - key: default-npm - paths: - - .npm - - node_modules +node dependencies: stage: setup script: - - ./.npmrc.gup - - npm ci --cache .npm --prefer-offline - before_script: &npm - - rm -rvf /etc/apt/sources.list /etc/apt/sources.list.d - - install -v -T -m 0644 ${APT_SOURCES_LIST} /etc/apt/sources.list - - apt-get update -y - - npm install -g n - - n 14.8.0 - - export PATH="${N_PREFIX}/bin:$PATH" - - npm install -g npm - - hash -r - - apt-get -y install openssh-client exiftool - - install -v -m 0700 -d ~/.ssh - - install -v -T -m 0644 ${SSH_KNOWN_HOSTS} ~/.ssh/known_hosts - - install -v -T -m 0400 ${SSH_DEPLOY_KEY} ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; - artifacts: - paths: - - node_modules/ - name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" - expire_in: "1 day" - retry: 2 - interruptible: true - -frontend:build: - cache: - - &frontend-cache - key: default-frontend - paths: - - .well-known-cache - - stage: frontend:build - script: - - npm run frontend:build - before_script: *npm - needs: - - job: npm install - artifacts: true - artifacts: - paths: - - static - - well-known - - config/webpack.yml - name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" - expire_in: "1 day" - retry: 2 - interruptible: true - -frontend:lint: - stage: lint - script: - - npm run frontend:lint - before_script: *npm - needs: - - job: npm install - artifacts: true - retry: 2 - interruptible: true - -yesod:build:dev: - cache: - - &stack-dev-cache - key: default-stack-dev - paths: - - .stack - - .stack-work - - stage: yesod:build - script: - - stack build --test --copy-bins --local-bin-path $(pwd)/bin --fast --flag uniworx:-library-only --flag uniworx:dev --flag uniworx:pedantic --no-strip --no-run-tests - - cp $(stack path --dist-dir)/build/hlint/hlint bin/test-hlint - - cp $(stack path --dist-dir)/build/yesod/yesod bin/test-yesod - needs: - - job: npm install # transitive - artifacts: false - - job: frontend:build - artifacts: true - before_script: &haskell - - rm -rvf /etc/apt/sources.list /etc/apt/sources.list.d - - install -v -T -m 0644 ${APT_SOURCES_LIST} /etc/apt/sources.list - - apt-get update -y - - apt-get install -y --no-install-recommends locales-all openssh-client git-restore-mtime - - wget http://newmirror.rz.ifi.lmu.de/ubuntu/archive/pool/main/libs/libsodium/libsodium-dev_1.0.18-1_amd64.deb http://newmirror.rz.ifi.lmu.de/ubuntu/archive/pool/main/libs/libsodium/libsodium23_1.0.18-1_amd64.deb - - apt install ./libsodium23_1.0.18-1_amd64.deb ./libsodium-dev_1.0.18-1_amd64.deb - - rm -v libsodium23_1.0.18-1_amd64.deb libsodium-dev_1.0.18-1_amd64.deb - - install -v -m 0700 -d ~/.ssh - - install -v -T -m 0644 ${SSH_KNOWN_HOSTS} ~/.ssh/known_hosts - - install -v -T -m 0400 ${SSH_DEPLOY_KEY} ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; - - stack install happy - - export PATH="${HOME}/.local/bin:$PATH" - - hash -r - - git restore-mtime - artifacts: - paths: - - bin/ - name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" - expire_in: "1 day" - - rules: - - if: $CI_COMMIT_REF_NAME =~ /(^v[0-9].*)|((^|\/)profile($|\/))/ - when: manual - allow_failure: true - - when: on_success - - retry: 2 - interruptible: true - -yesod:build: - cache: - - &stack-cache - key: default-stack - paths: - - .stack - - .stack-work - - stage: yesod:build - script: - - stack build --test --copy-bins --local-bin-path $(pwd)/bin --flag uniworx:-library-only --flag uniworx:-dev --flag uniworx:pedantic --no-strip --no-run-tests - - cp $(stack path --dist-dir)/build/hlint/hlint bin/test-hlint - - cp $(stack path --dist-dir)/build/yesod/yesod bin/test-yesod - needs: - - job: npm install # transitive - artifacts: false - - job: frontend:build - artifacts: true - before_script: *haskell - artifacts: - paths: - - bin/ - name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" - expire_in: "1 day" - - rules: - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9].*/ - when: on_success - - when: manual - allow_failure: true - - retry: 2 - interruptible: true - resource_group: ram - -yesod:build:profile: - cache: - - &stack-profile-cache - key: default-stack-profile - paths: - - .stack - - .stack-work - - stage: yesod:build - script: - - stack build --profile --copy-bins --local-bin-path $(pwd)/bin --flag uniworx:-library-only --flag uniworx:-dev --flag uniworx:pedantic --no-strip - needs: - - job: npm install # transitive - artifacts: false - - job: frontend:build - artifacts: true - before_script: *haskell - artifacts: - paths: - - bin/ - name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" - expire_in: "1 day" - - rules: - - if: $CI_COMMIT_REF_NAME =~ /(^|\/)profile($|\/)/ - when: on_success - - when: manual - allow_failure: true - - retry: 2 - interruptible: true - resource_group: ram - -yesod:test:yesod: - stage: test - - services: &test-services - - name: postgres:10.10 - alias: postgres - - name: minio/minio:RELEASE.2020-08-27T05-16-20Z - alias: minio - command: ["minio", "server", "/data"] - - needs: - - job: npm install # transitive - artifacts: false - - job: frontend:build #transitive - artifacts: false - - job: yesod:build - artifacts: true - - rules: - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9].*/ - when: on_success - - script: - - bin/test-yesod - - retry: 2 - interruptible: true - -yesod:test:yesod:dev: - stage: test - - services: *test-services - - needs: - - job: npm install # transitive - artifacts: false - - job: frontend:build #transitive - artifacts: false - - job: yesod:build:dev - artifacts: true - - rules: - - if: $CI_COMMIT_REF_NAME =~ /(^v[0-9].*)|((^|\/)profile($|\/))/ - when: manual - allow_failure: true - - when: on_success - - script: - - bin/test-yesod - - retry: 2 - interruptible: true - -yesod:test:hlint: - stage: lint - cache: &hlint_cache - key: hlint - paths: - - .stack - - .stack-work - - needs: - - job: npm install # transitive - artifacts: false - - job: frontend:build #transitive - artifacts: false - - job: yesod:build - artifacts: false - - rules: - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9].*/ - when: on_success - - before_script: *haskell - script: - - stack install hlint - - stack exec -- hlint --cc src > gl-code-quality-report.json || jq . gl-code-quality-report.json - - artifacts: - paths: - - gl-code-quality-report.json - name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" - expire_in: "1 day" - - reports: - codequality: gl-code-quality-report.json - - retry: 2 - interruptible: true - -yesod:test:hlint:dev: - stage: lint - cache: *hlint_cache - - needs: - - job: npm install # transitive - artifacts: false - - job: frontend:build #transitive - artifacts: false - - job: yesod:build:dev - artifacts: false - - rules: - - if: $CI_COMMIT_REF_NAME =~ /(^v[0-9].*)|((^|\/)profile($|\/))/ - when: manual - allow_failure: true - - when: on_success - - before_script: *haskell - script: - - stack install hlint - - stack exec -- hlint --cc src > gl-code-quality-report.json || jq . gl-code-quality-report.json - - artifacts: - paths: - - gl-code-quality-report.json - name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" - expire_in: "1 day" - - reports: - codequality: gl-code-quality-report.json - - retry: 2 - interruptible: true - -frontend:test: - stage: test - script: - - npm run frontend:test - needs: - - job: npm install - artifacts: true + - nix build -o node-dependencies uniworxNodeDependencies before_script: - - rm -rvf /etc/apt/sources.list /etc/apt/sources.list.d - - install -v -T -m 0644 ${APT_SOURCES_LIST} /etc/apt/sources.list - - apt-get update -y - - npm install -g n - - n 14.8.0 - - export PATH="${N_PREFIX}/bin:$PATH" - - npm install -g npm - - hash -r - - apt-get install -y --no-install-recommends chromium-browser - retry: 2 - interruptible: true - -parse-changelog: - stage: prepare release - needs: - - job: npm install - artifacts: true - rules: - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9].*/ - when: on_success - before_script: - - rm -rvf /etc/apt/sources.list /etc/apt/sources.list.d - - install -v -T -m 0644 ${APT_SOURCES_LIST} /etc/apt/sources.list - - apt-get update -y - - apt-get install -y --no-install-recommends jq - script: - - npm run parse-changelog - - | - jq -r '.versions[0].version' changelog.json > .current-version - - | - jq -r '.versions[0].body' changelog.json > .current-changelog.md + - nix-env -iA envsubst + - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf artifacts: paths: - - .current-version - - .current-changelog.md - name: "changelog-${CI_COMMIT_SHORT_SHA}" + - node-dependencies + name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" expire_in: "1 day" retry: 2 interruptible: true - -upload: - variables: - GIT_STRATEGY: none - stage: upload packages - image: curlimages/curl:latest - needs: - - job: npm install # transitive - artifacts: false - - job: frontend:build # transitive - artifacts: false - - job: yesod:build - artifacts: true - - job: parse-changelog - artifacts: true - - job: frontend:lint # validation - artifacts: false - - job: frontend:test # validation - artifacts: false - - job: yesod:test:hlint # validation - artifacts: false - - job: yesod:test:yesod # validation - artifacts: false - rules: - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9].*/ - when: on_success - before_script: - - export VERSION="$(cat .current-version)" - script: - - | - curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/uniworx ${PACKAGE_REGISTRY_URL}/${VERSION}/uniworx - - | - curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/uniworxdb ${PACKAGE_REGISTRY_URL}/${VERSION}/uniworxdb - - | - curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/uniworxload ${PACKAGE_REGISTRY_URL}/${VERSION}/uniworxload - - | - curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/uniworx-wflint ${PACKAGE_REGISTRY_URL}/${VERSION}/uniworx-wflint - -release: - variables: - GIT_STRATEGY: none - stage: release - image: registry.gitlab.com/gitlab-org/release-cli:latest - needs: - - job: upload - artifacts: false - - job: npm install # transitive - artifacts: false - - job: frontend:build # transitive - artifacts: false - - job: yesod:build # transitive - artifacts: false - - job: parse-changelog - artifacts: true - - job: frontend:lint # validation - artifacts: false - - job: frontend:test # validation - artifacts: false - - job: yesod:test:hlint # validation - artifacts: false - - job: yesod:test:yesod # validation - artifacts: false - rules: - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9].*/ - when: on_success - before_script: - - export VERSION="$(cat .current-version)" - script: - - | - release-cli create --name "${VERSION}" --tag-name $CI_COMMIT_TAG --description .current-changelog.md \ - --assets-link "{\"name\":\"uniworx\",\"url\":\"${PACKAGE_REGISTRY_URL}/${VERSION}/uniworx\",\"filepath\":\"/uniworx\"}" \ - --assets-link "{\"name\":\"uniworxdb\",\"url\":\"${PACKAGE_REGISTRY_URL}/${VERSION}/uniworxdb\",\"filepath\":\"/uniworxdb\"}" \ - --assets-link "{\"name\":\"uniworxload\",\"url\":\"${PACKAGE_REGISTRY_URL}/${VERSION}/uniworxload\",\"filepath\":\"/uniworxload\"}" \ - --assets-link "{\"name\":\"uniworx-wflint\",\"url\":\"${PACKAGE_REGISTRY_URL}/${VERSION}/uniworx-wflint\",\"filepath\":\"/uniworx-wflint\"}" - - -# deploy:uniworx3: -# cache: {} -# stage: deploy -# variables: -# GIT_STRATEGY: none -# script: -# - zip -qj - bin/uniworx bin/uniworxdb | ssh root@uniworx3.ifi.lmu.de /root/bin/accept_uni2work -# needs: -# - yesod:build -# - frontend:test # For sanity -# before_script: -# - rm -rvf /etc/apt/sources.list /etc/apt/sources.list.d -# - install -v -T -m 0644 ${APT_SOURCES_LIST} /etc/apt/sources.list -# - apt-get update -y -# - apt-get install -y --no-install-recommends openssh-client -# - install -v -m 0700 -d ~/.ssh -# - install -v -T -m 0644 ${SSH_KNOWN_HOSTS} ~/.ssh/known_hosts -# - install -v -T -m 0400 ${SSH_PRIVATE_KEY_UNIWORX3} ~/.ssh/uniworx3; echo "IdentityFile ~/.ssh/uniworx3" >> ~/.ssh/config; -# dependencies: -# - yesod:build - -# only: -# variables: -# - $CI_COMMIT_REF_NAME =~ /^v[0-9].*/ -# resource_group: uniworx3 diff --git a/.gitlab-ci/nix.conf b/.gitlab-ci/nix.conf new file mode 100644 index 000000000..06af3c2d5 --- /dev/null +++ b/.gitlab-ci/nix.conf @@ -0,0 +1 @@ +sandbox = true \ No newline at end of file From c63ed968e2f78ebfb3aa5a1ec614aca96810bf01 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Tue, 14 Sep 2021 12:14:06 +0200 Subject: [PATCH 09/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 2 +- .gitlab-ci/nix.conf | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 60c723369..3c12234f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,7 @@ node dependencies: script: - nix build -o node-dependencies uniworxNodeDependencies before_script: - - nix-env -iA envsubst + - nix-env -f '' -iA envsubst - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf artifacts: paths: diff --git a/.gitlab-ci/nix.conf b/.gitlab-ci/nix.conf index 06af3c2d5..036c2fc5e 100644 --- a/.gitlab-ci/nix.conf +++ b/.gitlab-ci/nix.conf @@ -1 +1,5 @@ -sandbox = true \ No newline at end of file +sandbox = true +experimental-features = nix-command flakes ca-references +substituters = https://hydra.iohk.io https://cache.nixos.org/ +trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= +require-sigs = true \ No newline at end of file From 9de7649feb0786d29a5d5d63f7e75fac2f4d6c14 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Tue, 14 Sep 2021 12:15:55 +0200 Subject: [PATCH 10/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c12234f8..5e50722c5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,9 +14,9 @@ stages: node dependencies: stage: setup script: - - nix build -o node-dependencies uniworxNodeDependencies + - nix build -o node-dependencies .#uniworxNodeDependencies before_script: - - nix-env -f '' -iA envsubst + - nix-env -f '' -iA envsubst nixUnstable - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf artifacts: paths: From 8e2369f5987dff52195d6c2e03acd06d74174543 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Tue, 14 Sep 2021 12:17:10 +0200 Subject: [PATCH 11/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e50722c5..68c4f2aa1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,7 @@ node dependencies: script: - nix build -o node-dependencies .#uniworxNodeDependencies before_script: - - nix-env -f '' -iA envsubst nixUnstable + - nix-env -f '' -iA envsubst nixUnstable git - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf artifacts: paths: From feaedfded43fb46dd356fb425accc46c76eebe78 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Tue, 14 Sep 2021 12:21:40 +0200 Subject: [PATCH 12/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68c4f2aa1..0644c2fa2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,11 @@ node dependencies: before_script: - nix-env -f '' -iA envsubst nixUnstable git - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf + - git config --global init.defaultBranch master + - install -v -m 0700 -d ~/.ssh + - install -v -T -m 0644 ${SSH_KNOWN_HOSTS} ~/.ssh/known_hosts + - install -v -T -m 0400 ${SSH_DEPLOY_KEY} ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; + - install -v -T -m 0644 ${FONTAWESOME_NPM_AUTH_FILE} /etc/fontawesome-token artifacts: paths: - node-dependencies From 229e5077ad430bdcd2bb8cf5b75d34123c1467fe Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Tue, 14 Sep 2021 12:24:26 +0200 Subject: [PATCH 13/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 3 ++- .gitlab-ci/nix-bootstrap.conf | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .gitlab-ci/nix-bootstrap.conf diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0644c2fa2..535ba2fd6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,8 @@ node dependencies: script: - nix build -o node-dependencies .#uniworxNodeDependencies before_script: - - nix-env -f '' -iA envsubst nixUnstable git + - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf + - nix-env -f '' -iA envsubst nixUnstable git coreutils - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf - git config --global init.defaultBranch master - install -v -m 0700 -d ~/.ssh diff --git a/.gitlab-ci/nix-bootstrap.conf b/.gitlab-ci/nix-bootstrap.conf new file mode 100644 index 000000000..036c2fc5e --- /dev/null +++ b/.gitlab-ci/nix-bootstrap.conf @@ -0,0 +1,5 @@ +sandbox = true +experimental-features = nix-command flakes ca-references +substituters = https://hydra.iohk.io https://cache.nixos.org/ +trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= +require-sigs = true \ No newline at end of file From 5958c452bcb90fcac22a9484acab8dfde3fd43ee Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Tue, 14 Sep 2021 12:26:28 +0200 Subject: [PATCH 14/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 535ba2fd6..75db5fe10 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,7 @@ node dependencies: before_script: - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf - nix-env -f '' -iA envsubst nixUnstable git coreutils + - nix-env -e busybox - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf - git config --global init.defaultBranch master - install -v -m 0700 -d ~/.ssh From 531cd028cdad4c0e419cbe59d7419d150b56360a Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Tue, 14 Sep 2021 12:32:23 +0200 Subject: [PATCH 15/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 75db5fe10..65929f501 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,8 +17,8 @@ node dependencies: - nix build -o node-dependencies .#uniworxNodeDependencies before_script: - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf - - nix-env -f '' -iA envsubst nixUnstable git coreutils - - nix-env -e busybox + - nix-channel --update + - nix-env -f '' -riA envsubst nixUnstable git coreutils - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf - git config --global init.defaultBranch master - install -v -m 0700 -d ~/.ssh From 7d66c8629cd2143e6eff2bc0cafb830f8d8cd853 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Tue, 14 Sep 2021 12:45:02 +0200 Subject: [PATCH 16/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 9 +++++---- .gitlab-ci/nix-bootstrap.conf | 1 - 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 65929f501..b5a8f5468 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,10 @@ workflow: default: image: - name: nixos/nix:2.3.12 + name: nixpkgs/nix-flakes:nixos-21.05 + +variables: + NIX_PATH: "nixpkgs=http://nixos.org/channels/nixos-21.05/nixexprs.tar.xz" stages: - setup @@ -16,9 +19,7 @@ node dependencies: script: - nix build -o node-dependencies .#uniworxNodeDependencies before_script: - - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf - - nix-channel --update - - nix-env -f '' -riA envsubst nixUnstable git coreutils + - nix-env -f '' -riA envsubst - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf - git config --global init.defaultBranch master - install -v -m 0700 -d ~/.ssh diff --git a/.gitlab-ci/nix-bootstrap.conf b/.gitlab-ci/nix-bootstrap.conf index 036c2fc5e..b35f18d99 100644 --- a/.gitlab-ci/nix-bootstrap.conf +++ b/.gitlab-ci/nix-bootstrap.conf @@ -1,5 +1,4 @@ sandbox = true -experimental-features = nix-command flakes ca-references substituters = https://hydra.iohk.io https://cache.nixos.org/ trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= require-sigs = true \ No newline at end of file From fcffa4808ced270827104164e72beae0850177fe Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Tue, 14 Sep 2021 12:51:32 +0200 Subject: [PATCH 17/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b5a8f5468..6095e829d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,15 +17,16 @@ stages: node dependencies: stage: setup script: - - nix build -o node-dependencies .#uniworxNodeDependencies + - nix -L build -o node-dependencies .#uniworxNodeDependencies before_script: + - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf - nix-env -f '' -riA envsubst - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf - git config --global init.defaultBranch master - install -v -m 0700 -d ~/.ssh - - install -v -T -m 0644 ${SSH_KNOWN_HOSTS} ~/.ssh/known_hosts - - install -v -T -m 0400 ${SSH_DEPLOY_KEY} ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; - - install -v -T -m 0644 ${FONTAWESOME_NPM_AUTH_FILE} /etc/fontawesome-token + - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts + - install -v -T -m 0400 "${SSH_DEPLOY_KEY}" ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; + - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token artifacts: paths: - node-dependencies From 72143a95f11bf89210a0fee4c31e7adec7efea9b Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Wed, 15 Sep 2021 10:57:26 +0200 Subject: [PATCH 18/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 142 +++++++++++++++++++++++++++++++++++++++++++++++-- flake.nix | 6 ++- 2 files changed, 144 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6095e829d..ea245ac00 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,12 +12,15 @@ variables: NIX_PATH: "nixpkgs=http://nixos.org/channels/nixos-21.05/nixexprs.tar.xz" stages: - - setup + - frontend:build + - backend:build + - docker:build node dependencies: - stage: setup + stage: frontend:build script: - nix -L build -o node-dependencies .#uniworxNodeDependencies + - nix-store --export $(nix-store -qR node-dependencies) > node-dependencies.nar before_script: - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf - nix-env -f '' -riA envsubst @@ -29,7 +32,140 @@ node dependencies: - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token artifacts: paths: - - node-dependencies + - node-dependencies.nar + name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" + expire_in: "1 day" + retry: 2 + interruptible: true + +well known: + stage: frontend:build + script: + - nix-store --import well-known.nar + before_script: + - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf + - nix-env -f '' -riA envsubst + - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf + - git config --global init.defaultBranch master + - install -v -m 0700 -d ~/.ssh + - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts + - install -v -T -m 0400 "${SSH_DEPLOY_KEY}" ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; + - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token + needs: + - job: node dependencies + artifacts: true + artifacts: + paths: + - well-known.nar + name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" + expire_in: "1 day" + retry: 2 + interruptible: true + +frontend: + stage: frontend:build + script: + - nix-store --import frontend.nar + before_script: + - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf + - nix-env -f '' -riA envsubst + - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf + - git config --global init.defaultBranch master + - install -v -m 0700 -d ~/.ssh + - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts + - install -v -T -m 0400 "${SSH_DEPLOY_KEY}" ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; + - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token + needs: + - job: node dependencies + artifacts: true + - job: well-known + artifacts: true + artifacts: + paths: + - frontend.nar + name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" + expire_in: "1 day" + retry: 2 + interruptible: true + +backend: + stage: backend:build + script: + - nix-store --import ' -riA envsubst + - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf + - git config --global init.defaultBranch master + - install -v -m 0700 -d ~/.ssh + - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts + - install -v -T -m 0400 "${SSH_DEPLOY_KEY}" ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; + - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token + needs: + - job: frontend + artifacts: true + artifacts: + paths: + - bin/ + name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" + expire_in: "1 day" + retry: 2 + interruptible: true + +demo docker: + stage: docker:build + script: + - nix-store --import ' -riA envsubst + - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf + - git config --global init.defaultBranch master + - install -v -m 0700 -d ~/.ssh + - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts + - install -v -T -m 0400 "${SSH_DEPLOY_KEY}" ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; + - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token + needs: + - job: backend + artifacts: true + artifacts: + paths: + - uniworx-demo.tar.gz + name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" + expire_in: "1 day" + retry: 2 + interruptible: true + +docker: + stage: docker:build + script: + - nix-store --import ' -riA envsubst + - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf + - git config --global init.defaultBranch master + - install -v -m 0700 -d ~/.ssh + - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts + - install -v -T -m 0400 "${SSH_DEPLOY_KEY}" ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; + - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token + needs: + - job: backend + artifacts: true + artifacts: + paths: + - uniworx.tar.gz name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" expire_in: "1 day" retry: 2 diff --git a/flake.nix b/flake.nix index c0e07cded..47f5fee9f 100644 --- a/flake.nix +++ b/flake.nix @@ -106,9 +106,13 @@ [[ -n "''${1}" ]] && shift ${pkgs.skopeo}/bin/skopeo ''${@} --insecure-policy copy docker-archive://${dockerImage} ''${target} ''; + uniworxBackend = pkgs.symlinkJoin { + name = "uniworx"; + paths = with pkgs; [ "uniworx:exe:uniworx" "uniworx:exe:uniworxdb" "uniworx:exe:uniworxload" "uniworx:exe:uniworx-wflint" ]; + }; in { packages = haskellFlake.packages // { - inherit (pkgs) uniworxNodeDependencies uniworxWellKnown uniworxFrontend uniworxDemoDocker uniworxDocker; + inherit (pkgs) uniworxNodeDependencies uniworxWellKnown uniworxFrontend uniworxBackend uniworxDemoDocker uniworxDocker; }; apps = haskellFlake.apps // { pushUniworxDemoDocker = flake-utils.lib.mkApp { drv = mkPushUniworxDocker pkgs.uniworxDemoDocker; }; From 6950b533f460860498fff9f8d6c30cf2bbec332e Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Wed, 15 Sep 2021 10:59:48 +0200 Subject: [PATCH 19/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 54 +++++++------------------------------------------- 1 file changed, 7 insertions(+), 47 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ea245ac00..29ddb819e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ node dependencies: script: - nix -L build -o node-dependencies .#uniworxNodeDependencies - nix-store --export $(nix-store -qR node-dependencies) > node-dependencies.nar - before_script: + before_script: &nix - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf - nix-env -f '' -riA envsubst - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf @@ -44,15 +44,7 @@ well known: - nix-store --import well-known.nar - before_script: - - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf - - nix-env -f '' -riA envsubst - - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf - - git config --global init.defaultBranch master - - install -v -m 0700 -d ~/.ssh - - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts - - install -v -T -m 0400 "${SSH_DEPLOY_KEY}" ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; - - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token + before_script: *nix needs: - job: node dependencies artifacts: true @@ -71,19 +63,11 @@ frontend: - nix-store --import frontend.nar - before_script: - - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf - - nix-env -f '' -riA envsubst - - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf - - git config --global init.defaultBranch master - - install -v -m 0700 -d ~/.ssh - - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts - - install -v -T -m 0400 "${SSH_DEPLOY_KEY}" ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; - - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token + before_script: *nix needs: - job: node dependencies artifacts: true - - job: well-known + - job: well known artifacts: true artifacts: paths: @@ -99,15 +83,7 @@ backend: - nix-store --import ' -riA envsubst - - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf - - git config --global init.defaultBranch master - - install -v -m 0700 -d ~/.ssh - - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts - - install -v -T -m 0400 "${SSH_DEPLOY_KEY}" ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; - - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token + before_script: *nix needs: - job: frontend artifacts: true @@ -125,15 +101,7 @@ demo docker: - nix-store --import ' -riA envsubst - - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf - - git config --global init.defaultBranch master - - install -v -m 0700 -d ~/.ssh - - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts - - install -v -T -m 0400 "${SSH_DEPLOY_KEY}" ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; - - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token + before_script: *nix needs: - job: backend artifacts: true @@ -151,15 +119,7 @@ docker: - nix-store --import ' -riA envsubst - - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf - - git config --global init.defaultBranch master - - install -v -m 0700 -d ~/.ssh - - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts - - install -v -T -m 0400 "${SSH_DEPLOY_KEY}" ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; - - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token + before_script: *nix needs: - job: backend artifacts: true From e1e5693a2a32b6bd1884002c91ec62c95e27fe55 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Wed, 15 Sep 2021 11:29:00 +0200 Subject: [PATCH 20/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 36 +++++++++++++++++------------------ .gitlab-ci/nix-bootstrap.conf | 4 ---- 2 files changed, 17 insertions(+), 23 deletions(-) delete mode 100644 .gitlab-ci/nix-bootstrap.conf diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29ddb819e..941d3d6ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,12 +19,10 @@ stages: node dependencies: stage: frontend:build script: - - nix -L build -o node-dependencies .#uniworxNodeDependencies - - nix-store --export $(nix-store -qR node-dependencies) > node-dependencies.nar - before_script: &nix - - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf - - nix-env -f '' -riA envsubst - - envsubst -i .gitlab-ci/nix.conf -o /etc/nix/nix.conf + - nix -L build -o result .#uniworxNodeDependencies + - nix-store --export $(nix-store -qR result) > node-dependencies.nar + before_script: &nix-before + - install -m 0644 .gitlab-ci/nix.conf /etc/nix/nix.conf - git config --global init.defaultBranch master - install -v -m 0700 -d ~/.ssh - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts @@ -42,9 +40,9 @@ well known: stage: frontend:build script: - nix-store --import well-known.nar - before_script: *nix + - nix -L build -o result .#uniworxWellKnown + - nix-store --export $(nix-store -qR result) > well-known.nar + before_script: *nix-before needs: - job: node dependencies artifacts: true @@ -61,9 +59,9 @@ frontend: script: - nix-store --import frontend.nar - before_script: *nix + - nix -L build -o result .#uniworxFrontend + - nix-store --export $(nix-store -qR result) > frontend.nar + before_script: *nix-before needs: - job: node dependencies artifacts: true @@ -81,9 +79,9 @@ backend: stage: backend:build script: - nix-store --import Date: Wed, 15 Sep 2021 12:13:15 +0200 Subject: [PATCH 21/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 3 +++ flake.nix | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 941d3d6ca..0d90ec014 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,9 @@ node dependencies: - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts - install -v -T -m 0400 "${SSH_DEPLOY_KEY}" ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token + after_script: &nix-after + - nix-env -f '' -riA tree + - tree -a /minio-gitlab-runner-cache artifacts: paths: - node-dependencies.nar diff --git a/flake.nix b/flake.nix index 47f5fee9f..7b4fd1270 100644 --- a/flake.nix +++ b/flake.nix @@ -112,7 +112,8 @@ }; in { packages = haskellFlake.packages // { - inherit (pkgs) uniworxNodeDependencies uniworxWellKnown uniworxFrontend uniworxBackend uniworxDemoDocker uniworxDocker; + inherit (pkgs) uniworxNodeDependencies uniworxWellKnown uniworxFrontend uniworxDemoDocker uniworxDocker; + inherit uniworxBackend; }; apps = haskellFlake.apps // { pushUniworxDemoDocker = flake-utils.lib.mkApp { drv = mkPushUniworxDocker pkgs.uniworxDemoDocker; }; From 0c185b2eb05a49b4b03589df6e6be4508e6d290b Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Wed, 15 Sep 2021 13:33:32 +0200 Subject: [PATCH 22/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 7 ++++--- .gitlab-ci/make-minio-bucket.sh | 6 ++++++ .gitlab-ci/nix-bootstrap.conf | 5 +++++ .gitlab-ci/nix.conf | 7 ++++--- .gitlab-ci/upload-to-cache.sh | 9 +++++++++ .gitlab-ci/write-minio-creds.sh | 10 ++++++++++ 6 files changed, 38 insertions(+), 6 deletions(-) create mode 100755 .gitlab-ci/make-minio-bucket.sh create mode 100644 .gitlab-ci/nix-bootstrap.conf create mode 100755 .gitlab-ci/upload-to-cache.sh create mode 100755 .gitlab-ci/write-minio-creds.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d90ec014..d4b4b02c5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,15 +22,16 @@ node dependencies: - nix -L build -o result .#uniworxNodeDependencies - nix-store --export $(nix-store -qR result) > node-dependencies.nar before_script: &nix-before + - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf + - .gitlab-ci/write-minio-creds.sh + - .gitlab-ci/make-minio-bucket.sh + - cp -pr --reflink=auto -L .gitlab-ci/upload-to-cache.sh /etc/nix/upload-to-cache.sh - install -m 0644 .gitlab-ci/nix.conf /etc/nix/nix.conf - git config --global init.defaultBranch master - install -v -m 0700 -d ~/.ssh - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts - install -v -T -m 0400 "${SSH_DEPLOY_KEY}" ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token - after_script: &nix-after - - nix-env -f '' -riA tree - - tree -a /minio-gitlab-runner-cache artifacts: paths: - node-dependencies.nar diff --git a/.gitlab-ci/make-minio-bucket.sh b/.gitlab-ci/make-minio-bucket.sh new file mode 100755 index 000000000..176c3cc16 --- /dev/null +++ b/.gitlab-ci/make-minio-bucket.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p bash -p coreutils -p minio-client + +export MC_HOST_minio-gitlab-runner-cache=https://$(cat /minio-gitlab-runner-cache/accesskey):$(cat /minio-gitlab-runner-cache/secretkey)@minio-gitlab-runner-cache:9000 + +mc mb --ignore-existing minio-gitlab-runner-cache/nix-cache diff --git a/.gitlab-ci/nix-bootstrap.conf b/.gitlab-ci/nix-bootstrap.conf new file mode 100644 index 000000000..036c2fc5e --- /dev/null +++ b/.gitlab-ci/nix-bootstrap.conf @@ -0,0 +1,5 @@ +sandbox = true +experimental-features = nix-command flakes ca-references +substituters = https://hydra.iohk.io https://cache.nixos.org/ +trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= +require-sigs = true \ No newline at end of file diff --git a/.gitlab-ci/nix.conf b/.gitlab-ci/nix.conf index 036c2fc5e..c427db666 100644 --- a/.gitlab-ci/nix.conf +++ b/.gitlab-ci/nix.conf @@ -1,5 +1,6 @@ sandbox = true experimental-features = nix-command flakes ca-references -substituters = https://hydra.iohk.io https://cache.nixos.org/ -trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= -require-sigs = true \ No newline at end of file +substituters = https://hydra.iohk.io https://cache.nixos.org/ s3://nix-cache?scheme=https&endpoint=minio-gitlab-runner-cache:9000 +trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= minio-gitlab-runner-cache:ZN5neq93MHqpnieHSBlxGgWvCL0WKrZ2S9QS+U5Bnro= +require-sigs = true +post-build-hook = /etc/nix/upload-to-cache.sh \ No newline at end of file diff --git a/.gitlab-ci/upload-to-cache.sh b/.gitlab-ci/upload-to-cache.sh new file mode 100755 index 000000000..b176fcf26 --- /dev/null +++ b/.gitlab-ci/upload-to-cache.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p bash -p coreutils + +set -eu +set -f # disable globbing +export IFS=' ' + +echo "Signing and uploading paths" $OUT_PATHS +exec nix copy --to "s3://nix-cache?scheme=https&endpoint=minio-gitlab-runner-cache:9000&secret-key=${NIX_CACHE_KEYFILE}" $OUT_PATHS diff --git a/.gitlab-ci/write-minio-creds.sh b/.gitlab-ci/write-minio-creds.sh new file mode 100755 index 000000000..f706d4607 --- /dev/null +++ b/.gitlab-ci/write-minio-creds.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p bash -p coreutils + +umask 0750 +mkdir -p ~/.aws +cat > ~/.aws/credentials < Date: Wed, 15 Sep 2021 13:36:30 +0200 Subject: [PATCH 23/52] chore(gitlab-ci): ... --- .gitlab-ci/make-minio-bucket.sh | 4 ++-- .gitlab-ci/nix.conf | 2 +- .gitlab-ci/upload-to-cache.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci/make-minio-bucket.sh b/.gitlab-ci/make-minio-bucket.sh index 176c3cc16..1287696ba 100755 --- a/.gitlab-ci/make-minio-bucket.sh +++ b/.gitlab-ci/make-minio-bucket.sh @@ -1,6 +1,6 @@ #!/usr/bin/env nix-shell #!nix-shell -i bash -p bash -p coreutils -p minio-client -export MC_HOST_minio-gitlab-runner-cache=https://$(cat /minio-gitlab-runner-cache/accesskey):$(cat /minio-gitlab-runner-cache/secretkey)@minio-gitlab-runner-cache:9000 +export MC_HOST_minio=https://$(cat /minio-gitlab-runner-cache/accesskey):$(cat /minio-gitlab-runner-cache/secretkey)@minio-gitlab-runner-cache:9000 -mc mb --ignore-existing minio-gitlab-runner-cache/nix-cache +mc mb --ignore-existing minio/nix-cache diff --git a/.gitlab-ci/nix.conf b/.gitlab-ci/nix.conf index c427db666..5c3f43fea 100644 --- a/.gitlab-ci/nix.conf +++ b/.gitlab-ci/nix.conf @@ -1,6 +1,6 @@ sandbox = true experimental-features = nix-command flakes ca-references -substituters = https://hydra.iohk.io https://cache.nixos.org/ s3://nix-cache?scheme=https&endpoint=minio-gitlab-runner-cache:9000 +substituters = https://hydra.iohk.io https://cache.nixos.org/ s3://nix-cache?scheme=http&endpoint=minio-gitlab-runner-cache:9000 trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= minio-gitlab-runner-cache:ZN5neq93MHqpnieHSBlxGgWvCL0WKrZ2S9QS+U5Bnro= require-sigs = true post-build-hook = /etc/nix/upload-to-cache.sh \ No newline at end of file diff --git a/.gitlab-ci/upload-to-cache.sh b/.gitlab-ci/upload-to-cache.sh index b176fcf26..68c7f1548 100755 --- a/.gitlab-ci/upload-to-cache.sh +++ b/.gitlab-ci/upload-to-cache.sh @@ -6,4 +6,4 @@ set -f # disable globbing export IFS=' ' echo "Signing and uploading paths" $OUT_PATHS -exec nix copy --to "s3://nix-cache?scheme=https&endpoint=minio-gitlab-runner-cache:9000&secret-key=${NIX_CACHE_KEYFILE}" $OUT_PATHS +exec nix copy --to "s3://nix-cache?scheme=http&endpoint=minio-gitlab-runner-cache:9000&secret-key=${NIX_CACHE_KEYFILE}" $OUT_PATHS From 9c8284f70dd3ffda3a3006679e0ae134cf9761a1 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Wed, 15 Sep 2021 13:40:01 +0200 Subject: [PATCH 24/52] chore(gitlab-ci): ... --- .gitlab-ci/make-minio-bucket.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci/make-minio-bucket.sh b/.gitlab-ci/make-minio-bucket.sh index 1287696ba..3797ba0b0 100755 --- a/.gitlab-ci/make-minio-bucket.sh +++ b/.gitlab-ci/make-minio-bucket.sh @@ -1,6 +1,6 @@ #!/usr/bin/env nix-shell #!nix-shell -i bash -p bash -p coreutils -p minio-client -export MC_HOST_minio=https://$(cat /minio-gitlab-runner-cache/accesskey):$(cat /minio-gitlab-runner-cache/secretkey)@minio-gitlab-runner-cache:9000 +export MC_HOST_minio=http://$(cat /minio-gitlab-runner-cache/accesskey):$(cat /minio-gitlab-runner-cache/secretkey)@minio-gitlab-runner-cache:9000 mc mb --ignore-existing minio/nix-cache From f5d311ae14e66144292fa398062b00e22993e5dd Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Wed, 15 Sep 2021 13:48:26 +0200 Subject: [PATCH 25/52] chore(gitlab-ci): ... --- .gitlab-ci/upload-to-cache.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci/upload-to-cache.sh b/.gitlab-ci/upload-to-cache.sh index 68c7f1548..ca79c1f02 100755 --- a/.gitlab-ci/upload-to-cache.sh +++ b/.gitlab-ci/upload-to-cache.sh @@ -1,5 +1,4 @@ -#!/usr/bin/env nix-shell -#!nix-shell -i bash -p bash -p coreutils +#!/usr/bin/bash set -eu set -f # disable globbing From e403bdc4bfa407fbf42a6f09cca6552ea3606474 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Wed, 15 Sep 2021 13:55:49 +0200 Subject: [PATCH 26/52] chore(gitlab-ci): ... --- .gitlab-ci/make-minio-bucket.sh | 2 +- .gitlab-ci/upload-to-cache.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci/make-minio-bucket.sh b/.gitlab-ci/make-minio-bucket.sh index 3797ba0b0..e5e94c3fe 100755 --- a/.gitlab-ci/make-minio-bucket.sh +++ b/.gitlab-ci/make-minio-bucket.sh @@ -1,6 +1,6 @@ #!/usr/bin/env nix-shell #!nix-shell -i bash -p bash -p coreutils -p minio-client -export MC_HOST_minio=http://$(cat /minio-gitlab-runner-cache/accesskey):$(cat /minio-gitlab-runner-cache/secretkey)@minio-gitlab-runner-cache:9000 +export MC_HOST_minio=http://$(cat /minio-gitlab-runner-cache/accesskey):$(cat /minio-gitlab-runner-cache/secretkey)@minio-gitlab-runner-cache mc mb --ignore-existing minio/nix-cache diff --git a/.gitlab-ci/upload-to-cache.sh b/.gitlab-ci/upload-to-cache.sh index ca79c1f02..8687983d7 100755 --- a/.gitlab-ci/upload-to-cache.sh +++ b/.gitlab-ci/upload-to-cache.sh @@ -5,4 +5,4 @@ set -f # disable globbing export IFS=' ' echo "Signing and uploading paths" $OUT_PATHS -exec nix copy --to "s3://nix-cache?scheme=http&endpoint=minio-gitlab-runner-cache:9000&secret-key=${NIX_CACHE_KEYFILE}" $OUT_PATHS +exec nix copy --to "s3://nix-cache?scheme=http&endpoint=minio-gitlab-runner-cache&secret-key=${NIX_CACHE_KEYFILE}" $OUT_PATHS From f748be98d102eb5bdbc820578646302c89d2274e Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Wed, 15 Sep 2021 14:01:01 +0200 Subject: [PATCH 27/52] chore(gitlab-ci): ... --- .gitlab-ci/nix.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci/nix.conf b/.gitlab-ci/nix.conf index 5c3f43fea..29c91aba8 100644 --- a/.gitlab-ci/nix.conf +++ b/.gitlab-ci/nix.conf @@ -1,6 +1,6 @@ sandbox = true experimental-features = nix-command flakes ca-references -substituters = https://hydra.iohk.io https://cache.nixos.org/ s3://nix-cache?scheme=http&endpoint=minio-gitlab-runner-cache:9000 +substituters = https://hydra.iohk.io https://cache.nixos.org/ s3://nix-cache?scheme=http&endpoint=minio-gitlab-runner-cache trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= minio-gitlab-runner-cache:ZN5neq93MHqpnieHSBlxGgWvCL0WKrZ2S9QS+U5Bnro= require-sigs = true post-build-hook = /etc/nix/upload-to-cache.sh \ No newline at end of file From 1303bf04d079e4478aaa545eda81a45083e0615c Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Wed, 15 Sep 2021 14:28:06 +0200 Subject: [PATCH 28/52] chore(nix): fix backend symlink-join --- flake.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 7b4fd1270..3c2109663 100644 --- a/flake.nix +++ b/flake.nix @@ -108,7 +108,12 @@ ''; uniworxBackend = pkgs.symlinkJoin { name = "uniworx"; - paths = with pkgs; [ "uniworx:exe:uniworx" "uniworx:exe:uniworxdb" "uniworx:exe:uniworxload" "uniworx:exe:uniworx-wflint" ]; + paths = [ + pkgs."uniworx:exe:uniworx" + pkgs."uniworx:exe:uniworxdb" + pkgs."uniworx:exe:uniworxload" + pkgs."uniworx:exe:uniworx-wflint" + ]; }; in { packages = haskellFlake.packages // { From 91a7d1788fbbe81cd8c11ed1b0de4ce8e44562b4 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Wed, 15 Sep 2021 15:15:26 +0200 Subject: [PATCH 29/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 1 + .gitlab-ci/write-minio-creds.sh | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d4b4b02c5..03e98869a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,6 +10,7 @@ default: variables: NIX_PATH: "nixpkgs=http://nixos.org/channels/nixos-21.05/nixexprs.tar.xz" + AWS_SHARED_CREDENTIALS_FILE: "/etc/aws/credentials" stages: - frontend:build diff --git a/.gitlab-ci/write-minio-creds.sh b/.gitlab-ci/write-minio-creds.sh index f706d4607..d978cdfc8 100755 --- a/.gitlab-ci/write-minio-creds.sh +++ b/.gitlab-ci/write-minio-creds.sh @@ -1,9 +1,9 @@ #!/usr/bin/env nix-shell #!nix-shell -i bash -p bash -p coreutils -umask 0750 -mkdir -p ~/.aws -cat > ~/.aws/credentials < /etc/aws/credentials < Date: Wed, 15 Sep 2021 15:19:32 +0200 Subject: [PATCH 30/52] chore(gitlab-ci): ... --- .gitlab-ci/write-minio-creds.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci/write-minio-creds.sh b/.gitlab-ci/write-minio-creds.sh index d978cdfc8..797365c00 100755 --- a/.gitlab-ci/write-minio-creds.sh +++ b/.gitlab-ci/write-minio-creds.sh @@ -1,10 +1,11 @@ #!/usr/bin/env nix-shell #!nix-shell -i bash -p bash -p coreutils -umask 7155 +umask 7333 mkdir -p /etc/aws cat > /etc/aws/credentials < Date: Wed, 15 Sep 2021 15:24:51 +0200 Subject: [PATCH 31/52] chore(gitlab-ci): ... --- .gitlab-ci/write-minio-creds.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci/write-minio-creds.sh b/.gitlab-ci/write-minio-creds.sh index 797365c00..fe94e2e67 100755 --- a/.gitlab-ci/write-minio-creds.sh +++ b/.gitlab-ci/write-minio-creds.sh @@ -1,11 +1,11 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p bash -p coreutils +#!nix-shell -i bash -p bash -p coreutils -p aws -umask 7333 -mkdir -p /etc/aws -cat > /etc/aws/credentials < ${AWS_SHARED_CREDENTIALS_FILE} < Date: Wed, 15 Sep 2021 15:28:07 +0200 Subject: [PATCH 32/52] chore(gitlab-ci): ... --- .gitlab-ci/write-minio-creds.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci/write-minio-creds.sh b/.gitlab-ci/write-minio-creds.sh index fe94e2e67..cc841c8d4 100755 --- a/.gitlab-ci/write-minio-creds.sh +++ b/.gitlab-ci/write-minio-creds.sh @@ -8,4 +8,4 @@ aws_access_key_id = $(cat /minio-gitlab-runner-cache/accesskey) aws_secret_access_key = $(cat /minio-gitlab-runner-cache/secretkey) EOF -aws --endpoint-url http://minio-gitlab-runner-cache ls +aws s3 ls --endpoint-url http://minio-gitlab-runner-cache From d6a63bd9ca1b53c1347bcb37bc0b603bfaad01ee Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Wed, 15 Sep 2021 15:36:21 +0200 Subject: [PATCH 33/52] chore(gitlab-ci): ... --- .gitlab-ci/write-minio-creds.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci/write-minio-creds.sh b/.gitlab-ci/write-minio-creds.sh index cc841c8d4..e179a9614 100755 --- a/.gitlab-ci/write-minio-creds.sh +++ b/.gitlab-ci/write-minio-creds.sh @@ -1,5 +1,5 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p bash -p coreutils -p aws +#!nix-shell -i bash -p bash -p coreutils -p awscli2 mkdir -p $(dirname ${AWS_SHARED_CREDENTIALS_FILE}) cat > ${AWS_SHARED_CREDENTIALS_FILE} < Date: Wed, 15 Sep 2021 15:50:45 +0200 Subject: [PATCH 34/52] chore(gitlab-ci): ... --- .gitlab-ci/nix.conf | 2 +- .gitlab-ci/write-minio-creds.sh | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci/nix.conf b/.gitlab-ci/nix.conf index 29c91aba8..884284cee 100644 --- a/.gitlab-ci/nix.conf +++ b/.gitlab-ci/nix.conf @@ -1,6 +1,6 @@ sandbox = true experimental-features = nix-command flakes ca-references -substituters = https://hydra.iohk.io https://cache.nixos.org/ s3://nix-cache?scheme=http&endpoint=minio-gitlab-runner-cache +substituters = https://hydra.iohk.io https://cache.nixos.org/ s3://nix-cache?profile=default&scheme=http&endpoint=minio-gitlab-runner-cache trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= minio-gitlab-runner-cache:ZN5neq93MHqpnieHSBlxGgWvCL0WKrZ2S9QS+U5Bnro= require-sigs = true post-build-hook = /etc/nix/upload-to-cache.sh \ No newline at end of file diff --git a/.gitlab-ci/write-minio-creds.sh b/.gitlab-ci/write-minio-creds.sh index e179a9614..90f0a57b8 100755 --- a/.gitlab-ci/write-minio-creds.sh +++ b/.gitlab-ci/write-minio-creds.sh @@ -1,5 +1,5 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p bash -p coreutils -p awscli2 +#!nix-shell -i bash -p bash -p coreutils mkdir -p $(dirname ${AWS_SHARED_CREDENTIALS_FILE}) cat > ${AWS_SHARED_CREDENTIALS_FILE} < ${AWS_SHARED_CREDENTIALS_FILE} < Date: Wed, 15 Sep 2021 16:17:34 +0200 Subject: [PATCH 35/52] chore(gitlab-ci): ... --- .gitlab-ci/nix.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci/nix.conf b/.gitlab-ci/nix.conf index 884284cee..713c109a0 100644 --- a/.gitlab-ci/nix.conf +++ b/.gitlab-ci/nix.conf @@ -1,6 +1,6 @@ sandbox = true experimental-features = nix-command flakes ca-references -substituters = https://hydra.iohk.io https://cache.nixos.org/ s3://nix-cache?profile=default&scheme=http&endpoint=minio-gitlab-runner-cache +substituters = https://hydra.iohk.io https://cache.nixos.org/ trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= minio-gitlab-runner-cache:ZN5neq93MHqpnieHSBlxGgWvCL0WKrZ2S9QS+U5Bnro= require-sigs = true post-build-hook = /etc/nix/upload-to-cache.sh \ No newline at end of file From 9390eb75b77a59f0beca47f08576fce55083d3bf Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Wed, 15 Sep 2021 16:24:31 +0200 Subject: [PATCH 36/52] chore(gitlab-ci): ... --- .gitlab-ci/upload-to-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci/upload-to-cache.sh b/.gitlab-ci/upload-to-cache.sh index 8687983d7..e012caa93 100755 --- a/.gitlab-ci/upload-to-cache.sh +++ b/.gitlab-ci/upload-to-cache.sh @@ -5,4 +5,4 @@ set -f # disable globbing export IFS=' ' echo "Signing and uploading paths" $OUT_PATHS -exec nix copy --to "s3://nix-cache?scheme=http&endpoint=minio-gitlab-runner-cache&secret-key=${NIX_CACHE_KEYFILE}" $OUT_PATHS +exec nix --debug copy --to "s3://nix-cache?scheme=http&endpoint=minio-gitlab-runner-cache&secret-key=${NIX_CACHE_KEYFILE}" $OUT_PATHS From 59007c32db859c92d13adc8218e1314d91d6e9e3 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Wed, 15 Sep 2021 16:29:01 +0200 Subject: [PATCH 37/52] chore(gitlab-ci): ... --- .gitlab-ci/upload-to-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci/upload-to-cache.sh b/.gitlab-ci/upload-to-cache.sh index e012caa93..171a4f00f 100755 --- a/.gitlab-ci/upload-to-cache.sh +++ b/.gitlab-ci/upload-to-cache.sh @@ -5,4 +5,4 @@ set -f # disable globbing export IFS=' ' echo "Signing and uploading paths" $OUT_PATHS -exec nix --debug copy --to "s3://nix-cache?scheme=http&endpoint=minio-gitlab-runner-cache&secret-key=${NIX_CACHE_KEYFILE}" $OUT_PATHS +exec nix --debug copy --to "s3://nix-cache?region=us-east-1&scheme=http&endpoint=minio-gitlab-runner-cache:80&secret-key=${NIX_CACHE_KEYFILE}" $OUT_PATHS From 7331ba3a5bc99d74ddeb6c579fa03bb4eed6490a Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Thu, 16 Sep 2021 12:15:57 +0200 Subject: [PATCH 38/52] chore(gitlab-ci): debug --- .gitlab-ci.yml | 1 + .gitlab-ci/debug-aws.patch | 12 ++++++++++++ .gitlab-ci/nix.conf | 2 +- .gitlab-ci/patched-nix.nix | 4 ++++ .gitlab-ci/upload-to-cache.sh | 2 +- 5 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 .gitlab-ci/debug-aws.patch create mode 100644 .gitlab-ci/patched-nix.nix diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 03e98869a..8e4a51874 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,6 +26,7 @@ node dependencies: - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf - .gitlab-ci/write-minio-creds.sh - .gitlab-ci/make-minio-bucket.sh + - nix-env -f .gitlab-ci/patched-nix.nix -iE 'f: f {}' - cp -pr --reflink=auto -L .gitlab-ci/upload-to-cache.sh /etc/nix/upload-to-cache.sh - install -m 0644 .gitlab-ci/nix.conf /etc/nix/nix.conf - git config --global init.defaultBranch master diff --git a/.gitlab-ci/debug-aws.patch b/.gitlab-ci/debug-aws.patch new file mode 100644 index 000000000..a7583c295 --- /dev/null +++ b/.gitlab-ci/debug-aws.patch @@ -0,0 +1,12 @@ +diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc +index 6bfbee044..ff406e5e4 100644 +--- a/src/libstore/s3-binary-cache-store.cc ++++ b/src/libstore/s3-binary-cache-store.cc +@@ -126,6 +126,7 @@ ref S3Helper::makeConfig(const string & region + initAWS(); + auto res = make_ref(); + res->region = region; ++ debug("configuring scheme %s", scheme); + if (!scheme.empty()) { + res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str()); + } diff --git a/.gitlab-ci/nix.conf b/.gitlab-ci/nix.conf index 713c109a0..29c91aba8 100644 --- a/.gitlab-ci/nix.conf +++ b/.gitlab-ci/nix.conf @@ -1,6 +1,6 @@ sandbox = true experimental-features = nix-command flakes ca-references -substituters = https://hydra.iohk.io https://cache.nixos.org/ +substituters = https://hydra.iohk.io https://cache.nixos.org/ s3://nix-cache?scheme=http&endpoint=minio-gitlab-runner-cache trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= minio-gitlab-runner-cache:ZN5neq93MHqpnieHSBlxGgWvCL0WKrZ2S9QS+U5Bnro= require-sigs = true post-build-hook = /etc/nix/upload-to-cache.sh \ No newline at end of file diff --git a/.gitlab-ci/patched-nix.nix b/.gitlab-ci/patched-nix.nix new file mode 100644 index 000000000..7b5f707af --- /dev/null +++ b/.gitlab-ci/patched-nix.nix @@ -0,0 +1,4 @@ +{ pkgs ? import {} }: +pkgs.nixUnstable.overrideAttrs (oldAttrs: { + patches = oldAttrs.patches or [] ++ [ ./debug-aws.patch ]; +}) diff --git a/.gitlab-ci/upload-to-cache.sh b/.gitlab-ci/upload-to-cache.sh index 171a4f00f..5adbd5cb8 100755 --- a/.gitlab-ci/upload-to-cache.sh +++ b/.gitlab-ci/upload-to-cache.sh @@ -5,4 +5,4 @@ set -f # disable globbing export IFS=' ' echo "Signing and uploading paths" $OUT_PATHS -exec nix --debug copy --to "s3://nix-cache?region=us-east-1&scheme=http&endpoint=minio-gitlab-runner-cache:80&secret-key=${NIX_CACHE_KEYFILE}" $OUT_PATHS +exec nix --debug copy --to "s3://nix-cache?region=us-east-1&scheme=http&endpoint=minio-gitlab-runner-cache&secret-key=${NIX_CACHE_KEYFILE}" $OUT_PATHS From 8706d00d3aa4e57a4bb39d419b382c0f6c46c883 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Thu, 16 Sep 2021 12:17:44 +0200 Subject: [PATCH 39/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8e4a51874..adcb8a932 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,7 +26,7 @@ node dependencies: - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf - .gitlab-ci/write-minio-creds.sh - .gitlab-ci/make-minio-bucket.sh - - nix-env -f .gitlab-ci/patched-nix.nix -iE 'f: f {}' + - "nix-env -f .gitlab-ci/patched-nix.nix -iE 'f: f {}'" - cp -pr --reflink=auto -L .gitlab-ci/upload-to-cache.sh /etc/nix/upload-to-cache.sh - install -m 0644 .gitlab-ci/nix.conf /etc/nix/nix.conf - git config --global init.defaultBranch master From a5631f30960a1f0f0b6fc2ffb3d354e474e1c979 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Thu, 16 Sep 2021 12:44:34 +0200 Subject: [PATCH 40/52] chore(gitlab-ci): ... --- .gitlab-ci/debug-aws.patch | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.gitlab-ci/debug-aws.patch b/.gitlab-ci/debug-aws.patch index a7583c295..f4a250b96 100644 --- a/.gitlab-ci/debug-aws.patch +++ b/.gitlab-ci/debug-aws.patch @@ -1,3 +1,16 @@ +diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc +index 514ab3bf9..25dab18bb 100644 +--- a/src/libstore/filetransfer.cc ++++ b/src/libstore/filetransfer.cc +@@ -696,6 +696,8 @@ struct curlFileTransfer : public FileTransfer + std::string scheme = get(params, "scheme").value_or(""); + std::string endpoint = get(params, "endpoint").value_or(""); + ++ debug("enqueueFileTransfer: scheme: %s", scheme); ++ + S3Helper s3Helper(profile, region, scheme, endpoint); + + // FIXME: implement ETag diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 6bfbee044..ff406e5e4 100644 --- a/src/libstore/s3-binary-cache-store.cc From ea749b3ad132512b7e961f761da9c3e0d558e81c Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Fri, 17 Sep 2021 10:35:34 +0200 Subject: [PATCH 41/52] chore(gitlab-ci): ... --- .gitlab-ci/fix-aws-scheme.patch | 13 +++++++++++++ .gitlab-ci/patched-nix.nix | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 .gitlab-ci/fix-aws-scheme.patch diff --git a/.gitlab-ci/fix-aws-scheme.patch b/.gitlab-ci/fix-aws-scheme.patch new file mode 100644 index 000000000..ac9416262 --- /dev/null +++ b/.gitlab-ci/fix-aws-scheme.patch @@ -0,0 +1,13 @@ +diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc +index 6bfbee044..51d86c4e6 100644 +--- a/src/libstore/s3-binary-cache-store.cc ++++ b/src/libstore/s3-binary-cache-store.cc +@@ -209,7 +209,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual + S3Helper s3Helper; + + S3BinaryCacheStoreImpl( +- const std::string & scheme, ++ const std::string & uriScheme, + const std::string & bucketName, + const Params & params) + : StoreConfig(params) diff --git a/.gitlab-ci/patched-nix.nix b/.gitlab-ci/patched-nix.nix index 7b5f707af..84f2d601e 100644 --- a/.gitlab-ci/patched-nix.nix +++ b/.gitlab-ci/patched-nix.nix @@ -1,4 +1,4 @@ { pkgs ? import {} }: pkgs.nixUnstable.overrideAttrs (oldAttrs: { - patches = oldAttrs.patches or [] ++ [ ./debug-aws.patch ]; + patches = oldAttrs.patches or [] ++ [ ./fix-aws-scheme.patch ]; }) From d12066045551d8476131728f8e0ab5dc3afcf90c Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Fri, 17 Sep 2021 10:41:59 +0200 Subject: [PATCH 42/52] chore(gitlab-ci): ... --- .gitlab-ci/upload-to-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci/upload-to-cache.sh b/.gitlab-ci/upload-to-cache.sh index 5adbd5cb8..bbadb87b2 100755 --- a/.gitlab-ci/upload-to-cache.sh +++ b/.gitlab-ci/upload-to-cache.sh @@ -5,4 +5,4 @@ set -f # disable globbing export IFS=' ' echo "Signing and uploading paths" $OUT_PATHS -exec nix --debug copy --to "s3://nix-cache?region=us-east-1&scheme=http&endpoint=minio-gitlab-runner-cache&secret-key=${NIX_CACHE_KEYFILE}" $OUT_PATHS +exec nix copy --to "s3://nix-cache?region=us-east-1&scheme=http&endpoint=minio-gitlab-runner-cache&secret-key=${NIX_CACHE_KEYFILE}" $OUT_PATHS From 17ccec3d1adbd20e24d089f864686c7e8c3eec15 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Fri, 17 Sep 2021 12:04:54 +0200 Subject: [PATCH 43/52] chore(gitlab-ci): ... --- flake.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index 3c2109663..2ace1e91d 100644 --- a/flake.nix +++ b/flake.nix @@ -102,17 +102,17 @@ mkPushUniworxDocker = dockerImage: pkgs.writeScriptBin "push-${dockerImage.name}" '' #!${pkgs.zsh}/bin/zsh -xe - target=''${1-docker://registry.gitlab.com/fradrive/fradrive/uniworx-demo:${dockerImage.imageTag}} + target=''${1-docker://registry.gitlab.com/fradrive/fradrive/${dockerImage.name}:${dockerImage.imageTag}} [[ -n "''${1}" ]] && shift ${pkgs.skopeo}/bin/skopeo ''${@} --insecure-policy copy docker-archive://${dockerImage} ''${target} ''; uniworxBackend = pkgs.symlinkJoin { name = "uniworx"; paths = [ - pkgs."uniworx:exe:uniworx" - pkgs."uniworx:exe:uniworxdb" - pkgs."uniworx:exe:uniworxload" - pkgs."uniworx:exe:uniworx-wflint" + haskellFlake.packages."uniworx:exe:uniworx" + haskellFlake.packages."uniworx:exe:uniworxdb" + haskellFlake.packages."uniworx:exe:uniworxload" + haskellFlake.packages."uniworx:exe:uniworx-wflint" ]; }; in { From bad9afc1b8d0537bc5551281f6c4867ea5b66695 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Fri, 17 Sep 2021 13:59:55 +0200 Subject: [PATCH 44/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 3 +-- flake.lock | 17 +++++++++++++++++ flake.nix | 23 +++++++++++++++++------ nix/aws-patch.nix | 11 +++++++++++ nix/docker/default.nix | 24 +++++++++++++++++------- 5 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 nix/aws-patch.nix diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index adcb8a932..55a948019 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,7 @@ workflow: default: image: - name: nixpkgs/nix-flakes:nixos-21.05 + name: registry.gitlab.com/fradrive/fradrive/nix-unstable variables: NIX_PATH: "nixpkgs=http://nixos.org/channels/nixos-21.05/nixexprs.tar.xz" @@ -26,7 +26,6 @@ node dependencies: - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf - .gitlab-ci/write-minio-creds.sh - .gitlab-ci/make-minio-bucket.sh - - "nix-env -f .gitlab-ci/patched-nix.nix -iE 'f: f {}'" - cp -pr --reflink=auto -L .gitlab-ci/upload-to-cache.sh /etc/nix/upload-to-cache.sh - install -m 0644 .gitlab-ci/nix.conf /etc/nix/nix.conf - git config --global init.defaultBranch master diff --git a/flake.lock b/flake.lock index 9d1b7a81e..ed9736994 100644 --- a/flake.lock +++ b/flake.lock @@ -151,6 +151,22 @@ "url": "ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/cryptonite.git" } }, + "docker-nixpkgs": { + "flake": false, + "locked": { + "lastModified": 1631525111, + "narHash": "sha256-dJKw280B1Hp0f6tSkBtkY0nrMT0DR/bmerrS3cQdBjI=", + "owner": "nix-community", + "repo": "docker-nixpkgs", + "rev": "df7f64139a0b2384fb2448dc2a406815f8d03867", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "docker-nixpkgs", + "type": "github" + } + }, "encoding": { "flake": false, "locked": { @@ -465,6 +481,7 @@ "conduit-resumablesink": "conduit-resumablesink", "cryptoids": "cryptoids", "cryptonite": "cryptonite", + "docker-nixpkgs": "docker-nixpkgs", "encoding": "encoding", "esqueleto": "esqueleto", "flake-utils": [ diff --git a/flake.nix b/flake.nix index 2ace1e91d..141118d40 100644 --- a/flake.nix +++ b/flake.nix @@ -3,6 +3,11 @@ inputs.nixpkgs.follows = "haskell-nix/nixpkgs-unstable"; inputs.flake-utils.follows = "haskell-nix/flake-utils"; + inputs.docker-nixpkgs = { + url = "github:nix-community/docker-nixpkgs"; + flake = false; + }; + inputs.encoding = { url = "git+ssh://git@gitlab2.rz.ifi.lmu.de/uni2work/encoding.git?ref=uni2work"; flake = false; @@ -69,7 +74,7 @@ flake = false; }; - outputs = inputs@{ self, nixpkgs, flake-utils, haskell-nix, ... }: flake-utils.lib.eachSystem ["x86_64-linux"] + outputs = inputs@{ self, nixpkgs, flake-utils, haskell-nix, docker-nixpkgs, ... }: flake-utils.lib.eachSystem ["x86_64-linux"] (system: let frontendSource = pkgs.lib.sourceByRegex ./. [ "^(assets|frontend)(/.*)?$" @@ -92,6 +97,11 @@ config.allowUnfree = true; }; overlays = [ + # remove once https://github.com/NixOS/nix/pull/5266 has landed in https://hub.docker.com/r/nixpkgs/nix-unstable + (import ./nix/aws-patch.nix) + (import "${docker-nixpkgs}/overlay.nix") + # end remove + (import ./nix/maildev) haskell-nix.overlay (import ./nix/uniworx { inherit inputs frontendSource backendSource; }) @@ -99,10 +109,10 @@ ]; haskellFlake = pkgs.uniworx.flake {}; - mkPushUniworxDocker = dockerImage: pkgs.writeScriptBin "push-${dockerImage.name}" '' + mkPushDocker = imageName: dockerImage: pkgs.writeScriptBin "push-${dockerImage.imageName}" '' #!${pkgs.zsh}/bin/zsh -xe - target=''${1-docker://registry.gitlab.com/fradrive/fradrive/${dockerImage.name}:${dockerImage.imageTag}} + target=''${1-docker://registry.gitlab.com/fradrive/fradrive/${imageName}:${dockerImage.imageTag}} [[ -n "''${1}" ]] && shift ${pkgs.skopeo}/bin/skopeo ''${@} --insecure-policy copy docker-archive://${dockerImage} ''${target} ''; @@ -117,12 +127,13 @@ }; in { packages = haskellFlake.packages // { - inherit (pkgs) uniworxNodeDependencies uniworxWellKnown uniworxFrontend uniworxDemoDocker uniworxDocker; + inherit (pkgs) uniworxNodeDependencies uniworxWellKnown uniworxFrontend uniworxDemoDocker uniworxDocker ciDocker; inherit uniworxBackend; }; apps = haskellFlake.apps // { - pushUniworxDemoDocker = flake-utils.lib.mkApp { drv = mkPushUniworxDocker pkgs.uniworxDemoDocker; }; - pushUniworxDocker = flake-utils.lib.mkApp { drv = mkPushUniworxDocker pkgs.uniworxDocker; }; + pushUniworxDemoDocker = flake-utils.lib.mkApp { drv = mkPushDocker "uniworx-demo" pkgs.uniworxDemoDocker; }; + pushUniworxDocker = flake-utils.lib.mkApp { drv = mkPushDocker "uniworx" pkgs.uniworxDocker; }; + pushCIDocker = flake-utils.lib.mkApp { drv = mkPushDocker "nix-unstable" pkgs.ciDocker; }; calculateMaterializedSha = flake-utils.lib.mkApp { drv = pkgs.uniworx.stack-nix.passthru.calculateMaterializedSha; exePath = ""; }; }; checks = haskellFlake.checks // { diff --git a/nix/aws-patch.nix b/nix/aws-patch.nix new file mode 100644 index 000000000..250192c67 --- /dev/null +++ b/nix/aws-patch.nix @@ -0,0 +1,11 @@ +final: prev: +let + awsPatch = prev.fetchurl { + url = "https://github.com/gkleen/nix/commit/fd67a0f927ec0711eba59714939ff939fc95db38.diff"; + hash = "sha256-1dJ9zGQvYu5b47O2NjdggSSinlGQDcqBwXoZcKUGfYQ="; + }; +in { + nixUnstable = prev.nixUnstable.overrideAttrs (oldAttrs: { + patches = oldAttrs.patches or [] ++ [ awsPatch ]; + }); +} diff --git a/nix/docker/default.nix b/nix/docker/default.nix index 558dadfde..85f6f8fbf 100644 --- a/nix/docker/default.nix +++ b/nix/docker/default.nix @@ -3,18 +3,20 @@ with prev.lib; let + created = + let + fromDate = builtins.readFile (prev.runCommand "date" { nativeBuildInputs = with final; [ coreutils ]; } '' + printf '%s' $(date -Is -d '@${toString self.lastModified}') > $out + ''); + in if self ? lastModified then fromDate else "1970-01-01T00:00:01Z"; + mkUniworxDocker = { isDemo }: prev.dockerTools.buildImage { name = "uniworx${optionalString isDemo "-demo"}"; tag = let versionFile = if isDemo then ./demo-version.json else ./version.json; in (builtins.fromJSON (prev.lib.readFile versionFile)).version; - created = - let - fromDate = builtins.readFile (prev.runCommand "date" { nativeBuildInputs = with final; [ coreutils ]; } '' - printf '%s' $(date -Is -d '@${toString self.lastModified}') > $out - ''); - in if self ? lastModified then fromDate else "1970-01-01T00:00:01Z"; + inherit created; contents = with final; [ uniworx.uniworx.components.exes.uniworx @@ -121,7 +123,15 @@ let }; }; }; -in mapAttrs (_name: mkUniworxDocker) { +in +{ + ciDocker = prev.docker-nixpkgs.nix-flakes.overrideAttrs (oldAttrs: { + inherit created; + contents = oldAttrs.contents or [] ++ [ + final.bash final.coreutils final.minio-client + ]; + }); +} // mapAttrs (_name: mkUniworxDocker) { uniworxDemoDocker = { isDemo = true; }; uniworxDocker = { isDemo = false; }; } From 814ebfad2772bb2f958c6a2ed74fa3f52dddd8c0 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Fri, 17 Sep 2021 14:02:27 +0200 Subject: [PATCH 45/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 55a948019..43c6b825e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,7 @@ workflow: default: image: - name: registry.gitlab.com/fradrive/fradrive/nix-unstable + name: registry.gitlab.com/fradrive/fradrive/nix-unstable:kkq79z2lyip9wasq9sfd8mgqgifj9prs variables: NIX_PATH: "nixpkgs=http://nixos.org/channels/nixos-21.05/nixexprs.tar.xz" From 2e09c25085f55047d3d12d2c213a3fc5f7853edc Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Fri, 17 Sep 2021 14:07:14 +0200 Subject: [PATCH 46/52] chore(gitlab-ci): ... --- .gitlab-ci/make-minio-bucket.sh | 3 +-- .gitlab-ci/write-minio-creds.sh | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci/make-minio-bucket.sh b/.gitlab-ci/make-minio-bucket.sh index e5e94c3fe..2bba31234 100755 --- a/.gitlab-ci/make-minio-bucket.sh +++ b/.gitlab-ci/make-minio-bucket.sh @@ -1,5 +1,4 @@ -#!/usr/bin/env nix-shell -#!nix-shell -i bash -p bash -p coreutils -p minio-client +#!/usr/bin/bash export MC_HOST_minio=http://$(cat /minio-gitlab-runner-cache/accesskey):$(cat /minio-gitlab-runner-cache/secretkey)@minio-gitlab-runner-cache diff --git a/.gitlab-ci/write-minio-creds.sh b/.gitlab-ci/write-minio-creds.sh index 90f0a57b8..356c367f1 100755 --- a/.gitlab-ci/write-minio-creds.sh +++ b/.gitlab-ci/write-minio-creds.sh @@ -1,5 +1,4 @@ -#!/usr/bin/env nix-shell -#!nix-shell -i bash -p bash -p coreutils +#!/usr/bin/bash mkdir -p $(dirname ${AWS_SHARED_CREDENTIALS_FILE}) cat > ${AWS_SHARED_CREDENTIALS_FILE} < Date: Fri, 17 Sep 2021 14:29:06 +0200 Subject: [PATCH 47/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 2 +- nix/docker/default.nix | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 43c6b825e..78d273513 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,7 @@ workflow: default: image: - name: registry.gitlab.com/fradrive/fradrive/nix-unstable:kkq79z2lyip9wasq9sfd8mgqgifj9prs + name: registry.gitlab.com/fradrive/fradrive/nix-unstable:hcdba12p7ajl9xylgmw4kfn6p9c185xy variables: NIX_PATH: "nixpkgs=http://nixos.org/channels/nixos-21.05/nixexprs.tar.xz" diff --git a/nix/docker/default.nix b/nix/docker/default.nix index 85f6f8fbf..beb2d9354 100644 --- a/nix/docker/default.nix +++ b/nix/docker/default.nix @@ -125,12 +125,16 @@ let }; in { - ciDocker = prev.docker-nixpkgs.nix-flakes.overrideAttrs (oldAttrs: { + ciDocker = prev.dockerTools.buildImageWithNixDb rec { + name = "uniworx-ci"; inherit created; - contents = oldAttrs.contents or [] ++ [ + tag = fromImage.imageTag; + fromImage = prev.docker-nixpkgs.nix-unstable; + + contents = [ final.bash final.coreutils final.minio-client ]; - }); + }; } // mapAttrs (_name: mkUniworxDocker) { uniworxDemoDocker = { isDemo = true; }; uniworxDocker = { isDemo = false; }; From e3984e6ab1679611ae324ae2c9118477ad9ff929 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Fri, 17 Sep 2021 14:30:13 +0200 Subject: [PATCH 48/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 78d273513..1ee94b2cb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,7 @@ node dependencies: - nix -L build -o result .#uniworxNodeDependencies - nix-store --export $(nix-store -qR result) > node-dependencies.nar before_script: &nix-before + - mkdir -p /etc/nix - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf - .gitlab-ci/write-minio-creds.sh - .gitlab-ci/make-minio-bucket.sh From 80625f5ef75c0e687c039bb474e0b084f3a6a658 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Fri, 17 Sep 2021 20:10:00 +0200 Subject: [PATCH 49/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1ee94b2cb..15d5ab267 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -87,6 +87,7 @@ backend: - nix-store --import backend.nar before_script: *nix-before needs: - job: frontend @@ -94,6 +95,7 @@ backend: artifacts: paths: - bin/ + - backend.nar name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" expire_in: "1 day" retry: 2 From 74490ce7ac8cc34bedcb9591e98726cc3edbc98e Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Fri, 17 Sep 2021 21:51:49 +0200 Subject: [PATCH 50/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 29 ++++++++++++++--------------- nix/docker/default.nix | 6 ++++-- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 15d5ab267..d22e8f975 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ node dependencies: stage: frontend:build script: - nix -L build -o result .#uniworxNodeDependencies - - nix-store --export $(nix-store -qR result) > node-dependencies.nar + - nix-store --export $(nix-store -qR result) | xz > node-dependencies.nar.xz before_script: &nix-before - mkdir -p /etc/nix - install -m 0644 .gitlab-ci/nix-bootstrap.conf /etc/nix/nix.conf @@ -36,7 +36,7 @@ node dependencies: - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token artifacts: paths: - - node-dependencies.nar + - node-dependencies.nar.xz name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" expire_in: "1 day" retry: 2 @@ -45,16 +45,16 @@ node dependencies: well known: stage: frontend:build script: - - nix-store --import well-known.nar + - nix-store --export $(nix-store -qR result) | xz > well-known.nar.xz before_script: *nix-before needs: - job: node dependencies artifacts: true artifacts: paths: - - well-known.nar + - well-known.nar.xz name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" expire_in: "1 day" retry: 2 @@ -63,10 +63,10 @@ well known: frontend: stage: frontend:build script: - - nix-store --import frontend.nar + - nix-store --export $(nix-store -qR result) | xz > frontend.nar.xz before_script: *nix-before needs: - job: node dependencies @@ -75,7 +75,7 @@ frontend: artifacts: true artifacts: paths: - - frontend.nar + - frontend.nar.xz name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" expire_in: "1 day" retry: 2 @@ -84,18 +84,17 @@ frontend: backend: stage: backend:build script: - - nix-store --import backend.nar + - nix-store --export $(nix-store -qR result) | xz > backend.nar.xz before_script: *nix-before needs: - job: frontend artifacts: true artifacts: paths: - - bin/ - - backend.nar + - backend.nar.xz name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" expire_in: "1 day" retry: 2 @@ -104,7 +103,7 @@ backend: demo docker: stage: docker:build script: - - nix-store --import Date: Sat, 18 Sep 2021 14:29:39 +0200 Subject: [PATCH 51/52] chore(gitlab-ci): ... --- .gitlab-ci.yml | 182 ++++++++++++++++++++++++++++++++++++++-- flake.nix | 14 +--- nix/parse-changelog.nix | 13 +++ 3 files changed, 189 insertions(+), 20 deletions(-) create mode 100644 nix/parse-changelog.nix diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d22e8f975..02ee9c3f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,11 @@ variables: stages: - frontend:build - backend:build + - test - docker:build + - prepare release + - upload + - release node dependencies: stage: frontend:build @@ -34,6 +38,11 @@ node dependencies: - install -v -T -m 0644 "${SSH_KNOWN_HOSTS}" ~/.ssh/known_hosts - install -v -T -m 0400 "${SSH_DEPLOY_KEY}" ~/.ssh/deploy && echo "IdentityFile ~/.ssh/deploy" >> ~/.ssh/config; - install -v -T -m 0644 "${FONTAWESOME_NPM_AUTH_FILE}" /etc/fontawesome-token + - | + if [ -n "${REGISTRY_AUTHFILE}" ]; then + mkdir -p ~/.docker + install -v -T -m 0400 "${REGISTRY_AUTHFILE}" ~/.docker/config.json + fi artifacts: paths: - node-dependencies.nar.xz @@ -81,37 +90,127 @@ frontend: retry: 2 interruptible: true -backend: +uniworx:lib:uniworx: stage: backend:build script: - xzcat frontend.nar.xz | nix-store --import - - nix -L build -o result .#uniworxBackend - - cp -pr --reflink=auto -L result/bin . - - nix-store --export $(nix-store -qR result) | xz > backend.nar.xz + - nix -L build -o result .#uniworx:lib:uniworx + - nix-store --export $(nix-store -qR result) | xz > uniworx:lib:uniworx.nar.xz before_script: *nix-before needs: - job: frontend artifacts: true artifacts: paths: - - backend.nar.xz + - uniworx:lib:uniworx.nar.xz name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" expire_in: "1 day" retry: 2 interruptible: true +uniworx:exe:uniworx: + stage: backend:build + script: + - xzcat uniworx:lib:uniworx.nar.xz | nix-store --import + - nix -L build -o result .#uniworx:exe:uniworx + - nix-store --export $(nix-store -qR result) | xz > uniworx:exe:uniworx.nar.xz + before_script: *nix-before + needs: + - job: uniworx:lib:uniworx + artifacts: true + artifacts: + paths: + - uniworx:exe:uniworx.nar.xz + name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" + expire_in: "1 day" + retry: 2 + interruptible: true + +uniworx:exe:uniworx-wflint: + stage: backend:build + script: + - xzcat uniworx:lib:uniworx.nar.xz | nix-store --import + - nix -L build -o result .#uniworx:exe:uniworx-wflint + - nix-store --export $(nix-store -qR result) | xz > uniworx:exe:uniworx-wflint.nar.xz + before_script: *nix-before + needs: + - job: uniworx:lib:uniworx + artifacts: true + artifacts: + paths: + - uniworx:exe:uniworx-wflint.nar.xz + name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" + expire_in: "1 day" + retry: 2 + interruptible: true + +uniworx:exe:uniworxdb: + stage: backend:build + script: + - xzcat uniworx:lib:uniworx.nar.xz | nix-store --import + - nix -L build -o result .#uniworx:exe:uniworxdb + - nix-store --export $(nix-store -qR result) | xz > uniworx:exe:uniworxdb.nar.xz + before_script: *nix-before + needs: + - job: uniworx:lib:uniworx + artifacts: true + artifacts: + paths: + - uniworx:exe:uniworxdb.nar.xz + name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" + expire_in: "1 day" + retry: 2 + interruptible: true + +uniworx:exe:uniworxload: + stage: backend:build + script: + - xzcat uniworx:lib:uniworx.nar.xz | nix-store --import + - nix -L build -o result .#uniworx:exe:uniworxload + - nix-store --export $(nix-store -qR result) | xz > uniworx:exe:uniworxload.nar.xz + before_script: *nix-before + needs: + - job: uniworx:lib:uniworx + artifacts: true + artifacts: + paths: + - uniworx:exe:uniworxload.nar.xz + name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" + expire_in: "1 day" + retry: 2 + interruptible: true + +check: + stage: test + script: + - xzcat frontend.nar.xz | nix-store --import + - xzcat uniworx:lib:uniworx.nar.xz | nix-store --import + - nix -L flake check + before_script: *nix-before + needs: + - job: frontend + artifacts: true + - job: uniworx:lib:uniworx + artifacts: true + retry: 2 + interruptible: true + demo docker: stage: docker:build script: - - xzcat backend.nar.xz | nix-store --import + - xzcat uniworx:exe:uniworx.nar.xz | nix-store --import - nix -L build -o result .#uniworxDemoDocker + - nix-store --export $(nix-store -qR result) > uniworx-demo.nar # image is already compressed, no need for xz - cp -pr --reflink=auto -L result uniworx-demo.tar.gz before_script: *nix-before needs: - - job: backend + - job: uniworx:exe:uniworx artifacts: true + - job: check + artifacts: false artifacts: paths: + - uniworx-demo.nar - uniworx-demo.tar.gz name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" expire_in: "1 day" @@ -121,17 +220,82 @@ demo docker: docker: stage: docker:build script: - - xzcat backend.nar.xz | nix-store --import + - xzcat uniworx:exe:uniworx.nar.xz | nix-store --import - nix -L build -o result .#uniworxDocker + - nix-store --export $(nix-store -qR result) > uniworx.nar # image is already compressed, no need for xz - cp -pr --reflink=auto -L result uniworx.tar.gz before_script: *nix-before needs: - - job: backend + - job: uniworx:exe:uniworx artifacts: true + - job: check + artifacts: false artifacts: paths: + - uniworx.nar - uniworx.tar.gz name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" expire_in: "1 day" retry: 2 interruptible: true + +parse-changelog: + stage: prepare release + needs: + - job: node dependencies + artifacts: true + rules: + - if: $CI_COMMIT_REF_NAME =~ /^v[0-9].*/ + when: on_success + before_script: *nix-before + script: + - xzcat node-dependencies.nar.xz | nix-store --import + - nix -L run .#jqChangelogJson -- -r '.versions[0].version' > .current-version + - nix -L run .#jqChangelogJson -- -r '.versions[0].body' > .current-changelog.md + artifacts: + paths: + - .current-version + - .current-changelog.md + name: "changelog-${CI_COMMIT_SHORT_SHA}" + expire_in: "1 day" + retry: 2 + interruptible: true + +upload docker: + stage: upload + script: + - nix-store --import < uniworx.nar + - nix -L run .#pushUniworxDocker + before_script: *nix-before + needs: + - job: docker + artifacts: true + rules: + - if: $CI_COMMIT_REF_NAME =~ /^v[0-9].*/ + when: on_success + +upload demo docker: + stage: upload + script: + - nix-store --import < uniworx-demo.nar + - nix -L run .#pushUniworxDemoDocker + before_script: *nix-before + needs: + - job: demo docker + artifacts: true + rules: + - if: $CI_COMMIT_REF_NAME =~ /^v[0-9].*/ + when: on_success + +release: + variables: + GIT_STRATEGY: none + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + rules: + - if: $CI_COMMIT_REF_NAME =~ /^v[0-9].*/ + when: on_success + before_script: + - export VERSION="$(cat .current-version)" + script: + - release-cli create --name "${VERSION}" --tag-name $CI_COMMIT_TAG --description .current-changelog.md diff --git a/flake.nix b/flake.nix index 141118d40..7f9ffa33d 100644 --- a/flake.nix +++ b/flake.nix @@ -106,6 +106,7 @@ haskell-nix.overlay (import ./nix/uniworx { inherit inputs frontendSource backendSource; }) (import ./nix/docker { inherit self; }) + (import ./nix/parse-changelog.nix {}) ]; haskellFlake = pkgs.uniworx.flake {}; @@ -116,25 +117,16 @@ [[ -n "''${1}" ]] && shift ${pkgs.skopeo}/bin/skopeo ''${@} --insecure-policy copy docker-archive://${dockerImage} ''${target} ''; - uniworxBackend = pkgs.symlinkJoin { - name = "uniworx"; - paths = [ - haskellFlake.packages."uniworx:exe:uniworx" - haskellFlake.packages."uniworx:exe:uniworxdb" - haskellFlake.packages."uniworx:exe:uniworxload" - haskellFlake.packages."uniworx:exe:uniworx-wflint" - ]; - }; in { packages = haskellFlake.packages // { - inherit (pkgs) uniworxNodeDependencies uniworxWellKnown uniworxFrontend uniworxDemoDocker uniworxDocker ciDocker; - inherit uniworxBackend; + inherit (pkgs) uniworxNodeDependencies uniworxWellKnown uniworxFrontend uniworxDemoDocker uniworxDocker ciDocker changelogJson; }; apps = haskellFlake.apps // { pushUniworxDemoDocker = flake-utils.lib.mkApp { drv = mkPushDocker "uniworx-demo" pkgs.uniworxDemoDocker; }; pushUniworxDocker = flake-utils.lib.mkApp { drv = mkPushDocker "uniworx" pkgs.uniworxDocker; }; pushCIDocker = flake-utils.lib.mkApp { drv = mkPushDocker "nix-unstable" pkgs.ciDocker; }; calculateMaterializedSha = flake-utils.lib.mkApp { drv = pkgs.uniworx.stack-nix.passthru.calculateMaterializedSha; exePath = ""; }; + jqChangelogJson = flake-utils.lib.mkApp { drv = pkgs.jqChangelogJson; }; }; checks = haskellFlake.checks // { uniworxFrontend = pkgs.uniworxFrontend.check; diff --git a/nix/parse-changelog.nix b/nix/parse-changelog.nix new file mode 100644 index 000000000..62dc6102d --- /dev/null +++ b/nix/parse-changelog.nix @@ -0,0 +1,13 @@ +{}: prev: final: rec { + changelogJson = prev.runCommand "changelog.json" { + } '' + ln -s ${final.uniworxNodeDependencies}/lib/node_modules ./node_modules + export PATH="${final.uniworxNodeDependencies}/bin:$PATH" + + changelog-parser ${../CHANGELOG.md} > $out + ''; + + jqChangelogJson = prev.writeShellScriptBin "jq-changelog" '' + exec -- ${final.jq}/bin/jq $@ < ${changelogJson} + ''; +} From be70d90402974a64d52e582afcf3cc57c4518b04 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Mon, 20 Sep 2021 17:12:36 +0200 Subject: [PATCH 52/52] chore(docker): no root & fill --- flake.nix | 6 ++++-- nix/docker/default.nix | 34 ++++++++++++++++------------------ nix/docker/demo-version.json | 2 +- package.yaml | 5 +++++ test/Database/Fill.hs | 25 ++++++++++++++----------- 5 files changed, 40 insertions(+), 32 deletions(-) diff --git a/flake.nix b/flake.nix index 7f9ffa33d..99125eb0b 100644 --- a/flake.nix +++ b/flake.nix @@ -90,6 +90,7 @@ "^(assets|app|hlint|load|messages|models|src|templates|test|testdata|wflint)(/.*)?$" "^config(/(archive-types|mimetypes|personalised-sheet-files-collate|settings\.yml|submission-blacklist|test-settings\.yml|video-types|wordlist\.txt))?$" "^routes$" + "^testdata(/.*)?$" ]; pkgs = import nixpkgs { @@ -97,11 +98,12 @@ config.allowUnfree = true; }; overlays = [ - # remove once https://github.com/NixOS/nix/pull/5266 has landed in https://hub.docker.com/r/nixpkgs/nix-unstable + # remove once https://github.com/NixOS/nix/pull/5266 has landed in nixpkgs used here (see flake.lock) (import ./nix/aws-patch.nix) - (import "${docker-nixpkgs}/overlay.nix") # end remove + (import "${docker-nixpkgs}/overlay.nix") + (import ./nix/maildev) haskell-nix.overlay (import ./nix/uniworx { inherit inputs frontendSource backendSource; }) diff --git a/nix/docker/default.nix b/nix/docker/default.nix index 514e82bf1..5d76b5d9b 100644 --- a/nix/docker/default.nix +++ b/nix/docker/default.nix @@ -20,8 +20,8 @@ let contents = with final; [ uniworx.uniworx.components.exes.uniworx - prev.dockerTools.binSh - ] ++ optionals isDemo [ postgresql_12 memcached ]; + prev.dockerTools.binSh findutils coreutils + ] ++ optionals isDemo [ postgresql_12 memcached uniworx.uniworx.components.exes.uniworxdb ]; runAsRoot = '' #!${final.stdenv.shell} @@ -38,21 +38,15 @@ let install -d -g uniworx -o uniworx -m 0755 /var/log/uniworx ${optionalString isDemo '' - groupadd -r postgres - useradd -r -g postgres -d /var/lib/postgres -M postgres - install -d -g postgres -o postgres -m 0750 /var/lib/postgres + install -d -g uniworx -o uniworx -m 0750 /var/lib/postgres - groupadd -r memcached - useradd -r -g memcached -d /var/lib/memcached -M memcached - install -d -g memcached -o memcached -m 0750 /var/lib/memcached + install -d -g uniworx -o uniworx -m 0750 /var/lib/memcached - gpasswd -a uniworx postgres - - install -d -g postgres -o postgres -m 0755 /var/log/postgres - install -d -g memcached -o memcached -m 0755 /var/log/memcached + install -d -g uniworx -o uniworx -m 0755 /var/log/postgres + install -d -g uniworx -o uniworx -m 0755 /var/log/memcached mkdir -p /run - install -d -g postgres -o postgres -m 0755 /run/postgres + install -d -g uniworx -o uniworx -m 0755 /run/postgres ''} ''; @@ -60,7 +54,6 @@ let let entrypoint = prev.writeScriptBin "uniworx-entrypoint" '' #!${final.zsh}/bin/zsh -xe - export PATH=${final.su}/bin:${final.findutils}/bin:${final.coreutils}/bin:/bin cTime=$(date -Is) @@ -76,11 +69,11 @@ let pgNew=1 fi - [[ -z "''${pgNew}" ]] || su postgres -c "initdb --no-locale --encoding=UTF8 -D ''${pgDir}" - su postgres -c "pg_ctl start -D ''${pgDir} -l ''${pgLogFile} -w -o '-k ''${pgSockDir} -c listen_addresses= -c hba_file=${postgresHba} -c unix_socket_permissions=0777 -c max_connections=9990 -c shared_preload_libraries=pg_stat_statements -c auto_explain.log_min_duration=100ms'" + [[ -z "''${pgNew}" ]] || initdb --no-locale --encoding=UTF8 --username postgres --pgdata ''${pgDir} + pg_ctl start -D ''${pgDir} -l ''${pgLogFile} -w -o "-k ''${pgSockDir} -c listen_addresses= -c hba_file=${postgresHba} -c unix_socket_permissions=0777 -c max_connections=9990 -c shared_preload_libraries=pg_stat_statements -c auto_explain.log_min_duration=100ms" [[ -z "''${pgNew}" ]] || psql -f ${postgresSchema} postgres postgres - su memcached -c "cd /var/lib/memcached; memcached -p 11212" &>/var/log/memcached/''${cTime}.log & + ( cd /var/lib/memcached; memcached -p 11212 ) &>/var/log/memcached/''${cTime}.log & export SESSION_MEMCACHED_HOST=localhost export SESSION_MEMCACHED_PORT=11212 ''} @@ -95,7 +88,11 @@ let done < <(find "''${configDir}" \( -name '*.yml' -o -name '*.yaml' \) -print0 | sort -rz) fi configs+=('${uniworxConfig}') - exec -- su uniworx -c "cd /var/lib/uniworx; uniworx ''${configs}" + cd /var/lib/uniworx + ${optionalString isDemo '' + [[ -z "''${pgNew}" ]] || uniworxdb -f ''${configs} + ''} + exec -- uniworx ''${configs} ''; postgresSchema = prev.writeText "schema.sql" '' CREATE USER uniworx WITH SUPERUSER; @@ -112,6 +109,7 @@ let ''; in { Cmd = [ "${entrypoint}/bin/uniworx-entrypoint" ]; + User = "uniworx:uniworx"; ExposedPorts = { "8080/tcp" = {}; }; diff --git a/nix/docker/demo-version.json b/nix/docker/demo-version.json index 10e64e00d..dfcc3af19 100644 --- a/nix/docker/demo-version.json +++ b/nix/docker/demo-version.json @@ -1,3 +1,3 @@ { - "version": "0.2.0" + "version": "0.3.0" } diff --git a/package.yaml b/package.yaml index 2793c89b4..710b808e1 100644 --- a/package.yaml +++ b/package.yaml @@ -253,6 +253,10 @@ when: else: ghc-options: - -O -fllvm + +data-files: + - testdata/** + library: source-dirs: src executables: @@ -276,6 +280,7 @@ executables: - uniworx other-modules: - Database.Fill + - Paths_uniworx when: - condition: flag(library-only) buildable: false diff --git a/test/Database/Fill.hs b/test/Database/Fill.hs index b57095456..c793ec982 100644 --- a/test/Database/Fill.hs +++ b/test/Database/Fill.hs @@ -38,13 +38,15 @@ import qualified Data.Yaml as Yaml import Utils.Workflow import Utils.Workflow.Lint -import System.Directory (getModificationTime, doesFileExist) +import System.Directory (getModificationTime, doesFileExist, doesDirectoryExist) import System.FilePath.Glob (glob) import System.IO (hPutStrLn) import qualified Data.List.NonEmpty as NonEmpty +import Paths_uniworx (getDataFileName) + data WorkflowIndexItem = WorkflowIndexItem { wiiGraphFile :: FilePath @@ -59,13 +61,13 @@ deriveJSON defaultOptions } ''WorkflowIndexItem -testdataDir :: FilePath -testdataDir = "testdata" - +testdataFile :: MonadIO m => FilePath -> m FilePath +testdataFile = liftIO . getDataFileName . ("testdata" ) insertFile :: ( HasFileReference fRef, PersistRecordBackend fRef SqlBackend ) => FileReferenceResidual fRef -> FilePath -> DB (Key fRef) insertFile residual fileTitle = do - let fileContent = Just . C.sourceFile $ testdataDir fileTitle + filepath <- testdataFile fileTitle + let fileContent = Just $ C.sourceFile filepath fileModified <- liftIO getCurrentTime sinkFile' File{..} residual >>= insert @@ -1399,8 +1401,6 @@ fillDb = do return . pure . AllocationPriorityNumericRecord matr . fromList $ sortOn Down prios _other -> return mempty - liftIO . LBS.writeFile (testdataDir "bigAlloc_numeric.csv") $ Csv.encode numericPriorities - ordinalPriorities <- do manyUsers' <- shuffleM manyUsers flip foldMapM manyUsers' $ \uid -> do @@ -1410,16 +1410,19 @@ fillDb = do return . pure $ Csv.Only matr _other -> return mempty - liftIO . LBS.writeFile (testdataDir "bigAlloc_ordinal.csv") $ Csv.encode ordinalPriorities + liftIO . handle (\(_ :: IOException) -> return ()) $ do + haveTestdata <- doesDirectoryExist "testdata" + LBS.writeFile (bool id ("testdata" ) haveTestdata "bigAlloc_numeric.csv") $ Csv.encode numericPriorities + LBS.writeFile (bool id ("testdata" ) haveTestdata "bigAlloc_ordinal.csv") $ Csv.encode ordinalPriorities - whenM (liftIO . doesFileExist $ testdataDir "workflows" "_index.yaml") $ do + whenM (liftIO . doesFileExist <=< testdataFile $ "workflows" "_index.yaml") $ do let displayLinterIssue :: MonadIO m => WorkflowGraphLinterIssue -> m () displayLinterIssue = liftIO . hPutStrLn stderr . displayException - wfIndex <- Yaml.decodeFileThrow @_ @(Map WorkflowDefinitionName WorkflowIndexItem) $ testdataDir "workflows" "_index.yaml" + wfIndex <- Yaml.decodeFileThrow @_ @(Map WorkflowDefinitionName WorkflowIndexItem) =<< testdataFile ("workflows" "_index.yaml") iforM_ wfIndex $ \wiName WorkflowIndexItem{..} -> handleSql displayLinterIssue $ do - graph <- Yaml.decodeFileThrow $ testdataDir "workflows" wiiGraphFile + graph <- Yaml.decodeFileThrow =<< testdataFile ("workflows" wiiGraphFile) for_ (lintWorkflowGraph graph) $ mapM_ throwM workflowDefinitionGraph <- insertSharedWorkflowGraph graph let workflowDef = WorkflowDefinition{..}