From 570cfc238bdccd3438124f96290b9272c8e82f0f Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Fri, 14 Mar 2025 20:19:26 +0100 Subject: [PATCH 1/5] fix(static): fix addStaticContent by using memcached again to supply static files --- src/Foundation/Instances.hs | 4 +- src/Foundation/Yesod/StaticContent.hs | 53 +++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 src/Foundation/Yesod/StaticContent.hs diff --git a/src/Foundation/Instances.hs b/src/Foundation/Instances.hs index 02de76927..7620d6cdc 100644 --- a/src/Foundation/Instances.hs +++ b/src/Foundation/Instances.hs @@ -29,6 +29,7 @@ import qualified Foundation.Yesod.Middleware as UniWorX import qualified Foundation.Yesod.ErrorHandler as UniWorX import qualified Foundation.Yesod.Persist as UniWorX import qualified Foundation.Yesod.Auth as UniWorX +import qualified Foundation.Yesod.StaticContent as UniWorX import Foundation.Instances.ButtonClass import Foundation.SiteLayout @@ -90,8 +91,7 @@ instance Yesod UniWorX where isAuthorized :: HasCallStack => Route UniWorX -> Bool -> HandlerFor UniWorX AuthResult isAuthorized r w = runDBRead $ evalAccess r w - -- TODO: minify on production builds using ifdef DEVELOP instead of bundler-based minify - addStaticContent = embedStaticContent appStatic StaticR Right + addStaticContent = UniWorX.addStaticContent fileUpload _site _length = FileUploadMemory lbsBackEnd diff --git a/src/Foundation/Yesod/StaticContent.hs b/src/Foundation/Yesod/StaticContent.hs new file mode 100644 index 000000000..79fbf726d --- /dev/null +++ b/src/Foundation/Yesod/StaticContent.hs @@ -0,0 +1,53 @@ +-- SPDX-FileCopyrightText: 2022-2025 Sarah Vaupel , Gregor Kleen , Sarah Vaupel +-- +-- SPDX-License-Identifier: AGPL-3.0-or-later + +module Foundation.Yesod.StaticContent + ( addStaticContent + ) where + +import Import.NoFoundation hiding (addStaticContent) + +import Foundation.Type + +import qualified Database.Memcached.Binary.IO as Memcached + +import qualified Data.ByteString.Lazy as Lazy +import qualified Data.ByteString.Base64.URL as Base64 (encodeUnpadded) +import Data.ByteArray (convert) +import Crypto.Hash (SHAKE256) +import Crypto.Hash.Conduit (sinkHash) +import Data.Bits (Bits(zeroBits)) + +import qualified Data.Conduit.Combinators as C + + +addStaticContent :: Text + -> Text + -> Lazy.ByteString + -> HandlerFor UniWorX (Maybe (Either Text (Route UniWorX, [(Text, Text)]))) +addStaticContent ext _mime content = do + UniWorX{appWidgetMemcached, appSettings'} <- getYesod + for ((,) <$> appWidgetMemcached <*> appWidgetMemcachedConf appSettings') $ \(mConn, WidgetMemcachedConf{ widgetMemcachedConf = MemcachedConf { memcachedExpiry }, widgetMemcachedBaseUrl }) -> do + let expiry = maybe 0 ceiling memcachedExpiry + touch = liftIO $ Memcached.touch expiry (encodeUtf8 $ pack fileName) mConn + addItem = liftIO $ Memcached.add zeroBits expiry (encodeUtf8 $ pack fileName) content mConn + absoluteLink = unpack widgetMemcachedBaseUrl fileName + catchIf Memcached.isKeyNotFound touch . const $ + handleIf Memcached.isKeyExists (const $ return ()) addItem + return . Left $ pack absoluteLink + where + -- Generate a unique filename based on the content itself, this is used + -- for deduplication so a collision resistant hash function is required + -- + -- SHA-3 (SHAKE256) seemed to be a future-proof choice + -- + -- Length of hash is 144 bits ~~instead of MD5's 128, so as to avoid + -- padding after base64-conversion~~ for backwards compatibility + fileName = (<.> unpack ext) + . unpack + . decodeUtf8 + . Base64.encodeUnpadded + . (convert :: Digest (SHAKE256 144) -> ByteString) + . runConduitPure + $ C.sourceLazy content .| sinkHash From 9ce3b5d146bfa3053db260cdeeee3ff3618251e9 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 17 Mar 2025 09:41:54 +0100 Subject: [PATCH 2/5] chore(release): 27.4.59-0.0.18+145-build-system-rewrite --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2fb6b069..27e1df4c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [27.4.59-0.0.18+145-build-system-rewrite](https://fraport@dev.azure.com/fraport/Fahrerausbildung/_git/FRADrive//compare/v27.4.59-test-g0.0.17...27.4.59-0.0.18+145-build-system-rewrite) (2025-03-17) + +### Bug Fixes + +* **static:** fix addStaticContent by using memcached again to supply static files ([570cfc2](https://fraport@dev.azure.com/fraport/Fahrerausbildung/_git/FRADrive/commit/570cfc238bdccd3438124f96290b9272c8e82f0f)) + ## [v27.4.59-test-g0.0.17](https://fraport@dev.azure.com/fraport/Fahrerausbildung/_git/FRADrive//compare/v27.4.59-test-f0.0.17...v27.4.59-test-g0.0.17) (2025-02-18) ## [v27.4.59-test-f0.0.17](https://fraport@dev.azure.com/fraport/Fahrerausbildung/_git/FRADrive//compare/v27.4.59-test-g0.0.16...v27.4.59-test-f0.0.17) (2025-02-17) From d33a792045d04a1dcb6e773f1b9a9f104d3b67cf Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 17 Mar 2025 10:15:16 +0100 Subject: [PATCH 3/5] build(Makefile): remove SET_IMAGE in favour of LOCAL_CONTAINERFILE; use non-local postgres with launch args --- Makefile | 33 +++++++++++++++++++++------------ docker/postgres/Dockerfile | 9 --------- 2 files changed, 21 insertions(+), 21 deletions(-) delete mode 100644 docker/postgres/Dockerfile diff --git a/Makefile b/Makefile index efce641e0..ba298ca21 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,8 @@ export IMAGE_REGISTRY = docker.io export MEMCACHED_IMAGE = $(IMAGE_REGISTRY)/memcached:latest export MINIO_IMAGE = $(IMAGE_REGISTRY)/minio/minio:latest export MAILDEV_IMAGE = $(IMAGE_REGISTRY)/maildev/maildev:latest # TODO: needs different port than 1025 to avoid conflicts +export POSTGRES_IMAGE = $(IMAGE_REGISTRY)/library/postgres:12 +export LOCAL_CONTAINERFILE export IN_CONTAINER ?= false export IN_CI ?= false @@ -31,7 +33,7 @@ export SERVICE export SERVICE_VARIANT ?= $(SERVICE) export JOB export IMAGE -export SET_IMAGE +export MAKECALL export ENTRYPOINT export EXEC_OPTS @@ -113,6 +115,7 @@ start: .PHONY: %-backend %-backend: SERVICE=backend %-backend: SERVICE_VARIANT=backend +%-backend: LOCAL_CONTAINERFILE=true %-backend: IMAGE=localhost/fradrive/backend %-backend: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443" @@ -120,17 +123,20 @@ start: %-uniworxdb: SERVICE=backend %-uniworxdb: SERVICE_VARIANT=uniworxdb %-uniworxdb: IMAGE=localhost/fradrive/backend +%-uniworxdb: LOCAL_CONTAINERFILE=true .PHONY: %-ghci %-ghci: SERVICE=backend %-ghci: SERVICE_VARIANT=ghci %-ghci: IMAGE=localhost/fradrive/backend +%-ghci: LOCAL_CONTAINERFILE=true .PHONY: %-hoogle %-hoogle: SERVICE=backend %-hoogle: SERVICE_VARIANT=hoogle %-hoogle: BASE_PORTS = "HOOGLE_PORT=8081" %-hoogle: IMAGE=localhost/fradrive/backend +%-hoogle: LOCAL_CONTAINERFILE=true --start-hoogle: HOOGLE_PORT=`cat $(CONTAINER_FILE) | grep 'HOOGLE_PORT=' | sed 's/HOOGLE_PORT=//'` ; \ stack $(STACK_CORES) hoogle -- server --local --port $${HOOGLE_PORT} @@ -138,24 +144,28 @@ start: .PHONY: %-frontend %-frontend: SERVICE=frontend %-frontend: SERVICE_VARIANT=frontend +%-frontend: LOCAL_CONTAINERFILE=true %-frontend: IMAGE=localhost/fradrive/frontend .PHONY: %-postgres %-postgres: SERVICE=postgres %-postgres: SERVICE_VARIANT=postgres %-postgres: BASE_PORTS = "PGPORT=5432" -%-postgres: SET_IMAGE=localhost/fradrive/postgres +%-postgres: IMAGE=--env POSTGRES_HOST_AUTH_METHOD=trust -v ./docker/postgres/pg_hba.conf:/tmp/pg_hba.conf -v ./docker/postgres/postgresql.conf:/tmp/postgresql.conf -v ./docker/postgres/pgconfig.sh:/docker-entrypoint-initdb.d/_pgconfig.sh -v ./docker/postgres/schema.sql:/docker-entrypoint-initdb.d/schema.sql $$(POSTGRES_IMAGE) +%-postgres: LOCAL_CONTAINERFILE=false .PHONY: %-memcached %-memcached: SERVICE=memcached %-memcached: SERVICE_VARIANT=memcached -%-memcached: SET_IMAGE=$$(MEMCACHED_IMAGE) --port=`cat $$(CONTAINER_FILE) | grep 'MEMCACHED_PORT=' | sed 's/MEMCACHED_PORT=//'` +%-memcached: IMAGE=$$(MEMCACHED_IMAGE) --port=`cat $$(CONTAINER_FILE) | grep 'MEMCACHED_PORT=' | sed 's/MEMCACHED_PORT=//'` +%-memcached: LOCAL_CONTAINERFILE=false %-memcached: BASE_PORTS = "MEMCACHED_PORT=11211" .PHONY: %-maildev %-maildev: SERVICE=maildev %-maildev: SERVICE_VARIANT=maildev -%-maildev: SET_IMAGE=$$(MAILDEV_IMAGE) --port=`cat $$(CONTAINER_FILE) | grep 'MAILDEV_PORT=' | sed 's/MAILDEV_PORT=//'` +%-maildev: IMAGE=$$(MAILDEV_IMAGE) --port=`cat $$(CONTAINER_FILE) | grep 'MAILDEV_PORT=' | sed 's/MAILDEV_PORT=//'` +%-maildev: LOCAL_CONTAINERFILE=false %-maildev: BASE_PORTS = "MAILDEV_PORT=1025" .PHONY: %-release @@ -163,11 +173,13 @@ start: %-release: SERVICE=fradrive %-release: SERVICE_VARIANT=fradrive %-release: IMAGE=localhost/fradrive/fradrive +%-release: LOCAL_CONTAINERFILE=true .PHONY: %-minio %-minio: SERVICE=minio %-minio: SERVICE_VARIANT=minio -%-minio: SET_IMAGE=$$(MINIO_IMAGE) -- server `mktemp` --address=:`cat $$(CONTAINER_FILE) | grep 'UPLOAD_S3_PORT=' | sed 's/UPLOAD_S3_PORT=//'` +%-minio: IMAGE=$$(MINIO_IMAGE) -- server `mktemp` --address=:`cat $$(CONTAINER_FILE) | grep 'UPLOAD_S3_PORT=' | sed 's/UPLOAD_S3_PORT=//'` +%-minio: LOCAL_CONTAINERFILE=false %-minio: BASE_PORTS = "UPLOAD_S3_PORT=9000" .PHONY: start-% @@ -229,7 +241,7 @@ ghci: shell-ghci; rebuild-%: $(MAKE) -- --image-build SERVICE=$* NO_CACHE=--no-cache --image-build: -ifeq "$(IMAGE)" "localhost/fradrive/$(SERVICE)" +ifeq "$(LOCAL_CONTAINERFILE)" "true" rm -f .Dockerfile ln -s docker/$(SERVICE)/Dockerfile .Dockerfile PROJECT_DIR=/fradrive; \ @@ -250,12 +262,9 @@ endif DEVELOP=`cat develop/.current` ; \ ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" "$(CONTAINER_INIT)" "$(CONTAINER_CLEANUP)" & \ CONTAINER_NAME=fradrive.$(CURR_DEV).$(CONTAINER_IDENT) ; \ - if ! [ -z "$(SET_IMAGE)" ] ; \ + if [ "$(LOCAL_CONTAINERFILE)" = "true" ] ; \ then \ - IMAGE="$(SET_IMAGE)" ; \ - else \ - IMAGE=$(IMAGE) ; \ - MAKECALL="make -- --$(JOB)-$(SERVICE_VARIANT) IN_CONTAINER=true" ; \ + MAKECALL="make -- --$(JOB)-$(SERVICE_VARIANT) IN_CONTAINER=true" ; \ fi ; \ CONTAINER_ID=`$(CONTAINER_BGRUN) \ -v $(PWD):$(PROJECT_DIR):rw \ @@ -265,7 +274,7 @@ endif --env JOB=$(JOB) \ --env SRC=$(SRC) \ --name $${CONTAINER_NAME} \ - $${IMAGE} \ + $(IMAGE) \ $${MAKECALL} \ ` ; \ printf "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)" ; \ diff --git a/docker/postgres/Dockerfile b/docker/postgres/Dockerfile deleted file mode 100644 index c3a14c0b9..000000000 --- a/docker/postgres/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM docker.io/postgres:12 - -# Allow for connecting to database without password authentication -ENV POSTGRES_HOST_AUTH_METHOD=trust - -COPY --chown=postgres:postgres docker/postgres/pg_hba.conf /tmp/pg_hba.conf -COPY --chown=postgres:postgres docker/postgres/postgresql.conf /tmp/postgresql.conf -COPY docker/postgres/pgconfig.sh /docker-entrypoint-initdb.d/_pgconfig.sh -COPY --chown=postgres:postgres docker/postgres/schema.sql /docker-entrypoint-initdb.d/schema.sql From f40818c1ccf3ea84f5ce1634ef25c6cb9a17b2f0 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 17 Mar 2025 10:18:06 +0100 Subject: [PATCH 4/5] build(docker/backend): tlmgr init-usertree --- docker/backend/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 699e79c04..19c24662a 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -24,6 +24,7 @@ RUN apt-get -y update && apt-get -y install fonts-roboto RUN apt-get -y update && apt-get -y install texlive # RUN ls /usr/local/texlive # RUN chown -hR root /usr/local/texlive/2018 +RUN tlmgr init-usertree RUN tlmgr option repository ftp://tug.org/historic/systems/texlive/2018/tlnet-final RUN tlmgr update --self --all From 0ac972bf22aae6021a5647514f75b88937b20676 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 17 Mar 2025 10:18:25 +0100 Subject: [PATCH 5/5] chore(release): 27.4.59-0.0.19+145-build-system-rewrite --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27e1df4c7..3361e6db0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [27.4.59-0.0.19+145-build-system-rewrite](https://fraport@dev.azure.com/fraport/Fahrerausbildung/_git/FRADrive//compare/27.4.59-0.0.18+145-build-system-rewrite...27.4.59-0.0.19+145-build-system-rewrite) (2025-03-17) + ## [27.4.59-0.0.18+145-build-system-rewrite](https://fraport@dev.azure.com/fraport/Fahrerausbildung/_git/FRADrive//compare/v27.4.59-test-g0.0.17...27.4.59-0.0.18+145-build-system-rewrite) (2025-03-17) ### Bug Fixes