From 8724fec74f9e428394e81d44126a90e83a595995 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 28 Oct 2024 13:26:42 +0100 Subject: [PATCH 01/64] ci(azure-pipelines): create initial azure-pipelines.yml --- azure-pipelines.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..75a8f78c8 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,14 @@ +pool: 'Prod Private Agent Pool' + +jobs: +- job: Hello-World + container: + image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' + endpoint: devfra + steps: + - displayName: 'Run a one-line script' + script: echo Hello, world! + - displayName: 'Run a multi-line script' + script: | + echo Add other tasks to build, test, and deploy your project. + echo See https://aka.ms/yaml \ No newline at end of file From 4b8fd3f4d8e55052cf328ed89028eee75540e91d Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 28 Oct 2024 13:32:26 +0100 Subject: [PATCH 02/64] ci(azure-pipelines): rename pipelines yaml --- azure-pipelines.yml => azure-pipelines.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename azure-pipelines.yml => azure-pipelines.yaml (100%) diff --git a/azure-pipelines.yml b/azure-pipelines.yaml similarity index 100% rename from azure-pipelines.yml rename to azure-pipelines.yaml From 9767070046e8b89bc0ea74591126a84b85559d6a Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 28 Oct 2024 13:34:49 +0100 Subject: [PATCH 03/64] ci(azure-pipelines): modify test steps --- azure-pipelines.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 75a8f78c8..f99f113cb 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -6,9 +6,9 @@ jobs: image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' endpoint: devfra steps: - - displayName: 'Run a one-line script' - script: echo Hello, world! - - displayName: 'Run a multi-line script' - script: | + - script: echo Hello, world! + displayName: 'Run a one-line script' + - script: | echo Add other tasks to build, test, and deploy your project. - echo See https://aka.ms/yaml \ No newline at end of file + echo See https://aka.ms/yaml + displayName: 'Run a multi-line script' \ No newline at end of file From 569eab24d59119617519abb8f42b39046bcf9444 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 28 Oct 2024 13:35:30 +0100 Subject: [PATCH 04/64] ci(azure-pipelines): modify test job name --- azure-pipelines.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index f99f113cb..7918e7a8c 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -1,7 +1,7 @@ pool: 'Prod Private Agent Pool' jobs: -- job: Hello-World +- job: HelloWorld container: image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' endpoint: devfra From 2eac288c25d8f4baebee0655965744a4f1b3059c Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 28 Oct 2024 13:53:19 +0100 Subject: [PATCH 05/64] ci(azure-pipelines): add license copyright identifier --- azure-pipelines.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 7918e7a8c..139560748 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2024 Sarah Vaupel +# +# SPDX-License-Identifier: AGPL-3.0-or-later + pool: 'Prod Private Agent Pool' jobs: From 1510116083527ec58000f8768bd61e4c815e4a64 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Tue, 29 Oct 2024 14:54:19 +0100 Subject: [PATCH 06/64] build(Makefile): add fontawesome-source json as icons dependency --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 718cd320a..33937362f 100644 --- a/Makefile +++ b/Makefile @@ -151,7 +151,7 @@ package-lock.json: package.json assets: assets/favicons assets/icons; assets/favicons: ./utils/faviconize.pl assets/favicon.svg long assets/favicons -assets/icons: node_modules +assets/icons: node_modules utils/rename-fa.json ./utils/renamer.pl node_modules/@fortawesome/fontawesome-free/svgs/solid utils/rename-fa.json assets/icons/fradrive ./utils/renamer.pl node_modules/@fortawesome/fontawesome-free/svgs/regular utils/rename-fa.json assets/icons/fradrive From 06f48fe4083b3037a661ff8a8edec6f7be7b6e37 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Tue, 29 Oct 2024 14:56:50 +0100 Subject: [PATCH 07/64] build(Makefile): remove deprecated stackopts --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 33937362f..cb48e5804 100644 --- a/Makefile +++ b/Makefile @@ -181,15 +181,15 @@ well-known: static; # HELP(compile-backend): compile backend --compile-backend: - stack build --fast --profile --library-profiling --executable-profiling --flag uniworx:-library-only --local-bin-path $$(pwd)/bin $(stackopts) + stack build --fast --profile --library-profiling --executable-profiling --flag uniworx:-library-only --local-bin-path $$(pwd)/bin # HELP(lint-backend): lint backend --lint-backend: - stack build --test --fast --work-dir=.stack-work-test --flag uniworx:library-only uniworx:test:hlint $(stackopts) + stack build --test --fast --work-dir=.stack-work-test --flag uniworx:library-only uniworx:test:hlint # HELP(test-backend): test backend --test-backend: - stack build --test --coverage --fast --work-dir=.stack-work-test --flag uniworx:library-only $(stackopts) + stack build --test --coverage --fast --work-dir=.stack-work-test --flag uniworx:library-only # TODO: better name .PHONY: db @@ -211,7 +211,7 @@ db: compile-frontend --image-build --containerized---db; $(MAKE) -- --image-run---.stack ; \ fi --.stack: stack.yaml stack.yaml.lock package.yaml - stack build --fast --only-dependencies $(stackopts) + stack build --fast --only-dependencies ##### BACKEND TARGETS ##### ########################### From 90b8af8cc1ba3d387de12f071645bc84a56b4781 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Tue, 29 Oct 2024 15:13:42 +0100 Subject: [PATCH 08/64] build(Makefile): set DEVELOPMENT ghc option based on PROD make var --- Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index cb48e5804..3dbff319e 100644 --- a/Makefile +++ b/Makefile @@ -32,8 +32,12 @@ export DEVELOP export CONTAINER_FILE export CONTAINER_FILE_CONTENT -export PROD ?= false export LOGSIZE ?= 1024 +export PROD ?= false + +ifneq ($(PROD),true) + export --DEVELOPMENT=--flag uniworx:dev +endif # HELP HEADER START # To see the definition of all available targets, take a look into the Makefile. @@ -181,7 +185,7 @@ well-known: static; # HELP(compile-backend): compile backend --compile-backend: - stack build --fast --profile --library-profiling --executable-profiling --flag uniworx:-library-only --local-bin-path $$(pwd)/bin + stack build --fast --profile --library-profiling --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin # HELP(lint-backend): lint backend --lint-backend: @@ -189,7 +193,7 @@ well-known: static; # HELP(test-backend): test backend --test-backend: - stack build --test --coverage --fast --work-dir=.stack-work-test --flag uniworx:library-only + stack build --test --coverage --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) # TODO: better name .PHONY: db From 9c27263d636718949cfff6b5745684650aa6e87d Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Wed, 30 Oct 2024 01:09:30 +0100 Subject: [PATCH 09/64] build(Makefile): do not automatically create new develop on start; document for manual usage --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3dbff319e..50b33fe2f 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ release: .PHONY: start # HELP: start database services, frontend and backend -start: new-develop start-database start-memcached start-minio start-frontend start-backend; +start: start-database start-memcached start-minio start-frontend start-backend; .PHONY: compile # HELP: compile frontend and backend @@ -397,6 +397,7 @@ SET_DEVELOP = $(eval DEVELOP=develop/$$(CURR_DEV)) NEW_DEVELOP = $(eval DEVELOP=develop/$$(DATE)) .PHONY: new-develop +# HELP: instantiate new development bundle, i.e. create new directory under develop/ new-develop: $(NEW_DEVELOP) mkdir -p $(DEVELOP) From 4d7f6158c91d93ca985cb3a81d1503bae59b29e9 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Wed, 30 Oct 2024 01:09:57 +0100 Subject: [PATCH 10/64] build(Makefile): ensure develop for compile-backend --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 50b33fe2f..28a534549 100644 --- a/Makefile +++ b/Makefile @@ -109,6 +109,8 @@ compile: $(MAKE) compile-frontend $(MAKE) compile-backend +compile-%: --develop + .PHONY: lint # HELP: lint frontend and backend lint: lint-frontend lint-backend; From e20aa0139a87a4fa7547865f015df6d56e2a0b8a Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Wed, 30 Oct 2024 03:03:16 +0100 Subject: [PATCH 11/64] build(Makefile): restructure icons generation --- Makefile | 6 +++--- utils/rename-fa.json => assets/icons-src/fontawesome.json | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename utils/rename-fa.json => assets/icons-src/fontawesome.json (100%) diff --git a/Makefile b/Makefile index 28a534549..bbfe5177f 100644 --- a/Makefile +++ b/Makefile @@ -158,9 +158,9 @@ assets: assets/favicons assets/icons; assets/favicons: ./utils/faviconize.pl assets/favicon.svg long assets/favicons assets/icons: node_modules utils/rename-fa.json - ./utils/renamer.pl node_modules/@fortawesome/fontawesome-free/svgs/solid utils/rename-fa.json assets/icons/fradrive - ./utils/renamer.pl node_modules/@fortawesome/fontawesome-free/svgs/regular utils/rename-fa.json assets/icons/fradrive - + ./utils/renamer.pl node_modules/@fortawesome/fontawesome-free/svgs/solid assets/icons-src/fontawesome.json assets/icons/fradrive + ./utils/renamer.pl node_modules/@fortawesome/fontawesome-free/svgs/regular assets/icons-src/fontawesome.json assets/icons/fradrive + cp assets/icons-src/*.svg assets/icons/fradrive static: node_modules assets esbuild.config.mjs npm run build well-known: static; diff --git a/utils/rename-fa.json b/assets/icons-src/fontawesome.json similarity index 100% rename from utils/rename-fa.json rename to assets/icons-src/fontawesome.json From cd6e98f915a56efd01d908609918ce90de553419 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Wed, 30 Oct 2024 03:04:43 +0100 Subject: [PATCH 12/64] build(Makefile): add uniworx:dev flag to more stack calls --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index bbfe5177f..34bf1cb43 100644 --- a/Makefile +++ b/Makefile @@ -179,7 +179,7 @@ well-known: static; --start-backend: DEV_PORT_HTTP=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTP=' | sed 's/DEV_PORT_HTTP=//'`; \ DEV_PORT_HTTPS=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTPS=' | sed 's/DEV_PORT_HTTPS=//'`; \ - stack --work-dir=.stack-work-run exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" + stack --work-dir=.stack-work-run $(--DEVELOPMENT) exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" --start-hoogle-backend: HOOGLE_PORT=`cat $(CONTAINER_FILE) | grep 'HOOGLE_PORT=' | sed 's/HOOGLE_PORT=//'` ; \ echo "${HOOGLE_PORT}" ; \ @@ -191,7 +191,7 @@ well-known: static; # HELP(lint-backend): lint backend --lint-backend: - stack build --test --fast --work-dir=.stack-work-test --flag uniworx:library-only uniworx:test:hlint + stack build --test --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) uniworx:test:hlint # HELP(test-backend): test backend --test-backend: @@ -207,7 +207,7 @@ db: compile-frontend --image-build --containerized---db; --db-backend: .stack --compile-backend SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true} ; \ AVSPASS=${AVSPASS:-nopasswordset} ; \ - stack exec uniworxdb -- $(UNIWORXDB_OPTS) + stack $(--DEVELOPMENT) exec uniworxdb -- $(UNIWORXDB_OPTS) .PHONY: .stack .stack: From 4bbab689cfc3b6081a1c2bc8169c01bcf1f825bb Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Wed, 30 Oct 2024 14:47:40 +0100 Subject: [PATCH 13/64] build(Makefile): switch-develop target for manual develop switching --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index 34bf1cb43..89cda4a4e 100644 --- a/Makefile +++ b/Makefile @@ -404,6 +404,14 @@ new-develop: $(NEW_DEVELOP) mkdir -p $(DEVELOP) $(MAKE) develop/.current +.PHONY: switch-develop +# HELP: switch current develop instance to DEVELOP=... +switch-develop: + if ! [ -e develop/$(DEVELOP) ]; then \ + echo "Specified develop $(DEVELOP) does not exist! Not switching." ; \ + exit 1 ; \ + fi ; \ + echo "$(DEVELOP)" > develop/.current .PHONY: --develop --develop: if ! [[ -e develop ]]; then \ From efda54ad8989aa260e428b1cf5606d3c090c02dd Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Fri, 1 Nov 2024 07:05:29 +0100 Subject: [PATCH 14/64] build(Makefile): Rewrite target structured names. --- Makefile | 211 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 136 insertions(+), 75 deletions(-) diff --git a/Makefile b/Makefile index 89cda4a4e..404f949e7 100644 --- a/Makefile +++ b/Makefile @@ -7,17 +7,19 @@ export CONTAINER_COMMAND ?= podman export CONTAINER_BGRUN ?= $(CONTAINER_COMMAND) run -dit --network=host --replace export CONTAINER_FGRUN ?= $(CONTAINER_COMMAND) run -it --network=host --replace -export CONTAINER_NAME ?= $(FRADRIVE_SERVICE) +#export CONTAINER_NAME ?= $(FRADRIVE_SERVICE) export ENTRYPOINT export IN_CONTAINER ?= false export IN_CI ?= false +export SINGLETON export WATCH export UNIWORXDB_OPTS ?= -cf export CONTAINER_ID export FRADRIVE_SERVICE +export SERVICE_VARIANT export DEV_PORT_HTTP export DEV_PORT_HTTPS @@ -89,12 +91,65 @@ release: .PHONY: %-shell # HELP: launch shell (bash) inside a currently running container %-shell: --%-shell; ---%-shell: +--shell: $(CONTAINER_COMMAND) exec -it $(EXEC_OPTS) fradrive.$(CURR_DEV).$* $(if $(ENTRYPOINT),$(ENTRYPOINT),/bin/bash) +compile-%: --%-compile; + ##### GENERAL TARGETS ##### ########################### +############################################## +##### SERVICE DEPENDENCIES AND VARIABLES ##### + +--frontend-%: FRADRIVE_SERVICE=frontend +--frontend-%: SERVICE_VARIANT=frontend +--frontend-%: SINGLETON = false +--frontend-%: WATCH = false +--frontend-%: +# TODO --containerized-% is not yet working; comment this in again as soon as it does +# $(MAKE) -- --containerized-static --containerized-well-known + $(MAKE) -- --$* + +--backend-%: FRADRIVE_SERVICE=backend +--backend-%: SERVICE_VARIANT=backend +--backend-%: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443" +--backend-%: SINGLETON = false +--backend-%: + $(MAKE) -- --frontend-compile + $(MAKE) -- --$* + +--hoogle-%: FRADRIVE_SERVICE=backend +--hoogle-%: SERVICE_VARIANT=hoogle +--hoogle-%: BASE_PORTS = "HOOGLE_PORT=8081" +--hoogle-%: SINGLETON = true +--hoogle-%: + $(MAKE) -- --$* + +# --uniworxdb was --db in old versions +--uniworxdb-%: FRADRIVE_SERVICE=backend +--uniworxdb-%: SERVICE_VARIANT=uniworxdb +--uniworxdb-%: + $(MAKE) -- --$* + +--minio-%: FRADRIVE_SERVICE=minio +--minio-%: SERVICE_VARIANT=minio +--minio-%: + $(MAKE) -- --$* + +--memcached-%: FRADRIVE_SERVICE=memcached +--memcached-%: SERVICE_VARIANT=memcached +--memcached-%: + $(MAKE) -- --$* + +# --postgres was --database in old versions +--postgres-%: BASE_PORTS = "PGPORT=5432" +--postgres-%: SINGLETON = true +--postgres-%: + $(MAKE) -- --$* + +##### SERVICE DEPENDENCIES AND VARIABLES ##### +############################################## ############################################ ##### UNIFIED FRONTEND/BACKEND TARGETS ##### @@ -109,7 +164,7 @@ compile: $(MAKE) compile-frontend $(MAKE) compile-backend -compile-%: --develop +--compile-%: --checkvarsset --develop --% .PHONY: lint # HELP: lint frontend and backend @@ -122,31 +177,53 @@ test: start-database test-frontend test-backend i18n-check; ##### UNIFIED FRONTEND/BACKEND TARGETS ##### ############################################ +--checkvarsset: + @: +ifndef ($(FRADRIVE_SERVICE)) + >&2 echo "Variable FRADRIVE_SERVICE is not set; this hints to an internal error in the Makefile unless you have manually requested a '--'-target" + exit 1 +endif +ifndef ($(SERVICE_VARIANT)) + >&2 echo "Variable SERVICE_VARIANT is not set; this hints to an internal error in the Makefile unless you have manually requested a '--'-target" + exit 1 +endif +ifndef ($(DEVELOP)) + >&2 echo "Variable DEVELOP is not set; this hints to an internal error in the Makefile unless you have manually requested a '--'-target" + exit 1 +endif +ifndef ($(CONTAINER_FILE)) + >&2 echo "Variable CONTAINER_FILE is not set; this hints to an internal error in the Makefile unless you have manually requested a '--'-target" + exit 1 +endif + ############################ ##### FRONTEND TARGETS ##### -.PHONY: %-frontend -%-frontend: FRADRIVE_SERVICE=frontend -%-frontend: --image-build --containerized---%-frontend; +#--frontend-%: --image-build --containerized---%-frontend; ---%-frontend: --containerized-static-frontend --containerized-well-known-frontend +#--%-frontend: --containerized-static-frontend --containerized-well-known-frontend ---start-frontend: static +#--frontend-start: static +# npm run start + +ifeq ($(SERVICE_VARIANT), frontend) +--start: --checkvarsset npm run start - + # HELP(compile-frontend): compile frontend ---compile-frontend: static well-known; +--compile: static well-known --checkvarsset; # HELP(lint-frontend): lint frontend ---lint-frontend: eslint.config.js +--lint: eslint.config.js --checkvarsset npx -- eslint frontend/src $(FIX) @echo Hooray! There are no hints. # HELP(test-frontend): test frontend ---test-frontend: karma.conf.cjs +--test: karma.conf.cjs --checkvarsset @echo Karma frontend tests are currently broken after npm update and have therefor been temporarily disabled. # npx -- karma start --conf karma.conf.cjs $(WATCH) +endif node_modules: package.json package-lock.json npm ci --cache .npm --prefer-offline @@ -157,7 +234,7 @@ package-lock.json: package.json assets: assets/favicons assets/icons; assets/favicons: ./utils/faviconize.pl assets/favicon.svg long assets/favicons -assets/icons: node_modules utils/rename-fa.json +assets/icons: node_modules assets/icons-src/fontawesome.json ./utils/renamer.pl node_modules/@fortawesome/fontawesome-free/svgs/solid assets/icons-src/fontawesome.json assets/icons/fradrive ./utils/renamer.pl node_modules/@fortawesome/fontawesome-free/svgs/regular assets/icons-src/fontawesome.json assets/icons/fradrive cp assets/icons-src/*.svg assets/icons/fradrive @@ -172,42 +249,43 @@ well-known: static; ########################### ##### BACKEND TARGETS ##### -.PHONY: %-backend -%-backend: FRADRIVE_SERVICE=backend -%-backend: --image-build --containerized---%-backend; ---start-backend: +ifeq ($(SERVICE_VARIANT), backend) +--start: --checkvarsset DEV_PORT_HTTP=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTP=' | sed 's/DEV_PORT_HTTP=//'`; \ DEV_PORT_HTTPS=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTPS=' | sed 's/DEV_PORT_HTTPS=//'`; \ stack --work-dir=.stack-work-run $(--DEVELOPMENT) exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" ---start-hoogle-backend: +endif +ifeq ($(SERVICE_VARIANT), hoogle) +--start: HOOGLE_PORT=`cat $(CONTAINER_FILE) | grep 'HOOGLE_PORT=' | sed 's/HOOGLE_PORT=//'` ; \ echo "${HOOGLE_PORT}" ; \ stack --work-dir=.stack-work-doc hoogle -- server --local --port $${HOOGLE_PORT} +endif # HELP(compile-backend): compile backend ---compile-backend: +ifeq ($(SERVICE_VARIANT), backend) +--compile: --checkvarsset stack build --fast --profile --library-profiling --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin # HELP(lint-backend): lint backend ---lint-backend: +--lint: --checkvarsset stack build --test --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) uniworx:test:hlint # HELP(test-backend): test backend ---test-backend: +--test: --checkvarsset stack build --test --coverage --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) +endif -# TODO: better name -.PHONY: db -# HELP: clear and fill database. requires running postgres -db: FRADRIVE_SERVICE=backend -db: CONTAINER_NAME=uniworxdb -db: compile-frontend --image-build --containerized---db; +# HELP(uniworxdb): clear and fill database. requires running postgres +uniworxdb: --frontend-compile --backend-imagebuild-containerized-uniworxdb-start; # TODO (db-m-$MIGRATION-backend): apply migration (see src/Model/Migration/Definition.hs for list of available migrations) ---db-backend: .stack --compile-backend +ifeq ($(SERVICE_VARIANT), uniworxdb) +--start: .stack SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true} ; \ AVSPASS=${AVSPASS:-nopasswordset} ; \ stack $(--DEVELOPMENT) exec uniworxdb -- $(UNIWORXDB_OPTS) +endif .PHONY: .stack .stack: @@ -226,13 +304,13 @@ db: compile-frontend --image-build --containerized---db; ############################ ##### DATABASE TARGETS ##### ---containerized---db: CONTAINER_NAME=uniworxdb ---containerized---db: --containerized-db-backend; - ---containerized-%-database: FRADRIVE_SERVICE=database ---containerized-%-database: CONTAINER_NAME=database ---containerized-%-database: docker/database/initdb.sh docker/database/pg_hba.conf docker/database/postgresql.conf docker/database/schema.sql --image-build - $(MAKE) -- --image-run-$*-database +ifeq ($(SERVICE_VARIANT), postgres) +--containerized-%: docker/database/initdb.sh docker/database/pg_hba.conf docker/database/postgresql.conf docker/database/schema.sql --image-build + $(MAKE) -- --imagerun-$* +else +--containerized-%: + $(MAKE) -- --imagerun-$* +endif .PHONY: psql # HELP: enter psql (postgresql) cli inside a currently running database container @@ -247,38 +325,42 @@ psql: --database-shell; ############################# ##### CONTAINER TARGETS ##### ---containerized-%-frontend: FRADRIVE_SERVICE=frontend ---containerized-%-frontend: CONTAINER_NAME=frontend ---containerized-%-frontend: --image-build - $(MAKE) -- --image-run-$*-frontend - ---containerized-%-backend: FRADRIVE_SERVICE=backend ---containerized-%-backend: CONTAINER_NAME?=backend ---containerized-%-backend: --image-build - $(MAKE) -- --image-run-$*-backend ---containerized-%-hoogle: FRADRIVE_SERVICE=backend ---containerized-%-hoogle: CONTAINER_NAME=hoogle ---containerized-%-hoogle: --image-build - $(MAKE) -- --image-run-$*-hoogle-backend +#--containerized-%-frontend: FRADRIVE_SERVICE=frontend +#--containerized-%-frontend: SERVICE_VARIANT=frontend +#--containerized-%-frontend: --image-build +# $(MAKE) -- --image-run-$*-frontend +# +#--containerized-%-backend: FRADRIVE_SERVICE=backend +#--containerized-%-backend: SERVICE_VARIANT?=backend +#--containerized-%-backend: --image-build +# $(MAKE) -- --image-run-$*-backend +#--containerized-%-hoogle: FRADRIVE_SERVICE=backend +#--containerized-%-hoogle: SERVICE_VARIANT=hoogle +#--containerized-%-hoogle: --image-build +# $(MAKE) -- --image-run-$*-hoogle-backend # --containerized-%-minio: FRADRIVE_SERVICE=minio # --containerized-%-minio: --image-build # $(MAKE) -- --image-run-$*-minio ---containerized---start-minio: +ifeq ($(SERVICE_VARIANT), minio) +--containerized-start: UPLOAD_S3_PORT=`cat $(CONTAINER_FILE) | grep 'UPLOAD_S3_PORT=' | sed 's/UPLOAD_S3_PORT=//'`; \ MINIO_DIR=`mktemp` ; \ ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" "" "rm -rf $${MINIO_DIR}" & \ CONTAINER_ID=`$(CONTAINER_BGRUN) --name fradrive.$(CURR_DEV).minio $(MINIO_IMAGE) -- server $${MINIO_DIR} --address=:$${UPLOAD_S3_PORT}` ; \ printf "CONTAINER_ID=$${CONTAINER_ID}\nUPLOAD_S3_PORT=$${UPLOAD_S3_PORT}\nMINIO_DIR=$${MINIO_DIR}" >> $(CONTAINER_FILE) +endif # --containerized-%-memcached: FRADRIVE_SERVICE=memcached # --containerized-%-memcached: --image-build # $(MAKE) -- --image-run-$*-memcached ---containerized---start-memcached: +ifeq ($(SERVICE_VARIANT), memcached) +--containerized-start: MEMCACHED_PORT=`cat $(CONTAINER_FILE) | grep 'MEMCACHED_PORT=' | sed 's/MEMCACHED_PORT=//'`; \ ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ CONTAINER_ID=`$(CONTAINER_BGRUN) --name fradrive.$(CURR_DEV).memcached $(MEMCACHED_IMAGE) --port=$${MEMCACHED_PORT}` ; \ printf "CONTAINER_ID=$${CONTAINER_ID}\nMEMCACHED_PORT=$${MEMCACHED_PORT}" >> $(CONTAINER_FILE) +endif .PHONY: image-rebuild # HELP(image-rebuild-{backend,frontend,database,memcached,minio}): force-rebuild the stated docker image @@ -296,42 +378,21 @@ image-rebuild-%: $(CONTAINER_COMMAND) build $(NO_CACHE) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --build-arg MOUNT_DIR=$${MOUNT_DIR} --build-arg PROJECT_DIR=$${PROJECT_DIR} --tag fradrive/$(FRADRIVE_SERVICE) --file $(PWD)/.Dockerfile ; \ fi ---image-run-%: CONTAINER_NAME ?= $(FRADRIVE_SERVICE) ---image-run-%: docker/$(FRADRIVE_SERVICE)/Dockerfile +--imagerun-%: --checkvarsset docker/$(FRADRIVE_SERVICE)/Dockerfile MOUNT_DIR=/mnt/fradrive; \ if [ "$(IN_CONTAINER)" == "true" ] ; then \ $(MAKE) -- $* ; \ else \ if [ -z "$(CONTAINER_FILE)" ] ; then \ - $(CONTAINER_FGRUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(CONTAINER_NAME) localhost/fradrive/$(FRADRIVE_SERVICE) ; \ + $(CONTAINER_FGRUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(SERVICE_VARIANT) localhost/fradrive/$(FRADRIVE_SERVICE) ; \ else \ ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ - CONTAINER_ID=`$(CONTAINER_BGRUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(CONTAINER_NAME) localhost/fradrive/$(FRADRIVE_SERVICE)` ; \ + CONTAINER_ID=`$(CONTAINER_BGRUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(SERVICE_VARIANT) localhost/fradrive/$(FRADRIVE_SERVICE)` ; \ echo "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)"; \ fi \ fi -# TODO: move starts below to respective entries, or leave together? - -start-database: BASE_PORTS = "PGPORT=5432" -start-database: SINGLETON = true - -start-memcached: BASE_PORTS = "MEMCACHED_PORT=11211" -start-memcached: SINGLETON = true - -start-minio: BASE_PORTS = "UPLOAD_S3_PORT=9000" -start-minio: SINGLETON = true - -start-backend: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443" -start-backend: SINGLETON = false -start-backend: compile-frontend - -start-hoogle: BASE_PORTS = "HOOGLE_PORT=8081" -start-hoogle: SINGLETON = true - -start-frontend: SINGLETON = false -start-frontend: WATCH = false # HELP(start-database): start postgres server # HELP(start-memcached): start memcached server @@ -441,4 +502,4 @@ i18n-check: --image-run---i18n-check flock -en $@ true .PHONY: --% -.SUFFIXES: # Delete all default suffixes \ No newline at end of file +.SUFFIXES: # Delete all default suffixes From 03dfe31b50172de51f3e75ae86eca4364e4700ea Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Sun, 3 Nov 2024 02:10:00 +0100 Subject: [PATCH 15/64] build(Makefile): Rewrite target structured names, continued. --- Makefile | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 404f949e7..63c8610bf 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ release: --shell: $(CONTAINER_COMMAND) exec -it $(EXEC_OPTS) fradrive.$(CURR_DEV).$* $(if $(ENTRYPOINT),$(ENTRYPOINT),/bin/bash) -compile-%: --%-compile; +compile-%: --%-containerized-compile; ##### GENERAL TARGETS ##### ########################### @@ -116,7 +116,7 @@ compile-%: --%-compile; --backend-%: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443" --backend-%: SINGLETON = false --backend-%: - $(MAKE) -- --frontend-compile + $(MAKE) -- --frontend-containerized-compile $(MAKE) -- --$* --hoogle-%: FRADRIVE_SERVICE=backend @@ -178,23 +178,24 @@ test: start-database test-frontend test-backend i18n-check; ############################################ --checkvarsset: + echo "FS:$(FRADRIVE_SERVICE) SV:$(SERVICE_VARIANT) DE:$(DEVELOP) CF:$(CONTAINER_FILE)" @: -ifndef ($(FRADRIVE_SERVICE)) +ifeq "$(FRADRIVE_SERVICE)" "" >&2 echo "Variable FRADRIVE_SERVICE is not set; this hints to an internal error in the Makefile unless you have manually requested a '--'-target" exit 1 endif -ifndef ($(SERVICE_VARIANT)) +ifeq "$(SERVICE_VARIANT)" "" >&2 echo "Variable SERVICE_VARIANT is not set; this hints to an internal error in the Makefile unless you have manually requested a '--'-target" exit 1 endif -ifndef ($(DEVELOP)) - >&2 echo "Variable DEVELOP is not set; this hints to an internal error in the Makefile unless you have manually requested a '--'-target" - exit 1 -endif -ifndef ($(CONTAINER_FILE)) - >&2 echo "Variable CONTAINER_FILE is not set; this hints to an internal error in the Makefile unless you have manually requested a '--'-target" - exit 1 -endif +#ifeq "$(DEVELOP)" "" +# >&2 echo "Variable DEVELOP is not set; this hints to an internal error in the Makefile unless you have manually requested a '--'-target" +# exit 1 +#endif +#ifeq "$(CONTAINER_FILE)" "" +# >&2 echo "Variable CONTAINER_FILE is not set; this hints to an internal error in the Makefile unless you have manually requested a '--'-target" +# exit 1 +#endif ############################ @@ -237,7 +238,7 @@ assets/favicons: assets/icons: node_modules assets/icons-src/fontawesome.json ./utils/renamer.pl node_modules/@fortawesome/fontawesome-free/svgs/solid assets/icons-src/fontawesome.json assets/icons/fradrive ./utils/renamer.pl node_modules/@fortawesome/fontawesome-free/svgs/regular assets/icons-src/fontawesome.json assets/icons/fradrive - cp assets/icons-src/*.svg assets/icons/fradrive + -cp assets/icons-src/*.svg assets/icons/fradrive static: node_modules assets esbuild.config.mjs npm run build well-known: static; @@ -305,10 +306,10 @@ endif ##### DATABASE TARGETS ##### ifeq ($(SERVICE_VARIANT), postgres) ---containerized-%: docker/database/initdb.sh docker/database/pg_hba.conf docker/database/postgresql.conf docker/database/schema.sql --image-build +--containerized-%: --checkvarsset docker/database/initdb.sh docker/database/pg_hba.conf docker/database/postgresql.conf docker/database/schema.sql --image-build $(MAKE) -- --imagerun-$* else ---containerized-%: +--containerized-%: --checkvarsset $(MAKE) -- --imagerun-$* endif @@ -378,16 +379,17 @@ image-rebuild-%: $(CONTAINER_COMMAND) build $(NO_CACHE) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --build-arg MOUNT_DIR=$${MOUNT_DIR} --build-arg PROJECT_DIR=$${PROJECT_DIR} --tag fradrive/$(FRADRIVE_SERVICE) --file $(PWD)/.Dockerfile ; \ fi +# runs --% in the container. Targets meant to be called manually cannot be run in the container via this target. If that is ever needed split this target into two targets, one for manually one for automatic (one may use the other) so that not at every usage a '---' has to be after containerized --imagerun-%: --checkvarsset docker/$(FRADRIVE_SERVICE)/Dockerfile MOUNT_DIR=/mnt/fradrive; \ if [ "$(IN_CONTAINER)" == "true" ] ; then \ $(MAKE) -- $* ; \ else \ if [ -z "$(CONTAINER_FILE)" ] ; then \ - $(CONTAINER_FGRUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(SERVICE_VARIANT) localhost/fradrive/$(FRADRIVE_SERVICE) ; \ + $(CONTAINER_FGRUN) -v $(PWD):$${MOUNT_DIR} --env FRADRIVE_SERVICE=$(FRADRIVE_SERVICE) --env SERVICE_VARIANT=$(SERVICE_VARIANT) --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=--$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(SERVICE_VARIANT) localhost/fradrive/$(FRADRIVE_SERVICE) ; \ else \ ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ - CONTAINER_ID=`$(CONTAINER_BGRUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(SERVICE_VARIANT) localhost/fradrive/$(FRADRIVE_SERVICE)` ; \ + CONTAINER_ID=`$(CONTAINER_BGRUN) -v $(PWD):$${MOUNT_DIR} --env FRADRIVE_SERVICE=$(FRADRIVE_SERVICE) --env SERVICE_VARIANT=$(SERVICE_VARIANT) --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=--$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(SERVICE_VARIANT) localhost/fradrive/$(FRADRIVE_SERVICE)` ; \ echo "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)"; \ fi \ fi From 47cd2f3b1fc541e51bde6ad713dd308f902a0b35 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Sun, 3 Nov 2024 02:12:51 +0100 Subject: [PATCH 16/64] build: remove unused debris --- config/keter.yml | 1 - config/keter_testworx.yml | 77 -------------------------------------- config/keter_uni2work.yml | 79 --------------------------------------- gup/Gupfile | 4 -- gup/Gupfile.license | 3 -- gup/cabal2nix.gup | 8 ---- gup/cabal2nix.gup.license | 3 -- gup/hpack.gup | 6 --- gup/hpack.gup.license | 3 -- 9 files changed, 184 deletions(-) delete mode 120000 config/keter.yml delete mode 100644 config/keter_testworx.yml delete mode 100644 config/keter_uni2work.yml delete mode 100644 gup/Gupfile delete mode 100644 gup/Gupfile.license delete mode 100755 gup/cabal2nix.gup delete mode 100644 gup/cabal2nix.gup.license delete mode 100755 gup/hpack.gup delete mode 100644 gup/hpack.gup.license diff --git a/config/keter.yml b/config/keter.yml deleted file mode 120000 index 3b8c9db84..000000000 --- a/config/keter.yml +++ /dev/null @@ -1 +0,0 @@ -keter_testworx.yml \ No newline at end of file diff --git a/config/keter_testworx.yml b/config/keter_testworx.yml deleted file mode 100644 index 91d90e156..000000000 --- a/config/keter_testworx.yml +++ /dev/null @@ -1,77 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Gregor Kleen ,Steffen Jost -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -root: .. - -stanzas: - - type: webapp - - # Name of your executable. You are unlikely to need to change this. - # Note that all file paths are relative to the keter.yml file. - # - # The path given is for Stack projects. If you're still using cabal, change - # to - # exec: ../dist/build/uniworx/uniworx - exec: ../dist/bin/uniworx - - # Command line options passed to your application. - args: [] - - hosts: - - testworx.tcs.ifi.lmu.de - - ssl: true - - forward-env: - - LDAPHOST - - LDAPTLS - - LDAPPORT - - LDAPUSER - - LDAPPASS - - LDAPBASE - - LDAPSCOPE - - LDAPSEARCHTIME - - LDAPSTRIPES - - LDAPTIMEOUT - - LDAPLIMIT - - DUMMY_LOGIN - - DETAILED_LOGGING - - LOG_ALL - - LOGLEVEL - - ALLOW_DEPRECATED - - PWFILE - - CRYPTOID_KEYFILE - - IP_FROM_HEADER - - MAILFROM_NAME - - MAILFROM_EMAIL - - MAILOBJECT_DOMAIN - - SMTPHOST - - SMTPPORT - - SMTPSSL - - SMTPUSER - - SMTPPASS - - SMTPTIMEOUT - - SMTPLIMIT - - MAILSUPPORT - - MAILSUPPORT_NAME - - INSTANCE_ID - - MEMCACHEDHOST - - MEMCACHEDPORT - - MEMCACHEDLIMIT - - MEMCACHEDTIMEOUT - - MEMCACHEDROOT - - MEMCACHEDEXPIRATION - -# Use the following to automatically copy your bundle upon creation via `yesod -# keter`. Uses `scp` internally, so you can set it to a remote destination -# copy-to: user@host:/opt/keter/incoming/ -copy-to: keter@testworx.tcs.ifi.lmu.de:/opt/keter/incoming/ -copy-to-args: - - "-P 30363" - -# If you would like to have Keter automatically create a PostgreSQL database -# and set appropriate environment variables for it to be discovered, uncomment -# the following line. -plugins: - postgres: true diff --git a/config/keter_uni2work.yml b/config/keter_uni2work.yml deleted file mode 100644 index f21122ec3..000000000 --- a/config/keter_uni2work.yml +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Gregor Kleen ,Steffen Jost -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -root: .. - -stanzas: - - type: webapp - - # Name of your executable. You are unlikely to need to change this. - # Note that all file paths are relative to the keter.yml file. - # - # The path given is for Stack projects. If you're still using cabal, change - # to - # exec: ../dist/build/uniworx/uniworx - exec: ../dist/bin/uniworx - - # Command line options passed to your application. - args: [] - - hosts: - - uni2work.ifi.lmu.de - - ssl: true - - forward-env: - - LDAPHOST - - LDAPTLS - - LDAPPORT - - LDAPUSER - - LDAPPASS - - LDAPBASE - - LDAPSCOPE - - LDAPSEARCHTIME - - LDAPSTRIPES - - LDAPTIMEOUT - - LDAPLIMIT - - DETAILED_LOGGING - - LOG_ALL - - LOGLEVEL - - ALLOW_DEPRECATED - - PWFILE - - CRYPTOID_KEYFILE - - IP_FROM_HEADER - - MAILFROM_NAME - - MAILFROM_EMAIL - - MAILOBJECT_DOMAIN - - SMTPHOST - - SMTPPORT - - SMTPSSL - - SMTPUSER - - SMTPPASS - - SMTPTIMEOUT - - SMTPLIMIT - - MAILSUPPORT - - MAILSUPPORT_NAME - - INSTANCE_ID - - MEMCACHEDHOST - - MEMCACHEDPORT - - MEMCACHEDLIMIT - - MEMCACHEDTIMEOUT - - MEMCACHEDROOT - - MEMCACHEDEXPIRATION - - -# Use the following to automatically copy your bundle upon creation via `yesod -# keter`. Uses `scp` internally, so you can set it to a remote destination -# copy-to: user@host:/opt/keter/incoming/ -copy-to: root@uni2work.ifi.lmu.de:/opt/keter/incoming/ -copy-to-args: [] - - -# If you would like to have Keter automatically create a PostgreSQL database -# and set appropriate environment variables for it to be discovered, uncomment -# the following line. -plugins: - postgres: - - server: uniworxdb - port: 5432 diff --git a/gup/Gupfile b/gup/Gupfile deleted file mode 100644 index d2db73f2f..000000000 --- a/gup/Gupfile +++ /dev/null @@ -1,4 +0,0 @@ -cabal2nix.gup: - *.nix -hpack.gup: - *.cabal \ No newline at end of file diff --git a/gup/Gupfile.license b/gup/Gupfile.license deleted file mode 100644 index 0544bf968..000000000 --- a/gup/Gupfile.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2022 Gregor Kleen - -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/gup/cabal2nix.gup b/gup/cabal2nix.gup deleted file mode 100755 index 9854cec0e..000000000 --- a/gup/cabal2nix.gup +++ /dev/null @@ -1,8 +0,0 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i zsh -p zsh haskellPackages.cabal2nix - -gup -u ${2:r}.cabal - -cd ${2:h} - -cabal2nix . > $1 diff --git a/gup/cabal2nix.gup.license b/gup/cabal2nix.gup.license deleted file mode 100644 index 0544bf968..000000000 --- a/gup/cabal2nix.gup.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2022 Gregor Kleen - -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/gup/hpack.gup b/gup/hpack.gup deleted file mode 100755 index 9e1052b02..000000000 --- a/gup/hpack.gup +++ /dev/null @@ -1,6 +0,0 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i sh -p haskellPackages.hpack - -gup -u package.yaml - -hpack - >$1 \ No newline at end of file diff --git a/gup/hpack.gup.license b/gup/hpack.gup.license deleted file mode 100644 index 0544bf968..000000000 --- a/gup/hpack.gup.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2022 Gregor Kleen - -SPDX-License-Identifier: AGPL-3.0-or-later From e38adb42b3a96663b4eb91df6c33c2a04d76445f Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Sun, 3 Nov 2024 16:33:18 +0100 Subject: [PATCH 17/64] build(Makefile): Rewrite target structured names, continued. --- Makefile | 96 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 51 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index 63c8610bf..5a1da6d7e 100644 --- a/Makefile +++ b/Makefile @@ -90,11 +90,11 @@ release: .PHONY: %-shell # HELP: launch shell (bash) inside a currently running container -%-shell: --%-shell; +%-shell: --%_shell; --shell: $(CONTAINER_COMMAND) exec -it $(EXEC_OPTS) fradrive.$(CURR_DEV).$* $(if $(ENTRYPOINT),$(ENTRYPOINT),/bin/bash) -compile-%: --%-containerized-compile; +compile-%: --%_containerized_--compile; ##### GENERAL TARGETS ##### ########################### @@ -102,50 +102,50 @@ compile-%: --%-containerized-compile; ############################################## ##### SERVICE DEPENDENCIES AND VARIABLES ##### ---frontend-%: FRADRIVE_SERVICE=frontend ---frontend-%: SERVICE_VARIANT=frontend ---frontend-%: SINGLETON = false ---frontend-%: WATCH = false ---frontend-%: +--frontend_%: FRADRIVE_SERVICE=frontend +--frontend_%: SERVICE_VARIANT=frontend +--frontend_%: SINGLETON = false +--frontend_%: WATCH = false +--frontend_%: # TODO --containerized-% is not yet working; comment this in again as soon as it does # $(MAKE) -- --containerized-static --containerized-well-known $(MAKE) -- --$* ---backend-%: FRADRIVE_SERVICE=backend ---backend-%: SERVICE_VARIANT=backend ---backend-%: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443" ---backend-%: SINGLETON = false ---backend-%: - $(MAKE) -- --frontend-containerized-compile +--backend_%: FRADRIVE_SERVICE=backend +--backend_%: SERVICE_VARIANT=backend +--backend_%: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443" +--backend_%: SINGLETON = false +--backend_%: + $(MAKE) -- --frontend_containerized_--compile $(MAKE) -- --$* ---hoogle-%: FRADRIVE_SERVICE=backend ---hoogle-%: SERVICE_VARIANT=hoogle ---hoogle-%: BASE_PORTS = "HOOGLE_PORT=8081" ---hoogle-%: SINGLETON = true ---hoogle-%: +--hoogle_%: FRADRIVE_SERVICE=backend +--hoogle_%: SERVICE_VARIANT=hoogle +--hoogle_%: BASE_PORTS = "HOOGLE_PORT=8081" +--hoogle_%: SINGLETON = true +--hoogle_%: $(MAKE) -- --$* # --uniworxdb was --db in old versions ---uniworxdb-%: FRADRIVE_SERVICE=backend ---uniworxdb-%: SERVICE_VARIANT=uniworxdb ---uniworxdb-%: +--uniworxdb_%: FRADRIVE_SERVICE=backend +--uniworxdb_%: SERVICE_VARIANT=uniworxdb +--uniworxdb_%: $(MAKE) -- --$* ---minio-%: FRADRIVE_SERVICE=minio ---minio-%: SERVICE_VARIANT=minio ---minio-%: +--minio_%: FRADRIVE_SERVICE=minio +--minio_%: SERVICE_VARIANT=minio +--minio_%: $(MAKE) -- --$* ---memcached-%: FRADRIVE_SERVICE=memcached ---memcached-%: SERVICE_VARIANT=memcached ---memcached-%: +--memcached_%: FRADRIVE_SERVICE=memcached +--memcached_%: SERVICE_VARIANT=memcached +--memcached_%: $(MAKE) -- --$* # --postgres was --database in old versions ---postgres-%: BASE_PORTS = "PGPORT=5432" ---postgres-%: SINGLETON = true ---postgres-%: +--postgres_%: BASE_PORTS = "PGPORT=5432" +--postgres_%: SINGLETON = true +--postgres_%: $(MAKE) -- --$* ##### SERVICE DEPENDENCIES AND VARIABLES ##### @@ -164,7 +164,7 @@ compile: $(MAKE) compile-frontend $(MAKE) compile-backend ---compile-%: --checkvarsset --develop --% +--compile_%: --checkvarsset --develop --% .PHONY: lint # HELP: lint frontend and backend @@ -279,7 +279,7 @@ ifeq ($(SERVICE_VARIANT), backend) endif # HELP(uniworxdb): clear and fill database. requires running postgres -uniworxdb: --frontend-compile --backend-imagebuild-containerized-uniworxdb-start; +uniworxdb: --frontend_compile --backend_imagebuild_containerized_--uniworxdb_start; # TODO (db-m-$MIGRATION-backend): apply migration (see src/Model/Migration/Definition.hs for list of available migrations) ifeq ($(SERVICE_VARIANT), uniworxdb) --start: .stack @@ -293,7 +293,7 @@ endif if [ "$(IN_CONTAINER)" == "true" ] ; then \ $(MAKE) -- --.stack ; \ else \ - $(MAKE) -- --image-run---.stack ; \ + $(MAKE) -- --imagerun_--.stack ; \ fi --.stack: stack.yaml stack.yaml.lock package.yaml stack build --fast --only-dependencies @@ -306,11 +306,11 @@ endif ##### DATABASE TARGETS ##### ifeq ($(SERVICE_VARIANT), postgres) ---containerized-%: --checkvarsset docker/database/initdb.sh docker/database/pg_hba.conf docker/database/postgresql.conf docker/database/schema.sql --image-build - $(MAKE) -- --imagerun-$* +--containerized_%: --checkvarsset docker/database/initdb.sh docker/database/pg_hba.conf docker/database/postgresql.conf docker/database/schema.sql --image-build + $(MAKE) -- --imagerun_$* else ---containerized-%: --checkvarsset - $(MAKE) -- --imagerun-$* +--containerized_%: --checkvarsset + $(MAKE) -- --imagerun_$* endif .PHONY: psql @@ -344,7 +344,7 @@ psql: --database-shell; # --containerized-%-minio: --image-build # $(MAKE) -- --image-run-$*-minio ifeq ($(SERVICE_VARIANT), minio) ---containerized-start: +--containerized_start: UPLOAD_S3_PORT=`cat $(CONTAINER_FILE) | grep 'UPLOAD_S3_PORT=' | sed 's/UPLOAD_S3_PORT=//'`; \ MINIO_DIR=`mktemp` ; \ ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" "" "rm -rf $${MINIO_DIR}" & \ @@ -356,7 +356,7 @@ endif # --containerized-%-memcached: --image-build # $(MAKE) -- --image-run-$*-memcached ifeq ($(SERVICE_VARIANT), memcached) ---containerized-start: +--containerized_start: MEMCACHED_PORT=`cat $(CONTAINER_FILE) | grep 'MEMCACHED_PORT=' | sed 's/MEMCACHED_PORT=//'`; \ ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ CONTAINER_ID=`$(CONTAINER_BGRUN) --name fradrive.$(CURR_DEV).memcached $(MEMCACHED_IMAGE) --port=$${MEMCACHED_PORT}` ; \ @@ -365,7 +365,7 @@ endif .PHONY: image-rebuild # HELP(image-rebuild-{backend,frontend,database,memcached,minio}): force-rebuild the stated docker image -image-rebuild-%: +image-rebuild_%: $(MAKE) -- --image-build FRADRIVE_SERVICE=$* NO_CACHE=--no-cache --image-build: rm -f .Dockerfile @@ -380,16 +380,16 @@ image-rebuild-%: fi # runs --% in the container. Targets meant to be called manually cannot be run in the container via this target. If that is ever needed split this target into two targets, one for manually one for automatic (one may use the other) so that not at every usage a '---' has to be after containerized ---imagerun-%: --checkvarsset docker/$(FRADRIVE_SERVICE)/Dockerfile +--imagerun_%: --checkvarsset docker/$(FRADRIVE_SERVICE)/Dockerfile MOUNT_DIR=/mnt/fradrive; \ if [ "$(IN_CONTAINER)" == "true" ] ; then \ $(MAKE) -- $* ; \ else \ if [ -z "$(CONTAINER_FILE)" ] ; then \ - $(CONTAINER_FGRUN) -v $(PWD):$${MOUNT_DIR} --env FRADRIVE_SERVICE=$(FRADRIVE_SERVICE) --env SERVICE_VARIANT=$(SERVICE_VARIANT) --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=--$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(SERVICE_VARIANT) localhost/fradrive/$(FRADRIVE_SERVICE) ; \ + $(CONTAINER_FGRUN) -v $(PWD):$${MOUNT_DIR} --env FRADRIVE_SERVICE=$(FRADRIVE_SERVICE) --env SERVICE_VARIANT=$(SERVICE_VARIANT) --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(SERVICE_VARIANT) localhost/fradrive/$(FRADRIVE_SERVICE) ; \ else \ ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ - CONTAINER_ID=`$(CONTAINER_BGRUN) -v $(PWD):$${MOUNT_DIR} --env FRADRIVE_SERVICE=$(FRADRIVE_SERVICE) --env SERVICE_VARIANT=$(SERVICE_VARIANT) --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=--$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(SERVICE_VARIANT) localhost/fradrive/$(FRADRIVE_SERVICE)` ; \ + CONTAINER_ID=`$(CONTAINER_BGRUN) -v $(PWD):$${MOUNT_DIR} --env FRADRIVE_SERVICE=$(FRADRIVE_SERVICE) --env SERVICE_VARIANT=$(SERVICE_VARIANT) --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(SERVICE_VARIANT) localhost/fradrive/$(FRADRIVE_SERVICE)` ; \ echo "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)"; \ fi \ fi @@ -402,7 +402,13 @@ image-rebuild-%: # HELP(start-backend): serve yesod development site # HELP(start-frontend): start frontend watcher. Watches frontend source files for changes and recompiles on change. (TODO: watcher functionality currently broken!) # HELP(start-hoogle): serve local hoogle instance +# TODO reformulate target to use the '--%_'-mechanism +# --autostart: use the SERVICE_VARIANT for the decision which service to start +--autostart: --checkvarsset + $(MAKE) -- start-$(SERVICE_VARIANT) + start-%: FRADRIVE_SERVICE = % +start-%: SERVICE_VARIANT = % start-%: PORTS = $(foreach PORT,$(BASE_PORTS),$(shell utils/next_free_port.pl $(PORT))) start-%: --develop echo "$*" @@ -421,7 +427,7 @@ start-%: --develop CONTAINER_FILE=$${DEVDIR}-$$(($${I}+1)); \ fi ; \ echo "$(PORTS)" | sed 's/ /\n/g' > $${CONTAINER_FILE} ; \ - $(MAKE) -- --containerized---start-$* CONTAINER_FILE=$${CONTAINER_FILE} + $(MAKE) -- --containerized_--autostart CONTAINER_FILE=$${CONTAINER_FILE} .PHONY: stop # HELP: stop all currently running develop instances @@ -494,7 +500,7 @@ hoogle: start-hoogle; .PHONY: i18n-check # HELP: check internationalization -i18n-check: --image-run---i18n-check +i18n-check: --imagerun_--i18n-check --i18n-check: ./missing-translations.sh @echo No missing translations. From 6239978a6ac2cc1465d2786e8674c2ff94e90226 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 4 Nov 2024 14:46:40 +0100 Subject: [PATCH 18/64] build(Makefile): rework Makefile call structure --- Makefile | 546 ++++++------------ docker/{database => postgres}/Dockerfile | 0 docker/{database => postgres}/initdb.sh | 0 docker/{database => postgres}/pg_hba.conf | 0 docker/{database => postgres}/pgconfig.sh | 0 docker/{database => postgres}/postgresql.conf | 0 docker/{database => postgres}/schema.sql | 0 7 files changed, 173 insertions(+), 373 deletions(-) rename docker/{database => postgres}/Dockerfile (100%) rename docker/{database => postgres}/initdb.sh (100%) rename docker/{database => postgres}/pg_hba.conf (100%) rename docker/{database => postgres}/pgconfig.sh (100%) rename docker/{database => postgres}/postgresql.conf (100%) rename docker/{database => postgres}/schema.sql (100%) diff --git a/Makefile b/Makefile index 5a1da6d7e..52734e06c 100644 --- a/Makefile +++ b/Makefile @@ -1,57 +1,43 @@ -include .gnumake/*.Mak -SHELL=bash -# MAKE=make -d +export SHELL=bash export CONTAINER_COMMAND ?= podman export CONTAINER_BGRUN ?= $(CONTAINER_COMMAND) run -dit --network=host --replace export CONTAINER_FGRUN ?= $(CONTAINER_COMMAND) run -it --network=host --replace -#export CONTAINER_NAME ?= $(FRADRIVE_SERVICE) -export ENTRYPOINT - -export IN_CONTAINER ?= false -export IN_CI ?= false - -export SINGLETON -export WATCH -export UNIWORXDB_OPTS ?= -cf - -export CONTAINER_ID -export FRADRIVE_SERVICE -export SERVICE_VARIANT - -export DEV_PORT_HTTP -export DEV_PORT_HTTPS -export HOOGLE_PORT - export IMAGE_REGISTRY = docker.io export MEMCACHED_IMAGE = $(IMAGE_REGISTRY)/memcached:latest export MINIO_IMAGE = $(IMAGE_REGISTRY)/minio/minio:latest export MAILDEV_IMAGE = $(IMAGE_REGISTRY)/maildev/maildev:latest # TODO: needs different port than 1025 to avoid conflicts -export DEVELOP +export IN_CONTAINER ?= false +export IN_CI ?= false export CONTAINER_FILE -export CONTAINER_FILE_CONTENT +export CF_PREFIX +export DEVELOP +export MOUNT_DIR=/mnt/fradrive -export LOGSIZE ?= 1024 +export SERVICE +export SERVICE_VARIANT ?= $(SERVICE) +export SERVICE_PARALLEL ?= false +export JOB +export JOB_ID +export CONTAINER_CMD + +export BASE_PORTS +export UNIWORXDB_OPTS ?= -cf export PROD ?= false ifneq ($(PROD),true) export --DEVELOPMENT=--flag uniworx:dev endif -# HELP HEADER START -# To see the definition of all available targets, take a look into the Makefile. -# Targets starting with '--' are not meant to be directly called. -# If you want to do so anyway please use 'make -- --target' to avoid '--target' being treated as option to 'make'. -# -# -# Targets meant to be used by humans are: -# HELP HEADER END +export DATE := $(shell date +'%Y-%m-%dT%H-%M-%S') + +export CURR_DEV = $(shell cat develop/.current) +export SET_DEVELOP = $(eval DEVELOP=develop/$$(CURR_DEV)) +export NEW_DEVELOP = $(eval DEVELOP=develop/$$(DATE)) -########################### -##### GENERAL TARGETS ##### .PHONY: help # HELP: print out this help message @@ -65,7 +51,7 @@ help: .PHONY: clean # HELP: stop all running containers and remove all compilation results in the directory (but leave images including dependencies unharmed) clean: - $(MAKE) stop + rm -rf develop -rm -rf node_modules .npm .cache assets/icons assets/favicons static well-known config/manifest.json -rm -rf .stack-work .stack-work-run .stack-work-test .stack-work-doc .stack-work.lock -rm -rf bin .Dockerfile develop @@ -86,145 +72,151 @@ release: VERSION=`.gitlab-ci/version.pl` git tag $${VERSION} git commit -m "chore(release): $${VERSION}" - git push - -.PHONY: %-shell -# HELP: launch shell (bash) inside a currently running container -%-shell: --%_shell; ---shell: - $(CONTAINER_COMMAND) exec -it $(EXEC_OPTS) fradrive.$(CURR_DEV).$* $(if $(ENTRYPOINT),$(ENTRYPOINT),/bin/bash) - -compile-%: --%_containerized_--compile; - -##### GENERAL TARGETS ##### -########################### - -############################################## -##### SERVICE DEPENDENCIES AND VARIABLES ##### - ---frontend_%: FRADRIVE_SERVICE=frontend ---frontend_%: SERVICE_VARIANT=frontend ---frontend_%: SINGLETON = false ---frontend_%: WATCH = false ---frontend_%: -# TODO --containerized-% is not yet working; comment this in again as soon as it does -# $(MAKE) -- --containerized-static --containerized-well-known - $(MAKE) -- --$* - ---backend_%: FRADRIVE_SERVICE=backend ---backend_%: SERVICE_VARIANT=backend ---backend_%: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443" ---backend_%: SINGLETON = false ---backend_%: - $(MAKE) -- --frontend_containerized_--compile - $(MAKE) -- --$* - ---hoogle_%: FRADRIVE_SERVICE=backend ---hoogle_%: SERVICE_VARIANT=hoogle ---hoogle_%: BASE_PORTS = "HOOGLE_PORT=8081" ---hoogle_%: SINGLETON = true ---hoogle_%: - $(MAKE) -- --$* - -# --uniworxdb was --db in old versions ---uniworxdb_%: FRADRIVE_SERVICE=backend ---uniworxdb_%: SERVICE_VARIANT=uniworxdb ---uniworxdb_%: - $(MAKE) -- --$* - ---minio_%: FRADRIVE_SERVICE=minio ---minio_%: SERVICE_VARIANT=minio ---minio_%: - $(MAKE) -- --$* - ---memcached_%: FRADRIVE_SERVICE=memcached ---memcached_%: SERVICE_VARIANT=memcached ---memcached_%: - $(MAKE) -- --$* - -# --postgres was --database in old versions ---postgres_%: BASE_PORTS = "PGPORT=5432" ---postgres_%: SINGLETON = true ---postgres_%: - $(MAKE) -- --$* - -##### SERVICE DEPENDENCIES AND VARIABLES ##### -############################################## - -############################################ -##### UNIFIED FRONTEND/BACKEND TARGETS ##### - -.PHONY: start -# HELP: start database services, frontend and backend -start: start-database start-memcached start-minio start-frontend start-backend; +# git push .PHONY: compile -# HELP: compile frontend and backend compile: $(MAKE) compile-frontend $(MAKE) compile-backend ---compile_%: --checkvarsset --develop --% +.PHONY: start +start: start-postgres start-memcached start-minio start-frontend start-backend; -.PHONY: lint -# HELP: lint frontend and backend -lint: lint-frontend lint-backend; +.PHONY: %-backend +%-backend: SERVICE=backend +%-backend: SERVICE_VARIANT=backend +%-backend: SERVICE_PARALLEL=true +%-backend: CONTAINER_CMD=localhost/fradrive/backend +%-backend: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443" -.PHONY: test -# HELP: test frontend, backend, and check internationalization -test: start-database test-frontend test-backend i18n-check; +.PHONY: %-uniworxdb +%-uniworxdb: SERVICE=backend +%-uniworxdb: SERVICE_VARIANT=uniworxdb +%-uniworxdb: CONTAINER_CMD=localhost/fradrive/backend -##### UNIFIED FRONTEND/BACKEND TARGETS ##### -############################################ +.PHONY: %-hoogle +%-hoogle: SERVICE=backend +%-hoogle: SERVICE_VARIANT=hoogle +%-hoogle: BASE_PORTS = "HOOGLE_PORT=8081" +%-hoogle: CONTAINER_CMD=localhost/fradrive/backend +--start-hoogle: + HOOGLE_PORT=`cat $(CONTAINER_FILE) | grep 'HOOGLE_PORT=' | sed 's/HOOGLE_PORT=//'` ; \ + stack --work-dir=.stack-work-doc hoogle -- server --local --port $${HOOGLE_PORT} ---checkvarsset: - echo "FS:$(FRADRIVE_SERVICE) SV:$(SERVICE_VARIANT) DE:$(DEVELOP) CF:$(CONTAINER_FILE)" - @: -ifeq "$(FRADRIVE_SERVICE)" "" - >&2 echo "Variable FRADRIVE_SERVICE is not set; this hints to an internal error in the Makefile unless you have manually requested a '--'-target" - exit 1 +.PHONY: %-frontend +%-frontend: SERVICE=frontend +%-frontend: CONTAINER_CMD=localhost/fradrive/frontend + +.PHONY: %-postgres +%-postgres: SERVICE=postgres +%-postgres: BASE_PORTS = "PGPORT=5432" +%-postgres: CONTAINER_CMD=localhost/fradrive/postgres + +.PHONY: %-memcached +%-memcached: SERVICE=memcached +%-memcached: CONTAINER_CMD=$(MEMCACHED_IMAGE) --port=`cat $(CONTAINER_FILE) | grep 'MEMCACHED_PORT=' | sed 's/MEMCACHED_PORT=//'` +%-memcached: BASE_PORTS = "MEMCACHED_PORT=11211" + +.PHONY: %-minio +%-minio: SERVICE=minio +%-minio: CONTAINER_CMD=$(MINIO_IMAGE) -- server `mktemp` --address=:`cat $(CONTAINER_FILE) | grep 'UPLOAD_S3_PORT=' | sed 's/UPLOAD_S3_PORT=//'`: +%-minio: BASE_PORTS = "UPLOAD_S3_PORT=9000" + +.PHONY: start-% +start-%: JOB=start +start-%: CF_PREFIX = "start-" +start-%: --act ; + +.PHONY: compile-% +compile-%: JOB=compile +compile-%: CF_PREFIX = "compile-" +compile-%: --act ; + +.PHONY: test-% +test-%: JOB=test +test-%: CF_PREFIX = "test-" +test-%: --act ; + +.PHONY: lint-% +lint-%: JOB=lint +lint-%: CF_PREFIX = "lint-" +lint-%: --act ; + +--act: --develop_containerized; + +--develop_%: PORTS = $(foreach PORT,$(BASE_PORTS),$(shell utils/next_free_port.pl $(PORT))) +--develop_%: --ensure-develop + echo "$(SERVICE_PARALLEL)" ; \ + if [[ "$(SERVICE_PARALLEL)" == "false" ]] ; then \ + CONTAINER_FILE=$(DEVELOP)/$(CF_PREFIX)$(SERVICE_VARIANT) ; \ + JOB_ID=$(JOB) ; \ + if [[ -e $${CONTAINER_FILE} ]]; then \ + >&2 echo "Another $* service is already running! Use \"make new-develop\" to start a new develop instance despite currently running services." ; \ + exit 1 ; \ + fi ; \ + else \ + DEVDIR=$(DEVELOP)/$(SERVICE_VARIANT) ; \ + I=`ls $(DEVELOP) | grep '$(SERVICE_VARIANT)' | sed 's/$(SERVICE_VARIANT)-//' | sort -n | tail -n1` ; \ + echo "$${I}" ; \ + J=$$(($${I}+1)) ; \ + CONTAINER_FILE=$${DEVDIR}-$${J} ; \ + JOB_ID=$(JOB)-$${J} ; \ + fi ; \ + echo "$(PORTS)" | sed 's/ /\n/g' > $${CONTAINER_FILE} ; \ + $(MAKE) -- --$* CONTAINER_FILE=$${CONTAINER_FILE} JOB_ID=$${JOB_ID} + +.PHONY: image-rebuild_% +# HELP(image-rebuild_{backend,frontend,database,memcached,minio}): force-rebuild the stated docker image +image-rebuild_%: + $(MAKE) -- --image-build SERVICE=$* NO_CACHE=--no-cache +--image-build: +ifeq "$(CONTAINER_CMD)" "localhost/fradrive/$(SERVICE)" + rm -f .Dockerfile + ln -s docker/$(SERVICE)/Dockerfile .Dockerfile + MOUNT_DIR=/mnt/fradrive; \ + PROJECT_DIR=/mnt/fradrive; \ + if [ "$(IN_CI)" == "true" ] ; then \ + PROJECT_DIR=/fradrive; \ + fi; \ + if [ "$(IN_CONTAINER)" == "false" ] ; then \ + $(CONTAINER_COMMAND) build $(NO_CACHE) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --build-arg MOUNT_DIR=$(MOUNT_DIR) --build-arg PROJECT_DIR=$${PROJECT_DIR} --tag fradrive/$(SERVICE) --file $(PWD)/.Dockerfile ; \ + fi +else + : endif -ifeq "$(SERVICE_VARIANT)" "" - >&2 echo "Variable SERVICE_VARIANT is not set; this hints to an internal error in the Makefile unless you have manually requested a '--'-target" - exit 1 -endif -#ifeq "$(DEVELOP)" "" -# >&2 echo "Variable DEVELOP is not set; this hints to an internal error in the Makefile unless you have manually requested a '--'-target" -# exit 1 -#endif -#ifeq "$(CONTAINER_FILE)" "" -# >&2 echo "Variable CONTAINER_FILE is not set; this hints to an internal error in the Makefile unless you have manually requested a '--'-target" -# exit 1 -#endif +--containerized: --image-build + ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ + CONTAINER_NAME=fradrive.$(CURR_DEV).$(SERVICE_VARIANT).$(JOB_ID) ; \ + CONTAINER_ID=`$(CONTAINER_BGRUN) \ + -v $(PWD):$(MOUNT_DIR) \ + --env IN_CONTAINER=true \ + --env FRADRIVE_MAKE_TARGET="--$(JOB)-$(SERVICE_VARIANT)" \ + --env CONTAINER_FILE=$(CONTAINER_FILE) \ + --env CONTAINER_NAME=$${CONTAINER_NAME} \ + --name $${CONTAINER_NAME} \ + $(CONTAINER_CMD) \ + ` ; \ + printf "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)" -############################ -##### FRONTEND TARGETS ##### +# HELP(start-backend): start yesod-devel instance +--start-backend: + DEV_PORT_HTTP=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTP=' | sed 's/DEV_PORT_HTTP=//'`; \ + DEV_PORT_HTTPS=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTPS=' | sed 's/DEV_PORT_HTTPS=//'`; \ + stack --work-dir=.stack-work-run $(--DEVELOPMENT) exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" +# HELP(compile-backend): compile backend binaries +--compile-backend: + stack build --fast --profile --library-profiling --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin +# HELP(lint-backend): lint backend +--lint-backend: + stack build --test --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) uniworx:test:hlint +# HELP(test-backend): test backend +--test-backend: + stack build --test --coverage --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) -#--frontend-%: --image-build --containerized---%-frontend; - -#--%-frontend: --containerized-static-frontend --containerized-well-known-frontend - -#--frontend-start: static -# npm run start - -ifeq ($(SERVICE_VARIANT), frontend) ---start: --checkvarsset - npm run start - -# HELP(compile-frontend): compile frontend ---compile: static well-known --checkvarsset; - -# HELP(lint-frontend): lint frontend ---lint: eslint.config.js --checkvarsset - npx -- eslint frontend/src $(FIX) - @echo Hooray! There are no hints. - -# HELP(test-frontend): test frontend ---test: karma.conf.cjs --checkvarsset - @echo Karma frontend tests are currently broken after npm update and have therefor been temporarily disabled. -# npx -- karma start --conf karma.conf.cjs $(WATCH) -endif +# HELP(compile-frontend): compile frontend assets +--compile-frontend: node_modules assets esbuild.config.mjs + npm run build node_modules: package.json package-lock.json npm ci --cache .npm --prefer-offline @@ -243,192 +235,40 @@ static: node_modules assets esbuild.config.mjs npm run build well-known: static; -##### FRONTEND TARGETS ##### -############################ - - -########################### -##### BACKEND TARGETS ##### - - -ifeq ($(SERVICE_VARIANT), backend) ---start: --checkvarsset - DEV_PORT_HTTP=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTP=' | sed 's/DEV_PORT_HTTP=//'`; \ - DEV_PORT_HTTPS=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTPS=' | sed 's/DEV_PORT_HTTPS=//'`; \ - stack --work-dir=.stack-work-run $(--DEVELOPMENT) exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" -endif -ifeq ($(SERVICE_VARIANT), hoogle) ---start: - HOOGLE_PORT=`cat $(CONTAINER_FILE) | grep 'HOOGLE_PORT=' | sed 's/HOOGLE_PORT=//'` ; \ - echo "${HOOGLE_PORT}" ; \ - stack --work-dir=.stack-work-doc hoogle -- server --local --port $${HOOGLE_PORT} -endif - -# HELP(compile-backend): compile backend -ifeq ($(SERVICE_VARIANT), backend) ---compile: --checkvarsset - stack build --fast --profile --library-profiling --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin - -# HELP(lint-backend): lint backend ---lint: --checkvarsset - stack build --test --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) uniworx:test:hlint - -# HELP(test-backend): test backend ---test: --checkvarsset - stack build --test --coverage --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) -endif # HELP(uniworxdb): clear and fill database. requires running postgres -uniworxdb: --frontend_compile --backend_imagebuild_containerized_--uniworxdb_start; # TODO (db-m-$MIGRATION-backend): apply migration (see src/Model/Migration/Definition.hs for list of available migrations) -ifeq ($(SERVICE_VARIANT), uniworxdb) ---start: .stack +--start-uniworxdb: SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true} ; \ AVSPASS=${AVSPASS:-nopasswordset} ; \ stack $(--DEVELOPMENT) exec uniworxdb -- $(UNIWORXDB_OPTS) -endif -.PHONY: .stack -.stack: - if [ "$(IN_CONTAINER)" == "true" ] ; then \ - $(MAKE) -- --.stack ; \ - else \ - $(MAKE) -- --imagerun_--.stack ; \ - fi ---.stack: stack.yaml stack.yaml.lock package.yaml - stack build --fast --only-dependencies +# HELP(start-minio): start minio service -##### BACKEND TARGETS ##### -########################### +.PHONY: status +# HELP: print develop status: running containers, used ports +status: + @./utils/develop-status.pl +# TODO: rework logs +.PHONY: log-% +# HELP(log-{database,memcached,minio,backend,frontend,hoogle}): inspect output of a given (currently running) service. When a service supports multiple running instances in one develop (i.e. backend), you need to specify the exact instance by its associated file (e.g. backend-1, backend-2, etc.), please check the contents of the develop/ directory for a list of running instances. +log-%: + $(CONTAINER_COMMAND) logs --follow `cat $(DEVELOP)/$* | grep CONTAINER_ID= | sed 's/^CONTAINER_ID=//'` -############################ -##### DATABASE TARGETS ##### - -ifeq ($(SERVICE_VARIANT), postgres) ---containerized_%: --checkvarsset docker/database/initdb.sh docker/database/pg_hba.conf docker/database/postgresql.conf docker/database/schema.sql --image-build - $(MAKE) -- --imagerun_$* -else ---containerized_%: --checkvarsset - $(MAKE) -- --imagerun_$* -endif - +# TODO: rework shells +.PHONY: %-shell +# HELP: launch shell (bash) inside a currently running container +%-shell: --%_shell; +--shell: + $(CONTAINER_COMMAND) exec -it $(EXEC_OPTS) fradrive.$(CURR_DEV).$* $(if $(ENTRYPOINT),$(ENTRYPOINT),/bin/bash) .PHONY: psql # HELP: enter psql (postgresql) cli inside a currently running database container psql: ENTRYPOINT=/usr/bin/psql -d uniworx psql: EXEC_OPTS=--user postgres psql: --database-shell; -##### DATABASE TARGETS ##### -############################ - - -############################# -##### CONTAINER TARGETS ##### - -#--containerized-%-frontend: FRADRIVE_SERVICE=frontend -#--containerized-%-frontend: SERVICE_VARIANT=frontend -#--containerized-%-frontend: --image-build -# $(MAKE) -- --image-run-$*-frontend -# -#--containerized-%-backend: FRADRIVE_SERVICE=backend -#--containerized-%-backend: SERVICE_VARIANT?=backend -#--containerized-%-backend: --image-build -# $(MAKE) -- --image-run-$*-backend -#--containerized-%-hoogle: FRADRIVE_SERVICE=backend -#--containerized-%-hoogle: SERVICE_VARIANT=hoogle -#--containerized-%-hoogle: --image-build -# $(MAKE) -- --image-run-$*-hoogle-backend - -# --containerized-%-minio: FRADRIVE_SERVICE=minio -# --containerized-%-minio: --image-build -# $(MAKE) -- --image-run-$*-minio -ifeq ($(SERVICE_VARIANT), minio) ---containerized_start: - UPLOAD_S3_PORT=`cat $(CONTAINER_FILE) | grep 'UPLOAD_S3_PORT=' | sed 's/UPLOAD_S3_PORT=//'`; \ - MINIO_DIR=`mktemp` ; \ - ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" "" "rm -rf $${MINIO_DIR}" & \ - CONTAINER_ID=`$(CONTAINER_BGRUN) --name fradrive.$(CURR_DEV).minio $(MINIO_IMAGE) -- server $${MINIO_DIR} --address=:$${UPLOAD_S3_PORT}` ; \ - printf "CONTAINER_ID=$${CONTAINER_ID}\nUPLOAD_S3_PORT=$${UPLOAD_S3_PORT}\nMINIO_DIR=$${MINIO_DIR}" >> $(CONTAINER_FILE) -endif - -# --containerized-%-memcached: FRADRIVE_SERVICE=memcached -# --containerized-%-memcached: --image-build -# $(MAKE) -- --image-run-$*-memcached -ifeq ($(SERVICE_VARIANT), memcached) ---containerized_start: - MEMCACHED_PORT=`cat $(CONTAINER_FILE) | grep 'MEMCACHED_PORT=' | sed 's/MEMCACHED_PORT=//'`; \ - ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ - CONTAINER_ID=`$(CONTAINER_BGRUN) --name fradrive.$(CURR_DEV).memcached $(MEMCACHED_IMAGE) --port=$${MEMCACHED_PORT}` ; \ - printf "CONTAINER_ID=$${CONTAINER_ID}\nMEMCACHED_PORT=$${MEMCACHED_PORT}" >> $(CONTAINER_FILE) -endif - -.PHONY: image-rebuild -# HELP(image-rebuild-{backend,frontend,database,memcached,minio}): force-rebuild the stated docker image -image-rebuild_%: - $(MAKE) -- --image-build FRADRIVE_SERVICE=$* NO_CACHE=--no-cache ---image-build: - rm -f .Dockerfile - ln -s docker/$(FRADRIVE_SERVICE)/Dockerfile .Dockerfile - MOUNT_DIR=/mnt/fradrive; \ - PROJECT_DIR=/mnt/fradrive; \ - if [ "$(IN_CI)" == "true" ] ; then \ - PROJECT_DIR=/fradrive; \ - fi; \ - if [ "$(IN_CONTAINER)" == "false" ] ; then \ - $(CONTAINER_COMMAND) build $(NO_CACHE) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --build-arg MOUNT_DIR=$${MOUNT_DIR} --build-arg PROJECT_DIR=$${PROJECT_DIR} --tag fradrive/$(FRADRIVE_SERVICE) --file $(PWD)/.Dockerfile ; \ - fi - -# runs --% in the container. Targets meant to be called manually cannot be run in the container via this target. If that is ever needed split this target into two targets, one for manually one for automatic (one may use the other) so that not at every usage a '---' has to be after containerized ---imagerun_%: --checkvarsset docker/$(FRADRIVE_SERVICE)/Dockerfile - MOUNT_DIR=/mnt/fradrive; \ - if [ "$(IN_CONTAINER)" == "true" ] ; then \ - $(MAKE) -- $* ; \ - else \ - if [ -z "$(CONTAINER_FILE)" ] ; then \ - $(CONTAINER_FGRUN) -v $(PWD):$${MOUNT_DIR} --env FRADRIVE_SERVICE=$(FRADRIVE_SERVICE) --env SERVICE_VARIANT=$(SERVICE_VARIANT) --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(SERVICE_VARIANT) localhost/fradrive/$(FRADRIVE_SERVICE) ; \ - else \ - ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ - CONTAINER_ID=`$(CONTAINER_BGRUN) -v $(PWD):$${MOUNT_DIR} --env FRADRIVE_SERVICE=$(FRADRIVE_SERVICE) --env SERVICE_VARIANT=$(SERVICE_VARIANT) --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(SERVICE_VARIANT) localhost/fradrive/$(FRADRIVE_SERVICE)` ; \ - echo "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)"; \ - fi \ - fi - - - -# HELP(start-database): start postgres server -# HELP(start-memcached): start memcached server -# HELP(start-minio): start minio server -# HELP(start-backend): serve yesod development site -# HELP(start-frontend): start frontend watcher. Watches frontend source files for changes and recompiles on change. (TODO: watcher functionality currently broken!) -# HELP(start-hoogle): serve local hoogle instance -# TODO reformulate target to use the '--%_'-mechanism -# --autostart: use the SERVICE_VARIANT for the decision which service to start ---autostart: --checkvarsset - $(MAKE) -- start-$(SERVICE_VARIANT) - -start-%: FRADRIVE_SERVICE = % -start-%: SERVICE_VARIANT = % -start-%: PORTS = $(foreach PORT,$(BASE_PORTS),$(shell utils/next_free_port.pl $(PORT))) -start-%: --develop - echo "$*" - echo "$(DATE)" - echo "DEV $(DEVELOP)" - if [[ "$(SINGLETON)" = "true" ]]; then \ - CONTAINER_FILE=$(DEVELOP)/$* ; \ - if [[ -e $${CONTAINER_FILE} ]]; then \ - >&2 echo "Another $* service is already running! Use \"make new-develop\" or \"make start\" to start a new develop instance despite currently running services." ; \ - exit 1 ; \ - fi \ - else \ - DEVDIR=$(DEVELOP)/$*; \ - I=`ls $${DEVDIR} | sort -n | tail -n1`; \ - echo "$${I}"; \ - CONTAINER_FILE=$${DEVDIR}-$$(($${I}+1)); \ - fi ; \ - echo "$(PORTS)" | sed 's/ /\n/g' > $${CONTAINER_FILE} ; \ - $(MAKE) -- --containerized_--autostart CONTAINER_FILE=$${CONTAINER_FILE} - +# TODO: rework stops .PHONY: stop # HELP: stop all currently running develop instances stop: @@ -445,28 +285,6 @@ stop-container-by-id: # CONTAINER_ID=`grep 'CONTAINER_ID=' $(CONTAINER_FILE) | sed 's/CONTAINER_ID=//'` ; \ # $(MAKE) stop-container-by-id CONTAINER_ID=$${CONTAINER_ID} -.PHONY: status -# HELP: print develop status: running containers, used ports -status: - @./utils/develop-status.pl - -.PHONY: log-% -# HELP(log-{database,memcached,minio,backend,frontend,hoogle}): inspect output of a given (currently running) service. When a service supports multiple running instances in one develop (i.e. backend), you need to specify the exact instance by its associated file (e.g. backend-1, backend-2, etc.), please check the contents of the develop/ directory for a list of running instances. -log-%: --develop - $(CONTAINER_COMMAND) logs --follow --tail=$(LOGSIZE) `cat $(DEVELOP)/$* | grep CONTAINER_ID= | sed 's/^CONTAINER_ID=//'` - -##### CONTAINER TARGETS ##### -############################# - - -# TODO: move targets below to better location - -DATE := $(shell date +'%Y-%m-%dT%H-%M-%S') - -CURR_DEV = $(shell ls -1 develop | tail -n1) -SET_DEVELOP = $(eval DEVELOP=develop/$$(CURR_DEV)) -NEW_DEVELOP = $(eval DEVELOP=develop/$$(DATE)) - .PHONY: new-develop # HELP: instantiate new development bundle, i.e. create new directory under develop/ new-develop: @@ -481,33 +299,15 @@ switch-develop: exit 1 ; \ fi ; \ echo "$(DEVELOP)" > develop/.current -.PHONY: --develop ---develop: +--ensure-develop: if ! [[ -e develop ]]; then \ $(MAKE) new-develop; \ fi - $(SET_DEVELOP) $(MAKE) develop/.current + $(SET_DEVELOP) .PHONY: develop/.current develop/.current: - $(SET_DEVELOP) - echo "$(DEVELOP)" > develop/.current - -# Some convenience aliases: -.PHONY: hoogle -# HELP: alias for start-hoogle -hoogle: start-hoogle; - -.PHONY: i18n-check -# HELP: check internationalization -i18n-check: --imagerun_--i18n-check ---i18n-check: - ./missing-translations.sh - @echo No missing translations. - -%.lock: - [ -e $@ ] || touch $@ - flock -en $@ true + ls -1 develop | tail -n1 > develop/.current .PHONY: --% .SUFFIXES: # Delete all default suffixes diff --git a/docker/database/Dockerfile b/docker/postgres/Dockerfile similarity index 100% rename from docker/database/Dockerfile rename to docker/postgres/Dockerfile diff --git a/docker/database/initdb.sh b/docker/postgres/initdb.sh similarity index 100% rename from docker/database/initdb.sh rename to docker/postgres/initdb.sh diff --git a/docker/database/pg_hba.conf b/docker/postgres/pg_hba.conf similarity index 100% rename from docker/database/pg_hba.conf rename to docker/postgres/pg_hba.conf diff --git a/docker/database/pgconfig.sh b/docker/postgres/pgconfig.sh similarity index 100% rename from docker/database/pgconfig.sh rename to docker/postgres/pgconfig.sh diff --git a/docker/database/postgresql.conf b/docker/postgres/postgresql.conf similarity index 100% rename from docker/database/postgresql.conf rename to docker/postgres/postgresql.conf diff --git a/docker/database/schema.sql b/docker/postgres/schema.sql similarity index 100% rename from docker/database/schema.sql rename to docker/postgres/schema.sql From da35d404dee1b622d727bc70fbfac64a30ac73c8 Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Mon, 4 Nov 2024 14:54:56 +0100 Subject: [PATCH 19/64] build(Makefile): change 'make status' to show also dead container --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 52734e06c..3b4f56ffa 100644 --- a/Makefile +++ b/Makefile @@ -248,7 +248,7 @@ well-known: static; .PHONY: status # HELP: print develop status: running containers, used ports status: - @./utils/develop-status.pl + @./utils/develop-status.pl -a # TODO: rework logs .PHONY: log-% From c9fbfc7e193b85c4ecfcc61a8a71d7bf0a2ceb46 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 4 Nov 2024 15:17:47 +0100 Subject: [PATCH 20/64] build(docker/database): fix postgres file paths --- docker/postgres/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/postgres/Dockerfile b/docker/postgres/Dockerfile index 75714c7a3..c3a14c0b9 100644 --- a/docker/postgres/Dockerfile +++ b/docker/postgres/Dockerfile @@ -3,7 +3,7 @@ FROM docker.io/postgres:12 # Allow for connecting to database without password authentication ENV POSTGRES_HOST_AUTH_METHOD=trust -COPY --chown=postgres:postgres docker/database/pg_hba.conf /tmp/pg_hba.conf -COPY --chown=postgres:postgres docker/database/postgresql.conf /tmp/postgresql.conf -COPY docker/database/pgconfig.sh /docker-entrypoint-initdb.d/_pgconfig.sh -COPY --chown=postgres:postgres docker/database/schema.sql /docker-entrypoint-initdb.d/schema.sql +COPY --chown=postgres:postgres docker/postgres/pg_hba.conf /tmp/pg_hba.conf +COPY --chown=postgres:postgres docker/postgres/postgresql.conf /tmp/postgresql.conf +COPY docker/postgres/pgconfig.sh /docker-entrypoint-initdb.d/_pgconfig.sh +COPY --chown=postgres:postgres docker/postgres/schema.sql /docker-entrypoint-initdb.d/schema.sql From b89ed41b5b70447407f3051d72f0f18a13a3be86 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 4 Nov 2024 15:18:17 +0100 Subject: [PATCH 21/64] build(Makefile): add missing SERVICE_VARIANT def --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 3b4f56ffa..65539d931 100644 --- a/Makefile +++ b/Makefile @@ -105,6 +105,7 @@ start: start-postgres start-memcached start-minio start-frontend start-backend; .PHONY: %-frontend %-frontend: SERVICE=frontend +%-backend: SERVICE_VARIANT=frontend %-frontend: CONTAINER_CMD=localhost/fradrive/frontend .PHONY: %-postgres From a5d270ccf495f210c060a8455ec3a4687ddbd06b Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Mon, 4 Nov 2024 15:34:00 +0100 Subject: [PATCH 22/64] build(Makefile): Reattach container that are meant to be foreground. --- Makefile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3b4f56ffa..24e74e52f 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,10 @@ export CONTAINER_FILE export CF_PREFIX export DEVELOP export MOUNT_DIR=/mnt/fradrive +# CONTAINER_RUNMODE variants: +# detached: detached container +# attached: attached container +export CONTAINER_RUNMODE ?= detached export SERVICE export SERVICE_VARIANT ?= $(SERVICE) @@ -130,16 +134,19 @@ start-%: --act ; .PHONY: compile-% compile-%: JOB=compile compile-%: CF_PREFIX = "compile-" +compile-%: CONTAINER_RUNMODE = "attached" compile-%: --act ; .PHONY: test-% test-%: JOB=test test-%: CF_PREFIX = "test-" +test-%: CONTAINER_RUNMODE = "attached" test-%: --act ; .PHONY: lint-% lint-%: JOB=lint lint-%: CF_PREFIX = "lint-" +lint-%: CONTAINER_RUNMODE = "attached" lint-%: --act ; --act: --develop_containerized; @@ -197,7 +204,10 @@ endif --name $${CONTAINER_NAME} \ $(CONTAINER_CMD) \ ` ; \ - printf "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)" + printf "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)" ; \ + if [[ "attached" -eq $(CONTAINER_RUNMODE) ]] ; then \ + $(CONTAINER_COMMAND) attach $${CONTAINER_ID} ; \ + fi # HELP(start-backend): start yesod-devel instance --start-backend: From 80849f8c5ecd490c653d61fee74caa9a75e80c60 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 4 Nov 2024 15:57:14 +0100 Subject: [PATCH 23/64] build(Makefile): fix shell; introduce enter for launching shells in running containers --- Makefile | 25 ++++++++++++++++++------- enter | 3 +++ 2 files changed, 21 insertions(+), 7 deletions(-) create mode 100755 enter diff --git a/Makefile b/Makefile index 65539d931..3c69de5c4 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,8 @@ export SERVICE_PARALLEL ?= false export JOB export JOB_ID export CONTAINER_CMD +export ENTRYPOINT +export EXEC_OPTS export BASE_PORTS export UNIWORXDB_OPTS ?= -cf @@ -257,17 +259,26 @@ status: log-%: $(CONTAINER_COMMAND) logs --follow `cat $(DEVELOP)/$* | grep CONTAINER_ID= | sed 's/^CONTAINER_ID=//'` -# TODO: rework shells -.PHONY: %-shell -# HELP: launch shell (bash) inside a currently running container -%-shell: --%_shell; ---shell: - $(CONTAINER_COMMAND) exec -it $(EXEC_OPTS) fradrive.$(CURR_DEV).$* $(if $(ENTRYPOINT),$(ENTRYPOINT),/bin/bash) +.PHONY: enter +# HELP: launch (bash) shell inside a currently running container. Use ./enter shell wrapper for more convenient usage, possibly with tab-completion in the future +enter: --ensure-develop + $(MAKE) -- --shell + .PHONY: psql # HELP: enter psql (postgresql) cli inside a currently running database container psql: ENTRYPOINT=/usr/bin/psql -d uniworx psql: EXEC_OPTS=--user postgres -psql: --database-shell; +psql: --ensure-develop + $(MAKE) -- --shell CONTAINER_FILE=develop/`cat develop/.current`/start-postgres + +.PHONY: ghci +# HELP: launch new backend instance and enter interactive ghci shell (WIP) +ghci: ENTRYPOINT=stack ghci +ghci: --shell; + +--shell: + CONTAINER_ID=`cat $(CONTAINER_FILE) | grep 'CONTAINER_ID=' | sed 's/CONTAINER_ID=//'` ; \ + $(CONTAINER_COMMAND) exec -it $(EXEC_OPTS) $${CONTAINER_ID} $(if $(ENTRYPOINT),$(ENTRYPOINT),/bin/bash) # TODO: rework stops .PHONY: stop diff --git a/enter b/enter new file mode 100755 index 000000000..427d93c10 --- /dev/null +++ b/enter @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +make enter CONTAINER_FILE=develop/`cat develop/.current`/"$1" \ No newline at end of file From 7b544990136f98963537494cc8c874acfebf00bd Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Mon, 4 Nov 2024 16:23:54 +0100 Subject: [PATCH 24/64] build(Makefile): Ignore reattach errors. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 69994e246..78ea19a1e 100644 --- a/Makefile +++ b/Makefile @@ -207,7 +207,7 @@ endif ` ; \ printf "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)" ; \ if [[ "attached" -eq $(CONTAINER_RUNMODE) ]] ; then \ - $(CONTAINER_COMMAND) attach $${CONTAINER_ID} ; \ + $(CONTAINER_COMMAND) attach $${CONTAINER_ID} || : ; \ fi # HELP(start-backend): start yesod-devel instance From 0be36ca82aae498c5f9cc6a617c564c315d6e646 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 4 Nov 2024 16:33:26 +0100 Subject: [PATCH 25/64] build(Makefile): fix SET_DEVELOP --- Makefile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 28120ba3c..6ac7ba6b8 100644 --- a/Makefile +++ b/Makefile @@ -156,6 +156,7 @@ lint-%: --act ; --develop_%: PORTS = $(foreach PORT,$(BASE_PORTS),$(shell utils/next_free_port.pl $(PORT))) --develop_%: --ensure-develop + $(SET_DEVELOP) echo "$(SERVICE_PARALLEL)" ; \ if [[ "$(SERVICE_PARALLEL)" == "false" ]] ; then \ CONTAINER_FILE=$(DEVELOP)/$(CF_PREFIX)$(SERVICE_VARIANT) ; \ @@ -230,13 +231,10 @@ endif # HELP(compile-frontend): compile frontend assets --compile-frontend: node_modules assets esbuild.config.mjs npm run build - node_modules: package.json package-lock.json npm ci --cache .npm --prefer-offline - package-lock.json: package.json npm install --cache .npm --prefer-offline - assets: assets/favicons assets/icons; assets/favicons: ./utils/faviconize.pl assets/favicon.svg long assets/favicons @@ -248,7 +246,6 @@ static: node_modules assets esbuild.config.mjs npm run build well-known: static; - # HELP(uniworxdb): clear and fill database. requires running postgres # TODO (db-m-$MIGRATION-backend): apply migration (see src/Model/Migration/Definition.hs for list of available migrations) --start-uniworxdb: @@ -290,7 +287,6 @@ ghci: --shell; CONTAINER_ID=`cat $(CONTAINER_FILE) | grep 'CONTAINER_ID=' | sed 's/CONTAINER_ID=//'` ; \ $(CONTAINER_COMMAND) exec -it $(EXEC_OPTS) $${CONTAINER_ID} $(if $(ENTRYPOINT),$(ENTRYPOINT),/bin/bash) -# TODO: rework stops .PHONY: stop # HELP: stop all currently running develop instances stop: From bdca968818c18781b680e2c3f6f0e308f078ffa3 Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Mon, 4 Nov 2024 16:55:04 +0100 Subject: [PATCH 26/64] build(Makefile): fix typos in date and SERVICE_VARIANT --- Makefile | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 13286eb10..b980da707 100644 --- a/Makefile +++ b/Makefile @@ -16,10 +16,7 @@ export CONTAINER_FILE export CF_PREFIX export DEVELOP export MOUNT_DIR=/mnt/fradrive -# CONTAINER_RUNMODE variants: -# detached: detached container -# attached: attached container -export CONTAINER_RUNMODE ?= detached +export CONTAINER_ATTACHED export SERVICE export SERVICE_VARIANT ?= $(SERVICE) @@ -111,7 +108,7 @@ start: start-postgres start-memcached start-minio start-frontend start-backend; .PHONY: %-frontend %-frontend: SERVICE=frontend -%-backend: SERVICE_VARIANT=frontend +%-frontend: SERVICE_VARIANT=frontend %-frontend: CONTAINER_CMD=localhost/fradrive/frontend .PHONY: %-postgres @@ -131,43 +128,44 @@ start: start-postgres start-memcached start-minio start-frontend start-backend; .PHONY: start-% start-%: JOB=start -start-%: CF_PREFIX = "start-" +start-%: CF_PREFIX = start- +start-%: CONTAINER_ATTACHED = false start-%: --act ; .PHONY: compile-% compile-%: JOB=compile -compile-%: CF_PREFIX = "compile-" -compile-%: CONTAINER_RUNMODE = "attached" +compile-%: CF_PREFIX = compile- +compile-%: CONTAINER_ATTACHED = true compile-%: --act ; .PHONY: test-% test-%: JOB=test -test-%: CF_PREFIX = "test-" -test-%: CONTAINER_RUNMODE = "attached" +test-%: CF_PREFIX = test- +test-%: CONTAINER_ATTACHED = true test-%: --act ; .PHONY: lint-% lint-%: JOB=lint -lint-%: CF_PREFIX = "lint-" -lint-%: CONTAINER_RUNMODE = "attached" +lint-%: CF_PREFIX = lint- +lint-%: CONTAINER_ATTACHED = true lint-%: --act ; --act: --develop_containerized; --develop_%: PORTS = $(foreach PORT,$(BASE_PORTS),$(shell utils/next_free_port.pl $(PORT))) --develop_%: --ensure-develop - $(SET_DEVELOP) + DEVELOP=develop/`cat develop/.current` ; \ echo "$(SERVICE_PARALLEL)" ; \ if [[ "$(SERVICE_PARALLEL)" == "false" ]] ; then \ - CONTAINER_FILE=$(DEVELOP)/$(CF_PREFIX)$(SERVICE_VARIANT) ; \ + CONTAINER_FILE=$${DEVELOP}/$(CF_PREFIX)$(SERVICE_VARIANT) ; \ JOB_ID=$(JOB) ; \ if [[ -e $${CONTAINER_FILE} ]]; then \ >&2 echo "Another $* service is already running! Use \"make new-develop\" to start a new develop instance despite currently running services." ; \ exit 1 ; \ fi ; \ else \ - DEVDIR=$(DEVELOP)/$(SERVICE_VARIANT) ; \ - I=`ls $(DEVELOP) | grep '$(SERVICE_VARIANT)' | sed 's/$(SERVICE_VARIANT)-//' | sort -n | tail -n1` ; \ + DEVDIR=$${DEVELOP}/$(SERVICE_VARIANT) ; \ + I=`ls $${DEVELOP} | grep '$(SERVICE_VARIANT)' | sed 's/$(SERVICE_VARIANT)-//' | sort -n | tail -n1` ; \ echo "$${I}" ; \ J=$$(($${I}+1)) ; \ CONTAINER_FILE=$${DEVDIR}-$${J} ; \ @@ -197,6 +195,7 @@ else endif --containerized: --image-build + DEVELOP=`cat develop/.current` ; \ ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ CONTAINER_NAME=fradrive.$(CURR_DEV).$(SERVICE_VARIANT).$(JOB_ID) ; \ CONTAINER_ID=`$(CONTAINER_BGRUN) \ @@ -209,7 +208,7 @@ endif $(CONTAINER_CMD) \ ` ; \ printf "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)" ; \ - if [[ "attached" -eq $(CONTAINER_RUNMODE) ]] ; then \ + if [[ "true" -eq "$(CONTAINER_ATTACHED)" ]] ; then \ $(CONTAINER_COMMAND) attach $${CONTAINER_ID} || : ; \ fi From 043753782467ad99b77f80739a87bc17be4ece9f Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Mon, 4 Nov 2024 17:28:01 +0100 Subject: [PATCH 27/64] build(Makefile): make 'make log' work; mv containerfiles to .exited. --- Makefile | 14 ++++++++++++-- utils/watchcontainerrun.sh | 4 +++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b980da707..a6853228e 100644 --- a/Makefile +++ b/Makefile @@ -259,11 +259,21 @@ well-known: static; status: @./utils/develop-status.pl -a -# TODO: rework logs +# TODO: rework logs, because they do not work .PHONY: log-% # HELP(log-{database,memcached,minio,backend,frontend,hoogle}): inspect output of a given (currently running) service. When a service supports multiple running instances in one develop (i.e. backend), you need to specify the exact instance by its associated file (e.g. backend-1, backend-2, etc.), please check the contents of the develop/ directory for a list of running instances. log-%: - $(CONTAINER_COMMAND) logs --follow `cat $(DEVELOP)/$* | grep CONTAINER_ID= | sed 's/^CONTAINER_ID=//'` + DEVELOP=develop/`cat develop/.current` ; \ + SEARCH_FILE="$${DEVELOP}/$*" ; \ + if [[ ! -e "$${SEARCH_FILE}" ]] ; then \ + SEARCH_FILE="$${DEVELOP}/.exited.$*" ; \ + fi ; \ + if [[ -e "$${SEARCH_FILE}" ]] ; then \ + $(CONTAINER_COMMAND) logs --follow `cat "$${SEARCH_FILE}" | grep CONTAINER_ID= | sed 's/^CONTAINER_ID=//'` ; \ + else \ + >&2 echo "Cannot show log: No develop file found for '$*'" ; \ + exit 1 ; \ + fi .PHONY: enter # HELP: launch (bash) shell inside a currently running container. Use ./enter shell wrapper for more convenient usage, possibly with tab-completion in the future diff --git a/utils/watchcontainerrun.sh b/utils/watchcontainerrun.sh index eb2073024..3e0bc91a0 100755 --- a/utils/watchcontainerrun.sh +++ b/utils/watchcontainerrun.sh @@ -29,7 +29,9 @@ $STARTSCRIPT while [ -e "$FILENAME" ] ; do sleep 1 if ! $RUNNER ps --no-trunc | grep -q "$CONTAINER_ID" ; then - rm "$FILENAME" + #rm "$FILENAME" + EXITED_FILENAME=`echo "$FILENAME" | sed 's/\([^\/]*\)$/.exited.\1/'` + mv "$FILENAME" "$EXITED_FILENAME" exit fi inotifywait -e ATTRIB -t 10 "$FILENAME" > /dev/null 2>/dev/null From ba6a50259146e4cb1998c222000e44c03940a4c3 Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Mon, 4 Nov 2024 17:35:15 +0100 Subject: [PATCH 28/64] build(Makefile): fix path of containerfile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a6853228e..8289b5e21 100644 --- a/Makefile +++ b/Makefile @@ -164,7 +164,7 @@ lint-%: --act ; exit 1 ; \ fi ; \ else \ - DEVDIR=$${DEVELOP}/$(SERVICE_VARIANT) ; \ + DEVDIR=$${DEVELOP}/$(CF_PREFIX)$(SERVICE_VARIANT) ; \ I=`ls $${DEVELOP} | grep '$(SERVICE_VARIANT)' | sed 's/$(SERVICE_VARIANT)-//' | sort -n | tail -n1` ; \ echo "$${I}" ; \ J=$$(($${I}+1)) ; \ From ad8edfd448fd72097ead31267aaf2b2bf8eb0eb4 Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Mon, 4 Nov 2024 17:52:56 +0100 Subject: [PATCH 29/64] build(Makefile): Fixed start postgres --- Makefile | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8289b5e21..2301638a7 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,12 @@ compile: $(MAKE) compile-backend .PHONY: start -start: start-postgres start-memcached start-minio start-frontend start-backend; +start: + $(MAKE) start-postgres + $(MAKE) start-memcached + $(MAKE) start-minio + $(MAKE) start-frontend + $(MAKE) start-backend .PHONY: %-backend %-backend: SERVICE=backend @@ -113,16 +118,19 @@ start: start-postgres start-memcached start-minio start-frontend start-backend; .PHONY: %-postgres %-postgres: SERVICE=postgres +%-postgres: SERVICE_VARIANT=postgres %-postgres: BASE_PORTS = "PGPORT=5432" %-postgres: CONTAINER_CMD=localhost/fradrive/postgres .PHONY: %-memcached %-memcached: SERVICE=memcached +%-memcached: SERVICE_VARIANT=memcached %-memcached: CONTAINER_CMD=$(MEMCACHED_IMAGE) --port=`cat $(CONTAINER_FILE) | grep 'MEMCACHED_PORT=' | sed 's/MEMCACHED_PORT=//'` %-memcached: BASE_PORTS = "MEMCACHED_PORT=11211" .PHONY: %-minio %-minio: SERVICE=minio +%-minio: SERVICE_VARIANT=minio %-minio: CONTAINER_CMD=$(MINIO_IMAGE) -- server `mktemp` --address=:`cat $(CONTAINER_FILE) | grep 'UPLOAD_S3_PORT=' | sed 's/UPLOAD_S3_PORT=//'`: %-minio: BASE_PORTS = "UPLOAD_S3_PORT=9000" @@ -154,6 +162,7 @@ lint-%: --act ; --develop_%: PORTS = $(foreach PORT,$(BASE_PORTS),$(shell utils/next_free_port.pl $(PORT))) --develop_%: --ensure-develop + echo "--develop ################## ATTACHED: $(CONTAINER_ATTACHED)" DEVELOP=develop/`cat develop/.current` ; \ echo "$(SERVICE_PARALLEL)" ; \ if [[ "$(SERVICE_PARALLEL)" == "false" ]] ; then \ @@ -166,8 +175,9 @@ lint-%: --act ; else \ DEVDIR=$${DEVELOP}/$(CF_PREFIX)$(SERVICE_VARIANT) ; \ I=`ls $${DEVELOP} | grep '$(SERVICE_VARIANT)' | sed 's/$(SERVICE_VARIANT)-//' | sort -n | tail -n1` ; \ - echo "$${I}" ; \ + echo "I=$${I}" ; \ J=$$(($${I}+1)) ; \ + echo "J=$${J}" ; \ CONTAINER_FILE=$${DEVDIR}-$${J} ; \ JOB_ID=$(JOB)-$${J} ; \ fi ; \ @@ -195,6 +205,7 @@ else endif --containerized: --image-build + echo "--containerized ################## ATTACHED: $(CONTAINER_ATTACHED)" DEVELOP=`cat develop/.current` ; \ ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ CONTAINER_NAME=fradrive.$(CURR_DEV).$(SERVICE_VARIANT).$(JOB_ID) ; \ @@ -208,7 +219,8 @@ endif $(CONTAINER_CMD) \ ` ; \ printf "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)" ; \ - if [[ "true" -eq "$(CONTAINER_ATTACHED)" ]] ; then \ + if [[ "true" == "$(CONTAINER_ATTACHED)" ]] ; then \ + echo "###!!! postgres shall not go here!" ; \ $(CONTAINER_COMMAND) attach $${CONTAINER_ID} || : ; \ fi From e8d8aafccd065e48aaa2a9ed8de6a1afb87413e4 Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Mon, 4 Nov 2024 18:57:45 +0100 Subject: [PATCH 30/64] build(Makefile): Work on stack-calls. --- Makefile | 6 +++--- docker/backend/Dockerfile | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 2301638a7..4ba85bcbf 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ export UNIWORXDB_OPTS ?= -cf export PROD ?= false ifneq ($(PROD),true) - export --DEVELOPMENT=--flag uniworx:dev + export --DEVELOPMENT=--flag=uniworx:dev endif export DATE := $(shell date +'%Y-%m-%dT%H-%M-%S') @@ -228,10 +228,10 @@ endif --start-backend: DEV_PORT_HTTP=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTP=' | sed 's/DEV_PORT_HTTP=//'`; \ DEV_PORT_HTTPS=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTPS=' | sed 's/DEV_PORT_HTTPS=//'`; \ - stack --work-dir=.stack-work-run $(--DEVELOPMENT) exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" + stack --work-dir=.stack-work-run --lock-file ignore exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" # HELP(compile-backend): compile backend binaries --compile-backend: - stack build --fast --profile --library-profiling --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin + stack build --fast --profile --library-profiling --lock-file ignore --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin # HELP(lint-backend): lint backend --lint-backend: stack build --test --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) uniworx:test:hlint diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index f8f1fa183..ac56909b4 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -21,11 +21,11 @@ RUN mkdir -p "${PROJECT_DIR}/.stack" ENV STACK_ROOT="${PROJECT_DIR}/.stack" RUN make -- --.stack STACK_ROOT=${STACK_ROOT} IN_CONTAINER=true -RUN stack --work-dir .stack-work-run build yesod-bin +RUN stack --work-dir .stack-work-run install yesod-bin RUN chmod -R 777 .stack .stack-work-build .stack-work-run .stack-work-test .stack-work-doc || : ENV FRADRIVE_MAKE_TARGET=--start-backend ENTRYPOINT make -- ${FRADRIVE_MAKE_TARGET} STACK_ROOT="${STACK_ROOT}" IN_CONTAINER=true CONTAINER_FILE="${CONTAINER_FILE}" EXPOSE 3000/tcp -EXPOSE 3443/tcp \ No newline at end of file +EXPOSE 3443/tcp From b99472288d7b21691d7756636d54d64c10372683 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Wed, 6 Nov 2024 15:28:16 +0100 Subject: [PATCH 31/64] build(Makefile): implement dependencies-backend target --- Makefile | 9 +++++++++ docker/backend/Dockerfile | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4ba85bcbf..eca2652db 100644 --- a/Makefile +++ b/Makefile @@ -146,6 +146,12 @@ compile-%: CF_PREFIX = compile- compile-%: CONTAINER_ATTACHED = true compile-%: --act ; +.PHONY: dependencies-% +compile-%: JOB=dependencies +compile-%: CF_PREFIX = dependencies- +compile-%: CONTAINER_ATTACHED = true +compile-%: --act ; + .PHONY: test-% test-%: JOB=test test-%: CF_PREFIX = test- @@ -232,6 +238,9 @@ endif # HELP(compile-backend): compile backend binaries --compile-backend: stack build --fast --profile --library-profiling --lock-file ignore --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin +# HELP(dependencies-backend): (re-)build backend dependencies +--dependencies-backend: stack.yaml stack.yaml.lock package.yaml + stack build --fast --only-dependencies # HELP(lint-backend): lint backend --lint-backend: stack build --test --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) uniworx:test:hlint diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index ac56909b4..d8e0b9003 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -20,7 +20,7 @@ ENV HOME="${PROJECT_DIR}" RUN mkdir -p "${PROJECT_DIR}/.stack" ENV STACK_ROOT="${PROJECT_DIR}/.stack" -RUN make -- --.stack STACK_ROOT=${STACK_ROOT} IN_CONTAINER=true +RUN make -- --dependencies-backend STACK_ROOT=${STACK_ROOT} IN_CONTAINER=true RUN stack --work-dir .stack-work-run install yesod-bin RUN chmod -R 777 .stack .stack-work-build .stack-work-run .stack-work-test .stack-work-doc || : From 6ab3983e804f7f59654ae4a424d065778b4206a4 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Wed, 6 Nov 2024 16:08:04 +0100 Subject: [PATCH 32/64] build(Makefile): add frontend dependencies target --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index eca2652db..88da4e5c9 100644 --- a/Makefile +++ b/Makefile @@ -251,6 +251,7 @@ endif # HELP(compile-frontend): compile frontend assets --compile-frontend: node_modules assets esbuild.config.mjs npm run build +--dependencies-frontend: node_modules assets static well-known; node_modules: package.json package-lock.json npm ci --cache .npm --prefer-offline package-lock.json: package.json From 5e49b73a0a4850e437ea54bd7c08b19ad47d7c4e Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Wed, 6 Nov 2024 16:09:16 +0100 Subject: [PATCH 33/64] build(Makefile): fix dependencies targets --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 88da4e5c9..f87225257 100644 --- a/Makefile +++ b/Makefile @@ -147,10 +147,10 @@ compile-%: CONTAINER_ATTACHED = true compile-%: --act ; .PHONY: dependencies-% -compile-%: JOB=dependencies -compile-%: CF_PREFIX = dependencies- -compile-%: CONTAINER_ATTACHED = true -compile-%: --act ; +dependencies-%: JOB=dependencies +dependencies-%: CF_PREFIX = dependencies- +dependencies-%: CONTAINER_ATTACHED = true +dependencies-%: --act ; .PHONY: test-% test-%: JOB=test From 125b999dd0d5760db15b7b881b69383b670ccbeb Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Wed, 6 Nov 2024 16:27:44 +0100 Subject: [PATCH 34/64] build(Makefile): restrict stack builds to half the number of local cpu cores --- Makefile | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index f87225257..6d198631d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ export SHELL=bash +# System information +export CPU_CORES = $(shell cat /proc/cpuinfo | grep '^processor' | wc -l) + export CONTAINER_COMMAND ?= podman export CONTAINER_BGRUN ?= $(CONTAINER_COMMAND) run -dit --network=host --replace export CONTAINER_FGRUN ?= $(CONTAINER_COMMAND) run -it --network=host --replace @@ -27,6 +30,7 @@ export CONTAINER_CMD export ENTRYPOINT export EXEC_OPTS +export STACK_CORES = $(shell echo $(($(CPU_CORES)/2))) export BASE_PORTS export UNIWORXDB_OPTS ?= -cf export PROD ?= false @@ -109,7 +113,7 @@ start: %-hoogle: CONTAINER_CMD=localhost/fradrive/backend --start-hoogle: HOOGLE_PORT=`cat $(CONTAINER_FILE) | grep 'HOOGLE_PORT=' | sed 's/HOOGLE_PORT=//'` ; \ - stack --work-dir=.stack-work-doc hoogle -- server --local --port $${HOOGLE_PORT} + stack --work-dir=.stack-work-doc $(STACK_CORES) hoogle -- server --local --port $${HOOGLE_PORT} .PHONY: %-frontend %-frontend: SERVICE=frontend @@ -234,19 +238,19 @@ endif --start-backend: DEV_PORT_HTTP=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTP=' | sed 's/DEV_PORT_HTTP=//'`; \ DEV_PORT_HTTPS=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTPS=' | sed 's/DEV_PORT_HTTPS=//'`; \ - stack --work-dir=.stack-work-run --lock-file ignore exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" + stack --work-dir=.stack-work-run $(STACK_CORES) --lock-file ignore exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" # HELP(compile-backend): compile backend binaries --compile-backend: - stack build --fast --profile --library-profiling --lock-file ignore --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin + stack build $(STACK_CORES) --fast --profile --library-profiling --lock-file ignore --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin # HELP(dependencies-backend): (re-)build backend dependencies --dependencies-backend: stack.yaml stack.yaml.lock package.yaml - stack build --fast --only-dependencies + stack build $(STACK_CORES) --fast --only-dependencies # HELP(lint-backend): lint backend --lint-backend: - stack build --test --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) uniworx:test:hlint + stack build $(STACK_CORES) --test --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) uniworx:test:hlint # HELP(test-backend): test backend --test-backend: - stack build --test --coverage --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) + stack build $(STACK_CORES) --test --coverage --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) # HELP(compile-frontend): compile frontend assets --compile-frontend: node_modules assets esbuild.config.mjs From 25d1d5d14ca80c8e7885a9a3ac43aefd3d04ca45 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 7 Nov 2024 00:36:20 +0100 Subject: [PATCH 35/64] build(Makefile): do not warn/error on non-existing current-dev --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6d198631d..ef232a4c6 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ endif export DATE := $(shell date +'%Y-%m-%dT%H-%M-%S') -export CURR_DEV = $(shell cat develop/.current) +export CURR_DEV = $(shell cat develop/.current 2>/dev/null) export SET_DEVELOP = $(eval DEVELOP=develop/$$(CURR_DEV)) export NEW_DEVELOP = $(eval DEVELOP=develop/$$(DATE)) From 402a710fbf8487363dd38c169c1fc161b886e238 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 7 Nov 2024 00:36:43 +0100 Subject: [PATCH 36/64] build(Makefile): build stack deps on missing stack.yaml.lock --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index ef232a4c6..25a332c95 100644 --- a/Makefile +++ b/Makefile @@ -362,5 +362,7 @@ switch-develop: develop/.current: ls -1 develop | tail -n1 > develop/.current +stack.yaml.lock: --dependencies-backend; + .PHONY: --% .SUFFIXES: # Delete all default suffixes From fba71e5e384bb20cc8f7680a67d0d859f7a88934 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 7 Nov 2024 00:56:06 +0100 Subject: [PATCH 37/64] build(Makefile): rewrite log-% documentation --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 25a332c95..7b669c535 100644 --- a/Makefile +++ b/Makefile @@ -285,9 +285,8 @@ well-known: static; status: @./utils/develop-status.pl -a -# TODO: rework logs, because they do not work .PHONY: log-% -# HELP(log-{database,memcached,minio,backend,frontend,hoogle}): inspect output of a given (currently running) service. When a service supports multiple running instances in one develop (i.e. backend), you need to specify the exact instance by its associated file (e.g. backend-1, backend-2, etc.), please check the contents of the develop/ directory for a list of running instances. +# HELP(log-$(JOB)-$(SERVICE)): inspect output of a given service. The service must be currently running When a service supports multiple running instances in one develop (i.e. backend), you need to specify the exact instance by its associated file (e.g. backend-1, backend-2, etc.), please check the contents of the develop/ directory for a list of running instances. log-%: DEVELOP=develop/`cat develop/.current` ; \ SEARCH_FILE="$${DEVELOP}/$*" ; \ From 0af2e452ef6b18801cd907f5dce4007cb7084445 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 7 Nov 2024 01:27:45 +0100 Subject: [PATCH 38/64] build(Makefile): make container name suffix identical to container file; remove some debug echoes --- Makefile | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 7b669c535..223c450c0 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,7 @@ export MAILDEV_IMAGE = $(IMAGE_REGISTRY)/maildev/maildev:latest # TODO: needs di export IN_CONTAINER ?= false export IN_CI ?= false export CONTAINER_FILE +export CONTAINER_IDENT export CF_PREFIX export DEVELOP export MOUNT_DIR=/mnt/fradrive @@ -25,7 +26,6 @@ export SERVICE export SERVICE_VARIANT ?= $(SERVICE) export SERVICE_PARALLEL ?= false export JOB -export JOB_ID export CONTAINER_CMD export ENTRYPOINT export EXEC_OPTS @@ -176,23 +176,19 @@ lint-%: --act ; DEVELOP=develop/`cat develop/.current` ; \ echo "$(SERVICE_PARALLEL)" ; \ if [[ "$(SERVICE_PARALLEL)" == "false" ]] ; then \ - CONTAINER_FILE=$${DEVELOP}/$(CF_PREFIX)$(SERVICE_VARIANT) ; \ - JOB_ID=$(JOB) ; \ - if [[ -e $${CONTAINER_FILE} ]]; then \ + CONTAINER_IDENT=$(CF_PREFIX)$(SERVICE_VARIANT) ; \ + if [[ -e $${DEVELOP}/$${CONTAINER_IDENT} ]]; then \ >&2 echo "Another $* service is already running! Use \"make new-develop\" to start a new develop instance despite currently running services." ; \ exit 1 ; \ fi ; \ else \ - DEVDIR=$${DEVELOP}/$(CF_PREFIX)$(SERVICE_VARIANT) ; \ I=`ls $${DEVELOP} | grep '$(SERVICE_VARIANT)' | sed 's/$(SERVICE_VARIANT)-//' | sort -n | tail -n1` ; \ - echo "I=$${I}" ; \ J=$$(($${I}+1)) ; \ - echo "J=$${J}" ; \ - CONTAINER_FILE=$${DEVDIR}-$${J} ; \ - JOB_ID=$(JOB)-$${J} ; \ + CONTAINER_IDENT=$(CF_PREFIX)$(SERVICE_VARIANT)-$${J} ; \ fi ; \ + CONTAINER_FILE=$${DEVELOP}/$${CONTAINER_IDENT} ; \ echo "$(PORTS)" | sed 's/ /\n/g' > $${CONTAINER_FILE} ; \ - $(MAKE) -- --$* CONTAINER_FILE=$${CONTAINER_FILE} JOB_ID=$${JOB_ID} + $(MAKE) -- --$* CONTAINER_FILE=$${CONTAINER_FILE} CONTAINER_IDENT=$${CONTAINER_IDENT} .PHONY: image-rebuild_% # HELP(image-rebuild_{backend,frontend,database,memcached,minio}): force-rebuild the stated docker image @@ -215,10 +211,9 @@ else endif --containerized: --image-build - echo "--containerized ################## ATTACHED: $(CONTAINER_ATTACHED)" DEVELOP=`cat develop/.current` ; \ ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ - CONTAINER_NAME=fradrive.$(CURR_DEV).$(SERVICE_VARIANT).$(JOB_ID) ; \ + CONTAINER_NAME=fradrive.$(CURR_DEV).$(CONTAINER_IDENT) ; \ CONTAINER_ID=`$(CONTAINER_BGRUN) \ -v $(PWD):$(MOUNT_DIR) \ --env IN_CONTAINER=true \ @@ -230,7 +225,6 @@ endif ` ; \ printf "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)" ; \ if [[ "true" == "$(CONTAINER_ATTACHED)" ]] ; then \ - echo "###!!! postgres shall not go here!" ; \ $(CONTAINER_COMMAND) attach $${CONTAINER_ID} || : ; \ fi From 6e721a55cfe19f6d90644721ebd17998ede409e7 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 7 Nov 2024 02:22:45 +0100 Subject: [PATCH 39/64] build(Makefile): fix CONTAINER_CMD quoting --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 223c450c0..49bca8064 100644 --- a/Makefile +++ b/Makefile @@ -129,13 +129,13 @@ start: .PHONY: %-memcached %-memcached: SERVICE=memcached %-memcached: SERVICE_VARIANT=memcached -%-memcached: CONTAINER_CMD=$(MEMCACHED_IMAGE) --port=`cat $(CONTAINER_FILE) | grep 'MEMCACHED_PORT=' | sed 's/MEMCACHED_PORT=//'` +%-memcached: CONTAINER_CMD=$$(MEMCACHED_IMAGE) --port=`cat $$(CONTAINER_FILE) | grep 'MEMCACHED_PORT=' | sed 's/MEMCACHED_PORT=//'` %-memcached: BASE_PORTS = "MEMCACHED_PORT=11211" .PHONY: %-minio %-minio: SERVICE=minio %-minio: SERVICE_VARIANT=minio -%-minio: CONTAINER_CMD=$(MINIO_IMAGE) -- server `mktemp` --address=:`cat $(CONTAINER_FILE) | grep 'UPLOAD_S3_PORT=' | sed 's/UPLOAD_S3_PORT=//'`: +%-minio: CONTAINER_CMD=$$(MINIO_IMAGE) -- server `mktemp` --address=:`cat $$(CONTAINER_FILE) | grep 'UPLOAD_S3_PORT=' | sed 's/UPLOAD_S3_PORT=//'` %-minio: BASE_PORTS = "UPLOAD_S3_PORT=9000" .PHONY: start-% From 191afbd2c87b2f2188b590bd8a890f70ac80aa19 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 7 Nov 2024 02:23:12 +0100 Subject: [PATCH 40/64] build(Makefile): remove some more debug echoes --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 49bca8064..718063eef 100644 --- a/Makefile +++ b/Makefile @@ -172,9 +172,7 @@ lint-%: --act ; --develop_%: PORTS = $(foreach PORT,$(BASE_PORTS),$(shell utils/next_free_port.pl $(PORT))) --develop_%: --ensure-develop - echo "--develop ################## ATTACHED: $(CONTAINER_ATTACHED)" DEVELOP=develop/`cat develop/.current` ; \ - echo "$(SERVICE_PARALLEL)" ; \ if [[ "$(SERVICE_PARALLEL)" == "false" ]] ; then \ CONTAINER_IDENT=$(CF_PREFIX)$(SERVICE_VARIANT) ; \ if [[ -e $${DEVELOP}/$${CONTAINER_IDENT} ]]; then \ From 88e984d4e53485f04bafafdf8a843a41ab73f101 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 7 Nov 2024 02:25:58 +0100 Subject: [PATCH 41/64] build(Makefile): use SET_CONTAINER_CMD to splice CONTAINER_CMD inside podman run --- Makefile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 718063eef..4e4f0361b 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ export SERVICE_VARIANT ?= $(SERVICE) export SERVICE_PARALLEL ?= false export JOB export CONTAINER_CMD +export SET_CONTAINER_CMD export ENTRYPOINT export EXEC_OPTS @@ -129,13 +130,13 @@ start: .PHONY: %-memcached %-memcached: SERVICE=memcached %-memcached: SERVICE_VARIANT=memcached -%-memcached: CONTAINER_CMD=$$(MEMCACHED_IMAGE) --port=`cat $$(CONTAINER_FILE) | grep 'MEMCACHED_PORT=' | sed 's/MEMCACHED_PORT=//'` +%-memcached: SET_CONTAINER_CMD=$$(MEMCACHED_IMAGE) --port=`cat $$(CONTAINER_FILE) | grep 'MEMCACHED_PORT=' | sed 's/MEMCACHED_PORT=//'` %-memcached: BASE_PORTS = "MEMCACHED_PORT=11211" .PHONY: %-minio %-minio: SERVICE=minio %-minio: SERVICE_VARIANT=minio -%-minio: CONTAINER_CMD=$$(MINIO_IMAGE) -- server `mktemp` --address=:`cat $$(CONTAINER_FILE) | grep 'UPLOAD_S3_PORT=' | sed 's/UPLOAD_S3_PORT=//'` +%-minio: SET_CONTAINER_CMD=$$(MINIO_IMAGE) -- server `mktemp` --address=:`cat $$(CONTAINER_FILE) | grep 'UPLOAD_S3_PORT=' | sed 's/UPLOAD_S3_PORT=//'` %-minio: BASE_PORTS = "UPLOAD_S3_PORT=9000" .PHONY: start-% @@ -212,6 +213,12 @@ endif DEVELOP=`cat develop/.current` ; \ ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ CONTAINER_NAME=fradrive.$(CURR_DEV).$(CONTAINER_IDENT) ; \ + if ! [ -z "$(SET_CONTAINER_CMD)" ] ; \ + then \ + CONTAINER_CMD="$(SET_CONTAINER_CMD)" ; \ + else \ + CONTAINER_CMD=$(CONTAINER_CMD) ; \ + fi ; \ CONTAINER_ID=`$(CONTAINER_BGRUN) \ -v $(PWD):$(MOUNT_DIR) \ --env IN_CONTAINER=true \ @@ -219,7 +226,7 @@ endif --env CONTAINER_FILE=$(CONTAINER_FILE) \ --env CONTAINER_NAME=$${CONTAINER_NAME} \ --name $${CONTAINER_NAME} \ - $(CONTAINER_CMD) \ + $${CONTAINER_CMD} \ ` ; \ printf "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)" ; \ if [[ "true" == "$(CONTAINER_ATTACHED)" ]] ; then \ From 6cd3cb88af9fb1390526e150b13e67bf36a47c7c Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 7 Nov 2024 16:47:12 +0100 Subject: [PATCH 42/64] build(Makefile): add start-frontend target --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 4e4f0361b..36c7c78b4 100644 --- a/Makefile +++ b/Makefile @@ -254,6 +254,7 @@ endif # HELP(compile-frontend): compile frontend assets --compile-frontend: node_modules assets esbuild.config.mjs npm run build +--start-frontend: --compile-frontend; --dependencies-frontend: node_modules assets static well-known; node_modules: package.json package-lock.json npm ci --cache .npm --prefer-offline From 365b7024c72178b5c552c7c5d2fc5330c85acd8c Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 7 Nov 2024 16:48:40 +0100 Subject: [PATCH 43/64] build(Makefile): remove .stack-work differentiation --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 36c7c78b4..b23b94144 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ start: %-hoogle: CONTAINER_CMD=localhost/fradrive/backend --start-hoogle: HOOGLE_PORT=`cat $(CONTAINER_FILE) | grep 'HOOGLE_PORT=' | sed 's/HOOGLE_PORT=//'` ; \ - stack --work-dir=.stack-work-doc $(STACK_CORES) hoogle -- server --local --port $${HOOGLE_PORT} + stack $(STACK_CORES) hoogle -- server --local --port $${HOOGLE_PORT} .PHONY: %-frontend %-frontend: SERVICE=frontend @@ -237,7 +237,7 @@ endif --start-backend: DEV_PORT_HTTP=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTP=' | sed 's/DEV_PORT_HTTP=//'`; \ DEV_PORT_HTTPS=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTPS=' | sed 's/DEV_PORT_HTTPS=//'`; \ - stack --work-dir=.stack-work-run $(STACK_CORES) --lock-file ignore exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" + stack $(STACK_CORES) --lock-file ignore exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" # HELP(compile-backend): compile backend binaries --compile-backend: stack build $(STACK_CORES) --fast --profile --library-profiling --lock-file ignore --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin @@ -246,10 +246,10 @@ endif stack build $(STACK_CORES) --fast --only-dependencies # HELP(lint-backend): lint backend --lint-backend: - stack build $(STACK_CORES) --test --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) uniworx:test:hlint + stack build $(STACK_CORES) --test --fast --flag uniworx:library-only $(--DEVELOPMENT) uniworx:test:hlint # HELP(test-backend): test backend --test-backend: - stack build $(STACK_CORES) --test --coverage --fast --work-dir=.stack-work-test --flag uniworx:library-only $(--DEVELOPMENT) + stack build $(STACK_CORES) --test --coverage --fast --flag uniworx:library-only $(--DEVELOPMENT) # HELP(compile-frontend): compile frontend assets --compile-frontend: node_modules assets esbuild.config.mjs @@ -273,7 +273,7 @@ well-known: static; # HELP(uniworxdb): clear and fill database. requires running postgres # TODO (db-m-$MIGRATION-backend): apply migration (see src/Model/Migration/Definition.hs for list of available migrations) ---start-uniworxdb: +--start-uniworxdb: --compile-backend SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true} ; \ AVSPASS=${AVSPASS:-nopasswordset} ; \ stack $(--DEVELOPMENT) exec uniworxdb -- $(UNIWORXDB_OPTS) From 8bc133f0edd4e8a63cb0cf262369cc003f202a28 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 7 Nov 2024 16:53:48 +0100 Subject: [PATCH 44/64] build(Makefile): remove parallelization option of services --- Makefile | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index b23b94144..b8c047f07 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,6 @@ export CONTAINER_ATTACHED export SERVICE export SERVICE_VARIANT ?= $(SERVICE) -export SERVICE_PARALLEL ?= false export JOB export CONTAINER_CMD export SET_CONTAINER_CMD @@ -98,7 +97,6 @@ start: .PHONY: %-backend %-backend: SERVICE=backend %-backend: SERVICE_VARIANT=backend -%-backend: SERVICE_PARALLEL=true %-backend: CONTAINER_CMD=localhost/fradrive/backend %-backend: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443" @@ -174,18 +172,12 @@ lint-%: --act ; --develop_%: PORTS = $(foreach PORT,$(BASE_PORTS),$(shell utils/next_free_port.pl $(PORT))) --develop_%: --ensure-develop DEVELOP=develop/`cat develop/.current` ; \ - if [[ "$(SERVICE_PARALLEL)" == "false" ]] ; then \ - CONTAINER_IDENT=$(CF_PREFIX)$(SERVICE_VARIANT) ; \ - if [[ -e $${DEVELOP}/$${CONTAINER_IDENT} ]]; then \ - >&2 echo "Another $* service is already running! Use \"make new-develop\" to start a new develop instance despite currently running services." ; \ - exit 1 ; \ - fi ; \ - else \ - I=`ls $${DEVELOP} | grep '$(SERVICE_VARIANT)' | sed 's/$(SERVICE_VARIANT)-//' | sort -n | tail -n1` ; \ - J=$$(($${I}+1)) ; \ - CONTAINER_IDENT=$(CF_PREFIX)$(SERVICE_VARIANT)-$${J} ; \ - fi ; \ + CONTAINER_IDENT=$(CF_PREFIX)$(SERVICE_VARIANT) ; \ CONTAINER_FILE=$${DEVELOP}/$${CONTAINER_IDENT} ; \ + if [[ -e $${CONTAINER_FILE} ]]; then \ + >&2 echo "Another $* service is already running! Use \"make new-develop\" to start a new develop instance despite currently running services." ; \ + exit 1 ; \ + fi ; \ echo "$(PORTS)" | sed 's/ /\n/g' > $${CONTAINER_FILE} ; \ $(MAKE) -- --$* CONTAINER_FILE=$${CONTAINER_FILE} CONTAINER_IDENT=$${CONTAINER_IDENT} From 54e55fc36b7d14b57d3ebf61fd5e8f7f54910048 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 7 Nov 2024 16:59:19 +0100 Subject: [PATCH 45/64] build(Makefile): stop ignoring stack lock files --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b8c047f07..37efc318b 100644 --- a/Makefile +++ b/Makefile @@ -229,10 +229,10 @@ endif --start-backend: DEV_PORT_HTTP=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTP=' | sed 's/DEV_PORT_HTTP=//'`; \ DEV_PORT_HTTPS=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTPS=' | sed 's/DEV_PORT_HTTPS=//'`; \ - stack $(STACK_CORES) --lock-file ignore exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" + stack $(STACK_CORES) exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" # HELP(compile-backend): compile backend binaries --compile-backend: - stack build $(STACK_CORES) --fast --profile --library-profiling --lock-file ignore --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin + stack build $(STACK_CORES) --fast --profile --library-profiling --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin # HELP(dependencies-backend): (re-)build backend dependencies --dependencies-backend: stack.yaml stack.yaml.lock package.yaml stack build $(STACK_CORES) --fast --only-dependencies From cf0bd0211a6b323603c2189079b762f3d9d4fe3f Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Thu, 7 Nov 2024 17:03:25 +0100 Subject: [PATCH 46/64] build(utils/makehelp.pl): Introduced awareness for Make variables. --- utils/makehelp.pl | 141 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 133 insertions(+), 8 deletions(-) diff --git a/utils/makehelp.pl b/utils/makehelp.pl index aa742a7ea..15ef1a2b1 100755 --- a/utils/makehelp.pl +++ b/utils/makehelp.pl @@ -4,22 +4,35 @@ use strict; use warnings; my %msg = (); -my @start = (); my %reorder = (help=>-1); +# We do not want every Makefile syntax error to break help completely, +# so we just disable the kind of features where the syntax error occurs +# In that case we store also the error message here +my %brokenFeatures = (); + +my @blocks = (); +my @plainCode = (); + READ: while(<>) { + my $lnum = $.; if(m/^# HELP HEADER START/) { + my @start = (); while(<>) { - next READ if m/^# HELP HEADER END/; + do { + push @blocks, { kind=>'start', text=>\@start, line=>$lnum }; + next READ + } if m/^# HELP HEADER END/; if(m/# (.*)/) { push @start, $1 } else { die "$0: Invalid HELP HEADER section in Makefile!\n"; } } + push @blocks, { kind=>'start', text=>\@start, line=>$lnum }; next READ } - if(m/# HELP((?:\([^\(\)]+\))?):\s*(.*)/) { + if(m/^# HELP((?:\((?:[^\(\)]|\([^\(\)]+\))+\))?):\s*(.*)/) { my ($target, $message) = ($1, $2); if($target=~m/\((.*)\)/) { $target = $1; @@ -29,13 +42,125 @@ READ: while(<>) { $line=~m/^([^:]+):/ or die "$0: HELP marker expects target but no target found!\n"; $target = $1 } - $msg{$target} .= $message + push @blocks, { kind=>'target', target=>$target, message=>$message, line=>$lnum }; + next READ + } + if(m/^# HELPVAR\((.*)\)/) { + push @blocks, { kind=>'helpvar', content=>$1, line=>$lnum }; + next READ; + } + push @plainCode, [$lnum, $_] +} + +my %kindBlocks = (); +for(@blocks) { + push @{$kindBlocks{$_->{kind}}}, $_ +} + +my @matchies = (); +HELPVARS: { + for my $hv(@{$kindBlocks{helpvar}}) { + my $reVar = qr((?:\$\((?[A-Z]+)(?:,(?[^\(\)]+))?\))); + my $reVarNoGroups = qr((?:\$\((?:[A-Z]+)(?:,(?:[^\(\)]+))?\))); + my @parts = split /($reVarNoGroups)/, $hv->{content}; + my %setvars = (); + my @reParts = (); + for my $p(@parts) { + if($p=~m#^$reVar$#) { + my $name = $+{name}; + my $re = $+{re}; + if(defined $re) { + if(exists $setvars{$name}) { + $brokenFeatures{helpvars} = "Makefile line $hv->{line}: Multiple definitions of variable $name found!"; + last HELPVARS + } + $setvars{$name} = $re; + push @reParts, [1,qr((?<$name>$re))]; + } else { + if(not exists $setvars{$name}) { + $brokenFeatures{helpvars} = "Makefile line $hv->{line}: Variable $name used without definition!"; + last HELPVARS + } + push @reParts, [0,$name]; #qr((?:\k<$name>)); -- problem: cannot reference variable when not defined + } + } else { + push @reParts, [1,qr((?:\Q$p\E))]; + } + } + my $reCol = qr((?:)); + for my $r(@reParts) { + my ($isRe, $reg) = @$r; + if($isRe) { + $reCol = qr((?:$reCol$reg)); + } else { + eval { + $reCol = qr((?:$reCol(?:\k<$reg>))); + }; + if($@) { + $brokenFeatures{helpvars} = "Makefile line $hv->{line}: Variable $reg caused an internal error ($@)!"; + last HELPVARS + } + } + } + push @matchies, $reCol } } -print "$_\n" for @start; -print "\n" if @start; -for my $tar(sort {($reorder{$a}||0) <=> ($reorder{$b}||0) || $a cmp $b } keys %msg) { - print "$tar\n $msg{$tar}\n\n" +my %helpvars = (); + +for my $rowr(@plainCode) { + my ($lnum, $row) = @$rowr; + my @matches = (); + for my $re(@matchies) { + if($row=~m#^$re$#) { + my %h = %+; + push @matches, \%h + } + } + if(@matches>1) { + $brokenFeatures{helpvars} = "Makefile line $lnum: Multiple variable definitions match on row '$row'\n"; + } + if(1==@matches) { + my %h = %{$matches[0]}; + for my $k(keys %h) { + push @{$helpvars{$k}}, $h{$k} + } + } +} + +my @start = (); +@start = @{$kindBlocks{start}} if exists $kindBlocks{start}; + +for my $tg(@{$kindBlocks{target}}) { + my $tgv = $tg->{target}; + my @build = (); + $tgv=~s#\$\(([A-Z]+)(?:,[^\(\)]*)?\)#push @build, {name=>$1,elm=>$helpvars{$1}}; $1#ge; + unshift @build, $tg->{message}; + $msg{$tgv} = \@build; +} + +sub msgprinter { + my $el = shift; + if('HASH' eq ref $el) { + print " Where $el->{name} is one of\n"; + print " $_\n" for @{$el->{elm}}; + return + } + $el =~ s#(.{40,76})\s+#$1\n #g; + print " $el\n"; +} + +for(@start) { + print "$_\n" for @{$_->{text}} +} +print "\n" if @start; +for my $tar(sort {($reorder{$a}||0) <=> ($reorder{$b}||0) || $a cmp $b } keys %msg) { + print "$tar\n"; + msgprinter($_) for @{$msg{$tar}}; + print "\n"; +} + +for my $k(keys %brokenFeatures) { + warn "\n!!!!!\nFailed to complete construct the help message from the Makefile.\nFeature '$k' disabled, because: $brokenFeatures{$k}\n\n"; } From c8cb73d366de98792d6a6cccecfaeeb9141766f8 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 7 Nov 2024 17:42:54 +0100 Subject: [PATCH 47/64] build(Makefile): CONTAINER_INIT and CONTAINER_CLEANUP for stack-work dir symlinking --- Makefile | 13 ++++++++++++- utils/stack-work-init.sh | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100755 utils/stack-work-init.sh diff --git a/Makefile b/Makefile index 37efc318b..402ddf433 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,11 @@ export CF_PREFIX export DEVELOP export MOUNT_DIR=/mnt/fradrive export CONTAINER_ATTACHED +export CONTAINER_INIT +export CONTAINER_CLEANUP + +export STACK_INIT="./utils/stack-work-init.sh" +export STACK_CLEANUP="rm -f .stack-work.lock" export SERVICE export SERVICE_VARIANT ?= $(SERVICE) @@ -98,18 +103,24 @@ start: %-backend: SERVICE=backend %-backend: SERVICE_VARIANT=backend %-backend: CONTAINER_CMD=localhost/fradrive/backend +%-backend: CONTAINER_INIT="$(STACK_INIT)" +%-backend: CONTAINER_CLEANUP="$(STACK_CLEANUP)" %-backend: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443" .PHONY: %-uniworxdb %-uniworxdb: SERVICE=backend %-uniworxdb: SERVICE_VARIANT=uniworxdb %-uniworxdb: CONTAINER_CMD=localhost/fradrive/backend +%-uniworxdb: CONTAINER_INIT="$(STACK_INIT)" +%-uniworxdb: CONTAINER_CLEANUP="$(STACK_CLEANUP)" .PHONY: %-hoogle %-hoogle: SERVICE=backend %-hoogle: SERVICE_VARIANT=hoogle %-hoogle: BASE_PORTS = "HOOGLE_PORT=8081" %-hoogle: CONTAINER_CMD=localhost/fradrive/backend +%-hoogle: CONTAINER_INIT="$(STACK_INIT)" +%-hoogle: CONTAINER_CLEANUP="$(STACK_CLEANUP)" --start-hoogle: HOOGLE_PORT=`cat $(CONTAINER_FILE) | grep 'HOOGLE_PORT=' | sed 's/HOOGLE_PORT=//'` ; \ stack $(STACK_CORES) hoogle -- server --local --port $${HOOGLE_PORT} @@ -203,7 +214,7 @@ endif --containerized: --image-build DEVELOP=`cat develop/.current` ; \ - ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ + ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" "$(CONTAINER_INIT)" "$(CONTAINER_CLEANUP)" & \ CONTAINER_NAME=fradrive.$(CURR_DEV).$(CONTAINER_IDENT) ; \ if ! [ -z "$(SET_CONTAINER_CMD)" ] ; \ then \ diff --git a/utils/stack-work-init.sh b/utils/stack-work-init.sh new file mode 100755 index 000000000..f88fabad1 --- /dev/null +++ b/utils/stack-work-init.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +STACK_WORK_SRC=.stack-work-${JOB} + +if [[ -e .stack-work.lock ]]; then + echo ".stack-work directory is locked (lock file exists); is another stack currently running?" + exit 1 +fi + +touch .stack-work.lock +rm -rf .stack-work + +if [[ ! -e "$STACK_WORK_SRC" ]]; then + mkdir "$STACK_WORK_SRC" + chmod 777 "$STACK_WORK_SRC" +fi + +ln -s "$STACK_WORK_SRC" .stack-work \ No newline at end of file From 70fa1961ca6f9cdcb13a710dc127dde66e11d473 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 7 Nov 2024 17:44:07 +0100 Subject: [PATCH 48/64] chore: gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index eec19fc73..c6e28b106 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ bin/ cabal-dev/ .cache/ .stack/ -.stack-work/ +.stack-work .dev-port-http .dev-port-https .bash_history From ef69845fc4eaab301657cd70f1ea1cd38a1b1e64 Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Thu, 7 Nov 2024 18:52:50 +0100 Subject: [PATCH 49/64] build(utils/dockerizedmake.pl): Helper tool to symlink a whole directory content. --- utils/dirsymlink.pl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 utils/dirsymlink.pl diff --git a/utils/dirsymlink.pl b/utils/dirsymlink.pl new file mode 100755 index 000000000..c7f4dd22b --- /dev/null +++ b/utils/dirsymlink.pl @@ -0,0 +1,19 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +my ($shadowdir, @keepdirs) = @ARGV; + +mkdir($shadowdir); + +system(qq#rm -f "$shadowdir"/*#); + +for my $k(@keepdirs) { + unlink("$shadowdir/$k"); + mkdir("$shadowdir/$k"); +} + +system(qq#ln -rs .* * "$shadowdir"#); + + From 02877e89b5fef74a179ae125bf0246d52e172b9f Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Fri, 8 Nov 2024 02:15:46 +0100 Subject: [PATCH 50/64] build(utils/makefilelogger.pl): Debug tools for Makefiles. --- utils/makefilelogger.pl | 53 +++++++++++++++++++++++++++++++++++++++++ utils/wrapmakelog.sh | 8 +++++++ 2 files changed, 61 insertions(+) create mode 100755 utils/makefilelogger.pl create mode 100755 utils/wrapmakelog.sh diff --git a/utils/makefilelogger.pl b/utils/makefilelogger.pl new file mode 100755 index 000000000..7054187a5 --- /dev/null +++ b/utils/makefilelogger.pl @@ -0,0 +1,53 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +my $fh = undef; +open($fh, '<', $ARGV[0]) or die "Could not read '$ARGV[0]', because: $!"; +my @cont = <$fh>; + +my %exports = (); +for(@cont) { + m#^export\s+([^=\s]+)# and $exports{$1} = 1 +} +my @exports = sort keys %exports; + +my @lined = (); +for(0..$#cont) { + $lined[$_] = [$_, $cont[$_]] +} +@lined = grep { $_->[1]!~m&^\s*#& } @lined; + +for my $i(0..$#lined-1) { + my $this = $lined[$i][1]; + my $next = $lined[$i+1][1]; + if($this=~m#^([^\s:]+):#) { + my $targetPattern = $1; + if($next=~m#^\t#) { + my $cl = $lined[$i]; + push @$cl, qq#\t\@echo "== PATTERN $targetPattern TARGET \$@" >> makelogger\n#; + push @$cl, qq#\t@ echo " \\\$\$($_) = '\$($_)'" >> makelogger\n# for @exports; + push @$cl, qq#\t@ echo "" >> makelogger\n#; + } + } +} + +my %out = (); +for my $i(0..$#cont) { + $out{$i} = $cont[$i] +} + +for(@lined) { + $out{$_->[0]} = $_ +} + +for my $i(0..$#cont) { + my $tc = $out{$i}; + if(ref $tc) { + print $tc->[$_] for 1..$#$tc + } else { + print $tc; + } +} + diff --git a/utils/wrapmakelog.sh b/utils/wrapmakelog.sh new file mode 100755 index 000000000..2fe1bbc7d --- /dev/null +++ b/utils/wrapmakelog.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +: > makelogger + +utils/makefilelogger.pl Makefile > Makefile-loggingsymbols + +make -f Makefile-loggingsymbols "$@" + From b5e7b970e309a026c5c8a3d07af7a857a2e0a4d8 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Fri, 8 Nov 2024 02:30:30 +0100 Subject: [PATCH 51/64] build(Makefile): make uniworxdb a compile- target instead of start- --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 402ddf433..6b356381f 100644 --- a/Makefile +++ b/Makefile @@ -274,9 +274,9 @@ static: node_modules assets esbuild.config.mjs npm run build well-known: static; -# HELP(uniworxdb): clear and fill database. requires running postgres +# HELP(compile-uniworxdb): clear and fill database. requires running postgres # TODO (db-m-$MIGRATION-backend): apply migration (see src/Model/Migration/Definition.hs for list of available migrations) ---start-uniworxdb: --compile-backend +--compile-uniworxdb: --compile-backend SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true} ; \ AVSPASS=${AVSPASS:-nopasswordset} ; \ stack $(--DEVELOPMENT) exec uniworxdb -- $(UNIWORXDB_OPTS) From 2691925557d451118af41070754bf38d635b3f88 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Fri, 8 Nov 2024 02:30:47 +0100 Subject: [PATCH 52/64] build(Makefile): cleanup new stack-work dirs --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6b356381f..aece0bf18 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ help: clean: rm -rf develop -rm -rf node_modules .npm .cache assets/icons assets/favicons static well-known config/manifest.json - -rm -rf .stack-work .stack-work-run .stack-work-test .stack-work-doc .stack-work.lock + -rm -rf .stack-work .stack-work.lock .stack-work-compile .stack-work-start .stack-work-test .stack-work-lint .stack-work-hoogle -rm -rf bin .Dockerfile develop -$(CONTAINER_COMMAND) container prune --force .PHONY: clean-all From edf850cb5034b8c88bf47010be2c4b006596f144 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Sun, 10 Nov 2024 01:39:27 +0100 Subject: [PATCH 53/64] build(Makefile): remove DEVELOPMENT flag on uniworxdb exec --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index aece0bf18..4773dfc59 100644 --- a/Makefile +++ b/Makefile @@ -279,7 +279,7 @@ well-known: static; --compile-uniworxdb: --compile-backend SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true} ; \ AVSPASS=${AVSPASS:-nopasswordset} ; \ - stack $(--DEVELOPMENT) exec uniworxdb -- $(UNIWORXDB_OPTS) + stack exec uniworxdb -- $(UNIWORXDB_OPTS) # HELP(start-minio): start minio service From 3699d06c5ad57a43bfe701f33ccc769b98c031d8 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Sun, 10 Nov 2024 01:42:43 +0100 Subject: [PATCH 54/64] build(docker/backend): chmod new stack-work dirs --- docker/backend/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index d8e0b9003..4ae02188e 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -21,8 +21,8 @@ RUN mkdir -p "${PROJECT_DIR}/.stack" ENV STACK_ROOT="${PROJECT_DIR}/.stack" RUN make -- --dependencies-backend STACK_ROOT=${STACK_ROOT} IN_CONTAINER=true -RUN stack --work-dir .stack-work-run install yesod-bin -RUN chmod -R 777 .stack .stack-work-build .stack-work-run .stack-work-test .stack-work-doc || : +RUN stack install yesod-bin +RUN chmod -R 777 .stack .stack-work .stack-work-compile .stack-work-start .stack-work-test .stack-work-lint .stack-work-hoogle || : ENV FRADRIVE_MAKE_TARGET=--start-backend ENTRYPOINT make -- ${FRADRIVE_MAKE_TARGET} STACK_ROOT="${STACK_ROOT}" IN_CONTAINER=true CONTAINER_FILE="${CONTAINER_FILE}" From 8d8d78dc81d299db97ba5f0191340a730a4a50ef Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Sun, 10 Nov 2024 01:43:21 +0100 Subject: [PATCH 55/64] build(Makefile): disable buggy custom stack-work locking for now --- utils/stack-work-init.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/stack-work-init.sh b/utils/stack-work-init.sh index f88fabad1..c9628bf3b 100755 --- a/utils/stack-work-init.sh +++ b/utils/stack-work-init.sh @@ -2,10 +2,10 @@ STACK_WORK_SRC=.stack-work-${JOB} -if [[ -e .stack-work.lock ]]; then - echo ".stack-work directory is locked (lock file exists); is another stack currently running?" - exit 1 -fi +#if [[ -e .stack-work.lock ]]; then +# echo ".stack-work directory is locked (lock file exists); is another stack currently running?" +# exit 1 +#fi touch .stack-work.lock rm -rf .stack-work From a2bacf8e1289b7642cb62e699c36d7ac8ed732b9 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Sun, 10 Nov 2024 02:07:36 +0100 Subject: [PATCH 56/64] ci(azure-pipelines): create basic Docker@2 task for building test image --- azure-pipelines.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 139560748..d05ab127c 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -6,7 +6,7 @@ pool: 'Prod Private Agent Pool' jobs: - job: HelloWorld - container: + container: &ubuntu image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' endpoint: devfra steps: @@ -15,4 +15,15 @@ jobs: - script: | echo Add other tasks to build, test, and deploy your project. echo See https://aka.ms/yaml - displayName: 'Run a multi-line script' \ No newline at end of file + displayName: 'Run a multi-line script' +- job: ContainerTest + container: *ubuntu + steps: + - task: Docker@2 + displayName: Image build test + inputs: + command: buildAndPush + Dockerfile: docker/backend/Dockerfile + buildContext: . + tags: backend + \ No newline at end of file From 29edf2207976574b2f3bf3ebaf08de96236d4490 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Sun, 10 Nov 2024 02:09:00 +0100 Subject: [PATCH 57/64] ci(azure-pipelines): remove yaml anchor --- azure-pipelines.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index d05ab127c..9b9e75d1b 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -6,7 +6,7 @@ pool: 'Prod Private Agent Pool' jobs: - job: HelloWorld - container: &ubuntu + container: image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' endpoint: devfra steps: @@ -17,7 +17,9 @@ jobs: echo See https://aka.ms/yaml displayName: 'Run a multi-line script' - job: ContainerTest - container: *ubuntu + container: + image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' + endpoint: devfra steps: - task: Docker@2 displayName: Image build test @@ -26,4 +28,3 @@ jobs: Dockerfile: docker/backend/Dockerfile buildContext: . tags: backend - \ No newline at end of file From 38efd413d2dc1bff2304403a675f0cfba5e356dc Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Sun, 10 Nov 2024 02:29:19 +0100 Subject: [PATCH 58/64] ci(azure-pipelines): add BuildKit test job --- azure-pipelines.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 9b9e75d1b..6a2ae3a1f 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -16,7 +16,7 @@ jobs: echo Add other tasks to build, test, and deploy your project. echo See https://aka.ms/yaml displayName: 'Run a multi-line script' -- job: ContainerTest +- job: DockerTaskTest container: image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' endpoint: devfra @@ -28,3 +28,13 @@ jobs: Dockerfile: docker/backend/Dockerfile buildContext: . tags: backend +- job: BuildKitTest + container: + image: 'devfra.azurecr.io/de.fraport.trusted/buildkit:0.12.1' + endpoint: devfra + steps: + - script: buildctl build \ + --frontend=dockerfile.v0 \ + --local context=. \ + --local dockerfile=docker/backend/Dockerfile + displayName: BuildKit test \ No newline at end of file From a97036ea7bfaedc19f3c15d455b8bd87c8ad9e5f Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Sun, 10 Nov 2024 02:32:01 +0100 Subject: [PATCH 59/64] ci(azure-pipelines): disable DockerTaskTest --- azure-pipelines.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 6a2ae3a1f..2aea0e366 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -16,18 +16,18 @@ jobs: echo Add other tasks to build, test, and deploy your project. echo See https://aka.ms/yaml displayName: 'Run a multi-line script' -- job: DockerTaskTest - container: - image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' - endpoint: devfra - steps: - - task: Docker@2 - displayName: Image build test - inputs: - command: buildAndPush - Dockerfile: docker/backend/Dockerfile - buildContext: . - tags: backend +# - job: DockerTaskTest +# container: +# image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' +# endpoint: devfra +# steps: +# - task: Docker@2 +# displayName: Image build test +# inputs: +# command: buildAndPush +# Dockerfile: docker/backend/Dockerfile +# buildContext: . +# tags: backend - job: BuildKitTest container: image: 'devfra.azurecr.io/de.fraport.trusted/buildkit:0.12.1' From aac5b0f17fca1663b64bcb36ba30b10ba558b98f Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Sun, 10 Nov 2024 02:54:23 +0100 Subject: [PATCH 60/64] ci(azure-pipelines): add buildah install attempt; disable BuildKit test --- azure-pipelines.yaml | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 2aea0e366..a55c25af7 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -28,13 +28,23 @@ jobs: # Dockerfile: docker/backend/Dockerfile # buildContext: . # tags: backend -- job: BuildKitTest +# - job: BuildKitTest +# container: +# image: 'devfra.azurecr.io/de.fraport.trusted/buildkit:0.12.1' +# endpoint: devfra +# steps: +# - script: buildctl build \ +# --frontend=dockerfile.v0 \ +# --local context=. \ +# --local dockerfile=docker/backend/Dockerfile +# displayName: BuildKit test +- job: CustomBuildahTest container: - image: 'devfra.azurecr.io/de.fraport.trusted/buildkit:0.12.1' + image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' endpoint: devfra steps: - - script: buildctl build \ - --frontend=dockerfile.v0 \ - --local context=. \ - --local dockerfile=docker/backend/Dockerfile - displayName: BuildKit test \ No newline at end of file + - script: | + apt-get -y update + apt-get -y install buildah + buildah bud -t fradrive-backend-test --volume .:/mnt/fradrive --file docker/backend/Dockerfile + displayName: Build buildah image \ No newline at end of file From 4519918292a327307bf0e904fb7e78829d040eed Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Sun, 10 Nov 2024 03:07:44 +0100 Subject: [PATCH 61/64] ci(azure-pipelines): add debugging info to buildah install --- azure-pipelines.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index a55c25af7..afa404e03 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -44,6 +44,7 @@ jobs: endpoint: devfra steps: - script: | + id apt-get -y update apt-get -y install buildah buildah bud -t fradrive-backend-test --volume .:/mnt/fradrive --file docker/backend/Dockerfile From 66c2c0bd18db7d64dc06ba5afbf27432379b83f2 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Sun, 10 Nov 2024 03:08:34 +0100 Subject: [PATCH 62/64] ci(azure-pipelines): disable HelloWorld job --- azure-pipelines.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index afa404e03..89506a4d1 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -5,17 +5,17 @@ pool: 'Prod Private Agent Pool' jobs: -- job: HelloWorld - container: - image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' - endpoint: devfra - steps: - - script: echo Hello, world! - displayName: 'Run a one-line script' - - script: | - echo Add other tasks to build, test, and deploy your project. - echo See https://aka.ms/yaml - displayName: 'Run a multi-line script' +# - job: HelloWorld +# container: +# image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' +# endpoint: devfra +# steps: +# - script: echo Hello, world! +# displayName: 'Run a one-line script' +# - script: | +# echo Add other tasks to build, test, and deploy your project. +# echo See https://aka.ms/yaml +# displayName: 'Run a multi-line script' # - job: DockerTaskTest # container: # image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' From 5013bcb8e5a7be3a71012b359e2ad2cf4c3d3ea5 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Sun, 10 Nov 2024 03:09:14 +0100 Subject: [PATCH 63/64] ci(azure-pipelines): use sudo --- azure-pipelines.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 89506a4d1..753c66727 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -45,7 +45,7 @@ jobs: steps: - script: | id - apt-get -y update - apt-get -y install buildah + sudo apt-get -y update + sudo apt-get -y install buildah buildah bud -t fradrive-backend-test --volume .:/mnt/fradrive --file docker/backend/Dockerfile displayName: Build buildah image \ No newline at end of file From ef7102ef9f6e0bbf824eaa417da3c31837e139fa Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Sun, 10 Nov 2024 03:10:56 +0100 Subject: [PATCH 64/64] ci(azure-pipelines): try docker build --- azure-pipelines.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 753c66727..7da1e371e 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -45,6 +45,7 @@ jobs: steps: - script: | id + docker build --help sudo apt-get -y update sudo apt-get -y install buildah buildah bud -t fradrive-backend-test --volume .:/mnt/fradrive --file docker/backend/Dockerfile