From 1f41aa8b371f185df52bfb5e0800fe613f4b0db1 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Mon, 11 Nov 2024 17:28:09 +0100 Subject: [PATCH 1/8] build(Makefile): fix PHONY def of job-% --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1ef34a0ed..e70f33218 100644 --- a/Makefile +++ b/Makefile @@ -280,7 +280,7 @@ endif # HELP(test-backend): test backend --test-backend: stack build $(STACK_CORES) --test --coverage --fast --flag uniworx:library-only $(--DEVELOPMENT) -.PHONY: .job-%: +.PHONY: .job-% .job-%: ./utils/dirsymlink.pl ".job-$*" .stack-work From 3008a34b78db377a2768eae94e1f7908dd34acf3 Mon Sep 17 00:00:00 2001 From: Stephan Barth Date: Mon, 11 Nov 2024 17:32:45 +0100 Subject: [PATCH 2/8] build(utils/dirsymlink.pl): Fixed handling of new files. --- utils/dirsymlink.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/dirsymlink.pl b/utils/dirsymlink.pl index d81b66cff..b88d9c180 100755 --- a/utils/dirsymlink.pl +++ b/utils/dirsymlink.pl @@ -7,7 +7,7 @@ my ($shadowdir, @keepdirs) = @ARGV; mkdir($shadowdir); -system(qq#rm -f "$shadowdir"/* "$shadowdir"/.*#); +# system(qq#rm -f "$shadowdir"/* "$shadowdir"/.*#); for my $k(@keepdirs) { unlink("$shadowdir/$k"); From 4d197732239bbd0250008dea17d37139795c073a Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Tue, 12 Nov 2024 01:15:02 +0100 Subject: [PATCH 3/8] build(Makefile): resort to default .stack-work for all jobs --- Makefile | 16 ++-------------- docker/backend/Dockerfile | 7 +++---- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index e70f33218..5c3ab799c 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ help: clean: rm -rf develop -rm -rf node_modules .npm .cache assets/icons assets/favicons static well-known config/manifest.json - -rm -rf .job-* + -rm -rf .stack-work .stack-work.lock -rm -rf bin .Dockerfile develop -$(CONTAINER_COMMAND) container prune --force .PHONY: clean-all @@ -106,24 +106,18 @@ start: %-backend: SERVICE=backend %-backend: SERVICE_VARIANT=backend %-backend: CONTAINER_CMD=localhost/fradrive/backend -# %-backend: CONTAINER_INIT="$(STACK_INIT)" -# %-backend: CONTAINER_CLEANUP="$(STACK_CLEANUP)" %-backend: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443" .PHONY: %-uniworxdb %-uniworxdb: SERVICE=backend %-uniworxdb: SERVICE_VARIANT=uniworxdb %-uniworxdb: CONTAINER_CMD=localhost/fradrive/backend -# %-uniworxdb: CONTAINER_INIT="$(STACK_INIT)" -# %-uniworxdb: CONTAINER_CLEANUP="$(STACK_CLEANUP)" .PHONY: %-hoogle %-hoogle: SERVICE=backend %-hoogle: SERVICE_VARIANT=hoogle %-hoogle: BASE_PORTS = "HOOGLE_PORT=8081" %-hoogle: CONTAINER_CMD=localhost/fradrive/backend -# %-hoogle: CONTAINER_INIT="$(STACK_INIT)" -# %-hoogle: CONTAINER_CLEANUP="$(STACK_CLEANUP)" --start-hoogle: HOOGLE_PORT=`cat $(CONTAINER_FILE) | grep 'HOOGLE_PORT=' | sed 's/HOOGLE_PORT=//'` ; \ stack $(STACK_CORES) hoogle -- server --local --port $${HOOGLE_PORT} @@ -203,7 +197,6 @@ rebuild-%: ifeq "$(CONTAINER_CMD)" "localhost/fradrive/$(SERVICE)" rm -f .Dockerfile ln -s docker/$(SERVICE)/Dockerfile .Dockerfile - $(MAKE) .job-$(JOB) MOUNT_DIR=/mnt/fradrive; \ PROJECT_DIR=/mnt/fradrive; \ if [ "$(IN_CI)" == "true" ] ; then \ @@ -223,9 +216,7 @@ else : endif ---containerized: - $(MAKE) .job-$(JOB) - $(MAKE) -- --image-build +--containerized: --image-build DEVELOP=`cat develop/.current` ; \ ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" "$(CONTAINER_INIT)" "$(CONTAINER_CLEANUP)" & \ CONTAINER_NAME=fradrive.$(CURR_DEV).$(CONTAINER_IDENT) ; \ @@ -280,9 +271,6 @@ endif # HELP(test-backend): test backend --test-backend: stack build $(STACK_CORES) --test --coverage --fast --flag uniworx:library-only $(--DEVELOPMENT) -.PHONY: .job-% -.job-%: - ./utils/dirsymlink.pl ".job-$*" .stack-work # HELP(compile-frontend): compile frontend assets --compile-frontend: node_modules assets esbuild.config.mjs diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 999c83c8c..21081debd 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -18,12 +18,11 @@ RUN if [ "${PROJECT_DIR}" != "${MOUNT_DIR}" ] ; then cp -r "${MOUNT_DIR}"/* "${P RUN mkdir -p "${PROJECT_DIR}/.stack" ENV STACK_ROOT="${PROJECT_DIR}/.stack" -WORKDIR "${PROJECT_DIR}/.job-${JOB}" -ENV HOME="${PROJECT_DIR}/.job-${JOB}" +WORKDIR ${PROJECT_DIR} +ENV HOME=${PROJECT_DIR} RUN make -- --dependencies-backend STACK_ROOT=${STACK_ROOT} IN_CONTAINER=true JOB=${JOB} -RUN cd "${PROJECT_DIR}" && make .job-start -RUN cd "${PROJECT_DIR}/.job-start" && STACK_ROOT=${STACK_ROOT} stack install yesod-bin +RUN stack install yesod-bin ENV FRADRIVE_MAKE_TARGET=--start-backend ENTRYPOINT make -- ${FRADRIVE_MAKE_TARGET} STACK_ROOT="${STACK_ROOT}" IN_CONTAINER=true CONTAINER_FILE="${CONTAINER_FILE}" JOB="${JOB}" From cbf167d197f2af63a3904158a42cd20211fba419 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Wed, 13 Nov 2024 03:37:46 +0100 Subject: [PATCH 4/8] build(Makefile): add/reimplement shell-% and ghci targets --- .gitignore | 1 + Makefile | 33 +++++++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 320302b1c..0c90b8a6f 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,4 @@ tunnel.log **/result-* .develop.cmd /.vscode +.ghc/ghci_history diff --git a/Makefile b/Makefile index 5c3ab799c..5e00fe41a 100644 --- a/Makefile +++ b/Makefile @@ -113,6 +113,11 @@ start: %-uniworxdb: SERVICE_VARIANT=uniworxdb %-uniworxdb: CONTAINER_CMD=localhost/fradrive/backend +.PHONY: %-ghci +%-ghci: SERVICE=backend +%-ghci: SERVICE_VARIANT=ghci +%-ghci: CONTAINER_CMD=localhost/fradrive/backend + .PHONY: %-hoogle %-hoogle: SERVICE=backend %-hoogle: SERVICE_VARIANT=hoogle @@ -175,6 +180,16 @@ lint-%: CF_PREFIX = lint- lint-%: CONTAINER_ATTACHED = true lint-%: --act ; +.PHONY: shell-% +# HELP(shell-$SERVICE): launch (bash) shell inside a new $SERVICE container +shell-%: JOB=shell +shell-%: CF_PREFIX=shell- +shell-%: CONTAINER_ATTACHED=true +shell-%: --act ; +.PHONY: ghci +# HELP: launch new backend instance and enter interactive ghci shell (WIP) +ghci: shell-ghci; + --act: --develop_containerized; --develop_%: PORTS = $(foreach PORT,$(BASE_PORTS),$(shell utils/next_free_port.pl $(PORT))) @@ -299,6 +314,13 @@ well-known: static; AVSPASS=${AVSPASS:-nopasswordset} ; \ stack exec uniworxdb -- $(UNIWORXDB_OPTS) +--shell-ghci: + stack exec -- ghci +# --main-is uniworx:exe:uniworx + +--shell-%: + /bin/bash + # HELP(start-minio): start minio service .PHONY: status @@ -324,21 +346,16 @@ log-%: .PHONY: enter # HELP: launch (bash) shell inside a currently running container. Use ./enter shell wrapper for more convenient usage, possibly with tab-completion in the future enter: --ensure-develop - $(MAKE) -- --shell + $(MAKE) -- --enter .PHONY: psql # HELP: enter psql (postgresql) cli inside a currently running database container psql: ENTRYPOINT=/usr/bin/psql -d uniworx psql: EXEC_OPTS=--user postgres psql: --ensure-develop - $(MAKE) -- --shell CONTAINER_FILE=develop/`cat develop/.current`/start-postgres + $(MAKE) -- --enter CONTAINER_FILE=develop/`cat develop/.current`/start-postgres -.PHONY: ghci -# HELP: launch new backend instance and enter interactive ghci shell (WIP) -ghci: ENTRYPOINT=stack ghci -ghci: --shell; - ---shell: +--enter: CONTAINER_ID=`cat $(CONTAINER_FILE) | grep 'CONTAINER_ID=' | sed 's/CONTAINER_ID=//'` ; \ $(CONTAINER_COMMAND) exec -it $(EXEC_OPTS) $${CONTAINER_ID} $(if $(ENTRYPOINT),$(ENTRYPOINT),/bin/bash) From 9f862bcfd63c56df75e093cf81fdf1a1b0816a14 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Wed, 13 Nov 2024 03:48:57 +0100 Subject: [PATCH 5/8] build(Makefile): explicitely execute hpack on missing uniworx.cabal for robustness --- Makefile | 6 +++--- docker/backend/Dockerfile | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5e00fe41a..33e2e5e97 100644 --- a/Makefile +++ b/Makefile @@ -278,7 +278,7 @@ endif --compile-backend: stack build $(STACK_CORES) --fast --profile --library-profiling --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin # HELP(dependencies-backend): (re-)build backend dependencies ---dependencies-backend: stack.yaml stack.yaml.lock package.yaml +--dependencies-backend: uniworx.cabal stack build $(STACK_CORES) --fast --only-dependencies # HELP(lint-backend): lint backend --lint-backend: @@ -286,6 +286,8 @@ endif # HELP(test-backend): test backend --test-backend: stack build $(STACK_CORES) --test --coverage --fast --flag uniworx:library-only $(--DEVELOPMENT) +uniworx.cabal: + stack exec -- hpack --force # HELP(compile-frontend): compile frontend assets --compile-frontend: node_modules assets esbuild.config.mjs @@ -399,7 +401,5 @@ switch-develop: develop/.current: ls -1 develop | tail -n1 > develop/.current -stack.yaml.lock: --dependencies-backend; - .PHONY: --% .SUFFIXES: # Delete all default suffixes diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 21081debd..54a0c5d55 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -23,6 +23,7 @@ ENV HOME=${PROJECT_DIR} RUN make -- --dependencies-backend STACK_ROOT=${STACK_ROOT} IN_CONTAINER=true JOB=${JOB} RUN stack install yesod-bin +RUN stack install hpack ENV FRADRIVE_MAKE_TARGET=--start-backend ENTRYPOINT make -- ${FRADRIVE_MAKE_TARGET} STACK_ROOT="${STACK_ROOT}" IN_CONTAINER=true CONTAINER_FILE="${CONTAINER_FILE}" JOB="${JOB}" From 81f689cfdf250e2fc9367287eb49f059012b9692 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Wed, 13 Nov 2024 04:00:52 +0100 Subject: [PATCH 6/8] build(Makefile): update documentation --- Makefile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 33e2e5e97..730e52ecf 100644 --- a/Makefile +++ b/Makefile @@ -27,9 +27,6 @@ export CONTAINER_ATTACHED export CONTAINER_INIT export CONTAINER_CLEANUP -# export STACK_INIT="./utils/stack-work-init.sh" -# export STACK_CLEANUP="rm -f .stack-work.lock" - export SERVICE export SERVICE_VARIANT ?= $(SERVICE) export JOB @@ -187,7 +184,7 @@ shell-%: CF_PREFIX=shell- shell-%: CONTAINER_ATTACHED=true shell-%: --act ; .PHONY: ghci -# HELP: launch new backend instance and enter interactive ghci shell (WIP) +# HELP(ghci): launch new backend instance and enter interactive ghci shell ghci: shell-ghci; --act: --develop_containerized; @@ -205,7 +202,7 @@ ghci: shell-ghci; $(MAKE) -- --$* CONTAINER_FILE=$${CONTAINER_FILE} CONTAINER_IDENT=$${CONTAINER_IDENT} JOB=$(JOB) .PHONY: rebuild-% -# HELP(rebuild-{backend,frontend,database,memcached,minio}): force-rebuild the stated docker image +# HELP(rebuild-{backend,frontend,database,memcached,minio}): force-rebuild a given container image rebuild-%: $(MAKE) -- --image-build SERVICE=$* NO_CACHE=--no-cache --image-build: @@ -309,7 +306,7 @@ static: node_modules assets esbuild.config.mjs npm run build well-known: static; -# HELP(compile-uniworxdb): clear and fill database. requires running postgres +# HELP(compile-uniworxdb): clear and fill database. requires running postgres instance (use "make start-postgres" to start one) # TODO (db-m-$MIGRATION-backend): apply migration (see src/Model/Migration/Definition.hs for list of available migrations) --compile-uniworxdb: --compile-backend SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true} ; \ @@ -320,6 +317,7 @@ well-known: static; stack exec -- ghci # --main-is uniworx:exe:uniworx +# HELP(shell-{backend,frontend,memcached,minio,postgres}): enter (bash) shell inside a new container of a given service --shell-%: /bin/bash From d646a033258114daddc0ab64ad81727d9052407d Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 14 Nov 2024 00:02:41 +0100 Subject: [PATCH 7/8] build(backend): use localhost with IPv6-enabled pg_hba.conf --- config/settings.yml | 8 ++++---- docker/postgres/pg_hba.conf | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/config/settings.yml b/config/settings.yml index 043905fbb..04f69e649 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -123,7 +123,7 @@ auth-pw-hash: database: user: "_env:PGUSER:uniworx" password: "_env:PGPASS:uniworx" - host: "_env:PGHOST:host.docker.internal" + host: "_env:PGHOST:localhost" port: "_env:PGPORT:5432" # See config/test-settings.yml for an override during tests database: "_env:PGDATABASE:uniworx" @@ -181,7 +181,7 @@ smtp: limit: "_env:SMTPLIMIT:10" widget-memcached: - host: "_env:WIDGET_MEMCACHED_HOST:host.docker.internal" + host: "_env:WIDGET_MEMCACHED_HOST:localhost" port: "_env:WIDGET_MEMCACHED_PORT:11211" auth: [] limit: "_env:WIDGET_MEMCACHED_LIMIT:1024" @@ -190,7 +190,7 @@ widget-memcached: expiration: "_env:WIDGET_MEMCACHED_EXPIRATION:3600" session-memcached: - host: "_env:SESSION_MEMCACHED_HOST:host.docker.internal" + host: "_env:SESSION_MEMCACHED_HOST:localhost" port: "_env:SESSION_MEMCACHED_PORT:11211" auth: [] limit: "_env:SESSION_MEMCACHED_LIMIT:1024" @@ -200,7 +200,7 @@ session-memcached: # TODO: this one seems to be required! memcached: - host: "_env:MEMCACHED_HOST:host.docker.internal" + host: "_env:MEMCACHED_HOST:localhost" port: "_env:MEMCACHED_PORT:11211" auth: [] limit: "_env:MEMCACHED_LIMIT:1024" diff --git a/docker/postgres/pg_hba.conf b/docker/postgres/pg_hba.conf index c11da7bfe..b826e433d 100644 --- a/docker/postgres/pg_hba.conf +++ b/docker/postgres/pg_hba.conf @@ -1,2 +1,3 @@ local all all trust -host all all 0.0.0.0/0 trust \ No newline at end of file +host all all 0.0.0.0/0 trust +host all all ::1/128 trust \ No newline at end of file From 1018e9dac53788b77333773b772f10423353d3e6 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 14 Nov 2024 00:31:20 +0100 Subject: [PATCH 8/8] build(Makefile): abandon yesod-devel for basic uniworx binary execution in start-backend --- Makefile | 8 ++++---- stack.yaml | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 730e52ecf..ff73bd03e 100644 --- a/Makefile +++ b/Makefile @@ -253,7 +253,7 @@ endif $(CONTAINER_COMMAND) attach $${CONTAINER_ID} || : ; \ fi -# HELP(start-backend): start yesod-devel instance +# HELP(start-backend): start development instance --start-backend: export DEV_PORT_HTTP=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTP=' | sed 's/DEV_PORT_HTTP=//'`; \ export DEV_PORT_HTTPS=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTPS=' | sed 's/DEV_PORT_HTTPS=//'`; \ @@ -270,10 +270,10 @@ endif export RIBBON=$${RIBBON:-$${HOST:-localhost}} ; \ export APPROOT=$${APPROOT:-http://localhost:$${DEV_PORT_HTTP}} ; \ export AVSPASS=$${AVSPASS:-nopasswordset} ; \ - stack $(STACK_CORES) exec --local-bin-path $$(pwd)/bin -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" + ./bin/uniworx # HELP(compile-backend): compile backend binaries --compile-backend: - stack build $(STACK_CORES) --fast --profile --library-profiling --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin + stack build $(STACK_CORES) --fast --profile --library-profiling --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin --copy-bins # HELP(dependencies-backend): (re-)build backend dependencies --dependencies-backend: uniworx.cabal stack build $(STACK_CORES) --fast --only-dependencies @@ -311,7 +311,7 @@ well-known: static; --compile-uniworxdb: --compile-backend SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true} ; \ AVSPASS=${AVSPASS:-nopasswordset} ; \ - stack exec uniworxdb -- $(UNIWORXDB_OPTS) + ./bin/uniworxdb $(UNIWORXDB_OPTS) --shell-ghci: stack exec -- ghci diff --git a/stack.yaml b/stack.yaml index 3c331178f..c881ba7b1 100644 --- a/stack.yaml +++ b/stack.yaml @@ -10,6 +10,8 @@ rebuild-ghc-options: true #ghc-options: # "$everything": -fno-prof-auto +local-bin-path: ./bin + build: library-profiling: true executable-profiling: true