diff --git a/.gitignore b/.gitignore index ed502f847..08249c6c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .Dockerfile dist* +develop node_modules/ assets/icons assets/favicons diff --git a/Makefile b/Makefile index 055ea674e..5f40064c8 100644 --- a/Makefile +++ b/Makefile @@ -2,21 +2,30 @@ SHELL=bash # MAKE=make -d export CONTAINER_COMMAND ?= podman -export CONTAINER_RUN ?= $(CONTAINER_COMMAND) run -it --network=host +export CONTAINER_RUN ?= $(CONTAINER_COMMAND) run -it -d --network=host export IN_CONTAINER ?= false export IN_CI ?= false +# TODO: deprecated export WATCH export db ?= -cf +export CONTAINER_ID +export FRADRIVE_SERVICE + +# TODO: to be deprecated export DEV_PORT_HTTP export DEV_PORT_HTTPS export IMAGE_REGISTRY = docker.io export MEMCACHED_IMAGE = $(IMAGE_REGISTRY)/memcached:latest export MINIO_IMAGE = $(IMAGE_REGISTRY)/minio/minio:latest -export MAILDEV_IMAGE = $(IMAGE_REGISTRY)/maildev/maildev:latest +export MAILDEV_IMAGE = $(IMAGE_REGISTRY)/maildev/maildev:latest # TODO: needs different port than 1025 to avoid conflicts + +export DEVELOP +export CONTAINER_FILE +export CONTAINER_FILE_CONTENT # HELP HEADER START # To see the definition of all available targets, take a look into the Makefile. @@ -31,8 +40,6 @@ export MAILDEV_IMAGE = $(IMAGE_REGISTRY)/maildev/maildev:latest ########################### ##### GENERAL TARGETS ##### -.PHONY: --% - .PHONY: help # HELP: print out this help message help: @@ -56,7 +63,7 @@ clean-all: clean $(CONTAINER_COMMAND) volume prune --force .PHONY: release -# HELP: unfinished +# HELP: create, commit and push a new release release: ./.gitlab-ci/version.pl -changelog CHANGELOG.md git add CHANGELOG.md @@ -65,12 +72,16 @@ release: git commit -m "chore(release): $${VERSION}" git push +# TODO: rewrite shell targets to enter shell inside currently running containers, do not just launch new containers! .PHONY: %-shell -%-shell: +# HELP: launches bash-shell inside backend/frontend container +%-shell: ENTRYPOINT=/bin/bash +%-shell: --%-shell; +--%-shell: MOUNT_DIR=/mnt/fradrive ; \ FRADRIVE_SERVICE=$* ; \ $(MAKE) -- --image-build FRADRIVE_SERVICE=$${FRADRIVE_SERVICE} ; \ - $(CONTAINER_RUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --entrypoint /bin/bash --name fradrive.$${FRADRIVE_SERVICE}.interactive.$$(date +'%Y-%m-%dT%H-%M-%S') fradrive/$${FRADRIVE_SERVICE} + $(CONTAINER_RUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env CONTAINER_FILE=$(CONTAINER_FILE) --entrypoint $(ENTRYPOINT) --name fradrive.$${FRADRIVE_SERVICE}.interactive.$$(date +'%Y-%m-%dT%H-%M-%S') fradrive/$${FRADRIVE_SERVICE} ##### GENERAL TARGETS ##### ########################### @@ -86,25 +97,19 @@ release: # - additionally provide targets for termination of running containers .PHONY: serve # HELP: serve frontend, backend, and database -serve: - $(MAKE) serve-database & - $(MAKE) serve-memcached & - $(MAKE) serve-minio & -# $(MAKE) serve-maildev & # TODO: needs different port than 1025 to avoid conflicts - $(MAKE) serve-frontend & - $(MAKE) serve-backend +serve: serve-database serve-memcached serve-minio serve-frontend serve-backend; .PHONY: compile # HELP: compile frontend and backend -compile: compile-frontend compile-backend +compile: compile-frontend compile-backend; .PHONY: lint # HELP: lint frontend and backend -lint: lint-frontend lint-backend +lint: lint-frontend lint-backend; .PHONY: test # HELP: test frontend, backend, and check internationalization -test: test-frontend test-backend i18n-check +test: test-frontend test-backend i18n-check; ##### UNIFIED FRONTEND/BACKEND TARGETS ##### ############################################ @@ -120,8 +125,7 @@ test: test-frontend test-backend i18n-check --%-frontend: --containerized-static-frontend --containerized-well-known-frontend; # HELP(compile-frontend): compile frontend ---compile-frontend: static well-known - npm run build +--compile-frontend: static well-known; .PHONY: serve-frontend # HELP: serve frontend (watch file changes) @@ -181,16 +185,21 @@ well-known: static; --%-prod-backend: stackopts=--flag uniworx:-dev --%-prod-backend: --image-build --containerized-%-backend; +# TODO: to be deprecated by start-% .PHONY: serve-backend # HELP: serve backend serve-backend: - DEV_PORT_HTTP=`docker/backend/dev_port.pl 3000 | tee .dev-port-http`; \ - DEV_PORT_HTTPS=`docker/backend/dev_port.pl 3443 | tee .dev-port-https`; \ + DEV_PORT_HTTP=`utils/next_free_port.pl 3000 | tee .dev-port-http`; \ + DEV_PORT_HTTPS=`utils/next_free_port.pl 3443 | tee .dev-port-https`; \ $(MAKE) -- --containerized---serve-dev-backend DEV_PORT_HTTP=$${DEV_PORT_HTTP} DEV_PORT_HTTPS=$${DEV_PORT_HTTPS} +# DEV_PORT_HTTP=`grep 'https:' develop/$(DATE)/backend/$(CONTAINER_ID) | sed 's/.*://'`; \ +# DEV_PORT_HTTPS=`grep 'https:' develop/$(DATE)/backend/$(CONTAINER_ID) | sed 's/.*://'`; \ +# DEVELOP=$(MAKE) develop/ --serve-dev-backend: start.sh DEV_PORT_HTTP=`cat .dev-port-http`; \ DEV_PORT_HTTPS=`cat .dev-port-https`; \ ./start.sh +# ./utils/watchrun.sh develop/ ./start.sh # HELP(compile-backend): compile backend --compile-backend: @@ -206,7 +215,6 @@ serve-backend: # HELP(db-cf-backend): clear and fill database. requires running postgres # TODO (db-m-$MIGRATION-backend): apply migration (see src/Model/Migration/Definition.hs for list of available migrations) -PHONY: --db-%-backend --db-%-dev-backend: .stack SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true} ; \ AVSPASS=${AVSPASS:-nopasswordset} ; \ @@ -225,6 +233,7 @@ PHONY: --db-%-backend ############################ ##### DATABASE TARGETS ##### +# TODO: to be deprecated by start-% .PHONY: serve-database # HELP: serve database serve-database: --containerized-database; @@ -236,14 +245,6 @@ serve-database: --containerized-database; $(CONTAINER_RUN) --name fradrive.$(FRADRIVE_SERVICE).$$(date +'%Y-%m-%dT%H-%M-%S') fradrive/$(FRADRIVE_SERVICE) ; \ fi -.PHONY: serve-memcached -serve-memcached: - $(CONTAINER_RUN) $(MEMCACHED_IMAGE) - -.PHONY: serve-minio -serve-minio: - $(CONTAINER_RUN) $(MINIO_IMAGE) -- server $$(mktemp) - ##### DATABASE TARGETS ##### ############################ @@ -251,7 +252,6 @@ serve-minio: ############################# ##### CONTAINER TARGETS ##### -.PHONY: --containerized-%-frontend --containerized-%-frontend: FRADRIVE_SERVICE=frontend --containerized-%-frontend: --image-build $(MAKE) -- --image-run-$*-frontend @@ -260,6 +260,14 @@ serve-minio: --containerized-%-backend: --image-build $(MAKE) -- --image-run-$*-backend +--containerized-%-minio: FRADRIVE_SERVICE=minio +--containerized-%-minio: --image-build + $(MAKE) -- --image-run-$*-minio + +--containerized-%-memcached: FRADRIVE_SERVICE=memcached +--containerized-%-memcached: --image-build + $(MAKE) -- --image-run-$*-memcached + .PHONY: image-rebuild # HELP: rebuild the stated docker image (frontend, backend, database instead of %) image-rebuild-%: @@ -281,13 +289,127 @@ image-rebuild-%: if [ "$(IN_CONTAINER)" == "true" ] ; then \ $(MAKE) -- $* ; \ else \ - $(CONTAINER_RUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env WATCH=$(WATCH) --name fradrive.$(FRADRIVE_SERVICE).$$(date +'%Y-%m-%dT%H-%M-%S') localhost/fradrive/$(FRADRIVE_SERVICE) ; \ + CONTAINER_ID=`$(CONTAINER_RUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(FRADRIVE_SERVICE).$$(date +'%Y-%m-%dT%H-%M-%S') localhost/fradrive/$(FRADRIVE_SERVICE)` ; \ + echo "CONTAINER_ID=$${CONTAINER_ID}" >> $(CONTAINER_FILE); \ fi +# .PHONY: new-backend +# new-backend: +# LAST_DATE=`ls -1 develop | tail -n1` +# if [[ -z "$${LAST_DATE}" || -e `ls develop/$${LAST_DATE}/database` ]]; then \ +# mkdir -p develop/$$(date +'%Y-%m-%dT%H-%M-%S') \ +# fi +# serve-backend + +#new-database: +# mkdir -p develop/$$(date +'%Y-%m-%dT%H-%M-%S') +# $(MAKE) serve-database + +# TODO: work in progress +# - compute used ports and provide as env vars +# - start container in detached mode +# - save container id with ports as content as file +# .PHONY: serve-% +# serve-backend: +# DEV_PORT_HTTP=`docker/backend/dev_port.pl 3000 | tee develop/$${DATE}/fradrive.backend.$${DATE}`; \ +# DEV_PORT_HTTPS=`docker/backend/dev_port.pl 3443 | tee .dev-port-https`; \ +# $(MAKE) -- --containerized---start-dev-backend DEV_PORT_HTTP=$${DEV_PORT_HTTP} DEV_PORT_HTTPS=$${DEV_PORT_HTTPS} +#--serve-dev-backend: start.sh +# DEV_PORT_HTTP=`grep 'https:' develop/$(DATE)/backend/$(CONTAINER_ID) | sed 's/.*://'`; \ +# DEV_PORT_HTTPS=`grep 'https:' develop/$(DATE)/backend/$(CONTAINER_ID) | sed 's/.*://'`; \ +# ./utils/watchrun.sh develop/ ./start.sh + + +#CURRENT_DEVELOP = $(shell mkdir -p develop && (ls -1 develop | tail -n1)) +CURRENT_DEVELOP = $(shell if [[ ! -e develop ]] ; then mkdir develop; $(MAKE) develop ; fi && (ls -1 develop | tail -n1)) +DEVELOP = develop/$(CURRENT_DEVELOP) +DATE := $(shell date +'%Y-%m-%dT%H-%M-%S') +SET_DEVELOP = $(eval DEVELOP=develop/$$(DATE)) + +.PHONY: develop +develop: + $(SET_DEVELOP) + mkdir -p develop/$(DATE) +--ensure-develop: + if ! [[ -e develop ]]; then \ + $(MAKE) develop; \ + fi + +start: develop start-database start-memcached start-minio start-frontend start-backend; + +start-database: BASE_PORTS = "PGPORT=5432" +start-database: SINGLETON = true +start-memcached: BASE_PORTS = "MEMCACHED_PORT=11211" +start-memcached: SINGLETON = true +start-minio: BASE_PORTS = "UPLOAD_S3_PORT=9000" +start-minio: SINGLETON = true + +start-backend: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443" +start-backend: SINGLETON = false + +start-frontend: SINGLETON = false + +start-%: FRADRIVE_SERVICE = % +start-%: PORTS = $(foreach PORT,$(BASE_PORTS),$(shell utils/next_free_port.pl $(PORT))) +start-%: --ensure-develop + echo "$*" + echo "$(DATE)" + echo "DEV $(DEVELOP)" + if [[ "$(SINGLETON)" = "true" ]]; then \ + CONTAINER_FILE=$(DEVELOP)/$* ; \ + else \ + DEVDIR=$(DEVELOP)/$*; \ + mkdir -p $${DEVDIR}; \ + I=`ls $${DEVDIR} | sort -n | tail -n1`; \ + echo "$${I}"; \ + CONTAINER_FILE=$${DEVDIR}/$$(($${I}+1)); \ + fi ; \ + echo "$(PORTS)" | sed 's/ /\n/g' > $${CONTAINER_FILE} ; \ + $(MAKE) -- --containerized---start-$* CONTAINER_FILE=$${CONTAINER_FILE} + +--start-backend: start.sh + DEV_PORT_HTTP=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTP=' | sed 's/DEV_PORT_HTTP=//'`; \ + DEV_PORT_HTTPS=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTPS=' | sed 's/DEV_PORT_HTTPS=//'`; \ + ./utils/watchrun-container.sh "$(CONTAINER_FILE)" ./start.sh + +--containerized---start-database: FRADRIVE_SERVICE=database +# port forwarding is disabled in --network=host mode; nevertheless it is stated here for documentation reasons +--containerized---start-database: docker/database/initdb.sh docker/database/pg_hba.conf docker/database/postgresql.conf docker/database/schema.sql --image-build + if [ "$(IN_CONTAINER)" == "false" ] ; then \ + $(CONTAINER_RUN) --name fradrive.$(FRADRIVE_SERVICE).$$(date +'%Y-%m-%dT%H-%M-%S') fradrive/$(FRADRIVE_SERVICE) ; \ + fi + +# TODO: mkdir inside develop/$(DATE)/minio instead of mktemp +# TODO: remove dir in stop-minio +#--containerized---start-minio: +# MINIO_DIR=`mktemp` ; \ +# echo "MINIO_DIR=$${MINIO_DIR}" > $(DEVELOP)/minio ; \ +# CONTAINER_ID=`$(CONTAINER_RUN) -v $(PWD):/mnt/fradrive --file docker/minio/Dockerfile -- (server $${MINIO_DIR})` ; \ +# echo "CONTAINER_ID=$${CONTAINER_ID}" >> $(DEVELOP)/minio + +# TODO: work in progress +# - kill container and remove file from .develop +# TODO: find way to stop a single container, e.g. by removing the file +.PHONY: stop-% +stop-%: --stop-% + $(MAKE) stop-container CONTAINER_FILE=TODO +.PHONY: --stop-minio +--stop-minio: + MINIO_DIR=`cat $(DEVELOP)/minio` + rm -rf $${MINIO_DIR} + +stop-container-by-id: + podman stop $(CONTAINER_ID) +stop-container-by-file: + CONTAINER_ID=`grep 'CONTAINER_ID=' $(CONTAINER_FILE) | sed 's/CONTAINER_ID=//'` ; \ + $(MAKE) stop-container-by-id CONTAINER_ID=$${CONTAINER_ID} + ##### CONTAINER TARGETS ##### ############################# +# TODO: move targets below to better location + .PHONY: i18n-check # HELP: check internationalization i18n-check: --image-run---i18n-check @@ -298,3 +420,5 @@ i18n-check: --image-run---i18n-check %.lock: [ -e $@ ] || touch $@ flock -en $@ true + +.PHONY: --% \ No newline at end of file diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 088d60bd2..0b1709105 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -23,7 +23,7 @@ RUN make .stack STACK_ROOT=${STACK_ROOT} IN_CONTAINER=true RUN stack build yesod-bin ENV FRADRIVE_MAKE_TARGET=serve-backend -ENTRYPOINT make -- ${FRADRIVE_MAKE_TARGET} STACK_ROOT="${STACK_ROOT}" IN_CONTAINER=true +ENTRYPOINT cat ${CONTAINER_FILE} > /container-file && make -- ${FRADRIVE_MAKE_TARGET} STACK_ROOT="${STACK_ROOT}" IN_CONTAINER=true CONTAINER_FILE="${CONTAINER_FILE}" EXPOSE 3000/tcp EXPOSE 3443/tcp \ No newline at end of file diff --git a/docker/backend/start.sh b/docker/backend/start.sh new file mode 100755 index 000000000..223bdfe0b --- /dev/null +++ b/docker/backend/start.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -e + +# SPDX-FileCopyrightText: 2022-2024 Sarah Vaupel , Gregor Kleen , Sarah Vaupel , Steffen Jost +# +# SPDX-License-Identifier: AGPL-3.0-or-later + + +__HOST=${HOST:-$(hostname -s | awk '{ print $0; }')} + +export DETAILED_LOGGING=${DETAILED_LOGGING:-true} +export LOG_ALL=${LOG_ALL:-false} +export LOGLEVEL=${LOGLEVEL:-info} +export DUMMY_LOGIN=${DUMMY_LOGIN:-true} +export SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true} +export SERVER_SESSION_COOKIES_SECURE=${SERVER_SESSION_COOKIES_SECURE:-false} +export COOKIES_SECURE=${COOKIES_SECURE:-false} +export ALLOW_DEPRECATED=${ALLOW_DEPRECATED:-true} +export ENCRYPT_ERRORS=${ENCRYPT_ERRORS:-false} +export RIBBON=${RIBBON:-${__HOST:-localhost}} +export APPROOT=${APPROOT:-http://localhost:$DEV_PORT_HTTP} +export AVSPASS=${AVSPASS:-nopasswordset} +unset HOST + +# export PORT_OFFSET=$(((16#$(sha256sum <<<"$(hostname -f):''${basePath}" | head -c 16)) % 1000)) + +# move-back() { +# mv -vT .stack-work .stack-work-run +# [[ -d .stack-work-build ]] && mv -vT .stack-work-build .stack-work +# } + +# if [[ -d .stack-work-run ]]; then +# [[ -d .stack-work ]] && mv -vT .stack-work .stack-work-build +# mv -vT .stack-work-run .stack-work +# trap move-back EXIT +# fi + +if ! [ -z "$(which yesod)" ] +then + yesod devel -p "$DEV_PORT_HTTP" -q "$DEV_PORT_HTTPS" $@ +elif ! [ -z "$(which stack)" ] +then + stack exec -- yesod devel -p "$DEV_PORT_HTTP" -q "$DEV_PORT_HTTPS" $@ +else + exit 1 +fi diff --git a/docker/memcached/Dockerfile b/docker/memcached/Dockerfile new file mode 100644 index 000000000..87e018409 --- /dev/null +++ b/docker/memcached/Dockerfile @@ -0,0 +1,4 @@ +FROM docker.io/memcached:latest + +USER root +ENTRYPOINT /mnt/fradrive/utils/watchrun.sh "/mnt/fradrive/${CONTAINER_FILE}" "su memcache -c /usr/local/bin/docker-entrypoint.sh" \ No newline at end of file diff --git a/docker/minio/Dockerfile b/docker/minio/Dockerfile new file mode 100644 index 000000000..0b9ad664c --- /dev/null +++ b/docker/minio/Dockerfile @@ -0,0 +1,6 @@ +FROM docker.io/minio/minio:latest + +ENV MINIO_DIR = $(mktemp) + +WORKDIR /mnt/fradrive +ENTRYPOINT ./utils/watchrun.sh "${CONTAINER_FILE}" "/usr/bin/docker-entrypoint.sh server ${MINIO_DIR}" "rm -rf ${MINIO_DIR}" \ No newline at end of file diff --git a/utils/next_free_port.pl b/utils/next_free_port.pl index 0d8faa0db..ab4dda59e 100755 --- a/utils/next_free_port.pl +++ b/utils/next_free_port.pl @@ -3,7 +3,9 @@ use strict; use warnings; -my $port = $ARGV[0]; +my $portDef = $ARGV[0]; +$portDef=~m/^(\D*)(\d+)(.*)/ or die "$0: We need a start port as parameter but got none!\n"; +my ($pre, $port, $post) = ($1, $2, $3); my $used_ports = `netstat -tulan`; my %p=(); @@ -16,4 +18,4 @@ for(split m/\R/, $used_ports) { $port++ while $p{$port}; -print $port \ No newline at end of file +print "$pre$port$post" \ No newline at end of file diff --git a/utils/runwrapper.sh b/utils/runwrapper.sh new file mode 100755 index 000000000..bea1c787c --- /dev/null +++ b/utils/runwrapper.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +echo $$ > /tmp/pidwrapper + +$1 \ No newline at end of file diff --git a/utils/watchrun-container.sh b/utils/watchrun-container.sh new file mode 100755 index 000000000..9c31324ce --- /dev/null +++ b/utils/watchrun-container.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +STOPSCRIPT="make stop-container-by-id CONTAINFER_FILE=${CONTAINER_FILE} ; $3" + +./utils/watchrun.sh "$1" "$2" \ No newline at end of file diff --git a/utils/watchrun.sh b/utils/watchrun.sh index bd3d7fbc4..343c92149 100755 --- a/utils/watchrun.sh +++ b/utils/watchrun.sh @@ -6,13 +6,14 @@ STOPSCRIPT=$3 touch "$FILENAME" -$STARTSCRIPT & - +( while [ -e "$FILENAME" ] ; do inotifywait -e ATTRIB -t 10 "$FILENAME" > /dev/null 2&>/dev/null done -kill %1 - $STOPSCRIPT +kill `cat /tmp/pidwrapper` +) & + +/mnt/fradrive/utils/runwrapper.sh "$STARTSCRIPT"