fraport/fradrive-old
fraport/fradrive-old
Archived
3
1
Fork 0
Code Issues 65 Pull Requests 5 Actions Packages Projects Releases 197 Wiki Activity

build(Makefile): work on start targets; custom minio and memcached Dockerfiles, and more

Browse Source
This commit is contained in:
Sarah Vaupel 2024-10-11 02:44:28 +02:00
parent b9153c0767
commit 3a5a169ca0
10 changed files with 234 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,5 +1,6 @@
.Dockerfile
dist*
develop
node_modules/
assets/icons
assets/favicons

188
Makefile
View File

@ -2,21 +2,30 @@ SHELL=bash
# MAKE=make -d
export CONTAINER_COMMAND ?= podman
export CONTAINER_RUN ?= $(CONTAINER_COMMAND) run -it --network=host
export CONTAINER_RUN ?= $(CONTAINER_COMMAND) run -it -d --network=host
export IN_CONTAINER ?= false
export IN_CI ?= false
# TODO: deprecated
export WATCH
export db ?= -cf
export CONTAINER_ID
export FRADRIVE_SERVICE
# TODO: to be deprecated
export DEV_PORT_HTTP
export DEV_PORT_HTTPS
export IMAGE_REGISTRY = docker.io
export MEMCACHED_IMAGE = $(IMAGE_REGISTRY)/memcached:latest
export MINIO_IMAGE = $(IMAGE_REGISTRY)/minio/minio:latest
export MAILDEV_IMAGE = $(IMAGE_REGISTRY)/maildev/maildev:latest
export MAILDEV_IMAGE = $(IMAGE_REGISTRY)/maildev/maildev:latest # TODO: needs different port than 1025 to avoid conflicts
export DEVELOP
export CONTAINER_FILE
export CONTAINER_FILE_CONTENT
# HELP HEADER START
# To see the definition of all available targets, take a look into the Makefile.
@ -31,8 +40,6 @@ export MAILDEV_IMAGE = $(IMAGE_REGISTRY)/maildev/maildev:latest
###########################
##### GENERAL TARGETS #####
.PHONY: --%
.PHONY: help
# HELP: print out this help message
help:
@ -56,7 +63,7 @@ clean-all: clean
$(CONTAINER_COMMAND) volume prune --force
.PHONY: release
# HELP: unfinished
# HELP: create, commit and push a new release
release:
./.gitlab-ci/version.pl -changelog CHANGELOG.md
git add CHANGELOG.md
@ -65,12 +72,16 @@ release:
git commit -m "chore(release): $${VERSION}"
git push
# TODO: rewrite shell targets to enter shell inside currently running containers, do not just launch new containers!
.PHONY: %-shell
%-shell:
# HELP: launches bash-shell inside backend/frontend container
%-shell: ENTRYPOINT=/bin/bash
%-shell: --%-shell;
--%-shell:
MOUNT_DIR=/mnt/fradrive ; \
FRADRIVE_SERVICE=$* ; \
$(MAKE) -- --image-build FRADRIVE_SERVICE=$${FRADRIVE_SERVICE} ; \
$(CONTAINER_RUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --entrypoint /bin/bash --name fradrive.$${FRADRIVE_SERVICE}.interactive.$$(date +'%Y-%m-%dT%H-%M-%S') fradrive/$${FRADRIVE_SERVICE}
$(CONTAINER_RUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env CONTAINER_FILE=$(CONTAINER_FILE) --entrypoint $(ENTRYPOINT) --name fradrive.$${FRADRIVE_SERVICE}.interactive.$$(date +'%Y-%m-%dT%H-%M-%S') fradrive/$${FRADRIVE_SERVICE}
##### GENERAL TARGETS #####
###########################
@ -86,25 +97,19 @@ release:
# - additionally provide targets for termination of running containers
.PHONY: serve
# HELP: serve frontend, backend, and database
serve:
$(MAKE) serve-database &
$(MAKE) serve-memcached &
$(MAKE) serve-minio &
# $(MAKE) serve-maildev & # TODO: needs different port than 1025 to avoid conflicts
$(MAKE) serve-frontend &
$(MAKE) serve-backend
serve: serve-database serve-memcached serve-minio serve-frontend serve-backend;
.PHONY: compile
# HELP: compile frontend and backend
compile: compile-frontend compile-backend
compile: compile-frontend compile-backend;
.PHONY: lint
# HELP: lint frontend and backend
lint: lint-frontend lint-backend
lint: lint-frontend lint-backend;
.PHONY: test
# HELP: test frontend, backend, and check internationalization
test: test-frontend test-backend i18n-check
test: test-frontend test-backend i18n-check;
##### UNIFIED FRONTEND/BACKEND TARGETS #####
############################################
@ -120,8 +125,7 @@ test: test-frontend test-backend i18n-check
--%-frontend: --containerized-static-frontend --containerized-well-known-frontend;
# HELP(compile-frontend): compile frontend
--compile-frontend: static well-known
npm run build
--compile-frontend: static well-known;
.PHONY: serve-frontend
# HELP: serve frontend (watch file changes)
@ -181,16 +185,21 @@ well-known: static;
--%-prod-backend: stackopts=--flag uniworx:-dev
--%-prod-backend: --image-build --containerized-%-backend;
# TODO: to be deprecated by start-%
.PHONY: serve-backend
# HELP: serve backend
serve-backend:
DEV_PORT_HTTP=`docker/backend/dev_port.pl 3000 | tee .dev-port-http`; \
DEV_PORT_HTTPS=`docker/backend/dev_port.pl 3443 | tee .dev-port-https`; \
DEV_PORT_HTTP=`utils/next_free_port.pl 3000 | tee .dev-port-http`; \
DEV_PORT_HTTPS=`utils/next_free_port.pl 3443 | tee .dev-port-https`; \
$(MAKE) -- --containerized---serve-dev-backend DEV_PORT_HTTP=$${DEV_PORT_HTTP} DEV_PORT_HTTPS=$${DEV_PORT_HTTPS}
# DEV_PORT_HTTP=`grep 'https:' develop/$(DATE)/backend/$(CONTAINER_ID) | sed 's/.*://'`; \
# DEV_PORT_HTTPS=`grep 'https:' develop/$(DATE)/backend/$(CONTAINER_ID) | sed 's/.*://'`; \
# DEVELOP=$(MAKE) develop/
--serve-dev-backend: start.sh
DEV_PORT_HTTP=`cat .dev-port-http`; \
DEV_PORT_HTTPS=`cat .dev-port-https`; \
./start.sh
# ./utils/watchrun.sh develop/ ./start.sh
# HELP(compile-backend): compile backend
--compile-backend:
@ -206,7 +215,6 @@ serve-backend:
# HELP(db-cf-backend): clear and fill database. requires running postgres
# TODO (db-m-$MIGRATION-backend): apply migration (see src/Model/Migration/Definition.hs for list of available migrations)
PHONY: --db-%-backend
--db-%-dev-backend: .stack
SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true} ; \
AVSPASS=${AVSPASS:-nopasswordset} ; \
@ -225,6 +233,7 @@ PHONY: --db-%-backend
############################
##### DATABASE TARGETS #####
# TODO: to be deprecated by start-%
.PHONY: serve-database
# HELP: serve database
serve-database: --containerized-database;
@ -236,14 +245,6 @@ serve-database: --containerized-database;
$(CONTAINER_RUN) --name fradrive.$(FRADRIVE_SERVICE).$$(date +'%Y-%m-%dT%H-%M-%S') fradrive/$(FRADRIVE_SERVICE) ; \
fi
.PHONY: serve-memcached
serve-memcached:
$(CONTAINER_RUN) $(MEMCACHED_IMAGE)
.PHONY: serve-minio
serve-minio:
$(CONTAINER_RUN) $(MINIO_IMAGE) -- server $$(mktemp)
##### DATABASE TARGETS #####
############################
@ -251,7 +252,6 @@ serve-minio:
#############################
##### CONTAINER TARGETS #####
.PHONY: --containerized-%-frontend
--containerized-%-frontend: FRADRIVE_SERVICE=frontend
--containerized-%-frontend: --image-build
$(MAKE) -- --image-run-$*-frontend
@ -260,6 +260,14 @@ serve-minio:
--containerized-%-backend: --image-build
$(MAKE) -- --image-run-$*-backend
--containerized-%-minio: FRADRIVE_SERVICE=minio
--containerized-%-minio: --image-build
$(MAKE) -- --image-run-$*-minio
--containerized-%-memcached: FRADRIVE_SERVICE=memcached
--containerized-%-memcached: --image-build
$(MAKE) -- --image-run-$*-memcached
.PHONY: image-rebuild
# HELP: rebuild the stated docker image (frontend, backend, database instead of %)
image-rebuild-%:
@ -281,13 +289,127 @@ image-rebuild-%:
if [ "$(IN_CONTAINER)" == "true" ] ; then \
$(MAKE) -- $* ; \
else \
$(CONTAINER_RUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env WATCH=$(WATCH) --name fradrive.$(FRADRIVE_SERVICE).$$(date +'%Y-%m-%dT%H-%M-%S') localhost/fradrive/$(FRADRIVE_SERVICE) ; \
CONTAINER_ID=`$(CONTAINER_RUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(FRADRIVE_SERVICE).$$(date +'%Y-%m-%dT%H-%M-%S') localhost/fradrive/$(FRADRIVE_SERVICE)` ; \
echo "CONTAINER_ID=$${CONTAINER_ID}" >> $(CONTAINER_FILE); \
fi
# .PHONY: new-backend
# new-backend:
# LAST_DATE=`ls -1 develop | tail -n1`
# if [[ -z "$${LAST_DATE}" || -e `ls develop/$${LAST_DATE}/database` ]]; then \
# mkdir -p develop/$$(date +'%Y-%m-%dT%H-%M-%S') \
# fi
# serve-backend
#new-database:
# mkdir -p develop/$$(date +'%Y-%m-%dT%H-%M-%S')
# $(MAKE) serve-database
# TODO: work in progress
# - compute used ports and provide as env vars
# - start container in detached mode
# - save container id with ports as content as file
# .PHONY: serve-%
# serve-backend:
# DEV_PORT_HTTP=`docker/backend/dev_port.pl 3000 | tee develop/$${DATE}/fradrive.backend.$${DATE}`; \
# DEV_PORT_HTTPS=`docker/backend/dev_port.pl 3443 | tee .dev-port-https`; \
# $(MAKE) -- --containerized---start-dev-backend DEV_PORT_HTTP=$${DEV_PORT_HTTP} DEV_PORT_HTTPS=$${DEV_PORT_HTTPS}
#--serve-dev-backend: start.sh
# DEV_PORT_HTTP=`grep 'https:' develop/$(DATE)/backend/$(CONTAINER_ID) | sed 's/.*://'`; \
# DEV_PORT_HTTPS=`grep 'https:' develop/$(DATE)/backend/$(CONTAINER_ID) | sed 's/.*://'`; \
# ./utils/watchrun.sh develop/ ./start.sh
#CURRENT_DEVELOP = $(shell mkdir -p develop && (ls -1 develop | tail -n1))
CURRENT_DEVELOP = $(shell if [[ ! -e develop ]] ; then mkdir develop; $(MAKE) develop ; fi && (ls -1 develop | tail -n1))
DEVELOP = develop/$(CURRENT_DEVELOP)
DATE := $(shell date +'%Y-%m-%dT%H-%M-%S')
SET_DEVELOP = $(eval DEVELOP=develop/$$(DATE))
.PHONY: develop
develop:
$(SET_DEVELOP)
mkdir -p develop/$(DATE)
--ensure-develop:
if ! [[ -e develop ]]; then \
$(MAKE) develop; \
fi
start: develop start-database start-memcached start-minio start-frontend start-backend;
start-database: BASE_PORTS = "PGPORT=5432"
start-database: SINGLETON = true
start-memcached: BASE_PORTS = "MEMCACHED_PORT=11211"
start-memcached: SINGLETON = true
start-minio: BASE_PORTS = "UPLOAD_S3_PORT=9000"
start-minio: SINGLETON = true
start-backend: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443"
start-backend: SINGLETON = false
start-frontend: SINGLETON = false
start-%: FRADRIVE_SERVICE = %
start-%: PORTS = $(foreach PORT,$(BASE_PORTS),$(shell utils/next_free_port.pl $(PORT)))
start-%: --ensure-develop
echo "$*"
echo "$(DATE)"
echo "DEV $(DEVELOP)"
if [[ "$(SINGLETON)" = "true" ]]; then \
CONTAINER_FILE=$(DEVELOP)/$* ; \
else \
DEVDIR=$(DEVELOP)/$*; \
mkdir -p $${DEVDIR}; \
I=`ls $${DEVDIR} | sort -n | tail -n1`; \
echo "$${I}"; \
CONTAINER_FILE=$${DEVDIR}/$$(($${I}+1)); \
fi ; \
echo "$(PORTS)" | sed 's/ /\n/g' > $${CONTAINER_FILE} ; \
$(MAKE) -- --containerized---start-$* CONTAINER_FILE=$${CONTAINER_FILE}
--start-backend: start.sh
DEV_PORT_HTTP=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTP=' | sed 's/DEV_PORT_HTTP=//'`; \
DEV_PORT_HTTPS=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTPS=' | sed 's/DEV_PORT_HTTPS=//'`; \
./utils/watchrun-container.sh "$(CONTAINER_FILE)" ./start.sh
--containerized---start-database: FRADRIVE_SERVICE=database
# port forwarding is disabled in --network=host mode; nevertheless it is stated here for documentation reasons
--containerized---start-database: docker/database/initdb.sh docker/database/pg_hba.conf docker/database/postgresql.conf docker/database/schema.sql --image-build
if [ "$(IN_CONTAINER)" == "false" ] ; then \
$(CONTAINER_RUN) --name fradrive.$(FRADRIVE_SERVICE).$$(date +'%Y-%m-%dT%H-%M-%S') fradrive/$(FRADRIVE_SERVICE) ; \
fi
# TODO: mkdir inside develop/$(DATE)/minio instead of mktemp
# TODO: remove dir in stop-minio
#--containerized---start-minio:
# MINIO_DIR=`mktemp` ; \
# echo "MINIO_DIR=$${MINIO_DIR}" > $(DEVELOP)/minio ; \
# CONTAINER_ID=`$(CONTAINER_RUN) -v $(PWD):/mnt/fradrive --file docker/minio/Dockerfile -- (server $${MINIO_DIR})` ; \
# echo "CONTAINER_ID=$${CONTAINER_ID}" >> $(DEVELOP)/minio
# TODO: work in progress
# - kill container and remove file from .develop
# TODO: find way to stop a single container, e.g. by removing the file
.PHONY: stop-%
stop-%: --stop-%
$(MAKE) stop-container CONTAINER_FILE=TODO
.PHONY: --stop-minio
--stop-minio:
MINIO_DIR=`cat $(DEVELOP)/minio`
rm -rf $${MINIO_DIR}
stop-container-by-id:
podman stop $(CONTAINER_ID)
stop-container-by-file:
CONTAINER_ID=`grep 'CONTAINER_ID=' $(CONTAINER_FILE) | sed 's/CONTAINER_ID=//'` ; \
$(MAKE) stop-container-by-id CONTAINER_ID=$${CONTAINER_ID}
##### CONTAINER TARGETS #####
#############################
# TODO: move targets below to better location
.PHONY: i18n-check
# HELP: check internationalization
i18n-check: --image-run---i18n-check
@ -298,3 +420,5 @@ i18n-check: --image-run---i18n-check
%.lock:
[ -e $@ ] || touch $@
flock -en $@ true
.PHONY: --%

2
docker/backend/Dockerfile
View File

@ -23,7 +23,7 @@ RUN make .stack STACK_ROOT=${STACK_ROOT} IN_CONTAINER=true
RUN stack build yesod-bin
ENV FRADRIVE_MAKE_TARGET=serve-backend
ENTRYPOINT make -- ${FRADRIVE_MAKE_TARGET} STACK_ROOT="${STACK_ROOT}" IN_CONTAINER=true
ENTRYPOINT cat ${CONTAINER_FILE} > /container-file && make -- ${FRADRIVE_MAKE_TARGET} STACK_ROOT="${STACK_ROOT}" IN_CONTAINER=true CONTAINER_FILE="${CONTAINER_FILE}"
EXPOSE 3000/tcp
EXPOSE 3443/tcp

47
docker/backend/start.sh Executable file
View File

@ -0,0 +1,47 @@
#!/usr/bin/env bash
set -e
# SPDX-FileCopyrightText: 2022-2024 Sarah Vaupel <sarah.vaupel@uniworx.de>, Gregor Kleen <gregor.kleen@ifi.lmu.de>, Sarah Vaupel <vaupel.sarah@campus.lmu.de>, Steffen Jost <jost@tcs.ifi.lmu.de>
#
# SPDX-License-Identifier: AGPL-3.0-or-later
__HOST=${HOST:-$(hostname -s | awk '{ print $0; }')}
export DETAILED_LOGGING=${DETAILED_LOGGING:-true}
export LOG_ALL=${LOG_ALL:-false}
export LOGLEVEL=${LOGLEVEL:-info}
export DUMMY_LOGIN=${DUMMY_LOGIN:-true}
export SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true}
export SERVER_SESSION_COOKIES_SECURE=${SERVER_SESSION_COOKIES_SECURE:-false}
export COOKIES_SECURE=${COOKIES_SECURE:-false}
export ALLOW_DEPRECATED=${ALLOW_DEPRECATED:-true}
export ENCRYPT_ERRORS=${ENCRYPT_ERRORS:-false}
export RIBBON=${RIBBON:-${__HOST:-localhost}}
export APPROOT=${APPROOT:-http://localhost:$DEV_PORT_HTTP}
export AVSPASS=${AVSPASS:-nopasswordset}
unset HOST
# export PORT_OFFSET=$(((16#$(sha256sum <<<"$(hostname -f):''${basePath}" | head -c 16)) % 1000))
# move-back() {
# mv -vT .stack-work .stack-work-run
# [[ -d .stack-work-build ]] && mv -vT .stack-work-build .stack-work
# }
# if [[ -d .stack-work-run ]]; then
# [[ -d .stack-work ]] && mv -vT .stack-work .stack-work-build
# mv -vT .stack-work-run .stack-work
# trap move-back EXIT
# fi
if ! [ -z "$(which yesod)" ]
then
yesod devel -p "$DEV_PORT_HTTP" -q "$DEV_PORT_HTTPS" $@
elif ! [ -z "$(which stack)" ]
then
stack exec -- yesod devel -p "$DEV_PORT_HTTP" -q "$DEV_PORT_HTTPS" $@
else
exit 1
fi

4
docker/memcached/Dockerfile Normal file
View File

@ -0,0 +1,4 @@
FROM docker.io/memcached:latest
USER root
ENTRYPOINT /mnt/fradrive/utils/watchrun.sh "/mnt/fradrive/${CONTAINER_FILE}" "su memcache -c /usr/local/bin/docker-entrypoint.sh"

6
docker/minio/Dockerfile Normal file
View File

@ -0,0 +1,6 @@
FROM docker.io/minio/minio:latest
ENV MINIO_DIR = $(mktemp)
WORKDIR /mnt/fradrive
ENTRYPOINT ./utils/watchrun.sh "${CONTAINER_FILE}" "/usr/bin/docker-entrypoint.sh server ${MINIO_DIR}" "rm -rf ${MINIO_DIR}"

6
utils/next_free_port.pl
View File

@ -3,7 +3,9 @@
use strict;
use warnings;
my $port = $ARGV[0];
my $portDef = $ARGV[0];
$portDef=~m/^(\D*)(\d+)(.*)/ or die "$0: We need a start port as parameter but got none!\n";
my ($pre, $port, $post) = ($1, $2, $3);
my $used_ports = `netstat -tulan`;
my %p=();
@ -16,4 +18,4 @@ for(split m/\R/, $used_ports) {
$port++ while $p{$port};
print $port
print "$pre$port$post"

5
utils/runwrapper.sh Executable file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env bash
echo $$ > /tmp/pidwrapper
$1

5
utils/watchrun-container.sh Executable file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env bash
STOPSCRIPT="make stop-container-by-id CONTAINFER_FILE=${CONTAINER_FILE} ; $3"
./utils/watchrun.sh "$1" "$2"

9
utils/watchrun.sh
View File

@ -6,13 +6,14 @@ STOPSCRIPT=$3
touch "$FILENAME"
$STARTSCRIPT &
(
while [ -e "$FILENAME" ] ; do
inotifywait -e ATTRIB -t 10 "$FILENAME" > /dev/null 2&>/dev/null
done
kill %1
$STOPSCRIPT
kill `cat /tmp/pidwrapper`
) &
/mnt/fradrive/utils/runwrapper.sh "$STARTSCRIPT"