diff --git a/.gitignore b/.gitignore index eec19fc73..c6e28b106 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ bin/ cabal-dev/ .cache/ .stack/ -.stack-work/ +.stack-work .dev-port-http .dev-port-https .bash_history diff --git a/Makefile b/Makefile index 718cd320a..4773dfc59 100644 --- a/Makefile +++ b/Makefile @@ -1,51 +1,55 @@ -include .gnumake/*.Mak -SHELL=bash -# MAKE=make -d +export SHELL=bash + +# System information +export CPU_CORES = $(shell cat /proc/cpuinfo | grep '^processor' | wc -l) export CONTAINER_COMMAND ?= podman export CONTAINER_BGRUN ?= $(CONTAINER_COMMAND) run -dit --network=host --replace export CONTAINER_FGRUN ?= $(CONTAINER_COMMAND) run -it --network=host --replace -export CONTAINER_NAME ?= $(FRADRIVE_SERVICE) -export ENTRYPOINT - -export IN_CONTAINER ?= false -export IN_CI ?= false - -export WATCH -export UNIWORXDB_OPTS ?= -cf - -export CONTAINER_ID -export FRADRIVE_SERVICE - -export DEV_PORT_HTTP -export DEV_PORT_HTTPS -export HOOGLE_PORT - export IMAGE_REGISTRY = docker.io export MEMCACHED_IMAGE = $(IMAGE_REGISTRY)/memcached:latest export MINIO_IMAGE = $(IMAGE_REGISTRY)/minio/minio:latest export MAILDEV_IMAGE = $(IMAGE_REGISTRY)/maildev/maildev:latest # TODO: needs different port than 1025 to avoid conflicts -export DEVELOP +export IN_CONTAINER ?= false +export IN_CI ?= false export CONTAINER_FILE -export CONTAINER_FILE_CONTENT +export CONTAINER_IDENT +export CF_PREFIX +export DEVELOP +export MOUNT_DIR=/mnt/fradrive +export CONTAINER_ATTACHED +export CONTAINER_INIT +export CONTAINER_CLEANUP +export STACK_INIT="./utils/stack-work-init.sh" +export STACK_CLEANUP="rm -f .stack-work.lock" + +export SERVICE +export SERVICE_VARIANT ?= $(SERVICE) +export JOB +export CONTAINER_CMD +export SET_CONTAINER_CMD +export ENTRYPOINT +export EXEC_OPTS + +export STACK_CORES = $(shell echo $(($(CPU_CORES)/2))) +export BASE_PORTS +export UNIWORXDB_OPTS ?= -cf export PROD ?= false -export LOGSIZE ?= 1024 -# HELP HEADER START -# To see the definition of all available targets, take a look into the Makefile. -# Targets starting with '--' are not meant to be directly called. -# If you want to do so anyway please use 'make -- --target' to avoid '--target' being treated as option to 'make'. -# -# -# Targets meant to be used by humans are: -# HELP HEADER END +ifneq ($(PROD),true) + export --DEVELOPMENT=--flag=uniworx:dev +endif + +export DATE := $(shell date +'%Y-%m-%dT%H-%M-%S') + +export CURR_DEV = $(shell cat develop/.current 2>/dev/null) +export SET_DEVELOP = $(eval DEVELOP=develop/$$(CURR_DEV)) +export NEW_DEVELOP = $(eval DEVELOP=develop/$$(DATE)) -########################### -##### GENERAL TARGETS ##### .PHONY: help # HELP: print out this help message @@ -59,9 +63,9 @@ help: .PHONY: clean # HELP: stop all running containers and remove all compilation results in the directory (but leave images including dependencies unharmed) clean: - $(MAKE) stop + rm -rf develop -rm -rf node_modules .npm .cache assets/icons assets/favicons static well-known config/manifest.json - -rm -rf .stack-work .stack-work-run .stack-work-test .stack-work-doc .stack-work.lock + -rm -rf .stack-work .stack-work.lock .stack-work-compile .stack-work-start .stack-work-test .stack-work-lint .stack-work-hoogle -rm -rf bin .Dockerfile develop -$(CONTAINER_COMMAND) container prune --force .PHONY: clean-all @@ -80,279 +84,245 @@ release: VERSION=`.gitlab-ci/version.pl` git tag $${VERSION} git commit -m "chore(release): $${VERSION}" - git push - -.PHONY: %-shell -# HELP: launch shell (bash) inside a currently running container -%-shell: --%-shell; ---%-shell: - $(CONTAINER_COMMAND) exec -it $(EXEC_OPTS) fradrive.$(CURR_DEV).$* $(if $(ENTRYPOINT),$(ENTRYPOINT),/bin/bash) - -##### GENERAL TARGETS ##### -########################### - - -############################################ -##### UNIFIED FRONTEND/BACKEND TARGETS ##### - -.PHONY: start -# HELP: start database services, frontend and backend -start: new-develop start-database start-memcached start-minio start-frontend start-backend; +# git push .PHONY: compile -# HELP: compile frontend and backend compile: $(MAKE) compile-frontend $(MAKE) compile-backend -.PHONY: lint -# HELP: lint frontend and backend -lint: lint-frontend lint-backend; - -.PHONY: test -# HELP: test frontend, backend, and check internationalization -test: start-database test-frontend test-backend i18n-check; - -##### UNIFIED FRONTEND/BACKEND TARGETS ##### -############################################ - - -############################ -##### FRONTEND TARGETS ##### - -.PHONY: %-frontend -%-frontend: FRADRIVE_SERVICE=frontend -%-frontend: --image-build --containerized---%-frontend; - ---%-frontend: --containerized-static-frontend --containerized-well-known-frontend - ---start-frontend: static - npm run start - -# HELP(compile-frontend): compile frontend ---compile-frontend: static well-known; - -# HELP(lint-frontend): lint frontend ---lint-frontend: eslint.config.js - npx -- eslint frontend/src $(FIX) - @echo Hooray! There are no hints. - -# HELP(test-frontend): test frontend ---test-frontend: karma.conf.cjs - @echo Karma frontend tests are currently broken after npm update and have therefor been temporarily disabled. -# npx -- karma start --conf karma.conf.cjs $(WATCH) - -node_modules: package.json package-lock.json - npm ci --cache .npm --prefer-offline - -package-lock.json: package.json - npm install --cache .npm --prefer-offline - -assets: assets/favicons assets/icons; -assets/favicons: - ./utils/faviconize.pl assets/favicon.svg long assets/favicons -assets/icons: node_modules - ./utils/renamer.pl node_modules/@fortawesome/fontawesome-free/svgs/solid utils/rename-fa.json assets/icons/fradrive - ./utils/renamer.pl node_modules/@fortawesome/fontawesome-free/svgs/regular utils/rename-fa.json assets/icons/fradrive - -static: node_modules assets esbuild.config.mjs - npm run build -well-known: static; - -##### FRONTEND TARGETS ##### -############################ - - -########################### -##### BACKEND TARGETS ##### +.PHONY: start +start: + $(MAKE) start-postgres + $(MAKE) start-memcached + $(MAKE) start-minio + $(MAKE) start-frontend + $(MAKE) start-backend .PHONY: %-backend -%-backend: FRADRIVE_SERVICE=backend -%-backend: --image-build --containerized---%-backend; +%-backend: SERVICE=backend +%-backend: SERVICE_VARIANT=backend +%-backend: CONTAINER_CMD=localhost/fradrive/backend +%-backend: CONTAINER_INIT="$(STACK_INIT)" +%-backend: CONTAINER_CLEANUP="$(STACK_CLEANUP)" +%-backend: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443" ---start-backend: - DEV_PORT_HTTP=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTP=' | sed 's/DEV_PORT_HTTP=//'`; \ - DEV_PORT_HTTPS=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTPS=' | sed 's/DEV_PORT_HTTPS=//'`; \ - stack --work-dir=.stack-work-run exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" ---start-hoogle-backend: +.PHONY: %-uniworxdb +%-uniworxdb: SERVICE=backend +%-uniworxdb: SERVICE_VARIANT=uniworxdb +%-uniworxdb: CONTAINER_CMD=localhost/fradrive/backend +%-uniworxdb: CONTAINER_INIT="$(STACK_INIT)" +%-uniworxdb: CONTAINER_CLEANUP="$(STACK_CLEANUP)" + +.PHONY: %-hoogle +%-hoogle: SERVICE=backend +%-hoogle: SERVICE_VARIANT=hoogle +%-hoogle: BASE_PORTS = "HOOGLE_PORT=8081" +%-hoogle: CONTAINER_CMD=localhost/fradrive/backend +%-hoogle: CONTAINER_INIT="$(STACK_INIT)" +%-hoogle: CONTAINER_CLEANUP="$(STACK_CLEANUP)" +--start-hoogle: HOOGLE_PORT=`cat $(CONTAINER_FILE) | grep 'HOOGLE_PORT=' | sed 's/HOOGLE_PORT=//'` ; \ - echo "${HOOGLE_PORT}" ; \ - stack --work-dir=.stack-work-doc hoogle -- server --local --port $${HOOGLE_PORT} + stack $(STACK_CORES) hoogle -- server --local --port $${HOOGLE_PORT} -# HELP(compile-backend): compile backend ---compile-backend: - stack build --fast --profile --library-profiling --executable-profiling --flag uniworx:-library-only --local-bin-path $$(pwd)/bin $(stackopts) +.PHONY: %-frontend +%-frontend: SERVICE=frontend +%-frontend: SERVICE_VARIANT=frontend +%-frontend: CONTAINER_CMD=localhost/fradrive/frontend -# HELP(lint-backend): lint backend ---lint-backend: - stack build --test --fast --work-dir=.stack-work-test --flag uniworx:library-only uniworx:test:hlint $(stackopts) +.PHONY: %-postgres +%-postgres: SERVICE=postgres +%-postgres: SERVICE_VARIANT=postgres +%-postgres: BASE_PORTS = "PGPORT=5432" +%-postgres: CONTAINER_CMD=localhost/fradrive/postgres -# HELP(test-backend): test backend ---test-backend: - stack build --test --coverage --fast --work-dir=.stack-work-test --flag uniworx:library-only $(stackopts) +.PHONY: %-memcached +%-memcached: SERVICE=memcached +%-memcached: SERVICE_VARIANT=memcached +%-memcached: SET_CONTAINER_CMD=$$(MEMCACHED_IMAGE) --port=`cat $$(CONTAINER_FILE) | grep 'MEMCACHED_PORT=' | sed 's/MEMCACHED_PORT=//'` +%-memcached: BASE_PORTS = "MEMCACHED_PORT=11211" -# TODO: better name -.PHONY: db -# HELP: clear and fill database. requires running postgres -db: FRADRIVE_SERVICE=backend -db: CONTAINER_NAME=uniworxdb -db: compile-frontend --image-build --containerized---db; -# TODO (db-m-$MIGRATION-backend): apply migration (see src/Model/Migration/Definition.hs for list of available migrations) ---db-backend: .stack --compile-backend - SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true} ; \ - AVSPASS=${AVSPASS:-nopasswordset} ; \ - stack exec uniworxdb -- $(UNIWORXDB_OPTS) +.PHONY: %-minio +%-minio: SERVICE=minio +%-minio: SERVICE_VARIANT=minio +%-minio: SET_CONTAINER_CMD=$$(MINIO_IMAGE) -- server `mktemp` --address=:`cat $$(CONTAINER_FILE) | grep 'UPLOAD_S3_PORT=' | sed 's/UPLOAD_S3_PORT=//'` +%-minio: BASE_PORTS = "UPLOAD_S3_PORT=9000" -.PHONY: .stack -.stack: - if [ "$(IN_CONTAINER)" == "true" ] ; then \ - $(MAKE) -- --.stack ; \ - else \ - $(MAKE) -- --image-run---.stack ; \ - fi ---.stack: stack.yaml stack.yaml.lock package.yaml - stack build --fast --only-dependencies $(stackopts) +.PHONY: start-% +start-%: JOB=start +start-%: CF_PREFIX = start- +start-%: CONTAINER_ATTACHED = false +start-%: --act ; -##### BACKEND TARGETS ##### -########################### +.PHONY: compile-% +compile-%: JOB=compile +compile-%: CF_PREFIX = compile- +compile-%: CONTAINER_ATTACHED = true +compile-%: --act ; +.PHONY: dependencies-% +dependencies-%: JOB=dependencies +dependencies-%: CF_PREFIX = dependencies- +dependencies-%: CONTAINER_ATTACHED = true +dependencies-%: --act ; -############################ -##### DATABASE TARGETS ##### +.PHONY: test-% +test-%: JOB=test +test-%: CF_PREFIX = test- +test-%: CONTAINER_ATTACHED = true +test-%: --act ; ---containerized---db: CONTAINER_NAME=uniworxdb ---containerized---db: --containerized-db-backend; +.PHONY: lint-% +lint-%: JOB=lint +lint-%: CF_PREFIX = lint- +lint-%: CONTAINER_ATTACHED = true +lint-%: --act ; ---containerized-%-database: FRADRIVE_SERVICE=database ---containerized-%-database: CONTAINER_NAME=database ---containerized-%-database: docker/database/initdb.sh docker/database/pg_hba.conf docker/database/postgresql.conf docker/database/schema.sql --image-build - $(MAKE) -- --image-run-$*-database +--act: --develop_containerized; -.PHONY: psql -# HELP: enter psql (postgresql) cli inside a currently running database container -psql: ENTRYPOINT=/usr/bin/psql -d uniworx -psql: EXEC_OPTS=--user postgres -psql: --database-shell; +--develop_%: PORTS = $(foreach PORT,$(BASE_PORTS),$(shell utils/next_free_port.pl $(PORT))) +--develop_%: --ensure-develop + DEVELOP=develop/`cat develop/.current` ; \ + CONTAINER_IDENT=$(CF_PREFIX)$(SERVICE_VARIANT) ; \ + CONTAINER_FILE=$${DEVELOP}/$${CONTAINER_IDENT} ; \ + if [[ -e $${CONTAINER_FILE} ]]; then \ + >&2 echo "Another $* service is already running! Use \"make new-develop\" to start a new develop instance despite currently running services." ; \ + exit 1 ; \ + fi ; \ + echo "$(PORTS)" | sed 's/ /\n/g' > $${CONTAINER_FILE} ; \ + $(MAKE) -- --$* CONTAINER_FILE=$${CONTAINER_FILE} CONTAINER_IDENT=$${CONTAINER_IDENT} -##### DATABASE TARGETS ##### -############################ - - -############################# -##### CONTAINER TARGETS ##### - ---containerized-%-frontend: FRADRIVE_SERVICE=frontend ---containerized-%-frontend: CONTAINER_NAME=frontend ---containerized-%-frontend: --image-build - $(MAKE) -- --image-run-$*-frontend - ---containerized-%-backend: FRADRIVE_SERVICE=backend ---containerized-%-backend: CONTAINER_NAME?=backend ---containerized-%-backend: --image-build - $(MAKE) -- --image-run-$*-backend ---containerized-%-hoogle: FRADRIVE_SERVICE=backend ---containerized-%-hoogle: CONTAINER_NAME=hoogle ---containerized-%-hoogle: --image-build - $(MAKE) -- --image-run-$*-hoogle-backend - -# --containerized-%-minio: FRADRIVE_SERVICE=minio -# --containerized-%-minio: --image-build -# $(MAKE) -- --image-run-$*-minio ---containerized---start-minio: - UPLOAD_S3_PORT=`cat $(CONTAINER_FILE) | grep 'UPLOAD_S3_PORT=' | sed 's/UPLOAD_S3_PORT=//'`; \ - MINIO_DIR=`mktemp` ; \ - ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" "" "rm -rf $${MINIO_DIR}" & \ - CONTAINER_ID=`$(CONTAINER_BGRUN) --name fradrive.$(CURR_DEV).minio $(MINIO_IMAGE) -- server $${MINIO_DIR} --address=:$${UPLOAD_S3_PORT}` ; \ - printf "CONTAINER_ID=$${CONTAINER_ID}\nUPLOAD_S3_PORT=$${UPLOAD_S3_PORT}\nMINIO_DIR=$${MINIO_DIR}" >> $(CONTAINER_FILE) - -# --containerized-%-memcached: FRADRIVE_SERVICE=memcached -# --containerized-%-memcached: --image-build -# $(MAKE) -- --image-run-$*-memcached ---containerized---start-memcached: - MEMCACHED_PORT=`cat $(CONTAINER_FILE) | grep 'MEMCACHED_PORT=' | sed 's/MEMCACHED_PORT=//'`; \ - ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ - CONTAINER_ID=`$(CONTAINER_BGRUN) --name fradrive.$(CURR_DEV).memcached $(MEMCACHED_IMAGE) --port=$${MEMCACHED_PORT}` ; \ - printf "CONTAINER_ID=$${CONTAINER_ID}\nMEMCACHED_PORT=$${MEMCACHED_PORT}" >> $(CONTAINER_FILE) - -.PHONY: image-rebuild -# HELP(image-rebuild-{backend,frontend,database,memcached,minio}): force-rebuild the stated docker image -image-rebuild-%: - $(MAKE) -- --image-build FRADRIVE_SERVICE=$* NO_CACHE=--no-cache +.PHONY: image-rebuild_% +# HELP(image-rebuild_{backend,frontend,database,memcached,minio}): force-rebuild the stated docker image +image-rebuild_%: + $(MAKE) -- --image-build SERVICE=$* NO_CACHE=--no-cache --image-build: +ifeq "$(CONTAINER_CMD)" "localhost/fradrive/$(SERVICE)" rm -f .Dockerfile - ln -s docker/$(FRADRIVE_SERVICE)/Dockerfile .Dockerfile + ln -s docker/$(SERVICE)/Dockerfile .Dockerfile MOUNT_DIR=/mnt/fradrive; \ PROJECT_DIR=/mnt/fradrive; \ if [ "$(IN_CI)" == "true" ] ; then \ PROJECT_DIR=/fradrive; \ fi; \ if [ "$(IN_CONTAINER)" == "false" ] ; then \ - $(CONTAINER_COMMAND) build $(NO_CACHE) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --build-arg MOUNT_DIR=$${MOUNT_DIR} --build-arg PROJECT_DIR=$${PROJECT_DIR} --tag fradrive/$(FRADRIVE_SERVICE) --file $(PWD)/.Dockerfile ; \ + $(CONTAINER_COMMAND) build $(NO_CACHE) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --build-arg MOUNT_DIR=$(MOUNT_DIR) --build-arg PROJECT_DIR=$${PROJECT_DIR} --tag fradrive/$(SERVICE) --file $(PWD)/.Dockerfile ; \ fi +else + : +endif ---image-run-%: CONTAINER_NAME ?= $(FRADRIVE_SERVICE) ---image-run-%: docker/$(FRADRIVE_SERVICE)/Dockerfile - MOUNT_DIR=/mnt/fradrive; \ - if [ "$(IN_CONTAINER)" == "true" ] ; then \ - $(MAKE) -- $* ; \ - else \ - if [ -z "$(CONTAINER_FILE)" ] ; then \ - $(CONTAINER_FGRUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(CONTAINER_NAME) localhost/fradrive/$(FRADRIVE_SERVICE) ; \ +--containerized: --image-build + DEVELOP=`cat develop/.current` ; \ + ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" "$(CONTAINER_INIT)" "$(CONTAINER_CLEANUP)" & \ + CONTAINER_NAME=fradrive.$(CURR_DEV).$(CONTAINER_IDENT) ; \ + if ! [ -z "$(SET_CONTAINER_CMD)" ] ; \ + then \ + CONTAINER_CMD="$(SET_CONTAINER_CMD)" ; \ else \ - ./utils/watchcontainerrun.sh "$(CONTAINER_COMMAND)" "$(CONTAINER_FILE)" & \ - CONTAINER_ID=`$(CONTAINER_BGRUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(CURR_DEV).$(CONTAINER_NAME) localhost/fradrive/$(FRADRIVE_SERVICE)` ; \ - echo "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)"; \ - fi \ + CONTAINER_CMD=$(CONTAINER_CMD) ; \ + fi ; \ + CONTAINER_ID=`$(CONTAINER_BGRUN) \ + -v $(PWD):$(MOUNT_DIR) \ + --env IN_CONTAINER=true \ + --env FRADRIVE_MAKE_TARGET="--$(JOB)-$(SERVICE_VARIANT)" \ + --env CONTAINER_FILE=$(CONTAINER_FILE) \ + --env CONTAINER_NAME=$${CONTAINER_NAME} \ + --name $${CONTAINER_NAME} \ + $${CONTAINER_CMD} \ + ` ; \ + printf "CONTAINER_ID=$${CONTAINER_ID}" >> "$(CONTAINER_FILE)" ; \ + if [[ "true" == "$(CONTAINER_ATTACHED)" ]] ; then \ + $(CONTAINER_COMMAND) attach $${CONTAINER_ID} || : ; \ fi +# HELP(start-backend): start yesod-devel instance +--start-backend: + DEV_PORT_HTTP=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTP=' | sed 's/DEV_PORT_HTTP=//'`; \ + DEV_PORT_HTTPS=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTPS=' | sed 's/DEV_PORT_HTTPS=//'`; \ + stack $(STACK_CORES) exec -- yesod devel -p "$${DEV_PORT_HTTP}" -q "$${DEV_PORT_HTTPS}" +# HELP(compile-backend): compile backend binaries +--compile-backend: + stack build $(STACK_CORES) --fast --profile --library-profiling --executable-profiling --flag uniworx:-library-only $(--DEVELOPMENT) --local-bin-path $$(pwd)/bin +# HELP(dependencies-backend): (re-)build backend dependencies +--dependencies-backend: stack.yaml stack.yaml.lock package.yaml + stack build $(STACK_CORES) --fast --only-dependencies +# HELP(lint-backend): lint backend +--lint-backend: + stack build $(STACK_CORES) --test --fast --flag uniworx:library-only $(--DEVELOPMENT) uniworx:test:hlint +# HELP(test-backend): test backend +--test-backend: + stack build $(STACK_CORES) --test --coverage --fast --flag uniworx:library-only $(--DEVELOPMENT) -# TODO: move starts below to respective entries, or leave together? +# HELP(compile-frontend): compile frontend assets +--compile-frontend: node_modules assets esbuild.config.mjs + npm run build +--start-frontend: --compile-frontend; +--dependencies-frontend: node_modules assets static well-known; +node_modules: package.json package-lock.json + npm ci --cache .npm --prefer-offline +package-lock.json: package.json + npm install --cache .npm --prefer-offline +assets: assets/favicons assets/icons; +assets/favicons: + ./utils/faviconize.pl assets/favicon.svg long assets/favicons +assets/icons: node_modules assets/icons-src/fontawesome.json + ./utils/renamer.pl node_modules/@fortawesome/fontawesome-free/svgs/solid assets/icons-src/fontawesome.json assets/icons/fradrive + ./utils/renamer.pl node_modules/@fortawesome/fontawesome-free/svgs/regular assets/icons-src/fontawesome.json assets/icons/fradrive + -cp assets/icons-src/*.svg assets/icons/fradrive +static: node_modules assets esbuild.config.mjs + npm run build +well-known: static; -start-database: BASE_PORTS = "PGPORT=5432" -start-database: SINGLETON = true +# HELP(compile-uniworxdb): clear and fill database. requires running postgres +# TODO (db-m-$MIGRATION-backend): apply migration (see src/Model/Migration/Definition.hs for list of available migrations) +--compile-uniworxdb: --compile-backend + SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true} ; \ + AVSPASS=${AVSPASS:-nopasswordset} ; \ + stack exec uniworxdb -- $(UNIWORXDB_OPTS) -start-memcached: BASE_PORTS = "MEMCACHED_PORT=11211" -start-memcached: SINGLETON = true +# HELP(start-minio): start minio service -start-minio: BASE_PORTS = "UPLOAD_S3_PORT=9000" -start-minio: SINGLETON = true +.PHONY: status +# HELP: print develop status: running containers, used ports +status: + @./utils/develop-status.pl -a -start-backend: BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443" -start-backend: SINGLETON = false -start-backend: compile-frontend - -start-hoogle: BASE_PORTS = "HOOGLE_PORT=8081" -start-hoogle: SINGLETON = true - -start-frontend: SINGLETON = false -start-frontend: WATCH = false - -# HELP(start-database): start postgres server -# HELP(start-memcached): start memcached server -# HELP(start-minio): start minio server -# HELP(start-backend): serve yesod development site -# HELP(start-frontend): start frontend watcher. Watches frontend source files for changes and recompiles on change. (TODO: watcher functionality currently broken!) -# HELP(start-hoogle): serve local hoogle instance -start-%: FRADRIVE_SERVICE = % -start-%: PORTS = $(foreach PORT,$(BASE_PORTS),$(shell utils/next_free_port.pl $(PORT))) -start-%: --develop - echo "$*" - echo "$(DATE)" - echo "DEV $(DEVELOP)" - if [[ "$(SINGLETON)" = "true" ]]; then \ - CONTAINER_FILE=$(DEVELOP)/$* ; \ - if [[ -e $${CONTAINER_FILE} ]]; then \ - >&2 echo "Another $* service is already running! Use \"make new-develop\" or \"make start\" to start a new develop instance despite currently running services." ; \ - exit 1 ; \ - fi \ - else \ - DEVDIR=$(DEVELOP)/$*; \ - I=`ls $${DEVDIR} | sort -n | tail -n1`; \ - echo "$${I}"; \ - CONTAINER_FILE=$${DEVDIR}-$$(($${I}+1)); \ +.PHONY: log-% +# HELP(log-$(JOB)-$(SERVICE)): inspect output of a given service. The service must be currently running When a service supports multiple running instances in one develop (i.e. backend), you need to specify the exact instance by its associated file (e.g. backend-1, backend-2, etc.), please check the contents of the develop/ directory for a list of running instances. +log-%: + DEVELOP=develop/`cat develop/.current` ; \ + SEARCH_FILE="$${DEVELOP}/$*" ; \ + if [[ ! -e "$${SEARCH_FILE}" ]] ; then \ + SEARCH_FILE="$${DEVELOP}/.exited.$*" ; \ fi ; \ - echo "$(PORTS)" | sed 's/ /\n/g' > $${CONTAINER_FILE} ; \ - $(MAKE) -- --containerized---start-$* CONTAINER_FILE=$${CONTAINER_FILE} + if [[ -e "$${SEARCH_FILE}" ]] ; then \ + $(CONTAINER_COMMAND) logs --follow `cat "$${SEARCH_FILE}" | grep CONTAINER_ID= | sed 's/^CONTAINER_ID=//'` ; \ + else \ + >&2 echo "Cannot show log: No develop file found for '$*'" ; \ + exit 1 ; \ + fi + +.PHONY: enter +# HELP: launch (bash) shell inside a currently running container. Use ./enter shell wrapper for more convenient usage, possibly with tab-completion in the future +enter: --ensure-develop + $(MAKE) -- --shell + +.PHONY: psql +# HELP: enter psql (postgresql) cli inside a currently running database container +psql: ENTRYPOINT=/usr/bin/psql -d uniworx +psql: EXEC_OPTS=--user postgres +psql: --ensure-develop + $(MAKE) -- --shell CONTAINER_FILE=develop/`cat develop/.current`/start-postgres + +.PHONY: ghci +# HELP: launch new backend instance and enter interactive ghci shell (WIP) +ghci: ENTRYPOINT=stack ghci +ghci: --shell; + +--shell: + CONTAINER_ID=`cat $(CONTAINER_FILE) | grep 'CONTAINER_ID=' | sed 's/CONTAINER_ID=//'` ; \ + $(CONTAINER_COMMAND) exec -it $(EXEC_OPTS) $${CONTAINER_ID} $(if $(ENTRYPOINT),$(ENTRYPOINT),/bin/bash) .PHONY: stop # HELP: stop all currently running develop instances @@ -370,60 +340,31 @@ stop-container-by-id: # CONTAINER_ID=`grep 'CONTAINER_ID=' $(CONTAINER_FILE) | sed 's/CONTAINER_ID=//'` ; \ # $(MAKE) stop-container-by-id CONTAINER_ID=$${CONTAINER_ID} -.PHONY: status -# HELP: print develop status: running containers, used ports -status: - @./utils/develop-status.pl - -.PHONY: log-% -# HELP(log-{database,memcached,minio,backend,frontend,hoogle}): inspect output of a given (currently running) service. When a service supports multiple running instances in one develop (i.e. backend), you need to specify the exact instance by its associated file (e.g. backend-1, backend-2, etc.), please check the contents of the develop/ directory for a list of running instances. -log-%: --develop - $(CONTAINER_COMMAND) logs --follow --tail=$(LOGSIZE) `cat $(DEVELOP)/$* | grep CONTAINER_ID= | sed 's/^CONTAINER_ID=//'` - -##### CONTAINER TARGETS ##### -############################# - - -# TODO: move targets below to better location - -DATE := $(shell date +'%Y-%m-%dT%H-%M-%S') - -CURR_DEV = $(shell ls -1 develop | tail -n1) -SET_DEVELOP = $(eval DEVELOP=develop/$$(CURR_DEV)) -NEW_DEVELOP = $(eval DEVELOP=develop/$$(DATE)) - .PHONY: new-develop +# HELP: instantiate new development bundle, i.e. create new directory under develop/ new-develop: $(NEW_DEVELOP) mkdir -p $(DEVELOP) $(MAKE) develop/.current -.PHONY: --develop ---develop: +.PHONY: switch-develop +# HELP: switch current develop instance to DEVELOP=... +switch-develop: + if ! [ -e develop/$(DEVELOP) ]; then \ + echo "Specified develop $(DEVELOP) does not exist! Not switching." ; \ + exit 1 ; \ + fi ; \ + echo "$(DEVELOP)" > develop/.current +--ensure-develop: if ! [[ -e develop ]]; then \ $(MAKE) new-develop; \ fi - $(SET_DEVELOP) $(MAKE) develop/.current + $(SET_DEVELOP) .PHONY: develop/.current develop/.current: - $(SET_DEVELOP) - echo "$(DEVELOP)" > develop/.current + ls -1 develop | tail -n1 > develop/.current -# Some convenience aliases: -.PHONY: hoogle -# HELP: alias for start-hoogle -hoogle: start-hoogle; - -.PHONY: i18n-check -# HELP: check internationalization -i18n-check: --image-run---i18n-check ---i18n-check: - ./missing-translations.sh - @echo No missing translations. - -%.lock: - [ -e $@ ] || touch $@ - flock -en $@ true +stack.yaml.lock: --dependencies-backend; .PHONY: --% -.SUFFIXES: # Delete all default suffixes \ No newline at end of file +.SUFFIXES: # Delete all default suffixes diff --git a/utils/rename-fa.json b/assets/icons-src/fontawesome.json similarity index 100% rename from utils/rename-fa.json rename to assets/icons-src/fontawesome.json diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml new file mode 100644 index 000000000..7da1e371e --- /dev/null +++ b/azure-pipelines.yaml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: 2024 Sarah Vaupel +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +pool: 'Prod Private Agent Pool' + +jobs: +# - job: HelloWorld +# container: +# image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' +# endpoint: devfra +# steps: +# - script: echo Hello, world! +# displayName: 'Run a one-line script' +# - script: | +# echo Add other tasks to build, test, and deploy your project. +# echo See https://aka.ms/yaml +# displayName: 'Run a multi-line script' +# - job: DockerTaskTest +# container: +# image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' +# endpoint: devfra +# steps: +# - task: Docker@2 +# displayName: Image build test +# inputs: +# command: buildAndPush +# Dockerfile: docker/backend/Dockerfile +# buildContext: . +# tags: backend +# - job: BuildKitTest +# container: +# image: 'devfra.azurecr.io/de.fraport.trusted/buildkit:0.12.1' +# endpoint: devfra +# steps: +# - script: buildctl build \ +# --frontend=dockerfile.v0 \ +# --local context=. \ +# --local dockerfile=docker/backend/Dockerfile +# displayName: BuildKit test +- job: CustomBuildahTest + container: + image: 'devfra.azurecr.io/de.fraport.trusted/ubuntu:22.04' + endpoint: devfra + steps: + - script: | + id + docker build --help + sudo apt-get -y update + sudo apt-get -y install buildah + buildah bud -t fradrive-backend-test --volume .:/mnt/fradrive --file docker/backend/Dockerfile + displayName: Build buildah image \ No newline at end of file diff --git a/config/keter.yml b/config/keter.yml deleted file mode 120000 index 3b8c9db84..000000000 --- a/config/keter.yml +++ /dev/null @@ -1 +0,0 @@ -keter_testworx.yml \ No newline at end of file diff --git a/config/keter_testworx.yml b/config/keter_testworx.yml deleted file mode 100644 index 91d90e156..000000000 --- a/config/keter_testworx.yml +++ /dev/null @@ -1,77 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Gregor Kleen ,Steffen Jost -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -root: .. - -stanzas: - - type: webapp - - # Name of your executable. You are unlikely to need to change this. - # Note that all file paths are relative to the keter.yml file. - # - # The path given is for Stack projects. If you're still using cabal, change - # to - # exec: ../dist/build/uniworx/uniworx - exec: ../dist/bin/uniworx - - # Command line options passed to your application. - args: [] - - hosts: - - testworx.tcs.ifi.lmu.de - - ssl: true - - forward-env: - - LDAPHOST - - LDAPTLS - - LDAPPORT - - LDAPUSER - - LDAPPASS - - LDAPBASE - - LDAPSCOPE - - LDAPSEARCHTIME - - LDAPSTRIPES - - LDAPTIMEOUT - - LDAPLIMIT - - DUMMY_LOGIN - - DETAILED_LOGGING - - LOG_ALL - - LOGLEVEL - - ALLOW_DEPRECATED - - PWFILE - - CRYPTOID_KEYFILE - - IP_FROM_HEADER - - MAILFROM_NAME - - MAILFROM_EMAIL - - MAILOBJECT_DOMAIN - - SMTPHOST - - SMTPPORT - - SMTPSSL - - SMTPUSER - - SMTPPASS - - SMTPTIMEOUT - - SMTPLIMIT - - MAILSUPPORT - - MAILSUPPORT_NAME - - INSTANCE_ID - - MEMCACHEDHOST - - MEMCACHEDPORT - - MEMCACHEDLIMIT - - MEMCACHEDTIMEOUT - - MEMCACHEDROOT - - MEMCACHEDEXPIRATION - -# Use the following to automatically copy your bundle upon creation via `yesod -# keter`. Uses `scp` internally, so you can set it to a remote destination -# copy-to: user@host:/opt/keter/incoming/ -copy-to: keter@testworx.tcs.ifi.lmu.de:/opt/keter/incoming/ -copy-to-args: - - "-P 30363" - -# If you would like to have Keter automatically create a PostgreSQL database -# and set appropriate environment variables for it to be discovered, uncomment -# the following line. -plugins: - postgres: true diff --git a/config/keter_uni2work.yml b/config/keter_uni2work.yml deleted file mode 100644 index f21122ec3..000000000 --- a/config/keter_uni2work.yml +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Gregor Kleen ,Steffen Jost -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -root: .. - -stanzas: - - type: webapp - - # Name of your executable. You are unlikely to need to change this. - # Note that all file paths are relative to the keter.yml file. - # - # The path given is for Stack projects. If you're still using cabal, change - # to - # exec: ../dist/build/uniworx/uniworx - exec: ../dist/bin/uniworx - - # Command line options passed to your application. - args: [] - - hosts: - - uni2work.ifi.lmu.de - - ssl: true - - forward-env: - - LDAPHOST - - LDAPTLS - - LDAPPORT - - LDAPUSER - - LDAPPASS - - LDAPBASE - - LDAPSCOPE - - LDAPSEARCHTIME - - LDAPSTRIPES - - LDAPTIMEOUT - - LDAPLIMIT - - DETAILED_LOGGING - - LOG_ALL - - LOGLEVEL - - ALLOW_DEPRECATED - - PWFILE - - CRYPTOID_KEYFILE - - IP_FROM_HEADER - - MAILFROM_NAME - - MAILFROM_EMAIL - - MAILOBJECT_DOMAIN - - SMTPHOST - - SMTPPORT - - SMTPSSL - - SMTPUSER - - SMTPPASS - - SMTPTIMEOUT - - SMTPLIMIT - - MAILSUPPORT - - MAILSUPPORT_NAME - - INSTANCE_ID - - MEMCACHEDHOST - - MEMCACHEDPORT - - MEMCACHEDLIMIT - - MEMCACHEDTIMEOUT - - MEMCACHEDROOT - - MEMCACHEDEXPIRATION - - -# Use the following to automatically copy your bundle upon creation via `yesod -# keter`. Uses `scp` internally, so you can set it to a remote destination -# copy-to: user@host:/opt/keter/incoming/ -copy-to: root@uni2work.ifi.lmu.de:/opt/keter/incoming/ -copy-to-args: [] - - -# If you would like to have Keter automatically create a PostgreSQL database -# and set appropriate environment variables for it to be discovered, uncomment -# the following line. -plugins: - postgres: - - server: uniworxdb - port: 5432 diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index f8f1fa183..4ae02188e 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -20,12 +20,12 @@ ENV HOME="${PROJECT_DIR}" RUN mkdir -p "${PROJECT_DIR}/.stack" ENV STACK_ROOT="${PROJECT_DIR}/.stack" -RUN make -- --.stack STACK_ROOT=${STACK_ROOT} IN_CONTAINER=true -RUN stack --work-dir .stack-work-run build yesod-bin -RUN chmod -R 777 .stack .stack-work-build .stack-work-run .stack-work-test .stack-work-doc || : +RUN make -- --dependencies-backend STACK_ROOT=${STACK_ROOT} IN_CONTAINER=true +RUN stack install yesod-bin +RUN chmod -R 777 .stack .stack-work .stack-work-compile .stack-work-start .stack-work-test .stack-work-lint .stack-work-hoogle || : ENV FRADRIVE_MAKE_TARGET=--start-backend ENTRYPOINT make -- ${FRADRIVE_MAKE_TARGET} STACK_ROOT="${STACK_ROOT}" IN_CONTAINER=true CONTAINER_FILE="${CONTAINER_FILE}" EXPOSE 3000/tcp -EXPOSE 3443/tcp \ No newline at end of file +EXPOSE 3443/tcp diff --git a/docker/database/Dockerfile b/docker/database/Dockerfile deleted file mode 100644 index 75714c7a3..000000000 --- a/docker/database/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM docker.io/postgres:12 - -# Allow for connecting to database without password authentication -ENV POSTGRES_HOST_AUTH_METHOD=trust - -COPY --chown=postgres:postgres docker/database/pg_hba.conf /tmp/pg_hba.conf -COPY --chown=postgres:postgres docker/database/postgresql.conf /tmp/postgresql.conf -COPY docker/database/pgconfig.sh /docker-entrypoint-initdb.d/_pgconfig.sh -COPY --chown=postgres:postgres docker/database/schema.sql /docker-entrypoint-initdb.d/schema.sql diff --git a/docker/postgres/Dockerfile b/docker/postgres/Dockerfile new file mode 100644 index 000000000..c3a14c0b9 --- /dev/null +++ b/docker/postgres/Dockerfile @@ -0,0 +1,9 @@ +FROM docker.io/postgres:12 + +# Allow for connecting to database without password authentication +ENV POSTGRES_HOST_AUTH_METHOD=trust + +COPY --chown=postgres:postgres docker/postgres/pg_hba.conf /tmp/pg_hba.conf +COPY --chown=postgres:postgres docker/postgres/postgresql.conf /tmp/postgresql.conf +COPY docker/postgres/pgconfig.sh /docker-entrypoint-initdb.d/_pgconfig.sh +COPY --chown=postgres:postgres docker/postgres/schema.sql /docker-entrypoint-initdb.d/schema.sql diff --git a/docker/database/initdb.sh b/docker/postgres/initdb.sh similarity index 100% rename from docker/database/initdb.sh rename to docker/postgres/initdb.sh diff --git a/docker/database/pg_hba.conf b/docker/postgres/pg_hba.conf similarity index 100% rename from docker/database/pg_hba.conf rename to docker/postgres/pg_hba.conf diff --git a/docker/database/pgconfig.sh b/docker/postgres/pgconfig.sh similarity index 100% rename from docker/database/pgconfig.sh rename to docker/postgres/pgconfig.sh diff --git a/docker/database/postgresql.conf b/docker/postgres/postgresql.conf similarity index 100% rename from docker/database/postgresql.conf rename to docker/postgres/postgresql.conf diff --git a/docker/database/schema.sql b/docker/postgres/schema.sql similarity index 100% rename from docker/database/schema.sql rename to docker/postgres/schema.sql diff --git a/enter b/enter new file mode 100755 index 000000000..427d93c10 --- /dev/null +++ b/enter @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +make enter CONTAINER_FILE=develop/`cat develop/.current`/"$1" \ No newline at end of file diff --git a/gup/Gupfile b/gup/Gupfile deleted file mode 100644 index d2db73f2f..000000000 --- a/gup/Gupfile +++ /dev/null @@ -1,4 +0,0 @@ -cabal2nix.gup: - *.nix -hpack.gup: - *.cabal \ No newline at end of file diff --git a/gup/Gupfile.license b/gup/Gupfile.license deleted file mode 100644 index 0544bf968..000000000 --- a/gup/Gupfile.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2022 Gregor Kleen - -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/gup/cabal2nix.gup b/gup/cabal2nix.gup deleted file mode 100755 index 9854cec0e..000000000 --- a/gup/cabal2nix.gup +++ /dev/null @@ -1,8 +0,0 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i zsh -p zsh haskellPackages.cabal2nix - -gup -u ${2:r}.cabal - -cd ${2:h} - -cabal2nix . > $1 diff --git a/gup/cabal2nix.gup.license b/gup/cabal2nix.gup.license deleted file mode 100644 index 0544bf968..000000000 --- a/gup/cabal2nix.gup.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2022 Gregor Kleen - -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/gup/hpack.gup b/gup/hpack.gup deleted file mode 100755 index 9e1052b02..000000000 --- a/gup/hpack.gup +++ /dev/null @@ -1,6 +0,0 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i sh -p haskellPackages.hpack - -gup -u package.yaml - -hpack - >$1 \ No newline at end of file diff --git a/gup/hpack.gup.license b/gup/hpack.gup.license deleted file mode 100644 index 0544bf968..000000000 --- a/gup/hpack.gup.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2022 Gregor Kleen - -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/utils/dirsymlink.pl b/utils/dirsymlink.pl new file mode 100755 index 000000000..c7f4dd22b --- /dev/null +++ b/utils/dirsymlink.pl @@ -0,0 +1,19 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +my ($shadowdir, @keepdirs) = @ARGV; + +mkdir($shadowdir); + +system(qq#rm -f "$shadowdir"/*#); + +for my $k(@keepdirs) { + unlink("$shadowdir/$k"); + mkdir("$shadowdir/$k"); +} + +system(qq#ln -rs .* * "$shadowdir"#); + + diff --git a/utils/makefilelogger.pl b/utils/makefilelogger.pl new file mode 100755 index 000000000..7054187a5 --- /dev/null +++ b/utils/makefilelogger.pl @@ -0,0 +1,53 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +my $fh = undef; +open($fh, '<', $ARGV[0]) or die "Could not read '$ARGV[0]', because: $!"; +my @cont = <$fh>; + +my %exports = (); +for(@cont) { + m#^export\s+([^=\s]+)# and $exports{$1} = 1 +} +my @exports = sort keys %exports; + +my @lined = (); +for(0..$#cont) { + $lined[$_] = [$_, $cont[$_]] +} +@lined = grep { $_->[1]!~m&^\s*#& } @lined; + +for my $i(0..$#lined-1) { + my $this = $lined[$i][1]; + my $next = $lined[$i+1][1]; + if($this=~m#^([^\s:]+):#) { + my $targetPattern = $1; + if($next=~m#^\t#) { + my $cl = $lined[$i]; + push @$cl, qq#\t\@echo "== PATTERN $targetPattern TARGET \$@" >> makelogger\n#; + push @$cl, qq#\t@ echo " \\\$\$($_) = '\$($_)'" >> makelogger\n# for @exports; + push @$cl, qq#\t@ echo "" >> makelogger\n#; + } + } +} + +my %out = (); +for my $i(0..$#cont) { + $out{$i} = $cont[$i] +} + +for(@lined) { + $out{$_->[0]} = $_ +} + +for my $i(0..$#cont) { + my $tc = $out{$i}; + if(ref $tc) { + print $tc->[$_] for 1..$#$tc + } else { + print $tc; + } +} + diff --git a/utils/makehelp.pl b/utils/makehelp.pl index aa742a7ea..15ef1a2b1 100755 --- a/utils/makehelp.pl +++ b/utils/makehelp.pl @@ -4,22 +4,35 @@ use strict; use warnings; my %msg = (); -my @start = (); my %reorder = (help=>-1); +# We do not want every Makefile syntax error to break help completely, +# so we just disable the kind of features where the syntax error occurs +# In that case we store also the error message here +my %brokenFeatures = (); + +my @blocks = (); +my @plainCode = (); + READ: while(<>) { + my $lnum = $.; if(m/^# HELP HEADER START/) { + my @start = (); while(<>) { - next READ if m/^# HELP HEADER END/; + do { + push @blocks, { kind=>'start', text=>\@start, line=>$lnum }; + next READ + } if m/^# HELP HEADER END/; if(m/# (.*)/) { push @start, $1 } else { die "$0: Invalid HELP HEADER section in Makefile!\n"; } } + push @blocks, { kind=>'start', text=>\@start, line=>$lnum }; next READ } - if(m/# HELP((?:\([^\(\)]+\))?):\s*(.*)/) { + if(m/^# HELP((?:\((?:[^\(\)]|\([^\(\)]+\))+\))?):\s*(.*)/) { my ($target, $message) = ($1, $2); if($target=~m/\((.*)\)/) { $target = $1; @@ -29,13 +42,125 @@ READ: while(<>) { $line=~m/^([^:]+):/ or die "$0: HELP marker expects target but no target found!\n"; $target = $1 } - $msg{$target} .= $message + push @blocks, { kind=>'target', target=>$target, message=>$message, line=>$lnum }; + next READ + } + if(m/^# HELPVAR\((.*)\)/) { + push @blocks, { kind=>'helpvar', content=>$1, line=>$lnum }; + next READ; + } + push @plainCode, [$lnum, $_] +} + +my %kindBlocks = (); +for(@blocks) { + push @{$kindBlocks{$_->{kind}}}, $_ +} + +my @matchies = (); +HELPVARS: { + for my $hv(@{$kindBlocks{helpvar}}) { + my $reVar = qr((?:\$\((?[A-Z]+)(?:,(?[^\(\)]+))?\))); + my $reVarNoGroups = qr((?:\$\((?:[A-Z]+)(?:,(?:[^\(\)]+))?\))); + my @parts = split /($reVarNoGroups)/, $hv->{content}; + my %setvars = (); + my @reParts = (); + for my $p(@parts) { + if($p=~m#^$reVar$#) { + my $name = $+{name}; + my $re = $+{re}; + if(defined $re) { + if(exists $setvars{$name}) { + $brokenFeatures{helpvars} = "Makefile line $hv->{line}: Multiple definitions of variable $name found!"; + last HELPVARS + } + $setvars{$name} = $re; + push @reParts, [1,qr((?<$name>$re))]; + } else { + if(not exists $setvars{$name}) { + $brokenFeatures{helpvars} = "Makefile line $hv->{line}: Variable $name used without definition!"; + last HELPVARS + } + push @reParts, [0,$name]; #qr((?:\k<$name>)); -- problem: cannot reference variable when not defined + } + } else { + push @reParts, [1,qr((?:\Q$p\E))]; + } + } + my $reCol = qr((?:)); + for my $r(@reParts) { + my ($isRe, $reg) = @$r; + if($isRe) { + $reCol = qr((?:$reCol$reg)); + } else { + eval { + $reCol = qr((?:$reCol(?:\k<$reg>))); + }; + if($@) { + $brokenFeatures{helpvars} = "Makefile line $hv->{line}: Variable $reg caused an internal error ($@)!"; + last HELPVARS + } + } + } + push @matchies, $reCol } } -print "$_\n" for @start; -print "\n" if @start; -for my $tar(sort {($reorder{$a}||0) <=> ($reorder{$b}||0) || $a cmp $b } keys %msg) { - print "$tar\n $msg{$tar}\n\n" +my %helpvars = (); + +for my $rowr(@plainCode) { + my ($lnum, $row) = @$rowr; + my @matches = (); + for my $re(@matchies) { + if($row=~m#^$re$#) { + my %h = %+; + push @matches, \%h + } + } + if(@matches>1) { + $brokenFeatures{helpvars} = "Makefile line $lnum: Multiple variable definitions match on row '$row'\n"; + } + if(1==@matches) { + my %h = %{$matches[0]}; + for my $k(keys %h) { + push @{$helpvars{$k}}, $h{$k} + } + } +} + +my @start = (); +@start = @{$kindBlocks{start}} if exists $kindBlocks{start}; + +for my $tg(@{$kindBlocks{target}}) { + my $tgv = $tg->{target}; + my @build = (); + $tgv=~s#\$\(([A-Z]+)(?:,[^\(\)]*)?\)#push @build, {name=>$1,elm=>$helpvars{$1}}; $1#ge; + unshift @build, $tg->{message}; + $msg{$tgv} = \@build; +} + +sub msgprinter { + my $el = shift; + if('HASH' eq ref $el) { + print " Where $el->{name} is one of\n"; + print " $_\n" for @{$el->{elm}}; + return + } + $el =~ s#(.{40,76})\s+#$1\n #g; + print " $el\n"; +} + +for(@start) { + print "$_\n" for @{$_->{text}} +} +print "\n" if @start; +for my $tar(sort {($reorder{$a}||0) <=> ($reorder{$b}||0) || $a cmp $b } keys %msg) { + print "$tar\n"; + msgprinter($_) for @{$msg{$tar}}; + print "\n"; +} + +for my $k(keys %brokenFeatures) { + warn "\n!!!!!\nFailed to complete construct the help message from the Makefile.\nFeature '$k' disabled, because: $brokenFeatures{$k}\n\n"; } diff --git a/utils/stack-work-init.sh b/utils/stack-work-init.sh new file mode 100755 index 000000000..c9628bf3b --- /dev/null +++ b/utils/stack-work-init.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +STACK_WORK_SRC=.stack-work-${JOB} + +#if [[ -e .stack-work.lock ]]; then +# echo ".stack-work directory is locked (lock file exists); is another stack currently running?" +# exit 1 +#fi + +touch .stack-work.lock +rm -rf .stack-work + +if [[ ! -e "$STACK_WORK_SRC" ]]; then + mkdir "$STACK_WORK_SRC" + chmod 777 "$STACK_WORK_SRC" +fi + +ln -s "$STACK_WORK_SRC" .stack-work \ No newline at end of file diff --git a/utils/watchcontainerrun.sh b/utils/watchcontainerrun.sh index eb2073024..3e0bc91a0 100755 --- a/utils/watchcontainerrun.sh +++ b/utils/watchcontainerrun.sh @@ -29,7 +29,9 @@ $STARTSCRIPT while [ -e "$FILENAME" ] ; do sleep 1 if ! $RUNNER ps --no-trunc | grep -q "$CONTAINER_ID" ; then - rm "$FILENAME" + #rm "$FILENAME" + EXITED_FILENAME=`echo "$FILENAME" | sed 's/\([^\/]*\)$/.exited.\1/'` + mv "$FILENAME" "$EXITED_FILENAME" exit fi inotifywait -e ATTRIB -t 10 "$FILENAME" > /dev/null 2>/dev/null diff --git a/utils/wrapmakelog.sh b/utils/wrapmakelog.sh new file mode 100755 index 000000000..2fe1bbc7d --- /dev/null +++ b/utils/wrapmakelog.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +: > makelogger + +utils/makefilelogger.pl Makefile > Makefile-loggingsymbols + +make -f Makefile-loggingsymbols "$@" +