diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e255d66b8..851d09b5b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,19 @@ # # SPDX-License-Identifier: AGPL-3.0-or-later +### IMPORTANT NOTICE ### +# Our pipeline consists of static and dynamic parts. +# +# This file only contains the static parts of our pipeline. +# Dynamic jobs are defined in .gitlab-ci/dynamic-ci.yml. +# These are used as a template to generate jobs during the pipeline runtime. +# +# The marker "#dyn#" (without quotes) will be replaced by concrete values. +# +# TODO: documentation about dynamic stage +### + + # workflow: # rules: # - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH @@ -10,30 +23,12 @@ # - if: $CI_COMMIT_TAG =~ /^t/ # - if: $CI_COMMIT_TAG =~ /^d/ -variables: - # TODO: old vars; review and maybe remove - AWS_SHARED_CREDENTIALS_FILE: "/etc/aws/credentials" - TRANSFER_METER_FREQUENCY: "2s" - CONTAINER_REGISTRY: registry.uniworx.de/fradrive/fradrive - - default: image: - name: registry.uniworx.de/uniworx/containers/debian:12.5 + name: ${CI_REGISTRY}/uniworx/containers/debian:12.5 entrypoint: [""] docker: platform: x86_64 - # name: registry.uniworx.de/fradrive/fradrive/debian-podman:latest - # name: registry.uniworx.de/fradrive/fradrive/docker:dind - # name: registry.uniworx.de/fradrive/fradrive/fradrive-env:latest # Debian 12.5 Bookworm - # entrypoint: [""] - # docker: - # platform: x86_64 - # # user: ${ENV_USER} - # before_script: - # - apt-get -y update - # - apt-get -y install make podman - # - chown -R ${ENV_USER} . # Change project directory ownership to (non-root) user inside docker container artifacts: name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}" expire_in: "1 day" @@ -42,115 +37,30 @@ default: stages: - - containers - - compile - - lint - - test + - dynamic - release # TODO -# TODO: apply rules:changes for image jobs to only execute jobs when Dockerfiles change - -frontend image: - stage: containers - image: &buildah quay.io/buildah/stable:latest +# TODO: documentation +dynamic:prepare: + stage: dynamic script: - - buildah bud -t frontend docker/frontend/Dockerfile - - buildah push --creds "${CI_REGISTRY_USER}:${CI_JOB_TOKEN}" frontend $CONTAINER_REGISTRY/frontend - rules: - - changes: - - docker/frontend/Dockerfile - - -backend image: - stage: containers - image: *buildah - script: - - buildah bud -t backend docker/backend/Dockerfile - - buildah push --creds "${CI_REGISTRY_USER}:${CI_JOB_TOKEN}" backend $CONTAINER_REGISTRY/backend - rules: - - changes: - - docker/backend/Dockerfile - - -frontend build: - stage: compile - needs: - - job: frontend image - optional: true - script: - - &load-frontend-image zcat docker/frontend/image.tar.gz | podman image load - - make -- --docker-run-frontend-build FRADRIVE_SERVICE=frontend CONTAINER_RUNNER=podman-compose + - FRONTEND_IMAGE_VERSION=`git log docker/frontend/Dockerfile | grep '^commit' | wc --lines` + - BACKEND_IMAGE_VERSION=`git log docker/backend/Dockerfile | grep '^commit' | wc --lines` + - cat .gitlab-ci/dynamic-gitlab-ci.yml | .gitlab-ci/dynamci.pl FRONTEND_IMAGE_VERSION=${FRONTEND_IMAGE_VERSION} BACKEND_IMAGE_VERSION=${BACKEND_IMAGE_VERSION} > dynamic-gitlab-ci.yml artifacts: paths: - - /fradrive/node_modules - - /fradrive/well-known - cache: - - &frontend-cache - key: default-frontend - paths: - - /fradrive/.npm/ - - /fradrive/.well-known-cache/ + - dynamic-gitlab-ci.yml -frontend lint: - stage: lint +# TODO: documentation +dynamic:execute: + stage: dynamic needs: - - job: frontend image - optional: true - script: - - *load-frontend-image - - make -- --docker-run-frontend-lint FRADRIVE_SERVICE=frontend CONTAINER_RUNNER=podman-compose - cache: *frontend-cache - -frontend test: - stage: test - needs: - - job: frontend image - optional: true - - job: frontend build - artifacts: true - script: - - *load-frontend-image - - make -- --docker-run-frontend-test FRADRIVE_SERVICE=frontend CONTAINER_RUNNER=podman-compose - - cache: *frontend-cache - -backend build: - stage: compile - needs: - - job: backend image - optional: true - - job: frontend build - artifacts: true - script: - - &load-backend-image zcat docker/backend/image.tar.gz | podman image load - - make -- --docker-run-backend-build FRADRIVE_SERVICE=backend CONTAINER_RUNNER=podman-compose - artifacts: - paths: - - /fradrive/.stack/ - - /fradrive/.stack-work/ - cache: &backend-cache - -backend lint: - stage: lint - needs: - - job: backend image - optional: true - script: - - *load-backend-image - - make -- --docker-run-backend-lint FRADRIVE_SERVICE=backend CONTAINER_RUNNER=podman-compose - cache: *backend-cache - -backend test: - stage: test - needs: - - job: backend image - optional: true - - job: backend build - artifacts: true - script: - - *load-backend-image - - make -- --docker-run-backend-test FRADRIVE_SERVICE=backend CONTAINER_RUNNER=podman-compose - cache: *backend-cache + - dynamic:prepare + trigger: + strategy: depend + include: + - artifact: dynamic-gitlab-ci.yml + job: dynamic:prepare # frontend dependencies: # stage: setup diff --git a/.gitlab-ci/dynamci.pl b/.gitlab-ci/dynamci.pl new file mode 100644 index 000000000..6ae2e763b --- /dev/null +++ b/.gitlab-ci/dynamci.pl @@ -0,0 +1,43 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +my %kv = (); + +for(@ARGV) { + if(!m#^\s*([^=\s]+)\s*=\s*(.*?)\s*$#) { + die "$0: Bad parameter '$_': Not a key=value pair\n" + } + $kv{$1} = $2; +} + +my $invar = 0; + +LOOP: while(my $line = ) { + if(1==$invar) { + if($line=~m/^(\s+)(\S+)(\s*:\s*)(\S+)(.*)/) { + my ($pre1, $key, $pre2, $var, $post) = ($1, $2, $3, $4, $5); + if('#dyn#' eq $var) { + if(not exists $kv{$key}) { + die "$0: No value given for key '$key' in the parameters but it is defined in input!\n" + } + my $v = $kv{$key}; + delete $kv{$key}; + print "$pre1$key$pre2$v$post\n"; + next LOOP; + } + } elsif($line=~m/^[^#\t ]/) { + $invar = 2 + } + } + if(0==$invar and $line=~m#^\s*variables\s*:\s*$#) { + $invar = 1; + } + print $line; +} + +my @rem = sort keys %kv; + +die "$0: Variables occur in parameter but not in input: @rem!\n" if @rem; + diff --git a/.gitlab-ci/dynamic-gitlab--ci.yml b/.gitlab-ci/dynamic-gitlab--ci.yml new file mode 100644 index 000000000..9ff85c702 --- /dev/null +++ b/.gitlab-ci/dynamic-gitlab--ci.yml @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: 2024 Sarah Vaupel +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +### IMPORTANT NOTICE ### +# Our pipeline consists of static and dynamic parts. +# +# This file only contains the dynamic parts of our pipeline. +# Static jobs are defined in .gitlab-ci.yml. +# +# The marker "#dyn#" (without quotes) will be replaced by concrete values. +### + +variables: + FRONTEND_IMAGE_VERSION: #dyn# + BACKEND_IMAGE_VERSION: #dyn# + +stages: + - containers + - compile + - lint + - test + + +containers:frontend: + stage: containers + image: &buildah quay.io/buildah/stable:latest + script: + # - IMAGE_VERSION=`git log docker/frontend/Dockerfile | grep '^commit' | wc --lines` + - buildah bud -t frontend/${CI_COMMIT_REF_SLUG}:${FRONTEND_IMAGE_VERSION} docker/frontend/Dockerfile + - buildah push --creds "${CI_REGISTRY_USER}:${CI_JOB_TOKEN}" frontend/${CI_COMMIT_REF_SLUG}:${FRONTEND_IMAGE_VERSION} ${CI_REGISTRY_IMAGE}/frontend/${CI_COMMIT_REF_SLUG}:${FRONTEND_IMAGE_VERSION} + rules: + - changes: + - docker/frontend/Dockerfile + interruptible: false + +containers:backend: + stage: containers + image: *buildah + script: + # - IMAGE_VERSION=`git log docker/backend/Dockerfile | grep '^commit' | wc --lines` + - buildah bud -t backend/${CI_COMMIT_REF_SLUG}:${BACKEND_IMAGE_VERSION} docker/backend/Dockerfile + - buildah push --creds "${CI_REGISTRY_USER}:${CI_JOB_TOKEN}" backend/${CI_COMMIT_REF_SLUG}:${BACKEND_IMAGE_VERSION} ${CI_REGISTRY_IMAGE}/backend/${CI_COMMIT_REF_SLUG}:${BACKEND_IMAGE_VERSION} + rules: + - changes: + - docker/backend/Dockerfile + interruptible: false + + +frontend build: + stage: compile + needs: + - job: containers:frontend + optional: true + image: &frontend-image ${CI_REGISTRY_IMAGE}/frontend/${CI_COMMIT_REF_SLUG}:${FRONTEND_IMAGE_VERSION} + script: + - &load-frontend-image zcat docker/frontend/image.tar.gz | podman image load + - make -- --docker-run-frontend-build FRADRIVE_SERVICE=frontend CONTAINER_RUNNER=podman-compose + artifacts: + paths: + - /fradrive/node_modules + - /fradrive/well-known + cache: + - &frontend-cache + key: default-frontend + paths: + - /fradrive/.npm/ + - /fradrive/.well-known-cache/ + +frontend lint: + stage: lint + needs: + - job: containers:frontend + optional: true + image: *frontend-image + script: + - *load-frontend-image + - make -- --docker-run-frontend-lint FRADRIVE_SERVICE=frontend CONTAINER_RUNNER=podman-compose + cache: *frontend-cache + +frontend test: + stage: test + needs: + - job: containers:frontend + optional: true + - job: frontend build + artifacts: true + image: *frontend-image + script: + - *load-frontend-image + - make -- --docker-run-frontend-test FRADRIVE_SERVICE=frontend CONTAINER_RUNNER=podman-compose + + cache: *frontend-cache + +backend build: + stage: compile + needs: + - job: containers:backend + optional: true + - job: frontend build + artifacts: true + image: &backend-image ${CI_REGISTRY_IMAGE}/backend/${CI_COMMIT_REF_SLUG}:${BACKEND_IMAGE_VERSION} + script: + - &load-backend-image zcat docker/backend/image.tar.gz | podman image load + - make -- --docker-run-backend-build FRADRIVE_SERVICE=backend CONTAINER_RUNNER=podman-compose + artifacts: + paths: + - /fradrive/.stack/ + - /fradrive/.stack-work/ + cache: &backend-cache + +backend lint: + stage: lint + needs: + - job: containers:backend + optional: true + image: *backend-image + script: + - *load-backend-image + - make -- --docker-run-backend-lint FRADRIVE_SERVICE=backend CONTAINER_RUNNER=podman-compose + cache: *backend-cache + +backend test: + stage: test + needs: + - job: containers:backend + optional: true + - job: backend build + artifacts: true + image: *backend-image + script: + - *load-backend-image + - make -- --docker-run-backend-test FRADRIVE_SERVICE=backend CONTAINER_RUNNER=podman-compose + cache: *backend-cache \ No newline at end of file