From ef9f4a9345919a579c6f244bfc210d56df001ca8 Mon Sep 17 00:00:00 2001 From: Sarah Vaupel Date: Thu, 6 Feb 2025 18:04:55 +0100 Subject: [PATCH] ci(azure-pipelines): restructure pipeline workflow --- .../templates/{ => jobs}/release.yaml | 7 +- .../templates/jobs/setup_dependencies.yaml | 81 +++++++++++++++++ .azure-pipelines/templates/steps/make.yaml | 2 +- azure-pipelines.yaml | 87 +++++++++++++------ 4 files changed, 148 insertions(+), 29 deletions(-) rename .azure-pipelines/templates/{ => jobs}/release.yaml (88%) create mode 100644 .azure-pipelines/templates/jobs/setup_dependencies.yaml diff --git a/.azure-pipelines/templates/release.yaml b/.azure-pipelines/templates/jobs/release.yaml similarity index 88% rename from .azure-pipelines/templates/release.yaml rename to .azure-pipelines/templates/jobs/release.yaml index 916117a08..0f7454a85 100644 --- a/.azure-pipelines/templates/release.yaml +++ b/.azure-pipelines/templates/jobs/release.yaml @@ -8,6 +8,9 @@ parameters: - name: releaseEndpoint type: string default: 'devfra' + values: + - 'devfra' + - 'prodfra' jobs: - job: Release @@ -37,8 +40,8 @@ jobs: script: | cp docker/fradrive/Dockerfile . docker build \ - --tag $(imageUpstream)/fradrive:$(Build.BuildNumber) \ - --tag $(imageUpstream)/fradrive:${{parameters.releaseTag}} \ + --tag $(buildImageUpstream)/fradrive:$(Build.BuildNumber) \ + --tag $(buildImageUpstream)/fradrive:${{parameters.releaseTag}} \ --build-arg FROM_IMG=devfra.azurecr.io/de.fraport.trusted/ubuntu \ --build-arg FROM_TAG=20.04 \ --build-arg PROJECT_DIR=$(Build.Repository.LocalPath) \ diff --git a/.azure-pipelines/templates/jobs/setup_dependencies.yaml b/.azure-pipelines/templates/jobs/setup_dependencies.yaml new file mode 100644 index 000000000..8a0ba798b --- /dev/null +++ b/.azure-pipelines/templates/jobs/setup_dependencies.yaml @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: 2024-2025 Sarah Vaupel +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +parameters: + - name: service + type: string + - name: dependenciesArtifacts + type: string + - name: dependenciesBuildPool + type: string + default: 'Prod Private Agent Pool' + values: + - 'Prod Private Agent Pool' + - 'Prod Private Agent Pool DS2' + - 'Prod Private Agent Pool DS3' + - name: dependenciesBuildTimeout + type: number + default: 60 + - name: dependenciesSource + type: string + default: 'current' + values: + - 'current' + - 'specific' + - name: dependenciesVersion + type: string + default: 'latest' + values: + - 'latest' + - 'latestFromBranch' + +jobs: +- job: SetupDependencies_${{parameters.service}} + displayName: Install ${{parameters.service}} dependencies + dependsOn: SetupImage_${{parameters.service}} + pool: '${{parameters.dependenciesBuildPool}}' + container: + # TODO: do not use latest on update branches + image: $(buildImageUpstream)/${{parameters.service}}:latest + endpoint: devfra + env: + PROJECT_DIR: $(Build.Repository.LocalPath) + IN_CONTAINER: true + IN_CI: true + steps: + # Download previously-built dependency artifacts + - task: DownloadPipelineArtifact@2 + displayName: Download previously built ${{parameters.service}} dependencies + condition: eq(variables.useCachedDependencies, true) + continueOnError: true + inputs: + artifactName: '${{service}}-dependencies' + source: ${{ dependenciesSource }} + project: 'Fahrerausbildung' # TODO: hardcoded for now, could not figure out which predefined variable to use + pipeline: $(System.DefinitionId) + buildVersionToDownload: '${{dependenciesVersion}}' + # tags: '${{dependenciesArtifacts}}' + allowPartiallySucceededBuilds: true + allowFailedBuilds: true + patterns: '${{dependenciesArtifacts}}' + targetPath: '$(Build.Repository.LocalPath)' + + # Compile dependencies + - template: .azure-pipelines/templates/steps/make.yaml + parameters: + makeJob: dependencies + makeService: ${{parameters.service}} + + # Upload newly-built dependencies as artifacts + - task: CopyFiles@2 + displayName: Copy ${{parameters.service}} dependencies for upload + inputs: + Contents: ${{ parameters.dependenciesArtifacts }} + TargetFolder: '$(Build.ArtifactStagingDirectory)' + - task: PublishBuildArtifacts@1 + displayName: Upload ${{parameters.service}} dependencies as artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: '${{parameters.service}}-dependencies' + publishLocation: 'Container' \ No newline at end of file diff --git a/.azure-pipelines/templates/steps/make.yaml b/.azure-pipelines/templates/steps/make.yaml index 2b77bf718..f7143e56c 100644 --- a/.azure-pipelines/templates/steps/make.yaml +++ b/.azure-pipelines/templates/steps/make.yaml @@ -19,7 +19,7 @@ parameters: steps: - task: Bash@3 name: ${{parameters.makeJob}}_${{parameters.makeService}} - displayName: make ${{parameters.makeJob}} ${{parameters.makeService}} + displayName: make ${{parameters.makeJob}}-${{parameters.makeService}} env: HTTPS_PROXY: http://proxy.frankfurt-airport.de:8080 HTTP_PROXY: http://proxy.frankfurt-airport.de:8080 diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 3d0e7f36f..b2e607619 100755 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -60,32 +60,67 @@ pool: 'Prod Private Agent Pool' stages: -- ${{ each service in parameters.services }}: - - template: .azure-pipelines/templates/service.yaml - parameters: - serviceName: ${{ service.name }} - serviceBase: ${{ service.base }} - servicePool: ${{ service.pool }} - serviceTimeout: ${{ service.timeout }} - serviceDependsOn: ${{ service.dependsOn }} - serviceRequiredArtifacts: ${{ service.requiredArtifacts }} - serviceArtifacts: ${{ service.artifacts }} - ${{ if eq(variables.skipTests, false) }}: - buildSteps: - - dependencies - - compile - - lint - - test - ${{ else }}: - buildSteps: - - dependencies - - compile - -- stage: release - condition: or(eq(variables.forceRelease, true), startsWith(variables['Build.SourceBranch'], 'refs/tags/')) - dependsOn: - - backend +- stage: Setup jobs: - - template: .azure-pipelines/templates/release.yaml + - ${{ each service in parameters.services }}: + - template: .azure-pipelines/templates/jobs/setup_image.yaml + parameters: + imageName: ${{service.name}} + imageBase: ${{service.imageBase}} + - template: .azure-pipelines/templates/jobs/setup_dependencies.yaml + parameters: + dependenciesName: ${{service.name}} + dependenciesBuildPool: ${{service.buildPool}} + dependenciesBuildTimeout: ${{service.buildTimeout}} + dependenciesArtifacts: ${{service.dependenciesArtifacts}} + +- stage: Build + jobs: + - ${{ each service in parameters.services }}: + - job: Build_${{service.name}} + displayName: Compile ${{service.name}} + dependsOn: ${{service.dependsOn}} + steps: + - template: .azure-pipelines/templates/steps/make.yaml + parameters: + makeJob: compile + makeService: ${{service.name}} + - task: CopyFiles@2 + displayName: Prepare ${{service.name}} build artifacts for upload + inputs: + Contents: ${{service.artifacts}} + TargetFolder: '$(Build.ArtifactStagingDirectory)' + - task: PublishBuildArtifacts@1 + displayName: Publish ${{service.name}} build artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: '${{service.name}}' + publishLocation: 'Container' + +- stage: Test + condition: eq(variables.skipTests, false) + jobs: + - ${{ each service in parameters.services }}: + - job: Test_${{service.name}} + displayName: Compile ${{service.name}} + dependsOn: ${{service.dependsOn}} + steps: + - template: .azure-pipelines/templates/steps/make.yaml + parameters: + makeJob: lint + makeService: ${{service.name}} + - template: .azure-pipelines/templates/steps/make.yaml + parameters: + makeJob: test + makeService: ${{service.name}} + - job: TestReport_${{service.name}} + displayName: Upload test reports for ${{service.name}} + steps: + - script: echo "Work in progress" # TODO + +- stage: Release + condition: or(eq(variables.forceRelease, true), startsWith(variables['Build.SourceBranch'], 'refs/tags/')) + jobs: + - template: .azure-pipelines/templates/jobs/release.yaml parameters: releaseTag: ${{split(variables['Build.SourceBranch'], '/')[2]}} \ No newline at end of file