Merge pull request #261 from fpco/helm-chart

Setup Helm chart for deployment instead of kube config
2026-01-11 19:58:28 +01:00 · 2019-02-04 18:00:09 +02:00 · 2019-02-04 18:00:09 +02:00 · 276f4f7f04
commit 276f4f7f04
parent 40e551a6f2 c4e2dd6603
14 changed files with 459 additions and 27 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -1,4 +1,4 @@
-image: registry.gitlab.fpcomplete.com/fpco/default-build-image:1954
+image: registry.gitlab.fpcomplete.com/fpco/default-build-image:5826

 cache:
  key: "$CI_BUILD_NAME"
@ -13,7 +13,8 @@ stages:

 variables:
    STACK_ROOT: "${CI_PROJECT_DIR}/.stack-root"
-    DEPLOYMENT_IMAGE: "${CI_REGISTRY_IMAGE}:${CI_BUILD_REF_SLUG}_${CI_PIPELINE_ID}"
+    #DEPLOYMENT_IMAGE: "${CI_REGISTRY_IMAGE}:${CI_BUILD_REF_SLUG}_${CI_PIPELINE_ID}"
+    DEPLOYMENT_IMAGE: "registry.gitlab.fpcomplete.com/fpco-mirrors/stackage-server:master_4296"
    DEPLOYMENT_NAME: "stackage-server-prod"
    HOOGLE_DEPLOYMENT_NAME: "stackage-server-hoogle-prod"
    CRON_DEPLOYMENT_NAME: "stackage-server-cron-prod"
@ -35,24 +36,30 @@ variables:
    kubectl apply -f <(envsubst <etc/kube/deployment_template.yaml) &&
    kubectl apply -f <(envsubst <etc/kube/cron_deployment_template.yaml) &&
    kubectl apply -f <(envsubst <etc/kube/hoogle_deployment_template.yaml)
+  - &HELMUPGRADE
+    helm --tiller-namespace ${KUBE_NAMESPACE} upgrade -i "${DEPLOYMENT_NAME}" etc/helm --set name="${DEPLOYMENT_NAME}" --set app="${DEPLOYMENT_APP}" --set hoogleName="${HOOGLE_DEPLOYMENT_NAME}" --set hoogleApp="${HOOGLE_DEPLOYMENT_APP}" --set cronName="${CRON_DEPLOYMENT_NAME}" --set cronApp="${HOOGLE_DEPLOYMENT_APP}" --set image.image="${DEPLOYMENT_IMAGE}" --values etc/helm/values/$CI_ENVIRONMENT_NAME.yaml --namespace ${KUBE_NAMESPACE}
+  - &HELMCHECK
+    helm --tiller-namespace ${KUBE_NAMESPACE} ls &&
+    helm --tiller-namespace ${KUBE_NAMESPACE} status "${DEPLOYMENT_NAME}"

-build:
-  stage: build
-  script:
-    # Clear *_TOKEN variables during code build so that compile-time code can't access them
-    - CI_BUILD_TOKEN="" KUBE_TOKEN="" PROD_KUBE_TOKEN="" PROD_DOCKER_PASSWORD="" etc/scripts/stage_docker.sh --install-ghc --test
-    - docker build -t "${DEPLOYMENT_IMAGE}" etc/docker
-    - docker login -u gitlab-ci-token -p "${CI_BUILD_TOKEN}" "${CI_REGISTRY}"
-    - docker push "${DEPLOYMENT_IMAGE}"
-    - |
-      if [[ "$CI_BUILD_REF_NAME" == "master" ]]; then
-        docker tag "${DEPLOYMENT_IMAGE}" "${CI_REGISTRY_IMAGE}:latest"
-        docker push "${CI_REGISTRY_IMAGE}:latest"
-      fi
-      if [[ "$CI_BUILD_REF_NAME" == "ci-cron" ]]; then
-        docker tag "${DEPLOYMENT_IMAGE}" "${CI_REGISTRY_IMAGE}:ci-cron"
-        docker push "${CI_REGISTRY_IMAGE}:ci-cron"
-      fi
+    #build:
+    #  stage: build
+    #  script:
+    #    # Clear *_TOKEN variables during code build so that compile-time code can't access them
+    #    - apt-get update && apt-get install -y --no-install-recommends libpq-dev git rsync
+    #    - CI_BUILD_TOKEN="" KUBE_TOKEN="" PROD_KUBE_TOKEN="" PROD_DOCKER_PASSWORD="" etc/scripts/stage_docker.sh --install-ghc --test
+    #    - docker build -t "${DEPLOYMENT_IMAGE}" etc/docker
+    #    - docker login -u gitlab-ci-token -p "${CI_BUILD_TOKEN}" "${CI_REGISTRY}"
+    #    - docker push "${DEPLOYMENT_IMAGE}"
+    #    - |
+    #      if [[ "$CI_BUILD_REF_NAME" == "master" ]]; then
+    #        docker tag "${DEPLOYMENT_IMAGE}" "${CI_REGISTRY_IMAGE}:latest"
+    #        docker push "${CI_REGISTRY_IMAGE}:latest"
+    #      fi
+    #      if [[ "$CI_BUILD_REF_NAME" == "ci-cron" ]]; then
+    #        docker tag "${DEPLOYMENT_IMAGE}" "${CI_REGISTRY_IMAGE}:ci-cron"
+    #        docker push "${CI_REGISTRY_IMAGE}:ci-cron"
+    #      fi

 deploy_prod:
  stage: deploy
@ -88,12 +95,13 @@ deploy_prod:

 deploy_ci:
  stage: deploy
-  only:
-    - ci
+  #only:
+  #  - ci
  environment:
-    name: stackage-server-ci
+    name: staging
    url: https://ci.stackage.org/
  variables:
+    KUBE_NAMESPACE: "fpco-public"
    DEPLOYMENT_NAME: "stackage-server-ci"
    HOOGLE_DEPLOYMENT_NAME: "stackage-server-hoogle-ci"
    CRON_DEPLOYMENT_NAME: "stackage-server-cron-ci"
@ -103,8 +111,5 @@ deploy_ci:
    HOST: ci.stackage.org
  script:
    - *KUBELOGIN
-    - *KUBEAPPLY
-    - kubectl apply -f <(envsubst <etc/kube/ingress_template.yaml)
-    - kubectl rollout status "deployment/$DEPLOYMENT_NAME"
-    - kubectl rollout status "deployment/$HOOGLE_DEPLOYMENT_NAME"
-    - kubectl rollout status "deployment/$CRON_DEPLOYMENT_NAME"
+    - *HELMUPGRADE
+    - *HELMCHECK
--- a/etc/helm/.helmignore
+++ b/etc/helm/.helmignore
@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
--- a/etc/helm/Chart.yaml
+++ b/etc/helm/Chart.yaml
@ -0,0 +1,4 @@
+apiVersion: v1
+description: A Helm chart for Kubernetes
+name: stackage-server
+version: 0.1.0
--- a/etc/helm/templates/NOTES.txt
+++ b/etc/helm/templates/NOTES.txt
@ -0,0 +1,19 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range .Values.ingress.hosts }}
+  http://{{ . }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "stackage-server.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get svc -w {{ template "stackage-server.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "stackage-server.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$SERVICE_IP:{{ .Values.service.externalPort }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "stackage-server.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl port-forward $POD_NAME 8080:{{ .Values.service.internalPort }}
+{{- end }}
--- a/etc/helm/templates/_helpers.tpl
+++ b/etc/helm/templates/_helpers.tpl
@ -0,0 +1,16 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "stackage-server.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "stackage-server.fullname" -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
--- a/etc/helm/templates/cron_deployment.yaml
+++ b/etc/helm/templates/cron_deployment.yaml
@ -0,0 +1,64 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: {{ .Values.cronName }}
+spec:
+  replicas: 1
+  minReadySeconds: 5
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.cronApp }}
+        release: {{ .Release.Name }}
+    spec:
+      imagePullSecrets:
+        - name: registry-key
+      volumes:
+      - name: stackage-server-cron-volume
+        secret:
+          secretName:  {{ .Values.cronName }}-secret
+      containers:
+        - name: stackage-server-cron
+          image: {{ .Values.image.image }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+          - name: PGSTRING
+            valueFrom:
+              secretKeyRef:
+                name: {{ .Values.name }}
+                key: PGCONN
+          command:
+            - bash
+            - "-c"
+            - |
+              set -eux
+              source /secret/aws.sh
+              mkdir -p /work
+              cd /work
+              while true
+              do
+                date
+                stack update
+                /usr/local/bin/stackage-server-cron
+                sleep 5m
+              done
+          volumeMounts:
+            - name: stackage-server-cron-volume
+              readOnly: true
+              mountPath: /secret
+          resources:
+{{ toYaml .Values.cronResources.resources | indent 12 }}
+    {{- with .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+    {{- end }}
--- a/etc/helm/templates/deployment.yaml
+++ b/etc/helm/templates/deployment.yaml
@ -0,0 +1,71 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: {{ .Values.name }}
+spec:
+  replicas: {{ .Values.image.replicas }}
+  minReadySeconds: 50
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.app }}
+        release: {{ .Release.Name }}
+    spec:
+      imagePullSecrets:
+        - name: registry-key
+      containers:
+        - name: stackage-server
+          image: {{ .Values.image.image }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: 3000
+          command: ["stackage-server"]
+          workingDir: /app
+          env:
+            - name: APPROOT
+              value: {{ .Values.image.env.approot | quote }}
+            - name: PGSTRING
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.name }}
+                  key: PGCONN
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 3000
+              httpHeaders:
+                # Works around stackage-server's `forceSSL` redirect
+                - name: HTTPS
+                  value: "on"
+            initialDelaySeconds: 5
+            timeoutSeconds: 1
+            periodSeconds: 5
+            successThreshold: 1
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 3000
+              httpHeaders:
+                # Works around stackage-server's `forceSSL` redirect
+                - name: HTTPS
+                  value: "on"
+            initialDelaySeconds: 120
+            timeoutSeconds: 1
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+          resources:
+{{ toYaml .Values.stackageResources.resources | indent 12 }}
+    {{- with .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+    {{- end }}
--- a/etc/helm/templates/hoogle_deployment.yaml
+++ b/etc/helm/templates/hoogle_deployment.yaml
@ -0,0 +1,71 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: {{ .Values.hoogleName }}
+spec:
+  replicas: {{ .Values.image.replicasHoggle }}
+  minReadySeconds: 5
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.hoogleApp }}
+        release: {{ .Release.Name }}
+    spec:
+      imagePullSecrets:
+        - name: registry-key
+      containers:
+        - name: stackage-server-hoogle
+          image: {{ .Values.image.image }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: 3000
+          command: ["stackage-server"]
+          workingDir: /app
+          env:
+            - name: APPROOT
+              value: {{ .Values.image.env.approot | quote }}
+            - name: PGSTRING
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.name }}
+                  key: PGCONN
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 3000
+              httpHeaders:
+                # Works around stackage-server's `forceSSL` redirect
+                - name: HTTPS
+                  value: "on"
+            initialDelaySeconds: 5
+            timeoutSeconds: 1
+            periodSeconds: 5
+            successThreshold: 1
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 3000
+              httpHeaders:
+                # Works around stackage-server's `forceSSL` redirect
+                - name: HTTPS
+                  value: "on"
+            initialDelaySeconds: 120
+            timeoutSeconds: 1
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+          resources:
+{{ toYaml .Values.stackageResources.resources | indent 12 }}
+    {{- with .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+    {{- end }}
--- a/etc/helm/templates/hoogle_service.yaml
+++ b/etc/helm/templates/hoogle_service.yaml
@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.hoogleName }}
+  labels:
+    app: {{ .Values.hoogleApp }}
+    release: {{ .Release.Name }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
+      protocol: TCP
+    - name: https
+      port: 443
+      targetPort: http
+      protocol: TCP
+  selector:
+    app: {{ .Values.hoogleApp }}
+    release: {{ .Release.Name }}
--- a/etc/helm/templates/ingress.yaml
+++ b/etc/helm/templates/ingress.yaml
@ -0,0 +1,39 @@
+{{- if .Values.ingress.enabled -}}
+{{- $name := .Values.name -}}
+{{- $hoogleName := .Values.hoogleName -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: {{ $name }}
+  labels:
+    app: {{ .Values.app }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+  annotations:
+    {{- range $key, $value := .Values.ingress.annotations }}
+      {{ $key }}: {{ $value | quote }}
+    {{- end }}
+spec:
+  {{- range .Values.ingress.hosts }}
+  rules:
+    - host: {{ . }}
+      http:
+        paths:
+          - path: /haddock.*
+            backend:
+              serviceName: {{ $name }}
+              servicePort: 80
+          - path: /.+/hoogle
+            backend:
+              serviceName: {{ $hoogleName }}
+              servicePort: 80
+          - backend:
+              serviceName: {{ $name }}
+              servicePort: 80
+  {{- end }}
+  tls:
+    - hosts:
+      {{- range .Values.ingress.hosts }}
+        - {{ . }}
+      {{- end }}
+      secretName: {{ $name }}-tls
+{{- end }}
--- a/etc/helm/templates/service.yaml
+++ b/etc/helm/templates/service.yaml
@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.name }}
+  labels:
+    app: {{ .Values.app }}
+    release: {{ .Release.Name }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
+      protocol: TCP
+    - name: https
+      port: 443
+      targetPort: http
+      protocol: TCP
+  selector:
+    app: {{ .Values.app }}
+    release: {{ .Release.Name }}
--- a/etc/helm/values.yaml
+++ b/etc/helm/values.yaml
@ -0,0 +1,16 @@
+image:
+  pullPolicy: Always
+  minReadySeconds: 5
+
+service:
+  type: ClusterIP
+
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    kubernetes.io/tls-acme: "true"
+    nginx.ingress.kubernetes.io/use-regex: "true"
+  path: /
+
+resources: {}
--- a/etc/helm/values/production.yaml
+++ b/etc/helm/values/production.yaml
@ -0,0 +1,30 @@
+#name: stackage-server-prod
+#app: stackage-server-prod
+#hoogleName: stackage-server-hoogle-prod
+#hoogleApp: stackage-server-hoogle-prod
+#cronName: stackage-server-cron-prod
+#cronApp: stackage-server-cron-prod
+
+image:
+  replicas: 1
+  replicasHoggle: 2
+  env:
+    approot: "stackage.org"
+
+stackageResources:
+  resources:
+    requests:
+      cpu: 100m
+      memory: 1.0Gi
+    limits:
+      cpu: 150m
+      memory: 1.5Gi
+
+cronResources:
+  resources:
+    requests:
+      cpu: 150m
+      memory: 2048Mi
+    limits:
+     cpu: 300m
+     memory: 4096Mi
--- a/etc/helm/values/staging.yaml
+++ b/etc/helm/values/staging.yaml
@ -0,0 +1,34 @@
+#name: stackage-server-ci
+#app: stackage-server-ci
+#hoogleName: stackage-server-hoogle-ci
+#hoogleApp: stackage-server-hoogle-ci
+#cronName: stackage-server-cron-ci
+#cronApp: stackage-server-cron-ci
+
+image:
+  replicas: 1
+  replicasHoggle: 2
+  env:
+    approot: "https://ci.stackage.org"
+
+ingress:
+  hosts:
+    - ci.stackage.org
+
+stackageResources:
+  resources:
+    requests:
+      cpu: 100m
+      memory: 1.0Gi
+    limits:
+      cpu: 150m
+      memory: 1.5Gi
+
+cronResources:
+  resources:
+    requests:
+      cpu: 150m
+      memory: 1048Mi
+    limits:
+     cpu: 200m
+     memory: 2096Mi