gitlab-ci-helm.yml

# =========================================================================================
# Copyright (C) 2021 Orange & contributors
#
# This program is free software; you can redistribute it and/or modify it under the terms
# of the GNU Lesser General Public License as published by the Free Software Foundation;
# either version 3 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along with this
# program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth
# Floor, Boston, MA  02110-1301, USA.
# =========================================================================================
# default workflow rules: Merge Request pipelines
workflow:
  rules:
    # prevent branch pipeline when an MR is open (prefer MR pipeline)
    - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS'
      when: never
    - when: always

# test job prototype: implement adaptive pipeline rules
.test-policy:
  rules:
    # on tag: auto & failing
    - if: $CI_COMMIT_TAG
    # on ADAPTIVE_PIPELINE_DISABLED: auto & failing
    - if: '$ADAPTIVE_PIPELINE_DISABLED == "true"'
    # on production or integration branch(es): auto & failing
    - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF || $CI_COMMIT_REF_NAME =~ $INTEG_REF'
    # early stage (dev branch, no MR): manual & non-failing
    - if: '$CI_MERGE_REQUEST_ID == null && $CI_OPEN_MERGE_REQUESTS == null'
      when: manual
      allow_failure: true
    # Draft MR: auto & non-failing
    - if: '$CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/'
      allow_failure: true
    # else (Ready MR): auto & failing
    - when: on_success

variables:
  # variabilized tracking image
  TBC_TRACKING_IMAGE: "$CI_REGISTRY/to-be-continuous/tools/tracking:master"

  # Docker Image with Helm CLI tool (can be overridden)
  HELM_CLI_IMAGE: "alpine/helm"
  HELM_YAMLLINT_IMAGE: "cytopia/yamllint"
#  HELM_LINT_DISABLED: "true"
#  HELM_YAMLLINT_DISABLED: "true"
  HELM_YAMLLINT_CONFIG: "{extends: relaxed, rules: {line-length: {max: 160}}}"
  HELM_YAMLLINT_ARGS: "-f colored --strict"
  HELM_LINT_ARGS: "lint --strict"
  HELM_DEPENDENCY_ARGS: "dependency update"
  HELM_KUBE_SCORE_IMAGE: "zegl/kube-score:latest-helm3"

  HELM_CHART_DIR: "."
  HELM_SCRIPTS_DIR: "."
  HELM_PACKAGE_ARGS: "package --dependency-update"
  # to GitLab's container registry (OCI-compliant)
  HELM_PUBLISH_URL: "oci://$CI_REGISTRY/$CI_PROJECT_PATH/charts"
  # to GitLab's packages repository
  # HELM_PUBLISH_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/release"
  # HELM_PUBLISH_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/release"
  HELM_PUBLISH_METHOD: "auto"

  HELM_REPOS: "stable@https://charts.helm.sh/stable bitnami@https://charts.bitnami.com/bitnami"

  HELM_ENV_VALUE_NAME: environmentType
  HELM_HOSTNAME_VALUE_NAME: hostname

  # Will work with gitlab Kubernetes integration (per env variables)
#  KUBE_NAMESPACE: "default"
#  KUBECONFIG: ""

#  HELM_COMMON_VALUES: "values-common.yml"

#  HELM_REVIEW_DISABLED: "true"
#  HELM_REVIEW_VALUES: "values-review.yml"
#  HELM_REVIEW_NAMESPACE: ""
#  HELM_REVIEW_KUBE_CONFIG: ""

#  HELM_INTEG_DISABLED: "true"
#  HELM_INTEG_VALUES: "values-review.yml"
#  HELM_INTEG_NAMESPACE: ""
#  HELM_INTEG_KUBE_CONFIG: ""

#  HELM_STAGING_DISABLED: "true"
#  HELM_STAGING_VALUES: "values-staging.yml"
#  HELM_STAGING_NAMESPACE: ""
#  HELM_STAGING_KUBE_CONFIG: ""

#  HELM_PROD_VALUES: "values-prod.yml"
#  HELM_PROD_NAMESPACE: ""
#  HELM_PROD_KUBE_CONFIG: ""

  HELM_DEPLOY_ARGS: "upgrade --install --atomic --timeout 120s"
  HELM_DELETE_ARGS: "uninstall"
  HELM_TEST_ARGS: "test"

#  HELM_DEPLOY_CHART: ""
#  HELM_DEPLOY_REPO_NAME: "my-repo-name"
#  HELM_DEPLOY_REPO_URL: "https://my.repo.com"

  # [optional]  HELM_BASE_APP_NAME    : the base application name (defaults to $CI_PROJECT_NAME)
  # [optional]  HELM_REVIEW_APP_NAME  : specific Helm application name in review env (defaults to $HELM_BASE_APP_NAME-$CI_COMMIT_REF_SLUG)
  # [optional]  HELM_STAGING_APP_NAME : specific Helm application name in staging env (defaults to $HELM_BASE_APP_NAME-staging)
  # [optional]  HELM_PROD_APP_NAME    : specific Helm application name in production env (defaults to $HELM_BASE_APP_NAME)

  HELM_BASE_APP_NAME: "$CI_PROJECT_NAME"
  HELM_REVIEW_ENVIRONMENT_SCHEME: "https"

  # default production ref name (pattern)
  PROD_REF: '/^(master|main)$/'
  # default integration ref name (pattern)
  INTEG_REF: '/^develop$/'

stages:
  - test
  - package-build
  - package-test
  - deploy
  - publish
  - production
  - acceptance

.helm-scripts: &helm-scripts |
  # BEGSCRIPT
  set -e

  function log_info() {
      echo -e "[\\e[1;94mINFO\\e[0m] $*"
  }

  function log_warn() {
      echo -e "[\\e[1;93mWARN\\e[0m] $*"
  }

  function log_error() {
      echo -e "[\\e[1;91mERROR\\e[0m] $*"
  }

  function fail() {
    log_error "$*"
    exit 1
  }

  function assert_defined() {
    if [[ -z "$1" ]]
    then
      log_error "$2"
      exit 1
    fi
  }

  function install_ca_certs() {
    certs=$1
    if [[ -z "$certs" ]]
    then
      return
    fi

    # import in system
    if echo "$certs" >> /etc/ssl/certs/ca-certificates.crt
    then
      log_info "CA certificates imported in \\e[33;1m/etc/ssl/certs/ca-certificates.crt\\e[0m"
    fi
    if echo "$certs" >> /etc/ssl/cert.pem
    then
      log_info "CA certificates imported in \\e[33;1m/etc/ssl/cert.pem\\e[0m"
    fi
  }

  function unscope_variables() {
    _scoped_vars=$(env | awk -F '=' "/^scoped__[a-zA-Z0-9_]+=/ {print \$1}" | sort)
    if [[ -z "$_scoped_vars" ]]; then return; fi
    log_info "Processing scoped variables..."
    for _scoped_var in $_scoped_vars
    do
      _fields=${_scoped_var//__/:}
      _condition=$(echo "$_fields" | cut -d: -f3)
      case "$_condition" in
      if) _not="";;
      ifnot) _not=1;;
      *)
        log_warn "... unrecognized condition \\e[1;91m$_condition\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"
        continue
      ;;
      esac
      _target_var=$(echo "$_fields" | cut -d: -f2)
      _cond_var=$(echo "$_fields" | cut -d: -f4)
      _cond_val=$(eval echo "\$${_cond_var}")
      _test_op=$(echo "$_fields" | cut -d: -f5)
      case "$_test_op" in
      defined)
        if [[ -z "$_not" ]] && [[ -z "$_cond_val" ]]; then continue;
        elif [[ "$_not" ]] && [[ "$_cond_val" ]]; then continue;
        fi
        ;;
      equals|startswith|endswith|contains|in|equals_ic|startswith_ic|endswith_ic|contains_ic|in_ic)
        # comparison operator
        # sluggify actual value
        _cond_val=$(echo "$_cond_val" | tr '[:punct:]' '_')
        # retrieve comparison value
        _cmp_val_prefix="scoped__${_target_var}__${_condition}__${_cond_var}__${_test_op}__"
        _cmp_val=${_scoped_var#"$_cmp_val_prefix"}
        # manage 'ignore case'
        if [[ "$_test_op" == *_ic ]]
        then
          # lowercase everything
          _cond_val=$(echo "$_cond_val" | tr '[:upper:]' '[:lower:]')
          _cmp_val=$(echo "$_cmp_val" | tr '[:upper:]' '[:lower:]')
        fi
        case "$_test_op" in
        equals*)
          if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val" ]]; then continue;
          elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val" ]]; then continue;
          fi
          ;;
        startswith*)
          if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val"* ]]; then continue;
          elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val"* ]]; then continue;
          fi
          ;;
        endswith*)
          if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val" ]]; then continue;
          elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val" ]]; then continue;
          fi
          ;;
        contains*)
          if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val"* ]]; then continue;
          elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val"* ]]; then continue;
          fi
          ;;
        in*)
          if [[ -z "$_not" ]] && [[ "__${_cmp_val}__" != *"__${_cond_val}__"* ]]; then continue;
          elif [[ "$_not" ]] && [[ "__${_cmp_val}__" == *"__${_cond_val}__"* ]]; then continue;
          fi
          ;;
        esac
        ;;
      *)
        log_warn "... unrecognized test operator \\e[1;91m${_test_op}\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"
        continue
        ;;
      esac
      # matches
      _val=$(eval echo "\$${_target_var}")
      log_info "... apply \\e[32m${_target_var}\\e[0m from \\e[32m\$${_scoped_var}\\e[0m${_val:+ (\\e[33;1moverwrite\\e[0m)}"
      _val=$(eval echo "\$${_scoped_var}")
      export "${_target_var}"="${_val}"
    done
    log_info "... done"
  }

  # evaluate and export a secret
  # - $1: secret variable name
  function eval_secret() {
    name=$1
    value=$(eval echo "\$${name}")
    case "$value" in
    @b64@*)
      decoded=$(mktemp)
      errors=$(mktemp)
      if echo "$value" | cut -c6- | base64 -d > "${decoded}" 2> "${errors}"
      then
        # shellcheck disable=SC2086
        export ${name}="$(cat ${decoded})"
        log_info "Successfully decoded base64 secret \\e[33;1m${name}\\e[0m"
      else
        fail "Failed decoding base64 secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
      fi
      ;;
    @hex@*)
      decoded=$(mktemp)
      errors=$(mktemp)
      if echo "$value" | cut -c6- | sed 's/\([0-9A-F]\{2\}\)/\\\\x\1/gI' | xargs printf > "${decoded}" 2> "${errors}"
      then
        # shellcheck disable=SC2086
        export ${name}="$(cat ${decoded})"
        log_info "Successfully decoded hexadecimal secret \\e[33;1m${name}\\e[0m"
      else
        fail "Failed decoding hexadecimal secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
      fi
      ;;
    @url@*)
      url=$(echo "$value" | cut -c6-)
      if command -v curl > /dev/null
      then
        decoded=$(mktemp)
        errors=$(mktemp)
        if curl -s -S -f --connect-timeout 5 -o "${decoded}" "$url" 2> "${errors}"
        then
          # shellcheck disable=SC2086
          export ${name}="$(cat ${decoded})"
          log_info "Successfully curl'd secret \\e[33;1m${name}\\e[0m"
        else
          log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
        fi
      elif command -v wget > /dev/null
      then
        decoded=$(mktemp)
        errors=$(mktemp)
        if wget -T 5 -O "${decoded}" "$url" 2> "${errors}"
        then
          # shellcheck disable=SC2086
          export ${name}="$(cat ${decoded})"
          log_info "Successfully wget'd secret \\e[33;1m${name}\\e[0m"
        else
          log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
        fi
      else
        log_warn "Couldn't get secret \\e[33;1m${name}\\e[0m: no http client found"
      fi
      ;;
    esac
  }

  function eval_all_secrets() {
    encoded_vars=$(env | grep -Ev '(^|.*_ENV_)scoped__' | awk -F '=' '/^[a-zA-Z0-9_]*=@(b64|hex|url)@/ {print $1}')
    for var in $encoded_vars
    do
      eval_secret "$var"
    done
  }

  function setup_kubeconfig() {
    if [ -n "$1" ]; then
      export KUBECONFIG="$CI_PROJECT_DIR/.kubeconfig"
      echo "$1" > "$KUBECONFIG"
      log_info "--- using \\e[32mKUBECONFIG\\e[0m provided by env variables"
    elif [ -n "$KUBECONFIG" ]; then
      log_info "--- using \\e[32mKUBECONFIG\\e[0m provided by GitLab"
    else
      log_warn "No \\e[32mKUBECONFIG\\e[0m configuration found!"
    fi
  }

  function add_helm_repositories() {
    # Use cacheable folders
    mkdir -p "$CI_PROJECT_DIR/.config/helm/"
    mkdir -p "$CI_PROJECT_DIR/.cache/helm/repository/"

    # Install helm repositories
    for repo in $HELM_REPOS
    do
      repo_name=$(echo "$repo" | cut -d@ -f 1)
      repo_url=$(echo "$repo" | cut -d@ -f 2)
      repo_name_ssc=$(echo "$repo_name" | tr '[:lower:]' '[:upper:]' | tr '[:punct:]' '_')
      repo_user=$(eval echo "\$HELM_REPO_${repo_name_ssc}_USER")
      repo_password=$(eval echo "\$HELM_REPO_${repo_name_ssc}_PASSWORD")

      if [[ "$repo_url" =~ oci://.* ]]
      then
        if [[ "$repo_user" ]] && [[ "$repo_password" ]]
        then
          registry_host=$(echo "$repo_url" | awk -F[/:] '{print $4}')
          log_info "--- login to OCI-registry \\e[32m${repo_name}\\e[0m: \\e[33;1m${registry_host}\\e[0m"
          export HELM_EXPERIMENTAL_OCI=1
          # shellcheck disable=SC2086
          echo "$repo_password" | helm ${TRACE+--debug} registry login "$registry_host" --username "$repo_user" --password-stdin
        else
          log_warn "--- OCI-registry \\e[32m${repo_name}\\e[0m (\\e[33;1m${repo_url}\\e[0m) defined, but no credentials found (\$HELM_REPO_${repo_name_ssc}_USER/\$HELM_REPO_${repo_name_ssc}_PASSWORD)"
        fi
      else
        if [[ "$repo_user" ]] && [[ "$repo_password" ]]
        then
          log_info "--- add repository \\e[32m${repo_name}\\e[0m: \\e[33;1m${repo_url}\\e[0m (with user/password auth)"
          # shellcheck disable=SC2086
          echo "$repo_password" | helm ${TRACE+--debug} repo add "$repo_name" "$repo_url" --username "$repo_user" --password-stdin
        else
          log_info "--- add repository \\e[32m${repo_name}\\e[0m: \\e[33;1m${repo_url}\\e[0m (unauthenticated)"
          # shellcheck disable=SC2086
          helm ${TRACE+--debug} repo add "$repo_name" "$repo_url"
        fi
      fi
    done

    # shellcheck disable=SC2086
    helm ${TRACE+--debug} repo update
  }

  function awkenvsubst() {
    awk '{while(match($0,"[$%]{[^}]*}")) {var=substr($0,RSTART+2,RLENGTH-3);val=ENVIRON[var]; gsub(/["\\]/,"\\\\&",val); gsub("\n","\\n",val);gsub("\r","\\r",val); gsub("[$%]{"var"}",val)}}1'
  }

  function exec_hook() {
    if [[ ! -x "$1" ]] && ! chmod +x "$1"
    then
      log_warn "... could not make \\e[33;1m${1}\\e[0m executable: please do it (chmod +x)"
      # fallback technique
      sh "$1"
    else
      "$1"
    fi
  }

  # deploy application
  function deploy() {
    export environment_type=$1
    export environment_name=$2
    namespace=$3
    values_files=$4
    environment_url=$5

    # variables expansion in $environment_url
    environment_url=$(echo "$environment_url" | awkenvsubst)
    export environment_url
    # extract hostname from $environment_url
    hostname=$(echo "$environment_url" | awk -F[/:] '{print $4}')
    export hostname

    log_info "--- \\e[32mdeploy\\e[0m"
    log_info "--- \$namespace: \\e[33;1m${namespace}\\e[0m"
    log_info "--- \$environment_type: \\e[33;1m${environment_type}\\e[0m (Helm variable '$HELM_ENV_VALUE_NAME')"
    log_info "--- \$environment_name: \\e[33;1m${environment_name}\\e[0m (used as release name)"
    log_info "--- \$hostname: \\e[33;1m${hostname}\\e[0m (Helm variable '$HELM_HOSTNAME_VALUE_NAME')"

    # unset any upstream deployment env & artifacts
    rm -f helm.env
    rm -f environment_url.txt

    # maybe execute pre deploy script
    prescript="$HELM_SCRIPTS_DIR/helm-pre-deploy.sh"
    if [[ -f "$prescript" ]]; then
      log_info "--- \\e[32mpre-deploy hook\\e[0m (\\e[33;1m${prescript}\\e[0m) found: execute"
      exec_hook "$prescript"
    else
      log_info "--- \\e[32mpre-deploy hook\\e[0m (\\e[33;1m${prescript}\\e[0m) not found: skip"
    fi

    helm_opts=${TRACE+--debug}
    
    helm_opts="$helm_opts --set ${HELM_ENV_VALUE_NAME}=$environment_type"
    helm_opts="$helm_opts --set ${HELM_HOSTNAME_VALUE_NAME}=$hostname"

    if [ -n "$HELM_COMMON_VALUES" ]; then
      log_info "--- using \\e[32mcommon values\\e[0m file: \\e[33;1m${HELM_COMMON_VALUES}\\e[0m"
      awkenvsubst < "$HELM_COMMON_VALUES" > generated-values-common.yml
      helm_opts="$helm_opts --values generated-values-common.yml"
    fi

    if [ -n "$values_files" ]; then
      log_info "--- using \\e[32mvalues\\e[0m file: \\e[33;1m${values_files}\\e[0m"
      awkenvsubst < "$values_files" > generated-values.yml
      helm_opts="$helm_opts --values generated-values.yml"
    fi

    if [ -f "$CI_PROJECT_DIR/.kubeconfig" ]; then
      log_info "--- using \\e[32mkubeconfig\\e[0m: \\e[33;1m$CI_PROJECT_DIR/.kubeconfig\\e[0m"
      helm_opts="$helm_opts --kubeconfig $CI_PROJECT_DIR/.kubeconfig"
    fi

    if [ -n "$namespace" ]; then
      log_info "--- using \\e[32mnamespace\\e[0m: \\e[33;1m${namespace}\\e[0m"
      helm_opts="$helm_opts --namespace $namespace"
    fi

    chart=${HELM_DEPLOY_CHART:-$HELM_CHART_DIR}
    if [ -z "${chart}" ]; then
      log_error "No Chart to deploy! Please use \\e[32m\$HELM_DEPLOY_CHART\\e[0m to deploy a chart from a repository"
      log_error "Or check the provided variables to package your own chart!"
      exit 1
    fi
    log_info "--- using \\e[32mchart\\e[0m: \\e[33;1m${chart}\\e[0m"

    # shellcheck disable=SC2086
    helm $helm_opts $HELM_DEPLOY_ARGS $environment_name $chart

    # maybe execute post deploy script
    postscript="$HELM_SCRIPTS_DIR/helm-post-deploy.sh"
    if [[ -f "$postscript" ]]; then
      log_info "--- \\e[32mpost-deploy hook\\e[0m (\\e[33;1m${postscript}\\e[0m) found: execute"
      exec_hook "$postscript"
    else
      log_info "--- \\e[32mpost-deploy hook\\e[0m (\\e[33;1m${postscript}\\e[0m) not found: skip"
    fi

    # persist environment url
    if [[ -f environment_url.txt ]]
    then
      environment_url=$(cat environment_url.txt)
      export environment_url
      log_info "--- dynamic environment url found: (\\e[33;1m$environment_url\\e[0m)"
    else
      echo "$environment_url" > environment_url.txt
    fi
    echo -e "environment_type=$environment_type\\nenvironment_name=$environment_name\\nenvironment_url=$environment_url" > helm.env
  }

  # delete application (and dependencies)
  function delete() {
    export environment_type=$1
    export environment_name=$2
    namespace=$3

    log_info "--- \\e[32mdelete"
    log_info "--- \$namespace: \\e[33;1m${namespace}\\e[0m"
    log_info "--- \$environment_type: \\e[33;1m${environment_type}\\e[0m"
    log_info "--- \$environment_name: \\e[33;1m${environment_name}\\e[0m (used as release name)"

    # maybe execute pre delete script
    prescript="$HELM_SCRIPTS_DIR/helm-pre-delete.sh"
    if [[ -f "$prescript" ]]; then
      log_info "--- \\e[32mpre-delete hook\\e[0m (\\e[33;1m${prescript}\\e[0m) found: execute"
      exec_hook "$prescript"
    else
      log_info "--- \\e[32mpre-delete hook\\e[0m (\\e[33;1m${prescript}\\e[0m) not found: skip"
    fi

    helm_opts=${TRACE+--debug}

    if [ -f "$CI_PROJECT_DIR/.kubeconfig" ]; then
      log_info "--- using \\e[32mkubeconfig\\e[0m: \\e[33;1m$CI_PROJECT_DIR/.kubeconfig\\e[0m"
      helm_opts="$helm_opts --kubeconfig $CI_PROJECT_DIR/.kubeconfig"
    fi

    if [ -n "$namespace" ]; then
      log_info "--- using \\e[32mnamespace\\e[0m: \\e[33;1m${namespace}\\e[0m"
      helm_opts="$helm_opts --namespace $namespace"
    fi

    # shellcheck disable=SC2086
    helm $helm_opts $HELM_DELETE_ARGS $environment_name

    # maybe execute post delete script
    postscript="$HELM_SCRIPTS_DIR/helm-post-delete.sh"
    if [[ -f "$postscript" ]]; then
      log_info "--- \\e[32mpost-delete hook\\e[0m (\\e[33;1m${postscript}\\e[0m) found: execute"
      exec_hook "$postscript"
    else
      log_info "--- \\e[32mpost-delete hook\\e[0m (\\e[33;1m${postscript}\\e[0m) not found: skip"
    fi
  }

  # test application (and dependencies)
  function test() {
    export environment_type=$1
    export environment_name=$2
    namespace=$3

    log_info "--- \\e[32mtest\\e[0m (env: ${environment_type})"
    log_info "--- \$namespace: \\e[33;1m${namespace}\\e[0m"
    log_info "--- \$environment_name: \\e[33;1m${environment_name}\\e[0m"
    log_info "--- \$environment_type: \\e[33;1m${environment_type}\\e[0m"

    helm_opts=${TRACE+--debug}

    if [ -f "$CI_PROJECT_DIR/.kubeconfig" ]; then
      log_info "--- using \\e[32mkubeconfig\\e[0m: \\e[33;1m$CI_PROJECT_DIR/.kubeconfig\\e[0m"
      helm_opts="$helm_opts --kubeconfig $CI_PROJECT_DIR/.kubeconfig"
    fi

    if [ -n "$namespace" ]; then
      log_info "--- using \\e[32mnamespace\\e[0m: \\e[33;1m${namespace}\\e[0m"
      helm_opts="$helm_opts --namespace $namespace"
    fi

    # shellcheck disable=SC2086
    helm $helm_opts $HELM_TEST_ARGS $environment_name
  }

  function helm_package() {
    # semantic-release integration
    if [[ "${SEMREL_INFO_ON}" && "${DOCKER_SEMREL_RELEASE_DISABLED}" != "true" ]]
    then
      if [[ -z "${SEMREL_INFO_NEXT_VERSION}" ]]
      then
        log_warn "[semantic-release] no new version to release: skip"
        exit 0
      else
        log_info "[semantic-release] use computed next version: \\e[1;94m${SEMREL_INFO_NEXT_VERSION}\\e[0m"
        helm_version_opts="--version ${SEMREL_INFO_NEXT_VERSION}"
      fi
    fi

    add_helm_repositories

    # helm package
    # shellcheck disable=SC2086
    helm ${TRACE+--debug} $HELM_PACKAGE_ARGS $helm_version_opts $HELM_CHART_DIR --destination helm_packages
  }

  function helm_publish() {
    helm_package=$(ls -1 ./helm_packages/*.tgz 2>/dev/null || echo "")
    if [[ -z "$helm_package" ]]; then
      log_error "No package found to deploy"
      exit 1
    fi
    helm_package_name=$(basename "$helm_package")
    log_info "--- Publishing Helm package ${helm_package_name} to: ${HELM_PUBLISH_URL}..."

    # method to lowercase
    HELM_PUBLISH_METHOD=$(echo "$HELM_PUBLISH_METHOD" | tr '[:upper:]' '[:lower:]')

    # auto-detect method
    if [[ "$HELM_PUBLISH_METHOD" == "auto" ]]
    then
        log_info "--- trying to auto detect publish method..."
        pubscript="$HELM_SCRIPTS_DIR/helm-publish.sh"
        if [[ -f "$pubscript" ]]
        then
          log_info "--- ... custom publish script (\\e[33;1m${postscript}\\e[0m) found: will use"
          HELM_PUBLISH_METHOD=custom
        elif [[ "$HELM_PUBLISH_URL" =~ oci://.* ]]
        then
          log_info "--- ... publish url looks like an OCI registry: will use helm push"
          HELM_PUBLISH_METHOD=push
        else
          log_info "--- ... publish url looks like a Chart repository: will use push method (uses cm-push plugin)"
          log_info "--- ... if auto-selected method is not suited, override with \$HELM_PUBLISH_METHOD or provide a custom publish script"
          HELM_PUBLISH_METHOD=push
        fi
    fi

    username="${HELM_PUBLISH_USER:-$CI_REGISTRY_USER}"
    password="${HELM_PUBLISH_PASSWORD:-$CI_REGISTRY_PASSWORD}"
    case "$HELM_PUBLISH_METHOD" in
    push)
      if [[ "$HELM_PUBLISH_URL" =~ oci://.* ]]
      then
        registry_host=$(echo "$HELM_PUBLISH_URL" | awk -F[/:] '{print $4}')
        # shellcheck disable=SC2086
        echo "$password" | helm ${TRACE+--debug} registry login "$registry_host" --username "$username" --password-stdin
        # enable OCI support prior to v3.8.0
        export HELM_EXPERIMENTAL_OCI=1
        # shellcheck disable=SC2086
        helm ${TRACE+--debug} push "$helm_package" "$HELM_PUBLISH_URL"
      else
        log_info "Installing cm-push plugin (version ${HELM_CM_PUSH_PLUGIN_VERSION:-latest})..."
        # shellcheck disable=SC2086
        helm ${TRACE+--debug} plugin install ${HELM_CM_PUSH_PLUGIN_VERSION:+--version "$HELM_CM_PUSH_PLUGIN_VERSION"} https://github.com/chartmuseum/helm-push || true
        # shellcheck disable=SC2086
        helm ${TRACE+--debug} cm-push --username "$username" --password "$password" "$helm_package" "$HELM_PUBLISH_URL"
      fi
      ;;
    post)
      if ! command -v curl > /dev/null
      then
        log_info "--- installing curl (required to publish Helm charts)..."
        apk add --no-cache curl
      fi
      curl --fail --request POST --form "chart=@$helm_package" --user "$username:$password" "$HELM_PUBLISH_URL"
      ;;
    put)
      wget -v --method=PUT --user="$username" --password="$password" --body-file="$helm_package" "$HELM_PUBLISH_URL/$helm_package_name" -O -
      ;;
    custom)
      pubscript="$HELM_SCRIPTS_DIR/helm-publish.sh"
      log_info "--- run custom publish script (\\e[33;1m${pubscript}\\e[0m)"
      exec_hook "$pubscript"
      ;;
    *)
      log_error "Unsupported publish method: $HELM_PUBLISH_METHOD"
      exit 1
      ;;
    esac
  }

  unscope_variables
  eval_all_secrets

  # ENDSCRIPT

# job prototype
# defines default Docker image, tracking probe, cache policy and tags
.helm-base:
  image:
    name: $HELM_CLI_IMAGE
    entrypoint: [""]
  services:
    - name: "$TBC_TRACKING_IMAGE"
      command: ["--service", "helm", "3.3.2" ]
  variables:
    HELM_CACHE_HOME: $CI_PROJECT_DIR/.cache/helm
    HELM_CONFIG_HOME: $CI_PROJECT_DIR/.config/helm
  before_script:
    - *helm-scripts
    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
  cache:
    key: "$CI_COMMIT_REF_SLUG-helm"
    paths:
      - .cache
      - .config

.helm-values-lint:
  extends: .helm-base
  image:
    name: $HELM_YAMLLINT_IMAGE
    entrypoint: [""]
  stage: test

.helm-score:
  extends: .helm-base
  image:
    name: $HELM_KUBE_SCORE_IMAGE
    entrypoint: [""]
  stage: package-test
  before_script:
    - *helm-scripts
    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
    - |
      if [ -f "$HELM_CHART_DIR/Chart.yaml" ]
      then
        helm $HELM_DEPENDENCY_ARGS $HELM_CHART_DIR
        helm_package=$HELM_CHART_DIR
      elif [ ! -z "${HELM_DEPLOY_CHART}" ]
      then
        add_helm_repositories
        helm_package=$HELM_DEPLOY_CHART
      else
        log_error "You need at least one Chart.yaml or external deploy chart reference"
        exit 1
      fi

# ==================================================
# Stage: check
# ==================================================
# lint-job is used to check the syntax of the Helm Chart for best practices.
helm-lint:
  extends: .helm-base
  stage: test
  before_script:
    - *helm-scripts
    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
    - add_helm_repositories
  script:
    - helm $HELM_DEPENDENCY_ARGS $HELM_CHART_DIR
    - helm ${TRACE+--debug} $HELM_LINT_ARGS $HELM_CHART_DIR
  rules:
    - if: '$HELM_LINT_DISABLED == "true"'
      when: never
    - exists:
        - "**/Chart.yaml"

# yamllint-job is used to check the syntax of the values files.
helm-values-common-lint:
  extends: .helm-values-lint
  script:
    - awkenvsubst < "$HELM_COMMON_VALUES" > generated-values-common.yml
    - yamllint -d "$HELM_YAMLLINT_CONFIG" $HELM_YAMLLINT_ARGS generated-values-common.yml
  rules:
    - if: '$HELM_YAMLLINT_DISABLED == "true"'
      when: never
    - if: '$HELM_COMMON_VALUES == null || $HELM_COMMON_VALUES == ""'
      when: never
    - !reference [.test-policy, rules]

helm-values-review-lint:
  extends: .helm-values-lint
  script:
    - awkenvsubst < "$HELM_REVIEW_VALUES" > generated-values-review.yml
    - yamllint -d "$HELM_YAMLLINT_CONFIG" $HELM_YAMLLINT_ARGS generated-values-review.yml
  rules:
    - if: '$HELM_YAMLLINT_DISABLED == "true"'
      when: never
    - if: '$HELM_REVIEW_VALUES == null || $HELM_REVIEW_VALUES == ""'
      when: never
    # only on non-production, non-integration branches
    - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF || $CI_COMMIT_REF_NAME =~ $INTEG_REF'
      when: never
    - !reference [.test-policy, rules]

helm-values-integration-lint:
  extends: .helm-values-lint
  script:
    - awkenvsubst < "$HELM_INTEG_VALUES" > generated-values-integration.yml
    - yamllint -d "$HELM_YAMLLINT_CONFIG" $HELM_YAMLLINT_ARGS generated-values-integration.yml
  rules:
    - if: '$HELM_YAMLLINT_DISABLED == "true"'
      when: never
    - if: '$HELM_INTEG_VALUES == null || $HELM_INTEG_VALUES == ""'
      when: never
    # only on non-production branches
    - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF'
      when: never
    - !reference [.test-policy, rules]

helm-values-staging-lint:
  extends: .helm-values-lint
  script:
    - awkenvsubst < "$HELM_STAGING_VALUES" > generated-values-staging.yml
    - yamllint -d "$HELM_YAMLLINT_CONFIG" $HELM_YAMLLINT_ARGS generated-values-staging.yml
  rules:
    - if: '$HELM_YAMLLINT_DISABLED == "true"'
      when: never
    - if: '$HELM_STAGING_VALUES == null || $HELM_STAGING_VALUES == ""'
      when: never
    - !reference [.test-policy, rules]

helm-values-prod-lint:
  extends: .helm-values-lint
  script:
    - awkenvsubst < "$HELM_PROD_VALUES" > generated-values-prod.yml
    - yamllint -d "$HELM_YAMLLINT_CONFIG" $HELM_YAMLLINT_ARGS generated-values-prod.yml
  rules:
    - if: '$HELM_YAMLLINT_DISABLED == "true"'
      when: never
    - if: '$HELM_PROD_VALUES == null || $HELM_PROD_VALUES == ""'
      when: never
    - !reference [.test-policy, rules]

helm-review-score:
  extends: .helm-score
  script:
    - if [ -z "$HELM_COMMON_VALUES" ]; then HELM_COMMON_VALUES=/dev/null; fi
    - awkenvsubst < "$HELM_COMMON_VALUES" > generated-values-common.yml
    - awkenvsubst < "$HELM_REVIEW_VALUES" > generated-values-review.yml
    - helm template $helm_package --values generated-values-common.yml --values generated-values-review.yml | kube-score score ${HELM_KUBE_SCORE_ARGS} -
  rules:
    # exclude when $HELM_KUBE_SCORE_DISABLED is set
    - if: '$HELM_KUBE_SCORE_DISABLED == "true"'
      when: never
    - if: '$HELM_REVIEW_VALUES == null || $HELM_REVIEW_VALUES == ""'
      when: never
    # only on non-production, non-integration branches
    - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF || $CI_COMMIT_REF_NAME =~ $INTEG_REF'
      when: never
    - !reference [.test-policy, rules]

helm-integration-score:
  extends: .helm-score
  script:
    - if [ -z "$HELM_COMMON_VALUES" ]; then HELM_COMMON_VALUES=/dev/null; fi
    - awkenvsubst < "$HELM_COMMON_VALUES" > generated-values-common.yml
    - awkenvsubst < "$HELM_INTEG_VALUES" > generated-values-integration.yml
    - helm template $helm_package --values generated-values-common.yml --values generated-values-integration.yml | kube-score score ${HELM_KUBE_SCORE_ARGS} -
  rules:
    # exclude when $HELM_SCORE_DISABLED is set
    - if: '$HELM_KUBE_SCORE_DISABLED == "true"'
      when: never
    - if: '$HELM_INTEG_VALUES == null || $HELM_INTEG_VALUES == ""'
      when: never
    # only on non-production branches
    - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF'
      when: never
    - !reference [.test-policy, rules]

helm-staging-score:
  extends: .helm-score
  script:
    - if [ -z "$HELM_COMMON_VALUES" ]; then HELM_COMMON_VALUES=/dev/null; fi
    - awkenvsubst < "$HELM_COMMON_VALUES" > generated-values-common.yml
    - awkenvsubst < "$HELM_STAGING_VALUES" > generated-values-staging.yml
    - helm template $helm_package --values generated-values-common.yml --values generated-values-staging.yml | kube-score score ${HELM_KUBE_SCORE_ARGS} -
  rules:
    # exclude when $HELM_SCORE_DISABLED is set
    - if: '$HELM_KUBE_SCORE_DISABLED == "true"'
      when: never
    - if: '$HELM_STAGING_VALUES == null || $HELM_STAGING_VALUES == ""'
      when: never
    - !reference [.test-policy, rules]

helm-prod-score:
  extends: .helm-score
  script:
    - if [ -z "$HELM_COMMON_VALUES" ]; then HELM_COMMON_VALUES=/dev/null; fi
    - awkenvsubst < "$HELM_COMMON_VALUES" > generated-values-common.yml
    - awkenvsubst < "$HELM_PROD_VALUES" > generated-values-prod.yml
    - helm template $helm_package --values generated-values-common.yml --values generated-values-prod.yml | kube-score score ${HELM_KUBE_SCORE_ARGS} -
  rules:
    # exclude when $HELM_SCORE_DISABLED is set
    - if: '$HELM_KUBE_SCORE_DISABLED == "true"'
      when: never
    - if: '$HELM_PROD_VALUES == null || $HELM_PROD_VALUES == ""'
      when: never
    - !reference [.test-policy, rules]

# ==================================================
# Stage: package-build
# ==================================================
helm-package:
  extends: .helm-base
  stage: package-build
  before_script:
    - *helm-scripts
    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
    - add_helm_repositories
  script:
    - helm_package
  rules:
    - exists:
        - "**/Chart.yaml"
  artifacts:
    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
    expire_in: 1 week
    paths:
      - helm_packages/

# ==================================================
# Stage: publish
# ==================================================
helm-publish:
  extends: .helm-base
  stage: publish
  script:
    - helm_publish
  rules:
    - if: '$HELM_PUBLISH_URL == null || $HELM_PUBLISH_URL == "" || $CI_COMMIT_REF_NAME !~ $PROD_REF || $HELM_PUBLISH_METHOD == "disabled"'
      when: never
    - if: '$AUTODEPLOY_TO_PROD == "true"'
      exists:
        - "**/Chart.yaml"
    # else: manual + blocking
    - exists:
        - "**/Chart.yaml"
      when: manual

# Deploy job prototype
# Can be extended to define a concrete environment
#
# @arg ENV_TYPE      : environment type
# @arg ENV_APP_NAME  : env-specific application name
# @arg ENV_APP_SUFFIX: env-specific application suffix
# @arg ENV_URL       : env-specific application url
# @arg ENV_KUBE_CONFIG: env-specific Kubeconfig
# @arg ENV_NAMESPACE : env-specific Kubernetes namespace
# @arg ENV_VALUES    : env-specific Helm values
.helm-deploy:
  extends: .helm-base
  stage: deploy
  variables:
    ENV_APP_SUFFIX: "-$CI_ENVIRONMENT_SLUG"
  before_script:
    - *helm-scripts
    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
    - assert_defined "${ENV_KUBE_CONFIG:-${HELM_DEFAULT_KUBE_CONFIG:-${KUBECONFIG}}}" 'Missing required env $ENV_KUBE_CONFIG or $HELM_DEFAULT_KUBE_CONFIG'
    - add_helm_repositories
    - setup_kubeconfig "${ENV_KUBE_CONFIG:-${HELM_DEFAULT_KUBE_CONFIG}}"
  script:
    - deploy $ENV_TYPE "${ENV_APP_NAME:-${HELM_BASE_APP_NAME}${ENV_APP_SUFFIX}}" "${ENV_NAMESPACE:-${KUBE_NAMESPACE}}" "$ENV_VALUES" "${ENV_URL:-${HELM_ENVIRONMENT_URL:-$ENV_URL_LEGACY}}"
  artifacts:
    name: "$ENV_TYPE env url for $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
    paths:
      - environment_url.txt
    reports:
      dotenv: helm.env
  resource_group: $CI_ENVIRONMENT_NAME
  environment:
    url: "$environment_url" # can be either static or dynamic

# Cleanup job prototype
# Can be extended for each deletable environment
#
# @arg ENV_TYPE      : environment type
# @arg ENV_APP_NAME  : env-specific application name
# @arg ENV_APP_SUFFIX: env-specific application suffix
# @arg ENV_KUBE_CONFIG: env-specific Kubeconfig
# @arg ENV_NAMESPACE : env-specific Kubernetes namespace
.helm-cleanup:
  extends: .helm-base
  stage: deploy
  # force no dependencies
  dependencies: []
  variables:
    ENV_APP_SUFFIX: "-$CI_ENVIRONMENT_SLUG"
  before_script:
    - *helm-scripts
    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
    - assert_defined "${ENV_KUBE_CONFIG:-${HELM_DEFAULT_KUBE_CONFIG:-${KUBECONFIG}}}" 'Missing required Kubeconfig'
    - setup_kubeconfig "${ENV_KUBE_CONFIG:-${HELM_DEFAULT_KUBE_CONFIG}}"
  script:
    - delete $ENV_TYPE "${ENV_APP_NAME:-${HELM_BASE_APP_NAME}${ENV_APP_SUFFIX}}" "${ENV_NAMESPACE:-${KUBE_NAMESPACE}}"
  environment:
    action: stop
  resource_group: $CI_ENVIRONMENT_NAME

# Test job prototype
# Can be extended to define a concrete environment
#
# @arg ENV_TYPE      : environment type
# @arg ENV_KUBE_CONFIG: env-specific Kubeconfig
# @arg ENV_NAMESPACE : env-specific Kubernetes namespace
.helm-test:
  extends: .helm-base
  stage: acceptance
  before_script:
    - *helm-scripts
    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
    - assert_defined "${ENV_KUBE_CONFIG:-${HELM_DEFAULT_KUBE_CONFIG:-${KUBECONFIG}}}" 'Missing required Kubeconfig'
    - setup_kubeconfig "${ENV_KUBE_CONFIG:-${HELM_DEFAULT_KUBE_CONFIG}}"
  script:
    - test "$environment_type" "$environment_name" "${ENV_NAMESPACE:-${KUBE_NAMESPACE}}"

# ==================================================
# Stage: review
# ==================================================
# deploy to review env (only for feature branches)
# enabled by default, disable this job by setting $HELM_REVIEW_DISABLED
helm-review:
  extends: .helm-deploy
  variables:
    ENV_TYPE: review
    ENV_APP_NAME: "$HELM_REVIEW_APP_NAME"
    ENV_URL: "${HELM_REVIEW_ENVIRONMENT_URL}"
    ENV_URL_LEGACY: "${HELM_REVIEW_ENVIRONMENT_SCHEME}://${CI_PROJECT_NAME}-${CI_ENVIRONMENT_SLUG}.${HELM_REVIEW_ENVIRONMENT_DOMAIN}"
    ENV_KUBE_CONFIG: "$HELM_REVIEW_KUBE_CONFIG"
    ENV_NAMESPACE: "$HELM_REVIEW_NAMESPACE"
    ENV_VALUES: "$HELM_REVIEW_VALUES"