gitlab-ci-gcloud.yml

# =========================================================================================
# Copyright (C) 2021 Orange & contributors
#
# This program is free software; you can redistribute it and/or modify it under the terms 
# of the GNU Lesser General Public License as published by the Free Software Foundation; 
# either version 3 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along with this 
# program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth 
# Floor, Boston, MA  02110-1301, USA.
# =========================================================================================
# default workflow rules: Merge Request pipelines
workflow:
  rules:
    # prevent branch pipeline when an MR is open (prefer MR pipeline)
    - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS'
      when: never
    - when: always

variables:
  # variabilized tracking image
  TBC_TRACKING_IMAGE: "$CI_REGISTRY/to-be-continuous/tools/tracking:master"

  # Default Docker image (can be overridden)
  GCP_CLI_IMAGE: "google/cloud-sdk:latest"

  GCP_SCRIPTS_DIR: "."
  
  GCP_BASE_APP_NAME: "$CI_PROJECT_NAME"
  GCP_REVIEW_ENVIRONMENT_SCHEME: "https"


  # default production ref name (pattern)
  PROD_REF: '/^(master|main)$/'
  # default integration ref name (pattern)
  INTEG_REF: '/^develop$/'

# allowed stages depend on your template type (see: to-be-continuous.gitlab.io/doc/dev-guidelines/#stages)
stages:
  - deploy
  - production

.gcp-scripts: &gcp-scripts |
  # BEGSCRIPT
  set -e

  function log_info() {
      echo -e "[\\e[1;94mINFO\\e[0m] $*"
  }

  function log_warn() {
      echo -e "[\\e[1;93mWARN\\e[0m] $*"
  }

  function log_error() {
      echo -e "[\\e[1;91mERROR\\e[0m] $*"
  }

  function fail() {
    log_error "$*"
    exit 1
  }

  function assert_defined() {
    if [[ -z "$1" ]]
    then
      log_error "$2"
      exit 1
    fi
  }

  function install_ca_certs() {
    certs=$1
    if [[ -z "$certs" ]]
    then
      return
    fi

    # import in system
    if echo "$certs" >> /etc/ssl/certs/ca-certificates.crt
    then
      log_info "CA certificates imported in \\e[33;1m/etc/ssl/certs/ca-certificates.crt\\e[0m"
    fi
    if echo "$certs" >> /etc/ssl/cert.pem
    then
      log_info "CA certificates imported in \\e[33;1m/etc/ssl/cert.pem\\e[0m"
    fi
  }

  function as_content() {
    file_or_content=$1
    if [[ -f ${file_or_content} ]]; then
      cat "${file_or_content}"
    else
      echo "${file_or_content}"
    fi
  }

  function unscope_variables() {
    _scoped_vars=$(env | awk -F '=' "/^scoped__[a-zA-Z0-9_]+=/ {print \$1}" | sort)
    if [[ -z "$_scoped_vars" ]]; then return; fi
    log_info "Processing scoped variables..."
    for _scoped_var in $_scoped_vars
    do
      _fields=${_scoped_var//__/:}
      _condition=$(echo "$_fields" | cut -d: -f3)
      case "$_condition" in
      if) _not="";;
      ifnot) _not=1;;
      *)
        log_warn "... unrecognized condition \\e[1;91m$_condition\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"
        continue
      ;;
      esac
      _target_var=$(echo "$_fields" | cut -d: -f2)
      _cond_var=$(echo "$_fields" | cut -d: -f4)
      _cond_val=$(eval echo "\$${_cond_var}")
      _test_op=$(echo "$_fields" | cut -d: -f5)
      case "$_test_op" in
      defined)
        if [[ -z "$_not" ]] && [[ -z "$_cond_val" ]]; then continue; 
        elif [[ "$_not" ]] && [[ "$_cond_val" ]]; then continue; 
        fi
        ;;
      equals|startswith|endswith|contains|in|equals_ic|startswith_ic|endswith_ic|contains_ic|in_ic)
        # comparison operator
        # sluggify actual value
        _cond_val=$(echo "$_cond_val" | tr '[:punct:]' '_')
        # retrieve comparison value
        _cmp_val_prefix="scoped__${_target_var}__${_condition}__${_cond_var}__${_test_op}__"
        _cmp_val=${_scoped_var#"$_cmp_val_prefix"}
        # manage 'ignore case'
        if [[ "$_test_op" == *_ic ]]
        then
          # lowercase everything
          _cond_val=$(echo "$_cond_val" | tr '[:upper:]' '[:lower:]')
          _cmp_val=$(echo "$_cmp_val" | tr '[:upper:]' '[:lower:]')
        fi
        case "$_test_op" in
        equals*)
          if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val" ]]; then continue; 
          elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val" ]]; then continue; 
          fi
          ;;
        startswith*)
          if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val"* ]]; then continue; 
          elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val"* ]]; then continue; 
          fi
          ;;
        endswith*)
          if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val" ]]; then continue; 
          elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val" ]]; then continue; 
          fi
          ;;
        contains*)
          if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val"* ]]; then continue; 
          elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val"* ]]; then continue; 
          fi
          ;;
        in*)
          if [[ -z "$_not" ]] && [[ "__${_cmp_val}__" != *"__${_cond_val}__"* ]]; then continue; 
          elif [[ "$_not" ]] && [[ "__${_cmp_val}__" == *"__${_cond_val}__"* ]]; then continue; 
          fi
          ;;
        esac
        ;;
      *)
        log_warn "... unrecognized test operator \\e[1;91m${_test_op}\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"
        continue
        ;;
      esac
      # matches
      _val=$(eval echo "\$${_target_var}")
      log_info "... apply \\e[32m${_target_var}\\e[0m from \\e[32m\$${_scoped_var}\\e[0m${_val:+ (\\e[33;1moverwrite\\e[0m)}"
      _val=$(eval echo "\$${_scoped_var}")
      export "${_target_var}"="${_val}"
    done
    log_info "... done"
  }

  # evaluate and export a secret
  # - $1: secret variable name
  function eval_secret() {
    name=$1
    value=$(eval echo "\$${name}")
    case "$value" in
    @b64@*)
      decoded=$(mktemp)
      errors=$(mktemp)
      if echo "$value" | cut -c6- | base64 -d > "${decoded}" 2> "${errors}"
      then
        # shellcheck disable=SC2086
        export ${name}="$(cat ${decoded})"
        log_info "Successfully decoded base64 secret \\e[33;1m${name}\\e[0m"
      else
        fail "Failed decoding base64 secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
      fi
      ;;
    @hex@*)
      decoded=$(mktemp)
      errors=$(mktemp)
      if echo "$value" | cut -c6- | sed 's/\([0-9A-F]\{2\}\)/\\\\x\1/gI' | xargs printf > "${decoded}" 2> "${errors}"
      then
        # shellcheck disable=SC2086
        export ${name}="$(cat ${decoded})"
        log_info "Successfully decoded hexadecimal secret \\e[33;1m${name}\\e[0m"
      else
        fail "Failed decoding hexadecimal secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
      fi
      ;;
    @url@*)
      url=$(echo "$value" | cut -c6-)
      if command -v curl > /dev/null
      then
        decoded=$(mktemp)
        errors=$(mktemp)
        if curl -s -S -f --connect-timeout 5 -o "${decoded}" "$url" 2> "${errors}"
        then
          # shellcheck disable=SC2086
          export ${name}="$(cat ${decoded})"
          log_info "Successfully curl'd secret \\e[33;1m${name}\\e[0m"
        else
          log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
        fi
      elif command -v wget > /dev/null
      then
        decoded=$(mktemp)
        errors=$(mktemp)
        if wget -T 5 -O "${decoded}" "$url" 2> "${errors}"
        then
          # shellcheck disable=SC2086
          export ${name}="$(cat ${decoded})"
          log_info "Successfully wget'd secret \\e[33;1m${name}\\e[0m"
        else
          log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
        fi
      else
        log_warn "Couldn't get secret \\e[33;1m${name}\\e[0m: no http client found"
      fi
      ;;
    esac
  }

  function eval_all_secrets() {
    encoded_vars=$(env | grep -v '^scoped__' | awk -F '=' '/^[a-zA-Z0-9_]*=@(b64|hex|url)@/ {print $1}')
    for var in $encoded_vars
    do
      eval_secret "$var"
    done
  }
  
  function awkenvsubst() {
    awk '{while(match($0,"[$]{[^}]*}")) {var=substr($0,RSTART+2,RLENGTH -3);val=ENVIRON[var];gsub(/["\\]/,"\\\\&", val);gsub("\n", "\\n", val);gsub("\r", "\\r", val);gsub("[$]{"var"}",val)}}1'
  }

  
  # Google Cloud Authentication
  function gcp_auth() {
    gcp_key_file="$1"
    oidc_provider="$2"
    oidc_account="$3"

    if [[ "$oidc_provider" ]]
    then
      # Use Workload Identity Federation to authenticate
      # see: https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/
      log_info "Authenticating with OpenID Connect..."
      assert_defined "$oidc_account" 'Missing required OpenID Connect service account'
      echo "${CI_JOB_JWT_V2}" > /tmp/.ci_job_jwt_file 
      gcloud iam workload-identity-pools create-cred-config "$oidc_provider" \
        --service-account="$oidc_account" \
        --output-file=/tmp/.gcp_temp_cred.json \
        --credential-source-file=/tmp/.ci_job_jwt_file
      gcloud auth login --cred-file=/tmp/.gcp_temp_cred.json
    else
      # Use gcp_key_file to authenticate
      log_info "Authenticating with Service Account key file..."
      assert_defined "$gcp_key_file" 'Missing required GCP key file (JSON)'
      as_content "$gcp_key_file" > /tmp/gcp_key.json
      gcloud auth activate-service-account --key-file /tmp/gcp_key.json    
    fi
  }


  # application deployment function
  function deploy() {
    export env=$1
    export appname=$2
    export gcp_project_id=$3
    export environment_url=$4
    # extract hostname from $environment_url
    hostname=$(echo "$environment_url" | awk -F[/:] '{print $4}')
    export hostname

    log_info "--- \\e[32mdeploy\\e[0m (env: \\e[33;1m${env}\\e[0m)"
    log_info "--- \$appname: \\e[33;1m${appname}\\e[0m"
    log_info "--- \$env: \\e[33;1m${env}\\e[0m"
    log_info "--- \$hostname: \\e[33;1m${hostname}\\e[0m"
    log_info "--- \$gcp_project_id: \\e[33;1m${gcp_project_id}\\e[0m"

    # unset any upstream deployment env & artifacts
    unset environment_name
    unset environment_type
    rm -f gcloud.env
    rm -f environment_url.txt

    deployscript=$(ls -1 "$GCP_SCRIPTS_DIR/gcp-deploy-${env}.sh" 2>/dev/null || ls -1 "$GCP_SCRIPTS_DIR/gcp-deploy.sh" 2>/dev/null || echo "")
    if [[ -f "$deployscript" ]]
    then
      log_info "--- deploy script (\\e[33;1m${deployscript}\\e[0m) found: execute"
      chmod +x "$deployscript"
      "$deployscript"
    else
      log_error "--- no deploy script found: abort"
      exit 1
    fi

    # finally persist environment url
    if [[ -f environment_url.txt ]]
    then
      environment_url=$(cat environment_url.txt)
      export environment_url
      log_info "--- dynamic environment url found: (\\e[33;1m$environment_url\\e[0m)"
    else
      echo "$environment_url" > environment_url.txt
    fi
    echo -e "environment_type=$env\\nenvironment_name=$appname\\nenvironment_url=$environment_url" > gcloud.env
  }

  # environment cleanup function
  function delete() {
    export env=$1
    export appname=$2
    export gcp_project_id=$3

    log_info "--- \\e[32mdelete\\e[0m (env: ${env})"
    log_info "--- \$appname: \\e[33;1m${appname}\\e[0m"
    log_info "--- \$env: \\e[33;1m${env}\\e[0m"
    log_info "--- \$gcp_project_id: \\e[33;1m${gcp_project_id}\\e[0m"

    cleanupscript=$(ls -1 "$GCP_SCRIPTS_DIR/gcp-cleanup-${env}.sh" 2>/dev/null || ls -1 "$GCP_SCRIPTS_DIR/gcp-cleanup.sh" 2>/dev/null || echo "")
    if [[ -f "$cleanupscript" ]]
    then
      log_info "--- cleanup script (\\e[33;1m${cleanupscript}\\e[0m) found: execute"
      chmod +x "$cleanupscript"
      "$cleanupscript"
    else
      log_error "--- no cleanup script found: abort"
      exit 1
    fi
  }


  # export tool functions (might be used in after_script)
  export -f log_info log_warn log_error assert_defined awkenvsubst

  unscope_variables
  eval_all_secrets

  # ENDSCRIPT

# job prototype
# defines default Docker image, tracking probe, cache policy and tags
.gcp-base:
  image: $GCP_CLI_IMAGE
  services:
    - name: "$TBC_TRACKING_IMAGE"
      command: ["--service", "gcloud", "2.1.0" ]
  before_script:
    - *gcp-scripts
    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"

# Deploy job prototype
# Can be extended to define a concrete environment
#
# @arg ENV_TYPE       : environment type
# @arg ENV_APP_NAME  : env-specific application name
# @arg ENV_APP_SUFFIX: env-specific application suffix
# @arg ENV_PROJECT   : env-specific GCP Project ID
# @arg ENV_KEY_FILE  : env-specific GCP API key file (JSON)
# @arg ENV_URL       : env-specific application url
.gcp-deploy:
  extends: .gcp-base
  stage: deploy 
  variables:
    ENV_APP_SUFFIX: "-$CI_ENVIRONMENT_SLUG"
  before_script:
    - *gcp-scripts
    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
    - gcp_auth "${ENV_KEY_FILE:-$GCP_KEY_FILE}" "${ENV_OIDC_PROVIDER:-$GCP_OIDC_PROVIDER}" "${ENV_OIDC_ACCOUNT:-$GCP_OIDC_ACCOUNT}"

  script:
    - deploy "$ENV_TYPE" "${ENV_APP_NAME:-${GCP_BASE_APP_NAME}${ENV_APP_SUFFIX}}" "$ENV_PROJECT" "$ENV_URL"
  artifacts:
    name: "$ENV_TYPE env url for $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
    paths:
      - environment_url.txt
    reports:
      dotenv: gcloud.env
  environment:
    url: "$environment_url" # can be either static or dynamic

# Cleanup job prototype
# Can be extended for each deletable environment
#
# @arg ENV_TYPE      : environment type
# @arg ENV_APP_NAME  : env-specific application name
# @arg ENV_APP_SUFFIX: env-specific application suffix
# @arg ENV_PROJECT   : env-specific GCP Project ID
# @arg ENV_KEY_FILE  : env-specific GCP API key file (JSON)
.gcp-cleanup:
  extends: .gcp-base
  stage: deploy
  # force no dependencies
  dependencies: []
  variables:
    ENV_APP_SUFFIX: "-$CI_ENVIRONMENT_SLUG"
  before_script:
    - *gcp-scripts
    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
    - gcp_auth "${ENV_KEY_FILE:-$GCP_KEY_FILE}" "${ENV_OIDC_PROVIDER:-$GCP_OIDC_PROVIDER}" "${ENV_OIDC_ACCOUNT:-$GCP_OIDC_ACCOUNT}"
  script:
    - delete "$ENV_TYPE" "${ENV_APP_NAME:-${GCP_BASE_APP_NAME}${ENV_APP_SUFFIX}}" "$ENV_PROJECT"
  environment:
    action: stop


# deploy to review env (only on feature branches)
# disabled by default, enable this job by setting $GCP_REVIEW_PROJECT.
gcp-review:
  extends: .gcp-deploy
  variables:
    ENV_TYPE: review
    ENV_APP_NAME: "$GCP_REVIEW_APP_NAME"
    ENV_PROJECT: "$GCP_REVIEW_PROJECT"
    ENV_OIDC_PROVIDER: "$GCP_REVIEW_OIDC_PROVIDER"
    ENV_OIDC_ACCOUNT: "$GCP_REVIEW_OIDC_ACCOUNT"
    ENV_KEY_FILE: "$GCP_REVIEW_KEY_FILE"
    ENV_URL: "${GCP_REVIEW_ENVIRONMENT_SCHEME}://${CI_PROJECT_NAME}-${CI_ENVIRONMENT_SLUG}.${GCP_REVIEW_ENVIRONMENT_DOMAIN}"
  environment:
    name: review/$CI_COMMIT_REF_NAME
    on_stop: gcp-cleanup-review
  resource_group: review/$CI_COMMIT_REF_NAME
  rules:
    # exclude tags
    - if: $CI_COMMIT_TAG
      when: never
    # exclude if $CLEANUP_ALL_REVIEW set to 'force'
    - if: '$CLEANUP_ALL_REVIEW == "force"'
      when: never
    # only on non-production, non-integration branches, with $GCP_REVIEW_PROJECT set
    - if: '$GCP_REVIEW_PROJECT && $CI_COMMIT_REF_NAME !~ $PROD_REF && $CI_COMMIT_REF_NAME !~ $INTEG_REF'

# cleanup review env (automatically triggered once branches are deleted)
gcp-cleanup-review:
  extends: .gcp-cleanup
  variables:
    ENV_TYPE: review
    ENV_APP_NAME: "$GCP_REVIEW_APP_NAME"
    ENV_PROJECT: "$GCP_REVIEW_PROJECT"
    ENV_OIDC_PROVIDER: "$GCP_REVIEW_OIDC_PROVIDER"
    ENV_OIDC_ACCOUNT: "$GCP_REVIEW_OIDC_ACCOUNT"
    ENV_KEY_FILE: "$GCP_REVIEW_KEY_FILE"
    ENV_URL: "${GCP_REVIEW_ENVIRONMENT_URL}"
  environment:
    name: review/$CI_COMMIT_REF_NAME
    action: stop
  resource_group: review/$CI_COMMIT_REF_NAME
  rules:
    # exclude tags
    - if: $CI_COMMIT_TAG
      when: never
    # only on non-production, non-integration branches, with $GCP_REVIEW_PROJECT set
    - if: '$GCP_REVIEW_PROJECT && $CI_COMMIT_REF_NAME !~ $PROD_REF && $CI_COMMIT_REF_NAME !~ $INTEG_REF'
      when: manual
      allow_failure: true

# deploy to integration env (only on develop branch)
gcp-integration:
  extends: .gcp-deploy
  variables:
    ENV_TYPE: integration
    ENV_APP_NAME: "$GCP_INTEG_APP_NAME"
    ENV_PROJECT: "$GCP_INTEG_PROJECT"
    ENV_OIDC_PROVIDER: "$GCP_INTEG_OIDC_PROVIDER"
    ENV_OIDC_ACCOUNT: "$GCP_INTEG_OIDC_ACCOUNT"
    ENV_KEY_FILE: "$GCP_INTEG_KEY_FILE"
    ENV_URL: "${GCP_INTEG_ENVIRONMENT_URL}"
  environment:
    name: integration
  resource_group: integration
  rules:
    # only on integration branch(es), with $GCP_INTEG_PROJECT set
    - if: '$GCP_INTEG_PROJECT && $CI_COMMIT_REF_NAME =~ $INTEG_REF'

# deploy to staging env (only on master branch)
gcp-staging:
  extends: .gcp-deploy
  variables:
    ENV_TYPE: staging
    ENV_APP_NAME: "$GCP_STAGING_APP_NAME"
    ENV_PROJECT: "$GCP_STAGING_PROJECT"
    ENV_OIDC_PROVIDER: "$GCP_STAGING_OIDC_PROVIDER"
    ENV_OIDC_ACCOUNT: "$GCP_STAGING_OIDC_ACCOUNT"
    ENV_KEY_FILE: "$GCP_STAGING_KEY_FILE"
    ENV_URL: "${GCP_STAGING_ENVIRONMENT_URL}"
  environment:
    name: staging
  resource_group: staging
  rules:
    # only on production branch(es), with $GCP_STAGING_PROJECT set
    - if: '$GCP_STAGING_PROJECT && $CI_COMMIT_REF_NAME =~ $PROD_REF'

# Deploy to production if on branch master and variable GCP_PROD_PROJECT defined and AUTODEPLOY_TO_PROD is set
gcp-production:
  extends: .gcp-deploy
  stage: production
  variables:
    ENV_TYPE: production
    ENV_APP_SUFFIX: "" # no suffix for prod
    ENV_APP_NAME: "$GCP_PROD_APP_NAME"
    ENV_PROJECT: "$GCP_PROD_PROJECT"
    ENV_OIDC_PROVIDER: "$GCP_PROD_OIDC_PROVIDER"
    ENV_OIDC_ACCOUNT: "$GCP_PROD_OIDC_ACCOUNT"
    ENV_KEY_FILE: "$GCP_PROD_KEY_FILE"
    ENV_URL: "${GCP_PROD_ENVIRONMENT_URL}"
  environment:
    name: production
  resource_group: production
  rules:
    # exclude non-production branches
    - if: '$CI_COMMIT_REF_NAME !~ $PROD_REF'
      when: never
    # exclude if $GCP_PROD_PROJECT not set
    - if: '$GCP_PROD_PROJECT == null || $GCP_PROD_PROJECT == ""'
      when: never
    # if $AUTODEPLOY_TO_PROD: auto
    - if: '$AUTODEPLOY_TO_PROD == "true"'
    # else if PUBLISH_ON_PROD enabled: auto (because the publish job was blocking)
    - if: '$PUBLISH_ON_PROD == "true"'
    # else: manual, blocking
    - if: $GCP_PROD_PROJECT # useless test, just to prevent GitLab warning
      when: manual