# ========================================================================================= # Copyright (C) 2021 Orange & contributors # # This program is free software; you can redistribute it and/or modify it under the terms # of the GNU Lesser General Public License as published by the Free Software Foundation; # either version 3 of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; # without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License along with this # program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth # Floor, Boston, MA 02110-1301, USA. # ========================================================================================= variables: # Default Docker image (can be overridden) GCP_CLI_IMAGE: "google/cloud-sdk:latest" GCP_SCRIPTS_DIR: "." GCP_BASE_APP_NAME: "$CI_PROJECT_NAME" GCP_REVIEW_ENVIRONMENT_SCHEME: "https" # default production ref name (pattern) PROD_REF: '/^master$/' # default integration ref name (pattern) INTEG_REF: '/^develop$/' # allowed stages depend on your template type (see: to-be-continuous.gitlab.io/doc/dev-guidelines/#stages) stages: - deploy - production .gcp-scripts: &gcp-scripts | # BEGSCRIPT set -e function log_info() { echo -e "[\\e[1;94mINFO\\e[0m] $*" } function log_warn() { echo -e "[\\e[1;93mWARN\\e[0m] $*" } function log_error() { echo -e "[\\e[1;91mERROR\\e[0m] $*" } function fail() { log_error "$*" exit 1 } function assert_defined() { if [[ -z "$1" ]] then log_error "$2" exit 1 fi } function install_ca_certs() { certs=$1 if [[ -z "$certs" ]] then return fi # import in system if echo "$certs" >> /etc/ssl/certs/ca-certificates.crt then log_info "CA certificates imported in \\e[33;1m/etc/ssl/certs/ca-certificates.crt\\e[0m" fi if echo "$certs" >> /etc/ssl/cert.pem then log_info "CA certificates imported in \\e[33;1m/etc/ssl/cert.pem\\e[0m" fi } function unscope_variables() { _scoped_vars=$(env | awk -F '=' "/^scoped__[a-zA-Z0-9_]+=/ {print \$1}" | sort) if [[ -z "$_scoped_vars" ]]; then return; fi log_info "Processing scoped variables..." for _scoped_var in $_scoped_vars do _fields=${_scoped_var//__/:} _condition=$(echo "$_fields" | cut -d: -f3) case "$_condition" in if) _not="";; ifnot) _not=1;; *) log_warn "... unrecognized condition \\e[1;91m$_condition\\e[0m in \\e[33;1m${_scoped_var}\\e[0m" continue ;; esac _target_var=$(echo "$_fields" | cut -d: -f2) _cond_var=$(echo "$_fields" | cut -d: -f4) _cond_val=$(eval echo "\$${_cond_var}") _test_op=$(echo "$_fields" | cut -d: -f5) case "$_test_op" in defined) if [[ -z "$_not" ]] && [[ -z "$_cond_val" ]]; then continue; elif [[ "$_not" ]] && [[ "$_cond_val" ]]; then continue; fi ;; equals|startswith|endswith|contains|in|equals_ic|startswith_ic|endswith_ic|contains_ic|in_ic) # comparison operator # sluggify actual value _cond_val=$(echo "$_cond_val" | tr '[:punct:]' '_') # retrieve comparison value _cmp_val_prefix="scoped__${_target_var}__${_condition}__${_cond_var}__${_test_op}__" _cmp_val=${_scoped_var#$_cmp_val_prefix} # manage 'ignore case' if [[ "$_test_op" == *_ic ]] then # lowercase everything _cond_val=$(echo "$_cond_val" | tr '[:upper:]' '[:lower:]') _cmp_val=$(echo "$_cmp_val" | tr '[:upper:]' '[:lower:]') fi case "$_test_op" in equals*) if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val" ]]; then continue; elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val" ]]; then continue; fi ;; startswith*) if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val"* ]]; then continue; elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val"* ]]; then continue; fi ;; endswith*) if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val" ]]; then continue; elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val" ]]; then continue; fi ;; contains*) if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val"* ]]; then continue; elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val"* ]]; then continue; fi ;; in*) if [[ -z "$_not" ]] && [[ "__${_cmp_val}__" != *"__${_cond_val}__"* ]]; then continue; elif [[ "$_not" ]] && [[ "__${_cmp_val}__" == *"__${_cond_val}__"* ]]; then continue; fi ;; esac ;; *) log_warn "... unrecognized test operator \\e[1;91m${_test_op}\\e[0m in \\e[33;1m${_scoped_var}\\e[0m" continue ;; esac # matches _val=$(eval echo "\$${_target_var}") log_info "... apply \\e[32m${_target_var}\\e[0m from \\e[32m\$${_scoped_var}\\e[0m${_val:+ (\\e[33;1moverwrite\\e[0m)}" _val=$(eval echo "\$${_scoped_var}") export "${_target_var}"="${_val}" done log_info "... done" } # evaluate and export a secret # - $1: secret variable name function eval_secret() { name=$1 value=$(eval echo "\$${name}") case "$value" in @b64@*) decoded=$(mktemp) errors=$(mktemp) if echo "$value" | cut -c6- | base64 -d > "${decoded}" 2> "${errors}" then # shellcheck disable=SC2086 export ${name}="$(cat ${decoded})" log_info "Successfully decoded base64 secret \\e[33;1m${name}\\e[0m" else fail "Failed decoding base64 secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")" fi ;; @hex@*) decoded=$(mktemp) errors=$(mktemp) if echo "$value" | cut -c6- | sed 's/\([0-9A-F]\{2\}\)/\\\\x\1/gI' | xargs printf > "${decoded}" 2> "${errors}" then # shellcheck disable=SC2086 export ${name}="$(cat ${decoded})" log_info "Successfully decoded hexadecimal secret \\e[33;1m${name}\\e[0m" else fail "Failed decoding hexadecimal secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")" fi ;; @url@*) url=$(echo "$value" | cut -c6-) if command -v curl > /dev/null then decoded=$(mktemp) errors=$(mktemp) if curl -s -S -f --connect-timeout 5 -o "${decoded}" "$url" 2> "${errors}" then # shellcheck disable=SC2086 export ${name}="$(cat ${decoded})" log_info "Successfully curl'd secret \\e[33;1m${name}\\e[0m" else fail "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")" fi elif command -v wget > /dev/null then decoded=$(mktemp) errors=$(mktemp) if wget -T 5 -O "${decoded}" "$url" 2> "${errors}" then # shellcheck disable=SC2086 export ${name}="$(cat ${decoded})" log_info "Successfully wget'd secret \\e[33;1m${name}\\e[0m" else fail "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")" fi else fail "Couldn't get secret \\e[33;1m${name}\\e[0m: no http client found" fi ;; esac } function eval_all_secrets() { encoded_vars=$(env | grep -v '^scoped__' | awk -F '=' '/^[a-zA-Z0-9_]*=@(b64|hex|url)@/ {print $1}') for var in $encoded_vars do eval_secret "$var" done } function awkenvsubst() { awk '{while(match($0,"[$]{[^}]*}")) {var=substr($0,RSTART+2,RLENGTH -3);val=ENVIRON[var];gsub(/["\\]/,"\\\\&", val);gsub("\n", "\\n", val);gsub("\r", "\\r", val);gsub("[$]{"var"}",val)}}1' } # application deployment function function deploy() { export env=$1 export appname=$2 export gcp_project_id=$3 # extract hostname from $CI_ENVIRONMENT_URL hostname=$(echo "$CI_ENVIRONMENT_URL" | awk -F[/:] '{print $4}') export hostname log_info "--- \\e[32mdeploy\\e[0m (env: \\e[33;1m${env}\\e[0m)" log_info "--- \$appname: \\e[33;1m${appname}\\e[0m" log_info "--- \$env: \\e[33;1m${env}\\e[0m" log_info "--- \$hostname: \\e[33;1m${hostname}\\e[0m" log_info "--- \$gcp_project_id: \\e[33;1m${gcp_project_id}\\e[0m" deployscript=$(ls -1 "$GCP_SCRIPTS_DIR/gcp-deploy-${env}.sh" 2>/dev/null || ls -1 "$GCP_SCRIPTS_DIR/gcp-deploy.sh" 2>/dev/null || echo "") if [[ -f "$deployscript" ]] then log_info "--- deploy script (\\e[33;1m${deployscript}\\e[0m) found: execute" sh "$deployscript" else log_error "--- no deploy script found: abort" exit 1 fi # finally persist environment url echo "$CI_ENVIRONMENT_URL" > environment_url.txt echo -e "environment_type=$env\\nenvironment_name=$appname\\nenvironment_url=$CI_ENVIRONMENT_URL" > gcloud.env } # environment cleanup function function delete() { export env=$1 export appname=$2 export gcp_project_id=$3 log_info "--- \\e[32mdelete\\e[0m (env: ${env})" log_info "--- \$appname: \\e[33;1m${appname}\\e[0m" log_info "--- \$env: \\e[33;1m${env}\\e[0m" log_info "--- \$gcp_project_id: \\e[33;1m${gcp_project_id}\\e[0m" cleanupscript=$(ls -1 "$GCP_SCRIPTS_DIR/gcp-cleanup-${env}.sh" 2>/dev/null || ls -1 "$GCP_SCRIPTS_DIR/gcp-cleanup.sh" 2>/dev/null || echo "") if [[ -f "$cleanupscript" ]] then log_info "--- cleanup script (\\e[33;1m${cleanupscript}\\e[0m) found: execute" sh "$cleanupscript" else log_error "--- no cleanup script found: abort" exit 1 fi } function get_latest_template_version() { tag_json=$(wget -T 5 -q -O - "$CI_API_V4_URL/projects/to-be-continuous%2F$1/repository/tags?per_page=1" 2> /dev/null || curl -s --connect-timeout 5 "$CI_API_V4_URL/projects/to-be-continuous%2F$1/repository/tags?per_page=1" 2> /dev/null || echo "") echo "$tag_json" | sed -rn 's/^.*"name":"([^"]*)".*$/\1/p' } function check_for_update() { template="$1" actual="$2" latest=$(get_latest_template_version "$template") if [[ -n "$latest" ]] && [[ "$latest" != "$actual" ]] then log_warn "\\e[1;93m=======================================================================================================\\e[0m" log_warn "\\e[93mThe template \\e[32m$template\\e[93m:\\e[33m$actual\\e[93m you're using is not up-to-date: consider upgrading to version \\e[32m$latest\\e[0m" log_warn "\\e[93m(set \$TEMPLATE_CHECK_UPDATE_DISABLED to disable this message)\\e[0m" log_warn "\\e[1;93m=======================================================================================================\\e[0m" fi } if [[ -z "$TEMPLATE_CHECK_UPDATE_DISABLED" ]]; then check_for_update gcloud "1.2.0"; fi # export tool functions (might be used in after_script) export -f log_info log_warn log_error assert_defined awkenvsubst unscope_variables eval_all_secrets # ENDSCRIPT # job prototype # defines default Docker image, tracking probe, cache policy and tags .gcp-base: image: $GCP_CLI_IMAGE services: - name: "$CI_REGISTRY/to-be-continuous/tools/tracking:master" command: ["--service", "gcloud", "1.2.0" ] before_script: - *gcp-scripts - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}" # Deploy job prototype # Can be extended to define a concrete environment # # @arg ENV_TYPE : environment type # @arg ENV_APP_NAME : env-specific application name # @arg ENV_APP_SUFFIX: env-specific application suffix # @arg ENV_PROJECT : env-specific GCP Project ID # @arg ENV_KEY_FILE : env-specific GCP API key file (JSON) .gcp-deploy: extends: .gcp-base stage: deploy variables: ENV_APP_SUFFIX: "-$CI_ENVIRONMENT_SLUG" before_script: - *gcp-scripts - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}" - assert_defined "${ENV_KEY_FILE:-$GCP_KEY_FILE}" 'Missing required GCP key file (JSON)' - gcloud auth activate-service-account --key-file ${ENV_KEY_FILE:-$GCP_KEY_FILE} script: - deploy "$ENV_TYPE" "${ENV_APP_NAME:-${GCP_BASE_APP_NAME}${ENV_APP_SUFFIX}}" "$ENV_PROJECT" artifacts: name: "$ENV_TYPE env url for $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG" paths: - environment_url.txt reports: dotenv: gcloud.env # Cleanup job prototype # Can be extended for each deletable environment # # @arg ENV_TYPE : environment type # @arg ENV_APP_NAME : env-specific application name # @arg ENV_APP_SUFFIX: env-specific application suffix # @arg ENV_PROJECT : env-specific GCP Project ID # @arg ENV_KEY_FILE : env-specific GCP API key file (JSON) .gcp-cleanup: extends: .gcp-base stage: deploy # force no dependencies dependencies: [] variables: ENV_APP_SUFFIX: "-$CI_ENVIRONMENT_SLUG" before_script: - *gcp-scripts - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}" - assert_defined "${ENV_KEY_FILE:-$GCP_KEY_FILE}" 'Missing required GCP key file (JSON)' - gcloud auth activate-service-account --key-file ${ENV_KEY_FILE:-$GCP_KEY_FILE} script: - delete "$ENV_TYPE" "${ENV_APP_NAME:-${GCP_BASE_APP_NAME}${ENV_APP_SUFFIX}}" "$ENV_PROJECT" environment: action: stop # deploy to review env (only on feature branches) # disabled by default, enable this job by setting $GCP_REVIEW_PROJECT. gcp-review: extends: .gcp-deploy variables: ENV_TYPE: review ENV_APP_NAME: "$GCP_REVIEW_APP_NAME" ENV_PROJECT: "$GCP_REVIEW_PROJECT" ENV_KEY_FILE: "$GCP_REVIEW_KEY_FILE" environment: name: review/$CI_COMMIT_REF_NAME url: "${GCP_REVIEW_ENVIRONMENT_SCHEME}://${CI_PROJECT_NAME}-${CI_ENVIRONMENT_SLUG}.${GCP_REVIEW_ENVIRONMENT_DOMAIN}" on_stop: gcp-cleanup-review resource_group: review/$CI_COMMIT_REF_NAME rules: # exclude merge requests - if: $CI_MERGE_REQUEST_ID when: never # exclude tags - if: $CI_COMMIT_TAG when: never # exclude if $CLEANUP_ALL_REVIEW set to 'force' - if: '$CLEANUP_ALL_REVIEW == "force"' when: never # only on non-production, non-integration branches, with $GCP_REVIEW_PROJECT set - if: '$GCP_REVIEW_PROJECT && $CI_COMMIT_REF_NAME !~ $PROD_REF && $CI_COMMIT_REF_NAME !~ $INTEG_REF' # cleanup review env (automatically triggered once branches are deleted) gcp-cleanup-review: extends: .gcp-cleanup variables: ENV_TYPE: review ENV_APP_NAME: "$GCP_REVIEW_APP_NAME" ENV_PROJECT: "$GCP_REVIEW_PROJECT" ENV_KEY_FILE: "$GCP_REVIEW_KEY_FILE" environment: name: review/$CI_COMMIT_REF_NAME action: stop resource_group: review/$CI_COMMIT_REF_NAME rules: # exclude merge requests - if: $CI_MERGE_REQUEST_ID when: never # exclude tags - if: $CI_COMMIT_TAG when: never # only on non-production, non-integration branches, with $GCP_REVIEW_PROJECT set - if: '$GCP_REVIEW_PROJECT && $CI_COMMIT_REF_NAME !~ $PROD_REF && $CI_COMMIT_REF_NAME !~ $INTEG_REF' when: manual allow_failure: true # deploy to integration env (only on develop branch) gcp-integration: extends: .gcp-deploy variables: ENV_TYPE: integration ENV_APP_NAME: "$GCP_INTEG_APP_NAME" ENV_PROJECT: "$GCP_INTEG_PROJECT" ENV_KEY_FILE: "$GCP_INTEG_KEY_FILE" environment: name: integration url: "${GCP_INTEG_ENVIRONMENT_URL}" resource_group: integration rules: # exclude merge requests - if: $CI_MERGE_REQUEST_ID when: never # only on integration branch(es), with $GCP_INTEG_PROJECT set - if: '$GCP_INTEG_PROJECT && $CI_COMMIT_REF_NAME =~ $INTEG_REF' # deploy to staging env (only on master branch) gcp-staging: extends: .gcp-deploy variables: ENV_TYPE: staging ENV_APP_NAME: "$GCP_STAGING_APP_NAME" ENV_PROJECT: "$GCP_STAGING_PROJECT" ENV_KEY_FILE: "$GCP_STAGING_KEY_FILE" environment: name: staging url: "${GCP_STAGING_ENVIRONMENT_URL}" resource_group: staging rules: # exclude merge requests - if: $CI_MERGE_REQUEST_ID when: never # only on production branch(es), with $GCP_STAGING_PROJECT set - if: '$GCP_STAGING_PROJECT && $CI_COMMIT_REF_NAME =~ $PROD_REF' # Deploy to production if on branch master and variable GCP_PROD_PROJECT defined and AUTODEPLOY_TO_PROD is set gcp-production: extends: .gcp-deploy stage: production variables: ENV_TYPE: production ENV_APP_SUFFIX: "" # no suffix for prod ENV_APP_NAME: "$GCP_PROD_APP_NAME" ENV_PROJECT: "$GCP_PROD_PROJECT" ENV_KEY_FILE: "$GCP_PROD_KEY_FILE" environment: name: production url: "${GCP_PROD_ENVIRONMENT_URL}" resource_group: production rules: # exclude merge requests - if: $CI_MERGE_REQUEST_ID when: never # exclude non-production branches - if: '$CI_COMMIT_REF_NAME !~ $PROD_REF' when: never # exclude if $GCP_PROD_PROJECT not set - if: '$GCP_PROD_PROJECT == null || $GCP_PROD_PROJECT == ""' when: never # if $AUTODEPLOY_TO_PROD: auto - if: $AUTODEPLOY_TO_PROD # else if PUBLISH_ON_PROD enabled: auto (because the publish job was blocking) - if: '$PUBLISH_ON_PROD == "true" || $PUBLISH_ON_PROD == "yes"' # else: manual, blocking - if: $GCP_PROD_PROJECT # useless test, just to prevent GitLab warning when: manual