-
Cédric OLIVIER authoredCédric OLIVIER authored
gitlab-ci-gcloud.yml 18.15 KiB
# =========================================================================================
# Copyright (C) 2021 Orange & contributors
#
# This program is free software; you can redistribute it and/or modify it under the terms
# of the GNU Lesser General Public License as published by the Free Software Foundation;
# either version 3 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along with this
# program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth
# Floor, Boston, MA 02110-1301, USA.
# =========================================================================================
variables:
# Default Docker image (can be overridden)
GCP_CLI_IMAGE: "google/cloud-sdk:latest"
GCP_SCRIPTS_DIR: "."
GCP_BASE_APP_NAME: "$CI_PROJECT_NAME"
GCP_REVIEW_ENVIRONMENT_SCHEME: "https"
# default production ref name (pattern)
PROD_REF: '/^(master|main)$/'
# default integration ref name (pattern)
INTEG_REF: '/^develop$/'
# allowed stages depend on your template type (see: to-be-continuous.gitlab.io/doc/dev-guidelines/#stages)
stages:
- deploy
- production
.gcp-scripts: &gcp-scripts |
# BEGSCRIPT
set -e
function log_info() {
echo -e "[\\e[1;94mINFO\\e[0m] $*"
}
function log_warn() {
echo -e "[\\e[1;93mWARN\\e[0m] $*"
}
function log_error() {
echo -e "[\\e[1;91mERROR\\e[0m] $*"
}
function fail() {
log_error "$*"
exit 1
}
function assert_defined() {
if [[ -z "$1" ]]
then
log_error "$2"
exit 1
fi
}
function install_ca_certs() {
certs=$1
if [[ -z "$certs" ]]
then
return
fi
# import in system
if echo "$certs" >> /etc/ssl/certs/ca-certificates.crt
then
log_info "CA certificates imported in \\e[33;1m/etc/ssl/certs/ca-certificates.crt\\e[0m"
fi
if echo "$certs" >> /etc/ssl/cert.pem
then
log_info "CA certificates imported in \\e[33;1m/etc/ssl/cert.pem\\e[0m"
fi
}
function as_content() {
file_or_content=$1
if [[ -f ${file_or_content} ]]; then
cat "${file_or_content}"
else
echo "${file_or_content}"
fi
}
function unscope_variables() {
_scoped_vars=$(env | awk -F '=' "/^scoped__[a-zA-Z0-9_]+=/ {print \$1}" | sort)
if [[ -z "$_scoped_vars" ]]; then return; fi
log_info "Processing scoped variables..."
for _scoped_var in $_scoped_vars
do
_fields=${_scoped_var//__/:}
_condition=$(echo "$_fields" | cut -d: -f3)
case "$_condition" in
if) _not="";;
ifnot) _not=1;;
*)
log_warn "... unrecognized condition \\e[1;91m$_condition\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"
continue
;;
esac
_target_var=$(echo "$_fields" | cut -d: -f2)
_cond_var=$(echo "$_fields" | cut -d: -f4)
_cond_val=$(eval echo "\$${_cond_var}")
_test_op=$(echo "$_fields" | cut -d: -f5)
case "$_test_op" in
defined)
if [[ -z "$_not" ]] && [[ -z "$_cond_val" ]]; then continue;
elif [[ "$_not" ]] && [[ "$_cond_val" ]]; then continue;
fi
;;
equals|startswith|endswith|contains|in|equals_ic|startswith_ic|endswith_ic|contains_ic|in_ic)
# comparison operator
# sluggify actual value
_cond_val=$(echo "$_cond_val" | tr '[:punct:]' '_')
# retrieve comparison value
_cmp_val_prefix="scoped__${_target_var}__${_condition}__${_cond_var}__${_test_op}__"
_cmp_val=${_scoped_var#"$_cmp_val_prefix"}
# manage 'ignore case'
if [[ "$_test_op" == *_ic ]]
then
# lowercase everything
_cond_val=$(echo "$_cond_val" | tr '[:upper:]' '[:lower:]')
_cmp_val=$(echo "$_cmp_val" | tr '[:upper:]' '[:lower:]')
fi
case "$_test_op" in
equals*)
if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val" ]]; then continue;
elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val" ]]; then continue;
fi
;;
startswith*)
if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val"* ]]; then continue;
elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val"* ]]; then continue;
fi ;;
endswith*)
if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val" ]]; then continue;
elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val" ]]; then continue;
fi
;;
contains*)
if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val"* ]]; then continue;
elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val"* ]]; then continue;
fi
;;
in*)
if [[ -z "$_not" ]] && [[ "__${_cmp_val}__" != *"__${_cond_val}__"* ]]; then continue;
elif [[ "$_not" ]] && [[ "__${_cmp_val}__" == *"__${_cond_val}__"* ]]; then continue;
fi
;;
esac
;;
*)
log_warn "... unrecognized test operator \\e[1;91m${_test_op}\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"
continue
;;
esac
# matches
_val=$(eval echo "\$${_target_var}")
log_info "... apply \\e[32m${_target_var}\\e[0m from \\e[32m\$${_scoped_var}\\e[0m${_val:+ (\\e[33;1moverwrite\\e[0m)}"
_val=$(eval echo "\$${_scoped_var}")
export "${_target_var}"="${_val}"
done
log_info "... done"
}
# evaluate and export a secret
# - $1: secret variable name
function eval_secret() {
name=$1
value=$(eval echo "\$${name}")
case "$value" in
@b64@*)
decoded=$(mktemp)
errors=$(mktemp)
if echo "$value" | cut -c6- | base64 -d > "${decoded}" 2> "${errors}"
then
# shellcheck disable=SC2086
export ${name}="$(cat ${decoded})"
log_info "Successfully decoded base64 secret \\e[33;1m${name}\\e[0m"
else
fail "Failed decoding base64 secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
fi
;;
@hex@*)
decoded=$(mktemp)
errors=$(mktemp)
if echo "$value" | cut -c6- | sed 's/\([0-9A-F]\{2\}\)/\\\\x\1/gI' | xargs printf > "${decoded}" 2> "${errors}"
then
# shellcheck disable=SC2086
export ${name}="$(cat ${decoded})"
log_info "Successfully decoded hexadecimal secret \\e[33;1m${name}\\e[0m"
else
fail "Failed decoding hexadecimal secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
fi
;;
@url@*)
url=$(echo "$value" | cut -c6-)
if command -v curl > /dev/null
then
decoded=$(mktemp)
errors=$(mktemp)
if curl -s -S -f --connect-timeout 5 -o "${decoded}" "$url" 2> "${errors}"
then # shellcheck disable=SC2086
export ${name}="$(cat ${decoded})"
log_info "Successfully curl'd secret \\e[33;1m${name}\\e[0m"
else
fail "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
fi
elif command -v wget > /dev/null
then
decoded=$(mktemp)
errors=$(mktemp)
if wget -T 5 -O "${decoded}" "$url" 2> "${errors}"
then
# shellcheck disable=SC2086
export ${name}="$(cat ${decoded})"
log_info "Successfully wget'd secret \\e[33;1m${name}\\e[0m"
else
fail "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
fi
else
fail "Couldn't get secret \\e[33;1m${name}\\e[0m: no http client found"
fi
;;
esac
}
function eval_all_secrets() {
encoded_vars=$(env | grep -v '^scoped__' | awk -F '=' '/^[a-zA-Z0-9_]*=@(b64|hex|url)@/ {print $1}')
for var in $encoded_vars
do
eval_secret "$var"
done
}
function awkenvsubst() {
awk '{while(match($0,"[$]{[^}]*}")) {var=substr($0,RSTART+2,RLENGTH -3);val=ENVIRON[var];gsub(/["\\]/,"\\\\&", val);gsub("\n", "\\n", val);gsub("\r", "\\r", val);gsub("[$]{"var"}",val)}}1'
}
# application deployment function
function deploy() {
export env=$1
export appname=$2
export gcp_project_id=$3
export environment_url=$4
# extract hostname from $environment_url
hostname=$(echo "$environment_url" | awk -F[/:] '{print $4}')
export hostname
log_info "--- \\e[32mdeploy\\e[0m (env: \\e[33;1m${env}\\e[0m)"
log_info "--- \$appname: \\e[33;1m${appname}\\e[0m"
log_info "--- \$env: \\e[33;1m${env}\\e[0m"
log_info "--- \$hostname: \\e[33;1m${hostname}\\e[0m"
log_info "--- \$gcp_project_id: \\e[33;1m${gcp_project_id}\\e[0m"
# unset any upstream deployment env & artifacts
unset environment_name
unset environment_type
rm -f gcloud.env
rm -f environment_url.txt
deployscript=$(ls -1 "$GCP_SCRIPTS_DIR/gcp-deploy-${env}.sh" 2>/dev/null || ls -1 "$GCP_SCRIPTS_DIR/gcp-deploy.sh" 2>/dev/null || echo "")
if [[ -f "$deployscript" ]]
then
log_info "--- deploy script (\\e[33;1m${deployscript}\\e[0m) found: execute"
chmod +x "$deployscript"
"$deployscript"
else
log_error "--- no deploy script found: abort"
exit 1
fi
# finally persist environment url
if [[ -f environment_url.txt ]]
then
environment_url=$(cat environment_url.txt)
export environment_url
log_info "--- dynamic environment url found: (\\e[33;1m$environment_url\\e[0m)"
else
echo "$environment_url" > environment_url.txt
fi
echo -e "environment_type=$env\\nenvironment_name=$appname\\nenvironment_url=$environment_url" > gcloud.env
}
# environment cleanup function
function delete() {
export env=$1
export appname=$2
export gcp_project_id=$3
log_info "--- \\e[32mdelete\\e[0m (env: ${env})"
log_info "--- \$appname: \\e[33;1m${appname}\\e[0m"
log_info "--- \$env: \\e[33;1m${env}\\e[0m"
log_info "--- \$gcp_project_id: \\e[33;1m${gcp_project_id}\\e[0m"
cleanupscript=$(ls -1 "$GCP_SCRIPTS_DIR/gcp-cleanup-${env}.sh" 2>/dev/null || ls -1 "$GCP_SCRIPTS_DIR/gcp-cleanup.sh" 2>/dev/null || echo "")
if [[ -f "$cleanupscript" ]]
then
log_info "--- cleanup script (\\e[33;1m${cleanupscript}\\e[0m) found: execute"
chmod +x "$cleanupscript"
"$cleanupscript"
else
log_error "--- no cleanup script found: abort"
exit 1
fi
}
function get_latest_template_version() {
tag_json=$(wget -T 5 -q -O - "$CI_API_V4_URL/projects/to-be-continuous%2F$1/repository/tags?per_page=1" 2> /dev/null || curl -s --connect-timeout 5 "$CI_API_V4_URL/projects/to-be-continuous%2F$1/repository/tags?per_page=1" 2> /dev/null || echo "")
echo "$tag_json" | sed -rn 's/^.*"name":"([^"]*)".*$/\1/p'
}
function check_for_update() {
template="$1"
actual="$2"
latest=$(get_latest_template_version "$template")
if [[ -n "$latest" ]] && [[ "$latest" != "$actual" ]]
then
log_warn "\\e[1;93m=======================================================================================================\\e[0m"
log_warn "\\e[93mThe template \\e[32m$template\\e[93m:\\e[33m$actual\\e[93m you're using is not up-to-date: consider upgrading to version \\e[32m$latest\\e[0m"
log_warn "\\e[93m(set \$TEMPLATE_CHECK_UPDATE_DISABLED to disable this message)\\e[0m"
log_warn "\\e[1;93m=======================================================================================================\\e[0m"
fi
}
if [[ "$TEMPLATE_CHECK_UPDATE_DISABLED" != "true" ]]; then check_for_update gcloud "1.5.1"; fi
# export tool functions (might be used in after_script)
export -f log_info log_warn log_error assert_defined awkenvsubst
unscope_variables
eval_all_secrets
# ENDSCRIPT
# job prototype
# defines default Docker image, tracking probe, cache policy and tags
.gcp-base:
image: $GCP_CLI_IMAGE
services:
- name: "$CI_REGISTRY/to-be-continuous/tools/tracking:master"
command: ["--service", "gcloud", "1.5.1" ] before_script:
- *gcp-scripts
- install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
# Deploy job prototype
# Can be extended to define a concrete environment
#
# @arg ENV_TYPE : environment type
# @arg ENV_APP_NAME : env-specific application name
# @arg ENV_APP_SUFFIX: env-specific application suffix
# @arg ENV_PROJECT : env-specific GCP Project ID
# @arg ENV_KEY_FILE : env-specific GCP API key file (JSON)
# @arg ENV_URL : env-specific application url
.gcp-deploy:
extends: .gcp-base
stage: deploy
variables:
ENV_APP_SUFFIX: "-$CI_ENVIRONMENT_SLUG"
before_script:
- *gcp-scripts
- install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
- assert_defined "${ENV_KEY_FILE:-$GCP_KEY_FILE}" 'Missing required GCP key file (JSON)'
- as_content "${ENV_KEY_FILE:-$GCP_KEY_FILE}" > /tmp/gcp.key
- gcloud auth activate-service-account --key-file /tmp/gcp.key
script:
- deploy "$ENV_TYPE" "${ENV_APP_NAME:-${GCP_BASE_APP_NAME}${ENV_APP_SUFFIX}}" "$ENV_PROJECT" "$ENV_URL"
artifacts:
name: "$ENV_TYPE env url for $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
paths:
- environment_url.txt
reports:
dotenv: gcloud.env
environment:
url: "$environment_url" # can be either static or dynamic
# Cleanup job prototype
# Can be extended for each deletable environment
#
# @arg ENV_TYPE : environment type
# @arg ENV_APP_NAME : env-specific application name
# @arg ENV_APP_SUFFIX: env-specific application suffix
# @arg ENV_PROJECT : env-specific GCP Project ID
# @arg ENV_KEY_FILE : env-specific GCP API key file (JSON)
.gcp-cleanup:
extends: .gcp-base
stage: deploy
# force no dependencies
dependencies: []
variables:
ENV_APP_SUFFIX: "-$CI_ENVIRONMENT_SLUG"
before_script:
- *gcp-scripts
- install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
- assert_defined "${ENV_KEY_FILE:-$GCP_KEY_FILE}" 'Missing required GCP key file (JSON)'
- as_content "${ENV_KEY_FILE:-$GCP_KEY_FILE}" > /tmp/gcp.key
- gcloud auth activate-service-account --key-file /tmp/gcp.key
script:
- delete "$ENV_TYPE" "${ENV_APP_NAME:-${GCP_BASE_APP_NAME}${ENV_APP_SUFFIX}}" "$ENV_PROJECT"
environment:
action: stop
# deploy to review env (only on feature branches)
# disabled by default, enable this job by setting $GCP_REVIEW_PROJECT.
gcp-review:
extends: .gcp-deploy
variables:
ENV_TYPE: review
ENV_APP_NAME: "$GCP_REVIEW_APP_NAME"
ENV_PROJECT: "$GCP_REVIEW_PROJECT" ENV_KEY_FILE: "$GCP_REVIEW_KEY_FILE"
ENV_URL: "${GCP_REVIEW_ENVIRONMENT_SCHEME}://${CI_PROJECT_NAME}-${CI_ENVIRONMENT_SLUG}.${GCP_REVIEW_ENVIRONMENT_DOMAIN}"
environment:
name: review/$CI_COMMIT_REF_NAME
on_stop: gcp-cleanup-review
resource_group: review/$CI_COMMIT_REF_NAME
rules:
# exclude merge requests
- if: $CI_MERGE_REQUEST_ID
when: never
# exclude tags
- if: $CI_COMMIT_TAG
when: never
# exclude if $CLEANUP_ALL_REVIEW set to 'force'
- if: '$CLEANUP_ALL_REVIEW == "force"'
when: never
# only on non-production, non-integration branches, with $GCP_REVIEW_PROJECT set
- if: '$GCP_REVIEW_PROJECT && $CI_COMMIT_REF_NAME !~ $PROD_REF && $CI_COMMIT_REF_NAME !~ $INTEG_REF'
# cleanup review env (automatically triggered once branches are deleted)
gcp-cleanup-review:
extends: .gcp-cleanup
variables:
ENV_TYPE: review
ENV_APP_NAME: "$GCP_REVIEW_APP_NAME"
ENV_PROJECT: "$GCP_REVIEW_PROJECT"
ENV_KEY_FILE: "$GCP_REVIEW_KEY_FILE"
environment:
name: review/$CI_COMMIT_REF_NAME
action: stop
resource_group: review/$CI_COMMIT_REF_NAME
rules:
# exclude merge requests
- if: $CI_MERGE_REQUEST_ID
when: never
# exclude tags
- if: $CI_COMMIT_TAG
when: never
# only on non-production, non-integration branches, with $GCP_REVIEW_PROJECT set
- if: '$GCP_REVIEW_PROJECT && $CI_COMMIT_REF_NAME !~ $PROD_REF && $CI_COMMIT_REF_NAME !~ $INTEG_REF'
when: manual
allow_failure: true
# deploy to integration env (only on develop branch)
gcp-integration:
extends: .gcp-deploy
variables:
ENV_TYPE: integration
ENV_APP_NAME: "$GCP_INTEG_APP_NAME"
ENV_PROJECT: "$GCP_INTEG_PROJECT"
ENV_KEY_FILE: "$GCP_INTEG_KEY_FILE"
ENV_URL: "${GCP_INTEG_ENVIRONMENT_URL}"
environment:
name: integration
resource_group: integration
rules:
# exclude merge requests
- if: $CI_MERGE_REQUEST_ID
when: never
# only on integration branch(es), with $GCP_INTEG_PROJECT set
- if: '$GCP_INTEG_PROJECT && $CI_COMMIT_REF_NAME =~ $INTEG_REF'
# deploy to staging env (only on master branch)
gcp-staging:
extends: .gcp-deploy
variables:
ENV_TYPE: staging
ENV_APP_NAME: "$GCP_STAGING_APP_NAME"
ENV_PROJECT: "$GCP_STAGING_PROJECT"
ENV_KEY_FILE: "$GCP_STAGING_KEY_FILE" ENV_URL: "${GCP_STAGING_ENVIRONMENT_URL}"
environment:
name: staging
resource_group: staging
rules:
# exclude merge requests
- if: $CI_MERGE_REQUEST_ID
when: never
# only on production branch(es), with $GCP_STAGING_PROJECT set
- if: '$GCP_STAGING_PROJECT && $CI_COMMIT_REF_NAME =~ $PROD_REF'
# Deploy to production if on branch master and variable GCP_PROD_PROJECT defined and AUTODEPLOY_TO_PROD is set
gcp-production:
extends: .gcp-deploy
stage: production
variables:
ENV_TYPE: production
ENV_APP_SUFFIX: "" # no suffix for prod
ENV_APP_NAME: "$GCP_PROD_APP_NAME"
ENV_PROJECT: "$GCP_PROD_PROJECT"
ENV_KEY_FILE: "$GCP_PROD_KEY_FILE"
ENV_URL: "${GCP_PROD_ENVIRONMENT_URL}"
environment:
name: production
resource_group: production
rules:
# exclude merge requests
- if: $CI_MERGE_REQUEST_ID
when: never
# exclude non-production branches
- if: '$CI_COMMIT_REF_NAME !~ $PROD_REF'
when: never
# exclude if $GCP_PROD_PROJECT not set
- if: '$GCP_PROD_PROJECT == null || $GCP_PROD_PROJECT == ""'
when: never
# if $AUTODEPLOY_TO_PROD: auto
- if: $AUTODEPLOY_TO_PROD
# else if PUBLISH_ON_PROD enabled: auto (because the publish job was blocking)
- if: '$PUBLISH_ON_PROD == "true" || $PUBLISH_ON_PROD == "yes"'
# else: manual, blocking
- if: $GCP_PROD_PROJECT # useless test, just to prevent GitLab warning
when: manual