Newer
Older
# =========================================================================================
#
# This program is free software; you can redistribute it and/or modify it under the terms
# of the GNU Lesser General Public License as published by the Free Software Foundation;
# either version 3 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along with this
# program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth
# Floor, Boston, MA 02110-1301, USA.
# =========================================================================================
# default workflow rules: Merge Request pipelines
# prevent branch pipeline when an MR is open (prefer MR pipeline)
- if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS'
# variabilized tracking image
TBC_TRACKING_IMAGE: "$CI_REGISTRY/to-be-continuous/tools/tracking:master"
GCP_CLI_IMAGE: "google/cloud-sdk:latest"
GCP_SCRIPTS_DIR: "."
GCP_BASE_APP_NAME: "$CI_PROJECT_NAME"
GCP_REVIEW_ENVIRONMENT_SCHEME: "https"
# default production ref name (pattern)
PROD_REF: '/^(master|main)$/'
# default integration ref name (pattern)
INTEG_REF: '/^develop$/'
# allowed stages depend on your template type (see: to-be-continuous.gitlab.io/doc/dev-guidelines/#stages)
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
stages:
- deploy
- production
.gcp-scripts: &gcp-scripts |
# BEGSCRIPT
set -e
function log_info() {
echo -e "[\\e[1;94mINFO\\e[0m] $*"
}
function log_warn() {
echo -e "[\\e[1;93mWARN\\e[0m] $*"
}
function log_error() {
echo -e "[\\e[1;91mERROR\\e[0m] $*"
}
function fail() {
log_error "$*"
exit 1
}
function assert_defined() {
if [[ -z "$1" ]]
then
log_error "$2"
exit 1
fi
}
function install_ca_certs() {
certs=$1
if [[ -z "$certs" ]]
then
return
fi
# import in system
if echo "$certs" >> /etc/ssl/certs/ca-certificates.crt
then
log_info "CA certificates imported in \\e[33;1m/etc/ssl/certs/ca-certificates.crt\\e[0m"
fi
if echo "$certs" >> /etc/ssl/cert.pem
then
log_info "CA certificates imported in \\e[33;1m/etc/ssl/cert.pem\\e[0m"
fi
}
function as_content() {
file_or_content=$1
if [[ -f ${file_or_content} ]]; then
cat "${file_or_content}"
else
echo "${file_or_content}"
fi
}
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
function unscope_variables() {
_scoped_vars=$(env | awk -F '=' "/^scoped__[a-zA-Z0-9_]+=/ {print \$1}" | sort)
if [[ -z "$_scoped_vars" ]]; then return; fi
log_info "Processing scoped variables..."
for _scoped_var in $_scoped_vars
do
_fields=${_scoped_var//__/:}
_condition=$(echo "$_fields" | cut -d: -f3)
case "$_condition" in
if) _not="";;
ifnot) _not=1;;
*)
log_warn "... unrecognized condition \\e[1;91m$_condition\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"
continue
;;
esac
_target_var=$(echo "$_fields" | cut -d: -f2)
_cond_var=$(echo "$_fields" | cut -d: -f4)
_cond_val=$(eval echo "\$${_cond_var}")
_test_op=$(echo "$_fields" | cut -d: -f5)
case "$_test_op" in
defined)
if [[ -z "$_not" ]] && [[ -z "$_cond_val" ]]; then continue;
elif [[ "$_not" ]] && [[ "$_cond_val" ]]; then continue;
fi
;;
equals|startswith|endswith|contains|in|equals_ic|startswith_ic|endswith_ic|contains_ic|in_ic)
# comparison operator
# sluggify actual value
_cond_val=$(echo "$_cond_val" | tr '[:punct:]' '_')
# retrieve comparison value
_cmp_val_prefix="scoped__${_target_var}__${_condition}__${_cond_var}__${_test_op}__"
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
# manage 'ignore case'
if [[ "$_test_op" == *_ic ]]
then
# lowercase everything
_cond_val=$(echo "$_cond_val" | tr '[:upper:]' '[:lower:]')
_cmp_val=$(echo "$_cmp_val" | tr '[:upper:]' '[:lower:]')
fi
case "$_test_op" in
equals*)
if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val" ]]; then continue;
elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val" ]]; then continue;
fi
;;
startswith*)
if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val"* ]]; then continue;
elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val"* ]]; then continue;
fi
;;
endswith*)
if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val" ]]; then continue;
elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val" ]]; then continue;
fi
;;
contains*)
if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val"* ]]; then continue;
elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val"* ]]; then continue;
fi
;;
in*)
if [[ -z "$_not" ]] && [[ "__${_cmp_val}__" != *"__${_cond_val}__"* ]]; then continue;
elif [[ "$_not" ]] && [[ "__${_cmp_val}__" == *"__${_cond_val}__"* ]]; then continue;
fi
;;
esac
;;
*)
log_warn "... unrecognized test operator \\e[1;91m${_test_op}\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"
continue
;;
esac
# matches
_val=$(eval echo "\$${_target_var}")
log_info "... apply \\e[32m${_target_var}\\e[0m from \\e[32m\$${_scoped_var}\\e[0m${_val:+ (\\e[33;1moverwrite\\e[0m)}"
_val=$(eval echo "\$${_scoped_var}")
export "${_target_var}"="${_val}"
done
log_info "... done"
}
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
# evaluate and export a secret
# - $1: secret variable name
function eval_secret() {
name=$1
value=$(eval echo "\$${name}")
case "$value" in
@b64@*)
decoded=$(mktemp)
errors=$(mktemp)
if echo "$value" | cut -c6- | base64 -d > "${decoded}" 2> "${errors}"
then
# shellcheck disable=SC2086
export ${name}="$(cat ${decoded})"
log_info "Successfully decoded base64 secret \\e[33;1m${name}\\e[0m"
else
fail "Failed decoding base64 secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
fi
;;
@hex@*)
decoded=$(mktemp)
errors=$(mktemp)
if echo "$value" | cut -c6- | sed 's/\([0-9A-F]\{2\}\)/\\\\x\1/gI' | xargs printf > "${decoded}" 2> "${errors}"
then
# shellcheck disable=SC2086
export ${name}="$(cat ${decoded})"
log_info "Successfully decoded hexadecimal secret \\e[33;1m${name}\\e[0m"
else
fail "Failed decoding hexadecimal secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
fi
;;
@url@*)
url=$(echo "$value" | cut -c6-)
if command -v curl > /dev/null
then
decoded=$(mktemp)
errors=$(mktemp)
if curl -s -S -f --connect-timeout 5 -o "${decoded}" "$url" 2> "${errors}"
then
# shellcheck disable=SC2086
export ${name}="$(cat ${decoded})"
log_info "Successfully curl'd secret \\e[33;1m${name}\\e[0m"
else
Pierre Smeyers
committed
log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
fi
elif command -v wget > /dev/null
then
decoded=$(mktemp)
errors=$(mktemp)
if wget -T 5 -O "${decoded}" "$url" 2> "${errors}"
then
# shellcheck disable=SC2086
export ${name}="$(cat ${decoded})"
log_info "Successfully wget'd secret \\e[33;1m${name}\\e[0m"
else
Pierre Smeyers
committed
log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
Pierre Smeyers
committed
log_warn "Couldn't get secret \\e[33;1m${name}\\e[0m: no http client found"
fi
;;
esac
}
function eval_all_secrets() {
encoded_vars=$(env | grep -v '^scoped__' | awk -F '=' '/^[a-zA-Z0-9_]*=@(b64|hex|url)@/ {print $1}')
for var in $encoded_vars
do
eval_secret "$var"
done
}
function awkenvsubst() {
awk '{while(match($0,"[$]{[^}]*}")) {var=substr($0,RSTART+2,RLENGTH -3);val=ENVIRON[var];gsub(/["\\]/,"\\\\&", val);gsub("\n", "\\n", val);gsub("\r", "\\r", val);gsub("[$]{"var"}",val)}}1'
}
# application deployment function
function deploy() {
export env=$1
export appname=$2
export gcp_project_id=$3
export environment_url=$4
# extract hostname from $environment_url
hostname=$(echo "$environment_url" | awk -F[/:] '{print $4}')
export hostname
log_info "--- \\e[32mdeploy\\e[0m (env: \\e[33;1m${env}\\e[0m)"
log_info "--- \$appname: \\e[33;1m${appname}\\e[0m"
log_info "--- \$env: \\e[33;1m${env}\\e[0m"
log_info "--- \$hostname: \\e[33;1m${hostname}\\e[0m"
log_info "--- \$gcp_project_id: \\e[33;1m${gcp_project_id}\\e[0m"
# unset any upstream deployment env & artifacts
unset environment_name
unset environment_type
rm -f gcloud.env
rm -f environment_url.txt
deployscript=$(ls -1 "$GCP_SCRIPTS_DIR/gcp-deploy-${env}.sh" 2>/dev/null || ls -1 "$GCP_SCRIPTS_DIR/gcp-deploy.sh" 2>/dev/null || echo "")
if [[ -f "$deployscript" ]]
then
log_info "--- deploy script (\\e[33;1m${deployscript}\\e[0m) found: execute"
chmod +x "$deployscript"
"$deployscript"
else
log_error "--- no deploy script found: abort"
exit 1
fi
# finally persist environment url
if [[ -f environment_url.txt ]]
then
environment_url=$(cat environment_url.txt)
export environment_url
log_info "--- dynamic environment url found: (\\e[33;1m$environment_url\\e[0m)"
else
echo "$environment_url" > environment_url.txt
fi
echo -e "environment_type=$env\\nenvironment_name=$appname\\nenvironment_url=$environment_url" > gcloud.env
}
# environment cleanup function
function delete() {
export env=$1
export appname=$2
export gcp_project_id=$3
log_info "--- \\e[32mdelete\\e[0m (env: ${env})"
log_info "--- \$appname: \\e[33;1m${appname}\\e[0m"
log_info "--- \$env: \\e[33;1m${env}\\e[0m"
log_info "--- \$gcp_project_id: \\e[33;1m${gcp_project_id}\\e[0m"
cleanupscript=$(ls -1 "$GCP_SCRIPTS_DIR/gcp-cleanup-${env}.sh" 2>/dev/null || ls -1 "$GCP_SCRIPTS_DIR/gcp-cleanup.sh" 2>/dev/null || echo "")
if [[ -f "$cleanupscript" ]]
then
log_info "--- cleanup script (\\e[33;1m${cleanupscript}\\e[0m) found: execute"
chmod +x "$cleanupscript"
"$cleanupscript"
else
log_error "--- no cleanup script found: abort"
exit 1
fi
}
# export tool functions (might be used in after_script)
export -f log_info log_warn log_error assert_defined awkenvsubst
eval_all_secrets
# ENDSCRIPT
# job prototype
# defines default Docker image, tracking probe, cache policy and tags
.gcp-base:
image: $GCP_CLI_IMAGE
services:
command: ["--service", "gcloud", "1.8.0" ]
before_script:
- *gcp-scripts
- install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
# Deploy job prototype
# Can be extended to define a concrete environment
#
# @arg ENV_TYPE : environment type
# @arg ENV_APP_NAME : env-specific application name
# @arg ENV_APP_SUFFIX: env-specific application suffix
# @arg ENV_PROJECT : env-specific GCP Project ID
# @arg ENV_KEY_FILE : env-specific GCP API key file (JSON)
# @arg ENV_URL : env-specific application url
.gcp-deploy:
extends: .gcp-base
stage: deploy
variables:
ENV_APP_SUFFIX: "-$CI_ENVIRONMENT_SLUG"
before_script:
- *gcp-scripts
- install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
- assert_defined "${ENV_KEY_FILE:-$GCP_KEY_FILE}" 'Missing required GCP key file (JSON)'
- as_content "${ENV_KEY_FILE:-$GCP_KEY_FILE}" > /tmp/gcp.key
- gcloud auth activate-service-account --key-file /tmp/gcp.key
- deploy "$ENV_TYPE" "${ENV_APP_NAME:-${GCP_BASE_APP_NAME}${ENV_APP_SUFFIX}}" "$ENV_PROJECT" "$ENV_URL"
artifacts:
name: "$ENV_TYPE env url for $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
paths:
- environment_url.txt
reports:
dotenv: gcloud.env
environment:
url: "$environment_url" # can be either static or dynamic
# Cleanup job prototype
# Can be extended for each deletable environment
#
# @arg ENV_TYPE : environment type
# @arg ENV_APP_NAME : env-specific application name
# @arg ENV_APP_SUFFIX: env-specific application suffix
# @arg ENV_PROJECT : env-specific GCP Project ID
# @arg ENV_KEY_FILE : env-specific GCP API key file (JSON)
.gcp-cleanup:
extends: .gcp-base
stage: deploy
# force no dependencies
dependencies: []
variables:
ENV_APP_SUFFIX: "-$CI_ENVIRONMENT_SLUG"
before_script:
- *gcp-scripts
- install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
- assert_defined "${ENV_KEY_FILE:-$GCP_KEY_FILE}" 'Missing required GCP key file (JSON)'
- as_content "${ENV_KEY_FILE:-$GCP_KEY_FILE}" > /tmp/gcp.key
- gcloud auth activate-service-account --key-file /tmp/gcp.key
script:
- delete "$ENV_TYPE" "${ENV_APP_NAME:-${GCP_BASE_APP_NAME}${ENV_APP_SUFFIX}}" "$ENV_PROJECT"
environment:
action: stop
# deploy to review env (only on feature branches)
# disabled by default, enable this job by setting $GCP_REVIEW_PROJECT.
gcp-review:
extends: .gcp-deploy
variables:
ENV_TYPE: review
ENV_APP_NAME: "$GCP_REVIEW_APP_NAME"
ENV_PROJECT: "$GCP_REVIEW_PROJECT"
ENV_KEY_FILE: "$GCP_REVIEW_KEY_FILE"
ENV_URL: "${GCP_REVIEW_ENVIRONMENT_SCHEME}://${CI_PROJECT_NAME}-${CI_ENVIRONMENT_SLUG}.${GCP_REVIEW_ENVIRONMENT_DOMAIN}"
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
environment:
name: review/$CI_COMMIT_REF_NAME
on_stop: gcp-cleanup-review
resource_group: review/$CI_COMMIT_REF_NAME
rules:
# exclude tags
- if: $CI_COMMIT_TAG
when: never
# exclude if $CLEANUP_ALL_REVIEW set to 'force'
- if: '$CLEANUP_ALL_REVIEW == "force"'
when: never
# only on non-production, non-integration branches, with $GCP_REVIEW_PROJECT set
- if: '$GCP_REVIEW_PROJECT && $CI_COMMIT_REF_NAME !~ $PROD_REF && $CI_COMMIT_REF_NAME !~ $INTEG_REF'
# cleanup review env (automatically triggered once branches are deleted)
gcp-cleanup-review:
extends: .gcp-cleanup
variables:
ENV_TYPE: review
ENV_APP_NAME: "$GCP_REVIEW_APP_NAME"
ENV_PROJECT: "$GCP_REVIEW_PROJECT"
ENV_KEY_FILE: "$GCP_REVIEW_KEY_FILE"
environment:
name: review/$CI_COMMIT_REF_NAME
action: stop
resource_group: review/$CI_COMMIT_REF_NAME
rules:
# exclude tags
- if: $CI_COMMIT_TAG
when: never
# only on non-production, non-integration branches, with $GCP_REVIEW_PROJECT set
- if: '$GCP_REVIEW_PROJECT && $CI_COMMIT_REF_NAME !~ $PROD_REF && $CI_COMMIT_REF_NAME !~ $INTEG_REF'
when: manual
allow_failure: true
# deploy to integration env (only on develop branch)
gcp-integration:
extends: .gcp-deploy
variables:
ENV_TYPE: integration
ENV_APP_NAME: "$GCP_INTEG_APP_NAME"
ENV_PROJECT: "$GCP_INTEG_PROJECT"
ENV_KEY_FILE: "$GCP_INTEG_KEY_FILE"
ENV_URL: "${GCP_INTEG_ENVIRONMENT_URL}"
environment:
name: integration
resource_group: integration
rules:
# only on integration branch(es), with $GCP_INTEG_PROJECT set
- if: '$GCP_INTEG_PROJECT && $CI_COMMIT_REF_NAME =~ $INTEG_REF'
# deploy to staging env (only on master branch)
gcp-staging:
extends: .gcp-deploy
variables:
ENV_TYPE: staging
ENV_APP_NAME: "$GCP_STAGING_APP_NAME"
ENV_PROJECT: "$GCP_STAGING_PROJECT"
ENV_KEY_FILE: "$GCP_STAGING_KEY_FILE"
ENV_URL: "${GCP_STAGING_ENVIRONMENT_URL}"
environment:
name: staging
resource_group: staging
rules:
# only on production branch(es), with $GCP_STAGING_PROJECT set
- if: '$GCP_STAGING_PROJECT && $CI_COMMIT_REF_NAME =~ $PROD_REF'
# Deploy to production if on branch master and variable GCP_PROD_PROJECT defined and AUTODEPLOY_TO_PROD is set
gcp-production:
extends: .gcp-deploy
stage: production
variables:
ENV_TYPE: production
ENV_APP_SUFFIX: "" # no suffix for prod
ENV_APP_NAME: "$GCP_PROD_APP_NAME"
ENV_PROJECT: "$GCP_PROD_PROJECT"
ENV_KEY_FILE: "$GCP_PROD_KEY_FILE"
ENV_URL: "${GCP_PROD_ENVIRONMENT_URL}"
environment:
name: production
resource_group: production
rules:
# exclude non-production branches
- if: '$CI_COMMIT_REF_NAME !~ $PROD_REF'
when: never
# exclude if $GCP_PROD_PROJECT not set
- if: '$GCP_PROD_PROJECT == null || $GCP_PROD_PROJECT == ""'
when: never
# if $AUTODEPLOY_TO_PROD: auto
- if: '$AUTODEPLOY_TO_PROD == "true"'
# else if PUBLISH_ON_PROD enabled: auto (because the publish job was blocking)
- if: '$PUBLISH_ON_PROD == "true"'
# else: manual, blocking
- if: $GCP_PROD_PROJECT # useless test, just to prevent GitLab warning
when: manual