gitlab-ci-golang.yml

# =========================================================================================
# Copyright (C) 2021 Orange & contributors
#
# This program is free software; you can redistribute it and/or modify it under the terms
# of the GNU Lesser General Public License as published by the Free Software Foundation;
# either version 3 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along with this
# program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth
# Floor, Boston, MA  02110-1301, USA.
# =========================================================================================
# default workflow rules: Merge Request pipelines
workflow:
  rules:
    # prevent branch pipeline when an MR is open (prefer MR pipeline)
    - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS'
      when: never
    - when: always

# test job prototype: implement adaptive pipeline rules
.test-policy:
  rules:
    # on tag: auto & failing
    - if: $CI_COMMIT_TAG
    # on ADAPTIVE_PIPELINE_DISABLED: auto & failing
    - if: '$ADAPTIVE_PIPELINE_DISABLED == "true"'
    # on production or integration branch(es): auto & failing
    - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF || $CI_COMMIT_REF_NAME =~ $INTEG_REF'
    # early stage (dev branch, no MR): manual & non-failing
    - if: '$CI_MERGE_REQUEST_ID == null && $CI_OPEN_MERGE_REQUESTS == null'
      when: manual
      allow_failure: true
    # Draft MR: auto & non-failing
    - if: '$CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/'
      allow_failure: true
    # else (Ready MR): auto & failing
    - when: on_success

variables:
  # variabilized tracking image
  TBC_TRACKING_IMAGE: "$CI_REGISTRY/to-be-continuous/tools/tracking:master"

  # Default Go project root directory
  GO_PROJECT_DIR: .

  # Default Docker image (can be overridden)
  GO_IMAGE: "golang:buster"

  # Default flags for 'build' command
  GO_BUILD_FLAGS: >-
    -mod=readonly

  # Default flags for go build linker
  GO_BUILD_LINKER_FLAGS: "-s -w"

  # Default packages for 'build' command
  GO_BUILD_PACKAGES: >-
    ./...

  # Default build mode (application/modules/auto)
  GO_BUILD_MODE: auto

  # Default flags for 'test' command
  GO_TEST_FLAGS: >-
    -mod=readonly
    -v
    -race

  # Default packages for 'test' command
  GO_TEST_PACKAGES: >-
    ./...

  # Default arguments for 'list' command
  GO_LIST_ARGS : >-
    list
    -u
    -m
    -mod=readonly
    -json all

  # Default arguments for go-mod-outdated command
  GO_MOD_OUTDATED_ARGS: '-update -direct'

  # Default golangci-lint Docker image (can be overridden)
  GO_CI_LINT_IMAGE: "golangci/golangci-lint:latest-alpine"

  # Default arguments for golangci-lint command
  GO_CI_LINT_ARGS: '-E gosec,goimports ./...'

  # Image of cyclonedx-gomod used for SBOM analysis
  GO_SBOM_IMAGE: cyclonedx/cyclonedx-gomod:latest
  # Options for cyclonedx-gomod used for SBOM analysis
  GO_SBOM_OPTS: "-main ."

  # default production ref name (pattern)
  PROD_REF: '/^(master|main)$/'
  # default integration ref name (pattern)
  INTEG_REF: '/^develop$/'

# allowed stages depend on your template type (see: to-be-continuous.gitlab.io/doc/dev-guidelines/#stages)
stages:
  - build
  - test

.go-scripts: &go-scripts |
  # BEGSCRIPT
  set -e

  function log_info() {
      echo -e "[\\e[1;94mINFO\\e[0m] $*"
  }

  function log_warn() {
      echo -e "[\\e[1;93mWARN\\e[0m] $*"
  }

  function log_error() {
      echo -e "[\\e[1;91mERROR\\e[0m] $*"
  }

  function output_coverage() {
    coverage_out=reports/go-coverage.native.out
    if [[ -f "$coverage_out" ]]
    then
      log_info "--- \\e[32mCoverage report(s) found\\e[0m (\\e[33;1m${coverage_out}\\e[0m): output"
      percent=$(go tool cover -func="$coverage_out" | tail -1 | awk -F" " '{print $NF}')
      echo "${percent} covered"

      go get github.com/boumenot/gocover-cobertura
      go run github.com/boumenot/gocover-cobertura < "$coverage_out" > reports/go-coverage.cobertura.xml
    else
      log_info "--- \\e[32mCoverage report(s) not found\\e[0m: skip"
    fi
  }

  function go_build() {
    case "${GO_BUILD_MODE}" in
      application)
        go_build_application
        ;;
      modules)
        go_build_modules
        ;;
      auto)
        go_build_auto
        ;;
      *)
        log_error "invalid \\e[94;1mGO_BUILD_MODE\\e[0m (expected values are \\e[96;1mapplication\\e[0m, \\e[96;1mmodules\\e[0m, \\e[96;1mauto\\e[0m)"
        exit 1
        ;;
    esac
  }

  function go_build_auto() {
    log_info "auto build mode, looking up for a \\e[33;1mmain\\e[0m package..."
    if grep -Rw --include "*.go" "^package main"
    then
      go_build_application
    else
      go_build_modules
    fi
  }

  function go_build_application() {
    log_info "building go application"
    GO_TARGET_OS="${GO_TARGET_OS:-$GOOS}"
    GO_TARGET_ARCH="${GO_TARGET_ARCH:-$GOARCH}"
    target_dir="$GOBIN/$GO_TARGET_OS/$GO_TARGET_ARCH"
    mkdir -p "$target_dir"
    # shellcheck disable=SC2086
    GOOS="$GO_TARGET_OS" GOARCH="$GO_TARGET_ARCH" go build -ldflags="$GO_BUILD_LINKER_FLAGS" $GO_BUILD_FLAGS -o "$target_dir" $GO_BUILD_PACKAGES
  }

  function go_build_modules() {
    log_info "building go modules"
    # shellcheck disable=SC2086
    go build -ldflags="$GO_BUILD_LINKER_FLAGS" $GO_BUILD_FLAGS $GO_BUILD_PACKAGES
  }

  function go_test() {
    mkdir -p -m 777 reports
    local go_text_report="reports/go-test.native.txt"

    set +e
    # shellcheck disable=SC2086
    go test $GO_TEST_FLAGS "-coverprofile=reports/go-coverage.native.out" $GO_TEST_PACKAGES > "$go_text_report"
    test_rc=$?
    set -e

    # dump text report in the console
    cat "$go_text_report"

    # compute and dump code coverage in the console
    output_coverage

    # produce JUnit report (for GitLab)
    install_go_junit_report
    "$GOBIN/go-junit-report" < "$go_text_report" > reports/go-test.xunit.xml

    # produce JSON report (for SonarQube)
    go tool test2json < "$go_text_report" > reports/go-test.native.json

    # maybe fail
    if [[ "$test_rc" != "0" ]]; then exit "$test_rc"; fi
  }

  function install_go_junit_report() {
    cd "$(mktemp -d)"
    go mod init go-junit-report
    go install github.com/jstemmer/go-junit-report@latest
    cd -
  }

  function install_go_mod_outdated() {
    cd "$(mktemp -d)"
    go mod init go-mod-outdated
    go install github.com/psampaz/go-mod-outdated@latest
    cd -
  }

  function install_ca_certs() {
    certs=$1
    if [[ -z "$certs" ]]
    then
      return
    fi

    # import in system
    if echo "$certs" >> /etc/ssl/certs/ca-certificates.crt
    then
      log_info "CA certificates imported in \\e[33;1m/etc/ssl/certs/ca-certificates.crt\\e[0m"
    fi
    if echo "$certs" >> /etc/ssl/cert.pem
    then
      log_info "CA certificates imported in \\e[33;1m/etc/ssl/cert.pem\\e[0m"
    fi
  }

  function unscope_variables() {
    _scoped_vars=$(env | awk -F '=' "/^scoped__[a-zA-Z0-9_]+=/ {print \$1}" | sort)
    if [[ -z "$_scoped_vars" ]]; then return; fi
    log_info "Processing scoped variables..."
    for _scoped_var in $_scoped_vars
    do
      _fields=${_scoped_var//__/:}
      _condition=$(echo "$_fields" | cut -d: -f3)
      case "$_condition" in
      if) _not="";;
      ifnot) _not=1;;
      *)
        log_warn "... unrecognized condition \\e[1;91m$_condition\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"
        continue
      ;;
      esac
      _target_var=$(echo "$_fields" | cut -d: -f2)
      _cond_var=$(echo "$_fields" | cut -d: -f4)
      _cond_val=$(eval echo "\$${_cond_var}")
      _test_op=$(echo "$_fields" | cut -d: -f5)
      case "$_test_op" in
      defined)
        if [[ -z "$_not" ]] && [[ -z "$_cond_val" ]]; then continue;
        elif [[ "$_not" ]] && [[ "$_cond_val" ]]; then continue;
        fi
        ;;
      equals|startswith|endswith|contains|in|equals_ic|startswith_ic|endswith_ic|contains_ic|in_ic)
        # comparison operator
        # sluggify actual value
        _cond_val=$(echo "$_cond_val" | tr '[:punct:]' '_')
        # retrieve comparison value
        _cmp_val_prefix="scoped__${_target_var}__${_condition}__${_cond_var}__${_test_op}__"
        _cmp_val=${_scoped_var#"$_cmp_val_prefix"}
        # manage 'ignore case'
        if [[ "$_test_op" == *_ic ]]
        then
          # lowercase everything
          _cond_val=$(echo "$_cond_val" | tr '[:upper:]' '[:lower:]')
          _cmp_val=$(echo "$_cmp_val" | tr '[:upper:]' '[:lower:]')
        fi
        case "$_test_op" in
        equals*)
          if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val" ]]; then continue;
          elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val" ]]; then continue;
          fi
          ;;
        startswith*)
          if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val"* ]]; then continue;
          elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val"* ]]; then continue;
          fi
          ;;
        endswith*)
          if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val" ]]; then continue;
          elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val" ]]; then continue;
          fi
          ;;
        contains*)
          if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val"* ]]; then continue;
          elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val"* ]]; then continue;
          fi
          ;;
        in*)
          if [[ -z "$_not" ]] && [[ "__${_cmp_val}__" != *"__${_cond_val}__"* ]]; then continue;
          elif [[ "$_not" ]] && [[ "__${_cmp_val}__" == *"__${_cond_val}__"* ]]; then continue;
          fi
          ;;
        esac
        ;;
      *)
        log_warn "... unrecognized test operator \\e[1;91m${_test_op}\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"
        continue
        ;;
      esac
      # matches
      _val=$(eval echo "\$${_target_var}")
      log_info "... apply \\e[32m${_target_var}\\e[0m from \\e[32m\$${_scoped_var}\\e[0m${_val:+ (\\e[33;1moverwrite\\e[0m)}"
      _val=$(eval echo "\$${_scoped_var}")
      export "${_target_var}"="${_val}"
    done
    log_info "... done"
  }

  unscope_variables

  # ENDSCRIPT

# job prototype
# defines default default docker image, tracking probe, cache policy and tags
.go-base:
  image: $GO_IMAGE
  services:
    - name: "$TBC_TRACKING_IMAGE"
      command: ["--service", "golang", "4.0.2" ]
  variables:
    # The directory where 'go install' will install a command.
    GOBIN: "$CI_PROJECT_DIR/$GO_PROJECT_DIR/bin"
    # The directory where the go command will store cached information for reuse in future builds.
    GOCACHE: "$CI_PROJECT_DIR/$GO_PROJECT_DIR/.cache"
  cache:
    key: "$CI_COMMIT_REF_SLUG-golang"
    paths:
      - $GO_PROJECT_DIR/.cache/
  before_script:
    - *go-scripts
    - |
      if command -v git > /dev/null
      then
        git config --global url.https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}.insteadOf https://${CI_SERVER_HOST}
      else
        log_warn "If you need to use private repository, you should provide an image with git executable"
      fi
    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
    - cd ${GO_PROJECT_DIR}

go-build:
  extends: .go-base
  stage: build
  script:
    - go_build
  artifacts:
    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
    expire_in: 1 day
    paths:
      - $GO_PROJECT_DIR/bin/
  rules:
    # if $GO_TEST_IMAGE set
    - if: '$GO_TEST_IMAGE'

go-test:
  extends: .go-base
  image: $GO_TEST_IMAGE
  stage: build
  script:
    - go_test
  coverage: '/^(\d+.\d+\%) covered$/'
  artifacts:
    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
    expire_in: 1 day
    when: always
    reports:
      junit:
        - "$GO_PROJECT_DIR/reports/go-test.xunit.xml"
      coverage_report:
        coverage_format: cobertura
        path: "$GO_PROJECT_DIR/reports/go-coverage.cobertura.xml"
    paths:
      - "$GO_PROJECT_DIR/reports/go-test.*"
      - "$GO_PROJECT_DIR/reports/go-coverage.*"
  rules:
    # if $GO_TEST_IMAGE set
    - if: '$GO_TEST_IMAGE == null'
      when: never
    - !reference [.test-policy, rules]

go-build-test:
  extends: .go-base
  stage: build
  script:
    - go_build
    - go_test
  coverage: '/^(\d+.\d+\%) covered$/'
  artifacts:
    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
    when: always
    expire_in: 1 day
    reports:
      junit:
        - "$GO_PROJECT_DIR/reports/go-test.xunit.xml"
      coverage_report:
        coverage_format: cobertura
        path: "$GO_PROJECT_DIR/reports/go-coverage.cobertura.xml"
    paths:
      - $GO_PROJECT_DIR/bin/
      - $GO_PROJECT_DIR/reports/
      - "$GO_PROJECT_DIR/reports/go-test.*"
      - "$GO_PROJECT_DIR/reports/go-coverage.*"
  rules:
    # if $GO_TEST_IMAGE not set
    - if: '$GO_TEST_IMAGE == null'

go-ci-lint:
  extends: .go-base
  stage: build
  image: $GO_CI_LINT_IMAGE
  script:
    - mkdir -p -m 777 reports
    # produce all reports at once
    - golangci-lint run --out-format "colored-line-number:stdout,code-climate:reports/go-ci-lint.codeclimate.json,checkstyle:reports/go-ci-lint.checkstyle.xml" $GO_CI_LINT_ARGS
  artifacts:
    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
    expire_in: 1 day
    when: always
    paths:
      - "$GO_PROJECT_DIR/reports/go-ci-lint.*"
    reports:
      codequality:
        - "$GO_PROJECT_DIR/reports/go-ci-lint.codeclimate.json"
  rules:
    # exclude if GO_CI_LINT_DISABLED set
    - if: '$GO_CI_LINT_DISABLED == "true"'
      when: never
    - !reference [.test-policy, rules]

go-mod-outdated:
  extends: .go-base
  stage: test
  dependencies: []
  script:
    - mkdir -p -m 777 reports
    # go list
    - go $GO_LIST_ARGS > reports/go-list.native.json
    - install_go_mod_outdated
    # console output (no fail)
    - $GOBIN/go-mod-outdated $GO_MOD_OUTDATED_ARGS < reports/go-list.native.json
    # text report (-ci fails)
    - $GOBIN/go-mod-outdated $GO_MOD_OUTDATED_ARGS -ci < reports/go-list.native.json > reports/go-mod-outdated.native.txt
  artifacts:
    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
    expire_in: 1 day
    when: always
    paths:
      - "$GO_PROJECT_DIR/reports/go-list.native.json"
      - "$GO_PROJECT_DIR/reports/go-mod-outdated.native.txt"
  rules:
    # on schedule: auto
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      allow_failure: true
    # else manual & non-blocking
    - when: manual
      allow_failure: true

go-sbom:
  extends: .go-base
  stage: test
  image:
    name: $GO_SBOM_IMAGE
    entrypoint: [""]
  # force no dependency
  dependencies: []
  needs: []
  script:
    - mkdir -p -m 777 reports/toto
    - cyclonedx-gomod app -json -output reports/go-sbom.cyclonedx.json $GO_SBOM_OPTS
    - chmod a+r reports/go-sbom.cyclonedx.json
  rules:
    # exclude if disabled
    - if: '$GO_SBOM_DISABLED == "true"'
      when: never
    - !reference [.test-policy, rules]
  artifacts:
    name: "SBOM for golang from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
    expire_in: 1 week
    when: always
    paths:
      - "$GO_PROJECT_DIR/reports/go-sbom.cyclonedx.json"