Merge branch 'refactor-ci-configuration' into 'master'

Align CI configuration with other Runner tools

See merge request !4

.gitignore

-tlsctl
+.tests/
+.tmp/
+build/

.gitlab-ci.yml

+stages:
+- prepare
+- test
+- compile
+- release
+
+variables:
+  DOCKER_VERSION: "19.03.5"
+  GO_VERSION: "1.13.8"
+  ALPINE_VERSION: "3.11"
+  GCLOUD_SDK_VERSION: "283.0.0"
+  RELEASE_INDEX_GEN_VERSION: "latest"
+  GOPATH: $CI_PROJECT_DIR/.go
+  CI_IMAGE: registry.gitlab.com/gitlab-org/ci-cd/runner-tools/tlsctl/ci:go${GO_VERSION}-alpine${ALPINE_VERSION}-2
+
 default:
   tags:
   - gitlab-org
 
-compile:
-  image: golang:1.13.7-alpine3.11
-  script:
-  - apk add -U bash git make
-  - make version_info compile_all
-  artifacts:
-    paths:
-    - ./build/
-    expire_in: 7d
+cache:
+  paths:
+  - .go/pkg/mod/
+
+image: ${CI_IMAGE}
+
+.merge_request_pipelines:
+  only:
+    refs:
+    - merge_requests
+    - master@gitlab-org/ci-cd/runner-tools/tlsctl
+    - /\Av[0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?\Z/@gitlab-org/ci-cd/runner-tools/tlsctl
+
+.docker_in_docker:
+  image: docker:${DOCKER_VERSION}-git
+  services:
+  - docker:${DOCKER_VERSION}-dind
+  variables:
+    DOCKER_HOST: tcp://docker:2376/
+    DOCKER_DRIVER: overlay2
+    DOCKER_TLS_CERTDIR: "/certs"
+  tags:
+  - gitlab-org-docker
 
+include:
+- local: .gitlab/ci/prepare.gitlab-ci.yml
+- local: .gitlab/ci/test.gitlab-ci.yml
+- local: .gitlab/ci/compile.gitlab-ci.yml
+- local: .gitlab/ci/release.gitlab-ci.yml

.gitlab/changelog.yml

+default_scope: other
+names:
+  new-feature: New features
+  security-fix: Security fixes
+  fix: Bug fixes
+  maintenance: Maintenance
+  documentation: Documentation changes
+  other: Other changes
+order:
+- new-feature
+- security-fix
+- fix
+- maintenance
+- documentation
+- other
+label_matchers:
+- labels:
+  - documentation
+  scope: documentation
+- labels:
+  - feature
+  scope: new-feature
+- labels:
+  - security
+  scope: security-fix
+- labels:
+  - bug
+  scope: fix
+- labels:
+  - technical debt
+  scope: maintenance
+- labels:
+  - backstage
+  scope: maintenance
+authorship_labels:
+- Community contribution

.gitlab/ci/compile.gitlab-ci.yml

+.compile: &compile
+  extends:
+  - .merge_request_pipelines
+  stage: compile
+  script:
+  - export platforms=$(echo $CI_JOB_NAME | sed 's|compile ||')
+  - make compile_all BUILD_PLATFORMS="-osarch='$platforms'"
+  dependencies: []
+  artifacts:
+    paths:
+    - build/
+    expire_in: 7d
+
+compile darwin/amd64: *compile
+compile linux/amd64: *compile

.gitlab/ci/prepare.gitlab-ci.yml

+prepare CI image:
+  extends:
+  - .docker_in_docker
+  stage: prepare
+  script:
+  - apk add --no-cache make
+  - make prepare_ci_image
+  only:
+    refs:
+    - merge_requests
+    changes:
+    - dockerfiles/ci/*
+    - .gitlab/ci/prepare.gitlab-ci.yml

.gitlab/ci/release.gitlab-ci.yml

+.release:
+  stage: release
+
+.release_development:
+  only:
+    refs:
+    - merge_requests@gitlab-org/ci-cd/runner-tools/tlsctl
+
+.release_beta:
+  only:
+    refs:
+    - master@gitlab-org/ci-cd/runner-tools/tlsctl
+    - /\Av[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+\Z/@gitlab-org/ci-cd/runner-tools/tlsctl
+
+.release_stable:
+  only:
+    refs:
+    - /\Av[0-9]+\.[0-9]+\.[0-9]+\Z/@gitlab-org/ci-cd/runner-tools/tlsctl
+
+.release_gcs_base:
+  before_script:
+  - gcloud auth activate-service-account --key-file "${GCP_SERVICE_KEY_FILE}"
+  - gcloud config set project ${GCP_PROJECT_ID}
+  only:
+    variables:
+    - $GCS_BUCKET
+    - $GCS_PATH
+
+.release_gcs:
+  extends:
+  - .release_gcs_base
+  script:
+  - make release_gcs
+
+.remove_release_gcs:
+  extends:
+  - .release_gcs_base
+  script:
+  - make remove_gcs_release
+  when: manual
+
+release development GCS:
+  extends:
+  - .release
+  - .release_development
+  - .release_gcs
+  environment:
+    name: development/GCS/${CI_COMMIT_REF_NAME}
+    url: https://storage.googleapis.com/${GCS_BUCKET}/${GCS_PATH}/${CI_COMMIT_REF_NAME}/index.html
+    on_stop: stop release development GCS
+
+stop release development GCS:
+  extends:
+  - .release
+  - .release_development
+  - .remove_release_gcs
+  environment:
+    name: development/GCS/${CI_COMMIT_REF_NAME}
+    action: stop
+
+release beta GCS:
+  extends:
+  - .release
+  - .release_beta
+  - .release_gcs
+  environment:
+    name: beta/GCS
+    url: https://storage.googleapis.com/${GCS_BUCKET}/${GCS_PATH}/${CI_COMMIT_REF_NAME}/index.html
+
+release stable GCS:
+  extends:
+  - .release
+  - .release_stable
+  - .release_gcs
+  environment:
+    name: stable/GCS
+    url: https://storage.googleapis.com/${GCS_BUCKET}/${GCS_PATH}/${CI_COMMIT_REF_NAME}/index.html

.gitlab/ci/test.gitlab-ci.yml

+tests:
+  extends:
+  - .merge_request_pipelines
+  stage: test
+  script:
+  - make tests
+  coverage: /^total:\s+\(statements\)\s+\d+.\d+\%/
+  artifacts:
+    reports:
+      junit: ./.tests/junit.xml
+    paths:
+    - ./.tests/coverage.html
+    - ./.tests/coverage.txt
+    expire_in: 7d
+    expose_as: Code coverage
+
+tests race:
+  extends:
+  - .merge_request_pipelines
+  stage: test
+  image: golang:${GO_VERSION}-buster
+  variables:
+    CGO_ENABLED: 1
+  script:
+  - make tests_race
+  artifacts:
+    reports:
+      junit: ./.tests/junit-race.xml
+
+code quality:
+  extends:
+  - .merge_request_pipelines
+  stage: test
+  image: golangci/golangci-lint:v1.25.0
+  variables:
+    CODECLIMATE_FORMAT: json
+    LINT_FLAGS: "--color never --deadline 15m"
+    OUT_FORMAT: code-climate
+  script:
+  - make lint | tee ${REPORT_FILE}
+  timeout: 15 minutes
+  artifacts:
+    reports:
+      codequality: ${REPORT_FILE}
+    paths:
+    - ${REPORT_FILE}
+    expire_in: 7d
+  cache: {}
+
+check mocks:
+  extends:
+  - .merge_request_pipelines
+  stage: test
+  script:
+  - make check_mocks
+
+check modules:
+  extends:
+  - .merge_request_pipelines
+  stage: test
+  cache: {}
+  script:
+  - make check_modules

Makefile

@@ -4,26 +4,27 @@ export BRANCH ?= $(shell git show-ref | grep "$(REVISION)" | grep -v HEAD | awk
 export BUILT ?= $(shell date -u +%Y-%m-%dT%H:%M:%S%z)
 export CGO_ENABLED ?= 0
 export GOPATH ?= ".go/"
-export BUILD_PLATFORMS ?= -osarch 'linux/amd64' -osarch 'darwin/amd64'
+export BUILD_PLATFORMS ?= -osarch 'linux/amd64' -osarch 'darwin/amd64' -osarch 'windows/amd64'
 
+export goJunitReport ?= $(GOPATH)/bin/go-junit-report
+mockery ?= $(GOPATH)/bin/mockery
 gox ?= $(GOPATH)/bin/gox
 
+RELEASE_INDEX_GEN_VERSION ?= master
+releaseIndexGen ?= .tmp/release-index-gen-$(RELEASE_INDEX_GEN_VERSION)
+
+GITLAB_CHANGELOG_VERSION ?= master
+gitlabChangelog = .tmp/gitlab-changelog-$(GITLAB_CHANGELOG_VERSION)
+
 PKG := $(shell go list .)
 PKGs := $(shell go list ./... | grep -vE "^/vendor/")
 
-GO_LDFLAGS := -X $(PKG)/version.VERSION=$(VERSION) \
-              -X $(PKG)/version.REVISION=$(REVISION) \
-              -X $(PKG)/version.BRANCH=$(BRANCH) \
-              -X $(PKG)/version.BUILT=$(BUILT) \
+GO_LDFLAGS := -X $(PKG).VERSION=$(VERSION) \
+              -X $(PKG).REVISION=$(REVISION) \
+              -X $(PKG).BRANCH=$(BRANCH) \
+              -X $(PKG).BUILT=$(BUILT) \
               -s -w
 
-.PHONY: version_info
-version_info:
-	# VERSION:  $(VERSION)
-	# REVISION: $(REVISION)
-	# BRANCH:   $(BRANCH)
-	# BUILT:    $(BUILT)
-
 .PHONY: compile
 compile:
 	go build \
@@ -39,12 +40,167 @@ compile_all: $(gox)
 			-output="build/tlsctl-{{.OS}}-{{.Arch}}" \
 			.
 
+export testsDir = ./.tests
+
+.PHONY: tests
+tests: $(testsDir) $(goJunitReport)
+	@./scripts/tests normal
+
+.PHONY: tests_race
+tests_race: $(testsDir) $(goJunitReport)
+	@./scripts/tests race
+
+.PHONY: lint
+lint: OUT_FORMAT ?= colored-line-number
+lint: LINT_FLAGS ?=
+lint:
+	@golangci-lint run ./... --out-format $(OUT_FORMAT) $(LINT_FLAGS)
+$(testsDir):
+	# Preparing tests output directory
+	@mkdir -p $@
+
 .PHONY: fmt
 fmt:
 	# Fixing project code formatting...
 	@go fmt $(PKGs) | awk '{if (NF > 0) {if (NR == 1) print "Please run go fmt for:"; print "- "$$1}} END {if (NF > 0) {if (NR > 0) exit 1}}'
 
+.PHONY: mocks
+mocks: $(mockery)
+	# Removing existing mocks
+	@find * -name "mock_*.go" -delete
+	# Generating new mocks
+	@$(mockery) -recursive -all -inpkg -dir ./
+
+.PHONY: check_mocks
+check_mocks:
+	# Checking if mocks are up-to-date
+	@$(MAKE) mocks
+	# Checking the differences
+	@git --no-pager diff --compact-summary --exit-code -- \
+		$(shell git ls-files **/mock_*.go) \
+		echo "Mocks up-to-date!"
+
+.PHONY: check_modules
+check_modules:
+	@git diff go.sum > /tmp/gosum-$${CI_JOB_ID}-before
+	@go mod tidy
+	@git diff go.sum > /tmp/gosum-$${CI_JOB_ID}-after
+	@diff -U0 /tmp/gosum-$${CI_JOB_ID}-before /tmp/gosum-$${CI_JOB_ID}-after
+
+.PHONY: prepare_ci_image
+prepare_ci_image: CI_IMAGE ?= tlsctl
+prepare_ci_image: CI_REGISTRY ?= ""
+prepare_ci_image:
+	# Builiding the $(CI_IMAGE) image
+	@docker build \
+			--pull \
+			--no-cache \
+			--build-arg GO_VERSION=$${GO_VERSION} \
+			--build-arg ALPINE_VERSION=$${ALPINE_VERSION} \
+			--build-arg GCLOUD_SDK_VERSION=$${GCLOUD_SDK_VERSION} \
+			-t $(CI_IMAGE) \
+			-f dockerfiles/ci/Dockerfile \
+			dockerfiles/ci/
+ifneq ($(CI_REGISTRY),)
+	# Pushing the $(CI_IMAGE) image to $(CI_REGISTRY)
+	@docker login --username $${CI_REGISTRY_USER} --password $${CI_REGISTRY_PASSWORD} $(CI_REGISTRY)
+	@docker push $(CI_IMAGE)
+	@docker logout $(CI_REGISTRY)
+else
+	# No CI_REGISTRY value, skipping image push
+endif
+
+latest_stable_tag := $(shell git -c versionsort.prereleaseSuffix="-rc" tag -l "v*.*.*" --sort=-v:refname | awk '!/rc/' | head -n 1)
+
+.PHONY: release_gcs
+release_gcs: CI_COMMIT_REF_NAME ?= $(BRANCH)
+release_gcs: CI_COMMIT_SHA ?= $(REVISION)
+release_gcs: GCS_BUCKET ?=
+release_gcs: GCS_PATH ?=
+release_gcs:
+	@$(MAKE) index_file
+ifneq ($(GCS_BUCKET),)
+	@$(MAKE) sync_gcs_release GCS_URL="gs://$(GCS_BUCKET)/$(GCS_PATH)/$(CI_COMMIT_REF_NAME)/"
+ifeq ($(shell git describe --exact-match --match $(latest_stable_tag) >/dev/null 2>&1; echo $$?), 0)
+	@$(MAKE) sync_gcs_release GCS_URL="gs://$(GCS_BUCKET)/$(GCS_PATH)/latest/";
+endif
+	@$(MAKE) release_gitlab
+endif
+
+.PHONY: sync_gcs_release
+sync_gcs_release: GCS_URL ?=
+sync_gcs_release:
+	# Syncing with $(GCS_URL)
+	@gsutil rsync build/ "$(GCS_URL)"
+	@gsutil acl set -r public-read "$(GCS_URL)"
+
+.PHONY: remove_gcs_release
+remove_gcs_release: CI_COMMIT_REF_NAME ?= $(BRANCH)
+remove_gcs_release: GCS_BUCKET ?=
+remove_gcs_release: GCS_PATH ?=
+remove_gcs_release:
+ifneq ($(GCS_BUCKET),)
+	@gsutil rm -r "gs://$(GCS_BUCKET)/$(GCS_PATH)/$(CI_COMMIT_REF_NAME)"
+endif
+
+.PHONY: release_gitlab
+release_gitlab: export CI_COMMIT_TAG ?=
+release_gitlab: export CI_PROJECT_URL ?=
+release_gitlab:
+ifneq ($(CI_COMMIT_TAG),)
+	# Saving as GitLab release at $(CI_PROJECT_URL)/-/releases
+	@./scripts/gitlab_release
+endif
+
+.PHONY: index_file
+index_file: export CI_COMMIT_REF_NAME ?= $(BRANCH)
+index_file: export CI_COMMIT_SHA ?= $(REVISION)
+index_file: $(releaseIndexGen)
+	# generating index.html file
+	@$(releaseIndexGen) \
+		-working-directory build/ \
+		-project-version $(VERSION) \
+		-project-git-ref $(CI_COMMIT_REF_NAME) \
+		-project-git-revision $(CI_COMMIT_SHA) \
+		-project-name "TLSCTL" \
+		-project-repo-url "https://gitlab.com/gitlab-org/ci-cd/runner-tools/tlsctl" \
+		-gpg-key-env GPG_KEY \
+		-gpg-password-env GPG_PASSPHRASE
+
+.PHONY: generate_changelog
+generate_changelog: export CHANGELOG_RELEASE ?= $(VERSION)
+generate_changelog: $(gitlabChangelog)
+	@$(gitlabChangelog) \
+			-changelog-file CHANGELOG.md \
+			-config-file .gitlab/changelog.yml \
+			-project-id 15065439 \
+			-release $(CHANGELOG_RELEASE) \
+			-starting-point-matcher "v[0-9]*.[0-9]*.[0-9]*"
+
+$(mockery):
+	# Installing $(mockery)
+	@go get github.com/vektra/mockery/cmd/mockery
+
+$(goJunitReport):
+	# Installing $(goJunitReport)
+	@go get github.com/jstemmer/go-junit-report
+
 $(gox):
-	# Installing gox
-	@go install github.com/mitchellh/gox
+	# Installing $(gox)
+	@go get github.com/mitchellh/gox
+
+$(releaseIndexGen): OS_TYPE ?= $(shell uname -s | tr '[:upper:]' '[:lower:]')
+$(releaseIndexGen): DOWNLOAD_URL = "https://storage.googleapis.com/gitlab-runner-tools/release-index-generator/$(RELEASE_INDEX_GEN_VERSION)/release-index-gen-$(OS_TYPE)-amd64"
+$(releaseIndexGen):
+	# Installing $(DOWNLOAD_URL) as $(releaseIndexGen)
+	@mkdir -p $(shell dirname $(releaseIndexGen))
+	@curl -sL "$(DOWNLOAD_URL)" -o "$(releaseIndexGen)"
+	@chmod +x "$(releaseIndexGen)"
 
+$(gitlabChangelog): OS_TYPE ?= $(shell uname -s | tr '[:upper:]' '[:lower:]')
+$(gitlabChangelog): DOWNLOAD_URL = "https://storage.googleapis.com/gitlab-runner-tools/gitlab-changelog/$(GITLAB_CHANGELOG_VERSION)/gitlab-changelog-$(OS_TYPE)-amd64"
+$(gitlabChangelog):
+	# Installing $(DOWNLOAD_URL) as $(gitlabChangelog)
+	@mkdir -p $(shell dirname $(gitlabChangelog))
+	@curl -sL "$(DOWNLOAD_URL)" -o "$(gitlabChangelog)"
+	@chmod +x "$(gitlabChangelog)"

chain.go

@@ -109,7 +109,10 @@ func chainAction(c *cli.Context) error {
 		}
 	}
 
-	fmt.Fprintf(os.Stdout, chainHead.String())
+	_, err = fmt.Fprint(os.Stdout, chainHead.String())
+	if err != nil {
+		return fmt.Errorf("pringing output: %w", err)
+	}
 
 	return nil
 }

dockerfiles/ci/Dockerfile

+ARG GO_VERSION
+ARG ALPINE_VERSION
+
+FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION}
+
+RUN apk add --no-cache make git py-pip bash curl ca-certificates openssl
+
+ARG GCLOUD_SDK_VERSION
+
+RUN curl -L https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GCLOUD_SDK_VERSION}-linux-x86_64.tar.gz -o /tmp/google-cloud-sdk.tar.gz && \
+    tar -xzvf /tmp/google-cloud-sdk.tar.gz -C /usr/local/lib && \
+    /usr/local/lib/google-cloud-sdk/install.sh --usage-reporting=false --path-update=true && \
+    rm -rf /tmp/*
+
+ENV PATH="/usr/local/lib/google-cloud-sdk/bin:${PATH}"
+
+RUN gcloud --quiet components update

dockerfiles/ci/force-build

+Tue May  5 17:57:23 CEST 2020

scripts/gitlab_release

+#!/usr/bin/env bash
+
+set -eo pipefail
+
+tag=${1:-$CI_COMMIT_TAG}
+
+if [[ -z "${tag}" ]]; then
+    echo -e "\033[0;31m****** gitlab publishing disabled ******\033[0m"
+    echo -e "usage:\n\t$0 tag"
+    exit 0
+fi
+
+if [[ -z "${DEPLOY_TOKEN}" ]]; then
+    echo -e "\033[0;31m****** Missing DEPLOY_TOKEN, cannot release ******\033[0m"
+    exit 0
+fi
+
+api=${CI_API_V4_URL:-https://gitlab.com/api/v4}
+projectID=${CI_PROJECT_ID:-15065439}
+gcs=${CI_ENVIRONMENT_URL/\/index.html/}
+
+release=$(cat <<EOS
+{
+  "name": "${tag}",
+  "tag_name": "${tag}",
+  "description": "The ${tag} release is here! :rocket:",
+  "assets": {
+    "links": [
+      { "name": "darwin amd64", "url": "${gcs}/tlsctl-darwin-amd64" },
+      { "name": "linux amd64", "url": "${gcs}/tlsctl-linux-amd64" },
+      { "name": "windows amd64", "url": "${gcs}/tlsctl-windows-amd64.exe" },
+      { "name": "SHA256 checksums", "url": "${gcs}/release.sha256" },
+      { "name": "SHA256 checksums - GPG signature", "url": "${gcs}/release.sha256.asc" },
+      { "name": "all release artifacts", "url": "${gcs}/index.html" }
+    ]
+  }
+}
+EOS
+)
+
+curl -f \
+     --header 'Content-Type: application/json' \
+     --header "PRIVATE-TOKEN: ${DEPLOY_TOKEN}" \
+     --data "${release}" \
+     --request POST \
+     "${api}/projects/${projectID}/releases"

scripts/tests

+#!/usr/bin/env bash
+
+set -eo pipefail
+
+testsDir=${testsDir:-"./.tests/"}
+goJunitReport=${goJunitReport:-"go-junit-report"}
+
+function generate_coverage_report() {
+	echo "Generating coverage HTML report"
+	echo "To open execute:"
+	echo "  x-www-browser ${testsDir}/coverage.html"
+
+	go tool cover -o "${testsDir}/coverage.html" -html="${testsDir}/coverage.out"
+
+	echo "Generating function report: ${testsDir}/coverage.txt"
+
+	go tool cover -o "${testsDir}/coverage.txt" -func="${testsDir}/coverage.out"
+	grep "^total:" "${testsDir}/coverage.txt"
+}
+
+function generate_junit_report() {
+  local junitFile=${1}
+
+  echo "Generating jUnit report: ${testsDir}/${junitFile}"
+
+	${goJunitReport} < "${testsDir}/output.txt" > "${testsDir}/${junitFile}"
+}
+
+function normal() {
+  local exitCode=0
+
+  go test -v ./... -cover -covermode=count -coverprofile="${testsDir}/coverage.out.source" | tee "${testsDir}/output.txt" || exitCode=1
+  grep -v -E "mock_.*\.go" "${testsDir}/coverage.out.source" > "${testsDir}/coverage.out"
+
+  generate_coverage_report
+  generate_junit_report "junit.xml"
+
+  exit ${exitCode}
+}
+
+function race() {
+  local exitCode=0
+
+  go test -v -race ./... | tee "${testsDir}/output.txt" || exitCode=1
+
+  generate_junit_report "junit-race.xml"
+
+  exit ${exitCode}
+}
+
+case "${1}" in
+  normal)