Commit c4f346d8 authored by Russell's avatar Russell
Browse files

Apply build_harness pipeline

parent 5a1370b4
image: ${DOCKER_REGISTRY}/docker_download_3gpp/blueskyjunkie-docker-download-3gpp:0.2.0
image: python:3.8-slim
stages:
- pre-package
- package
- test
- setup
- static-analysis
- tests
- publish
variables:
FLIT_ROOT_INSTALL: 1
# Fulfilled by Gitlab CI
FLIT_USERNAME: __token__
FLIT_PASSWORD: SECURE
PYPI_API_USER: __token__
PACKAGE_PATH: 'dist/${CI_PROJECT_NAME}*.whl'
# Don't forget to update the coverage threshold in .pre-commit-config.yaml
UNITTEST_COVERAGE_THRESHOLD: 95
# Bash doesn't have "\d" shortcut for decimal numbers and doesn't use "/" terminators.
# This release pattern also explicitly ignores the "-dryrun" suffix.
BASH_RELEASE_PATTERN: '^[0-9]+\.[0-9]+\.[0-9]+$'
VENV_BIN: .venv/bin
VERSION_PATH: ${CI_PROJECT_NAME}/VERSION
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS'
when: never
- if: '$CI_PIPELINE_SOURCE == "push"'
DOCKER_REGISTRY: registry.gitlab.com/blueskyjunkie
cache:
key: ${CI_COMMIT_REF_SLUG}
paths:
- .cache/pip
- .venv/
MAKE_PACKAGES: make -f build_harness/Makefile.packages
MAKE_STATIC_ANALYSIS: make -f build_harness/Makefile.static_analysis
MAKE_TESTS: make -f build_harness/Makefile.tests
build-package:
.build-harness-base:
artifacts:
expire_in: 1 week
paths:
- ${PACKAGE_PATH}
stage: package
- dist/
before_script:
- 'apt-get update
&& apt-get install -y
git'
- git config --global user.email "you@example.com"
- git config --global user.name "Your Name"
.build-harness-target:
script:
- '${MAKE_PACKAGES} build-package
GIT_REF_NAME=${CI_COMMIT_REF_NAME}
PIPELINE_ID=${CI_PIPELINE_ID}'
- ${VENV_BIN}/flit install -s
- '${VENV_BIN}/build-harness ${TARGET}'
extends:
- .build-harness-base
run-wheel-install:
dependencies:
- build-package
stage: test
install-dependencies:
stage: setup
script:
- '${MAKE_PACKAGES} test-wheel-install'
- python3 -m venv .venv
- ${VENV_BIN}/pip install flit pygments wheel
- ${VENV_BIN}/flit install -s
# This is redundant in light of the previous `flit install -s` step, but it serves
# to exercise the `build-harness install` command.
- ${VENV_BIN}/build-harness install
extends:
- .build-harness-base
run-unit-tests:
artifacts:
reports:
junit: junit_report.xml
stage: test
script:
- '${MAKE_TESTS} pytest'
formatting-check:
stage: static-analysis
variables:
TARGET: formatting --check
extends:
- .build-harness-target
run-black-check:
stage: test
script:
- '${MAKE_STATIC_ANALYSIS} black-check'
all-check:
stage: static-analysis
variables:
TARGET: static-analysis
extends:
- .build-harness-target
run-flake8-check:
stage: test
script:
- '${MAKE_STATIC_ANALYSIS} flake8'
flake8-check:
stage: static-analysis
variables:
TARGET: static-analysis --analysis flake8
extends:
- .build-harness-target
run-pydocstyle-check:
stage: test
script:
- '${MAKE_STATIC_ANALYSIS} pydocstyle'
mypy-check:
stage: static-analysis
variables:
TARGET: static-analysis --analysis mypy
extends:
- .build-harness-target
run-isort-check:
stage: test
script:
- '${MAKE_STATIC_ANALYSIS} isort-check'
pydocstyle-check:
stage: static-analysis
variables:
TARGET: static-analysis --analysis pydocstyle
after_script:
# log diffs for debugging
- '${MAKE_STATIC_ANALYSIS} isort-diffs'
extends:
- .build-harness-target
run-mypy-check:
stage: test
unit-tests:
stage: tests
variables:
TARGET: unit-test
script:
- '${MAKE_STATIC_ANALYSIS} mypy'
extends:
- .build-harness-target
run-coverage:
coverage: '/TOTAL.*?([0-9]{1,3})%/'
stage: test
acceptance-tests:
# NOTE: Remove or disable allow_failure once bdd features are implemented.
allow_failure: true
stage: tests
variables:
TARGET: acceptance tests --junitxml
script:
- '${MAKE_TESTS} coverage'
extends:
- .build-harness-target
run-coverage-threshold-test:
stage: test
acceptance-tags:
stage: tests
variables:
TARGET: acceptance tags
script:
- '${MAKE_TESTS} coverage-threshold-check'
after_script:
# log coverage report for debugging
- '${MAKE_TESTS} coverage'
extends:
- .build-harness-target
run-xml-coverage-report:
artifacts:
paths:
- dist/coverage.xml
stage: test
unit-tests-coverage:
stage: tests
variables:
TARGET: unit-test --check ${UNITTEST_COVERAGE_THRESHOLD}
script:
- '${MAKE_TESTS} coverage-xml'
extends:
- .build-harness-target
run-html-coverage-report:
artifacts:
paths:
- dist/htmlcov/*
stage: test
install-check:
stage: tests
variables:
TARGET: install
extends:
- .build-harness-target
build-packages:
stage: tests
script:
- '${MAKE_TESTS} coverage-html'
- export THIS_VERSION=$(${VENV_BIN}/release-flow)
# log THIS_VERSION to pipeline log for debugging
- echo ${THIS_VERSION}
- ${VENV_BIN}/build-harness package --release-id ${THIS_VERSION}
extends:
- .build-harness-base
publish-wheel:
dependencies:
- build-package
publish-release-packages:
needs:
- job: build-packages
artifacts: true
rules:
- if: '$CI_COMMIT_TAG =~ /^(\d+\.)*\d+$/'
stage: publish
variables:
TARGET: publish --user $PYPI_API_USER --password $PYPI_API_TOKEN
extends:
- .build-harness-target
publish-release-dryrun:
needs:
- job: build-packages
artifacts: true
rules:
- if: '$CI_COMMIT_TAG =~ /(\+dryrun\d*)$/'
stage: publish
variables:
TARGET: publish --user $PYPI_API_USER --password $PYPI_API_TOKEN --dryrun
extends:
- .build-harness-target
publish-nonrelease-packages:
needs:
- job: build-packages
artifacts: true
rules:
- if: '$CI_COMMIT_TAG !~ /(\+dryrun\d*)$/ &&
$CI_COMMIT_TAG !~ /^(\d+\.)*\d+$/'
stage: publish
script:
- '${MAKE_PACKAGES} publish-wheel
GIT_REF_NAME=${CI_COMMIT_REF_NAME}
PIPELINE_ID=${CI_PIPELINE_ID}'
- ls dist/
extends:
- .build-harness-base
repos:
- repo: local
hooks:
- entry: build-harness install --check
id: install-check
language: system
name: Virtual environment installed dependencies are current
pass_filenames: false
- repo: local
hooks:
- entry: build-harness formatting
id: formatting
language: system
name: Black, isort formatting passes
pass_filenames: false
- repo: local
hooks:
- entry: build-harness static-analysis --analysis flake8
id: flake8
language: system
name: Build harness flake8 analysis passes
pass_filenames: false
- repo: local
hooks:
- entry: build-harness static-analysis --analysis mypy
id: mypy
language: system
name: Build harness mypy analysis passes
pass_filenames: false
- repo: local
hooks:
- entry: build-harness static-analysis --analysis pydocstyle
id: static-analysis
language: system
name: Build harness pydocstyle analysis passes
pass_filenames: false
- repo: local
hooks:
- entry: build-harness unit-test
id: unit-test
language: system
name: Unit tests run clean
pass_filenames: false
- repo: local
hooks:
- entry: build-harness unit-test --check 95
id: unit-test-check
language: system
name: Coverage analysis passes
pass_filenames: false
#
# Copyright 2020 Russell Smiley
#
# This file is part of download_3gpp.
#
# download_3gpp is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# download_3gpp is distributed in the hope that it will be useful
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with download_3gpp. If not, see <http://www.gnu.org/licenses/>.
#
include build_harness/Makefile.project
# Bash doesn't have "\d" shortcut for decimal numbers and doesn't use "/" terminators.
# This release pattern also explicitly ignores the "-dryrun" suffix.
BASH_RELEASE_PATTERN=^[0-9]+\.[0-9]+\.[0-9]+$
PACKAGE_PATH=$(DIST_DIR)/$(PROJECT_NAME)*.whl
VERSION_PATH=$(PROJECT_NAME)/VERSION
.PHONY: build-package
build-package: install-dependencies update-version
$(VENV_BINDIR)/flit build
# Need to ensure that flit is looking for the correct version when it publishes
.PHONY: publish-wheel
publish-wheel: install-dependencies update-version
@$(call assert_is_defined,GIT_REF_NAME)
# debug logging
ls -l dist/
# On a strict release tag publish the package to pypi.org
# Otherwise just log a message
[[ $(GIT_REF_NAME) =~ $(BASH_RELEASE_PATTERN) ]] && \
($(VENV_BINDIR)/flit publish) || \
(echo "Publish dry run")
.PHONY: test-wheel-install
test-wheel-install: $(VENV_DIR)
$(VENV_PIP) install ${PACKAGE_PATH}
.PHONY: update-version
update-version: $(VERSION_PATH)
.PHONY: $(VERSION_PATH)
$(VERSION_PATH):
@$(call assert_is_defined,GIT_REF_NAME)
@$(call assert_is_defined,PIPELINE_ID)
$(VENV_BINDIR)/python build_harness/update_version.py \
$(PIPELINE_ID) \
$(GIT_REF_NAME) \
>$@
# debug logging
cat $@
#
# Copyright 2020 Russell Smiley
#
# This file is part of download_3gpp.
#
# download_3gpp is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# download_3gpp is distributed in the hope that it will be useful
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with download_3gpp. If not, see <http://www.gnu.org/licenses/>.
#
SHELL=/bin/bash
SYSTEM_PYTHON:=$(shell $$(python -c "import sys; assert sys.version_info.major==3" 2>&1 >/dev/null) && (echo $$(command -v python)) || (echo $$(command -v python3)))
ifdef CI_PROJECT_NAME
$(info Looks like Gitlab-CI is running)
PROJECT_NAME:=${CI_PROJECT_NAME}
else
$(info Looks like Gitlab-CI is not running. Interpolating the project name.)
PROJECT_NAME:=download_3gpp
endif
DIST_DIR:=dist
TEST_DIR:=tests
VENV_DIR:=.venv
VENV_BINDIR:=$(VENV_DIR)/bin
VENV_PIP:=$(VENV_BINDIR)/pip
DEPENDENCIES_INSTALLED_FILE:=.dependencies_installed
.PHONY: install-dependencies
install-dependencies: $(DEPENDENCIES_INSTALLED_FILE)
define assert_is_defined
[ ! -z "$($(1))" ] || (echo "FAILED variable not defined, $(1)" >/dev/stderr; exit 1)
endef
$(DEPENDENCIES_INSTALLED_FILE): $(VENV_DIR)
@$(VENV_PIP) install --upgrade pip
@$(VENV_PIP) install wheel pygments flit
@$(VENV_BINDIR)/flit install -s
@touch $@
$(DIST_DIR):
mkdir -p $@
$(VENV_DIR):
@[ ! -d "$(VENV_DIR)" ] && ($(SYSTEM_PYTHON) -m venv "$(VENV_DIR)")
#
# Copyright 2020 Russell Smiley
#
# This file is part of download_3gpp.
#
# download_3gpp is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# download_3gpp is distributed in the hope that it will be useful
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with download_3gpp. If not, see <http://www.gnu.org/licenses/>.
#
include build_harness/Makefile.project
.PHONY: assert-dependencies
assert-dependencies:
@$(call assert_is_defined,PROJECT_NAME)
@$(call assert_is_defined,TEST_DIR)
.PHONY: black-check
black-check: install-dependencies assert-dependencies
# Exit dirty on failure for a calling process such as CI pipeline to consume
@$(VENV_BINDIR)/black --check $(PROJECT_NAME) || (echo "FAILED black check $(PROJECT_NAME)"; exit 1)
@$(VENV_BINDIR)/black --check $(TEST_DIR) || (echo "FAILED black check $(TEST_DIR)"; exit 1)
.PHONY: black-formatting
black-formatting: install-dependencies assert-dependencies
$(VENV_BINDIR)/black $(PROJECT_NAME)
$(VENV_BINDIR)/black $(TEST_DIR)
.PHONY: flake8
flake8: install-dependencies assert-dependencies
@$(VENV_BINDIR)/flake8 $(PROJECT_NAME) || (echo "FAILED flake8 analysis"; exit 1)
.PHONY: isort-check
isort-check: install-dependencies assert-dependencies
# Exit dirty on failure for a calling process such as CI pipeline to consume
@[[ ! $$($(VENV_BINDIR)/isort --diff $(PROJECT_NAME)) ]] || (echo "FAILED isort check $(PROJECT_NAME)"; exit 1)
@[[ ! $$($(VENV_BINDIR)/isort --diff $(TEST_DIR)) ]] || (echo "FAILED isort check $(PROJECT_NAME)"; exit 1)
.PHONY: isort-diffs
isort-diffs: install-dependencies assert-dependencies
$(VENV_BINDIR)/isort --diff $(PROJECT_NAME)
$(VENV_BINDIR)/isort --diff $(TEST_DIR)
.PHONY: install-dependencies isort-formatting
isort-formatting: assert-dependencies
$(VENV_BINDIR)/isort $(PROJECT_NAME)
$(VENV_BINDIR)/isort $(TEST_DIR)
.PHONY: mypy
mypy: install-dependencies assert-dependencies
@$(VENV_BINDIR)/mypy \
--show-error-codes \
$(PROJECT_NAME) \
|| (echo "FAILED mypy analysis"; exit 1)
.PHONY: pydocstyle
pydocstyle: install-dependencies assert-dependencies
@$(VENV_BINDIR)/pydocstyle $(PROJECT_NAME) || (echo "FAILED pydocstyle analysis"; exit 1)
.PHONY: formatting
# Do isort before black formatting so black takes precedence in the event of disagreement
formatting: | isort-formatting black-formatting
.PHONY: analysis
analysis: pydocstyle flake8 mypy
.PHONY: static-analysis
static-analysis: | formatting analysis
#
# Copyright 2020 Russell Smiley
#
# This file is part of download_3gpp.
#
# download_3gpp is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# download_3gpp is distributed in the hope that it will be useful
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with download_3gpp. If not, see <http://www.gnu.org/licenses/>.
#
include build_harness/Makefile.project
COVERAGE_THRESHOLD=95
.PHONY: assert-dependencies
assert-dependencies:
$(call assert_is_defined,PROJECT_NAME)
.PHONY: coverage-html
coverage-html: install-dependencies assert-dependencies $(DIST_DIR)
$(VENV_BINDIR)/pytest \
--cov=$(PROJECT_NAME) \
--cov-report=html \
$(TEST_DIR)
mv htmlcov $(DIST_DIR)/
.PHONY: coverage
coverage: install-dependencies assert-dependencies
$(VENV_BINDIR)/pytest \
--cov=$(PROJECT_NAME) \
$(TEST_DIR)