Commit f29a0995 authored by Tristan Van Berkom's avatar Tristan Van Berkom

Support running test environments in parallel with `detox`

This patch namespaces the test temp directory and the output
coverage report file with the name of the environment under test,
such that separately run tests do not access the same files.

When running tests without tox, directly through setup.py,
then the tmp directory will still be `./tmp`.

  * .gitignore: Added new .coverage-reports/ directory

  * .gitlab-ci.yml: Rely on tox to combine and report coverage, only
    tell tox about the COVERAGE_PREFIX so that results can be namespaced
    by CI job name.

    This change also publishes the sources and final combined `.coverage`
    file in an output gitlab artifact for inspection, and lists some missing
    dependencies to the `coverage` job.

  * tox.ini: Add comments and refactor main [testenv] section so that
    other environments dont inherit too much unrelated cruft.

    Generate the coverate reports in the respective {envtmpdir} so that
    all per-process coverage files are prefixed with a full path, ensuring
    that concurrent runs don't mix reports and addressing concerns
    raised in #844.

    Also implemented new `tox -e coverage` environment to combine
    any found coverage and print a report.

  * .coveragerc: Omit .tox/ directory from coverage stats

Fixes issue #844
parent ed8bc5bc
......@@ -9,6 +9,8 @@ omit =
# Omit generated code
*/buildstream/_protos/*
*/.eggs/*
# Omit .tox directory
*/.tox/*
[report]
show_missing = True
......
......@@ -13,11 +13,12 @@ tests/**/*.pyc
integration-cache/
tmp
.coverage
.coverage-reports/
.coverage.*
.cache
.pytest_cache/
*.bst/
.tox
.tox/
# Pycache, in case buildstream is ran directly from within the source
# tree
......
......@@ -13,6 +13,7 @@ variables:
PYTEST_ADDOPTS: "--color=yes"
INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache"
TEST_COMMAND: "tox -- --color=yes --integration"
COVERAGE_PREFIX: "${CI_JOB_NAME}."
#####################################################
......@@ -24,9 +25,6 @@ variables:
.tests-template: &tests
stage: test
variables:
COVERAGE_DIR: coverage-linux
before_script:
# Diagnostics
- mount
......@@ -40,14 +38,11 @@ variables:
- su buildstream -c "${TEST_COMMAND}"
after_script:
# Collect our reports
- mkdir -p ${COVERAGE_DIR}
- cp .coverage ${COVERAGE_DIR}/coverage."${CI_JOB_NAME}"
except:
- schedules
artifacts:
paths:
- ${COVERAGE_DIR}
- .coverage-reports
tests-debian-9:
image: buildstream/testsuite-debian:9-5da27168-32c47d1c
......@@ -83,7 +78,6 @@ tests-unix:
<<: *tests
variables:
BST_FORCE_BACKEND: "unix"
COVERAGE_DIR: coverage-unix
script:
......@@ -239,22 +233,22 @@ coverage:
stage: post
coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
script:
- pip3 install -r requirements/requirements.txt -r requirements/dev-requirements.txt
- pip3 install --no-index .
- mkdir report
- cd report
- cp ../coverage-unix/coverage.* .
- cp ../coverage-linux/coverage.* .
- ls coverage.*
- coverage combine --rcfile=../.coveragerc -a coverage.*
- coverage report --rcfile=../.coveragerc -m
- cp -a .coverage-reports/ ./coverage-sources
- tox -e coverage
- cp -a .coverage-reports/ ./coverage-report
dependencies:
- tests-debian-9
- tests-fedora-27
- tests-fedora-28
- tests-fedora-missing-deps
- tests-ubuntu-18.04
- tests-unix
except:
- schedules
artifacts:
paths:
- coverage-sources/
- coverage-report/
# Deploy, only for merges which land on master branch.
#
......
#
# Tox global configuration
#
[tox]
envlist = py35,py36,py37
skip_missing_interpreters = true
#
# Defaults for all environments
#
# Anything specified here is iherited by the sections
#
[testenv]
commands = pytest {posargs}
commands =
pytest --basetemp {envtmpdir} {posargs}
mkdir -p .coverage-reports
mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
deps =
-rrequirements/requirements.txt
-rrequirements/dev-requirements.txt
......@@ -13,6 +24,32 @@ passenv =
GI_TYPELIB_PATH
INTEGRATION_CACHE
#
# These keys are not inherited by any other sections
#
setenv =
py{35,36,37}: COVERAGE_FILE = {envtmpdir}/.coverage
whitelist_externals =
py{35,36,37}:
mv
mkdir
#
# Coverage reporting
#
[testenv:coverage]
commands =
- coverage combine --rcfile={toxinidir}/.coveragerc {toxinidir}/.coverage-reports/
coverage report --rcfile={toxinidir}/.coveragerc -m
deps =
-rrequirements/requirements.txt
-rrequirements/dev-requirements.txt
setenv =
COVERAGE_FILE = {toxinidir}/.coverage-reports/.coverage
#
# Running linters
#
[testenv:lint]
commands =
pycodestyle
......@@ -22,6 +59,9 @@ deps =
-rrequirements/dev-requirements.txt
-rrequirements/plugin-requirements.txt
#
# Building documentation
#
[testenv:docs]
commands =
make -C doc
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment