.gitignore

@@ -13,11 +13,12 @@ tests/**/*.pyc
 integration-cache/
 tmp
 .coverage
+.coverage-reports/
 .coverage.*
 .cache
 .pytest_cache/
 *.bst/
-.tox
+.tox/
 
 # Pycache, in case buildstream is ran directly from within the source
 # tree

.gitlab-ci.yml

@@ -13,6 +13,7 @@ variables:
   PYTEST_ADDOPTS: "--color=yes"
   INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache"
   TEST_COMMAND: "tox -- --color=yes --integration"
+  COVERAGE_PREFIX: "${CI_JOB_NAME}."
 
 
 #####################################################
@@ -24,9 +25,6 @@ variables:
 .tests-template: &tests
   stage: test
 
-  variables:
-    COVERAGE_DIR: coverage-linux
-
   before_script:
   # Diagnostics
   - mount
@@ -40,14 +38,11 @@ variables:
   - su buildstream -c "${TEST_COMMAND}"
 
   after_script:
-  # Collect our reports
-  - mkdir -p ${COVERAGE_DIR}
-  - cp .coverage ${COVERAGE_DIR}/coverage."${CI_JOB_NAME}"
   except:
   - schedules
   artifacts:
     paths:
-    - ${COVERAGE_DIR}
+    - .coverage-reports
 
 tests-debian-9:
   image: buildstream/testsuite-debian:9-5da27168-32c47d1c
@@ -83,7 +78,6 @@ tests-unix:
   <<: *tests
   variables:
     BST_FORCE_BACKEND: "unix"
-    COVERAGE_DIR: coverage-unix
 
   script:
 
@@ -239,22 +233,22 @@ coverage:
   stage: post
   coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
   script:
-    - pip3 install -r requirements/requirements.txt -r requirements/dev-requirements.txt
-    - pip3 install --no-index .
-    - mkdir report
-    - cd report
-    - cp ../coverage-unix/coverage.* .
-    - cp ../coverage-linux/coverage.* .
-    - ls coverage.*
-    - coverage combine --rcfile=../.coveragerc -a coverage.*
-    - coverage report --rcfile=../.coveragerc -m
+    - cp -a .coverage-reports/ ./coverage-sources
+    - tox -e coverage
+    - cp -a .coverage-reports/ ./coverage-report
   dependencies:
   - tests-debian-9
   - tests-fedora-27
   - tests-fedora-28
+  - tests-fedora-missing-deps
+  - tests-ubuntu-18.04
   - tests-unix
   except:
   - schedules
+  artifacts:
+    paths:
+    - coverage-sources/
+    - coverage-report/
 
 # Deploy, only for merges which land on master branch.
 #

CONTRIBUTING.rst

@@ -1498,6 +1498,10 @@ option when running tox::
 
   tox -e py37
 
+If you do have multiple python versions installed and would like to test against
+multiple versions, then we recommend using `detox <https://github.com/tox-dev/detox>`_,
+just run it with the same arguments you would give `tox`.
+
 Linting is performed separately from testing. In order to run the linting step which
 consists of running the ``pycodestyle`` and ``pylint`` tools, run the following::
 
@@ -1574,6 +1578,19 @@ can run ``tox`` with ``-r`` or  ``--recreate`` option.
      ./setup.py test --addopts 'tests/frontend/buildtrack.py::test_build_track'
 
 
+Observing coverage
+~~~~~~~~~~~~~~~~~~
+Once you have run the tests using `tox` (or `detox`), some coverage reports will
+have been left behind.
+
+To view the coverage report of the last test run, simply run::
+
+  tox -e coverage
+
+This will collate any reports from separate python environments that may be
+under test before displaying the combined coverage.
+
+
 Adding tests
 ~~~~~~~~~~~~
 Tests are found in the tests subdirectory, inside of which

tox.ini

+#
+# Tox global configuration
+#
 [tox]
 envlist = py35,py36,py37
 skip_missing_interpreters = true
 
+#
+# Defaults for all environments
+#
+# Anything specified here is iherited by the sections
+#
 [testenv]
-commands = pytest {posargs}
+commands =
+    pytest --basetemp {envtmpdir} {posargs} {toxinidir}
+    mkdir -p {toxinidir}/.coverage-reports
 deps =
     -rrequirements/requirements.txt
     -rrequirements/dev-requirements.txt
@@ -13,6 +23,32 @@ passenv =
     GI_TYPELIB_PATH
     INTEGRATION_CACHE
 
+#
+# These keys are not inherited by any other sections
+#
+setenv =
+    py{35,36,37}: COVERAGE_FILE = .coverage.{env:COVERAGE_PREFIX:}{envname}
+whitelist_externals =
+    py{35,36,37}:
+        cp
+	mkdir
+
+#
+# Coverage reporting
+#
+[testenv:coverage]
+commands =
+    - coverage combine --rcfile={toxinidir}/.coveragerc {toxinidir}/.coverage-reports/
+    coverage report --rcfile={toxinidir}/.coveragerc -m
+deps =
+    -rrequirements/requirements.txt
+    -rrequirements/dev-requirements.txt
+setenv =
+    COVERAGE_FILE = {toxinidir}/.coverage-reports/.coverage
+
+#
+# Running linters
+#
 [testenv:lint]
 commands =
     pycodestyle
@@ -22,6 +58,9 @@ deps =
     -rrequirements/dev-requirements.txt
     -rrequirements/plugin-requirements.txt
 
+#
+# Building documentation
+#
 [testenv:docs]
 commands =
     make -C doc