Support running test environments in parallel with `detox`

This patch namespaces the test temp directory and the output
coverage report file with the name of the environment under test,
such that separately run tests do not access the same files.

When running tests without tox, directly through setup.py,
then the tmp directory will still be `./tmp`.

  * .gitignore: Added new .coverage-reports/ directory

  * .gitlab-ci.yml: Rely on tox to combine and report coverage, only
    tell tox about the COVERAGE_PREFIX so that results can be namespaced
    by CI job name.

    This change also publishes the sources and final combined `.coverage`
    file in an output gitlab artifact for inspection, and lists some missing
    dependencies to the `coverage` job.

  * tox.ini: Add comments and refactor main [testenv] section so that
    other environments dont inherit too much unrelated cruft.

    Set a separate working directory for each test environment,
    solving the parallelism concerns about coverage raised in #844.

    Also implemented new `tox -e coverage` environment to combine
    any found coverage and print a report.

Fixes issue #844

.gitignore

@@ -13,11 +13,12 @@ tests/**/*.pyc
 integration-cache/
 tmp
 .coverage
+.coverage-reports/
 .coverage.*
 .cache
 .pytest_cache/
 *.bst/
-.tox
+.tox/
 
 # Pycache, in case buildstream is ran directly from within the source
 # tree

.gitlab-ci.yml

@@ -13,6 +13,7 @@ variables:
   PYTEST_ADDOPTS: "--color=yes"
   INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache"
   TEST_COMMAND: "tox -- --color=yes --integration"
+  COVERAGE_PREFIX: "${CI_JOB_NAME}."
 
 
 #####################################################
@@ -24,9 +25,6 @@ variables:
 .tests-template: &tests
   stage: test
 
-  variables:
-    COVERAGE_DIR: coverage-linux
-
   before_script:
   # Diagnostics
   - mount
@@ -40,14 +38,11 @@ variables:
   - su buildstream -c "${TEST_COMMAND}"
 
   after_script:
-  # Collect our reports
-  - mkdir -p ${COVERAGE_DIR}
-  - cp .coverage ${COVERAGE_DIR}/coverage."${CI_JOB_NAME}"
   except:
   - schedules
   artifacts:
     paths:
-    - ${COVERAGE_DIR}
+    - .coverage-reports
 
 tests-debian-9:
   image: buildstream/testsuite-debian:9-5da27168-32c47d1c
@@ -83,7 +78,6 @@ tests-unix:
   <<: *tests
   variables:
     BST_FORCE_BACKEND: "unix"
-    COVERAGE_DIR: coverage-unix
 
   script:
 
@@ -239,22 +233,22 @@ coverage:
   stage: post
   coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
   script:
-    - pip3 install -r requirements/requirements.txt -r requirements/dev-requirements.txt
-    - pip3 install --no-index .
-    - mkdir report
-    - cd report
-    - cp ../coverage-unix/coverage.* .
-    - cp ../coverage-linux/coverage.* .
-    - ls coverage.*
-    - coverage combine --rcfile=../.coveragerc -a coverage.*
-    - coverage report --rcfile=../.coveragerc -m
+    - cp -a .coverage-reports/ ./coverage-sources
+    - tox -e coverage
+    - cp -a .coverage-reports/ ./coverage-report
   dependencies:
   - tests-debian-9
   - tests-fedora-27
   - tests-fedora-28
+  - tests-fedora-missing-deps
+  - tests-ubuntu-18.04
   - tests-unix
   except:
   - schedules
+  artifacts:
+    paths:
+    - coverage-sources/
+    - coverage-report/
 
 # Deploy, only for merges which land on master branch.
 #

tox.ini

+#
+# Tox global configuration
+#
 [tox]
 envlist = py35,py36,py37
 skip_missing_interpreters = true
 
+#
+# Defaults for all environments
+#
+# Anything specified here is iherited by the sections
+#
 [testenv]
-commands = pytest {posargs}
+commands =
+    pytest --basetemp {envtmpdir} {posargs} {toxinidir}
+    mkdir -p {toxinidir}/.coverage-reports
+    cp .coverage.{env:COVERAGE_PREFIX:}{envname} {toxinidir}/.coverage-reports
 deps =
     -rrequirements/requirements.txt
     -rrequirements/dev-requirements.txt
@@ -13,6 +24,32 @@ passenv =
     GI_TYPELIB_PATH
     INTEGRATION_CACHE
 
+#
+# These keys are not inherited by any other sections
+#
+changedir =
+    py{35,36,37}: {envdir}
+setenv =
+    py{35,36,37}: COVERAGE_FILE = .coverage.{env:COVERAGE_PREFIX:}{envname}
+whitelist_externals =
+    py{35,36,37}: cp mkdir
+
+#
+# Coverage reporting
+#
+[testenv:coverage]
+commands =
+    - coverage combine --rcfile={toxinidir}/.coveragerc -a {toxinidir}/.coverage-reports/
+    coverage report --rcfile={toxinidir}/.coveragerc -m
+deps =
+    -rrequirements/requirements.txt
+    -rrequirements/dev-requirements.txt
+setenv =
+    COVERAGE_FILE = {toxinidir}/.coverage-reports/.coverage
+
+#
+# Running linters
+#
 [testenv:lint]
 commands =
     pycodestyle
@@ -22,6 +59,9 @@ deps =
     -rrequirements/dev-requirements.txt
     -rrequirements/plugin-requirements.txt
 
+#
+# Building documentation
+#
 [testenv:docs]
 commands =
     make -C doc