.gitignore

@@ -17,6 +17,7 @@ tmp
 .cache
 .pytest_cache/
 *.bst/
+.tox
 
 # Pycache, in case buildstream is ran directly from within the source
 # tree

.gitlab-ci.yml

@@ -13,7 +13,7 @@ stages:
 variables:
   PYTEST_ADDOPTS: "--color=yes"
   INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache"
-  TEST_COMMAND: 'python3 setup.py test --index-url invalid://uri --addopts --integration'
+  TEST_COMMAND: 'tox -- --color=yes --integration'
 
 #####################################################
 #                  Prepare stage                    #
@@ -67,10 +67,10 @@ source_dist:
   # Diagnostics
   - mount
   - df -h
-
-  # Unpack
-  - cd dist && ./unpack.sh
-  - cd buildstream
+  # FIXME: this deps should ideally be provided by the testsuite images
+  - pip3 install tox
+  - |
+    bash -c '( apt-get update && apt-get install -y python3-dev gcc libcairo2-dev pkg-config libgirepository1.0-dev ) || ( dnf install -y gcc python3-devel gobject-introspection-devel glib2-devel cairo-gobject-devel )'
 
   script:
   - useradd -Um buildstream
@@ -83,7 +83,7 @@ source_dist:
   after_script:
   # Collect our reports
   - mkdir -p ${COVERAGE_DIR}
-  - cp dist/buildstream/.coverage ${COVERAGE_DIR}/coverage."${CI_JOB_NAME}"
+  - cp .coverage ${COVERAGE_DIR}/coverage."${CI_JOB_NAME}"
   except:
   - schedules
   artifacts:

CONTRIBUTING.rst

@@ -1476,50 +1476,96 @@ The elaborate documentation for pytest can be found here: http://doc.pytest.org/
 Don't get lost in the docs if you don't need to, follow existing examples instead.
 
 
+Installing build dependencies
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Some of BuildStream's dependencies have non-python build dependencies, so in
+order to be able to run BuildStream's tests, you will first need to install
+these dependencies. Exact steps to install these will depend on your
+oprtation systemm. Commands for installing them for some common distributions
+are lised below.
+
+For Fedora-based systems::
+
+  dnf install gcc pkg-config python3-devel cairo-gobject-devel glib2-devel gobject-introspection-devel
+
+
+For Debian-based systems::
+
+  apt install gcc pkg-config python3-dev libcairo2-dev libgirepository1.0-dev
+
+
 Running tests
 ~~~~~~~~~~~~~
-To run the tests, just type::
+We use `tox <https://tox.readthedocs.org/>`_ as a frontend run the tests which
+are implemented using `pytest <https://pytest.org/>`_. To run the tests, simply
+navigate to the toplevel directory of your buildstream checkout and run::
+
+  tox
 
-  ./setup.py test
+By default, the test suite will be run against every supported python version
+found on your host. If you have multiple python versions installed, you may
+want to run tests against only one version and you can do that using the ``-e``
+option when running tox::
 
-At the toplevel.
+  tox -e py37
 
-When debugging a test, it can be desirable to see the stdout
-and stderr generated by a test, to do this use the ``--addopts``
-function to feed arguments to pytest as such::
+The output of all failing tests will always be printed in the summary, but
+if you want to observe the stdout and stderr generated by a passing test,
+you can pass the ``-s`` option to pytest as such::
 
-  ./setup.py test --addopts -s
+  tox -- -s
+
+.. tip::
+
+   The ``-s`` option is `a pytest option <https://docs.pytest.org/latest/usage.html>`_.
+
+   Any options specified before the ``--`` separator are consumed by ``tox``,
+   and any options after the ``--`` separator will be passed along to pytest.
 
 You can always abort on the first failure by running::
 
-  ./setup.py test --addopts -x
+  tox -- -x
 
 If you want to run a specific test or a group of tests, you
 can specify a prefix to match. E.g. if you want to run all of
 the frontend tests you can do::
 
-  ./setup.py test --addopts 'tests/frontend/'
+  tox -- tests/frontend/
 
 Specific tests can be chosen by using the :: delimeter after the test module.
 If you wanted to run the test_build_track test within frontend/buildtrack.py you could do::
 
-  ./setup.py test --addopts 'tests/frontend/buildtrack.py::test_build_track'
+  tox -- tests/frontend/buildtrack.py::test_build_track
 
 We also have a set of slow integration tests that are disabled by
 default - you will notice most of them marked with SKIP in the pytest
 output. To run them, you can use::
 
-  ./setup.py test --addopts '--integration'
+  tox -- --integration
 
 By default, buildstream also runs pylint on all files. Should you want
 to run just pylint (these checks are a lot faster), you can do so
 with::
 
-  ./setup.py test --addopts '-m pylint'
+  tox -- -m pylint
 
 Alternatively, any IDE plugin that uses pytest should automatically
 detect the ``.pylintrc`` in the project's root directory.
 
+.. note::
+
+   While using ``tox`` is practical for developers running tests in
+   more predictable execution environments, it is still possible to
+   execute the test suite against a specific installation environment
+   using pytest directly::
+
+     ./setup.py test
+
+   Specific options can be passed to ``pytest`` using the ``--addopts``
+   option::
+
+     ./setup.py test --addopts 'tests/frontend/buildtrack.py::test_build_track'
+
 
 Adding tests
 ~~~~~~~~~~~~

MANIFEST.in

@@ -24,6 +24,7 @@ recursive-include doc/sessions *.run
 # Tests
 recursive-include tests *
 include conftest.py
+include tox.ini
 include .coveragerc
 include .pylintrc
 

tox.ini

+[tox]
+envlist = py35,py36,py37
+skip_missing_interpreters = true
+
+[testenv]
+commands = pytest {posargs}
+deps =
+    -rrequirements.txt
+    -rdev-requirements.txt
+    -rplugin-requirements.txt
+passenv =
+    GI_TYPELIB_PATH