.gitlab-ci.yml

@@ -276,7 +276,7 @@ coverage:
   coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
   script:
     - cd dist && ./unpack.sh && cd buildstream
-    - pip3 install -r tools/requirements.txt -r tools/dev-requirements.txt
+    - pip3 install -r requirements/requirements.txt -r requirements/dev-requirements.txt
     - pip3 install --no-index .
     - mkdir report
     - cd report

CONTRIBUTING.rst

@@ -1736,10 +1736,8 @@ obtain profiles::
       ForceCommand BST_PROFILE=artifact-receive cd /tmp && bst-artifact-receive --pull-url https://example.com/ /home/artifacts/artifacts
 
 
-The MANIFEST.in and setup.py
-----------------------------
-When adding a dependency to BuildStream, it's important to update the setup.py accordingly.
-
+Managing data files
+-------------------
 When adding data files which need to be discovered at runtime by BuildStream, update setup.py accordingly.
 
 When adding data files for the purpose of docs or tests, or anything that is not covered by
@@ -1749,3 +1747,23 @@ At any time, running the following command to create a source distribution shoul
 creating a tarball which contains everything we want it to include::
 
   ./setup.py sdist
+
+
+Updating BuildStream's Python dependencies
+------------------------------------------
+BuildStream's Python dependencies are listed in multiple
+`requirements files <https://pip.readthedocs.io/en/latest/reference/pip_install/#requirements-file-format>`
+present in the ``requirements`` directory.
+
+All ``.txt`` files in this directory are generated from the corresponding
+``.in`` file, and each ``.in`` file represents a set of dependencies. For
+example, ``requirements.in`` contains all runtime dependencies of BuildStream.
+``requirements.txt`` is generated from it, and contains pinned versions of all
+runtime dependencies (including transitive dependencies) of BuildStream.
+
+When adding a new dependency to BuildStream, or updating existing dependencies,
+it is important to update the appropriate requirements file accordingly. After
+changing the ``.in`` file, run the following to update the matching ``.txt``
+file::
+
+   make -C requirements

MANIFEST.in

@@ -32,12 +32,12 @@ include .pylintrc
 recursive-include buildstream/_protos *.proto
 
 # Requirements files
-include tools/requirements.in
-include tools/requirements.txt
-include tools/dev-requirements.in
-include tools/dev-requirements.txt
-include tools/plugin-requirements.in
-include tools/plugin-requirements.txt
+include requirements/requirements.in
+include requirements/requirements.txt
+include requirements/dev-requirements.in
+include requirements/dev-requirements.txt
+include requirements/plugin-requirements.in
+include requirements/plugin-requirements.txt
 
 # Versioneer
 include versioneer.py

requirements/Makefile

+# Makefile for updating BuildStream's requirements files.
+#
+
+REQUIREMENTS_IN := $(wildcard *.in)
+REQUIREMENTS_TXT := $(REQUIREMENTS_IN:.in=.txt)
+PYTHON := python3
+VENV := $(PYTHON) -m venv
+
+VENV_PIP = $(VENVDIR)/bin/pip
+
+
+.PHONY: all
+
+all: $(REQUIREMENTS_TXT)
+
+%.txt: %.in
+	$(eval VENVDIR := $(shell mktemp -d $(CURDIR)/.bst-venv.XXXXXX))
+	$(VENV) $(VENVDIR)
+	$(VENV_PIP) install -r $^
+	$(VENV_PIP) freeze -r $^ > $@
+	rm -rf $(VENVDIR)

setup.py

@@ -270,10 +270,10 @@ def get_cmdclass():
 #####################################################
 #               Gather requirements                 #
 #####################################################
-with open('tools/dev-requirements.in') as dev_reqs:
+with open('requirements/dev-requirements.in') as dev_reqs:
     dev_requires = dev_reqs.read().splitlines()
 
-with open('tools/requirements.in') as install_reqs:
+with open('requirements/requirements.in') as install_reqs:
     install_requires = install_reqs.read().splitlines()
 
 #####################################################

tox.ini

@@ -5,9 +5,9 @@ skip_missing_interpreters = true
 [testenv]
 commands = pytest {posargs}
 deps =
-    -rtools/requirements.txt
-    -rtools/dev-requirements.txt
-    -rtools/plugin-requirements.txt
+    -rrequirements/requirements.txt
+    -rrequirements/dev-requirements.txt
+    -rrequirements/plugin-requirements.txt
 passenv =
     BST_FORCE_BACKEND
     GI_TYPELIB_PATH
@@ -18,9 +18,9 @@ commands =
     pycodestyle
     pylint buildstream
 deps =
-    -rtools/requirements.txt
-    -rtools/dev-requirements.txt
-    -rtools/plugin-requirements.txt
+    -rrequirements/requirements.txt
+    -rrequirements/dev-requirements.txt
+    -rrequirements/plugin-requirements.txt
 
 [testenv:docs]
 commands =
@@ -30,8 +30,8 @@ deps =
     sphinx==1.7.9
     sphinx-click
     sphinx_rtd_theme
-    -rtools/requirements.txt
-    -rtools/plugin-requirements.txt
+    -rrequirements/requirements.txt
+    -rrequirements/plugin-requirements.txt
 passenv =
     BST_FORCE_SESSION_REBUILD
     BST_SOURCE_CACHE