.coveragerc

@@ -4,11 +4,15 @@ include =
   */buildstream/*
 
 omit =
-  # Omit profiling helper module
+  # Omit some internals
   */buildstream/_profile.py
+  */buildstream/__main__.py
+  */buildstream/_version.py
   # Omit generated code
   */buildstream/_protos/*
   */.eggs/*
+  # Omit .tox directory
+  */.tox/*
 
 [report]
 show_missing = True

.gitignore

@@ -13,9 +13,12 @@ tests/**/*.pyc
 integration-cache/
 tmp
 .coverage
+.coverage-reports/
 .coverage.*
 .cache
+.pytest_cache/
 *.bst/
+.tox/
 
 # Pycache, in case buildstream is ran directly from within the source
 # tree

.gitlab-ci.yml

-image: buildstream/testsuite-debian:9-master-114-4cab18e3
+image: buildstream/testsuite-debian:9-5da27168-32c47d1c
 
 cache:
   key: "$CI_JOB_NAME-"
@@ -6,49 +6,13 @@ cache:
     - cache/
 
 stages:
-  - prepare
   - test
   - post
 
-#####################################################
-#                  Prepare stage                    #
-#####################################################
-
-# Create a source distribution
-#
-source_dist:
-  stage: prepare
-  script:
-
-  # Generate the source distribution tarball
-  #
-  - python3 setup.py sdist
-  - tar -ztf dist/*
-  - tarball=$(cd dist && echo $(ls *))
-
-  # Create an installer script
-  - |
-    cat > dist/install.sh << EOF
-    #!/bin/sh
-    tar -zxf ${tarball}
-    cd ${tarball%.tar.gz}
-    pip3 install --no-index .
-    EOF
-
-  # unpack tarball as `dist/buildstream` directory
-  - |
-    cat > dist/unpack.sh << EOF
-    #!/bin/sh
-    tar -zxf ${tarball}
-    mv ${tarball%.tar.gz} buildstream
-    EOF
-
-  # Make our helpers executable
-  - chmod +x dist/install.sh
-  - chmod +x dist/unpack.sh
-  artifacts:
-    paths:
-    - dist/
+variables:
+  INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache"
+  TEST_COMMAND: "tox -- --color=yes --integration"
+  COVERAGE_PREFIX: "${CI_JOB_NAME}."
 
 
 #####################################################
@@ -57,66 +21,53 @@ source_dist:
 
 # Run premerge commits
 #
-.linux-tests-template: &linux-tests
+.tests-template: &tests
   stage: test
-  variables:
-    PYTEST_ADDOPTS: "--color=yes"
-  script:
+
+  before_script:
   # Diagnostics
   - mount
   - df -h
 
+  script:
+  - mkdir -p "${INTEGRATION_CACHE}"
   - useradd -Um buildstream
   - chown -R buildstream:buildstream .
 
-  - export INTEGRATION_CACHE="$(pwd)/cache/integration-cache"
-
-  # Unpack and get into dist/buildstream
-  - cd dist && ./unpack.sh
-  - chown -R buildstream:buildstream buildstream
-  - cd buildstream
-
-  # Run the tests from the source distribution, We run as a simple
-  # user to test for permission issues
-  - su buildstream -c 'python3 setup.py test --index-url invalid://uri --addopts --integration'
+  # Run the tests as a simple user to test for permission issues
+  - su buildstream -c "${TEST_COMMAND}"
 
-  # Go back to the toplevel and collect our reports
-  - cd ../..
-  - mkdir -p coverage-linux/
-  - cp dist/buildstream/.coverage.* coverage-linux/coverage."${CI_JOB_NAME}"
   except:
   - schedules
   artifacts:
     paths:
-    - coverage-linux/
+    - .coverage-reports
 
 tests-debian-9:
-  image: buildstream/testsuite-debian:9-master-117-aa3a33b3
-  <<: *linux-tests
+  image: buildstream/testsuite-debian:9-5da27168-32c47d1c
+  <<: *tests
 
 tests-fedora-27:
-  image: buildstream/testsuite-fedora:27-master-117-aa3a33b3
-  <<: *linux-tests
+  image: buildstream/testsuite-fedora:27-5da27168-32c47d1c
+  <<: *tests
 
 tests-fedora-28:
-  image: buildstream/testsuite-fedora:28-master-117-aa3a33b3
-  <<: *linux-tests
+  image: buildstream/testsuite-fedora:28-5da27168-32c47d1c
+  <<: *tests
 
 tests-ubuntu-18.04:
-  image: buildstream/testsuite-ubuntu:18.04-master-117-aa3a33b3
-  <<: *linux-tests
+  image: buildstream/testsuite-ubuntu:18.04-5da27168-32c47d1c
+  <<: *tests
 
 tests-unix:
   # Use fedora here, to a) run a test on fedora and b) ensure that we
   # can get rid of ostree - this is not possible with debian-8
-  image: buildstream/testsuite-fedora:27-master-117-aa3a33b3
-  stage: test
+  image: buildstream/testsuite-fedora:27-5da27168-32c47d1c
+  <<: *tests
   variables:
     BST_FORCE_BACKEND: "unix"
-    PYTEST_ADDOPTS: "--color=yes"
-  script:
 
-    - export INTEGRATION_CACHE="$(pwd)/cache/integration-cache"
+  script:
 
     # We remove the Bubblewrap and OSTree packages here so that we catch any
     # codepaths that try to use them. Removing OSTree causes fuse-libs to
@@ -124,39 +75,19 @@ tests-unix:
     - dnf mark install fuse-libs
     - dnf erase -y bubblewrap ostree
 
-    # Unpack and get into dist/buildstream
-    - cd dist && ./unpack.sh && cd buildstream
-
     # Since the unix platform is required to run as root, no user change required
-    - python3 setup.py test --index-url invalid://uri --addopts --integration
-
-    # Go back to the toplevel and collect our reports
-    - cd ../..
-    - mkdir -p coverage-unix/
-    - cp dist/buildstream/.coverage.* coverage-unix/coverage.unix
-  except:
-  - schedules
-  artifacts:
-    paths:
-    - coverage-unix/
-    - logs-unix/
+    - ${TEST_COMMAND}
 
 
 # Automatically build documentation for every commit, we want to know
 # if building documentation fails even if we're not deploying it.
-# Note: We still do not enforce a consistent installation of python3-sphinx,
-#       as it will significantly grow the backing image.
 docs:
   stage: test
+  variables:
+    BST_FORCE_SESSION_REBUILD: 1
   script:
-  - export BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources"
-  - pip3 install sphinx
-  - pip3 install sphinx-click
-  - pip3 install sphinx_rtd_theme
-  - cd dist && ./unpack.sh && cd buildstream
-  - make BST_FORCE_SESSION_REBUILD=1 -C doc
-  - cd ../..
-  - mv dist/buildstream/doc/build/html public
+  - env BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources" tox -e docs
+  - mv doc/build/html public
   except:
   - schedules
   artifacts:
@@ -170,7 +101,8 @@ docs:
     bst_ext_ref: 1d6ab71151b93c8cbc0a91a36ffe9270f3b835f1 # 0.5.1
     fd_sdk_ref: 88d7c22c2281b987faa02edd57df80d430eecf1f # 18.08.12
   before_script:
-  - (cd dist && ./unpack.sh && cd buildstream && pip3 install .)
+  - pip3 install -r requirements/requirements.txt -r requirements/plugin-requirements.txt
+  - pip3 install --no-index .
   - pip3 install --user -e ${bst_ext_url}@${bst_ext_ref}#egg=bst_ext
   - git clone https://gitlab.com/freedesktop-sdk/freedesktop-sdk.git
   - git -C freedesktop-sdk checkout ${fd_sdk_ref}
@@ -253,21 +185,15 @@ coverage:
   stage: post
   coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
   script:
-    - cd dist && ./unpack.sh && cd buildstream
-    - pip3 install --no-index .
-    - mkdir report
-    - cd report
-    - cp ../../../coverage-unix/coverage.unix .
-    - cp ../../../coverage-linux/coverage.* .
-    - ls coverage.*
-    - coverage combine --rcfile=../.coveragerc -a coverage.*
-    - coverage report --rcfile=../.coveragerc -m
+    - cp -a .coverage-reports/ ./coverage-sources
+    - tox -e coverage
+    - cp -a .coverage-reports/ ./coverage-report
   dependencies:
   - tests-debian-9
   - tests-fedora-27
   - tests-fedora-28
+  - tests-ubuntu-18.04
   - tests-unix
-  - source_dist
   except:
   - schedules
 
@@ -276,7 +202,6 @@ coverage:
 pages:
   stage: post
   dependencies:
-  - source_dist
   - docs
   script:
   - find public/

MANIFEST.in

@@ -23,4 +23,12 @@ recursive-include tests *.expected
 recursive-include buildstream/_protos *.proto
 
 # Requirements files
-include dev-requirements.txt
+include requirements/requirements.in
+include requirements/requirements.txt
+include requirements/dev-requirements.in
+include requirements/dev-requirements.txt
+include requirements/plugin-requirements.in
+include requirements/plugin-requirements.txt
+
+# Versioneer
+include versioneer.py

Makefile

+# Makefile for updating BuildStream's requirements files.
+#
+
+REQUIREMENTS_IN := $(wildcard *.in)
+REQUIREMENTS_TXT := $(REQUIREMENTS_IN:.in=.txt)
+PYTHON := python3
+VENV := $(PYTHON) -m venv
+
+VENV_PIP = $(VENVDIR)/bin/pip
+
+
+.PHONY: all
+
+all: $(REQUIREMENTS_TXT)
+
+%.txt: %.in
+	$(eval VENVDIR := $(shell mktemp -d $(CURDIR)/.bst-venv.XXXXXX))
+	$(VENV) $(VENVDIR)
+	$(VENV_PIP) install -r $^
+	$(VENV_PIP) freeze -r $^ > $@
+	rm -rf $(VENVDIR)

buildstream/_artifactcache/cascache.py

@@ -854,6 +854,7 @@ class CASCache(ArtifactCache):
         with tempfile.NamedTemporaryFile(dir=self.tmpdir) as f:
             self._fetch_blob(remote, digest, f)
 
+            os.chmod(f.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
             added_digest = self.add_object(path=f.name, link_directly=True)
             assert added_digest.hash == digest.hash
 

buildstream/plugins/elements/filter.py

@@ -47,7 +47,9 @@ from buildstream import Element, ElementError, Scope
 class FilterElement(Element):
     # pylint: disable=attribute-defined-outside-init
 
-    # The filter element's output is it's dependencies, so
+    BST_ARTIFACT_VERSION = 1
+
+    # The filter element's output is its dependencies, so
     # we must rebuild if the dependencies change even when
     # not in strict build plans.
     BST_STRICT_REBUILD = True
@@ -102,7 +104,7 @@ class FilterElement(Element):
 
     def assemble(self, sandbox):
         with self.timed_activity("Staging artifact", silent_nested=True):
-            for dep in self.dependencies(Scope.BUILD):
+            for dep in self.dependencies(Scope.BUILD, recurse=False):
                 dep.stage_artifact(sandbox, include=self.include,
                                    exclude=self.exclude, orphans=self.include_orphans)
         return ""

conftest.py

@@ -30,7 +30,7 @@ def pytest_addoption(parser):
 
 
 def pytest_runtest_setup(item):
-    if item.get_marker('integration') and not item.config.getvalue('integration'):
+    if item.get_closest_marker('integration') and not item.config.getvalue('integration'):
         pytest.skip('skipping integration test')
 
 

doc/bst2html.py

@@ -440,7 +440,7 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
 @click.option('--palette', '-p', default='tango',
               type=click.Choice(['solarized', 'solarized-xterm', 'tango', 'xterm', 'console']),
               help="Selects a palette for the output style")
-@click.argument('description', click.Path(file_okay=True, dir_okay=False, readable=True))
+@click.argument('description', type=click.Path(file_okay=True, dir_okay=False, readable=True))
 def run_bst(directory, force, source_cache, description, palette):
     """Run a bst command and capture stdout/stderr in html
 

requirements/dev-requirements.txt

+coverage==4.4
+pep8==1.7.1
+pylint==2.1.1
+pytest==4.1.1
+pytest-cov==2.6.1
+pytest-datafiles==2.0
+pytest-env==0.6.2
+pytest-pep8==1.0.6
+pytest-pylint==0.14.0
+pytest-xdist==1.26.0
+pytest-timeout==1.3.3
+## The following requirements were added by pip freeze:
+apipkg==1.5
+astroid==2.1.0
+atomicwrites==1.2.1
+attrs==18.2.0
+execnet==1.5.0
+isort==4.3.4
+lazy-object-proxy==1.3.1
+mccabe==0.6.1
+more-itertools==5.0.0
+pluggy==0.8.1
+py==1.7.0
+pytest-cache==1.0
+pytest-forked==1.0.1
+six==1.12.0
+wrapt==1.11.0

requirements/plugin-requirements.in

+arpy
+PyGObject

requirements/plugin-requirements.txt

+arpy==1.1.1
+PyGObject==3.30.4
+## The following requirements were added by pip freeze:
+pycairo==1.18.0

requirements/requirements.in

+Click
+grpcio >= 1.10
+jinja2 >= 2.10
+pluginbase
+protobuf >= 3.5
+psutil
+ruamel.yaml < 0.15.52
+setuptools

requirements/requirements.txt

+Click==7.0
+grpcio==1.18.0
+pluginbase==0.7
+protobuf==3.6.1
+psutil==5.4.8
+ruamel.yaml==0.15.51
+setuptools==40.6.2
+## The following requirements were added by pip freeze:
+Jinja2==2.10
+MarkupSafe==1.1.0
+six==1.12.0

setup.py

@@ -228,9 +228,12 @@ def get_cmdclass():
 #####################################################
 #               Gather requirements                 #
 #####################################################
-with open('dev-requirements.txt') as dev_reqs:
+with open('requirements/dev-requirements.in') as dev_reqs:
     dev_requires = dev_reqs.read().splitlines()
 
+with open('requirements/requirements.in') as install_reqs:
+    install_requires = install_reqs.read().splitlines()
+
 #####################################################
 #     Prepare package description from README       #
 #####################################################
@@ -291,16 +294,7 @@ setup(name='BuildStream',
               os.path.join('buildstream', 'data', 'bst')
           ])
       ],
-      install_requires=[
-          'setuptools',
-          'psutil',
-          'ruamel.yaml < 0.15.52',
-          'pluginbase',
-          'Click',
-          'jinja2 >= 2.10',
-          'protobuf >= 3.5',
-          'grpcio >= 1.10',
-      ],
+      install_requires=install_requires,
       entry_points=bst_install_entry_points,
       setup_requires=['pytest-runner'],
       tests_require=dev_requires,

tests/plugins/filter.py

@@ -452,3 +452,23 @@ def test_filter_track_multi_exclude(datafiles, cli, tmpdir):
     assert "ref" not in new_input["sources"][0]
     new_input2 = _yaml.load(input2_file)
     assert new_input2["sources"][0]["ref"] == ref
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+def test_filter_include_with_indirect_deps(datafiles, cli, tmpdir):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    result = cli.run(project=project, args=[
+        'build', 'output-include-with-indirect-deps.bst'])
+    result.assert_success()
+
+    checkout = os.path.join(tmpdir.dirname, tmpdir.basename, 'checkout')
+    result = cli.run(project=project, args=[
+        'checkout', 'output-include-with-indirect-deps.bst', checkout])
+    result.assert_success()
+
+    # direct dependencies should be staged and filtered
+    assert os.path.exists(os.path.join(checkout, "baz"))
+
+    # indirect dependencies shouldn't be staged and filtered
+    assert not os.path.exists(os.path.join(checkout, "foo"))
+    assert not os.path.exists(os.path.join(checkout, "bar"))

tests/plugins/filter/basic/elements/input-with-deps.bst

+kind: import
+
+depends:
+- filename: input.bst
+
+sources:
+- kind: local
+  path: files
+
+public:
+  bst:
+    split-rules:
+      baz:
+      - /baz

tests/plugins/filter/basic/elements/output-include-with-indirect-deps.bst

+kind: filter
+
+depends:
+- filename: input-with-deps.bst
+  type: build

tox.ini

+#
+# Tox global configuration
+#
+[tox]
+envlist = py35,py36,py37
+skip_missing_interpreters = true
+
+#
+# Defaults for all environments
+#
+# Anything specified here is iherited by the sections
+#
+[testenv]
+commands =
+    pytest --basetemp {envtmpdir} {posargs}
+    mkdir -p .coverage-reports
+    mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
+deps =
+    -rrequirements/requirements.txt
+    -rrequirements/dev-requirements.txt
+    -rrequirements/plugin-requirements.txt
+passenv =
+    BST_FORCE_BACKEND
+    GI_TYPELIB_PATH
+    INTEGRATION_CACHE
+
+#
+# These keys are not inherited by any other sections
+#
+setenv =
+    py{35,36,37}: COVERAGE_FILE = {envtmpdir}/.coverage
+whitelist_externals =
+    py{35,36,37}:
+        mv
+        mkdir
+
+#
+# Coverage reporting
+#
+[testenv:coverage]
+commands =
+    - coverage combine --rcfile={toxinidir}/.coveragerc {toxinidir}/.coverage-reports/
+    coverage report --rcfile={toxinidir}/.coveragerc -m
+deps =
+    -rrequirements/requirements.txt
+    -rrequirements/dev-requirements.txt
+setenv =
+    COVERAGE_FILE = {toxinidir}/.coverage-reports/.coverage
+
+#
+# Building documentation
+#
+[testenv:docs]
+commands =
+    make -C doc
+# sphinx_rtd_theme < 0.4.2 breaks search functionality for Sphinx >= 1.8
+deps =
+    sphinx
+    sphinx-click
+    sphinx_rtd_theme >= 0.4.2
+    -rrequirements/requirements.txt
+    -rrequirements/plugin-requirements.txt
+passenv =
+    BST_FORCE_SESSION_REBUILD
+    BST_SOURCE_CACHE
+    HOME
+    LANG
+    LC_ALL
+whitelist_externals =
+    make