Commit acde3ba8 authored by Valentin David's avatar Valentin David Committed by Tristan Van Berkom

Allow tracking dependencies within sub-projects.

--track-cross-junctions now concerns crossing junctions rather than
forbidding elements in sub-project to be tracked.

Part of #359.
parent 130bfbb8
Pipeline #23527384 passed with stages
in 29 minutes and 34 seconds
......@@ -346,6 +346,8 @@ class Pipeline():
# lists targetted at tracking.
# Args:
# project (Project): Project used for cross_junction filtering.
# All elements are expected to belong to that project.
# elements (list of Element): The list of elements to filter
# cross_junction_requested (bool): Whether the user requested
# cross junction tracking
......@@ -353,12 +355,11 @@ class Pipeline():
# Returns:
# (list of Element): The filtered or asserted result
def track_cross_junction_filter(self, elements, cross_junction_requested):
def track_cross_junction_filter(self, project, elements, cross_junction_requested):
# Filter out cross junctioned elements
if cross_junction_requested:
elements = self._filter_cross_junctions(elements)
if not cross_junction_requested:
elements = self._filter_cross_junctions(project, elements)
return elements
......@@ -403,16 +404,17 @@ class Pipeline():
# Filters out cross junction elements from the elements
# Args:
# project (Project): The project on which elements are allowed
# elements (list of Element): The list of elements to be tracked
# Returns:
# (list): A filtered list of `elements` which does
# not contain any cross junction elements.
def _filter_cross_junctions(self, elements):
def _filter_cross_junctions(self, project, elements):
return [
element for element in elements
if element._get_project() is self._project
if element._get_project() is project
# _assert_junction_tracking()
......@@ -831,12 +831,30 @@ class Stream():
# done before resolving element states.
assert track_selection != PipelineSelection.PLAN
track_selected = self._pipeline.get_selection(track_elements, track_selection)
# Tracked elements are split by owner projects in order to
# filter cross junctions tracking dependencies on their
# respective project.
track_projects = {}
for element in track_elements:
project = element._get_project()
if project not in track_projects:
track_projects[project] = [element]
track_selected = []
for project, project_elements in track_projects.items():
selected = self._pipeline.get_selection(project_elements, track_selection)
selected = self._pipeline.track_cross_junction_filter(project,
track_selected = self._pipeline.except_elements(track_elements,
track_selected = self._pipeline.track_cross_junction_filter(track_selected,
for element in track_selected:
......@@ -437,3 +437,46 @@ def test_junction_element(cli, tmpdir, datafiles, ref_storage):
# Now assert element state (via bst show under the hood) of the dep again
assert cli.get_element_state(project, 'junction-dep.bst') == 'waiting'
@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
def test_cross_junction(cli, tmpdir, datafiles, ref_storage, kind):
project = os.path.join(datafiles.dirname, datafiles.basename)
subproject_path = os.path.join(project, 'files', 'sub-project')
junction_path = os.path.join(project, 'elements', 'junction.bst')
etc_files = os.path.join(subproject_path, 'files', 'etc-files')
repo_element_path = os.path.join(subproject_path, 'elements',
configure_project(project, {
'ref-storage': ref_storage
repo = create_repo(kind, str(tmpdir.join('element_repo')))
ref = repo.create(etc_files)
generate_element(repo, repo_element_path)
subproject_path, junction_path, store_ref=False)
# Track the junction itself first.
result =, args=['track', 'junction.bst'])
assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'no reference'
# Track the cross junction element. -J is not given, it is implied.
result =, args=['track', 'junction.bst:import-etc-repo.bst'])
if ref_storage == 'inline':
# This is not allowed to track cross junction without project.refs.
result.assert_main_error(ErrorDomain.PIPELINE, 'untrackable-sources')
assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'buildable'
assert os.path.exists(os.path.join(project, 'project.refs'))
import os
import pytest
from tests.testutils import cli, create_repo, ALL_REPO_KINDS
from buildstream import _yaml
from . import generate_junction
def generate_element(repo, element_path, dep_name=None):
element = {
'kind': 'import',
'sources': [
if dep_name:
element['depends'] = [dep_name]
_yaml.dump(element, element_path)
def generate_import_element(tmpdir, kind, project, name):
element_name = 'import-{}.bst'.format(name)
repo_element_path = os.path.join(project, 'elements', element_name)
files = str(tmpdir.join("imported_files_{}".format(name)))
with open(os.path.join(files, '{}.txt'.format(name)), 'w') as f:
subproject_path = os.path.join(str(tmpdir.join('sub-project-{}'.format(name))))
repo = create_repo(kind, str(tmpdir.join('element_{}_repo'.format(name))))
ref = repo.create(files)
generate_element(repo, repo_element_path)
return element_name
def generate_project(tmpdir, name, config={}):
project_name = 'project-{}'.format(name)
subproject_path = os.path.join(str(tmpdir.join(project_name)))
os.makedirs(os.path.join(subproject_path, 'elements'))
project_conf = {
'name': name,
'element-path': 'elements'
_yaml.dump(project_conf, os.path.join(subproject_path, 'project.conf'))
return project_name, subproject_path
def generate_simple_stack(project, name, dependencies):
element_name = '{}.bst'.format(name)
element_path = os.path.join(project, 'elements', element_name)
element = {
'kind': 'stack',
'depends': dependencies
_yaml.dump(element, element_path)
return element_name
def generate_cross_element(project, subproject_name, import_name):
basename, _ = os.path.splitext(import_name)
return generate_simple_stack(project, 'import-{}-{}'.format(subproject_name, basename),
'junction': '{}.bst'.format(subproject_name),
'filename': import_name
@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
def test_cross_junction_multiple_projects(cli, tmpdir, datafiles, kind):
tmpdir = tmpdir.join(kind)
# Generate 3 projects: main, a, b
_, project = generate_project(tmpdir, 'main', {'ref-storage': 'project.refs'})
project_a, project_a_path = generate_project(tmpdir, 'a')
project_b, project_b_path = generate_project(tmpdir, 'b')
# Generate an element with a trackable source for each project
element_a = generate_import_element(tmpdir, kind, project_a_path, 'a')
element_b = generate_import_element(tmpdir, kind, project_b_path, 'b')
element_c = generate_import_element(tmpdir, kind, project, 'c')
# Create some indirections to the elements with dependencies to test --deps
stack_a = generate_simple_stack(project_a_path, 'stack-a', [element_a])
stack_b = generate_simple_stack(project_b_path, 'stack-b', [element_b])
# Create junctions for projects a and b in main.
junction_a = '{}.bst'.format(project_a)
junction_a_path = os.path.join(project, 'elements', junction_a)
generate_junction(tmpdir.join('repo_a'), project_a_path, junction_a_path, store_ref=False)
junction_b = '{}.bst'.format(project_b)
junction_b_path = os.path.join(project, 'elements', junction_b)
generate_junction(tmpdir.join('repo_b'), project_b_path, junction_b_path, store_ref=False)
# Track the junctions.
result =, args=['track', junction_a, junction_b])
# Import elements from a and b in to main.
imported_a = generate_cross_element(project, project_a, stack_a)
imported_b = generate_cross_element(project, project_b, stack_b)
# Generate a top level stack depending on everything
all_bst = generate_simple_stack(project, 'all', [imported_a, imported_b, element_c])
# Track without following junctions. But explicitly also track the elements in project a.
result =, args=['track', '--deps', 'all', all_bst, '{}:{}'.format(junction_a, stack_a)])
# Elements in project b should not be tracked. But elements in project a and main should.
expected = [element_c,
'{}:{}'.format(junction_a, element_a)]
assert set(result.get_tracked_elements()) == set(expected)
@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
def test_track_exceptions(cli, tmpdir, datafiles, kind):
tmpdir = tmpdir.join(kind)
_, project = generate_project(tmpdir, 'main', {'ref-storage': 'project.refs'})
project_a, project_a_path = generate_project(tmpdir, 'a')
element_a = generate_import_element(tmpdir, kind, project_a_path, 'a')
element_b = generate_import_element(tmpdir, kind, project_a_path, 'b')
all_bst = generate_simple_stack(project_a_path, 'all', [element_a,
junction_a = '{}.bst'.format(project_a)
junction_a_path = os.path.join(project, 'elements', junction_a)
generate_junction(tmpdir.join('repo_a'), project_a_path, junction_a_path, store_ref=False)
result =, args=['track', junction_a])
imported_b = generate_cross_element(project, project_a, element_b)
indirection = generate_simple_stack(project, 'indirection', [imported_b])
result =,
args=['track', '--deps', 'all',
'--except', indirection,
'{}:{}'.format(junction_a, all_bst), imported_b])
expected = ['{}:{}'.format(junction_a, element_a),
'{}:{}'.format(junction_a, element_b)]
assert set(result.get_tracked_elements()) == set(expected)
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment