Commit 4bcd3cc5 authored by Valentin David's avatar Valentin David Committed by Tiago Gomes
Browse files

Pass targets to Loader.load() instead of its constructor. 

This is required to be able to build a Loader before the list
of targets is known.
parent 70e5d36d

buildstream/_loader/loader.py

+16 −17
Original line number Diff line number Diff line
@@ -46,7 +46,6 @@ from . import MetaSource 
# Args:

#    context (Context): The Context object

#    project (Project): The toplevel Project object

#    filenames (list of str): Target, element-path relative bst filenames in the project

#    parent (Loader): A parent Loader object, in the case this is a junctioned Loader

#    tempdir (str): A directory to cleanup with the Loader, given to the loader by a parent

#                   loader in the case that this loader is a subproject loader.
@@ -54,22 +53,13 @@ from . import MetaSource 
#

class Loader():



    def __init__(self, context, project, filenames, *, parent=None, tempdir=None, fetch_subprojects=False):

    def __init__(self, context, project, *, parent=None, tempdir=None, fetch_subprojects=False):



        # Ensure we have an absolute path for the base directory

        basedir = project.element_path

        if not os.path.isabs(basedir):

            basedir = os.path.abspath(basedir)



        for filename in filenames:

            if os.path.isabs(filename):

                # XXX Should this just be an assertion ?

                # Expect that the caller gives us the right thing at least ?

                raise LoadError(LoadErrorReason.INVALID_DATA,

                                "Target '{}' was not specified as a relative "

                                "path to the base project directory: {}"

                                .format(filename, basedir))



        #

        # Public members

        #
@@ -82,7 +72,6 @@ class Loader(): 
        self._context = context

        self._options = project.options      # Project options (OptionPool)

        self._basedir = basedir              # Base project directory

        self._targets = filenames            # Target bst elements

        self._tempdir = tempdir              # A directory to cleanup

        self._parent = parent                # The parent loader
@@ -98,17 +87,27 @@ class Loader(): 
    #    rewritable (bool): Whether the loaded files should be rewritable

    #                       this is a bit more expensive due to deep copies

    #    ticker (callable): An optional function for tracking load progress

    #    targets (list of str): Target, element-path relative bst filenames in the project

    #

    # Raises: LoadError

    #

    # Returns: The toplevel LoadElement

    def load(self, rewritable=False, ticker=None):

    def load(self, targets, rewritable=False, ticker=None):



        for filename in targets:

            if os.path.isabs(filename):

                # XXX Should this just be an assertion ?

                # Expect that the caller gives us the right thing at least ?

                raise LoadError(LoadErrorReason.INVALID_DATA,

                                "Target '{}' was not specified as a relative "

                                "path to the base project directory: {}"

                                .format(filename, self._basedir))



        # First pass, recursively load files and populate our table of LoadElements

        #

        deps = []



        for target in self._targets:

        for target in targets:

            profile_start(Topics.LOAD_PROJECT, target)

            junction, name, loader = self._parse_name(target, rewritable, ticker)

            loader._load_file(name, rewritable, ticker)
@@ -126,7 +125,7 @@ class Loader(): 
        dummy = DummyTarget(name='', full_name='', deps=deps)

        self._elements[''] = dummy



        profile_key = "_".join(t for t in self._targets)

        profile_key = "_".join(t for t in targets)

        profile_start(Topics.CIRCULAR_CHECK, profile_key)

        self._check_circular_deps('')

        profile_end(Topics.CIRCULAR_CHECK, profile_key)
@@ -135,7 +134,7 @@ class Loader(): 
        #

        # Sort direct dependencies of elements by their dependency ordering

        #

        for target in self._targets:

        for target in targets:

            profile_start(Topics.SORT_DEPENDENCIES, target)

            junction, name, loader = self._parse_name(target, rewritable, ticker)

            loader._sort_dependencies(name)
@@ -546,7 +545,7 @@ class Loader(): 
            else:

                raise



        loader = Loader(self._context, project, [],

        loader = Loader(self._context, project,

                        parent=self,

                        tempdir=basedir,

                        fetch_subprojects=self._fetch_subprojects)

buildstream/_pipeline.py

+2 −2
Original line number Diff line number Diff line
@@ -109,11 +109,11 @@ class Pipeline(): 


        profile_start(Topics.LOAD_PIPELINE, "_".join(t.replace(os.sep, '-') for t in targets))



        self._loader = Loader(self._context, self._project, targets,

        self._loader = Loader(self._context, self._project,

                              fetch_subprojects=fetch_subprojects)



        with self._context.timed_activity("Loading pipeline", silent_nested=True):

            meta_elements = self._loader.load(rewritable, None)

            meta_elements = self._loader.load(targets, rewritable, None)



        # Resolve the real elements now that we've loaded the project

        with self._context.timed_activity("Resolving pipeline"):

tests/loader/__init__.py

+2 −2
Original line number Diff line number Diff line
@@ -8,7 +8,7 @@ from buildstream._loader import Loader 
# be removed in favor of testing the functionality via

# the CLI like in the frontend tests anyway.

#

def make_loader(basedir, targets):

def make_loader(basedir):

    context = Context()

    project = Project(basedir, context)

    return Loader(context, project, targets)
 
    return Loader(context, project)

tests/loader/basics.py

+14 −13
Original line number Diff line number Diff line
@@ -18,9 +18,9 @@ DATA_DIR = os.path.join( 
def test_one_file(datafiles):



    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/onefile.bst'])

    loader = make_loader(basedir)



    element = loader.load()[0]

    element = loader.load(['elements/onefile.bst'])[0]



    assert(isinstance(element, MetaElement))

    assert(element.kind == 'pony')
@@ -30,10 +30,10 @@ def test_one_file(datafiles): 
def test_missing_file(datafiles):



    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/missing.bst'])

    loader = make_loader(basedir)



    with pytest.raises(LoadError) as exc:

        element = loader.load()[0]

        element = loader.load(['elements/missing.bst'])[0]



    assert (exc.value.reason == LoadErrorReason.MISSING_FILE)
@@ -42,10 +42,10 @@ def test_missing_file(datafiles): 
def test_invalid_reference(datafiles):



    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/badreference.bst'])

    loader = make_loader(basedir)



    with pytest.raises(LoadError) as exc:

        element = loader.load()[0]

        element = loader.load(['elements/badreference.bst'])[0]



    assert (exc.value.reason == LoadErrorReason.INVALID_YAML)
@@ -54,10 +54,10 @@ def test_invalid_reference(datafiles): 
def test_invalid_yaml(datafiles):



    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/badfile.bst'])

    loader = make_loader(basedir)



    with pytest.raises(LoadError) as exc:

        element = loader.load()[0]

        element = loader.load(['elements/badfile.bst'])[0]



    assert (exc.value.reason == LoadErrorReason.INVALID_YAML)
@@ -69,7 +69,8 @@ def test_fail_fullpath_target(datafiles): 
    fullpath = os.path.join(basedir, 'elements', 'onefile.bst')



    with pytest.raises(LoadError) as exc:

        loader = make_loader(basedir, [fullpath])

        loader = make_loader(basedir)

        loader.load([fullpath])



    assert (exc.value.reason == LoadErrorReason.INVALID_DATA)
@@ -78,10 +79,10 @@ def test_fail_fullpath_target(datafiles): 
def test_invalid_key(datafiles):



    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/invalidkey.bst'])

    loader = make_loader(basedir)



    with pytest.raises(LoadError) as exc:

        element = loader.load()[0]

        element = loader.load(['elements/invalidkey.bst'])[0]



    assert (exc.value.reason == LoadErrorReason.INVALID_DATA)
@@ -90,9 +91,9 @@ def test_invalid_key(datafiles): 
def test_invalid_directory_load(datafiles):



    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/'])

    loader = make_loader(basedir)



    with pytest.raises(LoadError) as exc:

        element = loader.load()[0]

        element = loader.load(['elements/'])[0]



    assert (exc.value.reason == LoadErrorReason.LOADING_DIRECTORY)

tests/loader/dependencies.py

+20 −20
Original line number Diff line number Diff line
@@ -18,8 +18,8 @@ DATA_DIR = os.path.join( 
def test_two_files(datafiles):



    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/target.bst'])

    element = loader.load()[0]

    loader = make_loader(basedir)

    element = loader.load(['elements/target.bst'])[0]



    assert(isinstance(element, MetaElement))

    assert(element.kind == 'pony')
@@ -34,8 +34,8 @@ def test_two_files(datafiles): 
def test_shared_dependency(datafiles):



    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/shareddeptarget.bst'])

    element = loader.load()[0]

    loader = make_loader(basedir)

    element = loader.load(['elements/shareddeptarget.bst'])[0]



    # Toplevel is 'pony' with 2 dependencies

    #
@@ -77,8 +77,8 @@ def test_shared_dependency(datafiles): 
def test_dependency_dict(datafiles):



    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/target-depdict.bst'])

    element = loader.load()[0]

    loader = make_loader(basedir)

    element = loader.load(['elements/target-depdict.bst'])[0]



    assert(isinstance(element, MetaElement))

    assert(element.kind == 'pony')
@@ -92,10 +92,10 @@ def test_dependency_dict(datafiles): 
@pytest.mark.datafiles(DATA_DIR)

def test_invalid_dependency_declaration(datafiles):

    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/invaliddep.bst'])

    loader = make_loader(basedir)



    with pytest.raises(LoadError) as exc:

        element = loader.load()[0]

        element = loader.load(['elements/invaliddep.bst'])[0]



    assert (exc.value.reason == LoadErrorReason.INVALID_DATA)
@@ -103,10 +103,10 @@ def test_invalid_dependency_declaration(datafiles): 
@pytest.mark.datafiles(DATA_DIR)

def test_circular_dependency(datafiles):

    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/circulartarget.bst'])

    loader = make_loader(basedir)



    with pytest.raises(LoadError) as exc:

        element = loader.load()[0]

        element = loader.load(['elements/circulartarget.bst'])[0]



    assert (exc.value.reason == LoadErrorReason.CIRCULAR_DEPENDENCY)
@@ -114,10 +114,10 @@ def test_circular_dependency(datafiles): 
@pytest.mark.datafiles(DATA_DIR)

def test_invalid_dependency_type(datafiles):

    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/invaliddeptype.bst'])

    loader = make_loader(basedir)



    with pytest.raises(LoadError) as exc:

        element = loader.load()[0]

        element = loader.load(['elements/invaliddeptype.bst'])[0]



    assert (exc.value.reason == LoadErrorReason.INVALID_DATA)
@@ -125,8 +125,8 @@ def test_invalid_dependency_type(datafiles): 
@pytest.mark.datafiles(DATA_DIR)

def test_build_dependency(datafiles):

    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/builddep.bst'])

    element = loader.load()[0]

    loader = make_loader(basedir)

    element = loader.load(['elements/builddep.bst'])[0]



    assert(isinstance(element, MetaElement))

    assert(element.kind == 'pony')
@@ -141,8 +141,8 @@ def test_build_dependency(datafiles): 
@pytest.mark.datafiles(DATA_DIR)

def test_runtime_dependency(datafiles):

    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/runtimedep.bst'])

    element = loader.load()[0]

    loader = make_loader(basedir)

    element = loader.load(['elements/runtimedep.bst'])[0]



    assert(isinstance(element, MetaElement))

    assert(element.kind == 'pony')
@@ -157,8 +157,8 @@ def test_runtime_dependency(datafiles): 
@pytest.mark.datafiles(DATA_DIR)

def test_build_runtime_dependency(datafiles):

    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/target.bst'])

    element = loader.load()[0]

    loader = make_loader(basedir)

    element = loader.load(['elements/target.bst'])[0]



    assert(isinstance(element, MetaElement))

    assert(element.kind == 'pony')
@@ -174,8 +174,8 @@ def test_build_runtime_dependency(datafiles): 
@pytest.mark.datafiles(DATA_DIR)

def test_all_dependency(datafiles):

    basedir = os.path.join(datafiles.dirname, datafiles.basename)

    loader = make_loader(basedir, ['elements/alldep.bst'])

    element = loader.load()[0]

    loader = make_loader(basedir)

    element = loader.load(['elements/alldep.bst'])[0]



    assert(isinstance(element, MetaElement))

    assert(element.kind == 'pony')