Commit d2029444 authored by Tom Pollard's avatar Tom Pollard
Browse files

Provide configuration for the optional creation of buildtrees 

Artifacts can be cached explicitly with an empty `build tree` when
built via the cli main options or user config for all or only
successful build artifacts. Default behaviour is to still create
and cache all expected buildtrees.

element.py: _cache_artifact() Check if context for cache_buildtrees
has been set to always or failure with a corresponding build
result, if not skip attempting to export the build-root. Element
types without a build-root are cached with an empty buildtree
regardless. Update _stage_sources_at() to warn the user that the
buildtree import is empty.

tests/integration: Add test to artifact.py for the optional caching
of buildtree content from bst build. Rename build-tree.py to
shellbuildtrees.py to reflect included test cases, add test for
empty buildtree warning and failure option.

NEWS: Add entry for new option.
parent b546bac1

NEWS

+8 −0
Original line number Original line Diff line number Diff line
@@ -126,6 +126,14 @@ buildstream 1.3.1 
    Providing a remote will limit build's pull/push remote actions to the given

    Providing a remote will limit build's pull/push remote actions to the given

    remote specifically, ignoring those defined via user or project configuration.

    remote specifically, ignoring those defined via user or project configuration.





  o Artifacts can now be cached explicitly with an empty `build tree` when built.

    Element types without a build-root were already cached with an empty build tree

    directory, this can now be extended to all or successful artifacts to save on cache

    overheads. The cli main option '--cache-buildtrees' or the user configuration cache

    group option 'cache-buildtrees' can be set as 'always', 'failure' or 'never', with

    the default being always. Note, as the cache-key for the artifact is independant of

    the cached build tree input it will remain unaltered, however the availbility of the

    build tree content may differ.





=================

=================

buildstream 1.1.5

buildstream 1.1.5

buildstream/element.py

+19 −2
Original line number Original line Diff line number Diff line
@@ -1457,6 +1457,9 @@ class Element(Plugin): 
            elif usebuildtree:

            elif usebuildtree:

                artifact_base, _ = self.__extract()

                artifact_base, _ = self.__extract()

                import_dir = os.path.join(artifact_base, 'buildtree')

                import_dir = os.path.join(artifact_base, 'buildtree')

                if not os.listdir(import_dir):

                    detail = "Element type either does not expect a buildtree or it was explictily cached without one."

                    self.warn("WARNING: {} Artifact contains an empty buildtree".format(self.name), detail=detail)

            else:

            else:

                # No workspace or cached buildtree, stage source directly

                # No workspace or cached buildtree, stage source directly

                for source in self.sources():

                for source in self.sources():
@@ -1663,6 +1666,8 @@ class Element(Plugin): 
                # No collect directory existed

                # No collect directory existed

                collectvdir = None

                collectvdir = None





        context = self._get_context()



        # Create artifact directory structure

        # Create artifact directory structure

        assembledir = os.path.join(rootdir, 'artifact')

        assembledir = os.path.join(rootdir, 'artifact')

        filesdir = os.path.join(assembledir, 'files')

        filesdir = os.path.join(assembledir, 'files')
@@ -1680,6 +1685,16 @@ class Element(Plugin): 
        if collect is not None and collectvdir is not None:

        if collect is not None and collectvdir is not None:

            collectvdir.export_files(filesdir, can_link=True)

            collectvdir.export_files(filesdir, can_link=True)





        cache_buildtrees = context.cache_buildtrees

        build_success = self.__build_result[0]



        # cache_buildtrees defaults to 'always', as such the

        # default behaviour is to attempt to cache them. If only

        # caching failed artifact buildtrees, then query the build

        # result. Element types without a build-root dir will be cached

        # with an empty buildtreedir regardless of this configuration.



        if cache_buildtrees == 'always' or (cache_buildtrees == 'failure' and not build_success):

            try:

            try:

                sandbox_vroot = sandbox.get_virtual_directory()

                sandbox_vroot = sandbox.get_virtual_directory()

                sandbox_build_dir = sandbox_vroot.descend(

                sandbox_build_dir = sandbox_vroot.descend(
@@ -1693,7 +1708,7 @@ class Element(Plugin): 
                pass

                pass





        # Copy build log

        # Copy build log

        log_filename = self._get_context().get_log_filename()

        log_filename = context.get_log_filename()

        self._build_log_path = os.path.join(logsdir, 'build.log')

        self._build_log_path = os.path.join(logsdir, 'build.log')

        if log_filename:

        if log_filename:

            shutil.copyfile(log_filename, self._build_log_path)

            shutil.copyfile(log_filename, self._build_log_path)
@@ -1834,7 +1849,7 @@ class Element(Plugin): 
            return True

            return True





        # Do not push elements that aren't cached, or that are cached with a dangling buildtree

        # Do not push elements that aren't cached, or that are cached with a dangling buildtree

        # artifact unless element type is expected to have an an empty buildtree directory

        # ref unless element type is expected to have an an empty buildtree directory

        if not self._cached_buildtree():

        if not self._cached_buildtree():

            return True

            return True
@@ -2036,6 +2051,8 @@ class Element(Plugin): 
    # Returns:

    # Returns:

    #     (bool): True if artifact cached with buildtree, False if

    #     (bool): True if artifact cached with buildtree, False if

    #             element not cached or missing expected buildtree.

    #             element not cached or missing expected buildtree.

    #             Note this only confirms if a buildtree is present,

    #             not its contents.

    #

    #

    def _cached_buildtree(self):

    def _cached_buildtree(self):

        context = self._get_context()

        context = self._get_context()

tests/integration/artifact.py

+107 −1
Original line number Original line Diff line number Diff line
@@ -20,9 +20,12 @@ 




import os

import os

import pytest

import pytest

import shutil





from buildstream.plugintestutils import cli_integration as cli

from buildstream.plugintestutils import cli_integration as cli



from tests.testutils import create_artifact_share

from tests.testutils.site import HAVE_SANDBOX

from buildstream._exceptions import ErrorDomain





pytestmark = pytest.mark.integration

pytestmark = pytest.mark.integration
@@ -66,3 +69,106 @@ def test_artifact_log(cli, tmpdir, datafiles): 
    assert result.exit_code == 0

    assert result.exit_code == 0

    # The artifact is cached under both a strong key and a weak key

    # The artifact is cached under both a strong key and a weak key

    assert (log + log) == result.output

    assert (log + log) == result.output





# A test to capture the integration of the cachebuildtrees

# behaviour, which by default is to include the buildtree

# content of an element on caching.

@pytest.mark.integration

@pytest.mark.datafiles(DATA_DIR)

@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')

def test_cache_buildtrees(cli, tmpdir, datafiles):

    project = os.path.join(datafiles.dirname, datafiles.basename)

    element_name = 'autotools/amhello.bst'



    # Create artifact shares for pull & push testing

    with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\

        create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\

        create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3:

        cli.configure({

            'artifacts': {'url': share1.repo, 'push': True},

            'artifactdir': os.path.join(str(tmpdir), 'artifacts')

        })



        # Build autotools element with cache-buildtrees set via the

        # cli. The artifact should be successfully pushed to the share1 remote

        # and cached locally with an 'empty' buildtree digest, as it's not a

        # dangling ref

        result = cli.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])

        assert result.exit_code == 0

        assert cli.get_element_state(project, element_name) == 'cached'

        assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))



        # The extracted buildtree dir should be empty, as we set the config

        # to not cache buildtrees

        cache_key = cli.get_element_key(project, element_name)

        elementdigest = share1.has_artifact('test', element_name, cache_key)

        buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',

                                    elementdigest.hash, 'buildtree')

        assert os.path.isdir(buildtreedir)

        assert not os.listdir(buildtreedir)



        # Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees

        # that is was cached in share1 as expected with an empty buildtree dir

        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))

        assert cli.get_element_state(project, element_name) != 'cached'

        result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])

        assert element_name in result.get_pulled_elements()

        assert os.path.isdir(buildtreedir)

        assert not os.listdir(buildtreedir)

        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))



        # Assert that the default behaviour of pull to not include buildtrees on the artifact

        # in share1 which was purposely cached with an empty one behaves as expected. As such the

        # pulled artifact will have a dangling ref for the buildtree dir, regardless of content,

        # leading to no buildtreedir being extracted

        result = cli.run(project=project, args=['artifact', 'pull', element_name])

        assert element_name in result.get_pulled_elements()

        assert not os.path.isdir(buildtreedir)

        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))



        # Repeat building the artifacts, this time with the default behaviour of caching buildtrees,

        # as such the buildtree dir should not be empty

        cli.configure({

            'artifacts': {'url': share2.repo, 'push': True},

            'artifactdir': os.path.join(str(tmpdir), 'artifacts')

        })

        result = cli.run(project=project, args=['build', element_name])

        assert result.exit_code == 0

        assert cli.get_element_state(project, element_name) == 'cached'

        assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))



        # Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths

        elementdigest = share2.has_artifact('test', element_name, cache_key)

        buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',

                                    elementdigest.hash, 'buildtree')

        assert os.path.isdir(buildtreedir)

        assert os.listdir(buildtreedir) is not None



        # Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees

        # that it was cached in share2 as expected with a populated buildtree dir

        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))

        assert cli.get_element_state(project, element_name) != 'cached'

        result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])

        assert element_name in result.get_pulled_elements()

        assert os.path.isdir(buildtreedir)

        assert os.listdir(buildtreedir) is not None

        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))



        # Clarify that the user config option for cache-buildtrees works as the cli

        # main option does. Point to share3 which does not have the artifacts cached to force

        # a build

        cli.configure({

            'artifacts': {'url': share3.repo, 'push': True},

            'artifactdir': os.path.join(str(tmpdir), 'artifacts'),

            'cache': {'cache-buildtrees': 'never'}

        })

        result = cli.run(project=project, args=['build', element_name])

        assert result.exit_code == 0

        assert cli.get_element_state(project, element_name) == 'cached'

        cache_key = cli.get_element_key(project, element_name)

        elementdigest = share3.has_artifact('test', element_name, cache_key)

        buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',

                                    elementdigest.hash, 'buildtree')

        assert os.path.isdir(buildtreedir)

        assert not os.listdir(buildtreedir)

tests/integration/build-tree.py→tests/integration/shellbuildtrees.py

+71 −0
Original line number Original line Diff line number Diff line
@@ -52,6 +52,29 @@ def test_buildtree_staged_forced_true(cli_integration, tmpdir, datafiles): 
    assert 'Hi' in res.output

    assert 'Hi' in res.output









@pytest.mark.datafiles(DATA_DIR)

@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')

def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles):

    # Test that if we stage a cached and empty buildtree, we warn the user.

    project = os.path.join(datafiles.dirname, datafiles.basename)

    element_name = 'build-shell/buildtree.bst'



    # Switch to a temp artifact cache dir to ensure the artifact is rebuilt,

    # caching an empty buildtree

    cli_integration.configure({

        'artifactdir': os.path.join(os.path.join(str(tmpdir), 'artifacts'))

    })



    res = cli_integration.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])

    res.assert_success()



    res = cli_integration.run(project=project, args=[

        'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test'

    ])

    res.assert_shell_error()

    assert "Artifact contains an empty buildtree" in res.stderr





@pytest.mark.datafiles(DATA_DIR)

@pytest.mark.datafiles(DATA_DIR)

@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')

@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')

def test_buildtree_staged_if_available(cli_integration, tmpdir, datafiles):

def test_buildtree_staged_if_available(cli_integration, tmpdir, datafiles):
@@ -106,6 +129,54 @@ def test_buildtree_from_failure(cli_integration, tmpdir, datafiles): 
    assert 'Hi' in res.output

    assert 'Hi' in res.output









@pytest.mark.datafiles(DATA_DIR)

@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')

def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles):



    project = os.path.join(datafiles.dirname, datafiles.basename)

    element_name = 'build-shell/buildtree-fail.bst'



    # Switch to a temp artifact cache dir to ensure the artifact is rebuilt,

    # caching an empty buildtree

    cli_integration.configure({

        'artifactdir': os.path.join(os.path.join(str(tmpdir), 'artifacts'))

    })



    res = cli_integration.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])

    res.assert_main_error(ErrorDomain.STREAM, None)



    res = cli_integration.run(project=project, args=[

        'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'

    ])

    res.assert_shell_error()

    assert "Artifact contains an empty buildtree" in res.stderr





@pytest.mark.datafiles(DATA_DIR)

@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')

def test_buildtree_from_failure_option_failure(cli_integration, tmpdir, datafiles):



    project = os.path.join(datafiles.dirname, datafiles.basename)

    element_name = 'build-shell/buildtree-fail.bst'



    # build with  --cache-buildtrees set to 'failure', behaviour should match

    # default behaviour (which is always) as the buildtree will explicitly have been

    # cached with content.

    cli_integration.configure({

        'artifactdir': os.path.join(os.path.join(str(tmpdir), 'artifacts'))

    })



    res = cli_integration.run(project=project, args=['--cache-buildtrees', 'failure', 'build', element_name])

    res.assert_main_error(ErrorDomain.STREAM, None)



    res = cli_integration.run(project=project, args=[

        'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'

    ])

    res.assert_success()

    assert "WARNING: using a buildtree from a failed build" in res.stderr

    assert 'Hi' in res.output





# Check that build shells work when pulled from a remote cache

# Check that build shells work when pulled from a remote cache

# This is to roughly simulate remote execution

# This is to roughly simulate remote execution

@pytest.mark.datafiles(DATA_DIR)

@pytest.mark.datafiles(DATA_DIR)