Commit a46650d0 authored by Tom Pollard's avatar Tom Pollard
Browse files

Merge branch 'tpollard/buildremote' into 'master' 

Add --remote, -r option to bst build, inline with pull & push

See merge request !1119
parents 3ab09651 cd8e5e27

NEWS

+4 −0
Original line number Diff line number Diff line
@@ -122,6 +122,10 @@ buildstream 1.3.1 
    'shell', 'show', 'source-checkout', 'track', 'workspace close' and 'workspace reset'

    commands are affected.



  o bst 'build' now has '--remote, -r' option, inline with bst 'push' & 'pull'.

    Providing a remote will limit build's pull/push remote actions to the given

    remote specifically, ignoring those defined via user or project configuration.





=================

buildstream 1.1.5

buildstream/_frontend/cli.py

+6 −3
Original line number Diff line number Diff line
@@ -338,10 +338,12 @@ def init(app, project_name, format_version, element_path, force): 
              help="Allow tracking to cross junction boundaries")

@click.option('--track-save', default=False, is_flag=True,

              help="Deprecated: This is ignored")

@click.option('--remote', '-r', default=None,

              help="The URL of the remote cache (defaults to the first configured cache)")

@click.argument('elements', nargs=-1,

                type=click.Path(readable=False))

@click.pass_obj

def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions):

def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions, remote):

    """Build elements in a pipeline



    Specifying no elements will result in building the default targets
@@ -376,7 +378,8 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac 
                         track_except=track_except,

                         track_cross_junctions=track_cross_junctions,

                         ignore_junction_targets=ignore_junction_targets,

                         build_all=all_)

                         build_all=all_,

                         remote=remote)





##################################################################
@@ -1012,7 +1015,7 @@ def artifact_checkout(app, force, deps, integrate, hardlinks, tar, directory, el 
@click.option('--deps', '-d', default='none',

              type=click.Choice(['none', 'all']),

              help='The dependency artifacts to pull (default: none)')

@click.option('--remote', '-r',

@click.option('--remote', '-r', default=None,

              help="The URL of the remote cache (defaults to the first configured cache)")

@click.argument('elements', nargs=-1,

                type=click.Path(readable=False))

buildstream/_stream.py

+12 −2
Original line number Diff line number Diff line
@@ -197,26 +197,36 @@ class Stream(): 
    #    ignore_junction_targets (bool): Whether junction targets should be filtered out

    #    build_all (bool): Whether to build all elements, or only those

    #                      which are required to build the target.

    #    remote (str): The URL of a specific remote server to push to, or None

    #

    # If `remote` specified as None, then regular configuration will be used

    # to determine where to push artifacts to.

    #

    def build(self, targets, *,

              track_targets=None,

              track_except=None,

              track_cross_junctions=False,

              ignore_junction_targets=False,

              build_all=False):

              build_all=False,

              remote=None):



        if build_all:

            selection = PipelineSelection.ALL

        else:

            selection = PipelineSelection.PLAN



        use_config = True

        if remote:

            use_config = False



        elements, track_elements = \

            self._load(targets, track_targets,

                       selection=selection, track_selection=PipelineSelection.ALL,

                       track_except_targets=track_except,

                       track_cross_junctions=track_cross_junctions,

                       ignore_junction_targets=ignore_junction_targets,

                       use_artifact_config=True,

                       use_artifact_config=use_config,

                       artifact_remote_url=remote,

                       fetch_subprojects=True,

                       dynamic_plan=True)

tests/frontend/completions.py

+2 −1
Original line number Diff line number Diff line
@@ -141,7 +141,8 @@ def test_commands(cli, cmd, word_idx, expected): 
    ('bst --no-colors build -', 3, ['--all ', '--track ', '--track-all ',

                                    '--track-except ',

                                    '--track-cross-junctions ', '-J ',

                                    '--track-save ']),

                                    '--track-save ',

                                    '--remote ', '-r ']),



    # Test the behavior of completing after an option that has a

    # parameter that cannot be completed, vs an option that has

tests/frontend/pull.py

+53 −0
Original line number Diff line number Diff line
@@ -408,3 +408,56 @@ def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles): 


        assert "INFO    Remote ({}) does not have".format(share.repo) in result.stderr

        assert "SKIPPED Pull" in result.stderr





@pytest.mark.datafiles(DATA_DIR)

def test_build_remote_option(caplog, cli, tmpdir, datafiles):

    project = os.path.join(datafiles.dirname, datafiles.basename)

    caplog.set_level(1)



    with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\

        create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject,\

        create_artifact_share(os.path.join(str(tmpdir), 'artifactshare3')) as sharecli:



        # Add shareproject repo url to project.conf

        with open(os.path.join(project, "project.conf"), "a") as projconf:

            projconf.write("artifacts:\n  url: {}\n  push: True".format(shareproject.repo))



        # Configure shareuser remote in user conf

        cli.configure({

            'artifacts': {'url': shareuser.repo, 'push': True}

        })



        # Push the artifacts to the shareuser and shareproject remotes.

        # Assert that shareuser and shareproject have the artfifacts cached,

        # but sharecli doesn't, then delete locally cached elements

        result = cli.run(project=project, args=['build', 'target.bst'])

        result.assert_success()

        all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']

        for element_name in all_elements:

            assert element_name in result.get_pushed_elements()

            assert_not_shared(cli, sharecli, project, element_name)

            assert_shared(cli, shareuser, project, element_name)

            assert_shared(cli, shareproject, project, element_name)

            cli.remove_artifact_from_cache(project, element_name)



        # Now check that a build with cli set as sharecli results in nothing being pulled,

        # as it doesn't have them cached and shareuser/shareproject should be ignored. This

        # will however result in the artifacts being built and pushed to it

        result = cli.run(project=project, args=['build', '--remote', sharecli.repo, 'target.bst'])

        result.assert_success()

        for element_name in all_elements:

            assert element_name not in result.get_pulled_elements()

            assert_shared(cli, sharecli, project, element_name)

            cli.remove_artifact_from_cache(project, element_name)



        # Now check that a clean build with cli set as sharecli should result in artifacts only

        # being pulled from it, as that was provided via the cli and is populated

        result = cli.run(project=project, args=['build', '--remote', sharecli.repo, 'target.bst'])

        result.assert_success()

        for element_name in all_elements:

            assert cli.get_element_state(project, element_name) == 'cached'

            assert element_name in result.get_pulled_elements()

        assert shareproject.repo not in result.stderr

        assert shareuser.repo not in result.stderr

        assert sharecli.repo in result.stderr