Loading buildstream/plugins/sources/git.py +41 −4 Original line number Diff line number Diff line Loading @@ -74,6 +74,9 @@ This plugin provides the following configurable warnings: - 'git:inconsistent-submodule' - A submodule was found to be missing from the underlying git repository. This plugin also utilises the following configurable core plugin warnings: - 'ref-not-in-track' - The provided ref was not found in the provided track in the element's git repository. """ import os Loading @@ -87,6 +90,7 @@ from configparser import RawConfigParser from buildstream import Source, SourceError, Consistency, SourceFetcher from buildstream import utils from buildstream.plugin import CoreWarnings GIT_MODULES = '.gitmodules' Loading Loading @@ -203,7 +207,7 @@ class GitMirror(SourceFetcher): cwd=self.mirror) return output.rstrip('\n') def stage(self, directory): def stage(self, directory, track=None): fullpath = os.path.join(directory, self.path) # Using --shared here avoids copying the objects into the checkout, in any Loading @@ -217,10 +221,14 @@ class GitMirror(SourceFetcher): fail="Failed to checkout git ref {}".format(self.ref), cwd=fullpath) # Check that the user specified ref exists in the track if provided & not already tracked if track: self.assert_ref_in_track(fullpath, track) # Remove .git dir shutil.rmtree(os.path.join(fullpath, ".git")) def init_workspace(self, directory): def init_workspace(self, directory, track=None): fullpath = os.path.join(directory, self.path) url = self.source.translate_url(self.url) Loading @@ -236,6 +244,10 @@ class GitMirror(SourceFetcher): fail="Failed to checkout git ref {}".format(self.ref), cwd=fullpath) # Check that the user specified ref exists in the track if provided & not already tracked if track: self.assert_ref_in_track(fullpath, track) # List the submodules (path/url tuples) present at the given ref of this repo def submodule_list(self): modules = "{}:{}".format(self.ref, GIT_MODULES) Loading Loading @@ -300,6 +312,28 @@ class GitMirror(SourceFetcher): return None # Assert that ref exists in track, if track has been specified. def assert_ref_in_track(self, fullpath, track): _, branch = self.source.check_output([self.source.host_git, 'branch', '--list', track, '--contains', self.ref], cwd=fullpath,) if branch: return True else: _, tag = self.source.check_output([self.source.host_git, 'tag', '--list', track, '--contains', self.ref], cwd=fullpath,) if tag: return True detail = "The ref provided for the element does not exist locally in the provided track branch / tag " + \ "'{}'.\nYou may wish to track the element to update the ref from '{}' ".format(track, track) + \ "with `bst track`,\nor examine the upstream at '{}' for the specific ref.".format(self.url) self.source.warn("{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n" .format(self.source, self.ref, track, self.url), detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK) class GitSource(Source): # pylint: disable=attribute-defined-outside-init Loading Loading @@ -342,6 +376,7 @@ class GitSource(Source): self.submodule_checkout_overrides[path] = checkout self.mark_download_url(self.original_url) self.tracked = False def preflight(self): # Check if git is installed, get the binary at the same time Loading Loading @@ -407,6 +442,8 @@ class GitSource(Source): # Update self.mirror.ref and node.ref from the self.tracking branch ret = self.mirror.latest_commit(self.tracking) # Set tracked attribute, parameter for if self.mirror.assert_ref_in_track is needed self.tracked = True return ret def init_workspace(self, directory): Loading @@ -414,7 +451,7 @@ class GitSource(Source): self.refresh_submodules() with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True): self.mirror.init_workspace(directory) self.mirror.init_workspace(directory, track=(self.tracking if not self.tracked else None)) for mirror in self.submodules: mirror.init_workspace(directory) Loading @@ -430,7 +467,7 @@ class GitSource(Source): # Stage the main repo in the specified directory # with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True): self.mirror.stage(directory) self.mirror.stage(directory, track=(self.tracking if not self.tracked else None)) for mirror in self.submodules: if mirror.path in self.submodule_checkout_overrides: checkout = self.submodule_checkout_overrides[mirror.path] Loading tests/sources/git.py +68 −0 Original line number Diff line number Diff line Loading @@ -25,6 +25,7 @@ import pytest from buildstream._exceptions import ErrorDomain from buildstream import _yaml from buildstream.plugin import CoreWarnings from tests.testutils import cli, create_repo from tests.testutils.site import HAVE_GIT Loading Loading @@ -408,3 +409,70 @@ def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles): result = cli.run(project=project, args=['show', 'target.bst']) result.assert_main_error(ErrorDomain.SOURCE, "missing-track-and-ref") result.assert_task_error(None, None) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_ref_not_in_track_warn(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) # Create the repo from 'repofiles', create a branch without latest commit repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) gitsource = repo.source_config(ref=ref) # Overwrite the track value to the added branch gitsource['track'] = 'foo' # Write out our test target element = { 'kind': 'import', 'sources': [ gitsource ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Assert the warning is raised as ref is not in branch foo. # Assert warning not error to the user, when not set as fatal. result = cli.run(project=project, args=['build', 'target.bst']) assert "The ref provided for the element does not exist locally" in result.stderr @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_ref_not_in_track_warn_error(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) # Add fatal-warnings ref-not-in-track to project.conf project_template = { "name": "foo", "fatal-warnings": [CoreWarnings.REF_NOT_IN_TRACK] } _yaml.dump(project_template, os.path.join(project, 'project.conf')) # Create the repo from 'repofiles', create a branch without latest commit repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) gitsource = repo.source_config(ref=ref) # Overwrite the track value to the added branch gitsource['track'] = 'foo' # Write out our test target element = { 'kind': 'import', 'sources': [ gitsource ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Assert that build raises a warning here that is captured # as plugin error, due to the fatal warning being set result = cli.run(project=project, args=['build', 'target.bst']) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.REF_NOT_IN_TRACK) Loading
buildstream/plugins/sources/git.py +41 −4 Original line number Diff line number Diff line Loading @@ -74,6 +74,9 @@ This plugin provides the following configurable warnings: - 'git:inconsistent-submodule' - A submodule was found to be missing from the underlying git repository. This plugin also utilises the following configurable core plugin warnings: - 'ref-not-in-track' - The provided ref was not found in the provided track in the element's git repository. """ import os Loading @@ -87,6 +90,7 @@ from configparser import RawConfigParser from buildstream import Source, SourceError, Consistency, SourceFetcher from buildstream import utils from buildstream.plugin import CoreWarnings GIT_MODULES = '.gitmodules' Loading Loading @@ -203,7 +207,7 @@ class GitMirror(SourceFetcher): cwd=self.mirror) return output.rstrip('\n') def stage(self, directory): def stage(self, directory, track=None): fullpath = os.path.join(directory, self.path) # Using --shared here avoids copying the objects into the checkout, in any Loading @@ -217,10 +221,14 @@ class GitMirror(SourceFetcher): fail="Failed to checkout git ref {}".format(self.ref), cwd=fullpath) # Check that the user specified ref exists in the track if provided & not already tracked if track: self.assert_ref_in_track(fullpath, track) # Remove .git dir shutil.rmtree(os.path.join(fullpath, ".git")) def init_workspace(self, directory): def init_workspace(self, directory, track=None): fullpath = os.path.join(directory, self.path) url = self.source.translate_url(self.url) Loading @@ -236,6 +244,10 @@ class GitMirror(SourceFetcher): fail="Failed to checkout git ref {}".format(self.ref), cwd=fullpath) # Check that the user specified ref exists in the track if provided & not already tracked if track: self.assert_ref_in_track(fullpath, track) # List the submodules (path/url tuples) present at the given ref of this repo def submodule_list(self): modules = "{}:{}".format(self.ref, GIT_MODULES) Loading Loading @@ -300,6 +312,28 @@ class GitMirror(SourceFetcher): return None # Assert that ref exists in track, if track has been specified. def assert_ref_in_track(self, fullpath, track): _, branch = self.source.check_output([self.source.host_git, 'branch', '--list', track, '--contains', self.ref], cwd=fullpath,) if branch: return True else: _, tag = self.source.check_output([self.source.host_git, 'tag', '--list', track, '--contains', self.ref], cwd=fullpath,) if tag: return True detail = "The ref provided for the element does not exist locally in the provided track branch / tag " + \ "'{}'.\nYou may wish to track the element to update the ref from '{}' ".format(track, track) + \ "with `bst track`,\nor examine the upstream at '{}' for the specific ref.".format(self.url) self.source.warn("{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n" .format(self.source, self.ref, track, self.url), detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK) class GitSource(Source): # pylint: disable=attribute-defined-outside-init Loading Loading @@ -342,6 +376,7 @@ class GitSource(Source): self.submodule_checkout_overrides[path] = checkout self.mark_download_url(self.original_url) self.tracked = False def preflight(self): # Check if git is installed, get the binary at the same time Loading Loading @@ -407,6 +442,8 @@ class GitSource(Source): # Update self.mirror.ref and node.ref from the self.tracking branch ret = self.mirror.latest_commit(self.tracking) # Set tracked attribute, parameter for if self.mirror.assert_ref_in_track is needed self.tracked = True return ret def init_workspace(self, directory): Loading @@ -414,7 +451,7 @@ class GitSource(Source): self.refresh_submodules() with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True): self.mirror.init_workspace(directory) self.mirror.init_workspace(directory, track=(self.tracking if not self.tracked else None)) for mirror in self.submodules: mirror.init_workspace(directory) Loading @@ -430,7 +467,7 @@ class GitSource(Source): # Stage the main repo in the specified directory # with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True): self.mirror.stage(directory) self.mirror.stage(directory, track=(self.tracking if not self.tracked else None)) for mirror in self.submodules: if mirror.path in self.submodule_checkout_overrides: checkout = self.submodule_checkout_overrides[mirror.path] Loading
tests/sources/git.py +68 −0 Original line number Diff line number Diff line Loading @@ -25,6 +25,7 @@ import pytest from buildstream._exceptions import ErrorDomain from buildstream import _yaml from buildstream.plugin import CoreWarnings from tests.testutils import cli, create_repo from tests.testutils.site import HAVE_GIT Loading Loading @@ -408,3 +409,70 @@ def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles): result = cli.run(project=project, args=['show', 'target.bst']) result.assert_main_error(ErrorDomain.SOURCE, "missing-track-and-ref") result.assert_task_error(None, None) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_ref_not_in_track_warn(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) # Create the repo from 'repofiles', create a branch without latest commit repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) gitsource = repo.source_config(ref=ref) # Overwrite the track value to the added branch gitsource['track'] = 'foo' # Write out our test target element = { 'kind': 'import', 'sources': [ gitsource ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Assert the warning is raised as ref is not in branch foo. # Assert warning not error to the user, when not set as fatal. result = cli.run(project=project, args=['build', 'target.bst']) assert "The ref provided for the element does not exist locally" in result.stderr @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_ref_not_in_track_warn_error(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) # Add fatal-warnings ref-not-in-track to project.conf project_template = { "name": "foo", "fatal-warnings": [CoreWarnings.REF_NOT_IN_TRACK] } _yaml.dump(project_template, os.path.join(project, 'project.conf')) # Create the repo from 'repofiles', create a branch without latest commit repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) gitsource = repo.source_config(ref=ref) # Overwrite the track value to the added branch gitsource['track'] = 'foo' # Write out our test target element = { 'kind': 'import', 'sources': [ gitsource ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Assert that build raises a warning here that is captured # as plugin error, due to the fatal warning being set result = cli.run(project=project, args=['build', 'target.bst']) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.REF_NOT_IN_TRACK)
