Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • willsalmon/buildstream
  • CumHoleZH/buildstream
  • tchaik/buildstream
  • DCotyPortfolio/buildstream
  • jesusoctavioas/buildstream
  • patrickmmartin/buildstream
  • franred/buildstream
  • tintou/buildstream
  • alatiera/buildstream
  • martinblanchard/buildstream
  • neverdie22042524/buildstream
  • Mattlk13/buildstream
  • PServers/buildstream
  • phamnghia610909/buildstream
  • chiaratolentino/buildstream
  • eysz7-x-x/buildstream
  • kerrick1/buildstream
  • matthew-yates/buildstream
  • twofeathers/buildstream
  • mhadjimichael/buildstream
  • pointswaves/buildstream
  • Mr.JackWilson/buildstream
  • Tw3akG33k/buildstream
  • AlexFazakas/buildstream
  • eruidfkiy/buildstream
  • clamotion2/buildstream
  • nanonyme/buildstream
  • wickyjaaa/buildstream
  • nmanchev/buildstream
  • bojorquez.ja/buildstream
  • mostynb/buildstream
  • highpit74/buildstream
  • Demo112/buildstream
  • ba2014sheer/buildstream
  • tonimadrino/buildstream
  • usuario2o/buildstream
  • Angelika123456/buildstream
  • neo355/buildstream
  • corentin-ferlay/buildstream
  • coldtom/buildstream
  • wifitvbox81/buildstream
  • 358253885/buildstream
  • seanborg/buildstream
  • SotK/buildstream
  • DouglasWinship/buildstream
  • karansthr97/buildstream
  • louib/buildstream
  • bwh-ct/buildstream
  • robjh/buildstream
  • we88c0de/buildstream
  • zhengxian5555/buildstream
51 results
Show changes
Commits on Source (8)
Hide whitespace changes
Inline Side-by-side
buildstream/plugins/sources/git.py
View file @ fd844315
......@@ -404,8 +404,10 @@ class GitSource(Source):
mirror.stage(directory)
def get_source_fetchers(self):
yield self.mirror
self.refresh_submodules()
return [self.mirror] + self.submodules
for submodule in self.submodules:
yield submodule
###########################################################
# Local Functions #
......
buildstream/plugins/sources/ostree.py
View file @ fd844315
......@@ -71,7 +71,7 @@ class OSTreeSource(Source):
self.ref = self.node_get_member(node, str, 'ref', None)
self.tracking = self.node_get_member(node, str, 'track', None)
self.mirror = os.path.join(self.get_mirror_directory(),
utils.url_directory_name(self.url))
utils.url_directory_name(self.original_url))
# (optional) Not all repos are signed. But if they are, get the gpg key
self.gpg_key_path = None
......@@ -104,10 +104,11 @@ class OSTreeSource(Source):
return None
self.ensure()
remote_name = self.ensure_remote(self.url)
with self.timed_activity("Fetching tracking ref '{}' from origin: {}"
.format(self.tracking, self.url)):
try:
_ostree.fetch(self.repo, ref=self.tracking, progress=self.progress)
_ostree.fetch(self.repo, remote=remote_name, ref=self.tracking, progress=self.progress)
except OSTreeError as e:
raise SourceError("{}: Failed to fetch tracking ref '{}' from origin {}\n\n{}"
.format(self, self.tracking, self.url, e)) from e
......@@ -116,11 +117,12 @@ class OSTreeSource(Source):
def fetch(self):
self.ensure()
remote_name = self.ensure_remote(self.url)
if not _ostree.exists(self.repo, self.ref):
with self.timed_activity("Fetching remote ref: {} from origin: {}"
.format(self.ref, self.url)):
try:
_ostree.fetch(self.repo, ref=self.ref, progress=self.progress)
_ostree.fetch(self.repo, remote=remote_name, ref=self.ref, progress=self.progress)
except OSTreeError as e:
raise SourceError("{}: Failed to fetch ref '{}' from origin: {}\n\n{}"
.format(self, self.ref, self.url, e)) from e
......@@ -171,14 +173,22 @@ class OSTreeSource(Source):
self.status("Creating local mirror for {}".format(self.url))
self.repo = _ostree.ensure(self.mirror, True)
gpg_key = None
if self.gpg_key_path:
gpg_key = 'file://' + self.gpg_key_path
try:
_ostree.configure_remote(self.repo, "origin", self.url, key_url=gpg_key)
except OSTreeError as e:
raise SourceError("{}: Failed to configure origin {}\n\n{}".format(self, self.url, e)) from e
def ensure_remote(self, url):
if self.original_url == self.url:
remote_name = 'origin'
else:
remote_name = utils.url_directory_name(url)
gpg_key = None
if self.gpg_key_path:
gpg_key = 'file://' + self.gpg_key_path
try:
_ostree.configure_remote(self.repo, remote_name, url, key_url=gpg_key)
except OSTreeError as e:
raise SourceError("{}: Failed to configure origin {}\n\n{}".format(self, self.url, e)) from e
return remote_name
def progress(self, percent, message):
self.status(message)
......
tests/frontend/mirror.py
View file @ fd844315
......@@ -139,6 +139,67 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind):
if kind == 'ostree':
# FIXME: Mirroring fallback fails with ostree
pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
upstream_repodir = os.path.join(str(tmpdir), 'upstream')
mirror_repodir = os.path.join(str(tmpdir), 'mirror')
project_dir = os.path.join(str(tmpdir), 'project')
os.makedirs(project_dir)
element_dir = os.path.join(project_dir, 'elements')
# Create repo objects of the upstream and mirror
upstream_repo = create_repo(kind, upstream_repodir)
ref = upstream_repo.create(dev_files_path)
mirror_repo = upstream_repo.copy(mirror_repodir)
element = {
'kind': 'import',
'sources': [
upstream_repo.source_config(ref=ref)
]
}
element_name = 'test.bst'
element_path = os.path.join(element_dir, element_name)
full_repo = element['sources'][0]['url']
upstream_map, repo_name = os.path.split(full_repo)
alias = 'foo-' + kind
aliased_repo = alias + ':' + repo_name
element['sources'][0]['url'] = aliased_repo
full_mirror = mirror_repo.source_config()['url']
mirror_map, _ = os.path.split(full_mirror)
os.makedirs(element_dir)
_yaml.dump(element, element_path)
project = {
'name': 'test',
'element-path': 'elements',
'aliases': {
alias: 'http://www.example.com/'
},
'mirrors': [
{
'name': 'middle-earth',
'aliases': {
alias: [mirror_map + "/"],
},
},
]
}
project_file = os.path.join(project_dir, 'project.conf')
_yaml.dump(project, project_file)
result = cli.run(project=project_dir, args=['fetch', element_name])
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_mirror_fetch_multi(cli, tmpdir, datafiles):
output_file = os.path.join(str(tmpdir), "output.txt")
......@@ -405,14 +466,6 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
if kind == 'git':
# FIXME: Mirroring fallback does not work with git because it tries to
# fetch submodules on upstream.
pytest.skip("Bug #537 - Mirror fallback does not work for git")
if kind == 'ostree':
# FIXME: Mirroring fallback fails with ostree
pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
upstream_repodir = os.path.join(str(tmpdir), 'upstream')
mirror_repodir = os.path.join(str(tmpdir), 'mirror')
......@@ -559,3 +612,276 @@ def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind):
os.rename('{}.bak'.format(upstream_repo.repo), upstream_repo.repo)
result = cli.run(project=project_dir, args=['fetch', element_name])
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_mirror_git_submodule_fetch(cli, tmpdir, datafiles):
# Test that it behaves as expected with submodules, both defined in config
# and discovered when fetching.
foo_file = os.path.join(str(datafiles), 'files', 'foo')
bar_file = os.path.join(str(datafiles), 'files', 'bar')
bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
mirror_dir = os.path.join(str(datafiles), 'mirror')
defined_subrepo = create_repo('git', str(tmpdir), 'defined_subrepo')
defined_mirror_ref = defined_subrepo.create(bin_files_path)
defined_mirror = defined_subrepo.copy(mirror_dir)
defined_subref = defined_subrepo.add_file(foo_file)
found_subrepo = create_repo('git', str(tmpdir), 'found_subrepo')
found_subref = found_subrepo.create(dev_files_path)
main_repo = create_repo('git', str(tmpdir))
main_mirror_ref = main_repo.create(bin_files_path)
main_repo.add_submodule('defined', 'file://' + defined_subrepo.repo)
main_repo.add_submodule('found', 'file://' + found_subrepo.repo)
main_mirror = main_repo.copy(mirror_dir)
main_ref = main_repo.add_file(bar_file)
project_dir = os.path.join(str(tmpdir), 'project')
os.makedirs(project_dir)
element_dir = os.path.join(project_dir, 'elements')
os.makedirs(element_dir)
element = {
'kind': 'import',
'sources': [
main_repo.source_config(ref=main_mirror_ref)
]
}
element_name = 'test.bst'
element_path = os.path.join(element_dir, element_name)
# Alias the main repo
full_repo = element['sources'][0]['url']
_, repo_name = os.path.split(full_repo)
alias = 'foo'
aliased_repo = alias + ':' + repo_name
element['sources'][0]['url'] = aliased_repo
# Hide the found subrepo
del element['sources'][0]['submodules']['found']
# Alias the defined subrepo
subrepo = element['sources'][0]['submodules']['defined']['url']
_, repo_name = os.path.split(subrepo)
aliased_repo = alias + ':' + repo_name
element['sources'][0]['submodules']['defined']['url'] = aliased_repo
_yaml.dump(element, element_path)
full_mirror = main_mirror.source_config()['url']
mirror_map, _ = os.path.split(full_mirror)
project = {
'name': 'test',
'element-path': 'elements',
'aliases': {
alias: 'http://www.example.com/'
},
'mirrors': [
{
'name': 'middle-earth',
'aliases': {
alias: [mirror_map + "/"],
},
},
]
}
project_file = os.path.join(project_dir, 'project.conf')
_yaml.dump(project, project_file)
result = cli.run(project=project_dir, args=['fetch', element_name])
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_mirror_fallback_git_only_submodules(cli, tmpdir, datafiles):
# Main repo has no mirror or alias.
# One submodule is overridden to use a mirror.
# There is another submodules not overriden.
# Upstream for overriden submodule is down.
#
# We expect:
# - overriden submodule is fetched from mirror.
# - other submodule is fetched.
bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
upstream_bin_repodir = os.path.join(str(tmpdir), 'bin-upstream')
mirror_bin_repodir = os.path.join(str(tmpdir), 'bin-mirror')
upstream_bin_repo = create_repo('git', upstream_bin_repodir)
upstream_bin_repo.create(bin_files_path)
mirror_bin_repo = upstream_bin_repo.copy(mirror_bin_repodir)
dev_repodir = os.path.join(str(tmpdir), 'dev-upstream')
dev_repo = create_repo('git', dev_repodir)
dev_repo.create(dev_files_path)
main_files = os.path.join(str(tmpdir), 'main-files')
os.makedirs(main_files)
with open(os.path.join(main_files, 'README'), 'w') as f:
f.write("TEST\n")
main_repodir = os.path.join(str(tmpdir), 'main-upstream')
main_repo = create_repo('git', main_repodir)
main_repo.create(main_files)
upstream_url = 'file://{}'.format(upstream_bin_repo.repo)
main_repo.add_submodule('bin', url=upstream_url)
main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo))
# Unlist 'dev'.
del main_repo.submodules['dev']
main_ref = main_repo.latest_commit()
upstream_map, repo_name = os.path.split(upstream_url)
alias = 'foo'
aliased_repo = '{}:{}'.format(alias, repo_name)
main_repo.submodules['bin']['url'] = aliased_repo
full_mirror = mirror_bin_repo.source_config()['url']
mirror_map, _ = os.path.split(full_mirror)
project_dir = os.path.join(str(tmpdir), 'project')
os.makedirs(project_dir)
element_dir = os.path.join(project_dir, 'elements')
element = {
'kind': 'import',
'sources': [
main_repo.source_config(ref=main_ref, checkout_submodules=True)
]
}
element_name = 'test.bst'
element_path = os.path.join(element_dir, element_name)
os.makedirs(element_dir)
_yaml.dump(element, element_path)
project = {
'name': 'test',
'element-path': 'elements',
'aliases': {
alias: upstream_map + "/"
},
'mirrors': [
{
'name': 'middle-earth',
'aliases': {
alias: [mirror_map + "/"],
}
}
]
}
project_file = os.path.join(project_dir, 'project.conf')
_yaml.dump(project, project_file)
# Now make the upstream unavailable.
os.rename(upstream_bin_repo.repo, '{}.bak'.format(upstream_bin_repo.repo))
result = cli.run(project=project_dir, args=['fetch', element_name])
result.assert_success()
result = cli.run(project=project_dir, args=['build', element_name])
result.assert_success()
checkout = os.path.join(str(tmpdir), 'checkout')
result = cli.run(project=project_dir, args=['checkout', element_name, checkout])
result.assert_success()
assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello'))
assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h'))
@pytest.mark.datafiles(DATA_DIR)
def test_mirror_fallback_git_with_submodules(cli, tmpdir, datafiles):
# Main repo has mirror. But does not list submodules.
#
# We expect:
# - we will fetch submodules anyway
bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
bin_repodir = os.path.join(str(tmpdir), 'bin-repo')
bin_repo = create_repo('git', bin_repodir)
bin_repo.create(bin_files_path)
dev_repodir = os.path.join(str(tmpdir), 'dev-repo')
dev_repo = create_repo('git', dev_repodir)
dev_repo.create(dev_files_path)
main_files = os.path.join(str(tmpdir), 'main-files')
os.makedirs(main_files)
with open(os.path.join(main_files, 'README'), 'w') as f:
f.write("TEST\n")
upstream_main_repodir = os.path.join(str(tmpdir), 'main-upstream')
upstream_main_repo = create_repo('git', upstream_main_repodir)
upstream_main_repo.create(main_files)
upstream_main_repo.add_submodule('bin', url='file://{}'.format(bin_repo.repo))
upstream_main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo))
# Unlist submodules.
del upstream_main_repo.submodules['bin']
del upstream_main_repo.submodules['dev']
upstream_main_ref = upstream_main_repo.latest_commit()
mirror_main_repodir = os.path.join(str(tmpdir), 'main-mirror')
mirror_main_repo = upstream_main_repo.copy(mirror_main_repodir)
upstream_url = mirror_main_repo.source_config()['url']
upstream_map, repo_name = os.path.split(upstream_url)
alias = 'foo'
aliased_repo = '{}:{}'.format(alias, repo_name)
full_mirror = mirror_main_repo.source_config()['url']
mirror_map, _ = os.path.split(full_mirror)
project_dir = os.path.join(str(tmpdir), 'project')
os.makedirs(project_dir)
element_dir = os.path.join(project_dir, 'elements')
element = {
'kind': 'import',
'sources': [
upstream_main_repo.source_config(ref=upstream_main_ref, checkout_submodules=True)
]
}
element['sources'][0]['url'] = aliased_repo
element_name = 'test.bst'
element_path = os.path.join(element_dir, element_name)
os.makedirs(element_dir)
_yaml.dump(element, element_path)
project = {
'name': 'test',
'element-path': 'elements',
'aliases': {
alias: upstream_map + "/"
},
'mirrors': [
{
'name': 'middle-earth',
'aliases': {
alias: [mirror_map + "/"],
}
}
]
}
project_file = os.path.join(project_dir, 'project.conf')
_yaml.dump(project, project_file)
# Now make the upstream unavailable.
os.rename(upstream_main_repo.repo, '{}.bak'.format(upstream_main_repo.repo))
result = cli.run(project=project_dir, args=['fetch', element_name])
result.assert_success()
result = cli.run(project=project_dir, args=['build', element_name])
result.assert_success()
checkout = os.path.join(str(tmpdir), 'checkout')
result = cli.run(project=project_dir, args=['checkout', element_name, checkout])
result.assert_success()
assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello'))
assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h'))
tests/frontend/project/files/bar 0 → 100644
View file @ fd844315
tests/frontend/project/files/foo 0 → 100644
View file @ fd844315