Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • willsalmon/buildstream
  • CumHoleZH/buildstream
  • tchaik/buildstream
  • DCotyPortfolio/buildstream
  • jesusoctavioas/buildstream
  • patrickmmartin/buildstream
  • franred/buildstream
  • tintou/buildstream
  • alatiera/buildstream
  • martinblanchard/buildstream
  • neverdie22042524/buildstream
  • Mattlk13/buildstream
  • PServers/buildstream
  • phamnghia610909/buildstream
  • chiaratolentino/buildstream
  • eysz7-x-x/buildstream
  • kerrick1/buildstream
  • matthew-yates/buildstream
  • twofeathers/buildstream
  • mhadjimichael/buildstream
  • pointswaves/buildstream
  • Mr.JackWilson/buildstream
  • Tw3akG33k/buildstream
  • AlexFazakas/buildstream
  • eruidfkiy/buildstream
  • clamotion2/buildstream
  • nanonyme/buildstream
  • wickyjaaa/buildstream
  • nmanchev/buildstream
  • bojorquez.ja/buildstream
  • mostynb/buildstream
  • highpit74/buildstream
  • Demo112/buildstream
  • ba2014sheer/buildstream
  • tonimadrino/buildstream
  • usuario2o/buildstream
  • Angelika123456/buildstream
  • neo355/buildstream
  • corentin-ferlay/buildstream
  • coldtom/buildstream
  • wifitvbox81/buildstream
  • 358253885/buildstream
  • seanborg/buildstream
  • SotK/buildstream
  • DouglasWinship/buildstream
  • karansthr97/buildstream
  • louib/buildstream
  • bwh-ct/buildstream
  • robjh/buildstream
  • we88c0de/buildstream
  • zhengxian5555/buildstream
51 results
Show changes
Commits on Source (6)
Hide whitespace changes
Inline Side-by-side
buildstream/storage/_casbaseddirectory.py
View file @ 711c9847
......@@ -244,10 +244,8 @@ class CasBasedDirectory(Directory):
for collection in [self.pb2_directory.files, self.pb2_directory.symlinks, self.pb2_directory.directories]:
for thing in collection:
if thing.name == name:
print("Removing {} from PB2".format(name))
collection.remove(thing)
if name in self.index:
print("Removing {} from index".format(name))
del self.index[name]
def descend(self, subdirectory_spec, create=False):
......@@ -330,26 +328,7 @@ class CasBasedDirectory(Directory):
as a directory as long as it's within this directory tree.
"""
if isinstance(self.index[name].buildstream_object, Directory):
return self.index[name].buildstream_object
# OK then, it's a symlink
symlink = self._find_pb2_entry(name)
assert isinstance(symlink, remote_execution_pb2.SymlinkNode)
absolute = symlink.target.startswith(CasBasedDirectory._pb2_absolute_path_prefix)
if absolute:
root = self.find_root()
else:
root = self
directory = root
components = symlink.target.split(CasBasedDirectory._pb2_path_sep)
for c in components:
if c == ".":
pass
elif c == "..":
directory = directory.parent
else:
directory = directory.descend(c, create=True)
return directory
return self._resolve(name, force_create=True)
def _is_followable(self, name):
""" Returns true if this is a directory or symlink to a valid directory. """
......@@ -364,35 +343,16 @@ class CasBasedDirectory(Directory):
def _resolve_symlink(self, node, force_create=True):
"""Same as _resolve_symlink_or_directory but takes a SymlinkNode.
"""
# OK then, it's a symlink
symlink = node
absolute = symlink.target.startswith(CasBasedDirectory._pb2_absolute_path_prefix)
if absolute:
root = self.find_root()
else:
root = self
directory = root
components = symlink.target.split(CasBasedDirectory._pb2_path_sep)
for c in components:
if c == ".":
pass
elif c == "..":
directory = directory.parent
else:
if c in directory.index or force_create:
directory = directory.descend(c, create=True)
else:
return None
return directory
return self._resolve(node.name, force_create=True)
def _resolve(self, name, absolute_symlinks_resolve=True, force_create=False, first_seen_object = None):
""" Resolves any name to an object. If the name points to a symlink in
this directory, it returns the thing it points to,
recursively. Returns a CasBasedDirectory, FileNode or
None. Never creates a directory or otherwise alters the
directory.
None.
If force_create is on, will attempt to create directories to make symlinks and directories resolve.
If force_create is off, this will never alter this directory.
"""
# First check if it's a normal object and return that
......@@ -439,27 +399,23 @@ class CasBasedDirectory(Directory):
if c == ".":
pass
elif c == "..":
print(" resolving {}: up-dir".format(c))
if directory.parent is not None:
directory = directory.parent
# If directory.parent *is* None, this is an attempt to access
# '..' from the root, which is valid under POSIX; it just
# returns the root.
if directory.parent is not None:
directory = directory.parent
else:
if c in directory.index:
f = directory._resolve(c, absolute_symlinks_resolve, first_seen_object=first_seen_object)
# Ultimately f must now be a file or directory
if isinstance(f, CasBasedDirectory):
directory = f
print(" resolving {}: dir".format(c))
else:
# This is a file or None (i.e. broken symlink)
print(" resolving {}: file/broken link".format(c))
if f is None and force_create:
print("Creating target of broken link {}".format(c))
directory = directory.descend(c, create=True)
elif components:
elif components and force_create:
# Oh dear. We have components left to resolve, but the one we're trying to resolve points to a file.
print("Trying to resolve {}, but found {} was a file.".format(symlink.target, c))
self.delete_entry(c)
......@@ -468,9 +424,7 @@ class CasBasedDirectory(Directory):
else:
return f
else:
print(" resolving {}: Non-existent file; must be from a broken symlink.".format(c))
if force_create:
print("Creating target of broken link {} (2)".format(c))
directory = directory.descend(c, create=True)
else:
return None
......@@ -534,9 +488,6 @@ class CasBasedDirectory(Directory):
""" Imports files from a traditional directory """
result = FileListResult()
for entry in files:
print("Importing {} from file system".format(entry))
print("...Order of elements was {}".format(", ".join(self.index.keys())))
split_path = entry.split(os.path.sep)
# The actual file on the FS we're importing
import_file = os.path.join(source_directory, entry)
......@@ -547,7 +498,6 @@ class CasBasedDirectory(Directory):
# Hand this off to the importer for that subdir. This will only do one file -
# a better way would be to hand off all the files in this subdir at once.
# failed here because directory_name didn't point to a directory...
print("Attempting to import into {} from {}".format(directory_name, source_directory))
subdir_result = self._import_directory_recursively(directory_name, source_directory,
split_path[1:], path_prefix)
result.combine(subdir_result)
......@@ -563,8 +513,6 @@ class CasBasedDirectory(Directory):
if self._check_replacement(entry, path_prefix, result):
self._add_new_file(source_directory, entry)
result.files_written.append(relative_pathname)
print("...Order of elements is now {}".format(", ".join(self.index.keys())))
return result
......@@ -621,17 +569,6 @@ class CasBasedDirectory(Directory):
x = self._resolve_symlink(symlink_node, force_create=False)
return isinstance(x, CasBasedDirectory)
def _verify_unique(self):
# Verifies that there are no duplicate names in this directory or subdirectories.
names = []
for entrylist in [self.pb2_directory.files, self.pb2_directory.directories, self.pb2_directory.symlinks]:
for e in entrylist:
if e.name in names:
raise VirtualDirectoryError("Duplicate entry for name {} found".format(e.name))
names.append(e.name)
for d in self.pb2_directory.directories:
self.index[d.name].buildstream_object._verify_unique()
def _partial_import_cas_into_cas(self, source_directory, files, path_prefix="", file_list_required=True):
""" Import only the files and symlinks listed in 'files' from source_directory to this one.
Args:
......@@ -640,11 +577,9 @@ class CasBasedDirectory(Directory):
path_prefix (str): Prefix used to add entries to the file list result.
file_list_required: Whether to update the file list while processing.
"""
print("Beginning partial import of {} into {}. Files are: >{}<".format(source_directory, self, ", ".join(files)))
result = FileListResult()
processed_directories = set()
for f in files:
#if f == ".": continue
fullname = os.path.join(path_prefix, f)
components = f.split(os.path.sep)
if len(components)>1:
......@@ -664,12 +599,9 @@ class CasBasedDirectory(Directory):
else:
dest_subdir = x
else:
print("Importing {}: {} does not exist in {}, so it is created as a directory".format(f, dirname, self))
self.create_directory(dirname)
dest_subdir = self._resolve_symlink_or_directory(dirname)
src_subdir = source_directory.descend(dirname)
print("Now recursing into {} to continue adding {}".format(src_subdir, f))
import_result = dest_subdir._partial_import_cas_into_cas(src_subdir, subcomponents,
path_prefix=fullname, file_list_required=file_list_required)
result.combine(import_result)
......@@ -693,23 +625,17 @@ class CasBasedDirectory(Directory):
else:
# We're importing a file or symlink - replace anything with the same name.
print("Import of file/symlink {} into this directory. Removing anything existing...".format(f))
print(" ... ordering of nodes in this dir was: {}".format(self.index.keys()))
print(" ... symlinks were {}".format([x.name for x in self.pb2_directory.symlinks]))
importable = self._check_replacement(f, path_prefix, result)
if importable:
print(" ... after replacement of '{}', symlinks are now {}".format(f, [x.name for x in self.pb2_directory.symlinks]))
item = source_directory.index[f].pb_object
if isinstance(item, remote_execution_pb2.FileNode):
print(" ... importing file")
filenode = self.pb2_directory.files.add(digest=item.digest, name=f,
is_executable=item.is_executable)
self.index[f] = IndexEntry(filenode, modified=(fullname in result.overwritten))
else:
print(" ... importing symlink")
assert(isinstance(item, remote_execution_pb2.SymlinkNode))
self._add_new_link_direct(name=f, target=item.target)
print(" ... symlinks are now {}".format([x.name for x in self.pb2_directory.symlinks]))
print(" ... ordering of nodes in this dir is now: {}".format(self.index.keys()))
return result
def transfer_node_contents(destination, source):
......@@ -754,12 +680,12 @@ class CasBasedDirectory(Directory):
replace one directory with another's hash, without doing any recursion.
"""
if files is None:
#return self._full_import_cas_into_cas(source_directory, can_hardlink=True)
files = list(source_directory.list_relative_paths())
print("Extracted all files from source directory '{}': {}".format(source_directory, files))
files = source_directory.list_relative_paths()
# You must pass a list into _partial_import (not a generator)
return self._partial_import_cas_into_cas(source_directory, list(files))
def _describe(self, thing):
""" Only used by showdiff, and as such, not called """
# Describes protocol buffer objects
if isinstance(thing, remote_execution_pb2.DirectoryNode):
return "directory called {}".format(thing.name)
......@@ -770,10 +696,8 @@ class CasBasedDirectory(Directory):
else:
return "strange thing"
def showdiff(self, other):
print("Diffing {} and {}:".format(self, other))
""" An old function used to show differences between two directories. No longer in use. """
def compare_list(l1, l2, name):
item2 = None
index = 0
......@@ -848,39 +772,10 @@ class CasBasedDirectory(Directory):
print("Directory before import: {}".format(self.show_files_recursive()))
# Sync self
self._recalculate_recursing_down()
if self.parent:
self.parent._recalculate_recursing_up(self)
duplicate_test = False
print("Original CAS before CAS-based import: {}".format(self.show_files_recursive()))
print("Original CAS hash: {}".format(self.ref.hash))
duplicate_cas = None
self._verify_unique()
if isinstance(external_pathspec, CasBasedDirectory):
if duplicate_test:
duplicate_cas = CasBasedDirectory(self.context, ref=copy.copy(self.ref))
duplicate_cas._verify_unique()
print("Duplicated CAS before file-based import: {}".format(duplicate_cas.show_files_recursive()))
print("Duplicate CAS hash: {}".format(duplicate_cas.ref.hash))
print("-"*80 + "Performing direct CAS-to-CAS import")
result = self._import_cas_into_cas(external_pathspec, files=files)
self._verify_unique()
print("Result of cas-to-cas import: {}".format(self.show_files_recursive()))
print("-"*80 + "Performing round-trip import via file system")
if duplicate_test:
with tempfile.TemporaryDirectory(prefix="roundtrip") as tmpdir:
external_pathspec.export_files(tmpdir)
if files is None:
files = list(list_relative_paths(tmpdir))
print("Importing from filesystem: filelist is: {}".format(files))
duplicate_cas._import_files_from_directory(tmpdir, files=files)
duplicate_cas._recalculate_recursing_down()
if duplicate_cas.parent:
duplicate_cas.parent._recalculate_recursing_up(duplicate_cas)
print("Result of direct import: {}".format(duplicate_cas.show_files_recursive()))
else:
print("-"*80 + "Performing initial import")
if isinstance(external_pathspec, FileBasedDirectory):
......@@ -904,10 +799,6 @@ class CasBasedDirectory(Directory):
self._recalculate_recursing_down()
if self.parent:
self.parent._recalculate_recursing_up(self)
if duplicate_cas:
if duplicate_cas.ref.hash != self.ref.hash:
self.showdiff(duplicate_cas)
raise VirtualDirectoryError("Mismatch between file-imported result {} and cas-to-cas imported result {}.".format(duplicate_cas.ref.hash,self.ref.hash))
return result
......
tests/storage/virtual_directory_import.py
View file @ 711c9847
import os
import pytest
import random
import copy
import tempfile
from tests.testutils import cli
from buildstream.storage import CasBasedDirectory
from buildstream.storage import FileBasedDirectory
from buildstream import utils
class FakeContext():
def __init__(self):
......@@ -84,7 +87,6 @@ def generate_random_root(rootno, directory):
os.symlink(symlink_destination, target)
description = "symlink pointing to {}".format(symlink_destination)
things.append(os.path.join(location, thingname))
print("Generated {}/{}, a {}".format(rootdir, things[-1], description))
def file_contents(path):
......@@ -160,15 +162,24 @@ def _import_test(tmpdir, original, overlay, generator_function, verify_contents=
generator_function(overlay, tmpdir)
d = create_new_casdir(original, fake_context, tmpdir)
#duplicate_cas = CasBasedDirectory(fake_context, ref=copy.copy(d.ref))
duplicate_cas = create_new_casdir(original, fake_context, tmpdir)
assert duplicate_cas.ref.hash == d.ref.hash
d2 = create_new_casdir(overlay, fake_context, tmpdir)
print("Importing dir {} into {}".format(overlay, original))
d.import_files(d2)
d.export_files(os.path.join(tmpdir, "output"))
export_dir = os.path.join(tmpdir, "output")
roundtrip_dir = os.path.join(tmpdir, "roundtrip")
d2.export_files(roundtrip_dir)
d.export_files(export_dir)
if verify_contents:
for item in root_filesets[overlay - 1]:
(path, typename, content) = item
realpath = resolve_symlinks(path, os.path.join(tmpdir, "output"))
realpath = resolve_symlinks(path, export_dir)
if typename == 'F':
if os.path.isdir(realpath) and directory_not_empty(realpath):
# The file should not have overwritten the directory in this case.
......@@ -189,10 +200,21 @@ def _import_test(tmpdir, original, overlay, generator_function, verify_contents=
assert os.path.lexists(realpath)
# Now do the same thing with filebaseddirectories and check the contents match
d3 = create_new_casdir(original, fake_context, tmpdir)
d4 = create_new_filedir(overlay, tmpdir)
d3.import_files(d2)
assert d.ref.hash == d3.ref.hash
files = list(utils.list_relative_paths(roundtrip_dir))
print("Importing from filesystem: filelist is: {}".format(files))
duplicate_cas._import_files_from_directory(roundtrip_dir, files=files)
duplicate_cas._recalculate_recursing_down()
if duplicate_cas.parent:
duplicate_cas.parent._recalculate_recursing_up(duplicate_cas)
print("Result of direct import: {}".format(duplicate_cas.show_files_recursive()))
assert duplicate_cas.ref.hash == d.ref.hash
#d3 = create_new_casdir(original, fake_context, tmpdir)
#d4 = create_new_filedir(overlay, tmpdir)
#d3.import_files(d2)
#assert d.ref.hash == d3.ref.hash
@pytest.mark.parametrize("original,overlay", combinations(range(1,len(root_filesets)+1)))
def test_fixed_cas_import(cli, tmpdir, original, overlay):
......