Skip to content
Snippets Groups Projects
Commit 6e1883e0 authored by Tristan Maat's avatar Tristan Maat
Browse files

tarcache.py: Improve tar interface

parent e1cf9ec3
No related branches found
No related tags found
Hide whitespace changes
Inline Side-by-side
buildstream/_artifactcache/tarcache.py
+ 245
− 167
View file @ 6e1883e0
......@@ -20,17 +20,239 @@ def buildref(element, key):
# assume project and element names are not allowed to contain slashes
return '{0}/{1}/{2}'.format(project.name, element_name, key)
def tarpath(element, key, compression=None):
if compression is None:
compression = ''
else:
compression = '.' + compression
def tarpath(element, key):
project = element.get_project()
return os.path.join(project.name, element.normal_name, key + '.tar')
return os.path.join(project.name, element.normal_name, key + '.tar' + compression)
# A helper class that contains tar archive/extract functions
class Tar():
compression_flags = {
'bz2': 'j',
'xz': 'J',
'gz': 'z',
'': ''
}
# archive()
#
# Attempt to archive the given tarfile with the `tar` command,
# falling back to python's `tarfile` if this fails.
#
# Args:
# location (str): The path to the tar to create
# content (str): The path to the content to archvive
# compression (str): The compression method to use
#
# This is done since AIX tar does not support 2G+ files.
#
@classmethod
def archive(cls, location, content, compression=None):
if compression is None:
compression = ''
try:
cls._archive_with_tar(location, content, compression)
return
except tarfile.TarError:
pass
except ProgramNotFoundError:
pass
# If the former did not complete successfully, we try with
# python's tar implementation (since it's hard to detect
# specific issues with specific tar implementations - a
# fallback).
try:
cls._archive_with_python(location, content, compression)
except tarfile.TarError as e:
raise _ArtifactError("Failed to archive {}: {}"
.format(location, e)) from e
# extract()
#
# Attempt to extract the given tarfile with the `tar` command,
# falling back to python's `tarfile` if this fails.
#
# Args:
# location (str): The path to the tar to extract
# dest (str): The destination path to extract to
#
# This is done since python tarfile extraction is horrendously
# slow (2 hrs+ for base images).
#
@classmethod
def extract(cls, location, dest):
try:
cls._extract_with_tar(location, dest)
return
except tarfile.TarError:
pass
except ProgramNotFoundError:
pass
try:
cls._extract_with_python(location, dest)
except tarfile.TarError as e:
raise _ArtifactError("Failed to extract {}: {}"
.format(location, e)) from e
# _get_host_tar()
#
# Get the host tar command.
#
# Raises:
# ProgramNotFoundError: If the tar executable cannot be
# located
#
@classmethod
def _get_host_tar(cls):
tar_cmd = None
for potential_tar_cmd in ['gtar', 'tar']:
try:
tar_cmd = utils.get_host_tool(potential_tar_cmd)
break
except ProgramNotFoundError:
continue
# If we still couldn't find tar, raise the ProgramNotfounderror
if tar_cmd is None:
raise ProgramNotFoundError("Did not find tar in PATH: {}"
.format(os.environ.get('PATH')))
return tar_cmd
# _archive_with_tar()
#
# Archive with an implementation of the `tar` command
#
# Args:
# location (str): The path to the tar to create
# content (str): The path to the content to archvive
# compression (str): The compression method to use
#
# Raises:
# CompressionError: If the compression algorithm is not
# supported
# TarError: If an error occurs during extraction
# ProgramNotFoundError: If the tar executable cannot be
# located
#
@classmethod
def _archive_with_tar(cls, location, content, compression):
tar_cmd = cls._get_host_tar()
try:
flag = cls.compression_flags[compression]
except KeyError:
raise tarfile.CompressionError("Compression method '{}' is not supported"
.format(compression))
process = subprocess.Popen(
[tar_cmd, flag + 'caf', location, content],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
_, err = process.communicate()
if process.poll() != 0:
# Clean up in case the command failed in a broken state
try:
os.remove(location)
except FileNotFoundError:
pass
raise tarfile.TarError("Failed to archive '{}': {}"
.format(content, err.decode('utf8')))
# _archive_with_python()
#
# Archive with the python `tarfile` module
#
# Args:
# location (str): The path to the tar to create
# content (str): The path to the content to archvive
# compression (str): The compression method to use
#
# Raises:
# TarError: If an error occurs during extraction
#
@classmethod
def _archive_with_python(cls, location, content, compression):
with tarfile.open(location, mode='w:' + compression) as tar:
tar.add(content)
# _extract_with_tar()
#
# Extract with an implementation of the `tar` command
#
# Args:
# location (str): The path to the tar to extract
# dest (str): The destination path to extract to
#
# Raises:
# CompressionError: If the compression algorithm is not
# supported
# TarError: If an error occurs during extraction
#
@classmethod
def _extract_with_tar(cls, location, dest):
tar_cmd = cls._get_host_tar()
# Some tar implementations do not support compression auto-detection
compression = os.path.splitext(location)
try:
flag = cls.compression_flags[compression]
except KeyError:
raise tarfile.CompressionError("Compression method '{}' is not supported".format(compression))
# Some tar implementations do not support '-C'
with utils._pushd(dest):
process = subprocess.Popen(
[tar_cmd, flag + 'xf', location],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
_, err = process.communicate()
if process.poll() != 0:
raise tarfile.TarError("Failed to extract '{}': {}"
.format(location, err.decode('utf8')))
# _extract_with_python()
#
# Extract with the python `tarfile` module
#
# Args:
# location (str): The path to the tar to extract
# dest (str): The destination path to extract to
#
# Raises:
# TarError: If an error occurs during extraction
#
@classmethod
def _extract_with_python(cls, location, dest):
with tarfile.open(location) as tar:
tar.extractall(path=dest)
class TarCache(ArtifactCache):
def __init__(self, context):
super().__init__(context)
self.compression = 'bz2'
self.tardir = os.path.join(context.artifactdir, 'tar')
os.makedirs(self.tardir, exist_ok=True)
......@@ -64,7 +286,7 @@ class TarCache(ArtifactCache):
if not key:
return False
path = os.path.join(self.tardir, tarpath(element, key))
path = os.path.join(self.tardir, tarpath(element, key, compression=self.compression))
return os.path.isfile(path)
# remove()
......@@ -80,7 +302,7 @@ class TarCache(ArtifactCache):
if not key:
return
path = (os.path.join(self.tardir, tarpath(element, key)))
path = (os.path.join(self.tardir, tarpath(element, key, compression=self.compression)))
shutil.rmtree(path)
# commit()
......@@ -92,34 +314,28 @@ class TarCache(ArtifactCache):
# content () - The content to commit
#
def commit(self, element, content):
ref = tarpath(element, element._get_cache_key_for_build())
weak_ref = tarpath(element, element._get_cache_key(strength=_KeyStrength.WEAK))
ref = tarpath(element, element._get_cache_key_for_build(), compression=self.compression)
weak_ref = tarpath(element, element._get_cache_key(strength=_KeyStrength.WEAK), compression=self.compression)
os.makedirs(os.path.join(self.tardir, element.get_project().name, element.normal_name), exist_ok=True)
# Avoid using 'tar' on AIX due to a 2G limit on extracted files
platform = self.context._platform
if platform.get_platform() == platform.SupportedPlatforms.aix:
with utils._tempdir() as temp:
contained = os.path.join(temp, element._get_cache_key_for_build())
shutil.copytree(content, contained)
with utils._tempdir() as temp:
refdir = os.path.join(temp, element._get_cache_key_for_build())
shutil.copytree(content, refdir, symlinks=True)
self.tar_archive(element, os.path.join(self.tardir, ref), contained)
if ref != weak_ref:
weak_refdir = os.path.join(temp, element._get_cache_key(strength=_KeyStrength.WEAK))
shutil.copytree(content, weak_refdir, symlinks=True)
if ref == weak_ref:
return
with utils._pushd(temp):
Tar.archive(os.path.join(self.tardir, ref),
element._get_cache_key_for_build(),
compression=self.compression)
contained = os.path.join(temp, element._get_cache_key(strength=_KeyStrength.WEAK))
shutil.copytree(content, os.path.join(contained, os.path.basename(content)))
self.tar_archive(element, os.path.join(self.tardir, weak_ref), contained)
else:
with tarfile.open(os.path.join(self.tardir, ref), mode='a') as tar:
tar.add(content, element._get_cache_key_for_build())
with tarfile.open(os.path.join(self.tardir, weak_ref), mode='a') as tar:
tar.add(content, element._get_cache_key(strength=_KeyStrength.WEAK))
if ref != weak_ref:
Tar.archive(os.path.join(self.tardir, weak_ref),
element._get_cache_key(strength=_KeyStrength.WEAK),
compression=self.compression)
# extract()
#
......@@ -135,12 +351,12 @@ class TarCache(ArtifactCache):
key = element._get_cache_key()
ref = buildref(element, key)
path = tarpath(element, key)
path = tarpath(element, key, compression=self.compression)
if not os.path.isfile(os.path.join(self.tardir, path)):
key = element._get_cache_key(strength=_KeyStrength.WEAK)
ref = buildref(element, key)
path = tarpath(element, key)
path = tarpath(element, key, compression=self.compression)
if not os.path.isfile(os.path.join(self.tardir, path)):
raise _ArtifactError("Artifact missing for {}".format(ref))
......@@ -153,7 +369,7 @@ class TarCache(ArtifactCache):
os.makedirs(self.extractdir, exist_ok=True)
with utils._tempdir(dir=self.extractdir) as tmpdir:
self.tar_extract(os.path.join(self.tardir, path), tmpdir)
Tar.extract(os.path.join(self.tardir, path), tmpdir)
os.makedirs(os.path.join(self.extractdir, element.get_project().name, element.normal_name),
exist_ok=True)
......@@ -170,141 +386,3 @@ class TarCache(ArtifactCache):
.format(ref, e)) from e
return dest
def tar_archive(self, element, location, content):
tar_cmd = None
for potential_tar_cmd in ['gtar', 'tar']:
try:
tar_cmd = utils.get_host_tool(potential_tar_cmd)
break
except ProgramNotFoundError:
continue
# If this does not complete successfully, we try with
# python's tar implementation
if tar_cmd is not None:
try:
self.archive_with_tar(tar_cmd, location, content)
return
except tarfile.TarError:
pass
try:
self.archive_with_python(element, location, content)
except tarfile.TarError as e:
raise _ArtifactError("Failed to archive {}: {}"
.format(content, e)) from e
def archive_with_tar(self, tar_cmd, location, content):
process = subprocess.Popen(
[tar_cmd, 'caf', location, content],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
_, err = process.communicate()
if process.poll() != 0:
raise tarfile.TarError("Failed to archive '{}': {}"
.format(content, err.decode('utf8')))
def archive_with_python(self, element, location, content):
with tarfile.open(location, mode='a') as tar:
tar.add(content, element._get_cache_key_for_build())
with tarfile.open(location, mode='a') as tar:
tar.add(content, element._get_cache_key(strength=_KeyStrength.WEAK))
# tar_extract()
#
# Attempt to extract the given tarfile with the `tar` command,
# falling back to python's `tarfile` if this fails.
#
# Args:
# location (str): The path to the tar to extract
# dest (str): The destination path to extract to
#
# This is done since python tarfile extraction is horrendously
# slow (2 hrs+ for base images).
#
def tar_extract(self, location, dest):
tar_cmd = None
for potential_tar_cmd in ['gtar', 'tar']:
try:
tar_cmd = utils.get_host_tool(potential_tar_cmd)
break
except ProgramNotFoundError:
continue
# If this does not complete successfully, we try with
# python's tar implementation
if tar_cmd is not None:
try:
self.extract_with_tar(tar_cmd, location, dest)
return
except tarfile.TarError:
pass
try:
self.extract_with_python(location, dest)
except tarfile.TarError as e:
raise _ArtifactError("Failed to extract {}: {}"
.format(location, e)) from e
# extract_with_tar()
#
# Extract with an implementation of the `tar` command
#
# Args:
# location (str): The path to the tar to extract
# dest (str): The destination path to extract to
#
# Raises:
# CompressionError: If the compression algorithm is not
# supported
# TarError: If an error occurs during extraction
#
def extract_with_tar(self, tar_cmd, location, dest):
# Some tar implementations do not support compression auto-detection
compression = os.path.splitext(location)
flags = {
'bz2': 'j',
'xz': 'J',
'gz': 'z',
'tar': ''
}
try:
flag = flags[compression]
except KeyError:
raise tarfile.CompressionError("Compression method is not supported")
# Some tar implementations do not support '-C'
with utils._pushd(dest):
process = subprocess.Popen(
[tar_cmd, flag + 'xf', location],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
_, err = process.communicate()
if process.poll() != 0:
raise tarfile.TarError("Failed to extract '{}': {}"
.format(location, err.decode('utf8')))
# extract_with_python()
#
# Extract with the python `tarfile` module
#
# Args:
# location (str): The path to the tar to extract
# dest (str): The destination path to extract to
#
# Raises:
# TarError: If an error occurs during extraction
#
def extract_with_python(self, location, dest):
with tarfile.open(location) as tar:
tar.extractall(path=dest)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment