diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 04089f9a8363ac0a8fbe32fb6812494d30da513b..f272d7f1b6a7e77d69666c31a9c5b520aa969ec7 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -18,8 +18,6 @@
# Tristan Maat <tristan.maat@codethink.co.uk>
import os
-from collections.abc import Mapping
-import string
from ._basecache import BaseCache
from .types import _KeyStrength
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index fe25efce6c0a1aa6cddb2e644811cc5160d0067a..c9bd92f70526a80b3202e47fa2de3e770be27858 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -144,21 +144,22 @@ class CASCache():
# ref (str): The ref whose directory to extract
# path (str): The destination path
# subdir (str): Optional specific dir to extract
+ # hash_dir (bool): whether to extract to a directory named the directories hash
#
# Raises:
# CASCacheError: In cases there was an OSError, or if the ref did not exist.
#
# Returns: path to extracted directory
#
- def extract(self, ref, path, subdir=None):
+ def extract(self, ref, path, subdir=None, hash_dir=True):
tree = self.resolve_ref(ref, update_mtime=True)
- originaldest = dest = os.path.join(path, tree.hash)
+ originaldest = dest = os.path.join(path, tree.hash) if hash_dir else str(path)
# If artifact is already extracted, check if the optional subdir
# has also been extracted. If the artifact has not been extracted
# a full extraction would include the optional subdir
- if os.path.isdir(dest):
+ if os.path.isdir(dest) and hash_dir is True:
if subdir:
if not os.path.isdir(os.path.join(dest, subdir)):
dest = os.path.join(dest, subdir)
diff --git a/buildstream/_context.py b/buildstream/_context.py
index 369a05a0533b909655902507b4c6f6ac2b53ef09..1588a86d48467c05c6891097808ad409ee8ded99 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -31,6 +31,7 @@ from ._exceptions import LoadError, LoadErrorReason, BstError
from ._message import Message, MessageType
from ._profile import Topics, profile_start, profile_end
from ._artifactcache import ArtifactCache
+from ._sourcecache import SourceCache
from ._cas import CASCache, CASQuota, CASCacheUsage
from ._workspaces import Workspaces, WorkspaceProjectCache
from .plugin import _plugin_lookup
@@ -64,6 +65,9 @@ class Context():
# The directory where various sources are stored
self.sourcedir = None
+ # specs for source cache remotes
+ self.source_cache_specs = None
+
# The directory where build sandboxes will be created
self.builddir = None
@@ -151,6 +155,7 @@ class Context():
self._message_handler = None
self._message_depth = deque()
self._artifactcache = None
+ self._sourcecache = None
self._projects = []
self._project_overrides = {}
self._workspaces = None
@@ -168,6 +173,7 @@ class Context():
# Args:
# config (filename): The user specified configuration file, if any
#
+
# Raises:
# LoadError
#
@@ -253,6 +259,9 @@ class Context():
# Load artifact share configuration
self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
+ # Load source cache config
+ self.source_cache_specs = SourceCache.specs_from_config_node(defaults)
+
self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults)
# Load pull build trees configuration
@@ -334,6 +343,13 @@ class Context():
def get_cache_usage(self):
return CASCacheUsage(self.get_casquota())
+ @property
+ def sourcecache(self):
+ if not self._sourcecache:
+ self._sourcecache = SourceCache(self)
+
+ return self._sourcecache
+
# add_project():
#
# Add a project to the context.
diff --git a/buildstream/_exceptions.py b/buildstream/_exceptions.py
index 8728f6e6904e4c46ec576558711cef5d790bc378..f2d34bcba2a8206d731971a2a95df718e802f6d8 100644
--- a/buildstream/_exceptions.py
+++ b/buildstream/_exceptions.py
@@ -272,6 +272,15 @@ class SandboxError(BstError):
super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason)
+# SourceCacheError
+#
+# Raised when errors are encountered in the source caches
+#
+class SourceCacheError(BstError):
+ def __init__(self, message, detail=None, reason=None):
+ super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason)
+
+
# ArtifactError
#
# Raised when errors are encountered in the artifact caches
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index 17d7c63fb2592f9f33d4400d4aa8eef3957ccbd4..7c2ba1d5592e6a5ef6dfc35a61f8e727cccdb1f5 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -507,6 +507,7 @@ class Loader():
element = Element._new_from_meta(meta_element)
element._preflight()
+ last_source = None
sources = list(element.sources())
for idx, source in enumerate(sources):
# Handle the case where a subproject needs to be fetched
@@ -529,6 +530,12 @@ class Loader():
raise LoadError(LoadErrorReason.SUBPROJECT_INCONSISTENT,
"Subproject has no ref for junction: {}".format(filename),
detail=detail)
+ last_source = source
+
+ if last_source:
+ previous_sources = list(sources)
+ previous_sources.pop()
+ last_source._cache(previous_sources)
workspace = element._get_workspace()
if workspace:
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 92ac8cab5ebda2a2ca51178d54338757d5bc7e79..0c49aff55663e381afdb89f655b73abfa1b7b883 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -31,6 +31,7 @@ from ._profile import Topics, profile_start, profile_end
from ._exceptions import LoadError, LoadErrorReason
from ._options import OptionPool
from ._artifactcache import ArtifactCache
+from ._sourcecache import SourceCache
from .sandbox import SandboxRemote
from ._elementfactory import ElementFactory
from ._sourcefactory import SourceFactory
@@ -137,6 +138,7 @@ class Project():
self._shell_host_files = [] # A list of HostMount objects
self.artifact_cache_specs = None
+ self.source_cache_specs = None
self.remote_execution_specs = None
self._sandbox = None
self._splits = None
@@ -236,7 +238,7 @@ class Project():
'artifacts', 'options',
'fail-on-overlap', 'shell', 'fatal-warnings',
'ref-storage', 'sandbox', 'mirrors', 'remote-execution',
- 'sources', '(@)'
+ 'sources', 'source-caches', '(@)'
])
# create_element()
@@ -572,6 +574,9 @@ class Project():
parent = self.junction._get_project()
self.artifact_cache_specs = parent.artifact_cache_specs + self.artifact_cache_specs
+ # Load source caches with pull/push config
+ self.source_cache_specs = SourceCache.specs_from_config_node(config, self.directory)
+
# Load remote-execution configuration for this project
project_specs = SandboxRemote.specs_from_config_node(config, self.directory)
override_specs = SandboxRemote.specs_from_config_node(
diff --git a/buildstream/_scheduler/queues/fetchqueue.py b/buildstream/_scheduler/queues/fetchqueue.py
index fc11fd1d19f21540bfab9a75e71b37972d3fb85b..92cd9fb3beb9f1bc98c0039375ae869cd51d9ca7 100644
--- a/buildstream/_scheduler/queues/fetchqueue.py
+++ b/buildstream/_scheduler/queues/fetchqueue.py
@@ -62,7 +62,7 @@ class FetchQueue(Queue):
# This will automatically skip elements which
# have no sources.
- if element._get_consistency() == Consistency.CACHED:
+ if element._get_consistency() == Consistency.STAGED:
return QueueStatus.SKIP
return QueueStatus.READY
@@ -74,5 +74,5 @@ class FetchQueue(Queue):
element._update_state()
- # Successful fetch, we must be CACHED now
- assert element._get_consistency() == Consistency.CACHED
+ # Successful fetch, we must be CACHED or STAGED now
+ assert element._get_consistency() >= Consistency.CACHED
diff --git a/buildstream/_sourcecache.py b/buildstream/_sourcecache.py
new file mode 100644
index 0000000000000000000000000000000000000000..961f17a3c3a361073529d9b85df363e6c8bb2b66
--- /dev/null
+++ b/buildstream/_sourcecache.py
@@ -0,0 +1,95 @@
+#
+# Copyright (C) 2019 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Raoul Hidalgo Charman <raoul.hidalgocharman@codethink.co.uk>
+#
+from ._cas import CASRemoteSpec
+from .storage._casbaseddirectory import CasBasedDirectory
+from ._basecache import BaseCache
+from ._exceptions import CASCacheError, SourceCacheError
+from . import utils
+from ._cachekey import generate_key
+
+
+class SourceCacheSpec(CASRemoteSpec):
+ """Holds configuration for a remote used for the source cache.
+
+ Args:
+ url (str): Location of the remote source cache
+ push (bool): Whether we should attempt to push sources to this cache,
+ in addition to pulling from it.
+ instance-name (str): Name if any, of instance of server
+ """
+
+
+class SourceCache(BaseCache):
+ """SourceCache()
+
+ Class that keeps config of remotes and deals with caching of sources.
+ """
+
+ spec_class = SourceCacheSpec
+ spec_name = "source_cache_specs"
+ spec_error = SourceCacheError
+ config_node_name = "source-caches"
+
+ def get_source_fullname(self, source):
+ """Get a string for the sources CAS ref"""
+ # Get previous sources
+ return "{}/{}/{}".format(
+ 'source',
+ source.get_kind(),
+ source.get_key)
+
+ def contains(self, source):
+ """Checks the local CAS for source"""
+ ref = self.get_source_fullname(source)
+ return self.cas.contains(ref)
+
+ def commit(self, source, previous_sources):
+ """Stages and captures a source into the local CAS"""
+ ref = self.get_source_fullname(source)
+
+ with utils._tempdir(dir=self.context.tmpdir) as tmpdir:
+ for previous_source in previous_sources:
+ previous_source.stage(tmpdir)
+ source.stage(tmpdir)
+ self.cas.commit([ref], tmpdir)
+
+ def export(self, source):
+ """Exports a source in the CAS to a virtual directory
+
+ Args:
+ source (Source): source we want to export
+ previous_sources ([Source]): previous sources
+ Returns:
+ CASBasedDirectory
+ """
+ ref = self.get_source_fullname(source)
+
+ try:
+ digest = self.cas.resolve_ref(ref)
+ except CASCacheError as e:
+ raise SourceCacheError("Error with ref: {}".format(e))
+
+ return CasBasedDirectory(self.cas, ref=digest)
+
+ def fetch(self, source, previous_sources, *, progress=None):
+ """Tries to fetch source from remote CAS's"""
+
+ def push(self, source):
+ """Pushes a source to configured remote CAS's"""
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index caaa48908c8de45181a117e3950135a3cf091df7..8e4095a828813b9a1afde68ec90ec9257bcf1242 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -616,7 +616,7 @@ class Stream():
raise StreamError("Element '{}' already has workspace defined at: {}"
.format(target.name, workspace.get_absolute_path()))
- if not no_checkout and target._get_consistency() != Consistency.CACHED:
+ if not no_checkout and target._get_consistency() != Consistency.STAGED:
raise StreamError("Could not stage uncached source. For {} ".format(target.name) +
"Use `--track` to track and " +
"fetch the latest version of the " +
diff --git a/buildstream/element.py b/buildstream/element.py
index 74f97df6f7ff901448e9562529224183f78d8118..7386e3df85f74d828b890e31640e77b3dc405d29 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -944,11 +944,16 @@ class Element(Plugin):
element = meta.project.create_element(meta, first_pass=meta.first_pass)
cls.__instantiated_elements[meta] = element
- # Instantiate sources
+ # Instantiate sources and generate their keys
+ previous_sources = []
for meta_source in meta.sources:
meta_source.first_pass = meta.kind == "junction"
source = meta.project.create_source(meta_source,
first_pass=meta.first_pass)
+
+ source._generate_key(previous_sources)
+ previous_sources.append(source)
+
redundant_ref = source._load_ref()
element.__sources.append(source)
@@ -1068,7 +1073,7 @@ class Element(Plugin):
# (bool): Whether this element can currently be built
#
def _buildable(self):
- if self._get_consistency() != Consistency.CACHED:
+ if self._get_consistency() < Consistency.CACHED:
return False
for dependency in self.dependencies(Scope.BUILD):
@@ -1476,9 +1481,24 @@ class Element(Plugin):
detail = "Element type either does not expect a buildtree or it was explictily cached without one."
self.warn("WARNING: {} Artifact contains an empty buildtree".format(self.name), detail=detail)
else:
- # No workspace or cached buildtree, stage source directly
- for source in self.sources():
- source._stage(temp_staging_directory)
+ # stage last source if that's staged
+ last_source = None
+ for last_source in self.sources():
+ pass
+ if last_source and last_source._get_consistency() == Consistency.STAGED:
+ self.info("new fangled staging")
+ import_dir = self._get_context().sourcecache.export(last_source)
+
+ # Cant import CasBasedDirectories into FileBasedDirectories
+ if isinstance(vdirectory, FileBasedDirectory):
+ import_dir.export_files(temp_staging_directory)
+ import_dir = temp_staging_directory
+
+ # No workspace, cached buildtree or staged sources, stage source directly
+ else:
+ self.warn("old fangled staging should no longer be called")
+ for source in self.sources():
+ source._stage(temp_staging_directory)
vdirectory.import_files(import_dir)
@@ -2091,11 +2111,17 @@ class Element(Plugin):
#
def _fetch(self):
previous_sources = []
+ source = None
for source in self.sources():
- if source._get_consistency() < Consistency.CACHED:
+ if source._get_consistency() < Consistency.STAGED:
source._fetch(previous_sources)
previous_sources.append(source)
+ # cache the last source
+ if source:
+ previous_sources.pop()
+ source._cache(previous_sources)
+
# _calculate_cache_key():
#
# Calculates the cache key
@@ -2173,11 +2199,16 @@ class Element(Plugin):
else:
# Determine overall consistency of the element
+ source_consistency = None
for source in self.__sources:
source._update_state()
source_consistency = source._get_consistency()
self.__consistency = min(self.__consistency, source_consistency)
+ # If the last source consistency is staged it's all good
+ if source_consistency == Consistency.STAGED:
+ self.__consistency = Consistency.STAGED
+
# __can_build_incrementally()
#
# Check if the element can be built incrementally, this
diff --git a/buildstream/source.py b/buildstream/source.py
index 97995a8da8a307f37c531f47949980f33b7b8ef3..94044123980ae6eb1169d626469afb2d05b26490 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -170,6 +170,10 @@ from . import _yaml, utils
from ._exceptions import BstError, ImplError, ErrorDomain
from ._loader.metasource import MetaSource
from ._projectrefs import ProjectRefStorage
+from .storage._casbaseddirectory import CasBasedDirectory
+from ._cachekey import generate_key
+
+from ._message import Message, MessageType
class SourceError(BstError):
@@ -289,6 +293,9 @@ class Source(Plugin):
super().__init__("{}-{}".format(meta.element_name, meta.element_index),
context, project, provenance, "source")
+ self.__source_cache = context.sourcecache
+ self.__key = None # Cache key used for the source cache
+
self.__element_name = meta.element_name # The name of the element owning this source
self.__element_index = meta.element_index # The index of the source in the owning element's source list
self.__element_kind = meta.element_kind # The kind of the element owning this source
@@ -461,7 +468,7 @@ class Source(Plugin):
Implementors should raise :class:`.SourceError` when encountering
some system error.
"""
- self.stage(directory)
+ self._stage(directory)
def get_source_fetchers(self):
"""Get the objects that are used for fetching
@@ -667,7 +674,7 @@ class Source(Plugin):
#
def _update_state(self):
- if self.__consistency < Consistency.CACHED:
+ if self.__consistency < Consistency.STAGED:
# Source consistency interrogations are silent.
context = self._get_context()
@@ -679,6 +686,10 @@ class Source(Plugin):
if self.__consistency == Consistency.CACHED:
self.validate_cache()
+ if (self.__consistency < Consistency.STAGED and
+ self.__source_cache.contains(self)):
+ self.__consistency = Consistency.STAGED
+
# Return cached consistency
#
def _get_consistency(self):
@@ -688,8 +699,14 @@ class Source(Plugin):
#
# Args:
# previous_sources (list): List of Sources listed prior to this source
+ # fetch_original (bool): whether to fetch full source, or use local CAS
#
- def _fetch(self, previous_sources):
+ def _fetch(self, previous_sources, fetch_original=False):
+
+ # return if we've got the source
+ if self.__source_cache.contains(self) and fetch_original is False:
+ self._get_context().message(Message(None, MessageType.INFO, "source cached"))
+ return
if self.BST_REQUIRES_PREVIOUS_SOURCES_FETCH:
self.__ensure_previous_sources(previous_sources)
@@ -700,6 +717,11 @@ class Source(Plugin):
else:
self.__do_fetch()
+ def _cache(self, previous_sources):
+ # stage the source into the source cache
+ self.info("Commiting to source cache")
+ self.__source_cache.commit(self, previous_sources)
+
# Wrapper for stage() api which gives the source
# plugin a fully constructed path considering the
# 'directory' option
@@ -707,7 +729,8 @@ class Source(Plugin):
def _stage(self, directory):
staging_directory = self.__ensure_directory(directory)
- self.stage(staging_directory)
+ cdir = self.__source_cache.export(self)
+ cdir.export_file(staging_directory)
# Wrapper for init_workspace()
def _init_workspace(self, directory):
@@ -956,6 +979,18 @@ class Source(Plugin):
else:
return None
+ def _generate_key(self, previous_sources):
+ keys = [self._get_unique_key(True)]
+
+ for previous_source in previous_sources:
+ keys.append(previous_sources._get_unique_key(True))
+
+ self.__key = generate_key(keys)
+
+ @property
+ def get_key(self):
+ return self.__key
+
#############################################################
# Local Private Methods #
#############################################################
diff --git a/buildstream/types.py b/buildstream/types.py
index 23d78b08cdde67716e508376ef096eed61bdbb8f..8955394c82aa80b779b63000facf08c2ea33271d 100644
--- a/buildstream/types.py
+++ b/buildstream/types.py
@@ -80,6 +80,13 @@ class Consistency():
source cache. Only cached sources can be staged.
"""
+ STAGED = 3
+ """STAGED
+
+ Sources are staged in the local CAS, but are not present unstaged in the
+ source cache.
+ """
+
class CoreWarnings():
"""CoreWarnings()
diff --git a/tests/sourcecache/__init__.py b/tests/sourcecache/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/tests/sourcecache/config.py b/tests/sourcecache/config.py
new file mode 100644
index 0000000000000000000000000000000000000000..8f70b537407ba296e5e57a77fe87465805a329f0
--- /dev/null
+++ b/tests/sourcecache/config.py
@@ -0,0 +1,57 @@
+#
+# Copyright (C) 2019 Bloomberg Finance L.P.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Raoul Hidalgo Chraman <raoul.hidalgocharman@codethink.co.uk>
+import os
+import pytest
+
+from buildstream import _yaml
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
+
+from buildstream.plugintestutils.runcli import cli
+
+DATA_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+# Assert that if either the client key or client cert is specified
+# without specifying its counterpart, we get a comprehensive LoadError
+# instead of an unhandled exception.
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize('config_key, config_value', [
+ ('client-cert', 'client.crt'),
+ ('client-key', 'client.key')
+])
+def test_missing_certs(cli, datafiles, config_key, config_value):
+ project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
+
+ project_conf = {
+ 'name': 'test',
+
+ 'source-caches': {
+ 'url': 'https://cache.example.com:12345',
+ 'push': 'true',
+ config_key: config_value
+ }
+ }
+ project_conf_file = os.path.join(project, 'project.conf')
+ _yaml.dump(project_conf, project_conf_file)
+
+ # Use `pull` here to ensure we try to initialize the remotes, triggering the error
+ #
+ # This does not happen for a simple `bst show`.
+ result = cli.run(project=project, args=['source', 'fetch', 'element.bst'])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
diff --git a/tests/sourcecache/consistency.py b/tests/sourcecache/consistency.py
new file mode 100644
index 0000000000000000000000000000000000000000..19bd7e5f5bedbc35863f6ad1795a7b0f272cdd14
--- /dev/null
+++ b/tests/sourcecache/consistency.py
@@ -0,0 +1,76 @@
+#
+# Copyright (C) 2019 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Raoul Hidalgo Charman <raoul.hidalgocharman@codethink.co.uk>
+#
+import filecmp
+import os
+import pytest
+
+from buildstream._context import Context
+from buildstream._project import Project
+from buildstream.source import Source
+from buildstream import _yaml
+
+from buildstream.plugintestutils.runcli import cli
+from tests.testutils.artifactshare import create_artifact_share
+
+DATA_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+def dummy_message_handler(message, context):
+ pass
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_source_staged(tmpdir, cli, datafiles):
+ project_dir = os.path.join(datafiles.dirname, datafiles.basename, 'project')
+ casdir = os.path.join(datafiles.dirname, datafiles.basename, 'cache', 'cas')
+ cachedir = os.path.join(str(tmpdir), 'cache')
+
+ cli.configure({
+ 'cachedir': cachedir
+ })
+
+ # set up minimal context
+ context = Context()
+ context.load()
+
+ # load project and sourcecache
+ project = Project(project_dir, context)
+ project.ensure_fully_loaded()
+ context.cachedir = cachedir
+ context.set_message_handler(dummy_message_handler)
+ sourcecache = context.sourcecache
+ cas = context.get_cascache()
+
+ cli.run(project=project_dir, args=["build", "import-bin.bst"])
+
+ # now check that the source is in the refs file, this is pretty messy but
+ # seems to be the only way to get the sources?
+ source = list(project.load_elements(["import-bin.bst"])[0].sources())[0]
+ assert sourcecache.contains(source)
+
+ # Extract the file and check it's the same as the one we imported
+ refname = sourcecache.get_source_fullname(source)
+ extractdir = os.path.join(str(tmpdir), "extract")
+ extract = cas.extract(refname, extractdir)
+ hellopath = os.path.join("usr", "bin", "hello")
+ file1 = os.path.join(extract, hellopath)
+ file2 = os.path.join(project_dir, "files", "bin-files", hellopath)
+
+ assert filecmp.cmp(file1, file2) is True
diff --git a/tests/sourcecache/missing-certs/certificates/client.crt b/tests/sourcecache/missing-certs/certificates/client.crt
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/tests/sourcecache/missing-certs/certificates/client.key b/tests/sourcecache/missing-certs/certificates/client.key
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/tests/sourcecache/missing-certs/element.bst b/tests/sourcecache/missing-certs/element.bst
new file mode 100644
index 0000000000000000000000000000000000000000..3c29b4ea1334556c03b724ce505bc234e80a1892
--- /dev/null
+++ b/tests/sourcecache/missing-certs/element.bst
@@ -0,0 +1 @@
+kind: autotools
diff --git a/tests/sourcecache/project/elements/compose-all.bst b/tests/sourcecache/project/elements/compose-all.bst
new file mode 100644
index 0000000000000000000000000000000000000000..ba47081b3ce496804ca4a236a6c021451c4c076e
--- /dev/null
+++ b/tests/sourcecache/project/elements/compose-all.bst
@@ -0,0 +1,12 @@
+kind: compose
+
+depends:
+- filename: import-bin.bst
+ type: build
+- filename: import-dev.bst
+ type: build
+
+config:
+ # Dont try running the sandbox, we dont have a
+ # runtime to run anything in this context.
+ integrate: False
diff --git a/tests/sourcecache/project/elements/import-bin.bst b/tests/sourcecache/project/elements/import-bin.bst
new file mode 100644
index 0000000000000000000000000000000000000000..a847c0c23de84a3b792fccf8506cb9cebc5aa1ea
--- /dev/null
+++ b/tests/sourcecache/project/elements/import-bin.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+- kind: local
+ path: files/bin-files
diff --git a/tests/sourcecache/project/elements/import-dev.bst b/tests/sourcecache/project/elements/import-dev.bst
new file mode 100644
index 0000000000000000000000000000000000000000..152a54667fe9de84c37971819fbb7fafb5df23c1
--- /dev/null
+++ b/tests/sourcecache/project/elements/import-dev.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+- kind: local
+ path: files/dev-files
diff --git a/tests/sourcecache/project/elements/target.bst b/tests/sourcecache/project/elements/target.bst
new file mode 100644
index 0000000000000000000000000000000000000000..ba489f1e856633bd929db86bdac09dfca00f6415
--- /dev/null
+++ b/tests/sourcecache/project/elements/target.bst
@@ -0,0 +1,9 @@
+kind: stack
+description: |
+
+ Main stack target for the bst build test
+
+depends:
+- import-bin.bst
+- import-dev.bst
+- compose-all.bst
diff --git a/tests/sourcecache/project/files/bin-files/usr/bin/hello b/tests/sourcecache/project/files/bin-files/usr/bin/hello
new file mode 100755
index 0000000000000000000000000000000000000000..f534a40837ced35eed6c6079228387302a4c9d65
--- /dev/null
+++ b/tests/sourcecache/project/files/bin-files/usr/bin/hello
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "Hello !"
diff --git a/tests/sourcecache/project/files/dev-files/usr/include/pony.h b/tests/sourcecache/project/files/dev-files/usr/include/pony.h
new file mode 100644
index 0000000000000000000000000000000000000000..40bd0c2e768fbd9238ce3a7f332c245fd950b64f
--- /dev/null
+++ b/tests/sourcecache/project/files/dev-files/usr/include/pony.h
@@ -0,0 +1,12 @@
+#ifndef __PONY_H__
+#define __PONY_H__
+
+#define PONY_BEGIN "Once upon a time, there was a pony."
+#define PONY_END "And they lived happily ever after, the end."
+
+#define MAKE_PONY(story) \
+ PONY_BEGIN \
+ story \
+ PONY_END
+
+#endif /* __PONY_H__ */
diff --git a/tests/sourcecache/project/project.conf b/tests/sourcecache/project/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..854e38693f391e65b8f9af10ee35679e4907ccb0
--- /dev/null
+++ b/tests/sourcecache/project/project.conf
@@ -0,0 +1,4 @@
+# Project config for frontend build test
+name: test
+
+element-path: elements
diff --git a/tests/sourcecache/project/project/elements/compose-all.bst b/tests/sourcecache/project/project/elements/compose-all.bst
new file mode 100644
index 0000000000000000000000000000000000000000..ba47081b3ce496804ca4a236a6c021451c4c076e
--- /dev/null
+++ b/tests/sourcecache/project/project/elements/compose-all.bst
@@ -0,0 +1,12 @@
+kind: compose
+
+depends:
+- filename: import-bin.bst
+ type: build
+- filename: import-dev.bst
+ type: build
+
+config:
+ # Dont try running the sandbox, we dont have a
+ # runtime to run anything in this context.
+ integrate: False
diff --git a/tests/sourcecache/project/project/elements/import-bin.bst b/tests/sourcecache/project/project/elements/import-bin.bst
new file mode 100644
index 0000000000000000000000000000000000000000..a847c0c23de84a3b792fccf8506cb9cebc5aa1ea
--- /dev/null
+++ b/tests/sourcecache/project/project/elements/import-bin.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+- kind: local
+ path: files/bin-files
diff --git a/tests/sourcecache/project/project/elements/import-dev.bst b/tests/sourcecache/project/project/elements/import-dev.bst
new file mode 100644
index 0000000000000000000000000000000000000000..152a54667fe9de84c37971819fbb7fafb5df23c1
--- /dev/null
+++ b/tests/sourcecache/project/project/elements/import-dev.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+- kind: local
+ path: files/dev-files
diff --git a/tests/sourcecache/project/project/elements/target.bst b/tests/sourcecache/project/project/elements/target.bst
new file mode 100644
index 0000000000000000000000000000000000000000..ba489f1e856633bd929db86bdac09dfca00f6415
--- /dev/null
+++ b/tests/sourcecache/project/project/elements/target.bst
@@ -0,0 +1,9 @@
+kind: stack
+description: |
+
+ Main stack target for the bst build test
+
+depends:
+- import-bin.bst
+- import-dev.bst
+- compose-all.bst
diff --git a/tests/sourcecache/project/project/files/bin-files/usr/bin/hello b/tests/sourcecache/project/project/files/bin-files/usr/bin/hello
new file mode 100755
index 0000000000000000000000000000000000000000..f534a40837ced35eed6c6079228387302a4c9d65
--- /dev/null
+++ b/tests/sourcecache/project/project/files/bin-files/usr/bin/hello
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "Hello !"
diff --git a/tests/sourcecache/project/project/files/dev-files/usr/include/pony.h b/tests/sourcecache/project/project/files/dev-files/usr/include/pony.h
new file mode 100644
index 0000000000000000000000000000000000000000..40bd0c2e768fbd9238ce3a7f332c245fd950b64f
--- /dev/null
+++ b/tests/sourcecache/project/project/files/dev-files/usr/include/pony.h
@@ -0,0 +1,12 @@
+#ifndef __PONY_H__
+#define __PONY_H__
+
+#define PONY_BEGIN "Once upon a time, there was a pony."
+#define PONY_END "And they lived happily ever after, the end."
+
+#define MAKE_PONY(story) \
+ PONY_BEGIN \
+ story \
+ PONY_END
+
+#endif /* __PONY_H__ */
diff --git a/tests/sourcecache/project/project/project.conf b/tests/sourcecache/project/project/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..854e38693f391e65b8f9af10ee35679e4907ccb0
--- /dev/null
+++ b/tests/sourcecache/project/project/project.conf
@@ -0,0 +1,4 @@
+# Project config for frontend build test
+name: test
+
+element-path: elements