Merge branch 'jonathan/junction-no-tmpdir' into 'master'

Stage junctions into .bst instead of a tmpdir

Closes #895

See merge request !1134

buildstream/_loader/loader.py

@@ -20,8 +20,6 @@
 import os
 from functools import cmp_to_key
 from collections.abc import Mapping
-import tempfile
-import shutil
 
 from .._exceptions import LoadError, LoadErrorReason
 from .. import Consistency
@@ -49,12 +47,10 @@ from .._message import Message, MessageType
 #    context (Context): The Context object
 #    project (Project): The toplevel Project object
 #    parent (Loader): A parent Loader object, in the case this is a junctioned Loader
-#    tempdir (str): A directory to cleanup with the Loader, given to the loader by a parent
-#                   loader in the case that this loader is a subproject loader.
 #
 class Loader():
 
-    def __init__(self, context, project, *, parent=None, tempdir=None):
+    def __init__(self, context, project, *, parent=None):
 
         # Ensure we have an absolute path for the base directory
         basedir = project.element_path
@@ -73,7 +69,6 @@ class Loader():
         self._options = project.options      # Project options (OptionPool)
         self._basedir = basedir              # Base project directory
         self._first_pass_options = project.first_pass_config.options  # Project options (OptionPool)
-        self._tempdir = tempdir              # A directory to cleanup
         self._parent = parent                # The parent loader
 
         self._meta_elements = {}  # Dict of resolved meta elements by name
@@ -159,30 +154,6 @@ class Loader():
 
         return ret
 
-    # cleanup():
-    #
-    # Remove temporary checkout directories of subprojects
-    #
-    def cleanup(self):
-        if self._parent and not self._tempdir:
-            # already done
-            return
-
-        # recurse
-        for loader in self._loaders.values():
-            # value may be None with nested junctions without overrides
-            if loader is not None:
-                loader.cleanup()
-
-        if not self._parent:
-            # basedir of top-level loader is never a temporary directory
-            return
-
-        # safe guard to not accidentally delete directories outside builddir
-        if self._tempdir.startswith(self._context.builddir + os.sep):
-            if os.path.exists(self._tempdir):
-                shutil.rmtree(self._tempdir)
-
     ###########################################
     #            Private Methods              #
     ###########################################
@@ -540,23 +511,28 @@ class Loader():
                                 "Subproject has no ref for junction: {}".format(filename),
                                 detail=detail)
 
-        if len(sources) == 1 and sources[0]._get_local_path():
+        workspace = element._get_workspace()
+        if workspace:
+            # If a workspace is open, load it from there instead
+            basedir = workspace.get_absolute_path()
+        elif len(sources) == 1 and sources[0]._get_local_path():
             # Optimization for junctions with a single local source
             basedir = sources[0]._get_local_path()
-            tempdir = None
         else:
             # Stage sources
-            os.makedirs(self._context.builddir, exist_ok=True)
-            basedir = tempfile.mkdtemp(prefix="{}-".format(element.normal_name), dir=self._context.builddir)
-            element._stage_sources_at(basedir, mount_workspaces=False)
-            tempdir = basedir
+            element._update_state()
+            basedir = os.path.join(self.project.directory, ".bst", "staged-junctions",
+                                   filename, element._get_cache_key())
+            if not os.path.exists(basedir):
+                os.makedirs(basedir, exist_ok=True)
+                element._stage_sources_at(basedir, mount_workspaces=False)
 
         # Load the project
         project_dir = os.path.join(basedir, element.path)
         try:
             from .._project import Project
             project = Project(project_dir, self._context, junction=element,
-                              parent_loader=self, tempdir=tempdir)
+                              parent_loader=self)
         except LoadError as e:
             if e.reason == LoadErrorReason.MISSING_PROJECT_CONF:
                 raise LoadError(reason=LoadErrorReason.INVALID_JUNCTION,

buildstream/_project.py

@@ -91,7 +91,7 @@ class ProjectConfig:
 class Project():
 
     def __init__(self, directory, context, *, junction=None, cli_options=None,
-                 default_mirror=None, parent_loader=None, tempdir=None):
+                 default_mirror=None, parent_loader=None):
 
         # The project name
         self.name = None
@@ -147,7 +147,7 @@ class Project():
         self._project_includes = None
 
         profile_start(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-'))
-        self._load(parent_loader=parent_loader, tempdir=tempdir)
+        self._load(parent_loader=parent_loader)
         profile_end(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-'))
 
         self._partially_loaded = True
@@ -389,8 +389,6 @@ class Project():
     # Cleans up resources used loading elements
     #
     def cleanup(self):
-        self.loader.cleanup()
-
         # Reset the element loader state
         Element._reset_load_state()
 
@@ -439,7 +437,7 @@ class Project():
     #
     # Raises: LoadError if there was a problem with the project.conf
     #
-    def _load(self, parent_loader=None, tempdir=None):
+    def _load(self, parent_loader=None):
 
         # Load builtin default
         projectfile = os.path.join(self.directory, _PROJECT_CONF_FILE)
@@ -505,8 +503,7 @@ class Project():
         self._fatal_warnings = _yaml.node_get(pre_config_node, list, 'fatal-warnings', default_value=[])
 
         self.loader = Loader(self._context, self,
-                             parent=parent_loader,
-                             tempdir=tempdir)
+                             parent=parent_loader)
 
         self._project_includes = Includes(self.loader, copy_tree=False)