NEWS

@@ -27,6 +27,9 @@ buildstream 1.3.1
   o Generate Docker images from built artifacts using
     `contrib/bst-docker-import` script.
 
+  o Added Documentation on how to create out of source builds. This includes the
+    new the `conf-root` variable to make the process easier. And there has been
+    a bug fix to workspaces so they can be build in workspaces too.
 
 =================
 buildstream 1.1.5

buildstream/_versions.py

@@ -23,7 +23,7 @@
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 16
+BST_FORMAT_VERSION = 17
 
 
 # The base BuildStream artifact version

buildstream/buildelement.py

@@ -23,6 +23,50 @@ BuildElement - Abstract class for build elements
 The BuildElement class is a convenience element one can derive from for
 implementing the most common case of element.
 
+Built-in functionality
+----------------------
+
+The BuildElement base class provides built in functionality that could be
+overridden by the individual plugins.
+
+This section will give a brief summary of how some of the common features work,
+some of them or the variables they use will be further detailed in the following
+sections.
+
+Location for running commands
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The ``command-subdir`` variable sets where the build commands will be executed,
+if the directory does not exist it will be created, it is defined relative to
+the buildroot.
+
+Location for configuring the project
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The ``conf-root`` is defined by default as ``.`` and is the location that
+specific build element can use to look for build configuration files. This is
+used by elements such as autotools, cmake, distutils, meson, pip and qmake.
+
+The configuration commands are run in ``command-subdir`` and by default
+``conf-root`` is ``.`` so if ``conf-root`` is not set the configuration files
+in ``command-subdir`` will be used.
+
+By setting ``conf-root`` to ``"%{build-root}/Source/conf_location"`` and your
+source elements ``directory`` variable to ``Source`` then the configuration
+files in the directory ``conf_location`` with in your Source will be used.
+The current working directory when your configuration command is run will still
+be wherever you set your ``command-subdir`` to be, regardless of where the
+configure scripts are set with ``conf-root``.
+
+.. note::
+
+   The ``conf-root`` variable is available since :ref:`format version 17 <project_format_version>`
+
+Install Location
+~~~~~~~~~~~~~~~~
+
+You should not change the ``install-root`` variable as it is a special
+writeable location in the sandbox but it is useful when writing custom
+install instructions as it may need to be supplied as the ``DESTDIR``, please
+see the :mod:`cmake <elements.cmake>` build element for example.
 
 Abstract method implementations
 -------------------------------

buildstream/data/projectconfig.yaml

@@ -38,6 +38,9 @@ variables:
   # normally staged
   build-root: /buildstream/%{project-name}/%{element-name}
 
+  # Indicates where the build system should look for configuration files  
+  conf-root: .
+  
   # Indicates the build installation directory in the sandbox
   install-root: /buildstream-install
 

buildstream/plugins/elements/autotools.yaml

@@ -6,11 +6,11 @@ variables:
     export NOCONFIGURE=1;
 
     if [ -x %{conf-cmd} ]; then true;
-    elif [ -x autogen ]; then ./autogen;
-    elif [ -x autogen.sh ]; then ./autogen.sh;
-    elif [ -x bootstrap ]; then ./bootstrap;
-    elif [ -x bootstrap.sh ]; then ./bootstrap.sh;
-    else autoreconf -ivf;
+    elif [ -x %{conf-root}/autogen ]; then %{conf-root}/autogen;
+    elif [ -x %{conf-root}/autogen.sh ]; then %{conf-root}/autogen.sh;
+    elif [ -x %{conf-root}/bootstrap ]; then %{conf-root}/bootstrap;
+    elif [ -x %{conf-root}/bootstrap.sh ]; then %{conf-root}/bootstrap.sh;
+    else autoreconf -ivf %{conf-root};
     fi
 
   # Project-wide extra arguments to be passed to `configure`
@@ -22,7 +22,8 @@ variables:
   # For backwards compatibility only, do not use.
   conf-extra: ''
 
-  conf-cmd: ./configure
+  conf-cmd: "%{conf-root}/configure"
+  
   conf-args: |
 
     --prefix=%{prefix} \

buildstream/plugins/elements/cmake.yaml

@@ -23,7 +23,7 @@ variables:
 
   cmake: |
 
-    cmake -B%{build-dir} -H. -G"%{generator}" %{cmake-args}
+    cmake -B%{build-dir} -H"%{conf-root}" -G"%{generator}" %{cmake-args}
 
   make: cmake --build %{build-dir} -- ${JOBS}
   make-install: env DESTDIR="%{install-root}" cmake --build %{build-dir} --target install

buildstream/plugins/elements/distutils.yaml

@@ -8,7 +8,7 @@ variables:
 
   python-build: |
 
-    %{python} setup.py build
+    %{python} %{conf-root}/setup.py build
 
   install-args: |
 
@@ -17,7 +17,7 @@ variables:
 
   python-install: |
 
-    %{python} setup.py install %{install-args}
+    %{python} %{conf-root}/setup.py install %{install-args}
 
 
 config:

buildstream/plugins/elements/meson.yaml

@@ -28,7 +28,7 @@ variables:
     --mandir=%{mandir} \
     --infodir=%{infodir} %{meson-extra} %{meson-global} %{meson-local}
 
-  meson: meson %{build-dir} %{meson-args}
+  meson: meson %{conf-root} %{build-dir} %{meson-args}
 
   ninja: |
     ninja -j ${NINJAJOBS} -C %{build-dir}

buildstream/plugins/elements/pip.yaml

@@ -14,7 +14,7 @@ config:
   #
   install-commands:
   - |
-    %{pip} install --no-deps --root=%{install-root} --prefix=%{prefix} .
+    %{pip} install --no-deps --root=%{install-root} --prefix=%{prefix} %{conf-root} 
 
   # Commands for stripping debugging information out of
   # installed binaries

buildstream/plugins/elements/qmake.yaml

@@ -2,7 +2,7 @@
 
 variables:
 
-  qmake: qmake -makefile
+  qmake: qmake -makefile %{conf-root}
   make: make
   make-install: make -j1 INSTALL_ROOT="%{install-root}" install
 

buildstream/sandbox/_sandboxbwrap.py

@@ -108,9 +108,6 @@ class SandboxBwrap(Sandbox):
             bwrap_command += ['--unshare-uts', '--hostname', 'buildstream']
             bwrap_command += ['--unshare-ipc']
 
-        if cwd is not None:
-            bwrap_command += ['--chdir', cwd]
-
         # Give it a proc and tmpfs
         bwrap_command += [
             '--proc', '/proc',
@@ -151,6 +148,10 @@ class SandboxBwrap(Sandbox):
         if flags & SandboxFlags.ROOT_READ_ONLY:
             bwrap_command += ["--remount-ro", "/"]
 
+        if cwd is not None:
+            bwrap_command += ['--dir', cwd]
+            bwrap_command += ['--chdir', cwd]
+
         # Set UID and GUI
         if self.user_ns_available:
             bwrap_command += ['--unshare-user']
@@ -179,11 +180,6 @@ class SandboxBwrap(Sandbox):
         with ExitStack() as stack:
             stack.enter_context(mount_map.mounted(self))
 
-            # Ensure the cwd exists
-            if cwd is not None:
-                workdir = os.path.join(root_mount_source, cwd.lstrip(os.sep))
-                os.makedirs(workdir, exist_ok=True)
-
             # If we're interactive, we want to inherit our stdin,
             # otherwise redirect to /dev/null, ensuring process
             # disconnected from terminal.

buildstream/sandbox/_sandboxchroot.py

@@ -100,9 +100,8 @@ class SandboxChroot(Sandbox):
 
             # Ensure the cwd exists
             if cwd is not None:
-                workdir = os.path.join(root_mount_source, cwd.lstrip(os.sep))
+                workdir = os.path.join(rootfs, cwd.lstrip(os.sep))
                 os.makedirs(workdir, exist_ok=True)
-
             status = self.chroot(rootfs, command, stdin, stdout,
                                  stderr, cwd, env, flags)
 

buildstream/sandbox/sandbox.py

@@ -223,7 +223,9 @@ class Sandbox():
         .. note::
 
            The optional *cwd* argument will default to the value set with
-           :func:`~buildstream.sandbox.Sandbox.set_work_directory`
+           :func:`~buildstream.sandbox.Sandbox.set_work_directory` and this
+           function must make sure the directory will be created if it does
+           not exist yet, even if a workspace is being used.
         """
         raise ImplError("Sandbox of type '{}' does not implement run()"
                         .format(type(self).__name__))

buildstream/source.py

@@ -20,6 +20,19 @@
 Source - Base source class
 ==========================
 
+Built-in functionality
+----------------------
+
+The Source base class provides built in functionality that may be overridden
+by individual plugins.
+
+* Directory
+
+  The ``directory`` variable can be set for all sources of a type in project.conf
+  or per source within a element.
+
+  This sets the location within the build root that the content of the source
+  will be loaded in to. If the location does not exist, it will be created.
 
 .. _core_source_abstract_methods:
 

tests/format/variables.py

@@ -19,10 +19,10 @@ DATA_DIR = os.path.join(
 @pytest.mark.parametrize("target,varname,expected", [
     ('autotools.bst', 'make-install', "make -j1 DESTDIR=\"/buildstream-install\" install"),
     ('cmake.bst', 'cmake',
-     "cmake -B_builddir -H. -G\"Unix Makefiles\" -DCMAKE_INSTALL_PREFIX:PATH=\"/usr\" \\\n" +
+     "cmake -B_builddir -H\".\" -G\"Unix Makefiles\" " + "-DCMAKE_INSTALL_PREFIX:PATH=\"/usr\" \\\n" +
      "-DCMAKE_INSTALL_LIBDIR=lib   "),
     ('distutils.bst', 'python-install',
-     "python3 setup.py install --prefix \"/usr\" \\\n" +
+     "python3 ./setup.py install --prefix \"/usr\" \\\n" +
      "--root \"/buildstream-install\""),
     ('makemaker.bst', 'configure', "perl Makefile.PL PREFIX=/buildstream-install/usr"),
     ('modulebuild.bst', 'configure', "perl Build.PL --prefix \"/buildstream-install/usr\""),
@@ -45,10 +45,10 @@ def test_defaults(cli, datafiles, tmpdir, target, varname, expected):
 @pytest.mark.parametrize("target,varname,expected", [
     ('autotools.bst', 'make-install', "make -j1 DESTDIR=\"/custom/install/root\" install"),
     ('cmake.bst', 'cmake',
-     "cmake -B_builddir -H. -G\"Ninja\" -DCMAKE_INSTALL_PREFIX:PATH=\"/opt\" \\\n" +
+     "cmake -B_builddir -H\".\" -G\"Ninja\" " + "-DCMAKE_INSTALL_PREFIX:PATH=\"/opt\" \\\n" +
      "-DCMAKE_INSTALL_LIBDIR=lib   "),
     ('distutils.bst', 'python-install',
-     "python3 setup.py install --prefix \"/opt\" \\\n" +
+     "python3 ./setup.py install --prefix \"/opt\" \\\n" +
      "--root \"/custom/install/root\""),
     ('makemaker.bst', 'configure', "perl Makefile.PL PREFIX=/custom/install/root/opt"),
     ('modulebuild.bst', 'configure', "perl Build.PL --prefix \"/custom/install/root/opt\""),

tests/integration/autotools.py

@@ -38,6 +38,30 @@ def test_autotools_build(cli, tmpdir, datafiles):
                                '/usr/share/doc/amhello/README'])
 
 
+# Test that an autotools build 'works' - we use the autotools sample
+# amhello project for this.
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+def test_autotools_confroot_build(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    checkout = os.path.join(cli.directory, 'checkout')
+    element_name = 'autotools/amhelloconfroot.bst'
+
+    result = cli.run(project=project, args=['build', element_name])
+    assert result.exit_code == 0
+
+    result = cli.run(project=project, args=['checkout', element_name, checkout])
+    assert result.exit_code == 0
+
+    assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin',
+                               '/usr/share', '/usr/lib/debug',
+                               '/usr/lib/debug/usr', '/usr/lib/debug/usr/bin',
+                               '/usr/lib/debug/usr/bin/hello',
+                               '/usr/bin/hello', '/usr/share/doc',
+                               '/usr/share/doc/amhello',
+                               '/usr/share/doc/amhello/README'])
+
+
 # Test running an executable built with autotools
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_run(cli, tmpdir, datafiles):

tests/integration/cmake.py

@@ -32,6 +32,24 @@ def test_cmake_build(cli, tmpdir, datafiles):
                                '/usr/lib/debug/usr/bin/hello'])
 
 
+@pytest.mark.datafiles(DATA_DIR)
+def test_cmake_confroot_build(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    checkout = os.path.join(cli.directory, 'checkout')
+    element_name = 'cmake/cmakeconfroothello.bst'
+
+    result = cli.run(project=project, args=['build', element_name])
+    assert result.exit_code == 0
+
+    result = cli.run(project=project, args=['checkout', element_name, checkout])
+    assert result.exit_code == 0
+
+    assert_contains(checkout, ['/usr', '/usr/bin', '/usr/bin/hello',
+                               '/usr/lib/debug', '/usr/lib/debug/usr',
+                               '/usr/lib/debug/usr/bin',
+                               '/usr/lib/debug/usr/bin/hello'])
+
+
 @pytest.mark.datafiles(DATA_DIR)
 def test_cmake_run(cli, tmpdir, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)

tests/integration/project/elements/autotools/amhelloconfroot.bst

+kind: autotools
+description: Autotools test
+
+depends:
+- base.bst
+
+sources:
+- kind: tar
+  url: project_dir:/files/amhello.tar.gz
+  ref: 9ba123fa4e660929e9a0aa99f0c487b7eee59c5e7594f3284d015640b90f5590
+  directory: SourceFile
+
+variables:
+  conf-root: "%{build-root}/SourceFile"
+  command-subdir: build

tests/integration/project/elements/cmake/cmakeconfroothello.bst

+kind: cmake
+description: Cmake test
+
+depends:
+  - base.bst
+
+sources:
+  - kind: tar
+    directory: Source
+    url: project_dir:/files/cmakehello.tar.gz
+    ref: 508266f40dbc5875293bd24c4e50a9eb6b88cbacab742033f7b92f8c087b64e5
+
+variables:
+  conf-root: "%{build-root}/Source"
+  command-subdir: build

tests/integration/project/elements/workspace/workspace-commanddir.bst

+kind: manual
+description: Workspace mount test
+
+depends:
+  - filename: base.bst
+    type: build
+
+sources:
+  - kind: local
+    path: files/workspace-mount-src/
+
+variables:
+  command-subdir: build
+
+config:
+  build-commands:
+    - cc -c ../hello.c