.gitlab-ci.yml

@@ -143,7 +143,6 @@ docs:
   - pip3 install sphinx-click
   - pip3 install sphinx_rtd_theme
   - cd dist && ./unpack.sh && cd buildstream
-  - pip3 install .
   - make BST_FORCE_SESSION_REBUILD=1 -C doc
   - cd ../..
   - mv dist/buildstream/doc/build/html public

.gitlab/issue_templates/bst_task.md

@@ -6,9 +6,9 @@
 
 [//]: # (Short summary of the action to be executed)
 
-* [  ] Action 1
-* [  ] Action 2
-* [  ] Action 3
+* [ ] Action 1
+* [ ] Action 2
+* [ ] Action 3
 
 ## Acceptance Criteria
 

HACKING.rst

@@ -261,9 +261,6 @@ using pip or some other mechanism::
   # Additional optional dependencies required
   pip3 install --user arpy
 
-Furthermore, the documentation build requires that BuildStream itself
-be installed, as it will be used in the process of generating its docs.
-
 To build the documentation, just run the following::
 
   make -C doc

buildstream/__main__.py

+##################################################################
+#                      Private Entry Point                       #
+##################################################################
+#
+# This allows running the cli when BuildStream is uninstalled,
+# as long as BuildStream repo is in PYTHONPATH, one can run it
+# with:
+#
+#    python3 -m buildstream [program args]
+#
+# This is used when we need to run BuildStream before installing,
+# like when we build documentation.
+#
+if __name__ == '__main__':
+    # pylint: disable=no-value-for-parameter
+    from ._frontend.cli import cli
+    cli()

buildstream/_scheduler/queues/pullqueue.py

@@ -29,7 +29,7 @@ class PullQueue(Queue):
 
     action_name = "Pull"
     complete_name = "Pulled"
-    resources = [ResourceType.UPLOAD]
+    resources = [ResourceType.DOWNLOAD]
 
     def process(self, element):
         # returns whether an artifact was downloaded or not

buildstream/_versions.py

@@ -23,7 +23,7 @@
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 12
+BST_FORMAT_VERSION = 13
 
 
 # The base BuildStream artifact version

buildstream/data/userconfig.yaml

@@ -35,13 +35,13 @@ cache:
 #
 scheduler:
 
-  # Maximum number of simultaneous source downloading tasks.
+  # Maximum number of simultaneous downloading tasks.
   fetchers: 10
 
   # Maximum number of simultaneous build tasks.
   builders: 4
 
-  # Maximum number of simultaneous artifact uploading tasks.
+  # Maximum number of simultaneous uploading tasks.
   pushers: 4
 
   # Maximum number of retries for network tasks.

buildstream/element.py

@@ -369,8 +369,8 @@ class Element(Plugin):
         generated script is run:
 
         - All element variables have been exported.
-        - The cwd is `self.get_variable('build_root')/self.normal_name`.
-        - $PREFIX is set to `self.get_variable('install_root')`.
+        - The cwd is `self.get_variable('build-root')/self.normal_name`.
+        - $PREFIX is set to `self.get_variable('install-root')`.
         - The directory indicated by $PREFIX is an empty directory.
 
         Files are expected to be installed to $PREFIX.

buildstream/plugins/sources/remote.py

@@ -35,6 +35,10 @@ remote - stage files from remote urls
    # If not specified, the basename of the url will be used.
    # filename: customfilename
 
+   # Optionally specify whether the downloaded file should be
+   # marked executable.
+   # executable: true
+
    # Specify the url. Using an alias defined in your project
    # configuration is encouraged. 'bst track' will update the
    # sha256sum in 'ref' to the downloaded file's sha256sum.
@@ -43,6 +47,8 @@ remote - stage files from remote urls
    # Specify the ref. It's a sha256sum of the file you download.
    ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
 
+
+
 .. note::
 
    The ``remote`` plugin is available since :ref:`format version 10 <project_format_version>`
@@ -60,22 +66,31 @@ class RemoteSource(DownloadableFileSource):
         super().configure(node)
 
         self.filename = self.node_get_member(node, str, 'filename', os.path.basename(self.url))
+        self.executable = self.node_get_member(node, bool, 'executable', False)
 
         if os.sep in self.filename:
             raise SourceError('{}: filename parameter cannot contain directories'.format(self),
                               reason="filename-contains-directory")
-        self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['filename'])
+        self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['filename', 'executable'])
 
     def get_unique_key(self):
-        return super().get_unique_key() + [self.filename]
+        return super().get_unique_key() + [self.filename, self.executable]
 
     def stage(self, directory):
         # Same as in local plugin, don't use hardlinks to stage sources, they
         # are not write protected in the sandbox.
         dest = os.path.join(directory, self.filename)
         with self.timed_activity("Staging remote file to {}".format(dest)):
+
             utils.safe_copy(self._get_mirror_file(), dest)
 
+            # To prevent user's umask introducing variability here, explicitly set
+            # file modes.
+            if self.executable:
+                os.chmod(dest, 0o755)
+            else:
+                os.chmod(dest, 0o644)
+
 
 def setup():
     return RemoteSource

doc/Makefile

@@ -31,6 +31,9 @@ ifneq ($(strip $(BST_FORCE_SESSION_REBUILD)),)
 BST2HTMLOPTS = --force
 endif
 
+# Help Python find buildstream and its plugins
+PYTHONPATH=$(CURDIR)/..:$(CURDIR)/../buildstream/plugins
+
 
 .PHONY: all clean templates templates-clean sessions sessions-prep sessions-clean html devhelp
 
@@ -65,7 +68,6 @@ define plugin-doc-skeleton
 endef
 
 
-# We set PYTHONPATH here because source/conf.py sys.modules hacks dont seem to help sphinx-build import the plugins
 all: html devhelp
 
 clean: templates-clean sessions-clean
@@ -103,7 +105,7 @@ sessions-prep:
 #
 sessions: sessions-prep
 	for file in $(wildcard sessions/*.run); do		\
-	    $(BST2HTML) $(BST2HTMLOPTS) --description $$file;	\
+	    PYTHONPATH=$(PYTHONPATH) $(BST2HTML) $(BST2HTMLOPTS) $$file;	\
 	done
 
 sessions-clean:
@@ -114,7 +116,7 @@ sessions-clean:
 #
 html devhelp: templates sessions
 	@echo "Building $@..."
-	PYTHONPATH=$(CURDIR)/../buildstream/plugins \
+	PYTHONPATH=$(PYTHONPATH) \
 	    $(SPHINXBUILD) -b $@ $(ALLSPHINXOPTS) "$(BUILDDIR)/$@" \
 	    $(wildcard source/*.rst) \
 	    $(wildcard source/tutorial/*.rst) \

doc/bst2html.py

@@ -204,7 +204,7 @@ def workdir(source_cache=None):
         yield (tempdir, bst_config_file, source_cache)
 
 
-# run_command()
+# run_bst_command()
 #
 # Runs a command
 #
@@ -216,10 +216,30 @@ def workdir(source_cache=None):
 # Returns:
 #    (str): The colorized combined stdout/stderr of BuildStream
 #
-def run_command(config_file, directory, command):
-    click.echo("Running command in directory '{}': bst {}".format(directory, command), err=True)
+def run_bst_command(config_file, directory, command):
+    click.echo("Running bst command in directory '{}': bst {}".format(directory, command), err=True)
 
-    argv = ['bst', '--colors', '--config', config_file] + shlex.split(command)
+    argv = ['python3', '-m', 'buildstream', '--colors', '--config', config_file] + shlex.split(command)
+    p = subprocess.Popen(argv, cwd=directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    out, _ = p.communicate()
+    return out.decode('utf-8').strip()
+
+
+# run_shell_command()
+#
+# Runs a command
+#
+# Args:
+#    directory (str): The project directory
+#    command (str): A shell command
+#
+# Returns:
+#    (str): The combined stdout/stderr of the shell command
+#
+def run_shell_command(directory, command):
+    click.echo("Running shell command in directory '{}': {}".format(directory, command), err=True)
+
+    argv = shlex.split(command)
     p = subprocess.Popen(argv, cwd=directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     out, _ = p.communicate()
     return out.decode('utf-8').strip()
@@ -373,12 +393,18 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
         # Get the command string
         command_str = _yaml.node_get(command, str, 'command')
 
+        # Check whether this is a shell command and not a bst command
+        is_shell = _yaml.node_get(command, bool, 'shell', default_value=False)
+
         # Check if there is fake output
         command_fake_output = _yaml.node_get(command, str, 'fake-output', default_value=None)
 
         # Run the command, or just use the fake output
         if command_fake_output is None:
-            command_out = run_command(config_file, directory, command_str)
+            if is_shell:
+                command_out = run_shell_command(directory, command_str)
+            else:
+                command_out = run_bst_command(config_file, directory, command_str)
         else:
             command_out = command_fake_output
 
@@ -414,14 +440,8 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
 @click.option('--palette', '-p', default='tango',
               type=click.Choice(['solarized', 'solarized-xterm', 'tango', 'xterm', 'console']),
               help="Selects a palette for the output style")
-@click.option('--output', '-o',
-              type=click.Path(file_okay=True, dir_okay=False, writable=True),
-              help="A file to store the output")
-@click.option('--description', '-d',
-              type=click.Path(file_okay=True, dir_okay=False, readable=True),
-              help="A file describing what to do")
-@click.argument('command', type=click.STRING, nargs=-1)
-def run_bst(directory, force, source_cache, description, palette, output, command):
+@click.argument('description', click.Path(file_okay=True, dir_okay=False, readable=True))
+def run_bst(directory, force, source_cache, description, palette):
     """Run a bst command and capture stdout/stderr in html
 
     This command normally takes a description yaml file, see the HACKING
@@ -430,45 +450,8 @@ def run_bst(directory, force, source_cache, description, palette, output, comman
     if not source_cache and os.environ.get('BST_SOURCE_CACHE'):
         source_cache = os.environ['BST_SOURCE_CACHE']
 
-    if output is not None and not check_needs_build(None, output, force=force):
-        click.echo("No need to rebuild {}".format(output))
-        return 0
-
     with workdir(source_cache=source_cache) as (tempdir, config_file, source_cache):
-
-        if description:
-            run_session(description, tempdir, source_cache, palette, config_file, force)
-            return 0
-
-        # Run a command specified on the CLI
-        #
-        if not directory:
-            directory = os.getcwd()
-        else:
-            directory = os.path.abspath(directory)
-            directory = os.path.realpath(directory)
-
-        if not command:
-            command = []
-        command_str = " ".join(command)
-
-        # Run the command
-        #
-        command_out = run_command(config_file, directory, command_str)
-
-        # Generate a nice html div for this output
-        #
-        converted = generate_html(command_out, directory, config_file,
-                                  source_cache, tempdir, palette,
-                                  command_str)
-
-    if output is None:
-        click.echo(converted)
-    else:
-        outdir = os.path.dirname(output)
-        os.makedirs(outdir, exist_ok=True)
-        with open(output, 'wb') as f:
-            f.write(converted.encode('utf-8'))
+        run_session(description, tempdir, source_cache, palette, config_file, force)
 
     return 0
 

doc/sessions/developing.run

@@ -16,7 +16,8 @@ commands:
 
 # Apply a patch in the workspace
 - directory: ../examples/developing/
-  command: show hello.bst; patch workspace_hello/files/src/hello.c update.patch;
+  shell: True
+  command: patch workspace_hello/hello.c update.patch
 
 # Rebuild
 - directory: ../examples/developing/

doc/source/advanced-features/junction-elements.rst

@@ -41,7 +41,7 @@ This element consists of a script which calls hello.bst's hello command.
 Building callHello.bst,
 
 .. raw:: html
-   :file: ../sessions-stored/junctions-build.html
+   :file: ../sessions/junctions-build.html
 
 You can see that the hello.bst element and it's dependencies from the autotools
 project have been build as part of the pipeline for callHello.bst.
@@ -49,17 +49,18 @@ project have been build as part of the pipeline for callHello.bst.
 We can now invoke `bst shell`
 
 .. raw:: html
-   :file: ../sessions-stored/junctions-shell.html
+   :file: ../sessions/junctions-shell.html
 
 This runs the script files/callHello.sh which will makes use of the hello command from the hello.bst element in the autotools project.
 
+
 Cross-junction workspaces
 -------------------------
 You can open workspaces for elements in the project refered to by the junction
 using the syntax `bst open ${junction-name}:{element-name}`. In this example,
 
 .. raw:: html
-   :file: ../sessions-stored/junctions-workspace-open.html
+   :file: ../sessions/junctions-workspace-open.html
 
 This has opened a workspace for the hello.bst element from the autotools project.
 This workspace can now be used as normal.

doc/source/install_linux_distro.rst

@@ -132,8 +132,8 @@ For the default plugins::
 
 Ubuntu 16.04 LTS
 ^^^^^^^^^^^^^^^^
-On Ubuntu 16.04, neither `bubblewrap<https://github.com/projectatomic/bubblewrap/>`
-or `ostree<https://github.com/ostreedev/ostree>` are available in the official repositories.
+On Ubuntu 16.04, neither `bubblewrap <https://github.com/projectatomic/bubblewrap/>`_
+or `ostree <https://github.com/ostreedev/ostree>`_ are available in the official repositories.
 You will need to install them in whichever way you see fit. Refer the the upstream documentation
 for advice on this.
 
@@ -221,3 +221,15 @@ from `AUR <https://wiki.archlinux.org/index.php/Arch_User_Repository#Installing_
 Alternatively, use
 `buildstream-git <https://aur.archlinux.org/packages/buildstream-git>`_
 for the lastest version of the development branch.
+
+Fedora
+~~~~~~
+
+BuildStream is not yet in the official Fedora repositories, but you can
+install it from a Copr:
+
+  sudo dnf copr enable bochecha/buildstream
+  sudo dnf install buildstream
+
+Optionally, install the ``buildstream-docs`` package to have the BuildStream
+documentation in Devhelp or GNOME Builder.

tests/sources/remote.py

 import os
+import stat
 import pytest
 
 from buildstream._exceptions import ErrorDomain
@@ -82,7 +83,14 @@ def test_simple_file_build(cli, tmpdir, datafiles):
     result.assert_success()
     # Note that the url of the file in target.bst is actually /dir/file
     # but this tests confirms we take the basename
-    assert(os.path.exists(os.path.join(checkoutdir, 'file')))
+    checkout_file = os.path.join(checkoutdir, 'file')
+    assert(os.path.exists(checkout_file))
+
+    mode = os.stat(checkout_file).st_mode
+    # Assert not executable by anyone
+    assert(not (mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)))
+    # Assert not writeable by anyone other than me
+    assert(not (mode & (stat.S_IWGRP | stat.S_IWOTH)))
 
 
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file-custom-name'))
@@ -119,6 +127,7 @@ def test_unique_key(cli, tmpdir, datafiles):
     generate_project(project, tmpdir)
     assert cli.get_element_state(project, 'target.bst') == "fetch needed"
     assert cli.get_element_state(project, 'target-custom.bst') == "fetch needed"
+    assert cli.get_element_state(project, 'target-custom-executable.bst') == "fetch needed"
     # Try to fetch it
     result = cli.run(project=project, args=[
         'fetch', 'target.bst'
@@ -127,7 +136,31 @@ def test_unique_key(cli, tmpdir, datafiles):
     # We should download the file only once
     assert cli.get_element_state(project, 'target.bst') == 'buildable'
     assert cli.get_element_state(project, 'target-custom.bst') == 'buildable'
+    assert cli.get_element_state(project, 'target-custom-executable.bst') == 'buildable'
 
     # But the cache key is different because the 'filename' is different.
     assert cli.get_element_key(project, 'target.bst') != \
-        cli.get_element_key(project, 'target-custom.bst')
+        cli.get_element_key(project, 'target-custom.bst') != \
+        cli.get_element_key(project, 'target-custom-executable.bst')
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'unique-keys'))
+def test_executable(cli, tmpdir, datafiles):
+    '''This test confirms that the 'ecxecutable' parameter is honoured.
+    '''
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    checkoutdir = os.path.join(str(tmpdir), "checkout")
+    assert cli.get_element_state(project, 'target-custom-executable.bst') == "fetch needed"
+    # Try to fetch it
+    result = cli.run(project=project, args=[
+        'build', 'target-custom-executable.bst'
+    ])
+
+    result = cli.run(project=project, args=[
+        'checkout', 'target-custom-executable.bst', checkoutdir
+    ])
+    mode = os.stat(os.path.join(checkoutdir, 'some-custom-file')).st_mode
+    assert (mode & stat.S_IEXEC)
+    # Assert executable by anyone
+    assert(mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH))

tests/sources/remote/unique-keys/target-custom-executable.bst

+kind: import
+description: test
+sources:
+- kind: remote
+  url: tmpdir:/dir/file
+  ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  filename: some-custom-file
+  executable: true