Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • willsalmon/buildstream
  • CumHoleZH/buildstream
  • tchaik/buildstream
  • DCotyPortfolio/buildstream
  • jesusoctavioas/buildstream
  • patrickmmartin/buildstream
  • franred/buildstream
  • tintou/buildstream
  • alatiera/buildstream
  • martinblanchard/buildstream
  • neverdie22042524/buildstream
  • Mattlk13/buildstream
  • PServers/buildstream
  • phamnghia610909/buildstream
  • chiaratolentino/buildstream
  • eysz7-x-x/buildstream
  • kerrick1/buildstream
  • matthew-yates/buildstream
  • twofeathers/buildstream
  • mhadjimichael/buildstream
  • pointswaves/buildstream
  • Mr.JackWilson/buildstream
  • Tw3akG33k/buildstream
  • AlexFazakas/buildstream
  • eruidfkiy/buildstream
  • clamotion2/buildstream
  • nanonyme/buildstream
  • wickyjaaa/buildstream
  • nmanchev/buildstream
  • bojorquez.ja/buildstream
  • mostynb/buildstream
  • highpit74/buildstream
  • Demo112/buildstream
  • ba2014sheer/buildstream
  • tonimadrino/buildstream
  • usuario2o/buildstream
  • Angelika123456/buildstream
  • neo355/buildstream
  • corentin-ferlay/buildstream
  • coldtom/buildstream
  • wifitvbox81/buildstream
  • 358253885/buildstream
  • seanborg/buildstream
  • SotK/buildstream
  • DouglasWinship/buildstream
  • karansthr97/buildstream
  • louib/buildstream
  • bwh-ct/buildstream
  • robjh/buildstream
  • we88c0de/buildstream
  • zhengxian5555/buildstream
51 results
Show changes
Commits on Source (12)
Hide whitespace changes
Inline Side-by-side
Showing
with 397 additions and 221 deletions
NEWS
View file @ a322d5c0
......@@ -2,6 +2,12 @@
buildstream 1.3.1
=================
o BREAKING CHANGE: The bst source-bundle command has been removed. The
functionality it provided has been replaced by the `--include-build-scripts`
option of the `bst source-checkout` command. To produce a tarball containing
an element's sources and generated build scripts you can do the command
`bst source-checkout --include-build-scripts --tar foo.bst some-file.tar`
o BREAKING CHANGE: Default strip-commands have been removed as they are too
specific. Recommendation if you are building in Linux is to use the
ones being used in freedesktop-sdk project, for example
......@@ -86,6 +92,11 @@ buildstream 1.3.1
o Opening a workspace now creates a .bstproject.yaml file that allows buildstream
commands to be run from a workspace that is not inside a project.
o Specifying an element is now optional for some commands when buildstream is run
from inside a workspace - the 'build', 'checkout', 'fetch', 'pull', 'push',
'shell', 'show', 'source-checkout', 'track', 'workspace close' and 'workspace reset'
commands are affected.
=================
buildstream 1.1.5
......
buildstream/_context.py
View file @ a322d5c0
......@@ -32,7 +32,7 @@ from ._message import Message, MessageType
from ._profile import Topics, profile_start, profile_end
from ._artifactcache import ArtifactCache
from ._artifactcache.cascache import CASCache
from ._workspaces import Workspaces, WorkspaceProjectCache
from ._workspaces import Workspaces, WorkspaceProjectCache, WORKSPACE_PROJECT_FILE
from .plugin import _plugin_lookup
......@@ -47,9 +47,12 @@ from .plugin import _plugin_lookup
# verbosity levels and basically anything pertaining to the context
# in which BuildStream was invoked.
#
# Args:
# directory (str): The directory that buildstream was invoked in
#
class Context():
def __init__(self):
def __init__(self, directory=None):
# Filename indicating which configuration file was used, or None for the defaults
self.config_origin = None
......@@ -148,6 +151,7 @@ class Context():
self._log_handle = None
self._log_filename = None
self._cascache = None
self._directory = directory
# load()
#
......@@ -645,6 +649,20 @@ class Context():
self._cascache = CASCache(self.artifactdir)
return self._cascache
# guess_element()
#
# Attempts to interpret which element the user intended to run commands on
#
# Returns:
# (str) The name of the element, or None if no element can be guessed
def guess_element(self):
workspace_project_dir, _ = utils._search_upward_for_files(self._directory, [WORKSPACE_PROJECT_FILE])
if workspace_project_dir:
workspace_project = self._workspace_project_cache.get(workspace_project_dir)
return workspace_project.get_default_element()
else:
return None
# _node_get_option_str()
#
......
buildstream/_frontend/app.py
View file @ a322d5c0
......@@ -164,7 +164,7 @@ class App():
# Load the Context
#
try:
self.context = Context()
self.context = Context(directory)
self.context.load(config)
except BstError as e:
self._error_exit(e, "Error loading user configuration")
......
buildstream/_frontend/cli.py
View file @ a322d5c0
......@@ -316,10 +316,15 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
if track_save:
click.echo("WARNING: --track-save is deprecated, saving is now unconditional", err=True)
if track_all:
track_ = elements
with app.initialized(session_name="Build"):
if not all_ and not elements:
guessed_target = app.context.guess_element()
if guessed_target:
elements = (guessed_target,)
if track_all:
track_ = elements
app.stream.build(elements,
track_targets=track_,
track_except=track_except,
......@@ -371,6 +376,11 @@ def fetch(app, elements, deps, track_, except_, track_cross_junctions):
deps = PipelineSelection.ALL
with app.initialized(session_name="Fetch"):
if not elements:
guessed_target = app.context.guess_element()
if guessed_target:
elements = (guessed_target,)
app.stream.fetch(elements,
selection=deps,
except_targets=except_,
......@@ -407,6 +417,11 @@ def track(app, elements, deps, except_, cross_junctions):
all: All dependencies of all specified elements
"""
with app.initialized(session_name="Track"):
if not elements:
guessed_target = app.context.guess_element()
if guessed_target:
elements = (guessed_target,)
# Substitute 'none' for 'redirect' so that element redirections
# will be done
if deps == 'none':
......@@ -442,7 +457,13 @@ def pull(app, elements, deps, remote):
none: No dependencies, just the element itself
all: All dependencies
"""
with app.initialized(session_name="Pull"):
if not elements:
guessed_target = app.context.guess_element()
if guessed_target:
elements = (guessed_target,)
app.stream.pull(elements, selection=deps, remote=remote)
......@@ -475,6 +496,11 @@ def push(app, elements, deps, remote):
all: All dependencies
"""
with app.initialized(session_name="Push"):
if not elements:
guessed_target = app.context.guess_element()
if guessed_target:
elements = (guessed_target,)
app.stream.push(elements, selection=deps, remote=remote)
......@@ -545,6 +571,11 @@ def show(app, elements, deps, except_, order, format_):
$'---------- %{name} ----------\\n%{vars}'
"""
with app.initialized():
if not elements:
guessed_target = app.context.guess_element()
if guessed_target:
elements = (guessed_target,)
dependencies = app.stream.load_selection(elements,
selection=deps,
except_targets=except_)
......@@ -573,7 +604,7 @@ def show(app, elements, deps, except_, order, format_):
help="Mount a file or directory into the sandbox")
@click.option('--isolate', is_flag=True, default=False,
help='Create an isolated build sandbox')
@click.argument('element',
@click.argument('element', required=False,
type=click.Path(readable=False))
@click.argument('command', type=click.STRING, nargs=-1)
@click.pass_obj
......@@ -604,6 +635,11 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
scope = Scope.RUN
with app.initialized():
if not element:
element = app.context.guess_element()
if not element:
raise AppError('Missing argument "ELEMENT".')
dependencies = app.stream.load_selection((element,), selection=PipelineSelection.NONE)
element = dependencies[0]
prompt = app.shell_prompt(element)
......@@ -641,15 +677,24 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
help="Create a tarball from the artifact contents instead "
"of a file tree. If LOCATION is '-', the tarball "
"will be dumped to the standard output.")
@click.argument('element',
@click.argument('element', required=False,
type=click.Path(readable=False))
@click.argument('location', type=click.Path())
@click.argument('location', type=click.Path(), required=False)
@click.pass_obj
def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
"""Checkout a built artifact to the specified location
"""
from ..element import Scope
if not element and not location:
click.echo("ERROR: LOCATION is not specified", err=True)
sys.exit(-1)
if element and not location:
# Nasty hack to get around click's optional args
location = element
element = None
if hardlinks and tar:
click.echo("ERROR: options --hardlinks and --tar conflict", err=True)
sys.exit(-1)
......@@ -662,6 +707,11 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
scope = Scope.NONE
with app.initialized():
if not element:
element = app.context.guess_element()
if not element:
raise AppError('Missing argument "ELEMENT".')
app.stream.checkout(element,
location=location,
force=force,
......@@ -675,6 +725,8 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
# Source Checkout Command #
##################################################################
@cli.command(name='source-checkout', short_help='Checkout sources for an element')
@click.option('--force', '-f', default=False, is_flag=True,
help="Allow files to be overwritten")
@click.option('--except', 'except_', multiple=True,
type=click.Path(readable=False),
help="Except certain dependencies")
......@@ -683,19 +735,40 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
help='The dependencies whose sources to checkout (default: none)')
@click.option('--fetch', 'fetch_', default=False, is_flag=True,
help='Fetch elements if they are not fetched')
@click.argument('element',
type=click.Path(readable=False))
@click.argument('location', type=click.Path())
@click.option('--tar', 'tar', default=False, is_flag=True,
help='Create a tarball from the element\'s sources instead of a '
'file tree.')
@click.option('--include-build-scripts', 'build_scripts', is_flag=True)
@click.argument('element', required=False, type=click.Path(readable=False))
@click.argument('location', type=click.Path(), required=False)
@click.pass_obj
def source_checkout(app, element, location, deps, fetch_, except_):
def source_checkout(app, element, location, force, deps, fetch_, except_,
tar, build_scripts):
"""Checkout sources of an element to the specified location
"""
if not element and not location:
click.echo("ERROR: LOCATION is not specified", err=True)
sys.exit(-1)
if element and not location:
# Nasty hack to get around click's optional args
location = element
element = None
with app.initialized():
if not element:
element = app.context.guess_element()
if not element:
raise AppError('Missing argument "ELEMENT".')
app.stream.source_checkout(element,
location=location,
force=force,
deps=deps,
fetch=fetch_,
except_targets=except_)
except_targets=except_,
tar=tar,
include_build_scripts=build_scripts)
##################################################################
......@@ -747,11 +820,15 @@ def workspace_open(app, no_checkout, force, track_, directory, elements):
def workspace_close(app, remove_dir, all_, elements):
"""Close a workspace"""
if not (all_ or elements):
click.echo('ERROR: no elements specified', err=True)
sys.exit(-1)
with app.initialized():
if not (all_ or elements):
# NOTE: I may need to revisit this when implementing multiple projects
# opening one workspace.
element = app.context.guess_element()
if element:
elements = (element,)
else:
raise AppError('No elements specified')
# Early exit if we specified `all` and there are no workspaces
if all_ and not app.stream.workspace_exists():
......@@ -808,7 +885,11 @@ def workspace_reset(app, soft, track_, all_, elements):
with app.initialized():
if not (all_ or elements):
raise AppError('No elements specified to reset')
element = app.context.guess_element()
if element:
elements = (element,)
else:
raise AppError('No elements specified to reset')
if all_ and not app.stream.workspace_exists():
raise AppError("No open workspaces to reset")
......@@ -834,34 +915,3 @@ def workspace_list(app):
with app.initialized():
app.stream.workspace_list()
##################################################################
# Source Bundle Command #
##################################################################
@cli.command(name="source-bundle", short_help="Produce a build bundle to be manually executed")
@click.option('--except', 'except_', multiple=True,
type=click.Path(readable=False),
help="Elements to except from the tarball")
@click.option('--compression', default='gz',
type=click.Choice(['none', 'gz', 'bz2', 'xz']),
help="Compress the tar file using the given algorithm.")
@click.option('--track', 'track_', default=False, is_flag=True,
help="Track new source references before bundling")
@click.option('--force', '-f', default=False, is_flag=True,
help="Overwrite an existing tarball")
@click.option('--directory', default=os.getcwd(),
help="The directory to write the tarball to")
@click.argument('element',
type=click.Path(readable=False))
@click.pass_obj
def source_bundle(app, element, force, directory,
track_, compression, except_):
"""Produce a source bundle to be manually executed
"""
with app.initialized():
app.stream.source_bundle(element, directory,
track_first=track_,
force=force,
compression=compression,
except_targets=except_)
buildstream/_stream.py
View file @ a322d5c0
......@@ -25,8 +25,8 @@ import stat
import shlex
import shutil
import tarfile
from contextlib import contextmanager
from tempfile import TemporaryDirectory
import tempfile
from contextlib import contextmanager, suppress
from ._exceptions import StreamError, ImplError, BstError, set_last_task_error
from ._message import Message, MessageType
......@@ -449,11 +449,14 @@ class Stream():
#
def source_checkout(self, target, *,
location=None,
force=False,
deps='none',
fetch=False,
except_targets=()):
except_targets=(),
tar=False,
include_build_scripts=False):
self._check_location_writable(location)
self._check_location_writable(location, force=force, tar=tar)
elements, _ = self._load((target,), (),
selection=deps,
......@@ -467,7 +470,8 @@ class Stream():
# Stage all sources determined by scope
try:
self._write_element_sources(location, elements)
self._source_checkout(elements, location, force, deps,
fetch, tar, include_build_scripts)
except BstError as e:
raise StreamError("Error while writing sources"
": '{}'".format(e), detail=e.detail, reason=e.reason) from e
......@@ -544,7 +548,8 @@ class Stream():
if len(elements) != 1:
raise StreamError("Exactly one element can be given if --directory is used",
reason='directory-with-multiple-elements')
expanded_directories = [custom_dir, ]
directory = os.path.abspath(custom_dir)
expanded_directories = [directory, ]
else:
# If this fails it is a bug in what ever calls this, usually cli.py and so can not be tested for via the
# run bst test mechanism.
......@@ -727,87 +732,6 @@ class Stream():
'workspaces': workspaces
})
# source_bundle()
#
# Create a host buildable tarball bundle for the given target.
#
# Args:
# target (str): The target element to bundle
# directory (str): The directory to output the tarball
# track_first (bool): Track new source references before bundling
# compression (str): The compression type to use
# force (bool): Overwrite an existing tarball
#
def source_bundle(self, target, directory, *,
track_first=False,
force=False,
compression="gz",
except_targets=()):
if track_first:
track_targets = (target,)
else:
track_targets = ()
elements, track_elements = self._load((target,), track_targets,
selection=PipelineSelection.ALL,
except_targets=except_targets,
track_selection=PipelineSelection.ALL,
fetch_subprojects=True)
# source-bundle only supports one target
target = self.targets[0]
self._message(MessageType.INFO, "Bundling sources for target {}".format(target.name))
# Find the correct filename for the compression algorithm
tar_location = os.path.join(directory, target.normal_name + ".tar")
if compression != "none":
tar_location += "." + compression
# Attempt writing a file to generate a good error message
# early
#
# FIXME: A bit hackish
try:
open(tar_location, mode="x")
os.remove(tar_location)
except IOError as e:
raise StreamError("Cannot write to {0}: {1}"
.format(tar_location, e)) from e
# Fetch and possibly track first
#
self._fetch(elements, track_elements=track_elements)
# We don't use the scheduler for this as it is almost entirely IO
# bound.
# Create a temporary directory to build the source tree in
builddir = self._context.builddir
os.makedirs(builddir, exist_ok=True)
prefix = "{}-".format(target.normal_name)
with TemporaryDirectory(prefix=prefix, dir=builddir) as tempdir:
source_directory = os.path.join(tempdir, 'source')
try:
os.makedirs(source_directory)
except OSError as e:
raise StreamError("Failed to create directory: {}"
.format(e)) from e
# Any elements that don't implement _write_script
# should not be included in the later stages.
elements = [
element for element in elements
if self._write_element_script(source_directory, element)
]
self._write_element_sources(os.path.join(tempdir, "source"), elements)
self._write_build_script(tempdir, elements)
self._collect_sources(tempdir, tar_location,
target.normal_name, compression)
# redirect_element_names()
#
# Takes a list of element names and returns a list where elements have been
......@@ -1188,6 +1112,54 @@ class Stream():
sandbox_vroot.export_files(directory, can_link=True, can_destroy=True)
# Helper function for source_checkout()
def _source_checkout(self, elements,
location=None,
force=False,
deps='none',
fetch=False,
tar=False,
include_build_scripts=False):
location = os.path.abspath(location)
location_parent = os.path.abspath(os.path.join(location, ".."))
# Stage all our sources in a temporary directory. The this
# directory can be used to either construct a tarball or moved
# to the final desired location.
temp_source_dir = tempfile.TemporaryDirectory(dir=location_parent)
try:
self._write_element_sources(temp_source_dir.name, elements)
if include_build_scripts:
self._write_build_scripts(temp_source_dir.name, elements)
if tar:
self._create_tarball(temp_source_dir.name, location)
else:
self._move_directory(temp_source_dir.name, location, force)
except OSError as e:
raise StreamError("Failed to checkout sources to {}: {}"
.format(location, e)) from e
finally:
with suppress(FileNotFoundError):
temp_source_dir.cleanup()
# Move a directory src to dest. This will work across devices and
# may optionaly overwrite existing files.
def _move_directory(self, src, dest, force=False):
def is_empty_dir(path):
return os.path.isdir(dest) and not os.listdir(dest)
try:
os.rename(src, dest)
return
except OSError:
pass
if force or is_empty_dir(dest):
try:
utils.link_files(src, dest)
except utils.UtilError as e:
raise StreamError("Failed to move directory: {}".format(e)) from e
# Write the element build script to the given directory
def _write_element_script(self, directory, element):
try:
......@@ -1204,8 +1176,28 @@ class Stream():
os.makedirs(element_source_dir)
element._stage_sources_at(element_source_dir, mount_workspaces=False)
# Create a tarball from the content of directory
def _create_tarball(self, directory, tar_name):
try:
with utils.save_file_atomic(tar_name, mode='wb') as f:
# This TarFile does not need to be explicitly closed
# as the underlying file object will be closed be the
# save_file_atomic contect manager
tarball = tarfile.open(fileobj=f, mode='w')
for item in os.listdir(str(directory)):
file_to_add = os.path.join(directory, item)
tarball.add(file_to_add, arcname=item)
except OSError as e:
raise StreamError("Failed to create tar archive: {}".format(e)) from e
# Write all the build_scripts for elements in the directory location
def _write_build_scripts(self, location, elements):
for element in elements:
self._write_element_script(location, element)
self._write_master_build_script(location, elements)
# Write a master build script to the sandbox
def _write_build_script(self, directory, elements):
def _write_master_build_script(self, directory, elements):
module_string = ""
for element in elements:
......
doc/sessions/developing.run
View file @ a322d5c0
......@@ -24,6 +24,11 @@ commands:
output: ../source/sessions/developing-build-after-changes.html
command: build hello.bst
# Rebuild, from the workspace
- directory: ../examples/developing/workspace_hello
output: ../source/sessions/developing-build-after-changes-workspace.html
command: build
# Capture shell output with changes
- directory: ../examples/developing/
output: ../source/sessions/developing-shell-after-changes.html
......
doc/source/developing/workspaces.rst
View file @ a322d5c0
......@@ -50,11 +50,16 @@ We can open workspace_hello/hello.c and make the following change:
.. literalinclude:: ../../examples/developing/update.patch
:language: diff
Now, rebuild the hello.bst element
Now, rebuild the hello.bst element.
.. raw:: html
:file: ../sessions/developing-build-after-changes.html
Note that if you run the command from inside the workspace, the element name is optional.
.. raw:: html
:file: ../sessions/developing-build-after-changes-workspace.html
Now running the hello command using bst shell:
.. raw:: html
......
doc/source/sessions-stored/developing-build-after-changes-workspace.html 0 → 100644
View file @ a322d5c0
<!--
WARNING: This file was generated with bst2html.py
-->
<div class="highlight" style="font-size:x-small"><pre>
<span style="color:#C4A000;font-weight:bold">user@host</span>:<span style="color:#3456A4;font-weight:bold">~/workspace_hello</span>$ bst build
<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#3465A4"><span style=""><span style="opacity:0.5">START </span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Build
<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#3465A4"><span style=""><span style="opacity:0.5">START </span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Loading elements
<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#4E9A06"><span style=""><span style="opacity:0.5">SUCCESS</span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Loading elements
<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#3465A4"><span style=""><span style="opacity:0.5">START </span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Resolving elements
<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#4E9A06"><span style=""><span style="opacity:0.5">SUCCESS</span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Resolving elements
<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#3465A4"><span style=""><span style="opacity:0.5">START </span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Resolving cached state
<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#4E9A06"><span style=""><span style="opacity:0.5">SUCCESS</span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Resolving cached state
<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#3465A4"><span style=""><span style="opacity:0.5">START </span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Checking sources
<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#4E9A06"><span style=""><span style="opacity:0.5">SUCCESS</span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Checking sources
<span style="color:#C4A000"><span style="">BuildStream Version 1.3.0+1032.g41813d3a.dirty
</span></span><span style="color:#06989A"><span style="opacity:0.5"> Session Start: </span></span><span style="color:#C4A000">Wednesday, 05-12-2018 at 16:47:38</span>
<span style="color:#06989A"><span style="opacity:0.5"> Project: </span></span><span style="color:#C4A000">developing (/home/user/workspace/buildstream/buildstream/doc/examples/developing)</span>
<span style="color:#06989A"><span style="opacity:0.5"> Targets: </span></span><span style="color:#C4A000">hello.bst</span>
<span style="color:#C4A000"><span style="">User Configuration
</span></span><span style="color:#06989A"><span style="opacity:0.5"> Configuration File: </span></span><span style="color:#C4A000">/home/user/workspace/buildstream/buildstream/doc/run-bst-7ocq4_a7/buildstream.conf</span>
<span style="color:#06989A"><span style="opacity:0.5"> Log Files: </span></span><span style="color:#C4A000">/home/user/workspace/buildstream/buildstream/doc/run-bst-7ocq4_a7/logs</span>
<span style="color:#06989A"><span style="opacity:0.5"> Source Mirrors: </span></span><span style="color:#C4A000">/home/user/workspace/buildstream/buildstream/doc/run-bst-7ocq4_a7/sources</span>
<span style="color:#06989A"><span style="opacity:0.5"> Build Area: </span></span><span style="color:#C4A000">/home/user/workspace/buildstream/buildstream/doc/run-bst-7ocq4_a7/build</span>
<span style="color:#06989A"><span style="opacity:0.5"> Artifact Cache: </span></span><span style="color:#C4A000">/home/user/workspace/buildstream/buildstream/doc/run-bst-7ocq4_a7/artifacts</span>
<span style="color:#06989A"><span style="opacity:0.5"> Strict Build Plan: </span></span><span style="color:#C4A000">Yes</span>
<span style="color:#06989A"><span style="opacity:0.5"> Maximum Fetch Tasks: </span></span><span style="color:#C4A000">10</span>
<span style="color:#06989A"><span style="opacity:0.5"> Maximum Build Tasks: </span></span><span style="color:#C4A000">4</span>
<span style="color:#06989A"><span style="opacity:0.5"> Maximum Push Tasks: </span></span><span style="color:#C4A000">4</span>
<span style="color:#06989A"><span style="opacity:0.5"> Maximum Network Retries: </span></span><span style="color:#C4A000">2</span>
<span style="color:#C4A000"><span style="">Pipeline
</span></span><span style="color:#75507B"> cached</span> <span style="color:#C4A000">9afe69d645f0bee106749bc2101aae16ef437bb51e1b343ef1f16f04f0572efb</span> <span style="color:#3465A4"><span style="">base/alpine.bst</span></span>
<span style="color:#75507B"> cached</span> <span style="color:#C4A000">19f7c50c7a1db9ae4babe9d1f34f4cdbbf2428827d48673861fd1452d6c7e16b</span> <span style="color:#3465A4"><span style="">base.bst</span></span>
<span style="color:#75507B"> cached</span> <span style="color:#C4A000">faa419610e7309d36d15926a81a8d75bbc113443c23c8162e63843dd86b5f56a</span> <span style="color:#3465A4"><span style="">hello.bst</span></span> Workspace: /home/user/workspace/buildstream/buildstream/doc/examples/developing/workspace_hello
<span style="color:#06989A"><span style="opacity:0.5">===============================================================================
</span></span><span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#4E9A06"><span style=""><span style="opacity:0.5">SUCCESS</span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Build
<span style="color:#C4A000"><span style="">Pipeline Summary
</span></span><span style="color:#06989A"><span style="opacity:0.5"> Total: </span></span><span style="color:#C4A000">3</span>
<span style="color:#06989A"><span style="opacity:0.5"> Session: </span></span><span style="color:#C4A000">0</span>
<span style="color:#06989A"><span style="opacity:0.5"> Fetch Queue: </span></span><span style="color:#C4A000">processed </span><span style="color:#4E9A06">0</span><span style="color:#06989A"><span style="opacity:0.5">, </span></span><span style="color:#C4A000">skipped </span><span style="color:#C4A000">0</span><span style="color:#06989A"><span style="opacity:0.5">, </span></span><span style="color:#C4A000">failed </span><span style="color:#CC0000"><span style="opacity:0.5">0</span></span>
<span style="color:#06989A"><span style="opacity:0.5"> Build Queue: </span></span><span style="color:#C4A000">processed </span><span style="color:#4E9A06">0</span><span style="color:#06989A"><span style="opacity:0.5">, </span></span><span style="color:#C4A000">skipped </span><span style="color:#C4A000">0</span><span style="color:#06989A"><span style="opacity:0.5">, </span></span><span style="color:#C4A000">failed </span><span style="color:#CC0000"><span style="opacity:0.5">0</span></span>
</pre></div>
doc/source/using_commands.rst
View file @ a322d5c0
......@@ -86,13 +86,6 @@ project's main directory.
----
.. _invoking_source_bundle:
.. click:: buildstream._frontend.cli:source_bundle
:prog: bst source bundle
----
.. _invoking_workspace:
.. click:: buildstream._frontend.cli:workspace
......
tests/completions/completions.py
View file @ a322d5c0
......@@ -16,7 +16,6 @@ MAIN_COMMANDS = [
'shell ',
'show ',
'source-checkout ',
'source-bundle ',
'track ',
'workspace '
]
......
tests/frontend/help.py
View file @ a322d5c0
......@@ -25,7 +25,6 @@ def test_help_main(cli):
('push'),
('shell'),
('show'),
('source-bundle'),
('track'),
('workspace')
])
......
tests/frontend/source_bundle.py deleted 100644 → 0
View file @ 4219a6b4
#
# Copyright (C) 2018 Bloomberg Finance LP
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
# Authors: Chandan Singh <csingh43@bloomberg.net>
#
import os
import tarfile
import pytest
from tests.testutils import cli
# Project directory
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"project",
)
@pytest.mark.datafiles(DATA_DIR)
def test_source_bundle(cli, tmpdir, datafiles):
project_path = os.path.join(datafiles.dirname, datafiles.basename)
element_name = 'source-bundle/source-bundle-hello.bst'
normal_name = 'source-bundle-source-bundle-hello'
# Verify that we can correctly produce a source-bundle
args = ['source-bundle', element_name, '--directory', str(tmpdir)]
result = cli.run(project=project_path, args=args)
result.assert_success()
# Verify that the source-bundle contains our sources and a build script
with tarfile.open(os.path.join(str(tmpdir), '{}.tar.gz'.format(normal_name))) as bundle:
assert os.path.join(normal_name, 'source', normal_name, 'llamas.txt') in bundle.getnames()
assert os.path.join(normal_name, 'build.sh') in bundle.getnames()
tests/frontend/source_checkout.py
View file @ a322d5c0
import os
import pytest
import tarfile
from pathlib import Path
from tests.testutils import cli
......@@ -28,24 +30,66 @@ def generate_remote_import_element(input_path, output_path):
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize('with_workspace', [('workspace'), ('no-workspace')])
def test_source_checkout(datafiles, tmpdir_factory, cli, with_workspace):
@pytest.mark.parametrize(
"with_workspace,guess_element",
[(True, True), (True, False), (False, False)],
ids=["workspace-guess", "workspace-no-guess", "no-workspace-no-guess"]
)
def test_source_checkout(datafiles, cli, tmpdir_factory, with_workspace, guess_element):
tmpdir = tmpdir_factory.mktemp("")
project = os.path.join(datafiles.dirname, datafiles.basename)
checkout = os.path.join(cli.directory, 'source-checkout')
target = 'checkout-deps.bst'
workspace = os.path.join(str(tmpdir), 'workspace')
elm_cmd = [target] if not guess_element else []
if with_workspace == "workspace":
result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, target])
if with_workspace:
ws_cmd = ['-C', workspace]
result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, target])
result.assert_success()
else:
ws_cmd = []
result = cli.run(project=project, args=['source-checkout', target, '--deps', 'none', checkout])
args = ws_cmd + ['source-checkout', '--deps', 'none'] + elm_cmd + [checkout]
result = cli.run(project=project, args=args)
result.assert_success()
assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize('force_flag', ['--force', '-f'])
def test_source_checkout_force(datafiles, cli, force_flag):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkout = os.path.join(cli.directory, 'source-checkout')
target = 'checkout-deps.bst'
os.makedirs(os.path.join(checkout, 'some-thing'))
# Path(os.path.join(checkout, 'some-file')).touch()
result = cli.run(project=project, args=['source-checkout', force_flag, target, '--deps', 'none', checkout])
result.assert_success()
assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_tar(datafiles, cli):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkout = os.path.join(cli.directory, 'source-checkout.tar')
target = 'checkout-deps.bst'
result = cli.run(project=project, args=['source-checkout', '--tar', target, '--deps', 'none', checkout])
result.assert_success()
assert os.path.exists(checkout)
with tarfile.open(checkout) as tf:
expected_content = os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config')
tar_members = [f.name for f in tf]
for member in tar_members:
assert member in expected_content
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize('deps', [('build'), ('none'), ('run'), ('all')])
def test_source_checkout_deps(datafiles, cli, deps):
......@@ -126,3 +170,38 @@ def test_source_checkout_fetch(datafiles, cli, fetch):
assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
else:
result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
project_path = os.path.join(datafiles.dirname, datafiles.basename)
element_name = 'source-bundle/source-bundle-hello.bst'
normal_name = 'source-bundle-source-bundle-hello'
checkout = os.path.join(str(tmpdir), 'source-checkout')
args = ['source-checkout', '--include-build-scripts', element_name, checkout]
result = cli.run(project=project_path, args=args)
result.assert_success()
# There sould be a script for each element (just one in this case) and a top level build script
expected_scripts = ['build.sh', 'build-' + normal_name]
for script in expected_scripts:
assert script in os.listdir(checkout)
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
project_path = os.path.join(datafiles.dirname, datafiles.basename)
element_name = 'source-bundle/source-bundle-hello.bst'
normal_name = 'source-bundle-source-bundle-hello'
tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
args = ['source-checkout', '--include-build-scripts', '--tar', element_name, tar_file]
result = cli.run(project=project_path, args=args)
result.assert_success()
expected_scripts = ['build.sh', 'build-' + normal_name]
with tarfile.open(tar_file, 'r') as tf:
for script in expected_scripts:
assert script in tf.getnames()
tests/frontend/workspace.py
View file @ a322d5c0
......@@ -616,12 +616,16 @@ def test_list(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("kind", repo_kinds)
@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
@pytest.mark.parametrize("call_from", [("project"), ("workspace")])
def test_build(cli, tmpdir_factory, datafiles, kind, strict, call_from):
@pytest.mark.parametrize(
"from_workspace,guess_element",
[(False, False), (True, True), (True, False)],
ids=["project-no-guess", "workspace-guess", "workspace-no-guess"])
def test_build(cli, tmpdir_factory, datafiles, kind, strict, from_workspace, guess_element):
tmpdir = tmpdir_factory.mktemp('')
element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
checkout = os.path.join(str(tmpdir), 'checkout')
args_pre = ['-C', workspace] if call_from == "workspace" else []
args_dir = ['-C', workspace] if from_workspace else []
args_elm = [element_name] if not guess_element else []
# Modify workspace
shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
......@@ -644,14 +648,14 @@ def test_build(cli, tmpdir_factory, datafiles, kind, strict, call_from):
# Build modified workspace
assert cli.get_element_state(project, element_name) == 'buildable'
assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
result = cli.run(project=project, args=args_pre + ['build', element_name])
result = cli.run(project=project, args=args_dir + ['build'] + args_elm)
result.assert_success()
assert cli.get_element_state(project, element_name) == 'cached'
assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
# Checkout the result
result = cli.run(project=project,
args=args_pre + ['checkout', element_name, checkout])
args=args_dir + ['checkout'] + args_elm + [checkout])
result.assert_success()
# Check that the pony.conf from the modified workspace exists
......@@ -1062,29 +1066,36 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize('subdir', [True, False], ids=["subdir", "no-subdir"])
def test_external_fetch(cli, datafiles, tmpdir_factory, subdir):
@pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
def test_external_fetch(cli, datafiles, tmpdir_factory, subdir, guess_element):
# Fetching from a workspace outside a project doesn't fail horribly
tmpdir = tmpdir_factory.mktemp('')
element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
arg_elm = [element_name] if not guess_element else []
if subdir:
call_dir = os.path.join(workspace, 'usr')
else:
call_dir = workspace
result = cli.run(project=project, args=['-C', call_dir, 'fetch', element_name])
result = cli.run(project=project, args=['-C', call_dir, 'fetch'] + arg_elm)
result.assert_success()
# We already fetched it by opening the workspace, but we're also checking
# `bst show` works here
assert cli.get_element_state(project, element_name) == 'buildable'
result = cli.run(project=project,
args=['-C', call_dir, 'show', '--deps', 'none', '--format', '%{state}'] + arg_elm)
result.assert_success()
assert result.output.strip() == 'buildable'
@pytest.mark.datafiles(DATA_DIR)
def test_external_push_pull(cli, datafiles, tmpdir_factory):
@pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
def test_external_push_pull(cli, datafiles, tmpdir_factory, guess_element):
# Pushing and pulling to/from an artifact cache works from an external workspace
tmpdir = tmpdir_factory.mktemp('')
element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
arg_elm = [element_name] if not guess_element else []
with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
result = cli.run(project=project, args=['-C', workspace, 'build', element_name])
......@@ -1094,22 +1105,24 @@ def test_external_push_pull(cli, datafiles, tmpdir_factory):
'artifacts': {'url': share.repo, 'push': True}
})
result = cli.run(project=project, args=['-C', workspace, 'push', element_name])
result = cli.run(project=project, args=['-C', workspace, 'push'] + arg_elm)
result.assert_success()
result = cli.run(project=project, args=['-C', workspace, 'pull', '--deps', 'all', element_name])
result = cli.run(project=project, args=['-C', workspace, 'pull', '--deps', 'all'] + arg_elm)
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_external_track(cli, datafiles, tmpdir_factory):
@pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
def test_external_track(cli, datafiles, tmpdir_factory, guess_element):
# Tracking does not get horribly confused
tmpdir = tmpdir_factory.mktemp('')
element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", True)
arg_elm = [element_name] if not guess_element else []
# The workspace is necessarily already tracked, so we only care that
# there's no weird errors.
result = cli.run(project=project, args=['-C', workspace, 'track', element_name])
result = cli.run(project=project, args=['-C', workspace, 'track'] + arg_elm)
result.assert_success()
......@@ -1147,15 +1160,17 @@ def test_external_close_other(cli, datafiles, tmpdir_factory):
@pytest.mark.datafiles(DATA_DIR)
def test_external_close_self(cli, datafiles, tmpdir_factory):
@pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
def test_external_close_self(cli, datafiles, tmpdir_factory, guess_element):
# From inside an external workspace, close it
tmpdir1 = tmpdir_factory.mktemp('')
tmpdir2 = tmpdir_factory.mktemp('')
# Making use of the assumption that it's the same project in both invocations of open_workspace
alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
arg_elm = [alpha_element] if not guess_element else []
result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', alpha_element])
result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close'] + arg_elm)
result.assert_success()
......@@ -1172,11 +1187,13 @@ def test_external_reset_other(cli, datafiles, tmpdir_factory):
@pytest.mark.datafiles(DATA_DIR)
def test_external_reset_self(cli, datafiles, tmpdir):
@pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
def test_external_reset_self(cli, datafiles, tmpdir, guess_element):
element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
arg_elm = [element] if not guess_element else []
# Command succeeds
result = cli.run(project=project, args=['-C', workspace, 'workspace', 'reset', element])
result = cli.run(project=project, args=['-C', workspace, 'workspace', 'reset'] + arg_elm)
result.assert_success()
# Successive commands still work (i.e. .bstproject.yaml hasn't been deleted)
......
tests/integration/shell.py
View file @ a322d5c0
......@@ -358,13 +358,22 @@ def test_integration_devices(cli, tmpdir, datafiles):
# Test that a shell can be opened from an external workspace
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("build_shell", [("build"), ("nobuild")])
@pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell):
def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell, guess_element):
tmpdir = tmpdir_factory.mktemp("")
project = os.path.join(datafiles.dirname, datafiles.basename)
element_name = 'autotools/amhello.bst'
workspace_dir = os.path.join(str(tmpdir), 'workspace')
if guess_element:
# Mutate the project.conf to use a default shell command
project_file = os.path.join(project, 'project.conf')
config_text = "shell:\n"\
" command: ['true']\n"
with open(project_file, 'a') as f:
f.write(config_text)
result = cli.run(project=project, args=[
'workspace', 'open', '--directory', workspace_dir, element_name
])
......@@ -373,9 +382,10 @@ def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_sh
result = cli.run(project=project, args=['-C', workspace_dir, 'build', element_name])
result.assert_success()
command = ['shell']
command = ['-C', workspace_dir, 'shell']
if build_shell == 'build':
command.append('--build')
command.extend([element_name, '--', 'true'])
if not guess_element:
command.extend([element_name, '--', 'true'])
result = cli.run(project=project, cwd=workspace_dir, args=command)
result.assert_success()