Commit 76c5d2f8 authored by Chandan Singh's avatar Chandan Singh Committed by Chandan Singh

Add `bst source-checkout` command

As discussed in
https://mail.gnome.org/archives/buildstream-list/2018-September/msg00064.html,
add `bst source-checkout` command. This will allow users to checkout
sources for a given target.

* _frontend/cli.py: Add source-checkout command
* _pipeline.py: Add assert_sources_cached() method
* _stream.py: Add source_checkout method, abstract out
  __check_location_writable() method that used to part of checkout()
parent 30b72244
Pipeline #37109928 passed with stages
in 27 minutes and 40 seconds
......@@ -664,6 +664,33 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
tar=tar)
##################################################################
# Source Checkout Command #
##################################################################
@cli.command(name='source-checkout', short_help='Checkout sources for an element')
@click.option('--except', 'except_', multiple=True,
type=click.Path(readable=False),
help="Except certain dependencies")
@click.option('--deps', '-d', default='none',
type=click.Choice(['build', 'none', 'run', 'all']),
help='The dependencies whose sources to checkout (default: none)')
@click.option('--fetch', 'fetch_', default=False, is_flag=True,
help='Fetch elements if they are not fetched')
@click.argument('element',
type=click.Path(readable=False))
@click.argument('location', type=click.Path())
@click.pass_obj
def source_checkout(app, element, location, deps, fetch_, except_):
"""Checkout sources of an element to the specified location
"""
with app.initialized():
app.stream.source_checkout(element,
location=location,
deps=deps,
fetch=fetch_,
except_targets=except_)
##################################################################
# Workspace Command #
##################################################################
......
......@@ -383,6 +383,33 @@ class Pipeline():
detail += " " + element._get_full_name() + "\n"
raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline-workspaced")
# assert_sources_cached()
#
# Asserts that sources for the given list of elements are cached.
#
# Args:
# elements (list): The list of elements
#
def assert_sources_cached(self, elements):
uncached = []
with self._context.timed_activity("Checking sources"):
for element in elements:
if element._get_consistency() != Consistency.CACHED:
uncached.append(element)
if uncached:
detail = "Sources are not cached for the following elements:\n\n"
for element in uncached:
detail += " Following sources for element: {} are not cached:\n".format(element._get_full_name())
for source in element.sources():
if source._get_consistency() != Consistency.CACHED:
detail += " {}\n".format(source)
detail += '\n'
detail += "Try fetching these elements first with `bst fetch`,\n" + \
"or run this command with `--fetch` option\n"
raise PipelineError("Uncached sources", detail=detail, reason="uncached-sources")
#############################################################
# Private Methods #
#############################################################
......
......@@ -379,27 +379,7 @@ class Stream():
elements, _ = self._load((target,), (), fetch_subprojects=True)
target = elements[0]
if not tar:
try:
os.makedirs(location, exist_ok=True)
except OSError as e:
raise StreamError("Failed to create checkout directory: '{}'"
.format(e)) from e
if not tar:
if not os.access(location, os.W_OK):
raise StreamError("Checkout directory '{}' not writable"
.format(location))
if not force and os.listdir(location):
raise StreamError("Checkout directory '{}' not empty"
.format(location))
elif os.path.exists(location) and location != '-':
if not os.access(location, os.W_OK):
raise StreamError("Output file '{}' not writable"
.format(location))
if not force and os.path.exists(location):
raise StreamError("Output file '{}' already exists"
.format(location))
self._check_location_writable(location, force=force, tar=tar)
# Stage deps into a temporary sandbox first
try:
......@@ -443,6 +423,42 @@ class Stream():
raise StreamError("Error while staging dependencies into a sandbox"
": '{}'".format(e), detail=e.detail, reason=e.reason) from e
# source_checkout()
#
# Checkout sources of the target element to the specified location
#
# Args:
# target (str): The target element whose sources to checkout
# location (str): Location to checkout the sources to
# deps (str): The dependencies to checkout
# fetch (bool): Whether to fetch missing sources
# except_targets (list): List of targets to except from staging
#
def source_checkout(self, target, *,
location=None,
deps='none',
fetch=False,
except_targets=()):
self._check_location_writable(location)
elements, _ = self._load((target,), (),
selection=deps,
except_targets=except_targets,
fetch_subprojects=True)
# Assert all sources are cached
if fetch:
self._fetch(elements)
self._pipeline.assert_sources_cached(elements)
# Stage all sources determined by scope
try:
self._write_element_sources(location, elements)
except BstError as e:
raise StreamError("Error while writing sources"
": '{}'".format(e), detail=e.detail, reason=e.reason) from e
# workspace_open
#
# Open a project workspace
......@@ -726,7 +742,7 @@ class Stream():
if self._write_element_script(source_directory, element)
]
self._write_element_sources(tempdir, elements)
self._write_element_sources(os.path.join(tempdir, "source"), elements)
self._write_build_script(tempdir, elements)
self._collect_sources(tempdir, tar_location,
target.normal_name, compression)
......@@ -1068,6 +1084,39 @@ class Stream():
self._enqueue_plan(fetch_plan)
self._run()
# _check_location_writable()
#
# Check if given location is writable.
#
# Args:
# location (str): Destination path
# force (bool): Allow files to be overwritten
# tar (bool): Whether destination is a tarball
#
# Raises:
# (StreamError): If the destination is not writable
#
def _check_location_writable(self, location, force=False, tar=False):
if not tar:
try:
os.makedirs(location, exist_ok=True)
except OSError as e:
raise StreamError("Failed to create destination directory: '{}'"
.format(e)) from e
if not os.access(location, os.W_OK):
raise StreamError("Destination directory '{}' not writable"
.format(location))
if not force and os.listdir(location):
raise StreamError("Destination directory '{}' not empty"
.format(location))
elif os.path.exists(location) and location != '-':
if not os.access(location, os.W_OK):
raise StreamError("Output file '{}' not writable"
.format(location))
if not force and os.path.exists(location):
raise StreamError("Output file '{}' already exists"
.format(location))
# Helper function for checkout()
#
def _checkout_hardlinks(self, sandbox_vroot, directory):
......@@ -1089,11 +1138,10 @@ class Stream():
# Write all source elements to the given directory
def _write_element_sources(self, directory, elements):
for element in elements:
source_dir = os.path.join(directory, "source")
element_source_dir = os.path.join(source_dir, element.normal_name)
os.makedirs(element_source_dir)
element._stage_sources_at(element_source_dir)
element_source_dir = self._get_element_dirname(directory, element)
if list(element.sources()):
os.makedirs(element_source_dir)
element._stage_sources_at(element_source_dir)
# Write a master build script to the sandbox
def _write_build_script(self, directory, elements):
......@@ -1122,3 +1170,25 @@ class Stream():
with tarfile.open(tar_name, permissions) as tar:
tar.add(directory, arcname=element_name)
# _get_element_dirname()
#
# Get path to directory for an element based on its normal name.
#
# For cross-junction elements, the path will be prefixed with the name
# of the junction element.
#
# Args:
# directory (str): path to base directory
# element (Element): the element
#
# Returns:
# (str): Path to directory for this element
#
def _get_element_dirname(self, directory, element):
parts = [element.normal_name]
while element._get_project() != self._project:
element = element._get_project().junction
parts.append(element.normal_name)
return os.path.join(directory, *reversed(parts))
......@@ -15,6 +15,7 @@ MAIN_COMMANDS = [
'push ',
'shell ',
'show ',
'source-checkout ',
'source-bundle ',
'track ',
'workspace '
......
kind: import
description: It is important for this element to have both build and runtime dependencies
sources:
- kind: local
path: files/etc-files
depends:
- filename: import-dev.bst
type: build
- filename: import-bin.bst
type: runtime
import os
import pytest
from tests.testutils import cli
from buildstream import utils, _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
# Project directory
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'project',
)
def generate_remote_import_element(input_path, output_path):
return {
'kind': 'import',
'sources': [
{
'kind': 'remote',
'url': 'file://{}'.format(input_path),
'filename': output_path,
'ref': utils.sha256sum(input_path),
}
]
}
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout(datafiles, cli):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkout = os.path.join(cli.directory, 'source-checkout')
target = 'checkout-deps.bst'
result = cli.run(project=project, args=['source-checkout', target, '--deps', 'none', checkout])
result.assert_success()
assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize('deps', [('build'), ('none'), ('run'), ('all')])
def test_source_checkout_deps(datafiles, cli, deps):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkout = os.path.join(cli.directory, 'source-checkout')
target = 'checkout-deps.bst'
result = cli.run(project=project, args=['source-checkout', target, '--deps', deps, checkout])
result.assert_success()
# Sources of the target
if deps == 'build':
assert not os.path.exists(os.path.join(checkout, 'checkout-deps'))
else:
assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
# Sources of the target's build dependencies
if deps in ('build', 'all'):
assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
else:
assert not os.path.exists(os.path.join(checkout, 'import-dev'))
# Sources of the target's runtime dependencies
if deps in ('run', 'all'):
assert os.path.exists(os.path.join(checkout, 'import-bin', 'usr', 'bin', 'hello'))
else:
assert not os.path.exists(os.path.join(checkout, 'import-bin'))
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_except(datafiles, cli):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkout = os.path.join(cli.directory, 'source-checkout')
target = 'checkout-deps.bst'
result = cli.run(project=project, args=['source-checkout', target,
'--deps', 'all',
'--except', 'import-bin.bst',
checkout])
result.assert_success()
# Sources for the target should be present
assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
# Sources for import-bin.bst should not be present
assert not os.path.exists(os.path.join(checkout, 'import-bin'))
# Sources for other dependencies should be present
assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize('fetch', [(False), (True)])
def test_source_checkout_fetch(datafiles, cli, fetch):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkout = os.path.join(cli.directory, 'source-checkout')
target = 'remote-import-dev.bst'
target_path = os.path.join(project, 'elements', target)
# Create an element with remote source
element = generate_remote_import_element(
os.path.join(project, 'files', 'dev-files', 'usr', 'include', 'pony.h'),
'pony.h')
_yaml.dump(element, target_path)
# Testing --fetch option requires that we do not have the sources
# cached already
assert cli.get_element_state(project, target) == 'fetch needed'
args = ['source-checkout']
if fetch:
args += ['--fetch']
args += [target, checkout]
result = cli.run(project=project, args=args)
if fetch:
result.assert_success()
assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
else:
result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment