Compare revisions

39e23ecf · baf6b578 · baf6b578 · baf6b578 · baf6b578 · 39e23ecf
--- a/tests/integration/project/files/canary
+++ b/tests/integration/project/files/canary
+alive
--- a/tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz
+++ b/tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz
--- a/tests/integration/project/files/pypi-repo/app2/index.html
+++ b/tests/integration/project/files/pypi-repo/app2/index.html
-<html>
-  <head>
-    <title>Links for app1</title>
-  </head>
-  <body>
-    <a href='App2-0.1.tar.gz'>App2-0.1.tar.gz</a><br />
-  </body>
-</html>
--- a/tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz
+++ b/tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz
--- a/tests/integration/project/files/pypi-repo/hellolib/index.html
+++ b/tests/integration/project/files/pypi-repo/hellolib/index.html
-<html>
-  <head>
-    <title>Links for app1</title>
-  </head>
-  <body>
-    <a href='HelloLib-0.1.tar.gz'>HelloLib-0.1.tar.gz</a><br />
-  </body>
-</html>
--- a/tests/integration/script.py
+++ b/tests/integration/script.py
@@ -155,3 +155,32 @@ def test_script_layout(cli, tmpdir, datafiles):
        text = f.read()

    assert text == "Hi\n"
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_regression_cache_corruption(cli, tmpdir, datafiles):
+    project = str(datafiles)
+    checkout_original = os.path.join(cli.directory, 'checkout-original')
+    checkout_after = os.path.join(cli.directory, 'checkout-after')
+    element_name = 'script/corruption.bst'
+    canary_element_name = 'script/corruption-image.bst'
+
+    res = cli.run(project=project, args=['build', canary_element_name])
+    assert res.exit_code == 0
+
+    res = cli.run(project=project, args=['checkout', canary_element_name,
+                                         checkout_original])
+    assert res.exit_code == 0
+
+    with open(os.path.join(checkout_original, 'canary')) as f:
+        assert f.read() == 'alive\n'
+
+    res = cli.run(project=project, args=['build', element_name])
+    assert res.exit_code == 0
+
+    res = cli.run(project=project, args=['checkout', canary_element_name,
+                                         checkout_after])
+    assert res.exit_code == 0
+
+    with open(os.path.join(checkout_after, 'canary')) as f:
+        assert f.read() == 'alive\n'
--- a/tests/integration/shell.py
+++ b/tests/integration/shell.py
@@ -27,11 +27,22 @@ DATA_DIR = os.path.join(
 #    config (dict): A project.conf dictionary to composite over the default
 #    mount (tuple): A (host, target) tuple for the `--mount` option
 #    element (str): The element to build and run a shell with
+#    elements (list): Other elements to build and run a shell with
 #    isolate (bool): Whether to pass --isolate to `bst shell`
 #
-def execute_shell(cli, project, command, *, config=None, mount=None, element='base.bst', isolate=False):
+def execute_shell(cli, project, command, *, config=None, mount=None, element=None, elements=None, isolate=False):
+    assert element is None or elements is None, "Cannot mix single element and multi-element list options"
    # Ensure the element is built
-    result = cli.run(project=project, project_config=config, args=['build', element])
+    if element is None and elements is None:
+        element = 'base.bst'
+    if elements is None:
+        elements = []
+
+    args = ['build']
+    args.extend(elements)
+    if element is not None:
+        args.append(element)
+    result = cli.run(project=project, project_config=config, args=args)
    assert result.exit_code == 0

    args = ['shell']
@@ -40,7 +51,10 @@ def execute_shell(cli, project, command, *, config=None, mount=None, element='ba
    if mount is not None:
        host_path, target_path = mount
        args += ['--mount', host_path, target_path]
-    args += [element, '--'] + command
+    args += ["-e" + e for e in elements]
+    if element is not None:
+        args.append(element)
+    args += ['--'] + command

    return cli.run(project=project, project_config=config, args=args)

@@ -302,53 +316,81 @@ def test_workspace_visible(cli, tmpdir, datafiles):
    assert result.output == workspace_hello


-# Test that we can see the workspace files in a shell
-@pytest.mark.integration
+# Test that '--sysroot' works
 @pytest.mark.datafiles(DATA_DIR)
-def test_sysroot_workspace_visible(cli, tmpdir, datafiles):
+def test_sysroot(cli, tmpdir, datafiles):
    project = os.path.join(datafiles.dirname, datafiles.basename)
-    workspace = os.path.join(cli.directory, 'workspace')
-    element_name = 'workspace/workspace-mount-fail.bst'
+    base_element = "base/base-alpine.bst"
+    # test element only needs to be something lightweight for this test
+    test_element = "script/script.bst"
+    checkout_dir = os.path.join(str(tmpdir), 'alpine-sysroot')
+    test_file = 'hello'
+
+    # Build and check out a sysroot
+    res = cli.run(project=project, args=['build', base_element])
+    res.assert_success()
+    res = cli.run(project=project, args=['checkout', base_element, checkout_dir])
+    res.assert_success()
+
+    # Mutate the sysroot
+    test_path = os.path.join(checkout_dir, test_file)
+    with open(test_path, 'w') as f:
+        f.write('hello\n')
+
+    # Shell into the sysroot and check the test file exists
+    res = cli.run(project=project, args=[
+        'shell', '--build', '--sysroot', checkout_dir, test_element, '--',
+        'grep', '-q', 'hello', '/' + test_file
+    ])
+    res.assert_success()

-    # Open a workspace on our build failing element
-    #
-    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
-    assert res.exit_code == 0

-    # Ensure the dependencies of our build failing element are built
-    result = cli.run(project=project, args=['build', element_name])
-    result.assert_main_error(ErrorDomain.STREAM, None)
+# Test system integration commands can access devices in /dev
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+def test_integration_devices(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    element_name = 'shell/integration.bst'

-    # Discover the sysroot of the failed build directory, after one
-    # failed build, there should be only one directory there.
-    #
-    build_base = os.path.join(cli.directory, 'build')
-    build_dirs = os.listdir(path=build_base)
-    assert len(build_dirs) == 1
-    build_dir = os.path.join(build_base, build_dirs[0])
+    result = execute_shell(cli, project, ["true"], element=element_name)
+    assert result.exit_code == 0

-    # Obtain a copy of the hello.c content from the workspace
-    #
-    workspace_hello_path = os.path.join(cli.directory, 'workspace', 'hello.c')
-    assert os.path.exists(workspace_hello_path)
-    with open(workspace_hello_path, 'r') as f:
-        workspace_hello = f.read()

-    # Cat the hello.c file from a bst shell command, and assert
-    # that we got the same content here
-    #
-    result = cli.run(project=project, args=[
-        'shell', '--build', '--sysroot', build_dir, element_name, '--', 'cat', 'hello.c'
-    ])
+# Test multiple element shell
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+def test_shell_multiple_elements(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+
+    result = execute_shell(cli, project, ["sh", "-c", "foo && bar"],
+                           elements=["shell/adds-foo.bst", "shell/adds-bar.bst"])
    assert result.exit_code == 0
-    assert result.output == workspace_hello


-# Test system integration commands can access devices in /dev
+# Test multiple element build shell
+@pytest.mark.integration
 @pytest.mark.datafiles(DATA_DIR)
-def test_integration_devices(cli, tmpdir, datafiles):
+def test_shell_multiple_workspace(cli, tmpdir, datafiles):
    project = os.path.join(datafiles.dirname, datafiles.basename)
-    element_name = 'integration.bst'
+    elements = {'workspace/workspace-mount.bst': os.path.join(cli.directory, 'workspace-mount'),
+                'make/makehello.bst': os.path.join(cli.directory, 'makehello')}

-    result = execute_shell(cli, project, ["true"], element=element_name)
+    for element, workspace in elements.items():
+        res = cli.run(project=project, args=['workspace', 'open', element, workspace])
+        assert res.exit_code == 0
+
+    for workspace in elements.values():
+        with open(os.path.join(workspace, "workspace-exists"), "w") as f:
+            pass
+
+    # Ensure the dependencies of our build failing element are built
+    result = cli.run(project=project, args=['build', 'base.bst'])
+    assert result.exit_code == 0
+
+    args = ['shell', '--build'] + ['-e' + e for e in elements]
+    args += ['--', 'sh', '-c',
+             'test -e /buildstream/test/workspace/workspace-mount.bst/workspace-exists && \
+              test -e /buildstream/test/make/makehello.bst/workspace-exists']
+    result = cli.run(project=project, args=args)
    assert result.exit_code == 0
+    assert result.output == ''
--- a/tests/sources/pip.py
+++ b/tests/sources/pip.py
@@ -3,6 +3,7 @@ import pytest

 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
+from buildstream.plugins.sources.pip import _match_package_name
 from tests.testutils import cli

 DATA_DIR = os.path.join(
@@ -45,3 +46,22 @@ def test_no_packages(cli, tmpdir, datafiles):
        'show', 'target.bst'
    ])
    result.assert_main_error(ErrorDomain.SOURCE, None)
+
+
+# Test that pip source parses tar ball names correctly for the ref
+@pytest.mark.parametrize(
+    'tarball, expected_name, expected_version',
+    [
+        ('dotted.package-0.9.8.tar.gz', 'dotted.package', '0.9.8'),
+        ('hyphenated-package-2.6.0.tar.gz', 'hyphenated-package', '2.6.0'),
+        ('underscore_pkg-3.1.0.tar.gz', 'underscore_pkg', '3.1.0'),
+        ('numbers2and5-1.0.1.tar.gz', 'numbers2and5', '1.0.1'),
+        ('multiple.dots.package-5.6.7.tar.gz', 'multiple.dots.package', '5.6.7'),
+        ('multiple-hyphens-package-1.2.3.tar.gz', 'multiple-hyphens-package', '1.2.3'),
+        ('multiple_underscore_pkg-3.4.5.tar.gz', 'multiple_underscore_pkg', '3.4.5'),
+        ('shortversion-1.0.tar.gz', 'shortversion', '1.0'),
+        ('longversion-1.2.3.4.tar.gz', 'longversion', '1.2.3.4')
+    ])
+def test_match_package_name(tarball, expected_name, expected_version):
+    name, version = _match_package_name(tarball)
+    assert (expected_name, expected_version) == (name, version)
--- a/tests/testutils/__init__.py
+++ b/tests/testutils/__init__.py
@@ -29,3 +29,4 @@ from .artifactshare import create_artifact_share
 from .element_generators import create_element_size, update_element_size
 from .junction import generate_junction
 from .runner_integration import wait_for_cache_granularity
+from .python_repo import setup_pypi_repo
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -13,7 +13,7 @@ import pytest_cov
 from buildstream import _yaml
 from buildstream._artifactcache.casserver import create_server
 from buildstream._context import Context
-from buildstream._exceptions import ArtifactError
+from buildstream._exceptions import CASError
 from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2


@@ -48,7 +48,7 @@ class ArtifactShare():
        context = Context()
        context.artifactdir = self.repodir

-        self.cas = context.artifactcache
+        self.cas = context.artifactcache.cas

        self.total_space = total_space
        self.free_space = free_space
@@ -135,7 +135,7 @@ class ArtifactShare():
        try:
            tree = self.cas.resolve_ref(artifact_key)
            return True
-        except ArtifactError:
+        except CASError:
            return False

    # close():

--- a/tests/testutils/mock_os.py
+++ b/tests/testutils/mock_os.py
+from contextlib import contextmanager
+import os
+
+
+# MockAttributeResult
+#
+# A class to take a dictionary of kwargs and make them accessible via
+# attributes of the object.
+#
+class MockAttributeResult(dict):
+    __getattr__ = dict.get
+
+
+# mock_statvfs():
+#
+# Gets a function which mocks statvfs and returns a statvfs result with the kwargs accessible.
+#
+# Returns:
+#    func(path) -> object: object will have all the kwargs accessible via object.kwarg
+#
+# Example:
+#    statvfs = mock_statvfs(f_blocks=10)
+#    result = statvfs("regardless/of/path")
+#    assert result.f_blocks == 10 # True
+def mock_statvfs(**kwargs):
+    def statvfs(path):
+        return MockAttributeResult(kwargs)
+    return statvfs
+
+
+# monkey_patch()
+#
+# with monkey_patch("statvfs", custom_statvfs):
+#    assert os.statvfs == custom_statvfs # True
+# assert os.statvfs == custom_statvfs # False
+#
+@contextmanager
+def monkey_patch(to_patch, patched_func):
+    orig = getattr(os, to_patch)
+    setattr(os, to_patch, patched_func)
+    try:
+        yield
+    finally:
+        setattr(os, to_patch, orig)
--- a/tests/testutils/python_repo.py
+++ b/tests/testutils/python_repo.py
+from setuptools.sandbox import run_setup
+import os
+import pytest
+import re
+import shutil
+
+
+SETUP_TEMPLATE = '''\
+from setuptools import setup
+
+setup(
+    name='{name}',
+    version='{version}',
+    description='{name}',
+    packages=['{pkgdirname}'],
+    entry_points={{
+        'console_scripts': [
+            '{pkgdirname}={pkgdirname}:main'
+        ]
+    }}
+)
+'''
+
+# All packages generated via generate_pip_package will have the functions below
+INIT_TEMPLATE = '''\
+def main():
+    print('This is {name}')
+
+def hello(actor='world'):
+    print('Hello {{}}! This is {name}'.format(actor))
+'''
+
+HTML_TEMPLATE = '''\
+<html>
+  <head>
+    <title>Links for {name}</title>
+  </head>
+  <body>
+    <a href='{name}-{version}.tar.gz'>{name}-{version}.tar.gz</a><br />
+  </body>
+</html>
+'''
+
+
+# Creates a simple python source distribution and copies this into a specified
+# directory which is to serve as a mock python repository
+#
+# Args:
+#    tmpdir (str): Directory in which the source files will be created
+#    pypi (str): Directory serving as a mock python repository
+#    name (str): The name of the package to be created
+#    version (str): The version of the package to be created
+#
+# Returns:
+#    None
+#
+def generate_pip_package(tmpdir, pypi, name, version='0.1'):
+    # check if package already exists in pypi
+    pypi_package = os.path.join(pypi, re.sub('[^0-9a-zA-Z]+', '-', name))
+    if os.path.exists(pypi_package):
+        return
+
+    # create the package source files in tmpdir resulting in a directory
+    # tree resembling the following structure:
+    #
+    # tmpdir
+    # |-- setup.py
+    # `-- package
+    #     `-- __init__.py
+    #
+    setup_file = os.path.join(tmpdir, 'setup.py')
+    pkgdirname = re.sub('[^0-9a-zA-Z]+', '', name)
+    with open(setup_file, 'w') as f:
+        f.write(
+            SETUP_TEMPLATE.format(
+                name=name,
+                version=version,
+                pkgdirname=pkgdirname
+            )
+        )
+    os.chmod(setup_file, 0o755)
+
+    package = os.path.join(tmpdir, pkgdirname)
+    os.makedirs(package)
+
+    main_file = os.path.join(package, '__init__.py')
+    with open(main_file, 'w') as f:
+        f.write(INIT_TEMPLATE.format(name=name))
+    os.chmod(main_file, 0o644)
+
+    run_setup(setup_file, ['sdist'])
+
+    # create directory for this package in pypi resulting in a directory
+    # tree resembling the following structure:
+    #
+    # pypi
+    # `-- pypi_package
+    #     |-- index.html
+    #     `-- foo-0.1.tar.gz
+    #
+    os.makedirs(pypi_package)
+
+    # add an index html page
+    index_html = os.path.join(pypi_package, 'index.html')
+    with open(index_html, 'w') as f:
+        f.write(HTML_TEMPLATE.format(name=name, version=version))
+
+    # copy generated tarfile to pypi package
+    dist_dir = os.path.join(tmpdir, 'dist')
+    for tar in os.listdir(dist_dir):
+        tarpath = os.path.join(dist_dir, tar)
+        shutil.copy(tarpath, pypi_package)
+
+
+@pytest.fixture
+def setup_pypi_repo(tmpdir):
+    def create_pkgdir(package):
+        pkgdirname = re.sub('[^0-9a-zA-Z]+', '', package)
+        pkgdir = os.path.join(str(tmpdir), pkgdirname)
+        os.makedirs(pkgdir)
+        return pkgdir
+
+    def add_packages(packages, pypi_repo):
+        for package in packages:
+            pkgdir = create_pkgdir(package)
+            generate_pip_package(pkgdir, pypi_repo, package)
+
+    return add_packages
--- a/tests/utils/misc.py
+++ b/tests/utils/misc.py
+from buildstream import _yaml
+from ..testutils import mock_os
+from ..testutils.runcli import cli
+
+import os
+import pytest
+
+
+KiB = 1024
+MiB = (KiB * 1024)
+GiB = (MiB * 1024)
+TiB = (GiB * 1024)
+
+
+def test_parse_size_over_1024T(cli, tmpdir):
+    BLOCK_SIZE = 4096
+    cli.configure({
+        'cache': {
+            'quota': 2048 * TiB
+        }
+    })
+    project = tmpdir.join("main")
+    os.makedirs(str(project))
+    _yaml.dump({'name': 'main'}, str(project.join("project.conf")))
+
+    bavail = (1025 * TiB) / BLOCK_SIZE
+    patched_statvfs = mock_os.mock_statvfs(f_bavail=bavail, f_bsize=BLOCK_SIZE)
+    with mock_os.monkey_patch("statvfs", patched_statvfs):
+        result = cli.run(project, args=["build", "file.bst"])
+        assert "1025T of available system storage" in result.stderr
No results found