buildstream/_versions.py

@@ -23,7 +23,7 @@
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 11
+BST_FORMAT_VERSION = 12
 
 
 # The base BuildStream artifact version

buildstream/plugins/sources/pip.py

+#
+#  Copyright Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Chandan Singh <csingh43@bloomberg.net>
+
+"""
+pip - stage python packages using pip
+=====================================
+
+**Usage:**
+
+.. code:: yaml
+
+   # Specify the pip source kind
+   kind: pip
+
+   # Optionally specify the python executable, defaults to system "python"
+   # Note that either the venv module or the virtualenv CLI tool must be
+   # available
+   python-exe: python3.6
+
+   # Optionally specify index url, defaults to PyPi
+   index-url: https://mypypi.example.com/simple
+
+   # Optionally specify the path to requirements files
+   # Note that either 'requirements-files' or 'packages' must be defined
+   requirements-files:
+   - requirements.txt
+
+   # Optionally specify a list of additional packages
+   # Note that either 'requirements-files' or 'packages' must be defined
+   packages:
+   - flake8
+
+   # Optionally specify a relative staging directory
+   directory: path/to/stage
+
+   # Specify the ref. It is a list of strings of format
+   # ``<package-name>==<version>`` separated by ``\n``.
+   # Usually this will be contents of a requirements.txt file where all
+   # package versions have been frozen.
+   ref: "flake8==3.5.0\nmccabe==0.6.1\npkg-resources==0.0.0\npycodestyle==2.3.1\npyflakes==1.6.0"
+
+.. note::
+
+   The ``pip`` plugin is available since :ref:`format version 12 <project_format_version>`
+
+"""
+
+import hashlib
+import os
+
+from buildstream import Consistency, Source, SourceError, utils
+
+_PYPI_INDEX_URL = 'https://pypi.org/simple/'
+
+
+class PipSource(Source):
+    # pylint: disable=attribute-defined-outside-init
+
+    # We need access to previous sources at track time to use requirements.txt
+    # but not at fetch time as self.ref should contain sufficient information
+    # for this plugin
+    requires_previous_sources_track = True
+
+    def configure(self, node):
+        self.node_validate(node, ['index-url', 'packages', 'python-exe', 'ref', 'requirements-files'] +
+                           Source.COMMON_CONFIG_KEYS)
+        self.ref = self.node_get_member(node, str, 'ref', None)
+        self.python_exe = self.node_get_member(node, str, 'python-exe', 'python')
+        self.index_url = self.node_get_member(node, str, 'index-url', _PYPI_INDEX_URL)
+        self.packages = self.node_get_member(node, list, 'packages', [])
+        self.requirements_files = self.node_get_member(node, list, 'requirements-files', [])
+
+        if not (self.packages or self.requirements_files):
+            raise SourceError("{}: Either 'packages' and 'requirements-files' must be specified". format(self))
+
+    def preflight(self):
+        # Try to find a way to open virtual environments on the host
+        try:
+            # Look for the virtualenv CLI first
+            venv = utils.get_host_tool('virtualenv')
+            self.venv_cmd = [venv, '--python', self.python_exe]
+        except utils.ProgramNotFoundError:
+            # Fall back to venv module if it is installed
+            python_exe = utils.get_host_tool(self.python_exe)
+            rc = self.call([python_exe, '-m', 'venv', '--help'])
+            if rc == 0:
+                self.venv_cmd = [python_exe, '-m', 'venv']
+            else:
+                raise SourceError("{}: venv module not found using python: {}"
+                                  .format(self, python_exe))
+
+    def get_unique_key(self):
+        return [self.venv_cmd, self.index_url, self.ref]
+
+    def get_consistency(self):
+        if not self.ref:
+            return Consistency.INCONSISTENT
+        # FIXME add a stronger consistency check
+        # Currently we take the presence of "something" as an indication that
+        # we have the right things in cache
+        if os.path.exists(self.mirror) and os.listdir(self.mirror):
+            return Consistency.CACHED
+        return Consistency.RESOLVED
+
+    def get_ref(self):
+        return self.ref
+
+    def load_ref(self, node):
+        self.ref = self.node_get_member(node, str, 'ref', None)
+
+    def set_ref(self, ref, node):
+        node['ref'] = self.ref = ref
+
+    def track(self, previous_sources_dir):
+        # XXX pip does not offer any public API other than the CLI tool so it
+        # is not feasible to correctly parse the requirements file or to check
+        # which package versions pip is going to install.
+        # See https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program
+        # for details.
+        # As a result, we have to wastefully install the packages during track.
+        with self.tempdir() as tmpdir:
+            pip = self._venv_pip(tmpdir)
+
+            install_args = [pip, 'install', '--index-url', self.index_url]
+            for requirement_file in self.requirements_files:
+                fpath = os.path.join(previous_sources_dir, requirement_file)
+                install_args += ['-r', fpath]
+            install_args += self.packages
+
+            self.call(install_args, fail="Failed to install python packages")
+            _, reqs = self.check_output([pip, 'freeze'])
+
+        return reqs.strip()
+
+    def fetch(self):
+        with self.tempdir() as tmpdir:
+            pip = self._venv_pip(tmpdir)
+            packages = self.ref.strip().split('\n')
+            self.call([pip, 'install',
+                       '--index-url', self.index_url,
+                       '--prefix', self.mirror] +
+                      packages,
+                      fail="Failed to install python packages: {}".format(packages))
+
+    def stage(self, directory):
+        with self.timed_activity("Staging Python packages", silent_nested=True):
+            utils.copy_files(self.mirror, directory)
+
+    @property
+    def mirror(self):
+        """Directory where this source should stage its files
+
+        """
+        path = os.path.join(self.get_mirror_directory(),
+                            self.index_url,
+                            hashlib.sha256(self.ref.encode()).hexdigest())
+        os.makedirs(path, exist_ok=True)
+        return path
+
+    def _venv_pip(self, directory):
+        """Open a virtual environment in given directory and return pip path
+
+        """
+        self.call(self.venv_cmd + [directory], fail="Failed to initialize virtual environment")
+        pip_exe = os.path.join(directory, 'bin', 'pip')
+        if not os.path.isfile(pip_exe):
+            raise SourceError("Failed to initialize virtual environment")
+        return pip_exe
+
+
+def setup():
+    return PipSource

tests/integration/pip_source.py

+import os
+import pytest
+
+from buildstream import _yaml
+
+from tests.testutils import cli_integration as cli
+from tests.testutils.integration import assert_contains
+
+
+pytestmark = pytest.mark.integration
+
+
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    "project"
+)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_pip_source(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    checkout = os.path.join(cli.directory, 'checkout')
+    element_path = os.path.join(project, 'elements')
+    element_name = 'pip/hello.bst'
+
+    element = {
+        'kind': 'import',
+        'sources': [
+            {
+                'kind': 'local',
+                'path': 'files/pip-source'
+            },
+            {
+                'kind': 'pip',
+                'python-exe': 'python3',
+                'index-url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
+                'requirements-files': ['myreqs.txt'],
+                'packages': ['app2']
+            }
+        ]
+    }
+    os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
+    _yaml.dump(element, os.path.join(element_path, element_name))
+
+    result = cli.run(project=project, args=['track', element_name])
+    assert result.exit_code == 0
+
+    result = cli.run(project=project, args=['build', element_name])
+    assert result.exit_code == 0
+
+    result = cli.run(project=project, args=['checkout', element_name, checkout])
+    assert result.exit_code == 0
+
+    assert_contains(checkout, ['/bin', '/bin/app1', '/bin/app2'])

tests/integration/project/files/pip-source/myreqs.txt

+app1

tests/integration/project/files/pypi-repo/app1/index.html

+<html>
+  <head>
+    <title>Links for app1</title>
+  </head>
+  <body>
+    <a href='App1-0.1.tar.gz'>App1-0.1.tar.gz</a><br />
+  </body>
+</html>

tests/integration/project/files/pypi-repo/app2/index.html

+<html>
+  <head>
+    <title>Links for app1</title>
+  </head>
+  <body>
+    <a href='App2-0.1.tar.gz'>App2-0.1.tar.gz</a><br />
+  </body>
+</html>

tests/sources/pip.py

+import os
+import pytest
+
+from buildstream._exceptions import ErrorDomain
+from buildstream import _yaml
+from tests.testutils import cli
+
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'pip',
+)
+
+
+def generate_project(project_dir, tmpdir):
+    project_file = os.path.join(project_dir, "project.conf")
+    _yaml.dump({'name': 'foo'}, project_file)
+
+
+# Test that without ref, consistency is set appropriately.
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
+def test_no_ref(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    assert cli.get_element_state(project, 'target.bst') == 'no reference'
+
+
+# Test that pip is not allowed to be the first source
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'first-source-pip'))
+def test_first_source(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    result = cli.run(project=project, args=[
+        'show', 'target.bst'
+    ])
+    result.assert_main_error(ErrorDomain.ELEMENT, None)
+
+
+# Test that error is raised when neither packges nor requirements files
+# have been specified
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-packages'))
+def test_no_packages(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    result = cli.run(project=project, args=[
+        'show', 'target.bst'
+    ])
+    result.assert_main_error(ErrorDomain.SOURCE, None)

tests/sources/pip/first-source-pip/target.bst

+kind: import
+description: pip should not be allowed to be the first source
+sources:
+- kind: pip
+  python-exe: python3
+  packages:
+  - flake8

tests/sources/pip/no-packages/file

+Hello World!

tests/sources/pip/no-packages/target.bst

+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: local
+  path: file
+- kind: pip
+  python-exe: python3

tests/sources/pip/no-ref/file

+Hello World!

tests/sources/pip/no-ref/target.bst

+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: local
+  path: file
+- kind: pip
+  python-exe: python3
+  packages:
+  - flake8