build.py 7.98 KB
Newer Older
Stefan Scherfke's avatar
Stefan Scherfke committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269
"""
Build multiple Conda recipes at once and in the order of their dependencies.

Copy the created packages into a local Conda index (by default
:file:`./artifacts`).

To build the current project, just run:

.. code-block:: console

   $ ownconda build

You can, for example, build all external packages at once:

.. code-block:: console

   $ ownconda build-recipes ../external-recipes

Another use-case is to only build the packages that are new then the once in
the Conda repository.  For this, you need to pipe the output of the
``show-updated-recipes`` command to ``build-recipes``:

.. code-block:: console

   $ ownconda show-updated-recipes ../external-recipes/ | xargs ownconda build-recipes


You can also pass a ``--channel`` option which is forwarded to ``conda build``.

"""
import distutils.dir_util
import os
import pathlib
import subprocess

import click


from .. import click_util, constants, recipes, util


def get_build_dir(root_prefix: pathlib.Path):
    """Return the path of the Conda build directory.

    Use ``$CONDA_BLD_PATH`` if set, else *<conda_prefix>/conda-bld*.

    """
    try:
        return pathlib.Path(os.environ['CONDA_BLD_PATH'])
    except KeyError:
        return root_prefix / 'conda-bld'


def filter_secondary_deps(dep_list, dep_graph):
    """Remove nodes from *dep_list* that are not marked as ``primary`` in
    *dep_graph*.

    ``primary`` are those nodes, that were explicitly collected as recipe
    but not added implicitly as dependency.

    """
    primary = {node for node, data in dep_graph.node.items()
               if data.get('primary', False)}
    return [d for d in dep_list if d in primary]


def sort_and_filter(recipe_data, pkg_list):
    """Sort *recipe_data* like *pkg_list* and return the new list.

    *recipe_data* is a list of *(recipe, path)* tuples.

    *pkg_list* is the sorted list of package names.

    """
    len_before = len(recipe_data)

    # Convert the list of "(recipe, path)" tuples into a dict mapping
    # package names to their "(recipe, path)" tuples:
    recipe_dict = {}
    for meta, path in recipe_data:
        recipe_dict.setdefault(meta['package']['name'], []).append((meta, path))

    # Convert the values of the dict back to a list sorted like *pkg_list*:
    build_recipes = [r for pkg_name in pkg_list for r in recipe_dict[pkg_name]]

    assert len(build_recipes) == len_before
    return build_recipes


def build_pkgs(
        recipe_data, pythons, conda_exe, xvfb_prefix, extra_channels, interactive=False,
        no_test=False, quiet=False,
):
    """Run ``conda build`` with all packages in *pkg_list*.

    If *pythons* is a list of python versions (e.g., ``['3.5', '3.6']``) to
    build Python packages against.  It will be ignored for non-Python packages.

    Set the ``--channel`` flags to the local *build_dir* and, optionally, to
    *extra_channels*.

    The list *xvfb_prefix* is either empty or contains a command to initialize
    a xvfb session.

    """
    for meta, path in recipe_data:
        # "builds" is a list of extra-args lists
        if meta['package']['name'] == 'python':
            v = '.'.join(meta['package']['version'].split('.')[:2])
            builds = [[f'--python={v}']]
        elif 'python' in meta.get('requirements', {}).get('build', []):
            builds = [[f'--python={p}'] for p in pythons]
        else:
            builds = [[]]  # One build without extra arguments

        for build_args in builds:
            cmd = xvfb_prefix + [
                conda_exe,
                'build',
                '--error-overlinking',
            ]
            cmd.extend(build_args)
            cmd.extend(extra_channels)
            if no_test:
                cmd.append('--no-test')
            cmd.append(str(path))

            util.run(cmd, quiet=quiet, interactive=interactive)


def init_noarch_dir(conda_exe: str, path: pathlib.Path):
    """Initialize the conda repo "noarch" directory in *path*.

    We don't produce *noarch* packages, but the directoy must still be present.

    """
    path.mkdir(exist_ok=True)
    subprocess.call([conda_exe, 'index', str(path)])


def copy_pkgs(src, dest, *, conda_exe, overwrite=False):
    """Copy the conda index from *src* into *dest*.

    If *dest* does not exist, create it (and missing parent directories).

    Raise an :exc:`FileExistsError` if *dest* already exists and *overwrite* is
    ``False``.

    If *overwrite* is True, files *dest* will be overwritten.

    """
    if dest.exists():
        if overwrite:
            click_util.info(
                f'Target directory "{dest}" already exists.  '
                f'Writing new packages into it.',
            )
        else:
            click_util.warning(
                f'Target directory "{dest}" already exists and will not '
                f'be overwritten.  Packages are left in "{src}".',
            )
    else:
        click_util.ok(f'Copying packages into "{dest}".')

    dest_platform = dest / src.name
    distutils.dir_util.copy_tree(str(src), str(dest_platform))
    # subprocess.call([conda_exe, 'index', dest])  # conda-build = 3.15
    subprocess.call([conda_exe, 'index', str(dest_platform)])  # conda-build < 3.15


@click.command()
@click.pass_obj
@click_util.argument.recipe_root()
@click_util.option.channel(auto_ci_staging=True)
@click_util.option.gui()
@click_util.option.python(multiple=True)
@click.option(
    '--ignore-run-requirements',
    is_flag=True,
    default=False,
    help='Ignore runtime requirements (only use build requirements) when building the '
         'dependency graph.  This is necessary if packages have circular dependencies.',
)
@click.option(
    '--interactive',
    is_flag=True,
    default=False,
    help='Ask users if they want to retry building broken packages.',
)
@click.option(
    '--no-test',
    is_flag=True,
    default=False,
    help='Skip Conda package tests.',
)
@click.option(
    '--overwrite-artifacts',
    is_flag=True,
    default=False,
    help='Overwrite the contents of the "artifacts" direcoty if it '
         'already exists instead of aborting with an error',
)
@click.option(
    '--quiet',
    '-q',
    is_flag=True,
    default=False,
    help='Only print output of "conda build" if an error occurs',
)
def cli(info, recipe_root, channel, gui, python, ignore_run_requirements, interactive,
        no_test, overwrite_artifacts, quiet):
    """Build all recipes in RECIPE_ROOT in the correct order.

    You can specify multiple recipe roots which are all searched recursively
    for recipes.  If no directory is is given, nothing will be built.

    The build dependencies between all recipes ware resolved first so that they
    can be built in the correct order (e.g., packages with no dependies first).

    Copy all packages ("*.tar.bz2") into "./artifacts".

    """
    if not recipe_root:
        return

    conda_exe = info.conda_exe
    build_dir = get_build_dir(info.root_prefix)
    platform_dir = build_dir.joinpath(info.platform)
    extra_channels = (f'--channel=file://{build_dir}',) + channel

    recipe_data = list(recipes.load_recipes(recipe_root))

    # Get sorted list of dependencies
    if ignore_run_requirements:
        collect_types = recipes.BUILD
    else:
        collect_types = recipes.BUILD | recipes.RUN

    dep_graph = recipes.get_dep_graph(
        recipe_data,
        ignore_versions=True,
        collect=collect_types,
    )
    dep_list = recipes.sort_graph(dep_graph)
    dep_list = filter_secondary_deps(dep_list, dep_graph)

    # Build all packages
    recipe_data = sort_and_filter(recipe_data, dep_list)
    build_pkgs(
        recipe_data,
        python,
        conda_exe=conda_exe,
        xvfb_prefix=gui,
        extra_channels=extra_channels,
        interactive=interactive,
        no_test=no_test,
        quiet=quiet,
    )

    copy_pkgs(
        platform_dir,
        constants.artifacts_dir(),
        conda_exe=conda_exe,
        overwrite=overwrite_artifacts,
    )
    # Remove with conda-build >= 3.15:
    init_noarch_dir(conda_exe, build_dir / 'noarch')
    init_noarch_dir(conda_exe, constants.artifacts_dir() / 'noarch')