fdroid_basebox.py 16.9 KB
Newer Older
Michael Pöhn's avatar
Michael Pöhn committed
1 2
#! /usr/bin/env python3
#
3
# fdroid_basebox.py - bootstrapping VMs for F-Droid buildserver
Michael Pöhn's avatar
Michael Pöhn committed
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
# Copyright (C) 2018 Michael Poehn <michael.poehn@fsfe.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import re
import os
import json
import math
import stat
24
import shutil
Michael Pöhn's avatar
Michael Pöhn committed
25 26 27 28 29 30 31 32
import logging
import tarfile
import tempfile
import subprocess
from argparse import ArgumentParser


SUPPORTED_PROVIDERS = ('libvirt', 'virtualbox')
33
SUPPORTED_DEBVERS = ('stretch64',)
Michael Pöhn's avatar
Michael Pöhn committed
34 35 36 37 38 39 40 41 42 43 44 45


class BaseboxException(Exception):
    pass


def vm_size_str_to_bytes(size):
    ssize = str(size)
    if re.match('[0-9]+G', ssize):
        return int(float(ssize[:-1]) * (2**30))
    if re.match('[0-9]+M', ssize):
        return int(float(ssize[:-1]) * (2**20))
46 47
    raise BaseboxException("Could not convert, size '{}' to bytes. "
                           "(Try something like: '100G')".format(ssize))
Michael Pöhn's avatar
Michael Pöhn committed
48 49


50
def init_params(provider, debver, workdir='.', verbose=False, dry_run=False):
Michael Pöhn's avatar
Michael Pöhn committed
51 52
    """creates dict with carefully chosen parameters for buildservers"""

53 54 55 56
    params = {'vm_name': 'basebox-{}'.format(debver),
              'hostname': 'basebox-{}'.format(debver),
              'vm_ram': 1024,
              'vm_cpu': 1,
Michael Pöhn's avatar
Michael Pöhn committed
57 58
              'size': '1000G',
              'deb_mirror': 'http://deb.debian.org/debian',
59
              'deb_distro': debver[:-2],
60 61 62 63
              'deb_packages': ['apt-transport-https',  # enable HTTPS \
                               'ca-certificates',      # for Debian repos
                               'locales',              # install locale-gen
                               'openssh-server'],      # vagrant requires SSH
Michael Pöhn's avatar
Michael Pöhn committed
64 65 66 67
              'boostrapscript': os.path.join(workdir,
                                             'fdroidserver-customize.sh'),
              'username': 'vagrant',
              'password': 'vagrant',
68
              'boxfile': 'basebox-{debver}-{provider}.box'.format(
69 70 71
                  debver=debver, provider=provider),
              'verbose': verbose,
              'dry_run': dry_run}
Michael Pöhn's avatar
Michael Pöhn committed
72 73 74 75 76 77
    params['size_bytes'] = vm_size_str_to_bytes(params['size'])
    params['size_megs'] = math.ceil(params['size_bytes'] / (2.**20))
    params['size_gigs'] = math.ceil(params['size_bytes'] / (2.**30))
    params['img_name_raw'] = params['vm_name'] + '.raw'
    params['img_path_raw'] = os.path.join(workdir, params['img_name_raw'])
    params['vagrantfile_path'] = os.path.join(workdir, 'Vagrantfile')
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101
    # add some packages which are missing in makebuildserver:
    # (because they have been preinstalled on jessie64.box)
    params['deb_packages'] += ('perl',
                               'git',
                               'git-man',
                               'libcurl3-gnutls',
                               'liberror-perl',
                               'libexpat1',
                               'libldap-2.4-2',
                               'librtmp1',
                               'libsasl2-2',
                               'libsasl2-modules-db',
                               'libssh2-1',
                               'libjs-excanvas',
                               'libpython-stdlib',
                               'libpython2.7-minimal',
                               'libpython2.7-stdlib',
                               'libsqlite3-0',
                               'mercurial-common',
                               'mime-support',
                               'python',
                               'python-minimal',
                               'python2.7',
                               'python2.7-minimal')
102 103 104 105 106 107 108 109 110 111 112

    if provider == 'libvirt':
        params['img_name_qcow2'] = params['vm_name'] + '.qcow2'
        params['img_path_qcow2'] = os.path.join(workdir,
                                                params['img_name_qcow2'])
        params['metadata_json_path'] = os.path.join(workdir, 'metadata.json')
    elif provider == 'virtualbox':
        params['img_name_vmdk'] = params['vm_name'] + '.vmdk'
        params['img_path_vmdk'] = os.path.join(workdir,
                                               params['img_name_vmdk'])
        params['ovf_path'] = os.path.join(workdir, 'box.ovf')
Michael Pöhn's avatar
Michael Pöhn committed
113 114 115
    return params


116 117 118 119 120 121 122 123
def write_bootstrap_script(params, provider):
    if not params['dry_run']:
        path = params['boostrapscript']
        with open(path, 'w') as f:
            f.write(get_resource_as_string('debootstrap-custom.sh')
                    .format(username=params['username'],
                            deb_mirror=params['deb_mirror'],
                            deb_distro=params['deb_distro'],
124
                            verbose=params['verbose'],
125 126 127 128 129 130
                            provider=provider))
        st = os.stat(path)
        os.chmod(path, st.st_mode |
                 stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
    else:
        logging.info("(dry run) Skip writing bootstrap script to workdir.")
Michael Pöhn's avatar
Michael Pöhn committed
131 132


133
def exec_vmdebootstrap(params, provider):
Michael Pöhn's avatar
Michael Pöhn committed
134

135
    write_bootstrap_script(params,
136
                           provider)
Michael Pöhn's avatar
Michael Pöhn committed
137

138
    cmd = [shutil.which('vmdebootstrap') or 'vmdebootstrap',
Michael Pöhn's avatar
Michael Pöhn committed
139 140 141 142 143 144 145 146 147 148 149 150 151 152
           '--grub',
           '--image={}'.format(params['img_path_raw']),
           '--size={}'.format(params['size']),
           '--mirror={}'.format(params['deb_mirror']),
           '--distribution={}'.format(params['deb_distro']),
           '--hostname={}'.format(params['hostname']),
           '--enable-dhcp',
           '--user={}/{}'.format(params['username'], params['password']),
           '--package={}'.format(','.join(params['deb_packages'])),
           '--customize={}'.format(params['boostrapscript']),
           '--lock-root-password',
           '--systemd-network',
           '--sudo',
           '--sparse']
153
    if params['verbose']:
154
        cmd += ['--verbose']
Michael Pöhn's avatar
Michael Pöhn committed
155 156 157
    if provider == 'virtualbox':
        cmd += ['--debootstrapopts=components=main,contrib']
    logging.debug('> {}'.format(' '.join(cmd)))
158 159 160 161 162 163 164
    if not params['dry_run']:
        logging.info('running vmdebootstrap (please be patient, '
                     'this may take several minutes) ...')
        subprocess.call(cmd)
    else:
        logging.info("(dry run) Skip bootstraping to Debian "
                     "to raw disk image.")
Michael Pöhn's avatar
Michael Pöhn committed
165 166


167
def libvirt_convert_raw_to_qcow2(params):
168
    cmd = (shutil.which('qemu-img') or 'qemu-img',
Michael Pöhn's avatar
Michael Pöhn committed
169 170 171 172 173 174
           'convert',
           '-f', 'raw',
           '-O', 'qcow2',
           params['img_path_raw'],
           params['img_path_qcow2'])
    logging.debug('> {}'.format(' '.join(cmd)))
175
    if not params['dry_run']:
176 177 178 179
        if os.path.isfile(params['img_path_qcow2']):
            os.remove(params['img_path_qcow2'])
            logging.debug('removed already existing file: {}'.format(
                params['img_path_qcow2']))
Michael Pöhn's avatar
Michael Pöhn committed
180 181 182 183 184 185
        logging.info('converting raw image to qcow2 ...')
        subprocess.call(cmd)
    else:
        logging.info('(dry run) Skip converting raw image to qcow2')


186 187
def libvirt_write_metadata_json(params):
    if not params['dry_run']:
Michael Pöhn's avatar
Michael Pöhn committed
188 189 190 191 192 193 194 195 196 197
        with open(params['metadata_json_path'], 'w') as f:
            f.write(json.dumps({'provider': 'libvirt',
                                'format': 'qcow2',
                                'virtual_size': str(params['size_gigs'])},
                               indent=2))
    else:
        logging.info("(dry run) skipped generating '{}'"
                     .format(params['metadata_json_path']))


198 199
def libvirt_write_vagrantfile(params):
    if not params['dry_run']:
Michael Pöhn's avatar
Michael Pöhn committed
200
        with open(params['vagrantfile_path'], 'w') as f:
201 202
            f.write(get_resource_as_string('libvirt.Vagrantfile')
                    .format(**params))
Michael Pöhn's avatar
Michael Pöhn committed
203 204 205 206 207
    else:
        logging.info("(dry run) skipped generating '{}'"
                     .format(params['vagrantfile_path']))


208 209
def libvirt_package_box(params):
    if not params['dry_run']:
210 211 212
        logging.info("building vagrant box file: '{boxfile}' "
                     "(please be patient, this may take several minutes)"
                     .format(**params))
Michael Pöhn's avatar
Michael Pöhn committed
213 214 215 216 217 218 219 220 221 222
        with tarfile.open(params['boxfile'], 'w:gz') as f:
            logging.debug('packaging box file (metadata.json) ...')
            with open(params['metadata_json_path'], 'r') as mf:
                logging.debug('contents:\n{}'.format(mf.read()))
            f.add(params['metadata_json_path'], arcname='metadata.json')
            logging.debug('packaging box file (Vagrantfile) ...')
            with open(params['vagrantfile_path'], 'r') as vf:
                logging.debug('contents:\n{}'.format(vf.read()))
            f.add(params['vagrantfile_path'], arcname='Vagrantfile')
            logging.debug('packaging box file (box.img) ...')
223 224
            f.add(os.path.join(os.getcwd(), params['img_path_qcow2']),
                  arcname='box.img')
Michael Pöhn's avatar
Michael Pöhn committed
225 226 227 228 229 230 231 232 233 234 235 236
    else:
        logging.info('(dry run) Skip packaging box file.')


def get_resource_as_string(resource_name):
    path = os.path.dirname(os.path.abspath(__file__))
    path = os.path.join(path, 'resource', resource_name)
    logging.debug("loading resource '{}'".format(resource_name))
    with open(path, 'r') as f:
        return f.read()


237
def vbox_convert_raw_to_vmdk(params):
238
    cmd = (shutil.which('VBoxManage') or 'VBoxManage',
Michael Pöhn's avatar
Michael Pöhn committed
239 240 241 242 243 244
           'convertfromraw',
           params['img_path_raw'],
           params['img_path_vmdk'],
           '--format',
           'VMDK')
    logging.debug('> {}'.format(' '.join(cmd)))
245
    if not params['dry_run']:
246 247 248 249
        if os.path.isfile(params['img_path_vmdk']):
            os.remove(params['img_path_vmdk'])
            logging.debug('removed already existing file: {}'.format(
                params['img_path_vmdk']))
Michael Pöhn's avatar
Michael Pöhn committed
250 251 252 253 254 255
        logging.info('converting raw image to vmdk ...')
        subprocess.call(cmd)
    else:
        logging.info('(dry run) Skip converting raw image to vmdk')


256
def vbox_write_ovf(params):
257
    if params['deb_distro'] == 'stretch':
258 259
        ovf_template = get_resource_as_string('stretch.box.ovf')

260
    if not params['dry_run']:
Michael Pöhn's avatar
Michael Pöhn committed
261 262 263 264 265 266 267
        with open(params['ovf_path'], 'w') as f:
            f.write(ovf_template.format(**params))
    else:
        logging.info("(dry run) skipped generating '{}'"
                     .format(params['ovf_path']))


268 269
def vbox_write_vagrantfile(params):
    if not params['dry_run']:
Michael Pöhn's avatar
Michael Pöhn committed
270
        with open(params['vagrantfile_path'], 'w') as f:
271 272
            f.write(get_resource_as_string('virtualbox.Vagrantfile')
                    .format(**params))
Michael Pöhn's avatar
Michael Pöhn committed
273 274 275 276 277
    else:
        logging.info("(dry run) skipped generating '{}'"
                     .format(params['vagrantfile_path']))


278 279
def vbox_package_box(params):
    if not params['dry_run']:
Michael Pöhn's avatar
Michael Pöhn committed
280 281 282 283 284 285 286 287 288
        logging.info("building vagrant box file: '{boxfile}' "
                     "(please be patient, this may take several minutes)"
                     .format(**params))
        with tarfile.open(params['boxfile'], 'w:gz') as f:
            logging.debug('packaging box file (Vagrantfile) ...')
            with open(params['vagrantfile_path'], 'r') as vf:
                logging.debug('contents:\n{}'.format(vf.read()))
            f.add(params['vagrantfile_path'], arcname='./Vagrantfile')
            logging.debug('packaging box file (box-disk1.vmdk) ...')
289 290
            f.add(os.path.join(os.getcwd(), params['img_path_vmdk']),
                  arcname='./box-disk1.vmdk')
Michael Pöhn's avatar
Michael Pöhn committed
291 292 293 294 295 296 297 298
            logging.debug('packaging box file (box.ovf) ...')
            with open(params['ovf_path'], 'r') as mf:
                logging.debug('contents:\n{}'.format(mf.read()))
            f.add(params['ovf_path'], arcname='./box.ovf')
    else:
        logging.info('(dry run) Skip packaging box file.')


299
def main(provider='virtualbox', debver='stretch64', workdir=None,
300
         verbose=False, dry_run=False, skip_checks=False):
Michael Pöhn's avatar
Michael Pöhn committed
301

302
    if not os.geteuid() == 0 and not skip_checks:
Michael Pöhn's avatar
Michael Pöhn committed
303 304 305 306 307 308 309 310 311 312 313
        raise BaseboxException('This script requires super user privileges. '
                               '(Please use sudo or run as root.)')

    if provider not in SUPPORTED_PROVIDERS:
        providers = ', '.join(SUPPORTED_PROVIDERS)
        raise BaseboxException("provider '{selected}' not supported. "
                               "Supported providers are: {supported}"
                               .format(selected=provider,
                                       supported=providers))
    logging.info("selected provider: '{}'".format(provider))

314 315 316 317 318 319 320
    if debver not in SUPPORTED_DEBVERS:
        supvers = ', '.join(SUPPORTED_DEBVERS)
        raise BaseboxException("target debian version '{debver}' not "
                               "supported. Supported versions are: "
                               "{supvers}".format(debver=debver,
                                                  supvers=supvers))

321
    if not shutil.which('vmdebootstrap') and not skip_checks:
322 323
        raise BaseboxException("Can not find 'vmdebootstrap' executable. "
                               "(Please install vmdebootstrap.)")
324
    if provider == 'virtualbox' and not skip_checks:
325 326 327
        if not shutil.which('VBoxManage'):
            raise BaseboxException("Can not find 'VBoxManage' executable. "
                                   "(Please install virtualbox.)")
328
    elif provider == 'libvirt' and not skip_checks:
329
        if not shutil.which('qemu-img'):
330
            raise BaseboxException("Can not find 'qemu-img' executable. "
331
                                   "(Please install qemu-utils.)")
Michael Pöhn's avatar
Michael Pöhn committed
332

333
    if provider == 'virtualbox' and debver == 'stretch64' and not skip_checks:
334 335 336 337
        if not shutil.which('systemd-nspawn'):
            raise BaseboxException("Can not find 'systemd-nspawn' executable. "
                                   "(Please install systemd-container.)")

Michael Pöhn's avatar
Michael Pöhn committed
338 339
    with tempfile.TemporaryDirectory() as tmpdir:

340
        if not workdir:
Michael Pöhn's avatar
Michael Pöhn committed
341 342 343
            workdir = tmpdir

        # main parameters for this image
344 345
        params = init_params(provider, debver, workdir=workdir,
                             verbose=verbose, dry_run=dry_run)
346
        logging.debug('image parameters: {}'
347
                      .format(json.dumps(params, indent=2, sort_keys=True)))
Michael Pöhn's avatar
Michael Pöhn committed
348

349
        exec_vmdebootstrap(params, provider)
Michael Pöhn's avatar
Michael Pöhn committed
350 351

        if provider == 'libvirt':
352 353 354 355
            libvirt_convert_raw_to_qcow2(params)
            libvirt_write_metadata_json(params)
            libvirt_write_vagrantfile(params)
            libvirt_package_box(params)
Michael Pöhn's avatar
Michael Pöhn committed
356
        elif provider == 'virtualbox':
357 358 359 360
            vbox_convert_raw_to_vmdk(params)
            vbox_write_ovf(params)
            vbox_write_vagrantfile(params)
            vbox_package_box(params)
Michael Pöhn's avatar
Michael Pöhn committed
361 362 363 364 365


if __name__ == '__main__':

    parser = ArgumentParser()
366
    parser.add_argument("-v", "--verbose", action="store_true", default=False,
Michael Pöhn's avatar
Michael Pöhn committed
367 368 369
                        help="Spew out even more information than normal")
    parser.add_argument("-q", "--quiet", action="store_true", default=False,
                        help="Restrict output to warnings and errors")
Michael Pöhn's avatar
Michael Pöhn committed
370
    parser.add_argument('-t', '--target', default="stretch64",
371
                        help="target debian version. "
Michael Pöhn's avatar
Michael Pöhn committed
372
                             "defaults to 'stretch64'. "
373
                             "(supported: " +
374
                             ', '.join(SUPPORTED_DEBVERS) + ')')
Michael Pöhn's avatar
Michael Pöhn committed
375
    parser.add_argument('-p', '--provider', default='virtualbox',
376 377 378 379
                        help="target vagrant provider. "
                             "defaults to 'virtualbox'. "
                             "(supported: " +
                             ', '.join(SUPPORTED_PROVIDERS) + ")")
Michael Pöhn's avatar
Michael Pöhn committed
380 381 382 383 384
    parser.add_argument('-d', '--dry-run', action='store_true', default=False,
                        help='don\'t actually bulild the image')
    parser.add_argument('--workdir', default=None,
                        help="put intermediate steps into requested path, "
                             "instead of temporary dir")
385 386 387 388
    parser.add_argument('-S', '--skip-checks',
                        action='store_true', default=False,
                        help="do not check for dependencies, "
                        "user permissions, etc.")
Michael Pöhn's avatar
Michael Pöhn committed
389 390 391 392 393 394 395 396 397 398 399 400
    options = parser.parse_args()

    # Helpful to differentiate warnings from errors even when on quiet
    logformat = '%(levelname)s: %(message)s'
    loglevel = logging.INFO
    if options.verbose:
        loglevel = logging.DEBUG
    elif options.quiet:
        loglevel = logging.WARN
    logging.basicConfig(format=logformat, level=loglevel)

    try:
401 402
        main(dry_run=options.dry_run,
             provider=options.provider,
403
             debver=options.target,
404 405 406
             verbose=options.verbose,
             workdir=options.workdir,
             skip_checks=options.skip_checks)
Michael Pöhn's avatar
Michael Pöhn committed
407 408
    except BaseboxException as e:
        logging.critical(e)
409
        exit(1)