bst2html.py 17.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
#!/usr/bin/env python3
#
# Copyright (c) 2013 German M. Bravo (Kronuz)
# Copyright (c) 2018 Codethink Limited
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# This file is substantially based on German's work, obtained at:
#     https://github.com/Kronuz/ansi2html.git
#
import os
28
import sys
29
import re
30
import shlex
31
import subprocess
32
from collections.abc import Mapping
33
from contextlib import contextmanager
34 35 36 37 38
from tempfile import TemporaryDirectory

import click

from buildstream import _yaml
39
from buildstream import utils
40
from buildstream._exceptions import BstError
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98


_ANSI2HTML_STYLES = {}
ANSI2HTML_CODES_RE = re.compile('(?:\033\\[(\d+(?:;\d+)*)?([cnRhlABCDfsurgKJipm]))')
ANSI2HTML_PALETTE = {
    # See http://ethanschoonover.com/solarized
    'solarized': ['#073642', '#D30102', '#859900', '#B58900', '#268BD2', '#D33682', '#2AA198', '#EEE8D5',
                  '#002B36', '#CB4B16', '#586E75', '#657B83', '#839496', '#6C71C4', '#93A1A1', '#FDF6E3'],
    # Above mapped onto the xterm 256 color palette
    'solarized-xterm': ['#262626', '#AF0000', '#5F8700', '#AF8700', '#0087FF', '#AF005F', '#00AFAF', '#E4E4E4',
                        '#1C1C1C', '#D75F00', '#585858', '#626262', '#808080', '#5F5FAF', '#8A8A8A', '#FFFFD7'],
    # Gnome default:
    'tango': ['#000000', '#CC0000', '#4E9A06', '#C4A000', '#3465A4', '#75507B', '#06989A', '#D3D7CF',
              '#555753', '#EF2929', '#8AE234', '#FCE94F', '#729FCF', '#AD7FA8', '#34E2E2', '#EEEEEC'],
    # xterm:
    'xterm': ['#000000', '#CD0000', '#00CD00', '#CDCD00', '#0000EE', '#CD00CD', '#00CDCD', '#E5E5E5',
              '#7F7F7F', '#FF0000', '#00FF00', '#FFFF00', '#5C5CFF', '#FF00FF', '#00FFFF', '#FFFFFF'],
    'console': ['#000000', '#AA0000', '#00AA00', '#AA5500', '#0000AA', '#AA00AA', '#00AAAA', '#AAAAAA',
                '#555555', '#FF5555', '#55FF55', '#FFFF55', '#5555FF', '#FF55FF', '#55FFFF', '#FFFFFF'],
}


def _ansi2html_get_styles(palette):
    if palette not in _ANSI2HTML_STYLES:
        p = ANSI2HTML_PALETTE.get(palette, ANSI2HTML_PALETTE['console'])

        regular_style = {
            '1': '',  # bold
            '2': 'opacity:0.5',
            '4': 'text-decoration:underline',
            '5': 'font-weight:bold',
            '7': '',
            '8': 'display:none',
        }
        bold_style = regular_style.copy()
        for i in range(8):
            regular_style['3%s' % i] = 'color:%s' % p[i]
            regular_style['4%s' % i] = 'background-color:%s' % p[i]

            bold_style['3%s' % i] = 'color:%s' % p[i + 8]
            bold_style['4%s' % i] = 'background-color:%s' % p[i + 8]

        # The default xterm 256 colour p:
        indexed_style = {}
        for i in range(16):
            indexed_style['%s' % i] = p[i]

        for rr in range(6):
            for gg in range(6):
                for bb in range(6):
                    i = 16 + rr * 36 + gg * 6 + bb
                    r = (rr * 40 + 55) if rr else 0
                    g = (gg * 40 + 55) if gg else 0
                    b = (bb * 40 + 55) if bb else 0
                    indexed_style['%s' % i] = ''.join('%02X' % c if 0 <= c <= 255 else None for c in (r, g, b))

        for g in range(24):
            i = g + 232
Javier Jardón's avatar
Javier Jardón committed
99 100
            L = g * 10 + 8
            indexed_style['%s' % i] = ''.join('%02X' % c if 0 <= c <= 255 else None for c in (L, L, L))
101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177

        _ANSI2HTML_STYLES[palette] = (regular_style, bold_style, indexed_style)
    return _ANSI2HTML_STYLES[palette]


def ansi2html(text, palette='solarized'):
    def _ansi2html(m):
        if m.group(2) != 'm':
            return ''
        import sys
        state = None
        sub = ''
        cs = m.group(1)
        cs = cs.strip() if cs else ''
        for c in cs.split(';'):
            c = c.strip().lstrip('0') or '0'
            if c == '0':
                while stack:
                    sub += '</span>'
                    stack.pop()
            elif c in ('38', '48'):
                extra = [c]
                state = 'extra'
            elif state == 'extra':
                if c == '5':
                    state = 'idx'
                elif c == '2':
                    state = 'r'
            elif state:
                if state == 'idx':
                    extra.append(c)
                    state = None
                    # 256 colors
                    color = indexed_style.get(c)  # TODO: convert index to RGB!
                    if color is not None:
                        sub += '<span style="%s:%s">' % ('color' if extra[0] == '38' else 'background-color', color)
                        stack.append(extra)
                elif state in ('r', 'g', 'b'):
                    extra.append(c)
                    if state == 'r':
                        state = 'g'
                    elif state == 'g':
                        state = 'b'
                    else:
                        state = None
                        try:
                            color = '#' + ''.join(
                                '%02X' % c if 0 <= c <= 255 else None
                                for x in extra for c in [int(x)]
                            )
                        except (ValueError, TypeError):
                            pass
                        else:
                            sub += '<span style="{}:{}">'.format(
                                'color' if extra[0] == '38' else 'background-color',
                                color)
                            stack.append(extra)
            else:
                if '1' in stack:
                    style = bold_style.get(c)
                else:
                    style = regular_style.get(c)
                if style is not None:
                    sub += '<span style="%s">' % style
                    # Still needs to be added to the stack even if style is empty
                    # (so it can check '1' in stack above, for example)
                    stack.append(c)
        return sub
    stack = []
    regular_style, bold_style, indexed_style = _ansi2html_get_styles(palette)
    sub = ANSI2HTML_CODES_RE.sub(_ansi2html, text)
    while stack:
        sub += '</span>'
        stack.pop()
    return sub


178
# workdir()
179
#
180 181 182 183 184 185 186 187 188 189 190 191
# Sets up a new temp directory with a config file
#
# Args:
#    work_directory (str): The directory where to create a tempdir first
#    source_cache (str): The directory of a source cache to share with, or None
#
# Yields:
#    The buildstream.conf full path
#
@contextmanager
def workdir(source_cache=None):
    with TemporaryDirectory(prefix='run-bst-', dir=os.getcwd()) as tempdir:
192 193
        if not source_cache:
            source_cache = os.path.join(tempdir, 'sources')
194 195 196

        bst_config_file = os.path.join(tempdir, 'buildstream.conf')
        config = {
197
            'cachedir': tempdir,
198 199 200 201 202
            'sourcedir': source_cache,
            'logdir': os.path.join(tempdir, 'logs'),
        }
        _yaml.dump(config, bst_config_file)

203
        yield (tempdir, bst_config_file, source_cache)
204 205


206
# run_bst_command()
207 208 209 210 211 212 213 214 215 216 217
#
# Runs a command
#
# Args:
#    config_file (str): The path to the config file to use
#    directory (str): The project directory
#    command (str): A command string
#
# Returns:
#    (str): The colorized combined stdout/stderr of BuildStream
#
218 219
def run_bst_command(config_file, directory, command):
    click.echo("Running bst command in directory '{}': bst {}".format(directory, command), err=True)
220

221 222 223 224
    argv = ['python3', '-m', 'buildstream', '--colors', '--config', config_file] + shlex.split(command)
    p = subprocess.Popen(argv, cwd=directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    out, _ = p.communicate()
    return out.decode('utf-8').strip()
225 226


227 228 229 230 231 232 233 234 235 236 237 238 239
# run_shell_command()
#
# Runs a command
#
# Args:
#    directory (str): The project directory
#    command (str): A shell command
#
# Returns:
#    (str): The combined stdout/stderr of the shell command
#
def run_shell_command(directory, command):
    click.echo("Running shell command in directory '{}': {}".format(directory, command), err=True)
240

241 242 243 244
    argv = shlex.split(command)
    p = subprocess.Popen(argv, cwd=directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    out, _ = p.communicate()
    return out.decode('utf-8').strip()
245 246 247 248 249 250 251 252 253 254 255 256 257 258


# generate_html
#
# Generate html based on the output
#
# Args:
#    output (str): The output of the BuildStream command
#    directory (str): The project directory
#    config_file (str): The config file
#    source_cache (str): The source cache
#    tempdir (str): The base work directory
#    palette (str): The rendering color style
#    command (str): The command
259
#    fake_output (bool): Whether the provided output is faked or not
260 261 262 263
#
# Returns:
#    (str): The html formatted output
#
264
def generate_html(output, directory, config_file, source_cache, tempdir, palette, command, fake_output):
265 266

    test_base_name = os.path.basename(directory)
267 268 269 270
    if fake_output:
        show_command = command
    else:
        show_command = 'bst ' + command
271 272 273 274 275 276 277 278 279 280 281 282

    # Substitute some things we want normalized for the docs
    output = re.sub(os.environ.get('HOME'), '/home/user', output)
    output = re.sub(config_file, '/home/user/.config/buildstream.conf', output)
    output = re.sub(source_cache, '/home/user/.cache/buildstream/sources', output)
    output = re.sub(tempdir, '/home/user/.cache/buildstream', output)
    output = re.sub(directory, '/home/user/{}'.format(test_base_name), output)

    # Now convert to HTML and add some surrounding sugar
    output = ansi2html(output, palette=palette)

    # Finally format it nicely into a <div>
283 284 285 286 287 288 289 290 291 292 293
    final_output = '<!--\n' + \
                   '    WARNING: This file was generated with bst2html.py\n' + \
                   '-->\n' + \
                   '<div class="highlight" style="font-size:x-small">' + \
                   '<pre>\n' + \
                   '<span style="color:#C4A000;font-weight:bold">user@host</span>:' + \
                   '<span style="color:#3456A4;font-weight:bold">~/{}</span>$ '.format(test_base_name) + \
                   show_command + '\n'

    if output:
        final_output += '\n' + output + '\n'
294

295 296 297
    final_output += '</pre></div>\n'

    return final_output
298 299


300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
# check_needs_build()
#
# Checks whether filename, specified relative to basedir,
# needs to be built (based on whether it exists).
#
# Args:
#    basedir (str): The base directory to check relative of, or None for CWD
#    filename (str): The basedir relative path to the file
#    force (bool): Whether force rebuilding of existing things is enabled
#
# Returns:
#    (bool): Whether the file needs to be built
#
def check_needs_build(basedir, filename, force=False):
    if force:
        return True

    if basedir is None:
        basedir = os.getcwd()

    filename = os.path.join(basedir, filename)
    filename = os.path.realpath(filename)
    if not os.path.exists(filename):
        return True
324

325 326 327 328
    return False


def run_session(description, tempdir, source_cache, palette, config_file, force):
329 330 331
    desc = _yaml.load(description, shortname=os.path.basename(description))
    desc_dir = os.path.dirname(description)

332 333 334 335 336 337 338 339 340 341 342 343 344 345
    # Preflight commands and check if we can skip this session
    #
    if not force:
        needs_build = False
        commands = _yaml.node_get(desc, list, 'commands')
        for command in commands:
            output = _yaml.node_get(command, str, 'output', default_value=None)
            if output is not None and check_needs_build(desc_dir, output, force=False):
                needs_build = True
                break
        if not needs_build:
            click.echo("Skipping '{}' as no files need to be built".format(description), err=True)
            return

346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
    # FIXME: Workaround a setuptools bug where the symlinks
    #        we store in git dont get carried into a release
    #        tarball. This workaround lets us build docs from
    #        a source distribution tarball.
    #
    symlinks = _yaml.node_get(desc, Mapping, 'workaround-symlinks', default_value={})
    for symlink, target in _yaml.node_items(symlinks):

        # Resolve real path to where symlink should be
        symlink = os.path.join(desc_dir, symlink)

        # Ensure dir exists
        symlink_dir = os.path.dirname(symlink)
        os.makedirs(symlink_dir, exist_ok=True)

        click.echo("Generating symlink at: {} (target: {})".format(symlink, target), err=True)

        # Generate a symlink
        try:
            os.symlink(target, symlink)
        except FileExistsError:
            # If the files exist, we're running from a git checkout and
            # not a source distribution, no need to complain
            pass

371 372 373 374 375 376 377 378 379 380
    remove_files = _yaml.node_get(desc, list, 'remove-files', default_value=[])
    for remove_file in remove_files:
        remove_file = os.path.join(desc_dir, remove_file)
        remove_file = os.path.realpath(remove_file)

        if os.path.isdir(remove_file):
            utils._force_rmtree(remove_file)
        else:
            utils.safe_remove(remove_file)

381 382 383 384 385 386 387 388 389 390 391
    # Run commands
    #
    commands = _yaml.node_get(desc, list, 'commands')
    for c in commands:
        command = _yaml.node_get(desc, Mapping, 'commands', indices=[commands.index(c)])

        # Get the directory where this command should be run
        directory = _yaml.node_get(command, str, 'directory')
        directory = os.path.join(desc_dir, directory)
        directory = os.path.realpath(directory)

392
        # Get the command string
393
        command_str = _yaml.node_get(command, str, 'command')
394

395 396 397
        # Check whether this is a shell command and not a bst command
        is_shell = _yaml.node_get(command, bool, 'shell', default_value=False)

398 399 400 401 402
        # Check if there is fake output
        command_fake_output = _yaml.node_get(command, str, 'fake-output', default_value=None)

        # Run the command, or just use the fake output
        if command_fake_output is None:
403 404 405 406
            if is_shell:
                command_out = run_shell_command(directory, command_str)
            else:
                command_out = run_bst_command(config_file, directory, command_str)
407 408
        else:
            command_out = command_fake_output
409 410 411 412 413 414 415

        # Encode and save the output if that was asked for
        output = _yaml.node_get(command, str, 'output', default_value=None)
        if output is not None:
            # Convert / Generate a nice <div>
            converted = generate_html(command_out, directory, config_file,
                                      source_cache, tempdir, palette,
416
                                      command_str, command_fake_output is not None)
417 418 419 420 421 422 423 424 425
            # Save it
            filename = os.path.join(desc_dir, output)
            filename = os.path.realpath(filename)
            output_dir = os.path.dirname(filename)
            os.makedirs(output_dir, exist_ok=True)
            with open(filename, 'wb') as f:
                f.write(converted.encode('utf-8'))

            click.echo("Saved session at '{}'".format(filename), err=True)
426 427 428 429 430 431


@click.command(short_help="Run a bst command and capture stdout/stderr in html")
@click.option('--directory', '-C',
              type=click.Path(file_okay=False, dir_okay=True),
              help="The project directory where to run the command")
432 433
@click.option('--force', is_flag=True, default=False,
              help="Force rebuild, even if the file exists")
434 435 436 437 438 439
@click.option('--source-cache',
              type=click.Path(file_okay=False, dir_okay=True),
              help="A shared source cache")
@click.option('--palette', '-p', default='tango',
              type=click.Choice(['solarized', 'solarized-xterm', 'tango', 'xterm', 'console']),
              help="Selects a palette for the output style")
440
@click.argument('description', type=click.Path(file_okay=True, dir_okay=False, readable=True))
441
def run_bst(directory, force, source_cache, description, palette):
442 443
    """Run a bst command and capture stdout/stderr in html

Adam Jones's avatar
Adam Jones committed
444
    This command normally takes a description yaml file, see the CONTRIBUTING
445
    file for information on its format.
446
    """
447 448
    if not source_cache and os.environ.get('BST_SOURCE_CACHE'):
        source_cache = os.environ['BST_SOURCE_CACHE']
449

450
    with workdir(source_cache=source_cache) as (tempdir, config_file, source_cache):
451
        run_session(description, tempdir, source_cache, palette, config_file, force)
452

453
    return 0
454

Javier Jardón's avatar
Javier Jardón committed
455

456
if __name__ == '__main__':
457 458 459 460 461
    try:
        run_bst()
    except BstError as e:
        click.echo("Error: {}".format(e), err=True)
        sys.exit(-1)