.gitlab-ci.yml: Fix and refactor the whole pipeline

[run]

concurrency = multiprocessing

data_file = .coverage

include =

  */buildgrid/*

source =

  buildgrid/*

omit =

  # Omit profiling helper module

  # Omit generated code

  */buildgrid/google/*

  buildgrid/_app/*

  buildgrid/_protos/*

  *_pb2.py

  *_pb2_grpc.py

  */.eggs/*

  */_app/*

[report]

show_missing = True

precision = 2

[paths]

source =

    buildgrid/

    */site-packages/buildgrid/

    */buildgrid/buildgrid/

# Use a debian image with python 3.5 (and pip3) pre-installed

# Debian Stretch base image with python 3.5:

#

image: python:3.5-stretch

variables:

  BGD: bgd

stages:

  - test

  - post

  - deploy

before_script:

  - python3 -m pip install --upgrade setuptools pip

  - export PATH=~/.local/bin:${PATH}

  - python3 -m pip install --user --editable ".[tests]"

# Templates and global variables.

#

variables:

  BGD: "${CI_PROJECT_DIR}/env/bin/bgd"

  PYTHON: "${CI_PROJECT_DIR}/env/bin/python"

  TOOLS: "${CI_PROJECT_DIR}/env/bin"

.build-template:

  before_script: &install-base

    - apt-get update && apt-get install -y python3 python3-venv python3-pip graphviz

    - python3 -m venv env  # Creates a virtual environment

    - ${PYTHON} -m pip --cache-dir=.pip install --upgrade setuptools pip wheel

    - ${PYTHON} -m pip --cache-dir=.pip install --editable ".[auth,docs,tests]"

  cache:

    paths:

      - .pip

.tests-template: &linux-tests

# Test stage, build and test the code.

#

unit-tests:

  stage: test

  variables:

    PYTEST_ADDOPTS: "--color=yes"

  extends: .build-template

  script:

    - python3 -m pip install --user --editable ".[auth]"

    - python3 setup.py test

    - ${PYTHON} setup.py test

  after_script:

    - mkdir -p coverage/

    - cp .coverage coverage/coverage."${CI_JOB_NAME}"

  variables:

    PYTEST_ADDOPTS: "--color=yes"

  artifacts:

    paths:

      - coverage/

.run-dummy-job-template: &dummy-job

dummy-tests:

  stage: test

  extends: .build-template

  script:

    - ${BGD} server start data/config/default.conf &

    - sleep 1 # Allow server to boot

    - sleep 1  # Allows server to boot

    - ${BGD} bot dummy &

    - ${BGD} cas upload-dummy

    - ${BGD} execute request-dummy --wait-for-completion

# Test stage, build and test the code.

#

tests-debian-stretch:

  <<: *linux-tests

run-dummy-job-debian:

  <<: *dummy-job

buildbox-e2e:

  image: registry.gitlab.com/buildgrid/buildbox/buildbox-e2e:latest

e2e-tests:

  stage: test

  before_script: []

  image: registry.gitlab.com/buildgrid/buildbox/buildbox-e2e:latest

  script:

    - BUILDGRID_SOURCE_ROOT=`pwd` end-to-end-test.sh

# Post-build stage, documentation, coverage report...

#

documentation:

  stage: post

  extends: .build-template

  script:

    - apt-get update && apt-get install -y graphviz

    - python3 -m pip install --user --editable ".[docs]"

    - make -C docs html

    - PATH="${PATH}:${TOOLS}" make -C docs html

  after_script:

    - mkdir -p documentation/

    - cp -a docs/build/html/. documentation/

  artifacts:

coverage:

  stage: post

  extends: .build-template

  dependencies:

  - tests-debian-stretch

    - unit-tests

  coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'

  script:

    - python3 -m pip install --user --editable ".[tests]"

    - cd coverage/

    - ls -l .

    - python3 -m coverage combine --rcfile=../.coveragerc --append coverage.*

    - python3 -m coverage html --rcfile=../.coveragerc --directory .

    - python3 -m coverage report --rcfile=../.coveragerc --show-missing

    - python3 -m coverage erase --rcfile=../.coveragerc

    - cd coverage/ && ls -l .

    - ${PYTHON} -m coverage combine --rcfile=../.coveragerc --append coverage.*

    - ${PYTHON} -m coverage html --rcfile=../.coveragerc --directory .

    - ${PYTHON} -m coverage report --rcfile=../.coveragerc --show-missing

  artifacts:

    paths:

      - coverage/

  only:

    - master

trigger-buildbox-e2e-rebuild:

triggers:

  stage: deploy

  variables:

    GIT_STRATEGY: none

  script:

  before_script:

    - apt-get update && apt-get install -y curl

  script:

    - curl --request POST --form "token=$CI_JOB_TOKEN" --form ref=master https://gitlab.com/api/v4/projects/buildgrid%2Fbuildbox%2Fbuildbox-e2e/trigger/pipeline

  variables:

    GIT_STRATEGY: none

  only:

    - master