Commit be30995c authored by Kelvin Loh's avatar Kelvin Loh 🖖
Browse files

Merge branch 'create_docker_pipeline' into 'develop'

Create docker pipeline

See merge request !160
parents 0427dabf 971b0061
Loading
Loading
Loading
Loading
Loading
+263 −83
Original line number Diff line number Diff line
workflow:
  # these rules define if a pipeline will be created for certain events
  # note that each job needs its on rules that define when it will run
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_PIPELINE_SOURCE == "push"'
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
      when: never  # avoid duplicate pipelines for pushes for branches with a MR opened
    - if: '$CI_OPEN_MERGE_REQUESTS || $CI_PIPELINE_SOURCE == "push"'
    - if: '$CI_PIPELINE_SOURCE == "external_pull_request_event"'
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - when: always


.shared-windows-runners:
  tags:
@@ -14,28 +20,27 @@ workflow:
# https://hub.docker.com/r/library/python/tags/
image: python:3.7

# Change pip's cache directory to be inside the project directory since we can
# only cache local items.
variables:
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"

# Pip's cache doesn't store the python packages
# https://pip.pypa.io/en/stable/reference/pip_install/#caching
#
# If you want to also cache the installed packages, you have to install
# them in a virtualenv and cache it as well.
cache:
  paths:
    - .cache/pip
    - venv/
  QT_DEBUG_PLUGINS: 0  # change to `1` when facing issues with Qt
  QT_QPA_PLATFORM: offscreen
  DOCKER_BUILDKIT: 1
  DOCKER_HOST: tcp://docker:2375
  DOCKER_DRIVER: overlay

stages:
  - Build unix images manual
  - Build unix images nightly
  - Build windows images manual
  - Static Analysis
  - Test Unix
  - Docs build
  - Test/docs win
  - Docs Unix
  - Test/docs win manual
  - Test/docs win nighly
  - Deploy

services:
  - docker:dind

# ######################################################################################
# Reusable code
# ######################################################################################
@@ -49,14 +54,44 @@ stages:
      - frozen-requirements.txt
    reports:
      cobertura: coverage.xml
    when: always

.run-for-develop-only:
# START rules

# Controlling when a job runs and when not it is a bit messy
# Every job should be either manual or extend (only) one of the following.

.run-nightly-docker-images-build:
  rules:
    # variable set in GitLab -> CI/CD -> Schedules
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $QUANTIFY_BUILD_DOCKER_IMAGES == "true"'

.run-for-develop-and-nighly-only:
  # make any job extend from this if it should run only on develop
  rules:
    # variable set in GitLab -> CI/CD -> Schedules
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $QUANTIFY_BUILD_DOCKER_IMAGES == "true"'
      when: never  # do not run if we are building docker images
    # Only run when merging to develop branch
    - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop"'
    - if: '$CI_COMMIT_BRANCH == "develop"'  # Also run for direct commits on develop
    # variable set in GitLab -> CI/CD -> Schedules
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $QUANTIFY_NIGHTLY_RUN == "true"'
    - when: always

.run-nighly-only:
  # make any job extend from this if it should run only on develop and nightly runs
  rules:
    # variable set in GitLab -> CI/CD -> Schedules
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $QUANTIFY_NIGHTLY_RUN == "true"'

.run-always-except-docker-images:
  rules:
    # variable set in GitLab -> CI/CD -> Schedules
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $QUANTIFY_BUILD_DOCKER_IMAGES == "true"'
      when: never  # do not run if we are building docker images
    - when: always

# END rules

.docs-unix-and-artifacts:
  needs: []  # allow running in parallel with previous stage
@@ -79,20 +114,16 @@ stages:
# this uses YML anchors syntax to be able to reuse this snippets
# anchor usage https://docs.gitlab.com/ee/ci/yaml/#anchors
# Create an anchor called `pip_install_and_freeze`
.pip-install-and-freeze: &pip_install_and_freeze
  - pip install --upgrade --upgrade-strategy eager -e .
  - pip install --upgrade --upgrade-strategy eager -r requirements_dev.txt
.pip-install-and-freeze-unix: &pip_install_and_freeze_unix
  # do not force the upgrade to latest version of packages here, update docker images instead
  - pip install -e . -r requirements_dev.txt
  # save the version of the packages installed for CI debugging
  - pip freeze --all > frozen-requirements.txt; cat frozen-requirements.txt

.unix-before-script-no-install: &unix_before_script_no_install
  - python -V  # Print out python version for debugging
  - pip install virtualenv
  - virtualenv venv
  - source venv/bin/activate
  - apt update
  - apt install -y xvfb  # Sets up a headless display
  - sleep 3
.pip-install-and-freeze-windows: &pip_install_and_freeze_windows
  - pip install -e . -r requirements_dev.txt --upgrade --upgrade-strategy eager
  # save the version of the packages installed for CI debugging
  - pip freeze --all > frozen-requirements.txt; cat frozen-requirements.txt

.upgrade-pip: &upgrade_pip
  - python -V
@@ -100,9 +131,10 @@ stages:
  - python -m pip install --upgrade pip

.unix-before-script:
  variables:
    XDG_RUNTIME_DIR: /tmp/runtime-root  # solves an issue with PyQt5
  before_script:
    - *unix_before_script_no_install
    - *pip_install_and_freeze
    - *pip_install_and_freeze_unix

.win-before-script-3.7.9:
  # version 3.7.10 not available on chocolatey
@@ -110,14 +142,14 @@ stages:
    - choco install python --version=3.7.9 -y -f
    - $env:PATH+=";C:\Python37;C:\Python37\Scripts"
    - *upgrade_pip
    - *pip_install_and_freeze
    - *pip_install_and_freeze_windows

.win-before-script-3.8.9:
  before_script:
    - choco install python --version=3.8.9 -y -f
    - $env:PATH+=";C:\Python38;C:\Python38\Scripts"
    - *upgrade_pip
    - *pip_install_and_freeze
    - *pip_install_and_freeze_windows

.win-before-script-latest:
  before_script:
@@ -127,7 +159,7 @@ stages:
    # assume here the latest is 3.9 will fail when a 3.10 is available
    - $env:PATH+=";C:\Python39;C:\Python39\Scripts"
    - *upgrade_pip
    - *pip_install_and_freeze
    - *pip_install_and_freeze_windows

.pytest-command: &pytest_command
  - pytest -n 2 --mpl --mpl-baseline-path=tests/baseline_images -s --cov=quantify_scheduler --cov-report xml --cov-report html --cov-report term --cov-config=.coveragerc --color=yes
@@ -148,106 +180,122 @@ stages:

test-unix-3.7:
  stage: Test Unix
  image: "python:3.7"
  image: $CI_REGISTRY_IMAGE:python_3.7
  extends:
    - .unix-before-script
    - .unix-run-pytest
    - .save-test-artifacts
    - .run-for-develop-only
    - .run-for-develop-and-nighly-only

test-unix-3.8:
test-unix-3.8: # no rules, always runs such that a minimal test is done on every push
  stage: Test Unix
  image: "python:3.8"
  image: $CI_REGISTRY_IMAGE:python_3.8
  extends:
    - .unix-before-script
    - .unix-run-pytest
    - .save-test-artifacts
    - .run-always-except-docker-images

test-unix-latest:
  stage: Test Unix
  image: "python:latest"
  image: $CI_REGISTRY_IMAGE:python_latest
  extends:
    - .unix-before-script
    - .unix-run-pytest
    - .save-test-artifacts
    - .run-for-develop-only
    - .run-for-develop-and-nighly-only
  allow_failure: true

alternative-install-unix:
  needs: []  # allow running in parallel with previous stage
  stage: Test Unix
  extends:
    - .run-for-develop-only
  script:
    - *unix_before_script_no_install
    - python setup.py bdist_wheel
    # an alternative approach is to install and run:
    - pip install dist/*
    # run the command here
  artifacts:
    paths:
      - dist/*.whl

# ######################################################################################
# Test Unix Windows
# ######################################################################################

test-win-3.7.9:
  stage: Test/docs win
.test-win-3.7.9:
  extends:
    - .shared-windows-runners
    - .win-before-script-3.7.9
    - .win-run-pytest
    - .save-test-artifacts

test-win-3.7.9-manual:
  stage: Test/docs win manual
  extends:
    - .test-win-3.7.9
  when: manual

test-win-3.8.9:
  stage: Test/docs win
test-win-3.7.9-nightly:
  stage: Test/docs win nighly
  extends:
    - .test-win-3.7.9
    - .run-nighly-only

.test-win-3.8.9:
  extends:
    - .shared-windows-runners
    - .win-before-script-3.8.9
    - .win-run-pytest
    - .save-test-artifacts

test-win-3.8.9-manual:
  stage: Test/docs win manual
  extends:
    - .test-win-3.8.9
  when: manual

test-win-latest:
  stage: Test/docs win
test-win-3.8.9-nightly:
  stage: Test/docs win nighly
  extends:
    - .test-win-3.8.9
    - .run-nighly-only

.test-win-latest:
  extends:
    - .shared-windows-runners
    - .win-before-script-latest
    - .win-run-pytest
    - .save-test-artifacts
  when: manual
  # latest version always allowed to fail due to unforeseen changes
  allow_failure: true

test-win-latest-manual:
  stage: Test/docs win manual
  extends:
    - .test-win-latest
  when: manual

test-win-latest-nightly:
  stage: Test/docs win nighly
  extends:
    - .test-win-latest
    - .run-nighly-only

# ######################################################################################
# Docs build Unix
# ######################################################################################

docs-unix-make-html-3.7:
  stage: Docs build
  image: "python:3.7"
  stage: Docs Unix
  image: $CI_REGISTRY_IMAGE:python_3.7
  extends:
    - .unix-before-script
    - .docs-unix-and-artifacts
    - .run-for-develop-only
    - .run-for-develop-and-nighly-only

docs-unix-make-html-3.8:
  stage: Docs build
  image: "python:3.8"
  stage: Docs Unix
  image: $CI_REGISTRY_IMAGE:python_3.8
  extends:
    - .unix-before-script
    - .docs-unix-and-artifacts
    - .run-for-develop-only
    - .run-for-develop-and-nighly-only

docs-unix-make-html-latest:
  stage: Docs build
  image: "python:latest"
  stage: Docs Unix
  image: $CI_REGISTRY_IMAGE:python_latest
  extends:
    - .unix-before-script
    - .docs-unix-and-artifacts
    - .run-for-develop-only
    - .run-for-develop-and-nighly-only
  # latest version always allowed to fail due to unforeseen changes
  allow_failure: true

@@ -255,42 +303,174 @@ docs-unix-make-html-latest:
# Docs build Windows
# ######################################################################################

docs-win-make-html-3.7.9:
  stage: Test/docs win
.docs-win-make-html-3.7.9:
  extends:
    - .shared-windows-runners
    - .win-before-script-3.7.9
    - .docs-win-and-artifacts
  # Allowed to fail since the build was successful with font warnings.
  # See quantify-core#196
  allow_failure: true

docs-win-make-html-3.7.9-manual:
  stage: Test/docs win manual
  extends:
    - .docs-win-make-html-3.7.9
  when: manual

docs-win-make-html-3.8.9:
  stage: Test/docs win
docs-win-make-html-3.7.9-nightly:
  stage: Test/docs win nighly
  extends:
    - .docs-win-make-html-3.7.9
    - .run-nighly-only

.docs-win-make-html-3.8.9:
  extends:
    - .shared-windows-runners
    - .win-before-script-3.8.9
    - .docs-win-and-artifacts
  when: manual
  # Allowed to fail due to asyncio problems (quantify-core#182)
  allow_failure: true

docs-win-make-html-latest:
  stage: Test/docs win
docs-win-make-html-3.8.9-manual:
  stage: Test/docs win manual
  extends:
    - .docs-win-make-html-3.8.9
  when: manual

docs-win-make-html-3.8.9-nightly:
  stage: Test/docs win nighly
  extends:
    - .docs-win-make-html-3.8.9
    - .run-nighly-only

.docs-win-make-html-latest:
  extends:
    - .shared-windows-runners
    - .win-before-script-latest
    - .docs-win-and-artifacts
  when: manual
  # Allowed to fail due to asyncio problems (quantify-core#182)
  allow_failure: true

docs-win-make-html-latest-manual:
  stage: Test/docs win manual
  extends:
    - .docs-win-make-html-latest
  when: manual

docs-win-make-html-latest-nightly:
  stage: Test/docs win nighly
  extends:
    - .docs-win-make-html-latest
    - .run-nighly-only

# ######################################################################################
# Static Analysis
# ######################################################################################

black:
  stage: Static Analysis
  image: python:3.8
  extends:
    - .unix-before-script
  image: $CI_REGISTRY_IMAGE:python_3.8
  script:
    - black --check --fast .
  extends:
    - .run-always-except-docker-images

# ######################################################################################
# Build Docker images
# ######################################################################################

.build-unix-3.7-image:
  image: docker:stable
  variables:
    IMAGE_TAG: $CI_REGISTRY_IMAGE:python_3.7
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    - docker build -t $IMAGE_TAG -f $CI_PROJECT_DIR/docker/unix/Dockerfile_python37 .
    - docker push $IMAGE_TAG

build-unix-3.7-image-manual:
  stage: Build unix images manual
  extends:
    - .build-unix-3.7-image
  when: manual

build-unix-3.7-image-nightly:
  stage: Build unix images nightly
  extends:
    - .build-unix-3.7-image
    - .run-nightly-docker-images-build


.build-unix-3.8-image:
  image: docker:stable
  variables:
    IMAGE_TAG: $CI_REGISTRY_IMAGE:python_3.8
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    - docker build -t $IMAGE_TAG -f $CI_PROJECT_DIR/docker/unix/Dockerfile_python38 .
    - docker push $IMAGE_TAG

build-unix-3.8-image-manual:
  stage: Build unix images manual
  extends:
    - .build-unix-3.8-image
  when: manual

build-unix-3.8-image-nightly:
  stage: Build unix images nightly
  extends:
    - .build-unix-3.8-image
    - .run-nightly-docker-images-build


.build-unix-latest-image:
  image: docker:stable
  variables:
    IMAGE_TAG: $CI_REGISTRY_IMAGE:python_latest
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    - docker build -t $IMAGE_TAG -f $CI_PROJECT_DIR/docker/unix/Dockerfile_python_latest .
    - docker push $IMAGE_TAG

build-unix-latest-image-manual:
  stage: Build unix images manual
  extends:
    - .build-unix-latest-image
  when: manual

build-unix-latest-image-nightly:
  stage: Build unix images nightly
  extends:
    - .build-unix-latest-image
    - .run-nightly-docker-images-build

# Commented out because not being used yet, keeping here for future reference

.build-win-3.7-image:
  extends:
    - .shared-windows-runners
  variables:
    IMAGE_TAG: $CI_REGISTRY_IMAGE:win_python_3.7
    DOCKER_HOST: ""
    DOCKER_BUILDKIT: 0
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    - docker build -t $IMAGE_TAG -f $CI_PROJECT_DIR\docker\windows\Dockerfile_python37 .
    - docker push $IMAGE_TAG

# build-win-3.7-image-manual:
#   stage: Build windows images manual
#   extends:
#     - .build-win-3.7-image
#   when: manual

# build-win-3.7-image-nightly:
#   stage: Build windows images nightly
#   extends:
#     - .build-win-3.7-image
#     - .run-nightly-docker-images-build
+31 −0
Original line number Diff line number Diff line
FROM registry.gitlab.com/quantify-os/quantify-core:python_3.7

LABEL maintainer="Quantify Consortium"
ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
ARG DEBIAN_FRONTEND=noninteractive
ENV TZ=Europe/Amsterdam

# only necessary for unix that runs with headless display
ENV MPLBACKEND=agg

SHELL ["/bin/bash", "-c"]

WORKDIR /pip_env

COPY requirements.txt requirements.txt
COPY requirements_dev.txt requirements_dev.txt

# Print out python version for debugging
RUN python -V
RUN python -m pip --version
RUN python -m pip install --upgrade pip
RUN pip install virtualenv
RUN virtualenv venv
RUN source venv/bin/activate
RUN apt update
# Sets up a headless display
RUN apt install -y xvfb

RUN pip install --upgrade --upgrade-strategy eager -r requirements.txt -r requirements_dev.txt
#save the version of the packages installed for CI debugging
RUN pip freeze --all > frozen-requirements.txt; cat frozen-requirements.txt
+31 −0
Original line number Diff line number Diff line
FROM registry.gitlab.com/quantify-os/quantify-core:python_3.8

LABEL maintainer="Quantify Consortium"
ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
ARG DEBIAN_FRONTEND=noninteractive
ENV TZ=Europe/Amsterdam

# only necessary for unix that runs with headless display
ENV MPLBACKEND=agg

SHELL ["/bin/bash", "-c"]

WORKDIR /pip_env

COPY requirements.txt requirements.txt
COPY requirements_dev.txt requirements_dev.txt

# Print out python version for debugging
RUN python -V
RUN python -m pip --version
RUN python -m pip install --upgrade pip
RUN pip install virtualenv
RUN virtualenv venv
RUN source venv/bin/activate
RUN apt update
# Sets up a headless display
RUN apt install -y xvfb

RUN pip install --upgrade --upgrade-strategy eager -r requirements.txt -r requirements_dev.txt
#save the version of the packages installed for CI debugging
RUN pip freeze --all > frozen-requirements.txt; cat frozen-requirements.txt
+31 −0
Original line number Diff line number Diff line
FROM registry.gitlab.com/quantify-os/quantify-core:python_3.9

LABEL maintainer="Quantify Consortium"
ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
ARG DEBIAN_FRONTEND=noninteractive
ENV TZ=Europe/Amsterdam

# only necessary for unix that runs with headless display
ENV MPLBACKEND=agg

SHELL ["/bin/bash", "-c"]

WORKDIR /pip_env

COPY requirements.txt requirements.txt
COPY requirements_dev.txt requirements_dev.txt

# Print out python version for debugging
RUN python -V
RUN python -m pip --version
RUN python -m pip install --upgrade pip
RUN pip install virtualenv
RUN virtualenv venv
RUN source venv/bin/activate
RUN apt update
# Sets up a headless display
RUN apt install -y xvfb

RUN pip install --upgrade --upgrade-strategy eager -r requirements.txt -r requirements_dev.txt
#save the version of the packages installed for CI debugging
RUN pip freeze --all > frozen-requirements.txt; cat frozen-requirements.txt
+31 −0
Original line number Diff line number Diff line
FROM registry.gitlab.com/quantify-os/quantify-core:python_latest

LABEL maintainer="Quantify Consortium"
ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
ARG DEBIAN_FRONTEND=noninteractive
ENV TZ=Europe/Amsterdam

# only necessary for unix that runs with headless display
ENV MPLBACKEND=agg

SHELL ["/bin/bash", "-c"]

WORKDIR /pip_env

COPY requirements.txt requirements.txt
COPY requirements_dev.txt requirements_dev.txt

# Print out python version for debugging
RUN python -V
RUN python -m pip --version
RUN python -m pip install --upgrade pip
RUN pip install virtualenv
RUN virtualenv venv
RUN source venv/bin/activate
RUN apt update
# Sets up a headless display
RUN apt install -y xvfb

RUN pip install --upgrade --upgrade-strategy eager -r requirements.txt -r requirements_dev.txt
#save the version of the packages installed for CI debugging
RUN pip freeze --all > frozen-requirements.txt; cat frozen-requirements.txt
Loading