.gitlab-ci.yml 7.88 KB
Newer Older
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
1
stages:
2
  - prebuild
François Chabot's avatar
François Chabot committed
3
  - lint
4
  - test
Sylvain Witmeyer's avatar
fix ci  
Sylvain Witmeyer committed
5
  - build
6
  - deploy
7
  - release
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
8

9 10 11
variables:
  DOCKER_HOST: tcp://docker:2375/
  DOCKER_DRIVER: overlay2
François Chabot's avatar
François Chabot committed
12 13

# PREBUILD STAGES
14 15
docker_builder:
  stage: prebuild
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
16
  image: docker:18.09-dind
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
17 18
  services:
    - docker:dind
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
19 20
  variables:
    TAG: builder
21 22
  before_script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
23 24
  script:
    - apk add make
25 26
    - docker pull $CI_REGISTRY_IMAGE:builder
    - docker pull $CI_REGISTRY_IMAGE:cpp_builder
François Chabot's avatar
François Chabot committed
27 28
    - make cogment-builder
    - make cogment-cpp-builder
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
29
    - docker push $CI_REGISTRY_IMAGE:builder
30
    - docker push $CI_REGISTRY_IMAGE:cpp_builder
31
  only:
32
    - master
33

François Chabot's avatar
François Chabot committed
34
# LINT STAGES
François Chabot's avatar
François Chabot committed
35 36 37
cpp_lint:
  stage: lint
  image: docker:18.09-dind
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
38 39
  services:
    - docker:dind
François Chabot's avatar
François Chabot committed
40
  script:
François Chabot's avatar
François Chabot committed
41
    - apk add make git
François Chabot's avatar
François Chabot committed
42
    - make cpp_format
François Chabot's avatar
François Chabot committed
43
    - git diff --exit-code
François Chabot's avatar
François Chabot committed
44

45 46
python_lint:
  stage: lint
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
47
  image: python:3.7
48
  script:
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
49
    - pip install pycodestyle mypy
50
    - pycodestyle --exclude=*pb2.py,*pb2_grpc.py sdk_python/cogment/ log_exporter/
Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
51
    - mypy --config-file sdk_python/mypy.ini sdk_python/cogment/
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
52
    - mypy --config-file log_exporter/mypy.ini log_exporter/
53

Sylvain WITMEYER's avatar
Doc3  
Sylvain WITMEYER committed
54 55 56 57 58 59 60
# orchestrator_chart_lint:
#   stage: lint
#   image: 
#     name: alpine/helm
#     entrypoint: [""]
#   script:
#     - helm lint charts/orchestrator/
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
61

62 63 64 65 66 67 68 69 70
cli_test:
  stage: test
  image: golang:1.13
  script: |
    apt-get update -y
    apt-get install -y protobuf-compiler
    cd cli
    go test -v ./...

François Chabot's avatar
François Chabot committed
71
# BUILD STAGES
72 73 74 75 76 77
orchestrator:
  stage: build
  image: docker:18.09-dind
  services:
    - docker:dind
  variables:
François Chabot's avatar
François Chabot committed
78
    IMAGE_NAME: orchestrator
79 80 81 82 83 84
  before_script:
    - apk add make
  artifacts:
    expire_in: 1 week
    paths:
      - _images
85
  script:
86
    - mkdir _images
87
    - make cogment-orchestrator
88 89
    - docker save $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest > _images/orchestrator.tar
    - cat _images/orchestrator.tar | sha256sum -b > _images/orchestrator.hash
90

Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
91
test_py_sdk:
François Chabot's avatar
François Chabot committed
92
  stage: build
Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
93 94 95 96
  image: docker:18.09-dind
  services:
    - docker:dind
  before_script:
François Chabot's avatar
François Chabot committed
97
    - cd sdk_python
98
  script:
Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
99 100
    - docker build -t sdk_py:latest .
    - docker run --rm sdk_py:latest python -m pytest
François Chabot's avatar
François Chabot committed
101

102 103 104 105 106 107 108 109 110 111 112
    # test_log_exporter_todo:
    #   stage: build
    #   image: docker:18.09-dind
    #   services:
    #     - docker:dind
    #   variables:
    #     IMAGE_NAME: log_exporter
    #   before_script:
    #     - cd log_exporter
    #   script:
    #     - docker build -t log_exporter:latest .
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
113 114 115
    # - docker run --rm log_exporter:latest python -m pytest


116 117 118 119 120 121 122 123 124 125 126
autodocs_sdk:
  stage: build
  image: docker:18.09-dind
  services:
    - docker:dind
  before_script:
    - apk add make
  artifacts:
    expire_in: 1 week
    paths:
      - docs/sdk
127
  script:
128 129
    - make docs_grpc

François Chabot's avatar
François Chabot committed
130 131 132 133
# DEPLOY STAGES
deploy_orchestrator:
  stage: deploy
  image: docker:18.09-dind
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
134 135
  services:
    - docker:dind
François Chabot's avatar
François Chabot committed
136
  variables:
François Chabot's avatar
François Chabot committed
137
    IMAGE_NAME: orchestrator
François Chabot's avatar
François Chabot committed
138 139 140 141
  before_script:
    - apk add bash
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
  script:
142
    - docker load -i _images/orchestrator.tar
François Chabot's avatar
François Chabot committed
143
    - ./orchestrator/deploy.sh $CI_REGISTRY_IMAGE/$IMAGE_NAME
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
144
  allow_failure: true
145
  only:
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
    - master

deploy_log_exporter:
  stage: deploy
  image: docker:18.09-dind
  services:
    - docker:dind
  variables:
    IMAGE_NAME: log_exporter
  before_script:
    - apk add bash
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - cd log_exporter
  script:
    - docker build -t $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest .
    - docker run --rm $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest python /exporter/version.py > version.txt
    - docker tag $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest $CI_REGISTRY_IMAGE/$IMAGE_NAME:$(cat version.txt)
    - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest
    - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:$(cat version.txt)
165
  only:
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
166 167
    - master

Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
168

169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
#deploy_chart_orchestrator:
#  stage: deploy
#  image: docker:18.09-dind
#  services:
#    - docker:dind
#  before_script:
#    - apk add bash curl openssl jq
#    - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get > get_helm.sh
#    - chmod +x get_helm.sh
#    - ./get_helm.sh
#    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
#  script:
#    - helm init --client-only
#    - helm package charts/orchestrator/
#
#    - curl --request DELETE --header "Private-Token:${CI_ACCESS_TOKEN}" "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/releases/$CI_COMMIT_TAG"
#    - FILENAME=$(ls | grep orchestrator*.tgz)
#    - URL=$(curl --request POST --header "Private-Token:${CI_ACCESS_TOKEN}" --form "file=@${FILENAME}" "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/uploads" | jq --raw-output '.url')
#
#    - curl --request POST --header "Content-Type:application/json" --header "Private-Token:${CI_ACCESS_TOKEN}"
#      --data "{
#      \"name\":\"Chart Release\",
#      \"tag_name\":\"${CI_COMMIT_TAG}\",
#      \"description\":\"Release of the orchestrator chart\",
#      \"assets\":{
#      \"links\":[{
#      \"name\":\"chart-${FILENAME}\",
#      \"url\":\"${CI_PROJECT_URL}${URL}\"
#      }]
#      }
#      }"
#      ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/releases
#  only:
#    - tags
#
#js_sdk:
#  stage: deploy
#  image: node:10
#  script:
#    - cd sdk_js
#    - npm install
#    - echo "//registry.npmjs.org/:_authToken=$NPM_AUTH_TOKEN" > ~/.npmrc
#    - npm publish
#  allow_failure: true
#  only:
#    - master
François Chabot's avatar
François Chabot committed
215

216 217 218 219 220 221 222 223
python_sdk:
  stage: deploy
  image: python:3.6-alpine
  variables:
    TWINE_USERNAME: $PYPI_USERNAME
    TWINE_PASSWORD: $PYPI_PASSWORD
  script:
    - pip install twine
224
    - cd sdk_python
225 226
    - python setup.py sdist
    - twine upload dist/*
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
227
  allow_failure: true
228
  only:
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
229
    - master
230

Sylvain Witmeyer's avatar
test  
Sylvain Witmeyer committed
231
pages:
Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
232
  stage: deploy
233
  image:
Sylvain Witmeyer's avatar
doc  
Sylvain Witmeyer committed
234 235
    name: squidfunk/mkdocs-material
    entrypoint: [""]
Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
236
  script:
Sylvain Witmeyer's avatar
docs  
Sylvain Witmeyer committed
237
    - mkdocs build
Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
238 239
  artifacts:
    paths:
240 241 242 243 244 245 246 247 248 249 250
      - public
  only:
    - master

cli_release:
  stage: release
  image: golang:1.13
  script: |
    apt-get update -y
    apt-get install -y jq

Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
251 252
    cd cli

253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
    RELEASE_TAG=${CI_COMMIT_TAG:-latest}

    CLI_VERSION=${RELEASE_TAG} make release

    LINUX_BUILD_FILENAME="cogment-linux-amd64"
    MACOS_BUILD_FILENAME="cogment-macOS-amd64"
    WINDOWS_BUILD_FILENAME="cogment-windows-amd64.exe"

    curl --request DELETE --header "Private-Token:${CI_ACCESS_TOKEN}" "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/releases/${RELEASE_TAG}"

    LINUX_BUILD_URL=$(curl --request POST --header "Private-Token:${CI_ACCESS_TOKEN}" --form "file=@build/${LINUX_BUILD_FILENAME}" "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/uploads" | jq --raw-output '.url')
    MACOS_BUILD_URL=$(curl --request POST --header "Private-Token:${CI_ACCESS_TOKEN}" --form "file=@build/${MACOS_BUILD_FILENAME}" "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/uploads" | jq --raw-output '.url')
    WINDOWS_BUILD_URL=$(curl --request POST --header "Private-Token:${CI_ACCESS_TOKEN}" --form "file=@build/${WINDOWS_BUILD_FILENAME}" "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/uploads" | jq --raw-output '.url')

    curl --request POST --header "Content-Type:application/json" --header "Private-Token:${CI_ACCESS_TOKEN}" \
              --data "{
                  \"name\":\"Cogment CLI\",
                  \"tag_name\":\"${RELEASE_TAG}\",
                  \"ref\":\"${CI_COMMIT_SHA}\",
                  \"description\":\"Cogment CLI release\",
                  \"assets\":{
                      \"links\":[{
                          \"name\":\"${LINUX_BUILD_FILENAME}\",
                          \"url\":\"${CI_PROJECT_URL}${LINUX_BUILD_URL}\"
                      },{
                          \"name\":\"${MACOS_BUILD_FILENAME}\",
                          \"url\":\"${CI_PROJECT_URL}${MACOS_BUILD_URL}\"
                      },{
                          \"name\":\"${WINDOWS_BUILD_FILENAME}\",
                          \"url\":\"${CI_PROJECT_URL}${WINDOWS_BUILD_URL}\"
                      }]
                    }
                }" \
              ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/releases

Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
288
  only:
289
    - tags