.gitlab-ci.yml 8.7 KB
Newer Older
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
1
stages:
2
  - prebuild
François Chabot's avatar
François Chabot committed
3
  - lint
Sylvain Witmeyer's avatar
fix ci  
Sylvain Witmeyer committed
4
  - build
François Chabot's avatar
François Chabot committed
5
  - test
6
  - deploy
7
  - release
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
8

9 10 11
variables:
  DOCKER_HOST: tcp://docker:2375/
  DOCKER_DRIVER: overlay2
François Chabot's avatar
François Chabot committed
12 13

# PREBUILD STAGES
14 15
docker_builder:
  stage: prebuild
Yves's avatar
Yves committed
16
  needs: []
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
17
  image: docker:18.09-dind
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
18 19
  services:
    - docker:dind
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
20 21
  variables:
    TAG: builder
22 23
  before_script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
24 25
  script:
    - apk add make
26 27
    - docker pull $CI_REGISTRY_IMAGE:builder
    - docker pull $CI_REGISTRY_IMAGE:cpp_builder
François Chabot's avatar
François Chabot committed
28 29
    - make cogment-builder
    - make cogment-cpp-builder
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
30
    - docker push $CI_REGISTRY_IMAGE:builder
31
    - docker push $CI_REGISTRY_IMAGE:cpp_builder
32
  only:
33
    - master
34

François Chabot's avatar
François Chabot committed
35
# LINT STAGES
François Chabot's avatar
François Chabot committed
36 37
cpp_lint:
  stage: lint
Yves's avatar
Yves committed
38
  needs: []
François Chabot's avatar
François Chabot committed
39
  image: docker:18.09-dind
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
40 41
  services:
    - docker:dind
François Chabot's avatar
François Chabot committed
42
  script:
François Chabot's avatar
François Chabot committed
43
    - apk add make git
François Chabot's avatar
François Chabot committed
44
    - make cpp_format
François Chabot's avatar
François Chabot committed
45
    - git diff --exit-code
François Chabot's avatar
François Chabot committed
46

47 48
python_lint:
  stage: lint
Yves's avatar
Yves committed
49
  needs: []
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
50
  image: python:3.7
51
  script:
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
52
    - pip install pycodestyle mypy
53
    - pycodestyle --exclude=*pb2.py,*pb2_grpc.py sdk_python/cogment/ log_exporter/
Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
54
    - mypy --config-file sdk_python/mypy.ini sdk_python/cogment/
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
55
    - mypy --config-file log_exporter/mypy.ini log_exporter/
56

Sylvain WITMEYER's avatar
Doc3  
Sylvain WITMEYER committed
57 58 59 60 61 62 63
# orchestrator_chart_lint:
#   stage: lint
#   image: 
#     name: alpine/helm
#     entrypoint: [""]
#   script:
#     - helm lint charts/orchestrator/
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
64

65 66
cli_build:
  stage: build
Yves's avatar
Yves committed
67
  needs: []
68 69 70 71 72 73 74 75 76 77 78
  image: golang:1.13
  script: |
    apt-get update -y
    apt-get install -y protobuf-compiler
    cd cli
    CLI_VERSION=${CI_COMMIT_TAG} make release
  artifacts:
    expire_in: 1 week
    paths:
      - cli/build

79 80
cli_test:
  stage: test
Yves's avatar
Yves committed
81
  needs: ["cli_build"]
82 83 84 85 86 87 88
  image: golang:1.13
  script: |
    apt-get update -y
    apt-get install -y protobuf-compiler
    cd cli
    go test -v ./...

François Chabot's avatar
François Chabot committed
89
# BUILD STAGES
90 91
orchestrator:
  stage: build
Yves's avatar
Yves committed
92
  needs: []
93 94 95 96
  image: docker:18.09-dind
  services:
    - docker:dind
  variables:
François Chabot's avatar
François Chabot committed
97
    IMAGE_NAME: orchestrator
Sagar Kurandwad's avatar
Sagar Kurandwad committed
98
    REGISTRY_IMAGE: $CI_REGISTRY_IMAGE
99 100 101 102 103 104
  before_script:
    - apk add make
  artifacts:
    expire_in: 1 week
    paths:
      - _images
105
  script:
106
    - mkdir _images
107
    - make cogment-orchestrator
108 109
    - docker save $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest > _images/orchestrator.tar
    - cat _images/orchestrator.tar | sha256sum -b > _images/orchestrator.hash
110

Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
111
test_py_sdk:
François Chabot's avatar
François Chabot committed
112
  stage: build
Yves's avatar
Yves committed
113
  needs: []
Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
114 115 116 117
  image: docker:18.09-dind
  services:
    - docker:dind
  before_script:
François Chabot's avatar
François Chabot committed
118
    - cd sdk_python
119
  script:
Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
120
    - docker build -t sdk_py:latest .
121
    - docker run --rm sdk_py:latest python -m pytest -v
François Chabot's avatar
François Chabot committed
122

123 124 125 126 127 128 129 130 131 132 133
    # test_log_exporter_todo:
    #   stage: build
    #   image: docker:18.09-dind
    #   services:
    #     - docker:dind
    #   variables:
    #     IMAGE_NAME: log_exporter
    #   before_script:
    #     - cd log_exporter
    #   script:
    #     - docker build -t log_exporter:latest .
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
134 135 136
    # - docker run --rm log_exporter:latest python -m pytest


137 138 139 140 141 142 143 144 145 146 147
# integration_tests:
#   stage: test
#   image: docker:18.09-dind
#   services:
#     - docker:dind
#   before_script:
#     - docker load -i _images/orchestrator.tar
#     - docker tag $CI_REGISTRY_IMAGE/orchestrator:latest cogment_orchestrator:local
#   script:
#     - docker build -t cogment_tests -f tests/Dockerfile .
#     - docker run --rm cogment_tests
François Chabot's avatar
François Chabot committed
148

149 150
autodocs_sdk:
  stage: build
Yves's avatar
Yves committed
151
  needs: []
152 153 154 155 156 157 158 159 160
  image: docker:18.09-dind
  services:
    - docker:dind
  before_script:
    - apk add make
  artifacts:
    expire_in: 1 week
    paths:
      - docs/sdk
161
  script:
162 163
    - make docs_grpc

François Chabot's avatar
François Chabot committed
164 165 166
# DEPLOY STAGES
deploy_orchestrator:
  stage: deploy
Yves's avatar
Yves committed
167
  needs: ["orchestrator"]
François Chabot's avatar
François Chabot committed
168
  image: docker:18.09-dind
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
169 170
  services:
    - docker:dind
François Chabot's avatar
François Chabot committed
171
  variables:
François Chabot's avatar
François Chabot committed
172
    IMAGE_NAME: orchestrator
François Chabot's avatar
François Chabot committed
173 174 175 176
  before_script:
    - apk add bash
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
  script:
177
    - docker load -i _images/orchestrator.tar
François Chabot's avatar
François Chabot committed
178
    - ./orchestrator/deploy.sh $CI_REGISTRY_IMAGE/$IMAGE_NAME
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
179
  allow_failure: true
180
  only:
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
181 182 183 184
    - master

deploy_log_exporter:
  stage: deploy
Yves's avatar
Yves committed
185
  needs: []
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
  image: docker:18.09-dind
  services:
    - docker:dind
  variables:
    IMAGE_NAME: log_exporter
  before_script:
    - apk add bash
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - cd log_exporter
  script:
    - docker build -t $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest .
    - docker run --rm $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest python /exporter/version.py > version.txt
    - docker tag $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest $CI_REGISTRY_IMAGE/$IMAGE_NAME:$(cat version.txt)
    - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest
    - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:$(cat version.txt)
201
  only:
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
202 203
    - master

Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
204

205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
#deploy_chart_orchestrator:
#  stage: deploy
#  image: docker:18.09-dind
#  services:
#    - docker:dind
#  before_script:
#    - apk add bash curl openssl jq
#    - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get > get_helm.sh
#    - chmod +x get_helm.sh
#    - ./get_helm.sh
#    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
#  script:
#    - helm init --client-only
#    - helm package charts/orchestrator/
#
#    - curl --request DELETE --header "Private-Token:${CI_ACCESS_TOKEN}" "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/releases/$CI_COMMIT_TAG"
#    - FILENAME=$(ls | grep orchestrator*.tgz)
#    - URL=$(curl --request POST --header "Private-Token:${CI_ACCESS_TOKEN}" --form "file=@${FILENAME}" "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/uploads" | jq --raw-output '.url')
#
#    - curl --request POST --header "Content-Type:application/json" --header "Private-Token:${CI_ACCESS_TOKEN}"
#      --data "{
#      \"name\":\"Chart Release\",
#      \"tag_name\":\"${CI_COMMIT_TAG}\",
#      \"description\":\"Release of the orchestrator chart\",
#      \"assets\":{
#      \"links\":[{
#      \"name\":\"chart-${FILENAME}\",
#      \"url\":\"${CI_PROJECT_URL}${URL}\"
#      }]
#      }
#      }"
#      ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/releases
#  only:
#    - tags
#
Irv Lustigman's avatar
Irv Lustigman committed
240 241
js_sdk:
 stage: deploy
Yves's avatar
Yves committed
242
 needs: []
Irv Lustigman's avatar
Irv Lustigman committed
243 244 245 246 247 248 249 250 251
 image: node:10
 script:
   - cd sdk_js
   - npm install
   - echo "//registry.npmjs.org/:_authToken=$NPM_AUTH_TOKEN" > ~/.npmrc
   - npm publish
 allow_failure: true
 only:
   - master
François Chabot's avatar
François Chabot committed
252

253 254
python_sdk:
  stage: deploy
Yves's avatar
Yves committed
255
  needs: []
256
  image: python:3.6-slim
257 258 259 260 261
  variables:
    TWINE_USERNAME: $PYPI_USERNAME
    TWINE_PASSWORD: $PYPI_PASSWORD
  script:
    - pip install twine
262
    - cd sdk_python
263 264
    - python setup.py sdist
    - twine upload dist/*
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
265
  allow_failure: true
266
  only:
Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
267
    - master
268

Sylvain Witmeyer's avatar
test  
Sylvain Witmeyer committed
269
pages:
Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
270
  stage: deploy
Yves's avatar
Yves committed
271
  needs: []
272
  image:
Yves's avatar
Yves committed
273
    name: squidfunk/mkdocs-material:5.5.14
Sylvain Witmeyer's avatar
doc  
Sylvain Witmeyer committed
274
    entrypoint: [""]
Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
275
  script:
Sylvain Witmeyer's avatar
docs  
Sylvain Witmeyer committed
276
    - mkdocs build
Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
277 278
  artifacts:
    paths:
279 280 281 282 283 284
      - public
  only:
    - master

cli_release:
  stage: release
Yves's avatar
Yves committed
285
  needs: ["cli_build"]
286 287 288 289 290
  image: golang:1.13
  script: |
    apt-get update -y
    apt-get install -y jq

Sylvain WITMEYER's avatar
Sylvain WITMEYER committed
291 292
    cd cli

293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
    RELEASE_TAG=${CI_COMMIT_TAG:-latest}

    CLI_VERSION=${RELEASE_TAG} make release

    LINUX_BUILD_FILENAME="cogment-linux-amd64"
    MACOS_BUILD_FILENAME="cogment-macOS-amd64"
    WINDOWS_BUILD_FILENAME="cogment-windows-amd64.exe"

    curl --request DELETE --header "Private-Token:${CI_ACCESS_TOKEN}" "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/releases/${RELEASE_TAG}"

    LINUX_BUILD_URL=$(curl --request POST --header "Private-Token:${CI_ACCESS_TOKEN}" --form "file=@build/${LINUX_BUILD_FILENAME}" "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/uploads" | jq --raw-output '.url')
    MACOS_BUILD_URL=$(curl --request POST --header "Private-Token:${CI_ACCESS_TOKEN}" --form "file=@build/${MACOS_BUILD_FILENAME}" "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/uploads" | jq --raw-output '.url')
    WINDOWS_BUILD_URL=$(curl --request POST --header "Private-Token:${CI_ACCESS_TOKEN}" --form "file=@build/${WINDOWS_BUILD_FILENAME}" "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/uploads" | jq --raw-output '.url')

    curl --request POST --header "Content-Type:application/json" --header "Private-Token:${CI_ACCESS_TOKEN}" \
              --data "{
                  \"name\":\"Cogment CLI\",
                  \"tag_name\":\"${RELEASE_TAG}\",
                  \"ref\":\"${CI_COMMIT_SHA}\",
                  \"description\":\"Cogment CLI release\",
                  \"assets\":{
                      \"links\":[{
                          \"name\":\"${LINUX_BUILD_FILENAME}\",
                          \"url\":\"${CI_PROJECT_URL}${LINUX_BUILD_URL}\"
                      },{
                          \"name\":\"${MACOS_BUILD_FILENAME}\",
                          \"url\":\"${CI_PROJECT_URL}${MACOS_BUILD_URL}\"
                      },{
                          \"name\":\"${WINDOWS_BUILD_FILENAME}\",
                          \"url\":\"${CI_PROJECT_URL}${WINDOWS_BUILD_URL}\"
                      }]
                    }
                }" \
              ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/releases

Sylvain Witmeyer's avatar
Sylvain Witmeyer committed
328
  only:
329
    - tags
Yves's avatar
Yves committed
330