.gitlab-ci.yml 16.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
# This file is a template, and might need editing before it works on your project.
# Auto DevOps
# This CI/CD configuration provides a standard pipeline for
# * building a Docker image (using a buildpack if necessary),
# * storing the image in the container registry,
# * running tests from a buildpack,
# * running code quality analysis,
# * creating a review app for each topic branch,
# * and continuous deployment to production
#
# In order to deploy, you must have a Kubernetes cluster configured either
# via a project integration, or via group/project variables.
# AUTO_DEVOPS_DOMAIN must also be set as a variable at the group or project
# level, or manually added below.
#
# If you want to deploy to staging first, or enable canary deploys,
# uncomment the relevant jobs in the pipeline below.
#
# If Auto DevOps fails to detect the proper buildpack, or if you want to
# specify a custom buildpack, set a project variable `BUILDPACK_URL` to the
# repository URL of the buildpack.
# e.g. BUILDPACK_URL=https://github.com/heroku/heroku-buildpack-ruby.git#v142
# If you need multiple buildpacks, add a file to your project called
# `.buildpacks` that contains the URLs, one on each line, in order.
# Note: Auto CI does not work with multiple buildpacks yet

Marin Jankovski's avatar
Marin Jankovski committed
27
image: registry.gitlab.com/charts/gitlab:latest
28 29

variables:
30
  GOOGLE_APPLICATION_CREDENTIALS: ${CI_PROJECT_DIR}/.google_keyfile.json
31 32 33 34 35 36 37 38 39
  # AUTO_DEVOPS_DOMAIN is the application deployment domain and should be set as a variable at the group or project level.
  # AUTO_DEVOPS_DOMAIN: domain.example.com

  POSTGRES_USER: user
  POSTGRES_PASSWORD: testing-password
  POSTGRES_ENABLED: "false"
  POSTGRES_DB: $CI_ENVIRONMENT_SLUG

stages:
40
  - changelog
41 42 43 44
  - test
  - review
  - staging
  - canary
Ahmad Hassan's avatar
Ahmad Hassan committed
45
  - stable
Ahmad Hassan's avatar
Ahmad Hassan committed
46
  - specs
Ahmad Hassan's avatar
Ahmad Hassan committed
47
  - qa
48
  - package
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64
  - cleanup

.test:
  services:
    - postgres:latest
  variables:
    POSTGRES_DB: test
  stage: test
  image: gliderlabs/herokuish:latest
  script:
    - setup_test_db
    - cp -R . /tmp/app
    - /bin/herokuish buildpack test
  only:
    - branches

65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83
lint_package:
  stage: package
  when: always
  script:
    - helm init --client-only
    - helm repo add gitlab https://charts.gitlab.io
    - helm dependencies update
    - helm lint --set certmanager-issuer.email=support@gitlab.com
    - mkdir -p build
    - helm package -d build .
  artifacts:
    expire_in: 3d
    paths:
    - build
  except:
    - tags

release_package:
  stage: package
84
  when: always
85 86 87 88 89 90 91 92
  script:
    - curl --request POST --form "token=$CI_JOB_TOKEN" --form ref=master
        --form "variables[CHART_NAME]=$CI_PROJECT_NAME"
        --form "variables[RELEASE_REF]=$CI_COMMIT_REF_NAME"
        https://gitlab.com/api/v4/projects/2860651/trigger/pipeline
  only:
    - tags

93 94 95
review:
  stage: review
  script:
96
    - terraform_init
97
    - check_kube_domain
98
    - terraform_up
99 100 101 102 103
#    - download_chart
    - ensure_namespace
    - install_tiller
    - create_secret
    - deploy
Ahmad Hassan's avatar
Ahmad Hassan committed
104
    - echo "export QA_ENVIRONMENT_URL=gitlab-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN" >> variables
Ahmad Hassan's avatar
Ahmad Hassan committed
105 106 107 108
    - echo "export GITLAB_ROOT_DOMAIN=$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN"        >> variables
    - echo "export GITLAB_URL=gitlab-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN"         >> variables
    - echo "export REGISTRY_URL=registry-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN"     >> variables
    - echo "export S3_ENDPOINT=https://minio-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN" >> variables
Ahmad Hassan's avatar
Ahmad Hassan committed
109 110 111
  artifacts:
    paths:
    - variables
112 113 114 115 116
  environment:
    name: review/$CI_COMMIT_REF_NAME
    url: https://gitlab-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN
    on_stop: stop_review
  variables:
117
    HOST_SUFFIX: "$CI_ENVIRONMENT_SLUG"
118 119 120 121 122 123 124 125 126 127
    DOMAIN: "-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN"
  only:
    refs:
      - branches
    kubernetes: active
  except:
    - master

stop_review:
  stage: cleanup
128 129
  variables:
    GIT_CHECKOUT: "false"
130
  script:
131
    - git checkout master
132
    - terraform_init
133
    - delete
134
    - cleanup
135
    - terraform_down
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208
  environment:
    name: review/$CI_COMMIT_REF_NAME
    action: stop
  when: manual
  allow_failure: true
  only:
    refs:
      - branches
    kubernetes: active
  except:
    - master

# Keys that start with a dot (.) will not be processed by GitLab CI.
# Staging and canary jobs are disabled by default, to enable them
# remove the dot (.) before the job name.
# https://docs.gitlab.com/ee/ci/yaml/README.html#hidden-keys

# Staging deploys are disabled by default since
# continuous deployment to production is enabled by default
# If you prefer to automatically deploy to staging and
# only manually promote to production, enable this job by removing the dot (.),
# and uncomment the `when: manual` line in the `production` job.

.staging:
  stage: staging
  script:
    - check_kube_domain
    - check_domain_ip
#    - download_chart
    - ensure_namespace
    - install_tiller
    - create_secret
    - deploy
  environment:
    name: staging
    url: https://gitlab-staging.$AUTO_DEVOPS_DOMAIN
  variables:
    DOMAIN: -staging.$AUTO_DEVOPS_DOMAIN
  only:
    refs:
      - master
    kubernetes: active

# Canaries are disabled by default, but if you want them,
# and know what the downsides are, enable this job by removing the dot (.),
# and uncomment the `when: manual` line in the `production` job.

.canary:
  stage: canary
  script:
    - check_kube_domain
    - check_domain_ip
#    - download_chart
    - ensure_namespace
    - install_tiller
    - create_secret
    - deploy canary
  environment:
    name: production
    url: https://gitlab.$AUTO_DEVOPS_DOMAIN
  variables:
    DOMAIN: ".$AUTO_DEVOPS_DOMAIN"
  when: manual
  only:
    refs:
      - master
    kubernetes: active

# This job continuously deploys to production on every push to `master`.
# To make this a manual process, either because you're enabling `staging`
# or `canary` deploys, or you simply want more control over when you deploy
# to production, uncomment the `when: manual` line in the `production` job.

Ahmad Hassan's avatar
Ahmad Hassan committed
209 210
stable:
  stage: stable
211 212 213 214 215 216 217 218 219
  script:
    - check_kube_domain
    - check_domain_ip
    - download_chart
    - ensure_namespace
    - install_tiller
    - create_secret
    - deploy
    - delete canary
Ahmad Hassan's avatar
Ahmad Hassan committed
220
    - echo "export QA_ENVIRONMENT_URL=gitlab.$AUTO_DEVOPS_DOMAIN" >> variables
Ahmad Hassan's avatar
Ahmad Hassan committed
221 222
    - echo "export GITLAB_ROOT_DOMAIN=$AUTO_DEVOPS_DOMAIN"        >> variables
    - echo "export S3_ENDPOINT=https://minio.$AUTO_DEVOPS_DOMAIN" >> variables
Ahmad Hassan's avatar
Ahmad Hassan committed
223 224 225
  artifacts:
    paths:
    - variables
226
  environment:
227
    name: production
228 229 230 231 232 233 234 235 236
    url: https://gitlab.$AUTO_DEVOPS_DOMAIN
  variables:
    DOMAIN: ".$AUTO_DEVOPS_DOMAIN"
#  when: manual
  only:
    refs:
      - master
    kubernetes: active

Ahmad Hassan's avatar
Ahmad Hassan committed
237 238 239 240 241 242 243 244 245 246 247
.specs: &specs
  image: gunesmes/docker-capybara-chrome
  stage: specs
  services:
  - docker:dind
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375
    GITLAB_PASSWORD: $ROOT_PASSWORD
    RELEASE_NAME: $CI_ENVIRONMENT_SLUG
    S3_CONFIG_PATH: /etc/gitlab/minio
248
    LANG: C.UTF-8
Ahmad Hassan's avatar
Ahmad Hassan committed
249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
  script:
    - source variables
    - apt-get update && apt-get install -y --no-install-recommends curl ca-certificates
    - curl -sSL https://get.docker.com/ | sh
    - curl -LsO https://storage.googleapis.com/kubernetes-release/release/v1.9.3/bin/linux/amd64/kubectl
    - chmod +x kubectl
    - mv kubectl /usr/local/bin/kubectl
    - mkdir -p /etc/gitlab/minio
    - kubectl get secret ${CI_ENVIRONMENT_SLUG}-minio-secret -o jsonpath='{.data.accesskey}' | base64 --decode > /etc/gitlab/minio/accesskey
    - kubectl get secret ${CI_ENVIRONMENT_SLUG}-minio-secret -o jsonpath='{.data.secretkey}' | base64 --decode > /etc/gitlab/minio/secretkey
    - bundle install --without non_test
    - bundle exec rspec -c -f d spec

review_specs:
  <<: *specs
  environment:
    name: review/$CI_COMMIT_REF_NAME
    url: https://gitlab-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN
  except:
    refs:
      - master

production_specs:
  <<: *specs
  environment:
    name: production
    url: https://gitlab.$AUTO_DEVOPS_DOMAIN
  only:
    refs:
      - master
Ahmad Hassan's avatar
Ahmad Hassan committed
279 280 281 282 283 284 285 286 287 288 289 290 291
qa:
  image: registry.gitlab.com/gitlab-org/gitlab-omnibus-builder:ruby_docker-0.0.7
  stage: qa
  services:
  - docker:dind
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375
    QA_SCREENSHOTS_DIR: "$CI_PROJECT_DIR/screenshots"
  script:
    - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
    - gem install gitlab-qa
    - source variables
292
    - GITLAB_USERNAME=root GITLAB_PASSWORD=$ROOT_PASSWORD gitlab-qa Test::Instance::Any EE:nightly https://$QA_ENVIRONMENT_URL
Ahmad Hassan's avatar
Ahmad Hassan committed
293 294 295 296 297 298

  artifacts:
    when: on_failure
    expire_in: 7d
    paths:
    - screenshots
299 300 301
  only:
    refs:
      - branches
Ahmad Hassan's avatar
Ahmad Hassan committed
302

303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
debug_review:
  stage: qa
  when: on_failure
  script:
    - kubectl -n "$KUBE_NAMESPACE" describe pod
    - kubectl -n "$KUBE_NAMESPACE" get pod,jobs,secret,ing,cm,sa,svc,role,rolebinding,pvc
  artifacts:
    paths:
    - variables
  environment:
    name: review/$CI_COMMIT_REF_NAME
    url: https://gitlab-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN
  variables:
    HOST_SUFFIX: "$CI_ENVIRONMENT_SLUG"
    DOMAIN: "-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN"
  only:
    refs:
      - branches
    kubernetes: active
  except:
    - master

325 326 327 328
changelog_manager:
  stage: changelog
  script:
    - scripts/changelog_manager.rb .
329
  allow_failure: true
330 331 332 333
  only:
    refs:
      - master

334 335 336 337 338 339 340 341 342 343 344 345
# ---------------------------------------------------------------------------

.auto_devops: &auto_devops |
  # Auto DevOps variables and functions
  [[ "$TRACE" ]] && set -x
  auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB}
  export DATABASE_URL=${DATABASE_URL-$auto_database_url}
  export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG
  export CI_APPLICATION_TAG=$CI_COMMIT_SHA
  export CI_CONTAINER_NAME=ci_job_build_${CI_JOB_ID}
  export TILLER_NAMESPACE=$KUBE_NAMESPACE

346 347
  function previousDeployFailed() {
    set +e
348
    echo "Checking for previous deployment of $CI_ENVIRONMENT_SLUG"
349 350 351 352
    deployment_status=$(helm status $CI_ENVIRONMENT_SLUG >/dev/null 2>&1)
    status=$?
    # if `status` is `0`, deployment exists, has a status
    if [ $status -eq 0 ]; then
353
      echo "Previous deployment found, checking status"
354 355
      deployment_status=$(helm status $CI_ENVIRONMENT_SLUG | grep ^STATUS | cut -d' ' -f2)
      echo "Previous deployment state: $deployment_status"
356
      if [[ "$deployment_status" == "FAILED" || "$deployment_status" == "PENDING_UPGRADE" || "$deployment_status" == "PENDING_INSTALL" ]]; then
357
        status=0;
358 359
      else
        status=1;
360 361 362 363 364 365 366 367
      fi
    else
      echo "Previous deployment NOT found."
    fi
    set -e
    return $status
  }

368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
  function deploy() {
    track="${1-stable}"
    name="$CI_ENVIRONMENT_SLUG"

    if [[ "$track" != "stable" ]]; then
      name="$name-$track"
    fi

    replicas="1"
    service_enabled="false"
    postgres_enabled="$POSTGRES_ENABLED"
    # canary uses stable db
    [[ "$track" == "canary" ]] && postgres_enabled="false"

    env_track=$( echo $track | tr -s  '[:lower:]'  '[:upper:]' )
    env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s  '[:lower:]'  '[:upper:]' )

    if [[ "$track" == "stable" ]]; then
      # for stable track get number of replicas from `PRODUCTION_REPLICAS`
      eval new_replicas=\$${env_slug}_REPLICAS
      service_enabled="true"
    else
      # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS`
      eval new_replicas=\$${env_track}_${env_slug}_REPLICAS
    fi
    if [[ -n "$new_replicas" ]]; then
      replicas="$new_replicas"
    fi

    #ROOT_PASSWORD=$(cat /dev/urandom | LC_TYPE=C tr -dc "[:alpha:]" | head -c 16)
    #echo "Generated root login: $ROOT_PASSWORD"
399 400
    kubectl create secret generic "${CI_ENVIRONMENT_SLUG}-gitlab-initial-root-password" --from-literal=password=$ROOT_PASSWORD -o yaml --dry-run | kubectl replace --force -f -

401
    # YAML_FILE=""${AUTO_DEVOPS_DOMAIN//\./-}.yaml"
402
    # Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
403
    if [ "$CI_ENVIRONMENT_SLUG" != "production" ] && previousDeployFailed ; then
404 405
      echo "Deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG"
      delete
406
      cleanup
407
    fi
Ahmad Hassan's avatar
Ahmad Hassan committed
408
    helm repo add gitlab https://charts.gitlab.io/
409 410
    helm dep update .

411 412
    helm upgrade --install \
      --wait \
413
      --timeout 600 \
414
      --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
415 416
      --set global.hosts.hostSuffix="$HOST_SUFFIX" \
      --set global.hosts.domain="$AUTO_DEVOPS_DOMAIN" \
417
      --set global.hosts.externalIP="$DOMAIN_IP" \
418
      --set global.ingress.tls.secretName=helm-charts-win-tls \
419 420
      --set global.ingress.configureCertmanager=false \
      --set certmanager.install=false \
421
      --set gitlab.unicorn.resources.requests.cpu=400m \
422
      --set gitlab.unicorn.maxReplicas=3 \
423
      --set gitlab.sidekiq.resources.requests.cpu=200m \
424 425
      --set gitlab.sidekiq.maxReplicas=2 \
      --set gitlab.gitlab-shell.resources.requests.cpu=100m \
Ahmad Hassan's avatar
Ahmad Hassan committed
426
      --set gitlab.task-runner.enabled=true \
427 428 429
      --set gitlab.gitlab-shell.maxReplicas=3 \
      --set redis.resources.requests.cpu=100m \
      --set minio.resources.requests.cpu=100m \
430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
      --namespace="$KUBE_NAMESPACE" \
      --version="$CI_PIPELINE_ID-$CI_JOB_ID" \
      "$name" \
      .
  }

  function setup_test_db() {
    if [ -z ${KUBERNETES_PORT+x} ]; then
      DB_HOST=postgres
    else
      DB_HOST=localhost
    fi
    export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}"
  }

  function download_chart() {
    if [[ ! -d chart ]]; then
      auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app}
      auto_chart_name=$(basename $auto_chart)
      auto_chart_name=${auto_chart_name%.tgz}
    else
      auto_chart="chart"
      auto_chart_name="chart"
    fi

    helm init --client-only
    helm repo add gitlab https://charts.gitlab.io
    if [[ ! -d "$auto_chart" ]]; then
      helm fetch ${auto_chart} --untar
    fi
    if [ "$auto_chart_name" != "chart" ]; then
      mv ${auto_chart_name} chart
    fi

    helm dependency update chart/
    helm dependency build chart/
  }

  function ensure_namespace() {
    kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"
  }

  function check_kube_domain() {
    if [ -z ${AUTO_DEVOPS_DOMAIN+x} ]; then
      echo "In order to deploy, AUTO_DEVOPS_DOMAIN must be set as a variable at the group or project level, or manually added in .gitlab-cy.yml"
      false
    else
      true
    fi
  }

  function check_domain_ip() {
    # Expect the `DOMAIN` is a wildcard.
483
    domain_ip=$(nslookup gitlab$DOMAIN 2>/dev/null | grep "Address 1:" | cut -d' ' -f3)
484
    if [ -z $domain_ip ]; then
485
      echo "There was a problem resolving the IP of 'gitlab$DOMAIN'. Be sure you have configured a DNS entry."
486 487 488
      false
    else
      export DOMAIN_IP=$domain_ip
489
      echo "Found IP for gitlab$DOMAIN: $DOMAIN_IP"
490 491 492 493 494 495
      true
    fi
  }

  function install_tiller() {
    echo "Checking Tiller..."
DJ Mountney's avatar
DJ Mountney committed
496
    helm init --upgrade --service-account tiller
497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521
    kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy"
    if ! helm version --debug; then
      echo "Failed to init Tiller."
      return 1
    fi
    echo ""
  }

  function create_secret() {
    kubectl create secret -n "$KUBE_NAMESPACE" \
      docker-registry gitlab-registry-docker \
      --docker-server="$CI_REGISTRY" \
      --docker-username="$CI_REGISTRY_USER" \
      --docker-password="$CI_REGISTRY_PASSWORD" \
      --docker-email="$GITLAB_USER_EMAIL" \
      -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f -
  }

  function delete() {
    track="${1-stable}"
    name="$CI_ENVIRONMENT_SLUG"

    if [[ "$track" != "stable" ]]; then
      name="$name-$track"
    fi
DJ Mountney's avatar
DJ Mountney committed
522
    helm delete --purge "$name" || true
523 524
  }

525
  function cleanup() {
526
    kubectl -n "$KUBE_NAMESPACE" get ingress,svc,pdb,hpa,deploy,statefulset,job,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1 \
527
      | grep "$CI_ENVIRONMENT_SLUG" \
528 529
      | awk '{print $1}' \
      | xargs kubectl -n "$KUBE_NAMESPACE" delete \
530
      || true
531 532
  }

533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556
  function terraform_up() {
    pushd ci/terraform/
    terraform apply -input=false -auto-approve -var environment=${CI_ENVIRONMENT_SLUG}
    export DOMAIN_IP=$(terraform output loadBalancerIP)
    popd
  }

  function terraform_down() {
    pushd ci/terraform
    terraform destroy -input=false -force -var environment=${CI_ENVIRONMENT_SLUG}
    popd
  }

  function terraform_init() {
    pushd ci/terraform
    echo ${GOOGLE_CLOUD_KEYFILE_JSON} > ${GOOGLE_APPLICATION_CREDENTIALS}
    # gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
    # gcloud config set project $GOOGLE_PROJECT_ID
    terraform init -input=false \
      -backend-config="bucket=${GOOGLE_STORAGE_BUCKET}" \
      -backend-config="prefix=terraform/${CI_ENVIRONMENT_SLUG}"
    popd
  }

557 558
before_script:
  - *auto_devops