Commit 931d84ac authored by Kamil Trzciński's avatar Kamil Trzciński 🔴

Refactor .gitlab-ci.yml

parent ffb6995f
Pipeline #56849611 failed with stages
in 57 minutes and 45 seconds
This diff is collapsed.
cloud-native-image:
image: ruby:2.5-alpine
before_script: []
dependencies: []
stage: post-test
allow_failure: true
variables:
GIT_DEPTH: "1"
cache: {}
when: manual
script:
- gem install gitlab --no-document
- CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng
only:
- tags@gitlab-org/gitlab-ce
- tags@gitlab-org/gitlab-ee
.review-docs: &review-docs
extends: .single-script-job-dedicated-runner
variables:
SCRIPT_NAME: trigger-build-docs
environment:
name: review-docs/$CI_COMMIT_REF_SLUG
# DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are CI variables
# Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693
url: http://$CI_ENVIRONMENT_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX
on_stop: review-docs-cleanup
# Trigger a manual docs build in gitlab-docs only on non docs-only branches.
# Useful to preview the docs changes live.
review-docs-deploy-manual:
<<: *review-docs
stage: build
script:
- gem install gitlab --no-document
- ./$SCRIPT_NAME deploy
when: manual
only:
- branches@gitlab-org/gitlab-ce
- branches@gitlab-org/gitlab-ee
except:
- /(^docs[\/-].*|.*-docs$)/
- /(^qa[\/-].*|.*-qa$)/
# Always trigger a docs build in gitlab-docs only on docs-only branches.
# Useful to preview the docs changes live.
review-docs-deploy:
<<: *review-docs
stage: post-test
script:
- gem install gitlab --no-document
- ./$SCRIPT_NAME deploy
only:
- /(^docs[\/-].*|.*-docs$)/@gitlab-org/gitlab-ce
- /(^docs[\/-].*|.*-docs$)/@gitlab-org/gitlab-ee
except:
- /(^qa[\/-].*|.*-qa$)/
# Cleanup remote environment of gitlab-docs
review-docs-cleanup:
<<: *review-docs
stage: post-cleanup
environment:
name: review-docs/$CI_COMMIT_REF_SLUG
action: stop
script:
- gem install gitlab --no-document
- ./$SCRIPT_NAME cleanup
when: manual
only:
- branches@gitlab-org/gitlab-ce
- branches@gitlab-org/gitlab-ee
docs lint:
extends: .dedicated-runner
image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint"
stage: test
cache: {}
dependencies: []
before_script: []
script:
- scripts/lint-doc.sh
- scripts/lint-changelog-yaml
- mv doc/ /tmp/gitlab-docs/content/$DOCS_GITLAB_REPO_SUFFIX
- cd /tmp/gitlab-docs
# Build HTML from Markdown
- bundle exec nanoc
# Check the internal links
- bundle exec nanoc check internal_links
# Check the internal anchor links
- bundle exec nanoc check internal_anchors
except:
- /(^qa[\/-].*|.*-qa$)/
.assets-compile-cache: &assets-compile-cache
cache:
key: "assets-compile:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v5"
paths:
- vendor/ruby/
- .yarn-cache/
- tmp/cache/assets/sprockets
policy: pull-push
.use-pg: &use-pg
services:
- name: postgres:9.6
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
- name: redis:alpine
gitlab:assets:compile:
<<: *assets-compile-cache
extends: .dedicated-no-docs-pull-cache-job
image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.5.3-git-2.18-chrome-71.0-node-8.x-yarn-1.12-graphicsmagick-1.3.29-docker-18.06.1
dependencies:
- setup-test-env
services:
- docker:stable-dind
variables:
NODE_ENV: "production"
RAILS_ENV: "production"
SETUP_DB: "false"
SKIP_STORAGE_VALIDATION: "true"
WEBPACK_REPORT: "true"
# we override the max_old_space_size to prevent OOM errors
NODE_OPTIONS: --max_old_space_size=3584
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375
script:
- node --version
- yarn install --frozen-lockfile --production --cache-folder .yarn-cache
- free -m
- bundle exec rake gitlab:assets:compile
- time scripts/build_assets_image
- scripts/clean-old-cached-assets
artifacts:
name: webpack-report
expire_in: 31d
paths:
- webpack-report/
- public/assets/
only:
- /.+/@gitlab-org/gitlab-ce
- /.+/@gitlab-org/gitlab-ee
- /.+/@gitlab/gitlabhq
- /.+/@gitlab/gitlab-ee
tags:
- docker
- gitlab-org
compile-assets:
extends: .dedicated-runner
<<: *use-pg
<<: *assets-compile-cache
stage: prepare
script:
- node --version
- yarn install --frozen-lockfile --cache-folder .yarn-cache
- free -m
- bundle exec rake gitlab:assets:compile
- scripts/clean-old-cached-assets
variables:
# we override the max_old_space_size to prevent OOM errors
NODE_OPTIONS: --max_old_space_size=3584
artifacts:
expire_in: 7d
paths:
- node_modules
- public/assets
except:
refs:
- /(^docs[\/-].*|.*-docs$)/
gitlab:ui:visual:
extends: .dedicated-runner
before_script: []
allow_failure: true
dependencies:
- compile-assets
script:
# Remove node modules from GitLab that may conflict with gitlab-ui
- rm -r node_modules
- git clone https://gitlab.com/gitlab-org/gitlab-ui.git
- cp public/assets/application-*.css gitlab-ui/styles/application.css
- cd gitlab-ui
- yarn install
- CSS_URL=./application.css yarn test
only:
changes:
- app/assets/stylesheets/*.scss
- app/assets/stylesheets/**/*.scss
- app/assets/stylesheets/**/**/*.scss
except:
refs:
- /(^docs[\/-].*|.*-docs$)/
- master
variables:
- $CI_COMMIT_MESSAGE =~ /\[skip visual\]/i
artifacts:
paths:
- tests/__image_snapshots__/
karma:
extends: .dedicated-no-docs-pull-cache-job
<<: *use-pg
dependencies:
- compile-assets
- setup-test-env
variables:
# we override the max_old_space_size to prevent OOM errors
NODE_OPTIONS: --max_old_space_size=3584
script:
- export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log
- date
- scripts/gitaly-test-spawn
- date
- bundle exec rake karma
coverage: '/^Statements *: (\d+\.\d+%)/'
artifacts:
name: coverage-javascript
expire_in: 31d
when: always
paths:
- chrome_debug.log
- coverage-javascript/
reports:
junit: junit_karma.xml
jest:
extends: .dedicated-no-docs-and-no-qa-pull-cache-job
<<: *use-pg
dependencies:
- compile-assets
- setup-test-env
script:
- scripts/gitaly-test-spawn
- date
- bundle exec rake karma:fixtures
- date
- yarn jest --ci --coverage
artifacts:
name: coverage-frontend
expire_in: 31d
when: always
paths:
- coverage-frontend/
- junit_jest.xml
reports:
junit: junit_jest.xml
cache:
key: jest
paths:
- tmp/jest/jest/
policy: pull-push
qa:internal:
extends: .dedicated-no-docs-no-db-pull-cache-job
services: []
script:
- cd qa/
- bundle install
- bundle exec rspec
dependencies:
- setup-test-env
qa:selectors:
extends: .dedicated-no-docs-no-db-pull-cache-job
services: []
script:
- cd qa/
- bundle install
- bundle exec bin/qa Test::Sanity::Selectors
dependencies:
- setup-test-env
.qa-frontend-node: &qa-frontend-node
extends: .dedicated-no-docs-no-db-pull-cache-job
stage: test
cache:
key: "$CI_JOB_NAME"
paths:
- .yarn-cache/
policy: pull-push
dependencies: []
before_script: []
script:
- date
- yarn install --frozen-lockfile --cache-folder .yarn-cache
- date
- yarn run webpack-prod
qa-frontend-node:8:
<<: *qa-frontend-node
image: node:8-alpine
qa-frontend-node:10:
<<: *qa-frontend-node
image: node:10-alpine
qa-frontend-node:latest:
<<: *qa-frontend-node
image: node:alpine
allow_failure: true
lint:javascript:report:
extends: .dedicated-no-docs-no-db-pull-cache-job
stage: post-test
dependencies: []
before_script: []
script:
- date
- yarn run eslint-report || true # ignore exit code
artifacts:
name: eslint-report
expire_in: 31d
paths:
- eslint-report.html
jsdoc:
extends: .dedicated-no-docs-no-db-pull-cache-job
stage: post-test
dependencies:
- compile-assets
before_script: []
script:
- date
- yarn run jsdoc || true # ignore exit code
artifacts:
name: jsdoc
expire_in: 31d
paths:
- jsdoc/
\ No newline at end of file
.dedicated-runner:
retry:
max: 1 # This is confusing but this means "2 runs at max".
when:
- unknown_failure
- api_failure
- runner_system_failure
tags:
- gitlab-org
.default-cache: &default-cache
key: "debian-stretch-ruby-2.5.3-node-10.x"
paths:
- vendor/ruby
- .yarn-cache/
- vendor/gitaly-ruby
.dedicated-runner-default-cache:
extends: .dedicated-runner
cache:
<<: *default-cache
# Jobs that only need to pull cache
.dedicated-pull-cache-job:
extends: .dedicated-runner
cache:
<<: *default-cache
policy: pull
stage: test
.dedicated-no-docs-pull-cache-job:
extends: .dedicated-pull-cache-job
except:
- /(^docs[\/-].*|.*-docs$)/
.dedicated-no-docs-and-no-qa-pull-cache-job:
extends: .dedicated-pull-cache-job
except:
- /(^docs[\/-].*|.*-docs$)/
- /(^qa[\/-].*|.*-qa$)/
# Jobs that do not need a DB
.dedicated-no-docs-no-db-pull-cache-job:
extends: .dedicated-no-docs-pull-cache-job
variables:
SETUP_DB: "false"
.single-script-job-dedicated-runner:
extends: .dedicated-runner
image: ruby:2.5-alpine
stage: test
cache: {}
dependencies: []
variables:
GIT_STRATEGY: none
before_script:
# We don't clone the repo by using GIT_STRATEGY: none and only download the
# single script we need here so it's much faster than cloning.
- export SCRIPT_NAME="${SCRIPT_NAME:-$CI_JOB_NAME}"
- apk add --update openssl
- wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/$SCRIPT_NAME
- chmod 755 $(basename $SCRIPT_NAME)
pages:
extends: .dedicated-no-docs-no-db-pull-cache-job
before_script: []
stage: pages
dependencies:
- coverage
- karma
- gitlab:assets:compile
- lint:javascript:report
- jsdoc
script:
- mv public/ .public/
- mkdir public/
- mv coverage/ public/coverage-ruby/ || true
- mv coverage-javascript/ public/coverage-javascript/ || true
- mv eslint-report.html public/ || true
- mv webpack-report/ public/webpack-report/ || true
- cp .public/assets/application-*.css public/application.css || true
- cp .public/assets/application-*.css.gz public/application.css.gz || true
- mv jsdoc/ public/jsdoc/ || true
artifacts:
paths:
- public
only:
- master@gitlab-org/gitlab-ce
- master@gitlab-org/gitlab-ee
package-and-qa:
image: ruby:2.5-alpine
stage: test
before_script: []
dependencies: []
cache: {}
variables:
GIT_DEPTH: "1"
API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}"
retry: 0
script:
- apk add --update openssl curl jq
- gem install gitlab --no-document
- source ./scripts/review_apps/review-apps.sh
- wait_for_job_to_be_done "gitlab:assets:compile"
- ./scripts/trigger-build omnibus
when: manual
only:
- /.+/@gitlab-org/gitlab-ce
- /.+/@gitlab-org/gitlab-ee
.use-pg: &use-pg
services:
- name: postgres:9.6
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
- name: redis:alpine
.use-pg-10: &use-pg-10
services:
- postgres:10.0
- redis:alpine
.use-mysql: &use-mysql
services:
- mysql:5.7
- redis:alpine
.only-schedules-master: &only-schedules-master
only:
- schedules@gitlab-org/gitlab-ce
- schedules@gitlab-org/gitlab-ee
- master@gitlab-org/gitlab-ce
- master@gitlab-org/gitlab-ee
- master@gitlab/gitlabhq
- master@gitlab/gitlab-ee
.gitlab-setup: &gitlab-setup
extends: .dedicated-no-docs-and-no-qa-pull-cache-job
<<: *use-pg
variables:
SETUP_DB: "false"
script:
# Manually clone gitlab-test and only seed this project in
# db/fixtures/development/04_project.rb thanks to SIZE=1 below
- git clone https://gitlab.com/gitlab-org/gitlab-test.git
/home/git/repositories/gitlab-org/gitlab-test.git
- scripts/gitaly-test-spawn
- force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup
artifacts:
when: on_failure
expire_in: 1d
paths:
- log/development.log
.rake-exec: &rake-exec
extends: .dedicated-no-docs-no-db-pull-cache-job
script:
- bundle exec rake $CI_JOB_NAME
.rspec-metadata: &rspec-metadata
extends: .dedicated-pull-cache-job
stage: test
script:
- JOB_NAME=( $CI_JOB_NAME )
- TEST_TOOL=${JOB_NAME[0]}
- export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
- export KNAPSACK_GENERATE_REPORT=true
- export SUITE_FLAKY_RSPEC_REPORT_PATH=${FLAKY_RSPEC_SUITE_REPORT_PATH}
- export FLAKY_RSPEC_REPORT_PATH=rspec_flaky/all_${TEST_TOOL}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
- export NEW_FLAKY_RSPEC_REPORT_PATH=rspec_flaky/new_${TEST_TOOL}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
- export FLAKY_RSPEC_GENERATE_REPORT=true
- export CACHE_CLASSES=true
- cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
- '[[ -f $FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_REPORT_PATH}'
- '[[ -f $NEW_FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${NEW_FLAKY_RSPEC_REPORT_PATH}'
- scripts/gitaly-test-spawn
- knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml"
artifacts:
expire_in: 31d
when: always
paths:
- coverage/
- knapsack/
- rspec_flaky/
- rspec_profiling/
- tmp/capybara/
reports:
junit: junit_rspec.xml
except:
- /(^docs[\/-].*|.*-docs$)/
- /(^qa[\/-].*|.*-qa$)/
.rspec-metadata-pg: &rspec-metadata-pg
<<: *rspec-metadata
<<: *use-pg
.rspec-metadata-pg-10: &rspec-metadata-pg-10
<<: *rspec-metadata
<<: *use-pg-10
image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.5.3-golang-1.11-git-2.18-chrome-71.0-node-10.x-yarn-1.12-postgresql-10-graphicsmagick-1.3.29"
.rspec-metadata-mysql: &rspec-metadata-mysql
<<: *rspec-metadata
<<: *use-mysql
# DB migration, rollback, and seed jobs
.db-migrate-reset: &db-migrate-reset
extends: .dedicated-no-docs-and-no-qa-pull-cache-job
script:
- bundle exec rake db:migrate:reset
dependencies:
- setup-test-env
.migration-paths: &migration-paths
extends: .dedicated-no-docs-and-no-qa-pull-cache-job
variables:
SETUP_DB: "false"
script:
- git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v9.3.0
- git checkout -f FETCH_HEAD
- sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile
- bundle update google-protobuf grpc
- bundle install $BUNDLE_INSTALL_FLAGS
- date
- cp config/gitlab.yml.example config/gitlab.yml
- bundle exec rake db:drop db:create db:schema:load db:seed_fu
- date
- git checkout -f $CI_COMMIT_SHA
- bundle install $BUNDLE_INSTALL_FLAGS
- date
- . scripts/prepare_build.sh
- date
- bundle exec rake db:migrate
dependencies:
- setup-test-env
setup-test-env:
extends: .dedicated-runner-default-cache
<<: *use-pg
stage: prepare
script:
- bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
- scripts/gitaly-test-build # Do not use 'bundle exec' here
artifacts:
expire_in: 7d
paths:
- tmp/tests
- config/secrets.yml
- vendor/gitaly-ruby
except:
- /(^docs[\/-].*|.*-docs$)/
rspec-pg:
<<: *rspec-metadata-pg
parallel: 50
rspec-pg-10:
<<: *rspec-metadata-pg-10
<<: *only-schedules-master
parallel: 50
rspec-mysql:
<<: *rspec-metadata-mysql
<<: *only-schedules-master
parallel: 50
rspec-fast-spec-helper:
<<: *rspec-metadata-pg
script:
- bundle exec rspec spec/fast_spec_helper.rb
.rspec-quarantine: &rspec-quarantine
<<: *only-schedules-master
script:
- export CACHE_CLASSES=true
- scripts/gitaly-test-spawn
- bin/rspec --color --format documentation --tag quarantine spec/
rspec-pg-quarantine:
<<: *rspec-metadata-pg
<<: *rspec-quarantine
allow_failure: true
rspec-mysql-quarantine:
<<: *rspec-metadata-mysql
<<: *rspec-quarantine
allow_failure: true
static-analysis:
extends: .dedicated-no-docs-no-db-pull-cache-job
dependencies:
- compile-assets
- setup-test-env
script:
- scripts/static-analysis
cache:
key: "debian-stretch-ruby-2.5.3-node-10.x-and-rubocop"
paths:
- vendor/ruby
- .yarn-cache/
- tmp/rubocop_cache
policy: pull-push
downtime_check:
<<: *rake-exec
except:
- master
- tags
- /^[\d-]+-stable(-ee)?$/
- /(^docs[\/-].*|.*-docs$)/
- /(^qa[\/-].*|.*-qa$)/
dependencies:
- setup-test-env
ee_compat_check:
<<: *rake-exec
dependencies: []
except:
- master
- tags
- /[\d-]+-stable(-ee)?/
- /^security-/
- branches@gitlab-org/gitlab-ee
- branches@gitlab/gitlab-ee
retry: 0
artifacts:
name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}"
when: always
expire_in: 10d
paths:
- ee_compat_check/patches/*.patch
db:migrate:reset-pg:
<<: *db-migrate-reset
<<: *use-pg
db:migrate:reset-mysql:
<<: *db-migrate-reset
<<: *use-mysql
db:check-schema-pg:
<<: *db-migrate-reset
<<: *use-pg
script:
- source scripts/schema_changed.sh
migration:path-pg:
<<: *migration-paths
<<: *use-pg
migration:path-mysql:
<<: *migration-paths
<<: *use-mysql
.db-rollback: &db-rollback
extends: .dedicated-no-docs-and-no-qa-pull-cache-job
script:
- bundle exec rake db:migrate VERSION=20170523121229
- bundle exec rake db:migrate
dependencies:
- setup-test-env
db:rollback-pg:
<<: *db-rollback
<<: *use-pg
db:rollback-mysql:
<<: *db-rollback
<<: *use-mysql
gitlab:setup-pg:
<<: *gitlab-setup
<<: *use-pg
dependencies:
- setup-test-env
gitlab:setup-mysql:
<<: *gitlab-setup
<<: *use-mysql
dependencies:
- setup-test-env
coverage:
# Don't include dedicated-no-docs-no-db-pull-cache-job here since we need to
# download artifacts from all the rspec jobs instead of from setup-test-env only
extends: .dedicated-runner-default-cache
cache:
policy: pull
variables:
SETUP_DB: "false"
stage: post-test
script:
- bundle exec scripts/merge-simplecov
coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
artifacts:
name: coverage
expire_in: 31d
paths:
- coverage/index.html
- coverage/assets/
except:
- /(^docs[\/-].*|.*-docs$)/
- /(^qa[\/-].*|.*-qa$)/