.gitlab-ci.yml 28.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# De-duplication/DRY: A combination of YAML anchors and the `extends` keyword are used to remove duplication:
#   * `extends` can be used for composing mappings of different keys, but not composing different
#      values into the same array.
#   * YAML anchors are to compose/override entries into the same array. For example,
#     * Multiple `rules` entries, which uses `- <<: *anchor` to introduce a reusable mapping
#       entry like `if:` as an array entry.
#     * Composing a reusable string command as a step of a `script:` entry, which uses `- *anchor` to
#       introduce a string as an array entry.
#     * Composing a reusable array of commands, which uses `- *anchor` to introduce a sub-array
#       as an array entry.  Note that this works because up to one level of nested arrays will
#       still be successfully interpreted as sequential commands in a Gitlab CI `script` entry.
#     * TIP: Use https://yaml-online-parser.appspot.com/ to experiment and test out your YAML
#     * TIP: Use https://gitlab.com/gitlab-com/www-gitlab-com/-/ci/lint to validate your YAML
#   * Also note that not everything is de-duplicated or DRYed up. For example, direct invocations
#     of a script with no arguments, such as `scripts/deploy`, are simple enough that they are
#     listed directly. Extracting them to a YAML anchor would actually add more total characters
#     and unnecessarily increase complexity for no real benefit.
18
#
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
# * Forked repos and deploys: Forked repos do not have permission to deploy, nor run review apps.
#   Since we now use combined build-and-deploy jobs to deploy from the original repo, we need to exclude
#   forks from attempting to run these jobs.  This is done via the `if-merge-request-forked-repo` rule,
#   which runs the `...-build-only` versions of the jobs, while the `...-build-and-deploy` versions
#   use the `if-master-original-repo` and `if-merge-request-original-repo` rules.
#
# * Rules for test jobs: Jobs in the test stage will always be run for original repo master branch,
#   and all MRs on the original repo and forks (see 'Note A' below). Previously, tests had been
#   excluded from running on the original repo master branch, to avoid increasing pipeline run
#   times, and since they were mostly unnecessary due to merge trains. However, since currently
#   there are no excessively-long-running test jobs (see 'Note B' below), we run them always, which
#   allows the rules definitions to be simplified and DRYed up.
#     (Note A): If you want a job to always run (master of original repo, and any MRs whether on the main
#               repo or a fork), you must still specify the `if-master-original-repo-or-merge-request` rule,
#               in order to avoid duplicate pipeline runs for the same commit on branches and MRs.
#               (see https://docs.gitlab.com/ee/ci/troubleshooting.html#job-may-allow-multiple-pipelines-to-run-for-a-single-action)
#               In other words, the master branch is the only branch which triggers pipelines without a
#               merge request; all other pipelines are only triggered via merge requests on the original
#               repo or a fork.
#     (Note B): All excessively-long-running test jobs are currently commented out or set to
#               be run manually. There is also one exception for the `check-handbook-edit-links`
#               job, which is long-running/blocking, but only runs for certain changes, and only
#               for merge requests on the original repo.
42
#     
43
44
45
46
47
48
49
50
51
52
# Job key consistent order (keys which exist only in `default:` are not listed):
#   extends
#   image
#   services
#   interruptible
#   timeout
#   tags
#   stage
#   trigger
#   rules
53
#   needs
54
55
56
57
58
59
60
61
62
#   variables
#   environment
#   cache
#   artifacts
#   parallel
#   before_script
#   script
#   after_script

Chad Woolley's avatar
Chad Woolley committed
63
64
###################################
#
65
# GENERAL/DEFAULT CONFIG:
Chad Woolley's avatar
Chad Woolley committed
66
67
#
###################################
68

69
# Stages define groups of jobs. These are defined globally for the pipeline.
70
# The order defined below defines the execution order for the jobs.
71
72
# Jobs in the same stage will be run in parallel, and jobs in the other stages 
# happen sequentially - after the previous stage. 
73
stages:
74
  - prepare
75
76
  - content-generation
  - build-and-deploy
77
  - test
78
  - cleanup
79

80
# The default keyword sets its child keywords as global defaults for all jobs.
81
# These defaults can be overridden by job-specific configuration.
82
83
84
85
default:
  # Note that the rspec job below uses a different image that also
  # includes chromedriver. If we update the Ruby version for this image,
  # we should also update it for the rspec job.
86
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:www-gitlab-com-3.0
87
88
  tags:
    - gitlab-org
89
  interruptible: true # All jobs are interruptible by default
90
91
92
93
94
95
96
97
  retry:
    max: 2 # This is confusing but this means "3 runs at max".
    when:
      - unknown_failure
      - api_failure
      - runner_system_failure
      - job_execution_timeout
      - stuck_or_timeout_failure
98

99
# These variables are global and apply to all jobs. Specific job variables will override them.
100
variables:
101
102
  ### PERFORMANCE ###
  # GIT_* variables to speed up repo cloning/fetching
Achilleas Pipinellis's avatar
Achilleas Pipinellis committed
103
  GIT_DEPTH: "10"
104
  GIT_STRATEGY: "fetch"
105
  GIT_SUBMODULE_STRATEGY: "none"
106
  # Disabling LFS speeds up jobs, because runners don't have to perform the LFS steps during repo clone/fetch
107
  GIT_LFS_SKIP_SMUDGE: "1"
108
109
110
111
  # NO_CONTRACTS speeds up middleman builds
  NO_CONTRACTS: "true"

  ### RELIABILITY ###
Chad Woolley's avatar
Chad Woolley committed
112
  # Reduce flaky builds via https://docs.gitlab.com/ee/ci/yaml/#job-stages-attempts variables
113
114
115
116
  GET_SOURCES_ATTEMPTS: "3"
  ARTIFACT_DOWNLOAD_ATTEMPTS: "3"
  RESTORE_CACHE_ATTEMPTS: "3"
  EXECUTOR_JOB_SECTION_ATTEMPTS: "3"
117
118
  # Performs an error check after each Bash script command is executed, and exits if the previously executed command returned a non-zero exit code
  # https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/2671
119
  FF_ENABLE_BASH_EXIT_CODE_CHECK: "true" 
120

121
### RULES REUSED VIA YAML ANCHORS:
122
# YAML anchors reference: https://docs.gitlab.com/ee/ci/yaml/README.html#anchors
123

124
125
126
127
# master branch in the original repo, NOT a master branch MR for a forked repo.
# NOTE: We must make sure to exclude forked repo MRs by checking `CI_MERGE_REQUEST_SOURCE_PROJECT_ID is null,
# because the pipeline might be in an MR against a fork's 'master' branch, and in this case this rule
# would incorrectly trigger when the "Pipeline for Merged Results" is run for the MR.
128
.if-master-original-repo: &if-master-original-repo
129
  if: '$CI_COMMIT_REF_NAME == "master" && $CI_PROJECT_ID == "7764" && $CI_MERGE_REQUEST_SOURCE_PROJECT_ID == null'
130

131
132
133
# merge request, ONLY for a branch in the original repo
.if-merge-request-original-repo: &if-merge-request-original-repo
  if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_SOURCE_PROJECT_ID == "7764"'
134

135
136
137
# merge request, ONLY for a branch in a forked repo
.if-merge-request-forked-repo: &if-merge-request-forked-repo
  if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_SOURCE_PROJECT_ID != "7764"'
138

139
140
141
142
143
144
# "always" - master branch in the original repo, OR any merge request
.if-master-original-repo-or-merge-request: &if-master-original-repo-or-merge-request
  if: '($CI_COMMIT_REF_NAME == "master" && $CI_PROJECT_ID == "7764") || $CI_MERGE_REQUEST_IID'

# if triggered by the scheduled job to cleanup old deleted files, only on master branch in the original repo
.if-schedule-deploy-cleanup-old-deleted-files-master-original-repo: &if-schedule-deploy-cleanup-old-deleted-files-master-original-repo
145
146
  if: '$CI_PIPELINE_SOURCE == "schedule" && $RUN_SCHEDULED_DEPLOY_CLEANUP_OLD_DELETED_FILES == "true" && $CI_COMMIT_REF_NAME == "master" && $CI_PROJECT_ID == "7764"'

147
### COMMON JOBS REUSED VIA `extends`:
148
# Extends reference: https://docs.gitlab.com/ee/ci/yaml/README.html#extends
149

150
151
.ruby-cache:
  cache:
152
    key: "web_ruby-3.0-buster"
153
    policy: pull
154
155
156
    paths:
      - vendor

157
.bundle-install:
158
  extends: .ruby-cache
159
  before_script:
160
161
    - bundle config set path vendor
    - bundle install --quiet --jobs 4
162

163
164
165
# Artifacts upload: To avoid adding extra run time to jobs, this should only be
# used by jobs which require artifacts.  Currently, this is only
# `deploy-cleanup-old-deleted-files` and `check-handbook-edit-links`
166
167
168
169
170
171
.upload-artifacts:
  artifacts:
    expire_in: 7 days
    paths:
      - public/

Chad Woolley's avatar
Chad Woolley committed
172
173
174
175
176
###################################
#
# PREPARE STAGE
#
###################################
177

Chad Woolley's avatar
Chad Woolley committed
178
# Only push the cache from this job, to save time on all other jobs.
179
180
ruby-push-cache:
  extends: .bundle-install
181
  stage: prepare
182
  rules:
183
    - <<: *if-master-original-repo
184
185
186
  cache:
    policy: pull-push
  script:
187
    - echo "Pushing updated ruby cache..."
188

189
190
191
# Prevents pipeline from being interrupted by subsequent commits.
# Pipelines on the master branch of the original repo are never interruptible.
# MR pipelines (on original repo and forks) may be made non-interruptible by manually running the job.
192
dont-interrupt-me:
193
194
195
  image: alpine:edge
  interruptible: false
  stage: prepare
196
  rules:
197
198
199
    - <<: *if-master-original-repo
      allow_failure: true
    - <<: *if-merge-request-original-repo
200
      when: manual
201
      allow_failure: true
202
203
    - <<: *if-merge-request-forked-repo
      when: manual
204
      allow_failure: true
205
206
207
208
  variables:
    GIT_STRATEGY: none
  script:
    - echo "# This job makes sure this pipeline won't be interrupted on master. It can also be triggered manually to prevent a pipeline from being interrupted. See https://docs.gitlab.com/ee/ci/yaml/#interruptible."
209

210
# This is a manual job for debugging any unexpected behavior encountered while refactoring the CI config
211
expose-ci-rules-variables:
212
  image: alpine:edge
213
  stage: prepare
214
  rules:
215
    - <<: *if-master-original-repo-or-merge-request
216
      when: manual
217
      allow_failure: true
218
219
  variables:
    GIT_STRATEGY: none
220
221
222
  script:
    - echo "CI_COMMIT_REF_NAME = ${CI_COMMIT_REF_NAME}"
    - echo "CI_PROJECT_ID = ${CI_PROJECT_ID}"
223
    - echo "CI_MERGE_REQUEST_IID = ${CI_MERGE_REQUEST_IID}"
224
    - echo "CI_MERGE_REQUEST_TITLE = ${CI_MERGE_REQUEST_TITLE}"
225
    - echo "CI_MERGE_REQUEST_SOURCE_PROJECT_ID = ${CI_MERGE_REQUEST_SOURCE_PROJECT_ID}"
226
    - echo "CI_COMMIT_REF_SLUG = ${CI_COMMIT_REF_SLUG}"
227
228
    - echo "CI_PIPELINE_SOURCE = ${CI_PIPELINE_SOURCE}"
    - echo "RUN_SCHEDULED_DEPLOY_CLEANUP_OLD_DELETED_FILES = ${RUN_SCHEDULED_DEPLOY_CLEANUP_OLD_DELETED_FILES}"
229

Chad Woolley's avatar
Chad Woolley committed
230
231
###################################
#
232
# CONTENT-GENERATION STAGE
Chad Woolley's avatar
Chad Woolley committed
233
234
235
#
###################################

236
237
# SCHEDULED OR TRIGGERED CONTENT-GENERATION JOBS:

238
generate-handbook-changelog:
239
240
241
  extends:
    - .bundle-install
    - .production-environment
242
  timeout: 3h
243
  stage: content-generation
244
245
246
247
248
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $CHANGELOG_MD == "true"'
  needs: []
  script:
    - bundle exec bin/generate_handbook_changelog
249
250
    # Ensure we build and deploy only the generated `sites/handbook/source/handbook/CHANGELOG.html.md
    - cd sites/handbook && bundle exec middleman build --bail --clean --glob='handbook/CHANGELOG.html' && cd -
251
    - scripts/deploy
252
253

generate-handbook-changelog-rss:
254
255
256
  extends:
    - .bundle-install
    - .production-environment
257
  timeout: 3h
258
  stage: content-generation
259
260
261
262
263
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $CHANGELOG_RSS == "true"'
  needs: []
  script:
    - bundle exec bin/generate_handbook_changelog_rss
264
    - scripts/deploy
265

266
267
268
release-post-build:
  extends: .bundle-install
  timeout: 3h
269
  stage: content-generation
270
271
272
273
274
275
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RELEASE_POST_BUILD == "true"'
  needs: []
  script:
    - bundle exec bin/release-post-build

276
277
278
generate-release-post-items:
  extends: .bundle-install
  timeout: 1h
279
  stage: content-generation
280
281
282
283
284
285
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RELEASE_POST_ITEM == "true"'
  needs: []
  script:
    - bundle exec bin/release-post-item --no-local

286
287
288
289
290
# Update https://gitlab.com/gitlab-org/gitlab/-/releases
# and https://gitlab.com/gitlab-org/gitlab-foss/-/releases
# when release posts are published
update-gitlab-project-releases-page:
  extends: .bundle-install
291
  stage: content-generation
292
  rules:
293
    - <<: *if-master-original-repo
294
      changes:
295
        - sites/uncategorized/source/releases/posts/*-released.html.md
296
297
298
299
300
      allow_failure: true
  script:
    - bundle exec rake release:ee:update_project_releases_page
    - bundle exec rake release:foss:update_project_releases_page

301
302
303
304
305
###################################
#
# BUILD-AND-DEPLOY STAGE
#
###################################
306
307
308

# SHARED BUILD AND DEPLOY LOGIC USED VIA `extends` AND YAML ANCHORS:

309
.build-base:
310
  extends: .bundle-install
311
  stage: build-and-deploy
312
313
  needs: []

314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
.review-environment:
  variables:
    DEPLOY_TYPE: review
  environment:
    name: review/$CI_COMMIT_REF_SLUG
    url: https://$CI_COMMIT_REF_SLUG.about.gitlab-review.app
    on_stop: review-stop
    auto_stop_in: 30 days

.production-environment:
  variables:
    DEPLOY_TYPE: production
  environment:
    name: production
    url: https://about.gitlab.com

330
331
332
333
334
335
336
# IMAGES BUILD AND DEPLOY JOBS:

.images-build-script: &images-build-script
  - mkdir -p public/
  - rsync -vlaP --exclude='/team/' --exclude='.gitkeep' source/images/ public/images

images-build-only:
337
  extends: 
338
339
    - .build-base
    - .upload-artifacts
340
  rules:
341
    - <<: *if-schedule-deploy-cleanup-old-deleted-files-master-original-repo
342
343
344
345
346
347
348
349
  script:
    - *images-build-script

images-build-and-review-deploy:
  extends:
    - .build-base
    - .review-environment
  rules:
350
    - <<: *if-merge-request-original-repo
351
352
353
354
355
356
357
358
359
  script:
    - *images-build-script
    - scripts/deploy

images-build-and-prod-deploy:
  extends:
    - .build-base
    - .production-environment
  rules:
360
    - <<: *if-master-original-repo
361
362
363
364
  script:
    - *images-build-script
    - scripts/deploy

Tyler Williams's avatar
Tyler Williams committed
365
# WEBPACK BUILD AND DEPLOY JOBS
366

367
.packaged-assets-webpack-build-script: &packaged-assets-webpack-build-script
368
369
370
  - mkdir -p public/
  - yarn install
  - yarn run build-webpack
371
  - ls tmp 
372
  - mv tmp/dist/stylesheets public/
373
  - mv tmp/dist/javascripts public/
374
  - ls public
375
376
  - ls public/stylesheets
  - ls public/javascripts
377
  
378
packaged-assets-webpack-build-only:
379
  extends: 
380
381
    - .build-base
    - .upload-artifacts
382
383
384
385
  rules:
    - <<: *if-merge-request-forked-repo
    - <<: *if-schedule-deploy-cleanup-old-deleted-files-master-original-repo
  script:
386
    - *packaged-assets-webpack-build-script
387

388
packaged-assets-webpack-build-and-review-deploy:
389
390
391
392
393
394
  extends:
    - .build-base
    - .review-environment
  rules:
    - <<: *if-merge-request-original-repo
  script:
395
    - *packaged-assets-webpack-build-script
396
397
    - scripts/deploy

398
packaged-assets-webpack-build-and-prod-deploy:
399
400
401
402
403
404
  extends:
    - .build-base
    - .production-environment
  rules:
    - <<: *if-master-original-repo
  script:
405
    - *packaged-assets-webpack-build-script
406
407
    - scripts/deploy

408
# ASSETS BUILD AND DEPLOY JOBS:
409
410
411
412

.assets-build-base:
  extends: .build-base
  variables:
413
    DESTINATION_PATH_REGEXES: '^ico/'
414
415
416
417
418
419
    MIDDLEMAN_CONFIG_FILE_NAME: 'config_assets.rb'

.assets-build-script: &assets-build-script
  - bundle exec middleman build --bail

assets-build-only:
420
  extends: 
421
422
    - .assets-build-base
    - .upload-artifacts
423
  rules:
424
425
    - <<: *if-merge-request-forked-repo
    - <<: *if-schedule-deploy-cleanup-old-deleted-files-master-original-repo
426
427
428
429
430
431
432
433
  script:
    - *assets-build-script

assets-build-and-review-deploy:
  extends:
    - .assets-build-base
    - .review-environment
  rules:
434
    - <<: *if-merge-request-original-repo
435
436
437
438
439
440
441
442
443
444
  script:
    - *assets-build-script
    - scripts/review-replace-urls
    - scripts/deploy

assets-build-and-prod-deploy:
  extends:
    - .assets-build-base
    - .production-environment
  rules:
445
    - <<: *if-master-original-repo
446
447
  script:
    - *assets-build-script
448
449
    - scripts/deploy

450
451
452
453
# HANDBOOK BUILD AND DEPLOY JOBS (PARALLEL VIA PartialBuildHandbook EXTENSION):

.handbook-build-base:
  extends: .build-base
454
  parallel: 2
455

456
.handbook-build-script: &handbook-build-script
457
  - cd sites/handbook && bundle exec middleman build --bail && cd -
458
459

handbook-build-only:
460
  extends: 
461
462
    - .handbook-build-base
    - .upload-artifacts
463
  rules:
464
465
    - <<: *if-merge-request-forked-repo
    - <<: *if-schedule-deploy-cleanup-old-deleted-files-master-original-repo
466
467
468
469
470
  script:
    - *handbook-build-script

handbook-build-and-review-deploy:
  extends:
471
    - .handbook-build-base
472
    - .review-environment
473
474
    # NOTE: this `.upload-artifacts` is only needed by `check-handbook-edit-links`
    - .upload-artifacts
475
  rules:
476
   - <<: *if-merge-request-original-repo
477
478
479
480
481
482
483
  script:
    - *handbook-build-script
    - scripts/review-replace-urls
    - scripts/deploy

handbook-build-and-prod-deploy:
  extends:
484
    - .handbook-build-base
485
486
    - .production-environment
  rules:
487
    - <<: *if-master-original-repo
488
  script:
489
490
    - *handbook-build-script
    - scripts/deploy
491

492
# UNCATEGORIZED BUILD AND DEPLOY JOBS (PARALLEL VIA PartialBuildUncategorized EXTENSION):
493

494
.uncategorized-build-base:
495
  extends: .build-base
496
  parallel: 5
497

498
.uncategorized-build-script: &uncategorized-build-script
499
  - if [[ "$CI_NODE_INDEX" == "4" ]]; then bin/crop-team-pictures; fi
500
  - cd sites/uncategorized
501
502
  - find source/images/team -type f ! -name '*-crop.jpg' -delete
  - bundle exec middleman build --bail
503
  - cd -
504

505
uncategorized-build-only:
506
  extends: 
507
    - .uncategorized-build-base
508
    - .upload-artifacts
509
  rules:
510
511
    - <<: *if-merge-request-forked-repo
    - <<: *if-schedule-deploy-cleanup-old-deleted-files-master-original-repo
512
  script:
513
    - *uncategorized-build-script
514

515
uncategorized-build-and-review-deploy:
516
  extends:
517
    - .uncategorized-build-base
518
519
    - .review-environment
  rules:
520
    - <<: *if-merge-request-original-repo
521
  script:
522
    - *uncategorized-build-script
523
524
525
    - scripts/review-replace-urls
    - scripts/deploy

526
uncategorized-build-and-prod-deploy:
527
  extends:
528
    - .uncategorized-build-base
529
530
    - .production-environment
  rules:
531
    - <<: *if-master-original-repo
532
  script:
533
    - *uncategorized-build-script
534
535
    - scripts/deploy

536
# UNCATEGORIZED PROXY RESOURCE BUILD AND DEPLOY JOBS:
537

538
539
# Generators should be cached every 24 hours. We need a dedicated cache key to make
# sure the cache doesn't get blown away by other non-proxy-resource build jobs.
540
.uncategorized-proxy-resource-build-base:
541
  extends: .build-base
542
543
544
545
  # occasionally this job's script can run 11+ minutes if there are a
  # lot of cache misses, putting the total job runtime over the current
  # 15 minute default timeout. So, it is increased to 30m to be safe.
  timeout: 30m
546
547
548
  variables:
    INCLUDE_GENERATORS: 'true'
    CI_BUILD_PROXY_RESOURCE: 'true'
549
  cache:
550
    key: "build_proxy_resource_ruby-3.0-buster"
551
    policy: pull-push
552
553
554
    paths:
      - tmp/cache
      - vendor
555

556
557
.uncategorized-proxy-resource-build-script: &uncategorized-proxy-resource-build-script
  - cd sites/uncategorized && bundle exec middleman build --bail && cd -
558

559
uncategorized-proxy-resource-build-only:
560
  extends: 
561
    - .uncategorized-proxy-resource-build-base
562
    - .upload-artifacts
563
  rules:
564
565
    - <<: *if-merge-request-forked-repo
    - <<: *if-schedule-deploy-cleanup-old-deleted-files-master-original-repo
566
  script:
567
    - *uncategorized-proxy-resource-build-script
568

569
uncategorized-proxy-resource-build-and-review-deploy:
570
  extends:
571
    - .uncategorized-proxy-resource-build-base
572
    - .review-environment
573
  rules:
574
    - <<: *if-merge-request-original-repo
575
  script:
576
    - *uncategorized-proxy-resource-build-script
577
    - scripts/deploy
578

579
uncategorized-proxy-resource-build-and-prod-deploy:
580
  extends:
581
    - .uncategorized-proxy-resource-build-base
582
    - .production-environment
583
  rules:
584
    - <<: *if-master-original-repo
585
  script:
586
    - *uncategorized-proxy-resource-build-script
587
    - scripts/deploy
588

589
# APPLY REDIRECTS JOB:
Lauren Barker's avatar
Lauren Barker committed
590
591
592
.apply-redirects-script: &apply-redirects-script
  - bundle exec bin/apply-exact-match-redirects
  - bundle exec bin/apply-regex-redirects
Lauren Barker's avatar
Lauren Barker committed
593

Lauren Barker's avatar
Lauren Barker committed
594
apply-redirects-review:
Lauren Barker's avatar
Lauren Barker committed
595
  extends: .bundle-install
596
597
  stage: build-and-deploy
  rules:
598
    - <<: *if-merge-request-original-repo
Lauren Barker's avatar
Lauren Barker committed
599
      when: manual
Lauren Barker's avatar
Lauren Barker committed
600
      allow_failure: true
601
  script:
Lauren Barker's avatar
Lauren Barker committed
602
603
604
605
606
607
608
609
610
611
612
    - export FASTLY_SRV_ID=$FASTLY_SRV_ID_REVIEW
    - export FASTLY_SRV_VER=$FASTLY_SRV_VER_REVIEW
    - export FASTLY_DICT_ID=$FASTLY_DICT_ID_REVIEW
    - export FASTLY_EXACT_ERR_SNIPPET_ID=$FASTLY_EXACT_ERR_SNIPPET_ID_REVIEW
    - export FASTLY_EXACT_RECV_SNIPPET_ID=$FASTLY_EXACT_RECV_SNIPPET_ID_REVIEW
    - export FASTLY_LITERAL_ERR_SNIPPET_ID=$FASTLY_LITERAL_ERR_SNIPPET_ID_REVIEW
    - export FASTLY_LITERAL_RECV_SNIPPET_ID=$FASTLY_LITERAL_RECV_SNIPPET_ID_REVIEW
    - export FASTLY_REGEX_ERR_SNIPPET_ID=$FASTLY_REGEX_ERR_SNIPPET_ID_REVIEW
    - export FASTLY_REGEX_RECV_SNIPPET_ID=$FASTLY_REGEX_RECV_SNIPPET_ID_REVIEW
    - export FASTLY_API_KEY=$FASTLY_API_KEY_REVIEW
    - *apply-redirects-script
613

614
615
616
apply-redirects-prod:
  extends: .bundle-install
  stage: build-and-deploy
617
  rules:
618
    - <<: *if-master-original-repo
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
      changes:
        - data/redirects.yml
  environment:
    name: production
  script:
    - export FASTLY_SRV_ID=$FASTLY_SRV_ID_PROD
    - export FASTLY_SRV_VER=$FASTLY_SRV_VER_PROD
    - export FASTLY_DICT_ID=$FASTLY_DICT_ID_PROD
    - export FASTLY_EXACT_ERR_SNIPPET_ID=$FASTLY_EXACT_ERR_SNIPPET_ID_PROD
    - export FASTLY_EXACT_RECV_SNIPPET_ID=$FASTLY_EXACT_RECV_SNIPPET_ID_PROD
    - export FASTLY_LITERAL_ERR_SNIPPET_ID=$FASTLY_LITERAL_ERR_SNIPPET_ID_PROD
    - export FASTLY_LITERAL_RECV_SNIPPET_ID=$FASTLY_LITERAL_RECV_SNIPPET_ID_PROD
    - export FASTLY_REGEX_ERR_SNIPPET_ID=$FASTLY_REGEX_ERR_SNIPPET_ID_PROD
    - export FASTLY_REGEX_RECV_SNIPPET_ID=$FASTLY_REGEX_RECV_SNIPPET_ID_PROD
    - export FASTLY_API_KEY=$FASTLY_API_KEY_PROD
Lauren Barker's avatar
Lauren Barker committed
634
    - *apply-redirects-script
635

Chad Woolley's avatar
Chad Woolley committed
636
637
638
639
640
641
642
###################################
#
# TEST STAGE
#
###################################

# LINTER JOBS:
643

644
root-files-checker:
645
  image: debian:stable-slim
646
  stage: test
647
  rules:
648
    - <<: *if-master-original-repo-or-merge-request
649
  needs: []
650
  script:
651
    - (diff -u FILES <(find . -maxdepth 1 -mindepth 1 | sort) && /bin/echo "No files/directories are added or removed")
652
      || ( /bin/echo "It looks like you've added files to the root directory. If this was intentional, please update FILES to allow this file. If this was not intentional, please remove the file from Git and try again."; exit 1 )
653

654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
# The CSP is sometimes disabled for testing and we must not forget
# to renable it before merging. If you believe this check fails for
# a legitimate change, please ping @gitlab-com/gl-security/appsec
content-security-policy-checker:
  image: debian:stable-slim
  stage: test
  rules:
    - <<: *if-master-original-repo-or-merge-request
      changes:
        - "source/includes/layout/head.html.haml"
  needs: []
  script:
    - |
      grep '^%meta{"http-equiv": "Content-Security-Policy", content: "default-src .* script-src .*"}' source/includes/layout/head.html.haml || (echo "The Content Security Policy appears to be disabled, please re-enable it before merging." ; exit 1)

Lukas 'Eipi' Eipert's avatar
Lukas 'Eipi' Eipert committed
669
lint 0 2:
670
  extends: .bundle-install
671
  stage: test
672
  rules:
673
    - <<: *if-master-original-repo-or-merge-request
674
  needs: []
675
676
677
  script:
    - bundle exec rake lint

Lukas 'Eipi' Eipert's avatar
Lukas 'Eipi' Eipert committed
678
lint 1 2:
679
  stage: test
680
  rules:
681
    - <<: *if-master-original-repo-or-merge-request
682
  needs: []
683
684
685
686
  script:
    - yarn install
    - yarn run eslint

687
lint release-post-items:
688
  extends: .bundle-install
689
  stage: test
690
  rules:
691
    - <<: *if-master-original-repo-or-merge-request
692
693
694
695
      changes:
        - "data/release_posts/unreleased/*"
        - "data/categories.yml"
        - "data/stages.yml"
696
        - "source/images/unreleased/*"
697
698
  needs: []
  script:
699
    - bundle exec bin/validate-release-post-item
James Ramsay's avatar
James Ramsay committed
700

Matija Čupić's avatar
Matija Čupić committed
701
rubocop:
702
  extends: .bundle-install
703
  stage: test
704
  rules:
705
    - <<: *if-master-original-repo-or-merge-request
706
707
708
709
      changes:
        - "*.rb"
        - "**/*.rb"
        - ".rubocop.yml"
710
        - "Dangerfile"
Lauren Barker's avatar
Lauren Barker committed
711
712
        - "*.rake"
        - "**/*.rake"
713
714
715
  needs: []
  script:
    - bundle exec rubocop
Matija Čupić's avatar
Matija Čupić committed
716

717
718
include:
  - template: Code-Quality.gitlab-ci.yml
719
720
  - template: Security/Secret-Detection.gitlab-ci.yml
  - template: Dependency-Scanning.gitlab-ci.yml
721

Philippe Lafoucrière's avatar
Philippe Lafoucrière committed
722
code_quality:
723
724
  tags: [gitlab-org-docker]
  stage: test
725
  rules:
726
    - <<: *if-master-original-repo-or-merge-request
727
      when: manual
728
  needs: []
Olivier Gonzalez's avatar
Olivier Gonzalez committed
729
730
  artifacts:
    paths:
731
732
      - coffeelint.json
      - gl-code-quality-report.json
Olivier Gonzalez's avatar
Olivier Gonzalez committed
733

734
735
# Override Dependency Scanning Jobs until https://gitlab.com/gitlab-org/gitlab/-/issues/217668 is closed
gemnasium-dependency_scanning:
736
  needs: []
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
  rules:
    - if: $DEPENDENCY_SCANNING_DISABLED
      when: never
    - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/
      when: never
    - <<: *if-master-original-repo-or-merge-request
      exists:
        - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}'
        - '{composer.lock,*/composer.lock,*/*/composer.lock}'
        - '{gems.locked,*/gems.locked,*/*/gems.locked}'
        - '{go.sum,*/go.sum,*/*/go.sum}'
        - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}'
        - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}'
        - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}'
        - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}'
        - '{conan.lock,*/conan.lock,*/*/conan.lock}'

bundler-audit-dependency_scanning:
755
  needs: []
756
757
758
759
760
761
762
763
764
765
  rules:
    - if: $DEPENDENCY_SCANNING_DISABLED
      when: never
    - if: $DS_EXCLUDED_ANALYZERS =~ /bundler-audit/
      when: never
    - <<: *if-master-original-repo-or-merge-request
      exists:
        - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}'

retire-js-dependency_scanning:
766
  needs: []
767
768
769
770
771
772
773
774
775
776
777
778
779
780
  rules:
    - if: $DEPENDENCY_SCANNING_DISABLED
      when: never
    - if: $DS_EXCLUDED_ANALYZERS =~ /retire.js/
      when: never
    - <<: *if-master-original-repo-or-merge-request
      exists:
        - '{package.json,*/package.json,*/*/package.json}'

# == END Dependency Scanning jobs

# Override Secret Detection job until https://gitlab.com/gitlab-org/gitlab/-/issues/217668 is closed

secret_detection:
781
  needs: []
782
  rules:
783
    - <<: *if-master-original-repo-or-merge-request
784

785
check-handbook-edit-links:
786
  extends: .bundle-install
787
# NOTE: Because of the time it would takes to download all artifacts, and because the non-handbook portions of the site may change in the future,
788
#       we are only checking links in the handbook instead of all build jobs, even though there are edit links on non-handbook pages
789
#       which are still shared for now.  So, this only provides coverage for major breakages or regressions in the edit
790
791
#       links logic for pages in the handbook, not the rest of the site.
#
792
793
#       Note that the source for the `lib/lint/check_handbook_edit_links.rb` could still be used for checking the rest of the site
#       (if the proper skip regexes were added), but this job is only running it against the files generated by the handbook build job.
794
  needs:
795
    - handbook-build-and-review-deploy
796
  stage: test
797
  rules:
798
    - <<: *if-merge-request-original-repo
799
      changes:
800
801
802
        # To avoid this job from running too often and blocking builds (since its dependency on the handbook job is currently one of the longest-running jobs),
        # the list of changes to trigger it are restricted to files currently involved in rendering the edit links.
        - data/monorepo.yml
803
        - helpers/custom_helpers.rb
804
805
806
        - '**/edit_page.html.haml'
        - '**/footer.html.haml'
        - '**/handbook-page-toc.haml'
807
        - '**/check_handbook_edit_links.rb'
808
  script:
809
    - scripts/check-handbook-edit-links.rb
810

Chad Woolley's avatar
Chad Woolley committed
811
812
813
814
815
816
# TEST SUITE JOBS:

rspec-unit:
  extends: .bundle-install
  stage: test
  rules:
817
    - <<: *if-master-original-repo-or-merge-request
818
819
820
821
      changes:
        - "spec/**/*"
        - "**/*.{js,json,rb,yml}"
        - ".rspec"
Chad Woolley's avatar
Chad Woolley committed
822
823
824
825
826
827
  needs: []
  script:
    - bundle exec rspec --tag ~@feature

###################################
#
828
# CLEANUP STAGE
Chad Woolley's avatar
Chad Woolley committed
829
830
831
#
###################################

832
# REVIEW STOP JOB:
833

834
review-stop:
835
  extends: .review-environment
836
  stage: cleanup
837
  rules:
838
    - <<: *if-merge-request-original-repo
839
840
      when: manual
      allow_failure: true
841
842
  variables:
    DEPLOY_DELETE_APP: 'true'
843
844
  environment:
    action: stop
845
  script:
846
    - scripts/deploy
847

848
# PROD DEPLOY JOBS:
849

850
851
# This uses artifacts and is part of a scheduled pipeline, so it needs to depend on the "build-only" versions of jobs
deploy-cleanup-old-deleted-files:
852
  extends:
853
854
855
    - .production-environment
  stage: cleanup
  rules:
856
    - <<: *if-schedule-deploy-cleanup-old-deleted-files-master-original-repo
857
  needs:
858
859
    - uncategorized-build-only
    - uncategorized-proxy-resource-build-only
860
    - handbook-build-only
861
    - images-build-only
862
    - packaged-assets-webpack-build-only
863
    - assets-build-only
864
865
  variables:
    DEPLOY_CLEANUP_OLD_DELETED_FILES: 'true'
866
867
  script:
    - scripts/deploy
868
869
870
871
872
873
874
875
876

danger-review:
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
  rules:
    - if: '$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID'
  stage: test
  needs: []
  script:
    - danger --fail-on-errors=true --verbose
877
878
879
880
  
dependency_scanning:
  variables:
    DS_EXCLUDED_ANALYZERS: "gemnasium-maven, gemnasium-python"