.csscomb.json

-{
-  "exclude": [
-    "app/assets/stylesheets/framework/tw_bootstrap_variables.scss",
-    "app/assets/stylesheets/framework/fonts.scss"
-  ],
-  "always-semicolon": true,
-  "color-case": "lower",
-  "block-indent": "  ",
-  "color-shorthand": false,
-  "element-case": "lower",
-  "space-before-colon": "",
-  "space-after-colon": " ",
-  "space-before-combinator": " ",
-  "space-after-combinator": " ",
-  "space-between-declarations": "\n",
-  "space-before-opening-brace": " ",
-  "space-after-opening-brace": "\n",
-  "space-before-closing-brace": "\n",
-  "unitless-zero": true
-}

.gitignore

@@ -70,6 +70,7 @@ eslint-report.html
 /tags
 /vendor/bundle/*
 /vendor/gitaly-ruby
+/vendor/package_metadata_db/
 /builds*
 /.gitlab_workhorse_secret
 /.gitlab_pages_secret

.gitlab-ci.yml

 stages:
   - sync
+  - preflight
   - prepare
   - build-images
   - fixtures
@@ -11,6 +12,7 @@ stages:
   - post-qa
   - pages
   - notify
+  - release-environments
 
 # always use `gitlab-org` runners, however
 # in cases where jobs require Docker-in-Docker, the job
@@ -31,12 +33,15 @@ default:
 
 .ruby2-variables: &ruby2-variables
   RUBY_VERSION: "2.7"
+  OMNIBUS_GITLAB_RUBY2_BUILD: "true"
+  OMNIBUS_GITLAB_CACHE_EDITION: "GITLAB_RUBY2"
 
-.default-branch-incident-variables: &default-branch-incident-variables
+.default-branch-pipeline-failure-variables: &default-branch-pipeline-failure-variables
   CREATE_INCIDENT_FOR_PIPELINE_FAILURE: "true"
   NOTIFY_PIPELINE_FAILURE_CHANNEL: "master-broken"
   BROKEN_BRANCH_INCIDENTS_PROJECT: "gitlab-org/quality/engineering-productivity/master-broken-incidents"
   BROKEN_BRANCH_INCIDENTS_PROJECT_TOKEN: "${BROKEN_MASTER_INCIDENTS_PROJECT_TOKEN}"
+  CREATE_ISSUES_FOR_FAILING_TESTS: "true"
 
 workflow:
   name: '$PIPELINE_NAME'
@@ -72,7 +77,7 @@ workflow:
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule"'
       variables:
         <<: *ruby3-variables
-        <<: *default-branch-incident-variables
+        <<: *default-branch-pipeline-failure-variables
         CRYSTALBALL: "true"
         PIPELINE_NAME: 'Scheduled Ruby 3 $CI_COMMIT_BRANCH branch pipeline'
     # Run pipelines for ruby2 branch
@@ -86,14 +91,14 @@ workflow:
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $GITLAB_USER_LOGIN =~ /project_\d+_bot\d*/'
       variables:
         <<: *ruby3-variables
-        <<: *default-branch-incident-variables
+        <<: *default-branch-pipeline-failure-variables
         GITLAB_DEPENDENCY_PROXY_ADDRESS: ""
-        PIPELINE_NAME: 'Ruby 3 $CI_COMMIT_BRANCH branch pipeline (trigerred by a project token)'
+        PIPELINE_NAME: 'Ruby 3 $CI_COMMIT_BRANCH branch pipeline (triggered by a project token)'
     # For `$CI_DEFAULT_BRANCH` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       variables:
         <<: *ruby3-variables
-        <<: *default-branch-incident-variables
+        <<: *default-branch-pipeline-failure-variables
         PIPELINE_NAME: 'Ruby 3 $CI_COMMIT_BRANCH branch pipeline'
     # For tags, create a pipeline.
     - if: '$CI_COMMIT_TAG'
@@ -123,7 +128,7 @@ workflow:
 
 variables:
   PG_VERSION: "12"
-  DEFAULT_CI_IMAGE: "${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}.patched-golang-${GO_VERSION}-node-16.14-postgresql-${PG_VERSION}:rubygems-3.2-git-2.36-lfs-2.9-chrome-${CHROME_VERSION}-yarn-1.22-graphicsmagick-1.3.36"
+  DEFAULT_CI_IMAGE: "${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}.patched-golang-${GO_VERSION}-rust-${RUST_VERSION}-node-16.14-postgresql-${PG_VERSION}:rubygems-3.2-git-2.36-lfs-2.9-chrome-${CHROME_VERSION}-yarn-1.22-graphicsmagick-1.3.36"
   # We set $GITLAB_DEPENDENCY_PROXY to another variable (since it's set at the group level and has higher precedence than .gitlab-ci.yml)
   # so that we can override $GITLAB_DEPENDENCY_PROXY_ADDRESS in workflow rules.
   GITLAB_DEPENDENCY_PROXY_ADDRESS: "${GITLAB_DEPENDENCY_PROXY}"
@@ -140,10 +145,11 @@ variables:
   GIT_SUBMODULE_STRATEGY: "none"
   GET_SOURCES_ATTEMPTS: "3"
   DEBIAN_VERSION: "bullseye"
-  CHROME_VERSION: "106"
-  DOCKER_VERSION: "20.10.14"
+  CHROME_VERSION: "109"
+  DOCKER_VERSION: "23.0.1"
   RUBY_VERSION: "2.7"
   GO_VERSION: "1.18"
+  RUST_VERSION: "1.65"
 
   FLAKY_RSPEC_SUITE_REPORT_PATH: rspec/flaky/report-suite.json
   FRONTEND_FIXTURES_MAPPING_PATH: crystalball/frontend_fixtures_mapping.json
@@ -152,7 +158,7 @@ variables:
   JUNIT_RETRY_FILE: rspec/junit_rspec-retry.xml
   KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/report-master.json
   RSPEC_CHANGED_FILES_PATH: rspec/changed_files.txt
-  RSPEC_FOSS_IMPACT_PIPELINE_YML: rspec-foss-impact-pipeline.yml
+  RSPEC_FOSS_IMPACT_PIPELINE_TEMPLATE_YML: .gitlab/ci/rails/rspec-foss-impact.gitlab-ci.yml.erb
   RSPEC_LAST_RUN_RESULTS_FILE: rspec/rspec_last_run_results.txt
   RSPEC_MATCHING_JS_FILES_PATH: rspec/js_matching_files.txt
   RSPEC_MATCHING_TESTS_PATH: rspec/matching_tests.txt
@@ -169,7 +175,7 @@ variables:
   CACHE_CLASSES: "true"
   CHECK_PRECOMPILED_ASSETS: "true"
   FF_USE_FASTZIP: "true"
-  SKIP_FLAKY_TESTS_AUTOMATICALLY: "true"
+  SKIP_FLAKY_TESTS_AUTOMATICALLY: "false"
   RETRY_FAILED_TESTS_IN_NEW_PROCESS: "true"
   # Run with decomposed databases by default
   DECOMPOSED_DB: "true"
@@ -192,6 +198,9 @@ variables:
   # See https://gitlab.com/gitlab-com/gl-security/engineering-and-research/inventory/-/issues/827#note_1203181407
   DISABLE_OPENCOLLECTIVE: "true"
 
+  # This is set at the gitlab-org level, but we set it here for forks
+  DANGER_DO_NOT_POST_INVALID_DANGERFILE_ERROR: "1"
+
 include:
   - local: .gitlab/ci/*.gitlab-ci.yml
   - remote: 'https://gitlab.com/gitlab-org/frontend/untamper-my-lockfile/-/raw/main/templates/merge_request_pipelines.yml'

.gitlab/ci/as-if-jh.gitlab-ci.yml

@@ -38,12 +38,12 @@ prepare-as-if-jh-branch:
   needs:
     - add-jh-files
   script:
-    - git checkout -b "${AS_IF_JH_BRANCH}"
-    - git add ${JH_FILES_TO_COMMIT}
-    - git commit -m 'Add JH files'  # TODO: Mark which SHA we add
     # Fetch for the history of the branch so it does not cause the following error:
     #   ! [remote rejected] ref -> ref (shallow update not allowed)
     - git fetch --unshallow --filter=tree:0 origin "${CI_COMMIT_SHA}"
+    - git checkout -b "${AS_IF_JH_BRANCH}"
+    - git add ${JH_FILES_TO_COMMIT}
+    - git commit -m 'Add JH files'  # TODO: Mark which SHA we add
     - git push -f "${SANDBOX_REPOSITORY}" "${AS_IF_JH_BRANCH}"
 
 start-as-if-jh:

.gitlab/ci/build-images.gitlab-ci.yml

@@ -29,6 +29,33 @@ build-qa-image as-if-foss:
     - .as-if-foss
     - .build-images:rules:build-qa-image-as-if-foss
 
+# Prepares an image with GDK configured based on code in master. This saves some time in MRs because some installation
+# and complilation will have already been performed.
+build-qa-on-gdk-master-image:
+  extends:
+    - .base-image-build-buildx
+    - .build-images:rules:build-qa-on-gdk-master-image
+  tags:
+    - e2e
+  stage: build-images
+  needs: []
+  variables:
+    QA_GDK_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-qa-gdk"
+  before_script:
+    - !reference [.use-buildx, before_script]
+    - sysctl -n -w fs.inotify.max_user_watches=524288
+  script:
+    - |
+      docker buildx build \
+        --cache-to=type=inline \
+        --cache-from ${QA_GDK_IMAGE}:master \
+        --platform=${ARCH:-amd64} \
+        --add-host gdk.test:127.0.0.1 \
+        --tag ${QA_GDK_IMAGE}:master \
+        --file="qa/gdk/Dockerfile" \
+        --push \
+        ${CI_PROJECT_DIR}
+
 build-assets-image:
   extends:
     - .base-image-build

.gitlab/ci/database.gitlab-ci.yml

 include:
   - local: .gitlab/ci/rails/shared.gitlab-ci.yml
 
+db:rollback single-db-ci-connection:
+  extends:
+    - db:rollback
+    - .single-db-ci-connection
+    - .rails:rules:single-db-ci-connection
+
+db:migrate:reset single-db-ci-connection:
+  extends:
+    - db:migrate:reset
+    - .single-db-ci-connection
+    - .rails:rules:single-db-ci-connection
+
+db:check-schema-single-db-ci-connection:
+  extends:
+    - db:check-schema
+    - .single-db-ci-connection
+    - .rails:rules:single-db-ci-connection
+
+db:post_deployment_migrations_validator-single-db-ci-connection:
+  extends:
+    - db:post_deployment_migrations_validator
+    - .single-db-ci-connection
+    - .rails:rules:db:check-migrations-single-db-ci-connection
+
+db:backup_and_restore single-db-ci-connection:
+  extends:
+    - db:backup_and_restore
+    - .single-db-ci-connection
+    - .rails:rules:db-backup
+
 db:rollback:
-  extends: .db-job-base
+  extends:
+    - .db-job-base
+    - .rails:rules:db-rollback
   script:
-    - scripts/db_tasks db:migrate VERSION=20220502173045  # 14.10 (last 14.x version)
-    - scripts/db_tasks db:migrate
+    - bundle exec rake db:migrate VERSION=20220502173045  # 14.10 (last 14.x version)
+    - bundle exec rake db:migrate
 
 db:rollback single-db:
   extends:
@@ -29,8 +61,7 @@ db:check-schema:
     - .db-job-base
     - .rails:rules:ee-mr-and-default-branch-only
   script:
-    - run_timed_command "bundle exec rake db:drop db:create"
-    - run_timed_command "scripts/db_tasks db:migrate"
+    - run_timed_command "bundle exec rake db:drop db:create db:migrate"
 
 db:check-schema-single-db:
   extends:
@@ -95,7 +126,7 @@ db:backup_and_restore:
     GITLAB_ASSUME_YES: "1"
   script:
     - . scripts/prepare_build.sh
-    - bundle exec rake db:drop db:create db:structure:load db:seed_fu
+    - bundle exec rake db:drop db:create db:schema:load db:seed_fu
     - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,terraform_state,registry,packages}
     - bundle exec rake gitlab:backup:create
     - date

.gitlab/ci/docs.gitlab-ci.yml

@@ -42,7 +42,7 @@ review-docs-cleanup:
 docs-lint links:
   extends:
     - .docs:rules:docs-lint
-  image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-docs/lint-html:alpine-3.16-ruby-3.0.5-869cfc5d
+  image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-docs/lint-html:alpine-3.16-ruby-3.0.5-2a7e4d1c
   stage: lint
   needs: []
   script:
@@ -58,18 +58,31 @@ docs-lint links:
 
 .docs-markdown-lint-image:
   # When updating the image version here, update it in /scripts/lint-doc.sh too.
-  image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-docs/lint-markdown:alpine-3.16-vale-2.22.0-markdownlint-0.32.2-markdownlint2-0.6.0
+  image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-docs/lint-markdown:alpine-3.16-vale-2.23.0-markdownlint-0.33.0-markdownlint2-0.6.0
 
 docs-lint markdown:
   extends:
     - .default-retry
     - .docs:rules:docs-lint
     - .docs-markdown-lint-image
+    - .yarn-cache
   stage: lint
   needs: []
   script:
+    - source ./scripts/utils.sh
+    - yarn_install_script
     - scripts/lint-doc.sh
 
+docs-lint blueprint:
+  extends:
+    - .default-retry
+    - .docs:rules:docs-blueprints-lint
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}-slim
+  stage: lint
+  needs: []
+  script:
+    - scripts/lint-docs-blueprints.rb
+
 docs code_quality:
   extends:
     - .reports:rules:code_quality

.gitlab/ci/frontend.gitlab-ci.yml

@@ -141,6 +141,21 @@ rspec-all frontend_fixture as-if-foss:
     - !reference [.frontend-fixtures-base, needs]
     - "compile-test-assets as-if-foss"
 
+# Uploads EE fixtures in the EE project.
+# Uploads FOSS fixtures in the FOSS project.
+upload-frontend-fixtures:
+  extends:
+    - .frontend-fixtures-base
+    - .frontend:rules:upload-frontend-fixtures
+  stage: fixtures
+  needs: ["rspec-all frontend_fixture"]
+  script:
+    - source scripts/gitlab_component_helpers.sh
+    - 'fixtures_archive_doesnt_exist || { echoinfo "INFO: Exiting early as package exists."; exit 0; }'
+    - run_timed_command "create_fixtures_package"
+    - run_timed_command "upload_fixtures_package"
+  artifacts: {}
+
 graphql-schema-dump:
   variables:
     SETUP_DB: "false"

.gitlab/ci/global.gitlab-ci.yml

@@ -229,14 +229,6 @@
     - *node-modules-cache  # We don't push this cache as it's already rebuilt by `update-assets-compile-*-cache`
     - *storybook-node-modules-cache-push
 
-.use-pg11:
-  services:
-    - name: postgres:11.6
-      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
-    - name: redis:5.0-alpine
-  variables:
-    POSTGRES_HOST_AUTH_METHOD: trust
-    PG_VERSION: "11"
 
 .use-pg12:
   services:
@@ -256,21 +248,6 @@
     POSTGRES_HOST_AUTH_METHOD: trust
     PG_VERSION: "13"
 
-.use-pg11-es7-ee:
-  services:
-    - name: postgres:11.6
-      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
-    - name: redis:5.0-alpine
-    - name: elasticsearch:7.17.6
-      command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"]
-    - name: registry.gitlab.com/gitlab-org/gitlab-build-images:zoekt-ci-image-1.0
-      alias: zoekt-ci-image
-  variables:
-    POSTGRES_HOST_AUTH_METHOD: trust
-    PG_VERSION: "11"
-    ZOEKT_INDEX_BASE_URL: http://zoekt-ci-image:6060
-    ZOEKT_SEARCH_BASE_URL: http://zoekt-ci-image:6070
-
 .use-pg12-es7-ee:
   services:
     - name: postgres:12
@@ -278,7 +255,7 @@
     - name: redis:6.0-alpine
     - name: elasticsearch:7.17.6
       command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"]
-    - name: registry.gitlab.com/gitlab-org/gitlab-build-images:zoekt-ci-image-1.0
+    - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
       alias: zoekt-ci-image
   variables:
     POSTGRES_HOST_AUTH_METHOD: trust
@@ -293,7 +270,7 @@
     - name: redis:6.2-alpine
     - name: elasticsearch:7.17.6
       command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"]
-    - name: registry.gitlab.com/gitlab-org/gitlab-build-images:zoekt-ci-image-1.0
+    - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
       alias: zoekt-ci-image
   variables:
     POSTGRES_HOST_AUTH_METHOD: trust
@@ -307,7 +284,7 @@
       command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
     - name: redis:6.0-alpine
     - name: elasticsearch:8.5.3
-    - name: registry.gitlab.com/gitlab-org/gitlab-build-images:zoekt-ci-image-1.0
+    - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
       alias: zoekt-ci-image
   variables:
     POSTGRES_HOST_AUTH_METHOD: trust
@@ -325,7 +302,7 @@
     - name: opensearchproject/opensearch:1.3.5
       alias: elasticsearch
       command: ["bin/opensearch", "-E", "discovery.type=single-node", "-E", "plugins.security.disabled=true"]
-    - name: registry.gitlab.com/gitlab-org/gitlab-build-images:zoekt-ci-image-1.0
+    - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
       alias: zoekt-ci-image
   variables:
     POSTGRES_HOST_AUTH_METHOD: trust
@@ -341,7 +318,7 @@
     - name: opensearchproject/opensearch:2.2.1
       alias: elasticsearch
       command: ["bin/opensearch", "-E", "discovery.type=single-node", "-E", "plugins.security.disabled=true"]
-    - name: registry.gitlab.com/gitlab-org/gitlab-build-images:zoekt-ci-image-1.0
+    - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
       alias: zoekt-ci-image
   variables:
     POSTGRES_HOST_AUTH_METHOD: trust
@@ -376,7 +353,7 @@
 
 .use-buildx:
   extends: .use-docker-in-docker
-  image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-slim:docker-${DOCKER_VERSION}-buildx-0.8
+  image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-slim:docker-${DOCKER_VERSION}
   variables:
     QEMU_IMAGE: tonistiigi/binfmt:qemu-v7.0.0
   before_script:

.gitlab/ci/notify.gitlab-ci.yml

@@ -43,7 +43,7 @@ notify-pipeline-failure:
     - .notify:rules:notify-pipeline-failure
   image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}
   variables:
-    INCIDENT_PROJECT: "#{BROKEN_BRANCH_INCIDENTS_PROJECT}"
+    INCIDENT_PROJECT: "${BROKEN_BRANCH_INCIDENTS_PROJECT}"
     BROKEN_BRANCH_PROJECT_TOKEN: "${BROKEN_BRANCH_INCIDENTS_PROJECT_TOKEN}"
     INCIDENT_JSON: "${CI_PROJECT_DIR}/incident.json"
     SLACK_CHANNEL: "${NOTIFY_PIPELINE_FAILURE_CHANNEL}"
@@ -55,13 +55,14 @@ notify-pipeline-failure:
   script:
     - |
       if [[ "${CREATE_INCIDENT_FOR_PIPELINE_FAILURE}" == "true" ]]; then
-        scripts/create-pipeline-failure-incident.rb -p ${INCIDENT_PROJECT} -f ${INCIDENT_JSON} -t ${BROKEN_BRANCH_PROJECT_TOKEN};
+        scripts/create-pipeline-failure-incident.rb -p ${INCIDENT_PROJECT} -f ${INCIDENT_JSON} -t ${BROKEN_BRANCH_PROJECT_TOKEN} ||
+          scripts/slack ${SLACK_CHANNEL} "☠️  Broken pipeline incident creation failed! ☠️  See ${CI_JOB_URL}" ci_failing "Failed pipeline reporter";
         echosuccess "Created incident $(jq '.web_url' ${INCIDENT_JSON})";
       fi
     - |
       scripts/generate-failed-pipeline-slack-message.rb -i ${INCIDENT_JSON} -f ${FAILED_PIPELINE_SLACK_MESSAGE_FILE};
       curl -X POST -H 'Content-Type: application/json' --data @${FAILED_PIPELINE_SLACK_MESSAGE_FILE} "$CI_SLACK_WEBHOOK_URL" ||
-        scripts/slack ${SLACK_CHANNEL} "☠️  Broken pipeline notification failed! ☠️  See ${CI_JOB_URL}" ci_failing "Failed pipeline reporter"
+        scripts/slack ${SLACK_CHANNEL} "☠️  Broken pipeline notification failed! ☠️  See ${CI_JOB_URL}" ci_failing "Failed pipeline reporter";
 
   artifacts:
     paths:
@@ -69,3 +70,28 @@ notify-pipeline-failure:
       - ${FAILED_PIPELINE_SLACK_MESSAGE_FILE}
     when: always
     expire_in: 2 days
+
+create-issues-for-failing-tests:
+  extends:
+    - .notify-defaults
+    - .notify:rules:create-issues-for-failing-tests
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}
+  variables:
+    FAILED_TESTS_DIR: "${CI_PROJECT_DIR}/tmp/failed_tests"
+    FAILING_ISSUES_PROJECT: "gitlab-org/quality/engineering-productivity/flaky-tests-playground"
+    FAILING_ISSUE_JSON_DIR: "${CI_PROJECT_DIR}/tmp/issues"
+  before_script:
+    - source ./scripts/utils.sh
+    - source ./scripts/rspec_helpers.sh
+    - install_gitlab_gem
+  script:
+    - mkdir -p "${FAILING_ISSUE_JSON_DIR}"
+    - retrieve_failed_tests "${FAILED_TESTS_DIR}" "json" "latest"
+    - scripts/pipeline/create_test_failure_issues.rb --project "${FAILING_ISSUES_PROJECT}" --tests-report-file "${FAILED_TESTS_DIR}/rspec_failed_tests.json" --issues-json-folder "${FAILING_ISSUE_JSON_DIR}" --api-token "${FAILING_ISSUES_PROJECT_TOKEN}"
+    - scripts/pipeline/create_test_failure_issues.rb --project "${FAILING_ISSUES_PROJECT}" --tests-report-file "${FAILED_TESTS_DIR}/rspec_ee_failed_tests.json" --issues-json-folder "${FAILING_ISSUE_JSON_DIR}" --api-token "${FAILING_ISSUES_PROJECT_TOKEN}"
+  artifacts:
+    paths:
+      - ${FAILED_TESTS_DIR}/
+      - ${FAILING_ISSUE_JSON_DIR}/
+    when: always
+    expire_in: 2 days

.gitlab/ci/package-and-test/main.gitlab-ci.yml

@@ -8,7 +8,7 @@ include:
   - local: .gitlab/ci/package-and-test/rules.gitlab-ci.yml
   - local: .gitlab/ci/package-and-test/variables.gitlab-ci.yml
   - project: gitlab-org/quality/pipeline-common
-    ref: 1.14.1
+    ref: 2.2.0
     file:
       - /ci/base.gitlab-ci.yml
       - /ci/allure-report.yml
@@ -58,15 +58,16 @@ stages:
       optional: true
     - job: download-knapsack-report
       artifacts: true
+      optional: true
+    - job: check-release-set
   variables:
     QA_GENERATE_ALLURE_REPORT: "true"
     QA_CAN_TEST_PRAEFECT: "false"
     QA_INTERCEPT_REQUESTS: "true"
-    QA_RUN_TYPE: e2e-package-and-test
-    TEST_LICENSE_MODE: $QA_TEST_LICENSE_MODE
     EE_LICENSE: $QA_EE_LICENSE
     GITHUB_ACCESS_TOKEN: $QA_GITHUB_ACCESS_TOKEN
     GITLAB_QA_ADMIN_ACCESS_TOKEN: $QA_ADMIN_ACCESS_TOKEN
+    GITLAB_LICENSE_MODE: test
 
 # ==========================================
 # Prepare stage
@@ -121,6 +122,7 @@ trigger-omnibus-env:
       echo "OMNIBUS_GITLAB_CACHE_UPDATE=${OMNIBUS_GITLAB_CACHE_UPDATE:-false}" >> $BUILD_ENV
       for version_file in *_VERSION; do echo "$version_file=$(cat $version_file)" >> $BUILD_ENV; done
       echo "OMNIBUS_GITLAB_RUBY3_BUILD=${OMNIBUS_GITLAB_RUBY3_BUILD:-false}" >> $BUILD_ENV
+      echo "OMNIBUS_GITLAB_RUBY2_BUILD=${OMNIBUS_GITLAB_RUBY2_BUILD:-false}" >> $BUILD_ENV
       echo "OMNIBUS_GITLAB_CACHE_EDITION=${OMNIBUS_GITLAB_CACHE_EDITION:-GITLAB}" >> $BUILD_ENV
       echo "GITLAB_ASSETS_TAG=$(assets_image_tag)" >> $BUILD_ENV
       echo "Built environment file for omnibus build:"
@@ -151,8 +153,8 @@ trigger-omnibus:
     SECURITY_SOURCES: $SECURITY_SOURCES
     CACHE_UPDATE: $OMNIBUS_GITLAB_CACHE_UPDATE
     RUBY3_BUILD: $OMNIBUS_GITLAB_RUBY3_BUILD
+    RUBY2_BUILD: $OMNIBUS_GITLAB_RUBY2_BUILD
     CACHE_EDITION: $OMNIBUS_GITLAB_CACHE_EDITION
-    SKIP_QA_DOCKER: "true"
     SKIP_QA_TEST: "true"
     ee: "true"
   trigger:
@@ -162,7 +164,7 @@ trigger-omnibus:
 download-knapsack-report:
   extends:
     - .gitlab-qa-image
-    - .rules:prepare
+    - .rules:download-knapsack
   stage: .pre
   variables:
     KNAPSACK_DIR: ${CI_PROJECT_DIR}/qa/knapsack
@@ -212,18 +214,36 @@ _ee:quarantine:
   variables:
     QA_RSPEC_TAGS: --tag quarantine
 
+# Temporary test job to support the effort of migrating to Super Sidebar
+# https://gitlab.com/groups/gitlab-org/-/epics/9044
+_ee:super-sidebar-nav:
+  extends:
+    - .qa
+    - .parallel
+  variables:
+    QA_SCENARIO: Test::Instance::Image
+    QA_KNAPSACK_REPORT_NAME: ee-instance
+    QA_TESTS: ""
+    QA_SUPER_SIDEBAR_ENABLED: "true"
+    QA_ALLURE_RESULTS_DIRECTORY: tmp/allure-results-super-sidebar
+    GITLAB_QA_OPTS: --set-feature-flags super_sidebar_nav=enabled
+  allow_failure: true
+  rules:
+    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
+    - !reference [.rules:test:manual, rules]
+
 # ------------------------------------------
 # FF changes
 # ------------------------------------------
 
 # Run specs with feature flags set to the opposite of the default state
-ee:instance-parallel-ff-inverse:
+ee:instance-ff-inverse:
   extends:
     - .qa
     - .parallel
   variables:
     QA_SCENARIO: Test::Instance::Image
-    QA_KNAPSACK_REPORT_NAME: ee-instance-parallel
+    QA_KNAPSACK_REPORT_NAME: ee-instance
     GITLAB_QA_OPTS: --set-feature-flags $QA_FEATURE_FLAGS
   rules:
     - !reference [.rules:test:feature-flags-set, rules]
@@ -231,140 +251,141 @@ ee:instance-parallel-ff-inverse:
 # ------------------------------------------
 # Jobs with parallel variant
 # ------------------------------------------
-ee:instance:
+ee:instance-selective:
   extends: .qa
   variables:
     QA_SCENARIO: Test::Instance::Image
   rules:
-    - !reference [.rules:test:qa-non-parallel, rules]
+    - !reference [.rules:test:qa-selective, rules]
     - if: $QA_SUITES =~ /Test::Instance::All/
-ee:instance-parallel:
+ee:instance:
   extends:
     - .parallel
-    - ee:instance
+    - ee:instance-selective
   rules:
-    - !reference [.rules:test:feature-flags-set, rules]  # always run instance-parallel to validate ff change
+    - !reference [.rules:test:feature-flags-set, rules]  # always run ee:instance to validate ff change
     - !reference [.rules:test:qa-parallel, rules]
     - if: $QA_SUITES =~ /Test::Instance::All/
 
-ee:praefect:
+ee:praefect-selective:
   extends: .qa
   variables:
     QA_SCENARIO: Test::Integration::Praefect
     QA_CAN_TEST_PRAEFECT: "true"
   rules:
-    - !reference [.rules:test:qa-non-parallel, rules]
+    - !reference [.rules:test:qa-selective, rules]
     - if: $QA_SUITES =~ /Test::Instance::All/
-ee:praefect-parallel:
+ee:praefect:
   extends:
     - .parallel
-    - ee:praefect
+    - ee:praefect-selective
   rules:
     - !reference [.rules:test:qa-parallel, rules]
     - if: $QA_SUITES =~ /Test::Instance::All/
 
-ee:relative-url:
+ee:relative-url-selective:
   extends: .qa
   variables:
     QA_SCENARIO: Test::Instance::RelativeUrl
   rules:
-    - !reference [.rules:test:qa-non-parallel, rules]
+    - !reference [.rules:test:qa-selective, rules]
     - if: $QA_SUITES =~ /Test::Instance::All/
-ee:relative-url-parallel:
+ee:relative-url:
   extends:
     - .parallel
-    - ee:relative-url
+    - ee:relative-url-selective
   rules:
     - !reference [.rules:test:qa-parallel, rules]
     - if: $QA_SUITES =~ /Test::Instance::All/
 
-ee:decomposition-single-db:
+ee:decomposition-single-db-selective:
   extends: .qa
   variables:
     QA_SCENARIO: Test::Instance::Image
     GITLAB_QA_OPTS: --omnibus-config decomposition_single_db
   rules:
-    - !reference [.rules:test:qa-non-parallel, rules]
+    - !reference [.rules:test:qa-selective, rules]
     - if: $QA_SUITES =~ /Test::Instance::All/
-ee:decomposition-single-db-parallel:
+ee:decomposition-single-db:
   extends:
     - .parallel
-    - ee:decomposition-single-db
+    - ee:decomposition-single-db-selective
   rules:
     - !reference [.rules:test:qa-parallel, rules]
     - if: $QA_SUITES =~ /Test::Instance::All/
 
-ee:decomposition-multiple-db:
+ee:decomposition-multiple-db-selective:
   extends: .qa
   variables:
     QA_SCENARIO: Test::Instance::Image
+    GITLAB_ALLOW_SEPARATE_CI_DATABASE: "true"
     GITLAB_QA_OPTS: --omnibus-config decomposition_multiple_db
   rules:
-    - !reference [.rules:test:qa-non-parallel, rules]
+    - !reference [.rules:test:qa-selective, rules]
     - if: $QA_SUITES =~ /Test::Instance::All/
-ee:decomposition-multiple-db-parallel:
+ee:decomposition-multiple-db:
   extends:
     - .parallel
-    - ee:decomposition-multiple-db
+    - ee:decomposition-multiple-db-selective
   rules:
     - !reference [.rules:test:qa-parallel, rules]
     - if: $QA_SUITES =~ /Test::Instance::All/
 
-ee:object-storage:
+ee:object-storage-selective:
   extends: .qa
   variables:
     QA_SCENARIO: Test::Instance::Image
     QA_RSPEC_TAGS: --tag object_storage
     GITLAB_QA_OPTS: --omnibus-config object_storage
   rules:
-    - !reference [.rules:test:qa-non-parallel, rules]
+    - !reference [.rules:test:qa-selective, rules]
     - if: $QA_SUITES =~ /Test::Instance::ObjectStorage/
-ee:object-storage-parallel:
-  extends: ee:object-storage
+ee:object-storage:
+  extends: ee:object-storage-selective
   parallel: 2
   rules:
     - !reference [.rules:test:qa-parallel, rules]
     - if: $QA_SUITES =~ /Test::Instance::ObjectStorage/
 
-ee:object-storage-aws:
-  extends: ee:object-storage
+ee:object-storage-aws-selective:
+  extends: ee:object-storage-selective
   variables:
     AWS_S3_ACCESS_KEY: $QA_AWS_S3_ACCESS_KEY
     AWS_S3_BUCKET_NAME: $QA_AWS_S3_BUCKET_NAME
     AWS_S3_KEY_ID: $QA_AWS_S3_KEY_ID
     AWS_S3_REGION: $QA_AWS_S3_REGION
     GITLAB_QA_OPTS: --omnibus-config object_storage_aws
-ee:object-storage-aws-parallel:
-  extends: ee:object-storage-aws
+ee:object-storage-aws:
+  extends: ee:object-storage-aws-selective
   parallel: 2
   rules:
-    - !reference [ee:object-storage-parallel, rules]
+    - !reference [ee:object-storage, rules]
 
-ee:object-storage-gcs:
-  extends: ee:object-storage
+ee:object-storage-gcs-selective:
+  extends: ee:object-storage-selective
   variables:
     GCS_BUCKET_NAME: $QA_GCS_BUCKET_NAME
     GOOGLE_PROJECT: $QA_GOOGLE_PROJECT
     GOOGLE_JSON_KEY: $QA_GOOGLE_JSON_KEY
     GOOGLE_CLIENT_EMAIL: $QA_GOOGLE_CLIENT_EMAIL
     GITLAB_QA_OPTS: --omnibus-config object_storage_gcs
-ee:object-storage-gcs-parallel:
-  extends: ee:object-storage-gcs
+ee:object-storage-gcs:
+  extends: ee:object-storage-gcs-selective
   parallel: 2
   rules:
-    - !reference [ee:object-storage-parallel, rules]
+    - !reference [ee:object-storage, rules]
 
-ee:packages:
+ee:packages-selective:
   extends: .qa
   variables:
     QA_SCENARIO: Test::Instance::Image
     QA_RSPEC_TAGS: --tag packages
     GITLAB_QA_OPTS: --omnibus-config packages
   rules:
-    - !reference [.rules:test:qa-non-parallel, rules]
+    - !reference [.rules:test:qa-selective, rules]
     - if: $QA_SUITES =~ /Test::Instance::Packages/
-ee:packages-parallel:
-  extends: ee:packages
+ee:packages:
+  extends: ee:packages-selective
   parallel: 2
   rules:
     - !reference [.rules:test:qa-parallel, rules]
@@ -585,7 +606,6 @@ ee:elasticsearch:
   variables:
     QA_SCENARIO: "Test::Integration::Elasticsearch"
   before_script:
-    - unset ELASTIC_URL  # unset url which is globally defined in .gitlab-ci.yml
     - !reference [.qa, before_script]
   rules:
     - !reference [.rules:test:qa, rules]
@@ -593,7 +613,7 @@ ee:elasticsearch:
     - !reference [.rules:test:manual, rules]
 
 ee:registry-object-storage-tls:
-  extends: ee:object-storage-aws
+  extends: ee:object-storage-aws-selective
   variables:
     QA_SCENARIO: Test::Integration::RegistryTLS
     QA_RSPEC_TAGS: ""
@@ -623,9 +643,24 @@ e2e-test-report:
     GITLAB_AUTH_TOKEN: $PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE
     ALLURE_PROJECT_PATH: $CI_PROJECT_PATH
     ALLURE_MERGE_REQUEST_IID: $CI_MERGE_REQUEST_IID
-    ALLURE_JOB_NAME: e2e-package-and-test
     GIT_STRATEGY: none
 
+# Temporary separate test report for super-sidebar test job
+# TODO: remove once super-sidebar is on by default and enabled in tests
+# https://gitlab.com/groups/gitlab-org/-/epics/9044
+e2e-test-report-super-sidebar:
+  extends:
+    - .generate-allure-report-base
+  stage: report
+  needs:
+    - _ee:super-sidebar-nav
+  variables:
+    ALLURE_JOB_NAME: e2e-super-sidebar
+    ALLURE_RESULTS_GLOB: gitlab-qa-run-*/**/allure-results-super-sidebar
+  rules:
+    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
+    - !reference [.rules:test:manual, rules]
+
 upload-knapsack-report:
   extends:
     - .generate-knapsack-report-base

.gitlab/ci/package-and-test/rules.gitlab-ci.yml

@@ -61,6 +61,15 @@
       when: never
     - when: always
 
+# This job requires project access token with api permissions to detect parallel jobs,
+# it is problematic to set for every project that would include this template
+# Because parallel jobs themselves can download knapsack report, skip for non canonical runs
+.rules:download-knapsack:
+  rules:
+    - <<: *not-canonical-project
+      when: never
+    - when: always
+
 # ------------------------------------------
 # Test
 # ------------------------------------------
@@ -80,7 +89,7 @@
 
 # parallel and non parallel rules are used for jobs that require parallel execution and thus need to switch
 # between parallel and non parallel when only certain specs are executed
-.rules:test:qa-non-parallel:
+.rules:test:qa-selective:
   rules:
     # always run parallel with full suite when framework changes present or ff state changed
     - <<: *qa-run-all-tests

.gitlab/ci/package-and-test/variables.gitlab-ci.yml

@@ -6,7 +6,12 @@ variables:
   SKIP_REPORT_IN_ISSUES: "true"
   OMNIBUS_GITLAB_CACHE_UPDATE: "false"
   OMNIBUS_GITLAB_RUBY3_BUILD: "false"
+  OMNIBUS_GITLAB_RUBY2_BUILD: "false"
   OMNIBUS_GITLAB_CACHE_EDITION: "GITLAB"
+  ALLURE_JOB_NAME: $CI_PROJECT_NAME
   QA_LOG_LEVEL: "info"
   QA_TESTS: ""
   QA_FEATURE_FLAGS: ""
+  # run all tests by default when package-and-test is included natively in other projects
+  # this will be overridden when selective test execution is used in gitlab canonical project
+  QA_RUN_ALL_TESTS: "true"

.gitlab/ci/preflight.gitlab-ci.yml

+rails-production-environment:
+  extends:
+    - .default-before_script
+    - .production
+    - .ruby-cache
+    - .setup:rules:rails-production-environment
+    - .use-pg12
+  stage: preflight
+  variables:
+    BUNDLE_WITHOUT: "development:test"
+    BUNDLE_WITH: "production"
+  needs: []
+  script:
+    - bundle exec rails runner --environment=production 'puts Rails.env'

.gitlab/ci/qa.gitlab-ci.yml

@@ -67,6 +67,7 @@ qa:update-qa-cache:
 
 e2e:package-and-test:
   extends:
+    - .production  # this makes sure GITLAB_ALLOW_SEPARATE_CI_DATABASE is passed to the child pipeline
     - .qa:rules:package-and-test
   stage: qa
   needs:
@@ -81,6 +82,16 @@ e2e:package-and-test:
     GITLAB_QA_IMAGE: "${CI_REGISTRY_IMAGE}/gitlab-ee-qa:${CI_COMMIT_SHA}"
     RUN_WITH_BUNDLE: "true"  # instructs pipeline to install and run gitlab-qa gem via bundler
     QA_PATH: qa  # sets the optional path for bundler to run from
+  inherit:
+    variables:
+      - CHROME_VERSION
+      - RUBY_VERSION
+      - DOCKER_VERSION
+      - REGISTRY_GROUP
+      - REGISTRY_HOST
+      - OMNIBUS_GITLAB_CACHE_EDITION
+      - OMNIBUS_GITLAB_RUBY3_BUILD
+      - OMNIBUS_GITLAB_RUBY2_BUILD
   trigger:
     strategy: depend
     forward:
@@ -89,3 +100,20 @@ e2e:package-and-test:
     include:
       - artifact: package-and-test-pipeline.yml
         job: e2e-test-pipeline-generate
+
+e2e:test-on-gdk:
+  extends:
+    - .qa:rules:e2e:test-on-gdk
+  stage: qa
+  needs:
+    # In scheduled master pipelines we wait for the image to be built.
+    # In MRs we assume the last scheduled master pipeline built the image already.
+    - job: build-qa-on-gdk-master-image
+      optional: true
+  allow_failure: true
+  trigger:
+    strategy: depend
+    forward:
+      yaml_variables: true
+      pipeline_variables: true
+    include: .gitlab/ci/test-on-gdk/main.gitlab-ci.yml

.gitlab/ci/rails.gitlab-ci.yml

@@ -105,6 +105,18 @@ rspec background_migration pg12 single-db:
     - .single-db-rspec
     - .rails:rules:single-db
 
+rspec migration pg12 single-db-ci-connection:
+  extends:
+    - rspec migration pg12
+    - .single-db-ci-connection-rspec
+    - .rails:rules:single-db-ci-connection
+
+rspec background_migration pg12 single-db-ci-connection:
+  extends:
+    - rspec background_migration pg12
+    - .single-db-ci-connection-rspec
+    - .rails:rules:single-db-ci-connection
+
 rspec migration pg12 praefect:
   extends:
     - rspec migration pg12
@@ -191,16 +203,6 @@ rspec system pg12 praefect:
     - .praefect-with-db
     - .rails:rules:praefect-with-db
 
-# Dedicated job to test DB library code against PG11.
-# Note that these are already tested against PG12 in the `rspec unit pg12` / `rspec-ee unit pg12` jobs.
-rspec db-library-code pg11:
-  extends:
-    - .rspec-base-pg11
-    - .rails:rules:ee-and-foss-db-library-code
-  script:
-    - !reference [.base-script, script]
-    - rspec_db_library_code
-
 rspec fast_spec_helper:
   extends:
     - .rspec-base-pg12
@@ -438,6 +440,18 @@ rspec background_migration pg12-as-if-foss single-db:
     - .single-db-rspec
     - .rails:rules:single-db
 
+rspec migration pg12-as-if-foss single-db-ci-connection:
+  extends:
+    - rspec migration pg12-as-if-foss
+    - .single-db-ci-connection-rspec
+    - .rails:rules:single-db-ci-connection
+
+rspec background_migration pg12-as-if-foss single-db-ci-connection:
+  extends:
+    - rspec background_migration pg12-as-if-foss
+    - .single-db-ci-connection-rspec
+    - .rails:rules:single-db-ci-connection
+
 rspec unit pg12-as-if-foss:
   extends:
     - .rspec-base-pg12-as-if-foss
@@ -530,6 +544,18 @@ rspec-ee background_migration pg12 single-db:
     - .single-db-rspec
     - .rails:rules:single-db
 
+rspec-ee migration pg12 single-db-ci-connection:
+  extends:
+    - rspec-ee migration pg12
+    - .single-db-ci-connection-rspec
+    - .rails:rules:single-db-ci-connection
+
+rspec-ee background_migration pg12 single-db-ci-connection:
+  extends:
+    - rspec-ee background_migration pg12
+    - .single-db-ci-connection-rspec
+    - .rails:rules:single-db-ci-connection
+
 rspec-ee migration pg12 praefect:
   extends:
     - rspec migration pg12
@@ -616,39 +642,6 @@ rspec-ee system pg12 single-db:
 ##########################################
 # EE/FOSS: default branch nightly scheduled jobs #
 
-# PG11
-rspec migration pg11:
-  extends:
-    - .rspec-base-pg11
-    - .rspec-base-migration
-    - .rails:rules:rspec-on-pg11
-    - .rspec-migration-parallel
-
-rspec background_migration pg11:
-  extends:
-    - .rspec-base-pg11
-    - .rspec-base-migration
-    - .rails:rules:rspec-on-pg11
-    - .rspec-background-migration-parallel
-
-rspec unit pg11:
-  extends:
-    - .rspec-base-pg11
-    - .rails:rules:rspec-on-pg11
-    - .rspec-unit-parallel
-
-rspec integration pg11:
-  extends:
-    - .rspec-base-pg11
-    - .rails:rules:rspec-on-pg11
-    - .rspec-integration-parallel
-
-rspec system pg11:
-  extends:
-    - .rspec-base-pg11
-    - .rails:rules:rspec-on-pg11
-    - .rspec-system-parallel
-
 # PG13
 rspec migration pg13:
   extends:
@@ -687,39 +680,6 @@ rspec system pg13:
 #####################################
 # EE: default branch nightly scheduled jobs #
 
-# PG11
-rspec-ee migration pg11:
-  extends:
-    - .rspec-ee-base-pg11
-    - .rspec-base-migration
-    - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only
-    - .rspec-ee-migration-parallel
-
-rspec-ee background_migration pg11:
-  extends:
-    - .rspec-ee-base-pg11
-    - .rspec-base-migration
-    - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only
-    - .rspec-ee-background-migration-parallel
-
-rspec-ee unit pg11:
-  extends:
-    - .rspec-ee-base-pg11
-    - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only
-    - .rspec-ee-unit-parallel
-
-rspec-ee integration pg11:
-  extends:
-    - .rspec-ee-base-pg11
-    - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only
-    - .rspec-ee-integration-parallel
-
-rspec-ee system pg11:
-  extends:
-    - .rspec-ee-base-pg11
-    - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only
-    - .rspec-ee-system-parallel
-
 # PG12
 rspec-ee unit pg12 opensearch1:
   extends:
@@ -802,22 +762,18 @@ rspec-ee system pg13:
   script:
     - !reference [.base-script, script]
     - rspec_fail_fast "${MATCHING_TESTS_PATH}" "--tag ~quarantine --tag ~zoekt"
-  artifacts:
-    expire_in: 7d
-    paths:
-      - tmp/capybara/
 
 rspec fail-fast:
   extends:
-    - .rspec-fail-fast
     - .rspec-base-pg12
+    - .rspec-fail-fast  # extends from .rspec-fail-fast last to override script from .rspec-base-pg12
   variables:
     MATCHING_TESTS_PATH: "${RSPEC_MATCHING_TESTS_FOSS_PATH}"
 
 rspec-ee fail-fast:
   extends:
-    - .rspec-fail-fast
-    - .rspec-base-pg12
+    - .rspec-ee-base-pg12
+    - .rspec-fail-fast  # extends from .rspec-fail-fast last to override script from .rspec-ee-base-pg12
   variables:
     MATCHING_TESTS_PATH: "${RSPEC_MATCHING_TESTS_EE_PATH}"
 
@@ -825,13 +781,14 @@ rspec-foss-impact:pipeline-generate:
   extends:
     - .rails:rules:rspec-foss-impact
   stage: prepare
-  needs: ["detect-tests"]
+  needs: ["detect-tests", "retrieve-tests-metadata"]
   script:
-    - scripts/generate-rspec-foss-impact-pipeline "${RSPEC_MATCHING_TESTS_FOSS_PATH}" "${RSPEC_FOSS_IMPACT_PIPELINE_YML}"
+    - scripts/generate_rspec_pipeline.rb -f "${RSPEC_MATCHING_TESTS_FOSS_PATH}" -t "${RSPEC_FOSS_IMPACT_PIPELINE_TEMPLATE_YML}" -k "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+    - cat "${RSPEC_FOSS_IMPACT_PIPELINE_TEMPLATE_YML}.yml"
   artifacts:
     expire_in: 1 day
     paths:
-      - $RSPEC_FOSS_IMPACT_PIPELINE_YML
+      - "${RSPEC_FOSS_IMPACT_PIPELINE_TEMPLATE_YML}.yml"
 
 rspec-foss-impact:trigger:
   extends:
@@ -854,7 +811,7 @@ rspec-foss-impact:trigger:
       yaml_variables: true
       pipeline_variables: true
     include:
-      - artifact: $RSPEC_FOSS_IMPACT_PIPELINE_YML
+      - artifact: "${RSPEC_FOSS_IMPACT_PIPELINE_TEMPLATE_YML}.yml"
         job: rspec-foss-impact:pipeline-generate
 
 fail-pipeline-early:

.gitlab/ci/rails/rspec-foss-impact.gitlab-ci.yml.erb

@@ -21,7 +21,7 @@ dont-interrupt-me:
   script:
     - echo "This jobs makes sure this pipeline won't be interrupted! See https://docs.gitlab.com/ee/ci/yaml/#interruptible."
 
-rspec foss-impact:
+.base-rspec-foss-impact:
   extends: .rspec-base-pg12-as-if-foss
   needs:
     - pipeline: $PARENT_PIPELINE_ID
@@ -37,9 +37,6 @@ rspec foss-impact:
   variables:
     RSPEC_TESTS_FILTER_FILE: "${RSPEC_MATCHING_TESTS_FOSS_PATH}"
     RSPEC_TESTS_MAPPING_ENABLED: "true"
-<% if Integer(parallel_value) > 1 %>
-  parallel: <%= parallel_value %>
-<% end %>
   script:
     - !reference [.base-script, script]
     - rspec_paralellized_job "--tag ~quarantine --tag ~level:migration --tag ~zoekt"
@@ -48,3 +45,46 @@ rspec foss-impact:
     paths:
       - "${RSPEC_MATCHING_TESTS_FOSS_PATH}"
       - tmp/capybara/
+
+<% if rspec_files_per_test_level[:migration][:files].size > 0 %>
+rspec migration foss-impact:
+  extends: .base-rspec-foss-impact
+<% if rspec_files_per_test_level[:migration][:parallelization] > 1 %>
+  parallel: <%= rspec_files_per_test_level[:migration][:parallelization] %>
+<% end %>
+  script:
+    - !reference [.base-script, script]
+    - rspec_paralellized_job "--tag ~quarantine --tag ~zoekt"
+<% end %>
+
+<% if rspec_files_per_test_level[:background_migration][:files].size > 0 %>
+rspec background_migration foss-impact:
+  extends: .base-rspec-foss-impact
+<% if rspec_files_per_test_level[:background_migration][:parallelization] > 1 %>
+  parallel: <%= rspec_files_per_test_level[:background_migration][:parallelization] %>
+<% end %>
+<% end %>
+
+<% if rspec_files_per_test_level[:unit][:files].size > 0 %>
+rspec unit foss-impact:
+  extends: .base-rspec-foss-impact
+<% if rspec_files_per_test_level[:unit][:parallelization] > 1 %>
+  parallel: <%= rspec_files_per_test_level[:unit][:parallelization] %>
+<% end %>
+<% end %>
+
+<% if rspec_files_per_test_level[:integration][:files].size > 0 %>
+rspec integration foss-impact:
+  extends: .base-rspec-foss-impact
+<% if rspec_files_per_test_level[:integration][:parallelization] > 1 %>
+  parallel: <%= rspec_files_per_test_level[:integration][:parallelization] %>
+<% end %>
+<% end %>
+
+<% if rspec_files_per_test_level[:system][:files].size > 0 %>
+rspec system foss-impact:
+  extends: .base-rspec-foss-impact
+<% if rspec_files_per_test_level[:system][:parallelization] > 1 %>
+  parallel: <%= rspec_files_per_test_level[:system][:parallelization] %>
+<% end %>
+<% end %>

.gitlab/ci/rails/shared.gitlab-ci.yml

@@ -36,9 +36,17 @@ include:
   variables:
     DECOMPOSED_DB: "false"
 
+.single-db-ci-connection:
+  extends: .single-db
+  variables:
+    CI_CONNECTION_DB: "true"
+
 .single-db-rspec:
   extends: .single-db
 
+.single-db-ci-connection-rspec:
+  extends: .single-db-ci-connection
+
 .praefect-with-db:
   variables:
     GITALY_PRAEFECT_WITH_DB: '1'
@@ -92,11 +100,6 @@ include:
     - !reference [.base-script, script]
     - rspec_paralellized_job "--tag ~quarantine --tag ~zoekt"
 
-.rspec-base-pg11:
-  extends:
-    - .rspec-base
-    - .use-pg11
-
 .rspec-base-pg12:
   extends:
     - .rspec-base
@@ -119,11 +122,6 @@ include:
     - .rspec-base
     - .use-pg13
 
-.rspec-ee-base-pg11:
-  extends:
-    - .rspec-base
-    - .use-pg11-es7-ee
-
 .rspec-ee-base-pg12:
   extends:
     - .rspec-base

.gitlab/ci/release-environments.gitlab-ci.yml

+---
+start-release-environments-pipeline:
+  allow_failure: true
+  extends:
+    - .release-environments:rules:start-release-environments-pipeline
+  stage: release-environments
+  # We do not want to have ALL global variables passed as trigger variables,
+  # as they cannot be overridden. See this issue for more context:
+  #
+  # https://gitlab.com/gitlab-org/gitlab/-/issues/387183
+  inherit:
+    variables: false
+
+  # These variables are set in the pipeline schedules.
+  # They need to be explicitly passed on to the child pipeline.
+  # https://docs.gitlab.com/ee/ci/pipelines/multi_project_pipelines.html#pass-cicd-variables-to-a-downstream-pipeline-by-using-the-variables-keyword
+  variables:
+    # This is needed by `release-environments-build-cng-env` (`.gitlab/ci/release-environments/main.gitlab-ci.yml`).
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+  trigger:
+    strategy: depend
+    include: .gitlab/ci/release-environments/main.gitlab-ci.yml