Follow up to native e2e testing pipeline

.gitlab/ci/package-and-test/main.gitlab-ci.yml

@@ -4,7 +4,7 @@ include:
   - local: .gitlab/ci/global.gitlab-ci.yml
   - local: .gitlab/ci/package-and-test/rules.gitlab-ci.yml
   - project: gitlab-org/quality/pipeline-common
-    ref: 1.0.1
+    ref: 1.2.0
     file:
       - /ci/base.gitlab-ci.yml
       - /ci/allure-report.yml
@@ -22,11 +22,12 @@ include:
   image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-${RUBY_VERSION}:bundler-2.3
 
 .bundle-install:
-  extends: .qa-cache
+  extends:
+    - .gitlab-qa-install
+    - .qa-cache
   variables:
-    RUN_WITH_BUNDLE: "true" # adds 'bundle exec' prefix for gitlab-qa commands in pipeline-common job templates
-  before_script:
-    - cd qa && bundle install
+    RUN_WITH_BUNDLE: "true" # installs and runs gitlab-qa via bundler
+    QA_PATH: qa
 
 .update-script:
   script:
@@ -35,15 +36,8 @@ include:
     - echo "Running - '$QA_COMMAND'"
     - eval "$QA_COMMAND"
 
-.manual-qa:
-  stage: test
-  allow_failure: true
-  needs:
-    - trigger-omnibus
-
 .qa:
   extends:
-    - .use-docker-in-docker
     - .qa-base
     - .bundle-install
     - .gitlab-qa-report
@@ -90,6 +84,19 @@ download-knapsack-report:
       - qa/knapsack/ee-*.json
     expire_in: 1 day
 
+# e2e test jobs run on separate runner which has separate cache setup
+cache-gems:
+  extends:
+    - .ruby-image
+    - .bundle-install
+    - .qa-cache-push
+    - .rules:prepare
+  stage: .pre
+  tags:
+    - e2e
+  script:
+    - echo "Populated qa cache"
+
 # ==========================================
 # Test stage
 # ==========================================
@@ -104,8 +111,11 @@ download-knapsack-report:
 _ee:quarantine:
   extends:
     - .qa
-    - .manual-qa
     - .rules:test:quarantine
+  needs:
+    - trigger-omnibus
+  stage: test
+  allow_failure: true
   variables:
     QA_RSPEC_TAGS: --tag quarantine
 
@@ -446,7 +456,7 @@ notify-slack:
     - .rules:report:process-results
   stage: .post
   variables:
-    ALLURE_JOB_NAME: package-and-qa
+    ALLURE_JOB_NAME: package-and-e2e
     SLACK_ICON_EMOJI: ci_failing
     STATUS_SYM: ☠️
     STATUS: failed

.gitlab/ci/qa.gitlab-ci.yml

@@ -69,7 +69,7 @@ qa:update-qa-cache:
 populate-e2e-test-vars:
   extends:
     - .qa-job-base
-    - .qa:rules:determine-qa-tests
+    - .qa:rules:determine-e2e-tests
   stage: prepare
   variables:
     ENV_FILE: $CI_PROJECT_DIR/qa_tests_vars.env
@@ -83,7 +83,7 @@ populate-e2e-test-vars:
 
 e2e-test-pipeline-generate:
   extends:
-    - .qa:rules:determine-qa-tests
+    - .qa:rules:determine-e2e-tests
   stage: prepare
   when: on_success
   needs:
@@ -99,7 +99,7 @@ e2e-test-pipeline-generate:
 
 e2e:package-and-test:
   extends:
-    - .qa:rules:package-and-qa
+    - .qa:rules:package-and-test
   stage: qa
   when: on_success
   needs:

.gitlab/ci/review-apps/qa.gitlab-ci.yml

@@ -162,7 +162,7 @@ allure-report-qa-blocking:
     - review-qa-smoke
     - review-qa-reliable
   variables:
-    ALLURE_JOB_NAME: review-qa-blocking
+    ALLURE_JOB_NAME: e2e-review-qa-blocking
 
 allure-report-qa-all:
   extends:
@@ -170,7 +170,7 @@ allure-report-qa-all:
     - .review:rules:review-qa-all-report
   needs: ["review-qa-all"]
   variables:
-    ALLURE_JOB_NAME: review-qa-all
+    ALLURE_JOB_NAME: e2e-review-qa-all
 
 upload-knapsack-report:
   extends:

.gitlab/ci/rules.gitlab-ci.yml

@@ -133,7 +133,7 @@
 .if-dot-com-gitlab-org-and-security-merge-request-and-qa-tests-specified: &if-dot-com-gitlab-org-and-security-merge-request-and-qa-tests-specified
   if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/security$)/ && $CI_MERGE_REQUEST_IID && $QA_TESTS'
 
-.if-dot-com-gitlab-org-and-security-merge-request-manual-ff-package-and-qa: &if-dot-com-gitlab-org-and-security-merge-request-manual-ff-package-and-qa
+.if-dot-com-gitlab-org-and-security-merge-request-manual-ff-package-and-e2e: &if-dot-com-gitlab-org-and-security-merge-request-manual-ff-package-and-e2e
   if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/security$)/ && $CI_MERGE_REQUEST_IID && $QA_MANUAL_FF_PACKAGE_AND_QA'
 
 .if-dot-com-gitlab-org-and-security-tag: &if-dot-com-gitlab-org-and-security-tag
@@ -916,7 +916,7 @@
     - <<: *if-default-refs
       changes: *qa-patterns
 
-.qa:rules:determine-qa-tests:
+.qa:rules:determine-e2e-tests:
   rules:
     - <<: *if-not-ee
       when: never
@@ -930,7 +930,7 @@
     - <<: *if-force-ci
       allow_failure: true
 
-.qa:rules:package-and-qa:
+.qa:rules:package-and-test:
   rules:
     - <<: *if-fork-merge-request
       when: never
@@ -940,7 +940,7 @@
       when: never
     - <<: *if-merge-request-targeting-stable-branch
       allow_failure: true
-    - <<: *if-dot-com-gitlab-org-and-security-merge-request-manual-ff-package-and-qa
+    - <<: *if-dot-com-gitlab-org-and-security-merge-request-manual-ff-package-and-e2e
       changes: *feature-flag-development-config-patterns
       when: manual
       allow_failure: true

.gitlab/merge_request_templates/Quarantine End to End Test.md

@@ -26,7 +26,6 @@ the noise (due to constantly failing tests, flaky tests, and so on) so that new
 - [ ] Dequarantine test check-list
   - [ ] Follow the [Dequarantining Tests guide](https://about.gitlab.com/handbook/engineering/quality/quality-engineering/debugging-qa-test-failures/#dequarantining-tests).
   - [ ] Confirm the test consistently passes on the target GitLab environment(s).
-    - [ ] (Optionally) [Trigger a manual GitLab-QA pipeline](https://about.gitlab.com/handbook/engineering/quality/quality-engineering/tips-and-tricks/#running-gitlab-qa-pipeline-against-a-specific-gitlab-release) against a specific GitLab environment using the `RELEASE` variable from the `package-and-qa` job of the current merge request.
 - [ ] To ensure a faster turnaround, ask in the `#quality` Slack channel for someone to review and merge the merge request, rather than assigning it directly.
 
 <!-- Base labels. -->

.gitlab/merge_request_templates/Security Release.md

@@ -21,7 +21,7 @@ See [the general developer security release guidelines](https://gitlab.com/gitla
   - [ ] Ensure it's approved according to our [Approval Guidelines].
   - [ ] Ensure it's approved by an AppSec engineer.
     - Please see the security release [Code reviews and Approvals] documentation for details on which AppSec team member to ping for approval.
-    - Trigger the [`package-and-qa` build]. The docker image generated will be used by the AppSec engineer to validate the security vulnerability has been remediated.
+    - Trigger the [`e2e:package-and-test` job]. The docker image generated will be used by the AppSec engineer to validate the security vulnerability has been remediated.
 - [ ] For a backport MR targeting a versioned stable branch (`X-Y-stable-ee`).
   - [ ] Milestone is set to the version this backport applies to. A closed milestone can be assigned via [quick actions].
   - [ ] Ensure it's approved by a maintainer.

doc/development/fips_compliance.md

@@ -578,6 +578,6 @@ Merge requests that can trigger Package and QA, can trigger a FIPS package and a
 Reference Architecture test pipeline. The base image used for the trigger is
 Ubuntu 20.04 FIPS:
 
-1. Trigger `package-and-qa`, if not already triggered.
+1. Trigger `e2e:package-and-test` job, if not already triggered.
 1. On the `gitlab-omnibus-mirror` child pipeline, manually trigger `Trigger:package:fips`.
 1. When the package job is complete, manually trigger the `RAT:FIPS` job.

doc/development/pipelines.md

@@ -103,7 +103,7 @@ When you need to revert a merge request, to get accelerated feedback, you can ad
 
 When this label is assigned, the following steps of the CI/CD pipeline are skipped:
 
-- The `package-and-qa` job.
+- The `e2e:package-and-test` job.
 - The `rspec:undercoverage` job.
 - The entire [Review Apps process](testing_guide/review_apps.md).
 
@@ -489,7 +489,7 @@ graph RL;
   class 2_3-1 criticalPath;
   2_3-1 --> 1-5
 
-  2_4-1["package-and-qa (102 minutes)"];
+  2_4-1["e2e:package-and-test (102 minutes)"];
   class 2_4-1 criticalPath;
   click 2_4-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914305&udv=0"
   2_4-1 --> 1-2 & 2_3-1 & 1-15;

doc/development/service_ping/implement.md

@@ -806,7 +806,7 @@ This is the recommended approach to test Prometheus-based Service Ping.
 To verify your change, build a new Omnibus image from your code branch using CI/CD, download the image,
 and run a local container instance:
 
-1. From your merge request, select the `qa` stage, then trigger the `package-and-qa` job. This job triggers an Omnibus
+1. From your merge request, select the `qa` stage, then trigger the `e2e:package-and-test` job. This job triggers an Omnibus
    build in a [downstream pipeline of the `omnibus-gitlab-mirror` project](https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/pipelines).
 1. In the downstream pipeline, wait for the `gitlab-docker` job to finish.
 1. Open the job logs and locate the full container name including the version. It takes the following form: `registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>`.

doc/development/testing_guide/end_to_end/feature_flags.md

@@ -195,12 +195,10 @@ End-to-end tests should pass with a feature flag enabled before it is enabled on
 There are two ways to confirm that end-to-end tests pass:
 
 - If a merge request adds or edits a [feature flag definition file](../../feature_flags/index.md#feature-flag-definition-and-validation),
-  two `package-and-qa` jobs (`package-and-qa-ff-enabled` and `package-and-qa-ff-disabled`) are included automatically in the merge request
-  pipeline. One job enables the defined feature flag and the other job disables it. The jobs execute the same suite of tests to confirm
-  that they pass with the feature flag either enabled or disabled.
-- In some cases, if `package-and-qa` hasn't been triggered automatically, or if it has run the tests with the default feature flag values
-  (which might not be desired), you can create a Draft MR that enables the feature flag to ensure that all E2E tests pass with the feature
-  flag enabled.
+  two `e2e:package-and-test` jobs (`ee:instance-parallel` and `ee:instance-parallel-ff-inverse`) are included automatically in the merge request pipeline. 
+  One job runs the application with default feature flag state and another sets it to inverse value. The jobs execute the same suite of tests to confirm that they pass with the feature flag either enabled or disabled.
+- In some cases, if end-to-end test jobs didn't trigger automatically, or if it has run the tests with the default feature flag values (which might not be desired),
+  you can create a Draft MR that enables the feature flag to ensure that all E2E tests pass with the feature flag enabled and disabled.
 
 ### Troubleshooting end-to-end test failures with feature flag enabled
 
@@ -216,7 +214,7 @@ If enabling the feature flag results in E2E test failures, you can browse the ar
 ### Test execution during feature development
 
 If an end-to-end test enables a feature flag, the end-to-end test suite can be used to test changes in a merge request
-by running the `package-and-qa` job in the merge request pipeline. If the feature flag and relevant changes have already been merged, you can confirm that the tests
+by running the `e2e:package-and-test` job in the merge request pipeline. If the feature flag and relevant changes have already been merged, you can confirm that the tests
 pass on the default branch. The end-to-end tests run on the default branch every two hours, and the results are posted to a
 [Test Session Report, which is available in the testcase-sessions project](https://gitlab.com/gitlab-org/quality/testcase-sessions/-/issues?label_name%5B%5D=found%3Amain).
 

doc/development/testing_guide/end_to_end/index.md

@@ -56,7 +56,7 @@ graph TB
 
 subgraph "`gitlab-org/gitlab` pipeline"
     A1[`build-images` stage<br>`build-qa-image` and `build-assets-image` jobs]
-    A2[`qa` stage<br>`package-and-qa` job]
+    A2[`qa` stage<br>`e2e:package-and-test` job]
     end
 
 subgraph "`gitlab-org/build/omnibus-gitlab-mirror` pipeline"
@@ -186,7 +186,7 @@ Use these environment variables to configure metrics export:
 | -------- | -------- | ----------- |
 | `QA_INFLUXDB_URL` | `true` | Should be set to `https://influxdb.quality.gitlab.net`. No default value. |
 | `QA_INFLUXDB_TOKEN` | `true` | InfluxDB write token that can be found under `Influxdb auth tokens` document in `Gitlab-QA` `1Password` vault. No default value. |
-| `QA_RUN_TYPE` | `false` | Arbitrary name for test execution, like `package-and-qa`. Automatically inferred from the project name for live environment test executions. No default value. |
+| `QA_RUN_TYPE` | `false` | Arbitrary name for test execution, like `package-and-test`. Automatically inferred from the project name for live environment test executions. No default value. |
 | `QA_EXPORT_TEST_METRICS` | `false` | Flag to enable or disable metrics export. Defaults to `true`. |
 
 ## Test reports

scripts/trigger-build.rb

@@ -163,7 +163,7 @@ def version_file_variables
 
   class Omnibus < Base
     def self.access_token
-      # Default to "Multi-pipeline (from 'gitlab-org/gitlab' 'package-and-qa' job)" at https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/settings/access_tokens
+      # Default to "Multi-pipeline (from 'gitlab-org/gitlab' 'e2e:package-and-test' job)" at https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/settings/access_tokens
       ENV['OMNIBUS_GITLAB_PROJECT_ACCESS_TOKEN'] || super
     end