Merge branch 'andrey-simplify-qa-gem-caching' into 'master'

Simplify and improve qa gem caching setup

See merge request !137631



Merged-by: Rémy Coutable <remy@rymai.me>
Approved-by: Rémy Coutable <remy@rymai.me>
Reviewed-by: Rémy Coutable <remy@rymai.me>
Co-authored-by: Andrejs Cunskis <acunskis@gitlab.com>

.gitlab/ci/package-and-test/main.gitlab-ci.yml

@@ -30,19 +30,6 @@ download-fast-quarantine-report:
     - .download-fast-quarantine-report
     - .rules:download-fast-quarantine-report
 
-cache-gems:
-  extends:
-    - .qa-install
-    - .ruby-image
-    - .rules:update-cache
-  stage: .pre
-  tags:
-    - e2e
-  script:
-    - echo "Populated qa cache"
-  cache:
-    policy: pull-push
-
 # ==========================================
 # Test stage
 # ==========================================

.gitlab/ci/qa-common/rules.gitlab-ci.yml

@@ -64,10 +64,6 @@
   rules:
     - when: always
 
-.rules:update-cache:
-  rules:
-    - if: '$UPDATE_QA_CACHE == "true"'
-
 .rules:download-knapsack:
   rules:
     - when: always

.gitlab/ci/qa.gitlab-ci.yml

@@ -62,15 +62,21 @@ qa:master-auto-quarantine-dequarantine:
     - bundle exec confiner -r .confiner/master.yml
   allow_failure: true
 
-qa:update-qa-cache:
+cache-qa-gems:
   extends:
     - .qa-job-base
     - .qa-cache-push
-    - .shared:rules:update-cache
+    - .qa:rules:update-gem-cache
   stage: prepare
   script:
     - echo "Cache has been updated and ready to be uploaded."
 
+# E2E runners have separate infra setup and different cache bucket
+cache-qa-gems-e2e-runners:
+  extends: cache-qa-gems
+  tags:
+    - e2e
+
 trigger-omnibus:
   stage: qa
   extends:

.gitlab/ci/rules.gitlab-ci.yml

@@ -1443,6 +1443,16 @@
 ############
 # QA rules #
 ############
+.qa:rules:update-gem-cache:
+  rules:
+    - <<: *if-default-refs
+      changes:
+        - qa/Gemfile.lock
+    - <<: *if-schedule-maintenance
+    - <<: *if-security-schedule
+    - <<: *if-foss-schedule
+    - <<: *if-merge-request-labels-update-caches
+
 .qa:rules:code-merge-request-manual:
   rules:
     - <<: *if-merge-request
@@ -1592,11 +1602,6 @@
     - <<: *if-merge-request
       changes: *ci-qa-patterns
       allow_failure: true
-    - <<: *if-merge-request
-      changes:
-        - qa/Gemfile.lock  # qa/Gemfile.lock is a part of *qa-patterns, so this rule must be placed before the one with *qa-patterns changes
-      variables:
-        UPDATE_QA_CACHE: "true"
     - <<: *if-merge-request
       changes: *qa-patterns
       allow_failure: true
@@ -1636,11 +1641,6 @@
       when: manual
     - <<: *if-merge-request
       changes: *nodejs-patterns
-    - <<: *if-merge-request
-      changes:
-        - qa/Gemfile.lock  # qa/Gemfile.lock is a part of *qa-patterns, so this rule must be placed before the one with *qa-patterns changes
-      variables:
-        UPDATE_QA_CACHE: "true"
     - <<: *if-dot-com-gitlab-org-and-security-merge-request-and-qa-tests-specified
       changes: *code-patterns
     - <<: *if-merge-request
@@ -1657,7 +1657,6 @@
     CREATE_TEST_FAILURE_ISSUES: "true"
     PROCESS_TEST_RESULTS: "true"
     KNAPSACK_GENERATE_REPORT: "true"
-    UPDATE_QA_CACHE: "true"
     QA_SAVE_TEST_METRICS: "true"
     QA_EXPORT_TEST_METRICS: "false"  # on main runs, metrics are exported to separate bucket via rake task for better consistency
 

.gitlab/ci/test-on-gdk/main.gitlab-ci.yml

@@ -124,17 +124,6 @@ download-knapsack-report:
     - .download-knapsack-report
     - .rules:download-knapsack
 
-cache-gems:
-  extends:
-    - .ruby-image
-    - .qa-cache-push
-    - .rules:update-cache
-  stage: .pre
-  tags:
-    - e2e
-  script:
-    - cd qa && bundle install
-
 # Take the existing GDK docker image and reconfigure it with Postgres load
 # balancing. Adding 5s lag to 1 of the replicas to validate robustness of
 # the load balancer.