Raise RSpec parallelism based on average duration

.gitlab/ci/artifact-collector/as-if-foss.gitlab-ci.yml

@@ -16,9 +16,9 @@ rspec:artifact-collector as-if-foss:
       # optional: true
     - project: "${CI_PROJECT_NAMESPACE}/gitlab-foss"
       ref: as-if-foss/${CI_COMMIT_REF_NAME}
-      job: rspec:artifact-collector system
+      job: rspec:artifact-collector part-a
       # optional: true
     - project: "${CI_PROJECT_NAMESPACE}/gitlab-foss"
       ref: as-if-foss/${CI_COMMIT_REF_NAME}
-      job: rspec:artifact-collector remainder
+      job: rspec:artifact-collector part-b
       # optional: true

.gitlab/ci/rails.gitlab-ci.yml

@@ -436,29 +436,29 @@ rspec:artifact-collector unit:
     - .artifact-collector
     - .rails:rules:artifact-collector-unit
   needs:
-    - job: rspec unit pg14  # 32 jobs
+    - job: rspec unit pg14  # 44 jobs
       optional: true
     - job: rspec unit clickhouse  # 1 job
       optional: true
 
-rspec:artifact-collector system:
+rspec:artifact-collector part-a:
   extends:
     - .artifact-collector
-    - .rails:rules:artifact-collector-system
+    - .rails:rules:artifact-collector-part-a
   needs:
     - job: rspec system pg14  # 32 jobs
       optional: true
+    - job: rspec background_migration pg14  # 10 jobs
+      optional: true
 
-rspec:artifact-collector remainder:
+rspec:artifact-collector part-b:
   extends:
     - .artifact-collector
-    - .rails:rules:artifact-collector-remainder
+    - .rails:rules:artifact-collector-part-b
   needs:
-    - job: rspec integration pg14  # 16 jobs
-      optional: true
-    - job: rspec migration pg14  # 15 jobs
+    - job: rspec integration pg14  # 20 jobs
       optional: true
-    - job: rspec background_migration pg14  # 5 jobs
+    - job: rspec migration pg14  # 26 jobs
       optional: true
 
 rspec:artifact-collector unit single-redis:
@@ -466,7 +466,7 @@ rspec:artifact-collector unit single-redis:
     - .artifact-collector
     - .rails:rules:single-redis
   needs:
-    - job: rspec unit pg14 single-redis  # 32 jobs
+    - job: rspec unit pg14 single-redis  # 44 jobs
       optional: true
 
 rspec:artifact-collector system single-redis:
@@ -482,26 +482,32 @@ rspec:artifact-collector remainder single-redis:
     - .artifact-collector
     - .rails:rules:single-redis
   needs:
-    - job: rspec integration pg14 single-redis  # 16 jobs
+    - job: rspec integration pg14 single-redis  # 20 jobs
       optional: true
 
-rspec:artifact-collector ee single-redis:
+rspec:artifact-collector ee unit single-redis:
   extends:
     - .artifact-collector
     - .rails:rules:single-redis
   needs:
-    - job: rspec-ee unit pg14 single-redis  # 28 jobs
+    - job: rspec-ee unit pg14 single-redis  # 39 jobs
       optional: true
-    - job: rspec-ee integration pg14 single-redis  # 7 jobs
+
+rspec:artifact-collector ee remainder single-redis:
+  extends:
+    - .artifact-collector
+    - .rails:rules:single-redis
+  needs:
+    - job: rspec-ee integration pg14 single-redis  # 9 jobs
       optional: true
-    - job: rspec-ee system pg14 single-redis  # 14 jobs
+    - job: rspec-ee system pg14 single-redis  # 16 jobs
       optional: true
 
 rspec:artifact-collector ee unit:
   extends:
     - .artifact-collector
   needs:
-    - job: rspec-ee unit pg14  # 28 jobs
+    - job: rspec-ee unit pg14  # 39 jobs
       optional: true
   rules:
     - !reference ['.rails:rules:ee-only-unit', rules]
@@ -512,13 +518,13 @@ rspec:artifact-collector ee remainder:
   needs:
     - job: rspec-ee migration pg14  # 2 jobs
       optional: true
-    - job: rspec-ee background_migration pg14  # 2 jobs
+    - job: rspec-ee background_migration pg14  # 3 jobs
       optional: true
     - job: rspec-ee unit clickhouse  # 1 job
       optional: true
-    - job: rspec-ee integration pg14  # 7 jobs
+    - job: rspec-ee integration pg14  # 9 jobs
       optional: true
-    - job: rspec-ee system pg14  # 14 jobs
+    - job: rspec-ee system pg14  # 16 jobs
       optional: true
   rules:
     - !reference ['.rails:rules:ee-only-migration', rules]
@@ -538,9 +544,9 @@ rspec:coverage:
     # FOSS/EE jobs
     - job: rspec:artifact-collector unit
       optional: true
-    - job: rspec:artifact-collector system
+    - job: rspec:artifact-collector part-a
       optional: true
-    - job: rspec:artifact-collector remainder
+    - job: rspec:artifact-collector part-b
       optional: true
     - job: rspec:artifact-collector unit single-redis
       optional: true
@@ -552,7 +558,9 @@ rspec:coverage:
     - job: rspec:artifact-collector as-if-foss
       optional: true
     # EE jobs
-    - job: rspec:artifact-collector ee single-redis
+    - job: rspec:artifact-collector ee unit single-redis
+      optional: true
+    - job: rspec:artifact-collector ee remainder single-redis
       optional: true
     - job: rspec:artifact-collector ee unit
       optional: true

.gitlab/ci/rails/shared.gitlab-ci.yml

@@ -339,33 +339,51 @@ include:
 # rspec job parallel configs
 # Adjusting these parallel job counts has an impact on the
 # rspec:artifact-collector jobs in .gitlab/ci/rails.gitlab-ci.yml
-# Please double-check and adjust accordingly
+# Please double-check and adjust accordingly. At the moment,
+# each job can only have needs from up to 50 jobs.
+#
+# To calculate the number of jobs we need for each RSpec, follow this formula:
+#
+#     parallel_job_count = ceil(current_job_count * (average_duration / target_duration))
+#
+# At the moment, we set the target to 30 minutes
+#
+# https://app.snowflake.com/ys68254/gitlab/w2woEkSvsfYa/chart
 .rspec-migration-parallel:
-  parallel: 15
+  parallel: 26
 
+# https://app.snowflake.com/ys68254/gitlab/wpUmrAF0RJL/chart
 .rspec-background-migration-parallel:
-  parallel: 5
+  parallel: 10
 
+# https://app.snowflake.com/ys68254/gitlab/w4cxPH8ySQlP/chart
 .rspec-ee-migration-parallel:
   parallel: 2
 
+# https://app.snowflake.com/ys68254/gitlab/w20QT8KSVZAi/chart
 .rspec-ee-background-migration-parallel:
-  parallel: 2
+  parallel: 3
 
+# https://app.snowflake.com/ys68254/gitlab/w1tkkoZAXSsZ/chart
 .rspec-unit-parallel:
-  parallel: 32
+  parallel: 44
 
+# https://app.snowflake.com/ys68254/gitlab/w2GlD4YqHSbO/chart
 .rspec-ee-unit-parallel:
-  parallel: 28
+  parallel: 39
 
+# https://app.snowflake.com/ys68254/gitlab/w56UXaCA4xkG/chart
 .rspec-integration-parallel:
-  parallel: 16
+  parallel: 20
 
+# https://app.snowflake.com/ys68254/gitlab/w4tXhUkSsFj/chart
 .rspec-ee-integration-parallel:
-  parallel: 7
+  parallel: 9
 
+# https://app.snowflake.com/ys68254/gitlab/w3Y3rkdbS25v/chart
 .rspec-system-parallel:
   parallel: 32
 
+# https://app.snowflake.com/ys68254/gitlab/w4zGboX1YRrY/chart
 .rspec-ee-system-parallel:
-  parallel: 14
+  parallel: 16

.gitlab/ci/rules.gitlab-ci.yml

@@ -2323,17 +2323,17 @@
     - if: '$START_AS_IF_FOSS == "true"'
     - !reference [".rails:rules:ee-and-foss-unit", rules]
 
-.rails:rules:artifact-collector-system:
+.rails:rules:artifact-collector-part-a:
   rules:
     - if: '$START_AS_IF_FOSS == "true"'
     - !reference [".rails:rules:ee-and-foss-system", rules]
+    - !reference ['.rails:rules:ee-and-foss-background-migration', rules]
 
-.rails:rules:artifact-collector-remainder:
+.rails:rules:artifact-collector-part-b:
   rules:
     - if: '$START_AS_IF_FOSS == "true"'
     - !reference ['.rails:rules:ee-and-foss-integration', rules]
     - !reference ['.rails:rules:ee-and-foss-migration', rules]
-    - !reference ['.rails:rules:ee-and-foss-background-migration', rules]
 
 .rails:rules:detect-tests:
   rules: