Run FOSS system tests

* We also use `needs` to speed it up
* To simplify the dependencies, we also pull cache on master

.gitlab/ci/frontend.gitlab-ci.yml

@@ -53,7 +53,7 @@
     - gitlab-org
     - docker
 
-gitlab:assets:compile:
+gitlab:assets:compile pull-push-cache:
   extends: .gitlab:assets:compile-metadata
   only:
     refs:
@@ -63,9 +63,6 @@ gitlab:assets:compile:
 
 gitlab:assets:compile pull-cache:
   extends: .gitlab:assets:compile-metadata
-  except:
-    refs:
-      - master
   cache:
     policy: pull
 
@@ -89,14 +86,14 @@ gitlab:assets:compile pull-cache:
     # we override the max_old_space_size to prevent OOM errors
     NODE_OPTIONS: --max_old_space_size=3584
   cache:
-    key: "assets-compile:test:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6"
+    key: "assets-compile:v7"
   artifacts:
     expire_in: 7d
     paths:
       - node_modules
       - public/assets
 
-compile-assets:
+compile-assets pull-push-cache:
   extends: .compile-assets-metadata
   only:
     refs:
@@ -104,13 +101,25 @@ compile-assets:
   cache:
     policy: pull-push
 
-compile-assets pull-cache:
-  extends: .compile-assets-metadata
-  except:
+compile-assets pull-push-cache foss:
+  extends: [".compile-assets-metadata", ".only-ee-as-if-foss"]
+  only:
     refs:
       - master
+  cache:
+    policy: pull-push
+    key: "assets-compile:v7:foss"
+
+compile-assets pull-cache:
+  extends: .compile-assets-metadata
+  cache:
+    policy: pull
+
+compile-assets pull-cache foss:
+  extends: [".compile-assets-metadata", ".only-ee-as-if-foss"]
   cache:
     policy: pull
+    key: "assets-compile:v7:foss"
 
 .only-code-frontend-job-base:
   extends:
@@ -121,7 +130,9 @@ compile-assets pull-cache:
     - .default-before_script
     - .only-code-changes
     - .use-pg9
-  dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"]
+  stage: test
+  needs: ["setup-test-env", "compile-assets pull-cache"]
+  dependencies: ["setup-test-env", "compile-assets pull-cache"]
 
 .karma-base:
   extends: .only-code-frontend-job-base
@@ -195,6 +206,7 @@ jest-foss:
     - .default-cache
     - .default-only
     - .only-code-changes
+  stage: test
   dependencies: []
   cache:
     key: "$CI_JOB_NAME"
@@ -227,7 +239,9 @@ webpack-dev-server:
     - .default-cache
     - .default-only
     - .only-code-changes
-  dependencies: ["setup-test-env", "compile-assets", "compile-assets pull-cache"]
+  stage: test
+  needs: ["setup-test-env", "compile-assets pull-cache"]
+  dependencies: ["setup-test-env", "compile-assets pull-cache"]
   variables:
     WEBPACK_MEMORY_TEST: "true"
   script:

.gitlab/ci/pages.gitlab-ci.yml

@@ -11,7 +11,7 @@ pages:
     variables:
       - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
   stage: pages
-  dependencies: ["coverage", "karma", "gitlab:assets:compile"]
+  dependencies: ["coverage", "karma", "gitlab:assets:compile pull-cache"]
   script:
     - mv public/ .public/
     - mkdir public/

.gitlab/ci/qa.gitlab-ci.yml

@@ -71,4 +71,4 @@ schedule:package-and-qa:
     - .package-and-qa-base
     - .only-code-qa-changes
     - .only-canonical-schedules
-  needs: ["build-qa-image", "gitlab:assets:compile"]
+  needs: ["build-qa-image", "gitlab:assets:compile pull-cache"]

.gitlab/ci/rails.gitlab-ci.yml

@@ -53,6 +53,8 @@ setup-test-env:
 .rspec-base:
   extends: .only-code-rails-job-base
   stage: test
+  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache"]
+  dependencies: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache"]
   script:
     - source scripts/rspec_helpers.sh
     - rspec_paralellized_job "--tag ~quarantine --tag ~geo"
@@ -69,6 +71,11 @@ setup-test-env:
     reports:
       junit: junit_rspec.xml
 
+.rspec-base-foss:
+  extends: [".rspec-base", ".only-ee-as-if-foss"]
+  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache foss"]
+  dependencies: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache foss"]
+
 .rspec-base-pg9:
   extends:
     - .rspec-base
@@ -76,9 +83,8 @@ setup-test-env:
 
 .rspec-base-pg9-foss:
   extends:
-    - .rspec-base
+    - .rspec-base-foss
     - .use-pg9
-    - .only-ee-as-if-foss
 
 .rspec-base-pg10:
   extends:
@@ -106,10 +112,9 @@ rspec system pg9:
   extends: .rspec-base-pg9
   parallel: 24
 
-# TODO: This requires FOSS assets
-# rspec system pg9-foss:
-#   extends: .rspec-base-pg9-foss
-#   parallel: 24
+rspec system pg9-foss:
+  extends: .rspec-base-pg9-foss
+  parallel: 24
 
 rspec unit pg10:
   extends: .rspec-base-pg10
@@ -229,7 +234,9 @@ rspec fast_spec_helper:
 
 static-analysis:
   extends: .only-code-qa-rails-job-base
-  dependencies: ["setup-test-env", "compile-assets", "compile-assets pull-cache"]
+  stage: test
+  needs: ["setup-test-env", "compile-assets pull-cache"]
+  dependencies: ["setup-test-env", "compile-assets pull-cache"]
   variables:
     SETUP_DB: "false"
   script:
@@ -252,16 +259,16 @@ downtime_check:
     variables:
       - $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/
   stage: test
-  dependencies: ["setup-test-env"]
   needs: ["setup-test-env"]
+  dependencies: ["setup-test-env"]
 
 .db-job-base:
   extends:
     - .only-code-rails-job-base
     - .use-pg9
   stage: test
-  dependencies: ["setup-test-env"]
   needs: ["setup-test-env"]
+  dependencies: ["setup-test-env"]
 
 # DB migration, rollback, and seed jobs
 db:migrate:reset:

.gitlab/ci/review.gitlab-ci.yml

@@ -81,7 +81,7 @@ schedule:review-build-cng:
   extends:
     - .review-build-cng-base
     - .only-review-schedules
-  needs: ["gitlab:assets:compile"]
+  needs: ["gitlab:assets:compile pull-cache"]
 
 .review-deploy-base:
   extends:

doc/development/pipelines.md

@@ -127,7 +127,7 @@ execute jobs out of order for the following jobs:
 ```mermaid
 graph RL;
   A[setup-test-env];
-  B["gitlab:assets:compile<br/>(master only)"];
+  B["gitlab:assets:compile pull-push-cache<br/>(master only)"];
   C[gitlab:assets:compile pull-cache];
   D["cache gems<br/>(master and tags only)"];
   E[review-build-cng];
@@ -136,7 +136,7 @@ graph RL;
   G2["schedule:review-deploy<br/>(master only)"];
   H[karma];
   I[jest];
-  J["compile-assets<br/>(master only)"];
+  J["compile-assets pull-push-cache<br/>(master only)"];
   K[compile-assets pull-cache];
   L[webpack-dev-server];
   M[coverage];
@@ -145,39 +145,41 @@ graph RL;
   P["schedule:package-and-qa<br/>(master schedule only)"];
   Q[package-and-qa];
   R[package-and-qa-manual];
+  S["RSpec<br/>(e.g. rspec unit pg9)"]
+  T[retrieve-tests-metadata];
 
 subgraph "`prepare` stage"
     A
     F
-    J
     K
+    J
+    T
     end
 
 subgraph "`test` stage"
     B --> |needs| A;
     C --> |needs| A;
     D --> |needs| A;
-    H -.-> |depends on| A;
-    H -.-> |depends on| J;
-    H -.-> |depends on| K;
-    I -.-> |depends on| A;
-    I -.-> |depends on| J;
-    I -.-> |depends on| K;
-    L -.-> |depends on| A;
-    L -.-> |depends on| J;
-    L -.-> |depends on| K;
+    H -.-> |needs and depends on| A;
+    H -.-> |needs and depends on| K;
+    I -.-> |needs and depends on| A;
+    I -.-> |needs and depends on| K;
+    L -.-> |needs and depends on| A;
+    L -.-> |needs and depends on| K;
+    O -.-> |needs and depends on| A;
+    O -.-> |needs and depends on| K;
+    S -.-> |needs and depends on| A;
+    S -.-> |needs and depends on| K;
+    S -.-> |needs and depends on| T;
     downtime_check --> |needs and depends on| A;
     db:* --> |needs| A;
     gitlab:setup --> |needs| A;
-    O -.-> |depends on| A;
-    O -.-> |depends on| B;
-    O -.-> |depends on| C;
     downtime_check --> |needs and depends on| A;
     end
 
 subgraph "`review-prepare` stage"
     E --> |needs| C;
-    X["schedule:review-build-cng<br/>(master schedule only)"] --> |needs| B;
+    X["schedule:review-build-cng<br/>(master schedule only)"] --> |needs| C;
     end
 
 subgraph "`review` stage"
@@ -190,7 +192,7 @@ subgraph "`qa` stage"
     Q --> |needs| F;
     R --> |needs| C;
     R --> |needs| F;
-    P --> |needs| B;
+    P --> |needs| C;
     P --> |needs| F;
     review-qa-smoke -.-> |needs and depends on| G;
     review-qa-all -.-> |needs and depends on| G;
@@ -209,7 +211,7 @@ subgraph "`post-test` stage"
     end
 
 subgraph "`pages` stage"
-    N -.-> |depends on| B;
+    N -.-> |depends on| C;
     N -.-> |depends on| H;
     N -.-> |depends on| M;
     end

spec/features/projects/environments/environment_spec.rb

@@ -175,7 +175,7 @@
                     #
                     # In EE we have to stub EE::Environment since it overwrites
                     # the "terminals" method.
-                    allow_any_instance_of(defined?(EE) ? EE::Environment : Environment)
+                    allow_any_instance_of(Gitlab.ee? ? EE::Environment : Environment)
                       .to receive(:terminals) { nil }
 
                     visit terminal_project_environment_path(project, environment)