Merge branch '351250-parallelize-ci-remote-includes' into 'master'

Add CI remote include parallelization

See merge request gitlab-org/gitlab!136715



Merged-by: Furkan Ayhan <furkanayhn@gmail.com>

config/feature_flags/development/ci_parallel_remote_includes.yml

+---
+name: ci_parallel_remote_includes
+introduced_by_url:
+rollout_issue_url:
+milestone: '16.7'
+type: development
+group: group::pipeline authoring
+default_enabled: false

lib/gitlab/ci/config/external/file/remote.rb

@@ -14,8 +14,25 @@ def initialize(params, context)
               super
             end
 
+            def preload_content
+              return super if ::Feature.disabled?(:ci_parallel_remote_includes, context.project)
+
+              # It starts fetching the remote content in a separate thread and returns a promise immediately.
+              @content_lazy_response = Gitlab::HTTP.get(location, async: true).execute
+            end
+
             def content
-              strong_memoize(:content) { fetch_remote_content }
+              strong_memoize(:content) do
+                next fetch_remote_content if ::Feature.disabled?(:ci_parallel_remote_includes, context.project)
+
+                # Normally, `preload_content` is called and @content_lazy_response is set before `content` is called.
+                # Here, we call `preload_content` again if it's not called yet somehow.
+                preload_content unless @content_lazy_response
+
+                fetch_with_error_handling do
+                  @content_lazy_response.value
+                end
+              end
             end
 
             def metadata
@@ -43,9 +60,15 @@ def validate_location!
             private
 
             def fetch_remote_content
+              fetch_with_error_handling do
+                Gitlab::HTTP.get(location)
+              end
+            end
+
+            def fetch_with_error_handling
               begin
                 response = context.logger.instrument(:config_file_fetch_remote_content) do
-                  Gitlab::HTTP.get(location)
+                  yield
                 end
               rescue SocketError
                 errors.push("Remote file `#{masked_location}` could not be fetched because of a socket error!")

lib/gitlab/ci/config/external/mapper/verifier.rb

@@ -25,7 +25,7 @@ def process_without_instrumentation(files)
                 file.preload_context if file.valid?
               end
 
-              # We do not combine the loops because we need to load the context of all files via `BatchLoader`.
+              # We do not combine the loops because we need to preload the context of all files via `BatchLoader`.
               files.each do |file| # rubocop:disable Style/CombinableLoops
                 verify_execution_time!
 
@@ -33,7 +33,8 @@ def process_without_instrumentation(files)
                 file.preload_content if file.valid?
               end
 
-              # We do not combine the loops because we need to load the content of all files via `BatchLoader`.
+              # We do not combine the loops because we need to preload the content of all files via `BatchLoader`
+              # or `Concurrent::Promise`.
               files.each do |file| # rubocop:disable Style/CombinableLoops
                 verify_execution_time!
 

spec/lib/gitlab/ci/config/external/file/remote_spec.rb

@@ -75,7 +75,9 @@
 
     context 'with a timeout' do
       before do
-        allow(Gitlab::HTTP).to receive(:get).and_raise(Timeout::Error)
+        allow_next_instance_of(HTTParty::Request) do |instance|
+          allow(instance).to receive(:perform).and_raise(Timeout::Error)
+        end
       end
 
       it { is_expected.to be_falsy }
@@ -94,7 +96,9 @@
     end
   end
 
-  describe "#content" do
+  # When the FF ci_parallel_remote_includes is removed,
+  # convert this `shared_context` to `describe` and remove `rubocop:disable`.
+  shared_context "#content" do # rubocop:disable RSpec/ContextWording
     context 'with a valid remote file' do
       before do
         stub_full_request(location).to_return(body: remote_file_content)
@@ -107,7 +111,9 @@
 
     context 'with a timeout' do
       before do
-        allow(Gitlab::HTTP).to receive(:get).and_raise(Timeout::Error)
+        allow_next_instance_of(HTTParty::Request) do |instance|
+          allow(instance).to receive(:perform).and_raise(Timeout::Error)
+        end
       end
 
       it 'is falsy' do
@@ -136,6 +142,16 @@
     end
   end
 
+  it_behaves_like "#content"
+
+  context 'when the FF ci_parallel_remote_includes is disabled' do
+    before do
+      stub_feature_flags(ci_parallel_remote_includes: false)
+    end
+
+    it_behaves_like "#content"
+  end
+
   describe "#error_message" do
     subject(:error_message) do
       Gitlab::Ci::Config::External::Mapper::Verifier.new(context).process([remote_file])