From d9b56bc13b94bdc69a14a2b4201e25e141a62ce4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Fri, 26 Oct 2018 22:02:09 +0200
Subject: [PATCH 01/24] Add parallel keyword to CI config

---
 lib/gitlab/ci/config/entry/job.rb           | 14 +++++----
 spec/lib/gitlab/ci/config/entry/job_spec.rb | 33 +++++++++++++++++++--
 2 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index f290ff3a565..368b5eb2a89 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -12,7 +12,7 @@ module Gitlab
           ALLOWED_KEYS = %i[tags script only except type image services
                             allow_failure type stage when start_in artifacts cache
                             dependencies before_script after_script variables
-                            environment coverage retry extends].freeze
+                            environment coverage retry parallel extends].freeze
 
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS
@@ -27,6 +27,8 @@ module Gitlab
               validates :retry, numericality: { only_integer: true,
                                                 greater_than_or_equal_to: 0,
                                                 less_than_or_equal_to: 2 }
+              validates :parallel, numericality: { only_integer: true,
+                                                   greater_than_or_equal_to: 1 }
               validates :when,
                 inclusion: { in: %w[on_success on_failure always manual delayed],
                              message: 'should be on_success, on_failure, ' \
@@ -77,17 +79,18 @@ module Gitlab
             description: 'Artifacts configuration for this job.'
 
           entry :environment, Entry::Environment,
-               description: 'Environment configuration for this job.'
+            description: 'Environment configuration for this job.'
 
           entry :coverage, Entry::Coverage,
-               description: 'Coverage configuration for this job.'
+            description: 'Coverage configuration for this job.'
 
           helpers :before_script, :script, :stage, :type, :after_script,
                   :cache, :image, :services, :only, :except, :variables,
-                  :artifacts, :commands, :environment, :coverage, :retry
+                  :artifacts, :commands, :environment, :coverage, :retry,
+                  :parallel
 
           attributes :script, :tags, :allow_failure, :when, :dependencies,
-                     :retry, :extends, :start_in
+                     :retry, :parallel, :extends, :start_in
 
           def compose!(deps = nil)
             super do
@@ -156,6 +159,7 @@ module Gitlab
               environment_name: environment_defined? ? environment_value[:name] : nil,
               coverage: coverage_defined? ? coverage_value : nil,
               retry: retry_defined? ? retry_value.to_i : nil,
+              parallel: parallel_defined? ? parallel_value.to_i : nil,
               artifacts: artifacts_value,
               after_script: after_script_value,
               ignore: ignored? }
diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb
index 1169938b80c..718098c364e 100644
--- a/spec/lib/gitlab/ci/config/entry/job_spec.rb
+++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb
@@ -1,5 +1,4 @@
-require 'fast_spec_helper'
-require_dependency 'active_model'
+require 'spec_helper'
 
 describe Gitlab::Ci::Config::Entry::Job do
   let(:entry) { described_class.new(config, name: :rspec) }
@@ -138,6 +137,36 @@ describe Gitlab::Ci::Config::Entry::Job do
         end
       end
 
+      context 'when parallel value is not correct' do
+        context 'when it is not a numeric value' do
+          let(:config) { { parallel: true } }
+
+          it 'returns error about invalid type' do
+            expect(entry).not_to be_valid
+            expect(entry.errors).to include 'job parallel is not a number'
+          end
+        end
+
+        context 'when it is lower than one' do
+          let(:config) { { parallel: 0 } }
+
+          it 'returns error about value too low' do
+            expect(entry).not_to be_valid
+            expect(entry.errors)
+              .to include 'job parallel must be greater than or equal to 1'
+          end
+        end
+
+        context 'when it is not an integer' do
+          let(:config) { { parallel: 1.5 } }
+
+          it 'returns error about wrong value' do
+            expect(entry).not_to be_valid
+            expect(entry.errors).to include 'job parallel must be an integer'
+          end
+        end
+      end
+
       context 'when delayed job' do
         context 'when start_in is specified' do
           let(:config) { { script: 'echo', when: 'delayed', start_in: '1 day' } }
-- 
2.24.1


From 44b740f99dfe6a4344c918fd4c94972aa6f9237e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Fri, 26 Oct 2018 23:23:58 +0200
Subject: [PATCH 02/24] Implement POC for job parallelization

---
 lib/gitlab/ci/pipeline/seed/build.rb | 24 +++++++++++++++++++++++-
 lib/gitlab/ci/yaml_processor.rb      |  1 +
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 6980b0b7aff..c302acdf073 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -24,6 +24,24 @@ module Gitlab
             end
           end
 
+          def parallelized?
+            @attributes[:options].include?(:parallel)
+          end
+
+          def parallelize_build
+            builds = []
+
+            total = @attributes[:options][:parallel]
+
+            total.times do |i|
+              build = ::Ci::Build.new(attributes.merge(options: { variables: { CI_NODE_INDEX: i + 1, CI_NODE_TOTAL: total } }))
+              build.name = build.name + "_#{i + 1}/#{total}"
+              builds << build
+            end
+
+            builds
+          end
+
           def attributes
             @attributes.merge(
               pipeline: @pipeline,
@@ -38,7 +56,11 @@ module Gitlab
 
           def to_resource
             strong_memoize(:resource) do
-              ::Ci::Build.new(attributes)
+              if parallelized?
+                parallelize_build
+              else
+                ::Ci::Build.new(attributes)
+              end
             end
           end
         end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index a427aa30683..612d733ad49 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -50,6 +50,7 @@ module Gitlab
             after_script: job[:after_script],
             environment: job[:environment],
             retry: job[:retry],
+            parallel: job[:parallel],
             start_in: job[:start_in]
           }.compact }
       end
-- 
2.24.1


From 72430483ded51e9a7d01ef70c3dce3dda174fac1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Sat, 27 Oct 2018 03:38:36 +0200
Subject: [PATCH 03/24] Refactor parallelization implementation

* Move the variables to ::Ci::Build#predefined_variables
* Tweak pipeline build seed implementation
---
 app/models/ci/build.rb               |  6 ++++++
 lib/gitlab/ci/pipeline/seed/build.rb | 22 +++++-----------------
 2 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index cdfe8175a42..21e2f289e1e 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -801,10 +801,16 @@ module Ci
         variables.append(key: "CI_COMMIT_TAG", value: ref) if tag?
         variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request
         variables.append(key: "CI_JOB_MANUAL", value: 'true') if action?
+        variables.append(key: "CI_NODE_INDEX", value: node_index.to_s) if self.options[:parallel]
+        variables.append(key: "CI_NODE_TOTAL", value: self.options.fetch(:parallel, 1).to_s)
         variables.concat(legacy_variables)
       end
     end
 
+    def node_index
+      name.match(%r{(\d+)/\d+$}).captures[0]
+    end
+
     def gitlab_version_info
       @gitlab_version_info ||= Gitlab::VersionInfo.parse(Gitlab::VERSION)
     end
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index c302acdf073..4b1116ced92 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -24,22 +24,14 @@ module Gitlab
             end
           end
 
-          def parallelized?
-            @attributes[:options].include?(:parallel)
+          def parallel?
+            !!@attributes.dig(:options, :parallel)
           end
 
           def parallelize_build
-            builds = []
-
             total = @attributes[:options][:parallel]
-
-            total.times do |i|
-              build = ::Ci::Build.new(attributes.merge(options: { variables: { CI_NODE_INDEX: i + 1, CI_NODE_TOTAL: total } }))
-              build.name = build.name + "_#{i + 1}/#{total}"
-              builds << build
-            end
-
-            builds
+            Array.new(total) { ::Ci::Build.new(attributes) }
+              .each_with_index { |build, idx| build.name = "#{build.name} #{idx + 1}/#{total}" }
           end
 
           def attributes
@@ -56,11 +48,7 @@ module Gitlab
 
           def to_resource
             strong_memoize(:resource) do
-              if parallelized?
-                parallelize_build
-              else
-                ::Ci::Build.new(attributes)
-              end
+              parallel? ? parallelize_build : ::Ci::Build.new(attributes)
             end
           end
         end
-- 
2.24.1


From c2d49565cf787c592c4f8bd9f24843babd2a6c9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Sat, 27 Oct 2018 18:04:09 +0200
Subject: [PATCH 04/24] Add build specs

---
 app/models/ci/build.rb       |  4 +--
 spec/models/ci/build_spec.rb | 48 ++++++++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 21e2f289e1e..86569f9a9c3 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -801,8 +801,8 @@ module Ci
         variables.append(key: "CI_COMMIT_TAG", value: ref) if tag?
         variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request
         variables.append(key: "CI_JOB_MANUAL", value: 'true') if action?
-        variables.append(key: "CI_NODE_INDEX", value: node_index.to_s) if self.options[:parallel]
-        variables.append(key: "CI_NODE_TOTAL", value: self.options.fetch(:parallel, 1).to_s)
+        variables.append(key: "CI_NODE_INDEX", value: node_index.to_s) if self.options&.include?(:parallel)
+        variables.append(key: "CI_NODE_TOTAL", value: (self.options&.dig(:parallel) || 1).to_s)
         variables.concat(legacy_variables)
       end
     end
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index a046541031e..41c3c37a7f2 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -1880,6 +1880,7 @@ describe Ci::Build do
         { key: 'CI_COMMIT_BEFORE_SHA', value: build.before_sha, public: true },
         { key: 'CI_COMMIT_REF_NAME', value: build.ref, public: true },
         { key: 'CI_COMMIT_REF_SLUG', value: build.ref_slug, public: true },
+        { key: 'CI_NODE_TOTAL', value: '1', public: true },
         { key: 'CI_BUILD_REF', value: build.sha, public: true },
         { key: 'CI_BUILD_BEFORE_SHA', value: build.before_sha, public: true },
         { key: 'CI_BUILD_REF_NAME', value: build.ref, public: true },
@@ -2341,6 +2342,28 @@ describe Ci::Build do
       end
     end
 
+    context 'when build is parallelized' do
+      let(:total) { 5 }
+      let(:index) { 3 }
+
+      before do
+        build.options[:parallel] = total
+        build.name = "#{build.name} #{index}/#{total}"
+      end
+
+      it 'includes CI_NODE_INDEX' do
+        is_expected.to include(
+          { key: 'CI_NODE_INDEX', value: index.to_s, public: true }
+        )
+      end
+
+      it 'includes correct CI_NODE_TOTAL' do
+        is_expected.to include(
+          { key: 'CI_NODE_TOTAL', value: total.to_s, public: true }
+        )
+      end
+    end
+
     describe 'variables ordering' do
       context 'when variables hierarchy is stubbed' do
         let(:build_pre_var) { { key: 'build', value: 'value', public: true } }
@@ -2447,6 +2470,31 @@ describe Ci::Build do
         end
       end
     end
+
+    describe '#node_index' do
+      subject { build.send(:node_index) }
+      let(:index) { 4 }
+
+      context 'when build has only one index' do
+        before do
+          build.name = "#{build.name} #{index}/5"
+        end
+
+        it 'returns the index' do
+          expect(subject).to eq(index.to_s)
+        end
+      end
+
+      context 'when build has more than one one index' do
+        before do
+          build.name = "test_build 1/3 #{index}/5"
+        end
+
+        it 'returns the last index' do
+          expect(subject).to eq(index.to_s)
+        end
+      end
+    end
   end
 
   describe '#scoped_variables' do
-- 
2.24.1


From a12da215c96fc15fb27753f18ab2106c6714bbb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Sat, 27 Oct 2018 18:19:58 +0200
Subject: [PATCH 05/24] Add YamlProcessor specs

---
 spec/lib/gitlab/ci/yaml_processor_spec.rb | 32 +++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb
index 85b23edce9f..544e421d571 100644
--- a/spec/lib/gitlab/ci/yaml_processor_spec.rb
+++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb
@@ -136,6 +136,19 @@ module Gitlab
             end
           end
         end
+
+        describe 'parallel entry' do
+          context 'when parallel is defined' do
+            let(:config) do
+              YAML.dump(rspec: { script: 'rspec',
+                                 parallel: 1 })
+            end
+
+            it 'has the attributes' do
+              expect(subject[:options][:parallel]).to eq 1
+            end
+          end
+        end
       end
 
       describe '#stages_attributes' do
@@ -645,6 +658,25 @@ module Gitlab
         end
       end
 
+      describe 'Parallel' do
+        context 'when job is parallelized' do
+          let(:parallel) { 5 }
+
+          let(:config) do
+            YAML.dump(rspec: { script: 'rspec',
+                               parallel: parallel })
+          end
+
+          it 'returns parallelized job' do
+            config_processor = Gitlab::Ci::YamlProcessor.new(config)
+            builds = config_processor.stage_builds_attributes("test")
+
+            expect(builds.size).to eq(1)
+            expect(builds.first[:options][:parallel]).to eq(parallel)
+          end
+        end
+      end
+
       describe 'cache' do
         context 'when cache definition has unknown keys' do
           it 'raises relevant validation error' do
-- 
2.24.1


From 03bc722ea1797a6b2b09f2897215477f5b269632 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Sat, 27 Oct 2018 18:52:47 +0200
Subject: [PATCH 06/24] Add Build seed specs

---
 .../lib/gitlab/ci/pipeline/seed/build_spec.rb | 59 ++++++++++++++++++-
 1 file changed, 56 insertions(+), 3 deletions(-)

diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb
index fffa727c2ed..d75f385f368 100644
--- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb
@@ -13,6 +13,46 @@ describe Gitlab::Ci::Pipeline::Seed::Build do
     described_class.new(pipeline, attributes)
   end
 
+  describe '#parallel?' do
+    context 'when build is not parallelized' do
+      it 'should be false' do
+        expect(subject.parallel?).to eq(false)
+      end
+    end
+
+    context 'when build is parallelized' do
+      before do
+        attributes[:options] = { parallel: 5 }
+      end
+
+      it 'should be true' do
+        expect(subject.parallel?).to eq(true)
+      end
+    end
+  end
+
+  describe '#parallelize_build' do
+    let(:total) { 5 }
+
+    before do
+      attributes[:options] = { parallel: total }
+    end
+
+    it 'returns duplicated builds' do
+      builds = subject.parallelize_build
+
+      expect(builds.size).to eq(total)
+    end
+
+    it 'returns builds with indexed names' do
+      builds = subject.parallelize_build
+
+      base_name = builds.first.name.split(' ')[0]
+      names = builds.map(&:name)
+      expect(names).to all(match(%r{^#{base_name} \d+/\d+$}))
+    end
+  end
+
   describe '#attributes' do
     it 'returns hash attributes of a build' do
       expect(subject.attributes).to be_a Hash
@@ -22,9 +62,22 @@ describe Gitlab::Ci::Pipeline::Seed::Build do
   end
 
   describe '#to_resource' do
-    it 'returns a valid build resource' do
-      expect(subject.to_resource).to be_a(::Ci::Build)
-      expect(subject.to_resource).to be_valid
+    context 'when build is not parallelized' do
+      it 'returns a valid build resource' do
+        expect(subject.to_resource).to be_a(::Ci::Build)
+        expect(subject.to_resource).to be_valid
+      end
+    end
+
+    context 'when build is parallelized' do
+      before do
+        attributes[:options] = { parallel: 5 }
+      end
+
+      it 'returns a group of valid build resources' do
+        expect(subject.to_resource).to all(be_a(::Ci::Build))
+        expect(subject.to_resource).to all(be_valid)
+      end
     end
 
     it 'memoizes a resource object' do
-- 
2.24.1


From ff1795713e7029e40b11888debe577294dd9754f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Sat, 27 Oct 2018 18:53:23 +0200
Subject: [PATCH 07/24] Add CHANGELOG entry

---
 changelogs/unreleased/21480-parallel-job-keyword-mvc.yml | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 changelogs/unreleased/21480-parallel-job-keyword-mvc.yml

diff --git a/changelogs/unreleased/21480-parallel-job-keyword-mvc.yml b/changelogs/unreleased/21480-parallel-job-keyword-mvc.yml
new file mode 100644
index 00000000000..7ac2410b18c
--- /dev/null
+++ b/changelogs/unreleased/21480-parallel-job-keyword-mvc.yml
@@ -0,0 +1,5 @@
+---
+title: Implement parallel job keyword.
+merge_request: 22631
+author:
+type: added
-- 
2.24.1


From 9ba72fe09aaf3bf903494f09fe8896012e962c98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Sat, 27 Oct 2018 19:06:46 +0200
Subject: [PATCH 08/24] Change minimum parallel value to 2

---
 lib/gitlab/ci/config/entry/job.rb           | 2 +-
 spec/lib/gitlab/ci/config/entry/job_spec.rb | 6 +++---
 spec/lib/gitlab/ci/yaml_processor_spec.rb   | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 368b5eb2a89..c73c88583b0 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -28,7 +28,7 @@ module Gitlab
                                                 greater_than_or_equal_to: 0,
                                                 less_than_or_equal_to: 2 }
               validates :parallel, numericality: { only_integer: true,
-                                                   greater_than_or_equal_to: 1 }
+                                                   greater_than_or_equal_to: 2 }
               validates :when,
                 inclusion: { in: %w[on_success on_failure always manual delayed],
                              message: 'should be on_success, on_failure, ' \
diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb
index 718098c364e..f1a2946acda 100644
--- a/spec/lib/gitlab/ci/config/entry/job_spec.rb
+++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb
@@ -147,13 +147,13 @@ describe Gitlab::Ci::Config::Entry::Job do
           end
         end
 
-        context 'when it is lower than one' do
-          let(:config) { { parallel: 0 } }
+        context 'when it is lower than two' do
+          let(:config) { { parallel: 1 } }
 
           it 'returns error about value too low' do
             expect(entry).not_to be_valid
             expect(entry.errors)
-              .to include 'job parallel must be greater than or equal to 1'
+              .to include 'job parallel must be greater than or equal to 2'
           end
         end
 
diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb
index 544e421d571..8289a6b000b 100644
--- a/spec/lib/gitlab/ci/yaml_processor_spec.rb
+++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb
@@ -141,11 +141,11 @@ module Gitlab
           context 'when parallel is defined' do
             let(:config) do
               YAML.dump(rspec: { script: 'rspec',
-                                 parallel: 1 })
+                                 parallel: 2 })
             end
 
             it 'has the attributes' do
-              expect(subject[:options][:parallel]).to eq 1
+              expect(subject[:options][:parallel]).to eq 2
             end
           end
         end
-- 
2.24.1


From 3c2acb3acfcc95eead03403f6593201391ead8d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Sat, 27 Oct 2018 19:12:48 +0200
Subject: [PATCH 09/24] Add documentation entries

---
 doc/ci/variables/README.md |  2 ++
 doc/ci/yaml/README.md      | 26 +++++++++++++++++++++++++-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md
index 2d23bf6d2fd..0ddbc828d1a 100644
--- a/doc/ci/variables/README.md
+++ b/doc/ci/variables/README.md
@@ -65,6 +65,8 @@ future GitLab releases.**
 | **CI_JOB_NAME**                 | 9.0    | 0.5    | The name of the job as defined in `.gitlab-ci.yml` |
 | **CI_JOB_STAGE**                | 9.0    | 0.5    | The name of the stage as defined in `.gitlab-ci.yml` |
 | **CI_JOB_TOKEN**                | 9.0    | 1.2    | Token used for authenticating with the [GitLab Container Registry][registry] and downloading [dependent repositories][dependent-repositories] |
+| **CI_NODE_INDEX**               | 11.5   | all    | The index of the job in the whole set. If the job is not paralellized, this is not set. |
+| **CI_NODE_TOTAL**               | 11.5   | all    | The total number of instances of this job running in parallel. If the job is not paralellized, this is set to 1. |
 | **CI_JOB_URL**                  | 11.1   | 0.5    | Job details URL |
 | **CI_REPOSITORY_URL**           | 9.0    | all    | The URL to clone the Git repository |
 | **CI_RUNNER_DESCRIPTION**       | 8.10   | 0.5    | The description of the runner as saved in GitLab |
diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md
index 981aa101dd3..4801d5d42e7 100644
--- a/doc/ci/yaml/README.md
+++ b/doc/ci/yaml/README.md
@@ -75,6 +75,7 @@ A job is defined by a list of parameters that define the job behavior.
 | environment   | no       | Defines a name of environment to which deployment is done by this job |
 | coverage      | no       | Define code coverage settings for a given job |
 | retry         | no       | Define how many times a job can be auto-retried in case of a failure |
+| parallel      | no       | Define how many duplicates of a job should be run in parallel |
 
 ### `extends`
 
@@ -1451,6 +1452,28 @@ test:
   retry: 2
 ```
 
+## `parallel`
+
+> [Introduced][ce-22631] in GitLab 11.5.
+
+`parallel` allows you to configure how many duplicates of a job will be run in
+parallel. This value has to be greater or equal to two (2).
+
+This creates N duplicates of the same job that run in parallel. They're named
+sequentially from `job_name 1/N` to `job_name N/N`.
+
+For every job `CI_NODE_INDEX` and `CI_NODE_TOTAL` environment variables are set.
+`CI_NODE_TOTAL` represents the total number of jobs while `CI_NODE_INDEX` is the
+index of the job in the whole set.
+
+A simple example:
+
+```yaml
+test:
+  script: rspec
+  parallel: 5
+```
+
 ## `include`
 
 > Introduced in [GitLab Edition Premium][ee] 10.5.
@@ -2031,7 +2054,8 @@ CI with various languages.
 [ce-7983]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7983
 [ce-7447]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7447
 [ce-12909]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/12909
+[ce-22631]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22631
 [schedules]: ../../user/project/pipelines/schedules.md
 [variables-expressions]: ../variables/README.md#variables-expressions
 [ee]: https://about.gitlab.com/gitlab-ee/
-[gitlab-versions]: https://about.gitlab.com/products/
\ No newline at end of file
+[gitlab-versions]: https://about.gitlab.com/products/
-- 
2.24.1


From 3cbea3b95d2d20de7b65ef0775959c1c153c4e15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Mon, 29 Oct 2018 16:58:35 +0100
Subject: [PATCH 10/24] Copyedit documentation updates

---
 doc/ci/variables/README.md |  4 ++--
 doc/ci/yaml/README.md      | 15 ++++++---------
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md
index 0ddbc828d1a..bdbcf8c9435 100644
--- a/doc/ci/variables/README.md
+++ b/doc/ci/variables/README.md
@@ -65,8 +65,8 @@ future GitLab releases.**
 | **CI_JOB_NAME**                 | 9.0    | 0.5    | The name of the job as defined in `.gitlab-ci.yml` |
 | **CI_JOB_STAGE**                | 9.0    | 0.5    | The name of the stage as defined in `.gitlab-ci.yml` |
 | **CI_JOB_TOKEN**                | 9.0    | 1.2    | Token used for authenticating with the [GitLab Container Registry][registry] and downloading [dependent repositories][dependent-repositories] |
-| **CI_NODE_INDEX**               | 11.5   | all    | The index of the job in the whole set. If the job is not paralellized, this is not set. |
-| **CI_NODE_TOTAL**               | 11.5   | all    | The total number of instances of this job running in parallel. If the job is not paralellized, this is set to 1. |
+| **CI_NODE_INDEX**               | 11.5   | all    | Index of the job in the job set. If the job is not parallelized, this variable is not set. |
+| **CI_NODE_TOTAL**               | 11.5   | all    | Total number of instances of this job running in parallel. If the job is not parallelized, this variable is set to `1`. |
 | **CI_JOB_URL**                  | 11.1   | 0.5    | Job details URL |
 | **CI_REPOSITORY_URL**           | 9.0    | all    | The URL to clone the Git repository |
 | **CI_RUNNER_DESCRIPTION**       | 8.10   | 0.5    | The description of the runner as saved in GitLab |
diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md
index 4801d5d42e7..b3a55e48f4e 100644
--- a/doc/ci/yaml/README.md
+++ b/doc/ci/yaml/README.md
@@ -75,7 +75,7 @@ A job is defined by a list of parameters that define the job behavior.
 | environment   | no       | Defines a name of environment to which deployment is done by this job |
 | coverage      | no       | Define code coverage settings for a given job |
 | retry         | no       | Define how many times a job can be auto-retried in case of a failure |
-| parallel      | no       | Define how many duplicates of a job should be run in parallel |
+| parallel      | no       | Defines how many instances of a job should be run in parallel |
 
 ### `extends`
 
@@ -1454,17 +1454,15 @@ test:
 
 ## `parallel`
 
-> [Introduced][ce-22631] in GitLab 11.5.
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22631) in GitLab 11.5.
 
-`parallel` allows you to configure how many duplicates of a job will be run in
-parallel. This value has to be greater or equal to two (2).
+`parallel` allows you to configure how many instances of a job to run in
+parallel. This value has to be greater than or equal to two (2).
 
-This creates N duplicates of the same job that run in parallel. They're named
+This creates N instances of the same job that run in parallel. They're named
 sequentially from `job_name 1/N` to `job_name N/N`.
 
-For every job `CI_NODE_INDEX` and `CI_NODE_TOTAL` environment variables are set.
-`CI_NODE_TOTAL` represents the total number of jobs while `CI_NODE_INDEX` is the
-index of the job in the whole set.
+For every job, `CI_NODE_INDEX` and `CI_NODE_TOTAL` [environment variables](../variables/README.html#predefined-variables-environment-variables) are set.
 
 A simple example:
 
@@ -2054,7 +2052,6 @@ CI with various languages.
 [ce-7983]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7983
 [ce-7447]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7447
 [ce-12909]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/12909
-[ce-22631]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22631
 [schedules]: ../../user/project/pipelines/schedules.md
 [variables-expressions]: ../variables/README.md#variables-expressions
 [ee]: https://about.gitlab.com/gitlab-ee/
-- 
2.24.1


From 94923328fdf2904e5a31ad8b9c40adcf15428bb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Tue, 30 Oct 2018 15:04:25 +0100
Subject: [PATCH 11/24] Revert Seed based parallelization implementation

Revert "Add Build seed specs"

This reverts commit 03bc722ea1797a6b2b09f2897215477f5b269632.

Revert "Add build specs"

This reverts commit c2d49565cf787c592c4f8bd9f24843babd2a6c9a.

Revert "Refactor parallelization implementation"

This reverts commit 72430483ded51e9a7d01ef70c3dce3dda174fac1.

Revert "Implement POC for job parallelization"

This reverts commit 44b740f99dfe6a4344c918fd4c94972aa6f9237e.
---
 app/models/ci/build.rb                        |  6 --
 lib/gitlab/ci/pipeline/seed/build.rb          | 12 +---
 lib/gitlab/ci/yaml_processor.rb               |  1 -
 .../lib/gitlab/ci/pipeline/seed/build_spec.rb | 59 +------------------
 spec/models/ci/build_spec.rb                  | 48 ---------------
 5 files changed, 4 insertions(+), 122 deletions(-)

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 86569f9a9c3..cdfe8175a42 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -801,16 +801,10 @@ module Ci
         variables.append(key: "CI_COMMIT_TAG", value: ref) if tag?
         variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request
         variables.append(key: "CI_JOB_MANUAL", value: 'true') if action?
-        variables.append(key: "CI_NODE_INDEX", value: node_index.to_s) if self.options&.include?(:parallel)
-        variables.append(key: "CI_NODE_TOTAL", value: (self.options&.dig(:parallel) || 1).to_s)
         variables.concat(legacy_variables)
       end
     end
 
-    def node_index
-      name.match(%r{(\d+)/\d+$}).captures[0]
-    end
-
     def gitlab_version_info
       @gitlab_version_info ||= Gitlab::VersionInfo.parse(Gitlab::VERSION)
     end
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 4b1116ced92..6980b0b7aff 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -24,16 +24,6 @@ module Gitlab
             end
           end
 
-          def parallel?
-            !!@attributes.dig(:options, :parallel)
-          end
-
-          def parallelize_build
-            total = @attributes[:options][:parallel]
-            Array.new(total) { ::Ci::Build.new(attributes) }
-              .each_with_index { |build, idx| build.name = "#{build.name} #{idx + 1}/#{total}" }
-          end
-
           def attributes
             @attributes.merge(
               pipeline: @pipeline,
@@ -48,7 +38,7 @@ module Gitlab
 
           def to_resource
             strong_memoize(:resource) do
-              parallel? ? parallelize_build : ::Ci::Build.new(attributes)
+              ::Ci::Build.new(attributes)
             end
           end
         end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 612d733ad49..a427aa30683 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -50,7 +50,6 @@ module Gitlab
             after_script: job[:after_script],
             environment: job[:environment],
             retry: job[:retry],
-            parallel: job[:parallel],
             start_in: job[:start_in]
           }.compact }
       end
diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb
index d75f385f368..fffa727c2ed 100644
--- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb
@@ -13,46 +13,6 @@ describe Gitlab::Ci::Pipeline::Seed::Build do
     described_class.new(pipeline, attributes)
   end
 
-  describe '#parallel?' do
-    context 'when build is not parallelized' do
-      it 'should be false' do
-        expect(subject.parallel?).to eq(false)
-      end
-    end
-
-    context 'when build is parallelized' do
-      before do
-        attributes[:options] = { parallel: 5 }
-      end
-
-      it 'should be true' do
-        expect(subject.parallel?).to eq(true)
-      end
-    end
-  end
-
-  describe '#parallelize_build' do
-    let(:total) { 5 }
-
-    before do
-      attributes[:options] = { parallel: total }
-    end
-
-    it 'returns duplicated builds' do
-      builds = subject.parallelize_build
-
-      expect(builds.size).to eq(total)
-    end
-
-    it 'returns builds with indexed names' do
-      builds = subject.parallelize_build
-
-      base_name = builds.first.name.split(' ')[0]
-      names = builds.map(&:name)
-      expect(names).to all(match(%r{^#{base_name} \d+/\d+$}))
-    end
-  end
-
   describe '#attributes' do
     it 'returns hash attributes of a build' do
       expect(subject.attributes).to be_a Hash
@@ -62,22 +22,9 @@ describe Gitlab::Ci::Pipeline::Seed::Build do
   end
 
   describe '#to_resource' do
-    context 'when build is not parallelized' do
-      it 'returns a valid build resource' do
-        expect(subject.to_resource).to be_a(::Ci::Build)
-        expect(subject.to_resource).to be_valid
-      end
-    end
-
-    context 'when build is parallelized' do
-      before do
-        attributes[:options] = { parallel: 5 }
-      end
-
-      it 'returns a group of valid build resources' do
-        expect(subject.to_resource).to all(be_a(::Ci::Build))
-        expect(subject.to_resource).to all(be_valid)
-      end
+    it 'returns a valid build resource' do
+      expect(subject.to_resource).to be_a(::Ci::Build)
+      expect(subject.to_resource).to be_valid
     end
 
     it 'memoizes a resource object' do
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index 41c3c37a7f2..a046541031e 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -1880,7 +1880,6 @@ describe Ci::Build do
         { key: 'CI_COMMIT_BEFORE_SHA', value: build.before_sha, public: true },
         { key: 'CI_COMMIT_REF_NAME', value: build.ref, public: true },
         { key: 'CI_COMMIT_REF_SLUG', value: build.ref_slug, public: true },
-        { key: 'CI_NODE_TOTAL', value: '1', public: true },
         { key: 'CI_BUILD_REF', value: build.sha, public: true },
         { key: 'CI_BUILD_BEFORE_SHA', value: build.before_sha, public: true },
         { key: 'CI_BUILD_REF_NAME', value: build.ref, public: true },
@@ -2342,28 +2341,6 @@ describe Ci::Build do
       end
     end
 
-    context 'when build is parallelized' do
-      let(:total) { 5 }
-      let(:index) { 3 }
-
-      before do
-        build.options[:parallel] = total
-        build.name = "#{build.name} #{index}/#{total}"
-      end
-
-      it 'includes CI_NODE_INDEX' do
-        is_expected.to include(
-          { key: 'CI_NODE_INDEX', value: index.to_s, public: true }
-        )
-      end
-
-      it 'includes correct CI_NODE_TOTAL' do
-        is_expected.to include(
-          { key: 'CI_NODE_TOTAL', value: total.to_s, public: true }
-        )
-      end
-    end
-
     describe 'variables ordering' do
       context 'when variables hierarchy is stubbed' do
         let(:build_pre_var) { { key: 'build', value: 'value', public: true } }
@@ -2470,31 +2447,6 @@ describe Ci::Build do
         end
       end
     end
-
-    describe '#node_index' do
-      subject { build.send(:node_index) }
-      let(:index) { 4 }
-
-      context 'when build has only one index' do
-        before do
-          build.name = "#{build.name} #{index}/5"
-        end
-
-        it 'returns the index' do
-          expect(subject).to eq(index.to_s)
-        end
-      end
-
-      context 'when build has more than one one index' do
-        before do
-          build.name = "test_build 1/3 #{index}/5"
-        end
-
-        it 'returns the last index' do
-          expect(subject).to eq(index.to_s)
-        end
-      end
-    end
   end
 
   describe '#scoped_variables' do
-- 
2.24.1


From 8f8a89f98edc4cb901f4107cbf38288576849d9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Wed, 31 Oct 2018 15:27:39 +0100
Subject: [PATCH 12/24] Implement POC config based parallelization

---
 app/models/ci/build.rb             |  6 ++++
 lib/gitlab/ci/config/entry/jobs.rb | 10 +++++++
 lib/gitlab/ci/yaml_processor.rb    |  1 +
 spec/models/ci/build_spec.rb       | 48 ++++++++++++++++++++++++++++++
 4 files changed, 65 insertions(+)

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index cdfe8175a42..86569f9a9c3 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -801,10 +801,16 @@ module Ci
         variables.append(key: "CI_COMMIT_TAG", value: ref) if tag?
         variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request
         variables.append(key: "CI_JOB_MANUAL", value: 'true') if action?
+        variables.append(key: "CI_NODE_INDEX", value: node_index.to_s) if self.options&.include?(:parallel)
+        variables.append(key: "CI_NODE_TOTAL", value: (self.options&.dig(:parallel) || 1).to_s)
         variables.concat(legacy_variables)
       end
     end
 
+    def node_index
+      name.match(%r{(\d+)/\d+$}).captures[0]
+    end
+
     def gitlab_version_info
       @gitlab_version_info ||= Gitlab::VersionInfo.parse(Gitlab::VERSION)
     end
diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb
index 96b6f2e5d6c..fc4b8637ee2 100644
--- a/lib/gitlab/ci/config/entry/jobs.rb
+++ b/lib/gitlab/ci/config/entry/jobs.rb
@@ -29,6 +29,16 @@ module Gitlab
           # rubocop: disable CodeReuse/ActiveRecord
           def compose!(deps = nil)
             super do
+              @config = @config.map do |name, config|
+                total = config[:parallel]
+                if total
+                  Array.new(total) { { name => config } }
+                    .each_with_index { |build, idx| build["#{name} #{idx + 1}/#{total}".to_sym] = build.delete(name) }
+                else
+                  { name => config }
+                end
+              end.flatten.reduce(:merge)
+
               @config.each do |name, config|
                 node = hidden?(name) ? Entry::Hidden : Entry::Job
 
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index a427aa30683..612d733ad49 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -50,6 +50,7 @@ module Gitlab
             after_script: job[:after_script],
             environment: job[:environment],
             retry: job[:retry],
+            parallel: job[:parallel],
             start_in: job[:start_in]
           }.compact }
       end
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index a046541031e..41c3c37a7f2 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -1880,6 +1880,7 @@ describe Ci::Build do
         { key: 'CI_COMMIT_BEFORE_SHA', value: build.before_sha, public: true },
         { key: 'CI_COMMIT_REF_NAME', value: build.ref, public: true },
         { key: 'CI_COMMIT_REF_SLUG', value: build.ref_slug, public: true },
+        { key: 'CI_NODE_TOTAL', value: '1', public: true },
         { key: 'CI_BUILD_REF', value: build.sha, public: true },
         { key: 'CI_BUILD_BEFORE_SHA', value: build.before_sha, public: true },
         { key: 'CI_BUILD_REF_NAME', value: build.ref, public: true },
@@ -2341,6 +2342,28 @@ describe Ci::Build do
       end
     end
 
+    context 'when build is parallelized' do
+      let(:total) { 5 }
+      let(:index) { 3 }
+
+      before do
+        build.options[:parallel] = total
+        build.name = "#{build.name} #{index}/#{total}"
+      end
+
+      it 'includes CI_NODE_INDEX' do
+        is_expected.to include(
+          { key: 'CI_NODE_INDEX', value: index.to_s, public: true }
+        )
+      end
+
+      it 'includes correct CI_NODE_TOTAL' do
+        is_expected.to include(
+          { key: 'CI_NODE_TOTAL', value: total.to_s, public: true }
+        )
+      end
+    end
+
     describe 'variables ordering' do
       context 'when variables hierarchy is stubbed' do
         let(:build_pre_var) { { key: 'build', value: 'value', public: true } }
@@ -2447,6 +2470,31 @@ describe Ci::Build do
         end
       end
     end
+
+    describe '#node_index' do
+      subject { build.send(:node_index) }
+      let(:index) { 4 }
+
+      context 'when build has only one index' do
+        before do
+          build.name = "#{build.name} #{index}/5"
+        end
+
+        it 'returns the index' do
+          expect(subject).to eq(index.to_s)
+        end
+      end
+
+      context 'when build has more than one one index' do
+        before do
+          build.name = "test_build 1/3 #{index}/5"
+        end
+
+        it 'returns the last index' do
+          expect(subject).to eq(index.to_s)
+        end
+      end
+    end
   end
 
   describe '#scoped_variables' do
-- 
2.24.1


From 3c8dd4ccbfcbb20de88005d43c93cd476de6d5c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Wed, 31 Oct 2018 16:40:52 +0100
Subject: [PATCH 13/24] Move parallelization to Ci::Config::Normalizer

---
 lib/gitlab/ci/config/entry/jobs.rb | 10 +-----
 lib/gitlab/ci/config/normalizer.rb | 51 ++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 9 deletions(-)
 create mode 100644 lib/gitlab/ci/config/normalizer.rb

diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb
index fc4b8637ee2..1eab5927b56 100644
--- a/lib/gitlab/ci/config/entry/jobs.rb
+++ b/lib/gitlab/ci/config/entry/jobs.rb
@@ -29,15 +29,7 @@ module Gitlab
           # rubocop: disable CodeReuse/ActiveRecord
           def compose!(deps = nil)
             super do
-              @config = @config.map do |name, config|
-                total = config[:parallel]
-                if total
-                  Array.new(total) { { name => config } }
-                    .each_with_index { |build, idx| build["#{name} #{idx + 1}/#{total}".to_sym] = build.delete(name) }
-                else
-                  { name => config }
-                end
-              end.flatten.reduce(:merge)
+              @config = Ci::Config::Normalizer.normalize_jobs(@config)
 
               @config.each do |name, config|
                 node = hidden?(name) ? Entry::Hidden : Entry::Job
diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb
new file mode 100644
index 00000000000..aa69dd44c5f
--- /dev/null
+++ b/lib/gitlab/ci/config/normalizer.rb
@@ -0,0 +1,51 @@
+module Gitlab
+  module Ci
+    class Config
+      class Normalizer
+        class << self
+          def normalize_jobs(jobs_config)
+            parallelized_jobs = {}
+
+            parallelized_config = jobs_config.map do |name, config|
+              if config&.[](:parallel)
+                total = config[:parallel]
+                names = parallelize_job_names(name, total)
+                parallelized_jobs[name] = names
+                Hash[names.collect { |job_name| [job_name.to_sym, config] }]
+              else
+                { name => config }
+              end
+            end.reduce(:merge)
+
+            parallelized_config.each do |name, config|
+              next unless config[:dependencies]
+
+              deps = config[:dependencies].map do |dep|
+                if parallelized_jobs.keys.include?(dep.to_sym)
+                  config[:dependencies].delete(dep)
+                  parallelized_jobs[dep.to_sym]
+                else
+                  dep
+                end
+              end.flatten
+
+              config[:dependencies] = deps
+            end
+          end
+
+          private
+
+          def parallelize_job_names(name, total)
+            jobs = []
+
+            total.times do |idx|
+              jobs << "#{name} #{idx + 1}/#{total}"
+            end
+
+            jobs
+          end
+        end
+      end
+    end
+  end
+end
-- 
2.24.1


From 56a08d233fe18892ae7a7c723662007e260a2c2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Thu, 1 Nov 2018 01:05:42 +0100
Subject: [PATCH 14/24] Parallelize jobs in Gitlab::Ci::YamlProcessor

---
 lib/gitlab/ci/config/entry/jobs.rb        |  2 --
 lib/gitlab/ci/config/normalizer.rb        |  4 ++--
 lib/gitlab/ci/yaml_processor.rb           |  2 +-
 spec/lib/gitlab/ci/yaml_processor_spec.rb | 21 ++++-----------------
 4 files changed, 7 insertions(+), 22 deletions(-)

diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb
index 1eab5927b56..96b6f2e5d6c 100644
--- a/lib/gitlab/ci/config/entry/jobs.rb
+++ b/lib/gitlab/ci/config/entry/jobs.rb
@@ -29,8 +29,6 @@ module Gitlab
           # rubocop: disable CodeReuse/ActiveRecord
           def compose!(deps = nil)
             super do
-              @config = Ci::Config::Normalizer.normalize_jobs(@config)
-
               @config.each do |name, config|
                 node = hidden?(name) ? Entry::Hidden : Entry::Job
 
diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb
index aa69dd44c5f..55efc7439c1 100644
--- a/lib/gitlab/ci/config/normalizer.rb
+++ b/lib/gitlab/ci/config/normalizer.rb
@@ -7,11 +7,11 @@ module Gitlab
             parallelized_jobs = {}
 
             parallelized_config = jobs_config.map do |name, config|
-              if config&.[](:parallel)
+              if config[:parallel]
                 total = config[:parallel]
                 names = parallelize_job_names(name, total)
                 parallelized_jobs[name] = names
-                Hash[names.collect { |job_name| [job_name.to_sym, config] }]
+                Hash[names.collect { |job_name| [job_name.to_sym, config.merge(name: job_name)] }]
               else
                 { name => config }
               end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 612d733ad49..4977fa2ae24 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -103,7 +103,7 @@ module Gitlab
         ##
         # Jobs
         #
-        @jobs = @ci_config.jobs
+        @jobs = Ci::Config::Normalizer.normalize_jobs(@ci_config.jobs)
 
         @jobs.each do |name, job|
           # logical validation for job
diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb
index 8289a6b000b..4882087fcd3 100644
--- a/spec/lib/gitlab/ci/yaml_processor_spec.rb
+++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb
@@ -136,19 +136,6 @@ module Gitlab
             end
           end
         end
-
-        describe 'parallel entry' do
-          context 'when parallel is defined' do
-            let(:config) do
-              YAML.dump(rspec: { script: 'rspec',
-                                 parallel: 2 })
-            end
-
-            it 'has the attributes' do
-              expect(subject[:options][:parallel]).to eq 2
-            end
-          end
-        end
       end
 
       describe '#stages_attributes' do
@@ -667,12 +654,12 @@ module Gitlab
                                parallel: parallel })
           end
 
-          it 'returns parallelized job' do
+          it 'returns parallelized jobs' do
             config_processor = Gitlab::Ci::YamlProcessor.new(config)
-            builds = config_processor.stage_builds_attributes("test")
+            builds = config_processor.stage_builds_attributes('test')
 
-            expect(builds.size).to eq(1)
-            expect(builds.first[:options][:parallel]).to eq(parallel)
+            expect(builds.size).to eq(5)
+            expect(builds.map { |build| build[:options] }).to all(include(parallel: parallel))
           end
         end
       end
-- 
2.24.1


From 77715e47d633f3db3b6b58c1d67b4ddbe3668177 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Thu, 1 Nov 2018 15:24:29 +0100
Subject: [PATCH 15/24] Add Normalizer specs

---
 spec/lib/gitlab/ci/config/normalizer_spec.rb | 36 ++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 spec/lib/gitlab/ci/config/normalizer_spec.rb

diff --git a/spec/lib/gitlab/ci/config/normalizer_spec.rb b/spec/lib/gitlab/ci/config/normalizer_spec.rb
new file mode 100644
index 00000000000..5890d4f193c
--- /dev/null
+++ b/spec/lib/gitlab/ci/config/normalizer_spec.rb
@@ -0,0 +1,36 @@
+require 'fast_spec_helper'
+
+describe Gitlab::Ci::Config::Normalizer do
+  let(:job_name) { :rspec }
+  let(:job_config) { { script: 'rspec', parallel: 5 } }
+  let(:config) { { job_name => job_config } }
+
+  describe '.normalize_jobs' do
+    subject { described_class.normalize_jobs(config) }
+
+    it 'does not have original job' do
+      is_expected.not_to include(job_name)
+    end
+
+    it 'has parallelized jobs' do
+      job_names = described_class.send(:parallelize_job_names, job_name, 5).map(&:to_sym)
+
+      is_expected.to include(*job_names)
+    end
+
+    it 'parallelizes jobs with original config' do
+      original_config = config[job_name].except(:name)
+      configs = subject.values.map { |config| config.except(:name) }
+
+      expect(configs).to all(eq(original_config))
+    end
+  end
+
+  describe '.parallelize_job_names' do
+    subject { described_class.send(:parallelize_job_names, job_name, 5) }
+
+    it 'returns parallelized names' do
+      is_expected.to all(match(%r{#{job_name} \d+/\d+}))
+    end
+  end
+end
-- 
2.24.1


From 73e17446ef400a8f2a4e629c79749d7feb9866f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Thu, 1 Nov 2018 16:17:08 +0100
Subject: [PATCH 16/24] Move parallelized node index to job options

---
 app/models/ci/build.rb                       |  6 +----
 lib/gitlab/ci/config/normalizer.rb           |  6 ++---
 lib/gitlab/ci/yaml_processor.rb              |  1 +
 spec/lib/gitlab/ci/config/normalizer_spec.rb | 12 ++++++---
 spec/lib/gitlab/ci/yaml_processor_spec.rb    | 10 +++++++-
 spec/models/ci/build_spec.rb                 | 26 +-------------------
 6 files changed, 23 insertions(+), 38 deletions(-)

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 86569f9a9c3..6f2b3543493 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -801,16 +801,12 @@ module Ci
         variables.append(key: "CI_COMMIT_TAG", value: ref) if tag?
         variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request
         variables.append(key: "CI_JOB_MANUAL", value: 'true') if action?
-        variables.append(key: "CI_NODE_INDEX", value: node_index.to_s) if self.options&.include?(:parallel)
+        variables.append(key: "CI_NODE_INDEX", value: self.options[:instance].to_s) if self.options&.include?(:instance)
         variables.append(key: "CI_NODE_TOTAL", value: (self.options&.dig(:parallel) || 1).to_s)
         variables.concat(legacy_variables)
       end
     end
 
-    def node_index
-      name.match(%r{(\d+)/\d+$}).captures[0]
-    end
-
     def gitlab_version_info
       @gitlab_version_info ||= Gitlab::VersionInfo.parse(Gitlab::VERSION)
     end
diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb
index 55efc7439c1..fc1efb67560 100644
--- a/lib/gitlab/ci/config/normalizer.rb
+++ b/lib/gitlab/ci/config/normalizer.rb
@@ -10,8 +10,8 @@ module Gitlab
               if config[:parallel]
                 total = config[:parallel]
                 names = parallelize_job_names(name, total)
-                parallelized_jobs[name] = names
-                Hash[names.collect { |job_name| [job_name.to_sym, config.merge(name: job_name)] }]
+                parallelized_jobs[name] = names.map(&:first)
+                Hash[names.collect { |job_name, index| [job_name.to_sym, config.merge(name: job_name, instance: index)] }]
               else
                 { name => config }
               end
@@ -39,7 +39,7 @@ module Gitlab
             jobs = []
 
             total.times do |idx|
-              jobs << "#{name} #{idx + 1}/#{total}"
+              jobs << ["#{name} #{idx + 1}/#{total}", idx + 1]
             end
 
             jobs
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 4977fa2ae24..63b55c57913 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -51,6 +51,7 @@ module Gitlab
             environment: job[:environment],
             retry: job[:retry],
             parallel: job[:parallel],
+            instance: job[:instance],
             start_in: job[:start_in]
           }.compact }
       end
diff --git a/spec/lib/gitlab/ci/config/normalizer_spec.rb b/spec/lib/gitlab/ci/config/normalizer_spec.rb
index 5890d4f193c..f1096cd3931 100644
--- a/spec/lib/gitlab/ci/config/normalizer_spec.rb
+++ b/spec/lib/gitlab/ci/config/normalizer_spec.rb
@@ -2,7 +2,7 @@ require 'fast_spec_helper'
 
 describe Gitlab::Ci::Config::Normalizer do
   let(:job_name) { :rspec }
-  let(:job_config) { { script: 'rspec', parallel: 5 } }
+  let(:job_config) { { script: 'rspec', parallel: 5, name: 'rspec' } }
   let(:config) { { job_name => job_config } }
 
   describe '.normalize_jobs' do
@@ -13,14 +13,18 @@ describe Gitlab::Ci::Config::Normalizer do
     end
 
     it 'has parallelized jobs' do
-      job_names = described_class.send(:parallelize_job_names, job_name, 5).map(&:to_sym)
+      job_names = described_class.send(:parallelize_job_names, job_name, 5).map { |job_name, index| job_name.to_sym }
 
       is_expected.to include(*job_names)
     end
 
+    it 'sets job instance in options' do
+      expect(subject.values).to all(include(:instance))
+    end
+
     it 'parallelizes jobs with original config' do
       original_config = config[job_name].except(:name)
-      configs = subject.values.map { |config| config.except(:name) }
+      configs = subject.values.map { |config| config.except(:name, :instance) }
 
       expect(configs).to all(eq(original_config))
     end
@@ -30,7 +34,7 @@ describe Gitlab::Ci::Config::Normalizer do
     subject { described_class.send(:parallelize_job_names, job_name, 5) }
 
     it 'returns parallelized names' do
-      is_expected.to all(match(%r{#{job_name} \d+/\d+}))
+      expect(subject.map(&:first)).to all(match(%r{#{job_name} \d+/\d+}))
     end
   end
 end
diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb
index 4882087fcd3..dcfd54107a3 100644
--- a/spec/lib/gitlab/ci/yaml_processor_spec.rb
+++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb
@@ -657,9 +657,17 @@ module Gitlab
           it 'returns parallelized jobs' do
             config_processor = Gitlab::Ci::YamlProcessor.new(config)
             builds = config_processor.stage_builds_attributes('test')
+            build_options = builds.map { |build| build[:options] }
 
             expect(builds.size).to eq(5)
-            expect(builds.map { |build| build[:options] }).to all(include(parallel: parallel))
+            expect(build_options).to all(include(:instance, parallel: parallel))
+          end
+
+          it 'does not have the original job' do
+            config_processor = Gitlab::Ci::YamlProcessor.new(config)
+            builds = config_processor.stage_builds_attributes('test')
+
+            expect(builds).not_to include(:rspec)
           end
         end
       end
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index 41c3c37a7f2..957219b3985 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -2348,6 +2348,7 @@ describe Ci::Build do
 
       before do
         build.options[:parallel] = total
+        build.options[:instance] = index
         build.name = "#{build.name} #{index}/#{total}"
       end
 
@@ -2470,31 +2471,6 @@ describe Ci::Build do
         end
       end
     end
-
-    describe '#node_index' do
-      subject { build.send(:node_index) }
-      let(:index) { 4 }
-
-      context 'when build has only one index' do
-        before do
-          build.name = "#{build.name} #{index}/5"
-        end
-
-        it 'returns the index' do
-          expect(subject).to eq(index.to_s)
-        end
-      end
-
-      context 'when build has more than one one index' do
-        before do
-          build.name = "test_build 1/3 #{index}/5"
-        end
-
-        it 'returns the last index' do
-          expect(subject).to eq(index.to_s)
-        end
-      end
-    end
   end
 
   describe '#scoped_variables' do
-- 
2.24.1


From 8a6a312db8148433fca78da0b5c9db8f966e16fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Fri, 2 Nov 2018 14:20:27 +0100
Subject: [PATCH 17/24] Make Rubocop and Danger happy

---
 app/models/ci/build.rb                       | 2 +-
 lib/gitlab/ci/config/normalizer.rb           | 2 ++
 spec/lib/gitlab/ci/config/normalizer_spec.rb | 2 ++
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 6f2b3543493..d3abb28d704 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -781,7 +781,7 @@ module Ci
       end
     end
 
-    def predefined_variables
+    def predefined_variables # rubocop:disable Metrics/AbcSize
       Gitlab::Ci::Variables::Collection.new.tap do |variables|
         variables.append(key: 'CI', value: 'true')
         variables.append(key: 'GITLAB_CI', value: 'true')
diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb
index fc1efb67560..2dc408ac392 100644
--- a/lib/gitlab/ci/config/normalizer.rb
+++ b/lib/gitlab/ci/config/normalizer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
diff --git a/spec/lib/gitlab/ci/config/normalizer_spec.rb b/spec/lib/gitlab/ci/config/normalizer_spec.rb
index f1096cd3931..48b8217f77c 100644
--- a/spec/lib/gitlab/ci/config/normalizer_spec.rb
+++ b/spec/lib/gitlab/ci/config/normalizer_spec.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'fast_spec_helper'
 
 describe Gitlab::Ci::Config::Normalizer do
-- 
2.24.1


From a8da5238474f8f7ad4dab5461c5c5605894535db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Fri, 2 Nov 2018 15:48:25 +0100
Subject: [PATCH 18/24] Refactor Gitlab::Ci::Config::Normalizer

---
 lib/gitlab/ci/config/normalizer.rb | 44 ++++++++++++++++++------------
 1 file changed, 27 insertions(+), 17 deletions(-)

diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb
index 2dc408ac392..43331b28e2b 100644
--- a/lib/gitlab/ci/config/normalizer.rb
+++ b/lib/gitlab/ci/config/normalizer.rb
@@ -6,6 +6,13 @@ module Gitlab
       class Normalizer
         class << self
           def normalize_jobs(jobs_config)
+            parallelized_config, parallelized_jobs = parallelize_jobs(jobs_config)
+            parallelize_dependencies(parallelized_config, parallelized_jobs)
+          end
+
+          private
+
+          def parallelize_jobs(jobs_config)
             parallelized_jobs = {}
 
             parallelized_config = jobs_config.map do |name, config|
@@ -19,29 +26,32 @@ module Gitlab
               end
             end.reduce(:merge)
 
-            parallelized_config.each do |name, config|
-              next unless config[:dependencies]
-
-              deps = config[:dependencies].map do |dep|
-                if parallelized_jobs.keys.include?(dep.to_sym)
-                  config[:dependencies].delete(dep)
-                  parallelized_jobs[dep.to_sym]
-                else
-                  dep
-                end
-              end.flatten
-
-              config[:dependencies] = deps
-            end
+            [parallelized_config, parallelized_jobs]
           end
 
-          private
+          def parallelize_dependencies(jobs_config, parallelized_jobs)
+            jobs_config.map do |name, config|
+              if config[:dependencies]
+                deps = config[:dependencies].map do |dep|
+                  if parallelized_jobs.keys.include?(dep.to_sym)
+                    parallelized_jobs[dep.to_sym]
+                  else
+                    dep
+                  end
+                end.flatten
+
+                { name => config.merge(dependencies: deps) }
+              else
+                { name => config }
+              end
+            end.reduce(:merge)
+          end
 
           def parallelize_job_names(name, total)
             jobs = []
 
-            total.times do |idx|
-              jobs << ["#{name} #{idx + 1}/#{total}", idx + 1]
+            1.upto(total) do |idx|
+              jobs << ["#{name} #{idx}/#{total}", idx]
             end
 
             jobs
-- 
2.24.1


From 5fd8933d97b694fb8ae6a57acaae8fb4e7922e73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Mon, 5 Nov 2018 14:36:52 +0100
Subject: [PATCH 19/24] Use instance based approach for Normalizer

---
 lib/gitlab/ci/config/normalizer.rb           | 96 ++++++++++----------
 lib/gitlab/ci/yaml_processor.rb              |  2 +-
 spec/lib/gitlab/ci/config/normalizer_spec.rb |  2 +-
 3 files changed, 48 insertions(+), 52 deletions(-)

diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb
index 43331b28e2b..0eae6b9770b 100644
--- a/lib/gitlab/ci/config/normalizer.rb
+++ b/lib/gitlab/ci/config/normalizer.rb
@@ -4,58 +4,54 @@ module Gitlab
   module Ci
     class Config
       class Normalizer
-        class << self
-          def normalize_jobs(jobs_config)
-            parallelized_config, parallelized_jobs = parallelize_jobs(jobs_config)
-            parallelize_dependencies(parallelized_config, parallelized_jobs)
-          end
-
-          private
-
-          def parallelize_jobs(jobs_config)
-            parallelized_jobs = {}
-
-            parallelized_config = jobs_config.map do |name, config|
-              if config[:parallel]
-                total = config[:parallel]
-                names = parallelize_job_names(name, total)
-                parallelized_jobs[name] = names.map(&:first)
-                Hash[names.collect { |job_name, index| [job_name.to_sym, config.merge(name: job_name, instance: index)] }]
-              else
-                { name => config }
-              end
-            end.reduce(:merge)
-
-            [parallelized_config, parallelized_jobs]
-          end
-
-          def parallelize_dependencies(jobs_config, parallelized_jobs)
-            jobs_config.map do |name, config|
-              if config[:dependencies]
-                deps = config[:dependencies].map do |dep|
-                  if parallelized_jobs.keys.include?(dep.to_sym)
-                    parallelized_jobs[dep.to_sym]
-                  else
-                    dep
-                  end
-                end.flatten
-
-                { name => config.merge(dependencies: deps) }
-              else
-                { name => config }
-              end
-            end.reduce(:merge)
-          end
-
-          def parallelize_job_names(name, total)
-            jobs = []
-
-            1.upto(total) do |idx|
-              jobs << ["#{name} #{idx}/#{total}", idx]
+        def initialize(jobs_config)
+          @jobs_config = jobs_config
+        end
+
+        def normalize_jobs
+          parallelized_jobs = parallelize_jobs
+          parallelize_dependencies(parallelized_jobs)
+        end
+
+        private
+
+        def parallelize_jobs
+          parallelized_jobs = {}
+
+          @jobs_config = @jobs_config.map do |name, config|
+            if config[:parallel]
+              total = config[:parallel]
+              names = self.class.parallelize_job_names(name, total)
+              parallelized_jobs[name] = names.map(&:first)
+              Hash[names.collect { |job_name, index| [job_name.to_sym, config.merge(name: job_name, instance: index)] }]
+            else
+              { name => config }
             end
+          end.reduce(:merge)
+
+          parallelized_jobs
+        end
+
+        def parallelize_dependencies(parallelized_jobs)
+          @jobs_config.map do |name, config|
+            if config[:dependencies]
+              deps = config[:dependencies].map do |dep|
+                if parallelized_jobs.keys.include?(dep.to_sym)
+                  parallelized_jobs[dep.to_sym]
+                else
+                  dep
+                end
+              end.flatten
+
+              { name => config.merge(dependencies: deps) }
+            else
+              { name => config }
+            end
+          end.reduce(:merge)
+        end
 
-            jobs
-          end
+        def self.parallelize_job_names(name, total)
+          Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] }
         end
       end
     end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 63b55c57913..f65a4e198cc 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -104,7 +104,7 @@ module Gitlab
         ##
         # Jobs
         #
-        @jobs = Ci::Config::Normalizer.normalize_jobs(@ci_config.jobs)
+        @jobs = Ci::Config::Normalizer.new(@ci_config.jobs).normalize_jobs
 
         @jobs.each do |name, job|
           # logical validation for job
diff --git a/spec/lib/gitlab/ci/config/normalizer_spec.rb b/spec/lib/gitlab/ci/config/normalizer_spec.rb
index 48b8217f77c..84f8ea3e861 100644
--- a/spec/lib/gitlab/ci/config/normalizer_spec.rb
+++ b/spec/lib/gitlab/ci/config/normalizer_spec.rb
@@ -8,7 +8,7 @@ describe Gitlab::Ci::Config::Normalizer do
   let(:config) { { job_name => job_config } }
 
   describe '.normalize_jobs' do
-    subject { described_class.normalize_jobs(config) }
+    subject { described_class.new(config).normalize_jobs }
 
     it 'does not have original job' do
       is_expected.not_to include(job_name)
-- 
2.24.1


From 4b2b154c4e5ddfc397bc59f3a38b8f09d44903e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Mon, 5 Nov 2018 15:53:37 +0100
Subject: [PATCH 20/24] Refactor Gitlab::Ci::Config::Normalizer

Use Hash#each_with_object to manipulate job hashes.
---
 lib/gitlab/ci/config/normalizer.rb           | 86 ++++++++++----------
 lib/gitlab/ci/yaml_processor.rb              |  2 +-
 spec/lib/gitlab/ci/config/normalizer_spec.rb | 12 ++-
 3 files changed, 56 insertions(+), 44 deletions(-)

diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb
index 0eae6b9770b..7bd0e08e817 100644
--- a/lib/gitlab/ci/config/normalizer.rb
+++ b/lib/gitlab/ci/config/normalizer.rb
@@ -4,54 +4,56 @@ module Gitlab
   module Ci
     class Config
       class Normalizer
-        def initialize(jobs_config)
-          @jobs_config = jobs_config
-        end
-
-        def normalize_jobs
-          parallelized_jobs = parallelize_jobs
-          parallelize_dependencies(parallelized_jobs)
-        end
+        class << self
+          def normalize_jobs(jobs_config)
+            parallelized_jobs = extract_parallelized_jobs(jobs_config)
+            parallelized_config = parallelize_jobs(jobs_config, parallelized_jobs)
+            parallelize_dependencies(parallelized_config, parallelized_jobs)
+          end
+
+          private
+
+          def extract_parallelized_jobs(jobs_config)
+            parallelized_jobs = {}
+
+            jobs_config.each do |job_name, config|
+              if config[:parallel]
+                parallelized_jobs[job_name] = parallelize_job_names(job_name, config[:parallel])
+              end
+            end
 
-        private
+            parallelized_jobs
+          end
 
-        def parallelize_jobs
-          parallelized_jobs = {}
+          def parallelize_jobs(jobs_config, parallelized_jobs)
+            jobs_config.each_with_object({}) do |(job_name, config), hash|
+              if parallelized_jobs.keys.include?(job_name)
+                parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) }
+              else
+                hash[job_name] = config
+              end
 
-          @jobs_config = @jobs_config.map do |name, config|
-            if config[:parallel]
-              total = config[:parallel]
-              names = self.class.parallelize_job_names(name, total)
-              parallelized_jobs[name] = names.map(&:first)
-              Hash[names.collect { |job_name, index| [job_name.to_sym, config.merge(name: job_name, instance: index)] }]
-            else
-              { name => config }
+              hash
             end
-          end.reduce(:merge)
-
-          parallelized_jobs
-        end
-
-        def parallelize_dependencies(parallelized_jobs)
-          @jobs_config.map do |name, config|
-            if config[:dependencies]
-              deps = config[:dependencies].map do |dep|
-                if parallelized_jobs.keys.include?(dep.to_sym)
-                  parallelized_jobs[dep.to_sym]
-                else
-                  dep
-                end
-              end.flatten
-
-              { name => config.merge(dependencies: deps) }
-            else
-              { name => config }
+          end
+
+          def parallelize_dependencies(parallelized_config, parallelized_jobs)
+            parallelized_config.each_with_object({}) do |(job_name, config), hash|
+              intersection = config[:dependencies] & parallelized_jobs.keys.map(&:to_s)
+              if intersection && intersection.any?
+                deps = intersection.map { |dep| parallelized_jobs[dep.to_sym].map(&:first) }.flatten
+                hash[job_name] = config.merge(dependencies: deps)
+              else
+                hash[job_name] = config
+              end
+
+              hash
             end
-          end.reduce(:merge)
-        end
+          end
 
-        def self.parallelize_job_names(name, total)
-          Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] }
+          def parallelize_job_names(name, total)
+            Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] }
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index f65a4e198cc..63b55c57913 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -104,7 +104,7 @@ module Gitlab
         ##
         # Jobs
         #
-        @jobs = Ci::Config::Normalizer.new(@ci_config.jobs).normalize_jobs
+        @jobs = Ci::Config::Normalizer.normalize_jobs(@ci_config.jobs)
 
         @jobs.each do |name, job|
           # logical validation for job
diff --git a/spec/lib/gitlab/ci/config/normalizer_spec.rb b/spec/lib/gitlab/ci/config/normalizer_spec.rb
index 84f8ea3e861..2c8396199d0 100644
--- a/spec/lib/gitlab/ci/config/normalizer_spec.rb
+++ b/spec/lib/gitlab/ci/config/normalizer_spec.rb
@@ -8,7 +8,7 @@ describe Gitlab::Ci::Config::Normalizer do
   let(:config) { { job_name => job_config } }
 
   describe '.normalize_jobs' do
-    subject { described_class.new(config).normalize_jobs }
+    subject { described_class.normalize_jobs(config) }
 
     it 'does not have original job' do
       is_expected.not_to include(job_name)
@@ -30,6 +30,16 @@ describe Gitlab::Ci::Config::Normalizer do
 
       expect(configs).to all(eq(original_config))
     end
+
+    context 'when jobs depend on parallelized jobs' do
+      let(:config) { { job_name => job_config, other_job: { script: 'echo 1', dependencies: [job_name.to_s] } } }
+
+      it 'parallelizes dependencies' do
+        job_names = described_class.send(:parallelize_job_names, job_name, 5).map(&:first)
+
+        expect(subject[:other_job][:dependencies]).to include(*job_names)
+      end
+    end
   end
 
   describe '.parallelize_job_names' do
-- 
2.24.1


From f48261a409f4f4d73118621330796ed92623f08e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Tue, 6 Nov 2018 18:23:45 +0100
Subject: [PATCH 21/24] Refactor Normalizer specs

---
 lib/gitlab/ci/config/normalizer.rb           |  2 +-
 spec/lib/gitlab/ci/config/normalizer_spec.rb | 12 ++----------
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb
index 7bd0e08e817..bfe976c04bf 100644
--- a/lib/gitlab/ci/config/normalizer.rb
+++ b/lib/gitlab/ci/config/normalizer.rb
@@ -27,7 +27,7 @@ module Gitlab
 
           def parallelize_jobs(jobs_config, parallelized_jobs)
             jobs_config.each_with_object({}) do |(job_name, config), hash|
-              if parallelized_jobs.keys.include?(job_name)
+              if parallelized_jobs.key?(job_name)
                 parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) }
               else
                 hash[job_name] = config
diff --git a/spec/lib/gitlab/ci/config/normalizer_spec.rb b/spec/lib/gitlab/ci/config/normalizer_spec.rb
index 2c8396199d0..2e3d3cc1b2b 100644
--- a/spec/lib/gitlab/ci/config/normalizer_spec.rb
+++ b/spec/lib/gitlab/ci/config/normalizer_spec.rb
@@ -15,7 +15,7 @@ describe Gitlab::Ci::Config::Normalizer do
     end
 
     it 'has parallelized jobs' do
-      job_names = described_class.send(:parallelize_job_names, job_name, 5).map { |job_name, index| job_name.to_sym }
+      job_names = [:"rspec 1/5", :"rspec 2/5", :"rspec 3/5", :"rspec 4/5", :"rspec 5/5"]
 
       is_expected.to include(*job_names)
     end
@@ -35,18 +35,10 @@ describe Gitlab::Ci::Config::Normalizer do
       let(:config) { { job_name => job_config, other_job: { script: 'echo 1', dependencies: [job_name.to_s] } } }
 
       it 'parallelizes dependencies' do
-        job_names = described_class.send(:parallelize_job_names, job_name, 5).map(&:first)
+        job_names = ["rspec 1/5", "rspec 2/5", "rspec 3/5", "rspec 4/5", "rspec 5/5"]
 
         expect(subject[:other_job][:dependencies]).to include(*job_names)
       end
     end
   end
-
-  describe '.parallelize_job_names' do
-    subject { described_class.send(:parallelize_job_names, job_name, 5) }
-
-    it 'returns parallelized names' do
-      expect(subject.map(&:first)).to all(match(%r{#{job_name} \d+/\d+}))
-    end
-  end
 end
-- 
2.24.1


From 4ba960c17d07e5863045c74fb554ebe71575d54d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Tue, 6 Nov 2018 18:35:44 +0100
Subject: [PATCH 22/24] Use instance based approach for Normalizer

---
 lib/gitlab/ci/config/normalizer.rb           | 76 ++++++++++----------
 lib/gitlab/ci/yaml_processor.rb              |  2 +-
 spec/lib/gitlab/ci/config/normalizer_spec.rb |  2 +-
 3 files changed, 41 insertions(+), 39 deletions(-)

diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb
index bfe976c04bf..ef263c4933c 100644
--- a/lib/gitlab/ci/config/normalizer.rb
+++ b/lib/gitlab/ci/config/normalizer.rb
@@ -4,57 +4,59 @@ module Gitlab
   module Ci
     class Config
       class Normalizer
-        class << self
-          def normalize_jobs(jobs_config)
-            parallelized_jobs = extract_parallelized_jobs(jobs_config)
-            parallelized_config = parallelize_jobs(jobs_config, parallelized_jobs)
-            parallelize_dependencies(parallelized_config, parallelized_jobs)
-          end
+        def initialize(jobs_config)
+          @jobs_config = jobs_config
+        end
+
+        def normalize_jobs
+          extract_parallelized_jobs
+          parallelized_config = parallelize_jobs
+          parallelize_dependencies(parallelized_config)
+        end
 
-          private
+        private
 
-          def extract_parallelized_jobs(jobs_config)
-            parallelized_jobs = {}
+        def extract_parallelized_jobs
+          @parallelized_jobs = {}
 
-            jobs_config.each do |job_name, config|
-              if config[:parallel]
-                parallelized_jobs[job_name] = parallelize_job_names(job_name, config[:parallel])
-              end
+          @jobs_config.each do |job_name, config|
+            if config[:parallel]
+              @parallelized_jobs[job_name] = self.class.parallelize_job_names(job_name, config[:parallel])
             end
-
-            parallelized_jobs
           end
 
-          def parallelize_jobs(jobs_config, parallelized_jobs)
-            jobs_config.each_with_object({}) do |(job_name, config), hash|
-              if parallelized_jobs.key?(job_name)
-                parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) }
-              else
-                hash[job_name] = config
-              end
+          @parallelized_jobs
+        end
 
-              hash
+        def parallelize_jobs
+          @jobs_config.each_with_object({}) do |(job_name, config), hash|
+            if @parallelized_jobs.key?(job_name)
+              @parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) }
+            else
+              hash[job_name] = config
             end
+
+            hash
           end
+        end
 
-          def parallelize_dependencies(parallelized_config, parallelized_jobs)
-            parallelized_config.each_with_object({}) do |(job_name, config), hash|
-              intersection = config[:dependencies] & parallelized_jobs.keys.map(&:to_s)
-              if intersection && intersection.any?
-                deps = intersection.map { |dep| parallelized_jobs[dep.to_sym].map(&:first) }.flatten
-                hash[job_name] = config.merge(dependencies: deps)
-              else
-                hash[job_name] = config
-              end
-
-              hash
+        def parallelize_dependencies(parallelized_config)
+          parallelized_config.each_with_object({}) do |(job_name, config), hash|
+            intersection = config[:dependencies] & @parallelized_jobs.keys.map(&:to_s)
+            if intersection && intersection.any?
+              deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten
+              hash[job_name] = config.merge(dependencies: deps)
+            else
+              hash[job_name] = config
             end
-          end
 
-          def parallelize_job_names(name, total)
-            Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] }
+            hash
           end
         end
+
+        def self.parallelize_job_names(name, total)
+          Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] }
+        end
       end
     end
   end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 63b55c57913..f65a4e198cc 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -104,7 +104,7 @@ module Gitlab
         ##
         # Jobs
         #
-        @jobs = Ci::Config::Normalizer.normalize_jobs(@ci_config.jobs)
+        @jobs = Ci::Config::Normalizer.new(@ci_config.jobs).normalize_jobs
 
         @jobs.each do |name, job|
           # logical validation for job
diff --git a/spec/lib/gitlab/ci/config/normalizer_spec.rb b/spec/lib/gitlab/ci/config/normalizer_spec.rb
index 2e3d3cc1b2b..65718d1a432 100644
--- a/spec/lib/gitlab/ci/config/normalizer_spec.rb
+++ b/spec/lib/gitlab/ci/config/normalizer_spec.rb
@@ -8,7 +8,7 @@ describe Gitlab::Ci::Config::Normalizer do
   let(:config) { { job_name => job_config } }
 
   describe '.normalize_jobs' do
-    subject { described_class.normalize_jobs(config) }
+    subject { described_class.new(config).normalize_jobs }
 
     it 'does not have original job' do
       is_expected.not_to include(job_name)
-- 
2.24.1


From 2adfbf1c663e3cf1aa4ec82a90eed12e72c87309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Tue, 6 Nov 2018 18:44:01 +0100
Subject: [PATCH 23/24] Avoid creating intersection if there's no array

---
 lib/gitlab/ci/config/normalizer.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb
index ef263c4933c..969ae093e8b 100644
--- a/lib/gitlab/ci/config/normalizer.rb
+++ b/lib/gitlab/ci/config/normalizer.rb
@@ -42,8 +42,8 @@ module Gitlab
 
         def parallelize_dependencies(parallelized_config)
           parallelized_config.each_with_object({}) do |(job_name, config), hash|
-            intersection = config[:dependencies] & @parallelized_jobs.keys.map(&:to_s)
-            if intersection && intersection.any?
+            parallelized_job_names = @parallelized_jobs.keys.map(&:to_s)
+            if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any?
               deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten
               hash[job_name] = config.merge(dependencies: deps)
             else
-- 
2.24.1


From 7366c319df12900e40dbed679feab146d1092d89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Tue, 6 Nov 2018 18:56:06 +0100
Subject: [PATCH 24/24] Add additional specs for Normalizer

---
 spec/lib/gitlab/ci/config/normalizer_spec.rb | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/spec/lib/gitlab/ci/config/normalizer_spec.rb b/spec/lib/gitlab/ci/config/normalizer_spec.rb
index 65718d1a432..7c558cacdd5 100644
--- a/spec/lib/gitlab/ci/config/normalizer_spec.rb
+++ b/spec/lib/gitlab/ci/config/normalizer_spec.rb
@@ -31,6 +31,16 @@ describe Gitlab::Ci::Config::Normalizer do
       expect(configs).to all(eq(original_config))
     end
 
+    context 'when there is a job with a slash in it' do
+      let(:job_name) { :"rspec 35/2" }
+
+      it 'properly parallelizes job names' do
+        job_names = [:"rspec 35/2 1/5", :"rspec 35/2 2/5", :"rspec 35/2 3/5", :"rspec 35/2 4/5", :"rspec 35/2 5/5"]
+
+        is_expected.to include(*job_names)
+      end
+    end
+
     context 'when jobs depend on parallelized jobs' do
       let(:config) { { job_name => job_config, other_job: { script: 'echo 1', dependencies: [job_name.to_s] } } }
 
@@ -39,6 +49,10 @@ describe Gitlab::Ci::Config::Normalizer do
 
         expect(subject[:other_job][:dependencies]).to include(*job_names)
       end
+
+      it 'does not include original job name in dependencies' do
+        expect(subject[:other_job][:dependencies]).not_to include(job_name)
+      end
     end
   end
 end
-- 
2.24.1