Merge branch '300031-associate-jira-deploys-with-last-commits-to-env' into 'master'

Extract Jira issue keys for deploy from commits

See merge request !123455



Merged-by: Andy Soiron <asoiron@gitlab.com>
Approved-by: Ashraf Khamis (OOO) <akhamis@gitlab.com>
Approved-by: Andy Soiron <asoiron@gitlab.com>
Approved-by: Bojan Marjanovic <bmarjanovic@gitlab.com>
Reviewed-by: Vasilii Iakliushin <viakliushin@gitlab.com>
Reviewed-by: Bojan Marjanovic <bmarjanovic@gitlab.com>
Co-authored-by: Bojan Marjanovic <bmarjanovic@gitlab.com>
Co-authored-by: Luke Duncalfe <lduncalfe@eml.cc>

.rubocop_todo/rspec/context_wording.yml

@@ -1535,7 +1535,6 @@ RSpec/ContextWording:
     - 'spec/lib/api/validations/validators/untrusted_regexp_spec.rb'
     - 'spec/lib/atlassian/jira_connect/jwt/asymmetric_spec.rb'
     - 'spec/lib/atlassian/jira_connect/jwt/symmetric_spec.rb'
-    - 'spec/lib/atlassian/jira_connect/serializers/deployment_entity_spec.rb'
     - 'spec/lib/atlassian/jira_connect/serializers/feature_flag_entity_spec.rb'
     - 'spec/lib/atlassian/jira_connect_spec.rb'
     - 'spec/lib/backup/gitaly_backup_spec.rb'

config/feature_flags/development/jira_deployment_issue_keys.yml

+---
+name: jira_deployment_issue_keys
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/123455
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/415025
+milestone: '16.2'
+type: development
+group: group::import and integrate
+default_enabled: false

doc/integration/jira/development_panel.md

@@ -51,9 +51,9 @@ depends on where you mention the Jira issue ID in GitLab.
 
 | GitLab: where you mention the Jira issue ID    | Jira development panel: what information is displayed |
 |------------------------------------------------|-------------------------------------------------------|
-| Merge request title or description             | Link to the merge request<br>Link to the branch ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/354373) in GitLab 15.11) |
-| Branch name                                    | Link to the branch                                    |
-| Commit message                                 | Link to the commit                                    |
+| Merge request title or description             | Link to the merge request<br>Link to the deployment<br>Link to the pipeline by title only and by description ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/390888) in GitLab 15.10)<br>Link to the branch ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/354373) in GitLab 15.11) |
+| Branch name                                    | Link to the branch<br>Link to the deployment                                    |
+| Commit message                                 | Link to the commit<br>Link to the deployment from up to 5,000 commits after the last successful deployment to the environment ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/300031) in GitLab 16.2 [with a flag](../../administration/feature_flags.md) named `jira_deployment_issue_keys`. Disabled by default) |
 | [Jira Smart Commit](#jira-smart-commits)       | Custom comment, logged time, or workflow transition   |
 
 ## Jira Smart Commits

doc/integration/jira/index.md

@@ -46,7 +46,7 @@ This table shows the capabilities available with the Jira issue integration and
 | [View a list of Jira issues](issues.md#view-jira-issues) | **{check-circle}** Yes | **{dotted-circle}** No |
 | [Create a Jira issue for a vulnerability](../../user/application_security/vulnerabilities/index.md#create-a-jira-issue-for-a-vulnerability) | **{check-circle}** Yes | **{dotted-circle}** No |
 | Create a GitLab branch from a Jira issue | **{dotted-circle}** No | **{check-circle}** Yes, in the issue's development panel |
-| Mention a Jira issue ID in a GitLab merge request, and deployments are synced | **{dotted-circle}** No | **{check-circle}** Yes, in the issue's development panel |
+| Sync GitLab deployments to Jira issues | **{dotted-circle}** No | **{check-circle}** Yes, in the issue's development panel. Mention a Jira issue ID in a GitLab merge request, branch name, or any of the last 5,000 commits made to the branch after the last successful deployment to the environment |
 
 ## Privacy considerations
 

lib/atlassian/jira_connect/serializers/build_entity.rb

@@ -22,13 +22,7 @@ class BuildEntity < Grape::Entity
         expose :references
 
         def issue_keys
-          commit_message_issue_keys = JiraIssueKeyExtractor.new(pipeline.git_commit_message).issue_keys
-
-          # extract Jira issue keys from either the source branch/ref or the merge request title.
-          @issue_keys ||= commit_message_issue_keys + pipeline.all_merge_requests.flat_map do |mr|
-            src = "#{mr.source_branch} #{mr.title} #{mr.description}"
-            JiraIssueKeyExtractor.new(src).issue_keys
-          end.uniq
+          @issue_keys ||= (pipeline_commit_issue_keys + pipeline_mrs_issue_keys).uniq
         end
 
         private
@@ -89,6 +83,18 @@ def references
         def update_sequence_id
           options[:update_sequence_id] || Client.generate_update_sequence_id
         end
+
+        def pipeline_commit_issue_keys
+          JiraIssueKeyExtractor.new(pipeline.git_commit_message).issue_keys
+        end
+
+        # Extract Jira issue keys from either the source branch/ref, merge request title or merge request description.
+        def pipeline_mrs_issue_keys
+          pipeline.all_merge_requests.flat_map do |mr|
+            src = "#{mr.source_branch} #{mr.title} #{mr.description}"
+            JiraIssueKeyExtractor.new(src).issue_keys
+          end
+        end
       end
     end
   end

lib/atlassian/jira_connect/serializers/deployment_entity.rb

@@ -6,6 +6,8 @@ module Serializers
       class DeploymentEntity < Grape::Entity
         include Gitlab::Routing
 
+        COMMITS_LIMIT = 5_000
+
         format_with(:iso8601, &:iso8601)
 
         expose :schema_version, as: :schemaVersion
@@ -22,9 +24,7 @@ class DeploymentEntity < Grape::Entity
         expose :environment_entity, as: :environment
 
         def issue_keys
-          return [] unless build&.pipeline.present?
-
-          @issue_keys ||= BuildEntity.new(build.pipeline).issue_keys
+          @issue_keys ||= (issue_keys_from_pipeline + issue_keys_from_commits_since_last_deploy).uniq
         end
 
         private
@@ -74,7 +74,7 @@ def schema_version
         end
 
         def pipeline_entity
-          PipelineEntity.new(build.pipeline) if build&.pipeline.present?
+          PipelineEntity.new(build.pipeline) if pipeline?
         end
 
         def environment_entity
@@ -84,6 +84,44 @@ def environment_entity
         def update_sequence_id
           options[:update_sequence_id] || Client.generate_update_sequence_id
         end
+
+        def pipeline?
+          build&.pipeline.present?
+        end
+
+        def issue_keys_from_pipeline
+          return [] unless pipeline?
+
+          BuildEntity.new(build.pipeline).issue_keys
+        end
+
+        # Extract Jira issue keys from commits made to the deployment's branch or tag
+        # since the last successful deployment was made to the environment.
+        def issue_keys_from_commits_since_last_deploy
+          return [] if Feature.disabled?(:jira_deployment_issue_keys, project)
+
+          last_deployed_commit = environment
+            .successful_deployments
+            .id_not_in(deployment.id)
+            .ordered
+            .find_by_ref(deployment.ref)
+            &.commit
+
+          commits = project.repository.commits(
+            deployment.ref,
+            before: deployment.commit.created_at,
+            after: last_deployed_commit&.created_at,
+            skip_merges: true,
+            limit: COMMITS_LIMIT
+          )
+
+          # Include this deploy's commit, as the `before:` param in `Repository#list_commits_by` excluded it.
+          commits << deployment.commit
+
+          commits.flat_map do |commit|
+            JiraIssueKeyExtractor.new(commit.message).issue_keys
+          end.compact
+        end
       end
     end
   end

spec/factories/integrations.rb

@@ -136,6 +136,8 @@
           jira_auth_type: evaluator.jira_auth_type,
           jira_issue_transition_automatic: evaluator.jira_issue_transition_automatic,
           jira_issue_transition_id: evaluator.jira_issue_transition_id,
+          jira_issue_prefix: evaluator.jira_issue_prefix,
+          jira_issue_regex: evaluator.jira_issue_regex,
           username: evaluator.username, password: evaluator.password, issues_enabled: evaluator.issues_enabled,
           project_key: evaluator.project_key, vulnerabilities_enabled: evaluator.vulnerabilities_enabled,
           vulnerabilities_issuetype: evaluator.vulnerabilities_issuetype, deployment_type: evaluator.deployment_type

spec/lib/atlassian/jira_connect/client_spec.rb

@@ -214,13 +214,7 @@ def expected_headers(path)
   end
 
   describe '#store_deploy_info' do
-    let_it_be(:environment) { create(:environment, name: 'DEV', project: project) }
-    let_it_be(:deployments) do
-      pipelines.map do |p|
-        build = create(:ci_build, environment: environment.name, pipeline: p, project: project)
-        create(:deployment, deployable: build, environment: environment)
-      end
-    end
+    let_it_be(:deployments) { create_list(:deployment, 1) }
 
     let(:schema) do
       Atlassian::Schemata.deploy_info_payload
@@ -252,18 +246,22 @@ def expected_headers(path)
       subject.send(:store_deploy_info, project: project, deployments: deployments)
     end
 
-    it 'only sends information about relevant MRs' do
+    it 'calls the API if issue keys are found' do
       expect(subject).to receive(:post).with(
-        '/rest/deployments/0.1/bulk', { deployments: have_attributes(size: 8) }
+        '/rest/deployments/0.1/bulk', { deployments: have_attributes(size: 1) }
       ).and_call_original
 
       subject.send(:store_deploy_info, project: project, deployments: deployments)
     end
 
-    it 'does not call the API if there is nothing to report' do
+    it 'does not call the API if no issue keys are found' do
+      allow_next_instances_of(Atlassian::JiraConnect::Serializers::DeploymentEntity, nil) do |entity|
+        allow(entity).to receive(:issue_keys).and_return([])
+      end
+
       expect(subject).not_to receive(:post)
 
-      subject.send(:store_deploy_info, project: project, deployments: deployments.take(1))
+      subject.send(:store_deploy_info, project: project, deployments: deployments)
     end
 
     context 'when there are errors' do

spec/lib/atlassian/jira_connect/serializers/deployment_entity_spec.rb

@@ -6,18 +6,16 @@
   let_it_be(:user) { create_default(:user) }
   let_it_be(:project) { create_default(:project, :repository) }
   let_it_be(:environment) { create(:environment, name: 'prod', project: project) }
-  let_it_be_with_reload(:deployment) { create(:deployment, environment: environment) }
+  let_it_be_with_refind(:deployment) { create(:deployment, environment: environment) }
 
   subject { described_class.represent(deployment) }
 
-  context 'when the deployment does not belong to any Jira issue' do
-    describe '#issue_keys' do
-      it 'is empty' do
-        expect(subject.issue_keys).to be_empty
+  describe '#to_json' do
+    context 'when the deployment does not belong to any Jira issue' do
+      before do
+        allow(subject).to receive(:issue_keys).and_return([])
       end
-    end
 
-    describe '#to_json' do
       it 'can encode the object' do
         expect(subject.to_json).to be_valid_json
       end
@@ -26,9 +24,19 @@
         expect(subject.to_json).not_to match_schema(Atlassian::Schemata.deployment_info)
       end
     end
+
+    context 'when the deployment belongs to Jira issue' do
+      before do
+        allow(subject).to receive(:issue_keys).and_return(['JIRA-1'])
+      end
+
+      it 'is valid according to the deployment info schema' do
+        expect(subject.to_json).to be_valid_json.and match_schema(Atlassian::Schemata.deployment_info)
+      end
+    end
   end
 
-  context 'this is an external deployment' do
+  context 'when deployment is an external deployment' do
     before do
       deployment.update!(deployable: nil)
     end
@@ -36,10 +44,6 @@
     it 'does not raise errors when serializing' do
       expect { subject.to_json }.not_to raise_error
     end
-
-    it 'returns an empty list of issue keys' do
-      expect(subject.issue_keys).to be_empty
-    end
   end
 
   describe 'environment type' do
@@ -62,27 +66,137 @@
     end
   end
 
-  context 'when the deployment can be linked to a Jira issue' do
-    let(:pipeline) { create(:ci_pipeline, merge_request: merge_request) }
-
+  describe '#issue_keys' do
+    # For these tests, use a Jira issue key regex that matches a set of commit messages
+    # in the test repo.
+    #
+    # Relevant commits in this test from https://gitlab.com/gitlab-org/gitlab-test/-/commits/master:
+    #
+    # 1) 5f923865dde3436854e9ceb9cdb7815618d4e849 GitLab currently doesn't support patches [...]: add a commit here
+    # 2) 4cd80ccab63c82b4bad16faa5193fbd2aa06df40 add directory structure for tree_helper spec
+    # 3) ae73cb07c9eeaf35924a10f713b364d32b2dd34f Binary file added
+    # 4) 33f3729a45c02fc67d00adb1b8bca394b0e761d9 Image added
     before do
-      subject.deployable.update!(pipeline: pipeline)
+      allow(Gitlab::Regex).to receive(:jira_issue_key_regex).and_return(/add.[a-d]/)
+    end
+
+    let(:expected_issue_keys) { ['add a', 'add d', 'added'] }
+
+    it 'extracts issue keys from the commits' do
+      expect(subject.issue_keys).to contain_exactly(*expected_issue_keys)
+    end
+
+    it 'limits the number of commits scanned' do
+      stub_const("#{described_class}::COMMITS_LIMIT", 10)
+
+      expect(subject.issue_keys).to contain_exactly('add a')
+    end
+
+    context 'when `jira_deployment_issue_keys` flag is disabled' do
+      before do
+        stub_feature_flags(jira_deployment_issue_keys: false)
+      end
+
+      it 'does not extract issue keys from commits' do
+        expect(subject.issue_keys).to be_empty
+      end
+    end
+
+    context 'when deploy happened at an older commit' do
+      before do
+        # SHA is from a commit between 1) and 2) in the commit list above.
+        deployment.update!(sha: 'c7fbe50c7c7419d9701eebe64b1fdacc3df5b9dd')
+      end
+
+      it 'extracts only issue keys from that commit or older' do
+        expect(subject.issue_keys).to contain_exactly('add d', 'added')
+      end
     end
 
-    %i[jira_branch jira_title jira_description].each do |trait|
-      context "because it belongs to an MR with a #{trait}" do
-        let(:merge_request) { create(:merge_request, trait) }
+    context 'when the deployment has an associated merge request' do
+      let_it_be(:pipeline) do
+        create(:ci_pipeline,
+          merge_request: create(:merge_request,
+            title: 'Title addxa',
+            description: "Description\naddxa\naddya",
+            source_branch: 'feature/addza'
+          )
+        )
+      end
+
+      before do
+        subject.deployable.update!(pipeline: pipeline)
+      end
+
+      it 'includes issue keys extracted from the merge request' do
+        expect(subject.issue_keys).to contain_exactly(
+          *(expected_issue_keys + %w[addxa addya addza])
+        )
+      end
+    end
+
+    context 'when there was a successful deploy to the environment' do
+      let_it_be_with_reload(:last_deploy) do
+        # SHA is from a commit between 2) and 3) in the commit list above.
+        sha = '5937ac0a7beb003549fc5fd26fc247adbce4a52e'
+        create(:deployment, :success, sha: sha, environment: environment, finished_at: 1.hour.ago)
+      end
+
+      shared_examples 'extracts only issue keys from commits made since that deployment' do
+        specify do
+          expect(subject.issue_keys).to contain_exactly('add a', 'add d')
+        end
+      end
+
+      shared_examples 'ignores that deployment' do
+        specify do
+          expect(subject.issue_keys).to contain_exactly(*expected_issue_keys)
+        end
+      end
+
+      it_behaves_like 'extracts only issue keys from commits made since that deployment'
+
+      context 'when the deploy was for a different environment' do
+        before do
+          last_deploy.update!(environment: create(:environment))
+        end
+
+        it_behaves_like 'ignores that deployment'
+      end
+
+      context 'when the deploy was for a different branch or tag' do
+        before do
+          last_deploy.update!(ref: 'foo')
+        end
+
+        it_behaves_like 'ignores that deployment'
+      end
+
+      context 'when the deploy was not successful' do
+        before do
+          last_deploy.drop!
+        end
+
+        it_behaves_like 'ignores that deployment'
+      end
+
+      context 'when the deploy commit cannot be found' do
+        before do
+          last_deploy.update!(sha: 'foo')
+        end
+
+        it_behaves_like 'ignores that deployment'
+      end
 
-        describe '#issue_keys' do
-          it 'is not empty' do
-            expect(subject.issue_keys).not_to be_empty
-          end
+      context 'when there is a more recent deployment' do
+        let_it_be(:more_recent_last_deploy) do
+          # SHA is from a commit between 1) and 2) in the commit list above.
+          sha = 'c7fbe50c7c7419d9701eebe64b1fdacc3df5b9dd'
+          create(:deployment, :success, sha: sha, environment: environment, finished_at: 1.minute.ago)
         end
 
-        describe '#to_json' do
-          it 'is valid according to the deployment info schema' do
-            expect(subject.to_json).to be_valid_json.and match_schema(Atlassian::Schemata.deployment_info)
-          end
+        it 'extracts only issue keys from commits made since that deployment' do
+          expect(subject.issue_keys).to contain_exactly('add a')
         end
       end
     end

spec/support/rspec_order_todo.yml

@@ -5248,7 +5248,6 @@
 - './spec/lib/atlassian/jira_connect/serializers/base_entity_spec.rb'
 - './spec/lib/atlassian/jira_connect/serializers/branch_entity_spec.rb'
 - './spec/lib/atlassian/jira_connect/serializers/build_entity_spec.rb'
-- './spec/lib/atlassian/jira_connect/serializers/deployment_entity_spec.rb'
 - './spec/lib/atlassian/jira_connect/serializers/feature_flag_entity_spec.rb'
 - './spec/lib/atlassian/jira_connect/serializers/pull_request_entity_spec.rb'
 - './spec/lib/atlassian/jira_connect/serializers/repository_entity_spec.rb'