Velocity chart (BE)

Summary

This is the backend implementation issue for Velocity chart.

Proposal

See the proposal in the implementation issue.

Proof of concept

velocity_by_month.rb

require 'date'
require 'json'
require 'net/http'

# Purpose
# Get data on velocity (weight of closed issues).

# Instructions
# Customize access_token and query_params in the initialize method.

class VelocityByMonth

    attr_accessor(
      :access_token,
      :details,
      :query_params,
    )

    GITLAB_BOT_USER_ID = 1786152
    GITLAB_ORG_GROUP_ID = 9970 # gitlab-org

    def initialize(details = false)
      self.access_token = 'PAT_GOES_HERE'
      self.details = details
      self.query_params = {
        # Documentation at https://docs.gitlab.com/ee/api/issues.html#list-issues
        # assignee_username: 'djensen', # weight is divided by assignee count on issues with multiple assignees
        # created_after: '2020-01-01', # for testing
        labels: 'group::analytics,frontend', # Separate multiple with commas
        not: {
          labels: 'triage%20report',
        }
      }
    end

    def data
      {
        average: average_velocity_per_month,
        by_yearmonth: metrics_by_yearmonth.map do |yearmonth, metrics|
          [ yearmonth, metrics[:closed_weight] ]
        end.to_h
      }
    end

    def metrics_by_yearmonth
      return @metrics_by_yearmonth if @metrics_by_yearmonth
      
      @metrics_by_yearmonth = {}
      closed_issues.each do |issue|
        increment_metrics_for_issue(issue: issue, metrics_by_yearmonth: metrics_by_yearmonth)
      end
      @metrics_by_yearmonth = @metrics_by_yearmonth.sort.to_h
    end

    def missing_weight_issue_references
      get_all_of_metric(:missing_weight_issue_references)
    end

    def multiple_assignee_issue_references
      get_all_of_metric(:multiple_assignee_issue_references)
    end

    private

      def assignee_specified?
        query_params.has_key?(:assignee_id) || query_params.has_key?(:assignee_username)
      end

      def average_velocity_per_month
        velocities = metrics_by_yearmonth.map { |yearmonth, metrics| metrics[:closed_weight] }
        total_velocity = velocities.sum
        month_count = metrics_by_yearmonth.keys.size
        (total_velocity.to_f / month_count).round(1)
      end

      def build_query_string(params)
        query_string_default_params.merge(params).map do |key, value|
          if value.kind_of?(Hash)
            value.map do |subkey, subvalue|
              "#{key}[#{subkey}]=#{subvalue}" # see https://docs.gitlab.com/ee/api/#hash
            end.join('&')
          else
            "#{key}=#{value}"
          end
        end.join('&')
      end

      def closed_issues
        params = query_params.merge({ state: 'closed' })
        get_all_response_results(group_issues_url, params)
      end

      def current_month_start_date
        @current_month_start_date ||= (Date.today - Date.today.mday + 1)
      end

      def datetime_from_string(string)
        DateTime.parse(string)
      end

      def earliest_allowed_closed_at
        @earliest_allowed_closed_at ||= current_month_start_date - 365 # days
      end

      def get_all_of_metric(metric)
        metrics_by_yearmonth.map { |yearmonth, metrics| metrics[metric] }.flatten.sort
      end

      def get_all_response_results(url, params)
        results = []
        target_page = 1

        loop do
          params[:page] = target_page
          response = get_response(url, params)
          results.concat(JSON.parse(response.body))
          break if target_page >= response.header['X-Total-Pages'].to_i # Documented at https://docs.gitlab.com/ee/api/README.html#other-pagination-headers

          target_page += 1
        end

        return results
      end

      def get_response(url, params)
        sleep(0.1)
        query_string = build_query_string(params)
        uri = URI("#{url}?#{query_string}")
        request = Net::HTTP::Get.new(uri)
        Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
          http.request(request)
        end
      end

      def group_issues_url
        @group_issues_url ||= "https://gitlab.com/api/v4/groups/#{GITLAB_ORG_GROUP_ID}/issues"
      end

      def increment_metrics_for_issue(issue:, metrics_by_yearmonth:)
        return unless issue_countable?(issue)

        closed_at = datetime_from_string(issue['closed_at'])
        return if closed_at < earliest_allowed_closed_at
        # return if closed_at > current_month_start_date

        assignee_count = issue['assignees'].size
        closed_yearmonth = yearmonth(closed_at)
        weight = (issue['weight'] || 0).to_f
        weight = (weight / assignee_count) if assignee_count > 1 && assignee_specified?
puts "##{issue['id']}: #{weight}"

        metrics_by_yearmonth[closed_yearmonth] = metrics_default unless metrics_by_yearmonth.has_key?(closed_yearmonth)
        metrics_by_yearmonth[closed_yearmonth][:closed_weight] += weight
        if details
          metrics_by_yearmonth[closed_yearmonth][:closed_issues] += 1
          metrics_by_yearmonth[closed_yearmonth][:issue_references] << issue_reference(issue)
          metrics_by_yearmonth[closed_yearmonth][:missing_weight_issue_references] << issue_reference(issue) if weight.zero?
          metrics_by_yearmonth[closed_yearmonth][:multiple_assignee_issue_references] << issue_reference(issue) if assignee_count > 1
        end
      end

      def issue_countable?(issue)
        return false if issue['assignee'].nil? # Un-assigned means closed without execution
        return false if issue['author_id'] == GITLAB_BOT_USER_ID # Triage report
        return false if security_issue?(issue) # security issues duplicate gitlab issues

        return true
      end

      def issue_reference(issue)
        issue['references']['relative']
      end

      def last_complete_yearmonth
        @last_complete_yearmonth ||= team_metrics_by_yearmonth.sort.to_h.keys.last
      end

      def metrics_default
        hash = {
          # assigned_issues: 0,
          # assigned_weight: 0,
          closed_weight: 0.0,
        }
        if details
          hash.merge!({
            closed_issues: 0,
            issue_references: [],
            missing_weight_issue_references: [],
            multiple_assignee_issue_references: [],
          })
        end
        return hash
      end

      def query_string_default_params
        {
          per_page: 50, # Max is 100
          private_token: access_token,
        }
      end

      def security_issue?(issue)
        issue_reference(issue).match?(/^gitlab-org\/security\/gitlab/)
      end

      def yearmonth(datetime)
        date = datetime.to_date
        date.strftime('%Y%m').to_i
      end

end

velocity = VelocityByMonth.new(true)
puts velocity.data

puts "metrics_by_yearmonth:"
puts velocity.metrics_by_yearmonth

if velocity.details
  puts "missing_weight_issue_references (#{velocity.missing_weight_issue_references.size})"
  puts velocity.missing_weight_issue_references

  puts "multiple_assignee_issue_references (#{velocity.multiple_assignee_issue_references.size})"
  puts velocity.multiple_assignee_issue_references
end