Add more usage data to EE ping. Load it asynchronously to

prevent blocking a page load.

Closes #997

CHANGELOG-EE

 Please view this file on the master branch, on stable branches it's out of date.
 v 8.12.0 (Unreleased)
+  - Include more data in EE usage ping
   - Reduce UPDATE queries when moving between import states on projects
   - [ES] Instrument Elasticsearch::Git::Repository
   - Request only the LDAP attributes we need

app/controllers/admin/application_settings_controller.rb

 class Admin::ApplicationSettingsController < Admin::ApplicationController
+  include UsageDataHelper
+
   before_action :set_application_setting
 
   def show
@@ -13,6 +15,13 @@ def update
     end
   end
 
+  def usage_data
+    respond_to do |format|
+      format.html { render html: Gitlab::Highlight.highlight('payload.json', all_usage_data.to_json) }
+      format.json { render json: all_usage_data.to_json }
+    end
+  end
+
   def reset_runners_token
     @application_setting.reset_runners_registration_token!
     flash[:notice] = 'New runners registration token has been generated!'

app/helpers/usage_data_helper.rb

+module UsageDataHelper
+  include LicenseHelper
+
+  def all_usage_data
+    Rails.cache.fetch('usage_data', expires_in: 1.hour) { uncached_usage_data }
+  end
+
+  def uncached_usage_data
+    license_usage_data.merge(system_usage_data)
+  end
+
+  private
+
+  def system_usage_data
+    {
+      board_count: Board.count,
+      ci_build_count: Ci::Build.count,
+      ci_pipeline_count: Ci::Pipeline.count,
+      ci_runner_count: Ci::Runner.count,
+      ci_trigger_count: Ci::Trigger.count,
+      deploy_key_count: DeployKey.count,
+      deployment_count: Deployment.count,
+      environment_count: Environment.count,
+      geo_node_count: GeoNode.count,
+      group_count: Group.count,
+      issue_count: Issue.count,
+      key_count: Key.count,
+      label_count: Label.count,
+      ldap_group_link_count: LdapGroupLink.count,
+      ldap_key_count: LDAPKey.count,
+      ldap_user_count: User.ldap.count,
+      lfs_object_count: LfsObject.count,
+      merge_request_count: MergeRequest.count,
+      milestone_count: Milestone.count,
+      note_count: Note.count,
+      push_count: Event.code_push.count,
+      pages_domain_count: PagesDomain.count,
+      project_count: Project.count,
+      protected_branches_count: ProtectedBranch.count,
+      release_count: Release.count,
+      remote_mirror_count: RemoteMirror.count,
+      service_count: Service.where(active: true).count,
+      snippet_count: Snippet.count,
+      todo_count: Todo.count,
+      web_hook_count: WebHook.count
+    }
+  end
+end

app/views/admin/application_settings/_form.html.haml

@@ -64,7 +64,7 @@
           .help-block
             Every week GitLab will report license usage back to GitLab, Inc.
             Disable this option if you do not want this to occur. This is the JSON payload that will be sent:
-            %pre.js-syntax-highlight.code.highlight= Gitlab::Highlight.highlight('payload.json', license_usage_data.to_json)
+            .usage-data.pre.js-syntax-highlight.code.highlight
     .form-group
       .col-sm-offset-2.col-sm-10
         .checkbox
@@ -461,3 +461,13 @@
 
   .form-actions
     = f.submit 'Save', class: 'btn btn-save'
+
+:javascript
+  $.ajax({
+    type: "GET",
+    url: "#{usage_data_admin_application_settings_path(format: :html)}",
+    dataType: "html",
+    success: function (html) {
+      $(".usage-data").html("<pre>" + html + "</pre>");
+    }
+  });

app/workers/gitlab_usage_ping_worker.rb

 class GitlabUsagePingWorker
   LEASE_TIMEOUT = 86400
 
-  include LicenseHelper
+  include UsageDataHelper
   include Sidekiq::Worker
   include HTTParty
 
@@ -16,7 +16,7 @@ def perform
 
     begin
       HTTParty.post(url,
-                    body: license_usage_data.to_json,
+                    body: all_usage_data.to_json,
                     headers: { 'Content-type' => 'application/json' }
                    )
     rescue HTTParty::Error => e

config/routes.rb

@@ -320,6 +320,7 @@
 
     resource :application_settings, only: [:show, :update] do
       resources :services, only: [:index, :edit, :update]
+      get :usage_data
       put :reset_runners_token
       put :reset_health_check_token
       put :clear_repository_check_states

spec/controllers/admin/application_settings_controller_spec.rb

+require 'spec_helper'
+
+describe Admin::ApplicationSettingsController do
+  let(:group) { create(:group) }
+  let(:project) { create(:project, namespace: group) }
+  let(:admin) { create(:admin) }
+  let(:user) { create(:user)}
+  let(:body) { JSON.parse(response.body) }
+
+  describe 'GET #usage_data with no access' do
+    before do
+      sign_in(user)
+    end
+
+    it 'returns 404' do
+      get :usage_data, format: :html
+
+      expect(response.status).to eq(404)
+    end
+  end
+
+  describe 'GET #usage_data' do
+    before do
+      sign_in(admin)
+    end
+
+    it 'returns HTML data' do
+      get :usage_data, format: :html
+
+      html = Nokogiri::HTML.fragment(response.body)
+      expect(html.fragment?).to be_truthy
+    end
+
+    it 'returns JSON data' do
+      get :usage_data, format: :json
+
+      expect(body["version"]).to eq(Gitlab::VERSION)
+      expect(body).to include('push_count')
+    end
+  end
+end

spec/helpers/usage_data_helper_spec.rb

+require 'spec_helper'
+
+describe UsageDataHelper do
+  let!(:project) { create(:project) }
+  let!(:project2) { create(:project) }
+  let!(:board) { create(:board, project: project) }
+
+  describe '#all_usage_data' do
+    it "gathers usage data" do
+      data = all_usage_data
+
+      expect(data[:board_count]).to eq(1)
+      expect(data[:project_count]).to eq(2)
+
+      expect(data).to include(:ci_build_count)
+      expect(data).to include(:ci_pipeline_count)
+      expect(data).to include(:ci_runner_count)
+      expect(data).to include(:ci_trigger_count)
+      expect(data).to include(:deploy_key_count)
+      expect(data).to include(:deployment_count)
+      expect(data).to include(:environment_count)
+      expect(data).to include(:geo_node_count)
+      expect(data).to include(:group_count)
+      expect(data).to include(:issue_count)
+      expect(data).to include(:key_count)
+      expect(data).to include(:label_count)
+      expect(data).to include(:ldap_group_link_count)
+      expect(data).to include(:ldap_key_count)
+      expect(data).to include(:ldap_user_count)
+      expect(data).to include(:lfs_object_count)
+      expect(data).to include(:merge_request_count)
+      expect(data).to include(:milestone_count)
+      expect(data).to include(:note_count)
+      expect(data).to include(:push_count)
+      expect(data).to include(:pages_domain_count)
+      expect(data).to include(:protected_branches_count)
+      expect(data).to include(:release_count)
+      expect(data).to include(:remote_mirror_count)
+      expect(data).to include(:service_count)
+      expect(data).to include(:snippet_count)
+      expect(data).to include(:todo_count)
+      expect(data).to include(:web_hook_count)
+
+      license = License.current
+      expect(data[:license_md5]).to eq(Digest::MD5.hexdigest(license.data))
+      expect(data[:version]).to eq(Gitlab::VERSION)
+      expect(data[:licensee]).to eq(license.licensee)
+      expect(data[:active_user_count]).to eq(User.active.count)
+      expect(data[:licensee]).to eq(license.licensee)
+      expect(data[:license_user_count]).to eq(license.user_count)
+      expect(data[:license_starts_at]).to eq(license.starts_at)
+      expect(data[:license_expires_at]).to eq(license.expires_at)
+      expect(data[:license_add_ons]).to eq(license.add_ons)
+      expect(data[:recorded_at]).to be_a(Time)
+    end
+  end
+end