Extract EE specific files/lines for configuration files
We have the following files containing EE specific code. We should move them to ee/
- config/README.md => https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/14551
- config/application.rb
- config/brakeman.ignore (file only)
- config/database.yml.postgresql (file only)
- config/database_geo.yml.mysql (file only) ~Geo => Removed in https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/14549
- config/database_geo.yml.postgresql (file only) ~Geo
- config/gitlab.yml.example
- config/settings.rb
- config/sidekiq_queues.yml
- config/unicorn.rb.example
-
config/webpack.config.js=> https://gitlab.com/gitlab-org/gitlab-ee/issues/10328
The following files would have to be modified somehow so they're the same for both CE and EE:
- .gitlab-ci.yml
- crowdin.yml
- app/workers/all_queues.yml
Differences:
config/application.rb: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/29557
diff --git a/config/application.rb b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/application.rb
index cddd91f267a..67cd766ec6d 100644
--- a/config/application.rb
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/application.rb
@@ -4,6 +4,8 @@ require 'rails/all'
Bundler.require(:default, Rails.env)
+require 'elasticsearch/rails/instrumentation'
+
module Gitlab
class Application < Rails::Application
require_dependency Rails.root.join('lib/gitlab/redis/wrapper')
@@ -47,6 +49,21 @@ module Gitlab
config.generators.templates.push("#{config.root}/generator_templates")
+ ## EE-specific paths config START
+ ee_paths = config.eager_load_paths.each_with_object([]) do |path, memo|
+ ee_path = config.root.join('ee', Pathname.new(path).relative_path_from(config.root))
+ memo << ee_path.to_s if ee_path.exist?
+ end
+
+ # Eager load should load CE first
+ config.eager_load_paths.push(*ee_paths)
+ config.helpers_paths.push "#{config.root}/ee/app/helpers"
+
+ # Other than Ruby modules we load EE first
+ config.paths['lib/tasks'].unshift "#{config.root}/ee/lib/tasks"
+ config.paths['app/views'].unshift "#{config.root}/ee/app/views"
+ ## EE-specific paths config END
+
# Rake tasks ignore the eager loading settings, so we need to set the
# autoload paths explicitly
config.autoload_paths = config.eager_load_paths.dup
@@ -161,6 +178,25 @@ module Gitlab
config.assets.paths << "#{config.root}/node_modules/xterm/src/"
config.assets.precompile << "xterm.css"
+ ## EE-specific assets config START
+ %w[images javascripts stylesheets].each do |path|
+ config.assets.paths << "#{config.root}/ee/app/assets/#{path}"
+ config.assets.precompile << "jira_connect.js"
+ config.assets.precompile << "pages/jira_connect.css"
+ end
+
+ config.assets.paths << "#{config.root}/vendor/assets/javascripts/"
+ config.assets.precompile << "snowplow/sp.js"
+
+ # Compile non-JS/CSS assets in the ee/app/assets folder by default
+ # Mimic sprockets-rails default: https://github.com/rails/sprockets-rails/blob/v3.2.1/lib/sprockets/railtie.rb#L84-L87
+ LOOSE_EE_APP_ASSETS = lambda do |logical_path, filename|
+ filename.start_with?(config.root.join("ee/app/assets").to_s) &&
+ !['.js', '.css', ''].include?(File.extname(logical_path))
+ end
+ config.assets.precompile << LOOSE_EE_APP_ASSETS
+ ## EE-specific assets config END
+
# Version of your assets, change this if you want to expire all your assets
config.assets.version = '1.0'
config/database.yml.postgresql: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/29606
diff --git a/config/database.yml.postgresql b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/database.yml.postgresql
index baded682e46..5b3b35c9226 100644
--- a/config/database.yml.postgresql
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/database.yml.postgresql
@@ -9,6 +9,15 @@ production:
username: git
password: "secure password"
host: localhost
+ # load_balancing:
+ # hosts:
+ # - host1.example.com
+ # - host2.example.com
+ # discover:
+ # nameserver: 1.2.3.4
+ # port: 8600
+ # record: secondary.postgresql.service.consul
+ # interval: 300
#
# Development specific
config/dependency_decisions.yml: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/29606
diff --git a/config/dependency_decisions.yml b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/dependency_decisions.yml
index af76bace577..40a80429afa 100644
--- a/config/dependency_decisions.yml
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/dependency_decisions.yml
@@ -599,3 +599,10 @@
:why: https://github.com/apache/incubator-echarts/blob/master/LICENSE
:versions: []
:when: 2018-12-07 20:46:12.421256000 Z
+- - :license
+ - contracts
+ - BSD
+ - :who: Jarka Košanová
+ :why: https://github.com/egonSchiele/contracts.ruby/blob/master/LICENSE
+ :versions: []
+ :when: 2019-04-01 11:29:39.361015000 Z
config/gitlab.yml.example: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/29606
diff --git a/config/gitlab.yml.example b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/gitlab.yml.example
index c83f569d885..1aec8300fb3 100644
--- a/config/gitlab.yml.example
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/gitlab.yml.example
@@ -230,7 +230,45 @@ production: &base
## Packages (maven repository, npm registry, etc...)
packages:
- enabled: false
+ enabled: true
+ # The location where build packages are stored (default: shared/packages).
+ # storage_path: shared/packages
+ object_store:
+ enabled: false
+ remote_directory: packages # The bucket name
+ # direct_upload: false # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false)
+ # background_upload: false # Temporary option to limit automatic upload (Default: true)
+ # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage
+ connection:
+ provider: AWS
+ aws_access_key_id: AWS_ACCESS_KEY_ID
+ aws_secret_access_key: AWS_SECRET_ACCESS_KEY
+ region: us-east-1
+ # host: 'localhost' # default: s3.amazonaws.com
+ # endpoint: 'http://127.0.0.1:9000' # default: nil
+ # aws_signature_version: 4 # For creation of signed URLs. Set to 2 if provider does not support v4.
+ # path_style: true # Use 'host/bucket_name/object' instead of 'bucket_name.host/object'
+
+ ## Dependency Proxy
+ dependency_proxy:
+ enabled: true
+ # The location where build packages are stored (default: shared/dependency_proxy).
+ # storage_path: shared/dependency_proxy
+ object_store:
+ enabled: false
+ remote_directory: dependency_proxy # The bucket name
+ # direct_upload: false # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false)
+ # background_upload: false # Temporary option to limit automatic upload (Default: true)
+ # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage
+ connection:
+ provider: AWS
+ aws_access_key_id: AWS_ACCESS_KEY_ID
+ aws_secret_access_key: AWS_SECRET_ACCESS_KEY
+ region: us-east-1
+ # host: 'localhost' # default: s3.amazonaws.com
+ # endpoint: 'http://127.0.0.1:9000' # default: nil
+ # aws_signature_version: 4 # For creation of signed URLs. Set to 2 if provider does not support v4.
+ # path_style: true # Use 'host/bucket_name/object' instead of 'bucket_name.host/object'
## GitLab Pages
pages:
@@ -306,6 +344,53 @@ production: &base
schedule_migrate_external_diffs_worker:
cron: "15 * * * *"
+ ##
+ # GitLab EE only jobs:
+
+ # Snapshot active users statistics
+ historical_data_worker:
+ cron: "0 12 * * *"
+
+ # In addition to refreshing users when they log in,
+ # periodically refresh LDAP users membership.
+ # NOTE: This will only take effect if LDAP is enabled
+ ldap_sync_worker:
+ cron: "30 1 * * *"
+
+ # Periodically refresh LDAP groups membership.
+ # NOTE: This will only take effect if LDAP is enabled
+ ldap_group_sync_worker:
+ cron: "0 * * * *"
+
+ # GitLab Geo metrics update worker
+ # NOTE: This will only take effect if Geo is enabled
+ geo_metrics_update_worker:
+ cron: "*/1 * * * *"
+
+ # GitLab Geo prune event log worker
+ # NOTE: This will only take effect if Geo is enabled (primary node only)
+ geo_prune_event_log_worker:
+ cron: "*/5 * * * *"
+
+ # GitLab Geo repository sync worker
+ # NOTE: This will only take effect if Geo is enabled (secondary nodes only)
+ geo_repository_sync_worker:
+ cron: "*/1 * * * *"
+
+ # GitLab Geo file download dispatch worker
+ # NOTE: This will only take effect if Geo is enabled (secondary nodes only)
+ geo_file_download_dispatch_worker:
+ cron: "*/1 * * * *"
+
+ # GitLab Geo migrated local files clean up worker
+ # NOTE: This will only take effect if Geo is enabled (secondary nodes only)
+ geo_migrated_local_files_clean_up_worker:
+ cron: "15 */6 * * *"
+
+ # Export pseudonymized data in CSV format for analysis
+ pseudonymizer_worker:
+ cron: "0 * * * *"
+
registry:
# enabled: true
# host: registry.example.com
@@ -323,6 +408,16 @@ production: &base
# clientside_dsn: https://<key>@sentry.io/<project>
# environment: 'production' # e.g. development, staging, production
+ ## Geo
+ # NOTE: These settings will only take effect if Geo is enabled
+ geo:
+ # This is an optional identifier which Geo nodes can use to identify themselves.
+ # For example, if external_url is the same for two secondaries, you must specify
+ # a unique Geo node name for those secondaries.
+ #
+ # If it is blank, it defaults to external_url.
+ node_name: ''
+
#
# 2. GitLab CI settings
# ==========================
@@ -349,6 +444,19 @@ production: &base
# bundle exec rake gitlab:ldap:check RAILS_ENV=production
ldap:
enabled: false
+
+ # This setting controls the number of seconds between LDAP permission checks
+ # for each user. After this time has expired for a given user, their next
+ # interaction with GitLab (a click in the web UI, a git pull, etc.) will be
+ # slower because the LDAP permission check is being performed. How much
+ # slower depends on your LDAP setup, but it is not uncommon for this check
+ # to add seconds of waiting time. The default value is to have a "slow
+ # click" once every 3600 seconds (i.e., once per hour).
+ #
+ # Warning: if you set this value too low, every click in GitLab will be a
+ # "slow click" for all of your LDAP users.
+ # sync_time: 3600
+
servers:
##########################################################################
#
@@ -446,6 +554,10 @@ production: &base
# A value of 0 means there is no timeout.
timeout: 10
+ # Enable smartcard authentication against the LDAP server. Valid values
+ # are "false", "optional", and "required".
+ smartcard_auth: false
+
# This setting specifies if LDAP server is Active Directory LDAP server.
# For non AD servers it skips the AD specific queries.
# If your LDAP server is not AD, set this to false.
@@ -485,6 +597,31 @@ production: &base
#
user_filter: ''
+ # Base where we can search for groups
+ #
+ # Ex. ou=Groups,dc=gitlab,dc=example
+ #
+ group_base: ''
+
+ # LDAP group of users who should be admins in GitLab
+ #
+ # Ex. GLAdmins
+ #
+ admin_group: ''
+
+ # LDAP group of users who should be marked as external users in GitLab
+ #
+ # Ex. ['Contractors', 'Interns']
+ #
+ external_groups: []
+
+ # Name of attribute which holds a ssh public key of the user object.
+ # If false or nil, SSH key syncronisation will be disabled.
+ #
+ # Ex. sshpublickey
+ #
+ sync_ssh_keys: false
+
# LDAP attributes that GitLab will use to create an account for the LDAP user.
# The specified attribute can either be the attribute name as a string (e.g. 'mail'),
# or an array of attribute names to try in order (e.g. ['mail', 'email']).
@@ -516,6 +653,38 @@ production: &base
# host:
# ....
+ ## Smartcard authentication settings
+ smartcard:
+ # Allow smartcard authentication
+ enabled: false
+
+ # Path to a file containing a CA certificate
+ ca_file: '/etc/ssl/certs/CA.pem'
+
+ # Port where the client side certificate is requested by the webserver (NGINX/Apache)
+ # client_certificate_required_port: 3444
+
+ ## Kerberos settings
+ kerberos:
+ # Allow the HTTP Negotiate authentication method for Git clients
+ enabled: false
+
+ # Kerberos 5 keytab file. The keytab file must be readable by the GitLab user,
+ # and should be different from other keytabs in the system.
+ # (default: use default keytab from Krb5 config)
+ # keytab: /etc/http.keytab
+
+ # The Kerberos service name to be used by GitLab.
+ # (default: accept any service name in keytab file)
+ # service_principal_name: HTTP/gitlab.example.com@EXAMPLE.COM
+
+ # Dedicated port: Git before 2.4 does not fall back to Basic authentication if Negotiate fails.
+ # To support both Basic and Negotiate methods with older versions of Git, configure
+ # nginx to proxy GitLab on an extra port (e.g. 8443) and uncomment the following lines
+ # to dedicate this port to Kerberos authentication. (default: false)
+ # use_dedicated_port: true
+ # port: 8443
+ # https: true
## OmniAuth settings
omniauth:
@@ -635,6 +804,8 @@ production: &base
# name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient'
# } }
#
+ # - { name: 'group_saml' }
+ #
# - { name: 'crowd',
# args: {
# crowd_server_url: 'CROWD SERVER URL',
@@ -707,6 +878,20 @@ production: &base
# # Specifies Amazon S3 storage class to use for backups, this is optional
# # storage_class: 'STANDARD'
+ ## Pseudonymizer exporter
+ pseudonymizer:
+ # Tables manifest that specifies the fields to extract and pseudonymize.
+ manifest: config/pseudonymizer.yml
+ upload:
+ remote_directory: 'gitlab-elt'
+ # Fog storage connection settings, see http://fog.io/storage/ .
+ connection:
+ # provider: AWS
+ # region: eu-west-1
+ # aws_access_key_id: AKIAKIAKI
+ # aws_secret_access_key: 'secret123'
+ # # The remote 'directory' to store the CSV files. For S3, this would be the bucket name.
+
## GitLab Shell settings
gitlab_shell:
path: /home/git/gitlab-shell/
@@ -872,6 +1057,17 @@ test:
token: secret
backup:
path: tmp/tests/backups
+ pseudonymizer:
+ manifest: config/pseudonymizer.yml
+ upload:
+ # The remote 'directory' to store the CSV files. For S3, this would be the bucket name.
+ remote_directory: gitlab-elt.test
+ # Fog storage connection settings, see http://fog.io/storage/
+ connection:
+ provider: AWS # Only AWS supported at the moment
+ aws_access_key_id: AWS_ACCESS_KEY_ID
+ aws_secret_access_key: AWS_SECRET_ACCESS_KEY
+ region: us-east-1
gitlab_shell:
path: tmp/tests/gitlab-shell/
authorized_keys_file: tmp/tests/authorized_keys
@@ -892,6 +1088,7 @@ test:
external_providers: []
providers:
+ - { name: 'group_saml' }
- { name: 'cas3',
label: 'cas3',
args: { url: 'https://sso.example.com',
config/initializers/0_inflections.rb: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30041
diff --git a/config/initializers/0_inflections.rb b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/initializers/0_inflections.rb
index 1ad9ddca877..4d1f4917275 100644
--- a/config/initializers/0_inflections.rb
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/initializers/0_inflections.rb
@@ -14,6 +14,14 @@ ActiveSupport::Inflector.inflections do |inflect|
award_emoji
project_statistics
system_note_metadata
+ event_log
project_auto_devops
+ project_registry
+ file_registry
+ job_artifact_registry
+ vulnerability_feedback
+ vulnerabilities_feedback
+ group_view
)
+ inflect.acronym 'EE'
end
config/initializers/1_settings.rb: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/14443
diff --git a/config/initializers/1_settings.rb b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/initializers/1_settings.rb
index 0c8d94ccaed..cca195d7118 100644
--- a/config/initializers/1_settings.rb
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/initializers/1_settings.rb
@@ -4,6 +4,10 @@ require_relative '../object_store_settings'
# Default settings
Settings['ldap'] ||= Settingslogic.new({})
Settings.ldap['enabled'] = false if Settings.ldap['enabled'].nil?
+Settings.ldap['sync_time'] = 3600 if Settings.ldap['sync_time'].nil?
+Settings.ldap['schedule_sync_daily'] = 1 if Settings.ldap['schedule_sync_daily'].nil?
+Settings.ldap['schedule_sync_hour'] = 1 if Settings.ldap['schedule_sync_hour'].nil?
+Settings.ldap['schedule_sync_minute'] = 30 if Settings.ldap['schedule_sync_minute'].nil?
# backwards compatibility, we only have one host
if Settings.ldap['enabled'] || Rails.env.test?
@@ -23,11 +27,14 @@ if Settings.ldap['enabled'] || Rails.env.test?
server['timeout'] ||= 10.seconds
server['block_auto_created_users'] = false if server['block_auto_created_users'].nil?
server['allow_username_or_email_login'] = false if server['allow_username_or_email_login'].nil?
+ server['smartcard_auth'] = false unless %w[optional required].include?(server['smartcard_auth'])
server['active_directory'] = true if server['active_directory'].nil?
server['attributes'] = {} if server['attributes'].nil?
server['lowercase_usernames'] = false if server['lowercase_usernames'].nil?
server['provider_name'] ||= "ldap#{key}".downcase
server['provider_class'] = OmniAuth::Utils.camelize(server['provider_name'])
+ server['external_groups'] = [] if server['external_groups'].nil?
+ server['sync_ssh_keys'] = 'sshPublicKey' if server['sync_ssh_keys'].to_s == 'true'
# For backwards compatibility
server['encryption'] ||= server['method']
@@ -62,6 +69,10 @@ if Settings.ldap['enabled'] || Rails.env.test?
end
end
+Settings['smartcard'] ||= Settingslogic.new({})
+Settings.smartcard['enabled'] = false if Settings.smartcard['enabled'].nil?
+Settings.smartcard['client_certificate_required_port'] = 3444 if Settings.smartcard['client_certificate_required_port'].nil?
+
Settings['omniauth'] ||= Settingslogic.new({})
Settings.omniauth['enabled'] = true if Settings.omniauth['enabled'].nil?
Settings.omniauth['auto_sign_in_with_provider'] = false if Settings.omniauth['auto_sign_in_with_provider'].nil?
@@ -127,6 +138,7 @@ Settings['issues_tracker'] ||= {}
#
Settings['gitlab'] ||= Settingslogic.new({})
Settings.gitlab['default_project_creation'] ||= ::Gitlab::Access::DEVELOPER_MAINTAINER_PROJECT_ACCESS
+Settings.gitlab['default_project_deletion_protection'] ||= false
Settings.gitlab['default_projects_limit'] ||= 100000
Settings.gitlab['default_branch_protection'] ||= 2
Settings.gitlab['default_can_create_group'] = true if Settings.gitlab['default_can_create_group'].nil?
@@ -163,6 +175,9 @@ Settings.gitlab['webhook_timeout'] ||= 10
Settings.gitlab['max_attachment_size'] ||= 10
Settings.gitlab['session_expire_delay'] ||= 10080
Settings.gitlab['unauthenticated_session_expire_delay'] ||= 2.hours.to_i
+Settings.gitlab['mirror_max_delay'] ||= 300
+Settings.gitlab['mirror_max_capacity'] ||= 30
+Settings.gitlab['mirror_capacity_threshold'] ||= 15
Settings.gitlab.default_projects_features['issues'] = true if Settings.gitlab.default_projects_features['issues'].nil?
Settings.gitlab.default_projects_features['merge_requests'] = true if Settings.gitlab.default_projects_features['merge_requests'].nil?
Settings.gitlab.default_projects_features['wiki'] = true if Settings.gitlab.default_projects_features['wiki'].nil?
@@ -177,6 +192,13 @@ Settings.gitlab['no_todos_messages'] ||= YAML.load_file(Rails.root.join('config'
Settings.gitlab['impersonation_enabled'] ||= true if Settings.gitlab['impersonation_enabled'].nil?
Settings.gitlab['usage_ping_enabled'] = true if Settings.gitlab['usage_ping_enabled'].nil?
+#
+# Elasticseacrh
+#
+Settings['elasticsearch'] ||= Settingslogic.new({})
+Settings.elasticsearch['enabled'] = false if Settings.elasticsearch['enabled'].nil?
+Settings.elasticsearch['url'] = ENV['ELASTIC_URL'] || "http://localhost:9200"
+
#
# CI
#
@@ -245,6 +267,13 @@ Settings.pages['artifacts_server'] ||= Settings.pages['enabled'] if Settings.pa
Settings.pages['admin'] ||= Settingslogic.new({})
Settings.pages.admin['certificate'] ||= ''
+#
+# Geo
+#
+Settings['geo'] ||= Settingslogic.new({})
+# For backwards compatibility, default to gitlab_url and if so, ensure it ends with "/"
+Settings.geo['node_name'] = Settings.geo['node_name'].presence || Settings.gitlab['url'].chomp('/').concat('/')
+
#
# External merge request diffs
#
@@ -271,6 +300,27 @@ Settings.uploads['base_dir'] = Settings.uploads['base_dir'] || 'uploads/-/system
Settings.uploads['object_store'] = ObjectStoreSettings.parse(Settings.uploads['object_store'])
Settings.uploads['object_store']['remote_directory'] ||= 'uploads'
+#
+# Packages
+#
+Settings['packages'] ||= Settingslogic.new({})
+Settings.packages['enabled'] = true if Settings.packages['enabled'].nil?
+Settings.packages['storage_path'] = Settings.absolute(Settings.packages['storage_path'] || File.join(Settings.shared['path'], "packages"))
+Settings.packages['object_store'] = ObjectStoreSettings.parse(Settings.packages['object_store'])
+
+#
+# Dependency Proxy
+#
+Settings['dependency_proxy'] ||= Settingslogic.new({})
+Settings.dependency_proxy['enabled'] = true if Settings.dependency_proxy['enabled'].nil?
+Settings.dependency_proxy['storage_path'] = Settings.absolute(Settings.dependency_proxy['storage_path'] || File.join(Settings.shared['path'], "dependency_proxy"))
+Settings.dependency_proxy['object_store'] = ObjectStoreSettings.parse(Settings.dependency_proxy['object_store'])
+# For first iteration dependency proxy uses Rails server to download blobs.
+# To ensure acceptable performance we only allow feature to be used with
+# multithreaded web-server Puma. This will be removed once download logic is moved
+# to GitLab workhorse
+Settings.dependency_proxy['enabled'] = false unless defined?(::Puma)
+
#
# Mattermost
#
@@ -309,6 +359,36 @@ Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker'
Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *'
Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker'
+Settings.cron_jobs['historical_data_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['historical_data_worker']['cron'] ||= '0 12 * * *'
+Settings.cron_jobs['historical_data_worker']['job_class'] = 'HistoricalDataWorker'
+Settings.cron_jobs['ldap_sync_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['ldap_sync_worker']['cron'] ||= '30 1 * * *'
+Settings.cron_jobs['ldap_sync_worker']['job_class'] = 'LdapSyncWorker'
+Settings.cron_jobs['ldap_group_sync_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['ldap_group_sync_worker']['cron'] ||= '0 * * * *'
+Settings.cron_jobs['ldap_group_sync_worker']['job_class'] = 'LdapAllGroupsSyncWorker'
+Settings.cron_jobs['geo_metrics_update_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['geo_metrics_update_worker']['cron'] ||= '*/1 * * * *'
+Settings.cron_jobs['geo_metrics_update_worker']['job_class'] ||= 'Geo::MetricsUpdateWorker'
+Settings.cron_jobs['geo_repository_sync_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['geo_repository_sync_worker']['cron'] ||= '*/1 * * * *'
+Settings.cron_jobs['geo_repository_sync_worker']['job_class'] ||= 'Geo::RepositorySyncWorker'
+Settings.cron_jobs['geo_file_download_dispatch_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['geo_file_download_dispatch_worker']['cron'] ||= '*/1 * * * *'
+Settings.cron_jobs['geo_file_download_dispatch_worker']['job_class'] ||= 'Geo::FileDownloadDispatchWorker'
+Settings.cron_jobs['geo_prune_event_log_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['geo_prune_event_log_worker']['cron'] ||= '*/5 * * * *'
+Settings.cron_jobs['geo_prune_event_log_worker']['job_class'] ||= 'Geo::PruneEventLogWorker'
+Settings.cron_jobs['geo_repository_verification_primary_batch_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['geo_repository_verification_primary_batch_worker']['cron'] ||= '*/1 * * * *'
+Settings.cron_jobs['geo_repository_verification_primary_batch_worker']['job_class'] ||= 'Geo::RepositoryVerification::Primary::BatchWorker'
+Settings.cron_jobs['geo_repository_verification_secondary_scheduler_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['geo_repository_verification_secondary_scheduler_worker']['cron'] ||= '*/1 * * * *'
+Settings.cron_jobs['geo_repository_verification_secondary_scheduler_worker']['job_class'] ||= 'Geo::RepositoryVerification::Secondary::SchedulerWorker'
+Settings.cron_jobs['geo_migrated_local_files_clean_up_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['geo_migrated_local_files_clean_up_worker']['cron'] ||= '15 */6 * * *'
+Settings.cron_jobs['geo_migrated_local_files_clean_up_worker']['job_class'] ||= 'Geo::MigratedLocalFilesCleanUpWorker'
Settings.cron_jobs['import_export_project_cleanup_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['import_export_project_cleanup_worker']['cron'] ||= '0 * * * *'
Settings.cron_jobs['import_export_project_cleanup_worker']['job_class'] = 'ImportExportProjectCleanupWorker'
@@ -341,6 +421,14 @@ Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.__send__(:cron_for_usage_ping)
Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker'
+Settings.cron_jobs['pseudonymizer_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['pseudonymizer_worker']['cron'] ||= '0 23 * * *'
+Settings.cron_jobs['pseudonymizer_worker']['job_class'] ||= 'PseudonymizerWorker'
+
+Settings.cron_jobs['clear_shared_runners_minutes_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['clear_shared_runners_minutes_worker']['cron'] ||= '0 0 1 * *'
+Settings.cron_jobs['clear_shared_runners_minutes_worker']['job_class'] = 'ClearSharedRunnersMinutesWorker'
+
Settings.cron_jobs['stuck_merge_jobs_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['stuck_merge_jobs_worker']['cron'] ||= '0 */2 * * *'
Settings.cron_jobs['stuck_merge_jobs_worker']['job_class'] = 'StuckMergeJobsWorker'
@@ -365,6 +453,10 @@ Settings.cron_jobs['schedule_migrate_external_diffs_worker'] ||= Settingslogic.n
Settings.cron_jobs['schedule_migrate_external_diffs_worker']['cron'] ||= '15 * * * *'
Settings.cron_jobs['schedule_migrate_external_diffs_worker']['job_class'] = 'ScheduleMigrateExternalDiffsWorker'
+Settings.cron_jobs['update_max_seats_used_for_gitlab_com_subscriptions_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['update_max_seats_used_for_gitlab_com_subscriptions_worker']['cron'] ||= '0 12 * * *'
+Settings.cron_jobs['update_max_seats_used_for_gitlab_com_subscriptions_worker']['job_class'] = 'UpdateMaxSeatsUsedForGitlabComSubscriptionsWorker'
+
#
# Sidekiq
#
@@ -443,6 +535,14 @@ Settings.backup['upload']['encryption'] ||= nil
Settings.backup['upload']['encryption_key'] ||= ENV['GITLAB_BACKUP_ENCRYPTION_KEY']
Settings.backup['upload']['storage_class'] ||= nil
+#
+# Pseudonymizer
+#
+Settings['pseudonymizer'] ||= Settingslogic.new({})
+Settings.pseudonymizer['manifest'] = Settings.absolute(Settings.pseudonymizer['manifest'] || Rails.root.join("config/pseudonymizer.yml"))
+Settings.pseudonymizer['upload'] ||= Settingslogic.new({ 'remote_directory' => nil, 'connection' => nil })
+# Settings.pseudonymizer['upload']['multipart_chunk_size'] ||= 104857600
+
#
# Git
#
@@ -455,6 +555,21 @@ Settings.git['bin_path'] ||= '/usr/bin/git'
Settings['satellites'] ||= Settingslogic.new({})
Settings.satellites['path'] = Settings.absolute(Settings.satellites['path'] || "tmp/repo_satellites/")
+#
+# Kerberos
+#
+Settings['kerberos'] ||= Settingslogic.new({})
+Settings.kerberos['enabled'] = false if Settings.kerberos['enabled'].nil?
+Settings.kerberos['keytab'] = nil if Settings.kerberos['keytab'].blank? # nil means use default keytab
+Settings.kerberos['service_principal_name'] = nil if Settings.kerberos['service_principal_name'].blank? # nil means any SPN in keytab
+Settings.kerberos['use_dedicated_port'] = false if Settings.kerberos['use_dedicated_port'].nil?
+Settings.kerberos['https'] = Settings.gitlab.https if Settings.kerberos['https'].nil?
+Settings.kerberos['port'] ||= Settings.kerberos.https ? 8443 : 8088
+
+if Settings.kerberos['enabled'] && !Settings.omniauth.providers.map(&:name).include?('kerberos_spnego')
+ Settings.omniauth.providers << Settingslogic.new({ 'name' => 'kerberos_spnego' })
+end
+
#
# Extra customization
#
config/initializers/ar_speed_up_migration_checking.rb: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30529
diff --git a/config/initializers/ar_speed_up_migration_checking.rb b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/initializers/ar_speed_up_migration_checking.rb
index aae774daa35..f98b246db0b 100644
--- a/config/initializers/ar_speed_up_migration_checking.rb
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/initializers/ar_speed_up_migration_checking.rb
@@ -10,7 +10,8 @@ if Rails.env.test?
# it reads + parses `db/migrate/*` each time. Memoizing it can save 0.5
# seconds per spec.
def migrations(paths)
- (@migrations ||= migrations_unmemoized(paths)).dup
+ @migrations ||= {}
+ (@migrations[paths] ||= migrations_unmemoized(paths)).dup
end
end
end
config/initializers/console_message.rb: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30484
diff --git a/config/initializers/console_message.rb b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/initializers/console_message.rb
index 05eb395028d..3a23cc6d435 100644
--- a/config/initializers/console_message.rb
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/initializers/console_message.rb
@@ -6,5 +6,13 @@ if defined?(Rails::Console)
puts " GitLab:".ljust(justify) + "#{Gitlab::VERSION} (#{Gitlab.revision})"
puts " GitLab Shell:".ljust(justify) + "#{Gitlab::VersionInfo.parse(Gitlab::Shell.new.version)}"
puts " #{Gitlab::Database.human_adapter_name}:".ljust(justify) + Gitlab::Database.version
+
+ # EE-specific start
+ if Gitlab::Geo.enabled?
+ puts " Geo enabled:".ljust(justify) + 'yes'
+ puts " Geo server:".ljust(justify) + EE::GeoHelper.current_node_human_status
+ end
+ # EE specific end
+
puts "-------------------------------------------------------------------------------------"
end
config/initializers/health_check.rb: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30484
diff --git a/config/initializers/health_check.rb b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/initializers/health_check.rb
index 959daa93f78..b445cb77529 100644
--- a/config/initializers/health_check.rb
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/initializers/health_check.rb
@@ -1,4 +1,8 @@
HealthCheck.setup do |config|
config.standard_checks = %w(database migrations cache)
config.full_checks = %w(database migrations cache)
+
+ config.add_custom_check('geo') do
+ Gitlab::Geo::HealthCheck.new.perform_checks
+ end
end
config/initializers/sidekiq.rb: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30484
diff --git a/config/initializers/sidekiq.rb b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/initializers/sidekiq.rb
index 7b69cf11288..f1c6547e99c 100644
--- a/config/initializers/sidekiq.rb
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/initializers/sidekiq.rb
@@ -77,6 +77,10 @@ Sidekiq.configure_server do |config|
end
Sidekiq::Cron::Job.load_from_hash! cron_jobs
+ Gitlab::Mirror.configure_cron_job!
+
+ Gitlab::Geo.configure_cron_jobs!
+
Gitlab::SidekiqVersioning.install!
db_config = Gitlab::Database.config ||
@@ -85,6 +89,14 @@ Sidekiq.configure_server do |config|
ActiveRecord::Base.establish_connection(db_config)
Rails.logger.debug("Connection Pool size for Sidekiq Server is now: #{ActiveRecord::Base.connection.pool.instance_variable_get('@size')}")
+ # EE only
+ if Gitlab::Geo.geo_database_configured?
+ Rails.configuration.geo_database['pool'] = Sidekiq.options[:concurrency]
+ Geo::TrackingBase.establish_connection(Rails.configuration.geo_database)
+
+ Rails.logger.debug("Connection Pool size for Sidekiq Server is now: #{Geo::TrackingBase.connection_pool.size} (Geo tracking database)")
+ end
+
# Avoid autoload issue such as 'Mail::Parsers::AddressStruct'
# https://github.com/mikel/mail/issues/912#issuecomment-214850355
Mail.eager_autoload!
config/initializers/zz_metrics.rb: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30802
diff --git a/config/initializers/zz_metrics.rb b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/initializers/zz_metrics.rb
index 5aa6f73c5c5..b190012d535 100644
--- a/config/initializers/zz_metrics.rb
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/initializers/zz_metrics.rb
@@ -6,6 +6,7 @@
# that we can stub it for testing, as it is only called when metrics are
# enabled.
#
+# rubocop:disable Metrics/AbcSize
def instrument_classes(instrumentation)
instrumentation.instrument_instance_methods(Gitlab::Shell)
@@ -86,12 +87,40 @@ def instrument_classes(instrumentation)
instrumentation.instrument_methods(Gitlab::Highlight)
instrumentation.instrument_instance_methods(Gitlab::Highlight)
+ instrumentation.instrument_methods(Elasticsearch::Git::Repository)
+ instrumentation.instrument_instance_methods(Elasticsearch::Git::Repository)
+
+ instrumentation.instrument_instance_methods(Search::GlobalService)
+ instrumentation.instrument_instance_methods(Search::ProjectService)
+
+ instrumentation.instrument_instance_methods(Gitlab::Elastic::SearchResults)
+ instrumentation.instrument_instance_methods(Gitlab::Elastic::ProjectSearchResults)
+ instrumentation.instrument_instance_methods(Gitlab::Elastic::Indexer)
+ instrumentation.instrument_instance_methods(Gitlab::Elastic::SnippetSearchResults)
+ instrumentation.instrument_methods(Gitlab::Elastic::Helper)
+
+ instrumentation.instrument_instance_methods(Elastic::ApplicationSearch)
+ instrumentation.instrument_instance_methods(Elastic::IssuesSearch)
+ instrumentation.instrument_instance_methods(Elastic::MergeRequestsSearch)
+ instrumentation.instrument_instance_methods(Elastic::MilestonesSearch)
+ instrumentation.instrument_instance_methods(Elastic::NotesSearch)
+ instrumentation.instrument_instance_methods(Elastic::ProjectsSearch)
+ instrumentation.instrument_instance_methods(Elastic::RepositoriesSearch)
+ instrumentation.instrument_instance_methods(Elastic::SnippetsSearch)
+ instrumentation.instrument_instance_methods(Elastic::WikiRepositoriesSearch)
+
+ instrumentation.instrument_instance_methods(Gitlab::BitbucketImport::Importer)
+ instrumentation.instrument_instance_methods(Bitbucket::Connection)
+
+ instrumentation.instrument_instance_methods(Geo::RepositorySyncWorker)
+
# This is a Rails scope so we have to instrument it manually.
instrumentation.instrument_method(Project, :visible_to_user)
# Needed for https://gitlab.com/gitlab-org/gitlab-ce/issues/30224#note_32306159
instrumentation.instrument_instance_method(MergeRequestDiff, :load_commits)
end
+# rubocop:enable Metrics/AbcSize
# With prometheus enabled by default this breaks all specs
# that stubs methods using `any_instance_of` for the models reloaded here.
config/prometheus/common_metrics.yml: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/29606
diff --git a/config/prometheus/common_metrics.yml b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/prometheus/common_metrics.yml
index 3c67ca36791..f9ce5a6f365 100644
--- a/config/prometheus/common_metrics.yml
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/prometheus/common_metrics.yml
@@ -171,7 +171,7 @@ panel_groups:
weight: 2
metrics:
- id: system_metrics_kubernetes_container_memory_average
- query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024'
+ query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024'
label: Pod average
unit: MB
- title: "Canary: Memory Usage (Pod Average)"
@@ -190,7 +190,7 @@ panel_groups:
weight: 1
metrics:
- id: system_metrics_kubernetes_container_core_usage
- query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name))'
+ query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name))'
label: Pod average
unit: "cores"
- title: "Canary: Core Usage (Pod Average)"
config/settings.rb: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30529
diff --git a/config/settings.rb b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/settings.rb
index 1b94df785a7..3f3ea16c0fb 100644
--- a/config/settings.rb
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/settings.rb
@@ -56,6 +56,31 @@ class Settings < Settingslogic
(base_url(gitlab) + [gitlab.relative_url_root]).join('')
end
+ def kerberos_protocol
+ kerberos.https ? "https" : "http"
+ end
+
+ def kerberos_port
+ kerberos.use_dedicated_port ? kerberos.port : gitlab.port
+ end
+
+ # Curl expects username/password for authentication. However when using GSS-Negotiate not credentials should be needed.
+ # By inserting in the Kerberos dedicated URL ":@", we give to curl an empty username and password and GSS auth goes ahead
+ # Known bug reported in http://sourceforge.net/p/curl/bugs/440/ and http://curl.haxx.se/docs/knownbugs.html
+ def build_gitlab_kerberos_url
+ [
+ kerberos_protocol,
+ "://:@",
+ gitlab.host,
+ ":#{kerberos_port}",
+ gitlab.relative_url_root
+ ].join('')
+ end
+
+ def alternative_gitlab_kerberos_url?
+ kerberos.enabled && (build_gitlab_kerberos_url != build_gitlab_url)
+ end
+
# check that values in `current` (string or integer) is a contant in `modul`.
def verify_constant_array(modul, current, default)
values = default || []
config/sidekiq_queues.yml: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/29606
diff --git a/config/sidekiq_queues.yml b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/sidekiq_queues.yml
index fd9ce4d3374..5b42217d0ca 100644
--- a/config/sidekiq_queues.yml
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/sidekiq_queues.yml
@@ -93,3 +93,26 @@
- [migrate_external_diffs, 1]
- [update_project_statistics, 1]
- [phabricator_import_import_tasks, 1]
+
+ # EE-specific queues
+ - [ldap_group_sync, 2]
+ - [create_github_webhook, 2]
+ - [geo, 1]
+ - [repository_update_mirror, 1]
+ - [new_epic, 2]
+ - [project_import_schedule, 1]
+ - [project_update_repository_storage, 1]
+ - [admin_emails, 1]
+ - [elastic_batch_project_indexer, 1]
+ - [elastic_indexer, 1]
+ - [elastic_full_index, 1]
+ - [elastic_commit_indexer, 1]
+ - [elastic_namespace_indexer, 1]
+ - [export_csv, 1]
+ - [incident_management, 2]
+
+ # Deprecated queues: Remove after 10.7
+ - geo_base_scheduler
+ - geo_file_download
+ - geo_project_sync
+ - geo_repository_shard_sync
Edited by Robert Speicher