Commit 17595f2c authored by Toon Claes's avatar Toon Claes

Merge branch 'zj-rakefile-gitaly' into 'master'

Move Gitaly config to Rake tasks

Closes #546

See merge request gitlab-development-kit!717
parents 1096e87d 2cfbb9b7
......@@ -70,6 +70,7 @@ gitlab-development-kit/**
/.gdk-install-root
/builds
/gitaly/
praefect_enabled
/gitlab-workhorse/config.toml
/gitlab-workhorse/.cache
/go-gitlab-shell
......
......@@ -92,14 +92,7 @@ auto_devops_registry_port: auto_devops_gitlab_port
expr ${auto_devops_gitlab_port} + 5000 > $@
gitlab/config/gitlab.yml: support/templates/gitlab.yml.erb auto_devops_enabled auto_devops_gitlab_port auto_devops_registry_port
hostname=${hostname} port=${port} relative_url_root=${relative_url_root}\
https=${https}\
webpack_port=${webpack_port}\
registry_host=${registry_host} registry_external_port=${registry_external_port}\
registry_enabled=${registry_enabled} registry_port=${registry_port}\
object_store_enabled=${object_store_enabled} object_store_port=${object_store_port}\
gitlab_pages_port=${gitlab_pages_port}\
support/edit-gitlab-yml gitlab/config/gitlab.yml
rake gitlab/config/gitlab.yml
gitlab/config/database.yml: database.yml.example
bin/safe-sed "$@" \
......@@ -183,7 +176,7 @@ gitlab-shell/.gitlab_shell_secret:
# Set up gitaly
gitaly-setup: gitaly/bin/gitaly gitaly/config.toml ${gitaly_proto_clone_dir}/.git
gitaly-setup: gitaly/bin/gitaly gitaly/gitaly.config.toml ${gitaly_proto_clone_dir}/.git
${gitaly_clone_dir}/.git:
git clone --quiet --branch "${gitaly_version}" ${git_depth_param} ${gitaly_repo} ${gitaly_clone_dir}
......@@ -191,17 +184,8 @@ ${gitaly_clone_dir}/.git:
${gitaly_proto_clone_dir}/.git:
git clone ${git_depth_param} --quiet ${gitaly_proto_repo} ${gitaly_proto_clone_dir}
gitaly/config.toml: $(gitaly_clone_dir)/config.toml.example
bin/safe-sed "$@" \
-e "s|/home/git|${gitlab_development_root}|g" \
-e "s|^socket_path.*|socket_path = \"${gitlab_development_root}/gitaly.socket\"|" \
-e "s|^bin_dir.*|bin_dir = \"${gitlab_development_root}/gitaly/bin\"|" \
-e "s|# prometheus_listen_addr|prometheus_listen_addr|" \
-e "s|# \[logging\]|\[logging\]|" \
-e "s|# level = \"warn\"|level = \"warn\"|" \
-e "s|^#[^[]*\[git\].*|\[git\]|" \
-e "s|^# catfile_cache_size.*|catfile_cache_size = 5|" \
"$<"
gitaly/gitaly.config.toml: support/templates/gitaly.config.toml.erb
rake gitaly/gitaly.config.toml
prom-setup:
if [ "$(uname -s)" = "Linux" ]; then \
......@@ -627,14 +611,13 @@ clean-config:
.ruby-version \
Procfile \
gitlab-workhorse/config.toml \
gitaly/config.toml \
gitaly/gitaly.config.toml \
nginx/conf/nginx.conf \
registry/config.yml \
jaeger
touch-examples:
touch \
$(gitaly_clone_dir)/config.toml.example \
Procfile.erb \
database.yml.example \
database_geo.yml.example \
......@@ -650,6 +633,8 @@ touch-examples:
redis/redis.conf.example \
redis/resque.yml.example \
registry/config.yml.example \
support/templates/gitaly.toml.erb \
support/templates/praefect.toml.erb \
support/templates/gitlab.yml.erb
unlock-dependency-installers:
......
......@@ -19,7 +19,8 @@ postgresql: exec support/postgresql-signal-wrapper <%= config.postgresql.bin_dir
# Backend services
#
gitaly: exec gitaly/bin/gitaly <%= config.gdk_root %>/gitaly/config.toml
gitaly: exec gitaly/bin/gitaly <%= config.gdk_root %>/gitaly/gitaly.config.toml
<%= '#' unless config.praefect? %>praefect: exec gitaly/bin/praefect -config <%= config.gdk_root %>/gitaly/praefect.config.toml
gitlab-workhorse: exec /usr/bin/env PATH="<%= config.gdk_root %>/gitlab-workhorse/bin:$PATH" gitlab-workhorse -authSocket <%= config.gdk_root %>/gitlab.socket -listenAddr $host:<%= config.nginx? ? config.nginx.workhorse_port : '$port' %> -documentRoot <%= config.gdk_root %>/gitlab/public -developmentMode -secretPath <%= config.gdk_root %>/gitlab/.gitlab_workhorse_secret -config <%= config.gdk_root %>/gitlab-workhorse/config.toml
rails-background-jobs: exec /usr/bin/env SIDEKIQ_LOG_ARGUMENTS=1 RAILS_ENV=development RAILS_RELATIVE_URL_ROOT=$relative_url_root support/exec-cd gitlab bin/background_jobs start_foreground
......
......@@ -4,7 +4,9 @@ $LOAD_PATH.unshift('.')
require 'lib/gdk'
require 'rake/clean'
CLOBBER.include 'gdk.example.yml', 'Procfile', 'nginx/conf/nginx.conf'
CONFIGS = FileList['Procfile', 'nginx/conf/nginx.conf', 'gitlab/config/gitlab.yml']
CLOBBER.include *CONFIGS, 'gdk.example.yml'
def config
@config ||= GDK::Config.new
......@@ -27,6 +29,12 @@ file 'gdk.example.yml' => 'clobber:gdk.example.yml' do |t|
end
end
desc 'Regenerate all config files from scratch'
task reconfigure: [:clobber, :all]
desc 'Generate all config files'
task all: CONFIGS
task 'clobber:gdk.example.yml' do |t|
Rake::Cleaner.cleanup_files([t.name])
end
......@@ -40,3 +48,34 @@ desc 'Generate nginx configuration'
file 'nginx/conf/nginx.conf' => ['nginx/conf/nginx.conf.erb', GDK::Config::FILE] do |t|
GDK::ErbRenderer.new(t.source, t.name).safe_render!
end
desc 'Generate the gitlab.yml config file'
file 'gitlab/config/gitlab.yml' => ['support/templates/gitlab.yml.erb'] do |t|
GDK::ErbRenderer.new(t.source, t.name).render!
end
desc 'Generate Gitaly config toml'
file 'gitaly/gitaly.config.toml' => ['support/templates/gitaly.config.toml.erb'] do |t|
GDK::ErbRenderer.new(t.source, t.name).render!
end
desc 'Generate Praefects config toml'
file 'gitaly/praefect.config.toml' => ['support/templates/praefect.config.toml.erb'] do |t|
GDK::ErbRenderer.new(t.source, t.name).render!
end
namespace :praefect do
PRAEFECT_ENABLED_PATH = 'praefect_enabled'
desc 'Enable praefect and configure it to run'
task :enable => 'gitaly/praefect.config.toml' do
File.write(PRAEFECT_ENABLED_PATH, 'true')
Rake::Task[:reconfigure].invoke
end
desc 'Disable praefect and do not run it'
task :disable do
File.delete(PRAEFECT_ENABLED_PATH)
Rake::Task[:reconfigure].invoke
end
end
......@@ -42,6 +42,8 @@ postgresql:
replica_dir: "/home/git/gdk/postgresql-replica"
replication_user: gitlab_replication
protocol: https
praefect:
enabled: false
registry:
enabled: false
external_port: 5000
......
# frozen_string_literal: true
require 'etc'
require_relative 'config_settings'
module GDK
......@@ -16,7 +17,7 @@ module GDK
r.gitlab_docs 'https://gitlab.com/gitlab-com/gitlab-docs.git'
end
gdk_root { ENV['PWD'] }
gdk_root { Dir.pwd }
hostname do
next "#{config.auto_devops.gitlab.port}.qa-tunnel.gitlab.info" if config.auto_devops.enabled
......@@ -38,7 +39,7 @@ module GDK
protocol { config.https? ? 'https' : 'http' }
relative_url_root { read!('relative_url_root') || nil }
username { ENV['USERNAME'] }
username { Etc.getlogin }
webpack do |w|
w.port { read!('webpack_port') || 3808 }
......@@ -128,6 +129,17 @@ module GDK
gitaly do |g|
g.assembly_dir { "#{config.gdk_root}/gitaly/assembly" }
g.address do
if config.praefect?
File.join(config.gdk_root, 'praefect.socket')
else
File.join(config.gdk_root, 'gitaly.socket')
end
end
end
praefect do |p|
p.enabled { read!('praefect_enabled') || false }
end
sshd do |s|
......
......@@ -5,7 +5,7 @@ def main(argv)
when 'geo_db'
foreman_exec(%w[postgresql-geo])
when 'app'
svcs = %w[gitlab-workhorse nginx grafana sshd gitaly storage-check gitlab-pages]
svcs = %w[gitlab-workhorse nginx grafana sshd gitaly storage-check gitlab-pages praefect]
foreman_exec(svcs + %w[rails-web rails-background-jobs])
when 'grafana'
......
#!/usr/bin/env ruby
require 'erb'
require 'etc'
def main(file_path, template)
File.open(file_path, 'w') { |f| f.puts template.result(binding) }
end
def env_or_default(key, default)
ENV[key] || default
end
raw_template = File.read(File.join(__dir__, 'templates', 'gitlab.yml.erb'))
TEMPLATE = ERB.new(raw_template)
GDK_ROOT = Dir.pwd
GIT_BIN = `which git`
file_name = ARGV.first
abort "no file name as argument found" unless file_name
main(ARGV.first, TEMPLATE)
# Example Gitaly configuration file
socket_path = "<%= File.join(config.gdk_root, 'gitaly.socket') %>"
# The directory where Gitaly's executables are stored
bin_dir = "<%= File.join(config.gdk_root, 'gitaly', 'bin') %>"
# # Optional: listen on a TCP socket. This is insecure (no authentication)
# listen_addr = "localhost:9999"
# tls_listen_addr = "localhost:8888
# # Optional: export metrics via Prometheus
# prometheus_listen_addr = "localhost:9236"
# # Optional: authenticate Gitaly requests using a shared secret
# [auth]
# token = 'abc123secret'
# transitioning = false # Set `transitioning` to true to temporarily allow unauthenticated while rolling out authentication.
# [tls]
# certificate_path = '/home/git/cert.cert'
# key_path = '/home/git/key.pem'
# # Git settings
# [git]
bin_path = "<%= config.git.bin %>"
catfile_cache_size = 10
[[storage]]
name = "default"
path = "<%= File.join(config.gdk_root, 'repositories') %>"
# # You can optionally configure more storages for this Gitaly instance to serve up
#
# [[storage]]
# name = "other_storage"
# path = "/mnt/other_storage/repositories"
#
# # You can optionally configure Gitaly to output JSON-formatted log messages to stdout
# [logging]
# format = "json"
# # Optional: Set log level to only log entries with that severity or above
# # One of, in order: debug, info, warn, errror, fatal, panic
# # Defaults to "info"
level = "warn"
# # Additionally exceptions from the Go server can be reported to Sentry
# sentry_dsn = "https://<key>:<secret>@sentry.io/<project>"
# # Exceptions from gitaly-ruby can also be reported to Sentry
# ruby_sentry_dsn = "https://<key>:<secret>@sentry.io/<project>"
# # You can optionally configure Gitaly to record histogram latencies on GRPC method calls
# [prometheus]
# grpc_latency_buckets = [0.001, 0.005, 0.025, 0.1, 0.5, 1.0, 10.0, 30.0, 60.0, 300.0, 1500.0]
[gitaly-ruby]
# The directory where gitaly-ruby is installed
dir = "<%= File.join(config.gdk_root, 'gitaly', 'ruby') %>"
# # Gitaly-ruby resident set size (RSS) that triggers a memory restart (bytes)
# max_rss = 200000000
#
# # Grace period before a gitaly-ruby process is forcibly terminated after exceeding max_rss (seconds)
# graceful_restart_timeout = "10m"
#
# # Time that gitaly-ruby memory must remain high before a restart (seconds)
# restart_delay = "5m"
#
# # Number of gitaly-ruby worker processes
# num_workers = 2
[gitlab-shell]
# The directory where gitlab-shell is installed
dir = "<%= File.join(config.gdk_root, 'gitlab-shell') %>"
# # You can adjust the concurrency of each RPC endpoint
# [[concurrency]]
# rpc = "/gitaly.RepositoryService/GarbageCollect"
# max_per_repo = 1
......@@ -29,11 +29,11 @@ production: &base
## GitLab settings
gitlab:
## Web server settings (note: host is the FQDN, do not include http://)
host: <%= env_or_default('hostname', 'localhost') %>
port: <%= env_or_default('port', 3000).to_i %>
https: <%= env_or_default('https', false) %>
host: <%= config.hostname %>
port: <%= config.port %>
https: <%= config.https? %>
relative_url_root: "<%= env_or_default('relative_url_root', '') %>"
relative_url_root: "<%= config.relative_url_root %>"
# Trusted Proxies
# Customize if you have GitLab behind a reverse proxy which is running on a different machine.
......@@ -44,7 +44,7 @@ production: &base
#- 192.168.2.1
#- 2001:0db8::/32
user: <%= Etc.getlogin %>
user: <%= config.username %>
## Date & Time settings
# Uncomment and customize if you want to change the default time zone of GitLab application.
......@@ -140,14 +140,14 @@ production: &base
artifacts:
enabled: true
object_store:
enabled: <%= env_or_default('object_store_enabled', false) %>
enabled: <%= config.object_store? %>
remote_directory: artifacts
connection:
provider: AWS
aws_access_key_id: minio
aws_secret_access_key: gdk-minio
region: gdk
endpoint: 'http://127.0.0.1:<%= env_or_default('object_store_port', 9000) %>'
endpoint: 'http://127.0.0.1:<%= config.object_store.port %>'
path_style: true
## Merge request external diff storage
......@@ -172,7 +172,7 @@ production: &base
lfs:
enabled: true
object_store:
enabled: <%= env_or_default('object_store_enabled', false) %>
enabled: <%= config.object_store? %>
remote_directory: lfs-objects
direct_upload: true
connection:
......@@ -180,7 +180,7 @@ production: &base
aws_access_key_id: minio
aws_secret_access_key: gdk-minio
region: gdk
endpoint: 'http://127.0.0.1:<%= env_or_default('object_store_port', 9000) %>'
endpoint: 'http://127.0.0.1:<%= config.object_store.port %>'
path_style: true
## Uploads (attachments, avatars, etc...)
......@@ -189,7 +189,7 @@ production: &base
# storage_path: public/
# base_dir: uploads/-/system
object_store:
enabled: <%= env_or_default('object_store_enabled', false) %>
enabled: <%= config.object_store? %>
remote_directory: uploads
direct_upload: true
connection:
......@@ -197,14 +197,14 @@ production: &base
aws_access_key_id: minio
aws_secret_access_key: gdk-minio
region: gdk
endpoint: 'http://127.0.0.1:<%= env_or_default('object_store_port', 9000) %>'
endpoint: 'http://127.0.0.1:<%= config.object_store.port %>'
path_style: true
## Packages (maven repository, npm registry, etc...)
packages:
enabled: true
object_store:
enabled: <%= env_or_default('object_store_enabled', false) %>
enabled: <%= config.object_store? %>
remote_directory: packages
direct_upload: true
connection:
......@@ -212,7 +212,7 @@ production: &base
aws_access_key_id: minio
aws_secret_access_key: gdk-minio
region: gdk
endpoint: 'http://127.0.0.1:<%= env_or_default('object_store_port', 9000) %>'
endpoint: 'http://127.0.0.1:<%= config.object_store.port %>'
path_style: true
## GitLab Pages
......@@ -220,13 +220,13 @@ production: &base
enabled: true
access_control: false
host: 127.0.0.1.xip.io
port: <%= env_or_default('gitlab_pages_port', 3010) %>
port: <%= config.gitlab_pages.port %>
https: false # Set to true if you serve the pages with HTTPS
artifacts_server: true
# external_http: ["1.1.1.1:80", "[2001::1]:80"] # If defined, enables custom domain support in GitLab Pages
# external_https: ["1.1.1.1:443", "[2001::1]:443"] # If defined, enables custom domain and certificate support in GitLab Pages
admin:
address: unix:<%= GDK_ROOT %>/gitlab/tmp/sockets/private/pages-admin.socket # TCP connections are supported too (e.g. tcp://host:port)
address: unix:<%= config.gdk_root %>/gitlab/tmp/sockets/private/pages-admin.socket # TCP connections are supported too (e.g. tcp://host:port)
## Mattermost
## For enabling Add to Mattermost button
......@@ -284,10 +284,10 @@ production: &base
cron: "15 * * * *"
registry:
enabled: <%= env_or_default('registry_enabled', false) %>
host: <%= env_or_default('registry_host', '127.0.0.1') %>
port: <%= env_or_default('registry_external_port', 5000) %>
api_url: http://127.0.0.1:<%= env_or_default('registry_port', 5000) %>
enabled: <%= config.registry? %>
host: <%= config.registry.host %>
port: <%= config.registry.external_port %>
api_url: http://127.0.0.1:<%= config.registry.port %>
key: ../localhost.key
path: ../registry/storage/
issuer: gitlab-issuer
......@@ -634,7 +634,7 @@ production: &base
# Gitaly settings
gitaly:
# Path to the directory containing Gitaly client executables.
client_path: <%= File.join(GDK_ROOT, "gitaly", "bin") %>
client_path: <%= File.join(config.gdk_root, "gitaly", "bin") %>
# Default Gitaly authentication token. Can be overridden per storage. Can
# be left blank when Gitaly is running locally on a Unix socket, which
# is the normal way to deploy Gitaly.
......@@ -652,8 +652,8 @@ production: &base
# real path not the symlink.
storages: # You must have at least a `default` storage path.
default:
path: <%= File.join(GDK_ROOT, "repositories") %>
gitaly_address: unix:<%= File.join(GDK_ROOT, "gitaly.socket") %>
path: <%= File.join(config.gdk_root, "repositories") %>
gitaly_address: unix:<%= config.gitaly.address %>
## Backup settings
backup:
......@@ -684,8 +684,8 @@ production: &base
## GitLab Shell settings
gitlab_shell:
path: <%= File.join(GDK_ROOT, "gitlab-shell") %>
authorized_keys_file: <%= File.join(GDK_ROOT, ".ssh", "authorized_keys") %>
path: <%= File.join(config.gdk_root, "gitlab-shell") %>
authorized_keys_file: <%= File.join(config.gdk_root, ".ssh", "authorized_keys") %>
# Git over HTTP
upload_pack: true
......@@ -703,14 +703,14 @@ production: &base
# CAUTION!
# Use the default values unless you really know what you are doing
git:
bin_path: <%= GIT_BIN %>
bin_path: <%= config.git.bin %>
## Webpack settings
webpack:
dev_server:
enabled: true
host: localhost
port: <%= env_or_default('webpack_port', 3808).to_i %>
port: <%= config.webpack.port %>
## Monitoring
# Built in monitoring settings
......
# Example Praefect configuration file
# # TCP address to listen on
# listen_addr = "127.0.0.1:2305"
# # Praefect can listen on a socket when placed on the same machine as all clients
socket_path = "<%= File.join(config.gdk_root, 'praefect.socket') %>"
# # Praefect will only replicate whitelisted repositories
# whitelist = ["@hashed/3f/db/3fdba35f04dc8c462986c992bcf875546257113072a909c162f7e470e581e278.git"]
# # Optional: export metrics via Prometheus
# prometheus_listen_addr = "127.0.01:10101"
# # You can optionally configure Praefect to output JSON-formatted log messages to stdout
# [logging]
# format = "json"
# # Optional: Set log level to only log entries with that severity or above
# # One of, in order: debug, info, warn, errror, fatal, panic
# # Defaults to "info"
# level = "warn"
# # One or more Gitaly servers need to be configured to be managed. The names
# of each server are used to link multiple nodes, or `gitaly_server`s together
# as shard. listen_addr should be unique for all nodes.
# Requires the protocol to be defined, e.g. tcp://host.tld:1234
[primary_server]
name = "default"
listen_addr = "unix:<%= config.gitaly.address %>"
# [[secondary_server]]
# name = "default"
# listen_addr = "tcp://gitaly-backup1.example.com"
# [[secondary_server]]
# name = "backup"
# listen_addr = "tcp://gitaly-backup2.example.com"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment