Commit fcdbaedc authored by Stan Hu's avatar Stan Hu

Merge branch 'tc-gdk-yml-config' into 'master'

Compile Procfile & nginx.conf from gdk.yml if possible

See merge request gitlab-development-kit!642
parents d02e437c 8cd571bd
......@@ -43,7 +43,9 @@ gitlab-development-kit/**
/webpack_port
/.ruby-version
/.ruby-gemset
/localhost.*
*.key
*.crt
*.pem
/influxdb/
/grafana/
/openssh/*_key
......@@ -92,3 +94,4 @@ gitlab-development-kit/**
/auto_devops_registry_port
/jaeger-artifacts/
/jaeger/
/gdk.yml
......@@ -322,23 +322,11 @@ support-setup: .ruby-version foreman Procfile redis gitaly-setup jaeger-setup po
echo "*********************************************"; \
fi
gdk.yml:
touch $@
Procfile: Procfile.example auto_devops_enabled auto_devops_gitlab_port auto_devops_registry_port
bin/safe-sed "$@" \
-e "s|/home/git|${gitlab_development_root}|g"\
-e "s|/usr/sbin/sshd|${sshd_bin}|"\
-e "s|postgres |${postgres_bin_dir}/postgres |"\
-e "s|DEV_SERVER_PORT=3808 |DEV_SERVER_PORT=${webpack_port} |"\
-e "s|-listen-http \":3010\" |-listen-http \":${gitlab_pages_port}\" -artifacts-server http://${hostname}:${port}/api/v4 |"\
-e "s|jaeger-VERSION|jaeger-${jaeger_version}|" \
-e "$(if $(filter false,$(jaeger_server_enabled)),/^jaeger:/s/^/#/,/^#\s*jaeger:/s/^#\s*//)" \
-e "$(if $(filter true,$(auto_devops_enabled)),s|#tunnel_gitlab:.*|tunnel_gitlab: ssh -N -R $(auto_devops_gitlab_port):localhost:\$$port qa-tunnel.gitlab.info|g,/^#tunnel_gitlab:/s/^//)" \
-e "$(if $(filter true,$(auto_devops_enabled)),s|#tunnel_registry:.*|tunnel_registry: ssh -N -R ${auto_devops_registry_port}:localhost:${registry_port} qa-tunnel.gitlab.info|g,/^#tunnel_registry:/s/^//)" \
"$<"
if [ -f .vagrant_enabled ]; then \
echo "0.0.0.0" > host; \
echo "3000" > port; \
fi
Procfile: Procfile.erb gdk.yml auto_devops_enabled auto_devops_gitlab_port auto_devops_registry_port
rake $@
redis: redis/redis.conf
......@@ -647,7 +635,7 @@ clean-config:
touch-examples:
touch \
$(gitaly_clone_dir)/config.toml.example \
Procfile.example \
Procfile.erb \
database.yml.example \
database_geo.yml.example \
gitlab-shell/config.yml.example \
......
......@@ -5,29 +5,29 @@
# Datasources
#
redis: exec redis-server /home/git/redis/redis.conf
redis: exec redis-server <%= config.gdk_root %>/redis/redis.conf
minio: exec support/object-store
postgresql: exec support/postgresql-signal-wrapper postgres -D /home/git/postgresql/data -k /home/git/postgresql -h ''
#postgresql-replica: exec support/postgresql-signal-wrapper postgres -D /home/git/postgresql-replica/data -k /home/git/postgresql-replica -h ''
postgresql: exec support/postgresql-signal-wrapper <%= config.postgresql.bin_dir %>/postgres -D <%= config.postgresql.data_dir %> -k <%= config.postgresql.dir %> -h ''
#postgresql-replica: exec support/postgresql-signal-wrapper <%= config.postgresql.bin_dir %>/postgres -D <%= config.postgresql.replica_dir %>/data -k <%= config.postgresql.replica_dir %> -h ''
#openldap: exec support/exec-cd gitlab-openldap libexec/slapd -F slapd.d -d2 -h "ldap://$host:3890"
#storage-check: exec /usr/bin/env RAILS_ENV=development RAILS_RELATIVE_URL_ROOT=$relative_url_root support/exec-cd gitlab bin/storage_check -t unix:///home/git/gitlab.socket
#storage-check: exec /usr/bin/env RAILS_ENV=development RAILS_RELATIVE_URL_ROOT=$relative_url_root support/exec-cd gitlab bin/storage_check -t unix://<%= config.gdk_root %>/gitlab.socket
# Geo
#
#postgresql-geo: exec support/postgresql-signal-wrapper postgres -D /home/git/postgresql-geo/data -k /home/git/postgresql-geo -h ''
#geo-cursor: exec /usr/bin/env RAILS_ENV=development RAILS_RELATIVE_URL_ROOT=$relative_url_root support/geo-logcursor
<%= '#' unless config.geo? %>postgresql-geo: exec support/postgresql-signal-wrapper <%= config.postgresql.bin_dir %>/postgres -D <%= config.postgresql.geo_dir %>/data -k <%= config.postgresql.geo_dir %> -h ''
<%= '#' unless config.geo? %>geo-cursor: exec /usr/bin/env RAILS_ENV=development RAILS_RELATIVE_URL_ROOT=$relative_url_root support/geo-logcursor
# Backend services
#
gitaly: exec gitaly/bin/gitaly /home/git/gitaly/config.toml
gitlab-workhorse: exec /usr/bin/env PATH="/home/git/gitlab-workhorse/bin:$PATH" gitlab-workhorse -authSocket /home/git/gitlab.socket -listenAddr $host:$port -documentRoot /home/git/gitlab/public -developmentMode -secretPath /home/git/gitlab/.gitlab_workhorse_secret -config /home/git/gitlab-workhorse/config.toml
gitaly: exec gitaly/bin/gitaly <%= config.gdk_root %>/gitaly/config.toml
gitlab-workhorse: exec /usr/bin/env PATH="<%= config.gdk_root %>/gitlab-workhorse/bin:$PATH" gitlab-workhorse -authSocket <%= config.gdk_root %>/gitlab.socket -listenAddr $host:<%= config.nginx? ? config.nginx.workhorse_port : '$port' %> -documentRoot <%= config.gdk_root %>/gitlab/public -developmentMode -secretPath <%= config.gdk_root %>/gitlab/.gitlab_workhorse_secret -config <%= config.gdk_root %>/gitlab-workhorse/config.toml
rails-background-jobs: exec /usr/bin/env SIDEKIQ_LOG_ARGUMENTS=1 RAILS_ENV=development RAILS_RELATIVE_URL_ROOT=$relative_url_root support/exec-cd gitlab bin/background_jobs start_foreground
# Frontend services
#
webpack: exec /usr/bin/env NODE_ENV=development DEV_SERVER_PORT=3808 support/exec-cd gitlab yarn dev-server
webpack: exec /usr/bin/env NODE_ENV=development DEV_SERVER_PORT=<%= config.webpack.port %> DEV_SERVER_HOST=$host support/exec-cd gitlab yarn dev-server
rails-web: exec /usr/bin/env RAILS_ENV=development RAILS_RELATIVE_URL_ROOT=$relative_url_root support/exec-cd gitlab bin/web start_foreground
gitlab-pages: exec gitlab-pages/bin/gitlab-pages -listen-http ":3010" -pages-root /home/git/gitlab/shared/pages/ -pages-domain 127.0.0.1.xip.io
<%= '#' unless config.gitlab_pages? %>gitlab-pages: exec gitlab-pages/bin/gitlab-pages -listen-http ":<%= config.gitlab_pages.port %>" -artifacts-server <%= config.protocol %>://<%= config.hostname %>:<%= config.port %>/api/v4 -pages-root <%= config.gdk_root %>/gitlab/shared/pages/ -pages-domain 127.0.0.1.xip.io
# Docker
#
......@@ -37,16 +37,16 @@ registry: exec support/docker-registry
#
#influxdb: exec influxdb/bin/influxd -config influxdb/influxdb.conf
#grafana: exec support/wait-postgresql-ready support/exec-cd grafana bin/grafana-server -config grafana.ini
#prometheus: exec docker run --net host -p 9090:9090 -v /home/git/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus:v2.8.1
jaeger: exec jaeger/jaeger-VERSION/jaeger-all-in-one --memory.max-traces 512
#prometheus: exec docker run --net host -p 9090:9090 -v <%= config.gdk_root %>/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus:v2.8.1
<%= '#' unless config.tracer.jaeger? %>jaeger: exec jaeger/jaeger-<%= config.tracer.jaeger.version %>/jaeger-all-in-one --memory.max-traces 512
# Additional services
#
#sshd: exec /usr/sbin/sshd -e -D -f /home/git/openssh/sshd_config
#nginx: exec nginx -p /home/git/nginx -c conf/nginx.conf
#sshd: exec <%= config.sshd.bin %> -e -D -f <%= config.gdk_root %>/openssh/sshd_config
<%= '#' unless config.nginx? %>nginx: exec <%= config.nginx.bin %> -p <%= config.gdk_root %>/nginx -c conf/nginx.conf
#elasticsearch: exec elasticsearch/bin/elasticsearch
# Tunneling
#
#tunnel_gitlab: ssh -N -R $auto_devops_gitlab_port:localhost:$port qa-tunnel.gitlab.info
#tunnel_registry: ssh -N -R $auto_devops_registry_port:localhost:5000 qa-tunnel.gitlab.info
<%= '#' unless config.auto_devops? %>tunnel_gitlab: ssh -N -R $auto_devops_gitlab_port:localhost:$port qa-tunnel.gitlab.info
<%= '#' unless config.auto_devops? %>tunnel_registry: ssh -N -R $auto_devops_registry_port:localhost:5000 qa-tunnel.gitlab.info
# frozen_string_literal: true
$LOAD_PATH.unshift('.')
require 'lib/gdk'
require 'rake/clean'
CLOBBER.include 'gdk.example.yml', 'Procfile', 'nginx/conf/nginx.conf'
def config
@config ||= GDK::Config.new
end
desc 'Dump the configured settings'
task 'dump_config' do
GDK::Config.new.dump!(STDOUT)
end
desc 'Generate an example config file with all the defaults'
file 'gdk.example.yml' => 'clobber:gdk.example.yml' do |t|
File.open(t.name, File::CREAT|File::TRUNC|File::WRONLY) do |file|
config = Class.new(GDK::Config)
config.define_method(:gdk_root) { '/home/git/gdk' }
config.define_method(:username) { 'git' }
config.define_method(:read!) { |_| nil }
config.new(yaml: {}).dump!(file)
end
end
task 'clobber:gdk.example.yml' do |t|
Rake::Cleaner.cleanup_files([t.name])
end
desc 'Generate Procfile for Foreman'
file 'Procfile' => ['Procfile.erb', GDK::Config::FILE] do |t|
GDK::ErbRenderer.new(t.source, t.name).safe_render!
end
desc 'Generate nginx configuration'
file 'nginx/conf/nginx.conf' => ['nginx/conf/nginx.conf.erb', GDK::Config::FILE] do |t|
GDK::ErbRenderer.new(t.source, t.name).safe_render!
end
---
auto_devops:
enabled: false
gitlab:
port: 20000
registry:
port: 25000
elasticsearch:
checksum: 5903e1913a7c96aad96a8227517c40490825f672
version: 6.5.1
gdk_root: "/home/git/gdk"
geo:
enabled: false
git:
bin: "/usr/bin/git"
gitaly:
assembly_dir: "/home/git/gdk/gitaly/assembly"
gitlab_pages:
enabled: true
port: 3034
hostname: localhost
https:
enabled: true
nginx:
bin: "/usr/sbin/nginx"
enabled: false
http:
enabled: false
port: 80
ssl:
certificate: localhost.crt
workhorse_port: 3333
object_store:
enabled: false
port: 9000
port: 3000
postgresql:
bin_dir: "/usr/lib/postgresql/9.6/bin"
data_dir: "/home/git/gdk/postgresql/data"
dir: "/home/git/gdk/postgresql"
geo_dir: "/home/git/gdk/postgresql-geo"
replica_dir: "/home/git/gdk/postgresql-replica"
replication_user: gitlab_replication
protocol: https
registry:
enabled: false
external_port: 5000
host: 127.0.0.1
port: 5000
relative_url_root:
repositories:
gitaly: https://gitlab.com/gitlab-org/gitaly.git
gitaly_proto: https://gitlab.com/gitlab-org/gitaly-proto.git
gitlab: https://gitlab.com/gitlab-org/gitlab-ce.git
gitlab_docs: https://gitlab.com/gitlab-com/gitlab-docs.git
gitlab_pages: https://gitlab.com/gitlab-org/gitlab-pages.git
gitlab_shell: https://gitlab.com/gitlab-org/gitlab-shell.git
gitlab_workhorse: https://gitlab.com/gitlab-org/gitlab-workhorse.git
sshd:
bin: "/usr/sbin/sshd"
tracer:
build_tags: tracer_static tracer_static_jaeger
jaeger:
enabled: true
version: 1.10.1
username: git
webpack:
port: 3806
......@@ -4,6 +4,8 @@
# part of the gitlab-development-kit gem so that we can iterate faster.
require_relative 'gdk/env'
require_relative 'gdk/config'
require_relative 'gdk/erb_renderer'
module GDK
PROGNAME = 'gdk'.freeze
......
# frozen_string_literal: true
require_relative 'config_settings'
module GDK
class Config < ConfigSettings
FILE = 'gdk.yml'
repositories do |r|
r.gitlab 'https://gitlab.com/gitlab-org/gitlab-ce.git'
r.gitlab_shell 'https://gitlab.com/gitlab-org/gitlab-shell.git'
r.gitlab_workhorse 'https://gitlab.com/gitlab-org/gitlab-workhorse.git'
r.gitaly 'https://gitlab.com/gitlab-org/gitaly.git'
r.gitaly_proto 'https://gitlab.com/gitlab-org/gitaly-proto.git'
r.gitlab_pages 'https://gitlab.com/gitlab-org/gitlab-pages.git'
r.gitlab_docs 'https://gitlab.com/gitlab-com/gitlab-docs.git'
end
gdk_root { ENV['PWD'] }
hostname do
next "#{config.auto_devops.gitlab.port}.qa-tunnel.gitlab.info" if config.auto_devops.enabled
read!('hostname') || 'localhost'
end
port do
next 443 if config.auto_devops.enabled
read!('port') || 3000
end
https do |h|
h.enabled do
next true if config.auto_devops.enabled
read!('https_enabled') || false
end
end
protocol { config.https? ? 'https' : 'http' }
relative_url_root { read!('relative_url_root') || nil }
username { ENV['USERNAME'] }
webpack do |w|
w.port { read!('webpack_port') || 3808 }
end
registry do |r|
r.enabled do
next true if config.auto_devops.enabled
read!('registry_enabled') || false
end
r.host do
next "#{config.auto_devops.registry.port}.qa-tunnel.gitlab.info" if config.auto_devops.enabled
'127.0.0.1'
end
r.port do
next config.auto_devops.registry.port if config.auto_devops.enabled
read!('registry_port') || 5000
end
r.external_port do
next 443 if config.auto_devops.enabled
5000
end
end
object_store do |o|
o.enabled { read!('object_store_enabled') || false }
o.port { read!('object_store_port') || 9000 }
end
gitlab_pages do |p|
p.enabled true
p.port { read!('gitlab_pages_port') || 3010 }
end
auto_devops do |a|
a.enabled { read!('auto_devops_enabled') || false }
a.gitlab do |g|
g.port { read_or_write!('auto_devops_gitlab_port', rand(20000..24999)) }
end
a.registry do |r|
r.port { read!('auto_devops_registry_port') || (config.auto_devops.gitlab.port + 5000) }
end
end
geo do |g|
g.enabled false
end
elasticsearch do |e|
e.version '6.5.1'
e.checksum '5903e1913a7c96aad96a8227517c40490825f672'
end
tracer do |t|
t.build_tags 'tracer_static tracer_static_jaeger'
t.jaeger do |j|
j.enabled true
j.version '1.10.1'
end
end
nginx do |n|
n.enabled false
n.bin { find_executable!('nginx') || '/usr/sbin/nginx' }
n.workhorse_port 3333
n.ssl do |s|
s.certificate 'localhost.crt'
s.key 'localhost.key'
end
n.http do |h|
h.enabled false
h.port 80
end
end
postgresql do |p|
p.bin_dir { cmd!('support/pg_bindir') }
p.replication_user 'gitlab_replication'
p.dir { "#{config.gdk_root}/postgresql" }
p.data_dir { "#{config.postgresql.dir}/data" }
p.replica_dir { "#{config.gdk_root}/postgresql-replica" }
p.geo_dir { "#{config.gdk_root}/postgresql-geo" }
end
gitaly do |g|
g.assembly_dir { "#{config.gdk_root}/gitaly/assembly" }
end
sshd do |s|
s.bin { find_executable!('sshd') || '/usr/sbin/sshd' }
end
git do |g|
g.bin { find_executable!('git') }
end
end
end
# frozen_string_literal: true
require 'yaml'
module GDK
class ConfigSettings
SettingUndefined = Class.new(StandardError)
attr_reader :parent, :yaml, :key
def self.method_missing(name, *args, &blk)
if !args.empty?
define_method(name) do
yaml.fetch(name.to_s, args.first)
end
elsif block_given?
define_method(name) do
# return the result of the block if it didn't take an argument
# otherwise return an instance of the sub ConfigSettings
return yaml.fetch(name.to_s, instance_eval(&blk)) if blk.arity.zero?
sub = Class.new(ConfigSettings)
blk.call(sub)
sub.new(parent: self, yaml: yaml.fetch(name.to_s, {}), key: [key, name].compact.join('.'))
end
else
raise SettingUndefined, "Could not find the setting '#{name}'"
end
end
def initialize(parent: nil, yaml: nil, key: nil)
@parent = parent
@key = key
@yaml = yaml || load_yaml!
end
def dump!(file = nil)
base_methods = ConfigSettings.new.methods
yaml = (methods - base_methods).sort.inject({}) do |hash, method|
value = public_send(method)
if value.is_a?(ConfigSettings)
hash[method.to_s] = value.dump!
else
hash[method.to_s] = value
end
hash
end
file.puts(yaml.to_yaml) if file
yaml
end
def cmd!(cmd)
`#{cmd}`.chomp
end
def find_executable!(bin)
result = cmd!("which #{bin}")
result.empty? ? nil : result
end
def read!(filename)
sanitized_read!(filename)
rescue Errno::ENOENT
nil
end
def read_or_write!(filename, value)
sanitized_read!(filename)
rescue Errno::ENOENT
File.write(filename, value)
value
end
def root
parent&.root || self
end
alias_method :config, :root
def inspect
"#<GDK::ConfigSettings key:#{key}>"
end
# Provide a shorter form for `config.setting.enabled` as `config.setting?`
def method_missing(method_name, *args, &blk)
enabled = enabled_value(method_name)
return super if enabled.nil?
enabled
end
def respond_to_missing?(method_name, include_private = false)
!enabled_value(method_name).nil? || super
end
private
def enabled_value(method_name)
chopped_name = method_name.to_s.chop.to_sym
return public_send(chopped_name).enabled if method_name.to_s.end_with?('?') &&
respond_to?(chopped_name) &&
public_send(chopped_name).respond_to?(:enabled)
nil
end
def load_yaml!
return {} unless defined?(self.class::FILE) && File.exist?(self.class::FILE)
@yaml = YAML.load_file(self.class::FILE) || {}
end
def from_yaml(key, default: nil)
yaml.has_key?(key) ? yaml[key] : default
end
def sanitized_read!(filename)
value = File.read(filename).chomp
return true if value == "true"
return false if value == "false"
return value.to_i if value == value.to_i.to_s
value
end
end
end
# frozen_string_literal: true
require 'pathname'
require 'shellwords'
......
# frozen_string_literal: true
require 'erb'
require 'tempfile'
module GDK
class ErbRenderer
attr_reader :source, :target
def initialize(source, target)
@source = source
@target = target
end
def render!(target = @target)
str = File.read(source)
result = ERB.new(str).result
File.write(target, result)
end
def safe_render!
temp_file = Tempfile.new(target)
render!(temp_file.path)
return File.rename(temp_file.path, target) unless File.exist?(target)
warn!(temp_file) unless FileUtils.identical?(target, temp_file.path)
ensure
temp_file.close!
end
private
def warn!(temp_file)
diff = `git --no-pager diff --no-index #{colors_arg} -u "#{target}" "#{temp_file.path}"`
puts <<~EOF
-------------------------------------------------------------------------------------------------------------
Warning: Your `#{target}` is outdated. These are the changes GDK wanted to apply.
-------------------------------------------------------------------------------------------------------------
#{diff}
-------------------------------------------------------------------------------------------------------------
- To apply these changes run: `rm #{target}` and re-run `gdk update`.
- To silence this warning (at your own peril): `touch #{target}`
... Waiting 5 seconds for previous warning to be noticed.
-------------------------------------------------------------------------------------------------------------
EOF
sleep 5
end
def colors?
@colors_supported ||= (`tput colors`.chomp.to_i >= 8)
end
def colors_arg
'--color' if colors?
end
end
end
worker_processes 1;
daemon off;
pid tmp/nginx.pid;
error_log stderr;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
access_log logs/access.log;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
proxy_temp_path proxy_temp;
client_body_temp_path client_body_temp;
uwsgi_temp_path uwsgi_temp;
fastcgi_temp_path fastcgi_temp;
scgi_temp_path scgi_temp;
client_max_body_size 0;
upstream gitlab-workhorse {
server <%= config.hostname %>:<%= config.nginx.workhorse_port %> fail_timeout=0;
}
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
<% if config.https? %>
server {
listen <%= config.hostname %>:<%= config.port %> ssl;
ssl_certificate <%= config.gdk_root %>/<%= config.nginx.ssl.certificate %>;
ssl_certificate_key <%= config.gdk_root %>/<%= config.nginx.ssl.key %>;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_read_timeout 300;
proxy_pass http://gitlab-workhorse;
}
}
<% if config.nginx.http? %>
server {
listen <%= config.hostname %>:<%= config.nginx.http.port %>;
return 301 https://$host$request_uri;
}
<% end %>
<% end %>
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment