Verified Commit 9e04c964 authored by Alessio Caiazza's avatar Alessio Caiazza 🌴

Add object-store support

Objects store can be tested locally using [minio](https://minio.io)

Howto:

1. install minio binary (no docker) -
https://docs.minio.io/docs/minio-quickstart-guide
2. echo true > object_store_enabled
3. gdk reconfigure
parent 5b79c547
Pipeline #17271159 passed with stages
in 47 minutes and 28 seconds
......@@ -69,3 +69,6 @@
/registry/config.yml
/postgresql_port
/postgresql_geo_port
/object_store_enabled
/object_store_port
/minio
......@@ -26,8 +26,10 @@ registry_port = $(shell cat registry_port 2>/dev/null || echo '5000')
gitlab_from_container = $(shell [ "$(uname)" = "Linux" ] && echo 'localhost' || echo 'docker.for.mac.localhost')
postgresql_port = $(shell cat postgresql_port 2>/dev/null || echo '5432')
postgresql_geo_port = $(shell cat postgresql_geo_port 2>/dev/null || echo '5432')
object_store_enabled = $(shell cat object_store_enabled 2>/dev/null || echo 'false')
object_store_port = $(shell cat object_store_port 2>/dev/null || echo '9000')
all: gitlab-setup gitlab-shell-setup gitlab-workhorse-setup support-setup gitaly-setup prom-setup
all: gitlab-setup gitlab-shell-setup gitlab-workhorse-setup support-setup gitaly-setup prom-setup object-storage-setup
# Set up the GitLab Rails app
......@@ -42,7 +44,10 @@ gitlab/config/gitlab.yml:
sed -e "s|/home/git|${gitlab_development_root}|"\
-e "s|/usr/bin/git|${git_bin}|"\
gitlab/config/gitlab.yml.example > gitlab/config/gitlab.yml
hostname=${hostname} port=${port} webpack_port=${webpack_port} registry_enabled=${registry_enabled} registry_port=${registry_port} support/edit-gitlab.yml gitlab/config/gitlab.yml
hostname=${hostname} port=${port} webpack_port=${webpack_port}\
registry_enabled=${registry_enabled} registry_port=${registry_port}\
object_store_enabled=${object_store_enabled} object_store_port=${object_store_port}\
support/edit-gitlab.yml gitlab/config/gitlab.yml
gitlab/config/database.yml:
sed -e "s|/home/git|${gitlab_development_root}|"\
......@@ -426,6 +431,11 @@ registry/config.yml:
cp registry/config.yml.example $@
gitlab_host=${gitlab_from_container} gitlab_port=${port} registry_port=${registry_port} support/edit-registry-config.yml $@
object-storage-setup: minio/data/lfs-objects minio/data/artifacts
minio/data/%:
mkdir -p $@
pry:
grep '^#rails-web:' Procfile || (printf ',s/^rails-web/#rails-web/\nwq\n' | ed -s Procfile)
@echo ""
......
......@@ -20,3 +20,4 @@ webpack: exec /usr/bin/env NODE_ENV=development DEV_SERVER_PORT=3808 support/exe
registry: exec support/docker-registry
#prometheus: exec docker run --net host -p 9090:9090 -v /home/git/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus:v2.0.0
#storage-check: exec /usr/bin/env RAILS_ENV=development support/exec-cd gitlab bin/storage_check -t unix:///home/git/gitlab.socket
minio: exec support/object-store
......@@ -4,49 +4,18 @@ GitLab Enterprise Edition has Object Storage integration. In this
document we explain how to set this up in your development
environment.
## minio Setup
In order to take advantage of the GDK integration you must first install
[minio](https://docs.minio.io/docs/minio-quickstart-guide) binary (no docker image).
1. Spin up minio container https://github.com/minio/minio
You can enable the object store writing `true` in `object_store_enabled` file and
reconfiguring your `gdk` installation.
```sh
docker pull minio/minio
docker run -p 9000:9000 minio/minio server /data
```
```sh
echo true > object_store_enabled
gdk reconfigure
```
1. Copy AccessKey and SecretKey printed in terminal to 'gitlab.yml' as described below
Object store port defaults to `9000` but it can be changed writing the desired value
in `object_store_port`.
```sh
AccessKey: XXXXXXXXXXXXXXX
SecretKey: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
```
1. Visit the console on http://127.0.0.1:9000
1. Create a new bucket from the right-bottom button. Bucket name is "artifacts".
## GDK Setup
1. Configure gitlab/config/gitlab.yml
```yml
development:
<<: *base
## Build Artifacts
artifacts:
enabled: true
# The location where build artifacts are stored (default: shared/artifacts).
# path: shared/artifacts
object_store:
enabled: true
remote_directory: artifacts # The bucket name
background_upload: true # Temporary option to limit automatic upload (Default: true)
connection:
provider: AWS # Only AWS supported at the moment
aws_access_key_id: XXXXXXXXXXXXXXX # AccessKey
aws_secret_access_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX # SecretKey
region: eu-central-1
host: 'localhost'
endpoint: 'http://127.0.0.1:9000'
path_style: true
```
1. Run GDK
Changing port number requires `gdk reconfigure`.
......@@ -36,7 +36,7 @@ foreman_start() {
}
db() {
foreman_start -m all=0,redis=1,postgresql=1,openldap=1,influxdb=1,webpack=1,registry=1
foreman_start -m all=0,redis=1,postgresql=1,openldap=1,influxdb=1,webpack=1,registry=1,minio=1
}
geo_db() {
......
......@@ -49,5 +49,52 @@ a
path: ../registry/storage/
issuer: gitlab-issuer
.
1
/^ artifacts:/
/^ enabled:/
c
enabled: true
.
a
object_store:
enabled: ${object_store_enabled:=false}
remote_directory: artifacts
connection:
provider: AWS
aws_access_key_id: minio
aws_secret_access_key: gdk-minio
region: gdk
endpoint: 'http://127.0.0.1:${object_store_port:-9000}'
path_style: true
.
/^ lfs:/
/^ enabled:/
c
enabled: true
.
/^ object_store:/
/^ enabled:/
c
enabled: ${object_store_enabled:=false}
.
/^ remote_directory:/
c
remote_directory: lfs-objects
.
/^ connection:/
/^ aws_access_key_id:/
c
aws_access_key_id: minio
.
/^ aws_secret_access_key:/
c
aws_secret_access_key: gdk-minio
.
/^ region:/
c
region: gdk
endpoint: 'http://127.0.0.1:${object_store_port:-9000}'
path_style: true
.
wq
EOF
#!/usr/bin/env bash
os_enabled=$(cat object_store_enabled 2>/dev/null)
os_port=$(cat object_store_port 2>/dev/null)
if [[ "${os_enabled}" == "true" ]]; then
exec env MINIO_REGION=gdk MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=gdk-minio minio server \
-C minio/config --address "127.0.0.1:${os_port:-9000}" minio/data
else
echo "Object Storage disabled."
# If we exit now, Foreman will shut down all our sibling process too.
# We want to block until Foreman signals us to exit.
tail -f /dev/null
fi
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment