Copy production data from experimental node on Azure to the experimental node on GCP
- This is change is being done for https://gitlab.com/gitlab-com/infrastructure/issues/4611, the next step after https://gitlab.com/gitlab-com/infrastructure/issues/4626 (clone of https://gitlab.com/gitlab-com/production/issues/368)
- The idea is to get data from the Postgres experimental instance on Azure/staging (Azure instance is already created and has production data, see https://gitlab.com/gitlab-com/production/issues/368), using pg_basebackup + replication
-
Clone / setup a replica following the Azure experimental server
########
# Make the local Postgres server to follow the specified remote host
#
# Globals:
# SOURCE_HOST
# SLOT_NAME
# Arguments:
# none
# Returns:
# none
#
setupReplica() {
date
if sudo test -f "/var/opt/gitlab/postgresql/data/postgresql.conf"; then
>&2 echo "ERROR: /var/opt/gitlab/postgresql/data/postgresql.conf exists, cannot continue"
exit 1
fi
cd /tmp
# Create a new replication slot $SLOT_NAME on $SOURCE_HOST
sudo -u gitlab-psql gitlab-psql \
-h "$SOURCE_HOST" -U gitlab_repmgr postgres \
-c "select * from pg_create_physical_replication_slot('$SLOT_NAME');"
sudo -u gitlab-psql PGSSLMODE=disable /opt/gitlab/embedded/bin/pg_basebackup \
-D /var/opt/gitlab/postgresql/data \
-c fast -X stream -P \
--slot=dbteam_experimental_stream_azure_to_gcp \
--host="$SOURCE_HOST" -p 5432 --username=gitlab_repmgr
# edit recovery.conf
## TODO edit primary_conninfo !
echo "primary_slot_name = $SLOT_NAME" >> /var/opt/gitlab/postgresql/data/recovery.conf
echo "trigger_file = '/var/opt/gitlab/postgresql/data/trigger_file'" >> /var/opt/gitlab/postgresql/data/recovery.conf
date
}
export SOURCE_HOST="postgres-dbteam-01.db.stg.gitlab.com"
export SLOT_NAME="dbteam_experimental_stream_azure_to_gcp"
set -uxo pipefail
cd /tmp
setupReplica # be prepared to enter the password
cat /var/opt/gitlab/postgresql/data/recovery.conf
sudo gitlab-ctl start postgresql
sudo tail -f /var/log/gitlab/postgresql/current
-
Promote
touch /var/opt/gitlab/postgresql/data/trigger_file
-
Check
sudo -u gitlab-psql gitlab-psql postgres -c 'select pg_is_in_recovery();'
-
add DB user with statement_timeout = 0
psql -U gitlab_repmgr postgres -h localhost -c 'create role nikolays superuser login;'
psql -h /var/opt/gitlab/postgresql postgres -c 'alter role nikolays set statement_timeout to 0;'
Edited by Nikolay Samokhvalov