Automate the setup of storage logging buckets in GCS
Currently this is not done as part of the terraform environment setup script.
I did it manually, using the following script:
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'
ENV=${ENV:-staging}
case $ENV in
staging)
GCP_PROJECT_ID=gitlab-staging-1
GS_LOGGING_BUCKET_URL=gs://gitlab-staging-storage-logs
GS_TARGET_BUCKETS=(gs://gitlab-staging-artifacts gs://gitlab-staging-lfs-objects gs://gitlab-staging-uploads)
;;
production)
GCP_PROJECT_ID=gitlab-production
GS_LOGGING_BUCKET_URL=gs://gitlab-storage-logs
GS_TARGET_BUCKETS=(gs://gitlab-artifacts gs://gitlab-lfs-objects gs://gitlab-uploads)
;;
esac
# Create the bucket
gsutil mb -p ${GCP_PROJECT_ID} ${GS_LOGGING_BUCKET_URL} || true
# Add labels
gsutil label ch -l env:${ENV} ${GS_LOGGING_BUCKET_URL}
# Set permissions
gsutil acl ch -g cloud-storage-analytics@google.com:W ${GS_LOGGING_BUCKET_URL}
gsutil defacl set project-private ${GS_LOGGING_BUCKET_URL}
for i in ${GS_TARGET_BUCKETS[@]}; do
gsutil logging set on -b "${GS_LOGGING_BUCKET_URL}" "${i}"
done
CONFIG_FILE=$(mktemp)
trap "rm -rf ${CONFIG_FILE}" EXIT
cat <<EOD > ${CONFIG_FILE}
{
"rule":[{
"action": {"type": "Delete"},
"condition": {"age": 7}
}]
}
EOD
gsutil lifecycle set ${CONFIG_FILE} ${GS_LOGGING_BUCKET_URL}
# Summary
echo Created ${GS_LOGGING_BUCKET_URL}
echo ---------
echo ACLS:
gsutil acl get ${GS_LOGGING_BUCKET_URL}
echo ---------
echo Labels:
gsutil label get ${GS_LOGGING_BUCKET_URL}
echo ---------
echo Lifecycle:
gsutil lifecycle get ${GS_LOGGING_BUCKET_URL}
cc @jarv and @ahanselka
Edited by Andrew Newdigate