Commit 10f59ced authored by Emeric Verschuur's avatar Emeric Verschuur Committed by Stephane Bausseron

[env-setup] Modular setup

* add ability to limit setup on specified elements
* add ci-tool setup
* add gentoo support
parent b8e89a40
# Cangelog
## 3.2.0
* [env-setup] Modular setup
+ add ability to limit setup on specified elements
+ add ci-tool setup
+ add gentoo support
## 3.1.0
* [FIX/bashcomp] Fix number calculation in the _citbx4gitlab_compgen function
* Add disable-service parameter (CITBX_DISABLED_SERVICES) to disable specified services
......
# CI lib for Gitlab properties
# CI toolbox for Gitlab properties
#
# Default CI registry
#DEFAULT_CI_REGISTRY="registry.gitlab.example.com:5005"
......@@ -21,3 +21,10 @@ CITBX_DEFAULT_JOB_SHELL="/bin/bash"
# Enable git lfs by default
#CITBX_DEFAULT_GIT_LFS_ENABLED="true"
#
# Default DNS list
# * OpenDNS:
CITBX_DOCKER_DEFAULT_DNS=(208.67.222.123 208.67.220.123)
# * Quad9
#CITBX_DOCKER_DEFAULT_DNS=(9.9.9.9)
# * Google
#CITBX_DOCKER_DEFAULT_DNS=(8.8.8.8 8.8.4.4)
sudo_sponge() {
local out=${1:-/dev/stdout}
_sudo awk -v o="${out}" '
{
b = (NR > 1 ? b""ORS""$0 : $0);
}
END{
print b > o;
}'
}
if [ $(id -u) -eq 0 ]; then
_sudo() {
"$@"
}
else
if ! which sudo > /dev/null 2>&1; then
print_critical "In user mode, sudo with suitable system rights is required"
fi
_sudo() {
sudo "$@"
}
fi
write_daemon_json() {
bashopts_process_option -n CITBX_DOCKER_DNS_LIST -r
# Setup docker0 bridge with:
# - NET: 192.168.255.0/24 (by default)
# - DNS: Use system dns instead of Google one
_sudo mkdir -p /etc/docker
if ! jq '' /etc/docker/daemon.json > /dev/null 2>&1; then
if [ -f /etc/docker/daemon.json ]; then
_sudo mv /etc/docker/daemon.json{,.bak}
print_warning "Invalid file /etc/docker/daemon.json, moving it to /etc/docker/daemon.json.bak"
fi
_sudo bash -c 'echo {} > /etc/docker/daemon.json'
fi
for dns in $CITBX_DOCKER_DNS_LIST; do
if [ -n "$dnslist" ]; then
dnslist="$dnslist, \"$dns\""
else
dnslist="\"$dns\""
fi
done
_sudo cat /etc/docker/daemon.json |
jq '. + {
"bip": "'"$CITBX_DOCKER_BIP"'",
"fixed-cidr": "'"$CITBX_DOCKER_FIXED_CIDR"'",
"dns": '"$(bashopts_dump_array "string" "${CITBX_DOCKER_DNS_LIST[@]}")"',
"storage-driver": "'"$CITBX_DOCKER_STORAGE_DRIVER"'"
}' | sudo_sponge /etc/docker/daemon.json
}
install_ci_toolbox() {
curl -ksL https://gitlab.com/ercom/citbx4gitlab/raw/master/tools/gitlab-ci/citbx4gitlab/bashcomp \
| sed 's/\bcitbx4gitlab\b/'"$CITBX_TOOLBOX_NAME"'/' \
| sudo_sponge /etc/bash_completion.d/$CITBX_TOOLBOX_NAME
_sudo curl -ksLo /usr/local/bin/$CITBX_TOOLBOX_NAME https://gitlab.com/ercom/citbx4gitlab/raw/master/tools/gitlab-ci/citbx4gitlab/citbx4gitlab
_sudo chmod +x /usr/local/bin/$CITBX_TOOLBOX_NAME
}
. $CITBX_ABS_DIR/env-setup/common.sh
INSTALL_PKGS=()
if setup_component_enabled base-pkgs; then
for pkg in app-emulation/docker sys-apps/gawk dev-python/pyyaml app-misc/jq; do
if ! equery -q list $pkg > /dev/null; then
INSTALL_PKGS+=($pkg)
fi
done
fi
if setup_component_enabled git-lfs \
&& [ "$CITBX_GIT_LFS_SUPPORT_ENABLED" == "true" ]; then
if ! equery -q list dev-vcs/git-lfs > /dev/null; then
INSTALL_PKGS+=(dev-vcs/git-lfs)
fi
INSTALL_PKGS+=(git-lfs)
fi
if [ "${#INSTALL_PKGS[@]}" -gt 0 ]; then
print_info "Installing packages..."
_sudo emerge -av "${INSTALL_PKGS[@]}"
fi
if setup_component_enabled base-pkgs; then
if [ "${USER}" != "root" ]; then
_sudo gpasswd -a ${USER} docker
fi
fi
if setup_component_enabled ca-certs; then
print_info "Installing CA certificates..."
# Add user SSL ROOT CA
if [ -d $CITBX_ABS_DIR/ca-certificates ]; then
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /usr/local/share/ca-certificates/
_sudo update-ca-certificates
_sudo mkdir -p /etc/docker/certs.d
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /etc/docker/certs.d/
fi
fi
if setup_component_enabled docker-cfg; then
print_info "Configuring docker..."
write_daemon_json
# Put in comment the docker default options
if grep -q '^DOCKER_OPTS=.*' /etc/conf.d/docker \
&& ! grep -q '^DOCKER_OPTS=""$' /etc/conf.d/docker; then
_sudo sed -i 's/^DOCKER_OPTS=.*$/DOCKER_OPTS=""/g' /etc/conf.d/docker
fi
_sudo ip link del docker0 2>/dev/null || true
_sudo /etc/init.d/docker restart
fi
if setup_component_enabled ci-toolbox; then
print_info "Installing the CI toolbox $CITBX_TOOLBOX_NAME..."
install_ci_toolbox
fi
. $CITBX_ABS_DIR/env-setup/common.sh
ubuntu_is_installed() {
if [ "$(dpkg -s "$1" 2>/dev/null | grep -o 'installed' | head -n 1)" == "installed" ]; then
......@@ -6,64 +7,45 @@ ubuntu_is_installed() {
return 1
}
sudo_sponge() {
local out=${1:-/dev/stdout}
_sudo awk -v o="${out}" '
{
b = (NR > 1 ? b""ORS""$0 : $0);
}
END{
print b > o;
}'
}
if [ $(id -u) -eq 0 ]; then
_sudo() {
"$@"
}
else
if ! which sudo > /dev/null 2>&1; then
print_critical "In user mode, sudo with suitable system rights is required"
fi
_sudo() {
sudo "$@"
}
fi
INSTALL_PKGS=()
INSTALL_PKGS=(docker-ce gawk python-yaml jq)
if setup_component_enabled base-pkgs; then
INSTALL_PKGS+=(docker-ce gawk python-yaml jq)
# remove old versions...
if ubuntu_is_installed docker.io; then
print_note "Removing old docker.io package..."
_sudo /etc/init.d/docker stop
_sudo apt-get remove -y --allow-change-held-packages docker.io
fi
if ubuntu_is_installed docker-engine; then
print_note "Removing old docker-engine package..."
_sudo /etc/init.d/docker stop
_sudo apt-get remove -y --allow-change-held-packages docker-engine
fi
_sudo apt-get update
_sudo apt-get install -y aufs-tools \
linux-image-extra-virtual \
apt-transport-https \
ca-certificates \
curl \
software-properties-common
if grep -qr 'download.docker.com' /etc/apt/; then
print_note "Docker apt repository is already present."
else
print_note "Adding docker apt repository..."
# setup - pre install
# add docker repo
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | _sudo apt-key add -
_sudo add-apt-repository \
"deb [arch=amd64] http://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable"
# remove old versions...
if ubuntu_is_installed docker.io; then
print_note "Removing old docker.io package..."
_sudo /etc/init.d/docker stop
_sudo apt-get remove -y --allow-change-held-packages docker.io
fi
if ubuntu_is_installed docker-engine; then
print_note "Removing old docker-engine package..."
_sudo /etc/init.d/docker stop
_sudo apt-get remove -y --allow-change-held-packages docker-engine
fi
_sudo apt-get update
_sudo apt-get install -y aufs-tools \
apt-transport-https \
ca-certificates \
curl \
software-properties-common
if grep -qr 'download.docker.com' /etc/apt/; then
print_note "Docker apt repository is already present."
else
print_note "Adding docker apt repository..."
# setup - pre install
# add docker repo
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | _sudo apt-key add -
_sudo add-apt-repository \
"deb [arch=amd64] http://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable"
_sudo apt-get update
fi
fi
if [ "$CITBX_GIT_LFS_SUPPORT_ENABLED" == "true" ]; then
if setup_component_enabled git-lfs \
&& [ "$CITBX_GIT_LFS_SUPPORT_ENABLED" == "true" ]; then
if grep -qr 'git-lfs' /etc/apt/; then
print_note "GIT LFS apt repository is already present."
else
......@@ -77,50 +59,43 @@ if [ "$CITBX_GIT_LFS_SUPPORT_ENABLED" == "true" ]; then
fi
INSTALL_PKGS+=(git-lfs)
fi
_sudo apt-get -y install "${INSTALL_PKGS[@]}"
if [ "${USER}" != "root" ]; then
_sudo gpasswd -a ${USER} docker
if [ "${#INSTALL_PKGS[@]}" -gt 0 ]; then
print_info "Installing packages..."
_sudo apt-get -y install "${INSTALL_PKGS[@]}"
fi
# Setup docker0 bridge with:
# - NET: 192.168.255.0/24 (by default)
# - DNS: Use system dns instead of Google one
_sudo ip link del docker0 2>/dev/null || true
_sudo mkdir -p /etc/docker
if ! jq '' /etc/docker/daemon.json > /dev/null 2>&1; then
if [ -f /etc/docker/daemon.json ]; then
_sudo mv /etc/docker/daemon.json{,.bak}
print_warning "Invalid file /etc/docker/daemon.json, moving it to /etc/docker/daemon.json.bak"
if setup_component_enabled base-pkgs; then
if [ "${USER}" != "root" ]; then
_sudo gpasswd -a ${USER} docker
fi
_sudo bash -c 'echo {} > /etc/docker/daemon.json'
fi
for dns in $CITBX_DOCKER_DNS_LIST; do
if [ -n "$dnslist" ]; then
dnslist="$dnslist, \"$dns\""
else
dnslist="\"$dns\""
fi
done
# Put in comment the docker default options
if grep -q '^ *\<DOCKER_OPTS\>' /etc/default/docker; then
_sudo sed '/^ *\<DOCKER_OPTS\>/s/^/#/' -i /etc/default/docker
if setup_component_enabled ca-certs; then
print_info "Installing CA certificates..."
# Add user SSL ROOT CA
if [ -d $CITBX_ABS_DIR/ca-certificates ]; then
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /usr/local/share/ca-certificates/
_sudo update-ca-certificates
_sudo mkdir -p /etc/docker/certs.d
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /etc/docker/certs.d/
fi
fi
_sudo cat /etc/docker/daemon.json |
jq '. + {
"bip": "'"$CITBX_DOCKER_BIP"'",
"fixed-cidr": "'"$CITBX_DOCKER_FIXED_CIDR"'",
"dns": '"$(bashopts_dump_array "string" "${CITBX_DOCKER_DNS_LIST[@]}")"',
"storage-driver": "'"$CITBX_DOCKER_STORAGE_DRIVER"'"
}' | sudo_sponge /etc/docker/daemon.json
if setup_component_enabled docker-cfg; then
print_info "Configuring docker..."
write_daemon_json
# Put in comment the docker default options
if grep -q '^ *\<DOCKER_OPTS\>' /etc/default/docker; then
_sudo sed '/^ *\<DOCKER_OPTS\>/s/^/#/' -i /etc/default/docker
fi
_sudo ip link del docker0 2>/dev/null || true
_sudo service docker restart
fi
# Add user SSL ROOT CA
if [ -d $CITBX_ABS_DIR/ca-certificates ]; then
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /usr/local/share/ca-certificates/
_sudo update-ca-certificates
_sudo mkdir -p /etc/docker/certs.d
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /etc/docker/certs.d/
if setup_component_enabled ci-toolbox; then
print_info "Installing the CI toolbox $CITBX_TOOLBOX_NAME..."
install_ci_toolbox
fi
_sudo service docker restart
#!/bin/bash -e
# citools4gitlab: CI tools for Gitlab
# citbx4gitlab: CI toolbox for Gitlab
# Copyright (C) 2017 ERCOM - Emeric Verschuur <emeric@mbedsys.org>
#
# This program is free software: you can redistribute it and/or modify
......@@ -15,7 +15,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
CITBX_VERSION=3.1.0
CITBX_VERSION=3.2.0
# display a message
print_log() {
......@@ -285,6 +285,13 @@ citbx_check_env() {
;;
esac
}
setup_component_enabled() {
local pattern='\b'"$1"'\b'
if [[ "${CITBX_SETUP_COMPONENT[*]}" =~ $pattern ]]; then
return 0
fi
return 1
}
bashopts_process_option -n CITBX_DOCKER_DNS_LIST -r -k check_dns
. "$setupsh"
print_info "System setup complete" "On a first install, a system reboot may be necessary"
......@@ -461,32 +468,47 @@ $(for j in "${CITBX_JOB_LIST[@]}"; do echo " $j"; done | sort -u)"
setup)
bashopts_tool_usage="$CITBX_TOOL_NAME $command [arguments...]
=> type '$CITBX_TOOL_NAME help' to display the global help"
bashopts_declare -n CITBX_SETUP_COMPONENT -l component \
-t enum -m add -d "Setup only specified components" \
-e base-pkgs -e docker-cfg -e git-lfs -e ca-certs -e ci-toolbox \
-x '(base-pkgs docker-cfg git-lfs ca-certs ci-toolbox)'
check_tool_name() {
if [[ "$1" =~ ^[a-zA-Z0-9_-]+$ ]]; then
echo $1
return 0
fi
bashopts_log E "'$1' is not a valid tool name"
return 1
}
bashopts_declare -n CITBX_TOOLBOX_NAME -l toolbox-name \
-t string -d "CI toolbox name" -k check_tool_name -v ci-toolbox
bashopts_declare -n CITBX_DOCKER_BIP -l docker-bip -v "$(
val=$(jq -r '.bip' /etc/docker/daemon.json 2> /dev/null || true)
echo ${val:-"192.168.255.254/24"}
)" -t string -i -d "Local docker network IPV4 host adress"
)" -t string -d "Local docker network IPV4 host adress"
bashopts_declare -n CITBX_DOCKER_FIXED_CIDR -l docker-cdir -v "$(
val=$(jq -r '."fixed-cidr"' /etc/docker/daemon.json 2> /dev/null || true)
echo ${val:-"192.168.255.0/24"}
)" -t string -i -d "Local docker network IPV4 prefix"
)" -t string -d "Local docker network IPV4 prefix"
bashopts_declare -n CITBX_DOCKER_DNS_LIST -l docker-dns -m add \
-x "($(
if [ "0$(jq -e '.dns | length' /etc/docker/daemon.json 2> /dev/null || true)" -gt 0 ]; then
jq -r '.dns[]' /etc/docker/daemon.json 2> /dev/null | tr '\n' ' '
else
cat /etc/resolv.conf | awk '/^nameserver/ {
RESOLV_CONF_DNS="$(cat /etc/resolv.conf | awk '/^nameserver/ {
if ($2 !~ /^127\..*/ && $2 != "::1" ) {
printf(" %s", $2);
}
}'
}' 2> /dev/null || true)"
echo "${RESOLV_CONF_DNS:-${CITBX_DOCKER_DEFAULT_DNS[*]}}"
fi
) )" \
-t string -i -d "Docker DNS"
-t string -d "Docker DNS"
bashopts_declare -n CITBX_DOCKER_STORAGE_DRIVER -l docker-storage-driver -v "$(
val=$(jq -r '."storage-driver"' /etc/docker/daemon.json 2> /dev/null || true)
echo ${val:-"overlay2"}
)" -e 'o|overlay2' -e 'overlay' -e 'a|aufs' -e 'd|devicemapper' -e 'b|btrfs' -e 'z|zfs' \
-t enum -i -d "Docker storage driver"
-t enum -d "Docker storage driver"
;;
update)
;;
......@@ -628,20 +650,17 @@ bashopts_process_opts
citbx_check_env $(test "$command" != "setup" || echo "true")
if [ "$command" == "update" ]; then
fetch_file() {
local file=$1
local tg=${2:-"$CITBX_ABS_DIR/$1"}
local version=${bashopts_commands[0]:-"master"}
CITBX_REMOTE_URL=${CITBX_REMOTE_URL:-"https://gitlab.com/ercom/citbx4gitlab/raw/$version/tools/gitlab-ci"}
print_note "Fetching file $CITBX_REMOTE_URL/$file..."
mkdir -p $(dirname $tg)
curl -fSsLo $tg $CITBX_REMOTE_URL/$file \
|| print_warning "Unable to fetch file $CITBX_REMOTE_URL/$file"
}
fetch_file run.sh
tmpdir=$(mktemp -d)
version=${bashopts_commands[0]:-"master"}
print_note "Downloading $version archive from gitlab.com..."
curl -fSsL https://gitlab.com/ercom/citbx4gitlab/repository/$version/archive.tar.bz2 | tar -C $tmpdir -xj
srcdir="$tmpdir/$(ls -1 $tmpdir)"
cp -av $srcdir/tools/gitlab-ci/run.sh $CITBX_ABS_DIR/run.sh
chmod +x $CITBX_ABS_DIR/run.sh
fetch_file env-setup/ubuntu.sh
fetch_file 3rdparty/bashopts.sh $BASHOPTS_FILE_PATH
mkdir -p $CITBX_ABS_DIR/env-setup
cp -av $srcdir/tools/gitlab-ci/env-setup/* $CITBX_ABS_DIR/env-setup/
cp -av $srcdir/tools/gitlab-ci/3rdparty/bashopts.sh $BASHOPTS_FILE_PATH
rm -rf $tmpdir
print_info "Update done!"
exit 0
fi
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment