Commit f909deff authored by Stephane Bausseron's avatar Stephane Bausseron

Merge branch 'feature/3-3' into 'master'

New features/improvement and minor fixes

See merge request ercom/citbx4gitlab!17
parents 6bc2c5a8 15b3e288
job-with-volumes:
image: ubuntu:16.04
volumes_from_images:
- registry.gitlab.com/ercom/cijw4gitlab:volume-sample
job-test-services-ext:
image: postgres:9.4
volumes_from_images:
- registry.gitlab.com/ercom/cijw4gitlab:volume-sample
docker_run_args:
network: bridge
services:
- mysql
- name: postgres:9.4
alias: db-postgres
wait_before_run_job: 1
entrypoint: ["docker-entrypoint.sh"]
command: ["postgres"]
docker_run_args:
privileged: true
network: bridge
......@@ -2,7 +2,7 @@
image: ubuntu:16.04
variables:
GLOBALJOBVAR: "blobal variable"
GLOBALJOBVAR: "global variable"
after_script:
- echo "job ${CI_JOB_NAME} end"
......@@ -34,23 +34,12 @@ job-advanced:
- echo ${JOBADVAR}
- tools/gitlab-ci/run.sh
job-with-volumes:
image: registry.gitlab.com/ercom/cijw4gitlab:2.0.1
stage: build
only:
- /^$/
script:
- test -f /path/to/my/volume/content
- 'echo "/path/to/my/volume/content file content: $(cat /path/to/my/volume/content)"'
job-test-services-mysql:
stage: build
variables:
MYSQL_DATABASE: test
MYSQL_ROOT_PASSWORD: password
CITBX_WAIT_FOR_SERVICE_START: 10
only:
- /^$/
image: mysql
services:
- mysql
......@@ -63,8 +52,6 @@ job-test-services-postgres:
stage: build
variables:
CITBX_WAIT_FOR_SERVICE_START: 10
only:
- /^$/
image: postgres:9.4
services:
- name: postgres:9.4
......@@ -75,16 +62,3 @@ job-test-services-postgres:
- test
script:
- psql -h db-postgres -U postgres -c 'select version();'
job-test-services-ext:
image: registry.gitlab.com/ercom/cijw4gitlab:2.0.1
stage: build
variables:
MYSQL_DATABASE: test
MYSQL_ROOT_PASSWORD: password
only:
- /^$/
tags:
- test
script:
- "true"
# Cangelog
## 3.3.0
* [env-setup] Improve ca-certificates setup part
* [modules/dockerimg] Update dockerimg module
* [modules/example] minor fix on variable export
* Remove ci-job-wrapper jobs and update the documentation
* Update docker-login option
## 3.2.0
* [env-setup] Modular setup
+ add ability to limit setup on specified elements
......
......@@ -7,46 +7,45 @@ This toolbox can be used to
## Table of contents
* [The project](#the-project)
* [Use case: Standard Gitlab-CI pipeline job](#use-case-standard-gitlab-ci-pipeline-job)
* [Run a specific job locally](#run-a-specific-job-locally)
+ [Fist solution: using gitlab-runner tool](#fist-solution-using-gitlab-runner-tool)
+ [Other solution: Use run.sh tool](#other-solution-use-runsh-tool)
* [Write a job or a module script using CI toolbox](#write-a-job-or-a-module-script-using-ci-toolbox)
+ [Write a job](#write-a-job)
+ [Write a module](#write-a-module)
+ [Useful function list](#useful-function-list)
* [Job examples](#job-examples)
+ [job-minimal](#job-minimal)
+ [job-with-before-after-script](#job-with-before-after-script)
+ [job-advanced](#job-advanced)
+ [job-with-volumes](#job-with-volumes)
* [The project](#the-project)
* [Use case: Standard Gitlab-CI pipeline job](#use-case-standard-gitlab-ci-pipeline-job)
* [Run a specific job locally](#run-a-specific-job-locally)
* [Fist solution: using gitlab-runner tool](#fist-solution-using-gitlab-runner-tool)
* [Other solution: Use ci-toolbox command](#other-solution-use-ci-toolbox-command)
* [Write a job or a module script using CI toolbox](#write-a-job-or-a-module-script-using-ci-toolbox)
* [Write a job](#write-a-job)
* [Write a module](#write-a-module)
* [Useful function list](#useful-function-list)
* [Job examples](#job-examples)
* [job-minimal](#job-minimal)
* [job-with-before-after-script](#job-with-before-after-script)
* [job-advanced](#job-advanced)
* [job-test-services-mysql](#job-test-services-mysql)
* [job-test-services-postgres](#job-test-services-postgres)
## The project
The citbx4gitlab project tree
A citbx4gitlab integration example into a project tree
```
├── .ci-job-wrapper.yml
├── .git
├── .gitlab-ci.yml
├── doc
├── LICENSE
├── README.md
└── tools
└── gitlab-ci
├── 3rdparty
│ └── bashopts.sh
├── citbx.properties
├── env-setup
│ └── ubuntu.sh
├── modules
│ ├── ccache.sh
└── example.sh
├── run -> run.sh
├── run.d
└── job-advanced.sh
└── run.sh
├── tools
│   └── gitlab-ci
│   ├── 3rdparty
│   │   └── bashopts.sh
│   ├── citbx.properties
│   ├── env-setup
│   │   ├── common.sh
│   │   ├── gentoo.sh
│   │   └── ubuntu.sh
│   ├── modules
│   │   ├── ccache.sh
│   │   ├── dockerimg.sh
│   │   └── example.sh
│   ├── run -> run.sh
│   ├── run.d
│   │   └── job-advanced.sh
│   └── run.sh
```
The minimal element list:
......@@ -55,13 +54,16 @@ The minimal element list:
* ```run.sh```: tool to run Gitlab-CI job on the workstation
* ```bashopts.sh```: dependency required by run.sh
The additional element list:
The recommended additional element list:
* ```run.d```: directory containing job specific functions
* ```modules```: directory containing the modules
* ```env-setup```: directory containing the workstation environment setup script
* ```citbx.properties```: CI toolbox project properties
The advanced additional element list:
* ```run.d```: directory containing job specific functions
* ```modules```: directory containing the modules
## Use case: Standard Gitlab-CI pipeline job
The following schema describe the standard execution of a job called "J" in a Gitlab-CI pipeline
......@@ -88,7 +90,7 @@ In addition you have to:
* Commit the changes to be taken in board by the gitlab-runner
* Add gitlab-runner executor option like ```--docker-image```
### Other solution: Use run.sh tool
### Other solution: Use ci-toolbox command
This tool is able to:
......@@ -101,6 +103,17 @@ Additional advanced run.sh tool functionalities:
* Add ability to run a shell into the suitable job docker environment
* Split job parts in modular format (Job parts, modules, ...)
Usage:
* ```ci-toolbox <command> [arguments...]``` from anywhere in the project directory, ... or:
* ```path/to/run.sh <command> [arguments...]```
The ```ci-toolbox``` command, installed into the system during the ```path/to/run.sh setup```, add the following functionalities:
* This tool can be run anywhere in a project containing the run.sh script
* This tool is like a shortcut to find and run run.sh script without have to specify the absolute/relative path
* This tool add BASH auto completion support
Know limitations:
* Only applicable for docker and shell executor
......@@ -230,25 +243,47 @@ Job run:
![Job advanced](doc/TestCaseJobAvanced.png)
### job-with-volumes
### job-test-services-mysql
Dockerized job definition example with an additional volume
Dockerized job definition example with a mysql service
Job definition:
```yaml
job-with-volumes:
image: registry.gitlab.com/ercom/cijw4gitlab:1.0
stage: all-in-one
only:
- /^$/
job-test-services-mysql:
stage: build
variables:
MYSQL_DATABASE: test
MYSQL_ROOT_PASSWORD: password
CITBX_WAIT_FOR_SERVICE_START: 10
image: mysql
services:
- mysql
tags:
- test
script:
- test -f /path/to/my/volume/content
- 'echo "/path/to/my/volume/content file content: $(cat /path/to/my/volume/content)"'
- mysql -h mysql -u root -ppassword test -e 'SHOW VARIABLES LIKE "%version%";'
```
This project use:
### job-test-services-postgres
Dockerized job definition example with a postgresql service
Job definition:
* the functionality described at: [https://gitlab.com/ercom/cijw4gitlab](https://gitlab.com/ercom/cijw4gitlab)
* and the following volume: [https://gitlab.com/ercom/cijw4gitlab/tree/image-volume-example](https://gitlab.com/ercom/cijw4gitlab/tree/image-volume-example)
* This job need a runner with a volume /var/run/docker.sock to run an other container on the same docker daemon that gitlab-runner
```yaml
job-test-services-postgres:
stage: build
variables:
CITBX_WAIT_FOR_SERVICE_START: 10
image: postgres:9.4
services:
- name: postgres:9.4
alias: db-postgres
entrypoint: ["docker-entrypoint.sh"]
command: ["postgres"]
tags:
- test
script:
- psql -h db-postgres -U postgres -c 'select version();'
```
# CI toolbox for Gitlab properties
# CI toolbox for Gitlab - properties
#
# Default CI registry
#DEFAULT_CI_REGISTRY="registry.gitlab.example.com:5005"
DEFAULT_CI_REGISTRY="registry.gitlab.com"
#
# Default docker-login mode (enabled, disabled or auto)
#CITBX_DEFAULT_DOCKER_LOGIN_MODE=auto
#
# Settings file path
#CITBX_RC_PATH="$HOME/.config/myciprojectrc"
......@@ -13,7 +16,7 @@
CITBX_DEFAULT_JOB_SHELL="/bin/bash"
#
# Set service dockers in privileged mode as default
#CITBX_DEFAULT_SERVICE_DOCKER_PRIVILEGED="true"
CITBX_DEFAULT_SERVICE_DOCKER_PRIVILEGED="true"
#
# Enable support of git lfs
#CITBX_GIT_LFS_SUPPORT_ENABLED="true"
......
......@@ -61,3 +61,15 @@ install_ci_toolbox() {
_sudo curl -ksLo /usr/local/bin/$CITBX_TOOLBOX_NAME https://gitlab.com/ercom/citbx4gitlab/raw/master/tools/gitlab-ci/citbx4gitlab/citbx4gitlab
_sudo chmod +x /usr/local/bin/$CITBX_TOOLBOX_NAME
}
install_ca_certificates_system() {
test -d $CITBX_ABS_DIR/ca-certificates || return 0
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /usr/local/share/ca-certificates/
_sudo update-ca-certificates
}
install_ca_certificates_docker() {
test -d $CITBX_ABS_DIR/ca-certificates || return 0
_sudo mkdir -p /etc/docker/certs.d
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /etc/docker/certs.d/
}
......@@ -3,7 +3,7 @@
INSTALL_PKGS=()
if setup_component_enabled base-pkgs; then
for pkg in app-emulation/docker sys-apps/gawk dev-python/pyyaml app-misc/jq; do
for pkg in app-emulation/docker sys-apps/gawk dev-python/pyyaml app-misc/jq app-misc/ca-certificates; do
if ! equery -q list $pkg > /dev/null; then
INSTALL_PKGS+=($pkg)
fi
......@@ -29,13 +29,9 @@ fi
if setup_component_enabled ca-certs; then
print_info "Installing CA certificates..."
# Add user SSL ROOT CA
if [ -d $CITBX_ABS_DIR/ca-certificates ]; then
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /usr/local/share/ca-certificates/
_sudo update-ca-certificates
_sudo mkdir -p /etc/docker/certs.d
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /etc/docker/certs.d/
fi
# Add custom SSL ROOT CAs
install_ca_certificates_system
install_ca_certificates_docker
fi
if setup_component_enabled docker-cfg; then
......
......@@ -73,13 +73,9 @@ fi
if setup_component_enabled ca-certs; then
print_info "Installing CA certificates..."
# Add user SSL ROOT CA
if [ -d $CITBX_ABS_DIR/ca-certificates ]; then
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /usr/local/share/ca-certificates/
_sudo update-ca-certificates
_sudo mkdir -p /etc/docker/certs.d
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /etc/docker/certs.d/
fi
# Add custom SSL ROOT CAs
install_ca_certificates_system
install_ca_certificates_docker
fi
if setup_component_enabled docker-cfg; then
......
......@@ -12,10 +12,11 @@ citbx_module_dockerimg_define() {
}')
local declare_opts=()
if [ -n "$prj_name" ]; then
declare_opts+=(-e "\"\$CI_REGISTRY/$prj_name\"")
declare_opts+=(-x "\"\$CI_REGISTRY/$prj_name\"")
fi
bashopts_declare -n CI_REGISTRY_IMAGE -l image-name -d "Registry image name" -t string "${declare_opts[@]}"
bashopts_declare -n CI_COMMIT_TAG -l image-tag -d "Image tag" -t string -v "test"
bashopts_declare -n USE_LOCAL_DOCKER -l use-local-docker -d "Use the local docker instance instead of the dind service" -t boolean
CITBX_UID=0
CITBX_JOB_SHELL=${CITBX_JOB_SHELL:-/bin/sh}
citbx_export CI_REGISTRY_IMAGE CI_COMMIT_TAG
......@@ -23,4 +24,12 @@ citbx_module_dockerimg_define() {
citbx_module_dockerimg_setup() {
bashopts_process_option -n CI_REGISTRY_IMAGE -r
if [ "$USE_LOCAL_DOCKER" == "true" ]; then
CITBX_DISABLED_SERVICES+=(docker)
fi
pattern='\bdocker\b'
if [[ "${CITBX_DISABLED_SERVICES[*]}" =~ $pattern ]]; then
DOCKER_HOST="unix:///var/run/docker.sock"
citbx_export DOCKER_HOST
fi
}
citbx_module_example_define() {
bashopts_declare -n EXAMPLE_VALUE -l example -t string -d "Example option" -v "my example value"
X_CI_ENV_EXPORT_LIST+=(EXAMPLE_VALUE)
citbx_export EXAMPLE_VALUE
}
citbx_module_example_setup() {
......
......@@ -15,7 +15,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
CITBX_VERSION=3.2.0
CITBX_VERSION=3.3.0
# display a message
print_log() {
......@@ -569,7 +569,8 @@ $(for j in "${CITBX_JOB_LIST[@]}"; do echo " $j"; done | sort -u)"
fi
bashopts_declare -n CI_REGISTRY -l docker-registry -d "Docker registry" -t string -s "${declare_opts[@]}"
unset declare_opts
bashopts_declare -n CITBX_DOCKER_LOGIN -l docker-login -o l -d "Execute docker login" -t boolean
bashopts_declare -n CITBX_DOCKER_LOGIN_MODE -l docker-login -d "Execute docker login" -t enum \
-e "enabled" -e "disabled" -e "auto" -v "${CITBX_DEFAULT_DOCKER_LOGIN_MODE:-auto}"
bashopts_declare -n CITBX_JOB_EXECUTOR -l job-executor -o e \
-d "Job executor type (only docker or shell is sypported yet)" -t enum \
-v "$(test -n "$CITBX_DEFAULT_DOCKER_IMAGE" && echo "docker" || echo "shell" )" \
......@@ -671,8 +672,9 @@ fi
# Login to the registry if needed
if [ -n "$CI_REGISTRY" ] \
&& ( [ -z "$(jq -r '."auths"."'$CI_REGISTRY'"."auth"' $HOME/.docker/config.json 2> /dev/null)" ] \
|| [ "$CITBX_DOCKER_LOGIN" == "true" ] ); then
&& ( ( [ -z "$(jq -r '."auths"."'$CI_REGISTRY'" | select(.auth != null)' $HOME/.docker/config.json 2> /dev/null)" ] \
&& [ "$CITBX_DOCKER_LOGIN_MODE" == "auto" ] ) \
|| [ "$CITBX_DOCKER_LOGIN_MODE" == "true" ] ); then
print_info "You are not authenticated against the gitlab docker registry" \
"> Please enter your gitlab user id and password:"
docker login $CI_REGISTRY
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment