Commit f909deff authored by Stephane Bausseron's avatar Stephane Bausseron

Merge branch 'feature/3-3' into 'master'

New features/improvement and minor fixes

See merge request ercom/citbx4gitlab!17
parents 6bc2c5a8 15b3e288
job-with-volumes:
image: ubuntu:16.04
volumes_from_images:
- registry.gitlab.com/ercom/cijw4gitlab:volume-sample
job-test-services-ext:
image: postgres:9.4
volumes_from_images:
- registry.gitlab.com/ercom/cijw4gitlab:volume-sample
docker_run_args:
network: bridge
services:
- mysql
- name: postgres:9.4
alias: db-postgres
wait_before_run_job: 1
entrypoint: ["docker-entrypoint.sh"]
command: ["postgres"]
docker_run_args:
privileged: true
network: bridge
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
image: ubuntu:16.04 image: ubuntu:16.04
variables: variables:
GLOBALJOBVAR: "blobal variable" GLOBALJOBVAR: "global variable"
after_script: after_script:
- echo "job ${CI_JOB_NAME} end" - echo "job ${CI_JOB_NAME} end"
...@@ -34,23 +34,12 @@ job-advanced: ...@@ -34,23 +34,12 @@ job-advanced:
- echo ${JOBADVAR} - echo ${JOBADVAR}
- tools/gitlab-ci/run.sh - tools/gitlab-ci/run.sh
job-with-volumes:
image: registry.gitlab.com/ercom/cijw4gitlab:2.0.1
stage: build
only:
- /^$/
script:
- test -f /path/to/my/volume/content
- 'echo "/path/to/my/volume/content file content: $(cat /path/to/my/volume/content)"'
job-test-services-mysql: job-test-services-mysql:
stage: build stage: build
variables: variables:
MYSQL_DATABASE: test MYSQL_DATABASE: test
MYSQL_ROOT_PASSWORD: password MYSQL_ROOT_PASSWORD: password
CITBX_WAIT_FOR_SERVICE_START: 10 CITBX_WAIT_FOR_SERVICE_START: 10
only:
- /^$/
image: mysql image: mysql
services: services:
- mysql - mysql
...@@ -63,8 +52,6 @@ job-test-services-postgres: ...@@ -63,8 +52,6 @@ job-test-services-postgres:
stage: build stage: build
variables: variables:
CITBX_WAIT_FOR_SERVICE_START: 10 CITBX_WAIT_FOR_SERVICE_START: 10
only:
- /^$/
image: postgres:9.4 image: postgres:9.4
services: services:
- name: postgres:9.4 - name: postgres:9.4
...@@ -75,16 +62,3 @@ job-test-services-postgres: ...@@ -75,16 +62,3 @@ job-test-services-postgres:
- test - test
script: script:
- psql -h db-postgres -U postgres -c 'select version();' - psql -h db-postgres -U postgres -c 'select version();'
job-test-services-ext:
image: registry.gitlab.com/ercom/cijw4gitlab:2.0.1
stage: build
variables:
MYSQL_DATABASE: test
MYSQL_ROOT_PASSWORD: password
only:
- /^$/
tags:
- test
script:
- "true"
# Cangelog # Cangelog
## 3.3.0
* [env-setup] Improve ca-certificates setup part
* [modules/dockerimg] Update dockerimg module
* [modules/example] minor fix on variable export
* Remove ci-job-wrapper jobs and update the documentation
* Update docker-login option
## 3.2.0 ## 3.2.0
* [env-setup] Modular setup * [env-setup] Modular setup
+ add ability to limit setup on specified elements + add ability to limit setup on specified elements
......
...@@ -7,46 +7,45 @@ This toolbox can be used to ...@@ -7,46 +7,45 @@ This toolbox can be used to
## Table of contents ## Table of contents
* [The project](#the-project) * [The project](#the-project)
* [Use case: Standard Gitlab-CI pipeline job](#use-case-standard-gitlab-ci-pipeline-job) * [Use case: Standard Gitlab-CI pipeline job](#use-case-standard-gitlab-ci-pipeline-job)
* [Run a specific job locally](#run-a-specific-job-locally) * [Run a specific job locally](#run-a-specific-job-locally)
+ [Fist solution: using gitlab-runner tool](#fist-solution-using-gitlab-runner-tool) * [Fist solution: using gitlab-runner tool](#fist-solution-using-gitlab-runner-tool)
+ [Other solution: Use run.sh tool](#other-solution-use-runsh-tool) * [Other solution: Use ci-toolbox command](#other-solution-use-ci-toolbox-command)
* [Write a job or a module script using CI toolbox](#write-a-job-or-a-module-script-using-ci-toolbox) * [Write a job or a module script using CI toolbox](#write-a-job-or-a-module-script-using-ci-toolbox)
+ [Write a job](#write-a-job) * [Write a job](#write-a-job)
+ [Write a module](#write-a-module) * [Write a module](#write-a-module)
+ [Useful function list](#useful-function-list) * [Useful function list](#useful-function-list)
* [Job examples](#job-examples) * [Job examples](#job-examples)
+ [job-minimal](#job-minimal) * [job-minimal](#job-minimal)
+ [job-with-before-after-script](#job-with-before-after-script) * [job-with-before-after-script](#job-with-before-after-script)
+ [job-advanced](#job-advanced) * [job-advanced](#job-advanced)
+ [job-with-volumes](#job-with-volumes) * [job-test-services-mysql](#job-test-services-mysql)
* [job-test-services-postgres](#job-test-services-postgres)
## The project ## The project
The citbx4gitlab project tree A citbx4gitlab integration example into a project tree
``` ```
├── .ci-job-wrapper.yml
├── .git
├── .gitlab-ci.yml ├── .gitlab-ci.yml
├── doc ├── tools
├── LICENSE │   └── gitlab-ci
├── README.md │   ├── 3rdparty
└── tools │   │   └── bashopts.sh
└── gitlab-ci │   ├── citbx.properties
├── 3rdparty │   ├── env-setup
│ └── bashopts.sh │   │   ├── common.sh
├── citbx.properties │   │   ├── gentoo.sh
├── env-setup │   │   └── ubuntu.sh
│ └── ubuntu.sh │   ├── modules
├── modules │   │   ├── ccache.sh
│ ├── ccache.sh │   │   ├── dockerimg.sh
└── example.sh │   │   └── example.sh
├── run -> run.sh │   ├── run -> run.sh
├── run.d │   ├── run.d
└── job-advanced.sh │   │   └── job-advanced.sh
└── run.sh │   └── run.sh
``` ```
The minimal element list: The minimal element list:
...@@ -55,13 +54,16 @@ The minimal element list: ...@@ -55,13 +54,16 @@ The minimal element list:
* ```run.sh```: tool to run Gitlab-CI job on the workstation * ```run.sh```: tool to run Gitlab-CI job on the workstation
* ```bashopts.sh```: dependency required by run.sh * ```bashopts.sh```: dependency required by run.sh
The additional element list: The recommended additional element list:
* ```run.d```: directory containing job specific functions
* ```modules```: directory containing the modules
* ```env-setup```: directory containing the workstation environment setup script * ```env-setup```: directory containing the workstation environment setup script
* ```citbx.properties```: CI toolbox project properties * ```citbx.properties```: CI toolbox project properties
The advanced additional element list:
* ```run.d```: directory containing job specific functions
* ```modules```: directory containing the modules
## Use case: Standard Gitlab-CI pipeline job ## Use case: Standard Gitlab-CI pipeline job
The following schema describe the standard execution of a job called "J" in a Gitlab-CI pipeline The following schema describe the standard execution of a job called "J" in a Gitlab-CI pipeline
...@@ -88,7 +90,7 @@ In addition you have to: ...@@ -88,7 +90,7 @@ In addition you have to:
* Commit the changes to be taken in board by the gitlab-runner * Commit the changes to be taken in board by the gitlab-runner
* Add gitlab-runner executor option like ```--docker-image``` * Add gitlab-runner executor option like ```--docker-image```
### Other solution: Use run.sh tool ### Other solution: Use ci-toolbox command
This tool is able to: This tool is able to:
...@@ -101,6 +103,17 @@ Additional advanced run.sh tool functionalities: ...@@ -101,6 +103,17 @@ Additional advanced run.sh tool functionalities:
* Add ability to run a shell into the suitable job docker environment * Add ability to run a shell into the suitable job docker environment
* Split job parts in modular format (Job parts, modules, ...) * Split job parts in modular format (Job parts, modules, ...)
Usage:
* ```ci-toolbox <command> [arguments...]``` from anywhere in the project directory, ... or:
* ```path/to/run.sh <command> [arguments...]```
The ```ci-toolbox``` command, installed into the system during the ```path/to/run.sh setup```, add the following functionalities:
* This tool can be run anywhere in a project containing the run.sh script
* This tool is like a shortcut to find and run run.sh script without have to specify the absolute/relative path
* This tool add BASH auto completion support
Know limitations: Know limitations:
* Only applicable for docker and shell executor * Only applicable for docker and shell executor
...@@ -230,25 +243,47 @@ Job run: ...@@ -230,25 +243,47 @@ Job run:
![Job advanced](doc/TestCaseJobAvanced.png) ![Job advanced](doc/TestCaseJobAvanced.png)
### job-with-volumes ### job-test-services-mysql
Dockerized job definition example with an additional volume Dockerized job definition example with a mysql service
Job definition: Job definition:
```yaml ```yaml
job-with-volumes: job-test-services-mysql:
image: registry.gitlab.com/ercom/cijw4gitlab:1.0 stage: build
stage: all-in-one variables:
only: MYSQL_DATABASE: test
- /^$/ MYSQL_ROOT_PASSWORD: password
CITBX_WAIT_FOR_SERVICE_START: 10
image: mysql
services:
- mysql
tags:
- test
script: script:
- test -f /path/to/my/volume/content - mysql -h mysql -u root -ppassword test -e 'SHOW VARIABLES LIKE "%version%";'
- 'echo "/path/to/my/volume/content file content: $(cat /path/to/my/volume/content)"'
``` ```
This project use: ### job-test-services-postgres
Dockerized job definition example with a postgresql service
Job definition:
* the functionality described at: [https://gitlab.com/ercom/cijw4gitlab](https://gitlab.com/ercom/cijw4gitlab) ```yaml
* and the following volume: [https://gitlab.com/ercom/cijw4gitlab/tree/image-volume-example](https://gitlab.com/ercom/cijw4gitlab/tree/image-volume-example) job-test-services-postgres:
* This job need a runner with a volume /var/run/docker.sock to run an other container on the same docker daemon that gitlab-runner stage: build
variables:
CITBX_WAIT_FOR_SERVICE_START: 10
image: postgres:9.4
services:
- name: postgres:9.4
alias: db-postgres
entrypoint: ["docker-entrypoint.sh"]
command: ["postgres"]
tags:
- test
script:
- psql -h db-postgres -U postgres -c 'select version();'
```
# CI toolbox for Gitlab properties # CI toolbox for Gitlab - properties
# #
# Default CI registry # Default CI registry
#DEFAULT_CI_REGISTRY="registry.gitlab.example.com:5005" DEFAULT_CI_REGISTRY="registry.gitlab.com"
#
# Default docker-login mode (enabled, disabled or auto)
#CITBX_DEFAULT_DOCKER_LOGIN_MODE=auto
# #
# Settings file path # Settings file path
#CITBX_RC_PATH="$HOME/.config/myciprojectrc" #CITBX_RC_PATH="$HOME/.config/myciprojectrc"
...@@ -13,7 +16,7 @@ ...@@ -13,7 +16,7 @@
CITBX_DEFAULT_JOB_SHELL="/bin/bash" CITBX_DEFAULT_JOB_SHELL="/bin/bash"
# #
# Set service dockers in privileged mode as default # Set service dockers in privileged mode as default
#CITBX_DEFAULT_SERVICE_DOCKER_PRIVILEGED="true" CITBX_DEFAULT_SERVICE_DOCKER_PRIVILEGED="true"
# #
# Enable support of git lfs # Enable support of git lfs
#CITBX_GIT_LFS_SUPPORT_ENABLED="true" #CITBX_GIT_LFS_SUPPORT_ENABLED="true"
......
...@@ -61,3 +61,15 @@ install_ci_toolbox() { ...@@ -61,3 +61,15 @@ install_ci_toolbox() {
_sudo curl -ksLo /usr/local/bin/$CITBX_TOOLBOX_NAME https://gitlab.com/ercom/citbx4gitlab/raw/master/tools/gitlab-ci/citbx4gitlab/citbx4gitlab _sudo curl -ksLo /usr/local/bin/$CITBX_TOOLBOX_NAME https://gitlab.com/ercom/citbx4gitlab/raw/master/tools/gitlab-ci/citbx4gitlab/citbx4gitlab
_sudo chmod +x /usr/local/bin/$CITBX_TOOLBOX_NAME _sudo chmod +x /usr/local/bin/$CITBX_TOOLBOX_NAME
} }
install_ca_certificates_system() {
test -d $CITBX_ABS_DIR/ca-certificates || return 0
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /usr/local/share/ca-certificates/
_sudo update-ca-certificates
}
install_ca_certificates_docker() {
test -d $CITBX_ABS_DIR/ca-certificates || return 0
_sudo mkdir -p /etc/docker/certs.d
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /etc/docker/certs.d/
}
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
INSTALL_PKGS=() INSTALL_PKGS=()
if setup_component_enabled base-pkgs; then if setup_component_enabled base-pkgs; then
for pkg in app-emulation/docker sys-apps/gawk dev-python/pyyaml app-misc/jq; do for pkg in app-emulation/docker sys-apps/gawk dev-python/pyyaml app-misc/jq app-misc/ca-certificates; do
if ! equery -q list $pkg > /dev/null; then if ! equery -q list $pkg > /dev/null; then
INSTALL_PKGS+=($pkg) INSTALL_PKGS+=($pkg)
fi fi
...@@ -29,13 +29,9 @@ fi ...@@ -29,13 +29,9 @@ fi
if setup_component_enabled ca-certs; then if setup_component_enabled ca-certs; then
print_info "Installing CA certificates..." print_info "Installing CA certificates..."
# Add user SSL ROOT CA # Add custom SSL ROOT CAs
if [ -d $CITBX_ABS_DIR/ca-certificates ]; then install_ca_certificates_system
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /usr/local/share/ca-certificates/ install_ca_certificates_docker
_sudo update-ca-certificates
_sudo mkdir -p /etc/docker/certs.d
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /etc/docker/certs.d/
fi
fi fi
if setup_component_enabled docker-cfg; then if setup_component_enabled docker-cfg; then
......
...@@ -73,13 +73,9 @@ fi ...@@ -73,13 +73,9 @@ fi
if setup_component_enabled ca-certs; then if setup_component_enabled ca-certs; then
print_info "Installing CA certificates..." print_info "Installing CA certificates..."
# Add user SSL ROOT CA # Add custom SSL ROOT CAs
if [ -d $CITBX_ABS_DIR/ca-certificates ]; then install_ca_certificates_system
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /usr/local/share/ca-certificates/ install_ca_certificates_docker
_sudo update-ca-certificates
_sudo mkdir -p /etc/docker/certs.d
_sudo cp $CITBX_ABS_DIR/ca-certificates/*.crt /etc/docker/certs.d/
fi
fi fi
if setup_component_enabled docker-cfg; then if setup_component_enabled docker-cfg; then
......
...@@ -12,10 +12,11 @@ citbx_module_dockerimg_define() { ...@@ -12,10 +12,11 @@ citbx_module_dockerimg_define() {
}') }')
local declare_opts=() local declare_opts=()
if [ -n "$prj_name" ]; then if [ -n "$prj_name" ]; then
declare_opts+=(-e "\"\$CI_REGISTRY/$prj_name\"") declare_opts+=(-x "\"\$CI_REGISTRY/$prj_name\"")
fi fi
bashopts_declare -n CI_REGISTRY_IMAGE -l image-name -d "Registry image name" -t string "${declare_opts[@]}" bashopts_declare -n CI_REGISTRY_IMAGE -l image-name -d "Registry image name" -t string "${declare_opts[@]}"
bashopts_declare -n CI_COMMIT_TAG -l image-tag -d "Image tag" -t string -v "test" bashopts_declare -n CI_COMMIT_TAG -l image-tag -d "Image tag" -t string -v "test"
bashopts_declare -n USE_LOCAL_DOCKER -l use-local-docker -d "Use the local docker instance instead of the dind service" -t boolean
CITBX_UID=0 CITBX_UID=0
CITBX_JOB_SHELL=${CITBX_JOB_SHELL:-/bin/sh} CITBX_JOB_SHELL=${CITBX_JOB_SHELL:-/bin/sh}
citbx_export CI_REGISTRY_IMAGE CI_COMMIT_TAG citbx_export CI_REGISTRY_IMAGE CI_COMMIT_TAG
...@@ -23,4 +24,12 @@ citbx_module_dockerimg_define() { ...@@ -23,4 +24,12 @@ citbx_module_dockerimg_define() {
citbx_module_dockerimg_setup() { citbx_module_dockerimg_setup() {
bashopts_process_option -n CI_REGISTRY_IMAGE -r bashopts_process_option -n CI_REGISTRY_IMAGE -r
if [ "$USE_LOCAL_DOCKER" == "true" ]; then
CITBX_DISABLED_SERVICES+=(docker)
fi
pattern='\bdocker\b'
if [[ "${CITBX_DISABLED_SERVICES[*]}" =~ $pattern ]]; then
DOCKER_HOST="unix:///var/run/docker.sock"
citbx_export DOCKER_HOST
fi
} }
citbx_module_example_define() { citbx_module_example_define() {
bashopts_declare -n EXAMPLE_VALUE -l example -t string -d "Example option" -v "my example value" bashopts_declare -n EXAMPLE_VALUE -l example -t string -d "Example option" -v "my example value"
X_CI_ENV_EXPORT_LIST+=(EXAMPLE_VALUE) citbx_export EXAMPLE_VALUE
} }
citbx_module_example_setup() { citbx_module_example_setup() {
......
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
CITBX_VERSION=3.2.0 CITBX_VERSION=3.3.0
# display a message # display a message
print_log() { print_log() {
...@@ -569,7 +569,8 @@ $(for j in "${CITBX_JOB_LIST[@]}"; do echo " $j"; done | sort -u)" ...@@ -569,7 +569,8 @@ $(for j in "${CITBX_JOB_LIST[@]}"; do echo " $j"; done | sort -u)"
fi fi
bashopts_declare -n CI_REGISTRY -l docker-registry -d "Docker registry" -t string -s "${declare_opts[@]}" bashopts_declare -n CI_REGISTRY -l docker-registry -d "Docker registry" -t string -s "${declare_opts[@]}"
unset declare_opts unset declare_opts
bashopts_declare -n CITBX_DOCKER_LOGIN -l docker-login -o l -d "Execute docker login" -t boolean bashopts_declare -n CITBX_DOCKER_LOGIN_MODE -l docker-login -d "Execute docker login" -t enum \
-e "enabled" -e "disabled" -e "auto" -v "${CITBX_DEFAULT_DOCKER_LOGIN_MODE:-auto}"
bashopts_declare -n CITBX_JOB_EXECUTOR -l job-executor -o e \ bashopts_declare -n CITBX_JOB_EXECUTOR -l job-executor -o e \
-d "Job executor type (only docker or shell is sypported yet)" -t enum \ -d "Job executor type (only docker or shell is sypported yet)" -t enum \
-v "$(test -n "$CITBX_DEFAULT_DOCKER_IMAGE" && echo "docker" || echo "shell" )" \ -v "$(test -n "$CITBX_DEFAULT_DOCKER_IMAGE" && echo "docker" || echo "shell" )" \
...@@ -671,8 +672,9 @@ fi ...@@ -671,8 +672,9 @@ fi
# Login to the registry if needed # Login to the registry if needed
if [ -n "$CI_REGISTRY" ] \ if [ -n "$CI_REGISTRY" ] \
&& ( [ -z "$(jq -r '."auths"."'$CI_REGISTRY'"."auth"' $HOME/.docker/config.json 2> /dev/null)" ] \ && ( ( [ -z "$(jq -r '."auths"."'$CI_REGISTRY'" | select(.auth != null)' $HOME/.docker/config.json 2> /dev/null)" ] \
|| [ "$CITBX_DOCKER_LOGIN" == "true" ] ); then && [ "$CITBX_DOCKER_LOGIN_MODE" == "auto" ] ) \
|| [ "$CITBX_DOCKER_LOGIN_MODE" == "true" ] ); then
print_info "You are not authenticated against the gitlab docker registry" \ print_info "You are not authenticated against the gitlab docker registry" \
"> Please enter your gitlab user id and password:" "> Please enter your gitlab user id and password:"
docker login $CI_REGISTRY docker login $CI_REGISTRY
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment