Commit 958e2a12 authored by Fabien Catteau's avatar Fabien Catteau Committed by Philippe Lafoucrière

Rewrite using Go, common lib and shared orchestrator

parent 59ec62d7
*.gem
*.rbc
# binary that can be downloaded during dev
/bin/gemnasium
/bin/find-sec-bugs-launcher
/.config
/coverage/
/InstalledFiles
/pkg/
/spec/reports/
/spec/examples.txt
/test/tmp/
/test/version_tmp/
/tmp/
# Used by dotenv library to load environment variables.
# .env
## Specific to RubyMotion:
.dat*
.repl_history
build/
*.bridgesupport
build-iPhoneOS/
build-iPhoneSimulator/
## Specific to RubyMotion (use of CocoaPods):
#
# We recommend against adding the Pods directory to your .gitignore. However
# you should judge for yourself, the pros and cons are mentioned at:
# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
#
# vendor/Pods/
## Documentation cache and generated files:
/.yardoc/
/_yardoc/
/doc/
/rdoc/
## Environment normalization:
/.bundle/
/vendor/bundle
/lib/bundler/man/
# for a library or gem, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# Gemfile.lock
# .ruby-version
# .ruby-gemset
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
.rvmrc
dependency-scanning
test/fixtures/gl-dependency-scanning-report.json
include:
- https://gitlab.com/gitlab-org/security-products/ci-templates/raw/master/includes/sast.yml
- https://gitlab.com/gitlab-org/security-products/ci-templates/raw/master/includes/dependency_scanning.yml
- https://gitlab.com/gitlab-org/security-products/ci-templates/raw/master/includes/container_scanning.yml
- https://gitlab.com/gitlab-org/security-products/ci-templates/raw/master/includes/codequality.yml
image: alpine:latest
# When using dind, it's wise to use the overlayfs driver for
# improved performance.
variables:
POSTGRES_ENABLED: "false"
DOCKER_DRIVER: overlay2
MAJOR: 1
services:
- docker:stable-dind
stages:
- go
- test
- release
- build
- deploy
test:
stage: test
image: docker:stable
services:
- docker:dind
.go:
image: golang:1.11
stage: go
variables:
GO111MODULE: "on"
before_script:
- mkdir -p /go/src/gitlab.com/${CI_PROJECT_PATH}
- cp -r . /go/src/gitlab.com/${CI_PROJECT_PATH}
- cd /go/src/gitlab.com/${CI_PROJECT_PATH}
go build:
extends: .go
variables:
DOCKER_DRIVER: overlay2
CGO_ENABLED: 0
script:
- setup_docker
- docker run
--volume "$PWD:/app"
--volume /var/run/docker.sock:/var/run/docker.sock
ruby:2.3 sh -c "cd /app && bundle install && rspec spec"
only:
- branches
- go get ./...
- go build -o ${CI_PROJECT_DIR}/dependency-scanning
artifacts:
paths:
- dependency-scanning
rubocop:
stage: test
image: ruby:2.3
go test:
extends: .go
script:
- bundle install
- bundle exec rubocop
only:
- branches
- go get ./...
- go test -race -cover -v ./...
container_scanning:
go lint:
extends: .go
script:
- go get -u golang.org/x/lint/golint
- golint -set_exit_status
test:
image: docker:stable
allow_failure: true
before_script: []
cache: {}
dependencies: []
tags: []
services:
- docker:stable-dind
stage: test
variables:
DOCKER_DRIVER: overlay2
## Define two new variables based on GitLab's CI/CD predefined variables
## https://docs.gitlab.com/ee/ci/variables/#predefined-variables-environment-variables
CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG
CI_APPLICATION_TAG: $CI_COMMIT_SHA
DS_ANALYZER_IMAGE_TAG: $MAJOR
script:
# Build image locally as it doesn't exist on registry
- docker build -t "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" .
- docker run -d --name db arminc/clair-db:latest
- docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.1
- apk add -U wget ca-certificates
# - docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG}
- wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
- mv clair-scanner_linux_amd64 clair-scanner
- chmod +x clair-scanner
- touch clair-whitelist.yml
- retries=0
- echo "Waiting for clair daemon to start"
- while( ! wget -T 10 -q -O /dev/null http://docker:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done
- ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-sast-container-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true
artifacts:
paths:
- gl-sast-container-report.json
- ./test.sh
dependencies:
- go build
release-latest-and-next-version:
stage: release
image: docker:git
services:
- docker:dind
variables:
DOCKER_DRIVER: overlay2
.build:
image: docker:stable
stage: build
script:
- echo "Building Docker image..."
- docker build -t "$CI_REGISTRY_IMAGE:latest" .
- docker tag "$CI_REGISTRY_IMAGE:latest" "$CI_REGISTRY_IMAGE:$(cat VERSION)"
- echo "Logging to GitLab Container Registry with CI credentials..."
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
- echo "Pushing to GitLab Container Registry..."
- docker push "$CI_REGISTRY_IMAGE:latest"
- docker push "$CI_REGISTRY_IMAGE:$(cat VERSION)"
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- export IMAGE=$CI_REGISTRY_IMAGE:$IMAGE_TAG
- docker build -t $IMAGE --build-arg DS_ANALYZER_IMAGE_TAG=$DS_ANALYZER_IMAGE_TAG .
- docker push $IMAGE
dependencies:
- go build
build branch:
extends: .build
variables:
IMAGE_TAG: $CI_COMMIT_REF_NAME
DS_ANALYZER_IMAGE_TAG: $MAJOR
only:
- branches
except:
- master
release-stable:
stage: release
image: docker:git
services:
- docker:dind
build edge:
extends: .build
variables:
DOCKER_DRIVER: overlay2
script:
- echo "Building Docker image..."
- docker build -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" .
- echo "Logging to GitLab Container Registry with CI credentials..."
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
- echo "Pushing to GitLab Container Registry..."
- docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
IMAGE_TAG: edge
DS_ANALYZER_IMAGE_TAG: edge
only:
- /^\d+-\d+-stable$/
# ---------------------------------------------------------------------------
.auto_devops: &auto_devops |
# Auto DevOps variables and functions
[[ "$TRACE" ]] && set -x
auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB}
export DATABASE_URL=${DATABASE_URL-$auto_database_url}
export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG
export CI_APPLICATION_TAG=$CI_COMMIT_SHA
export CI_CONTAINER_NAME=ci_job_build_${CI_JOB_ID}
export TILLER_NAMESPACE=$KUBE_NAMESPACE
function deploy() {
track="${1-stable}"
name="$CI_ENVIRONMENT_SLUG"
if [[ "$track" != "stable" ]]; then
name="$name-$track"
fi
replicas="1"
service_enabled="false"
postgres_enabled="$POSTGRES_ENABLED"
# canary uses stable db
[[ "$track" == "canary" ]] && postgres_enabled="false"
env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' )
env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' )
if [[ "$track" == "stable" ]]; then
# for stable track get number of replicas from `PRODUCTION_REPLICAS`
eval new_replicas=\$${env_slug}_REPLICAS
service_enabled="true"
else
# for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS`
eval new_replicas=\$${env_track}_${env_slug}_REPLICAS
fi
if [[ -n "$new_replicas" ]]; then
replicas="$new_replicas"
fi
if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then
secret_name='gitlab-registry'
else
secret_name=''
fi
helm upgrade --install \
--wait \
--set service.enabled="$service_enabled" \
--set releaseOverride="$CI_ENVIRONMENT_SLUG" \
--set image.repository="$CI_APPLICATION_REPOSITORY" \
--set image.tag="$CI_APPLICATION_TAG" \
--set image.pullPolicy=IfNotPresent \
--set image.secrets[0].name="$secret_name" \
--set application.track="$track" \
--set application.database_url="$DATABASE_URL" \
--set service.url="$CI_ENVIRONMENT_URL" \
--set replicaCount="$replicas" \
--set postgresql.enabled="$postgres_enabled" \
--set postgresql.nameOverride="postgres" \
--set postgresql.postgresUser="$POSTGRES_USER" \
--set postgresql.postgresPassword="$POSTGRES_PASSWORD" \
--set postgresql.postgresDatabase="$POSTGRES_DB" \
--namespace="$KUBE_NAMESPACE" \
--version="$CI_PIPELINE_ID-$CI_JOB_ID" \
"$name" \
chart/
}
function install_dependencies() {
apk add -U openssl curl tar gzip bash ca-certificates git
wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub
wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.23-r3/glibc-2.23-r3.apk
apk add glibc-2.23-r3.apk
rm glibc-2.23-r3.apk
curl "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx
mv linux-amd64/helm /usr/bin/
helm version --client
curl -L -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
chmod +x /usr/bin/kubectl
kubectl version --client
}
function setup_docker() {
if ! docker info &>/dev/null; then
if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
export DOCKER_HOST='tcp://localhost:2375'
fi
fi
}
function setup_test_db() {
if [ -z ${KUBERNETES_PORT+x} ]; then
DB_HOST=postgres
else
DB_HOST=localhost
fi
export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}"
}
function download_chart() {
if [[ ! -d chart ]]; then
auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app}
auto_chart_name=$(basename $auto_chart)
auto_chart_name=${auto_chart_name%.tgz}
else
auto_chart="chart"
auto_chart_name="chart"
fi
helm init --client-only
helm repo add gitlab https://charts.gitlab.io
if [[ ! -d "$auto_chart" ]]; then
helm fetch ${auto_chart} --untar
fi
if [ "$auto_chart_name" != "chart" ]; then
mv ${auto_chart_name} chart
fi
helm dependency update chart/
helm dependency build chart/
}
function ensure_namespace() {
kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"
}
function check_kube_domain() {
if [ -z ${AUTO_DEVOPS_DOMAIN+x} ]; then
echo "In order to deploy or use Review Apps, AUTO_DEVOPS_DOMAIN variable must be set"
echo "You can do it in Auto DevOps project settings or defining a secret variable at group or project level"
echo "You can also manually add it in .gitlab-ci.yml"
false
else
true
fi
}
function build() {
if [[ -n "$CI_REGISTRY_USER" ]]; then
echo "Logging to GitLab Container Registry with CI credentials..."
docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
echo ""
fi
if [[ -f Dockerfile ]]; then
echo "Building Dockerfile-based application..."
docker build -t "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" .
else
echo "Building Heroku-based application using gliderlabs/herokuish docker image..."
docker run -i --name="$CI_CONTAINER_NAME" -v "$(pwd):/tmp/app:ro" gliderlabs/herokuish /bin/herokuish buildpack build
docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"
docker rm "$CI_CONTAINER_NAME" >/dev/null
echo ""
echo "Configuring $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG docker image..."
docker create --expose 5000 --env PORT=5000 --name="$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" /bin/herokuish procfile start web
docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"
docker rm "$CI_CONTAINER_NAME" >/dev/null
echo ""
fi
echo "Pushing to GitLab Container Registry..."
docker push "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"
echo ""
}
function install_tiller() {
echo "Checking Tiller..."
helm init --upgrade
kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy"
if ! helm version --debug; then
echo "Failed to init Tiller."
return 1
fi
echo ""
}
function create_secret() {
echo "Create secret..."
if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then
return
fi
kubectl create secret -n "$KUBE_NAMESPACE" \
docker-registry gitlab-registry \
--docker-server="$CI_REGISTRY" \
--docker-username="$CI_REGISTRY_USER" \
--docker-password="$CI_REGISTRY_PASSWORD" \
--docker-email="$GITLAB_USER_EMAIL" \
-o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f -
}
function dast() {
export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
mkdir /zap/wrk/
/zap/zap-baseline.py -J gl-dast-report.json -t "$CI_ENVIRONMENT_URL" || true
cp /zap/wrk/gl-dast-report.json .
}
function performance() {
export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
mkdir gitlab-exporter
wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js
mkdir sitespeed-results
if [ -f .gitlab-urls.txt ]
then
sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt
docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt
else
docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL"
fi
mv sitespeed-results/data/performance.json performance.json
}
- master
function persist_environment_url() {
echo $CI_ENVIRONMENT_URL > environment_url.txt
}
build tag:
extends: .build
before_script:
- export IMAGE_TAG=${CI_COMMIT_TAG/v/}
variables:
DS_ANALYZER_IMAGE_TAG: $MAJOR
only:
- tags
function delete() {
track="${1-stable}"
name="$CI_ENVIRONMENT_SLUG"
build major:
extends: .build
variables:
IMAGE_TAG: $MAJOR
DS_ANALYZER_IMAGE_TAG: $MAJOR
only:
- tags
when: manual
allow_failure: false
if [[ "$track" != "stable" ]]; then
name="$name-$track"
fi
tag latest:
image: docker:stable
stage: deploy
before_script:
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- export SOURCE_IMAGE=$CI_REGISTRY_IMAGE:$MAJOR
- export TARGET_IMAGE=$CI_REGISTRY_IMAGE:latest
script:
- docker pull $SOURCE_IMAGE
- docker tag $SOURCE_IMAGE $TARGET_IMAGE
- docker push $TARGET_IMAGE
only:
- tags
if [[ -n "$(helm ls -q "^$name$")" ]]; then
helm delete "$name"
fi
}
.deploy:
image: docker:stable
stage: deploy
before_script:
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- export IMAGE=$CI_REGISTRY_IMAGE:$CI_JOB_NAME
script:
- docker build -t $IMAGE -f $DOCKERFILE --build-arg DS_ANALYZER_IMAGE_TAG=$MAJOR .
- docker push $IMAGE
only:
- tags
before_script:
- *auto_devops
11-6-stable:
extends: .deploy
variables:
DOCKERFILE: Dockerfile.v1
--require spec_helper
AllCops:
DisabledByDefault: false
Exclude:
- 'tmp/**/*'
- 'vendor/**/*'
- 'coverage/**/*'
Metrics/LineLength:
Max: 100
Exclude:
- 'spec/**/*'
Metrics/ModuleLength:
Max: 100
Exclude:
- 'spec/**/*'
Metrics/MethodLength:
Max: 50
Metrics/BlockLength:
Max: 25
Exclude:
- 'spec/**/*'
Lint/AmbiguousBlockAssociation:
Exclude:
- 'spec/**/*'
Metrics/AbcSize:
Enabled: false
Metrics/CyclomaticComplexity:
Enabled: false
Metrics/PerceivedComplexity:
Enabled: false
# GitLab Dependency Scanning
# GitLab Dependency Scanning changelog
GitLab Dependency Scanning follows versioning of GitLab (`MAJOR.MINOR` only) and generates a `MAJOR-MINOR-stable` [Docker image](https://gitlab.com/gitlab-org/security-products/dependency-scanning/container_registry).
## v1.4.0
- Introduce customizable analyzers based on Docker images
These "stable" Docker images may be updated after release date, changes are added to the corresponding section bellow.
## 11-6-stable
## 11-5-stable
## v1.3.0
- Vulnerabilities reported by Gemnasium now include a solution.
## 11-4-stable
- Fix dependency scanning ignoring the variable DEP_SCAN_DISABLE_REMOTE_CHECKS.
## 11-3-stable
## 11-2-stable
## v1.2.0
- Fix dependency scanning ignoring the variable `DEP_SCAN_DISABLE_REMOTE_CHECKS`.
## 11-1-stable
## 11-0-stable
## v1.1.0
- Fix missing cve value for some vulnerabilities (frontend workaround)
## 10-8-stable
## 10-7-stable
## v1.0.0
- Initial release
## 10-6-stable
- **Backport:** Initial release
## 10-5-stable
- **Backport:** Initial release
......@@ -31,3 +31,38 @@ open the issue in order to keep track of it and then open the relevant merge
request that potentially fixes it.