Skip to content

GitLab Next

D
Dependency Scanning
Commit 958e2a12 authored by Fabien Catteau's avatar Fabien Catteau Committed by Philippe Lafoucrière
Browse files
Options

Rewrite using Go, common lib and shared orchestrator

parent 59ec62d7
Hide whitespace changes
Inline Side-by-side
Showing with 2086 additions and 2963 deletions
.gitignore
View file @ 958e2a12
*.gem
*.rbc
# binary that can be downloaded during dev
/bin/gemnasium
/bin/find-sec-bugs-launcher
/.config
/coverage/
/InstalledFiles
/pkg/
/spec/reports/
/spec/examples.txt
/test/tmp/
/test/version_tmp/
/tmp/
# Used by dotenv library to load environment variables.
# .env
## Specific to RubyMotion:
.dat*
.repl_history
build/
*.bridgesupport
build-iPhoneOS/
build-iPhoneSimulator/
## Specific to RubyMotion (use of CocoaPods):
#
# We recommend against adding the Pods directory to your .gitignore. However
# you should judge for yourself, the pros and cons are mentioned at:
# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
#
# vendor/Pods/
## Documentation cache and generated files:
/.yardoc/
/_yardoc/
/doc/
/rdoc/
## Environment normalization:
/.bundle/
/vendor/bundle
/lib/bundler/man/
# for a library or gem, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# Gemfile.lock
# .ruby-version
# .ruby-gemset
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
.rvmrc
dependency-scanning
test/fixtures/gl-dependency-scanning-report.json
.gitlab-ci.yml
View file @ 958e2a12
include:
- https://gitlab.com/gitlab-org/security-products/ci-templates/raw/master/includes/sast.yml
- https://gitlab.com/gitlab-org/security-products/ci-templates/raw/master/includes/dependency_scanning.yml
- https://gitlab.com/gitlab-org/security-products/ci-templates/raw/master/includes/container_scanning.yml
- https://gitlab.com/gitlab-org/security-products/ci-templates/raw/master/includes/codequality.yml
image: alpine:latest
# When using dind, it's wise to use the overlayfs driver for
# improved performance.
variables:
POSTGRES_ENABLED: "false"
DOCKER_DRIVER: overlay2
MAJOR: 1
services:
- docker:stable-dind
stages:
- go
- test
- release
- build
- deploy
test:
stage: test
image: docker:stable
services:
- docker:dind
.go:
image: golang:1.11
stage: go
variables:
GO111MODULE: "on"
before_script:
- mkdir -p /go/src/gitlab.com/${CI_PROJECT_PATH}
- cp -r . /go/src/gitlab.com/${CI_PROJECT_PATH}
- cd /go/src/gitlab.com/${CI_PROJECT_PATH}
go build:
extends: .go
variables:
DOCKER_DRIVER: overlay2
CGO_ENABLED: 0
script:
- setup_docker
- docker run
--volume "$PWD:/app"
--volume /var/run/docker.sock:/var/run/docker.sock
ruby:2.3 sh -c "cd /app && bundle install && rspec spec"
only:
- branches
- go get ./...
- go build -o ${CI_PROJECT_DIR}/dependency-scanning
artifacts:
paths:
- dependency-scanning
rubocop:
stage: test
image: ruby:2.3
go test:
extends: .go
script:
- bundle install
- bundle exec rubocop
only:
- branches
- go get ./...
- go test -race -cover -v ./...
container_scanning:
go lint:
extends: .go
script:
- go get -u golang.org/x/lint/golint
- golint -set_exit_status
test:
image: docker:stable
allow_failure: true
before_script: []
cache: {}
dependencies: []
tags: []
services:
- docker:stable-dind
stage: test
variables:
DOCKER_DRIVER: overlay2
## Define two new variables based on GitLab's CI/CD predefined variables
## https://docs.gitlab.com/ee/ci/variables/#predefined-variables-environment-variables
CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG
CI_APPLICATION_TAG: $CI_COMMIT_SHA
DS_ANALYZER_IMAGE_TAG: $MAJOR
script:
# Build image locally as it doesn't exist on registry
- docker build -t "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" .
- docker run -d --name db arminc/clair-db:latest
- docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.1
- apk add -U wget ca-certificates
# - docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG}
- wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
- mv clair-scanner_linux_amd64 clair-scanner
- chmod +x clair-scanner
- touch clair-whitelist.yml
- retries=0
- echo "Waiting for clair daemon to start"
- while( ! wget -T 10 -q -O /dev/null http://docker:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done
- ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-sast-container-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true
artifacts:
paths:
- gl-sast-container-report.json
- ./test.sh
dependencies:
- go build
release-latest-and-next-version:
stage: release
image: docker:git
services:
- docker:dind
variables:
DOCKER_DRIVER: overlay2
.build:
image: docker:stable
stage: build
script:
- echo "Building Docker image..."
- docker build -t "$CI_REGISTRY_IMAGE:latest" .
- docker tag "$CI_REGISTRY_IMAGE:latest" "$CI_REGISTRY_IMAGE:$(cat VERSION)"
- echo "Logging to GitLab Container Registry with CI credentials..."
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
- echo "Pushing to GitLab Container Registry..."
- docker push "$CI_REGISTRY_IMAGE:latest"
- docker push "$CI_REGISTRY_IMAGE:$(cat VERSION)"
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- export IMAGE=$CI_REGISTRY_IMAGE:$IMAGE_TAG
- docker build -t $IMAGE --build-arg DS_ANALYZER_IMAGE_TAG=$DS_ANALYZER_IMAGE_TAG .
- docker push $IMAGE
dependencies:
- go build
build branch:
extends: .build
variables:
IMAGE_TAG: $CI_COMMIT_REF_NAME
DS_ANALYZER_IMAGE_TAG: $MAJOR
only:
- branches
except:
- master
release-stable:
stage: release
image: docker:git
services:
- docker:dind
build edge:
extends: .build
variables:
DOCKER_DRIVER: overlay2
script:
- echo "Building Docker image..."
- docker build -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" .
- echo "Logging to GitLab Container Registry with CI credentials..."
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
- echo "Pushing to GitLab Container Registry..."
- docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
IMAGE_TAG: edge
DS_ANALYZER_IMAGE_TAG: edge
only:
- /^\d+-\d+-stable$/
# ---------------------------------------------------------------------------
.auto_devops: &auto_devops |
# Auto DevOps variables and functions
[[ "$TRACE" ]] && set -x
auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB}
export DATABASE_URL=${DATABASE_URL-$auto_database_url}
export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG
export CI_APPLICATION_TAG=$CI_COMMIT_SHA
export CI_CONTAINER_NAME=ci_job_build_${CI_JOB_ID}
export TILLER_NAMESPACE=$KUBE_NAMESPACE
function deploy() {
track="${1-stable}"
name="$CI_ENVIRONMENT_SLUG"
if [[ "$track" != "stable" ]]; then
name="$name-$track"
fi
replicas="1"
service_enabled="false"
postgres_enabled="$POSTGRES_ENABLED"
# canary uses stable db
[[ "$track" == "canary" ]] && postgres_enabled="false"
env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' )
env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' )
if [[ "$track" == "stable" ]]; then
# for stable track get number of replicas from `PRODUCTION_REPLICAS`
eval new_replicas=\$${env_slug}_REPLICAS
service_enabled="true"
else
# for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS`
eval new_replicas=\$${env_track}_${env_slug}_REPLICAS
fi
if [[ -n "$new_replicas" ]]; then
replicas="$new_replicas"
fi
if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then
secret_name='gitlab-registry'
else
secret_name=''
fi
helm upgrade --install \
--wait \
--set service.enabled="$service_enabled" \
--set releaseOverride="$CI_ENVIRONMENT_SLUG" \
--set image.repository="$CI_APPLICATION_REPOSITORY" \
--set image.tag="$CI_APPLICATION_TAG" \
--set image.pullPolicy=IfNotPresent \
--set image.secrets[0].name="$secret_name" \
--set application.track="$track" \
--set application.database_url="$DATABASE_URL" \
--set service.url="$CI_ENVIRONMENT_URL" \
--set replicaCount="$replicas" \
--set postgresql.enabled="$postgres_enabled" \
--set postgresql.nameOverride="postgres" \
--set postgresql.postgresUser="$POSTGRES_USER" \
--set postgresql.postgresPassword="$POSTGRES_PASSWORD" \
--set postgresql.postgresDatabase="$POSTGRES_DB" \
--namespace="$KUBE_NAMESPACE" \
--version="$CI_PIPELINE_ID-$CI_JOB_ID" \
"$name" \
chart/
}
function install_dependencies() {
apk add -U openssl curl tar gzip bash ca-certificates git
wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub
wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.23-r3/glibc-2.23-r3.apk
apk add glibc-2.23-r3.apk
rm glibc-2.23-r3.apk
curl "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx
mv linux-amd64/helm /usr/bin/
helm version --client
curl -L -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
chmod +x /usr/bin/kubectl
kubectl version --client
}
function setup_docker() {
if ! docker info &>/dev/null; then
if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
export DOCKER_HOST='tcp://localhost:2375'
fi
fi
}
function setup_test_db() {
if [ -z ${KUBERNETES_PORT+x} ]; then
DB_HOST=postgres
else
DB_HOST=localhost
fi
export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}"
}
function download_chart() {
if [[ ! -d chart ]]; then
auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app}
auto_chart_name=$(basename $auto_chart)
auto_chart_name=${auto_chart_name%.tgz}
else
auto_chart="chart"
auto_chart_name="chart"
fi
helm init --client-only
helm repo add gitlab https://charts.gitlab.io
if [[ ! -d "$auto_chart" ]]; then
helm fetch ${auto_chart} --untar
fi
if [ "$auto_chart_name" != "chart" ]; then
mv ${auto_chart_name} chart
fi
helm dependency update chart/
helm dependency build chart/
}
function ensure_namespace() {
kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"
}
function check_kube_domain() {
if [ -z ${AUTO_DEVOPS_DOMAIN+x} ]; then
echo "In order to deploy or use Review Apps, AUTO_DEVOPS_DOMAIN variable must be set"
echo "You can do it in Auto DevOps project settings or defining a secret variable at group or project level"
echo "You can also manually add it in .gitlab-ci.yml"
false
else
true
fi
}
function build() {
if [[ -n "$CI_REGISTRY_USER" ]]; then
echo "Logging to GitLab Container Registry with CI credentials..."
docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
echo ""
fi
if [[ -f Dockerfile ]]; then
echo "Building Dockerfile-based application..."
docker build -t "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" .
else
echo "Building Heroku-based application using gliderlabs/herokuish docker image..."
docker run -i --name="$CI_CONTAINER_NAME" -v "$(pwd):/tmp/app:ro" gliderlabs/herokuish /bin/herokuish buildpack build
docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"
docker rm "$CI_CONTAINER_NAME" >/dev/null
echo ""
echo "Configuring $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG docker image..."
docker create --expose 5000 --env PORT=5000 --name="$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" /bin/herokuish procfile start web
docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"
docker rm "$CI_CONTAINER_NAME" >/dev/null
echo ""
fi
echo "Pushing to GitLab Container Registry..."
docker push "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"
echo ""
}
function install_tiller() {
echo "Checking Tiller..."
helm init --upgrade
kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy"
if ! helm version --debug; then
echo "Failed to init Tiller."
return 1
fi
echo ""
}
function create_secret() {
echo "Create secret..."
if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then
return
fi
kubectl create secret -n "$KUBE_NAMESPACE" \
docker-registry gitlab-registry \
--docker-server="$CI_REGISTRY" \
--docker-username="$CI_REGISTRY_USER" \
--docker-password="$CI_REGISTRY_PASSWORD" \
--docker-email="$GITLAB_USER_EMAIL" \
-o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f -
}
function dast() {
export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
mkdir /zap/wrk/
/zap/zap-baseline.py -J gl-dast-report.json -t "$CI_ENVIRONMENT_URL" || true
cp /zap/wrk/gl-dast-report.json .
}
function performance() {
export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
mkdir gitlab-exporter
wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js
mkdir sitespeed-results
if [ -f .gitlab-urls.txt ]
then
sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt
docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt
else
docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL"
fi
mv sitespeed-results/data/performance.json performance.json
}
- master
function persist_environment_url() {
echo $CI_ENVIRONMENT_URL > environment_url.txt
}
build tag:
extends: .build
before_script:
- export IMAGE_TAG=${CI_COMMIT_TAG/v/}
variables:
DS_ANALYZER_IMAGE_TAG: $MAJOR
only:
- tags
function delete() {
track="${1-stable}"
name="$CI_ENVIRONMENT_SLUG"
build major:
extends: .build
variables:
IMAGE_TAG: $MAJOR
DS_ANALYZER_IMAGE_TAG: $MAJOR
only:
- tags
when: manual
allow_failure: false
if [[ "$track" != "stable" ]]; then
name="$name-$track"
fi
tag latest:
image: docker:stable
stage: deploy
before_script:
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- export SOURCE_IMAGE=$CI_REGISTRY_IMAGE:$MAJOR
- export TARGET_IMAGE=$CI_REGISTRY_IMAGE:latest
script:
- docker pull $SOURCE_IMAGE
- docker tag $SOURCE_IMAGE $TARGET_IMAGE
- docker push $TARGET_IMAGE
only:
- tags
if [[ -n "$(helm ls -q "^$name$")" ]]; then
helm delete "$name"
fi
}
.deploy:
image: docker:stable
stage: deploy
before_script:
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- export IMAGE=$CI_REGISTRY_IMAGE:$CI_JOB_NAME
script:
- docker build -t $IMAGE -f $DOCKERFILE --build-arg DS_ANALYZER_IMAGE_TAG=$MAJOR .
- docker push $IMAGE
only:
- tags
before_script:
- *auto_devops
11-6-stable:
extends: .deploy
variables:
DOCKERFILE: Dockerfile.v1
.rspec deleted 100644 → 0
View file @ 59ec62d7
--require spec_helper
.rubocop.yml deleted 100644 → 0
View file @ 59ec62d7
AllCops:
DisabledByDefault: false
Exclude:
- 'tmp/**/*'
- 'vendor/**/*'
- 'coverage/**/*'
Metrics/LineLength:
Max: 100
Exclude:
- 'spec/**/*'
Metrics/ModuleLength:
Max: 100
Exclude:
- 'spec/**/*'
Metrics/MethodLength:
Max: 50
Metrics/BlockLength:
Max: 25
Exclude:
- 'spec/**/*'
Lint/AmbiguousBlockAssociation:
Exclude:
- 'spec/**/*'
Metrics/AbcSize:
Enabled: false
Metrics/CyclomaticComplexity:
Enabled: false
Metrics/PerceivedComplexity:
Enabled: false
CHANGELOG.md
View file @ 958e2a12
# GitLab Dependency Scanning
# GitLab Dependency Scanning changelog
GitLab Dependency Scanning follows versioning of GitLab (`MAJOR.MINOR` only) and generates a `MAJOR-MINOR-stable` [Docker image](https://gitlab.com/gitlab-org/security-products/dependency-scanning/container_registry).
## v1.4.0
- Introduce customizable analyzers based on Docker images
These "stable" Docker images may be updated after release date, changes are added to the corresponding section bellow.
## 11-6-stable
## 11-5-stable
## v1.3.0
- Vulnerabilities reported by Gemnasium now include a solution.
## 11-4-stable
- Fix dependency scanning ignoring the variable DEP_SCAN_DISABLE_REMOTE_CHECKS.
## 11-3-stable
## 11-2-stable
## v1.2.0
- Fix dependency scanning ignoring the variable `DEP_SCAN_DISABLE_REMOTE_CHECKS`.
## 11-1-stable
## 11-0-stable
## v1.1.0
- Fix missing cve value for some vulnerabilities (frontend workaround)
## 10-8-stable
## 10-7-stable
## v1.0.0
- Initial release
## 10-6-stable
- **Backport:** Initial release
## 10-5-stable
- **Backport:** Initial release
CONTRIBUTING.md
View file @ 958e2a12
......@@ -31,3 +31,38 @@ open the issue in order to keep track of it and then open the relevant merge
request that potentially fixes it.