Commit 3be46bdf authored by Tiger Watson's avatar Tiger Watson

Split Security Auto DevOps jobs into templates

Container Scanning, Dependency Scanning, License Management
and SAST parts of Auto DevOps now use the preexisting templates.

Auto DevOps and the DAST job template will now use a
shared job template instead of maintaining two copies
of the job. This also allows Auto DevOps to use custom
authentication with DAST.
parent ec5e032a
...@@ -78,217 +78,8 @@ include: ...@@ -78,217 +78,8 @@ include:
- template: Jobs/Code-Quality.gitlab-ci.yml - template: Jobs/Code-Quality.gitlab-ci.yml
- template: Jobs/Deploy.gitlab-ci.yml - template: Jobs/Deploy.gitlab-ci.yml
- template: Jobs/Browser-Performance-Testing.gitlab-ci.yml - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml
- template: Jobs/DAST.gitlab-ci.yml
license_management: - template: Security/Container-Scanning.gitlab-ci.yml
stage: test - template: Security/Dependency-Scanning.gitlab-ci.yml
image: - template: Security/License-Management.gitlab-ci.yml
name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" - template: Security/SAST.gitlab-ci.yml
entrypoint: [""]
allow_failure: true
script:
- license_management
artifacts:
paths: [gl-license-management-report.json]
only:
refs:
- branches
- tags
variables:
- $GITLAB_FEATURES =~ /\blicense_management\b/
except:
variables:
- $LICENSE_MANAGEMENT_DISABLED
sast:
stage: test
image: docker:stable
allow_failure: true
services:
- docker:stable-dind
script:
- setup_docker
- sast
artifacts:
reports:
sast: gl-sast-report.json
only:
refs:
- branches
- tags
variables:
- $GITLAB_FEATURES =~ /\bsast\b/
except:
variables:
- $SAST_DISABLED
dependency_scanning:
stage: test
image: docker:stable
allow_failure: true
services:
- docker:stable-dind
script:
- setup_docker
- dependency_scanning
artifacts:
reports:
dependency_scanning: gl-dependency-scanning-report.json
only:
refs:
- branches
- tags
variables:
- $GITLAB_FEATURES =~ /\bdependency_scanning\b/
except:
variables:
- $DEPENDENCY_SCANNING_DISABLED
container_scanning:
stage: test
image: docker:stable
allow_failure: true
services:
- docker:stable-dind
script:
- setup_docker
- container_scanning
artifacts:
paths: [gl-container-scanning-report.json]
only:
refs:
- branches
- tags
variables:
- $GITLAB_FEATURES =~ /\bcontainer_scanning\b/
except:
variables:
- $CONTAINER_SCANNING_DISABLED
dast:
stage: dast
allow_failure: true
image: registry.gitlab.com/gitlab-org/security-products/zaproxy
variables:
POSTGRES_DB: "false"
script:
- dast
artifacts:
paths: [gl-dast-report.json]
only:
refs:
- branches
- tags
kubernetes: active
variables:
- $GITLAB_FEATURES =~ /\bdast\b/
except:
refs:
- master
variables:
- $DAST_DISABLED
# ---------------------------------------------------------------------------
.auto_devops: &auto_devops |
# Auto DevOps variables and functions
[[ "$TRACE" ]] && set -x
auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB}
export DATABASE_URL=${DATABASE_URL-$auto_database_url}
if [[ -z "$CI_COMMIT_TAG" ]]; then
export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG
export CI_APPLICATION_TAG=$CI_COMMIT_SHA
else
export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE
export CI_APPLICATION_TAG=$CI_COMMIT_TAG
fi
export TILLER_NAMESPACE=$KUBE_NAMESPACE
# Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products
export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
function registry_login() {
if [[ -n "$CI_REGISTRY_USER" ]]; then
echo "Logging to GitLab Container Registry with CI credentials..."
docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
echo ""
fi
}
function container_scanning() {
registry_login
docker run -d --name db arminc/clair-db:latest
docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:v2.0.6
apk add -U wget ca-certificates
docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG}
wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
mv clair-scanner_linux_amd64 clair-scanner
chmod +x clair-scanner
touch clair-whitelist.yml
retries=0
echo "Waiting for clair daemon to start"
while( ! wget -T 10 -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done
./clair-scanner -c http://${DOCKER_SERVICE}:6060 --ip $(hostname -i) -r gl-container-scanning-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true
}
function license_management() {
/run.sh analyze .
}
function sast() {
case "$CI_SERVER_VERSION" in
*-ee)
# Deprecation notice for CONFIDENCE_LEVEL variable
if [ -z "$SAST_CONFIDENCE_LEVEL" -a "$CONFIDENCE_LEVEL" ]; then
SAST_CONFIDENCE_LEVEL="$CONFIDENCE_LEVEL"
echo "WARNING: CONFIDENCE_LEVEL is deprecated and MUST be replaced with SAST_CONFIDENCE_LEVEL"
fi
docker run --env SAST_CONFIDENCE_LEVEL="${SAST_CONFIDENCE_LEVEL:-3}" \
--volume "$PWD:/code" \
--volume /var/run/docker.sock:/var/run/docker.sock \
"registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code
;;
*)
echo "GitLab EE is required"
;;
esac
}
function dependency_scanning() {
case "$CI_SERVER_VERSION" in
*-ee)
docker run --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}" \
--volume "$PWD:/code" \
--volume /var/run/docker.sock:/var/run/docker.sock \
"registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code
;;
*)
echo "GitLab EE is required"
;;
esac
}
# With the Kubernetes executor, 'localhost' must be used instead
# https://docs.gitlab.com/runner/executors/kubernetes.html
function setup_docker() {
if ! docker info &>/dev/null; then
if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
export DOCKER_HOST='tcp://localhost:2375'
export DOCKER_SERVICE="localhost"
else
export DOCKER_SERVICE="docker"
fi
fi
}
function dast() {
export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
mkdir /zap/wrk/
/zap/zap-baseline.py -J gl-dast-report.json -t "$CI_ENVIRONMENT_URL" || true
cp /zap/wrk/gl-dast-report.json .
}
before_script:
- *auto_devops
dast:
stage: dast
image: docker:stable
variables:
DOCKER_DRIVER: overlay2
allow_failure: true
services:
- docker:stable-dind
script:
- export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
- export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
- |
if ! docker info &>/dev/null; then
if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
export DOCKER_HOST='tcp://localhost:2375'
fi
fi
- |
function dast_run() {
docker run \
--env DAST_TARGET_AVAILABILITY_TIMEOUT \
--volume "$PWD:/output" \
--volume /var/run/docker.sock:/var/run/docker.sock \
-w /output \
"registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \
/analyze -t $DAST_WEBSITE \
"$@"
}
- |
if [ -n "$DAST_AUTH_URL" ]
then
dast_run \
--auth-url $DAST_AUTH_URL \
--auth-username $DAST_USERNAME \
--auth-password $DAST_PASSWORD \
--auth-username-field $DAST_USERNAME_FIELD \
--auth-password-field $DAST_PASSWORD_FIELD
else
dast_run
fi
artifacts:
reports:
dast: gl-dast-report.json
only:
refs:
- branches
- tags
variables:
- $GITLAB_FEATURES =~ /\bdast\b/
except:
refs:
- master
variables:
- $DAST_DISABLED
...@@ -28,6 +28,12 @@ container_scanning: ...@@ -28,6 +28,12 @@ container_scanning:
- docker:stable-dind - docker:stable-dind
script: script:
- if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then { export DOCKER_SERVICE="localhost" ; export DOCKER_HOST="tcp://${DOCKER_SERVICE}:2375" ; } fi - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then { export DOCKER_SERVICE="localhost" ; export DOCKER_HOST="tcp://${DOCKER_SERVICE}:2375" ; } fi
- |
if [[ -n "$CI_REGISTRY_USER" ]]; then
echo "Logging to GitLab Container Registry with CI credentials..."
docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
echo ""
fi
- docker run -d --name db arminc/clair-db:latest - docker run -d --name db arminc/clair-db:latest
- docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:${CLAIR_LOCAL_SCAN_VERSION} - docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:${CLAIR_LOCAL_SCAN_VERSION}
- apk add -U wget ca-certificates - apk add -U wget ca-certificates
...@@ -36,7 +42,6 @@ container_scanning: ...@@ -36,7 +42,6 @@ container_scanning:
- mv clair-scanner_linux_amd64 clair-scanner - mv clair-scanner_linux_amd64 clair-scanner
- chmod +x clair-scanner - chmod +x clair-scanner
- touch clair-whitelist.yml - touch clair-whitelist.yml
- while( ! wget -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; done
- retries=0 - retries=0
- echo "Waiting for clair daemon to start" - echo "Waiting for clair daemon to start"
- while( ! wget -T 10 -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done - while( ! wget -T 10 -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done
......
...@@ -4,6 +4,9 @@ ...@@ -4,6 +4,9 @@
# List of the variables: https://gitlab.com/gitlab-org/security-products/dast#settings # List of the variables: https://gitlab.com/gitlab-org/security-products/dast#settings
# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
include:
- template: Jobs/DAST.gitlab-ci.yml
variables: variables:
DAST_WEBSITE: http://example.com # Please edit to be your website to scan for vulnerabilities DAST_WEBSITE: http://example.com # Please edit to be your website to scan for vulnerabilities
...@@ -14,46 +17,10 @@ stages: ...@@ -14,46 +17,10 @@ stages:
- dast - dast
dast: dast:
stage: dast
image: docker:stable
variables:
DOCKER_DRIVER: overlay2
allow_failure: true
services:
- docker:stable-dind
script:
- export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
- |
function dast_run() {
docker run \
--env DAST_TARGET_AVAILABILITY_TIMEOUT \
--volume "$PWD:/output" \
--volume /var/run/docker.sock:/var/run/docker.sock \
-w /output \
"registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \
/analyze -t $DAST_WEBSITE \
"$@"
}
- |
if [ -n "$DAST_AUTH_URL" ]
then
dast_run \
--auth-url $DAST_AUTH_URL \
--auth-username $DAST_USERNAME \
--auth-password $DAST_PASSWORD \
--auth-username-field $DAST_USERNAME_FIELD \
--auth-password-field $DAST_PASSWORD_FIELD
else
dast_run
fi
artifacts:
reports:
dast: gl-dast-report.json
only: only:
refs: refs:
- branches - branches
variables:
- $GITLAB_FEATURES =~ /\bdast\b/
except: except:
refs: [] # Override default from template
variables: variables:
- $DAST_DISABLED - $DAST_DISABLED
...@@ -14,6 +14,12 @@ dependency_scanning: ...@@ -14,6 +14,12 @@ dependency_scanning:
- docker:stable-dind - docker:stable-dind
script: script:
- export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
- |
if ! docker info &>/dev/null; then
if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
export DOCKER_HOST='tcp://localhost:2375'
fi
fi
- | - |
docker run \ docker run \
--env DS_ANALYZER_IMAGES \ --env DS_ANALYZER_IMAGES \
......
...@@ -14,6 +14,12 @@ sast: ...@@ -14,6 +14,12 @@ sast:
- docker:stable-dind - docker:stable-dind
script: script:
- export SAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - export SAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
- |
if ! docker info &>/dev/null; then
if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
export DOCKER_HOST='tcp://localhost:2375'
fi
fi
- | - |
docker run \ docker run \
--env SAST_ANALYZER_IMAGES \ --env SAST_ANALYZER_IMAGES \
......
...@@ -4,6 +4,7 @@ require 'spec_helper' ...@@ -4,6 +4,7 @@ require 'spec_helper'
describe "CI YML Templates" do describe "CI YML Templates" do
ABSTRACT_TEMPLATES = %w[Serverless].freeze ABSTRACT_TEMPLATES = %w[Serverless].freeze
PROJECT_DEPENDENT_TEMPLATES = %w[Auto-DevOps].freeze
def self.concrete_templates def self.concrete_templates
Gitlab::Template::GitlabCiYmlTemplate.all.reject do |template| Gitlab::Template::GitlabCiYmlTemplate.all.reject do |template|
...@@ -20,7 +21,10 @@ describe "CI YML Templates" do ...@@ -20,7 +21,10 @@ describe "CI YML Templates" do
describe 'concrete templates with CI/CD jobs' do describe 'concrete templates with CI/CD jobs' do
concrete_templates.each do |template| concrete_templates.each do |template|
it "#{template.name} template should be valid" do it "#{template.name} template should be valid" do
expect { Gitlab::Ci::YamlProcessor.new(template.content) } # Trigger processing of included files
project = create(:project, :test_repo) if PROJECT_DEPENDENT_TEMPLATES.include?(template.name)
expect { Gitlab::Ci::YamlProcessor.new(template.content, project: project) }
.not_to raise_error .not_to raise_error
end end
end end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment