Commit ef80bee8 authored by mo's avatar mo Committed by Can Eldem
Browse files

Add functional tests to the pipeline

* Move jobs to gitlab-ci.yml files that match the stage they belong to.
* Move linter job to build stage
* Add functional test jobs
* Reduce max size to 2 GB
* Pass the current pipeline image to the downstream jobs
* Replace `edge` tag with `latest`
* Use $CI_DEFAULT_BRANCH instead of master
* Move `.env*` files to config dir and update RELEASE instructions
* Combine `tag` and `release` stages into `deploy` stage as recommended in GitLab docs
* Make the `build-docker-image` job interruptible
* Fix issues found in code quality report
parent 7e044195
......@@ -5,3 +5,4 @@ Dockerfile
spec
tags
tmp
config/.env*
......@@ -3,21 +3,21 @@
variables:
DOCKER_DRIVER: overlay2
MAJOR: 3
TMP_IMAGE: $CI_REGISTRY_IMAGE/tmp:$CI_COMMIT_SHA
TMP_IMAGE: $CI_REGISTRY_IMAGE/license-finder:$CI_COMMIT_SHA
include:
- template: Container-Scanning.gitlab-ci.yml
- template: Dependency-Scanning.gitlab-ci.yml
- template: Jobs/Code-Quality.gitlab-ci.yml
- template: License-Scanning.gitlab-ci.yml
- local: .gitlab/build.yml
- local: .gitlab/test.yml
- local: .gitlab/release.yml
- local: .gitlab/deploy.yml
stages:
- build
- test
- tag
- release
- deploy
default:
image:
......@@ -25,76 +25,9 @@ default:
entrypoint: [""]
tags: [gitlab-org]
build commit:
image: docker:stable
stage: build
tags: [gitlab-org-docker]
services:
- docker:stable-dind
script:
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker pull $CI_REGISTRY_IMAGE:latest || true
- docker build --cache-from $CI_REGISTRY_IMAGE:latest -t $TMP_IMAGE .
- docker image inspect $TMP_IMAGE --format='{{.Size}}'
- docker push $TMP_IMAGE
build mvn:
image: maven:3.3.9-jdk-8
stage: build
script:
- cd spec/fixtures/java/example/ && mvn deploy -s settings.xml
allow_failure: true
container_scanning:
variables:
CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/tmp # only predefined variables are parameter-expanded (no $TMP_IMAGE)
.docker_tag:
image: docker:stable
stage: tag
tags: [gitlab-org-docker]
services:
- docker:stable-dind
script:
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- export SOURCE_IMAGE=$TMP_IMAGE
- export TARGET_IMAGE=$CI_REGISTRY_IMAGE:${IMAGE_TAG:-$CI_JOB_NAME}
- |
if [ -z "$DOTENV" ]; then
docker pull $SOURCE_IMAGE
docker tag $SOURCE_IMAGE $TARGET_IMAGE
else
./bin/docker-dotenv
fi
- docker push $TARGET_IMAGE
branch:
extends: .docker_tag
variables:
IMAGE_TAG: $CI_COMMIT_REF_SLUG
only:
- branches
except:
- master
edge:
extends: .docker_tag
variables:
IMAGE_TAG: edge
only:
- master
version:
extends: .docker_tag
before_script:
- export IMAGE_TAG=${CI_COMMIT_TAG/v/}
- echo "Checking that $CI_COMMIT_TAG is last in the changelog"
- test "$(grep '^## v' CHANGELOG.md |head -n 1)" = "## $CI_COMMIT_TAG"
only:
- tags
allow_failure: false
CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/license-finder # only predefined variables are parameter-expanded (no $TMP_IMAGE)
code_quality:
before_script:
......
build-docker-image:
image: docker:stable
interruptible: true
stage: build
tags: [gitlab-org-docker]
services:
- docker:stable-dind
script:
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker pull $CI_REGISTRY_IMAGE:latest || true
- docker build --cache-from $CI_REGISTRY_IMAGE:latest -t $TMP_IMAGE .
- docker image inspect $TMP_IMAGE --format='{{.Size}}'
- docker push $TMP_IMAGE
build-mvn-pkg:
image: maven:3.3.9-jdk-8
stage: build
script:
- cd spec/fixtures/java/example/ && mvn deploy -s settings.xml
allow_failure: true
lint:
stage: build
image: ruby:alpine
script:
- apk add build-base shellcheck
- bin/setup
- bin/lint
.docker_tag:
image: docker:stable
stage: deploy
tags: [gitlab-org-docker]
services:
- docker:stable-dind
script:
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- export SOURCE_IMAGE=$TMP_IMAGE
- export TARGET_IMAGE=$CI_REGISTRY_IMAGE:${IMAGE_TAG:-$CI_JOB_NAME}
- |
if [ -z "$DOTENV" ]; then
docker pull $SOURCE_IMAGE
docker tag $SOURCE_IMAGE $TARGET_IMAGE
else
./bin/docker-dotenv
fi
- docker push $TARGET_IMAGE
latest:
extends: .docker_tag
variables:
IMAGE_TAG: latest
only:
- $CI_DEFAULT_BRANCH
version:
extends: .docker_tag
before_script:
- export IMAGE_TAG=${CI_COMMIT_TAG/v/}
- echo "Checking that $CI_COMMIT_TAG is last in the changelog"
- test "$(grep '^## v' CHANGELOG.md |head -n 1)" = "## $CI_COMMIT_TAG"
only:
- tags
allow_failure: false
.release:
extends: .docker_tag
stage: release
when: manual
only:
- tags
......@@ -15,125 +51,122 @@ major:
variables:
IMAGE_TAG: $MAJOR
latest:
extends: .release
13-0-stable:
extends: .release
variables:
DOTENV: ".env.13-0-stable"
DOTENV: "config/.env.13-0-stable"
12-10-stable:
extends: .release
variables:
DOTENV: ".env.12-10-stable"
DOTENV: "config/.env.12-10-stable"
12-9-stable:
extends: .release
variables:
DOTENV: ".env.12-9-stable"
DOTENV: "config/.env.12-9-stable"
12-8-stable:
extends: .release
variables:
DOTENV: ".env.12-8-stable"
DOTENV: "config/.env.12-8-stable"
12-7-stable:
extends: .release
variables:
DOTENV: ".env.12-7-stable"
DOTENV: "config/.env.12-7-stable"
12-6-stable:
extends: .release
variables:
DOTENV: ".env.12-6-stable"
DOTENV: "config/.env.12-6-stable"
12-5-stable:
extends: .release
variables:
DOTENV: ".env.12-5-stable"
DOTENV: "config/.env.12-5-stable"
12-4-stable:
extends: .release
variables:
DOTENV: ".env.12-4-stable"
DOTENV: "config/.env.12-4-stable"
12-3-stable:
extends: .release
variables:
DOTENV: ".env.12-3-stable"
DOTENV: "config/.env.12-3-stable"
12-2-stable:
extends: .release
variables:
DOTENV: ".env.12-2-stable"
DOTENV: "config/.env.12-2-stable"
12-1-stable:
extends: .release
variables:
DOTENV: ".env.legacy"
DOTENV: "config/.env.legacy"
12-0-stable:
extends: .release
variables:
DOTENV: ".env.legacy"
DOTENV: "config/.env.legacy"
11-11-stable:
extends: .release
variables:
DOTENV: ".env.legacy"
DOTENV: "config/.env.legacy"
11-10-stable:
extends: .release
variables:
DOTENV: ".env.legacy"
DOTENV: "config/.env.legacy"
11-9-stable:
extends: .release
variables:
DOTENV: ".env.legacy"
DOTENV: "config/.env.legacy"
11-8-stable:
extends: .release
variables:
DOTENV: ".env.legacy"
DOTENV: "config/.env.legacy"
11-7-stable:
extends: .release
variables:
DOTENV: ".env.legacy"
DOTENV: "config/.env.legacy"
11-6-stable:
extends: .release
variables:
DOTENV: ".env.legacy"
DOTENV: "config/.env.legacy"
11-5-stable:
extends: .release
variables:
DOTENV: ".env.legacy"
DOTENV: "config/.env.legacy"
11-4-stable:
extends: .release
variables:
DOTENV: ".env.legacy"
DOTENV: "config/.env.legacy"
11-3-stable:
extends: .release
variables:
DOTENV: ".env.legacy"
DOTENV: "config/.env.legacy"
11-2-stable:
extends: .release
variables:
DOTENV: ".env.legacy"
DOTENV: "config/.env.legacy"
11-1-stable:
extends: .release
variables:
DOTENV: ".env.legacy"
DOTENV: "config/.env.legacy"
11-0-stable:
extends: .release
variables:
DOTENV: ".env.legacy"
DOTENV: "config/.env.legacy"
size:
image: docker:stable
stage: test
allow_failure: true # temporary until we can shrink the image size.
services:
- docker:stable-dind
tags: [gitlab-org-docker]
timeout: 3 minutes
variables:
GIT_STRATEGY: none
MAX_SIZE: 2684354560
MAX_SIZE: 2147483648
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker pull $TMP_IMAGE
......@@ -18,14 +17,6 @@ size:
- echo $CURRENT_SIZE
- test "$MAX_SIZE" -gt "$CURRENT_SIZE"
lint:
stage: test
image: ruby:alpine
script:
- apk add build-base shellcheck
- bin/setup
- bin/lint
.rspec:
stage: test
script:
......@@ -86,3 +77,60 @@ integration-ruby:
extends: .rspec
variables:
RSPEC_DIR: spec/integration/ruby
.functional:
stage: test
trigger: gitlab-org/security-products/tests/common
variables:
CONTAINER_SCANNING_DISABLED: 'true'
DAST_DISABLED: 'true'
DEPENDENCY_SCANNING_DISABLED: 'true'
LICENSE_MANAGEMENT_VERSION: $CI_COMMIT_SHA
SAST_DISABLED: 'true'
SECURE_ANALYZERS_PREFIX: $CI_REGISTRY_IMAGE
only:
- $CI_DEFAULT_BRANCH
functional-csharp-nuget-dotnetcore:
extends: .functional
trigger: gitlab-org/security-products/tests/csharp-nuget-dotnetcore
functional-go-modules:
extends: .functional
trigger: gitlab-org/security-products/tests/go-modules
functional-java-maven-multimodules:
extends: .functional
trigger: gitlab-org/security-products/tests/java-maven-multimodules
functional-java-maven:
extends: .functional
trigger: gitlab-org/security-products/tests/java-maven
functional-js-bower:
extends: .functional
trigger: gitlab-org/security-products/tests/js-bower
functional-js-npm:
extends: .functional
trigger: gitlab-org/security-products/tests/js-npm
functional-js-yarn:
extends: .functional
trigger: gitlab-org/security-products/tests/js-yarn
functional-ruby-bundler_js-yarn:
extends: .functional
trigger: gitlab-org/security-products/tests/ruby-bundler_js-yarn
functional-php-composer:
extends: .functional
trigger: gitlab-org/security-products/tests/php-composer
functional-python-pip:
extends: .functional
trigger: gitlab-org/security-products/tests/python-pip
functional-ruby-bundler:
extends: .functional
trigger: gitlab-org/security-products/tests/ruby-bundler
......@@ -114,22 +114,6 @@ or its shorthand form (`--env MY_SETTING_VAR`) if the configuration comes from a
## Versioning and release process
1. Create a new entry in the `.gitlab/release.yml` file for the new version to release.
```yaml
12-x-stable:
extends: .release
variables:
DOTENV: ".env.12-x-stable"
```
2. Create a new `.env.*` that corresponds to the version to release and specify the default configuration.
```text
LM_PYTHON_VERSION '3'
LM_REPORT_VERSION '2.0'
```
Please check the [Release Process documentation](https://gitlab.com/gitlab-org/security-products/release/blob/master/docs/release_process.md).
## Upgrading to the latest version of LicenseFinder
......
......@@ -12,24 +12,28 @@ module LicenseFinder
end
def current_packages
stdout, _stderr, status = shell.execute([
:go, :list, '-m', '-f', "'{{.Path}},{{.Version}},{{.Dir}}'", :all
])
stdout, _stderr, status = shell.execute(go_list_command)
return [] unless status.success?
stdout.each_line.map do |line|
name, version, dir = line.chomp.split(',')
next if dir.nil?
next if Pathname(dir).cleanpath == absolute_project_path
Package.new(name, version, install_path: dir)
end.compact
stdout.each_line.map { |line| map_from(line) }.compact
end
private
def go_list_command
[:go, :list, '-m', '-f', "'{{.Path}},{{.Version}},{{.Dir}}'", :all]
end
def absolute_project_path
@absolute_project_path ||= Pathname(project_path).cleanpath
end
def map_from(line)
name, version, dir = line.chomp.split(',')
return if dir.nil?
return if Pathname(dir).cleanpath == absolute_project_path
Package.new(name, version, install_path: dir)
end
end
end
......@@ -9,7 +9,7 @@ module License
def initialize(logger: License::Management.logger, certificate: ENV['ADDITIONAL_CA_CERT_BUNDLE'])
@logger = logger
@custom_certificate_path = Pathname.new('/usr/local/share/ca-certificates/custom.crt')
trust!(certificate)
trust!(certificate) if present?(certificate)
end
def execute(command, env: {})
......@@ -18,8 +18,7 @@ module License
stdout, stderr, status = Open3.capture3(env, expanded_command)
logger.debug(stdout) unless stdout.nil? || stdout.empty?
logger.error(stderr) unless stderr.nil? || stderr.empty?
record(stdout, stderr)
[stdout, stderr, status]
end
......@@ -38,8 +37,6 @@ module License
end
def trust!(certificate)
return unless present?(certificate)
custom_certificate_path.write(certificate)
Dir.chdir custom_certificate_path.dirname do
execute([:awk, SPLIT_SCRIPT, '<', custom_certificate_path])
......@@ -48,29 +45,41 @@ module License
Dir.glob('custom.*.crt').each do |path|
full_path = File.expand_path(path)
execute([:openssl, :x509, '-in', full_path, '-text', '-noout'])
keystore_path = "#{ENV['JAVA_HOME']}/jre/lib/security/cacerts"
execute([
:keytool,
'-importcert',
'-alias', Time.now.to_i,
'-file', full_path,
'-trustcacerts',
'-noprompt',
'-storepass', 'changeit',
'-keystore', keystore_path
])
execute([
:keytool, '-list', '-v',
'-storepass changeit',
'-keystore', keystore_path
])
execute(keytool_import_command(full_path))
execute(keytool_list_command)
end
end
end
def keytool_import_command(file_path)
[
:keytool,
'-importcert',
'-alias', Time.now.to_i,
'-file', file_path,
'-trustcacerts',
'-noprompt',
'-storepass', 'changeit',
'-keystore', keystore_path
]
end
def keytool_list_command
[:keytool, '-list', '-v', '-storepass changeit', '-keystore', keystore_path]
end
def keystore_path
"#{ENV['JAVA_HOME']}/jre/lib/security/cacerts"
end
def present?(item)
!item.nil? && !item.empty?
end
def record(stdout, stderr)
logger.debug(stdout) if present?(stdout)
logger.error(stderr) if present?(stderr)
end
end
end
end
{
"licenses": [
{
"count": 7,
"name": "MIT"
},
{
"count": 1,
"name": "ISC"
},
{
"count": 1,
"name": "New BSD, MIT, ISC, Apache 2.0"
},
{
"count": 1,
"name": "unknown"
}
],
"dependencies": [
{
"license": {
"name": "New BSD, MIT, ISC, Apache 2.0",