Verified Commit bcfd6f15 authored by mo's avatar mo
Browse files

Convert python qa jobs to integration tests

parent 59573996
......@@ -10,7 +10,6 @@ include:
- template: Container-Scanning.gitlab-ci.yml
- template: Jobs/Code-Quality.gitlab-ci.yml
- local: .gitlab/test.yml
- local: .gitlab/qa.yml
- local: .gitlab/release.yml
services:
......
.QA:
image: docker:stable
stage: test
variables:
LM_PYTHON_VERSION: 3
LM_REPORT_VERSION: 1
script:
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker pull $TMP_IMAGE
- mkdir results
- IMAGE_NAME=$TMP_IMAGE ./bin/test
artifacts:
paths:
- results/
when: always
QA:python-pip:
extends: .QA
variables:
LM_PYTHON_VERSION: 2
QA_PROJECT: python-pip
QA_REF: 04dce91b
QA:python-pip-v1-1:
extends: QA:python-pip
variables:
LM_REPORT_VERSION: '1.1'
QA_RESULTS: python-pip-v1.1
QA:python-pip-v2:
extends: QA:python-pip
variables:
LM_REPORT_VERSION: '2.0'
QA_RESULTS: python-pip-v2
QA:python3-pip:
extends: .QA
variables:
QA_RESULTS: python3-pip-v1
QA_PROJECT: python-pip
QA_REF: 48e250a1
QA:python3-pip-v1-1:
extends: QA:python3-pip
variables:
LM_REPORT_VERSION: '1.1'
QA_RESULTS: python3-pip-v1.1
QA:python3-pip-v2:
extends: QA:python3-pip
variables:
LM_REPORT_VERSION: 2
QA_RESULTS: python3-pip-v2
......@@ -19,7 +19,7 @@ only, but this may change in the future.
registry.gitlab.com/gitlab-org/security-products/license-management:latest analyze /code
```
1. The results will be stored in the `gl-license-management-report.json` file in the application directory. `gl-license-management-report.html` is also available with a human readable report.
1. The results will be stored in the `gl-license-management-report.json` file in the application directory.
## Development
......@@ -38,24 +38,12 @@ You can then run License Management on some target directory:
docker run --rm --volume "/path/to/my/project":/code license-management analyze /code
```
You can run the QA tests using the following command:
You can run the tests using the following command:
```sh
./bin/test-all
./bin/test
```
Or you can run a specific QA test using the following command:
```sh
QA_PROJECT=project_name QA_REF=git_ref ./bin/test
```
where:
- `project_name` is the project slug among the
[test projects for security products](https://gitlab.com/gitlab-org/security-products/tests/) (e.g. `java-maven`)
- `git_ref` is the Git ref to checkout and scan against (e.g. `master`)
### Updating the SPDX index
We will need to periodically update the SPDX index. This can be achieved with
......@@ -149,7 +137,7 @@ unless a new package manager has been added and additional tooling needs to be i
1. Bump the license management version in [CHANGELOG.md](https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/CHANGELOG.md) and in [version.rb](https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/lib/license/management/version.rb)
1. Update the `license_finder` version constraint in the [gemspec](https://gitlab.com/gitlab-org/security-products/license-management/-/blob/39afdc40296871d9f7c1732ff3f7e1e3b7b28bbe/license-management.gemspec#L30).
1. Run `bundle update license_finder`
1. Test the changes locally using the `bin/test-all` script.
1. Test the changes locally using the `bin/test` script.
1. Submit a merge request.
### Upgrading the Docker image
......@@ -159,7 +147,7 @@ unless a new package manager has been added and additional tooling needs to be i
1. If an update is available, create a branch
1. Bump the license management version in [CHANGELOG.md](https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/CHANGELOG.md) and in [version.rb](https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/lib/license/management/version.rb)
1. Edit the version specified in the `DockerFile` to match the desired `license_finder` Docker image tag.
1. Test the changes locally using the `bin/test-all` script.
1. Test the changes locally using the `bin/test` script.
1. Submit a merge request.
# Contributing
......
......@@ -4,15 +4,4 @@ set -e
cd "$(dirname "$0")/.."
export LM_PYTHON_VERSION=${LM_PYTHON_VERSION:-2}
export LM_REPORT_VERSION=${LM_REPORT_VERSION:-1}
export QA_RESULTS=${QA_RESULTS:-$QA_PROJECT-v$LM_REPORT_VERSION}
export RESULTS_DIR=${RESULTS_DIR:-$(pwd)/results}
export QA_REF=${QA_REF:-master}
docker run \
--rm \
--env LM_PYTHON_VERSION \
--env LM_REPORT_VERSION \
--volume "$RESULTS_DIR":/results \
"$IMAGE_NAME" test "$QA_PROJECT" "$QA_RESULTS" "$QA_REF"
bundle exec rspec
#!/bin/bash
set -e
cd "$(dirname "$0")/.."
RESULTS_DIR=$(pwd)/tmp
export RESULTS_DIR
IMAGE_NAME=$(basename "$PWD"):latest
export IMAGE_NAME
./bin/docker-build
REPORT_VERSIONS=(2 1.1 1)
for version in "${REPORT_VERSIONS[@]}"
do
export LM_REPORT_VERSION=$version
echo "Report Version $LM_REPORT_VERSION"
QA_PROJECT=go-modules ./bin/test
QA_PROJECT=php-composer ./bin/test
QA_PROJECT=java-maven QA_REF=831c7a04 ./bin/test
QA_PROJECT=python-pip QA_REF=04dce91b LM_PYTHON_VERSION=2 ./bin/test
QA_PROJECT=python-pip QA_RESULTS="python3-pip-v$version" QA_REF=48e250a1 LM_PYTHON_VERSION=3 ./bin/test
QA_PROJECT=ruby-bundler ./bin/test
QA_PROJECT=js-yarn ./bin/test
QA_PROJECT=js-npm ./bin/test
QA_PROJECT=csharp-nuget-dotnetcore ./bin/test
QA_PROJECT=python-pipenv QA_REF=pip-file-lock ./bin/test
QA_PROJECT=java-gradle ./bin/test
done
......@@ -68,27 +68,4 @@ function scan_project() {
license_management report "${options[@]}"
}
function test_project() {
project=$1
results=$2
ref=$3
url="https://gitlab.com/gitlab-org/security-products/tests/$project.git"
echo "$project" "$results" "$ref" "$url"
echo "Cloning the test project $project from $url"
git clone "$url" "/code/$project"
echo "Running license management on the $project with ref $ref"
cd "/code/$project"
git checkout "$ref"
scan_project "/code/$project"
mkdir -p /results/
cp "/code/$project/gl-license-management-report.json" "/results/$project-gl-license-management-report.json"
diff -u "/test/results/$results.json" "/code/$project/gl-license-management-report.json"
echo "OK"
}
echo "$@"
scan_project "$1"
scan_project "$2"
......@@ -46,4 +46,23 @@ RSpec.describe "pip" do
expect(report[:dependencies]).not_to be_empty
end
end
['2', '3'].each do |python_version|
['1.0', '1.1', '2.0'].each do |report_version|
context "when generating a `#{report_version}` report using Python `#{python_version}`" do
let(:url) { "https://gitlab.com/gitlab-org/security-products/tests/#{language}-#{package_manager}.git" }
let(:language) { 'python' }
let(:package_manager) { 'pip' }
let(:environment) { { 'LM_REPORT_VERSION' => report_version, 'LM_PYTHON_VERSION' => python_version } }
it 'matches the expected report' do
runner.clone(url, branch: '48e250a1')
report = runner.scan(env: environment)
#expect(report).to match_schema(version: report_version)
expected = JSON.parse(fixture_file_content("expected/#{language}/#{python_version}/#{package_manager}/v#{report_version}.json"))
expect(report).to eql(expected)
end
end
end
end
end
......@@ -18,7 +18,7 @@ module IntegrationTestHelper
end
def scan(env: {})
return {} unless execute(env, './bin/test-local', project_path)
return {} unless execute(env, './bin/docker-test', project_path)
report_path = "#{project_path}/gl-license-management-report.json"
return {} unless File.exist?(report_path)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment