Skip to content

GitLab Next

Verified Commit 526a1888 authored by mo's avatar mo
Browse files

Add V2 report

parent fa7a0dd6
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 22280 additions and 12 deletions
.dockerignore
View file @ 526a1888
.git
.env.*
.git*
tmp
Dockerfile
.dockerignore
.gitlab-ci.yml
View file @ 526a1888
......@@ -55,6 +55,13 @@ QA:java-maven:
QA_PROJECT: java-maven
QA_REF: 831c7a04
QA:java-maven-v2:
extends: QA:java-maven
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: 2
QA_RESULTS: java-maven-v2
QA:python-pip:
extends: .QA
variables:
......@@ -62,6 +69,13 @@ QA:python-pip:
QA_PROJECT: python-pip
QA_REF: 04dce91b
QA:python-pip-v2:
extends: QA:python-pip
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: '2.0'
QA_RESULTS: python-pip-v2
QA:python3-pip:
extends: .QA
variables:
......@@ -69,24 +83,52 @@ QA:python3-pip:
QA_PROJECT: python-pip
QA_REF: 48e250a1
QA:python3-pip-v2:
extends: QA:python3-pip
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: 2
QA_RESULTS: python3-pip-v2
QA:ruby-bundler:
extends: .QA
variables:
QA_PROJECT: ruby-bundler
QA_REF: 6b858821
QA:ruby-bundler-v2:
extends: QA:ruby-bundler
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: 2
QA_RESULTS: ruby-bundler-v2
QA:go:
extends: .QA
variables:
QA_PROJECT: go-modules
QA_REF: master
QA:go-v2:
extends: QA:go
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: 2
QA_RESULTS: go-modules-v2
QA:js:
extends: .QA
variables:
QA_PROJECT: js-yarn
QA_REF: master
QA:js-v2:
extends: QA:js
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: 2
QA_RESULTS: js-yarn-v2
.docker_tag:
image: docker:stable
stage: tag
......
CHANGELOG.md
View file @ 526a1888
......@@ -3,6 +3,10 @@
## v1.7.0
- Convert HTML to JSON transformation to generating a JSON report directly.
- Introduce version 2 of the gl-license-management-report.json.
- Add support for multiple licenses.
- Remove license duplication in V2 report.
- Normalize license names to produce consistent results.
## v1.6.1
......
Dockerfile
View file @ 526a1888
......@@ -22,9 +22,6 @@ RUN apt-get update && \
libpq-dev libmysqlclient-dev realpath python3-dev python3-pip && \
rm -rf /var/lib/apt/lists/*
COPY . /opt/license-management/
RUN bash -lc "cd /opt/license-management && gem build *.gemspec && gem install *.gem"
# Don't load RVM automatically, it doesn't work with GitLab-CI
RUN mv /etc/profile.d/rvm.sh /rvm.sh
......@@ -50,5 +47,7 @@ ENV LM_REPORT_VERSION ${LM_REPORT_VERSION:-1}
COPY test /test
COPY run.sh html2json.js /
COPY . /opt/license-management/
RUN bash -lc "source /rvm.sh && cd /opt/license-management && gem build *.gemspec && gem install *.gem"
ENTRYPOINT ["/run.sh"]
README.md
View file @ 526a1888
......@@ -49,6 +49,15 @@ where:
[test projects for security products](https://gitlab.com/gitlab-org/security-products/tests/) (e.g. `java-maven`)
- `git_ref` is the Git ref to checkout and scan against (e.g. `master`)
### Updating the SPDX index
We will need to periodically update the SPDX index. This can be achieved with
the following command.
```bash
$ wget --timestamping https://spdx.org/licenses/licenses.json
```
## Supported languages and package managers
The following table shows which languages and package managers are supported.
......
bin/test_all
View file @ 526a1888
......@@ -2,12 +2,14 @@
set -e
cd "$(dirname "$0")/.."
export RESULTS_DIR=`pwd`/tmp
export TMP_IMAGE=$(basename $PWD):latest
docker build . -t $TMP_IMAGE --pull
REPORT_VERSIONS=(1)
REPORT_VERSIONS=(1 2)
for version in "${REPORT_VERSIONS[@]}"
do
export LM_REPORT_VERSION=$version
......@@ -17,4 +19,5 @@ do
QA_PROJECT=python-pip QA_REF=04dce91b LM_PYTHON_VERSION=2 ./bin/test
QA_PROJECT=python-pip QA_RESULTS="python3-pip-v$version" QA_REF=48e250a1 LM_PYTHON_VERSION=3 ./bin/test
QA_PROJECT=ruby-bundler QA_REF=6b858821 ./bin/test
QA_PROJECT=js-yarn QA_REF=master ./bin/test
done
lib/license/management/report.rb
View file @ 526a1888
......@@ -2,6 +2,7 @@
require 'license/management/report/base'
require 'license/management/report/v1'
require 'license/management/report/v2'
module License
module Management
......@@ -11,7 +12,9 @@ module License
nil => V1,
'' => V1,
'1' => V1,
'1.0' => V1
'1.0' => V1,
'2' => V2,
'2.0' => V2
}.freeze
# This method overrides the method defined in `LicenseFinder::JsonReport` to
......
lib/license/management/report/v1.rb
View file @ 526a1888
......@@ -67,11 +67,11 @@ module License
end
def license_data(license)
return repository.item_for(license) if canonicalize?
return repository.item_for(license, spdx: false) if canonicalize?
{
'name' => license.name.split(/[\r\n]+/)[0],
'url' => license.url
'url' => license.url || ''
}
end
......
lib/license/management/report/v2.rb 0 → 100644
View file @ 526a1888
# frozen_string_literal: true
module License
module Management
module Report
class V2 < Base
def to_h
{
version: '2.0',
licenses: license_summary,
dependencies: dependencies.sort_by(&:name).map { |x| map_from(x) }
}
end
private
def all_licenses
dependencies.map { |x| x.licenses.to_a }.flatten
end
def license_summary
all_licenses
.group_by { |x| data_for(x)['name'] }
.sort_by { |x, y| [-y.size, x] }
.map { |_name, items| data_for(items[0]).merge(count: items.count) }
end
def data_for(license)
repository.item_for(license, spdx: true)
end
def map_from(dependency)
{
name: dependency.name,
url: dependency.homepage,
description: description_for(dependency),
paths: paths_from(dependency),
licenses: dependency.licenses.map { |x| data_for(x)['id'] }
}
end
end
end
end
end
lib/license/management/repository.rb
View file @ 526a1888
......@@ -7,12 +7,19 @@ module License
include Verifiable
def initialize(
compatibility_path: License::Management.root.join('normalized-licenses.yml')
compatibility_path: License::Management.root.join('normalized-licenses.yml'),
spdx_path: License::Management.root.join('spdx-licenses.json')
)
@compatibility_data = YAML.safe_load(IO.read(compatibility_path))
@spdx_data = load_spdx_data_from(spdx_path)
end
def item_for(license)
def item_for(license, spdx: true)
if spdx
item = spdx_data_for(license)
return item if item
end
id = id_for(license)
item = id ? compatibility_data['licenses'][id] : nil
item ? { 'id' => id }.merge(item) : generate_item_for(license)
......@@ -20,7 +27,22 @@ module License
private
attr_reader :compatibility_data
attr_reader :spdx_data, :compatibility_data
def spdx_data_for(license)
id = id_for(license)
data = id ? spdx_data[id] : spdx_data[license.send(:short_name)]
if data
{
'id' => data['licenseId'],
'name' => data['name'],
'url' => data['detailsUrl']
}
else
log_info("could not find license named `#{license.send(:short_name)}` in SPDX index")
nil
end
end
def id_for(license)
ids = compatibility_data['ids']
......@@ -44,6 +66,17 @@ module License
'url' => present?(license.url) ? license.url : ''
}
end
def load_spdx_data_from(path)
content = IO.read(path)
json = JSON.parse(content)
licenses = json['licenses']
licenses.inject({}) do |memo, license|
memo[license['licenseId']] = license
memo
end
end
end
end
end
spdx-licenses.json 0 → 100644
View file @ 526a1888
This diff is collapsed.
test/results/go-modules-v2.json 0 → 100644
View file @ 526a1888
{
"version": "2.0",
"licenses": [
{
"id": "unknown",
"name": "unknown",
"url": "",
"count": 1
}
],
"dependencies": [
{
"name": "golang.org/x/crypto",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"unknown"
]
}
]
}
test/results/java-maven-v2.json 0 → 100644
View file @ 526a1888
{
"version": "2.0",
"licenses": [
{
"id": "Apache-2.0",
"name": "Apache License 2.0",
"url": "http://spdx.org/licenses/Apache-2.0.json",
"count": 27
},
{
"id": "MIT",
"name": "MIT License",
"url": "http://spdx.org/licenses/MIT.json",
"count": 3
},
{
"id": "unknown",
"name": "CDDL + GPLv2 with classpath exception",
"url": "",
"count": 2
},
{
"id": "LGPL-2.1",
"name": "GNU Lesser General Public License v2.1 only",
"url": "http://spdx.org/licenses/LGPL-2.1.json",
"count": 2
},
{
"id": "BSD-4-Clause",
"name": "BSD 4-Clause \"Original\" or \"Old\" License",
"url": "http://spdx.org/licenses/BSD-4-Clause.json",
"count": 1
},
{
"id": "unknown",
"name": "BSD style",
"url": "",
"count": 1
},
{
"id": "unknown",
"name": "Common Public License Version 1.0",
"url": "",
"count": 1
},
{
"id": "MPL-1.1",
"name": "Mozilla Public License 1.1",
"url": "http://spdx.org/licenses/MPL-1.1.json",
"count": 1
},
{
"id": "MPL-2.0",
"name": "Mozilla Public License 2.0",
"url": "http://spdx.org/licenses/MPL-2.0.json",
"count": 1
}
],
"dependencies": [
{
"name": "antlr",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"BSD-4-Clause"
]
},
{
"name": "commons-beanutils",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"Apache-2.0"
]
},
{
"name": "commons-io",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"Apache-2.0"
]
},
{
"name": "commons-lang",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"Apache-2.0"
]
},
{
"name": "commons-lang3",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"Apache-2.0"
]
},
{
"name": "fastutil",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"Apache-2.0"
]
},
{
"name": "findbugs-annotations",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"Apache-2.0"
]
},
{
"name": "geode-common",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"Apache-2.0"
]
},
{
"name": "geode-core",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"Apache-2.0"
]
},
{
"name": "geode-json",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"Apache-2.0"
]
},
{
"name": "hamcrest-core",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"unknown"
]
},
{
"name": "jackson-annotations",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"Apache-2.0"
]
},
{
"name": "jackson-core",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"Apache-2.0"
]
},
{
"name": "jackson-databind",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"Apache-2.0"
]
},
{
"name": "javassist",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"MPL-1.1",
"LGPL-2.1",
"Apache-2.0"
]
},
{
"name": "javax.resource-api",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"unknown"
]
},
{
"name": "javax.transaction-api",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"unknown"
]
},
{
"name": "jgroups",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"Apache-2.0"
]
},
{
"name": "jna",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"LGPL-2.1",
"Apache-2.0"
]
},
{
"name": "jopt-simple",
"url": "",
"description": "",
"paths": [
"."
],
"licenses": [
"MIT"