Verified Commit e265eb6a authored by mo's avatar mo
Browse files

Remove feature flags and temporary mappings

parent 7f175952
FEATURE_RUBY_REPORT 'false'
LM_REPORT_VERSION 1
LM_V1_CANONICALIZE 'false'
FEATURE_RUBY_REPORT 'true'
LM_REPORT_VERSION 1
LM_V1_CANONICALIZE 'false'
FEATURE_RUBY_REPORT 'true'
LM_REPORT_VERSION '1.1'
LM_V1_CANONICALIZE 'false'
FEATURE_RUBY_REPORT 'true'
LM_REPORT_VERSION '1.1'
LM_V1_CANONICALIZE 'false'
FEATURE_RUBY_REPORT 'false'
LM_PYTHON_VERSION 2.7
LM_REPORT_VERSION 1
LM_V1_CANONICALIZE 'false'
......@@ -35,7 +35,6 @@ container_scanning:
image: docker:stable
stage: test
variables:
FEATURE_RUBY_REPORT: 'false'
LM_PYTHON_VERSION: 3
LM_REPORT_VERSION: 1
script:
......@@ -65,14 +64,12 @@ QA:java-maven:
QA:java-maven-v1-1:
extends: QA:java-maven
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: '1.1'
QA_RESULTS: java-maven-v1.1
QA:java-maven-v2:
extends: QA:java-maven
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: 2
QA_RESULTS: java-maven-v2
......@@ -86,14 +83,12 @@ QA:python-pip:
QA:python-pip-v1-1:
extends: QA:python-pip
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: '1.1'
QA_RESULTS: python-pip-v1.1
QA:python-pip-v2:
extends: QA:python-pip
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: '2.0'
QA_RESULTS: python-pip-v2
......@@ -107,14 +102,12 @@ QA:python3-pip:
QA:python3-pip-v1-1:
extends: QA:python3-pip
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: '1.1'
QA_RESULTS: python3-pip-v1.1
QA:python3-pip-v2:
extends: QA:python3-pip
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: 2
QA_RESULTS: python3-pip-v2
......@@ -127,14 +120,12 @@ QA:ruby-bundler:
QA:ruby-bundler-v1-1:
extends: QA:ruby-bundler
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: '1.1'
QA_RESULTS: ruby-bundler-v1.1
QA:ruby-bundler-v2:
extends: QA:ruby-bundler
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: 2
QA_RESULTS: ruby-bundler-v2
......@@ -147,14 +138,12 @@ QA:go:
QA:go-v1-1:
extends: QA:go
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: '1.1'
QA_RESULTS: go-modules-v1.1
QA:go-v2:
extends: QA:go
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: 2
QA_RESULTS: go-modules-v2
......@@ -167,14 +156,12 @@ QA:js:
QA:js-v1-1:
extends: QA:js
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: '1.1'
QA_RESULTS: js-yarn-v1.1
QA:js-v2:
extends: QA:js
variables:
FEATURE_RUBY_REPORT: 'true'
LM_REPORT_VERSION: 2
QA_RESULTS: js-yarn-v2
......
......@@ -12,8 +12,7 @@ RUN cd /tmp && \
tar xf /tmp/jdk-11.tgz && \
mv jdk-11.0.2+9 /usr/lib/jvm/adoptopen_jdk11 && \
rm /tmp/jdk-11.tgz
RUN npm install npm-install-peers cheerio
RUN npm install npm-install-peers
# Don't let Rubygem fail with the numerous projects using PG or MySQL,
# install realpath, includes for python3, and pip for python3
......@@ -46,7 +45,7 @@ ENV LM_PYTHON_VERSION $LM_PYTHON_VERSION
ENV LM_REPORT_VERSION ${LM_REPORT_VERSION:-1}
COPY test /test
COPY run.sh html2json.js /
COPY run.sh /
COPY . /opt/license-management/
RUN bash -lc "source /rvm.sh && cd /opt/license-management && gem build *.gemspec && gem install *.gem"
......
......@@ -118,10 +118,8 @@ to the docker command.
2. Create a new `.env.*` that corresponds to the version to release and specify the default configuration.
```text
FEATURE_RUBY_REPORT 'true'
LM_PYTHON_VERSION '3.5'
LM_REPORT_VERSION '2.0'
LM_V1_CANONICALIZE 'true'
```
Please check the [Release Process documentation](https://gitlab.com/gitlab-org/security-products/release/blob/master/docs/release_process.md).
......
......@@ -2,7 +2,6 @@
set -e
export FEATURE_RUBY_REPORT=${FEATURE_RUBY_REPORT:-true}
export LM_PYTHON_VERSION=${LM_PYTHON_VERSION:-2}
export LM_REPORT_VERSION=${LM_REPORT_VERSION:-1}
export QA_RESULTS=${QA_RESULTS:-$QA_PROJECT-v$LM_REPORT_VERSION}
......@@ -10,7 +9,6 @@ export RESULTS_DIR=${RESULTS_DIR:-`pwd`/results}
docker run \
--rm \
--env FEATURE_RUBY_REPORT \
--env LM_PYTHON_VERSION \
--env LM_REPORT_VERSION \
--volume $RESULTS_DIR:/results \
......
var cheerio = require('cheerio')
var path = require('path')
// Read the HTML report
fs = require('fs')
var htmlContent;
try {
htmlContent = fs.readFileSync(process.argv[2], 'utf8');
} catch(e) {
console.log('Error:', e.stack);
process.exit(1);
}
// Get the directory containing the results to make pathes relative to it later.
report_directory = path.dirname(process.argv[2])
const $ = cheerio.load(htmlContent)
// Map that keeps the total tally of the license occurrences
var licenses = {};
// Extract dependencies info.
var dependencies = []
$('div.dependencies div').each(function(i, doc) {
// Get license name.
license = $(this).find('blockquote p').text().trim();
license = license.split("\n")[0];
if(licenses[license]) {
licenses[license].count += 1;
} else {
licenses[license] = { count: 1, name: license }
}
// Get URL.
license_url = $(this).find('blockquote p a[href]').attr('href');
// Get dependency name.
dependency_name = $(this).find('h2').text().trim();
dependency_name = dependency_name.split("\n")[0];
// Get dependency URL.
dependency_url = $(this).find('h2 a[href]').attr('href');
// Get dependency description.
dependency_description = $(this).find('dl').first().next().text().trim();
// Get dependency location relative to the project root path
dependency_pathes = []
$(this).find('dl').first().find('dd').each(function(i, doc) {
dependency_path = path.relative(report_directory, $(this).text().trim());
// Whitespace path means current directory
if (!dependency_path) {
dependency_path = ".";
}
dependency_pathes.push(dependency_path);
})
dependencies.push({
license: {
name: license,
url: license_url
},
dependency: {
name: dependency_name,
url: dependency_url,
description: dependency_description,
pathes: dependency_pathes
}
})
})
// Stable sort of licenses. First license count descending, then license name ascending
licenses = Object.values(licenses)
.sort(function (a, b) {
if (a.count === b.count) {
return a.name > b.name ? 1 : -1;
}
return a.count < b.count ? 1 : -1;
});
console.log(JSON.stringify({
licenses: licenses,
dependencies: dependencies}, null, 2))
......@@ -14,7 +14,7 @@ module License
private
# when a dependency has multiple licenses, this will join the licenses into a single name
# this defect was backported from the html2json version of this script.
# this defect was backported from the [html2json](https://gitlab.com/gitlab-org/security-products/license-management/blob/7f175952a5a047d785b5ea72c15a10642523c62a/html2json.js) version of this script.
def license_summary
dependencies
.map { |dependency| join_license_names(dependency.licenses) }
......@@ -67,17 +67,11 @@ module License
end
def license_data(license)
return repository.item_for(license, spdx: false) if canonicalize?
{
'name' => license.name.split(/[\r\n]+/)[0],
'url' => license.url || ''
}
end
def canonicalize?
ENV.fetch('LM_V1_CANONICALIZE', 'false') == 'true'
end
end
end
end
......
......@@ -26,7 +26,7 @@ module License
end
def data_for(license)
repository.item_for(license, spdx: true)
repository.item_for(license)
end
def map_from(dependency)
......
......@@ -14,24 +14,18 @@ module License
@spdx_data = load_spdx_data_from(spdx_path)
end
def item_for(license, spdx: true)
if spdx
item = spdx_data_for(license)
return item if item
end
id = id_for(license)
item = id ? compatibility_data['licenses'][id] : nil
item ? { 'id' => id }.merge(item) : generate_item_for(license)
def item_for(license)
spdx_data_for(id_for(license)) ||
spdx_data_for(license.send(:short_name)) ||
generate_item_for(license)
end
private
attr_reader :spdx_data, :compatibility_data
def spdx_data_for(license)
id = id_for(license)
data = id ? spdx_data[id] : spdx_data[license.send(:short_name)]
def spdx_data_for(id)
data = spdx_data[id]
if data
{
'id' => data['licenseId'],
......@@ -39,7 +33,7 @@ module License
'url' => data['seeAlso'][-1]
}
else
log_info("could not find license named `#{license.send(:short_name)}` in SPDX index")
log_info("Could not find license `#{id}` in SPDX")
nil
end
end
......
......@@ -31,58 +31,3 @@ ids:
http://www.apache.org/licenses/LICENSE-2.0: Apache-2.0
ruby: Ruby
unknown: unknown
licenses:
GPL-3.0:
name: GPL 3.0
url: https://www.gnu.org/licenses/gpl-3.0.txt
BSD-2-Clause:
name: Simplified BSD
url: https://opensource.org/licenses/bsd-license
ISC:
name: ISC
url: https://en.wikipedia.org/wiki/ISC_license
Apache-1.1:
name: Apache 1.1
url: https://www.apache.org/licenses/LICENSE-1.1.txt
MPL-2.0:
name: Mozilla Public License 2.0
url: https://www.mozilla.org/media/MPL/2.0/index.815ca599c9df.txt
LGPL-3.0-only:
name: LGPL
url: https://www.gnu.org/licenses/lgpl.txt
CC0-1.0:
name: CC0 1.0
url: https://creativecommons.org/publicdomain/zero/1.0/
unknown:
name: unknown
url: ''
GPL-2.0:
name: GPL 2.0
url: https://www.gnu.org/licenses/gpl-2.0.txt
EPL-1.0:
name: EPL 1.0
url: https://www.eclipse.org/legal/epl-v10.html
Python-2.0:
name: Python Software Foundation License
url: https://hg.python.org/cpython/raw-file/89ce323357db/LICENSE
BSD-3-Clause:
name: New BSD License
url: https://opensource.org/licenses/BSD-3-Clause
MIT:
name: MIT
url: https://opensource.org/licenses/mit-license
LGPL-2.1:
name: LGPL 2.1
url: https://opensource.org/licenses/LGPL-2.1
Ruby:
name: Ruby
url: https://www.ruby-lang.org/en/about/license.txt
Apache-2.0:
name: Apache 2.0
url: https://www.apache.org/licenses/LICENSE-2.0.txt
MPL-1.1:
name: MPL 1.1
url: https://www.mozilla.org/en-US/MPL/1.1/
BSD-4-Clause:
name: BSD
url: https://en.wikipedia.org/wiki/BSD_licenses#4-clause_license_.28original_.22BSD_License.22.29
......@@ -193,21 +193,11 @@ case "$COMMAND" in
if [ "$skip_prepare" != true ]; then
prepare="--prepare"
fi
if [ "$FEATURE_RUBY_REPORT" = 'true' ]; then
echo "Preparing JSON report..."
license_management report ${prepare} --format=json --save=gl-license-management-report.json
else
echo "Preparing HTML report..."
license_finder report ${prepare} --format=html --save=gl-license-management-report.html
fi
echo "Preparing JSON report..."
license_management report ${prepare} --format=json --save=gl-license-management-report.json
# rvm removes trap in bash: https://github.com/rvm/rvm/issues/4416
declare -f restore_lockfile > /dev/null && restore_lockfile
popd > /dev/null
if [ -f "gl-license-management-report.html" ]; then
# Extract data from the HTML report and put it into a JSON file
node /html2json.js $APP_PATH/gl-license-management-report.html > $APP_PATH/gl-license-management-report.json
fi
;;
*)
......
......@@ -2037,7 +2037,7 @@
},
"dependency": {
"name": "acorn",
"url": "https://github.com/ternjs/acorn",
"url": "https://github.com/acornjs/acorn",
"description": "",
"pathes": [
"."
......@@ -2079,7 +2079,7 @@
},
"dependency": {
"name": "acorn",
"url": "https://github.com/acornjs/acorn",
"url": "https://github.com/ternjs/acorn",
"description": "",
"pathes": [
"."
......@@ -2759,7 +2759,8 @@
},
{
"license": {
"name": "MIT*"
"name": "MIT",
"url": "http://opensource.org/licenses/mit-license"
},
"dependency": {
"name": "assert-plus",
......@@ -2772,8 +2773,7 @@
},
{
"license": {
"name": "MIT",
"url": "http://opensource.org/licenses/mit-license"
"name": "MIT*"
},
"dependency": {
"name": "assert-plus",
......@@ -2833,7 +2833,7 @@
},
"dependency": {
"name": "async",
"url": "Unknown",
"url": "https://caolan.github.io/async/",
"description": "",
"pathes": [
"."
......@@ -2847,7 +2847,7 @@
},
"dependency": {
"name": "async",
"url": "https://caolan.github.io/async/",
"url": "Unknown",
"description": "",
"pathes": [
"."
......@@ -2953,7 +2953,8 @@
},
{
"license": {
"name": "Apache*"
"name": "Apache 2.0",
"url": "http://www.apache.org/licenses/LICENSE-2.0.txt"
},
"dependency": {
"name": "aws-sign2",
......@@ -2966,8 +2967,7 @@
},
{
"license": {
"name": "Apache 2.0",
"url": "http://www.apache.org/licenses/LICENSE-2.0.txt"
"name": "Apache*"
},
"dependency": {
"name": "aws-sign2",
......@@ -3419,7 +3419,7 @@
},
"dependency": {
"name": "boom",
"url": "http://hueniverse.com",
"url": "Unknown",
"description": "",
"pathes": [
"."
......@@ -3433,7 +3433,7 @@
},
"dependency": {
"name": "boom",
"url": "Unknown",
"url": "http://hueniverse.com",
"description": "",
"pathes": [
"."
......@@ -3489,7 +3489,7 @@
},
"dependency": {
"name": "braces",
"url": "https://github.com/jonschlinkert/braces",
"url": "https://github.com/micromatch/braces",
"description": "",
"pathes": [
"."
......@@ -3503,7 +3503,7 @@
},
"dependency": {
"name": "braces",
"url": "https://github.com/micromatch/braces",
"url": "https://github.com/jonschlinkert/braces",
"description": "",
"pathes": [
"."
......@@ -3923,7 +3923,7 @@
},
"dependency": {
"name": "callsites",
"url": "http://sindresorhus.com",
"url": "sindresorhus.com",
"description": "",
"pathes": [
"."
......@@ -3937,7 +3937,7 @@
},
"dependency": {
"name": "callsites",
"url": "sindresorhus.com",
"url": "http://sindresorhus.com",
"description": "",
"pathes": [
"."
......@@ -5279,7 +5279,7 @@
},
"dependency": {
"name": "cryptiles",
"url": "http://hueniverse.com",
"url": "Unknown",
"description": "",
"pathes": [
"."
......@@ -5293,7 +5293,7 @@
},
"dependency": {
"name": "cryptiles",
"url": "Unknown",
"url": "http://hueniverse.com",
"description": "",
"pathes": [
"."
......@@ -6305,8 +6305,8 @@
},
{
"license": {
"name": "Simplified BSD",
"url": "http://opensource.org/licenses/bsd-license"
"name": "Apache 2.0",
"url": "http://www.apache.org/licenses/LICENSE-2.0.txt"
},
"dependency": {
"name": "doctrine",
......@@ -6319,8 +6319,8 @@
},
{
"license": {
"name": "Apache 2.0",
"url": "http://www.apache.org/licenses/LICENSE-2.0.txt"
"name": "Simplified BSD",
"url": "http://opensource.org/licenses/bsd-license"
},
"dependency": {
"name": "doctrine",
......@@ -7859,7 +7859,8 @@
},
{
"license": {
"name": "Apache*"
"name": "Apache 2.0",
"url": "http://www.apache.org/licenses/LICENSE-2.0.txt"
},
"dependency": {
"name": "forever-agent",
......@@ -7872,8 +7873,7 @@
},
{
"license": {
"name": "Apache 2.0",
"url": "http://www.apache.org/licenses/LICENSE-2.0.txt"
"name": "Apache*"
},
"dependency": {
"name": "forever-agent",
......@@ -8335,7 +8335,7 @@
},
"dependency": {
"name": "glob-parent",
"url": "https://github.com/es128/glob-parent",
"url": "https://gulpjs.com/",
"description": "",
"pathes": [
"."
......@@ -8349,7 +8349,7 @@
},
"dependency": {
"name": "glob-parent",
"url": "https://gulpjs.com/",
"url": "https://github.com/es128/glob-parent",
"description": "",
"pathes": [
"."