Unverified Commit d57e6160 authored by mo's avatar mo
Browse files

insert shim to generate report via ruby API

parent 19f8fdcd
......@@ -35,13 +35,15 @@ container_scanning:
image: docker:stable
stage: test
variables:
FEATURE_RUBY_REPORT: 'false'
LM_PYTHON_VERSION: 2
LM_REPORT_VERSION: 1
script:
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker pull $TMP_IMAGE
- mkdir results
- docker run --env LM_PYTHON_VERSION --volume `pwd`/results:/results $TMP_IMAGE test $QA_PROJECT ${QA_RESULTS:-$QA_PROJECT} $QA_REF
- ./bin/test
artifacts:
paths:
- results/
......@@ -63,7 +65,7 @@ QA:python3-pip:
extends: .QA
variables:
LM_PYTHON_VERSION: 3
QA_RESULTS: python3-pip
QA_RESULTS: python3-pip-v1
QA_PROJECT: python-pip
QA_REF: 48e250a1
......
# GitLab License management changelog
## v1.6.0
- Convert HTML to JSON transformation to generating a JSON report directly.
## v1.5.0
- Reverts 1.4.0
......
......@@ -22,6 +22,9 @@ RUN apt-get update && \
libpq-dev libmysqlclient-dev realpath python3-dev python3-pip && \
rm -rf /var/lib/apt/lists/*
COPY . /opt/license-management/
RUN bash -lc "cd /opt/license-management && gem build *.gemspec && gem install *.gem"
# Don't load RVM automatically, it doesn't work with GitLab-CI
RUN mv /etc/profile.d/rvm.sh /rvm.sh
......@@ -43,6 +46,7 @@ RUN pip3 install --disable-pip-version-check setuptools==$SETUPTOOLS_VERSION &&
# Version of Python, defaults to Python 2.7
ARG LM_PYTHON_VERSION
ENV LM_PYTHON_VERSION ${LM_PYTHON_VERSION:-2.7}
ENV LM_REPORT_VERSION ${LM_REPORT_VERSION:-1}
COPY test /test
COPY run.sh html2json.js /
......
......@@ -40,7 +40,7 @@ docker run --rm --volume "/path/to/my/project":/code license-management analyze
You can run integration tests on the image like this:
```sh
docker run --rm license-management /test/test.sh project_name git_ref
./bin/test_all
```
where:
......
#!/bin/sh
set -e
export FEATURE_RUBY_REPORT=${FEATURE_RUBY_REPORT:-true}
export LM_PYTHON_VERSION=${LM_PYTHON_VERSION:-2}
export LM_REPORT_VERSION=${LM_REPORT_VERSION:-1}
export QA_RESULTS=${QA_RESULTS:-$QA_PROJECT-v$LM_REPORT_VERSION}
export RESULTS_DIR=${RESULTS_DIR:-`pwd`/results}
docker run \
--rm \
--env FEATURE_RUBY_REPORT \
--env LM_PYTHON_VERSION \
--env LM_REPORT_VERSION \
--volume $RESULTS_DIR:/results \
$TMP_IMAGE test $QA_PROJECT ${QA_RESULTS} $QA_REF
#!/bin/sh
set -e
export RESULTS_DIR=`pwd`/tmp
export TMP_IMAGE=$(basename $PWD):latest
docker pull licensefinder/license_finder:5.6.2
docker build . -t $TMP_IMAGE
REPORT_VERSIONS=(1)
for version in "${REPORT_VERSIONS[@]}"
do
export LM_REPORT_VERSION=$version
echo "Report Version $LM_REPORT_VERSION"
QA_PROJECT=go-modules QA_RESULTS="go-modules-v$version" QA_REF=master ./bin/test
QA_PROJECT=java-maven QA_RESULTS="java-maven-v$version" QA_REF=831c7a04 ./bin/test
QA_PROJECT=python-pip QA_RESULTS="python-pip-v$version" QA_REF=04dce91b LM_PYTHON_VERSION=2 ./bin/test
QA_PROJECT=python-pip QA_RESULTS="python3-pip-v$version" QA_REF=48e250a1 LM_PYTHON_VERSION=3 ./bin/test
QA_PROJECT=ruby-bundler QA_RESULTS="ruby-bundler-v$version" QA_REF=6b858821 ./bin/test
done
#!/usr/bin/env ruby
# frozen_string_literal: true
require 'license/management'
LicenseFinder::CLI::Main.start(ARGV)
# frozen_string_literal: true
require 'pathname'
require 'yaml'
require 'license_finder'
require 'license/management/loggable'
require 'license/management/verifiable'
require 'license/management/repository'
require 'license/management/versions/base'
require 'license/management/versions/v1'
require 'license/management/json_report'
require 'license/management/version'
module License
module Management
def self.root
Pathname.new(File.dirname(__FILE__)).join('../..')
end
end
end
# frozen_string_literal: true
# This is a monkey patch of the JsonReport found in `license_finder`
module LicenseFinder
class JsonReport < CsvReport
DEFAULT_VERSION = '1'
VERSIONS = {
nil => ::License::Management::Versions::V1,
'' => ::License::Management::Versions::V1,
'1' => ::License::Management::Versions::V1,
'1.0' => ::License::Management::Versions::V1
}.freeze
def to_s
JSON.pretty_generate(version_for(report_version).to_h) + "\n"
end
private
def report_version
ENV.fetch('LM_REPORT_VERSION', DEFAULT_VERSION)
end
def version_for(version)
VERSIONS.fetch(version.to_s).new(dependencies)
end
end
end
# frozen_string_literal: true
module License
module Management
module Loggable
def logger
::LicenseFinder::Core.default_logger
end
def log_info(message)
logger.info(self.class, message)
end
end
end
end
# frozen_string_literal: true
module License
module Management
class Repository
include Loggable
include Verifiable
def initialize(
compatibility_path: License::Management.root.join('licenses.yml')
)
@compatibility_data = YAML.safe_load(IO.read(compatibility_path))
end
def item_for(license)
id = id_for(license)
item = id ? compatibility_data['items'][id] : nil
item ? { 'id' => id }.merge(item) : generate_item_for(license)
end
private
attr_reader :compatibility_data
def id_for(license)
ids = compatibility_data['ids']
ids[license.send(:short_name)] || ids[license.url]
end
def take_first_line_from(content)
return '' if blank?(content)
content.split(/[\r\n]+/)[0]
end
def generate_item_for(license)
log_info("detected unknown license named `#{license.send(:short_name)}`")
{
'id' => 'unknown',
'name' => take_first_line_from(license.name),
'url' => present?(license.url) ? license.url : ''
}
end
end
end
end
# frozen_string_literal: true
module License
module Management
module Verifiable
def blank?(item)
item.nil? || item.empty?
end
def present?(item)
!blank?(item)
end
end
end
end
# frozen_string_literal: true
module License
module Management
VERSION = '1.6.0'
end
end
# frozen_string_literal: true
module License
module Management
module Versions
class Base
include Loggable
include Verifiable
attr_reader :dependencies, :repository
def initialize(dependencies)
@dependencies = dependencies
@repository = License::Management::Repository.new
end
private
def paths_from(dependency)
return [] unless dependency.respond_to?(:aggregate_paths)
paths = dependency.aggregate_paths
return [] if blank?(paths)
paths.map { |x| x.gsub(Dir.pwd, '.') }
end
def description_for(dependency)
present?(dependency.summary) ? dependency.summary : dependency.description
end
end
end
end
end
# frozen_string_literal: true
module License
module Management
module Versions
class V1 < Base
def to_h
{
licenses: license_summary,
dependencies: formatted_dependencies(dependencies)
}
end
private
# when a dependency has multiple licenses, this will join the licenses into a single name
# this defect was backported from the htmltojson version of this script.
def license_summary
dependencies
.map { |x| x.licenses.map { |y| best_name_for(y) }.sort.reverse.join(', ') }
.flatten
.group_by { |name| name }
.map { |(license, items)| { count: items.count, name: license } }
.sort_by { |x| [-x[:count], x[:name]] }
end
# when a dependency has more than one license
# this method chooses one of the urls.
# to maintain backwards compatibility this bug has been carried forward.
def license_for(dependency)
name = join_license_names(dependency.licenses)
urls = dependency.licenses.map(&:url).reject { |x| blank?(x) }.uniq.sort
log_info("multiple urls detected: #{urls.inspect}") if urls.size > 1
url = urls[0] || data_for(dependency.licenses.first)['url']
blank?(url) ? { name: name } : { name: name, url: url }
end
def join_license_names(licenses)
licenses.map { |x| best_name_for(x) }.join(', ')
end
def map_from_dependency(dependency)
result = {
license: license_for(dependency),
dependency: {
name: dependency.name,
url: dependency.homepage,
description: description_for(dependency),
pathes: paths_from(dependency)
}
}
result[:dependency].delete(:url) if blank?(dependency.homepage)
result
end
def formatted_dependencies(dependencies)
dependencies
.sort_by(&:name)
.map { |x| map_from_dependency(x) }
end
def best_name_for(license)
data_for(license)['name']
end
def data_for(license)
repository.item_for(license)
end
end
end
end
end
# frozen_string_literal: true
lib = File.expand_path('lib', __dir__)
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
require 'license/management/version'
Gem::Specification.new do |spec|
spec.name = 'license-management'
spec.version = License::Management::VERSION
spec.authors = ['Fabien Catteau', 'Olivier Gonzalez']
spec.email = ['fcatteau@gitlab.com', 'ogonzalez@gitlab.com']
spec.summary = 'License Management job for GitLab CI.'
spec.description = 'License Management job for GitLab CI.'
spec.homepage = 'https://gitlab.com/gitlab-org/security-products/license-management'
spec.license = 'GitLab EE'
spec.metadata['allowed_push_host'] = 'https://example.com'
spec.metadata['homepage_uri'] = spec.homepage
spec.metadata['source_code_uri'] = 'https://gitlab.com/gitlab-org/security-products/license-management'
spec.metadata['changelog_uri'] = 'https://gitlab.com/gitlab-org/security-products/license-management/blob/master/CHANGELOG.md'
spec.files = Dir.chdir(File.expand_path(__dir__)) do
Dir.glob('exe/*') + Dir.glob('lib/**/**/*.{rb,yml}') + Dir.glob('*.{md,yml,json}')
end
spec.bindir = 'exe'
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
spec.require_paths = ['lib']
spec.add_dependency 'license_finder', ENV.fetch('LICENSE_FINDER_VERSION', '5.6.2')
end
---
ids:
ASL, version 2: Apache-2.0
Apache1_1: Apache-1.1
Apache2: Apache-2.0
Apache 2.0: Apache-2.0
BSD: BSD-4-Clause
CC01: CC0-1.0
EPL1: EPL-1.0
GPLv2: GPL-2.0
GPLv3: GPL-3.0
ISC: ISC
LGPL 2.1: LGPL-2.1
LGPL, version 2.1: LGPL-2.1
LGPL2_1: LGPL-2.1
LGPL: LGPL-3.0-only
MIT: MIT
MPL 1.1: MPL-1.1
MPL1_1: MPL-1.1
MPL2: MPL-2.0
New BSD License: BSD-3-Clause
NewBSD: BSD-3-Clause
Python: Python-2.0
Ruby: Ruby
SimplifiedBSD: BSD-2-Clause
http://www.apache.org/licenses/LICENSE-2.0: Apache-2.0
unknown: unknown
items:
GPL-3.0:
name: GPL 3.0
url: https://www.gnu.org/licenses/gpl-3.0.txt
BSD-2-Clause:
name: Simplified BSD
url: https://opensource.org/licenses/bsd-license
ISC:
name: ISC
url: https://en.wikipedia.org/wiki/ISC_license
Apache-1.1:
name: Apache 1.1
url: https://www.apache.org/licenses/LICENSE-1.1.txt
MPL-2.0:
name: Mozilla Public License 2.0
url: https://www.mozilla.org/media/MPL/2.0/index.815ca599c9df.txt
LGPL-3.0-only:
name: LGPL
url: https://www.gnu.org/licenses/lgpl.txt
CC0-1.0:
name: CC0 1.0
url: https://creativecommons.org/publicdomain/zero/1.0/
unknown:
name: unknown
url: ''
GPL-2.0:
name: GPL 2.0
url: https://www.gnu.org/licenses/gpl-2.0.txt
EPL-1.0:
name: EPL 1.0
url: https://www.eclipse.org/legal/epl-v10.html
Python-2.0:
name: Python Software Foundation License
url: https://hg.python.org/cpython/raw-file/89ce323357db/LICENSE
BSD-3-Clause:
name: New BSD License
url: https://opensource.org/licenses/BSD-3-Clause
MIT:
name: MIT
url: https://opensource.org/licenses/mit-license
LGPL-2.1:
name: LGPL 2.1
url: https://opensource.org/licenses/LGPL-2.1
Ruby:
name: Ruby
url: https://www.ruby-lang.org/en/about/license.txt
Apache-2.0:
name: Apache 2.0
url: https://www.apache.org/licenses/LICENSE-2.0.txt
MPL-1.1:
name: MPL 1.1
url: https://www.mozilla.org/en-US/MPL/1.1/
BSD-4-Clause:
name: BSD
url: https://en.wikipedia.org/wiki/BSD_licenses#4-clause_license_.28original_.22BSD_License.22.29
......@@ -115,12 +115,13 @@ case "$COMMAND" in
gem install bundler
# We need to install the license_finder gem into this Ruby version too.
gem install license_finder -v "$LICENSE_FINDER_VERSION"
gem install --no-document /opt/license-management/*.gem
fi
# Ignore test and development dependencies.
license_finder ignored_groups add development
license_finder ignored_groups add test
bundle install --without "development test"
bundle install --without "development test" --jobs $(nproc)
skip_prepare=true
fi
......@@ -189,17 +190,24 @@ case "$COMMAND" in
# Run License Finder.
echo "Running license_finder $@ in $PWD"
if [ "$skip_prepare" != true ]; then
prepare="--prepare"
fi
license_finder report ${prepare} --format=html --save=gl-license-management-report.html
if [ "$FEATURE_RUBY_REPORT" = 'true' ]; then
echo "Preparing JSON report..."
license_management report ${prepare} --format=json --save=gl-license-management-report.json
else
echo "Preparing HTML report..."
license_finder report ${prepare} --format=html --save=gl-license-management-report.html
fi
# rvm removes trap in bash: https://github.com/rvm/rvm/issues/4416
declare -f restore_lockfile > /dev/null && restore_lockfile
popd > /dev/null
# Extract data from the HTML report and put it into a JSON file
node /html2json.js $APP_PATH/gl-license-management-report.html > $APP_PATH/gl-license-management-report.json
if [ -f "gl-license-management-report.html" ]; then
# Extract data from the HTML report and put it into a JSON file
node /html2json.js $APP_PATH/gl-license-management-report.html > $APP_PATH/gl-license-management-report.json
fi
;;
*)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment