Verified Commit 80fa34dd authored by mo's avatar mo
Browse files

Scan Conan projects

* Scan the conan examples project
* Split licenses by comma
* Add integration test job for c projects
* Add package_manager, version and path to v2.1 report
* Detect collisions between names from different package managers
* Add CHANGELOG entry
* Update README to indicate Conan support
* Print timestamps in install script
parent bff873a9
......@@ -43,6 +43,11 @@ unit:
variables:
RSPEC_DIR: spec/unit
integration-c:
extends: .rspec
variables:
RSPEC_DIR: spec/integration/c
integration-dotnet:
extends: .rspec
variables:
......
......@@ -13,6 +13,7 @@ AllCops:
Naming/ClassAndModuleCamelCase:
Exclude:
- 'lib/license/management/report/v1_1.rb'
- 'lib/license/management/report/v2_1.rb'
Layout/IndentFirstArrayElement:
EnforcedStyle: consistent
......
# GitLab License management changelog
## v3.10.0
- Add initial support for the [Conan](https://conan.io/) package manger. (!156)
- Add preview of report version `2.1`. (!156)
## v3.9.2
- Pass `bower_ca` to bower install step. (!151)
......
PATH
remote: .
specs:
license-management (3.9.2)
license-management (3.10.0)
license_finder (~> 6.0.0)
GEM
......
......@@ -77,6 +77,7 @@ The following table shows which languages and package managers are supported.
| Language | Package managers |
|------------|-------------------------------------------------------------------|
| .NET | [.NET Core CLI][dotnet_core], [Nuget][nuget] |
| C/C++ | [Conan][conan] |
| Go | [Go modules][gomod], [Godep][godep], go get |
| Java | [Gradle][gradle], [Maven][maven] |
| JavaScript | [npm][npm], [yarn][yarn], [Bower][bower] |
......@@ -135,20 +136,21 @@ If an unknown license is detected, please consider updating the mapping defined
in [normalized-licenses.yml](https://gitlab.com/gitlab-org/security-products/license-management/blob/master/normalized-licenses.yml). A mapping can be for a detected name or url and must correspond to an SPDX identifier found in [spdx-licenses.json](https://gitlab.com/gitlab-org/security-products/license-management/blob/master/spdx-licenses.json).
[bower]: https://bower.io/
[bundler]: https://bundler.io/
[changelog]: https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/CHANGELOG.md
[composer]: https://getcomposer.org
[conan]: https://conan.io/
[dotnet_core]: https://docs.microsoft.com/en-us/dotnet/core/tools/
[gemfile_lock]: https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/Gemfile.lock
[gemspec]: https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/license-management.gemspec
[license_finder]: https://rubygems.org/gems/license_finder
[npm]: https://www.npmjs.com/
[version_rb]: https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/lib/license/management/version.rb
[yarn]: https://yarnpkg.com/
[gomod]: https://github.com/golang/go/wiki/Modules
[godep]: https://github.com/tools/godep
[gomod]: https://github.com/golang/go/wiki/Modules
[gradle]: https://gradle.org/
[license_finder]: https://rubygems.org/gems/license_finder
[maven]: https://maven.apache.org/
[npm]: https://www.npmjs.com/
[nuget]: https://www.nuget.org/
[dotnet_core]: https://docs.microsoft.com/en-us/dotnet/core/tools/
[pip]: https://pip.pypa.io/en/stable/
[pipenv]: https://github.com/pypa/pipenv
[bundler]: https://bundler.io/
[composer]: https://getcomposer.org
[version_rb]: https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/lib/license/management/version.rb
[yarn]: https://yarnpkg.com/
......@@ -3,6 +3,7 @@ set -euo pipefail
export DEBIAN_FRONTEND=noninteractive
echo ["$(date "+%H:%M:%S")"] "==> Installing packages…"
apt-get clean
apt-get update -q
apt-get install -y --no-install-recommends \
......@@ -76,10 +77,12 @@ wget -q -O /etc/apt/sources.list.d/microsoft-prod.list https://packages.microsof
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list
echo ["$(date "+%H:%M:%S")"] "==> Installing dotnet/mono…"
apt-get update -q
apt-get install -y --no-install-recommends dotnet-sdk-3.1 mono-complete &
curl -o /usr/local/bin/nuget.exe https://dist.nuget.org/win-x86-commandline/latest/nuget.exe &
echo ["$(date "+%H:%M:%S")"] "==> Installing asdf…"
mkdir -p "$ASDF_DATA_DIR"
git clone https://github.com/asdf-vm/asdf.git "$ASDF_DATA_DIR"
cd "$ASDF_DATA_DIR"
......@@ -102,6 +105,8 @@ for version in $(asdf list python); do
pip download -d "$HOME/.config/virtualenv/app-data" pip-licenses pip setuptools wheel
done
wait
echo ["$(date "+%H:%M:%S")"] "==> Beginning cleanup…"
rm -fr /tmp
mkdir -p /tmp
chmod 777 /tmp
......@@ -159,6 +164,7 @@ rm -fr "$ASDF_DATA_DIR/docs" \
/var/lib/systemd/* \
/var/log/*
echo ["$(date "+%H:%M:%S")"] "==> Starting compression…"
zstd_command="/usr/bin/zstd -19 -T0"
cd /opt
tar --use-compress-program "$zstd_command" -cf /opt/asdf.tar.zst asdf &
......@@ -182,4 +188,4 @@ rm -fr \
/usr/lib/mono \
/usr/lib/rustlib \
/usr/share/dotnet
echo "Done"
echo ["$(date "+%H:%M:%S")"] "==> Done"
# frozen_string_literal: true
require 'license/finder/ext/bower'
require 'license/finder/ext/conan'
require 'license/finder/ext/dependency'
require 'license/finder/ext/go_modules'
require 'license/finder/ext/gradle'
require 'license/finder/ext/license'
......
......@@ -34,14 +34,16 @@ module LicenseFinder
meta = bower_module.fetch('pkgMeta', {})
endpoint = bower_module.fetch('endpoint', {})
Package.new(
Dependency.new(
'Bower',
meta['name'] || endpoint['name'],
meta['version'] || endpoint['target'],
summary: meta['description'],
description: meta['readme'],
detection_path: detected_package_path,
homepage: meta['homepage'],
install_path: bower_module['canonicalDir'],
spec_licenses: Package.license_names_from_standard_spec(meta),
install_path: bower_module['canonicalDir']
summary: meta['description']
)
end
......
# frozen_string_literal: true
module LicenseFinder
class Conan
def possible_package_paths
[project_path.join('conanfile.txt')]
end
def current_packages
Dir.chdir(project_path) do
shell.execute([:conan, :install, '.'])
end
stdout, _stderr, status = Dir.chdir(project_path) do
shell.execute([:conan, :info, '-j', '/dev/stdout', '.'])
end
return [] unless status.success?
parse(stdout.lines[0]).map { |dependency| map_from(dependency) }.compact
end
private
def extract_name_version_from(name)
name.split('@', 2).first.split('/', 2)
end
def map_from(dependency)
name, version = extract_name_version_from(dependency['reference'])
return if name == 'conanfile.txt'
Dependency.new('Conan', name, version, spec_licenses: licenses_for(dependency), detection_path: detected_package_path)
end
def licenses_for(dependency)
dependency['license']
end
def parse(line)
JSON.parse(line)
end
end
end
# frozen_string_literal: true
module LicenseFinder
class Dependency < Package
attr_accessor :detection_path, :package_manager
def initialize(package_manager, name, version, options = {})
@package_manager = package_manager
@detection_path = options[:detection_path] || Pathname.pwd
super(name, version, options)
end
def self.from(other, detection_path)
new(
other.package_manager,
other.name,
other.version,
description: other.description,
detection_path: detection_path,
homepage: other.homepage,
install_path: other.install_path,
spec_licenses: other.license_names_from_spec,
summary: other.summary
)
end
end
end
......@@ -8,7 +8,7 @@ module LicenseFinder
end
def active?
Dir[project_path.join('go.sum')].any?
sum_files.any?
end
def current_packages
......@@ -20,6 +20,10 @@ module LicenseFinder
private
def sum_files
Pathname.glob(project_path.join('go.sum'))
end
def go_list_command
[:go, :list, '-m', '-f', "'{{.Path}},{{.Version}},{{.Dir}}'", :all]
end
......@@ -33,7 +37,7 @@ module LicenseFinder
return if dir.nil?
return if Pathname(dir).cleanpath == absolute_project_path
Package.new(name, version, install_path: dir)
Dependency.new('Go', name, version, install_path: dir, detection_path: sum_files.find(&:exist?))
end
end
end
......@@ -44,7 +44,7 @@ module LicenseFinder
end
def map_from(hash)
GradlePackage.new(hash, include_groups: @include_groups)
Dependency.from(GradlePackage.new(hash, include_groups: @include_groups), detected_package_path)
end
end
end
......@@ -38,7 +38,7 @@ module LicenseFinder
::License::Management.logger.debug(xml)
XmlSimple
.xml_in(xml, XML_PARSE_OPTIONS)['dependencies']
.map { |dependency| MavenPackage.new(dependency) }
.map { |dependency| Dependency.from(MavenPackage.new(dependency), detected_package_path) }
end
end
end
......@@ -3,7 +3,9 @@
module LicenseFinder
class NPM
def current_packages
NpmPackage.packages_from_json(npm_json, detected_package_path)
NpmPackage.packages_from_json(npm_json, detected_package_path).map do |item|
Dependency.from(item, detected_package_path)
end
end
def prepare_command
......
......@@ -5,7 +5,7 @@ module LicenseFinder
def current_packages
return legacy_results unless virtual_env?
dependencies = python.pip_licenses
dependencies = python.pip_licenses(detection_path: detected_package_path)
dependencies.any? ? dependencies : legacy_results
end
......@@ -58,10 +58,12 @@ module LicenseFinder
def legacy_results
pip_output.map do |name, version, _children, _location|
spec = PyPI.definition(name, version)
Package.new(
Dependency.new(
'Pip',
name,
version,
description: spec['description'],
detection_path: detected_package_path,
homepage: spec['home_page'],
spec_licenses: PipPackage.license_names_from_spec(spec)
)
......
......@@ -12,7 +12,7 @@ module LicenseFinder
def current_packages
return legacy_results unless pipfile?
python.pip_licenses
python.pip_licenses(detection_path: detected_package_path)
end
private
......
......@@ -54,13 +54,11 @@ module LicenseFinder
end
def map_from(hash)
name = hash['Name']
YarnPackage.new(
name,
build(
hash['Name'],
hash['Version'],
spec_licenses: [hash['License']],
install_path: install_path_for(name).to_s,
install_path: install_path_for(hash['Name']).to_s,
homepage: hash['VendorUrl']
)
end
......@@ -88,7 +86,11 @@ module LicenseFinder
matches = json['data'].to_s.match(INCOMPATIBLE_PACKAGE_REGEX)
return [] unless matches
[YarnPackage.new(matches['name'], matches['version'], spec_licenses: ['unknown'])]
[build(matches['name'], matches['version'], spec_licenses: ['unknown'])]
end
def build(name, version, options = {})
Dependency.new('Yarn', name, version, options.merge(detection_path: detected_package_path))
end
def default_env
......
......@@ -24,7 +24,7 @@ module License
ENV.fetch('PIP_INDEX_URL', 'https://pypi.org/simple/')
end
def pip_licenses(venv: '.venv')
def pip_licenses(venv: '.venv', detection_path:)
_stdout, _stderr, status = shell.sh([
". #{venv}/bin/activate &&",
:pip, :install,
......@@ -41,10 +41,12 @@ module License
return [] unless status.success?
JSON.parse(IO.read('pip-licenses.json')).map do |dependency|
::LicenseFinder::Package.new(
::LicenseFinder::Dependency.new(
'Pip',
dependency['Name'],
dependency['Version'],
description: dependency['Description'],
detection_path: detection_path,
homepage: dependency['URL'],
spec_licenses: [dependency['License']]
)
......
......@@ -4,19 +4,21 @@ require 'license/management/report/base'
require 'license/management/report/v1'
require 'license/management/report/v1_1'
require 'license/management/report/v2'
require 'license/management/report/v2_1'
module License
module Management
module Report
DEFAULT_VERSION = '2'
DEFAULT_VERSION = '2.0'
VERSIONS = {
nil => V1,
'' => V1,
'1' => V1,
'1.0' => V1,
'1.1' => V1_1,
'2' => V2,
'2.0' => V2
'2' => V2_1,
'2.0' => V2,
'2.1' => V2_1
}.freeze
# This method overrides the method defined in `LicenseFinder::JsonReport` to
......
......@@ -30,18 +30,21 @@ module License
end
def map_from(dependency)
licenses = dependency.licenses.map { |license| data_for(license)['id'] }.sort
log(dependency, licenses)
{
name: dependency.name,
url: dependency.homepage,
description: description_for(dependency),
paths: paths_from(dependency),
licenses: licenses
licenses: licenses_for(dependency)
}
end
def licenses_for(dependency)
licenses = dependency.licenses.map { |license| data_for(license)['id'] }.sort
log(dependency, licenses)
licenses
end
def log(dependency, licenses)
logger.info { [dependency.name, dependency.version, licenses].flatten.join(' ') }
return unless licenses == ['unknown']
......
# frozen_string_literal: true
module License
module Management
module Report
class V2_1 < V2
def to_h
super.merge(version: '2.1')
end
private
def license_summary
all_licenses
.map { |license| data_for(license) }
.uniq { |data| data['id'] }
.sort_by { |data| data['id'] }
end
def map_from(dependency)
{
name: dependency.name,
version: dependency.version,
package_manager: dependency.package_manager.downcase.to_sym,
path: detection_path_for(dependency),
licenses: licenses_for(dependency)
}
end
def detection_path_for(dependency)
dependency = dependency.dependency if dependency.instance_of?(LicenseFinder::MergedPackage)
if dependency.respond_to?(:detection_path)
dependency.detection_path.relative_path_from(Pathname.pwd).to_s
else
'.'
end
end
end
end
end
end
......@@ -2,6 +2,6 @@
module License
module Management
VERSION = '3.9.2'
VERSION = '3.10.0'
end
end
......@@ -6,6 +6,7 @@ ids:
Apache2: Apache-2.0
Apache License v2.0: Apache-2.0
ASL, version 2: Apache-2.0
BSD 3-Clause: BSD-3-Clause
BSD (3 clause): BSD-3-Clause
BSD: BSD-4-Clause
BSD-derived (http://www.repoze.org/LICENSE.txt): BSD-2-Clause
......
This diff is collapsed.
{
"version": "2.1",
"licenses": [
{
"id": "BSD-3-Clause",
"name": "BSD 3-Clause \"New\" or \"Revised\" License",
"url": "https://opensource.org/licenses/BSD-3-Clause"
},
{
"id": "ISC",
"name": "ISC License",
"url": "https://opensource.org/licenses/ISC"
},
{
"id": "MIT",
"name": "MIT License",
"url": "https://opensource.org/licenses/MIT"
},
{
"id": "unknown",
"name": "unknown",
"url": ""
}
],
"dependencies": [
{
"name": "github.com/davecgh/go-spew",
"version": "v1.1.1",
"package_manager": "go",
"path": "go.sum",
"licenses": [
"ISC"
]
},
{
"name": "github.com/konsorten/go-windows-terminal-sequences",
"version": "v1.0.1",
"package_manager": "go",
"path": "go.sum",
"licenses": [
"MIT"
]
},
{
"name": "github.com/pmezard/go-difflib",
"version": "v1.0.0",
"package_manager": "go",
"path": "go.sum",
"licenses": [
"unknown"
]
},
{
"name": "github.com/sirupsen/logrus",
"version": "v1.4.2",
"package_manager": "go",
"path": "go.sum",
"licenses": [
"MIT"
]
},
{
"name": "github.com/stretchr/testify",
"version": "v1.2.2",
"package_manager": "go",
"path": "go.sum",
"licenses": [
"MIT"
]
},
{
"name": "golang.org/x/sys",
"version": "v0.0.0-20190422165155-953cdadca894",
"package_manager": "go",
"path": "go.sum",
"licenses": [
"BSD-3-Clause"
]
}
]
}
{
"version": "2.1",
"licenses": [
{
"id": "Apache-2.0",
"name": "Apache License 2.0",
"url": "https://opensource.org/licenses/Apache-2.0"
},
{