Commit 1bc270fe authored by mo's avatar mo

Merge branch '273155-recursive' into 'main'

Detect maven/gradle wrapper in sub directory

See merge request !15
parents f8200b26 82338d97
Pipeline #214011956 passed with stages
in 45 minutes and 46 seconds
......@@ -16,7 +16,7 @@
cache:
key: ${CI_JOB_NAME}
paths:
- tmp/omnibus
- tmp/omnibus/cache
asdf-0.7.8:
extends: .deb
......
# GitLab License management changelog
## v3.28.2
- Detect maven wrapper in nested directories. (https://gitlab.com/gitlab-org/security-products/analyzers/license-finder/-/merge_requests/15)
- Detect gradle wrapper in nested directories. (https://gitlab.com/gitlab-org/security-products/analyzers/license-finder/-/merge_requests/15)
## v3.28.1
- Set `golang` version to `1.15.2` in `.tool-versions` file. (https://gitlab.com/gitlab-org/security-products/analyzers/license-finder/-/merge_requests/6)
......
......@@ -8,7 +8,7 @@ GIT
PATH
remote: .
specs:
license-management (3.28.1)
license-management (3.28.2)
license_finder (~> 6.7)
spandx (~> 0.13)
......
......@@ -5,9 +5,12 @@ set -e
cd "$(dirname "$0")/.."
IMAGE_NAME=${IMAGE_NAME:-$(basename "$PWD"):latest}
root_dir=/builds/gitlab-org/security-products/analyzers/license-finder
docker run --rm -it \
--entrypoint='' \
--entrypoint='bash' \
--network=host \
--volume "$PWD":/builds/gitlab-org/security-products/analyzers/license-finder \
"$IMAGE_NAME" sh -c 'cd /builds/gitlab-org/security-products/analyzers/license-finder && exec bash -l'
--env=RUBYLIB="${root_dir}/lib" \
--workdir="${root_dir}" \
--volume "$PWD":"${root_dir}" \
"$IMAGE_NAME" -l
......@@ -4,7 +4,7 @@ set -e
cd "$(dirname "$0")/.."
export PATH="/builds/gitlab-org/security-products/analyzers/license-finder/exe:/opt/gitlab/embedded/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
export PATH="$PWD/exe:/opt/gitlab/embedded/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
if [ ! -f /usr/sbin/haproxy ] && command -v apt-get; then
curl https://haproxy.debian.net/bernat.debian.org.gpg | apt-key add -
......
......@@ -4,10 +4,6 @@ set -e
cd "$(dirname "$0")/.."
export PATH="/builds/gitlab-org/security-products/analyzers/license-finder/exe:/opt/gitlab/embedded/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
export RUBYLIB="/builds/gitlab-org/security-products/analzyers/license-finder/lib"
export PATH="$PWD/exe:/opt/gitlab/embedded/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
if ! command -v bundle; then
./bin/setup
fi
bundle exec rspec "$@" --format=progress
......@@ -65,15 +65,15 @@ build do
when /^3\.3\./
command "curl https://bootstrap.pypa.io/3.3/get-pip.py -o #{project_dir}/get-pip.py", env: env
command "#{install_dir}/bin/python #{project_dir}/get-pip.py \"pip==10.0.1\"", env: env
command "#{install_dir}/bin/pip install pipenv virtualenv", env: env
command "#{install_dir}/bin/pip install pipenv==2018.6.25 virtualenv==16.0.0", env: env
when /^3\.4\./
command "#{install_dir}/bin/pip install pipenv==11.1.6 virtualenv", env: env
command "#{install_dir}/bin/pip install pipenv==11.1.6 virtualenv==20.0.33", env: env
when /^2\.7\./
command "curl https://bootstrap.pypa.io/2.6/get-pip.py -o #{project_dir}/get-pip.py", env: env
command "#{install_dir}/bin/python #{project_dir}/get-pip.py \"pip<20.0\"", env: env
command "#{install_dir}/bin/pip install pipenv virtualenv", env: env
command "#{install_dir}/bin/pip install pipenv==2020.8.13 virtualenv==20.0.33", env: env
else
command "#{install_dir}/bin/pip install pipenv virtualenv", env: env
command "#{install_dir}/bin/pip install pipenv==2020.8.13 virtualenv==20.0.33", env: env
end
if version == default_version
......
......@@ -18,7 +18,7 @@ module LicenseFinder
end
def package_management_command
wrapper? ? './gradlew' : :gradle
wrapper? ? project_path.join('gradlew') : :gradle
end
private
......
......@@ -10,6 +10,10 @@ module LicenseFinder
}
}.freeze
def active?
project_path.join('pom.xml').exist?
end
def prepare
within_project_path do
tool_box.install(tool: :java, version: java_version, env: default_env)
......@@ -38,9 +42,8 @@ module LicenseFinder
end
def detect_licenses_command
mvn_wrapper = project_path.join('mvnw')
[
mvn_wrapper.exist? ? mvn_wrapper : :mvn,
package_management_command,
"-e",
"org.codehaus.mojo:license-maven-plugin:aggregate-download-licenses",
"-Dlicense.excludedScopes=#{@ignored_groups.to_a.join(',')}",
......@@ -59,5 +62,13 @@ module LicenseFinder
.xml_in(xml, XML_PARSE_OPTIONS)['dependencies']
.map { |dependency| Dependency.from(MavenPackage.new(dependency), detected_package_path) }
end
def package_management_command
wrapper? ? project_path.join('mvnw') : :mvn
end
def wrapper?
project_path.join('mvnw').exist?
end
end
end
......@@ -11,7 +11,7 @@ module LicenseFinder
tool_box.install(tool: :nodejs, env: default_env)
if lockfile?
shell.execute([:npm, :ci, "--production"], env: default_env)
shell.execute([:npm, :ci, "--production"], env: default_env, capture: false)
else
shell.execute([:npm, :install, '--no-save', "--production"], env: default_env)
end
......
......@@ -54,7 +54,7 @@ module LicenseFinder
within_project_path do
tool_box.install(tool: :python, version: python_version, env: default_env)
shell.execute(["/opt/asdf/installs/python/#{python_version}/bin/virtualenv", '-p', 'python', '--activators=bash --seeder=app-data .venv'])
shell.execute([:virtualenv, '-p', 'python', '--activators=bash', '--seeder=app-data', '.venv'], capture: false)
shell.sh([". .venv/bin/activate", "&&", 'pip', 'install', '-v', '-r', @requirements_path], env: default_env)
end
end
......
......@@ -17,7 +17,9 @@ module License
mono: '/opt/asdf/installs/mono/6.8.0.123/bin/mono',
mvn: '/opt/asdf/bin/asdf exec mvn',
nuget: '/opt/asdf/installs/mono/6.8.0.123/bin/nuget.exe',
pip: '/opt/asdf/bin/asdf exec pip',
ruby: '/opt/asdf/bin/asdf exec ruby',
virtualenv: '/opt/asdf/bin/asdf exec virtualenv',
yarn: '/opt/asdf/bin/asdf exec yarn'
}.freeze
......@@ -31,14 +33,15 @@ module License
trust!(certificate) if present?(certificate)
end
def execute(command, env: {})
def execute(command, env: {}, capture: true)
expanded_command = expand(command)
collapsible_section(expanded_command) do
logger.debug(expanded_command)
stdout, stderr, status = Open3.capture3(default_env.merge(env), expanded_command)
record(stdout, stderr, status)
[stdout, stderr, status]
end
expanded_env = default_env.merge(env)
return system(expanded_env, expanded_command) unless capture
logger.debug(expanded_command)
stdout, stderr, status = Open3.capture3(expanded_env, expanded_command)
record(stdout, stderr, status)
[stdout, stderr, status]
end
def sh(command, env: {})
......@@ -85,14 +88,6 @@ module License
def flush(message, severity)
logger.add(severity, message) if present?(message)
end
def collapsible_section(header)
id = header.downcase.gsub(/[[:space:]]/, '_').gsub(/[^0-9a-z ]/i, '_')
logger.debug("\nsection_start:#{Time.now.to_i}:#{id}\r\e[0K#{header}")
yield
ensure
logger.debug("\nsection_end:#{Time.now.to_i}:#{id}\r\e[0K")
end
end
end
end
......@@ -17,17 +17,17 @@ module License
deb = deb_for(tool, version)
if deb&.exist?
::License::Management.logger.error("Installing #{deb} ...")
shell.execute([:dpkg, '-i', deb])
shell.execute([:dpkg, '-i', deb], capture: false)
else
::License::Management.logger.error("Installing #{version} via asdf ...")
shell.execute([:asdf, "plugin-update", tool.to_s], env: env)
shell.execute(['/opt/asdf/plugins/nodejs/bin/import-release-team-keyring']) if tool == :nodejs
end
shell.execute([:asdf, :install, tool.to_s, version], env: env)
install_common_libraries(env: env) if C_BASED_TOOLS.include?(tool.to_sym)
shell.execute([:asdf, :install, tool.to_s, version], env: env, capture: false)
shell.execute([:asdf, :local, tool.to_s, version], env: env)
shell.execute([:asdf, :reshim], env: env)
end
install_common_libraries(env: env) if C_BASED_TOOLS.include?(tool.to_sym)
install_certificates_into_java_keystore(env, version) if tool == :java
end
......
......@@ -2,6 +2,6 @@
module License
module Management
VERSION = '3.28.1'
VERSION = '3.28.2'
end
end
......@@ -50,9 +50,7 @@ function prepare_project() {
license_management ignored_groups add test
}
echo -e "section_start:$(date +%s):prepare_project\r\e[0KPrepare"
prepare_project
echo -e "section_end:$(date +%s):prepare_project\r\e[0K"
scan_project "$PREPARE" \
--format=json \
......
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
id("org.springframework.boot") version "2.3.4.RELEASE"
id("io.spring.dependency-management") version "1.0.10.RELEASE"
kotlin("jvm") version "1.3.72"
kotlin("plugin.spring") version "1.3.72"
}
group = "com.example"
version = "0.0.1-SNAPSHOT"
java.sourceCompatibility = JavaVersion.VERSION_1_8
repositories {
mavenCentral()
}
dependencies {
runtime("org.postgresql:postgresql:42.1.4")
implementation("org.springframework.boot:spring-boot-starter")
implementation("org.jetbrains.kotlin:kotlin-reflect")
implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8")
testImplementation("org.springframework.boot:spring-boot-starter-test") {
exclude(group = "org.junit.vintage", module = "junit-vintage-engine")
}
}
tasks.withType<Test> {
useJUnitPlatform()
}
tasks.withType<KotlinCompile> {
kotlinOptions {
freeCompilerArgs = listOf("-Xjsr305=strict")
jvmTarget = "1.8"
}
}
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
#!/usr/bin/env sh
#
# Copyright 2015 the original author or authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
##############################################################################
##
## Gradle start up script for UN*X
##
##############################################################################
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
PRG="$0"
# Need this for relative symlinks.
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`"/$link"
fi
done
SAVED="`pwd`"
cd "`dirname \"$PRG\"`/" >/dev/null
APP_HOME="`pwd -P`"
cd "$SAVED" >/dev/null
APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
warn () {
echo "$*"
}
die () {
echo
echo "$*"
echo
exit 1
}
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
nonstop=false
case "`uname`" in
CYGWIN* )
cygwin=true
;;
Darwin* )
darwin=true
;;
MINGW* )
msys=true
;;
NONSTOP* )
nonstop=true
;;
esac
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD="java"
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
MAX_FD_LIMIT=`ulimit -H -n`
if [ $? -eq 0 ] ; then
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
MAX_FD="$MAX_FD_LIMIT"
fi
ulimit -n $MAX_FD
if [ $? -ne 0 ] ; then
warn "Could not set maximum file descriptor limit: $MAX_FD"
fi
else
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
fi
fi
# For Darwin, add options to specify how the application appears in the dock
if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi
# For Cygwin or MSYS, switch paths to Windows format before running java
if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"`
# We build the pattern for arguments to be converted via cygpath
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
SEP=""
for dir in $ROOTDIRSRAW ; do
ROOTDIRS="$ROOTDIRS$SEP$dir"
SEP="|"
done
OURCYGPATTERN="(^($ROOTDIRS))"
# Add a user-defined pattern to the cygpath arguments
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
fi
# Now convert the arguments - kludge to limit ourselves to /bin/sh
i=0
for arg in "$@" ; do
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
else
eval `echo args$i`="\"$arg\""
fi
i=`expr $i + 1`
done
case $i in
0) set -- ;;
1) set -- "$args0" ;;
2) set -- "$args0" "$args1" ;;
3) set -- "$args0" "$args1" "$args2" ;;
4) set -- "$args0" "$args1" "$args2" "$args3" ;;
5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
# Escape application args
save () {
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
echo " "
}
APP_ARGS=`save "$@"`
# Collect all arguments for the java command, following the shell quoting and substitution rules
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
exec "$JAVACMD" "$@"
package com.example.demo
import org.springframework.boot.autoconfigure.SpringBootApplication
import org.springframework.boot.runApplication
@SpringBootApplication
class DemoApplication
fun main(args: Array<String>) {
runApplication<DemoApplication>(*args)
}
package com.example.demo
import org.junit.jupiter.api.Test
import org.springframework.boot.test.context.SpringBootTest
@SpringBootTest
class DemoApplicationTests {
@Test
fun contextLoads() {
}
}
/*
* Copyright 2007-present the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.net.*;
import java.io.*;
import java.nio.channels.*;
import java.util.Properties;
public class MavenWrapperDownloader {
private static final String WRAPPER_VERSION = "0.5.6";
private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
private static final String MAVEN_WRAPPER_PROPERTIES_PATH = ".mvn/wrapper/maven-wrapper.properties";
private static final String MAVEN_WRAPPER_JAR_PATH = ".mvn/wrapper/maven-wrapper.jar";
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
public static void main(String args[]) {
System.out.println("- Downloader started");
File baseDirectory = new File(args[0]);
System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
String url = DEFAULT_DOWNLOAD_URL;
if(mavenWrapperPropertyFile.exists()) {
FileInputStream mavenWrapperPropertyFileInputStream = null;
try {
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
Properties mavenWrapperProperties = new Properties();
mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
} catch (IOException e) {
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
} finally {
try {
if(mavenWrapperPropertyFileInputStream != null) {
mavenWrapperPropertyFileInputStream.close();
}
} catch (IOException e) {