Verified Commit 36cdb004 authored by mo's avatar mo
Browse files

Isolate license_management ruby from project

* Target ruby version 2.7.1
* Add spec to fetch gems from a custom source
* Add proxy to rubygems.org config
* Specify default env vars to support offline environment
* Cleanup custom certificates after spec
* Inline docker-test script
* Do not install license_finder with each installed ruby
* Increase gem log verbosity and include backtrace
* Extract test fixtures for the different ruby scenarios
* Find *.gemspec files in gems dir
* Use RUBYLIB to hijack src path
* Run scan from project path dir
parent c7385965
bin
coverage
Dockerfile
.dockerignore
.git*
pkg
spec
tags
tmp
config/.env*
vendor
......@@ -5,3 +5,4 @@ Dockerfile.env
pkg
tmp
coverage
vendor
......@@ -4,7 +4,6 @@ variables:
DOCKER_DRIVER: overlay2
GIT_DEPTH: "1"
GIT_STRATEGY: fetch
LATEST_IMAGE: registry.gitlab.com/gitlab-org/security-products/license-management:latest
MAJOR: 3
TMP_IMAGE: $CI_REGISTRY_IMAGE/license-finder:$CI_COMMIT_SHA
......
......@@ -22,11 +22,8 @@ build-docker-image:
services:
- docker:stable-dind
script:
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker pull $LATEST_IMAGE || true
- docker build --cache-from $LATEST_IMAGE -t $TMP_IMAGE .
- docker image inspect $TMP_IMAGE --format='{{.Size}}'
- IMAGE_NAME="$TMP_IMAGE" bin/docker-build
- docker push $TMP_IMAGE
build-mvn-pkg:
......
......@@ -22,7 +22,7 @@ lint:
stage: test
image: ruby:alpine
script:
- apk add build-base shellcheck
- apk add bash build-base git shellcheck
- bin/setup
- bin/lint
needs: []
......@@ -30,17 +30,16 @@ lint:
.rspec:
stage: test
script:
- bash -lc './bin/test $RSPEC_DIR --format RspecJunitFormatter --out rspec.xml'
- ./bin/setup
- ./bin/test $RSPEC_DIR --format RspecJunitFormatter --out rspec.xml
variables:
BUNDLE_JOBS: '2'
BUNDLE_PATH: 'vendor/bundle'
GIT_DEPTH: "10"
GIT_STRATEGY: fetch
LOG_LEVEL: debug
cache:
key: ${CI_COMMIT_REF_SLUG}
paths:
- vendor/bundle
- vendor
artifacts:
paths:
- coverage/coverage.xml
......
......@@ -6,10 +6,14 @@ require:
- rubocop-rspec
AllCops:
TargetRubyVersion: 2.4
TargetRubyVersion: 2.7
Exclude:
- 'tmp/**/*'
- 'spec/fixtures/**/*'
- 'vendor/**/*'
Cop/GemFetcher:
Enabled: false
Naming/ClassAndModuleCamelCase:
Exclude:
......@@ -21,3 +25,6 @@ Layout/IndentFirstArrayElement:
Layout/IndentFirstHashElement:
EnforcedStyle: consistent
Rails/SkipsModelValidations:
Enabled: false
# GitLab License management changelog
## v3.17.0
- Isolate the embedded LicenseFinder Ruby from the target project's Ruby (!181)
## v3.16.0
- Install `dotnet` and `mono` at scan time to decrease size of Docker image. (!185)
......
FROM debian:stable-slim AS gem-builder
ENV LM_HOME=/opt/license-management
WORKDIR $LM_HOME
COPY exe exe/
COPY lib lib/
COPY *.gemspec ./
COPY *.json ./
COPY *.md ./
COPY *.yml ./
RUN apt-get update -q \
&& apt-get install -y --no-install-recommends ruby \
&& gem build *.gemspec
# syntax = docker/dockerfile:experimental
FROM debian:stable AS deb-builder
WORKDIR /build
COPY . ./
RUN ./bin/omnibus setup
RUN ./bin/omnibus build license_management
# Install org.codehaus.mojo:license-maven-plugin to $HOME/.m2/repository
# Install gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin to $HOME/.m2/repository
FROM debian:stable AS license-maven-plugin-builder
RUN apt-get update -q \
&& apt-get install -y --no-install-recommends maven \
&& mvn license:license-list \
&& mvn dependency:get -Dartifact=gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin:0.15.0 -DremoteRepositories=https://plugins.gradle.org/m2 \
&& mvn dependency:get -Dartifact=org.codehaus.plexus:plexus-utils:2.0.6
FROM debian:stable-slim as tools-builder
FROM debian:stable-slim
ENV ASDF_DATA_DIR="/opt/asdf"
ENV HOME=/root
ENV PATH="${ASDF_DATA_DIR}/shims:${ASDF_DATA_DIR}/bin:${HOME}/.local/bin:${PATH}"
ENV PATH="${ASDF_DATA_DIR}/shims:${ASDF_DATA_DIR}/bin:/opt/gitlab/.local/bin:${PATH}"
ENV TERM="xterm"
WORKDIR $HOME
COPY config /root
WORKDIR /opt/gitlab
COPY config/01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc
RUN bash /root/install.sh
FROM tools-builder
ENV LM_HOME=/opt/license-management
COPY --from=license-maven-plugin-builder /root/.m2/repository /root/.m2/repository
COPY --from=gem-builder /opt/license-management/*.gem $LM_HOME/pkg/
RUN mkdir -p /opt/toolcache
COPY --from=deb-builder /build/pkg/*.deb /opt/toolcache/
COPY config/install.sh /opt/install.sh
RUN bash /opt/install.sh
COPY run.sh /
ENTRYPOINT ["/run.sh"]
GIT
remote: https://github.com/chef/omnibus-software.git
revision: 2cf96c6c07de7d05ded6b45a0531feb10ae7cd9e
specs:
omnibus-software (4.0.0)
omnibus (>= 5.6.1)
PATH
remote: .
specs:
license-management (3.16.0)
license-management (3.17.0)
license_finder (~> 6.6.0)
GEM
......@@ -10,28 +17,109 @@ GEM
addressable (2.7.0)
public_suffix (>= 2.0.2, < 5.0)
ast (2.4.0)
awesome_print (1.8.0)
aws-eventstream (1.1.0)
aws-partitions (1.336.0)
aws-sdk-core (3.102.1)
aws-eventstream (~> 1, >= 1.0.2)
aws-partitions (~> 1, >= 1.239.0)
aws-sigv4 (~> 1.1)
jmespath (~> 1.0)
aws-sdk-kms (1.35.0)
aws-sdk-core (~> 3, >= 3.99.0)
aws-sigv4 (~> 1.1)
aws-sdk-s3 (1.72.0)
aws-sdk-core (~> 3, >= 3.102.1)
aws-sdk-kms (~> 1)
aws-sigv4 (~> 1.1)
aws-sigv4 (1.2.1)
aws-eventstream (~> 1, >= 1.0.2)
byebug (11.1.3)
chef-cleanroom (1.0.2)
chef-config (16.2.50)
addressable
chef-utils (= 16.2.50)
fuzzyurl
mixlib-config (>= 2.2.12, < 4.0)
mixlib-shellout (>= 2.0, < 4.0)
tomlrb (~> 1.2)
chef-sugar (5.1.9)
chef-utils (16.2.50)
citrus (3.0.2)
diff-lcs (1.3)
docile (1.3.2)
ffi (1.13.1)
ffi-yajl (2.3.3)
libyajl2 (~> 1.2)
fuzzyurl (0.9.0)
gitlab-styles (3.1.0)
rubocop (~> 0.74.0)
rubocop-gitlab-security (~> 0.1.0)
rubocop-performance (~> 1.4.1)
rubocop-rails (~> 2.0)
rubocop-rspec (~> 1.36)
iostruct (0.0.4)
ipaddress (0.8.3)
jaro_winkler (1.5.4)
jmespath (1.4.0)
json-schema (2.8.1)
addressable (>= 2.4)
license_finder (6.6.1)
libyajl2 (1.2.0)
license_finder (6.6.2)
bundler
rubyzip (>= 1, < 3)
thor (~> 1.0.1)
tomlrb (~> 1.3.0)
with_env (= 1.1.0)
xml-simple (~> 1.1.5)
license_scout (1.1.8)
ffi-yajl (~> 2.2)
mixlib-shellout (>= 2.2, < 4.0)
toml-rb (>= 1, < 3)
mixlib-cli (2.1.6)
mixlib-config (3.0.6)
tomlrb
mixlib-log (3.0.8)
mixlib-shellout (3.0.9)
mixlib-versioning (1.2.12)
multipart-post (2.1.1)
ohai (16.2.1)
chef-config (>= 12.8, < 17)
chef-utils (>= 16.0, < 17)
ffi (~> 1.9)
ffi-yajl (~> 2.2)
ipaddress
mixlib-cli (>= 1.7.0)
mixlib-config (>= 2.0, < 4.0)
mixlib-log (>= 2.0.1, < 4.0)
mixlib-shellout (>= 2.0, < 4.0)
plist (~> 3.1)
systemu (~> 2.6.4)
wmi-lite (~> 1.0)
omnibus (7.0.13)
aws-sdk-s3 (~> 1)
chef-cleanroom (~> 1.0)
chef-sugar (>= 3.3)
ffi-yajl (~> 2.2)
license_scout (~> 1.0)
mixlib-shellout (>= 2.0, < 4.0)
mixlib-versioning
ohai (>= 13, < 17)
pedump
ruby-progressbar (~> 1.7)
thor (>= 0.18, < 2.0)
parallel (1.19.1)
parser (2.7.0.4)
ast (~> 2.4.0)
pedump (0.5.4)
awesome_print
iostruct (>= 0.0.4)
multipart-post (>= 2.0.0)
progressbar
rainbow
zhexdump (>= 0.0.2)
plist (3.5.0)
progressbar (1.10.1)
public_suffix (4.0.3)
rack (2.2.2)
rainbow (3.0.0)
......@@ -74,11 +162,16 @@ GEM
simplecov-cobertura (1.3.1)
simplecov (~> 0.8)
simplecov-html (0.12.2)
systemu (2.6.5)
thor (1.0.1)
toml-rb (2.0.1)
citrus (~> 3.0, > 3.0)
tomlrb (1.3.0)
unicode-display_width (1.6.1)
with_env (1.1.0)
wmi-lite (1.0.5)
xml-simple (1.1.5)
zhexdump (0.0.2)
PLATFORMS
ruby
......@@ -88,6 +181,8 @@ DEPENDENCIES
gitlab-styles (~> 3.1)
json-schema (~> 2.8)
license-management!
omnibus (~> 7.0)
omnibus-software!
rspec (~> 3.9)
rspec_junit_formatter (~> 0.4)
simplecov (~> 0.18)
......
......@@ -44,7 +44,7 @@ You can run the tests from inside a docker container:
```sh
$ ./bin/docker-build
$ ./bin/docker-shell
$ cd /opt/license-management/
$ ./bin/setup
$ ./bin/test
```
......@@ -54,12 +54,11 @@ following these steps:
```sh
$ ./bin/docker-build
$ ./bin/docker-shell
$ cd /opt/license-management/
$ enable_dev_mode
$ bundle open license_finder
```
The `docker-shell` script will mount the current project as a volume into `/opt/license-management`.
The `docker-shell` script will mount the current project as a volume into `/builds/gitlab-org/security-products/license-management`.
This allows you to edit code from your host machine using your preferred editor and
see the affect of those changes from within the running docker container.
......
......@@ -6,10 +6,10 @@ cd "$(dirname "$0")/.."
LATEST_IMAGE=${LATEST_IMAGE:='registry.gitlab.com/gitlab-org/security-products/license-management:latest'}
IMAGE_NAME=${IMAGE_NAME:-$(basename "$PWD"):latest}
export DOCKER_BUILDKIT=1
if command -v docker; then
docker pull $LATEST_IMAGE
docker build --network=host --cache-from "$LATEST_IMAGE" -t "$IMAGE_NAME" .
docker build --progress=plain --network=host --cache-from "$LATEST_IMAGE" -t "$IMAGE_NAME" .
else
echo "Install docker: https://docs.docker.com/engine/installation/"
exit 1
......
......@@ -9,5 +9,5 @@ IMAGE_NAME=${IMAGE_NAME:-$(basename "$PWD"):latest}
docker run --rm -it \
--entrypoint='' \
--network=host \
--volume "$PWD":/opt/license-management \
"$IMAGE_NAME" /bin/bash -l
--volume "$PWD":/builds/gitlab-org/security-products/license-management \
"$IMAGE_NAME" sh -c 'cd /builds/gitlab-org/security-products/license-management && exec bash -l'
......@@ -5,8 +5,8 @@ set -e
cd "$(dirname "$0")/.."
shellcheck bin/*
shellcheck config/.bashrc
shellcheck config/.profile
shellcheck config/files/.bashrc
shellcheck config/files/.profile
shellcheck config/*.sh
shellcheck run.sh
bundle exec rubocop
#!/bin/sh
set -e
cd "$(dirname "$0")/.."
case $1 in
setup)
apt-get update -q
apt-get install -y --no-install-recommends \
autoconf \
automake \
bison \
build-essential \
ca-certificates \
cmake \
curl \
fakeroot \
gettext \
git \
libdb-dev \
libffi-dev \
libgdbm-dev \
libgdbm6 \
libncurses5-dev \
libreadline6-dev \
libssl-dev \
libtool \
libtool-bin \
libyaml-dev \
maven \
python3 \
ruby \
ruby-dev \
zlib1g-dev \
zstd
gem install bundler
bundle install
;;
build)
for i in "$@"; do :; done
bundle exec omnibus build -l debug "$i"
;;
esac
#!/bin/sh
#!/bin/bash -l
set -e
cd "$(dirname "$0")/.."
gem install bundler --conservative -v '~> 2.0' -q
bundle install --quiet
export PATH="/builds/gitlab-org/security-products/license-management/exe:/opt/gitlab/embedded/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
if [ ! -f /usr/sbin/haproxy ] && command -v apt-get; then
apt-get update -y && apt-get install -y --no-install-recommends haproxy
fi
[[ -z "$CI_JOB_ID" ]] && enable_dev_mode
bundle config --local path vendor
bundle config --local jobs "$(nproc)"
bundle install
#!/bin/sh
#!/bin/bash -l
set -e
cd "$(dirname "$0")/.."
./bin/setup
export PATH="/builds/gitlab-org/security-products/license-management/exe:/opt/gitlab/embedded/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
export RUBYLIB="/builds/gitlab-org/security-products/license-management/lib"
if ! command -v bundle; then
./bin/setup
fi
bundle exec rspec "$@" --format=progress --tag ~environment:offline
:verbose: true
:sources:
- https://rubygems.org/
gem: --no-document
#!/bin/bash
export ASDF_DATA_DIR="/opt/asdf"
export PATH="${ASDF_DATA_DIR}/shims:${ASDF_DATA_DIR}/bin:/opt/gitlab/.local/bin:${PATH}"
export HOME="/opt/gitlab"
alias nuget='mono /usr/local/bin/nuget.exe'
set -o vi
function inflate() {
local file=$1
local to_dir=$2
......@@ -35,7 +40,7 @@ function switch_to() {
local tool=$1
local major_version=$2
local version
version="$(grep "$tool" "$HOME/.tool-versions"| tr ' ' '\n' | grep "^$major_version")"
version="$(grep "$tool" "/opt/gitlab/.tool-versions"| tr ' ' '\n' | grep "^$major_version")"
switch_to_exact "$tool" "$version"
}
......
bundler ~>1.7
bundler ~>2.0
license_finder ~>6.6.0
backtrace: true
benchmark: false
gem: --no-ri --no-rdoc --no-document --suggestions
verbose: true
#!/bin/sh
# shellcheck source=/dev/null
. "$HOME/.bashrc"
. "/opt/gitlab/.bashrc"
......@@ -81,10 +81,14 @@ wget -q -O /etc/apt/sources.list.d/microsoft-prod.list https://packages.microsof
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list
curl -o /usr/local/bin/nuget.exe https://dist.nuget.org/win-x86-commandline/latest/nuget.exe &
curl -o /usr/local/bin/nuget.exe https://dist.nuget.org/win-x86-commandline/latest/nuget.exe
echo -e "section_end:$(date +%s):install_dotnet\r\e[0K"
echo -e "section_start:$(date +%s):install_asdf\r\e[0K==> Installing asdf…"
dpkg --install "$(find /opt/toolcache/ -name "license-management*.deb")"
rm -fr /root
ln -s /opt/gitlab /root
mkdir -p "$ASDF_DATA_DIR"
git clone https://github.com/asdf-vm/asdf.git "$ASDF_DATA_DIR"
cd "$ASDF_DATA_DIR"
......@@ -96,7 +100,7 @@ git checkout "$(git describe --abbrev=0 --tags)"
while IFS= read -r line; do
tool=$(echo "$line" | cut -d' ' -f1)
asdf plugin-add "$tool"
done < "$HOME/.tool-versions"
done < "/opt/gitlab/.tool-versions"
bash "$ASDF_DATA_DIR/plugins/nodejs/bin/import-release-team-keyring"
asdf install
asdf reshim
......@@ -104,7 +108,7 @@ asdf current
for version in $(asdf list python); do
asdf shell python "$version"
pip download -d "$HOME/.config/virtualenv/app-data" pip-licenses pip setuptools wheel
pip download -d "/opt/gitlab/.config/virtualenv/app-data" pip-licenses pip setuptools wheel
done
wait
echo -e "section_end:$(date +%s):install_asdf\r\e[0K"
......@@ -127,11 +131,11 @@ rm -fr "$ASDF_DATA_DIR/docs" \
"$ASDF_DATA_DIR"/installs/ruby/**/lib/ruby/gems/**/cache \
"$ASDF_DATA_DIR"/installs/**/**/share \
"$ASDF_DATA_DIR"/test \
"$HOME"/.config/configstore/update-notifier-npm.json \
"$HOME"/.config/pip/selfcheck.json \
"$HOME"/.gem \
"$HOME"/.npm \
"$HOME"/.wget-hsts \
/opt/gitlab/.config/configstore/update-notifier-npm.json \
/opt/gitlab/.config/pip/selfcheck.json \
/opt/gitlab/.gem \
/opt/gitlab/.npm \
/opt/gitlab/.wget-hsts \
/etc/apache2/* \
/etc/bash_completion.d/* \
/etc/calendar/* \
......@@ -186,7 +190,5 @@ wait
rm -fr \
/opt/asdf/ \
/usr/lib/gcc \
/usr/lib/mono \
/usr/lib/rustlib \
/usr/share/dotnet
/usr/lib/rustlib
echo -e "section_end:$(date +%s):compress_files\r\e[0K"
require_relative '../../lib/license/management/version.rb'
name "license-management"
maintainer "mkhan@gitlab.com"
homepage "https://gitlab.com/gitlab-org/security-products/license-management"
license_file "LICENSE"