Isolate license_management ruby from project

* Target ruby version 2.7.1
* Add spec to fetch gems from a custom source
* Add proxy to rubygems.org config
* Specify default env vars to support offline environment
* Cleanup custom certificates after spec
* Inline docker-test script
* Do not install license_finder with each installed ruby
* Increase gem log verbosity and include backtrace
* Extract test fixtures for the different ruby scenarios
* Find *.gemspec files in gems dir
* Use RUBYLIB to hijack src path
* Run scan from project path dir

.dockerignore

-bin
+coverage
 Dockerfile
 .dockerignore
 .git*
+pkg
 spec
 tags
 tmp
-config/.env*
+vendor

.gitignore

@@ -5,3 +5,4 @@ Dockerfile.env
 pkg
 tmp
 coverage
+vendor

.gitlab-ci.yml

@@ -4,7 +4,6 @@ variables:
   DOCKER_DRIVER: overlay2
   GIT_DEPTH: "1"
   GIT_STRATEGY: fetch
-  LATEST_IMAGE: registry.gitlab.com/gitlab-org/security-products/license-management:latest
   MAJOR: 3
   TMP_IMAGE: $CI_REGISTRY_IMAGE/license-finder:$CI_COMMIT_SHA
 

.gitlab/build.yml

@@ -22,11 +22,8 @@ build-docker-image:
   services:
     - docker:stable-dind
   script:
-    - docker info
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-    - docker pull $LATEST_IMAGE || true
-    - docker build --cache-from $LATEST_IMAGE -t $TMP_IMAGE .
-    - docker image inspect $TMP_IMAGE --format='{{.Size}}'
+    - IMAGE_NAME="$TMP_IMAGE" bin/docker-build
     - docker push $TMP_IMAGE
 
 build-mvn-pkg:

.gitlab/test.yml

@@ -22,7 +22,7 @@ lint:
   stage: test
   image: ruby:alpine
   script:
-    - apk add build-base shellcheck
+    - apk add bash build-base git shellcheck
     - bin/setup
     - bin/lint
   needs: []
@@ -30,17 +30,16 @@ lint:
 .rspec:
   stage: test
   script:
-    - bash -lc './bin/test $RSPEC_DIR --format RspecJunitFormatter --out rspec.xml'
+    - ./bin/setup
+    - ./bin/test $RSPEC_DIR --format RspecJunitFormatter --out rspec.xml
   variables:
-    BUNDLE_JOBS: '2'
-    BUNDLE_PATH: 'vendor/bundle'
     GIT_DEPTH: "10"
     GIT_STRATEGY: fetch
     LOG_LEVEL: debug
   cache:
     key: ${CI_COMMIT_REF_SLUG}
     paths:
-      - vendor/bundle
+      - vendor
   artifacts:
     paths:
       - coverage/coverage.xml

.rubocop.yml

@@ -6,10 +6,14 @@ require:
   - rubocop-rspec
 
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.7
   Exclude:
     - 'tmp/**/*'
     - 'spec/fixtures/**/*'
+    - 'vendor/**/*'
+
+Cop/GemFetcher:
+  Enabled: false
 
 Naming/ClassAndModuleCamelCase:
   Exclude:
@@ -21,3 +25,6 @@ Layout/IndentFirstArrayElement:
 
 Layout/IndentFirstHashElement:
   EnforcedStyle: consistent
+
+Rails/SkipsModelValidations:
+  Enabled: false

CHANGELOG.md

 # GitLab License management changelog
 
+## v3.17.0
+
+- Isolate the embedded LicenseFinder Ruby from the target project's Ruby (!181)
+
 ## v3.16.0
 
 - Install `dotnet` and `mono` at scan time to decrease size of Docker image. (!185)

Dockerfile

-FROM debian:stable-slim AS gem-builder
-ENV LM_HOME=/opt/license-management
-WORKDIR $LM_HOME
-COPY exe exe/
-COPY lib lib/
-COPY *.gemspec ./
-COPY *.json ./
-COPY *.md ./
-COPY *.yml ./
-RUN apt-get update -q \
-  && apt-get install -y --no-install-recommends ruby \
-  && gem build *.gemspec
+# syntax = docker/dockerfile:experimental
+FROM debian:stable AS deb-builder
+WORKDIR /build
+COPY . ./
+RUN ./bin/omnibus setup
+RUN ./bin/omnibus build license_management
 
-# Install org.codehaus.mojo:license-maven-plugin to $HOME/.m2/repository
-# Install gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin to $HOME/.m2/repository
-FROM debian:stable AS license-maven-plugin-builder
-RUN apt-get update -q \
-  && apt-get install -y --no-install-recommends maven \
-  && mvn license:license-list \
-  && mvn dependency:get -Dartifact=gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin:0.15.0 -DremoteRepositories=https://plugins.gradle.org/m2 \
-  && mvn dependency:get -Dartifact=org.codehaus.plexus:plexus-utils:2.0.6
-
-FROM debian:stable-slim as tools-builder
+FROM debian:stable-slim
 ENV ASDF_DATA_DIR="/opt/asdf"
-ENV HOME=/root
-ENV PATH="${ASDF_DATA_DIR}/shims:${ASDF_DATA_DIR}/bin:${HOME}/.local/bin:${PATH}"
+ENV PATH="${ASDF_DATA_DIR}/shims:${ASDF_DATA_DIR}/bin:/opt/gitlab/.local/bin:${PATH}"
 ENV TERM="xterm"
-WORKDIR $HOME
-COPY config /root
+WORKDIR /opt/gitlab
 COPY config/01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc
-RUN bash /root/install.sh
-
-FROM tools-builder
-ENV LM_HOME=/opt/license-management
-COPY --from=license-maven-plugin-builder /root/.m2/repository /root/.m2/repository
-COPY --from=gem-builder /opt/license-management/*.gem $LM_HOME/pkg/
+RUN mkdir -p /opt/toolcache
+COPY --from=deb-builder /build/pkg/*.deb /opt/toolcache/
+COPY config/install.sh /opt/install.sh
+RUN bash /opt/install.sh
 COPY run.sh /
 ENTRYPOINT ["/run.sh"]

Gemfile

 source 'https://rubygems.org'
 
 gemspec
+gem 'omnibus-software', git: 'https://github.com/chef/omnibus-software.git'

Gemfile.lock

+GIT
+  remote: https://github.com/chef/omnibus-software.git
+  revision: 2cf96c6c07de7d05ded6b45a0531feb10ae7cd9e
+  specs:
+    omnibus-software (4.0.0)
+      omnibus (>= 5.6.1)
+
 PATH
   remote: .
   specs:
-    license-management (3.16.0)
+    license-management (3.17.0)
       license_finder (~> 6.6.0)
 
 GEM
@@ -10,28 +17,109 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.0)
+    awesome_print (1.8.0)
+    aws-eventstream (1.1.0)
+    aws-partitions (1.336.0)
+    aws-sdk-core (3.102.1)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sdk-kms (1.35.0)
+      aws-sdk-core (~> 3, >= 3.99.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-s3 (1.72.0)
+      aws-sdk-core (~> 3, >= 3.102.1)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.2.1)
+      aws-eventstream (~> 1, >= 1.0.2)
     byebug (11.1.3)
+    chef-cleanroom (1.0.2)
+    chef-config (16.2.50)
+      addressable
+      chef-utils (= 16.2.50)
+      fuzzyurl
+      mixlib-config (>= 2.2.12, < 4.0)
+      mixlib-shellout (>= 2.0, < 4.0)
+      tomlrb (~> 1.2)
+    chef-sugar (5.1.9)
+    chef-utils (16.2.50)
+    citrus (3.0.2)
     diff-lcs (1.3)
     docile (1.3.2)
+    ffi (1.13.1)
+    ffi-yajl (2.3.3)
+      libyajl2 (~> 1.2)
+    fuzzyurl (0.9.0)
     gitlab-styles (3.1.0)
       rubocop (~> 0.74.0)
       rubocop-gitlab-security (~> 0.1.0)
       rubocop-performance (~> 1.4.1)
       rubocop-rails (~> 2.0)
       rubocop-rspec (~> 1.36)
+    iostruct (0.0.4)
+    ipaddress (0.8.3)
     jaro_winkler (1.5.4)
+    jmespath (1.4.0)
     json-schema (2.8.1)
       addressable (>= 2.4)
-    license_finder (6.6.1)
+    libyajl2 (1.2.0)
+    license_finder (6.6.2)
       bundler
       rubyzip (>= 1, < 3)
       thor (~> 1.0.1)
       tomlrb (~> 1.3.0)
       with_env (= 1.1.0)
       xml-simple (~> 1.1.5)
+    license_scout (1.1.8)
+      ffi-yajl (~> 2.2)
+      mixlib-shellout (>= 2.2, < 4.0)
+      toml-rb (>= 1, < 3)
+    mixlib-cli (2.1.6)
+    mixlib-config (3.0.6)
+      tomlrb
+    mixlib-log (3.0.8)
+    mixlib-shellout (3.0.9)
+    mixlib-versioning (1.2.12)
+    multipart-post (2.1.1)
+    ohai (16.2.1)
+      chef-config (>= 12.8, < 17)
+      chef-utils (>= 16.0, < 17)
+      ffi (~> 1.9)
+      ffi-yajl (~> 2.2)
+      ipaddress
+      mixlib-cli (>= 1.7.0)
+      mixlib-config (>= 2.0, < 4.0)
+      mixlib-log (>= 2.0.1, < 4.0)
+      mixlib-shellout (>= 2.0, < 4.0)
+      plist (~> 3.1)
+      systemu (~> 2.6.4)
+      wmi-lite (~> 1.0)
+    omnibus (7.0.13)
+      aws-sdk-s3 (~> 1)
+      chef-cleanroom (~> 1.0)
+      chef-sugar (>= 3.3)
+      ffi-yajl (~> 2.2)
+      license_scout (~> 1.0)
+      mixlib-shellout (>= 2.0, < 4.0)
+      mixlib-versioning
+      ohai (>= 13, < 17)
+      pedump
+      ruby-progressbar (~> 1.7)
+      thor (>= 0.18, < 2.0)
     parallel (1.19.1)
     parser (2.7.0.4)
       ast (~> 2.4.0)
+    pedump (0.5.4)
+      awesome_print
+      iostruct (>= 0.0.4)
+      multipart-post (>= 2.0.0)
+      progressbar
+      rainbow
+      zhexdump (>= 0.0.2)
+    plist (3.5.0)
+    progressbar (1.10.1)
     public_suffix (4.0.3)
     rack (2.2.2)
     rainbow (3.0.0)
@@ -74,11 +162,16 @@ GEM
     simplecov-cobertura (1.3.1)
       simplecov (~> 0.8)
     simplecov-html (0.12.2)
+    systemu (2.6.5)
     thor (1.0.1)
+    toml-rb (2.0.1)
+      citrus (~> 3.0, > 3.0)
     tomlrb (1.3.0)
     unicode-display_width (1.6.1)
     with_env (1.1.0)
+    wmi-lite (1.0.5)
     xml-simple (1.1.5)
+    zhexdump (0.0.2)
 
 PLATFORMS
   ruby
@@ -88,6 +181,8 @@ DEPENDENCIES
   gitlab-styles (~> 3.1)
   json-schema (~> 2.8)
   license-management!
+  omnibus (~> 7.0)
+  omnibus-software!
   rspec (~> 3.9)
   rspec_junit_formatter (~> 0.4)
   simplecov (~> 0.18)

README.md

@@ -44,7 +44,7 @@ You can run the tests from inside a docker container:
 ```sh
 $ ./bin/docker-build
 $ ./bin/docker-shell
-$ cd /opt/license-management/
+$ ./bin/setup
 $ ./bin/test
 ```
 
@@ -54,12 +54,11 @@ following these steps:
 ```sh
 $ ./bin/docker-build
 $ ./bin/docker-shell
-$ cd /opt/license-management/
 $ enable_dev_mode
 $ bundle open license_finder
 ```
 
-The `docker-shell` script will mount the current project as a volume into `/opt/license-management`.
+The `docker-shell` script will mount the current project as a volume into `/builds/gitlab-org/security-products/license-management`.
 This allows you to edit code from your host machine using your preferred editor and
 see the affect of those changes from within the running docker container.
 

bin/docker-build

@@ -6,10 +6,10 @@ cd "$(dirname "$0")/.."
 
 LATEST_IMAGE=${LATEST_IMAGE:='registry.gitlab.com/gitlab-org/security-products/license-management:latest'}
 IMAGE_NAME=${IMAGE_NAME:-$(basename "$PWD"):latest}
+export DOCKER_BUILDKIT=1
 
 if command -v docker; then
-  docker pull $LATEST_IMAGE
-  docker build --network=host --cache-from "$LATEST_IMAGE" -t "$IMAGE_NAME" .
+  docker build --progress=plain --network=host --cache-from "$LATEST_IMAGE" -t "$IMAGE_NAME" .
 else
   echo "Install docker: https://docs.docker.com/engine/installation/"
   exit 1

bin/docker-shell

@@ -9,5 +9,5 @@ IMAGE_NAME=${IMAGE_NAME:-$(basename "$PWD"):latest}
 docker run --rm -it \
   --entrypoint='' \
   --network=host \
-  --volume "$PWD":/opt/license-management \
-  "$IMAGE_NAME" /bin/bash -l
+  --volume "$PWD":/builds/gitlab-org/security-products/license-management \
+  "$IMAGE_NAME" sh -c 'cd /builds/gitlab-org/security-products/license-management && exec bash -l'

bin/lint

@@ -5,8 +5,8 @@ set -e
 cd "$(dirname "$0")/.."
 
 shellcheck bin/*
-shellcheck config/.bashrc
-shellcheck config/.profile
+shellcheck config/files/.bashrc
+shellcheck config/files/.profile
 shellcheck config/*.sh
 shellcheck run.sh
 bundle exec rubocop

bin/omnibus

+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")/.."
+
+case $1 in
+  setup)
+    apt-get update -q
+    apt-get install -y --no-install-recommends \
+      autoconf \
+      automake \
+      bison \
+      build-essential \
+      ca-certificates \
+      cmake \
+      curl \
+      fakeroot \
+      gettext \
+      git \
+      libdb-dev \
+      libffi-dev \
+      libgdbm-dev \
+      libgdbm6 \
+      libncurses5-dev \
+      libreadline6-dev \
+      libssl-dev \
+      libtool \
+      libtool-bin \
+      libyaml-dev \
+      maven \
+      python3 \
+      ruby \
+      ruby-dev \
+      zlib1g-dev \
+      zstd
+
+    gem install bundler
+    bundle install
+    ;;
+
+  build)
+    for i in "$@"; do :; done
+    bundle exec omnibus build -l debug "$i"
+    ;;
+esac

bin/setup

-#!/bin/sh
+#!/bin/bash -l
 
 set -e
 
 cd "$(dirname "$0")/.."
 
-gem install bundler --conservative -v '~> 2.0' -q
-bundle install --quiet
+export PATH="/builds/gitlab-org/security-products/license-management/exe:/opt/gitlab/embedded/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
 if [ ! -f /usr/sbin/haproxy ] && command -v apt-get; then
   apt-get update -y && apt-get install -y --no-install-recommends haproxy
 fi
+
+[[ -z "$CI_JOB_ID" ]] && enable_dev_mode
+bundle config --local path vendor
+bundle config --local jobs "$(nproc)"
+bundle install

bin/test

-#!/bin/sh
+#!/bin/bash -l
 
 set -e
 
 cd "$(dirname "$0")/.."
 
-./bin/setup
+export PATH="/builds/gitlab-org/security-products/license-management/exe:/opt/gitlab/embedded/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+export RUBYLIB="/builds/gitlab-org/security-products/license-management/lib"
+
+if ! command -v bundle; then
+  ./bin/setup
+fi
 bundle exec rspec "$@" --format=progress --tag ~environment:offline

config/.gemrc

-:verbose: true
-:sources:
-- https://rubygems.org/
-gem: --no-document

config/.bashrc → config/files/.bashrc

 #!/bin/bash
 
+export ASDF_DATA_DIR="/opt/asdf"
+export PATH="${ASDF_DATA_DIR}/shims:${ASDF_DATA_DIR}/bin:/opt/gitlab/.local/bin:${PATH}"
+export HOME="/opt/gitlab"
+
 alias nuget='mono /usr/local/bin/nuget.exe'
 set -o vi
 
+
 function inflate() {
   local file=$1
   local to_dir=$2
@@ -35,7 +40,7 @@ function switch_to() {
   local tool=$1
   local major_version=$2
   local version
-  version="$(grep "$tool" "$HOME/.tool-versions"| tr ' ' '\n' | grep "^$major_version")"
+  version="$(grep "$tool" "/opt/gitlab/.tool-versions"| tr ' ' '\n' | grep "^$major_version")"
 
   switch_to_exact "$tool" "$version"
 }

config/.default-gems → config/files/.default-gems

 bundler ~>1.7
 bundler ~>2.0
-license_finder ~>6.6.0

config/files/.gemrc

+backtrace: true
+benchmark: false
+gem: --no-ri --no-rdoc --no-document --suggestions
+verbose: true

config/.profile → config/files/.profile

 #!/bin/sh
 # shellcheck source=/dev/null
-. "$HOME/.bashrc"
+. "/opt/gitlab/.bashrc"

config/install.sh

@@ -81,10 +81,14 @@ wget -q -O /etc/apt/sources.list.d/microsoft-prod.list https://packages.microsof
 apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
 echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list
 
-curl -o /usr/local/bin/nuget.exe https://dist.nuget.org/win-x86-commandline/latest/nuget.exe &
+curl -o /usr/local/bin/nuget.exe https://dist.nuget.org/win-x86-commandline/latest/nuget.exe
 echo -e "section_end:$(date +%s):install_dotnet\r\e[0K"
 
 echo -e "section_start:$(date +%s):install_asdf\r\e[0K==> Installing asdf…"
+dpkg --install "$(find /opt/toolcache/ -name "license-management*.deb")"
+rm -fr /root
+ln -s /opt/gitlab /root
+
 mkdir -p "$ASDF_DATA_DIR"
 git clone https://github.com/asdf-vm/asdf.git "$ASDF_DATA_DIR"
 cd "$ASDF_DATA_DIR"
@@ -96,7 +100,7 @@ git checkout "$(git describe --abbrev=0 --tags)"
 while IFS= read -r line; do
   tool=$(echo "$line" | cut -d' ' -f1)
   asdf plugin-add "$tool"
-done < "$HOME/.tool-versions"
+done < "/opt/gitlab/.tool-versions"
 bash "$ASDF_DATA_DIR/plugins/nodejs/bin/import-release-team-keyring"
 asdf install
 asdf reshim
@@ -104,7 +108,7 @@ asdf current
 
 for version in $(asdf list python); do
   asdf shell python "$version"
-  pip download -d "$HOME/.config/virtualenv/app-data" pip-licenses pip setuptools wheel
+  pip download -d "/opt/gitlab/.config/virtualenv/app-data" pip-licenses pip setuptools wheel
 done
 wait
 echo -e "section_end:$(date +%s):install_asdf\r\e[0K"
@@ -127,11 +131,11 @@ rm -fr "$ASDF_DATA_DIR/docs" \
   "$ASDF_DATA_DIR"/installs/ruby/**/lib/ruby/gems/**/cache \
   "$ASDF_DATA_DIR"/installs/**/**/share \
   "$ASDF_DATA_DIR"/test \
-  "$HOME"/.config/configstore/update-notifier-npm.json \
-  "$HOME"/.config/pip/selfcheck.json \
-  "$HOME"/.gem \
-  "$HOME"/.npm \
-  "$HOME"/.wget-hsts \
+  /opt/gitlab/.config/configstore/update-notifier-npm.json \
+  /opt/gitlab/.config/pip/selfcheck.json \
+  /opt/gitlab/.gem \
+  /opt/gitlab/.npm \
+  /opt/gitlab/.wget-hsts \
   /etc/apache2/* \
   /etc/bash_completion.d/* \
   /etc/calendar/* \
@@ -186,7 +190,5 @@ wait
 rm -fr \
   /opt/asdf/ \
   /usr/lib/gcc \
-  /usr/lib/mono \
-  /usr/lib/rustlib \
-  /usr/share/dotnet
+  /usr/lib/rustlib
 echo -e "section_end:$(date +%s):compress_files\r\e[0K"

config/projects/license_management.rb

+require_relative '../../lib/license/management/version.rb'
+name "license-management"
+maintainer "mkhan@gitlab.com"
+homepage "https://gitlab.com/gitlab-org/security-products/license-management"
+license_file "LICENSE"
+