Install root certificate

* Install certificate in root ca trust store
* Use PIP_CERT environment variable to specify path to the certificate bundle
* Do not override user provided PIP_CERT
* Perform shallow clone when possible
* Update CHANGELOG and bump version

CHANGELOG.md

 # GitLab License management changelog
 
+## v3.7.0
+
+- Add `ADDITIONAL_CA_CERT_BUNDLE` to list of trusted root certificates. (!126)
+
 ## v3.6.0
 
 - Change default log level to `warn`. (!132)

Gemfile.lock

 PATH
   remote: .
   specs:
-    license-management (3.6.0)
+    license-management (3.7.0)
       license_finder (~> 6.0.0)
       spandx (~> 0.1)
 

README.md

@@ -108,6 +108,7 @@ The License Management tool can be customized with environments variables for so
 
 | Environment variable | Project type | Function |
 |----------------------|--------------|----------|
+| ADDITIONAL_CA_CERT_BUNDLE | * | Additional certificate chain to install in the trusted store. |
 | MAVEN_CLI_OPTS       | Java (Maven) | Additional arguments for the mvn executable. If not supplied, defaults to `-DskipTests`. |
 | LICENSE_FINDER_CLI_OPTS | * | Additional arguments for the `license_finder` executable. |
 | LM_JAVA_VERSION      | Java (Maven) | Version of Java. If set to `11`, Maven and Gradle use Java 11 instead of Java 8. |

lib/license/finder/ext/go_modules.rb

@@ -25,10 +25,6 @@ module LicenseFinder
 
     private
 
-    def shell
-      @shell ||= ::License::Management::Shell.new
-    end
-
     def absolute_project_path
       @absolute_project_path ||= Pathname(project_path).cleanpath
     end

lib/license/finder/ext/pip.rb

@@ -51,10 +51,6 @@ module LicenseFinder
       Dir.chdir(project_path) { yield }
     end
 
-    def shell
-      @shell ||= ::License::Management::Shell.new
-    end
-
     def pypi
       @pypi ||= Spandx::Python::PyPI.new(sources: [
         Spandx::Python::Source.new({

lib/license/finder/ext/pipenv.rb

@@ -5,8 +5,8 @@ module LicenseFinder
     def prepare
       return unless pipfile?
 
-      shell.execute([:pipenv, '--python', python.version])
-      shell.execute([:pipenv, :run, :pipenv, :sync, '--pypi-mirror', python.pip_index_url])
+      shell.execute([:pipenv, '--python', python.version], env: default_env)
+      shell.execute([:pipenv, :run, :pipenv, :sync, '--pypi-mirror', python.pip_index_url], env: default_env)
     end
 
     def current_packages
@@ -17,10 +17,6 @@ module LicenseFinder
 
     private
 
-    def shell
-      @shell ||= ::License::Management::Shell.new
-    end
-
     def python
       @python ||= ::License::Management::Python.new
     end
@@ -52,5 +48,12 @@ module LicenseFinder
     def lockfile_hash
       @lockfile_hash ||= JSON.parse(IO.read(detected_package_path))
     end
+
+    def default_env
+      return {} unless shell.custom_certificate_installed?
+      return {} if ENV['PIP_CERT']
+
+      { 'PIP_CERT' => shell.custom_certificate_path.to_s }
+    end
   end
 end

lib/license/finder/ext/shared_helpers.rb

@@ -2,10 +2,13 @@
 
 module LicenseFinder
   module SharedHelpers
+    def shell
+      ::License::Management.shell
+    end
+
     class Cmd
       def self.run(command)
-        @shell ||= ::License::Management::Shell.new
-        @shell.execute(command)
+        ::License::Management.shell.execute(command)
       end
     end
   end

lib/license/management.rb

@@ -26,5 +26,9 @@ module License
     def self.logger
       @logger ||= Logger.new(STDOUT, level: ENV.fetch('LOG_LEVEL', Logger::WARN))
     end
+
+    def self.shell
+      @shell ||= Shell.new
+    end
   end
 end

lib/license/management/python.rb

@@ -5,7 +5,7 @@ module License
     class Python
       attr_reader :shell
 
-      def initialize(shell: Shell.new)
+      def initialize(shell: ::License::Management.shell)
         @shell = shell
       end
 

lib/license/management/shell.rb

@@ -3,10 +3,12 @@
 module License
   module Management
     class Shell
-      attr_reader :logger
+      attr_reader :custom_certificate_path, :logger
 
-      def initialize(logger: License::Management.logger)
+      def initialize(logger: License::Management.logger, certificate: ENV['ADDITIONAL_CA_CERT_BUNDLE'])
         @logger = logger
+        @custom_certificate_path = Pathname.new('/usr/local/share/ca-certificates/custom.crt')
+        trust!(certificate)
       end
 
       def execute(command, env: {})
@@ -24,11 +26,26 @@ module License
         execute("sh -c '#{expand(command)}'", env: env)
       end
 
+      def custom_certificate_installed?
+        present?(ENV['ADDITIONAL_CA_CERT_BUNDLE']) && custom_certificate_path.exist?
+      end
+
       private
 
       def expand(command)
         Array(command).map(&:to_s).join(' ')
       end
+
+      def trust!(certificate)
+        return unless present?(certificate)
+
+        custom_certificate_path.write(certificate)
+        execute('update-ca-certificates -v')
+      end
+
+      def present?(item)
+        !item.nil? && !item.empty?
+      end
     end
   end
 end

lib/license/management/version.rb

@@ -2,6 +2,6 @@
 
 module License
   module Management
-    VERSION = '3.6.0'
+    VERSION = '3.7.0'
   end
 end

spec/fixtures/python/airgap-Pipfile.erb

+[[source]]
+name = "pypi"
+url = "<%= index_url %>"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+requests = "*"

spec/fixtures/python/airgap-Pipfile.lock.erb

+{
+  "_meta": {
+    "hash": {
+      "sha256": "5b488a008aa3a3189ebb41e224ca36bb2ca6e5d4d420ad136bb660d15fd27a14"
+    },
+    "pipfile-spec": 6,
+    "requires": {},
+    "sources": [
+      {
+        "name": "pypi",
+        "url": "<%= index_url %>",
+        "verify_ssl": true
+      }
+    ]
+  },
+  "default": {
+    "certifi": {
+      "hashes": [
+        "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3"
+      ],
+      "version": "==2019.11.28"
+    },
+    "chardet": {
+      "hashes": [
+        "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+      ],
+      "version": "==3.0.4"
+    },
+    "idna": {
+      "hashes": [
+        "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"
+      ],
+      "version": "==2.9"
+    },
+    "requests": {
+      "hashes": [
+        "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"
+      ],
+      "index": "pypi",
+      "version": "==2.23.0"
+    },
+    "urllib3": {
+      "hashes": [
+        "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc"
+      ],
+      "version": "==1.25.8"
+    }
+  },
+  "develop": {}
+}

spec/fixtures/python/pypi.crt

+-----BEGIN CERTIFICATE-----
+MIID+zCCAuOgAwIBAgIUWxGpSPKNbbHUxh32y0cGgvmSwZIwDQYJKoZIhvcNAQEL
+BQAwgYwxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQK
+DBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxRTBDBgNVBAMMPGdpdGxhYi1haXJn
+YXAtcHlwaS51cy13ZXN0MS1iLmMuZ3JvdXAtc2VjdXJlLWE4OWZlNy5pbnRlcm5h
+bDAeFw0yMDA0MDIwMTQ0MTFaFw0yMTA0MDIwMTQ0MTFaMIGMMQswCQYDVQQGEwJB
+VTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0
+cyBQdHkgTHRkMUUwQwYDVQQDDDxnaXRsYWItYWlyZ2FwLXB5cGkudXMtd2VzdDEt
+Yi5jLmdyb3VwLXNlY3VyZS1hODlmZTcuaW50ZXJuYWwwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQCsguLHmkpWR13l06HrLze+sv/4eI3jyveBc0xkUGFh
+WKHAOCPQvfHhavK2W1w2mKA69mEixkncUwU3Q5KT8bapnUYo2sks62vSfuibat1F
+ZzoII35xk71zgkXwGgEAy/h4izEubbsP0JMYIE0uZKuuytylax7KDy5Tskbh79Gq
+Ye42N1j77T6rxfB06nrENokZMb9EoOtUdW/jU4BMBGUO1ZwQHh1QOawENtGBay24
+j05HbOURZzl5SxfJbbFSeQbGqaY/ujaCDdJRRfacbmkjs+qaZ0QF/fEvxg9/xP/y
+VwGwLSzIXfkuRyw2KSmKBSdy36DLIJ/TBdmLoMgHjouhAgMBAAGjUzBRMB0GA1Ud
+DgQWBBS2dpl46jX7D8PTwLhdulc++IzBRTAfBgNVHSMEGDAWgBS2dpl46jX7D8PT
+wLhdulc++IzBRTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBd
+SN3o3SEEl3sfbnzKPzQusBbg2njpp9zmnRt26DvHWXPM2BYyBPs0YY8jF4NmwoK4
+ijbGolsOPJy/8gQ478WLizN0Mdhcqh7+R5FvbhghyO66/I6WsPvOR+XNOuaYXe4S
+Jg9dfTFOTzndTJwbTJqhe+QwM3Ns+jw8uDE3zAtsGJ1rcgXOKX8B24AvPdIttu1h
+z0ahOUbRIT5MsTxCMBZZqQTFdhBOeBiLEYUtK789NsWjb3JdKwuBeKyhnJhcPnUo
+YK3HLoDrUfpHveFsg5CWUm7euJ/HXljJm+Ct1+fHbRC1jUQ3tY8JsvbmvkXCVHXv
+f7j8RXMzKBldTaHzsVj2
+-----END CERTIFICATE-----

spec/integration/python/pipenv_spec.rb

@@ -201,4 +201,29 @@ RSpec.describe "pipenv" do
       end
     end
   end
+
+  context "when connecting to a private package repository with self signed certificate" do
+    let(:index_url) { "https://gitlab-airgap-pypi.us-west1-b.c.group-secure-a89fe7.internal/simple" }
+    let(:bundle) { fixture_file_content('python/pypi.crt') }
+
+    def install_airgap_hosts
+      add_host('gitlab-airgap-test.us-west1-b.c.group-secure-a89fe7.internal', '34.82.7.216')
+      add_host('gitlab-airgap-pypi.us-west1-b.c.group-secure-a89fe7.internal', '35.227.149.218')
+    end
+
+    before do
+      runner.add_file('Pipfile', fixture_file_content('python/airgap-Pipfile.erb', index_url: index_url))
+      runner.add_file('Pipfile.lock', fixture_file_content('python/airgap-Pipfile.lock.erb', index_url: index_url))
+    end
+
+    pending 'downloads the packages and trusts the certificate' do
+      report = runner.scan(env: {
+        'ADDITIONAL_CA_CERT_BUNDLE' => bundle,
+        'PIP_INDEX_URL' => index_url
+      })
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report.dependency_names).to include('requests')
+    end
+  end
 end

spec/support/fixture_file_helper.rb

 module FixtureFileHelper
-  def fixture_file_content(path)
-    IO.read(fixture_file(path))
+  def fixture_file_content(path, data = {})
+    content = IO.read(fixture_file(path))
+    return content unless path.end_with?('.erb')
+
+    ERB
+      .new(content)
+      .result(OpenStruct.new(data).send(:binding))
   end
 
   def fixture_file(path)

spec/support/integration_test_helper.rb

@@ -56,9 +56,13 @@ module IntegrationTestHelper
     end
 
     def clone(repo, branch: 'master')
-      execute({}, "git", "clone", '--quiet', repo, project_path)
-      Dir.chdir project_path do
-        execute({}, "git", "checkout", branch)
+      if branch.match?(/\b[0-9a-f]{5,40}\b/)
+        execute({}, 'git', 'clone', '--quiet', repo, project_path)
+        Dir.chdir project_path do
+          execute({}, 'git', 'checkout', branch)
+        end
+      else
+        execute({}, 'git', 'clone', '--quiet', '--depth=1', '--single-branch', '--branch', branch, repo, project_path)
       end
     end
 
@@ -85,4 +89,11 @@ module IntegrationTestHelper
   def runner(*args)
     @runner ||= IntegrationTestRunner.new(*args)
   end
+
+  def add_host(name, ip)
+    return unless ENV['LM_HOME']
+    return if system("grep #{name} /etc/hosts")
+
+    system("echo '#{ip} #{name}' >> /etc/hosts")
+  end
 end