Merge branch 'go-modules-subdir' into 'master'

Scan sub directories of Golang projects

Closes gitlab-org/gitlab#255605

See merge request gitlab-org/security-products/license-management!222

.gitlab-ci.yml

@@ -6,7 +6,7 @@ variables:
   GIT_STRATEGY: fetch
   MAJOR: 3
   TMP_IMAGE: $CI_REGISTRY_IMAGE/license-finder:$CI_COMMIT_SHA
-  
+
 include:
   - template: Container-Scanning.gitlab-ci.yml
   - template: Dependency-Scanning.gitlab-ci.yml
@@ -31,6 +31,8 @@ default:
     name: $TMP_IMAGE
     entrypoint: [""]
   tags: [gitlab-org]
+  retry:
+    max: 2
 
 container_scanning:
   variables:

.gitlab/build.yml

@@ -13,6 +13,8 @@ build-conan-pkg:
     CONAN_LOGIN_USERNAME: 'ci_user'
     CONAN_PASSWORD: $CI_JOB_TOKEN
   allow_failure: true
+  dependencies: []
+  needs: []
 
 build-docker-image:
   image: docker:stable
@@ -32,3 +34,5 @@ build-mvn-pkg:
   script:
     - cd spec/fixtures/java/maven/example/ && mvn deploy -s settings.xml
   allow_failure: true
+  dependencies: []
+  needs: []

.gitlab/deb.yml

@@ -7,6 +7,7 @@
   artifacts:
     paths:
       - pkg/
+    expire_in: 1 day
   cache:
     key: ${CI_JOB_NAME}
     paths:

.gitlab/deploy.yml

@@ -13,6 +13,7 @@
       docker pull $SOURCE_IMAGE
       docker tag $SOURCE_IMAGE $TARGET_IMAGE
     - docker push $TARGET_IMAGE
+  dependencies: []
 
 latest:
   extends: .docker_tag

.gitlab/test.yml

@@ -17,6 +17,7 @@ size:
     - echo $CURRENT_SIZE
     - test "$MAX_SIZE" -gt "$CURRENT_SIZE"
   needs: ['build-docker-image']
+  dependencies: []
 
 lint:
   stage: test
@@ -26,6 +27,7 @@ lint:
     - bin/setup
     - bin/lint
   needs: []
+  dependencies: []
 
 .rspec:
   stage: test
@@ -43,10 +45,12 @@ lint:
     paths:
       - coverage/coverage.xml
       - rspec.xml
+    expire_in: 1 week
     reports:
       cobertura: coverage/coverage.xml
       junit: rspec.xml
   needs: ['build-docker-image']
+  dependencies: []
 
 unit:
   extends: .rspec

CHANGELOG.md

 # GitLab License management changelog
 
+## v3.26.1
+
+- Switch to working directory that contains the `go.mod` file. !222
+
 ## v3.26.0
 
 - Add Ruby version 2.7.2 to Docker image. !226

Gemfile.lock

@@ -8,7 +8,7 @@ GIT
 PATH
   remote: .
   specs:
-    license-management (3.26.0)
+    license-management (3.26.1)
       license_finder (~> 6.7)
 
 GEM

lib/license/finder/ext/go_modules.rb

@@ -15,8 +15,10 @@ module LicenseFinder
     end
 
     def current_packages
-      modules = vendored? ? parse_go_sum : go_list_all
-      modules.map { |hash| map_from(hash) }.compact
+      within_project_path do
+        modules = vendored? ? parse_go_sum : go_list_all
+        modules.map { |hash| map_from(hash) }.compact
+      end
     end
 
     private

lib/license/management/tool_box.rb

@@ -16,8 +16,10 @@ module License
         Dir.chdir(project_path) do
           deb = deb_for(tool, version)
           if deb&.exist?
+            ::License::Management.logger.error("Installing #{deb} ...")
             shell.execute([:dpkg, '-i', deb])
           else
+            ::License::Management.logger.error("Installing #{version} via asdf ...")
             shell.execute([:asdf, "plugin-update", tool.to_s], env: env)
             shell.execute(['/opt/asdf/plugins/nodejs/bin/import-release-team-keyring']) if tool == :nodejs
           end

lib/license/management/version.rb

@@ -2,6 +2,6 @@
 
 module License
   module Management
-    VERSION = '3.26.0'
+    VERSION = '3.26.1'
   end
 end

spec/fixtures/go/1.15-subdir/src/go.mod

+module example
+
+go 1.15
+
+require (
+	github.com/julienschmidt/httprouter v1.3.0
+	github.com/urfave/cli v1.22.4
+	go.uber.org/zap v1.16.0
+)

spec/fixtures/go/1.15-subdir/src/go.sum

+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA=
+github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
+go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A=
+go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM=
+go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=

spec/fixtures/python/pipenv/python-3.4/Pipfile

+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+docutils = "==0.13.1"
+
+[requires]
+python_version = "3.4"

spec/fixtures/python/pipenv/python-3.4/Pipfile.lock

+{
+  "_meta": {
+    "hash": {
+      "sha256": "ec82d5e7c10fd591aeebbc9b7b62d730f7fd70dc52e4e4818834891aa4194c73"
+    },
+    "pipfile-spec": 6,
+    "requires": {
+      "python_version": "3.4"
+    },
+    "sources": [
+      {
+        "name": "pypi",
+        "url": "https://pypi.org/simple",
+        "verify_ssl": true
+      }
+    ]
+  },
+  "default": {
+    "docutils": {
+      "hashes": [
+        "sha256:718c0f5fb677be0f34b781e04241c4067cbd9327b66bdd8e763201130f5175be",
+        "sha256:cb3ebcb09242804f84bdbf0b26504077a054da6772c6f4d625f335cc53ebf94d",
+        "sha256:de454f1015958450b72641165c08afe7023cd7e3944396448f2fb1b0ccba9d77"
+      ],
+      "index": "pypi",
+      "version": "==0.13.1"
+    }
+  },
+  "develop": {}
+}

spec/integration/go/modules_spec.rb

@@ -138,4 +138,52 @@ RSpec.describe "modules" do
       specify { expect(subject).to match_schema }
     end
   end
+
+  context "when scanning a go.mod file located in a sub directory" do
+    let(:env) { { 'LICENSE_FINDER_CLI_OPTS' => '--recursive' } }
+
+    before do
+      runner.mount(dir: fixture_file('go/1.15-subdir'))
+    end
+
+    it 'produces the proper report' do
+      expect(subject).to match_schema
+      expect(subject.dependency_names).to match_array([
+        "github.com/BurntSushi/toml",
+        "github.com/cpuguy83/go-md2man/v2",
+        "github.com/davecgh/go-spew",
+        "github.com/google/renameio",
+        "github.com/julienschmidt/httprouter",
+        "github.com/kisielk/gotool",
+        "github.com/kr/pretty",
+        "github.com/kr/pty",
+        "github.com/kr/text",
+        "github.com/pkg/errors",
+        "github.com/pmezard/go-difflib",
+        "github.com/rogpeppe/go-internal",
+        "github.com/russross/blackfriday/v2",
+        "github.com/shurcooL/sanitized_anchor_name",
+        "github.com/stretchr/objx",
+        "github.com/stretchr/testify",
+        "github.com/urfave/cli",
+        "go.uber.org/atomic",
+        "go.uber.org/multierr",
+        "go.uber.org/tools",
+        "go.uber.org/zap",
+        "golang.org/x/crypto",
+        "golang.org/x/lint",
+        "golang.org/x/mod",
+        "golang.org/x/net",
+        "golang.org/x/sync",
+        "golang.org/x/sys",
+        "golang.org/x/text",
+        "golang.org/x/tools",
+        "golang.org/x/xerrors",
+        "gopkg.in/check.v1",
+        "gopkg.in/errgo.v2",
+        "gopkg.in/yaml.v2",
+        "honnef.co/go/tools"
+      ])
+    end
+  end
 end

spec/integration/python/pipenv_spec.rb

@@ -239,4 +239,16 @@ RSpec.describe "pipenv" do
       expect(subject.dependency_names).to match_array(%w[Django docutils pytz requests])
     end
   end
+
+  context "when scanning a Python 3.4 project" do
+    before do
+      runner.mount(dir: fixture_file('python/pipenv/python-3.4/'))
+    end
+
+    specify do
+      expect(subject).to match_schema
+      expect(subject.dependency_names).to match_array(%w[docutils])
+      expect(subject.licenses_for('docutils')).to match_array(['public domain, python, 2-clause bsd, gpl 3 (see copying.txt)'])
+    end
+  end
 end