Merge branch '14630-use-gemnasium-db' into 'master'

Connect to gemnasium-db repo

See merge request !25

CHANGELOG.md

 # Gemnasium analyzer changelog
 
+## v2.3.0
+- Use gemnasium-db git repo instead of the Gemnasium API (!25)
+
 ## v2.2.6
 - Fix The engine "node" is incompatible with this module. error (!22)
 

Dockerfile

 FROM node:11-alpine
-RUN apk add --no-cache git
+
+ENV PYTHON_PIP_VERSION 19.3
+ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/65986a26949050d26e6ec98915da4aade8d8679d/get-pip.py
+ENV PYTHON_GET_PIP_SHA256 8d412752ae26b46a39a201ec618ef9ef7656c5b2d8529cdcbe60cd70dc94f40c
+
+COPY vrange /vrange
+ENV VRANGE_DIR="/vrange"
+
+ARG GEMNASIUM_DB_LOCAL_PATH="/gemnasium-db"
+ARG GEMNASIUM_DB_REMOTE_URL="https://gitlab.com/gitlab-org/security-products/gemnasium-db.git"
+ARG GEMNASIUM_DB_REF_NAME="master"
+
+ENV GEMNASIUM_DB_LOCAL_PATH $GEMNASIUM_DB_LOCAL_PATH
+ENV GEMNASIUM_DB_REMOTE_URL $GEMNASIUM_DB_REMOTE_URL
+ENV GEMNASIUM_DB_REF_NAME $GEMNASIUM_DB_REF_NAME
+
+RUN \
+	# gemnasium-db
+	apk add --no-cache git && \
+	git clone --branch $GEMNASIUM_DB_REF_NAME $GEMNASIUM_DB_REMOTE_URL $GEMNASIUM_DB_LOCAL_PATH && \
+	\
+	# vrange/php dependencies
+	apk add --no-cache php7 php7-dom php7-ctype php7-tokenizer php7-xmlwriter php7-xml composer && \
+	composer install -d "$VRANGE_DIR/php" && \
+	\
+	# vrange/gem dependencies
+	apk add --no-cache git ruby ruby-json && \
+	\
+	# vrange/python dependencies
+	apk add --no-cache git python3 && \
+	wget -O get-pip.py "$PYTHON_GET_PIP_URL" && \
+	echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum -c - && \
+	python3 get-pip.py --disable-pip-version-check --no-cache-dir "pip==$PYTHON_PIP_VERSION" && \
+	pip install -r "$VRANGE_DIR/python/requirements.txt" && \
+	\
+	# vrange/npm dependencies
+	yarn --cwd "$VRANGE_DIR/npm/yarn.lock" && \
+	\
+	echo "done."
+
 COPY analyzer /
+
 ENTRYPOINT []
 CMD ["/analyzer", "run"]

advisory/advisory.go

+// Package advisory implements structs for manipulating
+// security advisories affecting project dependencies
+// as a collection of YAML files.
+package advisory
+
+// Advisory is a security advisory published for a package.
+type Advisory struct {
+	// Identifier is CVE id (preferred) or any public identifier.
+	Identifier string `yaml:"identifier,omitempty" json:"identifier,omitempty"`
+
+	// Title is a short description of the security flaw.
+	Title string `yaml:"title" json:"title"`
+
+	// Description is a long description of the security flaw and the possible risks.
+	Description string `yaml:"description" json:"description"`
+
+	// DisclosureDate is the date on which the advisory was made public, in ISO-8601 format.
+	DisclosureDate string `yaml:"date,omitempty" json:"date,omitempty"`
+
+	// AffectedRange is the range of affected versions. Machine-readable syntax used by the package manager.
+	AffectedRange string `yaml:"affected_range" json:"affected_range"`
+
+	// FixedVersions are the versions fixing the vulnerability. The order is not relevant.
+	FixedVersions []string `yaml:"fixed_versions,omitempty" json:"fixed_versions,omitempty"`
+
+	// ImpactedVersions is the range of affected versions. Human-readable version for display.
+	ImpactedVersions string `yaml:"affected_versions,omitempty" json:"affected_versions,omitempty"`
+
+	// NotImpacted describes the environments not affected by the vulnerability.
+	NotImpacted string `yaml:"not_impacted,omitempty" json:"not_impacted,omitempty"`
+
+	// Solution tells how to remediate the vulnerability.
+	Solution string `yaml:"solution,omitempty" json:"solution,omitempty"`
+
+	// Credit gives the names of the people who reported the vulnerability or helped fixing it.
+	Credit string `yaml:"credit,omitempty" json:"credit,omitempty"`
+
+	// Links are the URLs of: detailed advisory, documented exploit, vulnerable source code, etc.
+	Links []string `yaml:"urls" json:"urls"`
+
+	// Package is the affected package.
+	Package Package `yaml:"package_slug" json:"package_slug"`
+
+	// UUID is the identifier in Gemnasium DB. It's no longer used.
+	UUID string `yaml:"uuid,omitempty" json:"uuid,omitempty"`
+}

advisory/advisory_test.go

+package advisory
+
+import (
+	"os"
+	"reflect"
+	"testing"
+
+	"gopkg.in/yaml.v2"
+)
+
+func TestAdvisory_Decode(t *testing.T) {
+	tcs := []struct {
+		name string
+		path string
+		want Advisory
+	}{
+		{
+			"Minimal",
+			"testdata/gem/actionmovie/GMS-2019-minimal.yml",
+			Advisory{
+				Identifier:     "GMS-2019-minimal",
+				Title:          "Unsafe UPDATE query",
+				Description:    "There is a vulnerability in some UPDATE query.",
+				DisclosureDate: "2019-10-30",
+				AffectedRange:  "*",
+				Package: Package{
+					Type: "gem",
+					Name: "actionmovie",
+				},
+			},
+		},
+		{
+
+			"Full",
+			"testdata/gem/activerecord/CVE-2016-6317.yml",
+			Advisory{
+				Identifier:       "CVE-2016-6317",
+				Title:            "Unsafe Query Generation Risk",
+				Description:      "There is a vulnerability when Active Record is used in conjunction with JSON\r\nparameter parsing. This vulnerability is similar to CVE-2012-2660,\r\nCVE-2012-2694 and CVE-2013-0155.",
+				DisclosureDate:   "2016-08-11",
+				AffectedRange:    ">=4.0.0.alpha <4.2.7.1",
+				FixedVersions:    []string{"4.2.7.1"},
+				ImpactedVersions: "4.x",
+				NotImpacted:      "5.x, 3.x and earlier",
+				Solution:         "Upgrade to latest or use workaround; see provided link.",
+				Credit:           "joernchen of Phenoelit",
+				Links: []string{
+					"https://groups.google.com/forum/#!topic/rubyonrails-security/rgO20zYW33s",
+				},
+				Package: Package{
+					Type: "gem",
+					Name: "activerecord",
+				},
+				UUID: "51b1a6d4-3eb3-48d4-8d25-ae62c2dbb3d4",
+			},
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			f, err := os.Open(tc.path)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer f.Close()
+
+			got := Advisory{}
+			err = yaml.NewDecoder(f).Decode(&got)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if !reflect.DeepEqual(tc.want, got) {
+				t.Errorf("Wrong result. Expected %#v but got %#v", tc.want, got)
+			}
+		})
+	}
+}

advisory/error.go

+package advisory
+
+import "fmt"
+
+// ErrNoAdvisoryForPackageType is the error returned when there are no advisories
+// corresponding to a given package type.
+type ErrNoAdvisoryForPackageType struct {
+	PackageType string
+}
+
+func (err ErrNoAdvisoryForPackageType) Error() string {
+	return fmt.Sprintf("no advisory for package type %s", err.PackageType)
+}
+
+// ErrNoPackageTypeDir is the error returned when the directory corresponding
+// to a package type is missing, or cannot be read.
+type ErrNoPackageTypeDir struct {
+	PackageType string
+}
+
+func (err ErrNoPackageTypeDir) Error() string {
+	return fmt.Sprintf("cannot read directory for package type %s", err.PackageType)
+}
+
+// ErrNoPackageDir is the error returned when the directory containing
+// package advisories is missing, or cannot be read.
+type ErrNoPackageDir struct {
+	Package Package
+}
+
+func (err ErrNoPackageDir) Error() string {
+	return fmt.Sprintf("cannot read directory for package %s", err.Package.Slug())
+}

advisory/package.go

+package advisory
+
+import (
+	"strings"
+)
+
+// Package contains the information needed to resolve a package on the server side.
+type Package struct {
+	Type string `json:"type"`
+	Name string `json:"name"`
+}
+
+// Slug returns the package slug/path.
+func (p Package) Slug() string {
+	return p.Type + "/" + p.Name
+}
+
+// UnmarshalYAML decodes a package slug.
+func (p *Package) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var slug string
+	if err := unmarshal(&slug); err != nil {
+		return err
+	}
+
+	parts := strings.SplitN(slug, "/", 2)
+	if len(parts) > 1 {
+		p.Name = parts[1]
+	}
+	p.Type = parts[0]
+
+	return nil
+}

advisory/repo.go

+package advisory
+
+import (
+	"errors"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+)
+
+// defaultRefName is the git ref that is checked out by default when updating the repo.
+const defaultRefName = "master"
+
+// Repo is a local git clone of the gemnasium-db repository.
+// It is a collection of YAML files containing security advisories for packages.
+type Repo struct {
+	Path string // Path of local git clone of gemnasium-db.
+}
+
+// SatisfyPackageTypes ensures that the repo provides advisories
+// for all the given package types.
+func (r Repo) SatisfyPackageTypes(pkgTypes ...string) error {
+	errAdvisoryFileFound := errors.New("advisory file found")
+	for _, pkgType := range pkgTypes {
+
+		// check directory corresponding to package type
+		pkgDir := filepath.Join(r.Path, pkgType)
+		info, err := os.Stat(pkgDir)
+		if err != nil || !info.IsDir() {
+			return ErrNoPackageTypeDir{pkgType}
+		}
+
+		// look for advisory file
+		err = filepath.Walk(pkgDir, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err // error accessing path
+			}
+			if !info.IsDir() && hasAdvisoryExt(info.Name()) {
+				return errAdvisoryFileFound // advisory found
+			}
+			return nil // keep searching for advisories
+		})
+		switch err {
+		case errAdvisoryFileFound:
+			continue // advisory found, move on to next package type
+		case nil:
+			return ErrNoAdvisoryForPackageType{pkgType}
+		default:
+			return err // error accessing path
+		}
+	}
+	return nil // all package types are satisfied
+}
+
+// PackageAdvisories returns the paths of the advisories affecting the given package.
+// It excludes directories and files that don't match the file extension of the advisories.
+// Paths are relative to the repository.
+func (r Repo) PackageAdvisories(pkg Package) ([]string, error) {
+	pkgSlug := pkg.Slug()
+	files, err := ioutil.ReadDir(filepath.Join(r.Path, pkgSlug))
+	if err != nil {
+		if _, ok := err.(*os.PathError); ok {
+			return nil, ErrNoPackageDir{Package: pkg}
+		}
+		return nil, err
+	}
+	relPaths := []string{}
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+		if !hasAdvisoryExt(file.Name()) {
+			continue
+		}
+		relPaths = append(relPaths, filepath.Join(pkgSlug, file.Name()))
+	}
+	return relPaths, nil
+}
+
+func hasAdvisoryExt(path string) bool {
+	switch strings.ToLower(filepath.Ext(path)) {
+	case ".yml", ".yaml":
+		return true
+	default:
+		return false
+	}
+}
+
+// Advisory decodes the given advisory.
+// Advisory path is relative to the repository path.
+func (r Repo) Advisory(relPath string) (*Advisory, error) {
+	f, err := os.Open(filepath.Join(r.Path, relPath))
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	adv := Advisory{}
+	err = yaml.NewDecoder(f).Decode(&adv)
+	return &adv, err
+}
+
+// UpdateOptions configures the update of the local git repository of gemnasium-db.
+type UpdateOptions struct {
+	RemoteURL string
+	RefName   string
+}
+
+// Update updates the local gemnasium-db repository with a git remote.
+func (r Repo) Update(opts UpdateOptions) error {
+	gitCommand := func(args ...string) *exec.Cmd {
+		args = append([]string{"-C", r.Path}, args...)
+		cmd := exec.Command("git", args...)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		return cmd
+	}
+
+	// list of git commands
+	argsList := [][]string{}
+
+	// add "git remote set-url" command if needed
+	if remoteURL := opts.RemoteURL; remoteURL != "" {
+		argsList = append(argsList, []string{"remote", "set-url", "origin", remoteURL})
+	}
+
+	// set git ref
+	ref := opts.RefName
+	if ref == "" {
+		ref = defaultRefName
+	}
+
+	// add "git fetch", "git reset" commands
+	argsList = append(argsList,
+		[]string{"fetch", "origin", ref},
+		[]string{"reset", "--hard", "origin/" + ref},
+	)
+
+	// run all commands
+	for _, args := range argsList {
+		if err := gitCommand(args...).Run(); err != nil {
+			return err
+		}
+	}
+	return nil
+}

advisory/repo_test.go

+package advisory
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestRepo_SatisfyPackageTypes(t *testing.T) {
+	repo := Repo{Path: "testdata"}
+
+	t.Run("ok", func(t *testing.T) {
+		if err := repo.SatisfyPackageTypes("gem", "pypi"); err != nil {
+			t.Error(err)
+		}
+	})
+
+	t.Run("no advisory", func(t *testing.T) {
+		err := repo.SatisfyPackageTypes("gem", "pypi", "no_advisory")
+		want := ErrNoAdvisoryForPackageType{PackageType: "no_advisory"}
+		if !reflect.DeepEqual(want, err) {
+			t.Errorf("Wrong error. Expected %#v but got %#v", want, err)
+		}
+	})
+
+	t.Run("no directory", func(t *testing.T) {
+		err := repo.SatisfyPackageTypes("gem", "pypi", "missing")
+		want := ErrNoPackageTypeDir{PackageType: "missing"}
+		if !reflect.DeepEqual(want, err) {
+			t.Errorf("Wrong error. Expected %#v but got %#v", want, err)
+		}
+	})
+}
+
+func TestRepo_PackageAdvisories(t *testing.T) {
+	repo := Repo{Path: "testdata"}
+
+	t.Run("found", func(t *testing.T) {
+		pkg := Package{Type: "pypi", Name: "cryptography"}
+		got, err := repo.PackageAdvisories(pkg)
+		if err != nil {
+			t.Fatal(err)
+		}
+		want := []string{
+			"pypi/cryptography/CVE-2016-9243.YAML",
+			"pypi/cryptography/CVE-2018-10903.yml",
+		}
+		if !reflect.DeepEqual(want, got) {
+			t.Errorf("Wrong result. Expected %#v but got %#v", want, got)
+		}
+	})
+
+	t.Run("missing", func(t *testing.T) {
+		pkg := Package{Type: "pypi", Name: "xyz"}
+		_, err := repo.PackageAdvisories(pkg)
+		want := ErrNoPackageDir{Package: pkg}
+		if !reflect.DeepEqual(want, err) {
+			t.Errorf("Wrong error. Expected %#v but got %#v", want, err)
+		}
+	})
+}
+
+func TestRepo_Advisory(t *testing.T) {
+	repo := Repo{Path: "testdata"}
+	got, err := repo.Advisory("gem/actionmovie/GMS-2019-minimal.yml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := &Advisory{
+		Identifier:     "GMS-2019-minimal",
+		Title:          "Unsafe UPDATE query",
+		Description:    "There is a vulnerability in some UPDATE query.",
+		DisclosureDate: "2019-10-30",
+		AffectedRange:  "*",
+		Package: Package{
+			Type: "gem",
+			Name: "actionmovie",
+		},
+	}
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("Wrong result. Expected %#v but got %#v", want, got)
+	}
+}

advisory/testdata/gem/actionmovie/GMS-2019-minimal.yml

+identifier: GMS-2019-minimal
+title: Unsafe UPDATE query
+description: "There is a vulnerability in some UPDATE query."
+date: "2019-10-30"
+affected_range: '*'
+package_slug: gem/actionmovie

advisory/testdata/gem/activerecord/CVE-2016-6317.yml

+identifier: CVE-2016-6317
+title: Unsafe Query Generation Risk
+description: "There is a vulnerability when Active Record is used in conjunction with
+  JSON\r\nparameter parsing. This vulnerability is similar to CVE-2012-2660,\r\nCVE-2012-2694
+  and CVE-2013-0155."
+date: "2016-08-11"
+affected_range: '>=4.0.0.alpha <4.2.7.1'
+fixed_versions:
+- 4.2.7.1
+affected_versions: 4.x
+not_impacted: 5.x, 3.x and earlier
+solution: Upgrade to latest or use workaround; see provided link.
+credit: joernchen of Phenoelit
+urls:
+- https://groups.google.com/forum/#!topic/rubyonrails-security/rgO20zYW33s
+uuid: 51b1a6d4-3eb3-48d4-8d25-ae62c2dbb3d4
+package_slug: gem/activerecord

advisory/testdata/no_advisory/package/readme

+This is a readme file.

advisory/testdata/pypi/cryptography/CVE-2016-9243.YAML

+identifier: CVE-2016-9243
+title: HKDF might return an empty byte-string
+description: There's a bug where HKDF would return an empty byte-string if used with
+  a length less than `algorithm.digest_size`.
+date: "2016-11-05"
+affected_range: <1.5.3
+fixed_versions:
+- 1.5.3
+affected_versions: All versions
+solution: Upgrade to latest version.
+credit: Markus Döring
+urls:
+- http://seclists.org/oss-sec/2016/q4/360
+- https://github.com/pyca/cryptography/commit/b924696b2e8731f39696584d12cceeb3aeb2d874
+- https://github.com/pyca/cryptography/issues/3211
+uuid: 8a342fb1-daed-4a91-a217-9ad6c19d7bc9
+package_slug: pypi/cryptography

advisory/testdata/pypi/cryptography/CVE-2018-10903.yml

+identifier: CVE-2018-10903
+title: GCM tag forgery via truncated tag in finalize_with_tag API
+description: The `finalize_with_tag` API did not enforce a minimum tag length. If
+  a user did not validate the input length prior to passing it to `finalize_with_tag`
+  an attacker could craft an invalid payload with a shortened tag (e.g. 1 byte) such
+  that they would have a 1 in 256 chance of passing the MAC check. GCM tag forgeries
+  can cause key leakage.
+date: "2018-07-30"
+affected_range: '>=1.9.0,<2.3'
+fixed_versions:
+- "2.3"
+affected_versions: From 1.9 to 2.2.2
+not_impacted: 1.8 and earlier
+solution: Upgrade to latest version
+urls:
+- http://cwe.mitre.org/data/definitions/20.html
+- https://access.redhat.com/errata/RHSA-2018:3600
+- https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2018-10903
+- https://github.com/pyca/cryptography/pull/4342/commits/688e0f673bfbf43fa898994326c6877f00ab19ef
+- https://usn.ubuntu.com/3720-1/
+uuid: 004b3de5-5aa8-4977-b6c5-3b9b31b28dcd
+package_slug: pypi/cryptography

convert/convert.go

@@ -20,9 +20,9 @@ const (
 	gemnasiumURL = "https://deps.sec.gitlab.com"
 )
 
-// Convert converts the output of the Gemnasium client (list of affected sources) to a report.
+// Convert converts the output of the Gemnasium (list of dependency files) to a report.
 func Convert(reader io.Reader, prependPath string) (*issue.Report, error) {
-	var result []scanner.AffectedSource
+	var result []scanner.File
 	err := json.NewDecoder(reader).Decode(&result)
 	if err != nil {
 		return nil, err
@@ -30,13 +30,13 @@ func Convert(reader io.Reader, prependPath string) (*issue.Report, error) {
 	return ToReport(result, prependPath), nil
 }
 
-// ToReport converts affected sources returned by the Gemnasium client to a report.
-func ToReport(sources []scanner.AffectedSource, prependPath string) *issue.Report {
+// ToReport converts dependency files returned by the Gemnasium scanner to a report.
+func ToReport(scanFiles []scanner.File, prependPath string) *issue.Report {
 	issues := []issue.Issue{}
-	depfiles := make([]issue.DependencyFile, len(sources))
-	for i, source := range sources {
-		issues = append(issues, toIssues(source, prependPath)...)
-		depfiles[i] = toDependencyFile(source, prependPath)
+	depfiles := make([]issue.DependencyFile, len(scanFiles))
+	for i, scanFile := range scanFiles {
+		issues = append(issues, toIssues(scanFile, prependPath)...)
+		depfiles[i] = toDependencyFile(scanFile, prependPath)
 	}
 	report := issue.NewReport()
 	report.Vulnerabilities = issues
@@ -44,11 +44,11 @@ func ToReport(sources []scanner.AffectedSource, prependPath string) *issue.Repor
 	return &report
 }
 
-func toDependencyFile(source scanner.AffectedSource, prependPath string) issue.DependencyFile {
+func toDependencyFile(scanFile scanner.File, prependPath string) issue.DependencyFile {
 	return issue.DependencyFile{
-		Path:           filepath.Join(prependPath, source.FilePath),
-		Dependencies:   toDependencies(source.Deps),
-		PackageManager: issue.PackageManager(source.PackageManager),
+		Path:           filepath.Join(prependPath, scanFile.Path),
+		Dependencies:   toDependencies(scanFile.Deps),
+		PackageManager: issue.PackageManager(scanFile.PackageManager),
 	}
 }
 
@@ -61,12 +61,12 @@ func toDependencies(in []parser.Dependency) []issue.Dependency {
 	return out
 }
 
-func toIssues(source scanner.AffectedSource, prependPath string) []issue.Issue {
-	issues := make([]issue.Issue, len(source.Affections))
-	for i, affection := range source.Affections {
+func toIssues(scanFile scanner.File, prependPath string) []issue.Issue {
+	issues := make([]issue.Issue, len(scanFile.Affections))
+	for i, affection := range scanFile.Affections {
 		issues[i] = VulnerabilityConverter{
 			PrependPath: prependPath,
-			Source:      source,
+			File:        scanFile,
 			Advisory:    affection.Advisory,
 			Dependency:  affection.Dependency,
 		}.Issue()

convert/convert_test.go

@@ -7,36 +7,33 @@ import (
 	"testing"
 
 	"gitlab.com/gitlab-org/security-products/analyzers/common/v2/issue"
+	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/advisory"
 	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner"
 	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/parser"
-	"gopkg.in/guregu/null.v3"
 )
 
 func TestConvert(t *testing.T) {
 
 	// advisories
-	pgAdvisory := scanner.Advisory{
-		UUID:             "7d9ba955-fd99-4503-936e-f6833768f76e",
-		Package:          scanner.Package{Type: "gem", Name: "pg"},
-		AffectedVersions: []string{"0.8.0"},
-		Identifier:       null.StringFrom("CVE-1234"),
-		Title:            "Regular Expression Denial of Service",
-		Description:      "Xyz is vulnerable to ReDoS in the Xyz parameter.",
-		Solution:         null.StringFrom("Upgrade to latest version."),
-		Links:            []string{"https://security.io/advisories/119", "https://security.io/advisories/117", "https://security.io/advisories/118"},
+	pgAdvisory := advisory.Advisory{
+		UUID:        "7d9ba955-fd99-4503-936e-f6833768f76e",
+		Package:     advisory.Package{Type: "gem", Name: "pg"},
+		Identifier:  "CVE-1234",
+		Title:       "Regular Expression Denial of Service",
+		Description: "Xyz is vulnerable to ReDoS in the Xyz parameter.",
+		Solution:    "Upgrade to latest version.",
+		Links:       []string{"https://security.io/advisories/119", "https://security.io/advisories/117", "https://security.io/advisories/118"},
 	}
 
 	// analyzer output
-	affected := []scanner.AffectedSource{
+	affected := []scanner.File{
 		{
-			Source: scanner.Source{
-				FilePath:       "rails/Gemfile.lock",
-				PackageManager: issue.PackageManagerBundler,
-				PackageType:    "gem",
-				Deps: []parser.Dependency{
-					{Name: "pg", Version: "0.8.0"},
-					{Name: "puma", Version: "2.16.0"},
-				},
+			Path:           "rails/Gemfile.lock",
+			PackageManager: issue.PackageManagerBundler,
+			PackageType:    "gem",
+			Deps: []parser.Dependency{
+				{Name: "pg", Version: "0.8.0"},
+				{Name: "puma", Version: "2.16.0"},
 			},
 			Affections: []scanner.Affection{
 				{
@@ -46,16 +43,14 @@ func TestConvert(t *testing.T) {
 			},
 		},
 		{
-			Source: scanner.Source{
-				FilePath:       "node/yarn.lock",
-				PackageManager: issue.PackageManagerYarn,
-				PackageType:    "npm",
-				Deps: []parser.Dependency{
-					{Name: "acorn", Version: "4.0.4"},
-					{Name: "acorn", Version: "3.3.0"},
-					{Name: "acorn", Version: "4.0.11"},
-					{Name: "@angular/animations", Version: "4.4.6"},
-				},
+			Path:           "node/yarn.lock",
+			PackageManager: issue.PackageManagerYarn,
+			PackageType:    "npm",
+			Deps: []parser.Dependency{
+				{Name: "acorn", Version: "4.0.4"},
+				{Name: "acorn", Version: "3.3.0"},
+				{Name: "acorn", Version: "4.0.11"},
+				{Name: "@angular/animations", Version: "4.4.6"},
 			},
 		},
 	}

convert/vulnerability_converter.go

@@ -6,6 +6,7 @@ import (
 	"sort"
 
 	"gitlab.com/gitlab-org/security-products/analyzers/common/v2/issue"
+	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/advisory"
 	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner"
 	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/parser"
 )
@@ -14,8 +15,8 @@ import (
 // It also converts information relative to the affected source file.
 type VulnerabilityConverter struct {
 	PrependPath string
-	Source      scanner.AffectedSource
-	Advisory    scanner.Advisory
+	File        scanner.File
+	Advisory    advisory.Advisory
 	Dependency  parser.Dependency
 }
 
@@ -32,7 +33,7 @@ func (c VulnerabilityConverter) Issue() issue.Issue {
 		Name:        c.Advisory.Title,
 		Description: c.Advisory.Description,
 		Severity:    issue.SeverityLevelUnknown,
-		Solution:    c.Advisory.Solution.String,
+		Solution:    c.Advisory.Solution,
 		Identifiers: c.identifiers(),
 		Links:       issue.NewLinks(c.Advisory.Links...),
 		Location: issue.Location{
@@ -48,13 +49,13 @@ func (c VulnerabilityConverter) Issue() issue.Issue {
 }
 
 func (c VulnerabilityConverter) filePath() string {
-	return filepath.Join(c.PrependPath, c.Source.FilePath)
+	return filepath.Join(c.PrependPath, c.File.Path)
 }
 
 func (c VulnerabilityConverter) identifiers() []issue.Identifier {
 	ids := []issue.Identifier{c.primaryIdentifier()}
-	if id := c.Advisory.Identifier; id.Valid {
-		if identifier, ok := issue.ParseIdentifierID(id.String); ok {
+	if id := c.Advisory.Identifier; id != "" {
+		if identifier, ok := issue.ParseIdentifierID(id); ok {
 			ids = append(ids, identifier)
 		}
 	}
@@ -71,7 +72,7 @@ func (c VulnerabilityConverter) primaryIdentifier() issue.Identifier {
 }
 
 func (c VulnerabilityConverter) advisoriesURL() string {
-	ptype := string(c.Source.PackageType)
+	ptype := string(c.File.PackageType)
 	name, version := c.Dependency.Name, c.Dependency.Version
 	return fmt.Sprintf("%s/packages/%s/%s/versions/%s/advisories", gemnasiumURL, ptype, name, version)
 }

go.mod

@@ -3,5 +3,5 @@ module gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2
 require (
 	github.com/urfave/cli v1.20.0
 	gitlab.com/gitlab-org/security-products/analyzers/common/v2 v2.4.2
-	gopkg.in/guregu/null.v3 v3.4.0
+	gopkg.in/yaml.v2 v2.2.4
 )

go.sum

@@ -2,5 +2,7 @@ github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 gitlab.com/gitlab-org/security-products/analyzers/common/v2 v2.4.2 h1:BzrBV677ShBVw5htCceGWg5C1sK7teqGAKF/PE97Nls=
 gitlab.com/gitlab-org/security-products/analyzers/common/v2 v2.4.2/go.mod h1:k+W8TS8696BIs1hhnVGbJbAaAXjvUkVAixhxUOV+KVs=
-gopkg.in/guregu/null.v3 v3.4.0 h1:AOpMtZ85uElRhQjEDsFx21BkXqFPwA7uoJukd4KErIs=
-gopkg.in/guregu/null.v3 v3.4.0/go.mod h1:E4tX2Qe3h7QdL+uZ3a0vqvYwKQsRSQKM5V4YltdgH9Y=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

main.go

@@ -10,6 +10,8 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
+	"strings"
 	"time"
 
 	"github.com/urfave/cli"
@@ -20,8 +22,9 @@ import (
 	"gitlab.com/gitlab-org/security-products/analyzers/common/v2/search"
 	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/convert"
 	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/plugin"
+	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/remediate"
 	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner"
-	scannercli "gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/cli"
+	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/vrange"
 
 	_ "gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/parser/composer"
 	_ "gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/parser/gemfile"
@@ -37,6 +40,13 @@ const (
 	flagRemediate        = "remediate"
 	flagRemediateTimeout = "remediate-timeout"
 
+	flagVrangeDir       = "vrange-dir"
+	flagVrangeGemCmd    = "vrange-gem-cmd"
+	flagVrangeMavenCmd  = "vrange-maven-cmd"
+	flagVrangeNpmCmd    = "vrange-npm-cmd"
+	flagVrangePhpCmd    = "vrange-php-cmd"
+	flagVrangePythonCmd = "vrange-python-cmd"
+
 	defaultTimeoutRemediate = 5 * time.Minute
 )
 
@@ -75,10 +85,48 @@ func runCommand() cli.Command {
 			Usage:  "Time limit for vulnerabilities auto-remediation",
 			Value:  defaultTimeoutRemediate,
 		},
+
+		// vrange CLIs
+		cli.StringFlag{
+			Name:   flagVrangeDir,
+			Usage:  "Path of the vrange directory",
+			EnvVar: "VRANGE_DIR",
+			Value:  "vrange",
+		},
+		cli.StringFlag{
+			Name:   flagVrangeGemCmd,
+			Usage:  "vrange command for Rubygem",
+			EnvVar: "VRANGE_GEM_CMD",
+			Value:  "gem/vrange.rb",
+		},
+		cli.StringFlag{
+			Name:   flagVrangeMavenCmd,
+			Usage:  "vrange command for Maven",
+			EnvVar: "VRANGE_MAVEN_CMD",
+			Value:  "semver/vrange-" + runtime.GOOS + " maven",
+		},
+		cli.StringFlag{
+			Name:   flagVrangeNpmCmd,
+			Usage:  "vrange command for npm",
+			EnvVar: "VRANGE_NPM_CMD",
+			Value:  "npm/rangecheck.js",
+		},
+		cli.StringFlag{
+			Name:   flagVrangePhpCmd,
+			Usage:  "vrange command for PHP",
+			EnvVar: "VRANGE_PHP_CMD",
+			Value:  "php/rangecheck.php",
+		},
+		cli.StringFlag{
+			Name:   flagVrangePythonCmd,
+			Usage:  "vrange command for Python",
+			EnvVar: "VRANGE_PYTHON_CMD",
+			Value:  "python/rangecheck.py",
+		},
 	}
+
 	flags = append(flags, search.NewFlags()...)
-	flags = append(flags, scannercli.ClientFlags()...)
-	flags = append(flags, scannercli.FinderFlags()...)
+	flags = append(flags, scanner.Flags()...)
 	flags = append(flags, pathfilter.MakeFlags("DS_")...)
 
 	return cli.Command{
@@ -93,6 +141,11 @@ func runCommand() cli.Command {
 				return errors.New("Invalid number of arguments")
 			}
 
+			// register version range resolvers
+			if err := registerResolvers(c); err != nil {
+				return err
+			}
+
 			// parse excluded paths
 			filter, err := pathfilter.NewFilter(c)
 			if err != nil {
@@ -113,33 +166,19 @@ func runCommand() cli.Command {
 			}
 			fmt.Fprintln(c.App.Writer, "Found project in "+matchPath)
 
-			// find compatible files
-			finder := scannercli.NewFinder(c)
-			files, err := finder.FindFiles(targetDir)
+			// scan target directory
+			scanner, err := scanner.NewScanner(c)
 			if err != nil {
 				return err
 			}
-
-			// get sources of dependencies
-			sources, err := scanner.ParseFiles(files)
+			result, err := scanner.ScanDir(targetDir)
 			if err != nil {
 				return err
 			}
 
-			// get advisories
-			client, err := scannercli.NewClient(c)
-			if err != nil {
-				return err
-			}
-			advisories, err := client.Advisories(sources.Packages())
-			if err != nil {
-				return err
-			}
-
-			// convert to vulnerabilities and dependency files
-			var affectedSources = scanner.AffectedSources(advisories, sources...)
+			// convert to generic report
 			var prependPath = "" // empty because the analyzer scans the root directory
-			var report = convert.ToReport(affectedSources, prependPath)
+			var report = convert.ToReport(result, prependPath)
 
 			// remediate vulnerabilities
 			if c.BoolT(flagRemediate) {
@@ -149,7 +188,7 @@ func runCommand() cli.Command {
 					var t = c.Duration(flagRemediateTimeout)
 					ctx, cancel := context.WithTimeout(context.Background(), t)
 					defer cancel()
-					report.Remediations = remediations(ctx, affectedSources...)
+					report.Remediations = remediations(ctx, result...)
 				}
 			}
 
@@ -171,12 +210,31 @@ func runCommand() cli.Command {
 	}
 }
 
-// remediations attempts to cure affected sources and returns remediations.
-func remediations(ctx context.Context, sources ...scanner.AffectedSource) []issue.Remediation {
+func registerResolvers(c *cli.Context) error {
+	syntaxToFlag := map[string]string{
+		"gem":    flagVrangeGemCmd,
+		"maven":  flagVrangeMavenCmd,
+		"npm":    flagVrangeNpmCmd,
+		"php":    flagVrangePhpCmd,
+		"python": flagVrangePythonCmd,
+	}
+	for syntax, flag := range syntaxToFlag {
+		cmd := strings.SplitN(c.String(flag), " ", 2)
+		path := filepath.Join(c.String(flagVrangeDir), cmd[0])
+		args := cmd[1:]
+		if err := vrange.RegisterCmd(syntax, path, args...); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// remediations attempts to cure affected dependency files and returns remediations.
+func remediations(ctx context.Context, scanFiles ...scanner.File) []issue.Remediation {
 	var allRems = make([]issue.Remediation, 0)
-	for _, source := range sources {
+	for _, file := range scanFiles {
 		// attempt to cure
-		cures, err := cureSource(ctx, source)
+		cures, err := remediate.Remediate(ctx, file)
 		switch err {
 		case nil:
 			// proceed
@@ -191,14 +249,14 @@ func remediations(ctx context.Context, sources ...scanner.AffectedSource) []issu
 		}
 
 		// convert cures to remediations;
-		// it can't be extracted out of this loop because source is required.
+		// it can't be extracted out of this loop because dependency file is required.
 		var rems = make([]issue.Remediation, len(cures))
 		for i, cure := range cures {
 			var refs = make([]issue.IssueRef, len(cure.Affections))
 			for j, a := range cure.Affections {
 				// HACK convert to an issue to get the exact compare key
 				var ckey = convert.VulnerabilityConverter{
-					Source:     source,
+					File:       file,
 					Advisory:   a.Advisory,
 					Dependency: a.Dependency,
 				}.Issue().CompareKey