Merge branch '14630-use-gemnasium-db' into 'master'

Connect to gemnasium-db repo

See merge request !25

scanner/package.go

-package scanner
-
-import "gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/parser"
-
-// Package contains the information needed to resolve a package on the server side.
-type Package struct {
-	Type parser.PackageType `json:"type"`
-	Name string             `json:"name"`
-}

scanner/parse.go

-package scanner
-
-import (
-	"io"
-	"os"
-
-	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/parser"
-)
-
-// ErrParserNotFound is raised when parser lookup fails.
-type ErrParserNotFound struct {
-	Filename string
-}
-
-// Error returns the error message.
-func (e ErrParserNotFound) Error() string {
-	return "No compatible parser for " + e.Filename
-}
-
-// Parse parses a reader with a parser capable of handling fileName.
-// The FilePath of the returned Source is set to the given filePath.
-func Parse(r io.Reader, fileName string, filePath string) (*Source, error) {
-	parser := parser.Lookup(fileName)
-	if parser == nil {
-		return nil, ErrParserNotFound{filePath}
-	}
-
-	deps, err := parser.Parse(r)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Source{
-		FilePath:       filePath,
-		PackageManager: parser.PackageManager,
-		PackageType:    parser.PackageType,
-		Deps:           deps,
-	}, nil
-}
-
-// ParseFiles parses the compatible files found by FindFiles.
-// The FilePath of the source is set to the relative path of the dependency file.
-func ParseFiles(files []File) (Sources, error) {
-	sources := make([]Source, len(files))
-	for i, file := range files {
-		// open file
-		f, err := os.Open(file.AbsPath())
-		if err != nil {
-			return nil, err
-		}
-		defer f.Close()
-
-		// parse
-		deps, err := file.Parser.Parse(f)
-		if err != nil {
-			return nil, err
-		}
-		sources[i] = Source{
-			FilePath:       file.Path,
-			RootDir:        file.Root,
-			PackageManager: file.Parser.PackageManager,
-			PackageType:    file.Parser.PackageType,
-			Deps:           deps,
-		}
-	}
-	return Sources(sources), nil
-}

scanner/parse_test.go

-package scanner
-
-import (
-	"os"
-	"reflect"
-	"testing"
-
-	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/parser"
-	_ "gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/parser/gemfile"
-	_ "gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/parser/yarn"
-)
-
-func TestParse(t *testing.T) {
-	f, err := os.Open("parser/gemfile/fixtures/simple/Gemfile.lock")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer f.Close()
-
-	got, err := Parse(f, "Gemfile.lock", "xyz")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	want := &Source{
-		FilePath:       "xyz",
-		PackageType:    "gem",
-		PackageManager: "bundler",
-		Deps: []parser.Dependency{
-			{Name: "pg", Version: "1.0.0"},
-			{Name: "puma", Version: "2.16.0"},
-		},
-	}
-
-	if !reflect.DeepEqual(got, want) {
-		t.Errorf("Wrong result. Expecting:\n%#v\nbut got:\n%#v", want, got)
-	}
-}
-
-func TestParseFiles(t *testing.T) {
-	getParser := func(filename string) parser.Parser {
-		if p := parser.Lookup(filename); p != nil {
-			return *p
-		}
-		t.Fatal(ErrParserNotFound{filename})
-		return parser.Parser{}
-	}
-
-	files := []File{
-		{
-			Path:   "simple/Gemfile.lock",
-			Root:   "parser/gemfile/fixtures",
-			Parser: getParser("Gemfile.lock"),
-		},
-		{
-			Path:   "simple/yarn.lock",
-			Root:   "parser/yarn/fixtures",
-			Parser: getParser("yarn.lock"),
-		},
-	}
-
-	got, err := ParseFiles(files)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	want := Sources{
-		{
-			FilePath:       "simple/Gemfile.lock",
-			RootDir:        "parser/gemfile/fixtures",
-			PackageType:    "gem",
-			PackageManager: "bundler",
-			Deps: []parser.Dependency{
-				{Name: "pg", Version: "1.0.0"},
-				{Name: "puma", Version: "2.16.0"},
-			},
-		},
-		{
-			FilePath:       "simple/yarn.lock",
-			RootDir:        "parser/yarn/fixtures",
-			PackageType:    "npm",
-			PackageManager: "yarn",
-			Deps: []parser.Dependency{
-				{Name: "acorn", Version: "4.0.4"},
-				{Name: "acorn", Version: "3.3.0"},
-				{Name: "acorn", Version: "4.0.11"},
-				{Name: "@angular/animations", Version: "4.4.6"},
-			},
-		},
-	}
-
-	if !reflect.DeepEqual(got, want) {
-		t.Errorf("Wrong result. Expecting:\n%#v\nbut got:\n%#v", want, got)
-	}
-}

scanner/parser/gemfile/gemfile.go

@@ -50,14 +50,14 @@ func Parse(r io.Reader) ([]parser.Dependency, error) {
 	document := Document{}
 	document.Dependencies = []Dependency{}
 
-	scanner := bufio.NewScanner(r)
+	depfile := bufio.NewScanner(r)
 
 	var line string
 	var lineNumber int64
 	var state = stateUnknown
 	var currentSourceType = SourceType{}
-	for scanner.Scan() {
-		line = scanner.Text()
+	for depfile.Scan() {
+		line = depfile.Text()
 		lineNumber++
 
 		switch {

scanner/parser/mvnplugin/mvnplugin.go

@@ -32,6 +32,6 @@ func init() {
 		Parse:          Parse,
 		PackageManager: "maven",
 		PackageType:    parser.PackageTypeMaven,
-		Filenames:      []string{"maven-dependencies.json"},
+		Filenames:      []string{"maven-dependencies.json", "gemnasium-maven-plugin.json"},
 	})
 }

scanner/parser/npm/npm.go

@@ -68,7 +68,7 @@ func removeDuplicates(dependencies []parser.Dependency) []parser.Dependency {
 	keys := make(map[string]bool)
 	list := []parser.Dependency{}
 	for _, dep := range dependencies {
-		key:= dep.Name+"@"+dep.Version
+		key := dep.Name + "@" + dep.Version
 		if _, found := keys[key]; !found {
 			keys[key] = true
 			list = append(list, dep)

scanner/parser/registry.go

@@ -43,3 +43,24 @@ func Parsers() []string {
 	sort.Strings(list)
 	return list
 }
+
+// PackageTypes returns all the registered package types, without duplicates.
+func PackageTypes() []string {
+	parsersMu.Lock()
+	defer parsersMu.Unlock()
+
+	// collect package types in a map to remove duplicates
+	pkgTypeMap := map[string]bool{}
+	for _, parser := range parsers {
+		pkgTypeMap[string(parser.PackageType)] = true
+	}
+
+	// turn map keys into a slice of strings
+	pkgTypes := []string{}
+	for pkgType, _ := range pkgTypeMap {
+		pkgTypes = append(pkgTypes, pkgType)
+	}
+
+	sort.Strings(pkgTypes)
+	return pkgTypes
+}

scanner/parser/registry_test.go

+package parser
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"reflect"
+	"testing"
+)
+
+// Test_Registry is a combined tests where Register is tested indirectly.
+func Test_Registry(t *testing.T) {
+
+	// clear registry
+	parsersMu.Lock()
+	parsers = make(map[string]Parser)
+	parsersMu.Unlock()
+
+	// unsorted list of package types
+	pkgTypes := []PackageType{PackageTypeNpm, PackageTypeMaven, PackageTypePackagist, PackageTypeGem}
+
+	// register parsers with duplicates
+	for _, pkgType := range pkgTypes {
+		name := string(pkgType)
+		parse := func(r io.Reader) ([]Dependency, error) { return nil, errors.New("no parser for " + name) }
+		Register(name+"1", Parser{
+			Parse:          parse,
+			PackageManager: fmt.Sprintf("package manager 1 for %ss", name),
+			PackageType:    pkgType,
+			Filenames:      []string{name + "file"},
+		})
+		Register(name+"2", Parser{
+			Parse:          parse,
+			PackageManager: fmt.Sprintf("package manager 2 for %ss", name),
+			PackageType:    pkgType,
+			Filenames:      []string{"package." + name, "manifest." + name},
+		})
+	}
+
+	t.Run("Lookup", func(t *testing.T) {
+		t.Run("found", func(t *testing.T) {
+			p := Lookup("manifest.npm")
+			want := "package manager 2 for npms"
+			if p == nil {
+				t.Fatalf("Expected '%s' but got nil", want)
+			}
+			got := p.PackageManager
+			if got != want {
+				t.Errorf("Expected '%s' but got '%s'", want, got)
+			}
+		})
+		t.Run("missing", func(t *testing.T) {
+			p := Lookup("missing.npm")
+			if p != nil {
+				t.Errorf("Expected no parser but got '%s'", p.PackageManager)
+			}
+		})
+	})
+
+	t.Run("Parsers", func(t *testing.T) {
+		want := []string{"gem1", "gem2", "maven1", "maven2", "npm1", "npm2", "packagist1", "packagist2"}
+		got := Parsers()
+		if !reflect.DeepEqual(got, want) {
+			t.Errorf("Expected %v but got %v", want, got)
+		}
+	})
+
+	t.Run("PackageTypes", func(t *testing.T) {
+		want := []string{"gem", "maven", "npm", "packagist"}
+		got := PackageTypes()
+		if !reflect.DeepEqual(got, want) {
+			t.Errorf("Expected %v but got %v", want, got)
+		}
+	})
+}

scanner/parser/yarn/yarn.go

@@ -32,13 +32,13 @@ func Parse(r io.Reader) ([]parser.Dependency, error) {
 	document.Version = "1"
 	document.Dependencies = []Dependency{}
 
-	scanner := bufio.NewScanner(r)
+	depfile := bufio.NewScanner(r)
 
 	var line string
 	var dep Dependency
 	var lineNumber int64
-	for scanner.Scan() {
-		line = scanner.Text()
+	for depfile.Scan() {
+		line = depfile.Text()
 		lineNumber++
 
 		switch {

scanner/scanner.go

+package scanner
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/urfave/cli"
+
+	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/advisory"
+	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/finder"
+	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/parser"
+	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/vrange"
+)
+
+const (
+	flagPrefix      = "gemnasium-"
+	flagDBUpdate    = flagPrefix + "db-update"
+	flagDBLocalPath = flagPrefix + "db-local-path"
+	flagDBRemoteURL = flagPrefix + "db-remote-url"
+	flagDBRefName   = flagPrefix + "db-ref-name"
+	flagIgnoredDirs = flagPrefix + "ignore-dirs"
+
+	envVarPrefix      = "GEMNASIUM_"
+	envVarDBUpdate    = envVarPrefix + "DB_UPDATE"
+	envVarDBLocalPath = envVarPrefix + "DB_LOCAL_PATH"
+	envVarDBRemoteURL = envVarPrefix + "DB_REMOTE_URL"
+	envVarDBRefName   = envVarPrefix + "DB_REF_NAME"
+	envVarIgnoredDirs = envVarPrefix + "IGNORED_DIRS,SEARCH_IGNORED_DIRS"
+)
+
+var defaultIgnoredDirs = cli.StringSlice([]string{"node_modules", ".bundle", "vendor", ".git"})
+
+func Flags() []cli.Flag {
+	return []cli.Flag{
+		cli.BoolTFlag{
+			Name:   flagDBUpdate,
+			EnvVar: envVarDBUpdate,
+			Usage:  "Update gemnasium-db git repo before scanning (default: true)",
+		},
+		cli.StringFlag{
+			Name:   flagDBLocalPath,
+			EnvVar: envVarDBLocalPath,
+			Usage:  "Path of gemnasium-db git repo",
+			Value:  "gemnasium-db",
+		},
+		cli.StringFlag{
+			Name:   flagDBRemoteURL,
+			EnvVar: envVarDBRemoteURL,
+			Usage:  "Remote URL the local gemnasium-db git repo is synced with",
+		},
+		cli.StringFlag{
+			Name:   flagDBRefName,
+			EnvVar: envVarDBRefName,
+			Usage:  "git reference the local gemnasium-db git repo is synced with",
+		},
+		cli.StringSliceFlag{
+			Name:   flagIgnoredDirs,
+			EnvVar: envVarIgnoredDirs,
+			Usage:  "Directory to be ignored",
+			Value:  &defaultIgnoredDirs,
+		},
+	}
+}
+
+func NewScanner(c *cli.Context) (*Scanner, error) {
+	f := finder.Finder{IgnoredDirs: c.StringSlice(flagIgnoredDirs)}
+	r := advisory.Repo{Path: c.String(flagDBLocalPath)}
+
+	// update advisory repo if requested
+	if c.Bool(flagDBUpdate) {
+		updateOpts := advisory.UpdateOptions{
+			RemoteURL: c.String(flagDBRemoteURL),
+			RefName:   c.String(flagDBRefName),
+		}
+		if err := r.Update(updateOpts); err != nil {
+			return nil, err
+		}
+	}
+
+	// check advisory repo
+	if err := r.SatisfyPackageTypes(parser.PackageTypes()...); err != nil {
+		return nil, err
+	}
+
+	return &Scanner{f, r}, nil
+}
+
+type Scanner struct {
+	finder finder.Finder
+	repo   advisory.Repo
+}
+
+func (s Scanner) ScanDir(dir string) ([]File, error) {
+
+	// find dependency files
+	files, err := s.finder.FindFiles(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	// parse dependency files
+	depfiles := []File{}
+	for _, file := range files {
+		path := filepath.Join(file.RootDir, file.Path)
+		f, err := os.Open(path)
+		if err != nil {
+			return nil, err
+		}
+		depfile, err := parseReader(f, file.Parser, file.Path)
+		if err != nil {
+			return nil, err
+		}
+		depfile.RootDir = dir
+		depfiles = append(depfiles, *depfile)
+	}
+
+	// add affections to dependency files
+	for i, depfile := range depfiles {
+		aff, err := fileAffections(depfile, s.repo)
+		if err != nil {
+			return nil, err
+		}
+		depfiles[i].Affections = aff
+	}
+
+	return depfiles, nil
+}
+
+func (s Scanner) ScanFile(path, alias string) (*File, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	fileType := filepath.Base(path)
+	return s.ScanReader(f, fileType, alias)
+}
+
+func (s Scanner) ScanReader(reader io.Reader, fileType, filePath string) (*File, error) {
+	depParser := parser.Lookup(fileType)
+	if depParser == nil {
+		return nil, fmt.Errorf("no parser for file type %s, path %s", fileType, filePath)
+	}
+	depfile, err := parseReader(reader, *depParser, filePath)
+	if err != nil {
+		return nil, err
+	}
+	aff, err := fileAffections(*depfile, s.repo)
+	if err != nil {
+		return nil, err
+	}
+	depfile.Affections = aff
+	return depfile, nil
+}
+
+func parseReader(reader io.Reader, depParser parser.Parser, filePath string) (*File, error) {
+	deps, err := depParser.Parse(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	return &File{
+		Path:           filePath,
+		PackageManager: depParser.PackageManager,
+		PackageType:    string(depParser.PackageType),
+		Deps:           deps,
+	}, nil
+}
+
+func fileAffections(depfile File, repo advisory.Repo) ([]Affection, error) {
+	pkgType := depfile.PackageType
+
+	// collect possible affections, list version range queries
+	affections := []Affection{}
+	queries := []vrange.Query{}
+	for _, dep := range depfile.Deps {
+
+		// list package advisories
+		pkg := advisory.Package{Type: pkgType, Name: dep.Name}
+		paths, err := repo.PackageAdvisories(pkg)
+		if _, ok := err.(advisory.ErrNoPackageDir); ok {
+			continue // no advisories
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		// fetch advisories, append affections and queries
+		for _, path := range paths {
+			adv, err := repo.Advisory(path)
+			if err != nil {
+				return nil, err
+			}
+			aff := Affection{dep, *adv}
+			affections = append(affections, aff)
+			queries = append(queries, affectionToQuery(aff))
+		}
+	}
+
+	// use resolver to tell if dependency versions are in affected ranges
+	resolver, err := vrange.NewResolver(pkgTypeToResolverName(pkgType))
+	if err != nil {
+		return nil, err
+	}
+	result, err := resolver.Resolve(queries)
+	if err != nil {
+		return nil, err
+	}
+
+	// filter affections and keep those where the dependency version is in affected range
+	filtered := []Affection{}
+	for _, aff := range affections {
+		ok, err := result.Satisfies(affectionToQuery(aff))
+		if err != nil {
+			// TODO report error
+			continue
+		}
+		if ok {
+			filtered = append(filtered, aff)
+		}
+	}
+	return filtered, nil
+}
+
+// pkgTypeToResolverName converts a package type to the name of a vrange resolver.
+func pkgTypeToResolverName(pkgType string) string {
+	switch pkgType {
+	case "npm", "maven", "gem":
+		return pkgType
+	case "packagist":
+		return "php"
+	case "pypi":
+		return "python"
+	default:
+		return pkgType
+	}
+}
+
+// affectionToQuery returns a version range query for given affection.
+func affectionToQuery(aff Affection) vrange.Query {
+	versionRange, version := aff.Advisory.AffectedRange, aff.Dependency.Version
+	return vrange.Query{Range: versionRange, Version: version}
+}

scanner/scanner_test.go

+package scanner
+
+import (
+	"encoding/json"
+	"flag"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"testing"
+
+	"github.com/urfave/cli"
+
+	_ "gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/parser/gemfile"
+	_ "gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/parser/yarn"
+
+	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/vrange"
+)
+
+func TestScanner(t *testing.T) {
+
+	// register version range resolvers
+	for _, syntax := range []string{"npm", "gem"} {
+		if err := vrange.RegisterCmd(syntax, "../vrange/semver/vrange-"+runtime.GOOS, "npm"); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// scanner
+	set := flag.NewFlagSet("test", 0)
+	set.String("gemnasium-db-local-path", "testdata/gemnasium-db", "doc")
+	set.Bool("gemnasium-db-update", false, "doc")
+	c := cli.NewContext(nil, set, nil)
+	scanner, err := NewScanner(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("ScanDir", func(t *testing.T) {
+		dir := "testdata/bundler-yarn"
+		got, err := scanner.ScanDir(dir)
+		if err != nil {
+			t.Fatal(err)
+		}
+		want := []File{}
+		checkExpectedFile(t, &want, &got, filepath.Join(dir, "scandir.json"))
+	})
+
+	t.Run("ScanFile", func(t *testing.T) {
+		dir := "testdata/bundler-yarn"
+		name := "bundler/Gemfile.lock"
+		path := filepath.Join(dir, name)
+		alias := "bundler/Gemfile"
+		got, err := scanner.ScanFile(path, alias)
+		if err != nil {
+			t.Fatal(err)
+		}
+		want := File{}
+		checkExpectedFile(t, &want, got, filepath.Join(dir, "scanfile.json"))
+	})
+
+	t.Run("ScanReader", func(t *testing.T) {
+		dir := "testdata/bundler-yarn"
+		path := "yarn/yarn.lock"
+		f, err := os.Open(filepath.Join(dir, path))
+		if err != nil {
+			t.Fatal(err)
+		}
+		got, err := scanner.ScanReader(f, "yarn.lock", path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		want := File{}
+		checkExpectedFile(t, &want, got, filepath.Join(dir, "scanreader.json"))
+	})
+
+}
+
+func checkExpectedFile(t *testing.T, want, got interface{}, path string) {
+	// look for file containing expected JSON document
+	if _, err := os.Stat(path); err != nil {
+		createExpectedFile(t, got, path)
+	} else {
+		compareToExpectedFile(t, want, got, path)
+	}
+}
+
+func compareToExpectedFile(t *testing.T, want, got interface{}, path string) {
+	f, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	err = json.NewDecoder(f).Decode(want)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("result doesn't match content of %s", path)
+	}
+}
+
+func createExpectedFile(t *testing.T, got interface{}, path string) {
+	// make test fail
+	t.Errorf("creating JSON document: %s", path)
+
+	// create directory for file
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	// create file
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	// write JSON doc
+	encoder := json.NewEncoder(f)
+	encoder.SetIndent("", "  ")
+	encoder.SetEscapeHTML(false)
+	if err := encoder.Encode(got); err != nil {
+		t.Fatal(err)
+	}
+}

scanner/source.go

-package scanner
-
-import (
-	"path/filepath"
-
-	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/parser"
-)
-
-// Source is a source of dependencies. It combines dependencies with a file.
-type Source struct {
-	FilePath       string              `json:"filepath"`
-	PackageManager string              `json:"package_manager"`
-	RootDir        string              `json:"root_dir,omitempty"`
-	PackageType    parser.PackageType  `json:"package_type"`
-	Deps           []parser.Dependency `json:"dependencies"`
-}
-
-// AbsFilePath returns the absolute file path.
-func (s Source) AbsFilePath() string {
-	return filepath.Join(s.RootDir, s.FilePath)
-}
-
-// Packages returns all the packages found in the dependencies. It may contain duplicates.
-func (s Source) Packages() []Package {
-	pkgs := make([]Package, len(s.Deps))
-	for i, d := range s.Deps {
-		pkgs[i] = Package{Type: s.PackageType, Name: d.Name}
-	}
-	return pkgs
-}
-
-// Sources extends a slice of FileDeps with helper methods.
-type Sources []Source
-
-// Packages return the packages for all the dependencies listed in the files.
-func (sources Sources) Packages() []Package {
-	packages := []Package{}
-	for _, s := range sources {
-		packages = append(packages, s.Packages()...)
-	}
-	return packages
-}

scanner/source_test.go

-package scanner
-
-import (
-	"reflect"
-	"testing"
-
-	"gitlab.com/gitlab-org/security-products/analyzers/gemnasium/v2/scanner/parser"
-)
-
-func TestSources(t *testing.T) {
-	t.Run("Packages", func(t *testing.T) {
-		// Sources.Package depends on Source.Package,
-		// so both are tested at the same time.
-
-		sources := Sources{
-			{
-				PackageType: "gem",
-				Deps: []parser.Dependency{
-					{Name: "pg", Version: "0.8.0"},
-					{Name: "puma", Version: "2.16.0"},
-				},
-			},
-			{
-				PackageType: "npm",
-				Deps: []parser.Dependency{
-					{Name: "acorn", Version: "4.0.4"},
-					{Name: "acorn", Version: "3.3.0"},
-					{Name: "acorn", Version: "4.0.11"},
-					{Name: "@angular/animations", Version: "4.4.6"},
-				},
-			},
-		}
-
-		want := []Package{
-			{Name: "pg", Type: "gem"},
-			{Name: "puma", Type: "gem"},
-			{Name: "acorn", Type: "npm"},
-			{Name: "acorn", Type: "npm"},
-			{Name: "acorn", Type: "npm"},
-			{Name: "@angular/animations", Type: "npm"},
-		}
-
-		got := sources.Packages()
-		if !reflect.DeepEqual(got, want) {
-			t.Errorf("Wrong result. Expecting:\n%v\nbut got:\n%v", want, got)
-		}
-	})
-}

scanner/testdata/bundler-yarn/bundler/Gemfile

+source 'https://rubygems.org'
+ruby "2.4.5"
+
+gem 'puma', '~> 3.12'
+gem 'pg'
+
+# Background Job
+gem 'sidekiq', '~> 4.0'
+gem 'sinatra', require: false
+gem 'slim'
+
+gem 'nokogiri', '1.6.7.2'
+
+group :development, :test do
+  gem 'byebug'
+end
+
+group :development do
+  gem 'spring'
+  gem 'guard-rspec', require: false
+end
+
+group :test do
+  gem 'faker', '~> 1.6.1'
+  gem 'rspec'
+end

scanner/testdata/bundler-yarn/bundler/Gemfile.lock

+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.1.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    byebug (10.0.0)
+    coderay (1.1.2)
+    concurrent-ruby (1.0.5)
+    connection_pool (2.2.1)
+    diff-lcs (1.3)
+    faker (1.6.6)
+      i18n (~> 0.5)
+    ffi (1.9.21)
+    formatador (0.2.5)
+    guard (2.14.2)
+      formatador (>= 0.2.4)
+      listen (>= 2.7, < 4.0)
+      lumberjack (>= 1.0.12, < 2.0)
+      nenv (~> 0.1)
+      notiffany (~> 0.0)
+      pry (>= 0.9.12)
+      shellany (~> 0.0)
+      thor (>= 0.18.1)
+    guard-compat (1.2.1)
+    guard-rspec (4.7.3)
+      guard (~> 2.1)
+      guard-compat (~> 1.1)
+      rspec (>= 2.99.0, < 4.0)
+    i18n (0.9.5)
+      concurrent-ruby (~> 1.0)
+    listen (3.1.5)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+      ruby_dep (~> 1.2)
+    lumberjack (1.0.12)
+    method_source (0.9.0)
+    mini_portile2 (2.0.0)
+    minitest (5.11.3)
+    mustermann (1.0.1)
+    nenv (0.3.0)
+    nokogiri (1.6.7.2)
+      mini_portile2 (~> 2.0.0.rc2)
+    notiffany (0.1.1)
+      nenv (~> 0.1)
+      shellany (~> 0.0)
+    pg (1.0.0)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    puma (3.12.0)
+    rack (2.0.4)
+    rack-protection (2.0.0)
+      rack
+    rb-fsevent (0.10.2)
+    rb-inotify (0.9.10)
+      ffi (>= 0.5.0, < 2)
+    redis (3.3.5)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.1)
+    ruby_dep (1.5.0)
+    shellany (0.0.1)
+    sidekiq (4.2.10)
+      concurrent-ruby (~> 1.0)
+      connection_pool (~> 2.2, >= 2.2.0)
+      rack-protection (>= 1.5.0)
+      redis (~> 3.2, >= 3.2.1)
+    sinatra (2.0.0)
+      mustermann (~> 1.0)
+      rack (~> 2.0)
+      rack-protection (= 2.0.0)
+      tilt (~> 2.0)
+    slim (3.0.9)
+      temple (>= 0.7.6, < 0.9)
+      tilt (>= 1.3.3, < 2.1)
+    spring (2.0.2)
+      activesupport (>= 4.2)
+    temple (0.8.0)
+    thor (0.20.0)
+    thread_safe (0.3.6)
+    tilt (2.0.8)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  byebug
+  faker (~> 1.6.1)
+  guard-rspec
+  nokogiri (= 1.6.7.2)
+  pg
+  puma (~> 3.12)
+  rspec
+  sidekiq (~> 4.0)
+  sinatra
+  slim
+  spring
+
+RUBY VERSION
+   ruby 2.4.5p335
+
+BUNDLED WITH
+   1.17.3

scanner/testdata/bundler-yarn/scanfile.json

+{
+  "filepath": "bundler/Gemfile",
+  "package_manager": "bundler",
+  "package_type": "gem",
+  "dependencies": [
+    {
+      "name": "activesupport",
+      "version": "5.1.4"
+    },
+    {
+      "name": "byebug",
+      "version": "10.0.0"
+    },
+    {
+      "name": "coderay",
+      "version": "1.1.2"
+    },
+    {
+      "name": "concurrent-ruby",
+      "version": "1.0.5"
+    },
+    {
+      "name": "connection_pool",
+      "version": "2.2.1"
+    },
+    {
+      "name": "diff-lcs",
+      "version": "1.3"
+    },
+    {
+      "name": "faker",
+      "version": "1.6.6"
+    },
+    {
+      "name": "ffi",
+      "version": "1.9.21"
+    },
+    {
+      "name": "formatador",
+      "version": "0.2.5"
+    },
+    {
+      "name": "guard",
+      "version": "2.14.2"
+    },
+    {
+      "name": "guard-compat",
+      "version": "1.2.1"
+    },
+    {
+      "name": "guard-rspec",
+      "version": "4.7.3"
+    },
+    {
+      "name": "i18n",
+      "version": "0.9.5"
+    },
+    {
+      "name": "listen",
+      "version": "3.1.5"
+    },
+    {
+      "name": "lumberjack",
+      "version": "1.0.12"
+    },
+    {
+      "name": "method_source",
+      "version": "0.9.0"
+    },
+    {
+      "name": "mini_portile2",
+      "version": "2.0.0"
+    },
+    {
+      "name": "minitest",
+      "version": "5.11.3"
+    },
+    {
+      "name": "mustermann",
+      "version": "1.0.1"
+    },
+    {
+      "name": "nenv",
+      "version": "0.3.0"
+    },
+    {
+      "name": "nokogiri",
+      "version": "1.6.7.2"
+    },
+    {
+      "name": "notiffany",
+      "version": "0.1.1"
+    },
+    {
+      "name": "pg",
+      "version": "1.0.0"
+    },
+    {
+      "name": "pry",
+      "version": "0.11.3"
+    },
+    {
+      "name": "puma",
+      "version": "3.12.0"
+    },
+    {
+      "name": "rack",
+      "version": "2.0.4"
+    },
+    {
+      "name": "rack-protection",
+      "version": "2.0.0"
+    },
+    {
+      "name": "rb-fsevent",
+      "version": "0.10.2"
+    },
+    {
+      "name": "rb-inotify",
+      "version": "0.9.10"
+    },
+    {
+      "name": "redis",
+      "version": "3.3.5"
+    },
+    {
+      "name": "rspec",
+      "version": "3.7.0"
+    },
+    {
+      "name": "rspec-core",
+      "version": "3.7.1"
+    },
+    {
+      "name": "rspec-expectations",
+      "version": "3.7.0"
+    },
+    {
+      "name": "rspec-mocks",
+      "version": "3.7.0"
+    },
+    {
+      "name": "rspec-support",
+      "version": "3.7.1"
+    },
+    {
+      "name": "ruby_dep",
+      "version": "1.5.0"
+    },
+    {
+      "name": "shellany",
+      "version": "0.0.1"
+    },
+    {
+      "name": "sidekiq",
+      "version": "4.2.10"
+    },
+    {
+      "name": "sinatra",
+      "version": "2.0.0"
+    },
+    {
+      "name": "slim",
+      "version": "3.0.9"
+    },
+    {
+      "name": "spring",
+      "version": "2.0.2"
+    },
+    {
+      "name": "temple",
+      "version": "0.8.0"
+    },
+    {
+      "name": "thor",
+      "version": "0.20.0"
+    },
+    {
+      "name": "thread_safe",
+      "version": "0.3.6"
+    },
+    {
+      "name": "tilt",
+      "version": "2.0.8"
+    },
+    {
+      "name": "tzinfo",
+      "version": "1.2.5"
+    }
+  ],
+  "affections": []
+}

scanner/testdata/bundler-yarn/yarn/package.json

+{
+  "name": "js-yarn",
+  "version": "1.0.0",
+  "description": "Test project for js-yarn",
+  "dependencies": {
+    "@nuxtjs/axios": "latest",
+    "@nuxtjs/browserconfig": "^0.0.6",
+    "@nuxtjs/icon": "^1.1.0",
+    "@nuxtjs/manifest": "^1.7.0",
+    "@nuxtjs/markdownit": "^1.1.2",
+    "@nuxtjs/optimize": "^1.1.1",
+    "@nuxtjs/sitemap": "^0.0.3",
+    "@nuxtjs/workbox": "^1.2.0",
+    "emitter": "git+https://github.com/Mango/emitter.git#0.0.7",
+    "highlight.js": "^9.12.0",
+    "moment": "^2.18.1",
+    "normalize.css": "^7.0.0",
+    "nuxt": "^2.7.1",
+    "nuxtent": "latest",
+    "v-tooltip": "^2.0.0-rc.5",
+    "vue-disqus": "^2.0.3",
+    "vue-moment": "^2.0.2",
+    "vue-scrollto": "^2.7.8",
+    "vue-slideout": "^1.7.0"
+  },
+  "devDependencies": {
+    "babel-eslint": "^7.2.3",
+    "browser-env": "^3.2.0",
+    "chai": "^4.1.2",
+    "coveralls": "^3.0.0",
+    "eslint": "^4.3.0",
+    "eslint-config-standard": "^10.2.1",
+    "eslint-loader": "^1.9.0",
+    "eslint-plugin-html": "^3.1.1",
+    "eslint-plugin-import": "^2.7.0",
+    "eslint-plugin-node": "^5.1.1",
+    "eslint-plugin-promise": "^3.5.0",
+    "eslint-plugin-standard": "^3.0.1",
+    "husky": "^0.14.3",
+    "jsdom": "^11.2.0",
+    "mocha": "^3.5.3",
+    "mocha-webpack": "^0.7.0",
+    "nyc": "^11.2.1",
+    "rimraf": "^2.6.2",
+    "rupture": "^0.6.2",
+    "stylus": "^0.54.5",
+    "stylus-loader": "^3.0.1"
+  },
+  "scripts": {
+    "test": "echo 'test'"
+  },
+  "engines": {
+    "node": ">=10.0.0 <11.0.0"
+  }
+}