Implement Proxy Interfacer for Handling NPM Data from deps.dev and repo data sources
Problem
The NPM License Interfacer lacks the capability to handle data from both deps.dev
and registry
sources as it does not differentiate between data source origins.
Solution
Implement a new NPM License Interfacer to act strictly as a proxy, forwarding data directly to the Processor
without any filtering or alteration.
Implementation Plan
-
Extend Package
struct with a source field https://gitlab.com/gitlab-org/security-products/license-db/license-interfacer/-/merge_requests/106+s [ ] renamenpm.Registry
tonpm.RepoRegistry
-
Introduce new NPM registry struct to proxy messages to npm.RepoRegistry
(https://gitlab.com/gitlab-org/security-products/license-db/license-interfacer/-/merge_requests/106+s) -
Extend proxy registry's Handle
implementation to return on the data we get as a*data.License
whenmsg.Source == "deps.dev"
(https://gitlab.com/gitlab-org/security-products/license-db/license-interfacer/-/merge_requests/106+s) -
Modify the License
struct in the NPM License Interfacer to incorporate asource
field, ensuring that data origin is captured for each entry (https://gitlab.com/gitlab-org/security-products/license-db/license-interfacer/-/merge_requests/107) -
Add the value from msg.Source
to theSource
of*data.License
(https://gitlab.com/gitlab-org/security-products/license-db/license-interfacer/-/merge_requests/107) -
Update the system documentation to clearly articulate the proxy role of the Interfacer
and illustrate how it handles data routing (https://gitlab.com/gitlab-org/security-products/license-db/deployment/-/merge_requests/265) -
Extend existing unit tests and add new ones for the new forwarding behaviour (https://gitlab.com/gitlab-org/security-products/license-db/license-interfacer/-/merge_requests/106) -
Create new release (https://gitlab.com/gitlab-org/security-products/license-db/deployment/-/merge_requests/265) -
Deploy to dev and prod
Outline
Here is a guide indicating the essential code changes that act as an entry point:
commit bd918d119620affd8bca566062a418c1b3b63a6a
Author: Philip Cunningham <pcunningham@gitlab.com>
Date: Tue Apr 30 16:50:06 2024 +0100
Add scaffolding for NPM deps.dev
diff --git a/data/license.go b/data/license.go
index 3f1b7d3..dc765ec 100644
--- a/data/license.go
+++ b/data/license.go
@@ -20,6 +20,8 @@ type License struct {
PackageRegistry string `json:"package_registry"`
// Versions the versions
Versions []*Version `json:"versions"`
+ // Source repo or deps.dev
+ Source string `json:"source"`
}
// NewLicense creates a new license for the specified packageRegistry
diff --git a/data/package.go b/data/package.go
index de468d1..fa0659d 100644
--- a/data/package.go
+++ b/data/package.go
@@ -49,4 +49,5 @@ type Package struct {
Version string `json:"version,omitempty"`
Licenses []string `json:"licenses,omitempty"`
Metadata []byte `json:"metadata,omitempty"`
+ Source string `json:"source"` // repo or deps.dev
}
diff --git a/interfacers/npm/npm.go b/interfacers/npm/npm.go
index 37282f7..234e21a 100644
--- a/interfacers/npm/npm.go
+++ b/interfacers/npm/npm.go
@@ -8,89 +8,56 @@ import (
"gitlab.com/gitlab-org/security-products/license-db/license-interfacer/data"
)
-const (
- // DefaultRegistryURL points at the public NPM CouchDB registry server
- DefaultRegistryURL = "https://replicate.npmjs.com"
- // DefaultRegistryDatabase points at the public NPM CouchDB registry database
- DefaultRegistryDatabase = "registry"
-)
-
-// Registry the NPM registry struct
-type Registry struct {
- url string
- database string
- auth *Auth
-}
+type ProxyRegistry struct{ repoRegistry *RepoRegistry }
-// NewRegistry creates a new NPM registry struct
-func NewRegistry(opts ...func(opts *Registry)) *Registry {
- npm := &Registry{url: DefaultRegistryURL, database: DefaultRegistryDatabase, auth: NoAuth}
+func NewRegistry(opts ...func(opts *RepoRegistry)) *ProxyRegistry {
+ npm := &RepoRegistry{url: DefaultRegistryURL, database: DefaultRegistryDatabase, auth: NoAuth}
for _, opt := range opts {
opt(npm)
}
- return npm
+ return &ProxyRegistry{repoRegistry: npm}
}
-// RegistryName returns the registry name
-func (npm *Registry) RegistryName() string {
+func (npm *ProxyRegistry) RegistryName() string {
return "npm"
}
-// Handle deals with incoming package messages from the NPM feeder
-func (npm *Registry) Handle(_ context.Context, pkgLogger *zerolog.Logger, msg *data.Package) (*data.License, error) {
- couchDB, err := NewCouchDB(pkgLogger, npm.url, npm.database, npm.auth.user, npm.auth.password)
- if err != nil {
- return nil, err
- }
-
- start := msg.Package
- end := string(msg.Metadata)
-
- result := data.NewLicense(npm.RegistryName())
-
- for event := range couchDB.AllDocs(start, end, 1000) {
- if event.Error != nil {
- pkgLogger.Error().Str("start", start).Str("end", start).Err(event.Error).Msg("couchdb error event received")
-
- return result, event.Error
- }
-
- versionLicenses := event.Message.versionLicenses()
- if len(versionLicenses) == 0 {
- pkgLogger.Warn().Str("package", event.Message.Name).Msg("empty version licenses")
- }
-
- for version, licenses := range versionLicenses {
- result.Versions = append(result.Versions, &data.Version{
- PackageName: event.Message.Name,
- Version: version,
- License: licenses,
- })
- }
+func (npm *ProxyRegistry) Handle(ctx context.Context, pkgLogger *zerolog.Logger, msg *data.Package) (*data.License, error) {
+ // Simply forward the message on
+ if msg.Source == "deps.dev" {
+ lic := data.NewLicense(npm.RegistryName())
+ lic.Source = msg.Source
+ lic.Versions = append(lic.Versions, &data.Version{
+ PackageName: msg.Package,
+ Version: msg.Version,
+ License: msg.Licenses,
+ })
+ return lic, nil
}
- return result, nil
+ // Process the message
+ return npm.repoRegistry.handle(ctx, pkgLogger, msg)
}
// WithNpmRegistryURL sets the NPM registry URL which is an optional argument
-func WithNpmRegistryURL(npmRegistryURL string) func(*Registry) {
- return func(f *Registry) {
+func WithNpmRegistryURL(npmRegistryURL string) func(*RepoRegistry) {
+ return func(f *RepoRegistry) {
f.url = npmRegistryURL
}
}
// WithNpmRegistryDatabase sets the NPM registry database which is an optional argument
-func WithNpmRegistryDatabase(database string) func(*Registry) {
- return func(f *Registry) {
+func WithNpmRegistryDatabase(database string) func(*RepoRegistry) {
+ return func(f *RepoRegistry) {
f.database = database
}
}
// WithNpmRegistryAuth sets the NPM registry password which is an optional argument
-func WithNpmRegistryAuth(auth *Auth) func(*Registry) {
- return func(f *Registry) {
+func WithNpmRegistryAuth(auth *Auth) func(*RepoRegistry) {
+ return func(f *RepoRegistry) {
f.auth = auth
}
}
diff --git a/interfacers/npm/repo.go b/interfacers/npm/repo.go
new file mode 100644
index 0000000..d7930e3
--- /dev/null
+++ b/interfacers/npm/repo.go
@@ -0,0 +1,69 @@
+package npm
+
+import (
+ "context"
+
+ "github.com/rs/zerolog"
+
+ "gitlab.com/gitlab-org/security-products/license-db/license-interfacer/data"
+)
+
+const (
+ // DefaultRegistryURL points at the public NPM CouchDB registry server
+ DefaultRegistryURL = "https://replicate.npmjs.com"
+ // DefaultRegistryDatabase points at the public NPM CouchDB registry database
+ DefaultRegistryDatabase = "registry"
+)
+
+// RepoRegistry the NPM registry struct
+type RepoRegistry struct {
+ url string
+ database string
+ auth *Auth
+}
+
+func newRepoRegistry(opts ...func(opts *RepoRegistry)) *RepoRegistry {
+ npm := &RepoRegistry{url: DefaultRegistryURL, database: DefaultRegistryDatabase, auth: NoAuth}
+
+ for _, opt := range opts {
+ opt(npm)
+ }
+
+ return npm
+}
+
+func (npm *RepoRegistry) handle(_ context.Context, pkgLogger *zerolog.Logger, msg *data.Package) (*data.License, error) {
+ couchDB, err := NewCouchDB(pkgLogger, npm.url, npm.database, npm.auth.user, npm.auth.password)
+ if err != nil {
+ return nil, err
+ }
+
+ start := msg.Package
+ end := string(msg.Metadata)
+
+ result := data.NewLicense("npm")
+ result.Source = "repo"
+
+ for event := range couchDB.AllDocs(start, end, 1000) {
+ if event.Error != nil {
+ pkgLogger.Error().Str("start", start).Str("end", start).Err(event.Error).Msg("couchdb error event received")
+
+ return result, event.Error
+ }
+
+ versionLicenses := event.Message.versionLicenses()
+ if len(versionLicenses) == 0 {
+ pkgLogger.Warn().Str("package", event.Message.Name).Msg("empty version licenses")
+ }
+
+ for version, licenses := range versionLicenses {
+ result.Versions = append(result.Versions, &data.Version{
+ PackageName: event.Message.Name,
+ Version: version,
+ License: licenses,
+ })
+ }
+ }
+
+ return result, nil
+}
Edited by Philip Cunningham