Skip to content
Commits on Source (29)
Expand all Hide whitespace changes
Inline Side-by-side
CHANGELOG.md
View file @ 7f45f273
## [3.88.0](https://gitlab.com/gitlab-org/container-registry/compare/v3.87.0-gitlab...v3.88.0-gitlab) (2023-12-19)
### ✨ Features ✨
* add referrers data to internal List Tags API ([fa28ee6](https://gitlab.com/gitlab-org/container-registry/commit/fa28ee6d333f265fc80aeda127f61a3ceb6462df))
* **datastore:** importer: use progressbar to show import progress ([3be86b9](https://gitlab.com/gitlab-org/container-registry/commit/3be86b9622c635b5b446d763d08758923f991656))
* **importer:** import command: expose tag concurrency option gcs ([8a9fdcd](https://gitlab.com/gitlab-org/container-registry/commit/8a9fdcd044cdd4149dc258e9a0522f25bd10712d))
### ⚙️ Build ⚙️
* **deps:** update module cloud.google.com/go/storage to v1.36.0 ([117dad0](https://gitlab.com/gitlab-org/container-registry/commit/117dad048a2c467e8b89284e326974d24db1b483))
* **deps:** update module github.com/data-dog/go-sqlmock to v1.5.1 ([d7763ea](https://gitlab.com/gitlab-org/container-registry/commit/d7763ea0792fbd169bc5e6b0e8d352c319dcafcf))
* **deps:** update module github.com/jackc/pgx/v5 to v5.5.1 ([10b8550](https://gitlab.com/gitlab-org/container-registry/commit/10b85509deda2ff37a8cff78e500d87e7653bdf1))
* **deps:** update module github.com/jszwec/csvutil to v1.9.0 ([8e181b1](https://gitlab.com/gitlab-org/container-registry/commit/8e181b17501220a84c2c2e6287c724e151682231))
* **deps:** update module github.com/spf13/viper to v1.18.1 ([1bc5bca](https://gitlab.com/gitlab-org/container-registry/commit/1bc5bca7c74663417fa75f1276f1971792626127))
* **deps:** update module github.com/xanzy/go-gitlab to v0.95.1 ([2c10193](https://gitlab.com/gitlab-org/container-registry/commit/2c101935a9b936aec3f549899a0eea001525966a))
* **deps:** update module github.com/xanzy/go-gitlab to v0.95.2 ([7f1d8ec](https://gitlab.com/gitlab-org/container-registry/commit/7f1d8ec9e2a3c689ce0532bad61765d03ee09e4d))
* **deps:** update module google.golang.org/api to v0.153.0 ([1a9edc1](https://gitlab.com/gitlab-org/container-registry/commit/1a9edc11d4ae3ebb61de69c92d3a42b26a894cf7))
* **deps:** update module google.golang.org/api to v0.154.0 ([bf21ad7](https://gitlab.com/gitlab-org/container-registry/commit/bf21ad75336e11015f8bb37226197de44e05d95e))
## [3.87.0](https://gitlab.com/gitlab-org/container-registry/compare/v3.86.2-gitlab...v3.87.0-gitlab) (2023-12-05)
......
cmd/internal/release-cli/client/client.go
View file @ 7f45f273
......@@ -50,7 +50,7 @@ func (g *Client) CreateCommit(projectID int, change []byte, fileName, commitMess
return commit, err
}
func (g *Client) CreateMergeRequest(projectID int, sourceBranch *gitlab.Branch, description, targetBranch, title string, labels *gitlab.Labels, reviwerIDs []int) (*gitlab.MergeRequest, error) {
func (g *Client) CreateMergeRequest(projectID int, sourceBranch *gitlab.Branch, description, targetBranch, title string, labels *gitlab.LabelOptions, reviwerIDs []int) (*gitlab.MergeRequest, error) {
mr, _, err := g.client.MergeRequests.CreateMergeRequest(projectID, &gitlab.CreateMergeRequestOptions{
SourceBranch: gitlab.String(sourceBranch.Name),
TargetBranch: &targetBranch,
......
cmd/internal/release-cli/cmd/gdk.go
View file @ 7f45f273
......@@ -36,7 +36,7 @@ var gdkCmd = &cobra.Command{
log.Fatal(err)
}
labels := &gitlab.Labels{
labels := &gitlab.LabelOptions{
"workflow::ready for review",
"group::container registry",
"devops::package",
......
cmd/internal/release-cli/cmd/k8s.go
View file @ 7f45f273
......@@ -42,7 +42,7 @@ var k8sCmd = &cobra.Command{
log.Fatal(err)
}
labels := &gitlab.Labels{
labels := &gitlab.LabelOptions{
"workflow::ready for review",
"team::Delivery",
"Service::Container Registry",
......
docs/spec/gitlab/api.md
View file @ 7f45f273
......@@ -133,14 +133,15 @@ information about each tag and not just their name.
GET /gitlab/v1/repositories/<path>/tags/list/
```
| Attribute | Type | Required | Default | Description |
|------------|--------|----------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `path` | String | Yes | | The full path of the target repository. Equivalent to the `name` parameter in the `/v2/` API, described in the [OCI Distribution Spec](https://github.com/opencontainers/distribution-spec/blob/main/spec.md). The same pattern validation applies. |
| `before` | String | No | | Query parameter used as marker for pagination. Set this to the tag name lexicographically _before_ which (exclusive) you want the requested page to start. The value of this query parameter must be a valid tag name. More precisely, it must respect the `[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}` pattern as defined in the OCI Distribution spec [here](https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-manifests). Otherwise, an `INVALID_QUERY_PARAMETER_VALUE` error is returned. Cannot be used in conjunction with `last`. |
| `last` | String | No | | Query parameter used as marker for pagination. Set this to the tag name lexicographically after which (exclusive) you want the requested page to start. The value of this query parameter must be a valid tag name. More precisely, it must respect the `[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}` pattern as defined in the OCI Distribution spec [here](https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-manifests). Otherwise, an `INVALID_QUERY_PARAMETER_VALUE` error is returned. |
| `n` | String | No | 100 | Query parameter used as limit for pagination. Defaults to 100. Must be a positive integer between `1` and `1000` (inclusive). If the value is not a valid integer, the `INVALID_QUERY_PARAMETER_TYPE` error is returned. If the value is a valid integer but is out of the rage then an `INVALID_QUERY_PARAMETER_VALUE` error is returned. |
| `name` | String | No | | Tag name filter. If set, tags are filtered using a partial match against its value. Does not support regular expressions. Only lowercase and uppercase letters, digits, underscores, periods, and hyphen characters are allowed. Maximum of 128 characters. It must respect the `[a-zA-Z0-9._-]{1,128}` pattern. If the value is not valid, the `INVALID_QUERY_PARAMETER_VALUE` error is returned. |
| `sort` | String | No | "name" | Sort tags by field in ascending or descending order. Prefix field with the `-` sign to sort in descending order according to the [JSON API spec](https://jsonapi.org/format/#fetching-sorting). |
| Attribute | Type | Required | Default | Description |
|-------------|--------|----------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `path` | String | Yes | | The full path of the target repository. Equivalent to the `name` parameter in the `/v2/` API, described in the [OCI Distribution Spec](https://github.com/opencontainers/distribution-spec/blob/main/spec.md). The same pattern validation applies. |
| `before` | String | No | | Query parameter used as marker for pagination. Set this to the tag name lexicographically _before_ which (exclusive) you want the requested page to start. The value of this query parameter must be a valid tag name. More precisely, it must respect the `[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}` pattern as defined in the OCI Distribution spec [here](https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-manifests). Otherwise, an `INVALID_QUERY_PARAMETER_VALUE` error is returned. Cannot be used in conjunction with `last`. |
| `last` | String | No | | Query parameter used as marker for pagination. Set this to the tag name lexicographically after which (exclusive) you want the requested page to start. The value of this query parameter must be a valid tag name. More precisely, it must respect the `[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}` pattern as defined in the OCI Distribution spec [here](https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-manifests). Otherwise, an `INVALID_QUERY_PARAMETER_VALUE` error is returned. |
| `n` | String | No | 100 | Query parameter used as limit for pagination. Defaults to 100. Must be a positive integer between `1` and `1000` (inclusive). If the value is not a valid integer, the `INVALID_QUERY_PARAMETER_TYPE` error is returned. If the value is a valid integer but is out of the rage then an `INVALID_QUERY_PARAMETER_VALUE` error is returned. |
| `name` | String | No | | Tag name filter. If set, tags are filtered using a partial match against its value. Does not support regular expressions. Only lowercase and uppercase letters, digits, underscores, periods, and hyphen characters are allowed. Maximum of 128 characters. It must respect the `[a-zA-Z0-9._-]{1,128}` pattern. If the value is not valid, the `INVALID_QUERY_PARAMETER_VALUE` error is returned. |
| `sort` | String | No | "name" | Sort tags by field in ascending or descending order. Prefix field with the `-` sign to sort in descending order according to the [JSON API spec](https://jsonapi.org/format/#fetching-sorting). |
| `referrers` | String | No | | When set to "true", each tag details object is appended with a `referrers` collection, containing one object per referrer of the corresponding tagged manifest. A referrer is an OCI manifest descriptor which has a `subject` relationship with the specified manifest, generally used to link artifacts to that manifest. |
#### Pagination
......
go.mod
View file @ 7f45f273
......@@ -3,9 +3,9 @@ module github.com/docker/distribution
go 1.18
require (
cloud.google.com/go/storage v1.35.1
cloud.google.com/go/storage v1.36.0
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/DATA-DOG/go-sqlmock v1.5.1
github.com/Shopify/toxiproxy/v2 v2.7.0
github.com/alicebob/miniredis/v2 v2.31.0
github.com/aws/aws-sdk-go v1.46.7
......@@ -23,8 +23,8 @@ require (
github.com/gorilla/mux v1.8.1
github.com/hashicorp/go-multierror v1.1.1
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa
github.com/jackc/pgx/v5 v5.5.0
github.com/jszwec/csvutil v1.8.0
github.com/jackc/pgx/v5 v5.5.1
github.com/jszwec/csvutil v1.9.0
github.com/mitchellh/mapstructure v1.5.0
github.com/ncw/swift v1.0.53
github.com/olekukonko/tablewriter v0.0.5
......@@ -33,20 +33,21 @@ require (
github.com/prometheus/client_golang v1.17.0
github.com/redis/go-redis/v9 v9.3.0
github.com/rubenv/sql-migrate v1.5.2
github.com/schollz/progressbar/v3 v3.14.1
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.8.0
github.com/spf13/viper v1.17.0
github.com/spf13/viper v1.18.1
github.com/stretchr/testify v1.8.4
github.com/trim21/go-redis-prometheus v0.0.0
github.com/vmihailenco/msgpack/v5 v5.4.1
github.com/xanzy/go-gitlab v0.94.0
github.com/xanzy/go-gitlab v0.95.2
gitlab.com/gitlab-org/labkit v1.21.0
go.uber.org/automaxprocs v1.5.3
golang.org/x/crypto v0.16.0
golang.org/x/oauth2 v0.15.0
golang.org/x/sync v0.5.0
golang.org/x/time v0.5.0
google.golang.org/api v0.152.0
google.golang.org/api v0.154.0
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
gopkg.in/yaml.v2 v2.4.0
)
......@@ -71,9 +72,11 @@ require (
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dnaeon/go-vcr v1.0.1 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-gorp/gorp/v3 v3.1.0 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gofrs/uuid v4.4.0+incompatible // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
......@@ -98,6 +101,7 @@ require (
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-runewidth v0.0.12 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/oklog/ulid/v2 v2.0.2 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
......@@ -105,30 +109,36 @@ require (
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/sagikazarmark/locafero v0.3.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.10.0 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/yuin/gopher-lua v1.1.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect
go.opentelemetry.io/otel v1.21.0 // indirect
go.opentelemetry.io/otel/metric v1.21.0 // indirect
go.opentelemetry.io/otel/trace v1.21.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/term v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect
google.golang.org/grpc v1.59.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
......
go.sum
View file @ 7f45f273
This diff is collapsed.
log/log.go
View file @ 7f45f273
......@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"io"
"runtime"
"strings"
......@@ -76,8 +77,9 @@ func WithLogger(ctx context.Context, logger Logger) context.Context {
}
type logOptions struct {
ctx context.Context
keys []interface{}
ctx context.Context
keys []interface{}
writer io.Writer
}
type logOpt func(o *logOptions)
......@@ -100,6 +102,12 @@ func WithKeys(keys ...interface{}) logOpt {
}
}
func WithWriter(w io.Writer) logOpt {
return func(o *logOptions) {
o.writer = w
}
}
// GetLogger returns a Logger based on a logrus Entry.
func GetLogger(opts ...logOpt) Logger {
cfg := &logOptions{ctx: context.Background()}
......@@ -107,7 +115,12 @@ func GetLogger(opts ...logOpt) Logger {
o(cfg)
}
return &wrapper{getLogrusLogger(cfg.ctx, cfg.keys...)}
l := getLogrusLogger(cfg.ctx, cfg.keys...)
if cfg.writer != nil {
l.Logger.Out = cfg.writer
}
return &wrapper{l}
}
// GetLogrusLogger returns the logrus logger for the context. If one more keys
......
registry/datastore/importer.go
View file @ 7f45f273
......@@ -7,6 +7,7 @@ import (
"net"
"net/http"
"os"
"path/filepath"
"sync"
"syscall"
"time"
......@@ -23,12 +24,28 @@ import (
"github.com/jackc/pgx/v5/pgconn"
"github.com/opencontainers/go-digest"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/schollz/progressbar/v3"
"google.golang.org/api/googleapi"
)
var (
errNegativeTestingDelay = errors.New("negative testing delay")
errManifestSkip = errors.New("the manifest is invalid and its (pre)import should be skipped")
commonBarOptions = []progressbar.Option{
progressbar.OptionSetElapsedTime(true),
progressbar.OptionShowCount(),
progressbar.OptionSetPredictTime(false),
progressbar.OptionShowElapsedTimeOnFinish(),
progressbar.OptionShowDescriptionAtLineEnd(),
progressbar.OptionShowIts(),
progressbar.OptionSetTheme(progressbar.Theme{
Saucer: "=",
SaucerHead: ">",
SaucerPadding: " ",
BarStart: "[",
BarEnd: "]",
}),
}
)
const mtOctetStream = "application/octet-stream"
......@@ -49,6 +66,7 @@ type Importer struct {
dryRun bool
tagConcurrency int
rowCount bool
showProgressBar bool
testingDelay time.Duration
preImportRetryTimeout time.Duration
}
......@@ -117,6 +135,11 @@ func WithPreImportRetryTimeout(d time.Duration) ImporterOption {
}
}
// WithProgressBar shows a progress bar and writes detailed logs to a file.
func WithProgressBar(imp *Importer) {
imp.showProgressBar = true
}
// NewImporter creates a new Importer.
func NewImporter(db *DB, registry distribution.Namespace, opts ...ImporterOption) *Importer {
imp := &Importer{
......@@ -607,6 +630,13 @@ func (imp *Importer) importTags(ctx context.Context, fsRepo distribution.Reposit
close(tagResChan)
}()
opts := append(commonBarOptions, progressbar.OptionSetDescription(fmt.Sprintf("importing tags in %s", dbRepo.Path)), progressbar.OptionSetItsString("tags"))
bar := progressbar.NewOptions(total, opts...)
defer func() {
bar.Finish()
bar.Close()
}()
// Consume the tag lookup details serially. In the ideal case, we only need
// retrieve the manifest from the database and associate it with a tag. This
// is fast enough that concurrency really isn't warranted here as well.
......@@ -616,6 +646,7 @@ func (imp *Importer) importTags(ctx context.Context, fsRepo distribution.Reposit
fsTag := tRes.name
desc := tRes.desc
err := tRes.err
bar.Add(1)
l := l.WithFields(log.Fields{"tag_name": fsTag, "count": i, "total": total, "digest": desc.Digest})
l.Info("importing tag")
......@@ -733,9 +764,17 @@ func (imp *Importer) preImportTaggedManifests(ctx context.Context, fsRepo distri
l := log.GetLogger(log.WithContext(ctx)).WithFields(log.Fields{"repository": dbRepo.Path, "total": total})
l.Info("processing tags")
opts := append(commonBarOptions, progressbar.OptionSetDescription(fmt.Sprintf("pre importing manifests in %s", dbRepo.Path)), progressbar.OptionSetItsString("manifests"))
bar := progressbar.NewOptions(total, opts...)
defer func() {
bar.Finish()
bar.Close()
}()
for i, fsTag := range fsTags {
l := l.WithFields(log.Fields{"tag_name": fsTag, "count": i + 1})
l.Info("processing tag")
bar.Add(1)
// read tag details from the filesystem
desc, err := tagService.Get(ctx, fsTag)
......@@ -1030,12 +1069,44 @@ func (imp *Importer) doImport(ctx context.Context, required step, steps ...step)
}
}
l := log.GetLogger(log.WithContext(ctx)).WithFields(log.Fields{
var f *os.File
bar := progressbar.NewOptions(-1,
progressbar.OptionShowElapsedTimeOnFinish(),
progressbar.OptionShowDescriptionAtLineEnd(),
progressbar.OptionSetVisibility(imp.showProgressBar),
)
defer bar.Close()
commonBarOptions = append(commonBarOptions, progressbar.OptionSetVisibility(imp.showProgressBar))
if imp.showProgressBar {
fn := fmt.Sprintf("%s-registry-import.log", time.Now().Format(time.RFC3339))
f, err = os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
if err != nil {
return fmt.Errorf("opening log file: %w", err)
}
wd, err := os.Getwd()
if err != nil {
return fmt.Errorf("getting working directory: %w", err)
}
// A little hacky, but we can use a progress bar to show the overall import
// progress by printing the bar with different descriptions. Otherwise, the
// progress bars and a regular logger would step over one another.
bar.Add(1) // Give the bar some state so we can print it.
imp.printBar(bar, fmt.Sprintf("registry import starting, detailed log written to: %s", filepath.Join(wd, fn)))
} else {
f = os.Stdout
}
l := log.GetLogger(log.WithContext(ctx), log.WithWriter(f)).WithFields(log.Fields{
"pre_import": pre,
"repository_import": repos,
"common_blobs": blobs,
"dry_run": imp.dryRun,
})
ctx = log.WithLogger(ctx, l)
if imp.requireEmptyDatabase {
......@@ -1061,21 +1132,31 @@ func (imp *Importer) doImport(ctx context.Context, required step, steps ...step)
l.Info("starting metadata import")
if pre {
imp.printBar(bar, "step one: import manifests")
if err := imp.preImportAllRepositories(ctx); err != nil {
return fmt.Errorf("pre importing all repositories: %w", err)
}
}
if repos {
imp.printBar(bar, "step two: import tags")
if err := imp.importAllRepositories(ctx); err != nil {
return fmt.Errorf("importing all repositories: %w", err)
}
}
if blobs {
imp.printBar(bar, "step three: import blobs")
if err := imp.importBlobs(ctx); err != nil {
return fmt.Errorf("importing blobs: %w", err)
}
}
imp.printBar(bar, "registry import complete")
t := time.Since(start).Seconds()
if imp.rowCount {
......@@ -1172,8 +1253,16 @@ func (imp *Importer) importBlobs(ctx context.Context) error {
l := log.GetLogger(log.WithContext(ctx))
l.Info("importing all blobs")
opts := append(commonBarOptions, progressbar.OptionSetDescription("importing blobs"), progressbar.OptionSetItsString("blobs"))
bar := progressbar.NewOptions(-1, opts...)
defer func() {
bar.Finish()
bar.Close()
}()
if err := imp.registry.Blobs().Enumerate(ctx, func(desc distribution.Descriptor) error {
index++
bar.Add(1)
l.WithFields(log.Fields{"digest": desc.Digest, "count": index, "size": desc.Size}).Info("importing blob")
dbBlob, err := imp.blobStore.FindByDigest(ctx, desc.Digest)
......@@ -1413,3 +1502,18 @@ func (imp *Importer) PreImport(ctx context.Context, path string) error {
return nil
}
// printBar prints the bar if showProgressBar is enabled. Optionally, sets the
// bar description with the passed string. Passing in a string will mutate the
// bar description.
func (imp *Importer) printBar(b *progressbar.ProgressBar, s ...string) {
if !imp.showProgressBar {
return
}
if len(s) > 0 {
b.Describe(s[0])
}
fmt.Println(b)
}
registry/datastore/models/models.go
View file @ 7f45f273
......@@ -124,6 +124,12 @@ type TagDetail struct {
CreatedAt time.Time
UpdatedAt sql.NullTime
PublishedAt time.Time
Referrers []TagReferrerDetail
}
type TagReferrerDetail struct {
Digest string
ArtifactType string
}
type Blob struct {
......
registry/datastore/repository.go
View file @ 7f45f273
......@@ -44,13 +44,14 @@ const (
// FilterParams contains the specific filters used to get
// the request results from the repositoryStore.
type FilterParams struct {
SortOrder SortOrder
OrderBy string
Name string
BeforeEntry string
LastEntry string
PublishedAt string
MaxEntries int
SortOrder SortOrder
OrderBy string
Name string
BeforeEntry string
LastEntry string
PublishedAt string
MaxEntries int
IncludeReferrers bool
}
// RepositoryReader is the interface that defines read operations for a repository store.
......@@ -720,6 +721,98 @@ func scanFullTagsDetail(rows *sql.Rows) ([]*models.TagDetail, error) {
return tt, nil
}
// The query for this method takes a list of TagDetails and returns a list of
// manifests which are referrers to those tags - i.e. `subject_id` points to
// one of the tags. The method modifies the `tags` collection by populating
// the `Referrers` field with the query results.
func (s *repositoryStore) appendTagsDetailReferrers(ctx context.Context, r *models.Repository, tags []*models.TagDetail) error {
if len(tags) == 0 {
return nil
}
sbjDigests := make([]string, 0, len(tags))
for _, tag := range tags {
nd, err := NewDigest(tag.Digest)
if err != nil {
return err
}
sbjDigests = append(sbjDigests, fmt.Sprintf("'%s'", nd))
}
q := fmt.Sprintf(
`SELECT
encode(m.digest, 'hex') AS digest,
COALESCE(at.media_type, cmt.media_type) AS artifact_type,
encode(ms.digest, 'hex') AS subject_digest
FROM
manifests AS m
JOIN manifests AS ms ON m.top_level_namespace_id = ms.top_level_namespace_id
AND m.subject_id = ms.id
LEFT JOIN media_types AS at ON at.id = m.artifact_media_type_id
LEFT JOIN media_types AS cmt ON cmt.id = m.configuration_media_type_id
WHERE
m.top_level_namespace_id = $1
AND m.repository_id = $2
AND m.subject_id IN (
SELECT
id
FROM
manifests
WHERE
top_level_namespace_id = $1
AND repository_id = $2
AND digest IN (
SELECT
decode(n, 'hex')
FROM
unnest(ARRAY[%s]) AS n))`,
strings.Join(sbjDigests, ","))
rows, err := s.db.QueryContext(ctx, q, r.NamespaceID, r.ID)
if err != nil {
return err
}
defer rows.Close()
refMap := make(map[string][]models.TagReferrerDetail)
var (
dgst, sbjDgst Digest
at, sbjStr string
)
for rows.Next() {
if err = rows.Scan(&dgst, &at, &sbjDgst); err != nil {
return fmt.Errorf("scanning referrer: %w", err)
}
d, err := dgst.Parse()
if err != nil {
return err
}
sbj, err := sbjDgst.Parse()
if err != nil {
return err
}
sbjStr = sbj.String()
if refMap[sbjStr] == nil {
refMap[sbjStr] = make([]models.TagReferrerDetail, 0)
}
refMap[sbjStr] = append(refMap[sbjStr], models.TagReferrerDetail{
Digest: d.String(),
ArtifactType: at,
})
}
if err := rows.Err(); err != nil {
return fmt.Errorf("scanning referrers: %w", err)
}
for _, tag := range tags {
tag.Referrers = refMap[tag.Digest.String()]
}
return nil
}
func sortTagsDesc(tags []*models.TagDetail) {
sort.SliceStable(tags, func(i int, j int) bool {
return tags[i].Name < tags[j].Name
......@@ -752,7 +845,17 @@ func (s *repositoryStore) TagsDetailPaginated(ctx context.Context, r *models.Rep
return nil, fmt.Errorf("finding tags detail with pagination: %w", err)
}
return scanFullTagsDetail(rows)
tags, err := scanFullTagsDetail(rows)
if err != nil {
return nil, err
}
if filters.IncludeReferrers {
if err := s.appendTagsDetailReferrers(ctx, r, tags); err != nil {
return nil, fmt.Errorf("populating referrers: %w", err)
}
}
return tags, nil
}
func tagsDetailPaginatedQuery(r *models.Repository, filters FilterParams) (string, []any) {
......
registry/handlers/api_gitlab_integration_test.go
View file @ 7f45f273
......@@ -24,6 +24,7 @@ import (
dbtestutil "github.com/docker/distribution/registry/datastore/testutil"
"github.com/docker/distribution/registry/handlers"
"github.com/docker/distribution/registry/internal/testutil"
"github.com/opencontainers/go-digest"
"github.com/stretchr/testify/require"
)
......@@ -1071,6 +1072,126 @@ func TestGitlabAPI_RepositoryTagsList_OmitEmptyConfigDigest(t *testing.T) {
require.NotContains(t, string(payload), "config_digest")
}
func TestGitlabAPI_RepositoryTagsList_IncludeReferrers(t *testing.T) {
env := newTestEnv(t)
t.Cleanup(env.Shutdown)
env.requireDB(t)
repoRef, err := reference.WithName("foo/bar")
require.NoError(t, err)
artifactType := "application/vnd.dev.cosign.artifact.sbom.v1+json"
mfst := seedRandomOCIManifest(t, env, repoRef.Name(), putByTag("apple"))
mfstRef1 := seedRandomOCIManifest(t, env, repoRef.Name(), putByTag("apple-sig-1"), withSubject(mfst))
mfstRef2 := seedRandomOCIManifest(t, env, repoRef.Name(), putByTag("apple-sig-2"),
withSubject(mfst), withArtifactType(artifactType))
seedRandomOCIManifest(t, env, repoRef.Name(), putByTag("banana"))
params := url.Values{
"referrers": {"true"},
}
tagsURL, err := env.builder.BuildGitlabV1RepositoryTagsURL(repoRef, params)
require.NoError(t, err)
resp, err := http.Get(tagsURL)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
var list []handlers.RepositoryTagResponse
dec := json.NewDecoder(resp.Body)
err = dec.Decode(&list)
require.NoError(t, err)
m := make(map[string]handlers.RepositoryTagResponse)
for _, tag := range list {
m[tag.Name] = tag
}
require.Equal(t, 2, len(m["apple"].Referrers))
require.Equal(t, 0, len(m["apple-sig"].Referrers))
require.Equal(t, 0, len(m["banana"].Referrers))
// check ref digests match signature digests
_, mb1, err := mfstRef1.Payload()
require.NoError(t, err)
_, mb2, err := mfstRef2.Payload()
require.NoError(t, err)
dgst1, dgst2 := digest.FromBytes(mb1), digest.FromBytes(mb2)
require.Contains(t, m["apple"].Referrers, handlers.RepositoryTagReferrerResponse{
Digest: dgst1.String(),
ArtifactType: mfstRef1.Manifest.Config.MediaType,
})
require.Contains(t, m["apple"].Referrers, handlers.RepositoryTagReferrerResponse{
Digest: dgst2.String(),
ArtifactType: artifactType,
})
}
func TestGitlabAPI_RepositoryTagsList_DoNotIncludeReferrersByDefault(t *testing.T) {
env := newTestEnv(t)
t.Cleanup(env.Shutdown)
env.requireDB(t)
repoRef, err := reference.WithName("foo/bar")
require.NoError(t, err)
artifactType := "application/vnd.dev.cosign.artifact.sbom.v1+json"
mfst := seedRandomOCIManifest(t, env, repoRef.Name(), putByTag("apple"))
seedRandomOCIManifest(t, env, repoRef.Name(), putByTag("apple-sig-1"), withSubject(mfst))
seedRandomOCIManifest(t, env, repoRef.Name(), putByTag("apple-sig-2"),
withSubject(mfst), withArtifactType(artifactType))
seedRandomOCIManifest(t, env, repoRef.Name(), putByTag("banana"))
// no referrers returned by default
tagsURL, err := env.builder.BuildGitlabV1RepositoryTagsURL(repoRef)
require.NoError(t, err)
resp, err := http.Get(tagsURL)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
var list []handlers.RepositoryTagResponse
dec := json.NewDecoder(resp.Body)
err = dec.Decode(&list)
require.NoError(t, err)
m := make(map[string]handlers.RepositoryTagResponse)
for _, tag := range list {
m[tag.Name] = tag
}
require.Equal(t, 0, len(m["apple"].Referrers))
require.Equal(t, 0, len(m["apple-sig"].Referrers))
require.Equal(t, 0, len(m["banana"].Referrers))
// no referrers returned if `referrers` param is set to something other than "true"
params := url.Values{
"referrers": {"false"},
}
tagsURL, err = env.builder.BuildGitlabV1RepositoryTagsURL(repoRef, params)
require.NoError(t, err)
resp, err = http.Get(tagsURL)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
dec = json.NewDecoder(resp.Body)
err = dec.Decode(&list)
require.NoError(t, err)
m = make(map[string]handlers.RepositoryTagResponse)
for _, tag := range list {
m[tag.Name] = tag
}
require.Equal(t, 0, len(m["apple"].Referrers))
require.Equal(t, 0, len(m["apple-sig"].Referrers))
require.Equal(t, 0, len(m["banana"].Referrers))
}
func TestGitlabAPI_SubRepositoryList(t *testing.T) {
env := newTestEnv(t)
t.Cleanup(env.Shutdown)
......
registry/handlers/repositories.go
View file @ 7f45f273
......@@ -75,6 +75,7 @@ const (
sortQueryParamKey = "sort"
publishedAtQueryParamKey = "published_at"
sortOrderDescPrefix = "-"
referrersQueryParamKey = "referrers"
defaultDryRunRenameOperationTimeout = 5 * time.Second
maxRepositoriesToRename = 1000
)
......@@ -315,14 +316,20 @@ func repositoryTagsDispatcher(ctx *Context, _ *http.Request) http.Handler {
// implementation details (such as sql.NullTime) without having to implement custom JSON serializers (and having to use
// our own implementations) for these types. This is therefore a precise representation of the API response structure.
type RepositoryTagResponse struct {
Name string `json:"name"`
Name string `json:"name"`
Digest string `json:"digest"`
ConfigDigest string `json:"config_digest,omitempty"`
MediaType string `json:"media_type"`
Size int64 `json:"size_bytes"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at,omitempty"`
PublishedAt string `json:"published_at,omitempty"`
Referrers []RepositoryTagReferrerResponse `json:"referrers,omitempty"`
}
type RepositoryTagReferrerResponse struct {
ArtifactType string `json:"artifactType"`
Digest string `json:"digest"`
ConfigDigest string `json:"config_digest,omitempty"`
MediaType string `json:"media_type"`
Size int64 `json:"size_bytes"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at,omitempty"`
PublishedAt string `json:"published_at,omitempty"`
}
func tagNameQueryParamValue(r *http.Request) string {
......@@ -413,6 +420,10 @@ func filterParamsFromRequest(r *http.Request) (datastore.FilterParams, error) {
filters.OrderBy, filters.SortOrder = getSortOrderParams(sort)
}
if q.Has(referrersQueryParamKey) && q.Get(referrersQueryParamKey) == "true" {
filters.IncludeReferrers = true
}
return filters, nil
}
......@@ -517,6 +528,15 @@ func (h *repositoryTagsHandler) GetTags(w http.ResponseWriter, r *http.Request)
if t.UpdatedAt.Valid {
d.UpdatedAt = timeToString(t.UpdatedAt.Time)
}
if t.Referrers != nil {
d.Referrers = make([]RepositoryTagReferrerResponse, 0, len(t.Referrers))
for _, td := range t.Referrers {
d.Referrers = append(d.Referrers, RepositoryTagReferrerResponse{
Digest: td.Digest,
ArtifactType: td.ArtifactType,
})
}
}
resp = append(resp, d)
}
......
registry/root.go
View file @ 7f45f273
......@@ -64,7 +64,9 @@ func init() {
ImportCmd.Flags().BoolVarP(&importAllRepos, "step-two", "2", false, "perform step two of a multi-step import: alias for `all-repositories`")
ImportCmd.Flags().BoolVarP(&importCommonBlobs, "common-blobs", "B", false, "import all blob metadata from common storage")
ImportCmd.Flags().BoolVarP(&importCommonBlobs, "step-three", "3", false, "perform step three of a multi-step import: alias for `common-blobs`")
ImportCmd.Flags().BoolVarP(&logToSTDOUT, "log-to-stdout", "l", false, "write detailed log to std instead of showing progress bars")
ImportCmd.Flags().StringVarP(&debugAddr, "debug-server", "s", "", "run a pprof debug server at <address:port>")
ImportCmd.Flags().VarP(nullableInt{&tagConcurrency}, "tag-concurrency", "t", "limit the number of tags to retrieve concurrently, only applicable on gcs backed storage")
InventoryCmd.Flags().StringVarP(&format, "format", "f", "text", "which format to write output to, text output produces an additional summary for convenience, options: text, json, csv")
InventoryCmd.Flags().BoolVarP(&countTags, "tag-count", "t", true, "count repository tags, set this to false to increase inventory speed")
......@@ -88,6 +90,7 @@ var (
importCommonBlobs bool
importAllRepos bool
tagConcurrency *int
logToSTDOUT bool
)
var parallelwalkKey = "parallelwalk"
......@@ -540,10 +543,16 @@ var ImportCmd = &cobra.Command{
if rowCount {
opts = append(opts, datastore.WithRowCount)
}
if tagConcurrency != nil {
if config.Storage.Type() != "gcs" {
fmt.Fprintf(os.Stderr, "the tag concurrency option is only compatible with a gcs backed registry storage")
os.Exit(1)
}
opts = append(opts, datastore.WithTagConcurrency(*tagConcurrency))
}
if !logToSTDOUT {
opts = append(opts, datastore.WithProgressBar)
}
p := datastore.NewImporter(db, registry, opts...)
......
registry/storage/garbagecollect.go
View file @ 7f45f273
......@@ -90,6 +90,9 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis
"dry_run": opts.DryRun,
})
// Always log this info message at the end of MarkAndSweep, even if there was an error or the process was interrupted.
defer l.Info("zero-downtime continuous garbage collection is now available as part of a beta feature. Please see the feedback issue for more information: https://gitlab.com/gitlab-org/gitlab/-/issues/423459")
// Check that the database does **not** manage this filesystem.
// We should both log and exit during the failure cases becase we send the
// final error to stderr and this is not always captured in user reports.
......