Commit bf9e1faf authored by João Pereira's avatar João Pereira 🔴 Committed by Pawel Rozlach
Browse files

refactor: filter transient errors from Sentry

parent 6c175d79

registry/bbm/bbm.go

+9 −9
Original line number Diff line number Diff line
@@ -18,8 +18,8 @@ import ( 
	"github.com/docker/distribution/log"

	"github.com/docker/distribution/registry/datastore"

	"github.com/docker/distribution/registry/datastore/models"

	"github.com/docker/distribution/registry/internal/errorreporting"

	"gitlab.com/gitlab-org/labkit/correlation"

	"gitlab.com/gitlab-org/labkit/errortracking"

)



type runResult struct {
@@ -346,7 +346,7 @@ func (jw *Worker) run(ctx context.Context) (runResult, error) { 
	tx, err := jw.db.BeginTx(ctx, nil)

	if err != nil {

		jw.logger.WithError(err).Error("failed to create database transaction")

		errortracking.Capture(err, errortracking.WithContext(ctx), errortracking.WithStackTrace())

		errorreporting.Capture(ctx, err)

		return runRes, err

	}

	defer tx.Rollback()
@@ -358,7 +358,7 @@ func (jw *Worker) run(ctx context.Context) (runResult, error) { 
	if err = jw.wh.GrabLock(ctx, bbmStore); err != nil {

		jw.logger.WithError(err).Info("failed to obtain lock")

		if !errors.Is(err, datastore.ErrBackgroundMigrationLockInUse) {

			errortracking.Capture(err, errortracking.WithContext(ctx), errortracking.WithStackTrace())

			errorreporting.Capture(ctx, err)

			return runRes, err

		}
@@ -372,7 +372,7 @@ func (jw *Worker) run(ctx context.Context) (runResult, error) { 
		throttle, err := jw.wh.ShouldThrottle(ctx, bbmStore)

		if err != nil {

			jw.logger.WithError(err).Info("WAL throttle check failed")

			errortracking.Capture(err, errortracking.WithContext(ctx), errortracking.WithStackTrace())

			errorreporting.Capture(ctx, err)

			return runRes, err

		}
@@ -387,14 +387,14 @@ func (jw *Worker) run(ctx context.Context) (runResult, error) { 
	job, err := jw.wh.FindJob(ctx, bbmStore)

	if err != nil {

		jw.logger.WithError(err).Error("failed to find job")

		errortracking.Capture(err, errortracking.WithContext(ctx), errortracking.WithStackTrace())

		errorreporting.Capture(ctx, err)

		return runRes, err

	}

	if job == nil {

		jw.logger.Info("no jobs to run...")

		if err = tx.Commit(); err != nil {

			jw.logger.WithError(err).Error("failed to commit database transaction")

			errortracking.Capture(err, errortracking.WithContext(ctx), errortracking.WithStackTrace())

			errorreporting.Capture(ctx, err)

			return runRes, err

		}

		return runRes, nil
@@ -420,7 +420,7 @@ func (jw *Worker) run(ctx context.Context) (runResult, error) { 
	err = jw.wh.ExecuteJob(ctx, bbmStore, job)

	if err != nil {

		l.WithError(err).Error("failed to execute job")

		errortracking.Capture(err, errortracking.WithContext(ctx), errortracking.WithStackTrace())

		errorreporting.Capture(ctx, err)

		return runRes, err

	}
@@ -432,7 +432,7 @@ func (jw *Worker) run(ctx context.Context) (runResult, error) { 
	// We should revisit this when redis is generally available. https://gitlab.com/gitlab-org/container-registry/-/issues/1639

	if err = tx.Commit(); err != nil {

		jw.logger.WithError(err).Error("failed to commit database transaction")

		errortracking.Capture(err, errortracking.WithContext(ctx), errortracking.WithStackTrace())

		errorreporting.Capture(ctx, err)

		return runRes, err

	}
@@ -500,7 +500,7 @@ func (jw *Worker) FindJob(ctx context.Context, bbmStore datastore.BackgroundMigr 
	err = validateMigration(ctx, bbmStore, jw.Work, bbm)

	if err != nil {

		l.WithError(err).Error("background migration failed validation")

		errortracking.Capture(err, errortracking.WithContext(ctx))

		errorreporting.Capture(ctx, err)



		var migrationErr *migrationFailureError

		if errors.As(err, &migrationErr) {

registry/datastore/loadbalancing.go

+8 −6
Original line number Diff line number Diff line
@@ -14,10 +14,10 @@ import ( 
	"github.com/docker/distribution/log"

	"github.com/docker/distribution/registry/datastore/metrics"

	"github.com/docker/distribution/registry/datastore/models"

	"github.com/docker/distribution/registry/internal/errorreporting"

	"github.com/hashicorp/go-multierror"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/sirupsen/logrus"

	"gitlab.com/gitlab-org/labkit/errortracking"

	"gitlab.com/gitlab-org/labkit/metrics/sqlmetrics"

)
@@ -514,7 +514,7 @@ func (lb *DBLoadBalancer) ProcessQueryError(ctx context.Context, db *DB, query s 
			// This is not supposed to happen, log and report

			err := fmt.Errorf("unknown database host type: %w", err)

			l.Error(err)

			errortracking.Capture(err, errortracking.WithContext(ctx), errortracking.WithStackTrace())

			errorreporting.Capture(ctx, err)

		}

	}

}
@@ -617,7 +617,7 @@ func (lb *DBLoadBalancer) ResolveReplicas(ctx context.Context) error { 
				if err := lb.promRegisterer.Register(collector); err != nil {

					l.WithError(err).WithFields(log.Fields{"db_host_addr": r.Address()}).

						Error("failed to register collector for database replica metrics")

					errortracking.Capture(err, errortracking.WithContext(ctx), errortracking.WithStackTrace())

					errorreporting.Capture(ctx, err)

				}

				lb.replicaPromCollectors[r.Address()] = collector

			}
@@ -637,11 +637,13 @@ func (lb *DBLoadBalancer) ResolveReplicas(ctx context.Context) error { 
			lb.unregisterReplicaMetricsCollector(r)



			// Close handlers for retired replicas

			l.WithFields(log.Fields{"db_host_addr": r.Address()}).Info("closing connection handler for retired replica")

			rl := l.WithFields(log.Fields{"db_host_addr": r.Address()})

			rl.Info("closing connection handler for retired replica")

			if err := r.Close(); err != nil {

				err = fmt.Errorf("failed to close retired replica %q connection: %w", r.Address(), err)

				rl.WithError(err).Error("failed to close retired replica connection")

				result = multierror.Append(result, err)

				errortracking.Capture(err, errortracking.WithContext(ctx), errortracking.WithStackTrace())

				errorreporting.Capture(ctx, err)

			}

		}

	}
@@ -814,7 +816,7 @@ func NewDBLoadBalancer(ctx context.Context, primaryDSN *DSN, opts ...Option) (*D 


		if err := lb.ResolveReplicas(ctx); err != nil {

			lb.logger(ctx).WithError(err).Error("failed to resolve database load balancing replicas")

			errortracking.Capture(err, errortracking.WithContext(ctx), errortracking.WithStackTrace())

			errorreporting.Capture(ctx, err)

		}

	}

registry/datastore/metrics/bbm_progress.go

+2 −6
Original line number Diff line number Diff line
@@ -11,9 +11,9 @@ import ( 
	dlog "github.com/docker/distribution/log"

	"github.com/docker/distribution/metrics"

	"github.com/docker/distribution/registry/datastore/models"

	"github.com/docker/distribution/registry/internal/errorreporting"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/redis/go-redis/v9"

	"gitlab.com/gitlab-org/labkit/errortracking"

)



const (
@@ -114,11 +114,7 @@ func (c *BBMProgressCollector) run(ctx context.Context) { 
	// Ensure metrics are registered

	if err := c.metricsRegistrar.Register(); err != nil {

		c.logger.WithError(err).Error("failed to register bbm progress metrics")

		errortracking.Capture(

			fmt.Errorf("bbm progress metrics: failed to register metrics: %w", err),

			errortracking.WithContext(ctx),

			errortracking.WithStackTrace(),

		)

		errorreporting.Capture(ctx, fmt.Errorf("bbm progress metrics: failed to register metrics: %w", err))

		return

	}

	defer c.metricsRegistrar.Unregister()

registry/datastore/metrics/rowcount.go

+4 −11
Original line number Diff line number Diff line
@@ -11,10 +11,10 @@ import ( 
	"github.com/bsm/redislock"

	dlog "github.com/docker/distribution/log"

	"github.com/docker/distribution/metrics"

	"github.com/docker/distribution/registry/internal/errorreporting"

	"github.com/docker/distribution/testutil"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/redis/go-redis/v9"

	"gitlab.com/gitlab-org/labkit/errortracking"

)



const (
@@ -300,11 +300,8 @@ func (c *RowCountCollector) run(ctx context.Context) { 


	// Register metrics when gaining leadership

	if err := c.registerMetrics(); err != nil {

		errortracking.Capture(

			fmt.Errorf("database row count metrics: failed to register metrics after obtaining lock: %w", err),

			errortracking.WithContext(ctx),

			errortracking.WithStackTrace(),

		)

		c.logger.WithError(err).Error("failed to register metrics after obtaining lock")

		errorreporting.Capture(ctx, fmt.Errorf("database row count metrics: failed to register metrics after obtaining lock: %w", err))



		// Release lock before returning

		if releaseErr := lock.Release(ctx); releaseErr != nil {
@@ -413,11 +410,7 @@ func (c *RowCountCollector) runLockRefresh(ctx context.Context, lock *redislock. 
				// nolint:revive // max-control-nesting - acceptable for error handling logic

				if consecutiveFailures >= lockExtensionMaxRetries {

					c.logger.WithError(err).Error("failed to extend lock after retries, releasing leadership")

					errortracking.Capture(

						fmt.Errorf("database row count metrics: failed to extend lock after %d retries, releasing leadership: %w", consecutiveFailures, err),

						errortracking.WithContext(ctx),

						errortracking.WithStackTrace(),

					)

					errorreporting.Capture(ctx, fmt.Errorf("database row count metrics: failed to extend lock after %d retries, releasing leadership: %w", consecutiveFailures, err))

					return // Too many failures, release leadership

				}

				continue

registry/datastore/repository.go

+4 −4
Original line number Diff line number Diff line
@@ -17,13 +17,13 @@ import ( 
	"github.com/docker/distribution/manifest/manifestlist"

	"github.com/docker/distribution/registry/datastore/metrics"

	"github.com/docker/distribution/registry/datastore/models"

	"github.com/docker/distribution/registry/internal/errorreporting"

	iredis "github.com/docker/distribution/registry/internal/redis"

	"github.com/jackc/pgerrcode"

	"github.com/jackc/pgx/v5/pgconn"

	"github.com/opencontainers/go-digest"

	v1 "github.com/opencontainers/image-spec/specs-go/v1"

	"github.com/redis/go-redis/v9"

	"gitlab.com/gitlab-org/labkit/errortracking"

)



type SortOrder string
@@ -484,7 +484,7 @@ func (c *centralRepositoryCache) InvalidateSize(ctx context.Context, r *models.R 
		detail := "failed to invalidate repository size in cache for repo: " + r.Path

		log.GetLogger(log.WithContext(ctx)).WithError(err).Warn(detail)

		err := fmt.Errorf("%q: %q", detail, err)

		errortracking.Capture(err, errortracking.WithContext(ctx), errortracking.WithStackTrace())

		errorreporting.Capture(ctx, err)

	}

}
@@ -493,7 +493,7 @@ func (c *centralRepositoryCache) Invalidate(ctx context.Context, path string) { 
	if path == "" {

		err := errors.New("can not invalidate an empty path")

		log.GetLogger(log.WithContext(ctx)).WithError(err).Warn("failed to delete repository cache keys")

		errortracking.Capture(err, errortracking.WithContext(ctx), errortracking.WithStackTrace())

		errorreporting.Capture(ctx, err)

		return

	}

	delCtx, cancel := context.WithTimeout(ctx, cacheOpTimeout)
@@ -512,7 +512,7 @@ func (c *centralRepositoryCache) Invalidate(ctx context.Context, path string) { 
			"path": path,

			"keys": keys,

		}).Warn("failed to delete repository cache keys")

		errortracking.Capture(err, errortracking.WithContext(ctx), errortracking.WithStackTrace())

		errorreporting.Capture(ctx, err)

	}

}