Commit 4d4d3390 authored by Pawel Rozlach's avatar Pawel Rozlach 💬 Committed by João Pereira
Browse files

feat: rewrite s3_v2 driver from deprecated aws-sdk-go to aws-sdk-go-v2

parent 0196b3df

.gitlab/ci/integration.yml

+2 −0
Original line number Diff line number Diff line
@@ -53,6 +53,8 @@ s3:minio: 
    MINIO_SECRET_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"

    REGION_ENDPOINT: "http://minio:9000"

    PACKAGE: "github.com/docker/distribution/registry/storage/driver/s3-aws"

    # NOTE(prozlach): see docs/storage-drivers/s3_v2.md#minio for explanation why:

    S3_CHECKSUM_DISABLED: "true"

  services:

    - name: minio/minio:latest

      alias: "minio"

docs/storage-drivers/s3_v2.md

+4 −0
Original line number Diff line number Diff line
@@ -48,6 +48,10 @@ This method uses an access key and secret key to authenticate with S3: 
| `checksum_algorithm` | no | The algorithm to use for checksums when uploading objects to S3. Can be one of: `CRC32`, `CRC32C`, `SHA1`, `SHA256`, or `CRC64NVME`. Defaults to `CRC64NVME`. |

| `checksum_disabled` | no | If set to true, disables checksum calculation for S3 objects. Defaults to false. This parameter takes precedence over `checksum_algorithm` if both are provided. |



## MinIO



Due to issues with checksum handling in MinIO ([issue](https://github.com/minio/minio/issues/20649), [issue](https://github.com/minio/minio/issues/17013)), `checksum_disabled` must be set to `true` in order to run container registry using MinIO as a backend.



## Storage Class Options



The `storageclass` parameter allows you to configure the storage class used for registry storage. The following options are available:

go.mod

+15 −0
Original line number Diff line number Diff line
@@ -15,6 +15,8 @@ require ( 
	github.com/alicebob/miniredis/v2 v2.34.0

	github.com/aws/aws-sdk-go v1.55.5

	github.com/aws/aws-sdk-go-v2 v1.36.3

	github.com/aws/aws-sdk-go-v2/config v1.29.12

	github.com/aws/aws-sdk-go-v2/credentials v1.17.65

	github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2

	github.com/aws/smithy-go v1.22.3

	github.com/benbjohnson/clock v1.3.5
@@ -78,6 +80,19 @@ require ( 
	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect

	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect

	github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect

	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect

	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect

	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect

	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect

	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect

	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect

	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect

	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect

	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect

	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect

	github.com/aws/aws-sdk-go-v2/service/sso v1.25.2 // indirect

	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0 // indirect

	github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 // indirect

	github.com/beorn7/perks v1.0.1 // indirect

	github.com/cespare/xxhash/v2 v2.3.0 // indirect

	github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect

go.sum

+30 −0
Original line number Diff line number Diff line
@@ -142,8 +142,38 @@ github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQ 
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=

github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=

github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=

github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=

github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=

github.com/aws/aws-sdk-go-v2/config v1.29.12 h1:Y/2a+jLPrPbHpFkpAAYkVEtJmxORlXoo5k2g1fa2sUo=

github.com/aws/aws-sdk-go-v2/config v1.29.12/go.mod h1:xse1YTjmORlb/6fhkWi8qJh3cvZi4JoVNhc+NbJt4kI=

github.com/aws/aws-sdk-go-v2/credentials v1.17.65 h1:q+nV2yYegofO/SUXruT+pn4KxkxmaQ++1B/QedcKBFM=

github.com/aws/aws-sdk-go-v2/credentials v1.17.65/go.mod h1:4zyjAuGOdikpNYiSGpsGz8hLGmUzlY8pc8r9QQ/RXYQ=

github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw=

github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=

github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=

github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=

github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=

github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=

github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=

github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=

github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM=

github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs=

github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=

github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=

github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 h1:lguz0bmOoGzozP9XfRJR1QIayEYo+2vP/No3OfLF0pU=

github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0=

github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=

github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=

github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg=

github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA=

github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2 h1:tWUG+4wZqdMl/znThEk9tcCy8tTMxq8dW0JTgamohrY=

github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc=

github.com/aws/aws-sdk-go-v2/service/sso v1.25.2 h1:pdgODsAhGo4dvzC3JAG5Ce0PX8kWXrTZGx+jxADD+5E=

github.com/aws/aws-sdk-go-v2/service/sso v1.25.2/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=

github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0 h1:90uX0veLKcdHVfvxhkWUQSCi5VabtwMLFutYiRke4oo=

github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=

github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 h1:PZV5W8yk4OtH1JAuhV2PXwwO9v5G5Aoj+eMCn4T+1Kc=

github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=

github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k=

github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=

github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o=

registry/storage/driver/s3-aws/common/parser.go

+3 −4
Original line number Diff line number Diff line
@@ -216,7 +216,7 @@ func ParseLogLevelParamV1(logger dcontext.Logger, param any) aws.LogLevelType { 
		case LogLevelOff:

			// LogLevelOff in the list of loglevels overrides all other log

			// levels and disables logging

			logger.Infof("S3 logging level set to %q", LogLevelOff)

			logger.Debugf("S3 logging level set to %q", LogLevelOff)

			return aws.LogOff

		case LogLevelDebug:

			res |= aws.LogDebug
@@ -274,7 +274,7 @@ func ParseLogLevelParamV2(logger dcontext.Logger, param any) v2_aws.ClientLogMod 
		// LogLevelOff in the list of loglevels overrides all other log levels

		// and disables logging

		case LogLevelOff:

			logger.Infof("S3 logging level set to %q", LogLevelOff)

			logger.Debugf("S3 logging level set to %q", LogLevelOff)

			// aws sdk v2 does not have a constant for this:

			return v2_aws.ClientLogMode(0)

		case LogSigning:
@@ -631,8 +631,7 @@ func ParseParameters(driverVersion string, parameters map[string]any) (*DriverPa 
	case V1DriverName, V1DriverNameAlt:

		res.LogLevel = uint64(ParseLogLevelParamV1(logger, parameters[ParamLogLevel]))

	case V2DriverName:

		// res.LogLevel = uint64(ParseLogLevelParamV2(logger, parameters[ParamLogLevel]))

		res.LogLevel = uint64(ParseLogLevelParamV1(logger, parameters[ParamLogLevel]))

		res.LogLevel = uint64(ParseLogLevelParamV2(logger, parameters[ParamLogLevel]))

	}



	return res, nil