Skip to content
Snippets Groups Projects
Verified Commit 7c1c10dc authored by Tomasz Maczukin's avatar Tomasz Maczukin :speech_balloon:
Browse files

Refactorize structure and interface of ./shells/cache/...

parent 8d8600a2
No related branches found
No related tags found
1 merge request!968Introduce GCS adapter for remote cache
This commit is part of merge request !968. Comments created here will be created in the context of that merge request.
Showing
with 534 additions and 368 deletions
......@@ -11,6 +11,7 @@ import (
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/tls"
"gitlab.com/gitlab-org/gitlab-runner/shells/cache"
)
type AbstractShell struct {
......@@ -237,7 +238,7 @@ func (b *AbstractShell) cacheExtractor(w ShellWriter, info common.ShellScriptInf
}
// Generate cache download address
if url := getCacheDownloadURL(info.Build, cacheKey); url != nil {
if url := cache.GetCacheDownloadURL(info.Build, cacheKey); url != nil {
args = append(args, "--url", url.String())
}
......@@ -467,7 +468,7 @@ func (b *AbstractShell) cacheArchiver(w ShellWriter, info common.ShellScriptInfo
args = append(args, archiverArgs...)
// Generate cache upload address
if url := getCacheUploadURL(info.Build, cacheKey); url != nil {
if url := cache.GetCacheUploadURL(info.Build, cacheKey); url != nil {
args = append(args, "--url", url.String())
}
......
package cache
import (
"net/url"
"time"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/shells/cache/gcs"
"gitlab.com/gitlab-org/gitlab-runner/shells/cache/s3"
)
type Adapter interface {
SetConfig(config *common.CacheConfig)
SetTimeout(timeout time.Duration)
SetObjectName(objectName string)
GetDownloadURL() *url.URL
GetUploadURL() *url.URL
}
type initializer func() Adapter
var adapters = map[string]initializer{
"s3": func() Adapter { return new(s3.Adapter) },
"gcs": func() Adapter { return new(gcs.Adapter) },
}
var Factory = func(cacheConfig *common.CacheConfig, timeout time.Duration, objectName string) Adapter {
init, ok := adapters[cacheConfig.Type]
if !ok {
logrus.Errorf("Cache adapter of type %q is unknown", cacheConfig.Type)
return nil
}
adapter := init()
adapter.SetConfig(cacheConfig)
adapter.SetTimeout(timeout)
adapter.SetObjectName(objectName)
return adapter
}
package adapter
import (
"fmt"
"net/url"
"sync"
"time"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type Adapter interface {
GetDownloadURL() *url.URL
GetUploadURL() *url.URL
}
type Factory func(config *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error)
type FactoriesMap struct {
sync.Mutex
internal map[string]Factory
}
func (m *FactoriesMap) Register(typeName string, factory Factory) error {
m.Lock()
defer m.Unlock()
if len(m.internal) == 0 {
m.internal = make(map[string]Factory)
}
_, ok := m.internal[typeName]
if ok {
return fmt.Errorf("adapter %q already registered", typeName)
}
m.internal[typeName] = factory
return nil
}
func (m *FactoriesMap) Find(typeName string) (Factory, error) {
factory := m.internal[typeName]
if factory == nil {
return nil, fmt.Errorf("factory for cache adapter %q was not registered", typeName)
}
return factory, nil
}
var factories = &FactoriesMap{}
func Factories() *FactoriesMap {
return factories
}
func FactorizeAdapter(cacheConfig *common.CacheConfig, timeout time.Duration, objectName string) Adapter {
log := logrus.WithField("type", cacheConfig.Type)
factorize, err := Factories().Find(cacheConfig.Type)
if err != nil {
log.WithError(err).Error("Cache factory not found")
return nil
}
adapter, err := factorize(cacheConfig, timeout, objectName)
if err != nil {
log.WithError(err).Error("Cache adapter could not be initialized")
return nil
}
return adapter
}
package adapter
import (
"bytes"
"errors"
"testing"
"time"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
var defaultTimeout = 1 * time.Hour
func runOnHijackedLogrusOutput(t *testing.T, handler func(output *bytes.Buffer)) {
oldOutput := logrus.StandardLogger().Out
defer func() { logrus.StandardLogger().Out = oldOutput }()
buf := bytes.NewBuffer([]byte{})
logrus.StandardLogger().Out = buf
handler(buf)
}
type factorizeTestCase struct {
adapter Adapter
errorOnFactorize error
expectedAdapter Adapter
expectedOutput []string
}
func runOnMockedFactoriesMap(handler func(factories *FactoriesMap)) {
oldFactories := factories
defer func() {
factories = oldFactories
}()
factories = &FactoriesMap{}
handler(factories)
}
func TestFactorize(t *testing.T) {
adapterMock := new(MockAdapter)
tests := map[string]factorizeTestCase{
"adapter doesn't exist": {
adapter: nil,
errorOnFactorize: nil,
expectedAdapter: nil,
expectedOutput: []string{
"Cache factory not found",
`factory for cache adapter \"test\" was not registered`,
},
},
"adapter exists": {
adapter: adapterMock,
errorOnFactorize: nil,
expectedAdapter: adapterMock,
expectedOutput: []string{},
},
"adapter errors on factorize": {
adapter: adapterMock,
errorOnFactorize: errors.New("test error"),
expectedAdapter: nil,
expectedOutput: []string{
"Cache adapter could not be initialized",
"test error",
},
},
}
for name, test := range tests {
t.Run(name, func(t *testing.T) {
runOnHijackedLogrusOutput(t, func(output *bytes.Buffer) {
runOnMockedFactoriesMap(func(factories *FactoriesMap) {
adapterTypeName := "test"
if test.adapter != nil {
factories.Register(adapterTypeName, func(config *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error) {
if test.errorOnFactorize != nil {
return nil, test.errorOnFactorize
}
return test.adapter, nil
})
}
config := &common.CacheConfig{
Type: adapterTypeName,
}
adapter := FactorizeAdapter(config, defaultTimeout, "key")
assert.Equal(t, test.expectedAdapter, adapter)
if len(test.expectedOutput) == 0 {
assert.Empty(t, output.String())
} else {
for _, expectedOutput := range test.expectedOutput {
assert.Contains(t, output.String(), expectedOutput)
}
}
})
})
})
}
}
func TestDoubledRegistration(t *testing.T) {
runOnMockedFactoriesMap(func(factories *FactoriesMap) {
adapterTypeName := "test"
fakeFactory := func(config *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error) {
return nil, nil
}
err := factories.Register(adapterTypeName, fakeFactory)
assert.NoError(t, err)
assert.Len(t, factories.internal, 1)
err = factories.Register(adapterTypeName, fakeFactory)
assert.Error(t, err)
assert.Len(t, factories.internal, 1)
})
}
......@@ -2,11 +2,9 @@
// This comment works around https://github.com/vektra/mockery/issues/155
package cache
package adapter
import common "gitlab.com/gitlab-org/gitlab-runner/common"
import mock "github.com/stretchr/testify/mock"
import time "time"
import url "net/url"
// MockAdapter is an autogenerated mock type for the Adapter type
......@@ -45,18 +43,3 @@ func (_m *MockAdapter) GetUploadURL() *url.URL {
return r0
}
// SetConfig provides a mock function with given fields: config
func (_m *MockAdapter) SetConfig(config *common.CacheConfig) {
_m.Called(config)
}
// SetObjectName provides a mock function with given fields: objectName
func (_m *MockAdapter) SetObjectName(objectName string) {
_m.Called(objectName)
}
// SetTimeout provides a mock function with given fields: timeout
func (_m *MockAdapter) SetTimeout(timeout time.Duration) {
_m.Called(timeout)
}
package cache
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
var defaultTimeout = 1 * time.Hour
func TestFactoryCreateExisting(t *testing.T) {
adapterMock := new(MockAdapter)
adapters = map[string]initializer{
"test": func() Adapter { return adapterMock },
}
config := &common.CacheConfig{
Type: "test",
}
adapterMock.On("SetConfig", config).Once()
adapterMock.On("SetTimeout", defaultTimeout).Once()
adapterMock.On("SetObjectName", mock.Anything).Once()
defer adapterMock.AssertExpectations(t)
adapter := Factory(config, defaultTimeout, "key")
assert.Equal(t, adapterMock, adapter)
}
func TestFactoryCreateUnexisting(t *testing.T) {
adapters = map[string]initializer{}
config := &common.CacheConfig{
Type: "test",
}
adapter := Factory(config, defaultTimeout, "key")
assert.Nil(t, adapter)
}
package gcs
import (
"fmt"
"net/http"
"net/url"
"time"
......@@ -10,81 +11,48 @@ import (
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/shells/cache/adapter"
)
type SignedURLGenerator interface {
GetSignedURL(bucket string, name string, opts *storage.SignedURLOptions) (string, error)
}
type DefaultSignedURLGenerator struct{}
type signedURLGenerator func(bucket string, name string, opts *storage.SignedURLOptions) (string, error)
func (dsug *DefaultSignedURLGenerator) GetSignedURL(bucket string, name string, opts *storage.SignedURLOptions) (string, error) {
return storage.SignedURL(bucket, name, opts)
}
var NewSignedURLGenerator = func() SignedURLGenerator {
return &DefaultSignedURLGenerator{}
}
type Adapter struct {
type gcsAdapter struct {
timeout time.Duration
config *common.CacheConfig
config *common.CacheGCSConfig
objectName string
}
func (a *Adapter) SetTimeout(timeout time.Duration) {
a.timeout = timeout
generateSignedURL signedURLGenerator
credentialsResolver credentialsResolver
}
func (a *Adapter) SetConfig(config *common.CacheConfig) {
a.config = config
}
func (a *Adapter) SetObjectName(objectName string) {
a.objectName = objectName
}
func (a *Adapter) GetDownloadURL() *url.URL {
func (a *gcsAdapter) GetDownloadURL() *url.URL {
return a.getGCSURL(http.MethodGet, "")
}
func (a *Adapter) GetUploadURL() *url.URL {
func (a *gcsAdapter) GetUploadURL() *url.URL {
return a.getGCSURL(http.MethodPut, "application/octet-stream")
}
func (a *Adapter) getGCSURL(method string, contentType string) (URL *url.URL) {
gcs := a.config.GCS
if gcs == nil {
logrus.Errorln("Missing GCS configuration")
return
}
cr, err := NewCredentialsResolver(gcs)
if err != nil {
logrus.Errorf("error while initializing GCS credentials resolver: %v", err)
return
}
err = cr.Resolve()
func (a *gcsAdapter) getGCSURL(method string, contentType string) *url.URL {
err := a.credentialsResolver.Resolve()
if err != nil {
logrus.Errorf("error while resolving GCS credentials: %v", err)
return
return nil
}
credentials := cr.Credentials()
credentials := a.credentialsResolver.Credentials()
var privateKey []byte
if credentials.PrivateKey != "" {
privateKey = []byte(credentials.PrivateKey)
}
if gcs.BucketName == "" {
if a.config.BucketName == "" {
logrus.Error("BucketName can't be empty")
return
return nil
}
generator := NewSignedURLGenerator()
rawURL, err := generator.GetSignedURL(gcs.BucketName, a.objectName, &storage.SignedURLOptions{
rawURL, err := a.generateSignedURL(a.config.BucketName, a.objectName, &storage.SignedURLOptions{
GoogleAccessID: credentials.AccessID,
PrivateKey: privateKey,
Method: method,
......@@ -93,14 +61,42 @@ func (a *Adapter) getGCSURL(method string, contentType string) (URL *url.URL) {
})
if err != nil {
logrus.Errorf("error while generating GCS pre-signed URL: %v", err)
return
return nil
}
URL, err = url.Parse(rawURL)
URL, err := url.Parse(rawURL)
if err != nil {
logrus.Errorf("error while parsing generated URL: %v", err)
return
return nil
}
return
return URL
}
func NewAdapter(config *common.CacheConfig, timeout time.Duration, objectName string) (adapter.Adapter, error) {
gcs := config.GCS
if gcs == nil {
return nil, fmt.Errorf("missing GCS configuration")
}
cr, err := newDefaultCredentialsResolver(gcs)
if err != nil {
return nil, fmt.Errorf("error while initializing GCS credentials resolver: %v", err)
}
a := new(gcsAdapter)
a.config = gcs
a.timeout = timeout
a.objectName = objectName
a.generateSignedURL = storage.SignedURL
a.credentialsResolver = cr
return a, nil
}
func init() {
err := adapter.Factories().Register("gcs", NewAdapter)
if err != nil {
panic(err)
}
}
......@@ -9,10 +9,8 @@ import (
"time"
"cloud.google.com/go/storage"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-runner/common"
......@@ -73,9 +71,8 @@ func runOnHijackedLogrusOutput(t *testing.T, handler func(t *testing.T, output *
}
type adapterOperationInvalidConfigTestCase struct {
noGCSConfig bool
credentialsResolverInitializationError bool
credentialsResolverResolveError bool
noGCSConfig bool
credentialsResolverResolveError bool
accessID string
privateKey string
......@@ -83,48 +80,30 @@ type adapterOperationInvalidConfigTestCase struct {
expectedError string
}
func onMockedCredentialsResolverForInvalidConfig(t *testing.T, tc adapterOperationInvalidConfigTestCase, handler func(t *testing.T)) {
if tc.credentialsResolverInitializationError {
oldNewCredentialsResolver := NewCredentialsResolver
defer func() {
NewCredentialsResolver = oldNewCredentialsResolver
}()
func onMockedCredentialsResolverForInvalidConfig(t *testing.T, adapter *gcsAdapter, tc adapterOperationInvalidConfigTestCase, handler func(t *testing.T)) {
cr := &mockCredentialsResolver{}
NewCredentialsResolver = func(config *common.CacheGCSConfig) (CredentialsResolver, error) {
return nil, fmt.Errorf("test error")
}
resolveCall := cr.On("Resolve")
if tc.credentialsResolverResolveError {
resolveCall.Return(fmt.Errorf("test error"))
} else {
cr := &MockCredentialsResolver{}
resolveCall := cr.On("Resolve")
if tc.credentialsResolverResolveError {
resolveCall.Return(fmt.Errorf("test error"))
} else {
resolveCall.Return(nil)
}
cr.On("Credentials").Return(&common.CacheGCSCredentials{
AccessID: tc.accessID,
PrivateKey: tc.privateKey,
})
resolveCall.Return(nil)
}
oldNewCredentialsResolver := NewCredentialsResolver
defer func() {
NewCredentialsResolver = oldNewCredentialsResolver
}()
cr.On("Credentials").Return(&common.CacheGCSCredentials{
AccessID: tc.accessID,
PrivateKey: tc.privateKey,
})
NewCredentialsResolver = func(config *common.CacheGCSConfig) (CredentialsResolver, error) {
return cr, nil
}
}
adapter.credentialsResolver = cr
handler(t)
}
func testAdapterOperationWithInvalidConfig(t *testing.T, name string, tc adapterOperationInvalidConfigTestCase, operation func() *url.URL) {
func testAdapterOperationWithInvalidConfig(t *testing.T, name string, tc adapterOperationInvalidConfigTestCase, adapter *gcsAdapter, operation func() *url.URL) {
t.Run(name, func(t *testing.T) {
runOnHijackedLogrusOutput(t, func(t *testing.T, output *bytes.Buffer) {
onMockedCredentialsResolverForInvalidConfig(t, tc, func(t *testing.T) {
onMockedCredentialsResolverForInvalidConfig(t, adapter, tc, func(t *testing.T) {
u := operation()
assert.Nil(t, u)
......@@ -136,13 +115,12 @@ func testAdapterOperationWithInvalidConfig(t *testing.T, name string, tc adapter
func TestAdapterOperation_InvalidConfig(t *testing.T) {
tests := map[string]adapterOperationInvalidConfigTestCase{
"no-gcs-config": {noGCSConfig: true, bucketName: bucketName, expectedError: "Missing GCS configuration"},
"credentials-resolver-initialization-error": {credentialsResolverInitializationError: true, bucketName: bucketName, expectedError: "error while initializing GCS credentials resolver: test error"},
"credentials-resolver-resolve-error": {credentialsResolverResolveError: true, bucketName: bucketName, expectedError: "error while resolving GCS credentials: test error"},
"no-credentials": {bucketName: bucketName, expectedError: "storage: missing required GoogleAccessID"},
"no-access-id": {privateKey: privateKey, bucketName: bucketName, expectedError: "storage: missing required GoogleAccessID"},
"no-private-key": {accessID: accessID, bucketName: bucketName, expectedError: "storage: exactly one of PrivateKey or SignedBytes must be set"},
"bucket-not-specified": {accessID: "access-id", privateKey: privateKey, expectedError: "BucketName can't be empty"},
"no-gcs-config": {noGCSConfig: true, bucketName: bucketName, expectedError: "Missing GCS configuration"},
"credentials-resolver-resolve-error": {credentialsResolverResolveError: true, bucketName: bucketName, expectedError: "error while resolving GCS credentials: test error"},
"no-credentials": {bucketName: bucketName, expectedError: "storage: missing required GoogleAccessID"},
"no-access-id": {privateKey: privateKey, bucketName: bucketName, expectedError: "storage: missing required GoogleAccessID"},
"no-private-key": {accessID: accessID, bucketName: bucketName, expectedError: "storage: exactly one of PrivateKey or SignedBytes must be set"},
"bucket-not-specified": {accessID: "access-id", privateKey: privateKey, expectedError: "BucketName can't be empty"},
}
for name, tc := range tests {
......@@ -154,13 +132,21 @@ func TestAdapterOperation_InvalidConfig(t *testing.T) {
config.GCS.BucketName = tc.bucketName
}
adapter := new(Adapter)
adapter.SetTimeout(defaultTimeout)
adapter.SetConfig(config)
adapter.SetObjectName(objectName)
a, err := NewAdapter(config, defaultTimeout, objectName)
if tc.noGCSConfig {
assert.Nil(t, a)
assert.EqualError(t, err, "missing GCS configuration")
return
}
require.NotNil(t, a)
require.NoError(t, err)
testAdapterOperationWithInvalidConfig(t, "GetDownloadURL", tc, adapter.GetDownloadURL)
testAdapterOperationWithInvalidConfig(t, "GetUploadURL", tc, adapter.GetUploadURL)
adapter, ok := a.(*gcsAdapter)
require.True(t, ok, "Adapter should be properly casted to *adapter type")
testAdapterOperationWithInvalidConfig(t, "GetDownloadURL", tc, adapter, a.GetDownloadURL)
testAdapterOperationWithInvalidConfig(t, "GetUploadURL", tc, adapter, a.GetUploadURL)
})
}
}
......@@ -171,8 +157,8 @@ type adapterOperationTestCase struct {
expectedError string
}
func onMockedCredentialsResolver(t *testing.T, handler func(t *testing.T)) {
cr := &MockCredentialsResolver{}
func onMockedCredentialsResolver(t *testing.T, adapter *gcsAdapter, handler func(t *testing.T)) {
cr := &mockCredentialsResolver{}
cr.On("Resolve").Return(nil)
cr.On("Credentials").Return(&common.CacheGCSCredentials{
AccessID: accessID,
......@@ -180,62 +166,29 @@ func onMockedCredentialsResolver(t *testing.T, handler func(t *testing.T)) {
})
defer cr.AssertExpectations(t)
oldNewCredentialsResolver := NewCredentialsResolver
defer func() {
NewCredentialsResolver = oldNewCredentialsResolver
}()
NewCredentialsResolver = func(config *common.CacheGCSConfig) (CredentialsResolver, error) {
return cr, nil
}
adapter.credentialsResolver = cr
handler(t)
}
func optsMatcher(expectedMethod string, expectedContentType string) interface{} {
return mock.MatchedBy(func(opts *storage.SignedURLOptions) bool {
if opts.GoogleAccessID != accessID {
return false
}
if string(opts.PrivateKey) != privateKey {
return false
}
if opts.Method != expectedMethod {
return false
}
if opts.ContentType != expectedContentType {
return false
}
return true
})
}
func onMockedSigndeURLGenerator(t *testing.T, tc adapterOperationTestCase, expectedMethod string, expectedContentType string, handler func(t *testing.T)) {
ug := &MockSignedURLGenerator{}
ug.On("GetSignedURL", bucketName, objectName, optsMatcher(expectedMethod, expectedContentType)).Return(tc.returnedURL, tc.returnedError)
defer ug.AssertExpectations(t)
func onMockedSigndeURLGenerator(t *testing.T, tc adapterOperationTestCase, expectedMethod string, expectedContentType string, adapter *gcsAdapter, handler func(t *testing.T)) {
adapter.generateSignedURL = func(bucket string, name string, opts *storage.SignedURLOptions) (string, error) {
require.Equal(t, accessID, opts.GoogleAccessID)
require.Equal(t, privateKey, string(opts.PrivateKey))
require.Equal(t, expectedMethod, opts.Method)
require.Equal(t, expectedContentType, opts.ContentType)
oldNewSignedURLGenerator := NewSignedURLGenerator
defer func() {
NewSignedURLGenerator = oldNewSignedURLGenerator
}()
NewSignedURLGenerator = func() SignedURLGenerator {
return ug
return tc.returnedURL, tc.returnedError
}
handler(t)
}
func testAdapterOperation(t *testing.T, tc adapterOperationTestCase, name string, expectedMethod string, expectedContentType string, operation func() *url.URL) {
func testAdapterOperation(t *testing.T, tc adapterOperationTestCase, name string, expectedMethod string, expectedContentType string, adapter *gcsAdapter, operation func() *url.URL) {
t.Run(name, func(t *testing.T) {
runOnHijackedLogrusOutput(t, func(t *testing.T, output *bytes.Buffer) {
onMockedCredentialsResolver(t, func(t *testing.T) {
onMockedSigndeURLGenerator(t, tc, expectedMethod, expectedContentType, func(t *testing.T) {
onMockedCredentialsResolver(t, adapter, func(t *testing.T) {
onMockedSigndeURLGenerator(t, tc, expectedMethod, expectedContentType, adapter, func(t *testing.T) {
u := operation()
if tc.expectedError != "" {
......@@ -275,13 +228,14 @@ func TestAdapterOperation(t *testing.T) {
t.Run(name, func(t *testing.T) {
config := defaultGCSCache()
adapter := new(Adapter)
adapter.SetTimeout(defaultTimeout)
adapter.SetConfig(config)
adapter.SetObjectName(objectName)
a, err := NewAdapter(config, defaultTimeout, objectName)
require.NoError(t, err)
adapter, ok := a.(*gcsAdapter)
require.True(t, ok, "Adapter should be properly casted to *adapter type")
testAdapterOperation(t, tc, "GetDownloadURL", http.MethodGet, "", adapter.GetDownloadURL)
testAdapterOperation(t, tc, "GetUploadURL", http.MethodPut, "application/octet-stream", adapter.GetUploadURL)
testAdapterOperation(t, tc, "GetDownloadURL", http.MethodGet, "", adapter, a.GetDownloadURL)
testAdapterOperation(t, tc, "GetUploadURL", http.MethodPut, "application/octet-stream", adapter, a.GetUploadURL)
})
}
}
......@@ -10,7 +10,7 @@ import (
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type CredentialsResolver interface {
type credentialsResolver interface {
Credentials() *common.CacheGCSCredentials
Resolve() error
}
......@@ -23,16 +23,16 @@ type credentialsFile struct {
PrivateKey string `json:"private_key"`
}
type DefaultCredentialsResolver struct {
type defaultCredentialsResolver struct {
config *common.CacheGCSConfig
credentials *common.CacheGCSCredentials
}
func (cr *DefaultCredentialsResolver) Credentials() *common.CacheGCSCredentials {
func (cr *defaultCredentialsResolver) Credentials() *common.CacheGCSCredentials {
return cr.credentials
}
func (cr *DefaultCredentialsResolver) Resolve() error {
func (cr *defaultCredentialsResolver) Resolve() error {
if cr.config.CredentialsFile != "" {
return cr.readCredentialsFromFile()
}
......@@ -40,7 +40,7 @@ func (cr *DefaultCredentialsResolver) Resolve() error {
return cr.readCredentialsFromConfig()
}
func (cr *DefaultCredentialsResolver) readCredentialsFromFile() error {
func (cr *defaultCredentialsResolver) readCredentialsFromFile() error {
data, err := ioutil.ReadFile(cr.config.CredentialsFile)
if err != nil {
return fmt.Errorf("error while reading credentials file: %v", err)
......@@ -64,7 +64,7 @@ func (cr *DefaultCredentialsResolver) readCredentialsFromFile() error {
return nil
}
func (cr *DefaultCredentialsResolver) readCredentialsFromConfig() error {
func (cr *defaultCredentialsResolver) readCredentialsFromConfig() error {
if cr.config.CacheGCSCredentials == nil {
return fmt.Errorf("GCS config present, but credentials are not configured")
}
......@@ -75,19 +75,15 @@ func (cr *DefaultCredentialsResolver) readCredentialsFromConfig() error {
return nil
}
func NewDefaultCredentialsResolver(config *common.CacheGCSConfig) (*DefaultCredentialsResolver, error) {
func newDefaultCredentialsResolver(config *common.CacheGCSConfig) (*defaultCredentialsResolver, error) {
if config == nil {
return nil, fmt.Errorf("config can't be nil")
}
credentials := &DefaultCredentialsResolver{
credentials := &defaultCredentialsResolver{
config: config,
credentials: &common.CacheGCSCredentials{},
}
return credentials, nil
}
var NewCredentialsResolver = func(config *common.CacheGCSConfig) (CredentialsResolver, error) {
return NewDefaultCredentialsResolver(config)
}
......@@ -131,7 +131,7 @@ func TestDefaultCredentialsResolver(t *testing.T) {
for name, testCase := range cases {
t.Run(name, func(t *testing.T) {
onStubbedCredentialsFile(t, testCase, func(t *testing.T, testCase credentialsResolverTestCase) {
cr, err := NewCredentialsResolver(testCase.config)
cr, err := newDefaultCredentialsResolver(testCase.config)
if testCase.errorExpectedOnInitialization {
assert.Error(t, err)
......
......@@ -7,13 +7,13 @@ package gcs
import common "gitlab.com/gitlab-org/gitlab-runner/common"
import mock "github.com/stretchr/testify/mock"
// MockCredentialsResolver is an autogenerated mock type for the CredentialsResolver type
type MockCredentialsResolver struct {
// mockCredentialsResolver is an autogenerated mock type for the credentialsResolver type
type mockCredentialsResolver struct {
mock.Mock
}
// Credentials provides a mock function with given fields:
func (_m *MockCredentialsResolver) Credentials() *common.CacheGCSCredentials {
func (_m *mockCredentialsResolver) Credentials() *common.CacheGCSCredentials {
ret := _m.Called()
var r0 *common.CacheGCSCredentials
......@@ -29,7 +29,7 @@ func (_m *MockCredentialsResolver) Credentials() *common.CacheGCSCredentials {
}
// Resolve provides a mock function with given fields:
func (_m *MockCredentialsResolver) Resolve() error {
func (_m *mockCredentialsResolver) Resolve() error {
ret := _m.Called()
var r0 error
......
package s3
import (
"fmt"
"net/url"
"time"
......@@ -9,6 +10,7 @@ import (
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/shells/cache/adapter"
)
type fakeIAMCredentialsProvider interface {
......@@ -21,54 +23,32 @@ var iamFactory = func() *credentials.Credentials {
type s3URLGenerator func(scl *minio.Client, bucketName string, objectName string, expires time.Duration) (*url.URL, error)
type Adapter struct {
type s3Adapter struct {
timeout time.Duration
config *common.CacheConfig
config *common.CacheS3Config
objectName string
}
func (a *Adapter) SetTimeout(timeout time.Duration) {
a.timeout = timeout
}
func (a *Adapter) SetConfig(config *common.CacheConfig) {
a.config = deprecatedConfigHandler(config)
}
func (a *Adapter) SetObjectName(objectName string) {
a.objectName = objectName
}
func (a *Adapter) GetDownloadURL() *url.URL {
return a.getS3URL(getS3DownloadURL)
func (a *s3Adapter) GetDownloadURL() *url.URL {
return a.getS3URL(func(scl *minio.Client, bucketName string, objectName string, expires time.Duration) (*url.URL, error) {
return scl.PresignedGetObject(bucketName, objectName, expires, nil)
})
}
func getS3DownloadURL(scl *minio.Client, bucketName string, objectName string, expires time.Duration) (*url.URL, error) {
return scl.PresignedGetObject(bucketName, objectName, expires, nil)
func (a *s3Adapter) GetUploadURL() *url.URL {
return a.getS3URL(func(scl *minio.Client, bucketName string, objectName string, expires time.Duration) (*url.URL, error) {
return scl.PresignedPutObject(bucketName, objectName, expires)
})
}
func (a *Adapter) GetUploadURL() *url.URL {
return a.getS3URL(getS3UploadURL)
}
func getS3UploadURL(scl *minio.Client, bucketName string, objectName string, expires time.Duration) (*url.URL, error) {
return scl.PresignedPutObject(bucketName, objectName, expires)
}
func (a *Adapter) getS3URL(generator s3URLGenerator) (url *url.URL) {
s3 := a.config.S3
if s3 == nil {
logrus.Errorln("Missing S3 configuration")
return
}
scl, err := a.getCacheStorageClient(s3)
func (a *s3Adapter) getS3URL(generator s3URLGenerator) (url *url.URL) {
scl, err := a.getCacheStorageClient(a.config)
if err != nil {
logrus.Errorf("error while creating S3 cache storage client: %v", err)
return
}
url, err = generator(scl, s3.BucketName, a.objectName, a.timeout)
url, err = generator(scl, a.config.BucketName, a.objectName, a.timeout)
if err != nil {
logrus.Errorf("error while generating S3 pre-signed URL: %v", err)
return
......@@ -76,7 +56,10 @@ func (a *Adapter) getS3URL(generator s3URLGenerator) (url *url.URL) {
return
}
func (a *Adapter) getCacheStorageClient(s3 *common.CacheS3Config) (scl *minio.Client, err error) {
func (a *s3Adapter) getCacheStorageClient(s3 *common.CacheS3Config) (*minio.Client, error) {
var scl *minio.Client
var err error
// If the server address or credentials aren't specified then use IAM
// instance profile credentials and talk to "real" S3.
if s3.ServerAddress == "" || s3.AccessKey == "" || s3.SecretKey == "" {
......@@ -87,12 +70,28 @@ func (a *Adapter) getCacheStorageClient(s3 *common.CacheS3Config) (scl *minio.Cl
}
if err != nil {
return
return scl, err
}
scl.SetCustomTransport(&bucketLocationTripper{s3.BucketLocation})
return
return scl, nil
}
func NewAdapter(config *common.CacheConfig, timeout time.Duration, objectName string) (adapter.Adapter, error) {
c := deprecatedConfigHandler(config)
s3 := c.S3
if s3 == nil {
return nil, fmt.Errorf("missing S3 configuration")
}
a := new(s3Adapter)
a.config = s3
a.timeout = timeout
a.objectName = objectName
return a, nil
}
// TODO: Remove in 12.0
......@@ -115,3 +114,10 @@ var deprecatedConfigHandler = func(config *common.CacheConfig) *common.CacheConf
return config
}
func init() {
err := adapter.Factories().Register("s3", NewAdapter)
if err != nil {
panic(err)
}
}
......@@ -57,10 +57,9 @@ func TestS3CacheUploadURL(t *testing.T) {
s3Cache := defaultS3CacheFactory()
s3Cache.S3.Insecure = false
adapter := new(Adapter)
adapter.SetTimeout(defaultTimeout)
adapter.SetConfig(s3Cache)
adapter.SetObjectName("key")
adapter, err := NewAdapter(s3Cache, defaultTimeout, "key")
require.NoError(t, err)
url := adapter.GetUploadURL()
require.NotNil(t, url)
......@@ -82,10 +81,9 @@ func TestS3CacheUploadURLForIamCredentials(t *testing.T) {
s3Cache.S3.Insecure = false
adapter := new(Adapter)
adapter.SetTimeout(defaultTimeout)
adapter.SetConfig(s3Cache)
adapter.SetObjectName("key")
adapter, err := NewAdapter(s3Cache, defaultTimeout, "key")
require.NoError(t, err)
url := adapter.GetUploadURL()
require.NotNil(t, url)
......@@ -98,10 +96,9 @@ func TestS3CacheUploadInsecureURL(t *testing.T) {
s3Cache := defaultS3CacheFactory()
s3Cache.S3.Insecure = true
adapter := new(Adapter)
adapter.SetTimeout(defaultTimeout)
adapter.SetConfig(s3Cache)
adapter.SetObjectName("key")
adapter, err := NewAdapter(s3Cache, defaultTimeout, "key")
require.NoError(t, err)
url := adapter.GetUploadURL()
require.NotNil(t, url)
......@@ -113,10 +110,9 @@ func TestS3CacheDownloadURL(t *testing.T) {
s3Cache := defaultS3CacheFactory()
s3Cache.S3.Insecure = false
adapter := new(Adapter)
adapter.SetTimeout(defaultTimeout)
adapter.SetConfig(s3Cache)
adapter.SetObjectName("key")
adapter, err := NewAdapter(s3Cache, defaultTimeout, "key")
require.NoError(t, err)
url := adapter.GetDownloadURL()
require.NotNil(t, url)
......@@ -128,10 +124,9 @@ func TestS3CacheDownloadInsecureURL(t *testing.T) {
s3Cache := defaultS3CacheFactory()
s3Cache.S3.Insecure = true
adapter := new(Adapter)
adapter.SetTimeout(defaultTimeout)
adapter.SetConfig(s3Cache)
adapter.SetObjectName("key")
adapter, err := NewAdapter(s3Cache, defaultTimeout, "key")
require.NoError(t, err)
url := adapter.GetDownloadURL()
require.NotNil(t, url)
......@@ -154,10 +149,8 @@ func TestS3DeprecatedConfigFormatDetection(t *testing.T) {
runOnHijackedLogrusOutput(t, func(t *testing.T, output *bytes.Buffer) {
s3Cache := deprecatedS3CacheFactory()
adapter := new(Adapter)
adapter.SetTimeout(defaultTimeout)
adapter.SetConfig(s3Cache)
adapter.SetObjectName("key")
adapter, err := NewAdapter(s3Cache, defaultTimeout, "key")
require.NoError(t, err)
url := adapter.GetDownloadURL()
......@@ -179,14 +172,8 @@ func TestNoConfiguration(t *testing.T) {
s3Cache := defaultS3CacheFactory()
s3Cache.S3 = nil
adapter := new(Adapter)
adapter.SetTimeout(defaultTimeout)
adapter.SetConfig(s3Cache)
adapter.SetObjectName("key")
url := adapter.GetDownloadURL()
assert.Nil(t, url)
assert.Contains(t, output.String(), "Missing S3 configuration")
adapter, err := NewAdapter(s3Cache, defaultTimeout, "key")
assert.Nil(t, adapter)
assert.EqualError(t, err, "missing S3 configuration")
})
}
// Code generaadapterted by mockery v1.0.0. DO NOT EDIT.
// Code generated by mockery v1.0.0. DO NOT EDIT.
// This comment works around https://github.com/vektra/mockery/issues/155
......
package shells
package cache
import (
"net/url"
......@@ -8,9 +8,13 @@ import (
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/shells/cache"
"gitlab.com/gitlab-org/gitlab-runner/shells/cache/adapter"
_ "gitlab.com/gitlab-org/gitlab-runner/shells/cache/adapters/gcs"
_ "gitlab.com/gitlab-org/gitlab-runner/shells/cache/adapters/s3"
)
var factorizeAdapter = adapter.FactorizeAdapter
func getCacheConfig(build *common.Build) *common.CacheConfig {
if build == nil || build.Runner == nil || build.Runner.Cache == nil {
return nil
......@@ -32,20 +36,20 @@ func generateObjectName(build *common.Build, config *common.CacheConfig, key str
return path.Join(config.Path, runnerSegment, "project", strconv.Itoa(build.JobInfo.ProjectID), key)
}
func onAdapter(build *common.Build, key string, handler func(adapter cache.Adapter) *url.URL) *url.URL {
func onAdapter(build *common.Build, key string, handler func(adapter adapter.Adapter) *url.URL) *url.URL {
config := getCacheConfig(build)
if config == nil {
logrus.Warnln("Cache config not defined. Skipping cache operation.")
logrus.Warning("Cache config not defined. Skipping cache operation.")
return nil
}
objectName := generateObjectName(build, config, key)
if objectName == "" {
logrus.Warnln("ObjectName is empty. Skipping adapter selection.")
logrus.Warning("ObjectName is empty. Skipping adapter selection.")
return nil
}
adapter := cache.Factory(config, build.GetBuildTimeout(), objectName)
adapter := factorizeAdapter(config, build.GetBuildTimeout(), objectName)
if adapter == nil {
return nil
}
......@@ -53,14 +57,14 @@ func onAdapter(build *common.Build, key string, handler func(adapter cache.Adapt
return handler(adapter)
}
func getCacheDownloadURL(build *common.Build, key string) *url.URL {
return onAdapter(build, key, func(adapter cache.Adapter) *url.URL {
func GetCacheDownloadURL(build *common.Build, key string) *url.URL {
return onAdapter(build, key, func(adapter adapter.Adapter) *url.URL {
return adapter.GetDownloadURL()
})
}
func getCacheUploadURL(build *common.Build, key string) *url.URL {
return onAdapter(build, key, func(adapter cache.Adapter) *url.URL {
func GetCacheUploadURL(build *common.Build, key string) *url.URL {
return onAdapter(build, key, func(adapter adapter.Adapter) *url.URL {
return adapter.GetUploadURL()
})
}
package shells
package cache
import (
"bytes"
"net/url"
"testing"
"time"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/shells/cache"
"gitlab.com/gitlab-org/gitlab-runner/shells/cache/adapter"
)
func runOnHijackedLogrusOutput(handler func(output *bytes.Buffer)) {
oldOutput := logrus.StandardLogger().Out
defer func() { logrus.StandardLogger().Out = oldOutput }()
buf := bytes.NewBuffer([]byte{})
logrus.StandardLogger().Out = buf
handler(buf)
}
func TestCacheOperations(t *testing.T) {
type test struct {
testedOperation func(build *common.Build, key string) (url *url.URL)
key string
configExists bool
testedOperation func(build *common.Build, key string) *url.URL
mockedOperation string
adapterExists bool
adapterURL *url.URL
expectedURL *url.URL
expectedOutput string
}
exampleURL, err := url.Parse("example.com")
require.NoError(t, err)
tests := map[string]test{
"no-config": {
key: "key",
testedOperation: GetCacheDownloadURL,
mockedOperation: "GetDownloadURL",
adapterExists: true,
adapterURL: nil,
expectedURL: nil,
expectedOutput: "Cache config not defined. Skipping cache operation.",
},
"key-not-specified": {
configExists: true,
testedOperation: GetCacheDownloadURL,
mockedOperation: "GetDownloadURL",
adapterExists: true,
adapterURL: nil,
expectedURL: nil,
expectedOutput: "ObjectName is empty. Skipping adapter selection.",
},
"download-url-adapter-exists": {
testedOperation: getCacheDownloadURL,
key: "key",
configExists: true,
testedOperation: GetCacheDownloadURL,
mockedOperation: "GetDownloadURL",
adapterExists: true,
adapterURL: exampleURL,
expectedURL: exampleURL,
},
"upload-url-adapter-exists": {
testedOperation: getCacheUploadURL,
key: "key",
configExists: true,
testedOperation: GetCacheUploadURL,
mockedOperation: "GetUploadURL",
adapterExists: true,
adapterURL: exampleURL,
expectedURL: exampleURL,
},
"download-url-adapter-doesnt-exists": {
testedOperation: getCacheDownloadURL,
key: "key",
configExists: true,
testedOperation: GetCacheDownloadURL,
mockedOperation: "GetDownloadURL",
adapterExists: false,
adapterURL: exampleURL,
expectedURL: nil,
},
"upload-url-adapter-doesnt-exists": {
testedOperation: getCacheUploadURL,
key: "key",
configExists: true,
testedOperation: GetCacheUploadURL,
mockedOperation: "GetUploadURL",
adapterExists: false,
adapterURL: exampleURL,
......@@ -57,30 +98,42 @@ func TestCacheOperations(t *testing.T) {
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
build := &common.Build{
Runner: &common.RunnerConfig{
RunnerSettings: common.RunnerSettings{
Cache: &common.CacheConfig{},
runOnHijackedLogrusOutput(func(output *bytes.Buffer) {
build := &common.Build{
Runner: &common.RunnerConfig{
RunnerSettings: common.RunnerSettings{},
},
},
}
key := "key"
}
if tc.configExists {
build.Runner.Cache = &common.CacheConfig{}
}
var cacheAdapter adapter.Adapter
if tc.adapterExists {
a := new(adapter.MockAdapter)
if tc.adapterURL != nil {
a.On(tc.mockedOperation).Return(tc.adapterURL)
}
defer a.AssertExpectations(t)
var adapter cache.Adapter
if tc.adapterExists {
a := new(cache.MockAdapter)
a.On(tc.mockedOperation).Return(tc.adapterURL)
defer a.AssertExpectations(t)
cacheAdapter = a
}
adapter = a
}
factorizeAdapter = func(cacheConfig *common.CacheConfig, timeout time.Duration, objectName string) adapter.Adapter {
return cacheAdapter
}
cache.Factory = func(cacheConfig *common.CacheConfig, timeout time.Duration, objectName string) cache.Adapter {
return adapter
}
generatedURL := tc.testedOperation(build, tc.key)
assert.Equal(t, tc.expectedURL, generatedURL)
generatedURL := tc.testedOperation(build, key)
assert.Equal(t, tc.expectedURL, generatedURL)
if tc.expectedOutput == "" {
assert.Empty(t, output.String())
} else {
assert.Contains(t, output.String(), tc.expectedOutput)
}
})
})
}
}
......
// Code generated by mockery v1.0.0. DO NOT EDIT.
// This comment works around https://github.com/vektra/mockery/issues/155
package gcs
import mock "github.com/stretchr/testify/mock"
import storage "cloud.google.com/go/storage"
// MockSignedURLGenerator is an autogenerated mock type for the SignedURLGenerator type
type MockSignedURLGenerator struct {
mock.Mock
}
// GetSignedURL provides a mock function with given fields: bucket, name, opts
func (_m *MockSignedURLGenerator) GetSignedURL(bucket string, name string, opts *storage.SignedURLOptions) (string, error) {
ret := _m.Called(bucket, name, opts)
var r0 string
if rf, ok := ret.Get(0).(func(string, string, *storage.SignedURLOptions) string); ok {
r0 = rf(bucket, name, opts)
} else {
r0 = ret.Get(0).(string)
}
var r1 error
if rf, ok := ret.Get(1).(func(string, string, *storage.SignedURLOptions) error); ok {
r1 = rf(bucket, name, opts)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment