Commit 0177466a authored by Alessio Caiazza's avatar Alessio Caiazza 💡
Browse files

Merge branch 'introduce-gcs-cache-support' into 'master'

Introduce GCS cache support

Closes #1773

See merge request !968
parents 91038c92 ebaf658c
Pipeline #29384606 passed with stages
in 42 minutes and 7 seconds
This diff is collapsed.
......@@ -115,7 +115,7 @@ ignored = ["test", "appengine"]
[[constraint]]
name = "golang.org/x/net"
revision = "f2499483f923065a842d38eb4c7f1927e6fc6e6d"
revision = "d0887baf81f4598189d4e12a37c6da86f0bba4d0"
[[constraint]]
# branch = "v2"
......@@ -138,6 +138,17 @@ ignored = ["test", "appengine"]
name = "github.com/mattn/go-zglob"
branch = "master"
[[constraint]]
name = "github.com/docker/go-units"
revision = "eb879ae3e2b84e2a142af415b679ddeda47ec71c"
[[constraint]]
name = "cloud.google.com/go"
version = "v0.25.0"
[[constraint]]
name = "gitlab.com/gitlab-org/gitlab-terminal"
revision = "d523b4fd2bb3c8728724dce365809e09113430a9"
##
## Refrain innovations ;)
......@@ -147,10 +158,6 @@ ignored = ["test", "appengine"]
name = "github.com/Azure/go-autorest"
revision = "d4e6b95c12a08b4de2d48b45d5b4d594e5d32fab"
[[override]]
name = "cloud.google.com/go"
revision = "05253f6a829103296c351b643f6815aedd81a3fb"
[[override]]
name = "github.com/Azure/go-ansiterm"
revision = "19f72df4d05d31cbe1c56bfc8045c96babff6c7e"
......@@ -175,10 +182,6 @@ ignored = ["test", "appengine"]
name = "github.com/docker/engine-api"
revision = "4290f40c056686fcaa5c9caf02eac1dde9315adf"
[[override]]
name = "github.com/docker/go-units"
revision = "eb879ae3e2b84e2a142af415b679ddeda47ec71c"
[[override]]
name = "github.com/docker/spdystream"
revision = "449fdfce4d962303d702fec724ef0ad181c92528"
......@@ -197,11 +200,11 @@ ignored = ["test", "appengine"]
[[override]]
name = "github.com/gogo/protobuf"
revision = "f20a1444730c7d9949b880a0309e737d007def25"
version = "v1.1.0"
[[override]]
name = "github.com/golang/protobuf"
revision = "f592bd283e9ef86337a432eb50e592278c3d534d"
version = "v1.1.0"
[[override]]
name = "github.com/google/cadvisor"
......@@ -271,16 +274,38 @@ ignored = ["test", "appengine"]
[[override]]
name = "golang.org/x/oauth2"
revision = "3b966c7f301c0c71c53d94dc632a62df0a682cd7"
revision = "ef147856a6ddbb60760db74283d2424e98c87bff"
[[override]]
name = "golang.org/x/sys"
revision = "042a8f53ce82bbe081222da955159491e32146a0"
[[override]]
name = "golang.org/x/text"
version = "v0.3.0"
[[override]]
name = "google.golang.org/api"
revision = "0025a57598c017c1b9f7bc916c4fb8ae77315f9c"
[[override]]
name = "google.golang.org/appengine"
revision = "e951d3868b377b14f4e60efa3a301532ee3c1ebf"
[[constraint]]
name = "gitlab.com/gitlab-org/gitlab-terminal"
revision = "d523b4fd2bb3c8728724dce365809e09113430a9"
[[override]]
name = "google.golang.org/genproto"
revision = "2731d4fa720b67f9fe38e9051a2a9b38e4609260"
[[override]]
name = "github.com/googleapis/gax-go"
version = "v2.0.0"
[[override]]
name = "go.opencensus.io"
version = "v0.14.0"
[[override]]
name = "google.golang.org/grpc"
# v1.14.0 introduced a compilation error with one of it dependencies,
# so forcing v1.13.0 which works
version = "=v1.13.0"
......@@ -154,11 +154,11 @@ mocks: $(MOCKERY)
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./vendor/github.com/ayufan/golang-kardianos-service -output=./helpers/service/mocks -name='(Interface|Logger)'
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./helpers/docker -all -inpkg
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./helpers/certificate -all -inpkg
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./cache -all -inpkg
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./common -all -inpkg
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./session -all -inpkg
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./shells -all -inpkg
test-docker:
make test-docker-image IMAGE=centos:6 TYPE=rpm
make test-docker-image IMAGE=centos:7 TYPE=rpm
......
package cache
import (
"fmt"
"net/url"
"sync"
"time"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type Adapter interface {
GetDownloadURL() *url.URL
GetUploadURL() *url.URL
}
type Factory func(config *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error)
type FactoriesMap struct {
internal map[string]Factory
lock sync.RWMutex
}
func (m *FactoriesMap) Register(typeName string, factory Factory) error {
m.lock.Lock()
defer m.lock.Unlock()
if len(m.internal) == 0 {
m.internal = make(map[string]Factory)
}
_, ok := m.internal[typeName]
if ok {
return fmt.Errorf("adapter %q already registered", typeName)
}
m.internal[typeName] = factory
return nil
}
func (m *FactoriesMap) Find(typeName string) (Factory, error) {
m.lock.RLock()
defer m.lock.RUnlock()
factory := m.internal[typeName]
if factory == nil {
return nil, fmt.Errorf("factory for cache adapter %q was not registered", typeName)
}
return factory, nil
}
var factories = &FactoriesMap{}
func Factories() *FactoriesMap {
return factories
}
func CreateAdapter(cacheConfig *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error) {
create, err := Factories().Find(cacheConfig.Type)
if err != nil {
return nil, fmt.Errorf("cache factory not found: %v", err)
}
adapter, err := create(cacheConfig, timeout, objectName)
if err != nil {
return nil, fmt.Errorf("cache adapter could not be initialized: %v", err)
}
return adapter, nil
}
package cache
import (
"errors"
"testing"
"time"
"github.com/stretchr/testify/assert"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
var defaultTimeout = 1 * time.Hour
type factorizeTestCase struct {
adapter Adapter
errorOnFactorize error
expectedError string
expectedAdapter Adapter
}
func prepareMockedFactoriesMap() func() {
oldFactories := factories
factories = &FactoriesMap{}
return func() {
factories = oldFactories
}
}
func makeTestFactory(test factorizeTestCase) Factory {
return func(config *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error) {
if test.errorOnFactorize != nil {
return nil, test.errorOnFactorize
}
return test.adapter, nil
}
}
func TestCreateAdapter(t *testing.T) {
adapterMock := new(MockAdapter)
tests := map[string]factorizeTestCase{
"adapter doesn't exist": {
adapter: nil,
errorOnFactorize: nil,
expectedAdapter: nil,
expectedError: `cache factory not found: factory for cache adapter \"test\" was not registered`,
},
"adapter exists": {
adapter: adapterMock,
errorOnFactorize: nil,
expectedAdapter: adapterMock,
expectedError: "",
},
"adapter errors on factorize": {
adapter: adapterMock,
errorOnFactorize: errors.New("test error"),
expectedAdapter: nil,
expectedError: `cache adapter could not be initialized: test error`,
},
}
for name, test := range tests {
t.Run(name, func(t *testing.T) {
cleanupFactoriesMap := prepareMockedFactoriesMap()
defer cleanupFactoriesMap()
adapterTypeName := "test"
if test.adapter != nil {
factories.Register(adapterTypeName, makeTestFactory(test))
}
factories.Register("additional-adapter", func(config *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error) {
return new(MockAdapter), nil
})
config := &common.CacheConfig{
Type: adapterTypeName,
}
adapter, err := CreateAdapter(config, defaultTimeout, "key")
if test.expectedError == "" {
assert.NoError(t, err)
} else {
assert.Error(t, err)
}
assert.Equal(t, test.expectedAdapter, adapter)
})
}
}
func TestDoubledRegistration(t *testing.T) {
adapterTypeName := "test"
fakeFactory := func(config *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error) {
return nil, nil
}
f := &FactoriesMap{}
err := f.Register(adapterTypeName, fakeFactory)
assert.NoError(t, err)
assert.Len(t, f.internal, 1)
err = f.Register(adapterTypeName, fakeFactory)
assert.Error(t, err)
assert.Len(t, f.internal, 1)
}
package cache
import (
"net/url"
"path"
"strconv"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
var createAdapter = CreateAdapter
func getCacheConfig(build *common.Build) *common.CacheConfig {
if build == nil || build.Runner == nil || build.Runner.Cache == nil {
return nil
}
return build.Runner.Cache
}
func generateObjectName(build *common.Build, config *common.CacheConfig, key string) string {
if key == "" {
return ""
}
runnerSegment := ""
if !config.Shared {
runnerSegment = path.Join("runner", build.Runner.ShortDescription())
}
return path.Join(config.Path, runnerSegment, "project", strconv.Itoa(build.JobInfo.ProjectID), key)
}
func onAdapter(build *common.Build, key string, handler func(adapter Adapter) *url.URL) *url.URL {
config := getCacheConfig(build)
if config == nil {
logrus.Warning("Cache config not defined. Skipping cache operation.")
return nil
}
objectName := generateObjectName(build, config, key)
if objectName == "" {
logrus.Warning("Empty cache key. Skipping adapter selection.")
return nil
}
adapter, err := createAdapter(config, build.GetBuildTimeout(), objectName)
if err != nil {
logrus.WithError(err).Error("Could not create cache adapter")
}
if adapter == nil {
return nil
}
return handler(adapter)
}
func GetCacheDownloadURL(build *common.Build, key string) *url.URL {
return onAdapter(build, key, func(adapter Adapter) *url.URL {
return adapter.GetDownloadURL()
})
}
func GetCacheUploadURL(build *common.Build, key string) *url.URL {
return onAdapter(build, key, func(adapter Adapter) *url.URL {
return adapter.GetUploadURL()
})
}
package cache
import (
"errors"
"net/url"
"testing"
"time"
"github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type cacheOperationTest struct {
key string
configExists bool
adapterExists bool
errorOnAdapterCreation bool
adapterURL *url.URL
expectedURL *url.URL
expectedOutput []string
}
func prepareFakeCreateAdapter(t *testing.T, operationName string, tc cacheOperationTest) func() {
assertAdapterExpectations := func(t mock.TestingT) bool { return true }
var cacheAdapter Adapter
if tc.adapterExists {
a := new(MockAdapter)
if tc.adapterURL != nil {
a.On(operationName).Return(tc.adapterURL)
}
assertAdapterExpectations = a.AssertExpectations
cacheAdapter = a
}
var cacheAdapterCreationError error
if tc.errorOnAdapterCreation {
cacheAdapterCreationError = errors.New("test error")
}
oldCreateAdapter := createAdapter
createAdapter = func(cacheConfig *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error) {
return cacheAdapter, cacheAdapterCreationError
}
return func() {
createAdapter = oldCreateAdapter
assertAdapterExpectations(t)
}
}
func prepareFakeBuild(tc cacheOperationTest) *common.Build {
build := &common.Build{
Runner: &common.RunnerConfig{
RunnerSettings: common.RunnerSettings{},
},
}
if tc.configExists {
build.Runner.Cache = &common.CacheConfig{}
}
return build
}
func testCacheOperation(t *testing.T, operationName string, operation func(build *common.Build, key string) *url.URL, tc cacheOperationTest) {
t.Run(operationName, func(t *testing.T) {
hook := test.NewGlobal()
cleanupCreateAdapter := prepareFakeCreateAdapter(t, operationName, tc)
defer cleanupCreateAdapter()
build := prepareFakeBuild(tc)
generatedURL := operation(build, tc.key)
assert.Equal(t, tc.expectedURL, generatedURL)
if len(tc.expectedOutput) == 0 {
assert.Len(t, hook.AllEntries(), 0)
} else {
for _, expectedOutput := range tc.expectedOutput {
message, err := hook.LastEntry().String()
require.NoError(t, err)
assert.Contains(t, message, expectedOutput)
}
}
})
}
func TestCacheOperations(t *testing.T) {
exampleURL, err := url.Parse("example.com")
require.NoError(t, err)
tests := map[string]cacheOperationTest{
"no-config": {
key: "key",
adapterExists: true,
adapterURL: nil,
expectedURL: nil,
expectedOutput: []string{"Cache config not defined. Skipping cache operation."},
},
"key-not-specified": {
configExists: true,
adapterExists: true,
adapterURL: nil,
expectedURL: nil,
expectedOutput: []string{"Empty cache key. Skipping adapter selection."},
},
"adapter-doesnt-exists": {
key: "key",
configExists: true,
adapterExists: false,
adapterURL: exampleURL,
expectedURL: nil,
},
"adapter-error-on-factorization": {
key: "key",
configExists: true,
errorOnAdapterCreation: true,
adapterURL: exampleURL,
expectedURL: nil,
expectedOutput: []string{
"Could not create cache adapter",
"test error",
},
},
"adapter-exists": {
key: "key",
configExists: true,
adapterExists: true,
adapterURL: exampleURL,
expectedURL: exampleURL,
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
testCacheOperation(t, "GetDownloadURL", GetCacheDownloadURL, tc)
testCacheOperation(t, "GetUploadURL", GetCacheUploadURL, tc)
})
}
}
func defaultCacheConfig() *common.CacheConfig {
return &common.CacheConfig{
Type: "test",
}
}
func defaultBuild(cacheConfig *common.CacheConfig) *common.Build {
return &common.Build{
JobResponse: common.JobResponse{
JobInfo: common.JobInfo{
ProjectID: 10,
},
RunnerInfo: common.RunnerInfo{
Timeout: 3600,
},
},
Runner: &common.RunnerConfig{
RunnerCredentials: common.RunnerCredentials{
Token: "longtoken",
},
RunnerSettings: common.RunnerSettings{
Cache: cacheConfig,
},
},
}
}
func TestGenerateObjectNameWhenKeyIsEmptyResultIsAlsoEmpty(t *testing.T) {
cache := defaultCacheConfig()
cacheBuild := defaultBuild(cache)
url := generateObjectName(cacheBuild, cache, "")
assert.Empty(t, url)
}
func TestGetCacheObjectName(t *testing.T) {
cache := defaultCacheConfig()
cacheBuild := defaultBuild(cache)
url := generateObjectName(cacheBuild, cache, "key")
assert.Equal(t, "runner/longtoke/project/10/key", url)
}
func TestGetCacheObjectNameWhenPathIsSetThenUrlContainsIt(t *testing.T) {
cache := defaultCacheConfig()
cache.Path = "whatever"
cacheBuild := defaultBuild(cache)
url := generateObjectName(cacheBuild, cache, "key")
assert.Equal(t, "whatever/runner/longtoke/project/10/key", url)
}
func TestGetCacheObjectNameWhenPathHasMultipleSegmentIsSetThenUrlContainsIt(t *testing.T) {
cache := defaultCacheConfig()
cache.Path = "some/other/path/goes/here"
cacheBuild := defaultBuild(cache)