Introduce GCS cache support

Gopkg.toml

@@ -115,7 +115,7 @@ ignored = ["test", "appengine"]
 
 [[constraint]]
   name = "golang.org/x/net"
-  revision = "f2499483f923065a842d38eb4c7f1927e6fc6e6d"
+  revision = "d0887baf81f4598189d4e12a37c6da86f0bba4d0"
 
 [[constraint]]
   # branch = "v2"
@@ -138,6 +138,17 @@ ignored = ["test", "appengine"]
   name = "github.com/mattn/go-zglob"
   branch = "master"
 
+[[constraint]]
+  name = "github.com/docker/go-units"
+  revision = "eb879ae3e2b84e2a142af415b679ddeda47ec71c"
+
+[[constraint]]
+  name = "cloud.google.com/go"
+  version = "v0.25.0"
+
+[[constraint]]
+  name = "gitlab.com/gitlab-org/gitlab-terminal"
+  revision = "d523b4fd2bb3c8728724dce365809e09113430a9"
 
 ##
 ## Refrain innovations ;)
@@ -147,10 +158,6 @@ ignored = ["test", "appengine"]
   name = "github.com/Azure/go-autorest"
   revision = "d4e6b95c12a08b4de2d48b45d5b4d594e5d32fab"
 
-[[override]]
-  name = "cloud.google.com/go"
-  revision = "05253f6a829103296c351b643f6815aedd81a3fb"
-
 [[override]]
   name = "github.com/Azure/go-ansiterm"
   revision = "19f72df4d05d31cbe1c56bfc8045c96babff6c7e"
@@ -175,10 +182,6 @@ ignored = ["test", "appengine"]
   name = "github.com/docker/engine-api"
   revision = "4290f40c056686fcaa5c9caf02eac1dde9315adf"
 
-[[override]]
-  name = "github.com/docker/go-units"
-  revision = "eb879ae3e2b84e2a142af415b679ddeda47ec71c"
-
 [[override]]
   name = "github.com/docker/spdystream"
   revision = "449fdfce4d962303d702fec724ef0ad181c92528"
@@ -197,11 +200,11 @@ ignored = ["test", "appengine"]
 
 [[override]]
   name = "github.com/gogo/protobuf"
-  revision = "f20a1444730c7d9949b880a0309e737d007def25"
+  version = "v1.1.0"
 
 [[override]]
   name = "github.com/golang/protobuf"
-  revision = "f592bd283e9ef86337a432eb50e592278c3d534d"
+  version = "v1.1.0"
 
 [[override]]
   name = "github.com/google/cadvisor"
@@ -271,16 +274,38 @@ ignored = ["test", "appengine"]
 
 [[override]]
   name = "golang.org/x/oauth2"
-  revision = "3b966c7f301c0c71c53d94dc632a62df0a682cd7"
+  revision = "ef147856a6ddbb60760db74283d2424e98c87bff"
 
 [[override]]
   name = "golang.org/x/sys"
   revision = "042a8f53ce82bbe081222da955159491e32146a0"
 
+[[override]]
+  name = "golang.org/x/text"
+  version = "v0.3.0"
+
+[[override]]
+  name = "google.golang.org/api"
+  revision = "0025a57598c017c1b9f7bc916c4fb8ae77315f9c"
+
 [[override]]
   name = "google.golang.org/appengine"
   revision = "e951d3868b377b14f4e60efa3a301532ee3c1ebf"
 
-[[constraint]]
-  name = "gitlab.com/gitlab-org/gitlab-terminal"
-  revision = "d523b4fd2bb3c8728724dce365809e09113430a9"
+[[override]]
+  name = "google.golang.org/genproto"
+  revision = "2731d4fa720b67f9fe38e9051a2a9b38e4609260"
+
+[[override]]
+  name = "github.com/googleapis/gax-go"
+  version = "v2.0.0"
+
+[[override]]
+  name = "go.opencensus.io"
+  version = "v0.14.0"
+
+[[override]]
+  name = "google.golang.org/grpc"
+  # v1.14.0 introduced a compilation error with one of it dependencies,
+  # so forcing v1.13.0 which works
+  version = "=v1.13.0"

Makefile

@@ -154,11 +154,11 @@ mocks: $(MOCKERY)
 	GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./vendor/github.com/ayufan/golang-kardianos-service -output=./helpers/service/mocks -name='(Interface|Logger)'
 	GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./helpers/docker -all -inpkg
 	GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./helpers/certificate -all -inpkg
+	GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./cache -all -inpkg
 	GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./common -all -inpkg
 	GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./session -all -inpkg
 	GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./shells -all -inpkg
 
-
 test-docker:
 	make test-docker-image IMAGE=centos:6 TYPE=rpm
 	make test-docker-image IMAGE=centos:7 TYPE=rpm

cache/adapter.go

+package cache
+
+import (
+	"fmt"
+	"net/url"
+	"sync"
+	"time"
+
+	"gitlab.com/gitlab-org/gitlab-runner/common"
+)
+
+type Adapter interface {
+	GetDownloadURL() *url.URL
+	GetUploadURL() *url.URL
+}
+
+type Factory func(config *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error)
+
+type FactoriesMap struct {
+	internal map[string]Factory
+	lock     sync.RWMutex
+}
+
+func (m *FactoriesMap) Register(typeName string, factory Factory) error {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	if len(m.internal) == 0 {
+		m.internal = make(map[string]Factory)
+	}
+
+	_, ok := m.internal[typeName]
+	if ok {
+		return fmt.Errorf("adapter %q already registered", typeName)
+	}
+
+	m.internal[typeName] = factory
+
+	return nil
+}
+
+func (m *FactoriesMap) Find(typeName string) (Factory, error) {
+	m.lock.RLock()
+	defer m.lock.RUnlock()
+
+	factory := m.internal[typeName]
+	if factory == nil {
+		return nil, fmt.Errorf("factory for cache adapter %q was not registered", typeName)
+	}
+
+	return factory, nil
+}
+
+var factories = &FactoriesMap{}
+
+func Factories() *FactoriesMap {
+	return factories
+}
+
+func CreateAdapter(cacheConfig *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error) {
+	create, err := Factories().Find(cacheConfig.Type)
+	if err != nil {
+		return nil, fmt.Errorf("cache factory not found: %v", err)
+	}
+
+	adapter, err := create(cacheConfig, timeout, objectName)
+	if err != nil {
+		return nil, fmt.Errorf("cache adapter could not be initialized: %v", err)
+	}
+
+	return adapter, nil
+}

cache/adapter_test.go

+package cache
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"gitlab.com/gitlab-org/gitlab-runner/common"
+)
+
+var defaultTimeout = 1 * time.Hour
+
+type factorizeTestCase struct {
+	adapter          Adapter
+	errorOnFactorize error
+	expectedError    string
+	expectedAdapter  Adapter
+}
+
+func prepareMockedFactoriesMap() func() {
+	oldFactories := factories
+	factories = &FactoriesMap{}
+
+	return func() {
+		factories = oldFactories
+	}
+}
+
+func makeTestFactory(test factorizeTestCase) Factory {
+	return func(config *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error) {
+		if test.errorOnFactorize != nil {
+			return nil, test.errorOnFactorize
+		}
+
+		return test.adapter, nil
+	}
+}
+
+func TestCreateAdapter(t *testing.T) {
+	adapterMock := new(MockAdapter)
+
+	tests := map[string]factorizeTestCase{
+		"adapter doesn't exist": {
+			adapter:          nil,
+			errorOnFactorize: nil,
+			expectedAdapter:  nil,
+			expectedError:    `cache factory not found: factory for cache adapter \"test\" was not registered`,
+		},
+		"adapter exists": {
+			adapter:          adapterMock,
+			errorOnFactorize: nil,
+			expectedAdapter:  adapterMock,
+			expectedError:    "",
+		},
+		"adapter errors on factorize": {
+			adapter:          adapterMock,
+			errorOnFactorize: errors.New("test error"),
+			expectedAdapter:  nil,
+			expectedError:    `cache adapter could not be initialized: test error`,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			cleanupFactoriesMap := prepareMockedFactoriesMap()
+			defer cleanupFactoriesMap()
+
+			adapterTypeName := "test"
+
+			if test.adapter != nil {
+				factories.Register(adapterTypeName, makeTestFactory(test))
+			}
+
+			factories.Register("additional-adapter", func(config *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error) {
+				return new(MockAdapter), nil
+			})
+
+			config := &common.CacheConfig{
+				Type: adapterTypeName,
+			}
+
+			adapter, err := CreateAdapter(config, defaultTimeout, "key")
+
+			if test.expectedError == "" {
+				assert.NoError(t, err)
+			} else {
+				assert.Error(t, err)
+			}
+
+			assert.Equal(t, test.expectedAdapter, adapter)
+		})
+	}
+}
+
+func TestDoubledRegistration(t *testing.T) {
+	adapterTypeName := "test"
+	fakeFactory := func(config *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error) {
+		return nil, nil
+	}
+
+	f := &FactoriesMap{}
+
+	err := f.Register(adapterTypeName, fakeFactory)
+	assert.NoError(t, err)
+	assert.Len(t, f.internal, 1)
+
+	err = f.Register(adapterTypeName, fakeFactory)
+	assert.Error(t, err)
+	assert.Len(t, f.internal, 1)
+}

cache/cache.go

+package cache
+
+import (
+	"net/url"
+	"path"
+	"strconv"
+
+	"github.com/sirupsen/logrus"
+
+	"gitlab.com/gitlab-org/gitlab-runner/common"
+)
+
+var createAdapter = CreateAdapter
+
+func getCacheConfig(build *common.Build) *common.CacheConfig {
+	if build == nil || build.Runner == nil || build.Runner.Cache == nil {
+		return nil
+	}
+
+	return build.Runner.Cache
+}
+
+func generateObjectName(build *common.Build, config *common.CacheConfig, key string) string {
+	if key == "" {
+		return ""
+	}
+
+	runnerSegment := ""
+	if !config.Shared {
+		runnerSegment = path.Join("runner", build.Runner.ShortDescription())
+	}
+
+	return path.Join(config.Path, runnerSegment, "project", strconv.Itoa(build.JobInfo.ProjectID), key)
+}
+
+func onAdapter(build *common.Build, key string, handler func(adapter Adapter) *url.URL) *url.URL {
+	config := getCacheConfig(build)
+	if config == nil {
+		logrus.Warning("Cache config not defined. Skipping cache operation.")
+		return nil
+	}
+
+	objectName := generateObjectName(build, config, key)
+	if objectName == "" {
+		logrus.Warning("Empty cache key. Skipping adapter selection.")
+		return nil
+	}
+
+	adapter, err := createAdapter(config, build.GetBuildTimeout(), objectName)
+	if err != nil {
+		logrus.WithError(err).Error("Could not create cache adapter")
+	}
+
+	if adapter == nil {
+		return nil
+	}
+
+	return handler(adapter)
+}
+
+func GetCacheDownloadURL(build *common.Build, key string) *url.URL {
+	return onAdapter(build, key, func(adapter Adapter) *url.URL {
+		return adapter.GetDownloadURL()
+	})
+}
+
+func GetCacheUploadURL(build *common.Build, key string) *url.URL {
+	return onAdapter(build, key, func(adapter Adapter) *url.URL {
+		return adapter.GetUploadURL()
+	})
+}

cache/cache_test.go

+package cache
+
+import (
+	"errors"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-runner/common"
+)
+
+type cacheOperationTest struct {
+	key                    string
+	configExists           bool
+	adapterExists          bool
+	errorOnAdapterCreation bool
+	adapterURL             *url.URL
+	expectedURL            *url.URL
+	expectedOutput         []string
+}
+
+func prepareFakeCreateAdapter(t *testing.T, operationName string, tc cacheOperationTest) func() {
+	assertAdapterExpectations := func(t mock.TestingT) bool { return true }
+
+	var cacheAdapter Adapter
+	if tc.adapterExists {
+		a := new(MockAdapter)
+
+		if tc.adapterURL != nil {
+			a.On(operationName).Return(tc.adapterURL)
+		}
+
+		assertAdapterExpectations = a.AssertExpectations
+		cacheAdapter = a
+	}
+
+	var cacheAdapterCreationError error
+	if tc.errorOnAdapterCreation {
+		cacheAdapterCreationError = errors.New("test error")
+	}
+
+	oldCreateAdapter := createAdapter
+	createAdapter = func(cacheConfig *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error) {
+		return cacheAdapter, cacheAdapterCreationError
+	}
+
+	return func() {
+		createAdapter = oldCreateAdapter
+		assertAdapterExpectations(t)
+	}
+}
+
+func prepareFakeBuild(tc cacheOperationTest) *common.Build {
+	build := &common.Build{
+		Runner: &common.RunnerConfig{
+			RunnerSettings: common.RunnerSettings{},
+		},
+	}
+
+	if tc.configExists {
+		build.Runner.Cache = &common.CacheConfig{}
+	}
+
+	return build
+}
+
+func testCacheOperation(t *testing.T, operationName string, operation func(build *common.Build, key string) *url.URL, tc cacheOperationTest) {
+	t.Run(operationName, func(t *testing.T) {
+		hook := test.NewGlobal()
+
+		cleanupCreateAdapter := prepareFakeCreateAdapter(t, operationName, tc)
+		defer cleanupCreateAdapter()
+
+		build := prepareFakeBuild(tc)
+		generatedURL := operation(build, tc.key)
+		assert.Equal(t, tc.expectedURL, generatedURL)
+
+		if len(tc.expectedOutput) == 0 {
+			assert.Len(t, hook.AllEntries(), 0)
+		} else {
+			for _, expectedOutput := range tc.expectedOutput {
+				message, err := hook.LastEntry().String()
+				require.NoError(t, err)
+				assert.Contains(t, message, expectedOutput)
+			}
+		}
+	})
+}
+
+func TestCacheOperations(t *testing.T) {
+	exampleURL, err := url.Parse("example.com")
+	require.NoError(t, err)
+
+	tests := map[string]cacheOperationTest{
+		"no-config": {
+			key:            "key",
+			adapterExists:  true,
+			adapterURL:     nil,
+			expectedURL:    nil,
+			expectedOutput: []string{"Cache config not defined. Skipping cache operation."},
+		},
+		"key-not-specified": {
+			configExists:   true,
+			adapterExists:  true,
+			adapterURL:     nil,
+			expectedURL:    nil,
+			expectedOutput: []string{"Empty cache key. Skipping adapter selection."},
+		},
+		"adapter-doesnt-exists": {
+			key:           "key",
+			configExists:  true,
+			adapterExists: false,
+			adapterURL:    exampleURL,
+			expectedURL:   nil,
+		},
+		"adapter-error-on-factorization": {
+			key:                    "key",
+			configExists:           true,
+			errorOnAdapterCreation: true,
+			adapterURL:             exampleURL,
+			expectedURL:            nil,
+			expectedOutput: []string{
+				"Could not create cache adapter",
+				"test error",
+			},
+		},
+		"adapter-exists": {
+			key:           "key",
+			configExists:  true,
+			adapterExists: true,
+			adapterURL:    exampleURL,
+			expectedURL:   exampleURL,
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			testCacheOperation(t, "GetDownloadURL", GetCacheDownloadURL, tc)
+			testCacheOperation(t, "GetUploadURL", GetCacheUploadURL, tc)
+		})
+	}
+}
+
+func defaultCacheConfig() *common.CacheConfig {
+	return &common.CacheConfig{
+		Type: "test",
+	}
+}
+
+func defaultBuild(cacheConfig *common.CacheConfig) *common.Build {
+	return &common.Build{
+		JobResponse: common.JobResponse{
+			JobInfo: common.JobInfo{
+				ProjectID: 10,
+			},
+			RunnerInfo: common.RunnerInfo{
+				Timeout: 3600,
+			},
+		},
+		Runner: &common.RunnerConfig{
+			RunnerCredentials: common.RunnerCredentials{
+				Token: "longtoken",
+			},
+			RunnerSettings: common.RunnerSettings{
+				Cache: cacheConfig,
+			},
+		},
+	}
+}
+
+func TestGenerateObjectNameWhenKeyIsEmptyResultIsAlsoEmpty(t *testing.T) {
+	cache := defaultCacheConfig()
+	cacheBuild := defaultBuild(cache)
+
+	url := generateObjectName(cacheBuild, cache, "")
+	assert.Empty(t, url)
+}
+
+func TestGetCacheObjectName(t *testing.T) {
+	cache := defaultCacheConfig()
+	cacheBuild := defaultBuild(cache)
+
+	url := generateObjectName(cacheBuild, cache, "key")
+	assert.Equal(t, "runner/longtoke/project/10/key", url)
+}
+
+func TestGetCacheObjectNameWhenPathIsSetThenUrlContainsIt(t *testing.T) {
+	cache := defaultCacheConfig()
+	cache.Path = "whatever"
+	cacheBuild := defaultBuild(cache)
+
+	url := generateObjectName(cacheBuild, cache, "key")
+	assert.Equal(t, "whatever/runner/longtoke/project/10/key", url)
+}
+
+func TestGetCacheObjectNameWhenPathHasMultipleSegmentIsSetThenUrlContainsIt(t *testing.T) {
+	cache := defaultCacheConfig()
+	cache.Path = "some/other/path/goes/here"
+	cacheBuild := defaultBuild(cache)
+
+	url := generateObjectName(cacheBuild, cache, "key")
+	assert.Equal(t, "some/other/path/goes/here/runner/longtoke/project/10/key", url)
+}
+