Verified Commit b8d50d67 authored by Tomasz Maczukin's avatar Tomasz Maczukin

Merge branch 'use-iam-instance-profile-creds-for-s3-caches'

* use-iam-instance-profile-creds-for-s3-caches:
  Add tests
  Add note to docs to explain how to use IAM instance profiles to authenticate S3 access for cache configuration
  Use IAM instance profile credentials if S3 cache keys not provided
parents 1c0fe99d b798917c
Pipeline #17535122 passed with stages
in 37 minutes and 38 seconds
......@@ -207,6 +207,7 @@ mocks: $(MOCKERY)
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./vendor/github.com/ayufan/golang-kardianos-service -output=./helpers/service/mocks -name='(Interface|Logger)'
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./helpers/docker -all -inpkg
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./common -all -inpkg
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./shells -name fakeIAMCredentialsProvider -inpkg
test-docker:
make test-docker-image IMAGE=centos:6 TYPE=rpm
......
......@@ -74,7 +74,7 @@ Example:
In cases where the GitLab instance is exposed to an URL which can't be used
by the runner, a `clone_url` can be configured. For example; GitLab is exposed
to `https://gitlab.example.com`, but the runner can't reach that because of
a firewall setup. If the runner can reach the node on `192.168.1.23`,
a firewall setup. If the runner can reach the node on `192.168.1.23`,
the `clone_url` should be set to `"http://192.168.1.23`.
Only if the `clone_url` is set, the runner will construct a clone URL in the form
......@@ -464,6 +464,9 @@ Example:
> **Note:** For Amazon's S3 service the `ServerAddress` should always be `s3.amazonaws.com`. Minio S3 client will
> get bucket metadata and modify the URL to point to the valid region (eg. `s3-eu-west-1.amazonaws.com`) itself.
> **Note:** If any of `ServerAddress`, `AccessKey` or `SecretKey` aren't specified then the S3 client will use the
> IAM instance profile available to the instance.
## The [runners.kubernetes] section
> **Note:**
......
......@@ -11,6 +11,7 @@ import (
"time"
"github.com/minio/minio-go"
"github.com/minio/minio-go/pkg/credentials"
"github.com/Sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
......@@ -51,8 +52,23 @@ func getCacheObjectName(build *common.Build, cache *common.CacheConfig, key stri
return path.Join(cache.Path, runnerSegment, "project", strconv.Itoa(build.JobInfo.ProjectID), key)
}
type fakeIAMCredentialsProvider interface {
credentials.Provider
}
var iamFactory = func() *credentials.Credentials {
return credentials.NewIAM("")
}
func getCacheStorageClient(cache *common.CacheConfig) (scl *minio.Client, err error) {
scl, err = minio.New(cache.ServerAddress, cache.AccessKey, cache.SecretKey, !cache.Insecure)
//If the server address or credentials aren't specified then use IAM
//instance profile credentials and talk to "real" S3.
if cache.ServerAddress == "" || cache.AccessKey == "" || cache.SecretKey == "" {
iam := iamFactory()
scl, err = minio.NewWithCredentials("s3.amazonaws.com", iam, true, "")
} else {
scl, err = minio.New(cache.ServerAddress, cache.AccessKey, cache.SecretKey, !cache.Insecure)
}
if err != nil {
logrus.Warningln(err)
return
......
......@@ -3,6 +3,7 @@ package shells
import (
"testing"
"github.com/minio/minio-go/pkg/credentials"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
......@@ -12,14 +13,28 @@ import (
func defaultS3CacheFactory() *common.CacheConfig {
return &common.CacheConfig{
Type: "s3",
BucketName: "test",
BucketLocation: "location",
ServerAddress: "server.com",
AccessKey: "access",
SecretKey: "key",
BucketName: "test",
BucketLocation: "location",
}
}
func iamS3CacheFactory(t *testing.T) (*common.CacheConfig, *mockFakeIAMCredentialsProvider) {
cacheConfig := defaultS3CacheFactory()
cacheConfig.ServerAddress = ""
cacheConfig.AccessKey = ""
cacheConfig.SecretKey = ""
iamProvider := &mockFakeIAMCredentialsProvider{}
iamFactory = func() *credentials.Credentials {
return credentials.New(iamProvider)
}
return cacheConfig, iamProvider
}
func defaults3CacheBuild(cacheConfig *common.CacheConfig) *common.Build {
return &common.Build{
JobResponse: common.JobResponse{
......@@ -49,6 +64,27 @@ func TestS3CacheUploadURL(t *testing.T) {
require.NotNil(t, url)
assert.Equal(t, s3Cache.ServerAddress, url.Host)
assert.Regexp(t, "^https://", url)
assert.Contains(t, url.String(), "X-Amz-Credential=access%2F")
}
func TestS3CacheUploadURLForIamCredentials(t *testing.T) {
s3Cache, iamProvider := iamS3CacheFactory(t)
fakeValue := credentials.Value{
AccessKeyID: "access-from-iam",
SecretAccessKey: "secret-from-iam",
}
iamProvider.On("Retrieve").Return(fakeValue, nil).Once()
iamProvider.On("IsExpired").Return(false)
defer iamProvider.AssertExpectations(t)
s3Cache.Insecure = false
s3CacheBuild := defaults3CacheBuild(s3Cache)
url := getCacheUploadURL(s3CacheBuild, "key")
require.NotNil(t, url)
assert.Equal(t, "test.s3.amazonaws.com", url.Host)
assert.Regexp(t, "^https://", url)
assert.Contains(t, url.String(), "X-Amz-Credential=access-from-iam%2F")
}
func TestS3CacheUploadInsecureURL(t *testing.T) {
......
package shells
import credentials "github.com/minio/minio-go/pkg/credentials"
import mock "github.com/stretchr/testify/mock"
// mockFakeIAMCredentialsProvider is an autogenerated mock type for the fakeIAMCredentialsProvider type
type mockFakeIAMCredentialsProvider struct {
mock.Mock
}
// IsExpired provides a mock function with given fields:
func (_m *mockFakeIAMCredentialsProvider) IsExpired() bool {
ret := _m.Called()
var r0 bool
if rf, ok := ret.Get(0).(func() bool); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(bool)
}
return r0
}
// Retrieve provides a mock function with given fields:
func (_m *mockFakeIAMCredentialsProvider) Retrieve() (credentials.Value, error) {
ret := _m.Called()
var r0 credentials.Value
if rf, ok := ret.Get(0).(func() credentials.Value); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(credentials.Value)
}
var r1 error
if rf, ok := ret.Get(1).(func() error); ok {
r1 = rf()
} else {
r1 = ret.Error(1)
}
return r0, r1
}
var _ fakeIAMCredentialsProvider = (*mockFakeIAMCredentialsProvider)(nil)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment