Skip to content
Snippets Groups Projects

Introduce GCS adapter for remote cache

Merged Tomasz Maczukin requested to merge introduce-gcs-cache-support into master
10 files
+ 632
282
Compare changes
  • Side-by-side
  • Inline
Files
10
+ 90
0
package s3
import (
"net/url"
"time"
"github.com/minio/minio-go"
"github.com/minio/minio-go/pkg/credentials"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type fakeIAMCredentialsProvider interface {
credentials.Provider
}
var iamFactory = func() *credentials.Credentials {
return credentials.NewIAM("")
}
type s3URLGenerator func(scl *minio.Client, bucketName string, objectName string, expires time.Duration) (*url.URL, error)
type Adapter struct {
build *common.Build
config *common.CacheConfig
objectName string
}
func (a *Adapter) SetBuild(build *common.Build) {
a.build = build
}
func (a *Adapter) SetConfig(config *common.CacheConfig) {
a.config = config
}
func (a *Adapter) SetObjectName(objectName string) {
a.objectName = objectName
}
func (a *Adapter) GetDownloadURL() *url.URL {
return a.getS3URL(getS3DownloadURL)
}
func getS3DownloadURL(scl *minio.Client, bucketName string, objectName string, expires time.Duration) (*url.URL, error) {
return scl.PresignedGetObject(bucketName, objectName, expires, nil)
}
func (a *Adapter) GetUploadURL() *url.URL {
return a.getS3URL(getS3UploadURL)
}
func getS3UploadURL(scl *minio.Client, bucketName string, objectName string, expires time.Duration) (*url.URL, error) {
return scl.PresignedPutObject(bucketName, objectName, expires)
}
func (a *Adapter) getS3URL(generator s3URLGenerator) (url *url.URL) {
scl, err := a.getCacheStorageClient()
if err != nil {
logrus.Errorf("error while creating S3 cache storage client: %v", err)
return
}
url, err = generator(scl, a.config.BucketName, a.objectName, time.Second*time.Duration(a.build.RunnerInfo.Timeout))
if err != nil {
logrus.Errorf("error while generating S3 pre-signed URL: %v", err)
return
}
return
}
func (a *Adapter) getCacheStorageClient() (scl *minio.Client, err error) {
// If the server address or credentials aren't specified then use IAM
// instance profile credentials and talk to "real" S3.
if a.config.ServerAddress == "" || a.config.AccessKey == "" || a.config.SecretKey == "" {
iam := iamFactory()
scl, err = minio.NewWithCredentials("s3.amazonaws.com", iam, true, "")
} else {
scl, err = minio.New(a.config.ServerAddress, a.config.AccessKey, a.config.SecretKey, !a.config.Insecure)
}
if err != nil {
return
}
scl.SetCustomTransport(&bucketLocationTripper{a.config.BucketLocation})
return
}
Loading