Commit e397c4bc authored by Kamil Trzciński's avatar Kamil Trzciński

Allow to use S3 buckets to store the caches

parent d71852cd
package helpers
import (
"fmt"
"net/http"
"os"
"path/filepath"
"time"
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
......@@ -11,6 +17,42 @@ import (
type CacheArchiverCommand struct {
fileArchiver
File string `long:"file" description:"The path to file"`
URL string `long:"url" description:"Download artifacts instead of uploading them"`
}
func (c *CacheArchiverCommand) upload() error {
logrus.Infoln("Uploading", filepath.Base(c.File))
file, err := os.Open(c.File)
if err != nil {
return err
}
defer file.Close()
fi, err := file.Stat()
if err != nil {
return err
}
req, err := http.NewRequest("PUT", c.URL, file)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))
req.ContentLength = fi.Size()
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return fmt.Errorf("Received: %s", resp.Status)
}
return nil
}
func (c *CacheArchiverCommand) Execute(*cli.Context) {
......@@ -35,6 +77,18 @@ func (c *CacheArchiverCommand) Execute(*cli.Context) {
if err != nil {
logrus.Fatalln(err)
}
// Upload archive if needed
if c.URL != "" {
for i := 0; i < 3; i++ {
err := c.upload()
if err == nil {
break
}
logrus.Warningln(err)
time.Sleep(time.Second)
}
}
}
func init() {
......
package helpers
import (
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"time"
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
......@@ -13,6 +19,49 @@ import (
type CacheExtractorCommand struct {
File string `long:"file" description:"The file containing your cache artifacts"`
URL string `long:"url" description:"Download artifacts instead of uploading them"`
}
func (c *CacheExtractorCommand) download() error {
os.MkdirAll(filepath.Dir(c.File), 0600)
file, err := ioutil.TempFile(filepath.Dir(c.File), "cache")
if err != nil {
return err
}
defer file.Close()
defer os.Remove(file.Name())
resp, err := http.Get(c.URL)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == 404 {
return os.ErrNotExist
} else if resp.StatusCode != 200 {
return fmt.Errorf("Received: %s", resp.Status)
}
fi, _ := os.Lstat(c.File)
date, _ := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified"))
if fi != nil && !date.After(fi.ModTime()) {
logrus.Infoln(filepath.Base(c.File), "is up to date")
return nil
}
logrus.Infoln("Downloading", filepath.Base(c.File))
_, err = io.Copy(file, resp.Body)
if err != nil {
return err
}
os.Chtimes(file.Name(), time.Now(), date)
err = os.Rename(file.Name(), c.File)
if err != nil {
return err
}
return nil
}
func (c *CacheExtractorCommand) Execute(context *cli.Context) {
......@@ -22,6 +71,13 @@ func (c *CacheExtractorCommand) Execute(context *cli.Context) {
logrus.Fatalln("Missing cache file")
}
if c.URL != "" {
err := c.download()
if err != nil && !os.IsNotExist(err) {
logrus.Warningln(err)
}
}
err := archives.ExtractZipFile(c.File)
if err != nil && !os.IsNotExist(err) {
logrus.Fatalln(err)
......
......@@ -50,6 +50,16 @@ type RunnerCredentials struct {
TLSCAFile string `toml:"tls-ca-file,omitempty" json:"tls-ca-file" long:"tls-ca-file" env:"CI_SERVER_TLS_CA_FILE" description:"File containing the certificates to verify the peer when using HTTPS"`
}
type CacheConfig struct {
UseS3 bool
ServerAddress string
AccessKey string
SecretKey string
BucketName string
BucketLocation string
Insecure bool
}
type RunnerSettings struct {
Executor string `toml:"executor" json:"executor" long:"executor" env:"RUNNER_EXECUTOR" required:"true" description:"Select executor, eg. shell, docker, etc."`
BuildsDir string `toml:"builds_dir,omitempty" json:"builds_dir" long:"builds-dir" env:"RUNNER_BUILDS_DIR" description:"Directory where builds are stored"`
......@@ -63,6 +73,7 @@ type RunnerSettings struct {
Docker *DockerConfig `toml:"docker" json:"docker" group:"docker executor" namespace:"docker"`
Parallels *ParallelsConfig `toml:"parallels" json:"parallels" group:"parallels executor" namespace:"parallels"`
VirtualBox *VirtualBoxConfig `toml:"virtualbox" json:"virtualbox" group:"virtualbox executor" namespace:"virtualbox"`
Cache *CacheConfig `toml:"cache" json:"cache" group:"cache configuration" namespace:"cache"`
}
type RunnerConfig struct {
......
......@@ -116,6 +116,11 @@ func (b *AbstractShell) cacheExtractor(w ShellWriter, info common.ShellScriptInf
"--file", b.cacheFile(cacheKey, info),
}
// Generate cache download address
if url := getCacheDownloadURL(info.Build, cacheKey); url != "" {
args = append(args, "--url", url)
}
// Execute archive command
w.Notice("Checking cache for %s...", cacheKey)
w.Command(info.RunnerCommand, args...)
......@@ -209,6 +214,11 @@ func (b *AbstractShell) cacheArchiver(w ShellWriter, list interface{}, info comm
}
args = append(args, archiverArgs...)
// Generate cache upload address
if url := getCacheUploadURL(info.Build, cacheKey); url != "" {
args = append(args, "--url", url)
}
// Execute archive command
w.Notice("Creating cache %s...", cacheKey)
w.Command(info.RunnerCommand, args...)
......
package shells
import (
"bytes"
"encoding/xml"
"io/ioutil"
"net/http"
"path"
"strconv"
"time"
"github.com/minio/minio-go"
"github.com/Sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
)
type bucketLocationTripper struct {
bucketLocation string
}
func (b *bucketLocationTripper) RoundTrip(req *http.Request) (res *http.Response, err error) {
var buffer bytes.Buffer
xml.NewEncoder(&buffer).Encode(b.bucketLocation)
res = &http.Response{
StatusCode: http.StatusOK,
Body: ioutil.NopCloser(&buffer),
}
return
}
func getCacheObjectName(build *common.Build, cache *common.CacheConfig, key string) string {
if key == "" || cache == nil || !cache.UseS3 {
return ""
}
return path.Join("runner", build.Runner.ShortDescription(), "project", strconv.Itoa(build.ProjectID), key)
}
func getCacheStorageClient(cache *common.CacheConfig) (scl minio.CloudStorageClient, err error) {
scl, err = minio.New(cache.ServerAddress, cache.AccessKey, cache.SecretKey, cache.Insecure)
if err != nil {
logrus.Warningln(err)
return
}
scl.SetCustomTransport(&bucketLocationTripper{cache.BucketLocation})
return
}
func getCacheDownloadURL(build *common.Build, key string) (url string) {
cache := build.Runner.Cache
objectName := getCacheObjectName(build, cache, key)
if objectName == "" {
return
}
scl, err := getCacheStorageClient(cache)
if err != nil {
logrus.Warningln(err)
return
}
url, err = scl.PresignedGetObject(cache.BucketName, key, time.Second*time.Duration(build.Timeout))
if err != nil {
logrus.Warningln(err)
return
}
return
}
func getCacheUploadURL(build *common.Build, key string) (url string) {
cache := build.Runner.Cache
objectName := getCacheObjectName(build, cache, key)
if objectName == "" {
return
}
scl, err := getCacheStorageClient(cache)
if err != nil {
logrus.Warningln(err)
return
}
url, err = scl.PresignedPutObject(cache.BucketName, key, time.Second*time.Duration(build.Timeout))
if err != nil {
logrus.Warningln(err)
return
}
return
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment