Introduce built-int support for caching

CHANGELOG.md

@@ -2,6 +2,7 @@ v 0.7.0 (unreleased)
 - Refactor code structure
 - Refactor bash script adding pre-build and post-build steps
 - Add support for build artifacts
+- Add support for caching build directories
 
 v 0.6.1
 - Revert: Fix tags handling when using git fetch: fetch all tags and prune the old ones

common/build.go

@@ -31,6 +31,7 @@ type Build struct {
 	BuildAbort       chan os.Signal `json:"-" yaml:"-"`
 	RootDir          string         `json:"-" yaml:"-"`
 	BuildDir         string         `json:"-" yaml:"-"`
+	CacheDir         string         `json:"-" yaml:"-"`
 	Hostname         string         `json:"-" yaml:"-"`
 	Runner           *RunnerConfig  `json:"runner"`
 
@@ -137,11 +138,27 @@ func (b *Build) FullProjectDir() string {
 	return b.BuildDir
 }
 
-func (b *Build) StartBuild(rootDir string, sharedDir bool) {
+func (b *Build) CacheFileForRef(ref string) string {
+	if b.CacheDir != "" {
+		return filepath.Join(b.CacheDir, ref, b.Name+".tgz")
+	}
+	return ""
+}
+
+func (b *Build) CacheFile() string {
+	// For tags we don't create cache
+	if b.Tag {
+		return ""
+	}
+	return b.CacheFileForRef(b.RefName)
+}
+
+func (b *Build) StartBuild(rootDir, cacheDir string, sharedDir bool) {
 	b.BuildStarted = time.Now()
 	b.BuildState = Pending
 	b.RootDir = rootDir
 	b.BuildDir = filepath.Join(rootDir, b.ProjectUniqueDir(sharedDir))
+	b.CacheDir = filepath.Join(cacheDir, b.ProjectUniqueDir(false))
 }
 
 func (b *Build) FinishBuild(buildState BuildState) {

common/config.go

@@ -49,6 +49,7 @@ type RunnerConfig struct {
 	Limit          *int    `toml:"limit" json:"limit" long:"limit" env:"RUNNER_LIMIT" description:"Maximum number of builds processed by this runner"`
 	Executor       string  `toml:"executor" json:"executor" long:"executor" env:"RUNNER_EXECUTOR" required:"true" description:"Select executor, eg. shell, docker, etc."`
 	BuildsDir      *string `toml:"builds_dir" json:"builds_dir" long:"builds-dir" env:"RUNNER_BUILDS_DIR" description:"Directory where builds are stored"`
+	CacheDir       *string `toml:"cache_dir" json:"cache_dir" long:"cache-dir" env:"RUNNER_CACHE_DIR" description:"Directory where build cache is stored"`
 
 	Environment    []string `toml:"environment" json:"environment" long:"env" env:"RUNNER_ENV" description:"Custom environment variables injected to build environment"`
 

common/network.go

@@ -25,6 +25,7 @@ type FeaturesInfo struct {
 	Image     bool `json:"image"`
 	Services  bool `json:"services"`
 	Artifacts bool `json:"features"`
+	Cache     bool `json:"cache"`
 }
 
 type VersionInfo struct {
@@ -63,6 +64,9 @@ type GetBuildResponse struct {
 	Variables     []BuildVariable `json:"variables"`
 	Options       BuildOptions    `json:"options"`
 	Token         string          `json:"token"`
+	Name          string          `json:"name"`
+	Stage         string          `json:"stage"`
+	Tag           bool            `json:"tag"`
 }
 
 type RegisterRunnerRequest struct {

executors/docker/executor_docker_command.go

@@ -64,6 +64,7 @@ func (s *DockerCommandExecutor) Start() error {
 func init() {
 	options := executors.ExecutorOptions{
 		DefaultBuildsDir: "/builds",
+		DefaultCacheDir:  "/cache",
 		SharedBuildsDir:  false,
 		Shell: common.ShellScriptInfo{
 			Shell: "bash",
@@ -90,7 +91,7 @@ func init() {
 	}
 
 	common.RegisterExecutor("docker", executors.DefaultExecutorProvider{
-		Creator: creator,
+		Creator:         creator,
 		FeaturesUpdater: featuresUpdater,
 	})
 }

executors/executor_abstract.go

@@ -14,6 +14,7 @@ import (
 
 type ExecutorOptions struct {
 	DefaultBuildsDir string
+	DefaultCacheDir  string
 	SharedBuildsDir  bool
 	Shell            common.ShellScriptInfo
 	ShowHostname     bool
@@ -79,7 +80,8 @@ func (e *AbstractExecutor) startBuild() error {
 
 	// Start actual build
 	rootDir := helpers.StringOrDefault(e.Config.BuildsDir, e.DefaultBuildsDir)
-	e.Build.StartBuild(rootDir, e.SharedBuildsDir)
+	cacheDir := helpers.StringOrDefault(e.Config.CacheDir, e.DefaultCacheDir)
+	e.Build.StartBuild(rootDir, cacheDir, e.SharedBuildsDir)
 	return nil
 }
 

executors/shell/executor_shell.go

@@ -93,6 +93,7 @@ func (s *ShellExecutor) Cleanup() {
 func init() {
 	options := executors.ExecutorOptions{
 		DefaultBuildsDir: "builds",
+		DefaultCacheDir:  "cache",
 		SharedBuildsDir:  true,
 		Shell: common.ShellScriptInfo{
 			Shell: common.GetDefaultShell(),
@@ -114,7 +115,7 @@ func init() {
 	}
 
 	common.RegisterExecutor("shell", executors.DefaultExecutorProvider{
-		Creator: creator,
+		Creator:         creator,
 		FeaturesUpdater: featuresUpdater,
 	})
 }

shells/bash.go

@@ -24,6 +24,7 @@ func (b *BashShell) GetName() string {
 
 func (b *BashShell) GetFeatures(features *common.FeaturesInfo) {
 	features.Artifacts = true
+	features.Cache = true
 }
 
 func (b *BashShell) executeCommand(w io.Writer, cmd string, arguments ...string) {
@@ -141,6 +142,32 @@ func (b *BashShell) generatePreBuildScript(info common.ShellScriptInfo) string {
 	}
 
 	b.writeCheckoutCmd(w, build)
+
+	cacheFile := info.Build.CacheFile()
+	cacheFile2 := info.Build.CacheFileForRef("master")
+	if cacheFile == "" {
+		cacheFile = cacheFile2
+		cacheFile2 = ""
+	}
+
+	// Try to restore from main cache, if not found cache for master
+	if cacheFile != "" {
+		// If we have cache, restore it
+		b.writeIfFile(w, cacheFile)
+		b.echoColored(w, "Restoring cache...")
+		b.executeCommand(w, "tar", "-zxfv", "-f", cacheFile)
+		if cacheFile2 != "" {
+			b.writeElse(w)
+
+			// If we have cache, restore it
+			b.writeIfFile(w, cacheFile2)
+			b.echoColored(w, "Restoring cache...")
+			b.executeCommand(w, "tar", "-zxfv", "-f", cacheFile2)
+			b.writeEndIf(w)
+		}
+		b.writeEndIf(w)
+	}
+
 	w.Flush()
 
 	return buffer.String()
@@ -206,6 +233,18 @@ func (b *BashShell) generatePostBuildScript(info common.ShellScriptInfo) string
 	w := bufio.NewWriter(&buffer)
 	b.writeCdBuildDir(w, info)
 
+	if cacheFile := info.Build.CacheFile(); cacheFile != "" {
+		// Find files to cache
+		b.findFiles(w, info.Build.Options["caches"], "caches.files")
+
+		// If we have list of files create archive
+		b.writeIfFile(w, "cache.files")
+		b.echoColored(w, "Archiving caches...")
+		b.executeCommand(w, "mkdir", "-p", filepath.Dir(cacheFile))
+		b.executeCommand(w, "tar", "-zcv", "-T", "caches.files", "-f", cacheFile)
+		b.writeEndIf(w)
+	}
+
 	// Find artifacts
 	b.findFiles(w, info.Build.Options["artifacts"], "artifacts.files")
 
@@ -277,7 +316,7 @@ func (b *BashShell) IsDefault() bool {
 func init() {
 	common.RegisterShell(&BashShell{
 		AbstractShell: AbstractShell{
-			SupportedOptions: []string{"artifacts"},
+			SupportedOptions: []string{"artifacts", "cache"},
 		},
 	})
 }