Commit 22b73564 authored by Kamil Trzciński's avatar Kamil Trzciński 💬

Introduce built-int support for caching

parent bf32491c
Pipeline #238906 passed with stage
......@@ -2,6 +2,7 @@ v 0.7.0 (unreleased)
- Refactor code structure
- Refactor bash script adding pre-build and post-build steps
- Add support for build artifacts
- Add support for caching build directories
v 0.6.1
- Revert: Fix tags handling when using git fetch: fetch all tags and prune the old ones
......
......@@ -31,6 +31,7 @@ type Build struct {
BuildAbort chan os.Signal `json:"-" yaml:"-"`
RootDir string `json:"-" yaml:"-"`
BuildDir string `json:"-" yaml:"-"`
CacheDir string `json:"-" yaml:"-"`
Hostname string `json:"-" yaml:"-"`
Runner *RunnerConfig `json:"runner"`
......@@ -137,11 +138,27 @@ func (b *Build) FullProjectDir() string {
return b.BuildDir
}
func (b *Build) StartBuild(rootDir string, sharedDir bool) {
func (b *Build) CacheFileForRef(ref string) string {
if b.CacheDir != "" {
return filepath.Join(b.CacheDir, ref, b.Name+".tgz")
}
return ""
}
func (b *Build) CacheFile() string {
// For tags we don't create cache
if b.Tag {
return ""
}
return b.CacheFileForRef(b.RefName)
}
func (b *Build) StartBuild(rootDir, cacheDir string, sharedDir bool) {
b.BuildStarted = time.Now()
b.BuildState = Pending
b.RootDir = rootDir
b.BuildDir = filepath.Join(rootDir, b.ProjectUniqueDir(sharedDir))
b.CacheDir = filepath.Join(cacheDir, b.ProjectUniqueDir(false))
}
func (b *Build) FinishBuild(buildState BuildState) {
......
......@@ -49,6 +49,7 @@ type RunnerConfig struct {
Limit *int `toml:"limit" json:"limit" long:"limit" env:"RUNNER_LIMIT" description:"Maximum number of builds processed by this runner"`
Executor string `toml:"executor" json:"executor" long:"executor" env:"RUNNER_EXECUTOR" required:"true" description:"Select executor, eg. shell, docker, etc."`
BuildsDir *string `toml:"builds_dir" json:"builds_dir" long:"builds-dir" env:"RUNNER_BUILDS_DIR" description:"Directory where builds are stored"`
CacheDir *string `toml:"cache_dir" json:"cache_dir" long:"cache-dir" env:"RUNNER_CACHE_DIR" description:"Directory where build cache is stored"`
Environment []string `toml:"environment" json:"environment" long:"env" env:"RUNNER_ENV" description:"Custom environment variables injected to build environment"`
......
......@@ -25,6 +25,7 @@ type FeaturesInfo struct {
Image bool `json:"image"`
Services bool `json:"services"`
Artifacts bool `json:"features"`
Cache bool `json:"cache"`
}
type VersionInfo struct {
......@@ -63,6 +64,9 @@ type GetBuildResponse struct {
Variables []BuildVariable `json:"variables"`
Options BuildOptions `json:"options"`
Token string `json:"token"`
Name string `json:"name"`
Stage string `json:"stage"`
Tag bool `json:"tag"`
}
type RegisterRunnerRequest struct {
......
......@@ -64,6 +64,7 @@ func (s *DockerCommandExecutor) Start() error {
func init() {
options := executors.ExecutorOptions{
DefaultBuildsDir: "/builds",
DefaultCacheDir: "/cache",
SharedBuildsDir: false,
Shell: common.ShellScriptInfo{
Shell: "bash",
......@@ -90,7 +91,7 @@ func init() {
}
common.RegisterExecutor("docker", executors.DefaultExecutorProvider{
Creator: creator,
Creator: creator,
FeaturesUpdater: featuresUpdater,
})
}
......@@ -14,6 +14,7 @@ import (
type ExecutorOptions struct {
DefaultBuildsDir string
DefaultCacheDir string
SharedBuildsDir bool
Shell common.ShellScriptInfo
ShowHostname bool
......@@ -79,7 +80,8 @@ func (e *AbstractExecutor) startBuild() error {
// Start actual build
rootDir := helpers.StringOrDefault(e.Config.BuildsDir, e.DefaultBuildsDir)
e.Build.StartBuild(rootDir, e.SharedBuildsDir)
cacheDir := helpers.StringOrDefault(e.Config.CacheDir, e.DefaultCacheDir)
e.Build.StartBuild(rootDir, cacheDir, e.SharedBuildsDir)
return nil
}
......
......@@ -93,6 +93,7 @@ func (s *ShellExecutor) Cleanup() {
func init() {
options := executors.ExecutorOptions{
DefaultBuildsDir: "builds",
DefaultCacheDir: "cache",
SharedBuildsDir: true,
Shell: common.ShellScriptInfo{
Shell: common.GetDefaultShell(),
......@@ -114,7 +115,7 @@ func init() {
}
common.RegisterExecutor("shell", executors.DefaultExecutorProvider{
Creator: creator,
Creator: creator,
FeaturesUpdater: featuresUpdater,
})
}
......@@ -24,6 +24,7 @@ func (b *BashShell) GetName() string {
func (b *BashShell) GetFeatures(features *common.FeaturesInfo) {
features.Artifacts = true
features.Cache = true
}
func (b *BashShell) executeCommand(w io.Writer, cmd string, arguments ...string) {
......@@ -141,6 +142,32 @@ func (b *BashShell) generatePreBuildScript(info common.ShellScriptInfo) string {
}
b.writeCheckoutCmd(w, build)
cacheFile := info.Build.CacheFile()
cacheFile2 := info.Build.CacheFileForRef("master")
if cacheFile == "" {
cacheFile = cacheFile2
cacheFile2 = ""
}
// Try to restore from main cache, if not found cache for master
if cacheFile != "" {
// If we have cache, restore it
b.writeIfFile(w, cacheFile)
b.echoColored(w, "Restoring cache...")
b.executeCommand(w, "tar", "-zxfv", "-f", cacheFile)
if cacheFile2 != "" {
b.writeElse(w)
// If we have cache, restore it
b.writeIfFile(w, cacheFile2)
b.echoColored(w, "Restoring cache...")
b.executeCommand(w, "tar", "-zxfv", "-f", cacheFile2)
b.writeEndIf(w)
}
b.writeEndIf(w)
}
w.Flush()
return buffer.String()
......@@ -206,6 +233,18 @@ func (b *BashShell) generatePostBuildScript(info common.ShellScriptInfo) string
w := bufio.NewWriter(&buffer)
b.writeCdBuildDir(w, info)
if cacheFile := info.Build.CacheFile(); cacheFile != "" {
// Find files to cache
b.findFiles(w, info.Build.Options["caches"], "caches.files")
// If we have list of files create archive
b.writeIfFile(w, "cache.files")
b.echoColored(w, "Archiving caches...")
b.executeCommand(w, "mkdir", "-p", filepath.Dir(cacheFile))
b.executeCommand(w, "tar", "-zcv", "-T", "caches.files", "-f", cacheFile)
b.writeEndIf(w)
}
// Find artifacts
b.findFiles(w, info.Build.Options["artifacts"], "artifacts.files")
......@@ -277,7 +316,7 @@ func (b *BashShell) IsDefault() bool {
func init() {
common.RegisterShell(&BashShell{
AbstractShell: AbstractShell{
SupportedOptions: []string{"artifacts"},
SupportedOptions: []string{"artifacts", "cache"},
},
})
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment