Introduce new commands to handle artifacts and caching

- artifacts-downloader: download artifacts from previous stages
- artifacts-uploader: create and upload artifacts
- cache-extractor: restore cache
- cache-archiver: create and store cache

commands/helpers/artifacts_downloader.go

+package commands_helpers
+
+import (
+	"errors"
+	"io/ioutil"
+	"os"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/codegangsta/cli"
+
+	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
+	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/archives"
+	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/formatter"
+	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/network"
+)
+
+type ArtifactsDownloaderCommand struct {
+	common.BuildCredentials
+}
+
+func (c *ArtifactsDownloaderCommand) download(file string) error {
+	gl := network.GitLabClient{}
+
+	// If the download fails, exit with a non-zero exit code to indicate an issue?
+retry:
+	for i := 0; i < 3; i++ {
+		switch gl.DownloadArtifacts(c.BuildCredentials, file) {
+		case common.DownloadSucceeded:
+			return nil
+		case common.DownloadNotFound:
+			return os.ErrNotExist
+		case common.DownloadForbidden:
+			break retry
+		case common.DownloadFailed:
+			// wait one second to retry
+			logrus.Warningln("Retrying...")
+			time.Sleep(time.Second)
+			break
+		}
+	}
+	return errors.New("Failed to download artifacts")
+}
+
+func (c *ArtifactsDownloaderCommand) Execute(context *cli.Context) {
+	formatter.SetRunnerFormatter()
+
+	if len(c.URL) == 0 || len(c.Token) == 0 {
+		logrus.Fatalln("Missing runner credentials")
+	}
+	if c.ID <= 0 {
+		logrus.Fatalln("Missing build ID")
+	}
+
+	// Create temporary file
+	file, err := ioutil.TempFile("", "artifacts")
+	if err != nil {
+		logrus.Fatalln(err)
+	}
+	file.Close()
+	defer os.Remove(file.Name())
+
+	// Download artifacts file
+	err = c.download(file.Name())
+	if err != nil {
+		logrus.Fatalln(err)
+	}
+
+	// Extract artifacts file
+	err = archives.ExtractZipFile(file.Name())
+	if err != nil {
+		logrus.Fatalln(err)
+	}
+}
+
+func init() {
+	common.RegisterCommand2("artifacts-downloader", "download and extract build artifacts (internal)", &ArtifactsDownloaderCommand{})
+}

commands/helpers/artifacts.go → commands/helpers/artifacts_uploader.go

 package commands_helpers
 
 import (
+	"io"
+	"os"
+	"time"
+
 	"github.com/Sirupsen/logrus"
 	"github.com/codegangsta/cli"
+
 	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
+	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/archives"
 	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/formatter"
 	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/network"
-	"os"
-	"time"
 )
 
-type ArtifactCommand struct {
+type ArtifactsUploaderCommand struct {
 	common.BuildCredentials
-	File     string `long:"file" description:"The file containing your build artifacts"`
-	Download bool   `long:"download" description:"Download artifacts instead of uploading them"`
+	FileArchiver
 }
 
-func (c *ArtifactCommand) upload() {
+func (c *ArtifactsUploaderCommand) createAndUpload(network common.Network) common.UploadState {
+	pr, pw := io.Pipe()
+	defer pr.Close()
+
+	// Create the archive
+	go func() {
+		err := archives.CreateZipArchive(pw, c.sortedFiles())
+		pw.CloseWithError(err)
+	}()
+
+	// Upload the data
+	return network.UploadRawArtifacts(c.BuildCredentials, pr, "artifacts.zip")
+}
+
+func (c *ArtifactsUploaderCommand) Execute(*cli.Context) {
+	formatter.SetRunnerFormatter()
+
+	if len(c.URL) == 0 || len(c.Token) == 0 {
+		logrus.Fatalln("Missing runner credentials")
+	}
+	if c.ID <= 0 {
+		logrus.Fatalln("Missing build ID")
+	}
+
+	// Enumerate files
+	err := c.enumerate()
+	if err != nil {
+		logrus.Fatalln(err)
+	}
+
 	gl := network.GitLabClient{}
 
 	// If the upload fails, exit with a non-zero exit code to indicate an issue?
 retry:
 	for i := 0; i < 3; i++ {
-		switch gl.UploadArtifacts(c.BuildCredentials, c.File) {
+		switch c.createAndUpload(&gl) {
 		case common.UploadSucceeded:
-			os.Exit(0)
+			return
 		case common.UploadForbidden:
 			break retry
 		case common.UploadTooLarge:
@@ -36,52 +68,9 @@ retry:
 			break
 		}
 	}
-
-	os.Exit(1)
-}
-
-func (c *ArtifactCommand) download() {
-	gl := network.GitLabClient{}
-
-	// If the download fails, exit with a non-zero exit code to indicate an issue?
-retry:
-	for i := 0; i < 3; i++ {
-		switch gl.DownloadArtifacts(c.BuildCredentials, c.File) {
-		case common.DownloadSucceeded:
-			os.Exit(0)
-		case common.DownloadForbidden:
-			break retry
-		case common.DownloadFailed:
-			// wait one second to retry
-			logrus.Warningln("Retrying...")
-			time.Sleep(time.Second)
-			break
-		}
-	}
-
 	os.Exit(1)
 }
 
-func (c *ArtifactCommand) Execute(context *cli.Context) {
-	formatter.SetRunnerFormatter()
-
-	if len(c.File) == 0 {
-		logrus.Fatalln("Missing archive file")
-	}
-	if len(c.URL) == 0 || len(c.Token) == 0 {
-		logrus.Fatalln("Missing runner credentials")
-	}
-	if c.ID <= 0 {
-		logrus.Fatalln("Missing build ID")
-	}
-
-	if c.Download {
-		c.download()
-	} else {
-		c.upload()
-	}
-}
-
 func init() {
-	common.RegisterCommand2("artifacts", "download or upload build artifacts (internal)", &ArtifactCommand{})
+	common.RegisterCommand2("artifacts-uploader", "create and upload build artifacts (internal)", &ArtifactsUploaderCommand{})
 }

commands/helpers/cache_archiver.go

+package commands_helpers
+
+import (
+	"github.com/Sirupsen/logrus"
+	"github.com/codegangsta/cli"
+
+	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
+	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/archives"
+)
+
+type CacheArchiverCommand struct {
+	FileArchiver
+	File string `long:"file" description:"The path to file"`
+}
+
+func (c *CacheArchiverCommand) Execute(*cli.Context) {
+	if c.File == "" {
+		logrus.Fatalln("Missing --file")
+	}
+
+	// Enumerate files
+	err := c.enumerate()
+	if err != nil {
+		logrus.Fatalln(err)
+	}
+
+	// Check if list of files changed
+	if !c.isFileChanged(c.File) {
+		logrus.Infoln("Archive is up to date!")
+		return
+	}
+
+	// Create archive
+	err = archives.CreateZipFile(c.File, c.sortedFiles())
+	if err != nil {
+		logrus.Fatalln(err)
+	}
+}
+
+func init() {
+	common.RegisterCommand2("cache-archiver", "create and upload cache artifacts (internal)", &CacheArchiverCommand{})
+}

commands/helpers/cache_extractor.go

+package commands_helpers
+
+import (
+	"os"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/codegangsta/cli"
+
+	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
+	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/archives"
+	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/formatter"
+)
+
+type CacheExtractorCommand struct {
+	File string `long:"file" description:"The file containing your cache artifacts"`
+}
+
+func (c *CacheExtractorCommand) Execute(context *cli.Context) {
+	formatter.SetRunnerFormatter()
+
+	if len(c.File) == 0 {
+		logrus.Fatalln("Missing cache file")
+	}
+
+	err := archives.ExtractZipFile(c.File)
+	if err != nil && !os.IsNotExist(err) {
+		logrus.Fatalln(err)
+	}
+}
+
+func init() {
+	common.RegisterCommand2("cache-extractor", "download and extract cache artifacts (internal)", &CacheExtractorCommand{})
+}

commands/helpers/archive.go → commands/helpers/file_archiver.go

 package commands_helpers
 
 import (
-	"archive/zip"
 	"bufio"
 	"bytes"
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -16,36 +14,18 @@ import (
 	"time"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/codegangsta/cli"
-	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
 )
 
-type ArchiveCommand struct {
+type FileArchiver struct {
 	Paths     []string `long:"path" description:"Add paths to archive"`
 	Untracked bool     `long:"untracked" description:"Add git untracked files"`
-	File      string   `long:"file" description:"The path to file"`
 	Verbose   bool     `long:"verbose" description:"Detailed information"`
-	List      bool     `long:"list" description:"List files to archive"`
 
 	wd    string
 	files map[string]os.FileInfo
 }
 
-func isTarArchive(fileName string) bool {
-	if strings.HasSuffix(fileName, ".tgz") || strings.HasSuffix(fileName, ".tar.gz") {
-		return true
-	}
-	return false
-}
-
-func isZipArchive(fileName string) bool {
-	if strings.HasSuffix(fileName, ".zip") {
-		return true
-	}
-	return false
-}
-
-func (c *ArchiveCommand) isChanged(modTime time.Time) bool {
+func (c *FileArchiver) isChanged(modTime time.Time) bool {
 	for _, info := range c.files {
 		if modTime.Before(info.ModTime()) {
 			return true
@@ -54,33 +34,19 @@ func (c *ArchiveCommand) isChanged(modTime time.Time) bool {
 	return false
 }
 
-func (c *ArchiveCommand) zipArchiveChanged() bool {
-	archive, err := zip.OpenReader(c.File)
-	if err != nil {
-		logrus.Warningf("%s: %v", c.File, err)
-		return true
-	}
-	defer archive.Close()
-	for _, file := range archive.File {
-		_, err := os.Lstat(file.Name)
-		if os.IsNotExist(err) {
-			return true
+func (c *FileArchiver) isFileChanged(fileName string) bool {
+	ai, err := os.Stat(fileName)
+	if ai != nil {
+		if !c.isChanged(ai.ModTime()) {
+			return false
 		}
+	} else if !os.IsNotExist(err) {
+		logrus.Warningln(err)
 	}
-	return false
+	return true
 }
 
-func (c *ArchiveCommand) isFileListChanged() bool {
-	if isZipArchive(c.File) {
-		return c.zipArchiveChanged()
-	} else {
-		logrus.Warningln("The archive can't be verified if file list changed: operation not supported")
-		// TODO: this is not supported
-		return false
-	}
-}
-
-func (c *ArchiveCommand) sortedFiles() []string {
+func (c *FileArchiver) sortedFiles() []string {
 	files := make([]string, len(c.files))
 
 	i := 0
@@ -93,7 +59,7 @@ func (c *ArchiveCommand) sortedFiles() []string {
 	return files
 }
 
-func (c *ArchiveCommand) add(path string) (err error) {
+func (c *FileArchiver) add(path string) (err error) {
 	// Always use slashes
 	path = filepath.ToSlash(path)
 
@@ -105,7 +71,7 @@ func (c *ArchiveCommand) add(path string) (err error) {
 	return
 }
 
-func (c *ArchiveCommand) process(match string) bool {
+func (c *FileArchiver) process(match string) bool {
 	var absolute, relative string
 	var err error
 
@@ -133,7 +99,7 @@ func (c *ArchiveCommand) process(match string) bool {
 	return false
 }
 
-func (c *ArchiveCommand) processPaths() {
+func (c *FileArchiver) processPaths() {
 	for _, path := range c.Paths {
 		matches, err := filepath.Glob(path)
 		if err != nil {
@@ -163,7 +129,7 @@ func (c *ArchiveCommand) processPaths() {
 	}
 }
 
-func (c *ArchiveCommand) processUntracked() {
+func (c *FileArchiver) processUntracked() {
 	if !c.Untracked {
 		return
 	}
@@ -202,156 +168,10 @@ func (c *ArchiveCommand) processUntracked() {
 	}
 }
 
-func (c *ArchiveCommand) listFiles() {
-	if len(c.files) == 0 {
-		logrus.Infoln("No files to archive.")
-		return
-	}
-
-	for _, file := range c.sortedFiles() {
-		println(string(file))
-	}
-}
-
-func (c *ArchiveCommand) createZipArchive(w io.Writer, fileNames []string) error {
-	archive := zip.NewWriter(w)
-	defer archive.Close()
-
-	for _, fileName := range fileNames {
-		fi, err := os.Lstat(fileName)
-		if err != nil {
-			logrus.Warningln("File ignored: %q: %v", fileName, err)
-			continue
-		}
-
-		fh, err := zip.FileInfoHeader(fi)
-		fh.Name = fileName
-		fh.Extra = createZipExtra(fi)
-
-		switch fi.Mode() & os.ModeType {
-		case os.ModeDir:
-			fh.Name += "/"
-
-			_, err := archive.CreateHeader(fh)
-			if err != nil {
-				return err
-			}
-
-		case os.ModeSymlink:
-			fw, err := archive.CreateHeader(fh)
-			if err != nil {
-				return err
-			}
-
-			link, err := os.Readlink(fileName)
-			if err != nil {
-				return err
-			}
-
-			io.WriteString(fw, link)
-
-		case os.ModeNamedPipe, os.ModeSocket, os.ModeDevice:
-			// Ignore the files that of these types
-			logrus.Warningln("File ignored: %q", fileName)
-
-		default:
-			fh.Method = zip.Deflate
-			fw, err := archive.CreateHeader(fh)
-			if err != nil {
-				return err
-			}
-
-			file, err := os.Open(fileName)
-			if err != nil {
-				return err
-			}
-
-			_, err = io.Copy(fw, file)
-			file.Close()
-			if err != nil {
-				return err
-			}
-			break
-		}
-
-		if c.Verbose {
-			fmt.Printf("%v\t%d\t%s\n", fh.Mode(), fh.UncompressedSize64, fh.Name)
-		}
-	}
-
-	return nil
-}
-
-func (c *ArchiveCommand) createTarArchive(w io.Writer, files []string) error {
-	var list bytes.Buffer
-	for _, file := range c.sortedFiles() {
-		list.WriteString(string(file) + "\n")
-	}
-
-	flags := "-zcP"
-	if c.Verbose {
-		flags += "v"
-	}
-
-	cmd := exec.Command("tar", flags, "-T", "-", "--no-recursion")
-	cmd.Env = os.Environ()
-	cmd.Stdin = &list
-	cmd.Stdout = w
-	cmd.Stderr = os.Stderr
-	logrus.Debugln("Executing command:", strings.Join(cmd.Args, " "))
-	return cmd.Run()
-}
-
-func (c *ArchiveCommand) createArchive(w io.Writer, files []string) error {
-	if isTarArchive(c.File) {
-		return c.createTarArchive(w, files)
-	} else if isZipArchive(c.File) {
-		return c.createZipArchive(w, files)
-	} else {
-		return fmt.Errorf("Unsupported archive format: %q", c.File)
-	}
-}
-
-func (c *ArchiveCommand) archive() {
-	if len(c.files) == 0 {
-		logrus.Infoln("No files to archive.")
-		return
-	}
-
-	logrus.Infoln("Creating archive", filepath.Base(c.File), "...")
-
-	// create directories to store archive
-	os.MkdirAll(filepath.Dir(c.File), 0700)
-
-	tempFile, err := ioutil.TempFile(filepath.Dir(c.File), "archive_")
-	if err != nil {
-		logrus.Fatalln("Failed to create temporary archive", err)
-	}
-	defer tempFile.Close()
-	defer os.Remove(tempFile.Name())
-
-	logrus.Debugln("Temporary file:", tempFile.Name())
-	err = c.createArchive(tempFile, c.sortedFiles())
-	if err != nil {
-		logrus.Fatalln("Failed to create archive:", err)
-	}
-	tempFile.Close()
-
-	err = os.Rename(tempFile.Name(), c.File)
-	if err != nil {
-		logrus.Warningln("Failed to rename archive:", err)
-	}
-
-	logrus.Infoln("Done!")
-}
-
-func (c *ArchiveCommand) Execute(context *cli.Context) {
+func (c *FileArchiver) enumerate() error {
 	wd, err := os.Getwd()
 	if err != nil {
-		logrus.Fatalln("Failed to get current working directory:", err)
-	}
-	if c.File == "" && !c.List {
-		logrus.Fatalln("Missing archive file name!")
+		return fmt.Errorf("Failed to get current working directory: %v", err)
 	}
 
 	c.wd = wd
@@ -359,25 +179,5 @@ func (c *ArchiveCommand) Execute(context *cli.Context) {
 
 	c.processPaths()
 	c.processUntracked()
-
-	ai, err := os.Stat(c.File)
-	if err != nil && !os.IsNotExist(err) {
-		logrus.Fatalln("Failed to verify archive:", c.File, err)
-	}
-	if ai != nil {
-		if !c.isChanged(ai.ModTime()) && !c.zipArchiveChanged() {
-			logrus.Infoln("Archive is up to date!")
-			return
-		}
-	}
-
-	if c.List {
-		c.listFiles()
-	} else {
-		c.archive()
-	}
-}
-
-func init() {
-	common.RegisterCommand2("archive", "find and archive files (internal)", &ArchiveCommand{})
+	return nil
 }

commands/helpers/file_archiver_test.go

+package commands_helpers
+
+import (
+	"archive/zip"
+	"github.com/stretchr/testify/assert"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+const UntrackedFileName = "some_fancy_untracked_file"
+
+var currentDir, _ = os.Getwd()
+
+func randomTempFile(t *testing.T, format string) string {
+	file, err := ioutil.TempFile("", "archive_")
+	assert.NoError(t, err)
+	defer file.Close()
+	defer os.Remove(file.Name())
+	return file.Name() + format
+}
+
+func createArchiveCommand(t *testing.T) *ArchiveCommand {
+	err := os.Chdir(filepath.Join(currentDir, "..", ".."))
+	assert.NoError(t, err)
+
+	return &ArchiveCommand{
+		File:    randomTempFile(t, ".zip"),
+		Verbose: true,
+	}
+}
+
+func filesInFolder(path string) []string {
+	matches, _ := filepath.Glob(path)
+	return matches
+}
+
+func readArchiveContent(t *testing.T, c *ArchiveCommand) (resultMap map[string]bool) {
+	resultMap = make(map[string]bool)
+
+	archive, err := zip.OpenReader(c.File)
+	assert.NoError(t, err)
+	defer archive.Close()
+	for _, file := range archive.File {
+		resultMap[file.Name] = true
+	}
+	return
+}
+
+func verifyArchiveContent(t *testing.T, c *ArchiveCommand, files ...string) {
+	resultMap := readArchiveContent(t, c)
+	for _, file := range files {
+		assert.True(t, resultMap[file], "File should exist %q", file)
+		delete(resultMap, file)
+	}
+	assert.Len(t, resultMap, 0, "No extra file should exist")
+}
+
+func TestArchiveNotCreatingArchive(t *testing.T) {
+	cmd := createArchiveCommand(t)
+	defer os.Remove(cmd.File)
+	cmd.Execute(nil)
+	_, err := os.Stat(cmd.File)
+	assert.True(t, os.IsNotExist(err), "File should not exist", cmd.File, err)
+}
+
+func TestArchiveAddingSomeLocalFiles(t *testing.T) {
+	cmd := createArchiveCommand(t)
+	defer os.Remove(cmd.File)
+	cmd.Paths = []string{
+		"commands/helpers/*",
+	}
+	cmd.Execute(nil)
+	verifyArchiveContent(t, cmd, filesInFolder("commands/helpers/*")...)
+}
+
+func TestArchiveNotAddingDuplicateFiles(t *testing.T) {
+	cmd := createArchiveCommand(t)
+	defer os.Remove(cmd.File)
+	cmd.Paths = []string{
+		"commands/helpers/*",
+		"commands/helpers/archive.go",
+	}
+	cmd.Execute(nil)
+	verifyArchiveContent(t, cmd, filesInFolder("commands/helpers/*")...)
+}
+
+func TestArchiveAddingUntrackedFiles(t *testing.T) {
+	cmd := createArchiveCommand(t)
+	defer os.Remove(cmd.File)
+	err := ioutil.WriteFile(UntrackedFileName, []byte{}, 0700)
+	assert.NoError(t, err)
+	cmd.Untracked = true
+	cmd.Execute(nil)
+	files := readArchiveContent(t, cmd)
+	assert.NotEmpty(t, files)
+	assert.True(t, files[UntrackedFileName])
+}
+
+func TestArchiveUpdating(t *testing.T) {
+	tempFile := randomTempFile(t, ".zip")
+	defer os.Remove(tempFile)
+
+	err := ioutil.WriteFile(UntrackedFileName, []byte{}, 0700)
+	assert.NoError(t, err)
+
+	cmd := createArchiveCommand(t)
+	defer os.Remove(cmd.File)
+	cmd.Paths = []string{
+		"commands",
+		UntrackedFileName,
+	}
+
+	cmd.Execute(nil)
+	archive1, err := os.Stat(cmd.File)
+	assert.NoError(t, err, "Archive is created")
+
+	cmd.Execute(nil)
+	archive2, err := os.Stat(cmd.File)
+	assert.NoError(t, err, "Archive is created")
+	assert.Equal(t, archive1.ModTime(), archive2.ModTime(), "Archive should not be modified")
+
+	time.Sleep(time.Second)
+	err = ioutil.WriteFile(UntrackedFileName, []byte{}, 0700)
+	assert.NoError(t, err, "File is created")
+
+	cmd.Execute(nil)
+	archive3, err := os.Stat(cmd.File)
+	assert.NoError(t, err, "Archive is created")
+	assert.NotEqual(t, archive2.ModTime(), archive3.ModTime(), "File is added to archive")
+
+	time.Sleep(time.Second)
+	err = ioutil.WriteFile(UntrackedFileName, []byte{}, 0700)
+	assert.NoError(t, err, "File is updated")
+
+	cmd.Execute(nil)
+	archive4, err := os.Stat(cmd.File)
+	assert.NoError(t, err, "Archive is created")
+	assert.NotEqual(t, archive3.ModTime(), archive4.ModTime(), "File is updated in archive")
+}

common/build.go

@@ -141,7 +141,7 @@ func (b *Build) FullProjectDir() string {
 	return helpers.ToSlash(b.BuildDir)
 }
 
-func (b *Build) CacheFileForRef(ref string) string {
+func (b *Build) CacheKeyForRef(ref string) string {
 	if b.CacheDir != "" {
 		cacheKey := path.Join(b.Name, ref)
 
@@ -156,23 +156,17 @@ func (b *Build) CacheFileForRef(ref string) string {
 		if cacheKey == "" {
 			return ""
 		}
-
-		cacheFile := path.Join(b.CacheDir, cacheKey, "cache.zip")
-		cacheFile, err := filepath.Rel(b.BuildDir, cacheFile)
-		if err != nil {
-			return ""
-		}
-		return filepath.ToSlash(cacheFile)
+		return filepath.ToSlash(path.Join(cacheKey, "cache.zip"))
 	}
 	return ""
 }
 
-func (b *Build) CacheFile() string {
+func (b *Build) CacheKey() string {
 	// For tags we don't create cache
 	if b.Tag {
 		return ""
 	}
-	return b.CacheFileForRef(b.RefName)
+	return b.CacheKeyForRef(b.RefName)
 }
 
 func (b *Build) StartBuild(rootDir, cacheDir string, sharedDir bool) {

common/network.go

 package common
 
+import "io"
+
 type UpdateState int
 type UploadState int
 type DownloadState int
@@ -21,6 +23,7 @@ const (
 	DownloadSucceeded DownloadState = iota
 	DownloadForbidden
 	DownloadFailed
+	DownloadNotFound
 )
 
 type FeaturesInfo struct {
@@ -124,5 +127,6 @@ type Network interface {
 	VerifyRunner(config RunnerCredentials) bool
 	UpdateBuild(config RunnerConfig, id int, state BuildState, trace string) UpdateState
 	DownloadArtifacts(config BuildCredentials, artifactsFile string) DownloadState
+	UploadRawArtifacts(config BuildCredentials, reader io.Reader, baseName string) UploadState
 	UploadArtifacts(config BuildCredentials, artifactsFile string) UploadState
 }

helpers/archives/zip_create.go

+package archives
+
+import (
+	"archive/zip"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/Sirupsen/logrus"
+)
+
+func createZipEntry(archive *zip.Writer, fileName string) error {
+	fi, err := os.Lstat(fileName)
+	if err != nil {
+		logrus.Warningln("File ignored:", err)
+		return nil
+	}
+
+	fh, err := zip.FileInfoHeader(fi)
+	fh.Name = fileName
+	fh.Extra = createZipExtra(fi)
+
+	switch fi.Mode() & os.ModeType {
+	case os.ModeDir:
+		fh.Name += "/"
+
+		_, err := archive.CreateHeader(fh)
+		if err != nil {
+			return err
+		}
+
+	case os.ModeSymlink:
+		fw, err := archive.CreateHeader(fh)
+		if err != nil {
+			return err
+		}
+
+		link, err := os.Readlink(fileName)
+		if err != nil {
+			return err
+		}
+
+		io.WriteString(fw, link)
+
+	case os.ModeNamedPipe, os.ModeSocket, os.ModeDevice:
+		// Ignore the files that of these types
+		logrus.Warningln("File ignored:", fileName)
+
+	default:
+		fh.Method = zip.Deflate
+		fw, err := archive.CreateHeader(fh)
+		if err != nil {
+			return err
+		}
+
+		file, err := os.Open(fileName)
+		if err != nil {
+			return err
+		}
+
+		_, err = io.Copy(fw, file)
+		file.Close()
+		if err != nil {
+			return err
+		}
+		break
+	}
+	return nil
+}
+
+func CreateZipArchive(w io.Writer, fileNames []string) error {
+	archive := zip.NewWriter(w)
+	defer archive.Close()
+
+	for _, fileName := range fileNames {
+		err := createZipEntry(archive, fileName)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func CreateZipFile(fileName string, fileNames []string) error {
+	// create directories to store archive
+	os.MkdirAll(filepath.Dir(fileName), 0700)
+
+	tempFile, err := ioutil.TempFile(filepath.Dir(fileName), "archive_")
+	if err != nil {
+		return err
+	}
+	defer tempFile.Close()
+	defer os.Remove(tempFile.Name())
+
+	logrus.Debugln("Temporary file:", tempFile.Name())
+	err = CreateZipArchive(tempFile, fileNames)
+	if err != nil {
+		return err
+	}
+	tempFile.Close()
+
+	err = os.Rename(tempFile.Name(), fileName)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

helpers/archives/zip_create_test.go

+package archives
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"archive/zip"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+var testZipFileContent = []byte("test content")
+
+func createTestFile(t *testing.T) string {
+	err := ioutil.WriteFile("test_file.txt", testZipFileContent, 0640)
+	assert.NoError(t, err)
+	return "test_file.txt"
+}
+
+func createSymlinkFile(t *testing.T) string {
+	err := os.Symlink("old_symlink", "new_symlink")
+	assert.NoError(t, err)
+	return "new_symlink"
+}
+
+func createTestDirectory(t *testing.T) string {
+	err := os.Mkdir("test_directory", 0711)
+	assert.NoError(t, err)
+	return "test_directory"
+}
+
+func createTestPipe(t *testing.T) string {
+	err := syscall.Mkfifo("test_pipe", 0600)
+	assert.NoError(t, err)
+	return "test_pipe"
+}
+
+func TestZipCreate(t *testing.T) {
+	td, err := ioutil.TempDir("", "zip_create")
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	wd, err := os.Getwd()
+	assert.NoError(t, err)
+	defer os.Chdir(wd)
+
+	err = os.Chdir(td)
+	assert.NoError(t, err)
+
+	tempFile, err := ioutil.TempFile("", "archive")
+	if !assert.NoError(t, err) {
+		return
+	}
+	tempFile.Close()
+	defer os.Remove(tempFile.Name())
+
+	err = CreateZipFile(tempFile.Name(), []string{
+		createTestFile(t),
+		createSymlinkFile(t),
+		createTestDirectory(t),
+		createTestPipe(t),
+		"non_existing_file.txt",
+	})
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	archive, err := zip.OpenReader(tempFile.Name())
+	if !assert.NoError(t, err) {
+		return
+	}
+	defer archive.Close()
+
+	assert.Len(t, archive.File, 3)
+	assert.Equal(t, "test_file.txt", archive.File[0].Name)
+	assert.Equal(t, 0640, archive.File[0].Mode().Perm())
+	assert.NotEmpty(t, archive.File[0].Extra)
+	assert.Equal(t, "new_symlink", archive.File[1].Name)
+	assert.Equal(t, "test_directory/", archive.File[2].Name)
+	assert.NotEmpty(t, archive.File[2].Extra)
+	assert.True(t, archive.File[2].Mode().IsDir())
+}
+
+func TestZipOverwrite(t *testing.T) {
+	td, err := ioutil.TempDir("", "archive")
+	if !assert.NoError(t, err) {
+		return
+	}
+	defer os.RemoveAll(td)
+
+	tempFile := filepath.Join(td, "archive.zip")
+
+	err = CreateZipFile(tempFile, []string{})
+	assert.NoError(t, err)
+
+	err = os.Chmod(td, 0000)
+	assert.NoError(t, err)
+
+	err = CreateZipFile(tempFile, []string{})
+	assert.Error(t, err)
+}

commands/helpers/zip_extra.go → helpers/archives/zip_extra.go

-package commands_helpers
+package archives
 
 import (
 	"archive/zip"
@@ -47,9 +47,8 @@ func createZipTimestampField(w io.Writer, fi os.FileInfo) (err error) {
 	return
 }
 
-func processZipTimestampField(data []byte, file *zip.File) error {
-	fi := file.FileInfo()
-	if !fi.Mode().IsDir() && !fi.Mode().IsRegular() {
+func processZipTimestampField(data []byte, file *zip.FileHeader) error {
+	if !file.Mode().IsDir() && !file.Mode().IsRegular() {
 		return nil
 	}
 
@@ -80,7 +79,7 @@ func createZipExtra(fi os.FileInfo) []byte {
 	return nil
 }
 
-func processZipExtra(file *zip.File) error {
+func processZipExtra(file *zip.FileHeader) error {
 	if len(file.Extra) == 0 {
 		return nil
 	}

helpers/archives/zip_extra_test.go

+package archives
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"archive/zip"
+	"encoding/binary"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCreateZipExtra(t *testing.T) {
+	testFile, err := ioutil.TempFile("", "test")
+	assert.NoError(t, err)
+	defer testFile.Close()
+	defer os.Remove(testFile.Name())
+
+	fi, _ := testFile.Stat()
+	assert.NotNil(t, fi)
+
+	data := createZipExtra(fi)
+	assert.NotEmpty(t, data)
+	assert.Len(t, data, binary.Size(&ZipExtraField{})*2+
+		binary.Size(&ZipUidGidField{})+
+		binary.Size(&ZipTimestampField{}))
+}
+
+func TestProcessZipExtra(t *testing.T) {
+	testFile, err := ioutil.TempFile("", "test")
+	assert.NoError(t, err)
+	defer testFile.Close()
+	defer os.Remove(testFile.Name())
+
+	fi, _ := testFile.Stat()
+	assert.NotNil(t, fi)
+
+	zipFile, err := zip.FileInfoHeader(fi)
+	assert.NoError(t, err)
+	zipFile.Extra = createZipExtra(fi)
+
+	err = ioutil.WriteFile(fi.Name(), []byte{}, 0666)
+	assert.NoError(t, err)
+
+	err = processZipExtra(zipFile)
+	assert.NoError(t, err)
+
+	fi2, _ := testFile.Stat()
+	assert.NotNil(t, fi2)
+	assert.Equal(t, fi.Mode(), fi2.Mode())
+	assert.Equal(t, fi.ModTime(), fi2.ModTime())
+}

commands/helpers/zip_extra_unix.go → helpers/archives/zip_extra_unix.go

 // +build linux darwin freebsd
 
-package commands_helpers
+package archives
 
 import (
 	"archive/zip"
@@ -34,7 +34,7 @@ func createZipUidGidField(w io.Writer, fi os.FileInfo) (err error) {
 	return nil
 }
 
-func processZipUidGidField(data []byte, file *zip.File) error {
+func processZipUidGidField(data []byte, file *zip.FileHeader) error {
 	var ugField ZipUidGidField
 	err := binary.Read(bytes.NewReader(data), binary.LittleEndian, &ugField)
 	if err != nil {

commands/helpers/zip_extra_windows.go → helpers/archives/zip_extra_windows.go

-package commands_helpers
+package archives
 
 import (
 	"archive/zip"
@@ -11,7 +11,7 @@ func createZipUidGidField(w io.Writer, fi os.FileInfo) (err error) {
 	return nil
 }
 
-func processZipUidGidField(data []byte, file *zip.File) error {
+func processZipUidGidField(data []byte, file *zip.FileHeader) error {
 	// TODO: currently not supported
 	return nil
 }

commands/helpers/extract.go → helpers/archives/zip_extract.go

-package commands_helpers
+package archives
 
 import (
 	"archive/zip"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
-	"strings"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/codegangsta/cli"
-	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
-	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
 )
 
-type ExtractCommand struct {
-	File    string `long:"file" description:"The file to extract"`
-	List    bool   `long:"list" description:"List files in archive"`
-	Verbose bool   `long:"verbose" description:"Suppress archiving output"`
-}
-
-func (c *ExtractCommand) extractTarArchive() error {
-	flags := "-zP"
-	if c.List {
-		flags += "t"
-	} else {
-		flags += "x"
-	}
-	if c.Verbose {
-		flags += "v"
-	}
-
-	cmd := exec.Command("tar", flags, "-f", c.File)
-	cmd.Env = os.Environ()
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	logrus.Debugln("Executing command:", strings.Join(cmd.Args, " "))
-	return cmd.Run()
-}
-
-func (c *ExtractCommand) extractFile(file *zip.File) (err error) {
-	if c.Verbose && c.List {
-		fmt.Println(helpers.ToJson(*file))
-	} else if c.Verbose || c.List {
-		fmt.Printf("%v\t%d\t%s\n", file.Mode(), file.UncompressedSize64, file.Name)
-		if c.List {
-			return
-		}
-	}
-
+func extractZipFile(file *zip.File) (err error) {
 	fi := file.FileInfo()
 
 	// Create all parents to extract the file
@@ -101,15 +61,9 @@ func (c *ExtractCommand) extractFile(file *zip.File) (err error) {
 	return
 }
 
-func (c *ExtractCommand) extractZipArchive() error {
-	archive, err := zip.OpenReader(c.File)
-	if err != nil {
-		return err
-	}
-	defer archive.Close()
-
+func ExtractZipArchive(archive *zip.Reader) error {
 	for _, file := range archive.File {
-		if err := c.extractFile(file); err != nil {
+		if err := extractZipFile(file); err != nil {
 			logrus.Warningf("%s: %s", file.Name, err)
 		}
 	}
@@ -121,35 +75,19 @@ func (c *ExtractCommand) extractZipArchive() error {
 		}
 
 		// Process zip metadata
-		if err := processZipExtra(file); err != nil {
+		if err := processZipExtra(&file.FileHeader); err != nil {
 			logrus.Warningf("%s: %s", file.Name, err)
 		}
 	}
 	return nil
 }
 
-func (c *ExtractCommand) extractArchive() error {
-	if isTarArchive(c.File) {
-		return c.extractTarArchive()
-	} else if isZipArchive(c.File) {
-		return c.extractZipArchive()
-	} else {
-		return fmt.Errorf("Unsupported archive format: %q", c.File)
-	}
-}
-
-func (c *ExtractCommand) Execute(context *cli.Context) {
-	if c.File == "" {
-		logrus.Fatalln("Missing archive file name!")
-	}
-
-	err := c.extractArchive()
+func ExtractZipFile(fileName string) error {
+	archive, err := zip.OpenReader(fileName)
 	if err != nil {
-		logrus.Fatalln("Failed to create archive:", err)
+		return err
 	}
-	logrus.Infoln("Done!")
-}
+	defer archive.Close()
 
-func init() {
-	common.RegisterCommand2("extract", "extract files from an archive (internal)", &ExtractCommand{})
+	return ExtractZipArchive(&archive.Reader)
 }

commands/helpers/extract_test.go → helpers/archives/zip_extract_test.go

-package commands_helpers
+package archives
 
 import (
 	"archive/zip"
+	"io"
+	"io/ioutil"
 	"os"
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"io"
 )
 
-func createExtractCommand(t *testing.T) *ExtractCommand {
-	err := os.Chdir(filepath.Join(currentDir, "..", ".."))
-	assert.NoError(t, err)
+func writeArchive(t *testing.T, w io.Writer) {
+	archive := zip.NewWriter(w)
+	defer archive.Close()
 
-	return &ExtractCommand{
-		File: randomTempFile(t, ".zip"),
+	testFile, err := archive.Create("temporary_file.txt")
+	if !assert.NoError(t, err) {
+		return
 	}
+	io.WriteString(testFile, "test file")
 }
 
-func writeArchive(t *testing.T, c *ExtractCommand) {
-	file, err := os.Create(c.File)
+func TestExtractZipFile(t *testing.T) {
+	tempFile, err := ioutil.TempFile("", "archive")
 	if !assert.NoError(t, err) {
 		return
 	}
-	defer file.Close()
-
-	archive := zip.NewWriter(file)
-	defer archive.Close()
+	defer tempFile.Close()
+	defer os.Remove(tempFile.Name())
+	writeArchive(t, tempFile)
+	tempFile.Close()
 
-	testFile, err := archive.Create("test.txt")
+	err = ExtractZipFile(tempFile.Name())
 	if !assert.NoError(t, err) {
 		return
 	}
-	io.WriteString(testFile, "test file")
-}
-
-func TestExtract(t *testing.T) {
-	c := createExtractCommand(t)
-	writeArchive(t, c)
-	defer os.Remove(c.File)
 
-	c.Execute(nil)
-
-	stat, err := os.Stat("test.txt")
-	assert.False(t, os.IsNotExist(err), "Expected test.txt to exist")
+	stat, err := os.Stat("temporary_file.txt")
+	assert.False(t, os.IsNotExist(err), "Expected temporary_file.txt to exist")
 	if !os.IsNotExist(err) {
 		assert.NoError(t, err)
 	}
 
 	if stat != nil {
-		defer os.Remove("test.txt")
+		defer os.Remove("temporary_file.txt")
 		assert.Equal(t, int64(9), stat.Size())
 	}
 }
+
+func TestExtractZipFileNotFound(t *testing.T) {
+	err := ExtractZipFile("non_existing_zip_file.zip")
+	assert.Error(t, err)
+}

network/gitlab.go

 package network
 
 import (
-	"errors"
 	"fmt"
 	"github.com/Sirupsen/logrus"
 	. "gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
@@ -208,35 +207,20 @@ func (n *GitLabClient) UpdateBuild(config RunnerConfig, id int, state BuildState
 	}
 }
 
-func (n *GitLabClient) createArtifactsForm(mpw *multipart.Writer, artifactsFile string) error {
-	wr, err := mpw.CreateFormFile("file", filepath.Base(artifactsFile))
+func (n *GitLabClient) createArtifactsForm(mpw *multipart.Writer, reader io.Reader, baseName string) error {
+	wr, err := mpw.CreateFormFile("file", baseName)
 	if err != nil {
 		return err
 	}
 
-	file, err := os.Open(artifactsFile)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	fi, err := file.Stat()
-	if err != nil {
-		return err
-	}
-	if fi.IsDir() {
-		return errors.New("Failed to upload directories")
-	}
-
-	_, err = io.Copy(wr, file)
+	_, err = io.Copy(wr, reader)
 	if err != nil {
 		return err
 	}
-
 	return nil
 }
 
-func (n *GitLabClient) UploadArtifacts(config BuildCredentials, artifactsFile string) UploadState {
+func (n *GitLabClient) UploadRawArtifacts(config BuildCredentials, reader io.Reader, baseName string) UploadState {
 	pr, pw := io.Pipe()
 	defer pr.Close()
 
@@ -245,7 +229,7 @@ func (n *GitLabClient) UploadArtifacts(config BuildCredentials, artifactsFile st
 	go func() {
 		defer pw.Close()
 		defer mpw.Close()
-		err := n.createArtifactsForm(mpw, artifactsFile)
+		err := n.createArtifactsForm(mpw, reader, baseName)
 		if err != nil {
 			pw.CloseWithError(err)
 		}
@@ -289,6 +273,33 @@ func (n *GitLabClient) UploadArtifacts(config BuildCredentials, artifactsFile st
 	}
 }
 
+func (n *GitLabClient) UploadArtifacts(config BuildCredentials, artifactsFile string) UploadState {
+	log := logrus.WithFields(logrus.Fields{
+		"id":    config.ID,
+		"token": helpers.ShortenToken(config.Token),
+	})
+
+	file, err := os.Open(artifactsFile)
+	if err != nil {
+		log.Errorln("Uploading artifacts to coordinator...", "error", err.Error())
+		return UploadFailed
+	}
+	defer file.Close()
+
+	fi, err := file.Stat()
+	if err != nil {
+		log.Errorln("Uploading artifacts to coordinator...", "error", err.Error())
+		return UploadFailed
+	}
+	if fi.IsDir() {
+		log.Errorln("Uploading artifacts to coordinator...", "error", "cannot upload directories")
+		return UploadFailed
+	}
+
+	baseName := filepath.Base(artifactsFile)
+	return n.UploadRawArtifacts(config, file, baseName)
+}
+
 func (n *GitLabClient) DownloadArtifacts(config BuildCredentials, artifactsFile string) DownloadState {
 	// TODO: Create proper interface for `doRaw` that can use other types than RunnerCredentials
 	mappedConfig := RunnerCredentials{
@@ -330,6 +341,9 @@ func (n *GitLabClient) DownloadArtifacts(config BuildCredentials, artifactsFile
 	case 403:
 		log.Errorln("Downloading artifacts from coordinator...", "forbidden")
 		return DownloadForbidden
+	case 404:
+		log.Errorln("Downloading artifacts from coordinator...", "not found")
+		return DownloadNotFound
 	default:
 		log.Warningln("Downloading artifacts from coordinator...", "failed", res.Status)
 		return DownloadFailed

shells/abstract.go

 package shells
 
 import (
-	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
-	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
+
+	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
+	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
 )
 
 type AbstractShell struct {
 }
 
-type ShellWriter interface {
-	Variable(variable common.BuildVariable)
-	Command(command string, arguments ...string)
-	Line(text string)
-
-	IfDirectory(path string)
-	IfFile(file string)
-	Else()
-	EndIf()
-
-	Cd(path string)
-	RmDir(path string)
-	RmFile(path string)
-	Absolute(path string) string
-
-	Print(fmt string, arguments ...interface{})
-	Notice(fmt string, arguments ...interface{})
-	Warning(fmt string, arguments ...interface{})
-	Error(fmt string, arguments ...interface{})
-	EmptyLine()
-}
-
 func (b *AbstractShell) GetFeatures(features *common.FeaturesInfo) {
 	features.Artifacts = true
 	features.Cache = true
@@ -91,6 +71,76 @@ func (b *AbstractShell) writeCheckoutCmd(w ShellWriter, build *common.Build) {
 	w.Command("git", "checkout", build.Sha)
 }
 
+func (b *AbstractShell) cacheFile(cacheKey string, info common.ShellScriptInfo) string {
+	if cacheKey == "" {
+		return ""
+	}
+	cacheFile := path.Join(info.Build.CacheDir, cacheKey)
+	cacheFile, err := filepath.Rel(info.Build.BuildDir, cacheFile)
+	if err != nil {
+		return ""
+	}
+	return cacheFile
+}
+
+func (b *AbstractShell) encodeArchiverOptions(list interface{}) (args []string) {
+	hash, ok := helpers.ToConfigMap(list)
+	if !ok {
+		return
+	}
+
+	// Collect paths
+	if paths, ok := hash["paths"].([]interface{}); ok {
+		for _, artifactPath := range paths {
+			if file, ok := artifactPath.(string); ok {
+				args = append(args, "--path", file)
+			}
+		}
+	}
+
+	// Archive also untracked files
+	if untracked, ok := hash["untracked"].(bool); ok && untracked {
+		args = append(args, "--untracked")
+	}
+	return
+}
+
+func (b *AbstractShell) cacheExtractor(w ShellWriter, info common.ShellScriptInfo, cacheKey string) {
+	if info.RunnerCommand == "" {
+		w.Warning("The cache is not supported in this executor.")
+		return
+	}
+
+	args := []string{
+		"cache-extractor",
+		"--file", b.cacheFile(cacheKey, info),
+	}
+
+	// Execute archive command
+	w.Notice("Checking cache for %s...", cacheKey)
+	w.Command(info.RunnerCommand, args...)
+}
+
+func (b *AbstractShell) downloadArtifacts(w ShellWriter, build *common.BuildInfo, info common.ShellScriptInfo) {
+	if info.RunnerCommand == "" {
+		w.Warning("The artifacts downloading is not supported in this executor.")
+		return
+	}
+
+	args := []string{
+		"artifacts-downloader",
+		"--url",
+		info.Build.Runner.URL,
+		"--token",
+		build.Token,
+		"--id",
+		strconv.Itoa(build.ID),
+	}
+
+	w.Notice("Downloading artifacts for %s (%d)...", build.Name, build.ID)
+	w.Command(info.RunnerCommand, args...)
+}
+
 func (b *AbstractShell) GeneratePreBuild(w ShellWriter, info common.ShellScriptInfo) {
 	b.writeExports(w, info)
 
@@ -109,27 +159,9 @@ func (b *AbstractShell) GeneratePreBuild(w ShellWriter, info common.ShellScriptI
 
 	b.writeCheckoutCmd(w, build)
 
-	cacheFile := info.Build.CacheFile()
-	cacheFile2 := info.Build.CacheFileForRef("master")
-	if cacheFile == "" {
-		cacheFile = cacheFile2
-		cacheFile2 = ""
-	}
-
 	// Try to restore from main cache, if not found cache for master
-	if cacheFile != "" {
-		// If we have cache, restore it
-		w.IfFile(cacheFile)
-		b.extractFiles(w, info.RunnerCommand, "cache", cacheFile)
-		if cacheFile2 != "" {
-			w.Else()
-
-			// If we have cache, restore it
-			w.IfFile(cacheFile2)
-			b.extractFiles(w, info.RunnerCommand, "cache", cacheFile2)
-			w.EndIf()
-		}
-		w.EndIf()
+	if cacheKey := info.Build.CacheKey(); cacheKey != "" {
+		b.cacheExtractor(w, info, cacheKey)
 	}
 
 	// Process all artifacts
@@ -137,10 +169,7 @@ func (b *AbstractShell) GeneratePreBuild(w ShellWriter, info common.ShellScriptI
 		if otherBuild.Artifacts == nil || otherBuild.Artifacts.Filename == "" {
 			continue
 		}
-
-		b.downloadArtifacts(w, info.Build.Runner, &otherBuild, info.RunnerCommand, otherBuild.Artifacts.Filename)
-		b.extractFiles(w, info.RunnerCommand, otherBuild.Name, otherBuild.Artifacts.Filename)
-		w.RmFile(otherBuild.Artifacts.Filename)
+		b.downloadArtifacts(w, &otherBuild, info)
 	}
 }
 
@@ -161,107 +190,56 @@ func (b *AbstractShell) GenerateCommands(w ShellWriter, info common.ShellScriptI
 	}
 }
 
-func (b *AbstractShell) archiveFiles(w ShellWriter, list interface{}, runnerCommand, archiveType, archivePath string) {
-	hash, ok := helpers.ToConfigMap(list)
-	if !ok {
-		return
-	}
-
-	if runnerCommand == "" {
-		w.Warning("The %s is not supported in this executor.", archiveType)
+func (b *AbstractShell) cacheArchiver(w ShellWriter, list interface{}, info common.ShellScriptInfo, cacheKey string) {
+	if info.RunnerCommand == "" {
+		w.Warning("The cache is not supported in this executor.")
 		return
 	}
 
 	args := []string{
-		"archive",
-		"--file",
-		archivePath,
+		"cache-archiver",
+		"--file", b.cacheFile(cacheKey, info),
 	}
 
-	// Collect paths
-	if paths, ok := hash["paths"].([]interface{}); ok {
-		for _, artifactPath := range paths {
-			if file, ok := artifactPath.(string); ok {
-				args = append(args, "--path", file)
-			}
-		}
-	}
-
-	// Archive also untracked files
-	if untracked, ok := hash["untracked"].(bool); ok && untracked {
-		args = append(args, "--untracked")
-	}
-
-	// Skip creating archive
-	if len(args) <= 3 {
+	// Create list of files to archive
+	archiverArgs := b.encodeArchiverOptions(list)
+	if len(archiverArgs) == 0 {
+		// Skip creating archive
 		return
 	}
+	args = append(args, archiverArgs...)
 
 	// Execute archive command
-	w.Notice("Archiving %s...", archiveType)
-	w.Command(runnerCommand, args...)
+	w.Notice("Creating cache %s...", cacheKey)
+	w.Command(info.RunnerCommand, args...)
 }
 
-func (b *AbstractShell) extractFiles(w ShellWriter, runnerCommand, archiveType, archivePath string) {
-	if runnerCommand == "" {
-		w.Warning("The %s is not supported in this executor.", archiveType)
-		return
-	}
-
-	args := []string{
-		"extract",
-		"--file",
-		archivePath,
-	}
-
-	// Execute extract command
-	w.Notice("Restoring %s...", archiveType)
-	w.Command(runnerCommand, args...)
-}
-
-func (b *AbstractShell) downloadArtifacts(w ShellWriter, runner *common.RunnerConfig, build *common.BuildInfo, runnerCommand, archivePath string) {
-	if runnerCommand == "" {
-		w.Warning("The artifacts downloading is not supported in this executor.")
+func (b *AbstractShell) uploadArtifacts(w ShellWriter, list interface{}, info common.ShellScriptInfo) {
+	if info.RunnerCommand == "" {
+		w.Warning("The artifacts uploading is not supported in this executor.")
 		return
 	}
 
 	args := []string{
-		"artifacts",
-		"--download",
+		"artifacts-uploader",
 		"--url",
-		runner.URL,
+		info.Build.Runner.URL,
 		"--token",
-		build.Token,
+		info.Build.Token,
 		"--id",
-		strconv.Itoa(build.ID),
-		"--file",
-		archivePath,
+		strconv.Itoa(info.Build.ID),
 	}
 
-	w.Notice("Downloading artifacts for %s (%d)...", build.Name, build.ID)
-	w.Command(runnerCommand, args...)
-}
-
-func (b *AbstractShell) uploadArtifacts(w ShellWriter, build *common.Build, runnerCommand, archivePath string) {
-	if runnerCommand == "" {
-		w.Warning("The artifacts uploading is not supported in this executor.")
+	// Create list of files to archive
+	archiverArgs := b.encodeArchiverOptions(list)
+	if len(archiverArgs) == 0 {
+		// Skip creating archive
 		return
 	}
-
-	args := []string{
-		"artifacts",
-		"--url",
-		build.Runner.URL,
-		"--token",
-		build.Token,
-		"--id",
-		strconv.Itoa(build.ID),
-		"--file",
-		archivePath,
-	}
+	args = append(args, archiverArgs...)
 
 	w.Notice("Uploading artifacts...")
-	w.Command(runnerCommand, args...)
+	w.Command(info.RunnerCommand, args...)
 }
 
 func (b *AbstractShell) GeneratePostBuild(w ShellWriter, info common.ShellScriptInfo) {
@@ -270,18 +248,12 @@ func (b *AbstractShell) GeneratePostBuild(w ShellWriter, info common.ShellScript
 	b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")
 
 	// Find cached files and archive them
-	if cacheFile := info.Build.CacheFile(); cacheFile != "" {
-		b.archiveFiles(w, info.Build.Options["cache"], info.RunnerCommand, "cache", cacheFile)
+	if cacheKey := info.Build.CacheKey(); cacheKey != "" {
+		b.cacheArchiver(w, info.Build.Options["cache"], info, cacheKey)
 	}
 
+	// Upload artifacts
 	if info.Build.Network != nil {
-		// Find artifacts
-		b.archiveFiles(w, info.Build.Options["artifacts"], info.RunnerCommand, "artifacts", "artifacts.zip")
-
-		// If archive is created upload it
-		w.IfFile("artifacts.zip")
-		b.uploadArtifacts(w, info.Build, info.RunnerCommand, "artifacts.zip")
-		w.RmFile("aritfacts.zip")
-		w.EndIf()
+		b.uploadArtifacts(w, info.Build.Options["artifacts"], info)
 	}
 }

shells/shell_writer.go

+package shells
+
+import "gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
+
+type ShellWriter interface {
+	Variable(variable common.BuildVariable)
+	Command(command string, arguments ...string)
+	Line(text string)
+
+	IfDirectory(path string)
+	IfFile(file string)
+	Else()
+	EndIf()
+
+	Cd(path string)
+	RmDir(path string)
+	RmFile(path string)
+	Absolute(path string) string
+
+	Print(fmt string, arguments ...interface{})
+	Notice(fmt string, arguments ...interface{})
+	Warning(fmt string, arguments ...interface{})
+	Error(fmt string, arguments ...interface{})
+	EmptyLine()
+}