Commit 63b2cb01 authored by Kamil Trzciński's avatar Kamil Trzciński

Introduce new commands to handle artifacts and caching

- artifacts-downloader: download artifacts from previous stages
- artifacts-uploader: create and upload artifacts
- cache-extractor: restore cache
- cache-archiver: create and store cache
parent 177d4157
package commands_helpers
import (
"errors"
"io/ioutil"
"os"
"time"
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/archives"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/formatter"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/network"
)
type ArtifactsDownloaderCommand struct {
common.BuildCredentials
}
func (c *ArtifactsDownloaderCommand) download(file string) error {
gl := network.GitLabClient{}
// If the download fails, exit with a non-zero exit code to indicate an issue?
retry:
for i := 0; i < 3; i++ {
switch gl.DownloadArtifacts(c.BuildCredentials, file) {
case common.DownloadSucceeded:
return nil
case common.DownloadNotFound:
return os.ErrNotExist
case common.DownloadForbidden:
break retry
case common.DownloadFailed:
// wait one second to retry
logrus.Warningln("Retrying...")
time.Sleep(time.Second)
break
}
}
return errors.New("Failed to download artifacts")
}
func (c *ArtifactsDownloaderCommand) Execute(context *cli.Context) {
formatter.SetRunnerFormatter()
if len(c.URL) == 0 || len(c.Token) == 0 {
logrus.Fatalln("Missing runner credentials")
}
if c.ID <= 0 {
logrus.Fatalln("Missing build ID")
}
// Create temporary file
file, err := ioutil.TempFile("", "artifacts")
if err != nil {
logrus.Fatalln(err)
}
file.Close()
defer os.Remove(file.Name())
// Download artifacts file
err = c.download(file.Name())
if err != nil {
logrus.Fatalln(err)
}
// Extract artifacts file
err = archives.ExtractZipFile(file.Name())
if err != nil {
logrus.Fatalln(err)
}
}
func init() {
common.RegisterCommand2("artifacts-downloader", "download and extract build artifacts (internal)", &ArtifactsDownloaderCommand{})
}
package commands_helpers
import (
"io"
"os"
"time"
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/archives"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/formatter"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/network"
"os"
"time"
)
type ArtifactCommand struct {
type ArtifactsUploaderCommand struct {
common.BuildCredentials
File string `long:"file" description:"The file containing your build artifacts"`
Download bool `long:"download" description:"Download artifacts instead of uploading them"`
FileArchiver
}
func (c *ArtifactCommand) upload() {
func (c *ArtifactsUploaderCommand) createAndUpload(network common.Network) common.UploadState {
pr, pw := io.Pipe()
defer pr.Close()
// Create the archive
go func() {
err := archives.CreateZipArchive(pw, c.sortedFiles())
pw.CloseWithError(err)
}()
// Upload the data
return network.UploadRawArtifacts(c.BuildCredentials, pr, "artifacts.zip")
}
func (c *ArtifactsUploaderCommand) Execute(*cli.Context) {
formatter.SetRunnerFormatter()
if len(c.URL) == 0 || len(c.Token) == 0 {
logrus.Fatalln("Missing runner credentials")
}
if c.ID <= 0 {
logrus.Fatalln("Missing build ID")
}
// Enumerate files
err := c.enumerate()
if err != nil {
logrus.Fatalln(err)
}
gl := network.GitLabClient{}
// If the upload fails, exit with a non-zero exit code to indicate an issue?
retry:
for i := 0; i < 3; i++ {
switch gl.UploadArtifacts(c.BuildCredentials, c.File) {
switch c.createAndUpload(&gl) {
case common.UploadSucceeded:
os.Exit(0)
return
case common.UploadForbidden:
break retry
case common.UploadTooLarge:
......@@ -36,52 +68,9 @@ retry:
break
}
}
os.Exit(1)
}
func (c *ArtifactCommand) download() {
gl := network.GitLabClient{}
// If the download fails, exit with a non-zero exit code to indicate an issue?
retry:
for i := 0; i < 3; i++ {
switch gl.DownloadArtifacts(c.BuildCredentials, c.File) {
case common.DownloadSucceeded:
os.Exit(0)
case common.DownloadForbidden:
break retry
case common.DownloadFailed:
// wait one second to retry
logrus.Warningln("Retrying...")
time.Sleep(time.Second)
break
}
}
os.Exit(1)
}
func (c *ArtifactCommand) Execute(context *cli.Context) {
formatter.SetRunnerFormatter()
if len(c.File) == 0 {
logrus.Fatalln("Missing archive file")
}
if len(c.URL) == 0 || len(c.Token) == 0 {
logrus.Fatalln("Missing runner credentials")
}
if c.ID <= 0 {
logrus.Fatalln("Missing build ID")
}
if c.Download {
c.download()
} else {
c.upload()
}
}
func init() {
common.RegisterCommand2("artifacts", "download or upload build artifacts (internal)", &ArtifactCommand{})
common.RegisterCommand2("artifacts-uploader", "create and upload build artifacts (internal)", &ArtifactsUploaderCommand{})
}
package commands_helpers
import (
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/archives"
)
type CacheArchiverCommand struct {
FileArchiver
File string `long:"file" description:"The path to file"`
}
func (c *CacheArchiverCommand) Execute(*cli.Context) {
if c.File == "" {
logrus.Fatalln("Missing --file")
}
// Enumerate files
err := c.enumerate()
if err != nil {
logrus.Fatalln(err)
}
// Check if list of files changed
if !c.isFileChanged(c.File) {
logrus.Infoln("Archive is up to date!")
return
}
// Create archive
err = archives.CreateZipFile(c.File, c.sortedFiles())
if err != nil {
logrus.Fatalln(err)
}
}
func init() {
common.RegisterCommand2("cache-archiver", "create and upload cache artifacts (internal)", &CacheArchiverCommand{})
}
package commands_helpers
import (
"os"
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/archives"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/formatter"
)
type CacheExtractorCommand struct {
File string `long:"file" description:"The file containing your cache artifacts"`
}
func (c *CacheExtractorCommand) Execute(context *cli.Context) {
formatter.SetRunnerFormatter()
if len(c.File) == 0 {
logrus.Fatalln("Missing cache file")
}
err := archives.ExtractZipFile(c.File)
if err != nil && !os.IsNotExist(err) {
logrus.Fatalln(err)
}
}
func init() {
common.RegisterCommand2("cache-extractor", "download and extract cache artifacts (internal)", &CacheExtractorCommand{})
}
package commands_helpers
import (
"archive/zip"
"bufio"
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
......@@ -16,36 +14,18 @@ import (
"time"
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
)
type ArchiveCommand struct {
type FileArchiver struct {
Paths []string `long:"path" description:"Add paths to archive"`
Untracked bool `long:"untracked" description:"Add git untracked files"`
File string `long:"file" description:"The path to file"`
Verbose bool `long:"verbose" description:"Detailed information"`
List bool `long:"list" description:"List files to archive"`
wd string
files map[string]os.FileInfo
}
func isTarArchive(fileName string) bool {
if strings.HasSuffix(fileName, ".tgz") || strings.HasSuffix(fileName, ".tar.gz") {
return true
}
return false
}
func isZipArchive(fileName string) bool {
if strings.HasSuffix(fileName, ".zip") {
return true
}
return false
}
func (c *ArchiveCommand) isChanged(modTime time.Time) bool {
func (c *FileArchiver) isChanged(modTime time.Time) bool {
for _, info := range c.files {
if modTime.Before(info.ModTime()) {
return true
......@@ -54,33 +34,19 @@ func (c *ArchiveCommand) isChanged(modTime time.Time) bool {
return false
}
func (c *ArchiveCommand) zipArchiveChanged() bool {
archive, err := zip.OpenReader(c.File)
if err != nil {
logrus.Warningf("%s: %v", c.File, err)
return true
}
defer archive.Close()
for _, file := range archive.File {
_, err := os.Lstat(file.Name)
if os.IsNotExist(err) {
return true
func (c *FileArchiver) isFileChanged(fileName string) bool {
ai, err := os.Stat(fileName)
if ai != nil {
if !c.isChanged(ai.ModTime()) {
return false
}
} else if !os.IsNotExist(err) {
logrus.Warningln(err)
}
return false
return true
}
func (c *ArchiveCommand) isFileListChanged() bool {
if isZipArchive(c.File) {
return c.zipArchiveChanged()
} else {
logrus.Warningln("The archive can't be verified if file list changed: operation not supported")
// TODO: this is not supported
return false
}
}
func (c *ArchiveCommand) sortedFiles() []string {
func (c *FileArchiver) sortedFiles() []string {
files := make([]string, len(c.files))
i := 0
......@@ -93,7 +59,7 @@ func (c *ArchiveCommand) sortedFiles() []string {
return files
}
func (c *ArchiveCommand) add(path string) (err error) {
func (c *FileArchiver) add(path string) (err error) {
// Always use slashes
path = filepath.ToSlash(path)
......@@ -105,7 +71,7 @@ func (c *ArchiveCommand) add(path string) (err error) {
return
}
func (c *ArchiveCommand) process(match string) bool {
func (c *FileArchiver) process(match string) bool {
var absolute, relative string
var err error
......@@ -133,7 +99,7 @@ func (c *ArchiveCommand) process(match string) bool {
return false
}
func (c *ArchiveCommand) processPaths() {
func (c *FileArchiver) processPaths() {
for _, path := range c.Paths {
matches, err := filepath.Glob(path)
if err != nil {
......@@ -163,7 +129,7 @@ func (c *ArchiveCommand) processPaths() {
}
}
func (c *ArchiveCommand) processUntracked() {
func (c *FileArchiver) processUntracked() {
if !c.Untracked {
return
}
......@@ -202,156 +168,10 @@ func (c *ArchiveCommand) processUntracked() {
}
}
func (c *ArchiveCommand) listFiles() {
if len(c.files) == 0 {
logrus.Infoln("No files to archive.")
return
}
for _, file := range c.sortedFiles() {
println(string(file))
}
}
func (c *ArchiveCommand) createZipArchive(w io.Writer, fileNames []string) error {
archive := zip.NewWriter(w)
defer archive.Close()
for _, fileName := range fileNames {
fi, err := os.Lstat(fileName)
if err != nil {
logrus.Warningln("File ignored: %q: %v", fileName, err)
continue
}
fh, err := zip.FileInfoHeader(fi)
fh.Name = fileName
fh.Extra = createZipExtra(fi)
switch fi.Mode() & os.ModeType {
case os.ModeDir:
fh.Name += "/"
_, err := archive.CreateHeader(fh)
if err != nil {
return err
}
case os.ModeSymlink:
fw, err := archive.CreateHeader(fh)
if err != nil {
return err
}
link, err := os.Readlink(fileName)
if err != nil {
return err
}
io.WriteString(fw, link)
case os.ModeNamedPipe, os.ModeSocket, os.ModeDevice:
// Ignore the files that of these types
logrus.Warningln("File ignored: %q", fileName)
default:
fh.Method = zip.Deflate
fw, err := archive.CreateHeader(fh)
if err != nil {
return err
}
file, err := os.Open(fileName)
if err != nil {
return err
}
_, err = io.Copy(fw, file)
file.Close()
if err != nil {
return err
}
break
}
if c.Verbose {
fmt.Printf("%v\t%d\t%s\n", fh.Mode(), fh.UncompressedSize64, fh.Name)
}
}
return nil
}
func (c *ArchiveCommand) createTarArchive(w io.Writer, files []string) error {
var list bytes.Buffer
for _, file := range c.sortedFiles() {
list.WriteString(string(file) + "\n")
}
flags := "-zcP"
if c.Verbose {
flags += "v"
}
cmd := exec.Command("tar", flags, "-T", "-", "--no-recursion")
cmd.Env = os.Environ()
cmd.Stdin = &list
cmd.Stdout = w
cmd.Stderr = os.Stderr
logrus.Debugln("Executing command:", strings.Join(cmd.Args, " "))
return cmd.Run()
}
func (c *ArchiveCommand) createArchive(w io.Writer, files []string) error {
if isTarArchive(c.File) {
return c.createTarArchive(w, files)
} else if isZipArchive(c.File) {
return c.createZipArchive(w, files)
} else {
return fmt.Errorf("Unsupported archive format: %q", c.File)
}
}
func (c *ArchiveCommand) archive() {
if len(c.files) == 0 {
logrus.Infoln("No files to archive.")
return
}
logrus.Infoln("Creating archive", filepath.Base(c.File), "...")
// create directories to store archive
os.MkdirAll(filepath.Dir(c.File), 0700)
tempFile, err := ioutil.TempFile(filepath.Dir(c.File), "archive_")
if err != nil {
logrus.Fatalln("Failed to create temporary archive", err)
}
defer tempFile.Close()
defer os.Remove(tempFile.Name())
logrus.Debugln("Temporary file:", tempFile.Name())
err = c.createArchive(tempFile, c.sortedFiles())
if err != nil {
logrus.Fatalln("Failed to create archive:", err)
}
tempFile.Close()
err = os.Rename(tempFile.Name(), c.File)
if err != nil {
logrus.Warningln("Failed to rename archive:", err)
}
logrus.Infoln("Done!")
}
func (c *ArchiveCommand) Execute(context *cli.Context) {
func (c *FileArchiver) enumerate() error {
wd, err := os.Getwd()
if err != nil {
logrus.Fatalln("Failed to get current working directory:", err)
}
if c.File == "" && !c.List {
logrus.Fatalln("Missing archive file name!")
return fmt.Errorf("Failed to get current working directory: %v", err)
}
c.wd = wd
......@@ -359,25 +179,5 @@ func (c *ArchiveCommand) Execute(context *cli.Context) {
c.processPaths()
c.processUntracked()
ai, err := os.Stat(c.File)
if err != nil && !os.IsNotExist(err) {
logrus.Fatalln("Failed to verify archive:", c.File, err)
}
if ai != nil {
if !c.isChanged(ai.ModTime()) && !c.zipArchiveChanged() {
logrus.Infoln("Archive is up to date!")
return
}
}
if c.List {
c.listFiles()
} else {
c.archive()
}
}
func init() {
common.RegisterCommand2("archive", "find and archive files (internal)", &ArchiveCommand{})
return nil
}
package commands_helpers
import (
"archive/zip"
"github.com/stretchr/testify/assert"
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"
)
const UntrackedFileName = "some_fancy_untracked_file"
var currentDir, _ = os.Getwd()
func randomTempFile(t *testing.T, format string) string {
file, err := ioutil.TempFile("", "archive_")
assert.NoError(t, err)
defer file.Close()
defer os.Remove(file.Name())
return file.Name() + format
}
func createArchiveCommand(t *testing.T) *ArchiveCommand {
err := os.Chdir(filepath.Join(currentDir, "..", ".."))
assert.NoError(t, err)
return &ArchiveCommand{
File: randomTempFile(t, ".zip"),
Verbose: true,
}
}
func filesInFolder(path string) []string {
matches, _ := filepath.Glob(path)
return matches
}
func readArchiveContent(t *testing.T, c *ArchiveCommand) (resultMap map[string]bool) {
resultMap = make(map[string]bool)
archive, err := zip.OpenReader(c.File)
assert.NoError(t, err)
defer archive.Close()
for _, file := range archive.File {
resultMap[file.Name] = true
}
return
}
func verifyArchiveContent(t *testing.T, c *ArchiveCommand, files ...string) {
resultMap := readArchiveContent(t, c)
for _, file := range files {
assert.True(t, resultMap[file], "File should exist %q", file)
delete(resultMap, file)
}
assert.Len(t, resultMap, 0, "No extra file should exist")
}
func TestArchiveNotCreatingArchive(t *testing.T) {
cmd := createArchiveCommand(t)
defer os.Remove(cmd.File)
cmd.Execute(nil)
_, err := os.Stat(cmd.File)
assert.True(t, os.IsNotExist(err), "File should not exist", cmd.File, err)
}
func TestArchiveAddingSomeLocalFiles(t *testing.T) {
cmd := createArchiveCommand(t)
defer os.Remove(cmd.File)
cmd.Paths = []string{
"commands/helpers/*",
}
cmd.Execute(nil)
verifyArchiveContent(t, cmd, filesInFolder("commands/helpers/*")...)
}
func TestArchiveNotAddingDuplicateFiles(t *testing.T) {
cmd := createArchiveCommand(t)
defer os.Remove(cmd.File)
cmd.Paths = []string{
"commands/helpers/*",
"commands/helpers/archive.go",
}
cmd.Execute(nil)
verifyArchiveContent(t, cmd, filesInFolder("commands/helpers/*")...)
}
func TestArchiveAddingUntrackedFiles(t *testing.T) {
cmd := createArchiveCommand(t)
defer os.Remove(cmd.File)
err := ioutil.WriteFile(UntrackedFileName, []byte{}, 0700)
assert.NoError(t, err)
cmd.Untracked = true
cmd.Execute(nil)
files := readArchiveContent(t, cmd)
assert.NotEmpty(t, files)
assert.True(t, files[UntrackedFileName])
}
func TestArchiveUpdating(t *testing.T) {
tempFile := randomTempFile(t, ".zip")
defer os.Remove(tempFile)
err := ioutil.WriteFile(UntrackedFileName, []byte{}, 0700)
assert.NoError(t, err)
cmd := createArchiveCommand(t)
defer os.Remove(cmd.File)
cmd.Paths = []string{
"commands",
UntrackedFileName,
}
cmd.Execute(nil)
archive1, err := os.Stat(cmd.File)
assert.NoError(t, err, "Archive is created")
cmd.Execute(nil)
archive2, err := os.Stat(cmd.File)
assert.NoError(t, err, "Archive is created")
assert.Equal(t, archive1.ModTime(), archive2.ModTime(), "Archive should not be modified")