Commit a8abd7bd authored by Alessio Caiazza's avatar Alessio Caiazza

Merge branch 'support-raw-artifacts' into 'master'

Support RAW artifacts

See merge request gitlab-org/gitlab-runner!1057
parents e3390813 7a31ed24
......@@ -7,8 +7,10 @@ import (
"io"
"io/ioutil"
"os"
"testing"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
......@@ -100,6 +102,14 @@ func (m *testNetwork) consumeGzipUpload(config common.JobCredentials, reader io.
return m.uploadState
}
func (m *testNetwork) consumeRawUpload(config common.JobCredentials, reader io.Reader, options common.ArtifactsOptions) common.UploadState {
io.Copy(ioutil.Discard, reader)
m.uploadedFiles = append(m.uploadedFiles, "raw")
m.uploadFormat = common.ArtifactFormatRaw
return m.uploadState
}
func (m *testNetwork) UploadRawArtifacts(config common.JobCredentials, reader io.Reader, options common.ArtifactsOptions) common.UploadState {
m.uploadCalled++
......@@ -114,6 +124,9 @@ func (m *testNetwork) UploadRawArtifacts(config common.JobCredentials, reader io
case common.ArtifactFormatGzip:
return m.consumeGzipUpload(config, reader, options)
case common.ArtifactFormatRaw:
return m.consumeRawUpload(config, reader, options)
default:
return common.UploadForbidden
}
......@@ -121,3 +134,8 @@ func (m *testNetwork) UploadRawArtifacts(config common.JobCredentials, reader io
return m.uploadState
}
func writeTestFile(t *testing.T, fileName string) {
err := ioutil.WriteFile(fileName, nil, 0600)
require.NoError(t, err, "Writing file:", fileName)
}
......@@ -41,6 +41,15 @@ func (c *ArtifactsUploaderCommand) generateGzipStream(w *io.PipeWriter) {
w.CloseWithError(err)
}
func (c *ArtifactsUploaderCommand) openRawStream() (io.ReadCloser, error) {
fileNames := c.sortedFiles()
if len(fileNames) > 1 {
return nil, errors.New("only one file can be send as raw")
}
return os.Open(fileNames[0])
}
func (c *ArtifactsUploaderCommand) createReadStream() (string, io.ReadCloser, error) {
if len(c.files) == 0 {
return "", nil, nil
......@@ -62,6 +71,10 @@ func (c *ArtifactsUploaderCommand) createReadStream() (string, io.ReadCloser, er
go c.generateGzipStream(pw)
return name + ".gz", pr, nil
case common.ArtifactFormatRaw:
file, err := c.openRawStream()
return name, file, err
default:
return "", nil, fmt.Errorf("unsupported archive format: %s", c.Format)
}
......
package helpers
import (
"io/ioutil"
"os"
"testing"
......@@ -39,7 +38,7 @@ func TestArtifactsUploaderTooLarge(t *testing.T) {
},
}
ioutil.WriteFile(artifactsTestArchivedFile, nil, 0600)
writeTestFile(t, artifactsTestArchivedFile)
defer os.Remove(artifactsTestArchivedFile)
removeHook := helpers.MakeFatalToPanic()
......@@ -64,7 +63,7 @@ func TestArtifactsUploaderForbidden(t *testing.T) {
},
}
ioutil.WriteFile(artifactsTestArchivedFile, nil, 0600)
writeTestFile(t, artifactsTestArchivedFile)
defer os.Remove(artifactsTestArchivedFile)
removeHook := helpers.MakeFatalToPanic()
......@@ -92,7 +91,7 @@ func TestArtifactsUploaderRetry(t *testing.T) {
},
}
ioutil.WriteFile(artifactsTestArchivedFile, nil, 0600)
writeTestFile(t, artifactsTestArchivedFile)
defer os.Remove(artifactsTestArchivedFile)
removeHook := helpers.MakeFatalToPanic()
......@@ -117,7 +116,7 @@ func TestArtifactsUploaderDefaultSucceeded(t *testing.T) {
},
}
ioutil.WriteFile(artifactsTestArchivedFile, nil, 0600)
writeTestFile(t, artifactsTestArchivedFile)
defer os.Remove(artifactsTestArchivedFile)
cmd.Execute(nil)
......@@ -142,7 +141,7 @@ func TestArtifactsUploaderZipSucceeded(t *testing.T) {
},
}
ioutil.WriteFile(artifactsTestArchivedFile, nil, 0600)
writeTestFile(t, artifactsTestArchivedFile)
defer os.Remove(artifactsTestArchivedFile)
cmd.Execute(nil)
......@@ -168,10 +167,10 @@ func TestArtifactsUploaderGzipSendsMultipleFiles(t *testing.T) {
},
}
ioutil.WriteFile(artifactsTestArchivedFile, nil, 0600)
writeTestFile(t, artifactsTestArchivedFile)
defer os.Remove(artifactsTestArchivedFile)
ioutil.WriteFile(artifactsTestArchivedFile2, nil, 0600)
writeTestFile(t, artifactsTestArchivedFile2)
defer os.Remove(artifactsTestArchivedFile)
cmd.Execute(nil)
......@@ -183,6 +182,61 @@ func TestArtifactsUploaderGzipSendsMultipleFiles(t *testing.T) {
assert.Contains(t, network.uploadedFiles, artifactsTestArchivedFile2)
}
func TestArtifactsUploaderRawSucceeded(t *testing.T) {
network := &testNetwork{
uploadState: common.UploadSucceeded,
}
cmd := ArtifactsUploaderCommand{
JobCredentials: UploaderCredentials,
Format: common.ArtifactFormatRaw,
Name: "my-release",
Type: "my-type",
network: network,
fileArchiver: fileArchiver{
Paths: []string{artifactsTestArchivedFile},
},
}
writeTestFile(t, artifactsTestArchivedFile)
defer os.Remove(artifactsTestArchivedFile)
cmd.Execute(nil)
assert.Equal(t, 1, network.uploadCalled)
assert.Equal(t, common.ArtifactFormatRaw, network.uploadFormat)
assert.Equal(t, "my-release", network.uploadName)
assert.Equal(t, "my-type", network.uploadType)
assert.Contains(t, network.uploadedFiles, "raw")
}
func TestArtifactsUploaderRawDoesNotSendMultipleFiles(t *testing.T) {
network := &testNetwork{
uploadState: common.UploadSucceeded,
}
cmd := ArtifactsUploaderCommand{
JobCredentials: UploaderCredentials,
Format: common.ArtifactFormatRaw,
Name: "junit.xml",
Type: "junit",
network: network,
fileArchiver: fileArchiver{
Paths: []string{artifactsTestArchivedFile, artifactsTestArchivedFile2},
},
}
writeTestFile(t, artifactsTestArchivedFile)
defer os.Remove(artifactsTestArchivedFile)
writeTestFile(t, artifactsTestArchivedFile2)
defer os.Remove(artifactsTestArchivedFile)
removeHook := helpers.MakeFatalToPanic()
defer removeHook()
assert.Panics(t, func() {
cmd.Execute(nil)
})
}
func TestArtifactsUploaderNoFilesDoNotGenerateError(t *testing.T) {
network := &testNetwork{
uploadState: common.UploadSucceeded,
......
......@@ -2,7 +2,6 @@ package helpers
import (
"bytes"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
......@@ -19,7 +18,7 @@ const cacheArchiverArchive = "archive.zip"
const cacheArchiverTestArchivedFile = "archive_file"
func TestCacheArchiverIsUpToDate(t *testing.T) {
ioutil.WriteFile(cacheArchiverTestArchivedFile, nil, 0600)
writeTestFile(t, cacheArchiverTestArchivedFile)
defer os.Remove(cacheArchiverTestArchivedFile)
defer os.Remove(cacheArchiverArchive)
......
......@@ -3,7 +3,6 @@ package helpers
import (
"archive/zip"
"bytes"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
......@@ -47,7 +46,7 @@ func TestCacheExtractorValidArchive(t *testing.T) {
func TestCacheExtractorForInvalidArchive(t *testing.T) {
removeHook := helpers.MakeFatalToPanic()
defer removeHook()
ioutil.WriteFile(cacheExtractorArchive, nil, 0600)
writeTestFile(t, cacheExtractorArchive)
defer os.Remove(cacheExtractorArchive)
cmd := CacheExtractorCommand{
......
package helpers
import (
"io/ioutil"
"os"
"testing"
"time"
......@@ -19,10 +18,10 @@ const fileArchiverAbsoluteFile = "/absolute.txt"
const fileArchiverRelativeFile = "../../../relative.txt"
func TestCacheArchiverAddingUntrackedFiles(t *testing.T) {
ioutil.WriteFile(artifactsTestArchivedFile, nil, 0600)
writeTestFile(t, artifactsTestArchivedFile)
defer os.Remove(artifactsTestArchivedFile)
ioutil.WriteFile(artifactsTestArchivedFile2, nil, 0600)
writeTestFile(t, artifactsTestArchivedFile2)
defer os.Remove(artifactsTestArchivedFile2)
f := fileArchiver{
......@@ -36,7 +35,7 @@ func TestCacheArchiverAddingUntrackedFiles(t *testing.T) {
}
func TestCacheArchiverAddingUntrackedUnicodeFiles(t *testing.T) {
ioutil.WriteFile(fileArchiverUntrackedUnicodeFile, nil, 0600)
writeTestFile(t, fileArchiverUntrackedUnicodeFile)
defer os.Remove(fileArchiverUntrackedUnicodeFile)
f := fileArchiver{
......@@ -49,7 +48,7 @@ func TestCacheArchiverAddingUntrackedUnicodeFiles(t *testing.T) {
}
func TestCacheArchiverAddingFile(t *testing.T) {
ioutil.WriteFile(fileArchiverUntrackedFile, nil, 0600)
writeTestFile(t, fileArchiverUntrackedFile)
defer os.Remove(fileArchiverUntrackedFile)
f := fileArchiver{
......@@ -90,7 +89,7 @@ func TestFileArchiverToAddNotExistingFile(t *testing.T) {
}
func TestFileArchiverChanged(t *testing.T) {
ioutil.WriteFile(fileArchiverUntrackedFile, nil, 0600)
writeTestFile(t, fileArchiverUntrackedFile)
defer os.Remove(fileArchiverUntrackedFile)
now := time.Now()
......@@ -109,10 +108,10 @@ func TestFileArchiverChanged(t *testing.T) {
func TestFileArchiverFileIsNotChanged(t *testing.T) {
now := time.Now()
ioutil.WriteFile(fileArchiverUntrackedFile, nil, 0600)
writeTestFile(t, fileArchiverUntrackedFile)
defer os.Remove(fileArchiverUntrackedFile)
ioutil.WriteFile(fileArchiverArchiveZipFile, nil, 0600)
writeTestFile(t, fileArchiverArchiveZipFile)
defer os.Remove(fileArchiverArchiveZipFile)
f := fileArchiver{
......@@ -128,10 +127,10 @@ func TestFileArchiverFileIsNotChanged(t *testing.T) {
func TestFileArchiverFileIsChanged(t *testing.T) {
now := time.Now()
ioutil.WriteFile(fileArchiverUntrackedFile, nil, 0600)
writeTestFile(t, fileArchiverUntrackedFile)
defer os.Remove(fileArchiverUntrackedFile)
ioutil.WriteFile(fileArchiverArchiveZipFile, nil, 0600)
writeTestFile(t, fileArchiverArchiveZipFile)
defer os.Remove(fileArchiverArchiveZipFile)
f := fileArchiver{
......@@ -145,7 +144,7 @@ func TestFileArchiverFileIsChanged(t *testing.T) {
}
func TestFileArchiverFileDoesNotExist(t *testing.T) {
ioutil.WriteFile(fileArchiverUntrackedFile, nil, 0600)
writeTestFile(t, fileArchiverUntrackedFile)
defer os.Remove(fileArchiverUntrackedFile)
f := fileArchiver{
......
......@@ -57,6 +57,7 @@ type FeaturesInfo struct {
Cache bool `json:"cache"`
Shared bool `json:"shared"`
UploadMultipleArtifacts bool `json:"upload_multiple_artifacts"`
UploadRawArtifacts bool `json:"upload_raw_artifacts"`
Session bool `json:"session"`
Terminal bool `json:"terminal"`
}
......@@ -207,6 +208,7 @@ const (
ArtifactFormatDefault ArtifactFormat = ""
ArtifactFormatZip ArtifactFormat = "zip"
ArtifactFormatGzip ArtifactFormat = "gzip"
ArtifactFormatRaw ArtifactFormat = "raw"
)
type Artifact struct {
......
......@@ -20,6 +20,7 @@ type AbstractShell struct {
func (b *AbstractShell) GetFeatures(features *common.FeaturesInfo) {
features.Artifacts = true
features.UploadMultipleArtifacts = true
features.UploadRawArtifacts = true
features.Cache = true
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment