Commit 177d4157 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'download-artifacts'

parents beac8245 46c91b57
......@@ -12,22 +12,11 @@ import (
type ArtifactCommand struct {
common.BuildCredentials
File string `long:"file" description:"The file containing your build artifacts"`
File string `long:"file" description:"The file containing your build artifacts"`
Download bool `long:"download" description:"Download artifacts instead of uploading them"`
}
func (c *ArtifactCommand) Execute(context *cli.Context) {
formatter.SetRunnerFormatter()
if len(c.File) == 0 {
logrus.Fatalln("Missing archive file")
}
if len(c.URL) == 0 || len(c.Token) == 0 {
logrus.Fatalln("Missing runner credentials")
}
if c.ID <= 0 {
logrus.Fatalln("Missing build ID")
}
func (c *ArtifactCommand) upload() {
gl := network.GitLabClient{}
// If the upload fails, exit with a non-zero exit code to indicate an issue?
......@@ -51,6 +40,48 @@ retry:
os.Exit(1)
}
func (c *ArtifactCommand) download() {
gl := network.GitLabClient{}
// If the download fails, exit with a non-zero exit code to indicate an issue?
retry:
for i := 0; i < 3; i++ {
switch gl.DownloadArtifacts(c.BuildCredentials, c.File) {
case common.DownloadSucceeded:
os.Exit(0)
case common.DownloadForbidden:
break retry
case common.DownloadFailed:
// wait one second to retry
logrus.Warningln("Retrying...")
time.Sleep(time.Second)
break
}
}
os.Exit(1)
}
func (c *ArtifactCommand) Execute(context *cli.Context) {
formatter.SetRunnerFormatter()
if len(c.File) == 0 {
logrus.Fatalln("Missing archive file")
}
if len(c.URL) == 0 || len(c.Token) == 0 {
logrus.Fatalln("Missing runner credentials")
}
if c.ID <= 0 {
logrus.Fatalln("Missing build ID")
}
if c.Download {
c.download()
} else {
c.upload()
}
}
func init() {
common.RegisterCommand2("artifacts", "upload build artifacts (internal)", &ArtifactCommand{})
common.RegisterCommand2("artifacts", "download or upload build artifacts (internal)", &ArtifactCommand{})
}
......@@ -2,6 +2,7 @@ package common
type UpdateState int
type UploadState int
type DownloadState int
const (
UpdateSucceeded UpdateState = iota
......@@ -16,6 +17,12 @@ const (
UploadFailed
)
const (
DownloadSucceeded DownloadState = iota
DownloadForbidden
DownloadFailed
)
type FeaturesInfo struct {
Variables bool `json:"variables"`
Image bool `json:"image"`
......@@ -41,23 +48,40 @@ type GetBuildRequest struct {
type BuildOptions map[string]interface{}
type BuildArtifacts struct {
Filename string `json:"filename,omitempty"`
Size int64 `json:"size,omitempty"`
}
type BuildInfo struct {
ID int `json:"id,omitempty"`
Sha string `json:"sha,omitempty"`
RefName string `json:"ref,omitempty"`
Token string `json:"token"`
Name string `json:"name"`
Stage string `json:"stage"`
Tag bool `json:"tag"`
Artifacts *BuildArtifacts `json:"artifacts_file"`
}
type GetBuildResponse struct {
ID int `json:"id,omitempty"`
ProjectID int `json:"project_id,omitempty"`
Commands string `json:"commands,omitempty"`
RepoURL string `json:"repo_url,omitempty"`
Sha string `json:"sha,omitempty"`
RefName string `json:"ref,omitempty"`
BeforeSha string `json:"before_sha,omitempty"`
AllowGitFetch bool `json:"allow_git_fetch,omitempty"`
Timeout int `json:"timeout,omitempty"`
Variables BuildVariables `json:"variables"`
Options BuildOptions `json:"options"`
Token string `json:"token"`
Name string `json:"name"`
Stage string `json:"stage"`
Tag bool `json:"tag"`
TLSCAChain string `json:"-"`
ID int `json:"id,omitempty"`
ProjectID int `json:"project_id,omitempty"`
Commands string `json:"commands,omitempty"`
RepoURL string `json:"repo_url,omitempty"`
Sha string `json:"sha,omitempty"`
RefName string `json:"ref,omitempty"`
BeforeSha string `json:"before_sha,omitempty"`
AllowGitFetch bool `json:"allow_git_fetch,omitempty"`
Timeout int `json:"timeout,omitempty"`
Variables BuildVariables `json:"variables"`
Options BuildOptions `json:"options"`
Token string `json:"token"`
Name string `json:"name"`
Stage string `json:"stage"`
Tag bool `json:"tag"`
DependsOnBuilds []BuildInfo `json:"depends_on_builds"`
TLSCAChain string `json:"-"`
}
type RegisterRunnerRequest struct {
......@@ -99,5 +123,6 @@ type Network interface {
DeleteRunner(config RunnerCredentials) bool
VerifyRunner(config RunnerCredentials) bool
UpdateBuild(config RunnerConfig, id int, state BuildState, trace string) UpdateState
DownloadArtifacts(config BuildCredentials, artifactsFile string) DownloadState
UploadArtifacts(config BuildCredentials, artifactsFile string) UploadState
}
......@@ -121,15 +121,16 @@ func (n *client) getCAChain(tls *tls.ConnectionState) (certificates string) {
return
}
func (n *client) do(uri, method string, statusCode int, request io.Reader, requestType string, response interface{}, headers http.Header) (int, string, string) {
func (n *client) do(uri, method string, request io.Reader, requestType string, headers http.Header) (res *http.Response, err error) {
url, err := n.url.Parse(uri)
if err != nil {
return -1, err.Error(), ""
return
}
req, err := http.NewRequest(method, url.String(), request)
if err != nil {
return -1, fmt.Sprintf("failed to create NewRequest: %v", err), ""
err = fmt.Errorf("failed to create NewRequest: %v", err)
return
}
if headers != nil {
......@@ -140,15 +141,35 @@ func (n *client) do(uri, method string, statusCode int, request io.Reader, reque
req.Header.Set("Content-Type", requestType)
}
if response != nil {
req.Header.Set("Accept", "application/json")
n.ensureTlsConfig()
res, err = n.Do(req)
if err != nil {
err = fmt.Errorf("couldn't execute %v against %s: %v", req.Method, req.URL, err)
return
}
return
}
n.ensureTlsConfig()
func (n *client) doJson(uri, method string, statusCode int, request interface{}, response interface{}) (int, string, string) {
var body io.Reader
if request != nil {
requestBody, err := json.Marshal(request)
if err != nil {
return -1, fmt.Sprintf("failed to marshal project object: %v", err), ""
}
body = bytes.NewReader(requestBody)
}
headers := make(http.Header)
if response != nil {
headers.Set("Accept", "application/json")
}
res, err := n.Do(req)
res, err := n.do(uri, method, body, "application/json", headers)
if err != nil {
return -1, fmt.Sprintf("couldn't execute %v against %s: %v", req.Method, req.URL, err), ""
return -1, err.Error(), ""
}
defer res.Body.Close()
......@@ -169,20 +190,6 @@ func (n *client) do(uri, method string, statusCode int, request io.Reader, reque
return res.StatusCode, res.Status, n.getCAChain(res.TLS)
}
func (n *client) doJson(uri, method string, statusCode int, request interface{}, response interface{}) (int, string, string) {
var body io.Reader
if request != nil {
requestBody, err := json.Marshal(request)
if err != nil {
return -1, fmt.Sprintf("failed to marshal project object: %v", err), ""
}
body = bytes.NewReader(requestBody)
}
return n.do(uri, method, statusCode, body, "application/json", response, nil)
}
func (n *client) fullUrl(uri string, a ...interface{}) string {
url, err := n.url.Parse(fmt.Sprintf(uri, a...))
if err != nil {
......
......@@ -57,13 +57,13 @@ func (n *GitLabClient) getRunnerVersion(config RunnerConfig) VersionInfo {
return info
}
func (n *GitLabClient) doRaw(runner RunnerCredentials, method, uri string, statusCode int, request io.Reader, requestType string, response interface{}, headers http.Header) (int, string, string) {
func (n *GitLabClient) doRaw(runner RunnerCredentials, method, uri string, request io.Reader, requestType string, headers http.Header) (res *http.Response, err error) {
c, err := n.getClient(runner)
if err != nil {
return clientError, err.Error(), ""
return nil, err
}
return c.do(uri, method, statusCode, request, requestType, response, headers)
return c.do(uri, method, request, requestType, headers)
}
func (n *GitLabClient) doJson(runner RunnerCredentials, method, uri string, statusCode int, request interface{}, response interface{}) (int, string, string) {
......@@ -260,14 +260,20 @@ func (n *GitLabClient) UploadArtifacts(config BuildCredentials, artifactsFile st
headers := make(http.Header)
headers.Set("BUILD-TOKEN", config.Token)
result, statusText, _ := n.doRaw(mappedConfig, "POST", fmt.Sprintf("builds/%d/artifacts", config.ID), 201, pr, mpw.FormDataContentType(), nil, headers)
res, err := n.doRaw(mappedConfig, "POST", fmt.Sprintf("builds/%d/artifacts", config.ID), pr, mpw.FormDataContentType(), headers)
log := logrus.WithFields(logrus.Fields{
"id": config.ID,
"token": helpers.ShortenToken(config.Token),
})
switch result {
if err != nil {
log.Errorln("Uploading artifacts to coordinator...", "error", err.Error())
return UploadFailed
}
defer res.Body.Close()
switch res.StatusCode {
case 201:
log.Println("Uploading artifacts to coordinator...", "ok")
return UploadSucceeded
......@@ -277,11 +283,55 @@ func (n *GitLabClient) UploadArtifacts(config BuildCredentials, artifactsFile st
case 413:
log.Errorln("Uploading artifacts to coordinator...", "too large archive")
return UploadTooLarge
case clientError:
log.Errorln("Uploading artifacts to coordinator...", "error", statusText)
return UploadFailed
default:
log.Warningln("Uploading artifacts to coordinator...", "failed", statusText)
log.Warningln("Uploading artifacts to coordinator...", "failed", res.Status)
return UploadFailed
}
}
func (n *GitLabClient) DownloadArtifacts(config BuildCredentials, artifactsFile string) DownloadState {
// TODO: Create proper interface for `doRaw` that can use other types than RunnerCredentials
mappedConfig := RunnerCredentials{
URL: config.URL,
Token: config.Token,
TLSCAFile: config.TLSCAFile,
}
headers := make(http.Header)
headers.Set("BUILD-TOKEN", config.Token)
res, err := n.doRaw(mappedConfig, "GET", fmt.Sprintf("builds/%d/artifacts", config.ID), nil, "", headers)
log := logrus.WithFields(logrus.Fields{
"id": config.ID,
"token": helpers.ShortenToken(config.Token),
})
if err != nil {
log.Errorln("Uploading artifacts from coordinator...", "error", err.Error())
return DownloadFailed
}
defer res.Body.Close()
switch res.StatusCode {
case 200:
file, err := os.Create(artifactsFile)
if err == nil {
defer file.Close()
_, err = io.Copy(file, res.Body)
}
if err != nil {
file.Close()
os.Remove(file.Name())
log.Errorln("Uploading artifacts from coordinator...", "error", err.Error())
return DownloadFailed
}
log.Println("Downloading artifacts from coordinator...", "ok")
return DownloadSucceeded
case 403:
log.Errorln("Downloading artifacts from coordinator...", "forbidden")
return DownloadForbidden
default:
log.Warningln("Downloading artifacts from coordinator...", "failed", res.Status)
return DownloadFailed
}
}
......@@ -99,6 +99,7 @@ func (b *AbstractShell) GeneratePreBuild(w ShellWriter, info common.ShellScriptI
gitDir := path.Join(build.FullProjectDir(), ".git")
b.writeTLSCAInfo(w, info.Build, "GIT_SSL_CAINFO")
b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")
if build.AllowGitFetch {
b.writeFetchCmd(w, build, projectDir, gitDir)
......@@ -130,6 +131,17 @@ func (b *AbstractShell) GeneratePreBuild(w ShellWriter, info common.ShellScriptI
}
w.EndIf()
}
// Process all artifacts
for _, otherBuild := range info.Build.DependsOnBuilds {
if otherBuild.Artifacts == nil || otherBuild.Artifacts.Filename == "" {
continue
}
b.downloadArtifacts(w, info.Build.Runner, &otherBuild, info.RunnerCommand, otherBuild.Artifacts.Filename)
b.extractFiles(w, info.RunnerCommand, otherBuild.Name, otherBuild.Artifacts.Filename)
w.RmFile(otherBuild.Artifacts.Filename)
}
}
func (b *AbstractShell) GenerateCommands(w ShellWriter, info common.ShellScriptInfo) {
......@@ -207,6 +219,29 @@ func (b *AbstractShell) extractFiles(w ShellWriter, runnerCommand, archiveType,
w.Command(runnerCommand, args...)
}
func (b *AbstractShell) downloadArtifacts(w ShellWriter, runner *common.RunnerConfig, build *common.BuildInfo, runnerCommand, archivePath string) {
if runnerCommand == "" {
w.Warning("The artifacts downloading is not supported in this executor.")
return
}
args := []string{
"artifacts",
"--download",
"--url",
runner.URL,
"--token",
build.Token,
"--id",
strconv.Itoa(build.ID),
"--file",
archivePath,
}
w.Notice("Downloading artifacts for %s (%d)...", build.Name, build.ID)
w.Command(runnerCommand, args...)
}
func (b *AbstractShell) uploadArtifacts(w ShellWriter, build *common.Build, runnerCommand, archivePath string) {
if runnerCommand == "" {
w.Warning("The artifacts uploading is not supported in this executor.")
......@@ -226,13 +261,13 @@ func (b *AbstractShell) uploadArtifacts(w ShellWriter, build *common.Build, runn
}
w.Notice("Uploading artifacts...")
b.writeTLSCAInfo(w, build, "CI_SERVER_TLS_CA_FILE")
w.Command(runnerCommand, args...)
}
func (b *AbstractShell) GeneratePostBuild(w ShellWriter, info common.ShellScriptInfo) {
b.writeExports(w, info)
b.writeCdBuildDir(w, info)
b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")
// Find cached files and archive them
if cacheFile := info.Build.CacheFile(); cacheFile != "" {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment