Commit a828046d authored by Tomasz Maczukin's avatar Tomasz Maczukin

Add incremental trace update proof-of-concept

parent b8d12856
......@@ -2,6 +2,7 @@
config.toml
.project
out
builds/
executors/docker/bindata.go
dockerfiles/build/gitlab-runner-helper
.idea/
......@@ -305,4 +305,7 @@ s3-upload:
--target-paths "$(S3_UPLOAD_PATH)/" \
$(shell cd out/; find . -type f)
linux_amd64_build: BUILD_PLATFORMS := -os=linux -arch=amd64
linux_amd64_build: build
FORCE:
......@@ -120,7 +120,7 @@ type RunnerSettings struct {
type RunnerConfig struct {
Name string `toml:"name" json:"name" short:"name" long:"description" env:"RUNNER_NAME" description:"Runner name"`
Limit int `toml:"limit,omitzero" json:"limit" long:"limit" env:"RUNNER_LIMIT" description:"Maximum number of builds processed by this runner"`
OutputLimit int `toml:"output_limit,omitzero" long:"output-limit" env:"RUNNER_OUTPUT_LIMIT" description:"Maximum build trace size"`
OutputLimit int `toml:"output_limit,omitzero" long:"output-limit" env:"RUNNER_OUTPUT_LIMIT" description:"Maximum build trace size in kilobytes"`
RunnerCredentials
RunnerSettings
......
......@@ -113,6 +113,10 @@ type UpdateBuildRequest struct {
Trace string `json:"trace,omitempty"`
}
type SendTracePartRequest struct {
TracePart string
}
type BuildCredentials struct {
ID int `long:"id" env:"CI_BUILD_ID" description:"The build ID to upload artifacts for"`
Token string `long:"token" env:"CI_BUILD_TOKEN" required:"true" description:"Build token"`
......@@ -134,6 +138,8 @@ type Network interface {
DeleteRunner(config RunnerCredentials) bool
VerifyRunner(config RunnerCredentials) bool
UpdateBuild(config RunnerConfig, id int, state BuildState, trace string) UpdateState
UpdateBuildState(config RunnerConfig, id int, state BuildState) UpdateState
SendTracePart(config RunnerConfig, id int, tracePart string) UpdateState
DownloadArtifacts(config BuildCredentials, artifactsFile string) DownloadState
UploadRawArtifacts(config BuildCredentials, reader io.Reader, baseName string) UploadState
UploadArtifacts(config BuildCredentials, artifactsFile string) UploadState
......
......@@ -51,6 +51,20 @@ func (m *Network) UpdateBuild(config common.RunnerConfig, id int, state common.B
return r0
}
func (m *Network) UpdateBuildState(config common.RunnerConfig, id int, state common.BuildState) common.UpdateState {
ret := m.Called(config, id, state)
r0 := ret.Get(0).(common.UpdateState)
return r0
}
func (m *Network) SendTracePart(config common.RunnerConfig, id int, tracePart string) common.UpdateState {
ret := m.Called(config, id, tracePart)
r0 := ret.Get(0).(common.UpdateState)
return r0
}
func (m *Network) DownloadArtifacts(config common.BuildCredentials, artifactsFile string) common.DownloadState {
ret := m.Called(config, artifactsFile)
......
......@@ -29,7 +29,6 @@ type clientBuildTrace struct {
state common.BuildState
finished chan bool
sentTrace int
sentTime time.Time
sentState common.BuildState
}
......@@ -136,23 +135,27 @@ func (c *clientBuildTrace) process(pipe *io.PipeReader) {
func (c *clientBuildTrace) update() common.UpdateState {
c.lock.RLock()
state := c.state
trace := c.log.String()
tracePart := c.log.String()
c.lock.RUnlock()
if c.sentState == state &&
c.sentTrace == len(trace) &&
time.Since(c.sentTime) < traceForceSendInterval {
return common.UpdateSucceeded
}
upload := c.client.UpdateBuild(c.config, c.id, state, trace)
traceUpdate := c.client.SendTracePart(c.config, c.id, tracePart)
if traceUpdate == common.UpdateSucceeded {
c.lock.Lock()
c.log.Reset()
c.lock.Unlock()
}
if upload == common.UpdateSucceeded {
c.sentTrace = len(trace)
stateUpdate := c.client.UpdateBuildState(c.config, c.id, state)
if stateUpdate == common.UpdateSucceeded {
c.sentState = state
c.sentTime = time.Now()
}
return upload
return stateUpdate
}
func (c *clientBuildTrace) watch() {
......
......@@ -188,6 +188,20 @@ func (n *GitLabClient) UpdateBuild(config common.RunnerConfig, id int, state com
Trace: trace,
}
return n.updateBuild(config, id, request)
}
func (n *GitLabClient) UpdateBuildState(config common.RunnerConfig, id int, state common.BuildState) common.UpdateState {
request := common.UpdateBuildRequest{
Info: n.getRunnerVersion(config),
Token: config.Token,
State: state,
}
return n.updateBuild(config, id, request)
}
func (n *GitLabClient) updateBuild(config common.RunnerConfig, id int, request common.UpdateBuildRequest) common.UpdateState {
result, statusText, _ := n.doJSON(config.RunnerCredentials, "PUT", fmt.Sprintf("builds/%d.json", id), 200, &request, nil)
switch result {
case 200:
......@@ -208,6 +222,35 @@ func (n *GitLabClient) UpdateBuild(config common.RunnerConfig, id int, state com
}
}
func (n *GitLabClient) SendTracePart(config common.RunnerConfig, id int, tracePart string) common.UpdateState {
request := common.SendTracePartRequest{
TracePart: tracePart,
}
result, statusText, _ := n.doJSON(config.RunnerCredentials, "PATCH", fmt.Sprintf("builds/%d/trace.txt", id), 200, &request, nil)
logrus.Debugln(fmt.Sprintf("result=%s, statusText=%s", result, statusText))
return common.UpdateSucceeded
/*switch result {
case 200:
config.Log().Println(id, "Submitting build to coordinator...", "ok")
return common.UpdateSucceeded
case 404:
config.Log().Warningln(id, "Submitting build to coordinator...", "aborted")
return common.UpdateAbort
case 403:
config.Log().Errorln(id, "Submitting build to coordinator...", "forbidden")
return common.UpdateAbort
case clientError:
config.Log().WithField("status", statusText).Errorln(id, "Submitting build to coordinator...", "error")
return common.UpdateAbort
default:
config.Log().WithField("status", statusText).Warningln(id, "Submitting build to coordinator...", "failed")
return common.UpdateFailed
}*/
}
func (n *GitLabClient) createArtifactsForm(mpw *multipart.Writer, reader io.Reader, baseName string) error {
wr, err := mpw.CreateFormFile("file", baseName)
if err != nil {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment