Commit dd5995b1 authored by Kamil Trzciński's avatar Kamil Trzciński 🔴

Properly handle UTF-8 characters and store build in internal buffer

parent 41ef9484
package common
import (
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"sync"
"time"
)
......@@ -20,7 +20,6 @@ const (
type Build struct {
GetBuildResponse
BuildLog string `json:"-"`
BuildState BuildState `json:"build_state"`
BuildStarted time.Time `json:"build_started"`
BuildFinished time.Time `json:"build_finished"`
......@@ -39,6 +38,9 @@ type Build struct {
ProjectRunnerID int `json:"project_runner_id"`
ProjectRunnerName string `json:"name"`
buildLog bytes.Buffer `json:"-"`
buildLogLock sync.RWMutex
}
func (b *Build) AssignID(otherBuilds ...*Build) {
......@@ -99,10 +101,9 @@ func (b *Build) FullProjectDir() string {
return fmt.Sprintf("%s/%s", b.BuildsDir, b.ProjectDir())
}
func (b *Build) StartBuild(buildsDir string, buildLog string) {
func (b *Build) StartBuild(buildsDir string) {
b.BuildStarted = time.Now()
b.BuildState = Pending
b.BuildLog = buildLog
b.BuildsDir = buildsDir
}
......@@ -113,40 +114,34 @@ func (b *Build) FinishBuild(buildState BuildState, buildMessage string, args ...
b.BuildDuration = b.BuildFinished.Sub(b.BuildStarted)
}
func (b *Build) ReadBuildLog() (string, error) {
maxTraceSize := MaxTraceOutputSize
if b.BuildLog == "" {
return "", errors.New("no build log")
}
func (b *Build) BuildLog() string {
b.buildLogLock.RLock()
defer b.buildLogLock.RUnlock()
return b.buildLog.String()
}
file, err := os.Open(b.BuildLog)
if err != nil {
return "", err
}
defer file.Close()
func (b *Build) BuildLogLen() int {
b.buildLogLock.RLock()
defer b.buildLogLock.RUnlock()
return b.buildLog.Len()
}
limitReader := io.LimitReader(file, maxTraceSize)
data, err := ioutil.ReadAll(limitReader)
if err != nil {
return "", err
}
func (b *Build) WriteString(data string) (int, error) {
b.buildLogLock.Lock()
defer b.buildLogLock.Unlock()
return b.buildLog.WriteString(data)
}
buildTrace := string(data)
if fi, err := file.Stat(); err == nil && fi.Size() > maxTraceSize {
buildLogExceeded := fmt.Sprintf("\nBuild log exceed limit of %v bytes.", maxTraceSize)
buildTrace = buildTrace + buildLogExceeded
}
return buildTrace, nil
func (b *Build) WriteRune(r rune) (int, error) {
b.buildLogLock.Lock()
defer b.buildLogLock.Unlock()
return b.buildLog.WriteRune(r)
}
func (b *Build) SendBuildLog() {
var buildTrace string
buildTrace, err := b.ReadBuildLog()
if err != nil {
buildTrace = "Failed to read build trace: " + err.Error()
}
buildTrace = b.BuildLog()
if b.BuildMessage != "" {
buildTrace = buildTrace + b.BuildMessage
}
......
......@@ -10,4 +10,4 @@ const HealthyChecks = 3
const HealthCheckInterval = 3600
const DefaultWaitForServicesTimeout = 30
const ShutdownTimeout = 30
const MaxTraceOutputSize int64 = 1024 * 1024 // 1MB
const MaxTraceOutputSize = 1024 * 1024 // 1MB
......@@ -2,12 +2,13 @@ package executors
import (
"fmt"
"io/ioutil"
"os"
"time"
"bufio"
log "github.com/Sirupsen/logrus"
"github.com/ayufan/gitlab-ci-multi-runner/common"
"io"
)
type AbstractExecutor struct {
......@@ -19,11 +20,38 @@ type AbstractExecutor struct {
BuildCanceled chan bool
FinishLogWatcher chan bool
BuildFinish chan error
BuildLog *os.File
BuildLog *io.PipeWriter
ShellScript *common.ShellScript
}
func (e *AbstractExecutor) WatchTrace(config common.RunnerConfig, canceled chan bool, finished chan bool) {
func (e *AbstractExecutor) ReadTrace(pipe *io.PipeReader) {
defer e.Debugln("ReadTrace finished")
reader := bufio.NewReader(pipe)
for {
r, s, err := reader.ReadRune()
if s <= 0 {
break
} else if err == nil {
e.Build.WriteRune(r)
} else {
// ignore invalid characters
continue
}
if e.Build.BuildLogLen() > common.MaxTraceOutputSize {
output := fmt.Sprintf("\nBuild log exceed limit of %v bytes.", common.MaxTraceOutputSize)
e.Build.WriteString(output)
break
}
}
pipe.Close()
}
func (e *AbstractExecutor) PushTrace(config common.RunnerConfig, canceled chan bool, finished chan bool) {
defer e.Debugln("PushTrace finished")
buildLog := e.BuildLog
if buildLog == nil {
<-finished
......@@ -33,12 +61,7 @@ func (e *AbstractExecutor) WatchTrace(config common.RunnerConfig, canceled chan
for {
select {
case <-time.After(common.UpdateInterval * time.Second):
buildTrace, err := e.Build.ReadBuildLog()
if err != nil {
e.Debugln("updateBuildLog", "Failed to read build log...", err)
continue
}
buildTrace := e.Build.BuildLog()
switch common.UpdateBuild(config, e.Build.ID, common.Running, buildTrace) {
case common.UpdateSucceeded:
case common.UpdateAbort:
......@@ -64,8 +87,8 @@ func (e *AbstractExecutor) Debugln(args ...interface{}) {
}
func (e *AbstractExecutor) Println(args ...interface{}) {
if e.BuildLog != nil {
e.BuildLog.WriteString(fmt.Sprintln(args...))
if e.Build != nil {
e.Build.WriteString(fmt.Sprintln(args...))
}
args = append([]interface{}{e.Config.ShortDescription(), e.Build.ID}, args...)
......@@ -74,8 +97,8 @@ func (e *AbstractExecutor) Println(args ...interface{}) {
func (e *AbstractExecutor) Errorln(args ...interface{}) {
// write to log file
if e.BuildLog != nil {
e.BuildLog.WriteString(fmt.Sprintln(args...))
if e.Build != nil {
e.Build.WriteString(fmt.Sprintln(args...))
}
args = append([]interface{}{e.Config.ShortDescription(), e.Build.ID}, args...)
......@@ -103,12 +126,10 @@ func (e *AbstractExecutor) startBuild() error {
buildsDir = e.Config.BuildsDir
}
buildLog, err := ioutil.TempFile("", "build_log")
if err != nil {
return err
}
e.BuildLog = buildLog
e.Debugln("Created build log:", buildLog.Name())
// Craete pipe where data are read
reader, writer := io.Pipe()
go e.ReadTrace(reader)
e.BuildLog = writer
// Save hostname
if e.ShowHostname {
......@@ -116,7 +137,7 @@ func (e *AbstractExecutor) startBuild() error {
}
// Start actual build
e.Build.StartBuild(buildsDir, buildLog.Name())
e.Build.StartBuild(buildsDir)
return nil
}
......@@ -138,7 +159,7 @@ func (e *AbstractExecutor) Prepare(config *common.RunnerConfig, build *common.Bu
return err
}
go e.WatchTrace(*e.Config, e.BuildCanceled, e.FinishLogWatcher)
go e.PushTrace(*e.Config, e.BuildCanceled, e.FinishLogWatcher)
return nil
}
......@@ -197,7 +218,6 @@ func (e *AbstractExecutor) Finish(err error) {
func (e *AbstractExecutor) Cleanup() {
if e.BuildLog != nil {
os.Remove(e.BuildLog.Name())
e.BuildLog.Close()
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment