Commit 5a3aa812 authored by Kamil Trzciński's avatar Kamil Trzciński

Introduce Acquire/Release to Executor

- This allow to allocate resources in context of executor for time of fetching the builds
parent 867071e5
package commands
import (
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
"sync"
)
type buildsHelper struct {
builds []*common.Build
buildsLock sync.Mutex
}
func (b *buildsHelper) count(runner *common.RunnerConfig) int {
count := 0
for _, build := range b.builds {
if build.Runner.ShortDescription() == runner.ShortDescription() {
count++
}
}
return count
}
func (b *buildsHelper) acquire(runner *runnerAcquire) (build *common.Build) {
b.buildsLock.Lock()
defer b.buildsLock.Unlock()
// Check number of builds
count := b.count(&runner.RunnerConfig)
if runner.Limit > 0 && count >= runner.Limit {
// Too many builds
return
}
// Create a new build
build = &common.Build{
Runner: &runner.RunnerConfig,
ExecutorData: runner.data,
}
build.AssignID(b.builds...)
b.builds = append(b.builds, build)
return
}
func (b *buildsHelper) release(deleteBuild *common.Build) bool {
b.buildsLock.Lock()
defer b.buildsLock.Unlock()
for idx, build := range b.builds {
if build == deleteBuild {
b.builds = append(b.builds[0:idx], b.builds[idx+1:]...)
return true
}
}
return false
}
package commands
import (
"sync"
"time"
"github.com/Sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
)
type healthData struct {
failures int
lastCheck time.Time
}
type healthHelper struct {
healthy map[string]*healthData
healthyLock sync.Mutex
}
func (mr *healthHelper) getHealth(id string) *healthData {
mr.healthyLock.Lock()
defer mr.healthyLock.Unlock()
if mr.healthy == nil {
mr.healthy = map[string]*healthData{}
}
health := mr.healthy[id]
if health == nil {
health = &healthData{
lastCheck: time.Now(),
}
mr.healthy[id] = health
}
return health
}
func (mr *healthHelper) isHealthy(id string) bool {
health := mr.getHealth(id)
if health.failures < common.HealthyChecks {
return true
}
if time.Since(health.lastCheck) > common.HealthCheckInterval*time.Second {
logrus.Errorln("Runner", id, "is not healthy, but will be checked!")
health.failures = 0
health.lastCheck = time.Now()
return true
}
return false
}
func (mr *healthHelper) makeHealthy(id string, healthy bool) {
health := mr.getHealth(id)
if healthy {
health.failures = 0
health.lastCheck = time.Now()
} else {
health.failures++
if health.failures >= common.HealthyChecks {
logrus.Errorln("Runner", id, "is not healthy and will be disabled!")
}
}
}
......@@ -6,7 +6,6 @@ import (
"os"
"os/signal"
"runtime"
"sync"
"syscall"
"time"
......@@ -16,28 +15,32 @@ import (
log "github.com/Sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/service"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/network"
)
type RunnerHealth struct {
failures int
lastCheck time.Time
type runnerAcquire struct {
common.RunnerConfig
provider common.ExecutorProvider
data common.ExecutorData
}
func (r *runnerAcquire) Release() {
r.provider.Release(&r.RunnerConfig, r.data)
}
type RunCommand struct {
configOptions
network common.Network
healthHelper
buildsHelper
ServiceName string `short:"n" long:"service" description:"Use different names for different services"`
WorkingDirectory string `short:"d" long:"working-directory" description:"Specify custom working directory"`
User string `short:"u" long:"user" description:"Use specific user to execute shell scripts"`
Syslog bool `long:"syslog" description:"Log to syslog"`
builds []*common.Build
buildsLock sync.RWMutex
healthy map[string]*RunnerHealth
healthyLock sync.Mutex
finished bool
abortBuilds chan os.Signal
interruptSignal chan os.Signal
......@@ -49,148 +52,66 @@ func (mr *RunCommand) log() *log.Entry {
return log.WithField("builds", len(mr.builds))
}
func (mr *RunCommand) getHealth(runner *common.RunnerConfig) *RunnerHealth {
mr.healthyLock.Lock()
defer mr.healthyLock.Unlock()
if mr.healthy == nil {
mr.healthy = map[string]*RunnerHealth{}
}
health := mr.healthy[runner.UniqueID()]
if health == nil {
health = &RunnerHealth{
lastCheck: time.Now(),
}
mr.healthy[runner.UniqueID()] = health
}
return health
}
func (mr *RunCommand) isHealthy(runner *common.RunnerConfig) bool {
health := mr.getHealth(runner)
if health.failures < common.HealthyChecks {
return true
func (mr *RunCommand) feedRunner(runner *common.RunnerConfig, runners chan *runnerAcquire) {
if !mr.isHealthy(runner.UniqueID()) {
return
}
if time.Since(health.lastCheck) > common.HealthCheckInterval*time.Second {
mr.log().Errorln("Runner", runner.ShortDescription(), "is not healthy, but will be checked!")
health.failures = 0
health.lastCheck = time.Now()
return true
provider := common.GetExecutor(runner.Executor)
if provider == nil {
return
}
return false
}
func (mr *RunCommand) makeHealthy(runner *common.RunnerConfig) {
health := mr.getHealth(runner)
health.failures = 0
health.lastCheck = time.Now()
}
func (mr *RunCommand) makeUnhealthy(runner *common.RunnerConfig) {
health := mr.getHealth(runner)
health.failures++
if health.failures >= common.HealthyChecks {
mr.log().Errorln("Runner", runner.ShortDescription(), "is not healthy and will be disabled!")
data, err := provider.Acquire(runner)
if err != nil {
log.Warningln("Failed to update executor", runner.Executor, "for", runner.ShortDescription(), err)
return
}
}
func (mr *RunCommand) addBuild(newBuild *common.Build) {
mr.buildsLock.Lock()
defer mr.buildsLock.Unlock()
newBuild.AssignID(mr.builds...)
mr.builds = append(mr.builds, newBuild)
mr.log().Debugln("Added a new build", newBuild)
runners <- &runnerAcquire{*runner, provider, data}
}
func (mr *RunCommand) removeBuild(deleteBuild *common.Build) bool {
mr.buildsLock.Lock()
defer mr.buildsLock.Unlock()
for idx, build := range mr.builds {
if build == deleteBuild {
mr.builds = append(mr.builds[0:idx], mr.builds[idx+1:]...)
mr.log().Debugln("Build removed", deleteBuild)
return true
}
}
return false
}
func (mr *RunCommand) buildsForRunner(runner *common.RunnerConfig) int {
count := 0
for _, build := range mr.builds {
if build.Runner == runner {
count++
func (mr *RunCommand) feedRunners(runners chan *runnerAcquire) {
for !mr.finished {
mr.log().Debugln("Feeding runners to channel")
config := mr.config
for _, runner := range config.Runners {
mr.feedRunner(runner, runners)
}
time.Sleep(common.CheckInterval * time.Second)
}
return count
}
func (mr *RunCommand) requestBuild(runner *common.RunnerConfig) *common.Build {
if runner == nil {
return nil
}
if !mr.isHealthy(runner) {
return nil
}
count := mr.buildsForRunner(runner)
if runner.Limit > 0 && count >= runner.Limit {
return nil
}
func (mr *RunCommand) processRunner(id int, runner *runnerAcquire) {
defer runner.Release()
buildData, healthy := mr.network.GetBuild(*runner)
if healthy {
mr.makeHealthy(runner)
} else {
mr.makeUnhealthy(runner)
// Acquire build slot
build := mr.buildsHelper.acquire(runner)
if build == nil {
return
}
defer mr.buildsHelper.release(build)
// Receive a new build
buildData, healthy := mr.network.GetBuild(runner.RunnerConfig)
mr.makeHealthy(runner.UniqueID(), healthy)
if buildData == nil {
return nil
}
mr.log().Debugln("Received new build for", runner.ShortDescription(), "build", buildData.ID)
newBuild := &common.Build{
GetBuildResponse: *buildData,
Runner: runner,
BuildAbort: mr.abortBuilds,
Network: mr.network,
return
}
return newBuild
}
func (mr *RunCommand) feedRunners(runners chan *common.RunnerConfig) {
for !mr.finished {
mr.log().Debugln("Feeding runners to channel")
config := mr.config
for _, runner := range config.Runners {
runners <- runner
}
time.Sleep(common.CheckInterval * time.Second)
}
// Process a build
build.GetBuildResponse = *buildData
build.BuildAbort = mr.abortBuilds
build.Network = mr.network
build.Run(mr.config)
}
func (mr *RunCommand) processRunners(id int, stopWorker chan bool, runners chan *common.RunnerConfig) {
func (mr *RunCommand) processRunners(id int, stopWorker chan bool, runners chan *runnerAcquire) {
mr.log().Debugln("Starting worker", id)
for !mr.finished {
select {
case runner := <-runners:
mr.log().Debugln("Checking runner", runner, "on", id)
newJob := mr.requestBuild(runner)
if newJob == nil {
break
}
mr.addBuild(newJob)
newJob.Run(mr.config)
mr.removeBuild(newJob)
newJob = nil
mr.processRunner(id, runner)
// force GC cycle after processing build
runtime.GC()
......@@ -203,7 +124,7 @@ func (mr *RunCommand) processRunners(id int, stopWorker chan bool, runners chan
<-stopWorker
}
func (mr *RunCommand) startWorkers(startWorker chan int, stopWorker chan bool, runners chan *common.RunnerConfig) {
func (mr *RunCommand) startWorkers(startWorker chan int, stopWorker chan bool, runners chan *runnerAcquire) {
for !mr.finished {
id := <-startWorker
go mr.processRunners(id, stopWorker, runners)
......@@ -222,7 +143,7 @@ func (mr *RunCommand) loadConfig() error {
}
mr.healthy = nil
mr.log().Println("Config loaded.")
mr.log().Println("Config loaded:", helpers.ToYAML(mr.config))
return nil
}
......@@ -320,7 +241,7 @@ func (mr *RunCommand) updateConfig() os.Signal {
}
func (mr *RunCommand) Run() {
runners := make(chan *common.RunnerConfig)
runners := make(chan *runnerAcquire)
go mr.feedRunners(runners)
startWorker := make(chan int)
......
......@@ -50,7 +50,7 @@ func waitForInterrupts(finished *bool, abortSignal chan os.Signal, doneSignal ch
}
}
func (r *RunSingleCommand) processBuild(abortSignal chan os.Signal) {
func (r *RunSingleCommand) processBuild(data common.ExecutorData, abortSignal chan os.Signal) {
buildData, healthy := r.network.GetBuild(r.RunnerConfig)
if !healthy {
log.Println("Runner is not healthy!")
......@@ -76,6 +76,7 @@ func (r *RunSingleCommand) processBuild(abortSignal chan os.Signal) {
Runner: &r.RunnerConfig,
BuildAbort: abortSignal,
Network: r.network,
ExecutorData: data,
}
newBuild.AssignID()
newBuild.Run(config)
......@@ -92,6 +93,11 @@ func (r *RunSingleCommand) Execute(c *cli.Context) {
log.Fatalln("Missing Executor")
}
executorProvider := common.GetExecutor(r.Executor)
if executorProvider == nil {
log.Fatalln("Uknown executor:", r.Executor)
}
log.Println("Starting runner for", r.URL, "with token", r.ShortDescription(), "...")
finished := false
......@@ -101,7 +107,13 @@ func (r *RunSingleCommand) Execute(c *cli.Context) {
go waitForInterrupts(&finished, abortSignal, doneSignal)
for !finished {
r.processBuild(abortSignal)
data, err := executorProvider.Acquire(&r.RunnerConfig)
if err != nil {
log.Warningln("Executor update:", err)
}
r.processBuild(data, abortSignal)
executorProvider.Release(&r.RunnerConfig, data)
}
doneSignal <- 0
......
......@@ -4,6 +4,7 @@ import (
"bytes"
"errors"
"fmt"
"github.com/Sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
"net/url"
"os"
......@@ -37,6 +38,7 @@ type Build struct {
CacheDir string `json:"-" yaml:"-"`
Hostname string `json:"-" yaml:"-"`
Runner *RunnerConfig `json:"runner"`
ExecutorData ExecutorData
// Unique ID for all running builds on this runner
RunnerID int `json:"runner_id"`
......@@ -212,13 +214,21 @@ func (b *Build) SendBuildLog() {
}
func (b *Build) Run(globalConfig *Config) error {
executor := NewExecutor(b.Runner.Executor)
if executor == nil {
logrus.Debugln("Starting a new build:", helpers.ToYAML(b))
provider := GetExecutor(b.Runner.Executor)
if provider == nil {
b.WriteString("Executor not found: " + b.Runner.Executor)
b.SendBuildLog()
return errors.New("executor not found")
}
executor := provider.Create()
if executor == nil {
b.WriteString("Failed to create executor: " + b.Runner.Executor)
b.SendBuildLog()
return errors.New("executor not found")
}
err := executor.Prepare(globalConfig, b.Runner, b)
if err == nil {
err = executor.Start()
......
......@@ -4,6 +4,8 @@ import (
log "github.com/Sirupsen/logrus"
)
type ExecutorData interface{}
type Executor interface {
Prepare(globalConfig *Config, config *RunnerConfig, build *Build) error
Start() error
......@@ -15,6 +17,8 @@ type Executor interface {
type ExecutorProvider interface {
CanCreate() bool
Create() Executor
Acquire(config *RunnerConfig) (ExecutorData, error)
Release(config *RunnerConfig, data ExecutorData) error
GetFeatures(features *FeaturesInfo)
}
......
......@@ -18,6 +18,14 @@ func (e DefaultExecutorProvider) Create() common.Executor {
return e.Creator()
}
func (e DefaultExecutorProvider) Acquire(config *common.RunnerConfig) (common.ExecutorData, error) {
return nil, nil
}
func (e DefaultExecutorProvider) Release(config *common.RunnerConfig, data common.ExecutorData) error {
return nil
}
func (e DefaultExecutorProvider) GetFeatures(features *common.FeaturesInfo) {
if e.FeaturesUpdater != nil {
e.FeaturesUpdater(features)
......
package helpers
import (
"testing"
"github.com/BurntSushi/toml"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing"
)
func TestTOMLOmitEmpty(t *testing.T) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment