Commit 784b636c authored by Tomasz Maczukin's avatar Tomasz Maczukin

Add executor stage to ci_runner_builds metric's labels

parent f9060a58
......@@ -8,7 +8,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
)
var numBuildsDesc = prometheus.NewDesc("ci_runner_builds", "The current number of running builds.", []string{"state", "stage"}, nil)
var numBuildsDesc = prometheus.NewDesc("ci_runner_builds", "The current number of running builds.", []string{"state", "stage", "executor_stage"}, nil)
type runnerCounter struct {
builds int
......@@ -144,19 +144,23 @@ func (b *buildsHelper) buildsCount() int {
return len(b.builds)
}
func (b *buildsHelper) statesAndStages() map[common.BuildRuntimeState]map[common.BuildStage]int {
func (b *buildsHelper) statesAndStages() map[common.BuildRuntimeState]map[common.BuildStage]map[common.ExecutorStage]int {
b.lock.Lock()
defer b.lock.Unlock()
data := make(map[common.BuildRuntimeState]map[common.BuildStage]int)
data := make(map[common.BuildRuntimeState]map[common.BuildStage]map[common.ExecutorStage]int)
for _, build := range b.builds {
state := build.CurrentState
stage := build.CurrentStage
executorStage := build.CurrentExecutorStage()
if data[state] == nil {
data[state] = make(map[common.BuildStage]int)
data[state] = make(map[common.BuildStage]map[common.ExecutorStage]int)
}
data[state][stage]++
if data[state][stage] == nil {
data[state][stage] = make(map[common.ExecutorStage]int)
}
data[state][stage][executorStage]++
}
return data
}
......@@ -171,9 +175,11 @@ func (b *buildsHelper) Collect(ch chan<- prometheus.Metric) {
data := b.statesAndStages()
for state, scripts := range data {
for stage, count := range scripts {
for stage, executor_stages := range scripts {
for executor_stage, count := range executor_stages {
ch <- prometheus.MustNewConstMetric(numBuildsDesc, prometheus.GaugeValue, float64(count),
string(state), string(stage))
string(state), string(stage), string(executor_stage))
}
}
}
}
......@@ -75,6 +75,8 @@ type Build struct {
CurrentStage BuildStage
CurrentState BuildRuntimeState
executorStageResolver func() ExecutorStage
}
func (b *Build) Log() *logrus.Entry {
......@@ -297,6 +299,8 @@ func (b *Build) retryCreateExecutor(globalConfig *Config, provider ExecutorProvi
return
}
b.executorStageResolver = executor.GetCurrentStage
err = executor.Prepare(globalConfig, b.Runner, b)
if err == nil {
break
......@@ -316,6 +320,16 @@ func (b *Build) retryCreateExecutor(globalConfig *Config, provider ExecutorProvi
return
}
func (b *Build) CurrentExecutorStage() ExecutorStage {
if b.executorStageResolver == nil {
b.executorStageResolver = func() ExecutorStage {
return ExecutorStage("")
}
}
return b.executorStageResolver()
}
func (b *Build) Run(globalConfig *Config, trace JobTrace) (err error) {
var executor Executor
......
......@@ -12,12 +12,23 @@ type ExecutorCommand struct {
Abort chan interface{}
}
type ExecutorStage string
const (
ExecutorStageCreated ExecutorStage = "created"
ExecutorStagePrepare ExecutorStage = "prepare"
ExecutorStageFinish ExecutorStage = "finish"
ExecutorStageCleanup ExecutorStage = "cleanup"
)
type Executor interface {
Shell() *ShellScriptInfo
Prepare(globalConfig *Config, config *RunnerConfig, build *Build) error
Run(cmd ExecutorCommand) error
Finish(err error)
Cleanup()
GetCurrentStage() ExecutorStage
SetCurrentStage(stage ExecutorStage)
}
type ExecutorProvider interface {
......
......@@ -28,6 +28,17 @@ import (
"golang.org/x/net/context"
)
const (
DockerExecutorStagePrepare common.ExecutorStage = "docker_prepare"
DockerExecutorStageRun common.ExecutorStage = "docker_run"
DockerExecutorStageCleanup common.ExecutorStage = "docker_cleanup"
DockerExecutorStageCreatingBuildVolumes common.ExecutorStage = "docker_creating_build_volumes"
DockerExecutorStageCreatingServices common.ExecutorStage = "docker_creating_services"
DockerExecutorStageCreatingUserVolumes common.ExecutorStage = "docker_creating_user_volumes"
DockerExecutorStagePullingImage common.ExecutorStage = "docker_pulling_image"
)
var neverRestartPolicy = container.RestartPolicy{Name: "no"}
type dockerOptions struct {
......@@ -122,6 +133,7 @@ func (s *executor) getAuthConfig(imageName string) *types.AuthConfig {
}
func (s *executor) pullDockerImage(imageName string, ac *types.AuthConfig) (*types.ImageInspect, error) {
s.SetCurrentStage(DockerExecutorStagePullingImage)
s.Println("Pulling docker image", imageName, "...")
ref := imageName
......@@ -976,18 +988,21 @@ func (s *executor) createDependencies() (err error) {
return err
}
s.SetCurrentStage(DockerExecutorStageCreatingBuildVolumes)
s.Debugln("Creating build volume...")
err = s.createBuildVolume()
if err != nil {
return err
}
s.SetCurrentStage(DockerExecutorStageCreatingServices)
s.Debugln("Creating services...")
err = s.createServices()
if err != nil {
return err
}
s.SetCurrentStage(DockerExecutorStageCreatingUserVolumes)
s.Debugln("Creating user-defined volumes...")
err = s.createUserVolumes()
if err != nil {
......@@ -1016,6 +1031,7 @@ func (s *executor) Prepare(globalConfig *common.Config, config *common.RunnerCon
return errors.New("Missing docker configuration")
}
s.SetCurrentStage(DockerExecutorStagePrepare)
s.prepareOptions()
imageName, err := s.getImageName()
if err != nil {
......@@ -1061,8 +1077,9 @@ func (s *executor) prepareBuildsDir(config *common.RunnerConfig) error {
}
func (s *executor) Cleanup() {
var wg sync.WaitGroup
s.SetCurrentStage(DockerExecutorStageCleanup)
var wg sync.WaitGroup
remove := func(id string) {
wg.Add(1)
go func() {
......
......@@ -52,8 +52,9 @@ func (s *commandExecutor) Prepare(globalConfig *common.Config, config *common.Ru
}
func (s *commandExecutor) Run(cmd common.ExecutorCommand) error {
var runOn *types.ContainerJSON
s.SetCurrentStage(DockerExecutorStageRun)
var runOn *types.ContainerJSON
if cmd.Predefined {
runOn = s.predefinedContainer
} else {
......@@ -79,13 +80,15 @@ func init() {
}
creator := func() common.Executor {
return &commandExecutor{
e := &commandExecutor{
executor: executor{
AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: options,
},
},
}
e.SetCurrentStage(common.ExecutorStageCreated)
return e
}
featuresUpdater := func(features *common.FeaturesInfo) {
......
......@@ -68,6 +68,8 @@ func (s *sshExecutor) Prepare(globalConfig *common.Config, config *common.Runner
}
func (s *sshExecutor) Run(cmd common.ExecutorCommand) error {
s.SetCurrentStage(DockerExecutorStageRun)
err := s.sshCommand.Run(ssh.Command{
Environment: s.BuildShell.Environment,
Command: s.BuildShell.GetCommandWithArguments(),
......@@ -98,13 +100,15 @@ func init() {
}
creator := func() common.Executor {
return &sshExecutor{
e := &sshExecutor{
executor: executor{
AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: options,
},
},
}
e.SetCurrentStage(common.ExecutorStageCreated)
return e
}
featuresUpdater := func(features *common.FeaturesInfo) {
......
......@@ -12,12 +12,19 @@ import (
_ "gitlab.com/gitlab-org/gitlab-ci-multi-runner/executors/docker"
)
const (
DockerMachineExecutorStageUseMachine common.ExecutorStage = "docker_machine_use_machine"
DockerMachineExecutorStageReleaseMachine common.ExecutorStage = "docker_machine_release_machine"
)
type machineExecutor struct {
provider *machineProvider
executor common.Executor
build *common.Build
data common.ExecutorData
config common.RunnerConfig
currentStage common.ExecutorStage
}
func (e *machineExecutor) log() (log *logrus.Entry) {
......@@ -53,6 +60,7 @@ func (e *machineExecutor) Prepare(globalConfig *common.Config, config *common.Ru
e.build = build
// Use the machine
e.SetCurrentStage(DockerMachineExecutorStageUseMachine)
e.config, e.data, err = e.provider.Use(config, build.ExecutorData)
if err != nil {
return err
......@@ -98,11 +106,29 @@ func (e *machineExecutor) Cleanup() {
// Release allocated machine
if e.data != nil {
e.SetCurrentStage(DockerMachineExecutorStageReleaseMachine)
e.provider.Release(&e.config, e.data)
e.data = nil
}
}
func (e *machineExecutor) GetCurrentStage() common.ExecutorStage {
if e.executor == nil {
return common.ExecutorStage("")
}
return e.executor.GetCurrentStage()
}
func (e *machineExecutor) SetCurrentStage(stage common.ExecutorStage) {
if e.executor == nil {
e.currentStage = stage
return
}
e.executor.SetCurrentStage(stage)
}
func init() {
common.RegisterExecutor("docker+machine", newMachineProvider("docker_machines", "docker"))
common.RegisterExecutor("docker-ssh+machine", newMachineProvider("ssh_docker_machines", "docker-ssh"))
......
......@@ -21,6 +21,8 @@ type AbstractExecutor struct {
Build *common.Build
BuildTrace common.JobTrace
BuildShell *common.ShellConfiguration
currentStage common.ExecutorStage
}
func (e *AbstractExecutor) updateShell() error {
......@@ -70,6 +72,7 @@ func (e *AbstractExecutor) Shell() *common.ShellScriptInfo {
}
func (e *AbstractExecutor) Prepare(globalConfig *common.Config, config *common.RunnerConfig, build *common.Build) error {
e.currentStage = common.ExecutorStagePrepare
e.Config = *config
e.Build = build
e.BuildTrace = build.Trace
......@@ -93,7 +96,17 @@ func (e *AbstractExecutor) Prepare(globalConfig *common.Config, config *common.R
}
func (e *AbstractExecutor) Finish(err error) {
e.currentStage = common.ExecutorStageFinish
}
func (e *AbstractExecutor) Cleanup() {
e.currentStage = common.ExecutorStageCleanup
}
func (e *AbstractExecutor) GetCurrentStage() common.ExecutorStage {
return e.currentStage
}
func (e *AbstractExecutor) SetCurrentStage(stage common.ExecutorStage) {
e.currentStage = stage
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment