Commit a3e05df8 authored by Nick Thomas's avatar Nick Thomas 💃

Merge branch 'improvement/metrics/2333-add-executor-stage-to-builds-metric' into 'master'

Add executor stage to ci_runner_builds metric's labels

Closes #2333

See merge request !548
parents faffb538 9b1a7ca2
Pipeline #7707697 passed with stages
in 26 minutes and 9 seconds
......@@ -8,7 +8,21 @@ import (
"github.com/prometheus/client_golang/prometheus"
)
var numBuildsDesc = prometheus.NewDesc("ci_runner_builds", "The current number of running builds.", []string{"state", "stage"}, nil)
var numBuildsDesc = prometheus.NewDesc("ci_runner_builds", "The current number of running builds.", []string{"state", "stage", "executor_stage"}, nil)
type statePermutation struct {
buildState common.BuildRuntimeState
buildStage common.BuildStage
executorStage common.ExecutorStage
}
func newStatePermutationFromBuild(build *common.Build) statePermutation {
return statePermutation{
buildState: build.CurrentState,
buildStage: build.CurrentStage,
executorStage: build.CurrentExecutorStage(),
}
}
type runnerCounter struct {
builds int
......@@ -144,19 +158,18 @@ func (b *buildsHelper) buildsCount() int {
return len(b.builds)
}
func (b *buildsHelper) statesAndStages() map[common.BuildRuntimeState]map[common.BuildStage]int {
func (b *buildsHelper) statesAndStages() map[statePermutation]int {
b.lock.Lock()
defer b.lock.Unlock()
data := make(map[common.BuildRuntimeState]map[common.BuildStage]int)
data := make(map[statePermutation]int)
for _, build := range b.builds {
state := build.CurrentState
stage := build.CurrentStage
if data[state] == nil {
data[state] = make(map[common.BuildStage]int)
state := newStatePermutationFromBuild(build)
if _, ok := data[state]; ok {
data[state]++
} else {
data[state] = 1
}
data[state][stage]++
}
return data
}
......@@ -170,10 +183,14 @@ func (b *buildsHelper) Describe(ch chan<- *prometheus.Desc) {
func (b *buildsHelper) Collect(ch chan<- prometheus.Metric) {
data := b.statesAndStages()
for state, scripts := range data {
for stage, count := range scripts {
ch <- prometheus.MustNewConstMetric(numBuildsDesc, prometheus.GaugeValue, float64(count),
string(state), string(stage))
}
for state, count := range data {
ch <- prometheus.MustNewConstMetric(
numBuildsDesc,
prometheus.GaugeValue,
float64(count),
string(state.buildState),
string(state.buildStage),
string(state.executorStage),
)
}
}
......@@ -75,6 +75,8 @@ type Build struct {
CurrentStage BuildStage
CurrentState BuildRuntimeState
executorStageResolver func() ExecutorStage
}
func (b *Build) Log() *logrus.Entry {
......@@ -297,6 +299,8 @@ func (b *Build) retryCreateExecutor(globalConfig *Config, provider ExecutorProvi
return
}
b.executorStageResolver = executor.GetCurrentStage
err = executor.Prepare(globalConfig, b.Runner, b)
if err == nil {
break
......@@ -316,6 +320,16 @@ func (b *Build) retryCreateExecutor(globalConfig *Config, provider ExecutorProvi
return
}
func (b *Build) CurrentExecutorStage() ExecutorStage {
if b.executorStageResolver == nil {
b.executorStageResolver = func() ExecutorStage {
return ExecutorStage("")
}
}
return b.executorStageResolver()
}
func (b *Build) Run(globalConfig *Config, trace JobTrace) (err error) {
var executor Executor
......
......@@ -12,12 +12,23 @@ type ExecutorCommand struct {
Abort chan interface{}
}
type ExecutorStage string
const (
ExecutorStageCreated ExecutorStage = "created"
ExecutorStagePrepare ExecutorStage = "prepare"
ExecutorStageFinish ExecutorStage = "finish"
ExecutorStageCleanup ExecutorStage = "cleanup"
)
type Executor interface {
Shell() *ShellScriptInfo
Prepare(globalConfig *Config, config *RunnerConfig, build *Build) error
Run(cmd ExecutorCommand) error
Finish(err error)
Cleanup()
GetCurrentStage() ExecutorStage
SetCurrentStage(stage ExecutorStage)
}
type ExecutorProvider interface {
......
......@@ -36,3 +36,18 @@ func (m *MockExecutor) Finish(err error) {
func (m *MockExecutor) Cleanup() {
m.Called()
}
func (m *MockExecutor) GetCurrentStage() ExecutorStage {
ret := m.Called()
var r0 ExecutorStage
if ret.Get(0) != nil {
r0 = ret.Get(0).(ExecutorStage)
}
return r0
}
func (m *MockExecutor) SetCurrentStage(stage ExecutorStage) {
m.Called(stage)
}
......@@ -28,6 +28,17 @@ import (
"golang.org/x/net/context"
)
const (
DockerExecutorStagePrepare common.ExecutorStage = "docker_prepare"
DockerExecutorStageRun common.ExecutorStage = "docker_run"
DockerExecutorStageCleanup common.ExecutorStage = "docker_cleanup"
DockerExecutorStageCreatingBuildVolumes common.ExecutorStage = "docker_creating_build_volumes"
DockerExecutorStageCreatingServices common.ExecutorStage = "docker_creating_services"
DockerExecutorStageCreatingUserVolumes common.ExecutorStage = "docker_creating_user_volumes"
DockerExecutorStagePullingImage common.ExecutorStage = "docker_pulling_image"
)
var neverRestartPolicy = container.RestartPolicy{Name: "no"}
type dockerOptions struct {
......@@ -122,6 +133,7 @@ func (s *executor) getAuthConfig(imageName string) *types.AuthConfig {
}
func (s *executor) pullDockerImage(imageName string, ac *types.AuthConfig) (*types.ImageInspect, error) {
s.SetCurrentStage(DockerExecutorStagePullingImage)
s.Println("Pulling docker image", imageName, "...")
ref := imageName
......@@ -978,18 +990,21 @@ func (s *executor) createDependencies() (err error) {
return err
}
s.SetCurrentStage(DockerExecutorStageCreatingBuildVolumes)
s.Debugln("Creating build volume...")
err = s.createBuildVolume()
if err != nil {
return err
}
s.SetCurrentStage(DockerExecutorStageCreatingServices)
s.Debugln("Creating services...")
err = s.createServices()
if err != nil {
return err
}
s.SetCurrentStage(DockerExecutorStageCreatingUserVolumes)
s.Debugln("Creating user-defined volumes...")
err = s.createUserVolumes()
if err != nil {
......@@ -1018,6 +1033,7 @@ func (s *executor) Prepare(globalConfig *common.Config, config *common.RunnerCon
return errors.New("Missing docker configuration")
}
s.SetCurrentStage(DockerExecutorStagePrepare)
s.prepareOptions()
imageName, err := s.getImageName()
if err != nil {
......@@ -1063,8 +1079,9 @@ func (s *executor) prepareBuildsDir(config *common.RunnerConfig) error {
}
func (s *executor) Cleanup() {
var wg sync.WaitGroup
s.SetCurrentStage(DockerExecutorStageCleanup)
var wg sync.WaitGroup
remove := func(id string) {
wg.Add(1)
go func() {
......
......@@ -52,8 +52,9 @@ func (s *commandExecutor) Prepare(globalConfig *common.Config, config *common.Ru
}
func (s *commandExecutor) Run(cmd common.ExecutorCommand) error {
var runOn *types.ContainerJSON
s.SetCurrentStage(DockerExecutorStageRun)
var runOn *types.ContainerJSON
if cmd.Predefined {
runOn = s.predefinedContainer
} else {
......@@ -79,13 +80,15 @@ func init() {
}
creator := func() common.Executor {
return &commandExecutor{
e := &commandExecutor{
executor: executor{
AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: options,
},
},
}
e.SetCurrentStage(common.ExecutorStageCreated)
return e
}
featuresUpdater := func(features *common.FeaturesInfo) {
......
......@@ -68,6 +68,8 @@ func (s *sshExecutor) Prepare(globalConfig *common.Config, config *common.Runner
}
func (s *sshExecutor) Run(cmd common.ExecutorCommand) error {
s.SetCurrentStage(DockerExecutorStageRun)
err := s.sshCommand.Run(ssh.Command{
Environment: s.BuildShell.Environment,
Command: s.BuildShell.GetCommandWithArguments(),
......@@ -98,13 +100,15 @@ func init() {
}
creator := func() common.Executor {
return &sshExecutor{
e := &sshExecutor{
executor: executor{
AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: options,
},
},
}
e.SetCurrentStage(common.ExecutorStageCreated)
return e
}
featuresUpdater := func(features *common.FeaturesInfo) {
......
......@@ -12,12 +12,19 @@ import (
_ "gitlab.com/gitlab-org/gitlab-ci-multi-runner/executors/docker"
)
const (
DockerMachineExecutorStageUseMachine common.ExecutorStage = "docker_machine_use_machine"
DockerMachineExecutorStageReleaseMachine common.ExecutorStage = "docker_machine_release_machine"
)
type machineExecutor struct {
provider *machineProvider
executor common.Executor
build *common.Build
data common.ExecutorData
config common.RunnerConfig
currentStage common.ExecutorStage
}
func (e *machineExecutor) log() (log *logrus.Entry) {
......@@ -53,6 +60,7 @@ func (e *machineExecutor) Prepare(globalConfig *common.Config, config *common.Ru
e.build = build
// Use the machine
e.SetCurrentStage(DockerMachineExecutorStageUseMachine)
e.config, e.data, err = e.provider.Use(config, build.ExecutorData)
if err != nil {
return err
......@@ -98,11 +106,29 @@ func (e *machineExecutor) Cleanup() {
// Release allocated machine
if e.data != nil {
e.SetCurrentStage(DockerMachineExecutorStageReleaseMachine)
e.provider.Release(&e.config, e.data)
e.data = nil
}
}
func (e *machineExecutor) GetCurrentStage() common.ExecutorStage {
if e.executor == nil {
return common.ExecutorStage("")
}
return e.executor.GetCurrentStage()
}
func (e *machineExecutor) SetCurrentStage(stage common.ExecutorStage) {
if e.executor == nil {
e.currentStage = stage
return
}
e.executor.SetCurrentStage(stage)
}
func init() {
common.RegisterExecutor("docker+machine", newMachineProvider("docker_machines", "docker"))
common.RegisterExecutor("docker-ssh+machine", newMachineProvider("ssh_docker_machines", "docker-ssh"))
......
......@@ -21,6 +21,8 @@ type AbstractExecutor struct {
Build *common.Build
BuildTrace common.JobTrace
BuildShell *common.ShellConfiguration
currentStage common.ExecutorStage
}
func (e *AbstractExecutor) updateShell() error {
......@@ -70,6 +72,7 @@ func (e *AbstractExecutor) Shell() *common.ShellScriptInfo {
}
func (e *AbstractExecutor) Prepare(globalConfig *common.Config, config *common.RunnerConfig, build *common.Build) error {
e.currentStage = common.ExecutorStagePrepare
e.Config = *config
e.Build = build
e.BuildTrace = build.Trace
......@@ -93,7 +96,17 @@ func (e *AbstractExecutor) Prepare(globalConfig *common.Config, config *common.R
}
func (e *AbstractExecutor) Finish(err error) {
e.currentStage = common.ExecutorStageFinish
}
func (e *AbstractExecutor) Cleanup() {
e.currentStage = common.ExecutorStageCleanup
}
func (e *AbstractExecutor) GetCurrentStage() common.ExecutorStage {
return e.currentStage
}
func (e *AbstractExecutor) SetCurrentStage(stage common.ExecutorStage) {
e.currentStage = stage
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment