builds_helper.go 7.85 KB
Newer Older
1 2 3
package commands

import (
4 5
	"fmt"
	"net/http"
6
	"regexp"
7
	"strings"
8
	"sync"
9

10
	"gitlab.com/gitlab-org/gitlab-runner/common"
11
	"gitlab.com/gitlab-org/gitlab-runner/helpers"
12
	"gitlab.com/gitlab-org/gitlab-runner/session"
13 14

	"github.com/prometheus/client_golang/prometheus"
15 16
)

17
var numBuildsDesc = prometheus.NewDesc(
18
	"gitlab_runner_jobs",
19 20 21 22
	"The current number of running builds.",
	[]string{"runner", "state", "stage", "executor_stage"},
	nil,
)
23

24
var requestConcurrencyDesc = prometheus.NewDesc(
25
	"gitlab_runner_request_concurrency",
26 27 28 29 30 31
	"The current number of concurrent requests for a new job",
	[]string{"runner"},
	nil,
)

var requestConcurrencyExceededDesc = prometheus.NewDesc(
32
	"gitlab_runner_request_concurrency_exceeded_total",
33 34 35 36 37
	"Counter tracking exceeding of request concurrency",
	[]string{"runner"},
	nil,
)

38
type statePermutation struct {
39
	runner        string
40 41 42 43 44 45 46
	buildState    common.BuildRuntimeState
	buildStage    common.BuildStage
	executorStage common.ExecutorStage
}

func newStatePermutationFromBuild(build *common.Build) statePermutation {
	return statePermutation{
47
		runner:        build.Runner.ShortDescription(),
48 49 50 51 52 53
		buildState:    build.CurrentState,
		buildStage:    build.CurrentStage,
		executorStage: build.CurrentExecutorStage(),
	}
}

54 55 56
type runnerCounter struct {
	builds   int
	requests int
57 58

	requestConcurrencyExceeded int
59 60
}

61
type buildsHelper struct {
62 63 64
	counters map[string]*runnerCounter
	builds   []*common.Build
	lock     sync.Mutex
65

66 67
	jobsTotal            *prometheus.CounterVec
	jobDurationHistogram *prometheus.HistogramVec
68 69 70 71 72 73 74 75 76 77 78 79 80
}

func (b *buildsHelper) getRunnerCounter(runner *common.RunnerConfig) *runnerCounter {
	if b.counters == nil {
		b.counters = make(map[string]*runnerCounter)
	}

	counter, _ := b.counters[runner.Token]
	if counter == nil {
		counter = &runnerCounter{}
		b.counters[runner.Token] = counter
	}
	return counter
81 82
}

83 84 85 86 87 88 89 90 91 92 93 94 95
func (b *buildsHelper) findSessionByURL(url string) *session.Session {
	b.lock.Lock()
	defer b.lock.Unlock()

	for _, build := range b.builds {
		if strings.HasPrefix(url, build.Session.Endpoint+"/") {
			return build.Session
		}
	}

	return nil
}

96
func (b *buildsHelper) acquireBuild(runner *common.RunnerConfig) bool {
Kamil Trzciński's avatar
Kamil Trzciński committed
97 98
	b.lock.Lock()
	defer b.lock.Unlock()
99

100 101 102
	counter := b.getRunnerCounter(runner)

	if runner.Limit > 0 && counter.builds >= runner.Limit {
103
		// Too many builds
Kamil Trzciński's avatar
Kamil Trzciński committed
104
		return false
105 106
	}

107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129
	counter.builds++
	return true
}

func (b *buildsHelper) releaseBuild(runner *common.RunnerConfig) bool {
	b.lock.Lock()
	defer b.lock.Unlock()

	counter := b.getRunnerCounter(runner)
	if counter.builds > 0 {
		counter.builds--
		return true
	}
	return false
}

func (b *buildsHelper) acquireRequest(runner *common.RunnerConfig) bool {
	b.lock.Lock()
	defer b.lock.Unlock()

	counter := b.getRunnerCounter(runner)

	if counter.requests >= runner.GetRequestConcurrency() {
130 131
		counter.requestConcurrencyExceeded++

132
		return false
133
	}
134 135

	counter.requests++
136
	return true
Kamil Trzciński's avatar
Kamil Trzciński committed
137 138
}

139
func (b *buildsHelper) releaseRequest(runner *common.RunnerConfig) bool {
Kamil Trzciński's avatar
Kamil Trzciński committed
140 141 142
	b.lock.Lock()
	defer b.lock.Unlock()

143 144 145
	counter := b.getRunnerCounter(runner)
	if counter.requests > 0 {
		counter.requests--
Kamil Trzciński's avatar
Kamil Trzciński committed
146 147 148 149 150 151
		return true
	}
	return false
}

func (b *buildsHelper) addBuild(build *common.Build) {
152 153 154 155
	if build == nil {
		return
	}

Kamil Trzciński's avatar
Kamil Trzciński committed
156 157 158 159 160 161 162 163 164 165 166 167
	b.lock.Lock()
	defer b.lock.Unlock()

	runners := make(map[int]bool)
	projectRunners := make(map[int]bool)

	for _, otherBuild := range b.builds {
		if otherBuild.Runner.Token != build.Runner.Token {
			continue
		}
		runners[otherBuild.RunnerID] = true

168
		if otherBuild.JobInfo.ProjectID != build.JobInfo.ProjectID {
Kamil Trzciński's avatar
Kamil Trzciński committed
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
			continue
		}
		projectRunners[otherBuild.ProjectRunnerID] = true
	}

	for {
		if !runners[build.RunnerID] {
			break
		}
		build.RunnerID++
	}

	for {
		if !projectRunners[build.ProjectRunnerID] {
			break
		}
		build.ProjectRunnerID++
	}

188
	b.builds = append(b.builds, build)
189 190
	b.jobsTotal.WithLabelValues(build.Runner.ShortDescription()).Inc()

191 192 193
	return
}

Kamil Trzciński's avatar
Kamil Trzciński committed
194 195 196
func (b *buildsHelper) removeBuild(deleteBuild *common.Build) bool {
	b.lock.Lock()
	defer b.lock.Unlock()
197

198 199
	b.jobDurationHistogram.WithLabelValues(deleteBuild.Runner.ShortDescription()).Observe(deleteBuild.Duration().Seconds())

200 201 202
	for idx, build := range b.builds {
		if build == deleteBuild {
			b.builds = append(b.builds[0:idx], b.builds[idx+1:]...)
203

204 205 206
			return true
		}
	}
207

208 209
	return false
}
210 211 212 213 214 215 216

func (b *buildsHelper) buildsCount() int {
	b.lock.Lock()
	defer b.lock.Unlock()

	return len(b.builds)
}
217

218
func (b *buildsHelper) statesAndStages() map[statePermutation]int {
219 220 221
	b.lock.Lock()
	defer b.lock.Unlock()

222
	data := make(map[statePermutation]int)
223
	for _, build := range b.builds {
224 225 226 227 228
		state := newStatePermutationFromBuild(build)
		if _, ok := data[state]; ok {
			data[state]++
		} else {
			data[state] = 1
229 230 231 232 233
		}
	}
	return data
}

234 235 236 237 238 239 240 241 242 243 244 245
func (b *buildsHelper) runnersCounters() map[string]*runnerCounter {
	b.lock.Lock()
	defer b.lock.Unlock()

	data := make(map[string]*runnerCounter)
	for token, counter := range b.counters {
		data[helpers.ShortenToken(token)] = counter
	}

	return data
}

246 247 248
// Describe implements prometheus.Collector.
func (b *buildsHelper) Describe(ch chan<- *prometheus.Desc) {
	ch <- numBuildsDesc
249 250
	ch <- requestConcurrencyDesc
	ch <- requestConcurrencyExceededDesc
251 252

	b.jobsTotal.Describe(ch)
253
	b.jobDurationHistogram.Describe(ch)
254 255 256 257
}

// Collect implements prometheus.Collector.
func (b *buildsHelper) Collect(ch chan<- prometheus.Metric) {
258 259
	builds := b.statesAndStages()
	for state, count := range builds {
260 261 262 263
		ch <- prometheus.MustNewConstMetric(
			numBuildsDesc,
			prometheus.GaugeValue,
			float64(count),
264
			state.runner,
265 266 267 268
			string(state.buildState),
			string(state.buildStage),
			string(state.executorStage),
		)
269
	}
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286

	counters := b.runnersCounters()
	for runner, counter := range counters {
		ch <- prometheus.MustNewConstMetric(
			requestConcurrencyDesc,
			prometheus.GaugeValue,
			float64(counter.requests),
			runner,
		)

		ch <- prometheus.MustNewConstMetric(
			requestConcurrencyExceededDesc,
			prometheus.CounterValue,
			float64(counter.requestConcurrencyExceeded),
			runner,
		)
	}
287 288

	b.jobsTotal.Collect(ch)
289
	b.jobDurationHistogram.Collect(ch)
290
}
291

292 293 294 295 296
func (b *buildsHelper) ListJobsHandler(w http.ResponseWriter, r *http.Request) {
	version := r.URL.Query().Get("v")
	if version == "" {
		version = "1"
	}
297

298 299 300 301
	handlers := map[string]http.HandlerFunc{
		"1": b.listJobsHandlerV1,
		"2": b.listJobsHandlerV2,
	}
302

303 304 305 306 307 308 309 310
	handler, ok := handlers[version]
	if !ok {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Request version %q not supported", version)
		return
	}

	w.Header().Add("X-List-Version", version)
311
	w.Header().Add("Content-Type", "text/plain")
Tomasz Maczukin's avatar
Tomasz Maczukin committed
312
	w.WriteHeader(http.StatusOK)
313

314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
	handler(w, r)
}

func (b *buildsHelper) listJobsHandlerV1(w http.ResponseWriter, r *http.Request) {
	for _, job := range b.builds {
		fmt.Fprintf(
			w,
			"id=%d url=%s state=%s stage=%s executor_stage=%s\n",
			job.ID, job.RepoCleanURL(),
			job.CurrentState, job.CurrentStage, job.CurrentExecutorStage(),
		)
	}

}

func (b *buildsHelper) listJobsHandlerV2(w http.ResponseWriter, r *http.Request) {
330
	for _, job := range b.builds {
331 332
		url := CreateJobURL(job.RepoCleanURL(), job.ID)

333 334
		fmt.Fprintf(
			w,
Tomasz Maczukin's avatar
Tomasz Maczukin committed
335 336
			"url=%s state=%s stage=%s executor_stage=%s duration=%s\n",
			url, job.CurrentState, job.CurrentStage, job.CurrentExecutorStage(), job.Duration(),
337 338 339
		)
	}
}
340 341 342 343 344 345 346

func CreateJobURL(projectURL string, jobID int) string {
	r := regexp.MustCompile("(\\.git$)?")
	URL := r.ReplaceAllString(projectURL, "")

	return fmt.Sprintf("%s/-/jobs/%d", URL, jobID)
}
347 348 349 350 351 352 353 354 355 356

func newBuildsHelper() buildsHelper {
	return buildsHelper{
		jobsTotal: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Name: "gitlab_runner_jobs_total",
				Help: "Total number of handled jobs",
			},
			[]string{"runner"},
		),
357 358 359 360 361 362 363 364
		jobDurationHistogram: prometheus.NewHistogramVec(
			prometheus.HistogramOpts{
				Name:    "gitlab_runner_job_duration_seconds",
				Help:    "Histogram of job durations",
				Buckets: []float64{30, 60, 300, 600, 1800, 3600, 7200, 10800, 18000, 36000},
			},
			[]string{"runner"},
		),
365 366
	}
}