health_helper.go 1.32 KB
Newer Older
1 2 3 4 5 6
package commands

import (
	"sync"
	"time"

7
	"github.com/sirupsen/logrus"
8
	"gitlab.com/gitlab-org/gitlab-runner/common"
9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
)

type healthData struct {
	failures  int
	lastCheck time.Time
}

type healthHelper struct {
	healthy     map[string]*healthData
	healthyLock sync.Mutex
}

func (mr *healthHelper) getHealth(id string) *healthData {
	if mr.healthy == nil {
		mr.healthy = map[string]*healthData{}
	}
	health := mr.healthy[id]
	if health == nil {
		health = &healthData{
			lastCheck: time.Now(),
		}
		mr.healthy[id] = health
	}
	return health
}

func (mr *healthHelper) isHealthy(id string) bool {
36 37 38
	mr.healthyLock.Lock()
	defer mr.healthyLock.Unlock()

39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
	health := mr.getHealth(id)
	if health.failures < common.HealthyChecks {
		return true
	}

	if time.Since(health.lastCheck) > common.HealthCheckInterval*time.Second {
		logrus.Errorln("Runner", id, "is not healthy, but will be checked!")
		health.failures = 0
		health.lastCheck = time.Now()
		return true
	}

	return false
}

func (mr *healthHelper) makeHealthy(id string, healthy bool) {
55 56 57
	mr.healthyLock.Lock()
	defer mr.healthyLock.Unlock()

58 59 60 61 62 63 64 65 66 67 68
	health := mr.getHealth(id)
	if healthy {
		health.failures = 0
		health.lastCheck = time.Now()
	} else {
		health.failures++
		if health.failures >= common.HealthyChecks {
			logrus.Errorln("Runner", id, "is not healthy and will be disabled!")
		}
	}
}