Commit d0f94c53 authored by Tomasz Maczukin's avatar Tomasz Maczukin

Merge branch 'support-json-logging' into 'master'

Support json logging

Closes #2406 and #3336

See merge request gitlab-org/gitlab-runner!1020
parents f0df90c8 91fa16e1
......@@ -6,9 +6,9 @@ import (
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/cli"
"gitlab.com/gitlab-org/gitlab-runner/helpers/formatter"
"gitlab.com/gitlab-org/gitlab-runner/log"
_ "gitlab.com/gitlab-org/gitlab-runner/commands/helpers"
)
......@@ -24,9 +24,6 @@ func main() {
}
}()
formatter.SetRunnerFormatter()
cli_helpers.AddSecretsCleanupLogHook()
app := cli.NewApp()
app.Name = path.Base(os.Args[0])
app.Usage = "a GitLab Runner Helper"
......@@ -38,12 +35,14 @@ func main() {
Email: "support@gitlab.com",
},
}
cli_helpers.SetupLogLevelOptions(app)
app.Commands = common.GetCommands()
app.CommandNotFound = func(context *cli.Context, command string) {
logrus.Fatalln("Command", command, "not found")
}
log.AddSecretsCleanupLogHook(logrus.StandardLogger())
log.ConfigureLogging(app)
if err := app.Run(os.Args); err != nil {
logrus.Fatal(err)
}
......
......@@ -10,7 +10,7 @@ import (
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
"gitlab.com/gitlab-org/gitlab-runner/helpers/formatter"
"gitlab.com/gitlab-org/gitlab-runner/log"
"gitlab.com/gitlab-org/gitlab-runner/network"
)
......@@ -36,7 +36,7 @@ func (c *ArtifactsDownloaderCommand) download(file string) (bool, error) {
}
func (c *ArtifactsDownloaderCommand) Execute(context *cli.Context) {
formatter.SetRunnerFormatter()
log.SetRunnerFormatter()
if len(c.URL) == 0 || len(c.Token) == 0 {
logrus.Fatalln("Missing runner credentials")
......
......@@ -13,7 +13,7 @@ import (
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
"gitlab.com/gitlab-org/gitlab-runner/helpers/formatter"
"gitlab.com/gitlab-org/gitlab-runner/log"
"gitlab.com/gitlab-org/gitlab-runner/network"
)
......@@ -102,7 +102,7 @@ func (c *ArtifactsUploaderCommand) createAndUpload() (bool, error) {
}
func (c *ArtifactsUploaderCommand) Execute(*cli.Context) {
formatter.SetRunnerFormatter()
log.SetRunnerFormatter()
if len(c.URL) == 0 || len(c.Token) == 0 {
logrus.Fatalln("Missing runner credentials")
......
......@@ -13,6 +13,7 @@ import (
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
"gitlab.com/gitlab-org/gitlab-runner/helpers/url"
"gitlab.com/gitlab-org/gitlab-runner/log"
)
type CacheArchiverCommand struct {
......@@ -71,6 +72,8 @@ func (c *CacheArchiverCommand) upload() (bool, error) {
}
func (c *CacheArchiverCommand) Execute(*cli.Context) {
log.SetRunnerFormatter()
if c.File == "" {
logrus.Fatalln("Missing --file")
}
......
......@@ -14,8 +14,8 @@ import (
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
"gitlab.com/gitlab-org/gitlab-runner/helpers/formatter"
"gitlab.com/gitlab-org/gitlab-runner/helpers/url"
"gitlab.com/gitlab-org/gitlab-runner/log"
)
type CacheExtractorCommand struct {
......@@ -81,7 +81,7 @@ func (c *CacheExtractorCommand) download() (bool, error) {
}
func (c *CacheExtractorCommand) Execute(context *cli.Context) {
formatter.SetRunnerFormatter()
log.SetRunnerFormatter()
if len(c.File) == 0 {
logrus.Fatalln("Missing cache file")
......
......@@ -15,16 +15,16 @@ import (
"github.com/ayufan/golang-kardianos-service"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/helpers/certificate"
"gitlab.com/gitlab-org/gitlab-runner/helpers/cli"
prometheus_helper "gitlab.com/gitlab-org/gitlab-runner/helpers/prometheus"
"gitlab.com/gitlab-org/gitlab-runner/helpers/sentry"
"gitlab.com/gitlab-org/gitlab-runner/helpers/service"
"gitlab.com/gitlab-org/gitlab-runner/log"
"gitlab.com/gitlab-org/gitlab-runner/network"
"gitlab.com/gitlab-org/gitlab-runner/session"
)
......@@ -55,7 +55,7 @@ type RunCommand struct {
ServiceName string `short:"n" long:"service" description:"Use different names for different services"`
WorkingDirectory string `short:"d" long:"working-directory" description:"Specify custom working directory"`
User string `short:"u" long:"user" description:"Use specific user to execute shell scripts"`
Syslog bool `long:"syslog" description:"Log to syslog"`
Syslog bool `long:"syslog" description:"Log to system service logger" env:"LOG_SYSLOG"`
sentryLogHook sentry.LogHook
prometheusLogHook prometheus_helper.LogHook
......@@ -88,8 +88,8 @@ type RunCommand struct {
currentWorkers int
}
func (mr *RunCommand) log() *log.Entry {
return log.WithField("builds", mr.buildsHelper.buildsCount())
func (mr *RunCommand) log() *logrus.Entry {
return logrus.WithField("builds", mr.buildsHelper.buildsCount())
}
func (mr *RunCommand) feedRunner(runner *common.RunnerConfig, runners chan *common.RunnerConfig) {
......@@ -140,7 +140,7 @@ func (mr *RunCommand) processRunner(id int, runner *common.RunnerConfig, runners
context, err := provider.Acquire(runner)
if err != nil {
log.Warningln("Failed to update executor", runner.Executor, "for", runner.ShortDescription(), err)
logrus.Warningln("Failed to update executor", runner.Executor, "for", runner.ShortDescription(), err)
return
}
defer provider.Release(runner, context)
......@@ -254,12 +254,9 @@ func (mr *RunCommand) loadConfig() error {
}
// Set log level
if !cli_helpers.CustomLogLevelSet && mr.config.LogLevel != nil {
level, err := log.ParseLevel(*mr.config.LogLevel)
if err != nil {
log.Fatalf(err.Error())
}
log.SetLevel(level)
err = mr.updateLoggingConfiguration()
if err != nil {
return err
}
// pass user to execute scripts as specific user
......@@ -285,6 +282,34 @@ func (mr *RunCommand) loadConfig() error {
return nil
}
func (mr *RunCommand) updateLoggingConfiguration() error {
reloadNeeded := false
if mr.config.LogLevel != nil && !log.Configuration().IsLevelSetWithCli() {
err := log.Configuration().SetLevel(*mr.config.LogLevel)
if err != nil {
return err
}
reloadNeeded = true
}
if mr.config.LogFormat != nil && !log.Configuration().IsFormatSetWithCli() {
err := log.Configuration().SetFormat(*mr.config.LogFormat)
if err != nil {
return err
}
reloadNeeded = true
}
if reloadNeeded {
log.Configuration().ReloadConfiguration()
}
return nil
}
func (mr *RunCommand) checkConfig() (err error) {
info, err := os.Stat(mr.ConfigFile)
if err != nil {
......@@ -633,27 +658,21 @@ func (mr *RunCommand) Execute(context *cli.Context) {
},
}
service, err := service_helpers.New(mr, svcConfig)
svc, err := service_helpers.New(mr, svcConfig)
if err != nil {
log.Fatalln(err)
logrus.Fatalln(err)
}
if mr.Syslog {
log.SetFormatter(new(log.TextFormatter))
logger, err := service.SystemLogger(nil)
if err == nil {
log.AddHook(&ServiceLogHook{logger, log.InfoLevel})
} else {
log.Errorln(err)
}
log.SetSystemLogger(svc)
}
log.AddHook(&mr.sentryLogHook)
log.AddHook(&mr.prometheusLogHook)
logrus.AddHook(&mr.sentryLogHook)
logrus.AddHook(&mr.prometheusLogHook)
err = service.Run()
err = svc.Run()
if err != nil {
log.Fatalln(err)
logrus.Fatalln(err)
}
}
......
......@@ -8,6 +8,7 @@ import (
"github.com/ayufan/golang-kardianos-service"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/helpers/service"
......@@ -19,43 +20,6 @@ const (
defaultDescription = "GitLab Runner"
)
type ServiceLogHook struct {
service.Logger
Level logrus.Level
}
func (s *ServiceLogHook) Levels() []logrus.Level {
return []logrus.Level{
logrus.PanicLevel,
logrus.FatalLevel,
logrus.ErrorLevel,
logrus.WarnLevel,
logrus.InfoLevel,
}
}
func (s *ServiceLogHook) Fire(entry *logrus.Entry) error {
if entry.Level > s.Level {
return nil
}
msg, err := entry.String()
if err != nil {
return err
}
switch entry.Level {
case logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel:
s.Error(msg)
case logrus.WarnLevel:
s.Warning(msg)
case logrus.InfoLevel:
s.Info(msg)
}
return nil
}
type NullService struct {
}
......@@ -115,7 +79,11 @@ func getServiceArguments(c *cli.Context) (arguments []string) {
arguments = append(arguments, "--service", sn)
}
arguments = append(arguments, "--syslog")
syslog := !c.IsSet("syslog") || c.Bool("syslog")
if syslog {
arguments = append(arguments, "--syslog")
}
return
}
......@@ -183,16 +151,18 @@ func RunServiceControl(c *cli.Context) {
}
}
func init() {
flags := []cli.Flag{
func getFlags() []cli.Flag {
return []cli.Flag{
cli.StringFlag{
Name: "service, n",
Value: defaultServiceName,
Usage: "Specify service name to use",
},
}
}
installFlags := flags
func getInstallFlags() []cli.Flag {
installFlags := getFlags()
installFlags = append(installFlags, cli.StringFlag{
Name: "working-directory, d",
Value: helpers.GetCurrentWorkingDirectory(),
......@@ -203,6 +173,10 @@ func init() {
Value: getDefaultConfigFile(),
Usage: "Specify custom config file",
})
installFlags = append(installFlags, cli.BoolFlag{
Name: "syslog",
Usage: "Setup system logging integration",
})
if runtime.GOOS == "windows" {
installFlags = append(installFlags, cli.StringFlag{
......@@ -223,6 +197,13 @@ func init() {
})
}
return installFlags
}
func init() {
flags := getFlags()
installFlags := getInstallFlags()
common.RegisterCommand(cli.Command{
Name: "install",
Usage: "install service",
......
package commands
import (
"os"
"fmt"
"testing"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/helpers/service/mocks"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
)
func TestServiceLogHook(t *testing.T) {
formatter := new(logrus.TextFormatter)
formatter.DisableColors = true
formatter.DisableTimestamp = true
func newTestGetServiceArgumentsCommand(t *testing.T, expectedArgs []string) func(*cli.Context) {
return func(c *cli.Context) {
arguments := getServiceArguments(c)
logger := &logrus.Logger{
Formatter: formatter,
Hooks: make(logrus.LevelHooks),
Level: logrus.InfoLevel,
Out: os.Stderr,
for _, arg := range expectedArgs {
assert.Contains(t, arguments, arg)
}
}
}
mockServiceLogger := new(mocks.Logger)
mockServiceLogger.On("Info", "level=info msg=test\n").Return(nil)
func testServiceCommandRun(t *testing.T, command func(*cli.Context), args ...string) {
app := cli.NewApp()
app.Commands = []cli.Command{
{
Name: "test-command",
Action: command,
Flags: getInstallFlags(),
},
}
logHook := &ServiceLogHook{mockServiceLogger, logrus.InfoLevel}
args = append([]string{"binary", "test-command"}, args...)
app.Run(args)
}
logger.Hooks.Add(logHook)
type getServiceArgumentsTestCase struct {
cliFlags []string
expectedArgs []string
}
logger.Info("test")
func TestGetServiceArguments(t *testing.T) {
tests := []getServiceArgumentsTestCase{
{
expectedArgs: []string{
"--working-directory", helpers.GetCurrentWorkingDirectory(),
"--config", getDefaultConfigFile(),
"--service", "gitlab-runner",
"--syslog",
},
},
{
cliFlags: []string{
"--config", "/tmp/config.toml",
},
expectedArgs: []string{
"--working-directory", helpers.GetCurrentWorkingDirectory(),
"--config", "/tmp/config.toml",
"--service", "gitlab-runner",
"--syslog",
},
},
{
cliFlags: []string{
"--working-directory", "/tmp",
},
expectedArgs: []string{
"--working-directory", "/tmp",
"--config", getDefaultConfigFile(),
"--service", "gitlab-runner",
"--syslog",
},
},
{
cliFlags: []string{
"--service", "gitlab-runner-service-name",
},
expectedArgs: []string{
"--working-directory", helpers.GetCurrentWorkingDirectory(),
"--config", getDefaultConfigFile(),
"--service", "gitlab-runner-service-name",
"--syslog",
},
},
{
cliFlags: []string{
"--syslog=true",
},
expectedArgs: []string{
"--working-directory", helpers.GetCurrentWorkingDirectory(),
"--config", getDefaultConfigFile(),
"--service", "gitlab-runner",
"--syslog",
},
},
{
cliFlags: []string{
"--syslog=false",
},
expectedArgs: []string{
"--working-directory", helpers.GetCurrentWorkingDirectory(),
"--config", getDefaultConfigFile(),
"--service", "gitlab-runner",
},
},
}
mockServiceLogger.AssertExpectations(t)
for id, testCase := range tests {
t.Run(fmt.Sprintf("case-%d", id), func(t *testing.T) {
testServiceCommandRun(t, newTestGetServiceArgumentsCommand(t, testCase.expectedArgs), testCase.cliFlags...)
})
}
}
......@@ -307,6 +307,7 @@ type Config struct {
Concurrent int `toml:"concurrent" json:"concurrent"`
CheckInterval int `toml:"check_interval" json:"check_interval" description:"Define active checking interval of jobs"`
LogLevel *string `toml:"log_level" json:"log_level" description:"Define log level (one of: panic, fatal, error, warning, info, debug)"`
LogFormat *string `toml:"log_format" json:"log_format" description:"Define log format (one of: runner, text, json)"`
User string `toml:"user,omitempty" json:"user"`
Runners []*RunnerConfig `toml:"runners" json:"runners"`
SentryDSN *string `toml:"sentry_dsn"`
......
......@@ -306,10 +306,10 @@ service. Use them to install, uninstall, start and stop the runner service.
All service related commands accept these arguments:
| Parameter | Default | Description |
|-----------|---------|-------------|
| `--service` | `gitlab-runner` | Specify custom service name |
| `--config` | See the [configuration file](#configuration-file) | Specify a custom configuration file to use |
| Parameter | Default | Description |
|-------------|---------------------------------------------------|-------------|
| `--service` | `gitlab-runner` | Specify custom service name |
| `--config` | See the [configuration file](#configuration-file) | Specify a custom configuration file to use |
### gitlab-runner install
......@@ -319,12 +319,12 @@ arguments depending on which system it's run on.
When run on **Windows** or as super-user, it accepts the `--user` flag which
allows you to drop privileges of builds run with the **shell** executor.
| Parameter | Default | Description |
|-----------|---------|-------------|
| `--service` | `gitlab-runner` | Specify a custom name for the Runner |
| Parameter | Default | Description |
|-----------------------|-----------------------|-------------|
| `--syslog` | `true` | Specify if the service should integrate with system logging service |
| `--working-directory` | the current directory | Specify the root directory where all data will be stored when builds will be run with the **shell** executor |
| `--user` | `root` | Specify the user which will be used to execute builds |
| `--password` | none | Specify the password for the user that will be used to execute the builds |
| `--user` | `root` | Specify the user which will be used to execute builds |
| `--password` | none | Specify the password for the user that will be used to execute the builds |
### gitlab-runner uninstall
......
......@@ -17,7 +17,8 @@ This defines global settings of GitLab Runner.
| Setting | Description |
| ------- | ----------- |
| `concurrent` | limits how many jobs globally can be run concurrently. The most upper limit of jobs using all defined runners. `0` **does not** mean unlimited |
| `log_level` | Log level (options: debug, info, warn, error, fatal, panic). Note that this setting has lower priority than log-level set by command line argument --debug, -l or --log-level |
| `log_level` | Log level (options: debug, info, warn, error, fatal, panic). Note that this setting has lower priority than level set by command line argument `--debug`, `-l` or `--log-level` |
| `log_format` | Log format (options: runner, text, json). Note that this setting has lower priority than format set by command line argument `--log-format` |
| `check_interval` | defines the interval length, in seconds, between new jobs check. The default value is `3`; if set to `0` or lower, the default value will be used. |
| `sentry_dsn` | enable tracking of all system level errors to sentry |
| `listen_address` | address (`<host>:<port>`) on which the Prometheus metrics HTTP server should be listening |
......
// +build darwin dragonfly freebsd linux netbsd openbsd
package cli_helpers
import (
"os"
"os/signal"
"runtime"
"syscall"
"github.com/sirupsen/logrus"
)
func watchForGoroutinesDump() {
// On USR1 dump stacks of all go routines
dumpStacks := make(chan os.Signal, 1)
signal.Notify(dumpStacks, syscall.SIGUSR1)
for _ = range dumpStacks {
buf := make([]byte, 1<<20)
runtime.Stack(buf, true)
logrus.Printf("=== received SIGUSR1 ===\n*** goroutine dump...\n%s\n*** end\n", buf)
}
}
package cli_helpers
func watchForGoroutinesDump() {
}
package cli_helpers
import (
log "github.com/sirupsen/logrus"
"github.com/urfave/cli"
"os"
)
var (
DefaultLogLevel = log.InfoLevel
CustomLogLevelSet = false
)
func SetupLogLevelOptions(app *cli.App) {
newFlags := []cli.Flag{
cli.BoolFlag{
Name: "debug",
Usage: "debug mode",
EnvVar: "DEBUG",
},
cli.StringFlag{
Name: "log-level, l",
Usage: "Log level (options: debug, info, warn, error, fatal, panic)",
},
}
app.Flags = append(app.Flags, newFlags...)
appBefore := app.Before
// logs
app.Before = func(c *cli.Context) error {
log.SetOutput(os.Stderr)
log.SetLevel(DefaultLogLevel)
if c.IsSet("log-level") || c.IsSet("l") {
level, err := log.ParseLevel(c.String("log-level"))
if err != nil {
log.Fatalf(err.Error())
}
setCustomLevel(level)
} else if c.Bool("debug") {
setCustomLevel(log.DebugLevel)
go watchForGoroutinesDump()
}
if appBefore != nil {
return appBefore(c)
}
return nil
}
}
func setCustomLevel(level log.Level) {
log.SetLevel(level)
CustomLogLevelSet = true
}
package cli_helpers
import (
"os"
"runtime"
"gitlab.com/gitlab-org/gitlab-runner/common"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
func LogRuntimePlatform(app *cli.App) {
appBefore := app.Before
app.Before = func(c *cli.Context) error {
logrus.WithFields(logrus.Fields{
fields := logrus.Fields{
"os": runtime.GOOS,
"arch": runtime.GOARCH,
"version": common.VERSION,
"revision": common.REVISION,
}).Debugln("Runtime platform")
"pid": os.Getpid(),
}
logrus.WithFields(fields).Info("Runtime platform")
if appBefore != nil {
return appBefore(c)
......
package log
import (
"fmt"
"os"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
const (
FormatRunner = "runner"
FormatText = "text"
FormatJSON = "json"
)
var (
configuration = NewConfig(logrus.StandardLogger())
logFlags = []cli.Flag{
cli.BoolFlag{
Name: "debug",
Usage: "debug mode",
EnvVar: "DEBUG",
},
cli.StringFlag{
Name: "log-format",
Usage: "Choose log format (options: runner, text, json)",
EnvVar: "LOG_FORMAT",
},
cli.StringFlag{
Name: "log-level, l",
Usage: "Log level (options: debug, info, warn, error, fatal, panic)",
EnvVar: "LOG_LEVEL",
},
}
formats = map[string]logrus.Formatter{
FormatRunner: new(RunnerTextFormatter),
FormatText: new(logrus.TextFormatter),
FormatJSON: new(logrus.JSONFormatter),
}
)
func formatNames() []string {
formatNames := make([]string, 0)
for name := range formats {
formatNames = append(formatNames, name)
}
return formatNames
}
type Config struct {
logger *logrus.Logger
level logrus.Level
format logrus.Formatter
levelSetWithCli bool
formatSetWithCli bool
goroutinesDumpStopCh chan bool
}
func (l *Config) IsLevelSetWithCli() bool {
return l.levelSetWithCli
}
func (l *Config) IsFormatSetWithCli() bool {
return l.formatSetWithCli
}
func (l *Config) handleCliCtx(cliCtx *cli.Context) error {
if cliCtx.IsSet("log-level") || cliCtx.IsSet("l") {
err := l.SetLevel(cliCtx.String("log-level"))
if err != nil {
return err
}
l.levelSetWithCli = true
}