gitlab-runner that recieves a SIGQUIT will crash during artifact upload
Summary
A gitlab runner can receive a SIGQUIT which indicates that it should shutdown when it's no longer processing a job. This graceful shutdown is documented here: https://docs.gitlab.com/runner/commands/
However, when a SIGQUIT is received during artifact upload, the runner will core.
Steps to reproduce
Monitor the CI job and send a SIGQUIT to the runner process: pkill -QUIT gitlab-runner
when the runner is uploading artifacts.
This is easier to reproduce if the artifact is very large in order to have the time to manually send a SIGQUIT.
What is the current bug behavior?
Core.
What is the expected correct behavior?
No core. gitlab-runner should finish uploading artifacts then shutdown.
Relevant logs and/or screenshots
Example output in job logs:
Uploading artifacts...
*.afi: found 1 matching files
*.vm.tar.gz: found 1 matching files
SIGQUIT: quit
PC=0x4579c1 m=0 sigcode=0
goroutine 0 [idle]:
runtime.futex(0x1166050, 0x0, 0x0, 0x0, 0x0, 0x2b452b50, 0x0, 0x0, 0x7ffe2b452b60, 0x40d8cb, ...)
/usr/local/go/src/runtime/sys_linux_amd64.s:425 +0x21
runtime.futexsleep(0x1166050, 0x0, 0xffffffffffffffff)
/usr/local/go/src/runtime/os_linux.go:45 +0x62
runtime.notesleep(0x1166050)
/usr/local/go/src/runtime/lock_futex.go:145 +0x6b
runtime.stopm()
/usr/local/go/src/runtime/proc.go:1650 +0xad
runtime.findrunnable(0xc420024000, 0x0)
/usr/local/go/src/runtime/proc.go:2102 +0x2e4
runtime.schedule()
/usr/local/go/src/runtime/proc.go:2222 +0x14c
runtime.park_m(0xc4203ced00)
/usr/local/go/src/runtime/proc.go:2285 +0xab
runtime.mcall(0x7ffe2b452cf0)
/usr/local/go/src/runtime/asm_amd64.s:269 +0x5b
goroutine 1 [select]:
net/http.(*persistConn).roundTrip(0xc42013a000, 0xc4205b51a0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1898 +0x974
net/http.(*Transport).RoundTrip(0xc4201261e0, 0xc420098100, 0xc4201261e0, 0xed4679af2, 0x23038562)
/usr/local/go/src/net/http/transport.go:391 +0x74c
net/http.send(0xc4203a2700, 0x110edc0, 0xc4201261e0, 0xed4679af2, 0x23038562, 0x11651e0, 0x0, 0x8, 0xc42057e030, 0x40f4d8)
/usr/local/go/src/net/http/client.go:249 +0x162
net/http.(*Client).send(0xc420096c00, 0xc4203a2700, 0xed4679af2, 0x23038562, 0x11651e0, 0xc42057e030, 0x0, 0x1, 0x0)
/usr/local/go/src/net/http/client.go:173 +0x108
net/http.(*Client).Do(0xc420096c00, 0xc4203a2700, 0x0, 0xc420229200, 0xb12e92)
/usr/local/go/src/net/http/client.go:595 +0x254
gitlab.com/gitlab-org/gitlab-runner/network.(*client).doBackoffRequest(0xc420096c00, 0xc4203a2700, 0xa, 0xc4203e4140, 0x3a)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/network/client.go:232 +0x4f
gitlab.com/gitlab-org/gitlab-runner/network.(*client).do(0xc420096c00, 0xc4203b1c20, 0x4b, 0xcb8bd1, 0x4, 0x110e8c0, 0xc42000c600, 0xc4203ac9c0, 0x5a, 0xc4203d2330, ...)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/network/client.go:271 +0x1b3
gitlab.com/gitlab-org/gitlab-runner/network.(*GitLabClient).doRaw(0xc4203be1a0, 0x111b180, 0xc4203ac960, 0xcb8bd1, 0x4, 0xc4203b1c20, 0x4b, 0x110e8c0, 0xc42000c600, 0xc4203ac9c0, ...)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/network/gitlab.go:159 +0x101
gitlab.com/gitlab-org/gitlab-runner/network.(*GitLabClient).UploadRawArtifacts(0xc4203be1a0, 0x121dd, 0x7ffe2b453296, 0x14, 0x7ffe2b453273, 0x1a, 0xc420012106, 0x2c, 0x0, 0x0, ...)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/network/gitlab.go:452 +0x54c
gitlab.com/gitlab-org/gitlab-runner/commands/helpers.(*ArtifactsUploaderCommand).createAndUpload(0xc420126960, 0x0, 0x0)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/commands/helpers/artifacts_uploader.go:107 +0x25c
gitlab.com/gitlab-org/gitlab-runner/commands/helpers.(*ArtifactsUploaderCommand).(gitlab.com/gitlab-org/gitlab-runner/commands/helpers.createAndUpload)-fm(0xc42000a284, 0xc420261880)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/commands/helpers/artifacts_uploader.go:138 +0x2a
gitlab.com/gitlab-org/gitlab-runner/commands/helpers.(*retryHelper).doRetry(0xc4201269f0, 0xc4202298e0, 0x0, 0x5)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/commands/helpers/retry_helper.go:29 +0x2b
gitlab.com/gitlab-org/gitlab-runner/commands/helpers.(*ArtifactsUploaderCommand).Execute(0xc420126960, 0xc4203d4000)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/commands/helpers/artifacts_uploader.go:138 +0x16b
gitlab.com/gitlab-org/gitlab-runner/common.(Commander).Execute-fm(0xc4203d4000)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/common/command.go:24 +0x39
gitlab.com/gitlab-org/gitlab-runner/vendor/github.com/urfave/cli.HandleAction(0xb980c0, 0xc4203be1c0, 0xc4203d4000, 0xc4203ac700, 0x0)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/vendor/github.com/urfave/cli/app.go:492 +0x7c
gitlab.com/gitlab-org/gitlab-runner/vendor/github.com/urfave/cli.Command.Run(0xcc22b3, 0x12, 0x0, 0x0, 0x0, 0x0, 0x0, 0xcd5e29, 0x2c, 0x0, ...)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/vendor/github.com/urfave/cli/command.go:210 +0xb6e
gitlab.com/gitlab-org/gitlab-runner/vendor/github.com/urfave/cli.(*App).Run(0xc4200c3d40, 0xc42000e120, 0x12, 0x12, 0x0, 0x0)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/vendor/github.com/urfave/cli/app.go:255 +0x76f
main.main()
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/apps/gitlab-runner-helper/main.go:46 +0x298
goroutine 5 [chan receive]:
gitlab.com/gitlab-org/gitlab-runner/vendor/github.com/golang/glog.(*loggingT).flushDaemon(0x11654e0)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/vendor/github.com/golang/glog/glog.go:882 +0x7a
created by gitlab.com/gitlab-org/gitlab-runner/vendor/github.com/golang/glog.init.1
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/vendor/github.com/golang/glog/glog.go:410 +0x21d
goroutine 8 [syscall]:
os/signal.signal_recv(0x430e98)
/usr/local/go/src/runtime/sigqueue.go:116 +0x104
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.1
/usr/local/go/src/os/signal/signal_unix.go:28 +0x41
goroutine 9 [semacquire]:
sync.runtime_notifyListWait(0xc420096af8, 0xc400004ef4)
/usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420096ae8)
/usr/local/go/src/sync/cond.go:57 +0x89
io.(*pipe).write(0xc420096a80, 0xc4203f4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/io/pipe.go:90 +0x189
io.(*PipeWriter).Write(0xc42000c5f8, 0xc4203f4000, 0x1000, 0x1000, 0x0, 0x0, 0x2)
/usr/local/go/src/io/pipe.go:157 +0x4c
bufio.(*Writer).Flush(0xc42012e040, 0xc4205641c1, 0xffb)
/usr/local/go/src/bufio/bufio.go:566 +0x75
bufio.(*Writer).Write(0xc42012e040, 0xc4205641c1, 0x4000, 0x7e3f, 0x4, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:602 +0xdf
archive/zip.(*countWriter).Write(0xc42012c060, 0xc4205641c1, 0x4000, 0x7e3f, 0x4, 0x0, 0x0)
/usr/local/go/src/archive/zip/writer.go:373 +0x51
archive/zip.(*countWriter).Write(0xc42012c080, 0xc4205641c1, 0x4000, 0x7e3f, 0x4, 0x0, 0x0)
/usr/local/go/src/archive/zip/writer.go:373 +0x51
compress/flate.(*huffmanBitWriter).write(0xc42055a000, 0xc4205641c1, 0x4000, 0x7e3f)
/usr/local/go/src/compress/flate/huffman_bit_writer.go:144 +0x61
compress/flate.(*huffmanBitWriter).writeBytes(0xc42055a000, 0xc4205641c1, 0x4000, 0x7e3f)
/usr/local/go/src/compress/flate/huffman_bit_writer.go:193 +0xf7
compress/flate.(*huffmanBitWriter).writeBlock(0xc42055a000, 0xc42056c000, 0x4001, 0x4001, 0x0, 0xc4205641c1, 0x4000, 0x7e3f)
/usr/local/go/src/compress/flate/huffman_bit_writer.go:491 +0x44d
compress/flate.(*compressor).writeBlock(0xc420472000, 0xc42056c000, 0x4000, 0x4001, 0xc1c1, 0x3, 0x0)
/usr/local/go/src/compress/flate/deflate.go:170 +0xc4
compress/flate.(*compressor).deflate(0xc420472000)
/usr/local/go/src/compress/flate/deflate.go:506 +0x7ff
compress/flate.(*compressor).write(0xc420472000, 0xc420580000, 0x8000, 0x8000, 0xc420044c00, 0xc420044bf8, 0xc420464140)
/usr/local/go/src/compress/flate/deflate.go:551 +0x76
compress/flate.(*Writer).Write(0xc420472000, 0xc420580000, 0x8000, 0x8000, 0xa31efb76, 0xc420044c68, 0x8000)
/usr/local/go/src/compress/flate/deflate.go:709 +0x4b
archive/zip.(*pooledFlateWriter).Write(0xc42045e090, 0xc420580000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/archive/zip/register.go:52 +0x13b
archive/zip.(*countWriter).Write(0xc420462060, 0xc420580000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/usr/local/go/src/archive/zip/writer.go:373 +0x51
archive/zip.(*fileWriter).Write(0xc4201360a0, 0xc420580000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/usr/local/go/src/archive/zip/writer.go:315 +0x107
io.copyBuffer(0x110db40, 0xc4201360a0, 0x110f1c0, 0xc42057e008, 0xc420580000, 0x8000, 0x8000, 0xc42057e008, 0x0, 0x0)
/usr/local/go/src/io/io.go:392 +0x20b
io.Copy(0x110db40, 0xc4201360a0, 0x110f1c0, 0xc42057e008, 0x0, 0x0, 0xc420450070)
/usr/local/go/src/io/io.go:360 +0x68
gitlab.com/gitlab-org/gitlab-runner/helpers/archives.createZipFileEntry(0xc42012e080, 0xc420450000, 0xc420450090, 0x18)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/helpers/archives/zip_create.go:46 +0xc6
gitlab.com/gitlab-org/gitlab-runner/helpers/archives.createZipEntry(0xc42012e080, 0xc4203b1b93, 0x36, 0xc4203f4000, 0x1000)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/helpers/archives/zip_create.go:81 +0x36a
gitlab.com/gitlab-org/gitlab-runner/helpers/archives.CreateZipArchive(0x110e900, 0xc42000c5f8, 0xc42012c020, 0x2, 0x2, 0x0, 0x0)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/helpers/archives/zip_create.go:96 +0x251
gitlab.com/gitlab-org/gitlab-runner/commands/helpers.(*ArtifactsUploaderCommand).generateZipArchive(0xc420126960, 0xc42000c5f8)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/commands/helpers/artifacts_uploader.go:35 +0x6d
created by gitlab.com/gitlab-org/gitlab-runner/commands/helpers.(*ArtifactsUploaderCommand).createReadStream
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/commands/helpers/artifacts_uploader.go:66 +0x1bb
goroutine 10 [runnable]:
sync.runtime_notifyListWait(0xc420096ac0, 0xc40000351e)
/usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420096ab0)
/usr/local/go/src/sync/cond.go:57 +0x89
io.(*pipe).read(0xc420096a80, 0xc42063e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/io/pipe.go:47 +0x104
io.(*PipeReader).Read(0xc42000c5f0, 0xc42063e000, 0x8000, 0x8000, 0x3005, 0x0, 0x0)
/usr/local/go/src/io/pipe.go:130 +0x4c
io.copyBuffer(0x110ea00, 0xc42059a1e0, 0x110e8c0, 0xc42000c5f0, 0xc42063e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/io/io.go:390 +0x100
io.Copy(0x110ea00, 0xc42059a1e0, 0x110e8c0, 0xc42000c5f0, 0xd, 0x110ea00, 0xc42059a1e0)
/usr/local/go/src/io/io.go:360 +0x68
gitlab.com/gitlab-org/gitlab-runner/network.(*GitLabClient).createArtifactsForm(0xc4203be1a0, 0xc4203d22d0, 0x110e8c0, 0xc42000c5f0, 0xc4203d6250, 0xd, 0x0, 0x0)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/network/gitlab.go:408 +0x96
gitlab.com/gitlab-org/gitlab-runner/network.(*GitLabClient).UploadRawArtifacts.func1(0xc42000c608, 0xc4203d22d0, 0xc4203be1a0, 0x110e8c0, 0xc42000c5f0, 0xc4203d6250, 0xd, 0x7ffe2b4532e2, 0x2, 0x7ffe2b4532f7, ...)
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/network/gitlab.go:442 +0xb4
created by gitlab.com/gitlab-org/gitlab-runner/network.(*GitLabClient).UploadRawArtifacts
/builds/gitlab-org/gitlab-runner/.gopath/src/gitlab.com/gitlab-org/gitlab-runner/network/gitlab.go:446 +0x22b
goroutine 35 [select]:
net/http.setRequestCancel.func3(0x0, 0xc420590180, 0xc4205f6300, 0xc4205b711c, 0xc4205b07e0)
/usr/local/go/src/net/http/client.go:320 +0x17c
created by net/http.setRequestCancel
/usr/local/go/src/net/http/client.go:330 +0x287
goroutine 50 [IO wait]:
net.runtime_pollWait(0x7f544f6bf8c0, 0x72, 0x4)
/usr/local/go/src/runtime/netpoll.go:164 +0x59
net.(*pollDesc).wait(0xc4200d4d18, 0x72, 0x1110980, 0x110c300)
/usr/local/go/src/net/fd_poll_runtime.go:75 +0x38
net.(*pollDesc).waitRead(0xc4200d4d18, 0xc4200c9000, 0x400)
/usr/local/go/src/net/fd_poll_runtime.go:80 +0x34
net.(*netFD).Read(0xc4200d4cb0, 0xc4200c9000, 0x400, 0x400, 0x0, 0x1110980, 0x110c300)
/usr/local/go/src/net/fd_unix.go:250 +0x1b7
net.(*conn).Read(0xc42000c008, 0xc4200c9000, 0x400, 0x400, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:181 +0x70
crypto/tls.(*block).readFromUntil(0xc4203f0120, 0x7f544f6730e0, 0xc42000c008, 0x5, 0xc42000c008, 0x40204e)
/usr/local/go/src/crypto/tls/conn.go:488 +0x98
crypto/tls.(*Conn).readRecord(0xc420074700, 0xd0ef17, 0xc420074820, 0x22)
/usr/local/go/src/crypto/tls/conn.go:590 +0xc4
crypto/tls.(*Conn).Read(0xc420074700, 0xc420634000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1134 +0x11d
net/http.(*persistConn).Read(0xc42013a000, 0xc420634000, 0x1000, 0x1000, 0xc4203e7800, 0xac1b617f00cbddde, 0xc42004bb80)
/usr/local/go/src/net/http/transport.go:1316 +0x14b
bufio.(*Reader).fill(0xc42012a9c0)
/usr/local/go/src/bufio/bufio.go:97 +0x117
bufio.(*Reader).Peek(0xc42012a9c0, 0x1, 0xc4205b11a0, 0xc42002bc80, 0xc42002bc00, 0x5e21b6, 0x140)
/usr/local/go/src/bufio/bufio.go:129 +0x67
net/http.(*persistConn).readLoop(0xc42013a000)
/usr/local/go/src/net/http/transport.go:1474 +0x196
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1117 +0xa35
goroutine 51 [semacquire]:
sync.runtime_notifyListWait(0xc420096b80, 0xc400004d55)
/usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420096b70)
/usr/local/go/src/sync/cond.go:57 +0x89
io.(*pipe).read(0xc420096b40, 0xc420636000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/io/pipe.go:47 +0x104
io.(*PipeReader).Read(0xc42000c600, 0xc420636000, 0x8000, 0x8000, 0x3005, 0x0, 0x0)
/usr/local/go/src/io/pipe.go:130 +0x4c
io.copyBuffer(0x7f544f673530, 0xc4205931d0, 0x110e8c0, 0xc42000c600, 0xc420636000, 0x8000, 0x8000, 0xbe9e60, 0x0, 0x7f544f673530)
/usr/local/go/src/io/io.go:390 +0x100
io.Copy(0x7f544f673530, 0xc4205931d0, 0x110e8c0, 0xc42000c600, 0xc4205931d0, 0xc420593140, 0x1)
/usr/local/go/src/io/io.go:360 +0x68
net/http.(*transferWriter).WriteBody(0xc420398b80, 0x110dbc0, 0xc42033cb00, 0x2, 0x2)
/usr/local/go/src/net/http/transfer.go:312 +0x15c
net/http.(*Request).write(0xc420098100, 0x110dbc0, 0xc42033cb00, 0x0, 0xc4203f1530, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/request.go:622 +0x6e9
net/http.(*persistConn).writeLoop(0xc42013a000)
/usr/local/go/src/net/http/transport.go:1707 +0x1ad
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1118 +0xa5a
rax 0xca
rbx 0x0
rcx 0x4579c3
rdx 0x0
rdi 0x1166050
rsi 0x0
rbp 0x7ffe2b452b30
rsp 0x7ffe2b452ae8
r8 0x0
r9 0x0
r10 0x0
r11 0x286
r12 0x0
r13 0xc4203cf040
r14 0x43e200
r15 0xdd65a0
rip 0x4579c1
rflags 0x286
cs 0x33
fs 0x0
gs 0x0
ERROR: Job failed: exit code 1
Output of checks
# root@gitlab-runner-07:~# gitlab-runner --version
Version: 11.10.1
Git revision: 1f513601
Git branch: 11-10-stable
GO version: go1.8.7
Built: 2019-04-24T09:29:18+0000
OS/Arch: linux/amd64
# root@gitlab-runner-07:~# cat /etc/gitlab-runner/config.toml
concurrent = 1
check_interval = 0
listen_address = ":9252"
[session_server]
session_timeout = 1800
[[runners]]
name = "runner-07"
url = "https://gitlab.XXXXXXXX.com/"
token = "XXXXXXXXXXXXXXXXXX"
executor = "docker"
[runners.docker]
tls_verify = false
image = "XXXXXXXXX"
privileged = true
disable_entrypoint_overwrite = false
oom_kill_disable = false
disable_cache = true
volumes = ["/var/run/docker.sock:/var/run/docker.sock", "/cache", "/tmpfs:/tmpfs"]
shm_size = 0
[runners.cache]
[runners.cache.s3]
[runners.cache.gcs]
Our hourly cleanup script (since we're a docker executor and not a docker+machine)
root@gitlab-runner-07:~# cat /etc/cron.hourly/gitlab-runner-docker-cleanup
#!/bin/bash
logger -p daemon.notice --id=$$ "Stopping gitlab-runner"
# tell gitlab-runners to shutdown when able:
pkill -QUIT gitlab-runner
# wait for them to be shutdown
logger -p daemon.notice --id=$$ "Waiting for gitlab-runner to be stopped"
while pgrep -x gitlab-runner; do sleep 5s; done
# CLEANUP IS HERE and has been removed for your convenience.
logger -p daemon.notice --id=$$ "Restarting docker"
service docker restart
logger -p daemon.notice --id=$$ "Restarting gitlab-runner"
service gitlab-runner restart
Output in /var/log/daemon
May 10 17:17:01 gitlab-runner-07 root[15756]: Stopping gitlab-runner
May 10 17:17:01 gitlab-runner-07 gitlab-runner[26088]: #033[0;33mWARNING: Requested quit, waiting for builds to finish#033[0;m #033[0;33mbuilds#033[0;m=1
May 10 17:17:01 gitlab-runner-07 root[15756]: Waiting for gitlab-runner to be stopped
May 10 17:17:01 gitlab-runner-07 containerd[1397]: ...
May 10 17:17:01 gitlab-runner-07 dockerd[25955]: ...
May 10 17:17:01 gitlab-runner-07 gitlab-runner[26088]: #033[0;33mWARNING: Job failed: exit code 1 #033[0;m #033[0;33mduration#033[0;m=11m31.537953146s #033[0;33mjob#033[0;m
May 10 17:17:02 gitlab-runner-07 gitlab-runner[26088]: #033[31;1mERROR: Failed to process runner #033[0;m #033[31;1mbuilds#033[0;m=0 #033[31;1merror#033[0;m=exit code 1 #0
May 10 17:17:02 gitlab-runner-07 gitlab-runner[26088]: All workers stopped. Can exit now #033[0;m builds#033[0;m=0
May 10 17:17:06 gitlab-runner-07 root[15756]: Removing docker images [first pass]
May 10 17:17:08 gitlab-runner-07 root[15756]: Update existing docker images
May 10 17:17:08 gitlab-runner-07 root[15756]: Removing docker images [second pass]
May 10 17:17:08 gitlab-runner-07 root[15756]: Restarting docker