Spot Instance Runners are Terminated Immediately
Summary
When using [Amazon Linux - image - ami-062f7200baf2fa504] with spot instances, the runner is immediately terminated. The config.toml is configured appropriately according to our documentation, and works as expected under Ubuntu. Setting IdleCount to 2 from 1 according to this issue does not change the outcome.
Steps to reproduce
- Configure an AWS Spot Instance using [ami-062f7200baf2fa504] in MachineOptions and a docker+machine executor.
- Attempt to start the runner.
What is the current bug behavior?
The spot instance spins up and is terminated immediately.
What is the expected correct behavior?
The spot instance should spin up and not stop immediately.
Relevant logs and/or screenshots
concurrent = 5
check_interval = 0
[session_server]
session_timeout = 1800
[[runners]]
name = "runner-amazonec2"
url = "https://gitlab.com"
token = "xxxxxxxxxx"
executor = "docker+machine"
limit = 2
[runners.custom_build_dir]
[runners.cache]
[runners.cache.s3]
[runners.cache.gcs]
[runners.docker]
tls_verify = false
image = "xxxxxxxxxx"
privileged = true
disable_entrypoint_overwrite = false
oom_kill_disable = false
disable_cache = false
volumes = ["/cache"]
shm_size = 0
[runners.machine]
IdleCount = 1
IdleTime = 1800
MaxBuilds = 10
MachineDriver = "amazonec2"
MachineName = "gitlab-ec2--%s"
OffPeakTimezone = ""
OffPeakIdleCount = 0
OffPeakIdleTime = 1200
MachineOptions = [
"amazonec2-spot-price=0.05",
"amazonec2-device-name=/dev/xvda",
"amazonec2-ami=ami-062f7200baf2fa504",
"amazonec2-ssh-user=ec2-user",
"amazonec2-root-size=100",
"amazonec2-region=us-east-1",
"amazonec2-vpc-id=xxxxxxxx",
"amazonec2-subnet-id=xxxxxxxxxxx",
"amazonec2-zone=c",
"amazonec2-private-address-only=true",
"amazonec2-iam-instance-profile=xxxxx",
"amazonec2-tags=xxx",
"amazonec2-security-group=xxxx",
"amazonec2-instance-type=t3.medium",
"amazonec2-request-spot-instance=true"
]
[root@ip-xx-xxx-xxx-xxx gitlab-runner]# gitlab-runner --debug run
Runtime platform arch=amd64 os=linux pid=18397 revision=ce065b93 version=12.10.1
Starting multi-runner from /etc/gitlab-runner/config.toml... builds=0
Checking runtime mode GOOS=linux uid=0
Running in system-mode.
Configuration loaded builds=0
listenaddress: ""
sessionserver:
listenaddress: ""
advertiseaddress: ""
sessiontimeout: 1800
concurrent: 5
checkinterval: 0
loglevel: null
logformat: null
user: ""
runners:
- name: runner-amazonec2
limit: 2
outputlimit: 0
requestconcurrency: 0
runnercredentials:
url: https://gitlab.com
token: xxxxxx
tlscafile: ""
tlscertfile: ""
tlskeyfile: ""
runnersettings:
executor: docker+machine
buildsdir: ""
cachedir: ""
cloneurl: ""
environment: []
preclonescript: ""
prebuildscript: ""
postbuildscript: ""
debugtracedisabled: false
shell: ""
custombuilddir:
enabled: false
referees: null
cache:
type: ""
path: ""
shared: false
s3:
serveraddress: ""
accesskey: ""
secretkey: ""
bucketname: ""
bucketlocation: ""
insecure: false
gcs:
cachegcscredentials:
accessid: ""
privatekey: ""
credentialsfile: ""
bucketname: ""
ssh: null
docker:
credentials:
host: ""
certpath: ""
tlsverify: false
hostname: ""
image: xxxxxxxxxxxxxxxx.dkr.ecr.us-east-1.amazonaws.com/gitlab-ci
runtime: ""
memory: ""
memoryswap: ""
memoryreservation: ""
cpusetcpus: ""
cpus: ""
cpushares: 0
dns: []
dnssearch: []
privileged: true
disableentrypointoverwrite: false
usernsmode: ""
capadd: []
capdrop: []
oomkilldisable: false
oomscoreadjust: 0
securityopt: []
devices: []
disablecache: false
volumes:
- /cache
volumedriver: ""
cachedir: ""
extrahosts: []
volumesfrom: []
networkmode: ""
links: []
services: []
waitforservicestimeout: 0
allowedimages: []
allowedservices: []
pullpolicy: ""
shmsize: 0
tmpfs: {}
servicestmpfs: {}
sysctls: {}
helperimage: ""
parallels: null
virtualbox: null
machine:
idlecount: 1
idletime: 1800
maxbuilds: 10
machinedriver: amazonec2
machinename: gitlab-ec2--%s
machineoptions:
- amazonec2-spot-price=0.05
- amazonec2-device-name=/dev/xvda
- amazonec2-ami=xxxxxxxxxxxxxxxx
- amazonec2-ssh-user=ec2-user
- amazonec2-root-size=100
- amazonec2-region=us-east-1
- amazonec2-vpc-id=xxxxxxxxxxxxxx
- amazonec2-subnet-id=subnet-xxxxxxxxxxxxxx
- amazonec2-zone=c
- amazonec2-private-address-only=true
- amazonec2-iam-instance-profile=xxxxxxxxxxxxxx
- amazonec2-tags=xxxxxxxxx
- amazonec2-security-group=xxxxxx
- amazonec2-instance-type=t3.medium
- amazonec2-request-spot-instance=true
offpeakperiods: []
offpeaktimezone: ""
offpeakidlecount: 0
offpeakidletime: 1200
kubernetes: null
custom: null
sentrydsn: null
modtime: 2020-05-08T06:03:58.200121784Z
loaded: true
builds=0
Waiting for stop signal builds=0
listen_address not defined, metrics & debug endpoints disabled builds=0
[session_server].listen_address not defined, session endpoints disabled builds=0
Feeding runners to channel builds=0
Starting worker builds=0 worker=0
Starting worker builds=0 worker=2
Starting worker builds=0 worker=1
Docker Machine Details creating=0 idle=1 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:13.731846211 +0000 UTC m=+0.015285282 total=1 used=0
Starting worker builds=0 worker=4
Starting worker builds=0 worker=3
WARNING: Requesting machine removal lifetime=1.200600444s name=runner-hrdl7rsz-gitlab-ec2--1588917828-f3fc2576 now=2020-05-08 06:04:14.932448763 +0000 UTC m=+1.215887850 reason=machine is unavailable used=1.200603077s usedCount=1
Dialing: tcp gitlab.com:443 ...
WARNING: Stopping machine lifetime=1.219958957s name=runner-hrdl7rsz-gitlab-ec2--1588917828-f3fc2576 reason=machine is unavailable used=19.263542ms usedCount=1
Stopping "runner-hrdl7rsz-gitlab-ec2--1588917828-f3fc2576"... name=runner-hrdl7rsz-gitlab-ec2--1588917828-f3fc2576 operation=stop
Checking for jobs... nothing runner=hRDL7rsz
ERROR: UnsupportedOperation: You can't stop the Spot Instance 'i-xxxxxxxxxxxxxx because it is associated with a one-time Spot Instance request. You can only stop Spot Instances associated with persistent Spot Instance requests. name=runner-hrdl7rsz-gitlab-ec2--1588917828-f3fc2576 operation=stop
ERROR: status code: 400, request id: c773398e-8655-479e-97cd-dc033f056fc0 name=runner-hrdl7rsz-gitlab-ec2--1588917828-f3fc2576 operation=stop
WARNING: Error while stopping machine error=exit status 1 lifetime=1.48334715s name=runner-hrdl7rsz-gitlab-ec2--1588917828-f3fc2576 reason=machine is unavailable used=282.652449ms usedCount=1
WARNING: Removing machine lifetime=1.48340079s name=runner-hrdl7rsz-gitlab-ec2--1588917828-f3fc2576 reason=machine is unavailable used=282.705076ms usedCount=1
About to remove runner-hrdl7rsz-gitlab-ec2--1588917828-f3fc2576 name=runner-hrdl7rsz-gitlab-ec2--1588917828-f3fc2576 operation=remove
WARNING: This action will delete both local reference and remote instance. name=runner-hrdl7rsz-gitlab-ec2--1588917828-f3fc2576 operation=remove
Successfully removed runner-hrdl7rsz-gitlab-ec2--1588917828-f3fc2576 name=runner-hrdl7rsz-gitlab-ec2--1588917828-f3fc2576 operation=remove
Machine removed lifetime=1.87496395s name=runner-hrdl7rsz-gitlab-ec2--1588917828-f3fc2576 now=2020-05-08 06:04:15.606799912 +0000 UTC m=+1.890239026 reason=machine is unavailable retries=0 used=674.268818ms usedCount=1
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:16.732372837 +0000 UTC m=+3.015811902 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Executing /usr/local/bin/docker-machine [docker-machine --bugsnag-api-token=no-report create --driver amazonec2 --amazonec2-spot-price=0.05 --amazonec2-device-name=/dev/xvda --amazonec2-ami=ami-xxxxxxxxxxx --amazonec2-ssh-user=xxx-user --amazonec2-root-size=100 --amazonec2-region=us-east-1 --amazonec2-vpc-id=xxxxxxxxxxxxxx --amazonec2-subnet-id=subnet-xxxxxxxxxxxxxx --amazonec2-zone=c --amazonec2-private-address-only=true --amazonec2-iam-instance-profile=xxxxxxxxxxxxxx --amazonec2-tags=xxxxxxx --amazonec2-security-group=xxxxxx --amazonec2-instance-type=t3.medium --amazonec2-request-spot-instance=true runner-hrdl7rsz-gitlab-ec2--1588917856-bbbe59ec]
Running pre-create checks... driver=amazonec2 name=runner-hrdl7rsz-gitlab-ec2--1588917856-bbbe59ec operation=create
Creating machine... driver=amazonec2 name=runner-hrdl7rsz-gitlab-ec2--1588917856-bbbe59ec operation=create
(runner-hrdl7rsz-gitlab-ec2--1588917856-bbbe59ec) Launching instance... driver=amazonec2 name=runner-hrdl7rsz-gitlab-ec2--1588917856-bbbe59ec operation=create
(runner-hrdl7rsz-gitlab-ec2--1588917856-bbbe59ec) Waiting for spot instance... driver=amazonec2 name=runner-hrdl7rsz-gitlab-ec2--1588917856-bbbe59ec operation=create
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:19.732576319 +0000 UTC m=+6.016015370 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:22.732776196 +0000 UTC m=+9.016215233 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
(runner-hrdl7rsz-gitlab-ec2--1588917856-bbbe59ec) Created spot instance request sir-6sg9k69j driver=amazonec2 name=runner-hrdl7rsz-gitlab-ec2--1588917856-bbbe59ec operation=create
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:25.73302768 +0000 UTC m=+12.016466760 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:28.733182003 +0000 UTC m=+15.016621056 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Waiting for machine to be running, this may take a few minutes... driver=amazonec2 name=runner-hrdl7rsz-gitlab-ec2--1588917856-bbbe59ec operation=create
Detecting operating system of created instance... driver=amazonec2 name=runner-hrdl7rsz-gitlab-ec2--1588917856-bbbe59ec operation=create
Waiting for SSH to be available... driver=amazonec2 name=runner-hrdl7rsz-gitlab-ec2--1588917856-bbbe59ec operation=create
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:31.733332068 +0000 UTC m=+18.016771107 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:34.733599787 +0000 UTC m=+21.017038864 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:37.733786495 +0000 UTC m=+24.017225540 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:40.734012937 +0000 UTC m=+27.017451986 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:43.734141369 +0000 UTC m=+30.017580412 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:46.734406972 +0000 UTC m=+33.017846166 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:49.73451514 +0000 UTC m=+36.017954181 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:52.734698252 +0000 UTC m=+39.018137298 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:55.734879568 +0000 UTC m=+42.018318616 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:04:58.735105293 +0000 UTC m=+45.018544341 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:01.735301761 +0000 UTC m=+48.018740809 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:04.735455215 +0000 UTC m=+51.018894256 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:07.735592716 +0000 UTC m=+54.019031766 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:10.735837556 +0000 UTC m=+57.019276607 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:13.736001542 +0000 UTC m=+60.019440587 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:16.736256374 +0000 UTC m=+63.019695412 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:19.736404951 +0000 UTC m=+66.019843989 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:22.736589577 +0000 UTC m=+69.020028625 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:25.736789625 +0000 UTC m=+72.020228689 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:28.737042698 +0000 UTC m=+75.020481740 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:31.737299893 +0000 UTC m=+78.020738964 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:34.737481527 +0000 UTC m=+81.020920573 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:37.737733354 +0000 UTC m=+84.021172403 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:40.737939367 +0000 UTC m=+87.021378410 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:43.738200147 +0000 UTC m=+90.021639187 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:46.73837732 +0000 UTC m=+93.021816369 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
Feeding runners to channel builds=0
Docker Machine Details creating=1 idle=0 maxMachines=2 minIdleCount=1 removing=0 runner=hRDL7rsz time=2020-05-08 06:05:49.738621068 +0000 UTC m=+96.022060112 total=1 used=0
WARNING: Failed to process runner builds=0 error=failed to update executor: no free machines that can process builds executor=docker+machine runner=hRDL7rsz
^CWARNING: Graceful shutdown not finished properly builds=0 error=received: interrupt
WARNING: Starting forceful shutdown StopSignal=interrupt builds=0
Broadcasting interrupt signal builds=0
All workers stopped. Can exit now builds=0
Stopping worker builds=0 worker=4
[root@ip-xx-xxx-xxx-xxx gitlab-runner]#
Output of checks
This bug happens on GitLab.com 13.0.0-pre 54286ec1
https://gitlab.zendesk.com/agent/tickets/155832
ZD Ticket (Internal):Edited by 🤖 GitLab Bot 🤖