Skip to content

UnsupportedOperation: You can't stop the Spot Instance 'i-xxxxxxxxxxx' because it is associated with a one-time Spot Instance request. You can only stop Spot Instances associated with persistent Spot Instance requests.

The instance is removed anyway, but I'm keen to know if this error is anything I should be concerned about.

Here is a complete set of logs covering a CI run:

Feb  3 13:45:03 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:45:03Z" level=info msg="Checking for jobs... received" job=2049740326 repo_url="https://gitlab.com/<REDACTED>.git" runner=<REDACTED>
Feb  3 13:45:04 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:45:04Z" level=info msg="Running pre-create checks..." driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:45:04 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:45:04Z" level=info msg="Creating machine..." driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:45:04 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:45:04Z" level=info msg="(runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4) Launching instance..." driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:45:05 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:45:05Z" level=info msg="(runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4) Waiting for spot instance..." driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:45:20 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:45:20Z" level=info msg="(runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4) Created spot instance request sir-q47pbbiq" driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:45:21 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:45:21Z" level=info msg="Waiting for machine to be running, this may take a few minutes..." driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:45:21 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:45:21Z" level=info msg="Detecting operating system of created instance..." driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:45:21 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:45:21Z" level=info msg="Waiting for SSH to be available..." driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:45:32 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:45:32Z" level=info msg="Detecting the provisioner..." driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:45:33 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:45:33Z" level=info msg="Provisioning with ubuntu(systemd)..." driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:45:46 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:45:46Z" level=info msg="Installing Docker..." driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:46:31 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:46:31Z" level=info msg="Copying certs to the local machine directory..." driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:46:32 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:46:32Z" level=info msg="Copying certs to the remote machine..." driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:46:33 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:46:33Z" level=info msg="Setting Docker configuration on the remote daemon..." driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:46:34 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:46:34Z" level=info msg="Checking connection to Docker..." driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:46:35 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:46:35Z" level=info msg="Docker is up and running!" driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:46:35 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:46:35Z" level=info msg="To see how to connect your Docker Client to the Docker Engine running on this virtual machine, run: docker-machine env runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4" driver=amazonec2 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=create
Feb  3 13:46:35 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:46:35Z" level=info msg="Machine created" duration=1m31.162795444s name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 now="2022-02-03 13:46:35.043579486 +0000 UTC m=+3290.232000286" retries=0
Feb  3 13:46:35 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:46:35Z" level=info msg="Created docker-machine" created="2022-02-03 13:45:03.880326459 +0000 UTC m=+3199.068747249" docker="tcp://10.0.1.149:2376" job=2049740326 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 now="2022-02-03 13:46:35.59675464 +0000 UTC m=+3290.785175430" project=20267612 runner=<REDACTED> usedcount=1
Feb  3 13:47:21 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:47:21Z" level=info msg="Starting docker-machine build..." created="2022-02-03 13:45:03.880326459 +0000 UTC m=+3199.068747249" docker="tcp://10.0.1.149:2376" job=2049740326 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 now="2022-02-03 13:47:21.244971053 +0000 UTC m=+3336.433391843" project=20267612 runner=<REDACTED> usedcount=1
Feb  3 13:52:08 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:08Z" level=info msg="Finished docker-machine build: <nil>" created="2022-02-03 13:45:03.880326459 +0000 UTC m=+3199.068747249" docker="tcp://10.0.1.149:2376" job=2049740326 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 now="2022-02-03 13:52:08.886348592 +0000 UTC m=+3624.074769392" project=20267612 runner=<REDACTED> usedcount=1
Feb  3 13:52:13 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:13Z" level=info msg="Cleaned up docker-machine" created="2022-02-03 13:45:03.880326459 +0000 UTC m=+3199.068747249" docker="tcp://10.0.1.149:2376" job=2049740326 name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 now="2022-02-03 13:52:13.475857733 +0000 UTC m=+3628.664278523" project=20267612 runner=<REDACTED> usedcount=1
Feb  3 13:52:13 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:13Z" level=info msg="Job succeeded" duration_s=429.596598329 job=2049740326 project=20267612 runner=<REDACTED>
Feb  3 13:52:15 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:15Z" level=warning msg="Requesting machine removal" lifetime=7m11.378241163s name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 now="2022-02-03 13:52:15.258571272 +0000 UTC m=+3630.446992062" reason="too many idle machines" used=1.781907965s usedCount=1
Feb  3 13:52:15 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:15Z" level=warning msg="Stopping machine" lifetime=7m11.43003652s name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 reason="too many idle machines" used=51.272174ms usedCount=1
Feb  3 13:52:15 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:15Z" level=info msg="Stopping \"runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4\"..." name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=stop
Feb  3 13:52:15 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:15Z" level=error msg="UnsupportedOperation: You can't stop the Spot Instance 'i-<REDACTED>' because it is associated with a one-time Spot Instance request. You can only stop Spot Instances associated with persistent Spot Instance requests." name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=stop
Feb  3 13:52:15 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:15Z" level=error msg="\tstatus code: 400, request id: a44f52f8-952f-458d-9afa-99993d20d798" name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=stop
Feb  3 13:52:15 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:15Z" level=warning msg="Error while stopping machine" error="exit status 1" lifetime=7m11.836609506s name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 reason="too many idle machines" used=457.84405ms usedCount=1
Feb  3 13:52:15 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:15Z" level=warning msg="Removing machine" lifetime=7m11.836671946s name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 reason="too many idle machines" used=457.9061ms usedCount=1
Feb  3 13:52:15 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:15Z" level=info msg="About to remove runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4" name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=remove
Feb  3 13:52:15 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:15Z" level=info msg="WARNING: This action will delete both local reference and remote instance." name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=remove
Feb  3 13:52:16 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:16Z" level=info msg="Successfully removed runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4" name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 operation=remove
Feb  3 13:52:16 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:16Z" level=warning msg="Problem while reading command output" error="read |0: file already closed"
Feb  3 13:52:16 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:16Z" level=warning msg="Problem while reading command output" error="read |0: file already closed"
Feb  3 13:52:16 ip-10-0-1-80 gitlab-runner[1746290]: time="2022-02-03T13:52:16Z" level=info msg="Machine removed" lifetime=7m12.206280471s name=runner-<REDACTED>-gitlab-ml-ci-machine-1643895903-859a91f4 now="2022-02-03 13:52:16.08661039 +0000 UTC m=+3631.275031190" reason="too many idle machines" retries=0 used=827.514855ms usedCount=1

Note that as well as the

UnsupportedOperation: You can't stop the Spot Instance 'i-<REDACTED>' because it is associated with a one-time Spot Instance request. You can only stop Spot Instances associated with persistent Spot Instance requests

messsage I also get 2 of these:

Problem while reading command output" error="read |0: file already closed

Is this the expected behaviour?

Current gitlab-runner.toml:

listen_address = "[::]:9252"
concurrent = 4
check_interval = 3
log_format = "text"

[session_server]
  session_timeout = 1800

[[runners]]
  name = "gitlab-ml-autoscaler"
  limit = 4
  url = "https://gitlab.com/"
  token = "<REDACTED>"
  executor = "docker+machine"
  environment = ["DOCKER_AUTH_CONFIG={\"auths\":{\"registry-1.docker.io\":{\"auth\":\"<REDACTED>\"}}}"]
  [runners.cache]
    Type = "s3"
    Shared = true
    [runners.cache.s3]
      ServerAddress = "s3.amazonaws.com"
      BucketName = "<REDACTED>"
      BucketLocation = "eu-west-2"
  [runners.docker]
    tls_verify = false
    image = "docker:19.03.12"
    privileged = true
    disable_entrypoint_overwrite = false
    oom_kill_disable = false
    disable_cache = true
    shm_size = 0
  [runners.machine]
    IdleCount = 0
    MachineDriver = "amazonec2"
    MachineName = "gitlab-ml-ci-machine-%s"
    MachineOptions = [
        "amazonec2-iam-instance-profile=gitlab-runner-agent",
        "amazonec2-region=eu-west-2",
        "amazonec2-vpc-id=<REDACTED>",
        "amazonec2-subnet-id=<REDACTED>",
        "amazonec2-zone=a",
        "amazonec2-use-private-address=true",
        "amazonec2-tags=runner-manager-name,gitlab-ml-autoscaler,gitlab,true,gitlab-runner-autoscale,true,Project,Gitlab ML CI",
        "amazonec2-security-group=<REDACTED>,
        "amazonec2-instance-type=t3a.small",
        "amazonec2-private-address-only=true",
        "amazonec2-security-group-readonly=true",
        "amazonec2-request-spot-instance=true",
        "amazonec2-spot-price=",
    ]
    OffPeakTimezone = ""
    OffPeakIdleCount = 0
    OffPeakIdleTime = 0

Background info/versions:

$ gitlab-runner --version
Version:      14.7.0
Git revision: 98daeee0
Git branch:   14-7-stable
GO version:   go1.17.5
Built:        2022-01-19T17:11:48+0000
OS/Arch:      linux/amd64
$ docker-machine --version
docker-machine version 0.16.2-gitlab.12, build 84bc65e2
$ uname -a
Linux ip-10-0-1-80 5.4.0-1038-aws #40-Ubuntu SMP Fri Feb 5 23:50:40 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
$ cat /etc/issue
Ubuntu 20.04.3 LTS \n \l

Thanks