Skip to content

Machine creation failed ssh command error

Summary

Today I started experiencing a problem while creating a new runner EC2 machine using AWS spot instances. This setup was working fine for more than a year

When any of my GitLab jobs are starting gitlab-runner provisioning a new EC2 spot instance. After installing Docker, gitlab-runner tries to copy some certs to remote machine through ssh and here's the error Error creating machine: Error running provisioning: ssh command error: driver=amazonec2 name=runner-8x-zjxha-runner-1675346104-2d7914c4 operation=create

Relevant logs and/or screenshots

Feb  2 13:55:05 ip-10-255-3-240 gitlab-runner: Creating machine...                               #033[0;m  driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb  2 13:55:05 ip-10-255-3-240 gitlab-runner: (runner-8x-zjxha-runner-1675346104-2d7914c4) Launching instance...#033[0;m  driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb  2 13:55:06 ip-10-255-3-240 gitlab-runner: (runner-8x-zjxha-runner-1675346104-2d7914c4) Waiting for spot instance...#033[0;m  driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb  2 13:55:11 ip-10-255-3-240 gitlab-runner: (runner-8x-zjxha-runner-1675346104-2d7914c4) Created spot instance request sir-65tehfhj#033[0;m  driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb  2 13:55:11 ip-10-255-3-240 gitlab-runner: (runner-8x-zjxha-runner-1675346104-2d7914c4) Tags are not key value in pairs. 11 elements found#033[0;m  driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb  2 13:55:11 ip-10-255-3-240 gitlab-runner: Waiting for machine to be running, this may take a few minutes...#033[0;m  driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb  2 13:55:11 ip-10-255-3-240 gitlab-runner: Detecting operating system of created instance... #033[0;m  driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb  2 13:55:11 ip-10-255-3-240 gitlab-runner: Waiting for SSH to be available...                #033[0;m  driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb  2 13:55:30 ip-10-255-3-240 gitlab-runner: Detecting the provisioner...                      #033[0;m  driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb  2 13:55:30 ip-10-255-3-240 gitlab-runner: Provisioning with ubuntu(systemd)...              #033[0;m  driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb  2 13:55:40 ip-10-255-3-240 dhclient[1982]: XMT: Solicit on eth0, interval 121330ms.
Feb  2 13:55:40 ip-10-255-3-240 gitlab-runner: Installing Docker...                              #033[0;m  driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb  2 13:56:04 ip-10-255-3-240 gitlab-runner: Copying certs to the local machine directory...   #033[0;m  driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb  2 13:56:05 ip-10-255-3-240 gitlab-runner: Copying certs to the remote machine...            #033[0;m  driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb  2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: Error creating machine: Error running provisioning: ssh command error:#033[0;m  #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb  2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: command : printf '%s' '-----BEGIN CERTIFICATE-----#033[0;m  #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb  2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: -----END CERTIFICATE-----                  #033[0;m  #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb  2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: ' | sudo tee /etc/docker/ca.pem            #033[0;m  #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb  2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: err     : exit status 1                    #033[0;m  #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb  2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: output  : -----BEGIN CERTIFICATE-----      #033[0;m  #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb  2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: -----END CERTIFICATE-----                  #033[0;m  #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb  2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: tee: /etc/docker/ca.pem: No such file or directory#033[0;m  #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb  2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[0;33mWARNING: Problem while reading command output     #033[0;m  #033[0;33merror#033[0;m=read |0: file already closed
Feb  2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[0;33mWARNING: Problem while reading command output     #033[0;m  #033[0;33merror#033[0;m=read |0: file already closed

Environment description

concurrent = 10
check_interval = 0
sentry_dsn = ""

[[runners]]
  name = "gitlab-runner"
  url = "https://gitlab.com"
  token = "<token>"
  executor = "docker+machine"
  pre_build_script = ""
  post_build_script = ""
  pre_clone_script = ""
  request_concurrency = 1
  output_limit = 10000
  limit = 0
  [runners.docker]
    tls_verify = false
    image = "docker:18.03.1-ce"
    privileged = true
    disable_cache = false
    volumes = ["/cache","/certs/client"]
    shm_size = 1073741824
    pull_policy = "if-not-present"
    runtime = ""
    helper_image = ""
  [runners.docker.tmpfs]
    "/var/opt/cache" = "rw,noexec"
  [runners.docker.services_tmpfs]

  [runners.cache]
    Type = "s3"
    Shared = true
    [runners.cache.s3]
      ServerAddress = "s3.amazonaws.com"
      BucketName = "<bucket>"
      BucketLocation = "eu-west-1"
      Insecure = false
  [runners.machine]
    IdleCount = 0
    IdleTime = 1200

    MachineDriver = "amazonec2"
    MachineName = "runner-%s"
    MachineOptions = [
      "amazonec2-instance-type=c6i.xlarge",
      "amazonec2-region=eu-west-1",
      "amazonec2-zone=a",
      "amazonec2-vpc-id=<vpc>",
      "amazonec2-subnet-id=<subnet>",
      "amazonec2-private-address-only=true",
      "amazonec2-use-private-address=false",
      "amazonec2-request-spot-instance=true",
      "amazonec2-spot-price=0.10",
      "amazonec2-security-group=gitlab-runner-docker-machine20210924082328686300000002",
      "amazonec2-tags=Name,gitlab-runner-docker-machine,Environment,gitlab-runner,Name,gitlab-runner,tf-aws-gitlab-runner:instancelifecycle,spot:yes,tf-aws-gitlab-runner:main,runner-main,",
      "amazonec2-use-ebs-optimized-instance=true",
      "amazonec2-monitoring=false",
      "amazonec2-iam-instance-profile=gitlab-runner-docker-machine",
      "amazonec2-root-size=50",
      "amazonec2-ami=ami-0c68b55d1c875067e"

    ]

Used GitLab Runner version

# gitlab-runner --version
Version:      14.0.1
Git revision: c1edb478
Git branch:   refs/pipelines/326100216
GO version:   go1.13.8
Built:        2021-06-23T16:35:23+0000
OS/Arch:      linux/amd64

I tried to update amazonec2-ami to the latest, no luck