Job failed prepare environment invalid content type

Customer is testing the new autoscaling executors for Gitlab Runner with Windows, and they're having trouble with the instance executor.

It seems that whenever a new instance is created, connecting to it initially fails with an error like the following:

WARNING: Job failed: prepare environment: invalid content type. Check https://docs.gitlab.com/runner/shells/index.html#shell-profile-loading for more information
   duration_s=90.032858454 job=731 project=12 runner=xxxxxx

This causes the job to fail. However, immediately after this, retrying the job before the instance is removed seems to work fine. That suggests that gitlab-runner is trying to connect to the new instance too quickly, before it's truly ready.

It seems it waits for the instance to be in an InService state. As a workaround customer has added a lifecycle hook to the ASG, and that seems to have helped.

<powershell>
Write-Host "Starting Userdata script $(Get-Date)"
# Required for Gitlab runner
netsh advfirewall firewall add rule name="WinRM-HTTP" dir=in localport=5985 protocol=TCP action=allow
winrm set winrm/config/service/auth '@{Basic="True"}'
winrm set winrm/config/service '@{AllowUnencrypted="true"}'

# Install chocolatey, git and gitlab-runner for non-docker runners
if ("${executor_type}" -eq "instance") {
	Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
	Write-Host "$(Get-Date) Installing git..."
	choco install -y git | Out-Null
	Write-Host "$(Get-Date) Installing gitlab-runner..."
	choco install -y gitlab-runner | Out-Null
}
$id = Get-EC2InstanceMetadata -Category "InstanceId"
$asg = (Get-ASAutoScalingInstance -InstanceId $id).AutoScalingGroupName
Write-Host "Signalling completion for $($id) to $($asg)"
Complete-ASLifeCycleAction -InstanceId $id -AutoScalingGroupName $asg -LifecycleActionResult "CONTINUE" -LifecycleHookName "instance-ready"
</powershell>

Config.toml configuration:

concurrent = 10
check_interval = 0
log_level = "debug"
shutdown_timeout = 0

[session_server]
  session_timeout = 1800

[[runners]]
  name = "gitlab-runner"
  output_limit = 16384
  url = REDACTED
  id = 70
  token = "redacted"
  token_obtained_at = 2024-02-13T11:12:19Z
  token_expires_at = 0001-01-01T00:00:00Z
  executor = "docker-autoscaler"
  environment = ["FF_USE_POWERSHELL_PATH_RESOLVER=1"]
  shell = "powershell"
  [runners.cache]
    Type = "s3"
    Path = "runner-cache"
    Shared = true
    MaxUploadedArchiveSize = 0
    [runners.cache.s3]
      BucketName = REDACTED
      BucketLocation = REDACTED
  [runners.docker]
    tls_verify = false
    image = "mcr.microsoft.com/windows/servercore:ltsc2022"
    privileged = false
    disable_entrypoint_overwrite = false
    oom_kill_disable = false
    disable_cache = true
    shm_size = 0
    network_mtu = 0
  [runners.autoscaler]
    capacity_per_instance = 1
    max_use_count = 1000
    max_instances = 10
    plugin = "fleeting-plugin-aws"
    [runners.autoscaler.plugin_config]
      name = "gitlab-test-docker-runners"
    [runners.autoscaler.connector_config]
      username = REDACTED
      password = ""
      key_path = "/etc/gitlab-runner/windows-decryption-key.pem"
      use_static_credentials = false
      keepalive = "0s"
      timeout = "0s"
      use_external_addr = false

    [[runners.autoscaler.policy]]
      idle_count = 0
      idle_time = "10m0s"
      scale_factor = 0.0
      scale_factor_limit = 0

[[runners]]
  name = "gitlab-test-runner"
  output_limit = 16384
  url = REDACTED
  id = 71
  token = "redacted"
  token_obtained_at = 2024-02-13T11:12:20Z
  token_expires_at = 0001-01-01T00:00:00Z
  executor = "instance"
  environment = ["FF_USE_POWERSHELL_PATH_RESOLVER=1"]
  shell = "powershell"
  [runners.cache]
    Type = "s3"
    Path = "runner-cache"
    Shared = true
    MaxUploadedArchiveSize = 0
    [runners.cache.s3]
      BucketName = REDACTED
      BucketLocation = REDACTED
  [runners.autoscaler]
    capacity_per_instance = 1
    max_use_count = 1000
    max_instances = 10
    plugin = "fleeting-plugin-aws"
    [runners.autoscaler.plugin_config]
      name = "gitlab-test-runners"
    [runners.autoscaler.connector_config]
      username = REDACTED
      password = ""
      key_path = "/etc/gitlab-runner/windows-decryption-key.pem"
      use_static_credentials = false
      keepalive = "0s"
      timeout = "0s"
      use_external_addr = false

    [[runners.autoscaler.policy]]
      idle_count = 0
      idle_time = "10m0s"
      scale_factor = 0.0
      scale_factor_limit = 0

It might also useful to have a look at the error handling of the above so the error would give us more information.