Windows jobs hang until timeout when providing an exit code > 32767
when a Windows job (cmd or pwsh shell) exists with an error code > 32767, the runner cleanup fails and the job hangs until timeout
expected behavior: the job terminates, and the runner becomes available right away; Windows 32-bit exit codes shall be supported, at least to leading to hanging jobs on Windows runners
in the Windows Event Viewer logs of the runner, the following **500 Internal Server Error **can be seen:
[0;33mWARNING: Submitting job to coordinator... failed [0;m [0;33mbytesize[0;m=2196 [0;33mchecksum[0;m=crc32:d5b71ead [0;33mcode[0;m=500 [0;33mjob[0;m=1978894 [0;33mjob-status[0;m= [0;33mrunner[0;m=rPG8zoT1 [0;33mstatus[0;m=500 **Internal Server Error **[0;33mupdate-interval[0;m=0s
this reproduces with various Windows gitlab runners 15.x, 16.x, 17.x and a 16.11 gitlab self-hosted server
on the server-side, the follow log is quite indicative:
{
"severity": "ERROR",
"time": "2025-04-07T12:02:49.140Z",
"correlation_id": "01JR823FK8GDP6GXN7RH8G7JAN",
"meta.caller_id": "PUT /api/:version/jobs/:id",
"meta.remote_ip": "172.26.9.118",
"meta.feature_category": "continuous_integration",
"meta.project": "pmeerwald/lovely-gitlab",
"meta.root_namespace": "pmeerwald",
"meta.client_id": "user/41",
"meta.pipeline_id": 200006,
"meta.job_id": 1978907,
"exception.class": "ActiveModel::RangeError",
"exception.message": "1073750990 is out of range for ActiveModel::Type::Integer with limit 2 bytes",
"exception.backtrace": [
"activemodel (7.0.8.1) lib/active_model/type/integer.rb:52:in `ensure_in_range'",
"activemodel (7.0.8.1) lib/active_model/type/integer.rb:28:in `serialize'",
"activemodel (7.0.8.1) lib/active_model/attribute.rb:56:in `value_for_database'",
"activerecord (7.0.8.1) lib/arel/collectors/substitute_binds.rb:19:in `add_bind'",
"activerecord (7.0.8.1) lib/arel/visitors/to_sql.rb:743:in `visit_ActiveModel_Attribute'",
"activerecord (7.0.8.1) lib/arel/visitors/visitor.rb:30:in `visit'",
"activerecord (7.0.8.1) lib/arel/visitors/to_sql.rb:635:in `visit_Arel_Nodes_Assignment'",
"activerecord (7.0.8.1) lib/arel/visitors/to_sql.rb:44:in `visit_Arel_Nodes_UpdateStatement'",
"activerecord (7.0.8.1) lib/arel/visitors/visitor.rb:30:in `visit'",
"activerecord (7.0.8.1) lib/arel/visitors/visitor.rb:11:in `accept'",
"activerecord (7.0.8.1) lib/arel/visitors/to_sql.rb:18:in `compile'",
"activerecord (7.0.8.1) lib/active_record/connection_adapters/abstract/database_statements.rb:37:in `to_sql_and_binds'",
"activerecord (7.0.8.1) lib/active_record/connection_adapters/abstract/database_statements.rb:174:in `update'",
"activerecord (7.0.8.1) lib/active_record/connection_adapters/abstract/query_cache.rb:22:in `update'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `public_send'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `block in write_using_load_balancer'",
"lib/gitlab/database/load_balancing/load_balancer.rb:141:in `block in read_write'",
"lib/gitlab/database/load_balancing/load_balancer.rb:228:in `retry_with_backoff'",
"lib/gitlab/database/load_balancing/load_balancer.rb:130:in `read_write'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `write_using_load_balancer'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:61:in `block (2 levels) in \u003cclass:ConnectionProxy\u003e'",
"activerecord (7.0.8.1) lib/active_record/persistence.rb:513:in `_update_record'",
"activerecord (7.0.8.1) lib/active_record/persistence.rb:1060:in `_update_row'",
"activerecord (7.0.8.1) lib/active_record/locking/optimistic.rb:93:in `_update_row'",
"activerecord (7.0.8.1) lib/active_record/persistence.rb:1082:in `_update_record'",
"activerecord (7.0.8.1) lib/active_record/attribute_methods/dirty.rb:216:in `_update_record'",
"activerecord (7.0.8.1) lib/active_record/callbacks.rb:463:in `block in _update_record'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:99:in `run_callbacks'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:929:in `_run_update_callbacks'",
"activerecord (7.0.8.1) lib/active_record/callbacks.rb:463:in `_update_record'",
"activerecord (7.0.8.1) lib/active_record/timestamp.rb:121:in `_update_record'",
"activerecord (7.0.8.1) lib/active_record/persistence.rb:1069:in `create_or_update'",
"activerecord (7.0.8.1) lib/active_record/callbacks.rb:455:in `block in create_or_update'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:107:in `run_callbacks'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:929:in `_run_save_callbacks'",
"activerecord (7.0.8.1) lib/active_record/callbacks.rb:455:in `create_or_update'",
"activerecord (7.0.8.1) lib/active_record/timestamp.rb:126:in `create_or_update'",
"activerecord (7.0.8.1) lib/active_record/persistence.rb:615:in `save'",
"activerecord (7.0.8.1) lib/active_record/validations.rb:47:in `save'",
"activerecord (7.0.8.1) lib/active_record/transactions.rb:298:in `block in save'",
"activerecord (7.0.8.1) lib/active_record/transactions.rb:354:in `block in with_transaction_returning_status'",
"activerecord (7.0.8.1) lib/active_record/connection_adapters/abstract/database_statements.rb:314:in `transaction'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `public_send'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `block in write_using_load_balancer'",
"lib/gitlab/database/load_balancing/load_balancer.rb:141:in `block in read_write'",
"lib/gitlab/database/load_balancing/load_balancer.rb:228:in `retry_with_backoff'",
"lib/gitlab/database/load_balancing/load_balancer.rb:130:in `read_write'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `write_using_load_balancer'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:78:in `transaction'",
"activerecord (7.0.8.1) lib/active_record/transactions.rb:350:in `with_transaction_returning_status'",
"activerecord (7.0.8.1) lib/active_record/transactions.rb:298:in `save'",
"activerecord (7.0.8.1) lib/active_record/suppressor.rb:50:in `save'",
"activerecord (7.0.8.1) lib/active_record/autosave_association.rb:455:in `save_has_one_association'",
"activerecord (7.0.8.1) lib/active_record/autosave_association.rb:198:in `block in add_autosave_association_callbacks'",
"activerecord (7.0.8.1) lib/active_record/autosave_association.rb:166:in `instance_eval'",
"activerecord (7.0.8.1) lib/active_record/autosave_association.rb:166:in `block in define_non_cyclic_method'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:400:in `block in make_lambda'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:236:in `block in halting_and_conditional'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:599:in `block in invoke_after'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:599:in `each'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:599:in `invoke_after'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:108:in `run_callbacks'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:929:in `_run_update_callbacks'",
"activerecord (7.0.8.1) lib/active_record/callbacks.rb:463:in `_update_record'",
"activerecord (7.0.8.1) lib/active_record/timestamp.rb:121:in `_update_record'",
"activerecord (7.0.8.1) lib/active_record/persistence.rb:1069:in `create_or_update'",
"activerecord (7.0.8.1) lib/active_record/callbacks.rb:455:in `block in create_or_update'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:118:in `block in run_callbacks'",
"activerecord (7.0.8.1) lib/active_record/autosave_association.rb:370:in `around_save_collection_association'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:138:in `run_callbacks'",
"activesupport (7.0.8.1) lib/active_support/callbacks.rb:929:in `_run_save_callbacks'",
"activerecord (7.0.8.1) lib/active_record/callbacks.rb:455:in `create_or_update'",
"activerecord (7.0.8.1) lib/active_record/timestamp.rb:126:in `create_or_update'",
"activerecord (7.0.8.1) lib/active_record/persistence.rb:615:in `save'",
"activerecord (7.0.8.1) lib/active_record/validations.rb:47:in `save'",
"activerecord (7.0.8.1) lib/active_record/transactions.rb:298:in `block in save'",
"activerecord (7.0.8.1) lib/active_record/transactions.rb:354:in `block in with_transaction_returning_status'",
"activerecord (7.0.8.1) lib/active_record/connection_adapters/abstract/database_statements.rb:314:in `transaction'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `public_send'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `block in write_using_load_balancer'",
"lib/gitlab/database/load_balancing/load_balancer.rb:141:in `block in read_write'",
"lib/gitlab/database/load_balancing/load_balancer.rb:228:in `retry_with_backoff'",
"lib/gitlab/database/load_balancing/load_balancer.rb:130:in `read_write'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `write_using_load_balancer'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:78:in `transaction'",
"activerecord (7.0.8.1) lib/active_record/transactions.rb:350:in `with_transaction_returning_status'",
"activerecord (7.0.8.1) lib/active_record/transactions.rb:298:in `save'",
"activerecord (7.0.8.1) lib/active_record/suppressor.rb:50:in `save'",
"state_machines-activerecord (0.8.0) lib/state_machines/integrations/active_record.rb:511:in `block in save'",
"state_machines-activerecord (0.8.0) lib/state_machines/integrations/active_record.rb:530:in `block in around_save'",
"state_machines (0.5.0) lib/state_machines/transition_collection.rb:150:in `block in run_actions'",
"state_machines (0.5.0) lib/state_machines/transition_collection.rb:170:in `catch_exceptions'",
"state_machines (0.5.0) lib/state_machines/transition_collection.rb:148:in `run_actions'",
"state_machines (0.5.0) lib/state_machines/transition_collection.rb:133:in `run_callbacks'",
"state_machines (0.5.0) lib/state_machines/transition_collection.rb:224:in `run_callbacks'",
"state_machines (0.5.0) lib/state_machines/transition_collection.rb:128:in `block in run_callbacks'",
"state_machines (0.5.0) lib/state_machines/transition.rb:371:in `before'",
"state_machines (0.5.0) lib/state_machines/transition.rb:190:in `block in run_callbacks'",
"state_machines (0.5.0) lib/state_machines/transition.rb:292:in `block in pausable'",
"state_machines (0.5.0) lib/state_machines/transition.rb:292:in `catch'",
"state_machines (0.5.0) lib/state_machines/transition.rb:292:in `pausable'",
"state_machines (0.5.0) lib/state_machines/transition.rb:190:in `run_callbacks'",
"state_machines (0.5.0) lib/state_machines/transition_collection.rb:127:in `run_callbacks'",
"state_machines (0.5.0) lib/state_machines/transition_collection.rb:212:in `run_callbacks'",
"state_machines (0.5.0) lib/state_machines/transition_collection.rb:63:in `block (2 levels) in perform'",
...Edited by 🤖 GitLab Bot 🤖