Skip to content

Workspaces cannot be created on CentOS 7 / Ubuntu 20.04 based GitLab instance after upgrading to GitLab 17.4

https://gitlab.com/gitlab-org/release/tasks/-/issues/16046+ This is the backport issue

MR:

Devfile Patch Tags:

Summary

Workspaces cannot be created on CentOS 7 based GitLab instance after upgrading to GitLab 17.4 due to the old version of glibc shipped with CentOS 7 or Ubuntu 20.04.

Steps to reproduce

  • Install GitLab 17.4.2 on CentOS 7 or Ubuntu 20.04 (the important part is that glibc version of OS should be less than 2.34)
  • Follow all the docs to configure Workspaces infrastructure including GitLab Agent configuration.
  • In one of your projects under the group where GitLab Agent is configured, click Edit > New workspace
  • In New workspace screen, click Create workspace

What is the current bug behavior?

Creation fails with the error:

Workspace create devfile flatten failed: /opt/gitlab/embedded/lib/ruby/gems/3.1.0/gems/devfile-0.0.28.pre.alpha1-x86_64-linux/bin/devfile:
/lib64/libc.so.6: version `GLIBC_2.34' not found (required by /opt/gitlab/embedded/lib/ruby/gems/3.1.0/gems/devfile-0.0.28.pre.alpha1-x86_64-linux/bin/devfile)
/opt/gitlab/embedded/lib/ruby/gems/3.1.0/gems/devfile-0.0.28.pre.alpha1-x86_64-linux/bin/devfile: /lib64/libc.so.6: version `GLIBC_2.32' not found 
(required by /opt/gitlab/embedded/lib/ruby/gems/3.1.0/gems/devfile-0.0.28.pre.alpha1-x86_64-linux/bin/devfile)

What is the expected correct behavior?

Creation of the workspace should be successful.

Relevant logs and/or screenshots

Logs related to the graphql request we are sending 
{
  "catfile.duration_ms": 0,
  "catfile.flush_count": 1,
  "catfile.flush_ms": 0,
  "catfile.read_object_count": 1,
  "catfile.read_object_ms": 0,
  "catfile.request_object_count": 1,
  "catfile.request_object_ms": 0,
  "command.count": 3,
  "command.cpu_time_ms": 2,
  "command.inblock": 0,
  "command.majflt": 0,
  "command.maxrss": 337740,
  "command.minflt": 1221,
  "command.oublock": 0,
  "command.real_time_ms": 6,
  "command.spawn_token_fork_ms": 8,
  "command.spawn_token_wait_ms": 0,
  "command.system_time_ms": 0,
  "command.user_time_ms": 2,
  "component": "gitaly.UnaryServerInterceptor",
  "correlation_id": "01JAA9Q93YKKY4D2GYNWK2AP7X",
  "grpc.code": "OK",
  "grpc.meta.auth_version": "v2",
  "grpc.meta.client_name": "gitlab-web",
  "grpc.meta.deadline_type": "regular",
  "grpc.meta.method_operation": "accessor",
  "grpc.meta.method_scope": "repository",
  "grpc.meta.method_type": "unary",
  "grpc.method": "FindCommit",
  "grpc.request.deadline": "2024-10-16T09:15:11.239",
  "grpc.request.fullMethod": "/gitaly.CommitService/FindCommit",
  "grpc.request.glProjectPath": "test-group/agent-project",
  "grpc.request.glRepository": "project-1",
  "grpc.request.payload_bytes": 139,
  "grpc.request.repoPath": "@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b.git",
  "grpc.request.repoStorage": "default",
  "grpc.response.payload_bytes": 301,
  "grpc.service": "gitaly.CommitService",
  "grpc.start_time": "2024-10-16T09:14:41.239",
  "grpc.time_ms": 11.202,
  "level": "info",
  "msg": "finished unary call with code OK",
  "pid": 18659,
  "remote_ip": "1.2.3.4",
  "span.kind": "server",
  "system": "grpc",
  "time": "2024-10-16T09:14:41.250Z",
  "user_id": "1",
  "username": "root"
}
{
  "catfile.duration_ms": 0,
  "catfile.flush_count": 3,
  "catfile.flush_ms": 0,
  "catfile.read_info_count": 1,
  "catfile.read_info_ms": 0,
  "catfile.read_object_count": 2,
  "catfile.read_object_ms": 0,
  "catfile.request_info_count": 1,
  "catfile.request_info_ms": 0,
  "catfile.request_object_count": 2,
  "catfile.request_object_ms": 0,
  "command.count": 3,
  "command.cpu_time_ms": 3,
  "command.inblock": 0,
  "command.majflt": 0,
  "command.maxrss": 337740,
  "command.minflt": 1221,
  "command.oublock": 0,
  "command.real_time_ms": 6,
  "command.spawn_token_fork_ms": 8,
  "command.spawn_token_wait_ms": 0,
  "command.system_time_ms": 3,
  "command.user_time_ms": 0,
  "component": "gitaly.StreamServerInterceptor",
  "correlation_id": "01JAA9Q93YKKY4D2GYNWK2AP7X",
  "grpc.code": "OK",
  "grpc.meta.auth_version": "v2",
  "grpc.meta.client_name": "gitlab-web",
  "grpc.meta.deadline_type": "regular",
  "grpc.meta.method_operation": "accessor",
  "grpc.meta.method_scope": "repository",
  "grpc.meta.method_type": "server_stream",
  "grpc.method": "TreeEntry",
  "grpc.request.deadline": "2024-10-16T09:15:11.253",
  "grpc.request.fullMethod": "/gitaly.CommitService/TreeEntry",
  "grpc.request.glProjectPath": "test-group/agent-project",
  "grpc.request.glRepository": "project-1",
  "grpc.request.payload_bytes": 195,
  "grpc.request.repoPath": "@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b.git",
  "grpc.request.repoStorage": "default",
  "grpc.response.payload_bytes": 454,
  "grpc.service": "gitaly.CommitService",
  "grpc.start_time": "2024-10-16T09:14:41.253",
  "grpc.time_ms": 12.052,
  "level": "info",
  "msg": "finished streaming call with code OK",
  "pid": 18659,
  "remote_ip": "1.2.3.4",
  "span.kind": "server",
  "system": "grpc",
  "time": "2024-10-16T09:14:41.265Z",
  "user_id": "1",
  "username": "root"
}
{
  "severity": "INFO",
  "time": "2024-10-16T09:14:41.273Z",
  "correlation_id": "01JAA9Q93YKKY4D2GYNWK2AP7X",
  "meta.caller_id": "graphql:workspaceCreate",
  "meta.feature_category": "workspaces",
  "meta.remote_ip": "1.2.3.4",
  "meta.user": "root",
  "meta.user_id": 1,
  "meta.client_id": "user/1",
  "trace_type": "execute_query",
  "query_fingerprint": "workspaceCreate/Qf6U91yWMtXHvl5VWwHyZ-bpsSctUk9fa97GTL7qQNA=/1/7RLcF5bugvddeswST8rPQaZT9fDmQN5AJDa75OyBAjY=",
  "duration_s": 0.08520205800596159,
  "operation_name": "workspaceCreate",
  "operation_fingerprint": "workspaceCreate/Qf6U91yWMtXHvl5VWwHyZ-bpsSctUk9fa97GTL7qQNA=",
  "is_mutation": true,
  "variables": "{\"input\"=>{\"projectId\"=>\"gid://gitlab/Project/1\", \"clusterAgentId\"=>\"gid://gitlab/Clusters::Agent/1\", \"editor\"=>\"webide\", \"desiredState\"=>\"Running\", \"devfileRef\"=>\"main\", \"devfilePath\"=>\".devfile.yaml\", \"maxHoursBeforeTermination\"=>24, \"variables\"=>\"[FILTERED]\"}}",
  "query_string": "mutation workspaceCreate {\n  workspaceCreate(input: {projectId: \"\", clusterAgentId: \"\", editor: \"\", desiredState: \"\", devfileRef: \"\", devfilePath: \"\", maxHoursBeforeTermination: 24, variables: []}) {\n    workspace {\n      ...WorkspaceItem\n      __typename\n    }\n    errors\n    __typename\n  }\n}\n\nfragment WorkspaceItem on Workspace {\n  id\n  name\n  namespace\n  projectId\n  desiredState\n  actualState\n  url\n  devfileRef\n  devfilePath\n  devfileWebUrl\n  maxHoursBeforeTermination\n  createdAt\n  __typename\n}",
  "query_analysis.duration_s": 0.0022897870076121762,
  "query_analysis.complexity": 17,
  "query_analysis.depth": 3,
  "query_analysis.used_fields": [
    "Workspace.id",
    "Workspace.name",
    "Workspace.namespace",
    "Workspace.projectId",
    "Workspace.desiredState",
    "Workspace.actualState",
    "Workspace.url",
    "Workspace.devfileRef",
    "Workspace.devfilePath",
    "Workspace.devfileWebUrl",
    "Workspace.maxHoursBeforeTermination",
    "Workspace.createdAt",
    "Workspace.__typename",
    "WorkspaceCreatePayload.workspace",
    "WorkspaceCreatePayload.errors",
    "WorkspaceCreatePayload.__typename",
    "Mutation.workspaceCreate"
  ],
  "query_analysis.used_deprecated_fields": [],
  "query_analysis.used_deprecated_arguments": []
}
{
  "method": "POST",
  "path": "/api/graphql",
  "format": "*/*",
  "controller": "GraphqlController",
  "action": "execute",
  "status": 200,
  "time": "2024-10-16T09:14:41.275Z",
  "params": [
    {
      "key": "operationName",
      "value": "workspaceCreate"
    },
    {
      "key": "variables",
      "value": "[FILTERED]"
    },
    {
      "key": "query",
      "value": "mutation workspaceCreate($input: WorkspaceCreateInput!) {\n  workspaceCreate(input: $input) {\n    workspace {\n      ...WorkspaceItem\n      __typename\n    }\n    errors\n    __typename\n  }\n}\n\nfragment WorkspaceItem on Workspace {\n  id\n  name\n  namespace\n  projectId\n  desiredState\n  actualState\n  url\n  devfileRef\n  devfilePath\n  devfileWebUrl\n  maxHoursBeforeTermination\n  createdAt\n  __typename\n}\n"
    },
    {
      "key": "graphql",
      "value": {
        "operationName": "workspaceCreate",
        "variables": "[FILTERED]",
        "query": "mutation workspaceCreate($input: WorkspaceCreateInput!) {\n  workspaceCreate(input: $input) {\n    workspace {\n      ...WorkspaceItem\n      __typename\n    }\n    errors\n    __typename\n  }\n}\n\nfragment WorkspaceItem on Workspace {\n  id\n  name\n  namespace\n  projectId\n  desiredState\n  actualState\n  url\n  devfileRef\n  devfilePath\n  devfileWebUrl\n  maxHoursBeforeTermination\n  createdAt\n  __typename\n}\n"
      }
    }
  ],
  "correlation_id": "01JAA9Q93YKKY4D2GYNWK2AP7X",
  "meta.caller_id": "GraphqlController#execute",
  "meta.feature_category": "workspaces",
  "meta.remote_ip": "1.2.3.4",
  "meta.user": "root",
  "meta.user_id": 1,
  "meta.client_id": "user/1",
  "graphql": [
    {
      "complexity": 17,
      "depth": 3,
      "used_fields": [
        "Workspace.id",
        "Workspace.name",
        "Workspace.namespace",
        "Workspace.projectId",
        "Workspace.desiredState",
        "Workspace.actualState",
        "Workspace.url",
        "Workspace.devfileRef",
        "Workspace.devfilePath",
        "Workspace.devfileWebUrl",
        "Workspace.maxHoursBeforeTermination",
        "Workspace.createdAt",
        "Workspace.__typename",
        "WorkspaceCreatePayload.workspace",
        "WorkspaceCreatePayload.errors",
        "WorkspaceCreatePayload.__typename",
        "Mutation.workspaceCreate"
      ],
      "used_deprecated_fields": [],
      "used_deprecated_arguments": [],
      "variables": "{\"input\"=>{\"projectId\"=>\"gid://gitlab/Project/1\", \"clusterAgentId\"=>\"gid://gitlab/Clusters::Agent/1\", \"editor\"=>\"webide\", \"desiredState\"=>\"Running\", \"devfileRef\"=>\"main\", \"devfilePath\"=>\".devfile.yaml\", \"maxHoursBeforeTermination\"=>24, \"variables\"=>\"[FILTERED]\"}}",
      "operation_name": "workspaceCreate"
    }
  ],
  "remote_ip": "1.2.3.4",
  "user_id": 1,
  "username": "root",
  "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
  "queue_duration_s": 0.015803,
  "request_urgency": "low",
  "target_duration_s": 5,
  "gitaly_calls": 2,
  "gitaly_duration_s": 0.027907,
  "redis_calls": 10,
  "redis_allowed_cross_slot_calls": 1,
  "redis_duration_s": 0.001521,
  "redis_read_bytes": 297,
  "redis_write_bytes": 885,
  "redis_db_load_balancing_calls": 2,
  "redis_db_load_balancing_duration_s": 0.000243,
  "redis_db_load_balancing_write_bytes": 104,
  "redis_repository_cache_calls": 1,
  "redis_repository_cache_duration_s": 0.000191,
  "redis_repository_cache_read_bytes": 101,
  "redis_repository_cache_write_bytes": 50,
  "redis_sessions_calls": 3,
  "redis_sessions_allowed_cross_slot_calls": 1,
  "redis_sessions_duration_s": 0.000661,
  "redis_sessions_read_bytes": 180,
  "redis_sessions_write_bytes": 607,
  "redis_shared_state_calls": 4,
  "redis_shared_state_duration_s": 0.000426,
  "redis_shared_state_read_bytes": 16,
  "redis_shared_state_write_bytes": 124,
  "db_count": 12,
  "db_write_count": 0,
  "db_cached_count": 2,
  "db_txn_count": 0,
  "db_replica_txn_count": 0,
  "db_primary_txn_count": 0,
  "db_main_txn_count": 0,
  "db_ci_txn_count": 0,
  "db_main_replica_txn_count": 0,
  "db_ci_replica_txn_count": 0,
  "db_replica_count": 0,
  "db_primary_count": 12,
  "db_main_count": 12,
  "db_ci_count": 0,
  "db_main_replica_count": 0,
  "db_ci_replica_count": 0,
  "db_replica_cached_count": 0,
  "db_primary_cached_count": 2,
  "db_main_cached_count": 2,
  "db_ci_cached_count": 0,
  "db_main_replica_cached_count": 0,
  "db_ci_replica_cached_count": 0,
  "db_replica_wal_count": 0,
  "db_primary_wal_count": 0,
  "db_main_wal_count": 0,
  "db_ci_wal_count": 0,
  "db_main_replica_wal_count": 0,
  "db_ci_replica_wal_count": 0,
  "db_replica_wal_cached_count": 0,
  "db_primary_wal_cached_count": 0,
  "db_main_wal_cached_count": 0,
  "db_ci_wal_cached_count": 0,
  "db_main_replica_wal_cached_count": 0,
  "db_ci_replica_wal_cached_count": 0,
  "db_replica_txn_max_duration_s": 0,
  "db_primary_txn_max_duration_s": 0,
  "db_main_txn_max_duration_s": 0,
  "db_ci_txn_max_duration_s": 0,
  "db_main_replica_txn_max_duration_s": 0,
  "db_ci_replica_txn_max_duration_s": 0,
  "db_replica_txn_duration_s": 0,
  "db_primary_txn_duration_s": 0,
  "db_main_txn_duration_s": 0,
  "db_ci_txn_duration_s": 0,
  "db_main_replica_txn_duration_s": 0,
  "db_ci_replica_txn_duration_s": 0,
  "db_replica_duration_s": 0,
  "db_primary_duration_s": 0.01,
  "db_main_duration_s": 0.01,
  "db_ci_duration_s": 0,
  "db_main_replica_duration_s": 0,
  "db_ci_replica_duration_s": 0,
  "cpu_s": 0.09055,
  "mem_objects": 20745,
  "mem_bytes": 2010312,
  "mem_mallocs": 6042,
  "mem_total_bytes": 2840112,
  "pid": 21945,
  "worker_id": "puma_0",
  "rate_limiting_gates": [],
  "db_duration_s": 0.00845,
  "view_duration_s": 0.00027,
  "duration_s": 0.09929
}
{
  "backend_id": "rails",
  "content_type": "application/json; charset=utf-8",
  "correlation_id": "01JAA9Q93YKKY4D2GYNWK2AP7X",
  "duration_ms": 129,
  "host": "g2.atanayno-4bfbd21c.gcp.gitlabsandbox.net",
  "level": "info",
  "method": "POST",
  "msg": "access",
  "proto": "HTTP/1.1",
  "referrer": "https://g2.atanayno-4bfbd21c.gcp.gitlabsandbox.net/-/remote_development/workspaces/new?project=test-group%2Fagent-project",
  "remote_addr": "1.2.3.4:0",
  "remote_ip": "1.2.3.4",
  "route": "^/api/graphql\\z",
  "route_id": "api_graphql",
  "status": 200,
  "system": "http",
  "time": "2024-10-16T09:14:41Z",
  "ttfb_ms": 129,
  "uri": "/api/graphql",
  "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
  "written_bytes": 658
}

Additional information

  • We mention that workspaces containers require glibc 2.28 or later in the docs at Workspace container requirements, but we do not say anything about GitLab instance. glibc 2.17 is shipped with CentOS 7.
  • While CentOS 7 is EOL, we still ship packages for it till 17.6 (omnibus-gitlab#8447 (closed)), so I suppose it should still work.
  • The version of devfile gem was bumped to 0.0.28 in 17.4: !165888 (merged), it's likely related to the issue.
  • We also bumped golang version in gitlab-org/ruby/gems/devfile-gem!59 (merged) for devfile gem.
  • It was working fine in GitLab 17.3 for the customer that reported this issue.
Edited by zli