Gitlab 18.6 fails to import LFS files due to HTTP 429
Summary
When importing a Github repo via the Gitlab Web UI, the import fails to import any LFS files.
In the importer errors, we see:
{
"feature_category": "importers",
"import_type": "github",
"severity": "ERROR",
"time": "2025-11-26T16:28:19.694Z",
"correlation_id": "01KAZ5QGK950P1QS4C5MRVHCEB",
"meta.caller_id": "Gitlab::GithubImport::Stage::ImportLfsObjectsWorker",
"meta.feature_category": "importers",
"meta.organization_id": 1,
"meta.remote_ip": "REDACTED_IP",
"meta.user": "REDACTED_USER",
"meta.gl_user_id": 5,
"meta.project": "REDACTED_GROUP/REDACTED_REPO",
"meta.root_namespace": "REDACTED_GROUP",
"meta.client_id": "user/5",
"meta.root_caller_id": "Import::GithubController#create",
"project_id": 4,
"source": "Gitlab::GithubImport::Importer::LfsObjectImporter",
"external_identifiers": {},
"message": "importer failed",
"exception.message": "The LFS objects download list couldn't be imported. Error: Too Many Requests"
}Associated trace
{
"severity": "ERROR",
"time": "2025-11-26T16:28:19.695Z",
"correlation_id": "01KAZ5QGK950P1QS4C5MRVHCEB",
"meta.caller_id": "Gitlab::GithubImport::Stage::ImportLfsObjectsWorker",
"meta.feature_category": "importers",
"meta.organization_id": 1,
"meta.remote_ip": "REDACTED_IP",
"meta.user": "REDACTED_USER",
"meta.gl_user_id": 5,
"meta.project": "REDACTED_GROUP/REDACTED_PROJECT",
"meta.root_namespace": "REDACTED_GROUP",
"meta.client_id": "user/5",
"meta.root_caller_id": "Import::GithubController#create",
"exception.class": "Projects::LfsPointers::LfsObjectDownloadListService::LfsObjectDownloadListError",
"exception.message": "The LFS objects download list couldn't be imported. Error: Too Many Requests",
"exception.backtrace": [
"app/services/projects/lfs_pointers/lfs_object_download_list_service.rb:26:in `rescue in each_list_item'",
"app/services/projects/lfs_pointers/lfs_object_download_list_service.rb:17:in `each_list_item'",
"lib/gitlab/github_import/importer/lfs_objects_importer.rb:32:in `each_object_to_import'",
"lib/gitlab/github_import/parallel_scheduling.rb:99:in `spread_parallel_import'",
"lib/gitlab/github_import/parallel_scheduling.rb:92:in `parallel_import'",
"lib/gitlab/github_import/parallel_scheduling.rb:47:in `execute'",
"app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb:22:in `import'",
"app/workers/concerns/gitlab/github_import/stage_methods.rb:65:in `try_import'",
"app/workers/concerns/gitlab/github_import/stage_methods.rb:47:in `perform'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:220:in `execute_job'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:185:in `block (4 levels) in process'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:180:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"ee/lib/gitlab/sidekiq_middleware/set_session/server.rb:21:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/current_organization/server.rb:21:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/identity/restore.rb:12:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/resource_usage_limit/middleware.rb:16:in `perform'",
"lib/gitlab/sidekiq_middleware/resource_usage_limit/server.rb:8:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb:35:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/skip_jobs.rb:51:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/throttling/middleware.rb:18:in `perform'",
"lib/gitlab/sidekiq_middleware/throttling/server.rb:8:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/pause_control/strategies/base.rb:31:in `perform'",
"lib/gitlab/sidekiq_middleware/pause_control/strategy_handler.rb:22:in `perform'",
"lib/gitlab/sidekiq_middleware/pause_control/server.rb:8:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/concurrency_limit/middleware.rb:44:in `perform'",
"lib/gitlab/sidekiq_middleware/concurrency_limit/server.rb:8:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/click_house/migration_support/sidekiq_middleware.rb:7:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'",
"lib/gitlab/application_context.rb:177:in `block in use'",
"gitlab-labkit (0.42.2) lib/labkit/context.rb:43:in `with_context'",
"lib/gitlab/application_context.rb:177:in `use'",
"lib/gitlab/application_context.rb:99:in `with_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:83:in `within'",
"lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/set_ip_address.rb:7:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:111:in `block in call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:139:in `block in instrument'",
"lib/gitlab/metrics/background_transaction.rb:33:in `run'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:139:in `instrument'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:110:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:8:in `block in call'",
"gems/gitlab-safe_request_store/lib/gitlab/safe_request_store.rb:66:in `enabling_request_store'",
"gems/gitlab-safe_request_store/lib/gitlab/safe_request_store.rb:59:in `ensure_request_store'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:7:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"gitlab-labkit (0.42.2) lib/labkit/middleware/sidekiq/server.rb:22:in `block in call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:180:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"gitlab-labkit (0.42.2) lib/labkit/middleware/sidekiq/user_experience_sli/server.rb:16:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"gitlab-labkit (0.42.2) lib/labkit/middleware/sidekiq/context/server.rb:16:in `block in call'",
"gitlab-labkit (0.42.2) lib/labkit/context.rb:43:in `with_context'",
"gitlab-labkit (0.42.2) lib/labkit/middleware/sidekiq/context/server.rb:15:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:173:in `invoke'",
"gitlab-labkit (0.42.2) lib/labkit/middleware/sidekiq/server.rb:21:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'",
"lib/gitlab/sidekiq_daemon/monitor.rb:46:in `within_job'",
"lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/shard_awareness_validator.rb:10:in `block in call'",
"lib/gitlab/sidekiq_sharding/validator.rb:42:in `enabled'",
"lib/gitlab/sidekiq_middleware/shard_awareness_validator.rb:9:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"marginalia (1.11.1) lib/marginalia/sidekiq_instrumentation.rb:9:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"sentry-sidekiq (5.23.0) lib/sentry/sidekiq/sentry_context_middleware.rb:54:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"vendor/gems/sidekiq/lib/sidekiq/job/interrupt_handler.rb:9:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
"vendor/gems/sidekiq/lib/sidekiq/metrics/tracking.rb:26:in `track'",
"vendor/gems/sidekiq/lib/sidekiq/metrics/tracking.rb:134:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
"vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:173:in `invoke'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:184:in `block (3 levels) in process'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:145:in `block (6 levels) in dispatch'",
"vendor/gems/sidekiq/lib/sidekiq/job_retry.rb:118:in `local'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:144:in `block (5 levels) in dispatch'",
"vendor/gems/sidekiq/lib/sidekiq/rails.rb:27:in `block in call'",
"activesupport (7.1.6) lib/active_support/reloader.rb:77:in `block in wrap'",
"activesupport (7.1.6) lib/active_support/execution_wrapper.rb:92:in `wrap'",
"activesupport (7.1.6) lib/active_support/reloader.rb:74:in `wrap'",
"vendor/gems/sidekiq/lib/sidekiq/rails.rb:26:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:139:in `block (4 levels) in dispatch'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:281:in `stats'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:134:in `block (3 levels) in dispatch'",
"lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:133:in `block (2 levels) in dispatch'",
"vendor/gems/sidekiq/lib/sidekiq/job_retry.rb:85:in `global'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:132:in `block in dispatch'",
"vendor/gems/sidekiq/lib/sidekiq/job_logger.rb:40:in `prepare'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:131:in `dispatch'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:183:in `block (2 levels) in process'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:182:in `handle_interrupt'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:182:in `block in process'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:181:in `handle_interrupt'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:181:in `process'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:86:in `process_one'",
"vendor/gems/sidekiq/lib/sidekiq/processor.rb:76:in `run'",
"vendor/gems/sidekiq/lib/sidekiq/component.rb:10:in `watchdog'",
"vendor/gems/sidekiq/lib/sidekiq/component.rb:19:in `block in safe_thread'"
],
"exception.cause_class": "Projects::LfsPointers::LfsDownloadLinkListService::DownloadLinksError",
"user.username": "REDACTED_USER",
"tags.queue": "default",
"tags.jid": "f74a8f0865d0b7936146bd09",
"tags.program": "sidekiq",
"tags.locale": "en",
"tags.feature_category": "importers",
"tags.correlation_id": "01KAZ5QGK950P1QS4C5MRVHCEB",
"extra.import_type": "github",
"extra.project_id": 4,
"extra.source": "Gitlab::GithubImport::Importer::LfsObjectImporter",
"extra.external_identifiers": {}
}It seems like the LFS import logic is failing to download the initial list of LFS files with an HTTP 429 and then never retries. This import is run overnight during off-peak periods where Github request traffic should be minimal - so the 429s are not expected and don't seem to be handled well.
we have also confirmed we have the mirroring_lfs_optimization feature flag turned off to avoid the issue in #580340 (closed):
irb(main):001> Feature.enabled?(:mirroring_lfs_optimization)
=> falseSteps to reproduce
- Have a Github repo with several thousands of LFS files and 10s of thousands of other objects
- Attempt to import the repo into Gitlab
What is the current bug behavior?
All git LFS files fail to import.
What is the expected correct behavior?
All git LFS files
Relevant logs and/or screenshots
Output of checks
This happens on self-hosted.
Results of GitLab environment info
System information
System: Ubuntu 22.04
Proxy: no
Current User: git
Using RVM: no
Ruby Version: 3.2.8
Gem Version: 3.7.1
Bundler Version:2.7.1
Rake Version: 13.0.6
Redis Version: 7.2.11
Sidekiq Version:7.3.9
Go Version: unknown
GitLab information
Version: 18.6.0-ee
Revision: 5927b4ad8ec
Directory: /opt/gitlab/embedded/service/gitlab-rails
DB Adapter: PostgreSQL
DB Version: 16.10
URL: https://REDACTED_HOST
HTTP Clone URL: https://REDACTED_HOST/some-group/some-project.git
SSH Clone URL: ssh://git@REDACTED_HOST:2222/some-group/some-project.git
Elasticsearch: no
Geo: no
Using LDAP: no
Using Omniauth: yes
Omniauth Providers: openid_connect
GitLab Shell
Version: 14.45.3
Repository storages:
- default: tcp://REDACTED_IP:8075
- storage1: tcp://REDACTED_IP:8075
GitLab Shell path: /opt/gitlab/embedded/service/gitlab-shell
Gitaly
- default Address: tcp://REDACTED_IP:8075
- default Version: 18.6.0
- default Git Version: 2.50.1
- storage1 Address: tcp://REDACTED_IP:8075
- storage1 Version: 18.6.0
- storage1 Git Version: 2.50.1Results of GitLab application Check
Expand for output related to the GitLab application check
Checking GitLab subtasks ... Checking GitLab Shell ... GitLab Shell: ... GitLab Shell version \\\\\\\\\\\\\\\>= 14.45.3 ? ... OK (14.45.3) Running /opt/gitlab/embedded/service/gitlab-shell/bin/gitlab-shell-check Internal API available: OK Redis available via internal API: OK gitlab-shell self-check successful Checking GitLab Shell ... Finished Checking Gitaly ... Gitaly: ... default ... OK storage1 ... OK Checking Gitaly ... Finished Checking Sidekiq ... Sidekiq: ... Running? ... no Try fixing it: sudo -u git -H RAILS_ENV=production bin/background_jobs start For more information see: doc/install/installation.md in section "Install Init Script" see log/sidekiq.log for possible errors Please fix the error above and rerun the checks. Checking Sidekiq ... Finished Checking Incoming Email ... Incoming Email: ... Reply by email is disabled in config/gitlab.yml Checking Incoming Email ... Finished Checking LDAP ... LDAP: ... LDAP is disabled in config/gitlab.yml Checking LDAP ... Finished Checking GitLab App ... Database config exists? ... yes Tables are truncated? ... skipped All migrations up? ... yes Database contains orphaned GroupMembers? ... no GitLab config exists? ... yes GitLab config up to date? ... yes Cable config exists? ... yes Resque config exists? ... yes Log directory writable? ... yes Tmp directory writable? ... yes Uploads directory exists? ... yes Uploads directory has correct permissions? ... yes Uploads directory tmp has correct permissions? ... yes Systemd unit files or init script exist? ... skipped (omnibus-gitlab has neither init script nor systemd units) Systemd unit files or init script up-to-date? ... skipped (omnibus-gitlab has neither init script nor systemd units) Projects have namespace: ... 3/1 ... yes 3/2 ... yes 10/3 ... yes 10/4 ... yes Redis version \\\\\\\\\\\\\\\>= 6.2.14? ... yes Ruby version \\\\\\\\\\\\\\\>= 3.0.6 ? ... yes (3.2.8) Git user has default SSH configuration? ... yes Active users: ... 3 Is authorized keys file accessible? ... yes GitLab configured to store new projects in hashed storage? ... yes All projects are in hashed storage? ... yes Elasticsearch version 7.x-9.x or OpenSearch version 1.x-3.x ... skipped (advanced search is disabled) All migrations must be finished before doing a major upgrade ... skipped (Advanced Search is disabled) Checking GitLab App ... Finished Checking GitLab subtasks ... Finished
Possible fixes
See traceback for the most info we have here.
Edited by Thiago Figueiró