Github importer failing with `undefined method 'id' for nil:NilClass`
Summary
We tested importing the nodejs library from GitHub, which contains about 24k merge requests and 12k issues on a 13.9 instance patched with !60668 (merged). While all 24k MRs were imported, many of the comments were still missing and none of the issues were imported.
Relevant logs and/or screenshots
We're seeing two kinds of entries that we think might be related. One is this undefined method 'id' for nil:NilClass
:
Click to see the log
{
"severity": "ERROR",
"time": "2021-05-06T18:02:42.184Z",
"correlation_id": "17f8d3c3aff0fb6da5f5e0feb1043ac1",
"extra.sidekiq": {
"class": "Gitlab::GithubImport::ImportPullRequestReviewWorker",
"args": [
"9",
"[FILTERED]",
"[FILTERED]"
],
"retry": 5,
"queue": "github_importer:github_import_import_pull_request_review",
"version": 0,
"queue_namespace": "github_importer",
"dead": false,
"jid": "b9d1f1fc87f744f945c72bca",
"created_at": 1620323455.9328947,
"meta.user": "root",
"meta.project": "node-with-queues/node",
"meta.root_namespace": "node-with-queues",
"meta.caller_id": "Gitlab::GithubImport::Stage::ImportPullRequestsReviewsWorker",
"meta.remote_ip": "96.231.211.84",
"meta.related_class": "Projects::CreateService",
"meta.feature_category": "importers",
"correlation_id": "17f8d3c3aff0fb6da5f5e0feb1043ac1",
"enqueued_at": 1620324162.1698716,
"error_message": "undefined method `id' for nil:NilClass",
"error_class": "NoMethodError",
"failed_at": 1620323455.9514377,
"retry_count": 4,
"retried_at": 1620323755.618828
},
"extra.import_source": "github",
"extra.project_id": 9,
"extra.importer": "Gitlab::GithubImport::Importer::PullRequestReviewImporter",
"exception.class": "NoMethodError",
"exception.message": "undefined method `id' for nil:NilClass",
"exception.backtrace": [
"lib/gitlab/github_import/user_finder.rb:66:in `user_id_for'",
"lib/gitlab/github_import/importer/pull_request_review_importer.rb:16:in `execute'",
"app/workers/concerns/gitlab/github_import/object_importer.rb:31:in `import'",
"app/workers/concerns/gitlab/github_import/rescheduling_methods.rb:31:in `try_import'",
"app/workers/concerns/gitlab/github_import/rescheduling_methods.rb:19:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:40:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
"lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `block in call'",
"lib/gitlab/application_context.rb:56:in `block in use'",
"lib/gitlab/application_context.rb:56:in `use'",
"lib/gitlab/application_context.rb:22:in `with_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:15:in `call'",
"lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
"lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/admin_mode/server.rb:8:in `call'",
"lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'",
"lib/gitlab/with_request_store.rb:17:in `enabling_request_store'",
"lib/gitlab/with_request_store.rb:10:in `with_request_store'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:37:in `call'",
"lib/gitlab/sidekiq_middleware/monitor.rb:8:in `block in call'",
"lib/gitlab/sidekiq_daemon/monitor.rb:49:in `within_job'",
"lib/gitlab/sidekiq_middleware/monitor.rb:7:in `call'",
"lib/gitlab/sidekiq_logging/structured_logger.rb:19:in `call'"
]
}
The other one is the Host cannot be resolved or invalid
:
Click to see the log
{ "severity": "ERROR", "time": "2021-05-06T19:03:07.744Z", "correlation_id": "fd379a0331526182e48523dccc242e8f", "extra.sidekiq": { "class": "Gitlab::GithubImport::Stage::ImportIssuesAndDiffNotesWorker", "args": [ "[FILTERED]" ], "retry": 5, "queue": "github_importer:github_import_stage_import_issues_and_diff_notes", "version": 0, "queue_namespace": "github_importer", "dead": false, "jid": "22add543b445beac412393ed", "created_at": 1620324398.1779258, "meta.user": "root", "meta.project": "node-with-queues/node", "meta.root_namespace": "node-with-queues", "meta.caller_id": "Gitlab::GithubImport::Stage::ImportIssuesAndDiffNotesWorker", "meta.remote_ip": "96.231.211.84", "meta.related_class": "Projects::CreateService", "meta.feature_category": "importers", "correlation_id": "fd379a0331526182e48523dccc242e8f", "enqueued_at": 1620327031.486363 }, "extra.import_source": "github", "extra.project_id": 9, "extra.import_stage": "Gitlab::GithubImport::Stage::ImportIssuesAndDiffNotesWorker", "exception.class": "Gitlab::UrlBlocker::BlockedUrlError", "exception.message": "Host cannot be resolved or invalid", "exception.backtrace": [ "lib/gitlab/url_blocker.rb:132:in `rescue in get_address_info'", "lib/gitlab/url_blocker.rb:115:in `get_address_info'", "lib/gitlab/url_blocker.rb:51:in `validate!'", "lib/gitlab/octokit/middleware.rb:11:in `call'", "lib/gitlab/github_import/client.rb:122:in `block in each_page'", "lib/gitlab/github_import/client.rb:156:in `with_rate_limit'", "lib/gitlab/github_import/client.rb:122:in `each_page'", "lib/gitlab/github_import/parallel_scheduling.rb:93:in `each_object_to_import'", "lib/gitlab/github_import/parallel_scheduling.rb:71:in `parallel_import'", "lib/gitlab/github_import/parallel_scheduling.rb:33:in `execute'", "app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb:23:in `block in import'", "app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb:21:in `each'", "app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb:21:in `each_with_object'", "app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb:21:in `import'", "app/workers/concerns/gitlab/github_import/stage_methods.rb:24:in `try_import'", "app/workers/concerns/gitlab/github_import/stage_methods.rb:14:in `perform'", "lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'", "lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:40:in `perform'", "lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'", "lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'", "lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `block in call'", "lib/gitlab/application_context.rb:56:in `block in use'", "lib/gitlab/application_context.rb:56:in `use'", "lib/gitlab/application_context.rb:22:in `with_context'", "lib/gitlab/sidekiq_middleware/worker_context/server.rb:15:in `call'", "lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'", "lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'", "lib/gitlab/sidekiq_middleware/admin_mode/server.rb:8:in `call'", "lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'", "lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'", "lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'", "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'", "lib/gitlab/with_request_store.rb:17:in `enabling_request_store'", "lib/gitlab/with_request_store.rb:10:in `with_request_store'", "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'", "lib/gitlab/sidekiq_middleware/server_metrics.rb:37:in `call'", "lib/gitlab/sidekiq_middleware/monitor.rb:8:in `block in call'", "lib/gitlab/sidekiq_daemon/monitor.rb:49:in `within_job'", "lib/gitlab/sidekiq_middleware/monitor.rb:7:in `call'", "lib/gitlab/sidekiq_logging/structured_logger.rb:19:in `call'" ] }
Edited by Kassio Borges