Project export should not fail hard if a single file goes missing in object storage
As seen in #292233 (comment 462762640), if a file goes missing in object storage, the project export will fail completely cause the whole export to fail. In the following case, the upload URL was a 403 from AWS:
{
"severity": "ERROR",
"time": "2020-12-08T23:54:00.971Z",
"correlation_id": "01ES2CDBBV28MF5EW42S44GZQA",
"extra.sidekiq": {
"class": "ProjectExportWorker",
"args": [
"1",
"14",
"",
"{}"
],
"retry": false,
"queue": "project_export",
"version": 0,
"backtrace": 5,
"dead": false,
"status_expiration": 21600,
"jid": "59cf0390cfdb8169eac54b50",
"created_at": 1607471639.9818864,
"meta.user": "root",
"meta.project": "root/simple-ci2",
"meta.root_namespace": "root",
"meta.caller_id": "ProjectsController#export",
"meta.feature_category": "importers",
"correlation_id": "01ES2CDBBV28MF5EW42S44GZQA",
"enqueued_at": 1607471639.9901834
},
"extra.importer": "Import/Export",
"extra.exportable_id": 14,
"extra.exportable_path": "root/simple-ci2",
"extra.import_jid": null,
"exception.class": "OpenURI::HTTPError",
"exception.message": "403 Forbidden",
"exception.backtrace": [
"lib/gitlab/import_export/command_line_util.rb:35:in `block in download'",
"lib/gitlab/import_export/command_line_util.rb:33:in `open'",
"lib/gitlab/import_export/command_line_util.rb:33:in `download'",
"lib/gitlab/import_export/command_line_util.rb:28:in `download_or_copy_upload'",
"lib/gitlab/import_export/uploads_manager.rb:88:in `download_and_copy'",
"lib/gitlab/import_export/uploads_manager.rb:55:in `block in copy_project_uploads'",
"lib/gitlab/import_export/uploads_manager.rb:71:in `block in each_uploader'",
"lib/gitlab/import_export/uploads_manager.rb:70:in `each_uploader'",
"lib/gitlab/import_export/uploads_manager.rb:47:in `copy_project_uploads'",
"lib/gitlab/import_export/uploads_manager.rb:17:in `save'",
"lib/gitlab/import_export/uploads_saver.rb:15:in `save'",
"app/services/projects/import_export/export_service.rb:58:in `all?'",
"app/services/projects/import_export/export_service.rb:58:in `save_exporters'",
"app/services/projects/import_export/export_service.rb:49:in `save_all!'",
"app/services/projects/import_export/export_service.rb:20:in `execute'",
"app/services/concerns/measurable.rb:35:in `execute'",
"app/workers/project_export_worker.rb:22:in `perform'",
"lib/gitlab/metrics/sidekiq_middleware.rb:18:in `block in call'",
"lib/gitlab/metrics/transaction.rb:56:in `run'",
"lib/gitlab/metrics/sidekiq_middleware.rb:18:in `call'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:40:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
"lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `block in call'",
"lib/gitlab/application_context.rb:54:in `block in use'",
"lib/gitlab/application_context.rb:54:in `use'",
"lib/gitlab/application_context.rb:21:in `with_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:15:in `call'",
"lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
"lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/admin_mode/server.rb:8:in `call'",
"lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'",
"lib/gitlab/with_request_store.rb:17:in `enabling_request_store'",
"lib/gitlab/with_request_store.rb:10:in `with_request_store'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:37:in `call'",
"lib/gitlab/sidekiq_middleware/monitor.rb:8:in `block in call'",
"lib/gitlab/sidekiq_daemon/monitor.rb:49:in `within_job'",
"lib/gitlab/sidekiq_middleware/monitor.rb:7:in `call'",
"lib/gitlab/sidekiq_logging/structured_logger.rb:18:in `call'"
]
}
Proposal:
- If we download and fail, we should log a warning and continue (MVC, this issue).
- Bundle a collection of warnings in some separate file with the export (separate issue, still needs to be discussed).
Edited by Martin Wortschack