Project export should not fail hard if a single file goes missing in object storage

As seen in #292233 (comment 462762640), if a file goes missing in object storage, the project export will fail completely cause the whole export to fail. In the following case, the upload URL was a 403 from AWS:

{
  "severity": "ERROR",
  "time": "2020-12-08T23:54:00.971Z",
  "correlation_id": "01ES2CDBBV28MF5EW42S44GZQA",
  "extra.sidekiq": {
    "class": "ProjectExportWorker",
    "args": [
      "1",
      "14",
      "",
      "{}"
    ],
    "retry": false,
    "queue": "project_export",
    "version": 0,
    "backtrace": 5,
    "dead": false,
    "status_expiration": 21600,
    "jid": "59cf0390cfdb8169eac54b50",
    "created_at": 1607471639.9818864,
    "meta.user": "root",
    "meta.project": "root/simple-ci2",
    "meta.root_namespace": "root",
    "meta.caller_id": "ProjectsController#export",
    "meta.feature_category": "importers",
    "correlation_id": "01ES2CDBBV28MF5EW42S44GZQA",
    "enqueued_at": 1607471639.9901834
  },
  "extra.importer": "Import/Export",
  "extra.exportable_id": 14,
  "extra.exportable_path": "root/simple-ci2",
  "extra.import_jid": null,
  "exception.class": "OpenURI::HTTPError",
  "exception.message": "403 Forbidden",
  "exception.backtrace": [
    "lib/gitlab/import_export/command_line_util.rb:35:in `block in download'",
    "lib/gitlab/import_export/command_line_util.rb:33:in `open'",
    "lib/gitlab/import_export/command_line_util.rb:33:in `download'",
    "lib/gitlab/import_export/command_line_util.rb:28:in `download_or_copy_upload'",
    "lib/gitlab/import_export/uploads_manager.rb:88:in `download_and_copy'",
    "lib/gitlab/import_export/uploads_manager.rb:55:in `block in copy_project_uploads'",
    "lib/gitlab/import_export/uploads_manager.rb:71:in `block in each_uploader'",
    "lib/gitlab/import_export/uploads_manager.rb:70:in `each_uploader'",
    "lib/gitlab/import_export/uploads_manager.rb:47:in `copy_project_uploads'",
    "lib/gitlab/import_export/uploads_manager.rb:17:in `save'",
    "lib/gitlab/import_export/uploads_saver.rb:15:in `save'",
    "app/services/projects/import_export/export_service.rb:58:in `all?'",
    "app/services/projects/import_export/export_service.rb:58:in `save_exporters'",
    "app/services/projects/import_export/export_service.rb:49:in `save_all!'",
    "app/services/projects/import_export/export_service.rb:20:in `execute'",
    "app/services/concerns/measurable.rb:35:in `execute'",
    "app/workers/project_export_worker.rb:22:in `perform'",
    "lib/gitlab/metrics/sidekiq_middleware.rb:18:in `block in call'",
    "lib/gitlab/metrics/transaction.rb:56:in `run'",
    "lib/gitlab/metrics/sidekiq_middleware.rb:18:in `call'",
    "lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
    "lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:40:in `perform'",
    "lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
    "lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
    "lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `block in call'",
    "lib/gitlab/application_context.rb:54:in `block in use'",
    "lib/gitlab/application_context.rb:54:in `use'",
    "lib/gitlab/application_context.rb:21:in `with_context'",
    "lib/gitlab/sidekiq_middleware/worker_context/server.rb:15:in `call'",
    "lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
    "lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
    "lib/gitlab/sidekiq_middleware/admin_mode/server.rb:8:in `call'",
    "lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:7:in `call'",
    "lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
    "lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
    "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'",
    "lib/gitlab/with_request_store.rb:17:in `enabling_request_store'",
    "lib/gitlab/with_request_store.rb:10:in `with_request_store'",
    "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'",
    "lib/gitlab/sidekiq_middleware/server_metrics.rb:37:in `call'",
    "lib/gitlab/sidekiq_middleware/monitor.rb:8:in `block in call'",
    "lib/gitlab/sidekiq_daemon/monitor.rb:49:in `within_job'",
    "lib/gitlab/sidekiq_middleware/monitor.rb:7:in `call'",
    "lib/gitlab/sidekiq_logging/structured_logger.rb:18:in `call'"
  ]
}

Proposal:

  1. If we download and fail, we should log a warning and continue (MVC, this issue).
  2. Bundle a collection of warnings in some separate file with the export (separate issue, still needs to be discussed).
Edited Aug 30, 2022 by Martin Wortschack
Assignee Loading
Time tracking Loading