Skip to content

Add Export to Explore Dependency list page

mo khan requested to merge mokhax/432264/export-dependencies into master

What does this MR do and why?

This change adds the ability to export dependencies from the Dependency list on the explore page.

#438743 (closed)

SQL Queries

Each batch will produce the following queries.

Get the primary key for the start of the next batch:

SELECT "sbom_occurrences"."id"
FROM "sbom_occurrences"
INNER JOIN "projects" ON "sbom_occurrences"."project_id" = "projects"."id"
WHERE "projects"."organization_id" = 1
AND "sbom_occurrences"."id" >= 2382
ORDER BY "sbom_occurrences"."id" ASC LIMIT 1
OFFSET 1000

https://console.postgres.ai/gitlab/gitlab-production-tunnel-pg12/sessions/26172/commands/82300

Get the next batch of occurrences

SELECT
  "sbom_occurrences"."id",
  "sbom_occurrences"."created_at",
  "sbom_occurrences"."updated_at",
  "sbom_occurrences"."component_version_id",
  "sbom_occurrences"."project_id",
  "sbom_occurrences"."pipeline_id",
  "sbom_occurrences"."source_id",
  "sbom_occurrences"."commit_sha",
  "sbom_occurrences"."component_id",
  "sbom_occurrences"."uuid",
  "sbom_occurrences"."package_manager",
  "sbom_occurrences"."component_name",
  "sbom_occurrences"."input_file_path",
  "sbom_occurrences"."licenses",
  "sbom_occurrences"."highest_severity",
  "sbom_occurrences"."vulnerability_count",
  "sbom_occurrences"."source_package_id",
  "sbom_occurrences"."archived",
  "sbom_occurrences"."traversal_ids"
FROM "sbom_occurrences"
INNER JOIN "projects" ON "sbom_occurrences"."project_id" = "projects"."id"
WHERE "projects"."organization_id" = 1
AND "sbom_occurrences"."id" >= 2382
AND "sbom_occurrences"."id" < 3382

https://console.postgres.ai/gitlab/gitlab-production-tunnel-pg12/sessions/26172/commands/82301

Get the SBoM sources for the current batch of occurrences

SELECT "sbom_sources".*
FROM "sbom_sources"
WHERE "sbom_sources"."id" IN (556, 557)

Get the SBoM component versions for the current batch of occurences

This provides the data for the :version key in the export.

SELECT "sbom_component_versions".*
FROM "sbom_component_versions"
WHERE "sbom_component_versions"."id" IN (2382, 2383, 2384, 2385, 2386, 2387, 2388, 2389, 2390, 2391)

Get the Projects for the current batch of occurrences

This is used to build a URL to the file where the occurrence was detected.

SELECT
  "projects"."id",
  "projects"."name",
  "projects"."path",
  "projects"."description",
  "projects"."created_at",
  "projects"."updated_at",
  "projects"."creator_id",
  "projects"."namespace_id",
  "projects"."last_activity_at",
  "projects"."import_url",
  "projects"."visibility_level",
  "projects"."archived",
  "projects"."avatar",
  "projects"."merge_requests_template",
  "projects"."star_count",
  "projects"."merge_requests_rebase_enabled",
  "projects"."import_type",
  "projects"."import_source",
  "projects"."approvals_before_merge",
  "projects"."reset_approvals_on_push",
  "projects"."merge_requests_ff_only_enabled",
  "projects"."issues_template",
  "projects"."mirror",
  "projects"."mirror_last_update_at",
  "projects"."mirror_last_successful_update_at",
  "projects"."mirror_user_id",
  "projects"."shared_runners_enabled",
  "projects"."runners_token",
  "projects"."build_allow_git_fetch",
  "projects"."build_timeout",
  "projects"."mirror_trigger_builds",
  "projects"."pending_delete",
  "projects"."public_builds",
  "projects"."last_repository_check_failed",
  "projects"."last_repository_check_at",
  "projects"."only_allow_merge_if_pipeline_succeeds",
  "projects"."has_external_issue_tracker",
  "projects"."repository_storage",
  "projects"."repository_read_only",
  "projects"."request_access_enabled",
  "projects"."has_external_wiki",
  "projects"."ci_config_path",
  "projects"."lfs_enabled",
  "projects"."description_html",
  "projects"."only_allow_merge_if_all_discussions_are_resolved",
  "projects"."repository_size_limit",
  "projects"."printing_merge_request_link_enabled",
  "projects"."auto_cancel_pending_pipelines",
  "projects"."service_desk_enabled",
  "projects"."cached_markdown_version",
  "projects"."delete_error",
  "projects"."last_repository_updated_at",
  "projects"."disable_overriding_approvers_per_merge_request",
  "projects"."storage_version",
  "projects"."resolve_outdated_diff_discussions",
  "projects"."remote_mirror_available_overridden",
  "projects"."only_mirror_protected_branches",
  "projects"."pull_mirror_available_overridden",
  "projects"."jobs_cache_index",
  "projects"."external_authorization_classification_label",
  "projects"."mirror_overwrites_diverged_branches",
  "projects"."pages_https_only",
  "projects"."external_webhook_token",
  "projects"."packages_enabled",
  "projects"."merge_requests_author_approval",
  "projects"."pool_repository_id",
  "projects"."runners_token_encrypted",
  "projects"."bfg_object_map",
  "projects"."detected_repository_languages",
  "projects"."merge_requests_disable_committers_approval",
  "projects"."require_password_to_approve",
  "projects"."max_pages_size",
  "projects"."max_artifacts_size",
  "projects"."pull_mirror_branch_prefix",
  "projects"."remove_source_branch_after_merge",
  "projects"."marked_for_deletion_at",
  "projects"."marked_for_deletion_by_user_id",
  "projects"."autoclose_referenced_issues",
  "projects"."suggestion_commit_message",
  "projects"."project_namespace_id",
  "projects"."hidden",
  "projects"."organization_id"
FROM "projects"
WHERE "projects"."id" IN (4642, 4644)

Get the Namespace for each project in the current batch of occurrences

This is used to build a URL to the file where the occurrence was detected.

SELECT
  "namespaces"."id",
  "namespaces"."name",
  "namespaces"."path",
  "namespaces"."owner_id",
  "namespaces"."created_at",
  "namespaces"."updated_at",
  "namespaces"."type",
  "namespaces"."description",
  "namespaces"."avatar",
  "namespaces"."membership_lock",
  "namespaces"."share_with_group_lock",
  "namespaces"."visibility_level",
  "namespaces"."request_access_enabled",
  "namespaces"."ldap_sync_status",
  "namespaces"."ldap_sync_error",
  "namespaces"."ldap_sync_last_update_at",
  "namespaces"."ldap_sync_last_successful_update_at",
  "namespaces"."ldap_sync_last_sync_at",
  "namespaces"."description_html",
  "namespaces"."lfs_enabled",
  "namespaces"."parent_id",
  "namespaces"."shared_runners_minutes_limit",
  "namespaces"."repository_size_limit",
  "namespaces"."require_two_factor_authentication",
  "namespaces"."two_factor_grace_period",
  "namespaces"."cached_markdown_version",
  "namespaces"."project_creation_level",
  "namespaces"."runners_token",
  "namespaces"."file_template_project_id",
  "namespaces"."saml_discovery_token",
  "namespaces"."runners_token_encrypted",
  "namespaces"."custom_project_templates_group_id",
  "namespaces"."auto_devops_enabled",
  "namespaces"."extra_shared_runners_minutes_limit",
  "namespaces"."last_ci_minutes_notification_at",
  "namespaces"."last_ci_minutes_usage_notification_level",
  "namespaces"."subgroup_creation_level",
  "namespaces"."emails_disabled",
  "namespaces"."max_pages_size",
  "namespaces"."max_artifacts_size",
  "namespaces"."mentions_disabled",
  "namespaces"."default_branch_protection",
  "namespaces"."max_personal_access_token_lifetime",
  "namespaces"."push_rule_id",
  "namespaces"."shared_runners_enabled",
  "namespaces"."allow_descendants_override_disabled_shared_runners",
  "namespaces"."traversal_ids",
  "namespaces"."organization_id"
FROM "namespaces"
WHERE "namespaces"."id" IN (9319, 9323)

Get the route for each Namespace in the current batch

This is used to build a URL to the file that the occurrence was detected in.

SELECT "routes".*
FROM "routes"
WHERE "routes"."source_type" = 'Namespace'
AND "routes"."source_id" IN (9319, 9323)

MR acceptance checklist

Please evaluate this MR against the MR acceptance checklist. It helps you analyze changes to reduce risks in quality, performance, reliability, security, and maintainability.

Screenshots or screen recordings

Before After
before after

How to set up and validate locally

  1. In rails console run the following
    ::Dependencies::CreateExportService.new(Organizations::Organization.default_organization, User.find(1)).execute.as_json
Edited by mo khan

Merge request reports