Potential for large # records loaded into memory during indexing
There are repeated WARN
logs in the background jobs logs generated from the preload_database_records
method in document_reference.rb
. The warnings are from ActiveRecord. We should look into using batches here to prevent very large amounts of records being loaded into memory.
Steps to reproduce
- verify that gdk has Elasticsearch enabled
- ensure you have projects with large number of issues, notes, and merge requests loaded into gdk
- follow the background jobs logs -
gdk tail rails-background-jobs
- recreate the Elasticsearch indexes from scratch -
bundle exec rake gitlab:elastic:index
(Note: this will delete and recreate the Elasticsearch indexes and make take awhile) - you should see warnings in the logs about the number of records being fetched during indexing
The following discussion from !79949 (merged) should be addressed:
-
@terrichu started a discussion: (+1 comment) I have noticed some warnings in the background jobs logs from the
preload_database_records
method indocument_reference.rb
. I can't tell where the warnings are being generated from and they appear to be related to the large number of records fetched by the queries. I'll do some more digging and report back.2022-04-05_19:44:57.94495 rails-background-jobs : {"severity":"WARN","time":"2022-04-05T19:44:57.944Z","message":"Query fetched 7887 Issue records: SELECT \"namespaces\".\"id\", \"namespaces\".\"name\", \"namespaces\".\"path\", \"namespaces\".\"owner_id\", \"namespaces\".\"created_at\", \"namespaces\".\"updated_at\", \"namespaces\".\"type\", \"namespaces\".\"description\", \"namespaces\".\"avatar\", \"namespaces\".\"membership_lock\", \"namespaces\".\"share_with_group_lock\", \"namespaces\".\"visibility_level\", \"namespaces\".\"request_access_enabled\", \"namespaces\".\"ldap_sync_status\", \"namespaces\".\"ldap_sync_error\", \"namespaces\".\"ldap_sync_last_update_at\", \"namespaces\".\"ldap_sync_last_successful_update_at\", \"namespaces\".\"ldap_sync_last_sync_at\", \"namespaces\".\"description_html\", \"namespaces\".\"lfs_enabled\", \"namespaces\".\"parent_id\", \"namespaces\".\"shared_runners_minutes_limit\", \"namespaces\".\"repository_size_limit\", \"namespaces\".\"require_two_factor_authentication\", \"namespaces\".\"two_factor_grace_period\", \"namespaces\".\"cached_markdown_version\", \"namespaces\".\"project_creation_level\", \"namespaces\".\"runners_token\", \"namespaces\".\"file_template_project_id\", \"namespaces\".\"saml_discovery_token\", \"namespaces\".\"runners_token_encrypted\", \"namespaces\".\"custom_project_templates_group_id\", \"namespaces\".\"auto_devops_enabled\", \"namespaces\".\"extra_shared_runners_minutes_limit\", \"namespaces\".\"last_ci_minutes_notification_at\", \"namespaces\".\"last_ci_minutes_usage_notification_level\", \"namespaces\".\"subgroup_creation_level\", \"namespaces\".\"emails_disabled\", \"namespaces\".\"max_pages_size\", \"namespaces\".\"max_artifacts_size\", \"namespaces\".\"mentions_disabled\", \"namespaces\".\"default_branch_protection\", \"namespaces\".\"unlock_membership_to_ldap\", \"namespaces\".\"max_personal_access_token_lifetime\", \"namespaces\".\"push_rule_id\", \"namespaces\".\"shared_runners_enabled\", \"namespaces\".\"allow_descendants_override_disabled_shared_runners\", \"namespaces\".\"traversal_ids\" FROM \"namespaces\" WHERE \"namespaces\".\"id\" IN (110, 22, 28, 1, 17, 25, 52, 27, 23, 5, 19, 26, 16, 24, 7, 2, 4, 15, 107, 20) /*application:sidekiq,correlation_id:17f8f1aae7bf73255fc2500fb6de6266,jid:a3da2b002b7ea6ef3565d8fd,endpoint_id:ElasticIndexInitialBulkCronWorker,db_config_name:main,line:/ee/lib/gitlab/elastic/document_reference.rb:35:in `block in preload_database_records'*/"} 2022-04-05_19:45:00.00834 rails-background-jobs : {"severity":"WARN","time":"2022-04-05T19:45:00.008Z","message":"Query fetched 3681 Note records: SELECT \"routes\".* FROM \"routes\" WHERE \"routes\".\"source_type\" = 'Project' AND \"routes\".\"source_id\" IN (2, 1, 43, 8, 5, 7, 3, 6, 4, 32) /*application:sidekiq,correlation_id:17f8f1aae7bf73255fc2500fb6de6266,jid:a3da2b002b7ea6ef3565d8fd,endpoint_id:ElasticIndexInitialBulkCronWorker,db_config_name:main,line:/ee/lib/gitlab/elastic/document_reference.rb:35:in `block in preload_database_records'*/"} 2022-04-05_19:45:00.47723 rails-background-jobs : {"severity":"INFO","time":"2022-04-05T19:45:00.476Z","retry":0,"queue":"default","backtrace":true,"version":0,"queue_namespace":"cronjob","args":[],"class":"ScheduleMergeRequestCleanupRefsWorker","jid":"98ec1afd27e77146e24c25ee","created_at":"2022-04-05T19:44:33.229Z","meta.caller_id":"Cronjob","correlation_id":"c43956edc25a02168f74d46380dde47c","meta.feature_category":"code_review","uber-trace-id":"35ce2a03d50f46fe:35ce2a03d50f46fe:0:1","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:default:33e8a9dcd4c9780ad0ea123ad7ccbabde1aa1e90ffcbb928434ba4b5800a5811","size_limiter":"validated","enqueued_at":"2022-04-05T19:44:33.230Z","job_size_bytes":2,"pid":39562,"message":"ScheduleMergeRequestCleanupRefsWorker JID-98ec1afd27e77146e24c25ee: done: 12.211456 sec","job_status":"done","scheduling_latency_s":15.034695,"redis_calls":2,"redis_duration_s":0.000527,"redis_read_bytes":213,"redis_write_bytes":373,"redis_cache_calls":1,"redis_cache_duration_s":0.000337,"redis_cache_read_bytes":203,"redis_cache_write_bytes":61,"redis_queues_calls":1,"redis_queues_duration_s":0.00019,"redis_queues_read_bytes":10,"redis_queues_write_bytes":312,"db_count":1,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":1,"db_main_count":1,"db_main_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_main_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_main_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.384,"db_main_duration_s":0.384,"db_main_replica_duration_s":0.0,"cpu_s":0.390895,"duration_s":12.211456,"completed_at":"2022-04-05T19:45:00.476Z","load_balancing_strategy":"primary","db_duration_s":0.00474} 2022-04-05_19:45:02.14660 rails-background-jobs : {"severity":"WARN","time":"2022-04-05T19:45:02.146Z","message":"Query fetched 4329 MergeRequest records: SELECT \"project_features\".* FROM \"project_features\" WHERE \"project_features\".\"project_id\" IN (2, 1, 43, 13, 11, 8, 5, 7, 3, 17, 18, 6, 12, 4, 10, 16, 9, 15, 26, 14) /*application:sidekiq,correlation_id:17f8f1aae7bf73255fc2500fb6de6266,jid:a3da2b002b7ea6ef3565d8fd,endpoint_id:ElasticIndexInitialBulkCronWorker,db_config_name:main,line:/ee/lib/gitlab/elastic/document_reference.rb:35:in `block in preload_database_records'*/"}
Edited by Terri Chu