500 errors 'GetRepoPath: not a git repository' when migrated to the praefect['configuration'] and gitaly['configuration']
GET recently was updated to migrate to a new Gitaly and Praefect config - praefect['configuration']
and gitaly['configuration']
at gitlab-environment-toolkit!986 (merged)
While the new projects are created and work without issues, when navigating to an old project - there are 500 errors - 5:cmdstart failed: GetRepoPath: not a git repository: \"/var/opt/gitlab/git-data/@hashed/e7/f6/e7f6c011776e8db7cd330b54174fd76f7d0216b612387a5ffcfb81e6f0919683.wiki.git\". debug_error_string:{\"created\":\"@1680779352.646158358\",\"description\":\"Error received from peer ipv4:10.142.15.240:2305\",\"file\":\"src/core/lib/surface/call.cc\",\"file_line\":1063,\"grpc_message\":\"cmd start failed: GetRepoPath: not a git repository: \"/var/opt/gitlab/git-data/@hashed/e7/f6/e7f6c011776e8db7cd330b54174fd76f7d0216b612387a5ffcfb81e6f0919683.wiki.git\"\"
:
- /var/log/gitlab/gitlab-rails/production_json.log
Click to expand
{
"method": "GET",
"path": "/gpt/many_groups_and_projects/gpt-subgroup-8/gpt-project-1",
"format": "html",
"controller": "ProjectsController",
"action": "show",
"status": 500,
"time": "2023-04-06T11:09:12.648Z",
"params": [
{
"key": "namespace_id",
"value": "gpt/many_groups_and_projects/gpt-subgroup-8"
},
{
"key": "id",
"value": "gpt-project-1"
}
],
"correlation_id": "01GXB435317N591Y43RBPW7KG6",
"meta.caller_id": "ProjectsController#show",
"meta.remote_ip": "213.10.34.7",
"meta.feature_category": "projects",
"meta.user": "root",
"meta.user_id": 1,
"meta.project": "gpt/many_groups_and_projects/gpt-subgroup-8/gpt-project-1",
"meta.root_namespace": "gpt",
"meta.client_id": "user/1",
"remote_ip": "213.10.34.7",
"user_id": 1,
"username": "root",
"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36",
"queue_duration_s": 0.024286,
"request_urgency": "low",
"target_duration_s": 5,
"gitaly_calls": 39,
"gitaly_duration_s": 0.138461,
"redis_calls": 96,
"redis_allowed_cross_slot_calls": 4,
"redis_duration_s": 0.05258,
"redis_read_bytes": 4613,
"redis_write_bytes": 8135,
"redis_cache_calls": 21,
"redis_cache_duration_s": 0.010747,
"redis_cache_read_bytes": 3752,
"redis_cache_write_bytes": 1304,
"redis_repository_cache_calls": 62,
"redis_repository_cache_duration_s": 0.02801,
"redis_repository_cache_read_bytes": 440,
"redis_repository_cache_write_bytes": 5930,
"redis_sessions_calls": 9,
"redis_sessions_allowed_cross_slot_calls": 4,
"redis_sessions_duration_s": 0.010005,
"redis_sessions_read_bytes": 341,
"redis_sessions_write_bytes": 761,
"redis_shared_state_calls": 4,
"redis_shared_state_duration_s": 0.003818,
"redis_shared_state_read_bytes": 80,
"redis_shared_state_write_bytes": 140,
"db_count": 45,
"db_write_count": 0,
"db_cached_count": 8,
"db_replica_count": 43,
"db_primary_count": 2,
"db_main_count": 2,
"db_main_replica_count": 43,
"db_replica_cached_count": 8,
"db_primary_cached_count": 0,
"db_main_cached_count": 0,
"db_main_replica_cached_count": 8,
"db_replica_wal_count": 0,
"db_primary_wal_count": 1,
"db_main_wal_count": 1,
"db_main_replica_wal_count": 0,
"db_replica_wal_cached_count": 0,
"db_primary_wal_cached_count": 0,
"db_main_wal_cached_count": 0,
"db_main_replica_wal_cached_count": 0,
"db_replica_duration_s": 0.255,
"db_primary_duration_s": 0.004,
"db_main_duration_s": 0.004,
"db_main_replica_duration_s": 0.255,
"cpu_s": 0.950531,
"mem_objects": 500161,
"mem_bytes": 74429606,
"mem_mallocs": 306184,
"mem_total_bytes": 94436046,
"pid": 5031,
"worker_id": "puma_16",
"rate_limiting_gates": [],
"exception.class": "ActionView::Template::Error",
"exception.message": "5:cmdstart failed: GetRepoPath: not a git repository: \"/var/opt/gitlab/git-data/@hashed/e7/f6/e7f6c011776e8db7cd330b54174fd76f7d0216b612387a5ffcfb81e6f0919683.wiki.git\". debug_error_string:{\"created\":\"@1680779352.646158358\",\"description\":\"Error received from peer ipv4:10.142.15.240:2305\",\"file\":\"src/core/lib/surface/call.cc\",\"file_line\":1063,\"grpc_message\":\"cmd start failed: GetRepoPath: not a git repository: \"/var/opt/gitlab/git-data/@hashed/e7/f6/e7f6c011776e8db7cd330b54174fd76f7d0216b612387a5ffcfb81e6f0919683.wiki.git\"\",\"grpc_status\":5}",
"exception.backtrace": [
"config/initializers/enumerator_next_patch.rb:9:in `block (2 levels) in <module:EnumeratorNextPatch>''",
"lib/gitlab/gitaly_client/call.rb:43:in `block (3 levels) in instrument_stream''",
"lib/gitlab/gitaly_client/call.rb:55:in `recording_request''",
"lib/gitlab/gitaly_client/call.rb:43:in `block (2 levels) in instrument_stream''",
"lib/gitlab/gitaly_client/call.rb:42:in `loop''",
"lib/gitlab/gitaly_client/call.rb:42:in`block in instrument_stream''",
"lib/gitlab/gitaly_client/repository_service.rb:321:in `each''",
"lib/gitlab/gitaly_client/repository_service.rb:321:in `each''",
"lib/gitlab/gitaly_client/repository_service.rb:321:in `flat_map''",
"lib/gitlab/gitaly_client/repository_service.rb:321:in `search_files_by_regexp''",
"lib/gitlab/git/repository.rb:1098:in `search_files_by_regexp''",
"app/models/wiki.rb:499:in `find_matched_file''",
"app/models/wiki.rb:254:in `find_page''",
"app/models/wiki.rb:189:in `has_home_page?''",
"app/presenters/project_presenter.rb:399:in `wiki_anchor_data''",
"app/presenters/project_presenter.rb:65:in `empty_repo_statistics_buttons''",
"app/views/projects/empty.html.haml:27",
"app/controllers/application_controller.rb:142:in `render''",
"app/controllers/projects_controller.rb:394:in `render_landing_page''",
"app/controllers/projects_controller.rb:188:in `block (2 levels) in show''",
"app/controllers/projects_controller.rb:184:in `show''",
"app/controllers/application_controller.rb:573:in`block in allow_gitaly_ref_name_caching''",
"lib/gitlab/gitaly_client.rb:352:in `allow_ref_name_caching''",
"app/controllers/application_controller.rb:572:in `allow_gitaly_ref_name_caching''",
"ee/lib/gitlab/ip_address_state.rb:10:in `with''",
"ee/app/controllers/ee/application_controller.rb:46:in `set_current_ip_address''",
"app/controllers/application_controller.rb:524:in `set_current_admin''",
"lib/gitlab/session.rb:11:in `with_session''",
"app/controllers/application_controller.rb:515:in `set_session_storage''",
"lib/gitlab/i18n.rb:107:in `with_locale''",
"lib/gitlab/i18n.rb:113:in `with_user_locale''",
"app/controllers/application_controller.rb:506:in `set_locale''",
"app/controllers/application_controller.rb:499:in `set_current_context''",
"lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call''",
"lib/gitlab/middleware/memory_report.rb:13:in `call''",
"lib/gitlab/middleware/speedscope.rb:13:in `call''",
"lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call''",
"lib/gitlab/middleware/rails_queue_duration.rb:33:in`call''",
"lib/gitlab/metrics/rack_middleware.rb:16:in `block in call''",
"lib/gitlab/metrics/web_transaction.rb:46:in `run''",
"lib/gitlab/metrics/rack_middleware.rb:16:in `call''",
"lib/gitlab/jira/middleware.rb:19:in `call''",
"lib/gitlab/middleware/go.rb:20:in `call''",
"lib/gitlab/etag_caching/middleware.rb:21:in `call''",
"lib/gitlab/middleware/query_analyzer.rb:11:in `block in call''",
"lib/gitlab/database/query_analyzer.rb:37:in `within''",
"lib/gitlab/middleware/query_analyzer.rb:11:in `call''",
"lib/gitlab/middleware/multipart.rb:173:in `call''",
"lib/gitlab/middleware/read_only/controller.rb:50:in `call''",
"lib/gitlab/middleware/read_only.rb:18:in `call''",
"lib/gitlab/middleware/same_site_cookies.rb:27:in `call''",
"lib/gitlab/middleware/basic_health_check.rb:25:in `call''",
"lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call''",
"lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call''",
"lib/gitlab/middleware/request_context.rb:21:in `call''",
"lib/gitlab/middleware/webhook_recursion_detection.rb:15:in`call''",
"config/initializers/fix_local_cache_middleware.rb:11:in `call''",
"lib/gitlab/middleware/compressed_json.rb:37:in `call''",
"lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call''",
"lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call''",
"lib/gitlab/metrics/requests_rack_middleware.rb:79:in `call''",
"lib/gitlab/middleware/release_env.rb:13:in `call''"
],
"exception.cause_class": "GRPC::NotFound",
"db_duration_s": 0.22372,
"view_duration_s": 0,
"duration_s": 0.98548
}
- /var/log/gitlab/gitaly/current:
Click to expand
{
"correlation_id": "01GXB435317N591Y43RBPW7KG6",
"error": "GetRepoPath: not a git repository: \"/var/opt/gitlab/git-data/@hashed/e7/f6/e7f6c011776e8db7cd330b54174fd76f7d0216b612387a5ffcfb81e6f0919683.git\"",
"grpc.code": "NotFound",
"grpc.meta.auth_version": "v2",
"grpc.meta.client_name": "gitlab-web",
"grpc.meta.deadline_type": "regular",
"grpc.meta.method_type": "unary",
"grpc.method": "FindDefaultBranchName",
"grpc.request.deadline": "2023-04-06T11:09:21.009",
"grpc.request.fullMethod": "/gitaly.RefService/FindDefaultBranchName",
"grpc.request.glProjectPath": "gpt/many_groups_and_projects/gpt-subgroup-8/gpt-project-1",
"grpc.request.glRepository": "project-6",
"grpc.request.payload_bytes": 167,
"grpc.request.repoPath": "@hashed/e7/f6/e7f6c011776e8db7cd330b54174fd76f7d0216b612387a5ffcfb81e6f0919683.git",
"grpc.request.repoStorage": "gitaly-1",
"grpc.response.payload_bytes": 0,
"grpc.service": "gitaly.RefService",
"grpc.start_time": "2023-04-06T11:09:11.810",
"grpc.time_ms": 0.168,
"level": "info",
"limit.concurrency_queue_ms": 0,
"msg": "finishedunary call with code NotFound",
"peer.address": "10.142.0.45:54722",
"pid": 3325,
"remote_ip": "213.10.34.7",
"span.kind": "server",
"system": "grpc",
"time": "2023-04-06T11:09:11.810Z",
"user_id": "1",
"username": "root"
}
Example Staging Ref sentry error - https://sentry.gitlab.net/gitlab/staging-ref/issues/4120934/?query=is%3Aunresolved
Affected 10k GPT pipeline at https://gitlab.com/gitlab-org/quality/performance/-/jobs/4070711935
Configurations
Gitaly configuration before update:
Click to expand
postgresql['enable'] = false
redis['enable'] = false
puma['enable'] = false
sidekiq['enable'] = false
gitlab_workhorse['enable'] = false
prometheus['enable'] = false
alertmanager['enable'] = false
grafana['enable'] = false
gitlab_exporter['enable'] = false
nginx['enable'] = false
gitlab_kas['enable'] = false
# Prevent database connections during 'gitlab-ctl reconfigure'
gitlab_rails['rake_cache_clear'] = false
gitlab_rails['auto_migrate'] = false
gitlab_rails['internal_api_url'] = 'x'
gitaly['listen_addr'] = "0.0.0.0:8075"
gitaly['ruby_num_workers'] = 3
gitaly['pack_objects_cache_enabled'] = true
# Consul Agent (Monitoring Auto Discovery)
consul['enable'] = true
consul['configuration'] = {
bind_addr: 'x',
retry_join: %w(x)
}
consul['monitoring_service_discovery'] = true
node_exporter['listen_address'] = '0.0.0.0:9100'
gitaly['prometheus_listen_addr'] = "0.0.0.0:9236"
gitaly['auth_token'] = 'xxxx'
git_data_dirs({
"gitaly-1" => {
"path" => "/var/opt/gitlab/git-data"
}
})
Gitaly configuration after update:
Click to expand
...
gitaly['configuration'] = {
listen_addr: '0.0.0.0:8075',
pack_objects_cache: {
enabled: true,
},
gitaly_ruby: {
num_workers: 3
},
prometheus_listen_addr: '0.0.0.0:9236',
auth: {
token: 'xxx',
},
storage: [
{
name: 'gitaly-1',
path: '/var/opt/gitlab/git-data'
}
],
}
Validated that errors hoes away if revert back to old configuration.