Transactionize FetchIntoObjectPool RPC manually
Closes #5922 (closed)
Recently, we transactionize requests from most RPCs, including FetchIntoObjectPool RPC. That RPC fetches new objects from the origin repository into the object pool. Afterward, it triggers a full set of housekeeping tasks. In some prior commits, we enabled WAL transaction support in the housekeeping manager. The manager initiates the WAL transaction and commits it after done. So, it means FetchIntoObjectPool results in two non-nested transactions. The housekeeping transaction is committed before the fetching one. Thus, the housekeeping task's effect is pushed to the next request.
This MR lets FetchIntoObjectPool RPC control the life cycle of the transaction manually. It allows the fetching transaction to finish before the housekeeping task commits, hence reserving the initial purpose.
Manual test
- Set up a repository and corresponding object pool.
-
FetchIntoObjectPool
is a baggage ofOptimizeRepository
. It is always triggered beforeOptimizeRepository
is called. As a result, we can trigger this call by clicking theOptimize repository
button in the admin setting or by pushing to the original repository N times. - The logs show each stage of this call:
- Cleaning stale data
- Dangling ref stats (before and after fetching)
- Run housekeeping (geometric repacking because we cannot remove objects from an object pool)
Full logs
{
"component": "gitaly.UnaryServerInterceptor",
"correlation_id": "01HTBXAZTRVW1XGA8GZP2GDZC3",
"grpc.meta.auth_version": "v2",
"grpc.meta.client_name": "gitlab-sidekiq",
"grpc.meta.method_operation": "mutator",
"grpc.meta.method_scope": "repository",
"grpc.meta.method_type": "unary",
"grpc.method": "FetchIntoObjectPool",
"grpc.request.deadline": "2024-04-01T17:05:35.566",
"grpc.request.fullMethod": "/gitaly.ObjectPoolService/FetchIntoObjectPool",
"grpc.request.pool.relativePath": "@pools/4e/07/4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce.git",
"grpc.request.pool.sourceProjectPath": "root/test-wal-2",
"grpc.request.pool.storage": "default",
"grpc.service": "gitaly.ObjectPoolService",
"grpc.start_time": "2024-04-01T11:05:35.566",
"level": "info",
"msg": "removed files",
"pid": 39322,
"remote_ip": "127.0.0.1",
"span.kind": "server",
"stale_data.configkeys": 0,
"stale_data.configsections": 0,
"stale_data.locks": 0,
"stale_data.objects": 0,
"stale_data.packedrefslock": 0,
"stale_data.packedrefsnew": 0,
"stale_data.packfilelocks": 0,
"stale_data.reflocks": 0,
"stale_data.refs": 0,
"stale_data.refsemptydir": 0,
"stale_data.reftablelocks": 0,
"stale_data.serverinfo": 0,
"system": "grpc",
"time": "2024-04-01T04:05:35.694Z",
"trace.sampled": "true",
"trace.spanid": "0359477fd89465d1",
"trace.traceid": "5f7f6d7903b5e4c5",
"user_id": "1",
"username": "root"
}
{
"component": "gitaly.UnaryServerInterceptor",
"correlation_id": "01HTBXAZTRVW1XGA8GZP2GDZC3",
"grpc.meta.auth_version": "v2",
"grpc.meta.client_name": "gitlab-sidekiq",
"grpc.meta.method_operation": "mutator",
"grpc.meta.method_scope": "repository",
"grpc.meta.method_type": "unary",
"grpc.method": "FetchIntoObjectPool",
"grpc.request.deadline": "2024-04-01T17:05:35.566",
"grpc.request.fullMethod": "/gitaly.ObjectPoolService/FetchIntoObjectPool",
"grpc.request.pool.relativePath": "@pools/4e/07/4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce.git",
"grpc.request.pool.sourceProjectPath": "root/test-wal-2",
"grpc.request.pool.storage": "default",
"grpc.service": "gitaly.ObjectPoolService",
"grpc.start_time": "2024-04-01T11:05:35.566",
"level": "info",
"msg": "pool dangling ref stats",
"pid": 39322,
"references.dangling": {
"blobs": 0,
"commits": 0,
"tags": 0,
"trees": 0
},
"references.normal": {
"blobs": 0,
"commits": 1,
"tags": 0,
"trees": 0
},
"remote_ip": "127.0.0.1",
"repository_info": {
"is_object_pool": true,
"loose_objects": {
"count": 3,
"size": 3014,
"stale_count": 0,
"stale_size": 0,
"garbage_count": 0,
"garbage_size": 0
},
"packfiles": {
"count": 0,
"size": 0,
"reverse_index_count": 0,
"cruft_count": 0,
"cruft_size": 0,
"keep_count": 0,
"keep_size": 0,
"garbage_count": 0,
"garbage_size": 0,
"bitmap": {
"exists": false,
"version": 0,
"has_hash_cache": false,
"has_lookup_table": false
},
"multi_pack_index": {
"exists": false,
"version": 0,
"packfile_count": 0
},
"multi_pack_index_bitmap": {
"exists": false,
"version": 0,
"has_hash_cache": false,
"has_lookup_table": false
},
"last_full_repack": "2024-04-01T11:00:49.731048+07:00"
},
"references": {
"loose_references_count": 0,
"packed_references_size": 103,
"reference_backend": "files"
},
"commit_graph": {
"exists": false,
"commit_graph_chain_length": 0,
"has_bloom_filters": false,
"has_generation_data": false,
"has_generation_data_overflow": false
},
"alternates": {
"exists": false,
"last_modified": "0001-01-01T00:00:00Z"
}
},
"span.kind": "server",
"system": "grpc",
"time": "2024-04-01T04:05:35.717Z",
"trace.sampled": "true",
"trace.spanid": "0359477fd89465d1",
"trace.traceid": "5f7f6d7903b5e4c5",
"user_id": "1",
"username": "root",
"when": "before fetch"
}
{
"component": "gitaly.UnaryServerInterceptor",
"correlation_id": "01HTBXAZTRVW1XGA8GZP2GDZC3",
"grpc.meta.auth_version": "v2",
"grpc.meta.client_name": "gitlab-sidekiq",
"grpc.meta.method_operation": "mutator",
"grpc.meta.method_scope": "repository",
"grpc.meta.method_type": "unary",
"grpc.method": "FetchIntoObjectPool",
"grpc.request.deadline": "2024-04-01T17:05:35.566",
"grpc.request.fullMethod": "/gitaly.ObjectPoolService/FetchIntoObjectPool",
"grpc.request.pool.relativePath": "@pools/4e/07/4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce.git",
"grpc.request.pool.sourceProjectPath": "root/test-wal-2",
"grpc.request.pool.storage": "default",
"grpc.service": "gitaly.ObjectPoolService",
"grpc.start_time": "2024-04-01T11:05:35.566",
"level": "info",
"msg": "pool dangling ref stats",
"pid": 39322,
"references.dangling": {
"blobs": 0,
"commits": 0,
"tags": 0,
"trees": 0
},
"references.normal": {
"blobs": 0,
"commits": 2,
"tags": 0,
"trees": 0
},
"remote_ip": "127.0.0.1",
"repository_info": {
"is_object_pool": true,
"loose_objects": {
"count": 3,
"size": 3014,
"stale_count": 0,
"stale_size": 0,
"garbage_count": 0,
"garbage_size": 0
},
"packfiles": {
"count": 0,
"size": 0,
"reverse_index_count": 0,
"cruft_count": 0,
"cruft_size": 0,
"keep_count": 0,
"keep_size": 0,
"garbage_count": 0,
"garbage_size": 0,
"bitmap": {
"exists": false,
"version": 0,
"has_hash_cache": false,
"has_lookup_table": false
},
"multi_pack_index": {
"exists": false,
"version": 0,
"packfile_count": 0
},
"multi_pack_index_bitmap": {
"exists": false,
"version": 0,
"has_hash_cache": false,
"has_lookup_table": false
},
"last_full_repack": "2024-04-01T11:00:49.731048+07:00"
},
"references": {
"loose_references_count": 1,
"packed_references_size": 103,
"reference_backend": "files"
},
"commit_graph": {
"exists": false,
"commit_graph_chain_length": 0,
"has_bloom_filters": false,
"has_generation_data": false,
"has_generation_data_overflow": false
},
"alternates": {
"exists": false,
"last_modified": "0001-01-01T00:00:00Z"
}
},
"span.kind": "server",
"system": "grpc",
"time": "2024-04-01T04:05:36.053Z",
"trace.sampled": "true",
"trace.spanid": "0359477fd89465d1",
"trace.traceid": "5f7f6d7903b5e4c5",
"user_id": "1",
"username": "root",
"when": "after fetch"
}
{
"component": "gitaly.UnaryServerInterceptor",
"correlation_id": "01HTBXAZTRVW1XGA8GZP2GDZC3",
"grpc.meta.auth_version": "v2",
"grpc.meta.client_name": "gitlab-sidekiq",
"grpc.meta.method_operation": "mutator",
"grpc.meta.method_scope": "repository",
"grpc.meta.method_type": "unary",
"grpc.method": "FetchIntoObjectPool",
"grpc.request.deadline": "2024-04-01T17:05:35.566",
"grpc.request.fullMethod": "/gitaly.ObjectPoolService/FetchIntoObjectPool",
"grpc.request.pool.relativePath": "@pools/4e/07/4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce.git",
"grpc.request.pool.sourceProjectPath": "root/test-wal-2",
"grpc.request.pool.storage": "default",
"grpc.service": "gitaly.ObjectPoolService",
"grpc.start_time": "2024-04-01T11:05:35.566",
"level": "info",
"msg": "repository info",
"pid": 39322,
"remote_ip": "127.0.0.1",
"repository_info": {
"is_object_pool": true,
"loose_objects": {
"count": 3,
"size": 3014,
"stale_count": 0,
"stale_size": 0,
"garbage_count": 0,
"garbage_size": 0
},
"packfiles": {
"count": 0,
"size": 0,
"reverse_index_count": 0,
"cruft_count": 0,
"cruft_size": 0,
"keep_count": 0,
"keep_size": 0,
"garbage_count": 0,
"garbage_size": 0,
"bitmap": {
"exists": false,
"version": 0,
"has_hash_cache": false,
"has_lookup_table": false
},
"multi_pack_index": {
"exists": false,
"version": 0,
"packfile_count": 0
},
"multi_pack_index_bitmap": {
"exists": false,
"version": 0,
"has_hash_cache": false,
"has_lookup_table": false
},
"last_full_repack": "2024-04-01T11:00:49.731048+07:00"
},
"references": {
"loose_references_count": 1,
"packed_references_size": 103,
"reference_backend": "files"
},
"commit_graph": {
"exists": false,
"commit_graph_chain_length": 0,
"has_bloom_filters": false,
"has_generation_data": false,
"has_generation_data_overflow": false
},
"alternates": {
"exists": false,
"last_modified": "0001-01-01T00:00:00Z"
}
},
"span.kind": "server",
"system": "grpc",
"time": "2024-04-01T04:05:36.132Z",
"trace.sampled": "true",
"trace.spanid": "0359477fd89465d1",
"trace.traceid": "5f7f6d7903b5e4c5",
"user_id": "1",
"username": "root"
}
{
"component": "gitaly.UnaryServerInterceptor",
"correlation_id": "01HTBXAZTRVW1XGA8GZP2GDZC3",
"grpc.meta.auth_version": "v2",
"grpc.meta.client_name": "gitlab-sidekiq",
"grpc.meta.method_operation": "mutator",
"grpc.meta.method_scope": "repository",
"grpc.meta.method_type": "unary",
"grpc.method": "FetchIntoObjectPool",
"grpc.request.deadline": "2024-04-01T17:05:35.566",
"grpc.request.fullMethod": "/gitaly.ObjectPoolService/FetchIntoObjectPool",
"grpc.request.pool.relativePath": "@pools/4e/07/4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce.git",
"grpc.request.pool.sourceProjectPath": "root/test-wal-2",
"grpc.request.pool.storage": "default",
"grpc.service": "gitaly.ObjectPoolService",
"grpc.start_time": "2024-04-01T11:05:35.566",
"level": "info",
"msg": "repository info",
"pid": 39322,
"remote_ip": "127.0.0.1",
"repository_info": {
"is_object_pool": true,
"loose_objects": {
"count": 0,
"size": 0,
"stale_count": 0,
"stale_size": 0,
"garbage_count": 0,
"garbage_size": 0
},
"packfiles": {
"count": 1,
"size": 2864,
"reverse_index_count": 1,
"cruft_count": 0,
"cruft_size": 0,
"keep_count": 0,
"keep_size": 0,
"garbage_count": 0,
"garbage_size": 0,
"bitmap": {
"exists": false,
"version": 0,
"has_hash_cache": false,
"has_lookup_table": false
},
"multi_pack_index": {
"exists": true,
"version": 1,
"packfile_count": 1
},
"multi_pack_index_bitmap": {
"exists": true,
"version": 1,
"has_hash_cache": true,
"has_lookup_table": true
},
"last_full_repack": "2024-04-01T11:00:49.731048+07:00"
},
"references": {
"loose_references_count": 1,
"packed_references_size": 103,
"reference_backend": "files"
},
"commit_graph": {
"exists": true,
"commit_graph_chain_length": 1,
"has_bloom_filters": true,
"has_generation_data": true,
"has_generation_data_overflow": false
},
"alternates": {
"exists": false,
"last_modified": "0001-01-01T00:00:00Z"
}
},
"span.kind": "server",
"system": "grpc",
"time": "2024-04-01T04:05:36.306Z",
"trace.sampled": "true",
"trace.spanid": "0359477fd89465d1",
"trace.traceid": "5f7f6d7903b5e4c5",
"user_id": "1",
"username": "root"
}
{"level":"info","msg":"optimized repository with WAL","optimizations":{"packed_objects_geometric":"success","written_bitmap":"success","written_commit_graph_full":"success","written_multi_pack_index":"success"},"pid":39322,"system":"housekeeping","time":"2024-04-01T04:05:36.304Z"}
{
"command.count": 19,
"command.cpu_time_ms": 260,
"command.inblock": 0,
"command.majflt": 1124,
"command.maxrss": 30310400,
"command.minflt": 27565,
"command.oublock": 0,
"command.real_time_ms": 558,
"command.spawn_token_fork_ms": 48,
"command.spawn_token_wait_ms": 0,
"command.system_time_ms": 148,
"command.user_time_ms": 112,
"component": "gitaly.UnaryServerInterceptor",
"correlation_id": "01HTBXAZTRVW1XGA8GZP2GDZC3",
"grpc.code": "OK",
"grpc.meta.auth_version": "v2",
"grpc.meta.client_name": "gitlab-sidekiq",
"grpc.meta.method_operation": "mutator",
"grpc.meta.method_scope": "repository",
"grpc.meta.method_type": "unary",
"grpc.method": "FetchIntoObjectPool",
"grpc.request.deadline": "2024-04-01T17:05:35.566",
"grpc.request.fullMethod": "/gitaly.ObjectPoolService/FetchIntoObjectPool",
"grpc.request.payload_bytes": 237,
"grpc.request.pool.relativePath": "@pools/4e/07/4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce.git",
"grpc.request.pool.sourceProjectPath": "root/test-wal-2",
"grpc.request.pool.storage": "default",
"grpc.response.payload_bytes": 0,
"grpc.service": "gitaly.ObjectPoolService",
"grpc.start_time": "2024-04-01T11:05:35.566",
"grpc.time_ms": 945.075,
"level": "info",
"msg": "finished unary call with code OK",
"pack_objects.enumerate_objects_ms": 1,
"pack_objects.prepare_pack_ms": 0,
"pack_objects.write_pack_file_ms": 0,
"pack_objects.written_object_count": 9,
"pid": 39322,
"remote_ip": "127.0.0.1",
"span.kind": "server",
"system": "grpc",
"time": "2024-04-01T04:05:36.511Z",
"trace.sampled": "true",
"trace.spanid": "0359477fd89465d1",
"trace.traceid": "5f7f6d7903b5e4c5",
"trace2.activated": "true",
"trace2.hooks": "tracing_exporter",
"user_id": "1",
"username": "root"
}