Skip to content

Transactionize FetchIntoObjectPool RPC manually

Quang-Minh Nguyen requested to merge qmnguyen0711/transactionize-object-pool-fetch into master

Closes #5922 (closed)

Recently, we transactionize requests from most RPCs, including FetchIntoObjectPool RPC. That RPC fetches new objects from the origin repository into the object pool. Afterward, it triggers a full set of housekeeping tasks. In some prior commits, we enabled WAL transaction support in the housekeeping manager. The manager initiates the WAL transaction and commits it after done. So, it means FetchIntoObjectPool results in two non-nested transactions. The housekeeping transaction is committed before the fetching one. Thus, the housekeeping task's effect is pushed to the next request.

This MR lets FetchIntoObjectPool RPC control the life cycle of the transaction manually. It allows the fetching transaction to finish before the housekeeping task commits, hence reserving the initial purpose.

Manual test

  • Set up a repository and corresponding object pool.
  • FetchIntoObjectPool is a baggage of OptimizeRepository. It is always triggered before OptimizeRepository is called. As a result, we can trigger this call by clicking the Optimize repository button in the admin setting or by pushing to the original repository N times.
  • The logs show each stage of this call:
    • Cleaning stale data
    • Dangling ref stats (before and after fetching)
    • Run housekeeping (geometric repacking because we cannot remove objects from an object pool)
Full logs 
{
  "component": "gitaly.UnaryServerInterceptor",
  "correlation_id": "01HTBXAZTRVW1XGA8GZP2GDZC3",
  "grpc.meta.auth_version": "v2",
  "grpc.meta.client_name": "gitlab-sidekiq",
  "grpc.meta.method_operation": "mutator",
  "grpc.meta.method_scope": "repository",
  "grpc.meta.method_type": "unary",
  "grpc.method": "FetchIntoObjectPool",
  "grpc.request.deadline": "2024-04-01T17:05:35.566",
  "grpc.request.fullMethod": "/gitaly.ObjectPoolService/FetchIntoObjectPool",
  "grpc.request.pool.relativePath": "@pools/4e/07/4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce.git",
  "grpc.request.pool.sourceProjectPath": "root/test-wal-2",
  "grpc.request.pool.storage": "default",
  "grpc.service": "gitaly.ObjectPoolService",
  "grpc.start_time": "2024-04-01T11:05:35.566",
  "level": "info",
  "msg": "removed files",
  "pid": 39322,
  "remote_ip": "127.0.0.1",
  "span.kind": "server",
  "stale_data.configkeys": 0,
  "stale_data.configsections": 0,
  "stale_data.locks": 0,
  "stale_data.objects": 0,
  "stale_data.packedrefslock": 0,
  "stale_data.packedrefsnew": 0,
  "stale_data.packfilelocks": 0,
  "stale_data.reflocks": 0,
  "stale_data.refs": 0,
  "stale_data.refsemptydir": 0,
  "stale_data.reftablelocks": 0,
  "stale_data.serverinfo": 0,
  "system": "grpc",
  "time": "2024-04-01T04:05:35.694Z",
  "trace.sampled": "true",
  "trace.spanid": "0359477fd89465d1",
  "trace.traceid": "5f7f6d7903b5e4c5",
  "user_id": "1",
  "username": "root"
}
{
  "component": "gitaly.UnaryServerInterceptor",
  "correlation_id": "01HTBXAZTRVW1XGA8GZP2GDZC3",
  "grpc.meta.auth_version": "v2",
  "grpc.meta.client_name": "gitlab-sidekiq",
  "grpc.meta.method_operation": "mutator",
  "grpc.meta.method_scope": "repository",
  "grpc.meta.method_type": "unary",
  "grpc.method": "FetchIntoObjectPool",
  "grpc.request.deadline": "2024-04-01T17:05:35.566",
  "grpc.request.fullMethod": "/gitaly.ObjectPoolService/FetchIntoObjectPool",
  "grpc.request.pool.relativePath": "@pools/4e/07/4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce.git",
  "grpc.request.pool.sourceProjectPath": "root/test-wal-2",
  "grpc.request.pool.storage": "default",
  "grpc.service": "gitaly.ObjectPoolService",
  "grpc.start_time": "2024-04-01T11:05:35.566",
  "level": "info",
  "msg": "pool dangling ref stats",
  "pid": 39322,
  "references.dangling": {
    "blobs": 0,
    "commits": 0,
    "tags": 0,
    "trees": 0
  },
  "references.normal": {
    "blobs": 0,
    "commits": 1,
    "tags": 0,
    "trees": 0
  },
  "remote_ip": "127.0.0.1",
  "repository_info": {
    "is_object_pool": true,
    "loose_objects": {
      "count": 3,
      "size": 3014,
      "stale_count": 0,
      "stale_size": 0,
      "garbage_count": 0,
      "garbage_size": 0
    },
    "packfiles": {
      "count": 0,
      "size": 0,
      "reverse_index_count": 0,
      "cruft_count": 0,
      "cruft_size": 0,
      "keep_count": 0,
      "keep_size": 0,
      "garbage_count": 0,
      "garbage_size": 0,
      "bitmap": {
        "exists": false,
        "version": 0,
        "has_hash_cache": false,
        "has_lookup_table": false
      },
      "multi_pack_index": {
        "exists": false,
        "version": 0,
        "packfile_count": 0
      },
      "multi_pack_index_bitmap": {
        "exists": false,
        "version": 0,
        "has_hash_cache": false,
        "has_lookup_table": false
      },
      "last_full_repack": "2024-04-01T11:00:49.731048+07:00"
    },
    "references": {
      "loose_references_count": 0,
      "packed_references_size": 103,
      "reference_backend": "files"
    },
    "commit_graph": {
      "exists": false,
      "commit_graph_chain_length": 0,
      "has_bloom_filters": false,
      "has_generation_data": false,
      "has_generation_data_overflow": false
    },
    "alternates": {
      "exists": false,
      "last_modified": "0001-01-01T00:00:00Z"
    }
  },
  "span.kind": "server",
  "system": "grpc",
  "time": "2024-04-01T04:05:35.717Z",
  "trace.sampled": "true",
  "trace.spanid": "0359477fd89465d1",
  "trace.traceid": "5f7f6d7903b5e4c5",
  "user_id": "1",
  "username": "root",
  "when": "before fetch"
}
{
  "component": "gitaly.UnaryServerInterceptor",
  "correlation_id": "01HTBXAZTRVW1XGA8GZP2GDZC3",
  "grpc.meta.auth_version": "v2",
  "grpc.meta.client_name": "gitlab-sidekiq",
  "grpc.meta.method_operation": "mutator",
  "grpc.meta.method_scope": "repository",
  "grpc.meta.method_type": "unary",
  "grpc.method": "FetchIntoObjectPool",
  "grpc.request.deadline": "2024-04-01T17:05:35.566",
  "grpc.request.fullMethod": "/gitaly.ObjectPoolService/FetchIntoObjectPool",
  "grpc.request.pool.relativePath": "@pools/4e/07/4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce.git",
  "grpc.request.pool.sourceProjectPath": "root/test-wal-2",
  "grpc.request.pool.storage": "default",
  "grpc.service": "gitaly.ObjectPoolService",
  "grpc.start_time": "2024-04-01T11:05:35.566",
  "level": "info",
  "msg": "pool dangling ref stats",
  "pid": 39322,
  "references.dangling": {
    "blobs": 0,
    "commits": 0,
    "tags": 0,
    "trees": 0
  },
  "references.normal": {
    "blobs": 0,
    "commits": 2,
    "tags": 0,
    "trees": 0
  },
  "remote_ip": "127.0.0.1",
  "repository_info": {
    "is_object_pool": true,
    "loose_objects": {
      "count": 3,
      "size": 3014,
      "stale_count": 0,
      "stale_size": 0,
      "garbage_count": 0,
      "garbage_size": 0
    },
    "packfiles": {
      "count": 0,
      "size": 0,
      "reverse_index_count": 0,
      "cruft_count": 0,
      "cruft_size": 0,
      "keep_count": 0,
      "keep_size": 0,
      "garbage_count": 0,
      "garbage_size": 0,
      "bitmap": {
        "exists": false,
        "version": 0,
        "has_hash_cache": false,
        "has_lookup_table": false
      },
      "multi_pack_index": {
        "exists": false,
        "version": 0,
        "packfile_count": 0
      },
      "multi_pack_index_bitmap": {
        "exists": false,
        "version": 0,
        "has_hash_cache": false,
        "has_lookup_table": false
      },
      "last_full_repack": "2024-04-01T11:00:49.731048+07:00"
    },
    "references": {
      "loose_references_count": 1,
      "packed_references_size": 103,
      "reference_backend": "files"
    },
    "commit_graph": {
      "exists": false,
      "commit_graph_chain_length": 0,
      "has_bloom_filters": false,
      "has_generation_data": false,
      "has_generation_data_overflow": false
    },
    "alternates": {
      "exists": false,
      "last_modified": "0001-01-01T00:00:00Z"
    }
  },
  "span.kind": "server",
  "system": "grpc",
  "time": "2024-04-01T04:05:36.053Z",
  "trace.sampled": "true",
  "trace.spanid": "0359477fd89465d1",
  "trace.traceid": "5f7f6d7903b5e4c5",
  "user_id": "1",
  "username": "root",
  "when": "after fetch"
}
{
  "component": "gitaly.UnaryServerInterceptor",
  "correlation_id": "01HTBXAZTRVW1XGA8GZP2GDZC3",
  "grpc.meta.auth_version": "v2",
  "grpc.meta.client_name": "gitlab-sidekiq",
  "grpc.meta.method_operation": "mutator",
  "grpc.meta.method_scope": "repository",
  "grpc.meta.method_type": "unary",
  "grpc.method": "FetchIntoObjectPool",
  "grpc.request.deadline": "2024-04-01T17:05:35.566",
  "grpc.request.fullMethod": "/gitaly.ObjectPoolService/FetchIntoObjectPool",
  "grpc.request.pool.relativePath": "@pools/4e/07/4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce.git",
  "grpc.request.pool.sourceProjectPath": "root/test-wal-2",
  "grpc.request.pool.storage": "default",
  "grpc.service": "gitaly.ObjectPoolService",
  "grpc.start_time": "2024-04-01T11:05:35.566",
  "level": "info",
  "msg": "repository info",
  "pid": 39322,
  "remote_ip": "127.0.0.1",
  "repository_info": {
    "is_object_pool": true,
    "loose_objects": {
      "count": 3,
      "size": 3014,
      "stale_count": 0,
      "stale_size": 0,
      "garbage_count": 0,
      "garbage_size": 0
    },
    "packfiles": {
      "count": 0,
      "size": 0,
      "reverse_index_count": 0,
      "cruft_count": 0,
      "cruft_size": 0,
      "keep_count": 0,
      "keep_size": 0,
      "garbage_count": 0,
      "garbage_size": 0,
      "bitmap": {
        "exists": false,
        "version": 0,
        "has_hash_cache": false,
        "has_lookup_table": false
      },
      "multi_pack_index": {
        "exists": false,
        "version": 0,
        "packfile_count": 0
      },
      "multi_pack_index_bitmap": {
        "exists": false,
        "version": 0,
        "has_hash_cache": false,
        "has_lookup_table": false
      },
      "last_full_repack": "2024-04-01T11:00:49.731048+07:00"
    },
    "references": {
      "loose_references_count": 1,
      "packed_references_size": 103,
      "reference_backend": "files"
    },
    "commit_graph": {
      "exists": false,
      "commit_graph_chain_length": 0,
      "has_bloom_filters": false,
      "has_generation_data": false,
      "has_generation_data_overflow": false
    },
    "alternates": {
      "exists": false,
      "last_modified": "0001-01-01T00:00:00Z"
    }
  },
  "span.kind": "server",
  "system": "grpc",
  "time": "2024-04-01T04:05:36.132Z",
  "trace.sampled": "true",
  "trace.spanid": "0359477fd89465d1",
  "trace.traceid": "5f7f6d7903b5e4c5",
  "user_id": "1",
  "username": "root"
}
{
  "component": "gitaly.UnaryServerInterceptor",
  "correlation_id": "01HTBXAZTRVW1XGA8GZP2GDZC3",
  "grpc.meta.auth_version": "v2",
  "grpc.meta.client_name": "gitlab-sidekiq",
  "grpc.meta.method_operation": "mutator",
  "grpc.meta.method_scope": "repository",
  "grpc.meta.method_type": "unary",
  "grpc.method": "FetchIntoObjectPool",
  "grpc.request.deadline": "2024-04-01T17:05:35.566",
  "grpc.request.fullMethod": "/gitaly.ObjectPoolService/FetchIntoObjectPool",
  "grpc.request.pool.relativePath": "@pools/4e/07/4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce.git",
  "grpc.request.pool.sourceProjectPath": "root/test-wal-2",
  "grpc.request.pool.storage": "default",
  "grpc.service": "gitaly.ObjectPoolService",
  "grpc.start_time": "2024-04-01T11:05:35.566",
  "level": "info",
  "msg": "repository info",
  "pid": 39322,
  "remote_ip": "127.0.0.1",
  "repository_info": {
    "is_object_pool": true,
    "loose_objects": {
      "count": 0,
      "size": 0,
      "stale_count": 0,
      "stale_size": 0,
      "garbage_count": 0,
      "garbage_size": 0
    },
    "packfiles": {
      "count": 1,
      "size": 2864,
      "reverse_index_count": 1,
      "cruft_count": 0,
      "cruft_size": 0,
      "keep_count": 0,
      "keep_size": 0,
      "garbage_count": 0,
      "garbage_size": 0,
      "bitmap": {
        "exists": false,
        "version": 0,
        "has_hash_cache": false,
        "has_lookup_table": false
      },
      "multi_pack_index": {
        "exists": true,
        "version": 1,
        "packfile_count": 1
      },
      "multi_pack_index_bitmap": {
        "exists": true,
        "version": 1,
        "has_hash_cache": true,
        "has_lookup_table": true
      },
      "last_full_repack": "2024-04-01T11:00:49.731048+07:00"
    },
    "references": {
      "loose_references_count": 1,
      "packed_references_size": 103,
      "reference_backend": "files"
    },
    "commit_graph": {
      "exists": true,
      "commit_graph_chain_length": 1,
      "has_bloom_filters": true,
      "has_generation_data": true,
      "has_generation_data_overflow": false
    },
    "alternates": {
      "exists": false,
      "last_modified": "0001-01-01T00:00:00Z"
    }
  },
  "span.kind": "server",
  "system": "grpc",
  "time": "2024-04-01T04:05:36.306Z",
  "trace.sampled": "true",
  "trace.spanid": "0359477fd89465d1",
  "trace.traceid": "5f7f6d7903b5e4c5",
  "user_id": "1",
  "username": "root"
}
{"level":"info","msg":"optimized repository with WAL","optimizations":{"packed_objects_geometric":"success","written_bitmap":"success","written_commit_graph_full":"success","written_multi_pack_index":"success"},"pid":39322,"system":"housekeeping","time":"2024-04-01T04:05:36.304Z"}
{
  "command.count": 19,
  "command.cpu_time_ms": 260,
  "command.inblock": 0,
  "command.majflt": 1124,
  "command.maxrss": 30310400,
  "command.minflt": 27565,
  "command.oublock": 0,
  "command.real_time_ms": 558,
  "command.spawn_token_fork_ms": 48,
  "command.spawn_token_wait_ms": 0,
  "command.system_time_ms": 148,
  "command.user_time_ms": 112,
  "component": "gitaly.UnaryServerInterceptor",
  "correlation_id": "01HTBXAZTRVW1XGA8GZP2GDZC3",
  "grpc.code": "OK",
  "grpc.meta.auth_version": "v2",
  "grpc.meta.client_name": "gitlab-sidekiq",
  "grpc.meta.method_operation": "mutator",
  "grpc.meta.method_scope": "repository",
  "grpc.meta.method_type": "unary",
  "grpc.method": "FetchIntoObjectPool",
  "grpc.request.deadline": "2024-04-01T17:05:35.566",
  "grpc.request.fullMethod": "/gitaly.ObjectPoolService/FetchIntoObjectPool",
  "grpc.request.payload_bytes": 237,
  "grpc.request.pool.relativePath": "@pools/4e/07/4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce.git",
  "grpc.request.pool.sourceProjectPath": "root/test-wal-2",
  "grpc.request.pool.storage": "default",
  "grpc.response.payload_bytes": 0,
  "grpc.service": "gitaly.ObjectPoolService",
  "grpc.start_time": "2024-04-01T11:05:35.566",
  "grpc.time_ms": 945.075,
  "level": "info",
  "msg": "finished unary call with code OK",
  "pack_objects.enumerate_objects_ms": 1,
  "pack_objects.prepare_pack_ms": 0,
  "pack_objects.write_pack_file_ms": 0,
  "pack_objects.written_object_count": 9,
  "pid": 39322,
  "remote_ip": "127.0.0.1",
  "span.kind": "server",
  "system": "grpc",
  "time": "2024-04-01T04:05:36.511Z",
  "trace.sampled": "true",
  "trace.spanid": "0359477fd89465d1",
  "trace.traceid": "5f7f6d7903b5e4c5",
  "trace2.activated": "true",
  "trace2.hooks": "tracing_exporter",
  "user_id": "1",
  "username": "root"
}
Edited by Quang-Minh Nguyen

Merge request reports