Skip to content
Snippets Groups Projects

Add cron worker to automatically rollout zoekt exact code search to paid namespaces

Merged John Mason requested to merge jm-namespace-assignment-service into master
1 file
+ 42
4
Compare changes
  • Side-by-side
  • Inline
@@ -14,6 +14,7 @@ def initialize(enabled_namespace, number_of_replicas)
@namespace = enabled_namespace.namespace
@number_of_replicas = number_of_replicas
@replica_plans = [] # Stores plans for each replica
@errors = [] # Tracks errors hierarchically
end
# Read-only operation to plan assignments
@@ -37,12 +38,16 @@ def simulate_replica_plan
project_statistics.each do |project_stats|
assign_project_to_index(indices, project_stats)
rescue StandardError => e
log_error(@replica_plans.size - 1, nil, :assignment_failed, e.message)
end
@replica_plans << {
id: replica_id, # nil if the replica is simulated as new
indices: indices.map { |index| format_index(index) }
}
rescue StandardError => e
log_error(@replica_plans.size - 1, nil, :replica_failed, e.message)
end
def find_replica
@@ -65,14 +70,21 @@ def find_or_simulate_index(indices, project_stats)
return index if index
# Simulate a new in-memory index if none are available
raise "Index limit exceeded for replica" if indices.size >= MAX_INDICES_PER_REPLICA
if indices.size >= MAX_INDICES_PER_REPLICA
log_error(@replica_plans.size - 1, indices.size - 1, :index_limit_exceeded,
"Reached max indices for a replica")
return
end
simulate_new_index(indices)
end
def simulate_new_index(indices)
node = find_best_node
raise "No suitable nodes available" unless node
unless node
log_error(@replica_plans.size - 1, nil, :node_unavailable, "No suitable nodes available")
return
end
existing_index = find_existing_index_on_node(node)
indices << if existing_index
@@ -94,6 +106,8 @@ def build_simulated_index(node)
node_id: node.id,
projects: { project_id_from: nil, project_id_to: nil },
required_storage_bytes: 0,
used_storage_bytes: 0, # For new indices, this is 0
reserved_storage_bytes: 0, # For new indices, this is also 0
max_storage_bytes: node.unclaimed_storage_bytes
}
end
@@ -107,11 +121,19 @@ def build_existing_index(existing_index)
project_id_to: existing_index.metadata["project_id_to"]
},
required_storage_bytes: 0, # Will be calculated as repositories are assigned
used_storage_bytes: existing_index.used_storage_bytes, # Actual value from the DB
reserved_storage_bytes: existing_index.reserved_storage_bytes, # Actual value from the DB
max_storage_bytes: existing_index.node.unclaimed_storage_bytes
}
end
def add_project_to_index(index, project_stats)
if index.nil?
log_error(@replica_plans.size - 1, nil, :index_not_found,
"No index available for project #{project_stats.project_id}")
return
end
# Update projects to reflect project ID range
index[:projects][:project_id_from] ||= project_stats.project_id
index[:projects][:project_id_to] = project_stats.project_id
@@ -125,7 +147,20 @@ def has_space_for?(index, project_stats)
end
def find_best_node
Node.order_by_unclaimed_space.first
# Collect nodes already assigned to indices across all replicas
used_node_ids = @replica_plans.flat_map { |replica| replica[:indices].map { |index| index[:node_id] } }
# Exclude nodes already used by previous replicas
Node.order_by_unclaimed_space.where.not(id: used_node_ids).first
end
def log_error(replica_idx, index_idx, type, details)
@errors << {
replica_idx: replica_idx,
index_idx: index_idx,
type: type,
details: details
}
end
def format_index(index)
@@ -134,6 +169,8 @@ def format_index(index)
node_id: index[:node_id],
projects: index[:projects],
required_storage_bytes: index[:required_storage_bytes],
used_storage_bytes: index[:used_storage_bytes],
reserved_storage_bytes: index[:reserved_storage_bytes],
max_storage_bytes: index[:max_storage_bytes]
}
end
@@ -142,7 +179,8 @@ def format_output
{
namespace_id: @namespace.id,
enabled_namespace_id: @enabled_namespace.id,
replicas: @replica_plans
replicas: @replica_plans,
errors: @errors
}
end
end
Loading