Commit c92034fb authored by Jean-Gabriel Doyon PTO until 2024-04-17's avatar Jean-Gabriel Doyon PTO until 2024-04-17
Browse files

refactor(indexer): unify module configurations

parent f71f3baf

config/default.yaml

+12 −15
Original line number Diff line number Diff line
@@ -27,22 +27,19 @@ graph: 


engine:

  max_concurrent_workers: 16



modules:

  sdlc:

    max_concurrency: 8

    max_retry_attempts: 5

    retry_interval_secs: 60

    datalake_batch_size: 1000000

  code:

    max_concurrency: 4

    max_retry_attempts: 5

    retry_interval_secs: 60



code_indexing:

    events_stream_name: siphon_stream_main_db



sdlc_indexing:

  datalake_batch_size: 1000000



health_check:

  bind_address: "0.0.0.0:4201"

  namespace: default

crates/gkg-server/src/config.rs

+2 −5
Original line number Diff line number Diff line
@@ -2,11 +2,10 @@ use std::net::SocketAddr; 
use std::sync::Arc;



use health_check::HealthCheckConfig;

use indexer::ModulesConfig;

use indexer::clickhouse::ClickHouseConfiguration;

use indexer::configuration::EngineConfiguration;

use indexer::modules::code::GitalyConfiguration;

use indexer::modules::code::config::CodeIndexingConfig;

use indexer::modules::sdlc::config::SdlcIndexingConfig;

use indexer::nats::NatsConfiguration;

use labkit_rs::metrics::MetricsConfig;

use serde::{Deserialize, Serialize};
@@ -46,9 +45,7 @@ pub struct AppConfig { 
    #[serde(default)]

    pub gitaly: Option<GitalyConfiguration>,

    #[serde(default)]

    pub code_indexing: CodeIndexingConfig,

    #[serde(default)]

    pub sdlc_indexing: SdlcIndexingConfig,

    pub modules: ModulesConfig,

    #[serde(default)]

    pub health_check: HealthCheckConfig,

    #[serde(default)]

crates/gkg-server/src/main.rs

+1 −2
Original line number Diff line number Diff line
@@ -42,8 +42,7 @@ async fn main() -> anyhow::Result<()> { 
                datalake: config.datalake.clone(),

                engine: config.engine.clone(),

                gitaly: config.gitaly.clone(),

                code_indexing: config.code_indexing.clone(),

                sdlc_indexing: config.sdlc_indexing.clone(),

                modules: config.modules.clone(),

            };

            indexer::run(&indexer_config, shutdown)

                .await

crates/gkg-server/tests/common/mod.rs

+6 −1
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ use chrono::{DateTime, Utc}; 
use indexer::clickhouse::{ArrowClickHouseClient, ClickHouseConfiguration, ClickHouseDestination};

use indexer::module::{Handler, HandlerContext, Module};

use indexer::modules::SdlcModule;

use indexer::modules::sdlc::config::SdlcIndexingConfig;

use indexer::testkit::{MockLockService, MockNatsServices};

use query_engine::ParameterizedQuery;

use serde_json::Value;
@@ -212,7 +213,11 @@ pub fn default_test_watermark() -> DateTime<Utc> { 


/// Get the namespace handler from an SdlcModule.

pub async fn get_namespace_handler(context: &TestContext) -> Box<dyn Handler> {

    let sdlc_module = SdlcModule::new(&context.config, &context.config, 1)

    let sdlc_config = SdlcIndexingConfig {

        datalake_batch_size: 1,

        ..Default::default()

    };

    let sdlc_module = SdlcModule::new(&context.config, &context.config, &sdlc_config)

        .await

        .expect("failed to create SDLC module");

    let handlers = sdlc_module.handlers();

crates/indexer/src/configuration.rs

+2 −1
Original line number Diff line number Diff line
@@ -17,7 +17,8 @@ pub struct EngineConfiguration { 
    pub max_concurrent_workers: usize,



    /// Per-module overrides, keyed by [`Module::name`](crate::module::Module::name).

    #[serde(default)]

    /// Populated programmatically from the unified module configs — not deserialized from YAML.

    #[serde(skip)]

    pub modules: HashMap<String, ModuleConfiguration>,

}