Verified Commit 2cc569ce authored by Michael Usachenko's avatar Michael Usachenko Committed by GitLab
Browse files

feat(xtask): deploy and configure gkg services for e2e harness

parent 4606a219

Cargo.lock

+1 −0
Original line number Diff line number Diff line
@@ -8162,6 +8162,7 @@ dependencies = [ 
 "anyhow",

 "clap",

 "cliclack",

 "const_format",

 "tempfile",

 "xshell",

]

crates/xtask/Cargo.toml

+1 −0
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ workspace = true 
[dependencies]

clap = { workspace = true }

anyhow = { workspace = true }

const_format = { workspace = true }

tempfile = { workspace = true }

xshell = { workspace = true }

cliclack = { workspace = true }

crates/xtask/src/e2e/config.rs

+25 −4
Original line number Diff line number Diff line
@@ -64,15 +64,25 @@ pub struct Config { 
    pub ch_url: String,

    pub ch_datalake_db: String,

    pub ch_graph_db: String,



    // -- Siphon ---------------------------------------------------------------

    pub siphon_publication: String,

    pub siphon_slot: String,

    pub siphon_poll_timeout: u64,



    // -- GKG ------------------------------------------------------------------

    pub gkg_server_image: String,

    pub gkg_dispatch_job: String,

    pub gkg_indexer_configmap: String,

}



impl Config {

    /// Build config from environment variables with sensible defaults.

    pub fn from_env() -> Self {

        let gkg_root = e::workspace_root();

        let cng_dir = gkg_root.join("e2e/cng");

        let tilt_dir = gkg_root.join("e2e/tilt");

        let log_dir = gkg_root.join(".dev");

        let cng_dir = gkg_root.join(c::CNG_DIR);

        let tilt_dir = gkg_root.join(c::TILT_DIR);

        let log_dir = gkg_root.join(c::LOG_DIR);



        let base_tag = e::env_or("BASE_TAG", c::BASE_TAG);

        let cng_registry = e::env_or("CNG_REGISTRY", c::CNG_REGISTRY);
@@ -81,7 +91,7 @@ impl Config { 
        let workhorse_image = format!("{cng_registry}/{}:{base_tag}", c::WORKHORSE_COMPONENT);



        let e2e_pod_dir = e::env_or("E2E_POD_DIR", c::E2E_POD_DIR);

        let manifest_pod_path = format!("{e2e_pod_dir}/manifest.json");

        let manifest_pod_path = format!("{e2e_pod_dir}/{}", c::MANIFEST_JSON);



        let ch_service_name = e::env_or("CH_SERVICE_NAME", c::CH_SERVICE_NAME);

        let default_ns_val = e::env_or("DEFAULT_NS", c::DEFAULT_NS);
@@ -128,6 +138,17 @@ impl Config { 
            ch_url,

            ch_datalake_db: e::env_or("CH_DATALAKE_DB", c::CH_DATALAKE_DB),

            ch_graph_db: e::env_or("CH_GRAPH_DB", c::CH_GRAPH_DB),



            siphon_publication: e::env_or("SIPHON_PUBLICATION", c::SIPHON_PUBLICATION),

            siphon_slot: e::env_or("SIPHON_SLOT", c::SIPHON_SLOT),

            siphon_poll_timeout: env::var("SIPHON_POLL_TIMEOUT")

                .ok()

                .and_then(|v| v.parse().ok())

                .unwrap_or(c::SIPHON_POLL_TIMEOUT),



            gkg_server_image: e::env_or("GKG_SERVER_IMAGE", c::GKG_SERVER_IMAGE),

            gkg_dispatch_job: e::env_or("GKG_DISPATCH_JOB", c::GKG_DISPATCH_JOB),

            gkg_indexer_configmap: e::env_or("GKG_INDEXER_CONFIGMAP", c::GKG_INDEXER_CONFIGMAP),

        }

    }

crates/xtask/src/e2e/constants.rs

+101 −8
Original line number Diff line number Diff line
@@ -7,6 +7,8 @@ 
//! User-specific paths (e.g. GITLAB_SRC) are intentionally absent —

//! those are required env vars with no fallback.



use const_format::concatcp;



// -- Colima / k8s -------------------------------------------------------------



pub const COLIMA_PROFILE: &str = "cng";
@@ -51,6 +53,7 @@ pub const PG_SUPERUSER: &str = "postgres"; 


/// Secret name for PG credentials bridged to the default namespace (for Siphon).

pub const PG_BRIDGE_SECRET_NAME: &str = "postgres-credentials";

pub const PG_GKG_ENABLED_TABLE: &str = "knowledge_graph_enabled_namespaces";



// -- Paths inside pods --------------------------------------------------------
@@ -96,15 +99,105 @@ pub const CH_DATALAKE_DB: &str = "gitlab_clickhouse_development"; 
pub const CH_GRAPH_DB: &str = "gkg-development";

pub const CH_DEFAULT_USER: &str = "default";



// -- Siphon -------------------------------------------------------------------



pub const SIPHON_PUBLICATION: &str = "siphon_publication_main_db";

pub const SIPHON_SLOT: &str = "siphon_slot_main_db";

pub const SIPHON_POLL_TIMEOUT: u64 = 600;



/// Tables polled during step 21 to confirm siphon data is flowing.

pub const SIPHON_MR_TABLE: &str = "hierarchy_merge_requests";

pub const SIPHON_KG_NS_TABLE: &str = "siphon_knowledge_graph_enabled_namespaces";



/// Poll intervals (seconds) for siphon data checks.

pub const SIPHON_MR_POLL_INTERVAL: u64 = 15;

pub const SIPHON_KG_POLL_INTERVAL: u64 = 10;

pub const SIPHON_KG_POLL_TIMEOUT: u64 = 300;



// -- GKG ----------------------------------------------------------------------



pub const GKG_SERVER_IMAGE: &str = "gkg-server";

pub const GKG_DISPATCH_JOB: &str = "gkg-dispatch-indexing";

pub const GKG_INDEXER_CONFIGMAP: &str = "gkg-indexer-config";



/// Image tag used by the dispatch-indexing k8s Job.

pub const GKG_DEV_TAG: &str = "dev";



/// k8s secret providing the ClickHouse password to dispatch-indexing.

pub const CH_CREDENTIALS_SECRET: &str = "clickhouse-credentials";

pub const CH_CREDENTIALS_KEY: &str = "password";



/// ClickHouse graph tables operated on by OPTIMIZE TABLE FINAL and row-count

/// verification.

pub const GL_TABLES: &[&str] = &[

    "gl_user",

    "gl_group",

    "gl_project",

    "gl_merge_request",

    "gl_work_item",

    "gl_note",

    "gl_milestone",

    "gl_label",

    "gl_edge",

];



/// Table polled after dispatch-indexing to confirm the indexer is working.

pub const GL_PROJECT_TABLE: &str = "gl_project";



// -- Timeouts -----------------------------------------------------------------



pub const CH_POD_TIMEOUT: &str = "300s";

pub const TILT_CI_TIMEOUT: &str = "20m";

pub const DISPATCH_JOB_TIMEOUT: &str = "120s";

pub const INDEXER_POLL_TIMEOUT: u64 = 300;

pub const INDEXER_POLL_INTERVAL: u64 = 10;

pub const INDEXER_SETTLE_SECS: u64 = 30;



// -- Tilt ---------------------------------------------------------------------



pub const TILT_CAPRONI_ENV: &str = "GKG_E2E_CAPRONI";



// -- Directories (relative to GKG repo root) ----------------------------------



pub const CNG_DIR: &str = "e2e/cng";

pub const TILT_DIR: &str = "e2e/tilt";

pub const LOG_DIR: &str = ".dev";

pub const E2E_TESTS_DIR: &str = "e2e/tests";



// -- Paths (relative to GKG repo root) ----------------------------------------



pub const GRAPH_SQL_PATH: &str = "fixtures/schema/graph.sql";

pub const TILTFILE_PATH: &str = concatcp!(TILT_DIR, "/Tiltfile");

pub const DISPATCH_JOB_TEMPLATE: &str = "e2e/templates/dispatch-indexing-job.yaml.tmpl";



// -- Filenames ----------------------------------------------------------------



/// Stem shared by Tilt CI log and PID files.

const TILT_CI_STEM: &str = "tilt-ci";

pub const TILT_CI_LOG: &str = concatcp!(TILT_CI_STEM, ".log");

pub const TILT_CI_PID: &str = concatcp!(TILT_CI_STEM, ".pid");



pub const CLICKHOUSE_YAML: &str = "clickhouse.yaml";

pub const CREATE_TEST_DATA_LOG: &str = "create-test-data.log";

pub const MANIFEST_JSON: &str = "manifest.json";

pub const SECRETS_FILE: &str = ".secrets";

pub const TRAEFIK_VALUES_YAML: &str = "traefik-values.yaml";

pub const GITLAB_VALUES_YAML: &str = "gitlab-values.yaml";

pub const DOCKERFILE_RAILS: &str = "Dockerfile.rails";

pub const COLIMA_START_LOG: &str = "colima-start.log";

pub const CH_MIGRATE_LOG: &str = "clickhouse-migrate.log";

pub const REDACTION_TEST_LOG: &str = "redaction-test.log";

pub const TILT_E2E_LOG: &str = "tilt-e2e.log";



// -- Log / artifact files cleaned during teardown -----------------------------



pub const TEARDOWN_LOG_FILES: &[&str] = &[

    "create-test-data.log",

    "manifest.json",

    "colima-start.log",

    "tilt-ci.log",

    "tilt-ci.pid",

    "clickhouse-migrate.log",

    "redaction-test.log",

    "tilt-e2e.log",

    CREATE_TEST_DATA_LOG,

    MANIFEST_JSON,

    COLIMA_START_LOG,

    TILT_CI_LOG,

    TILT_CI_PID,

    CH_MIGRATE_LOG,

    REDACTION_TEST_LOG,

    TILT_E2E_LOG,

];

crates/xtask/src/e2e/kubectl.rs

+65 −1
Original line number Diff line number Diff line
@@ -111,7 +111,7 @@ pub fn toolbox_exec(sh: &Shell, cfg: &Config, pod: &str, command: &[&str]) -> Re 
pub fn toolbox_rails_eval(sh: &Shell, cfg: &Config, pod: &str, ruby_cmd: &str) -> Result<String> {

    let ns = &cfg.gitlab_ns;

    let rails_root = &cfg.rails_root;

    let script = r#"cd "$0" && bundle exec rails runner "$1" RAILS_ENV=production"#;

    let script = r#"cd "$0" && bundle exec rails runner "$1" RAILS_ENV=production"#.to_string();



    let output = cmd!(

        sh,
@@ -194,6 +194,42 @@ pub fn pg_superuser_exec( 
    Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())

}



/// Run a psql query as superuser and return the scalar result (no headers).

///

/// Uses `-t` (tuples-only) so the output is just the value, no column

/// headers or row-count footers. Whitespace is stripped.

///

/// The password and SQL are passed as positional parameters to `bash -c`

/// (`$0` and `$1`) so they are never interpreted as shell syntax.

pub fn pg_superuser_query(

    sh: &Shell,

    cfg: &Config,

    pg_superpass: &str,

    sql: &str,

) -> Result<String> {

    let ns = &cfg.gitlab_ns;

    let pod = &cfg.pg_pod;

    let db = &cfg.pg_database;

    let superuser = c::PG_SUPERUSER;

    let script = format!(r#"PGPASSWORD="$0" psql -U {superuser} -d {db} -t -c "$1""#);



    let output = cmd!(

        sh,

        "kubectl exec -n {ns} {pod} -- bash -c {script} {pg_superpass} {sql}"

    )

    .quiet()

    .ignore_status()

    .ignore_stderr()

    .output()?;



    if !output.status.success() {

        let stderr = String::from_utf8_lossy(&output.stderr);

        bail!("psql query failed: {stderr}");

    }



    Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())

}



// -- ClickHouse ---------------------------------------------------------------



/// Resolve the ClickHouse pod name in the default namespace.
@@ -224,3 +260,31 @@ pub fn get_ch_pod(sh: &Shell, cfg: &Config) -> Result<String> { 
        ),

    }

}



/// Run a clickhouse-client query and return the output.

pub fn ch_query(

    sh: &Shell,

    cfg: &Config,

    ch_pod: &str,

    database: &str,

    query: &str,

) -> Result<String> {

    let ns = &cfg.default_ns;

    let ch_user = c::CH_DEFAULT_USER;

    let output = cmd!(

        sh,

        "kubectl exec -n {ns} {ch_pod} --

            clickhouse-client --user {ch_user} --database {database} --query {query}"

    )

    .quiet()

    .ignore_status()

    .ignore_stderr()

    .output()?;



    if !output.status.success() {

        let stderr = String::from_utf8_lossy(&output.stderr);

        bail!("clickhouse-client query failed: {stderr}");

    }



    Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())

}