Verified Commit 8ef98e2d authored by Michael Usachenko's avatar Michael Usachenko Committed by GitLab
Browse files

chore(xtask): remove tilt from e2e harness

parent fbfa6be0

.gitignore

+0 −1
Original line number Diff line number Diff line
@@ -23,7 +23,6 @@ datalake-generator-report.json 


# Tilt secrets (contains passwords)

.tilt-secrets

e2e/tilt/.secrets



# Dev logs, pid files, and runtime artifacts

.dev/

.tilt-secrets.example

deleted100644 → 0
+0 −7
Original line number Diff line number Diff line 
# Local development secrets for Tilt

# Copy this file: cp .tilt-secrets.example .tilt-secrets

# Then fill in passwords from GDK config



POSTGRES_PASSWORD=

CLICKHOUSE_PASSWORD=

GKG_JWT_SECRET=your-secret-key-at-least-32-bytes-long

crates/xtask/src/e2e/config.rs

+1 −4
Original line number Diff line number Diff line
@@ -128,7 +128,7 @@ pub struct PodReadiness { 
#[derive(Debug, Deserialize)]

pub struct Timeouts {

    pub ch_pod: String,

    pub tilt_ci: String,

    pub gkg_chart: String,

    pub dispatch_job: String,

    pub indexer_poll: u64,

    pub indexer_poll_interval: u64,
@@ -165,8 +165,6 @@ pub struct Config { 
    #[serde(skip)]

    pub cng_dir: PathBuf,

    #[serde(skip)]

    pub tilt_dir: PathBuf,

    #[serde(skip)]

    pub gitlab_src: PathBuf,

    #[serde(skip)]

    pub log_dir: PathBuf,
@@ -187,7 +185,6 @@ impl Config { 
            .with_context(|| format!("parsing {}", yaml_path.display()))?;



        cfg.cng_dir = gkg_root.join(c::CNG_DIR);

        cfg.tilt_dir = gkg_root.join(c::TILT_DIR);

        cfg.log_dir = gkg_root.join(c::LOG_DIR);

        cfg.gitlab_src = e::expand_home(&e::require("GITLAB_SRC"));

        cfg.gkg_root = gkg_root;

crates/xtask/src/e2e/constants.rs

+4 −24
Original line number Diff line number Diff line
@@ -6,16 +6,10 @@ 
//! All configurable defaults (namespaces, timeouts, image tags, etc.)

//! live in `e2e/config.yaml` and are loaded by `config.rs`.



use const_format::concatcp;



// -- Config file path ---------------------------------------------------------



pub const CONFIG_YAML: &str = "e2e/config.yaml";



// -- Tilt env var -------------------------------------------------------------



pub const TILT_CNG_ENV: &str = "GKG_E2E_CNG";



// -- Table lists (iteration targets, not config) ------------------------------



/// Datalake tables polled during step 21 to confirm siphon data is flowing
@@ -58,27 +52,23 @@ pub const GL_TABLES: &[&str] = &[ 
// -- Directories (relative to GKG repo root) ----------------------------------



pub const CNG_DIR: &str = "e2e/cng";

pub const TILT_DIR: &str = "e2e/tilt";

pub const LOG_DIR: &str = ".dev";

pub const E2E_TESTS_DIR: &str = "e2e/tests";



// -- Paths (relative to GKG repo root) ----------------------------------------



pub const GRAPH_SQL_PATH: &str = "fixtures/schema/graph.sql";

pub const TILTFILE_PATH: &str = concatcp!(TILT_DIR, "/Tiltfile");

pub const GKG_CHART_PATH: &str = "helm-dev/gkg";

pub const HELM_VALUES_YAML: &str = "e2e/helm-values.yaml";

pub const BUILD_DEV_SCRIPT: &str = "scripts/build-dev.sh";

pub const DISPATCH_JOB_TEMPLATE: &str = "e2e/templates/dispatch-indexing-job.yaml.tmpl";

pub const CLICK_HOUSE_YML_TEMPLATE: &str = "e2e/templates/click-house.yml.tmpl";



// -- Filenames ----------------------------------------------------------------



const TILT_CI_STEM: &str = "tilt-ci";

pub const TILT_CI_LOG: &str = concatcp!(TILT_CI_STEM, ".log");

pub const TILT_CI_PID: &str = concatcp!(TILT_CI_STEM, ".pid");



pub const CLICKHOUSE_YAML: &str = "clickhouse.yaml";

pub const CREATE_TEST_DATA_LOG: &str = "create-test-data.log";

pub const MANIFEST_JSON: &str = "manifest.json";

pub const SECRETS_FILE: &str = ".secrets";

pub const TRAEFIK_VALUES_YAML: &str = "traefik-values.yaml";

pub const GITLAB_VALUES_YAML: &str = "gitlab-values.yaml";

pub const DOCKERFILE_RAILS: &str = "Dockerfile.rails";
@@ -86,7 +76,6 @@ pub const COLIMA_START_LOG: &str = "colima-start.log"; 
pub const CH_MIGRATE_LOG: &str = "clickhouse-migrate.log";

pub const REDACTION_TEST_RB: &str = "redaction_test.rb";

pub const REDACTION_TEST_LOG: &str = "redaction-test.log";

pub const TILT_E2E_LOG: &str = "tilt-e2e.log";



// -- Log / artifact files cleaned during teardown -----------------------------
@@ -94,19 +83,10 @@ pub const TEARDOWN_LOG_FILES: &[&str] = &[ 
    CREATE_TEST_DATA_LOG,

    MANIFEST_JSON,

    COLIMA_START_LOG,

    TILT_CI_LOG,

    TILT_CI_PID,

    CH_MIGRATE_LOG,

    REDACTION_TEST_LOG,

    TILT_E2E_LOG,

];



/// Subset of log files removed during GKG-only teardown.

/// Omits CNG-phase logs (colima-start, create-test-data, manifest).

pub const GKG_TEARDOWN_LOG_FILES: &[&str] = &[

    TILT_CI_LOG,

    TILT_CI_PID,

    CH_MIGRATE_LOG,

    REDACTION_TEST_LOG,

    TILT_E2E_LOG,

];
 
pub const GKG_TEARDOWN_LOG_FILES: &[&str] = &[CH_MIGRATE_LOG, REDACTION_TEST_LOG];

crates/xtask/src/e2e/pipeline/cngsetup.rs

+12 −34
Original line number Diff line number Diff line
@@ -5,11 +5,10 @@ 
//!

//!   8.  Bridge PG credentials to default namespace (for Siphon)

//!   9.  Grant REPLICATION privilege to gitlab PG user (for Siphon WAL sender)

//!  10.  Extract JWT secret from toolbox pod -> e2e/tilt/.secrets

//!  11.  Run Rails db:migrate

//!  12.  Enable :knowledge_graph feature flag

//!  13.  Copy test scripts into toolbox pod

//!  14.  Create test data (users, groups, projects, MRs)

//!  10.  Run Rails db:migrate

//!  11.  Enable :knowledge_graph feature flag

//!  12.  Copy test scripts into toolbox pod

//!  13.  Create test data (users, groups, projects, MRs)



use std::fs;
@@ -33,7 +32,6 @@ pub fn run(sh: &Shell, cfg: &Config) -> Result<()> { 


    bridge_pg_credentials(sh, cfg)?;

    grant_replication(sh, cfg)?;

    extract_jwt_secret(sh, cfg, &toolbox_pod)?;

    run_db_migrate(sh, cfg, &toolbox_pod)?;

    enable_feature_flag(sh, cfg, &toolbox_pod)?;

    copy_test_scripts(sh, cfg, &toolbox_pod)?;
@@ -99,30 +97,10 @@ fn grant_replication(sh: &Shell, cfg: &Config) -> Result<()> { 
    Ok(())

}



// -- Step 10: Write Tilt secrets (.secrets file) ------------------------------



fn extract_jwt_secret(sh: &Shell, cfg: &Config, toolbox_pod: &str) -> Result<()> {

    ui::step(10, "Writing Tilt secrets")?;



    let pg_pass = kubectl::read_secret(

        sh,

        &cfg.namespaces.gitlab,

        &cfg.postgres.secret_name,

        &cfg.postgres.password_key,

    )?;



    match utils::write_tilt_secrets(sh, cfg, toolbox_pod, &pg_pass) {

        Ok(path) => ui::done(&format!("Written to {path}"))?,

        Err(_) => ui::warn("Could not extract JWT secret. You'll need to set it manually.")?,

    }



    Ok(())

}



// -- Step 11: Run Rails db:migrate --------------------------------------------

// -- Step 10: Run Rails db:migrate --------------------------------------------



fn run_db_migrate(sh: &Shell, cfg: &Config, toolbox_pod: &str) -> Result<()> {

    ui::step(11, "Running Rails db:migrate")?;

    ui::step(10, "Running Rails db:migrate")?;



    let ns = &cfg.namespaces.gitlab;

    let rails_root = &cfg.pod_paths.rails_root;
@@ -139,10 +117,10 @@ fn run_db_migrate(sh: &Shell, cfg: &Config, toolbox_pod: &str) -> Result<()> { 
    Ok(())

}



// -- Step 12: Enable feature flag ---------------------------------------------

// -- Step 11: Enable feature flag ---------------------------------------------



fn enable_feature_flag(sh: &Shell, cfg: &Config, toolbox_pod: &str) -> Result<()> {

    ui::step(12, "Enabling :knowledge_graph feature flag")?;

    ui::step(11, "Enabling :knowledge_graph feature flag")?;



    kubectl::toolbox_rails_eval(sh, cfg, toolbox_pod, "Feature.enable(:knowledge_graph)")?;
@@ -150,10 +128,10 @@ fn enable_feature_flag(sh: &Shell, cfg: &Config, toolbox_pod: &str) -> Result<() 
    Ok(())

}



// -- Step 13: Copy test scripts -----------------------------------------------

// -- Step 12: Copy test scripts -----------------------------------------------



fn copy_test_scripts(sh: &Shell, cfg: &Config, toolbox_pod: &str) -> Result<()> {

    ui::step(13, "Copying test scripts to toolbox pod")?;

    ui::step(12, "Copying test scripts to toolbox pod")?;



    let count = utils::copy_test_scripts(sh, cfg, toolbox_pod)?;

    if count == 0 {
@@ -165,10 +143,10 @@ fn copy_test_scripts(sh: &Shell, cfg: &Config, toolbox_pod: &str) -> Result<()> 
    Ok(())

}



// -- Step 14: Create test data ------------------------------------------------

// -- Step 13: Create test data ------------------------------------------------



fn create_test_data(sh: &Shell, cfg: &Config, toolbox_pod: &str) -> Result<()> {

    ui::step(14, "Creating test data")?;

    ui::step(13, "Creating test data")?;



    let ns = &cfg.namespaces.gitlab;

    let rails_root = &cfg.pod_paths.rails_root;