chore(e2e): drift management + documentation

BasedOnStyles = gitlab_base

gitlab_base.British = NO

[todo.md]

BasedOnStyles =

[CHANGELOG.md]

BasedOnStyles =

  memory: "12"              # GB

  cpus: "4"

  disk: "60"                # GB

  vm_type: vz                 # vz (macOS Virtualization.framework) or qemu

  k8s_version: v1.31.5+k3s1

# -- Kubernetes namespaces -----------------------------------------------------

  gitlab:

    release: gitlab

    chart: gitlab/gitlab

    chart_version: "9.8.1"    # must match cng.base_tag (v18.8.1 → chart 9.8.1)

    repo_name: gitlab

    repo_url: https://charts.gitlab.io

    timeout: 15m

  traefik:

    release: traefik

    chart: traefik/traefik

    chart_version: ""          # empty = latest

    repo_name: traefik

    repo_url: https://traefik.github.io/charts

    timeout: 5m

# -- ClickHouse ----------------------------------------------------------------

clickhouse:

  image: "clickhouse/clickhouse-server:25.1-alpine"

  service_name: gkg-e2e-clickhouse

  http_port: "8123"

  native_port: "9000"

  datalake_db: gitlab_clickhouse_development

  graph_db: gkg-development

  default_user: default

# xtask

Development task runner for the GitLab Knowledge Graph. 

## Features

- Automates E2E environment lifecycle: provisioning a local Kubernetes cluster, deploying GitLab and GKG, running tests, and tearing it all down.

## Quick start

```shell

# Full environment (Colima + GitLab + CNG setup)

GITLAB_SRC=~/path/to/gitlab cargo xtask e2e setup

# Add the GKG stack (ClickHouse, siphon, indexer, webserver)

GITLAB_SRC=~/path/to/gitlab cargo xtask e2e setup --gkg

# Run redaction tests

cargo xtask e2e test

# Rebuild after code changes

cargo xtask e2e rebuild --gkg      # GKG server image (~2-3min)

cargo xtask e2e rebuild --rails    # CNG images from GITLAB_SRC (~5-8min)

# Tear down

cargo xtask e2e teardown                # everything

cargo xtask e2e teardown --keep-colima  # keep VM, remove workloads

cargo xtask e2e teardown --gkg-only     # keep GitLab, remove GKG

```

Mise aliases are available: `mise e2e:setup`, `mise e2e:test`, `mise e2e:teardown`, `mise e2e:rebuild:gkg`, `mise e2e:rebuild:rails`, `mise e2e:rebuild:all`.

## Prerequisites

- [Colima](https://github.com/abiosoft/colima) with Kubernetes support

- Docker CLI

- Helm 3

- `GITLAB_SRC` environment variable pointing to a GitLab Rails checkout (required for `setup` and `rebuild --rails`)

## Configuration

All configurable values live in [`config/e2e.yaml`](../../config/e2e.yaml): Colima resources, Helm chart versions, image tags, namespaces, timeouts, ClickHouse settings, and pod readiness checks.

Structural constants (file paths, table lists, concurrency limits) are in `src/e2e/constants.rs`.

## Module structure

See the [E2E module README](src/e2e/README.md) for architecture details.

# E2E module

Orchestrates a local Kubernetes environment for end-to-end testing of the GKG stack against a real GitLab instance. Deploys Colima, GitLab (via CNG Helm chart), ClickHouse, siphon, and the GKG indexer/webserver, then runs redaction and permission tests.

## Architecture

```plaintext

pipeline/          -- orchestration steps (setup, teardown, rebuild, test)

  cng.rs           -- Colima VM, Traefik, GitLab Helm deploy (steps 1-6)

  cngsetup.rs      -- PG credentials, migrations, test data (steps 8-13)

  gkg.rs           -- ClickHouse, schema, GKG chart, siphon wait, dispatch-indexing, tests (steps 15-25)

  rebuild.rs       -- fast image rebuild + rollout restart

  teardown.rs      -- full or partial teardown

  test.rs          -- standalone test runner

infra/             -- infrastructure primitives (no domain logic)

  colima.rs        -- Colima VM lifecycle

  docker.rs        -- Bollard Docker client helpers

  helm.rs          -- Helm CLI: install, upgrade, uninstall, repo add

  kube.rs          -- kube-rs primitives: SSA apply, cp, exec, wait, delete

config.rs          -- deserializes config/e2e.yaml into Config

constants.rs       -- structural constants (paths, table lists, tool names)

template.rs        -- ${VAR} template renderer for YAML manifests

utils.rs           -- domain helpers (toolbox pod, PG queries, ClickHouse queries, secrets)

ui.rs              -- cliclack terminal output

cmd.rs             -- xshell helpers

env.rs             -- path utilities

```

## Static files

```plaintext

config/e2e.yaml                          -- all configurable values

e2e/cng/gitlab-values.yaml               -- GitLab Helm overrides

e2e/cng/traefik-values.yaml              -- Traefik Helm overrides

e2e/cng/clickhouse.yaml.tmpl             -- ClickHouse k8s manifest (templated)

e2e/cng/Dockerfile.rails                 -- CNG image overlay

e2e/helm-values.yaml                     -- GKG Helm overrides

e2e/templates/dispatch-indexing-job.yaml.tmpl

e2e/templates/rails-clickhouse-config.yml.tmpl

e2e/tests/{redaction_test,test_helper,create_test_data}.rb

```

For usage, configuration, and the full pipeline walkthrough, see [docs/dev/e2e.md](../../../../docs/dev/e2e.md).

//! E2E environment configuration.

//!

//! Deserializes `e2e/config.yaml` directly into [`Config`].  The only env var

//! Deserializes `config/e2e.yaml` directly into [`Config`].  The only env var

//! is `GITLAB_SRC` — an optional, user-specific path with no YAML default,

//! required only for commands that build CNG images (setup, rebuild --rails).

    pub memory: String,

    pub cpus: String,

    pub disk: String,

    pub vm_type: String,

    pub k8s_version: String,

}

pub struct HelmRelease {

    pub release: String,

    pub chart: String,

    #[serde(default)]

    pub chart_version: String,

    pub repo_name: String,

    pub repo_url: String,

    pub timeout: String,

#[derive(Debug, Deserialize)]

pub struct ClickHouse {

    pub image: String,

    pub service_name: String,

    pub http_port: String,

    pub native_port: String,

    pub datalake_db: String,

    pub graph_db: String,

    pub default_user: String,

}

impl Config {

    /// Load `e2e/config.yaml`, populate runtime paths, return Config.

    /// Load `config/e2e.yaml`, populate runtime paths, return Config.

    pub fn load() -> Result<Self> {

        let gkg_root = e::workspace_root();

        let yaml_path = gkg_root.join(c::CONFIG_YAML);