Verified Commit 856e0d31 authored by Jean-Gabriel Doyon's avatar Jean-Gabriel Doyon Committed by GitLab
Browse files

feat(indexer): leverage traversal_path projection in Group and Project ETL

parent 2a512bad

config/ontology/nodes/core/group.yaml

+1 −1
Original line number Diff line number Diff line
@@ -85,7 +85,7 @@ etl: 
    ON namespace.id = namespace_details.namespace_id

    INNER JOIN namespace_traversal_paths traversal_paths

    ON namespace.id = traversal_paths.id

  traversal_path_filter: "namespace.id IN (SELECT id FROM namespace_traversal_paths WHERE startsWith(traversal_path, {traversal_path:String}))"

  traversal_path_filter: "startsWith(traversal_paths.traversal_path, {traversal_path:String})"

  watermark: namespace._siphon_replicated_at

  deleted: namespace._siphon_deleted

  edges:

config/ontology/nodes/core/project.yaml

+1 −1
Original line number Diff line number Diff line
@@ -102,7 +102,7 @@ etl: 
    siphon_projects project

    INNER JOIN project_namespace_traversal_paths traversal_paths

    ON project.id = traversal_paths.id

  traversal_path_filter: "project.id IN (SELECT id FROM project_namespace_traversal_paths WHERE startsWith(traversal_path, {traversal_path:String}))"

  traversal_path_filter: "startsWith(traversal_paths.traversal_path, {traversal_path:String})"

  watermark: project._siphon_replicated_at

  deleted: project._siphon_deleted

  edges:

fixtures/siphon.sql

+14 −4
Original line number Diff line number Diff line
@@ -112,11 +112,16 @@ CREATE TABLE IF NOT EXISTS namespace_traversal_paths 
    `id` Int64 DEFAULT 0,

    `traversal_path` String DEFAULT '0/',

    `version` DateTime64(6, 'UTC') DEFAULT now(),

    `deleted` Bool DEFAULT false

    `deleted` Bool DEFAULT false,

    PROJECTION by_traversal_path

    (

        SELECT *

        ORDER BY traversal_path

    )

) ENGINE = ReplacingMergeTree(version, deleted)

PRIMARY KEY id

ORDER BY id

SETTINGS index_granularity = 512;

SETTINGS index_granularity = 512, deduplicate_merge_projection_mode = 'rebuild';



-- Siphon source tables for project data

CREATE TABLE IF NOT EXISTS siphon_projects
@@ -146,11 +151,16 @@ CREATE TABLE IF NOT EXISTS project_namespace_traversal_paths 
    `id` Int64 DEFAULT 0,

    `traversal_path` String DEFAULT '0/',

    `version` DateTime64(6, 'UTC') DEFAULT now(),

    `deleted` Bool DEFAULT false

    `deleted` Bool DEFAULT false,

    PROJECTION by_traversal_path

    (

        SELECT *

        ORDER BY traversal_path

    )

) ENGINE = ReplacingMergeTree(version, deleted)

PRIMARY KEY id

ORDER BY id

SETTINGS index_granularity = 512;

SETTINGS index_granularity = 512, deduplicate_merge_projection_mode = 'rebuild';



-- Siphon source tables for notes

CREATE TABLE IF NOT EXISTS siphon_notes