Verified Commit 2d784f8e authored by Michael Angelo Rivera's avatar Michael Angelo Rivera Committed by GitLab
Browse files

fix(ontology): compute full_path from routes table instead of slug

parent 0843ce30

config/ontology/nodes/core/group.yaml

+10 −2
Original line number Diff line number Diff line
@@ -25,7 +25,7 @@ properties: 


  full_path:

    type: string

    source: path

    source: full_path

    nullable: true

    description: "Full path for the group (e.g. gitlab-org/quality)."
@@ -76,7 +76,8 @@ etl: 
  select: >-

    namespace.id AS id, namespace.name AS name,

    namespace_details.description AS description,

    namespace.visibility_level AS visibility_level, namespace.path AS path,

    namespace.visibility_level AS visibility_level,

    if(isNotNull(route.source_id), route.path, namespace.path) AS full_path,

    namespace.parent_id AS parent_id, namespace.owner_id AS owner_id,

    namespace.created_at AS created_at, namespace.updated_at AS updated_at,

    traversal_paths.traversal_path AS traversal_path
@@ -90,6 +91,13 @@ etl: 
      WHERE startsWith(traversal_path, {traversal_path:String})

    ) traversal_paths

    ON namespace.id = traversal_paths.id

    LEFT JOIN (

      SELECT toNullable(source_id) AS source_id, argMax(path, _siphon_replicated_at) AS path

      FROM siphon_routes

      WHERE startsWith(traversal_path, {traversal_path:String})

        AND source_type = 'Namespace' AND NOT _siphon_deleted

      GROUP BY source_id

    ) route ON namespace.id = route.source_id

  traversal_path_filter: "startsWith(traversal_paths.traversal_path, {traversal_path:String})"

  watermark: namespace._siphon_replicated_at

  deleted: namespace._siphon_deleted

config/ontology/nodes/core/project.yaml

+10 −2
Original line number Diff line number Diff line
@@ -25,7 +25,7 @@ properties: 


  full_path:

    type: string

    source: path

    source: full_path

    nullable: true

    description: "Full path for the project (e.g. gitlab-org/gitlab)."
@@ -93,7 +93,8 @@ etl: 
  scope: namespaced

  select: >-

    project.id AS id, project.name AS name, project.description AS description,

    project.visibility_level AS visibility_level, project.path AS path,

    project.visibility_level AS visibility_level,

    if(isNotNull(route.source_id), route.path, project.path) AS full_path,

    project.namespace_id AS namespace_id, project.creator_id AS creator_id,

    project.created_at AS created_at, project.updated_at AS updated_at,

    project.archived AS archived, project.star_count AS star_count,
@@ -107,6 +108,13 @@ etl: 
      WHERE startsWith(traversal_path, {traversal_path:String})

    ) traversal_paths

    ON project.id = traversal_paths.id

    LEFT JOIN (

      SELECT toNullable(source_id) AS source_id, argMax(path, _siphon_replicated_at) AS path

      FROM siphon_routes

      WHERE startsWith(traversal_path, {traversal_path:String})

        AND source_type = 'Project' AND NOT _siphon_deleted

      GROUP BY source_id

    ) route ON project.id = route.source_id

  traversal_path_filter: "startsWith(traversal_paths.traversal_path, {traversal_path:String})"

  watermark: project._siphon_replicated_at

  deleted: project._siphon_deleted

config/seeds/data_correctness.sql

+12 −12
Original line number Diff line number Diff line
@@ -97,19 +97,19 @@ INSERT INTO gl_user (id, username, name, state, user_type, email) VALUES 
    (5, 'eve', 'Eve External', 'blocked', 'service_account', 'eve@example.com'),

    (6, '用户_émoji_🎉', 'Ünïcödé Üser', 'active', 'human', 'unicode@example.com');



INSERT INTO gl_group (id, name, visibility_level, traversal_path) VALUES

    (100, 'Public Group', 'public', '1/100/'),

    (101, 'Private Group', 'private', '1/101/'),

    (102, 'Internal Group', 'internal', '1/102/'),

    (200, 'Deep Group A', 'public', '1/100/200/'),

    (300, 'Deep Group B', 'public', '1/100/200/300/');

INSERT INTO gl_group (id, name, full_path, visibility_level, traversal_path) VALUES

    (100, 'Public Group', 'public-group', 'public', '1/100/'),

    (101, 'Private Group', 'private-group', 'private', '1/101/'),

    (102, 'Internal Group', 'internal-group', 'internal', '1/102/'),

    (200, 'Deep Group A', 'public-group/deep-a', 'public', '1/100/200/'),

    (300, 'Deep Group B', 'public-group/deep-a/deep-b', 'public', '1/100/200/300/');



INSERT INTO gl_project (id, name, visibility_level, traversal_path) VALUES

    (1000, 'Public Project', 'public', '1/100/1000/'),

    (1001, 'Private Project', 'private', '1/101/1001/'),

    (1002, 'Internal Project', 'internal', '1/100/1002/'),

    (1003, 'Secret Project', 'private', '1/101/1003/'),

    (1004, 'Shared Project', 'public', '1/102/1004/');

INSERT INTO gl_project (id, name, full_path, visibility_level, traversal_path) VALUES

    (1000, 'Public Project', 'public-group/public-project', 'public', '1/100/1000/'),

    (1001, 'Private Project', 'private-group/private-project', 'private', '1/101/1001/'),

    (1002, 'Internal Project', 'public-group/internal-project', 'internal', '1/100/1002/'),

    (1003, 'Secret Project', 'private-group/secret-project', 'private', '1/101/1003/'),

    (1004, 'Shared Project', 'internal-group/shared-project', 'public', '1/102/1004/');



INSERT INTO gl_merge_request (id, iid, title, state, source_branch, target_branch, traversal_path) VALUES

    (2000, 1, 'Add feature A', 'opened', 'feature-a', 'main', '1/100/1000/'),

crates/integration-tests/tests/indexer/common/mod.rs

+4 −1
Original line number Diff line number Diff line
@@ -11,4 +11,7 @@ pub use integration_testkit::{ 
    GRAPH_SCHEMA_SQL, SIPHON_SCHEMA_SQL, TestContext, assert_edge_count,

    assert_edge_count_for_traversal_path, assert_edges_have_traversal_path, assert_node_count,

};

pub use siphon::{create_member, create_namespace, create_project, create_user};
 
pub use siphon::{

    create_member, create_namespace, create_namespace_with_path, create_project,

    create_project_with_path, create_route, create_user,

};

crates/integration-tests/tests/indexer/common/siphon.rs

+76 −4
Original line number Diff line number Diff line 
use integration_testkit::TestContext;



fn sql_escape(s: &str) -> String {

    s.replace('\'', "''")

}



pub async fn create_namespace(

    ctx: &TestContext,

    id: i64,

    parent_id: Option<i64>,

    visibility_level: i32,

    traversal_path: &str,

) {

    create_namespace_with_path(ctx, id, parent_id, visibility_level, traversal_path, None).await;

}



pub async fn create_namespace_with_path(

    ctx: &TestContext,

    id: i64,

    parent_id: Option<i64>,

    visibility_level: i32,

    traversal_path: &str,

    slug: Option<&str>,

) {

    let parent_val = parent_id.map_or("NULL".to_string(), |p| p.to_string());

    let default_slug = format!("namespace-{id}");

    let path_slug = sql_escape(slug.unwrap_or(&default_slug));

    let traversal_ids: Vec<i64> = traversal_path

        .trim_end_matches('/')

        .split('/')

        .filter_map(|s| s.parse().ok())

        .collect();

    let traversal_ids_str = format!(

        "[{}]",

        traversal_ids

            .iter()

            .map(|id| id.to_string())

            .collect::<Vec<_>>()

            .join(",")

    );

    ctx.execute(&format!(

        "INSERT INTO siphon_namespaces \

         (id, name, path, visibility_level, parent_id, owner_id, created_at, updated_at, _siphon_replicated_at) \

         VALUES ({id}, 'namespace-{id}', 'namespace-{id}', {visibility_level}, {parent_val}, 1, \

                 '2023-01-01', '2024-01-15', '2024-01-20 12:00:00')"

         (id, name, path, visibility_level, parent_id, owner_id, traversal_ids, created_at, updated_at, _siphon_replicated_at) \

         VALUES ({id}, '{path_slug}', '{path_slug}', {visibility_level}, {parent_val}, 1, \

                 {traversal_ids_str}, '2023-01-01', '2024-01-15', '2024-01-20 12:00:00')"

    ))

    .await;

    ctx.execute(&format!(
@@ -33,11 +63,34 @@ pub async fn create_project( 
    visibility_level: i32,

    traversal_path: &str,

) {

    create_project_with_path(

        ctx,

        id,

        namespace_id,

        creator_id,

        visibility_level,

        traversal_path,

        None,

    )

    .await;

}



pub async fn create_project_with_path(

    ctx: &TestContext,

    id: i64,

    namespace_id: i64,

    creator_id: i64,

    visibility_level: i32,

    traversal_path: &str,

    slug: Option<&str>,

) {

    let default_slug = format!("project-{id}");

    let path_slug = sql_escape(slug.unwrap_or(&default_slug));

    ctx.execute(&format!(

        "INSERT INTO siphon_projects \

         (id, name, description, visibility_level, path, namespace_id, creator_id, \

          created_at, updated_at, archived, star_count, last_activity_at, _siphon_replicated_at) \

         VALUES ({id}, 'project-{id}', NULL, {visibility_level}, 'project-{id}', {namespace_id}, {creator_id}, \

         VALUES ({id}, '{path_slug}', NULL, {visibility_level}, '{path_slug}', {namespace_id}, {creator_id}, \

                 '2023-01-01', '2024-01-15', false, 0, '2024-01-15', '2024-01-20 12:00:00')"

    ))

    .await;
@@ -47,6 +100,25 @@ pub async fn create_project( 
    .await;

}



pub async fn create_route(

    ctx: &TestContext,

    id: i64,

    source_id: i64,

    source_type: &str,

    path: &str,

    namespace_id: i64,

    traversal_path: &str,

) {

    let escaped_path = sql_escape(path);

    ctx.execute(&format!(

        "INSERT INTO siphon_routes \

         (id, source_id, source_type, path, namespace_id, traversal_path, created_at, updated_at, _siphon_replicated_at) \

         VALUES ({id}, {source_id}, '{source_type}', '{escaped_path}', {namespace_id}, '{traversal_path}', \

                 '2023-01-01', '2024-01-15', '2024-01-20 12:00:00')"

    ))

    .await;

}



pub async fn create_user(ctx: &TestContext, id: i64) {

    ctx.execute(&format!(

        "INSERT INTO siphon_users \