Skip to content
Snippets Groups Projects
Commit e0a3414f authored by John Mason's avatar John Mason
Browse files

Merge branch '402984-start-using-gitlab-zoekt-indexer' into 'master'

Change Zoekt to use Gitaly for fetching code

See merge request !122308



Merged-by: default avatarJohn Mason <9717668-johnmason@users.noreply.gitlab.com>
Approved-by: default avatarMadelein van Niekerk <mvanniekerk@gitlab.com>
Approved-by: Ash McKenzie's avatarAsh McKenzie <amckenzie@gitlab.com>
Approved-by: default avatarJennifer Li <jli@gitlab.com>
Approved-by: default avatarJohn Mason <9717668-johnmason@users.noreply.gitlab.com>
Reviewed-by: Ash McKenzie's avatarAsh McKenzie <amckenzie@gitlab.com>
Co-authored-by: default avatarDmitry Gruzd <dgruzd@gitlab.com>
parents 44093e9e 2a7c777f
No related branches found
No related tags found
1 merge request!122308Change Zoekt to use Gitaly for fetching code
Pipeline #903399468 failed
Pipeline: E2E Omnibus GitLab EE

#903408196

    Pipeline: E2E GDK

    #903402690

      Pipeline: GitLab

      #903400406

        ......@@ -271,7 +271,7 @@
        - name: redis:6.0-alpine
        - name: elasticsearch:7.17.6
        command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"]
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1
        alias: zoekt-ci-image
        variables:
        POSTGRES_HOST_AUTH_METHOD: trust
        ......@@ -289,7 +289,7 @@
        - name: redis:6.2-alpine
        - name: elasticsearch:7.17.6
        command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"]
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1
        alias: zoekt-ci-image
        variables:
        POSTGRES_HOST_AUTH_METHOD: trust
        ......@@ -307,7 +307,7 @@
        - name: redis:6.2-alpine
        - name: elasticsearch:7.17.6
        command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"]
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1
        alias: zoekt-ci-image
        variables:
        POSTGRES_HOST_AUTH_METHOD: trust
        ......@@ -325,7 +325,7 @@
        - name: redis:6.2-alpine
        - name: elasticsearch:7.17.6
        command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"]
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1
        alias: zoekt-ci-image
        variables:
        POSTGRES_HOST_AUTH_METHOD: trust
        ......@@ -342,7 +342,7 @@
        alias: rediscluster # configure connections in config/redis.yml
        - name: redis:6.0-alpine
        - name: elasticsearch:8.6.2
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1
        alias: zoekt-ci-image
        variables:
        POSTGRES_HOST_AUTH_METHOD: trust
        ......@@ -361,7 +361,7 @@
        alias: rediscluster # configure connections in config/redis.yml
        - name: redis:6.0-alpine
        - name: elasticsearch:8.6.2
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1
        alias: zoekt-ci-image
        variables:
        POSTGRES_HOST_AUTH_METHOD: trust
        ......@@ -380,7 +380,7 @@
        alias: rediscluster # configure connections in config/redis.yml
        - name: redis:6.0-alpine
        - name: elasticsearch:8.6.2
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1
        alias: zoekt-ci-image
        variables:
        POSTGRES_HOST_AUTH_METHOD: trust
        ......@@ -401,7 +401,7 @@
        - name: opensearchproject/opensearch:1.3.5
        alias: elasticsearch
        command: ["bin/opensearch", "-E", "discovery.type=single-node", "-E", "plugins.security.disabled=true"]
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1
        alias: zoekt-ci-image
        variables:
        POSTGRES_HOST_AUTH_METHOD: trust
        ......@@ -420,7 +420,7 @@
        - name: opensearchproject/opensearch:2.2.1
        alias: elasticsearch
        command: ["bin/opensearch", "-E", "discovery.type=single-node", "-E", "plugins.security.disabled=true"]
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1
        alias: zoekt-ci-image
        variables:
        POSTGRES_HOST_AUTH_METHOD: trust
        ......@@ -439,7 +439,7 @@
        - name: opensearchproject/opensearch:1.3.5
        alias: elasticsearch
        command: ["bin/opensearch", "-E", "discovery.type=single-node", "-E", "plugins.security.disabled=true"]
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1
        alias: zoekt-ci-image
        variables:
        POSTGRES_HOST_AUTH_METHOD: trust
        ......@@ -458,7 +458,7 @@
        - name: opensearchproject/opensearch:2.2.1
        alias: elasticsearch
        command: ["bin/opensearch", "-E", "discovery.type=single-node", "-E", "plugins.security.disabled=true"]
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1
        alias: zoekt-ci-image
        variables:
        POSTGRES_HOST_AUTH_METHOD: trust
        ......@@ -477,7 +477,7 @@
        - name: opensearchproject/opensearch:1.3.5
        alias: elasticsearch
        command: ["bin/opensearch", "-E", "discovery.type=single-node", "-E", "plugins.security.disabled=true"]
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1
        alias: zoekt-ci-image
        variables:
        POSTGRES_HOST_AUTH_METHOD: trust
        ......@@ -496,7 +496,7 @@
        - name: opensearchproject/opensearch:2.2.1
        alias: elasticsearch
        command: ["bin/opensearch", "-E", "discovery.type=single-node", "-E", "plugins.security.disabled=true"]
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0
        - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1
        alias: zoekt-ci-image
        variables:
        POSTGRES_HOST_AUTH_METHOD: trust
        ......
        ......@@ -4,15 +4,23 @@ module Zoekt
        module SearchableRepository
        extend ActiveSupport::Concern
        READ_TIMEOUT_S = 10.minutes.to_i
        INDEXING_TIMEOUT_S = 10.minutes.to_i
        class_methods do
        def truncate_zoekt_index!(shard)
        ::Gitlab::HTTP.post(
        URI.join(shard.index_base_url, '/truncate'),
        URI.join(shard.index_base_url, zoekt_indexer_truncate_path),
        allow_local_requests: true
        )
        end
        def zoekt_indexer_truncate_path
        use_new_zoekt_indexer? ? '/indexer/truncate' : '/truncate'
        end
        def use_new_zoekt_indexer?
        ::Feature.enabled?(:use_new_zoekt_indexer)
        end
        end
        included do
        ......@@ -20,33 +28,73 @@ def use_zoekt?
        project&.use_zoekt?
        end
        def update_zoekt_index!(use_local_disk_path: false)
        repository_url = if use_local_disk_path
        path_to_repo
        else
        project.http_url_to_repo
        end
        def use_new_zoekt_indexer?
        self.class.use_new_zoekt_indexer?
        end
        payload = { CloneUrl: repository_url, RepoId: project.id }
        def update_zoekt_index!
        use_new_zoekt_indexer? ? use_new_indexer! : use_legacy_indexer!
        end
        response = ::Gitlab::HTTP.post(
        URI.join(zoekt_index_base_url, '/index'),
        def async_update_zoekt_index
        ::Zoekt::IndexerWorker.perform_async(project.id)
        end
        private
        def zoekt_indexer_post(path, payload)
        ::Gitlab::HTTP.post(
        URI.join(zoekt_index_base_url, path),
        headers: { "Content-Type" => "application/json" },
        body: payload.to_json,
        allow_local_requests: true,
        timeout: READ_TIMEOUT_S
        timeout: INDEXING_TIMEOUT_S
        )
        end
        def use_legacy_indexer!
        payload = { CloneUrl: project.http_url_to_repo, RepoId: project.id }
        response = zoekt_indexer_post('/index', payload)
        raise response['Error'] if response['Error']
        response
        end
        def async_update_zoekt_index
        ::Zoekt::IndexerWorker.perform_async(project.id)
        def use_new_indexer!
        response = zoekt_indexer_post('/indexer/index', indexing_payload)
        raise response['Error'] if response['Error']
        raise "Request failed with: #{response.inspect}" unless response.success?
        response
        end
        private
        def indexing_payload
        repository_storage = project.repository_storage
        connection_info = Gitlab::GitalyClient.connection_data(repository_storage)
        repository_path = "#{project.repository.disk_path}.git"
        address = connection_info['address']
        # This code is needed to support relative unix: connection strings. For example, specs
        if address.match?(%r{\Aunix:[^/.]})
        path = address.split('unix:').last
        address = "unix:#{Rails.root.join(path)}"
        end
        {
        GitalyConnectionInfo: {
        Address: address,
        Token: connection_info['token'],
        Storage: repository_storage,
        Path: repository_path
        },
        RepoId: project.id,
        FileSizeLimit: Gitlab::CurrentSettings.elasticsearch_indexed_file_size_limit_kb.kilobytes,
        Timeout: "#{INDEXING_TIMEOUT_S}s"
        }
        end
        def zoekt_index_base_url
        Zoekt::IndexedNamespace.where(namespace: project.root_namespace).first&.shard&.index_base_url
        ......
        ---
        name: use_new_zoekt_indexer
        introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/122308
        rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/414440
        milestone: '16.1'
        type: development
        group: group::global search
        default_enabled: false
        ......@@ -57,10 +57,22 @@ def search_for(term)
        expect { repository.update_zoekt_index! }.to raise_error(RuntimeError, 'command failed: exit status 128')
        end
        it 'raises an exception when response is not successful' do
        response = {}
        allow(response).to receive(:success?).and_return(false)
        allow(::Gitlab::HTTP).to receive(:post).and_return(response)
        expect { repository.update_zoekt_index! }.to raise_error(RuntimeError, /Request failed with/)
        end
        it 'sets http the correct timeout' do
        response = {}
        allow(response).to receive(:success?).and_return(true)
        expect(::Gitlab::HTTP).to receive(:post)
        .with(anything, hash_including(timeout: described_class::READ_TIMEOUT_S))
        .and_return({})
        .with(anything, hash_including(timeout: described_class::INDEXING_TIMEOUT_S))
        .and_return(response)
        repository.update_zoekt_index!
        end
        ......
        ......@@ -24,10 +24,7 @@ def zoekt_ensure_namespace_indexed!(namespace)
        def zoekt_ensure_project_indexed!(project)
        zoekt_ensure_namespace_indexed!(project.namespace)
        # TODO: We shouldn't be referencing files on disk but I don't think we
        # can git clone from rspec as Web/API is not running
        allow(::Gitlab::GitalyClient::StorageSettings).to receive(:disk_access_denied?).and_return(false)
        project.repository.update_zoekt_index!(use_local_disk_path: true)
        project.repository.update_zoekt_index!
        end
        end
        end
        ......
        0% Loading or .
        You are about to add 0 people to the discussion. Proceed with caution.
        Finish editing this message first!
        Please register or to comment