Cannot create new project from template

Summary

I'm having issues with project creation on a self hosted gitlab instance.

The gitlab instance is hosted on a EKS cluster and deployed with the Gitlab operator

When creating a new project from a template, I will get the following issue Screenshot_2023-05-31_at_3.22.45_pm

This is what the gitaly pod logs

[
  {
    "component": "gitaly",
    "subcomponent": "gitaly",
    "level": "info",
    "correlation_id": "01H1R3J4Y77Q02031JT00DCAZ2",
    "grpc.code": "OK",
    "grpc.meta.auth_version": "v2",
    "grpc.meta.client_name": "gitlab-web",
    "grpc.meta.deadline_type": "regular",
    "grpc.meta.method_type": "unary",
    "grpc.method": "RepositoryExists",
    "grpc.request.deadline": "2023-05-31T05:12:58.006",
    "grpc.request.fullMethod": "/gitaly.RepositoryService/RepositoryExists",
    "grpc.request.glProjectPath": "s2-infrastructure/test",
    "grpc.request.glRepository": "project-23",
    "grpc.request.payload_bytes": 132,
    "grpc.request.repoPath": "@hashed/53/5f/535fa30d7e25dd8a49f1536779734ec8286108d115da5045d77f3b4185d8f790.git",
    "grpc.request.repoStorage": "default",
    "grpc.response.payload_bytes": 0,
    "grpc.service": "gitaly.RepositoryService",
    "grpc.start_time": "2023-05-31T05:12:48.966",
    "grpc.time_ms": 0.31,
    "limit.concurrency_queue_ms": 0,
    "msg": "finished unary call with code OK",
    "peer.address": "10.0.45.225:33376",
    "pid": 1,
    "remote_ip": "104.28.196.199",
    "span.kind": "server",
    "system": "grpc",
    "time": "2023-05-31T05:12:48.967Z",
    "user_id": "4",
    "username": "javion"
  },
  {
    "component": "gitaly",
    "subcomponent": "gitaly",
    "level": "info",
    "correlation_id": "01H1R3J4Y77Q02031JT00DCAZ2",
    "error": "GetRepoPath: not a git repository: \"/home/git/repositories/@hashed/53/5f/535fa30d7e25dd8a49f1536779734ec8286108d115da5045d77f3b4185d8f790.git\"",
    "grpc.code": "NotFound",
    "grpc.meta.auth_version": "v2",
    "grpc.meta.client_name": "gitlab-web",
    "grpc.meta.deadline_type": "regular",
    "grpc.meta.method_type": "unary",
    "grpc.method": "FindDefaultBranchName",
    "grpc.request.deadline": "2023-05-31T05:12:59.003",
    "grpc.request.fullMethod": "/gitaly.RefService/FindDefaultBranchName",
    "grpc.request.glProjectPath": "s2-infrastructure/test",
    "grpc.request.glRepository": "project-23",
    "grpc.request.payload_bytes": 132,
    "grpc.request.repoPath": "@hashed/53/5f/535fa30d7e25dd8a49f1536779734ec8286108d115da5045d77f3b4185d8f790.git",
    "grpc.request.repoStorage": "default",
    "grpc.response.payload_bytes": 0,
    "grpc.service": "gitaly.RefService",
    "grpc.start_time": "2023-05-31T05:12:49.123",
    "grpc.time_ms": 0.23,
    "limit.concurrency_queue_ms": 0,
    "msg": "finished unary call with code NotFound",
    "peer.address": "10.0.45.225:33376",
    "pid": 1,
    "remote_ip": "104.28.196.199",
    "span.kind": "server",
    "system": "grpc",
    "time": "2023-05-31T05:12:49.123Z",
    "user_id": "4",
    "username": "javion"
  },
  {
    "component": "gitaly",
    "subcomponent": "gitaly",
    "level": "info",
    "correlation_id": "01H1R3J6DCMEGXCWX9W6YARCQ2",
    "error": "GetRepoPath: not a git repository: \"/home/git/repositories/@hashed/53/5f/535fa30d7e25dd8a49f1536779734ec8286108d115da5045d77f3b4185d8f790.git\"",
    "grpc.code": "NotFound",
    "grpc.meta.auth_version": "v2",
    "grpc.meta.client_name": "gitlab-web",
    "grpc.meta.deadline_type": "regular",
    "grpc.meta.method_type": "unary",
    "grpc.method": "FindDefaultBranchName",
    "grpc.request.deadline": "2023-05-31T05:13:00.007",
    "grpc.request.fullMethod": "/gitaly.RefService/FindDefaultBranchName",
    "grpc.request.glProjectPath": "s2-infrastructure/test",
    "grpc.request.glRepository": "project-23",
    "grpc.request.payload_bytes": 132,
    "grpc.request.repoPath": "@hashed/53/5f/535fa30d7e25dd8a49f1536779734ec8286108d115da5045d77f3b4185d8f790.git",
    "grpc.request.repoStorage": "default",
    "grpc.response.payload_bytes": 0,
    "grpc.service": "gitaly.RefService",
    "grpc.start_time": "2023-05-31T05:12:50.448",
    "grpc.time_ms": 0.274,
    "limit.concurrency_queue_ms": 0,
    "msg": "finished unary call with code NotFound",
    "peer.address": "10.0.45.225:33376",
    "pid": 1,
    "remote_ip": "104.28.196.199",
    "span.kind": "server",
    "system": "grpc",
    "time": "2023-05-31T05:12:50.448Z",
    "user_id": "4",
    "username": "javion"
  },
  {
    "component": "gitaly",
    "subcomponent": "gitaly",
    "level": "info",
    "correlation_id": "01H1R3J6DCMEGXCWX9W6YARCQ2",
    "error": "GetRepoPath: not a git repository: \"/home/git/repositories/@hashed/53/5f/535fa30d7e25dd8a49f1536779734ec8286108d115da5045d77f3b4185d8f790.git\"",
    "grpc.code": "NotFound",
    "grpc.meta.auth_version": "v2",
    "grpc.meta.client_name": "gitlab-web",
    "grpc.meta.deadline_type": "regular",
    "grpc.meta.method_type": "unary",
    "grpc.method": "FindDefaultBranchName",
    "grpc.request.deadline": "2023-05-31T05:13:00.002",
    "grpc.request.fullMethod": "/gitaly.RefService/FindDefaultBranchName",
    "grpc.request.glProjectPath": "s2-infrastructure/test",
    "grpc.request.glRepository": "project-23",
    "grpc.request.payload_bytes": 132,
    "grpc.request.repoPath": "@hashed/53/5f/535fa30d7e25dd8a49f1536779734ec8286108d115da5045d77f3b4185d8f790.git",
    "grpc.request.repoStorage": "default",
    "grpc.response.payload_bytes": 0,
    "grpc.service": "gitaly.RefService",
    "grpc.start_time": "2023-05-31T05:12:50.452",
    "grpc.time_ms": 0.189,
    "limit.concurrency_queue_ms": 0,
    "msg": "finished unary call with code NotFound",
    "peer.address": "10.0.45.225:33376",
    "pid": 1,
    "remote_ip": "104.28.196.199",
    "span.kind": "server",
    "system": "grpc",
    "time": "2023-05-31T05:12:50.452Z",
    "user_id": "4",
    "username": "javion"
  },
  {
    "component": "gitaly",
    "subcomponent": "gitaly",
    "level": "info",
    "correlation_id": "01H1R3J6DCMEGXCWX9W6YARCQ2",
    "grpc.code": "OK",
    "grpc.meta.auth_version": "v2",
    "grpc.meta.client_name": "gitlab-web",
    "grpc.meta.deadline_type": "regular",
    "grpc.meta.method_type": "unary",
    "grpc.method": "RepositoryExists",
    "grpc.request.deadline": "2023-05-31T05:13:00.001",
    "grpc.request.fullMethod": "/gitaly.RepositoryService/RepositoryExists",
    "grpc.request.glProjectPath": "s2-infrastructure/test",
    "grpc.request.glRepository": "project-23",
    "grpc.request.payload_bytes": 132,
    "grpc.request.repoPath": "@hashed/53/5f/535fa30d7e25dd8a49f1536779734ec8286108d115da5045d77f3b4185d8f790.git",
    "grpc.request.repoStorage": "default",
    "grpc.response.payload_bytes": 0,
    "grpc.service": "gitaly.RepositoryService",
    "grpc.start_time": "2023-05-31T05:12:50.462",
    "grpc.time_ms": 0.121,
    "limit.concurrency_queue_ms": 0,
    "msg": "finished unary call with code OK",
    "peer.address": "10.0.45.225:33376",
    "pid": 1,
    "remote_ip": "104.28.196.199",
    "span.kind": "server",
    "system": "grpc",
    "time": "2023-05-31T05:12:50.462Z",
    "user_id": "4",
    "username": "javion"
  },
  {
    "component": "gitaly",
    "subcomponent": "gitaly",
    "level": "info",
    "correlation_id": "01H1R3J6DCMEGXCWX9W6YARCQ2",
    "error": "GetRepoPath: not a git repository: \"/home/git/repositories/@hashed/53/5f/535fa30d7e25dd8a49f1536779734ec8286108d115da5045d77f3b4185d8f790.git\"",
    "grpc.code": "NotFound",
    "grpc.meta.auth_version": "v2",
    "grpc.meta.client_name": "gitlab-web",
    "grpc.meta.deadline_type": "regular",
    "grpc.meta.method_type": "unary",
    "grpc.method": "FindDefaultBranchName",
    "grpc.request.deadline": "2023-05-31T05:13:00.002",
    "grpc.request.fullMethod": "/gitaly.RefService/FindDefaultBranchName",
    "grpc.request.glProjectPath": "s2-infrastructure/test",
    "grpc.request.glRepository": "project-23",
    "grpc.request.payload_bytes": 132,
    "grpc.request.repoPath": "@hashed/53/5f/535fa30d7e25dd8a49f1536779734ec8286108d115da5045d77f3b4185d8f790.git",
    "grpc.request.repoStorage": "default",
    "grpc.response.payload_bytes": 0,
    "grpc.service": "gitaly.RefService",
    "grpc.start_time": "2023-05-31T05:12:50.632",
    "grpc.time_ms": 0.257,
    "limit.concurrency_queue_ms": 0,
    "msg": "finished unary call with code NotFound",
    "peer.address": "10.0.45.225:33376",
    "pid": 1,
    "remote_ip": "104.28.196.199",
    "span.kind": "server",
    "system": "grpc",
    "time": "2023-05-31T05:12:50.633Z",
    "user_id": "4",
    "username": "javion"
  }
]

In addition. Creating a blank project also had some minor issues, which seems to be related. When created a new blank project, even if the initialize with README checkbox is checked, the project will be created without a readme, nor a branch or anything. It is a completely empty project that I end up having to push a new master from a remote location to initialize.

I've also done some further debugging, but I'm not sure if the findings are relevant. My gitaly pod uses a PVC with efs-sc as the storage class.

Shelling into the gitaly pod

Defaulted container "gitaly" out of: gitaly, certificates (init), configure (init)
$ ls -laht /home/git/repositories/
total 16K
drwxr-x--- 12 50004 50004 6.0K May 31 04:39 @hashed
drwxr-xr-x  4 50004 50004 6.0K May 31 04:39 +gitaly
drwx------  4 50004 50004 6.0K May 28 22:58 .
drwxr-xr-x  1 git   git     26 May 10 11:00 ..
-rw-------  1 50004 50004   64 May  9 13:31 .gitaly-metadata
$ whoami
git

I also have a suspicion that it might be because the file permissions are not correct.

In addition,

$ cd @hashed
$ ls
3f  45  4b  4e  6b  6f  78  85  b1  d4  e6  f5

The directory from the log

error":"GetRepoPath: not a git repository: \"/home/git/repositories/@hastherhed/53/5f/535fa30d7e25dd8a49f1536779734ec8286108d115da5045d77f3b4185d8f790.git\""

indeed does not exist as we can see that there is no 53 directory inside @hashed

Help on this would be super appreciated

Steps to reproduce

  1. Create a new project
  2. Select any template
  3. Click create

Configuration used

(Please provide a sanitized version of the configuration used wrapped in a code block (```yaml))

            global:
                edition: ce
                nodeSelector:
                    kubernetes.io/arch: amd64
                batch:
                    cronJob:
                        apiVersion: batch/v1
                hpa:
                    apiVersion: autoscaling/v2
                shell:
                    authToken:
                        secret: <hidden>
                        key: password
                hosts:
                    domain: <hidden>
                    https: false
                    gitlab:
                        name: <hidden>
                        https: false
                ingress:
                    class: nginx
                    tls:
                        enabled: false
                psql:
                    password:
                        secret: <hidden>
                        key: password
                    host: <hidden>
                    port: <hidden>
                    username: <hidden>
                    database: gitlab
                minio:
                    # Using s3 instead
                    enabled: false
                redis:
                    password:
                        enabled: false
                    host: <hidden>
                    port: <hidden>
                registry:
                    enabled: false
                appConfig:
                    defaultProjectsFeatures:
                        containerRegistry: false
                    enableUsagePing: false
                    enableImpersonation: true
                    # For context on the ObjectStorage configuration below, see:
                    # https://docs.gitlab.com/ee/administration/reference_architectures/2k_users.html#configure-the-object-storage
                    object_store:
                        enabled: true
                    artifacts:
                        bucket: <hidden>
                    backups:
                        bucket: <hidden>
                        tmpBucket: <hidden>
                    dependencyProxy:
                        bucket: <hidden>
                    externalDiffs:
                        bucket: <hidden>
                        # Recommend to only store older External Diffs in Object Storage for performance reasons
                        when: outdated
                    lfs:
                        bucket: <hidden>
                    packages:
                        bucket: <hidden>
                    terraformState:
                        bucket: <hidden>
                    uploads:
                        bucket: <hidden>
                    omniauth: <hidden>
            certmanager:
                install: false
            nginx-ingress:
                enabled: false
            prometheus:
                install: false
            redis:
                install: false
            postgresql:
                install: false
            registry:
                # Use ECR
                enabled: false
            # https://docs.gitlab.com/ee/ci/runners/#configuring-runners-in-gitlab
            gitlab-runner:
                install: false
            gitlab:
                webservice:
                    # Min pods recommended to be set to 75% of max to start
                    maxReplicas: 3
                    maxUnavailable: 1
                    minReplicas: 2
                    registry:
                        enabled: false
                    # Increased to stabilize scaling and accommodate for higher CPU load on large environment
                    hpa:
                        cpu:
                            targetAverageValue: 1600m
                    ingress:
                        proxyBodySize: 0 # To allow large file uploads like imports
                    workerProcesses: 4
                sidekiq:
                    maxReplicas: 4
                    minReplicas: 3
                    registry:
                        enabled: false
                    # Increased to stabilize scaling and accommodate for higher CPU load on large environment
                    hpa:
                        cpu:
                            targetAverageValue: 700m
                    backups:
                        cron:
                            enabled: true
                            schedule: "0 15 * * *"
                        objectStorage:
                            backend: s3
                migrations:
                    # Migrations pod must point directly to PostgreSQL primary
                    psql:
                        password:
                            secret: <hidden>
                            key: password
                        host: <hidden>
                        port: <hidden>

Current behavior

Cannot create from template

Expected behavior

I should be able to create new project from template

Versions

  • Chart: 6.11.3
  • Platform:
    • Cloud: EKS
  • Kubernetes: (kubectl version)
    • Client: version.Info{Major:"1", Minor:"26", GitVersion:"v1.26.0", GitCommit:"b46a3f887ca979b1a5d14fd39cb1af43e7e5d12d", GitTreeState:"clean", BuildDate:"2022-12-08T19:51:43Z", GoVersion:"go1.19.4", Compiler:"gc", Platform:"darwin/arm64"}
    • Server: version.Info{Major:"1", Minor:"26+", GitVersion:"v1.26.4-eks-0a21954", GitCommit:"4a3479673cb6d9b63f1c69a67b57de30a4d9b781", GitTreeState:"clean", BuildDate:"2023-04-15T00:33:09Z", GoVersion:"go1.19.8", Compiler:"gc", Platform:"linux/amd64"}
  • Helm: (helm version)
    • Client: version.BuildInfo{Version:"v3.10.2", GitCommit:"50f003e5ee8704ec937a756c646870227d7c8b58", GitTreeState:"clean", GoVersion:"go1.18.8"}
    • Server:

Relevant logs

(Please provide any relevate log snippets you have collected, using code blocks (```) to format)

Edited by Javion Cai