Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • limansky/gitaly
  • cmingxu/gitaly
  • bugrargn/gitaly
  • asedge/gitaly
  • softwareplumber/gitaly
  • gitlab-org/gitaly
  • armbiant/gitlab-gitaly
  • sinwav/gitaly
  • Mattlk13/gitaly
  • Ruby-and-Friends/gitaly
  • kutelev/gitaly
  • bbodenmiller/gitaly
  • alejandro/gitaly
  • lukeshu/gitaly
  • eddiet/gitaly
  • mpoornima/gitaly
  • dturner_ts/gitaly
  • zj-gitlab/gitaly
  • nadiavu/gitaly
  • balasankarc/gitaly
  • mpszone/gitaly
  • sysu-liyanliang/gitaly
  • frenkel/gitaly
  • lfauts/gitaly
  • yang-zaixin/gitaly
  • mydigitalself/gitaly
  • litongdeng/gitaly
  • toon/gitaly
  • axot/gitaly
  • josecordaz/gitaly
  • sj14/gitaly
  • MaykonOliveira/gitaly
  • zjkyz8/gitaly
  • svankmajer/gitaly
  • qipa/gitaly
  • sptr/gitaly
  • kiameisomabes/gitaly
  • bradobro/gitaly
  • fcgravalos/gitaly
  • reprazent/gitaly
  • tbuida/gitaly
  • arjunmayilvaganan/gitaly
  • b1n/gitaly
  • timniederhausen/gitaly
  • maksim-paskal/gitaly
  • zhutiankai/gitaly
  • maxmati/gitaly
  • 358141721/gitaly
  • despinosa/gitaly
  • c.bianchi/gitaly
  • tamilselvan10021998/gitaly
  • Randomneo/gitaly
  • yemanaung/gitaly
  • siemens/gitaly
  • avar/gitaly
  • SeananXu/gitaly
  • smusavi/gitaly
  • johncai/gitaly
  • utkarsh2102/gitaly
  • jhenkens/gitaly
  • ba2014sheer/gitaly
  • jacobvosmaer-gitlab/gitaly
  • Esteban_Carnuccio/gitaly
  • d0c-s4vage/gitaly
  • thetradedesk/gitaly
  • AfolabiOlaoluwa/gitaly
  • njkevlani/gitaly
  • chriscool/gitaly
  • ethan.gitlab/gitaly
  • GeniusLearner/gitaly
  • wasphin/gitaly
  • CauhxMilloy/gitaly
  • nnelson/gitaly
  • simchatop/gitaly
  • htkaslan/gitaly
  • hemanthdev/gitaly
  • shovanmaity/gitaly
  • Temtaime/gitaly
  • Ash560/gitaly
  • gitlab-bot-readonly/gitaly
  • tylerthetester/gitaly
  • sushovan/gitaly
  • shy-Xu/gitaly
  • thmr1/gitaly
  • pravi/gitaly
  • EphremDms/gitaly
  • sluongng/gitaly
  • nejc/gitaly
  • ksashikumar/gitaly
  • motogami/gitaly
  • bs223617/gitaly
  • eagle_fly_sky/gitaly
  • hashworks/gitaly
  • anatol.pomozov/gitaly
  • T4cC0re/gitaly
  • ethan.reesor/contrib/gitaly
  • czh592/gitaly
  • wchandler/gitaly
  • diogo_nicoleti_ifood/gitaly
  • bulemka12345poczta/gitaly
  • taroguru/gitaly
  • dyrone/gitaly
  • inguin/gitaly
  • mathieuusoyan/gitaly
  • mfechner/gitaly
  • wwwicbd/icbd-gitaly
  • blanet/gitaly
  • paulsd2020/gitaly
  • harry-hov/gitaly
  • pranav/gitaly
  • abhitidarbar/gitaly
  • heygirlhey356/gitaly
  • maoqiang/gitaly
  • yyexplore/gitaly
  • cgsyam/gitaly
  • behrmann/gitaly
  • nfishe/gitaly
  • g4s8/gitaly
  • brett.higgins/gitaly
  • test11042/gitaly
  • jeffzhu503/gitaly
  • TheSide/gitaly
  • nanmu42/gitaly
  • van4elotti/gitaly
  • Cactusinhand/gitaly
  • yamini.bhaskar007/gitaly
  • icbd/gitaly
  • ooulwluoo/gitaly
  • nagypeter/gitaly
  • Abhilashsiyer/gitaly
  • jiangxin/gitaly
  • KimJi55/gitaly
  • flycutter/gitaly
  • leonard-adleman-sec/gitaly
  • ryan-qianchen/gitaly
  • feistel/gitaly
  • snowcrystall/gitaly
  • edith007/gitaly
  • zhanglinjie/gitaly
  • luminarrr/gitaly
  • stephankirsten/gitaly
  • etanot/gitaly
  • nikovega21/gitaly
  • m0rosan/gitaly
  • HaroldKnowlden/gitaly
  • iotcl/test/gitaly
  • okenwaonyebuchi103/gitaly
  • simpleclickers/gitaly
  • kdaudt/gitaly
  • YushuaiLI/gitaly
  • davebarr/gitaly
  • dgsdoug/gitaly
  • trakos/gitaly
  • XciD/gitaly
  • imskr/gitaly
  • mj.bruijns/gitaly
  • markoke256/gitaly
  • Sahil162/gitaly
  • Kartik1397/gitaly
  • akumar1503/gitaly
  • bill2022/gitaly
  • ncu_gsoc/gitaly
  • SURA907/gitaly
  • hongeinh/gitaly
  • joshua.sickmeyer/gitaly
  • adlternative/gitaly
  • amstal93/gitaly
  • gitlab-renovate-forks/gitaly
  • xiaowenxia/gitaly
  • checkscale-gitlab/gitaly
  • KyleFromKitware/gitaly
  • n0h4ppy/gitaly
  • pwn3/gitaly
  • jokerpwn/gitaly
  • salamacha7389/gitaly
  • L11R/gitaly
  • hanmingliang/gitaly
  • Isengart1/gitaly
  • cbj/gitaly
  • gitlab-community/gitaly
  • shamanthb90/gitaly
  • mjkalasky2/gitaly
  • VladPetriv/gitaly
  • arkn98/gitaly
  • scwang18/gitaly
  • sebicioacata/gitaly
  • edsonmichaque/gitaly
  • Kexin2000/gitaly
  • johnwparent/gitaly
  • kevin.rojas/wr-gitaly
  • marc.ratombotsoa/gitaly
  • igor.drozdov/gitaly
  • radityasurya1911/gitaly
  • CS2Us/gitaly
  • RryLee/gitaly
  • liruixin-coding/gitaly
  • gerardo/gitaly
  • josephburnett/gitaly
  • b.s.strelnikov_tinkoff/gitaly
  • OK_MF/gitaly-fork
  • zeb0x01/gitaly
  • 8bitlife/gitaly
  • nraj0408/gitaly
  • troyfox758/gitaly
  • ouladsine.saloua/gitaly
  • qeesung/gitaly
  • dzcdada123/gitaly
  • ashmckenzie/gitaly
  • Freedisch/gitaly
  • xsizxenjin/gitaly
  • eric.p.ju/gitaly
  • zzhzero/gitaly
  • lenghan1991/gitaly
  • jhammer101/gitaly
  • wheredidfranciscogo1/gitaly
  • zakiir206/gitaly
  • jonas.hogman/gitaly
  • sauravchanda9/gitaly
  • lzampier/gitaly
  • fkhe/gitaly
  • liaoxingju/gitaly
  • bufdev/gitaly
  • a.songer/gitaly
  • bcooksley/gitaly
  • Wangyadong1108/gitaly
  • MashyBasker/gitaly
  • nickaldwin/gitaly
  • avevlad/gitaly
  • sat-h/gitaly
  • wingred96399/gitaly-mine
  • chen_feng/gitaly
  • grootwang/gitaly
  • bhatianikhil551/gitaly
  • schuam_alice/gitaly
  • armbiant/hive-gitaly
  • ollevche/gitaly
  • HassanAkbar/gitaly
  • echui-gitlab/gitaly
  • ebrahim.poursadeghi/gitaly
239 results
Show changes
Commits on Source (38)
Showing
with 521 additions and 238 deletions
3.2.4 3.3.7
# Versions of Gitaly dependencies managed by asdf. # Versions of Gitaly dependencies managed by asdf.
golang 1.22.6 1.23.0 golang 1.22.6 1.23.0
ruby 3.2.4 ruby 3.3.7
# For linting documentation # For linting documentation
nodejs 20.18.0 # Dependency for markdownlint-cli2 nodejs 20.18.0 # Dependency for markdownlint-cli2
......
# Gitaly changelog # Gitaly changelog
## 17.8.1 (2025-01-22)
No changes.
## 17.8.0 (2025-01-15) ## 17.8.0 (2025-01-15)
No changes. No changes.
## 17.7.3 (2025-01-22)
No changes.
## 17.7.2 (2025-01-14) ## 17.7.2 (2025-01-14)
No changes. No changes.
...@@ -22,6 +30,10 @@ No changes. ...@@ -22,6 +30,10 @@ No changes.
- [gitaly: Query available archived WAL entries in recovery status](gitlab-org/gitaly@a8b11d9ce0a46d5d3a170bca355d665b373f1f9e) ([merge request](gitlab-org/gitaly!7404)) - [gitaly: Query available archived WAL entries in recovery status](gitlab-org/gitaly@a8b11d9ce0a46d5d3a170bca355d665b373f1f9e) ([merge request](gitlab-org/gitaly!7404))
## 17.6.4 (2025-01-22)
No changes.
## 17.6.3 (2025-01-08) ## 17.6.3 (2025-01-08)
No changes. No changes.
......
...@@ -143,7 +143,7 @@ GIT_VERSION ?= ...@@ -143,7 +143,7 @@ GIT_VERSION ?=
# GIT_VERSION_x_xx defines versions for each instance of bundled Git we ship. When a new # GIT_VERSION_x_xx defines versions for each instance of bundled Git we ship. When a new
# major version is added, be sure to update GIT_PACKED_EXECUTABLES, the *-bundled-git targets, # major version is added, be sure to update GIT_PACKED_EXECUTABLES, the *-bundled-git targets,
# and add new targets under the "# These targets build specific releases of Git." section. # and add new targets under the "# These targets build specific releases of Git." section.
GIT_VERSION_2_47 ?= v2.47.0 GIT_VERSION_2_47 ?= v2.47.2
GIT_VERSION_2_48 ?= v2.48.1 GIT_VERSION_2_48 ?= v2.48.1
# #
# OVERRIDE_GIT_VERSION allows you to specify a custom semver value to be reported by the # OVERRIDE_GIT_VERSION allows you to specify a custom semver value to be reported by the
...@@ -315,6 +315,12 @@ install: build ...@@ -315,6 +315,12 @@ install: build
## Build bundled Git binaries. ## Build bundled Git binaries.
build-bundled-git: build-bundled-git-v2.47 build-bundled-git-v2.48 build-bundled-git: build-bundled-git-v2.47 build-bundled-git-v2.48
build-bundled-git-v2.47: $(patsubst %,${BUILD_DIR}/bin/gitaly-%-v2.47,${GIT_EXECUTABLES}) build-bundled-git-v2.47: $(patsubst %,${BUILD_DIR}/bin/gitaly-%-v2.47,${GIT_EXECUTABLES})
# Use non-collision-detecting SHA1 implementation in non-cryptographic scenarios
# to improve performance. For now, this is only enabled for Git version 2.48 on
# Linux platforms.
ifeq ($(OS), Linux)
build-bundled-git-v2.48: override GIT_BUILD_OPTIONS += OPENSSL_SHA1_UNSAFE=YesPlease
endif
build-bundled-git-v2.48: $(patsubst %,${BUILD_DIR}/bin/gitaly-%-v2.48,${GIT_EXECUTABLES}) build-bundled-git-v2.48: $(patsubst %,${BUILD_DIR}/bin/gitaly-%-v2.48,${GIT_EXECUTABLES})
.PHONY: install-bundled-git .PHONY: install-bundled-git
...@@ -400,7 +406,7 @@ bench: ${BENCHMARK_REPO} prepare-tests ...@@ -400,7 +406,7 @@ bench: ${BENCHMARK_REPO} prepare-tests
## Since the reftable code isn't tagged in Git yet, to run it locally ## Since the reftable code isn't tagged in Git yet, to run it locally
## we have to specify the git version too: ## we have to specify the git version too:
## 'GIT_DEFAULT_REF_FORMAT=reftable OVERRIDE_GIT_VERSION="v99.99.99" GIT_VERSION="master" make test-go' ## 'GIT_DEFAULT_REF_FORMAT=reftable OVERRIDE_GIT_VERSION="v99.99.99" GIT_VERSION="master" make test-go'
test-with-reftable: export GIT_DEFAULT_REF_FORMAT = reftable test-with-reftable: export GITALY_TEST_REF_FORMAT = reftable
test-with-reftable: test-go test-with-reftable: test-go
.PHONY: test-with-sha256 .PHONY: test-with-sha256
......
...@@ -36,7 +36,7 @@ require ( ...@@ -36,7 +36,7 @@ require (
github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_golang v1.20.5
github.com/prometheus/client_model v0.6.1 github.com/prometheus/client_model v0.6.1
github.com/prometheus/common v0.61.0 github.com/prometheus/common v0.61.0
github.com/rubenv/sql-migrate v1.7.0 github.com/rubenv/sql-migrate v1.7.1
github.com/sirupsen/logrus v1.9.3 github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.10.0 github.com/stretchr/testify v1.10.0
github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-client-go v2.30.0+incompatible
...@@ -46,7 +46,7 @@ require ( ...@@ -46,7 +46,7 @@ require (
go.uber.org/automaxprocs v1.6.0 go.uber.org/automaxprocs v1.6.0
go.uber.org/goleak v1.3.0 go.uber.org/goleak v1.3.0
gocloud.dev v0.40.1-0.20241107185025-56954848c3aa gocloud.dev v0.40.1-0.20241107185025-56954848c3aa
golang.org/x/crypto v0.30.0 golang.org/x/crypto v0.31.0
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f
golang.org/x/sync v0.10.0 golang.org/x/sync v0.10.0
golang.org/x/sys v0.29.0 golang.org/x/sys v0.29.0
......
...@@ -588,8 +588,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L ...@@ -588,8 +588,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/rubenv/sql-migrate v1.7.0 h1:HtQq1xyTN2ISmQDggnh0c9U3JlP8apWh8YO2jzlXpTI= github.com/rubenv/sql-migrate v1.7.1 h1:f/o0WgfO/GqNuVg+6801K/KW3WdDSupzSjDYODmiUq4=
github.com/rubenv/sql-migrate v1.7.0/go.mod h1:S4wtDEG1CKn+0ShpTtzWhFpHHI5PvCUtiGI+C+Z2THE= github.com/rubenv/sql-migrate v1.7.1/go.mod h1:Ob2Psprc0/3ggbM6wCzyYVFFuc6FyZrb2AS+ezLDFb4=
github.com/rubyist/tracerx v0.0.0-20170927163412-787959303086 h1:mncRSDOqYCng7jOD+Y6+IivdRI6Kzv2BLWYkWkdQfu0= github.com/rubyist/tracerx v0.0.0-20170927163412-787959303086 h1:mncRSDOqYCng7jOD+Y6+IivdRI6Kzv2BLWYkWkdQfu0=
github.com/rubyist/tracerx v0.0.0-20170927163412-787959303086/go.mod h1:YpdgDXpumPB/+EGmGTYHeiW/0QVFRzBYTNFaxWfPDk4= github.com/rubyist/tracerx v0.0.0-20170927163412-787959303086/go.mod h1:YpdgDXpumPB/+EGmGTYHeiW/0QVFRzBYTNFaxWfPDk4=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
...@@ -717,8 +717,8 @@ golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2Uz ...@@ -717,8 +717,8 @@ golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2Uz
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
......
...@@ -2,61 +2,45 @@ package bundleuri ...@@ -2,61 +2,45 @@ package bundleuri
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"strconv"
"gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/gitcmd" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gitcmd"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage"
"gitlab.com/gitlab-org/gitaly/v16/internal/log"
) )
// CapabilitiesGitConfig returns a slice of gitcmd.ConfigPairs that can be injected // CapabilitiesGitConfig returns a slice of gitcmd.ConfigPairs that can be injected
// into the Git config to make it aware the bundle-URI capabilities are // into the Git config to enable or disable bundle-uri capability.
// supported. //
// This can be used when spawning git-upload-pack(1) --advertise-refs in // If the feature flag is OFF, this config disables bundle-uri regardless of the `enable` parameter.
//
// Explicitly disabling bundle-uri capability is used during calls to git-upload-pack(1)
// when no bundle exists for the given repository. This prevents the client
// from requesting a non-existing URI.
//
// This function is also used when spawning git-upload-pack(1) --advertise-refs in
// response to the GET /info/refs request. // response to the GET /info/refs request.
func CapabilitiesGitConfig(ctx context.Context) []gitcmd.ConfigPair { func CapabilitiesGitConfig(ctx context.Context, enable bool) []gitcmd.ConfigPair {
if featureflag.BundleURI.IsDisabled(ctx) { cfg := gitcmd.ConfigPair{
return []gitcmd.ConfigPair{} Key: "uploadpack.advertiseBundleURIs",
Value: strconv.FormatBool(enable),
} }
return []gitcmd.ConfigPair{ if featureflag.BundleURI.IsDisabled(ctx) {
{ cfg.Value = strconv.FormatBool(false)
Key: "uploadpack.advertiseBundleURIs",
Value: "true",
},
} }
return []gitcmd.ConfigPair{cfg}
} }
// UploadPackGitConfig return a slice of gitcmd.ConfigPairs you can inject into the // UploadPackGitConfig return a slice of gitcmd.ConfigPairs you can inject into the
// call to git-upload-pack(1) to advertise the available bundle to the client // call to git-upload-pack(1) to advertise the available bundle to the client
// who clones/fetches from the repository. // who clones/fetches from the repository.
func UploadPackGitConfig( func UploadPackGitConfig(ctx context.Context, signedURL string) []gitcmd.ConfigPair {
ctx context.Context, if featureflag.BundleURI.IsDisabled(ctx) || signedURL == "" {
manager *GenerationManager, return CapabilitiesGitConfig(ctx, false)
repo storage.Repository,
) ([]gitcmd.ConfigPair, error) {
if featureflag.BundleURI.IsDisabled(ctx) {
return []gitcmd.ConfigPair{}, nil
}
if manager == nil || manager.sink == nil {
return CapabilitiesGitConfig(ctx), errors.New("bundle-URI sink missing")
} }
uri, err := manager.SignedURL(ctx, repo) config := []gitcmd.ConfigPair{
if err != nil {
return nil, err
}
log.AddFields(ctx, log.Fields{"bundle_uri": true})
return []gitcmd.ConfigPair{
{
Key: "uploadpack.advertiseBundleURIs",
Value: "true",
},
{ {
Key: "bundle.version", Key: "bundle.version",
Value: "1", Value: "1",
...@@ -71,7 +55,7 @@ func UploadPackGitConfig( ...@@ -71,7 +55,7 @@ func UploadPackGitConfig(
}, },
{ {
Key: fmt.Sprintf("bundle.%s.uri", defaultBundle), Key: fmt.Sprintf("bundle.%s.uri", defaultBundle),
Value: uri, Value: signedURL,
}, },
{ {
// Gitaly uses only one bundle URI bundle, and it's a // Gitaly uses only one bundle URI bundle, and it's a
...@@ -91,5 +75,6 @@ func UploadPackGitConfig( ...@@ -91,5 +75,6 @@ func UploadPackGitConfig(
Key: fmt.Sprintf("bundle.%s.creationToken", defaultBundle), Key: fmt.Sprintf("bundle.%s.creationToken", defaultBundle),
Value: "1", Value: "1",
}, },
}, nil }
return append(config, CapabilitiesGitConfig(ctx, true)...)
} }
...@@ -2,118 +2,89 @@ package bundleuri ...@@ -2,118 +2,89 @@ package bundleuri
import ( import (
"context" "context"
"fmt"
"os"
"path/filepath"
"strings"
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/gitcmd" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gitcmd"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo"
"gitlab.com/gitlab-org/gitaly/v16/internal/structerr"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg"
) )
func TestUploadPackGitConfig(t *testing.T) { func TestCapabilitiesGitConfig(t *testing.T) {
testhelper.NewFeatureSets(featureflag.BundleURI).
Run(t, testUploadPackGitConfig)
}
func testUploadPackGitConfig(t *testing.T, ctx context.Context) {
t.Parallel() t.Parallel()
testhelper.NewFeatureSets(
featureflag.BundleURI,
).Run(t, testCapabilitiesGitConfig)
}
cfg := testcfg.Build(t) func testCapabilitiesGitConfig(t *testing.T, ctx context.Context) {
repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ tests := []struct {
SkipCreationViaService: true, desc string
}) enabled bool
repo := localrepo.NewTestRepo(t, cfg, repoProto) expected []gitcmd.ConfigPair
gittest.WriteCommit(t, cfg, repoPath,
gittest.WithTreeEntries(gittest.TreeEntry{Mode: "100644", Path: "README", Content: "much"}),
gittest.WithBranch("main"))
tempDir := testhelper.TempDir(t)
keyFile, err := os.Create(filepath.Join(tempDir, "secret.key"))
require.NoError(t, err)
_, err = keyFile.WriteString("super-secret-key")
require.NoError(t, err)
require.NoError(t, keyFile.Close())
type setupData struct {
manager *GenerationManager
}
for _, tc := range []struct {
desc string
setup func(t *testing.T) setupData
expectedConfig []gitcmd.ConfigPair
expectedErr error
}{ }{
{ {
desc: "no bundle found", desc: "when bundle-uri enabled",
setup: func(t *testing.T) setupData { enabled: true,
sinkDir := t.TempDir() expected: []gitcmd.ConfigPair{
sink, err := NewSink(ctx, "file://"+sinkDir+"?base_url=http://example.com&secret_key_path="+keyFile.Name()) {
require.NoError(t, err) Key: "uploadpack.advertiseBundleURIs",
Value: "true",
manager, err := NewGenerationManager(sink, testhelper.NewLogger(t), 1, 0, nil) },
require.NoError(t, err)
return setupData{
manager: manager,
}
}, },
expectedConfig: nil,
expectedErr: structerr.NewNotFound("no bundle available"),
}, },
{ {
desc: "not signed", desc: "when bundle-uri disabled",
setup: func(t *testing.T) setupData { enabled: false,
sinkDir := t.TempDir() expected: []gitcmd.ConfigPair{
sink, err := NewSink(ctx, "file://"+sinkDir) {
require.NoError(t, err) Key: "uploadpack.advertiseBundleURIs",
Value: "false",
manager, err := NewGenerationManager(sink, testhelper.NewLogger(t), 1, 0, nil) },
require.NoError(t, err)
require.NoError(t, manager.Generate(ctx, repo))
return setupData{
manager: manager,
}
}, },
expectedConfig: nil,
expectedErr: fmt.Errorf("signed URL: fileblob.SignedURL: bucket does not have an Options.URLSigner (code=Unimplemented)"),
}, },
{ }
desc: "success", for _, tt := range tests {
setup: func(t *testing.T) setupData { got := CapabilitiesGitConfig(ctx, tt.enabled)
sinkDir := t.TempDir() if featureflag.BundleURI.IsEnabled(ctx) {
sink, err := NewSink(ctx, "file://"+sinkDir+"?base_url=http://example.com&secret_key_path="+keyFile.Name()) require.ElementsMatch(t, tt.expected, got)
require.NoError(t, err) } else {
require.ElementsMatch(t, []gitcmd.ConfigPair{
manager, err := NewGenerationManager(sink, testhelper.NewLogger(t), 1, 0, nil)
require.NoError(t, err)
require.NoError(t, manager.Generate(ctx, repo))
return setupData{
manager: manager,
}
},
expectedConfig: []gitcmd.ConfigPair{
{ {
Key: "uploadpack.advertiseBundleURIs", Key: "uploadpack.advertiseBundleURIs",
Value: "true", Value: "false",
}, },
}, got)
}
}
}
func TestUploadPackGitConfig(t *testing.T) {
t.Parallel()
testhelper.NewFeatureSets(
featureflag.BundleURI,
).Run(t, testUploadPackGitConfig)
}
func testUploadPackGitConfig(t *testing.T, ctx context.Context) {
tests := []struct {
desc string
uri string
expected []gitcmd.ConfigPair
}{
{
desc: "should return all config",
uri: "https://example.com/bundle.git",
expected: []gitcmd.ConfigPair{
{ {
Key: "bundle.version", Key: "bundle.version",
Value: "1", Value: "1",
}, },
{
Key: "uploadpack.advertiseBundleURIs",
Value: "true",
},
{ {
Key: "bundle.mode", Key: "bundle.mode",
Value: "all", Value: "all",
...@@ -124,7 +95,7 @@ func testUploadPackGitConfig(t *testing.T, ctx context.Context) { ...@@ -124,7 +95,7 @@ func testUploadPackGitConfig(t *testing.T, ctx context.Context) {
}, },
{ {
Key: "bundle.default.uri", Key: "bundle.default.uri",
Value: "https://example.com/bundle.git?signed=ok", Value: "https://example.com/bundle.git",
}, },
{ {
Key: "bundle.default.creationToken", Key: "bundle.default.creationToken",
...@@ -132,35 +103,28 @@ func testUploadPackGitConfig(t *testing.T, ctx context.Context) { ...@@ -132,35 +103,28 @@ func testUploadPackGitConfig(t *testing.T, ctx context.Context) {
}, },
}, },
}, },
} { {
t.Run(tc.desc, func(t *testing.T) { desc: "empty uri should disable advertising bundle",
t.Parallel() uri: "",
expected: []gitcmd.ConfigPair{
data := tc.setup(t) {
require.NoError(t, err) Key: "uploadpack.advertiseBundleURIs",
Value: "false",
actual, err := UploadPackGitConfig(ctx, data.manager, repoProto) },
},
if featureflag.BundleURI.IsEnabled(ctx) { },
require.Equal(t, tc.expectedErr, err) }
for _, tt := range tests {
if tc.expectedConfig != nil { got := UploadPackGitConfig(ctx, tt.uri)
require.Equal(t, len(tc.expectedConfig), len(actual)) if featureflag.BundleURI.IsEnabled(ctx) {
require.ElementsMatch(t, tt.expected, got)
for i, c := range tc.expectedConfig { } else {
if strings.HasSuffix(c.Key, ".uri") { require.ElementsMatch(t, []gitcmd.ConfigPair{
// We cannot predict the exact signed URL Value, {
// so only check the Keys. Key: "uploadpack.advertiseBundleURIs",
require.Equal(t, c.Key, actual[i].Key) Value: "false",
} else { },
require.Equal(t, c, actual[i]) }, got)
} }
}
}
} else {
require.NoError(t, err)
require.Empty(t, actual)
}
})
} }
} }
...@@ -13,11 +13,14 @@ import ( ...@@ -13,11 +13,14 @@ import (
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"gitlab.com/gitlab-org/gitaly/v16/internal/backup" "gitlab.com/gitlab-org/gitaly/v16/internal/backup"
"gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/gitcmd"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage"
"gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/log"
"gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr"
"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
) )
var bundleGenerationLatency = prometheus.NewHistogram( var bundleGenerationLatency = prometheus.NewHistogram(
...@@ -210,6 +213,20 @@ func (g *GenerationManager) SignedURL(ctx context.Context, repo storage.Reposito ...@@ -210,6 +213,20 @@ func (g *GenerationManager) SignedURL(ctx context.Context, repo storage.Reposito
return g.sink.signedURL(ctx, relativePath) return g.sink.signedURL(ctx, relativePath)
} }
// UploadPackGitConfig is a helper function to provide all required, and computed, configurations
// to inject into the `git-upload-pack` command in order to advertise `bundle-uri` and provide
// the URI for the bundle for the given repository.
func (g *GenerationManager) UploadPackGitConfig(ctx context.Context, repo storage.Repository) []gitcmd.ConfigPair {
uri, err := g.SignedURL(ctx, repo)
if err != nil {
if st, ok := status.FromError(err); !ok || st.Code() != codes.NotFound {
g.logger.WithField("bundle_uri_error", err)
}
return CapabilitiesGitConfig(ctx, false)
}
return UploadPackGitConfig(ctx, uri)
}
// bundleRelativePath returns a relative path of the bundle-URI bundle inside the bucket. // bundleRelativePath returns a relative path of the bundle-URI bundle inside the bucket.
func bundleRelativePath(repo storage.Repository, name string) string { func bundleRelativePath(repo storage.Repository, name string) string {
repoPath := filepath.Join(repo.GetStorageName(), repo.GetRelativePath()) repoPath := filepath.Join(repo.GetStorageName(), repo.GetRelativePath())
......
...@@ -37,6 +37,7 @@ import ( ...@@ -37,6 +37,7 @@ import (
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue/databasemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue/databasemgr"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/mdfile" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/mdfile"
nodeimpl "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/node" nodeimpl "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/node"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/partition" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/partition"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/partition/migration" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/partition/migration"
...@@ -539,9 +540,15 @@ func run(appCtx *cli.Context, cfg config.Cfg, logger log.Logger) error { ...@@ -539,9 +540,15 @@ func run(appCtx *cli.Context, cfg config.Cfg, logger log.Logger) error {
return fmt.Errorf("resolve backup locator: %w", err) return fmt.Errorf("resolve backup locator: %w", err)
} }
} }
raftTransport := &raftmgr.GrpcTransport{}
if cfg.Raft.Enabled {
raftManagerRegistry := raftmgr.NewRaftManagerRegistry()
routingTable := raftmgr.NewStaticRaftRoutingTable()
raftTransport = raftmgr.NewGrpcTransport(logger, cfg, routingTable, raftManagerRegistry, conns)
}
var bundleURISink *bundleuri.Sink var bundleURISink *bundleuri.Sink
var bundleManager *bundleuri.GenerationManager var bundleURIManager *bundleuri.GenerationManager
if cfg.BundleURI.GoCloudURL != "" { if cfg.BundleURI.GoCloudURL != "" {
bundleURISink, err = bundleuri.NewSink(ctx, cfg.BundleURI.GoCloudURL) bundleURISink, err = bundleuri.NewSink(ctx, cfg.BundleURI.GoCloudURL)
if err != nil { if err != nil {
...@@ -554,7 +561,7 @@ func run(appCtx *cli.Context, cfg config.Cfg, logger log.Logger) error { ...@@ -554,7 +561,7 @@ func run(appCtx *cli.Context, cfg config.Cfg, logger log.Logger) error {
// Further tests and analysis would be required to come up with the // Further tests and analysis would be required to come up with the
// appropriate configuration. This will be done once we are ready to use this manager // appropriate configuration. This will be done once we are ready to use this manager
// to generate bundles based on this configuration (ie: calling GenerateIfAboveThreshold(...)) // to generate bundles based on this configuration (ie: calling GenerateIfAboveThreshold(...))
bundleManager, err = bundleuri.NewGenerationManager(bundleURISink, logger, 3, 1, bundleuri.NewInProgressTracker()) bundleURIManager, err = bundleuri.NewGenerationManager(bundleURISink, logger, 3, 1, bundleuri.NewInProgressTracker())
if err != nil { if err != nil {
return fmt.Errorf("error creating bundle manager: %w", err) return fmt.Errorf("error creating bundle manager: %w", err)
} }
...@@ -584,26 +591,27 @@ func run(appCtx *cli.Context, cfg config.Cfg, logger log.Logger) error { ...@@ -584,26 +591,27 @@ func run(appCtx *cli.Context, cfg config.Cfg, logger log.Logger) error {
} }
setup.RegisterAll(srv, &service.Dependencies{ setup.RegisterAll(srv, &service.Dependencies{
Logger: logger, Logger: logger,
Cfg: cfg, Cfg: cfg,
GitalyHookManager: hookManager, GitalyHookManager: hookManager,
TransactionManager: transactionManager, RaftGrpcTransport: raftTransport,
StorageLocator: locator, TransactionManager: transactionManager,
ClientPool: conns, StorageLocator: locator,
GitCmdFactory: gitCmdFactory, ClientPool: conns,
CatfileCache: catfileCache, GitCmdFactory: gitCmdFactory,
DiskCache: diskCache, CatfileCache: catfileCache,
PackObjectsCache: streamCache, DiskCache: diskCache,
PackObjectsLimiter: packObjectsLimiter, PackObjectsCache: streamCache,
RepositoryCounter: repoCounter, PackObjectsLimiter: packObjectsLimiter,
UpdaterWithHooks: updaterWithHooks, RepositoryCounter: repoCounter,
Node: node, UpdaterWithHooks: updaterWithHooks,
TransactionRegistry: txRegistry, Node: node,
HousekeepingManager: housekeepingManager, TransactionRegistry: txRegistry,
BackupSink: backupSink, HousekeepingManager: housekeepingManager,
BackupLocator: backupLocator, BackupSink: backupSink,
LocalRepositoryFactory: localrepoFactory, BackupLocator: backupLocator,
BundleGenerationManager: bundleManager, LocalRepositoryFactory: localrepoFactory,
BundleURIManager: bundleURIManager,
}) })
b.RegisterStarter(starter.New(c, srv, logger)) b.RegisterStarter(starter.New(c, srv, logger))
} }
......
...@@ -255,17 +255,35 @@ func recoveryReplayAction(ctx *cli.Context) (returnErr error) { ...@@ -255,17 +255,35 @@ func recoveryReplayAction(ctx *cli.Context) (returnErr error) {
} }
func processLogEntry(reader io.Reader, tempDir string, logWriter storage.LogWriter, lsn storage.LSN) (returnErr error) { func processLogEntry(reader io.Reader, tempDir string, logWriter storage.LogWriter, lsn storage.LSN) (returnErr error) {
path := filepath.Join(tempDir, lsn.String()) stagingDir, err := os.MkdirTemp(tempDir, "staging-*")
if err != nil {
return fmt.Errorf("create staging dir: %w", err)
}
defer func() {
if err := os.RemoveAll(stagingDir); err != nil {
returnErr = errors.Join(returnErr, fmt.Errorf("removing temp staging dir: %w", err))
}
}()
if err := downloadArchive(reader, path); err != nil { if err := downloadArchive(reader, stagingDir); err != nil {
return fmt.Errorf("download archive: %w", err) return fmt.Errorf("download archive: %w", err)
} }
if err := extractArchive(path); err != nil { if err := extractArchive(stagingDir); err != nil {
return fmt.Errorf("extract archive: %w", err) return fmt.Errorf("extract archive: %w", err)
} }
if _, err := logWriter.CompareAndAppendLogEntry(lsn, path); err != nil { // Validate that WAL entry was extracted correctly to its own directory
entryPath := filepath.Join(stagingDir, lsn.String())
info, err := os.Stat(entryPath)
if err != nil {
return fmt.Errorf("WAL entry not found after archive extraction: %w", err)
}
if !info.IsDir() {
return fmt.Errorf("expected WAL entry path %s to be a directory", entryPath)
}
if _, err := logWriter.CompareAndAppendLogEntry(lsn, entryPath); err != nil {
return fmt.Errorf("append log entry: %w", err) return fmt.Errorf("append log entry: %w", err)
} }
logWriter.NotifyNewEntries() logWriter.NotifyNewEntries()
......
...@@ -480,9 +480,9 @@ Successfully processed log entries up to LSN %s ...@@ -480,9 +480,9 @@ Successfully processed log entries up to LSN %s
partitionPath := filepath.Join(repo.GetStorageName(), fmt.Sprintf("%d", storage.PartitionID(2))) partitionPath := filepath.Join(repo.GetStorageName(), fmt.Sprintf("%d", storage.PartitionID(2)))
testhelper.WriteFiles(t, opts.backupRoot, map[string]any{ testhelper.WriteFiles(t, opts.backupRoot, map[string]any{
filepath.Join(partitionPath, storage.LSN(1).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath()), filepath.Join(partitionPath, storage.LSN(1).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath(), storage.LSN(1)),
filepath.Join(partitionPath, storage.LSN(2).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath()), filepath.Join(partitionPath, storage.LSN(2).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath(), storage.LSN(2)),
filepath.Join(partitionPath, storage.LSN(3).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath()), filepath.Join(partitionPath, storage.LSN(3).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath(), storage.LSN(3)),
}) })
return setupData{ return setupData{
...@@ -535,9 +535,9 @@ Successfully processed log entries up to LSN %s ...@@ -535,9 +535,9 @@ Successfully processed log entries up to LSN %s
partitionPath := filepath.Join(repo.GetStorageName(), fmt.Sprintf("%d", storage.PartitionID(2))) partitionPath := filepath.Join(repo.GetStorageName(), fmt.Sprintf("%d", storage.PartitionID(2)))
testhelper.WriteFiles(t, opts.backupRoot, map[string]any{ testhelper.WriteFiles(t, opts.backupRoot, map[string]any{
filepath.Join(partitionPath, storage.LSN(1).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath()), filepath.Join(partitionPath, storage.LSN(1).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath(), storage.LSN(1)),
filepath.Join(partitionPath, storage.LSN(2).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath()), filepath.Join(partitionPath, storage.LSN(2).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath(), storage.LSN(2)),
filepath.Join(partitionPath, storage.LSN(4).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath()), filepath.Join(partitionPath, storage.LSN(4).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath(), storage.LSN(4)),
}) })
return setupData{ return setupData{
...@@ -592,9 +592,9 @@ Starting archived log entries import ...@@ -592,9 +592,9 @@ Starting archived log entries import
partitionPath := filepath.Join(repo.GetStorageName(), fmt.Sprintf("%d", storage.PartitionID(2))) partitionPath := filepath.Join(repo.GetStorageName(), fmt.Sprintf("%d", storage.PartitionID(2)))
testhelper.WriteFiles(t, opts.backupRoot, map[string]any{ testhelper.WriteFiles(t, opts.backupRoot, map[string]any{
filepath.Join(partitionPath, storage.LSN(1).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath()), filepath.Join(partitionPath, storage.LSN(1).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath(), storage.LSN(1)),
filepath.Join(partitionPath, storage.LSN(2).String()+".tar"): createInvalidLogEntryArchive(t, repo.GetRelativePath()), filepath.Join(partitionPath, storage.LSN(2).String()+".tar"): createInvalidLogEntryArchive(t, repo.GetRelativePath(), storage.LSN(2)),
filepath.Join(partitionPath, storage.LSN(3).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath()), filepath.Join(partitionPath, storage.LSN(3).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath(), storage.LSN(3)),
}) })
return setupData{ return setupData{
...@@ -715,12 +715,20 @@ Starting archived log entries import ...@@ -715,12 +715,20 @@ Starting archived log entries import
} }
} }
func createValidLogEntryArchive(t *testing.T, repoRelativePath string) []byte { func createValidLogEntryArchive(t *testing.T, repoRelativePath string, lsn storage.LSN) []byte {
t.Helper() t.Helper()
var buf bytes.Buffer var buf bytes.Buffer
tw := tar.NewWriter(&buf) tw := tar.NewWriter(&buf)
// First create the directory entry
err := tw.WriteHeader(&tar.Header{
Name: lsn.String() + "/", // Add trailing slash for directory
Mode: 0o755,
Typeflag: tar.TypeDir,
})
require.NoError(t, err)
// Create a dummy MANIFEST file // Create a dummy MANIFEST file
manifest := &gitalypb.LogEntry{ manifest := &gitalypb.LogEntry{
RelativePath: repoRelativePath, RelativePath: repoRelativePath,
...@@ -730,7 +738,7 @@ func createValidLogEntryArchive(t *testing.T, repoRelativePath string) []byte { ...@@ -730,7 +738,7 @@ func createValidLogEntryArchive(t *testing.T, repoRelativePath string) []byte {
require.NoError(t, err) require.NoError(t, err)
err = tw.WriteHeader(&tar.Header{ err = tw.WriteHeader(&tar.Header{
Name: "MANIFEST", Name: lsn.String() + "/MANIFEST",
Mode: 0o644, Mode: 0o644,
Size: int64(len(manifestBytes)), Size: int64(len(manifestBytes)),
}) })
...@@ -743,12 +751,20 @@ func createValidLogEntryArchive(t *testing.T, repoRelativePath string) []byte { ...@@ -743,12 +751,20 @@ func createValidLogEntryArchive(t *testing.T, repoRelativePath string) []byte {
return buf.Bytes() return buf.Bytes()
} }
func createInvalidLogEntryArchive(t *testing.T, repoRelativePath string) []byte { func createInvalidLogEntryArchive(t *testing.T, repoRelativePath string, lsn storage.LSN) []byte {
t.Helper() t.Helper()
var buf bytes.Buffer var buf bytes.Buffer
tw := tar.NewWriter(&buf) tw := tar.NewWriter(&buf)
// First create the directory entry
err := tw.WriteHeader(&tar.Header{
Name: lsn.String() + "/", // Add trailing slash for directory
Mode: 0o755,
Typeflag: tar.TypeDir,
})
require.NoError(t, err)
// Create a dummy MANIFEST file // Create a dummy MANIFEST file
manifest := &gitalypb.LogEntry{ manifest := &gitalypb.LogEntry{
RelativePath: repoRelativePath, RelativePath: repoRelativePath,
...@@ -767,7 +783,7 @@ func createInvalidLogEntryArchive(t *testing.T, repoRelativePath string) []byte ...@@ -767,7 +783,7 @@ func createInvalidLogEntryArchive(t *testing.T, repoRelativePath string) []byte
require.NoError(t, err) require.NoError(t, err)
err = tw.WriteHeader(&tar.Header{ err = tw.WriteHeader(&tar.Header{
Name: "MANIFEST", Name: lsn.String() + "/MANIFEST",
Mode: 0o644, Mode: 0o644,
Size: int64(len(manifestBytes)), Size: int64(len(manifestBytes)),
}) })
......
...@@ -5,5 +5,5 @@ var VerbatimPatchID = NewFeatureFlag( ...@@ -5,5 +5,5 @@ var VerbatimPatchID = NewFeatureFlag(
"verbatim_patch_id", "verbatim_patch_id",
"v17.7.0", "v17.7.0",
"https://gitlab.com/gitlab-org/gitaly/-/issues/6537", "https://gitlab.com/gitlab-org/gitaly/-/issues/6537",
false, true,
) )
...@@ -16,7 +16,7 @@ import ( ...@@ -16,7 +16,7 @@ import (
// DefaultReferenceBackend is the default reftable backend used for running tests. // DefaultReferenceBackend is the default reftable backend used for running tests.
var DefaultReferenceBackend = func() git.ReferenceBackend { var DefaultReferenceBackend = func() git.ReferenceBackend {
if val, enabled := os.LookupEnv("GIT_DEFAULT_REF_FORMAT"); enabled && val == "reftable" { if val, enabled := os.LookupEnv("GITALY_TEST_REF_FORMAT"); enabled && val == "reftable" {
return git.ReferenceBackendReftables return git.ReferenceBackendReftables
} }
......
...@@ -106,6 +106,13 @@ func (repo *Repo) CloneBundle(ctx context.Context, reader io.Reader) error { ...@@ -106,6 +106,13 @@ func (repo *Repo) CloneBundle(ctx context.Context, reader io.Reader) error {
}, },
gitcmd.WithStderr(&cloneErr), gitcmd.WithStderr(&cloneErr),
gitcmd.WithDisabledHooks(), gitcmd.WithDisabledHooks(),
// Starting in Git version 2.46.0, executing git-fetch(1) on a bundle performs fsck
// checks when `transfer.fsckObjects` is enabled. Prior to this, this configuration was
// always ignored and fsck checks were not run. Unfortunately, fsck message severity
// configuration is ignored by Git only for bundle fetches. Until this is supported by
// Git, disable `transfer.fsckObjects` so bundles containing fsck errors can continue to
// be fetched. This matches behavior prior to Git version 2.46.0.
gitcmd.WithConfig(gitcmd.ConfigPair{Key: "transfer.fsckObjects", Value: "false"}),
) )
if err != nil { if err != nil {
return fmt.Errorf("spawning git-clone: %w", err) return fmt.Errorf("spawning git-clone: %w", err)
......
...@@ -137,6 +137,7 @@ func TestRepo_CloneBundle(t *testing.T) { ...@@ -137,6 +137,7 @@ func TestRepo_CloneBundle(t *testing.T) {
reader io.Reader reader io.Reader
expectedHeadRef git.ReferenceName expectedHeadRef git.ReferenceName
expectedRefs []git.Reference expectedRefs []git.Reference
skipFsck bool
expectedErr error expectedErr error
} }
...@@ -189,6 +190,35 @@ func TestRepo_CloneBundle(t *testing.T) { ...@@ -189,6 +190,35 @@ func TestRepo_CloneBundle(t *testing.T) {
} }
}, },
}, },
{
desc: "bundle contains fsck error",
setup: func(t *testing.T, ctx context.Context, cfg config.Cfg) setupData {
_, sourceRepoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
SkipCreationViaService: true,
})
// Write a tree object that has the same path twice. When git-clone(1) executes with
// `transfer.fsckObjects=true`, it is expected that this objects causes the fetch to fail.
//
// The git-clone(1) performed when fetching bundles is configured to override
// `transfer.fsckObjects` to `false`. Therefore, fsck errors are ignored.
treeID := gittest.WriteTree(t, cfg, sourceRepoPath, []gittest.TreeEntry{
{Path: "duplicate", Mode: "100644", Content: "foo"},
{Path: "duplicate", Mode: "100644", Content: "bar"},
})
mainOid := gittest.WriteCommit(t, cfg, sourceRepoPath, gittest.WithBranch(git.DefaultBranch), gittest.WithTree(treeID))
return setupData{
reader: createBundle(t, cfg, sourceRepoPath),
expectedHeadRef: git.DefaultRef,
expectedRefs: []git.Reference{
git.NewReference(git.DefaultRef, mainOid),
},
skipFsck: true,
}
},
},
} { } {
t.Run(tc.desc, func(t *testing.T) { t.Run(tc.desc, func(t *testing.T) {
t.Parallel() t.Parallel()
...@@ -217,8 +247,10 @@ func TestRepo_CloneBundle(t *testing.T) { ...@@ -217,8 +247,10 @@ func TestRepo_CloneBundle(t *testing.T) {
repoPath, err := repo.Path(ctx) repoPath, err := repo.Path(ctx)
require.NoError(t, err) require.NoError(t, err)
// Verify connectivity and validity of the repository objects. if !data.skipFsck {
gittest.Exec(t, cfg, "-C", repoPath, "fsck") // Verify connectivity and validity of the repository objects.
gittest.Exec(t, cfg, "-C", repoPath, "fsck")
}
// Verify HEAD has been set correctly. // Verify HEAD has been set correctly.
headRef, err := repo.HeadReference(ctx) headRef, err := repo.HeadReference(ctx)
......
...@@ -13,6 +13,7 @@ import ( ...@@ -13,6 +13,7 @@ import (
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/hook/updateref" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/hook/updateref"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab"
...@@ -26,30 +27,31 @@ import ( ...@@ -26,30 +27,31 @@ import (
// Dependencies assembles set of components required by different kinds of services. // Dependencies assembles set of components required by different kinds of services.
type Dependencies struct { type Dependencies struct {
Logger log.Logger Logger log.Logger
Cfg config.Cfg Cfg config.Cfg
GitalyHookManager gitalyhook.Manager GitalyHookManager gitalyhook.Manager
TransactionManager transaction.Manager RaftGrpcTransport raftmgr.Transport
StorageLocator storage.Locator TransactionManager transaction.Manager
ClientPool *client.Pool StorageLocator storage.Locator
GitCmdFactory gitcmd.CommandFactory ClientPool *client.Pool
BackchannelRegistry *backchannel.Registry GitCmdFactory gitcmd.CommandFactory
GitlabClient gitlab.Client BackchannelRegistry *backchannel.Registry
CatfileCache catfile.Cache GitlabClient gitlab.Client
DiskCache cache.Cache CatfileCache catfile.Cache
PackObjectsCache streamcache.Cache DiskCache cache.Cache
PackObjectsLimiter limiter.Limiter PackObjectsCache streamcache.Cache
LimitHandler *limithandler.LimiterMiddleware PackObjectsLimiter limiter.Limiter
RepositoryCounter *counter.RepositoryCounter LimitHandler *limithandler.LimiterMiddleware
UpdaterWithHooks *updateref.UpdaterWithHooks RepositoryCounter *counter.RepositoryCounter
HousekeepingManager housekeepingmgr.Manager UpdaterWithHooks *updateref.UpdaterWithHooks
TransactionRegistry *storagemgr.TransactionRegistry HousekeepingManager housekeepingmgr.Manager
Node storage.Node TransactionRegistry *storagemgr.TransactionRegistry
BackupSink *backup.Sink Node storage.Node
BackupLocator backup.Locator BackupSink *backup.Sink
ProcReceiveRegistry *gitalyhook.ProcReceiveRegistry BackupLocator backup.Locator
BundleGenerationManager *bundleuri.GenerationManager ProcReceiveRegistry *gitalyhook.ProcReceiveRegistry
LocalRepositoryFactory localrepo.Factory BundleURIManager *bundleuri.GenerationManager
LocalRepositoryFactory localrepo.Factory
} }
// GetLogger returns the logger. // GetLogger returns the logger.
...@@ -72,6 +74,11 @@ func (dc *Dependencies) GetTxManager() transaction.Manager { ...@@ -72,6 +74,11 @@ func (dc *Dependencies) GetTxManager() transaction.Manager {
return dc.TransactionManager return dc.TransactionManager
} }
// GetRaftGrpcTransport returns raft transport.
func (dc *Dependencies) GetRaftGrpcTransport() raftmgr.Transport {
return dc.RaftGrpcTransport
}
// GetLocator returns storage locator. // GetLocator returns storage locator.
func (dc *Dependencies) GetLocator() storage.Locator { func (dc *Dependencies) GetLocator() storage.Locator {
return dc.StorageLocator return dc.StorageLocator
...@@ -167,7 +174,7 @@ func (dc *Dependencies) GetRepositoryFactory() localrepo.Factory { ...@@ -167,7 +174,7 @@ func (dc *Dependencies) GetRepositoryFactory() localrepo.Factory {
return dc.LocalRepositoryFactory return dc.LocalRepositoryFactory
} }
// GetBundleManager returns the RepositoryFactory // GetBundleURIManager returns the RepositoryFactory
func (dc *Dependencies) GetBundleManager() *bundleuri.GenerationManager { func (dc *Dependencies) GetBundleURIManager() *bundleuri.GenerationManager {
return dc.BundleGenerationManager return dc.BundleURIManager
} }
package raft
import (
"errors"
"io"
"gitlab.com/gitlab-org/gitaly/v16/internal/structerr"
"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
)
// SendMessage is a gRPC method for sending a Raft message across nodes.
func (s *Server) SendMessage(stream gitalypb.RaftService_SendMessageServer) error {
for {
req, err := stream.Recv()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return structerr.NewInternal("receive error: %w", err)
}
// The cluster ID protects Gitaly from cross-cluster interactions, which could potentially corrupt the clusters.
// This is particularly crucial after disaster recovery so that an identical cluster is restored from backup.
if req.GetClusterId() == "" {
return structerr.NewInvalidArgument("cluster_id is required")
}
// Let's assume we have a single cluster per node for now.
if req.GetClusterId() != s.cfg.Raft.ClusterID {
return structerr.NewPermissionDenied("message from wrong cluster: got %q, want %q",
req.GetClusterId(), s.cfg.Raft.ClusterID)
}
if req.GetAuthorityName() == "" {
return structerr.NewInvalidArgument("authority_name is required")
}
if req.GetPartitionId() == 0 {
return structerr.NewInvalidArgument("partition_id is required")
}
raftMsg := req.GetMessage()
if err := s.transport.Receive(stream.Context(), req.GetAuthorityName(), req.GetPartitionId(), *raftMsg); err != nil {
return structerr.NewInternal("receive error: %w", err)
}
}
return stream.SendAndClose(&gitalypb.RaftMessageResponse{})
}
package raft
import (
"context"
"testing"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver"
"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
"go.etcd.io/etcd/raft/v3/raftpb"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
)
func TestServer_SendMessage(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
cfg := testcfg.Build(t)
cfg.Raft.ClusterID = "test-cluster"
client := runRaftServer(t, ctx, cfg)
testCases := []struct {
desc string
req *gitalypb.RaftMessageRequest
expectedGrpcErr codes.Code
expectedError string
}{
{
desc: "successful message send",
req: &gitalypb.RaftMessageRequest{
ClusterId: "test-cluster",
AuthorityName: "test-authority",
PartitionId: 1,
Message: &raftpb.Message{
Type: raftpb.MsgApp,
To: 2,
},
},
},
{
desc: "missing cluster ID",
req: &gitalypb.RaftMessageRequest{
AuthorityName: "test-authority",
PartitionId: 1,
Message: &raftpb.Message{
Type: raftpb.MsgApp,
To: 2,
},
},
expectedGrpcErr: codes.InvalidArgument,
expectedError: "rpc error: code = InvalidArgument desc = cluster_id is required",
},
{
desc: "wrong cluster ID",
req: &gitalypb.RaftMessageRequest{
ClusterId: "wrong-cluster",
AuthorityName: "test-authority",
PartitionId: 1,
Message: &raftpb.Message{
Type: raftpb.MsgApp,
To: 2,
},
},
expectedGrpcErr: codes.PermissionDenied,
expectedError: `rpc error: code = PermissionDenied desc = message from wrong cluster: got "wrong-cluster", want "test-cluster"`,
},
{
desc: "missing authority name",
req: &gitalypb.RaftMessageRequest{
ClusterId: "test-cluster",
PartitionId: 1,
Message: &raftpb.Message{
Type: raftpb.MsgApp,
To: 2,
},
},
expectedGrpcErr: codes.InvalidArgument,
expectedError: "rpc error: code = InvalidArgument desc = authority_name is required",
},
{
desc: "missing partition ID",
req: &gitalypb.RaftMessageRequest{
ClusterId: "test-cluster",
AuthorityName: "test-authority",
Message: &raftpb.Message{
Type: raftpb.MsgApp,
To: 2,
},
},
expectedGrpcErr: codes.InvalidArgument,
expectedError: "rpc error: code = InvalidArgument desc = partition_id is required",
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
stream, err := client.SendMessage(ctx)
require.NoError(t, err)
require.NoError(t, stream.Send(tc.req))
_, err = stream.CloseAndRecv()
if tc.expectedGrpcErr == codes.OK {
require.NoError(t, err)
} else {
testhelper.RequireGrpcCode(t, err, tc.expectedGrpcErr)
require.Contains(t, err.Error(), tc.expectedError)
}
})
}
}
func runRaftServer(t *testing.T, ctx context.Context, cfg config.Cfg) gitalypb.RaftServiceClient {
serverSocketPath := testserver.RunGitalyServer(t, cfg, func(srv *grpc.Server, deps *service.Dependencies) {
transport := newMockTransport(t)
deps.RaftGrpcTransport = transport
deps.Cfg = cfg
gitalypb.RegisterRaftServiceServer(srv, NewServer(deps))
}, testserver.WithDisablePraefect())
cfg.SocketPath = serverSocketPath
conn := gittest.DialService(t, ctx, cfg)
return gitalypb.NewRaftServiceClient(conn)
}
package raft
import (
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr"
"gitlab.com/gitlab-org/gitaly/v16/internal/log"
"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
)
// Server is a gRPC server for the Raft service.
type Server struct {
gitalypb.UnimplementedRaftServiceServer
logger log.Logger
transport raftmgr.Transport
cfg config.Cfg
}
// NewServer creates a new Raft gRPC server.
func NewServer(deps *service.Dependencies) *Server {
return &Server{
logger: deps.GetLogger(),
transport: deps.GetRaftGrpcTransport(),
cfg: deps.GetCfg(),
}
}