Skip to content
Snippets Groups Projects
Commit 130b09d3 authored by Patrick Steinhardt's avatar Patrick Steinhardt
Browse files

Revert "Merge branch 'pks-objectpool-drop-ondisk-remotes' into 'master'"

This reverts commit 369fbdb1 (Merge branch
'pks-objectpool-drop-ondisk-remotes' into 'master', 2021-10-25), which
introduced a regression in Rails' test suite.
parent 713bcd2c
No related branches found
No related tags found
Loading
Showing
with 1175 additions and 197 deletions
package localrepo
import (
"bufio"
"bytes"
"context"
"errors"
......@@ -13,6 +14,169 @@ import (
"gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb"
)
// Remote provides functionality of the 'remote' git sub-command.
type Remote struct {
repo *Repo
}
// Add adds a new remote to the repository.
func (remote Remote) Add(ctx context.Context, name, url string, opts git.RemoteAddOpts) error {
if err := validateNotBlank(name, "name"); err != nil {
return err
}
if err := validateNotBlank(url, "url"); err != nil {
return err
}
var stderr bytes.Buffer
if err := remote.repo.ExecAndWait(ctx,
git.SubSubCmd{
Name: "remote",
Action: "add",
Flags: buildRemoteAddOptsFlags(opts),
Args: []string{name, url},
},
git.WithStderr(&stderr),
git.WithRefTxHook(ctx, remote.repo, remote.repo.cfg),
); err != nil {
switch {
case isExitWithCode(err, 3):
// In Git v2.30.0 and newer (https://gitlab.com/git-vcs/git/commit/9144ba4cf52)
return git.ErrAlreadyExists
case isExitWithCode(err, 128) && bytes.HasPrefix(stderr.Bytes(), []byte("fatal: remote "+name+" already exists")):
// ..in older versions we parse stderr
return git.ErrAlreadyExists
}
return err
}
return nil
}
func buildRemoteAddOptsFlags(opts git.RemoteAddOpts) []git.Option {
var flags []git.Option
for _, b := range opts.RemoteTrackingBranches {
flags = append(flags, git.ValueFlag{Name: "-t", Value: b})
}
if opts.DefaultBranch != "" {
flags = append(flags, git.ValueFlag{Name: "-m", Value: opts.DefaultBranch})
}
if opts.Fetch {
flags = append(flags, git.Flag{Name: "-f"})
}
if opts.Tags != git.RemoteAddOptsTagsDefault {
flags = append(flags, git.Flag{Name: opts.Tags.String()})
}
if opts.Mirror != git.RemoteAddOptsMirrorDefault {
flags = append(flags, git.ValueFlag{Name: "--mirror", Value: opts.Mirror.String()})
}
return flags
}
// Remove removes a named remote from the repository configuration.
func (remote Remote) Remove(ctx context.Context, name string) error {
if err := validateNotBlank(name, "name"); err != nil {
return err
}
var stderr bytes.Buffer
if err := remote.repo.ExecAndWait(ctx,
git.SubSubCmd{
Name: "remote",
Action: "remove",
Args: []string{name},
},
git.WithStderr(&stderr),
git.WithRefTxHook(ctx, remote.repo, remote.repo.cfg),
); err != nil {
switch {
case isExitWithCode(err, 2):
// In Git v2.30.0 and newer (https://gitlab.com/git-vcs/git/commit/9144ba4cf52)
return git.ErrNotFound
case isExitWithCode(err, 128) && strings.HasPrefix(stderr.String(), "fatal: No such remote"):
// ..in older versions we parse stderr
return git.ErrNotFound
}
return err
}
return nil
}
// SetURL sets the URL for a given remote.
func (remote Remote) SetURL(ctx context.Context, name, url string, opts git.SetURLOpts) error {
if err := validateNotBlank(name, "name"); err != nil {
return err
}
if err := validateNotBlank(url, "url"); err != nil {
return err
}
var stderr bytes.Buffer
if err := remote.repo.ExecAndWait(ctx,
git.SubSubCmd{
Name: "remote",
Action: "set-url",
Flags: buildSetURLOptsFlags(opts),
Args: []string{name, url},
},
git.WithStderr(&stderr),
git.WithRefTxHook(ctx, remote.repo, remote.repo.cfg),
); err != nil {
switch {
case isExitWithCode(err, 2):
// In Git v2.30.0 and newer (https://gitlab.com/git-vcs/git/commit/9144ba4cf52)
return git.ErrNotFound
case isExitWithCode(err, 128) && strings.HasPrefix(stderr.String(), "fatal: No such remote"):
// ..in older versions we parse stderr
return git.ErrNotFound
}
return err
}
return nil
}
// Exists determines whether a given named remote exists.
func (remote Remote) Exists(ctx context.Context, name string) (bool, error) {
cmd, err := remote.repo.Exec(ctx,
git.SubCmd{Name: "remote"},
git.WithRefTxHook(ctx, remote.repo, remote.repo.cfg),
)
if err != nil {
return false, err
}
found := false
scanner := bufio.NewScanner(cmd)
for scanner.Scan() {
if scanner.Text() == name {
found = true
break
}
}
return found, cmd.Wait()
}
func buildSetURLOptsFlags(opts git.SetURLOpts) []git.Option {
if opts.Push {
return []git.Option{git.Flag{Name: "--push"}}
}
return nil
}
// FetchOptsTags controls what tags needs to be imported on fetch.
type FetchOptsTags string
......
......@@ -9,43 +9,319 @@ import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/v14/internal/git"
"gitlab.com/gitlab-org/gitaly/v14/internal/git/catfile"
"gitlab.com/gitlab-org/gitaly/v14/internal/git/gittest"
"gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/config"
"gitlab.com/gitlab-org/gitaly/v14/internal/helper/text"
"gitlab.com/gitlab-org/gitaly/v14/internal/testhelper"
"gitlab.com/gitlab-org/gitaly/v14/internal/testhelper/testcfg"
"gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb"
)
func TestRepo_FetchRemote(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
func setupRepoRemote(t *testing.T, bare bool) (Remote, string) {
t.Helper()
cfg := testcfg.Build(t)
cfg.Ruby.Dir = "/var/empty"
var repoProto *gitalypb.Repository
var repoPath string
if bare {
repoProto, repoPath = gittest.InitRepo(t, cfg, cfg.Storages[0])
} else {
repoProto, repoPath = gittest.CloneRepo(t, cfg, cfg.Storages[0])
}
gitCmdFactory := git.NewExecCommandFactory(cfg)
catfileCache := catfile.NewCache(cfg)
defer catfileCache.Stop()
t.Cleanup(catfileCache.Stop)
return New(gitCmdFactory, catfileCache, repoProto, cfg).Remote(), repoPath
}
func TestRepo_Remote(t *testing.T) {
repository := &gitalypb.Repository{StorageName: "stub", RelativePath: "/stub"}
repo := New(nil, nil, repository, config.Cfg{})
require.Equal(t, Remote{repo: repo}, repo.Remote())
}
func TestBuildRemoteAddOptsFlags(t *testing.T) {
for _, tc := range []struct {
desc string
opts git.RemoteAddOpts
exp []git.Option
}{
{
desc: "none",
exp: nil,
},
{
desc: "all set",
opts: git.RemoteAddOpts{
Tags: git.RemoteAddOptsTagsNone,
Fetch: true,
RemoteTrackingBranches: []string{"branch-1", "branch-2"},
DefaultBranch: "develop",
Mirror: git.RemoteAddOptsMirrorPush,
},
exp: []git.Option{
git.ValueFlag{Name: "-t", Value: "branch-1"},
git.ValueFlag{Name: "-t", Value: "branch-2"},
git.ValueFlag{Name: "-m", Value: "develop"},
git.Flag{Name: "-f"},
git.Flag{Name: "--no-tags"},
git.ValueFlag{Name: "--mirror", Value: "push"},
},
},
{
desc: "with tags",
opts: git.RemoteAddOpts{Tags: git.RemoteAddOptsTagsAll},
exp: []git.Option{git.Flag{Name: "--tags"}},
},
} {
t.Run(tc.desc, func(t *testing.T) {
require.Equal(t, tc.exp, buildRemoteAddOptsFlags(tc.opts))
})
}
}
func TestRemote_Add(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
remote, repoPath := setupRepoRemote(t, false)
gittest.Exec(t, remote.repo.cfg, "-C", repoPath, "remote", "remove", "origin")
_, remoteRepoPath := gittest.CloneRepo(t, remote.repo.cfg, remote.repo.cfg.Storages[0])
t.Run("invalid argument", func(t *testing.T) {
for _, tc := range []struct {
desc string
name, url string
errMsg string
}{
{
desc: "name",
name: " ",
url: "http://some.com.git",
errMsg: `"name" is blank or empty`,
},
{
desc: "url",
name: "name",
url: "",
errMsg: `"url" is blank or empty`,
},
} {
t.Run(tc.desc, func(t *testing.T) {
err := remote.Add(ctx, tc.name, tc.url, git.RemoteAddOpts{})
require.Error(t, err)
assert.True(t, errors.Is(err, git.ErrInvalidArg))
assert.Contains(t, err.Error(), tc.errMsg)
})
}
})
t.Run("fetch", func(t *testing.T) {
require.NoError(t, remote.Add(ctx, "first", remoteRepoPath, git.RemoteAddOpts{Fetch: true}))
remotes := text.ChompBytes(gittest.Exec(t, remote.repo.cfg, "-C", repoPath, "remote", "--verbose"))
require.Equal(t,
"first "+remoteRepoPath+" (fetch)\n"+
"first "+remoteRepoPath+" (push)",
remotes,
)
latestSHA := text.ChompBytes(gittest.Exec(t, remote.repo.cfg, "-C", repoPath, "rev-parse", "refs/remotes/first/master"))
require.Equal(t, "1e292f8fedd741b75372e19097c76d327140c312", latestSHA)
})
t.Run("default branch", func(t *testing.T) {
require.NoError(t, remote.Add(ctx, "second", "http://some.com.git", git.RemoteAddOpts{DefaultBranch: "wip"}))
defaultRemote := text.ChompBytes(gittest.Exec(t, remote.repo.cfg, "-C", repoPath, "symbolic-ref", "refs/remotes/second/HEAD"))
require.Equal(t, "refs/remotes/second/wip", defaultRemote)
})
t.Run("remote tracking branches", func(t *testing.T) {
require.NoError(t, remote.Add(ctx, "third", "http://some.com.git", git.RemoteAddOpts{RemoteTrackingBranches: []string{"a", "b"}}))
defaultRemote := text.ChompBytes(gittest.Exec(t, remote.repo.cfg, "-C", repoPath, "config", "--get-all", "remote.third.fetch"))
require.Equal(t, "+refs/heads/a:refs/remotes/third/a\n+refs/heads/b:refs/remotes/third/b", defaultRemote)
})
t.Run("already exists", func(t *testing.T) {
require.NoError(t, remote.Add(ctx, "fourth", "http://some.com.git", git.RemoteAddOpts{}))
err := remote.Add(ctx, "fourth", "http://some.com.git", git.RemoteAddOpts{})
require.Equal(t, git.ErrAlreadyExists, err)
})
}
func TestRemote_Remove(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
remote, repoPath := setupRepoRemote(t, true)
t.Run("ok", func(t *testing.T) {
gittest.Exec(t, remote.repo.cfg, "-C", repoPath, "remote", "add", "first", "http://some.com.git")
require.NoError(t, remote.Remove(ctx, "first"))
remotes := text.ChompBytes(gittest.Exec(t, remote.repo.cfg, "-C", repoPath, "remote", "--verbose"))
require.Empty(t, remotes)
})
t.Run("not found", func(t *testing.T) {
err := remote.Remove(ctx, "second")
require.Equal(t, git.ErrNotFound, err)
})
t.Run("invalid argument: name", func(t *testing.T) {
err := remote.Remove(ctx, " ")
require.Error(t, err)
assert.True(t, errors.Is(err, git.ErrInvalidArg))
assert.Contains(t, err.Error(), `"name" is blank or empty`)
})
t.Run("don't remove local branches", func(t *testing.T) {
remote, repoPath := setupRepoRemote(t, false)
// configure remote as fetch mirror
gittest.Exec(t, remote.repo.cfg, "-C", repoPath, "config", "remote.origin.fetch", "+refs/*:refs/*")
gittest.Exec(t, remote.repo.cfg, "-C", repoPath, "fetch")
masterBeforeRemove := gittest.Exec(t, remote.repo.cfg, "-C", repoPath, "show-ref", "refs/heads/master")
require.NoError(t, remote.Remove(ctx, "origin"))
out := gittest.Exec(t, remote.repo.cfg, "-C", repoPath, "remote")
require.Len(t, out, 0)
out = gittest.Exec(t, remote.repo.cfg, "-C", repoPath, "show-ref", "refs/heads/master")
require.Equal(t, masterBeforeRemove, out)
})
}
func TestRemote_Exists(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
remote, _ := setupRepoRemote(t, false)
found, err := remote.Exists(ctx, "origin")
require.NoError(t, err)
require.True(t, found)
found, err = remote.Exists(ctx, "can-not-be-found")
require.NoError(t, err)
require.False(t, found)
}
func TestBuildSetURLOptsFlags(t *testing.T) {
for _, tc := range []struct {
desc string
opts git.SetURLOpts
exp []git.Option
}{
{
desc: "none",
exp: nil,
},
{
desc: "all set",
opts: git.SetURLOpts{Push: true},
exp: []git.Option{git.Flag{Name: "--push"}},
},
} {
t.Run(tc.desc, func(t *testing.T) {
require.Equal(t, tc.exp, buildSetURLOptsFlags(tc.opts))
})
}
}
func TestRemote_SetURL(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
remote, repoPath := setupRepoRemote(t, true)
t.Run("invalid argument", func(t *testing.T) {
for _, tc := range []struct {
desc string
name, url string
errMsg string
}{
{
desc: "name",
name: " ",
url: "http://some.com.git",
errMsg: `"name" is blank or empty`,
},
{
desc: "url",
name: "name",
url: "",
errMsg: `"url" is blank or empty`,
},
} {
t.Run(tc.desc, func(t *testing.T) {
err := remote.SetURL(ctx, tc.name, tc.url, git.SetURLOpts{})
require.Error(t, err)
assert.True(t, errors.Is(err, git.ErrInvalidArg))
assert.Contains(t, err.Error(), tc.errMsg)
})
}
})
t.Run("ok", func(t *testing.T) {
gittest.Exec(t, remote.repo.cfg, "-C", repoPath, "remote", "add", "first", "file:/"+repoPath)
require.NoError(t, remote.SetURL(ctx, "first", "http://some.com.git", git.SetURLOpts{Push: true}))
remotes := text.ChompBytes(gittest.Exec(t, remote.repo.cfg, "-C", repoPath, "remote", "--verbose"))
require.Equal(t,
"first file:/"+repoPath+" (fetch)\n"+
"first http://some.com.git (push)",
remotes,
)
})
t.Run("doesnt exist", func(t *testing.T) {
err := remote.SetURL(ctx, "second", "http://some.com.git", git.SetURLOpts{})
require.True(t, errors.Is(err, git.ErrNotFound), err)
})
}
func TestRepo_FetchRemote(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
remoteCmd, remoteRepoPath := setupRepoRemote(t, false)
cfg := remoteCmd.repo.cfg
initBareWithRemote := func(t *testing.T, remote string) (*Repo, string) {
t.Helper()
_, remoteRepoPath := gittest.CloneRepo(t, cfg, cfg.Storages[0])
clientRepo, clientRepoPath := gittest.InitRepo(t, cfg, cfg.Storages[0])
testRepo, testRepoPath := gittest.InitRepo(t, cfg, cfg.Storages[0])
cmd := exec.Command(cfg.Git.BinPath, "-C", clientRepoPath, "remote", "add", remote, remoteRepoPath)
cmd := exec.Command(cfg.Git.BinPath, "-C", testRepoPath, "remote", "add", remote, remoteRepoPath)
err := cmd.Run()
if err != nil {
require.NoError(t, err)
}
return New(gitCmdFactory, catfileCache, clientRepo, cfg), clientRepoPath
return New(remoteCmd.repo.gitCmdFactory, remoteCmd.repo.catfileCache, testRepo, cfg), testRepoPath
}
t.Run("invalid name", func(t *testing.T) {
repo := New(gitCmdFactory, catfileCache, nil, cfg)
repo := New(remoteCmd.repo.gitCmdFactory, remoteCmd.repo.catfileCache, nil, cfg)
err := repo.FetchRemote(ctx, " ", FetchOpts{})
require.True(t, errors.Is(err, git.ErrInvalidArg))
......@@ -53,9 +329,7 @@ func TestRepo_FetchRemote(t *testing.T) {
})
t.Run("unknown remote", func(t *testing.T) {
repoProto, _ := gittest.InitRepo(t, cfg, cfg.Storages[0])
repo := New(gitCmdFactory, catfileCache, repoProto, cfg)
repo := New(remoteCmd.repo.gitCmdFactory, remoteCmd.repo.catfileCache, remoteCmd.repo, cfg)
var stderr bytes.Buffer
err := repo.FetchRemote(ctx, "stub", FetchOpts{Stderr: &stderr})
require.Error(t, err)
......@@ -85,7 +359,7 @@ func TestRepo_FetchRemote(t *testing.T) {
_, sourceRepoPath := gittest.CloneRepo(t, cfg, cfg.Storages[0])
testRepo, testRepoPath := gittest.CloneRepo(t, cfg, cfg.Storages[0])
repo := New(gitCmdFactory, catfileCache, testRepo, cfg)
repo := New(remoteCmd.repo.gitCmdFactory, remoteCmd.repo.catfileCache, testRepo, cfg)
gittest.Exec(t, cfg, "-C", testRepoPath, "remote", "add", "source", sourceRepoPath)
var stderr bytes.Buffer
......@@ -97,7 +371,7 @@ func TestRepo_FetchRemote(t *testing.T) {
_, sourceRepoPath := gittest.CloneRepo(t, cfg, cfg.Storages[0])
testRepo, testRepoPath := gittest.CloneRepo(t, cfg, cfg.Storages[0])
repo := New(gitCmdFactory, catfileCache, testRepo, cfg)
repo := New(remoteCmd.repo.gitCmdFactory, remoteCmd.repo.catfileCache, testRepo, cfg)
gittest.Exec(t, cfg, "-C", testRepoPath, "remote", "add", "source", sourceRepoPath)
var stderr bytes.Buffer
......@@ -113,7 +387,7 @@ func TestRepo_FetchRemote(t *testing.T) {
_, sourceRepoPath := gittest.CloneRepo(t, cfg, cfg.Storages[0])
testRepo, testRepoPath := gittest.CloneRepo(t, cfg, cfg.Storages[0])
repo := New(gitCmdFactory, catfileCache, testRepo, cfg)
repo := New(remoteCmd.repo.gitCmdFactory, remoteCmd.repo.catfileCache, testRepo, cfg)
gittest.Exec(t, cfg, "-C", testRepoPath, "remote", "add", "source", sourceRepoPath)
require.NoError(t, repo.FetchRemote(ctx, "source", FetchOpts{}))
......@@ -140,7 +414,7 @@ func TestRepo_FetchRemote(t *testing.T) {
_, sourceRepoPath := gittest.CloneRepo(t, cfg, cfg.Storages[0])
testRepo, testRepoPath := gittest.CloneRepo(t, cfg, cfg.Storages[0])
repo := New(gitCmdFactory, catfileCache, testRepo, cfg)
repo := New(remoteCmd.repo.gitCmdFactory, remoteCmd.repo.catfileCache, testRepo, cfg)
gittest.Exec(t, cfg, "-C", testRepoPath, "remote", "add", "source", sourceRepoPath)
require.NoError(t, repo.FetchRemote(ctx, "source", FetchOpts{}))
......
......@@ -64,6 +64,11 @@ func (repo *Repo) ExecAndWait(ctx context.Context, cmd git.Cmd, opts ...git.CmdO
return command.Wait()
}
// Remote returns executor of the 'remote' sub-command.
func (repo *Repo) Remote() Remote {
return Remote{repo: repo}
}
func errorWithStderr(err error, stderr []byte) error {
if len(stderr) == 0 {
return err
......
......@@ -22,7 +22,10 @@ import (
"gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb"
)
const sourceRefNamespace = "refs/remotes/origin"
const (
sourceRemote = "origin"
sourceRefNamespace = "refs/remotes/" + sourceRemote
)
// FetchFromOrigin initializes the pool and fetches the objects from its origin repository
func (o *ObjectPool) FetchFromOrigin(ctx context.Context, origin *gitalypb.Repository) error {
......@@ -39,6 +42,23 @@ func (o *ObjectPool) FetchFromOrigin(ctx context.Context, origin *gitalypb.Repos
return err
}
remote := o.poolRepo.Remote()
remoteExists, err := remote.Exists(ctx, sourceRemote)
if err != nil {
return err
}
if remoteExists {
if err := remote.SetURL(ctx, sourceRemote, originPath, git.SetURLOpts{}); err != nil {
return err
}
} else {
if err := remote.Add(ctx, sourceRemote, originPath, git.RemoteAddOpts{}); err != nil {
return err
}
}
if err := o.logStats(ctx, "before fetch"); err != nil {
return err
}
......@@ -52,7 +72,7 @@ func (o *ObjectPool) FetchFromOrigin(ctx context.Context, origin *gitalypb.Repos
git.Flag{Name: "--quiet"},
git.Flag{Name: "--atomic"},
},
Args: []string{originPath, refSpec},
Args: []string{sourceRemote, refSpec},
},
git.WithRefTxHook(ctx, o.poolRepo, o.cfg),
git.WithStderr(&stderr),
......
......@@ -2,6 +2,7 @@ package objectpool
import (
"context"
"errors"
"fmt"
"io"
"os"
......@@ -160,3 +161,23 @@ func (o *ObjectPool) LinkedToRepository(repo *gitalypb.Repository) (bool, error)
return false, nil
}
// Unlink removes the remote from the object pool
func (o *ObjectPool) Unlink(ctx context.Context, repo *gitalypb.Repository) error {
if !o.Exists() {
return errors.New("pool does not exist")
}
remote := o.poolRepo.Remote()
// We need to use removeRemote, and can't leverage `git config --remove-section`
// as the latter doesn't clean up refs
remoteName := repo.GetGlRepository()
if err := remote.Remove(ctx, remoteName); err != nil {
if present, err2 := remote.Exists(ctx, remoteName); err2 != nil || present {
return err
}
}
return nil
}
......@@ -122,6 +122,23 @@ func listBitmaps(t *testing.T, repoPath string) []string {
return bitmaps
}
func TestUnlink(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
pool, testRepo := setupObjectPool(t)
require.Error(t, pool.Unlink(ctx, testRepo), "removing a non-existing pool should be an error")
require.NoError(t, pool.Create(ctx, testRepo), "create pool")
require.NoError(t, pool.Link(ctx, testRepo), "link test repo to pool")
require.False(t, gittest.RemoteExists(t, pool.cfg, pool.FullPath(), testRepo.GetGlRepository()), "pool remotes should include %v", testRepo)
require.NoError(t, pool.Unlink(ctx, testRepo), "unlink repo")
require.False(t, gittest.RemoteExists(t, pool.cfg, pool.FullPath(), testRepo.GetGlRepository()), "pool remotes should no longer include %v", testRepo)
}
func TestLinkAbsoluteLinkExists(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
......
package git
import (
"context"
)
// Remote represents 'remote' sub-command.
// https://git-scm.com/docs/git-remote
type Remote interface {
// Add creates a new remote repository if it doesn't exist.
// If such a remote already exists it returns an ErrAlreadyExists error.
// https://git-scm.com/docs/git-remote#Documentation/git-remote.txt-emaddem
Add(ctx context.Context, name, url string, opts RemoteAddOpts) error
// Remove removes the remote configured for the local repository and all configurations associated with it.
// https://git-scm.com/docs/git-remote#Documentation/git-remote.txt-emremoveem
Remove(ctx context.Context, name string) error
// SetURL sets a new url value for an existing remote.
// If remote doesn't exist it returns an ErrNotFound error.
// https://git-scm.com/docs/git-remote#Documentation/git-remote.txt-emset-urlem
SetURL(ctx context.Context, name, url string, opts SetURLOpts) error
}
// RemoteAddOptsMirror represents possible values for the '--mirror' flag value
type RemoteAddOptsMirror string
func (m RemoteAddOptsMirror) String() string {
return string(m)
}
var (
// RemoteAddOptsMirrorDefault allows to use a default behaviour.
RemoteAddOptsMirrorDefault = RemoteAddOptsMirror("")
// RemoteAddOptsMirrorFetch configures everything in refs/ on the remote to be
// directly mirrored into refs/ in the local repository.
RemoteAddOptsMirrorFetch = RemoteAddOptsMirror("fetch")
// RemoteAddOptsMirrorPush configures 'git push' to always behave as if --mirror was passed.
RemoteAddOptsMirrorPush = RemoteAddOptsMirror("push")
)
// RemoteAddOptsTags controls whether tags will be fetched.
type RemoteAddOptsTags string
func (t RemoteAddOptsTags) String() string {
return string(t)
}
var (
// RemoteAddOptsTagsDefault enables importing of tags only on fetched branches.
RemoteAddOptsTagsDefault = RemoteAddOptsTags("")
// RemoteAddOptsTagsAll enables importing of every tag from the remote repository.
RemoteAddOptsTagsAll = RemoteAddOptsTags("--tags")
// RemoteAddOptsTagsNone disables importing of tags from the remote repository.
RemoteAddOptsTagsNone = RemoteAddOptsTags("--no-tags")
)
// RemoteAddOpts is used to configure invocation of the 'git remote add' command.
// https://git-scm.com/docs/git-remote#Documentation/git-remote.txt-emaddem
type RemoteAddOpts struct {
// RemoteTrackingBranches controls what branches should be tracked instead of
// all branches which is a default refs/remotes/<name>.
// For each entry the refspec '+refs/heads/<branch>:refs/remotes/<remote>/<branch>' would be created and added to the configuration.
RemoteTrackingBranches []string
// DefaultBranch sets the default branch (i.e. the target of the symbolic-ref refs/remotes/<name>/HEAD)
// for the named remote.
// If set to 'develop' then: 'git symbolic-ref refs/remotes/<remote>/HEAD' call will result to 'refs/remotes/<remote>/develop'.
DefaultBranch string
// Fetch controls if 'git fetch <name>' is run immediately after the remote information is set up.
Fetch bool
// Tags controls whether tags will be fetched as part of the remote or not.
Tags RemoteAddOptsTags
// Mirror controls value used for '--mirror' flag.
Mirror RemoteAddOptsMirror
}
// SetURLOpts are the options for SetURL.
type SetURLOpts struct {
// Push URLs are manipulated instead of fetch URLs.
Push bool
}
......@@ -2,6 +2,8 @@ package objectpool
import (
"context"
"errors"
"fmt"
"gitlab.com/gitlab-org/gitaly/v14/internal/helper"
"gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb"
......@@ -29,3 +31,24 @@ func (s *server) LinkRepositoryToObjectPool(ctx context.Context, req *gitalypb.L
return &gitalypb.LinkRepositoryToObjectPoolResponse{}, nil
}
func (s *server) UnlinkRepositoryFromObjectPool(ctx context.Context, req *gitalypb.UnlinkRepositoryFromObjectPoolRequest) (*gitalypb.UnlinkRepositoryFromObjectPoolResponse, error) {
if req.GetRepository() == nil {
return nil, helper.ErrInvalidArgument(errors.New("no repository"))
}
pool, err := s.poolForRequest(req)
if err != nil {
return nil, helper.ErrInternal(err)
}
if !pool.Exists() {
return nil, helper.ErrNotFound(fmt.Errorf("pool repository not found: %s", pool.FullPath()))
}
if err := pool.Unlink(ctx, req.GetRepository()); err != nil {
return nil, helper.ErrInternal(err)
}
return &gitalypb.UnlinkRepositoryFromObjectPoolResponse{}, nil
}
......@@ -155,3 +155,127 @@ func TestLinkNoPool(t *testing.T) {
assert.True(t, storage.IsGitDirectory(poolRepoPath))
}
func TestUnlink(t *testing.T) {
cfg, repo, _, _, client := setup(t, testserver.WithDisablePraefect())
ctx, cancel := testhelper.Context()
defer cancel()
deletedRepo, deletedRepoPath := gittest.CloneRepo(t, cfg, cfg.Storages[0])
pool := initObjectPool(t, cfg, cfg.Storages[0])
require.NoError(t, pool.Create(ctx, repo), "create pool")
require.NoError(t, pool.Link(ctx, repo))
require.NoError(t, pool.Link(ctx, deletedRepo))
require.NoError(t, os.RemoveAll(deletedRepoPath))
require.NoFileExists(t, deletedRepoPath)
pool2 := initObjectPool(t, cfg, cfg.Storages[0])
require.NoError(t, pool2.Create(ctx, repo), "create pool 2")
require.False(t, gittest.RemoteExists(t, cfg, pool.FullPath(), repo.GlRepository), "sanity check: remote exists in pool")
require.False(t, gittest.RemoteExists(t, cfg, pool.FullPath(), deletedRepo.GlRepository), "sanity check: remote exists in pool")
testCases := []struct {
desc string
req *gitalypb.UnlinkRepositoryFromObjectPoolRequest
code codes.Code
}{
{
desc: "Successful request",
req: &gitalypb.UnlinkRepositoryFromObjectPoolRequest{
Repository: repo,
ObjectPool: pool.ToProto(),
},
code: codes.OK,
},
{
desc: "Not linked in the first place",
req: &gitalypb.UnlinkRepositoryFromObjectPoolRequest{
Repository: repo,
ObjectPool: pool2.ToProto(),
},
code: codes.OK,
},
{
desc: "No Repository",
req: &gitalypb.UnlinkRepositoryFromObjectPoolRequest{
Repository: nil,
ObjectPool: pool.ToProto(),
},
code: codes.InvalidArgument,
},
{
desc: "No ObjectPool",
req: &gitalypb.UnlinkRepositoryFromObjectPoolRequest{
Repository: repo,
ObjectPool: nil,
},
code: codes.InvalidArgument,
},
{
desc: "Repo not found",
req: &gitalypb.UnlinkRepositoryFromObjectPoolRequest{
Repository: deletedRepo,
ObjectPool: pool.ToProto(),
},
code: codes.OK,
},
{
desc: "Pool not found",
req: &gitalypb.UnlinkRepositoryFromObjectPoolRequest{
Repository: repo,
ObjectPool: &gitalypb.ObjectPool{
Repository: &gitalypb.Repository{
StorageName: repo.GetStorageName(),
RelativePath: gittest.NewObjectPoolName(t), // does not exist
},
},
},
code: codes.NotFound,
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
//nolint:staticcheck
_, err := client.UnlinkRepositoryFromObjectPool(ctx, tc.req)
if tc.code != codes.OK {
testhelper.RequireGrpcError(t, err, tc.code)
return
}
require.NoError(t, err, "call UnlinkRepositoryFromObjectPool")
remoteName := tc.req.Repository.GlRepository
require.False(t, gittest.RemoteExists(t, cfg, pool.FullPath(), remoteName), "remote should no longer exist in pool")
})
}
}
func TestUnlinkIdempotent(t *testing.T) {
cfg, repo, _, _, client := setup(t)
ctx, cancel := testhelper.Context()
defer cancel()
pool := initObjectPool(t, cfg, cfg.Storages[0])
require.NoError(t, pool.Create(ctx, repo))
require.NoError(t, pool.Link(ctx, repo))
request := &gitalypb.UnlinkRepositoryFromObjectPoolRequest{
Repository: repo,
ObjectPool: pool.ToProto(),
}
//nolint:staticcheck
_, err := client.UnlinkRepositoryFromObjectPool(ctx, request)
require.NoError(t, err)
//nolint:staticcheck
_, err = client.UnlinkRepositoryFromObjectPool(ctx, request)
require.NoError(t, err)
}
......@@ -41,6 +41,6 @@ func TestReduplicate(t *testing.T) {
_, err = client.ReduplicateRepository(ctx, &gitalypb.ReduplicateRepositoryRequest{Repository: repo})
require.NoError(t, err)
require.NoError(t, os.RemoveAll(altPath))
require.NoError(t, pool.Unlink(ctx, repo))
gittest.Exec(t, cfg, "-C", repoPath, "cat-file", "-e", existingObjectID)
}
......@@ -86,12 +86,13 @@ var transactionRPCs = map[string]transactionsCondition{
// The following RPCs currently aren't transactional, but we may consider making them
// transactional in the future if the need arises.
"/gitaly.ObjectPoolService/CreateObjectPool": transactionsDisabled,
"/gitaly.ObjectPoolService/DeleteObjectPool": transactionsDisabled,
"/gitaly.ObjectPoolService/DisconnectGitAlternates": transactionsDisabled,
"/gitaly.ObjectPoolService/LinkRepositoryToObjectPool": transactionsDisabled,
"/gitaly.ObjectPoolService/ReduplicateRepository": transactionsDisabled,
"/gitaly.RepositoryService/RenameRepository": transactionsDisabled,
"/gitaly.ObjectPoolService/CreateObjectPool": transactionsDisabled,
"/gitaly.ObjectPoolService/DeleteObjectPool": transactionsDisabled,
"/gitaly.ObjectPoolService/DisconnectGitAlternates": transactionsDisabled,
"/gitaly.ObjectPoolService/LinkRepositoryToObjectPool": transactionsDisabled,
"/gitaly.ObjectPoolService/ReduplicateRepository": transactionsDisabled,
"/gitaly.ObjectPoolService/UnlinkRepositoryFromObjectPool": transactionsDisabled,
"/gitaly.RepositoryService/RenameRepository": transactionsDisabled,
// The following list of RPCs are considered idempotent RPCs: while they write into the
// target repository, this shouldn't ever have any user-visible impact given that they're
......
......@@ -58,11 +58,12 @@ func TestNewProtoRegistry(t *testing.T) {
"NamespaceExists": protoregistry.OpAccessor,
},
"ObjectPoolService": {
"CreateObjectPool": protoregistry.OpMutator,
"DeleteObjectPool": protoregistry.OpMutator,
"LinkRepositoryToObjectPool": protoregistry.OpMutator,
"ReduplicateRepository": protoregistry.OpMutator,
"DisconnectGitAlternates": protoregistry.OpMutator,
"CreateObjectPool": protoregistry.OpMutator,
"DeleteObjectPool": protoregistry.OpMutator,
"LinkRepositoryToObjectPool": protoregistry.OpMutator,
"UnlinkRepositoryFromObjectPool": protoregistry.OpMutator,
"ReduplicateRepository": protoregistry.OpMutator,
"DisconnectGitAlternates": protoregistry.OpMutator,
},
"OperationService": {
"UserCreateBranch": protoregistry.OpMutator,
......
This diff is collapsed.
......@@ -22,6 +22,18 @@ type ObjectPoolServiceClient interface {
DeleteObjectPool(ctx context.Context, in *DeleteObjectPoolRequest, opts ...grpc.CallOption) (*DeleteObjectPoolResponse, error)
// Repositories are assumed to be stored on the same disk
LinkRepositoryToObjectPool(ctx context.Context, in *LinkRepositoryToObjectPoolRequest, opts ...grpc.CallOption) (*LinkRepositoryToObjectPoolResponse, error)
// Deprecated: Do not use.
// UnlinkRepositoryFromObjectPool does not unlink the repository from the
// object pool as you'd think, but all it really does is to remove the object
// pool's remote pointing to the repository. And even this is a no-op given
// that we'd try to remove the remote by the repository's `GlRepository()`
// name, which we never create in the first place. To unlink repositories
// from an object pool, you'd really want to execute DisconnectGitAlternates
// to remove the repository's link to the pool's object database.
//
// This function is never called by anyone and highly misleading. It's thus
// deprecated and will be removed in v14.4.
UnlinkRepositoryFromObjectPool(ctx context.Context, in *UnlinkRepositoryFromObjectPoolRequest, opts ...grpc.CallOption) (*UnlinkRepositoryFromObjectPoolResponse, error)
ReduplicateRepository(ctx context.Context, in *ReduplicateRepositoryRequest, opts ...grpc.CallOption) (*ReduplicateRepositoryResponse, error)
DisconnectGitAlternates(ctx context.Context, in *DisconnectGitAlternatesRequest, opts ...grpc.CallOption) (*DisconnectGitAlternatesResponse, error)
FetchIntoObjectPool(ctx context.Context, in *FetchIntoObjectPoolRequest, opts ...grpc.CallOption) (*FetchIntoObjectPoolResponse, error)
......@@ -63,6 +75,16 @@ func (c *objectPoolServiceClient) LinkRepositoryToObjectPool(ctx context.Context
return out, nil
}
// Deprecated: Do not use.
func (c *objectPoolServiceClient) UnlinkRepositoryFromObjectPool(ctx context.Context, in *UnlinkRepositoryFromObjectPoolRequest, opts ...grpc.CallOption) (*UnlinkRepositoryFromObjectPoolResponse, error) {
out := new(UnlinkRepositoryFromObjectPoolResponse)
err := c.cc.Invoke(ctx, "/gitaly.ObjectPoolService/UnlinkRepositoryFromObjectPool", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *objectPoolServiceClient) ReduplicateRepository(ctx context.Context, in *ReduplicateRepositoryRequest, opts ...grpc.CallOption) (*ReduplicateRepositoryResponse, error) {
out := new(ReduplicateRepositoryResponse)
err := c.cc.Invoke(ctx, "/gitaly.ObjectPoolService/ReduplicateRepository", in, out, opts...)
......@@ -107,6 +129,18 @@ type ObjectPoolServiceServer interface {
DeleteObjectPool(context.Context, *DeleteObjectPoolRequest) (*DeleteObjectPoolResponse, error)
// Repositories are assumed to be stored on the same disk
LinkRepositoryToObjectPool(context.Context, *LinkRepositoryToObjectPoolRequest) (*LinkRepositoryToObjectPoolResponse, error)
// Deprecated: Do not use.
// UnlinkRepositoryFromObjectPool does not unlink the repository from the
// object pool as you'd think, but all it really does is to remove the object
// pool's remote pointing to the repository. And even this is a no-op given
// that we'd try to remove the remote by the repository's `GlRepository()`
// name, which we never create in the first place. To unlink repositories
// from an object pool, you'd really want to execute DisconnectGitAlternates
// to remove the repository's link to the pool's object database.
//
// This function is never called by anyone and highly misleading. It's thus
// deprecated and will be removed in v14.4.
UnlinkRepositoryFromObjectPool(context.Context, *UnlinkRepositoryFromObjectPoolRequest) (*UnlinkRepositoryFromObjectPoolResponse, error)
ReduplicateRepository(context.Context, *ReduplicateRepositoryRequest) (*ReduplicateRepositoryResponse, error)
DisconnectGitAlternates(context.Context, *DisconnectGitAlternatesRequest) (*DisconnectGitAlternatesResponse, error)
FetchIntoObjectPool(context.Context, *FetchIntoObjectPoolRequest) (*FetchIntoObjectPoolResponse, error)
......@@ -127,6 +161,9 @@ func (UnimplementedObjectPoolServiceServer) DeleteObjectPool(context.Context, *D
func (UnimplementedObjectPoolServiceServer) LinkRepositoryToObjectPool(context.Context, *LinkRepositoryToObjectPoolRequest) (*LinkRepositoryToObjectPoolResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method LinkRepositoryToObjectPool not implemented")
}
func (UnimplementedObjectPoolServiceServer) UnlinkRepositoryFromObjectPool(context.Context, *UnlinkRepositoryFromObjectPoolRequest) (*UnlinkRepositoryFromObjectPoolResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method UnlinkRepositoryFromObjectPool not implemented")
}
func (UnimplementedObjectPoolServiceServer) ReduplicateRepository(context.Context, *ReduplicateRepositoryRequest) (*ReduplicateRepositoryResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ReduplicateRepository not implemented")
}
......@@ -206,6 +243,24 @@ func _ObjectPoolService_LinkRepositoryToObjectPool_Handler(srv interface{}, ctx
return interceptor(ctx, in, info, handler)
}
func _ObjectPoolService_UnlinkRepositoryFromObjectPool_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(UnlinkRepositoryFromObjectPoolRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ObjectPoolServiceServer).UnlinkRepositoryFromObjectPool(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/gitaly.ObjectPoolService/UnlinkRepositoryFromObjectPool",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ObjectPoolServiceServer).UnlinkRepositoryFromObjectPool(ctx, req.(*UnlinkRepositoryFromObjectPoolRequest))
}
return interceptor(ctx, in, info, handler)
}
func _ObjectPoolService_ReduplicateRepository_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ReduplicateRepositoryRequest)
if err := dec(in); err != nil {
......@@ -297,6 +352,10 @@ var ObjectPoolService_ServiceDesc = grpc.ServiceDesc{
MethodName: "LinkRepositoryToObjectPool",
Handler: _ObjectPoolService_LinkRepositoryToObjectPool_Handler,
},
{
MethodName: "UnlinkRepositoryFromObjectPool",
Handler: _ObjectPoolService_UnlinkRepositoryFromObjectPool_Handler,
},
{
MethodName: "ReduplicateRepository",
Handler: _ObjectPoolService_ReduplicateRepository_Handler,
......
......@@ -26,6 +26,23 @@ service ObjectPoolService {
};
}
// UnlinkRepositoryFromObjectPool does not unlink the repository from the
// object pool as you'd think, but all it really does is to remove the object
// pool's remote pointing to the repository. And even this is a no-op given
// that we'd try to remove the remote by the repository's `GlRepository()`
// name, which we never create in the first place. To unlink repositories
// from an object pool, you'd really want to execute DisconnectGitAlternates
// to remove the repository's link to the pool's object database.
//
// This function is never called by anyone and highly misleading. It's thus
// deprecated and will be removed in v14.4.
rpc UnlinkRepositoryFromObjectPool(UnlinkRepositoryFromObjectPoolRequest) returns (UnlinkRepositoryFromObjectPoolResponse) {
option deprecated = true;
option (op_type) = {
op: MUTATOR
};
}
rpc ReduplicateRepository(ReduplicateRepositoryRequest) returns (ReduplicateRepositoryResponse) {
option (op_type) = {
op: MUTATOR
......@@ -69,6 +86,14 @@ message LinkRepositoryToObjectPoolRequest {
}
message LinkRepositoryToObjectPoolResponse {}
// This RPC doesn't require the ObjectPool as it will remove the alternates file
// from the pool participant. The caller is responsible no data loss occurs.
message UnlinkRepositoryFromObjectPoolRequest {
Repository repository = 1 [(target_repository)=true]; // already specified as the target repo field
ObjectPool object_pool = 2 [(additional_repository)=true];
}
message UnlinkRepositoryFromObjectPoolResponse {}
message ReduplicateRepositoryRequest {
Repository repository = 1 [(target_repository)=true];
}
......
......@@ -24,6 +24,12 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
end
add_message "gitaly.LinkRepositoryToObjectPoolResponse" do
end
add_message "gitaly.UnlinkRepositoryFromObjectPoolRequest" do
optional :repository, :message, 1, "gitaly.Repository"
optional :object_pool, :message, 2, "gitaly.ObjectPool"
end
add_message "gitaly.UnlinkRepositoryFromObjectPoolResponse" do
end
add_message "gitaly.ReduplicateRepositoryRequest" do
optional :repository, :message, 1, "gitaly.Repository"
end
......@@ -57,6 +63,8 @@ module Gitaly
DeleteObjectPoolResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.DeleteObjectPoolResponse").msgclass
LinkRepositoryToObjectPoolRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.LinkRepositoryToObjectPoolRequest").msgclass
LinkRepositoryToObjectPoolResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.LinkRepositoryToObjectPoolResponse").msgclass
UnlinkRepositoryFromObjectPoolRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.UnlinkRepositoryFromObjectPoolRequest").msgclass
UnlinkRepositoryFromObjectPoolResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.UnlinkRepositoryFromObjectPoolResponse").msgclass
ReduplicateRepositoryRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.ReduplicateRepositoryRequest").msgclass
ReduplicateRepositoryResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.ReduplicateRepositoryResponse").msgclass
DisconnectGitAlternatesRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.DisconnectGitAlternatesRequest").msgclass
......
......@@ -18,6 +18,17 @@ module Gitaly
rpc :DeleteObjectPool, Gitaly::DeleteObjectPoolRequest, Gitaly::DeleteObjectPoolResponse
# Repositories are assumed to be stored on the same disk
rpc :LinkRepositoryToObjectPool, Gitaly::LinkRepositoryToObjectPoolRequest, Gitaly::LinkRepositoryToObjectPoolResponse
# UnlinkRepositoryFromObjectPool does not unlink the repository from the
# object pool as you'd think, but all it really does is to remove the object
# pool's remote pointing to the repository. And even this is a no-op given
# that we'd try to remove the remote by the repository's `GlRepository()`
# name, which we never create in the first place. To unlink repositories
# from an object pool, you'd really want to execute DisconnectGitAlternates
# to remove the repository's link to the pool's object database.
#
# This function is never called by anyone and highly misleading. It's thus
# deprecated and will be removed in v14.4.
rpc :UnlinkRepositoryFromObjectPool, Gitaly::UnlinkRepositoryFromObjectPoolRequest, Gitaly::UnlinkRepositoryFromObjectPoolResponse
rpc :ReduplicateRepository, Gitaly::ReduplicateRepositoryRequest, Gitaly::ReduplicateRepositoryResponse
rpc :DisconnectGitAlternates, Gitaly::DisconnectGitAlternatesRequest, Gitaly::DisconnectGitAlternatesResponse
rpc :FetchIntoObjectPool, Gitaly::FetchIntoObjectPoolRequest, Gitaly::FetchIntoObjectPoolResponse
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment