Skip to content
Snippets Groups Projects
Verified Commit 73eac5af authored by Sami Hiltunen's avatar Sami Hiltunen Committed by GitLab
Browse files

Merge branch '6607-relative-path' into 'master'

recovery: Run recovery commands with relative path

Closes #6607

See merge request !7621



Merged-by: Sami Hiltunen's avatarSami Hiltunen <shiltunen@gitlab.com>
Approved-by: default avatarJames Fargher <jfargher@gitlab.com>
Approved-by: Sami Hiltunen's avatarSami Hiltunen <shiltunen@gitlab.com>
Reviewed-by: default avatarJames Fargher <jfargher@gitlab.com>
Co-authored-by: default avatarMustafa Bayar <mbayar@gitlab.com>
parents f1205f97 107791a0
No related branches found
No related tags found
1 merge request!7621recovery: Run recovery commands with relative path
Pipeline #1679553747 failed
Pipeline: praefect-migration-testing

#1679553978

    ......@@ -36,9 +36,9 @@ import (
    )
    const (
    flagPartition = "partition"
    flagAll = "all"
    flagParallel = "parallel"
    flagPartition = "partition"
    )
    type recoveryContext struct {
    ......@@ -76,13 +76,17 @@ Example: gitaly recovery --config gitaly.config.toml status --storage default --
    Name: flagPartition,
    Usage: "partition ID",
    },
    &cli.StringFlag{
    Name: flagRepository,
    Usage: "relative path to the repository",
    },
    &cli.BoolFlag{
    Name: flagAll,
    Usage: "runs the command for all partitions in the storage",
    },
    &cli.IntFlag{
    Name: flagParallel,
    Usage: "maximum number of parallel restores per storage",
    Usage: "maximum number of parallel queries per storage",
    Value: 2,
    },
    },
    ......@@ -103,6 +107,10 @@ Example: gitaly recovery --config gitaly.config.toml status --storage default --
    Name: flagPartition,
    Usage: "partition ID",
    },
    &cli.StringFlag{
    Name: flagRepository,
    Usage: "relative path to the repository",
    },
    &cli.BoolFlag{
    Name: flagAll,
    Usage: "runs the command for all partitions in the storage",
    ......@@ -601,11 +609,31 @@ func setupRecoveryContext(ctx *cli.Context) (rc recoveryContext, returnErr error
    return recoveryContext, fmt.Errorf("partition iterator: %w", err)
    }
    } else {
    partitionString := ctx.String(flagPartition)
    repositoryPath := ctx.String(flagRepository)
    if partitionString != "" && repositoryPath != "" {
    return recoveryContext, fmt.Errorf("--partition and --repository flags can not be provided at the same time")
    }
    if partitionString == "" && repositoryPath == "" {
    return recoveryContext, fmt.Errorf("this command requires one of --all, --partition or --repository flags")
    }
    var err error
    var partitionID storage.PartitionID
    if err := parsePartitionID(&partitionID, ctx.String(flagPartition)); err != nil {
    return recoveryContext, fmt.Errorf("parse partition ID: %w", err)
    if partitionString != "" {
    if err = parsePartitionID(&partitionID, partitionString); err != nil {
    return recoveryContext, fmt.Errorf("parse partition ID: %w", err)
    }
    } else {
    partitionID, err = nodeStorage.GetAssignedPartitionID(repositoryPath)
    if err != nil {
    return recoveryContext, fmt.Errorf("partition ID not found for the given relative path: %w", err)
    }
    }
    if partitionID == 0 {
    if partitionID == storage.PartitionID(0) {
    return recoveryContext, fmt.Errorf("invalid partition ID %s", partitionID)
    }
    ......
    ......@@ -44,8 +44,7 @@ type setupOptions struct {
    type setupData struct {
    storageName string
    partitionID storage.PartitionID
    all bool
    args []string
    expectedErr error
    expectedOutputs []string
    expectedLSN map[storage.PartitionID]storage.LSN
    ......@@ -73,7 +72,7 @@ func TestRecoveryCLI_status(t *testing.T) {
    setup: func(tb testing.TB, ctx context.Context, opts setupOptions) setupData {
    return setupData{
    storageName: opts.cfg.Storages[0].Name,
    partitionID: 0,
    args: []string{"-partition", storage.PartitionID(0).String()},
    expectedErr: errors.New("exit status 1"),
    expectedOutputs: []string{fmt.Sprintf("invalid partition ID %s\n", storage.PartitionID(0))},
    }
    ......@@ -84,7 +83,7 @@ func TestRecoveryCLI_status(t *testing.T) {
    setup: func(tb testing.TB, ctx context.Context, opts setupOptions) setupData {
    return setupData{
    storageName: opts.cfg.Storages[0].Name,
    partitionID: 42,
    args: []string{"-partition", storage.PartitionID(42).String()},
    // TODO: This currently will create arbitrary partitions.
    // It should return an error instead.
    // https://gitlab.com/gitlab-org/gitaly/-/issues/6478
    ......@@ -98,6 +97,52 @@ recovery status completed: 1 succeeded, 0 failed`,
    }
    },
    },
    {
    desc: "not all necessary flags provided",
    setup: func(tb testing.TB, ctx context.Context, opts setupOptions) setupData {
    repo, err := createRepository(t, ctx, opts)
    require.NoError(t, err)
    partitionPath := filepath.Join(repo.GetStorageName(), fmt.Sprintf("%d", storage.PartitionID(2)))
    testhelper.WriteFiles(t, opts.backupRoot, map[string]any{
    filepath.Join(partitionPath, storage.LSN(1).String()+".tar"): "",
    filepath.Join(partitionPath, storage.LSN(2).String()+".tar"): "",
    filepath.Join(partitionPath, storage.LSN(3).String()+".tar"): "",
    })
    return setupData{
    storageName: repo.GetStorageName(),
    args: []string{},
    expectedErr: errors.New("exit status 1"),
    expectedOutputs: []string{
    "this command requires one of --all, --partition or --repository flags",
    },
    }
    },
    },
    {
    desc: "both partition ID and relative path provided",
    setup: func(tb testing.TB, ctx context.Context, opts setupOptions) setupData {
    repo, err := createRepository(t, ctx, opts)
    require.NoError(t, err)
    partitionPath := filepath.Join(repo.GetStorageName(), fmt.Sprintf("%d", storage.PartitionID(2)))
    testhelper.WriteFiles(t, opts.backupRoot, map[string]any{
    filepath.Join(partitionPath, storage.LSN(1).String()+".tar"): "",
    filepath.Join(partitionPath, storage.LSN(2).String()+".tar"): "",
    filepath.Join(partitionPath, storage.LSN(3).String()+".tar"): "",
    })
    return setupData{
    storageName: repo.GetStorageName(),
    args: []string{"-partition", storage.PartitionID(2).String(), "-repository", repo.GetRelativePath()},
    expectedErr: errors.New("exit status 1"),
    expectedOutputs: []string{
    "--partition and --repository flags can not be provided at the same time",
    },
    }
    },
    },
    {
    desc: "success, no backups",
    setup: func(tb testing.TB, ctx context.Context, opts setupOptions) setupData {
    ......@@ -106,7 +151,7 @@ recovery status completed: 1 succeeded, 0 failed`,
    return setupData{
    storageName: repo.GetStorageName(),
    partitionID: 2,
    args: []string{"-partition", storage.PartitionID(2).String()},
    expectedOutputs: []string{fmt.Sprintf(`---------------------------------------------
    Partition ID: %s - Applied LSN: %s
    Relative paths:
    ......@@ -135,7 +180,37 @@ recovery status completed: 1 succeeded, 0 failed`,
    return setupData{
    storageName: repo.GetStorageName(),
    partitionID: 2,
    args: []string{"-partition", storage.PartitionID(2).String()},
    expectedOutputs: []string{fmt.Sprintf(`---------------------------------------------
    Partition ID: %s - Applied LSN: %s
    Relative paths:
    - %s
    Available WAL backup entries: up to LSN: %s
    recovery status completed: 1 succeeded, 0 failed`,
    storage.PartitionID(2),
    storage.LSN(1),
    repo.GetRelativePath(),
    storage.LSN(3),
    )},
    }
    },
    },
    {
    desc: "success using relative path",
    setup: func(tb testing.TB, ctx context.Context, opts setupOptions) setupData {
    repo, err := createRepository(t, ctx, opts)
    require.NoError(t, err)
    partitionPath := filepath.Join(repo.GetStorageName(), fmt.Sprintf("%d", storage.PartitionID(2)))
    testhelper.WriteFiles(t, opts.backupRoot, map[string]any{
    filepath.Join(partitionPath, storage.LSN(1).String()+".tar"): "",
    filepath.Join(partitionPath, storage.LSN(2).String()+".tar"): "",
    filepath.Join(partitionPath, storage.LSN(3).String()+".tar"): "",
    })
    return setupData{
    storageName: repo.GetStorageName(),
    args: []string{"-repository", repo.GetRelativePath()},
    expectedOutputs: []string{fmt.Sprintf(`---------------------------------------------
    Partition ID: %s - Applied LSN: %s
    Relative paths:
    ......@@ -169,7 +244,7 @@ recovery status completed: 1 succeeded, 0 failed`,
    return setupData{
    storageName: repo.GetStorageName(),
    partitionID: 2,
    args: []string{"-partition", storage.PartitionID(2).String()},
    expectedOutputs: []string{fmt.Sprintf(`---------------------------------------------
    Partition ID: %s - Applied LSN: %s
    Relative paths:
    ......@@ -210,7 +285,7 @@ recovery status completed: 1 succeeded, 0 failed`,
    return setupData{
    storageName: opts.cfg.Storages[0].Name,
    all: true,
    args: []string{"-all", "-parallel", "2"},
    expectedOutputs: []string{
    fmt.Sprintf(`---------------------------------------------
    Partition ID: %s - Applied LSN: %s
    ......@@ -287,13 +362,7 @@ Available WAL backup entries: up to LSN: %s`,
    dbMgr.Close()
    args := []string{"recovery", "-config", configPath, "status", "-storage", data.storageName}
    if data.all {
    args = append(args, "-all", "-parallel", "2")
    } else {
    args = append(args, "-partition")
    args = append(args, data.partitionID.String())
    }
    args = append(args, data.args...)
    cmd := exec.Command(cfg.BinaryPath("gitaly"), args...)
    output, err := cmd.CombinedOutput()
    testhelper.RequireGrpcError(t, data.expectedErr, err)
    ......@@ -328,7 +397,7 @@ func TestRecoveryCLI_replay(t *testing.T) {
    setup: func(tb testing.TB, ctx context.Context, opts setupOptions) setupData {
    return setupData{
    storageName: opts.cfg.Storages[0].Name,
    partitionID: 0,
    args: []string{"-partition", storage.PartitionID(0).String()},
    expectedErr: errors.New("exit status 1"),
    expectedOutputs: []string{fmt.Sprintf("invalid partition ID %s\n", storage.PartitionID(0))},
    expectedLSN: nil,
    ......@@ -340,7 +409,7 @@ func TestRecoveryCLI_replay(t *testing.T) {
    setup: func(tb testing.TB, ctx context.Context, opts setupOptions) setupData {
    return setupData{
    storageName: opts.cfg.Storages[0].Name,
    partitionID: 42,
    args: []string{"-partition", storage.PartitionID(42).String()},
    // TODO: This currently will create arbitrary partitions.
    // It should return an error instead.
    // https://gitlab.com/gitlab-org/gitaly/-/issues/6478
    ......@@ -367,7 +436,7 @@ recovery replay completed: 1 succeeded, 0 failed`,
    return setupData{
    storageName: repo.GetStorageName(),
    partitionID: 2,
    args: []string{"-partition", storage.PartitionID(2).String()},
    expectedOutputs: []string{
    "started processing partition 2",
    fmt.Sprintf(`---------------------------------------------
    ......@@ -398,7 +467,38 @@ recovery replay completed: 1 succeeded, 0 failed`,
    return setupData{
    storageName: repo.GetStorageName(),
    partitionID: 2,
    args: []string{"-partition", storage.PartitionID(2).String()},
    expectedOutputs: []string{
    "started processing partition 2",
    fmt.Sprintf(`---------------------------------------------
    Partition ID: %s - Applied LSN: %s
    Successfully processed log entries up to LSN: %s
    recovery replay completed: 1 succeeded, 0 failed`,
    storage.PartitionID(2),
    storage.LSN(1),
    storage.LSN(3),
    ),
    },
    expectedLSN: map[storage.PartitionID]storage.LSN{2: 3},
    }
    },
    },
    {
    desc: "success using relative path, contiguous backups",
    setup: func(tb testing.TB, ctx context.Context, opts setupOptions) setupData {
    repo, err := createRepository(t, ctx, opts)
    require.NoError(t, err)
    partitionPath := filepath.Join(repo.GetStorageName(), fmt.Sprintf("%d", storage.PartitionID(2)))
    testhelper.WriteFiles(t, opts.backupRoot, map[string]any{
    filepath.Join(partitionPath, storage.LSN(1).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath(), storage.LSN(1)),
    filepath.Join(partitionPath, storage.LSN(2).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath(), storage.LSN(2)),
    filepath.Join(partitionPath, storage.LSN(3).String()+".tar"): createValidLogEntryArchive(t, repo.GetRelativePath(), storage.LSN(3)),
    })
    return setupData{
    storageName: repo.GetStorageName(),
    args: []string{"-repository", repo.GetRelativePath()},
    expectedOutputs: []string{
    "started processing partition 2",
    fmt.Sprintf(`---------------------------------------------
    ......@@ -429,7 +529,7 @@ recovery replay completed: 1 succeeded, 0 failed`,
    return setupData{
    storageName: repo.GetStorageName(),
    partitionID: 2,
    args: []string{"-partition", storage.PartitionID(2).String()},
    expectedErr: errors.New("exit status 1"),
    expectedOutputs: []string{
    "started processing partition 2",
    ......@@ -456,7 +556,7 @@ recovery replay completed: 1 succeeded, 0 failed`,
    return setupData{
    storageName: repo.GetStorageName(),
    partitionID: 2,
    args: []string{"-partition", storage.PartitionID(2).String()},
    expectedErr: errors.New("exit status 1"),
    expectedOutputs: []string{
    "started processing partition 2",
    ......@@ -491,7 +591,7 @@ recovery replay completed: 1 succeeded, 0 failed`,
    })
    return setupData{
    storageName: opts.cfg.Storages[0].Name,
    all: true,
    args: []string{"-all", "-parallel", "2"},
    expectedOutputs: []string{
    "started processing partition 2",
    fmt.Sprintf(`---------------------------------------------
    ......@@ -570,12 +670,7 @@ Successfully processed log entries up to LSN: %s`,
    dbMgr.Close()
    args := []string{"recovery", "-config", configPath, "replay", "-storage", data.storageName}
    if data.all {
    args = append(args, "-all", "-parallel", "2")
    } else {
    args = append(args, "-partition")
    args = append(args, data.partitionID.String())
    }
    args = append(args, data.args...)
    cmd := exec.Command(cfg.BinaryPath("gitaly"), args...)
    output, err := cmd.CombinedOutput()
    ......
    0% Loading or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment