Commit d48d6583 authored by Alessio Caiazza's avatar Alessio Caiazza Committed by Tomasz Maczukin

Add refspec fetching feature

Gitlab can now control the source getting process providing a set of
refspec.

If present runner will switch to the new method.
parent 694578b5
......@@ -595,6 +595,8 @@ func (b *Build) GetRemoteURL() string {
return fmt.Sprintf("%sgitlab-ci-token:%s@%s/%s.git", splits[0], ciJobToken, splits[1], ciProjectPath)
}
// GetGitDepth is deprecated and will be removed in 12.0, use build.GitInfo.Depth instead
// TODO: Remove in 12.0
func (b *Build) GetGitDepth() string {
return b.GetAllVariables().Get("GIT_DEPTH")
}
......@@ -706,6 +708,10 @@ func (b *Build) Duration() time.Duration {
return time.Since(b.createdAt)
}
func (b *Build) RefspecsAvailable() bool {
return len(b.GitInfo.Refspecs) > 0
}
func NewBuild(jobData JobResponse, runnerConfig *RunnerConfig, systemInterrupt chan os.Signal, executorData ExecutorData) (*Build, error) {
// Attempt to perform a deep copy of the RunnerConfig
runnerConfigCopy, err := runnerConfig.DeepCopy()
......
......@@ -61,6 +61,7 @@ type FeaturesInfo struct {
UploadRawArtifacts bool `json:"upload_raw_artifacts"`
Session bool `json:"session"`
Terminal bool `json:"terminal"`
Refspecs bool `json:"refspecs"`
}
type RegisterRunnerParameters struct {
......@@ -134,6 +135,8 @@ type GitInfo struct {
Sha string `json:"sha"`
BeforeSha string `json:"before_sha"`
RefType GitInfoRefType `json:"ref_type"`
Refspecs []string `json:"refspecs"`
Depth int `json:"depth"`
}
type RunnerInfo struct {
......
......@@ -54,17 +54,17 @@ func GetRemoteSuccessfulBuildWithAfterScript() (JobResponse, error) {
return jobResponse, err
}
func GetRemoteSuccessfulBuildWithDumpedVariables() (response JobResponse, err error) {
func GetRemoteSuccessfulBuildWithDumpedVariables() (JobResponse, error) {
variableName := "test_dump"
variableValue := "test"
response, err = GetRemoteBuildResponse(
response, err := GetRemoteBuildResponse(
fmt.Sprintf("[[ \"${%s}\" != \"\" ]]", variableName),
fmt.Sprintf("[[ $(cat $%s) == \"%s\" ]]", variableName, variableValue),
)
if err != nil {
return
return JobResponse{}, err
}
dumpedVariable := JobVariable{
......@@ -73,7 +73,7 @@ func GetRemoteSuccessfulBuildWithDumpedVariables() (response JobResponse, err er
}
response.Variables = append(response.Variables, dumpedVariable)
return
return response, nil
}
func GetFailedBuild() (JobResponse, error) {
......@@ -101,28 +101,28 @@ fi
`)
}
func GetRemoteBrokenTLSBuild() (job JobResponse, err error) {
func GetRemoteBrokenTLSBuild() (JobResponse, error) {
invalidCert, err := buildSnakeOilCert()
if err != nil {
return
return JobResponse{}, err
}
return getRemoteCustomTLSBuild(invalidCert)
}
func GetRemoteGitLabComTLSBuild() (job JobResponse, err error) {
func GetRemoteGitLabComTLSBuild() (JobResponse, error) {
cert, err := getGitLabComTLSChain()
if err != nil {
return
return JobResponse{}, err
}
return getRemoteCustomTLSBuild(cert)
}
func getRemoteCustomTLSBuild(chain string) (job JobResponse, err error) {
job, err = GetRemoteBuildResponse("echo Hello World")
func getRemoteCustomTLSBuild(chain string) (JobResponse, error) {
job, err := GetRemoteBuildResponse("echo Hello World")
if err != nil {
return
return JobResponse{}, err
}
job.TLSCAChain = chain
......@@ -130,17 +130,18 @@ func getRemoteCustomTLSBuild(chain string) (job JobResponse, err error) {
JobVariable{Key: "GIT_STRATEGY", Value: "clone"},
JobVariable{Key: "GIT_SUBMODULE_STRATEGY", Value: "normal"})
return
return job, nil
}
func GetRemoteBuildResponse(commands ...string) (response JobResponse, err error) {
response = JobResponse{
func getBuildResponse(repoURL string, commands []string) JobResponse {
return JobResponse{
GitInfo: GitInfo{
RepoURL: repoRemoteURL,
RepoURL: repoURL,
Sha: repoSHA,
BeforeSha: repoBeforeSHA,
Ref: repoRefName,
RefType: repoRefType,
Refspecs: []string{"+refs/heads/*:refs/origin/heads/*", "+refs/tags/*:refs/tags/*"},
},
Steps: Steps{
Step{
......@@ -151,35 +152,19 @@ func GetRemoteBuildResponse(commands ...string) (response JobResponse, err error
},
},
}
}
return
func GetRemoteBuildResponse(commands ...string) (JobResponse, error) {
return getBuildResponse(repoRemoteURL, commands), nil
}
func GetLocalBuildResponse(commands ...string) (response JobResponse, err error) {
func GetLocalBuildResponse(commands ...string) (JobResponse, error) {
localRepoURL, err := getLocalRepoURL()
if err != nil {
return
}
response = JobResponse{
GitInfo: GitInfo{
RepoURL: localRepoURL,
Sha: repoSHA,
BeforeSha: repoBeforeSHA,
Ref: repoRefName,
RefType: repoRefType,
},
Steps: Steps{
Step{
Name: StepNameScript,
Script: commands,
When: StepWhenAlways,
AllowFailure: false,
},
},
return JobResponse{}, err
}
return
return getBuildResponse(localRepoURL, commands), nil
}
func getLocalRepoURL() (string, error) {
......
......@@ -410,7 +410,7 @@ func TestDockerCommandOutput(t *testing.T) {
err = build.Run(&common.Config{}, &common.Trace{Writer: &buffer})
assert.NoError(t, err)
re, err := regexp.Compile("(?m)^Cloning into '/builds/gitlab-org/gitlab-test'...")
re, err := regexp.Compile("(?m)^Initialized empty Git repository in /builds/gitlab-org/gitlab-test/.git/")
assert.NoError(t, err)
assert.Regexp(t, re, buffer.String())
}
......@@ -898,7 +898,7 @@ func TestDockerCommandWithBrokenGitSSLCAInfo(t *testing.T) {
err = build.Run(&common.Config{}, &common.Trace{Writer: &buffer})
assert.Error(t, err)
out := buffer.String()
assert.Contains(t, out, "Cloning repository")
assert.Contains(t, out, "Created fresh repository")
assert.NotContains(t, out, "Updating/initializing submodules")
}
......@@ -930,7 +930,7 @@ func TestDockerCommandWithGitSSLCAInfo(t *testing.T) {
err = build.Run(&common.Config{}, &common.Trace{Writer: &buffer})
assert.NoError(t, err)
out := buffer.String()
assert.Contains(t, out, "Cloning repository")
assert.Contains(t, out, "Created fresh repository")
assert.Contains(t, out, "Updating/initializing submodules")
}
......
......@@ -2,6 +2,7 @@ package shell_test
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
......@@ -300,7 +301,7 @@ func TestBuildWithGitStrategyNone(t *testing.T) {
out, err := runBuildReturningOutput(t, build)
assert.NoError(t, err)
assert.NotContains(t, out, "pre-clone-script")
assert.NotContains(t, out, "Cloning repository")
assert.NotContains(t, out, "Created fresh repository")
assert.NotContains(t, out, "Fetching changes")
assert.Contains(t, out, "Skipping Git repository setup")
})
......@@ -318,7 +319,7 @@ func TestBuildWithGitStrategyFetch(t *testing.T) {
out, err := runBuildReturningOutput(t, build)
assert.NoError(t, err)
assert.Contains(t, out, "Cloning repository")
assert.Contains(t, out, "Created fresh repository")
assert.Regexp(t, "Checking out [a-f0-9]+ as", out)
out, err = runBuildReturningOutput(t, build)
......@@ -342,7 +343,7 @@ func TestBuildWithGitStrategyFetchNoCheckout(t *testing.T) {
out, err := runBuildReturningOutput(t, build)
assert.NoError(t, err)
assert.Contains(t, out, "Cloning repository")
assert.Contains(t, out, "Created fresh repository")
assert.Contains(t, out, "Skipping Git checkout")
out, err = runBuildReturningOutput(t, build)
......@@ -365,11 +366,11 @@ func TestBuildWithGitStrategyClone(t *testing.T) {
out, err := runBuildReturningOutput(t, build)
assert.NoError(t, err)
assert.Contains(t, out, "Cloning repository")
assert.Contains(t, out, "Created fresh repository")
out, err = runBuildReturningOutput(t, build)
assert.NoError(t, err)
assert.Contains(t, out, "Cloning repository")
assert.Contains(t, out, "Created fresh repository")
assert.Regexp(t, "Checking out [a-f0-9]+ as", out)
assert.Contains(t, out, "pre-clone-script")
})
......@@ -388,11 +389,11 @@ func TestBuildWithGitStrategyCloneNoCheckout(t *testing.T) {
out, err := runBuildReturningOutput(t, build)
assert.NoError(t, err)
assert.Contains(t, out, "Cloning repository")
assert.Contains(t, out, "Created fresh repository")
out, err = runBuildReturningOutput(t, build)
assert.NoError(t, err)
assert.Contains(t, out, "Cloning repository")
assert.Contains(t, out, "Created fresh repository")
assert.Contains(t, out, "Skipping Git checkout")
assert.Contains(t, out, "pre-clone-script")
})
......@@ -504,7 +505,7 @@ func TestBuildWithGitSubmoduleStrategyRecursiveAndGitStrategyNone(t *testing.T)
out, err := runBuildReturningOutput(t, build)
assert.NoError(t, err)
assert.NotContains(t, out, "Cloning repository")
assert.NotContains(t, out, "Created fresh repository")
assert.NotContains(t, out, "Fetching changes")
assert.Contains(t, out, "Skipping Git repository setup")
assert.NotContains(t, out, "Updating/initializing submodules...")
......@@ -623,7 +624,7 @@ func TestBuildWithBrokenGitSSLCAInfo(t *testing.T) {
out, err := runBuildReturningOutput(t, build)
assert.Error(t, err)
assert.Contains(t, out, "Cloning repository")
assert.Contains(t, out, "Created fresh repository")
assert.NotContains(t, out, "Updating/initializing submodules")
})
}
......@@ -639,7 +640,7 @@ func TestBuildWithGoodGitSSLCAInfo(t *testing.T) {
out, err := runBuildReturningOutput(t, build)
assert.NoError(t, err)
assert.Contains(t, out, "Cloning repository")
assert.Contains(t, out, "Created fresh repository")
assert.Contains(t, out, "Updating/initializing submodules")
})
}
......@@ -657,7 +658,7 @@ func TestBuildWithGitSSLAndStrategyFetch(t *testing.T) {
out, err := runBuildReturningOutput(t, build)
assert.NoError(t, err)
assert.Contains(t, out, "Cloning repository")
assert.Contains(t, out, "Created fresh repository")
assert.Regexp(t, "Checking out [a-f0-9]+ as", out)
out, err = runBuildReturningOutput(t, build)
......@@ -767,3 +768,47 @@ func TestInteractiveTerminal(t *testing.T) {
})
}
}
// TODO: Remove in 12.0
func TestBuildNoRefspecs(t *testing.T) {
strategies := []string{"clone", "fetch", "none"}
expectedOutputs := map[string]string{
"clone": "Cloning repository...",
"fetch": "Fetching changes...",
"none": "Skipping Git repository setup",
}
onEachShell(t, func(t *testing.T, shell string) {
for _, strategy := range strategies {
t.Run(fmt.Sprintf("GIT_STRATEGY %s", strategy), func(t *testing.T) {
apiBuild, err := common.GetSuccessfulBuild()
assert.NoError(t, err)
apiBuild.GitInfo.Refspecs = []string{}
build, cleanup := newBuild(t, apiBuild, shell)
defer cleanup()
build.Variables = append(build.Variables, common.JobVariable{Key: "GIT_STRATEGY", Value: strategy})
expectedOutput, ok := expectedOutputs[strategy]
require.True(t, ok, "missing expectancies for %s strategy", strategy)
err = runBuild(t, build)
require.NoError(t, err)
// run twice because script behavior changes based on build directory existence
out, err := runBuildReturningOutput(t, build)
require.NoError(t, err)
require.Contains(t, out, expectedOutput)
for s, notExpected := range expectedOutputs {
if s == strategy {
continue
}
require.NotContains(t, out, notExpected)
}
})
}
})
}
......@@ -22,6 +22,7 @@ func (b *AbstractShell) GetFeatures(features *common.FeaturesInfo) {
features.UploadMultipleArtifacts = true
features.UploadRawArtifacts = true
features.Cache = true
features.Refspecs = true
}
func (b *AbstractShell) writeCdBuildDir(w ShellWriter, info common.ShellScriptInfo) {
......@@ -62,11 +63,11 @@ func (b *AbstractShell) writeGitSSLConfig(w ShellWriter, build *common.Build, wh
return
}
// TODO: Remove in 12.0
func (b *AbstractShell) writeCloneCmd(w ShellWriter, build *common.Build, projectDir string) {
templateDir := w.MkTmpDir("git-template")
args := []string{"clone", "--no-checkout", build.GetRemoteURL(), projectDir, "--template", templateDir}
w.RmDir(projectDir)
templateFile := path.Join(templateDir, "config")
w.Command("git", "config", "-f", templateFile, "fetch.recurseSubmodules", "false")
if build.IsSharedEnv() {
......@@ -84,6 +85,25 @@ func (b *AbstractShell) writeCloneCmd(w ShellWriter, build *common.Build, projec
w.Cd(projectDir)
}
func (b *AbstractShell) writeGitCleanup(w ShellWriter) {
// Remove .git/{index,shallow,HEAD}.lock files from .git, which can fail the fetch command
// The file can be left if previous build was terminated during git operation
w.RmFile(".git/index.lock")
w.RmFile(".git/shallow.lock")
w.RmFile(".git/HEAD.lock")
w.RmFile(".git/hooks/post-checkout")
w.Command("git", "clean", "-ffdx")
w.IfCmd("git", "diff", "--no-ext-diff", "--quiet", "--exit-code")
// git 1.7 cannot reset before a checkout, if no diffs we can avoid git reset
w.Print("Clean repository")
w.Else()
w.Command("git", "reset", "--hard")
w.EndIf()
}
// TODO: Remove in 12.0
func (b *AbstractShell) writeFetchCmd(w ShellWriter, build *common.Build, projectDir string, gitDir string) {
depth := build.GetGitDepth()
......@@ -100,18 +120,8 @@ func (b *AbstractShell) writeFetchCmd(w ShellWriter, build *common.Build, projec
b.writeGitSSLConfig(w, build, nil)
}
// Remove .git/{index,shallow,HEAD}.lock files from .git, which can fail the fetch command
// The file can be left if previous build was terminated during git operation
w.RmFile(".git/index.lock")
w.RmFile(".git/shallow.lock")
w.RmFile(".git/HEAD.lock")
b.writeGitCleanup(w)
w.IfFile(".git/hooks/post-checkout")
w.RmFile(".git/hooks/post-checkout")
w.EndIf()
w.Command("git", "clean", "-ffdx")
w.Command("git", "reset", "--hard")
w.Command("git", "remote", "set-url", "origin", build.GetRemoteURL())
if depth != "" {
var refspec string
......@@ -129,6 +139,45 @@ func (b *AbstractShell) writeFetchCmd(w ShellWriter, build *common.Build, projec
w.EndIf()
}
func (b *AbstractShell) writeRefspecFetchCmd(w ShellWriter, build *common.Build, projectDir string, gitDir string) {
depth := build.GitInfo.Depth
// initializing
templateDir := w.MkTmpDir("git-template")
templateFile := path.Join(templateDir, "config")
w.Command("git", "config", "-f", templateFile, "fetch.recurseSubmodules", "false")
if build.IsSharedEnv() {
b.writeGitSSLConfig(w, build, []string{"-f", templateFile})
}
w.Command("git", "init", projectDir, "--template", templateDir)
w.Cd(projectDir)
// fetching
if depth > 0 {
w.Notice("Fetching changes with git depth set to %d...", depth)
} else {
w.Notice("Fetching changes...")
}
// Add `git remote` or update existing
w.IfCmdWithOutput("git", "remote", "add", "origin", build.GetRemoteURL())
w.Notice("Created fresh repository.")
w.Else()
w.Command("git", "remote", "set-url", "origin", build.GetRemoteURL())
b.writeGitCleanup(w)
w.EndIf()
fetchArgs := []string{"fetch", "origin", "--prune"}
fetchArgs = append(fetchArgs, build.GitInfo.Refspecs...)
if depth > 0 {
fetchArgs = append(fetchArgs, "--depth", strconv.Itoa(depth))
}
w.Command("git", fetchArgs...)
}
func (b *AbstractShell) writeCheckoutCmd(w ShellWriter, build *common.Build) {
w.Notice("Checking out %s as %s...", build.GitInfo.Sha[0:8], build.GitInfo.Ref)
w.Command("git", "checkout", "-f", "-q", build.GitInfo.Sha)
......@@ -302,14 +351,28 @@ func (b *AbstractShell) writePrepareScript(w ShellWriter, info common.ShellScrip
func (b *AbstractShell) writeCloneFetchCmds(w ShellWriter, info common.ShellScriptInfo) (err error) {
build := info.Build
hasRefspecs := build.RefspecsAvailable()
projectDir := build.FullProjectDir()
gitDir := path.Join(build.FullProjectDir(), ".git")
if !hasRefspecs {
w.Warning("DEPRECATION: this GitLab server doesn't support refspecs, gitlab-runner 12.0 will no longer work with this version of GitLab")
}
switch info.Build.GetGitStrategy() {
case common.GitFetch:
b.writeFetchCmd(w, build, projectDir, gitDir)
if hasRefspecs {
b.writeRefspecFetchCmd(w, build, projectDir, gitDir)
} else {
b.writeFetchCmd(w, build, projectDir, gitDir)
}
case common.GitClone:
b.writeCloneCmd(w, build, projectDir)
w.RmDir(projectDir)
if hasRefspecs {
b.writeRefspecFetchCmd(w, build, projectDir, gitDir)
} else {
b.writeCloneCmd(w, build, projectDir)
}
case common.GitNone:
w.Notice("Skipping Git repository setup")
w.MkDir(projectDir)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment