Commit 2390c5df authored by Tomasz Maczukin's avatar Tomasz Maczukin 🌴

Remove legacy JobResponse stuct fields - step 11

parent 882a9370
......@@ -20,6 +20,7 @@ import (
_ "gitlab.com/gitlab-org/gitlab-ci-multi-runner/executors/shell"
_ "gitlab.com/gitlab-org/gitlab-ci-multi-runner/executors/ssh"
_ "gitlab.com/gitlab-org/gitlab-ci-multi-runner/executors/virtualbox"
"strconv"
)
type ExecCommand struct {
......@@ -55,15 +56,6 @@ func (c *ExecCommand) getCommands(commands interface{}) (string, error) {
return "", nil
}
func (c *ExecCommand) supportedOption(key string, _ interface{}) bool {
switch key {
case "image", "services", "artifacts", "cache", "after_script":
return true
default:
return false
}
}
func (c *ExecCommand) buildCommands(configBeforeScript, jobConfigBeforeScript, jobScript interface{}) (commands string, err error) {
// get before_script
beforeScript, err := c.getCommands(configBeforeScript)
......@@ -129,25 +121,6 @@ func (c *ExecCommand) buildGlobalAndJobVariables(global, job interface{}) (build
return
}
func (c *ExecCommand) buildOptions(config, jobConfig common.BuildOptions) (options common.BuildOptions, err error) {
options = make(common.BuildOptions)
// parse global options
for key, value := range config {
if c.supportedOption(key, value) {
options[key] = value
}
}
// parse job options
for key, value := range jobConfig {
if c.supportedOption(key, value) {
options[key] = value
}
}
return
}
func (c *ExecCommand) parseYaml(job string, build *common.JobResponse) error {
data, err := ioutil.ReadFile(".gitlab-ci.yml")
if err != nil {
......@@ -184,9 +157,37 @@ func (c *ExecCommand) parseYaml(job string, build *common.JobResponse) error {
return err
}
build.Options, err = c.buildOptions(config, jobConfig)
build.Image.Name = getOption("image", config, jobConfig)
services := []string(getOption("services", config, jobConfig))
for _, service := range services {
build.Services = append(build.Services, common.JRImage{
Name: service,
})
}
artifacts := getOption("artifacts", config, jobConfig)
untracked, err := strconv.ParseBool(artifacts["untracked"])
if err != nil {
return err
untracked = false
}
build.Artifacts[0] = common.JRArtifact{
Name: artifacts["name"],
Untracted: untracked,
Paths: artifacts["paths"],
When: artifacts["when"],
ExpireIn: artifacts["expireIn"],
}
cache := getOption("cache", config, jobConfig)
untracked, err = strconv.ParseBool(cache["untracked"])
if err != nil {
untracked = false
}
build.Cache[0] = common.JRCache{
Key: cache["key"],
Untracted: untracked,
Paths: cache["paths"],
}
if stage, ok := jobConfig.GetString("stage"); ok {
......@@ -197,6 +198,24 @@ func (c *ExecCommand) parseYaml(job string, build *common.JobResponse) error {
return nil
}
func getOption(optionKey string, config, jobConfig common.BuildOptions) interface{} {
// parse job options
for key, value := range jobConfig {
if key == optionKey {
return value
}
}
// parse global options
for key, value := range config {
if key == optionKey {
return value
}
}
return
}
func (c *ExecCommand) createBuild(repoURL string, abortSignal chan os.Signal) (build *common.Build, err error) {
// Check if we have uncommitted changes
_, err = c.runCommand("git", "diff", "--quiet", "HEAD")
......
......@@ -170,16 +170,20 @@ func (b *Build) executeStage(buildStage BuildStage, executor Executor, abort cha
}
func (b *Build) executeUploadArtifacts(state error, executor Executor, abort chan interface{}) (err error) {
when, _ := b.Options.GetString("artifacts", "when")
if len(b.Artifacts) < 1 {
return
}
when := JRArtifact(b.Artifacts[0]).When
if state == nil {
// Previous stages were successful
if when == "" || when == "on_success" || when == "always" {
if when == "" || when == ArtifactWhenOnSuccess || when == ArtifactWhenAlways {
err = b.executeStage(BuildStageUploadArtifacts, executor, abort)
}
} else {
// Previous stage did fail
if when == "on_failure" || when == "always" {
if when == ArtifactWhenOnFailure || when == ArtifactWhenAlways {
err = b.executeStage(BuildStageUploadArtifacts, executor, abort)
}
}
......
......@@ -13,16 +13,6 @@ func (m *MockShell) GetName() string {
return r0
}
func (m *MockShell) GetSupportedOptions() []string {
ret := m.Called()
var r0 []string
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
return r0
}
func (m *MockShell) GetFeatures(features *FeaturesInfo) {
m.Called(features)
}
......
......@@ -205,8 +205,7 @@ type JobResponse struct {
Dependencies JRDependencies `json:"dependencies"`
// TODO: LEGACY KEYS - TO BE REFACTORIZED WITH USAGE!!!
Commands string `json:"commands,omitempty"` // TODO: This introduces a change in scripts execution!
Options BuildOptions `json:"options"`
Commands string `json:"commands,omitempty"` // TODO: This introduces a change in scripts execution!
TLSCAChain string `json:"-"`
}
......
......@@ -47,7 +47,6 @@ type ShellScriptInfo struct {
type Shell interface {
GetName() string
GetSupportedOptions() []string
GetFeatures(features *FeaturesInfo)
IsDefault() bool
......
......@@ -1011,11 +1011,7 @@ func (s *executor) Prepare(globalConfig *common.Config, config *common.RunnerCon
return errors.New("Missing docker configuration")
}
err = build.Options.Decode(&s.options)
if err != nil {
return err
}
s.prepareOptions()
imageName, err := s.getImageName()
if err != nil {
return err
......@@ -1035,6 +1031,19 @@ func (s *executor) Prepare(globalConfig *common.Config, config *common.RunnerCon
return nil
}
func (s *executor) prepareOptions() {
s.options = dockerOptions{}
s.options.Image = s.Build.Image.Name
for _, service := range s.Build.Services {
serviceName := service.Name
if serviceName == "" {
continue
}
s.options.Services = append(s.options.Services, serviceName)
}
}
func (s *executor) prepareBuildsDir(config *common.RunnerConfig) error {
rootDir := config.BuildsDir
if rootDir == "" {
......
......@@ -75,8 +75,7 @@ func init() {
Type: common.NormalShell,
RunnerCommand: "/usr/bin/gitlab-runner-helper",
},
ShowHostname: true,
SupportedOptions: []string{"image", "services"},
ShowHostname: true,
}
creator := func() common.Executor {
......
......@@ -249,11 +249,18 @@ func TestDockerPrivilegedServiceAccessingBuildsFolder(t *testing.T) {
},
},
}
build.Commands = strings.Join(commands, "\n")
build.Options = common.BuildOptions{
"image": "docker:git",
"services": []string{
"docker:dind",
build.Steps = common.JRSteps{
common.JRStep{
Name: "script",
Script: common.JRStepScript(commands),
When: common.StepWhenOnSuccess,
AllowFailure: false,
},
}
build.Image.Name = "docker:git"
build.Services = common.JRServices{
common.JRImage{
Name: "docker:dind",
},
}
build.Variables = append(build.Variables, common.BuildVariable{
......
......@@ -94,8 +94,7 @@ func init() {
Type: common.LoginShell,
RunnerCommand: "gitlab-runner",
},
ShowHostname: true,
SupportedOptions: []string{"image", "services"},
ShowHostname: true,
}
creator := func() common.Executor {
......
......@@ -6,27 +6,12 @@ import (
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
)
var (
excludedOptions = []string{"environment"}
)
func isOptionExcluded(option string) bool {
for _, excluded := range excludedOptions {
if option == excluded {
return true
}
}
return false
}
type ExecutorOptions struct {
DefaultBuildsDir string
DefaultCacheDir string
SharedBuildsDir bool
Shell common.ShellScriptInfo
ShowHostname bool
SupportedOptions []string
}
type AbstractExecutor struct {
......@@ -80,37 +65,6 @@ func (e *AbstractExecutor) startBuild() error {
return nil
}
func (e *AbstractExecutor) verifyOptions() error {
supportedOptions := e.SupportedOptions
if shell := common.GetShell(e.Shell().Shell); shell != nil {
supportedOptions = append(supportedOptions, shell.GetSupportedOptions()...)
}
for key, value := range e.Build.Options {
if value == nil {
continue
}
if isOptionExcluded(key) {
delete(e.Build.Options, key)
continue
}
found := false
for _, option := range supportedOptions {
if option == key {
found = true
break
}
}
if !found {
e.Warningln(key, "is not supported by selected executor and shell")
}
}
return nil
}
func (e *AbstractExecutor) Shell() *common.ShellScriptInfo {
return &e.ExecutorOptions.Shell
}
......@@ -131,11 +85,6 @@ func (e *AbstractExecutor) Prepare(globalConfig *common.Config, config *common.R
return err
}
err = e.verifyOptions()
if err != nil {
return err
}
err = e.generateShellConfiguration()
if err != nil {
return err
......
......@@ -21,8 +21,7 @@ var (
Type: common.NormalShell,
RunnerCommand: "/usr/bin/gitlab-runner-helper",
},
ShowHostname: true,
SupportedOptions: []string{"image", "services", "artifacts", "cache"},
ShowHostname: true,
}
)
......@@ -293,7 +292,7 @@ func (s *executor) prepareOptions(job *common.Build) {
s.options = &kubernetesOptions{}
s.options.Image = job.Image.Name
for _, service := range job.Services {
serviceName := common.JRImage(service).Name
serviceName := service.Name
if serviceName == "" {
continue
}
......
......@@ -595,9 +595,7 @@ func TestKubernetesSuccessRun(t *testing.T) {
successfulBuild, err := common.GetRemoteSuccessfulBuild()
assert.NoError(t, err)
successfulBuild.Options = map[string]interface{}{
"image": "docker:git",
}
successfulBuild.Image.Name = "docker:git"
build := &common.Build{
JobResponse: successfulBuild,
Runner: &common.RunnerConfig{
......@@ -628,9 +626,7 @@ func TestKubernetesBuildFail(t *testing.T) {
},
},
}
build.Options = map[string]interface{}{
"image": "docker:git",
}
build.Image.Name = "docker:git"
err = build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
require.Error(t, err, "error")
......@@ -654,9 +650,7 @@ func TestKubernetesMissingImage(t *testing.T) {
},
},
}
build.Options = map[string]interface{}{
"image": "some/non-existing/image",
}
build.Image.Name = "some/non-existing/image"
err = build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
require.Error(t, err)
......@@ -680,9 +674,7 @@ func TestKubernetesMissingTag(t *testing.T) {
},
},
}
build.Options = map[string]interface{}{
"image": "docker:missing-tag",
}
build.Image.Name = "docker:missing-tag"
err = build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
require.Error(t, err)
......@@ -707,9 +699,7 @@ func TestKubernetesBuildAbort(t *testing.T) {
},
SystemInterrupt: make(chan os.Signal, 1),
}
build.Options = map[string]interface{}{
"image": "docker:git",
}
build.Image.Name = "docker:git"
abortTimer := time.AfterFunc(time.Second, func() {
t.Log("Interrupt")
......@@ -744,9 +734,7 @@ func TestKubernetesBuildCancel(t *testing.T) {
},
SystemInterrupt: make(chan os.Signal, 1),
}
build.Options = map[string]interface{}{
"image": "docker:git",
}
build.Image.Name = "docker:git"
trace := &common.Trace{Writer: os.Stdout, Abort: make(chan interface{}, 1)}
......@@ -794,9 +782,7 @@ func TestOverwriteNamespaceNotMatch(t *testing.T) {
},
SystemInterrupt: make(chan os.Signal, 1),
}
build.Options = map[string]interface{}{
"image": "docker:git",
}
build.Image.Name = "docker:git"
err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
require.Error(t, err)
......
package shells
import (
"errors"
"path"
"path/filepath"
"strconv"
"strings"
"errors"
"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
)
......@@ -18,10 +18,6 @@ func (b *AbstractShell) GetFeatures(features *common.FeaturesInfo) {
features.Cache = true
}
func (b *AbstractShell) GetSupportedOptions() []string {
return []string{"artifacts", "cache", "dependencies", "after_script"}
}
func (b *AbstractShell) writeCdBuildDir(w ShellWriter, info common.ShellScriptInfo) {
w.Cd(info.Build.FullProjectDir())
}
......@@ -149,17 +145,6 @@ func (b *AbstractShell) cacheFile(build *common.Build, userKey string) (key, fil
return
}
func (o *archivingOptions) CommandArguments() (args []string) {
for _, path := range o.Paths {
args = append(args, "--path", path)
}
if o.Untracked {
args = append(args, "--untracked")
}
return
}
func (b *AbstractShell) guardRunnerCommand(w ShellWriter, runnerCommand string, action string, f func()) {
if runnerCommand == "" {
w.Warning("%s is not supported by this executor.", action)
......@@ -173,18 +158,30 @@ func (b *AbstractShell) guardRunnerCommand(w ShellWriter, runnerCommand string,
w.EndIf()
}
func (b *AbstractShell) cacheExtractor(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
if options == nil {
func (b *AbstractShell) cacheExtractor(w ShellWriter, info common.ShellScriptInfo) {
if len(info.Build.Cache) < 1 {
return
}
cacheOptions := info.Build.Cache[0]
// Create list of files to archive
var archiverArgs []string
for _, path := range cacheOptions.Paths {
archiverArgs = append(archiverArgs, "--path", path)
}
if cacheOptions.Untracted {
archiverArgs = append(archiverArgs, "--untracked")
}
// Skip restoring cache if no cache is defined
if archiverArgs := options.CommandArguments(); len(archiverArgs) == 0 {
if len(archiverArgs) < 1 {
return
}
// Skip archiving if no cache is defined
cacheKey, cacheFile := b.cacheFile(info.Build, options.Key)
cacheKey, cacheFile := b.cacheFile(info.Build, cacheOptions.Key)
if cacheKey == "" {
return
}
......@@ -318,19 +315,12 @@ func (b *AbstractShell) writeGetSourcesScript(w ShellWriter, info common.ShellSc
}
func (b *AbstractShell) writeRestoreCacheScript(w ShellWriter, info common.ShellScriptInfo) (err error) {
// Parse options
var options shellOptions
err = info.Build.Options.Decode(&options)
if err != nil {
return
}
b.writeExports(w, info)
b.writeCdBuildDir(w, info)
b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")
// Try to restore from main cache, if not found cache for master
b.cacheExtractor(w, options.Cache, info)
b.cacheExtractor(w, info)
return nil
}
......@@ -377,13 +367,15 @@ func (b *AbstractShell) writeUserScript(w ShellWriter, info common.ShellScriptIn
return nil
}
func (b *AbstractShell) cacheArchiver(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
if options == nil {
func (b *AbstractShell) cacheArchiver(w ShellWriter, info common.ShellScriptInfo) {
if len(info.Build.Cache) < 1 {
return
}
cacheOptions := info.Build.Cache[0]
// Skip archiving if no cache is defined
cacheKey, cacheFile := b.cacheFile(info.Build, options.Key)
cacheKey, cacheFile := b.cacheFile(info.Build, cacheOptions.Key)
if cacheKey == "" {
return
}
......@@ -394,8 +386,16 @@ func (b *AbstractShell) cacheArchiver(w ShellWriter, options *archivingOptions,
}
// Create list of files to archive
archiverArgs := options.CommandArguments()
if len(archiverArgs) == 0 {
var archiverArgs []string
for _, path := range cacheOptions.Paths {
archiverArgs = append(archiverArgs, "--path", path)
}
if cacheOptions.Untracted {
archiverArgs = append(archiverArgs, "--untracked")
}
if len(archiverArgs) < 1 {
// Skip creating archive
return
}
......@@ -417,8 +417,8 @@ func (b *AbstractShell) cacheArchiver(w ShellWriter, options *archivingOptions,
})
}
func (b *AbstractShell) uploadArtifacts(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
if options == nil {
func (b *AbstractShell) uploadArtifacts(w ShellWriter, info common.ShellScriptInfo) {
if len(info.Build.Artifacts) < 1 {
return
}
if info.Build.Runner.URL == "" {
......@@ -435,22 +435,30 @@ func (b *AbstractShell) uploadArtifacts(w ShellWriter, options *archivingOptions
strconv.Itoa(info.Build.ID),
}
artifacts := info.Build.Artifacts[0]
// Create list of files to archive
archiverArgs := options.CommandArguments()
if len(archiverArgs) == 0 {
var archiverArgs []string
for _, path := range artifacts.Paths {
archiverArgs = append(archiverArgs, "--path", path)
}
if artifacts.Untracted {
archiverArgs = append(archiverArgs, "--untracked")
}
if len(archiverArgs) < 1 {
// Skip creating archive
return
}
args = append(args, archiverArgs...)
// Get artifacts:name
if name, ok := info.Build.Options.GetString("artifacts", "name"); ok && name != "" {
args = append(args, "--name", name)
if artifacts.Name != "" {
args = append(args, "--name", artifacts.Name)
}
// Get artifacts:expire_in
if expireIn, ok := info.Build.Options.GetString("artifacts", "expire_in"); ok && expireIn != "" {
args = append(args, "--expire-in", expireIn)
if artifacts.ExpireIn != "" {
args = append(args, "--expire-in", artifacts.ExpireIn)
}
b.guardRunnerCommand(w, info.RunnerCommand, "Uploading artifacts", func() {
......@@ -460,15 +468,15 @@ func (b *AbstractShell) uploadArtifacts(w ShellWriter, options *archivingOptions
}
func (b *AbstractShell) writeAfterScript(w ShellWriter, info common.ShellScriptInfo) error {
shellOptions := struct {
AfterScript []string `json:"after_script"`
}{}
err := info.Build.Options.Decode(&shellOptions)
if err != nil {
return err
var afterScriptStep *common.JRStep
for _, step := range info.Build.Steps {
if step.Name != "after_script" {
continue
}
afterScriptStep = &step
}
if len(shellOptions.AfterScript) == 0 {
if afterScriptStep == nil {
return nil
}
......@@ -477,7 +485,7 @@ func (b *AbstractShell) writeAfterScript(w ShellWriter, info common.ShellScriptI
w.Notice("Running after script...")
for _, command := range shellOptions.AfterScript {
for _, command := range afterScriptStep.Script {
command = strings.TrimSpace(command)
if command != "" {
w.Notice("$ %s", command)
......@@ -492,36 +500,22 @@ func (b *AbstractShell) writeAfterScript(w ShellWriter, info common.ShellScriptI
}
func (b *AbstractShell) writeArchiveCacheScript(w ShellWriter, info common.ShellScriptInfo) (err error) {
// Parse options
var options shellOptions
err = info.Build.Options.Decode(&options)
if err != nil {
return
}
b.writeExports(w, info)
b.writeCdBuildDir(w, info)
b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")
// Find cached files and archive them
b.cacheArchiver(w, options.Cache, info)
b.cacheArchiver(w, info)
return
}
func (b *AbstractShell) writeUploadArtifactsScript(w ShellWriter, info common.ShellScriptInfo) (err error) {
// Parse options
var options shellOptions
err = info.Build.Options.Decode(&options)
if err != nil {
return
}
b.writeExports(w, info)
b.writeCdBuildDir(w, info)
b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")
// Upload artifacts
b.uploadArtifacts(w, options.Artifacts, info)
b.uploadArtifacts(w, info)
return
}
......
package shells
type archivingOptions struct {
Untracked bool `json:"untracked"`
Paths []string `json:"paths"`
Name string `json:"name"`
Key string `json:"key"`
}
type shellOptions struct {
Cache *archivingOptions `json:"cache"`
Artifacts *archivingOptions `json:"artifacts"`
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment