build.go 13.2 KB
Newer Older
Kamil Trzciński's avatar
Kamil Trzciński committed
1
package common
Kamil Trzciński's avatar
WIP  
Kamil Trzciński committed
2 3

import (
4
	"context"
Kamil Trzciński's avatar
Kamil Trzciński committed
5
	"errors"
Kamil Trzciński's avatar
WIP  
Kamil Trzciński committed
6
	"fmt"
7
	"net/url"
8
	"os"
9
	"path"
10
	"strconv"
11
	"strings"
12
	"time"
13

14
	"github.com/Sirupsen/logrus"
15
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
Kamil Trzciński's avatar
WIP  
Kamil Trzciński committed
16 17
)

18 19 20 21 22
type GitStrategy int

const (
	GitClone GitStrategy = iota
	GitFetch
Nick Thomas's avatar
Nick Thomas committed
23
	GitNone
24 25
)

26 27 28
type SubmoduleStrategy int

const (
29 30
	SubmoduleInvalid SubmoduleStrategy = iota
	SubmoduleNone
31 32 33 34
	SubmoduleNormal
	SubmoduleRecursive
)

35 36 37 38
type BuildRuntimeState string

const (
	BuildRunStatePending      BuildRuntimeState = "pending"
39 40 41 42 43
	BuildRunRuntimeRunning    BuildRuntimeState = "running"
	BuildRunRuntimeFinished   BuildRuntimeState = "finished"
	BuildRunRuntimeCanceled   BuildRuntimeState = "canceled"
	BuildRunRuntimeTerminated BuildRuntimeState = "terminated"
	BuildRunRuntimeTimedout   BuildRuntimeState = "timedout"
44 45 46 47
)

type BuildStage string

Kamil Trzciński's avatar
Kamil Trzciński committed
48
const (
49
	BuildStagePrepare           BuildStage = "prepare_script"
50 51 52 53 54 55 56
	BuildStageGetSources        BuildStage = "get_sources"
	BuildStageRestoreCache      BuildStage = "restore_cache"
	BuildStageDownloadArtifacts BuildStage = "download_artifacts"
	BuildStageUserScript        BuildStage = "build_script"
	BuildStageAfterScript       BuildStage = "after_script"
	BuildStageArchiveCache      BuildStage = "archive_cache"
	BuildStageUploadArtifacts   BuildStage = "upload_artifacts"
Kamil Trzciński's avatar
Kamil Trzciński committed
57 58
)

Kamil Trzciński's avatar
WIP  
Kamil Trzciński committed
59
type Build struct {
60
	JobResponse `yaml:",inline"`
61

62 63 64 65 66 67 68
	SystemInterrupt chan os.Signal `json:"-" yaml:"-"`
	RootDir         string         `json:"-" yaml:"-"`
	BuildDir        string         `json:"-" yaml:"-"`
	CacheDir        string         `json:"-" yaml:"-"`
	Hostname        string         `json:"-" yaml:"-"`
	Runner          *RunnerConfig  `json:"runner"`
	ExecutorData    ExecutorData
69

70
	// Unique ID for all running builds on this runner
Kamil Trzciński's avatar
Kamil Trzciński committed
71
	RunnerID int `json:"runner_id"`
72

73 74
	// Unique ID for all running builds on this runner and this project
	ProjectRunnerID int `json:"project_runner_id"`
75

76
	CurrentStage BuildStage
Kamil Trzciński's avatar
Kamil Trzciński committed
77
	CurrentState BuildRuntimeState
78 79

	executorStageResolver func() ExecutorStage
Kamil Trzciński's avatar
WIP  
Kamil Trzciński committed
80 81
}

82
func (b *Build) Log() *logrus.Entry {
83
	return b.Runner.Log().WithField("job", b.ID).WithField("project", b.JobInfo.ProjectID)
84 85
}

Kamil Trzciński's avatar
Kamil Trzciński committed
86
func (b *Build) ProjectUniqueName() string {
87
	return fmt.Sprintf("runner-%s-project-%d-concurrent-%d",
88
		b.Runner.ShortDescription(), b.JobInfo.ProjectID, b.ProjectRunnerID)
89 90
}

91
func (b *Build) ProjectSlug() (string, error) {
92
	url, err := url.Parse(b.GitInfo.RepoURL)
93 94 95 96 97
	if err != nil {
		return "", err
	}
	if url.Host == "" {
		return "", errors.New("only URI reference supported")
98 99
	}

100
	slug := url.Path
101
	slug = strings.TrimSuffix(slug, ".git")
102
	slug = path.Clean(slug)
103 104 105 106 107 108
	if slug == "." {
		return "", errors.New("invalid path")
	}
	if strings.Contains(slug, "..") {
		return "", errors.New("it doesn't look like a valid path")
	}
109
	return slug, nil
Kamil Trzciński's avatar
WIP  
Kamil Trzciński committed
110 111
}

112 113 114
func (b *Build) ProjectUniqueDir(sharedDir bool) string {
	dir, err := b.ProjectSlug()
	if err != nil {
115
		dir = fmt.Sprintf("project-%d", b.JobInfo.ProjectID)
116 117 118 119
	}

	// for shared dirs path is constructed like this:
	// <some-path>/runner-short-id/concurrent-id/group-name/project-name/
120
	// ex.<some-path>/01234567/0/group/repo/
121
	if sharedDir {
122
		dir = path.Join(
123
			fmt.Sprintf("%s", b.Runner.ShortDescription()),
124 125 126 127 128 129 130
			fmt.Sprintf("%d", b.ProjectRunnerID),
			dir,
		)
	}
	return dir
}

131
func (b *Build) FullProjectDir() string {
132
	return helpers.ToSlash(b.BuildDir)
Kamil Trzciński's avatar
Kamil Trzciński committed
133
}
Kamil Trzciński's avatar
Kamil Trzciński committed
134

135
func (b *Build) StartBuild(rootDir, cacheDir string, sharedDir bool) {
136
	b.RootDir = rootDir
137 138
	b.BuildDir = path.Join(rootDir, b.ProjectUniqueDir(sharedDir))
	b.CacheDir = path.Join(cacheDir, b.ProjectUniqueDir(false))
139 140
}

141
func (b *Build) executeStage(ctx context.Context, buildStage BuildStage, executor Executor) error {
142
	b.CurrentStage = buildStage
143

144 145 146 147 148
	shell := executor.Shell()
	if shell == nil {
		return errors.New("No shell defined")
	}

149
	script, err := GenerateShellScript(buildStage, *shell)
150 151 152 153 154 155 156 157 158 159
	if err != nil {
		return err
	}

	// Nothing to execute
	if script == "" {
		return nil
	}

	cmd := ExecutorCommand{
160
		Context: ctx,
Kamil Trzciński's avatar
Kamil Trzciński committed
161
		Script:  script,
162 163
	}

164 165
	switch buildStage {
	case BuildStageUserScript, BuildStageAfterScript: // use custom build environment
166 167 168 169 170 171 172 173
		cmd.Predefined = false
	default: // all other stages use a predefined build environment
		cmd.Predefined = true
	}

	return executor.Run(cmd)
}

174
func (b *Build) executeUploadArtifacts(ctx context.Context, state error, executor Executor) (err error) {
175 176 177 178 179 180 181
	jobState := state

	for _, artifacts := range b.Artifacts {
		when := artifacts.When
		if state == nil {
			// Previous stages were successful
			if when == "" || when == ArtifactWhenOnSuccess || when == ArtifactWhenAlways {
182
				state = b.executeStage(ctx, BuildStageUploadArtifacts, executor)
183 184 185 186
			}
		} else {
			// Previous stage did fail
			if when == ArtifactWhenOnFailure || when == ArtifactWhenAlways {
187
				err = b.executeStage(ctx, BuildStageUploadArtifacts, executor)
188
			}
189 190 191
		}
	}

192 193 194
	// Use job's error if set
	if jobState != nil {
		err = jobState
195 196 197 198
	}
	return
}

199
func (b *Build) executeScript(ctx context.Context, executor Executor) error {
200
	// Prepare stage
201
	err := b.executeStage(ctx, BuildStagePrepare, executor)
202

203
	if err == nil {
204
		err = b.attemptExecuteStage(ctx, BuildStageGetSources, executor, b.GetGetSourcesAttempts())
205 206
	}
	if err == nil {
207
		err = b.attemptExecuteStage(ctx, BuildStageRestoreCache, executor, b.GetRestoreCacheAttempts())
208 209
	}
	if err == nil {
210
		err = b.attemptExecuteStage(ctx, BuildStageDownloadArtifacts, executor, b.GetDownloadArtifactsAttempts())
211 212
	}

213
	if err == nil {
214
		// Execute user build script (before_script + script)
215
		err = b.executeStage(ctx, BuildStageUserScript, executor)
216 217

		// Execute after script (after_script)
218 219 220 221
		timeoutContext, timeoutCancel := context.WithTimeout(ctx, AfterScriptTimeout)
		defer timeoutCancel()

		b.executeStage(timeoutContext, BuildStageAfterScript, executor)
222 223 224 225
	}

	// Execute post script (cache store, artifacts upload)
	if err == nil {
226
		err = b.executeStage(ctx, BuildStageArchiveCache, executor)
227
	}
228
	err = b.executeUploadArtifacts(ctx, err, executor)
229 230 231
	return err
}

232
func (b *Build) attemptExecuteStage(ctx context.Context, buildStage BuildStage, executor Executor, attempts int) (err error) {
233 234 235 236
	if attempts < 1 || attempts > 10 {
		return fmt.Errorf("Number of attempts out of the range [1, 10] for stage: %s", buildStage)
	}
	for attempt := 0; attempt < attempts; attempt++ {
237
		if err = b.executeStage(ctx, buildStage, executor); err == nil {
238 239 240 241 242 243
			return
		}
	}
	return
}

244
func (b *Build) GetBuildTimeout() time.Duration {
245
	buildTimeout := b.RunnerInfo.Timeout
246 247 248
	if buildTimeout <= 0 {
		buildTimeout = DefaultTimeout
	}
249 250 251
	return time.Duration(buildTimeout) * time.Second
}

252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267
func (b *Build) handleError(err error) error {
	switch err {
	case context.Canceled:
		b.CurrentState = BuildRunRuntimeCanceled
		return &BuildError{Inner: errors.New("canceled")}

	case context.DeadlineExceeded:
		b.CurrentState = BuildRunRuntimeTimedout
		return &BuildError{Inner: fmt.Errorf("execution took longer than %v seconds", b.GetBuildTimeout())}

	default:
		b.CurrentState = BuildRunRuntimeFinished
		return err
	}
}

268 269
func (b *Build) run(ctx context.Context, executor Executor) (err error) {
	b.CurrentState = BuildRunRuntimeRunning
270

271
	buildFinish := make(chan error, 1)
272 273 274

	runContext, runCancel := context.WithCancel(context.Background())
	defer runCancel()
275 276 277

	// Run build script
	go func() {
278
		buildFinish <- b.executeScript(runContext, executor)
279 280 281
	}()

	// Wait for signals: cancel, timeout, abort or finish
282
	b.Log().Debugln("Waiting for signals...")
283
	select {
284
	case <-ctx.Done():
285
		err = b.handleError(ctx.Err())
286

287
	case signal := <-b.SystemInterrupt:
288
		err = fmt.Errorf("aborted: %v", signal)
289
		b.CurrentState = BuildRunRuntimeTerminated
290 291

	case err = <-buildFinish:
292
		b.CurrentState = BuildRunRuntimeFinished
293 294 295
		return err
	}

Kamil Trzciński's avatar
Kamil Trzciński committed
296
	b.Log().WithError(err).Debugln("Waiting for build to finish...")
297

298
	// Wait till we receive that build did finish
299 300 301
	runCancel()
	<-buildFinish
	return err
302 303
}

304
func (b *Build) retryCreateExecutor(options ExecutorPrepareOptions, provider ExecutorProvider, logger BuildLogger) (executor Executor, err error) {
305
	for tries := 0; tries < PreparationRetries; tries++ {
306 307 308 309 310 311
		executor = provider.Create()
		if executor == nil {
			err = errors.New("failed to create executor")
			return
		}

312 313
		b.executorStageResolver = executor.GetCurrentStage

314
		err = executor.Prepare(options)
315 316 317 318 319 320 321
		if err == nil {
			break
		}
		if executor != nil {
			executor.Cleanup()
			executor = nil
		}
322 323
		if _, ok := err.(*BuildError); ok {
			break
324 325
		} else if options.Context.Err() != nil {
			return nil, b.handleError(options.Context.Err())
326
		}
327

328 329 330 331 332 333 334
		logger.SoftErrorln("Preparation failed:", err)
		logger.Infoln("Will be retried in", PreparationRetryInterval, "...")
		time.Sleep(PreparationRetryInterval)
	}
	return
}

335 336 337 338 339 340 341 342 343 344
func (b *Build) CurrentExecutorStage() ExecutorStage {
	if b.executorStageResolver == nil {
		b.executorStageResolver = func() ExecutorStage {
			return ExecutorStage("")
		}
	}

	return b.executorStageResolver()
}

Tomasz Maczukin's avatar
Tomasz Maczukin committed
345
func (b *Build) Run(globalConfig *Config, trace JobTrace) (err error) {
346 347
	var executor Executor

Kamil Trzciński's avatar
Kamil Trzciński committed
348
	logger := NewBuildLogger(trace, b.Log())
349
	logger.Println(fmt.Sprintf("Running with %s\n  on %s (%s)", AppVersion.Line(), b.Runner.Name, b.Runner.ShortDescription()))
Kamil Trzciński's avatar
Kamil Trzciński committed
350

351
	b.CurrentState = BuildRunStatePending
352

353
	defer func() {
Kamil Trzciński's avatar
Kamil Trzciński committed
354
		if _, ok := err.(*BuildError); ok {
355
			logger.SoftErrorln("Job failed:", err)
Kamil Trzciński's avatar
Kamil Trzciński committed
356 357
			trace.Fail(err)
		} else if err != nil {
358
			logger.Errorln("Job failed (system failure):", err)
359
			trace.Fail(err)
360
		} else {
361
			logger.Infoln("Job succeeded")
362
			trace.Success()
363
		}
364 365 366
		if executor != nil {
			executor.Cleanup()
		}
367
	}()
368

369 370 371
	context, cancel := context.WithTimeout(context.Background(), b.GetBuildTimeout())
	defer cancel()

372 373
	trace.SetCancelFunc(cancel)

374
	options := ExecutorPrepareOptions{
Kamil Trzciński's avatar
Kamil Trzciński committed
375 376
		Config:  b.Runner,
		Build:   b,
377
		Trace:   trace,
Kamil Trzciński's avatar
Kamil Trzciński committed
378
		User:    globalConfig.User,
379 380
		Context: context,
	}
381

382
	provider := GetExecutor(b.Runner.Executor)
Kamil Trzciński's avatar
Kamil Trzciński committed
383
	if provider == nil {
384 385 386
		return errors.New("executor not found")
	}

387
	executor, err = b.retryCreateExecutor(options, provider, logger)
Kamil Trzciński's avatar
Kamil Trzciński committed
388
	if err == nil {
389
		err = b.run(context, executor)
Kamil Trzciński's avatar
Kamil Trzciński committed
390
	}
391 392 393
	if executor != nil {
		executor.Finish(err)
	}
Kamil Trzciński's avatar
Kamil Trzciński committed
394 395
	return err
}
396 397 398 399

func (b *Build) String() string {
	return helpers.ToYAML(b)
}
400

401 402
func (b *Build) GetDefaultVariables() JobVariables {
	return JobVariables{
403 404
		{Key: "CI_PROJECT_DIR", Value: b.FullProjectDir(), Public: true, Internal: true, File: false},
		{Key: "CI_SERVER", Value: "yes", Public: true, Internal: true, File: false},
405 406 407
	}
}

408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
func (b *Build) GetCITLSVariables() JobVariables {
	variables := JobVariables{}
	if b.TLSCAChain != "" {
		variables = append(variables, JobVariable{"CI_SERVER_TLS_CA_FILE", b.TLSCAChain, true, true, true})
	}
	if b.TLSAuthCert != "" && b.TLSAuthKey != "" {
		variables = append(variables, JobVariable{"CI_SERVER_TLS_CERT_FILE", b.TLSAuthCert, true, true, true})
		variables = append(variables, JobVariable{"CI_SERVER_TLS_KEY_FILE", b.TLSAuthKey, true, true, true})
	}
	return variables
}

func (b *Build) GetGitTLSVariables() JobVariables {
	variables := JobVariables{}
	if b.TLSCAChain != "" {
		variables = append(variables, JobVariable{"GIT_SSL_CAINFO", b.TLSCAChain, true, true, true})
	}
	if b.TLSAuthCert != "" && b.TLSAuthKey != "" {
		variables = append(variables, JobVariable{"GIT_SSL_CERT", b.TLSAuthCert, true, true, true})
		variables = append(variables, JobVariable{"GIT_SSL_KEY", b.TLSAuthKey, true, true, true})
	}
	return variables
}

432
func (b *Build) GetAllVariables() (variables JobVariables) {
433 434 435
	if b.Runner != nil {
		variables = append(variables, b.Runner.GetVariables()...)
	}
436
	variables = append(variables, b.GetDefaultVariables()...)
437
	variables = append(variables, b.GetCITLSVariables()...)
438
	variables = append(variables, b.Variables...)
Kamil Trzciński's avatar
Kamil Trzciński committed
439
	return variables.Expand()
440
}
441 442 443 444

func (b *Build) GetGitDepth() string {
	return b.GetAllVariables().Get("GIT_DEPTH")
}
445 446 447 448 449 450 451 452 453

func (b *Build) GetGitStrategy() GitStrategy {
	switch b.GetAllVariables().Get("GIT_STRATEGY") {
	case "clone":
		return GitClone

	case "fetch":
		return GitFetch

Nick Thomas's avatar
Nick Thomas committed
454 455 456
	case "none":
		return GitNone

457 458 459 460 461 462 463 464
	default:
		if b.AllowGitFetch {
			return GitFetch
		}

		return GitClone
	}
}
465

466
func (b *Build) GetSubmoduleStrategy() SubmoduleStrategy {
467 468 469
	if b.GetGitStrategy() == GitNone {
		return SubmoduleNone
	}
470 471 472 473 474 475 476 477 478 479 480 481
	switch b.GetAllVariables().Get("GIT_SUBMODULE_STRATEGY") {
	case "normal":
		return SubmoduleNormal

	case "recursive":
		return SubmoduleRecursive

	case "none", "":
		// Default (legacy) behavior is to not update/init submodules
		return SubmoduleNone

	default:
482 483
		// Will cause an error in AbstractShell) writeSubmoduleUpdateCmds
		return SubmoduleInvalid
484 485 486
	}
}

487 488 489 490 491 492 493 494
func (b *Build) IsDebugTraceEnabled() bool {
	trace, err := strconv.ParseBool(b.GetAllVariables().Get("CI_DEBUG_TRACE"))
	if err != nil {
		return false
	}

	return trace
}
495

496
func (b *Build) GetDockerAuthConfig() string {
Tomasz Maczukin's avatar
Tomasz Maczukin committed
497
	return b.GetAllVariables().Get("DOCKER_AUTH_CONFIG")
498
}
499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522

func (b *Build) GetGetSourcesAttempts() int {
	retries, err := strconv.Atoi(b.GetAllVariables().Get("GET_SOURCES_ATTEMPTS"))
	if err != nil {
		return DefaultGetSourcesAttempts
	}
	return retries
}

func (b *Build) GetDownloadArtifactsAttempts() int {
	retries, err := strconv.Atoi(b.GetAllVariables().Get("ARTIFACT_DOWNLOAD_ATTEMPTS"))
	if err != nil {
		return DefaultArtifactDownloadAttempts
	}
	return retries
}

func (b *Build) GetRestoreCacheAttempts() int {
	retries, err := strconv.Atoi(b.GetAllVariables().Get("RESTORE_CACHE_ATTEMPTS"))
	if err != nil {
		return DefaultRestoreCacheAttempts
	}
	return retries
}