build.go 13.7 KB
Newer Older
Kamil Trzciński's avatar
Kamil Trzciński committed
1
package common
Kamil Trzciński's avatar
Kamil Trzciński committed
2 3

import (
4
	"context"
5
	"errors"
Kamil Trzciński's avatar
Kamil Trzciński committed
6
	"fmt"
7
	"net/url"
8
	"os"
9
	"path"
10
	"strconv"
11
	"strings"
12
	"time"
13

14
	"github.com/Sirupsen/logrus"
15

16
	"gitlab.com/gitlab-org/gitlab-runner/helpers"
Kamil Trzciński's avatar
Kamil Trzciński committed
17 18
)

19 20 21 22 23
type GitStrategy int

const (
	GitClone GitStrategy = iota
	GitFetch
24
	GitNone
25 26
)

27 28 29
type SubmoduleStrategy int

const (
30 31
	SubmoduleInvalid SubmoduleStrategy = iota
	SubmoduleNone
32 33 34 35
	SubmoduleNormal
	SubmoduleRecursive
)

36 37 38 39
type BuildRuntimeState string

const (
	BuildRunStatePending      BuildRuntimeState = "pending"
40 41 42 43 44
	BuildRunRuntimeRunning    BuildRuntimeState = "running"
	BuildRunRuntimeFinished   BuildRuntimeState = "finished"
	BuildRunRuntimeCanceled   BuildRuntimeState = "canceled"
	BuildRunRuntimeTerminated BuildRuntimeState = "terminated"
	BuildRunRuntimeTimedout   BuildRuntimeState = "timedout"
45 46 47 48
)

type BuildStage string

Kamil Trzciński's avatar
Kamil Trzciński committed
49
const (
50
	BuildStagePrepare           BuildStage = "prepare_script"
51 52 53 54 55 56 57
	BuildStageGetSources        BuildStage = "get_sources"
	BuildStageRestoreCache      BuildStage = "restore_cache"
	BuildStageDownloadArtifacts BuildStage = "download_artifacts"
	BuildStageUserScript        BuildStage = "build_script"
	BuildStageAfterScript       BuildStage = "after_script"
	BuildStageArchiveCache      BuildStage = "archive_cache"
	BuildStageUploadArtifacts   BuildStage = "upload_artifacts"
Kamil Trzciński's avatar
Kamil Trzciński committed
58 59
)

Kamil Trzciński's avatar
Kamil Trzciński committed
60
type Build struct {
61
	JobResponse `yaml:",inline"`
62

63 64 65 66 67 68 69
	SystemInterrupt chan os.Signal `json:"-" yaml:"-"`
	RootDir         string         `json:"-" yaml:"-"`
	BuildDir        string         `json:"-" yaml:"-"`
	CacheDir        string         `json:"-" yaml:"-"`
	Hostname        string         `json:"-" yaml:"-"`
	Runner          *RunnerConfig  `json:"runner"`
	ExecutorData    ExecutorData
70

71
	// Unique ID for all running builds on this runner
Kamil Trzciński's avatar
Kamil Trzciński committed
72
	RunnerID int `json:"runner_id"`
73

74 75
	// Unique ID for all running builds on this runner and this project
	ProjectRunnerID int `json:"project_runner_id"`
76

77
	CurrentStage BuildStage
Kamil Trzciński's avatar
Kamil Trzciński committed
78
	CurrentState BuildRuntimeState
79 80

	executorStageResolver func() ExecutorStage
81
	logger                BuildLogger
Kamil Trzciński's avatar
Kamil Trzciński committed
82 83
}

84
func (b *Build) Log() *logrus.Entry {
85
	return b.Runner.Log().WithField("job", b.ID).WithField("project", b.JobInfo.ProjectID)
86 87
}

Kamil Trzciński's avatar
Kamil Trzciński committed
88
func (b *Build) ProjectUniqueName() string {
89
	return fmt.Sprintf("runner-%s-project-%d-concurrent-%d",
90
		b.Runner.ShortDescription(), b.JobInfo.ProjectID, b.ProjectRunnerID)
91 92
}

93
func (b *Build) ProjectSlug() (string, error) {
94
	url, err := url.Parse(b.GitInfo.RepoURL)
95 96 97 98 99
	if err != nil {
		return "", err
	}
	if url.Host == "" {
		return "", errors.New("only URI reference supported")
100 101
	}

102
	slug := url.Path
103
	slug = strings.TrimSuffix(slug, ".git")
104
	slug = path.Clean(slug)
105 106 107 108 109 110
	if slug == "." {
		return "", errors.New("invalid path")
	}
	if strings.Contains(slug, "..") {
		return "", errors.New("it doesn't look like a valid path")
	}
111
	return slug, nil
Kamil Trzciński's avatar
Kamil Trzciński committed
112 113
}

114 115 116
func (b *Build) ProjectUniqueDir(sharedDir bool) string {
	dir, err := b.ProjectSlug()
	if err != nil {
117
		dir = fmt.Sprintf("project-%d", b.JobInfo.ProjectID)
118 119 120 121
	}

	// for shared dirs path is constructed like this:
	// <some-path>/runner-short-id/concurrent-id/group-name/project-name/
122
	// ex.<some-path>/01234567/0/group/repo/
123
	if sharedDir {
124
		dir = path.Join(
125
			fmt.Sprintf("%s", b.Runner.ShortDescription()),
126 127 128 129 130 131 132
			fmt.Sprintf("%d", b.ProjectRunnerID),
			dir,
		)
	}
	return dir
}

133
func (b *Build) FullProjectDir() string {
134
	return helpers.ToSlash(b.BuildDir)
Kamil Trzciński's avatar
Kamil Trzciński committed
135
}
136

137
func (b *Build) StartBuild(rootDir, cacheDir string, sharedDir bool) {
138
	b.RootDir = rootDir
139 140
	b.BuildDir = path.Join(rootDir, b.ProjectUniqueDir(sharedDir))
	b.CacheDir = path.Join(cacheDir, b.ProjectUniqueDir(false))
141 142
}

143
func (b *Build) executeStage(ctx context.Context, buildStage BuildStage, executor Executor) error {
144
	b.CurrentStage = buildStage
145

146 147 148 149 150
	shell := executor.Shell()
	if shell == nil {
		return errors.New("No shell defined")
	}

151
	script, err := GenerateShellScript(buildStage, *shell)
152 153 154 155 156 157 158 159 160 161
	if err != nil {
		return err
	}

	// Nothing to execute
	if script == "" {
		return nil
	}

	cmd := ExecutorCommand{
162
		Context: ctx,
Kamil Trzciński's avatar
Kamil Trzciński committed
163
		Script:  script,
164 165
	}

166 167
	switch buildStage {
	case BuildStageUserScript, BuildStageAfterScript: // use custom build environment
168 169 170 171 172
		cmd.Predefined = false
	default: // all other stages use a predefined build environment
		cmd.Predefined = true
	}

Alessio Caiazza's avatar
Alessio Caiazza committed
173
	section := helpers.BuildSection{
174 175 176
		Name:        string(buildStage),
		SkipMetrics: !b.JobResponse.Features.TraceSections,
		Run:         func() error { return executor.Run(cmd) },
177
	}
178
	return section.Execute(&b.logger)
179 180
}

181
func (b *Build) executeUploadArtifacts(ctx context.Context, state error, executor Executor) (err error) {
182 183 184 185 186 187 188 189
	var uploadError error

	for _, artifact := range b.JobResponse.Artifacts {
		if artifact.ShouldUpload(state) {
			uploadError = b.executeStage(ctx, BuildStageUploadArtifacts, executor)
		}
		if uploadError != nil {
			err = uploadError
190 191 192 193 194 195
		}
	}

	return
}

196
func (b *Build) executeScript(ctx context.Context, executor Executor) error {
197
	// Prepare stage
198
	err := b.executeStage(ctx, BuildStagePrepare, executor)
199

200
	if err == nil {
201
		err = b.attemptExecuteStage(ctx, BuildStageGetSources, executor, b.GetGetSourcesAttempts())
202 203
	}
	if err == nil {
204
		err = b.attemptExecuteStage(ctx, BuildStageRestoreCache, executor, b.GetRestoreCacheAttempts())
205 206
	}
	if err == nil {
207
		err = b.attemptExecuteStage(ctx, BuildStageDownloadArtifacts, executor, b.GetDownloadArtifactsAttempts())
208 209
	}

210
	if err == nil {
211
		// Execute user build script (before_script + script)
212
		err = b.executeStage(ctx, BuildStageUserScript, executor)
213 214

		// Execute after script (after_script)
215 216 217 218
		timeoutContext, timeoutCancel := context.WithTimeout(ctx, AfterScriptTimeout)
		defer timeoutCancel()

		b.executeStage(timeoutContext, BuildStageAfterScript, executor)
219 220 221 222
	}

	// Execute post script (cache store, artifacts upload)
	if err == nil {
223
		err = b.executeStage(ctx, BuildStageArchiveCache, executor)
224
	}
225 226 227 228 229 230 231 232

	jobState := err
	err = b.executeUploadArtifacts(ctx, jobState, executor)

	// Use job's error if set
	if jobState != nil {
		err = jobState
	}
233 234 235
	return err
}

236
func (b *Build) attemptExecuteStage(ctx context.Context, buildStage BuildStage, executor Executor, attempts int) (err error) {
237 238 239 240
	if attempts < 1 || attempts > 10 {
		return fmt.Errorf("Number of attempts out of the range [1, 10] for stage: %s", buildStage)
	}
	for attempt := 0; attempt < attempts; attempt++ {
241
		if err = b.executeStage(ctx, buildStage, executor); err == nil {
242 243 244 245 246 247
			return
		}
	}
	return
}

248
func (b *Build) GetBuildTimeout() time.Duration {
249
	buildTimeout := b.RunnerInfo.Timeout
250 251 252
	if buildTimeout <= 0 {
		buildTimeout = DefaultTimeout
	}
253 254 255
	return time.Duration(buildTimeout) * time.Second
}

256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
func (b *Build) handleError(err error) error {
	switch err {
	case context.Canceled:
		b.CurrentState = BuildRunRuntimeCanceled
		return &BuildError{Inner: errors.New("canceled")}

	case context.DeadlineExceeded:
		b.CurrentState = BuildRunRuntimeTimedout
		return &BuildError{Inner: fmt.Errorf("execution took longer than %v seconds", b.GetBuildTimeout())}

	default:
		b.CurrentState = BuildRunRuntimeFinished
		return err
	}
}

272 273
func (b *Build) run(ctx context.Context, executor Executor) (err error) {
	b.CurrentState = BuildRunRuntimeRunning
274

275
	buildFinish := make(chan error, 1)
276 277 278

	runContext, runCancel := context.WithCancel(context.Background())
	defer runCancel()
279 280 281

	// Run build script
	go func() {
282
		buildFinish <- b.executeScript(runContext, executor)
283 284 285
	}()

	// Wait for signals: cancel, timeout, abort or finish
286
	b.Log().Debugln("Waiting for signals...")
287
	select {
288
	case <-ctx.Done():
289
		err = b.handleError(ctx.Err())
290

291
	case signal := <-b.SystemInterrupt:
292
		err = fmt.Errorf("aborted: %v", signal)
293
		b.CurrentState = BuildRunRuntimeTerminated
294 295

	case err = <-buildFinish:
296
		b.CurrentState = BuildRunRuntimeFinished
297 298 299
		return err
	}

Kamil Trzciński's avatar
Kamil Trzciński committed
300
	b.Log().WithError(err).Debugln("Waiting for build to finish...")
301

302
	// Wait till we receive that build did finish
303 304 305
	runCancel()
	<-buildFinish
	return err
306 307
}

308
func (b *Build) retryCreateExecutor(options ExecutorPrepareOptions, provider ExecutorProvider, logger BuildLogger) (executor Executor, err error) {
309
	for tries := 0; tries < PreparationRetries; tries++ {
310 311 312 313 314 315
		executor = provider.Create()
		if executor == nil {
			err = errors.New("failed to create executor")
			return
		}

316 317
		b.executorStageResolver = executor.GetCurrentStage

318
		err = executor.Prepare(options)
319 320 321 322 323 324 325
		if err == nil {
			break
		}
		if executor != nil {
			executor.Cleanup()
			executor = nil
		}
326 327
		if _, ok := err.(*BuildError); ok {
			break
328 329
		} else if options.Context.Err() != nil {
			return nil, b.handleError(options.Context.Err())
330
		}
331

332 333 334 335 336 337 338
		logger.SoftErrorln("Preparation failed:", err)
		logger.Infoln("Will be retried in", PreparationRetryInterval, "...")
		time.Sleep(PreparationRetryInterval)
	}
	return
}

339 340 341 342 343 344 345 346 347 348
func (b *Build) CurrentExecutorStage() ExecutorStage {
	if b.executorStageResolver == nil {
		b.executorStageResolver = func() ExecutorStage {
			return ExecutorStage("")
		}
	}

	return b.executorStageResolver()
}

349
func (b *Build) Run(globalConfig *Config, trace JobTrace) (err error) {
350 351
	var executor Executor

352 353
	b.logger = NewBuildLogger(trace, b.Log())
	b.logger.Println(fmt.Sprintf("Running with %s\n  on %s (%s)", AppVersion.Line(), b.Runner.Name, b.Runner.ShortDescription()))
Kamil Trzciński's avatar
Kamil Trzciński committed
354

355
	b.CurrentState = BuildRunStatePending
356

357
	defer func() {
Kamil Trzciński's avatar
Kamil Trzciński committed
358
		if _, ok := err.(*BuildError); ok {
359
			b.logger.SoftErrorln("Job failed:", err)
Kamil Trzciński's avatar
Kamil Trzciński committed
360 361
			trace.Fail(err)
		} else if err != nil {
362
			b.logger.Errorln("Job failed (system failure):", err)
363
			trace.Fail(err)
364
		} else {
365
			b.logger.Infoln("Job succeeded")
366
			trace.Success()
367
		}
368 369 370
		if executor != nil {
			executor.Cleanup()
		}
371
	}()
372

373 374 375
	context, cancel := context.WithTimeout(context.Background(), b.GetBuildTimeout())
	defer cancel()

376 377
	trace.SetCancelFunc(cancel)

378
	options := ExecutorPrepareOptions{
Kamil Trzciński's avatar
Kamil Trzciński committed
379 380
		Config:  b.Runner,
		Build:   b,
381
		Trace:   trace,
Kamil Trzciński's avatar
Kamil Trzciński committed
382
		User:    globalConfig.User,
383 384
		Context: context,
	}
385

386
	provider := GetExecutor(b.Runner.Executor)
Kamil Trzciński's avatar
Kamil Trzciński committed
387
	if provider == nil {
388 389 390
		return errors.New("executor not found")
	}

391
	executor, err = b.retryCreateExecutor(options, provider, b.logger)
392
	if err == nil {
393
		err = b.run(context, executor)
394
	}
395 396 397
	if executor != nil {
		executor.Finish(err)
	}
398 399
	return err
}
400 401 402 403

func (b *Build) String() string {
	return helpers.ToYAML(b)
}
404

405 406
func (b *Build) GetDefaultVariables() JobVariables {
	return JobVariables{
407 408
		{Key: "CI_PROJECT_DIR", Value: b.FullProjectDir(), Public: true, Internal: true, File: false},
		{Key: "CI_SERVER", Value: "yes", Public: true, Internal: true, File: false},
409 410 411
	}
}

412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
func (b *Build) GetCITLSVariables() JobVariables {
	variables := JobVariables{}
	if b.TLSCAChain != "" {
		variables = append(variables, JobVariable{"CI_SERVER_TLS_CA_FILE", b.TLSCAChain, true, true, true})
	}
	if b.TLSAuthCert != "" && b.TLSAuthKey != "" {
		variables = append(variables, JobVariable{"CI_SERVER_TLS_CERT_FILE", b.TLSAuthCert, true, true, true})
		variables = append(variables, JobVariable{"CI_SERVER_TLS_KEY_FILE", b.TLSAuthKey, true, true, true})
	}
	return variables
}

func (b *Build) GetGitTLSVariables() JobVariables {
	variables := JobVariables{}
	if b.TLSCAChain != "" {
		variables = append(variables, JobVariable{"GIT_SSL_CAINFO", b.TLSCAChain, true, true, true})
	}
	if b.TLSAuthCert != "" && b.TLSAuthKey != "" {
		variables = append(variables, JobVariable{"GIT_SSL_CERT", b.TLSAuthCert, true, true, true})
		variables = append(variables, JobVariable{"GIT_SSL_KEY", b.TLSAuthKey, true, true, true})
	}
	return variables
}

436
func (b *Build) GetAllVariables() (variables JobVariables) {
437 438 439
	if b.Runner != nil {
		variables = append(variables, b.Runner.GetVariables()...)
	}
440
	variables = append(variables, b.GetDefaultVariables()...)
441
	variables = append(variables, b.GetCITLSVariables()...)
442
	variables = append(variables, b.Variables...)
Kamil Trzciński's avatar
Kamil Trzciński committed
443
	return variables.Expand()
444
}
445 446 447 448

func (b *Build) GetGitDepth() string {
	return b.GetAllVariables().Get("GIT_DEPTH")
}
449 450 451 452 453 454 455 456 457

func (b *Build) GetGitStrategy() GitStrategy {
	switch b.GetAllVariables().Get("GIT_STRATEGY") {
	case "clone":
		return GitClone

	case "fetch":
		return GitFetch

458 459 460
	case "none":
		return GitNone

461 462 463 464 465 466 467 468
	default:
		if b.AllowGitFetch {
			return GitFetch
		}

		return GitClone
	}
}
469

470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486
func (b *Build) GetGitCheckout() bool {
	if b.GetGitStrategy() == GitNone {
		return false
	}

	strCheckout := b.GetAllVariables().Get("GIT_CHECKOUT")
	if len(strCheckout) == 0 {
		return true
	}

	checkout, err := strconv.ParseBool(strCheckout)
	if err != nil {
		return true
	}
	return checkout
}

487
func (b *Build) GetSubmoduleStrategy() SubmoduleStrategy {
488 489 490
	if b.GetGitStrategy() == GitNone {
		return SubmoduleNone
	}
491 492 493 494 495 496 497 498 499 500 501 502
	switch b.GetAllVariables().Get("GIT_SUBMODULE_STRATEGY") {
	case "normal":
		return SubmoduleNormal

	case "recursive":
		return SubmoduleRecursive

	case "none", "":
		// Default (legacy) behavior is to not update/init submodules
		return SubmoduleNone

	default:
503 504
		// Will cause an error in AbstractShell) writeSubmoduleUpdateCmds
		return SubmoduleInvalid
505 506 507
	}
}

508 509 510 511 512 513 514 515
func (b *Build) IsDebugTraceEnabled() bool {
	trace, err := strconv.ParseBool(b.GetAllVariables().Get("CI_DEBUG_TRACE"))
	if err != nil {
		return false
	}

	return trace
}
516

517
func (b *Build) GetDockerAuthConfig() string {
Tomasz Maczukin's avatar
Tomasz Maczukin committed
518
	return b.GetAllVariables().Get("DOCKER_AUTH_CONFIG")
519
}
520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543

func (b *Build) GetGetSourcesAttempts() int {
	retries, err := strconv.Atoi(b.GetAllVariables().Get("GET_SOURCES_ATTEMPTS"))
	if err != nil {
		return DefaultGetSourcesAttempts
	}
	return retries
}

func (b *Build) GetDownloadArtifactsAttempts() int {
	retries, err := strconv.Atoi(b.GetAllVariables().Get("ARTIFACT_DOWNLOAD_ATTEMPTS"))
	if err != nil {
		return DefaultArtifactDownloadAttempts
	}
	return retries
}

func (b *Build) GetRestoreCacheAttempts() int {
	retries, err := strconv.Atoi(b.GetAllVariables().Get("RESTORE_CACHE_ATTEMPTS"))
	if err != nil {
		return DefaultRestoreCacheAttempts
	}
	return retries
}
544 545 546 547 548 549 550 551

func (b *Build) GetCacheRequestTimeout() int {
	timeout, err := strconv.Atoi(b.GetAllVariables().Get("CACHE_REQUEST_TIMEOUT"))
	if err != nil {
		return DefaultCacheRequestTimeout
	}
	return timeout
}