Commit c2c9236c authored by Dmytro Zaporozhets (DZ)'s avatar Dmytro Zaporozhets (DZ)

Merge branch 'jobs_in_yml' into 'master'

CI configuration with .gitlab-ci.yml

https://dev.gitlab.org/gitlab/gitlab-ci/issues/245

Example:

```
# Refs to skip
skip_refs: “deploy*”

# Run before each script
before_script:
  - export PATH=$HOME/bin:/usr/local/bin:/usr/bin:/bin
  - gem install bundler
  - cp config/database.yml.mysql config/database.yml
  - cp config/gitlab.yml.example config/gitlab.yml
  - touch log/application.log
  - touch log/test.log
  - bundle install --without postgres production --jobs $(nproc)
  - bundle exec rake db:create RAILS_ENV=test

# Parallel jobs, each line is parallel build
jobs:
  - script: “rake spec”
    runner: “ruby,postgres”
    name: “Rspec”
  - script: “rake spinach”
    runner: “ruby,mysql”
    name: “Spinach”
    tags: true
    branches: false

# Parallel deploy jobs
deploy_jobs:
  “cap deploy production”
  “cap deploy staging”

```

See merge request !110
parents a5b1a3a5 27a3fe8b
......@@ -3,6 +3,7 @@ v7.12.0
- Add notification if there are no runners
- Fix pagination on dashboard
- Remove ID column from runners list in the admin area
- Using .gitlab-ci.yml file instead of jobs
v7.11.0
- Deploy Jobs API calls
......
class JobsController < ApplicationController
before_filter :authenticate_user!
before_filter :project
before_filter :authorize_access_project!
before_filter :authorize_manage_project!
layout 'project'
def index
end
def deploy_jobs
end
private
def project
@project ||= Project.find(params[:project_id])
end
end
......@@ -3,9 +3,9 @@ class ProjectsController < ApplicationController
before_filter :authenticate_user!, except: [:build, :badge, :index, :show]
before_filter :authenticate_public_page!, only: :show
before_filter :project, only: [:build, :integration, :show, :badge, :edit, :update, :destroy, :toggle_shared_runners]
before_filter :project, only: [:build, :integration, :show, :badge, :edit, :update, :destroy, :toggle_shared_runners, :dumped_yaml]
before_filter :authorize_access_project!, except: [:build, :gitlab, :badge, :index, :show, :new, :create]
before_filter :authorize_manage_project!, only: [:edit, :integration, :update, :destroy, :toggle_shared_runners]
before_filter :authorize_manage_project!, only: [:edit, :integration, :update, :destroy, :toggle_shared_runners, :dumped_yaml]
before_filter :authenticate_token!, only: [:build]
before_filter :no_cache, only: [:badge]
protect_from_forgery except: :build
......@@ -109,6 +109,10 @@ class ProjectsController < ApplicationController
redirect_to :back
end
def dumped_yaml
send_data @project.generated_yaml_config, filename: '.gitlab-ci.yml'
end
protected
def project
......@@ -123,8 +127,7 @@ class ProjectsController < ApplicationController
def project_params
params.require(:project).permit(:path, :timeout, :timeout_in_minutes, :default_ref, :always_build,
:polling_interval, :public, :ssh_url_to_repo, :allow_git_fetch, :skip_refs, :email_recipients,
:email_add_pusher, :email_only_broken_builds, :coverage_regex, :shared_runners_enabled, :token,
{ jobs_attributes: [:id, :name, :build_branches, :build_tags, :tag_list, :commands, :refs, :_destroy, :job_type] })
:polling_interval, :public, :ssh_url_to_repo, :allow_git_fetch, :email_recipients,
:email_add_pusher, :email_only_broken_builds, :coverage_regex, :shared_runners_enabled, :token)
end
end
......@@ -14,7 +14,6 @@
# commit_id :integer
# coverage :float
# commands :text
# job_id :integer
#
class Build < ActiveRecord::Base
......@@ -23,7 +22,6 @@ class Build < ActiveRecord::Base
belongs_to :commit
belongs_to :project
belongs_to :runner
belongs_to :job, -> { with_deleted }
validates :commit, presence: true
validates :status, presence: true
......@@ -64,15 +62,8 @@ class Build < ActiveRecord::Base
def retry(build)
new_build = Build.new(status: :pending)
if build.job
new_build.commands = build.job.commands
new_build.tag_list = build.job.tag_list
else
new_build.commands = build.commands
end
new_build.job_id = build.job_id
new_build.commands = build.commands
new_build.tag_list = build.tag_list
new_build.commit_id = build.commit_id
new_build.project_id = build.project_id
new_build.save
......@@ -109,8 +100,8 @@ class Build < ActiveRecord::Base
WebHookService.new.build_end(build)
end
if build.commit.success? && !(build.job && build.job.deploy?)
build.commit.create_deploy_builds(build.ref)
if build.commit.success? && !build.deploy?
build.commit.create_deploy_builds
end
project.execute_services(build)
......@@ -206,18 +197,4 @@ class Build < ActiveRecord::Base
# so we just silentrly ignore error for now
end
end
def job_name
if job
job.name
end
end
def for_tag?
if job && job.build_tags
true
else
false
end
end
end
......@@ -15,7 +15,6 @@
class Commit < ActiveRecord::Base
belongs_to :project
has_many :builds, dependent: :destroy
has_many :jobs, through: :builds
serialize :push_data
......@@ -93,31 +92,19 @@ class Commit < ActiveRecord::Base
end
def create_builds
project.jobs.where(build_branches: true).active.parallel.map do |job|
create_build_from_job(job)
end
end
def create_builds_for_tag(ref = '')
project.jobs.where(build_tags: true).active.parallel.map do |job|
create_build_from_job(job, ref)
filter_param = tag? ? :tags : :branches
config_processor.builds.each do |build_attrs|
if build_attrs[filter_param]
builds.create!({ project: project }.merge(build_attrs.extract!(:name, :commands, :tag_list)))
end
end
end
def create_build_from_job(job, ref = '')
build = builds.new(commands: job.commands)
build.tag_list = job.tag_list
build.project_id = project_id
build.job = job
build.save
build
end
def builds_without_retry
@builds_without_retry ||=
begin
grouped_builds = builds.group_by(&:job)
grouped_builds.map do |job, builds|
grouped_builds = builds.group_by(&:name)
grouped_builds.map do |name, builds|
builds.sort_by(&:id).last
end
end
......@@ -127,11 +114,9 @@ class Commit < ActiveRecord::Base
@retried_builds ||= (builds - builds_without_retry)
end
def create_deploy_builds(ref)
project.jobs.deploy.active.each do |job|
if job.run_for_ref?(ref)
create_build_from_job(job)
end
def create_deploy_builds
config_processor.deploy_builds_for_ref(ref).each do |build_attrs|
builds.create!({ project: project }.merge(build_attrs))
end
end
......@@ -194,4 +179,8 @@ class Commit < ActiveRecord::Base
def matrix?
builds_without_retry.size > 1
end
def config_processor
@config_processor ||= GitlabCiYamlProcessor.new(push_data[:ci_yaml_file])
end
end
# == Schema Information
#
# Table name: jobs
#
# id :integer not null, primary key
# project_id :integer not null
# commands :text
# active :boolean default(TRUE), not null
# created_at :datetime
# updated_at :datetime
# name :string(255)
# build_branches :boolean default(TRUE), not null
# build_tags :boolean default(FALSE), not null
# job_type :string(255) default("parallel")
# refs :string(255)
# deleted_at :datetime
#
class Job < ActiveRecord::Base
acts_as_paranoid
belongs_to :project
has_many :builds
acts_as_taggable
scope :active, ->() { where(active: true) }
scope :archived, ->() { where(active: false) }
scope :parallel, ->(){ where(job_type: "parallel") }
scope :deploy, ->(){ where(job_type: "deploy") }
validate :refs, length: { maximum: 255 }
def deploy?
job_type == "deploy"
end
def run_for_ref?(ref)
if refs.present?
refs.split(",").map(&:strip).each do |refs_val|
return true if File.fnmatch(refs_val, ref)
end
false
else
true
end
end
end
......@@ -32,7 +32,6 @@ class Project < ActiveRecord::Base
has_many :runner_projects, dependent: :destroy
has_many :runners, through: :runner_projects
has_many :web_hooks, dependent: :destroy
has_many :jobs, dependent: :destroy
has_many :events, dependent: :destroy
# Project services
......@@ -41,8 +40,6 @@ class Project < ActiveRecord::Base
has_one :slack_service, dependent: :destroy
has_one :mail_service, dependent: :destroy
accepts_nested_attributes_for :jobs, allow_destroy: true
#
# Validations
#
......@@ -55,8 +52,6 @@ class Project < ActiveRecord::Base
presence: true,
if: ->(project) { project.always_build.present? }
validate :validate_jobs
scope :public_only, ->() { where(public: true) }
before_validation :set_default_values
......@@ -165,39 +160,10 @@ ls -la
self.timeout = value.to_i * 60
end
def skip_ref?(ref_name)
if skip_refs.present?
skip_refs.delete(" ").split(",").each do |ref|
return true if File.fnmatch(ref, ref_name)
end
false
else
false
end
end
def create_commit_for_tag?(tag)
jobs.where(build_tags: true).active.parallel.any? ||
jobs.active.deploy.any?{ |job| job.run_for_ref?(tag)}
end
def coverage_enabled?
coverage_regex.present?
end
def build_default_job
jobs.build(commands: Project.base_build_script)
end
def validate_jobs
remaining_jobs = jobs.reject(&:marked_for_destruction?)
if remaining_jobs.empty?
errors.add(:jobs, "At least one foo")
end
end
# Build a clone-able repo url
# using http and basic auth
def repo_url_with_auth
......
......@@ -8,12 +8,14 @@ class HipChatMessage
def to_s
lines = Array.new
lines.push("<a href=\"#{RoutesHelper.project_url(project)}\">#{project.name}</a> - ")
if commit.matrix?
lines.push("<a href=\"#{RoutesHelper.project_ref_commit_url(project, commit.ref, commit.sha)}\">Commit ##{commit.id}</a></br>")
else
first_build = commit.builds_without_retry.first
lines.push("<a href=\"#{RoutesHelper.project_build_url(project, first_build)}\">Build '#{first_build.job_name}' ##{first_build.id}</a></br>")
lines.push("<a href=\"#{RoutesHelper.project_build_url(project, first_build)}\">Build '#{first_build.name}' ##{first_build.id}</a></br>")
end
lines.push("#{commit.short_sha} #{commit.git_author_name} - #{commit.git_commit_message}</br>")
lines.push("#{humanized_status(commit_status)} in #{commit.duration} second(s).")
lines.join('')
......
......@@ -24,7 +24,7 @@ class SlackMessage
commit.builds_without_retry.each do |build|
next unless build.failed?
fields << {
title: build.job_name,
title: build.name,
value: "Build <#{RoutesHelper.project_build_url(project, build)}|\##{build.id}> failed in #{build.duration.to_i} second(s)."
}
end
......
......@@ -3,6 +3,8 @@ class CreateCommitService
before_sha = params[:before]
sha = params[:checkout_sha] || params[:after]
origin_ref = params[:ref]
yaml_config = params[:ci_yaml_file] || project.generated_yaml_config
config_processor = build_config_processor(yaml_config)
unless origin_ref && sha.present?
return false
......@@ -19,11 +21,11 @@ class CreateCommitService
return false
end
if origin_ref.start_with?('refs/tags/') && !project.create_commit_for_tag?(ref)
if origin_ref.start_with?('refs/tags/') && !config_processor.create_commit_for_tag?(ref)
return false
end
if project.skip_ref?(ref)
if config_processor.skip_ref?(ref)
return false
end
......@@ -34,6 +36,7 @@ class CreateCommitService
data = {
ref: ref,
sha: sha,
tag: origin_ref.start_with?('refs/tags/'),
before_sha: before_sha,
push_data: {
before: before_sha,
......@@ -43,23 +46,26 @@ class CreateCommitService
user_email: params[:user_email],
repository: params[:repository],
commits: params[:commits],
total_commits_count: params[:total_commits_count]
total_commits_count: params[:total_commits_count],
ci_yaml_file: yaml_config
}
}
commit = project.commits.create(data)
end
if origin_ref.start_with?('refs/tags/')
commit.create_builds_for_tag(ref)
else
commit.create_builds
end
commit.create_builds
if commit.builds.empty?
commit.create_deploy_builds(ref)
commit.create_deploy_builds
end
commit
end
private
def build_config_processor(config_data)
@builds_config ||= GitlabCiYamlProcessor.new(config_data)
end
end
......@@ -5,7 +5,6 @@ class CreateProjectService
@project = Project.parse(params)
Project.transaction do
@project.build_default_job
@project.save!
opts = {
......@@ -19,11 +18,6 @@ class CreateProjectService
end
if forked_project
# Copy jobs
@project.jobs = forked_project.jobs.map do |job|
Job.new(job.attributes.except("id"))
end
# Copy settings
settings = forked_project.attributes.select do |attr_name, value|
["public", "shared_runners_enabled", "allow_git_fetch"].include? attr_name
......
......@@ -18,7 +18,7 @@ class WebHookService
data = {}
data.merge!({
build_id: build.id,
build_name: build.job_name,
build_name: build.name,
build_status: build.status,
build_started_at: build.started_at,
build_finished_at: build.finished_at,
......
......@@ -9,7 +9,7 @@
%span.runner-state.runner-state-specific
Specific
- if @runner.shared?
.bs-callout.bs-callout-success
......@@ -38,7 +38,7 @@
Tags
.col-sm-10
= f.text_field :tag_list, class: 'form-control'
.help-block You can setup jobs to only use runners with specific tags
.help-block You can setup builds to only use runners with specific tags
.form-actions
= f.submit 'Save', class: 'btn btn-save'
......
......@@ -7,17 +7,16 @@
%strong Build ##{build.id}
%td
- if build.job
- if build.for_tag?
Tag
&middot;
#{build.ref}
- else
Commit
&middot;
#{build.short_sha}
- if build.commit.tag?
Tag
&middot;
#{build.ref}
- else
Commit
&middot;
#{build.short_sha}
%td
= build.job_name
= build.name
- if build.tags.any?
.pull-right
- build.tag_list.each do |tag|
......
......@@ -2,12 +2,7 @@
= link_to @project.name, @project
@
= @commit.short_sha
- if current_user && current_user.can_manage_project?(@project.gitlab_id)
.pull-right
= link_to project_jobs_path(@project), class: "btn btn-default btn-small" do
%i.icon-edit.icon-white
Edit Job
%p
= link_to project_ref_commit_path(@project, @commit.ref, @commit.sha) do
&larr; Back to project commit
......@@ -21,9 +16,9 @@
%i{class: build_icon_css_class(build)}
%span
Build ##{build.id}
- if build.job_name
- if build.name
&middot;
= build.job_name
= build.name
- unless @commit.builds_without_retry.include?(@build)
%li.active
......@@ -37,7 +32,7 @@
.col-md-9
.build-head.alert{class: build_status_alert_class(@build)}
%h4
- if @build.for_tag?
- if @build.commit.tag?
Build for tag
%code #{@build.ref}
- else
......@@ -150,8 +145,8 @@
= link_to build_url(build) do
%span ##{build.id}
%td
- if build.job_name
= build.job_name
- if build.name
= build.name
%td.status= build.status
......
......@@ -2,12 +2,6 @@
= @project.name
@
#{gitlab_commit_link(@project, @commit.sha)}
- if current_user && current_user.can_manage_project?(@project.gitlab_id)
.pull-right
= link_to project_jobs_path(@project), class: "btn btn-default btn-small" do
%i.icon-edit.icon-white
Edit Job
%p
= link_to project_path(@project) do
&larr; Back to project commits
......
= nested_form_for @project, html: { class: 'form-horizontal' } do |f|
- if @project.errors.any?
#error_explanation
%p.lead= "#{pluralize(@project.errors.count, "error")} prohibited this project from being saved:"
.alert.alert-error
%ul
- @project.errors.full_messages.each do |msg|
%li= msg
= f.fields_for :jobs do |job_form|
- if job_form.object.job_type == "deploy" || job_form.object.new_record?
= job_form.hidden_field :job_type, value: "deploy"
.form-group
= job_form.label :name, 'Name', class: 'control-label'
.col-sm-10
= job_form.text_field :name, class: 'form-control', placeholder: "Staging"
.form-group
= job_form.label :tag_list, class: 'control-label' do
Tags
.col-sm-10
= job_form.text_field :tag_list, class: 'form-control'
.help-block
This job will only use runners that contain all these tags.
Leave blank if you want this job to use any runner
.form-group
= job_form.label :refs, class: 'control-label' do
Refs
.col-sm-10
= job_form.text_field :refs, class: 'form-control', placeholder: "master, staging, feature/*, tags/testing*"
.help-block
Run only when the above git refs strings match the branch or tag that was pushed.
%br
Accepts strings and glob pattern syntax
.form-group
= job_form.label :commands, 'Script', class: 'control-label'
.col-sm-10
= job_form.text_area :commands, class: 'form-control', rows: 10, placeholder: "bundle exec rake spec"
%p.light
All lines will be concatenated into one file and executed.
%br
If you change the working directory or the environment this will affect all following lines.
%br
= link_to("Example job scripts", "https://gitlab.com/gitlab-org/gitlab-ci/tree/master/doc/examples")
= job_form.link_to_remove "Remove this job", class: 'btn btn-danger pull-right'
%hr
.clearfix
= f.link_to_add "Add a job", :jobs, class: 'btn btn-save pull-right'
.form-actions
= f.submit 'Save changes', class: 'btn btn-success'
= nested_form_for @project, html: { class: 'form-horizontal' } do |f|
- if @project.errors.any?