Merge branch 'jobs_in_yml' into 'master'

CI configuration with .gitlab-ci.yml

https://dev.gitlab.org/gitlab/gitlab-ci/issues/245

Example:

```
# Refs to skip
skip_refs: “deploy*”

# Run before each script
before_script:
  - export PATH=$HOME/bin:/usr/local/bin:/usr/bin:/bin
  - gem install bundler
  - cp config/database.yml.mysql config/database.yml
  - cp config/gitlab.yml.example config/gitlab.yml
  - touch log/application.log
  - touch log/test.log
  - bundle install --without postgres production --jobs $(nproc)
  - bundle exec rake db:create RAILS_ENV=test

# Parallel jobs, each line is parallel build
jobs:
  - script: “rake spec”
    runner: “ruby,postgres”
    name: “Rspec”
  - script: “rake spinach”
    runner: “ruby,mysql”
    name: “Spinach”
    tags: true
    branches: false

# Parallel deploy jobs
deploy_jobs:
  “cap deploy production”
  “cap deploy staging”

```

See merge request !110

CHANGELOG

@@ -3,6 +3,7 @@ v7.12.0
   - Add notification if there are no runners
   - Fix pagination on dashboard
   - Remove ID column from runners list in the admin area
+  - Using .gitlab-ci.yml file instead of jobs
 
 v7.11.0
   - Deploy Jobs API calls

app/controllers/jobs_controller.rb

-class JobsController < ApplicationController
-  before_filter :authenticate_user!
-  before_filter :project
-  before_filter :authorize_access_project!
-  before_filter :authorize_manage_project!
-
-  layout 'project'
-
-  def index
-  end
-
-  def deploy_jobs
-  end
-
-  private
-
-  def project
-    @project ||= Project.find(params[:project_id])
-  end
-end

app/controllers/projects_controller.rb

@@ -3,9 +3,9 @@ class ProjectsController < ApplicationController
 
   before_filter :authenticate_user!, except: [:build, :badge, :index, :show]
   before_filter :authenticate_public_page!, only: :show
-  before_filter :project, only: [:build, :integration, :show, :badge, :edit, :update, :destroy, :toggle_shared_runners]
+  before_filter :project, only: [:build, :integration, :show, :badge, :edit, :update, :destroy, :toggle_shared_runners, :dumped_yaml]
   before_filter :authorize_access_project!, except: [:build, :gitlab, :badge, :index, :show, :new, :create]
-  before_filter :authorize_manage_project!, only: [:edit, :integration, :update, :destroy, :toggle_shared_runners]
+  before_filter :authorize_manage_project!, only: [:edit, :integration, :update, :destroy, :toggle_shared_runners, :dumped_yaml]
   before_filter :authenticate_token!, only: [:build]
   before_filter :no_cache, only: [:badge]
   protect_from_forgery except: :build
@@ -109,6 +109,10 @@ class ProjectsController < ApplicationController
     redirect_to :back
   end
 
+  def dumped_yaml
+    send_data @project.generated_yaml_config, filename: '.gitlab-ci.yml'
+  end
+
   protected
 
   def project
@@ -123,8 +127,7 @@ class ProjectsController < ApplicationController
 
   def project_params
     params.require(:project).permit(:path, :timeout, :timeout_in_minutes, :default_ref, :always_build,
-      :polling_interval, :public, :ssh_url_to_repo, :allow_git_fetch, :skip_refs, :email_recipients,
-      :email_add_pusher, :email_only_broken_builds, :coverage_regex, :shared_runners_enabled, :token,
-      { jobs_attributes: [:id, :name, :build_branches, :build_tags, :tag_list, :commands, :refs, :_destroy, :job_type] })
+      :polling_interval, :public, :ssh_url_to_repo, :allow_git_fetch, :email_recipients,
+      :email_add_pusher, :email_only_broken_builds, :coverage_regex, :shared_runners_enabled, :token)
   end
 end

app/models/build.rb

@@ -14,7 +14,6 @@
 #  commit_id   :integer
 #  coverage    :float
 #  commands    :text
-#  job_id      :integer
 #
 
 class Build < ActiveRecord::Base
@@ -23,7 +22,6 @@ class Build < ActiveRecord::Base
   belongs_to :commit
   belongs_to :project
   belongs_to :runner
-  belongs_to :job, -> { with_deleted }
 
   validates :commit, presence: true
   validates :status, presence: true
@@ -64,15 +62,8 @@ class Build < ActiveRecord::Base
 
     def retry(build)
       new_build = Build.new(status: :pending)
-
-      if build.job
-        new_build.commands = build.job.commands
-        new_build.tag_list = build.job.tag_list
-      else
-        new_build.commands = build.commands
-      end
-
-      new_build.job_id = build.job_id
+      new_build.commands = build.commands
+      new_build.tag_list = build.tag_list
       new_build.commit_id = build.commit_id
       new_build.project_id = build.project_id
       new_build.save
@@ -109,8 +100,8 @@ class Build < ActiveRecord::Base
         WebHookService.new.build_end(build)
       end
 
-      if build.commit.success? && !(build.job && build.job.deploy?)
-        build.commit.create_deploy_builds(build.ref)
+      if build.commit.success? && !build.deploy?
+        build.commit.create_deploy_builds
       end
 
       project.execute_services(build)
@@ -206,18 +197,4 @@ class Build < ActiveRecord::Base
       # so we just silentrly ignore error for now
     end
   end
-
-  def job_name
-    if job
-      job.name
-    end
-  end
-
-  def for_tag?
-    if job && job.build_tags
-      true
-    else
-      false
-    end
-  end
 end

app/models/commit.rb

@@ -15,7 +15,6 @@
 class Commit < ActiveRecord::Base
   belongs_to :project
   has_many :builds, dependent: :destroy
-  has_many :jobs, through: :builds
 
   serialize :push_data
 
@@ -93,31 +92,19 @@ class Commit < ActiveRecord::Base
   end
 
   def create_builds
-    project.jobs.where(build_branches: true).active.parallel.map do |job|
-      create_build_from_job(job)
-    end
-  end
-
-  def create_builds_for_tag(ref = '')
-    project.jobs.where(build_tags: true).active.parallel.map do |job|
-      create_build_from_job(job, ref)
+    filter_param = tag? ? :tags : :branches
+    config_processor.builds.each do |build_attrs|
+      if build_attrs[filter_param]
+        builds.create!({ project: project }.merge(build_attrs.extract!(:name, :commands, :tag_list)))
+      end
     end
   end
 
-  def create_build_from_job(job, ref = '')
-    build = builds.new(commands: job.commands)
-    build.tag_list = job.tag_list
-    build.project_id = project_id
-    build.job = job
-    build.save
-    build
-  end
-
   def builds_without_retry
     @builds_without_retry ||=
       begin
-        grouped_builds = builds.group_by(&:job)
-        grouped_builds.map do |job, builds|
+        grouped_builds = builds.group_by(&:name)
+        grouped_builds.map do |name, builds|
           builds.sort_by(&:id).last
         end
       end
@@ -127,11 +114,9 @@ class Commit < ActiveRecord::Base
     @retried_builds ||= (builds - builds_without_retry)
   end
 
-  def create_deploy_builds(ref)
-    project.jobs.deploy.active.each do |job|
-      if job.run_for_ref?(ref)
-        create_build_from_job(job)
-      end
+  def create_deploy_builds
+    config_processor.deploy_builds_for_ref(ref).each do |build_attrs|
+      builds.create!({ project: project }.merge(build_attrs))
     end
   end
 
@@ -194,4 +179,8 @@ class Commit < ActiveRecord::Base
   def matrix?
     builds_without_retry.size > 1
   end
+
+  def config_processor
+    @config_processor ||= GitlabCiYamlProcessor.new(push_data[:ci_yaml_file])
+  end
 end

app/models/job.rb

-# == Schema Information
-#
-# Table name: jobs
-#
-#  id             :integer          not null, primary key
-#  project_id     :integer          not null
-#  commands       :text
-#  active         :boolean          default(TRUE), not null
-#  created_at     :datetime
-#  updated_at     :datetime
-#  name           :string(255)
-#  build_branches :boolean          default(TRUE), not null
-#  build_tags     :boolean          default(FALSE), not null
-#  job_type       :string(255)      default("parallel")
-#  refs           :string(255)
-#  deleted_at     :datetime
-#
-
-class Job < ActiveRecord::Base
-  acts_as_paranoid
-  
-  belongs_to :project
-  has_many :builds
-
-  acts_as_taggable
-
-  scope :active, ->() { where(active: true) }
-  scope :archived, ->() { where(active: false) }
-  scope :parallel, ->(){ where(job_type: "parallel") }
-  scope :deploy, ->(){ where(job_type: "deploy") }
-
-  validate :refs, length: { maximum: 255 }
-
-  def deploy?
-    job_type == "deploy"
-  end
-
-  def run_for_ref?(ref)
-    if refs.present?
-      refs.split(",").map(&:strip).each do |refs_val|
-        return true if File.fnmatch(refs_val, ref)
-      end
-
-      false
-    else
-      true
-    end
-  end
-end

app/models/project.rb

@@ -32,7 +32,6 @@ class Project < ActiveRecord::Base
   has_many :runner_projects, dependent: :destroy
   has_many :runners, through: :runner_projects
   has_many :web_hooks, dependent: :destroy
-  has_many :jobs, dependent: :destroy
   has_many :events, dependent: :destroy
 
   # Project services
@@ -41,8 +40,6 @@ class Project < ActiveRecord::Base
   has_one :slack_service, dependent: :destroy
   has_one :mail_service, dependent: :destroy
 
-  accepts_nested_attributes_for :jobs, allow_destroy: true
-
   #
   # Validations
   #
@@ -55,8 +52,6 @@ class Project < ActiveRecord::Base
     presence: true,
     if: ->(project) { project.always_build.present? }
 
-  validate :validate_jobs
-
   scope :public_only, ->() { where(public: true) }
 
   before_validation :set_default_values
@@ -165,39 +160,10 @@ ls -la
     self.timeout = value.to_i * 60
   end
 
-  def skip_ref?(ref_name)
-    if skip_refs.present?
-      skip_refs.delete(" ").split(",").each do |ref|
-        return true if File.fnmatch(ref, ref_name)
-      end
-
-      false
-    else
-      false
-    end
-  end
-
-  def create_commit_for_tag?(tag)
-    jobs.where(build_tags: true).active.parallel.any? ||
-    jobs.active.deploy.any?{ |job| job.run_for_ref?(tag)}
-  end
-
   def coverage_enabled?
     coverage_regex.present?
   end
 
-  def build_default_job
-    jobs.build(commands: Project.base_build_script)
-  end
-
-  def validate_jobs
-    remaining_jobs = jobs.reject(&:marked_for_destruction?)
-
-    if remaining_jobs.empty?
-      errors.add(:jobs, "At least one foo")
-    end
-  end
-
   # Build a clone-able repo url
   # using http and basic auth
   def repo_url_with_auth

app/models/project_services/hip_chat_message.rb

@@ -8,12 +8,14 @@ class HipChatMessage
   def to_s
     lines = Array.new
     lines.push("<a href=\"#{RoutesHelper.project_url(project)}\">#{project.name}</a> - ")
+    
     if commit.matrix?
       lines.push("<a href=\"#{RoutesHelper.project_ref_commit_url(project, commit.ref, commit.sha)}\">Commit ##{commit.id}</a></br>")
     else
       first_build = commit.builds_without_retry.first
-      lines.push("<a href=\"#{RoutesHelper.project_build_url(project, first_build)}\">Build '#{first_build.job_name}' ##{first_build.id}</a></br>")
+      lines.push("<a href=\"#{RoutesHelper.project_build_url(project, first_build)}\">Build '#{first_build.name}' ##{first_build.id}</a></br>")
     end
+    
     lines.push("#{commit.short_sha} #{commit.git_author_name} - #{commit.git_commit_message}</br>")
     lines.push("#{humanized_status(commit_status)} in #{commit.duration} second(s).")
     lines.join('')

app/models/project_services/slack_message.rb

@@ -24,7 +24,7 @@ class SlackMessage
       commit.builds_without_retry.each do |build|
         next unless build.failed?
         fields << {
-          title: build.job_name,
+          title: build.name,
           value: "Build <#{RoutesHelper.project_build_url(project, build)}|\##{build.id}> failed in #{build.duration.to_i} second(s)."
         }
       end

app/services/create_commit_service.rb

@@ -3,6 +3,8 @@ class CreateCommitService
     before_sha = params[:before]
     sha = params[:checkout_sha] || params[:after]
     origin_ref = params[:ref]
+    yaml_config = params[:ci_yaml_file] || project.generated_yaml_config
+    config_processor = build_config_processor(yaml_config)
 
     unless origin_ref && sha.present?
       return false
@@ -19,11 +21,11 @@ class CreateCommitService
       return false
     end
 
-    if origin_ref.start_with?('refs/tags/') && !project.create_commit_for_tag?(ref)
+    if origin_ref.start_with?('refs/tags/') && !config_processor.create_commit_for_tag?(ref)
       return false
     end
 
-    if project.skip_ref?(ref)
+    if config_processor.skip_ref?(ref)
       return false
     end
 
@@ -34,6 +36,7 @@ class CreateCommitService
       data = {
         ref: ref,
         sha: sha,
+        tag: origin_ref.start_with?('refs/tags/'),
         before_sha: before_sha,
         push_data: {
           before: before_sha,
@@ -43,23 +46,26 @@ class CreateCommitService
           user_email: params[:user_email],
           repository: params[:repository],
           commits: params[:commits],
-          total_commits_count: params[:total_commits_count]
+          total_commits_count: params[:total_commits_count],
+          ci_yaml_file: yaml_config
         }
       }
 
       commit = project.commits.create(data)
     end
 
-    if origin_ref.start_with?('refs/tags/')
-      commit.create_builds_for_tag(ref)
-    else
-      commit.create_builds
-    end
+    commit.create_builds
 
     if commit.builds.empty?
-      commit.create_deploy_builds(ref)
+      commit.create_deploy_builds
     end
 
     commit
   end
+
+  private
+
+  def build_config_processor(config_data)
+    @builds_config ||= GitlabCiYamlProcessor.new(config_data)
+  end
 end

app/services/create_project_service.rb

@@ -5,7 +5,6 @@ class CreateProjectService
     @project = Project.parse(params)
 
     Project.transaction do
-      @project.build_default_job
       @project.save!
 
       opts = {
@@ -19,11 +18,6 @@ class CreateProjectService
     end
 
     if forked_project
-      # Copy jobs
-      @project.jobs = forked_project.jobs.map do |job|
-        Job.new(job.attributes.except("id"))
-      end
-
       # Copy settings
       settings = forked_project.attributes.select do |attr_name, value|
         ["public", "shared_runners_enabled", "allow_git_fetch"].include? attr_name

app/services/web_hook_service.rb

@@ -18,7 +18,7 @@ class WebHookService
     data = {}
     data.merge!({
       build_id: build.id,
-      build_name: build.job_name,
+      build_name: build.name,
       build_status: build.status,
       build_started_at: build.started_at,
       build_finished_at: build.finished_at,

app/views/admin/runners/show.html.haml

@@ -9,7 +9,7 @@
         %span.runner-state.runner-state-specific
           Specific
 
-  
+
 
 - if @runner.shared?
   .bs-callout.bs-callout-success
@@ -38,7 +38,7 @@
       Tags
     .col-sm-10
       = f.text_field :tag_list, class: 'form-control'
-      .help-block You can setup jobs to only use runners with specific tags
+      .help-block You can setup builds to only use runners with specific tags
   .form-actions
     = f.submit 'Save', class: 'btn btn-save'
 

app/views/builds/_build.html.haml

@@ -7,17 +7,16 @@
       %strong Build ##{build.id}
 
   %td
-    - if build.job
-      - if build.for_tag?
-        Tag
-        ·
-        #{build.ref}
-      - else
-        Commit
-        ·
-        #{build.short_sha}
+    - if build.commit.tag?
+      Tag
+      ·
+      #{build.ref}
+    - else
+      Commit
+      ·
+      #{build.short_sha}
   %td
-    = build.job_name
+    = build.name
     - if build.tags.any?
       .pull-right
         - build.tag_list.each do |tag|

app/views/builds/show.html.haml

@@ -2,12 +2,7 @@
   = link_to @project.name, @project
   @
   = @commit.short_sha
-  - if current_user && current_user.can_manage_project?(@project.gitlab_id)
-    .pull-right
-      = link_to project_jobs_path(@project), class: "btn btn-default btn-small" do
-        %i.icon-edit.icon-white
-        Edit Job
-
+  
 %p
   = link_to project_ref_commit_path(@project, @commit.ref, @commit.sha) do
     ← Back to project commit
@@ -21,9 +16,9 @@
           %i{class: build_icon_css_class(build)}
           %span
             Build ##{build.id}
-            - if build.job_name
+            - if build.name
               ·
-              = build.job_name
+              = build.name
 
     - unless @commit.builds_without_retry.include?(@build)
       %li.active
@@ -37,7 +32,7 @@
   .col-md-9
     .build-head.alert{class: build_status_alert_class(@build)}
       %h4
-        - if @build.for_tag?
+        - if @build.commit.tag?
           Build for tag
           %code #{@build.ref}
         - else
@@ -150,8 +145,8 @@
                 = link_to build_url(build) do
                   %span ##{build.id}
               %td
-                - if build.job_name
-                  = build.job_name
+                - if build.name
+                  = build.name
               %td.status= build.status
 
 

app/views/commits/show.html.haml

@@ -2,12 +2,6 @@
   = @project.name
   @
   #{gitlab_commit_link(@project, @commit.sha)}
-  - if current_user && current_user.can_manage_project?(@project.gitlab_id)
-    .pull-right
-      = link_to project_jobs_path(@project), class: "btn btn-default btn-small" do
-        %i.icon-edit.icon-white
-        Edit Job
-
 %p
   = link_to project_path(@project) do
     ← Back to project commits

app/views/jobs/_deploy_job_edit.html.haml

-= nested_form_for @project, html: { class: 'form-horizontal' } do |f|
-  - if @project.errors.any?
-    #error_explanation
-      %p.lead= "#{pluralize(@project.errors.count, "error")} prohibited this project from being saved:"
-      .alert.alert-error
-        %ul
-          - @project.errors.full_messages.each do |msg|
-            %li= msg
-
-  = f.fields_for :jobs do |job_form|
-    - if job_form.object.job_type == "deploy" || job_form.object.new_record?
-
-      = job_form.hidden_field :job_type, value: "deploy"
-      .form-group
-        = job_form.label :name, 'Name', class: 'control-label'
-        .col-sm-10
-          = job_form.text_field :name, class: 'form-control', placeholder: "Staging"
-      .form-group
-        = job_form.label :tag_list, class: 'control-label' do
-          Tags
-        .col-sm-10
-          = job_form.text_field :tag_list, class: 'form-control'
-          .help-block
-            This job will only use runners that contain all these tags.
-            Leave blank if you want this job to use any runner
-
-      .form-group
-        = job_form.label :refs, class: 'control-label' do
-          Refs
-        .col-sm-10
-          = job_form.text_field :refs, class: 'form-control', placeholder: "master, staging, feature/*, tags/testing*"
-          .help-block
-            Run only when the above git refs strings match the branch or tag that was pushed.
-            %br
-            Accepts strings and glob pattern syntax
-
-      .form-group
-        = job_form.label :commands, 'Script', class: 'control-label'
-        .col-sm-10
-          = job_form.text_area :commands, class: 'form-control', rows: 10, placeholder: "bundle exec rake spec"
-          %p.light
-            All lines will be concatenated into one file and executed.
-            %br
-            If you change the working directory or the environment this will affect all following lines.
-            %br
-            = link_to("Example job scripts", "https://gitlab.com/gitlab-org/gitlab-ci/tree/master/doc/examples")
-          = job_form.link_to_remove "Remove this job", class: 'btn btn-danger pull-right'
-      %hr
-  .clearfix
-    = f.link_to_add "Add a job", :jobs, class: 'btn btn-save pull-right'
-
-  .form-actions
-    = f.submit 'Save changes', class: 'btn btn-success'
-

app/views/jobs/_edit.html.haml

-= nested_form_for @project, html: { class: 'form-horizontal' } do |f|
-  - if @project.errors.any?
-    #error_explanation
-      %p.lead= "#{pluralize(@project.errors.count, "error")} prohibited this project from being saved:"
-      .alert.alert-error
-        %ul
-          - @project.errors.full_messages.each do |msg|
-            %li= msg
-
-  = f.fields_for :jobs do |job_form|
-    - if job_form.object.job_type == "parallel" || job_form.object.new_record?
-      .form-group
-        = job_form.label :name, 'Name', class: 'control-label'
-        .col-sm-10
-          = job_form.text_field :name, class: 'form-control', placeholder: "Ex. cucumber"
-      .form-group
-        = job_form.label :build_branches, 'Trigger', class: 'control-label'
-        .col-sm-10
-          .checkbox
-            = job_form.label :build_branches, 'Builds commits', class: ''
-            = job_form.check_box :build_branches
-          .checkbox
-            = job_form.label :build_tags, 'Build tags', class: ''