Added basic implementation of GitLab Chatops

The chatops solution is built on top of CI pipelines and essentially
acts as a different UI for scheduling pipelines manually. Pipelines
scheduled via chat have access to an environment variable called
"CHAT_INPUT". This variable stores the arguments that were passed to the
chatops command.

Output is retrieved by reading specific trace sections from the build
output. There are two sections supported (in this order):

1. chat_reply
2. build_script

The "chat_reply" is a trace section that one has to generate themselves,
"build_script" in turn is provided by default. The "build_script" trace
section also includes the output of any commands executed in the
"before_script" list. If one doesn't want to display this output they
should generate the custom "chat_reply" trace section.

Responses are sent back via Sidekiq using the ChatNotificationWorker
class. This ensures that existing CI related jobs won't get slowed down
if Slack (or another chat service) is not responding.
parent 68a13836
Pipeline #18426316 passed with stages
in 49 minutes and 49 seconds
......@@ -62,7 +62,8 @@ module Ci
schedule: 4,
api: 5,
external: 6,
pipeline: 7
pipeline: 7,
chat: 8
}
enum config_source: {
......
class SlackSlashCommandsService < SlashCommandsService
prepend EE::SlackSlashCommandsService
include TriggersHelper
def title
......
......@@ -3,6 +3,7 @@ module Ci
attr_reader :pipeline
SEQUENCE = [Gitlab::Ci::Pipeline::Chain::Build,
EE::Gitlab::Ci::Pipeline::Chain::RemoveUnwantedChatJobs,
Gitlab::Ci::Pipeline::Chain::Validate::Abilities,
Gitlab::Ci::Pipeline::Chain::Validate::Repository,
Gitlab::Ci::Pipeline::Chain::Validate::Config,
......@@ -29,7 +30,8 @@ module Ci
current_user: current_user,
# EE specific
allow_mirror_update: mirror_update
allow_mirror_update: mirror_update,
chat_data: params[:chat_data]
)
sequence = Gitlab::Ci::Pipeline::Chain::Sequence
......
......@@ -144,3 +144,4 @@
- rebase
- repository_update_mirror
- repository_update_remote_mirror
- chat_notification
class BuildFinishedWorker
prepend EE::BuildFinishedWorker
include ApplicationWorker
include PipelineQueue
......
......@@ -72,6 +72,7 @@
# EE-specific queues
- [ldap_group_sync, 2]
- [chat_notification, 2]
- [geo, 1]
- [repository_remove_remote, 1]
- [repository_update_mirror, 1]
......
......@@ -420,6 +420,14 @@ ActiveRecord::Schema.define(version: 20180301084653) do
add_index "ci_job_artifacts", ["job_id", "file_type"], name: "index_ci_job_artifacts_on_job_id_and_file_type", unique: true, using: :btree
add_index "ci_job_artifacts", ["project_id"], name: "index_ci_job_artifacts_on_project_id", using: :btree
create_table "ci_pipeline_chat_data", id: :bigserial, force: :cascade do |t|
t.integer "pipeline_id", null: false
t.integer "chat_name_id", null: false
t.text "response_url", null: false
end
add_index "ci_pipeline_chat_data", ["pipeline_id"], name: "index_ci_pipeline_chat_data_on_pipeline_id", unique: true, using: :btree
create_table "ci_pipeline_schedule_variables", force: :cascade do |t|
t.string "key", null: false
t.text "value"
......@@ -2505,6 +2513,8 @@ ActiveRecord::Schema.define(version: 20180301084653) do
add_foreign_key "ci_group_variables", "namespaces", column: "group_id", name: "fk_33ae4d58d8", on_delete: :cascade
add_foreign_key "ci_job_artifacts", "ci_builds", column: "job_id", on_delete: :cascade
add_foreign_key "ci_job_artifacts", "projects", on_delete: :cascade
add_foreign_key "ci_pipeline_chat_data", "chat_names", on_delete: :cascade
add_foreign_key "ci_pipeline_chat_data", "ci_pipelines", column: "pipeline_id", on_delete: :cascade
add_foreign_key "ci_pipeline_schedule_variables", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_41c35fda51", on_delete: :cascade
add_foreign_key "ci_pipeline_schedules", "projects", name: "fk_8ead60fcc4", on_delete: :cascade
add_foreign_key "ci_pipeline_schedules", "users", column: "owner_id", name: "fk_9ea99f58d2", on_delete: :nullify
......
module Ci
class PipelineChatData < ActiveRecord::Base
self.table_name = 'ci_pipeline_chat_data'
belongs_to :chat_name
validates :pipeline_id, presence: true
validates :chat_name_id, presence: true
validates :response_url, presence: true
end
end
module EE
module Ci
module Pipeline
extend ActiveSupport::Concern
EE_FAILURE_REASONS = {
activity_limit_exceeded: 20,
size_limit_exceeded: 21
}.freeze
included do
has_one :chat_data, class_name: 'Ci::PipelineChatData'
end
def predefined_variables
result = super
result << { key: 'CI_PIPELINE_SOURCE', value: source.to_s, public: true }
......
module EE
module SlackSlashCommandsService
def chat_responder
::Gitlab::Chat::Responder::Slack
end
end
end
......@@ -62,6 +62,7 @@ class License < ActiveRecord::Base
dast
epics
ide
chatops
].freeze
# List all features available for early adopters,
......@@ -316,6 +317,7 @@ class License < ActiveRecord::Base
def reset_current
self.class.reset_current
Gitlab::Chat.flush_available_cache
end
def reset_license
......
# frozen_string_literal: true
class ChatNotificationWorker
include ApplicationWorker
RESCHEDULE_INTERVAL = 2.seconds
def perform(build_id)
Ci::Build.find_by(id: build_id).try do |build|
send_response(build)
end
rescue Gitlab::Chat::Output::MissingBuildSectionError
# The creation of traces and sections appears to be eventually consistent.
# As a result it's possible for us to run the above code before the trace
# sections are present. To better handle such cases we'll just reschedule
# the job instead of producing an error.
self.class.perform_in(RESCHEDULE_INTERVAL, build_id)
end
def send_response(build)
Gitlab::Chat::Responder.responder_for(build).try do |responder|
if build.success?
output = Gitlab::Chat::Output.new(build)
responder.success(output.to_s)
else
responder.failure
end
end
end
end
module EE
module BuildFinishedWorker
def perform(build_id)
super
::Ci::Build.find_by(id: build_id).try do |build|
ChatNotificationWorker.perform_async(build_id) if build.pipeline.chat?
end
end
end
end
---
title: Added basic implementation of GitLab Chatops
merge_request:
author:
type: added
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class CreateChatopsTables < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
# Set this constant to true if this migration requires downtime.
DOWNTIME = false
def change
create_table :ci_pipeline_chat_data, id: :bigserial do |t|
t.integer :pipeline_id, null: false
t.references :chat_name, foreign_key: { on_delete: :cascade }, null: false
t.text :response_url, null: false
# A pipeline can only contain one row in this table, hence this index is
# unique.
t.index :pipeline_id, unique: true
end
add_foreign_key :ci_pipeline_chat_data, :ci_pipelines,
column: :pipeline_id,
on_delete: :cascade
end
end
module EE
module Gitlab
module Ci
module Pipeline
module Chain
class RemoveUnwantedChatJobs < ::Gitlab::Ci::Pipeline::Chain::Base
def perform!
return unless pipeline.config_processor && pipeline.chat?
# When scheduling a chat pipeline we only want to run the build
# that matches the chat command.
pipeline.config_processor.jobs.select! do |name, _|
name.to_s == command.chat_data[:command].to_s
end
end
def break?
false
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Chat
CACHE_TTL = 1.hour.to_i
AVAILABLE_CACHE_KEY = :gitlab_chat_available
# Returns `true` if Chatops is available for the current instance.
def self.available?
# We anticipate this code to be called rather frequently, especially on
# large instances such as GitLab.com. To reduce database load we cache the
# output for a while.
Rails.cache.fetch(AVAILABLE_CACHE_KEY, expires_in: CACHE_TTL) do
::License.feature_available?(:chatops)
end
end
def self.flush_available_cache
Rails.cache.delete(AVAILABLE_CACHE_KEY)
end
end
end
# frozen_string_literal: true
module Gitlab
module Chat
# Class for scheduling chat pipelines.
#
# A Command takes care of creating a `Ci::Pipeline` with all the data
# necessary to execute a chat command. This includes data such as the chat
# data (e.g. the response URL) and any environment variables that should be
# exposed to the chat command.
class Command
include Utils::StrongMemoize
attr_reader :project, :chat_name, :name, :arguments, :response_url,
:channel
# project - The Project to schedule the command for.
# chat_name - The ChatName belonging to the user that scheduled the
# command.
# name - The name of the chat command to run.
# arguments - The arguments (as a String) to pass to the command.
# channel - The channel the message was sent from.
# response_url - The URL to send the response back to.
def initialize(project:, chat_name:, name:, arguments:, channel:, response_url:)
@project = project
@chat_name = chat_name
@name = name
@arguments = arguments
@channel = channel
@response_url = response_url
end
# Tries to create a new pipeline.
#
# This method will return a pipeline that _may_ be persisted, or `nil` if
# the pipeline could not be created.
def try_create_pipeline
return unless valid?
create_pipeline
end
def create_pipeline
service = ::Ci::CreatePipelineService.new(
project,
chat_name.user,
ref: branch,
sha: commit,
chat_data: {
chat_name_id: chat_name.id,
command: name,
arguments: arguments,
response_url: response_url
}
)
service.execute(:chat) do |pipeline|
create_environment_variables(pipeline)
create_chat_data(pipeline)
end
end
# pipeline - The `Ci::Pipeline` to create the environment variables for.
def create_environment_variables(pipeline)
pipeline.variables.create!(
[
{ key: 'CHAT_INPUT', value: arguments },
{ key: 'CHAT_CHANNEL', value: channel }
]
)
end
# pipeline - The `Ci::Pipeline` to create the chat data for.
def create_chat_data(pipeline)
pipeline.create_chat_data!(
chat_name_id: chat_name.id,
response_url: response_url
)
end
def valid?
branch && commit
end
def branch
strong_memoize(:branch) { project.default_branch }
end
def commit
strong_memoize(:commit) do
project.commit(branch)&.id if branch
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Chat
# Class for gathering and formatting the output of a `Ci::Build`.
class Output
attr_reader :build
MissingBuildSectionError = Class.new(StandardError)
# The primary trace section to look for.
PRIMARY_SECTION = 'chat_reply'
# The backup trace section in case the primary one could not be found.
FALLBACK_SECTION = 'build_script'
# build - The `Ci::Build` to obtain the output from.
def initialize(build)
@build = build
end
# Returns a `String` containing the output of the build.
#
# The output _does not_ include the command that was executed.
def to_s
offset, length = read_offset_and_length
trace.read do |stream|
stream.seek(offset)
output = stream
.stream
.read(length)
.force_encoding(Encoding.default_external)
without_executed_command_line(output)
end
end
# Returns the offset to seek to and the number of bytes to read relative
# to the offset.
def read_offset_and_length
section = find_build_trace_section(PRIMARY_SECTION) ||
find_build_trace_section(FALLBACK_SECTION)
unless section
raise(
MissingBuildSectionError,
"The build_script trace section could not be found for build #{build.id}"
)
end
length = section[:byte_end] - section[:byte_start]
[section[:byte_start], length]
end
# Removes the line containing the executed command from the build output.
#
# output - A `String` containing the output of a trace section.
def without_executed_command_line(output)
output.split("\n")[1..-1].join("\n")
end
# Returns the trace section for the given name, or `nil` if the section
# could not be found.
#
# name - The name of the trace section to find.
def find_build_trace_section(name)
trace_sections.find { |s| s[:name] == name }
end
def trace_sections
@trace_sections ||= trace.extract_sections
end
def trace
@trace ||= build.trace
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Chat
module Responder
# Returns an instance of the responder to use for generating chat
# responses.
#
# This method will return `nil` if no formatter is available for the given
# build.
#
# build - A `Ci::Build` that executed a chat command.
def self.responder_for(build)
service = build.pipeline.chat_data&.chat_name&.service
if (responder = service.try(:chat_responder))
responder.new(build)
end
end
end
end
end
module Gitlab
module Chat
module Responder
class Base
attr_reader :build
# build - The `Ci::Build` that was executed.
def initialize(build)
@build = build
end
def pipeline
build.pipeline
end
def project
pipeline.project
end
def success(*)
raise NotImplementedError, 'You must implement #success(output)'
end
def failure
raise NotImplementedError, 'You must implement #failure'
end
def send_response(output)
raise NotImplementedError, 'You must implement #send_response(output)'
end
def scheduled_output
raise NotImplementedError, 'You must implement #scheduled_output'
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Chat
module Responder
class Slack < Responder::Base
SUCCESS_COLOR = '#B3ED8E'
FAILURE_COLOR = '#FF5640'
RESPONSE_TYPE = :in_channel
# Slack breaks messages apart if they're around 4 KB in size. We use a
# slightly smaller limit here to account for user mentions.
MESSAGE_SIZE_LIMIT = 3.5.kilobytes
# Sends a response back to Slack
#
# output - The output to send back to Slack, as a Hash.
def send_response(output)
HTTParty.post(
pipeline.chat_data.response_url,
{
headers: { Accept: 'application/json' },
body: output.to_json
}
)
end
# Sends the output for a build that completed successfully.
#
# output - The output produced by the chat command.
def success(output)
output =
if output.empty?
'The command successfully completed but did not ' \
'write any data to STDOUT or STDERR.'
else
limit_output(output)
end
send_response(
text: message_text(output),
response_type: RESPONSE_TYPE,
attachments: [
{
color: SUCCESS_COLOR,
actions: [
view_project_button,
view_pipeline_button,
view_build_button
]
}
]
)
end
# Sends the output for a build that failed.
def failure
send_response(
text: message_text('Sorry, the build failed!'),
response_type: RESPONSE_TYPE,
attachments: [
{
color: FAILURE_COLOR,
actions: [
view_project_button,
view_pipeline_button,
view_build_button
]
}
]
)
end
# Returns the output to send back after a command has been scheduled.
def scheduled_output
{
text: message_text('The command has been scheduled!'),
attachments: [
{
actions: [
view_project_button,
view_pipeline_button,
view_build_button
]
}
]
}
end
private
def limit_output(output)
if output.bytesize <= MESSAGE_SIZE_LIMIT
output
else
'The command output is too large to be sent back directly. ' \
"The full output can be found at #{build_url}"
end
end
def mention_user
"<@#{pipeline.chat_data.chat_name.chat_id}>"
end
def message_text(output)
"#{mention_user}: #{output}"
end
def view_project_button
{
type: :button,
text: 'View Project',
url: url_helpers.project_url(project)
}
end
def view_pipeline_button
{
type: :button,
text: 'View Pipeline',
url: url_helpers.project_pipeline_url(project, pipeline)
}
end
def view_build_button
{
type: :button,
text: 'View Build',
url: build_url
}
end
def build_url
url_helpers.project_build_url(project, build)
end
def url_helpers
::Gitlab::Routing.url_helpers
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module SlashCommands
module Presenters
class Run < Presenters::Base
def present(pipeline)
build = pipeline.builds.take
if build && (responder = Chat::Responder.responder_for(build))
in_channel_response(responder.scheduled_output)
else
unsupported_chat_service
end
end
def unsupported_chat_service
ephemeral_response(text: 'Sorry, this chat service is currently not supported by GitLab ChatOps.')
end
def failed_to_schedule(command)
ephemeral_response(
text: 'The command could not be scheduled. Make sure that your ' \
'project has a .gitlab-ci.yml that defines a job with the ' \
"name #{command.inspect}"
)
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module SlashCommands
# Slash command for triggering chatops jobs.
class Run < BaseCommand
def self.match(text)
/\Arun\s+(?<command>\S+)(\s+(?<arguments>.+))?\z/.match(text)
end
def self.help_message
'run <command> <arguments>'
end
def self.available?(project)
Chat.available? && project.builds_enabled?
end
def self.allowed?(project, user)
can?(user, :create_pipeline, project)
end
def execute(match)
command = Chat::Command.new(
project: project,
chat_name: chat_name,
name: match[:command],
arguments: match[:arguments],
channel: params[:channel_id],
response_url: params[:response_url]
)
presenter = Gitlab::SlashCommands::Presenters::Run.new
pipeline = command.try_create_pipeline