Commit 6e2f2f5d authored by Robert Speicher's avatar Robert Speicher

Merge branch 'dm-fix-web-edit-new-lines' into 'master'

Consistently create, update, and delete files, taking CRLF settings into account

See merge request !9207
parents 1f8d6c79 705f1558
Pipeline #6678669 passed with stages
in 83 minutes and 35 seconds
......@@ -746,136 +746,63 @@ class Repository
@tags ||= raw_repository.tags
end
# rubocop:disable Metrics/ParameterLists
def commit_dir(
user, path,
message:, branch_name:,
author_email: nil, author_name: nil,
start_branch_name: nil, start_project: project)
check_tree_entry_for_dir(branch_name, path)
if start_branch_name
start_project.repository.
check_tree_entry_for_dir(start_branch_name, path)
end
def create_dir(user, path, **options)
options[:user] = user
options[:actions] = [{ action: :create_dir, file_path: path }]
commit_file(
user,
"#{path}/.gitkeep",
'',
message: message,
branch_name: branch_name,
update: false,
author_email: author_email,
author_name: author_name,
start_branch_name: start_branch_name,
start_project: start_project)
multi_action(**options)
end
# rubocop:enable Metrics/ParameterLists
# rubocop:disable Metrics/ParameterLists
def commit_file(
user, path, content,
message:, branch_name:, update: true,
author_email: nil, author_name: nil,
start_branch_name: nil, start_project: project)
unless update
error_message = "Filename already exists; update not allowed"
def create_file(user, path, content, **options)
options[:user] = user
options[:actions] = [{ action: :create, file_path: path, content: content }]
if tree_entry_at(branch_name, path)
raise Gitlab::Git::Repository::InvalidBlobName.new(error_message)
end
multi_action(**options)
end
if start_branch_name &&
start_project.repository.tree_entry_at(start_branch_name, path)
raise Gitlab::Git::Repository::InvalidBlobName.new(error_message)
end
end
def update_file(user, path, content, **options)
previous_path = options.delete(:previous_path)
action = previous_path && previous_path != path ? :move : :update
multi_action(
user: user,
message: message,
branch_name: branch_name,
author_email: author_email,
author_name: author_name,
start_branch_name: start_branch_name,
start_project: start_project,
actions: [{ action: :create,
file_path: path,
content: content }])
end
# rubocop:enable Metrics/ParameterLists
options[:user] = user
options[:actions] = [{ action: action, file_path: path, previous_path: previous_path, content: content }]
# rubocop:disable Metrics/ParameterLists
def update_file(
user, path, content,
message:, branch_name:, previous_path:,
author_email: nil, author_name: nil,
start_branch_name: nil, start_project: project)
action = if previous_path && previous_path != path
:move
else
:update
end
multi_action(
user: user,
message: message,
branch_name: branch_name,
author_email: author_email,
author_name: author_name,
start_branch_name: start_branch_name,
start_project: start_project,
actions: [{ action: action,
file_path: path,
content: content,
previous_path: previous_path }])
multi_action(**options)
end
# rubocop:enable Metrics/ParameterLists
# rubocop:disable Metrics/ParameterLists
def remove_file(
user, path,
message:, branch_name:,
author_email: nil, author_name: nil,
start_branch_name: nil, start_project: project)
multi_action(
user: user,
message: message,
branch_name: branch_name,
author_email: author_email,
author_name: author_name,
start_branch_name: start_branch_name,
start_project: start_project,
actions: [{ action: :delete,
file_path: path }])
def delete_file(user, path, **options)
options[:user] = user
options[:actions] = [{ action: :delete, file_path: path }]
multi_action(**options)
end
# rubocop:enable Metrics/ParameterLists
# rubocop:disable Metrics/ParameterLists
def multi_action(
user:, branch_name:, message:, actions:,
author_email: nil, author_name: nil,
start_branch_name: nil, start_project: project)
GitOperationService.new(user, self).with_branch(
branch_name,
start_branch_name: start_branch_name,
start_project: start_project) do |start_commit|
index = rugged.index
parents = if start_commit
index.read_tree(start_commit.raw_commit.tree)
[start_commit.sha]
else
[]
end
index = Gitlab::Git::Index.new(raw_repository)
actions.each do |act|
git_action(index, act)
if start_commit
index.read_tree(start_commit.raw_commit.tree)
parents = [start_commit.sha]
else
parents = []
end
actions.each do |options|
index.public_send(options.delete(:action), options)
end
options = {
tree: index.write_tree(rugged),
tree: index.write_tree,
message: message,
parents: parents
}
......@@ -1166,30 +1093,6 @@ class Repository
blob_data_at(sha, '.gitlab-ci.yml')
end
protected
def tree_entry_at(branch_name, path)
branch_exists?(branch_name) &&
# tree_entry is private
raw_repository.send(:tree_entry, commit(branch_name), path)
end
def check_tree_entry_for_dir(branch_name, path)
return unless branch_exists?(branch_name)
entry = tree_entry_at(branch_name, path)
return unless entry
if entry[:type] == :blob
raise Gitlab::Git::Repository::InvalidBlobName.new(
"Directory already exists as a file")
else
raise Gitlab::Git::Repository::InvalidBlobName.new(
"Directory already exists")
end
end
private
def blob_data_at(sha, path)
......@@ -1200,58 +1103,6 @@ class Repository
blob.data
end
def git_action(index, action)
path = normalize_path(action[:file_path])
if action[:action] == :move
previous_path = normalize_path(action[:previous_path])
end
case action[:action]
when :create, :update, :move
mode =
case action[:action]
when :update
index.get(path)[:mode]
when :move
index.get(previous_path)[:mode]
end
mode ||= 0o100644
index.remove(previous_path) if action[:action] == :move
content = if action[:encoding] == 'base64'
Base64.decode64(action[:content])
else
action[:content]
end
detect = CharlockHolmes::EncodingDetector.new.detect(content) if content
unless detect && detect[:type] == :binary
# When writing to the repo directly as we are doing here,
# the `core.autocrlf` config isn't taken into account.
content.gsub!("\r\n", "\n") if self.autocrlf
end
oid = rugged.write(content, :blob)
index.add(path: path, oid: oid, mode: mode)
when :delete
index.remove(path)
end
end
def normalize_path(path)
pathname = Gitlab::Git::PathHelper.normalize_path(path)
if pathname.each_filename.include?('..')
raise Gitlab::Git::Repository::InvalidBlobName.new('Invalid path')
end
pathname.to_s
end
def refs_directory_exists?
return false unless path_with_namespace
......
module Files
class CreateDirService < Files::BaseService
def commit
repository.commit_dir(
repository.create_dir(
current_user,
@file_path,
message: @commit_message,
......
module Files
class CreateService < Files::BaseService
def commit
repository.commit_file(
repository.create_file(
current_user,
@file_path,
@file_content,
message: @commit_message,
branch_name: @target_branch,
update: false,
author_email: @author_email,
author_name: @author_name,
start_project: @start_project,
......@@ -17,6 +16,10 @@ module Files
def validate
super
if @file_content.nil?
raise_error("You must provide content.")
end
if @file_path =~ Gitlab::Regex.directory_traversal_regex
raise_error(
'Your changes could not be committed, because the file name ' +
......
module Files
class DestroyService < Files::BaseService
def commit
repository.remove_file(
repository.delete_file(
current_user,
@file_path,
message: @commit_message,
......
......@@ -2,6 +2,8 @@ module Files
class MultiService < Files::BaseService
class FileChangedError < StandardError; end
ACTIONS = %w[create update delete move].freeze
def commit
repository.multi_action(
user: current_user,
......@@ -21,10 +23,19 @@ module Files
super
params[:actions].each_with_index do |action, index|
if ACTIONS.include?(action[:action].to_s)
action[:action] = action[:action].to_sym
else
raise_error("Unknown action type `#{action[:action]}`.")
end
unless action[:file_path].present?
raise_error("You must specify a file_path.")
end
action[:file_path].slice!(0) if action[:file_path] && action[:file_path].start_with?('/')
action[:previous_path].slice!(0) if action[:previous_path] && action[:previous_path].start_with?('/')
regex_check(action[:file_path])
regex_check(action[:previous_path]) if action[:previous_path]
......@@ -43,8 +54,6 @@ module Files
validate_delete(action)
when :move
validate_move(action, index)
else
raise_error("Unknown action type `#{action[:action]}`.")
end
end
end
......@@ -92,6 +101,20 @@ module Files
if repository.blob_at_branch(params[:branch], action[:file_path])
raise_error("Your changes could not be committed because a file with the name `#{action[:file_path]}` already exists.")
end
if action[:content].nil?
raise_error("You must provide content.")
end
end
def validate_update(action)
if action[:content].nil?
raise_error("You must provide content.")
end
if file_has_changed?
raise FileChangedError.new("You are attempting to update a file `#{action[:file_path]}` that has changed since you started editing it.")
end
end
def validate_delete(action)
......@@ -114,11 +137,5 @@ module Files
params[:actions][index][:content] = blob.data
end
end
def validate_update(action)
if file_has_changed?
raise FileChangedError.new("You are attempting to update a file `#{action[:file_path]}` that has changed since you started editing it.")
end
end
end
end
......@@ -18,6 +18,10 @@ module Files
def validate
super
if @file_content.nil?
raise_error("You must provide content.")
end
if file_has_changed?
raise FileChangedError.new("You are attempting to update a file that has changed since you started editing it.")
end
......
......@@ -155,17 +155,9 @@ class Gitlab::Seeder::CycleAnalytics
issue.project.repository.add_branch(@user, branch_name, 'master')
options = {
committer: issue.project.repository.user_to_committer(@user),
author: issue.project.repository.user_to_committer(@user),
commit: { message: "Commit for ##{issue.iid}", branch: branch_name, update_ref: true },
file: { content: "content", path: filename, update: false }
}
commit_sha = Gitlab::Git::Blob.commit(issue.project.repository, options)
commit_sha = issue.project.repository.create_file(@user, filename, "content", options, message: "Commit for ##{issue.iid}", branch_name: branch_name)
issue.project.repository.commit(commit_sha)
GitPushService.new(issue.project,
@user,
oldrev: issue.project.repository.commit("master").sha,
......
......@@ -52,13 +52,6 @@ module API
attrs = declared_params.merge(start_branch: declared_params[:branch], target_branch: declared_params[:branch])
attrs[:actions].map! do |action|
action[:action] = action[:action].to_sym
action[:file_path].slice!(0) if action[:file_path] && action[:file_path].start_with?('/')
action[:previous_path].slice!(0) if action[:previous_path] && action[:previous_path].start_with?('/')
action
end
result = ::Files::MultiService.new(user_project, current_user, attrs).execute
if result[:status] == :success
......
......@@ -55,13 +55,6 @@ module API
branch = attrs.delete(:branch_name)
attrs.merge!(branch: branch, start_branch: branch, target_branch: branch)
attrs[:actions].map! do |action|
action[:action] = action[:action].to_sym
action[:file_path].slice!(0) if action[:file_path] && action[:file_path].start_with?('/')
action[:previous_path].slice!(0) if action[:previous_path] && action[:previous_path].start_with?('/')
action
end
result = ::Files::MultiService.new(user_project, current_user, attrs).execute
if result[:status] == :success
......
......@@ -93,163 +93,6 @@ module Gitlab
commit_id: sha,
)
end
# Commit file in repository and return commit sha
#
# options should contain next structure:
# file: {
# content: 'Lorem ipsum...',
# path: 'documents/story.txt',
# update: true
# },
# author: {
# email: 'user@example.com',
# name: 'Test User',
# time: Time.now
# },
# committer: {
# email: 'user@example.com',
# name: 'Test User',
# time: Time.now
# },
# commit: {
# message: 'Wow such commit',
# branch: 'master',
# update_ref: false
# }
#
# rubocop:disable Metrics/AbcSize
# rubocop:disable Metrics/CyclomaticComplexity
# rubocop:disable Metrics/PerceivedComplexity
def commit(repository, options, action = :add)
file = options[:file]
update = file[:update].nil? ? true : file[:update]
author = options[:author]
committer = options[:committer]
commit = options[:commit]
repo = repository.rugged
ref = commit[:branch]
update_ref = commit[:update_ref].nil? ? true : commit[:update_ref]
parents = []
mode = 0o100644
unless ref.start_with?('refs/')
ref = 'refs/heads/' + ref
end
path_name = Gitlab::Git::PathHelper.normalize_path(file[:path])
# Abort if any invalid characters remain (e.g. ../foo)
raise Gitlab::Git::Repository::InvalidBlobName.new("Invalid path") if path_name.each_filename.to_a.include?('..')
filename = path_name.to_s
index = repo.index
unless repo.empty?
rugged_ref = repo.references[ref]
raise Gitlab::Git::Repository::InvalidRef.new("Invalid branch name") unless rugged_ref
last_commit = rugged_ref.target
index.read_tree(last_commit.tree)
parents = [last_commit]
end
if action == :remove
index.remove(filename)
else
file_entry = index.get(filename)
if action == :rename
old_path_name = Gitlab::Git::PathHelper.normalize_path(file[:previous_path])
old_filename = old_path_name.to_s
file_entry = index.get(old_filename)
index.remove(old_filename) unless file_entry.blank?
end
if file_entry
raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists; update not allowed") unless update
# Preserve the current file mode if one is available
mode = file_entry[:mode] if file_entry[:mode]
end
content = file[:content]
detect = CharlockHolmes::EncodingDetector.new.detect(content) if content
unless detect && detect[:type] == :binary
# When writing to the repo directly as we are doing here,
# the `core.autocrlf` config isn't taken into account.
content.gsub!("\r\n", "\n") if repository.autocrlf
end
oid = repo.write(content, :blob)
index.add(path: filename, oid: oid, mode: mode)
end
opts = {}
opts[:tree] = index.write_tree(repo)
opts[:author] = author
opts[:committer] = committer
opts[:message] = commit[:message]
opts[:parents] = parents
opts[:update_ref] = ref if update_ref
Rugged::Commit.create(repo, opts)
end
# rubocop:enable Metrics/AbcSize
# rubocop:enable Metrics/CyclomaticComplexity
# rubocop:enable Metrics/PerceivedComplexity
# Remove file from repository and return commit sha
#
# options should contain next structure:
# file: {
# path: 'documents/story.txt'
# },
# author: {
# email: 'user@example.com',
# name: 'Test User',
# time: Time.now
# },
# committer: {
# email: 'user@example.com',
# name: 'Test User',
# time: Time.now
# },
# commit: {
# message: 'Remove FILENAME',
# branch: 'master'
# }
#
def remove(repository, options)
commit(repository, options, :remove)
end
# Rename file from repository and return commit sha
#
# options should contain next structure:
# file: {
# previous_path: 'documents/old_story.txt'
# path: 'documents/story.txt'
# content: 'Lorem ipsum...',
# update: true
# },
# author: {
# email: 'user@example.com',
# name: 'Test User',
# time: Time.now
# },
# committer: {
# email: 'user@example.com',
# name: 'Test User',
# time: Time.now
# },
# commit: {
# message: 'Rename FILENAME',
# branch: 'master'
# }
#
def rename(repository, options)
commit(repository, options, :rename)
end
end
def initialize(options)
......
module Gitlab
module Git
class Index
DEFAULT_MODE = 0o100644
attr_reader :repository, :raw_index
def initialize(repository)
@repository = repository
@raw_index = repository.rugged.index
end
delegate :read_tree, :get, to: :raw_index
def write_tree
raw_index.write_tree(repository.rugged)
end
def dir_exists?(path)
raw_index.find { |entry| entry[:path].start_with?("#{path}/") }
end
def create(options)
options = normalize_options(options)
file_entry = get(options[:file_path])
if file_entry
raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists")
end
add_blob(options)
end
def create_dir(options)
options = normalize_options(options)
file_entry = get(options[:file_path])
if file_entry
raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists as a file")
end
if dir_exists?(options[:file_path])
raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists")
end
options = options.dup
options[:file_path] += '/.gitkeep'
options[:content] = ''
add_blob(options)
end
def update(options)
options = normalize_options(options)
file_entry = get(options[:file_path])
unless file_entry
raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist")
end
add_blob(options, mode: file_entry[:mode])
end
def move(options)
options = normalize_options(options)
file_entry = get(options[:previous_path])
unless file_entry
raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist")
end
raw_index.remove(options[:previous_path])
add_blob(options, mode: file_entry[:mode])
end
def delete(options)
options = normalize_options(options)
file_entry = get(options[:file_path])
unless file_entry
raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist")
end
raw_index.remove(options[:file_path])
end
private
def normalize_options(options)
options = options.dup
options[:file_path] = normalize_path(options[:file_path]) if options[:file_path]
options[:previous_path] = normalize_path(options[:previous_path]) if options[:previous_path]
options
end
def normalize_path(path)
pathname = Gitlab::Git::PathHelper.normalize_path(path.dup)
if pathname.each_filename.include?('..')
raise Gitlab::Git::Repository::InvalidBlobName.new('Invalid path')
end
pathname.to_s
end
def add_blob(options, mode: nil)
content = options[:content]
content = Base64.decode64(content) if options[:encoding] == 'base64'
detect = CharlockHolmes::EncodingDetector.new.detect(content)
unless detect && detect[:type] == :binary
# When writing to the repo directly as we are doing here,
# the `core.autocrlf` config isn't taken into account.
content.gsub!("\r\n", "\n") if repository.autocrlf
end