Commit 0b849e8d authored by Dmitriy Zaporozhets's avatar Dmitriy Zaporozhets

Merge branch 'jej/commit-api-tracks-lfs' into 'master'

Create commit API and Web IDE obey LFS filters

Closes #42287

See merge request gitlab-org/gitlab-ce!16718
parents 40e9e440 c1e39421
Pipeline #19025726 passed with stages
in 65 minutes and 1 second
module Files
class CreateService < Files::BaseService
def create_commit!
handler = Lfs::FileModificationHandler.new(project, @branch_name)
transformer = Lfs::FileTransformer.new(project, @branch_name)
handler.new_file(@file_path, @file_content) do |content_or_lfs_pointer|
create_transformed_commit(content_or_lfs_pointer)
end
result = transformer.new_file(@file_path, @file_content)
create_transformed_commit(result.content)
end
private
......
......@@ -3,11 +3,33 @@ module Files
UPDATE_FILE_ACTIONS = %w(update move delete).freeze
def create_commit!
transformer = Lfs::FileTransformer.new(project, @branch_name)
actions = actions_after_lfs_transformation(transformer, params[:actions])
commit_actions!(actions)
end
private
def actions_after_lfs_transformation(transformer, actions)
actions.map do |action|
if action[:action] == 'create'
result = transformer.new_file(action[:file_path], action[:content], encoding: action[:encoding])
action[:content] = result.content
action[:encoding] = result.encoding
end
action
end
end
def commit_actions!(actions)
repository.multi_action(
current_user,
message: @commit_message,
branch_name: @branch_name,
actions: params[:actions],
actions: actions,
author_email: @author_email,
author_name: @author_name,
start_project: @start_project,
......@@ -17,8 +39,6 @@ module Files
raise_error(e)
end
private
def validate!
super
......
module Lfs
class FileModificationHandler
# Usage: Calling `new_file` check to see if a file should be in LFS and
# return a transformed result with `content` and `encoding` to commit.
#
# For LFS an LfsObject linked to the project is stored and an LFS
# pointer returned. If the file isn't in LFS the untransformed content
# is returned to save in the commit.
#
# transformer = Lfs::FileTransformer.new(project, @branch_name)
# content_or_lfs_pointer = transformer.new_file(file_path, content).content
# create_transformed_commit(content_or_lfs_pointer)
#
class FileTransformer
attr_reader :project, :branch_name
delegate :repository, to: :project
......@@ -9,24 +20,37 @@ module Lfs
@branch_name = branch_name
end
def new_file(file_path, file_content)
def new_file(file_path, file_content, encoding: nil)
if project.lfs_enabled? && lfs_file?(file_path)
file_content = Base64.decode64(file_content) if encoding == 'base64'
lfs_pointer_file = Gitlab::Git::LfsPointerFile.new(file_content)
lfs_object = create_lfs_object!(lfs_pointer_file, file_content)
content = lfs_pointer_file.pointer
success = yield(content)
link_lfs_object!(lfs_object)
link_lfs_object!(lfs_object) if success
Result.new(content: lfs_pointer_file.pointer, encoding: 'text')
else
yield(file_content)
Result.new(content: file_content, encoding: encoding)
end
end
class Result
attr_reader :content, :encoding
def initialize(content:, encoding:)
@content = content
@encoding = encoding
end
end
private
def lfs_file?(file_path)
repository.attributes_at(branch_name, file_path)['filter'] == 'lfs'
cached_attributes.attributes(file_path)['filter'] == 'lfs'
end
def cached_attributes
@cached_attributes ||= Gitlab::Git::AttributesAtRefParser.new(repository, branch_name)
end
def create_lfs_object!(lfs_pointer_file, file_content)
......
---
title: Create commit API and Web IDE obey LFS filters
merge_request: 16718
author:
type: fixed
module Gitlab
module Git
class LfsPointerFile
VERSION = "https://git-lfs.github.com/spec/v1".freeze
VERSION_LINE = "version #{VERSION}".freeze
def initialize(data)
@data = data
end
def pointer
@pointer ||= <<~FILE
version https://git-lfs.github.com/spec/v1
#{VERSION_LINE}
oid sha256:#{sha256}
size #{size}
FILE
......@@ -20,6 +23,10 @@ module Gitlab
def sha256
@sha256 ||= Digest::SHA256.hexdigest(@data)
end
def inspect
"#<#{self.class}:#{object_id} @size=#{size}, @sha256=#{sha256.inspect}>"
end
end
end
end
......@@ -1002,8 +1002,9 @@ module Gitlab
# This only checks the root .gitattributes file,
# it does not traverse subfolders to find additional .gitattributes files
#
# This method is around 30 times slower than `attributes`,
# which uses `$GIT_DIR/info/attributes`
# This method is around 30 times slower than `attributes`, which uses
# `$GIT_DIR/info/attributes`. Consider caching AttributesAtRefParser
# and reusing that for multiple calls instead of this method.
def attributes_at(ref, file_path)
parser = AttributesAtRefParser.new(self, ref)
parser.attributes(file_path)
......
......@@ -43,7 +43,7 @@ describe Files::CreateService do
blob = repository.blob_at('lfs', file_path)
expect(blob.data).not_to start_with('version https://git-lfs.github.com/spec/v1')
expect(blob.data).not_to start_with(Gitlab::Git::LfsPointerFile::VERSION_LINE)
expect(blob.data).to eq(file_content)
end
end
......@@ -58,7 +58,7 @@ describe Files::CreateService do
blob = repository.blob_at('lfs', file_path)
expect(blob.data).to start_with('version https://git-lfs.github.com/spec/v1')
expect(blob.data).to start_with(Gitlab::Git::LfsPointerFile::VERSION_LINE)
end
it "creates an LfsObject with the file's content" do
......
......@@ -4,28 +4,30 @@ describe Files::MultiService do
subject { described_class.new(project, user, commit_params) }
let(:project) { create(:project, :repository) }
let(:repository) { project.repository }
let(:user) { create(:user) }
let(:branch_name) { project.default_branch }
let(:original_file_path) { 'files/ruby/popen.rb' }
let(:new_file_path) { 'files/ruby/popen.rb' }
let(:file_content) { 'New content' }
let(:action) { 'update' }
let!(:original_commit_id) do
Gitlab::Git::Commit.last_for_path(project.repository, branch_name, original_file_path).sha
end
let(:actions) do
[
{
action: action,
file_path: new_file_path,
previous_path: original_file_path,
content: 'New content',
last_commit_id: original_commit_id
}
]
let(:default_action) do
{
action: action,
file_path: new_file_path,
previous_path: original_file_path,
content: file_content,
last_commit_id: original_commit_id
}
end
let(:actions) { [default_action] }
let(:commit_params) do
{
commit_message: "Update File",
......@@ -110,6 +112,56 @@ describe Files::MultiService do
end
end
context 'when creating a file matching an LFS filter' do
let(:action) { 'create' }
let(:branch_name) { 'lfs' }
let(:new_file_path) { 'test_file.lfs' }
before do
allow(project).to receive(:lfs_enabled?).and_return(true)
end
it 'creates an LFS pointer' do
subject.execute
blob = repository.blob_at('lfs', new_file_path)
expect(blob.data).to start_with(Gitlab::Git::LfsPointerFile::VERSION_LINE)
end
it "creates an LfsObject with the file's content" do
subject.execute
expect(LfsObject.last.file.read).to eq file_content
end
context 'with base64 encoded content' do
let(:raw_file_content) { 'Raw content' }
let(:file_content) { Base64.encode64(raw_file_content) }
let(:actions) { [default_action.merge(encoding: 'base64')] }
it 'creates an LFS pointer' do
subject.execute
blob = repository.blob_at('lfs', new_file_path)
expect(blob.data).to start_with(Gitlab::Git::LfsPointerFile::VERSION_LINE)
end
it "creates an LfsObject with the file's content" do
subject.execute
expect(LfsObject.last.file.read).to eq raw_file_content
end
end
it 'links the LfsObject to the project' do
expect do
subject.execute
end.to change { project.lfs_objects.count }.by(1)
end
end
context 'when file status validation is skipped' do
let(:action) { 'create' }
let(:new_file_path) { 'files/ruby/new_file.rb' }
......
require "spec_helper"
describe Lfs::FileTransformer do
let(:project) { create(:project, :repository) }
let(:repository) { project.repository }
let(:file_content) { 'Test file content' }
let(:branch_name) { 'lfs' }
let(:file_path) { 'test_file.lfs' }
subject { described_class.new(project, branch_name) }
describe '#new_file' do
context 'with lfs disabled' do
it 'skips gitattributes check' do
expect(repository.raw).not_to receive(:blob_at)
subject.new_file(file_path, file_content)
end
it 'returns untransformed content' do
result = subject.new_file(file_path, file_content)
expect(result.content).to eq(file_content)
end
it 'returns untransformed encoding' do
result = subject.new_file(file_path, file_content, encoding: 'base64')
expect(result.encoding).to eq('base64')
end
end
context 'with lfs enabled' do
before do
allow(project).to receive(:lfs_enabled?).and_return(true)
end
it 'reuses cached gitattributes' do
second_file = 'another_file.lfs'
expect(repository.raw).to receive(:blob_at).with(branch_name, '.gitattributes').once
subject.new_file(file_path, file_content)
subject.new_file(second_file, file_content)
end
it "creates an LfsObject with the file's content" do
subject.new_file(file_path, file_content)
expect(LfsObject.last.file.read).to eq file_content
end
it 'returns an LFS pointer' do
result = subject.new_file(file_path, file_content)
expect(result.content).to start_with(Gitlab::Git::LfsPointerFile::VERSION_LINE)
end
it 'returns LFS pointer encoding as text' do
result = subject.new_file(file_path, file_content, encoding: 'base64')
expect(result.encoding).to eq('text')
end
context "when doesn't use LFS" do
let(:file_path) { 'other.filetype' }
it "doesn't create LFS pointers" do
new_content = subject.new_file(file_path, file_content).content
expect(new_content).not_to start_with(Gitlab::Git::LfsPointerFile::VERSION_LINE)
expect(new_content).to eq(file_content)
end
end
it 'links LfsObjects to project' do
expect do
subject.new_file(file_path, file_content)
end.to change { project.lfs_objects.count }.by(1)
end
context 'when LfsObject already exists' do
let(:lfs_pointer) { Gitlab::Git::LfsPointerFile.new(file_content) }
before do
create(:lfs_object, oid: lfs_pointer.sha256, size: lfs_pointer.size)
end
it 'links LfsObjects to project' do
expect do
subject.new_file(file_path, file_content)
end.to change { project.lfs_objects.count }.by(1)
end
end
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment