Skip to content
Snippets Groups Projects
Verified Commit 485fd4aa authored by James Nutt's avatar James Nutt Committed by GitLab
Browse files

Allow relation tree restorer to import single relation

This MR lays groundwork for chosen relation import
(#425798) by updating
the relation tree restorer to allow importing of a single relation.

It also adds a service and class which will be called by an API endpoint
in a follow-up MR.

Changelog: changed
parent 0ca1d8a8
No related branches found
No related tags found
1 merge request!147904API endpoint for re-import of named relation from project export file
......@@ -268,6 +268,52 @@ curl --request POST \
The `Content-Length` header must return a valid number. The maximum file size is 10 GB.
The `Content-Type` header must be `application/gzip`.
## Import a single relation
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/425798) in GitLab 16.11.
This endpoints accepts a project export archive and a named relation (issues,
merge requests, pipelines, or milestones) and re-imports that relation, skipping
items that have already been imported.
The required project export file adheres to the same structure and size requirements described in
[Import a file](#import-a-file).
- The extracted files must adhere to the structure of a GitLab project export.
- The archive must not exceed the maximum import file size configured by the Administrator.
```plaintext
POST /projects/import-relation
```
| Attribute | Type | Required | Description |
|------------|--------|----------|----------------------------------------------------------------------------------------------------------------|
| `file` | string | yes | The file to be uploaded. |
| `path` | string | yes | Name and path for new project. |
| `relation` | string | yes | The name of the relation to import. Must be one of `issues`, `milestones`, `ci_pipelines` or `merge_requests`. |
To upload a file from your file system, use the `--form` option, which causes
cURL to post data using the header `Content-Type: multipart/form-data`.
The `file=` parameter must point to a file on your file system and be preceded
by `@`. For example:
```shell
curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" \
--form "path=api-project" \
--form "file=@/path/to/file" \
--form "relation=issues" \
"https://gitlab.example.com/api/v4/projects/import-relation"
```
```json
{
"id": 9,
"project_path": "namespace1/project1",
"relation": "issues",
"status": "finished"
}
```
## Import a file from AWS S3
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/348874) in GitLab 14.9 in [Beta](https://handbook.gitlab.com/handbook/product/gitlab-the-product/#experiment-beta-ga), [with a flag](../administration/feature_flags.md) named `import_project_from_remote_file_s3`. Disabled by default.
......
# frozen_string_literal: true
module API
module Entities
class RelationImportTracker < Grape::Entity
expose :id, documentation: { type: 'integer', example: 1 }
expose :project_path, documentation: { type: 'string', example: 'namespace1/project1' } do |tracker|
tracker.project.full_path
end
expose :relation, documentation: { type: 'string', example: 'issues' }
expose :status, documentation: { type: 'string', example: 'pending' }, &:status_name
expose :created_at, documentation: { type: "dateTime", example: "2022-01-31T15:10:45.080Z" }
expose :updated_at, documentation: { type: "dateTime", example: "2022-01-31T15:10:45.080Z" }
end
end
end
......@@ -187,6 +187,84 @@ def filtered_override_params(params)
end
end
desc 'Workhorse authorize the project relation import upload' do
detail 'This feature was introduced in GitLab 16.11'
tags ['project_import']
end
post 'import-relation/authorize' do
forbidden! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project')
require_gitlab_workhorse!
status 200
content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
ImportExportUploader.workhorse_authorize(
has_length: false,
maximum_size: Gitlab::CurrentSettings.max_import_size.megabytes
)
end
params do
requires :path, type: String, desc: 'The project path and name'
requires :file,
type: ::API::Validations::Types::WorkhorseFile,
desc: 'The project export file from which to extract the relation.',
documentation: { type: 'file' }
requires :relation,
type: String,
desc: 'The relation to import. Must be one of issues, merge_requests, ci_pipelines, or milestones.'
optional 'file.path', type: String, desc: 'Path to locally stored body (generated by Workhorse)'
optional 'file.name', type: String, desc: 'Real filename as sent in Content-Disposition (generated by Workhorse)'
optional 'file.type', type: String, desc: 'Real content type as send in Content-Type (generated by Workhorse)'
optional 'file.size', type: Integer, desc: 'Real size of file (generated by Workhorse)'
optional 'file.md5', type: String, desc: 'MD5 checksum of the file (generated by Workhorse)'
optional 'file.sha1', type: String, desc: 'SHA1 checksum of the file (generated by Workhorse)'
optional 'file.sha256', type: String, desc: 'SHA256 checksum of the file (generated by Workhorse)'
optional 'file.etag', type: String, desc: 'Etag of the file (generated by Workhorse)'
optional 'file.remote_id', type: String, desc: 'Remote_id of the file (generated by Workhorse)'
optional 'file.remote_url', type: String, desc: 'Remote_url of the file (generated by Workhorse)'
end
desc 'Re-import a relation into a project' do
detail 'This feature was introduced in GitLab 16.11.'
success code: 201, model: Entities::RelationImportTracker
failure [
{ code: 401, message: 'Unauthorized' },
{ code: 403, message: 'Forbidden' },
{ code: 400, message: 'Bad request' },
{ code: 404, message: 'Not found' },
{ code: 503, message: 'Service unavailable' }
]
tags ['project_import']
consumes ['multipart/form-data']
end
post 'import-relation' do
forbidden! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project')
require_gitlab_workhorse!
check_rate_limit! :project_import, scope: [current_user, :project_import]
Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/21041')
validate_file!
response = ::Projects::ImportExport::RelationImportService.new(
current_user: current_user,
params: {
path: import_params[:path],
file: import_params[:file],
relation: import_params[:relation]
}
).execute
if response.success?
present(response.payload, with: Entities::RelationImportTracker, current_user: current_user)
else
render_api_error!(response.message, response.http_status)
end
end
params do
requires :region, type: String, desc: 'AWS region'
requires :bucket_name, type: String, desc: 'Bucket name'
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe API::Entities::RelationImportTracker, feature_category: :importers do
subject(:entity) { described_class.new(tracker) }
let(:tracker) { build(:relation_import_tracker) }
describe '#as_json' do
subject { entity.as_json }
it 'exposes correct attributes' do
is_expected.to eq(
id: tracker.id,
project_path: tracker.project.full_path,
relation: tracker.relation,
status: :created,
created_at: tracker.created_at,
updated_at: tracker.updated_at
)
end
end
end
......@@ -512,6 +512,114 @@ def stub_import(namespace)
end
end
describe 'POST /projects/:id/import-relation' do
subject(:perform_relation_import) { upload_relation_archive(file_upload, workhorse_headers, params) }
let(:file_upload) { fixture_file_upload(file) }
let(:params) do
{
path: 'test-import',
relation: 'issues',
'file.size' => file_upload.size
}
end
before do
allow(ImportExportUploader).to receive(:workhorse_upload_path).and_return('/')
end
it_behaves_like 'requires authentication'
it_behaves_like 'requires import source to be enabled'
it 'executes a limited number of queries', :use_clean_rails_redis_caching do
control = ActiveRecord::QueryRecorder.new { perform_relation_import }
expect(control.count).to be <= 111
end
context 'when the project is valid' do
context 'and the user is a maintainer' do
it 'allows the import' do
project = create(
:project,
name: 'test-import',
path: 'test-import'
)
project.add_maintainer(user)
params[:path] = project.full_path
perform_relation_import
expect(response).to have_gitlab_http_status(:created)
end
end
end
it 'does not schedule a relation import for a project that does not exist' do
params[:path] = 'missing/test-import'
perform_relation_import
expect(response).to have_gitlab_http_status(:not_found)
expect(json_response['message']).to eq('Project not found')
end
it 'does not schedule a relation import if the user has no permission to the project' do
project = create(
:project,
name: 'test-import',
path: 'test-import'
)
params[:path] = project.full_path
perform_relation_import
expect(response).to have_gitlab_http_status(:forbidden)
expect(json_response['message']).to eq('You are not authorized to perform this action')
end
context 'if user uploads no valid file' do
let(:file) { 'README.md' }
it 'does not schedule a relation import' do
params[:path] = 'test-import'
perform_relation_import
expect(response).to have_gitlab_http_status(:unprocessable_entity)
expect(json_response['message']['error']).to eq('You need to upload a GitLab project export archive (ending in .gz).')
end
end
context 'when request exceeds the rate limit' do
before do
allow(::Gitlab::ApplicationRateLimiter).to receive(:throttled?).and_return(true)
end
it 'prevents users from importing relations' do
perform_relation_import
expect(response).to have_gitlab_http_status(:too_many_requests)
expect(json_response['message']['error']).to eq('This endpoint has been requested too many times. Try again later.')
end
end
def upload_relation_archive(file, headers = {}, params = {})
workhorse_finalize(
api("/projects/import-relation", user),
method: :post,
file_key: :file,
params: params.merge(file: file),
headers: headers,
send_rewritten_field: true
)
end
end
describe 'POST /projects/import/authorize' do
subject(:authorize_import) { post api('/projects/import/authorize', user), headers: workhorse_headers }
......@@ -570,4 +678,27 @@ def stub_import(namespace)
end
end
end
describe 'POST /projects/import-relation/authorize' do
subject(:authorize_import) { post api('/projects/import-relation/authorize', user), headers: workhorse_headers }
it_behaves_like 'requires authentication'
it_behaves_like 'requires import source to be enabled'
it 'authorizes importing project with workhorse header' do
authorize_import
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
expect(json_response['TempPath']).to eq(ImportExportUploader.workhorse_local_upload_path)
end
it 'rejects requests that bypassed gitlab-workhorse' do
workhorse_headers.delete(Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER)
authorize_import
expect(response).to have_gitlab_http_status(:forbidden)
end
end
end
......@@ -165,6 +165,8 @@ func TestAcceleratedUpload(t *testing.T) {
{"POST", `/api/v4/groups/import/`, true},
{"POST", `/api/v4/projects/import`, true},
{"POST", `/api/v4/projects/import/`, true},
{"POST", `/api/v4/projects/import-relation`, true},
{"POST", `/api/v4/projects/import-relation/`, true},
{"POST", `/import/gitlab_project`, true},
{"POST", `/import/gitlab_project/`, true},
{"POST", `/import/gitlab_group`, true},
......
......@@ -313,6 +313,7 @@ func configureRoutes(u *upstream) {
u.route("PUT", apiTopicPattern, tempfileMultipartProxy),
u.route("POST", apiPattern+`v4/groups/import`, mimeMultipartUploader),
u.route("POST", apiPattern+`v4/projects/import`, mimeMultipartUploader),
u.route("POST", apiPattern+`v4/projects/import-relation`, mimeMultipartUploader),
// Project Import via UI upload acceleration
u.route("POST", importPattern+`gitlab_project`, mimeMultipartUploader),
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment