Commit 4106b658 authored by James Lopez's avatar James Lopez

Merge branch 'georgekoltsov/add-github-gitea-filtering' into 'master'

Add GitHub & Gitea importer projects filtering

See merge request !16823
parents dc4df41c 17d47451
Pipeline #82973409 passed with stages
in 50 minutes and 17 seconds
<script>
import _ from 'underscore';
import { mapActions, mapState, mapGetters } from 'vuex';
import { GlLoadingIcon } from '@gitlab/ui';
import LoadingButton from '~/vue_shared/components/loading_button.vue';
......@@ -7,6 +8,8 @@ import ImportedProjectTableRow from './imported_project_table_row.vue';
import ProviderRepoTableRow from './provider_repo_table_row.vue';
import eventHub from '../event_hub';
const reposFetchThrottleDelay = 1000;
export default {
name: 'ImportProjectsTable',
components: {
......@@ -23,11 +26,11 @@ export default {
},
computed: {
...mapState(['importedProjects', 'providerRepos', 'isLoadingRepos']),
...mapState(['importedProjects', 'providerRepos', 'isLoadingRepos', 'filter']),
...mapGetters(['isImportingAnyRepo', 'hasProviderRepos', 'hasImportedProjects']),
emptyStateText() {
return sprintf(__('No %{providerTitle} repositories available to import'), {
return sprintf(__('No %{providerTitle} repositories found'), {
providerTitle: this.providerTitle,
});
},
......@@ -47,21 +50,38 @@ export default {
},
methods: {
...mapActions(['fetchRepos', 'fetchJobs', 'stopJobsPolling', 'clearJobsEtagPoll']),
...mapActions([
'fetchRepos',
'fetchReposFiltered',
'fetchJobs',
'stopJobsPolling',
'clearJobsEtagPoll',
'setFilter',
]),
importAll() {
eventHub.$emit('importAll');
},
handleFilterInput({ target }) {
this.setFilter(target.value);
},
throttledFetchRepos: _.throttle(function fetch() {
eventHub.$off('importAll');
this.fetchRepos();
}, reposFetchThrottleDelay),
},
};
</script>
<template>
<div>
<p class="light text-nowrap mt-2">
{{ s__('ImportProjects|Select the projects you want to import') }}
</p>
<div class="d-flex justify-content-between align-items-end flex-wrap mb-3">
<p class="light text-nowrap mt-2 my-sm-0">
{{ s__('ImportProjects|Select the projects you want to import') }}
</p>
<loading-button
container-class="btn btn-success js-import-all"
:loading="isImportingAnyRepo"
......@@ -70,6 +90,19 @@ export default {
type="button"
@click="importAll"
/>
<form novalidate @submit.prevent>
<input
:value="filter"
data-qa-selector="githubish_import_filter_field"
class="form-control"
name="filter"
:placeholder="__('Filter your projects by name')"
autofocus
size="40"
@input="handleFilterInput($event)"
@keyup.enter="throttledFetchRepos"
/>
</form>
</div>
<gl-loading-icon
v-if="isLoadingRepos"
......
......@@ -38,7 +38,7 @@ export default function mountImportProjectsTable(mountElement) {
},
methods: {
...mapActions(['setInitialData']),
...mapActions(['setInitialData', 'setFilter']),
},
render(createElement) {
......
......@@ -5,6 +5,7 @@ import Poll from '~/lib/utils/poll';
import createFlash from '~/flash';
import { s__, sprintf } from '~/locale';
import axios from '~/lib/utils/axios_utils';
import { jobsPathWithFilter, reposPathWithFilter } from './getters';
let eTagPoll;
......@@ -19,16 +20,20 @@ export const restartJobsPolling = () => {
};
export const setInitialData = ({ commit }, data) => commit(types.SET_INITIAL_DATA, data);
export const setFilter = ({ commit }, filter) => commit(types.SET_FILTER, filter);
export const requestRepos = ({ commit }, repos) => commit(types.REQUEST_REPOS, repos);
export const receiveReposSuccess = ({ commit }, repos) =>
commit(types.RECEIVE_REPOS_SUCCESS, repos);
export const receiveReposError = ({ commit }) => commit(types.RECEIVE_REPOS_ERROR);
export const fetchRepos = ({ state, dispatch }) => {
dispatch('stopJobsPolling');
dispatch('requestRepos');
const { provider } = state;
return axios
.get(state.reposPath)
.get(reposPathWithFilter(state))
.then(({ data }) =>
dispatch('receiveReposSuccess', convertObjectPropsToCamelCase(data, { deep: true })),
)
......@@ -36,7 +41,7 @@ export const fetchRepos = ({ state, dispatch }) => {
.catch(() => {
createFlash(
sprintf(s__('ImportProjects|Requesting your %{provider} repositories failed'), {
provider: state.provider,
provider,
}),
);
......@@ -77,16 +82,23 @@ export const fetchImport = ({ state, dispatch }, { newName, targetNamespace, rep
export const receiveJobsSuccess = ({ commit }, updatedProjects) =>
commit(types.RECEIVE_JOBS_SUCCESS, updatedProjects);
export const fetchJobs = ({ state, dispatch }) => {
if (eTagPoll) return;
const { filter } = state;
if (eTagPoll) {
stopJobsPolling();
clearJobsEtagPoll();
}
eTagPoll = new Poll({
resource: {
fetchJobs: () => axios.get(state.jobsPath),
fetchJobs: () => axios.get(jobsPathWithFilter(state)),
},
method: 'fetchJobs',
successCallback: ({ data }) =>
dispatch('receiveJobsSuccess', convertObjectPropsToCamelCase(data, { deep: true })),
errorCallback: () => createFlash(s__('ImportProjects|Updating the imported projects failed')),
errorCallback: () =>
createFlash(s__('ImportProjects|Update of imported projects with realtime changes failed')),
data: { filter },
});
if (!Visibility.hidden()) {
......
......@@ -20,3 +20,8 @@ export const isImportingAnyRepo = state => state.reposBeingImported.length > 0;
export const hasProviderRepos = state => state.providerRepos.length > 0;
export const hasImportedProjects = state => state.importedProjects.length > 0;
export const reposPathWithFilter = ({ reposPath, filter = '' }) =>
filter ? `${reposPath}?filter=${filter}` : reposPath;
export const jobsPathWithFilter = ({ jobsPath, filter = '' }) =>
filter ? `${jobsPath}?filter=${filter}` : jobsPath;
......@@ -9,3 +9,5 @@ export const RECEIVE_IMPORT_SUCCESS = 'RECEIVE_IMPORT_SUCCESS';
export const RECEIVE_IMPORT_ERROR = 'RECEIVE_IMPORT_ERROR';
export const RECEIVE_JOBS_SUCCESS = 'RECEIVE_JOBS_SUCCESS';
export const SET_FILTER = 'SET_FILTER';
......@@ -6,6 +6,10 @@ export default {
Object.assign(state, data);
},
[types.SET_FILTER](state, filter) {
state.filter = filter;
},
[types.REQUEST_REPOS](state) {
state.isLoadingRepos = true;
},
......
......@@ -12,4 +12,5 @@ export default () => ({
isLoadingRepos: false,
canSelectNamespace: false,
ciCdOnly: false,
filter: '',
});
......@@ -2,6 +2,7 @@
class Import::GithubController < Import::BaseController
include ImportHelper
include ActionView::Helpers::SanitizeHelper
before_action :verify_import_enabled
before_action :provider_auth, only: [:status, :realtime_changes, :create]
......@@ -55,7 +56,7 @@ class Import::GithubController < Import::BaseController
def realtime_changes
Gitlab::PollingInterval.set_header(response, interval: 3_000)
render json: find_jobs(provider)
render json: already_added_projects.to_json(only: [:id], methods: [:import_status])
end
private
......@@ -82,7 +83,7 @@ class Import::GithubController < Import::BaseController
end
def already_added_projects
@already_added_projects ||= find_already_added_projects(provider)
@already_added_projects ||= filtered(find_already_added_projects(provider))
end
def already_added_project_names
......@@ -104,7 +105,7 @@ class Import::GithubController < Import::BaseController
end
def client_repos
@client_repos ||= client.repos
@client_repos ||= filtered(client.repos)
end
def verify_import_enabled
......@@ -185,6 +186,20 @@ class Import::GithubController < Import::BaseController
def extra_import_params
{}
end
def sanitized_filter_param
@filter ||= sanitize(params[:filter])
end
def filter_attribute
:name
end
def filtered(collection)
return collection unless sanitized_filter_param
collection.select { |item| item[filter_attribute].include?(sanitized_filter_param) }
end
end
Import::GithubController.prepend_if_ee('EE::Import::GithubController')
---
title: Add GitHub & Gitea importers project filtering
merge_request: 16823
author:
type: added
......@@ -66,10 +66,14 @@ From there, you can see the import statuses of your Gitea repositories.
- whereas those that are not yet imported will have an **Import** button on the
right side of the table.
If you want, you can import all your Gitea projects in one go by hitting
**Import all projects** in the upper left corner.
You also can:
![Gitea importer page](img/import_projects_from_github_importer.png)
- Import all your Gitea projects in one go by hitting **Import all projects** in
the upper left corner
- Filter projects by name. If filter is applied, hitting **Import all projects**
will only import matched projects
![Gitea importer page](img/import_projects_from_gitea_importer_v12_3.png)
---
......
......@@ -115,11 +115,14 @@ your GitHub repositories are listed.
1. By default, the proposed repository namespaces match the names as they exist in GitHub, but based on your permissions,
you can choose to edit these names before you proceed to import any of them.
1. Select the **Import** button next to any number of repositories, or select **Import all repositories**.
1. Select the **Import** button next to any number of repositories, or select **Import all repositories**. Additionally,
you can filter projects by name. If filter is applied, **Import all repositories** only imports matched repositories.
1. The **Status** column shows the import status of each repository. You can choose to leave the page open and it will
update in realtime or you can return to it later.
1. Once a repository has been imported, click its GitLab path to open its GitLab URL.
![Github importer page](img/import_projects_from_github_importer_v12_3.png)
## Mirroring and pipeline status sharing
Depending your GitLab tier, [project mirroring](../../../workflow/repository_mirroring.md) can be set up to keep
......
......@@ -8261,7 +8261,7 @@ msgstr ""
msgid "ImportProjects|The repository could not be created."
msgstr ""
msgid "ImportProjects|Updating the imported projects failed"
msgid "ImportProjects|Update of imported projects with realtime changes failed"
msgstr ""
msgid "Improve Issue boards"
......@@ -10205,7 +10205,7 @@ msgstr ""
msgid "No %{header} for this request."
msgstr ""
msgid "No %{providerTitle} repositories available to import"
msgid "No %{providerTitle} repositories found"
msgstr ""
msgid "No Epic"
......
......@@ -93,7 +93,7 @@ describe('ImportProjectsTable', () => {
return vm.$nextTick().then(() => {
expect(vm.$el.querySelector('.js-loading-button-icon')).toBeNull();
expect(vm.$el.querySelector('.table')).toBeNull();
expect(vm.$el.innerText).toMatch(`No ${providerTitle} repositories available to import`);
expect(vm.$el.innerText).toMatch(`No ${providerTitle} repositories found`);
});
});
......@@ -182,4 +182,10 @@ describe('ImportProjectsTable', () => {
expect(vm.$el.querySelector(`.ic-status_${statusObject.icon}`)).not.toBeNull();
});
});
it('renders filtering input field', () => {
expect(
vm.$el.querySelector('input[data-qa-selector="githubish_import_filter_field"]'),
).not.toBeNull();
});
});
......@@ -97,6 +97,7 @@ describe('import_projects store actions', () => {
describe('fetchRepos', () => {
let mock;
const payload = { imported_projects: [{}], provider_repos: [{}], namespaces: [{}] };
beforeEach(() => {
localState.reposPath = `${TEST_HOST}/endpoint.json`;
......@@ -105,8 +106,7 @@ describe('import_projects store actions', () => {
afterEach(() => mock.restore());
it('dispatches requestRepos and receiveReposSuccess actions on a successful request', done => {
const payload = { imported_projects: [{}], provider_repos: [{}], namespaces: [{}] };
it('dispatches stopJobsPolling, requestRepos and receiveReposSuccess actions on a successful request', done => {
mock.onGet(`${TEST_HOST}/endpoint.json`).reply(200, payload);
testAction(
......@@ -115,6 +115,7 @@ describe('import_projects store actions', () => {
localState,
[],
[
{ type: 'stopJobsPolling' },
{ type: 'requestRepos' },
{
type: 'receiveReposSuccess',
......@@ -128,7 +129,7 @@ describe('import_projects store actions', () => {
);
});
it('dispatches requestRepos and receiveReposSuccess actions on an unsuccessful request', done => {
it('dispatches stopJobsPolling, requestRepos and receiveReposError actions on an unsuccessful request', done => {
mock.onGet(`${TEST_HOST}/endpoint.json`).reply(500);
testAction(
......@@ -136,10 +137,39 @@ describe('import_projects store actions', () => {
null,
localState,
[],
[{ type: 'requestRepos' }, { type: 'receiveReposError' }],
[{ type: 'stopJobsPolling' }, { type: 'requestRepos' }, { type: 'receiveReposError' }],
done,
);
});
describe('when filtered', () => {
beforeEach(() => {
localState.filter = 'filter';
});
it('fetches repos with filter applied', done => {
mock.onGet(`${TEST_HOST}/endpoint.json?filter=filter`).reply(200, payload);
testAction(
fetchRepos,
null,
localState,
[],
[
{ type: 'stopJobsPolling' },
{ type: 'requestRepos' },
{
type: 'receiveReposSuccess',
payload: convertObjectPropsToCamelCase(payload, { deep: true }),
},
{
type: 'fetchJobs',
},
],
done,
);
});
});
});
describe('requestImport', () => {
......@@ -249,6 +279,7 @@ describe('import_projects store actions', () => {
describe('fetchJobs', () => {
let mock;
const updatedProjects = [{ name: 'imported/project' }, { name: 'provider/repo' }];
beforeEach(() => {
localState.jobsPath = `${TEST_HOST}/endpoint.json`;
......@@ -263,7 +294,6 @@ describe('import_projects store actions', () => {
afterEach(() => mock.restore());
it('dispatches requestJobs and receiveJobsSuccess actions on a successful request', done => {
const updatedProjects = [{ name: 'imported/project' }, { name: 'provider/repo' }];
mock.onGet(`${TEST_HOST}/endpoint.json`).reply(200, updatedProjects);
testAction(
......@@ -280,5 +310,29 @@ describe('import_projects store actions', () => {
done,
);
});
describe('when filtered', () => {
beforeEach(() => {
localState.filter = 'filter';
});
it('fetches realtime changes with filter applied', done => {
mock.onGet(`${TEST_HOST}/endpoint.json?filter=filter`).reply(200, updatedProjects);
testAction(
fetchJobs,
null,
localState,
[],
[
{
type: 'receiveJobsSuccess',
payload: convertObjectPropsToCamelCase(updatedProjects, { deep: true }),
},
],
done,
);
});
});
});
});
......@@ -139,6 +139,38 @@ shared_examples 'a GitHub-ish import controller: GET status' do
expect { get :status, format: :json }
.not_to exceed_all_query_limit(control_count)
end
context 'when filtering' do
let(:repo_2) { OpenStruct.new(login: 'emacs', full_name: 'asd/emacs', name: 'emacs', owner: { login: 'owner' }) }
let(:project) { create(:project, import_type: provider, namespace: user.namespace, import_status: :finished, import_source: 'example/repo') }
let(:group) { create(:group) }
before do
group.add_owner(user)
stub_client(repos: [repo, repo_2, org_repo], orgs: [org], org_repos: [org_repo])
end
it 'filters list of repositories by name' do
get :status, params: { filter: 'emacs' }, format: :json
expect(response).to have_gitlab_http_status(200)
expect(json_response.dig("imported_projects").count).to eq(0)
expect(json_response.dig("provider_repos").count).to eq(1)
expect(json_response.dig("provider_repos", 0, "id")).to eq(repo_2.id)
expect(json_response.dig("namespaces", 0, "id")).to eq(group.id)
end
context 'when user input contains html' do
let(:expected_filter) { 'test' }
let(:filter) { "<html>#{expected_filter}</html>" }
it 'sanitizes user input' do
get :status, params: { filter: filter }, format: :json
expect(assigns(:filter)).to eq(expected_filter)
end
end
end
end
shared_examples 'a GitHub-ish import controller: POST create' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment