Commit 5d4531db authored by Kim Carlbäcker's avatar Kim Carlbäcker Committed by Rémy Coutable

Gogs Importer

parent 60f61096
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><g fill-rule="evenodd" transform="translate(0 1)"><path d="m14 15.01h1v-8.02c0-3.862-3.134-6.991-7-6.991-3.858 0-7 3.13-7 6.991v8.02h1v-8.02c0-3.306 2.691-5.991 6-5.991 3.314 0 6 2.682 6 5.991v8.02m-10.52-13.354c-.366-.402-.894-.655-1.48-.655-1.105 0-2 .895-2 2 0 .868.552 1.606 1.325 1.883.102-.321.226-.631.371-.93-.403-.129-.695-.507-.695-.953 0-.552.448-1 1-1 .306 0 .58.138.764.354.222-.25.461-.483.717-.699m9.04-.002c.366-.401.893-.653 1.479-.653 1.105 0 2 .895 2 2 0 .867-.552 1.606-1.324 1.883-.101-.321-.225-.632-.37-.931.403-.129.694-.507.694-.952 0-.552-.448-1-1-1-.305 0-.579.137-.762.353-.222-.25-.461-.483-.717-.699"/><path d="m5.726 7.04h1.557v.124c0 .283-.033.534-.1.752-.065.202-.175.391-.33.566-.35.394-.795.591-1.335.591-.527 0-.979-.19-1.355-.571-.376-.382-.564-.841-.564-1.377 0-.547.191-1.01.574-1.391.382-.382.848-.574 1.396-.574.295 0 .57.06.825.181.244.12.484.316.72.586l-.405.388c-.309-.412-.686-.618-1.13-.618-.399 0-.733.138-1 .413-.27.27-.405.609-.405 1.015 0 .42.151.766.452 1.037.282.252.587.378.915.378.28 0 .531-.094.754-.283.223-.19.347-.418.373-.683h-.94v-.535m2.884.061c0-.53.194-.986.583-1.367.387-.381.853-.571 1.396-.571.537 0 .998.192 1.382.576.386.384.578.845.578 1.384 0 .542-.194 1-.581 1.379-.389.379-.858.569-1.408.569-.487 0-.923-.168-1.311-.505-.426-.373-.64-.861-.64-1.465m.574.007c0 .417.14.759.42 1.028.278.269.6.403.964.403.395 0 .729-.137 1-.41.272-.277.408-.613.408-1.01 0-.402-.134-.739-.403-1.01-.267-.273-.597-.41-.991-.41-.392 0-.723.137-.993.41-.27.27-.405.604-.405 1m-.184 3.918c.525.026.812.063.812.063.271.025.324-.096.116-.273 0 0-.775-.813-1.933-.813-1.159 0-1.923.813-1.923.813-.211.174-.153.3.12.273 0 0 .286-.037.81-.063v.477c0 .268.224.5.5.5.268 0 .5-.223.5-.498v-.252.25c0 .268.224.5.5.5.268 0 .5-.223.5-.498v-.478m-1-1.023c.552 0 1-.224 1-.5 0-.276-.448-.5-1-.5-.552 0-1 .224-1 .5 0 .276.448.5 1 .5"/></g></svg>
\ No newline at end of file
......@@ -25,7 +25,7 @@ class ApplicationController < ActionController::Base
protect_from_forgery with: :exception
helper_method :can?, :current_application_settings
helper_method :import_sources_enabled?, :github_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled?
helper_method :import_sources_enabled?, :github_import_enabled?, :gogs_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled?
rescue_from Encoding::CompatibilityError do |exception|
log_exception(exception)
......@@ -245,6 +245,10 @@ class ApplicationController < ActionController::Base
current_application_settings.import_sources.include?('github')
end
def gogs_import_enabled?
current_application_settings.import_sources.include?('gogs')
end
def github_import_configured?
Gitlab::OAuth::Provider.enabled?(:github)
end
......
class Import::GogsController < Import::BaseController
before_action :verify_gogs_import_enabled
before_action :gogs_auth, only: [:status, :jobs, :create]
rescue_from Octokit::Unauthorized, with: :gogs_unauthorized
helper_method :logged_in_with_gogs?
def new
if session[:gogs_access_token]
redirect_to status_import_gogs_url
end
end
def personal_access_token
session[:gogs_access_token] = params[:personal_access_token]
session[:gogs_host_url] = params[:gogs_host_url]
redirect_to status_import_gogs_url
end
def status
@repos = client.repos
@already_added_projects = current_user.created_projects.where(import_type: "gogs")
already_added_projects_names = @already_added_projects.pluck(:import_source)
@gogs_root_url = session[:gogs_host_url]
@repos.reject!{ |repo| already_added_projects_names.include? repo.full_name }
end
def jobs
jobs = current_user.created_projects.where(import_type: "gogs").to_json(only: [:id, :import_status])
render json: jobs
end
def create
@repo_id = params[:repo_id].to_i
repo = client.repo(@repo_id)
@project_name = params[:new_name].presence || repo.name
namespace_path = params[:target_namespace].presence || current_user.namespace_path
@target_namespace = find_or_create_namespace(namespace_path, current_user.namespace_path)
if current_user.can?(:create_projects, @target_namespace)
@project = Gitlab::GithubImport::ProjectCreator.new(repo, @project_name, @target_namespace, current_user, access_params, type: 'gogs').execute
else
render 'unauthorized'
end
end
private
def client
@client ||= Gitlab::GithubImport::Client.new(session[:gogs_access_token], host: session[:gogs_host_url], api_version: 'v1')
end
def verify_gogs_import_enabled
render_404 unless gogs_import_enabled?
end
def gogs_auth
if session[:gogs_access_token].blank? || session[:gogs_host_url].blank?
redirect_to new_import_gogs_url,
alert: 'You need to specify both an Access Token and a Host URL.'
end
end
def gogs_unauthorized
session[:gogs_access_token] = nil
redirect_to new_import_gogs_url,
alert: 'Access denied to your Gogs account.'
end
def access_params
{ github_access_token: session[:gogs_access_token] }
end
end
......@@ -8,6 +8,10 @@ module ImportHelper
link_to path_with_namespace, github_project_url(path_with_namespace), target: '_blank'
end
def gogs_project_link(path_with_namespace)
link_to path_with_namespace, gogs_project_url(path_with_namespace), target: '_blank'
end
private
def github_project_url(path_with_namespace)
......@@ -20,4 +24,8 @@ module ImportHelper
provider = Gitlab.config.omniauth.providers.find { |p| p.name == 'github' }
@github_url = provider.fetch('url', 'https://github.com') if provider
end
def gogs_project_url(path_with_namespace)
"#{@gogs_root_url}/#{path_with_namespace}"
end
end
......@@ -5,6 +5,7 @@ module Projects
class Error < StandardError; end
ALLOWED_TYPES = [
'gogs',
'bitbucket',
'fogbugz',
'gitlab',
......
- page_title "Gogs Import"
- header_title "Projects", root_path
%h3.page-title
= image_tag(image_path('gogs-logo.svg'), alt: 'Gogs', size: "16x16")
Gogs
%p
To import a Gogs project, you can use a
= succeed '.' do
= link_to 'Personal Access Token', 'https://github.com/gogits/go-gogs-client/wiki#access-token'
= form_tag personal_access_token_import_gogs_path, method: :post, class: 'form-inline' do
.form-group
= text_field_tag :personal_access_token, '', class: 'form-control', placeholder: "Personal Access Token", size: 40
= text_field_tag :gogs_host_url, '', class: 'form-control', placeholder: "Gogs Host URL", size: 128
= submit_tag 'List Your Gogs Repositories', class: 'btn btn-success'
- page_title "Gogs import"
- header_title "Projects", root_path
%h3.page-title
%i.fa.fa-github
Import projects from Gogs
%p.light
Select projects you want to import.
%hr
%p
= button_tag class: "btn btn-import btn-success js-import-all" do
Import all projects
= icon("spinner spin", class: "loading-icon")
.table-responsive
%table.table.import-jobs
%colgroup.import-jobs-from-col
%colgroup.import-jobs-to-col
%colgroup.import-jobs-status-col
%thead
%tr
%th From Gogs
%th To GitLab
%th Status
%tbody
- @already_added_projects.each do |project|
%tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
%td
= gogs_project_link(project.import_source)
%td
= link_to project.path_with_namespace, [project.namespace.becomes(Namespace), project]
%td.job-status
- if project.import_status == 'finished'
%span
%i.fa.fa-check
done
- elsif project.import_status == 'started'
%i.fa.fa-spinner.fa-spin
started
- else
= project.human_import_status_name
- @repos.each do |repo|
%tr{id: "repo_#{repo.id}"}
%td
= gogs_project_link(repo.full_name)
%td.import-target
%fieldset.row
.input-group
.project-path.input-group-btn
- if current_user.can_select_namespace?
- selected = params[:namespace_id] || :current_user
- opts = current_user.can_create_group? ? { extra_group: Group.new(name: repo.owner.login, path: repo.owner.login) } : {}
= select_tag :namespace_id, namespaces_options(selected, opts.merge({ display_path: true })), { class: 'select2 js-select-namespace', tabindex: 1 }
- else
= text_field_tag :path, current_user.namespace_path, class: "input-large form-control", tabindex: 1, disabled: true
%span.input-group-addon /
= text_field_tag :path, repo.name, class: "input-mini form-control", tabindex: 2, autofocus: true, required: true
%td.import-actions.job-status
= button_tag class: "btn btn-import js-add-to-import" do
Import
= icon("spinner spin", class: "loading-icon")
.js-importer-status{ data: { jobs_import_path: "#{jobs_import_gogs_path}", import_path: "#{import_gogs_path}" } }
......@@ -48,6 +48,11 @@
- if github_import_enabled?
= link_to new_import_github_path, class: 'btn import_github' do
= icon('github', text: 'GitHub')
%div
- if gogs_import_enabled?
= link_to new_import_gogs_url, class: 'btn import_gogs' do
= image_tag(image_path('gogs-logo.svg'), alt: 'Gogs', size: "14x14")
Gogs
%div
- if bitbucket_import_enabled?
= link_to status_import_bitbucket_path, class: "btn import_bitbucket #{'how_to_import_link' unless bitbucket_import_configured?}", "data-no-turbolink" => "true" do
......
---
title: Gogs importer
merge_request: 6945
author: Kim Carlbäcker
......@@ -213,7 +213,7 @@ Settings.gitlab.default_projects_features['builds'] = true if Settin
Settings.gitlab.default_projects_features['container_registry'] = true if Settings.gitlab.default_projects_features['container_registry'].nil?
Settings.gitlab.default_projects_features['visibility_level'] = Settings.send(:verify_constant, Gitlab::VisibilityLevel, Settings.gitlab.default_projects_features['visibility_level'], Gitlab::VisibilityLevel::PRIVATE)
Settings.gitlab['domain_whitelist'] ||= []
Settings.gitlab['import_sources'] ||= %w[github bitbucket gitlab google_code fogbugz git gitlab_project]
Settings.gitlab['import_sources'] ||= %w[gogs github bitbucket gitlab google_code fogbugz git gitlab_project]
Settings.gitlab['trusted_proxies'] ||= []
Settings.gitlab['no_todos_messages'] ||= YAML.load_file(Rails.root.join('config', 'no_todos_messages.yml'))
......
......@@ -6,6 +6,12 @@ namespace :import do
get :jobs
end
resource :gogs, only: [:create, :new], controller: :gogs do
post :personal_access_token
get :status
get :jobs
end
resource :gitlab, only: [:create], controller: :gitlab do
get :status
get :callback
......
......@@ -45,7 +45,7 @@ module Gitlab
default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
domain_whitelist: Settings.gitlab['domain_whitelist'],
import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project],
import_sources: %w[gogs github bitbucket gitlab google_code fogbugz git gitlab_project],
shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
max_artifacts_size: Settings.artifacts['max_size'],
require_two_factor_authentication: false,
......
......@@ -4,24 +4,30 @@ module Gitlab
GITHUB_SAFE_REMAINING_REQUESTS = 100
GITHUB_SAFE_SLEEP_TIME = 500
attr_reader :access_token
attr_reader :access_token, :host, :api_version
def initialize(access_token)
def initialize(access_token, host: nil, api_version: 'v3')
@access_token = access_token
@host = host
@api_version = api_version
if access_token
::Octokit.auto_paginate = false
end
end
def api_endpoint
host.present? && api_version.present? ? "#{host}/api/#{api_version}" : github_options[:site]
end
def api
@api ||= ::Octokit::Client.new(
access_token: access_token,
api_endpoint: github_options[:site],
api_endpoint: api_endpoint,
# If there is no config, we're connecting to github.com and we
# should verify ssl.
connection_options: {
ssl: { verify: config ? config['verify_ssl'] : true }
ssl: { verify: config ? config['verify_ssl'] : false }
}
)
end
......
......@@ -60,7 +60,7 @@ module Gitlab
fetch_resources(:labels, repo, per_page: 100) do |labels|
labels.each do |raw|
begin
LabelFormatter.new(project, raw).create!
GithubImport::LabelFormatter.new(project, raw).create!
rescue => e
errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
end
......@@ -74,7 +74,7 @@ module Gitlab
fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones|
milestones.each do |raw|
begin
MilestoneFormatter.new(project, raw).create!
GithubImport::MilestoneFormatter.new(project, raw).create!
rescue => e
errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
end
......@@ -85,7 +85,7 @@ module Gitlab
def import_issues
fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues|
issues.each do |raw|
gh_issue = IssueFormatter.new(project, raw)
gh_issue = GithubImport::IssueFormatter.new(project, raw)
begin
issuable =
......@@ -106,7 +106,7 @@ module Gitlab
def import_pull_requests
fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests|
pull_requests.each do |raw|
pull_request = PullRequestFormatter.new(project, raw)
pull_request = GithubImport::PullRequestFormatter.new(project, raw)
next unless pull_request.valid?
begin
......@@ -179,7 +179,7 @@ module Gitlab
ActiveRecord::Base.no_touching do
comments.each do |raw|
begin
comment = CommentFormatter.new(project, raw)
comment = GithubImport::CommentFormatter.new(project, raw)
# GH does not return info about comment's parent, so we guess it by checking its URL!
*_, parent, iid = URI(raw.html_url).path.split('/')
issuable_class = parent == 'issues' ? Issue : MergeRequest
......@@ -198,7 +198,7 @@ module Gitlab
last_note_attrs = nil
cut_off_index = comments.find_index do |raw|
comment = CommentFormatter.new(project, raw)
comment = GithubImport::CommentFormatter.new(project, raw)
comment_attrs = comment.attributes
last_note_attrs ||= last_note.slice(*comment_attrs.keys)
......@@ -214,7 +214,7 @@ module Gitlab
def import_wiki
unless project.wiki.repository_exists?
wiki = WikiFormatter.new(project)
wiki = GithubImport::WikiFormatter.new(project)
gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url)
end
rescue Gitlab::Shell::Error => e
......@@ -230,7 +230,7 @@ module Gitlab
fetch_resources(:releases, repo, per_page: 100) do |releases|
releases.each do |raw|
begin
gh_release = ReleaseFormatter.new(project, raw)
gh_release = GithubImport::ReleaseFormatter.new(project, raw)
gh_release.create! if gh_release.valid?
rescue => e
errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
......
......@@ -70,7 +70,7 @@ module Gitlab
def milestone
if raw_data.milestone.present?
project.milestones.find_by(iid: raw_data.milestone.number)
project.milestones.find_by(iid: raw_data.milestone.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr))
end
end
......
......@@ -3,7 +3,7 @@ module Gitlab
class MilestoneFormatter < BaseFormatter
def attributes
{
iid: raw_data.number,
iid: raw_data.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr),
project: project,
title: raw_data.title,
description: raw_data.description,
......@@ -19,7 +19,11 @@ module Gitlab
end
def find_condition
{ iid: raw_data.number }
{ iid: raw_data.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr) }
end
def self.iid_attr
:number
end
private
......
module Gitlab
module GithubImport
class ProjectCreator
attr_reader :repo, :name, :namespace, :current_user, :session_data
attr_reader :repo, :name, :namespace, :current_user, :session_data, :type
def initialize(repo, name, namespace, current_user, session_data)
def initialize(repo, name, namespace, current_user, session_data, type: 'github')
@repo = repo
@name = name
@namespace = namespace
@current_user = current_user
@session_data = session_data
@type = type
end
def execute
......@@ -19,7 +20,7 @@ module Gitlab
description: repo.description,
namespace_id: namespace.id,
visibility_level: visibility_level,
import_type: "github",
import_type: type,
import_source: repo.full_name,
import_url: import_url,
skip_wiki: skip_wiki
......@@ -29,7 +30,7 @@ module Gitlab
private
def import_url
repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@")
repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@")
end
def visibility_level
......
......@@ -98,7 +98,7 @@ module Gitlab
def milestone
if raw_data.milestone.present?
project.milestones.find_by(iid: raw_data.milestone.number)
project.milestones.find_by(iid: raw_data.milestone.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr))
end
end
......
require 'uri'
module Gitlab
module GogsImport
class Importer < Gitlab::GithubImport::Importer
include Gitlab::ShellAdapter
attr_reader :client, :errors, :project, :repo, :repo_url
def initialize(project)
@project = project
@repo = project.import_source
@repo_url = project.import_url
@errors = []
@labels = {}
if credentials
uri = URI.parse(project.import_url)
host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(/[\w-]+\/[\w-]+\.git\z/, '')
@client = GithubImport::Client.new(credentials[:user], host: host, api_version: 'v1')
else
raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}"
end
end
def execute
import_labels
import_milestones
import_pull_requests
import_issues
import_comments(:issues)
import_comments(:pull_requests)
import_wiki
# NOTE: this is commented out since Gogs doesn't have release-API yet
# import_releases
handle_errors
true
end
def import_milestones
fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones|
milestones.each do |raw|
begin
GogsImport::MilestoneFormatter.new(project, raw).create!
rescue => e
errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
end
end
end
end
end
end
end
module Gitlab
module GogsImport
class MilestoneFormatter < GithubImport::MilestoneFormatter
def self.iid_attr
:id
end
end
end
end
......@@ -14,6 +14,7 @@ module Gitlab
def options
{
'Gogs' => 'gogs',
'GitHub' => 'github',
'Bitbucket' => 'bitbucket',
'GitLab.com' => 'gitlab',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment