Unverified Commit 5a9bcfbb authored by TheZ3ro's avatar TheZ3ro

add git repo mirroring

parent fc433c80
......@@ -155,8 +155,20 @@ The repository of older versions of applications from the main demo repository.
# 'bar.info:/var/www/fdroid',
# }
# The full URL to a git remote repository. You can include
# multiple servers to mirror to by wrapping the whole thing in {} or [], and
# including the servergitmirrors strings in a comma-separated list.
# Servers listed here will also be automatically inserted in the mirrors list.
# servergitmirrors = 'https://github.com/user/repo'
# servergitmirrors = {
# 'https://github.com/user/repo',
# 'https://gitlab.com/user/repo',
# }
# Any mirrors of this repo, for example all of the servers declared in
# serverwebroot, will automatically be used by the client. If one
# serverwebroot and all the servers declared in servergitmirrors,
# will automatically be used by the client. If one
# mirror is not working, then the client will try another. If the
# client has Tor enabled, then the client will prefer mirrors with
# .onion addresses. This base URL will be used for both the main repo
......@@ -262,6 +262,15 @@ def read_config(opts, config_file='config.py'):
rootlist.append(rootstr.replace('//', '/'))
config['serverwebroot'] = rootlist
if 'servergitmirrors' in config:
if isinstance(config['servergitmirrors'], str):
roots = [config['servergitmirrors']]
elif all(isinstance(item, str) for item in config['servergitmirrors']):
roots = config['servergitmirrors']
raise TypeError('only accepts strings, lists, and tuples')
config['servergitmirrors'] = roots
return config
......@@ -25,6 +25,7 @@ import pwd
import subprocess
from argparse import ArgumentParser
import logging
import shutil
from . import common
......@@ -191,6 +192,39 @@ def update_localcopy(repo_section, local_copy_dir):
_local_sync(repo_section, local_copy_dir)
def update_servergitmirrors(servergitmirrors, repo_section):
# depend on GitPython only if users set a git mirror
import git
# right now we support only 'repo' git-mirroring
if repo_section == 'repo':
# create a new git-mirror folder
repo_dir = os.path.join('.', 'git-mirror/')
# remove if already present
if os.path.isdir(repo_dir):
repo = git.Repo.init(repo_dir)
# take care of each mirror
for mirror in servergitmirrors:
hostname = mirror.split("/")[2]
repo.create_remote(hostname, mirror)
logging.info('Mirroring to: ' + mirror)
# copy local 'repo' to 'git-mirror/fdroid/repo directory' with _local_sync
fdroid_repo_path = os.path.join(repo_dir, "fdroid")
_local_sync(repo_section, fdroid_repo_path)
# sadly index.add don't allow the --all parameter
repo.index.commit("fdroidserver git-mirror")
# push for every remote. This will overwrite the git history
for remote in repo.remotes:
remote.push('master', force=True, set_upstream=True)
def main():
global config, options
......@@ -269,8 +303,9 @@ def main():
if not config.get('awsbucket') \
and not config.get('serverwebroot') \
and not config.get('servergitmirrors') \
and local_copy_dir is None:
logging.warn('No serverwebroot, local_copy_dir, or awsbucket set!'
logging.warn('No serverwebroot, local_copy_dir, or awsbucket set! '
+ 'Edit your config.py to set at least one.')
......@@ -313,6 +348,10 @@ def main():
update_localcopy(repo_section, local_copy_dir)
for serverwebroot in config.get('serverwebroot', []):
update_serverwebroot(serverwebroot, repo_section)
if config.get('servergitmirrors', []):
# update_servergitmirrors will take care of multiple mirrors so don't need a foreach
servergitmirrors = config.get('servergitmirrors', [])
update_servergitmirrors(servergitmirrors, repo_section)
if config.get('awsbucket'):
......@@ -961,6 +961,32 @@ def extract_pubkey():
return hexlify(pubkey)
# Get raw URL from git service for mirroring
def get_raw_mirror(url):
# Divide urls in parts
url = url.split("/")
# Get the hostname
hostname = url[2]
# fdroidserver will use always 'master' branch for git-mirroring
branch = "master"
folder = "fdroid"
if hostname == "github.com":
# Github like RAW url "https://raw.githubusercontent.com/user/repo/master/fdroid"
url[2] = "raw.githubusercontent.com"
url.extend([branch, folder])
elif hostname == "gitlab.com":
# Gitlab like RAW url "https://gitlab.com/user/repo/raw/master/fdroid"
url.extend(["raw", branch, folder])
return None
url = "/".join(url)
return url
def make_index(apps, sortedids, apks, repodir, archive):
"""Generate the repo index files.
......@@ -1012,6 +1038,10 @@ def make_index(apps, sortedids, apks, repodir, archive):
mirrors.append(mirror + '/')
for mirror in config.get('servergitmirrors', []):
mirror = get_raw_mirror(mirror)
if mirror is not None:
mirrors.append(mirror + '/')
if mirrorcheckfailed:
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment