support cloud storage with libcloud, starting with Amazon AWS S3

apache-libcloud enables uploading to basically any cloud storage service.
This is the first implementation that allows `fdroid server` to push a repo
up to a AWS S3 'bucket'.  Supporting other cloud storage services should
mostly be a matter of finding the libcloud "Provider" and setting the
access creditials.

fixes #3137
parent d1cd8177
......@@ -102,6 +102,15 @@ keyaliases['com.example.another.plugin'] = '@com.example.another'
# sub-directories (i.e. /var/www/packagerepos/fdroid).
#serverwebroot = '[email protected]:/var/www/fdroid'
# To upload the repo to an Amazon S3 bucket using `fdroid server update`.
# Warning, this deletes and recreates the whole fdroid/ directory each
# time. This is based on apache-libcloud, which supports basically all cloud
# storage services, so it should be easy to port the fdroid server tools to
# any of them.
#awsbucket = 'myawsfdroid'
#awsaccesskeyid = 'SEE0CHAITHEIMAUR2USA'
#awssecretkey = 'yourverysecretkeywordpassphraserighthere'
# If you want to force 'fdroid server' to use a non-standard serverwebroot
#nonstandardwebroot = False
......@@ -27,6 +27,53 @@ import common
config = None
options = None
def update_awsbucket(repo_section):
Upload the contents of the directory `repo_section` (including
subdirectories) to the AWS S3 "bucket". The contents of that subdir of the
bucket will first be deleted.
Requires AWS credentials set in awsaccesskeyid, awssecretkey
import = True
from import Provider
from import get_driver
if 'awsaccesskeyid' not in config or 'awssecretkey' not in config:
logging.error('To use awsbucket, you must set awssecretkey and awsaccesskeyid in!')
awsbucket = config['awsbucket']
cls = get_driver(Provider.S3)
driver = cls(config['awsaccesskeyid'], config['awssecretkey'])
container = driver.get_container(container_name=awsbucket)
upload_dir = 'fdroid/' + repo_section
if options.verbose:'Deleting existing repo on Amazon S3 bucket: "' + awsbucket
+ '/' + upload_dir + '"')
for obj in container.list_objects():
if + '/'):
if options.verbose:' deleted ' +
if options.verbose:'Uploading to Amazon S3 bucket: "' + awsbucket + '/' + upload_dir + '"')
for root, _, files in os.walk(os.path.join(os.getcwd(), repo_section)):
for name in files:
file_to_upload = os.path.join(root, name)
object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
if options.verbose:' ' + file_to_upload + '...')
extra = { 'acl': 'public-read' }
def update_serverwebroot(repo_section):
rsyncargs = ['rsync', '-u', '-r', '--delete']
......@@ -84,11 +131,8 @@ def main():
+ serverwebroot.rstrip('/') + '/fdroid\n\t'
+ serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid')
serverwebroot = None
if serverwebroot == None:
logging.warn('No serverwebroot set! Edit your to set one.')
elif 'awsbucket' not in config:
logging.warn('No serverwebroot or awsbucket set! Edit your to set one or both.')
repo_sections = ['repo']
......@@ -110,8 +154,10 @@ def main():
elif args[0] == 'update':
for repo_section in repo_sections:
if serverwebroot != None:
if 'serverwebroot' in config:
if 'awsbucket' in config:
......@@ -32,6 +32,7 @@ setup(name='fdroidserver',
'apache-libcloud >= 0.5',
'Development Status :: 3 - Alpha',
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment