support cloud storage with libcloud, starting with Amazon AWS S3

apache-libcloud enables uploading to basically any cloud storage service.
This is the first implementation that allows `fdroid server` to push a repo
up to a AWS S3 'bucket'.  Supporting other cloud storage services should
mostly be a matter of finding the libcloud "Provider" and setting the
access creditials.

fixes #3137 https://dev.guardianproject.info/issues/3137
parent d1cd8177
......@@ -102,6 +102,15 @@ keyaliases['com.example.another.plugin'] = '@com.example.another'
# sub-directories (i.e. /var/www/packagerepos/fdroid).
#serverwebroot = '[email protected]:/var/www/fdroid'
# To upload the repo to an Amazon S3 bucket using `fdroid server update`.
# Warning, this deletes and recreates the whole fdroid/ directory each
# time. This is based on apache-libcloud, which supports basically all cloud
# storage services, so it should be easy to port the fdroid server tools to
# any of them.
#awsbucket = 'myawsfdroid'
#awsaccesskeyid = 'SEE0CHAITHEIMAUR2USA'
#awssecretkey = 'yourverysecretkeywordpassphraserighthere'
# If you want to force 'fdroid server' to use a non-standard serverwebroot
#nonstandardwebroot = False
......
......@@ -27,6 +27,53 @@ import common
config = None
options = None
def update_awsbucket(repo_section):
'''
Upload the contents of the directory `repo_section` (including
subdirectories) to the AWS S3 "bucket". The contents of that subdir of the
bucket will first be deleted.
Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey
'''
import libcloud.security
libcloud.security.VERIFY_SSL_CERT = True
from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver
if 'awsaccesskeyid' not in config or 'awssecretkey' not in config:
logging.error('To use awsbucket, you must set awssecretkey and awsaccesskeyid in config.py!')
sys.exit(1)
awsbucket = config['awsbucket']
cls = get_driver(Provider.S3)
driver = cls(config['awsaccesskeyid'], config['awssecretkey'])
container = driver.get_container(container_name=awsbucket)
upload_dir = 'fdroid/' + repo_section
if options.verbose:
logging.info('Deleting existing repo on Amazon S3 bucket: "' + awsbucket
+ '/' + upload_dir + '"')
for obj in container.list_objects():
if obj.name.startswith(upload_dir + '/'):
obj.delete()
if options.verbose:
logging.info(' deleted ' + obj.name)
if options.verbose:
logging.info('Uploading to Amazon S3 bucket: "' + awsbucket + '/' + upload_dir + '"')
for root, _, files in os.walk(os.path.join(os.getcwd(), repo_section)):
for name in files:
file_to_upload = os.path.join(root, name)
object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
if options.verbose:
logging.info(' ' + file_to_upload + '...')
extra = { 'acl': 'public-read' }
driver.upload_object(file_path=file_to_upload,
container=container,
object_name=object_name,
extra=extra)
def update_serverwebroot(repo_section):
rsyncargs = ['rsync', '-u', '-r', '--delete']
......@@ -84,11 +131,8 @@ def main():
+ serverwebroot.rstrip('/') + '/fdroid\n\t'
+ serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid')
sys.exit(1)
else:
serverwebroot = None
if serverwebroot == None:
logging.warn('No serverwebroot set! Edit your config.py to set one.')
elif 'awsbucket' not in config:
logging.warn('No serverwebroot or awsbucket set! Edit your config.py to set one or both.')
sys.exit(1)
repo_sections = ['repo']
......@@ -110,8 +154,10 @@ def main():
sys.exit(1)
elif args[0] == 'update':
for repo_section in repo_sections:
if serverwebroot != None:
if 'serverwebroot' in config:
update_serverwebroot(repo_section)
if 'awsbucket' in config:
update_awsbucket(repo_section)
sys.exit(0)
......
......@@ -32,6 +32,7 @@ setup(name='fdroidserver',
'paramiko',
'PIL',
'python-magic',
'apache-libcloud >= 0.5',
],
classifiers=[
'Development Status :: 3 - Alpha',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment