Commit 82eceebd authored by Hans-Christoph Steiner's avatar Hans-Christoph Steiner
Browse files

Merge branch 'parse-donation-links-from-funding.yml' into 'master'

update: insert donation links based on FUNDING.yml

See merge request !754
parents 0b4112f7 62c8fd59
Pipeline #157460918 passed with stage
in 20 minutes and 1 second
......@@ -36,7 +36,7 @@ metadata_v0:
- cd fdroiddata
- ../tests/dump_internal_metadata_format.py
- sed -i
-e '/kivy:\sfalse/d'
-e '/Liberapay:/d'
-e '/OpenCollective/d'
metadata/dump_*/*.yaml
- diff -uw metadata/dump_*
......
......@@ -141,9 +141,11 @@ regex_checks = {
],
'Donate': http_checks + [
(re.compile(r'.*flattr\.com'),
_("Flattr donation methods belong in the FlattrID flag")),
_("Flattr donation methods belong in the FlattrID: field")),
(re.compile(r'.*liberapay\.com'),
_("Liberapay donation methods belong in the LiberapayID flag")),
_("Liberapay donation methods belong in the Liberapay: field")),
(re.compile(r'.*opencollective\.com'),
_("OpenCollective donation methods belong in the OpenCollective: field")),
],
'Changelog': http_checks,
'Author Name': [
......
......@@ -41,6 +41,10 @@ from fdroidserver.exception import MetaDataException, FDroidException
srclibs = None
warnings_action = None
# validates usernames based on a loose collection of rules from GitHub, GitLab,
# Liberapay and issuehunt. This is mostly to block abuse.
VALID_USERNAME_REGEX = re.compile(r'^[a-z\d](?:[a-z\d/._-]){0,38}$', re.IGNORECASE)
def warn_or_exception(value, cause=None):
'''output warning or Exception depending on -W'''
......@@ -73,6 +77,7 @@ app_fields = set([
'Changelog',
'Donate',
'FlattrID',
'Liberapay',
'LiberapayID',
'OpenCollective',
'Bitcoin',
......@@ -117,6 +122,7 @@ yaml_app_field_order = [
'Changelog',
'Donate',
'FlattrID',
'Liberapay',
'LiberapayID',
'OpenCollective',
'Bitcoin',
......@@ -177,6 +183,7 @@ class App(dict):
self.Changelog = ''
self.Donate = None
self.FlattrID = None
self.Liberapay = None
self.LiberapayID = None
self.OpenCollective = None
self.Bitcoin = None
......@@ -450,12 +457,16 @@ valuetypes = {
r'^[0-9a-z]+$',
['FlattrID']),
FieldValidator("Liberapay",
VALID_USERNAME_REGEX,
['Liberapay']),
FieldValidator("Liberapay ID",
r'^[0-9]+$',
['LiberapayID']),
FieldValidator("Open Collective",
r'^[0-9a-zA-Z_-]+$',
VALID_USERNAME_REGEX,
['OpenCollective']),
FieldValidator("HTTP link",
......
......@@ -31,10 +31,15 @@ import zipfile
import hashlib
import json
import time
import yaml
import copy
from datetime import datetime
from argparse import ArgumentParser
from base64 import urlsafe_b64encode
try:
from yaml import CSafeLoader as SafeLoader
except ImportError:
from yaml import SafeLoader
import collections
from binascii import hexlify
......@@ -854,6 +859,122 @@ def _get_base_hash_extension(f):
return base, None, extension
def sanitize_funding_yml_entry(entry):
"""FUNDING.yml comes from upstream repos, entries must be sanitized"""
if type(entry) not in (bytes, int, float, list, str):
return
if isinstance(entry, bytes):
entry = entry.decode()
elif isinstance(entry, list):
if entry:
entry = entry[0]
else:
return
try:
entry = str(entry)
except (TypeError, ValueError):
return
if len(entry) > 2048:
logging.warning(_('Ignoring FUNDING.yml entry longer than 2048: %s') % entry[:2048])
return
if '\n' in entry:
return
return entry.strip()
def sanitize_funding_yml_name(name):
"""Sanitize usernames that come from FUNDING.yml"""
entry = sanitize_funding_yml_entry(name)
if entry:
m = metadata.VALID_USERNAME_REGEX.match(entry)
if m:
return m.group()
return
def insert_funding_yml_donation_links(apps):
"""include donation links from FUNDING.yml in app's source repo
GitHub made a standard file format for declaring donation
links. This parses that format from upstream repos to include in
metadata here. GitHub supports mostly proprietary services, so
this logic adds proprietary services only as Donate: links.
FUNDING.yml can be either in the root of the project, or in the
".github" subdir.
https://help.github.com/en/articles/displaying-a-sponsor-button-in-your-repository#about-funding-files
"""
if not os.path.isdir('build'):
return # nothing to do
for packageName, app in apps.items():
sourcedir = os.path.join('build', packageName)
if not os.path.isdir(sourcedir):
continue
for f in ([os.path.join(sourcedir, 'FUNDING.yml'), ]
+ glob.glob(os.path.join(sourcedir, '.github', 'FUNDING.yml'))):
if not os.path.isfile(f):
continue
data = None
try:
with open(f) as fp:
data = yaml.load(fp, Loader=SafeLoader)
except yaml.YAMLError as e:
logging.error(_('Found bad funding file "{path}" for "{name}":')
.format(path=f, name=packageName))
logging.error(e)
if not data or type(data) != dict:
continue
if not app.get('Liberapay') and 'liberapay' in data:
s = sanitize_funding_yml_name(data['liberapay'])
if s:
app['Liberapay'] = s
if not app.get('OpenCollective') and 'open_collective' in data:
s = sanitize_funding_yml_name(data['open_collective'])
if s:
app['OpenCollective'] = s
if not app.get('Donate'):
del(data['liberapay'])
del(data['open_collective'])
# this tuple provides a preference ordering
for k in ('custom', 'github', 'patreon', 'community_bridge', 'ko_fi', 'issuehunt'):
v = data.get(k)
if not v:
continue
if k == 'custom':
s = sanitize_funding_yml_entry(v)
if s:
app['Donate'] = s
break
elif k == 'community_bridge':
s = sanitize_funding_yml_name(v)
if s:
app['Donate'] = 'https://funding.communitybridge.org/projects/' + s
break
elif k == 'github':
s = sanitize_funding_yml_name(v)
if s:
app['Donate'] = 'https://github.com/sponsors/' + s
break
elif k == 'issuehunt':
s = sanitize_funding_yml_name(v)
if s:
app['Donate'] = 'https://issuehunt.io/r/' + s
break
elif k == 'ko_fi':
s = sanitize_funding_yml_name(v)
if s:
app['Donate'] = 'https://ko-fi.com/' + s
break
elif k == 'patreon':
s = sanitize_funding_yml_name(v)
if s:
app['Donate'] = 'https://patreon.com/' + s
break
def copy_triple_t_store_metadata(apps):
"""Include store metadata from the app's source repo
......@@ -2179,6 +2300,7 @@ def main():
else:
logging.warning(msg + '\n\t' + _('Use `fdroid update -c` to create it.'))
insert_funding_yml_donation_links(apps)
copy_triple_t_store_metadata(apps)
insert_obbs(repodirs[0], apps, apks)
insert_localized_app_metadata(apps)
......
bad:
- "Robert'); DROP TABLE Students; --"
- ''
- -a-b
- '1234567890123456789012345678901234567890'
- ~derp@darp---++asdf
- foo@bar.com
- me++
- --me
bitcoin:
- 3Lbz4vdt15Fsa4wVD3Yk8uGf6ugKKY4zSc
community_bridge: []
custom:
- bc1qvll2mp5ndwd4sgycu4ad2ken4clhjac7mdlcaj
- http://www.roguetemple.com/z/donate.php
- https://donate.openfoodfacts.org
- https://email.faircode.eu/donate/
- https://etchdroid.depau.eu/donate/
- https://f-droid.org/about/
- https://flattr.com/github/bk138
- https://gultsch.de/donate.html
- https://jahir.dev/donate
- https://kodi.tv/contribute/donate
- https://link.xbrowsersync.org/cryptos
- https://manyver.se/donate
- https://paypal.me/DanielQuahShaoHian
- https://paypal.me/deletescape
- https://paypal.me/freaktechnik
- https://paypal.me/hpoul
- https://paypal.me/imkosh
- https://paypal.me/paphonb
- https://paypal.me/vocabletrainer
- https://pendulums.io/donation.html
- https://play.google.com/store/apps/details?id=de.dennisguse.opentracks.playstore
- https://play.google.com/store/apps/details?id=eu.faircode.email
- https://raw.githubusercontent.com/Blankj/AndroidUtilCode/master/art/donate.png
- https://raw.githubusercontent.com/CarGuo/GSYGithubAppFlutter/master/thanks.jpg
- https://raw.githubusercontent.com/GanZhiXiong/GZXTaoBaoAppFlutter/blob/master/preview_images/thanks.png
- https://seriesgui.de/whypay
- https://transportr.app/donate/
- https://www.bountysource.com/teams/nextcloud/issues?tracker_ids=38838206
- https://www.donationalerts.com/r/blidingmage835
- https://www.hellotux.com/f-droid
- https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=8UH5MBVYM3J36
- https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=E2FCXCT6837GL
- https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=FMLNN8GXZKJEE
- https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=K7HVLE6J7SXXA
- https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=ZD39ZE7MGEGBL&source=url
- https://www.paypal.me/SimpleMobileTools
- https://www.paypal.me/TheAlphamerc/
- https://www.paypal.me/avirias
- https://www.paypal.me/btimofeev
- https://www.paypal.me/enricocid
- https://www.paypal.me/gsnathan
- https://www.paypal.me/nikita36078
- https://www.paypal.me/sahdeep
- https://www.paypal.me/saulhenriquez
- https://www.simplemobiletools.com/donate
- https://www.youtube.com/watch?v=ZmrNc1ZhBkQ
- paypal.me/amangautam1
- paypal.me/pools/c/8lCZfNnU0u
- paypal.me/psoffritti
github:
- 00-Evan
- adrcotfas
- afollestad
- ar-
- BarnabyShearer
- CarGuo
- cketti
- eighthave
- emansih
- GanZhiXiong
- gpeal
- hpoul
- i--
- inorichi
- inputmice
- jahirfiquitiva
- johnjohndoe
- kaloudis
- kiwix
- ligi
- M66B
- mikepenz
- Mygod
- paroj
- PerfectSlayer
- sschueller
- tateisu
- tibbi
- westnordost
- x1unix
- xn--nding-jua
- zenorogue
issuehunt:
- bk138/multivnc
ko_fi:
- afollestad
- fennifith
- inorichi
- mastalab
- psoffritti
liberapay:
- ActivityDiary
- AndStatus
- BM835
- Briar
- DAVx5
- F-Droid-Data
- Feeel
- Fruit-Radar-Development
- Gadgetbridge
- GuardianProject
- Hocuri
- KOReader
- Kanedias
- Kunzisoft
- MaxK
- NovaVideoPlayer
- Phie
- Rudloff
- Schoumi
- Syncthing-Fork
- TeamNewPipe
- Telegram-FOSS
- Transportr
- Varlorg
- Wesnoth
- ZiiS
- ar-
- bk138
- btimofeev
- bubblineyuri
- dennis.guse
- developerfromjokela
- devgianlu
- eneiluj
- experiment322
- fdossena
- fennifith
- freaktechnik
- gsantner
- hisname
- hsn6
- iNPUTmice
- inputmice
- k9mail
- matrixdotorg
- mmarif
- moezbhatti
- proninyaroslav
- quite
- renyuneyun
- rocketnine.space
- sanskritbscs
- sschueller
- sschueller/donate
- stefan-niedermann
- tasks
- teamkodi
- thermatk
- tom79
- wallabag
- westnordost
- whyorean
- wilko
- xbrowsersync
- yeriomin
- zeh
open_collective:
- avirias
- curl
- libsodium
- manyverse
- mastalab
- tusky
otechie: []
patreon:
- BaldPhone
- Bm835
- FastHub
- Teamkodi
- andrestaltz
- bk138
- depau
- iamSahdeep
- ligi
- ogre1
- orhunp
- tiborkaputa
- tom79
- westnordost
- xbrowsersync
- yairm210
- zenorogue
tidelift: []
......@@ -100,6 +100,21 @@ class MetadataTest(unittest.TestCase):
self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
'tb1qw5r8drrejxrrg4y5rrrrrraryrrrrwrkxrjrsx', 'fake.app.id')
def test_valid_funding_yml_regex(self):
"""Check the regex can find all the cases"""
with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp:
data = yaml.safe_load(fp)
for k, entries in data.items():
for entry in entries:
m = fdroidserver.metadata.VALID_USERNAME_REGEX.match(entry)
if k == 'custom':
pass
elif k == 'bad':
self.assertIsNone(m, 'this is an invalid %s username: {%s}' % (k, entry))
else:
self.assertIsNotNone(m, 'this is a valid %s username: {%s}' % (k, entry))
def test_read_metadata(self):
def _build_yaml_representer(dumper, data):
......
......@@ -17,6 +17,7 @@ Disabled: null
Donate: null
FlattrID: null
IssueTracker: https://github.com/miguelvps/PoliteDroid/issues
Liberapay: null
LiberapayID: null
License: GPL-3.0-only
Litecoin: null
......
......@@ -40,7 +40,8 @@ Disabled: null
Donate: http://sufficientlysecure.org/index.php/adaway
FlattrID: '369138'
IssueTracker: https://github.com/dschuermann/ad-away/issues
LiberapayID: null
Liberapay: null
LiberapayID: '1234567890'
License: GPL-3.0-only
Litecoin: null
MaintainerNotes: ''
......
......@@ -37,6 +37,7 @@ Disabled: null
Donate: null
FlattrID: null
IssueTracker: https://github.com/SMSSecure/SMSSecure/issues
Liberapay: null
LiberapayID: null
License: GPL-3.0-only
Litecoin: null
......
......@@ -24,6 +24,7 @@ Disabled: null
Donate: http://www.videolan.org/contribute.html#money
FlattrID: null
IssueTracker: http://www.videolan.org/support/index.html#bugs
Liberapay: null
LiberapayID: null
License: GPL-3.0-only
Litecoin: null
......
......@@ -7,6 +7,7 @@ SourceCode: https://github.com/guardianproject/checkey
IssueTracker: https://dev.guardianproject.info/projects/checkey/issues
Translation: https://www.transifex.com/otf/checkey
Bitcoin: 1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
Liberapay: GuardianProject
AutoName: Checkey
......
......@@ -3,6 +3,7 @@ Categories:
License: GPL-3.0-only
SourceCode: https://github.com/eighthave/urzip
Bitcoin: 1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
Liberapay: 12334
AutoName: OBB Main Old Version
......
......@@ -8,6 +8,7 @@ IssueTracker: https://github.com/dschuermann/ad-away/issues
Translation: https://www.transifex.com/dominikschuermann/adaway
Donate: http://sufficientlysecure.org/index.php/adaway
FlattrID: '369138'
LiberapayID: '1234567890'
AutoName: AdAway
Summary: Block advertisements
......
......@@ -104,6 +104,7 @@
"Development"
],
"suggestedVersionCode": "99999999",
"liberapay": "12334",
"license": "GPL-3.0-only",
"name": "OBB Main Old Version",
"sourceCode": "https://github.com/eighthave/urzip",
......
......@@ -8,6 +8,7 @@ import inspect
import logging
import optparse
import os
import random
import shutil
import subprocess
import sys
......@@ -33,6 +34,13 @@ import fdroidserver.update
from fdroidserver.common import FDroidPopen
DONATION_FIELDS = (
'Donate',
'Liberapay',
'OpenCollective',
)
class UpdateTest(unittest.TestCase):
'''fdroid update'''
......@@ -972,6 +980,86 @@ class UpdateTest(unittest.TestCase):
'VercodeOperation': '',
'WebSite': ''})
def test_insert_funding_yml_donation_links(self):
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
os.chdir(testdir)
os.mkdir('build')
content = textwrap.dedent("""
community_bridge: ''
custom: [LINK1, LINK2]
github: USERNAME
issuehunt: USERNAME
ko_fi: USERNAME
liberapay: USERNAME
open_collective: USERNAME
otechie: USERNAME
patreon: USERNAME
""")
app = fdroidserver.metadata.App()
app.id = 'fake.app.id'
apps = {app.id: app}
os.mkdir(os.path.join('build', app.id))
fdroidserver.update.insert_funding_yml_donation_links(apps)
for field in DONATION_FIELDS:
self.assertFalse(app.get(field))
with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp:
fp.write(content)
fdroidserver.update.insert_funding_yml_donation_links(apps)
for field in DONATION_FIELDS:
self.assertIsNotNone(app.get(field), field)
self.assertEqual('LINK1', app.get('Donate'))
self.assertEqual('USERNAME', app.get('Liberapay'))
self.assertEqual('USERNAME', app.get('OpenCollective'))
app['Donate'] = 'keepme'
app['Liberapay'] = 'keepme'
app['OpenCollective'] = 'keepme'
fdroidserver.update.insert_funding_yml_donation_links(apps)
for field in DONATION_FIELDS:
self.assertEqual('keepme', app.get(field))
def test_insert_funding_yml_donation_links_with_corrupt_file(self):
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
os.chdir(testdir)
os.mkdir('build')
app = fdroidserver.metadata.App()
app.id = 'fake.app.id'
apps = {app.id: app}
os.mkdir(os.path.join('build', app.id))
with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp:
fp.write(textwrap.dedent("""
opencollective: foo
custom: []
liberapay: :
"""))
fdroidserver.update.insert_funding_yml_donation_links(apps)
for field in DONATION_FIELDS:
self.assertIsNone(app.get(field))
def test_sanitize_funding_yml(self):
with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp:
data = yaml.safe_load(fp)
for k, entries in data.items():
for entry in entries:
if k in 'custom':
m = fdroidserver.update.sanitize_funding_yml_entry(entry)
else:
m = fdroidserver.update.sanitize_funding_yml_name(entry)
if k == 'bad':
self.assertIsNone(m)
else:
self.assertIsNotNone(m)
self.assertIsNone(fdroidserver.update.sanitize_funding_yml_entry('foo\nbar'))
self.assertIsNone(fdroidserver.update.sanitize_funding_yml_entry(
''.join(chr(random.randint(65, 90)) for _ in range(2049))))
# not recommended but valid entries
self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(12345))