.issuebot.py

#!/usr/bin/env python3

import configparser
import git
import gitlab
import github
import glob
import hashlib
import os
import re
import requests
import shutil
import subprocess
import sys
import time
import traceback

from androguard.core.bytecodes.apk import APK
from fdroidserver import net
from urllib.parse import parse_qs, urlparse

JAVA_PACKAGENAME = '''(?:[a-zA-Z]+(?:\d*[a-zA-Z_]*)*)(?:\.[a-zA-Z]+(?:\d*[a-zA-Z_]*)*)+'''
GPLAY_PATTERN = re.compile('http[s]?://play\.google\.com/store/apps/details\?id=' + JAVA_PACKAGENAME)
GIT_PATTERN = re.compile('http[s]?://(codeberg.org|github.com|gitlab.com|bitbucket.org|git.code.sf.net)/[\w.-]+/[\w.-]+')
PACKAGE_ID_PATTERN = re.compile('(APPLICATION|PACKAGE) ?(ID|NAME):?\s*' + JAVA_PACKAGENAME, re.IGNORECASE | re.DOTALL)
# find all repositories that use plain HTTP urls (e.g. not HTTPS)
HTTP_GRADLE_PATTERN = re.compile('repositories\s*{[^}]*http://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+[^}]*}', re.DOTALL)
METADATA_PATTERN = re.compile('metadata/' + JAVA_PACKAGENAME + '\.yml$')
IZZYSOFT_PATTERN = re.compile(b'''.*<a [^>]* href=['"](.+)['"][^>]*>Download.*''')

HEADERS = {'User-Agent': 'F-Droid Issuebot'}

PERSONAL_ACCESS_TOKEN = os.getenv('PERSONAL_ACCESS_TOKEN')
VIRUSTOTAL_API_KEY = os.getenv('VIRUSTOTAL_API_KEY')

def _requests_get(url):
    return requests.get(url, headers=HEADERS)

def _requests_head(url):
    return requests.head(url, headers=HEADERS)

def download_file(url):
    try:
        net.download_file(url, dldir='.')
    except requests.exceptions.HTTPError as e:
        print(e)

def get_apk_from_github(url, labels):
    if url.startswith('https://github.com/'):
        token = os.getenv('GITHUB_TOKEN')
        g = github.Github(token)
        repo = g.get_repo('/'.join(url.split('/')[3:5]))
        try:
            release = repo.get_latest_release()
            for asset in release.get_assets():
                if asset.browser_download_url.endswith('.apk'):
                    print('Downloading from %s/releases/latest' % url)
                    download_file(asset.browser_download_url)
                    labels.add('in-github-releases')
        except github.GithubException as e:
            print(e)

def get_virustotal_status(apkfilename, sha256, headers=None, apikey=None):
    """Get the status of the APK on VirusTotal, uploading as needed

    returns the JSON info from VirusTotal
    """
    if not headers:
        headers = {
            "User-Agent": "F-Droid"
        }
    params = {
        'apikey': VIRUSTOTAL_API_KEY,
        'resource': sha256,
    }
    needs_file_upload = False
    response = dict()
    while True:
        r = requests.post('https://www.virustotal.com/vtapi/v2/file/report',
                          params=params, headers=headers)
        if r.status_code == 200:
            response = r.json()
            if response['response_code'] == 0:
                needs_file_upload = True
            break
        elif r.status_code == 204:
            print('Waiting for VirusTotal rate limiting...')
            time.sleep(10)  # wait for public API rate limiting
        elif r.status_code == 403:
            print('VirusTotal throws a 403 (API key invalid?)')
            break;
        else:
            try:
                r.raise_for_status()
            except Exception:
                print(re.sub(r'apikey=[0-9a-f]+', '', traceback.format_exc()))
                sys.exit(1)

    if needs_file_upload:
        print('Uploading ' + apkfilename + ' to virustotal')
        files = {
            'file': (os.path.basename(apkfilename), open(apkfilename, 'rb'))
        }
        r = requests.post('https://www.virustotal.com/vtapi/v2/file/scan',
                          params=params, headers=headers, files=files)
        response = r.json()
    return response


gl = gitlab.Gitlab('https://gitlab.com', api_version=4,
                   private_token=PERSONAL_ACCESS_TOKEN)
rfp = gl.projects.get('fdroid/rfp')

os.makedirs('build', exist_ok=True)
os.makedirs('metadata', exist_ok=True)
os.makedirs('repo', exist_ok=True)
with open('fdroid-icon.png', 'w') as fp:
    fp.write('')
with open('config.py', 'w') as fp:
    fp.write('repo_pubkey = "deadbeef"\n')
    fp.write('make_current_version_link = False\n')
    fp.write('androidobservatory = True\n')
    if VIRUSTOTAL_API_KEY:
        fp.write('virustotal_apikey = "%s"\n' % VIRUSTOTAL_API_KEY)

processed = 0
# per_page: by default, GitLab returns only 20 entries. At max unfortunately 100
for issue in rfp.issues.list(state='opened',order_by='updated_at',per_page=250):
    note = ''
    security = ''
    labels = set(issue.labels)
    if 'fdroid-bot' in labels:
        # print('Skipping %s (already has fdroid-bot label).' % issue.title)
        continue
    if processed > 21:
        print('Processed %d app, quitting.' % processed)
        break
    processed += 1
    labels.add('fdroid-bot')

    print('Checking', issue.title)

    git_urls = set()
    for m in GIT_PATTERN.finditer(issue.description):
        url = m.group(0)
        if url.startswith('http://'):
            url = 'https://' + url[7:]
        if not url.startswith('https://gitlab.com/fdroid/'):
            git_urls.add(url)
            labels.add('git-url')
    for git_url in git_urls:
        get_apk_from_github(git_url, labels)

    appid = None
    gplay_urls = set()
    for m in GPLAY_PATTERN.finditer(issue.description):
        gplay_urls.add(m.group(0))
    for url in sorted(gplay_urls):
        appid = parse_qs(urlparse(url).query)['id'][0]
        cmd = ['gplaycli', '-d', appid]
        print('$ %s' % ' '.join(cmd))
        sp = subprocess.run(cmd, stderr=subprocess.STDOUT)
        print(sp.stdout)

    if not appid:
        m = PACKAGE_ID_PATTERN.search(issue.description)
        if m:
            appid = m.group(0).split(':')[1].strip()

    if not appid:
        for url in git_urls:
            cmd = ['fdroid', 'import', '-Wwarn', '--url', url]
            print('$ %s' % ' '.join(cmd))
            p = subprocess.run(cmd, stderr=subprocess.STDOUT, universal_newlines=True)
            if p.returncode == 0:
                for f in glob.glob(os.path.join('metadata', '*.yml')):
                    with open(f, errors='surrogateescape') as fp:
                        if url in fp.read():
                            m = METADATA_PATTERN.search(f)
                            if m:
                                appid = m.group(0)[9:-4]
            elif p.stdout:
                    note += '\n\n## fdroid import\n\n'
                    note += '%s:\n\n```console\n$ %s\n' % (url, ' '.join(cmd))
                    note += p.stdout + '\n```\n\n'

    if not appid:
        for f in glob.glob('*.apk'):
            try:
                appid = APK(f).get_package()
                break
            except Exception as e:
                print(e)

    apkfilename = None
    if appid:
        apkfilename = appid + '.apk'
        fdroiddata_url = 'https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/' + appid + '.yml'
        r = _requests_head(fdroiddata_url)
        if r.status_code == 200:
            note += 'Closing issue, since I found %s in F-Droid.  ' % appid
            note += 'If you think I made a mistake, please reopen this issue! For more info:\n\n'
            note += '* [fdroiddata](%s)\n' % fdroiddata_url
            note += '* [F-Droid wiki](https://f-droid.org/wiki/page/%s)\n' % appid
            note += '* [F-Droid GitLab](%s)\n' \
                    % ('https://gitlab.com/search?group_id=28397&scope=issues&search=' + appid)
            labels.add('in-fdroiddata')
            issue.state_event = 'close'
        url = 'https://f-droid.org/packages/' + appid
        r = _requests_head(url)
        if r.status_code == 200:
            note += '* [f-droid.org](%s)\n' % url
            labels.add('in-f-droid.org')

        gplay_urls.add('https://play.google.com/store/apps/details?id=' + appid)
        urls = set(gplay_urls)
        izzysoft_url = 'https://apt.izzysoft.de/fdroid/index/apk/' + appid
        urls.add(izzysoft_url)
        for url in urls:
            r = _requests_head(url)
            if r.status_code == 200:
                note += '* found %s\n' %url
                if url.startswith('https://play.google.com'):
                    labels.add('in-google-play')
                elif url.startswith('https://apt.izzysoft.de'):
                    labels.add('in-izzysoft')
                    print('looking for', apkfilename, 'in izzysoft')
                    if not os.path.exists(apkfilename):
                        r = _requests_get(url)
                        m = IZZYSOFT_PATTERN.search(r.content)
                        if m:
                            dl_url = 'https://apt.izzysoft.de' + m.group(1).decode()
                            print('Downloading', dl_url)
                            # the stream=True parameter keeps memory usage low
                            r = requests.get(dl_url, stream=True, allow_redirects=True,
                                             headers=HEADERS)
                            if r.status_code == 200:
                                with open(apkfilename, 'wb') as fp:
                                    for chunk in r.iter_content(chunk_size=64*1024):
                                        if chunk:  # filter out keep-alive new chunks
                                            fp.write(chunk)
                                    fp.flush()

    repo = None
    for url in sorted(git_urls):
        r = _requests_head(url)
        if r.status_code == 200:
            if appid:
                metadata_file = os.path.join('metadata', appid + '.yml')
                if not os.path.exists(metadata_file):
                    with open(metadata_file, 'w') as fp:
                        fp.write('RepoType: git\nRepo: %s\n' % url)
                repo_path = os.path.join('build', appid)
                if os.path.exists(repo_path):
                    repo = git.Repo(repo_path)
                else:
                    print('$ git clone', url, repo_path)
                    repo = git.Repo.clone_from(url, repo_path)

                print('Checking for gradle')
                insecure_repositories = set()
                for root, dirs, files in os.walk(repo_path):
                    for f in files:
                        if f.endswith('.gradle') or f.endswith('.gradle.kts'):
                            labels.add('gradle')
                            path = os.path.join(root, f)
                            if os.path.exists(path):
                                with open(path, errors='surrogateescape') as fp:
                                    data = fp.read()
                                for url in HTTP_GRADLE_PATTERN.findall(data):
                                    print('Found plain HTTP URL for gradle repository:\n%s\n%s'
                                          % (path, url))
                                    insecure_repositories.add(url)
                        elif f.endswith('.java'):
                            labels.add('java')
                        elif f.endswith('.kt'):
                            labels.add('kotlin')
                        elif f == 'App.js' or f == 'App.jsx':
                            labels.add('react-native')
                        elif f == 'Android.mk' or f == 'Application.mk':
                            labels.add('ndk')
                        elif f.endswith('.cc') or f.endswith('.cpp'):
                            labels.add('c++')
                    for d in dirs:
                        if d == 'fastlane':
                            labels.add('fastlane')
                if insecure_repositories:
                    security += ('* gradle build uses %d plain HTTP URLs for repositories!  This is insecure! See: %s\n'
                                 % (len(insecure_repositories),
                                    'https://max.computer/blog/how-to-take-over-the-computer-of-any-java-or-clojure-or-scala-developer/'))
                    for url in insecure_repositories:
                        security += '  * %s\n' % url
                    labels.add('insecure-repositories')

                prop = os.path.join(repo_path, 'gradle', 'wrapper', 'gradle-wrapper.properties')
                if os.path.exists(prop):
                    labels.add('gradle')
                    with open(prop, errors='surrogateescape') as fp:
                        propdata = fp.read()
                    config = configparser.ConfigParser()
                    config.read_string('[DEFAULT]\n' + propdata)  # fake a INI file
                    gradle_url = config['DEFAULT']['distributionUrl'].replace('\\', '')

                    gradle_jar = os.path.join(repo_path, 'gradle', 'wrapper', 'gradle-wrapper.jar')
                    hasher = hashlib.sha256()
                    with open(gradle_jar, 'rb') as fp:
                        hasher.update(fp.read())
                    sha256 = hasher.hexdigest().encode()

                    wrapper_url = re.sub(r'(bin|all).zip', r'wrapper.jar', gradle_url)
                    r = requests.get(wrapper_url + '.sha256', allow_redirects=True, headers=HEADERS)
                    if r.status_code == 200:
                        if not r.content.startswith(sha256):
                            note += ('* _gradle/wrapper/gradle-wrapper.jar_ does not match the wrapper version ('
                                     + gradle_url + ').  See update and verify instructions: '
                                     + 'https://docs.gradle.org/current/userguide/gradle_wrapper.html#sec:upgrading_wrapper')
                            s = _requests_get('https://gradle.org/release-checksums/')
                            if s.status_code == 200:
                                if sha256 not in s.content:
                                    labels.add('insecure-gradlew')
                                    security += ('* Custom _gradle/wrapper/gradle-wrapper.jar_, not found '
                                                 + 'in https://gradle.org/release-checksums: '
                                                 + sha256.decode()+ '\n')
                            else:
                                security += ('* failed to fetch https://gradle.org/release-checksums\n')
                    else:
                        labels.add('insecure-gradlew')
                        security += ('* _gradle/wrapper/gradle-wrapper.properties_ contains wrapper not on'
                                     + 'official website: ' + wrapper_url + '\n')

                    if gradle_url.startswith('http:'):
                        labels.add('insecure-gradlew')
                        security += '* Insecure HTTP gradle download, use _%s_!\n' \
                                    % gradle_url.replace('http:', 'https:')
                    if not gradle_url.startswith('https://services.gradle.org/distributions/gradle-'):
                        labels.add('insecure-gradlew')
                        security += ('* _gradle/wrapper/gradle-wrapper.properties_ uses non-standard source '
                                     + 'for downloading gradle: ' + gradle_url)
                    if 'distributionSha256Sum' not in config['DEFAULT']:
                        labels.add('insecure-gradlew')
                        security += ('* _gradle/wrapper/gradle-wrapper.properties_ is missing [distributionSha256Sum]('
                                     + 'https://docs.gradle.org/current/userguide/gradle_wrapper.html#sec:verification), '
                                     + 'unverified gradle download!')
                        r = requests.get(gradle_url + '.sha256', allow_redirects=True, headers=HEADERS)
                        if r.status_code == 200:
                            security += (' Here is an example of how to fix this:\n\n'
                                         + '```properties\n{propdata}\ndistributionSha256Sum={sha256}\n```\n\n'
                                         .format(propdata=propdata.strip(), sha256=r.content.decode()))
                        else:
                            security += '\n\n'

                cmd = ['fdroid', 'scanner', '--verbose', appid]
                print('$ %s' % ' '.join(cmd))
                scanner_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode()
                if 'WARNING:' in scanner_output:
                    labels.add('scanner-warning')
                if 'ERROR:' in scanner_output:
                    labels.add('scanner-error')
                note += '\n\n## fdroid scanner\n\n'
                note += '%s:\n\n```console\n$ %s\n' % (url, ' '.join(cmd))
                note += scanner_output + '\n```\n\n'

                break

    report = ''
    if repo:
        print('Scanning tag/branch names')
        for tag in repo.tags:
            suspicious = set()
            for c in re.findall(r'[^a-zA-Z0-9.,()/+_#@-]', tag.name):
                suspicious.add(c)
            if suspicious:
                report += "* %s (%s)\n" % (tag, ''.join(sorted(suspicious)))
        if report:
            report = '\n\n### Suspicious tag/branch names\n\n' + report
            labels.add('suspicious-names')

    if security or report:
        note += '\n\n## Security Issues\n\n' + security + '\n\n' + report

    scanners = ''
    if appid:
        scanners += '* [APK Scan %s](https://apkscan.org/?searchby=pkg&q=%s)\n' \
                    % (appid, appid)
        if os.path.exists(apkfilename):
            hasher = hashlib.sha256()
            with open(apkfilename, 'rb') as fp:
                while True:
                    chunk = fp.read(65536)
                    if not chunk:
                        break
                    hasher.update(chunk)
            sha256 = hasher.hexdigest()
            scanners += '* [APK Scan](https://apkscan.org/?searchby=hash&q=%s)\n' % sha256

            print('Checking VirusTotal')
            response = get_virustotal_status(apkfilename, sha256, apikey=VIRUSTOTAL_API_KEY)
            if response.get('response_code') is None:
                print('No response from VirusTotal')
            elif response['response_code'] < 0:
                labels.add('in-virustotal')
                scanners += ('* ' + apkfilename + ': [' + response.get('verbose_msg')
                             + '](https://www.virustotal.com/file/' + response['resource'] + ')\n')
            elif response['response_code'] == 0:
                scanners += ('* ' + apkfilename + ' has been uploaded to VirusTotal: ['
                             + response['verbose_msg'] + '](' + response['permalink'] + ')\n')
            else:
                if response.get('positives', 0) == 0:
                    labels.add('in-virustotal')
                    scanners += ('* ' + apkfilename + ' has [not been flagged by VirusTotal]('
                                 + response['permalink'] + ')\n')
                else:
                    labels.add('in-virustotal')
                    labels.add('flagged-by-virustotal')
                    scanners += ('* ' + apkfilename + ' has been [flagged by VirusTotal '
                               + str(response['positives']) + ' times]('
                               + response['permalink'] + ')\n')

        t = _requests_get('https://reports.exodus-privacy.eu.org/api/trackers')
        r = _requests_get(('https://reports.exodus-privacy.eu.org/api/search/%s' % appid))
        if t.status_code == 200 and r.status_code == 200:
            try:
                trackers = t.json()['trackers']
                reports = r.json()
                if appid in reports:
                    exodus = ''
                    reports = r.json()[appid]['reports']
                    if len(reports) > 0:
                        for trackernum in reports[0]['trackers']:
                            tracker = trackers[str(trackernum)]
                            exodus += '* [%s](%s)\n' % (tracker['name'], tracker['website'])
                    if exodus:
                        labels.add('trackers')
                        scanners += '### [Exodus Privacy](https://reports.exodus-privacy.eu.org/search)\n\n'
                        scanners += exodus
            except Exception as e:
                print(appid, '-', e)

    if scanners:
        note += '\n\n## External Scanners\n\n' + scanners + '\n\n'

    emojis = issue.awardemojis.list()
    add_robot = True
    for emoji in emojis:
        if emoji.name == 'robot':
            add_robot = False
            break
    if add_robot:
        issue.awardemojis.create({'name': 'robot'})

    if appid:
        labels.add(appid)
    if labels:
        issue.labels = sorted(labels)

    if note:
        n = issue.notes.create({'body': note})
    issue.save()

# attempt to upload all APKs to APK Scan
cmd = 'fdroid update --create-metadata --nosign'
print('$ ' + cmd)
for f in glob.glob('*.apk'):
    shutil.move(f, 'repo')
subprocess.call(cmd.split())

cmd = 'fdroid server update --verbose'
print('$ ' + cmd)
subprocess.call(cmd.split())

print('DONE')