Commit 9607cdb6 authored by Hans-Christoph Steiner's avatar Hans-Christoph Steiner

Merge branch 'download-index-etag' into 'master'

Support ETag when downloading repository index

See merge request !264
parents e7e97654 8d424f19
Pipeline #7999031 passed with stage
in 19 minutes and 1 second
......@@ -35,9 +35,7 @@ from binascii import hexlify, unhexlify
from datetime import datetime
from xml.dom.minidom import Document
import requests
from fdroidserver import metadata, signindex, common
from fdroidserver import metadata, signindex, common, net
from fdroidserver.common import FDroidPopen, FDroidPopenBytes
from fdroidserver.metadata import MetaDataException
......@@ -557,14 +555,16 @@ class VerificationException(Exception):
pass
def download_repo_index(url_str, verify_fingerprint=True):
def download_repo_index(url_str, etag=None, verify_fingerprint=True):
"""
Downloads the repository index from the given :param url_str
and verifies the repository's fingerprint if :param verify_fingerprint is not False.
:raises: VerificationException() if the repository could not be verified
:return: The index in JSON format.
:return: A tuple consisting of:
- The index in JSON format or None if the index did not change
- The new eTag as returned by the HTTP request
"""
url = urllib.parse.urlsplit(url_str)
......@@ -576,11 +576,14 @@ def download_repo_index(url_str, verify_fingerprint=True):
fingerprint = query['fingerprint'][0]
url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
r = requests.get(url.geturl())
download, new_etag = net.http_get(url.geturl(), etag)
if download is None:
return None, new_etag
with tempfile.NamedTemporaryFile() as fp:
# write and open JAR file
fp.write(r.content)
fp.write(download)
jar = zipfile.ZipFile(fp)
# verify that the JAR signature is valid
......@@ -601,7 +604,7 @@ def download_repo_index(url_str, verify_fingerprint=True):
# turn the apps into App objects
index["apps"] = [metadata.App(app) for app in index["apps"]]
return index
return index, new_etag
def verify_jar_signature(file):
......
......@@ -34,3 +34,34 @@ def download_file(url, local_filename=None, dldir='tmp'):
f.write(chunk)
f.flush()
return local_filename
def http_get(url, etag=None):
"""
Downloads the content from the given URL by making a GET request.
If an ETag is given, it will do a HEAD request first, to see if the content changed.
:param url: The URL to download from.
:param etag: The last ETag to be used for the request (optional).
:return: A tuple consisting of:
- The raw content that was downloaded or None if it did not change
- The new eTag as returned by the HTTP request
"""
headers = {'User-Agent': 'F-Droid'}
# TODO disable TLS Session IDs and TLS Session Tickets
# (plain text cookie visible to anyone who can see the network traffic)
if etag:
r = requests.head(url, headers=headers)
r.raise_for_status()
if 'ETag' in r.headers and etag == r.headers['ETag']:
return None, etag
r = requests.get(url, headers=headers)
r.raise_for_status()
new_etag = None
if 'ETag' in r.headers:
new_etag = r.headers['ETag']
return r.content, new_etag
......@@ -6,6 +6,9 @@ import os
import sys
import unittest
import zipfile
from unittest.mock import patch
import requests
localmodule = os.path.realpath(
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
......@@ -18,6 +21,9 @@ import fdroidserver.index
import fdroidserver.signindex
GP_FINGERPRINT = 'B7C2EEFD8DAC7806AF67DFCD92EB18126BC08312A7F2D6F3862E46013C7A6135'
class IndexTest(unittest.TestCase):
def setUp(self):
......@@ -55,9 +61,7 @@ class IndexTest(unittest.TestCase):
'818E469465F96B704E27BE2FEE4C63AB' +
'9F83DDF30E7A34C7371A4728D83B0BC1')
if f == 'guardianproject.jar':
self.assertTrue(fingerprint ==
'B7C2EEFD8DAC7806AF67DFCD92EB1812' +
'6BC08312A7F2D6F3862E46013C7A6135')
self.assertTrue(fingerprint == GP_FINGERPRINT)
def test_get_public_key_from_jar_fails(self):
basedir = os.path.dirname(__file__)
......@@ -72,10 +76,43 @@ class IndexTest(unittest.TestCase):
fdroidserver.index.download_repo_index("http://example.org")
def test_download_repo_index_no_jar(self):
with self.assertRaises(zipfile.BadZipFile):
with self.assertRaises(requests.exceptions.HTTPError):
fdroidserver.index.download_repo_index("http://example.org?fingerprint=nope")
# TODO test_download_repo_index with an actual repository
@patch('requests.head')
def test_download_repo_index_same_etag(self, head):
url = 'http://example.org?fingerprint=test'
etag = '"4de5-54d840ce95cb9"'
head.return_value.headers = {'ETag': etag}
index, new_etag = fdroidserver.index.download_repo_index(url, etag=etag)
self.assertIsNone(index)
self.assertEqual(etag, new_etag)
@patch('requests.get')
@patch('requests.head')
def test_download_repo_index_new_etag(self, head, get):
url = 'http://example.org?fingerprint=' + GP_FINGERPRINT
etag = '"4de5-54d840ce95cb9"'
# fake HTTP answers
head.return_value.headers = {'ETag': 'new_etag'}
get.return_value.headers = {'ETag': 'new_etag'}
get.return_value.status_code = 200
testfile = os.path.join(os.path.dirname(__file__), 'signindex', 'guardianproject-v1.jar')
with open(testfile, 'rb') as file:
get.return_value.content = file.read()
index, new_etag = fdroidserver.index.download_repo_index(url, etag=etag)
# assert that the index was retrieved properly
self.assertEqual('Guardian Project Official Releases', index['repo']['name'])
self.assertEqual(GP_FINGERPRINT, index['repo']['fingerprint'])
self.assertTrue(len(index['repo']['pubkey']) > 500)
self.assertEqual(10, len(index['apps']))
self.assertEqual(10, len(index['packages']))
self.assertEqual('new_etag', new_etag)
if __name__ == "__main__":
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment