Use Cache for storing requests

Resolves #126
parent e6c6d5da
......@@ -2,7 +2,6 @@
This package contains an abstraction for a git repository.
"""
from base64 import b64encode
from collections import defaultdict
from datetime import timedelta
from enum import Enum
from json.decoder import JSONDecodeError
......@@ -16,9 +15,10 @@ from requests.auth import AuthBase
from requests.auth import HTTPBasicAuth
import requests
from IGitt.Utils import Cache
HEADERS = {'User-Agent': 'IGitt'}
_RESPONSES = defaultdict()
class IGittObject:
......@@ -138,6 +138,17 @@ def is_client_error_or_unmodified(exception):
return (400 <= exception.args[1] < 500) or (exception.args[1] == 304)
def parse_response(response: requests.Response):
"""
Parses the response object into JSON and link headers and returns them.
"""
try:
return response.json(), response.links
except JSONDecodeError:
# if the response body is pure text, for e.g. a git diff.
return response.text, response.links
@on_exception(expo, ConnectionError, max_tries=8)
@on_exception(expo,
RuntimeError,
......@@ -148,18 +159,36 @@ def get_response(method: Callable,
auth: AuthBase,
json: Optional[Dict]=frozenset()):
"""
Sends a request and checks the response for errors, and retries unless it's
a HTTP client error.
Sends a request and returns the response. Also checks the response for
errors, and keeps retrying unless it's a HTTP Client Error.
"""
headers = ({'If-None-Match': _RESPONSES[url].headers.get('ETag')}
if url in _RESPONSES else {})
response = method(url, auth=auth, json=dict(json or {}), headers=headers)
if response.status_code == 304 and url in _RESPONSES:
return _RESPONSES[url]
elif response.status_code >= 300:
raise RuntimeError(response.text, response.status_code)
_RESPONSES[url] = response
return response
if method.__name__.lower() != 'get':
resp = method(url, auth=auth, json=dict(json or {}))
if resp.status_code >= 300 and resp.status_code != 304:
raise RuntimeError(resp.text, resp.status_code)
return parse_response(resp)
# cache only GET requests
cached_resp, headers = Cache.get(url), {}
if cached_resp:
if cached_resp['fromWebhook']:
headers['If-Modified-Since'] = cached_resp.get('lastFetched')
else:
headers['If-None-Match'] = cached_resp.get('entityTag')
resp = method(url, auth=auth, json=dict(json or {}), headers=headers)
if resp.status_code == 304 and cached_resp:
return cached_resp.get('data'), cached_resp.get('links')
elif resp.status_code >= 300:
raise RuntimeError(resp.text, resp.status_code)
data, links = parse_response(resp)
# update entry in cache
Cache.set(url, {
'entityTag': resp.headers.get('ETag'),
'data': data,
'links': links
})
return data, links
def _fetch(url: str, req_type: str, token: Token, data: Optional[dict]=None,
......@@ -196,34 +225,32 @@ def _fetch(url: str, req_type: str, token: Token, data: Optional[dict]=None,
'patch': session.patch,
'delete': session.delete
}
method = req_methods[req_type]
resp = get_response(method, url, token.auth, json=data)
method = req_methods[req_type.lower()]
resp, links = get_response(method, url, token.auth, json=data)
# DELETE request returns no response
if not len(resp.text):
return []
# if the response body is pure text
if isinstance(resp, str):
# if the response body is empty, for e.g. in case of a DELETE request
if resp == '':
return []
return resp
while True:
try:
if isinstance(resp.json(), dict) and 'items' not in resp.json():
# if response is a single object
return resp.json()
if isinstance(resp, dict):
if 'items' in resp:
# if response is a dict with `items` key, i.e. a list of items
data_container.extend(resp['items'])
else:
if isinstance(resp.json(), list):
# if response is a list of objects
data_container.extend(resp.json())
elif 'items' in resp.json():
# if response is a dict with `items` key
data_container.extend(resp.json()['items'])
if not resp.links.get('next', False):
return data_container
resp = get_response(method,
resp.links.get('next')['url'],
token.auth,
json=data)
except JSONDecodeError:
# if the request has a text response, for e.g. a git diff.
return resp.text
# if response is a single item
return resp
elif isinstance(resp, list):
# if response is a list of items
data_container.extend(resp)
if not links.get('next', False):
return data_container
resp, links = get_response(
method, links.get('next')['url'], token.auth, json=data)
def get(token: Token, url: str, params: Optional[dict]=None,
headers: Optional[dict]=None):
......
......@@ -50,7 +50,8 @@ class Cache:
>>> from IGitt.Utils import Cache
>>> Cache.use(read_from, write_to)
For further details follow the specific method documentation below.
If not provided, IGitt uses a default in-memory cache. For further details
follow the specific method documentation below.
"""
__mem_store = LimitedSizeDict(size_limit=10 ** 6) # a million entries
_get = __mem_store.__getitem__
......@@ -225,6 +226,12 @@ class PossiblyIncompleteDict:
self._data = self._del_nul(self._refresh())
self.may_need_refresh = False
def get(self):
"""
Returns the stored data.
"""
return self._data
class CachedDataMixin:
"""
......
......@@ -101,4 +101,92 @@ interactions:
X-Runtime-rack: ['0.063448']
X-XSS-Protection: [1; mode=block]
status: {code: 304, message: Not Modified}
- request:
body: '{}'
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
Content-Length: ['2']
Content-Type: [application/json]
User-Agent: [IGitt]
method: GET
uri: https://api.github.com/rate_limit?per_page=100
response:
body:
string: !!binary |
H4sIAAAAAAAAA6tWKkotzi8tSk4tVrKqVkrOL0oF0TmZuZklSlamBgYGOkAVuYmZeZl56QiB4lSg
rKGpkamJsYm5pUGtjlJxamJRcgaSXmNUnRAukj5DI1OQvvSixIKMwhxyLAXqLkosIcu9tQBD7taa
+AAAAA==
headers:
Access-Control-Allow-Origin: ['*']
Access-Control-Expose-Headers: ['ETag, Link, Retry-After, X-GitHub-OTP, X-RateLimit-Limit,
X-RateLimit-Remaining, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes,
X-Poll-Interval']
Cache-Control: [no-cache]
Content-Encoding: [gzip]
Content-Security-Policy: [default-src 'none']
Content-Type: [application/json; charset=utf-8]
Date: ['Fri, 04 May 2018 10:53:10 GMT']
Referrer-Policy: ['origin-when-cross-origin, strict-origin-when-cross-origin']
Server: [GitHub.com]
Status: [200 OK]
Strict-Transport-Security: [max-age=31536000; includeSubdomains; preload]
X-Accepted-OAuth-Scopes: ['']
X-Content-Type-Options: [nosniff]
X-Frame-Options: [deny]
X-GitHub-Media-Type: [github.v3; format=json]
X-GitHub-Request-Id: ['35B0:1096:178C1BE:2F854AC:5AEC3B96']
X-OAuth-Scopes: ['admin:gpg_key, admin:org, admin:org_hook, admin:public_key,
admin:repo_hook, gist, notifications, repo, user']
X-RateLimit-Limit: ['5000']
X-RateLimit-Remaining: ['5000']
X-RateLimit-Reset: ['1525434790']
X-Runtime-rack: ['0.026001']
X-XSS-Protection: [1; mode=block]
status: {code: 200, message: OK}
- request:
body: '{}'
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
Content-Length: ['2']
Content-Type: [application/json]
User-Agent: [IGitt]
method: GET
uri: https://api.github.com/rate_limit?per_page=100
response:
body:
string: !!binary |
H4sIAAAAAAAAA6tWKkotzi8tSk4tVrKqVkrOL0oF0TmZuZklSlamBgYGOkAVuYmZeZl56QiB4lSg
rKGpkamJsamBoXmtjlJxamJRcgaSXmNUnRAukj5DE3OQvvSixIKMwhxyLAXqLkosIcu9tQDTtY8v
+AAAAA==
headers:
Access-Control-Allow-Origin: ['*']
Access-Control-Expose-Headers: ['ETag, Link, Retry-After, X-GitHub-OTP, X-RateLimit-Limit,
X-RateLimit-Remaining, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes,
X-Poll-Interval']
Cache-Control: [no-cache]
Content-Encoding: [gzip]
Content-Security-Policy: [default-src 'none']
Content-Type: [application/json; charset=utf-8]
Date: ['Fri, 04 May 2018 10:56:57 GMT']
Referrer-Policy: ['origin-when-cross-origin, strict-origin-when-cross-origin']
Server: [GitHub.com]
Status: [200 OK]
Strict-Transport-Security: [max-age=31536000; includeSubdomains; preload]
X-Accepted-OAuth-Scopes: ['']
X-Content-Type-Options: [nosniff]
X-Frame-Options: [deny]
X-GitHub-Media-Type: [github.v3; format=json]
X-GitHub-Request-Id: ['2F79:1094:B3A94E:1B519C2:5AEC3C78']
X-OAuth-Scopes: ['admin:gpg_key, admin:org, admin:org_hook, admin:public_key,
admin:repo_hook, gist, notifications, repo, user']
X-RateLimit-Limit: ['5000']
X-RateLimit-Remaining: ['5000']
X-RateLimit-Reset: ['1525435017']
X-Runtime-rack: ['0.020082']
X-XSS-Protection: [1; mode=block]
status: {code: 200, message: OK}
version: 1
import os
from IGitt.GitHub import BASE_URL as GITHUB_BASE_URL
from IGitt.GitHub import GitHubMixin
from IGitt.GitHub import GitHubToken
from IGitt.GitHub import GitHubJsonWebToken
from IGitt.GitHub import GitHubInstallationToken
from IGitt.GitHub.GitHubRepository import GitHubRepository
from IGitt.GitLab import BASE_URL as GITLAB_BASE_URL
from IGitt.GitLab import GitLabOAuthToken
from IGitt.Interfaces import _RESPONSES
from IGitt.Interfaces import _fetch
from IGitt.Interfaces import get
from IGitt.Interfaces import BasicAuthorizationToken
from IGitt.Utils import Cache
from tests import IGittTestCase
......@@ -67,17 +68,22 @@ class TestInterfacesInit(IGittTestCase):
@staticmethod
def test_github_conditional_request():
store = {}
Cache.use(store.__getitem__, store.__setitem__)
token = GitHubToken(os.environ.get('GITHUB_TEST_TOKEN', ''))
repo = GitHubRepository(token, os.environ.get('GITHUB_TEST_REPO',
'gitmate-test-user/test'))
repo.refresh()
prev_data = repo.data._data
prev_count = _RESPONSES[repo.url].headers.get('X-RateLimit-Remaining')
prev_data = repo.data.get()
prev_count = get(
token, GitHubMixin.absolute_url('/rate_limit'))['rate']['limit']
repo.refresh()
new_data = repo.data._data
new_count = _RESPONSES[repo.url].headers.get('X-RateLimit-Remaining')
new_data = repo.data.get()
new_count = get(
token, GitHubMixin.absolute_url('/rate_limit'))['rate']['limit']
# check that no reduction in rate limit is observed
assert prev_count == new_count
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment