Commit 9f1aff16 authored by Elger Jonker's avatar Elger Jonker

easier map data cache creation, + task to schedule every night


Former-commit-id: 440276f7
parent a8f7f15e
......@@ -9,7 +9,7 @@ from deepdiff import DeepDiff
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Q
from failmap.map.views import map_data
from failmap.map.views import get_map_data
from failmap.organizations.models import Organization, OrganizationType, Url
from failmap.scanners.models import Endpoint, EndpointGenericScan, TlsQualysScan, UrlGenericScan
from failmap.scanners.scanner.scanner import q_configurations_to_display
......@@ -1605,20 +1605,18 @@ def calculate_vulnerability_graphs(days: int = 366):
@app.task(queue='storage')
def calculate_map_data(days: int = 366):
log.info("calculate_map_data")
def calculate_map_data_today():
calculate_map_data.si(1).apply_async()
# fake request
from django.test.client import RequestFactory
rf = RequestFactory()
from django.conf import settings
get_request = rf.get('/', HTTP_HOST=settings.ALLOWED_HOSTS[0])
@app.task(queue='storage')
def calculate_map_data(days: int = 366):
log.info("calculate_map_data")
# all vulnerabilities * 14 days
# all vulnerabilities
filters = ["security_headers_strict_transport_security", "security_headers_x_content_type_options", "ftp", "DNSSEC",
"security_headers_x_frame_options", "security_headers_x_xss_protection", "tls_qualys", "plain_https", ""
]
"security_headers_x_frame_options", "security_headers_x_xss_protection", "tls_qualys", "plain_https",
'']
map_configurations = Configuration.objects.all().filter(
is_displayed=True).order_by('display_order').values('country', 'organization_type__name', 'organization_type')
......@@ -1635,10 +1633,23 @@ def calculate_map_data(days: int = 366):
filters=[filter]
).delete()
log.debug("Country: %s, Organization_type: %s, day: %s, filter: %s" % (
log.debug("Country: %s, Organization_type: %s, day: %s, date: %s, filter: %s" % (
map_configuration['country'], map_configuration['organization_type__name'],
days_back, filter
days_back, when, filter
))
map_data(get_request,
map_configuration['country'], map_configuration['organization_type__name'],
days_back, filter)
data = get_map_data(map_configuration['country'], map_configuration['organization_type__name'],
days_back, filter)
from django.db import OperationalError
try:
cached = MapDataCache()
cached.organization_type = OrganizationType(pk=map_configuration['organization_type'])
cached.country = map_configuration['country']
cached.filters = [filter]
cached.when = when
cached.dataset = data
cached.save()
except OperationalError as a:
# The public user does not have permission to run insert statements....
log.exception(a)
......@@ -1291,6 +1291,14 @@ def map_default(request, days_back: int = 0, displayed_issue: str = None):
def map_data(request, country: str = "NL", organization_type: str = "municipality", days_back: int = 0,
displayed_issue: str = None):
data = get_map_data(country, organization_type, days_back, displayed_issue)
return JsonResponse(data, encoder=JSEncoder)
def get_map_data(country: str = "NL", organization_type: str = "municipality", days_back: int = 0,
displayed_issue: str = None):
if not days_back:
when = datetime.now(pytz.utc)
else:
......@@ -1319,13 +1327,20 @@ def map_data(request, country: str = "NL", organization_type: str = "municipalit
desired_url_scans = possible_url_scans
desired_endpoint_scans = possible_endpoint_scans
# look if we have data in the cache, which will save some calculations and a slower query
cached = MapDataCache.objects.all().filter(country=country,
organization_type=get_organization_type(organization_type), when=when,
filters=desired_url_scans + desired_endpoint_scans).first()
# look if we have data in the cache, which will save some calculations and a slower query
cached = MapDataCache.objects.all().filter(country=country,
organization_type=get_organization_type(organization_type),
when=when,
filters=['']).first()
else:
# look if we have data in the cache, which will save some calculations and a slower query
cached = MapDataCache.objects.all().filter(country=country,
organization_type=get_organization_type(organization_type),
when=when,
filters=desired_url_scans + desired_endpoint_scans).first()
if cached:
return JsonResponse(cached.dataset, encoder=JSEncoder)
return cached.dataset
"""
Returns a json structure containing all current map data.
......@@ -1520,7 +1535,7 @@ def map_data(request, country: str = "NL", organization_type: str = "municipalit
# except OperationalError:
# # The public user does not have permission to run insert statements....
return JsonResponse(data, encoder=JSEncoder)
return data
def empty_response():
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment