[wip] drastically increased ratings of urls and organizations

parent 7557a56f
import logging
from failmap_admin.app.management.commands._private import TaskCommand
from ...rating import rerate_organizations
log = logging.getLogger(__name__)
class Command(TaskCommand):
"""Remove all organization and url ratings, then rebuild them from scratch."""
help = __doc__
task = rerate_organizations
......@@ -9,8 +9,9 @@ import logging
logger = logging.getLogger(__package__)
def points_and_calculation(scan, scan_type):
def points_and_calculation(scan):
# Can be probably more efficient by adding some methods to scan.
scan_type = getattr(scan, "type", "tls_qualys")
return calculation_methods[scan_type](scan)
This diff is collapsed.
import collections
import json
from datetime import datetime, timedelta
import pytz
......@@ -15,6 +14,7 @@ from django.shortcuts import render
from django.utils.translation import ugettext as _
from django.views.decorators.cache import cache_page
import simplejson as json
from failmap_admin.map.models import OrganizationRating, UrlRating
from failmap_admin.organizations.models import Organization, Promise, Url
from failmap_admin.scanners.models import EndpointGenericScan, TlsQualysScan
......@@ -1094,7 +1094,7 @@ def latest_scans(request, scan_type):
scans = list(EndpointGenericScan.objects.filter(type=scan_type).order_by('-rating_determined_on')[0:6])
for scan in scans:
points, calculation = points_and_calculation(scan, scan_type)
points, calculation = points_and_calculation(scan)
"url": scan.endpoint.url.url,
"service": "%s/%s (IPv%s)" % (scan.endpoint.protocol, scan.endpoint.port, scan.endpoint.ip_version),
......@@ -1149,7 +1149,7 @@ def latest_updates(organization_id):
for scan in scans:
scan_type = getattr(scan, "type", "tls_qualys") # todo: should always be a property of scan
points, calculation = points_and_calculation(scan, scan_type)
points, calculation = points_and_calculation(scan)
"organization": organization.name,
"organization_id": organization.pk,
......@@ -1225,20 +1225,22 @@ class UpdatesOnOrganizationFeed(Feed):
# @cache_page(ten_minutes), you can't cache this using the decorator.
Setting a parameter such as self.scan_type in the get_object will cause concurrency problems.
The manual is lacking how to get variables to the item_title and such functions: only to "items" it is somewhat
clear. This is probably because i don't know enough python. Why would this extra parameter work at the "items"
functions but not anywhere else? (signature issues).
class LatestScanFeed(Feed):
description = "Overview of the latest scans."
# magic
def get_object(self, request, *args, **kwargs):
# raunchy solution to get the scan_type to the item_description method
if kwargs['scan_type'] not in ["Strict-Transport-Security", "X-Content-Type-Options", "X-Frame-Options",
"X-XSS-Protection", "plain_https", "tls_qualys"]:
self.scan_type = "tls_qualys"
self.scan_type = kwargs['scan_type']
print("args: %s" % kwargs['scan_type'])
return kwargs['scan_type']
def title(self, scan_type):
......@@ -1255,6 +1257,7 @@ class LatestScanFeed(Feed):
# second parameter via magic
def items(self, scan_type):
if scan_type in ["Strict-Transport-Security", "X-Content-Type-Options", "X-Frame-Options", "X-XSS-Protection",
return EndpointGenericScan.objects.filter(type=scan_type).order_by('-last_scan_moment')[0:30]
......@@ -1262,7 +1265,7 @@ class LatestScanFeed(Feed):
return TlsQualysScan.objects.order_by('-last_scan_moment')[0:30]
def item_title(self, item):
points, calculation = points_and_calculation(item, self.scan_type)
points, calculation = points_and_calculation(item)
if not calculation:
return ""
......@@ -1275,7 +1278,7 @@ class LatestScanFeed(Feed):
return "%s %s - %s" % (badge, rating, item.endpoint.url.url)
def item_description(self, item):
points, calculation = points_and_calculation(item, self.scan_type)
points, calculation = points_and_calculation(item)
return _(calculation.get("explanation", ""))
def item_pubdate(self, item):
......@@ -2,8 +2,8 @@ import logging
from django.core.management.base import BaseCommand
from failmap_admin.map.rating import (add_organization_rating, create_timeline,
rerate_organizations, rerate_urls, show_timeline_console)
from failmap_admin.map.rating import (add_organization_rating, create_timeline, rerate_urls,
from failmap_admin.organizations.models import Organization, Url
from failmap_admin.scanners.models import Endpoint
from failmap_admin.scanners.scanner_security_headers import scan as scan_headers
......@@ -15,7 +15,9 @@ class Command(BaseCommand):
help = 'Development command'
def handle(self, *args, **options):
# develop_determineratings()
# test_can_connect_to_organization()
# as a task
# develop_determineratings()
......@@ -32,6 +34,11 @@ class Command(BaseCommand):
# Command.develop_celery_test_async_tasks()
def develop_rerate_urls_async():
from failmap_admin.map.rating import rerate_urls_async
def reset_onboard():
organization = Organization.objects.filter(name="Arnhem").get()
urls = Url.objects.all().filter(organization=organization)
......@@ -120,7 +127,12 @@ def develop_security_headers_scanner():
def develop_determineratings():
# DetermineRatings.default_ratings()
# return
from datetime import datetime
import pytz
from failmap_admin.map.rating import relevant_endpoints_at_timepoint
u = Url.objects.all().filter(url='www.arnhem.nl').get()
relevant_endpoints_at_timepoint(url=u, when=datetime(2016, 12, 31, 0, 0, tzinfo=pytz.utc))
# DetermineRatings.significant_times(organization=organization)
# urls = Url.objects.all().filter(organization=organization)
# for url in urls:
......@@ -128,9 +140,9 @@ def develop_determineratings():
# pyflakes when = datetime(2016, 12, 31, 0, 0, tzinfo=pytz.utc)
# when = datetime.now(pytz.utc)
organization = Organization.objects.filter(name="Zederik").get()
# organization = Organization.objects.filter(name="Zederik").get()
# rerate_urls(Url.objects.all().filter(organization=organization))
# rerate_organizations(organizations=[organization])
# ratings are always different since we now also save last scan date.
# only creates things for near midnight. Should check if today, and then save for now.
# add_organization_rating(organization, create_history=True)
......@@ -51,4 +51,9 @@ raven
# allow recursive comparison of dictionaries
\ No newline at end of file
# loading json is faster in simplejson
# https://stackoverflow.com/questions/712791/what-are-the-differences-between-json-and-simplejson-python-modules
# needed for mapping reasons.
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment