[wip] drastically increased ratings of urls and organizations

parent 7557a56f
import logging
from failmap_admin.app.management.commands._private import TaskCommand
from ...rating import rerate_organizations
log = logging.getLogger(__name__)
class Command(TaskCommand):
"""Remove all organization and url ratings, then rebuild them from scratch."""
help = __doc__
task = rerate_organizations
...@@ -9,8 +9,9 @@ import logging ...@@ -9,8 +9,9 @@ import logging
logger = logging.getLogger(__package__) logger = logging.getLogger(__package__)
def points_and_calculation(scan, scan_type): def points_and_calculation(scan):
# Can be probably more efficient by adding some methods to scan. # Can be probably more efficient by adding some methods to scan.
scan_type = getattr(scan, "type", "tls_qualys")
return calculation_methods[scan_type](scan) return calculation_methods[scan_type](scan)
......
This diff is collapsed.
import collections import collections
import json
from datetime import datetime, timedelta from datetime import datetime, timedelta
import pytz import pytz
...@@ -15,6 +14,7 @@ from django.shortcuts import render ...@@ -15,6 +14,7 @@ from django.shortcuts import render
from django.utils.translation import ugettext as _ from django.utils.translation import ugettext as _
from django.views.decorators.cache import cache_page from django.views.decorators.cache import cache_page
import simplejson as json
from failmap_admin.map.models import OrganizationRating, UrlRating from failmap_admin.map.models import OrganizationRating, UrlRating
from failmap_admin.organizations.models import Organization, Promise, Url from failmap_admin.organizations.models import Organization, Promise, Url
from failmap_admin.scanners.models import EndpointGenericScan, TlsQualysScan from failmap_admin.scanners.models import EndpointGenericScan, TlsQualysScan
...@@ -1094,7 +1094,7 @@ def latest_scans(request, scan_type): ...@@ -1094,7 +1094,7 @@ def latest_scans(request, scan_type):
scans = list(EndpointGenericScan.objects.filter(type=scan_type).order_by('-rating_determined_on')[0:6]) scans = list(EndpointGenericScan.objects.filter(type=scan_type).order_by('-rating_determined_on')[0:6])
for scan in scans: for scan in scans:
points, calculation = points_and_calculation(scan, scan_type) points, calculation = points_and_calculation(scan)
dataset["scans"].append({ dataset["scans"].append({
"url": scan.endpoint.url.url, "url": scan.endpoint.url.url,
"service": "%s/%s (IPv%s)" % (scan.endpoint.protocol, scan.endpoint.port, scan.endpoint.ip_version), "service": "%s/%s (IPv%s)" % (scan.endpoint.protocol, scan.endpoint.port, scan.endpoint.ip_version),
...@@ -1149,7 +1149,7 @@ def latest_updates(organization_id): ...@@ -1149,7 +1149,7 @@ def latest_updates(organization_id):
for scan in scans: for scan in scans:
scan_type = getattr(scan, "type", "tls_qualys") # todo: should always be a property of scan scan_type = getattr(scan, "type", "tls_qualys") # todo: should always be a property of scan
points, calculation = points_and_calculation(scan, scan_type) points, calculation = points_and_calculation(scan)
dataset["scans"].append({ dataset["scans"].append({
"organization": organization.name, "organization": organization.name,
"organization_id": organization.pk, "organization_id": organization.pk,
...@@ -1225,20 +1225,22 @@ class UpdatesOnOrganizationFeed(Feed): ...@@ -1225,20 +1225,22 @@ class UpdatesOnOrganizationFeed(Feed):
# @cache_page(ten_minutes), you can't cache this using the decorator. # @cache_page(ten_minutes), you can't cache this using the decorator.
"""
Setting a parameter such as self.scan_type in the get_object will cause concurrency problems.
The manual is lacking how to get variables to the item_title and such functions: only to "items" it is somewhat
clear. This is probably because i don't know enough python. Why would this extra parameter work at the "items"
functions but not anywhere else? (signature issues).
"""
class LatestScanFeed(Feed): class LatestScanFeed(Feed):
description = "Overview of the latest scans." description = "Overview of the latest scans."
# magic # magic
def get_object(self, request, *args, **kwargs): def get_object(self, request, *args, **kwargs):
print("args: %s" % kwargs['scan_type'])
# raunchy solution to get the scan_type to the item_description method
if kwargs['scan_type'] not in ["Strict-Transport-Security", "X-Content-Type-Options", "X-Frame-Options",
"X-XSS-Protection", "plain_https", "tls_qualys"]:
self.scan_type = "tls_qualys"
else:
self.scan_type = kwargs['scan_type']
return kwargs['scan_type'] return kwargs['scan_type']
def title(self, scan_type): def title(self, scan_type):
...@@ -1255,6 +1257,7 @@ class LatestScanFeed(Feed): ...@@ -1255,6 +1257,7 @@ class LatestScanFeed(Feed):
# second parameter via magic # second parameter via magic
def items(self, scan_type): def items(self, scan_type):
print(scan_type)
if scan_type in ["Strict-Transport-Security", "X-Content-Type-Options", "X-Frame-Options", "X-XSS-Protection", if scan_type in ["Strict-Transport-Security", "X-Content-Type-Options", "X-Frame-Options", "X-XSS-Protection",
"plain_https"]: "plain_https"]:
return EndpointGenericScan.objects.filter(type=scan_type).order_by('-last_scan_moment')[0:30] return EndpointGenericScan.objects.filter(type=scan_type).order_by('-last_scan_moment')[0:30]
...@@ -1262,7 +1265,7 @@ class LatestScanFeed(Feed): ...@@ -1262,7 +1265,7 @@ class LatestScanFeed(Feed):
return TlsQualysScan.objects.order_by('-last_scan_moment')[0:30] return TlsQualysScan.objects.order_by('-last_scan_moment')[0:30]
def item_title(self, item): def item_title(self, item):
points, calculation = points_and_calculation(item, self.scan_type) points, calculation = points_and_calculation(item)
if not calculation: if not calculation:
return "" return ""
...@@ -1275,7 +1278,7 @@ class LatestScanFeed(Feed): ...@@ -1275,7 +1278,7 @@ class LatestScanFeed(Feed):
return "%s %s - %s" % (badge, rating, item.endpoint.url.url) return "%s %s - %s" % (badge, rating, item.endpoint.url.url)
def item_description(self, item): def item_description(self, item):
points, calculation = points_and_calculation(item, self.scan_type) points, calculation = points_and_calculation(item)
return _(calculation.get("explanation", "")) return _(calculation.get("explanation", ""))
def item_pubdate(self, item): def item_pubdate(self, item):
......
...@@ -2,8 +2,8 @@ import logging ...@@ -2,8 +2,8 @@ import logging
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from failmap_admin.map.rating import (add_organization_rating, create_timeline, from failmap_admin.map.rating import (add_organization_rating, create_timeline, rerate_urls,
rerate_organizations, rerate_urls, show_timeline_console) show_timeline_console)
from failmap_admin.organizations.models import Organization, Url from failmap_admin.organizations.models import Organization, Url
from failmap_admin.scanners.models import Endpoint from failmap_admin.scanners.models import Endpoint
from failmap_admin.scanners.scanner_security_headers import scan as scan_headers from failmap_admin.scanners.scanner_security_headers import scan as scan_headers
...@@ -15,7 +15,9 @@ class Command(BaseCommand): ...@@ -15,7 +15,9 @@ class Command(BaseCommand):
help = 'Development command' help = 'Development command'
def handle(self, *args, **options): def handle(self, *args, **options):
test_can_connect_to_organization() develop_rerate_urls_async()
# develop_determineratings()
# test_can_connect_to_organization()
# as a task # as a task
# develop_determineratings() # develop_determineratings()
...@@ -32,6 +34,11 @@ class Command(BaseCommand): ...@@ -32,6 +34,11 @@ class Command(BaseCommand):
# Command.develop_celery_test_async_tasks() # Command.develop_celery_test_async_tasks()
def develop_rerate_urls_async():
from failmap_admin.map.rating import rerate_urls_async
rerate_urls_async()
def reset_onboard(): def reset_onboard():
organization = Organization.objects.filter(name="Arnhem").get() organization = Organization.objects.filter(name="Arnhem").get()
urls = Url.objects.all().filter(organization=organization) urls = Url.objects.all().filter(organization=organization)
...@@ -120,7 +127,12 @@ def develop_security_headers_scanner(): ...@@ -120,7 +127,12 @@ def develop_security_headers_scanner():
def develop_determineratings(): def develop_determineratings():
# DetermineRatings.default_ratings() # DetermineRatings.default_ratings()
# return # return
from datetime import datetime
import pytz
from failmap_admin.map.rating import relevant_endpoints_at_timepoint
u = Url.objects.all().filter(url='www.arnhem.nl').get()
relevant_endpoints_at_timepoint(url=u, when=datetime(2016, 12, 31, 0, 0, tzinfo=pytz.utc))
# DetermineRatings.significant_times(organization=organization) # DetermineRatings.significant_times(organization=organization)
# urls = Url.objects.all().filter(organization=organization) # urls = Url.objects.all().filter(organization=organization)
# for url in urls: # for url in urls:
...@@ -128,9 +140,9 @@ def develop_determineratings(): ...@@ -128,9 +140,9 @@ def develop_determineratings():
# pyflakes when = datetime(2016, 12, 31, 0, 0, tzinfo=pytz.utc) # pyflakes when = datetime(2016, 12, 31, 0, 0, tzinfo=pytz.utc)
# when = datetime.now(pytz.utc) # when = datetime.now(pytz.utc)
organization = Organization.objects.filter(name="Zederik").get() # organization = Organization.objects.filter(name="Zederik").get()
# rerate_urls(Url.objects.all().filter(organization=organization)) # rerate_urls(Url.objects.all().filter(organization=organization))
rerate_organizations(organizations=[organization]) # rerate_organizations(organizations=[organization])
# ratings are always different since we now also save last scan date. # ratings are always different since we now also save last scan date.
# only creates things for near midnight. Should check if today, and then save for now. # only creates things for near midnight. Should check if today, and then save for now.
# add_organization_rating(organization, create_history=True) # add_organization_rating(organization, create_history=True)
......
...@@ -51,4 +51,9 @@ raven ...@@ -51,4 +51,9 @@ raven
# allow recursive comparison of dictionaries # allow recursive comparison of dictionaries
deepdiff deepdiff
influxdb influxdb
urllib3 urllib3
\ No newline at end of file
# loading json is faster in simplejson
# https://stackoverflow.com/questions/712791/what-are-the-differences-between-json-and-simplejson-python-modules
# needed for mapping reasons.
simplejson
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment