Commit 3d19d098 authored by Elger Jonker's avatar Elger Jonker

vulnerability stats with endpoints + urls, days param in calculate commands


Former-commit-id: 755d0f79
parent 9f1aff16
......@@ -241,7 +241,7 @@ class ConfigurationAdmin(SortableAdminMixin, ImportExportModelAdmin, admin.Model
@admin.register(VulnerabilityStatistic)
class VulnerabilityStatisticAdmin(ImportExportModelAdmin, admin.ModelAdmin):
list_display = ('country', 'organization_type', 'scan_type', 'when', 'high', 'medium', 'low')
list_display = ('country', 'organization_type', 'scan_type', 'when', 'high', 'medium', 'low', 'urls', 'endpoints')
list_filter = ('country', 'organization_type', 'scan_type', 'when', 'high', 'medium', 'low')
search_fields = (['country', 'organization_type', 'scan_type'])
......
import logging
from argparse import ArgumentTypeError
from django.core.management.base import BaseCommand
......@@ -9,7 +10,26 @@ log = logging.getLogger(__package__)
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument("--days",
type=check_positive,
help="Number of days to go back in time.",
required=False)
def handle(self, *args, **options):
""" Short hand for the first time running this """
calculate_map_data()
if options['days']:
days = options['days']
else:
days = 366
calculate_map_data(days)
def check_positive(value):
ivalue = int(value)
if ivalue <= 0:
raise ArgumentTypeError("%s is an invalid positive int value" % value)
return ivalue
import logging
from argparse import ArgumentTypeError
from django.core.management.base import BaseCommand
from failmap.map.rating import calculate_vulnerability_graphs
from failmap.map.rating import calculate_vulnerability_statistics
log = logging.getLogger(__package__)
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument("--days",
type=check_positive,
help="Number of days to go back in time.",
required=False)
def handle(self, *args, **options):
""" Short hand for the first time running this """
calculate_vulnerability_graphs()
if options['days']:
days = options['days']
else:
days = 366
calculate_vulnerability_statistics(days)
def check_positive(value):
ivalue = int(value)
if ivalue <= 0:
raise ArgumentTypeError("%s is an invalid positive int value" % value)
return ivalue
# Generated by Django 2.0.8 on 2018-10-29 09:46
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('map', '0028_mapdatacache_cached_on'),
]
operations = [
migrations.AddField(
model_name='vulnerabilitystatistic',
name='endpoints',
field=models.PositiveIntegerField(
default=0, help_text='Makes only sense on the total number of vulnerabilities'),
),
migrations.AddField(
model_name='vulnerabilitystatistic',
name='urls',
field=models.PositiveIntegerField(
default=0, help_text='Makes only sense on the total number of vulnerabilities'),
),
]
......@@ -310,6 +310,10 @@ class VulnerabilityStatistic(models.Model):
high = models.PositiveIntegerField(default=0, blank=False, null=False)
medium = models.PositiveIntegerField(default=0, blank=False, null=False)
low = models.PositiveIntegerField(default=0, blank=False, null=False)
urls = models.PositiveIntegerField(default=0, blank=False, null=False,
help_text="Makes only sense on the total number of vulnerabilities")
endpoints = models.PositiveIntegerField(default=0, blank=False, null=False,
help_text="Makes only sense on the total number of vulnerabilities")
class MapDataCache(models.Model):
......
......@@ -86,7 +86,7 @@ def compose_task(
# finally, rebuild the graphs (which can mis-matchi a bit if the last reports aren't in yet. Will have to do for now
# mainly as we're trying to get away from canvas and it's buggyness.
tasks.append(calculate_vulnerability_graphs.si(days))
tasks.append(calculate_vulnerability_statistics.si(days))
# also try to speed up the map view
tasks.append(calculate_map_data.si(days))
......@@ -1486,13 +1486,15 @@ def default_organization_rating(organizations: List[Organization]):
@app.task(queue='storage')
def calculate_vulnerability_graphs(days: int = 366):
def calculate_vulnerability_statistics(days: int = 366):
log.info("Calculation vulnerability graphs")
# for everything that is displayed on the site:
map_configurations = Configuration.objects.all().filter(
is_displayed=True).order_by('display_order').values('country', 'organization_type')
# from django.db import connection
for map_configuration in map_configurations:
scan_types = set() # set instead of list to prevent checking if something is in there already.
scan_types.add('total') # the total would be separated per char if directly passed into set()
......@@ -1534,7 +1536,8 @@ def calculate_vulnerability_graphs(days: int = 366):
# This query will deliver double ratings for urls that are doubly listed, which is dubious.
# this happens because multiple organizations can have the same URL.
# It's fair that there are more issues if more organizations share the same url?
sql = """SELECT map_urlrating.id as id, map_urlrating2.calculation as calculation FROM
sql = """SELECT map_urlrating.id as id, map_urlrating.total_endpoints,
map_urlrating2.calculation as calculation FROM
map_urlrating
INNER JOIN
(SELECT MAX(id) as id2 FROM map_urlrating or2
......@@ -1546,16 +1549,18 @@ def calculate_vulnerability_graphs(days: int = 366):
INNER JOIN map_urlrating as map_urlrating2 ON map_urlrating2.id = map_urlrating.id
WHERE organization.type_id = '%(OrganizationTypeId)s'
AND organization.country = '%(country)s'
AND map_urlrating.total_endpoints > 0
""" % {"when": when, "OrganizationTypeId": organization_type_id, "country": country}
urlratings = UrlRating.objects.raw(sql)
number_of_endpoints = 0
# print(connection.queries)
# group by vulnerability type
for urlrating in urlratings:
# rare occasions there are no endpoints.
if "endpoints" not in urlrating.calculation:
continue
number_of_endpoints += urlrating.total_endpoints
# url reports
for rating in urlrating.calculation['ratings']:
......@@ -1590,6 +1595,8 @@ def calculate_vulnerability_graphs(days: int = 366):
measurement['total']['medium'] += rating['medium']
measurement['total']['low'] += rating['low']
number_of_urls = len(list(urlratings))
# store these results per scan type, and only retrieve this per scan type...
for scan_type in scan_types:
if scan_type in measurement:
......@@ -1601,6 +1608,8 @@ def calculate_vulnerability_graphs(days: int = 366):
vs.high = measurement[scan_type]['high']
vs.medium = measurement[scan_type]['medium']
vs.low = measurement[scan_type]['low']
vs.urls = number_of_urls
vs.endpoints = number_of_endpoints
vs.save()
......
......@@ -251,7 +251,7 @@ const report_mixin = {
}],
yAxes: [{
display: true,
stacked: true,
stacked: false,
scaleLabel: {
display: false,
labelString: 'Value'
......@@ -621,10 +621,86 @@ function views() {
// data.total
// security_headers_strict_transport_security
var urls = Array();
var endpoints = Array();
var labels = Array();
fetch('/data/vulnstats/' + this.country + '/' + this.category + '/0')
.then(response => response.json()).then(data => {
this.vulnerability_graph('timeline_all_vulnerabilities', data.total, 'hml');
for(let i=0; i<data.total.length; i++){
labels.push(data.total[i].date);
urls.push(data.total[i].urls);
endpoints.push(data.total[i].endpoints);
}
// and a single endpoint/url graph:
let context = document.getElementById("timeline_available_urls_and_endpoints").getContext('2d');
let myChart2 = new Chart(context, {
type: 'line',
data: {
labels: labels,
datasets: [{
label: '# Internet Adresses',
data: urls,
backgroundColor: 'rgba(0, 0, 0, 0.2)',
borderColor: 'rgba(0,0,0,1)',
borderWidth: 1,
lineTension: 0
},
{
label: '# Services',
data: endpoints,
backgroundColor: 'rgba(0, 40, 255, 0.2)',
borderColor: 'rgba(0,40,255,1)',
borderWidth: 1,
lineTension: 0
},
]
},
options: {
responsive: true,
maintainAspectRatio: false,
title: {
display: true,
text: 'Internet connectivity overview'
},
tooltips: {
mode: 'index',
intersect: false,
},
hover: {
mode: 'nearest',
intersect: true
},
scales: {
xAxes: [{
display: true,
type: 'time',
distribution: 'linear',
time: {
unit: 'month'
},
scaleLabel: {
display: false,
labelString: 'Month'
}
}],
yAxes: [{
display: true,
stacked: false,
scaleLabel: {
display: false,
labelString: 'Value'
}
}]
}
}
});
this.vulnerability_graph('timeline_tls_qualys_vulnerabilities', data.tls_qualys, 'hl');
this.vulnerability_graph('timeline_missing_https_encryption_vulnerabilities', data.plain_https, 'hm');
this.vulnerability_graph('timeline_hsts_vulnerabilities', data.security_headers_strict_transport_security, 'm');
......
......@@ -915,6 +915,9 @@
<div class="chart-container" style="position: relative; height:555px; width:100%">
<canvas id="timeline_all_vulnerabilities"></canvas>
</div>
<div class="chart-container" style="position: relative; height:300px; width:100%">
<canvas id="timeline_available_urls_and_endpoints"></canvas>
</div>
</div>
</div>
......
......@@ -984,7 +984,8 @@ def vulnerability_graphs(request, country: str = "NL", organization_type="munici
stats[statistic.scan_type] = []
stats[statistic.scan_type].append({'high': statistic.high, 'medium': statistic.medium,
'low': statistic.low, 'date': statistic.when.isoformat()})
'low': statistic.low, 'date': statistic.when.isoformat(),
'urls': statistic.urls, 'endpoints': statistic.endpoints})
return JsonResponse(stats, encoder=JSEncoder)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment