Commit a311a487 authored by Elger Jonker's avatar Elger Jonker

[WIP] O-Saft result saving, reporting, various test tools, comparison with other scanner

Former-commit-id: 519aa631
parent 810a8280
......@@ -7,7 +7,7 @@ from jet.filters import RelatedFieldAjaxListFilter
from import rate_url
from .models import (Endpoint, EndpointGenericScan, EndpointGenericScanScratchpad, Screenshot,
TlsQualysScan, TlsQualysScratchpad, UrlGenericScan, UrlIp)
TlsQualysScan, TlsQualysScratchpad, TlsScan, UrlGenericScan, UrlIp)
class TlsQualysScanAdminInline(CompactInline):
......@@ -105,6 +105,23 @@ class EndpointAdmin(ImportExportModelAdmin, admin.ModelAdmin):
scan_url.short_description = "Scan (url)"
class TlsScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
list_display = ('endpoint', 'rating', 'rating_no_trust', 'explanation',
'last_scan_moment', 'rating_determined_on')
search_fields = ('endpoint__url__url', 'rating', 'rating_no_trust',
'scan_date', 'rating_determined_on')
list_filter = ('rating', 'rating_no_trust', 'explanation',
'scan_date', 'rating_determined_on',
'endpoint__port', 'endpoint__ip_version', 'endpoint__discovered_on', 'endpoint__is_dead'
fields = ('endpoint', 'rating', 'rating_no_trust', 'explanation', 'evidence',
'rating_determined_on', 'last_scan_moment')
readonly_fields = ('scan_date', 'scan_time', 'last_scan_moment', 'endpoint')
class TlsQualysScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
list_display = ('endpoint', 'qualys_rating', 'qualys_rating_no_trust', 'qualys_message',
'last_scan_moment', 'rating_determined_on')
......@@ -205,6 +222,7 @@ class EndpointGenericScanScratchpadAdmin(ImportExportModelAdmin, admin.ModelAdmi, TlsQualysScanAdmin), TlsScanAdmin), TlsQualysScratchpadAdmin), EndpointAdmin), ScreenshotAdmin)
......@@ -38,14 +38,28 @@ class Command(BaseCommand):
def test_osaft():
from failmap.scanners.scanner_tls_osaft import scan_address, determine_grade, debug_grade
from failmap.scanners.scanner_tls_osaft import scan_address, determine_grade, grade_report, scan_url
from failmap.scanners.scanner import q_configurations_to_scan
urls = Url.objects.filter(
for url in urls:
address = ''
port = 443
report = scan_address(address, port)
grades, trust = determine_grade(report)
debug_grade(grades, trust)
print(grade_report(grades, trust))
def rebuild_ratings():
import logging
from import BaseCommand
from failmap.scanners.scanner_tls_osaft import (ammend_unsuported_issues, determine_grade,
grade_report, run_osaft_scan)
logger = logging.getLogger(__package__)
class Command(BaseCommand):
help = 'Development command'
def handle(self, *args, **options):
address = ""
port = 443
report = run_osaft_scan(address, port)
report = ammend_unsuported_issues(report, address, port)
grades, trust, report = determine_grade(report)
print(grade_report(grades, trust, report))
......@@ -2,7 +2,7 @@ import logging
from import ScannerTaskCommand
from failmap.scanners import (scanner_dnssec, scanner_ftp, scanner_http, scanner_plain_http,
scanner_security_headers, scanner_tls_qualys)
scanner_security_headers, scanner_tls_osaft, scanner_tls_qualys)
log = logging.getLogger(__name__)
......@@ -23,7 +23,8 @@ class Command(ScannerTaskCommand):
'headers': scanner_security_headers,
'plain': scanner_plain_http,
'endpoints': scanner_http,
'tls': scanner_tls_qualys,
'tls': scanner_tls_osaft,
'tlsq': scanner_tls_qualys,
'ftp': scanner_ftp
import logging
from import BaseCommand
logger = logging.getLogger(__package__)
class Command(BaseCommand):
help = 'Determine if there are major differences between Qualys and O-Saft scanner'
def handle(self, *args, **options):
from failmap.scanners.scanner_tls_osaft import compare_results
# Generated by Django 2.0.7 on 2018-07-12 14:11
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('scanners', '0040_auto_20180523_1856'),
operations = [
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('rating', models.CharField(default='', help_text='F, D, C, B, A-, A, A+, T', max_length=3)),
('rating_no_trust', models.CharField(default='', help_text='rating when trust issues are ignored', max_length=3)),
('explanation', models.CharField(default=0,
help_text='Short explanation from the scanner on how the rating came to be.', max_length=255)),
('evidence', models.TextField(default=0, help_text='Content that might help understanding the result.', max_length=9001)),
('scan_date', models.DateField(auto_now_add=True)),
('scan_time', models.TimeField(auto_now_add=True)),
('last_scan_moment', models.DateTimeField(auto_now_add=True, db_index=True)),
('rating_determined_on', models.DateTimeField()),
('endpoint', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='scanners.Endpoint')),
......@@ -196,6 +196,44 @@ class TlsQualysScan(models.Model):
return "%s - %s" % (self.scan_date, self.qualys_rating)
class TlsScan(models.Model):
Model for our own TLS scans
endpoint = models.ForeignKey(Endpoint, on_delete=models.CASCADE)
# result from the API
rating = models.CharField(
help_text="F, D, C, B, A-, A, A+, T")
rating_no_trust = models.CharField(
help_text="rating when trust issues are ignored"
explanation = models.CharField(
help_text="Short explanation from the scanner on how the rating came to be."
evidence = models.TextField(
help_text="Content that might help understanding the result."
scan_date = models.DateField(auto_now_add=True) # completed scan
scan_time = models.TimeField(auto_now_add=True) # For database indexes
last_scan_moment = models.DateTimeField(auto_now_add=True, db_index=True) # For database indexes
# This is when the rating was determined. Ratings don't change that much.
rating_determined_on = models.DateTimeField()
def __str__(self):
return "%s - %s" % (self.scan_date, self.rating)
class GenericScanMixin(models.Model):
This diff is collapsed.
......@@ -7,10 +7,11 @@ from celery.result import allow_join_result
from failmap.celery import app
from . import (scanner_dns, scanner_dnssec, scanner_dummy, scanner_ftp, scanner_http,
scanner_plain_http, scanner_security_headers, scanner_tls_qualys)
scanner_plain_http, scanner_security_headers, scanner_tls_osaft, scanner_tls_qualys)
# explicitly declare the imported modules as this modules 'content', prevents pyflakes issues
__all__ = [scanner_tls_qualys, scanner_security_headers, scanner_dummy, scanner_http, scanner_dnssec, scanner_ftp]
__all__ = [scanner_tls_qualys, scanner_security_headers, scanner_dummy, scanner_http, scanner_dnssec, scanner_ftp,
# This is the single source of truth regarding scanner configuration.
# Lists to be used elsewhere when tasks need to be composed, these lists contain compose functions.
import logging
from datetime import datetime
import pytz
from django.core.exceptions import ObjectDoesNotExist
from .models import Endpoint, TlsScan
logger = logging.getLogger(__package__)
class TlsScanManager:
Helps with data deduplication of scans. Helps storing scans in a more generic way.
def add_scan(endpoint: Endpoint, rating: str, rating_no_trust: str, message: str, evidence: str=""):
# Check if the latest scan has the same rating or not:
gs = TlsScan.objects.all().filter(
except ObjectDoesNotExist:
gs = TlsScan()
# last scan had exactly the same result, so don't create a new scan and just update the
# last scan date.
if gs.explanation == message and gs.rating == rating and gs.rating_no_trust == rating_no_trust:
logger.debug("Scan had the same rating and message, updating last_scan_moment only.")
gs.last_scan_moment =
# message and rating changed for this scan_type, so it's worth while to save the scan.
logger.debug("Message or rating changed: making a new generic scan.")
gs = TlsScan()
gs.endpoint = endpoint
gs.rating = rating
gs.rating_no_trust = rating_no_trust
gs.explanation = message
gs.evidence = evidence
gs.last_scan_moment =
gs.rating_determined_on =
def had_scan_with_points(endpoint: Endpoint):
Used for data deduplication. Don't save a scan that had zero points, but you can upgrade
to zero (or another rating)
:param scan_type:
:param endpoint:
from .models import EndpointGenericScan
gs = EndpointGenericScan.objects.all().filter(
if gs.rating:
return True
except ObjectDoesNotExist:
return False
......@@ -845,6 +845,7 @@ JET_SIDE_MENU_ITEMS = [ # A list of application or custom item dicts
{'app_label': 'scanners', 'label': _('🔬 scanners'), 'items': [
{'name': 'endpoint'},
{'name': 'endpointgenericscan'},
{'name': 'tlsscan'},
{'name': 'tlsqualysscan'},
{'name': 'urlgenericscan'},
{'name': 'screenshot'},
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment