Commit 07681fbb authored by Elger Jonker's avatar Elger Jonker

DNSSEC scanner, UrlGenericScan, Admin, started tests

parent a13a0267
......@@ -6,7 +6,7 @@ from jet.admin import CompactInline
from failmap.map.rating import rate_url
from .models import (Endpoint, EndpointGenericScan, EndpointGenericScanScratchpad, Screenshot,
State, TlsQualysScan, TlsQualysScratchpad, UrlIp)
State, TlsQualysScan, TlsQualysScratchpad, UrlGenericScan, UrlIp)
class TlsQualysScanAdminInline(CompactInline):
......@@ -167,19 +167,31 @@ class StateAdmin(ImportExportModelAdmin, admin.ModelAdmin):
class EndpointGenericScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
list_display = ('endpoint', 'type', 'domain', 'rating',
list_display = ('endpoint', 'domain', 'type', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on')
search_fields = ('endpoint__url__url', 'type', 'domain', 'rating',
search_fields = ('endpoint__url__url', 'type', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on')
list_filter = ('type', 'domain', 'rating',
list_filter = ('type', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on')
fields = ('endpoint', 'type', 'domain', 'rating',
fields = ('endpoint', 'type', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on')
# see tlsqualysscan why endpoint is here.
readonly_fields = ['last_scan_moment', 'endpoint']
class UrlGenericScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
list_display = ('url', 'domain', 'type', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on')
search_fields = ('url__url', 'type', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on')
list_filter = ('type', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on')
fields = ('url', 'type', 'rating',
'explanation', 'evidence', 'last_scan_moment', 'rating_determined_on')
readonly_fields = ['last_scan_moment', 'url']
class EndpointGenericScanScratchpadAdmin(ImportExportModelAdmin, admin.ModelAdmin):
list_display = ('type', 'domain', 'when', 'data')
search_fields = ('type', 'domain', 'when', 'data')
......@@ -193,5 +205,6 @@ admin.site.register(Endpoint, EndpointAdmin)
admin.site.register(Screenshot, ScreenshotAdmin)
admin.site.register(State, StateAdmin)
admin.site.register(EndpointGenericScan, EndpointGenericScanAdmin)
admin.site.register(UrlGenericScan, UrlGenericScanAdmin)
admin.site.register(EndpointGenericScanScratchpad, EndpointGenericScanScratchpadAdmin)
admin.site.register(UrlIp, UrlIpAdmin)
......@@ -16,7 +16,7 @@ class EndpointScanManager:
:return:
"""
@staticmethod
def add_scan(scan_type: str, endpoint: Endpoint, rating: str, message: str):
def add_scan(scan_type: str, endpoint: Endpoint, rating: str, message: str, evidence: str=""):
# Check if the latest scan has the same rating or not:
try:
......@@ -41,6 +41,7 @@ class EndpointScanManager:
gs.rating = rating
gs.endpoint = endpoint
gs.type = scan_type
gs.evidence = evidence
gs.last_scan_moment = datetime.now(pytz.utc)
gs.rating_determined_on = datetime.now(pytz.utc)
gs.save()
......
import logging
from failmap.app.management.commands._private import ScannerTaskCommand
from failmap.scanners import scanner_http
log = logging.getLogger(__name__)
class Command(ScannerTaskCommand):
"""Perform plain http scan on selected organizations."""
help = __doc__
scanner_module = scanner_http
# -*- coding: utf-8 -*-
# Generated by Django 1.11.8 on 2018-03-02 16:33
from __future__ import unicode_literals
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('organizations', '0022_auto_20180208_1318'),
('scanners', '0033_auto_20180301_2054'),
]
operations = [
migrations.CreateModel(
name='UrlGenericScan',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('type', models.CharField(db_index=True,
help_text='The type of scan that was performed. Instead of having different tables for eachscan, this label separates the scans.', max_length=60)),
('rating', models.CharField(
default=0, help_text="Preferably an integer, 'True' or 'False'. Keep ratings over time consistent.", max_length=6)),
('explanation', models.CharField(default=0,
help_text='Short explanation from the scanner on how the rating came to be.', max_length=255)),
('evidence', models.CharField(default=0, help_text='Content that might help understanding the result.', max_length=9001)),
('last_scan_moment', models.DateTimeField(auto_now_add=True, db_index=True,
help_text='This gets updated when all the other fields stay the same. If one changes, anew scan will be saved, obsoleting the older ones.')),
('rating_determined_on', models.DateTimeField(
help_text="This is when the current rating was first discovered. It may be obsoleted byanother rating or explanation (which might have the same rating). This date cannot change once it's set.")),
('domain', models.CharField(help_text='Deprecated', max_length=255)),
('url', models.ForeignKey(blank=True, null=True,
on_delete=django.db.models.deletion.CASCADE, to='organizations.Url')),
],
options={
'abstract': False,
},
),
migrations.AddField(
model_name='endpointgenericscan',
name='evidence',
field=models.CharField(
default=0, help_text='Content that might help understanding the result.', max_length=9001),
),
migrations.AlterField(
model_name='endpointgenericscan',
name='domain',
field=models.CharField(help_text='Deprecated', max_length=255),
),
migrations.AlterField(
model_name='endpointgenericscan',
name='explanation',
field=models.CharField(
default=0, help_text='Short explanation from the scanner on how the rating came to be.', max_length=255),
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.8 on 2018-03-02 17:22
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('scanners', '0034_auto_20180302_1633'),
]
operations = [
migrations.AlterField(
model_name='endpointgenericscan',
name='domain',
field=models.CharField(help_text='Deprecated. Text value representing the url scanned.', max_length=255),
),
migrations.AlterField(
model_name='urlgenericscan',
name='domain',
field=models.CharField(help_text='Deprecated. Text value representing the url scanned.', max_length=255),
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.8 on 2018-03-02 17:41
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('scanners', '0035_auto_20180302_1722'),
]
operations = [
migrations.AlterField(
model_name='endpointgenericscan',
name='evidence',
field=models.TextField(
default=0, help_text='Content that might help understanding the result.', max_length=9001),
),
migrations.AlterField(
model_name='urlgenericscan',
name='evidence',
field=models.TextField(
default=0, help_text='Content that might help understanding the result.', max_length=9001),
),
]
......@@ -190,7 +190,8 @@ class TlsQualysScan(models.Model):
return "%s - %s" % (self.scan_date, self.qualys_rating)
class EndpointGenericScan(models.Model):
# https://docs.djangoproject.com/en/dev/topics/db/models/#id6
class GenericScanMixin(models.Model):
"""
This is a fact, a point in time.
"""
......@@ -199,27 +200,21 @@ class EndpointGenericScan(models.Model):
db_index=True,
help_text="The type of scan that was performed. Instead of having different tables for each"
"scan, this label separates the scans.")
endpoint = models.ForeignKey(
Endpoint,
on_delete=models.CASCADE,
null=True,
blank=True,
)
domain = models.CharField(
max_length=255,
help_text="Deprecated. Used when there is no known endpoint.",
blank=True
)
rating = models.CharField(
max_length=6,
default=0,
help_text="Preferably an integer, 'True' or 'False'. Keep ratings over time consistent."
)
explanation = models.CharField(
max_length=9001,
max_length=255,
default=0,
help_text="Short explanation from the scanner on how the rating came to be."
)
evidence = models.TextField(
max_length=9001,
default=0,
help_text="Content that might help understanding the result."
)
last_scan_moment = models.DateTimeField(
auto_now_add=True,
db_index=True,
......@@ -231,11 +226,53 @@ class EndpointGenericScan(models.Model):
"another rating or explanation (which might have the same rating). This date "
"cannot change once it's set."
)
domain = models.CharField(
max_length=255,
help_text="Deprecated. Text value representing the url scanned."
)
class Meta:
"""
From the docs:
Django does make one adjustment to the Meta class of an abstract base class: before installing the Meta
attribute, it sets abstract=False. This means that children of abstract base classes don’t automatically
become abstract classes themselves. Of course, you can make an abstract base class that inherits from
another abstract base class. You just need to remember to explicitly set abstract=True each time.
"""
abstract = True
class EndpointGenericScan(GenericScanMixin):
"""
Only changes are saved as a scan.
"""
endpoint = models.ForeignKey(
Endpoint,
on_delete=models.CASCADE,
null=True,
blank=True,
)
def __str__(self):
return "%s: %s %s on %s" % (self.rating_determined_on.date(), self.type, self.rating, self.endpoint)
class UrlGenericScan(GenericScanMixin):
"""
Only changes are saved as a scan.
"""
url = models.ForeignKey(
Url,
null=True,
blank=True,
on_delete=models.CASCADE
)
def __str__(self):
return "%s: %s %s on %s" % (self.rating_determined_on.date(), self.type, self.rating, self.url)
class EndpointGenericScanScratchpad(models.Model):
"""
A debugging channel for generic scans.
......
......@@ -17,7 +17,6 @@ todo: You'll end up with DNSCheck missing in $... forget it, use docker!
"""
import logging
import random
import subprocess
from typing import List
......@@ -26,7 +25,7 @@ from django.conf import settings
from failmap.celery import ParentFailed, app
from failmap.organizations.models import Organization, Url
from failmap.scanners.endpoint_scan_manager import EndpointScanManager
from failmap.scanners.url_scan_manager import UrlScanManager
from .models import Endpoint
......@@ -44,45 +43,52 @@ def compose_task(
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),
) -> Task:
"""Compose taskset to scan specified endpoints.
""" Compose taskset to scan toplevel domains.
DNSSEC is implemented on a (top level) url. It's useless to scan per-endpoint.
This is the first scanner that uses the UrlGenericScan table, which looks nearly the same as the
endpoint variant.
"""
# apply filter to organizations (or if no filter, all organizations)
organizations = Organization.objects.filter(**organizations_filter)
# apply filter to urls in organizations (or if no filter, all urls)
urls = Url.objects.filter(organization__in=organizations, **urls_filter)
# select endpoints to scan based on filters
endpoints = Endpoint.objects.filter(
# apply filter to endpoints (or if no filter, all endpoints)
url__in=urls, **endpoints_filter,
# also apply manditory filters to only select valid endpoints for this action
is_dead=False, protocol__in=['http', 'https'])
if not endpoints:
# DNSSEC only works on top level urls
urls_filter = dict(urls_filter, **{"url__iregex": "^[^.]*\.[^.]*$"})
urls = []
# gather urls from organizations
if organizations_filter:
organizations = Organization.objects.filter(**organizations_filter)
urls += Url.objects.filter(organization__in=organizations, **urls_filter)
elif endpoints_filter:
# and now retrieve urls from endpoints
endpoints = Endpoint.objects.filter(**endpoints_filter)
urls += Url.objects.filter(endpoint__in=endpoints, **urls_filter)
else:
# now urls directly
urls += Url.objects.filter(**urls_filter)
if not urls:
raise Exception('Applied filters resulted in no tasks!')
log.info('Creating scan task for %s endpoints for %s urls for %s organizations.',
len(endpoints), len(urls), len(organizations))
# only unique urls
urls = list(set(urls))
log.info('Creating scan task for %s urls.', len(urls))
# todo: this is a poor mans solution for queue randomization, will be implemented in the queue manager
# make sure we're dealing with a list for the coming random function
endpoints = list(endpoints)
# randomize the endpoints so hosts are contacted in random order (less pressure)
random.shuffle(endpoints)
# The number of top level urls is negligible, so randomization is not needed.
# create tasks for scanning all selected endpoints as a single managable group
# Sending entire objects is possible. How signatures (.s and .si) work is documented:
# http://docs.celeryproject.org/en/latest/reference/celery.html#celery.signature
task = group(
scan_dnssec.s(endpoint.url.url) | store_dnssec.s(endpoint) for endpoint in endpoints
scan_dnssec.s(url.url) | store_dnssec.s(url) for url in urls
)
return task
@app.task(queue='storage')
def store_dnssec(result: List[str], endpoint: Endpoint):
def store_dnssec(result: List[str], url: Url):
"""
:param result: param endpoint:
......@@ -102,18 +108,16 @@ def store_dnssec(result: List[str], endpoint: Endpoint):
# /failmap/map/locale/*/django.po
# translate them and then run "failmap translate" again.
messages = {
'ERROR': 'DNSSEC is incorrectly or not configured. Use below information to debug: %s',
'WARNING': 'DNSSEC is incorrectly configured. Use below information to debug: %s',
'INFO': 'DNSSEC seems to be implemented sufficiently. %s'
'ERROR': 'DNSSEC is incorrectly or not configured (errors found).',
'WARNING': 'DNSSEC is incorrectly configured (warnings found).',
'INFO': 'DNSSEC seems to be implemented sufficiently.'
}
log.debug('Storing result: %s, for endpoint: %s.', result, endpoint)
log.debug('Storing result: %s, for url: %s.', result, url)
# You can save any (string) value and any (string) message.
# The EndpointScanManager deduplicates the data for you automatically.
if result:
EndpointScanManager.add_scan('DNSSEC', endpoint, level, messages[level] % relevant)
else:
EndpointScanManager.add_scan('DNSSEC', endpoint, level, messages[level] % "")
UrlScanManager.add_scan('DNSSEC', url, level, messages[level], evidence=",\n".join(relevant))
# return something informative
return {'status': 'success', 'result': level}
......@@ -143,8 +147,7 @@ def scan_dnssec(self, url: str):
log.info('Done scanning: %s, result: %s', url, content)
return content
# errors:
# subprocess.CalledProcessError non zero exit status
# subprocess.CalledProcessError: non zero exit status
# OSError: Incorrect permission, file doesn't exist, etc
except (subprocess.CalledProcessError, OSError) as e:
# If an expected error is encountered put this task back on the queue to be retried.
......
......@@ -47,7 +47,7 @@ def compose_task(
# apply filter to organizations (or if no filter, all organizations)
organizations = Organization.objects.filter(**organizations_filter)
# apply filter to urls in organizations (or if no filter, all urls)
# apply filter to urls in organizations (or if no filter, all urls (which is not wat below code does))
urls = Url.objects.filter(organization__in=organizations, **urls_filter)
# select endpoints to scan based on filters
......
This diff is collapsed.
......@@ -66,3 +66,5 @@ csscompressor # css compression
retry
sphinx_rtd_theme
IPython # make shell_plus better
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment