Commit 489de2f2 authored by Elger Jonker's avatar Elger Jonker

added internet.nl email scans


Former-commit-id: b68648a2
parent 8dd96a6f
......@@ -421,7 +421,7 @@ class UrlAdmin(ActionMixin, ImportExportModelAdmin, nested_admin.NestedModelAdmi
list_filter = ['is_dead', 'is_dead_since', 'is_dead_reason',
'not_resolvable', 'not_resolvable_since', 'not_resolvable_reason',
'uses_dns_wildcard', 'organization', 'onboarded', 'onboarding_stage', 'organization__type__name',
'organization__country',
'organization__country', 'dns_supports_mx',
HasEndpointScansListFilter][::-1]
fieldsets = (
......@@ -429,7 +429,7 @@ class UrlAdmin(ActionMixin, ImportExportModelAdmin, nested_admin.NestedModelAdmi
'fields': ('url', 'organization', 'internal_notes', 'created_on', 'onboarded', 'onboarding_stage')
}),
('DNS', {
'fields': ('uses_dns_wildcard', ),
'fields': ('uses_dns_wildcard', 'dns_supports_mx', ),
}),
('Resolvability', {
'description': 'Non resolving urls cannot be reached anymore.',
......
# Generated by Django 2.1.3 on 2018-11-22 12:37
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('organizations', '0039_auto_20181119_1105'),
]
operations = [
migrations.AddField(
model_name='url',
name='dns_supports_mx',
field=models.BooleanField(
default=False, help_text="If there is at least one MX record available, so we can perform mail generic mail scans. (for thesescans we don't need to know what mail-ports and protocols/endpoints are available)."),
),
]
......@@ -255,6 +255,11 @@ class Url(models.Model):
help_text="When true, this domain uses a DNS wildcard and any subdomain will resolve to "
"something on this host.")
dns_supports_mx = models.BooleanField(
default=False,
help_text="If there is at least one MX record available, so we can perform mail generic mail scans. (for these"
"scans we don't need to know what mail-ports and protocols/endpoints are available).")
onboarding_stage = models.CharField(
max_length=150,
blank=True,
......
......@@ -4,8 +4,8 @@ from import_export.admin import ImportExportModelAdmin
from jet.admin import CompactInline
from jet.filters import RelatedFieldAjaxListFilter
from .models import (Endpoint, EndpointGenericScan, EndpointGenericScanScratchpad, Screenshot,
TlsQualysScan, TlsQualysScratchpad, TlsScan, UrlGenericScan, UrlIp)
from .models import (Endpoint, EndpointGenericScan, EndpointGenericScanScratchpad, InternetNLScan,
Screenshot, TlsQualysScan, TlsQualysScratchpad, TlsScan, UrlGenericScan, UrlIp)
class TlsQualysScanAdminInline(CompactInline):
......@@ -269,3 +269,11 @@ class EndpointGenericScanScratchpadAdmin(ImportExportModelAdmin, admin.ModelAdmi
search_fields = ('type', 'domain', 'when', 'data')
list_filter = ['type', 'domain', 'when', 'data'][::-1]
fields = ('type', 'domain', 'when', 'data')
@admin.register(InternetNLScan)
class InternetNLScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
list_display = ('started_on', 'finished_on', 'success', 'message')
search_fields = ('message', 'status_url')
list_filter = ('started_on', 'finished_on', 'success', 'message', )
fields = ('started', 'started_on', 'finished', 'finished_on', 'success', 'message', 'status_url')
import logging
from failmap.app.management.commands._private import DiscoverTaskCommand
from failmap.scanners.scanner import dns, dns_known_subdomains, ftp, http
from failmap.scanners.scanner import dns, dns_known_subdomains, ftp, http, mail
log = logging.getLogger(__name__)
......@@ -16,7 +16,8 @@ class Command(DiscoverTaskCommand):
'ftp': ftp,
'http': http,
'subdomains': dns,
'known_subdomains': dns_known_subdomains
'known_subdomains': dns_known_subdomains,
'mail': mail
}
def add_arguments(self, parser):
......
import logging
from django.core.management.base import BaseCommand
from failmap.scanners.scanner.mail import check_running_scans
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""See if there are updates on the scans at internet.nl"""
help = __doc__
def handle(self, *args, **options):
log.info("Checking status of scans on internet.nl")
check_running_scans()
log.info("Done checking status on internet.nl")
import logging
from failmap.app.management.commands._private import ScannerTaskCommand
from failmap.scanners.scanner import (debug, dnssec, dummy, ftp, http, onboard, plain_http,
from failmap.scanners.scanner import (debug, dnssec, dummy, ftp, http, mail, onboard, plain_http,
screenshot, security_headers, tls_osaft, tls_qualys)
log = logging.getLogger(__name__)
......@@ -23,7 +23,8 @@ class Command(ScannerTaskCommand):
'screenshot': screenshot,
'onboard': onboard,
'dummy': dummy,
'debug': debug
'debug': debug,
'mail': mail
}
def add_arguments(self, parser):
......
# Generated by Django 2.1.3 on 2018-11-22 15:06
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('scanners', '0050_auto_20181114_0735'),
]
operations = [
migrations.CreateModel(
name='InternetNLScan',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('success', models.BooleanField(default=False, help_text='If the scan finished successfully.')),
('started', models.BooleanField(default=False, help_text='If the scan is started, normally this is a YES.')),
('started_on', models.DateTimeField(blank=True, null=True)),
('finished', models.BooleanField(default=False, help_text='If the scan is complete.')),
('finished_on', models.DateTimeField(blank=True, null=True)),
('url', models.CharField(blank=True,
help_text='The url where the status of the batch scan can be retrieved.', max_length=500, null=True)),
('message', models.CharField(blank=True,
help_text='A status message received from the service', max_length=500, null=True)),
],
),
]
# Generated by Django 2.1.3 on 2018-11-22 15:10
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('scanners', '0051_internetnlscan'),
]
operations = [
migrations.RenameField(
model_name='internetnlscan',
old_name='url',
new_name='status_url',
),
]
# Generated by Django 2.1.3 on 2018-11-22 15:14
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('scanners', '0052_auto_20181122_1510'),
]
operations = [
migrations.AddField(
model_name='internetnlscan',
name='type',
field=models.CharField(blank=True, help_text='mail or web', max_length=4, null=True),
),
migrations.AlterField(
model_name='internetnlscan',
name='message',
field=models.TextField(
blank=True, help_text='A status message received from the service', max_length=500, null=True),
),
migrations.AlterField(
model_name='internetnlscan',
name='status_url',
field=models.TextField(
blank=True, help_text='The url where the status of the batch scan can be retrieved.', max_length=500, null=True),
),
]
......@@ -524,6 +524,55 @@ class Screenshot(models.Model):
created_on = models.DateTimeField(auto_now_add=True, db_index=True)
class InternetNLScan(models.Model):
type = models.CharField(
max_length=4,
help_text="mail or web",
blank=True,
null=True
)
success = models.BooleanField(
default=False,
help_text="If the scan finished successfully."
)
started = models.BooleanField(
default=False,
help_text="If the scan is started, normally this is a YES."
)
started_on = models.DateTimeField(
blank=True,
null=True
)
finished = models.BooleanField(
default=False,
help_text="If the scan is complete."
)
finished_on = models.DateTimeField(
blank=True,
null=True
)
status_url = models.TextField(
max_length=500,
help_text="The url where the status of the batch scan can be retrieved.",
blank=True,
null=True
)
message = models.TextField(
max_length=500,
help_text="A status message received from the service",
blank=True,
null=True
)
# A debugging table to help with API interactions.
# This can be auto truncated after a few days.
# Not anymore, since it's used to see if there are DNS problems (unresolvable domains)
......
......@@ -27,11 +27,16 @@ class UrlScanManager:
except ObjectDoesNotExist:
gs = UrlGenericScan()
# here we figured out that you can still pass a bool while type hinting.
# log.debug("Explanation new: '%s', old: '%s' eq: %s, Rating new: '%s', old: '%s', eq: %s" %
# (message, gs.explanation, message == gs.explanation, rating, gs.rating, str(rating) == gs.rating))
# last scan had exactly the same result, so don't create a new scan and just update the last scan date.
if gs.explanation == message and gs.rating == rating:
# while we have type hinting, it's still possible to pass in a boolean and then you compare a str to a bool...
if gs.explanation == message and gs.rating == str(rating):
log.debug("Scan had the same rating and message, updating last_scan_moment only.")
gs.last_scan_moment = datetime.now(pytz.utc)
gs.save()
gs.save(update_fields=['last_scan_moment'])
else:
# message and rating changed for this scan_type, so it's worth while to save the scan.
log.debug("Message or rating changed: making a new generic scan.")
......
This diff is collapsed.
......@@ -43,6 +43,9 @@ def allowed_to_scan(scanner_name: str = ""):
if scanner_name == 'scanner_screenshot':
return config.CREATE_HTTP_SCREENSHOT
if scanner_name == 'scanner_mail_internet_nl':
return config.SCAN_MAIL_INTERNET_NL
if scanner_name == 'scanner_security_headers':
return (config.SCAN_HTTP_HEADERS_HSTS or
config.SCAN_HTTP_HEADERS_XFO or
......
......@@ -2,11 +2,11 @@
from celery import group
from failmap.scanners.scanner import (dns, dnssec, dummy, ftp, http, plain_http, screenshot,
from failmap.scanners.scanner import (dns, dnssec, dummy, ftp, http, mail, plain_http, screenshot,
security_headers, tls_osaft, tls_qualys)
# explicitly declare the imported modules as this modules 'content', prevents pyflakes issues
__all__ = [tls_qualys, security_headers, dummy, http, dnssec, ftp, tls_osaft, screenshot, dns]
__all__ = [tls_qualys, security_headers, dummy, http, dnssec, ftp, tls_osaft, screenshot, dns, mail]
# This is the single source of truth regarding scanner configuration.
# Lists to be used elsewhere when tasks need to be composed, these lists contain compose functions.
......
......@@ -757,6 +757,11 @@ CONSTANCE_CONFIG = {
'SCAN_HTTP_HEADERS_X_XSS': (True, 'Do you want to scan for missing X-XSS headers?', bool),
'SCAN_HTTP_HEADERS_X_CONTENT': (True, 'Do you want to scan for missing X-Content-Type issues?', bool),
'SCAN_FTP': (True, 'Do you want to scan for FTP servers that are missing encryption?', bool),
# todo username and password for internetnl, make separate category
'SCAN_MAIL_INTERNET_NL': (True, 'Do you want to scan for email security configuration using internet.nl?', bool),
'INTERNET_NL_API_USERNAME': ('', 'Username for the internet.nl API', str),
'INTERNET_NL_API_PASSWORD': ('', 'Password for the internet.nl API', str),
'CREATE_HTTP_SCREENSHOT': (True, 'Todo: Does not work yet! Do you want to create screenshots for HTTP endpoints?',
bool),
......@@ -820,6 +825,8 @@ CONSTANCE_CONFIG_FIELDSETS = OrderedDict([
('FTP Scans', ('SCAN_FTP', 'REPORT_INCLUDE_FTP', 'SHOW_FTP', )),
('Internet.nl Scans', ('SCAN_MAIL_INTERNET_NL', 'INTERNET_NL_API_USERNAME', 'INTERNET_NL_API_PASSWORD')),
('Security Headers Scans', ('SCAN_HTTP_HEADERS_HSTS', 'SCAN_HTTP_HEADERS_XFO', 'SCAN_HTTP_HEADERS_X_XSS',
'SCAN_HTTP_HEADERS_X_CONTENT', 'REPORT_INCLUDE_HTTP_HEADERS_HSTS',
'REPORT_INCLUDE_HTTP_HEADERS_XFO', 'REPORT_INCLUDE_HTTP_HEADERS_X_XSS',
......@@ -928,6 +935,7 @@ JET_SIDE_MENU_ITEMS = [ # A list of application or custom item dicts
{'name': 'tlsscan', 'permissions': ['scanners.change_tlsscan']},
{'name': 'tlsqualysscan', 'permissions': ['scanners.change_tlsqualysscan']},
{'name': 'urlgenericscan', 'permissions': ['scanners.change_urlgenericscan']},
{'name': 'internetnlscan', 'permissions': ['scanners.change_internetnlscan']},
{'name': 'screenshot', 'permissions': ['admin']},
{'name': 'urlip', 'permissions': ['admin']},
{'name': 'tlsqualysscratchpad', 'permissions': ['admin']},
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment