migration bugfix, admin interface improvements, logical argument order

parent dddb73b1
......@@ -30,3 +30,4 @@ failmap_dataset*
failmap_testdataset*
dev_db/
failmap_debug_dataset*
temp/*
\ No newline at end of file
......@@ -3,7 +3,6 @@ from datetime import datetime
import pytz
from django.contrib import admin
from django.urls import reverse
from jet.admin import CompactInline
from failmap_admin.map.determineratings import (OrganizationRating, UrlRating, rate_organization,
......@@ -11,14 +10,11 @@ from failmap_admin.map.determineratings import (OrganizationRating, UrlRating, r
from failmap_admin.scanners.models import Endpoint
from failmap_admin.scanners.scanner_dns import brute_known_subdomains, certificate_transparency
from failmap_admin.scanners.scanner_http import scan_urls_on_standard_ports
<<<<<<< HEAD
from failmap_admin.scanners.scanner_plain_http import scan_urls as plain_http_scan_urls
from failmap_admin.scanners.scanner_screenshot import screenshot_urls
from failmap_admin.scanners.scanner_security_headers import scan_urls as security_headers_scan_urls
from failmap_admin.scanners.scanner_tls_qualys import ScannerTlsQualys
=======
from failmap_admin.scanners.scanner_tls_qualys import scan
>>>>>>> [WIP] #61 tls scanner rewrite, migration script, http_scanner_rewrite
from failmap_admin.scanners.scanner_tls_qualys import scan_url_list
from failmap_admin.scanners.admin import UrlIpInline
from ..app.models import Job
from .models import Coordinate, Organization, OrganizationType, Url
......@@ -57,6 +53,9 @@ class UrlRatingAdminInline(CompactInline):
class OrganizationAdmin(admin.ModelAdmin):
class Media:
js = ('js/action_buttons.js', )
list_display = ('name', 'type', 'country')
search_fields = (['name', 'country', 'type__name'])
list_filter = ('name', 'type__name', 'country') # todo: type is now listed as name, confusing
......@@ -74,18 +73,8 @@ class OrganizationAdmin(admin.ModelAdmin):
self.message_user(request, "Organization(s) have been rated")
def scan_organization(self, request, queryset):
# it's best to add all url's in one go, resulting in the fastest processing
urls_to_scan = []
for organization in queryset:
urls = Url.objects.filter(organization=organization)
for url in urls:
urls_to_scan.append(url.url)
scan(urls_to_scan)
urls = Url.objects.filter(organization__in=list(queryset))
scan_url_list(list(urls))
self.message_user(request, "Organization(s) have been scanned")
rate_organization.short_description = \
......@@ -96,7 +85,11 @@ class OrganizationAdmin(admin.ModelAdmin):
class UrlAdmin(admin.ModelAdmin):
list_display = ('url', 'endpoints', 'onboarded', 'uses_dns_wildcard', 'is_dead', 'not_resolvable', 'created_on')
class Media:
js = ('js/action_buttons.js', )
list_display = ('url', 'endpoints', 'current_rating', 'onboarded', 'uses_dns_wildcard',
'is_dead', 'not_resolvable', 'created_on')
search_fields = ('url', )
list_filter = ('url', 'is_dead', 'is_dead_since', 'is_dead_reason',
'not_resolvable', 'uses_dns_wildcard', 'organization')
......@@ -120,7 +113,11 @@ class UrlAdmin(admin.ModelAdmin):
def endpoints(self, obj: Url):
return obj.endpoint_set.count()
inlines = [EndpointAdminInline, UrlRatingAdminInline]
@staticmethod
def current_rating(obj):
return UrlRating.objects.filter(url=obj).latest('when').rating
inlines = [EndpointAdminInline, UrlRatingAdminInline, UrlIpInline]
actions = []
......@@ -139,7 +136,6 @@ class UrlAdmin(admin.ModelAdmin):
url.onboarded = True
url.onboarded_on = datetime.now(pytz.utc)
url.save()
<<<<<<< HEAD
self.message_user(request, "Onboard: Done")
actions.append('onboard')
onboard.short_description = "🔮 Onboard (dns, endpoints, scans, screenshot)"
......@@ -155,27 +151,6 @@ class UrlAdmin(admin.ModelAdmin):
self.message_user(request, "Discover subdomains (using known subdomains): Done")
dns_known_subdomains.short_description = "🗺 Discover subdomains (using known subdomains)"
actions.append('dns_known_subdomains')
=======
self.message_user(request, "URL(s) have been declared dead")
def rate_url(self, request, queryset):
for url in queryset:
rate_url(url=url)
self.message_user(request, "URL(s) have been rated")
def scan_url(self, request, queryset):
urls_to_scan = []
for url in queryset:
urls_to_scan.append(url.url)
scan(urls_to_scan)
self.message_user(request, "URL(s) have been scanned on TLS")
>>>>>>> [WIP] #61 tls scanner rewrite, migration script, http_scanner_rewrite
def discover_http_endpoints(self, request, queryset):
scan_urls_on_standard_ports([url for url in queryset])
......@@ -184,7 +159,7 @@ class UrlAdmin(admin.ModelAdmin):
actions.append('discover_http_endpoints')
def scan_tls_qualys(self, request, queryset):
ScannerTlsQualys().scan([url.url for url in queryset])
scan_url_list(list(queryset))
self.message_user(request, "Scan TLS (qualys, slow): Scheduled with Priority")
scan_tls_qualys.short_description = "🔬 Scan TLS (qualys, slow)"
actions.append('scan_tls_qualys')
......@@ -200,13 +175,8 @@ class UrlAdmin(admin.ModelAdmin):
actions.append('security_headers')
def plain_http_scan(self, request, queryset):
urls = list(queryset)
task = plain_http_scan_urls(urls=urls, execute=False)
print(task)
name = "Scan Plain Http (%s) " % str(urls)
job = Job.create(task, name, request)
link = reverse('admin:app_job_change', args=(job.id,))
self.message_user(request, '%s: job created, id: <a href="%s">%s</a>' % (name, link, str(job)))
plain_http_scan_urls([url for url in queryset])
self.message_user(request, "Scan Plain Http: done")
plain_http_scan.short_description = "🔬 Scan Plain Http"
actions.append('plain_http_scan')
......
......@@ -14,11 +14,25 @@ class TlsQualysScanAdminInline(CompactInline):
show_change_link = True
ordering = ["-rating_determined_on"]
class EndpointGenericScanInline(CompactInline):
model = EndpointGenericScan
extra = 0
show_change_link = True
ordering = ["-rating_determined_on"]
# can't make this admin, there is no join. And there shouldn't be.
# class TlsQualysScratchpadAdminInline(admin.StackedInline):
# model = TlsQualysScratchpad
# extra = 0
class UrlIpInline(CompactInline):
model = UrlIp
extra = 0
show_change_link = True
ordering = ["-discovered_on"]
class UrlIpAdmin(admin.ModelAdmin):
list_display = ('url', 'ip', 'rdns_name', 'discovered_on', 'is_unused_since')
......@@ -29,8 +43,8 @@ class UrlIpAdmin(admin.ModelAdmin):
class EndpointAdmin(admin.ModelAdmin):
list_display = ('url', 'domain', 'discovered_on', 'ip_version', 'port', 'protocol', 'is_dead_since',
'tls_scan_count')
list_display = ('id', 'url', 'discovered_on', 'ip_version', 'port', 'protocol', 'is_dead_since',
'tls_scans', 'generic_scans')
search_fields = ('url__url', 'domain', 'server_name', 'ip_version', 'port', 'protocol', 'is_dead',
'is_dead_since', 'is_dead_reason')
list_filter = ('server_name', 'ip_version', 'ip', 'port', 'protocol', 'is_dead')
......@@ -46,10 +60,14 @@ class EndpointAdmin(admin.ModelAdmin):
readonly_fields = ['discovered_on']
@staticmethod
def tls_scan_count(inst):
def tls_scans(inst):
return TlsQualysScan.objects.filter(endpoint=inst.id).count()
inlines = [TlsQualysScanAdminInline]
@staticmethod
def generic_scans(inst):
return EndpointGenericScan.objects.filter(endpoint_id=inst.id).count()
inlines = [TlsQualysScanAdminInline, EndpointGenericScanInline]
save_as = True # Save as new is nice for duplicating endpoints.
actions = ['rate_url', 'scan_url']
......
import logging
from random import Random
from django.core.management.base import BaseCommand
......@@ -18,9 +17,10 @@ class Command(BaseCommand):
def add_arguments(self, parser):
add_organization_argument(parser)
add_discover_verify(parser) # default verify
add_discover_verify(parser)
def handle(self, *args, **options):
# some expansion magic to avoid using eval
func = "verify_existing_endpoints" if options['method'] == "verify" else "discover_endpoints"
functionlist = {"verify_existing_endpoints": verify_existing_endpoints,
......@@ -39,7 +39,7 @@ class Command(BaseCommand):
functionlist[func](organization=organization)
def verify_existing_endpoints(protocol=None, port=None, organization=None):
def verify_existing_endpoints(port=None, protocol=None, organization=None):
"""
Checks all http(s) endpoints if they still exist. This is to monitor changes in the existing
dataset, without contacting an organization too often. It can be checked every few days,
......@@ -66,13 +66,13 @@ def verify_existing_endpoints(protocol=None, port=None, organization=None):
endpoints = endpoints.filter(url__organization=organization)
for endpoint in endpoints:
# todo: add IP version?
scan_url(endpoint.protocol, endpoint.url, endpoint.port)
def discover_endpoints(protocol=None, port=None, organization=None):
def discover_endpoints(port=None, protocol=None, organization=None):
"""
:return: None
"""
urls = Url.objects.all().filter(is_dead=False, not_resolvable=False).filter()
......@@ -92,9 +92,6 @@ def discover_endpoints(protocol=None, port=None, organization=None):
# Don't underestimate the flexibility of the internet.
ports = [80, 81, 82, 88, 443, 8008, 8080, 8088, 8443, 8888, 9443]
# scan the urls in a semi-random order
urls = sorted(urls, key=lambda L: Random().random())
logger.debug("Going to scan %s urls." % len(urls))
logger.debug("Going to scan %s urls." % urls.count())
scan_urls(protocols, urls, ports)
......@@ -3,7 +3,7 @@ import logging
from django.core.management.base import BaseCommand
from failmap_admin.scanners.models import (Endpoint, EndpointGenericScan, Screenshot, TlsQualysScan,
UrlIp)
UrlIp, Url)
logger = logging.getLogger(__package__)
......@@ -65,7 +65,11 @@ failmap-admin createsuperuser
failmap-admin clear-database
failmap-admin load-dataset testdata # we've not deleted columns till here.
failmap-admin rebuild-ratings
failmap-admin migrate-61-endpoint-ip-separation
failmap-admin rebuild-ratings
# The ratings should be the same
"""
......@@ -82,13 +86,51 @@ def merge_duplicate_endpoints():
# ordered by newest first, so you'll not have to figure out the current is_dead situation.
endpoints = Endpoint.objects.all().order_by("-discovered_on")
for endpoint in endpoints:
similar_endpoints = Endpoint.objects.all().filter(ip_version=endpoint.ip_version,
port=endpoint.port,
protocol=endpoint.protocol,
url=endpoint.url).exclude(id=endpoint.id)
# check if this endpoint still exists... it could be deleted in a previous check
# it can mess up connecting deleted endpoints
if not Endpoint.objects.filter(id=endpoint.id).exists():
logger.debug('Endpoint does not exist anymore, probably merged previously. Continuing...')
continue
logger.debug("Endpoint: %s, Discovered on: %s" % (endpoint, endpoint.discovered_on))
similar_endpoints = list(Endpoint.objects.all().filter(
ip_version=endpoint.ip_version,
port=endpoint.port,
protocol=endpoint.protocol,
url=endpoint.url).exclude(id=endpoint.id).order_by('-discovered_on'))
# In some cases there are hundreds of endpoints due to IP switching.
# Using the first one, we can determine from when the endpoint existed.
# This is relevant for creating reports.
# EX:
"""
URL DOMAIN DISCOVERED ON ipv PORT PROTOCOL IS DEAD SINCE TLS SCAN COUNT
... and 200 more...
opendata.arnhem.nl opendata.arnhem.nl 27 april 2016 14:59 4 443 https 3 mei 2016 03:18 1
opendata.arnhem.nl opendata.arnhem.nl 27 april 2016 14:59 4 443 https 3 mei 2016 03:18 1
opendata.arnhem.nl opendata.arnhem.nl 27 april 2016 14:59 4 443 https 3 mei 2016 03:18 1
opendata.arnhem.nl opendata.arnhem.nl 8 april 2016 19:52 4 443 https 27 april 2016 14:59 1
opendata.arnhem.nl opendata.arnhem.nl 8 april 2016 19:52 4 443 https 27 april 2016 14:59 1
"""
if similar_endpoints:
first_similar = similar_endpoints[-1]
logger.debug("Last similar: %s, Discovered on: %s" % (first_similar, first_similar.discovered_on))
endpoint.discovered_on = first_similar.discovered_on
endpoint.save()
else:
logger.debug("There are no similar endpoints. Ignoring.")
for similar_endpoint in similar_endpoints:
# apperantly exclude doesn't work... there goes my faith in the data layer.
if similar_endpoint == endpoint:
continue
# migrate all scans to the same endpoint
logger.debug("Merging similar: %s, Discovered on: %s" % (similar_endpoint, similar_endpoint.discovered_on))
EndpointGenericScan.objects.all().filter(endpoint=similar_endpoint).update(endpoint=endpoint)
TlsQualysScan.objects.all().filter(endpoint=similar_endpoint).update(endpoint=endpoint)
Screenshot.objects.all().filter(endpoint=similar_endpoint).update(endpoint=endpoint)
......
# -*- coding: utf-8 -*-
# Generated by Django 1.11.6 on 2017-11-10 12:37
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('scanners', '0025_auto_20171030_2111'),
]
operations = [
migrations.AlterField(
model_name='endpoint',
name='discovered_on',
field=models.DateTimeField(blank=True, null=True),
),
]
......@@ -71,7 +71,7 @@ class Endpoint(models.Model):
"SSH and so on. For more, read here: "
"https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol")
discovered_on = models.DateTimeField(auto_now_add=True, blank=True, null=True)
discovered_on = models.DateTimeField(blank=True, null=True)
# Till when the endpoint existed and why it was deleted (or didn't exist at all).
is_dead = models.IntegerField(default=False,
......@@ -148,6 +148,9 @@ class UrlIp(models.Model):
is_unused_reason = models.CharField(max_length=255, blank=True, null=True)
def __str__(self):
return "%s %s" % (self.ip, self.discovered_on.date())
class TlsQualysScan(models.Model):
"""
......@@ -224,7 +227,7 @@ class EndpointGenericScan(models.Model):
)
def __str__(self):
return "%s: %s rated %s on %s" % (self.rating_determined_on, self.type, self.rating, self.endpoint)
return "%s: %s %s on %s" % (self.rating_determined_on.date(), self.type, self.rating, self.endpoint)
class EndpointGenericScanScratchpad(models.Model):
......
......@@ -54,11 +54,7 @@ def validate_protocol(protocol):
def scan_urls_on_standard_ports(urls):
<<<<<<< HEAD
scan_urls(urls, [80, 81, 82, 88, 443, 8008, 8080, 8088, 8443, 8888, 9443], ['http', 'https'])
=======
scan_urls(['http', 'https'], urls, [80, 81, 82, 88, 443, 8008, 8080, 8088, 8443, 8888, 9443])
>>>>>>> [WIP] #61 tls scanner rewrite, migration script, http_scanner_rewrite
def scan_urls(protocols, urls, ports):
......
......@@ -129,7 +129,7 @@ def verify_is_secure(url):
# i've seen qualys saying there is no TLS, while there is!
# This _might_ revive an endpoint.
scanner_http_scan_urls([url], [443], ['https'])
scanner_http_scan_urls(['https'], [url], [443])
endpoints = Endpoint.objects.all().filter(url=url, is_dead=False,
protocol="https", port=443)
......
......@@ -149,6 +149,8 @@ def screenshot_with_chrome(endpoint, skip_if_latest=False):
:param skip_if_latest:
:return:
"""
if not check_installation('chrome'):
return
logger.debug("Chrome Screenshot: %s over IPv%s" % (endpoint.uri_url(), endpoint.ip_version))
......@@ -209,6 +211,7 @@ def check_installation(browser):
logger.error('%s is not available for %s, please update the configuration with the correct binary.'
% (browser, platform.system()))
return False
if not os.path.exists(browser_binary):
logger.error('Supplied browser does not exist in configured path: %s' % browser_binary)
return False
......
......@@ -556,7 +556,7 @@ def scan_organization(organization):
log.info("There are no alive https urls for this organization: %s" % organization)
return
scan(urls)
scan_url_list(urls)
for url in urls:
rerate_url_with_timeline(url=url)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment