Commit 7ce736bd authored by Elger Jonker's avatar Elger Jonker

old code removal, unification in scan_types in js+python+db


Former-commit-id: 0eec91f9
parent 3cd6b26c
This diff is collapsed.
......@@ -959,19 +959,19 @@ msgid "report_header_ftp"
msgstr "File Transfer (FTP)"
#: failmap/map/templates/map/index.html
msgid "report_header_security_headers_strict_transport_security"
msgid "report_header_http_security_header_strict_transport_security"
msgstr "Strict-Transport-Security Header (HSTS)"
#: failmap/map/templates/map/index.html
msgid "report_header_security_headers_x_frame_options"
msgid "report_header_http_security_header_x_frame_options"
msgstr "X-Frame-Options Header (clickjacking)"
#: failmap/map/templates/map/index.html
msgid "report_header_security_headers_x_xss_protection"
msgid "report_header_http_security_header_x_xss_protection"
msgstr "X-XSS-Protection Header"
#: failmap/map/templates/map/index.html
msgid "report_header_security_headers_x_content_type_options"
msgid "report_header_http_security_header_x_content_type_options"
msgstr "X-Content-Type-Options Header"
#: failmap/map/templates/map/index.html
......
......@@ -231,19 +231,19 @@ msgid "report_header_plain_https"
msgstr "Missing transport encryption"
#: failmap/map/static/js/script.js
msgid "report_header_security_headers_x_xss_protection"
msgid "report_header_http_security_header_x_xss_protection"
msgstr "X-XSS-Protection Header"
#: failmap/map/static/js/script.js
msgid "report_header_security_headers_x_frame_options"
msgid "report_header_http_security_header_x_frame_options"
msgstr "X-Frame-Options Header (clickjacking)"
#: failmap/map/static/js/script.js
msgid "report_header_security_headers_x_content_type_options"
msgid "report_header_http_security_header_x_content_type_options"
msgstr "X-Content-Type-Options"
#: failmap/map/static/js/script.js
msgid "report_header_security_headers_strict_transport_security"
msgid "report_header_http_security_header_strict_transport_security"
msgstr "Strict-Transport-Security Header (HSTS)"
#: failmap/map/static/js/script.js
......
......@@ -977,19 +977,19 @@ msgid "report_header_ftp"
msgstr "Bestandsoverdracht (FTP)"
#: failmap/map/templates/map/index.html
msgid "report_header_security_headers_strict_transport_security"
msgid "report_header_http_security_header_strict_transport_security"
msgstr "Strict-Transport-Security Header (HSTS)"
#: failmap/map/templates/map/index.html
msgid "report_header_security_headers_x_frame_options"
msgid "report_header_http_security_header_x_frame_options"
msgstr "X-Frame-Options Header (clickjacking)"
#: failmap/map/templates/map/index.html
msgid "report_header_security_headers_x_xss_protection"
msgid "report_header_http_security_header_x_xss_protection"
msgstr "X-XSS-Protection Header"
#: failmap/map/templates/map/index.html
msgid "report_header_security_headers_x_content_type_options"
msgid "report_header_http_security_header_x_content_type_options"
msgstr "X-Content-Type-Options Header"
#: failmap/map/templates/map/index.html
......
......@@ -241,19 +241,19 @@ msgid "report_header_plain_https"
msgstr "Ontbrekende Transport Layer Security (TLS)"
#: failmap/map/static/js/script.js
msgid "report_header_security_headers_x_xss_protection"
msgid "report_header_http_security_header_x_xss_protection"
msgstr "X-XSS-Protection Header"
#: failmap/map/static/js/script.js
msgid "report_header_security_headers_x_frame_options"
msgid "report_header_http_security_header_x_frame_options"
msgstr "X-Frame-Options Header (clickjacking)"
#: failmap/map/static/js/script.js
msgid "report_header_security_headers_x_content_type_options"
msgid "report_header_http_security_header_x_content_type_options"
msgstr "X-Content-Type-Options"
#: failmap/map/static/js/script.js
msgid "report_header_security_headers_strict_transport_security"
msgid "report_header_http_security_header_strict_transport_security"
msgstr "Strict-Transport-Security Header (HSTS)"
#: failmap/map/static/js/script.js
......
......@@ -996,19 +996,19 @@ msgid "report_header_ftp"
msgstr "🌈"
#: failmap/map/templates/map/index.html
msgid "report_header_security_headers_strict_transport_security"
msgid "report_header_http_security_header_strict_transport_security"
msgstr "🌈"
#: failmap/map/templates/map/index.html
msgid "report_header_security_headers_x_frame_options"
msgid "report_header_http_security_header_x_frame_options"
msgstr "🌈"
#: failmap/map/templates/map/index.html
msgid "report_header_security_headers_x_xss_protection"
msgid "report_header_http_security_header_x_xss_protection"
msgstr "🌈"
#: failmap/map/templates/map/index.html
msgid "report_header_security_headers_x_content_type_options"
msgid "report_header_http_security_header_x_content_type_options"
msgstr "🌈"
#: failmap/map/templates/map/index.html
......
......@@ -229,19 +229,19 @@ msgid "report_header_plain_https"
msgstr "🌈"
#: failmap/map/static/js/script.js
msgid "report_header_security_headers_x_xss_protection"
msgid "report_header_https_security_header_x_xss_protection"
msgstr "🌈"
#: failmap/map/static/js/script.js
msgid "report_header_security_headers_x_frame_options"
msgid "report_header_https_security_header_x_frame_options"
msgstr "🌈"
#: failmap/map/static/js/script.js
msgid "report_header_security_headers_x_content_type_options"
msgid "report_header_https_security_header_x_content_type_options"
msgstr "🌈"
#: failmap/map/static/js/script.js
msgid "report_header_security_headers_strict_transport_security"
msgid "report_header_https_security_header_strict_transport_security"
msgstr "🌈"
#: failmap/map/static/js/script.js
......
......@@ -20,7 +20,7 @@ from .models import (Configuration, MapDataCache, OrganizationRating, UrlRating,
log = logging.getLogger(__package__)
from failmap.scanners.types import ENDPOINT_SCAN_TYPES, URL_SCAN_TYPES
from failmap.scanners.types import ENDPOINT_SCAN_TYPES, URL_SCAN_TYPES, ALL_SCAN_TYPES
FAILMAP_STARTED = datetime(year=2016, month=1, day=1, hour=13, minute=37, second=42, tzinfo=pytz.utc)
......@@ -237,13 +237,13 @@ def significant_moments(organizations: List[Organization] = None, urls: List[Url
if constance_cached_value('REPORT_INCLUDE_HTTP_MISSING_TLS'):
allowed_to_report.append("plain_https")
if constance_cached_value('REPORT_INCLUDE_HTTP_HEADERS_HSTS'):
allowed_to_report.append("Strict-Transport-Security")
allowed_to_report.append("http_security_header_strict_transport_security")
if constance_cached_value('REPORT_INCLUDE_HTTP_HEADERS_XFO'):
allowed_to_report.append("X-Frame-Options")
allowed_to_report.append("http_security_header_x_frame_options")
if constance_cached_value('REPORT_INCLUDE_HTTP_HEADERS_X_XSS'):
allowed_to_report.append("X-XSS-Protection")
allowed_to_report.append("http_security_header_x_xss_protection")
if constance_cached_value('REPORT_INCLUDE_HTTP_HEADERS_X_CONTENT'):
allowed_to_report.append("X-Content-Type-Options")
allowed_to_report.append("http_security_header_x_content_type_options")
if constance_cached_value('REPORT_INCLUDE_DNS_DNSSEC'):
allowed_to_report.append("DNSSEC")
if constance_cached_value('REPORT_INCLUDE_FTP'):
......@@ -1576,32 +1576,27 @@ def calculate_map_data_today():
def calculate_map_data(days: int = 366):
log.info("calculate_map_data")
# all vulnerabilities
filters = ["security_headers_strict_transport_security", "security_headers_x_content_type_options", "ftp", "DNSSEC",
"security_headers_x_frame_options", "security_headers_x_xss_protection", "tls_qualys", "plain_https",
'', 'tls_qualys_certificate_trusted', 'tls_qualys_encryption_quality']
map_configurations = Configuration.objects.all().filter(
is_displayed=True).order_by('display_order').values('country', 'organization_type__name', 'organization_type')
for map_configuration in map_configurations:
for days_back in list(reversed(range(0, days))):
when = datetime.now(pytz.utc) - timedelta(days=days_back)
for filter in filters:
for scan_type in ALL_SCAN_TYPES:
# You can expect something to change each day. Therefore just store the map data each day.
MapDataCache.objects.all().filter(
when=when, country=map_configuration['country'],
organization_type=OrganizationType(pk=map_configuration['organization_type']),
filters=[filter]
filters=[scan_type]
).delete()
log.debug("Country: %s, Organization_type: %s, day: %s, date: %s, filter: %s" % (
map_configuration['country'], map_configuration['organization_type__name'],
days_back, when, filter
days_back, when, scan_type
))
data = get_map_data(map_configuration['country'], map_configuration['organization_type__name'],
days_back, filter)
days_back, scan_type)
from django.db import OperationalError
......@@ -1609,7 +1604,7 @@ def calculate_map_data(days: int = 366):
cached = MapDataCache()
cached.organization_type = OrganizationType(pk=map_configuration['organization_type'])
cached.country = map_configuration['country']
cached.filters = [filter]
cached.filters = [scan_type]
cached.when = when
cached.dataset = data
cached.save()
......
......@@ -59,10 +59,10 @@ var dynamic_translations = function(){
// vulnerabilities:
gettext('report_header_tls_qualys');
gettext('report_header_plain_https');
gettext('report_header_security_headers_x_xss_protection');
gettext('report_header_security_headers_x_frame_options');
gettext('report_header_security_headers_x_content_type_options');
gettext('report_header_security_headers_strict_transport_security');
gettext('report_header_http_security_header_x_xss_protection');
gettext('report_header_http_security_header_x_frame_options');
gettext('report_header_http_security_header_x_content_type_options');
gettext('report_header_http_security_header_strict_transport_security');
gettext('report_header_DNSSEC');
gettext('report_header_ftp');
gettext('report_header_tls_qualys_certificate_trusted');
......
This diff is collapsed.
This diff is collapsed.
......@@ -34,6 +34,8 @@ from .. import __version__
from ..app.common import JSEncoder
from .calculate import get_calculation
from failmap.scanners.types import ENDPOINT_SCAN_TYPES, URL_SCAN_TYPES, ALL_SCAN_TYPES
log = logging.getLogger(__package__)
one_minute = 60
......@@ -1336,26 +1338,16 @@ def get_map_data(country: str = "NL", organization_type: str = "municipality", d
desired_url_scans = []
desired_endpoint_scans = []
possible_url_scans = ['DNSSEC']
possible_endpoint_scans = ['security_headers_strict_transport_security',
'security_headers_x_content_type_options',
'security_headers_x_frame_options',
'security_headers_x_xss_protection',
'tls_qualys_certificate_trusted',
'tls_qualys_encryption_quality',
'plain_https',
'ftp']
if displayed_issue in possible_url_scans:
if displayed_issue in URL_SCAN_TYPES:
desired_url_scans += [displayed_issue]
if displayed_issue in possible_endpoint_scans:
if displayed_issue in ENDPOINT_SCAN_TYPES:
desired_endpoint_scans += [displayed_issue]
# fallback if no data, which is the default.
if not desired_url_scans and not desired_endpoint_scans:
desired_url_scans = possible_url_scans
desired_endpoint_scans = possible_endpoint_scans
desired_url_scans = URL_SCAN_TYPES
desired_endpoint_scans = ENDPOINT_SCAN_TYPES
# look if we have data in the cache, which will save some calculations and a slower query
cached = MapDataCache.objects.all().filter(country=country,
......@@ -1589,26 +1581,17 @@ def latest_scans(request, scan_type, country: str = "NL", organization_type="mun
"remark": remark,
}
if scan_type not in ["tls_qualys_encryption_quality", "tls_qualys_certificate_trusted",
"Strict-Transport-Security", "X-Content-Type-Options", "X-Frame-Options", "X-XSS-Protection",
"plain_https", "ftp", 'DNSSEC']:
if scan_type not in ALL_SCAN_TYPES:
return empty_response()
if scan_type == "tls_qualys":
scans = list(TlsQualysScan.objects.filter(
endpoint__url__organization__type=get_organization_type(organization_type),
endpoint__url__organization__country=get_country(country)
).order_by('-rating_determined_on')[0:6])
if scan_type in ["Strict-Transport-Security", "X-Content-Type-Options", "X-Frame-Options", "X-XSS-Protection",
"plain_https", "ftp", 'tls_qualys_certificate_trusted', 'tls_qualys_encryption_quality']:
if scan_type in ENDPOINT_SCAN_TYPES:
scans = list(EndpointGenericScan.objects.filter(
type=scan_type,
endpoint__url__organization__type=get_organization_type(organization_type),
endpoint__url__organization__country=get_country(country)
).order_by('-rating_determined_on')[0:6])
if scan_type in ['DNSSEC']:
if scan_type in URL_SCAN_TYPES:
scans = list(UrlGenericScan.objects.filter(
type=scan_type,
url__organization__type=get_organization_type(organization_type),
......@@ -1618,12 +1601,12 @@ def latest_scans(request, scan_type, country: str = "NL", organization_type="mun
for scan in scans:
calculation = get_calculation(scan)
if scan_type in ['DNSSEC']:
if scan_type in URL_SCAN_TYPES:
# url scans
dataset["scans"].append({
"url": scan.url.url,
"service": "%s" % scan.url.url,
"protocol": "DNSSEC",
"protocol": scan_type,
"port": "-",
"ip_version": "-",
"explanation": calculation.get("explanation", ""),
......@@ -1946,11 +1929,10 @@ class LatestScanFeed(Feed):
# second parameter via magic
def items(self, scan_type):
# print(scan_type)
if scan_type in ["Strict-Transport-Security", "X-Content-Type-Options", "X-Frame-Options", "X-XSS-Protection",
"plain_https", "ftp", "tls_qualys_certificate_trusted", "tls_qualys_encryption_quality"]:
if scan_type in ENDPOINT_SCAN_TYPES:
return EndpointGenericScan.objects.filter(type=scan_type).order_by('-last_scan_moment')[0:30]
if scan_type in ["DNSSEC"]:
if scan_type in URL_SCAN_TYPES:
return UrlGenericScan.objects.filter(type=scan_type).order_by('-last_scan_moment')[0:30]
return TlsQualysScan.objects.order_by('-last_scan_moment')[0:30]
......
import logging
from failmap.app.management.commands._private import VerifyTaskCommand
from failmap.scanners.scanner import dns, ftp, http
log = logging.getLogger(__name__)
class Command(VerifyTaskCommand):
"""
Changes scan types with weird characters into a slug that can also be used in javascript.
This fixes a mismatch between these two worlds: and stops you thinking what scan is what.
"""
help = __doc__
def handle(self, *args, **options):
from failmap.scanners.models import EndpointGenericScan
scans = EndpointGenericScan.objects.all().filter(
type__in=['Strict-Transport-Security', 'X-Content-Type-Options', 'X-Frame-Options', 'X-XSS-Protection'])
for scan in scans:
old_type = scan.type
new_type = "http_security_header_%s" % str(old_type).lower().replace("-", "_")
log.debug("Old: %s, New: %s" % (old_type, new_type))
scan.type = new_type
scan.save(update_fields=['type'])
......@@ -71,6 +71,7 @@ def compose_task(
@app.task(queue="storage")
def analyze_headers(result: requests.Response, endpoint):
# todo: remove code paths, and make a more clear case per header type. That's easier to understand edge cases.
# todo: Content-Security-Policy, Referrer-Policy
# if scan task failed, ignore the result (exception) and report failed status
......@@ -144,11 +145,11 @@ def analyze_headers(result: requests.Response, endpoint):
else:
if 'Strict-Transport-Security' in response.headers:
log.debug('Has Strict-Transport-Security')
store_endpoint_scan_result('Strict-Transport-Security', endpoint, 'True',
store_endpoint_scan_result('http_security_header_strict_transport_security', endpoint, 'True',
response.headers['Strict-Transport-Security'])
else:
log.debug('Has no Strict-Transport-Security, yet offers no insecure http service.')
store_endpoint_scan_result('Strict-Transport-Security', endpoint, 'False',
store_endpoint_scan_result('http_security_header_strict_transport_security', endpoint, 'False',
"Security Header not present: Strict-Transport-Security, "
"yet offers no insecure http service.")
......@@ -157,46 +158,39 @@ def analyze_headers(result: requests.Response, endpoint):
def generic_check(endpoint: Endpoint, headers, header):
# this is case insensitive
scan_type = "http_security_header_%s" % header.lower().replace("-", "_")
if header in headers.keys():
log.debug('Has %s' % header)
store_endpoint_scan_result(header, endpoint, 'True', headers[header])
store_endpoint_scan_result(scan_type, endpoint, 'True', headers[header])
else:
log.debug('Has no %s' % header)
store_endpoint_scan_result(header, endpoint, 'False', "Security Header not present: %s" % header)
store_endpoint_scan_result(scan_type, endpoint, 'False', "Security Header not present: %s" % header)
def generic_check_using_csp_fallback(endpoint: Endpoint, headers, header):
scan_type = "http_security_header_%s" % header.lower().replace("-", "_")
# this is case insensitive
if header in headers.keys():
log.debug('Has %s' % header)
store_endpoint_scan_result(header, endpoint, 'True', headers[header])
store_endpoint_scan_result(scan_type, endpoint, 'True', headers[header])
else:
# CSP fallback:
if "Content-Security-Policy" in headers.keys():
store_endpoint_scan_result(
header, endpoint, 'Using CSP',
scan_type, endpoint, 'Using CSP',
"Content-Security-Policy header found, which can handle the security from %s. Value: %s." %
(header, headers["Content-Security-Policy"]))
else:
log.debug('Has no %s' % header)
store_endpoint_scan_result(
header, endpoint, 'False',
scan_type, endpoint, 'False',
"Security Header not present: %s, alternative header Content-Security-Policy not present." % header)
def error_response_400_500(endpoint):
# Set all headers for this endpoint to 400_500, which are not shown in the report.
# These are not shown in the report anymore. Not using this
store_endpoint_scan_result('X-XSS-Protection', endpoint, '400_500', "")
store_endpoint_scan_result('X-Frame-Options', endpoint, '400_500', "")
store_endpoint_scan_result('X-Content-Type-Options', endpoint, '400_500', "")
if endpoint.protocol == "https":
store_endpoint_scan_result('Strict-Transport-Security', endpoint, '400_500', "")
@app.task(bind=True, default_retry_delay=1, retry_kwargs={'max_retries': 3})
def get_headers(self, uri_uri):
try:
......
"""
A list of all scan types that are reliable and can be used in production environments.
A list of all scan types that are reliable and can be used in production environments (reporting, etc).
"""
ENDPOINT_SCAN_TYPES = [
'Strict-Transport-Security',
'X-Content-Type-Options',
'X-Frame-Options',
'X-XSS-Protection',
'http_security_header_strict_transport_security',
'http_security_header_x_content_type_options',
'http_security_header_x_frame_options',
'http_security_header_x_xss_protection',
'plain_https',
'ftp',
'tls_qualys_certificate_trusted',
......
......@@ -439,19 +439,19 @@ msgid "report_header_plain_https"
msgstr ""
#: static/js/script.js
msgid "report_header_security_headers_x_xss_protection"
msgid "report_header_http_security_header_x_xss_protection"
msgstr ""
#: static/js/script.js
msgid "report_header_security_headers_x_frame_options"
msgid "report_header_http_security_header_x_frame_options"
msgstr ""
#: static/js/script.js
msgid "report_header_security_headers_x_content_type_options"
msgid "report_header_http_security_header_x_content_type_options"
msgstr ""
#: static/js/script.js
msgid "report_header_security_headers_strict_transport_security"
msgid "report_header_http_security_header_strict_transport_security"
msgstr ""
#: static/js/script.js
......
......@@ -439,19 +439,19 @@ msgid "report_header_plain_https"
msgstr ""
#: static/js/script.js
msgid "report_header_security_headers_x_xss_protection"
msgid "report_header_http_security_header_x_xss_protection"
msgstr ""
#: static/js/script.js
msgid "report_header_security_headers_x_frame_options"
msgid "report_header_http_security_header_x_frame_options"
msgstr ""
#: static/js/script.js
msgid "report_header_security_headers_x_content_type_options"
msgid "report_header_http_security_header_x_content_type_options"
msgstr ""
#: static/js/script.js
msgid "report_header_security_headers_strict_transport_security"
msgid "report_header_http_security_header_strict_transport_security"
msgstr ""
#: static/js/script.js
......
......@@ -439,19 +439,19 @@ msgid "report_header_plain_https"
msgstr ""
#: static/js/script.js
msgid "report_header_security_headers_x_xss_protection"
msgid "report_header_http_security_header_x_xss_protection"
msgstr ""
#: static/js/script.js
msgid "report_header_security_headers_x_frame_options"
msgid "report_header_http_security_header_x_frame_options"
msgstr ""
#: static/js/script.js
msgid "report_header_security_headers_x_content_type_options"
msgid "report_header_http_security_header_x_content_type_options"
msgstr ""
#: static/js/script.js
msgid "report_header_security_headers_strict_transport_security"
msgid "report_header_http_security_header_strict_transport_security"
msgstr ""
#: static/js/script.js
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment