[WIP] FTP scanning

parent 36a1438f
Pipeline #25422070 failed with stage
in 14 minutes and 8 seconds
......@@ -156,3 +156,29 @@ class ScannerTaskCommand(TaskCommand):
# compose set of tasks to be executed
return self.scanner_module.compose_task(organization_filter)
# for discovery of services only
class DiscoverTaskCommand(TaskCommand):
"""Generic Task Command for scanners."""
scanner_module = None
def _add_arguments(self, parser):
"""Add command specific arguments."""
self.mutual_group.add_argument('-o', '--organization_names', nargs='*',
help="Perform scans on these organizations (default is all).")
def compose(self, *args, **options):
"""Compose set of tasks based on provided arguments."""
if not options['organization_names']:
# by default no filter means all organizations
organization_filter = dict()
else:
# create a case-insensitive filter to match organizations by name
regex = '^(' + '|'.join(options['organization_names']) + ')$'
organization_filter = {'name__iregex': regex}
# compose set of tasks to be executed
return self.scanner_module.compose_discover_task(organization_filter)
......@@ -151,6 +151,30 @@ def http_plain_rating_based_on_scan(scan):
return calculation
def ftp_rating_based_on_scan(scan):
# outdated, insecure
high = 0
# changed the ratings in the database. They are not really correct.
# When there is no https at all, it's worse than having broken https. So rate them the same.
if scan.rating == "outdated" or scan.rating == "insecure":
high += 1
# also here: the last scan moment increases with every scan. When you have a set of
# relevant dates (when scans where made) ....
calculation = {
"type": "ftp",
"explanation": scan.explanation,
"since": scan.rating_determined_on.isoformat(),
"last_scan": scan.last_scan_moment.isoformat(),
"high": high,
"medium": 0,
"low": 0,
}
return calculation
def dnssec_rating_based_on_scan(scan):
"""
See: https://en.wikipedia.org/wiki/Domain_Name_System_Security_Extensions
......@@ -237,5 +261,6 @@ calculation_methods = {
'X-XSS-Protection': security_headers_rating_based_on_scan,
'plain_https': http_plain_rating_based_on_scan,
'tls_qualys': tls_qualys_rating_based_on_scan,
'DNSSEC': dnssec_rating_based_on_scan
'DNSSEC': dnssec_rating_based_on_scan,
'ftp': ftp_rating_based_on_scan
}
......@@ -192,6 +192,8 @@ def significant_moments(organizations: List[Organization]=None, urls: List[Url]=
allowed_to_report.append("X-Content-Type-Options")
if config.REPORT_INCLUDE_DNS_DNSSEC:
allowed_to_report.append("DNSSEC")
if config.REPORT_INCLUDE_FTP:
allowed_to_report.append("ftp")
generic_scans = EndpointGenericScan.objects.all().filter(type__in=allowed_to_report, endpoint__url__in=urls).\
prefetch_related("endpoint").defer("endpoint__url")
......@@ -416,7 +418,7 @@ def rate_timeline(timeline, url: Url):
previous_endpoints.remove(dead_endpoint)
endpoint_scan_types = ["Strict-Transport-Security", "X-Content-Type-Options", "X-Frame-Options",
"X-XSS-Protection", "tls_qualys", "plain_https"]
"X-XSS-Protection", "tls_qualys", "plain_https", "ftp"]
for endpoint in relevant_endpoints:
url_was_once_rated = True
......@@ -429,7 +431,7 @@ def rate_timeline(timeline, url: Url):
these_endpoint_scans['tls_qualys'] = scan
if isinstance(scan, EndpointGenericScan):
if scan.type in ['Strict-Transport-Security', 'X-Content-Type-Options',
'X-Frame-Options', 'X-XSS-Protection', 'plain_https']:
'X-Frame-Options', 'X-XSS-Protection', 'plain_https', 'ftp']:
these_endpoint_scans[scan.type] = scan
# enrich the ratings with previous ratings, without overwriting them.
......@@ -635,6 +637,10 @@ def show_timeline_console(timeline, url: Url):
if item.type == "X-XSS-Protection":
calculation = get_calculation(item)
message += "| | |- %5s points: %s" % (calculation.high, item) + newline
for item in timeline[moment]['generic_scan']['scans']:
if item.type == "ftp":
calculation = get_calculation(item)
message += "| | |- %5s points: %s" % (calculation.high, item) + newline
if 'dead' in timeline[moment]:
message += "| |- dead endpoints" + newline
......@@ -871,7 +877,7 @@ def get_url_score_modular(url: Url, when: datetime=None):
continue
scan_types = ["Strict-Transport-Security", "X-Content-Type-Options", "X-Frame-Options", "X-XSS-Protection",
"tls_qualys", "plain_https"]
"tls_qualys", "plain_https", "ftp"]
calculations = []
for scan_type in scan_types:
......@@ -974,6 +980,9 @@ def endpoint_to_points_and_calculation(endpoint: Endpoint, when: datetime, scan_
if scan_type == "plain_https":
scan = EndpointGenericScan.objects.filter(endpoint=endpoint, rating_determined_on__lte=when,
type="plain_https").latest('rating_determined_on')
if scan_type == "ftp":
scan = EndpointGenericScan.objects.filter(endpoint=endpoint, rating_determined_on__lte=when,
type="ftp").latest('rating_determined_on')
if scan_type == "tls_qualys":
scan = TlsQualysScan.objects.filter(endpoint=endpoint, rating_determined_on__lte=when
).latest('rating_determined_on')
......
......@@ -566,6 +566,7 @@ function views() {
self.d3stats.stacked_area_chart("graph_total", error, data.total);
self.d3stats.stacked_area_chart("graph_tls_qualys", error, data.tls_qualys);
self.d3stats.stacked_area_chart("graph_plain_https", error, data.plain_https);
self.d3stats.stacked_area_chart("graph_ftp", error, data.ftp);
self.d3stats.stacked_area_chart("graph_security_headers_strict_transport_security", error, data.security_headers_strict_transport_security);
self.d3stats.stacked_area_chart("graph_security_headers_x_frame_options", error, data.security_headers_x_frame_options);
self.d3stats.stacked_area_chart("graph_security_headers_x_content_type_options", error, data.security_headers_x_content_type_options);
......@@ -881,6 +882,13 @@ function views() {
data: {scan: "plain_https"}
});
window.vueLatestFtp = new Vue({
name: "LatestFtp",
mixins: [latest_mixin, state_mixin],
el: '#latest_ftp',
data: {scan: "ftp"}
});
window.vueLatestHSTS = new Vue({
name: "LatestHSTS",
mixins: [latest_mixin, state_mixin],
......@@ -945,7 +953,8 @@ function views() {
security_headers_x_frame_options: false,
security_headers_x_xss_protection: false,
tls_qualys: false,
plain_https: false
plain_https: false,
ftp: false
},
computed: {
visibleweek: function () {
......@@ -965,6 +974,7 @@ function views() {
'"tls_qualys": "' + this.tls_qualys + '", ' +
'"security_headers_x_content_type_options": "' + this.security_headers_x_content_type_options + '", ' +
'"security_headers_x_xss_protection": "' + this.security_headers_x_xss_protection + '", ' +
'"ftp": "' + this.ftp + '", ' +
'"plain_https": "' + this.plain_https + '"}'
}
......@@ -993,6 +1003,9 @@ function views() {
plain_https: function(newsetting, oldsetting){
this.load(this.week)
},
ftp: function(newsetting, oldsetting){
this.load(this.week)
},
category: function (newCategory, oldCategory) {
if (newCategory === oldCategory)
......@@ -1026,6 +1039,7 @@ function views() {
vueTopwin.set_state(this.country, this.category);
vueStatistics.set_state(this.country, this.category);
vueLatestPlainHttps.set_state(this.country, this.category);
vueLatestFtp.set_state(this.country, this.category);
vueLatestTlsQualys.set_state(this.country, this.category);
vueLatestXContentTypeOptions.set_state(this.country, this.category);
vueLatestHSTS.set_state(this.country, this.category);
......@@ -1202,6 +1216,7 @@ function views() {
security_headers_x_xss_protection: {high: 0, medium:0, low: 0},
security_headers_x_frame_options: {high: 0, medium:0, low: 0},
plain_https: {high: 0, medium:0, low: 0},
ftp: {high: 0, medium:0, low: 0},
overall: {high: 0, medium:0, low: 0}
},
......@@ -1225,6 +1240,7 @@ function views() {
self.security_headers_x_xss_protection = {high: 0, medium:0, low: 0};
self.security_headers_x_frame_options = {high: 0, medium:0, low: 0};
self.plain_https = {high: 0, medium:0, low: 0};
self.ftp = {high: 0, medium:0, low: 0};
self.overall = {high: 0, medium:0, low: 0}
} else {
self.data = data;
......@@ -1240,6 +1256,8 @@ function views() {
self.security_headers_x_frame_options = data.security_headers_x_frame_options.improvements;
if (data.plain_https !== undefined)
self.plain_https = data.plain_https.improvements;
if (data.ftp !== undefined)
self.ftp = data.ftp.improvements;
if (data.overall !== undefined)
self.overall = data.overall.improvements;
}
......
......@@ -545,6 +545,32 @@
</td>
</tr>
{% endif %}
{% if config.SHOW_FTP %}
{% verbatim %}
<tr v-if="x['explained']['ftp']">
<td colspan="3">{% endverbatim %}{% trans "FTP" %}{% verbatim %}</td>
</tr>
<tr v-if="x['explained']['ftp']" class="redrow">
<td></td>
<td>{% endverbatim %}{% trans "FTP insecure" %}{% verbatim %}:</td>
<td>{{ x['explained']['ftp']['FTP Server does not support encrypted transport or has protocol issues.'] }}
</td>
</tr>
<tr v-if="x['explained']['ftp']" class="redrow">
<td></td>
<td>{% endverbatim %}{% trans "FTP outdated" %}{% verbatim %}:</td>
<td>{{ x['explained']['ftp']['FTP Server only supports insecure SSL protocol.'] }}
</td>
</tr>
<tr v-if="x['explained']['ftp']" class="greenrow">
<td></td>
<td>{% endverbatim %}{% trans "FTP secure" %}{% verbatim %}:</td>
<td>{{ x['explained']['ftp']['FTP Server supports TLS encryption protocol.'] }}{% endverbatim %}
</td>
</tr>
{% endif %}
{% if config.SHOW_HTTP_MISSING_TLS %}
{% verbatim %}
<tr v-if="x['explained']['plain_https']">
......@@ -791,6 +817,10 @@
{% if config.SHOW_HTTP_MISSING_TLS %}<td>{% trans "Missing encryption" %}</td>{% endif %}
{% if config.SHOW_HTTP_MISSING_TLS %}<td :class="'number-sm ' + goodbad(plain_https.high)">{% verbatim %}{{ plain_https.high }}{% endverbatim %}</td>{% endif %}
</tr>
<tr>
{% if config.SHOW_FTP %}<td>{% trans "Missing FTP encryption" %}</td>{% endif %}
{% if config.SHOW_FTP %}<td :class="'number-sm ' + goodbad(ftp.high)">{% verbatim %}{{ ftp.high }}{% endverbatim %}</td>{% endif %}
</tr>
</table>
</div>
......@@ -1252,6 +1282,9 @@
{% if config.SHOW_HTTP_MISSING_TLS %}
<input type='checkbox' v-model="plain_https" name="plain_https" id="plain_https"> <label for="plain_https">{% trans "report_header_plain_https" %}</label><br />
{% endif %}
{% if config.SHOW_FTP %}
<input type='checkbox' v-model="ftp" name="ftp" id="ftp"> <label for="ftp">{% trans "report_header_ftp" %}</label><br />
{% endif %}
{% if config.SHOW_HTTP_HEADERS_HSTS %}
<input type='checkbox' v-model="security_headers_strict_transport_security" name="HSTS" id="HSTS"> <label for="HSTS">{% trans "report_header_security_headers_strict_transport_security" %}</label><br />
{% endif %}
......
......@@ -1253,7 +1253,8 @@ def map_data(request, country: str="NL", organization_type: str="municipality",
"security_headers_x_frame_options",
"security_headers_x_xss_protection",
"tls_qualys",
"plain_https"]
"plain_https",
"ftp"]
# todo: add try except for standard json errors.
# this is a vulnerability, so we try to contain it
......@@ -1468,7 +1469,7 @@ def latest_scans(request, scan_type, country: str="NL", organization_type="munic
if scan_type not in ["tls_qualys",
"Strict-Transport-Security", "X-Content-Type-Options", "X-Frame-Options", "X-XSS-Protection",
"plain_https"]:
"plain_https", "ftp"]:
return empty_response()
if scan_type == "tls_qualys":
......@@ -1478,7 +1479,7 @@ def latest_scans(request, scan_type, country: str="NL", organization_type="munic
).order_by('-rating_determined_on')[0:6])
if scan_type in ["Strict-Transport-Security", "X-Content-Type-Options", "X-Frame-Options", "X-XSS-Protection",
"plain_https"]:
"plain_https", "ftp"]:
scans = list(EndpointGenericScan.objects.filter(
type=scan_type,
endpoint__url__organization__type=get_organization_type(organization_type),
......@@ -1651,7 +1652,7 @@ class LatestScanFeed(Feed):
def items(self, scan_type):
# print(scan_type)
if scan_type in ["Strict-Transport-Security", "X-Content-Type-Options", "X-Frame-Options", "X-XSS-Protection",
"plain_https"]:
"plain_https", "ftp"]:
return EndpointGenericScan.objects.filter(type=scan_type).order_by('-last_scan_moment')[0:30]
return TlsQualysScan.objects.order_by('-last_scan_moment')[0:30]
......
......@@ -52,7 +52,10 @@ class Command(DumpDataCommand):
"scanners.Endpoint",
"scanners.TlsQualysScan",
"scanners.EndpointGenericScan",
"scanners.UrlIp"
"scanners.UrlGenericScan",
"scanners.UrlIp",
"map.Configuration",
"map.AdministrativeRegion"
)
def handle(self, *app_labels, **options):
......
......@@ -40,7 +40,7 @@ class UrlIpAdmin(ImportExportModelAdmin, admin.ModelAdmin):
class EndpointAdmin(ImportExportModelAdmin, admin.ModelAdmin):
list_display = ('id', 'url', 'visit', 'discovered_on', 'ip_version', 'port', 'protocol', 'is_dead', 'is_dead_since',
'tls_scans', 'generic_scans')
'tls_scans', 'endpoint_generic_scans', 'url_generic_scans')
search_fields = ('url__url', 'ip_version', 'port', 'protocol', 'is_dead',
'is_dead_since', 'is_dead_reason')
list_filter = ('ip_version', 'port', 'protocol', 'is_dead', 'is_dead_reason', 'discovered_on')
......@@ -60,9 +60,13 @@ class EndpointAdmin(ImportExportModelAdmin, admin.ModelAdmin):
return TlsQualysScan.objects.filter(endpoint=inst.id).count()
@staticmethod
def generic_scans(inst):
def endpoint_generic_scans(inst):
return EndpointGenericScan.objects.filter(endpoint_id=inst.id).count()
@staticmethod
def url_generic_scans(inst):
return UrlGenericScan.objects.filter(url__endpoint=inst.id).count()
@staticmethod
def visit(inst):
url = "%s://%s:%s/" % (inst.protocol, inst.url.url, inst.port)
......@@ -169,7 +173,7 @@ class EndpointGenericScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
)
fields = ('endpoint', 'type', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on')
'explanation', 'evidence', 'last_scan_moment', 'rating_determined_on')
readonly_fields = ['last_scan_moment', 'endpoint']
......
import logging
from failmap.app.management.commands._private import DiscoverTaskCommand
from failmap.scanners import scanner_ftp
log = logging.getLogger(__name__)
class Command(DiscoverTaskCommand):
"""Can perform a host of scans. Run like: failmap scan [scanner_name] and then options."""
help = __doc__
def add_arguments(self, parser):
parser.add_argument('scanner', nargs=1, help='The scanner you want to use.')
super().add_arguments(parser)
def handle(self, *args, **options):
scanners = {
'ftp': scanner_ftp
}
if options['scanner'][0] not in scanners:
print("Scanner does not exist. Please specify a scanner: %s " % scanners.keys())
return
self.scanner_module = scanners[options['scanner'][0]]
super().handle(self, *args, **options)
import logging
from failmap.app.management.commands._private import ScannerTaskCommand
from failmap.scanners import (scanner_dnssec, scanner_http, scanner_plain_http,
from failmap.scanners import (scanner_dnssec, scanner_ftp, scanner_http, scanner_plain_http,
scanner_security_headers, scanner_tls_qualys)
log = logging.getLogger(__name__)
......@@ -23,7 +23,8 @@ class Command(ScannerTaskCommand):
'headers': scanner_security_headers,
'plain': scanner_plain_http,
'endpoints': scanner_http,
'tls': scanner_tls_qualys
'tls': scanner_tls_qualys,
'ftp': scanner_ftp
}
if options['scanner'][0] not in scanners:
......
......@@ -23,6 +23,9 @@ def allowed_to_scan(scanner_name: str=""):
if not config.SCAN_AT_ALL:
return False
if scanner_name == 'scanner_ftp':
return config.SCAN_FTP
if scanner_name == 'scanner_plain_http':
return config.SCAN_HTTP_MISSING_TLS
......@@ -87,6 +90,11 @@ def q_configurations_to_scan(level: str='url'):
qs.add(Q(organization__type=configuration['organization_type'],
organization__country=configuration['country']), Q.OR)
if level == 'endpoint':
for configuration in configurations:
qs.add(Q(url__organization__type=configuration['organization_type'],
url__organization__country=configuration['country']), Q.OR)
return qs
......
This diff is collapsed.
......@@ -6,11 +6,11 @@ from celery.result import allow_join_result
from failmap.celery import app
from . import (scanner_dns, scanner_dnssec, scanner_dummy, scanner_http, scanner_plain_http,
scanner_security_headers, scanner_tls_qualys)
from . import (scanner_dns, scanner_dnssec, scanner_dummy, scanner_ftp, scanner_http,
scanner_plain_http, scanner_security_headers, scanner_tls_qualys)
# explicitly declare the imported modules as this modules 'content', prevents pyflakes issues
__all__ = [scanner_tls_qualys, scanner_security_headers, scanner_dummy, scanner_http, scanner_dnssec]
__all__ = [scanner_tls_qualys, scanner_security_headers, scanner_dummy, scanner_http, scanner_dnssec, scanner_ftp]
# This is the single source of truth regarding scanner configuration.
# Lists to be used elsewhere when tasks need to be composed, these lists contain compose functions.
......
......@@ -674,6 +674,7 @@ CONSTANCE_CONFIG = {
'SHOW_HTTP_HEADERS_XFO': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
'SHOW_HTTP_HEADERS_X_XSS': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
'SHOW_HTTP_HEADERS_X_CONTENT': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
'SHOW_FTP': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
# todo: schedule this once per week by default.
'DISCOVER_URLS_USING_NSEC': (True, 'Discover new domains using DNSSEC NSEC1 hashes? (See docs)', bool),
......@@ -692,10 +693,12 @@ CONSTANCE_CONFIG = {
'SCAN_HTTP_HEADERS_XFO': (True, 'Do you want to scan for missing X-Frame-Options headers?', bool),
'SCAN_HTTP_HEADERS_X_XSS': (True, 'Do you want to scan for missing X-XSS headers?', bool),
'SCAN_HTTP_HEADERS_X_CONTENT': (True, 'Do you want to scan for missing X-Content-Type issues?', bool),
'SCAN_FTP': (True, 'Do you want to scan for FTP servers that are missing encryption?', bool),
'CREATE_HTTP_SCREENSHOT': (True, 'Todo: Does not work yet! Do you want to create screenshots for HTTP endpoints?',
bool),
# future: FTP, TLS_QUICK (way less robust and complete, much faster)
'REPORT_INCLUDE_FTP': (True, 'Do you want to add FTP encryption issues to the report?', bool),
'REPORT_INCLUDE_DNS_DNSSEC': (True, 'Do you want to add DNSSEC issues to the report?', bool),
'REPORT_INCLUDE_HTTP_TLS_QUALYS': (True, 'Do you want to show TLS results in the report?', bool),
'REPORT_INCLUDE_HTTP_MISSING_TLS': (True, 'Do you want to show missing TLS in the report?', bool),
......@@ -736,7 +739,7 @@ CONSTANCE_CONFIG_FIELDSETS = OrderedDict([
'SHOW_STATS_IMPROVEMENTS', 'SHOW_STATS_NUMBERS', 'SHOW_SERVICES', 'SHOW_STATS_CHANGES', 'SHOW_TICKER',
'SHOW_DNS_DNSSEC', 'SHOW_HTTP_TLS_QUALYS', 'SHOW_HTTP_MISSING_TLS',
'SHOW_HTTP_HEADERS_HSTS', 'SHOW_HTTP_HEADERS_XFO', 'SHOW_HTTP_HEADERS_X_XSS',
'SHOW_HTTP_HEADERS_X_CONTENT'
'SHOW_HTTP_HEADERS_X_CONTENT', 'SHOW_FTP'
)),
('Discovery', ('DISCOVER_URLS_USING_NSEC', 'DISCOVER_URLS_USING_KNOWN_SUBDOMAINS',
......@@ -746,7 +749,7 @@ CONSTANCE_CONFIG_FIELDSETS = OrderedDict([
('Scanning', ('SCAN_AT_ALL', 'SCAN_DNS_DNSSEC', 'SCAN_HTTP_TLS_QUALYS', 'SCAN_HTTP_MISSING_TLS',
'SCAN_HTTP_HEADERS_HSTS',
'SCAN_HTTP_HEADERS_XFO', 'SCAN_HTTP_HEADERS_X_XSS', 'SCAN_HTTP_HEADERS_X_CONTENT',
'SCAN_HTTP_HEADERS_XFO', 'SCAN_HTTP_HEADERS_X_XSS', 'SCAN_HTTP_HEADERS_X_CONTENT', 'SCAN_FTP',
'CREATE_HTTP_SCREENSHOT')),
('Reporting', ('REPORT_INCLUDE_DNS_DNSSEC', 'REPORT_INCLUDE_HTTP_TLS_QUALYS', 'REPORT_INCLUDE_HTTP_MISSING_TLS',
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment