Verified Commit 13f7b13e authored by Elger Jonker's avatar Elger Jonker

added verification commands for ftp and http(s)

parent 9b167766
......@@ -166,14 +166,52 @@ Which results in:
Perform scans on these urls (default is all).
Starting a development server
Reporting
---------
Reports can be updated with the following command:
.. code-block:: bash
failmap report
This command also takes in the organization and url filters. Filtering always updates the entire organization over the entire
timespan. This means it can take a while before the command has finished.
.. code-block:: bash
failmap report -o Arnhem
See failmap help report for more info.
Endpoint discovery and verification
-----------------------------------
.. code-block:: bash
failmap discover http -o Texel
failmap verify http -u www.texel.nl
failmap verify ftp -o Apeldoorn
See failmap help verify
Subdomain discovery
-------------------
(doesn't work atm)
.. code-block:: bash
failmap discover subdomains -o Texel
Running a development server
-----------------------------
To start a development server, that does not tamper with any data, and accepts connections from anywhere (which are then
filtered by settings.py), run:
.. code-block:: bash
failmap devserver --no-backend --no-data 0.0.0.0:8000
......
......@@ -177,9 +177,12 @@ class ScannerTaskCommand(TaskCommand):
endpoints_filter=endpoints_filter)
# for discovery of services only
class DiscoverTaskCommand(TaskCommand):
"""Generic Task Command for scanners."""
"""Discovery command, launces tasks that can discover services.
These tasks can take a lot of time as they might encounter many network timeout for non existing adresses.
If you just want to update the current set of information in the database, run a verification task. That will be
much faster (but will not discover anything new). The ratio between discovery and verification can be 1:7
"""
scanner_module = None
......@@ -201,3 +204,39 @@ class DiscoverTaskCommand(TaskCommand):
# compose set of tasks to be executed
return self.scanner_module.compose_discover_task(organization_filter)
class VerifyTaskCommand(TaskCommand):
"""Generic Task Command for scanners."""
scanner_module = None
def _add_arguments(self, parser):
"""Add command specific arguments."""
self.mutual_group.add_argument('-o', '--organization_names', nargs='*',
help="Perform scans on these organizations (default is all).")
self.mutual_group.add_argument('-u', '--url_addresses', nargs='*',
help="Perform scans on these urls (default is all).")
def compose(self, *args, **options):
"""Compose set of tasks based on provided arguments."""
# by default no filter means all of each.
organizations_filter = dict()
urls_filter = dict()
endpoints_filter = dict() # A choice of ports, ip-version and protocol
if options['organization_names']:
# create a case-insensitive filter to match organizations by name
regex = '^(' + '|'.join(options['organization_names']) + ')$'
organizations_filter = {'name__iregex': regex}
if options['url_addresses']:
# create a case-insensitive filter to match organizations by name
regex = '^(' + '|'.join(options['url_addresses']) + ')$'
urls_filter = {'url__iregex': regex}
# compose set of tasks to be executed
return self.scanner_module.compose_verify_task(organizations_filter=organizations_filter,
urls_filter=urls_filter, endpoints_filter=endpoints_filter)
from failmap.app.management.commands._private import TaskCommand
from ...rating import compose_task
class Command(TaskCommand):
"""Remove all organization and url ratings, then rebuild them from scratch."""
help = __doc__
task = compose_task()
from failmap.app.management.commands._private import ScannerTaskCommand
from ... import rating
class Command(ScannerTaskCommand):
"""Remove all organization and url ratings, then rebuild them from scratch."""
help = __doc__
def handle(self, *args, **options):
self.scanner_module = rating
return super().handle(self, *args, **options)
......@@ -39,6 +39,8 @@ def compose_task(
if endpoints_filter:
raise NotImplementedError('This scanner does not work on a endpoint level.')
log.info(organizations_filter)
# Only displayed configurations are reported. Because why have reports on things you don't display?
# apply filter to organizations (or if no filter, all organizations)
organizations = Organization.objects.filter(q_configurations_to_display('organization'), **organizations_filter)
......
import logging
from failmap.app.management.commands._private import ScannerTaskCommand
from failmap.scanners.scanner import (dnssec, dummy, ftp, http, onboard, plain_http, screenshot,
security_headers, tls_osaft, tls_qualys, debug)
from failmap.scanners.scanner import (debug, dnssec, dummy, ftp, http, onboard, plain_http,
screenshot, security_headers, tls_osaft, tls_qualys)
log = logging.getLogger(__name__)
......
import logging
from failmap.app.management.commands._private import VerifyTaskCommand
from failmap.scanners.scanner import dns, ftp, http
log = logging.getLogger(__name__)
class Command(VerifyTaskCommand):
"""Can perform a host of scans. Run like: failmap scan [scanner_name] and then options."""
help = __doc__
# todo: subdomains, from scanner.dns
scanners = {
'ftp': ftp,
'http': http,
}
def add_arguments(self, parser):
parser.add_argument('scanner', nargs=1, help='The scanner you want to use.', choices=self.scanners)
super().add_arguments(parser)
def handle(self, *args, **options):
if options['scanner'][0] not in self.scanners:
print("Scanner does not exist. Please specify a scanner: %s " % self.scanners.keys())
return
self.scanner_module = self.scanners[options['scanner'][0]]
return super().handle(self, *args, **options)
import logging
from ftplib import FTP, FTP_TLS, error_perm, error_proto, error_reply, error_temp
from celery import Task, group
from django.utils import timezone
from constance import config
from failmap.celery import ParentFailed, app
from failmap.organizations.models import Url
from failmap.celery import app
from failmap.scanners.models import Endpoint
from failmap.scanners.scanmanager.endpoint_scan_manager import EndpointScanManager
from failmap.scanners.scanner.scanner import (allowed_to_scan, endpoint_filters,
q_configurations_to_scan, url_filters)
from constance import config
from failmap.scanners.scanner.scanner import endpoint_filters, q_configurations_to_scan
log = logging.getLogger(__name__)
......@@ -23,7 +18,8 @@ def compose_task(
) -> Task:
"""
Helps with identifying issues with scanners. It shows the relevant permissions, configurations and lists the
organizations, urls and endpoints in a convenient way. This can only run in direct mode and will not result in tasks.
organizations, urls and endpoints in a convenient way. This can only run in direct mode and will not result in
tasks.
All messages are returned as log messages.
:param organizations_filter:
......
......@@ -98,6 +98,24 @@ def compose_discover_task(organizations_filter: dict = dict(), urls_filter: dict
return task
def compose_verify_task(organizations_filter: dict = dict(), urls_filter: dict = dict(),
endpoints_filter: dict = dict()) -> Task:
default_filter = {"protocol": "ftp"}
endpoints_filter = {**endpoints_filter, **default_filter}
endpoints = Endpoint.objects.all().filter(q_configurations_to_scan(level='endpoint'), **endpoints_filter)
endpoints = endpoint_filters(endpoints, organizations_filter, urls_filter, endpoints_filter)
task = group(
# first iterate through ports, so there is more time between different connection attempts. Which reduces load
# for the tested server. Also, the first port has the most hits :)
discover.si(endpoint.url.url, endpoint.port) | store_when_new_or_kill_if_gone.s(
endpoint.url, endpoint.port, 'ftp', 4) for endpoint in endpoints
)
return task
@app.task(queue='storage')
def store(result: dict, endpoint: Endpoint):
"""
......@@ -280,7 +298,7 @@ def scan(self, address: str, port: int):
# tries and discover FTP servers by A) trying to open an FTP connection to a standard port.
# it will do on all known urls, and try a range of well known FTP ports and alternative ports.
@app.task(queue='scanners')
def discover(url, port):
def discover(url: str, port: int):
# https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers
connected = False
......
......@@ -42,9 +42,10 @@ from requests.exceptions import ConnectionError
from failmap.celery import app
from failmap.organizations.models import Organization, Url
from failmap.scanners.models import Endpoint, UrlIp
from failmap.scanners.scanner.scanner import allowed_to_discover, q_configurations_to_scan
from failmap.scanners.scanner.scanner import allowed_to_discover, q_configurations_to_scan, endpoint_filters
from failmap.scanners.timeout import timeout
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
log = logging.getLogger(__package__)
......@@ -114,6 +115,33 @@ def compose_discover_task(
return group(tasks)
@app.task(queue="storage")
def compose_verify_task(
organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),
) -> Task:
"""Verifies existing https and http endpoints. Is pretty quick, as it will not stumble upon non-existing services
as much.
"""
if not allowed_to_discover("scanner_http"):
return group()
default_filter = {"protocol__in": ["https", "http"]}
endpoints_filter = {**endpoints_filter, **default_filter}
endpoints = Endpoint.objects.all().filter(q_configurations_to_scan(level='endpoint'), **endpoints_filter)
endpoints = endpoint_filters(endpoints, organizations_filter, urls_filter, endpoints_filter)
tasks = []
for endpoint in endpoints:
tasks.append(can_connect.si(protocol=endpoint.protocol, url=endpoint.url,
port=endpoint.port, ip_version=endpoint.ip_version)
| connect_result.s(protocol=endpoint.protocol, url=endpoint.url,
port=endpoint.port, ip_version=endpoint.ip_version))
return group(tasks)
@app.task(queue="storage")
def url_lives(ips, url):
"""
......
"""Generic Types for type hinting."""
from .celery import Task
import datetime
def compose_task(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment