Verified Commit 20722d26 authored by Elger Jonker's avatar Elger Jonker

Fixing known subdomain scans, made worker-safe

parent 110cbcc5
Pipeline #37340663 passed with stages
in 19 minutes and 25 seconds
......@@ -199,6 +199,9 @@ class DiscoverTaskCommand(TaskCommand):
self.mutual_group.add_argument('-o', '--organization_names', nargs='*',
help="Perform scans on these organizations (default is all).")
self.mutual_group.add_argument('-y', '--organization_type', nargs='*',
help="Perform scans on these organization types (default is all).")
def compose(self, *args, **options):
"""Compose set of tasks based on provided arguments."""
......@@ -210,6 +213,11 @@ class DiscoverTaskCommand(TaskCommand):
regex = '^(' + '|'.join(options['organization_names']) + ')$'
organization_filter = {'name__iregex': regex}
if options['organization_type']:
# create a case-insensitive filter to match organizations by name
regex = '^(' + '|'.join(options['organization_type']) + ')$'
organization_filter = {'type__name__iregex': regex}
# compose set of tasks to be executed
return self.scanner_module.compose_discover_task(organization_filter)
......
import logging
from failmap.app.management.commands._private import DiscoverTaskCommand
from failmap.scanners.scanner import dns, ftp, http
from failmap.scanners.scanner import dns, dns_known_subdomains, ftp, http
log = logging.getLogger(__name__)
......@@ -16,6 +16,7 @@ class Command(DiscoverTaskCommand):
'ftp': ftp,
'http': http,
'subdomains': dns,
'known_subdomains': dns_known_subdomains
}
def add_arguments(self, parser):
......
......@@ -19,7 +19,7 @@ def valid_organization(name):
if name in ["_ALL_", "*"]:
return "*"
try:
o = Organization.objects.get(name=name)
o = Organization.objects.get(name__iexact=name)
return o.name
except ObjectDoesNotExist:
raise argparse.ArgumentTypeError("%s is not a valid organization or _ALL_" % name)
This diff is collapsed.
"""
Performs a range of DNS scans:
- Using Search engines
- Using Wordlists
- Using Certificate Transparency
- Using NSEC
It separates the scans as it might be desirable to use different scanners.
Todo: the list of known subdomains might help (a lot) with breaking nsec3 hashes?
https://github.com/anonion0/nsec3map
"""
# todo: if ScannerHttp.has_internet_connection():
# todo: language matters, many of the NL subdomains don't make sense in other countries.
import logging
from celery import Task, group
from failmap.scanners.scanner.dns import get_subdomains, url_by_filters, wordlist_scan
from failmap.scanners.scanner.scanner import allowed_to_discover
log = logging.getLogger(__package__)
def compose_discover_task(organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(), **kwargs) -> Task:
if not allowed_to_discover("brute_known_subdomains_compose_task"):
return group()
urls = url_by_filters(organizations_filter=organizations_filter,
urls_filter=urls_filter,
endpoints_filter=endpoints_filter)
# a heuristic
if not urls:
log.info("Did not get any urls to discover known subdomains.")
return group()
log.debug("Going to scan subdomains for the following %s urls." % len(urls))
first_url = urls[0]
first_organization = first_url.organization.all().first()
# The country is more then enough to get a sort of feasible list of subdomains.
wordlist = get_subdomains([first_organization.country], None)
# The worker has no way to write / save things. A wordlist can be 10's of thousands of words.
task = group(wordlist_scan.si([url], wordlist) for url in urls)
return task
......@@ -65,10 +65,10 @@ def allowed_to_discover(scanner_name: str = ""):
return config.DISCOVER_URLS_USING_NSEC
if scanner_name == 'certificate_transparency_compose_task':
return config.DISCOVER_URLS_USING_KNOWN_SUBDOMAINS
return config.DISCOVER_URLS_USING_CERTIFICATE_TRANSPARENCY
if scanner_name == 'brute_known_subdomains_compose_task':
return config.DISCOVER_URLS_USING_CERTIFICATE_TRANSPARENCY
return config.DISCOVER_URLS_USING_KNOWN_SUBDOMAINS
return False
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment