lint and debugging hunt results

.gitignore

@@ -28,4 +28,5 @@ vendor/dnsrecon/
 vendor/theHarvester/
 vendor/Google Chrome.app/
 failmap_dataset*
-failmap_testdataset*
\ No newline at end of file
+failmap_testdataset*
+failmap_debug_dataset*
\ No newline at end of file

failmap_admin/scanners/management/commands/discover-endpoints-http-https.py

@@ -6,7 +6,7 @@ from failmap_admin.organizations.models import Organization, Url
 from failmap_admin.scanners.models import Endpoint
 from failmap_admin.scanners.scanner_http import scan_url, scan_urls
 
-from .support.arguments import add_organization_argument
+from .support.arguments import add_discover_verify, add_organization_argument
 
 logger = logging.getLogger(__package__)
 
@@ -17,20 +17,26 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         add_organization_argument(parser)
+        add_discover_verify(parser)
 
     def handle(self, *args, **options):
 
+        # some expansion magic to avoid using eval
+        func = "verify_existing_endpoints" if options['method'] == "verify" else "discover_endpoints"
+        functionlist = {"verify_existing_endpoints": verify_existing_endpoints,
+                        "discover_endpoints": discover_endpoints}
+
         if not options['organization']:
-            discover_endpoints()
+            functionlist[func]()
             return
 
         if options['organization'][0] == "_ALL_":
-            discover_endpoints()
+            functionlist[func]()
             return
 
         organization = Organization.objects.all().filter(name=options['organization'][0])
 
-        discover_endpoints(organization=organization)
+        functionlist[func](organization=organization)
 
 
 def verify_existing_endpoints(port=None, protocol=None, organization=None):
@@ -54,7 +60,7 @@ def verify_existing_endpoints(port=None, protocol=None, organization=None):
     if protocol:
         endpoints = endpoints.filter(protocol=protocol)
     else:
-        endpoints = endpoints.filter(endpoint__protocol__in=['http', 'https'])
+        endpoints = endpoints.filter(protocol__in=['http', 'https'])
 
     if organization:
         endpoints = endpoints.filter(url__organization=organization)

failmap_admin/scanners/management/commands/support/arguments.py

@@ -15,6 +15,17 @@ def add_organization_argument(parser):
     )
 
 
+def add_discover_verify(parser):
+    return parser.add_argument(
+        '--method', '-m',
+        help="verify|discover. Verify checks all existing ones, discover tries to find new ones.",
+        nargs='?',
+        required=False,
+        type=valid_discover_verify,
+        default="verify"
+    )
+
+
 def valid_organization(name):
     if "_ALL_" in name:
         return "_ALL_"
@@ -22,5 +33,10 @@ def valid_organization(name):
         o = Organization.objects.get(name=name)
         return o.name
     except ObjectDoesNotExist:
-        msg = "%s is not a valid organization or _ALL_" % name
-        raise argparse.ArgumentTypeError(msg)
+        raise argparse.ArgumentTypeError("%s is not a valid organization or _ALL_" % name)
+
+
+def valid_discover_verify(option):
+    if option == "verify" or option == "discover":
+        return option
+    raise argparse.ArgumentTypeError("Method can be either 'discover' or 'verify'. Given: " % option)

failmap_admin/scanners/scanner_http.py

@@ -34,7 +34,8 @@ from requests import ConnectTimeout, HTTPError, ReadTimeout, Timeout
 from requests.exceptions import ConnectionError
 
 from failmap_admin.celery import app
-from failmap_admin.scanners.models import Endpoint
+
+from .models import Endpoint
 
 logger = logging.getLogger(__package__)
 
@@ -76,10 +77,30 @@ def scan_urls(urls, ports, protocols):
 
 
 def scan_url(url, port=80, protocol="https"):
+
     task = scan_url_task.s(url, port, protocol)
     task.apply_async()
 
 
+def database_debug():
+    # had the wrong env.
+    from django.db import connection
+    from failmap_admin import settings
+
+    logger.error(dir(settings))
+    logger.error(settings.DATABASE)
+    logger.error(settings.DATABASES)
+
+    sql = "SELECT name FROM sqlite_master WHERE type='table';"
+    cursor = connection.cursor()
+    cursor.execute(sql)
+    rows = cursor.fetchall()
+    logger.error(rows)
+
+    for row in rows:
+        logger.error(row)
+
+
 @app.task
 def scan_url_task(url, port=80, protocol="https"):
     """
@@ -267,7 +288,6 @@ def endpoint_exists(url, port, protocol, ip):
 
 
 def kill_endpoint(url, port, protocol, ip):
-
     eps = Endpoint.objects.all().filter(url=url,
                                         port=port,
                                         ip=ip,

failmap_admin/settings.py

@@ -86,7 +86,7 @@ try:
     import django_extensions
     INSTALLED_APPS += ['django_extensions']
 except ImportError:
-    print("swag")
+    print("Django Extensions is not installed (not a dev setup?) Install requirements.dev.txt if needed.")
     pass