Commit 660b92ba authored by Elger Jonker's avatar Elger Jonker

added rabbitmq to readme, added another dataset export, improved some logging

parent bf33205d
......@@ -27,3 +27,5 @@ version
vendor/dnsrecon/
vendor/theHarvester/
vendor/Google Chrome.app/
failmap_dataset*
failmap_testdataset*
\ No newline at end of file
......@@ -27,7 +27,7 @@ Download and install below system requirements to get started:
- [git](https://git-scm.com/downloads) (download and install)
- [python3](https://www.python.org/downloads/) (download and install)
- [direnv](https://direnv.net/) (download and install, then follow [setup instructions](https://direnv.net/))
- [rabbitmq](http://www.rabbitmq.com/download.html) (for scanners only, download and install)
After installation of above tools, all following steps use the command line:
......@@ -77,6 +77,19 @@ Now visit the [map website](http://127.0.0.1:8000/) and/or the
# Scanning services (beta)
Todo: add celery beat information
Some scanners require RabbitMQ to be installed. We're currently in transition from running scanners
manually to supporting both manual scans and celery beat.
Each of the below commands requires their own command line window:
# start rabbitmq
rabbitmq-server
# start a worker
failmap-admin celery worker -ldebug
These services help fill the database with accurate up to date information. Run each one of them in
a separate command line window and keep them running.
......
......@@ -7,9 +7,11 @@
# from __future__ import absolute_import
import os
from celery import Celery # don't name your module celery, as Celery can not be found :)
from celery import Celery
from django.conf import settings
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "failmap_admin.settings")
app = Celery(__name__)
app.config_from_object('django.conf:settings')
# autodiscover all celery tasks in tasks.py files inside failmap_admin modules
......@@ -20,8 +22,3 @@ app.autodiscover_tasks([app for app in settings.INSTALLED_APPS if app.startswith
@app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
@app.task
def add(x, y):
return x + y
# urls for scanners, maybe in their own url files
import proxy.views
from django.conf import settings
from django.conf.urls import url
import proxy.views
from failmap_admin.map.views import (index, manifest_json, map_data, organization_report,
robots_txt, security_txt, stats, terrible_urls, topfail,
topwin, urlstats, wanted_urls)
......
......@@ -3,9 +3,8 @@ from datetime import datetime
import pytz
from django.core.management.commands.dumpdata import Command as DumpDataCommand
from django.db import connection
from failmap_admin import settings
from .datasethelpers import check_referential_integrity
logger = logging.getLogger(__package__)
......@@ -52,20 +51,7 @@ class Command(DumpDataCommand):
:return:
"""
# first check if there are reference issues. If there are, then we can't export.
# They must be fixed, otherwise
# we can't loaddata at a later stage without terrible hacks.
# this only works for ssqlite.
if "sqlite" in settings.DATABASES["default"]["ENGINE"]:
if not sqlite_has_correct_referential_integrity():
logger.error("Fix above issues. Not proceeding.")
return
else:
logger.error("This export might have incorrect integrity: no foreign key check for "
"this engine was implemented. Loaddata might not accept this import. "
"Perform a key check manually and then alter this code to continue.")
return
check_referential_integrity()
filename = "failmap_dataset_%s.yaml" % datetime.now(pytz.utc).strftime("%Y%m%d_%H%M%S")
......@@ -96,30 +82,3 @@ class Command(DumpDataCommand):
logger.debug(options)
super(Command, self).handle(*app_labels, **options)
def sqlite_has_correct_referential_integrity():
# http://www.sqlite.org/pragma.html#pragma_foreign_key_check
logger.info(
'Checking for foreign key issues, and generating possible SQL to remediate issues.')
cursor = connection.cursor()
cursor.execute('''PRAGMA foreign_key_check;''')
rows = cursor.fetchall()
if rows:
logger.error("Cannot create export. There are incomplete foreign keys. "
"See information above to fix this. "
"Please fix these issues manually and try again.")
for row in rows:
logger.info("%25s %6s %25s %6s" % (row[0], row[1], row[2], row[3]))
if rows:
logger.error(
"Here are some extremely crude SQL statements that might help fix the problem.")
for row in rows:
logger.info("DELETE FROM %s WHERE id = \"%s\";" % (row[0], row[1]))
if rows:
return False
return True
import logging
from datetime import datetime
import pytz
from django.core.management.commands.dumpdata import Command as DumpDataCommand
from .datasethelpers import check_referential_integrity
logger = logging.getLogger(__package__)
# Remove ALL organization and URL ratings and rebuild them
class Command(DumpDataCommand):
help = "Create a near complete export for debugging on another server."
def handle(self, *app_labels, **options):
check_referential_integrity()
filename = "failmap_debug_dataset_%s.yaml" % datetime.now(pytz.utc).strftime("%Y%m%d_%H%M%S")
# Override default options.
options["indent"] = 2
options["format"] = "yaml"
options["output"] = filename
options["verbosity"] = 1 # Show progress bar, it's not really helpful though :)
# Fill the list of things to export
if not app_labels:
app_labels = ('organizations', 'scanners', 'map', 'django_celery_beat')
logger.debug(app_labels)
logger.debug(options)
super(Command, self).handle(*app_labels, **options)
import logging
from django.db import connection
from failmap_admin import settings
logger = logging.getLogger(__package__)
def sqlite_has_correct_referential_integrity():
# http://www.sqlite.org/pragma.html#pragma_foreign_key_check
logger.info(
'Checking for foreign key issues, and generating possible SQL to remediate issues.')
cursor = connection.cursor()
cursor.execute('''PRAGMA foreign_key_check;''')
rows = cursor.fetchall()
if rows:
logger.error("Cannot create export. There are incomplete foreign keys. "
"See information above to fix this. "
"Please fix these issues manually and try again.")
for row in rows:
logger.info("%25s %6s %25s %6s" % (row[0], row[1], row[2], row[3]))
if rows:
logger.error(
"Here are some extremely crude SQL statements that might help fix the problem.")
for row in rows:
logger.info("DELETE FROM %s WHERE id = \"%s\";" % (row[0], row[1]))
if rows:
return False
return True
def check_referential_integrity():
# this only works for ssqlite.
if "sqlite" in settings.DATABASES["default"]["ENGINE"]:
if not sqlite_has_correct_referential_integrity():
logger.error("Fix above issues. Not proceeding.")
return
else:
logger.error("This export might have incorrect integrity: no foreign key check for "
"this engine was implemented. Loaddata might not accept this import. "
"Perform a key check manually and then alter this code to continue.")
return
About datasets:
== Dataset ==
A dataset that is free of things that are easy to recreate. Such things are all logs from scanners,
screenshots and such.
== Test dataset ==
The test-dataset exports 20 organizations, with their minimal information such as scans. Log info
will not be attached.
== Debug Dataset ==
The debug dataset is the most complete dataset that contains all database data, which results in
several hundreds or even thousands of megabytes of yaml.
import logging
from datetime import datetime
import pytz
from django.core.exceptions import ObjectDoesNotExist
logger = logging.getLogger(__package__)
class EndpointScanManager:
"""
......@@ -26,10 +29,12 @@ class EndpointScanManager:
# last scan had exactly the same result, so don't create a new scan and just update the
# last scan date.
if gs.explanation == message and gs.rating == rating:
logger.debug("Scan had the same rating and message, updating last_scan_moment only.")
gs.last_scan_moment = datetime.now(pytz.utc)
gs.save()
else:
# make a new one, please don't update the existing one :)
logger.debug("Message or rating changed: making a new generic scan.")
gs = EndpointGenericScan()
gs.explanation = message
gs.rating = rating
......
......@@ -4,7 +4,8 @@ from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization, Url
from failmap_admin.scanners.scanner_security_headers import scan_organization
from failmap_admin.scanners.scanner_security_headers import (scan_organization,
scan_organization_celery)
from failmap_admin.scanners.state_manager import StateManager
logger = logging.getLogger(__package__)
......@@ -17,4 +18,5 @@ class Command(BaseCommand):
organizations = StateManager.create_resumed_organizationlist(scanner="Security Headers")
for organization in organizations:
StateManager.set_state("Security Headers", organization.name)
scan_organization(organization)
scan_organization_celery(organization)
return
......@@ -9,7 +9,7 @@ import pytz
from celery import Celery
from celery.task import task
from celery_test import app
from failmap_admin.celery import app
from failmap_admin.organizations.models import Url
from failmap_admin.scanners.endpoint_scan_manager import EndpointScanManager
from failmap_admin.scanners.models import EndpointGenericScanScratchpad
......@@ -53,6 +53,14 @@ def scan_organization(organization):
scan_headers(url)
def scan_organization_celery(organization):
urls = Url.objects.all().filter(is_dead=False,
not_resolvable=False,
organization=organization)
for url in urls:
scan_header_celery(url)
def scan_headers(url):
# get website headers using request library
# don't redirect, otherwise count scores for _all_ redirections (which can be a lot :))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment