Verified Commit ca7e44aa authored by Elger Jonker's avatar Elger Jonker

more complete dataset export, generated datamodel documentation, command cleanup

parent 00ea3285
Pipeline #26302599 passed with stage
in 15 minutes and 1 second
Below image is autogenerated and shows the Failmap data model.
This is an autogenerated page about the FailMap data model.
# organizations
![Data Model](data_model/organizations_models.png)
# scanners
![Data Model](data_model/scanners_models.png)
# map
![Data Model](data_model/map_models.png)
# game
![Data Model](data_model/game_models.png)
# app
![Data Model](data_model/app_models.png)
# hypersh
![Data Model](data_model/hypersh_models.png)
# All in one
![Data Model](data_model/failmap_models.png)
![Data Model](data_model/failmap_models.png)
\ No newline at end of file
# Some help to translate the django part.
# This tries to help you avoid remembering the "messages" mess from Django.
import logging
from subprocess import call
from django.core.management.base import BaseCommand
try:
from pip._internal.utils.misc import get_installed_distributions
except ImportError: # pip<10
from pip import get_installed_distributions
# from pip.utils import get_installed_distributions
logger = logging.getLogger(__package__)
# todo: move to pipenv
class Command(BaseCommand):
def handle(self, *args, **options):
packages = [dist.project_name for dist in get_installed_distributions()]
call("pip install --upgrade " + ' '.join(packages), shell=True)
"""Basic tests to check nothing major is broken."""
from __future__ import absolute_import, unicode_literals
import logging
from django.core.management import call_command
from django.core.management.base import BaseCommand
log = logging.getLogger(__package__)
class Command(BaseCommand):
"""Celery command wrapper."""
help = __doc__
def add_arguments(self, parser):
parser.add_argument('--clear_entire_database', action='store_true',
help='Will remove ALL data from the database.')
super().add_arguments(parser)
def handle(self, *args, **options):
if options['clear_entire_database']:
run_clean()
test_all_scanners()
def run_clean():
# Another protection against idiocy
from django.conf import settings
if not settings.DEBUG:
log.error("Will not clear database in production environment.")
return
# requires manual verification
call_command("clear_database")
# Load the single organization (Internet Cleanup Foundation)
log.warning("load_dataset")
call_command('load_dataset', 'internet_cleanup_foundation')
def test_all_scanners():
"""
Tests various scanning commands from the command line.
:param admin_client:
:return:
"""
# check the network. No network, you'll fail. Todo: you should be able to reach a domain over the net. Build
# machines often cannot do this for reasons and things.
from failmap.scanners.scanner.http import check_network
log.warning("check_network")
check_network("real_scanner_test")
from failmap.organizations.models import Organization, Url
organization = Organization.objects.filter(name="Internet Cleanup Foundation").get()
toplevel_urls = Url.objects.all().filter(organization=organization, url__iregex="^[^.]*\.[^.]*$")
first_toplevel_url = toplevel_urls.first()
# Run this test here, given the qualys scanner does not rate limit when running standalone.
# Three urls at the same time will be fine. But if you run this test too often (every few minutes) you'll
# get errors.
call_command('scan_tls_qualys', '-v3', '-o', 'Internet Cleanup Foundation')
from failmap.scanners.scanner.dns import brute_known_subdomains, certificate_transparency, nsec
log.warning("brute_known_subdomains")
brute_known_subdomains(urls=[first_toplevel_url])
log.warning("certificate_transparency")
certificate_transparency(urls=[first_toplevel_url])
log.warning("nsec")
nsec(urls=[first_toplevel_url])
# for the first three urls, run all possible scans.
# We're testing that none of below commands crash, regardless of their output.
# Of course configurations and such can change.
# how to detect errors? Try except?
log.warning("scan_dnssec")
call_command('scan_dnssec', '-v3', '-o', 'Internet Cleanup Foundation')
log.warning("discover_endpoints_2")
call_command('discover_endpoints_2', '-v3', '-o', 'Internet Cleanup Foundation')
log.warning("scan_security_headers")
call_command('scan_security_headers', '-v3', '-o', 'Internet Cleanup Foundation')
log.warning("scan_plain_http")
call_command('scan_plain_http', '-v3', '-o', 'Internet Cleanup Foundation')
log.warning("scan_security_headers")
call_command('scan_security_headers', '-v3', '-o', 'Internet Cleanup Foundation')
log.warning("scan_tls_qualys")
log.warning("export_organization")
call_command('export_organization', 'Internet Cleanup Foundation')
log.warning("rebuild_ratings")
call_command('rebuild_ratings')
......@@ -10,6 +10,14 @@ logger = logging.getLogger(__package__)
class Command(BaseCommand):
def handle(self, *args, **options):
"""
This was created to update coordinates from older databases. Was a one-shot.
todo: Not needed anymore?
:param args:
:param options:
:return:
"""
coords = Coordinate.objects.all()
for coord in coords:
......
......@@ -32,7 +32,7 @@ class Command(DumpDataCommand):
# Scanners:
# - Endpoints Yes Needed for rebuild ratings, hard to gather
# - Screenshots No Can be recreated with ease (except history)
# - States No Just start somewhere
# - TLS Scans Yes
# - TLS Qualys Scans Yes Needed for rebuild ratings
# - TLS Qualys Scratchpa No This is mainly for debugging
# - Generic Scans Yes
......@@ -42,6 +42,12 @@ class Command(DumpDataCommand):
# Auth:
# - Users No Create this yourself
# - Groups No Create this yourself
#
# Game:
# - Team Yes
# - Contest Yes
# - OrganizationSubmission Yes Helps with debugging
# - UrlSubmission Yes Helps with debugging
APP_LABELS = (
"organizations.OrganizationType",
......@@ -51,11 +57,13 @@ class Command(DumpDataCommand):
"organizations.Promise",
"scanners.Endpoint",
"scanners.TlsQualysScan",
"scanners.TlsScan",
"scanners.EndpointGenericScan",
"scanners.UrlGenericScan",
"scanners.UrlIp",
"map.Configuration",
"map.AdministrativeRegion"
"map.AdministrativeRegion",
"game"
)
def handle(self, *app_labels, **options):
......
import logging
import os
from django.core.management import call_command
from django.core.management.base import BaseCommand
logger = logging.getLogger(__package__)
log = logging.getLogger(__package__)
class Command(BaseCommand):
help = 'Adds some documentation artifacts to this project.'
def handle(self, *args, **options):
raise NotImplementedError
# failmap graph_models organizations scanners map -o myapp_models.png
# it's posisble to also include auth and other installed things by not specifying an app.
apps = ['organizations', 'scanners', 'map', 'game', 'app', 'hypersh']
for app in apps:
log.info("creating image for: %s", app)
call_command('graph_models', app, '-o',
'docs/source/topics/development/data_model/%s_models.png' % app)
# not using -a, because it's hard to see where some of the models originate from (and then to remove them
# using -X). So making it more explicit.
log.info("creating image for all models in one.")
call_command('graph_models', 'organizations', 'scanners', 'map', 'game', 'app', 'hypersh', '-o',
'docs/source/topics/development/data_model/failmap_models.png')
# Creating a file that contains all above models...
contents = "This is an autogenerated page about the FailMap data model." + os.linesep
for app in apps:
contents += "# %s" % app + os.linesep
contents += "![Data Model](data_model/%s_models.png)" % app + os.linesep
contents += "" + os.linesep
contents += "# All in one" + os.linesep
contents += "![Data Model](data_model/failmap_models.png)" + os.linesep
contents += "" + os.linesep
with open("docs/source/topics/development/data_model.md", "w") as text_file:
text_file.write(contents)
......@@ -17,6 +17,7 @@ class Command(DiscoverTaskCommand):
def handle(self, *args, **options):
# todo: subdomains, from scanner.dns
scanners = {
'ftp': ftp,
'http': http,
......
......@@ -35,6 +35,7 @@ recommonmark
httmock
freezegun
colorama
pydotplus # generating graphs with the docs command...
# used to restart celery worker on file changes
# use current (03-2018) master as it includes a fix for macOS: https://github.com/gorakhargosh/watchdog/pull/329
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment