Verified Commit ca7e44aa authored by Elger Jonker's avatar Elger Jonker

more complete dataset export, generated datamodel documentation, command cleanup

parent 00ea3285
Pipeline #26302599 passed with stage
in 15 minutes and 1 second
Below image is autogenerated and shows the Failmap data model.
This is an autogenerated page about the FailMap data model.
# organizations
![Data Model](data_model/organizations_models.png)
# scanners
![Data Model](data_model/scanners_models.png)
# map
![Data Model](data_model/map_models.png)
# game
![Data Model](data_model/game_models.png)
# app
![Data Model](data_model/app_models.png)
# hypersh
![Data Model](data_model/hypersh_models.png)
# All in one
![Data Model](data_model/failmap_models.png)
![Data Model](data_model/failmap_models.png)
\ No newline at end of file
# Some help to translate the django part.
# This tries to help you avoid remembering the "messages" mess from Django.
import logging
from subprocess import call
from import BaseCommand
from pip._internal.utils.misc import get_installed_distributions
except ImportError: # pip<10
from pip import get_installed_distributions
# from pip.utils import get_installed_distributions
logger = logging.getLogger(__package__)
# todo: move to pipenv
class Command(BaseCommand):
def handle(self, *args, **options):
packages = [dist.project_name for dist in get_installed_distributions()]
call("pip install --upgrade " + ' '.join(packages), shell=True)
"""Basic tests to check nothing major is broken."""
from __future__ import absolute_import, unicode_literals
import logging
from import call_command
from import BaseCommand
log = logging.getLogger(__package__)
class Command(BaseCommand):
"""Celery command wrapper."""
help = __doc__
def add_arguments(self, parser):
parser.add_argument('--clear_entire_database', action='store_true',
help='Will remove ALL data from the database.')
def handle(self, *args, **options):
if options['clear_entire_database']:
def run_clean():
# Another protection against idiocy
from django.conf import settings
if not settings.DEBUG:
log.error("Will not clear database in production environment.")
# requires manual verification
# Load the single organization (Internet Cleanup Foundation)
call_command('load_dataset', 'internet_cleanup_foundation')
def test_all_scanners():
Tests various scanning commands from the command line.
:param admin_client:
# check the network. No network, you'll fail. Todo: you should be able to reach a domain over the net. Build
# machines often cannot do this for reasons and things.
from failmap.scanners.scanner.http import check_network
from failmap.organizations.models import Organization, Url
organization = Organization.objects.filter(name="Internet Cleanup Foundation").get()
toplevel_urls = Url.objects.all().filter(organization=organization, url__iregex="^[^.]*\.[^.]*$")
first_toplevel_url = toplevel_urls.first()
# Run this test here, given the qualys scanner does not rate limit when running standalone.
# Three urls at the same time will be fine. But if you run this test too often (every few minutes) you'll
# get errors.
call_command('scan_tls_qualys', '-v3', '-o', 'Internet Cleanup Foundation')
from failmap.scanners.scanner.dns import brute_known_subdomains, certificate_transparency, nsec
# for the first three urls, run all possible scans.
# We're testing that none of below commands crash, regardless of their output.
# Of course configurations and such can change.
# how to detect errors? Try except?
call_command('scan_dnssec', '-v3', '-o', 'Internet Cleanup Foundation')
call_command('discover_endpoints_2', '-v3', '-o', 'Internet Cleanup Foundation')
call_command('scan_security_headers', '-v3', '-o', 'Internet Cleanup Foundation')
call_command('scan_plain_http', '-v3', '-o', 'Internet Cleanup Foundation')
call_command('scan_security_headers', '-v3', '-o', 'Internet Cleanup Foundation')
call_command('export_organization', 'Internet Cleanup Foundation')
......@@ -10,6 +10,14 @@ logger = logging.getLogger(__package__)
class Command(BaseCommand):
def handle(self, *args, **options):
This was created to update coordinates from older databases. Was a one-shot.
todo: Not needed anymore?
:param args:
:param options:
coords = Coordinate.objects.all()
for coord in coords:
......@@ -32,7 +32,7 @@ class Command(DumpDataCommand):
# Scanners:
# - Endpoints Yes Needed for rebuild ratings, hard to gather
# - Screenshots No Can be recreated with ease (except history)
# - States No Just start somewhere
# - TLS Scans Yes
# - TLS Qualys Scans Yes Needed for rebuild ratings
# - TLS Qualys Scratchpa No This is mainly for debugging
# - Generic Scans Yes
......@@ -42,6 +42,12 @@ class Command(DumpDataCommand):
# Auth:
# - Users No Create this yourself
# - Groups No Create this yourself
# Game:
# - Team Yes
# - Contest Yes
# - OrganizationSubmission Yes Helps with debugging
# - UrlSubmission Yes Helps with debugging
......@@ -51,11 +57,13 @@ class Command(DumpDataCommand):
def handle(self, *app_labels, **options):
import logging
import os
from import call_command
from import BaseCommand
logger = logging.getLogger(__package__)
log = logging.getLogger(__package__)
class Command(BaseCommand):
help = 'Adds some documentation artifacts to this project.'
def handle(self, *args, **options):
raise NotImplementedError
# failmap graph_models organizations scanners map -o myapp_models.png
# it's posisble to also include auth and other installed things by not specifying an app.
apps = ['organizations', 'scanners', 'map', 'game', 'app', 'hypersh']
for app in apps:"creating image for: %s", app)
call_command('graph_models', app, '-o',
'docs/source/topics/development/data_model/%s_models.png' % app)
# not using -a, because it's hard to see where some of the models originate from (and then to remove them
# using -X). So making it more explicit."creating image for all models in one.")
call_command('graph_models', 'organizations', 'scanners', 'map', 'game', 'app', 'hypersh', '-o',
# Creating a file that contains all above models...
contents = "This is an autogenerated page about the FailMap data model." + os.linesep
for app in apps:
contents += "# %s" % app + os.linesep
contents += "![Data Model](data_model/%s_models.png)" % app + os.linesep
contents += "" + os.linesep
contents += "# All in one" + os.linesep
contents += "![Data Model](data_model/failmap_models.png)" + os.linesep
contents += "" + os.linesep
with open("docs/source/topics/development/", "w") as text_file:
......@@ -17,6 +17,7 @@ class Command(DiscoverTaskCommand):
def handle(self, *args, **options):
# todo: subdomains, from scanner.dns
scanners = {
'ftp': ftp,
'http': http,
......@@ -35,6 +35,7 @@ recommonmark
pydotplus # generating graphs with the docs command...
# used to restart celery worker on file changes
# use current (03-2018) master as it includes a fix for macOS:
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment