Refactor to make dataset creators more generic.

parent 97d1c761
......@@ -13,34 +13,48 @@ logger = logging.getLogger(__package__)
class Command(DumpDataCommand):
help = "Create a near complete export for testing and migrating to another server."
FILENAME = "failmap_dataset_{}.{options[format]}"
# Here is a list of what is and is not included.
#
# Included
# Map:
# - Url Rating No Can be rebuild with rebuildratings
# - Organization Rating No Can be rebuild with rebuildratings
#
# Organization:
# - Coordinates Yes Hard to create, is not scripted yet
# - Organizations Yes Hard to gather
# - OrganizationType Yes Foreign Keys
# - Urls Yes Even harder to gather
#
# Scanners:
# - Endpoints Yes Needed for rebuild ratings, hard to gather
# - Screenshots No Can be recreated with ease (except history)
# - States No Just start somewhere
# - TLS Qualys Scans Yes Needed for rebuild ratings
# - TLS Qualys Scratchpa No This is mainly for debugging (todo: check scan tls qual)
# - Generic Scans Yes
# - Generic Scans scratch No
#
# Auth:
# - Users No Create this yourself
# - Groups No Create this yourself
APP_LABELS = (
"organizations.OrganizationType",
"organizations.Organization",
"organizations.Coordinate",
"organizations.Url",
"scanners.Endpoint",
"scanners.TlsQualysScan",
"scanners.EndpointGenericScan"
)
def handle(self, *app_labels, **options):
"""
This function will make a YAML export of the data in the database that is not easily
recreateable. Here is a list of what is and is not included.
Included
Map:
- Url Rating No Can be rebuild with rebuildratings
- Organization Rating No Can be rebuild with rebuildratings
Organization:
- Coordinates Yes Hard to create, is not scripted yet
- Organizations Yes Hard to gather
- OrganizationType Yes Foreign Keys
- Urls Yes Even harder to gather
Scanners:
- Endpoints Yes Needed for rebuild ratings, hard to gather
- Screenshots No Can be recreated with ease (except history)
- States No Just start somewhere
- TLS Qualys Scans Yes Needed for rebuild ratings
- TLS Qualys Scratchpa No This is mainly for debugging (todo: check scan tls qual)
- Generic Scans Yes
- Generic Scans scratch No
Auth:
- Users No Create this yourself
- Groups No Create this yourself
recreateable.
Further docs:
https://docs.djangoproject.com/en/1.11/ref/django-admin/
......@@ -51,9 +65,11 @@ class Command(DumpDataCommand):
:return:
"""
# verify data is properly exportable
check_referential_integrity()
filename = "failmap_dataset_{}.{options[format]}".format(
# generate unique filename for every export
filename = self.FILENAME.format(
datetime.now(pytz.utc).strftime("%Y%m%d_%H%M%S"),
options=options
)
......@@ -62,24 +78,8 @@ class Command(DumpDataCommand):
if not options['output']:
options["output"] = filename
# Fill the list of things to export
# unless specified on the commandline, use default set of apps to export
if not app_labels:
# about 8 megabyte
app_labels = (
"organizations.OrganizationType",
"organizations.Organization",
"organizations.Coordinate",
"organizations.Url",
"scanners.Endpoint",
"scanners.TlsQualysScan",
"scanners.EndpointGenericScan"
)
# the rest:
# about 160 megabyte for the database
# screenshot files are about 6 gigabyte
# app_labels = ('organizations', 'scanners', 'map') # to check size of whole database
logger.debug(app_labels)
logger.debug(options)
app_labels = self.APP_LABELS
super(Command, self).handle(*app_labels, **options)
import logging
from datetime import datetime
import pytz
from django.core.management.commands.dumpdata import Command as DumpDataCommand
from .datasethelpers import check_referential_integrity
from .create_dataset import Command as CreateDatasetCommand
logger = logging.getLogger(__package__)
# Remove ALL organization and URL ratings and rebuild them
class Command(DumpDataCommand):
class Command(CreateDatasetCommand):
help = "Create a near complete export for debugging on another server."
def handle(self, *app_labels, **options):
check_referential_integrity()
filename = "failmap_debug_dataset_%s.yaml" % datetime.now(pytz.utc).strftime("%Y%m%d_%H%M%S")
# Override default options.
options["indent"] = 2
options["format"] = "yaml"
options["output"] = filename
options["verbosity"] = 1 # Show progress bar, it's not really helpful though :)
# Fill the list of things to export
if not app_labels:
app_labels = ('organizations', 'scanners', 'map', 'django_celery_beat')
logger.debug(app_labels)
logger.debug(options)
FILENAME = "failmap_debug_dataset_{}.{options[format]}"
super(Command, self).handle(*app_labels, **options)
APP_LABELS = ('organizations', 'scanners', 'map', 'django_celery_beat')
......@@ -8,12 +8,19 @@ from django.core.serializers import serialize
from failmap_admin.organizations.models import Coordinate, Organization, OrganizationType, Url
from failmap_admin.scanners.models import Endpoint, EndpointGenericScan, TlsQualysScan
logger = logging.getLogger(__package__)
log = logging.getLogger(__package__)
# Remove ALL organization and URL ratings and rebuild them
class Command(DumpDataCommand):
help = "Create a near complete export for testing and migrating to another server."
help = "Create a small export for testing."
FILENAME = "failmap_testdataset_{}.{options[format]}"
APP_LABELS = ('organizations', 'scanners', 'map', 'django_celery_beat')
# for testing it is nice to have a human editable serialization language
FORMAT = 'yaml'
def handle(self, *app_labels, **options):
"""
......@@ -21,7 +28,7 @@ class Command(DumpDataCommand):
easily recreateable. Django's own serializers are used.
Only organizations with the letter A will be included. This consists of large and small
organizations. At max 10 organizations.
organizations. At max 20 organizations.
Included
Organization:
......@@ -35,28 +42,42 @@ class Command(DumpDataCommand):
- TLS Qualys Scans Yes Needed for rebuild ratings
- Generic Scans Yes
"""
filename = "failmap_testdataset_%s.yaml" % datetime.now(pytz.utc).strftime("%Y%m%d_%H%M%S")
file = open(filename, "w")
# force desired format for testing
options['format'] = self.FORMAT
file.write(serialize('yaml', OrganizationType.objects.all()))
if options['output']:
filename = options['output']
else:
# generate unique filename for every export
filename = self.FILENAME.format(
datetime.now(pytz.utc).strftime("%Y%m%d_%H%M%S"),
options=options
)
objects = []
objects += OrganizationType.objects.all()
organizations = Organization.objects.all().filter(name__istartswith='A')[0:20]
file.write(serialize('yaml', organizations))
objects += organizations
coordinates = Coordinate.objects.all().filter(organization__in=organizations)
file.write(serialize('yaml', coordinates))
objects += coordinates
urls = Url.objects.all().filter(organization__in=organizations)
file.write(serialize('yaml', urls))
objects += urls
endpoints = Endpoint.objects.all().filter(url__in=urls)
file.write(serialize('yaml', endpoints))
objects += endpoints
tlsqualysscans = TlsQualysScan.objects.all().filter(endpoint__in=endpoints)
file.write(serialize('yaml', tlsqualysscans))
objects += tlsqualysscans
endpointgenericscans = EndpointGenericScan.objects.all().filter(endpoint__in=endpoints)
file.write(serialize('yaml', endpointgenericscans))
objects += endpointgenericscans
with open(filename, "w") as f:
f.write(serialize(self.FORMAT, objects))
file.close()
log.info('Wrote %s', filename)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment