parallelized the faster ratings, added export for one organization

parent 54108ae7
import logging
from django.core.management.base import BaseCommand
from failmap_admin.map.models import OrganizationRating, UrlRating
logger = logging.getLogger(__package__)
class Command(BaseCommand):
help = 'Deletes ratings from the database. They can be rebuild based on available scanner data.'
def handle(self, *args, **options):
askreset()
def askreset():
try:
print("Do you __REALLY__ want to delete all ratings?")
answer = input("Type 'YES' if you mean it: ")
if answer == "YES":
and_its_gone()
else:
nothing_happened()
except KeyboardInterrupt:
nothing_happened()
def nothing_happened():
print("Nothing was deleted.")
def and_its_gone():
"""
This is not a maintenance friendly way of deleting data.
There is a thing in django somewhere that determines the order of relationships in the model.
:return:
"""
# map
OrganizationRating.objects.all().delete()
UrlRating.objects.all().delete()
......@@ -2,7 +2,7 @@ import logging
from failmap_admin.app.management.commands._private import TaskCommand
from ...rating import rebuild_ratings
from ...rating import rebuild_ratings_async
log = logging.getLogger(__name__)
......@@ -12,4 +12,6 @@ class Command(TaskCommand):
help = __doc__
task = rebuild_ratings
# 6.5 minutes
# original: 1 hour
task = rebuild_ratings_async
......@@ -33,9 +33,11 @@ def rebuild_ratings():
rerate_organizations()
def rebuild_ratings_async():
@app.task
def rebuild_ratings_async(execute_locally: bool=True):
"""Remove all organization and url ratings, then rebuild them from scratch."""
rerate_organizations()
task = (rerate_urls_async.s(execute_locally=True) | rerate_organizations_async.si(execute_locally=True))
task.apply_async()
@app.task
......@@ -87,6 +89,11 @@ def delete_url_rating(url: Url):
UrlRating.objects.all().filter(url=url).delete()
@app.task
def delete_organization_rating(organization: Organization):
OrganizationRating.objects.all().filter(organization=organization).delete()
@app.task
# 2.5 minutes and it's done :)
def rerate_urls_async(urls: List[Url]=None, execute_locally: bool=True):
......@@ -98,7 +105,26 @@ def rerate_urls_async(urls: List[Url]=None, execute_locally: bool=True):
for url in urls:
tasks.append((delete_url_rating.s(url) | create_timeline.si(url) | rate_timeline.s(url)))
task = group([task for task in tasks])
task = group(tasks)
if execute_locally:
task.apply_async()
else:
return task
@app.task
def rerate_organizations_async(organizations: List[Organization]=None, execute_locally: bool=True):
if not organizations:
organizations = list(Organization.objects.all().order_by('name'))
# to not clear the whole map at once, do this per organization.
# could be more efficient, but since the process is so slow, you'll end up with people looking at empty maps.
tasks = [(default_ratings.si()
| delete_organization_rating.si(organization)
| add_organization_rating.si(organizations=[organization], build_history=True))
for organization in organizations]
task = group(tasks)
if execute_locally:
task.apply_async()
else:
......@@ -118,10 +144,6 @@ def rerate_organizations(organizations: List[Organization]=None):
add_organization_rating(organizations=[organization], build_history=True)
def rerate_urls_of_organizations(organizations: List[Organization]):
rerate_urls(Url.objects.filter(is_dead=False, organization__in=organizations).order_by('url'))
def significant_moments(organizations: List[Organization]=None, urls: List[Url]=None):
"""
Searches for all significant point in times that something changed. The goal is to save
......@@ -1098,6 +1120,7 @@ def relevant_endpoints_at_timepoint(url: Url, when: datetime):
# todo: use the organization creation date for this.
@app.task
def default_ratings():
"""
Generate default ratings so all organizations are on the map (as being grey). This prevents
......
......@@ -274,25 +274,25 @@
<tr v-if="x['explained']['tls_qualys']" class="redrow">
<td></td>
<td>{% endverbatim %}{% trans "No trust, C" %}{% verbatim %}:</td>
<td>{{ x['explained']['tls_qualys']['Could not establish trust. For the certificate installation: Less than optimal Transport Security, rated C.'] }}
<td>{{ x['explained']['tls_qualys']['Could not establish trust. For the certificate installation: Less than optimal Transport Security, rated C.'] }}
</td>
</tr>
<tr v-if="x['explained']['tls_qualys']" class="redrow">
<td></td>
<td>{% endverbatim %}{% trans "No trust, B" %}{% verbatim %}:</td>
<td>{{ x['explained']['tls_qualys']['Could not establish trust. For the certificate installation: Less than optimal Transport Security, rated B.'] }}
<td>{{ x['explained']['tls_qualys']['Could not establish trust. For the certificate installation: Less than optimal Transport Security, rated B.'] }}
</td>
</tr>
<tr v-if="x['explained']['tls_qualys']" class="redrow">
<td></td>
<td>{% endverbatim %}{% trans "No trust, A-" %}{% verbatim %}:</td>
<td>{{ x['explained']['tls_qualys']['Could not establish trust. For the certificate installation: Good Transport Security, rated A-.'] }}
<td>{{ x['explained']['tls_qualys']['Could not establish trust. For the certificate installation: Good Transport Security, rated A-.'] }}
</td>
</tr>
<tr v-if="x['explained']['tls_qualys']" class="redrow">
<td></td>
<td>{% endverbatim %}{% trans "No trust, A" %}{% verbatim %}:</td>
<td>{{ x['explained']['tls_qualys']['Could not establish trust. For the certificate installation: Good Transport Security, rated A.'] }}
<td>{{ x['explained']['tls_qualys']['Could not establish trust. For the certificate installation: Good Transport Security, rated A.'] }}
</td>
</tr>
<tr v-if="x['explained']['tls_qualys']" class="orangerow">
......
import logging
from datetime import datetime
import pytz
from django.core.management.commands.dumpdata import Command as DumpDataCommand
from django.core.serializers import serialize
from failmap_admin.map.models import OrganizationRating, UrlRating
from failmap_admin.organizations.models import (Coordinate, Organization, OrganizationType, Promise,
Url)
from failmap_admin.scanners.models import (Endpoint, EndpointGenericScan, Screenshot, TlsQualysScan,
UrlIp)
log = logging.getLogger(__package__)
class Command(DumpDataCommand):
help = "Create a smaller export for testing."
FILENAME = "failmap_organization_export_{}.{options[format]}"
APP_LABELS = ('organizations', 'scanners', 'map', 'django_celery_beat')
# for testing it is nice to have a human editable serialization language
FORMAT = 'yaml'
def add_arguments(self, parser):
parser.add_argument('organizations', type=str, nargs='+')
# https://stackoverflow.com/questions/8203622/argparse-store-false-if-unspecified#8203679
# https://edumaven.com/python-programming/argparse-boolean
# --all / "all" is used by super.
parser.add_argument('--yes', dest='yes', action='store_true')
super(Command, self).add_arguments(parser)
def handle(self, *app_labels, **options):
log.info(options['yes'])
options['format'] = self.FORMAT
if options['output']:
filename = options['output']
else:
# generate unique filename for every export
filename = self.FILENAME.format(
datetime.now(pytz.utc).strftime("%Y%m%d_%H%M%S"),
options=options
)
objects = []
objects += OrganizationType.objects.all()
for organization in options['organizations']:
organizations = Organization.objects.all().filter(name=organization)
objects += organizations
objects += Promise.objects.all().filter(organization__in=organizations)
objects += Coordinate.objects.all().filter(organization__in=organizations)
if options['yes']:
objects += OrganizationRating.objects.all().filter(organization__in=organizations)
urls = Url.objects.all().filter(organization__in=organizations)
objects += urls
objects += UrlIp.objects.all().filter(url__in=urls)
if options['yes']:
objects += UrlRating.objects.all().filter(url__in=urls)
endpoints = Endpoint.objects.all().filter(url__in=urls)
objects += endpoints
objects += TlsQualysScan.objects.all().filter(endpoint__in=endpoints)
objects += EndpointGenericScan.objects.all().filter(endpoint__in=endpoints)
if options['yes']:
objects += Screenshot.objects.all().filter(endpoint__in=endpoints)
with open(filename, "w") as f:
f.write(serialize(self.FORMAT, objects))
log.info('Wrote %s', filename)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment