Commit 23d65ccb authored by Elger Jonker's avatar Elger Jonker

supporting kwargs in periodic tasks, easy filtering for latest scans, tool to set latest scans


Former-commit-id: 07a11335
parent 0b37bbc1
......@@ -2,6 +2,7 @@
# from __future__ import unicode_literals
import importlib
import logging
import celery
from django.contrib.auth.models import User
......@@ -11,6 +12,8 @@ from jsonfield import JSONField
from ..celery import app
log = logging.getLogger(__name__)
class Job(models.Model):
"""Wrap any Celery task to easily 'manage' it from Django."""
......@@ -77,40 +80,40 @@ class Job(models.Model):
@app.task(queue='storage')
def create_job(task_module: str):
def create_job(task_module: str, **kwargs):
"""Helper to allow Jobs to be created using Celery Beat.
task_module: module from which to call `compose_discover_task` which results in the task to be executed
"""
module = importlib.import_module(task_module)
task = module.compose_task()
task = module.compose_task(**kwargs)
return Job.create(task, task_module, None)
@app.task(queue='storage')
def create_discover_job(task_module: str):
def create_discover_job(task_module: str, **kwargs):
"""Helper to allow Jobs to be created using Celery Beat.
task_module: module from which to call `compose_discover_task` which results in the task to be executed
"""
module = importlib.import_module(task_module)
task = module.compose_discover_task()
task = module.compose_discover_task(**kwargs)
return Job.create(task, task_module, None)
@app.task(queue='storage')
def create_verify_job(task_module: str):
def create_verify_job(task_module: str, **kwargs):
"""Helper to allow Jobs to be created using Celery Beat.
task_module: module from which to call `compose_discover_task` which results in the task to be executed
"""
module = importlib.import_module(task_module)
task = module.compose_verify_task()
task = module.compose_verify_task(**kwargs)
return Job.create(task, task_module, None)
......
......@@ -86,7 +86,7 @@ class EndpointAdmin(ImportExportModelAdmin, admin.ModelAdmin):
@admin.register(TlsScan)
class TlsScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
list_display = ('endpoint', 'rating', 'rating_no_trust', 'comply_or_explain_is_explained', 'compared_to_qualys',
'explanation', 'last_scan_moment', 'rating_determined_on', 'explain')
'explanation', 'last_scan_moment', 'rating_determined_on', 'explain', 'is_the_latest_scan')
search_fields = ('endpoint__url__url', 'rating', 'rating_no_trust',
'scan_date', 'rating_determined_on')
......@@ -96,13 +96,14 @@ class TlsScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
'endpoint__protocol',
'endpoint__port', 'endpoint__ip_version', 'endpoint__discovered_on', 'endpoint__is_dead',
'comply_or_explain_is_explained', 'comply_or_explain_explained_on',
'comply_or_explain_case_handled_by', 'comply_or_explain_explanation_valid_until'
'comply_or_explain_case_handled_by', 'comply_or_explain_explanation_valid_until',
'is_the_latest_scan'
][::-1]
fieldsets = (
(None, {
'fields': ('endpoint', 'rating', 'rating_no_trust', 'explanation', 'evidence',
'rating_determined_on', 'last_scan_moment')
'rating_determined_on', 'last_scan_moment', 'is_the_latest_scan')
}),
('comply or explain', {
'fields': ('comply_or_explain_is_explained', 'comply_or_explain_explanation_valid_until',
......@@ -123,13 +124,14 @@ class TlsScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
return "%s %s | %s %s" % (first, latest.qualys_rating, second, latest.qualys_rating_no_trust)
readonly_fields = ('endpoint', 'rating', 'rating_no_trust', 'explanation', 'evidence',
'rating_determined_on', 'last_scan_moment')
'rating_determined_on', 'last_scan_moment', 'is_the_latest_scan')
@admin.register(TlsQualysScan)
class TlsQualysScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
list_display = ('endpoint', 'qualys_rating', 'qualys_rating_no_trust', 'qualys_message',
'last_scan_moment', 'rating_determined_on', 'comply_or_explain_is_explained', 'explain')
'last_scan_moment', 'rating_determined_on', 'comply_or_explain_is_explained', 'explain',
'is_the_latest_scan')
search_fields = ('endpoint__url__url', 'qualys_rating', 'qualys_rating_no_trust',
'scan_date', 'rating_determined_on')
......@@ -140,7 +142,8 @@ class TlsQualysScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
'endpoint__protocol',
'endpoint__port', 'endpoint__ip_version', 'endpoint__discovered_on', 'endpoint__is_dead',
'comply_or_explain_is_explained', 'comply_or_explain_explained_on',
'comply_or_explain_case_handled_by', 'comply_or_explain_explanation_valid_until'
'comply_or_explain_case_handled_by', 'comply_or_explain_explanation_valid_until',
'is_the_latest_scan'
][::-1]
# loading related fields in django jet is not done in a smart way: everything is prefetched.
......@@ -150,7 +153,7 @@ class TlsQualysScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
fieldsets = (
(None, {
'fields': ('endpoint', 'qualys_rating', 'qualys_rating_no_trust',
'rating_determined_on', 'last_scan_moment')
'rating_determined_on', 'last_scan_moment', 'is_the_latest_scan')
}),
('comply or explain', {
'fields': ('comply_or_explain_is_explained', 'comply_or_explain_explanation_valid_until',
......@@ -161,7 +164,7 @@ class TlsQualysScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
)
readonly_fields = ('endpoint', 'qualys_rating', 'qualys_rating_no_trust',
'rating_determined_on', 'last_scan_moment')
'rating_determined_on', 'last_scan_moment', 'is_the_latest_scan')
def explain(self, object):
return format_html("<a href='./{}/change/#/tab/module_1/'>Explain</a>", object.pk)
......@@ -189,7 +192,7 @@ class ScreenshotAdmin(ImportExportModelAdmin, admin.ModelAdmin):
class EndpointGenericScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
list_display = ('endpoint', 'domain', 'type', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on',
'comply_or_explain_is_explained', 'explain')
'comply_or_explain_is_explained', 'explain', 'is_the_latest_scan')
search_fields = ('endpoint__url__url', 'type', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on')
list_filter = ['endpoint__url__organization__country', 'endpoint__url__organization__type__name',
......@@ -198,13 +201,14 @@ class EndpointGenericScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
'endpoint__protocol',
'endpoint__port', 'endpoint__ip_version', 'endpoint__discovered_on', 'endpoint__is_dead',
'comply_or_explain_is_explained', 'comply_or_explain_explained_on',
'comply_or_explain_case_handled_by', 'comply_or_explain_explanation_valid_until'
'comply_or_explain_case_handled_by', 'comply_or_explain_explanation_valid_until',
'is_the_latest_scan'
][::-1]
fieldsets = (
(None, {
'fields': ('endpoint', 'type', 'rating', 'explanation',
'evidence', 'last_scan_moment', 'rating_determined_on')
'evidence', 'last_scan_moment', 'rating_determined_on', 'is_the_latest_scan')
}),
('comply or explain', {
'fields': ('comply_or_explain_is_explained', 'comply_or_explain_explanation_valid_until',
......@@ -218,25 +222,27 @@ class EndpointGenericScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
return format_html("<a href='./{}/change/#/tab/module_1/'>Explain</a>", object.pk)
readonly_fields = ['endpoint', 'type', 'rating', 'explanation', 'evidence', 'last_scan_moment',
'rating_determined_on']
'rating_determined_on', 'is_the_latest_scan']
@admin.register(UrlGenericScan)
class UrlGenericScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
list_display = ('url', 'domain', 'type', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on',
'comply_or_explain_is_explained', 'explain')
'comply_or_explain_is_explained', 'explain', 'is_the_latest_scan')
search_fields = ('url__url', 'type', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on')
list_filter = ['url__organization__country', 'url__organization__type__name', 'type', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on',
'comply_or_explain_is_explained', 'comply_or_explain_explained_on',
'comply_or_explain_case_handled_by', 'comply_or_explain_explanation_valid_until'
'comply_or_explain_case_handled_by', 'comply_or_explain_explanation_valid_until',
'is_the_latest_scan'
][::-1]
fieldsets = (
(None, {
'fields': ('url', 'type', 'rating', 'explanation', 'evidence', 'last_scan_moment', 'rating_determined_on')
'fields': ('url', 'type', 'rating', 'explanation', 'evidence', 'last_scan_moment', 'rating_determined_on',
'is_the_latest_scan')
}),
('comply or explain', {
'fields': ('comply_or_explain_is_explained', 'comply_or_explain_explanation_valid_until',
......@@ -249,7 +255,8 @@ class UrlGenericScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
def explain(self, object):
return format_html("<a href='./{}/change/#/tab/module_1/'>Explain</a>", object.pk)
readonly_fields = ['url', 'type', 'rating', 'explanation', 'evidence', 'last_scan_moment', 'rating_determined_on']
readonly_fields = ['url', 'type', 'rating', 'explanation', 'evidence', 'last_scan_moment', 'rating_determined_on',
'is_the_latest_scan']
@admin.register(EndpointGenericScanScratchpad)
......
import logging
from datetime import datetime
import pytz
from django.core.management.base import BaseCommand
from failmap.scanners.models import EndpointGenericScan, TlsQualysScan, TlsScan, UrlGenericScan
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""Determines which scans are the latest and sets the latest scan flag. Normally this would happen automatically
using the scan manager. But the flags are empty in older systems.
The queries used allow sliding through time, for example setting the latest on a certain date. Which does not
make sense at all, but works."""
def handle(self, *args, **options):
reflag_tlssscan()
reflag_tls_qualysscan()
reflag_urlgenericscan(type="DNSSEC")
reflag_endpointgenericscan(type="X-XSS-Protection")
reflag_endpointgenericscan(type="Strict-Transport-Security")
reflag_endpointgenericscan(type="X-Frame-Options")
reflag_endpointgenericscan(type="X-Content-Type-Options")
def reflag_tlssscan():
log.debug("Setting flags on tlsscan type")
TlsScan.objects.all().update(is_the_latest_scan=False)
# get the latest scans
sql = '''
SELECT
id,
last_scan_moment,
is_the_latest_scan
FROM scanners_tlsscan
INNER JOIN
(SELECT MAX(id) as id2 FROM scanners_tlsscan egs2
WHERE `last_scan_moment` <= '%(when)s' GROUP BY endpoint_id
) as x
ON x.id2 = scanners_tlsscan.id
''' % {'when': datetime.now(pytz.utc)}
updatescans(TlsScan.objects.raw(sql))
def reflag_tls_qualysscan():
log.debug("Setting flags on tls_qualysscan type: %s")
TlsQualysScan.objects.all().update(is_the_latest_scan=False)
# get the latest scans
sql = '''
SELECT
id,
last_scan_moment,
is_the_latest_scan
FROM scanner_tls_qualys
INNER JOIN
(SELECT MAX(id) as id2 FROM scanner_tls_qualys egs2
WHERE `last_scan_moment` <= '%(when)s' GROUP BY endpoint_id
) as x
ON x.id2 = scanner_tls_qualys.id
''' % {'when': datetime.now(pytz.utc)}
updatescans(TlsQualysScan.objects.raw(sql))
def reflag_urlgenericscan(type):
log.debug("Setting flags on UrlGenericScan type: %s" % type)
UrlGenericScan.objects.all().filter(type=type).update(is_the_latest_scan=False)
# get the latest scans
sql = '''
SELECT
id,
last_scan_moment,
is_the_latest_scan
FROM scanners_urlgenericscan
INNER JOIN
(SELECT MAX(id) as id2 FROM scanners_urlgenericscan egs2
WHERE `last_scan_moment` <= '%(when)s' and type = '%(type)s' GROUP BY url_id
) as x
ON x.id2 = scanners_urlgenericscan.id
''' % {'when': datetime.now(pytz.utc), 'type': type}
updatescans(UrlGenericScan.objects.raw(sql))
def reflag_endpointgenericscan(type):
log.debug("Setting flags on EndpointGenericScan type: %s" % type)
EndpointGenericScan.objects.all().filter(type=type).update(is_the_latest_scan=False)
# get the latest endpointgenericscans
sql = '''
SELECT
id,
last_scan_moment,
is_the_latest_scan
FROM scanners_endpointgenericscan
INNER JOIN
(SELECT MAX(id) as id2 FROM scanners_endpointgenericscan egs2
WHERE `last_scan_moment` <= '%(when)s' and type = '%(type)s' GROUP BY endpoint_id
) as x
ON x.id2 = scanners_endpointgenericscan.id
''' % {'when': datetime.now(pytz.utc), 'type': type}
updatescans(EndpointGenericScan.objects.raw(sql))
def updatescans(scans):
log.debug("Updating %s scans" % len(list(scans)))
for scan in scans:
scan.is_the_latest_scan = True
scan.save()
# Generated by Django 2.0.8 on 2018-10-31 13:00
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('scanners', '0048_auto_20181009_1315'),
]
operations = [
migrations.AddField(
model_name='endpointgenericscan',
name='is_the_latest_scan',
field=models.BooleanField(
default=False, help_text='Notes if this was the latest scan for this url/endpoint. Scanmanagers set this value.'),
),
migrations.AddField(
model_name='tlsqualysscan',
name='is_the_latest_scan',
field=models.BooleanField(
default=False, help_text='Notes if this was the latest scan for this url/endpoint. Scanmanagers set this value.'),
),
migrations.AddField(
model_name='tlsscan',
name='is_the_latest_scan',
field=models.BooleanField(
default=False, help_text='Notes if this was the latest scan for this url/endpoint. Scanmanagers set this value.'),
),
migrations.AddField(
model_name='urlgenericscan',
name='is_the_latest_scan',
field=models.BooleanField(
default=False, help_text='Notes if this was the latest scan for this url/endpoint. Scanmanagers set this value.'),
),
]
......@@ -208,6 +208,30 @@ def one_year_in_the_future():
return datetime.now(pytz.utc) + timedelta(days=365)
class LatestScanMixin(models.Model):
"""
This contains a boolean field that notes if this was the latest scan for this url/endpoint.
This is needed to make sure scanners can easily filter on the latest result on a certain url. There is no
django ORM construct for this. The latest is automatically maintained by the scanmanager, which will both
set new scans as being the latest while unsetting all previous scans to not be the latest.
url: date: value: latest:
example.com 1 april F False
example.com 2 april B False
example.com 3 april A True
... etc
"""
is_the_latest_scan = models.BooleanField(
default=False,
help_text="Notes if this was the latest scan for this url/endpoint. Scanmanagers set this value.",
)
class Meta:
abstract = True
class ExplainMixin(models.Model):
"""
An explanation excludes the grading to impact the report. The result is added in the report,
......@@ -290,7 +314,7 @@ class ExplainMixin(models.Model):
abstract = True
class TlsQualysScan(ExplainMixin):
class TlsQualysScan(ExplainMixin, LatestScanMixin):
"""
Model for scanner tls qualys
"""
......@@ -330,7 +354,7 @@ class TlsQualysScan(ExplainMixin):
return "%s - %s" % (self.scan_date, self.qualys_rating)
class TlsScan(ExplainMixin):
class TlsScan(ExplainMixin, LatestScanMixin):
"""
Model for our own TLS scans
"""
......@@ -379,7 +403,7 @@ class TlsScan(ExplainMixin):
# https://docs.djangoproject.com/en/dev/topics/db/models/#id6
class GenericScanMixin(ExplainMixin):
class GenericScanMixin(ExplainMixin, LatestScanMixin):
"""
This is a fact, a point in time.
"""
......
......@@ -44,8 +44,12 @@ class EndpointScanManager:
gs.evidence = evidence
gs.last_scan_moment = datetime.now(pytz.utc)
gs.rating_determined_on = datetime.now(pytz.utc)
gs.is_the_latest_scan = True
gs.save()
EndpointGenericScan.objects.all().filter(endpoint=gs.endpoint, type=gs.type).exclude(
pk=gs.pk).update(is_the_latest_scan=False)
@staticmethod
def had_scan_with_points(scan_type: str, endpoint: Endpoint):
"""
......
......@@ -43,8 +43,11 @@ class TlsScanManager:
gs.evidence = evidence
gs.last_scan_moment = datetime.now(pytz.utc)
gs.rating_determined_on = datetime.now(pytz.utc)
gs.is_the_latest_scan = True
gs.save()
TlsScan.objects.all().filter(endpoint=gs.endpoint).exclude(pk=gs.pk).update(is_the_latest_scan=False)
@staticmethod
def had_scan_with_points(endpoint: Endpoint):
"""
......
......@@ -44,8 +44,11 @@ class TlsQualysScanManager:
gs.rating_determined_on = datetime.now(pytz.utc)
gs.scan_time = datetime.now(pytz.utc)
gs.scan_date = datetime.now(pytz.utc)
gs.is_the_latest_scan = True
gs.save()
TlsQualysScan.objects.all().filter(endpoint=gs.endpoint).exclude(pk=gs.pk).update(is_the_latest_scan=False)
@staticmethod
def had_scan_with_points(endpoint: Endpoint):
"""
......
......@@ -45,6 +45,9 @@ class UrlScanManager:
gs.rating_determined_on = datetime.now(pytz.utc)
gs.save()
UrlGenericScan.objects.all().filter(url=gs.url, type=gs.type).exclude(
pk=gs.pk).update(is_the_latest_scan=False)
@staticmethod
def had_scan_with_points(scan_type: str, url: Url):
"""
......
......@@ -13,6 +13,7 @@ def compose_task(
organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),
**kwargs
) -> Task:
"""
Helps with identifying issues with scanners. It shows the relevant permissions, configurations and lists the
......
......@@ -114,7 +114,7 @@ def url_by_filters(organizations_filter: dict = dict(), urls_filter: dict = dict
def nsec_compose_task(organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),) -> Task:
endpoints_filter: dict = dict(), **kwargs) -> Task:
if not allowed_to_discover("nsec_compose_task"):
return group()
......@@ -134,7 +134,7 @@ def certificate_transparency(organizations: List[Organization] = None, urls: Lis
def certificate_transparency_compose_task(organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),) -> Task:
endpoints_filter: dict = dict(), **kwargs) -> Task:
if not allowed_to_discover("certificate_transparency_compose_task"):
return group()
......@@ -149,7 +149,7 @@ def certificate_transparency_compose_task(organizations_filter: dict = dict(),
def compose_discover_task(organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),) -> Task:
endpoints_filter: dict = dict(), **kwargs) -> Task:
# these approaches have the highest chance of getting new subdomains.
if not allowed_to_discover("certificate_transparency_compose_task"):
......@@ -174,7 +174,7 @@ def compose_discover_task(organizations_filter: dict = dict(),
# it will not revive anything(!) Should that be a revive task?
def compose_verify_task(organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),) -> Task:
endpoints_filter: dict = dict(), **kwargs) -> Task:
# instead of only checking by domain, just accept the filters as they are handled in any other scenario...
......@@ -249,7 +249,7 @@ def brute_known_subdomains(organizations: List[Organization] = None, urls: List[
def brute_known_subdomains_compose_task(organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),) -> Task:
endpoints_filter: dict = dict(), **kwargs) -> Task:
if not allowed_to_discover("brute_known_subdomains_compose_task"):
return group()
......
......@@ -48,6 +48,7 @@ def compose_task(
organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),
**kwargs
) -> Task:
""" Compose taskset to scan toplevel domains.
......
......@@ -33,6 +33,7 @@ def compose_task(
organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),
**kwargs
) -> Task:
"""Compose taskset to scan specified endpoints.
......
......@@ -35,6 +35,7 @@ def compose_task(
organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),
**kwargs
) -> Task:
"""
Todo's:
......@@ -77,7 +78,7 @@ def compose_task(
def compose_discover_task(organizations_filter: dict = dict(), urls_filter: dict = dict(),
endpoints_filter: dict = dict()) -> Task:
endpoints_filter: dict = dict(), **kwargs) -> Task:
# ports = [21, 990, 2811, 5402, 6622, 20, 2121, 212121] # All types different default ports.
ports = [21]
urls = Url.objects.all().filter(q_configurations_to_scan(level='url'), not_resolvable=False, is_dead=False)
......@@ -96,7 +97,7 @@ def compose_discover_task(organizations_filter: dict = dict(), urls_filter: dict
def compose_verify_task(organizations_filter: dict = dict(), urls_filter: dict = dict(),
endpoints_filter: dict = dict()) -> Task:
endpoints_filter: dict = dict(), **kwargs) -> Task:
default_filter = {"protocol": "ftp", "is_dead": False}
endpoints_filter = {**endpoints_filter, **default_filter}
......
......@@ -62,6 +62,7 @@ def compose_discover_task(
organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),
**kwargs
) -> Task:
"""Compose taskset to scan specified endpoints.
......@@ -129,6 +130,7 @@ def compose_verify_task(
organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),
**kwargs
) -> Task:
"""Verifies existing https and http endpoints. Is pretty quick, as it will not stumble upon non-existing services
as much.
......
......@@ -17,6 +17,7 @@ def compose_task(
organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),
**kwargs
) -> Task:
"""Multi-stage onboarding."""
......
......@@ -31,6 +31,7 @@ def compose_task(
organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),
**kwargs
) -> Task:
# We might not be allowed to scan for this at all.
......
......@@ -62,6 +62,7 @@ def compose_task(
organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),
**kwargs
) -> Task:
# We might not be allowed to scan for this at all.
if not allowed_to_scan("scanner_screenshot"):
......
......@@ -25,6 +25,7 @@ def compose_task(
organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),
**kwargs
) -> Task:
"""Compose taskset to scan specified endpoints.
"""
......
......@@ -144,6 +144,7 @@ def compose_task(
organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),
**kwargs
) -> Task:
if not allowed_to_scan("scanner_tls_osaft"):
......
......@@ -59,6 +59,7 @@ def compose_task(
organizations_filter: dict = dict(),
urls_filter: dict = dict(),
endpoints_filter: dict = dict(),
**kwargs
) -> Task:
# for some reason declaring the chunks function outside of this function does not resolve. I mean... why?
......@@ -93,9 +94,7 @@ def compose_task(
endpoint__port=443,
endpoint__is_dead=False,
organization__in=organizations, # whem empty, no results...
**urls_filter,
).exclude(endpoint__tlsqualysscan__last_scan_moment__gte=datetime.now(tz=pytz.utc) - timedelta(days=7)
).order_by("?")
**urls_filter)
else:
urls = Url.objects.filter(
q_configurations_to_scan(),
......@@ -104,9 +103,15 @@ def compose_task(
endpoint__protocol="https",
endpoint__port=443,
endpoint__is_dead=False,
**urls_filter,
).exclude(endpoint__tlsqualysscan__last_scan_moment__gte=datetime.now(tz=pytz.utc) - timedelta(days=7)
).order_by("?")
**urls_filter)
# exclude everything that has been scanned in the last N days
# this is a separate filter, given you cannot perform date logic in json (aka code injection)
if kwargs.get('exclude_urls_scanned_in_the_last_n_days', 0):
days = int(kwargs.get('exclude_urls_scanned_in_the_last_n_days', 0))
log.debug("Skipping everything that has been scanned in the last %s days." % days)
urls = urls.exclude(
endpoint__tlsqualysscan__last_scan_moment__gte=datetime.now(tz=pytz.utc) - timedelta(days=days))
# Urls are ordered randomly.