Commit 5ce2ab96 authored by Elger Jonker's avatar Elger Jonker

unique urls only


Former-commit-id: 385a378b
parent d6a4dd5f
......@@ -81,16 +81,19 @@ def compose_discover_task(
organizations = Organization.objects.filter(**organizations_filter)
# apply filter to urls in organizations (or if no filter, all urls)
urls = Url.objects.filter(q_configurations_to_scan(), organization__in=organizations, **urls_filter)
urls = list(set(urls))
log.info('Creating http scan task for %s urls for %s organizations.', len(urls), len(organizations))
else:
urls = Url.objects.filter(q_configurations_to_scan(), **urls_filter)
urls = list(set(urls))
log.info('Creating http scan task for %s urls.', len(urls))
if endpoints_filter:
log.warning("Endpoint filters are not implemented: filter has no effect.")
# make sure we're dealing with a list for the coming random function
urls = list(urls)
# randomize the endpoints to better spread load over urls.
random.shuffle(urls)
tasks = []
......
......@@ -56,6 +56,9 @@ def compose_task(
'URL Filter: %s, endpoints_filter: %s', organizations_filter, urls_filter, endpoints_filter)
return group()
# unique endpoints only
endpoints = list(set(endpoints))
log.info('Creating security headers scan task for %s endpoints for %s urls for %s organizations.',
len(endpoints), len(urls), len(organizations))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment