Verified Commit 385a378b authored by Elger Jonker's avatar Elger Jonker

unique urls only

parent c4e65f0e
Pipeline #39599290 failed with stage
in 12 minutes and 19 seconds
......@@ -81,16 +81,19 @@ def compose_discover_task(
organizations = Organization.objects.filter(**organizations_filter)
# apply filter to urls in organizations (or if no filter, all urls)
urls = Url.objects.filter(q_configurations_to_scan(), organization__in=organizations, **urls_filter)
urls = list(set(urls))'Creating http scan task for %s urls for %s organizations.', len(urls), len(organizations))
urls = Url.objects.filter(q_configurations_to_scan(), **urls_filter)
urls = list(set(urls))'Creating http scan task for %s urls.', len(urls))
if endpoints_filter:
log.warning("Endpoint filters are not implemented: filter has no effect.")
# make sure we're dealing with a list for the coming random function
urls = list(urls)
# randomize the endpoints to better spread load over urls.
tasks = []
......@@ -56,6 +56,9 @@ def compose_task(
'URL Filter: %s, endpoints_filter: %s', organizations_filter, urls_filter, endpoints_filter)
return group()
# unique endpoints only
endpoints = list(set(endpoints))'Creating security headers scan task for %s endpoints for %s urls for %s organizations.',
len(endpoints), len(urls), len(organizations))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment