Verified Commit d7237a8d authored by Elger Jonker's avatar Elger Jonker

bugfixes, supporting suggested urls

parent f84d8cd7
Pipeline #39275515 passed with stages
in 19 minutes and 15 seconds
......@@ -5,6 +5,7 @@ from datetime import datetime, timedelta
from random import choice, choices, randint
import pytz
import tldextract
from constance import config
from django.contrib import admin
from django.db import transaction
......@@ -17,6 +18,7 @@ from failmap.app.admin import generate_game_user
from failmap.app.models import GameUser
from failmap.game.models import Contest, OrganizationSubmission, Team, UrlSubmission
from failmap.organizations.models import Coordinate, Organization, OrganizationType, Url
from failmap.scanners.scanner.http import resolves
log = logging.getLogger(__package__)
......@@ -463,6 +465,19 @@ class OrganizationSubmissionAdmin(ImportExportModelAdmin, admin.ModelAdmin):
new_coordinate.save()
log.debug('Saved matching coordinate.')
# add the toplevel urls if they exist.
if osm.suggested_urls:
# a disgusting way to parse this list, without using eval.
urls = osm.suggested_urls.replace("[", "").replace("'", "").replace("]", "").replace(",", "").split(" ")
urls = check_valid_urls(urls)
for url in urls:
new_url = Url()
new_url.url = url
new_url.save()
new_url.organization.add(new_org)
new_url.save()
# and save tracking information
osm.organization_in_system = new_org
osm.has_been_accepted = True
......@@ -482,3 +497,28 @@ class OrganizationSubmissionAdmin(ImportExportModelAdmin, admin.ModelAdmin):
self.message_user(request, "Organisation(s) have been rejected.")
reject.short_description = "❌ Reject"
actions.append('reject')
def check_valid_urls(urls):
valid = []
for url in urls:
url = url.lower()
url = url.replace("https://", "")
url = url.replace("http://", "")
extract = tldextract.extract(url)
if not extract.suffix:
continue
# tld extract has also removed ports, usernames, passwords and other nonsense.
url = "%s.%s" % (extract.domain, extract.suffix)
# see if the URL resolves at all:
if not resolves(url):
continue
if url not in valid:
valid.append(url)
return valid
......@@ -396,7 +396,9 @@ class UrlSubmissionForm(forms.Form):
not_resolvable.append(url)
continue
valid.append(url)
# todo: does this add the same url a few times?
if url not in valid:
valid.append(url)
return incomplete, not_resolvable, valid
......
......@@ -114,7 +114,7 @@ class Organization(models.Model):
def __str__(self):
if self.type_id not in self.organization_name_cache:
log.debug("caching...")
# log.debug("caching...")
self.organization_name_cache[self.type_id] = self.type.name
type_label = self.organization_name_cache[self.type_id]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment