Verified Commit 4e56c02a authored by Elger Jonker's avatar Elger Jonker

[wip] added geolocation, improved interface and handling

parent 0cac431c
......@@ -47,13 +47,13 @@ class ContestAdmin(ImportExportModelAdmin, admin.ModelAdmin):
@admin.register(Team)
class TeamAdmin(ImportExportModelAdmin, admin.ModelAdmin):
list_display = ('name', 'participating_in_contest', 'allowed_to_submit_things')
list_display = ('name', 'color', 'participating_in_contest', 'allowed_to_submit_things')
search_fields = ('name', 'participating_in_contest__name')
list_filter = ('name', 'participating_in_contest__name', 'participating_in_contest__target_country')
fieldsets = (
(None, {
'fields': ('name', 'participating_in_contest', 'allowed_to_submit_things')
'fields': ('name', 'color', 'participating_in_contest', 'allowed_to_submit_things')
}),
('secret', {
'fields': ('secret', ),
......
......@@ -4,9 +4,6 @@ from datetime import datetime
import pytz
import tldextract
from crispy_forms.bootstrap import AppendedText, FormActions, PrependedText
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Field, Layout, Submit
from dal import autocomplete
# from django.contrib.gis import forms # needs gdal, which...
from django import forms
......@@ -15,6 +12,8 @@ from django.forms import ValidationError
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from django_countries.fields import CountryField
from django_countries.widgets import CountrySelectWidget
from mapwidgets.widgets import GooglePointFieldWidget
from failmap.game.models import Contest, OrganizationSubmission, Team, UrlSubmission
from failmap.organizations.models import Organization, OrganizationType, Url
......@@ -92,33 +91,68 @@ class TeamForm(forms.Form):
# http://django-autocomplete-light.readthedocs.io/en/master/tutorial.html
class OrganisationSubmissionForm(forms.Form):
field_order = ('organization_type_name', 'organization_name', 'organization_address', 'organization_evidence')
field_order = ('organization_country', 'organization_type_name', 'organization_name',
'organization_address_geocoded', 'organization_address',
'organization_evidence')
organization_country = CountryField().formfield(
label="Country",
widget=CountrySelectWidget(),
initial="NL"
)
# todo: filter based on country and organization type.
# todo: but how to suggest new organizations?
organization_type_name = forms.ModelChoiceField(
queryset=OrganizationType.objects.all(),
widget=autocomplete.ModelSelect2(url='/game/autocomplete/organization-type-autocomplete/')
widget=autocomplete.ModelSelect2(url='/game/autocomplete/organization-type-autocomplete/'),
label="Type"
)
organization_name = forms.CharField()
organization_name = forms.CharField(
label="Name"
)
organization_wikipedia = forms.URLField()
organization_address_geocoded = forms.CharField(
widget=GooglePointFieldWidget,
label="Address"
)
organization_address = forms.CharField(widget=forms.Textarea)
organization_address = forms.CharField(
widget=forms.Textarea
)
organization_evidence = forms.CharField(widget=forms.Textarea)
organization_evidence = forms.CharField(
widget=forms.Textarea,
label="Sources verifying the existence of this organization"
)
organization_wikipedia = forms.URLField(
label="Wikipedia page",
help_text="To quickly find the correct wiki page, start a search by "
"clicking <a href='https://en.wikipedia.org/w/index.php?search="
"ministry+van+binnenlandse+zaken&title=Special:Search&go=Go'>here: search wikipedia</a>."
)
organization_wikidata = forms.CharField(
label="Wikidata code",
help_text="Find a Q code on <a href='https://www.wikidata.org/wiki/"
"Wikidata:Main_Page' target='_blank'>wikidata</a>."
)
# todo: clean the geolocated address to fit the rest of the system. The ugly
# POINT -() etc has to be formatted according our normal layout so it can be processed in the admin
# interface.
def clean(self):
# verify that an organization of this type is not in the database yet...
cleaned_data = super().clean()
organization_type_name = cleaned_data.get("organization_type_name")
name = cleaned_data.get("organization_name")
country = cleaned_data.get("organization_country")
# todo: allow adding for any country...
# todo: this is now a setting in the contest... should be availble somewhere...
exists = Organization.objects.all().filter(
type=organization_type_name, name=name, is_dead=False, country='NL').exists()
type=organization_type_name, name=name, is_dead=False, country=country).exists()
if exists:
raise ValidationError(
......@@ -129,11 +163,14 @@ class OrganisationSubmissionForm(forms.Form):
@transaction.atomic
def save(self, team):
organization_country = self.cleaned_data.get('organization_country', None)
organization_type_name = self.cleaned_data.get('organization_type_name', None)
organization_name = self.cleaned_data.get('organization_name', None)
organization_address = self.cleaned_data.get('organization_address', None)
organization_evidence = self.cleaned_data.get('organization_address', None)
organization_wikipedia = self.cleaned_data.get('organization_wikipedia', None)
organization_wikidata = self.cleaned_data.get('organization_wikidata', None)
organization_address_geocoded = self.cleaned_data.get('organization_address_geocoded', None)
if not all([organization_name, organization_type_name, organization_address, organization_evidence]):
raise forms.ValidationError(
......@@ -147,6 +184,9 @@ class OrganisationSubmissionForm(forms.Form):
organization_name=organization_name,
organization_type_name=organization_type_name,
organization_wikipedia=organization_wikipedia,
organization_wikidata_code=organization_wikidata,
organization_address_geocoded=organization_address_geocoded,
organization_country=organization_country,
added_on=timezone.now(),
has_been_accepted=False,
has_been_rejected=False
......@@ -158,11 +198,13 @@ class UrlSubmissionForm(forms.Form):
field_order = ('country', 'organization_type_name', 'for_organization', 'url',)
country = CountryField().formfield(
label="🔍 Filter organization by country",
required=False,
help_text="This only helps finding the correct organization."
)
organization_type_name = forms.ModelChoiceField(
label="🔍 Filter organization by organization type",
queryset=OrganizationType.objects.all(),
widget=autocomplete.ModelSelect2(url='/game/autocomplete/organization-type-autocomplete/',
forward=['country']),
......@@ -172,35 +214,23 @@ class UrlSubmissionForm(forms.Form):
for_organization = forms.ModelMultipleChoiceField(
queryset=Organization.objects.all(),
widget=autocomplete.ModelSelect2Multiple(url='/game/autocomplete/organization-autocomplete/',
forward=['organization_type_name', 'country'])
widget=autocomplete.ModelSelect2Multiple(
url='/game/autocomplete/organization-autocomplete/',
forward=['organization_type_name', 'country']),
help_text="Only approved organization are shown in this list. If your submitted organization is missing, please"
" ask the competition manager to verify your organization."
)
url = forms.CharField(
help_text="Do NOT enter http:// or https://"
)
helper = FormHelper()
helper.form_class = 'form-horizontal'
helper.layout = Layout(
Field('text_input', css_class='input-xlarge'),
Field('textarea', rows="3", css_class='input-xlarge'),
'radio_buttons',
Field('checkboxes', style="background: #FAFAFA; padding: 10px;"),
AppendedText('appended_text', '.00'),
PrependedText('prepended_text', '<input type="checkbox" checked="checked" value="" id="" name="">',
active=True),
PrependedText('prepended_text_two', '@'),
'multicolon_select',
FormActions(
Submit('save_changes', 'Save changes', css_class="btn-primary"),
Submit('cancel', 'Cancel'),
)
help_text=""
)
def clean_url(self):
url = self.cleaned_data['url']
url = url.replace("https://", "")
url = url.replace("http://", "")
extract = tldextract.extract(url)
if not extract.suffix:
raise ValidationError(
......@@ -229,10 +259,11 @@ class UrlSubmissionForm(forms.Form):
organisations = cleaned_data.get("for_organization")
url = cleaned_data.get("url")
if not organisations or not url:
raise forms.ValidationError(
"Fix the errors."
)
if not organisations:
raise forms.ValidationError("Organization missing!")
if not url:
raise forms.ValidationError("Url missing!")
log.info(organisations)
......
# Generated by Django 2.0.7 on 2018-07-24 13:04
import colorful.fields
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('game', '0008_organizationsubmission_organization_wikidata_code'),
]
operations = [
migrations.AddField(
model_name='team',
name='color',
field=colorful.fields.RGBColorField(blank=True, colors=['#FF0000', '#00FF00', '#0000FF'], null=True),
),
]
from colorful.fields import RGBColorField
from django.db import models
from django.utils.translation import gettext_lazy as _
from django_countries.fields import CountryField
......@@ -74,6 +75,13 @@ class Team(models.Model):
on_delete=models.CASCADE
)
color = RGBColorField(
colors=['#F2D7D5', '#FADBD8', '#EBDEF0', '#E8DAEF', '#D4E6F1', '#D6EAF8', '#D1F2EB', '#D0ECE7', '#D4EFDF',
'#D5F5E3', '#FCF3CF', '#FDEBD0', '#FAE5D3', '#F6DDCC'],
null=True,
blank=True,
)
allowed_to_submit_things = models.BooleanField(
default=False,
help_text="Disables teams from submitting things."
......
......@@ -77,8 +77,8 @@
<div class="navbar-collapse">
<ul class="nav navbar-nav navbar-left">
<li><a href="/game/scores/">Scores</a></li>
<li><a href="/game/contests/">Select Contest</a></li>
<li><a href="/game/team/">Select Team</a></li>
<li><a href="/game/contests/">Contest</a></li>
<li><a href="/game/team/">Teams</a></li>
<li class="dropdown">
<a href="/game/submit_organization/" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Submit Organization <span class="caret"></span></a>
<ul class="dropdown-menu">
......@@ -95,6 +95,7 @@
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li><a href="/game/rules_help/">? Rules / Help</a></li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Log on/off<span class="caret"></span></a>
<ul class="dropdown-menu">
......
......@@ -18,10 +18,6 @@
</div>
{% endif %}
<p>
Current Contest: {{ contest.name }}
</p>
{% comment "Find out how to re-use a list." %}
{% endcomment %}
......@@ -29,24 +25,23 @@
<table class="table">
<thead>
<tr>
<th>#</th>
<th></th>
<th>Name</th>
<th>Info</th>
<th>From</th>
<th>Until</th>
<th>Filters</th>
</tr>
</thead>
<tbody>
{% for contest in active_contests %}
<tr>
<td><form method="POST">{% csrf_token %}<input type="hidden" name="id" value="{{ contest.id }}">
{% for a_contest in active_contests %}
<tr style="{% if a_contest.name == contest.name %} font-weight: bold; background-color: #ddffdd {% endif %}">
<td><form method="POST">{% csrf_token %}<input type="hidden" name="id" value="{{ a_contest.id }}">
<input type="submit" value="join"></form></td>
<td><img src="{{ contest.logo }}" height="25"/></td>
<td>{{ contest.name }}</td>
<td>{{ contest.from_moment }}</td>
<td>{{ contest.until_moment }} <div id="countdown_{{ contest.id }}">countdown...</div></td>
<td>{{ contest.filters }}</td>
<td><img class="country-select-flag" id="{{ a_contest.target_country }}" style="margin: 6px 4px 0" src="/static/flags/{{ a_contest.target_country }}.gif">
{{ a_contest.name }}</td>
<td><a href="/game/scores/?contest={{ a_contest.id }}">view scores</a></td>
<td>{{ a_contest.from_moment }}</td>
<td>{{ a_contest.until_moment }}</td>
</tr>
{% empty %}
<tr><td colspan="6">-</td></tr>
......@@ -58,23 +53,23 @@
<table class="table">
<thead>
<tr>
<th>#</th>
<th>Logo</th>
<th></th>
<th>Name</th>
<th>Info</th>
<th>From</th>
<th>Until</th>
<th>Filters</th>
</tr>
</thead>
<tbody>
{% for contest in future_contests %}
<tr>
<td>{{ forloop.counter }}</td>
<td>{{ contest.logo }}</td>
<td>{{ contest.name }}</td>
<td>{{ contest.from }}</td>
<td>{{ contest.until }}</td>
<td>{{ contest.filters }}</td>
{% for a_contest in future_contests %}
<tr style="{% if a_contest.name == contest.name %} font-weight: bold; background-color: #ddffdd {% endif %}">
<td><form method="POST">{% csrf_token %}<input type="hidden" name="id" value="{{ a_contest.id }}">
<input type="submit" value="join"></form></td>
<td><img class="country-select-flag" id="{{ a_contest.target_country }}" style="margin: 6px 4px 0" src="/static/flags/{{ a_contest.target_country }}.gif">
{{ a_contest.name }}</td>
<td><a href="/game/scores/?contest={{ a_contest.id }}">view scores</a></td>
<td>{{ a_contest.from_moment }}</td>
<td>{{ a_contest.until_moment }}</td>
</tr>
{% empty %}
<tr><td colspan="6">-</td></tr>
......@@ -86,23 +81,19 @@
<table class="table">
<thead>
<tr>
<th>#</th>
<th>Logo</th>
<th>Name</th>
<th>From</th>
<th>Until</th>
<th>Filters</th>
</tr>
</thead>
<tbody>
{% for contest in expired_contests %}
{% for a_contest in expired_contests %}
<tr>
<td>{{ forloop.counter }}</td>
<td>{{ contest.logo }}</td>
<td>{{ contest.name }}</td>
<td>{{ contest.from }}</td>
<td>{{ contest.until }}</td>
<td>{{ contest.filters }}</td>
<td><img class="country-select-flag" id="{{ a_contest.target_country }}" style="margin: 6px 4px 0" src="/static/flags/{{ a_contest.target_country }}.gif">
{{ a_contest.name }}</td>
<td><a href="/game/scores/?contest={{ a_contest.id }}">view scores</a></td>
<td>{{ a_contest.from_moment }}</td>
<td>{{ a_contest.until_moment }}</td>
</tr>
{% empty %}
<tr><td colspan="6">-</td></tr>
......
{% extends 'game/base.html' %}
{% load crispy_forms_tags %}
{% block content %}
<h1>Rules</h1>
<p>This page outlines some help / rules for joining this session.</p>
<h3>Rule 1: Quality over quantity</h3>
<p>Having a great dataset will increase the likelyhood of this tool being taken serious. It also makes the downloadable
datadumps of failmap a pleasure to use. Please take care that you're adding information that is genuine and of quality.
To assist you, there are pages that list already
<a href="/game/submitted_organizations/">known organizations</a> and
<a href="/game/submitted_urls/">known urls</a>: you can CTRL+F through it.</p>
<h3>Rule 2: Be excellent to each other</h3>
<p>Please help each other and try to answer each others questions.</p>
<h3>Rule 3: Don't cause abuse</h3>
<p>Make sure that you're not abusing online services or online infrastructure in your quest to find more information.
Usually there is a better way to find what you're looking for.</p>
<h1>Help</h1>
<h3>What kind of organizations can i add?</h3>
<p>There are various organization types, which all fall under "governmental" or "vital infrastructure".</p>
<p>You can add all governmental organizations and "agencies" that are exclusively government appointed.</p>
<p><b>Do not</b> submit urls of non-governmental organizations such as 3rd party companies that deliver services to the
government: for example, do not add the url of the webdesigner for the organization.</p>
<p><b>Do not</b> add urls of specific people such as linked-in pages.</p>
<h3>What can i do if i find a serious vulnerability</h3>
<p>Responsible disclosure. Read this: https://www.ncsc.nl/english/current-topics/news/responsible-disclosure-guideline.html</p>
<h3>Can i use automated submissions: yes, and be warned.</h3>
<p>Yes, you can submit urls automatically. But do make sure that you're taking into account using the CSRF value
and make sure your submissions are quality. Any submission that is not of quality will cost someone a lot of
time validating. Be sure to check first, especially on wildcard domains.</p>
<p>Be warned though: if you submit too much rubbish data, you / your team get kicked and all submitted data baleeted.</p>
<h3>What happens after something is added?</h3>
<p>For each url that has been added, an onboarding process starts. This process consists of the following steps:</p>
<ol>
<li>Trying to contact the url: finding endpoints of various services.</li>
<li>Perform tests on the endpoints.</li>
<li>If url is top level: Automatically determine if there are more subdomains.</li>
<li>Update the report on the website.</li>
</ol>
<h3>Where can i download all submitted information?</h3>
<p>
The filtered, higher quality, dataset can be downloaded from the failmap website we're currently working for.
Look at the bottom of the page at datasets. You'll find ready made datasets for various purposes.
</p>
<h3>Where can i find more domains / subdomains?</h3>
<p>There are many sources for subdomains:</p>
<ul>
<li>Various online services index subdomains and may have some unique ones.</li>
<li>If you have access to DNS servers or services, you might be able to get subdomains from that.</li>
<li>Certificate transparency databases such as crt.sh have long lists of subdomains and domains for which
TLS certificates are requested. Note that some of these may be outdated, expired and will miss all sites that
do not use encryption (no encryption = lots of points).</li>
<li>Tools like theHarvester and DNSrecon can help you find subdomains: this is also what this website does
after onboarding: use various standard tools to find subdomains in a friendly way.</li>
<li>Crawling the site, or searching using smart search queries on the larger search engines.</li>
</ul>
{% endblock %}
{% extends 'game/base.html' %}
{% block content %}
<h1>Scores</h1>
<p>
Your Team: {{ team.name }}
</p>
<p>Contest: {{ contest.name }}, from {{ contest.from_moment }} until {{ contest.until_moment }}.</p>
<p>This contest targets {{ contest.target_country }}.</p>
<table class="table">
<thead>
<tr>
<th>&nbsp;</th>
<th>&nbsp;</th>
<th colspan="3" style="background-color: lightgrey">Issues found per severity</th>
<th>&nbsp;</th>
<th>&nbsp;</th>
<th>&nbsp;</th>
<th>&nbsp;</th>
</tr>
<tr>
<th>Rank</th>
<th>Team</th>
......@@ -15,18 +21,20 @@
<th>Medium</th>
<th>Low</th>
<th>Rejected</th>
<th>Added organizations</th>
<th>Total</th>
</tr>
</thead>
<tbody>
{% for score in scores %}
<tr style="{% if team.name == score.team %} font-weight: bold; background-color: #ddffdd {% endif %}">
<tr style="{% if team.name == score.team %} font-weight: bold; background-color: #ddffdd;{% endif %}">
<td>{{ forloop.counter }}</td>
<td>{{ score.team }}</td>
<td style="background-color: {{ score.team_color }};">{{ score.team }}</td>
<td>{{ score.high }} * {{ score.high_multiplier }} = {{ score.high_score }}</td>
<td>{{ score.medium }} * {{ score.medium_multiplier }} = {{ score.medium_score }}</td>
<td>{{ score.low }} * {{ score.low_multiplier }} = {{ score.low_score }}</td>
<td>{{ score.rejected }} * {{ score.rejected_multiplier }} = {{ score.rejected_score }}</td>
<td>{{ score.added_organizations }} * {{ score.added_organizations_multiplier }} = {{ score.added_organizations_score }}</td>
<td>{{ score.total_score }}</td>
</tr>
{% empty %}
......
{% extends 'game/base.html' %}
{% load crispy_forms_tags %}
{% block content %}
<style>
.mw-wrap {
width: 100% !important;
}
</style>
<script>
$(document).on("google_point_map_widget:marker_create", function (e, place, lat, lng, locationInputElem, mapWrapID, something) {
console.log("e");
console.log(e);
console.log("place");
console.log(place); // google place object
console.log("lat");
console.log(lat); // created marker coordinates
console.log("lng");
console.log(lng); // created marker coordinates
console.log("wdiget");
console.log(locationInputElem); // django widget textarea widget (hidden)
console.log("mapwrapid");
console.log(mapWrapID); // map widget wrapper element ID
});
</script>
{% if success %}
<div class="alert alert-success" role="alert">
<strong>Organization added!</strong> organization has been added!
......@@ -16,6 +37,11 @@
{% endif %}
<h1>Suggest a new organisation</h1>
<p>
Suggesting a new organization correctly can take some work, as you need to find enough sources to verify the
organization actually exist. Please specify their website in the evidence, their wikipedia page and wikidata Q
number to complete the process.
</p>
<form method="POST" class="uniForm">{% csrf_token %}
{{ form | crispy }}
<br />
......
......@@ -10,13 +10,21 @@
</div>
{% endif %}
{% comment %}
{% if error %}
<div class="alert alert-danger" role="alert">
<strong>Error!</strong> {{ error }}
</div>
{% endif %}
{% endcomment %}
<h1>Suggest a new url</h1>
<p>
It's possible to submit an URL to multiple organizations.
</p>
<p>
The country and organization type fields can be useful to more easily find the organization you're looking for.
</p>
<form method="POST" class="uniForm">{% csrf_token %}
{{ form | crispy }}
<br />
......
......@@ -2,27 +2,23 @@
{% load crispy_forms_tags %}
{% block content %}
<h1>Submitted urls</h1>
<p>This is a list of submitted urls during this contest. Below it also shows the urls that already exist.
This information can help you to quickly determine if an organization is already being investigated and if some
information is still missing.</p>
<h1>Submitted organizations</h1>
<p>
Current Contest: {{ contest.name }}
</p>
<h2>Submitted urls</h2>
<h2>Submitted organizations</h2>
<p>This is a list of submitted organizations during this contest. Below it also shows the urls that already exist.
This information can help you to quickly determine if an organization is already being investigated and if some
information is still missing.</p>
<table class="table">
<thead>
<tr>
<th>Team</th>
<th>Organization</th>
<th>Country</th>
<th>Type</th>
<th>Address</th>
<th>Wikipedia</th>
<th>Wikidata</th>
<th>Geocode</th>
<th>evidence</th>
<th>Accepted</th>
<th>Rejected</th>
<th>In system</th>
......@@ -31,15 +27,12 @@
</thead>
<tbody>
{% for organization in submitted_organizations %}
<tr>
<tr style="background-color: {{ organization.added_by_team.color }};">
<td>{{ organization.added_by_team.name }}</td>
<td>{{ organization.organization_name }}</td>
<td>{{ organization.organization_country }}</td>
<td>{{ organization.organization_type_name }}</td>
<td>{{ organization.organization_address }}</td>
<td>{{ organization.organization_wikipedia }}</td>
<td>{{ organization.organization_wikidata_code }}</td>
<td>{{ organization.organization_address_geocoded }}</td>
<td>{{ organization.evidence }}</td>
<td>{{ organization.has_been_accepted }}</td>
<td>{{ organization.has_been_rejected }}</td>
<td>{{ organization.organization_in_system }}</td>
......@@ -52,8 +45,8 @@
</table>
<h2>Already known urls</h2>
<p>These urls where already in the database before the contest started.</p>
<h2>Already known organizations</h2>
<p>These organizations where already in the database before the contest started.</p>
<table class="table">
<thead>
<tr>
......
......@@ -26,7 +26,7 @@
</thead>
<tbody>
{% for url in submitted_urls %}
<tr>
<tr style="background-color: {{ url.added_by_team.color }};">
<td>{{ url.added_by_team.name }}</td>
<td>{{ url.for_organization }}</td>
<td>{{ url.url }} </td>
......
......@@ -7,6 +7,7 @@ urlpatterns = [
path('scores/', views.scores),
path('team/', views.teams),
path('submitted_urls/', views.submitted_urls),
path('rules_help/', views.rules_help),
path('submitted_organizations/', views.submitted_organizations),
path('contests/', views.contests),
path('submit_url/', views.submit_url),
......
This diff is collapsed.
......@@ -105,7 +105,8 @@ INSTALLED_APPS = [
'bootstrapform', # Required for nicer formatting of forms with the default templates
'helpdesk', # This is us!
'mapwidgets',
'colorful',
# others:
# 'mapwidgets', no gdal available yet, try again later
# 'cachalot', # query cache, is not faster.
......@@ -782,12 +783,12 @@ LOGOUT_REDIRECT_URL = '/'
MAP_WIDGETS = {
"GooglePointFieldWidget": (
("zoom", 15),
("mapCenterLocationName", "utrecht"),
("GooglePlaceAutocompleteOptions", {'componentRestrictions': {'country': 'nl'}}),
("markerFitZoom", 12),
("zoom", 4),