Commit c535efc2 authored by Elger Jonker's avatar Elger Jonker Committed by Johan Bloemberg

Johans pull request, HSTS ratings (ipv4), TLS ratings 1xper v4 and v6, manual scans deleted

parent 8d550cf9
......@@ -16,3 +16,5 @@ db_backup_*.sqlite3
failmap_admin/map/static/images/screenshots/*
failmap_admin/scanners/resources/output/dnsrecon/*
failmap_admin/scanners/resources/output/theHarvester/*
dist/
build/
include requirements.txt
recursive-include failmap_admin/*/templates *.html
recursive-include failmap_admin/*/static *
recursive-include failmap_admin/*/fixtures *.yaml *.json
\ No newline at end of file
......@@ -27,8 +27,7 @@ To run code quality checks and unit tests run:
To make life easier you can use `autopep8`/`isort` before running `tox` to automatically fix most style issues:
autopep8 -ri failmap_admin tests
isort -rc failmap_admin tests
tox -e autofix
To run only a specific test use:
......
This diff is collapsed.
......@@ -18,11 +18,13 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='OrganizationRating',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('id', models.AutoField(auto_created=True,
primary_key=True, serialize=False, verbose_name='ID')),
('rating', models.IntegerField()),
('when', models.DateTimeField()),
('calculation', models.TextField()),
('organization', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='organizations.Organization')),
('organization', models.ForeignKey(
on_delete=django.db.models.deletion.PROTECT, to='organizations.Organization')),
],
options={
'managed': True,
......@@ -31,7 +33,8 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='UrlRating',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('id', models.AutoField(auto_created=True,
primary_key=True, serialize=False, verbose_name='ID')),
('rating', models.IntegerField()),
('when', models.DateTimeField()),
('calculation', models.TextField()),
......
......@@ -16,6 +16,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='urlrating',
name='url',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='organizations.Url'),
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to='organizations.Url'),
),
]
......@@ -129,6 +129,7 @@ body {
.reportlist {
display: table;
padding-bottom: 15px;
}
.screenshotlist {
......@@ -245,4 +246,54 @@ body {
border-bottom-right-radius: 10px;
padding-bottom: 15px;
padding-top: 9px;
}
.reportlist h3 {
font-size: 17px;
}
.awarded_points_orange {
border: 1px solid darkorange;
background-color: orange;
padding: 0px 3px;
font-weight: bold;
font-size: 11px;
}
.awarded_points_red {
border: 1px solid darkred;
background-color: red;
padding: 0px 3px;
font-weight: bold;
font-size: 11px;
}
.awarded_points_green {
border: 1px solid green;
background-color: lightgreen;
padding: 0px 3px;
font-weight: bold;
font-size: 11px;
}
.total_awarded_points_orange {
border: 2px solid darkorange;
background-color: orange;
padding: 1px 4px;
font-weight: bold;
}
.total_awarded_points_red {
border: 2px solid darkred;
background-color: red;
padding: 1px 4px;
font-weight: bold;
}
.total_awarded_points_green {
border: 2px solid green;
background-color: lightgreen;
padding: 1px 4px;
font-weight: bold;
}
\ No newline at end of file
......@@ -334,12 +334,12 @@ $(document).ready(function () {
},
methods: {
colorize: function (points) {
if (points < 100) return "green";
if (points < 199) return "green";
if (points < 400) return "orange";
if (points > 399) return "red";
},
colorizebg: function (points) {
if (points < 100) return "#dff9d7";
if (points < 199) return "#dff9d7";
if (points < 400) return "#ffefd3";
if (points > 399) return "#fbeaea";
},
......@@ -353,6 +353,32 @@ $(document).ready(function () {
},
humanize: function (date) {
return new Date(date).humanTimeStamp()
},
create_header: function(rating){
keyz = Object.keys(rating);
if (keyz[0] === "security_headers_strict_transport_security")
return "Strict Transport Security Header (HSTS)";
if (keyz[0] === "tls_qualys")
return "Transport Layer Security (TLS)";
if (keyz[0] === "http_plain")
return "Missing transport security (TLS)";
},
second_opinion_links: function(rating, url){
keyz = Object.keys(rating);
if (keyz[0] === "security_headers_strict_transport_security")
return '<a href="https://securityheaders.io/?q=' + url.url.url + '\" target=\"_blank\">Second Opinion (securityheaders.io)</a>';
if (keyz[0] === "tls_qualys")
return '<a href="https://www.ssllabs.com/ssltest/analyze.html?d=' + url.url.url + '&hideResults=on&latest\" target=\"_blank\">Second Opinion (Qualys)</a>';
},
total_awarded_points: function(points) {
if (points === "0")
marker = "✓ perfect";
else
marker = points;
return '<span class="total_awarded_points_'+ this.colorize(points) +'">' + marker + '</span>'
},
awarded_points: function(points) {
return '<span class="awarded_points_'+ this.colorize(points) +'">+ ' + points + '</span>'
}
}
});
......@@ -396,7 +422,7 @@ $(document).ready(function () {
data: {urls: Array},
methods: {
colorize: function (points) {
if (points < 100) return "green";
if (points < 199) return "green";
if (points < 400) return "orange";
if (points > 399) return "red";
}
......
......@@ -63,9 +63,9 @@
</p>
</div>
{% include "target_audience.html" %}
{% include "map/target_audience.html" %}
{% include "donation_row.html" %}
{% include "map/donation_row.html" %}
<div class="page-header">
<a name="map" class="jumptonav"></a>
......@@ -109,7 +109,7 @@
<div class="progress-bar progress-bar-basic" :style="{width:unknownpercentage}">
<span class='sr-only'>We didn't scan them then...</span>
</div>
</div>git
</div>
<div class="page-header">
<a name="numbers" class="jumptonav"></a>
......@@ -169,7 +169,7 @@
</div>
</div>
{% include "donation_row.html" %}
{% include "map/donation_row.html" %}
<div class="page-header" id="report">
<a name="report" class="jumptonav"></a>
......@@ -193,23 +193,23 @@
</div>
</div>
<div class="reportlist">
<span v-html="total_awarded_points(url.url.points)"> </span>
<span class="faildomain" v-bind:class="colorize(url.url.points)" v-bind:data-tooltip-content="idizetag(url.url.url)" >
{{ url.url.url }}</span>
{{ url.url.url }}</span><br />
<div v-for="(endpoint, key) in url.url.endpoints">
<div v-for="(myendpoint, mykey) in endpoint">
<div v-for="(myendpoint, mykey) in endpoint"><br />
&nbsp; {% endverbatim %}{% trans "Address" %}{% verbatim %}: {{ myendpoint.ip }}:{{ myendpoint.port }}
<div v-for="(rating, key) in myendpoint.ratings">
<h3>&nbsp; {{ create_header(rating) }}</h3>
<div v-for="(myrating, myratingkey) in rating">
&nbsp; &nbsp; {% endverbatim %}{% trans "Score" %}{% verbatim %}: {{ myrating.points }} {% endverbatim %}{% trans "points" %}{% verbatim %}<br />
&nbsp; &nbsp; {% endverbatim %}{% trans "since" %}{% verbatim %}: {{ humanize(myrating.since) }} <br />
&nbsp; &nbsp; {% endverbatim %}{% trans "Last scan" %}{% verbatim %}: {{ humanize(myrating.last_scan) }} <br />
&nbsp; &nbsp; {% endverbatim %}{% trans "Explanation" %}{% verbatim %}: {{ myrating.explanation }}<br />
&nbsp; &nbsp; <span v-html="awarded_points(myrating.points)"></span> {{ myrating.explanation }}<br />
&nbsp; &nbsp; {% endverbatim %}{% trans "Since" %}{% verbatim %}: {{ humanize(myrating.since) }}, {% endverbatim %}{% trans "Last check" %}{% verbatim %}: {{ humanize(myrating.last_scan) }} <br />
&nbsp; &nbsp; <span v-html="second_opinion_links(rating, url)"> </span>
<br />
</div>
</div>
</div>
</div>
<a :href="'https://www.ssllabs.com/ssltest/analyze.html?d=' + url.url.url + '&hideResults=on&latest'" target="_blank">Second Opinion</a>
</div>
</div>
{% endverbatim %}
......@@ -220,13 +220,13 @@
<br style="clear: both">
<div class="row">
{% include "topfail.html" %}
{% include "topwin.html" %}
{% include "map/topfail.html" %}
{% include "map/topwin.html" %}
</div>
<div class="row">
{% include "terrible_urls.html" %}
{% include "faq.html" %}
{% include "map/terrible_urls.html" %}
{% include "map/faq.html" %}
</div>
<div id="lastrow">
......
# urls for scanners, maybe in their own url files
from django.conf.urls import url
from failmap_admin.map.views import (index, map_data, organization_report, stats, topfail,
wanted_urls, topwin, terrible_urls)
from failmap_admin.map.views import (index, map_data, organization_report, stats, terrible_urls,
topfail, topwin, wanted_urls)
urlpatterns = [
url(r'^data/map/(?P<weeks_back>[0-9]{0,2})', map_data, name='map data'),
......
......@@ -8,6 +8,7 @@ from django.db import connection
from django.db.models import Count, Max
from django.http import JsonResponse
from django.shortcuts import render
from django.template.loader import get_template
from django.views.decorators.cache import cache_page
from failmap_admin.map.determineratings import DetermineRatings
......@@ -27,11 +28,8 @@ def index(request):
:param request:
:return:
"""
# today, used for refreshing data. / client side caching on a daily basis.
# timestamp
# now return the rendered template, it takes the wrong one... from another thing.
return render(request, 'map/templates/index.html',
return render(request, 'map/index.html',
{"timestamp": datetime.now(pytz.utc)})
......@@ -45,11 +43,11 @@ def organization_report(request, organization_id, weeks_back=0):
# getting the latest report.
try:
r = Organization.objects.filter(pk=organization_id, organizationrating__when__lte=when). \
values('organizationrating__rating',
'organizationrating__calculation',
'organizationrating__when',
'name',
'pk').latest('organizationrating__when')
values('organizationrating__rating',
'organizationrating__calculation',
'organizationrating__when',
'name',
'pk').latest('organizationrating__when')
# latest replaced: order_by('-organizationrating__when')[:1].get()
report_json = """
......@@ -101,7 +99,7 @@ def terrible_urls(request, weeks_back=0):
"urls":
[
]
]
}
cursor = connection.cursor()
......@@ -132,7 +130,6 @@ def terrible_urls(request, weeks_back=0):
GROUP BY url.url
HAVING(`rating`) > 999
ORDER BY `rating` DESC, `organization`.`name` ASC
''' % (when, when))
rows = cursor.fetchall()
......@@ -180,7 +177,7 @@ def topfail(request, weeks_back=0):
"ranking":
[
]
]
}
cursor = connection.cursor()
......@@ -256,7 +253,7 @@ def topwin(request, weeks_back=0):
"ranking":
[
]
]
}
cursor = connection.cursor()
......@@ -356,7 +353,8 @@ def stats(request, weeks_back=0):
rating = OrganizationRating.objects.filter(organization=o, rating__gt=-1)
rating = rating.earliest('when')
else:
rating = OrganizationRating.objects.filter(organization=o, when__lte=when, rating__gt=-1)
rating = OrganizationRating.objects.filter(
organization=o, when__lte=when, rating__gt=-1)
rating = rating.latest('when')
measurement["total_organizations"] += 1
......@@ -406,11 +404,11 @@ def stats(request, weeks_back=0):
if measurement["total_urls"]:
measurement["red url percentage"] = round((measurement["red_urls"] /
measurement["total_urls"]) * 100)
measurement["total_urls"]) * 100)
measurement["orange url percentage"] = round((measurement["orange_urls"] /
measurement["total_urls"]) * 100)
measurement["total_urls"]) * 100)
measurement["green url percentage"] = round((measurement["green_urls"] /
measurement["total_urls"]) * 100)
measurement["total_urls"]) * 100)
else:
measurement["red url percentage"] = 0
measurement["orange url percentage"] = 0
......@@ -531,11 +529,11 @@ def map_data(request, weeks_back=0):
{
"type": "name",
"properties": {"name": "urn:ogc:def:crs:OGC:1.3:CRS84"}
},
},
"features":
[
]
]
}
# todo: add comment to point to github repo
......
......@@ -6,10 +6,9 @@ from jet.admin import CompactInline
from failmap_admin.map.determineratings import DetermineRatings, OrganizationRating, UrlRating
from failmap_admin.scanners.models import Endpoint
from failmap_admin.scanners.scanner_tls_qualys import ScannerTlsQualys
from failmap_admin.scanners.scanner_dns import ScannerDns
from failmap_admin.scanners.scanner_http import ScannerHttp
from failmap_admin.scanners.scanner_tls_qualys import ScannerTlsQualys
from .models import Coordinate, Organization, Url
......@@ -97,7 +96,7 @@ class UrlAdmin(admin.ModelAdmin):
fieldsets = (
(None, {
'fields': ('url', 'organization','created_on')
'fields': ('url', 'organization', 'created_on')
}),
('DNS', {
'fields': ('uses_dns_wildcard', ),
......
......@@ -21,11 +21,13 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='coordinate',
name='geojsontype',
field=models.CharField(blank=True, choices=[('MultiPolygon', 'MultiPolygon'), ('MultiLineString', 'MultiLineString'), ('MultiPoint', 'MultiPoint'), ('Polygon', 'Polygon'), ('LineString', 'LineString'), ('Point', 'Point')], db_column='geoJsonType', max_length=20, null=True),
field=models.CharField(blank=True, choices=[('MultiPolygon', 'MultiPolygon'), ('MultiLineString', 'MultiLineString'), ('MultiPoint', 'MultiPoint'), (
'Polygon', 'Polygon'), ('LineString', 'LineString'), ('Point', 'Point')], db_column='geoJsonType', max_length=20, null=True),
),
migrations.AlterField(
model_name='url',
name='isdead',
field=models.BooleanField(db_column='isDead', default=False, help_text="Dead url's will not be rendered on the map. Scanners can set this check automatically (which might change in the future)"),
field=models.BooleanField(db_column='isDead', default=False,
help_text="Dead url's will not be rendered on the map. Scanners can set this check automatically (which might change in the future)"),
),
]
......@@ -15,6 +15,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='organization',
name='twitter_handle',
field=models.CharField(help_text='Used in the top lists to let visitors tweet to theorganization to wake them up.', max_length=150, null=True),
field=models.CharField(
help_text='Used in the top lists to let visitors tweet to theorganization to wake them up.', max_length=150, null=True),
),
]
......@@ -15,6 +15,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='organization',
name='twitter_handle',
field=models.CharField(blank=True, help_text='Used in the top lists to let visitors tweet to theorganization to wake them up.', max_length=150, null=True),
field=models.CharField(
blank=True, help_text='Used in the top lists to let visitors tweet to theorganization to wake them up.', max_length=150, null=True),
),
]
......@@ -17,6 +17,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='organization',
name='twitter_handle',
field=models.CharField(blank=True, help_text='Include the @ symbol. Used in the top lists to let visitors tweet to theorganization to wake them up.', max_length=150, null=True, validators=[failmap_admin.organizations.models.validate_twitter]),
field=models.CharField(blank=True, help_text='Include the @ symbol. Used in the top lists to let visitors tweet to theorganization to wake them up.',
max_length=150, null=True, validators=[failmap_admin.organizations.models.validate_twitter]),
),
]
......@@ -15,12 +15,14 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='url',
name='not_resolvable',
field=models.BooleanField(default=False, help_text='Url is not resolvable (anymore) and will not be picked up by scanners anymore.When the url is not resolvable, ratings from the past will still be shown(?)#'),
field=models.BooleanField(
default=False, help_text='Url is not resolvable (anymore) and will not be picked up by scanners anymore.When the url is not resolvable, ratings from the past will still be shown(?)#'),
),
migrations.AddField(
model_name='url',
name='not_resolvable_reason',
field=models.CharField(blank=True, help_text='A scanner might find this not resolvable, some details about that are placed here.', max_length=255, null=True),
field=models.CharField(
blank=True, help_text='A scanner might find this not resolvable, some details about that are placed here.', max_length=255, null=True),
),
migrations.AddField(
model_name='url',
......
......@@ -15,6 +15,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='url',
name='uses_dns_wildcard',
field=models.BooleanField(default=False, help_text='When true, this domain uses a DNS wildcard and any subdomain will resolve to something on this host.'),
field=models.BooleanField(
default=False, help_text='When true, this domain uses a DNS wildcard and any subdomain will resolve to something on this host.'),
),
]
......@@ -16,11 +16,13 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='coordinate',
name='area',
field=jsonfield.fields.JSONField(default=dict, help_text='GeoJson using the WGS84 (EPSG 4326) projection. Use simplified geometries to reduce the amount of data to transfer.', max_length=10000),
field=jsonfield.fields.JSONField(
default=dict, help_text='GeoJson using the WGS84 (EPSG 4326) projection. Use simplified geometries to reduce the amount of data to transfer.', max_length=10000),
),
migrations.AlterField(
model_name='url',
name='url',
field=models.CharField(help_text='Lowercase url name. For example: mydomain.tld or subdomain.domain.tld', max_length=150),
field=models.CharField(
help_text='Lowercase url name. For example: mydomain.tld or subdomain.domain.tld', max_length=150),
),
]
......@@ -16,6 +16,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='url',
name='organization',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='organizations.Organization'),
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE,
to='organizations.Organization'),
),
]
......@@ -15,6 +15,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='url',
name='organizations',
field=models.ManyToManyField(related_name='u_many_o_upgrade', to='organizations.Organization'),
field=models.ManyToManyField(related_name='u_many_o_upgrade',
to='organizations.Organization'),
),
]
......@@ -2,8 +2,8 @@
# Generated by Django 1.11.5 on 2017-09-27 13:34
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
......@@ -16,6 +16,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='url',
name='organization_old',
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='organizations.Organization'),
field=models.ForeignKey(
null=True, on_delete=django.db.models.deletion.CASCADE, to='organizations.Organization'),
),
]
......@@ -15,11 +15,13 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='url',
name='onboarded',
field=models.BooleanField(default=False, help_text='After adding a url, there is an onboarding process that runs a set of tests.These tests are usually run very quickly to get a first glimpse of the url.This test is run once.'),
field=models.BooleanField(
default=False, help_text='After adding a url, there is an onboarding process that runs a set of tests.These tests are usually run very quickly to get a first glimpse of the url.This test is run once.'),
),
migrations.AddField(
model_name='url',
name='onboarded_on',
field=models.DateTimeField(auto_now_add=True, help_text='The moment the onboard process finished.', null=True),
field=models.DateTimeField(
auto_now_add=True, help_text='The moment the onboard process finished.', null=True),
),
]
......@@ -77,6 +77,7 @@ class Coordinate(models.Model):
managed = True
db_table = 'coordinate'
class Url(models.Model):
organization_old = models.ForeignKey(Organization, null=True) # on_delete=models.PROTECT
......
......@@ -5,7 +5,7 @@ from failmap_admin.map.determineratings import DetermineRatings, OrganizationRat
from failmap_admin.scanners.scanner_tls_qualys import ScannerTlsQualys
from .models import (Endpoint, EndpointGenericScan, Screenshot, State, TlsQualysScan,
TlsQualysScratchpad, Url)
TlsQualysScratchpad, Url, EndpointGenericScanScratchpad)
class TlsQualysScanAdminInline(CompactInline):
......@@ -139,6 +139,12 @@ class EndpointGenericScanAdmin(admin.ModelAdmin):
fields = ('endpoint', 'type', 'domain', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on')
class EndpointGenericScanScratchpadAdmin(admin.ModelAdmin):
list_display = ('type', 'domain', 'when', 'data')
search_fields = ('type', 'domain', 'when', 'data')
list_filter = ('type', 'domain', 'when', 'data')
fields = ('type', 'domain', 'when', 'data')
admin.site.register(TlsQualysScan, TlsQualysScanAdmin)
admin.site.register(TlsQualysScratchpad, TlsQualysScratchpadAdmin)
......@@ -146,3 +152,4 @@ admin.site.register(Endpoint, EndpointAdmin)
admin.site.register(Screenshot, ScreenshotAdmin)
admin.site.register(State, StateAdmin)
admin.site.register(EndpointGenericScan, EndpointGenericScanAdmin)
admin.site.register(EndpointGenericScanScratchpad, EndpointGenericScanScratchpadAdmin)
from django.core.exceptions import ObjectDoesNotExist
from datetime import datetime
import pytz
from django.core.exceptions import ObjectDoesNotExist
class EndpointScanManager:
......
from dal import autocomplete
from django import forms
from failmap_admin.organizations.models import Url
# Dit moet een lijst worden uit de Urls die we al hebben.
# Het is een autocomplete en een scan button. Er moet een link bij naar de admin.
# Dit zou een actie kunnen zijn in de admin, maar flatpages staan dit niet toe.
# of wel? wss niet.
class QualysScanForm(forms.ModelForm):
url = forms.ModelChoiceField(
queryset=Url.objects.all(),
widget=autocomplete.ModelSelect2(url='url-autocomplete')
)
class Meta:
fields = ('url', )
model = Url
......@@ -86,7 +86,7 @@ opendata.arnhem.nl 2 december 2016 15:01 4 december 2016 15:50 1
endpoint.is_dead_since = Endpoint.objects.all().filter(
url=url,
discovered_on__gt=endpoint.discovered_on).earliest('discovered_on').\
discovered_on
discovered_on
logger.debug('With date: %s' % endpoint.is_dead_since)
endpoint.save()
except ObjectDoesNotExist:
......
......@@ -2,14 +2,14 @@ import logging
from datetime import datetime
import pytz
from django.core.cache import cache
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.map.determineratings import DetermineRatings
from failmap_admin.organizations.models import Organization, Url
from failmap_admin.scanners.state_manager import StateManager
from failmap_admin.scanners.scanner_dns import ScannerDns
from django.core.cache import cache
from failmap_admin.scanners.state_manager import StateManager
logger = logging.getLogger(__package__)
......
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization
......@@ -9,7 +10,7 @@ class Command(BaseCommand):
6 basic urls = 6
4 * 80 weekly urls = 320
390 * 81 organization urls = 31590
The url's that are slow and _really_ need precaching are the stats: 81 urls.
"""
......@@ -21,17 +22,19 @@ class Command(BaseCommand):
print('/data/topfail/')
print('/data/topwin/')
print('/data/wanted/')
print('/data/terrible_urls/')
weeks = range(0,80)
weeks = range(0, 99)
for week in weeks:
print('/data/stats/%s' % week)
print('/data/map/%s' % week)