Johans pull request, HSTS ratings (ipv4), TLS ratings 1xper v4 and v6, manual scans deleted

parent 1ab7f49e
......@@ -16,3 +16,5 @@ db_backup_*.sqlite3
failmap_admin/map/static/images/screenshots/*
failmap_admin/scanners/resources/output/dnsrecon/*
failmap_admin/scanners/resources/output/theHarvester/*
dist/
build/
include requirements.txt
recursive-include failmap_admin/*/templates *.html
recursive-include failmap_admin/*/static *
recursive-include failmap_admin/*/fixtures *.yaml *.json
\ No newline at end of file
......@@ -27,8 +27,7 @@ To run code quality checks and unit tests run:
To make life easier you can use `autopep8`/`isort` before running `tox` to automatically fix most style issues:
autopep8 -ri failmap_admin tests
isort -rc failmap_admin tests
tox -e autofix
To run only a specific test use:
......
This diff is collapsed.
......@@ -18,11 +18,13 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='OrganizationRating',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('id', models.AutoField(auto_created=True,
primary_key=True, serialize=False, verbose_name='ID')),
('rating', models.IntegerField()),
('when', models.DateTimeField()),
('calculation', models.TextField()),
('organization', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='organizations.Organization')),
('organization', models.ForeignKey(
on_delete=django.db.models.deletion.PROTECT, to='organizations.Organization')),
],
options={
'managed': True,
......@@ -31,7 +33,8 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='UrlRating',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('id', models.AutoField(auto_created=True,
primary_key=True, serialize=False, verbose_name='ID')),
('rating', models.IntegerField()),
('when', models.DateTimeField()),
('calculation', models.TextField()),
......
......@@ -16,6 +16,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='urlrating',
name='url',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='organizations.Url'),
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to='organizations.Url'),
),
]
......@@ -129,6 +129,7 @@ body {
.reportlist {
display: table;
padding-bottom: 15px;
}
.screenshotlist {
......@@ -245,4 +246,54 @@ body {
border-bottom-right-radius: 10px;
padding-bottom: 15px;
padding-top: 9px;
}
.reportlist h3 {
font-size: 17px;
}
.awarded_points_orange {
border: 1px solid darkorange;
background-color: orange;
padding: 0px 3px;
font-weight: bold;
font-size: 11px;
}
.awarded_points_red {
border: 1px solid darkred;
background-color: red;
padding: 0px 3px;
font-weight: bold;
font-size: 11px;
}
.awarded_points_green {
border: 1px solid green;
background-color: lightgreen;
padding: 0px 3px;
font-weight: bold;
font-size: 11px;
}
.total_awarded_points_orange {
border: 2px solid darkorange;
background-color: orange;
padding: 1px 4px;
font-weight: bold;
}
.total_awarded_points_red {
border: 2px solid darkred;
background-color: red;
padding: 1px 4px;
font-weight: bold;
}
.total_awarded_points_green {
border: 2px solid green;
background-color: lightgreen;
padding: 1px 4px;
font-weight: bold;
}
\ No newline at end of file
......@@ -334,12 +334,12 @@ $(document).ready(function () {
},
methods: {
colorize: function (points) {
if (points < 100) return "green";
if (points < 199) return "green";
if (points < 400) return "orange";
if (points > 399) return "red";
},
colorizebg: function (points) {
if (points < 100) return "#dff9d7";
if (points < 199) return "#dff9d7";
if (points < 400) return "#ffefd3";
if (points > 399) return "#fbeaea";
},
......@@ -353,6 +353,32 @@ $(document).ready(function () {
},
humanize: function (date) {
return new Date(date).humanTimeStamp()
},
create_header: function(rating){
keyz = Object.keys(rating);
if (keyz[0] === "security_headers_strict_transport_security")
return "Strict Transport Security Header (HSTS)";
if (keyz[0] === "tls_qualys")
return "Transport Layer Security (TLS)";
if (keyz[0] === "http_plain")
return "Missing transport security (TLS)";
},
second_opinion_links: function(rating, url){
keyz = Object.keys(rating);
if (keyz[0] === "security_headers_strict_transport_security")
return '<a href="https://securityheaders.io/?q=' + url.url.url + '\" target=\"_blank\">Second Opinion (securityheaders.io)</a>';
if (keyz[0] === "tls_qualys")
return '<a href="https://www.ssllabs.com/ssltest/analyze.html?d=' + url.url.url + '&hideResults=on&latest\" target=\"_blank\">Second Opinion (Qualys)</a>';
},
total_awarded_points: function(points) {
if (points === "0")
marker = "✓ perfect";
else
marker = points;
return '<span class="total_awarded_points_'+ this.colorize(points) +'">' + marker + '</span>'
},
awarded_points: function(points) {
return '<span class="awarded_points_'+ this.colorize(points) +'">+ ' + points + '</span>'
}
}
});
......@@ -396,7 +422,7 @@ $(document).ready(function () {
data: {urls: Array},
methods: {
colorize: function (points) {
if (points < 100) return "green";
if (points < 199) return "green";
if (points < 400) return "orange";
if (points > 399) return "red";
}
......
......@@ -63,9 +63,9 @@
</p>
</div>
{% include "target_audience.html" %}
{% include "map/target_audience.html" %}
{% include "donation_row.html" %}
{% include "map/donation_row.html" %}
<div class="page-header">
<a name="map" class="jumptonav"></a>
......@@ -109,7 +109,7 @@
<div class="progress-bar progress-bar-basic" :style="{width:unknownpercentage}">
<span class='sr-only'>We didn't scan them then...</span>
</div>
</div>git
</div>
<div class="page-header">
<a name="numbers" class="jumptonav"></a>
......@@ -169,7 +169,7 @@
</div>
</div>
{% include "donation_row.html" %}
{% include "map/donation_row.html" %}
<div class="page-header" id="report">
<a name="report" class="jumptonav"></a>
......@@ -193,23 +193,23 @@
</div>
</div>
<div class="reportlist">
<span v-html="total_awarded_points(url.url.points)"> </span>
<span class="faildomain" v-bind:class="colorize(url.url.points)" v-bind:data-tooltip-content="idizetag(url.url.url)" >
{{ url.url.url }}</span>
{{ url.url.url }}</span><br />
<div v-for="(endpoint, key) in url.url.endpoints">
<div v-for="(myendpoint, mykey) in endpoint">
<div v-for="(myendpoint, mykey) in endpoint"><br />
&nbsp; {% endverbatim %}{% trans "Address" %}{% verbatim %}: {{ myendpoint.ip }}:{{ myendpoint.port }}
<div v-for="(rating, key) in myendpoint.ratings">
<h3>&nbsp; {{ create_header(rating) }}</h3>
<div v-for="(myrating, myratingkey) in rating">
&nbsp; &nbsp; {% endverbatim %}{% trans "Score" %}{% verbatim %}: {{ myrating.points }} {% endverbatim %}{% trans "points" %}{% verbatim %}<br />
&nbsp; &nbsp; {% endverbatim %}{% trans "since" %}{% verbatim %}: {{ humanize(myrating.since) }} <br />
&nbsp; &nbsp; {% endverbatim %}{% trans "Last scan" %}{% verbatim %}: {{ humanize(myrating.last_scan) }} <br />
&nbsp; &nbsp; {% endverbatim %}{% trans "Explanation" %}{% verbatim %}: {{ myrating.explanation }}<br />
&nbsp; &nbsp; <span v-html="awarded_points(myrating.points)"></span> {{ myrating.explanation }}<br />
&nbsp; &nbsp; {% endverbatim %}{% trans "Since" %}{% verbatim %}: {{ humanize(myrating.since) }}, {% endverbatim %}{% trans "Last check" %}{% verbatim %}: {{ humanize(myrating.last_scan) }} <br />
&nbsp; &nbsp; <span v-html="second_opinion_links(rating, url)"> </span>
<br />
</div>
</div>
</div>
</div>
<a :href="'https://www.ssllabs.com/ssltest/analyze.html?d=' + url.url.url + '&hideResults=on&latest'" target="_blank">Second Opinion</a>
</div>
</div>
{% endverbatim %}
......@@ -220,13 +220,13 @@
<br style="clear: both">
<div class="row">
{% include "topfail.html" %}
{% include "topwin.html" %}
{% include "map/topfail.html" %}
{% include "map/topwin.html" %}
</div>
<div class="row">
{% include "terrible_urls.html" %}
{% include "faq.html" %}
{% include "map/terrible_urls.html" %}
{% include "map/faq.html" %}
</div>
<div id="lastrow">
......
# urls for scanners, maybe in their own url files
from django.conf.urls import url
from failmap_admin.map.views import (index, map_data, organization_report, stats, topfail,
wanted_urls, topwin, terrible_urls)
from failmap_admin.map.views import (index, map_data, organization_report, stats, terrible_urls,
topfail, topwin, wanted_urls)
urlpatterns = [
url(r'^data/map/(?P<weeks_back>[0-9]{0,2})', map_data, name='map data'),
......
......@@ -8,6 +8,7 @@ from django.db import connection
from django.db.models import Count, Max
from django.http import JsonResponse
from django.shortcuts import render
from django.template.loader import get_template
from django.views.decorators.cache import cache_page
from failmap_admin.map.determineratings import DetermineRatings
......@@ -27,11 +28,8 @@ def index(request):
:param request:
:return:
"""
# today, used for refreshing data. / client side caching on a daily basis.
# timestamp
# now return the rendered template, it takes the wrong one... from another thing.
return render(request, 'map/templates/index.html',
return render(request, 'map/index.html',
{"timestamp": datetime.now(pytz.utc)})
......@@ -45,11 +43,11 @@ def organization_report(request, organization_id, weeks_back=0):
# getting the latest report.
try:
r = Organization.objects.filter(pk=organization_id, organizationrating__when__lte=when). \
values('organizationrating__rating',
'organizationrating__calculation',
'organizationrating__when',
'name',
'pk').latest('organizationrating__when')
values('organizationrating__rating',
'organizationrating__calculation',
'organizationrating__when',
'name',
'pk').latest('organizationrating__when')
# latest replaced: order_by('-organizationrating__when')[:1].get()
report_json = """
......@@ -101,7 +99,7 @@ def terrible_urls(request, weeks_back=0):
"urls":
[
]
]
}
cursor = connection.cursor()
......@@ -132,7 +130,6 @@ def terrible_urls(request, weeks_back=0):
GROUP BY url.url
HAVING(`rating`) > 999
ORDER BY `rating` DESC, `organization`.`name` ASC
''' % (when, when))
rows = cursor.fetchall()
......@@ -180,7 +177,7 @@ def topfail(request, weeks_back=0):
"ranking":
[
]
]
}
cursor = connection.cursor()
......@@ -256,7 +253,7 @@ def topwin(request, weeks_back=0):
"ranking":
[
]
]
}
cursor = connection.cursor()
......@@ -356,7 +353,8 @@ def stats(request, weeks_back=0):
rating = OrganizationRating.objects.filter(organization=o, rating__gt=-1)
rating = rating.earliest('when')
else:
rating = OrganizationRating.objects.filter(organization=o, when__lte=when, rating__gt=-1)
rating = OrganizationRating.objects.filter(
organization=o, when__lte=when, rating__gt=-1)
rating = rating.latest('when')
measurement["total_organizations"] += 1
......@@ -406,11 +404,11 @@ def stats(request, weeks_back=0):
if measurement["total_urls"]:
measurement["red url percentage"] = round((measurement["red_urls"] /
measurement["total_urls"]) * 100)
measurement["total_urls"]) * 100)
measurement["orange url percentage"] = round((measurement["orange_urls"] /
measurement["total_urls"]) * 100)
measurement["total_urls"]) * 100)
measurement["green url percentage"] = round((measurement["green_urls"] /
measurement["total_urls"]) * 100)
measurement["total_urls"]) * 100)
else:
measurement["red url percentage"] = 0
measurement["orange url percentage"] = 0
......@@ -531,11 +529,11 @@ def map_data(request, weeks_back=0):
{
"type": "name",
"properties": {"name": "urn:ogc:def:crs:OGC:1.3:CRS84"}
},
},
"features":
[
]
]
}
# todo: add comment to point to github repo
......
......@@ -6,10 +6,9 @@ from jet.admin import CompactInline
from failmap_admin.map.determineratings import DetermineRatings, OrganizationRating, UrlRating
from failmap_admin.scanners.models import Endpoint
from failmap_admin.scanners.scanner_tls_qualys import ScannerTlsQualys
from failmap_admin.scanners.scanner_dns import ScannerDns
from failmap_admin.scanners.scanner_http import ScannerHttp
from failmap_admin.scanners.scanner_tls_qualys import ScannerTlsQualys
from .models import Coordinate, Organization, Url
......@@ -97,7 +96,7 @@ class UrlAdmin(admin.ModelAdmin):
fieldsets = (
(None, {
'fields': ('url', 'organization','created_on')
'fields': ('url', 'organization', 'created_on')
}),
('DNS', {
'fields': ('uses_dns_wildcard', ),
......
......@@ -21,11 +21,13 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='coordinate',
name='geojsontype',
field=models.CharField(blank=True, choices=[('MultiPolygon', 'MultiPolygon'), ('MultiLineString', 'MultiLineString'), ('MultiPoint', 'MultiPoint'), ('Polygon', 'Polygon'), ('LineString', 'LineString'), ('Point', 'Point')], db_column='geoJsonType', max_length=20, null=True),
field=models.CharField(blank=True, choices=[('MultiPolygon', 'MultiPolygon'), ('MultiLineString', 'MultiLineString'), ('MultiPoint', 'MultiPoint'), (
'Polygon', 'Polygon'), ('LineString', 'LineString'), ('Point', 'Point')], db_column='geoJsonType', max_length=20, null=True),
),
migrations.AlterField(
model_name='url',
name='isdead',
field=models.BooleanField(db_column='isDead', default=False, help_text="Dead url's will not be rendered on the map. Scanners can set this check automatically (which might change in the future)"),
field=models.BooleanField(db_column='isDead', default=False,
help_text="Dead url's will not be rendered on the map. Scanners can set this check automatically (which might change in the future)"),
),
]
......@@ -15,6 +15,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='organization',
name='twitter_handle',
field=models.CharField(help_text='Used in the top lists to let visitors tweet to theorganization to wake them up.', max_length=150, null=True),
field=models.CharField(
help_text='Used in the top lists to let visitors tweet to theorganization to wake them up.', max_length=150, null=True),
),
]
......@@ -15,6 +15,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='organization',
name='twitter_handle',
field=models.CharField(blank=True, help_text='Used in the top lists to let visitors tweet to theorganization to wake them up.', max_length=150, null=True),
field=models.CharField(
blank=True, help_text='Used in the top lists to let visitors tweet to theorganization to wake them up.', max_length=150, null=True),
),
]
......@@ -17,6 +17,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='organization',
name='twitter_handle',
field=models.CharField(blank=True, help_text='Include the @ symbol. Used in the top lists to let visitors tweet to theorganization to wake them up.', max_length=150, null=True, validators=[failmap_admin.organizations.models.validate_twitter]),
field=models.CharField(blank=True, help_text='Include the @ symbol. Used in the top lists to let visitors tweet to theorganization to wake them up.',
max_length=150, null=True, validators=[failmap_admin.organizations.models.validate_twitter]),
),
]
......@@ -15,12 +15,14 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='url',
name='not_resolvable',
field=models.BooleanField(default=False, help_text='Url is not resolvable (anymore) and will not be picked up by scanners anymore.When the url is not resolvable, ratings from the past will still be shown(?)#'),
field=models.BooleanField(
default=False, help_text='Url is not resolvable (anymore) and will not be picked up by scanners anymore.When the url is not resolvable, ratings from the past will still be shown(?)#'),
),
migrations.AddField(
model_name='url',
name='not_resolvable_reason',
field=models.CharField(blank=True, help_text='A scanner might find this not resolvable, some details about that are placed here.', max_length=255, null=True),
field=models.CharField(
blank=True, help_text='A scanner might find this not resolvable, some details about that are placed here.', max_length=255, null=True),
),
migrations.AddField(
model_name='url',
......
......@@ -15,6 +15,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='url',
name='uses_dns_wildcard',
field=models.BooleanField(default=False, help_text='When true, this domain uses a DNS wildcard and any subdomain will resolve to something on this host.'),
field=models.BooleanField(
default=False, help_text='When true, this domain uses a DNS wildcard and any subdomain will resolve to something on this host.'),
),
]
......@@ -16,11 +16,13 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='coordinate',
name='area',
field=jsonfield.fields.JSONField(default=dict, help_text='GeoJson using the WGS84 (EPSG 4326) projection. Use simplified geometries to reduce the amount of data to transfer.', max_length=10000),
field=jsonfield.fields.JSONField(
default=dict, help_text='GeoJson using the WGS84 (EPSG 4326) projection. Use simplified geometries to reduce the amount of data to transfer.', max_length=10000),
),
migrations.AlterField(
model_name='url',
name='url',
field=models.CharField(help_text='Lowercase url name. For example: mydomain.tld or subdomain.domain.tld', max_length=150),
field=models.CharField(
help_text='Lowercase url name. For example: mydomain.tld or subdomain.domain.tld', max_length=150),
),
]
......@@ -16,6 +16,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='url',
name='organization',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='organizations.Organization'),
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE,
to='organizations.Organization'),
),
]
......@@ -15,6 +15,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='url',
name='organizations',
field=models.ManyToManyField(related_name='u_many_o_upgrade', to='organizations.Organization'),
field=models.ManyToManyField(related_name='u_many_o_upgrade',
to='organizations.Organization'),
),
]
......@@ -2,8 +2,8 @@
# Generated by Django 1.11.5 on 2017-09-27 13:34
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
......@@ -16,6 +16,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='url',
name='organization_old',
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='organizations.Organization'),
field=models.ForeignKey(
null=True, on_delete=django.db.models.deletion.CASCADE, to='organizations.Organization'),
),
]
......@@ -15,11 +15,13 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='url',
name='onboarded',
field=models.BooleanField(default=False, help_text='After adding a url, there is an onboarding process that runs a set of tests.These tests are usually run very quickly to get a first glimpse of the url.This test is run once.'),
field=models.BooleanField(
default=False, help_text='After adding a url, there is an onboarding process that runs a set of tests.These tests are usually run very quickly to get a first glimpse of the url.This test is run once.'),
),
migrations.AddField(
model_name='url',
name='onboarded_on',
field=models.DateTimeField(auto_now_add=True, help_text='The moment the onboard process finished.', null=True),
field=models.DateTimeField(
auto_now_add=True, help_text='The moment the onboard process finished.', null=True),
),
]
......@@ -77,6 +77,7 @@ class Coordinate(models.Model):
managed = True
db_table = 'coordinate'
class Url(models.Model):
organization_old = models.ForeignKey(Organization, null=True) # on_delete=models.PROTECT
......
......@@ -5,7 +5,7 @@ from failmap_admin.map.determineratings import DetermineRatings, OrganizationRat
from failmap_admin.scanners.scanner_tls_qualys import ScannerTlsQualys
from .models import (Endpoint, EndpointGenericScan, Screenshot, State, TlsQualysScan,
TlsQualysScratchpad, Url)
TlsQualysScratchpad, Url, EndpointGenericScanScratchpad)
class TlsQualysScanAdminInline(CompactInline):
......@@ -139,6 +139,12 @@ class EndpointGenericScanAdmin(admin.ModelAdmin):
fields = ('endpoint', 'type', 'domain', 'rating',
'explanation', 'last_scan_moment', 'rating_determined_on')
class EndpointGenericScanScratchpadAdmin(admin.ModelAdmin):
list_display = ('type', 'domain', 'when', 'data')
search_fields = ('type', 'domain', 'when', 'data')
list_filter = ('type', 'domain', 'when', 'data')
fields = ('type', 'domain', 'when', 'data')
admin.site.register(TlsQualysScan, TlsQualysScanAdmin)
admin.site.register(TlsQualysScratchpad, TlsQualysScratchpadAdmin)
......@@ -146,3 +152,4 @@ admin.site.register(Endpoint, EndpointAdmin)
admin.site.register(Screenshot, ScreenshotAdmin)
admin.site.register(State, StateAdmin)
admin.site.register(EndpointGenericScan, EndpointGenericScanAdmin)
admin.site.register(EndpointGenericScanScratchpad, EndpointGenericScanScratchpadAdmin)
from django.core.exceptions import ObjectDoesNotExist
from datetime import datetime
import pytz
from django.core.exceptions import ObjectDoesNotExist
class EndpointScanManager:
......
from dal import autocomplete
from django import forms
from failmap_admin.organizations.models import Url
# Dit moet een lijst worden uit de Urls die we al hebben.
# Het is een autocomplete en een scan button. Er moet een link bij naar de admin.
# Dit zou een actie kunnen zijn in de admin, maar flatpages staan dit niet toe.
# of wel? wss niet.
class QualysScanForm(forms.ModelForm):
url = forms.ModelChoiceField(
queryset=Url.objects.all(),
widget=autocomplete.ModelSelect2(url='url-autocomplete')
)
class Meta:
fields = ('url', )
model = Url
......@@ -86,7 +86,7 @@ opendata.arnhem.nl 2 december 2016 15:01 4 december 2016 15:50 1
endpoint.is_dead_since = Endpoint.objects.all().filter(
url=url,
discovered_on__gt=endpoint.discovered_on).earliest('discovered_on').\
discovered_on
discovered_on
logger.debug('With date: %s' % endpoint.is_dead_since)
endpoint.save()
except ObjectDoesNotExist:
......
......@@ -2,14 +2,14 @@ import logging
from datetime import datetime
import pytz
from django.core.cache import cache
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.map.determineratings import DetermineRatings
from failmap_admin.organizations.models import Organization, Url
from failmap_admin.scanners.state_manager import StateManager
from failmap_admin.scanners.scanner_dns import ScannerDns
from django.core.cache import cache
from failmap_admin.scanners.state_manager import StateManager
logger = logging.getLogger(__package__)
......
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization
......@@ -9,7 +10,7 @@ class Command(BaseCommand):
6 basic urls = 6
4 * 80 weekly urls = 320
390 * 81 organization urls = 31590
The url's that are slow and _really_ need precaching are the stats: 81 urls.
"""
......@@ -21,17 +22,19 @@ class Command(BaseCommand):
print('/data/topfail/')
print('/data/topwin/')
print('/data/wanted/')
print('/data/terrible_urls/')
weeks = range(0,80)
weeks = range(0, 99)
for week in weeks:
print('/data/stats/%s' % week)
print('/data/map/%s' % week)