Merge remote-tracking branch 'origin/master' into 61-endpoint-ip-separation

.codeclimate.yml

@@ -24,6 +24,5 @@ ratings:
     - "**.md"
 
 exclude_paths:
-  - "failmap_admin/fail/migrations/*"
-  - "manage.py"
+  - "failmap_admin/*/migrations/*"
   - ".*"

docker-compose.yaml

@@ -71,7 +71,7 @@ services:
       # django decides what to log based on type of console
       TERM: xterm-color
       DEBUG: 1
-      # disable server static files from collectstatic during development
+      # disable server static files from collectstatic during development, passthrough to django staticfiles urlpatterns
       UWSGI_STATIC_MAP:
     # mount current source into container to allow changes to propagate without container rebuild
     volumes:

failmap_admin/app/common.py

+import datetime
 import json
 
 
@@ -13,3 +14,20 @@ class ResultEncoder(json.JSONEncoder):
             if value.__cause__:
                 error['cause'] = self.default(value.__cause__)
             return error
+        else:
+            return super(ResultEncoder, self).default(value)
+
+
+class JSEncoder(json.JSONEncoder):
+    """JSON encoder to serialize results to be consumed by Javascript web apps."""
+
+    def default(self, obj):
+        # convert python datetime objects into a standard parsable by javascript
+        if isinstance(obj, datetime.datetime):
+            return obj.isoformat()
+        elif isinstance(obj, datetime.date):
+            return obj.isoformat()
+        elif isinstance(obj, datetime.timedelta):
+            return (datetime.datetime.min + obj).time().isoformat()
+        else:
+            return super(JSEncoder, self).default(obj)

failmap_admin/app/templates/500.html

 <p>You've encountered an error, oh noes!</p>
 {% if request.sentry.id %}
-    <p>If you need assistance, you may reference this error as
-    <strong>{{ request.sentry.id }}</strong>.</p>
+<p>If you need assistance, you may reference this error as
+{% if admin_instance %}
+<a href="{{sentry_project_url}}/?query={{ request.sentry.id }}">{{ request.sentry.id }}</a>,
+{% else %}
+<strong>{{ request.sentry.id }}</strong>.</p>
+{% endif %}
+{% endif %}
+
+{% if exception_message %}
+<p>For what it's worth the error message was: {{ exception_message }}</p>
 {% endif %}

failmap_admin/map/static/js/script.js

@@ -559,7 +559,8 @@ $(document).ready(function () {
             urls: Array,
             mailto: document.head.querySelector("[name=mailto]").getAttribute('content'),
             selected: null,
-            loading: false
+            loading: false,
+            promise: false,
         },
         filters: {
             // you cannot run filters in rawHtml, so this doesn't work.
@@ -679,6 +680,8 @@ $(document).ready(function () {
                     vueReport.when = data.when;
                     vueReport.name = data.name;
                     vueReport.twitter_handle = data.twitter_handle;
+                    vueReport.promise = data.promise;
+
                     // include id in anchor to allow url sharing
                     let newHash = 'report-' + OrganizationID;
                     $('a#report-anchor').attr('name', newHash)
@@ -698,6 +701,9 @@ $(document).ready(function () {
                         return "<a role='button' class='btn btn-xs btn-info' target='_blank' href=\"https://twitter.com/intent/tweet?screen_name=" + twitter_handle + '&text=' + name + ' heeft alles op orde! 🌹&hashtags=' + name + ',win,faalkaart"><img src="/static/images/twitterwhite.png" width="14" /> Tweet!</a>';
                     }
                 }
+            },
+            formatDate: function(date){
+                return new Date(date).toISOString().substring(0, 10)
             }
         }
     });

failmap_admin/map/templates/map/index.html

@@ -312,12 +312,20 @@
                         Dit resultaat delen? {% verbatim %}<span v-html="create_twitter_link(name, twitter_handle, points)"></span>{% endverbatim %}<br />
                         <br />
                         {% trans "Data from" %}: {% verbatim %}{{ humanize(when) }}{% endverbatim %}<br />
-                        {% trans "Points" %}: {% verbatim %}{{ points }}{% endverbatim %}, {% trans "congratulations" %}!<br />
+                        {% trans "Points" %}: {% verbatim %}{{ points }}{% endverbatim %}<span v-if="promise"><strong>*</strong></span>, {% trans "congratulations" %}!
+                        <br />
                         <br />
                         Gaat faalkaart niet ver genoeg? <a v-bind:href="'mailto:' + mailto + '?subject=Pentest%20aanvraag%20voor%20'+name+'&body=Beste Faalkaart,%0D%0A%0D%0AWij hebben interesse in een pentest op de outward-facing IT van onze organisatie. Kunnen jullie daar bij helpen?%0D%0A%0D%0AMet vriendelijke groet,%0D%0A%0D%0A'">Vraag hier een echte pentest aan.</a><br/>
                         Ontbreken er domeinen? <a v-bind:href="'mailto:' + mailto + '?subject=Nieuwe%20domeinen%20voor%20'+name+'&body=Beste Faalkaart,%0D%0A%0D%0AGraag de volgende domeinen toevoegen aan de kaart:%0D%0A%0D%0A%0D%0A%0D%0A%0D%0A%0D%0ATip: stuur een zonefile mee met alle domeinen.%0D%0A%0D%0AMet vriendelijke groet,%0D%0A%0D%0A'">Stuur hier domeinen in.</a><br/>
                         <br />
                         {% verbatim %}
+                        <div v-if="promise" id="promise"><strong>*</strong> Deze organisatie heeft op {{formatDate(promise.created_on)}} laten weten dat veranderingen zijn doorgevoerd.
+                          De resultaten van deze verandering zullen uiterlijk {{formatDate(promise.expires_on)}} zichtbaar zijn. Faalkaart draait continu
+                          geautomatiseerd scans. Elke cyclus bevat tienduizenden scans en kost daarom enkele dagen.
+                        </div>
+                        {% endverbatim %}
+                        <br/>
+                        {% verbatim %}
                         <div v-for="url in urls" class="perurl" v-bind:style="'background: linear-gradient(' + colorizebg(url.points) + ', white);'">
                             <div class="screenshotlist">
                                 <div v-for="endpoint in url.endpoints" class="servicelink">

failmap_admin/map/views.py

@@ -13,9 +13,10 @@ from django.utils.translation import ugettext as _
 from django.views.decorators.cache import cache_page
 
 from failmap_admin.map.models import OrganizationRating, UrlRating
-from failmap_admin.organizations.models import Organization, Url
+from failmap_admin.organizations.models import Organization, Promise, Url
 
 from .. import __version__
+from ..app.common import JSEncoder
 
 one_minute = 60
 one_hour = 60 * 60
@@ -93,6 +94,12 @@ def organization_report(request, organization_id, weeks_back=0):
                    'twitter_handle').latest('organizationrating__when')
         # latest replaced: order_by('-organizationrating__when')[:1].get()
 
+        # get the most recent non-expired 'promise'
+        promise = Promise.objects.filter(organization_id=organization_id, expires_on__gt=datetime.now())
+        promise = promise.order_by('-expires_on')
+        promise = promise.values('created_on', 'expires_on')
+        promise = promise.first()
+
         report_json = """
 {
     "name": "%s",
@@ -100,7 +107,8 @@ def organization_report(request, organization_id, weeks_back=0):
     "twitter_handle": "%s",
     "rating": %s,
     "when": "%s",
-    "calculation": %s
+    "calculation": %s,
+    "promise": %s
 }
         """
         report_json = report_json % (
@@ -110,6 +118,7 @@ def organization_report(request, organization_id, weeks_back=0):
             r['organizationrating__rating'],
             r['organizationrating__when'].isoformat(),
             r['organizationrating__calculation'],
+            json.dumps(promise, cls=JSEncoder),
         )
         # print(report_json)
     except Organization.DoesNotExist:

failmap_admin/organizations/admin.py

@@ -17,11 +17,24 @@ from failmap_admin.scanners.scanner_security_headers import scan_urls as securit
 from failmap_admin.scanners.scanner_tls_qualys import scan_url_list
 
 from ..app.models import Job
-from .models import Coordinate, Organization, OrganizationType, Url
+from .models import Coordinate, Organization, OrganizationType, Promise, Url
 
 logger = logging.getLogger(__name__)
 
 
+PROMISE_DESCRIPTION = """
+<p>A 'promise' is an indication by an organisation representitive that an improvement
+has been made which will alter the organizations score. A generic message will be
+displayed on the organization report with the creation and expiry date of the promise
+until it expires.</p>
+<p>This indication is to overcome the problem of a negative score even though improvement
+are made, but the score cannot reflect them yet due to technical or bureaucratic reasons.</p>
+<p>It is not intended for long term promises of improvement that have not been applied or
+put in to progress. The promised improvement must be verifiable by Faalkaart within a
+handfull of days.</p>
+"""
+
+
 class UrlAdminInline(CompactInline):
     model = Url
     extra = 0
@@ -52,6 +65,19 @@ class UrlRatingAdminInline(CompactInline):
     ordering = ["-when"]
 
 
+class PromiseAdminInline(CompactInline):
+    model = Promise
+    extra = 0
+    ordering = ["-created_on"]
+
+    fieldsets = (
+        (None, {
+            'fields': ('organization', 'created_on', 'expires_on', 'notes'),
+            'description': PROMISE_DESCRIPTION,
+        }),
+    )
+
+
 class OrganizationAdmin(admin.ModelAdmin):
     class Media:
         js = ('js/action_buttons.js', )
@@ -61,7 +87,7 @@ class OrganizationAdmin(admin.ModelAdmin):
     list_filter = ('name', 'type__name', 'country')  # todo: type is now listed as name, confusing
     fields = ('name', 'type', 'country', 'twitter_handle')
 
-    inlines = [UrlAdminInline, CoordinateAdminInline, OrganizationRatingAdminInline]  #
+    inlines = [UrlAdminInline, CoordinateAdminInline, OrganizationRatingAdminInline, PromiseAdminInline]  #
 
     actions = ['rate_organization', 'scan_organization']
 
@@ -224,7 +250,21 @@ class CoordinateAdmin(admin.ModelAdmin):
     fields = ('organization', 'geojsontype', 'area')
 
 
+class PromiseAdmin(admin.ModelAdmin):
+    list_display = ('organization', 'created_on', 'expires_on')
+    search_fields = ('organization',)
+    list_filter = ('organization',)
+
+    fieldsets = (
+        (None, {
+            'fields': ('organization', 'created_on', 'expires_on', 'notes'),
+            'description': PROMISE_DESCRIPTION,
+        }),
+    )
+
+
 admin.site.register(Organization, OrganizationAdmin)
 admin.site.register(Url, UrlAdmin)
 admin.site.register(Coordinate, CoordinateAdmin)
 admin.site.register(OrganizationType, OrganizationTypeAdmin)
+admin.site.register(Promise, PromiseAdmin)

failmap_admin/organizations/dashboard.py

@@ -33,13 +33,13 @@ class CustomIndexDashboard(Dashboard):
             _('Failmap resources'),
             children=[
                 {
-                    'title': _('Github Repository'),
-                    'url': 'https://github.com/failmap/',
+                    'title': _('Gitlab Repository'),
+                    'url': 'https://gitlab.com/failmap/',
                     'external': True,
                 },
                 {
                     'title': _('Admin repository'),
-                    'url': 'https://github.com/failmap/admin',
+                    'url': 'https://gitlab.com/failmap/admin',
                     'external': True,
                 },
                 {

failmap_admin/organizations/migrations/0019_promise.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.7 on 2017-11-11 09:52
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('organizations', '0018_auto_20171017_1317'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Promise',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('notes', models.TextField(help_text='Context information about the promise (eg: ticket reference).')),
+                ('created_on', models.DateTimeField(auto_now_add=True, null=True)),
+                ('expires_on', models.DateTimeField(blank=True, null=True)),
+                ('organization', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='organizations.Organization')),
+            ],
+        ),
+    ]

failmap_admin/organizations/migrations/0020_auto_20171112_1149.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.7 on 2017-11-12 11:49
+from __future__ import unicode_literals
+
+import datetime
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('organizations', '0019_promise'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='promise',
+            name='created_on',
+            field=models.DateTimeField(blank=True, default=datetime.datetime.now, null=True),
+        ),
+        migrations.AlterField(
+            model_name='promise',
+            name='expires_on',
+            field=models.DateTimeField(blank=True, help_text='When in the future this promise is expected to be fulfilled.', null=True),
+        ),
+        migrations.AlterField(
+            model_name='promise',
+            name='notes',
+            field=models.TextField(blank=True, help_text='Context information about the promise (eg: ticket reference).', null=True),
+        ),
+    ]

failmap_admin/organizations/models.py

 # coding=UTF-8
 # from __future__ import unicode_literals
 
+from datetime import datetime, timedelta
+
 from django.core.exceptions import ValidationError
 from django.db import models
 from django_countries.fields import CountryField
@@ -161,3 +163,24 @@ class Url(models.Model):
 # so they are not used anymore.
 # class Port(models.Model):
 #    url = models.ForeignKey(Url, on_delete=models.PROTECT)
+
+
+class Promise(models.Model):
+    """Allow recording of organisation promises for improvement."""
+
+    organization = models.ForeignKey(Organization, on_delete=models.PROTECT)
+    notes = models.TextField(
+        blank=True,
+        null=True,
+        help_text="Context information about the promise (eg: ticket reference).")
+
+    created_on = models.DateTimeField(
+        default=datetime.today, blank=True, null=True)
+    expires_on = models.DateTimeField(
+        default=lambda: datetime.now() + timedelta(days=7),
+        blank=True,
+        null=True,
+        help_text="When in the future this promise is expected to be fulfilled.")
+
+    def __str__(self):
+        return '%s - %s' % (self.organization.name, self.created_on)

failmap_admin/settings.py

@@ -469,8 +469,7 @@ if DEBUG:
         # send statsd metrics to debug_toolbar
         STATSD_CLIENT = 'django_statsd.clients.toolbar'
     except ImportError:
-        # send statsd metrics to logging
-        STATSD_CLIENT = 'django_statsd.clients.log'
+        pass
 
 # is administrative backend enabled on this instance
 ADMIN = bool(APPNAME == 'failmap-admin')
@@ -491,4 +490,8 @@ if SENTRY_DSN:
     MIDDLEWARE_CLASSES.insert(0, 'raven.contrib.django.raven_compat.middleware.SentryResponseErrorIdMiddleware')
 
 # set javascript sentry token if provided
-SENTRY_TOKEN = os.environ.get('SENTRY_TOKEN', 'https://a4f72b82fc0742bc82b82560b340006b@sentry.io/242170')
+SENTRY_TOKEN = os.environ.get('SENTRY_TOKEN', '')
+
+SENTRY_ORGANIZATION = 'internet-cleanup-foundation'
+SENTRY_PROJECT = 'faalkaart'
+SENTRY_PROJECT_URL = 'https://sentry.io/%s/%s' % (SENTRY_ORGANIZATION, SENTRY_PROJECT)

failmap_admin/urls.py

@@ -13,9 +13,12 @@ Including another URLconf
     1. Import the include() function: from django.conf.urls import url, include
     2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
 """
+import sys
+
 from django.conf import settings
 from django.conf.urls import include, url
 from django.contrib import admin
+from django.contrib.staticfiles.urls import staticfiles_urlpatterns
 from django.template.response import TemplateResponse
 
 # Django 1.10 http://stackoverflow.com/questions/38744285/
@@ -39,6 +42,8 @@ if settings.ADMIN:
     urlpatterns += admin_urls
 
 if settings.DEBUG:
+    urlpatterns += staticfiles_urlpatterns()
+
     try:
         import debug_toolbar
         urlpatterns = [
@@ -63,6 +68,16 @@ if settings.SENTRY_DSN:
         Context: None
         """
 
-        context = {'request': request}
+        context = {
+            'request': request,
+            'admin_instance': settings.ADMIN,
+            'sentry_project_url': settings.SENTRY_PROJECT_URL,
+        }
+
+        # on privileged instance show the actual error message to hopefully be useful for the user
+        if settings.ADMIN:
+            _, value, _ = sys.exc_info()
+            context['exception_message'] = value
+
         template_name = '500.html'  # You need to create a 500.html template.
         return TemplateResponse(request, template_name, context, status=500)