Commit 30026b46 authored by Elger Jonker's avatar Elger Jonker

testing commands (beta), no relative import paths, bugfixes

parent 7c058775
......@@ -24,7 +24,8 @@ static/
version
vendor/Google Chrome.app/
failmap_dataset*
failmap_testdataset*
failmap_test_dataset*
failmap_organization_export*
dev_db/
failmap_debug_dataset*
temp/*
......
......@@ -13,7 +13,7 @@ from django_celery_beat.models import CrontabSchedule, IntervalSchedule, Periodi
from import_export import resources
from import_export.admin import ImportExportModelAdmin
from .models import Job, Volunteer
from failmap.app.models import Job, Volunteer
class JobAdmin(ImportExportModelAdmin, admin.ModelAdmin):
......
......@@ -3,7 +3,7 @@ from jet.dashboard import modules
from jet.dashboard.dashboard import Dashboard
from jet.dashboard.models import UserDashboardModule
from . import dashboard_modules
from failmap.app import dashboard_modules
class ResetUserWidgetConfiguration:
......
......@@ -2,7 +2,7 @@ from django.conf.urls import url
from django.http import JsonResponse
from jet.dashboard import dashboard
from ..celery import status
from failmap.celery import status
def task_processing_status(request):
......
......@@ -199,12 +199,17 @@ class DiscoverTaskCommand(TaskCommand):
self.mutual_group.add_argument('-o', '--organization_names', nargs='*',
help="Perform scans on these organizations (default is all).")
self.mutual_group.add_argument('-u', '--url_addresses', nargs='*',
help="Perform scans on these urls (default is all).")
self.mutual_group.add_argument('-y', '--organization_type', nargs='*',
help="Perform scans on these organization types (default is all).")
def compose(self, *args, **options):
"""Compose set of tasks based on provided arguments."""
urls_filter = dict()
if not options['organization_names']:
# by default no filter means all organizations
organization_filter = dict()
......@@ -213,13 +218,18 @@ class DiscoverTaskCommand(TaskCommand):
regex = '^(' + '|'.join(options['organization_names']) + ')$'
organization_filter = {'name__iregex': regex}
if options['url_addresses']:
# create a case-insensitive filter to match organizations by name
regex = '^(' + '|'.join(options['url_addresses']) + ')$'
urls_filter = {'url__iregex': regex}
if options['organization_type']:
# create a case-insensitive filter to match organizations by name
regex = '^(' + '|'.join(options['organization_type']) + ')$'
organization_filter = {'type__name__iregex': regex}
# compose set of tasks to be executed
return self.scanner_module.compose_discover_task(organization_filter)
return self.scanner_module.compose_discover_task(organization_filter, urls_filter)
class VerifyTaskCommand(TaskCommand):
......
from datetime import datetime
import pytz
from django.core.management import call_command
from django.core.management.base import BaseCommand
from failmap.organizations.models import Organization, OrganizationType, Url
from failmap.map.models import Configuration
import logging
from django.conf import settings
from constance import config
log = logging.getLogger(__package__)
class Command(BaseCommand):
"""
Runs various live scans and requests as a long integration test. It's reliable to check for missing includes
and other code that crashed when loading or running. It is not a complete test per scanner / per function. And
it requires internet connectivity for a lot of the scans.
It also requires devserver to be running.
todo: in the future, move this test-set to the tests folder.
"""
help = __doc__
def handle(self, *args, **options):
if not settings.DEBUG:
log.error("Not running in debug mode. Given risky operations, this code will not run now.")
# todo: move to test environment.
try:
log.info("Setting up test-data.")
organization_type, organization, url, configuration = setup()
log.info("Testing app commands")
test_app_commands()
log.info("Testing hypersh commands")
test_hypersh_commands()
log.info("Testing map commands")
test_map_commands(organization, url)
log.info("Testing organization commands, this can take a while.")
test_organization_commands(organization)
log.info("Testing scanner commands, which can take a while.")
test_commandline_scanners(url)
log.info("Testing map views")
test_map_views(organization_type, organization)
except KeyboardInterrupt:
log.info("Received interrupt. Quitting.")
def test_map_views(organization_type, organization):
"""
Tests all urls on the map, to see if it loads / does not crash etc.
The correctness of each of these views is not validated. It could be utter garbage.
For this devserver needs to be running.
Todo: how is this done in other tests? -> via a docker IP. So these tests will have to be rewritten.
"""
address = "http://localhost:8000/"
from django.test.client import RequestFactory
rf = RequestFactory()
request = rf.get('/')
request.csrf_processing_done = True
request.META['HTTP_HOST'] = '127.0.0.1'
request.LANGUAGE_CODE = 'NL'
# autofill known parameters with some generic values:
autofill = {
'organization_type_name': organization_type.name,
'country': organization.country,
'organization_type': organization_type.name,
'days_back': 0,
'weeks_duration': 0,
'weeks_back': 0,
'organization_id': organization.id,
'displayed_issue': '', # todo: all the known issues from scanners
'scan_type': 'plain_https', # todo: all known scans
'request': request,
'parameter': '',
'args': {}, # LatestScanFeed
'kwargs': {'scan_type': 'plain_https'}, # LatestScanFeed
'file_format': 'json',
'organization_name': organization.name,
'url': 'http://localhost:8000/',
'requests_args': {"params": {"access_token": config.MAPBOX_ACCESS_TOKEN}},
}
import inspect
from failmap.map.urls import urlpatterns
for pattern in urlpatterns:
# can't test this, also part of django, so why test it here?
if pattern.name == "javascript-catalog":
continue
callback = pattern.callback
log.debug("Testing: %s" % callback)
arguments = inspect.signature(callback).parameters.keys()
kwargs = {}
for argument in arguments:
kwargs[argument] = autofill[argument]
output = callback(**kwargs)
log.debug("Received: %s" % output.content[0:80])
def test_app_commands():
call_debug_command('translate')
call_debug_command('ver')
# celery tested in test cases
# call_debug_command('celery')
# ignore devserver, we use that all day, every day
# test_dataset is run during build.
# production is used in production, so well...
def test_hypersh_commands():
# containers will be rebuilt if live anyway. It's not awesome, but well...
# nuking can have a lot of devastating impact when ran with the wrong credentials.
# call_debug_command('hyper_nuke')
pass
def test_map_commands(organization, url):
# this is going to take a while
call_debug_command('rebuild_reports', '-o', organization.name)
call_debug_command('report', '-o', organization.name)
call_debug_command('timeline', '-u', url.url)
call_debug_command('calculate_map_data', '--days', '1')
call_debug_command('calculate_vulnerability_graphs', '--days', '1')
call_debug_command('check_default_ratings')
call_debug_command('clear_cache')
call_debug_command('clear_ratings')
call_debug_command('import_coordinates', '--list')
call_debug_command('import_coordinates', '--country', 'NL', '--region', 'province')
call_debug_command('update_coordinates', '--country', 'NL', '--region', 'province')
def test_organization_commands(organization):
call_debug_command('add_urls', '-u', 'doesnotresolve.faalkaart.nl')
call_debug_command('export_organization', '--include_generated', organization.name)
call_debug_command('clean_short_outages')
call_debug_command('reset_autocomputed_fields_in_urls')
call_debug_command('update_datamodel_documentation')
call_debug_command('create_test_dataset', '--output', 'failmap_test_dataset_test')
call_debug_command('create_debug_dataset', '--output', 'failmap_debug_dataset_test')
call_debug_command('create_dataset', '--output', 'failmap_dataset_test')
# will not test clear_database, because of disasterous consequences. Could be done in test-database.
# todo: move to real test environment. Do we have network there? How do we get packages? So yes?
# call_debug_command('clear_database')
# call_debug_command('load_dataset', 'failmap_dataset_test')
def test_commandline_scanners(url):
# Calls all scanner commands, except the ones prefixed with one_shot (as they will be removed in the future)
call_debug_command('check_network')
call_debug_command('reset_onboards')
call_debug_command('forward_onboards')
# will not perform any db action without the --accept parameter
# failmap revive endpoint 2018-10-15
call_debug_command('revive', 'endpoints', '2018-10-15')
call_debug_command('revive', 'urls', '2018-10-15')
from failmap.scanners.management.commands.discover import scanners
for scanner in scanners:
call_debug_command('discover', scanner, '-u', url.url)
from failmap.scanners.management.commands.verify import scanners
for scanner in scanners:
call_debug_command('verify', scanner, '-u', url.url)
from failmap.scanners.management.commands.scan import scanners
for scanner in scanners:
call_debug_command('scan', scanner, '-u', url.url)
call_debug_command('set_latest_scan')
def call_debug_command(command_name, *args):
log.warning("Calling command: %s %s" % (command_name, ' '.join(args)))
call_command(command_name, *args)
def setup():
# Make a test organization + url
test_type = OrganizationType.objects.all().filter(name="test").first()
if not test_type:
test_type = OrganizationType()
test_type.name = "test"
test_type.save()
# Make sure the test organization can be scanned etc
test_organization = Organization.objects.all().filter(name="test").first()
if not test_organization:
test_organization = Organization()
test_organization.name = "test"
test_organization.created_on = datetime.now(pytz.utc)
test_organization.country = "NL"
test_organization.internal_notes = "Created for testing purposes only."
test_organization.type = test_type
test_organization.save()
test_url = Url.objects.all().filter(organization=test_organization).first()
if not test_url:
test_url = Url()
test_url.url = "faalkaart.nl"
test_url.save()
test_url.organization.add(test_organization)
test_url.save()
# make sure the test organization can be scanned, doesn't have to be displayed on the map or something.
configuration = Configuration.objects.all().filter(country="NL", organization_type=test_type).first()
if not configuration:
configuration = Configuration()
configuration.organization_type = test_type
configuration.country = "NL"
configuration.is_reported = True
configuration.is_scanned = True
configuration.is_displayed = False
configuration.is_the_default_option = False
configuration.save()
return test_type, test_organization, test_url, configuration
......@@ -4,7 +4,7 @@ import platform
import django
from django.core.management.base import BaseCommand
from .... import __version__
from failmap import __version__
log = logging.getLogger(__package__)
......
......@@ -10,7 +10,7 @@ from django.db import models
from django.utils import timezone
from jsonfield import JSONField
from ..celery import app
from failmap.celery import app
log = logging.getLogger(__name__)
......
# expose tasks for discovery by django_celery
from .models import create_job # noqa
from failmap.app.models import create_job # noqa
import pytest
from django.contrib.auth.models import User
from ..celery import app
from .models import Job
from failmap.celery import app
from failmap.app.models import Job
@pytest.fixture
......
......@@ -10,7 +10,7 @@ import flower.utils.broker
from celery import Celery, Task
from django.conf import settings
from .worker import QUEUES_MATCHING_ROLES
from failmap.celery.worker import QUEUES_MATCHING_ROLES
import logging
log = logging.getLogger(__package__)
......
......@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2018-11-21 12:10+0000\n"
"POT-Creation-Date: 2018-11-26 16:01+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
......
......@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2018-11-21 12:10+0000\n"
"POT-Creation-Date: 2018-11-26 16:01+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
......
......@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2018-11-21 12:10+0000\n"
"POT-Creation-Date: 2018-11-26 16:01+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
......
......@@ -5,7 +5,7 @@ from django.utils.html import format_html
from django_fsm_log.admin import StateLogInline
from import_export.admin import ImportExportModelAdmin
from .models import ContainerConfiguration, ContainerEnvironment, ContainerGroup, Credential
from failmap.hypersh.models import ContainerConfiguration, ContainerEnvironment, ContainerGroup, Credential
log = logging.getLogger(__name__)
......
......@@ -16,7 +16,7 @@ from django_fsm_log.models import StateLog
from hyper_sh import Client
from raven.contrib.django.raven_compat.models import client
from ..celery import app
from failmap.celery import app
DEFAULT_IMAGE = 'failmap/failmap:latest'
DEFAULT_COMMAND = 'celery worker --loglevel=info --concurrency=1'
......
......@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2018-11-21 12:10+0000\n"
"POT-Creation-Date: 2018-11-26 16:01+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
......
......@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2018-11-21 12:10+0000\n"
"POT-Creation-Date: 2018-11-26 16:01+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
......
......@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2018-11-21 12:10+0000\n"
"POT-Creation-Date: 2018-11-26 16:01+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
......
......@@ -5,11 +5,11 @@ from django.urls import reverse
from django.utils.html import format_html
from import_export.admin import ImportExportModelAdmin
from ..app.models import Job
from ..celery import PRIO_HIGH, app
from .geojson import import_from_scratch, update_coordinates
from .models import (AdministrativeRegion, Configuration, MapDataCache, OrganizationRating,
UrlRating, VulnerabilityStatistic)
from failmap.app.models import Job
from failmap.celery import PRIO_HIGH, app
from failmap.map.geojson import import_from_scratch, update_coordinates
from failmap.map.models import (AdministrativeRegion, Configuration, MapDataCache, OrganizationRating,
UrlRating, VulnerabilityStatistic)
@admin.register(OrganizationRating)
......
......@@ -204,6 +204,12 @@ def tls_qualys_encryption_quality_rating_based_on_scan(scan):
return standard_calculation(scan, explanation, high, medium, low)
def dummy_calculated_values(scan):
explanation = "This is a dummy scan."
high, medium, low = 0, 0, 0
return standard_calculation(scan, explanation, high, medium, low)
def standard_calculation(scan, explanation, high, medium, low):
return {
"type": scan.type,
......@@ -226,7 +232,8 @@ calculation_methods = {
'DNSSEC': dnssec_rating_based_on_scan,
'ftp': ftp_rating_based_on_scan,
'tls_qualys_certificate_trusted': tls_qualys_certificate_trusted_rating_based_on_scan,
'tls_qualys_encryption_quality': tls_qualys_encryption_quality_rating_based_on_scan
'tls_qualys_encryption_quality': tls_qualys_encryption_quality_rating_based_on_scan,
'Dummy': dummy_calculated_values
}
......
......@@ -22,8 +22,8 @@ from wikidata.client import Client
from failmap.celery import app
from ..organizations.models import Coordinate, Organization, OrganizationType, Url
from .models import AdministrativeRegion
from failmap.organizations.models import Coordinate, Organization, OrganizationType, Url
from failmap.map.models import AdministrativeRegion
log = logging.getLogger(__package__)
......@@ -88,7 +88,10 @@ def import_from_scratch(countries: List[str] = None, organization_types: List[st
log.info("The combination of %s and %s does not exist in OSM. Skipping." % (country, organization_type))
continue
data = get_osm_data_wambachers(country, organization_type)
if config.WAMBACHERS_OSM_CLIKEY:
data = get_osm_data_wambachers(country, organization_type)
else:
data = get_osm_data(country, organization_type)
for feature in data["features"]:
if "properties" not in feature:
......@@ -117,7 +120,10 @@ def update_coordinates(country: str = "NL", organization_type: str = "municipali
log.info("Attempting to update coordinates for: %s %s " % (country, organization_type))
# you are about to load 50 megabyte of data. Or MORE! :)
data = get_osm_data_wambachers(country, organization_type)
if config.WAMBACHERS_OSM_CLIKEY:
data = get_osm_data_wambachers(country, organization_type)
else:
data = get_osm_data(country, organization_type)
log.info("Received coordinate data. Starting with: %s" % json.dumps(data)[0:200])
......@@ -422,10 +428,20 @@ def store_updates(feature: Dict, country: str = "NL", organization_type: str = "
def get_osm_data_wambachers(country: str = "NL", organization_type: str = "municipality"):
"""
When the config.WAMBACHERS_OSM_CLIKEY is configured, this routine will get the map data.
It's the same as OSM, but with major seas and lakes cut out. That makes a nicer terrain on the website.
# uses https://wambachers-osm.website/boundaries/ to download map data. Might not be the most updated, but it has
# a more complete and better set of queries. For example; it DOES get northern ireland and it clips out the sea,
# which makes it very nice to look at.
# yes, i've donated, and so should you :)
:param country:
:param organization_type:
:return:
"""
"""
curl -f -o NL_province.zip 'https://wambachers-osm.website/boundaries/exportBoundaries
?cliVersion=1.0
......
......@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2018-11-21 12:10+0000\n"
"POT-Creation-Date: 2018-11-26 16:01+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
......
......@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2018-11-21 12:11+0000\n"
"POT-Creation-Date: 2018-11-26 16:01+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
......
......@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2018-11-21 12:10+0000\n"
"POT-Creation-Date: 2018-11-26 16:01+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: Fail Map <info@faalkaart.nl>\n"
"Language-Team: LANGUAGE <info@faalkaart.nl>\n"
......
......@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2018-11-21 12:11+0000\n"
"POT-Creation-Date: 2018-11-26 16:01+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
......
......@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2018-11-21 12:10+0000\n"
"POT-Creation-Date: 2018-11-26 16:01+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: Fail Map <info@faalkaart.nl>\n"
"Language-Team: LANGUAGE <info@faalkaart.nl>\n"
......
......@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2018-11-21 12:11+0000\n"
"POT-Creation-Date: 2018-11-26 16:01+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
......@@ -229,19 +229,27 @@ msgid "report_header_plain_https"
msgstr "🌈"
#: failmap/map/static/js/script.js
msgid "report_header_https_security_header_x_xss_protection"
#, fuzzy
#| msgid "report_header_https_security_header_x_xss_protection"
msgid "report_header_http_security_header_x_xss_protection"
msgstr "🌈"
#: failmap/map/static/js/script.js
msgid "report_header_https_security_header_x_frame_options"
#, fuzzy
#| msgid "report_header_https_security_header_x_frame_options"
msgid "report_header_http_security_header_x_frame_options"
msgstr "🌈"
#: failmap/map/static/js/script.js
msgid "report_header_https_security_header_x_content_type_options"
#, fuzzy
#| msgid "report_header_https_security_header_x_content_type_options"
msgid "report_header_http_security_header_x_content_type_options"
msgstr "🌈"