settings.py 45.9 KB
Newer Older
1
"""Django settings for failmap project.
2

3 4 5 6
You do not need to edit the settings listed below.

For example, you should not need to change the DEBUG setting here, ever. For this you can use
direnv, which will change your environment settings when you enter this projects directory.
7 8 9 10 11 12 13 14

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""
import os
Elger Jonker's avatar
Elger Jonker committed
15 16
# required until fixed: https://github.com/jazzband/django-constance/issues/263
from collections import OrderedDict
17
from datetime import timedelta
18

Johan Bloemberg's avatar
Johan Bloemberg committed
19 20
import raven
import raven.contrib.celery
Elger Jonker's avatar
lint  
Elger Jonker committed
21
from django.utils.translation import gettext_lazy as _
Johan Bloemberg's avatar
Johan Bloemberg committed
22 23 24 25
from pkg_resources import get_distribution

__version__ = get_distribution(__name__.split('.', 1)[0]).version

26
# this application can run in 3 modes: admin, interactive and frontend
Johan Bloemberg's avatar
Johan Bloemberg committed
27
# 'admin' exposes all routes and uses no caching. It should be restricted in access.
28 29 30 31 32 33 34 35 36 37 38
# 'interactive' does not expose administrative urls, but does allow write access. Access should be
# restricted but can be less restricted then admin.
# 'frontend' only exposes the visitor facing routes and serves with cache headers. It does not
# allow write access to the database. Access can be unrestricted but is preferably behind caching
# proxy.

# for usability default to most functional mode
APPLICATION_MODE = os.environ.get('APPLICATION_MODE', 'admin')

ADMIN = bool(APPLICATION_MODE == 'admin')
INTERACTIVE = bool(APPLICATION_MODE == 'interactive')
39

40
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
41 42 43 44 45
# BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Changed BASE_DIR so templates need to include the module and such. The idea
# was that otherwise the wrong template could be used when they have the same name
# over different dirs. Is this correct?
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
46 47 48 49 50

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
51
SECRET_KEY = os.environ.get('SECRET_KEY', 'ditisgeengeheimvriendachtjedatditeenwachtwoordwas')
52 53

# SECURITY WARNING: don't run with debug turned on in production!
54
DEBUG = os.environ.get('DEBUG', False)
55

56
ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', 'localhost,127.0.0.1,::1').split(',')
57

Johan Bloemberg's avatar
Johan Bloemberg committed
58 59 60
# allow better debugging for these clients
# https://docs.djangoproject.com/en/1.11/ref/settings/#internal-ips
INTERNAL_IPS = ['localhost', '127.0.0.1', '::1']
61 62 63 64

# Application definition

INSTALLED_APPS = [
65
    # needs to be before jet and admin to extend admin/base.html template
66 67
    'dal',
    'dal_select2',
68
    'failmap.app',
69 70
    'failmap.hypersh',
    'django_fsm_log',
71
    'adminsortable2',
72
    # Jet admin dashboard
73 74
    'jet.dashboard',
    'jet',
75
    'nested_admin',
76 77 78 79 80 81
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
82
    'import_export',
83
    'failmap.fail',
84
    'failmap.organizations.apps.OrganizationsConfig',  # because some signals need this.
85 86
    'failmap.scanners',
    'failmap.map',
87
    'failmap.game',
88
    'failmap.monitoryour',
89 90
    'django_countries',
    'django.contrib.admindocs',
91
    'django.contrib.humanize',
92
    'compressor',
93
    'django_celery_beat',
Johan Bloemberg's avatar
Johan Bloemberg committed
94
    'proxy',
95
    'django_statsd',
96 97
    'constance',
    'constance.backends.database',
98
    'leaflet',
99
    'djgeojson',
Elger Jonker's avatar
lint  
Elger Jonker committed
100
    'crispy_forms',  # for the game
101 102 103 104

    # django helpdesk requirements:
    'django.contrib.sites',  # Required for determining domain url for use in emails
    'markdown_deux',  # Required for Knowledgebase item formatting
Elger Jonker's avatar
lint  
Elger Jonker committed
105
    'bootstrapform',  # Required for nicer formatting of forms with the default templates
106 107
    'helpdesk',  # This is us!

108
    # 'mapwidgets', # we don't support gdal, as it's not in alpine stable yet.
109
    'colorful',
110
    'django_select2',
111
    # others:
112
    # 'mapwidgets',  no gdal available yet, try again later
113
    # 'cachalot',  # query cache, is not faster.
114
    # 'silk'  # works great for debugging.
115 116
]

Johan Bloemberg's avatar
Johan Bloemberg committed
117
try:
118 119
    # hack to disable django_uwsgi app as it currently conflicts with compressor
    # https://github.com/django-compressor/django-compressor/issues/881
Johan Bloemberg's avatar
Johan Bloemberg committed
120
    if not os.environ.get('COMPRESS', False):
121
        import django_uwsgi  # NOQA
122

123
        INSTALLED_APPS += ['django_uwsgi', ]
Johan Bloemberg's avatar
Johan Bloemberg committed
124 125 126 127
except ImportError:
    # only configure uwsgi app if installed (ie: production environment)
    pass

128 129
# don't run this in production
try:
130
    import django_extensions  # NOQA
131

132 133 134 135
    INSTALLED_APPS += ['django_extensions']
except ImportError:
    pass

Elger Jonker's avatar
Elger Jonker committed
136
MIDDLEWARE = [
137 138
    'django_statsd.middleware.GraphiteRequestTimingMiddleware',
    'django_statsd.middleware.GraphiteMiddleware',
139
    'django.contrib.sessions.middleware.SessionMiddleware',
140 141 142 143 144 145

    # based on cookie or based on user agent. So people with Dutch browsers will see the Dutch version(?).
    # This project is going to be used in a variety of countries, with multiple languages, so auto-setting this
    # makes sense over having a fixed single option.
    # https://docs.djangoproject.com/en/2.0/topics/i18n/translation/#how-django-discovers-language-preference
    'django.middleware.locale.LocaleMiddleware',
146 147 148 149 150 151
    'django.middleware.security.SecurityMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
152
    'django.contrib.admindocs.middleware.XViewMiddleware',  # admindocs
153 154
]

155 156 157 158 159
if DEBUG:
    # usage:
    # http://localhost:8000/data/vulnstats/NL/municipality/0?prof&count=100000&sort=cumtime
    MIDDLEWARE += ['django_cprofile_middleware.middleware.ProfilerMiddleware']

160
ROOT_URLCONF = 'failmap.urls'
161

162 163
# template needed for admin template
# this step is missing in the django jet tutorial, maybe because it's fairly common.
164

165 166 167
TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
168 169 170
        'DIRS': [
            BASE_DIR + '/',
        ],
171 172 173
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
174
                'constance.context_processors.config',
175 176 177 178 179 180 181 182 183
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

184
WSGI_APPLICATION = 'failmap.wsgi.application'
185

186 187 188
# Assume traffic is proxied from frontend loadbalancers
USE_X_FORWARDED_HOST = True
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
189 190 191

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases
192
# hopefully fixes helpdesk issue https://github.com/django-helpdesk/django-helpdesk/issues/184
193
DATABASE_OPTIONS = {
Elger Jonker's avatar
Elger Jonker committed
194
    'mysql': {'init_command': "SET character_set_connection=utf8,"
195 196
                              "collation_connection=utf8_unicode_ci,"
                              "sql_mode='STRICT_ALL_TABLES';"},
197

198
}
Johan Bloemberg's avatar
Johan Bloemberg committed
199 200 201 202
DB_ENGINE = os.environ.get('DB_ENGINE', 'mysql')
DATABASE_ENGINES = {
    'mysql': 'failmap.app.backends.mysql',
}
203 204 205 206 207
DATABASES_SETTINGS = {
    # persisten local database used during development (runserver)
    'dev': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.environ.get('DB_NAME', 'db.sqlite3'),
Johan Bloemberg's avatar
Johan Bloemberg committed
208
    },
209 210 211 212 213 214 215
    # sqlite memory database for running tests without
    'test': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.environ.get('DB_NAME', 'db.sqlite3'),
    },
    # for production get database settings from environment (eg: docker)
    'production': {
Johan Bloemberg's avatar
Johan Bloemberg committed
216
        'ENGINE': DATABASE_ENGINES.get(DB_ENGINE, 'django.db.backends.' + DB_ENGINE),
217 218 219 220
        'NAME': os.environ.get('DB_NAME', 'failmap'),
        'USER': os.environ.get('DB_USER', 'failmap'),
        'PASSWORD': os.environ.get('DB_PASSWORD', 'failmap'),
        'HOST': os.environ.get('DB_HOST', 'mysql'),
221
        'OPTIONS': DATABASE_OPTIONS.get(os.environ.get('DB_ENGINE', 'mysql'), {})
222
    }
223
}
224 225 226
# allow database to be selected through environment variables
DATABASE = os.environ.get('DJANGO_DATABASE', 'dev')
DATABASES = {'default': DATABASES_SETTINGS[DATABASE]}
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

249 250 251 252 253
# Go to admin: language_code = language_code.replace('_', '-').lower()
# AttributeError: 'NoneType' object has no attribute 'replace'
# while settings are loaded and Django uses LANGUAGE_CODE as default. What overrides this?
# a possible undesired solution, http://source.mihelac.org/2009/11/12/django-set-language-for-admin/

254 255
# http://stackoverflow.com/questions/1832709/django-how-to-make-translation-work
# shoddy documentation on dashes and underscores... different than the "ll" suggestion.
256
# LANGUAGE_CODE = 'en-gb'
257
# Less text is better :) See: https://www.youtube.com/watch?v=0j74jcxSunY
258
LANGUAGES = (
259
    ('en', 'English'),
260
    ('nl', 'Dutch'),
261 262
    # This helps to find missing translations. Using this language the goal is to replace all text
    # strings with a single rainbow. In the end there should be no text, only rainbows! 🌈
263
    ('rainbowsandunicorns', '🌈'),
264 265
)

266
# There is no 🌈🦄 translation for humanize. Instead you'll get the english fallback values.
267 268
# language is now depending on the user agent.
# LANGUAGE_CODE = 'en'
269

270 271
LANGUAGE_COOKIE_NAME = 'language'

272 273 274 275
TIME_ZONE = 'UTC'

USE_I18N = True
USE_L10N = True
276

277 278 279 280
# Loaddata will show a massive amount of warnings, therefore use load-dataset. load-dataset will
# do exactly the same as loaddata, but will overwrite below flag preventing warnings.
USE_TZ = True

281 282 283
# https://docs.djangoproject.com/en/1.11/topics/i18n/translation/#how-django-discovers-translations
# In all cases the name of the directory containing the translation is expected to be named using
# locale name notation. E.g. de, pt_BR, es_AR, etc.
284
LOCALE_PATHS = ['locale']
285

286

287 288 289 290
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'
Johan Bloemberg's avatar
Johan Bloemberg committed
291

Johan Bloemberg's avatar
Johan Bloemberg committed
292
# Absolute path to aggregate to and serve static file from.
293 294 295
if DEBUG:
    STATIC_ROOT = 'static'
else:
296
    STATIC_ROOT = '/srv/failmap/static/'
Johan Bloemberg's avatar
Johan Bloemberg committed
297

298
TEST_RUNNER = 'failmap.testrunner.PytestTestRunner'
299

300
# From the Jet documentation, a different color for a different season.
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
JET_THEMES = [
    {
        'theme': 'default',  # theme folder name
        'color': '#47bac1',  # color of the theme's button in user menu
        'title': 'Default'  # theme title
    },
    {
        'theme': 'green',
        'color': '#44b78b',
        'title': 'Green'
    },
    {
        'theme': 'light-green',
        'color': '#2faa60',
        'title': 'Light Green'
    },
    {
        'theme': 'light-violet',
        'color': '#a464c4',
        'title': 'Light Violet'
    },
    {
        'theme': 'light-blue',
        'color': '#5EADDE',
        'title': 'Light Blue'
    },
    {
        'theme': 'light-gray',
        'color': '#222',
        'title': 'Light Gray'
    }
]
333

334 335
# see: https://github.com/geex-arts/django-jet/blob/
#   fea07040229d1b56800a7b8e6234e5f9419e2114/docs/config_file.rst
336
# required for custom modules
337 338 339 340 341 342
JET_APP_INDEX_DASHBOARD = 'failmap.app.dashboard.CustomAppIndexDashboard'

# Customize the Dashboard index page remove unneeded panels (eg: feeds) and add usefull stuff (actions).
JET_INDEX_DASHBOARD = 'failmap.app.dashboard.CustomIndexDashboard'

# expand menu items by default for quick access
343
JET_SIDE_MENU_COMPACT = False
344 345 346 347 348 349 350 351 352 353 354 355

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',  # sys.stdout
            'formatter': 'color',
        },
    },
    'formatters': {
        'debug': {
356 357
            'format': '%(asctime)s\t%(levelname)-8s - %(filename)-20s:%(lineno)-4s - '
                      '%(funcName)20s() - %(message)s',
358 359 360
        },
        'color': {
            '()': 'colorlog.ColoredFormatter',
361
            'format': '%(log_color)s%(asctime)s\t%(levelname)-8s - '
362
                      '%(message)s',
Johan Bloemberg's avatar
Johan Bloemberg committed
363
            'datefmt': '%Y-%m-%d %H:%M',
364
            'log_colors': {
365 366 367 368
                'DEBUG': 'green',
                'INFO': 'white',
                'WARNING': 'yellow',
                'ERROR': 'red',
369 370 371 372 373
                'CRITICAL': 'bold_red',
            },
        }
    },
    'loggers': {
Elger Jonker's avatar
Elger Jonker committed
374 375 376 377 378 379 380 381 382 383
        # Used when there is no log defined or loaded. Disabled given we always use __package__ to log.
        # Would you enable it, all logging messages will be logged twice.
        # '': {
        #     'handlers': ['console'],
        #     'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
        # },

        # Default Django logging, we expect django to work, and therefore only show INFO messages.
        # It can be smart to sometimes want to see what's going on here, but not all the time.
        # https://docs.djangoproject.com/en/2.1/topics/logging/#django-s-logging-extensions
384 385 386 387
        'django': {
            'handlers': ['console'],
            'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'),
        },
Elger Jonker's avatar
Elger Jonker committed
388 389 390 391 392 393

        # We expect to be able to debug failmap all of the time.
        'failmap': {
            'handlers': ['console'],
            'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
        },
394 395 396
    },
}

397 398 399 400
# Add a slash at the end so we know it's a directory. Tries to somewhat prevents doing things in root.
OUTPUT_DIR = os.environ.get('OUTPUT_DIR', os.path.abspath(os.path.dirname(__file__)) + '/')
VENDOR_DIR = os.environ.get('VENDOR_DIR', os.path.abspath(os.path.dirname(__file__) + '/../vendor/') + '/')

401 402 403 404
# the tools dir in this case are very small tools that build upon external dependencies, such as dnscheck.
# only use this if the vendor dir does not provide the needed command(s) in a simple way
TOOLS_DIR = os.environ.get('TOOLS_DIR', os.path.abspath(os.path.dirname(__file__) + '/../tools/') + '/')

405 406 407
# A number of tools and outputs are grouped to easier have access to all of them.
# Our vendor directory contains a number of small tools that are hard to install otherwise.

408
TOOLS = {
409
    # Chrome and firefox are special cases: they install very easily and therefore don't need further grouping.
410 411
    'chrome': {
        'executable': {
412 413 414 415 416
            # os.platform is used to see what binaries should be used on a worker.
            'Darwin': os.environ.get(
                'CHROME_EXECUTABLE_DARWIN', "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"),
            'Linux': os.environ.get(
                'CHROME_EXECUTABLE_LINUX', ""),
417
        },
418 419
        'screenshot_output_dir': OUTPUT_DIR + os.environ.get(
            'CHROME_SCREENSHOT_OUTPUT_DIR', 'map/static/images/screenshots/'),
420
    },
421
    # Chrome and firefox are special cases: they install very easily and therefore don't need further grouping.
422 423
    'firefox': {
        'executable': {
424 425 426 427 428
            # os.platform is used to see what binaries should be used on a worker.
            'Darwin': os.environ.get(
                'FIREFOX_EXECUTABLE_DARWIN', "/Applications/Firefox.app/Contents/MacOS/firefox"),
            'Linux': os.environ.get(
                'FIREFOX_EXECUTABLE_LINUX', ""),
429
        },
430 431
        'screenshot_output_dir': OUTPUT_DIR + os.environ.get(
            'FIREFOX_SCREENSHOT_OUTPUT_DIR', 'map/static/images/screenshots/'),
432
    },
433
    'theHarvester': {
434 435
        'executable': VENDOR_DIR + os.environ.get('THEHARVESTER_EXECUTABLE', "theHarvester/theHarvester.py"),
        'output_dir': OUTPUT_DIR + os.environ.get('THEHARVESTER_OUTPUT_DIR', "scanners/resources/output/theHarvester/"),
436 437
    },
    'dnsrecon': {
438 439 440 441 442
        'executable': VENDOR_DIR + os.environ.get('DNSRECON_EXECUTABLE', "dnsrecon/dnsrecon.py"),
        'output_dir': OUTPUT_DIR + os.environ.get('DNSRECON_OUTPUT_DIR', "scanners/resources/output/dnsrecon/"),

        # The most important wordlists are auto-generated by this software, and are thus output.
        'wordlist_dir': OUTPUT_DIR + os.environ.get('DNSRECON_WORDLIST_DIR', "scanners/resources/wordlists/"),
443
    },
444
    'openstreetmap': {
Elger Jonker's avatar
Elger Jonker committed
445 446
        'output_dir': OUTPUT_DIR + os.environ.get('OPENSTREETMAP_OUTPUT_DIR',
                                                  "scanners/resources/output/openstreetmap/"),
447
    },
448
    'sslscan': {
449 450
        # this is beta functionality and not supported in production
        # these are installed system wide and don't require a path (they might when development continues)
451 452 453 454
        'executable': {
            'Darwin': 'sslscan',
            'Linux': 'sslscan',
        },
455
        'report_output_dir': OUTPUT_DIR + "scanners/resources/output/sslscan/",
456 457
    },
    'openssl': {
458 459
        # this is beta functionality and not supported in production
        # these are installed system wide and don't require a path  (they might when development continues)
460 461 462 463 464 465
        'executable': {
            'Darwin': 'openssl',
            'Linux': 'openssl',
        },
    },
    'TLS': {
466
        # this is beta functionality and not supported in production
467 468 469 470 471
        'cve_2016_2107': VENDOR_DIR + 'CVE-2016-2107-padding-oracle/main.go',
        'cve_2016_9244': VENDOR_DIR + 'CVE-2016-9244-ticketbleed/ticketbleed.go',
        'cert_chain_resolver': {
            'Darwin': VENDOR_DIR + 'cert-chain-resolver/cert-chain-resolver-darwin',
            'Linux': VENDOR_DIR + 'cert-chain-resolver/cert-chain-resolver-linux',
472 473 474
        },
        'tls_check_output_dir': OUTPUT_DIR + os.environ.get('TLSCHECK_OUTPUT_DIR',
                                                            'scanners/resources/output/tls_check/'),
475 476 477
    },
    'dnscheck': {
        'executable': TOOLS_DIR + 'dnssec.pl'
478 479 480
    },
    'osaft': {
        'json': VENDOR_DIR + 'osaft/JSON-array.awk',
481
    }
482
}
483

484
# Compression
485 486 487
# Django-compressor is used to compress css and js files in production
# During development this is disabled as it does not provide any feature there
# Django-compressor configuration defaults take care of this.
488
# https://django-compressor.readthedocs.io/en/latest/usage/
489
# which plugins to use to find static files
490
STATICFILES_FINDERS = (
491
    # default static files finders
492 493 494 495 496
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    # other finders..
    'compressor.finders.CompressorFinder',
)
497

498 499 500 501 502 503 504 505 506 507 508 509
COMPRESS_CSS_FILTERS = ['compressor.filters.cssmin.CSSCompressorFilter']

# Slimit doesn't work with vue. Tried two versions. Had to rewrite some other stuff.
# Now using the default, so not explicitly adding that to the settings
# COMPRESS_JS_FILTERS = ['compressor.filters.jsmin.JSMinFilter']

# Brotli compress storage gives some issues.
# This creates the original compressed and a gzipped compressed file.
COMPRESS_STORAGE = (
    'compressor.storage.GzipCompressorFileStorage'
)

510 511
# Disable caching during development and production.
# Django only emits caching headers, the webserver/caching-proxy makes sure the rest of the caching is handled.
Elger Jonker's avatar
Elger Jonker committed
512 513


514 515 516
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
517
    }
518
}
519

Johan Bloemberg's avatar
Johan Bloemberg committed
520 521 522 523 524
# Enable static file (js/css) compression when not running debug
# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_OFFLINE
COMPRESS_OFFLINE = not DEBUG
# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_ENABLED
# Enabled when debug is off by default.
525

526 527 528 529 530 531 532 533 534
# Celery 4.0 settings
# Pickle can work, but you need to use certificates to communicate (to verify the right origin)
# It's preferable not to use pickle, yet it's overly convenient as the normal serializer can not
# even serialize dicts.
# http://docs.celeryproject.org/en/latest/userguide/configuration.html
CELERY_accept_content = ['pickle', 'yaml']
CELERY_task_serializer = 'pickle'
CELERY_result_serializer = 'pickle'

535

536
# Celery config
537
CELERY_BROKER_URL = os.environ.get('BROKER', 'redis://localhost:6379/0')
538 539 540 541
ENABLE_UTC = True

# Any data transfered with pickle needs to be over tls... you can inject arbitrary objects with
# this stuff... message signing makes it a bit better, not perfect as it peels the onion.
542
# this stuff... message signing makes it a bit better, not perfect as it peels the onion.
543 544 545 546 547 548
# see: https://blog.nelhage.com/2011/03/exploiting-pickle/
# Yet pickle is the only convenient way of transporting objects without having to lean in all kinds
# of directions to get the job done. Intermediate tables to store results could be an option.
CELERY_ACCEPT_CONTENT = ['pickle']
CELERY_TASK_SERIALIZER = 'pickle'
CELERY_RESULT_SERIALIZER = 'pickle'
549 550
CELERY_TIMEZONE = 'UTC'

Johan Bloemberg's avatar
Johan Bloemberg committed
551
CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler'
Johan Bloemberg's avatar
Johan Bloemberg committed
552

553 554
CELERY_BROKER_CONNECTION_MAX_RETRIES = 1
CELERY_BROKER_CONNECTION_RETRY = False
555
CELERY_RESULT_EXPIRES = timedelta(hours=4)
556

Johan Bloemberg's avatar
Johan Bloemberg committed
557 558 559 560 561 562 563 564 565 566 567 568
# Use the value of 2 for celery prefetch multiplier. Previous was 1. The
# assumption is that 1 will block a worker thread until the current (rate
# limited) task is completed. When using 2 (or higher) the assumption is that
# celery will drop further rate limited task from the internal worker queue and
# fetch other tasks tasks that could be executed (spooling other rate limited
# tasks through in the process but to no hard except for a slight drop in
# overall throughput/performance). A to high value for the prefetch multiplier
# might result in high priority tasks not being picked up as Celery does not
# seem to do prioritisation in worker queues but only on the broker
# queues. The value of 2 is currently selected because it higher than 1,
# behaviour needs to be observed to decide if raising this results in
# further improvements without impacting the priority feature.
569
CELERY_WORKER_PREFETCH_MULTIPLIER = 2
570 571 572 573

# numer of tasks to be executed in parallel by celery
CELERY_WORKER_CONCURRENCY = 10

574 575 576 577 578 579 580 581 582 583
# Workers will scale up and scale down depending on the number of tasks
# available. To prevent workers from scaling down while still doing work,
# the ACKS_LATE setting is used. This insures that a task is removed from
# the task queue after the task is performed. This might result in some
# issues where tasks that don't finish or crash keep being executed:
# thus for tasks that are not programmed perfectly it will raise a number
# of repeated exceptions which will need to be debugged.
CELERY_ACKS_LATE = True


584 585 586 587 588 589
# Settings for statsd metrics collection. Statsd defaults over UDP port 8125.
# https://django-statsd.readthedocs.io/en/latest/#celery-signals-integration
STATSD_HOST = os.environ.get('STATSD_HOST', '127.0.0.1')
STATSD_PREFIX = 'failmap'
# register hooks for selery tasks
STATSD_CELERY_SIGNALS = True
590 591 592
# send database query metric (in production, in development we have debug toolbar for this)
if not DEBUG:
    STATSD_PATCHES = ['django_statsd.patches.db', ]
593 594 595

# enable some features during debug
if DEBUG:
596 597
    # enable debug toolbar if available
    try:
Johan Bloemberg's avatar
Johan Bloemberg committed
598
        import debug_toolbar
599

600
        INSTALLED_APPS.append('debug_toolbar')
Elger Jonker's avatar
Elger Jonker committed
601
        MIDDLEWARE.append('debug_toolbar.middleware.DebugToolbarMiddleware')
602 603

        import debug_toolbar.settings
604

605
        DEBUG_TOOLBAR_PANELS = [
Elger Jonker's avatar
lint  
Elger Jonker committed
606 607 608 609
            'ddt_request_history.panels.request_history.RequestHistoryPanel',
        ] + debug_toolbar.settings.PANELS_DEFAULTS + [
            'django_statsd.panel.StatsdPanel',
        ]
610
        # send statsd metrics to debug_toolbar
611
        STATSD_CLIENT = 'django_statsd.clients.toolbar'
Johan Bloemberg's avatar
Johan Bloemberg committed
612
    except ImportError:
613
        pass
614

Johan Bloemberg's avatar
Johan Bloemberg committed
615
# if sentry DSN is provided register raven to emit events on exceptions
616 617
SENTRY_DSN = os.environ.get('SENTRY_DSN')
if SENTRY_DSN:
Johan Bloemberg's avatar
Johan Bloemberg committed
618 619
    INSTALLED_APPS += ('raven.contrib.django.raven_compat',)
    RAVEN_CONFIG = {
620
        'dsn': SENTRY_DSN,
Johan Bloemberg's avatar
Johan Bloemberg committed
621 622
        'release': __version__,
    }
623 624
    # add sentry ID to request for inclusion in templates
    # https://docs.sentry.io/clients/python/integrations/django/#message-references
Elger Jonker's avatar
Elger Jonker committed
625
    MIDDLEWARE.insert(0, 'raven.contrib.django.raven_compat.middleware.SentryResponseErrorIdMiddleware')
Johan Bloemberg's avatar
Johan Bloemberg committed
626

Johan Bloemberg's avatar
Johan Bloemberg committed
627 628 629 630 631
    # Celery specific handlers
    client = raven.Client(SENTRY_DSN)
    raven.contrib.celery.register_logger_signal(client)
    raven.contrib.celery.register_signal(client)

Johan Bloemberg's avatar
Johan Bloemberg committed
632
# set javascript sentry token if provided
633
SENTRY_TOKEN = os.environ.get('SENTRY_TOKEN', '')
634 635 636 637

SENTRY_ORGANIZATION = 'internet-cleanup-foundation'
SENTRY_PROJECT = 'faalkaart'
SENTRY_PROJECT_URL = 'https://sentry.io/%s/%s' % (SENTRY_ORGANIZATION, SENTRY_PROJECT)
638 639 640 641 642 643 644

# Some workers or (development) environments don't support both IP networks
# Note that not supporting either protocols can result in all endpoints being killed as they are unreachable by scanners
# We don't check these settings anywhere for sanity as some workers might not need a network at all.
# The defaults stem from our live environment, where we've set IPv4 being present on all containers and workers.
NETWORK_SUPPORTS_IPV4 = os.environ.get('NETWORK_SUPPORTS_IPV4', True)
NETWORK_SUPPORTS_IPV6 = os.environ.get('NETWORK_SUPPORTS_IPV6', False)
645 646 647

# atomic imports: fail completely, not half
IMPORT_EXPORT_USE_TRANSACTIONS = True
648

649 650
#########
# Begin constance settigns
651
# runtime configuration from database
652 653 654
# https://django-constance.readthedocs.io/en/latest/
CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend'

Elger Jonker's avatar
Elger Jonker committed
655 656 657 658 659 660 661 662

CONSTANCE_ADDITIONAL_FIELDS = {
    'yes_no_null_select': ['django.forms.fields.ChoiceField', {
        'widget': 'django.forms.Select',
        'choices': ((None, "-----"), ("1", "Yes"), ("0", "No"))
    }],
}

Elger Jonker's avatar
Elger Jonker committed
663

664
CONSTANCE_CONFIG = {
Elger Jonker's avatar
Elger Jonker committed
665
    'COMMENTS': ('', 'Some comments by you that say a little bit about why below settings are the way they are.', str),
666

667 668 669 670
    'SHOW_ANNOUNCEMENT': (False, 'Shows the announcement bar. Announcements do not show when empty anyway.', bool),
    'ANNOUNCEMENT': ('', 'Text / HTML message that should appear on the top of the site / app. For example with '
                         'important news / features or current issues. You can enter any HTML, JS, CSS.', str),

671
    'PROJECT_NAME': ('', 'The name of this mapping project, used for branding and promotion.', str),
672
    'PROJECT_COUNTRY': ('NL', 'Two letter ISO code of the country that should be shown.', str),
Elger Jonker's avatar
Elger Jonker committed
673
    'PROJECT_TAGLINE': ('', 'Tagline for this project.', str),
674
    'PROJECT_WEBSITE': ('', 'The url where this site is located. Without trailing slash. Eg: https://example.com', str),
675 676 677 678 679 680 681
    'PROJECT_MAIL': ('', 'The address where people can contact for more info about this project.', str),
    'PROJECT_ISSUE_MAIL': ('', 'The address where people can mail when they encounter issues, for example when they '
                               'are using the incorrect findings button.', str),
    'PROJECT_TWITTER': ('', 'The twitter address where people can follow this project. Include the @!', str),
    'PROJECT_FACEBOOK': ('', 'The facebook address where people can follow this project. Has to be a complete url.',
                         str),

682 683 684 685

    'GOOGLE_MAPS_API_KEY': ('AIzaSyBXJbEUxGW1dAB4hJOlmKdYelfoRY6_fjo',
                            'API Key of google maps that can be used in the game / adding section.', str),

686 687 688
    'MAPBOX_ACCESS_TOKEN': ('pk.eyJ1IjoibXJmYWlsIiwiYSI6ImNqMHRlNXloczAwMWQyd3FxY3JkMnUxb3EifQ.9nJBaedxrry91O1d90wfuw',
                            'Access token for Mapbox', str),

689 690
    'WAMBACHERS_OSM_CLIKEY': ('', 'CLI key for Wambachers OSM export feature', str),

691 692 693 694 695 696 697 698 699
    'RESPONSIBLE_ORGANIZATION_NAME': ('', 'The name of the organization running this project.', str),
    'RESPONSIBLE_ORGANIZATION_PROMO_TEXT': ('', 'Some text promoting this organization and it\s mission. This text will'
                                                ' not be translated.', str),
    'RESPONSIBLE_ORGANIZATION_WEBSITE': ('', 'The name of the organization running this project.', str),
    'RESPONSIBLE_ORGANIZATION_MAIL': ('', 'The name of the organization running this project.', str),
    'RESPONSIBLE_ORGANIZATION_TWITTER': ('', 'The twitter address where people can follow this project. Include '
                                             'the @!', str),
    'RESPONSIBLE_ORGANIZATION_FACEBOOK': ('', 'The facebook address where people can follow this project. Make sure'
                                              ' this is a complete url.', str),
Elger Jonker's avatar
Elger Jonker committed
700 701 702
    'RESPONSIBLE_ORGANIZATION_LINKEDIN': ('', 'Linkedin page url.', str),
    'RESPONSIBLE_ORGANIZATION_WHATSAPP': ('', 'Whatsapp number.', str),
    'RESPONSIBLE_ORGANIZATION_PHONE': ('', 'Phone number, displayed as a sip:// addres.', str),
703

Elger Jonker's avatar
Elger Jonker committed
704
    'SHOW_INTRO': (True, 'Shows the standard introduction.', bool),
705
    'SHOW_GOOD_BAD': (True, 'Shows the good / bad lists.', bool),
Elger Jonker's avatar
Elger Jonker committed
706 707 708 709 710 711 712 713 714 715 716
    'SHOW_EXTENSIVE_STATISTICS': (True, 'Shows extended statistics.', bool),
    'SHOW_STATS_GRAPHS': (True, 'Shows graphs in extended statistics. '
                                'Extended statistics needs to be enabled for this to have effect.', bool),
    'SHOW_STATS_IMPROVEMENTS': (True, 'Shows improvements in extended statistics. '
                                      'Extended statistics needs to be enabled for this to have effect.', bool),
    'SHOW_STATS_NUMBERS': (True, 'Shows numbers in extended statistics. '
                                 'Extended statistics needs to be enabled for this to have effect.', bool),
    'SHOW_STATS_CHANGES': (True, 'Shows changes in extended statistics. '
                                 'Extended statistics needs to be enabled for this to have effect.', bool),
    'SHOW_DATASETS': (True, 'Shows dataset downloads. Note: dataset downloads are always available, even if they '
                            'are not shown on the website.', bool),
Elger Jonker's avatar
Elger Jonker committed
717
    'SHOW_COMPLY_OR_EXPLAIN': (False, 'Shows comply or explain sections and the explain link in findings.', bool),
Elger Jonker's avatar
Elger Jonker committed
718 719
    'SHOW_COMPLY_OR_EXPLAIN_DISCUSS': (False, 'Shows a link to the comply or explain discussion forum. The url of this'
                                              'forum can be edited below.', bool),
Elger Jonker's avatar
Elger Jonker committed
720
    'SHOW_TICKER': (False, 'Shows stock-ticker with updates in the past month.', bool),
721
    'TICKER_SLOGAN': ('failmap.org - monitor everything', 'Text to show between every 10 changes.', str),
Elger Jonker's avatar
Elger Jonker committed
722
    'SHOW_SCAN_SCHEDULE': (False, 'Shows list of upcoming scans, so everyone knows what scan is due next.', bool),
Elger Jonker's avatar
Elger Jonker committed
723

Elger Jonker's avatar
Elger Jonker committed
724
    'SHOW_SERVICES': (True, 'Show table with how many services are scanned. Requires SHOW_STATS_NUMBERS.', bool),
Elger Jonker's avatar
Elger Jonker committed
725 726 727 728 729 730 731
    'SHOW_DNS_DNSSEC': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
    'SHOW_HTTP_TLS_QUALYS': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
    'SHOW_HTTP_MISSING_TLS': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
    'SHOW_HTTP_HEADERS_HSTS': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
    'SHOW_HTTP_HEADERS_XFO': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
    'SHOW_HTTP_HEADERS_X_XSS': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
    'SHOW_HTTP_HEADERS_X_CONTENT': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
Elger Jonker's avatar
Elger Jonker committed
732
    'SHOW_FTP': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
Elger Jonker's avatar
Elger Jonker committed
733
    'SHOW_DONATION': (True, 'Show donation buttons and links on the site.', bool),
Elger Jonker's avatar
Elger Jonker committed
734

Elger Jonker's avatar
Elger Jonker committed
735 736 737
    'COMPLY_OR_EXPLAIN_DISCUSSION_FORUM_LINK': ('', 'Link to the comply or explain discussion forum.', str),
    'COMPLY_OR_EXPLAIN_EMAIL_ADDRESS': ('', 'E-mail where to receive explanations.', str),

Elger Jonker's avatar
Elger Jonker committed
738 739 740 741 742 743 744 745 746 747 748
    # todo: schedule this once per week by default.
    'DISCOVER_URLS_USING_NSEC': (True, 'Discover new domains using DNSSEC NSEC1 hashes? (See docs)', bool),
    'DISCOVER_URLS_USING_KNOWN_SUBDOMAINS': (True, 'Discover new domains using known subdomains? (See docs)', bool),
    'DISCOVER_URLS_USING_CERTIFICATE_TRANSPARENCY': (True, 'Discover new domains using crt.sh '
                                                           'certificate transperancy database? (See docs)', bool),
    'DISCOVER_HTTP_ENDPOINTS': (True, 'Do you want to discover HTTP endpoints?', bool),

    'SCAN_AT_ALL': (True, 'Do you want to scan at all?', bool),
    'SCAN_DNS_DNSSEC': (True, 'Do you want to scan for DNSSEC issues?', bool),
    'SCAN_HTTP_TLS_QUALYS': (True, 'Do you want to scan for TLS issues through Qualys? This is about 1 scan per '
                                   'two minutes. Cloud scanning can improve this a bit.', bool),
749 750
    'SCAN_HTTP_TLS_OSAFT': (True, 'Experimental: Do you want to scan for TLS issues through OSaft? This is about '
                                  '3 scans per minute. Cloud scanning can improve this a bit.', bool),
Elger Jonker's avatar
Elger Jonker committed
751 752 753 754 755 756
    'SCAN_HTTP_MISSING_TLS': (True, 'Do you want to scan for endpoints that don\'t have a secure counterpart?', bool),
    'SCAN_HTTP_HEADERS_HSTS': (True, 'Do you want to scan for missing '
                                     'Hypertext Strict Transport Security headers?', bool),
    'SCAN_HTTP_HEADERS_XFO': (True, 'Do you want to scan for missing X-Frame-Options headers?', bool),
    'SCAN_HTTP_HEADERS_X_XSS': (True, 'Do you want to scan for missing X-XSS headers?', bool),
    'SCAN_HTTP_HEADERS_X_CONTENT': (True, 'Do you want to scan for missing X-Content-Type issues?', bool),
Elger Jonker's avatar
Elger Jonker committed
757
    'SCAN_FTP': (True, 'Do you want to scan for FTP servers that are missing encryption?', bool),
Elger Jonker's avatar
Elger Jonker committed
758 759 760 761
    'CREATE_HTTP_SCREENSHOT': (True, 'Todo: Does not work yet! Do you want to create screenshots for HTTP endpoints?',
                               bool),

    # future: FTP, TLS_QUICK (way less robust and complete, much faster)
Elger Jonker's avatar
Elger Jonker committed
762
    'REPORT_INCLUDE_FTP': (True, 'Do you want to add FTP encryption issues to the report?', bool),
Elger Jonker's avatar
Elger Jonker committed
763 764 765 766 767 768 769 770
    'REPORT_INCLUDE_DNS_DNSSEC': (True, 'Do you want to add DNSSEC issues to the report?', bool),
    'REPORT_INCLUDE_HTTP_TLS_QUALYS': (True, 'Do you want to show TLS results in the report?', bool),
    'REPORT_INCLUDE_HTTP_MISSING_TLS': (True, 'Do you want to show missing TLS in the report?', bool),
    'REPORT_INCLUDE_HTTP_HEADERS_HSTS': (True, 'Do you want to HSTS in the report?', bool),
    'REPORT_INCLUDE_HTTP_HEADERS_XFO': (True, 'Do you want to show XFO in the report?', bool),
    'REPORT_INCLUDE_HTTP_HEADERS_X_XSS': (True, 'Do you want to show X-XSS protection headers in the report?', bool),
    'REPORT_INCLUDE_HTTP_HEADERS_X_CONTENT': (True, 'Do you want to show X-Content-Type headers in the report?', bool),

Elger Jonker's avatar
Elger Jonker committed
771 772 773 774 775 776 777
    # scanning pre-requisites
    'CONNECTIVITY_TEST_DOMAIN': ('faalkaart.nl', 'A server that is reachable over IPv4. This is used by a worker '
                                                 'to determine what kind of scans it can do. Enter an '
                                                 'address that you own or manage.', str),
    'IPV6_TEST_DOMAIN': ('faalkaart.nl', 'A server that is reachable over IPv6. This is used by a worker to determine '
                                         'what kind of scans it can do. Enter an address that you own or manage.', str),

Elger Jonker's avatar
Elger Jonker committed
778 779
    'GITTER_CHAT_ENABLE': (False, 'Show the Gitter Chat option on the website (using gitter sidecar).', bool),
    'GITTER_CHAT_CHANNEL': ('internet-cleanup-foundation/support', 'Name of the channel chat takes place.', str),
Elger Jonker's avatar
Elger Jonker committed
780

Elger Jonker's avatar
Elger Jonker committed
781 782 783 784
    #
    # 'USE_CUSTOM_INTRO': (False, 'If you want to use an (untranslated) custom intro, enable this. Enabling this'
    #                             'will remove the standard intro and has several fields that can be filled with '
    #                             'custom HTML.', bool),
785
}
786

Elger Jonker's avatar
Elger Jonker committed
787
CONSTANCE_CONFIG_FIELDSETS = OrderedDict([
788
    ('General', ('COMMENTS', 'SHOW_ANNOUNCEMENT', 'ANNOUNCEMENT', 'MAPBOX_ACCESS_TOKEN', 'WAMBACHERS_OSM_CLIKEY')),
789

790 791
    ('Project', ('PROJECT_NAME', 'PROJECT_COUNTRY', 'PROJECT_TAGLINE', 'PROJECT_WEBSITE', 'PROJECT_MAIL',
                 'PROJECT_ISSUE_MAIL', 'PROJECT_TWITTER', 'PROJECT_FACEBOOK')),
792

Elger Jonker's avatar
Elger Jonker committed
793 794 795 796 797
    ('Responsible', ('RESPONSIBLE_ORGANIZATION_NAME', 'RESPONSIBLE_ORGANIZATION_PROMO_TEXT',
                     'RESPONSIBLE_ORGANIZATION_WEBSITE', 'RESPONSIBLE_ORGANIZATION_MAIL',
                     'RESPONSIBLE_ORGANIZATION_TWITTER', 'RESPONSIBLE_ORGANIZATION_FACEBOOK',
                     'RESPONSIBLE_ORGANIZATION_LINKEDIN', 'RESPONSIBLE_ORGANIZATION_WHATSAPP',
                     'RESPONSIBLE_ORGANIZATION_PHONE')),
798

Elger Jonker's avatar
Elger Jonker committed
799
    ('Website', ('SHOW_INTRO', 'SHOW_GOOD_BAD', 'SHOW_EXTENSIVE_STATISTICS', 'SHOW_DATASETS', 'SHOW_STATS_GRAPHS',
Elger Jonker's avatar
Elger Jonker committed
800
                 'SHOW_STATS_IMPROVEMENTS', 'SHOW_STATS_NUMBERS', 'SHOW_SERVICES', 'SHOW_STATS_CHANGES',
Elger Jonker's avatar
Elger Jonker committed
801 802
                 'SHOW_DNS_DNSSEC', 'SHOW_HTTP_TLS_QUALYS', 'SHOW_HTTP_MISSING_TLS',
                 'SHOW_HTTP_HEADERS_HSTS', 'SHOW_HTTP_HEADERS_XFO', 'SHOW_HTTP_HEADERS_X_XSS',
Elger Jonker's avatar
Elger Jonker committed
803
                 'SHOW_HTTP_HEADERS_X_CONTENT', 'SHOW_FTP', 'SHOW_SCAN_SCHEDULE', 'SHOW_DONATION'
Elger Jonker's avatar
Elger Jonker committed
804
                 )),
Elger Jonker's avatar
Elger Jonker committed
805

Elger Jonker's avatar
Elger Jonker committed
806 807 808 809
    ('Comply or Explain', ('SHOW_COMPLY_OR_EXPLAIN', 'SHOW_COMPLY_OR_EXPLAIN_DISCUSS',
                           'COMPLY_OR_EXPLAIN_DISCUSSION_FORUM_LINK', 'COMPLY_OR_EXPLAIN_EMAIL_ADDRESS'
                           )),

Elger Jonker's avatar
Elger Jonker committed
810 811
    ('Discovery', ('DISCOVER_URLS_USING_NSEC', 'DISCOVER_URLS_USING_KNOWN_SUBDOMAINS',
                   'DISCOVER_URLS_USING_CERTIFICATE_TRANSPARENCY', 'DISCOVER_HTTP_ENDPOINTS')),
Elger Jonker's avatar
Elger Jonker committed
812

Elger Jonker's avatar
Elger Jonker committed
813
    ('Scanner Capabilities', ('CONNECTIVITY_TEST_DOMAIN', 'IPV6_TEST_DOMAIN')),
Elger Jonker's avatar
Elger Jonker committed
814

815 816
    ('Scanning', ('SCAN_AT_ALL', 'SCAN_DNS_DNSSEC', 'SCAN_HTTP_TLS_QUALYS', 'SCAN_HTTP_TLS_OSAFT',
                  'SCAN_HTTP_MISSING_TLS', 'SCAN_HTTP_HEADERS_HSTS',
Elger Jonker's avatar
Elger Jonker committed
817
                  'SCAN_HTTP_HEADERS_XFO', 'SCAN_HTTP_HEADERS_X_XSS', 'SCAN_HTTP_HEADERS_X_CONTENT', 'SCAN_FTP',
Elger Jonker's avatar
Elger Jonker committed
818
                  'CREATE_HTTP_SCREENSHOT')),
Elger Jonker's avatar
Elger Jonker committed
819

Elger Jonker's avatar
Elger Jonker committed
820 821 822
    ('Reporting', ('REPORT_INCLUDE_DNS_DNSSEC', 'REPORT_INCLUDE_HTTP_TLS_QUALYS', 'REPORT_INCLUDE_HTTP_MISSING_TLS',
                   'REPORT_INCLUDE_HTTP_HEADERS_HSTS',
                   'REPORT_INCLUDE_HTTP_HEADERS_XFO', 'REPORT_INCLUDE_HTTP_HEADERS_X_XSS',
Elger Jonker's avatar
Elger Jonker committed
823
                   'REPORT_INCLUDE_HTTP_HEADERS_X_CONTENT', 'REPORT_INCLUDE_FTP')),
824 825 826

    ('Game', ('GOOGLE_MAPS_API_KEY',)),

827 828
    ('Ticker', ('SHOW_TICKER', 'TICKER_SLOGAN')),

Elger Jonker's avatar
Elger Jonker committed
829
    ('Chat (using gitter)', ('GITTER_CHAT_ENABLE', 'GITTER_CHAT_CHANNEL'))
Elger Jonker's avatar
Elger Jonker committed
830
])
Elger Jonker's avatar
Elger Jonker committed
831 832 833 834 835 836 837 838 839

# Check for constance configuration issues:
# See also: https://github.com/jazzband/django-constance/issues/293
variables_in_fieldsets = [i for sub in [CONSTANCE_CONFIG_FIELDSETS[x] for x in CONSTANCE_CONFIG_FIELDSETS] for i in sub]
variables_in_config = [x for x in CONSTANCE_CONFIG]
missing = set(variables_in_config) - set(variables_in_fieldsets)
if missing:
    raise EnvironmentError("Constance config variables %s are missing in constance config fieldsets." % missing)

840 841
# End constance settings
########
842

843
# https://docs.djangoproject.com/en/1.11/ref/settings/#data-upload-max-number-fields
844 845 846
# The default is far too low with various inlines (even on the test dataset).
# Yes, we happily exceed 1000 fields anyday. No problem :)
DATA_UPLOAD_MAX_NUMBER_FIELDS = 4242
Johan Bloemberg's avatar
Johan Bloemberg committed
847

848 849
########
# Begin game settings:
Johan Bloemberg's avatar
Johan Bloemberg committed
850 851
# login on the frontpage should redirect to the game landingpage
LOGIN_REDIRECT_URL = '/game/'
852 853
LOGIN_URL = '/authentication/login/'
LOGOUT_REDIRECT_URL = '/'
854

Elger Jonker's avatar
Elger Jonker committed
855
CRISPY_TEMPLATE_PACK = 'bootstrap3'
856

857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878
# End game settigns
#######

#######
# Begin helpdesk settings:
# Unfortunately we cannot manage these settings with constance. And therefore you should be still editing
# this file. We're sorry.
SITE_ID = 1  # For django-helpdesk. We only run one site...

# helps against spam, will leak all your data, so be careful. Spam could be protected against on your mailserver...
AKISMET_API_KEY = os.environ.get('AKISMET_API_KEY', '')
TYPEPAD_ANTISPAM_API_KEY = os.environ.get('TYPEPAD_ANTISPAM_API_KEY', '')

# If you use another server for sending mail, see config example here:
# http://django-helpdesk.readthedocs.io/en/0.2.x/configuration.html
# EMAIL_HOST = 'XXXXX'
# EMAIL_HOST_USER = '[email protected]'
# EMAIL_HOST_PASSWORD = '123456'
# Make sure the MEDIA_ROOT is NOT readable from the webserver directly. So no https://bla/media.
MEDIA_ROOT = os.environ.get('MEDIA_ROOT', os.path.abspath(os.path.dirname(__file__)) + '/../')

# End of helpdesk settings
Elger Jonker's avatar
Elger Jonker committed
879 880 881 882
#######


#######
Elger Jonker's avatar
Elger Jonker committed
883
# begin django jet menu configuration
Elger Jonker's avatar
Elger Jonker committed
884 885 886 887 888 889 890 891
# This helps making various resources and fewer used features less visible.


# todo: add permissions: 'permissions': ['core.user']

# The following items are hidden on purpose:
# core.site (we only have one site)
# scanners.state, will be deprecated and removed (if not already)
Elger Jonker's avatar
Elger Jonker committed
892
# See: http://jet.readthedocs.io/en/latest/config_file.html#custom-menu
893 894 895 896
# Permissions are AND-ed together.
# admin (a nonsense permission) has been added everywhere to avoid "empty arrows" when signing in with a role with
# limited permissions.
# For the default labels, see: https://docs.djangoproject.com/en/2.1/topics/auth/default/#topic-authorization
Elger Jonker's avatar
Elger Jonker committed
897
JET_SIDE_MENU_ITEMS = [  # A list of application or custom item dicts
Elger Jonker's avatar
Elger Jonker committed
898 899

    {'label': _('🔧 configuration'), 'items': [
Elger Jonker's avatar
Elger Jonker committed
900 901
        {'name': 'auth.user'},
        {'name': 'auth.group'},
Elger Jonker's avatar
Elger Jonker committed
902 903
        {'name': 'constance.config', 'label': _('configuration')},
        {'name': 'map.configuration', 'label': _('map configuration')},
Elger Jonker's avatar
Elger Jonker committed
904
        {'name': 'map.administrativeregion', 'label': _('import regions')},
905
    ], 'permissions': ['admin']},
Elger Jonker's avatar
Elger Jonker committed
906

Elger Jonker's avatar
Elger Jonker committed
907
    {'app_label': 'organizations', 'label': _('🏢 organizations'), 'items': [
Elger Jonker's avatar
Elger Jonker committed
908 909 910 911 912
        {'name': 'organization'},
        {'name': 'url'},
        {'name': 'promise'},
        {'name': 'coordinate'},
        {'name': 'organizationtype'},
913
    ], 'permissions': ['admin']},
Elger Jonker's avatar
Elger Jonker committed
914

915
    # todo: sort scan moment to show latest first.
Elger Jonker's avatar
Elger Jonker committed
916
    {'app_label': 'scanners', 'label': _('🔬 scanners'), 'items': [
917 918 919 920 921 922 923 924 925
        {'name': 'endpoint', 'permissions': ['admin']},
        {'name': 'endpointgenericscan', 'permissions': ['scanners.change_endpointgenericscan']},
        {'name': 'tlsscan', 'permissions': ['scanners.change_tlsscan']},
        {'name': 'tlsqualysscan', 'permissions': ['scanners.change_tlsqualysscan']},
        {'name': 'urlgenericscan', 'permissions': ['scanners.change_urlgenericscan']},
        {'name': 'screenshot', 'permissions': ['admin']},
        {'name': 'urlip', 'permissions': ['admin']},
        {'name': 'tlsqualysscratchpad', 'permissions': ['admin']},
        {'name': 'endpointgenericscanscratchpad', 'permissions': ['admin']},
Elger Jonker's avatar
Elger Jonker committed
926 927
    ]},

Elger Jonker's avatar
Elger Jonker committed
928 929 930
    {'label': _('🗺️ map (autogenerated)'), 'items': [
        {'name': 'map.organizationrating'},
        {'name': 'map.urlrating'},
931 932
        {'name': 'map.vulnerabilitystatistic'},
        {'name': 'map.mapdatacache'}
933
    ], 'permissions': ['admin']},
Elger Jonker's avatar
Elger Jonker committed
934 935

    {'label': _('🕒 periodic tasks'), 'items': [
Elger Jonker's avatar
Elger Jonker committed
936 937 938 939 940
        {'name': 'app.job'},
        {'name': 'django_celery_beat.periodictask'},
        {'name': 'django_celery_beat.crontabschedule'},
        {'name': 'django_celery_beat.intervalschedule'},
        {'name': 'django_celery_beat.solarschedule'},
941
    ], 'permissions': ['admin']},
Elger Jonker's avatar
Elger Jonker committed
942

Elger Jonker's avatar
Elger Jonker committed
943
    {'app_label': 'helpdesk', 'label': _('ℹ️ helpdesk'), 'items': [
Elger Jonker's avatar
Elger Jonker committed
944 945 946 947 948 949 950 951 952 953
        {'name': 'queue'},
        {'name': 'ticket'},
        {'name': 'followup'},
        {'name': 'customfield'},
        {'name': 'presetreply'},
        {'name': 'emailtemplate'},
        {'name': 'escalationexclusion'},
        {'name': 'ignoreemail'},
        {'name': 'kbcategory'},
        {'name': 'kbitem'},
954
    ], 'permissions': ['admin']},
Elger Jonker's avatar
Elger Jonker committed
955

Elger Jonker's avatar
Elger Jonker committed
956
    {'app_label': 'hypersh', 'label': _('☁️ hypersh cloud scans'), 'items': [
Elger Jonker's avatar
Elger Jonker committed
957 958 959
        {'name': 'containerenvironment', 'label': _('Environment variables'), },
        {'name': 'containerconfiguration', 'label': _('Container configuration'), },
        {'name': 'containergroup', 'label': _('Container instances'), },
Elger Jonker's avatar
Elger Jonker committed
960
        {'name': 'credential'},
961
    ], 'permissions': ['admin']},
Elger Jonker's avatar
Elger Jonker committed
962

Elger Jonker's avatar
Elger Jonker committed
963
    {'app_label': 'game', 'label': _('👾️ the game'), 'items': [
Elger Jonker's avatar
Elger Jonker committed
964 965 966
        {'name': 'contest'},
        {'name': 'team'},
        {'name': 'organizationsubmission'},
967 968 969
        {'label': _('New organizations'),
         'url': '/admin/game/organizationsubmission/?has_been_accepted__exact=0&has_been_rejected__exact=0&o=-5',
         'url_blank': False},
Elger Jonker's avatar
Elger Jonker committed
970
        {'name': 'urlsubmission'},
971 972 973
        {'label': _('New urls'),
         'url': '/admin/game/urlsubmission/?has_been_accepted__exact=0&has_been_rejected__exact=0&o=-6.2.3',
         'url_blank': False},
974
    ], 'permissions': ['admin']},
Elger Jonker's avatar
Elger Jonker committed
975
]
Elger Jonker's avatar
Elger Jonker committed
976 977
# end django jet menu configuration
########
978 979 980

########
# Begin Cacheops
981 982 983
#         # django cacheops doesn't work with raw.
#         # too bad https://github.com/Suor/django-cacheops
#         # it's the elephant in the room in the documentation: all are explained except this one.
984 985 986 987

# Cacheops has been added to improve the retrieval of graphs-queries. At the time of writing it's only in use
# there using a hack to improve querying speed.

Elger Jonker's avatar
Elger Jonker committed
988 989 990
# It was not an option to rewrite queries to tailor to specific caching schemes per database vendor, django ORM also
# does not support that.

991 992 993 994 995 996 997 998 999 1000 1001 1002
# It's a hack because out of the box cacheops doesn't support raw querysets (the only caveat without explanation).
# But we just need the list of data for displaying, thus we can wrap that in a function and use function result caching
#
# Note that lru_cache from functools does not support lists or complex types, while cahcheops does.
#
# I've chosen cacheops because it's actively maintained and has a lot of commits, while it's backlog is clean.
# Another reason is that it doesn't affect anything, except what you explicitly cache. So behaviour of the rest of the
# application is unchanged.

# If redis is not available, that is not a problem. For example: certain development scenario's or when redis fails.
# in that case a limited set of functions are performed without caching (and thus slower) but without crashes.
# If it's too slow, the webserver will kill it anyway, or in dev environments it will take longer.
1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
# CACHEOPS_DEGRADE_ON_FAILURE = True

# CACHEOPS_REDIS = {
#     'host': 'localhost',  # redis-server is on same machine
#     'port': 6379,        # default redis port
#     'db': 1,             # SELECT non-default redis database
#                          # using separate redis db or redis instance
#                          # is highly recommended
#
#     'socket_timeout': 3,   # connection timeout in seconds, optional
# }
1014 1015
# End cacheops
########