settings.py 46.3 KB
Newer Older
1
"""Django settings for failmap project.
2

3 4 5 6
You do not need to edit the settings listed below.

For example, you should not need to change the DEBUG setting here, ever. For this you can use
direnv, which will change your environment settings when you enter this projects directory.
7 8 9 10 11 12 13 14

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""
import os
Elger Jonker's avatar
Elger Jonker committed
15 16
# required until fixed: https://github.com/jazzband/django-constance/issues/263
from collections import OrderedDict
17
from datetime import timedelta
18

Johan Bloemberg's avatar
Johan Bloemberg committed
19 20
import raven
import raven.contrib.celery
Elger Jonker's avatar
lint  
Elger Jonker committed
21
from django.utils.translation import gettext_lazy as _
22 23 24 25
from pkg_resources import get_distribution

__version__ = get_distribution(__name__.split('.', 1)[0]).version

26
# this application can run in 3 modes: admin, interactive and frontend
Johan Bloemberg's avatar
Johan Bloemberg committed
27
# 'admin' exposes all routes and uses no caching. It should be restricted in access.
28 29 30 31 32 33 34 35 36 37 38
# 'interactive' does not expose administrative urls, but does allow write access. Access should be
# restricted but can be less restricted then admin.
# 'frontend' only exposes the visitor facing routes and serves with cache headers. It does not
# allow write access to the database. Access can be unrestricted but is preferably behind caching
# proxy.

# for usability default to most functional mode
APPLICATION_MODE = os.environ.get('APPLICATION_MODE', 'admin')

ADMIN = bool(APPLICATION_MODE == 'admin')
INTERACTIVE = bool(APPLICATION_MODE == 'interactive')
39

40
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
41 42 43 44 45
# BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Changed BASE_DIR so templates need to include the module and such. The idea
# was that otherwise the wrong template could be used when they have the same name
# over different dirs. Is this correct?
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
46 47 48 49 50

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
51
SECRET_KEY = os.environ.get('SECRET_KEY', 'ditisgeengeheimvriendachtjedatditeenwachtwoordwas')
52 53

# SECURITY WARNING: don't run with debug turned on in production!
54
DEBUG = os.environ.get('DEBUG', False)
55

56
ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', 'localhost,127.0.0.1,::1').split(',')
57

58 59 60
# allow better debugging for these clients
# https://docs.djangoproject.com/en/1.11/ref/settings/#internal-ips
INTERNAL_IPS = ['localhost', '127.0.0.1', '::1']
61 62 63 64

# Application definition

INSTALLED_APPS = [
65
    # needs to be before jet and admin to extend admin/base.html template
66 67
    'dal',
    'dal_select2',
68
    'failmap.app',
69 70
    'failmap.hypersh',
    'django_fsm_log',
71
    'adminsortable2',
72
    # Jet admin dashboard
73 74
    'jet.dashboard',
    'jet',
75
    'nested_admin',
76 77 78 79 80 81
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
82
    'import_export',
83
    'failmap.fail',
84
    'failmap.organizations.apps.OrganizationsConfig',  # because some signals need this.
85 86
    'failmap.scanners',
    'failmap.map',
87
    'failmap.game',
88 89
    'django_countries',
    'django.contrib.admindocs',
90
    'django.contrib.humanize',
91
    'compressor',
92
    'django_celery_beat',
Johan Bloemberg's avatar
Johan Bloemberg committed
93
    'proxy',
94
    'django_statsd',
95 96
    'constance',
    'constance.backends.database',
97
    'leaflet',
98
    'djgeojson',
Elger Jonker's avatar
lint  
Elger Jonker committed
99
    'crispy_forms',  # for the game
100 101

    # django helpdesk requirements:
102 103 104
    # We don't use this yet, as it had issues during build.
    # 'django.contrib.sites',  # Required for determining domain url for use in emails
    # 'markdown_deux',  # Required for Knowledgebase item formatting
105
    # 'bootstrapform',  # Required for nicer formatting of forms with the default templates
106
    # 'helpdesk',  # This is us!
107

108
    # 'mapwidgets', # we don't support gdal, as it's not in alpine stable yet.
109
    'colorful',
110
    'django_select2',
111
    # others:
112
    # 'mapwidgets',  no gdal available yet, try again later
113
    # 'cachalot',  # query cache, is not faster.
114
    # 'silk'  # works great for debugging.
115 116
]

Johan Bloemberg's avatar
Johan Bloemberg committed
117
try:
118 119
    # hack to disable django_uwsgi app as it currently conflicts with compressor
    # https://github.com/django-compressor/django-compressor/issues/881
Johan Bloemberg's avatar
Johan Bloemberg committed
120
    if not os.environ.get('COMPRESS', False):
121
        import django_uwsgi  # NOQA
122

123
        INSTALLED_APPS += ['django_uwsgi', ]
Johan Bloemberg's avatar
Johan Bloemberg committed
124 125 126 127
except ImportError:
    # only configure uwsgi app if installed (ie: production environment)
    pass

128 129
# don't run this in production
try:
130
    import django_extensions  # NOQA
131

132 133 134 135
    INSTALLED_APPS += ['django_extensions']
except ImportError:
    pass

Elger Jonker's avatar
Elger Jonker committed
136
MIDDLEWARE = [
137 138
    'django_statsd.middleware.GraphiteRequestTimingMiddleware',
    'django_statsd.middleware.GraphiteMiddleware',
139
    'django.contrib.sessions.middleware.SessionMiddleware',
140 141 142 143 144 145

    # based on cookie or based on user agent. So people with Dutch browsers will see the Dutch version(?).
    # This project is going to be used in a variety of countries, with multiple languages, so auto-setting this
    # makes sense over having a fixed single option.
    # https://docs.djangoproject.com/en/2.0/topics/i18n/translation/#how-django-discovers-language-preference
    'django.middleware.locale.LocaleMiddleware',
146 147 148 149 150 151
    'django.middleware.security.SecurityMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
152
    'django.contrib.admindocs.middleware.XViewMiddleware',  # admindocs
153 154
]

155 156 157 158 159
if DEBUG:
    # usage:
    # http://localhost:8000/data/vulnstats/NL/municipality/0?prof&count=100000&sort=cumtime
    MIDDLEWARE += ['django_cprofile_middleware.middleware.ProfilerMiddleware']

160
ROOT_URLCONF = 'failmap.urls'
161

162 163
# template needed for admin template
# this step is missing in the django jet tutorial, maybe because it's fairly common.
164

165 166 167
TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
168 169 170
        'DIRS': [
            BASE_DIR + '/',
        ],
171 172 173
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
174
                'constance.context_processors.config',
175 176 177 178 179 180 181 182 183
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

184
WSGI_APPLICATION = 'failmap.wsgi.application'
185

186 187 188
# Assume traffic is proxied from frontend loadbalancers
USE_X_FORWARDED_HOST = True
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
189 190 191

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases
192
# hopefully fixes helpdesk issue https://github.com/django-helpdesk/django-helpdesk/issues/184
193
# These settings also make sense if you don't use helpdesk, so they are still here.
194
DATABASE_OPTIONS = {
Elger Jonker's avatar
Elger Jonker committed
195
    'mysql': {'init_command': "SET character_set_connection=utf8,"
196 197
                              "collation_connection=utf8_unicode_ci,"
                              "sql_mode='STRICT_ALL_TABLES';"},
198

199
}
Johan Bloemberg's avatar
Johan Bloemberg committed
200 201 202 203
DB_ENGINE = os.environ.get('DB_ENGINE', 'mysql')
DATABASE_ENGINES = {
    'mysql': 'failmap.app.backends.mysql',
}
204 205 206 207 208
DATABASES_SETTINGS = {
    # persisten local database used during development (runserver)
    'dev': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.environ.get('DB_NAME', 'db.sqlite3'),
Johan Bloemberg's avatar
Johan Bloemberg committed
209
    },
210 211 212 213 214 215 216
    # sqlite memory database for running tests without
    'test': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.environ.get('DB_NAME', 'db.sqlite3'),
    },
    # for production get database settings from environment (eg: docker)
    'production': {
Johan Bloemberg's avatar
Johan Bloemberg committed
217
        'ENGINE': DATABASE_ENGINES.get(DB_ENGINE, 'django.db.backends.' + DB_ENGINE),
218 219 220 221
        'NAME': os.environ.get('DB_NAME', 'failmap'),
        'USER': os.environ.get('DB_USER', 'failmap'),
        'PASSWORD': os.environ.get('DB_PASSWORD', 'failmap'),
        'HOST': os.environ.get('DB_HOST', 'mysql'),
222
        'OPTIONS': DATABASE_OPTIONS.get(os.environ.get('DB_ENGINE', 'mysql'), {})
223
    }
224
}
225 226 227
# allow database to be selected through environment variables
DATABASE = os.environ.get('DJANGO_DATABASE', 'dev')
DATABASES = {'default': DATABASES_SETTINGS[DATABASE]}
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

250 251 252 253 254
# Go to admin: language_code = language_code.replace('_', '-').lower()
# AttributeError: 'NoneType' object has no attribute 'replace'
# while settings are loaded and Django uses LANGUAGE_CODE as default. What overrides this?
# a possible undesired solution, http://source.mihelac.org/2009/11/12/django-set-language-for-admin/

255 256
# http://stackoverflow.com/questions/1832709/django-how-to-make-translation-work
# shoddy documentation on dashes and underscores... different than the "ll" suggestion.
257
# LANGUAGE_CODE = 'en-gb'
258
# Less text is better :) See: https://www.youtube.com/watch?v=0j74jcxSunY
259
LANGUAGES = (
260
    ('en', 'English'),
261
    ('nl', 'Dutch'),
262 263
    # This helps to find missing translations. Using this language the goal is to replace all text
    # strings with a single rainbow. In the end there should be no text, only rainbows! 🌈
264
    ('rainbowsandunicorns', '🌈'),
265 266
)

267
# There is no 🌈🦄 translation for humanize. Instead you'll get the english fallback values.
268 269
# language is now depending on the user agent.
# LANGUAGE_CODE = 'en'
270

271 272
LANGUAGE_COOKIE_NAME = 'language'

273 274 275 276
TIME_ZONE = 'UTC'

USE_I18N = True
USE_L10N = True
277

278 279 280 281
# Loaddata will show a massive amount of warnings, therefore use load-dataset. load-dataset will
# do exactly the same as loaddata, but will overwrite below flag preventing warnings.
USE_TZ = True

282 283 284
# https://docs.djangoproject.com/en/1.11/topics/i18n/translation/#how-django-discovers-translations
# In all cases the name of the directory containing the translation is expected to be named using
# locale name notation. E.g. de, pt_BR, es_AR, etc.
285
LOCALE_PATHS = ['locale']
286

287

288 289 290 291
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'
Johan Bloemberg's avatar
Johan Bloemberg committed
292

Johan Bloemberg's avatar
Johan Bloemberg committed
293
# Absolute path to aggregate to and serve static file from.
294 295 296
if DEBUG:
    STATIC_ROOT = 'static'
else:
297
    STATIC_ROOT = '/srv/failmap/static/'
Johan Bloemberg's avatar
Johan Bloemberg committed
298

299
TEST_RUNNER = 'failmap.testrunner.PytestTestRunner'
300

301
# From the Jet documentation, a different color for a different season.
302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333
JET_THEMES = [
    {
        'theme': 'default',  # theme folder name
        'color': '#47bac1',  # color of the theme's button in user menu
        'title': 'Default'  # theme title
    },
    {
        'theme': 'green',
        'color': '#44b78b',
        'title': 'Green'
    },
    {
        'theme': 'light-green',
        'color': '#2faa60',
        'title': 'Light Green'
    },
    {
        'theme': 'light-violet',
        'color': '#a464c4',
        'title': 'Light Violet'
    },
    {
        'theme': 'light-blue',
        'color': '#5EADDE',
        'title': 'Light Blue'
    },
    {
        'theme': 'light-gray',
        'color': '#222',
        'title': 'Light Gray'
    }
]
334

335 336
# see: https://github.com/geex-arts/django-jet/blob/
#   fea07040229d1b56800a7b8e6234e5f9419e2114/docs/config_file.rst
337
# required for custom modules
338 339 340 341 342 343
JET_APP_INDEX_DASHBOARD = 'failmap.app.dashboard.CustomAppIndexDashboard'

# Customize the Dashboard index page remove unneeded panels (eg: feeds) and add usefull stuff (actions).
JET_INDEX_DASHBOARD = 'failmap.app.dashboard.CustomIndexDashboard'

# expand menu items by default for quick access
344
JET_SIDE_MENU_COMPACT = False
345 346 347 348 349 350 351 352 353 354 355 356

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',  # sys.stdout
            'formatter': 'color',
        },
    },
    'formatters': {
        'debug': {
357 358
            'format': '%(asctime)s\t%(levelname)-8s - %(filename)-20s:%(lineno)-4s - '
                      '%(funcName)20s() - %(message)s',
359 360 361
        },
        'color': {
            '()': 'colorlog.ColoredFormatter',
362
            'format': '%(log_color)s%(asctime)s\t%(levelname)-8s - '
363
                      '%(message)s',
Johan Bloemberg's avatar
Johan Bloemberg committed
364
            'datefmt': '%Y-%m-%d %H:%M',
365
            'log_colors': {
366 367 368 369
                'DEBUG': 'green',
                'INFO': 'white',
                'WARNING': 'yellow',
                'ERROR': 'red',
370 371 372 373 374
                'CRITICAL': 'bold_red',
            },
        }
    },
    'loggers': {
Elger Jonker's avatar
Elger Jonker committed
375 376 377 378 379 380 381 382 383 384
        # Used when there is no log defined or loaded. Disabled given we always use __package__ to log.
        # Would you enable it, all logging messages will be logged twice.
        # '': {
        #     'handlers': ['console'],
        #     'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
        # },

        # Default Django logging, we expect django to work, and therefore only show INFO messages.
        # It can be smart to sometimes want to see what's going on here, but not all the time.
        # https://docs.djangoproject.com/en/2.1/topics/logging/#django-s-logging-extensions
385 386 387 388
        'django': {
            'handlers': ['console'],
            'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'),
        },
Elger Jonker's avatar
Elger Jonker committed
389 390 391 392 393 394

        # We expect to be able to debug failmap all of the time.
        'failmap': {
            'handlers': ['console'],
            'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
        },
395 396 397
    },
}

398 399 400 401
# Add a slash at the end so we know it's a directory. Tries to somewhat prevents doing things in root.
OUTPUT_DIR = os.environ.get('OUTPUT_DIR', os.path.abspath(os.path.dirname(__file__)) + '/')
VENDOR_DIR = os.environ.get('VENDOR_DIR', os.path.abspath(os.path.dirname(__file__) + '/../vendor/') + '/')

402 403 404 405
# the tools dir in this case are very small tools that build upon external dependencies, such as dnscheck.
# only use this if the vendor dir does not provide the needed command(s) in a simple way
TOOLS_DIR = os.environ.get('TOOLS_DIR', os.path.abspath(os.path.dirname(__file__) + '/../tools/') + '/')

406 407 408
# A number of tools and outputs are grouped to easier have access to all of them.
# Our vendor directory contains a number of small tools that are hard to install otherwise.

409
TOOLS = {
410
    # Chrome and firefox are special cases: they install very easily and therefore don't need further grouping.
411 412
    'chrome': {
        'executable': {
413 414 415 416 417
            # os.platform is used to see what binaries should be used on a worker.
            'Darwin': os.environ.get(
                'CHROME_EXECUTABLE_DARWIN', "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"),
            'Linux': os.environ.get(
                'CHROME_EXECUTABLE_LINUX', ""),
418
        },
419 420
        'screenshot_output_dir': OUTPUT_DIR + os.environ.get(
            'CHROME_SCREENSHOT_OUTPUT_DIR', 'map/static/images/screenshots/'),
421
    },
422
    # Chrome and firefox are special cases: they install very easily and therefore don't need further grouping.
423 424
    'firefox': {
        'executable': {
425 426 427 428 429
            # os.platform is used to see what binaries should be used on a worker.
            'Darwin': os.environ.get(
                'FIREFOX_EXECUTABLE_DARWIN', "/Applications/Firefox.app/Contents/MacOS/firefox"),
            'Linux': os.environ.get(
                'FIREFOX_EXECUTABLE_LINUX', ""),
430
        },
431 432
        'screenshot_output_dir': OUTPUT_DIR + os.environ.get(
            'FIREFOX_SCREENSHOT_OUTPUT_DIR', 'map/static/images/screenshots/'),
433
    },
434
    'theHarvester': {
435 436
        'executable': VENDOR_DIR + os.environ.get('THEHARVESTER_EXECUTABLE', "theHarvester/theHarvester.py"),
        'output_dir': OUTPUT_DIR + os.environ.get('THEHARVESTER_OUTPUT_DIR', "scanners/resources/output/theHarvester/"),
437 438
    },
    'dnsrecon': {
439 440 441 442 443
        'executable': VENDOR_DIR + os.environ.get('DNSRECON_EXECUTABLE', "dnsrecon/dnsrecon.py"),
        'output_dir': OUTPUT_DIR + os.environ.get('DNSRECON_OUTPUT_DIR', "scanners/resources/output/dnsrecon/"),

        # The most important wordlists are auto-generated by this software, and are thus output.
        'wordlist_dir': OUTPUT_DIR + os.environ.get('DNSRECON_WORDLIST_DIR', "scanners/resources/wordlists/"),
444
    },
445
    'openstreetmap': {
Elger Jonker's avatar
Elger Jonker committed
446 447
        'output_dir': OUTPUT_DIR + os.environ.get('OPENSTREETMAP_OUTPUT_DIR',
                                                  "scanners/resources/output/openstreetmap/"),
448
    },
449
    'sslscan': {
450 451
        # this is beta functionality and not supported in production
        # these are installed system wide and don't require a path (they might when development continues)
452 453 454 455
        'executable': {
            'Darwin': 'sslscan',
            'Linux': 'sslscan',
        },
456
        'report_output_dir': OUTPUT_DIR + "scanners/resources/output/sslscan/",
457 458
    },
    'openssl': {
459 460
        # this is beta functionality and not supported in production
        # these are installed system wide and don't require a path  (they might when development continues)
461 462 463 464 465 466
        'executable': {
            'Darwin': 'openssl',
            'Linux': 'openssl',
        },
    },
    'TLS': {
467
        # this is beta functionality and not supported in production
468 469 470 471 472
        'cve_2016_2107': VENDOR_DIR + 'CVE-2016-2107-padding-oracle/main.go',
        'cve_2016_9244': VENDOR_DIR + 'CVE-2016-9244-ticketbleed/ticketbleed.go',
        'cert_chain_resolver': {
            'Darwin': VENDOR_DIR + 'cert-chain-resolver/cert-chain-resolver-darwin',
            'Linux': VENDOR_DIR + 'cert-chain-resolver/cert-chain-resolver-linux',
473 474 475
        },
        'tls_check_output_dir': OUTPUT_DIR + os.environ.get('TLSCHECK_OUTPUT_DIR',
                                                            'scanners/resources/output/tls_check/'),
476 477 478
    },
    'dnscheck': {
        'executable': TOOLS_DIR + 'dnssec.pl'
479 480 481
    },
    'osaft': {
        'json': VENDOR_DIR + 'osaft/JSON-array.awk',
482
    }
483
}
484

485
# Compression
486 487 488
# Django-compressor is used to compress css and js files in production
# During development this is disabled as it does not provide any feature there
# Django-compressor configuration defaults take care of this.
489
# https://django-compressor.readthedocs.io/en/latest/usage/
490
# which plugins to use to find static files
491
STATICFILES_FINDERS = (
492
    # default static files finders
493 494 495 496 497
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    # other finders..
    'compressor.finders.CompressorFinder',
)
498

499 500 501 502 503 504 505 506 507 508 509 510
COMPRESS_CSS_FILTERS = ['compressor.filters.cssmin.CSSCompressorFilter']

# Slimit doesn't work with vue. Tried two versions. Had to rewrite some other stuff.
# Now using the default, so not explicitly adding that to the settings
# COMPRESS_JS_FILTERS = ['compressor.filters.jsmin.JSMinFilter']

# Brotli compress storage gives some issues.
# This creates the original compressed and a gzipped compressed file.
COMPRESS_STORAGE = (
    'compressor.storage.GzipCompressorFileStorage'
)

511 512
# Disable caching during development and production.
# Django only emits caching headers, the webserver/caching-proxy makes sure the rest of the caching is handled.
Elger Jonker's avatar
Elger Jonker committed
513 514


515 516 517
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
518
    }
519
}
520

521 522 523 524 525
# Enable static file (js/css) compression when not running debug
# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_OFFLINE
COMPRESS_OFFLINE = not DEBUG
# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_ENABLED
# Enabled when debug is off by default.
526

527 528 529 530 531 532 533 534 535
# Celery 4.0 settings
# Pickle can work, but you need to use certificates to communicate (to verify the right origin)
# It's preferable not to use pickle, yet it's overly convenient as the normal serializer can not
# even serialize dicts.
# http://docs.celeryproject.org/en/latest/userguide/configuration.html
CELERY_accept_content = ['pickle', 'yaml']
CELERY_task_serializer = 'pickle'
CELERY_result_serializer = 'pickle'

536

537
# Celery config
538
CELERY_BROKER_URL = os.environ.get('BROKER', 'redis://localhost:6379/0')
539 540 541 542
ENABLE_UTC = True

# Any data transfered with pickle needs to be over tls... you can inject arbitrary objects with
# this stuff... message signing makes it a bit better, not perfect as it peels the onion.
543
# this stuff... message signing makes it a bit better, not perfect as it peels the onion.
544 545 546 547 548 549
# see: https://blog.nelhage.com/2011/03/exploiting-pickle/
# Yet pickle is the only convenient way of transporting objects without having to lean in all kinds
# of directions to get the job done. Intermediate tables to store results could be an option.
CELERY_ACCEPT_CONTENT = ['pickle']
CELERY_TASK_SERIALIZER = 'pickle'
CELERY_RESULT_SERIALIZER = 'pickle'
550 551
CELERY_TIMEZONE = 'UTC'

Johan Bloemberg's avatar
Johan Bloemberg committed
552
CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler'
Johan Bloemberg's avatar
Johan Bloemberg committed
553

554 555
CELERY_BROKER_CONNECTION_MAX_RETRIES = 1
CELERY_BROKER_CONNECTION_RETRY = False
556
CELERY_RESULT_EXPIRES = timedelta(hours=4)
557

Johan Bloemberg's avatar
Johan Bloemberg committed
558 559 560 561 562 563 564 565 566 567 568 569
# Use the value of 2 for celery prefetch multiplier. Previous was 1. The
# assumption is that 1 will block a worker thread until the current (rate
# limited) task is completed. When using 2 (or higher) the assumption is that
# celery will drop further rate limited task from the internal worker queue and
# fetch other tasks tasks that could be executed (spooling other rate limited
# tasks through in the process but to no hard except for a slight drop in
# overall throughput/performance). A to high value for the prefetch multiplier
# might result in high priority tasks not being picked up as Celery does not
# seem to do prioritisation in worker queues but only on the broker
# queues. The value of 2 is currently selected because it higher than 1,
# behaviour needs to be observed to decide if raising this results in
# further improvements without impacting the priority feature.
570
CELERY_WORKER_PREFETCH_MULTIPLIER = 2
571 572 573 574

# numer of tasks to be executed in parallel by celery
CELERY_WORKER_CONCURRENCY = 10

575 576 577 578 579 580 581 582 583 584
# Workers will scale up and scale down depending on the number of tasks
# available. To prevent workers from scaling down while still doing work,
# the ACKS_LATE setting is used. This insures that a task is removed from
# the task queue after the task is performed. This might result in some
# issues where tasks that don't finish or crash keep being executed:
# thus for tasks that are not programmed perfectly it will raise a number
# of repeated exceptions which will need to be debugged.
CELERY_ACKS_LATE = True


585 586 587 588 589 590
# Settings for statsd metrics collection. Statsd defaults over UDP port 8125.
# https://django-statsd.readthedocs.io/en/latest/#celery-signals-integration
STATSD_HOST = os.environ.get('STATSD_HOST', '127.0.0.1')
STATSD_PREFIX = 'failmap'
# register hooks for selery tasks
STATSD_CELERY_SIGNALS = True
591 592 593
# send database query metric (in production, in development we have debug toolbar for this)
if not DEBUG:
    STATSD_PATCHES = ['django_statsd.patches.db', ]
594 595 596

# enable some features during debug
if DEBUG:
597 598
    # enable debug toolbar if available
    try:
Johan Bloemberg's avatar
Johan Bloemberg committed
599
        import debug_toolbar
600

601
        INSTALLED_APPS.append('debug_toolbar')
Elger Jonker's avatar
Elger Jonker committed
602
        MIDDLEWARE.append('debug_toolbar.middleware.DebugToolbarMiddleware')
603 604

        import debug_toolbar.settings
605

606
        DEBUG_TOOLBAR_PANELS = [
Elger Jonker's avatar
lint  
Elger Jonker committed
607 608 609 610
            'ddt_request_history.panels.request_history.RequestHistoryPanel',
        ] + debug_toolbar.settings.PANELS_DEFAULTS + [
            'django_statsd.panel.StatsdPanel',
        ]
611
        # send statsd metrics to debug_toolbar
612
        STATSD_CLIENT = 'django_statsd.clients.toolbar'
Johan Bloemberg's avatar
Johan Bloemberg committed
613
    except ImportError:
614
        pass
615

Johan Bloemberg's avatar
Johan Bloemberg committed
616
# if sentry DSN is provided register raven to emit events on exceptions
617 618
SENTRY_DSN = os.environ.get('SENTRY_DSN')
if SENTRY_DSN:
619 620
    INSTALLED_APPS += ('raven.contrib.django.raven_compat',)
    RAVEN_CONFIG = {
621
        'dsn': SENTRY_DSN,
622 623
        'release': __version__,
    }
624 625
    # add sentry ID to request for inclusion in templates
    # https://docs.sentry.io/clients/python/integrations/django/#message-references
Elger Jonker's avatar
Elger Jonker committed
626
    MIDDLEWARE.insert(0, 'raven.contrib.django.raven_compat.middleware.SentryResponseErrorIdMiddleware')
Johan Bloemberg's avatar
Johan Bloemberg committed
627

Johan Bloemberg's avatar
Johan Bloemberg committed
628 629 630 631 632
    # Celery specific handlers
    client = raven.Client(SENTRY_DSN)
    raven.contrib.celery.register_logger_signal(client)
    raven.contrib.celery.register_signal(client)

Johan Bloemberg's avatar
Johan Bloemberg committed
633
# set javascript sentry token if provided
634
SENTRY_TOKEN = os.environ.get('SENTRY_TOKEN', '')
635 636 637 638

SENTRY_ORGANIZATION = 'internet-cleanup-foundation'
SENTRY_PROJECT = 'faalkaart'
SENTRY_PROJECT_URL = 'https://sentry.io/%s/%s' % (SENTRY_ORGANIZATION, SENTRY_PROJECT)
639 640 641 642 643 644 645

# Some workers or (development) environments don't support both IP networks
# Note that not supporting either protocols can result in all endpoints being killed as they are unreachable by scanners
# We don't check these settings anywhere for sanity as some workers might not need a network at all.
# The defaults stem from our live environment, where we've set IPv4 being present on all containers and workers.
NETWORK_SUPPORTS_IPV4 = os.environ.get('NETWORK_SUPPORTS_IPV4', True)
NETWORK_SUPPORTS_IPV6 = os.environ.get('NETWORK_SUPPORTS_IPV6', False)
646 647 648

# atomic imports: fail completely, not half
IMPORT_EXPORT_USE_TRANSACTIONS = True
649

650 651
#########
# Begin constance settigns
652
# runtime configuration from database
653 654 655
# https://django-constance.readthedocs.io/en/latest/
CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend'

Elger Jonker's avatar
Elger Jonker committed
656 657 658 659 660 661 662 663

CONSTANCE_ADDITIONAL_FIELDS = {
    'yes_no_null_select': ['django.forms.fields.ChoiceField', {
        'widget': 'django.forms.Select',
        'choices': ((None, "-----"), ("1", "Yes"), ("0", "No"))
    }],
}

Elger Jonker's avatar
Elger Jonker committed
664

665
CONSTANCE_CONFIG = {
Elger Jonker's avatar
Elger Jonker committed
666
    'COMMENTS': ('', 'Some comments by you that say a little bit about why below settings are the way they are.', str),
667

668 669 670 671
    'SHOW_ANNOUNCEMENT': (False, 'Shows the announcement bar. Announcements do not show when empty anyway.', bool),
    'ANNOUNCEMENT': ('', 'Text / HTML message that should appear on the top of the site / app. For example with '
                         'important news / features or current issues. You can enter any HTML, JS, CSS.', str),

672
    'PROJECT_NAME': ('', 'The name of this mapping project, used for branding and promotion.', str),
673
    'PROJECT_COUNTRY': ('NL', 'Two letter ISO code of the country that should be shown.', str),
Elger Jonker's avatar
Elger Jonker committed
674
    'PROJECT_TAGLINE': ('', 'Tagline for this project.', str),
675
    'PROJECT_WEBSITE': ('', 'The url where this site is located. Without trailing slash. Eg: https://example.com', str),
676 677 678 679 680 681 682
    'PROJECT_MAIL': ('', 'The address where people can contact for more info about this project.', str),
    'PROJECT_ISSUE_MAIL': ('', 'The address where people can mail when they encounter issues, for example when they '
                               'are using the incorrect findings button.', str),
    'PROJECT_TWITTER': ('', 'The twitter address where people can follow this project. Include the @!', str),
    'PROJECT_FACEBOOK': ('', 'The facebook address where people can follow this project. Has to be a complete url.',
                         str),

683 684 685 686

    'GOOGLE_MAPS_API_KEY': ('AIzaSyBXJbEUxGW1dAB4hJOlmKdYelfoRY6_fjo',
                            'API Key of google maps that can be used in the game / adding section.', str),

687 688 689
    'MAPBOX_ACCESS_TOKEN': ('pk.eyJ1IjoibXJmYWlsIiwiYSI6ImNqMHRlNXloczAwMWQyd3FxY3JkMnUxb3EifQ.9nJBaedxrry91O1d90wfuw',
                            'Access token for Mapbox', str),

690 691
    'WAMBACHERS_OSM_CLIKEY': ('', 'CLI key for Wambachers OSM export feature', str),

692 693 694 695 696 697 698 699 700
    'RESPONSIBLE_ORGANIZATION_NAME': ('', 'The name of the organization running this project.', str),
    'RESPONSIBLE_ORGANIZATION_PROMO_TEXT': ('', 'Some text promoting this organization and it\s mission. This text will'
                                                ' not be translated.', str),
    'RESPONSIBLE_ORGANIZATION_WEBSITE': ('', 'The name of the organization running this project.', str),
    'RESPONSIBLE_ORGANIZATION_MAIL': ('', 'The name of the organization running this project.', str),
    'RESPONSIBLE_ORGANIZATION_TWITTER': ('', 'The twitter address where people can follow this project. Include '
                                             'the @!', str),
    'RESPONSIBLE_ORGANIZATION_FACEBOOK': ('', 'The facebook address where people can follow this project. Make sure'
                                              ' this is a complete url.', str),
Elger Jonker's avatar
Elger Jonker committed
701 702 703
    'RESPONSIBLE_ORGANIZATION_LINKEDIN': ('', 'Linkedin page url.', str),
    'RESPONSIBLE_ORGANIZATION_WHATSAPP': ('', 'Whatsapp number.', str),
    'RESPONSIBLE_ORGANIZATION_PHONE': ('', 'Phone number, displayed as a sip:// addres.', str),
704

Elger Jonker's avatar
Elger Jonker committed
705
    'SHOW_INTRO': (True, 'Shows the standard introduction.', bool),
706
    'SHOW_GOOD_BAD': (True, 'Shows the good / bad lists.', bool),
Elger Jonker's avatar
Elger Jonker committed
707 708 709 710 711 712 713 714 715 716 717
    'SHOW_EXTENSIVE_STATISTICS': (True, 'Shows extended statistics.', bool),
    'SHOW_STATS_GRAPHS': (True, 'Shows graphs in extended statistics. '
                                'Extended statistics needs to be enabled for this to have effect.', bool),
    'SHOW_STATS_IMPROVEMENTS': (True, 'Shows improvements in extended statistics. '
                                      'Extended statistics needs to be enabled for this to have effect.', bool),
    'SHOW_STATS_NUMBERS': (True, 'Shows numbers in extended statistics. '
                                 'Extended statistics needs to be enabled for this to have effect.', bool),
    'SHOW_STATS_CHANGES': (True, 'Shows changes in extended statistics. '
                                 'Extended statistics needs to be enabled for this to have effect.', bool),
    'SHOW_DATASETS': (True, 'Shows dataset downloads. Note: dataset downloads are always available, even if they '
                            'are not shown on the website.', bool),
Elger Jonker's avatar
Elger Jonker committed
718
    'SHOW_COMPLY_OR_EXPLAIN': (False, 'Shows comply or explain sections and the explain link in findings.', bool),
719 720
    'SHOW_COMPLY_OR_EXPLAIN_DISCUSS': (False, 'Shows a link to the comply or explain discussion forum. The url of this'
                                              'forum can be edited below.', bool),
Elger Jonker's avatar
Elger Jonker committed
721
    'SHOW_TICKER': (False, 'Shows stock-ticker with updates in the past month.', bool),
722
    'TICKER_SLOGAN': ('failmap.org - monitor everything', 'Text to show between every 10 changes.', str),
723
    'TICKER_VISIBLE_VIA_JS_COMMAND': (False, 'Only show the ticker using the Javascript command "show_ticker()"', bool),
Elger Jonker's avatar
Elger Jonker committed
724
    'SHOW_SCAN_SCHEDULE': (False, 'Shows list of upcoming scans, so everyone knows what scan is due next.', bool),
Elger Jonker's avatar
Elger Jonker committed
725

Elger Jonker's avatar
Elger Jonker committed
726
    'SHOW_SERVICES': (True, 'Show table with how many services are scanned. Requires SHOW_STATS_NUMBERS.', bool),
Elger Jonker's avatar
Elger Jonker committed
727 728 729 730 731 732 733
    'SHOW_DNS_DNSSEC': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
    'SHOW_HTTP_TLS_QUALYS': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
    'SHOW_HTTP_MISSING_TLS': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
    'SHOW_HTTP_HEADERS_HSTS': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
    'SHOW_HTTP_HEADERS_XFO': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
    'SHOW_HTTP_HEADERS_X_XSS': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
    'SHOW_HTTP_HEADERS_X_CONTENT': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
Elger Jonker's avatar
Elger Jonker committed
734
    'SHOW_FTP': (True, 'Show graphs/stats of this? May cause empty spots on the site.', bool),
Elger Jonker's avatar
Elger Jonker committed
735
    'SHOW_DONATION': (True, 'Show donation buttons and links on the site.', bool),
Elger Jonker's avatar
Elger Jonker committed
736

737 738 739
    'COMPLY_OR_EXPLAIN_DISCUSSION_FORUM_LINK': ('', 'Link to the comply or explain discussion forum.', str),
    'COMPLY_OR_EXPLAIN_EMAIL_ADDRESS': ('', 'E-mail where to receive explanations.', str),

Elger Jonker's avatar
Elger Jonker committed
740 741 742 743 744 745 746 747 748 749 750
    # todo: schedule this once per week by default.
    'DISCOVER_URLS_USING_NSEC': (True, 'Discover new domains using DNSSEC NSEC1 hashes? (See docs)', bool),
    'DISCOVER_URLS_USING_KNOWN_SUBDOMAINS': (True, 'Discover new domains using known subdomains? (See docs)', bool),
    'DISCOVER_URLS_USING_CERTIFICATE_TRANSPARENCY': (True, 'Discover new domains using crt.sh '
                                                           'certificate transperancy database? (See docs)', bool),
    'DISCOVER_HTTP_ENDPOINTS': (True, 'Do you want to discover HTTP endpoints?', bool),

    'SCAN_AT_ALL': (True, 'Do you want to scan at all?', bool),
    'SCAN_DNS_DNSSEC': (True, 'Do you want to scan for DNSSEC issues?', bool),
    'SCAN_HTTP_TLS_QUALYS': (True, 'Do you want to scan for TLS issues through Qualys? This is about 1 scan per '
                                   'two minutes. Cloud scanning can improve this a bit.', bool),
751 752
    'SCAN_HTTP_TLS_OSAFT': (True, 'Experimental: Do you want to scan for TLS issues through OSaft? This is about '
                                  '3 scans per minute. Cloud scanning can improve this a bit.', bool),
Elger Jonker's avatar
Elger Jonker committed
753 754 755 756 757 758
    'SCAN_HTTP_MISSING_TLS': (True, 'Do you want to scan for endpoints that don\'t have a secure counterpart?', bool),
    'SCAN_HTTP_HEADERS_HSTS': (True, 'Do you want to scan for missing '
                                     'Hypertext Strict Transport Security headers?', bool),
    'SCAN_HTTP_HEADERS_XFO': (True, 'Do you want to scan for missing X-Frame-Options headers?', bool),
    'SCAN_HTTP_HEADERS_X_XSS': (True, 'Do you want to scan for missing X-XSS headers?', bool),
    'SCAN_HTTP_HEADERS_X_CONTENT': (True, 'Do you want to scan for missing X-Content-Type issues?', bool),
Elger Jonker's avatar
Elger Jonker committed
759
    'SCAN_FTP': (True, 'Do you want to scan for FTP servers that are missing encryption?', bool),
Elger Jonker's avatar
Elger Jonker committed
760 761 762 763
    'CREATE_HTTP_SCREENSHOT': (True, 'Todo: Does not work yet! Do you want to create screenshots for HTTP endpoints?',
                               bool),

    # future: FTP, TLS_QUICK (way less robust and complete, much faster)
Elger Jonker's avatar
Elger Jonker committed
764
    'REPORT_INCLUDE_FTP': (True, 'Do you want to add FTP encryption issues to the report?', bool),
Elger Jonker's avatar
Elger Jonker committed
765 766 767 768 769 770 771 772
    'REPORT_INCLUDE_DNS_DNSSEC': (True, 'Do you want to add DNSSEC issues to the report?', bool),
    'REPORT_INCLUDE_HTTP_TLS_QUALYS': (True, 'Do you want to show TLS results in the report?', bool),
    'REPORT_INCLUDE_HTTP_MISSING_TLS': (True, 'Do you want to show missing TLS in the report?', bool),
    'REPORT_INCLUDE_HTTP_HEADERS_HSTS': (True, 'Do you want to HSTS in the report?', bool),
    'REPORT_INCLUDE_HTTP_HEADERS_XFO': (True, 'Do you want to show XFO in the report?', bool),
    'REPORT_INCLUDE_HTTP_HEADERS_X_XSS': (True, 'Do you want to show X-XSS protection headers in the report?', bool),
    'REPORT_INCLUDE_HTTP_HEADERS_X_CONTENT': (True, 'Do you want to show X-Content-Type headers in the report?', bool),

Elger Jonker's avatar
Elger Jonker committed
773 774 775 776 777 778 779
    # scanning pre-requisites
    'CONNECTIVITY_TEST_DOMAIN': ('faalkaart.nl', 'A server that is reachable over IPv4. This is used by a worker '
                                                 'to determine what kind of scans it can do. Enter an '
                                                 'address that you own or manage.', str),
    'IPV6_TEST_DOMAIN': ('faalkaart.nl', 'A server that is reachable over IPv6. This is used by a worker to determine '
                                         'what kind of scans it can do. Enter an address that you own or manage.', str),

780 781
    'GITTER_CHAT_ENABLE': (False, 'Show the Gitter Chat option on the website (using gitter sidecar).', bool),
    'GITTER_CHAT_CHANNEL': ('internet-cleanup-foundation/support', 'Name of the channel chat takes place.', str),
Elger Jonker's avatar
Elger Jonker committed
782

Elger Jonker's avatar
Elger Jonker committed
783 784 785 786
    #
    # 'USE_CUSTOM_INTRO': (False, 'If you want to use an (untranslated) custom intro, enable this. Enabling this'
    #                             'will remove the standard intro and has several fields that can be filled with '
    #                             'custom HTML.', bool),
787
}
788

Elger Jonker's avatar
Elger Jonker committed
789
CONSTANCE_CONFIG_FIELDSETS = OrderedDict([
790
    ('General', ('COMMENTS', 'SHOW_ANNOUNCEMENT', 'ANNOUNCEMENT', 'MAPBOX_ACCESS_TOKEN', 'WAMBACHERS_OSM_CLIKEY')),
791

792 793
    ('Project', ('PROJECT_NAME', 'PROJECT_COUNTRY', 'PROJECT_TAGLINE', 'PROJECT_WEBSITE', 'PROJECT_MAIL',
                 'PROJECT_ISSUE_MAIL', 'PROJECT_TWITTER', 'PROJECT_FACEBOOK')),
794

Elger Jonker's avatar
Elger Jonker committed
795 796 797 798 799
    ('Responsible', ('RESPONSIBLE_ORGANIZATION_NAME', 'RESPONSIBLE_ORGANIZATION_PROMO_TEXT',
                     'RESPONSIBLE_ORGANIZATION_WEBSITE', 'RESPONSIBLE_ORGANIZATION_MAIL',
                     'RESPONSIBLE_ORGANIZATION_TWITTER', 'RESPONSIBLE_ORGANIZATION_FACEBOOK',
                     'RESPONSIBLE_ORGANIZATION_LINKEDIN', 'RESPONSIBLE_ORGANIZATION_WHATSAPP',
                     'RESPONSIBLE_ORGANIZATION_PHONE')),
800

Elger Jonker's avatar
Elger Jonker committed
801
    ('Website', ('SHOW_INTRO', 'SHOW_GOOD_BAD', 'SHOW_EXTENSIVE_STATISTICS', 'SHOW_DATASETS', 'SHOW_STATS_GRAPHS',
Elger Jonker's avatar
Elger Jonker committed
802
                 'SHOW_STATS_IMPROVEMENTS', 'SHOW_STATS_NUMBERS', 'SHOW_SERVICES', 'SHOW_STATS_CHANGES',
803
                 'SHOW_SCAN_SCHEDULE', 'SHOW_DONATION'
Elger Jonker's avatar
Elger Jonker committed
804
                 )),
Elger Jonker's avatar
Elger Jonker committed
805

Elger Jonker's avatar
Elger Jonker committed
806 807
    ('Discovery', ('DISCOVER_URLS_USING_NSEC', 'DISCOVER_URLS_USING_KNOWN_SUBDOMAINS',
                   'DISCOVER_URLS_USING_CERTIFICATE_TRANSPARENCY', 'DISCOVER_HTTP_ENDPOINTS')),
Elger Jonker's avatar
Elger Jonker committed
808

809 810
    ('Scanning', ('SCAN_AT_ALL', )),

Elger Jonker's avatar
Elger Jonker committed
811
    ('Scanner Capabilities', ('CONNECTIVITY_TEST_DOMAIN', 'IPV6_TEST_DOMAIN')),
Elger Jonker's avatar
Elger Jonker committed
812

813 814 815 816 817
    ('Scanning (beta)', ('SCAN_HTTP_TLS_OSAFT', 'CREATE_HTTP_SCREENSHOT')),

    ('TLS Scans (Qualys)', ('SCAN_HTTP_TLS_QUALYS', 'REPORT_INCLUDE_HTTP_TLS_QUALYS', 'SHOW_HTTP_TLS_QUALYS', )),

    ('Missing TLS Scans', ('SCAN_HTTP_MISSING_TLS', 'REPORT_INCLUDE_HTTP_MISSING_TLS', 'SHOW_HTTP_MISSING_TLS', )),
Elger Jonker's avatar
Elger Jonker committed
818

819 820 821 822 823 824 825 826 827 828 829 830 831
    ('DNSSEC Scans', ('SCAN_DNS_DNSSEC', 'REPORT_INCLUDE_DNS_DNSSEC', 'SHOW_DNS_DNSSEC')),

    ('FTP Scans', ('SCAN_FTP', 'REPORT_INCLUDE_FTP', 'SHOW_FTP', )),

    ('Security Headers Scans', ('SCAN_HTTP_HEADERS_HSTS', 'SCAN_HTTP_HEADERS_XFO', 'SCAN_HTTP_HEADERS_X_XSS',
                                'SCAN_HTTP_HEADERS_X_CONTENT', 'REPORT_INCLUDE_HTTP_HEADERS_HSTS',
                                'REPORT_INCLUDE_HTTP_HEADERS_XFO', 'REPORT_INCLUDE_HTTP_HEADERS_X_XSS',
                                'REPORT_INCLUDE_HTTP_HEADERS_X_CONTENT',  'SHOW_HTTP_HEADERS_HSTS',
                                'SHOW_HTTP_HEADERS_XFO',
                                'SHOW_HTTP_HEADERS_X_XSS', 'SHOW_HTTP_HEADERS_X_CONTENT', )),

    ('Comply or Explain', ('SHOW_COMPLY_OR_EXPLAIN', 'SHOW_COMPLY_OR_EXPLAIN_DISCUSS',
                           'COMPLY_OR_EXPLAIN_DISCUSSION_FORUM_LINK', 'COMPLY_OR_EXPLAIN_EMAIL_ADDRESS')),
832 833 834

    ('Game', ('GOOGLE_MAPS_API_KEY',)),

835
    ('Ticker', ('SHOW_TICKER', 'TICKER_SLOGAN', 'TICKER_VISIBLE_VIA_JS_COMMAND')),
836

837
    ('Chat (using gitter)', ('GITTER_CHAT_ENABLE', 'GITTER_CHAT_CHANNEL'))
Elger Jonker's avatar
Elger Jonker committed
838
])
Elger Jonker's avatar
Elger Jonker committed
839 840 841 842 843 844 845 846 847

# Check for constance configuration issues:
# See also: https://github.com/jazzband/django-constance/issues/293
variables_in_fieldsets = [i for sub in [CONSTANCE_CONFIG_FIELDSETS[x] for x in CONSTANCE_CONFIG_FIELDSETS] for i in sub]
variables_in_config = [x for x in CONSTANCE_CONFIG]
missing = set(variables_in_config) - set(variables_in_fieldsets)
if missing:
    raise EnvironmentError("Constance config variables %s are missing in constance config fieldsets." % missing)

848 849
# End constance settings
########
850

851
# https://docs.djangoproject.com/en/1.11/ref/settings/#data-upload-max-number-fields
852 853 854
# The default is far too low with various inlines (even on the test dataset).
# Yes, we happily exceed 1000 fields anyday. No problem :)
DATA_UPLOAD_MAX_NUMBER_FIELDS = 4242
Johan Bloemberg's avatar
Johan Bloemberg committed
855

856 857
########
# Begin game settings:
Johan Bloemberg's avatar
Johan Bloemberg committed
858 859
# login on the frontpage should redirect to the game landingpage
LOGIN_REDIRECT_URL = '/game/'
860 861
LOGIN_URL = '/authentication/login/'
LOGOUT_REDIRECT_URL = '/'
862

Elger Jonker's avatar
Elger Jonker committed
863
CRISPY_TEMPLATE_PACK = 'bootstrap3'
864

865 866 867 868 869
# End game settigns
#######

#######
# Begin helpdesk settings:
870
# Helpdesk removed.
871 872
# Unfortunately we cannot manage these settings with constance. And therefore you should be still editing
# this file. We're sorry.
873
# SITE_ID = 1  # For django-helpdesk. We only run one site...
874 875

# helps against spam, will leak all your data, so be careful. Spam could be protected against on your mailserver...
876 877
# AKISMET_API_KEY = os.environ.get('AKISMET_API_KEY', '')
# TYPEPAD_ANTISPAM_API_KEY = os.environ.get('TYPEPAD_ANTISPAM_API_KEY', '')
878 879 880 881 882 883 884

# If you use another server for sending mail, see config example here:
# http://django-helpdesk.readthedocs.io/en/0.2.x/configuration.html
# EMAIL_HOST = 'XXXXX'
# EMAIL_HOST_USER = '[email protected]'
# EMAIL_HOST_PASSWORD = '123456'
# Make sure the MEDIA_ROOT is NOT readable from the webserver directly. So no https://bla/media.
885
# MEDIA_ROOT = os.environ.get('MEDIA_ROOT', os.path.abspath(os.path.dirname(__file__)) + '/../')
886 887

# End of helpdesk settings
888 889 890 891
#######


#######
Elger Jonker's avatar
Elger Jonker committed
892
# begin django jet menu configuration
893 894 895 896 897 898 899 900
# This helps making various resources and fewer used features less visible.


# todo: add permissions: 'permissions': ['core.user']

# The following items are hidden on purpose:
# core.site (we only have one site)
# scanners.state, will be deprecated and removed (if not already)
Elger Jonker's avatar
Elger Jonker committed
901
# See: http://jet.readthedocs.io/en/latest/config_file.html#custom-menu
902 903 904 905
# Permissions are AND-ed together.
# admin (a nonsense permission) has been added everywhere to avoid "empty arrows" when signing in with a role with
# limited permissions.
# For the default labels, see: https://docs.djangoproject.com/en/2.1/topics/auth/default/#topic-authorization
906
JET_SIDE_MENU_ITEMS = [  # A list of application or custom item dicts
Elger Jonker's avatar
Elger Jonker committed
907 908

    {'label': _('🔧 configuration'), 'items': [
909 910
        {'name': 'auth.user'},
        {'name': 'auth.group'},
Elger Jonker's avatar
Elger Jonker committed
911 912
        {'name': 'constance.config', 'label': _('configuration')},
        {'name': 'map.configuration', 'label': _('map configuration')},
913
        {'name': 'map.administrativeregion', 'label': _('import regions')},
914
    ], 'permissions': ['admin']},
915

Elger Jonker's avatar
Elger Jonker committed
916
    {'app_label': 'organizations', 'label': _('🏢 organizations'), 'items': [
917 918 919 920 921
        {'name': 'organization'},
        {'name': 'url'},
        {'name': 'promise'},
        {'name': 'coordinate'},
        {'name': 'organizationtype'},
922
    ], 'permissions': ['admin']},
923

924
    # todo: sort scan moment to show latest first.
Elger Jonker's avatar
Elger Jonker committed
925
    {'app_label': 'scanners', 'label': _('🔬 scanners'), 'items': [
926 927 928 929 930 931 932 933 934
        {'name': 'endpoint', 'permissions': ['admin']},
        {'name': 'endpointgenericscan', 'permissions': ['scanners.change_endpointgenericscan']},
        {'name': 'tlsscan', 'permissions': ['scanners.change_tlsscan']},
        {'name': 'tlsqualysscan', 'permissions': ['scanners.change_tlsqualysscan']},
        {'name': 'urlgenericscan', 'permissions': ['scanners.change_urlgenericscan']},
        {'name': 'screenshot', 'permissions': ['admin']},
        {'name': 'urlip', 'permissions': ['admin']},
        {'name': 'tlsqualysscratchpad', 'permissions': ['admin']},
        {'name': 'endpointgenericscanscratchpad', 'permissions': ['admin']},
935 936
    ]},

Elger Jonker's avatar
Elger Jonker committed
937 938 939
    {'label': _('🗺️ map (autogenerated)'), 'items': [
        {'name': 'map.organizationrating'},
        {'name': 'map.urlrating'},
940 941
        {'name': 'map.vulnerabilitystatistic'},
        {'name': 'map.mapdatacache'}
942
    ], 'permissions': ['admin']},
Elger Jonker's avatar
Elger Jonker committed
943 944

    {'label': _('🕒 periodic tasks'), 'items': [
945 946 947 948 949
        {'name': 'app.job'},
        {'name': 'django_celery_beat.periodictask'},
        {'name': 'django_celery_beat.crontabschedule'},
        {'name': 'django_celery_beat.intervalschedule'},
        {'name': 'django_celery_beat.solarschedule'},
950
    ], 'permissions': ['admin']},
951

952 953 954 955 956 957 958 959 960 961 962 963 964
    #
    # {'app_label': 'helpdesk', 'label': _('ℹ️ helpdesk'), 'items': [
    #     {'name': 'queue'},
    #     {'name': 'ticket'},
    #     {'name': 'followup'},
    #     {'name': 'customfield'},
    #     {'name': 'presetreply'},
    #     {'name': 'emailtemplate'},
    #     {'name': 'escalationexclusion'},
    #     {'name': 'ignoreemail'},
    #     {'name': 'kbcategory'},
    #     {'name': 'kbitem'},
    # ], 'permissions': ['admin']},
965

Elger Jonker's avatar
Elger Jonker committed
966
    {'app_label': 'hypersh', 'label': _('☁️ hypersh cloud scans'), 'items': [
Elger Jonker's avatar
Elger Jonker committed
967 968 969
        {'name': 'containerenvironment', 'label': _('Environment variables'), },
        {'name': 'containerconfiguration', 'label': _('Container configuration'), },
        {'name': 'containergroup', 'label': _('Container instances'), },
970
        {'name': 'credential'},
971
    ], 'permissions': ['admin']},
972

Elger Jonker's avatar
Elger Jonker committed
973
    {'app_label': 'game', 'label': _('👾️ the game'), 'items': [
974 975 976
        {'name': 'contest'},
        {'name': 'team'},
        {'name': 'organizationsubmission'},
977 978 979
        {'label': _('New organizations'),
         'url': '/admin/game/organizationsubmission/?has_been_accepted__exact=0&has_been_rejected__exact=0&o=-5',
         'url_blank': False},
980
        {'name': 'urlsubmission'},
981 982 983
        {'label': _('New urls'),
         'url': '/admin/game/urlsubmission/?has_been_accepted__exact=0&has_been_rejected__exact=0&o=-6.2.3',
         'url_blank': False},
984
    ], 'permissions': ['admin']},
985
]
Elger Jonker's avatar
Elger Jonker committed
986 987
# end django jet menu configuration
########
988 989 990

########
# Begin Cacheops
991 992 993
#         # django cacheops doesn't work with raw.
#         # too bad https://github.com/Suor/django-cacheops
#         # it's the elephant in the room in the documentation: all are explained except this one.
994 995 996 997

# Cacheops has been added to improve the retrieval of graphs-queries. At the time of writing it's only in use
# there using a hack to improve querying speed.

Elger Jonker's avatar
Elger Jonker committed
998 999 1000
# It was not an option to rewrite queries to tailor to specific caching schemes per database vendor, django ORM also
# does not support that.

1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012
# It's a hack because out of the box cacheops doesn't support raw querysets (the only caveat without explanation).
# But we just need the list of data for displaying, thus we can wrap that in a function and use function result caching
#
# Note that lru_cache from functools does not support lists or complex types, while cahcheops does.
#
# I've chosen cacheops because it's actively maintained and has a lot of commits, while it's backlog is clean.
# Another reason is that it doesn't affect anything, except what you explicitly cache. So behaviour of the rest of the
# application is unchanged.

# If redis is not available, that is not a problem. For example: certain development scenario's or when redis fails.
# in that case a limited set of functions are performed without caching (and thus slower) but without crashes.
# If it's too slow, the webserver will kill it anyway, or in dev environments it will take longer.
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023
# CACHEOPS_DEGRADE_ON_FAILURE = True

# CACHEOPS_REDIS = {
#     'host': 'localhost',  # redis-server is on same machine
#     'port': 6379,        # default redis port
#     'db': 1,             # SELECT non-default redis database
#                          # using separate redis db or redis instance
#                          # is highly recommended
#
#     'socket_timeout': 3,   # connection timeout in seconds, optional
# }
1024 1025
# End cacheops
########