Commit 5fbf86ee authored by Johan Bloemberg's avatar Johan Bloemberg

wip

parent b9ef27fe
Pipeline #38233857 failed with stage
in 4 minutes and 1 second
......@@ -12,10 +12,14 @@ if test -z "$PYTHON_BIN"; then
exit 1
fi
# determine virtualenv that Pipenv will use
# https://pipenv.readthedocs.io/en/latest/install/#virtualenv-mapping-caveat
export VIRTUAL_ENV=$(python -c 'from pipenv.project import Project; print(Project(chdir=False).virtualenv_location)')
# use virtualenv created by Tox for development
export PATH=$PWD/.tox/default/bin/:$PATH
export VIRTUAL_ENV=$PWD/.tox/default/
export PATH=$VIRTUAL_ENV/bin:$PATH
# add tools to path
export PATH=$PWD/tools/:$PATH
export DJANGO_SETTINGS_MODULE=failmap.settings
$(info Running in virtualenv: ${VIRTUAL_ENV})
all: check test
.PHONY: check
check: | .pipfile_installed
pylama failmap tests setup.py
.PHONY: autofix
autofix: | .pipfile_installed
# fix trivial pep8 style issues
autopep8 -ri failmap tests setup.py
# remove unused imports
autoflake -ri --remove-all-unused-imports failmap tests setup.py
# sort imports
isort -rc failmap tests setup.py
# do a check after autofixing to show remaining problems
pylama failmap tests
.PHONY: test
test: check | .pipfile_installed
# run testsuite
coverage run --include 'failmap/*' -m pytest -v -k 'not integration and not system' ${test_args}
# generate coverage
coverage report
# and pretty html
coverage html
# ensure no model updates are commited without migrations
failmap makemigrations --check
.PHONY: datasets
test_datasets: | .pipfile_installed
# find all fixtures in source and verify loading each one
/bin/sh -ec "find failmap -path '*/fixtures/*.yaml' -print0 | \
xargs -0n1 basename -s .yaml | uniq | \
xargs -n1 failmap test_dataset"
.PHONY: integration
test_integration: | .pipfile_installed
nc localhost:redisport || exit 1
DB_NAME=test.sqlite3 pytest -v -k 'integration' ${test_args}
.PHONY: system
test_system: | .pipfile_installed
pytest -v tests/system ${test_args}
# creates virtualenv and installed all Pipfile requirements and the failmap project
.pipfile_installed: ${VIRTUAL_ENV}/.installed
${VIRTUAL_ENV}/.installed:
pipenv sync --dev
touch $@
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[dev-packages]
"autopep8" = "*"
autoflake = "*"
isort = "*"
pylama = "*"
coverage = "*"
pytest = "*"
pytest-cov = "*"
pytest-django = "*"
pytest-responses = "*"
pytest-logging = "*"
django-coverage = "*"
django-extensions = "*"
django-debug-toolbar = "*"
django-debug-toolbar-request-history = "*"
pytest-mock = "*"
pytest-docker = "*"
sphinx-autobuild = "*"
recommonmark = "*"
httmock = "*"
freezegun = "*"
colorama = "*"
pydotplus = "*"
watchdog = {git = "https://github.com/gorakhargosh/watchdog.git", ref = "d33bee0290e00ecb3570dd938db72a2550933586"}
ipython = "*"
django-cprofile-middleware = "*"
Sphinx = "*"
[packages]
"e1839a8" = {path = ".", editable = true}
django-jet = "*"
django-countries = "*"
django-leaflet = "*"
django-geojson = {extras = ["field"]}
jsonfield = "*"
django-import-export = "*"
django-constance = {extras = ["database"]}
typing = "*"
certifi = "*"
pytz = "*"
redis = "<3"
celery = {extras = ["redis", "eventlet"]}
flower = "*"
django-statsd-mozilla = "*"
django-uwsgi = "*"
requests = "*"
dnspython = "*"
netaddr = "*"
untangle = "*"
python-resize-image = "*"
tldextract = "*"
colorlog = "*"
django-proxy = "*"
raven = "*"
deepdiff = "*"
influxdb = "*"
"urllib3" = "*"
simplejson = "*"
csscompressor = "*"
retry = "*"
recommonmark = "*"
rdp = "*"
clint = "*"
django-nested-admin = "*"
django-fsm = "*"
django-fsm-log = "*"
django-crispy-forms = "*"
django-autocomplete-light = "*"
"django-admin-sortable2" = "*"
django-colorful = "*"
"iso3166" = "*"
docker = "*"
tenacity = "*"
Django = "*"
PyYAML = "*"
django_celery_beat = "*"
Pillow = "*"
pyOpenSSL = "*"
django_compressor = "*"
sphinx_rtd_theme = "*"
Wikidata = "*"
hyper-sh = {git = "https://github.com/berryp/hyper_sh.git", ref = "8a4c52ac88fdcc052f3c008377bdf77e70c30904"}
"Django-Select2" = "*"
[requires]
python_version = "3.6"
This diff is collapsed.
......@@ -47,19 +47,18 @@ brew install git python3 direnv
**Debian Linux (apt)**
```bash
apt-get install git python3 direnv
apt-get install git python3 direnv make
```
**Redhat/CentOS (yum)**
```bash
yum install git python3 direnv
yum install git python3 direnv make
```
Or download and install each package separately:
- [git](https://git-scm.com/downloads) (download and install)
- [python3.6](https://www.python.org/downloads/) (download and install)
- [Tox](http://tox.readthedocs.io/) (`pip3 install --user tox`)
- [direnv](https://direnv.net/) (download and install, then follow [setup instructions](https://direnv.net/), see Direnv section below)
- [Docker](https://docs.docker.com/engine/installation/) (recommended, follow instructions to install.)
......
......@@ -21,7 +21,7 @@ Once you're happy with your changes, and you've tested it on your environment us
(eg productiondata), then you can verify (and autofix) the code to our code standards:
```
tox -e autofix
make autofix
```
Fix any of the remarks it gives, otherwise your changes will not be added to the master branch.
......@@ -75,7 +75,8 @@ Make sure you've got an up to date development environment. You can do so by run
Rebuild the environment:
```
tox -r
make clean
make
```
Get all requirements and development requirements:
......@@ -90,19 +91,19 @@ This project sticks to default pycodestyle/pyflakes configuration to maintain co
To run code quality checks and unit tests run:
tox
make
For a comprehensive test run:
tox -e check,test,datasets
make check test test_datasets
To make life easier you can use `autopep8`/`isort` before running `tox` to automatically fix most style issues:
tox -e autofix
make autofix
To run only a specific test use:
tox -e test -- -k test_name
make test test_args="-k test_name"
To only run a specific test suite use for example:
......@@ -114,7 +115,7 @@ A coverage report is generated after running tests, on OSX it can be viewed usin
Pytest allows to drop into Python debugger when a tests fails. To enable run:
tox -- --pdb
make test_args="--pdb"
## Integration/system tests
Besides quality checks and unit tests there are also integration and system testing frameworks available.
......@@ -123,11 +124,11 @@ These frameworks will run in the CI system but not by default when running `tox`
To run these testsuites make sure Docker is installed and running and run either:
tox -e integration
make test_integration
or
tox -e system
make test_system
## Direnv / Virtualenv
......
......@@ -7,36 +7,42 @@ Including a nice admin interface:
![Admin Interface](shared/admin_interface.png)
## 1: Install dependencies on your system
Setup your system to run this software using your favourite package manager.
**MacOS (brew)**
```bash
brew install git python3 direnv
```
**Debian Linux (apt)**
```bash
apt-get install git python3 direnv
apt-get install git python3 direnv make
```
**Redhat/CentOS (yum)**
```bash
yum install git python3 direnv
yum install git python3 direnv make
```
Or download and install each package seperately:
- [git](https://git-scm.com/downloads) (download and install)
- [python3.6](https://www.python.org/downloads/) (download and install)
- [Tox](http://tox.readthedocs.io/) (`pip3 install --user tox`)
- [make](https://www.gnu.org/software/make/) (often preinstalled)
- [direnv](https://direnv.net/) (download and install, then follow [setup instructions](https://direnv.net/), see Direnv section below)
- [Docker](https://docs.docker.com/engine/installation/) (recommended, follow instructions to install.)
## 2: Install direnv correctly
Then set up direnv, the right command depends on your shell:
**BASH**
Add the following line at the end of the ~/.bashrc file:
```bash
eval "$(direnv hook bash)"
```
......@@ -45,6 +51,7 @@ Make sure it appears even after rvm, git-prompt and other shell extensions that
**ZSH**
Add the following line at the end of the ~/.zshrc file:
```bash
eval "$(direnv hook zsh)"
```
......@@ -63,13 +70,7 @@ Add the following line at the end of the ~/.cshrc file:
eval `direnv hook tcsh`
```
## 3: Generic install steps
Install Tox, which helps to install the rest of the dependancies of this project:
```bash
pip3 install --user tox
```
In a directory of your choosing:
......@@ -91,10 +92,10 @@ This prepares the shell environment for local development.
direnv allow
```
Running Tox once creates a development Virtualenv in .tox/default/ which is automatically used after creation due to Direnv setup. Running Tox without arguments by default also runs basic checks and tests to verify project code quality.
Running `make` once creates a development Virtualenv which is automatically used after creation due to Direnv setup. Running `make` without arguments by default also runs basic checks and tests to verify project code quality.
```bash
tox
make
```
After completing succesfully Failmap is available to run. For example, to show a list of commands:
......@@ -102,6 +103,7 @@ After completing succesfully Failmap is available to run. For example, to show a
```bash
failmap help
```
Now run the following command to start a full development server.
```bash
......@@ -112,6 +114,7 @@ Now visit the [map website](http://127.0.0.1:8000/) and/or the
[admin website](http://127.0.0.1:8000/admin/) at http://127.0.0.1:8000 (credentials: admin:faalkaart).
## 4. Optional Steps
This shows the current data on the map:
```bash
......@@ -135,8 +138,8 @@ https://www.youtube.com/watch?v=a14Y2V5zJlY
https://www.youtube.com/watch?v=eAwq2QV7f1k
```
## Troubleshooting
This repository uses [submodules](https://git-scm.com/docs/git-submodule) to pull in
external dependencies. If you have not cloned the repository with `--recursive` or you need to
restore the submodules to the expected state run:
......@@ -171,12 +174,12 @@ build/staging/latest tags).
For local development informal release or a special `dev0` build release is used which indicates a different state
from the formal releases.
## Known Issues
### Docker installation
#### ERROR: for failmap_database_1 Cannot start service database: Mounts denied:
As the error suggests, you're running the installation from a directory that is not shared with Docker. Change the
docker configuration or run the installation from your user directory. You might receive this error if you run
`docker-composer up` from /var/www/ or /srv/www/ as docker by default only has access to your user directory.
# Tools/dependencies required for development or running tests.
autopep8
autoflake
isort
pylama
coverage
pytest
pytest-cov
pytest-django
# counterpart of requests for mock responses
pytest-responses
# output logging when test fails
pytest-logging
django-coverage
django-extensions
django-debug-toolbar
django-debug-toolbar-request-history
pytest-mock
pytest-docker
# # dashboard for celery introspection
# # run failmap celery flower
# # go to http://localhost:5555/
# celery-flower
# docs / readthedocs.io
sphinx
sphinx-autobuild
recommonmark
# brotlipy # doesn't work, some vague errors not worth the time: compression handled elsewhere
# slimit does not work with vue.js
# tests (of scanners)
httmock
freezegun
colorama
pydotplus # generating graphs with the docs command...
# used to restart celery worker on file changes
# use current (03-2018) master as it includes a fix for macOS: https://github.com/gorakhargosh/watchdog/pull/329
git+https://github.com/gorakhargosh/watchdog.git@d33bee0290e00ecb3570dd938db72a2550933586#egg=watchdog
ipython # failmap shell -i ipython
django-cprofile-middleware
# This file contains all requirements for a release build of failmap.
# Any testing/debug only requirements should to into requirements.dev.txt.
# Requirements only needed for release should go in requirements.deploy.txt.
# Django jet is blocking migration to 2.0
# error: from django.views.i18n import javascript_catalog
# ImportError: cannot import name 'javascript_catalog'
django
django-jet
django-countries
django-leaflet # geojson editor in admin
django-geojson [field] # geojson editor in admin
jsonfield
# django-jsonfield
django-import-export
django-constance[database]
pyyaml
typing
# use mozzila certificate bundle by default
certifi
pytz
# task processing framework
redis<3 # forcing a specific version of redis, 3.0.1 gives timeout errors during build.
celery[redis,eventlet]
django-celery-beat
flower # used for queue statistics
# https://github.com/pi-bjl/celery-statsd/commit/5d61d7756f115dbf05a7eeb8314495b53ee1955e
django-statsd-mozilla
django_uwsgi
# scanner dependencies
requests
dnspython # dnsrecon
netaddr # dnsrecon
untangle # dns scans https://github.com/stchris/untangle
Pillow # screenshots
python-resize-image # screenshots
tldextract
# logging
colorlog
# for proxying external requests (mapbox)
django-proxy
# sentry client
raven
# allow recursive comparison of dictionaries
deepdiff
influxdb
urllib3
# loading json is faster in simplejson
# https://stackoverflow.com/questions/712791/what-are-the-differences-between-json-and-simplejson-python-modules
# needed for mapping reasons.
simplejson
# Remote worker TLS
pyopenssl
certifi
# static file collection/generation
django_compressor # bundle javascript and css
csscompressor # css compression
retry
sphinx_rtd_theme # readthedocs.io only looks at requirements.txt (default) and thus it's better to have it here.
# docs / readthedocs.io
recommonmark
# map, to reduce the number of points (Ramer-Douglas-Peucker algorithm)
rdp
clint # downloading stuff from overpass
wikidata # downloading data from wikidata
django-nested-admin # going deeper underground
sphinx_rtd_theme
# https://github.com/tardyp/hyper_sh/pull/14/
git+https://github.com/berryp/hyper_sh.git@8a4c52ac88fdcc052f3c008377bdf77e70c30904#egg=hyper_sh
django-fsm
django-fsm-log
# game requirements
django-crispy-forms
django-autocomplete-light
django-admin-sortable2 # nice sorting in the admin, used in map configurations.
# Django helpdesk started having migration issues during build. Given we don't use it yet i've just removed it.
# Helpdesk requirements
# Runs django-helpdesk, https://github.com/django-helpdesk
# django-helpdesk # more complete ticketing, might help a lot with interaction.
# django-bootstrap-form>=3.3,<4
# email-reply-parser
# django-markdown-deux
# beautifulsoup4
# lxml
# six
django-colorful # game team colors
django-select2 # game, add multiple urls in one go.
# function caching / query caching for slow stats graphs queries
# another way has to be found, but until then
# django-cacheops <-- doesn't do raw queries, so forget it.
# and when wrapping raw queries it is more heavy than just using djangos caching framework.
# django-redis # caching of graph resultsets
iso3166 # ISO country code conversion
docker # hyper scaling
tenacity # generic retry library
from __future__ import print_function
import os
import sys
from subprocess import check_output
......@@ -44,28 +42,10 @@ def get_version():
return '0.0.0'
def requirements(extra=None):
"""Return list of required package names from requirements.txt."""
# strip trailing comments, and extract package name from git urls.
if extra:
filename = 'requirements.' + extra + '.txt'
else:
filename = 'requirements.txt'
requirements = [r.strip().split(' ', 1)[0].split('egg=', 1)[-1]
for r in open(filename) if not r.startswith('#')]
return requirements
setup(
name='failmap',
version=get_version(),
packages=find_packages(),
install_requires=requirements(),
# allow extra packages to be installed, eg: `pip install -e .[deploy]`
extras_require={
'development': requirements(extra='dev'),
'deploy': requirements(extra='deploy'),
},
entry_points={
'console_scripts': [
'failmap = failmap.manage:main',
......
......@@ -45,7 +45,7 @@ def docker_ip():
if not docker_host:
return '127.0.0.1'
match = re.match('^tcp://(.+?):\d+$', docker_host)
match = re.match('^tcp://(.+?):[0-9]+$', docker_host)
if not match:
raise ValueError(
'Invalid value for DOCKER_HOST: "%s".' % (docker_host,)
......
......@@ -11,45 +11,39 @@ envdir = {toxworkdir}/default
# force use of specific Python version outside of Tox environment to be used by Tox itself
basepython = {env:PYTHON_BIN:python3.6}
usedevelop = True
# see requirements*.txt, hack to make git install work
deps =
git+https://github.com/gorakhargosh/watchdog.git@d33bee0290e00ecb3570dd938db72a2550933586#egg=watchdog
git+https://github.com/berryp/hyper_sh.git@8a4c52ac88fdcc052f3c008377bdf77e70c30904#egg=hyper_sh
extras =
development
# allow setuptools extras (eg: deploy requirements) to be enabled in CI
{env:TOX_EXTRAS:}
deps = pipenv
setenv =
DJANGO_SETTINGS_MODULE = failmap.settings
# allow broker url to be overriden for development
passenv = BROKER C_FORCE_ROOT DOCKER_HOST SYSTEM_TEST_TIMEOUT IMAGE
# install all requirements using Pipfile
commands_pre=pipenv sync --dev
# test set runs test suite and fixture integrity checks
[testenv:test]
commands =
# run testsuite
coverage run --include 'failmap/*' -m pytest -v -k 'not integration and not system' {posargs}
pipenv run coverage run --include 'failmap/*' -m pytest -v -k 'not integration and not system' {posargs}
# generate coverage
coverage report
pipenv run coverage report
# and pretty html
coverage html
pipenv run coverage html
# ensure no model updates are commited without migrations
failmap makemigrations --check
pipenv run failmap makemigrations --check
# run code quality check
[testenv:check]
commands =
# make sure code quality is up to par
pylama failmap tests setup.py
pipenv run pylama failmap tests setup.py
# ensure all datasets can be imported (and indirectly if all migrations work)
[testenv:datasets]
commands =
# find all fixtures in source and verify loading each one
/bin/sh -ec "find failmap -path '*/fixtures/*.yaml' -print0 | \
xargs -0n1 basename -s .yaml | uniq | \
xargs -n1 failmap test_dataset"
# find all fixtures in source and verify loading each one
pipenv run /bin/sh -ec "find failmap -path '*/fixtures/*.yaml' -print0 | \
xargs -0n1 basename -s .yaml | uniq | \
xargs -n1 failmap test_dataset"
# ensure rebuild-ratings is deterministic
[testenv:deterministic]
......@@ -60,19 +54,19 @@ commands = /bin/bash tools/compare_differences.sh HEAD HEAD tools/show_ratings.s
# install all dependencies so isort knows packages belong where
commands =
# fix trivial pep8 style issues
autopep8 -ri failmap tests setup.py
pipenv run autopep8 -ri failmap tests setup.py
# remove unused imports
autoflake -ri --remove-all-unused-imports failmap tests setup.py
pipenv run autoflake -ri --remove-all-unused-imports failmap tests setup.py
# sort imports
isort -rc failmap tests setup.py
pipenv run isort -rc failmap tests setup.py
# do a check after autofixing to show remaining problems
pylama failmap tests
pipenv run pylama failmap tests
[testenv:integration]
setenv =
{[testenv]setenv}
DB_NAME = test.sqlite3
commands = pytest -v -k 'integration' {posargs}
commands = pipenv run pytest -v -k 'integration' {posargs}
[testenv:system]
envdir = {toxworkdir}/testenv-system
......@@ -82,4 +76,4 @@ deps =
retry
usedevelop = False
skip_install = True
commands = pytest -v tests/system {posargs}
commands = pipenv run pytest -v tests/system {posargs}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment