Commit 7ec01907 authored by Kyle Johnson's avatar Kyle Johnson
Browse files

Initial commit

parents
.idea
venv
__pycache__
*.sqlite3
# Django Async Rivers
A Django async view example on running external API calls concurrently.
## Setup
1. Clone repository
2. Create virtual environment and install `requirements.txt`
3. Run tests with: `python manage.py test`
4. Run server with: `uvicorn django_async_rivers.asgi:application --reload`
Water data from the USGS Instantaneous Values Web Service
https://waterservices.usgs.gov/rest/IV-Service.html
"""
ASGI config for django_async_rivers project.
It exposes the ASGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.1/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'django_async_rivers.settings')
application = get_asgi_application()
"""
Django settings for django_async_rivers project.
Generated by 'django-admin startproject' using Django 3.1.2.
For more information on this file, see
https://docs.djangoproject.com/en/3.1/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.1/ref/settings/
"""
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '_____DO_NOT_USE_THIS_SECRET_KEY_IN_PRODUCTION_____'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
'rivers',
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'django_async_rivers.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'django_async_rivers.wsgi.application'
# Database
# https://docs.djangoproject.com/en/3.1/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': BASE_DIR / 'db.sqlite3',
}
}
# Password validation
# https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/3.1/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.1/howto/static-files/
STATIC_URL = '/static/'
"""django_async_rivers URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/3.1/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import include, path
from django.views.generic import RedirectView
urlpatterns = [
path('admin/', admin.site.urls),
path('weather/', include('rivers.urls', namespace='weather')),
path('', RedirectView.as_view(pattern_name='weather:dashboard_v3'))
]
"""
WSGI config for django_async_rivers project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.1/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'django_async_rivers.settings')
application = get_wsgi_application()
#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys
def main():
"""Run administrative tasks."""
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'django_async_rivers.settings')
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
if __name__ == '__main__':
main()
asgiref==3.2.10
certifi==2020.6.20
chardet==3.0.4
click==7.1.2
Django==3.1.3
h11==0.11.0
httpcore==0.12.0
httpx==0.16.1
idna==2.10
pytz==2020.1
requests==2.24.0
rfc3986==1.4.0
sniffio==1.2.0
sqlparse==0.4.1
urllib3==1.25.11
uvicorn==0.12.2
default_app_config = 'rivers.apps.RiversConfig'
from django.contrib import admin
# Register your models here.
from django.apps import AppConfig
class RiversConfig(AppConfig):
name = 'rivers'
from django.db import models
# Create your models here.
<html lang="en">
<head>
<title>Rivers</title>
<style>
main table th, main table td {
padding: 0.2em 1em;
text-align: left;
border: 1px solid #999;
}
main table td.numeric {
text-align: right;
}
</style>
</head>
<body>
<main>
<h1>River Dashboard {{ version }}</h1>
<p>Load Time: {{ load_time }} seconds</p>
<table>
<thead>
<tr>
<th>Site</th>
<th>Discharge (cfs)</th>
<th>Gage height (ft)</th>
</tr>
</thead>
<tbody>
{% for site, data in river_data %}
<tr>
<td>{{ site }}</td>
<td class="numeric">{{ data.00060|default_if_none:'' }}</td>
<td class="numeric">{{ data.00065|default_if_none:'' }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</main>
</body>
</html>
from unittest import mock
from django.test import TestCase
from django.urls import reverse
from .views import SITES
class RiverDashboardTests(TestCase):
@mock.patch('rivers.views.get_river_flow_and_height')
def test_dashboard_v1(self, mock_get_river_flow_and_height):
response = self.client.get(reverse('rivers:dashboard_v1'))
self.assertTemplateUsed(response, 'rivers/dashboard.html')
self.assertEqual(response.context['version'], 'v1')
mock_get_river_flow_and_height.assert_has_calls([
mock.call(site_id)
for site_id in SITES
])
@mock.patch('rivers.views.get_river_flow_and_height')
def test_dashboard_v2(self, mock_get_river_flow_and_height):
response = self.client.get(reverse('rivers:dashboard_v2'))
self.assertTemplateUsed(response, 'rivers/dashboard.html')
self.assertEqual(response.context['version'], 'v2')
mock_get_river_flow_and_height.assert_has_calls([
mock.call(site_id)
for site_id in SITES
], any_order=True)
@mock.patch('rivers.views.get_river_flow_and_height_async')
def test_dashboard_v3(self, mock_get_river_flow_and_height_async):
response = self.client.get(reverse('rivers:dashboard_v3'))
self.assertTemplateUsed(response, 'rivers/dashboard.html')
self.assertEqual(response.context['version'], 'v3')
mock_get_river_flow_and_height_async.assert_has_calls([
mock.call(site_id)
for site_id in SITES
], any_order=True)
from django.urls import path
from . import views
app_name = 'rivers'
urlpatterns = [
path('dashboard/v1/', views.dashboard_v1, name='dashboard_v1'),
path('dashboard/v2/', views.dashboard_v2, name='dashboard_v2'),
path('dashboard/v3/', views.dashboard_v3, name='dashboard_v3'),
]
import asyncio
import collections
import concurrent.futures
import time
import httpx
import requests
from django.shortcuts import render
SITES = collections.OrderedDict({
'06192500': 'Livingston',
'06214500': 'Billings',
'06295000': 'Forsyth',
'06309000': 'Miles City',
'06327500': 'Glendive',
'06329500': 'Sidney',
})
VARIABLES = {
'00060': 'Discharge, cubic feet per second',
'00065': 'Gage height, feet',
}
def parse_flow_and_height_from_json(json_data):
flow_and_height_data = {
'00060': None,
'00065': None,
}
for item in json_data['value']['timeSeries']:
variable_code = item['variable']['variableCode'][0]['value']
value = item['values'][0]['value'][0]['value']
if variable_code in flow_and_height_data:
flow_and_height_data[variable_code] = float(value)
return flow_and_height_data
def get_river_flow_and_height(site_id):
"""
Synchronous method to get river data from USGS
"""
response = requests.get(f'https://waterservices.usgs.gov/nwis/iv/?format=json&sites={site_id}&parameterCd=00060,00065&siteStatus=all')
data = parse_flow_and_height_from_json(response.json())
return data
async def get_river_flow_and_height_async(site_id):
"""
Asynchronous method to get river data from USGS
"""
async with httpx.AsyncClient() as client:
response = await client.get(f'https://waterservices.usgs.gov/nwis/iv/?format=json&sites={site_id}&parameterCd=00060,00065&siteStatus=all')
data = parse_flow_and_height_from_json(response.json())
return data
def dashboard_v1(request):
"""
Synchronous view that loads data one at a time
"""
start_time = time.time()
river_data = []
for site_id in SITES.keys():
river_data.append((SITES[site_id], get_river_flow_and_height(site_id)))
return render(request, 'rivers/dashboard.html', {
'river_data': river_data,
'version': 'v1',
'load_time': time.time() - start_time,
})
def dashboard_v2(request):
"""
Concurrent view that loads some data simultaneously
"""
start_time = time.time()
river_data = []
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor:
for site_id, data in zip(SITES.keys(), executor.map(get_river_flow_and_height, SITES.keys())):
river_data.append((SITES[site_id], data))
return render(request, 'rivers/dashboard.html', {
'river_data': river_data,
'version': 'v2',
'load_time': time.time() - start_time,
})
async def dashboard_v3(request):
"""
Asynchronous view that loads data using asyncio
"""
start_time = time.time()
river_data = []
datas = await asyncio.gather(*[get_river_flow_and_height_async(site_id) for site_id in SITES.keys()])
for site_id, data in zip(SITES.keys(), datas):
river_data.append((SITES[site_id], data))
return render(request, 'rivers/dashboard.html', {
'river_data': river_data,
'version': 'v3',
'load_time': time.time() - start_time,
})
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment