Verified Commit 9eb79940 authored by Elger Jonker's avatar Elger Jonker

a bit more friendly interaction with the gmaps API

parent 65bbc939
Pipeline #39964678 passed with stages
in 19 minutes and 46 seconds
......@@ -8,7 +8,10 @@ log = logging.getLogger(__package__)
importers = {
# failmap import_organizations dutch_government
'dutch_government': dutch_government,
# failmap import_organizations excel https://example.com/example.xlsx
'excel': excel
}
......
......@@ -28,6 +28,7 @@ import googlemaps
from datetime import datetime
import tldextract
import pytz
from time import sleep
from failmap.organizations.models import Organization, OrganizationType, Coordinate, Url
......@@ -116,7 +117,22 @@ def geolocate(organizations: List):
# continue
# you can only get the geometry with a lot of data attached, so it cannot be done cheaper :(
geocode_result = gmaps.geocode("%s, %s" % (organization['address'], organization['geocoding_hint']))
try:
geocode_result = gmaps.geocode("%s, %s" % (organization['address'], organization['geocoding_hint']))
# give a little slack, so the server is not overburdened.
sleep(0.1)
except googlemaps.exceptions.ApiError as e:
# sometimes the API is just obnoxiously blocking the request, saying the IP is not authorized,
# while it is.
# single retry would be enough. Otherwise there are real issues. We don't want to retry a lot because
# it costs money.
log.debug('Error received from API, trying again in 10 seconds.')
log.debug(e)
sleep(10)
geocode_result = gmaps.geocode("%s, %s" % (organization['address'], organization['geocoding_hint']))
"""
[{'address_components': [
......
......@@ -23,6 +23,10 @@ COUNTRY = 'NL'
# https://almanak-redactie.overheid.nl/archive/
# the xml plural / single are to help parsing, they don't need to be in your specification.
datasets = [
{'url': 'https://almanak-redactie.overheid.nl/archive/exportOO_ministeries.xml',
'description': 'Dutch ministries', 'layer': LAYER, 'country': COUNTRY,
'xml_plural': 'organisaties', 'xml_single': 'organisatie'},
{'url': 'https://almanak-redactie.overheid.nl/archive/exportOO_gemeenschappelijke_regelingen.xml',
'description': 'Gemeenschappelijke Regelingen', 'layer': LAYER, 'country': COUNTRY,
'xml_plural': 'gemeenschappelijkeRegelingen', 'xml_single': 'gemeenschappelijkeRegeling'},
......@@ -43,9 +47,6 @@ datasets = [
'description': 'Zelfstandige bestuursorganen', 'layer': LAYER, 'country': COUNTRY,
'xml_plural': 'zelfstandigeBestuursorganen', 'xml_single': 'zelfstandigBestuursorgaan'},
{'url': 'https://almanak-redactie.overheid.nl/archive/exportOO_ministeries.xml',
'description': 'Dutch ministries', 'layer': LAYER, 'country': COUNTRY,
'xml_plural': 'organisaties', 'xml_single': 'organisatie'},
]
namespaces = {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment