Add autoAWS v1.1

Updated source and additional css and db schema received from Jonathan in
zip bundle by email on 2018-08-16.
Sensitive information (account passwords) redacted.
parent 476db1ed
This diff is collapsed.
.error {
color: red;
font-weight: bold;
}
\ No newline at end of file
<!--
http://overpass-api.de/api/status
https://www.retsinformation.dk/Forms/R0710.aspx?id=200564
https://www.retsinformation.dk/eli/lta/2017/136
http://danmarksadresser.dk/adressedata
-->
<!DOCTYPE html>
<head>
<meta charset="uft-8"/>
......@@ -15,18 +21,22 @@
</head>
<?php
## Setting various settings ##
error_reporting(0);
ini_set('default_charset', 'utf-8');
ini_set('memory_limit', '2048M'); //standard memory size not sufficient when downloading AWS data
set_time_limit(3540); //59 minutes. a changeset will never be larger than ~9000 edits. based on testing, approximately 3.5 edits are done per second. so the script should never run for more than approximately 45 minutes. main bottleneck is the OSM API, which requires we call it once for every single node we want to update
define('THIS', 'autoAWS 1.0'); //name and version of this script
//define('URL', 'https://master.apis.dev.openstreetmap.org/api/0.6/'); //test API URL
define('DEBUG', FALSE); //disables all cURL calls to the OSM API if set to true
define('URL', 'https://api.openstreetmap.org/api/0.6/'); //url to the OSM API
define('PASSWORD', (redacted)); //password to log in to OSM
define('OUTPUT', FALSE); //whether or not to output script progress to the browser
define('CONTINUOUS', FALSE); //if set to true, will reload the html page 1 second after an update is finished, thus immediately starting a new one
set_time_limit(5400); //90 minutes
define('THIS', 'autoAWS 1.1'); //name and version of this script
define('URL', 'https://api.openstreetmap.org/api/0.6/'); //url to the OSM API
define('USERNAME', 'autoAWS'); //OSM username
define('PASSWORD', 'secret'); //OSM password
define('DEBUG', FALSE); //disables all cURL calls to the OSM API
define('OUTPUT', FALSE); //whether or not to output script progress to the browser (disabled by default for server version)
define('CONTINUOUS', FALSE); //reload the html page 1 second after an update is finished, thus immediately starting a new one (disabled by default for server version)
define('MANUAL', FALSE); //manually select which postcode to update (should only be used with CONTINUOUS=FALSE)
define('MANUAL_CODE', 4720); //the postcode to update if MANUAL is TRUE
define('UPDATE_ALL', FALSE); //update all address nodes, even if nothing has changed. Useful when changing the tagging scheme
define('UPDATE_BY_TAG', FALSE); //force an update of an address node if a specific tag (defined as UPDATE_TAG) is present
define('UPDATE_TAG', 'osak:revision'); //the tag to detect if UPDATE_BY_TAG is set to true
if(isset($_GET['postnr'])) {
......@@ -36,8 +46,8 @@ elseif(isset($_GET['cleardb'])) {
database_cleanup();
output(6);
}
elseif (redacted) {
(redacted) {
elseif(isset($_GET['passcode'])) {
if($_GET['passcode'] == 'secret') {
main();
}
}
......@@ -200,7 +210,10 @@ function postnummer_navn(int $nr): string {
function db_connect(): PDO {
//connects to database using PDO and returns a PDO object
(redacted)
$host = 'localhost';
$db = 'autoaws';
$user = '';
$pass = '';
$charset = 'utf8mb4';
$dsn = "mysql:host=$host;dbname=$db;charset=$charset";
......@@ -275,6 +288,9 @@ function download_osm_data(): array {
$string = file_get_contents('http://overpass-api.de/api/interpreter?data=[out:json];node[%22osak:identifier%22][%22addr:postcode%22=' . POSTNUMMER . '];out;');
if(!$string) {
output(9, 'Kunne ikke kontakte overpass API');
if(CONTINUOUS) {
echo '<script>setTimeout(location.reload.bind(location), 1000);</script>';
}
die();
}
$json = (array) json_decode($string, true);
......@@ -296,6 +312,9 @@ function download_osm_data(): array {
$c = (float) $addr['lat'];
$d = (float) $addr['lon'];
$e = isset($addr['tags']['addr:city']) ? $addr['tags']['addr:city'] : NULL;
if(UPDATE_BY_TAG) {
$e = isset($addr['tags'][UPDATE_TAG]) ? 'invalid' : $e; //by setting city name to "invalid" in the database, the DAR<=>OSM data check will fail for this address, thus triggering an update
}
$f = isset($addr['tags']['addr:country']) ? $addr['tags']['addr:country'] : NULL;
$g = isset($addr['tags']['addr:housenumber']) ? $addr['tags']['addr:housenumber'] : NULL;
$h = isset($addr['tags']['addr:postcode']) ? $addr['tags']['addr:postcode'] : NULL;
......@@ -435,7 +454,7 @@ function download_ois_fixes(): int {
$stmt->execute([$fixed_name, $mun_no, $str_no]);
}
catch(PDOException $ex) {
//do nothing
continue;
}
}
$antal = $db->query('select count(*) from `ois_fixes`')->fetchColumn();
......@@ -456,7 +475,7 @@ function create_changeset() {
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_HTTPAUTH => CURLAUTH_BASIC,
CURLOPT_USERPWD => "autoAWS:".PASSWORD,
CURLOPT_USERPWD => USERNAME.":".PASSWORD,
CURLOPT_CUSTOMREQUEST =>"PUT",
CURLOPT_POSTFIELDS => html_entity_decode($changeset->asXML()),
CURLOPT_URL => URL.'changeset/create')
......@@ -490,7 +509,7 @@ function close_changeset(int $chg = CHANGESET) {
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_HTTPAUTH => CURLAUTH_BASIC,
CURLOPT_USERPWD => "autoAWS:".PASSWORD,
CURLOPT_USERPWD => USERNAME.":".PASSWORD,
CURLOPT_CUSTOMREQUEST =>"PUT",
CURLOPT_URL => URL.'changeset/'.$chg.'/close')
);
......@@ -541,6 +560,16 @@ function detect_updates() {
)
AND `osmdata`.`autoaws_ignore` = 0;
';
//forces an update of all addresses, useful if the tagging structure has been changed
if(UPDATE_ALL) {
$sql = '
INSERT INTO `update_addr`(`osak:identifier`, `node_id`, `lat`, `lon`, `addr:city`, `addr:country`, `addr:housenumber`, `addr:postcode`, `addr:street`, `addr:municipality`, `addr:place`)
SELECT `awsdata`.`osak:identifier`, `osmdata`.`node_id`, `awsdata`.`lat`, `awsdata`.`lon`, `awsdata`.`postnummer_navn`, "DK", `awsdata`.`husnr`, `awsdata`.`postnummer_nr`, `awsdata`.`vejstykke_navn`, `awsdata`.`kommune_navn`, `awsdata`.`sup_bynavn`
FROM `awsdata`
INNER JOIN `osmdata` ON `awsdata`.`osak:identifier` = `osmdata`.`osak:identifier`
WHERE `osmdata`.`autoaws_ignore` = 0;
';
}
$db->exec($sql);
$db = NULL;
}
......@@ -685,6 +714,7 @@ function split_changeset(int $sum) {
function get_addr_tags(): array {
//return an array of all recognised address tags, i.e. tags we are comfortable deleting in case we need to remove an address
//NOTE!!! if updating this list, also update the $xpath_string variable in delete_nodes()
return array(
"addr:street",
"addr:housenumber",
......@@ -722,7 +752,7 @@ function create_nodes() {
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_HTTPAUTH => CURLAUTH_BASIC,
CURLOPT_USERPWD => "autoAWS:".PASSWORD,
CURLOPT_USERPWD => USERNAME.":".PASSWORD,
CURLOPT_CUSTOMREQUEST =>"PUT",
CURLOPT_URL => URL.'node/create')
);
......@@ -759,7 +789,7 @@ function create_nodes() {
$db = NULL;
}
function detect_parent_ways($node_id) {
function detect_parent_ways($node_id): bool {
//returns TRUE if a given node is part of a way, or FALSE if the given node is standalone (not part of a way)
$parent_ways = new SimpleXMLElement(file_get_contents('https://api.openstreetmap.org/api/0.6/node/' . $node_id . '/ways'));
if($parent_ways->children()[0] == NULL) {
......@@ -778,7 +808,7 @@ function update_node_in_way($result): bool {
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_HTTPAUTH => CURLAUTH_BASIC,
CURLOPT_USERPWD => "autoAWS:".PASSWORD,
CURLOPT_USERPWD => USERNAME.":".PASSWORD,
CURLOPT_CUSTOMREQUEST =>"PUT",
CURLOPT_URL => URL.'node/create')
);
......@@ -819,7 +849,7 @@ function update_node_in_way($result): bool {
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_HTTPAUTH => CURLAUTH_BASIC,
CURLOPT_USERPWD => "autoAWS:".PASSWORD,
CURLOPT_USERPWD => USERNAME.":".PASSWORD,
CURLOPT_CUSTOMREQUEST => "PUT"
));
if(!$node) {
......@@ -839,14 +869,14 @@ function update_node_in_way($result): bool {
return TRUE;
}
function delete_node_in_way($node, $id) {
//removes all address tags from a given node
function delete_node_in_way($node, $id): bool {
//remove all address tags from a node which is part of a way (delete the address without actually deleting the node)
$ch = curl_init();
curl_setopt_array($ch, array(
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_HTTPAUTH => CURLAUTH_BASIC,
CURLOPT_USERPWD => "autoAWS:".PASSWORD,
CURLOPT_USERPWD => USERNAME.":".PASSWORD,
CURLOPT_CUSTOMREQUEST => "PUT"
));
$node->node[0]['changeset'] = CHANGESET;
......@@ -874,7 +904,7 @@ function update_nodes() {
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_HTTPAUTH => CURLAUTH_BASIC,
CURLOPT_USERPWD => "autoAWS:".PASSWORD,
CURLOPT_USERPWD => USERNAME.":".PASSWORD,
CURLOPT_CUSTOMREQUEST => "PUT"
));
$error = array();
......@@ -978,7 +1008,7 @@ function update_node(string $xml_node, int $id) {
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_HTTPAUTH => CURLAUTH_BASIC,
CURLOPT_USERPWD => "autoAWS:".PASSWORD,
CURLOPT_USERPWD => USERNAME.":".PASSWORD,
CURLOPT_CUSTOMREQUEST => "PUT",
CURLOPT_URL => URL.'node/'.$id,
CURLOPT_POSTFIELDS => $xml_node
......@@ -1054,7 +1084,7 @@ function delete_node(string $xml_node, int $id) {
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_HTTPAUTH => CURLAUTH_BASIC,
CURLOPT_USERPWD => "autoAWS:".PASSWORD,
CURLOPT_USERPWD => USERNAME.":".PASSWORD,
CURLOPT_CUSTOMREQUEST => "DELETE",
CURLOPT_URL => URL.'node/'.$id,
CURLOPT_POSTFIELDS => $xml_node
......@@ -1118,26 +1148,30 @@ function opdater_postnumre () {
function select_postcode() {
//select and define the next postcode to update
if(MANUAL) {
define('POSTNUMMER', MANUAL_CODE); return;
}
$db = db_connect();
$stmt = $db->prepare('SELECT `postnummer`, `deleted` FROM `last_update` ORDER BY `deleted` DESC, `opdateret` ASC, `postnummer` ASC LIMIT 1');
try {
$stmt->execute();
$postnr = (int) $stmt->fetch()['postnummer'];
$fetch = $stmt->fetch();
$postnr = (int) $fetch['postnummer'];
if($postnr >= 1000 AND $postnr <= 9999) {
define('POSTNUMMER', $postnr);
}
else {
throw new Exception();
}
if($fetch['deleted'] == 1) {
$db->exec('DELETE FROM `last_update` WHERE `last_update`.`postnummer` = '.POSTNUMMER);
}
}
catch(Exception $ex) {
opdater_postnumre();
output(9, 'Kunne ikke indlæse postnumre');
die();
}
if($stmt->fetch()['deleted'] == 1) {
$db->exec('DELETE FROM `last_update` WHERE `last_update`.`postnummer` = '.POSTNUMMER);
}
if(POSTNUMMER % 1000 == 0) { //we need to make sure the list of postcodes is up to date
output(2, 'Opdaterer listen af postnumre');
opdater_postnumre();
......@@ -1159,4 +1193,4 @@ function database_cleanup() {
$db->exec('UPDATE `last_update` SET `last_update`.`opdateret` = "' . date("Y-m-d H:i:s") . '" WHERE `last_update`.`postnummer` = ' . POSTNUMMER);
}
$db = NULL;
}
\ No newline at end of file
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment