Commit 71129d17 authored by Rick Gruber-Riemer's avatar Rick Gruber-Riemer

Few changes in batch operation:

- Changed command line option -d to be based on Osmosis.
- Changed command line option -o to be optional and removed hard-coding
- Updated documentation
parent d746fd5f
......@@ -12,6 +12,7 @@ import stat
from _io import open
import calc_tile
import parameters
import setup
import tools
......@@ -36,10 +37,11 @@ def _open_file(name, directory):
return open(directory + name, "wb")
def _write_to_file(command, file_handle, python_exe):
file_handle.write(python_exe + ' ' + tools.get_osm2city_directory() + os.sep + command + ' -f ' + replacement_path + '/params.ini ')
def _write_to_file(command, file_handle, python_exe, params_out):
file_handle.write(python_exe + ' ' + tools.get_osm2city_directory() + os.sep + command)
file_handle.write(' -f ' + params_out)
if BASH_PARALLEL_PROCESS:
file_handle.write('&' + os.linesep + 'parallel_wait $max_parallel_process' + os.linesep)
file_handle.write(' &' + os.linesep + 'parallel_wait $max_parallel_process' + os.linesep)
else:
file_handle.write(os.linesep)
......@@ -52,7 +54,7 @@ if __name__ == '__main__':
parser.add_argument("-f", "--properties", dest="properties",
help="The name of the property file to be copied", required=True)
parser.add_argument("-o", "--out", dest="out",
help="The name of the property file to be generated", required=True)
help="The name of the property file to be generated", required=False)
parser.add_argument("-u", "--url",
help='Address of the api to download OSM data on the fly',
dest="api_url",
......@@ -73,11 +75,9 @@ if __name__ == '__main__':
required=False)
parser.add_argument("-x", dest="python_executable",
help="Path to specific Python executable",
default=False,
required=False)
parser.add_argument("-d", dest="orig_osm_data",
help="Use the OSM data as specified in params.ini instead of dynamic download",
action='store_true',
parser.add_argument("-d", dest="osmosis_executable",
help="Use the OSM data as specified in params.ini and split it with Osmosis using the specified path to Osmosis",
required=False)
args = parser.parse_args()
......@@ -86,6 +86,10 @@ if __name__ == '__main__':
if args.python_executable:
python_exe = args.python_executable
params_out_file_name = "params.ini"
if args.out:
params_out_file_name = args.out
logging.info('Generating directory structure for %s ', args.tile_name)
matched = re.match("([ew])([0-9]{3})([ns])([0-9]{2})", args.tile_name)
lon = int(matched.group(2))
......@@ -107,7 +111,25 @@ if __name__ == '__main__':
except OSError, e:
if e.errno != 17:
logging.exception("Unable to create path to output")
osmosis_command = None
if args.osmosis_executable:
osmosis_path = args.osmosis_executable
osm_xml_based = True # if False then the input is using pbf formatting
parameters.read_from_file(args.properties)
osmosis_command = args.osmosis_executable
if parameters.OSM_FILE.endswith(".pbf"):
osmosis_command += ' --read-pbf file="'
else:
osmosis_command += ' --read-xml file="'
sep_pos = args.properties.rfind(os.sep)
if sep_pos >= 0:
orig_path = args.properties[:sep_pos + 1]
osmosis_command += orig_path
osmosis_command += parameters.OSM_FILE + '" --bounding-box completeWays=yes '
osmosis_command += 'top=%f left=%f bottom=%f right=%f --wx file="%s"' + os.linesep
download_file = _open_file(_get_file_name("download_", args.tile_name), root_dir_name)
files = []
utils = ['tools', 'osm2city', 'osm2pylon', 'platforms', 'roads', 'piers', ]
......@@ -163,24 +185,28 @@ done
# Manipulate the properties file and write to new destination
with open(args.properties, "r") as sources:
lines = sources.readlines()
with open(path + os.sep + args.out, "w") as sources:
with open(path + os.sep + params_out_file_name, "w") as sources:
replacement = '\\1 "' + replacement_path + '"'
for line in lines:
if not args.orig_osm_data:
line = re.sub('^\s*(PREFIX\s*=)(.*)', replacement, line)
line = re.sub('^\s*(PREFIX\s*=)(.*)', replacement, line)
line = re.sub('^\s*(BOUNDARY_EAST\s*=)(.*)', '\\1 %f' % (calc_tile.get_east_lon(lon, lat, dx)), line)
line = re.sub('^\s*(BOUNDARY_WEST\s*=)(.*)', '\\1 %f' % (calc_tile.get_west_lon(lon, lat, dx)), line)
line = re.sub('^\s*(BOUNDARY_NORTH\s*=)(.*)', '\\1 %f' % (calc_tile.get_north_lat(lat, dy)), line)
line = re.sub('^\s*(BOUNDARY_SOUTH\s*=)(.*)', '\\1 %f' % (calc_tile.get_south_lat(lat, dy)), line)
if not args.orig_osm_data:
line = re.sub('^\s*(OSM_FILE\s*=)(.*)', '\\1 "%s"' % OSM_FILE_NAME, line)
line = re.sub('^\s*(OSM_FILE\s*=)(.*)', '\\1 "%s"' % OSM_FILE_NAME, line)
sources.write(line)
if args.new_download:
download_command = '%s %s%sdownload_tile.py -f %s/params.ini -d "%s"' % (python_exe
, tools.get_osm2city_directory()
, os.sep
, path, OSM_FILE_NAME)
, tools.get_osm2city_directory()
, os.sep
, path, OSM_FILE_NAME)
download_file.write(download_command + os.linesep)
elif args.osmosis_executable:
download_file.write(osmosis_command % (calc_tile.get_north_lat(lat, dy)
, calc_tile.get_west_lon(lon, lat, dx)
, calc_tile.get_south_lat(lat, dy)
, calc_tile.get_east_lon(lon, lat, dx)
, replacement_path + os.sep + OSM_FILE_NAME))
else:
download_command = 'curl -f --retry 6 --proxy-ntlm -o %s/%s -g %s*[bbox=%f,%f,%f,%f] '
if BASH_PARALLEL_PROCESS:
......@@ -194,7 +220,7 @@ done
, calc_tile.get_east_lon(lon, lat, dx)
, calc_tile.get_north_lat(lat, dy)))
for command in files:
_write_to_file(command[0], command[1], python_exe)
_write_to_file(command[0], command[1], python_exe, replacement_path + os.sep + params_out_file_name)
for command in files:
command[1].close()
......
......@@ -72,7 +72,9 @@ The default work flow is based on the sub-chapters of :ref:`Preparation <chapter
#. Adapt ``params.ini``. This will get copied to several subdirectories as part of the next process steps. Most importantly apapt the parameter ``PATH_TO_OUTPUT`` (in the example below "/home/fg_customscenery/CH_OSM"). The ``PREFIX`` and ``BOUNDARY_*`` parameters will automatically be updated.
#. :ref:`Call build_tiles.py <chapter-build-tiles-label>`. This step creates sub-directories including a set of shell / command scripts. The top directory will be created in your ``WORKING_DIRECTORY`` and have the same name as the lon/lat area specified with argument ``-t``
#. If needed adapt the params.ini files in the sub-directories if you need to change specific characteristics within one tile (e.g. parameters for building height etc.). In most situations this will not be needed.
#. Call the generated scripts starting with ``download_xxxxx.sh`` and ``tiles_xxxxx.sh`` (depending on the chosen elevation probing mode) and then the rest.
#. Call the generated scripts starting with ``download_xxxxx.sh``. Make sure you are still in the correct working directory, because path names are relative.
#. Call ``tiles_xxxxx.sh`` depending on the chosen elevation probing mode
#. Call ``osm2city_xxxxx.sh``, ``osm2pylons_xxxxx.sh`` etc. depending on your requirements.
#. :ref:`Copy textures <chapter-copy-textures-label>`
......@@ -84,13 +86,12 @@ Calling build_tiles.py
::
/usr/bin/python2.7 /home/pingu/develop_vcs/osm2city/batch_processing/build_tiles.py -t e009n47 -f CH_OSM/params_kp.ini -o params.ini
$ /usr/bin/python2.7 /home/pingu/develop_vcs/osm2city/batch_processing/build_tiles.py -t e009n47 -f CH_OSM/params_kp.ini -o params.ini
Mandatory command line arguments:
* -t: the name of the 1-degree lon/lat-area, e.g. w003n60 or e012s06 (you need to provide 3 digits for longitude and 2 digits for latitude). The lon/lat position is the lower left corner (e.g. e009n47 to cover most of the Lake of Constance region).
* -t: the name of the 1-degree lon/lat-area, e.g. w003n60 or e012s06 (you need to provide 3 digits for longitude and 2 digits for latitude). The lon/lat position is the lower left corner (e.g. e009n47 to cover most of the Lake of Constance region in Europe).
* -f: the relative path to the main params.ini file, which is the template copied to all sub-directories.
* -o: the name of the copied params.ini files in the sub-directories
Optional command line arguments:
......@@ -98,7 +99,15 @@ Optional command line arguments:
* -u: Which API to use to download OSM data on the fly.
* -n: There are two implementations of downloading data on the fly. If this option is used, then a download program is used, which has better support for retries (FIXME: does this work?)
* -x: If ``python`` is not in your executable path or you want to specify a specific Python version if you have installed several versions, then use this argument (e.g. ``/usr/bin/python2.7``).
* -d: Use the OSM data file as specified in the overall ``params.ini`` instead of dynamic download. This can be used if e.g. ``curl`` is not available (mostly on Windows) or if you have problems with dynamic download. Be sure to set parameter ``BOUNDARY_CLIPPING=True`` and consider ``BOUNDARY_CLIPPING_COMPLETE_WAYS=True``. While this is a possibility, it is not recommended, because reading a large OSM-file repeatedly costs more time than downloading dynamically or even running a tool like Osmosis and manually copy the files into the sub-folders. This can be especially problematic if running parallel threads, all of which accessing the same file.
* -d: Instead of dynamic download an existing OSM data file as specified in the overall ``params.ini`` will be used. This can be used if e.g. ``curl`` is not available (mostly on Windows) or if you have problems with dynamic download or if you need to manipulate the OSM data after download and before processing. A pre-requisite for this is that you have Osmosis installed on your computer (see also :ref:`Getting OpenStreetMap Data <chapter-getting-data-label>`) — the path to the Osmosis executable needs to be specified with this command line argument.
* -o: the name of the copied params.ini files in the sub-directories
Calling build_tiles.py with optional argument ``-d`` could look like the following:
::
$ /usr/bin/python2.7 /home/pingu/develop_vcs/osm2city/batch_processing/build_tiles.py -t e009n47 -f CH_OSM/params.ini -o params.ini -x /usr/bin/python2.7 -d /home/pingu/bin/osmosis-latest/bin/osmosis
``build_tiles.py`` creates a directory layout like the following:
......@@ -108,7 +117,7 @@ Optional command line arguments:
fg_customscenery/
projects/
e000n40/
download_e009n47.sh
download_e009n47.sh # If option -d was chosen, then the commands within will call Osmosis and not download stuff
osm2city_e009n47.sh
osm2pylon_e009n47.sh
piers_e009n47.sh
......
No preview for this file type
......@@ -87,11 +87,14 @@ Be aware that ``osm2city`` only accepts OSM data in xml-format, i.e. ``*.osm`` f
c:\> "C:\FlightGear\osmosis-latest\bin\osmosis.bat" --read-pbf file="C:\FlightGear\fg_customscenery\raw_data\switzerland-latest.osm.pbf"
--bounding-box completeWays=yes top=46.7 left=9.75 bottom=46.4 right=10.0 --wx file="C:\FlightGear\fg_customscenery\projects\LSZS\lszs_wider.osm"
The exception to the requirement of using OSM data in xml-format is if you use batch processing with the optional ``-d`` command line argument (see :ref:`Calling build_tiles.py <chapter-build-tiles-label>`). In that situation you might want to consider using the pbf-format_.
Please be aware of the `Tile Index Schema`_ in FlightGear. It is advised to set boundaries, which do not cross tiles. Otherwise the scenery objects can jitter and disappear / re-appear due to the clusters of facades crossing tiles. Another reason to keep within boundaries is the sheer amount of data that needs to be kept in memory.
.. _information: http://wiki.openstreetmap.org/wiki/Downloading_data
.. _Osmosis: http://wiki.openstreetmap.org/wiki/Osmosis
.. _`Tile Index Schema`: http://wiki.flightgear.org/Tile_Index_Scheme
.. _pbf-format: http://wiki.openstreetmap.org/wiki/PBF_Format
.. _chapter-setting-parameters-label:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment