Commit 3749edee authored by Alan Taylor's avatar Alan Taylor

add generic support for rendering to different filetypes; add specific support for OpenEXR

parent a92d5d23
......@@ -440,6 +440,7 @@ def main():
'render_order': 'CENTRE', # the order the blocks are rendered in
'spatial_splits': False, # use spatial splits: longer build time, faster render
'heatmap': False, # flag indicating whether to generate a heatmap post render
'filetype': 'OPEN_EXR', # filetype to render to
# RESERVED SETTINGS
'blocks_x': -1, # number of blocks, x axis
......@@ -458,7 +459,7 @@ def main():
}
# check any command line arguments supplied by the user
cl_args = init.check_arguments()
cl_args = init.check_arguments(image_config)
# check if we are restarting from an earlier interrupted render
if os.path.isfile(ds.FILENAME_RESTART_CONFIG) and os.path.isfile(ds.FILENAME_RESTART_PROGRESS):
......@@ -517,7 +518,7 @@ def main():
fio.create_final_image_from_blocks(image_config)
ana.create_heatmap(image_config, checked)
fio.backup_render(image_config)
fio.tidy_up_temporary_files(available_render_nodes)
fio.tidy_up_temporary_files(available_render_nodes, image_config)
ana.display_basic_render_info(script_start, time_taken_by_previous_renders, checked)
......
......@@ -49,8 +49,6 @@ import pickle # dump, load
import subprocess # DEVNULL, Popen
import sys # exit
import PIL.Image # alpha_composite, open
import dtr_data_struct as ds # FILENAME_BENCHMARK_CACHE,
# FILENAME_CONFIG_BLENDER,
# FILENAME_RESTART_CONFIG,
......@@ -179,6 +177,15 @@ def _check_setting_order(settings, name, val):
if val in ['CENTRE', 'BOTTOM_TO_TOP']:
settings[name] = val
def _check_setting_filetype(settings, name, val):
"""
the filetype to render to
provides a service to read_user_options()
"""
if val in ['PNG', 'OPEN_EXR']:
settings[name] = val
def read_user_options(render_nodes, settings):
"""
read user set render options from a configuration file on disk
......@@ -212,6 +219,7 @@ def read_user_options(render_nodes, settings):
'image_y': _check_setting_int,
'seed': _check_setting_int,
'frame': _check_setting_int,
'filetype': _check_setting_filetype,
'blocks_user': _check_setting_int,
'textures_directory': _check_setting_directory,
'library_directory': _check_setting_directory,
......@@ -378,6 +386,34 @@ render_data_read = functools.partial(_data_read, filename=ds.FILENAME_RENDER_DAT
#
##############################################################################
def filetype_defaults(settings):
"""
generate configuration values for user specified render filetype
--------------------------------------------------------------------------
args
settings : image_config dictionary
contains core information about the image to be rendered
--------------------------------------------------------------------------
returns : list
--------------------------------------------------------------------------
"""
ft_defaults = {\
'OPEN_EXR': [\
'scene.render.image_settings.file_format="OPEN_EXR"',\
'scene.render.image_settings.color_depth="16"',\
'scene.render.image_settings.color_mode="RGBA"',\
'scene.render.image_settings.exr_codec="ZIP"'],\
'PNG': [\
'scene.render.image_settings.file_format="PNG"',
'scene.render.image_settings.color_mode="RGBA"',
'scene.render.image_settings.compression=100']}
config = ft_defaults.get(settings['filetype'], None)
assert config != None, 'ASSERT: unrecognised filetype in filetype_defaults'
return config
def _write_blender_python_config(settings, block_number):
"""
write configuration python script to instruct blender to render a single
......@@ -397,7 +433,7 @@ def _write_blender_python_config(settings, block_number):
--------------------------------------------------------------------------
"""
block_coords = utils.render_block_normal(settings, block_number)
commands = '\n'.join([
part1 = [
'import bpy',
'scene=bpy.context.scene',
# enable border rendering so we can define the area of the block to be
......@@ -433,17 +469,16 @@ def _write_blender_python_config(settings, block_number):
'scene.render.resolution_x=' + str(settings['image_x']),
'scene.render.resolution_y=' + str(settings['image_y']),
'scene.render.resolution_percentage=100',
# set filetype
'scene.render.image_settings.file_format="PNG"',
'scene.render.image_settings.color_mode="RGBA"',
'scene.render.image_settings.compression=100',
# set frame
'scene.frame_set(' + str(settings['frame']) + ')',
# post processing should always be suppressed for block renders
'scene.render.use_compositing=False',
'scene.render.use_sequencer=False',
'scene.render.use_sequencer=False']
part2 = filetype_defaults(settings)
part3 = [
# instruction to render the image
'bpy.ops.render.render(write_still=True)'])
'bpy.ops.render.render(write_still=True)']
commands = '\n'.join(part1 + part2 + part3)
filename = ds.FILENAME_CONFIG_BLENDER + str(block_number).zfill(settings['padding']) + '.py'
with open(filename, 'w') as configuration:
......@@ -563,7 +598,7 @@ def backup_render(settings, initial_test=False):
def create_final_image_from_blocks(settings):
"""
stitch the rendered blocks together to form the final image
combine the rendered blocks together to form the final image
--------------------------------------------------------------------------
args
......@@ -575,41 +610,51 @@ def create_final_image_from_blocks(settings):
"""
print('>> compositing')
# stitch tiles together
tmp = PIL.Image.open(utils.block_render_filename(settings, 1))
if settings['blocks_required'] > 1:
for i in range(2, settings['blocks_required'] + 1):
image = PIL.Image.open(utils.block_render_filename(settings, i))
tmp = PIL.Image.alpha_composite(tmp, image)
extension = utils.file_extension(settings)
filename = 'composite_seed_' + str(settings['seed']) + '.' + extension
composite = 'convert -layers flatten -compress zip block_*_seed_*.' + extension + ' ' + filename
# the shell is needed to parse the wildcards
with subprocess.Popen(composite, shell=True, \
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) as proc:
proc.wait()
if proc.returncode == 0:
if settings['remove_alpha']:
strip_alpha = 'convert -alpha off -compress zip ' + filename + ' ' + filename
# discard alpha channel if requested
if settings['remove_alpha']:
tmp = tmp.convert("RGB")
# the shell is needed to parse the wildcards
with subprocess.Popen(strip_alpha, shell=True, \
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) as proc:
proc.wait()
# save final image
filename = 'composite_seed_' + str(settings['seed']) + '.png'
tmp.save(filename)
print('>> result written to: ' + filename)
print('>> result written to: ' + filename)
else:
print('>> result write to: ' + filename + ' failed')
##############################################################################
# tidy up
##############################################################################
def tidy_up_restart(full_clear=False):
def tidy_up_restart(settings, full_clear=False):
"""
on the local machine, remove temporary files associated with a
successfully completed, or previously interrupted render
--------------------------------------------------------------------------
args : none
args
settings : image_config dictionary
contains core information about the image to be rendered
full_clear : bool
flag to indicate whether to clear heatmap related data
--------------------------------------------------------------------------
returns : none
--------------------------------------------------------------------------
"""
command = [ \
'rm -f', \
'block_*_seed_*.png', \
'block_*_seed_*.' + utils.file_extension(settings), \
ds.FILENAME_CONFIG_BLENDER + '*.py', \
ds.FILENAME_RESTART_CONFIG, \
ds.FILENAME_RESTART_PROGRESS]
......@@ -624,7 +669,7 @@ def tidy_up_restart(full_clear=False):
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) as proc:
proc.wait()
def _tidy_files(node):
def _tidy_files(settings, node):
"""
tidy up files on a remote node
......@@ -638,6 +683,8 @@ def _tidy_files(node):
--------------------------------------------------------------------------
args
settings : image_config dictionary
contains core information about the image to be rendered
node : instance of RenderNode
contains information about the node we will send a block to
--------------------------------------------------------------------------
......@@ -651,7 +698,7 @@ def _tidy_files(node):
on_this_node = node.username + '@' + node.ip_address
delete_files = ' '.join([\
'rm -f', \
'block_*_seed_*.png', \
'block_*_seed_*.' + utils.file_extension(settings), \
'0001.png', \
ds.FILENAME_CONFIG_BLENDER + '*.py'])
......@@ -661,7 +708,7 @@ def _tidy_files(node):
return resp, node.ip_address
def tidy_up_temporary_files(render_nodes):
def tidy_up_temporary_files(nodes, settings):
"""
remove temporary files on local machine and remote nodes
......@@ -672,8 +719,10 @@ def tidy_up_temporary_files(render_nodes):
--------------------------------------------------------------------------
args
render_nodes : list of RenderNode instances
nodes : list of RenderNode instances
contains information about all the nodes in the cluster
settings : image_config dictionary
contains core information about the image to be rendered
--------------------------------------------------------------------------
returns : none
--------------------------------------------------------------------------
......@@ -687,13 +736,13 @@ def tidy_up_temporary_files(render_nodes):
num_threads = 5
with cf.ThreadPoolExecutor(max_workers=num_threads) as executor:
delete_status = {executor.submit(_tidy_files, node): node for node in render_nodes}
delete_status = {executor.submit(_tidy_files, settings, node): node for node in nodes}
for status in cf.as_completed(delete_status):
tidy_failed, node_ip = status.result()
if tidy_failed:
print('problem deleting temporary files on', node_ip)
# tidy up locally
tidy_up_restart()
tidy_up_restart(settings)
print('>> finished')
......@@ -337,8 +337,8 @@ def _block_layout_2(settings, min_blocks):
settings['blocks_x'], settings['blocks_y'] = found[0]
return False
else:
return True
return True
def _block_layout_3(settings, min_blocks):
"""
......@@ -420,8 +420,6 @@ def _block_layout(settings, min_blocks):
--------------------------------------------------------------------------
args
arn : list of RenderNode instances
contains information about all the nodes in the cluster
settings : image_config dictionary
contains core information about the image to be rendered
min_blocks : int
......@@ -798,7 +796,7 @@ def _value_range(val):
raise argparse.ArgumentTypeError(str(val) + ' not in range 0.0 - 1.0')
return val
def check_arguments():
def check_arguments(settings):
"""
handle command line options
......@@ -806,7 +804,9 @@ def check_arguments():
be deferred until image settings have been finalised
--------------------------------------------------------------------------
args : none
args
settings : image_config dictionary
contains core information about the image to be rendered
--------------------------------------------------------------------------
returns
args : user command line arguments as parsed by argparse
......@@ -842,7 +842,7 @@ def check_arguments():
fio.flush_cache()
if args.clean or args.flush:
fio.tidy_up_restart(full_clear=True)
fio.tidy_up_restart(settings, full_clear=True)
# sanity check only, handle deletion in restart_interrupted_render()
if args.delete:
......
......@@ -8,6 +8,7 @@ callable functions from this file:
check_node
check_python_version
despatch_order
file_extension
node_alive
no_dupes_ipu
package_filename_for_cygwin
......@@ -271,8 +272,8 @@ def package_filename_for_cygwin(node_op_sys, nix_filename):
"""
if 'CYGWIN' not in node_op_sys:
return nix_filename
else:
return '\"$(cygpath -aw \"' + nix_filename + '\")\"'
return '\"$(cygpath -aw \"' + nix_filename + '\")\"'
def block_render_filename(settings, block):
"""
......@@ -291,7 +292,7 @@ def block_render_filename(settings, block):
--------------------------------------------------------------------------
"""
return 'block_' + str(block).zfill(len(str(settings['blocks_required']))) + \
'_seed_' + str(settings['seed']) + '.png'
'_seed_' + str(settings['seed']) + '.' + file_extension(settings)
def node_alive(r_node):
"""
......@@ -543,6 +544,21 @@ def remove_dead_nodes(available_nodes, remove=True):
if not available_nodes:
sys.exit('exiting, as no viable render nodes were found')
def file_extension(settings):
"""
generate file extension for chosen filetype
--------------------------------------------------------------------------
args
settings : image_config dictionary
contains core information about the image to be rendered
--------------------------------------------------------------------------
returns
string containing file extension
--------------------------------------------------------------------------
"""
return {'OPEN_EXR': 'exr', 'PNG': 'png'}[settings['filetype']]
##############################################################################
# set the order that blocks are despatched to be rendered
......
......@@ -7,8 +7,8 @@ It is free and [Open Source](https://opensource.org/definition) software, and is
The design mindset was to create a script that:
* was fault tolerant (in terms of user error, network outages, crashed nodes)
* would run on any operating system
* would use render nodes of any operating system
* would run on any operating system supported by Blender
* would use render nodes of any operating system supported by Blender
* would self-configure to get as close to an optimally-efficient distributed render as possible
If you're looking to get a distributed render up and running very quickly and don't need all the features here, try using [GNU Parallel](https://www.gnu.org/software/parallel/) and [ImageMagick](https://imagemagick.org/script/convert.php) together, see [here](https://gitlab.com/skororu/scripts) for some examples.
......@@ -26,7 +26,7 @@ If you're looking to get a distributed render up and running very quickly and do
## Please note:
* renders to PNG
* renders to OpenEXR (default) and PNG
* post processing (compositing and sequencer) will be disabled by the script
---
......@@ -64,6 +64,7 @@ Check the [wiki](https://gitlab.com/skororu/dtr/wikis/branches) for more details
|[Blender](https://www.blender.org/)|no|yes|no|
|[Python](https://www.python.org)|yes|no|no|
|[Pillow](https://python-pillow.github.io)|yes|no|no|
|[ImageMagick](https://imagemagick.org/script/convert.php)|yes|no|no|
|[Rsync](https://rsync.samba.org/)|yes|yes|yes|
### (1) Python requirements
......@@ -389,7 +390,7 @@ tile 0 retrieved from 127.0.0.1
Files:
benchmark_cache.p : holds cached benchmark results (so the script doesn't have to benchmark nodes on every run)
bench.blend : file rendered when benchmarking remote nodes
composite_seed_*.png : final rendered image
composite_seed_*.FFF : final rendered image (file extension FFF as specified by user)
dtr.py : run this file to perform the render
dtr_benchmark.py : functions that support benchmarking the remote nodes
dtr_data_struct.py : data structures used by the script
......@@ -402,14 +403,14 @@ Files:
user_settings.conf : contains all user configuration options (this is the only user editable file - do not edit any other files)
Temporary files created on the local machine:
block_*_seed_*.png : individual block renders
block_*_seed_*.FFF : individual block renders (file extension FFF as specified by user)
render_block.py : file used to configure Blender to render a specific block
restart_config.p : holds the key render configuration data, in case we need to interrupt and restart the script
restart_progress.p : holds details about the work completed so far, in case we need to interrupt and restart the script
Temporary files created on remote machines:
0001.png : output from benchmark render (not used for anything)
block_*_seed_*.png : individual block renders
block_*_seed_*.FFF : individual block renders (file extension FFF as specified by user)
render_block_*.py : file used to configure Blender to render a specific block
```
......
......@@ -88,6 +88,15 @@ image_y = 256
#
#remove_alpha = True
##############################################################################
# specify filetype to render images to
#
# default value if unset is an OPEN_EXR
# all options are:
# OPEN_EXR, PNG
#
#filetype=PNG
##############################################################################
# list render nodes below, one per line
# either ip addresses or locally recognised host names are fine
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment