Commit 015055b6 authored by Alan Taylor's avatar Alan Taylor

merge experimental branch

parent 4df6e03f
This diff is collapsed.
This diff is collapsed.
......@@ -31,12 +31,13 @@ import datetime # timedelta
import hashlib # sha256
import multiprocessing as mp # cpu_count, Queue
import os # path.isdir, path.isfile
import subprocess # call
import subprocess # DEVNULL, Popen
import sys # exit
import time # time
import dtr_data_struct as ds # FILENAME_BENCHMARK, FILENAME_BENCHMARK_CACHE
import dtr_file_io as fio # backup_render, benchmark_cache_read, benchmark_cache_write
import dtr_file_io as fio # backup_render, benchmark_cache_read,
# benchmark_cache_write
import dtr_utils as utils # package_filename_for_cygwin
......@@ -216,16 +217,17 @@ def _benchmark_render_node(rni):
# transfer benchmark Blender file to remote node
on_this_node = rni.username + '@' + rni.ip_address
remote_filename = on_this_node + ':~'
subprocess.call(['rsync', '-acq', ds.FILENAME_BENCHMARK, remote_filename], \
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
with subprocess.Popen(['rsync', '-acq', ds.FILENAME_BENCHMARK, remote_filename], \
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) as proc:
proc.wait()
filename = utils.package_filename_for_cygwin(rni.os, ds.FILENAME_BENCHMARK)
filename = utils.package_filename_for_cygwin(rni.opsys, ds.FILENAME_BENCHMARK)
render_the_benchmark = rni.binloc + ' -b ' + filename + ' -f 1'
start = time.time()
# need to use subprocess.call here (blocking necessary)
subprocess.call(['ssh', on_this_node, render_the_benchmark], \
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
with subprocess.Popen(['ssh', on_this_node, render_the_benchmark], \
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) as proc:
proc.wait()
time_diff = int(time.time() - start)
return rni.ip_address, time_diff
......
......@@ -25,13 +25,15 @@ along with dtr. If not, see <http://www.gnu.org/licenses/>.
------------------------------------------------------------------------------
"""
import signal # signal
import multiprocessing as mp # Queue
import signal # signal
FILENAME_BENCHMARK = 'bench.blend'
FILENAME_BENCHMARK_CACHE = 'benchmark_cache.p'
FILENAME_CONFIG_BLENDER = 'render_block_'
FILENAME_FOR_RENDER = 'render.blend'
FILENAME_RENDER_DATA = 'block_data.p'
FILENAME_RESTART_CONFIG = 'restart_config.p'
FILENAME_RESTART_PROGRESS = 'restart_progress.p'
FILENAME_USER_SETTINGS = 'user_settings.conf'
......@@ -39,21 +41,26 @@ FILENAME_USER_SETTINGS = 'user_settings.conf'
class RenderNode:
"""define nodes available to render tiles"""
def __init__(self, ip_address, username='render', os='', binloc='', mean_duration=0.0, num_blocks=0, block_time_start=0.0, block_in_progress=0, first_use=True):
def __init__(self, ip_address, username='render', opsys='', binloc='', first_use=True):
self.ip_address = ip_address # ip address of node
self.username = username # username relating to the ip_address
self.os = os # operating system of the node is determined at runtime
self.opsys = opsys # operating system of the node is determined at runtime
self.binloc = binloc # location of Blender binary
self.mean_duration = mean_duration # the mean completion time of all tiles rendered by this node
self.num_blocks = num_blocks # the number of blocks rendered so far
self.block_time_start = block_time_start # the time the current block started rendering
self.block_in_progress = block_in_progress # the number of the block currrently being rendered by this node
self.first_use = first_use # flag to indicate whether the node has been used before (should we copy files to it)
self.first_use = first_use # flag to indicate if the node has received any blocks before
# using the equality test to establish whether we have duplicate nodes
def __eq__(self, other):
return self.ip_address == other.ip_address
class Production:
"""Queues to support render pipeline"""
blocks = mp.Queue()
lock = mp.Lock()
collect = mp.Queue()
check = mp.Queue()
terminate = mp.Queue()
terminate_info = mp.Queue()
inform = mp.Queue()
class UserExit:
"""handle user interrupts gracefully"""
......
This diff is collapsed.
This diff is collapsed.
......@@ -3,11 +3,17 @@ utilities
callable functions from this file:
area_in_blocks
block_render_filename
check_node
check_python_version
despatch_order
node_alive
no_dupes_ipu
package_filename_for_cygwin
render_block_normal
render_block_pillow
tidy_checked
------------------------------------------------------------------------------
Copyright 2015-2017 Alan Taylor
......@@ -28,9 +34,8 @@ You should have received a copy of the GNU General Public License
along with dtr. If not, see <http://www.gnu.org/licenses/>.
------------------------------------------------------------------------------
"""
import concurrent.futures as cf # ThreadPoolExecutor
import math # hypot
import math # floor, hypot
import multiprocessing as mp # cpu_count, Queue
import socket # AF_INET, SOCK_STREAM, socket
import subprocess # PIPE, Popen
......@@ -107,6 +112,69 @@ def render_block_normal(settings, block_number):
return (x_min, y_min, x_max, y_max)
def area_in_blocks(settings, cl_args):
"""
yield a series of blocks that will fully contain the area specified
expected coordinate system: bottom left is 0.0, 0.0 top right is 1.0, 1.0
--------------------------------------------------------------------------
args
settings : image_config dictionary
contains core information about the image to be rendered
cl_args : list [min_x, max_x, min_y, max_y]
where:
min_x : int, value between 0.0 and 1.0
max_x : int, value between 0.0 and 1.0
min_y : int, value between 0.0 and 1.0
max_y : int, value between 0.0 and 1.0
--------------------------------------------------------------------------
yields
generated series of ints, block numbers
--------------------------------------------------------------------------
"""
min_x, max_x, min_y, max_y = cl_args
blk_siz_x = 1 / settings['blocks_x']
blk_siz_y = 1 / settings['blocks_y']
col_st = math.floor(min_x / blk_siz_x)
col_en = math.floor(max_x / blk_siz_x)
row_st = math.floor(min_y / blk_siz_y)
row_en = math.floor(max_y / blk_siz_y)
for row in range(row_st, row_en + 1):
for col in range(col_st, col_en + 1):
yield row * settings['blocks_x'] + col + 1
def render_block_pillow(settings, block_number):
"""
a version of render_block_normal() that generates coordinates compatible
with Pillow (Python Imaging Library) where (0, 0) is at the top left
--------------------------------------------------------------------------
args
settings : image_config dictionary
contains core information about the image to be rendered
block_number : int
block number to be rendered
e.g. for a render with 8 blocks, blocks would be numbered
{1, 2, 3, 4, 5, 6, 7, 8}
--------------------------------------------------------------------------
returns
tuple (float, float, float, float)
lower and upper bounds of the given block in both axes
--------------------------------------------------------------------------
"""
x_min, y_min, x_max, y_max = render_block_normal(settings, block_number)
x_min = int(x_min * settings['image_x'])
y_min = int((1 - y_min) * settings['image_y'])
x_max = int(x_max * settings['image_x'])
y_max = int((1 - y_max) * settings['image_y'])
return x_min, y_min, x_max, y_max
def _dist_from_centre(settings, block_number):
"""
return the distance between the centre of the image and the
......@@ -160,6 +228,30 @@ def no_dupes_ipu(original):
ndup -= 1
o_ind -= 1
def tidy_checked(checked, missing):
"""
remove all traces of the missing blocks from the record, making sure
duplicates are removed too
--------------------------------------------------------------------------
args
checked : list of lists
[[x, y, z], ...]
where:
x = string, ip address
y = int, block number
z = int, time it took to render block in seconds
missing : list of ints
where each int represents a block render that was missing
--------------------------------------------------------------------------
returns : none
--------------------------------------------------------------------------
"""
for missing_block in missing:
all_matching_blocks = [x for x in checked if x[1] == missing_block]
for rendered_block in all_matching_blocks:
checked.remove(rendered_block)
def package_filename_for_cygwin(node_op_sys, nix_filename):
"""
Windows applications will expect to receive Windows filenames
......@@ -201,7 +293,7 @@ def block_render_filename(settings, block):
return 'block_' + str(block).zfill(len(str(settings['blocks_required']))) + \
'_seed_' + str(settings['seed']) + '.png'
def _node_alive(r_node):
def node_alive(r_node):
"""
establish whether the node is up by attempting to connect to port 22,
as this port is typically used for ssh connections which this script
......@@ -270,6 +362,21 @@ def _modern_blender_executable(node, test_blender):
return True
def check_python_version():
"""
simple check that the version of Python running this script is
3.4 or later, and exit if it is older
--------------------------------------------------------------------------
args : none
--------------------------------------------------------------------------
returns : none
--------------------------------------------------------------------------
"""
if sys.version_info < (3, 4):
py_vers = sys.version.partition(' ')[0]
sys.exit('this script requires Python 3.4 or later (' + py_vers + ' detected), exiting')
def _locate_blender(rni):
"""
find the location of the Blender binary on the remote render node
......@@ -296,7 +403,7 @@ def _locate_blender(rni):
'Darwin': '~/Blender/blender.app/Contents/MacOS/blender',
'Linux': '~/Blender/blender',
'CYGWIN' : '~/Blender/blender.exe'}
b_loc = blender_binloc.get(rni.os, '~/Blender/blender.exe')
b_loc = blender_binloc.get(rni.opsys, '~/Blender/blender.exe')
find_blender = 'if [ ! -f ' + b_loc + ' ]; then echo "MISSING"; fi'
find_response = _remote_command(rni, find_blender)
if 'MISSING' not in find_response:
......@@ -336,7 +443,7 @@ def _remote_command(node, execute_command):
return stdo.decode('utf-8', 'ignore').rstrip()
def _check_node(node):
def check_node(node):
"""
determine whether the node is up with a basic connection check,
only then if we get a reply should we try to run a test command using ssh
......@@ -359,14 +466,14 @@ def _check_node(node):
node_up = True
binloc = ''
if _node_alive(node):
if node_alive(node):
# node appears to be up and responsive (check 1 passed)
uname_response = _remote_command(node, 'uname')
if uname_response:
# successfully ran a command over ssh (check 2 passed)
node.os = 'CYGWIN' if 'CYGWIN' in uname_response else uname_response
node.opsys = 'CYGWIN' if 'CYGWIN' in uname_response else uname_response
blender_location = _locate_blender(node)
if blender_location:
......@@ -388,7 +495,7 @@ def _check_node(node):
return node, node_up, binloc
def remove_dead_nodes(available_nodes):
def remove_dead_nodes(available_nodes, remove=True):
"""
determine whether the node is up with a basic connection check,
only then if we get a reply should we try to run a test command using ssh
......@@ -404,6 +511,9 @@ def remove_dead_nodes(available_nodes):
args
available_nodes : list of RenderNode instances
contains information about all the nodes in the cluster
remove : bool
used indicate during a restarted render that we wish to warn the
user that a node is down, without actually removing it permanently
--------------------------------------------------------------------------
returns
available_nodes : list of RenderNode instances
......@@ -417,18 +527,17 @@ def remove_dead_nodes(available_nodes):
num_threads = 5
with cf.ThreadPoolExecutor(max_workers=num_threads) as executor:
node_status = {executor.submit(_check_node, anode): anode for anode in available_nodes}
node_status = {executor.submit(check_node, anode): anode for anode in available_nodes}
for status in cf.as_completed(node_status):
node, node_up, binloc = status.result()
if node_up:
node.binloc = binloc
else:
elif remove:
available_nodes.remove(node)
# if no nodes are up, there is no point continuing
if not available_nodes:
print('exiting, as none of the given render nodes appear to be responsive')
sys.exit()
sys.exit('exiting, as none of the given render nodes appear to be responsive')
##############################################################################
......@@ -441,7 +550,7 @@ def _do_bottom_to_top(ic_settings, blocks):
provides a service to despatch_order()
"""
blocks.sort(reverse=True)
blocks.sort()
def _do_centre(ic_settings, blocks):
"""
......@@ -449,7 +558,7 @@ def _do_centre(ic_settings, blocks):
provides a service to despatch_order()
"""
blocks.sort(key=lambda x: _dist_from_centre(ic_settings, x), reverse=True)
blocks.sort(key=lambda x: _dist_from_centre(ic_settings, x))
def despatch_order(ic_settings, blocks):
"""
......
......@@ -23,6 +23,10 @@ associated with a previously interrupted render.
.TP
.BR "-f, --flush"
Flush the benchmark cache, this option implies \fB--clean\fR.
.TP
.BR "-m, --map"
Generate a heatmap after the render has completed, to see which
areas of the image are taking longest to render.
.SH FILES
.BR user_settings.conf
.RS
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment