Commit 9aa3645a authored by Paul Shved's avatar Paul Shved

Add a hacky profile for the training and feature generationy

parent dc512356
import time
class AggregatedTimer(object):
def __init__(self, name="Timer", print_every=100):
self.total_timings = 0.0
self.total_runs = 0
self.name = name
self.print_every = print_every
def __str__(self):
return "Average time for {} is {:.4f} sec".format(self.name, self.avg())
def avg(self):
if self.total_runs == 0:
return 0
else:
return self.total_timings / self.total_runs
def start(self):
self.last_timer = time.perf_counter()
def stop(self):
t = time.perf_counter()
self.total_timings += t - self.last_timer
self.total_runs += 1
if self.total_runs % self.print_every == 0:
print(self)
......@@ -6,6 +6,8 @@ import os
from tqdm import tqdm
import tensorflow as tf
from learning.timer import AggregatedTimer
def and_not_test(fn):
def ant(source, row):
return not row.get('test', False) and fn(source,row)
......@@ -39,6 +41,8 @@ class Provider(object):
self.keys = self.make_shuffled_keys(self.db(), shuffle=False)
#self.normalize_and_shuffle()
self.feature_timer = AggregatedTimer(name="Features")
def db(self):
if self._cached_db: return self._cached_db
self._cached_db = self.truths.filtered_db(self.truths_filter)
......@@ -134,6 +138,7 @@ class Provider(object):
self.processed_boxes = {}
def generate_images():
for entry in dataset:
self.feature_timer.start()
key = decode_str(entry['key'].numpy())
original_image_key = self.original_image_key(key)
......@@ -171,6 +176,7 @@ class Provider(object):
entry['y_mask'] = gtf['y_mask']
entry['num_matched_boxes'] = gtf['num_matched_boxes']
self.feature_timer.stop()
yield entry
# Sadly, we have a choice here: either use map's parallelization, or write the numpy_function junk but have everything called serially...
......
......@@ -15,6 +15,7 @@ import timeit
from tensorboard import summary as summary_lib
from learning.timer import AggregatedTimer
from smilebot import augmentations, detections, features, gpu, metrics, focal_loss, profile
from smilebot.models import alexnet_like, simplified_alexnet, ssd_detector
from csvdb import truths
......@@ -310,11 +311,12 @@ class SmileBotTrainer(object):
# Shuffle again. This time we only need enough images to load up the GPU.
SHUFFLE_BUFFER_SIZE = 16 * 256
self.train_dataset = self.train_dataset.shuffle(SHUFFLE_BUFFER_SIZE, reshuffle_each_iteration=True).prefetch(32 * 256)
#self.train_dataset = self.train_dataset.shuffle(SHUFFLE_BUFFER_SIZE, reshuffle_each_iteration=True).prefetch(1000)
self.train_dataset = self.train_dataset.prefetch(1000)
# Use validation dataset for testing.
#self.test_dataset = tf.data.Dataset.from_tensor_slices((self.X_test, self.y_test))
self.test_dataset = p.load_detections(p.keys_as_dataset(split='val'), gt_features=feature_converter)
self.test_dataset = p.load_detections(p.keys_as_dataset(split='val'), gt_features=feature_converter).prefetch(1000)
......@@ -673,6 +675,7 @@ class SmileBotTrainer(object):
eval_set_size = 0
# Training loop
train_timer = AggregatedTimer(name="Training", print_every=100)
for epoch in range(num_epochs):
# Reset the metrics at the start of the next epoch
for metric in resettables:
......@@ -683,7 +686,10 @@ class SmileBotTrainer(object):
# The features are merged into this dict, and it's set in the load_decoder_train_test_set
gt_features = entry
train_timer.start()
train_step(images, gt_features)
train_timer.stop()
train_step_count += 1
with summary_writer.as_default():
tf.summary.scalar('train_loss', train_loss.result(), step=train_step_count)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment