Extend the flexibility of the metrics interface (9a97920e) · Commits · Desupervised / Borch

src/borch/metrics/init.py

+2 −0

Original line number	Diff line number	Diff line
		@@ -10,5 +10,7 @@ from borch.metrics.rv_metrics import (
		mean_squared_error,
		accuracy,
		module_metrics,
		suggest_metric_fns,
		calculate_metric,
		)
		from borch.metrics import metrics

src/borch/metrics/metrics.py

+8 −6

Original line number	Diff line number	Diff line
		@@ -9,6 +9,7 @@ import torch

		# pylint: disable=unsubscriptable-object


		def r2_mean_diff_ratio(pred, target):
		"""Calculate r2

		@@ -27,7 +28,7 @@ def r2_mean_diff_ratio(pred, target):
		return pred_diff / target_diff


		def r2_general(pred, target):
		def r2_score(pred, target):
		"""Calculate r2 using the standard definition

		Args:
		@@ -42,6 +43,7 @@ def r2_general(pred, target):
		ss_tot = torch.sum((pred - pred.mean()) ** 2)
		return 1 - ss_res / ss_tot


		def mean_squared_error(pred, target):
		"""
		Measures the averaged element-wise mean squared error.
		@@ -106,7 +108,7 @@ def accuracy_logit(logits, target):
		return (target == logits.argmax(-1)).float().mean()


		def confusion_matrix(target, pred):
		def confusion_matrix(pred, target):
		"""Compute confusion matrix to evaluate the accuracy of a classification
		By definition a confusion matrix :math:`C` is such that :math:`C_{i, j}`
		is equal to the number of observations known to be in group :math:`i` but
		@@ -126,10 +128,10 @@ def confusion_matrix(target, pred):
		--------
		>>> y_true = [2, 0, 2, 2, 0, 1]
		>>> y_pred = [0, 0, 2, 2, 0, 2]
		>>> cm = confusion_matrix(y_true, y_pred)
		>>> cm = confusion_matrix(y_pred, y_true)
		>>> y_true = ["cat", "ant", "cat", "cat", "ant", "bird"]
		>>> y_pred = ["ant", "ant", "cat", "cat", "ant", "cat"]
		>>> cm = confusion_matrix(y_true, y_pred)
		>>> cm = confusion_matrix(y_pred, y_true)
		"""

		if isinstance(target, torch.Tensor):
		@@ -159,7 +161,7 @@ def confusion_matrix(target, pred):
		return conf_mat, labels


		def binary_roc_auc(targets, preds):
		def binary_roc_auc(preds, targets):
		"""Compute the Area under the Receiver Operating Characteristics curve
		for a binary classification case.

		@@ -174,7 +176,7 @@ def binary_roc_auc(targets, preds):
		--------
		>>> y_pred = [0.9, 0.1, 0.2, 0.8, 0.7, 0.6]
		>>> y_true = [ 0, 0, 1, 1, 0, 1]
		>>> binary_roc_auc(y_true, y_pred)
		>>> binary_roc_auc(y_pred, y_true)
		0.4444444444444444
		"""
		if not len(targets) == len(preds):

src/borch/metrics/rv_metrics.py

+50 −19

Original line number	Diff line number	Diff line
		@@ -16,22 +16,50 @@ from borch.metrics import metrics
		from borch.module import Module

		METRICS = {
		constraints.real.__class__: (metrics.mean_squared_error, metrics.r2_general,),
		constraints.greater_than: (metrics.mean_squared_error, metrics.r2_general,),
		constraints.real_vector.__class__: (metrics.mean_squared_error, metrics.r2_general,),
		constraints.greater_than_eq: (metrics.mean_squared_error, metrics.r2_general,),
		constraints.less_than: (metrics.mean_squared_error, metrics.r2_general,),
		constraints.integer_interval: (metrics.accuracy, metrics.r2_general,),
		constraints.real.__class__: (
		metrics.mean_squared_error,
		metrics.r2_score,
		),
		constraints.greater_than: (
		metrics.mean_squared_error,
		metrics.r2_score,
		),
		constraints.real_vector.__class__: (
		metrics.mean_squared_error,
		metrics.r2_score,
		),
		constraints.greater_than_eq: (
		metrics.mean_squared_error,
		metrics.r2_score,
		),
		constraints.less_than: (
		metrics.mean_squared_error,
		metrics.r2_score,
		),
		constraints.integer_interval: (metrics.accuracy,),
		constraints.positive_integer.__class__: (metrics.accuracy,),
		constraints.unit_interval.__class__: (metrics.mean_squared_error, metrics.r2_general,),
		constraints.unit_interval.__class__: (metrics.mean_squared_error,),
		}


		def _call_metric(rv, metric):
		return metric(rv.tensor, rv.distribution.sample())
		def suggest_metric_fns(rv):
		"""Suggest metric functions to use for an RV"""
		return METRICS.get(type(rv.support), [])


		def module_metrics(mod):
		def calculate_metric(fn, rv):
		"""Calculate metrics

		Calculate a metric for an observed RV

		Args:
		fn: callable that takes two arguments equivalent to `pred`, `target`
		rv: a random variable
		"""
		return fn(rv.distribution.sample(), rv.tensor)


		def module_metrics(mod, metrics_fns=None):
		"""
		Get the metrics of all observed RV`s in the module

		@@ -42,17 +70,18 @@ def module_metrics(mod):
		if not isinstance(mod, Module):
		return {}
		return {
		key: all_metrics(getattr(mod.posterior, key))
		key: all_metrics(getattr(mod.posterior, key), metrics_fns=metrics_fns)
		for key in dict(mod.observed.items())
		}

		def network_metrics():


		def all_metrics(rv):
		def all_metrics(rv, metrics_fns=None):
		"""
		Calculates all valid performance metrics of an observed RandomVariable,
		rv (borch.RandomVariable): an observed `RandomVariable`.
		metrics_fns (optional, callable, List[callable])]: list of metrics
		functions to calculate, or a callable that takes a rv as argument
		and returns a list of callable

		Returns:
		dict, with performance measures
		@@ -68,9 +97,11 @@ def all_metrics(rv):
		>>> met = all_metrics(rv)

		"""
		return {
		met.__name__: _call_metric(rv, met) for met in METRICS.get(type(rv.support), [])
		}
		if metrics_fns is None:
		metrics_fns = suggest_metric_fns
		if callable(metrics_fns):
		metrics_fns = metrics_fns(rv)
		return {met.__name__: calculate_metric(met, rv) for met in metrics_fns}


		def mean_squared_error(rv):
		@@ -87,7 +118,7 @@ def mean_squared_error(rv):
		>>> rv = distributions.Normal(torch.randn(10), torch.randn(10).exp())
		>>> mse = mean_squared_error(rv)
		"""
		return _call_metric(rv, metrics.mean_squared_error)
		return calculate_metric(metrics.mean_squared_error, rv)


		def accuracy(rv):
		@@ -109,4 +140,4 @@ def accuracy(rv):
		Notes:
		This function does not support gradient trough it
		"""
		return _call_metric(rv, metrics.accuracy)
		return calculate_metric(metrics.accuracy, rv)

tests/borch/metrics/test_metrics.py

+10 −10

Original line number	Diff line number	Diff line
		@@ -12,7 +12,7 @@ from borch.utils.torch_utils import get_device
		DEVICE = get_device()


		@pytest.mark.parametrize("fn", [metrics.r2_mean_diff_ratio, metrics.r2_general])
		@pytest.mark.parametrize("fn", [metrics.r2_mean_diff_ratio, metrics.r2_score])
		def test_r2(fn):
		x = torch.ones(100)
		x[0] = 0
		@@ -111,28 +111,28 @@ class Test_confusion_matrix(unittest.TestCase):
		def test_call_with_ints(self):
		y_true = [2, 0, 2, 2, 0, 1]
		y_pred = [0, 0, 2, 2, 0, 2]
		conf_mat, _ = metrics.confusion_matrix(y_true, y_pred)
		conf_mat, _ = metrics.confusion_matrix(y_pred, y_true)
		truth = np.array([[1, 0, 0], [0, 0, 1], [1 / 3, 0, 2 / 3]])
		self.assertTrue(np.array_equal(conf_mat, truth))

		def test_call_with_strings(self):
		y_true = ["cat", "ant", "cat", "cat", "ant", "bird"]
		y_pred = ["ant", "ant", "cat", "cat", "ant", "cat"]
		conf_mat, _ = metrics.confusion_matrix(y_true, y_pred)
		conf_mat, _ = metrics.confusion_matrix(y_pred, y_true)
		truth = np.array([[1, 0, 0], [0, 0, 1], [1 / 3, 0, 2 / 3]])
		self.assertTrue(np.array_equal(conf_mat, truth))

		def test_call_with_torch(self):
		y_true = torch.tensor([2, 0, 2, 2, 0, 1])
		y_pred = torch.tensor([0, 0, 2, 2, 0, 2])
		conf_mat, _ = metrics.confusion_matrix(y_true, y_pred)
		conf_mat, _ = metrics.confusion_matrix(y_pred, y_true)
		truth = np.array([[1, 0, 0], [0, 0, 1], [1 / 3, 0, 2 / 3]])
		self.assertTrue(np.array_equal(conf_mat, truth))


		class Test_binary_roc_auc(unittest.TestCase):
		def try_correct_calculation(self, y_pred, y_true):
		self.assertEqual(metrics.binary_roc_auc(y_true, y_pred), 0.4444444444444444)
		self.assertEqual(metrics.binary_roc_auc(y_pred, y_true), 0.4444444444444444)

		def test_correct_calculation_list(self):
		y_pred = [0.9, 0.1, 0.2, 0.8, 0.7, 0.6]
		@@ -155,14 +155,14 @@ class Test_binary_roc_auc(unittest.TestCase):

		def test_bad_targets(self):
		with self.assertRaises(RuntimeError):
		metrics.binary_roc_auc([0, 0, 0], [0.8, 0.9, 0.4])
		metrics.binary_roc_auc([0.8, 0.9, 0.4], [0, 0, 0])
		with self.assertRaises(RuntimeError):
		metrics.binary_roc_auc([1, 1, 1], [0.4, 0.2, 0])
		metrics.binary_roc_auc([0.4, 0.2, 0], [1, 1, 1])
		with self.assertRaises(RuntimeError):
		metrics.binary_roc_auc([0, 1, 0.2], [0.4, 0.2, 0])
		metrics.binary_roc_auc([0.4, 0.2, 0], [0, 1, 0.2])

		def test_bad_predictions(self):
		with self.assertRaises(ValueError):
		metrics.binary_roc_auc([0, 1, 0], [1.3, 0.9, 0.4])
		metrics.binary_roc_auc([1.3, 0.9, 0.4], [0, 1, 0])
		with self.assertRaises(ValueError):
		metrics.binary_roc_auc([1, 0, 1], [-0.2, 0.2, 0])
		metrics.binary_roc_auc([-0.2, 0.2, 0], [1, 0, 1])

tests/borch/test_convergance.py

+2 −0

Original line number	Diff line number	Diff line
		@@ -7,6 +7,7 @@ from torch import optim
		import borch
		from borch import distributions as dist, infer
		from borch.random_variable import RVPair
		from borch.utils.torch_utils import seed

		# pylint: disable=attribute-defined-outside-init,too-many-arguments

		@@ -169,6 +170,7 @@ def test_mlp_linear_regression():


		def test_linear_regression_with_transform():
		seed(1)
		device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

		class Model(borch.Module):