Commit 0a86768b authored by Rémi Huguet's avatar Rémi Huguet
Browse files

feat: add memory usage estimation check at first distinguisher update

parent 7fe1014e

.recipe/meta.yaml

+1 −1
Original line number Diff line number Diff line
@@ -22,7 +22,7 @@ requirements: 
    - python

    - numpy

    - estraces



    - psutil

about:

  home: https://gitlab.com/eshard/scared

  license: GNU LGPL V3

scared/analysis.py

+1 −1
Original line number Diff line number Diff line
@@ -58,6 +58,7 @@ class _BaseAnalysis: 


        batch_size = self._compute_batch_size(container.batch_size)

        logger.info(f'Starting run on container {container}, with batch size {batch_size}.')



        for i, batch in enumerate(container.batches(batch_size=batch_size)):

            logger.info(f'Process batch number {i} starting.')

            self.process(batch)
@@ -93,7 +94,6 @@ class _BaseAnalysis: 
        intermediate_values = self.compute_intermediate_values(traces_batch.metadatas)



        logger.info(f'Will call distinguisher update with {traces_batch}.')



        self.update(

            data=intermediate_values,

            traces=traces_batch.samples

scared/distinguishers/base.py

+24 −0
Original line number Diff line number Diff line 
import abc

import numpy as _np

import logging

import psutil



logger = logging.getLogger(__name__)
@@ -42,8 +43,12 @@ class DistinguisherMixin(abc.ABC): 
        except AttributeError:

            logger.info(f'Initialize distinguisher state.')

            self._origin_shape = o_shape

            mem = psutil.virtual_memory().available / 2 ** 30

            logger.info(f'Memory usage before compute {mem} GB.')

            self._initialize(traces=traces, data=data)



        self._check(traces=traces, data=data)



        self.processed_traces += traces.shape[0]

        logger.info(f'Will call _update traces.')

        self._update(traces=traces, data=data)
@@ -57,6 +62,8 @@ class DistinguisherMixin(abc.ABC): 
        pass



    def compute(self):

        mem = psutil.virtual_memory().available / 2 ** 30

        logger.info(f'Memory usage before compute {mem} GB.')

        try:

            assert self.processed_traces > 0

        except (AttributeError, AssertionError):
@@ -68,6 +75,22 @@ class DistinguisherMixin(abc.ABC): 
    def _compute(self):

        pass



    def _check(self, traces, data):

        if not self._is_checked:

            data_dim = data.shape[1]

            dtype_size = _np.dtype(self.precision).itemsize

            needed_mem = dtype_size * data_dim * self._memory_usage_coefficient(trace_size=traces.shape[1]) / 2 ** 30

            available_mem = psutil.virtual_memory().available / 2 ** 30

            logger.info(f'Needed memory estimated to {needed_mem} GB, for available {available_mem}.')

            self._is_checked = True

            if needed_mem > 0.9 * available_mem:

                raise MemoryError(

                    f'This analysis will probably need more than 90% of your available memory - {available_mem} GB available against {needed_mem} GB needed.'

                )



    def _memory_usage_coefficient(self, trace_size):

        return 2 * trace_size





def _set_precision(obj, precision):

    try:
@@ -83,6 +106,7 @@ def _set_precision(obj, precision): 
def _initialize_distinguisher(obj, precision, processed_traces):

    _set_precision(obj, precision)

    obj.processed_traces = processed_traces

    obj._is_checked = False





class _StandaloneDistinguisher:

scared/distinguishers/mia.py

+3 −0
Original line number Diff line number Diff line
@@ -16,6 +16,9 @@ def _is_valid_bin_edges(bin_edges): 
class MIADistinguisherMixin(_PartitionnedDistinguisherBaseMixin):

    """This partitioned distinguisher mixin applies a mutual information computation."""



    def _memory_usage_coefficient(self, trace_size):

        return 3 * len(self.partitions) * self.bins_number * trace_size



    @property

    def bin_edges(self):

        return self._bin_edges

scared/distinguishers/partitioned.py

+4 −0
Original line number Diff line number Diff line
@@ -6,6 +6,10 @@ logger = logging.getLogger(__name__) 




class _PartitionnedDistinguisherBaseMixin(DistinguisherMixin):



    def _memory_usage_coefficient(self, trace_size):

        return 3 * len(self.partitions) * trace_size



    def _initialize(self, traces, data):

        maxdata = _np.nanmax(data)

        if self.partitions is None: