Commit 6bf37a09 authored by Joel Collins's avatar Joel Collins
Browse files

Threadpool for disk IO, greenlets for metadata

parent 7c9f42f7

openflexure_microscope/captures/capture.py

+17 −20
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ import atexit 


from gevent.fileobject import FileObjectThread

import gevent

import threading



from collections import OrderedDict
@@ -132,8 +133,6 @@ class CaptureObject(object): 
        # Event to notify when the stream has finished writing to disk

        #self.file_ready = Event()

        self.file_ready = gevent.event.Event()

        # Lock to control disk file access

        self.file_lock = gevent.lock.BoundedSemaphore()



        # Store a nice ID

        self.id = uuid.uuid4()  #: str: Unique capture ID
@@ -164,7 +163,7 @@ class CaptureObject(object): 


    def _stream_to_file(self):

        logging.info(f"Writing to disk {self.file}")

        with FileObjectThread(open(self.file, "wb"), 'wb')  as outfile, self.file_lock:

        with FileObjectThread(open(self.file, "wb"), 'wb')  as outfile:

            outfile.write(self.stream.getbuffer())

        self.stream.close()

        self.file_ready.set()
@@ -173,7 +172,7 @@ class CaptureObject(object): 


    def flush(self):

        logging.debug(f"Flushing {self.file}")

        gevent.spawn(self._stream_to_file)

        gevent.get_hub().threadpool.spawn(self._stream_to_file)

        gevent.sleep()

        logging.debug(f"Returning flushing {self.file}")
@@ -250,8 +249,7 @@ class CaptureObject(object): 
        self.save_metadata()



    def save_metadata(self) -> None:

        #gevent.get_hub().threadpool.spawn(self.synchronous_save_metadata)

        gevent.spawn(self.synchronous_save_metadata)

        gevent.get_hub().threadpool.spawn(self.synchronous_save_metadata)

        gevent.sleep()



    def synchronous_save_metadata(self) -> None:
@@ -261,7 +259,6 @@ class CaptureObject(object): 
        global EXIF_FORMATS



        if self.format.upper() in EXIF_FORMATS and self.exists:

            with self.file_lock:

            logging.debug("Writing exif data to capture file")

            # Extract current Exif data

            exif_dict = piexif.load(self.file)
@@ -321,7 +318,7 @@ class CaptureObject(object): 


        if self.exists:  # If data file exists

            logging.info("Opening from file {}".format(self.file))

            with open(self.file, "rb") as f, self.file_lock:

            with open(self.file, "rb") as f:

                d = io.BytesIO(f.read())  # Load bytes from file

            d.seek(0)  # Rewind loaded bytestream

            # Create a copy of the bytestream bytes

openflexure_microscope/microscope.py

+20 −21
Original line number Diff line number Diff line
@@ -288,6 +288,21 @@ class Microscope: 


        return system_metadata



    def add_metadata_to_capture(output, metadata, annotations, tags):

        logging.debug(f"Waiting for {output.file}")

        # Wait for the file to be written to disk

        output.file_ready.wait()

        logging.info(f"Asynchronously injecting EXIF data into {output.file}")

        # Inject system metadata

        output.put_metadata({"instrument": self.metadata})

        # Insert custom metadata

        output.put_metadata(metadata)

        # Insert custom metadata

        output.put_annotations(annotations)

        # Insert custom tags

        output.put_tags(tags)

        logging.info(f"Finished injecting EXIF data into {output.file}")



    def capture(

        self,

        filename: str = None,
@@ -324,26 +339,10 @@ class Microscope: 
                bayer=bayer,

                fmt=fmt,

            )

            logging.info("Finished microscope capture...")





        def inject_metadata():

            logging.debug(f"Waiting for {output.file}")

            # Wait for the file to be written to disk

            output.file_ready.wait()

            logging.info(f"Asynchronously injecting EXIF data into {output.file}")

            # Inject system metadata

            output.put_metadata({"instrument": self.metadata})

            # Insert custom metadata

            output.put_metadata(metadata)

            # Insert custom metadata

            output.put_annotations(annotations)

            # Insert custom tags

            output.put_tags(tags)

            logging.info(f"Finished injecting EXIF data into {output.file}")



        gevent.spawn(inject_metadata)

        # Gether metadata from hardware in a greenlet

        gevent.spawn(self.add_metadata_to_capture, output, metadata, annotations, tags)



        logging.debug(f"Finished capture to {output.file}")

        logging.info(f"Finished capture to {output.file}")



        return output