Optimised captures by using greenlets for disk IO (7c9f42f7) · Commits · OpenFlexure / openflexure-microscope-server

openflexure_microscope/api/app.py

+8 −1

Original line number	Diff line number	Diff line
		@@ -3,13 +3,20 @@ from gevent import monkey

		monkey.patch_all()

		import sys
		import time
		import atexit
		import logging, logging.handlers

		# Look for debug flag
		if "-d" in sys.argv or "--debug" in sys.argv:
		log_level = logging.DEBUG
		else:
		log_level = logging.INFO

		# Set root logger level
		logger = logging.getLogger()
		logger.setLevel(logging.INFO)
		logger.setLevel(log_level)

		import os
		import pkg_resources

openflexure_microscope/camera/mock.py

+5 −10

Original line number	Diff line number	Diff line
		@@ -197,19 +197,14 @@ class MissingCamera(BaseCamera):
		bayer (bool): Store raw bayer data in capture
		"""

		if isinstance(output, CaptureObject):
		target = output.file
		else:
		target = output

		with self.lock:
		if isinstance(target, str):
		target = open(target, "wb")
		if isinstance(output, str):
		output = open(output, "wb")

		target.write(self.stream.getvalue())
		output.write(self.stream.getvalue())

		if isinstance(target, str):
		target.close()
		if isinstance(output, str):
		output.close()

		# HANDLE STREAM FRAMES

openflexure_microscope/camera/pi.py

+3 −14

Original line number	Diff line number	Diff line
		@@ -115,11 +115,6 @@ class PiCameraStreamer(BaseCamera):
		"picamera_lst.npy"
		) #: str: Path of .npy lens shading table file

		# Create an empty stream
		self.stream = io.BytesIO()

		# Start streaming
		self.start_worker()

		@property
		def configuration(self):
		@@ -520,12 +515,6 @@ class PiCameraStreamer(BaseCamera):
		Returns:
		output_object (str/BytesIO): Target object.
		"""

		if isinstance(output, CaptureObject):
		target = output.file
		else:
		target = output

		with self.lock:
		logging.info("Capturing to {}".format(output))

		@@ -535,20 +524,20 @@ class PiCameraStreamer(BaseCamera):
		time.sleep(0.1)

		self.camera.capture(
		target,
		output,
		format=fmt,
		quality=100,
		resize=resize,
		bayer=(not use_video_port) and bayer,
		use_video_port=use_video_port,
		)
		time.sleep(0.1)
		#time.sleep(0.1)

		# Set resolution and start stream recording if necessary
		if not use_video_port:
		self.start_stream_recording()

		return target
		return output

		def yuv(
		self, use_video_port: bool = True, resize: Tuple[int, int] = None

openflexure_microscope/captures/capture.py

+49 −14

Original line number	Diff line number	Diff line
		@@ -10,6 +10,9 @@ from PIL import Image
		import dateutil.parser
		import atexit

		from gevent.fileobject import FileObjectThread
		import gevent

		from collections import OrderedDict

		from openflexure_microscope.camera import piexif
		@@ -118,12 +121,19 @@ def capture_from_exif(path, exif_dict):

		class CaptureObject(object):
		"""
		StreamObject used to store and process on-disk capture data, and metadata.
		File-like object used to store and process on-disk capture data, and metadata.
		Serves to simplify modifying properties of on-disk capture data.
		"""

		def __init__(self, filepath) -> None:
		"""Create a new StreamObject, to manage capture data."""
		# Stream for buffering capture data
		self.stream = io.BytesIO()
		# Event to notify when the stream has finished writing to disk
		#self.file_ready = Event()
		self.file_ready = gevent.event.Event()
		# Lock to control disk file access
		self.file_lock = gevent.lock.BoundedSemaphore()

		# Store a nice ID
		self.id = uuid.uuid4() #: str: Unique capture ID
		@@ -148,6 +158,25 @@ class CaptureObject(object):
		# Thumbnail (populated only for PIL captures)
		self.thumb_bytes = None

		def write(self, s):
		self.stream.write(s)
		gevent.sleep()

		def _stream_to_file(self):
		logging.info(f"Writing to disk {self.file}")
		with FileObjectThread(open(self.file, "wb"), 'wb') as outfile, self.file_lock:
		outfile.write(self.stream.getbuffer())
		self.stream.close()
		self.file_ready.set()
		logging.info(f"Finished writing to disk {self.file}")
		gevent.sleep()

		def flush(self):
		logging.debug(f"Flushing {self.file}")
		gevent.spawn(self._stream_to_file)
		gevent.sleep()
		logging.debug(f"Returning flushing {self.file}")

		def open(self, mode):
		return open(self.file, mode)

		@@ -221,12 +250,18 @@ class CaptureObject(object):
		self.save_metadata()

		def save_metadata(self) -> None:
		#gevent.get_hub().threadpool.spawn(self.synchronous_save_metadata)
		gevent.spawn(self.synchronous_save_metadata)
		gevent.sleep()

		def synchronous_save_metadata(self) -> None:
		"""
		Save metadata to exif, if supported
		"""
		global EXIF_FORMATS

		if self.format.upper() in EXIF_FORMATS and self.exists:
		with self.file_lock:
		logging.debug("Writing exif data to capture file")
		# Extract current Exif data
		exif_dict = piexif.load(self.file)
		@@ -286,7 +321,7 @@ class CaptureObject(object):

		if self.exists: # If data file exists
		logging.info("Opening from file {}".format(self.file))
		with open(self.file, "rb") as f:
		with open(self.file, "rb") as f, self.file_lock:
		d = io.BytesIO(f.read()) # Load bytes from file
		d.seek(0) # Rewind loaded bytestream
		# Create a copy of the bytestream bytes

openflexure_microscope/microscope.py

+17 −1

Original line number	Diff line number	Diff line
		@@ -7,6 +7,8 @@ import pkg_resources
		import uuid
		from typing import Tuple

		import gevent

		from openflexure_microscope.captures import CaptureManager

		from openflexure_microscope.stage.mock import MissingStage
		@@ -299,6 +301,7 @@ class Microscope:
		tags: list = None,
		metadata: dict = None,
		):
		logging.debug(f"Microscope capturing to {filename}")
		if not annotations:
		annotations = {}
		if not metadata:
		@@ -313,14 +316,22 @@ class Microscope:
		)

		# Capture to output object
		logging.info("Starting microscope capture...")
		self.camera.capture(
		output.file,
		output,
		use_video_port=use_video_port,
		resize=resize,
		bayer=bayer,
		fmt=fmt,
		)
		logging.info("Finished microscope capture...")


		def inject_metadata():
		logging.debug(f"Waiting for {output.file}")
		# Wait for the file to be written to disk
		output.file_ready.wait()
		logging.info(f"Asynchronously injecting EXIF data into {output.file}")
		# Inject system metadata
		output.put_metadata({"instrument": self.metadata})
		# Insert custom metadata
		@@ -329,5 +340,10 @@ class Microscope:
		output.put_annotations(annotations)
		# Insert custom tags
		output.put_tags(tags)
		logging.info(f"Finished injecting EXIF data into {output.file}")

		gevent.spawn(inject_metadata)

		logging.debug(f"Finished capture to {output.file}")

		return output