Overhauled config loader to allow split configs (f2a3261b) · Commits · OpenFlexure / openflexure-microscope-server

openflexure_microscope/api/app.py

+5 −8

Original line number	Diff line number	Diff line
		@@ -24,7 +24,7 @@ from flask_cors import CORS
		from openflexure_microscope.api.utilities import list_routes
		from openflexure_microscope.api.exceptions import JSONExceptionHandler

		from openflexure_microscope import Microscope, config
		from openflexure_microscope import Microscope
		from openflexure_microscope.exceptions import LockError
		from openflexure_microscope.camera.pi import StreamingCamera
		from openflexure_microscope.stage.openflexure import Stage
		@@ -35,10 +35,7 @@ import logging, sys
		logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)

		# Create a dummy microscope object, with no hardware attachments
		openflexurerc = config.load_config() # Load default user config
		api_microscope = Microscope(None, None, config=openflexurerc)
		logging.debug("Created an empty microscope in global.")

		api_microscope = Microscope(None, None)

		# Generate API URI based on version from filename
		def uri(suffix, api_version, base=None):
		@@ -61,11 +58,11 @@ handler = JSONExceptionHandler(app)
		@app.before_first_request
		def attach_microscope():
		# Create the microscope object globally (common to all spawned server threads)
		global api_microscope, openflexurerc
		global api_microscope
		logging.debug("First request made. Populating microscope with hardware...")

		logging.debug("Creating camera object...")
		api_camera = StreamingCamera(config=openflexurerc)
		api_camera = StreamingCamera(config=api_microscope.rc.config)

		logging.debug("Creating stage object...")
		api_stage = Stage("/dev/ttyUSB0")
		@@ -115,7 +112,7 @@ task_blueprint = blueprints.task.construct_blueprint(api_microscope)
		app.register_blueprint(task_blueprint, url_prefix=uri('/task', 'v1'))

		# List all routes
		list_routes(app)
		#list_routes(app)


		# Automatically clean up microscope at exit

openflexure_microscope/camera/base.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -97,7 +97,7 @@ class BaseCamera(object):
		self.stream_timeout_enabled = True #: bool: Enable or disable timing out the stream

		self.state = {} #: dict: Dictionary for capture state
		self._config = {} #: dict: Dictionary of base camera settings
		self._config = {} #: dict: Dictionary of base camera config
		self.paths = {
		'image': BASE_CAPTURE_PATH,
		'video': BASE_CAPTURE_PATH,

openflexure_microscope/camera/pi.py

+42 −68

Original line number	Diff line number	Diff line
		@@ -51,24 +51,28 @@ from .base import BaseCamera, CaptureObject
		# Richard's fix gain
		from .set_picamera_gain import set_analog_gain, set_digital_gain

		PICAMERA_KEYS = [
		'exposure_mode',
		'awb_mode',
		'awb_gains',
		'framerate',
		'shutter_speed',
		'saturation', ]

		CONFIG_KEYS = [
		'video_resolution',
		'image_resolution',
		'numpy_resolution',
		'jpeg_quality',
		'analog_gain',
		'digital_gain',
		'shading_table_path']
		# Handle config and picamera settings
		CONFIG_KEYS = {
		'video_resolution': (832, 624),
		'image_resolution': (2592, 1944),
		'numpy_resolution': (1312, 976),
		'jpeg_quality': 75,
		'picamera_settings': {
		'exposure_mode': None,
		'awb_mode': None,
		'awb_gains': None,
		'framerate': None,
		'shutter_speed': None,
		'saturation': None,
		'analog_gain': None,
		'digital_gain': None,
		'lens_shading_table': None,
		},
		}

		# Useful methods

		# TODO: Remove this
		def fractions_to_floats(value):
		"""Deal with horrible, horrible PiCamera fractions"""
		result = value
		@@ -81,6 +85,7 @@ def fractions_to_floats(value):
		result = float(value)
		return result

		# MAIN CLASS

		class StreamingCamera(BaseCamera):
		"""Raspberry Pi camera implementation of StreamingCamera.
		@@ -88,7 +93,9 @@ class StreamingCamera(BaseCamera):
		Args:
		config (dict): Dictionary of config parameters to apply on init. If None, will default to basic config.
		"""
		def __init__(self, config: dict=None):
		def __init__(self, config: dict=None, picamera_settings: dict=None):
		global CONFIG_KEYS

		# Run BaseCamera init
		BaseCamera.__init__(self)
		# Attach to Pi camera
		@@ -103,13 +110,8 @@ class StreamingCamera(BaseCamera):
		# Reset variable states
		self.set_zoom(1.0)

		# Default camera config
		self._config.update({
		'video_resolution': (832, 624),
		'image_resolution': (2592, 1944),
		'numpy_resolution': (1312, 976),
		'jpeg_quality': 75,
		})
		# Populate config and settings with all available keys
		self._config.update(CONFIG_KEYS)

		# Load config dictionary if passed
		if config:
		@@ -144,44 +146,27 @@ class StreamingCamera(BaseCamera):
		"""
		return hasattr(self.camera, "lens_shading_table")

		def apply_shading_table(self):
		if not self.supports_lens_shading:
		logging.error("the currently-installed picamera library does not support all "
		"the features of the openflexure microscope software. These features "
		"include lens shading control and setting the analog/digital gain.\n"
		"\n"
		"See the installation instructions for how to fix this:\n"
		"https://github.com/rwb27/openflexure_microscope_software")
		elif 'shading_table_path' not in self.config:
		logging.warning("No shading table path given in config.")
		else:
		logging.debug("Applying shading table from {}".format(self.config['shading_table_path']))
		npy = np.load(self.config['shading_table_path'])
		self.camera.lens_shading_table = npy

		@property
		def config(self):
		"""
		Return config dictionary of the StreamingCamera.
		"""
		global PICAMERA_KEYS, CONFIG_KEYS

		conf_dict = {
		'picamera_params': {},
		'picamera_settings': {},
		}

		# PiCamera parameters (obtained directly from PiCamera object)
		for key in PICAMERA_KEYS:
		for key, _ in self._config['picamera_settings'].items():
		try:
		value = getattr(self.camera, key)
		value = fractions_to_floats(value)
		conf_dict['picamera_params'][key] = value
		conf_dict['picamera_settings'][key] = value
		except AttributeError:
		logging.warning("Unable to read PiCamera attribute {}".format(key))

		# StreamingCamera parameters (obtained from StreamingCamera _config)
		for key in CONFIG_KEYS:
		if key in self._config:
		if (key != 'picamera_settings') and (key in self._config):
		conf_dict[key] = self._config[key]

		return conf_dict
		@@ -197,7 +182,6 @@ class StreamingCamera(BaseCamera):
		Args:
		config (dict): Dictionary of config parameters.
		"""
		global PICAMERA_KEYS, CONFIG_KEYS

		paused_stream = False

		@@ -216,27 +200,17 @@ class StreamingCamera(BaseCamera):
		paused_stream = True # Remember to unpause stream when done

		# PiCamera parameters (applied directly to PiCamera object)
		if 'picamera_params' in config:
		for key in PICAMERA_KEYS:
		if key in config['picamera_params']:
		logging.debug("Setting parameter {}: {}".format(key, config['picamera_params'][key]))
		setattr(self.camera, key, config['picamera_params'][key])
		if 'picamera_settings' in config: # If new settings are given
		for key, value in config['picamera_settings'].items(): # For each given setting
		self._config['picamera_settings'][key] = None # Add the key to the list of returned settings
		logging.debug("Setting parameter {}: {}".format(key, config['picamera_settings'][key]))
		setattr(self.camera, key, value) # Write setting to camera

		# StreamingCamera parameters (applied via StreamingCamera config)
		for key in CONFIG_KEYS:
		if key in config:
		logging.debug("Setting parameter {}: {}".format(key, config[key]))
		self._config[key] = config[key]

		# Richard's library to set analog and digital gains
		if 'analog_gain' in config:
		set_analog_gain(self.camera, config['analog_gain'])
		if 'digital_gain' in config:
		set_digital_gain(self.camera, config['digital_gain'])

		# Handle lens shading, if a new one is given
		if 'shading_table_path' in config:
		self.apply_shading_table()
		for key, value in config.items(): # For each provided setting
		if key != 'picamera_settings': # We already handled this
		logging.debug("Setting parameter {}: {}".format(key, value))
		self._config[key] = value

		# If stream was paused to update config, unpause
		if paused_stream:

openflexure_microscope/config.py

+154 −94

Original line number	Diff line number	Diff line
		import yaml
		import os
		import errno
		import logging
		import shutil
		import copy

		HERE = os.path.abspath(os.path.dirname(__file__))
		DEFAULT_CONFIG_PATH = os.path.join(HERE, 'microscoperc.default.yaml')
		@@ -9,73 +11,123 @@ DEFAULT_CONFIG_PATH = os.path.join(HERE, 'microscoperc.default.yaml')
		USER_CONFIG_DIR = os.path.join(os.path.expanduser("~"), ".openflexure") #: str: Default path of the user-config directory, containing runtime-config and calibration files. Obtained from ``os.path.join(os.path.expanduser("~"), ".openflexure")``.
		USER_CONFIG_FILE = os.path.join(USER_CONFIG_DIR, "microscoperc.yaml") #: str: Default path of the user microscoperc.yaml runtime-config file. Obtained from ``os.path.join(USER_CONFIG_DIR, "microscoperc.yaml")``

		TYPES = {
		'stream_resolution': tuple,
		'video_resolution': tuple,
		'image_resolution': tuple,
		'numpy_resolution': tuple,
		'jpeg_quality': int,
		'analog_gain': float,
		'digital_gain': float,
		} #: dict: Dictionary of expected types for common config parameters. Used by ``convert_config()``.
		with open(DEFAULT_CONFIG_PATH, 'r') as defaultrc:
		DEFAULT_CONFIG = defaultrc.read()


		def convert_config(config: dict) -> dict:
		"""
		Convert datatype of top-level config parameters based on the global TYPES dictionary.
		class OpenflexureConfig():
		expandable_keys = [
		'picamera_settings',
		'openflexure_stage_settings',
		] #: List of keys that can be passed as a file path string and expanded automatically

		Args:
		config (dict): Dictionary representation of a loaded runtime-config.
		"""
		global TYPES
		def __init__(self, config_path: str=None, expand: bool=True):
		global DEFAULT_CONFIG, USER_CONFIG_FILE

		# Set arguments
		self.config_path = config_path or USER_CONFIG_FILE
		self.expand = expand

		# Create empty config dictionaries
		self.raw_config = {}
		# Expanded config dictionary (used by server)
		self.config = copy.copy(self.raw_config)

		# Initialise basic config file with defaults if it doesn't exist
		self.initialise_file(self.config_path, populate=DEFAULT_CONFIG)

		# Load the config in, setting self._config and self.config
		self.load()


		def update(self, update_dict: dict):
		self.config.update(update_dict)

		for key in config:
		if key in TYPES:
		config[key] = TYPES[key](config[key])

		return config
		def overwrite(self, new_dict: dict):
		self.config = new_dict


		def load(self):
		# Unexpanded config dictionary (used at load/save time)
		self.raw_config = self.load_yaml_file(self.config_path)

		# If the loaded config is in contracted format
		if self.expand:
		# Expand self.raw_config into self._config
		self.expand_config()


		def save(self, backup: bool=True):
		# If the loaded config was in contracted format
		if self.expand:
		# Contract self._config into self.raw_config
		self.contract_config()

		if backup:
		if os.path.isfile(self.config_path):
		shutil.copyfile(self.config_path, self.config_path+".bk")

		self.save_yaml_file(self.config_path, self.raw_config)


		def load_config(config_path: str=None) -> dict:
		def expand_config(self):
		# For each value in the raw loaded config
		for key, value in self.raw_config.items():
		# If it's a valid expandable parameter
		if (key in OpenflexureConfig.expandable_keys and
		type(value) is str):

		logging.debug("Expanding {}".format(value))
		# Initialise, load and expand
		self.initialise_file(value)
		self.config[key] = self.load_yaml_file(value) or {}
		else:
		self.config[key] = value


		def contract_config(self):
		for key, value in self.config.items():
		# If it's a valid expandable parameter
		if (key in OpenflexureConfig.expandable_keys and
		type(value) is dict):

		# Create the file if it doesn't exist
		self.initialise_file(value)
		self.save_yaml_file(self.raw_config[key], value)
		else:
		self.raw_config = value


		def load_yaml_file(self, config_path) -> dict:
		"""
		Open openflexurerc.yaml runtime-config file and write to the microscope devices.
		Open a .yaml config file

		Args:
		config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
		"""
		global DEFAULT_CONFIG_PATH, USER_CONFIG_FILE

		if not config_path:
		if os.path.exists(USER_CONFIG_FILE): # If user config file already exists
		config_path = USER_CONFIG_FILE # Load it
		else: # If user config file doesn't yet exist
		logging.warning("No user config found. Loading system defaults...")
		if not os.path.exists(USER_CONFIG_DIR):
		logging.info("Making user config directory...")
		os.makedirs(USER_CONFIG_DIR)
		logging.info("Copying default config to user config...")
		shutil.copyfile(DEFAULT_CONFIG_PATH, USER_CONFIG_FILE)
		logging.info("Loading user config...")
		config_path = USER_CONFIG_FILE # Load defaults in
		config_path = os.path.expanduser(config_path)

		logging.info("Loading {}...".format(config_path))

		with open(config_path) as config_file:
		config_data = yaml.load(config_file)

		# Store config dictionary to self
		return convert_config(config_data)
		# Return loaded config dictionary
		return config_data


		def save_config(config_dict: dict, config_path: str=None, safe: bool=False):
		def save_yaml_file(self, config_path: str, config_dict: dict, safe: bool=False):
		"""
		Save current config dictionary to a YAML file.
		Save a .yaml config file

		Args:
		config_dict (dict): Dictionary of config data to save.
		config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
		config_path (str): Path to the config YAML file.
		"""
		global USER_CONFIG_FILE
		if not config_path:
		config_path = USER_CONFIG_FILE
		config_path = os.path.expanduser(config_path)

		logging.info("Saving {}...".format(config_path))

		with open(config_path, 'w') as outfile:
		if not safe:
		@@ -84,25 +136,33 @@ def save_config(config_dict: dict, config_path: str=None, safe: bool=False):
		yaml.safe_dump(config_dict, outfile)


		def merge_config(config_dict: dict, config_path: str=None, safe: bool=False, backup: bool=True):
		def initialise_file(self, config_path, populate: str=""):
		"""
		merge current config dictionary with an existing YAML file.
		Check if a file exists, and if not, create it
		and optionally populate it with content

		Args:
		config_dict (dict): Dictionary of config data to save.
		config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
		config_path (str): Path to the file.
		populate (str): String to dump to the file, if it is being newly created
		"""
		global USER_CONFIG_FILE
		if not config_path:
		config_path = USER_CONFIG_FILE
		config_path = os.path.expanduser(config_path)

		config_data = load_config(config_path=config_path)
		logging.debug("Initialising {}".format(config_path))
		logging.debug("Exists: {}".format(os.path.exists(config_path)))

		for key, value in config_dict.items():
		config_data[key] = value
		if not os.path.exists(config_path): # If user config file doesn't exist
		logging.warning("No config file found at {}. Creating...".format(config_path))
		self.create_file(config_path)

		logging.info("Populating {}...".format(config_path))
		with open(config_path, 'w') as outfile:
		outfile.write(populate)

		if backup:
		if os.path.isfile(config_path):
		shutil.copyfile(config_path, config_path+".bk")

		save_config(config_data, config_path=config_path, safe=safe)
		def create_file(self, config_path):
		if not os.path.exists(os.path.dirname(config_path)):
		try:
		os.makedirs(os.path.dirname(config_path))
		except OSError as exc: # Guard against race condition
		if exc.errno != errno.EEXIST:
		raise

openflexure_microscope/microscope.py

+25 −19

Original line number	Diff line number	Diff line
		@@ -11,10 +11,10 @@ from openflexure_stage import OpenFlexureStage
		from .camera.pi import StreamingCamera

		from .plugins import PluginMount
		from .config import load_config, save_config, convert_config
		from .utilities import axes_to_array
		from .task import TaskOrchestrator
		from .lock import CompositeLock
		from .config import OpenflexureConfig


		class Microscope(object):
		@@ -29,14 +29,14 @@ class Microscope(object):
		config (dict): Dictionary of the runtime config to apply to the microscope. Does not automatically apply to camera and stage.
		attach_plugins (bool): Should the microscope attach plugins listen in 'config' immediately.
		"""
		def __init__(self, camera: StreamingCamera, stage: OpenFlexureStage, config: dict={}, attach_plugins: bool=True):
		def __init__(self, camera: StreamingCamera, stage: OpenFlexureStage, config_path: str=None, attach_plugins: bool=True):

		# Create RC object
		self.rc = OpenflexureConfig(config_path=config_path, expand=True)

		# Attach initial hardware (may be NoneTypes)
		self.attach(camera, stage)

		# Store entire runtime-config
		self._config = config

		# Create a task orchestrator
		self.task = TaskOrchestrator() #: :py:class:`openflexure_microscope.task.TaskOrchestrator`: Threaded task orchestrator

		@@ -48,11 +48,11 @@ class Microscope(object):
		self.attach_plugins()

		# Set default parameters
		if not 'name' in self._config:
		self._config['name'] = uuid.uuid4().hex
		if not 'name' in self.rc.config:
		self.rc.config['name'] = uuid.uuid4().hex

		if not 'fov' in self._config:
		self._config['fov'] = [0, 0] # Assumes pi camera 2, and 40x objective
		if not 'fov' in self.rc.config:
		self.rc.config['fov'] = [0, 0]

		# Initialise with an empty composite lock
		self.lock = CompositeLock([]) #: :py:class:`openflexure_microscope.lock.CompositeLock`: Composite lock controlling thread access to multiple pieces of hardware
		@@ -73,7 +73,7 @@ class Microscope(object):
		if self.stage:
		self.stage.close()

		def attach(self, camera: StreamingCamera, stage: OpenFlexureStage, apply_config: bool=True):
		def attach(self, camera: StreamingCamera, stage: OpenFlexureStage):
		"""
		Retroactively attaches a camera and stage to the microscope object.

		@@ -128,8 +128,8 @@ class Microscope(object):
		"""
		Automatically search for plugin maps in self.config, and attach.
		"""
		if 'plugins' in self._config:
		self.attach_plugin_maps(self._config['plugins'])
		if 'plugins' in self.rc.config:
		self.attach_plugin_maps(self.rc.config['plugins'])
		else:
		logging.warning("No plugins specified in microscoperc.yaml. Skipping.")

		@@ -178,7 +178,7 @@ class Microscope(object):
		# If attached to a camera
		if self.camera:
		# Update camera config params from StreamingCamera object
		self._config.update(self.camera.config)
		self.rc.update(self.camera.config)

		# If attached to a stage
		if self.stage:
		@@ -186,13 +186,17 @@ class Microscope(object):
		else:
		backlash = [0, 0, 0]

		self._config['backlash'] = {
		backlash = {
		'backlash': {
		'x': backlash[0],
		'y': backlash[1],
		'z': backlash[2],
		}
		}

		self.rc.update(backlash)

		return self._config
		return self.rc.config

		@config.setter
		def config(self, config: dict) -> None:
		@@ -209,4 +213,6 @@ class Microscope(object):
		self.stage.backlash = backlash

		# Cache config
		self._config.update(config)
		No newline at end of file
		self.rc.update(config)
		# Save to disk
		self.rc.save()
		No newline at end of file