Commit a82b8121 authored by Hugh Brown's avatar Hugh Brown

Refactor PolarisConfig() class; change example config file to match

- Don't use OrderedDict for base class
- Break out properties for various things
- Set defaults in saner/more explicit way
- Use mergedeep module to properly merge nested dictionaries
Signed-off-by: Hugh Brown's avatarHugh Brown (Saint Aardvark the Carpeted) <[email protected]>
parent c7c29631
......@@ -2,34 +2,108 @@
"""
import json
from collections import OrderedDict
from mergedeep import merge
# Disabling check for public methods; the python_json_config class has
# all the methods we need, and we're explicitly deferring to it.
class PolarisConfig(OrderedDict): # pylint: disable=R0903
class PolarisConfig():
"""Class for Polaris configuration
"""
DEFAULT_CONFIGFILE = "polaris_config.json"
_DEFAULT_CONFIGFILE = 'polaris_config.json'
_DEFAULT_CACHEDIR = 'cache'
_DEFAULT_NORMALIZED_FILE = 'normalized_frames.json'
_DEFAULT_GRAPHDIR = 'graph'
_DEFAULT_OUTPUT_GRAPH_FILE = 'graph.json'
_DEFAULT_LOGDIR = 'log'
_DEFAULT_SETTINGS = {
"root_dir": "/tmp/polaris",
"cache_dir": "cache",
"graph_dir": "graph",
"log_dir": "log",
"batch": {
"learn": True,
"fetch": True,
"viz": False,
'file_layout': {
'root_dir': '/tmp/polaris',
},
'satellite': {
'batch': {
'learn': True,
'fetch': True,
'viz': False,
}
}
}
def __init__(self, file=DEFAULT_CONFIGFILE, defaults=None):
def __init__(self, file=_DEFAULT_CONFIGFILE, defaults=None):
"""Initialize Polaris configuration
"""
defaults = defaults or self._DEFAULT_SETTINGS
with open(file) as f_handle:
OrderedDict.__init__(self, defaults)
_data = json.load(f_handle)
OrderedDict.update(self, _data)
# data from file overrides the defaults
self._data = merge({}, defaults, json.load(f_handle))
@property
def root_dir(self):
"""Root directory for configuration
"""
return self._data['file_layout']['root_dir']
@root_dir.setter
def root_dir(self, value):
"""Set root directory
"""
self._data['file_layout']['root_dir'] = value
@property
def name(self):
"""Return name of satellite configuration
"""
return self._data['satellite']['name']
@name.setter
def name(self, value):
"""Set name of satellite configuration
"""
self._data['satellite']['name'] = value
@property
def cache_dir(self):
"""Return cache directory
"""
return f'{self.root_dir}/{self.name}/{self._DEFAULT_CACHEDIR}'
@property
def normalized_file_path(self):
"""Return path to satellite-specific normalized frames file
"""
# 'cache_dir': 'cache'
return f'{self.cache_dir}/{self._DEFAULT_NORMALIZED_FILE}'
@property
def graph_dir(self):
"""Return graph directory
"""
return f'{self.root_dir}/{self.name}/{self._DEFAULT_GRAPHDIR}'
@property
def output_graph_file(self):
"""Return path to graph file
"""
return f'{self.graph_dir}/{self._DEFAULT_OUTPUT_GRAPH_FILE}'
@property
def log_dir(self):
"""Return log directory
"""
return f'{self.root_dir}/{self._DEFAULT_LOGDIR}'
@property
def log_file(self):
"""Return path to log file
"""
return f'{self.log_dir}/{self.name}.log'
@property
def batch_settings(self):
"""Batch settings
"""
return self._data['satellite']['batch']
{
"default": {
"root_dir": "/tmp/polaris",
"cache_dir": "cache",
"graph_dir": "graph",
"log_dir": "log"
"file_layout": {
"root_dir": "/tmp/polaris"
},
"satellite": {
"lightsail2": {
"name": "LightSail-2",
"_comment": "Fields that begin with an underscore are ignored but preserved. _comment is suggested as the default way to include comments anywhere they might be needed.",
"batch": {
"fetch": true,
"learn": true,
"viz": false
}
"name": "LightSail-2",
"_comment": "Fields that begin with an underscore are ignored but preserved. _comment is suggested as the default way to include comments anywhere they might be needed.",
"batch": {
"fetch": true,
"learn": true,
"viz": false
}
}
}
......@@ -30,6 +30,7 @@ install_requires =
satnogs-decoders
fets
mlflow
mergedeep
[bdist_wheel]
universal = true
......
......@@ -2,6 +2,8 @@
Fixtures for pytest tests
"""
import json
import pytest
from polaris.dataset.dataset import PolarisDataset
......@@ -234,3 +236,41 @@ def pandas_dataset_dict():
"bat1_ctlflags": 0,
"time": 1563741837
}]
@pytest.fixture
def polaris_config():
"""Polaris configuration JSON to be used in testing
"""
return json.dumps({
'file_layout': {
'root_dir': '/tmp/polaris'
},
'satellite': {
'name': 'LightSail-2',
'_comment': 'This is a comment',
'batch': {
'fetch': True,
'learn': True,
'viz': False
}
}
})
@pytest.fixture
def polaris_config_defaults():
"""Polaris configuration defaults dict to be used in testing
"""
return {
'file_layout': {
'root_dir': '/default_root_dir'
},
'satellite': {
'batch': {
'fetch': True,
'learn': True,
'viz': False
}
}
}
......@@ -3,47 +3,69 @@
import json
import pytest
from polaris.common.config import PolarisConfig
def test_polaris_config_creation_non_existent_file(tmp_path):
"""Smoke test for object creation, non-existent file
def test_polaris_config_override_defaults(polaris_config_defaults, tmp_path):
"""Smoke test for overriding settings
"""
with pytest.raises(FileNotFoundError):
fullpath = tmp_path / 'does_not_exist.json'
_ = PolarisConfig(file=fullpath)
file_config = {
'file_layout': {
'root_dir': '/override'
},
'satellite': {
'name': 'AnotherSat',
'batch': {
'learn': False,
'viz': True
}
}
}
fullpath = tmp_path / 'simple_config.json'
with open(fullpath.as_posix(), 'w') as f_handle:
f_handle.write(json.dumps(file_config))
config_from_file = PolarisConfig(file=fullpath,
defaults=polaris_config_defaults)
def test_polaris_config_creation_simple_file(tmp_path):
"""Smoke test for object creation, simple config
"""
assert config_from_file.name == 'AnotherSat'
# The fetch setting is from the `polaris_config` fixture
assert config_from_file.batch_settings['fetch'] is True
assert config_from_file.batch_settings['learn'] is False
assert config_from_file.batch_settings['viz'] is True
assert config_from_file.root_dir == '/override'
simple_config = {'foo': 'bar', 'baz': 'bum', 'bling': 123}
def test_polaris_configuration_name(polaris_config, tmp_path):
"""Test getting name from satellite configuration
"""
fullpath = tmp_path / 'simple_config.json'
with open(fullpath.as_posix(), 'w') as f_handle:
json.dump(simple_config, f_handle)
f_handle.write(polaris_config)
config_from_file = PolarisConfig(file=fullpath)
for key in simple_config:
assert config_from_file[key] == simple_config[key]
assert config_from_file.name == "LightSail-2"
def test_polaris_config_creation_simple_file_plus_defaults(tmp_path):
"""Smoke test for overrriding settings
"""
defaults = {'default_to_override': 1, 'default_to_leave_alone': 2}
new_config = {'default_to_override': 'OVERRIDDEN', 'other_setting': 'foo'}
def test_polaris_configuration_root_dir(polaris_config, tmp_path):
"""Test getting name from satellite configuration
"""
fullpath = tmp_path / 'simple_config.json'
with open(fullpath.as_posix(), 'w') as f_handle:
json.dump(new_config, f_handle)
f_handle.write(polaris_config)
config_from_file = PolarisConfig(file=fullpath)
assert config_from_file.root_dir == '/tmp/polaris'
config_from_file = PolarisConfig(file=fullpath, defaults=defaults)
assert config_from_file['default_to_override'] == new_config[
'default_to_override']
assert config_from_file['default_to_leave_alone'] == defaults[
'default_to_leave_alone']
assert config_from_file['other_setting'] == new_config['other_setting']
def test_polaris_configuration_normalized_file_path(polaris_config, tmp_path):
"""Test getting name from satellite configuration
"""
fullpath = tmp_path / 'simple_config.json'
with open(fullpath.as_posix(), 'w') as f_handle:
f_handle.write(polaris_config)
config_from_file = PolarisConfig(file=fullpath)
expected_path = '/tmp/polaris/LightSail-2/cache/normalized_frames.json'
assert config_from_file.normalized_file_path == expected_path
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment