...
 
Commits (4)
......@@ -5,6 +5,7 @@ include LICENSE
include README.md
include requirements.txt
include requirements_dev.txt
include config.yml
recursive-include tests *
recursive-include requirements *
......
......@@ -75,6 +75,9 @@ setup(
'sirbot': 'sirbot',
'plugins': 'sirbot/plugins',
},
package_data={
'sirbot': ['config.yml']
},
# To provide executable scripts, use entry points in preference to the
# "scripts" keyword. Entry points provide cross-platform support and
# allow pip to create the appropriate form of executable for the
......
......@@ -10,7 +10,7 @@ from sirbot import SirBot
def parse_args(arguments):
parser = argparse.ArgumentParser(description='The good Sir-bot-a-lot')
parser.add_argument('-p', '--port', dest='port', action='store',
parser.add_argument('-P', '--port', dest='port', action='store',
type=int,
help='port where to run sirbot')
parser.add_argument('-c', '--config', action='store',
......@@ -18,22 +18,65 @@ def parse_args(arguments):
parser.add_argument('-u', '--update', help='Run update of plugins'
'if necessary',
action='store_true', dest='update')
parser.add_argument('-p', '--plugins', help='Plugins to load',
dest='plugins', nargs='+')
return parser.parse_args(arguments)
def load_config(path=None):
if path:
with open(path) as file:
return yaml.load(file)
return {}
if not path:
return dict()
if not os.path.isabs(path):
path = os.path.join(os.getcwd(), path)
def start(config, loop=None): # pragma: no cover
with open(path) as file:
return yaml.load(file)
def cli_plugin(args, config):
if args.plugins:
try:
config['sirbot']['plugins'].extend(args.plugins)
except KeyError:
if 'sirbot' not in config:
config['sirbot'] = {'plugins': []}
elif 'plugins' not in config['sirbot']:
config['sirbot']['plugins'] = list()
config['sirbot']['plugins'] = args.plugins
return config
def main(): # pragma: no cover
args = parse_args(sys.argv[1:])
logging.basicConfig()
config_file = args.config or os.getenv('SIRBOT_CONFIG')
config = load_config(config_file)
config = cli_plugin(args, config)
try:
port = args.port or config['sirbot']['port']
except KeyError:
port = 8080
try:
if args.update:
update(config)
else:
start(config, port=port)
except Exception as e:
raise
def start(config, port, loop=None): # pragma: no cover
if not loop:
loop = asyncio.get_event_loop()
bot = SirBot(config=config, loop=loop)
bot.run(port=int(config.get('port', 8080)))
bot.run(port=int(port))
return bot
......@@ -41,25 +84,9 @@ def update(config, loop=None):
if not loop:
loop = asyncio.get_event_loop()
bot = SirBot(config=config, loop=loop)
loop.run_until_complete(bot.update())
return bot
def main(): # pragma: no cover
args = parse_args(sys.argv[1:])
logging.basicConfig()
config_file = os.getenv('SIRBOT_CONFIG', args.config)
config = load_config(config_file)
config['port'] = os.getenv('SIRBOT_PORT') or args.port or config.get(
'port') or 8080
if args.update:
update(config)
else:
start(config)
if __name__ == '__main__':
main() # pragma: no cover
sirbot:
port: 8080
plugins: []
import asyncio
import functools
import importlib
import logging
import logging.config
from collections import defaultdict
from typing import Optional
import os
import aiohttp
import pluggy
import yaml
from aiohttp import web
from collections import defaultdict
from typing import Optional
from sirbot.utils import error_callback
from . import hookspecs
from .facade import MainFacade
from .utils import merge_dict
logger = logging.getLogger('sirbot.core')
class SirBot:
def __init__(self, config=None, *,
def __init__(self, config=None, debug=False, *,
loop: Optional[asyncio.AbstractEventLoop] = None):
self.config = config or {}
self._debug = debug
self._configure()
logger.info('Initializing Sir-bot-a-lot')
......@@ -31,7 +33,6 @@ class SirBot:
self._pm = None
self._session = aiohttp.ClientSession(loop=self._loop)
self._plugins = dict()
self._configure_future = None
self._start_priority = defaultdict(list)
self._facades = dict()
......@@ -44,7 +45,9 @@ class SirBot:
self._initialize_plugins()
self._registering_facades()
self._configure_plugins()
self._loop.run_until_complete(self._configure_plugins())
logger.info('Sir-bot-a-lot Initialized')
def _configure(self) -> None:
......@@ -54,6 +57,15 @@ class SirBot:
:return: None
"""
path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'config.yml'
)
with open(path) as file:
defaultconfig = yaml.load(file)
self.config = merge_dict(self.config, defaultconfig)
if 'logging' in self.config:
logging.config.dictConfig(self.config['logging'])
else:
......@@ -64,8 +76,6 @@ class SirBot:
Startup tasks
"""
logger.info('Starting Sir-bot-a-lot ...')
if self._configure_future:
await self._configure_future
await self._start_plugins()
logger.info('Sir-bot-a-lot fully started')
......@@ -92,18 +102,20 @@ class SirBot:
if plugins:
for plugin in plugins:
name = plugin.__name__
facade = getattr(plugin, '__facade__', name)
config = self.config.get(name, {})
priority = config.get('priority', True)
priority = config.get('priority', 50)
if priority:
self._plugins[name] = {'plugin': plugin,
'config': config,
'priority': priority
}
if type(priority) == int:
self._start_priority[priority].append(name)
elif priority:
self._start_priority[50].append(name)
self._plugins[name] = {
'plugin': plugin,
'config': config,
'priority': priority,
'facade': facade
}
self._start_priority[priority].append(name)
else:
logger.error('No plugins found')
......@@ -113,56 +125,47 @@ class SirBot:
if info['priority']:
plugin_facade = getattr(info['plugin'], 'facade', None)
if callable(plugin_facade):
self._facades[name] = info['plugin'].facade
self._facades[info['facade']] = info['plugin'].facade
def _configure_plugins(self) -> None:
funcs = list()
async def _configure_plugins(self) -> None:
logger.debug('Configuring plugins')
for name, info in self._plugins.items():
if info['priority']:
funcs.append(
info['plugin'].configure(
config=info['config'],
session=self._session,
facades=MainFacade(self._facades),
router=self.app.router
)
)
funcs = [
info['plugin'].configure(
config=info['config'],
session=self._session,
facades=MainFacade(self._facades),
router=self.app.router
)
for info in self._plugins.values()
]
if funcs:
self._configure_future = asyncio.wait(
funcs,
return_when=asyncio.ALL_COMPLETED,
loop=self._loop
)
if not self._loop.is_running():
self._loop.run_until_complete(self._configure_future)
await asyncio.gather(*funcs, loop=self._loop)
logger.debug('Plugins configured')
async def _start_plugins(self) -> None:
logger.debug('Starting plugins')
callback = functools.partial(error_callback, logger=logger)
max_start_time = 4
for priority in sorted(self._start_priority, reverse=True):
elapsed_time = 0
logger.debug('Starting plugins %s',
', '.join(self._start_priority[priority]))
logger.debug(
'Starting plugins %s',
', '.join(self._start_priority[priority])
)
for name in self._start_priority[priority]:
plugin = self._plugins[name]
self._tasks[name] = self._loop.create_task(
plugin['plugin'].start())
self._tasks[name].add_done_callback(callback)
plugin['plugin'].start()
)
while not all(self._plugins[name]['plugin'].started
for name in self._tasks):
while not all(self._plugins[name]['plugin'].started for name in
self._start_priority[priority]):
for task in self._tasks.values():
if task.done():
task.result()
await asyncio.sleep(0.2, loop=self._loop)
elapsed_time += 0.2
if elapsed_time >= max_start_time:
logger.warning('Timeout while starting one of %s',
', '.join(self._start_priority[priority]))
break
else:
logger.debug('Plugins %s started',
', '.join(self._start_priority[priority]))
......@@ -177,10 +180,9 @@ class SirBot:
self._pm = pluggy.PluginManager('sirbot')
self._pm.add_hookspecs(hookspecs)
if 'core' in self.config and 'plugins' in self.config['core']:
for plugin in self.config['core']['plugins']:
p = importlib.import_module(plugin)
self._pm.register(p)
for plugin in self.config['sirbot']['plugins']:
p = importlib.import_module(plugin)
self._pm.register(p)
async def _middleware_factory(self, app, handler):
async def middleware_handler(request):
......
......@@ -2,6 +2,10 @@ from abc import ABC
class Plugin(ABC): # pragma: no cover
__version__ = '0.0.1'
__name__ = 'test'
__facade__ = 'test'
def __init__(self, loop):
"""
Method called at the bot initialization.
......
......@@ -21,3 +21,20 @@ def error_callback(f, logger):
if error is not None:
logger.exception("Task exited with error",
exc_info=error)
def merge_dict(a, b, path=None):
"""
Merge dict b into a
"""
if not path:
path = []
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
merge_dict(a[key], b[key], path + [str(key)])
else:
pass
else:
a[key] = b[key]
return a
......@@ -14,7 +14,7 @@ def test_argument_parser_config():
def test_argument_parser_port():
args = cli.parse_args(['-p', '4567'])
args = cli.parse_args(['-P', '4567'])
assert args.port == 4567
......
......@@ -10,6 +10,6 @@ logging:
sirbot:
level: DEBUG
handlers: [console]
core:
sirbot:
plugins:
- tests.test_plugin.sirbot
\ No newline at end of file
- tests.test_plugin.sirbot
......@@ -17,7 +17,7 @@ from sirbot.facade import MainFacade
from tests.test_plugin.sirbot import PluginTest
CONFIG = {
'core': {
'sirbot': {
'plugins': ['tests.test_plugin.sirbot']
},
'test': {
......@@ -25,21 +25,24 @@ CONFIG = {
}
}
async def test_bot_is_starting(loop, test_server):
def test_bot_is_starting(loop, test_server):
bot = sirbot.SirBot(loop=loop)
await test_server(bot._app)
loop.run_until_complete(test_server(bot._app))
assert bot._app == bot.app
async def test_load_config(loop):
def test_load_config(loop):
config = {
'core': {
'sirbot': {
'plugins': ['tests.test_plugin']
}
}
bot = sirbot.SirBot(loop=loop, config=config)
assert bot.config == config
async def test_logging_config(loop):
def test_logging_config(loop):
config = {
'logging': {
'version': 1,
......@@ -52,71 +55,73 @@ async def test_logging_config(loop):
}
}
},
'core': {
'plugins': ['tests.test_plugin']
'sirbot': {
'plugins': ['tests.test_plugin.sirbot']
}
}
bot = sirbot.SirBot(loop=loop, config=config)
assert logging.getLogger('sirbot.core').level == 30
assert logging.getLogger('sirbot').level == 40
async def test_plugin_import(loop, test_server):
def test_plugin_import(loop, test_server):
bot = sirbot.SirBot(loop=loop, config=CONFIG)
await test_server(bot._app)
loop.run_until_complete(test_server(bot._app))
assert bot._pm.has_plugin('tests.test_plugin.sirbot')
async def test_plugin_import_error(loop):
def test_plugin_import_error(loop):
bot = sirbot.SirBot(loop=loop)
bot.config['core'] = {
bot.config['sirbot'] = {
'plugins': ['xxx', ]
}
with pytest.raises(ImportError):
bot._import_plugins()
async def test_initialize_plugins(loop):
def test_initialize_plugins(loop):
bot = sirbot.SirBot(loop=loop, config=CONFIG)
assert isinstance(bot._plugins['test']['plugin'], PluginTest)
async def test_plugin_configure(loop, test_server):
def test_plugin_configure(loop, test_server):
bot = sirbot.SirBot(loop=loop, config=CONFIG)
await test_server(bot._app)
loop.run_until_complete(test_server(bot._app))
assert bot._plugins['test']['plugin'].config == CONFIG['test']
async def test_start_plugins(loop, test_server):
def test_start_plugins(loop, test_server):
bot = sirbot.SirBot(loop=loop, config=CONFIG)
await test_server(bot._app)
loop.run_until_complete(test_server(bot._app))
assert 'test' in bot._tasks
async def test_plugin_task_error(loop, test_server, capsys):
def test_plugin_task_error(loop, test_server, capsys):
config = deepcopy(CONFIG)
config['core']['plugins'] = ['tests.test_plugin.sirbot_start_error']
config['sirbot']['plugins'] = ['tests.test_plugin.sirbot_start_error']
bot = sirbot.SirBot(loop=loop, config=config)
await test_server(bot._app)
out, err = capsys.readouterr()
del bot._tasks['test-error']
assert 'Task exited with error' in err
assert 'Timeout while starting' in err
with pytest.raises(ValueError):
loop.run_until_complete(test_server(bot._app))
async def test_plugin_priority(loop, test_server):
def test_plugin_priority(loop, test_server):
config = deepcopy(CONFIG)
config['test']['priority'] = 80
config['test-error'] = {'priority': 70}
config['core']['plugins'].append('tests.test_plugin.sirbot_start_error')
config['sirbot']['plugins'].append('tests.test_plugin.sirbot_start_error')
bot = sirbot.SirBot(loop=loop, config=config)
assert bot._start_priority[80] == ['test']
assert bot._start_priority[70] == ['test-error']
async def test_plugin_no_start(loop, test_server):
def test_plugin_no_start(loop, test_server):
config = deepcopy(CONFIG)
config['test']['priority'] = False
bot = sirbot.SirBot(loop=loop, config=config)
assert bot._start_priority == {}
assert bot._plugins == {}
async def test_middleware(loop, test_client):
def test_middleware(loop, test_client):
async def handler(request):
assert isinstance(request['facades'], MainFacade)
......@@ -124,7 +129,7 @@ async def test_middleware(loop, test_client):
bot = sirbot.SirBot(loop=loop, config=CONFIG)
bot._app.router.add_route('GET', '/', handler)
server = await test_client(bot._app)
rep = await server.get('/')
server = loop.run_until_complete(test_client(bot._app))
rep = loop.run_until_complete(server.get('/'))
assert 200 == rep.status
assert 'test' == (await rep.text())
assert 'test' == (loop.run_until_complete(rep.text()))
......@@ -9,7 +9,7 @@ from tests.test_plugin.sirbot import PluginTest, FacadeTest
config = {
'loglevel': 10,
'core': {
'sirbot': {
'loglevel': 20,
'plugins': ['tests.test_plugin.sirbot']
}
......@@ -52,14 +52,14 @@ def test_getitem_facade(loop):
facade = MainFacade(bot._facades)
assert isinstance(facade['test'], FacadeTest)
with pytest.raises(KeyError):
facade['foo']
_ = facade['foo']
def test_contains_facade(loop):
bot = sirbot.SirBot(loop=loop, config=config)
facade = MainFacade(bot._facades)
assert 'test' in facade
assert not 'foo' in facade
assert 'foo' not in facade
def test_len_facade(loop):
......
import asyncio
from sirbot.hookimpl import hookimpl
from .sirbot import PluginTest
......@@ -6,6 +8,7 @@ class PluginTestStartError(PluginTest):
__name__ = 'test-error'
async def start(self):
await asyncio.sleep(0.1, loop=self.loop)
raise ValueError
......