Commit 4bcb06dc authored by Thomas Phil's avatar Thomas Phil

Merge branch 'release/rel-1.0'

parents b94c4253 32412a2e
......@@ -8,11 +8,20 @@ root = true
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
; Python: PEP8 defines 4 spaces for indentation
[*.py]
indent_style = space
indent_size = 4
max_line_length = 160
[*.json]
indent_style = space
indent_size = 4
max_line_length = 160
; Salt state files, YAML format, 2 spaces
[*.sls, *.yaml, *.yml]
indent_style = space
indent_size = 2
max_line_length = 160
activate
bin/
strongr-tasks-lock
.idea/
config.*
temp
# VIM
*.swp
*.swo
# project-specific
strongr/master/config.js
# Byte-compiled / optimized / DLL files
__pycache__/
......
from cleo import Application
from cleo.inputs.argv_input import ArgvInput
from strongr.cli import DeploySingleCommand, ListDeployedVmsCommand,\
RunShellCodeCommand, DeployManyCommand,\
AddTaskCommand, GetTaskStatusCommand,\
RunResourceManager, PrintConfig,\
IsValidUserCommand, RunRestServerCommand,\
RunCeleryCommand
import strongr.core
import logging.config
# Use CLEO ArgvInput to extract some parameters
# this dependency gets injected into
# application.run(..) as well
argvInputs = ArgvInput()
# env is used for loading the right config environment
env = "develop"
if argvInputs.has_parameter_option('env'):
env = argvInputs.get_parameter_option('env')
core = strongr.core.getCore()
configDomain = core.domains().configDomain()
configDomain.configService().getCommandBus().handle(configDomain.commandFactory().newLoadConfig(env))
logging.config.dictConfig(core.config().logger.as_dict())
application = Application()
application.add(DeploySingleCommand(core))
application.add(ListDeployedVmsCommand(core))
application.add(RunShellCodeCommand(core))
application.add(DeployManyCommand(core))
application.add(AddTaskCommand(core))
application.add(GetTaskStatusCommand(core))
application.add(RunResourceManager(core))
application.add(PrintConfig(core))
application.add(IsValidUserCommand(core))
application.add(RunRestServerCommand(core))
application.add(RunCeleryCommand(core))
if __name__ == '__main__':
application.run(input_=argvInputs)
cleo==0.5.0
cmndr==1.0.4
dependency_injector==3.4.1
setuptools==25.1.1
setuptools>=25.1.1
cleo>=0.5.0
cmndr>=1.0.5
dependency_injector>=3.4.1
jsonpickle>=0.9.4
salt>=2016.11.3
filelock>=2.0.8
from setuptools import setup
setup(
name='commandr',
name='strongr',
version='1.0',
description='A python commandbus implementation',
description='An elastic cloud command runner',
author='Thomas Phil',
author_email='thomas@tphil.nl',
packages=['commndr'], #same as name
install_requires=['bar', 'greek'], #external packages as dependencies
packages=find_packages(), #same as name
install_requires=[], #external packages as dependencies
scripts=[]
)
from .queryfactory import QueryFactory
from strongr.authdomain.query import IsValidUser
from strongr.core.exception import InvalidParameterException
class QueryFactory:
def newIsValidUserQuery(self, username, password):
""" Generates a new IsValidUser query
:returns: An IsValidUser query object
:rtype: IsValidUser
"""
if username == None or len(username) == 0 or len(username.strip()) == 0:
raise InvalidParameterException('Username is invalid')
if password == None or len(password) == 0:
raise InvalidParameterException('Password is invalid')
return IsValidUser(username, password)
from .isvaliduserhandler import IsValidUserHandler
class IsValidUserHandler():
def __call__(self, command):
# use a hardcoded test-user for now
return (True if command.username == 'test' and command.password == 'test' else False)
from .isvaliduser import IsValidUser
class IsValidUser():
def __init__(self, username, password):
self.username = username
self.password = password
from .authservice import AuthService
from strongr.core.abstracts.abstractservice import AbstractService
from strongr.authdomain.query import IsValidUser
from strongr.authdomain.handler import IsValidUserHandler
class AuthService(AbstractService):
_command_bus = None
_query_bus = None
def getCommandBus(self):
if self._command_bus is None:
self._command_bus = self._make_default_commandbus({
})
return self._command_bus
def getQueryBus(self):
if self._query_bus is None:
self._query_bus = self._make_default_querybus({
IsValidUserHandler: IsValidUser,
})
return self._query_bus
......@@ -2,3 +2,10 @@ from .deploysinglecommand import DeploySingleCommand
from .runshellcodecommand import RunShellCodeCommand
from .listdeployedvmscommand import ListDeployedVmsCommand
from .deploymanycommand import DeployManyCommand
from .addtaskcommand import AddTaskCommand
from .gettaskstatuscommand import GetTaskStatusCommand
from .runresourcemanager import RunResourceManager
from .printconfig import PrintConfig
from .isvalidusercommand import IsValidUserCommand
from .runrestservercommand import RunRestServerCommand
from .runcelerycommand import RunCeleryCommand
from .wrapper import Command
import uuid
import base64
import json
import time
class AddTaskCommand(Command):
"""
Add a task to the resource manager
task:add
{shell : Shellcode to execute}
{cores : Amount of cores required for the task}
{mem : Amount of ram in GiB required for the task}
"""
def handle(self):
schedulerService = self.getDomains().schedulerDomain().schedulerService()
commandFactory = self.getDomains().schedulerDomain().commandFactory()
cmd = str(base64.b64decode(self.argument('shell')))
cores = int(self.argument('cores'))
ram = int(self.argument('mem'))
if not (cores > 0 and ram > 0 and len(cmd) > 0):
# TODO: put something sensible in here, this is just a placeholder
self.error('Invalid input')
return
taskid = str(int(time.time())) + '-' + str(uuid.uuid4())
command = commandFactory.newScheduleTaskCommand(taskid, cmd, cores, ram)
schedulerService.getCommandBus().handle(command)
print(json.dumps({'taskid': taskid}))
from .wrapper import Command
from services import CloudServices
from commands import DeployVm
from commands import DeployVms
import uuid
......@@ -12,6 +9,9 @@ class DeployManyCommand(Command):
deploy:many
"""
def handle(self):
cloudServices = self.getDomains().cloudDomain().cloudService()
commandFactory = self.getDomains().cloudDomain().commandFactory()
cores = int(self.ask('How many processing cores should the VM\'s have? (default 1): ', 1))
ram = int(self.ask('How much memory in GiB should the VM\'s have? (default 4): ', 4))
amount = int(self.ask('How many VM\'s should be deployed? (default 2)', 2))
......@@ -21,19 +21,18 @@ class DeployManyCommand(Command):
self.error('Invalid input')
return
deployVms = DeployVms()
deployVmList = []
while amount > 0:
deployVmCommand = DeployVm().name(str(uuid.uuid4())).cores(cores).ram(ram)
deployVms.append(deployVmCommand)
deployVmCommand = commandFactory.newDeployVmCommand(name='worker-' + str(uuid.uuid4()), cores=cores, ram=ram)
deployVmList.append(deployVmCommand)
amount -= 1
deployVms = commandFactory.newDeployVmsCommand(deployVmList)
cloudServices = CloudServices()
cloudNames = cloudServices.getCloudNames()
cloudProviderName = self.choice('Please select a cloud provider (default {0})'.format(cloudNames[0]), cloudNames, 0)
cloudProviderName = self.getContainer().config().clouddomain.driver
cloudService = cloudServices.getCloudServiceByName(cloudProviderName)
commandBus = cloudService.getCommandBus()
self.info('Deploying {0} VM\'s with cores={1} ram={2}GiB'.format(len(deployVms), cores, ram))
print(commandBus.handle(deployVms))
commandBus.handle(deployVms)
from .wrapper import Command
from services import CloudServices
from commands import DeployVm
import uuid
......@@ -11,6 +9,9 @@ class DeploySingleCommand(Command):
deploy:single
"""
def handle(self):
cloudServices = self.getDomains().cloudDomain().cloudService()
commandFactory = self.getDomains().cloudDomain().commandFactory()
cores = int(self.ask('How many processing cores should the VM have? (default 1): ', 1))
ram = int(self.ask('How much memory in GiB should the VM have? (default 4): ', 4))
name = self.ask('What is the name of the VM? (default generated): ', str(uuid.uuid4()))
......@@ -21,15 +22,13 @@ class DeploySingleCommand(Command):
return
deployVmCommand = DeployVm().name(name).cores(cores).ram(ram)
deployVmCommand = commandFactory.newDeployVmCommand(name='worker-' + name, cores=cores, ram=ram)
cloudServices = CloudServices()
cloudNames = cloudServices.getCloudNames()
cloudProviderName = self.choice('Please select a cloud provider (default {0})'.format(cloudNames[0]), cloudNames, 0)
cloudProviderName = self.getContainer().config().clouddomain.driver
cloudService = cloudServices.getCloudServiceByName(cloudProviderName)
commandBus = cloudService.getCommandBus()
self.info('Deploying VM {0} cores={1} ram={2}GiB'.format(name, cores, ram))
print(commandBus.handle(deployVmCommand))
commandBus.handle(deployVmCommand)
from .wrapper import Command
import json
class GetTaskStatusCommand(Command):
"""
Get task status. For now this returns json with all the enqueued tasks.
task:status
"""
def handle(self):
schedulerService = self.getDomains().schedulerDomain().schedulerService()
queryFactory = self.getDomains().schedulerDomain().queryFactory()
query = queryFactory.newRequestScheduledTasks()
result = schedulerService.getQueryBus().handle(query)
print(json.dumps(result))
from .wrapper import Command
import json
class IsValidUserCommand(Command):
"""
Check if the login is a valid user
authdomain:isvaliduser
{username : The username to be checked}
{password : The password to be checked}
"""
def handle(self):
authService = self.getDomains().authDomain().authService()
queryFactory = self.getDomains().authDomain().queryFactory()
query = queryFactory.newIsValidUserQuery(self.argument('username'), self.argument('password'))
result = authService.getQueryBus().handle(query)
print(json.dumps(result))
from .wrapper import Command
class ListDeployedVmsCommand(Command):
"""
List VMs deployed in the cloud.
deploy:list
"""
def handle(self):
cloudService = self.getDomains().cloudDomain().cloudService()
queryFactory = self.getDomains().cloudDomain().queryFactory()
cloudProviderName = self.getContainer().config().clouddomain.driver
cloudService = cloudService.getCloudServiceByName(cloudProviderName)
queryBus = cloudService.getQueryBus()
listDeployedVms = queryFactory.newListDeployedVms()
print(queryBus.handle(listDeployedVms))
from .wrapper import Command
import json
class PrintConfig(Command):
"""
Prints the config.
print:config
"""
def handle(self):
config = self.getContainer().config().as_dict()
del config['internal']
print(json.dumps(config, indent=2, sort_keys=True))
from .wrapper import Command
class RequestScheduledTasks(Command):
"""
Shows the task queue.
task:list
"""
def handle(self):
schedulerService = self.getDomains().schedulerDomain().schedulerService()
queryFactory = self.getDomains().schedulerDomain().schedulerQueryFactory()
query = commandFactory.newRequestScheduledTasks()
result = schedulerService.getQueryBus().handle(query)
print(json.dumps(result))
from .wrapper import Command
import strongr.core
class RunCeleryCommand(Command):
"""
Runs a celery worker.
celery:run
"""
def handle(self):
from celery import Celery
broker = 'amqp://guest:guest@localhost'
celery_test = Celery('celery_test', broker=broker, backend=broker)
scheduler = self.getDomains().schedulerDomain().schedulerService()
scheduler.getCommandBus() # we need to initiate this so that celery knows where to send its commands
scheduler.getQueryBus() # we need to initiate this so that celery knows where to send its commands
commands = [
'strongr.schedulerdomain.command.scheduletask.ScheduleTask',
'strongr.schedulerdomain.query.requestscheduledtasks.RequestScheduledTasks'
]
for command in commands:
strongr.core.getCore().commandRouter().enable_worker_route_for_command(celery_test, command)
argv = [
'worker',
'--loglevel=DEBUG',
'-Q=' + ','.join(commands)
]
celery_test.worker_main(argv)
from .wrapper import Command
import time
class RunResourceManager(Command):
"""
Runs the resource manager. This should be done on your salt master.
resourcemanager:run
"""
def handle(self):
schedulerService = self.getDomains().schedulerDomain().schedulerService()
commandFactory = self.getDomains().schedulerDomain().commandFactory()
commandBus = schedulerService.getCommandBus()
doDelayedTasksCommand = commandFactory.newDoDelayedTasks()
self.info('Running.')
while True:
commandBus.handle(doDelayedTasksCommand)
time.sleep(1)
from .wrapper import Command
from flask import Flask
class RunRestServerCommand(Command):
"""
Runs the strongr REST server that sits between FASTR and STRONGR
restdomain:startserver
"""
def handle(self):
config = self.getContainer().config()
host = config.restdomain.host
port = int(config.restdomain.port)
backend = config.restdomain.backend.strip().lower()
self.info("Starting server on {}:{} using {}".format(host, port, backend))
domain = self.getDomains().restDomain()
wsgiQueryFactory = domain.wsgiQueryFactory()
wsgiQueryBus = domain.wsgiService().getQueryBus()
blueprints = wsgiQueryBus.handle(wsgiQueryFactory.newRetrieveBlueprints())
app = Flask(__name__)
# the oauth2 lib can not work with templates,
# this hack was proposed as a temp fix on the
# libraries github. Use this for now, we
# should refactor this later.
# https://github.com/lepture/flask-oauthlib/issues/180
from strongr.restdomain.api.oauth2 import bind_oauth2
oauth2 = bind_oauth2(app)
app.oauth2 = oauth2
for blueprint in blueprints:
app.register_blueprint(blueprint)
if backend == 'flask':
app.run(**config.restdomain.flask.as_dict())
elif backend == 'gunicorn':
from gunicorn.app.base import BaseApplication
# put WSGIServer class here for now
# this should be refactored later
class WSGIServer(BaseApplication):
def __init__(self, app):
self.application = app
super(WSGIServer, self).__init__("%(prog)s [OPTIONS]")
def load_config(self):
for key in config.restdomain.gunicorn:
self.cfg.set(key, config[key])
def load(self):
return self.application
WSGIServer(app).run()
from .wrapper import Command
from services import CloudServices
from commands import RunShellCode
class RunShellCodeCommand(Command):
"""
Runs shellcode on a VM in the cloud.
......@@ -11,21 +8,23 @@ class RunShellCodeCommand(Command):
{--r|remote=? : If set, runs the code on the specified host}
{cmd* : shellcode to be run on the specified host(s)}
"""
def handle(self):
services = self.cloudDomain().getServicesContainer()
cloudServices = services.cloudServices()
commandFactory = services.cloudCommandFactory()
host = self.option('remote')
cmd = self.argument('cmd')
if not host:
host = '*'
runShellCode = RunShellCode().host(host).sh(cmd)
runShellCode = commandFactory.newRunShellCodeCommand(host=host, sh=cmd)
cloudServices = CloudServices()
cloudNames = cloudServices.getCloudNames()
cloudProviderName = self.choice('Please select a cloud provider (default {0})'.format(cloudNames[0]), cloudNames, 0)
cloudService = cloudServices.getCloudServiceByName(cloudProviderName)
commandBus = cloudService.getCommandBus()
print(commandBus.handle(runShellCode))
commandBus.handle(runShellCode)
from cleo import Command
class PatchedCommand(Command):
def __init__(self, coreContainer):
self._coreContainer = coreContainer
super(PatchedCommand, self).__init__()
def getDomains(self):
return self._coreContainer.domains()
def getContainer(self):
return self._coreContainer
def ask(self, question, default):
# since cleo has a bug that causes it not to return default values we need a wrapper that does exactly that
output = super(PatchedCommand, self).ask(question)
......@@ -8,12 +18,16 @@ class PatchedCommand(Command):
output = default
return output
def choice(self, question, options, default):
def choice(self, question, options, defaultIndex):
# for some reason cleo can not handle arrays with 1 el so we need a workaround
if len(options) > 1:
output = super(PatchedCommand, self).choice(question, options, default)
output = super(PatchedCommand, self).choice(question, options, defaultIndex)
else:
output = options[0]
return output
def _castToBool(self, val):
if isinstance(val, bool):
return val
return val.lower() in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup', 'certainly', 'uh-huh', 'yarr']
from .deployvm import DeployVm
from .runshellcode import RunShellCode
from .listdeployedvms import ListDeployedVms
from .deployvms import DeployVms
from .destroyvm import DestroyVm
class DeployVm:
# TODO: we should do propper input checking
def name(self, name):
def __init__(self, name, cores, ram):
self.name = name
return self
def cores(self, cores):
self.cores = cores
return self
def ram(self, ram):
self.ram = ram
return self
class DestroyVm:
def __init__(self, name):
self.name = name
class RunCommandInContainer:
def __init__(self, container, command):
pass
class RunShellCode:
def sh(self, sh):
def __init__(self, sh, host):
self.sh = sh
return self
def host(self, host