Commit 09e7701f authored by Thomas's avatar Thomas

Merge branch 'release/v1.0'

parents 882243a3 58f0c6db
......@@ -15,13 +15,14 @@ indent_style = space
indent_size = 4
max_line_length = 160
[*.json]
; Salt state files, YAML format, 2 spaces
[*.sls, *.yaml, *.yml]
indent_style = space
indent_size = 4
indent_size = 2
max_line_length = 160
; Salt state files, YAML format, 2 spaces
[*.sls, *.yaml, *.yml]
; json files 2 spaces
[*.json]
indent_style = space
indent_size = 2
max_line_length = 160
activate
bin/
strongr-tasks-lock
.idea/
config.*
......
#!/usr/bin/env python
from cleo import Application
from cleo.inputs.argv_input import ArgvInput
from strongr.cli import DeploySingleCommand, ListDeployedVmsCommand,\
RunShellCodeCommand, DeployManyCommand,\
AddTaskCommand, GetTaskStatusCommand,\
RunResourceManager, PrintConfig,\
IsValidUserCommand, RunRestServerCommand,\
RunCeleryCommand
import strongr.core
import logging.config
# Use CLEO ArgvInput to extract some parameters
# this dependency gets injected into
# application.run(..) as well
argvInputs = ArgvInput()
# env is used for loading the right config environment
......@@ -21,27 +17,45 @@ env = "develop"
if argvInputs.has_parameter_option('env'):
env = argvInputs.get_parameter_option('env')
core = strongr.core.getCore()
import strongr.core
core = strongr.core.Core
configDomain = core.domains().configDomain()
configDomain.configService().getCommandBus().handle(configDomain.commandFactory().newLoadConfig(env))
from strongr.core.domain.configdomain import ConfigDomain
ConfigDomain.configService().getCommandBus().handle(ConfigDomain.commandFactory().newLoadConfig(env))
logging.config.dictConfig(core.config().logger.as_dict())
application = Application()
from strongr.cli import DeploySingleCommand, ListDeployedVmsCommand,\
DeployManyCommand,\
AddJobCommand, GetFinishedJobsCommand,\
RunResourceManager, PrintConfig,\
IsValidUserCommand, RunRestServerCommand,\
RunWorkerCommand, DestroySingleCommand,\
DestroyManyCommand, EnsureMinAmountOfNodesCommand, \
MakeDbCommand, RunTestsCommand, TestCommand, CleanupCommand,\
GetJobStdOut
application.add(DeploySingleCommand(core))
application.add(ListDeployedVmsCommand(core))
application.add(RunShellCodeCommand(core))
application.add(DeployManyCommand(core))
application.add(AddTaskCommand(core))
application.add(GetTaskStatusCommand(core))
application.add(RunResourceManager(core))
application.add(PrintConfig(core))
application.add(IsValidUserCommand(core))
application.add(RunRestServerCommand(core))
application.add(RunCeleryCommand(core))
application = Application()
application.add(DeploySingleCommand())
application.add(ListDeployedVmsCommand())
application.add(DeployManyCommand())
application.add(AddJobCommand())
application.add(GetFinishedJobsCommand())
application.add(RunResourceManager())
application.add(PrintConfig())
application.add(IsValidUserCommand())
application.add(RunRestServerCommand())
application.add(RunWorkerCommand())
application.add(DestroySingleCommand())
application.add(DestroyManyCommand())
application.add(EnsureMinAmountOfNodesCommand())
application.add(MakeDbCommand())
application.add(RunTestsCommand())
application.add(TestCommand())
application.add(CleanupCommand())
application.add(GetJobStdOut())
if __name__ == '__main__':
application.run(input_=argvInputs)
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
SPHINXPROJ = StrongR
SOURCEDIR = source
BUILDDIR = build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
\ No newline at end of file
@ECHO OFF
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=source
set BUILDDIR=build
set SPHINXPROJ=StrongR
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
:end
popd
# -*- coding: utf-8 -*-
#
# Configuration file for the Sphinx documentation builder.
#
# This file does only contain a selection of the most common options. For a
# full list see the documentation:
# http://www.sphinx-doc.org/en/stable/config
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
# -- Project information -----------------------------------------------------
project = u'StrongR'
copyright = u'2018, Thomas Phil'
author = u'Thomas Phil'
# The short X.Y version
version = u''
# The full version, including alpha/beta/rc tags
release = u''
# -- General configuration ---------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.doctest',
'sphinx.ext.todo',
'sphinx.ext.coverage',
'sphinx.ext.mathjax',
'sphinx.ext.ifconfig',
'sphinx.ext.viewcode',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path .
exclude_patterns = []
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# The default sidebars (for documents that don't match any pattern) are
# defined by theme itself. Builtin themes are using these templates by
# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
# 'searchbox.html']``.
#
# html_sidebars = {}
# -- Options for HTMLHelp output ---------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'StrongRdoc'
# -- Options for LaTeX output ------------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'StrongR.tex', u'StrongR Documentation',
u'Thomas Phil', 'manual'),
]
# -- Options for manual page output ------------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'strongr', u'StrongR Documentation',
[author], 1)
]
# -- Options for Texinfo output ----------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'StrongR', u'StrongR Documentation',
author, 'StrongR', 'One line description of project.',
'Miscellaneous'),
]
# -- Options for Epub output -------------------------------------------------
# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright
# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#
# epub_identifier = ''
# A unique identification for the text.
#
# epub_uid = ''
# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']
# -- Extension configuration -------------------------------------------------
# -- Options for todo extension ----------------------------------------------
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True
\ No newline at end of file
.. StrongR documentation master file, created by
sphinx-quickstart on Fri Jun 29 11:41:49 2018.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to StrongR's documentation!
===================================
.. toctree::
:maxdepth: 2
:caption: Contents:
manuals/install
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
Installing StrongR
******************
StrongR currently supports two cloud-adapters. The OpenNebula and MockCloud adapters.
OpenNebula
==========
MockCloud
=========
The mockcloud is useful for testing containers localy.
Let's clone strongr and install it. Then create a scratch directory.
.. code-block:: shell
git clone https://gitlab.com/bbmri/infra/strongr.git
cd strongr
pip install -r requirements.txt
pip install -e .
mkdir /scratch
chmod 777 /scratch
Now copy over the config below to ~/.strongr/config.json.
.. code-block:: json
{
"defaults": {
},
"develop": {
"lock": {
"driver": "file",
"file": {
"path": "/tmp/strongr-locks"
}
},
"cache": {
"driver": "local"
},
"db": {
"engine": {
"url": "sqlite:////tmp/strongr.db"
}
},
"clouddomain": {
"driver": "MockCloud",
"MockCloud": {
"scratch": "/scratch"
}
},
"logger": {
"handlers": {
"default": {
"level": "DEBUG"
}
},
"loggers": {
"": {
"level": "DEBUG"
}
}
}
}
}
Finally, run the mockcloud.
.. code-block:: shell
strongr r:r
FROM bigr/fastr_worker
MAINTAINER Thomas Phil <thomas@tphil.nl>
version: '3'
services:
strongr-master:
build: ./strongr-master/
networks:
- strongr-int
strongr-worker:
build: ./strongr-worker/
networks:
- strongr-int
strongr-restserver:
build: ./strongr-restserver/
networks:
- strongr-int
redis:
image: "redis:4-alpine"
volumes:
- redis-data:/data
networks:
- strongr-int
rabbitmq:
image: "rabbitmq:3-alpine"
volumes:
- rabbitmq-data:/var/lib/rabbitmq
networks:
- strongr-int
mysql:
image: "mysql:5"
volumes:
- mysql-data:/var/lib/mysql
networks:
- strongr-int
volumes:
mysql-data: /opt/strongr/mysql
redis-data: /opt/strongr/redis
rabbitmq-data: /opt/strongr/rabbitmq
networks:
strongr-int:
FROM bigr/fastr_worker_develop
MAINTAINER T. Phil <thomas@tphil.nl>
RUN mkdir /opt/fastr-conf && mkdir /opt/strongr && touch /opt/strongr/init.sh
ENV FASTRHOME /opt/fastr-conf
ADD config.py /opt/fastr-conf/config.py
FROM bigr/fastr_worker
MAINTAINER T. Phil <thomas@tphil.nl>
RUN mkdir /opt/fastr-conf && mkdir /opt/strongr && touch /opt/strongr/init.sh
ENV FASTRHOME /opt/fastr-conf
ADD config.py /opt/fastr-conf/config.py
FROM ubuntu:16.04
MAINTAINER Thomas Phil <thomas@tphil.nl>
#ARG FASTRBRANCH develop
ARG FASTRBRANCH
RUN export DEBIAN_FRONTED=noninteractive
RUN apt-get update && apt-get install -y python python-pip mercurial && pip install --upgrade pip
RUN hg clone https://sikerdebaard@bitbucket.org/bigr_erasmusmc/fastr && cd fastr && hg checkout $FASTRBRANCH && hg pull && hg up && pip install -e .
ENTRYPOINT '/bin/bash'
FROM ubuntu:16.04
MAINTAINER Thomas Phil <thomas@tphil.nl>
ARG FASTRVERSION=develop
ENV FASTRVERSION ${FASTRVERSION}
RUN export DEBIAN_FRONTED=noninteractive
RUN apt-get update && apt-get install -y python python-pip && pip install fastr==$FASTRVERSION
ENTRYPOINT '/bin/bash'
FROM alpine:3.6
MAINTAINER Thomas Phil <thomas@tphil.nl>
ADD https://gitlab.com/bbmri/infra/strongr/repository/rel-1.0/archive.zip /tmp/strongr.zip
RUN apk add --no-cache unzip python2 python2-dev build-base
ENV ADDITIONAL_REQUIREMENTS flask flask_restplus celery
RUN python2 -m ensurepip && mkdir -p /opt/strongr && unzip -o /tmp/strongr.zip -d /tmp && mv /tmp/strongr-rel*/* /opt/strongr/ && rm -rf /tmp/strongr* && pip install setuptools && pip install -r /opt/strongr/requirements.txt && pip install $ADDITIONAL_REQUIREMENTS
RUN apk del unzip build-base python2-dev
WORKDIR /opt/strongr
FROM strongr:1.0
MAINTAINER Thomas Phil <thomas@tphil.nl>
RUN apk add --no-cache salt-master salt-cloud salt-ssh
EXPOSE 4505 4506
VOLUME ["/etc/salt/pki", "/var/cache/salt", "/var/logs/salt", "/etc/salt/master.d", "/srv/salt"]
CMD "python app.py r:r"
FROM ubuntu:16.04
MAINTAINER Thomas Phil <thomas@tphil.nl>
ADD https://gitlab.com/bbmri/infra/strongr/repository/rel-1.1/archive.zip /tmp/strongr.zip
ADD config.json /root/.strongr/config.json
ADD config.py /root/.fastr/config.py
ADD addtask /opt/strongr/addtask
ADD queryqueue /opt/strongr/queryqueue
RUN apt-get update && apt-get install -y unzip python2.7 python2.7-dev build-essential mercurial python-pip ca-certificates docker.io
RUN chmod +x /opt/strongr/queryqueue && chmod +x /opt/strongr/addtask
RUN mkdir -p /opt/strongr && mkdir -p /opt/fastr && unzip -o /tmp/strongr.zip -d /tmp && mv /tmp/strongr-rel*/* /opt/strongr/ && rm -rf /tmp/strongr* && pip install setuptools && pip install -r /opt/strongr/requirements.txt && hg clone --insecure https://bitbucket.org/bigr_erasmusmc/fastr /opt/fastr && cd /opt/fastr && hg checkout develop && pip install -e .
RUN cd /opt/strongr && python app.py d:create
WORKDIR /opt/strongr
CMD ["python", "app.py", "r:r"]
#!/bin/bash
cd /opt/strongr
python app.py j:a 'bigr/fastr_addints_worker' $1 y $2 $3 | tail -n1
#!/bin/bash
cd /opt/strongr
python app.py j:f | tail -n1
FROM strongr:1.0
MAINTAINER Thomas Phil <thomas@tphil.nl>
#CMD "python app.py r:r"
FROM strongr:1.0
MAINTAINER Thomas Phil <thomas@tphil.nl>
#CMD "python app.py r:r"
setuptools>=25.1.1
cleo>=0.5.0
cmndr>=1.0.5
dependency_injector>=3.4.1
jsonpickle>=0.9.4
salt>=2016.11.3
filelock>=2.0.8
jsonpickle==0.9.5
cleo==0.6.1
setuptools==36.5.0
Flask==0.12.2
dependency_injector==3.6.1
salt==2017.7.1
filelock==2.0.12
flask_restplus==0.10.1
Flask_OAuthlib==0.9.4