[ops] Move ella-cli to bin/ folder

parent 1b3ce8ab
......@@ -67,8 +67,12 @@ src/webui/dest
src/package
attachments
src/webui/build
src/webui/dev
vardb-schema.png
errorShots
_book
.hypothesis
.mypy_cache
......@@ -33,9 +33,14 @@ RUN apt-get update && \
rm -rf /var/lib/apt/lists/* && \
rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/groff/* /usr/share/info/* /tmp/* /var/cache/apt/* /root/.cache
# Add user
RUN useradd -ms /bin/bash ella-user
RUN mkdir -p /dist /logs /data /pg-data /socket && chown -R ella-user:ella-user /dist /logs /socket /data /pg-data
# Tab completion for ella-cli
ENV PATH=/ella/bin:${PATH}
RUN echo 'eval "$(_ELLA_CLI_COMPLETE=source ella-cli)"' >> /home/ella-user/.bashrc
####
# dev image
# (also compiles files for production)
......
......@@ -29,7 +29,7 @@ database in production.
In general we don't want to make more migration scripts than necessary, so make sure things are proper.
1. Make and enter a dev instance: `make dev` and `make shell`
1. Inside it do:
1. `/ella/ella-cli database ci-migration-head` (resets database to the migration base, then runs all the migrations)
1. `ella-cli database ci-migration-head` (resets database to the migration base, then runs all the migrations)
1. `cd /ella/src/vardb/datamodel/migration/`
1. `PYTHONPATH=../../.. alembic revision --autogenerate -m "Name of migration"`. This will look at the current datamodel
and compare it against the database state, generating a migration script from the differences.
......@@ -43,10 +43,10 @@ The `test-api-migration` part of the test suite will test also test database mig
### Manually testing the migrations
To manually test the migration scripts you can run the upgrade/downgrade parts of the various migrations:
- $ ./ella-cli database ci-migration-base
- $ ./ella-cli database upgrade e371dfeb38c1
- $ ./ella-cli database upgrade 94a80b8848df
- $ ./ella-cli database downgrade e371dfeb38c1
- $ ella-cli database ci-migration-base
- $ ella-cli database upgrade e371dfeb38c1
- $ ella-cli database upgrade 94a80b8848df
- $ ella-cli database downgrade e371dfeb38c1
For migrations involving user generated data, it would be useful to run the migrations (both upgrade and downgrade)
with the database populated through "real" use.
......
......@@ -43,4 +43,4 @@ echo "Starting PostgreSQL"
# KILL goes directly to kernel, so it's useless here.
trap shutdown HUP INT QUIT ABRT KILL ALRM TERM TSTP
exec /usr/lib/postgresql/10/bin/postgres -c listen_addresses=''
exec /usr/lib/postgresql/10/bin/postgres -c listen_addresses='127.0.0.1'
......@@ -3,8 +3,8 @@
# Sets up vardb-test for local integration testing
createdb vardb-test || true
export DB_URL=postgresql:///vardb-test
/ella/ella-cli database drop -f
/ella/ella-cli database make -f
ella-cli database drop -f
ella-cli database make -f
export TEST_DB_DUMP="/ella/local-integration-testdump.sql"
echo "Setting TEST_DB_DUMP=${TEST_DB_DUMP}"
......
......@@ -3,7 +3,7 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Abort if not migrated
${DIR}/../../ella-cli database compare
ella-cli database compare
export PORT=${PORT:-3114}
......
......@@ -11,7 +11,7 @@ yellow "Importing fixtures"
cat /ella/ops/test/report/report-fixture-sample-1.sql | psql postgres
yellow "Starting test of classifications report..."
./ella-cli export classifications --filename e2e-variant-export
ella-cli export classifications --filename e2e-variant-export
# Compare only a few fields. Include the report field to test handling html -> text and non-ascii characters:
diff <(cut -f1,2,3,4,12,13,15 e2e-variant-export.csv) <(cut -f1,2,3,4,5,6,7 ops/test/report/classifications-expected.csv) || \
......@@ -21,7 +21,7 @@ diff <(cut -f1,2,3,4,12,13,15 e2e-variant-export.csv) <(cut -f1,2,3,4,5,6,7 ops/
green "[OK]"
yellow "Starting test of classifications report with analysis names..."
./ella-cli export classifications --filename e2e-variant-export-with-analyses -with_analysis_names
ella-cli export classifications --filename e2e-variant-export-with-analyses -with_analysis_names
# Compare only a few fields. Include the report field to test handling html -> text and non-ascii characters:
# same verifications as the above with one extra column
......
......@@ -12,7 +12,7 @@ cat /ella/ops/test/report/report-fixture-sample-1.sql | psql postgres
yellow "Testing that Sanger report has variants"
./ella-cli export sanger --filename e2e-sanger-export testgroup01
ella-cli export sanger --filename e2e-sanger-export testgroup01
# don't include date column in diff (it changes on each test run)
diff <(cut -f2- e2e-sanger-export.csv) <(cut -f2- ops/test/report/sanger-expected.csv) || \
(red "Sanger report (csv) did not have the expected content. Check report/sanger-expected.csv";
......@@ -26,7 +26,7 @@ cat /ella/ops/test/report/report-fixture-sample-1-and-2.sql | psql postgres
yellow "Testing that Sanger report is empty when there are no unstarted analyses"
./ella-cli export sanger --filename e2e-sanger-export testgroup01
ella-cli export sanger --filename e2e-sanger-export testgroup01
grep "file is intentionally empty" e2e-sanger-export.csv || \
(red "File should could contain a single line with comment; no variants";
exit 1)
......
......@@ -7,22 +7,22 @@ dropdb --if-exists vardb-test
echo "creating 'vardb-test'"
createdb vardb-test
echo "created 'vardb-test'"
/ella/ella-cli database drop -f
/ella/ella-cli database make -f
ella-cli database drop -f
ella-cli database make -f
if [ "$1" = "" ]
then
# Run migration scripts test
### Downgrade not supported ###
# /ella/ella-cli database ci-migration-test -f
# ella-cli database ci-migration-test -f
# Run API test on migrated database
/ella/ella-cli database ci-migration-head -f
/ella/ella-cli database refresh -f
ella-cli database ci-migration-head -f
ella-cli database refresh -f
py.test --color=yes "/ella/src/api/" -s
else
/ella/ella-cli database ci-migration-head -f
/ella/ella-cli database refresh -f
ella-cli database ci-migration-head -f
ella-cli database refresh -f
$@
fi
......
......@@ -7,8 +7,8 @@ dropdb --if-exists vardb-test
echo "creating 'vardb-test'"
createdb vardb-test
echo "created 'vardb-test'"
/ella/ella-cli database drop -f
/ella/ella-cli database make -f
ella-cli database drop -f
ella-cli database make -f
if [ "$1" = "" ]
then
......
......@@ -7,8 +7,8 @@ dropdb --if-exists vardb-test
echo "creating 'vardb-test'"
createdb vardb-test
echo "created 'vardb-test'"
/ella/ella-cli database drop -f
/ella/ella-cli database make -f
ella-cli database drop -f
ella-cli database make -f
if [ "$1" = "" ]
then
......@@ -17,14 +17,6 @@ else
$@
fi
# /ella/ella-cli deposit genepanel --folder $PANEL_PATH/HBOC_v01
# /ella/ella-cli deposit genepanel --folder $PANEL_PATH/HBOCUTV_v01
# /ella/ella-cli users add_groups --name testgroup01 src/vardb/testdata/usergroups.json
# /ella/ella-cli users add_many --group testgroup01 src/vardb/testdata/users.json
# /ella/ella-cli users list --username testuser1 > cli_output
# grep "HBOC_v01" cli_output | grep "HBOCUTV_v01" | grep "testuser1" || \
# (echo "Missing genepanels for testuser1. CLI output is:"; cat cli_output; exit 1)
echo "exits $BASH_SOURCE"
......@@ -8,8 +8,8 @@ echo "creating 'vardb-test'"
createdb vardb-test
echo "created 'vardb-test'"
/ella/ella-cli database drop -f
/ella/ella-cli database make -f
ella-cli database drop -f
ella-cli database make -f
if [ "$1" = "" ]
then
......
......@@ -6,52 +6,52 @@
* +++++++++++++++
*
* In order to run the script you need to have ammonite installed:
*
*
* sudo curl -L -o /usr/local/bin/amm https://git.io/vdNv2 && sudo chmod +x /usr/local/bin/amm && amm
*
* Or try your default package manager, at least it exists in `brew`
*
*
* Or try your default package manager, at least it exists in `brew`
*
* Java is probably a prerequisite!
*
*
* +++++++++++
* How to run:
* +++++++++++
* The scripts needs a running docker container instance.
*
* If you have installed ammonite, it should be enough to start it like this:
*
*
* ProductionSimulator --dockerId <provide id of the running ella docker container>
* The script can also be started with amm directly:
* amm TestDataGenerator.sc --dockerId <provide id of the running ella docker container>
*
* You have to restart the docker container after the script has finished, or kill the
* You have to restart the docker container after the script has finished, or kill the
* `analyis_watcher.py` script manually.
*
*
* +++++++++++++
* What it does:
* +++++++++++++
*
* Simulates production environment to make it possible to test the analysis_watcher.py script.
* Simulates production environment to make it possible to test the analysis_watcher.py script.
*
* +++++++++++++
* +++++++++++++
* How it works:
* +++++++++++++
*
* The script is based on Streams created by fs2 scala library. It concatenates the following two Streams:
* 1.) preparations - database reset and creation
* 2.) Three parallell streams running in 3 threads, consisting of analysisWatcher, analysisStream, readyStream
* 2.) Three parallell streams running in 3 threads, consisting of analysisWatcher, analysisStream, readyStream
*
* The three parallel streams will start to run only after the preparations stream has finished.
*
* The three parallel streams will start to run only after the preparations stream has finished.
*
* - analysisWatcher runs the python script `analysis_watcher.py` in `/ella/src/vardb/watcher`.
* - analysisStream creates an analysis folder according to `TestAnalysis-001`.
* - readyStream adds the READY flag inside the folder after the analysisStream has created the analysis folder.
*
*
* IMPORTANT REMARKS: The analysisStream and readyStream runs with random delays given with the flags CONFIG_DELAY and READY_DELAY
*
* Ammonite is an amazing tool created by Li Haoyi!
*
*
* Ammonite is an amazing tool created by Li Haoyi!
*
* It combines the Scala language with shell scripting almost as nice as in Python
*
*/
......@@ -121,12 +121,12 @@ implicit val formats = Serialization.formats(NoTypeHints)
val namePostfix = "TestAnalysis-00"
case class Analysis(
id: Int,
folderName: String,
fileName: String,
json: String,
genepanelName: String,
genepanelVersion: String
id: Int,
folderName: String,
fileName: String,
json: String,
genepanelName: String,
genepanelVersion: String
)
// Pure Stream of analysis config data
......@@ -143,7 +143,7 @@ val datamodelStream: Stream[IO, Analysis] = Stream(2,3,4,5,6).map { n =>
Analysis(
n,
json = write(config),
folderName = analysisName,
folderName = analysisName,
fileName = s"$analysisName.analysis",
genepanelName = genepanelName,
genepanelVersion = genepanelVersion
......@@ -170,15 +170,15 @@ def createTestConfig(dockerContainerId: String): Sink[IO, Analysis] = _.evalMap
s"""insert into genepanel (name, version, genome_reference, config ) values (
'${a.genepanelName}', '${a.genepanelVersion}', 'GRCh37' , '{}'
)"""
// running the insert statement at the simplest possible way through bash
val result = %%('docker, 'exec, dockerContainerId, 'psql, "-U", 'postgres, "-d", 'postgres, "-c", insert)
val path = analysisFolder/a.folderName
val file = path/a.fileName
// the `bang` is Ammonite syntax for the same shell operation as in bash.
// The ammonite shell operations are usually more forceful, they don't
// The ammonite shell operations are usually more forceful, they don't
// care if files exists, or folders are missing in a path.
mkdir! path
......@@ -195,7 +195,7 @@ def createTestConfig(dockerContainerId: String): Sink[IO, Analysis] = _.evalMap
case class Program(dockerContainerId: String) {
// transforming the pure dataModelStream into one for
// transforming the pure dataModelStream into one for
// generating analysis data and one for setting the READY FLAG
val analysisStream = datamodelStream.covary[IO].to(createTestConfig(dockerContainerId))
val readyStream = datamodelStream.map { c => c.folderName }.through(randomDelays(READY_DELAY)).to(setReady)
......@@ -203,8 +203,8 @@ def createTestConfig(dockerContainerId: String): Sink[IO, Analysis] = _.evalMap
// resetting the database
val preparations = {
mkdir! wd/up/'testdata/'destination
%('docker, 'exec, dockerContainerId, "/ella/ella-cli", 'database, 'drop, "-f")
%('docker, 'exec, dockerContainerId, "/ella/ella-cli", 'database, 'make, "-f")
%('docker, 'exec, dockerContainerId, "ella-cli", 'database, 'drop, "-f")
%('docker, 'exec, dockerContainerId, "ella-cli", 'database, 'make, "-f")
%('docker, 'exec, dockerContainerId, 'make, 'dbreset)
}
......@@ -225,10 +225,10 @@ def main(dockerId: String @doc(
println("creating new analysis and setting the READY flag at random intervals ..")
// running analysisStream and readyStream in parallell on NR_OF_THREADS given.
// ready flag will always be set after the initial test data are created, but
// running analysisStream and readyStream in parallell on NR_OF_THREADS given.
// ready flag will always be set after the initial test data are created, but
// ready flag for test data 2 may be set after test data 1.
val run = Stream(program.preparations) ++
val run = Stream(program.preparations) ++
Stream(program.analysisWatcher, program.analysisStream, program.readyStream)
.join(NR_OF_THREADS)
.through(randomDelays(CONFIG_DELAY))
......@@ -236,6 +236,6 @@ println("creating new analysis and setting the READY flag at random intervals ..
run.run.unsafeRunAsync(println)
println("exiting ..")
// program.program.covary[IO].run.unsafeRunAsync(println)
// program.program.covary[IO].run.unsafeRunAsync(println)
}
......@@ -73,7 +73,7 @@ module.exports = function addCommands() {
browser.addCommand('resetDb', (testset = 'e2e') => {
console.log(`Resetting database with '${testset}' (this can take a while...)`)
try {
execSync(`/ella/ella-cli database drop -f`, {
execSync(`ella-cli database drop -f`, {
stdio: ['ignore', 'ignore', 'pipe']
})
execSync(`psql postgres < /ella/dbdump_${testset}.sql`, {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment