Commit 5e2daac4 authored by Brian Hatchet's avatar Brian Hatchet 💬

Merge branch 'goal/local-infra' into 'master'

Local infrastructure provisioner

Closes #1066

See merge request !173
parents 40689b79 d4aea0f3
......@@ -18,6 +18,7 @@
# Declare files that will always have CRLF line endings on checkout.
*.bat eol=crlf
*.ps1 eol=crlf
# Declare files that will always have LF line endings on checkout.
*.pem eol=lf
......
#!/bin/bash
# Exit script wit ERRORLEVEL if any command fails
set -e
rm -rf /usr/share/elasticsearch/data/*
exec /docker-entrypoint.sh elasticsearch
\ No newline at end of file
exec /docker-entrypoint.sh elasticsearch
......@@ -5,4 +5,4 @@ WORKDIR /provisioner
COPY wait-for.sh provision-elasticsearch.sh provision-elasticsearch-legacy.sh /provisioner/
COPY schema-legacy/ /provisioner/schema-legacy/
COPY schema/ /provisioner/schema/
ENTRYPOINT ["sh", "./provision-elasticsearch.sh"]
\ No newline at end of file
ENTRYPOINT ["sh", "./provision-elasticsearch.sh", "elasticsearch"]
#!/bin/bash
# Exit script wit ERRORLEVEL if any command fails
set -e
echo "Provisioning legacy elastic search";
echo "Waiting for legacy elastic search to come online..."
./wait-for.sh minds_elasticsearch-legacy_1:9200 --timeout=120 -- echo "Legacy elastic search is up and running"
./wait-for.sh $1:9200 --timeout=120 -- echo "Legacy elastic search is up and running"
echo "Putting mappings"
echo "Putting minds_badger"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds_badger -d @./schema-legacy/minds_badger.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds_badger -d @./schema-legacy/minds_badger.json --header "Content-Type: application/json"
echo "Putting minds-graph"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-graph -d @./schema-legacy/minds-graph.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-graph -d @./schema-legacy/minds-graph.json --header "Content-Type: application/json"
echo "Putting minds-helpdesk"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-helpdesk -d @./schema-legacy/minds-helpdesk.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-helpdesk -d @./schema-legacy/minds-helpdesk.json --header "Content-Type: application/json"
echo "Putting minds-kite"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-kite -d @./schema-legacy/minds-kite.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-kite -d @./schema-legacy/minds-kite.json --header "Content-Type: application/json"
echo "Putting minds-metrics"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-metrics-06-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-metrics-07-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-metrics-08-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-metrics-09-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-metrics-10-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-metrics-11-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-metrics-12-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-metrics-06-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-metrics-07-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-metrics-08-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-metrics-09-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-metrics-10-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-metrics-11-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-metrics-12-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
echo "Putting minds-moderation"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-moderation -d @./schema-legacy/minds-moderation.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-moderation -d @./schema-legacy/minds-moderation.json --header "Content-Type: application/json"
echo "Putting minds-trending-hashtags-shrunk"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-trending-hashtags-shrunk -d @./schema-legacy/minds-trending-hashtags-shrunk.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-trending-hashtags-shrunk -d @./schema-legacy/minds-trending-hashtags-shrunk.json --header "Content-Type: application/json"
echo "Stopping legacy elastic search container"
docker stop minds_elasticsearch-legacy_1
#!/bin/bash
# Exit script wit ERRORLEVEL if any command fails
set -e
echo "Provisioning elastic search";
echo "Waiting for elastic search to come online..."
./wait-for.sh minds_elasticsearch_1:9200 --timeout=120 -- echo "Elastic search is up and running"
./wait-for.sh $1:9200 --timeout=120 -- echo "Elastic search is up and running"
echo "Putting mappings"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch_1:9200/minds-views -d @./schema/minds-views.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch_1:9200/minds-boost -d @./schema/minds-boost.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch_1:9200/minds-offchain -d @./schema/minds-offchain.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch_1:9200/minds-transactions-onchain -d @./schema/minds-transactions-onchain.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-views -d @./schema/minds-views.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-boost -d @./schema/minds-boost.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-offchain -d @./schema/minds-offchain.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://$1:9200/minds-transactions-onchain -d @./schema/minds-transactions-onchain.json --header "Content-Type: application/json"
echo "elastic search is ready!"
#!/bin/bash
# Exit script wit ERRORLEVEL if any command fails
set -e
chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/data
exec /usr/local/bin/docker-entrypoint.sh elasticsearch
\ No newline at end of file
exec /usr/local/bin/docker-entrypoint.sh elasticsearch
......@@ -2,10 +2,12 @@ FROM nginx:1.13-alpine
WORKDIR /var/www/Minds
ENV UPSTREAM_ENDPOINT=front-live-server:4200
RUN rm /etc/nginx/conf.d/default.conf
COPY nginx.conf /etc/nginx/nginx.conf
COPY dev-ssr.conf /etc/nginx/conf.d/dev.conf
COPY nginx_entrypoint_dev.sh /nginx_entrypoint_dev.sh
COPY dev-ssr.conf.tpl /dev-ssr.conf.tpl
COPY nginx_entrypoint_dev_ssr.sh /nginx_entrypoint_dev_ssr.sh
ENTRYPOINT /nginx_entrypoint_dev.sh
ENTRYPOINT /nginx_entrypoint_dev_ssr.sh
......@@ -4,10 +4,12 @@ map $http_upgrade $connection_upgrade {
}
server {
resolver ${DOCKER_RESOLVER} ipv6=off;
listen 80;
listen [::]:80 default ipv6only=on;
listen 8080;
root /var/www/Minds/front/dist;
root /var/www/Minds/front/dist/en;
index index.php index.html;
server_name _;
......@@ -26,12 +28,14 @@ server {
sendfile off;
location / {
set $upstream http://${UPSTREAM_ENDPOINT};
port_in_redirect off;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_set_header Host nginx;
proxy_set_header X-NginX-Proxy true;
proxy_pass http://front:4200/;
proxy_pass $upstream;
proxy_redirect off;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
......@@ -49,12 +53,12 @@ server {
rewrite ^(.+)$ /index.php last;
}
location ~* \.(png|jpg|jpeg|gif|ico|js|css)$ {
location ~* \.(png|jpg|jpeg|svg|gif|ico|js|css)$ {
expires 1y;
log_not_found off;
}
location ~ (.woff|.tff) {
location ~* \.(woff|woff2|ttf|eot) {
add_header 'Access-Control-Allow-Origin' *;
}
......@@ -106,7 +110,7 @@ server {
fastcgi_param PATH_INFO $fastcgi_path_info;
}
location ~* \.(jpg|jpeg|gif|png|css|js|ico|xml)$ {
location ~* \.(jpg|jpeg|gif|png|css|js|ico|xml|woff|woff2|ttf|eot|svg)$ {
expires 5d;
}
......
#!/bin/sh
set -e
export DOCKER_RESOLVER=$(cat /etc/resolv.conf | grep -i '^nameserver' | head -n1 | cut -d ' ' -f2)
envsubst \$UPSTREAM_ENDPOINT,\$DOCKER_RESOLVER < /dev-ssr.conf.tpl > /etc/nginx/conf.d/dev.conf
cat /etc/nginx/conf.d/dev.conf
nginx -g "daemon off;"
version: "2.2"
services:
## EXTRA DEPENDENCY FOR WEB SERVER
nginx:
depends_on:
- front-live-server
## WEB APP
front-live-server:
build:
context: ./front/containers/live-server
dockerfile: ./Dockerfile
mem_limit: 2GB
depends_on:
- front-live-server-compiler
networks:
- app
volumes:
- ./front/:/var/www/Minds/front:cached
front-live-server-compiler:
build:
context: ./front/containers/live-server-compiler
dockerfile: ./Dockerfile
mem_limit: 2GB
networks:
- app
volumes:
- ./front/:/var/www/Minds/front:delegated
# PHPSpec docker-compose
version: "2.2"
services:
phpspec:
build:
context: ./engine
dockerfile: ./containers/phpspec/Dockerfile
volumes:
- ./front/:/var/www/Minds/front:cached
- ./engine/:/var/www/Minds/engine:cached
- keys:/.dev
networks:
- app
# Using v2.2 because v3+ is meant for swarm operations
# and removes health check conditions and memory limits
# Main docker-compose
# Suited for containerized SSR server.
version: "2.2"
services:
cassandra:
image: cassandra:3.11.1
environment:
- CASSANDRA_START_RPC=true
- MAX_HEAP_SIZE=768M
- HEAP_NEWSIZE=512M
networks:
- app
mem_limit: 1024MB
volumes:
- cassandra-data:/var/lib/cassandra:delegated
healthcheck:
test: "cqlsh -e 'DESC TABLE system.batches'"
retries: 10
timeout: 5s
interval: 15s
mongo:
image: mongo:3.4
networks:
- app
## WEB SERVER / GATEWAY
elasticsearch-legacy:
image: elasticsearch:5.6-alpine
entrypoint: /clear-data-and-start.sh
environment:
- "xpack.security.enabled=false"
- "http.host=0.0.0.0"
- "transport.host=127.0.0.1"
nginx:
build:
context: ./containers/nginx
dockerfile: ./Dockerfile.dev-ssr
mem_limit: 512MB
depends_on:
- php-fpm
networks:
- app
ports:
- "9200:9200"
volumes:
- elasticsearch-data:/usr/share/elasticsearch/data:delegated
- ./containers/elasticsearch-legacy/clear-data-and-start.sh:/clear-data-and-start.sh
elasticsearch:
image: elasticsearch:6.6.2
entrypoint: /set-permissions-and-start.sh
mem_limit: 512MB # keep an eye
ulimits:
nproc: 65536
memlock:
soft: 65536
hard: 65536
nofile:
soft: 65536
hard: 65536
- "8080:80"
environment:
- "ES_JAVA_OPTS=-Xms256m -Xmx256m"
networks:
- app
ports:
- "9200:9200"
- UPSTREAM_ENDPOINT
volumes:
- ./containers/elasticsearch/set-permissions-and-start.sh:/set-permissions-and-start.sh
- elasticsearch-data:/usr/share/elasticsearch/data:delegated
- ./front/:/var/www/Minds/front:cached
kibana:
image: docker.elastic.co/kibana/kibana-oss:6.6.2
## WEB APP
front-live-server:
build:
context: ./front/containers/live-server
dockerfile: ./Dockerfile
mem_limit: 2GB
depends_on:
- elasticsearch
environment:
ELASTICSEARCH_URL: http://minds_elasticsearch_1:9200
- front-live-server-compiler
networks:
- app
ports:
- "5601:5601"
volumes:
- ./front/:/var/www/Minds/front:delegated
redis:
image: redis:4.0-alpine
mem_limit: 100MB # keep an eye
front-live-server-compiler:
build:
context: ./front/containers/live-server-compiler
dockerfile: ./Dockerfile
mem_limit: 2GB
networks:
- app
volumes:
- ./front/:/var/www/Minds/front:delegated
rabbitmq:
image: rabbitmq:3.6-alpine
networks:
- app
# UNUSED (for now)
# - Disabled because docker complains if there's no dist/ folder
# front:
# # Static server
# build:
# context: ./front/dist
# dockerfile: ../containers/server/Dockerfile
# networks:
# - app
# volumes:
# - ./front/dist/:/dist:cached
## APP ENGINE
php-fpm:
build:
......@@ -95,13 +71,11 @@ services:
- elasticsearch
- rabbitmq
- redis
- sync-engine
networks:
- app
volumes:
## The following is for development environments only. Comment out on production. ##
- ./front/:/var/www/Minds/front:delegated
- ./engine:/var/www/Minds/engine:delegated
- ./front/:/var/www/Minds/front:cached
- ./engine/:/var/www/Minds/engine:cached
- keys:/.dev
runners:
......@@ -116,51 +90,89 @@ services:
networks:
- app
volumes:
- ./front/:/var/www/Minds/front:delegated
- "./plugins/:/var/www/Minds/plugins"
- "./languages/:/var/www/Minds/languages"
- ./engine:/var/www/Minds/engine:delegated
- ./front/:/var/www/Minds/front:cached
- ./engine/:/var/www/Minds/engine:cached
- keys:/.dev
front:
build:
context: ./front/dist
dockerfile: ../containers/server/Dockerfile
sockets:
image: minds/sockets
environment:
- PORT=3030
- REDIS_HOST=redis
- REDIS_PORT=6379
- JWT_SECRET=<!!! set this to jwt-token in your engine/settings.php !!!>
- CASSANDRA_SERVERS=cassandra
- CASSANDRA_KEYSPACE=minds
networks:
- app
volumes:
- ./front/dist:/dist:delegated
nginx:
build:
context: ./containers/nginx
dockerfile: ./Dockerfile.dev # Change to ./Dockerfile.dev-ssr to use SSR locally
mem_limit: 512MB
depends_on:
# - front # Uncomment out this line if you want to use SSR locally
- php-fpm
- redis
- cassandra
ports:
- "8080:80"
- 8010:3030
## DATABASES
cassandra:
image: cassandra:3.11.1
environment:
- CASSANDRA_START_RPC=true
- MAX_HEAP_SIZE=768M
- HEAP_NEWSIZE=512M
networks:
app:
aliases:
- minds.local
- app
mem_limit: 1024MB
volumes:
- ./front/:/var/www/Minds/front:delegated
- cassandra-data:/var/lib/cassandra
healthcheck:
test: "cqlsh -e 'DESC TABLE system.batches'"
retries: 10
timeout: 5s
interval: 15s
sync-engine:
build:
context: ./containers/alpine-rsync
command:
[
"watch",
"-n5",
"rsync -avz --no-perms --progress /engine-dir-host/ /engine-dir/ --exclude .git",
]
mongo:
image: mongo:3.4
networks:
- app
elasticsearch:
image: elasticsearch:6.6.2
entrypoint: /set-permissions-and-start.sh
mem_limit: 512MB # keep an eye
ulimits:
nproc: 65536
memlock:
soft: 65536
hard: 65536
nofile:
soft: 65536
hard: 65536
environment:
- "ES_JAVA_OPTS=-Xms256m -Xmx256m"
networks:
- app
ports:
- "9200:9200"
volumes:
- engine-dir:/engine-dir
- ./engine:/engine-dir-host
- ./containers/elasticsearch/set-permissions-and-start.sh:/set-permissions-and-start.sh:cached
- elasticsearch-data:/usr/share/elasticsearch/data
## CACHE
redis:
image: redis:4.0-alpine
mem_limit: 100MB # keep an eye
networks:
- app
## QUEUE
rabbitmq:
image: rabbitmq:3.6-alpine
networks:
- app
## INSTALLATION ARTIFACTS
installer:
build:
......@@ -169,17 +181,32 @@ services:
networks:
- app
volumes:
- "./engine/:/var/www/Minds/engine"
- "./front/:/var/www/Minds/front"
- keys:/.dev:delegated
- ./front/:/var/www/Minds/front:delegated
- ./engine/:/var/www/Minds/engine:delegated
- keys:/.dev
depends_on:
cassandra:
condition: service_healthy
elasticsearch-legacy:
image: elasticsearch:5.6-alpine
entrypoint: /clear-data-and-start.sh
environment:
- "xpack.security.enabled=false"
- "http.host=0.0.0.0"
- "transport.host=127.0.0.1"
networks:
- app
ports:
- "9200:9200"
volumes:
- ./containers/elasticsearch-legacy/clear-data-and-start.sh:/clear-data-and-start.sh:cached
- elasticsearch-data:/usr/share/elasticsearch/data
elasticsearch-legacy-provisioner:
build:
context: ./containers/elasticsearch-provisioner
entrypoint: sh /provisioner/provision-elasticsearch-legacy.sh
entrypoint: sh /provisioner/provision-elasticsearch-legacy.sh elasticsearch-legacy
networks:
- app
depends_on:
......@@ -195,28 +222,23 @@ services:
depends_on:
- elasticsearch
sockets:
image: minds/sockets
## UTILITIES
kibana:
image: docker.elastic.co/kibana/kibana-oss:6.6.2
depends_on:
- elasticsearch
environment:
- PORT=3030
- REDIS_HOST=redis
- REDIS_PORT=6379
- JWT_SECRET=<!!! set this to jwt-token in your engine/settings.php !!!>
- CASSANDRA_SERVERS=cassandra
- CASSANDRA_KEYSPACE=minds
ELASTICSEARCH_URL: http://minds_elasticsearch_1:9200
networks:
- app
depends_on:
- redis
- cassandra
ports:
- 8010:3030
- "5601:5601"
volumes:
cassandra-data:
elasticsearch-data:
keys:
engine-dir:
networks:
app:
......
node_modules/
package-lock.json
# Local Stack
## Requirements
- git
- docker 18 or higher, with docker-compose
- node 10.x or higher, with npm and npx
- port 8080 open
### Extra requirements for Windows
- Windows 10 Pro (needed for Docker)
- PowerShell
## Creating an alias
This is an optional step, but all examples in this document will be using the alias.
### Linux/macOS
Add to your ~/.bashrc (or ~/.zshrc) file
```sh
alias minds=/path/to/minds/local/local
alias minds-front-build=/path/to/minds/local/front-build
alias minds-ssr-build=/path/to/minds/local/ssr-build
alias minds-ssr-serve=/path/to/minds/local/ssr-serve
```
### Windows
Open PowerShell and run
```powershell
echo $profile
```
That command will output the location to your profile script. Edit it and add
```powershell
Set-Alias -Name minds -Value X:\Path\To\minds\local\local.ps1
Set-Alias -Name minds-front-build -Value X:\Path\To\minds\local\front-build.ps1
Set-Alias -Name minds-ssr-build -Value X:\Path\To\minds\local\ssr-build.ps1
Set-Alias -Name minds-ssr-serve -Value X:\Path\To\minds\local\ssr-serve.ps1
```
## Preparing your OS
### Linux
- Nothing to do.
### macOS
- Setup Docker VM to have at least 6.5GB and it uses at leas 2 CPUs.
### Windows
- Setup Docker VM to have at least 6.5GB and it uses at leas 2 CPUs.
- Enable Shared Drives availability to the drive that has the Minds repository (https://docs.docker.com/docker-for-windows/#file-sharing).
## Installing Minds
> **Important!**
>
> This operation will wipe out all your current data in the Minds containers.
>
> Ensure you run `docker-compose down` to dispose old Docker containers **before updating `master` or checking out this branch**.
Run
```sh
minds install
```
## Running
### Starting the containers
Run
```sh
minds up
```
### Stopping the containers
Run
```sh
minds down
```
### Restarting the containers
Run
```sh
minds restart
```
### Rebuilding the containers
After any infrastructure changes, run
```sh
minds rebuild
```
## Running the frontend stack
### Linux
#### App
Run
```sh
minds-front-build
```
#### SSR Server
SSR server runs inside two Docker containers: `front-live-server` and `front-live-server-compiler`.
To check out their activity, open a terminal in the `minds` directory and run
```sh
docker-compose logs -f --tail=40 front-live-server front-live-server-compiler
```
### macOS/Windows
#### App
Run
```sh
minds-front-build
```
#### SSR Server
Open two consoles and run in the first one:
```sh
minds-ssr-build
```
And in the seconds
```sh
minds-ssr-serve
```
The last one might show an error first, but it's normal as your computer might be re-building the server at the time your started it.
## PHPSpec
### Running test suite
Run
```sh
minds phpspec
```
#### Running a directory
Run
```sh
minds phpspec run --format=pretty --no-code-generation Spec/.../
```
#### Running a single file