Commit d3c79e85 authored by Brian Hatchet's avatar Brian Hatchet 💬 Committed by Mark Harding

(feature) elastic search provisioner

parent c58b45cd
......@@ -22,12 +22,22 @@ Minds is split into multiple repositories:
**Enabling full installation**
By default, we try not to eat your machine by running the full stack. You'll be able to run the app, but you won't have search and indexing. If you need everthing, be sure to uncomment the ```depends_on``` services in
### Setting up elasticsearch
* runners
* php-fpm
** Linux users **
To get elasticsearch 6 to run, you must make a settings change on the host machine.
Then you can run:
Run ```sudo sysctl -w vm.max_map_count=262144```
To make it permanent, modify the variable in /etc/sysctl.conf
#### Build the elasticsearch indexes
1. Make sure nothing is running: `docker-compose down`
2. Run the legacy provisioner: `docker-compose up elasticsearch-legacy-provisioner`
3. Run the legacy provisioner: `docker-compose up elasticsearch-provisioner`
### Running the stack the first time
1. Run `sh init.sh` in order to install the front and engine repositories
2. Run `docker-compose up -d nginx`
......@@ -35,7 +45,7 @@ Then you can run:
4. Run `docker-compose up front-build`
5. Navigate to `http://localhost:8080`
### Troubleshooting
# Troubleshooting
- Minds is already installed
- Ensure engine/settings.php does not exist and re-run `docker-compose up installer`
......
#!/bin/bash
rm -rf /usr/share/elasticsearch/data/*
exec /docker-entrypoint.sh elasticsearch
\ No newline at end of file
FROM alpine:latest
RUN apk add curl docker --no-cache
WORKDIR /provisioner
COPY wait-for.sh provision-elasticsearch.sh provision-elasticsearch-legacy.sh /provisioner/
COPY schema-legacy/ /provisioner/schema-legacy/
COPY schema/ /provisioner/schema/
ENTRYPOINT ["sh", "./provision-elasticsearch.sh"]
\ No newline at end of file
#!/bin/bash
echo "Provisioning legacy elastic search";
echo "Waiting for legacy elastic search to come online..."
./wait-for.sh minds_elasticsearch-legacy_1:9200 --timeout=120 -- echo "Legacy elastic search is up and running"
echo "Putting mappings"
echo "Putting minds_badger"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds_badger -d @./schema-legacy/minds_badger.json --header "Content-Type: application/json"
echo "Putting minds-graph"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-graph -d @./schema-legacy/minds-graph.json --header "Content-Type: application/json"
echo "Putting minds-helpdesk"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-helpdesk-2019 -d @./schema-legacy/minds-helpdesk.json --header "Content-Type: application/json"
echo "Putting minds-kite"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-kite-2019 -d @./schema-legacy/minds-kite.json --header "Content-Type: application/json"
echo "Putting minds-metrics"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-metrics-06-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-metrics-07-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-metrics-08-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-metrics-09-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-metrics-10-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-metrics-11-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-metrics-12-2019 -d @./schema-legacy/minds-metrics.json --header "Content-Type: application/json"
echo "Putting minds-moderation"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-moderation -d @./schema-legacy/minds-moderation.json --header "Content-Type: application/json"
echo "Putting minds-trending-hashtags-shrunk"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch-legacy_1:9200/minds-trending-hashtags-shrunk -d @./schema-legacy/minds-trending-hashtags-shrunk.json --header "Content-Type: application/json"
echo "Stopping legacy elastic search container"
docker stop minds_elasticsearch-legacy_1
#!/bin/bash
echo "Provisioning elastic search";
echo "Waiting for elastic search to come online..."
./wait-for.sh minds_elasticsearch_1:9200 --timeout=120 -- echo "Elastic search is up and running"
echo "Putting mappings"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch_1:9200/minds-views -d @./schema/minds-views.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch_1:9200/minds-boost -d @./schema/minds-boost.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch_1:9200/minds-offchain -d @./schema/minds-offchain.json --header "Content-Type: application/json"
curl -s --write-out ' Status: %{http_code}\n' -X PUT http://minds_elasticsearch_1:9200/minds-transactions-onchain -d @./schema/minds-transactions-onchain.json --header "Content-Type: application/json"
echo "elastic search is ready!"
{
"mappings": {
"pass": {
"_all": {
"enabled": true
},
"properties": {
"guids": {
"type": "long"
},
"user_guid": {
"type": "long"
}
}
},
"subscriptions": {
"_all": {
"enabled": true
},
"properties": {
"guids": {
"type": "long"
},
"user_guid": {
"type": "long"
}
}
}
}
}
{
"mappings": {
"question": {
"_all": {
"enabled": true
},
"properties": {
"answer": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"category_uuid": {
"type": "text"
},
"question": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"score": {
"type": "integer"
},
"suggest": {
"type": "completion",
"analyzer": "simple",
"preserve_separators": true,
"preserve_position_increments": true,
"max_input_length": 50
},
"uuid": {
"type": "text"
}
}
}
}
}
{
"mappings": {
"user-state": {
"_all": {
"enabled": true
},
"properties": {
"activity_percentage": {
"type": "double"
},
"previous_state": {
"type": "text"
},
"reference_date": {
"type": "date"
},
"state": {
"type": "keyword"
},
"user_guid": {
"type": "long"
}
}
},
"active_user": {
"_all": {
"enabled": true
},
"properties": {
"activity_percentage": {
"type": "double"
},
"previous_state": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reference_date": {
"type": "date"
},
"state": {
"type": "keyword"
},
"user_guid": {
"type": "long"
}
}
}
}
}
{
"mappings": {
"action": {
"properties": {
"@timestamp": {
"type": "date"
},
"action": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ban_reason": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"boost_entity_guid": {
"type": "long"
},
"boost_quality": {
"type": "long"
},
"boost_rating": {
"type": "long"
},
"boost_reject_reason": {
"type": "long"
},
"boost_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"comment_guid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"cookie_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"email_campaign": {
"type": "keyword"
},
"email_state": {
"type": "keyword"
},
"email_topic": {
"type": "keyword"
},
"entity": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"entityGuid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"entityOwnerGuid": {
"type": "long"
},
"entitySubtype": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"entityType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"entity_access_id": {
"type": "long"
},
"entity_container_guid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"entity_guid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"entity_membership": {
"type": "long"
},
"entity_owner_guid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"entity_subtype": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"entity_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ip_addr": {
"type": "ip"
},
"ip_hash": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ip_range_hash": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"is_remind": {
"type": "boolean"
},
"logged_in": {
"type": "boolean"
},
"mobile_version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"platform": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"product": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ratelimit_key": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ratelimit_period": {
"type": "long"
},
"referrer_uri": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"route_uri": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_agent": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_guid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_phone_number_hash": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
{
"mappings": {
"reports": {
"dynamic": "false",
"properties": {
"@appeal_jury_decided_timestamp": {
"type": "date"
},
"@appeal_timestamp": {
"type": "date"
},
"@initial_jury_decided_timestamp": {
"type": "date"
},
"action": {
"type": "keyword"
},
"appeal_jury": {
"type": "nested",
"properties": {
"@timestamp": {
"type": "date"
},
"accepted": {
"type": "boolean"
},
"juror_guid": {
"type": "long"
},
"juror_hash": {
"type": "keyword"
}
}
},
"appeal_jury_action": {
"type": "keyword"
},
"appeal_note": {
"type": "keyword"
},
"entity_guid": {
"type": "long"
},
"entity_owner_guid": {
"type": "long"
},
"initial_jury": {
"type": "nested",
"properties": {
"@timestamp": {
"type": "date"
},
"accepted": {
"type": "boolean"
},
"juror_guid": {
"type": "long"
},
"juror_hash": {
"type": "keyword"
}
}
},
"initial_jury_action": {
"type": "keyword"
},
"reports": {
"type": "nested",
"properties": {
"@timestamp": {
"type": "date"
},
"reason": {
"type": "integer"
},
"reporter_guid": {
"type": "long"
},
"reporter_hash": {
"type": "keyword"
}
}
}
}
}
}
}
{
"mappings": {
"tags": {
"_all": {
"enabled": true
},
"properties": {
"count": {
"type": "long"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"suggest": {
"type": "completion",