Commit c62dacec authored by Andrew Newdigate's avatar Andrew Newdigate

Simplify Docker

parent fee829fd
......@@ -47,6 +47,18 @@ package:
- output/assets.tar.gz
expire_in: 1 week
testing:
<<: *node_job
variables:
BLUEBIRD_DEBUG: 1
stage: build_unit_test
services:
- registry.gitlab.com/gitlab-org/gitter/webapp/mongo:latest
- registry.gitlab.com/gitlab-org/gitter/webapp/elasticsearch:latest
- redis:3.0
script:
- make ci-test
docker-base:
variables:
DOCKER_DRIVER: overlay
......@@ -94,3 +106,32 @@ deploy_beta_staging:
url: https://beta.gitter.im
only:
- /^feature\/.*$/
mongo-base:
variables:
DOCKER_DRIVER: overlay
stage: docker_images
only:
- schedules
image: docker:latest
services:
- docker:dind
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
- docker build -t registry.gitlab.com/gitlab-org/gitter/webapp/mongo:latest scripts/docker/mongo-image/
- docker push registry.gitlab.com/gitlab-org/gitter/webapp/mongo:latest
elasticsearch-base:
variables:
DOCKER_DRIVER: overlay
stage: docker_images
only:
- schedules
image: docker:latest
services:
- docker:dind
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
- docker build -t registry.gitlab.com/gitlab-org/gitter/webapp/elasticsearch:latest scripts/docker/elasticsearch-image/
- docker push registry.gitlab.com/gitlab-org/gitter/webapp/elasticsearch:latest
......@@ -22,10 +22,9 @@ clean:
# If gulp clean failed, it's almost certainly a problem
# with the npm folder, so nuke it and try again
ci-test: clean
ci-test:
mkdir -p output/
gulp test --test-coverage --test-suite docker --test-xunit-reports
echo "Docker tests completed"
gulp test --test-coverage --test-suite docker --test-xunit-reports --test-bail
test: clean
mkdir -p output/
......
......@@ -84,20 +84,12 @@
"mongo": {
"connection": {
"hosts": [
"gitter-mongo-dev:27017",
"gitter-mongo-dev:27018",
"gitter-mongo-dev:27019"
"gitter-mongo-dev:27017"
],
"database": "gitter",
"options": {
"replicaSet": "troupeSet",
"w": 1
}
},
"autoDiscovery": {
"host": "gitter-mongo-dev",
"port": 27017
},
"profileSlowQueries": true,
"logQueries": false,
"noAutoIndex": true
......@@ -141,9 +133,9 @@
"basepath": "http://localhost:8061",
"cacheBuster": "1"
},
"redis": "redis-sentinel://gitter-redis-dev:26379?db=1&master=gitter-master-dev",
"redis_nopersist": "redis-sentinel://gitter-redis-dev:26379?db=1&master=gitter-master-dev",
"redis_caching": "redis-sentinel://gitter-redis-dev:26379?master=gitter-master-dev",
"redis": "redis://gitter-redis-dev?db=1",
"redis_nopersist": "redis://gitter-redis-dev?db=1",
"redis_caching": "redis://gitter-redis-dev",
"elasticsearch": {
"hosts": [
"http://gitter-es-dev:9200"
......
......@@ -3,13 +3,10 @@
"onlyUsePrimary": true,
"connection": {
"hosts": [
"mongo1:27017",
"mongo2:27017"
"registry.gitlab.com-gitlab-org-gitter-webapp-mongo:27017"
],
"database": "gitter",
"options": {
"replicaSet": "troupeSet",
"w": 1
}
},
"profileSlowQueries": true,
......@@ -33,13 +30,13 @@
"irc": {
"clientKey": "irc-dev"
},
"redis": "redis-sentinel://sentinel1:26379?db=1&master=gitter-master-dev",
"redis_nopersist": "redis-sentinel://sentinel1:26379?db=1&master=gitter-master-dev",
"redis_caching": "redis-sentinel://sentinel1:26379?master=gitter-master-dev",
"redis": "redis://redis?db=1",
"redis_nopersist": "redis://redis?db=1",
"redis_caching": "redis://redis",
"elasticsearch": {
"comment": "Override with GITTER_ES_ENDPOINT (comma separated)",
"hosts": [
"http://elasticsearch:9200"
"http://registry.gitlab.com-gitlab-org-gitter-webapp-elasticsearch:9200"
]
},
"test": {
......
......@@ -8,71 +8,11 @@ redis1:
- "6379:6379"
entrypoint: [ "/usr/local/bin/redis-server", "--bind", "0.0.0.0", "--port", "6379", "--maxclients", "200", "--dir", "/tmp", "--save", ""]
redis2:
image: redis:3.0.3
restart: always
ports:
- "6380:6380"
links:
- redis1
extra_hosts:
- gitter-redis-dev:${DOCKER_HOST_IP}
entrypoint: [ "/usr/local/bin/redis-server", "--bind", "0.0.0.0", "--port", "6380", "--slaveof", "gitter-redis-dev", "6379", "--maxclients", "200", "--dir", "/tmp", "--save", ""]
sentinel1:
image: redis:3.0.3
restart: 'on-failure:5'
links:
- redis1
- redis2
ports:
- "26379:26379"
extra_hosts:
- gitter-redis-dev:${DOCKER_HOST_IP}
entrypoint: [ "/bin/bash", "-c", "echo 'sentinel monitor gitter-master-dev gitter-redis-dev 6379 1' > /redis-sentinel.conf && exec /usr/local/bin/redis-server /redis-sentinel.conf --sentinel --bind 0.0.0.0 --maxclients 200"]
#
# Mongo
#
mongo2:
image: mongo:3.2
restart: always
entrypoint: [ "/usr/bin/mongod", "--replSet", "troupeSet", "--journal", "--smallfiles", "--storageEngine=wiredTiger", "--oplogSize=20", "--port=27018" ]
ports:
- "27018:27018"
mongo3:
image: mongo:3.2
restart: always
entrypoint: [ "/usr/bin/mongod", "--replSet", "troupeSet", "--journal", "--smallfiles", "--storageEngine=wiredTiger", "--oplogSize=20", "--port=27019" ]
ports:
- "27019:27019"
mongo1:
image: mongo:3.2
image: registry.gitlab.com/gitlab-org/gitter/webapp/mongo:latest
restart: always
entrypoint: [ "/usr/bin/mongod", "--replSet", "troupeSet", "--journal", "--smallfiles", "--storageEngine=wiredTiger", "--oplogSize=20" ]
ports:
- "27017:27017"
links:
- mongo2
- mongo3
mongosetup:
image: mongo:3.2
restart: 'no'
links:
- mongo1
- mongo2
- mongo3
volumes:
- ${WORKSPACE}:/src
environment:
ANNOUNCE_MONGO1_HOST: ${DOCKER_HOST_IP}
ANNOUNCE_MONGO2_HOST: ${DOCKER_HOST_IP}
ANNOUNCE_MONGO3_HOST: ${DOCKER_HOST_IP}
working_dir: /src
entrypoint: [ "/src/scripts/docker/mongo/mongo-setup.sh" ]
neo4j:
image: neo4j:2.3
......@@ -85,30 +25,9 @@ neo4j:
# The official elasticsearch:1.4.2 image with mapper-attachments and river-mongodb plugins
# https://github.com/soldotno/elasticsearch-river-mongodb/blob/master/Dockerfile
elasticsearch:
image: elasticsearch:1.4.2
image: registry.gitlab.com/gitlab-org/gitter/webapp/elasticsearch:latest
ports:
- "9200:9200"
- "9300:9300"
links:
- mongo1
- mongo2
- mongo3
volumes:
- ${WORKSPACE}:/src
environment:
ANNOUNCE_ES_HOST: ${DOCKER_HOST_IP}
working_dir: /src
entrypoint: [ "/src/scripts/docker/es/start-elasticsearch.sh" ]
elasticsearchsetup:
image: elasticsearch:1.4.2
restart: 'no'
links:
- mongo1
- mongo2
- mongo3
- elasticsearch:elasticsearch
volumes:
- ${WORKSPACE}:/src
working_dir: /src
entrypoint: [ "/src/scripts/docker/es/setup-elasticsearch.sh" ]
......@@ -8,25 +8,6 @@ set -euo pipefail
DOCKER_MACHINE_NAME=${DOCKER_MACHINE_NAME-default}
function has_docker_for_mac {
if [[ "$OSTYPE" == "linux-gnu" ]]; then
return 1
fi
# Is Docker for Mac running?
if (pgrep -f /Applications/Docker.app/Contents/MacOS/Docker > /dev/null); then
return 0
fi
return 1
}
if has_docker_for_mac; then
echo "The docker-compose setup is not compatible with Docker for Mac. Please install Docker Toolbox for Mac: https://docs.docker.com/toolbox/toolbox_install_mac/"
exit 1
fi
function has_local_docker_install {
if [[ "$OSTYPE" == "linux-gnu" ]]; then
return 0
......@@ -40,18 +21,9 @@ function has_local_docker_install {
return 1
}
get_local_ip() {
local _ip _line
while IFS=$': \t' read -a _line ;do
[ -z "${_line%inet}" ] &&
_ip=${_line[${#_line[1]}>4?1:2]} &&
[ "${_ip#127.0.0.1}" ] && echo $_ip && return 0
done< <(LANG=C /sbin/ifconfig)
}
function get_ip {
if has_local_docker_install; then
get_local_ip
echo "127.0.0.1"
else
echo "$(docker-machine ip $DOCKER_MACHINE_NAME)"
fi
......
......@@ -9,6 +9,6 @@
"private": true,
"dependencies": {
"bluebird": "^3.2.1",
"mongoose": "~4.10.0"
"mongoose": "~4.6.0"
}
}
......@@ -14,6 +14,7 @@
"gitter-web-env": "file:../env",
"lodash": "^3.2.0",
"mongodb": "~2.2.19",
"mongodb-unique-ids": "^0.2.0"
"mongodb-unique-ids": "^0.2.0",
"mongoose": "~4.6.0"
}
}
......@@ -16,7 +16,7 @@
"gitter-web-env": "file:../env",
"gitter-web-mongoose-bluebird": "file:../mongoose-bluebird",
"lodash": "^3.2.0",
"mongoose": "~4.10.0",
"mongoose": "~4.6.0",
"mongoose-number": "^0.1.1",
"node-mongodb-debug-log": "^0.1.2"
}
......
......@@ -4,13 +4,23 @@
// its nasty, but if jsmockito uses a different instance of the library
// it won't work correctly
function getHamcrest() {
try {
return require('jshamcrest').JsHamcrest
} catch (e) { /* do nothing */ }
try {
return require('jsmockito/node_modules/jshamcrest').JsHamcrest;
} catch (e) {
return require('jsmockito').JsHamcrest;
return require('jsmockito/node_modules/jshamcrest').JsHamcrest; // eslint-disable-line
} catch(e) { /* do nothing */ }
var hamcrest = require('jsmockito').JsHamcrest;
if (!hamcrest) {
throw new Error('Unable to obtain hamcrest instance')
}
return hamcrest;
}
var assert = require('assert');
var presenceService = require('..');
var mockito = require('jsmockito')
......
......@@ -27,6 +27,15 @@ var configurationMappings = {
GITTER_INTEGRATION_REPO_WITH_COLLAB: 'integrationTests:collabRepos:repo1',
GITTER_INTEGRATION_REPO_WITH_COLLAB2: 'integrationTests:collabRepos:repo2',
GITTER_INTEGRATION_REPO_WITH_COLLAB_ONLY_READ: 'integrationTests:collabRepos:repoReadOnly',
GITHUB_PRIVATE_CLIENT_ID: "github:client_id",
GITHUB_PRIVATE_CLIENT_SECRET: "github:client_secret",
GITHUB_USER_CLIENT_ID: "github:user_client_id",
GITHUB_USER_SECRET: "github:user_client_secret",
GITHUB_ANON_CLIENT_ID: "github:anonymous_app:client_id",
GITHUB_ANON_CLIENT_SECRET: "github:anonymous_app:client_secret",
TWITTER_CONSUMER_KEY: "twitteroauth__consumer_key",
TWITTER_CONSUMER_SECRET: "twitteroauth__consumer_secret",
}
var configSets = {
......@@ -37,6 +46,16 @@ var configSets = {
'#integrationCollabUser1': [
'GITTER_INTEGRATION_COLLAB_USERNAME',
'GITTER_INTEGRATION_COLLAB_USER_SCOPE_TOKEN'
],
'#oauthTokens': [
'GITHUB_PRIVATE_CLIENT_ID',
'GITHUB_PRIVATE_CLIENT_SECRET',
'GITHUB_USER_CLIENT_ID',
'GITHUB_USER_SECRET',
'GITHUB_ANON_CLIENT_ID',
'GITHUB_ANON_CLIENT_SECRET',
'TWITTER_CONSUMER_KEY',
'TWITTER_CONSUMER_SECRET',
]
}
......
FROM elasticsearch:1.4.2
COPY logging.yml /usr/share/elasticsearch/config/
COPY elasticsearch.yml /usr/share/elasticsearch/config/
COPY scripts /scripts/
RUN mkdir -p /data/es/logs /data/es/data /data/es/plugins
RUN /usr/share/elasticsearch/bin/elasticsearch -d -p /tmp/elasticsearch-pid \
&& sleep 2 \
&& /usr/share/elasticsearch/bin/plugin --install elasticsearch/elasticsearch-mapper-attachments/2.5.0 \
&& /usr/share/elasticsearch/bin/plugin --install com.github.richardwilly98.elasticsearch/elasticsearch-river-mongodb/2.0.9 \
&& /usr/share/elasticsearch/bin/plugin --install mobz/elasticsearch-head \
&& /usr/share/elasticsearch/bin/plugin --install royrusso/elasticsearch-HQ \
&& /scripts/setup-elasticsearch.sh \
&& kill -15 $(cat /tmp/elasticsearch-pid) \
&& sleep 5
# Make the new dir a VOLUME to persists it
VOLUME /data/es
EXPOSE 9200
EXPOSE 9300
CMD ["/usr/share/elasticsearch/bin/elasticsearch"]
cluster.name: "gitter-docker-cluster"
network.host: 0.0.0.0
path:
logs: /data/es/logs
data: /data/es/data
plugins: /data/es/plugins
rootLogger: INFO,console
appender:
console:
type: console
layout:
type: consolePattern
conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
#!/bin/bash
set -euo pipefail
set -x
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
if ! curl -q --fail -vi -XGET "$ES_URL/$INDEX_NAME/_settings"; then
curl -q --fail -vi -XPUT "$ES_URL/$INDEX_NAME?pretty=true" -d @- << EOD
{
"settings": {
"number_of_shards": 4,
"number_of_replicas" : 1,
"mapper": {
"dynamic": false
}
}
}
EOD
fi
sleep 0.5
## Update the language analyzers
curl -q --fail -XPOST "$ES_URL/$INDEX_NAME/_close"
for l in $SCRIPT_DIR/languages/*.json; do
curl -q --fail -vi -XPUT "$ES_URL/$INDEX_NAME/_settings?pretty=true" -d @$l
done
curl -q --fail -XPOST "$ES_URL/$INDEX_NAME/_open"
curl --fail -vi -XPUT "$ES_URL/$INDEX_NAME/_mapping/user?pretty=true" -d @"${SCRIPT_DIR}/mappings/user-mapping.json"
curl --fail -vi -XPUT "$ES_URL/$INDEX_NAME/_mapping/chat?pretty=true" -d @"${SCRIPT_DIR}/mappings/chat-mapping.json"
curl --fail -vi -XPUT "$ES_URL/$INDEX_NAME/_mapping/group?pretty=true" -d @"${SCRIPT_DIR}/mappings/group-mapping.json"
curl --fail -vi -XPUT "$ES_URL/$INDEX_NAME/_mapping/room?pretty=true" -d @"${SCRIPT_DIR}/mappings/room-mapping.json"
#!/bin/bash
set -euo pipefail
set -x
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
function inline {
cat $1| perl -p -e 's/\n/\\n/g;s/"/\\"/g;'
}
# source $SCRIPT_DIR/vars
curl -q --fail -vi -XPUT "$ES_URL/_river/$USER_RIVER/_meta?pretty=true" -d @- << EOD
{
"type": "mongodb",
"mongodb": {
"servers": [
{ "host": "mongo1", "port": 27017 }
],
"options": { },
"db": "gitter",
"collection": "users",
"scriptType": "groovy",
"script": "$(inline $SCRIPT_DIR/scripts/river/user.groovy)"
},
"index": {
"name": "$INDEX_NAME",
"type": "user"
}
}
EOD
sleep 0.5
curl -q --fail -vi -XPUT "$ES_URL/_river/$CHAT_RIVER/_meta?pretty=true" -d @- << EOD
{
"type": "mongodb",
"mongodb": {
"servers": [
{ "host": "mongo1", "port": 27017 }
],
"options": { },
"db": "gitter",
"collection": "chatmessages",
"scriptType": "groovy",
"script": "$(inline $SCRIPT_DIR/scripts/river/chat.groovy)"
},
"index": {
"name": "$INDEX_NAME",
"type": "chat"
}
}
EOD
sleep 0.5
curl -q --fail -vi -XPUT "$ES_URL/_river/$GROUP_RIVER/_meta?pretty=true" -d @- << EOD
{
"type": "mongodb",
"mongodb": {
"servers": [
{ "host": "mongo1", "port": 27017 }
],
"options": { },
"db": "gitter",
"collection": "groups",
"scriptType": "groovy",
"script": "$(inline $SCRIPT_DIR/scripts/river/group.groovy)"
},
"index": {
"name": "$INDEX_NAME",
"type": "group"
}
}
EOD
sleep 0.5
curl -q --fail -vi -XPUT "$ES_URL/_river/$ROOM_RIVER/_meta?pretty=true" -d @- << EOD
{
"type": "mongodb",
"mongodb": {
"servers": [
{ "host": "mongo1", "port": 27017 }
],
"options": { },
"db": "gitter",
"collection": "troupes",
"scriptType": "groovy",
"script": "$(inline $SCRIPT_DIR/scripts/river/room.groovy)"
},
"index": {
"name": "$INDEX_NAME",
"type": "room"
}
}
EOD
#!/bin/bash
set -euo pipefail
set -x
curl -q --fail -vi -XPOST "$ES_URL/_aliases?pretty=true" -d @- << EOD
{
"actions": [{
"add": {
"index": "$INDEX_NAME",
"alias": "gitter-primary"
}
}]
}
EOD
{
"analysis": {
"analyzer": {
"default": {
"type": "custom",
"tokenizer": "uax_url_email",
"filter": [
"lowercase",
"asciifolding"
]
}
}
}
}
{
"analysis": {
"filter": {
"english_stop": {
"type": "stop",
"stopwords": "_english_"
},
"english_stemmer": {
"type": "stemmer",
"language": "light_english"
},
"english_possessive_stemmer": {
"type": "stemmer",
"language": "possessive_english"
}
},
"analyzer": {
"analyzer-en": {
"type": "custom",
"tokenizer": "uax_url_email",
"filter": [
"english_possessive_stemmer",
"lowercase",
"english_stop",
"english_stemmer",
"asciifolding"
]
}
}
}
}
{
"chat": {
"_parent": {
"type": "user"
},
"dynamic" : "strict",
"properties": {
"text": {
"type": "string",
"index": "analyzed",
"term_vector": "with_positions_offsets",
"norms": {
"enabled": false
}
},
"lang": {
"type": "string",
"index": "not_analyzed"
},
"pub": {
"type": "boolean",
"index": "not_analyzed"
},
"sent": {
"type": "date"
},
"toTroupeId": {
"type": "string",
"index": "not_analyzed"
},
"fromUserId": {
"type": "string",
"index": "not_analyzed"
},
"_analyzer": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
\ No newline at end of file
{
"group": {
"dynamic" : "strict",
"properties": {
"uri": {
"type": "string",
"norms": {
"enabled": false
}
},
"name": {
"type": "string",
"norms": {
"enabled": false
}
},
"sd": {
"type": "nested",
"properties": {
"type": {
"type": "string",
"index": "not_analyzed"
},
"members": {
"type": "string",
"index": "not_analyzed"
},
"admins": {
"type": "string",