Commit 61bdc733 authored by Matthias Larisch's avatar Matthias Larisch

Prepare Mail domain switch (including metrics)

parent 5d9ee60c
Pipeline #37925929 passed with stages
in 32 minutes and 54 seconds
......@@ -16,6 +16,7 @@
- it is now possible to open profiles in new tabs via middle mouse button !574 @peter.toennies
- Database functions can now be called with critera that contain arrays !559 @janopae
- Added a Rest controller for food baskets #345 !557 @alex.simm
- Allow platform mailing system to work with multiple email domains !583 @NerdyProjects
## Bugfixes
- Foodsavers list is now sorted by name and doesn't reshuffle !578 #54 @odedNea
......@@ -36,6 +37,7 @@
- Adding pictures to blog posts and Fairteiler work again !581 @NerdyProjects
- Redirect after joining a new region works again !581 @NerdyProjects
- Bell notifications for store fetch confirmations and for new fairteilers are now generated and stored in the database like normal ones #353 !559 @janopae
- More stability for internal email system as emails are handled like external ones now !583 @NerdyProjects
## Refactoring
......@@ -49,6 +51,7 @@
- Use webpack-dev-server instead of webpack-serve !582 @NerdyProjects
- updated webpack and switched to terser !584 @peter.toennies
- Updated whatwg-fetch to version 3 !585 @peter.toennies
- Gather statistics about incoming and outgoing emails !583 @NerdyProjects
# 2018-08-19 Hotfix
......
......@@ -54,7 +54,8 @@
"ext-gd": "*",
"ext-zlib": "*",
"ext-openssl": "*",
"ext-intl": "*"
"ext-intl": "*",
"influxdb/influxdb-php": "^1.14"
},
"autoload": {
"psr-4": {"Foodsharing\\": "src/",
......
This diff is collapsed.
......@@ -18,6 +18,8 @@ define('DB_DB', 'foodsharing');
define('ERROR_REPORT', E_ALL);
define('BASE_URL', $protocol . '://localhost:18080');
define('INFLUX_DSN', 'influxdb://influxdb:8086/foodsharing');
define('DEFAULT_EMAIL', 'noreply@lebensmittelretten.de');
define('SUPPORT_EMAIL', 'it@lebensmittelretten.de');
define('DEFAULT_EMAIL_NAME', 'Foodsharing Freiwillige');
......@@ -39,6 +41,10 @@ define('SOCK_URL', 'http://chat:1338/');
define('REDIS_HOST', 'redis');
define('REDIS_PORT', 6379);
define('IMAP', [
['host' => 'imap', 'user' => 'user', 'password' => 'pass']
]);
if (!defined('ROOT_DIR')) {
define('ROOT_DIR', './');
}
......@@ -20,6 +20,7 @@ define('DB_HOST', 'db');
define('DB_USER', 'root');
define('DB_PASS', 'root');
define('DB_DB', 'foodsharing');
define('INFLUX_DSN', 'influxdb://influxdb:8086/foodsharing');
define('ERROR_REPORT', E_ALL);
define('BASE_URL', $protocol . '://lmr.local/');
define('DEFAULT_EMAIL', 'noreply@lebensmittelretten.de');
......
......@@ -140,3 +140,7 @@ services:
- [ setAttribute, [!php/const PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION]]
- [ setAttribute, [!php/const PDO::MYSQL_ATTR_INIT_COMMAND, 'SET NAMES "utf8"']]
InfluxDB\Database:
factory: ['InfluxDB\Client', 'fromdsn']
arguments:
- !php/const INFLUX_DSN
FROM influxdb:1.6.4-alpine
COPY docker/conf/influxdb/influxdb.toml /etc/influxdb/influxdb.toml
RUN chmod 644 /etc/influxdb/influxdb.toml
reporting-disabled = true
bind-address = "0.0.0.0:8088"
[meta]
dir = "/var/lib/influxdb/meta"
retention-autocreate = true
logging-enabled = true
[data]
dir = "/var/lib/influxdb/data"
index-version = "inmem"
wal-dir = "/var/lib/influxdb/wal"
wal-fsync-delay = "0s"
query-log-enabled = true
cache-max-memory-size = 1073741824
cache-snapshot-memory-size = 26214400
cache-snapshot-write-cold-duration = "10m0s"
compact-full-write-cold-duration = "4h0m0s"
max-series-per-database = 1000000
max-values-per-tag = 100000
max-concurrent-compactions = 0
trace-logging-enabled = false
[coordinator]
write-timeout = "10s"
max-concurrent-queries = 0
query-timeout = "0s"
log-queries-after = "0s"
max-select-point = 0
max-select-series = 0
max-select-buckets = 0
[retention]
enabled = true
check-interval = "30m0s"
[shard-precreation]
enabled = true
check-interval = "10m0s"
advance-period = "30m0s"
[monitor]
store-enabled = true
store-database = "_internal"
store-interval = "10s"
[subscriber]
enabled = true
http-timeout = "30s"
insecure-skip-verify = false
ca-certs = ""
write-concurrency = 40
write-buffer-size = 1000
[http]
enabled = true
bind-address = ":8086"
auth-enabled = false
log-enabled = true
write-tracing = false
pprof-enabled = true
https-enabled = false
https-certificate = "/etc/ssl/influxdb.pem"
https-private-key = ""
max-row-limit = 0
max-connection-limit = 0
shared-secret = ""
realm = "InfluxDB"
unix-socket-enabled = false
bind-socket = "/var/run/influxdb.sock"
[[graphite]]
enabled = false
bind-address = ":2003"
database = "graphite"
retention-policy = ""
protocol = "tcp"
batch-size = 5000
batch-pending = 10
batch-timeout = "1s"
consistency-level = "one"
separator = "."
udp-read-buffer = 0
[[collectd]]
enabled = false
bind-address = ":25826"
database = "collectd"
retention-policy = ""
batch-size = 5000
batch-pending = 10
batch-timeout = "10s"
read-buffer = 0
typesdb = "/usr/share/collectd/types.db"
security-level = "none"
auth-file = "/etc/collectd/auth_file"
[[opentsdb]]
enabled = false
bind-address = ":4242"
database = "opentsdb"
retention-policy = ""
consistency-level = "one"
tls-enabled = false
certificate = "/etc/ssl/influxdb.pem"
batch-size = 1000
batch-pending = 5
batch-timeout = "1s"
log-point-errors = true
[[udp]]
enabled = false
bind-address = ":8089"
database = "udp"
retention-policy = ""
batch-size = 5000
batch-pending = 10
read-buffer = 0
batch-timeout = "1s"
precision = ""
[continuous_queries]
log-enabled = true
enabled = true
run-interval = "1s"
......@@ -38,6 +38,7 @@ services:
- redis:redis
- db:db
- chat:chat
- influxdb
environment:
FS_ENV: test
volumes:
......@@ -137,8 +138,21 @@ services:
image: djfarrelly/maildev
container_name: foodsharing_ci_maildev
influxdb:
container_name: foodsharing_ci_influxdb
build:
context: ..
dockerfile: docker/Dockerfile.influxdb.ci
image: influxdb:1.6.4-alpine
command: >
influxd -config /etc/influxdb/influxdb.toml
environment:
INFLUXDB_ADMIN_ENABLED: "true"
INFLUXDB_DB: "foodsharing"
volumes:
selenium_downloads:
vendor_cache:
client_node_modules_cache:
chat_node_modules_cache:
......@@ -34,6 +34,7 @@ services:
- db
- redis
- mailqueuerunner
- influxdb
volumes:
- ../:/app
- ${PWD}/docker/conf/app/php.dev.ini:/usr/local/etc/php/conf.d/dev-config.ini
......@@ -216,6 +217,29 @@ services:
ports:
- 18084:80
influxdb:
container_name: foodsharing_dev_influxdb
image: influxdb:1.6.4-alpine
command: >
influxd -config /etc/influxdb/influxdb.toml
ports:
- 8083:8083
- 18086:8086
environment:
INFLUXDB_ADMIN_ENABLED: "true"
INFLUXDB_DB: "foodsharing"
volumes:
- influxdb:/var/lib/influxdb
- ./conf/influxdb/influxdb.toml:/etc/influxdb/influxdb.toml
grafana:
image: grafana/grafana:5.3.2
ports:
- 4000:3000
volumes:
- grafana:/var/lib/grafana
volumes:
client-node-modules:
......@@ -224,3 +248,5 @@ volumes:
api-virtualenv:
api-pip-cache:
chat-node-modules:
influxdb:
grafana:
......@@ -32,6 +32,7 @@ services:
- redis:redis
- db:db
- chat:chat
- influxdb
depends_on:
- db
- redis
......@@ -166,6 +167,21 @@ services:
ports:
- 28083:80
influxdb:
container_name: foodsharing_test_influxdb
image: influxdb:1.6.4-alpine
command: >
influxd -config /etc/influxdb/influxdb.toml
ports:
- 28086:8086
environment:
INFLUXDB_ADMIN_ENABLED: "true"
INFLUXDB_DB: "foodsharing"
volumes:
- influxdb:/var/lib/influxdb
- ./conf/influxdb/influxdb.toml:/etc/influxdb/influxdb.toml
volumes:
selenium_downloads:
influxdb:
......@@ -3,6 +3,8 @@
use Foodsharing\FoodsharingKernel;
use Symfony\Component\DependencyInjection\Container;
$script_start_time = microtime(true);
require __DIR__ . '/../vendor/autoload.php';
\Doctrine\Common\Annotations\AnnotationRegistry::registerLoader('class_exists');
......
......@@ -21,7 +21,7 @@ SECONDS=0
pids=()
for name in app chat web db; do
for name in app chat web db influxdb; do
dc build $name &
pids+=($!)
done
......
#!/bin/bash
export FS_ENV=dev
set -e
. $(dirname "$0")/inc.sh
PARAMS="$@"
dc exec -T --user 33:33 app sh -c "HOME=./ php -f run.php $PARAMS"
......@@ -4,6 +4,7 @@ namespace Foodsharing\Lib\Cache;
use Foodsharing\Lib\Db\Mem;
use Foodsharing\Lib\Session;
use Foodsharing\Modules\Core\InfluxMetrics;
class Caching
{
......@@ -11,20 +12,25 @@ class Caching
private $cacheMode;
private $session;
private $mem;
private $metrics;
public function __construct($cache_rules, Session $session, Mem $mem)
public function __construct($cache_rules, Session $session, Mem $mem, InfluxMetrics $metrics)
{
$this->session = $session;
$this->mem = $mem;
$this->cacheRules = $cache_rules;
$this->cacheMode = $this->session->may() ? 'u' : 'g';
$this->metrics = $metrics;
}
public function lookup()
{
if (isset($this->cacheRules[$_SERVER['REQUEST_URI']][$this->cacheMode]) && ($page = $this->mem->getPageCache()) !== false && !isset($_GET['flush'])) {
$this->metrics->addPageStatData(['cached' => 1]);
echo $page;
exit();
} else {
$this->metrics->addPageStatData(['cached' => 0]);
}
}
......
......@@ -10,6 +10,7 @@ use Foodsharing\Lib\Db\Mem;
use Foodsharing\Lib\Mail\AsyncMail;
use Foodsharing\Lib\View\Utils;
use Foodsharing\Modules\Core\DBConstants\Region\Type;
use Foodsharing\Modules\Core\InfluxMetrics;
use Foodsharing\Modules\EmailTemplateAdmin\EmailTemplateGateway;
use Foodsharing\Modules\Region\RegionGateway;
use Foodsharing\Services\SanitizerService;
......@@ -58,16 +59,23 @@ class Func
*/
private $mem;
/**
* @var InfluxMetrics
*/
private $metrics;
public function __construct(
Utils $viewUtils,
SanitizerService $sanitizerService,
RegionGateway $regionGateway,
EmailTemplateGateway $emailTemplateGateway
EmailTemplateGateway $emailTemplateGateway,
InfluxMetrics $metrics
) {
$this->viewUtils = $viewUtils;
$this->sanitizerService = $sanitizerService;
$this->regionGateway = $regionGateway;
$this->emailTemplateGateway = $emailTemplateGateway;
$this->metrics = $metrics;
$this->content_main = '';
$this->content_right = '';
$this->content_left = '';
......@@ -678,6 +686,7 @@ Verantwortlich für den Inhalt nach § 55 Abs. 2 RStV:<br />
$mail->addRecipient($to);
$mail->send();
$this->metrics->addPoint('outgoing_email', ['template' => $tpl_id], ['count' => 1]);
}
public function dt($ts)
......
......@@ -59,6 +59,11 @@ abstract class Control
*/
private $foodsaverGateway;
/**
* @var InfluxMetrics
*/
private $metrics;
public function __construct()
{
global $container;
......@@ -68,6 +73,7 @@ abstract class Control
$this->v_utils = $container->get(Utils::class);
$this->legacyDb = $container->get(Db::class);
$this->foodsaverGateway = $container->get(FoodsaverGateway::class);
$this->metrics = $container->get(InfluxMetrics::class);
$reflection = new ReflectionClass($this);
$dir = dirname($reflection->getFileName()) . DIRECTORY_SEPARATOR;
......@@ -126,6 +132,7 @@ abstract class Control
}
}
$this->mem->updateActivity($this->session->id());
$this->metrics->addPageStatData(['controller' => $className]);
}
/**
......
<?php
namespace Foodsharing\Modules\Core;
use InfluxDB\Point;
class InfluxMetrics
{
private $influxdb;
private $points;
private $pageStatTags;
private $pageStatFields;
public function __construct(\InfluxDB\Database $influxdb)
{
$this->influxdb = $influxdb;
$this->points = [];
$this->pageStatTags = [];
$this->pageStatFields = [];
}
public function __destruct()
{
$this->generatePageStatistics();
$this->flush();
}
/**
* adds a point.
*
* @param string measurement name of the measurement
* @param array fields array of field => value for field values
* @param array tags array of tag => value for tag values
*/
public function addPoint($measurement, $tags = [], $fields = [])
{
$this->points[] = new Point($measurement,
null,
$tags,
$fields);
}
/**
* writes all collected points to influxDb.
*/
public function flush()
{
$this->influxdb->writePoints($this->points);
$this->points = [];
}
/**
* adds tags and fields used for per-execution statistics.
* Also enables generation of execution statistics as soon as this is called the first time.
*
* @param array $tags
* @param array $fields
*/
public function addPageStatData($tags = [], $fields = [])
{
$this->pageStatTags += $tags;
$this->pageStatFields += $fields;
}
private function generatePageStatistics()
{
global $script_start_time;
if (isset($script_start_time)) {
$now = microtime(true);
$executionTime = $now - $script_start_time;
$this->addPageStatData([], ['execution_time' => $executionTime]);
$this->addPoint('page', $this->pageStatTags, $this->pageStatFields);
}
}
}
......@@ -8,6 +8,7 @@ use Flourish\fFile;
use Flourish\fSMTP;
use Foodsharing\Modules\Console\ConsoleControl;
use Foodsharing\Modules\Core\Database;
use Foodsharing\Modules\Core\InfluxMetrics;
use Foodsharing\Modules\Mailbox\MailboxModel;
class MailsControl extends ConsoleControl
......@@ -19,8 +20,9 @@ class MailsControl extends ConsoleControl
public static $last_connect;
private $mailboxModel;
private $database;
private $metrics;
public function __construct(MailsModel $model, MailboxModel $mailboxModel, Database $database)
public function __construct(MailsModel $model, MailboxModel $mailboxModel, Database $database, InfluxMetrics $metrics)
{
echo "creating mailscontrl!!!!\n";
error_reporting(E_ALL);
......@@ -29,6 +31,7 @@ class MailsControl extends ConsoleControl
$this->model = $model;
$this->mailboxModel = $mailboxModel;
$this->database = $database;
$this->metrics = $metrics;
parent::__construct();
echo "-------------------------------------\n";
}
......@@ -56,16 +59,25 @@ class MailsControl extends ConsoleControl
}
}
public function fetchMails()
{
foreach (IMAP as $imap) {
$stats = $this->mailboxupdate($imap['host'], $imap['user'], $imap['password']);
$this->metrics->addPoint('fetch_mails', ['account' => $imap['user']], $stats);
}
}
/**
* This Method will check for new E-Mails and sort it to the mailboxes.
*/
public function mailboxupdate()
public function mailboxupdate($host, $user, $password)
{
$server = new Server(IMAP_HOST);
$connection = $server->authenticate(IMAP_USER, IMAP_PASS);
$server = new Server($host);
$connection = $server->authenticate($user, $password);
$mailbox = $connection->getMailbox('INBOX');
$messages = $mailbox->getMessages();
$stats = ['unknown-recipient' => 0, 'failure' => 0, 'delivered' => 0, 'has-attachment' => 0];
if (count($messages) > 0) {
self::info(count($messages) . ' in Inbox');
......@@ -94,6 +106,7 @@ class MailsControl extends ConsoleControl
if (!$mb_ids) {
$mb_ids = $this->model->getMailboxIds(array('lost'));
++$stats['unknown-recipient'];
}
if ($mb_ids) {
......@@ -119,6 +132,7 @@ class MailsControl extends ConsoleControl
$body = $text;
$html = nl2br($this->func->autolink($text));
} else {
++$stats['failure'];
continue;
}
}
......@@ -147,23 +161,31 @@ class MailsControl extends ConsoleControl
}
}
$attach = json_encode($attach);
if ($attach) {
++$stats['has-attachment'];
}
$date = null;
try {
$date = $msg->getDate();
} catch (\Exception $e) {
echo 'Error parsing date: ' . $e->getMessage() . ", continuing with 'now'\n";
}
if ($date === null) {
$date = new \DateTime();
}
$md = $date->format('Y-m-d H:i:s') . ':' . $msg->getSubject();
$delivered = false;
foreach ($mb_ids as $id) {
if (!isset($have_send[$id])) {
$have_send[$id] = [];
}
$date = null;
try {
$date = $msg->getDate();
} catch (\Exception $e) {
echo 'Error parsing date: ' . $e->getMessage() . ", continuing with 'now'\n";
}
if ($date === null) {
$date = new \DateTime();
}
$md = $date->format('Y-m-d H:i:s') . ':' . $msg->getSubject();
if (!isset($have_send[$id][$md])) {
$delivered = true;
$have_send[$id][$md] = true;
$from = [];
$from['mailbox'] = $msg->getFrom()->getMailbox();
......@@ -190,6 +212,11 @@ class MailsControl extends ConsoleControl
);
}
}
if ($delivered) {
++$stats['delivered'];
} else {
++$stats['failure'];
}
}
$msg->delete();
......@@ -203,6 +230,8 @@ class MailsControl extends ConsoleControl
echo "\n";
self::success('ready :o)');
}
return $stats;
}
private function getMailAddressParts($str)
......@@ -289,49 +318,16 @@ class MailsControl extends ConsoleControl
}
$has_recip = false;
foreach ($data['recipients'] as $r) {
// check is it own lmr email? put direct into db
$r[0] = strtolower($r[0]);
self::info(substr(
$r[0],
(strlen(DEFAULT_EMAIL_HOST) * -1),
strlen(DEFAULT_EMAIL_HOST)
));
if (
substr(
$r[0],
(strlen(DEFAULT_EMAIL_HOST) * -1),
strlen(DEFAULT_EMAIL_HOST)
) == DEFAULT_EMAIL_HOST
) {
self::info($r[0] . ' own host save direct into db');
$mailbox = str_replace('@' . DEFAULT_EMAIL_HOST, '', $r[0]);
$mb_id = $this->model->getMailboxId($mailbox);
if (!$mb_id) {
// lost mailbox id
$mb_id = 25631;
}
$toarr = array();
foreach ($data['recipients'] as $r) {
$toarr[] = self::parseEmailAddress($r[0], $r[1]);
}
$this->model->saveMessage(
$mb_id, // mailbox id
1, // folder inbox
json_encode(self::parseEmailAddress($data['from'][0], $data['from'][1])), // sender
json_encode($toarr), // to
$data['subject'], // subject
$data['body'],
$data['html'],
date('Y-m-d H:i:s') // time,
);
} else {
$email->addRecipient($r[0], $r[1]);
$has_recip = true;
self::info($r[0]);
$address = explode('@', $r[0]);
if (count($address) != 2) {
self::error('invalid address');
continue;
}
$email->addRecipient($r[0], $r[1]);
$has_recip = true;
}
if (!$has_recip) {
return true;
......
......@@ -13,6 +13,7 @@ $I->login($user['email'], $pass);
// request mail with link
$I->amOnPage('/?page=settings&sub=general');
$I->click('E-Mail-Adresse ändern');
$I->waitForElementVisible('#newmail');
$I->fillField('#newmail', $newmail);
$I->executeJS("$('button:contains(E-Mail-Adresse ändern)').click()");
$I->waitForElementVisible('#pulse-info', 4);
......@@ -28,6 +29,7 @@ $link = $matches[1];
// open link, fill in password and submit
$I->amOnPage($link);
$I->waitForElementVisible('#passcheck');
$I->fillField('#passcheck', $pass);
$I->executeJS("$('button:contains(Bestätigen)').click()");
$I->waitForElementVisible('#pulse-info', 4);
......
......@@ -35,6 +35,8 @@ if (isset($_GET['f'])) {
$xhr = $container->get(XhrMethods::class);
$func = 'xhr_' . $action;
if (method_exists($xhr, $func)) {
$metrics = $container->get(\Foodsharing\Modules\Core\InfluxMetrics::class);
$metrics->addPageStatData(['controller' => $func]);
/*
* check for page caching
*/
......
Markdown is supported
0% or