.gitlab-ci.yml

@@ -9,6 +9,7 @@ stages:
   - prepare
   - review
   - deploy:staging
+  - qa
   - deploy:canary
   - deploy:production
 
@@ -22,7 +23,7 @@ build:
   stage: build
   script:
     - apk update && apk add --no-cache git
-    - sh tools/setup.sh
+    - sh tools/setup.sh production
   cache:
     paths:
       - vendor
@@ -127,6 +128,17 @@ review:stop:
       - master
       - test/gitlab-ci
 
+qa:manual:
+  stage: qa
+  script:
+    - echo "Manually approved"
+  when: manual
+  only:
+    refs:
+      - master
+      - test/gitlab-ci
+  allow_failure: false
+
 staging:fpm:
   stage: deploy:staging
   image: minds/ci:latest

Controllers/Cli/EntityCentric.php

+<?php
+
+namespace Minds\Controllers\Cli;
+
+use Minds\Core;
+use Minds\Core\Analytics\EntityCentric\Manager;
+use Minds\Cli;
+use Minds\Interfaces;
+use Minds\Exceptions;
+use Minds\Entities;
+
+class EntityCentric extends Cli\Controller implements Interfaces\CliControllerInterface
+{
+    public function __construct()
+    {
+    }
+
+    public function help($command = null)
+    {
+        $this->out('TBD');
+    }
+    
+    public function exec()
+    {
+        $this->out('Missing subcommand');
+    }
+
+    public function sync()
+    {
+        error_reporting(E_ALL);
+        ini_set('display_errors', 1);
+
+        $daysAgo = $this->getOpt('daysAgo') ?: 0;
+        $from = $this->getOpt('from') ?: strtotime("midnight $daysAgo days ago");
+        $manager = new Manager();
+        $manager->setFrom($from);
+
+        $i = 0;
+        foreach ($manager->sync() as $record) {
+            $this->out(++$i);
+        }
+    }
+}

Controllers/Cli/User.php

@@ -50,4 +50,35 @@ class User extends Cli\Controller implements Interfaces\CliControllerInterface
             $this->out("Set feature flags for {$user->username}: " . implode(', ', $features));
         }
     }
+
+    /**
+     * Resets a users passwords.
+     * Requires username and password.
+     *
+     * Example call: php ./cli.php User password_reset --username=nemofin --password=password123
+     * @return void
+     */
+    public function password_reset()
+    {
+        try {
+            if (!$this->getOpt('username') || !$this->getOpt('password')) {
+                throw new Exceptions\CliException('Missing username / password');
+            }
+
+            $username = $this->getOpt('username');
+            $password = $this->getOpt('password');
+
+            $user = new Entities\User($username);
+        
+            $user->password = Core\Security\Password::generate($user, $password);
+            $user->password_reset_code = "";
+            $user->override_password = true;
+            $user->save();
+
+            $this->out("Password changed successfuly for user ".$username);
+        } catch (Exception $e) {
+            $this->out("An error has occured");
+            $this->out($e);
+        }
+    }
 }

Controllers/api/v1/authenticate.php

@@ -119,6 +119,7 @@ class authenticate implements Interfaces\Api, Interfaces\ApiIgnorePam
 
     public function delete($pages)
     {
+        /** @var Core\Sessions\Manager $sessions */
         $sessions = Di::_()->get('Sessions\Manager');
 
         if (isset($pages[0]) && $pages[0] === 'all') {

Controllers/api/v1/blog.php

@@ -362,7 +362,7 @@ class blog implements Interfaces\Api
         }
 
         if ($saved) {
-            if ($blog->isPublished() && $blog->getAccessId() == Access::PUBLIC) {
+            if ($blog->isPublished() && in_array($blog->getAccessId(), [Access::PUBLIC, Access::LOGGED_IN], false)) {
                 if (!$editing || ($editing && !$alreadyPublished) || ($editing && $oldAccessId == Access::UNLISTED)) {
                     (new CreateActivity())->save($blog);
                 }

Controllers/api/v1/comments.php

@@ -347,15 +347,22 @@ class comments implements Interfaces\Api
 
         $comment = $manager->getByLuid($pages[0]);
 
-        if ($comment && $comment->canEdit()) {
+        if (!$comment) {
+            return Factory::response([
+                'status' => 'error',
+                'message' => 'Comment not found',
+            ]);
+        }
+
+        if ($comment->canEdit()) {
             $manager->delete($comment);
             return Factory::response([]);
         }
         //check if owner of activity trying to remove
         $entity = Entities\Factory::build($comment->getEntityGuid());
-        
+
         if ($entity->owner_guid == Core\Session::getLoggedInUserGuid()) {
-            $manager->delete($comment, [ 'force' => true ]);
+            $manager->delete($comment, ['force' => true]);
             return Factory::response([]);
         }
 

Controllers/api/v1/media.php

@@ -38,10 +38,10 @@ class media implements Interfaces\Api, Interfaces\ApiIgnorePam
 
             switch ($entity->subtype) {
                 case "video":
-                    Helpers\Counters::increment($pages[0], 'plays');
+                    // Helpers\Counters::increment($pages[0], 'plays');
 
                     if (isset($pages[1]) && $pages[1] == 'play') {
-                        http_response_code(301);
+                        http_response_code(302);
 
                         if ($entity->subtype == 'audio') {
                             \forward($entity->getSourceUrl('128.mp3'));

Controllers/api/v1/newsfeed.php

@@ -514,9 +514,6 @@ class newsfeed implements Interfaces\Api
                     $activity->indexes = ["activity:$activity->owner_guid:edits"]; //don't re-index on edit
                     (new Core\Translation\Storage())->purge($activity->guid);
 
-                    $attachmentPaywallDelegate = new Core\Feeds\Activity\Delegates\AttachmentPaywallDelegate();
-                    $attachmentPaywallDelegate->onUpdate($activity);
-
                     if (isset($_POST['time_created']) && ($_POST['time_created'] != $activity->getTimeCreated())) {
                         try {
                             $timeCreatedDelegate = new Core\Feeds\Activity\Delegates\TimeCreatedDelegate();
@@ -532,6 +529,8 @@ class newsfeed implements Interfaces\Api
                     $save->setEntity($activity)
                         ->save();
 
+                    (new Core\Entities\PropagateProperties())->from($activity);
+
                     $activity->setExportContext(true);
                     return Factory::response(['guid' => $activity->guid, 'activity' => $activity->export(), 'edited' => true]);
                 }

Controllers/api/v2/analytics/views.php

@@ -67,46 +67,49 @@ class views implements Interfaces\Api
                 ]);
                 break;
             case 'activity':
-                $activity = new Entities\Activity($pages[1]);
+            case 'entity':
+                $entity = Entities\Factory::build($pages[1]);
 
-                if (!$activity->guid) {
+                if (!$entity) {
                     return Factory::response([
                         'status' => 'error',
-                        'message' => 'Could not find activity post'
+                        'message' => 'Could not the entity'
                     ]);
                 }
 
-                try {
-                    Core\Analytics\App::_()
+                if ($entity->type === 'activity') {
+                    try {
+                        Core\Analytics\App::_()
                         ->setMetric('impression')
-                        ->setKey($activity->guid)
+                        ->setKey($entity->guid)
                         ->increment();
 
-                    if ($activity->remind_object) {
-                        Core\Analytics\App::_()
+                        if ($entity->remind_object) {
+                            Core\Analytics\App::_()
                             ->setMetric('impression')
-                            ->setKey($activity->remind_object['guid'])
+                            ->setKey($entity->remind_object['guid'])
                             ->increment();
 
-                        Core\Analytics\App::_()
+                            Core\Analytics\App::_()
                             ->setMetric('impression')
-                            ->setKey($activity->remind_object['owner_guid'])
+                            ->setKey($entity->remind_object['owner_guid'])
                             ->increment();
-                    }
+                        }
 
-                    Core\Analytics\User::_()
+                        Core\Analytics\User::_()
                         ->setMetric('impression')
-                        ->setKey($activity->owner_guid)
+                        ->setKey($entity->owner_guid)
                         ->increment();
-                } catch (\Exception $e) {
-                    error_log($e->getMessage());
+                    } catch (\Exception $e) {
+                        error_log($e->getMessage());
+                    }
                 }
 
                 try {
                     $viewsManager->record(
                         (new Core\Analytics\Views\View())
-                            ->setEntityUrn($activity->getUrn())
-                            ->setOwnerGuid((string) $activity->getOwnerGuid())
+                            ->setEntityUrn($entity->getUrn())
+                            ->setOwnerGuid((string) $entity->getOwnerGuid())
                             ->setClientMeta($_POST['client_meta'] ?? [])
                     );
                 } catch (\Exception $e) {
@@ -114,7 +117,7 @@ class views implements Interfaces\Api
                 }
 
                 Di::_()->get('Referrals\Cookie')
-                    ->setEntity($activity)
+                    ->setEntity($entity)
                     ->create();
 
                 break;

Controllers/api/v2/blockchain/wallet.php

@@ -25,6 +25,8 @@ class wallet implements Interfaces\Api
      */
     public function get($pages)
     {
+        Factory::isLoggedIn();
+
         /** @var abstractCacher $cache */
         $cache = Di::_()->get('Cache');
 

Controllers/api/v2/boost/feed.php

@@ -124,10 +124,14 @@ class feed implements Interfaces\Api
                     //     $next = 0;
                     // }
                     $len = count($boosts);
-                    $next = $boosts[$len -1]->getTimestamp();
+                    if ($boosts[$len -1]) {
+                        $next = $boosts[$len -1]->getTimestamp();
+                    }
                 } elseif ($isBoostFeed) {
                     $len = count($boosts);
-                    $next = $boosts[$len -1]->getTimestamp();
+                    if ($boosts[$len -1]) {
+                        $next = $boosts[$len -1]->getTimestamp();
+                    }
                 }
 
                 // $ttl = 1800; // 30 minutes

Controllers/api/v2/payments/stripe/transactions.php

@@ -28,7 +28,14 @@ class transactions implements Interfaces\Api
                 'message' => 'There was an error returning the usd account',
             ]);
         }
- 
+
+        if (!$account) {
+            return Factory::response([
+                'status' => 'error',
+                'message' => 'Stripe account not found',
+            ]);
+        }
+
         $transactionsManger = new Stripe\Transactions\Manager();
         $transactions = $transactionsManger->getByAccount($account);
 

Controllers/api/v2/settings/emails.php

@@ -13,6 +13,12 @@ class emails implements Interfaces\Api
     public function get($pages)
     {
         $user = Core\Session::getLoggedInUser();
+        if (!$user) {
+            return Factory::response([
+                'status' => 'error',
+                'message' => 'User must be logged in.'
+            ]);
+        }
 
         $campaigns = [ 'when', 'with', 'global' ];
 

Core/Analytics/EntityCentric/ActiveUsersSynchroniser.php

+<?php
+namespace Minds\Core\Analytics\EntityCentric;
+
+use Minds\Core\Analytics\Metrics\Active;
+use DateTime;
+use Exception;
+
+class ActiveUsersSynchroniser
+{
+    /** @var array */
+    private $records = [];
+
+    /** @var Active */
+    private $activeMetric;
+
+    public function __construct($activeMetric = null)
+    {
+        $this->activeMetric = $activeMetric ?? new Active();
+    }
+
+    /**
+     * @param int $from
+     * @return self
+     */
+    public function setFrom($from): self
+    {
+        $this->from = $from;
+        return $this;
+    }
+
+    /**
+     * Convert to records
+     * @return iterable
+     */
+    public function toRecords(): iterable
+    {
+        $date = (new DateTime())->setTimestamp($this->from);
+        $now = new DateTime();
+        $days = (int) $date->diff($now)->format('%a');
+        $months = round($days / 28);
+
+        // Daily resolution
+        foreach ($this->activeMetric->get($days ?: 1) as $bucket) {
+            $record = new EntityCentricRecord();
+            $record->setEntityUrn("urn:metric:global")
+                ->setOwnerGuid((string) 0) // Site is owner
+                ->setTimestamp($bucket['timestamp'])
+                ->setResolution('day')
+                ->incrementSum('active::total', $bucket['total']);
+            $this->records[] = $record;
+        }
+
+        // Monthly resolution
+        foreach ($this->activeMetric->get($months ?: 1, 'month') as $bucket) {
+            $record = new EntityCentricRecord();
+            $record->setEntityUrn("urn:metric:global")
+                ->setOwnerGuid((string) 0) // Site is owner
+                ->setTimestamp($bucket['timestamp'])
+                ->setResolution('month')
+                ->incrementSum('active::total', $bucket['total']);
+            $this->records[] = $record;
+        }
+
+        foreach ($this->records as $record) {
+            yield $record;
+        }
+    }
+}

Core/Analytics/EntityCentric/EntityCentricRecord.php

+<?php
+/**
+ * EntityCentricRecord
+ * @author Mark
+ */
+
+namespace Minds\Core\Analytics\EntityCentric;
+
+use Minds\Traits\MagicAttributes;
+
+/**
+ * Class EntityCentricRecord
+ * @package Minds\Core\Analytics\EntityCentric
+ * @method EntityCentricRecord setResolution(int $year)
+ * @method string getResolution()
+ * @method EntityCentricRecord setEntityUrn(string $entityUrn)
+ * @method string getEntityUrn()
+ * @method EntityCentricRecord setOwnerGuid(string $ownerGuid)
+ * @method string getOwnerGuid()
+ * @method EntityCentricRecord setTimestampMs(int $timestampMs)
+ * @method int getTimestampMs()
+ * @method EntityCentricRecord setTimestamp(int $timestamp)
+ * @method int getTimestamp()
+ * @method EntityCentricRecord setSums(array $sums)
+ * @method int getSums()
+ */
+class EntityCentricRecord
+{
+    use MagicAttributes;
+
+    /** @var string */
+    private $resolution;
+
+    /** @var int */
+    protected $timestamp;
+
+    /** @var int */
+    protected $timestampMs;
+
+    /** @var string */
+    protected $entityUrn;
+
+    /** @var string */
+    protected $ownerGuid;
+
+    /** @var array */
+    private $sums;
+
+    /**
+     * Increment views
+     * @param string $metric
+     * @param int $value
+     * @return EntityCentricRecord
+     */
+    public function incrementSum($metric, $value = 1): EntityCentricRecord
+    {
+        if (!isset($this->sums[$metric])) {
+            $this->sums[$metric] = 0;
+        }
+        $this->sums[$metric] = $this->sums[$metric] + $value;
+        return $this;
+    }
+}

Core/Analytics/EntityCentric/Manager.php

+<?php
+/**
+ * EntityCentric Manager
+ * @author Mark
+ */
+
+namespace Minds\Core\Analytics\EntityCentric;
+
+use DateTime;
+use Exception;
+
+class Manager
+{
+    /** @var array */
+    const SYNCHRONISERS = [
+        SignupsSynchroniser::class,
+        ActiveUsersSynchroniser::class,
+        ViewsSynchroniser::class,
+    ];
+
+    /** @var Repository */
+    protected $repository;
+
+    /** @var int */
+    private $from;
+
+    /** @var int */
+    private $to;
+
+    public function __construct(
+        $repository = null
+    ) {
+        $this->repository = $repository ?: new Repository();
+    }
+
+    /**
+     * @param int $from
+     * @return self
+     */
+    public function setFrom($from): self
+    {
+        $this->from = $from;
+        return $this;
+    }
+
+    /**
+     * Synchronise views from cassandra to elastic
+     * @return iterable
+     */
+    public function sync(): iterable
+    {
+        foreach (Manager::SYNCHRONISERS as $synchroniserClass) {
+            $synchroniser = new $synchroniserClass;
+            $date = (new DateTime())->setTimestamp($this->from);
+            $synchroniser->setFrom($this->from);
+            foreach ($synchroniser->toRecords() as $record) {
+                $this->add($record);
+                yield $record;
+            }
+            // Call again incase any leftover
+            $this->repository->bulk();
+        }
+        echo "done";
+    }
+
+    /**
+     * Add an entity centric record to the database
+     * @param EntityCentricRecord $record
+     * @return bool
+     */
+    public function add(EntityCentricRecord $record): bool
+    {
+        return (bool) $this->repository->add($record);
+    }
+
+    /**
+     * Query aggregate
+     * @param array $query
+     * @return array
+     */
+    public function getAggregateByQuery(array $query): array
+    {
+    }
+}

Core/Analytics/EntityCentric/Repository.php

+<?php
+/**
+ * EntityCentric Repository
+ * @author Mark
+ */
+
+namespace Minds\Core\Analytics\EntityCentric;
+
+use DateTime;
+use DateTimeZone;
+use Exception;
+use Minds\Common\Repository\Response;
+use Minds\Core\Data\ElasticSearch\Client as ElasticClient;
+use Minds\Core\Di\Di;
+
+class Repository
+{
+    /** @var ElasticClient */
+    protected $es;
+
+    /** @var array $pendingBulkInserts * */
+    private $pendingBulkInserts = [];
+
+    /**
+     * Repository constructor.
+     * @param ElasticClient $es
+     */
+    public function __construct(
+        $es = null
+    ) {
+        $this->es = $es ?: Di::_()->get('Database\ElasticSearch');
+    }
+
+    /**
+     * @param array $opts
+     * @return Response
+     */
+    public function getList(array $opts = [])
+    {
+        $response = new Response();
+
+        return $response;
+    }
+
+    /**
+     * @param EntityCentricRecord $record
+     * @return bool
+     * @throws Exception
+     */
+    public function add(EntityCentricRecord $record)
+    {
+        $index = 'minds-entitycentric-' . date('m-Y', $record->getTimestamp());
+
+        $body = [
+            'resolution' => $record->getResolution(),
+            '@timestamp' => $record->getTimestamp() * 1000,
+            'entity_urn' => $record->getEntityUrn(),
+            'owner_guid' => $record->getOwnerGuid(),
+        ];
+
+        $body = array_merge($body, $record->getSums());
+
+        $body = array_filter($body, function ($val) {
+            if ($val === '' || $val === null) {
+                return false;
+            }
+            return true;
+        });
+
+        $this->pendingBulkInserts[] = [
+            'update' => [
+                '_id' => (string) implode('-', [ $record->getEntityUrn(), $record->getResolution(), $record->getTimestamp() ]),
+                '_index' => $index,
+                '_type' => '_doc',
+            ],
+        ];
+
+        $this->pendingBulkInserts[] = [
+            'doc' => $body,
+            'doc_as_upsert' => true,
+        ];
+
+        if (count($this->pendingBulkInserts) > 2000) { //1000 inserts
+            $this->bulk();
+        }
+    }
+
+    /**
+     * Bulk insert results
+     */
+    public function bulk()
+    {
+        if (count($this->pendingBulkInserts) > 0) {
+            $res = $this->es->bulk(['body' => $this->pendingBulkInserts]);
+            $this->pendingBulkInserts = [];
+        }
+    }
+}

Core/Analytics/EntityCentric/SignupsSynchroniser.php

+<?php
+namespace Minds\Core\Analytics\EntityCentric;
+
+use Minds\Core\Analytics\Metrics\Signup;
+use DateTime;
+use Exception;
+
+class SignupsSynchroniser
+{
+    /** @var array */
+    private $records = [];
+
+    /** @var Signup */
+    private $signupMetric;
+
+    public function __construct($signupMetric = null)
+    {
+        $this->signupMetric = $signupMetric ?? new Signup;
+    }
+
+    /**
+     * @param int $from
+     * @return self
+     */
+    public function setFrom($from): self
+    {
+        $this->from = $from;
+        return $this;
+    }
+
+    /**
+     * Convert to records
+     * @return iterable
+     */
+    public function toRecords(): iterable
+    {
+        $date = (new DateTime())->setTimestamp($this->from);
+        $now = new DateTime();
+        $days = (int) $date->diff($now)->format('%a');
+
+        foreach ($this->signupMetric->get($days) as $bucket) {
+            error_log($bucket['date']);
+            $record = new EntityCentricRecord();
+            $record->setEntityUrn("urn:metric:global")
+                ->setOwnerGuid((string) 0) // Site is owner
+                ->setTimestamp($bucket['timestamp'])
+                ->setResolution('day')
+                ->incrementSum('signups::total', $bucket['total']);
+            $this->records[] = $record;
+        }
+        
+        foreach ($this->records as $record) {
+            yield $record;
+        }
+    }
+}

Core/Analytics/EntityCentric/ViewsSynchroniser.php

+<?php
+namespace Minds\Core\Analytics\EntityCentric;
+
+use Minds\Core\Analytics\Views\Repository as ViewsRepository;
+use DateTime;
+use Exception;
+
+class ViewsSynchroniser
+{
+    /** @var array */
+    private $records = [];
+
+    /** @var ViewsRepository */
+    private $viewsRepository;
+
+    public function __construct($viewsRepository = null)
+    {
+        $this->viewsRepository = $viewsRepository ?: new ViewsRepository();
+    }
+
+    /**
+     * @param int $from
+     * @return self
+     */
+    public function setFrom($from): self
+    {
+        $this->from = $from;
+        return $this;
+    }
+
+    /**
+     * Convert to records
+     * @return iterable
+     */
+    public function toRecords(): iterable
+    {
+        $date = (new DateTime())->setTimestamp($this->from);
+
+        $opts['day'] = intval($date->format('d'));
+        $opts['month'] = intval($date->format('m'));
+        $opts['year'] = $date->format('Y');
+
+        $opts['from'] = $this->from;
+
+        $i = 0;
+        while (true) {
+            $result = $this->viewsRepository->getList($opts);
+
+            $opts['offset'] = $result->getPagingToken();
+
+            foreach ($result as $view) {
+                // if (!in_array($view->getSource(), [ 'single', 'feed/channel'])) {
+                //     continue;
+                // }
+                $this->downsampleViewToRecord($view);
+                error_log(++$i);
+            }
+
+            if ($result->isLastPage()) {
+                break;
+            }
+        }
+
+        foreach ($this->records as $record) {
+            yield $record;
+        }
+    }
+
+    /**
+     * Add entity to map
+     * @param View $view
+     * @return void
+     */
+    private function downsampleViewToRecord($view): void
+    {
+        $entityUrn = $view->getEntityUrn();
+
+        if (!isset($this->records[$view->getEntityUrn()])) {
+            $timestamp = (new \DateTime())->setTimestamp($view->getTimestamp())->setTime(0, 0, 0);
+            $record = new EntityCentricRecord();
+            $record->setEntityUrn($view->getEntityUrn())
+                ->setOwnerGuid($view->getOwnerGuid())
+                ->setTimestamp($timestamp->getTimestamp())
+                ->setResolution('day');
+
+            $this->records[$view->getEntityUrn()] = $record;
+        }
+        if ($view->getCampaign()) {
+            $this->records[$view->getEntityUrn()]->incrementSum('views::boosted');
+        } else {
+            $this->records[$view->getEntityUrn()]->incrementSum('views::organic');
+        }
+
+        if ($view->getSource() === 'single') {
+            $this->records[$view->getEntityUrn()]->incrementSum('views::single');
+        }
+
+        $this->records[$view->getEntityUrn()]->incrementSum('views::total');
+    }
+}

Core/Blockchain/Wallets/OffChain/Sums.php

@@ -78,6 +78,9 @@ class Sums
 
     public function getContractBalance($contract = '', $onlySpend = false)
     {
+        if (!$this->user) {
+            return 0;
+        }
         $cql = "SELECT SUM(amount) as balance from blockchain_transactions_mainnet WHERE user_guid = ? AND wallet_address = ?";
         $values = [
             new Varint($this->user->guid),