Commit 9acc6fd8 authored by Emma's avatar Emma 🏳🌈

optimise database retrieval

parent 6124028c
Pipeline #60643117 passed with stage
in 2 minutes and 28 seconds
......@@ -30,7 +30,6 @@ use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
*/
final class CommentController extends AbstractController {
public function list(CommentRepository $repository, int $page) {
// TODO: link this somewhere
return $this->render('comment/list.html.twig', [
'comments' => $repository->findRecentPaginated($page),
]);
......
......@@ -12,6 +12,7 @@ use App\Events;
use App\Form\DeleteReasonType;
use App\Form\Model\SubmissionData;
use App\Form\SubmissionType;
use App\Repository\CommentRepository;
use App\Utils\Slugger;
use Doctrine\ORM\EntityManager;
use Sensio\Bundle\FrameworkExtraBundle\Configuration\Cache;
......@@ -28,6 +29,15 @@ use Symfony\Component\HttpFoundation\Response;
* @Entity("comment", expr="repository.findOneBy({submission: submission, id: comment_id})")
*/
final class SubmissionController extends AbstractController {
/**
* @var CommentRepository
*/
private $comments;
public function __construct(CommentRepository $comments) {
$this->comments = $comments;
}
/**
* Show a submission's comment page.
*
......@@ -39,6 +49,8 @@ final class SubmissionController extends AbstractController {
* @return Response
*/
public function submission(Forum $forum, Submission $submission) {
$this->comments->hydrate(...$submission->getComments());
return $this->render('submission/submission.html.twig', [
'forum' => $forum,
'submission' => $submission,
......@@ -65,6 +77,8 @@ final class SubmissionController extends AbstractController {
Submission $submission,
Comment $comment
) {
$this->comments->hydrate(...$submission->getComments());
return $this->render('submission/comment.html.twig', [
'comment' => $comment,
'forum' => $forum,
......
......@@ -13,8 +13,10 @@ use App\Form\UserBlockType;
use App\Form\UserFilterType;
use App\Form\UserSettingsType;
use App\Form\UserType;
use App\Repository\CommentRepository;
use App\Repository\ForumBanRepository;
use App\Repository\NotificationRepository;
use App\Repository\SubmissionRepository;
use App\Repository\UserRepository;
use App\Security\AuthenticationHelper;
use Doctrine\ORM\EntityManager;
......@@ -28,12 +30,28 @@ use Symfony\Component\HttpFoundation\Response;
* @Entity("user", expr="repository.findOneOrRedirectToCanonical(username, 'username')")
*/
final class UserController extends AbstractController {
/**
* @var SubmissionRepository
*/
private $submissions;
/**
* @var CommentRepository
*/
private $comments;
/**
* @var string
*/
private $defaultLocale;
public function __construct(string $defaultLocale) {
public function __construct(
SubmissionRepository $submissions,
CommentRepository $comments,
string $defaultLocale
) {
$this->submissions = $submissions;
$this->comments = $comments;
$this->defaultLocale = $defaultLocale;
}
......@@ -72,8 +90,12 @@ final class UserController extends AbstractController {
* @return Response
*/
public function submissions(User $user, int $page) {
$submissions = $user->getPaginatedSubmissions($page);
$this->submissions->hydrate(...$submissions);
return $this->render('user/submissions.html.twig', [
'submissions' => $user->getPaginatedSubmissions($page),
'submissions' => $submissions,
'user' => $user,
]);
}
......@@ -85,8 +107,12 @@ final class UserController extends AbstractController {
* @return Response
*/
public function comments(User $user, int $page) {
$comments = $user->getPaginatedComments($page);
$this->comments->hydrate(...$comments);
return $this->render('user/comments.html.twig', [
'comments' => $user->getPaginatedComments($page),
'comments' => $comments,
'user' => $user,
]);
}
......
......@@ -143,6 +143,15 @@ class Comment extends Votable {
*/
private $mentions;
/**
* @ORM\Column(type="integer")
*
* @Groups({"comment:read"})
*
* @var int
*/
private $netScore;
/**
* @ORM\Column(type="tsvector", nullable=true)
*/
......@@ -158,11 +167,6 @@ class Comment extends Votable {
*/
protected $downvotes;
/**
* @Groups({"comment:read"})
*/
protected $netScore;
public function __construct(
string $body,
User $user,
......@@ -312,6 +316,8 @@ class Comment extends Votable {
}
parent::vote($user, $ip, $choice);
$this->updateNetScore();
}
public function isSoftDeleted(): bool {
......@@ -324,6 +330,9 @@ class Comment extends Votable {
public function softDelete() {
$this->softDeleted = true;
$this->body = '';
$this->submission->updateCommentCount();
$this->submission->updateRanking();
$this->submission->updateLastActive();
}
public function getIp(): ?string {
......@@ -383,4 +392,12 @@ class Comment extends Votable {
$receiver->sendNotification(new CommentNotification($receiver, $this));
}
public function getNetScore(): int {
return $this->netScore;
}
private function updateNetScore(): void {
$this->netScore = parent::getNetScore();
}
}
......@@ -14,7 +14,11 @@ use Symfony\Component\Serializer\Annotation\SerializedName;
/**
* @ORM\Entity(repositoryClass="App\Repository\SubmissionRepository")
* @ORM\Table(name="submissions", indexes={
* @ORM\Index(name="submissions_timestamp_idx", columns={"timestamp"}),
* @ORM\Index(name="submissions_ranking_id_idx", columns={"ranking", "id"}),
* @ORM\Index(name="submissions_last_active_id_idx", columns={"last_active", "id"}),
* @ORM\Index(name="submissions_comment_count_id_idx", columns={"comment_count", "id"}),
* @ORM\Index(name="submissions_net_score_id_idx", columns={"net_score", "id"}),
* @ORM\Index(name="submissions_search_idx", columns={"search_doc"}),
* })
*/
......@@ -113,6 +117,15 @@ class Submission extends Votable {
*/
private $comments;
/**
* @ORM\Column(type="integer")
*
* @Groups({"submission:read"})
*
* @var int
*/
private $commentCount = 0;
/**
* @ORM\Column(type="datetimetz")
*
......@@ -230,6 +243,14 @@ class Submission extends Votable {
*/
private $locked = false;
/**
* @ORM\Column(type="integer")
* @Groups({"submission:read"})
*
* @var int
*/
private $netScore = 0;
/**
* @ORM\Column(type="tsvector", nullable=true)
*
......@@ -247,11 +268,6 @@ class Submission extends Votable {
*/
protected $downvotes;
/**
* @Groups({"submission:read"})
*/
protected $netScore;
public function __construct(
string $title,
?string $url,
......@@ -322,15 +338,6 @@ class Submission extends Votable {
return $this->comments;
}
/**
* @Groups({"submission:read"})
*
* @return int
*/
public function getCommentCount(): int {
return \count($this->comments);
}
/**
* Get top-level comments, ordered by descending net score.
*
......@@ -356,6 +363,7 @@ class Submission extends Votable {
}
}
$this->updateCommentCount();
$this->updateRanking();
$this->updateLastActive();
}
......@@ -365,13 +373,27 @@ class Submission extends Votable {
$this->comments->get(-1);
foreach ($comments as $comment) {
$this->comments->removeElement($comment);
if ($this->comments->contains($comment)) {
$this->comments->removeElement($comment);
}
}
$this->updateCommentCount();
$this->updateRanking();
$this->updateLastActive();
}
public function getCommentCount(): int {
return $this->commentCount;
}
public function updateCommentCount(): void {
$criteria = Criteria::create()
->where(Criteria::expr()->eq('softDeleted', false));
$this->commentCount = \count($this->comments->matching($criteria));
}
public function getTimestamp(): \DateTime {
return $this->timestamp;
}
......@@ -427,6 +449,7 @@ class Submission extends Votable {
parent::vote($user, $ip, $choice);
$this->updateNetScore();
$this->updateRanking();
}
......@@ -477,18 +500,13 @@ class Submission extends Votable {
}
public function updateRanking() {
$criteria = Criteria::create()
->where(Criteria::expr()->eq('softDeleted', false));
$commentCount = \count($this->comments->matching($criteria));
$netScore = $this->getNetScore();
$netScoreAdvantage = $netScore * self::NETSCORE_MULTIPLIER;
if ($netScore > self::DOWNVOTED_CUTOFF) {
$commentAdvantage = $commentCount * self::COMMENT_MULTIPLIER;
$commentAdvantage = $this->getCommentCount() * self::COMMENT_MULTIPLIER;
} else {
$commentAdvantage = $commentCount * self::COMMENT_DOWNVOTED_MULTIPLIER;
$commentAdvantage = $this->getCommentCount() * self::COMMENT_DOWNVOTED_MULTIPLIER;
}
$advantage = max(min($netScoreAdvantage + $commentAdvantage, self::MAX_ADVANTAGE), -self::MAX_PENALTY);
......@@ -541,4 +559,12 @@ class Submission extends Votable {
public function setLocked(bool $locked) {
$this->locked = $locked;
}
public function getNetScore(): int {
return $this->netScore;
}
private function updateNetScore(): void {
$this->netScore = parent::getNetScore();
}
}
......@@ -101,6 +101,6 @@ abstract class Votable {
* multiple entities.
*/
private function hydrateVoteCollection(): void {
$this->getVotes()->getValues();
$this->getVotes()->get(-1);
}
}
<?php
namespace DoctrineMigrations;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\Migrations\AbstractMigration;
final class Version20190509152528 extends AbstractMigration {
public function getDescription(): string {
return 'Optimise submission and comment retrieval';
}
public function up(Schema $schema): void {
$this->abortIf($this->connection->getDatabasePlatform()->getName() !== 'postgresql', 'Migration can only be executed safely on \'postgresql\'.');
$this->addSql('ALTER TABLE submissions ADD comment_count INT');
$this->addSql('ALTER TABLE submissions ADD net_score INT');
$this->addSql('ALTER TABLE comments ADD net_score INT');
$this->addSql('UPDATE submissions s SET comment_count = (SELECT COUNT(*) FROM comments c WHERE c.submission_id = s.id)');
$this->addSql('UPDATE submissions s SET net_score = (SELECT COUNT(*) FILTER (WHERE upvote = TRUE) - COUNT(*) FILTER (WHERE upvote = FALSE) FROM submission_votes sv WHERE sv.submission_id = s.id)');
$this->addSql('UPDATE comments c SET net_score = (SELECT COUNT(*) FILTER (WHERE upvote = TRUE) - COUNT(*) FILTER (WHERE upvote = FALSE) FROM comment_votes cv WHERE cv.comment_id = c.id)');
$this->addSql('ALTER TABLE submissions ALTER comment_count SET NOT NULL');
$this->addSql('ALTER TABLE submissions ALTER net_score SET NOT NULL');
$this->addSql('ALTER TABLE comments ALTER net_score SET NOT NULL');
$this->addSql('CREATE INDEX submissions_timestamp_idx ON submissions (timestamp)');
$this->addSql('CREATE INDEX submissions_last_active_id_idx ON submissions (last_active, id)');
$this->addSql('CREATE INDEX submissions_net_score_id_idx ON submissions (net_score, id)');
$this->addSql('CREATE INDEX submissions_comment_count_id_idx ON submissions (comment_count, id)');
}
public function down(Schema $schema): void {
$this->abortIf($this->connection->getDatabasePlatform()->getName() !== 'postgresql', 'Migration can only be executed safely on \'postgresql\'.');
$this->addSql('ALTER TABLE submissions DROP comment_count');
$this->addSql('ALTER TABLE submissions DROP net_score');
$this->addSql('ALTER TABLE comments DROP net_score');
$this->addSql('DROP INDEX submissions_timestamp_idx');
$this->addSql('DROP INDEX submissions_last_active_id_idx');
}
}
......@@ -10,10 +10,21 @@ use Doctrine\Common\Persistence\ManagerRegistry;
use Pagerfanta\Adapter\DoctrineORMAdapter;
use Pagerfanta\Pagerfanta;
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
use Symfony\Component\Security\Core\Authorization\AuthorizationCheckerInterface;
class CommentRepository extends ServiceEntityRepository {
public function __construct(ManagerRegistry $registry) {
/**
* @var AuthorizationCheckerInterface
*/
private $authorizationChecker;
public function __construct(
ManagerRegistry $registry,
AuthorizationCheckerInterface $authorizationChecker
) {
parent::__construct($registry, Comment::class);
$this->authorizationChecker = $authorizationChecker;
}
/**
......@@ -56,7 +67,7 @@ class CommentRepository extends ServiceEntityRepository {
$pager->setMaxPerPage($maxPerPage);
$pager->setCurrentPage($page);
$this->hydrateComments(\iterator_to_array($pager));
$this->hydrate(...$pager);
return $pager;
}
......@@ -80,58 +91,46 @@ class CommentRepository extends ServiceEntityRepository {
$pager->setMaxPerPage($maxPerPage);
$pager->setCurrentPage($page);
$this->hydrateComments(\iterator_to_array($pager));
$this->hydrate(...$pager);
return $pager;
}
private function hydrateComments(array $comments): void {
// hydrate parent and parent's author
$this->createQueryBuilder('c')
->select('PARTIAL c.{id}')
->addSelect('p')
->addSelect('pu')
->leftJoin('c.parent', 'p')
->leftJoin('p.user', 'pu')
->where('c IN (?1)')
->setParameter(1, $comments)
->getQuery()
->execute();
// hydrate comment author/submission/submission author/forum
public function hydrate(Comment ...$comments): void {
$this->createQueryBuilder('c')
->select('PARTIAL c.{id}')
->addSelect('cu')
->addSelect('u')
->addSelect('s')
->addSelect('sf')
->addSelect('su')
->addSelect('f')
->join('c.user', 'cu')
->join('c.user', 'u')
->join('c.submission', 's')
->join('s.forum', 'sf')
->join('s.user', 'su')
->join('s.forum', 'f')
->where('c IN (?1)')
->setParameter(1, $comments)
->getQuery()
->execute();
// hydrate votes
$this->createQueryBuilder('c')
->select('PARTIAL c.{id}')
->addSelect('cv')
->leftJoin('c.votes', 'cv')
->addSelect('cc')
->leftJoin('c.children', 'cc')
->where('c IN (?1)')
->setParameter(1, $comments)
->getQuery()
->execute();
// hydrate children (for count only)
$this->createQueryBuilder('c')
->select('PARTIAL c.{id}')
->addSelect('PARTIAL r.{id}')
->leftJoin('c.children', 'r')
->where('c IN (?1)')
->setParameter(1, $comments)
->getQuery()
->execute();
// for fast retrieval of user vote
if ($this->authorizationChecker->isGranted('ROLE_USER')) {
$this->createQueryBuilder('c')
->select('PARTIAL c.{id}')
->addSelect('cv')
->leftJoin('c.votes', 'cv')
->where('c IN (?1)')
->setParameter(1, $comments)
->getQuery()
->execute();
}
}
}
......@@ -24,7 +24,7 @@ class SubmissionPager implements \IteratorAggregate {
$params = [];
foreach (SubmissionRepository::SORT_COLUMN_MAP[$sortBy] as $column) {
foreach (SubmissionRepository::SORT_COLUMN_MAP[$sortBy] as $column => $order) {
$value = $request->query->get('next_'.$column);
$type = SubmissionRepository::SORT_COLUMN_TYPES[$column];
......@@ -56,7 +56,7 @@ class SubmissionPager implements \IteratorAggregate {
foreach ($submissions as $submission) {
if (++$count > $maxPerPage) {
foreach (SubmissionRepository::SORT_COLUMN_MAP[$sortBy] as $column) {
foreach (SubmissionRepository::SORT_COLUMN_MAP[$sortBy] as $column => $order) {
$accessor = $this->columnNameToAccessor($column);
$value = $submission->{$accessor}();
......
......@@ -10,20 +10,26 @@ use Doctrine\Common\Persistence\ManagerRegistry;
use Doctrine\DBAL\Query\QueryBuilder;
use Doctrine\DBAL\Types\Type;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\Security\Core\Authorization\AuthorizationCheckerInterface;
class SubmissionRepository extends ServiceEntityRepository {
/**
* @var AuthorizationCheckerInterface
*/
private $authorizationChecker;
/**
* `$sortBy` -> ordered column name mapping.
*
* @var array[]
*/
public const SORT_COLUMN_MAP = [
Submission::SORT_ACTIVE => ['last_active', 'id'],
Submission::SORT_HOT => ['ranking', 'id'],
Submission::SORT_NEW => ['id'],
Submission::SORT_TOP => ['net_score', 'id'],
Submission::SORT_CONTROVERSIAL => ['downvotes', 'id'],
Submission::SORT_MOST_COMMENTED => ['comment_count', 'id'],
Submission::SORT_ACTIVE => ['last_active' => 'DESC', 'id' => 'DESC'],
Submission::SORT_HOT => ['ranking' => 'DESC', 'id' => 'DESC'],
Submission::SORT_NEW => ['id' => 'DESC'],
Submission::SORT_TOP => ['net_score' => 'DESC', 'id' => 'DESC'],
Submission::SORT_CONTROVERSIAL => ['net_score' => 'ASC', 'id' => 'ASC'],
Submission::SORT_MOST_COMMENTED => ['comment_count' => 'DESC', 'id' => 'DESC'],
];
public const SORT_COLUMN_TYPES = [
......@@ -31,36 +37,18 @@ class SubmissionRepository extends ServiceEntityRepository {
'ranking' => 'bigint',
'id' => 'bigint',
'net_score' => 'integer',
'downvotes' => 'integer',
'comment_count' => 'integer',
];
private const MAX_PER_PAGE = 25;
private const NET_SCORE_JOIN = '('.
'SELECT submission_id, '.
'COUNT(*) FILTER (WHERE upvote = TRUE) - '.
'COUNT(*) FILTER (WHERE upvote = FALSE) AS net_score '.
'FROM submission_votes '.
'GROUP BY submission_id'.
')';
// TODO: implement actually useful controversy metric
private const CONTROVERSIAL_JOIN = '('.
'SELECT submission_id, COUNT(*) AS downvotes '.
'FROM submission_votes '.
'WHERE upvote = FALSE '.
'GROUP BY submission_id'.
')';
private const COMMENT_COUNT_JOIN = '('.
'SELECT submission_id, COUNT(*) AS comment_count '.
'FROM comments '.
'GROUP BY submission_id'.
')';
public function __construct(ManagerRegistry $registry) {
public function __construct(
ManagerRegistry $registry,
AuthorizationCheckerInterface $authorizationChecker
) {
parent::__construct($registry, Submission::class);
$this->authorizationChecker = $authorizationChecker;
}
/**
......@@ -102,21 +90,7 @@ class SubmissionRepository extends ServiceEntityRepository {
->from('submissions', 's')
->setMaxResults($maxPerPage + 1);
switch ($sortBy) {
case Submission::SORT_ACTIVE:
case Submission::SORT_HOT:
case Submission::SORT_NEW:
break;
case Submission::SORT_TOP:
$qb->join('s', self::NET_SCORE_JOIN, 'ns', 's.id = ns.submission_id');
break;
case Submission::SORT_CONTROVERSIAL:
$qb->join('s', self::CONTROVERSIAL_JOIN, 'cn', 's.id = cn.submission_id');
break;
case Submission::SORT_MOST_COMMENTED:
$qb->join('s', self::COMMENT_COUNT_JOIN, 'cc', 's.id = cc.submission_id');
break;
default:
if (!\in_array($sortBy, Submission::SORT_OPTIONS, true)) {
throw new \InvalidArgumentException("Sort mode '$sortBy' not implemented");
}
......@@ -165,17 +139,17 @@ class SubmissionRepository extends ServiceEntityRepository {
}
}
foreach (self::SORT_COLUMN_MAP[$sortBy] as $column) {
$qb->addOrderBy($column, 'DESC');
foreach (self::SORT_COLUMN_MAP[$sortBy] as $column => $order) {
$qb->addOrderBy($column, $order);
}
if ($pager) {
$qb->andWhere(sprintf('(%s) <= (:next_%s)',
implode(', ', self::SORT_COLUMN_MAP[$sortBy]),
implode(', :next_', self::SORT_COLUMN_MAP[$sortBy])
implode(', ', \array_keys(self::SORT_COLUMN_MAP[$sortBy])),
implode(', :next_', \array_keys(self::SORT_COLUMN_MAP[$sortBy]))
));
foreach (self::SORT_COLUMN_MAP[$sortBy] as $column) {
foreach (self::SORT_COLUMN_MAP[$sortBy] as $column => $order) {
$qb->setParameter('next_'.$column, $pager[$column]);
}
}
......@@ -193,7 +167,7 @@ class SubmissionRepository extends ServiceEntityRepository {
$submissions = new SubmissionPager($results, $maxPerPage, $sortBy);
$this->hydrateAssociations($submissions);
$this->hydrate(...$submissions);
return $submissions;
}
......@@ -232,11 +206,7 @@ class SubmissionRepository extends ServiceEntityRepository {
}
}
private function hydrateAssociations(iterable $submissions): void {
if ($submissions instanceof \Traversable) {
$submissions = iterator_to_array($submissions);
}
public function hydrate(Submission ...$submissions): void {
$this->_em->createQueryBuilder()
->select('PARTIAL s.{id}')
->addSelect('u')
......@@ -249,24 +219,17 @@ class SubmissionRepository extends ServiceEntityRepository {
->getQuery()
->getResult();
$this->_em->createQueryBuilder()
->select('PARTIAL s.{id}')
->addSelect('sv')
->from(Submission::class, 's')
->leftJoin('s.votes', 'sv')
->where('s IN (?1)')
->setParameter(1, $submissions)
->getQuery()
->getResult();
$this->_em->createQueryBuilder()
->select('PARTIAL s.{id}')
->addSelect('PARTIAL c.{id}')
->from(Submission::class, 's')
->leftJoin('s.comments', 'c')
->where('s IN (?1)')
->setParameter(1, $submissions)
->getQuery()
->getResult();
if ($this->authorizationChecker->isGranted('ROLE_USER')) {
// hydrate submission votes for fast checking of user choice
$this->_em->createQueryBuilder()
->select('PARTIAL s.{id}')
->addSelect('sv')
->from(Submission::class, 's')
->leftJoin('s.votes', 'sv')
->where('s IN (?1)')
->setParameter(1, $submissions)
->getQuery()
->getResult();
}
}
}
......@@ -144,6 +144,9 @@ class UserRepository extends ServiceEntityRepository implements UserLoaderInterf
$combined = \array_merge($submissions, $comments);
$this->_em->getRepository(Submission::class)->hydrate(...$submissions);
$this->_em->getRepository(Comment::class)->hydrate(...$comments);
\usort($combined, function ($a, $b) {
return $b->getTimestamp() <=> $a->getTimestamp();
});
......
......@@ -147,14 +147,14 @@
{% filter spaceless %}
<a href="{{ path('submission', {forum_name: submission.forum.name, submission_id: submission.id, slug: submission.title|slugify}) }}"
class="text-sm">
<strong>{{ 'submissions.comments'|trans({'%count%': submission.comments|length}) }}</strong>
<strong>{{ 'submissions.comments'|trans({ '%count%': submission.commentCount }) }}</strong>
</a>
{% endfilter %}
{%- if not show_body -%}
<span class="js-display-new-comments submission__new-comments text-sm fg-green"
data-submission-id="{{ submission.id }}"
data-comment-count="{{ submission.comments|length }}">
data-comment-count="{{ submission.commentCount }}">
</span>
{%- endif -%}
</li>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment