Unverified Commit 24cafd73 authored by Eugen Rochko's avatar Eugen Rochko Committed by GitHub

Lists (#5703)

* Add structure for lists

* Add list timeline streaming API

* Add list APIs, bind list-account relation to follow relation

* Add API for adding/removing accounts from lists

* Add pagination to lists API

* Add pagination to list accounts API

* Adjust scopes for new APIs

- Creating and modifying lists merely requires "write" scope
- Fetching information about lists merely requires "read" scope

* Add test for wrong user context on list timeline

* Clean up tests
parent 4a2fc2d4
# frozen_string_literal: true
class Api::V1::Lists::AccountsController < Api::BaseController
before_action -> { doorkeeper_authorize! :read }, only: [:show]
before_action -> { doorkeeper_authorize! :write }, except: [:show]
before_action :require_user!
before_action :set_list
after_action :insert_pagination_headers, only: :show
def show
@accounts = @list.accounts.paginate_by_max_id(limit_param(DEFAULT_ACCOUNTS_LIMIT), params[:max_id], params[:since_id])
render json: @accounts, each_serializer: REST::AccountSerializer
end
def create
ApplicationRecord.transaction do
list_accounts.each do |account|
@list.accounts << account
end
end
render_empty
end
def destroy
ListAccount.where(list: @list, account_id: account_ids).destroy_all
render_empty
end
private
def set_list
@list = List.where(account: current_account).find(params[:list_id])
end
def list_accounts
Account.find(account_ids)
end
def account_ids
Array(resource_params[:account_ids])
end
def resource_params
params.permit(account_ids: [])
end
def insert_pagination_headers
set_pagination_headers(next_path, prev_path)
end
def next_path
if records_continue?
api_v1_list_accounts_url pagination_params(max_id: pagination_max_id)
end
end
def prev_path
unless @accounts.empty?
api_v1_list_accounts_url pagination_params(since_id: pagination_since_id)
end
end
def pagination_max_id
@accounts.last.id
end
def pagination_since_id
@accounts.first.id
end
def records_continue?
@accounts.size == limit_param(DEFAULT_ACCOUNTS_LIMIT)
end
def pagination_params(core_params)
params.permit(:limit).merge(core_params)
end
end
# frozen_string_literal: true
class Api::V1::ListsController < Api::BaseController
LISTS_LIMIT = 50
before_action -> { doorkeeper_authorize! :read }, only: [:index, :show]
before_action -> { doorkeeper_authorize! :write }, except: [:index, :show]
before_action :require_user!
before_action :set_list, except: [:index, :create]
after_action :insert_pagination_headers, only: :index
def index
@lists = List.where(account: current_account).paginate_by_max_id(limit_param(LISTS_LIMIT), params[:max_id], params[:since_id])
render json: @lists, each_serializer: REST::ListSerializer
end
def show
render json: @list, serializer: REST::ListSerializer
end
def create
@list = List.create!(list_params.merge(account: current_account))
render json: @list, serializer: REST::ListSerializer
end
def update
@list.update!(list_params)
render json: @list, serializer: REST::ListSerializer
end
def destroy
@list.destroy!
render_empty
end
private
def set_list
@list = List.where(account: current_account).find(params[:id])
end
def list_params
params.permit(:title)
end
def insert_pagination_headers
set_pagination_headers(next_path, prev_path)
end
def next_path
if records_continue?
api_v1_lists_url pagination_params(max_id: pagination_max_id)
end
end
def prev_path
unless @lists.empty?
api_v1_lists_url pagination_params(since_id: pagination_since_id)
end
end
def pagination_max_id
@lists.last.id
end
def pagination_since_id
@lists.first.id
end
def records_continue?
@lists.size == limit_param(LISTS_LIMIT)
end
def pagination_params(core_params)
params.permit(:limit).merge(core_params)
end
end
......@@ -31,7 +31,7 @@ class Api::V1::Timelines::HomeController < Api::BaseController
end
def account_home_feed
Feed.new(:home, current_account)
HomeFeed.new(current_account)
end
def insert_pagination_headers
......
# frozen_string_literal: true
class Api::V1::Timelines::ListController < Api::BaseController
before_action -> { doorkeeper_authorize! :read }
before_action :require_user!
before_action :set_list
before_action :set_statuses
after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
def show
render json: @statuses,
each_serializer: REST::StatusSerializer,
relationships: StatusRelationshipsPresenter.new(@statuses, current_user.account_id)
end
private
def set_list
@list = List.where(account: current_account).find(params[:id])
end
def set_statuses
@statuses = cached_list_statuses
end
def cached_list_statuses
cache_collection list_statuses, Status
end
def list_statuses
list_feed.get(
limit_param(DEFAULT_STATUSES_LIMIT),
params[:max_id],
params[:since_id]
)
end
def list_feed
ListFeed.new(@list)
end
def insert_pagination_headers
set_pagination_headers(next_path, prev_path)
end
def pagination_params(core_params)
params.permit(:limit).merge(core_params)
end
def next_path
api_v1_timelines_list_url params[:id], pagination_params(max_id: pagination_max_id)
end
def prev_path
api_v1_timelines_list_url params[:id], pagination_params(since_id: pagination_since_id)
end
def pagination_max_id
@statuses.last.id
end
def pagination_since_id
@statuses.first.id
end
end
......@@ -26,34 +26,42 @@ class FeedManager
end
end
def push(timeline_type, account, status)
return false unless add_to_feed(timeline_type, account, status)
trim(timeline_type, account.id)
PushUpdateWorker.perform_async(account.id, status.id) if push_update_required?(timeline_type, account.id)
def push_to_home(account, status)
return false unless add_to_feed(:home, account.id, status)
trim(:home, account.id)
PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}") if push_update_required?("timeline:#{account.id}")
true
end
def unpush(timeline_type, account, status)
return false unless remove_from_feed(timeline_type, account, status)
def unpush_from_home(account, status)
return false unless remove_from_feed(:home, account.id, status)
Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s))
true
end
payload = Oj.dump(event: :delete, payload: status.id.to_s)
Redis.current.publish("timeline:#{account.id}", payload)
def push_to_list(list, status)
return false unless add_to_feed(:list, list.id, status)
trim(:list, list.id)
PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}")
true
end
def unpush_from_list(list, status)
return false unless remove_from_feed(:list, list.id, status)
Redis.current.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s))
true
end
def trim(type, account_id)
timeline_key = key(type, account_id)
reblog_key = key(type, account_id, 'reblogs')
reblog_key = key(type, account_id, 'reblogs')
# Remove any items past the MAX_ITEMS'th entry in our feed
redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s)
# Get the score of the REBLOG_FALLOFF'th item in our feed, and stop
# tracking anything after it for deduplication purposes.
falloff_rank = FeedManager::REBLOG_FALLOFF - 1
falloff_rank = FeedManager::REBLOG_FALLOFF - 1
falloff_range = redis.zrevrange(timeline_key, falloff_rank, falloff_rank, with_scores: true)
falloff_score = falloff_range&.first&.last&.to_i || 0
......@@ -69,10 +77,6 @@ class FeedManager
end
end
def push_update_required?(timeline_type, account_id)
timeline_type != :home || redis.get("subscribed:timeline:#{account_id}").present?
end
def merge_into_timeline(from_account, into_account)
timeline_key = key(:home, into_account.id)
query = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4)
......@@ -84,28 +88,28 @@ class FeedManager
query.each do |status|
next if status.direct_visibility? || filter?(:home, status, into_account)
add_to_feed(:home, into_account, status)
add_to_feed(:home, into_account.id, status)
end
trim(:home, into_account.id)
end
def unmerge_from_timeline(from_account, into_account)
timeline_key = key(:home, into_account.id)
timeline_key = key(:home, into_account.id)
oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
from_account.statuses.select('id, reblog_of_id').where('id > ?', oldest_home_score).reorder(nil).find_each do |status|
remove_from_feed(:home, into_account, status)
remove_from_feed(:home, into_account.id, status)
end
end
def clear_from_timeline(account, target_account)
timeline_key = key(:home, account.id)
timeline_key = key(:home, account.id)
timeline_status_ids = redis.zrange(timeline_key, 0, -1)
target_statuses = Status.where(id: timeline_status_ids, account: target_account)
target_statuses = Status.where(id: timeline_status_ids, account: target_account)
target_statuses.each do |status|
unpush(:home, account, status)
unpush_from_home(account, status)
end
end
......@@ -122,7 +126,7 @@ class FeedManager
statuses.each do |status|
next if filter_from_home?(status, account)
added += 1 if add_to_feed(:home, account, status)
added += 1 if add_to_feed(:home, account.id, status)
end
break unless added.zero?
......@@ -137,6 +141,10 @@ class FeedManager
Redis.current
end
def push_update_required?(timeline_id)
redis.exists("subscribed:#{timeline_id}")
end
def filter_from_home?(status, receiver_id)
return false if receiver_id == status.account_id
return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?)
......@@ -182,9 +190,9 @@ class FeedManager
# added, and false if it was not added to the feed. Note that this is
# an internal helper: callers must call trim or push updates if
# either action is appropriate.
def add_to_feed(timeline_type, account, status)
timeline_key = key(timeline_type, account.id)
reblog_key = key(timeline_type, account.id, 'reblogs')
def add_to_feed(timeline_type, account_id, status)
timeline_key = key(timeline_type, account_id)
reblog_key = key(timeline_type, account_id, 'reblogs')
if status.reblog?
# If the original status or a reblog of it is within
......@@ -195,6 +203,7 @@ class FeedManager
return false if !rank.nil? && rank < FeedManager::REBLOG_FALLOFF
reblog_rank = redis.zrevrank(reblog_key, status.reblog_of_id)
if reblog_rank.nil?
# This is not something we've already seen reblogged, so we
# can just add it to the feed (and note that we're
......@@ -205,7 +214,7 @@ class FeedManager
# Another reblog of the same status was already in the
# REBLOG_FALLOFF most recent statuses, so we note that this
# is an "extra" reblog, by storing it in reblog_set_key.
reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}")
reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}")
redis.sadd(reblog_set_key, status.id)
return false
end
......@@ -220,8 +229,8 @@ class FeedManager
# with reblogs, and returning true if a status was removed. As with
# `add_to_feed`, this does not trigger push updates, so callers must
# do so if appropriate.
def remove_from_feed(timeline_type, account, status)
timeline_key = key(timeline_type, account.id)
def remove_from_feed(timeline_type, account_id, status)
timeline_key = key(timeline_type, account_id)
if status.reblog?
# 1. If the reblogging status is not in the feed, stop.
......@@ -229,7 +238,7 @@ class FeedManager
return false if status_rank.nil?
# 2. Remove reblog from set of this status's reblogs.
reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}")
reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}")
redis.srem(reblog_set_key, status.id)
# 3. Re-insert another reblog or original into the feed if one
......@@ -244,7 +253,7 @@ class FeedManager
# (outside conditional)
else
# If the original is getting deleted, no use for reblog references
redis.del(key(timeline_type, account.id, "reblogs:#{status.id}"))
redis.del(key(timeline_type, account_id, "reblogs:#{status.id}"))
end
redis.zrem(timeline_key, status.id)
......
......@@ -3,7 +3,7 @@
#
# Table name: accounts
#
# id :bigint not null, primary key
# id :integer not null, primary key
# username :string default(""), not null
# domain :string
# secret :string default(""), not null
......@@ -53,6 +53,7 @@ class Account < ApplicationRecord
include AccountInteractions
include Attachmentable
include Remotable
include Paginable
enum protocol: [:ostatus, :activitypub]
......@@ -95,6 +96,10 @@ class Account < ApplicationRecord
has_many :account_moderation_notes, dependent: :destroy
has_many :targeted_moderation_notes, class_name: 'AccountModerationNote', foreign_key: :target_account_id, dependent: :destroy
# Lists
has_many :list_accounts, inverse_of: :account, dependent: :destroy
has_many :lists, through: :list_accounts
scope :remote, -> { where.not(domain: nil) }
scope :local, -> { where(domain: nil) }
scope :without_followers, -> { where(followers_count: 0) }
......
......@@ -3,11 +3,11 @@
#
# Table name: account_domain_blocks
#
# id :integer not null, primary key
# domain :string
# created_at :datetime not null
# updated_at :datetime not null
# account_id :bigint
# id :bigint not null, primary key
# account_id :integer
#
class AccountDomainBlock < ApplicationRecord
......
......@@ -3,10 +3,10 @@
#
# Table name: account_moderation_notes
#
# id :bigint not null, primary key
# id :integer not null, primary key
# content :text not null
# account_id :bigint not null
# target_account_id :bigint not null
# account_id :integer not null
# target_account_id :integer not null
# created_at :datetime not null
# updated_at :datetime not null
#
......
......@@ -3,11 +3,11 @@
#
# Table name: blocks
#
# id :integer not null, primary key
# created_at :datetime not null
# updated_at :datetime not null
# account_id :bigint not null
# id :bigint not null, primary key
# target_account_id :bigint not null
# account_id :integer not null
# target_account_id :integer not null
#
class Block < ApplicationRecord
......
......@@ -3,7 +3,7 @@
#
# Table name: conversations
#
# id :bigint not null, primary key
# id :integer not null, primary key
# uri :string
# created_at :datetime not null
# updated_at :datetime not null
......
......@@ -3,9 +3,9 @@
#
# Table name: conversation_mutes
#
# conversation_id :bigint not null
# account_id :bigint not null
# id :bigint not null, primary key
# id :integer not null, primary key
# conversation_id :integer not null
# account_id :integer not null
#
class ConversationMute < ApplicationRecord
......
......@@ -3,7 +3,7 @@
#
# Table name: custom_emojis
#
# id :bigint not null, primary key
# id :integer not null, primary key
# shortcode :string default(""), not null
# domain :string
# image_file_name :string
......
......@@ -3,12 +3,12 @@
#
# Table name: domain_blocks
#
# id :integer not null, primary key
# domain :string default(""), not null
# created_at :datetime not null
# updated_at :datetime not null
# severity :integer default("silence")
# reject_media :boolean default(FALSE), not null
# id :bigint not null, primary key
#
class DomainBlock < ApplicationRecord
......
......@@ -3,7 +3,7 @@
#
# Table name: email_domain_blocks
#
# id :bigint not null, primary key
# id :integer not null, primary key
# domain :string default(""), not null
# created_at :datetime not null
# updated_at :datetime not null
......
......@@ -3,11 +3,11 @@
#
# Table name: favourites
#
# id :integer not null, primary key
# created_at :datetime not null
# updated_at :datetime not null
# account_id :bigint not null
# id :bigint not null, primary key
# status_id :bigint not null
# account_id :integer not null
# status_id :integer not null
#
class Favourite < ApplicationRecord
......
# frozen_string_literal: true
class Feed
def initialize(type, account)
@type = type
@account = account
def initialize(type, id)
@type = type
@id = id
end
def get(limit, max_id = nil, since_id = nil)
if redis.exists("account:#{@account.id}:regeneration")
from_database(limit, max_id, since_id)
else
from_redis(limit, max_id, since_id)
end
from_redis(limit, max_id, since_id)
end
private
protected
def from_redis(limit, max_id, since_id)
max_id = '+inf' if max_id.blank?
since_id = '-inf' if since_id.blank?
unhydrated = redis.zrevrangebyscore(key, "(#{max_id}", "(#{since_id}", limit: [0, limit], with_scores: true).map(&:first).map(&:to_i)
Status.where(id: unhydrated).cache_ids
end
def from_database(limit, max_id, since_id)
Status.as_home_timeline(@account)
.paginate_by_max_id(limit, max_id, since_id)
.reject { |status| FeedManager.instance.filter?(:home, status, @account.id) }
Status.where(id: unhydrated).cache_ids
end
def key
FeedManager.instance.key(@type, @account.id)
FeedManager.instance.key(@type, @id)
end
def redis
......
......@@ -3,11 +3,11 @@
#
# Table name: follows
#
# id :integer not null, primary key
# created_at :datetime not null
# updated_at :datetime not null
# account_id :bigint not null
# id :bigint not null, primary key
# target_account_id :bigint not null
# account_id :integer not null
# target_account_id :integer not null
#
class Follow < ApplicationRecord
......
......@@ -3,11 +3,11 @@
#
# Table name: follow_requests
#
# id :integer not null, primary key
# created_at :datetime not null
# updated_at :datetime not null
# account_id :bigint not null
# id :bigint not null, primary key
# target_account_id :bigint not null
# account_id :integer not null
# target_account_id :integer not null
#
class FollowRequest < ApplicationRecord
......
# frozen_string_literal: true
class HomeFeed < Feed
def initialize(account)
@type = :home
@id = account.id
@account = account
end
def get(limit, max_id = nil, since_id = nil)
if redis.exists("account:#{@account.id}:regeneration")
from_database(limit, max_id, since_id)
else
super
end
end
private
def from_database(limit, max_id, since_id)
Status.as_home_timeline(@account)
.paginate_by_max_id(limit, max_id, since_id)
.reject { |status| FeedManager.instance.filter?(:home, status, @account.id) }
end
end
......@@ -3,6 +3,7 @@
#
# Table name: imports
#
# id :integer not null, primary key
# type :integer not null
# approved :boolean default(FALSE), not null
# created_at :datetime not null
......@@ -11,8 +12,7 @@
# data_content_type :string
# data_file_size :integer
# data_updated_at :datetime
# account_id :bigint not null
# id :bigint not null, primary key
# account_id :integer not null
#
class Import < ApplicationRecord
......
# frozen_string_literal: true
# == Schema Information
#
# Table name: lists
#
# id :integer not null, primary key
# account_id :integer
# title :string default(""), not null
# created_at :datetime not null
# updated_at :datetime not null
#
class List < ApplicationRecord
include Paginable
belongs_to :account
has_many :list_accounts, inverse_of: :list, dependent: :destroy
has_many :accounts, through: :list_accounts
validates :title, presence: true
end