Commit 6d6a429a authored by Eugen Rochko's avatar Eugen Rochko Committed by GitHub

Rewrite Atom generation from stream entries to use Ox instead of Nokogiri (#1124)

* Rewrite Atom generation from stream entries to use Ox instead of Nokogiri::Builder

StreamEntry is now limited to only statuses, which allows some optimization. Removed
extra queries on AccountsController#show. AtomSerializer instead of AtomBuilderHelper
used in AccountsController#show, StreamEntriesController#show, StreamEntryRenderer
and PubSubHubbub::DistributionWorker

PubSubHubbub::DistributionWorker moves n+1 DomainBlock query to PubSubHubbub::DeliveryWorker
instead.

All Salmon slaps that aren't based on StreamEntry still use AtomBuilderHelper and Nokogiri

* All Salmon slaps now use Ox instead of Nokogiri. No touch from status on account
parent 31597fd3
......@@ -34,6 +34,7 @@ gem 'doorkeeper'
gem 'rabl'
gem 'rqrcode'
gem 'twitter-text'
gem 'ox'
gem 'oj'
gem 'hiredis'
gem 'redis', '~>3.2', require: ['redis', 'redis/connection/hiredis']
......
......@@ -240,6 +240,7 @@ GEM
addressable (~> 2.4)
http (~> 2.0)
nokogiri (~> 1.6)
ox (2.4.11)
paperclip (5.1.0)
activemodel (>= 4.2.0)
activesupport (>= 4.2.0)
......@@ -482,6 +483,7 @@ DEPENDENCIES
nokogiri
oj
ostatus2
ox
paperclip (~> 5.1)
paperclip-av-transcoder
pg
......
......@@ -16,7 +16,8 @@ class AccountsController < ApplicationController
end
format.atom do
@entries = @account.stream_entries.order('id desc').where(activity_type: 'Status').where(hidden: false).with_includes.paginate_by_max_id(20, params[:max_id], params[:since_id])
@entries = @account.stream_entries.order('id desc').where(hidden: false).with_includes.paginate_by_max_id(20, params[:max_id], params[:since_id])
render xml: AtomSerializer.render(AtomSerializer.new.feed(@account, @entries.to_a))
end
format.activitystreams2
......
......@@ -19,7 +19,9 @@ class StreamEntriesController < ApplicationController
end
end
format.atom
format.atom do
render xml: AtomSerializer.render(AtomSerializer.new.entry(@stream_entry, true))
end
end
end
......
# frozen_string_literal: true
class AtomSerializer
include RoutingHelper
class << self
def render(element)
document = Ox::Document.new(version: '1.0')
document << element
"<?xml version=\"1.0\"?>#{Ox.dump(element)}"
end
end
def author(account)
author = Ox::Element.new('author')
uri = TagManager.instance.uri_for(account)
append_element(author, 'id', uri)
append_element(author, 'activity:object-type', TagManager::TYPES[:person])
append_element(author, 'uri', uri)
append_element(author, 'name', account.username)
append_element(author, 'email', account.local? ? "#{account.acct}@#{Rails.configuration.x.local_domain}" : account.acct)
append_element(author, 'summary', account.note)
append_element(author, 'link', nil, rel: :alternate, type: 'text/html', href: TagManager.instance.url_for(account))
append_element(author, 'link', nil, rel: :avatar, type: account.avatar_content_type, 'media:width': 120, 'media:height': 120, href: full_asset_url(account.avatar.url(:original)))
append_element(author, 'link', nil, rel: :header, type: account.header_content_type, 'media:width': 700, 'media:height': 335, href: full_asset_url(account.header.url(:original)))
append_element(author, 'poco:preferredUsername', account.username)
append_element(author, 'poco:displayName', account.display_name) unless account.display_name.blank?
append_element(author, 'poco:note', Formatter.instance.simplified_format(account).to_str) unless account.note.blank?
append_element(author, 'mastodon:scope', account.locked? ? :private : :public)
author
end
def feed(account, stream_entries)
feed = Ox::Element.new('feed')
add_namespaces(feed)
append_element(feed, 'id', account_url(account, format: 'atom'))
append_element(feed, 'title', account.display_name)
append_element(feed, 'subtitle', account.note)
append_element(feed, 'updated', account.updated_at.iso8601)
append_element(feed, 'logo', full_asset_url(account.avatar.url(:original)))
feed << author(account)
append_element(feed, 'link', nil, rel: :alternate, type: 'text/html', href: TagManager.instance.url_for(account))
append_element(feed, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_url(account, format: 'atom'))
append_element(feed, 'link', nil, rel: :next, type: 'application/atom+xml', href: account_url(account, format: 'atom', max_id: stream_entries.last.id)) if stream_entries.size == 20
append_element(feed, 'link', nil, rel: :hub, href: api_push_url)
append_element(feed, 'link', nil, rel: :salmon, href: api_salmon_url(account.id))
stream_entries.each do |stream_entry|
feed << entry(stream_entry)
end
feed
end
def entry(stream_entry, root = false)
entry = Ox::Element.new('entry')
add_namespaces(entry) if root
append_element(entry, 'id', TagManager.instance.unique_tag(stream_entry.created_at, stream_entry.activity_id, stream_entry.activity_type))
append_element(entry, 'published', stream_entry.created_at.iso8601)
append_element(entry, 'updated', stream_entry.updated_at.iso8601)
append_element(entry, 'title', stream_entry&.status&.title)
append_element(entry, 'activity:object-type', TagManager::TYPES[stream_entry.object_type])
append_element(entry, 'activity:verb', TagManager::VERBS[stream_entry.verb])
entry << object(stream_entry.target) if stream_entry.targeted?
serialize_status_attributes(entry, stream_entry.status) unless stream_entry.status.nil?
append_element(entry, 'link', nil, rel: :alternate, type: 'text/html', href: account_stream_entry_url(stream_entry.account, stream_entry))
append_element(entry, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_stream_entry_url(stream_entry.account, stream_entry, format: 'atom'))
append_element(entry, 'thr:in-reply-to', nil, ref: TagManager.instance.uri_for(stream_entry.thread), href: TagManager.instance.url_for(stream_entry.thread)) if stream_entry.threaded?
entry
end
def object(status)
object = Ox::Element.new('activity:object')
append_element(object, 'id', TagManager.instance.uri_for(status))
append_element(object, 'published', status.created_at.iso8601)
append_element(object, 'updated', status.updated_at.iso8601)
append_element(object, 'title', status.title)
object << author(status.account)
append_element(object, 'activity:object-type', TagManager::TYPES[status.object_type])
append_element(object, 'activity:verb', TagManager::VERBS[status.verb])
serialize_status_attributes(object, status)
append_element(object, 'link', nil, rel: :alternate, type: 'text/html', href: TagManager.instance.url_for(status))
append_element(object, 'thr:in-reply-to', nil, ref: TagManager.instance.uri_for(status.thread), href: TagManager.instance.url_for(status.thread)) if status.reply?
object
end
def follow_salmon(follow)
entry = Ox::Element.new('entry')
add_namespaces(entry)
description = "#{follow.account.acct} started following #{follow.target_account.acct}"
append_element(entry, 'id', TagManager.instance.unique_tag(follow.created_at, follow.id, 'Follow'))
append_element(entry, 'title', description)
append_element(entry, 'content', description, type: :html)
entry << author(follow.account)
append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', TagManager::VERBS[:follow])
object = author(follow.target_account)
object.value = 'activity:object'
entry << object
entry
end
def follow_request_salmon(follow_request)
entry = Ox::Element.new('entry')
add_namespaces(entry)
append_element(entry, 'id', TagManager.instance.unique_tag(follow_request.created_at, follow_request.id, 'FollowRequest'))
append_element(entry, 'title', "#{follow_request.account.acct} requested to follow #{follow_request.target_account.acct}")
entry << author(follow_request.account)
append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', TagManager::VERBS[:request_friend])
object = author(follow_request.target_account)
object.value = 'activity:object'
entry << object
entry
end
def authorize_follow_request_salmon(follow_request)
entry = Ox::Element.new('entry')
add_namespaces(entry)
append_element(entry, 'id', TagManager.instance.unique_tag(Time.now.utc, follow_request.id, 'FollowRequest'))
append_element(entry, 'title', "#{follow_request.target_account.acct} authorizes follow request by #{follow_request.account.acct}")
entry << author(follow_request.target_account)
append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', TagManager::VERBS[:authorize])
object = Ox::Element.new('activity:object')
object << author(follow_request.account)
append_element(object, 'activity:object-type', TagManager::TYPES[:activity])
append_element(object, 'activity:verb', TagManager::VERBS[:request_friend])
inner_object = author(follow_request.target_account)
inner_object.value = 'activity:object'
object << inner_object
entry << object
entry
end
def reject_follow_request_salmon(follow_request)
entry = Ox::Element.new('entry')
add_namespaces(entry)
append_element(entry, 'id', TagManager.instance.unique_tag(Time.now.utc, follow_request.id, 'FollowRequest'))
append_element(entry, 'title', "#{follow_request.target_account.acct} rejects follow request by #{follow_request.account.acct}")
entry << author(follow_request.target_account)
append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', TagManager::VERBS[:reject])
object = Ox::Element.new('activity:object')
object << author(follow_request.account)
append_element(object, 'activity:object-type', TagManager::TYPES[:activity])
append_element(object, 'activity:verb', TagManager::VERBS[:request_friend])
inner_object = author(follow_request.target_account)
inner_object.value = 'activity:object'
object << inner_object
entry << object
entry
end
def unfollow_salmon(follow)
entry = Ox::Element.new('entry')
add_namespaces(entry)
description = "#{follow.account.acct} is no longer following #{follow.target_account.acct}"
append_element(entry, 'id', TagManager.instance.unique_tag(Time.now.utc, follow.id, 'Follow'))
append_element(entry, 'title', description)
append_element(entry, 'content', description, type: :html)
entry << author(follow.account)
append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', TagManager::VERBS[:unfollow])
object = author(follow.target_account)
object.value = 'activity:object'
entry << object
entry
end
def block_salmon(block)
entry = Ox::Element.new('entry')
add_namespaces(entry)
description = "#{block.account.acct} no longer wishes to interact with #{block.target_account.acct}"
append_element(entry, 'id', TagManager.instance.unique_tag(Time.now.utc, block.id, 'Block'))
append_element(entry, 'title', description)
entry << author(block.account)
append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', TagManager::VERBS[:block])
object = author(block.target_account)
object.value = 'activity:object'
entry << object
entry
end
def unblock_salmon(block)
entry = Ox::Element.new('entry')
add_namespaces(entry)
description = "#{block.account.acct} no longer blocks #{block.target_account.acct}"
append_element(entry, 'id', TagManager.instance.unique_tag(Time.now.utc, block.id, 'Block'))
append_element(entry, 'title', description)
entry << author(block.account)
append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', TagManager::VERBS[:unblock])
object = author(block.target_account)
object.value = 'activity:object'
entry << object
entry
end
def favourite_salmon(favourite)
entry = Ox::Element.new('entry')
add_namespaces(entry)
description = "#{favourite.account.acct} favourited a status by #{favourite.status.account.acct}"
append_element(entry, 'id', TagManager.instance.unique_tag(favourite.created_at, favourite.id, 'Favourite'))
append_element(entry, 'title', description)
append_element(entry, 'content', description, type: :html)
entry << author(favourite.account)
append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', TagManager::VERBS[:favorite])
entry << object(favourite.status)
append_element(entry, 'thr:in-reply-to', nil, ref: TagManager.instance.uri_for(favourite.status), href: TagManager.instance.url_for(favourite.status))
entry
end
def unfavourite_salmon(favourite)
entry = Ox::Element.new('entry')
add_namespaces(entry)
description = "#{favourite.account.acct} no longer favourites a status by #{favourite.status.account.acct}"
append_element(entry, 'id', TagManager.instance.unique_tag(Time.now.utc, favourite.id, 'Favourite'))
append_element(entry, 'title', description)
append_element(entry, 'content', description, type: :html)
entry << author(favourite.account)
append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', TagManager::VERBS[:unfavorite])
entry << object(favourite.status)
append_element(entry, 'thr:in-reply-to', nil, ref: TagManager.instance.uri_for(favourite.status), href: TagManager.instance.url_for(favourite.status))
entry
end
private
def append_element(parent, name, content = nil, attributes = {})
element = Ox::Element.new(name)
attributes.each { |k, v| element[k] = v.to_s }
element << content.to_s unless content.nil?
parent << element
end
def add_namespaces(parent)
parent['xmlns'] = TagManager::XMLNS
parent['xmlns:thr'] = TagManager::THR_XMLNS
parent['xmlns:activity'] = TagManager::AS_XMLNS
parent['xmlns:poco'] = TagManager::POCO_XMLNS
parent['xmlns:media'] = TagManager::MEDIA_XMLNS
parent['xmlns:ostatus'] = TagManager::OS_XMLNS
parent['xmlns:mastodon'] = TagManager::MTDN_XMLNS
end
def serialize_status_attributes(entry, status)
append_element(entry, 'summary', status.spoiler_text) unless status.spoiler_text.blank?
append_element(entry, 'content', Formatter.instance.format(status.reblog? ? status.reblog : status).to_str, type: 'html')
status.mentions.each do |mentioned|
append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': TagManager::TYPES[:person], href: TagManager.instance.uri_for(mentioned.account))
end
append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': TagManager::TYPES[:collection], href: TagManager::COLLECTIONS[:public]) if status.public_visibility?
status.tags.each do |tag|
append_element(entry, 'category', nil, term: tag.name)
end
append_element(entry, 'category', nil, term: 'nsfw') if status.sensitive?
status.media_attachments.each do |media|
append_element(entry, 'link', nil, rel: :enclosure, type: media.file_content_type, length: media.file_file_size, href: full_asset_url(media.file.url(:original, false)))
end
append_element(entry, 'mastodon:scope', status.visibility)
end
end
......@@ -78,6 +78,8 @@ class TagManager
case target.object_type
when :person
account_url(target)
when :note, :comment, :activity
unique_tag(target.created_at, target.id, 'Status')
else
unique_tag(target.stream_entry.created_at, target.stream_entry.activity_id, target.stream_entry.activity_type)
end
......
......@@ -5,25 +5,21 @@ class StreamEntry < ApplicationRecord
belongs_to :account, inverse_of: :stream_entries
belongs_to :activity, polymorphic: true
belongs_to :status, foreign_type: 'Status', foreign_key: 'activity_id', inverse_of: :stream_entry
validates :account, :activity, presence: true
STATUS_INCLUDES = [:account, :stream_entry, :media_attachments, :tags, mentions: :account, reblog: [:stream_entry, :account, mentions: :account], thread: [:stream_entry, :account]].freeze
STATUS_INCLUDES = [:account, :stream_entry, :media_attachments, :tags, mentions: :account, reblog: [:stream_entry, :account, :media_attachments, :tags, mentions: :account], thread: [:stream_entry, :account]].freeze
default_scope { where(activity_type: 'Status') }
scope :with_includes, -> { includes(:account, status: STATUS_INCLUDES) }
def object_type
if orphaned?
:activity
else
targeted? ? :activity : activity.object_type
end
orphaned? || targeted? ? :activity : status.object_type
end
def verb
orphaned? ? :delete : activity.verb
orphaned? ? :delete : status.verb
end
def targeted?
......@@ -31,15 +27,15 @@ class StreamEntry < ApplicationRecord
end
def target
orphaned? ? nil : activity.target
orphaned? ? nil : status.target
end
def title
orphaned? ? nil : activity.title
orphaned? ? nil : status.title
end
def content
orphaned? ? nil : activity.content
orphaned? ? nil : status.content
end
def threaded?
......@@ -47,20 +43,16 @@ class StreamEntry < ApplicationRecord
end
def thread
orphaned? ? nil : activity.thread
orphaned? ? nil : status.thread
end
def mentions
activity.respond_to?(:mentions) ? activity.mentions.map(&:account) : []
end
def activity
!new_record? ? send(activity_type.underscore) || super : super
orphaned? ? [] : status.mentions.map(&:account)
end
private
def orphaned?
activity.nil?
status.nil?
end
end
......@@ -9,20 +9,20 @@ class AfterBlockService < BaseService
private
def clear_timelines(account, target_account)
mentions_key = FeedManager.instance.key(:mentions, account.id)
home_key = FeedManager.instance.key(:home, account.id)
redis.pipelined do
target_account.statuses.select('id').find_each do |status|
redis.zrem(mentions_key, status.id)
redis.zrem(home_key, status.id)
end
end
end
def clear_notifications(account, target_account)
Notification.where(account: account).joins(:follow).where(activity_type: 'Follow', follows: { account_id: target_account.id }).destroy_all
Notification.where(account: account).joins(mention: :status).where(activity_type: 'Mention', statuses: { account_id: target_account.id }).destroy_all
Notification.where(account: account).joins(:favourite).where(activity_type: 'Favourite', favourites: { account_id: target_account.id }).destroy_all
Notification.where(account: account).joins(:status).where(activity_type: 'Status', statuses: { account_id: target_account.id }).destroy_all
Notification.where(account: account).joins(:follow).where(activity_type: 'Follow', follows: { account_id: target_account.id }).delete_all
Notification.where(account: account).joins(mention: :status).where(activity_type: 'Mention', statuses: { account_id: target_account.id }).delete_all
Notification.where(account: account).joins(:favourite).where(activity_type: 'Favourite', favourites: { account_id: target_account.id }).delete_all
Notification.where(account: account).joins(:status).where(activity_type: 'Status', statuses: { account_id: target_account.id }).delete_all
end
def redis
......
......@@ -10,31 +10,6 @@ class AuthorizeFollowService < BaseService
private
def build_xml(follow_request)
Nokogiri::XML::Builder.new do |xml|
entry(xml, true) do
unique_id xml, Time.now.utc, follow_request.id, 'FollowRequest'
title xml, "#{follow_request.target_account.acct} authorizes follow request by #{follow_request.account.acct}"
author(xml) do
include_author xml, follow_request.target_account
end
object_type xml, :activity
verb xml, :authorize
target(xml) do
author(xml) do
include_author xml, follow_request.account
end
object_type xml, :activity
verb xml, :request_friend
target(xml) do
include_author xml, follow_request.target_account
end
end
end
end.to_xml
AtomSerializer.render(AtomSerializer.new.authorize_follow_request_salmon(follow_request))
end
end
......@@ -18,22 +18,6 @@ class BlockService < BaseService
private
def build_xml(block)
Nokogiri::XML::Builder.new do |xml|
entry(xml, true) do
unique_id xml, block.created_at, block.id, 'Block'
title xml, "#{block.account.acct} no longer wishes to interact with #{block.target_account.acct}"
author(xml) do
include_author xml, block.account
end
object_type xml, :activity
verb xml, :block
target(xml) do
include_author xml, block.target_account
end
end
end.to_xml
AtomSerializer.render(AtomSerializer.new.block_salmon(block))
end
end
......@@ -2,7 +2,6 @@
module StreamEntryRenderer
def stream_entry_to_xml(stream_entry)
renderer = StreamEntriesController.renderer.new(method: 'get', http_host: Rails.configuration.x.local_domain, https: Rails.configuration.x.use_https)
renderer.render(:show, assigns: { stream_entry: stream_entry }, formats: [:atom])
AtomSerializer.render(AtomSerializer.new.entry(stream_entry, true))
end
end
......@@ -22,26 +22,6 @@ class FavouriteService < BaseService
private
def build_xml(favourite)
description = "#{favourite.account.acct} favourited a status by #{favourite.status.account.acct}"
Nokogiri::XML::Builder.new do |xml|
entry(xml, true) do
unique_id xml, favourite.created_at, favourite.id, 'Favourite'
title xml, description
content xml, description
author(xml) do
include_author xml, favourite.account
end
object_type xml, :activity
verb xml, :favorite
in_reply_to xml, TagManager.instance.uri_for(favourite.status), TagManager.instance.url_for(favourite.status)
target(xml) do
include_target xml, favourite.status
end
end
end.to_xml
AtomSerializer.render(AtomSerializer.new.favourite_salmon(favourite))
end
end
......@@ -55,48 +55,10 @@ class FollowService < BaseService
end
def build_follow_request_xml(follow_request)
description = "#{follow_request.account.acct} requested to follow #{follow_request.target_account.acct}"
Nokogiri::XML::Builder.new do |xml|
entry(xml, true) do
unique_id xml, follow_request.created_at, follow_request.id, 'FollowRequest'
title xml, description
content xml, description
author(xml) do
include_author xml, follow_request.account
end
object_type xml, :activity
verb xml, :request_friend
target(xml) do
include_author xml, follow_request.target_account
end
end
end.to_xml
AtomSerializer.render(AtomSerializer.new.follow_request_salmon(follow_request))
end
def build_follow_xml(follow)
description = "#{follow.account.acct} started following #{follow.target_account.acct}"
Nokogiri::XML::Builder.new do |xml|
entry(xml, true) do
unique_id xml, follow.created_at, follow.id, 'Follow'
title xml, description
content xml, description
author(xml) do
include_author xml, follow.account
end
object_type xml, :activity
verb xml, :follow
target(xml) do
include_author xml, follow.target_account
end
end
end.to_xml
AtomSerializer.render(AtomSerializer.new.follow_salmon(follow))
end
end
......@@ -10,31 +10,6 @@ class RejectFollowService < BaseService
private
def build_xml(follow_request)
Nokogiri::XML::Builder.new do |xml|
entry(xml, true) do
unique_id xml, Time.now.utc, follow_request.id, 'FollowRequest'
title xml, "#{follow_request.target_account.acct} rejects follow request by #{follow_request.account.acct}"
author(xml) do
include_author xml, follow_request.target_account
end
object_type xml, :activity
verb xml, :reject
target(xml) do
author(xml) do
include_author xml, follow_request.account
end
object_type xml, :activity
verb xml, :request_friend
target(xml) do
include_author xml, follow_request.target_account
end
end
end
end.to_xml
AtomSerializer.render(AtomSerializer.new.reject_follow_request_salmon(follow_request))
end
end
......@@ -11,22 +11,6 @@ class UnblockService < BaseService
private
def build_xml(block)
Nokogiri::XML::Builder.new do |xml|
entry(xml, true) do
unique_id xml, Time.now.utc, block.id, 'Block'