Commit 3663a209 authored by Michael Rose's avatar Michael Rose

story: add deleted_at column, workers respect it

parent 403cdd7c
......@@ -8,6 +8,10 @@ class Story < ActiveRecord::Base
Story.all.find_in_batches(start: 0, batch_size: 20) { |g| g.each(&:reindex) }
end
def deleted?
!deleted_at.nil?
end
def reindex
ReindexStoryWorker.perform_async(id, crossover?)
end
......
......@@ -17,13 +17,22 @@ class BrokenLinkWorker
if response.body.include?('Unable to locate story. Code 1')
logger.info "[#{story_id}] FFN response included 'Code 1', marking as deleted."
index = is_crossover ? 'ffn_index_crossover' : 'ffn_index'
story = Story.find_by(id: story_id)
if story.nil?
index = is_crossover ? 'ffn_index_crossover' : 'ffn_index'
es_doc = $elasticsearch.get index: index, id: story_id, type: 'story'
src = es_doc['_source']
src['deleted'] = true
es_doc = $elasticsearch.get index: index, id: story_id, type: 'story'
$elasticsearch.index index: index, type: es_doc['_type'], id: story_id, body: src
src = es_doc['_source']
src['deleted'] = true
$elasticsearch.index index: index, type: es_doc['_type'], id: story_id, body: src
else
story.deleted_at = DateTime.now
story.save
story.reindex
end
else
logger.info "[#{story_id}] Found story at #{url}, skip marking as deleted."
end
......
......@@ -7,17 +7,20 @@ class FetchAndIndexWorker
def perform(story_id)
url = "https://www.fanfiction.net/s/#{story_id}/1/"
logger.info "[#{story_id}] Checking #{url} for a broken link..."
logger.info "[#{story_id}] Live-indexing #{url}..."
response = Faraday.get(url) do |req|
req.headers['User-Agent'] = 'Accio!IndexFetch/1.0'
end
if response.body.include?('Unable to locate story. Code 1')
# story = Story.find_by(id: story_id)
# unless story.nil?
# story.deleted
# end
logger.info "[#{story_id}] Story appears deleted and row exists, de-indexing."
story = Story.find_by(id: story_id)
unless story.nil?
story.deleted_at = DateTime.now
story.save
story.reindex
end
else
logger.info "[#{story_id}] Found story at #{url}"
......
......@@ -21,7 +21,14 @@ class ReindexStoryWorker < StoryIndexBase
$elasticsearch.delete index: 'ffncrossover_index', id: story_id, ignore: 404, type: 'story'
end
logger.info "[#{story_id}] Reindexing story"
story = Story.find_by(id: story_id)
if story.deleted?
logger.info "[#{story_id}] Story is marked at deleted, removing from index."
$elasticsearch.delete index: index_name, id: story_id, ignore: 404, type: 'story'
return
end
logger.info "[#{story_id}] Reindexing story."
if is_crossover
story = story_crossover
......
class AddDeletedAtToStory < ActiveRecord::Migration[5.1]
def change
add_column :stories, :deleted_at, :timestamp
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment