Commit f10e4900 authored by Michael Rose's avatar Michael Rose

api: fetch unknown stories at request time

parent eb816ea3
......@@ -9,7 +9,12 @@ class Api::V1::StoriesController < ApplicationController
private
def set_story
@story = Story.find(params[:id])
@story = Story.find_by(id: params[:id])
if @story == nil
FetchAndIndexWorker.new.perform(params[:id])
@story = Story.find_by(id: params[:id])
end
end
def story_params
......
require 'pp'
class FetchAndIndexWorker
include Sidekiq::Worker
sidekiq_options :backtrace => true
def perform(story_id)
url = "https://www.fanfiction.net/s/#{story_id}/1/"
logger.info "[#{story_id}] Checking #{url} for a broken link..."
response = Faraday.get(url) do |req|
req.headers['User-Agent'] = 'Accio!IndexFetch/1.0'
end
if response.body.include?('Unable to locate story. Code 1')
# story = Story.find_by(id: story_id)
# unless story.nil?
# story.deleted
# end
else
logger.info "[#{story_id}] Found story at #{url}"
document = Nokogiri::HTML.parse(response.body)
parsed_doc = parse_document(story_id, document)
index = parsed_doc[:fandoms].size > 1 ? 'ffn_index_crossover' : 'ffn_index'
IndexStoryWorker.new.perform(story_id, parsed_doc, index)
end
end
def parse_document(story_id, document)
pre_story_link = document.css('#pre_story_links a').last
pre_story_url = pre_story_link.attr('href')
fandoms = []
if pre_story_link.text.include?('Crossover')
m = /\/([0-9]+)\/([0-9]+)\//.match(pre_story_url)
fandoms << {
fandom_id: m[1].to_i
}
fandoms << {
fandom_id: m[2].to_i
}
else
m = /\/([^\\]+)\/([^\\]+)\//.match(pre_story_url)
fandom_category = FandomCategory.find_by(slug: m[1])
fandom = Fandom.find_by(name: m[2].gsub('-', ' '), category_id: fandom_category.id)
fandoms << {
fandom_id: fandom.id
}
end
title = document.css('#profile_top b').text
author_doc = document.css('#profile_top a[href^=\'/u/\']')
author = author_doc.text
author_id = /\/u\/([0-9]+)\//.match(author_doc.first.attr('href'))[1].to_i
summary = document.css('#profile_top').css('div')[2].text
meta = document.css('#profile_top .xgray').text
times = document.css('#profile_top span[data-xutime]')
updated = DateTime.strptime(times[0].attr('data-xutime'),'%s').iso8601
published = DateTime.strptime(times[1].attr('data-xutime'),'%s').iso8601
last_chapter = document.css('#chap_select option').last.attr('value').to_i
url = "https://www.fanfiction.net/s/#{story_id}/1/"
url_latest = "https://www.fanfiction.net/s/#{story_id}/#{last_chapter}/"
{
story_id: story_id,
title: title,
author: author,
author_id: author_id,
fandoms: fandoms,
summary: summary,
meta: {
info_string: meta,
},
updated: updated,
published: published,
last_seen: DateTime.now.iso8601,
url: url,
url_latest: url_latest,
}
end
end
......@@ -53,6 +53,8 @@ module Scryer
true
when part.match(/Updated:/)
true
when part.match(/id:/)
true
when part == 'In-Progress'
m.status = part
true
......
......@@ -118,6 +118,40 @@ class MetadataParserTest < ActiveSupport::TestCase
actual = Scryer::FFN::MetadataParser.parse_info_string(str, [8])
assert_equal expected, actual
end
test 'handles_story_with_id' do
str = 'Rated: M - English - Chapters: 1 - Words: 33 - Published: - Darth Vader, Jango Fett, General Grievous, Darth Maul - id: 9704180 '
expected = Scryer::FFN::Metadata.new(
:categories => [],
:characters => [
{
:id => 66813,
:name => 'Darth Vader'
},
{
:id => 37718,
:name => 'Jango Fett'
},
{
:id => 16082,
:name => 'Darth Maul'
},
],
:chapters => 1,
:favs => 0,
:follows => 0,
:rated => 'M',
:relationships => [],
:reviews => 0,
:status => 'In-Progress',
:language => 'English',
:words => 33
)
actual = Scryer::FFN::MetadataParser.parse_info_string(str, [8])
assert_equal expected, actual
end
end
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment