Commit c9681e41 authored by HankG's avatar HankG

Bugfix for Twitter timeline interval hiccups

parent edd5bb27
......@@ -46,7 +46,8 @@ namespace MySocialPortalLib.Repository
ulong stopValue,
string serviceName = DefaultServiceName)
{
return TimelineRepository.Query<TimelineInterval>(serviceName)
//BSON doesn't have ulong type so need to bring the whole collection back and do the LINQ query at client
return TimelineRepository.Query<TimelineInterval>(serviceName).ToList()
.Where(i => (i.IntervalStart >= startValue && i.IntervalStop <= stopValue)
|| (i.IntervalStart < startValue && i.IntervalStop >= startValue && i.IntervalStop <= stopValue)
|| (i.IntervalStop > stopValue && i.IntervalStart >= startValue && i.IntervalStart <= stopValue))
......@@ -58,8 +59,9 @@ namespace MySocialPortalLib.Repository
ulong stopValue,
string serviceName = DefaultServiceName)
{
//BSON doesn't have ulong type so need to bring the whole collection back and do the LINQ query at client
return TimelineRepository.Query<TimelineInterval>(serviceName)
.Where(i => i.TimelineName == timelineName)
.Where(i => i.TimelineName == timelineName).ToList()
.Where(i => (i.IntervalStart >= startValue && i.IntervalStop <= stopValue)
|| (i.IntervalStart < startValue && i.IntervalStop >= startValue && i.IntervalStop <= stopValue)
|| (i.IntervalStop > stopValue && i.IntervalStart >= startValue && i.IntervalStart <= stopValue))
......
......@@ -172,39 +172,48 @@ namespace MySocialPortalLib.Service.SocialMediaConnectors
private IList<Post> PullHomeTweets(TimelineInterval interval, int maxPosts)
{
IQueryable<Status> query;
var updateUpper = true;
var removeIntervalIfEmpty = false;
Console.WriteLine($"PullHomeTweets {maxPosts} #posts for {interval}");
if (interval.IntervalStop == ulong.MaxValue)
{
Console.WriteLine("Pulling home tweets with no upper bound");
query = GetNoUpperHomeStatusQuery(interval, maxPosts);
removeIntervalIfEmpty = true;
}
else
{
Console.WriteLine("Pulling upper and lower bound tweets");
query = GetUpperLowerHomeStatusQuery(interval, maxPosts);
updateUpper = false;
}
return PullTweets(interval, maxPosts, query);
return PullTweets(interval, maxPosts, query, updateUpper, removeIntervalIfEmpty);
}
private IList<Post> PullUserTweets(string username, TimelineInterval interval, int maxPosts)
{
IQueryable<Status> query;
bool updateUpper = true;
bool removeIntervalIfEmpty = false;
if (interval.IntervalStop == ulong.MaxValue)
{
Console.WriteLine($"Pulling tweets for Twitter user {username} with no upper bounds: {interval}");
query = GetNoUpperUserStatusQuery(username, interval, maxPosts);
removeIntervalIfEmpty = true;
}
else
{
Console.WriteLine($"Pulling tweets for Twitter user {username} with no upper and lower bounds: {interval}");
Console.WriteLine($"Pulling tweets for Twitter user {username} with upper and lower bounds: {interval}");
query = GetUpperLowerUserStatusQuery(username, interval, maxPosts);
updateUpper = false;
}
return PullTweets(interval, maxPosts, query);
return PullTweets(interval, maxPosts, query, updateUpper, removeIntervalIfEmpty);
}
private IList<Post> PullTweets(TimelineInterval newInterval, int maxPosts, IQueryable<Status> query)
private IList<Post> PullTweets(TimelineInterval newInterval, int maxPosts, IQueryable<Status> query, bool updateUpper, bool removeRequestIfEmpty)
{
var posts = new List<Post>();
if (newInterval == null)
......@@ -217,21 +226,30 @@ namespace MySocialPortalLib.Service.SocialMediaConnectors
Console.WriteLine($"{tweets.Count} tweets returned");
if (tweets.Count == 0)
{
if (removeRequestIfEmpty)
{
TimelineManager.RemoveRequestInterval(newInterval);
}
return posts;
}
UpdateSampleInterval(tweets, newInterval, maxPosts);
UpdateSampleInterval(tweets, newInterval, maxPosts, updateUpper);
var converter = new TwitterConverter(PersonsRepository);
posts.AddRange(tweets.Select(t => converter.TweetToPost(t)));
return posts;
}
private void UpdateSampleInterval(IList<Status> tweets, TimelineInterval newInterval, int maxPosts)
private void UpdateSampleInterval(IList<Status> tweets, TimelineInterval newInterval, int maxPosts, bool updateUpper)
{
var earliestTweet = tweets.Min(t => t.StatusID);
var latestTweet = tweets.Max(t => t.StatusID);
Console.WriteLine($"TweetIDs range: {earliestTweet} to {latestTweet}");
newInterval.IntervalStop = latestTweet;
if (updateUpper)
{
newInterval.IntervalStop = latestTweet;
}
if (tweets.Count == maxPosts)
{
Console.WriteLine("Max posts returned therefore setting interval start as well");
......
......@@ -136,12 +136,18 @@ namespace MySocialPortalLib.Service
return success;
}
public bool RemoveRequestInterval(TimelineInterval interval)
{
return TimelineRepository.Remove(interval, ServiceName);
}
private ITimelineRepository TimelineRepository { get; }
private void CleanupIntervals(string timelineName)
{
var gapTolerance = 2UL;
var allIntervals = TimelineRepository.FindByTimelineName(timelineName, ServiceName);
var mergedIntervals = TimelineIntervalUtilities.CleanupList(allIntervals);
var mergedIntervals = TimelineIntervalUtilities.CleanupList(allIntervals, gapTolerance);
TimelineRepository.RemoveByTimelineName(timelineName, ServiceName);
TimelineRepository.AddOrUpdate(mergedIntervals, ServiceName);
}
......
......@@ -107,47 +107,48 @@ namespace MySocialPortalLibTest.Repository
public void TestFindByInterval()
{
var db = GetTempDb();
var baseIntervalValue = ulong.MaxValue - 10000;
var interval1 = new TimelineInterval
{
IntervalStart = 4,
IntervalStop = 6,
IntervalStart = baseIntervalValue + 4,
IntervalStop = baseIntervalValue + 6,
TimelineName = "Name1"
};
db.AddOrUpdate(interval1);
var interval2 = new TimelineInterval
{
IntervalStart = 8,
IntervalStop = 9,
IntervalStart = baseIntervalValue + 8,
IntervalStop = baseIntervalValue + 9,
TimelineName = "Name1"
};
db.AddOrUpdate(interval2);
var interval3 = new TimelineInterval
{
IntervalStart = 10,
IntervalStop = 12,
IntervalStart = baseIntervalValue + 10,
IntervalStop = baseIntervalValue + 12,
TimelineName = "Name1"
};
db.AddOrUpdate(interval3);
var interval4 = new TimelineInterval
{
IntervalStart = 0,
IntervalStop = 1,
IntervalStart = baseIntervalValue + 0,
IntervalStop = baseIntervalValue + 1,
TimelineName = "Name1"
};
db.AddOrUpdate(interval4);
var interval5 = new TimelineInterval
{
IntervalStart = 15,
IntervalStop = 17,
IntervalStart = baseIntervalValue + 15,
IntervalStop = baseIntervalValue + 17,
TimelineName = "Name1"
};
db.AddOrUpdate(interval5);
var results = db.FindByInterval(5, 11);
var results = db.FindByInterval(baseIntervalValue + 5, baseIntervalValue + 11);
Assert.Equal(3, results.Count);
Assert.True(results.Contains(interval1));
Assert.True(results.Contains(interval2));
......
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
......@@ -191,7 +192,7 @@ namespace MySocialPortalLibTest.Service
{
break;
}
resultsInterval = SimulateQuerySpan(requestInterval, maxInterval);
resultsInterval = SimulateQuerySpan(requestInterval, maxInterval, true, true);
service.UpdateRequestedInterval(resultsInterval);
PrintIntervals(db.FindByTimelineName(name, ServiceName));
}
......@@ -218,11 +219,17 @@ namespace MySocialPortalLibTest.Service
_testOutputHelper.WriteLine("");
}
private TimelineInterval SimulateQuerySpan(TimelineInterval requestInterval, ulong maxDelta = ulong.MaxValue, bool forward = true)
private TimelineInterval SimulateQuerySpan(TimelineInterval requestInterval, ulong maxDelta = ulong.MaxValue,
bool forward = true, bool simulateNone = false)
{
var newInterval = requestInterval.Copy();
var delta = requestInterval.IntervalStop - requestInterval.IntervalStart;
var actualDelta = delta > maxDelta ? maxDelta : delta;
if (simulateNone && new Random().NextDouble() > 0.5)
{
actualDelta = 0;
}
if (forward)
{
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment