Commit e7410308 authored by HankG's avatar HankG

Add TwitterConnector GetHomeTimeline (both directions) working w/tests

parent 1f7442fa
......@@ -38,6 +38,7 @@ namespace MySocialPortalLib.Converter
Body = string.IsNullOrWhiteSpace(tweet.Text) ? tweet.FullText : tweet.Text ?? "",
UserId = person.Id,
OriginalSocialMediaSystem = StandardSocialNetworkNames.Twitter,
OriginalLinkUrlOrId = tweet.StatusID.ToString(CultureInfo.InvariantCulture),
PostDateTime = tweet.CreatedAt
};
if (tweet?.Entities?.MediaEntities?.Count > 0)
......
......@@ -11,6 +11,7 @@ namespace MySocialPortalDesktop.Factory
private const string LinkPreviewRepositoryDbName = "link_preview.db";
private const string MainPeopleRepositoryDbName = "all_people.db";
private const string ProfileImageRepositoryDbName = "profile_images.db";
private const string TimelineRepositoryDbName = "timeline_cache.db";
private static readonly Lazy<RepositoryFactory> Singleton =
new Lazy<RepositoryFactory>(() => new RepositoryFactory());
......@@ -29,6 +30,8 @@ namespace MySocialPortalDesktop.Factory
public IPersonsRepository MainPeopleRepository => MainPeopleRepositoryLazy.Value;
public ITimelineRepository TimelineRepository => TimelineRepositoryLazy.Value;
public RepositoryFactory()
{
AllPostsRepositoryLazy = new Lazy<IPostsRepository>(BuildAllPostsRepository);
......@@ -36,6 +39,7 @@ namespace MySocialPortalDesktop.Factory
LinkPreviewImageCacheRepositoryLazy = new Lazy<IFileCacheRepository>(BuildLinkPreviewImageCacheRepository);
MainPeopleRepositoryLazy = new Lazy<IPersonsRepository>(BuildPeopleRepository);
ProfileImageCacheRepositoryLazy = new Lazy<IFileCacheRepository>(BuildProfileImageCacheRepository);
TimelineRepositoryLazy = new Lazy<ITimelineRepository>(BuildTimelineRepository);
}
private Lazy<IPostsRepository> AllPostsRepositoryLazy { get; }
......@@ -47,6 +51,8 @@ namespace MySocialPortalDesktop.Factory
private Lazy<IPersonsRepository> MainPeopleRepositoryLazy { get; }
private Lazy<IFileCacheRepository> ProfileImageCacheRepositoryLazy { get; }
private Lazy<ITimelineRepository> TimelineRepositoryLazy { get; }
private IPostsRepository BuildAllPostsRepository()
{
......@@ -73,6 +79,11 @@ namespace MySocialPortalDesktop.Factory
return new PersonsLiteDbRepository(BuildDbPath(MainPeopleRepositoryDbName));
}
private ITimelineRepository BuildTimelineRepository()
{
return new TimelineLiteDbRepository(BuildDbPath(TimelineRepositoryDbName));
}
public void Dispose()
{
Dispose(true);
......@@ -93,6 +104,7 @@ namespace MySocialPortalDesktop.Factory
(LinkPreviewImageCacheRepository as IDisposable)?.Dispose();
(MainPeopleRepository as IDisposable)?.Dispose();
(ProfileImageCacheRepository as IDisposable)?.Dispose();
(TimelineRepository as IDisposable)?.Dispose();
}
_disposed = true;
......
......@@ -20,6 +20,11 @@ namespace MySocialPortalDesktop.Factory
PreviewImageServiceLazy = new Lazy<ImageService>(BuildProfileImageService);
}
public static TimelineManagementService BuildNewTimelineManagementService(string serviceName)
{
return new TimelineManagementService(RepositoryFactory.Instance.TimelineRepository, serviceName);
}
private Lazy<LinkPreviewService> LinkPreviewServiceLazy { get; }
private Lazy<ImageService> PreviewImageServiceLazy { get; }
......
using System.Diagnostics.CodeAnalysis;
using MySocialPortalLib.Model;
using MySocialPortalLib.Service.SocialMediaConnectors;
namespace MySocialPortalDesktop.Factory
{
[SuppressMessage("ReSharper", "CA1822")]
public class SocialMediaConnectorFactory
public static class SocialMediaConnectorFactory
{
public SocialMediaConnectorFactory()
{
}
public TwitterConnector GetNewTwitterConnector()
public static TwitterConnector GetNewTwitterConnector()
{
return new TwitterConnector(RepositoryFactory.Instance.MainPeopleRepository);
return new TwitterConnector(RepositoryFactory.Instance.MainPeopleRepository,
ServiceFactory.BuildNewTimelineManagementService(StandardSocialNetworkNames.Twitter));
}
}
}
\ No newline at end of file
......@@ -18,7 +18,7 @@ namespace MySocialPortalLib.Model
public List<MediaData> Media { get; set; }
public string OriginalLinkUrl { get; set; }
public string OriginalLinkUrlOrId { get; set; }
public string OriginalSocialMediaSystem { get; set; }
......@@ -45,7 +45,7 @@ namespace MySocialPortalLib.Model
Id = Guid.NewGuid().ToString();
Links = new List<ExternalLink>();
Media = new List<MediaData>();
OriginalLinkUrl = "";
OriginalLinkUrlOrId = "";
OriginalSocialMediaSystem = "";
Places = new List<Place>();
Polls = new List<Poll>();
......@@ -72,7 +72,7 @@ namespace MySocialPortalLib.Model
&& Links.All(other.Links.Contains)
&& Media.Count == other.Media.Count
&& Media.All(other.Media.Contains)
&& OriginalLinkUrl == other.OriginalLinkUrl
&& OriginalLinkUrlOrId == other.OriginalLinkUrlOrId
&& OriginalSocialMediaSystem == other.OriginalSocialMediaSystem
&& Places.Count == other.Places.Count
&& Places.All(other.Places.Contains)
......
......@@ -6,26 +6,26 @@ namespace MySocialPortalLib.Repository
{
public interface ITimelineRepository
{
bool AddOrUpdate (TimelineInterval interval);
bool AddOrUpdate (TimelineInterval interval, string serviceName);
bool AddOrUpdate(IEnumerable<TimelineInterval> intervals);
bool AddOrUpdate(IEnumerable<TimelineInterval> intervals, string serviceName);
TimelineInterval? FindById(string id);
TimelineInterval? FindById(string id, string serviceName);
IList<TimelineInterval> FindByInterval(ulong startValue, ulong stopValue);
IList<TimelineInterval> FindByInterval(ulong startValue, ulong stopValue, string serviceName);
IList<TimelineInterval> FindByTimelineName(string timelineName);
IList<TimelineInterval> FindByTimelineName(string timelineName, string serviceName);
bool Remove (TimelineInterval interval);
bool Remove (TimelineInterval interval, string serviceName);
bool RemoveById(string id);
bool RemoveById(string id, string serviceName);
bool RemoveRange(IEnumerable<TimelineInterval> intervals);
bool RemoveRange(IEnumerable<TimelineInterval> intervals, string serviceName);
bool RemoveByTimelineName(string name);
bool RemoveByTimelineName(string name, string serviceName);
long Count();
long Count(string serviceName);
long CountByTimelineName(string timelineName);
long CountByTimelineName(string timelineName, string serviceName);
}
}
\ No newline at end of file
......@@ -10,53 +10,55 @@ namespace MySocialPortalLib.Repository
{
public class TimelineLiteDbRepository : ITimelineRepository, IDisposable
{
private const string DefaultTimelineCollectionName = "timeline";
private const string DefaultServiceName = "MySocialPortalInternal";
private bool _disposed = false;
public TimelineLiteDbRepository (Stream dbStream,
string postsCollectionName = DefaultTimelineCollectionName)
public TimelineLiteDbRepository (Stream dbStream)
{
TimelineRepository = new LiteRepository(dbStream);
TimelineCollectionName = postsCollectionName;
}
public TimelineLiteDbRepository(String filepath, string postsCollectionName = DefaultTimelineCollectionName)
public TimelineLiteDbRepository(String filepath)
{
TimelineRepository = new LiteRepository(filepath);
TimelineCollectionName = postsCollectionName;
}
public bool AddOrUpdate(TimelineInterval interval)
public bool AddOrUpdate(TimelineInterval interval, string serviceName = DefaultServiceName)
{
return TimelineRepository.Upsert(interval, TimelineCollectionName);
return TimelineRepository.Upsert(interval, serviceName);
}
public bool AddOrUpdate(IEnumerable<TimelineInterval> intervals)
public bool AddOrUpdate(IEnumerable<TimelineInterval> intervals, string serviceName = DefaultServiceName)
{
TimelineRepository.Upsert(intervals, TimelineCollectionName);
TimelineRepository.Upsert(intervals, serviceName);
return true;
}
public TimelineInterval? FindById(string id)
public TimelineInterval? FindById(string id, string serviceName = DefaultServiceName)
{
return TimelineRepository.Query<TimelineInterval>(TimelineCollectionName)
return TimelineRepository.Query<TimelineInterval>(serviceName)
.Where(i => i.Id == id)
.FirstOrDefault();
}
public IList<TimelineInterval> FindByInterval(ulong startValue, ulong stopValue)
public IList<TimelineInterval> FindByInterval(ulong startValue,
ulong stopValue,
string serviceName = DefaultServiceName)
{
return TimelineRepository.Query<TimelineInterval>(TimelineCollectionName)
return TimelineRepository.Query<TimelineInterval>(serviceName)
.Where(i => (i.IntervalStart >= startValue && i.IntervalStop <= stopValue)
|| (i.IntervalStart < startValue && i.IntervalStop >= startValue && i.IntervalStop <= stopValue)
|| (i.IntervalStop > stopValue && i.IntervalStart >= startValue && i.IntervalStart <= stopValue))
.ToList();
}
public IList<TimelineInterval> FindByTimelineInterval(string timelineName, ulong startValue, ulong stopValue)
public IList<TimelineInterval> FindByTimelineInterval(string timelineName,
ulong startValue,
ulong stopValue,
string serviceName = DefaultServiceName)
{
return TimelineRepository.Query<TimelineInterval>(TimelineCollectionName)
return TimelineRepository.Query<TimelineInterval>(serviceName)
.Where(i => i.TimelineName == timelineName)
.Where(i => (i.IntervalStart >= startValue && i.IntervalStop <= stopValue)
|| (i.IntervalStart < startValue && i.IntervalStop >= startValue && i.IntervalStop <= stopValue)
......@@ -64,24 +66,24 @@ namespace MySocialPortalLib.Repository
.ToList();
}
public IList<TimelineInterval> FindByTimelineName(string timelineName)
public IList<TimelineInterval> FindByTimelineName(string timelineName, string serviceName = DefaultServiceName)
{
return TimelineRepository.Query<TimelineInterval>(TimelineCollectionName)
return TimelineRepository.Query<TimelineInterval>(serviceName)
.Where(i => i.TimelineName == timelineName)
.ToList();
}
public bool Remove(TimelineInterval interval)
public bool Remove(TimelineInterval interval, string serviceName = DefaultServiceName)
{
return interval != null && RemoveById(interval.Id);
return interval != null && RemoveById(interval.Id, serviceName);
}
public bool RemoveById(string id)
public bool RemoveById(string id, string serviceName = DefaultServiceName)
{
return !string.IsNullOrWhiteSpace(id) && TimelineRepository.Delete<TimelineInterval>(id, TimelineCollectionName);
return !string.IsNullOrWhiteSpace(id) && TimelineRepository.Delete<TimelineInterval>(id, serviceName);
}
public bool RemoveByTimelineName(string name)
public bool RemoveByTimelineName(string name, string serviceName = DefaultServiceName)
{
if (string.IsNullOrWhiteSpace(name))
{
......@@ -90,24 +92,24 @@ namespace MySocialPortalLib.Repository
TimelineRepository.DeleteMany<TimelineInterval>(
interval => interval.TimelineName == name,
TimelineCollectionName);
serviceName);
return FindByTimelineName(name).Count == 0;
}
public bool RemoveRange(IEnumerable<TimelineInterval> intervals)
public bool RemoveRange(IEnumerable<TimelineInterval> intervals, string serviceName = DefaultServiceName)
{
return intervals.Aggregate(true, (current, interval) => current & Remove(interval));
return intervals.Aggregate(true, (current, interval) => current & Remove(interval, serviceName));
}
public long Count()
public long Count(string serviceName = DefaultServiceName)
{
return TimelineRepository.Query<TimelineInterval>(TimelineCollectionName)
return TimelineRepository.Query<TimelineInterval>(serviceName)
.LongCount();
}
public long CountByTimelineName(string timelineName)
public long CountByTimelineName(string timelineName, string serviceName = DefaultServiceName)
{
return TimelineRepository.Query<TimelineInterval>(TimelineCollectionName)
return TimelineRepository.Query<TimelineInterval>(serviceName)
.Where(i => i.TimelineName == timelineName)
.LongCount();
}
......@@ -134,7 +136,5 @@ namespace MySocialPortalLib.Repository
}
private LiteRepository TimelineRepository { get; }
private string TimelineCollectionName { get; }
}
}
\ No newline at end of file
......@@ -5,9 +5,13 @@ namespace MySocialPortalLib.Service
{
public interface ISocialMediaConnector
{
IEnumerable<Post> GetHomeTimeline(int maxPosts);
IEnumerable<Post> GetNewerHomeTimeline(int maxPosts);
IEnumerable<Post> GetOlderHomeTimeline(int maxPosts);
IEnumerable<Post> GetUserTimeline(Person person, int maxPosts);
IEnumerable<Post> GetNewerUserTimeline(Person person, int maxPosts);
IEnumerable<Post> GetOlderUserTimeline(Person person, int maxPosts);
bool UserOnSocialNetwork(Person person);
}
......
......@@ -13,16 +13,24 @@ namespace MySocialPortalLib.Service.SocialMediaConnectors
{
public class TwitterConnector : ISocialMediaConnector
{
public const int DefaultMaxPosts = 50;
public const string HomeTimelineName = "Home_B3F632BB-33D5-4E05-8BE7-2D61D69AE4D4";
public const ulong EarliestTweetValue = 1;
public const ulong LatestTweetValue = ulong.MaxValue;
public IPersonsRepository PersonsRepository { get; }
public TimelineManagementService TimelineManager { get; }
private TwitterConnector()
{
throw new MethodAccessException();
}
public TwitterConnector(IPersonsRepository personsRepository)
public TwitterConnector(IPersonsRepository personsRepository, TimelineManagementService timelineManager)
{
PersonsRepository = personsRepository;
TimelineManager = timelineManager;
Settings = new TwitterConnectorSettings();
if (File.Exists(Settings?.CredentialsPath ?? ""))
{
......@@ -35,22 +43,27 @@ namespace MySocialPortalLib.Service.SocialMediaConnectors
}
}
public IEnumerable<Post> GetHomeTimeline(int maxPosts)
public IEnumerable<Post> GetNewerHomeTimeline(int maxPosts = DefaultMaxPosts)
{
using var twitterCtx = new TwitterContext(DoSingleUserAuth());
var tweets =
(from tweet in twitterCtx.Status
where tweet.Type == StatusType.Home &&
tweet.TweetMode == TweetMode.Extended &&
tweet.Count == 10
select tweet)
.ToList();
var converter = new TwitterConverter(PersonsRepository);
var posts = tweets.Select(t => converter.TweetToPost(t)).ToList();
return posts;
var newInterval = TimelineManager.GetNextSamplingInterval(HomeTimelineName, EarliestTweetValue, LatestTweetValue);
return PullHomeTweets(newInterval, maxPosts);
}
public IEnumerable<Post> GetOlderHomeTimeline(int maxPosts = DefaultMaxPosts)
{
var newInterval = TimelineManager.GetPreviousSamplingInterval(HomeTimelineName, EarliestTweetValue, LatestTweetValue);
if (newInterval != null && newInterval.IntervalStop != ulong.MaxValue)
{
newInterval.IntervalStop -= 1;
}
return PullHomeTweets(newInterval, maxPosts);
}
public IEnumerable<Post> GetUserTimeline(Person person, int maxPosts)
public IEnumerable<Post> GetNewerUserTimeline(Person person, int maxPosts)
{
throw new System.NotImplementedException();
}
public IEnumerable<Post> GetOlderUserTimeline(Person person, int maxPosts)
{
throw new System.NotImplementedException();
}
......@@ -79,5 +92,78 @@ namespace MySocialPortalLib.Service.SocialMediaConnectors
return auth;
}
private IQueryable<Status> GetNoUpperHomeStatusQuery(TimelineInterval interval, int maxPosts)
{
using var twitterCtx = new TwitterContext(DoSingleUserAuth());
return (from tweet in twitterCtx.Status
where tweet.Type == StatusType.Home &&
tweet.TweetMode == TweetMode.Extended &&
tweet.Count == maxPosts &&
tweet.SinceID == interval.IntervalStart
select tweet);
}
private IQueryable<Status> GetUpperLowerHomeStatusQuery(TimelineInterval interval, int maxPosts)
{
using var twitterCtx = new TwitterContext(DoSingleUserAuth());
return (from tweet in twitterCtx.Status
where tweet.Type == StatusType.Home &&
tweet.TweetMode == TweetMode.Extended &&
tweet.Count == maxPosts &&
tweet.MaxID == interval.IntervalStop &&
tweet.SinceID == interval.IntervalStart
select tweet);
}
private IList<Post> PullHomeTweets(TimelineInterval interval, int maxPosts)
{
IQueryable<Status> query;
if (interval.IntervalStop == ulong.MaxValue)
{
query = GetNoUpperHomeStatusQuery(interval, maxPosts);
}
else
{
query = GetUpperLowerHomeStatusQuery(interval, maxPosts);
}
return PullTweets(interval, maxPosts, query);
}
private IList<Post> PullTweets(TimelineInterval newInterval, int maxPosts, IQueryable<Status> query)
{
var posts = new List<Post>();
if (newInterval == null)
{
return posts;
}
List<Status> tweets = query.ToList();
if (tweets.Count == 0)
{
return posts;
}
UpdateSampleInterval(tweets, newInterval, maxPosts);
var converter = new TwitterConverter(PersonsRepository);
posts.AddRange(tweets.Select(t => converter.TweetToPost(t)));
return posts;
}
private void UpdateSampleInterval(IList<Status> tweets, TimelineInterval newInterval, int maxPosts)
{
var earliestTweet = tweets.Min(t => t.StatusID);
var latestTweet = tweets.Max(t => t.StatusID);
newInterval.IntervalStop = latestTweet;
if (tweets.Count == maxPosts)
{
newInterval.IntervalStart = earliestTweet;
}
TimelineManager.UpdateRequestedInterval(newInterval);
}
}
}
\ No newline at end of file
......@@ -13,15 +13,18 @@ namespace MySocialPortalLib.Service
{
throw new MethodAccessException();
}
public string ServiceName { get; }
public TimelineManagementService(ITimelineRepository repository)
public TimelineManagementService(ITimelineRepository repository, string serviceName)
{
TimelineRepository = repository;
ServiceName = serviceName;
}
public TimelineInterval? GetNextSamplingInterval(string timelineName, ulong earliestValue, ulong forwardValue)
{
if (TimelineRepository.CountByTimelineName(timelineName) == 0)
if (TimelineRepository.CountByTimelineName(timelineName, ServiceName) == 0)
{
var interval = new TimelineInterval
{
......@@ -30,11 +33,11 @@ namespace MySocialPortalLib.Service
TimelineName = timelineName
};
TimelineRepository.AddOrUpdate(interval);
TimelineRepository.AddOrUpdate(interval, ServiceName);
return interval;
}
var intervals = new List<TimelineInterval>(TimelineRepository.FindByTimelineName(timelineName));
var intervals = new List<TimelineInterval>(TimelineRepository.FindByTimelineName(timelineName, ServiceName));
intervals.Sort((i1, i2) => -i1.IntervalStop.CompareTo(i2.IntervalStop));
var interval2 = new TimelineInterval
{
......@@ -43,14 +46,14 @@ namespace MySocialPortalLib.Service
TimelineName = timelineName
};
TimelineRepository.AddOrUpdate(interval2);
TimelineRepository.AddOrUpdate(interval2, ServiceName);
return interval2;
}
public TimelineInterval? GetPreviousSamplingInterval(string timelineName, ulong initialStart, ulong initialStop)
{
if (TimelineRepository.CountByTimelineName(timelineName) == 0)
if (TimelineRepository.CountByTimelineName(timelineName, ServiceName) == 0)
{
var interval = new TimelineInterval
{
......@@ -59,11 +62,11 @@ namespace MySocialPortalLib.Service
TimelineName = timelineName
};
TimelineRepository.AddOrUpdate(interval);
TimelineRepository.AddOrUpdate(interval, ServiceName);
return interval;
}
var intervals = new List<TimelineInterval>(TimelineRepository.FindByTimelineName(timelineName));
var intervals = new List<TimelineInterval>(TimelineRepository.FindByTimelineName(timelineName, ServiceName));
intervals.Sort((i1, i2) => i2.IntervalStop.CompareTo(i1.IntervalStop));
if (intervals.Count == 1)
......@@ -79,7 +82,7 @@ namespace MySocialPortalLib.Service
IntervalStop = intervals[0].IntervalStart,
TimelineName = timelineName
};
TimelineRepository.AddOrUpdate(newInterval);
TimelineRepository.AddOrUpdate(newInterval, ServiceName);
return newInterval;
}
......@@ -95,7 +98,7 @@ namespace MySocialPortalLib.Service
var gap = TimelineIntervalUtilities.Gap(interval1, interval2);
if (gap != null)
{
TimelineRepository.AddOrUpdate(gap);
TimelineRepository.AddOrUpdate(gap, ServiceName);
return gap;
}
}
......@@ -112,18 +115,18 @@ namespace MySocialPortalLib.Service
TimelineName = timelineName
};
TimelineRepository.AddOrUpdate(fillInterval);
TimelineRepository.AddOrUpdate(fillInterval, ServiceName);
return fillInterval;
}
public bool UpdateRequestedInterval(TimelineInterval updatedInterval, bool withCleanup = true)
{
if (TimelineRepository.FindById(updatedInterval.Id) == null)
if (TimelineRepository.FindById(updatedInterval.Id, ServiceName) == null)
{
return false;
}
bool success = TimelineRepository.AddOrUpdate(updatedInterval);
bool success = TimelineRepository.AddOrUpdate(updatedInterval, ServiceName);
if (withCleanup)
{
......@@ -137,10 +140,10 @@ namespace MySocialPortalLib.Service
private void CleanupIntervals(string timelineName)
{
var allIntervals = TimelineRepository.FindByTimelineName(timelineName);
var allIntervals = TimelineRepository.FindByTimelineName(timelineName, ServiceName);
var mergedIntervals = TimelineIntervalUtilities.CleanupList(allIntervals);
TimelineRepository.RemoveByTimelineName(timelineName);
TimelineRepository.AddOrUpdate(mergedIntervals);
TimelineRepository.RemoveByTimelineName(timelineName, ServiceName);
TimelineRepository.AddOrUpdate(mergedIntervals, ServiceName);
}
}
}
\ No newline at end of file
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using LinqToTwitter;
......@@ -24,6 +25,7 @@ namespace MySocialPortalLibTest.Converter
CompareTweet(tweet, post);
}
[Fact]
public void TestTweetToPostConverterFull()
{
var personDb = GetTempDb();
......@@ -39,6 +41,8 @@ namespace MySocialPortalLibTest.Converter
Assert.Equal(tweet.FullText, post.Body);
Assert.Equal(tweet.CreatedAt, post.PostDateTime);
Assert.Equal(tweet.Entities.MediaEntities.Count, post.Media.Count);
Assert.Equal(StandardSocialNetworkNames.Twitter, post.OriginalSocialMediaSystem);
Assert.Equal(tweet.StatusID.ToString(CultureInfo.InvariantCulture), post.OriginalLinkUrlOrId);
for (int i = 0; i < tweet.Entities.MediaEntities.Count; i++)
{
Assert.Equal(tweet.Entities.MediaEntities[i].AltText, post.Media[i].Description);
......@@ -50,7 +54,7 @@ namespace MySocialPortalLibTest.Converter
for (int i = 0; i < tweet.Entities.UrlEntities.Count; i++)
{
Assert.Equal(tweet.Entities.UrlEntities[i].ExpandedUrl, post.Links[i].Url);
Assert.Equal(StandardSocialNetworkNames.Twitter, post.Links[i].Url);
Assert.Equal(StandardSocialNetworkNames.Twitter, post.Links[i].Source);
}