Commit 0a04831a authored by Robert Rudman's avatar Robert Rudman

Organized the code a bit more. Implement backoff, make sure calls are not...

Organized the code a bit more. Implement backoff, make sure calls are not parallel, and added quota checking.
parent d8e069c6
using System;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Reactive.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StackExchangeChat.Authenticators;
using StackExchangeChat.Console.AppSettings;
using StackExchangeChat.Sites;
using StackExchangeChat.Utilities;
namespace StackExchangeChat.Console
{
......@@ -27,33 +27,37 @@ namespace StackExchangeChat.Console
var credentials = new Credentials();
config.Bind("ChatCredentials", credentials);
serviceCollection.AddTransient(_ =>
{
var handler = new HttpClientHandler();
var httpClient = new HttpClient(handler);
return new HttpClientWithHandler
{
HttpClient = httpClient,
HttpClientHandler = handler
};
});
serviceCollection.AddScoped<SiteAuthenticator>();
serviceCollection.AddScoped<ChatClient>();
serviceCollection.AddTransient<HttpClient>();
serviceCollection.AddTransient(_ => new HttpClient(new HttpClientHandler { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate }));
serviceCollection.AddTransient<HttpClientWithHandler>();
serviceCollection.AddSingleton(_ => config);
serviceCollection.AddSingleton(_ => credentials);
var serviceProvider = serviceCollection.BuildServiceProvider();
var chatClient = new ChatClient(serviceProvider);
chatClient.SubscribeToEvents(Site.StackOverflow, 167908)
.Where(c => c.EventType == EventType.MessagePosted || c.EventType == EventType.MessageEdited)
.Subscribe(async chatEvent =>
{
await chatClient.SendMessage(Site.StackOverflow, 167908, $":{chatEvent.MessageId} Replying to message..");
}, exception =>
{
System.Console.WriteLine(exception);
});
var apiThing = new StackExchangeApiHelper(serviceProvider);
apiThing.TotalQuestionsByTag("design").GetAwaiter().GetResult();
apiThing.TotalQuestionsByTag("design").GetAwaiter().GetResult();
apiThing.TotalQuestionsByTag("design").GetAwaiter().GetResult();
StackExchangeApiHelper.QuotaRemaining.Subscribe(System.Console.WriteLine);
//// var result = apiThing.TotalQuestionsByTag("design").GetAwaiter().GetResult();
//var chatClient = serviceProvider.GetService<ChatClient>();
//chatClient.SubscribeToEvents(Site.StackOverflow, 167908)
// .Where(c => c.EventType == EventType.MessagePosted || c.EventType == EventType.MessageEdited)
// .Subscribe(async chatEvent =>
// {
// await chatClient.SendMessage(Site.StackOverflow, 167908, $":{chatEvent.MessageId} Replying to message..");
// }, exception =>
// {
// System.Console.WriteLine(exception);
// });
System.Console.ReadKey();
}
......
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using HtmlAgilityPack;
using StackExchangeChat.Console.AppSettings;
using StackExchangeChat.Sites;
using StackExchangeChat.Utilities;
namespace StackExchangeChat.Authenticators
{
public class SiteAuthenticator
{
private readonly HttpClientWithHandler _authenticatingHttpClient;
private readonly HttpClientWithHandler _fkeyHttpClient;
private readonly Credentials _credentials;
private struct SiteRoomIdPair
{
public Site Site;
public int RoomId;
}
private DateTime? _cookieExpires;
private Task<Cookie> _authenticateTask;
private readonly object _locker = new object();
private readonly Dictionary<SiteRoomIdPair, Task<string>> m_CachedFKeys = new Dictionary<SiteRoomIdPair, Task<string>>();
public SiteAuthenticator(
HttpClientWithHandler authenticatingHttpClient,
HttpClientWithHandler fkeyHttpClient,
Credentials credentials)
{
if (ReferenceEquals(authenticatingHttpClient, fkeyHttpClient))
throw new ArgumentException("Must provide two distinct instance of HttpClient for the Authenticating client and the fkey client.");
_authenticatingHttpClient = authenticatingHttpClient;
_fkeyHttpClient = fkeyHttpClient;
_credentials = credentials;
}
public async Task AuthenticateClient(HttpClientWithHandler httpClient, Site site)
{
var acctCookie = await GetAccountCookie(site);
var cookieContainer = new CookieContainer();
httpClient.Handler.CookieContainer = cookieContainer;
cookieContainer.Add(acctCookie);
}
public async Task<string> GetFKeyForRoom(Site site, int roomId)
{
Task<string> task;
lock (_locker)
{
var pair = new SiteRoomIdPair {Site = site, RoomId = roomId};
if (!m_CachedFKeys.ContainsKey(pair))
m_CachedFKeys[pair] = GetFKeyForRoomInternal();
task = m_CachedFKeys[pair];
}
return await task;
async Task<string> GetFKeyForRoomInternal()
{
await AuthenticateClient(_fkeyHttpClient, site);
var result = await _fkeyHttpClient.GetAsync($"https://{site.ChatDomain}/rooms/{roomId}");
var resultStr = await result.Content.ReadAsStringAsync();
var doc = new HtmlDocument();
doc.LoadHtml(resultStr);
var fkeyElement = doc.DocumentNode.SelectSingleNode("//input[@id = 'fkey']");
var fkey = fkeyElement.Attributes["value"].Value;
return fkey;
}
}
private async Task<Cookie> GetAccountCookie(Site site)
{
lock (_locker)
{
if (_cookieExpires.HasValue && _cookieExpires.Value < DateTime.UtcNow)
_authenticateTask = null;
if (_authenticateTask == null)
_authenticateTask = GetAccountCookieInternal();
}
var cookie = await _authenticateTask;
_cookieExpires = cookie.Expires;
return cookie;
async Task<Cookie> GetAccountCookieInternal()
{
if (!string.IsNullOrWhiteSpace(_credentials.AcctCookie))
{
var expiry = DateTime.ParseExact(_credentials.AcctCookieExpiry, "yyyy-MM-dd HH:mm:ssZ", CultureInfo.InvariantCulture);
return new Cookie("acct", _credentials.AcctCookie, "/", site.LoginDomain)
{
Expires = expiry
};
}
var cookieContainer = new CookieContainer();
_authenticatingHttpClient.Handler.CookieContainer = cookieContainer;
var result = await _authenticatingHttpClient.GetAsync($"https://{site.LoginDomain}/users/login");
var content = await result.Content.ReadAsStringAsync();
var doc = new HtmlDocument();
doc.LoadHtml(content);
var fkeyElement = doc.DocumentNode.SelectSingleNode("//form[@id = 'login-form']/input[@name = 'fkey']");
var fkey = fkeyElement.Attributes["value"].Value;
var loginPayload = new FormUrlEncodedContent(new Dictionary<string, string>
{
{"fkey", fkey},
{"email", _credentials.Email},
{"password", _credentials.Password},
});
await _authenticatingHttpClient.PostAsync($"https://{site.LoginDomain}/users/login", loginPayload);
var cookies = cookieContainer.GetCookies(new Uri($"https://{site.LoginDomain}")).Cast<Cookie>().ToList();
var acctCookie = cookies.FirstOrDefault(c => c.Name == "acct");
if (acctCookie == null || acctCookie.Expired)
throw new Exception();
return acctCookie;
}
}
}
}
This diff is collapsed.
using System;
using System.Net.Http;
namespace StackExchangeChat
{
public class HttpClientWithHandler : IDisposable
{
public HttpClient HttpClient { get; set; }
public HttpClientHandler HttpClientHandler { get; set; }
public void Dispose()
{
HttpClient?.Dispose();
HttpClientHandler?.Dispose();
}
}
}
using System.Net.Http;
namespace StackExchangeChat.Utilities
{
public class HttpClientWithHandler : HttpClient
{
public HttpClientHandler Handler { get; }
public HttpClientWithHandler(HttpClientHandler handler) : base(handler)
{
Handler = handler;
}
}
}
using Newtonsoft.Json;
namespace StackExchangeChat.Utilities.Responses
{
public class ApiBaseResponse
{
public int? Backoff { get; set; }
[JsonProperty("quota_remaining")]
public int QuotaRemaining { get; set; }
[JsonProperty("error_id")]
public int ErrorId { get; set; }
[JsonProperty("error_message")]
public int ErrorMessage { get; set; }
[JsonProperty("error_name")]
public int ErrorName { get; set; }
}
}
namespace StackExchangeChat.Utilities.Responses
{
public class TotalResponse : ApiBaseResponse
{
public int Total { get; set; }
}
}
using System;
using System.Net.Http;
using System.Reactive.Disposables;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using Microsoft.Extensions.DependencyInjection;
using Newtonsoft.Json;
using StackExchangeChat.Utilities.Responses;
namespace StackExchangeChat.Utilities
{
public class StackExchangeApiHelper
{
private readonly IServiceProvider _serviceProvider;
public static IObservable<int> QuotaRemaining;
public static int CurrentQuotaRemaining = int.MaxValue;
public static object TaskLocker = new object();
public static Task ExecutingTask = Task.CompletedTask;
private static Action<int> m_UpdateQuota;
static StackExchangeApiHelper()
{
var replaySubject = new ReplaySubject<int>(1);
Observable.Create<int>(o =>
{
m_UpdateQuota = o.OnNext;
return Disposable.Empty;
}).Subscribe(replaySubject);
QuotaRemaining = replaySubject;
QuotaRemaining.Subscribe(remaining => { Interlocked.Exchange(ref CurrentQuotaRemaining, remaining); });
}
public StackExchangeApiHelper(IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
}
public async Task<TResponseType> MakeRequest<TResponseType>(string endpoint) where TResponseType: ApiBaseResponse
{
Task<TResponseType> nextTask;
lock (TaskLocker)
{
nextTask = MakeRequestInternal(ExecutingTask);
ExecutingTask = PostProcess(nextTask);
}
var result = await nextTask;
return result;
async Task PostProcess(Task<TResponseType> executingTask)
{
var returnedItem = await executingTask;
m_UpdateQuota(returnedItem.QuotaRemaining);
if (returnedItem.Backoff.HasValue)
await Task.Delay(TimeSpan.FromSeconds(returnedItem.Backoff.Value));
}
async Task<TResponseType> MakeRequestInternal(Task previousTask)
{
await previousTask;
if (CurrentQuotaRemaining <= 0)
throw new Exception("No more quota!");
using (var httpClient = _serviceProvider.GetService<HttpClient>())
{
var response = await httpClient.GetAsync(endpoint);
var content = await response.Content.ReadAsStringAsync();
var payload = JsonConvert.DeserializeObject<TResponseType>(content);
return payload;
}
}
}
public Task<TotalResponse> TotalQuestionsByTag(string tag)
{
var encodedTag = HttpUtility.HtmlEncode(tag);
var query = $"https://api.stackexchange.com/2.2/questions?order=desc&sort=activity&tagged={encodedTag}&site=stackoverflow&filter=!--s3oyShP3gx";
return MakeRequest<TotalResponse>(query);
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment