Verified Commit 1b9c4137 authored by lexi's avatar lexi
Browse files

various tweaks and fixes and lineendings

parent d4d2e155
from datetime import datetime, timedelta
from discord.ext import tasks
from cogs.utils.utils import *
from cogs.utils.bot import Semicolon
import humanize
def get_emoji(candidate):
party = candidate['party_id'].upper()
emojis = CONSTANTS['emojis']
return emojis[party] if party in emojis else ''
class ElectionCog(commands.Cog, name='Election'):
def __init__(self, bot: Semicolon):
self.bot = bot
self.output = self.bot.i18n.localize('election_loading')
self.update = datetime.now()
self.updater.start()
@tasks.loop(minutes=1.0)
async def updater(self):
url = "https://static01.nyt.com/elections-assets/2020/data/api/2020-11-03/votes-remaining-page/national/president.json"
localize = self.bot.i18n.localize
async with self.bot.session.get(url, headers={"User-Agent": "semicolon-bot:v1.2"}) as r:
if r.status == 200:
rawjson = await r.json()
else:
raise Exception(localize('connection_failure').format('nytimes', r.status, await r.text()))
data = rawjson['data']
races = data['races']
full_out = []
for race in races:
if not (race['race_rating'].startswith('lean') or race['race_rating'] == 'tossup'):
continue
if race['winnerCalledTimestamp'] is not None and (datetime.utcnow() - datetime.utcfromtimestamp(int(str(race['winnerCalledTimestamp'])[:-3]))) > timedelta(hours=2):
# print((datetime.utcnow() - datetime.utcfromtimestamp(int(str(race['winnerCalledTimestamp'])[:-3]))))
continue
republican = {}
democrat = {}
leader = None
trailer = None
for candidate in race['candidates']:
if candidate['party_id'] == "republican":
republican = candidate
elif candidate['party_id'] == "democrat":
democrat = candidate
else:
continue
if leader is None:
trailer = candidate
leader = candidate
if candidate['votes'] > leader['votes']:
leader = candidate
else:
trailer = candidate
if republican and democrat:
break
lead_symbol = get_emoji(leader)
trail_symbol = get_emoji(trailer)
# remaining_votes = sum(map(lambda n: n['tot_exp_vote'], race['counties'])) - race['votes']
remaining_votes = sum(map(lambda n: max(0, n['tot_exp_vote'] - n['votes']), race['counties']))
difference = leader['votes'] - trailer['votes']
percent_diff = leader['percent'] - trailer['percent']
lead_msg = localize('election_lead_win') if leader['winner'] else localize('election_lead_tbd')
trail_msg = localize('election_trail_lose') if leader['winner'] else localize('election_trail_tbd')
output = ["__**{}**__ - {} EVs - {} - {:,} votes remaining (est.)".format(race['state_name'], race['electoral_votes'], race['reporting_display'], remaining_votes),
lead_msg.format(lead_symbol, leader['name_display'], leader['votes'], leader['percent'], race['leader_margin_display']),
trail_msg.format(trail_symbol, trailer['name_display'], trailer['votes'], trailer['percent'], trailer['pronoun'].title(), difference+1, percent_diff, (((remaining_votes + difference) / 2) / remaining_votes) if remaining_votes > 0 else 0)]
full_out += output
full_out.append(localize('election_footer'))
self.output = "\n".join(full_out)
self.update = datetime.now()
@updater.before_loop
async def before_updater(self):
await self.bot.wait_until_ready()
@commands.command()
@commands.dm_only()
async def results(self, ctx: commands.Context):
"""Command for tracking the US presidential election results of contested states"""
update = humanize.naturaldelta(datetime.now()-self.update)
for msg in line_split("\n".join([self.output, "", f"Last updated {update} ago"])):
await ctx.send(msg)
def setup(bot):
bot.add_cog(ElectionCog(bot))
from datetime import datetime, timedelta
from discord.ext import tasks
from cogs.utils.utils import *
from cogs.utils.bot import Semicolon
import humanize
def get_emoji(candidate):
party = candidate['party_id'].upper()
emojis = CONSTANTS['emojis']
return emojis[party] if party in emojis else emojis['UNKNOWN']
def is_contentious(race):
return race['race_rating'].startswith('lean') or race['race_rating'] == 'tossup'
class ElectionCog(commands.Cog, name='Election'):
def __init__(self, bot: Semicolon):
self.bot = bot
self.output = self.bot.i18n.localize('election_loading')
self.update = datetime.now()
self.updater.start()
@tasks.loop(minutes=1.0)
async def updater(self):
url = "https://static01.nyt.com/elections-assets/2020/data/api/2020-11-03/votes-remaining-page/national/president.json"
localize = self.bot.i18n.localize
async with self.bot.session.get(url, headers={"User-Agent": "semicolon-bot:v1.2"}) as r:
if r.status != 200:
raise ConnectionFailure(localize('connection_failure').format('nytimes', r.status, await r.text()))
rawjson = await r.json()
data = rawjson['data']
races = data['races']
full_out = []
for race in races:
# we only want to track contentious states (ones that lean or tossup)
if not is_contentious(race):
continue
# skips races that have been called more than X hours ago
has_winner = race['winnerCalledTimestamp'] is not None
time_limit = timedelta(hours=2) # how long to display recently called races
if has_winner and (datetime.utcnow() - datetime.utcfromtimestamp(int(str(race['winnerCalledTimestamp'])[:-3]))) > time_limit:
continue
# get leading/losing parties
candidates = sorted(race['candidates'], key=lambda x: x['votes'], reverse=True)
leader = candidates[0]
trailer = candidates[1]
lead_symbol = get_emoji(leader)
trail_symbol = get_emoji(trailer)
remaining_votes = sum(map(lambda n: max(0, n['tot_exp_vote'] - n['votes']), race['counties']))
difference = leader['votes'] - trailer['votes']
percent_diff = leader['percent'] - trailer['percent']
percent_needed = (((remaining_votes + difference) / 2) / remaining_votes) if remaining_votes > 0 else 0
lead_msg = localize('election_lead_win') if leader['winner'] else localize('election_lead_tbd')
trail_msg = localize('election_trail_lose') if leader['winner'] else localize('election_trail_tbd')
output = [localize('election_header').format(race['state_name'], race['electoral_votes'], race['reporting_display'], remaining_votes),
lead_msg.format(lead_symbol, leader['name_display'], leader['votes'], leader['percent'], race['leader_margin_display']),
trail_msg.format(trail_symbol, trailer['name_display'], trailer['votes'], trailer['percent'], trailer['pronoun'].title(), difference+1, percent_diff, percent_needed)]
full_out += output
full_out.append(localize('election_footer'))
self.output = "\n".join(full_out)
self.update = datetime.now()
@updater.before_loop
async def before_updater(self):
await self.bot.wait_until_ready()
@commands.command()
@commands.dm_only()
async def results(self, ctx: commands.Context):
"""Command for tracking the US presidential election results of contested states"""
update = humanize.naturaldelta(datetime.now()-self.update)
for msg in line_split("\n".join([self.output, "", f"Last updated {update} ago"])):
await ctx.send(msg)
def setup(bot):
bot.add_cog(ElectionCog(bot))
import logging
import re
from discord.ext import tasks
import datetime
from cogs.utils.utils import *
from cogs.utils.bot import Semicolon
import random
from numpy import math
LOCK = asyncio.Lock()
newline_split = re.compile(r'\n?<\|endoftext\|>\n')
async def gen_sample(conditional_input: str = '', complete: bool = None):
"""Hacky function to generate samples from the gpt-2 instance"""
async with LOCK:
path = open('gpt2.txt', 'r').read().split('\n')[0].strip()
# path = os.path.join(os.path.expanduser('~'), 'GitHub', 'gpt-2')
cmd = "bash htstem_gen.sh"
if conditional_input:
cmd = "bash htstem_condition.sh"
inputpath = os.path.join(path, 'input.txt')
if not complete:
conditional_input += '\n<|endoftext|>\n'
with open(inputpath, 'w') as f:
f.write(conditional_input)
process = await asyncio.create_subprocess_shell(cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=path)
stdout, stderr = await process.communicate()
if process.returncode != 0:
logging.exception("Samples did not properly generate!")
return
samples = stdout.decode().split("=" * 40 + " SAMPLE 1 " + "=" * 40+'\n')[1]
return newline_split.split(samples)
async def is_stem(ctx: commands.Context) -> bool:
"""Returns bool of if current guild is STEM or not"""
return guild_check(ctx, 282219466589208576)
class GPT2(commands.Cog):
def __init__(self, bot: Semicolon):
self.bot = bot
self.random_prompt.start()
@tasks.loop(hours=5)
async def random_prompt(self):
dtnow = datetime.datetime.utcnow().replace(microsecond=0, second=0, minute=0)
post_timer_hours = 6
wait_hour = post_timer_hours * math.ceil(dtnow.hour / post_timer_hours)
wait_until = dtnow + datetime.timedelta(hours=wait_hour-dtnow.hour)
# TODO: sometimes this gives results that immediately post, should probably look into, adding this hack for now
if wait_until < datetime.datetime.utcnow():
return
logging.info(f'Sample sleeping until {wait_until}')
await discord.utils.sleep_until(wait_until)
logging.info('Sending sample')
samples = await gen_sample()
botespam = self.bot.get_channel(282500390891683841)
async with botespam.typing():
await botespam.send(samples[0])
@random_prompt.before_loop
async def before_random_prompt(self):
await self.bot.wait_until_ready()
def cog_unload(self):
self.random_prompt.cancel()
@commands.command()
@commands.is_owner()
async def get_training_data(self, ctx: commands.Context):
"""Gets training data for a GPT-2 model :)"""
async with ctx.typing():
datapath = data_path('{}.txt'.format(ctx.guild.id))
botself = ctx.guild.me
sadreplace = re.compile(r'no(ah|elle)kiq', re.IGNORECASE)
for chan in ctx.guild.text_channels:
if chan.category_id == 360588732333817857: # avoid htstem mod channels
continue
perms = chan.permissions_for(botself)
if not (perms.read_messages and perms.read_message_history):
continue
async for msg in chan.history(limit=None, oldest_first=True):
if random.random() > 0.2:
continue
msgcontent = msg.clean_content # or use .system_content if you want <@id>'s and @e's
if not msgcontent: # if msg content is blank
continue
with open(datapath, 'a') as f:
f.write('{}\n<|endoftext|>\n'.format(sadreplace.sub("lexikiq", msgcontent)))
await ctx.send('done lol')
@commands.group(aliases=['gpt2'])
@commands.check(is_stem)
async def prompt(self, ctx: commands.Context):
"""Communicate with a GPT-2 117M instance trained on HTwins STEM.
Specify "enable" as the first argument to have the AI finish your message instead of responding to it.
GPT-2 117M is trained on various text around the internet and may provide random results.
It was tuned overnight using a fifth of the messages in HTwins STEM (~95k) public channels.
(Only a fifth was used to avoid out of memory errors. I need more RAM...)
It went through 76k training steps and reached a loss of ~0.08, where it was stopped to avoid overfitting."""
if ctx.invoked_subcommand is None:
subcmds = [x.name for x in ctx.command.commands]
await ctx.send(embed=error_gen(ctx, 'Please use one of the subcommands: `{}`'.format('`, `'.join(subcmds))))
@prompt.command(usage='[optional: messages to generate] <input sentence(s)>')
@commands.check(is_stem)
async def respond(self, ctx: commands.Context, messages: typing.Optional[int] = 1, *, input_text: str):
"""Responds to your message"""
if LOCK.locked():
await ctx.send(embed=error_gen(ctx, 'Please wait for existing commands to finish.'), delete_after=7)
return
async with ctx.typing():
samples = await gen_sample(input_text)
messages = clamp(messages, 1, min(5, len(samples)))
if not samples:
await ctx.send(embed=error_gen(ctx, 'Sample output failed, please try again.'))
return
if messages == 1:
for msg in line_split('{}: {}'.format(ctx.author.mention, samples[0])):
await ctx.send(msg, allowed_mentions=ping_user(ctx))
return
for msg_c in range(0, messages):
for msg in line_split('**[{}]** {}'.format(msg_c+1, samples[msg_c])):
await ctx.send(msg)
@prompt.command(usage='<input sentence(s)>')
@commands.check(is_stem)
async def complete(self, ctx: commands.Context, *, input_text: str):
"""'Auto-completes' your input text. May not complete anything."""
if LOCK.locked():
await ctx.send(embed=error_gen(ctx, 'Please wait for existing commands to finish.'), delete_after=7)
return
async with ctx.typing():
samples = await gen_sample(input_text, True)
if not samples:
await ctx.send(embed=error_gen(ctx, 'Sample output failed, please try again.'))
return
for msg in line_split('{}: {}{}'.format(ctx.author.mention, input_text, samples[0])):
await ctx.send(msg, allowed_mentions=ping_user(ctx))
@prompt.command()
@commands.check(is_stem)
async def channel(self, ctx: commands.Context, channel: discord.TextChannel, messages: typing.Optional[int] = 1):
"""Responds to the last 10 messages of a specified channel.
[messages] refers to the number of messages that should be generated in response."""
if LOCK.locked():
await ctx.send(embed=error_gen(ctx, 'Please wait for existing commands to finish.'), delete_after=7)
return
async with ctx.typing():
g_perms = ctx.guild.default_role.permissions.read_messages
c_perms = channel.overwrites_for(ctx.guild.default_role).read_messages
if channel.category_id == 360588732333817857 or not (g_perms and (c_perms or c_perms is None)):
await ctx.send(embed=error_gen(ctx, ctx.bot.i18n.localize('error', ctx=ctx)+'Nice try, pal.'))
return
botself = ctx.guild.me
perms = channel.permissions_for(botself)
if not (perms.read_messages and perms.read_message_history):
await ctx.send(embed=error_gen(ctx, ctx.bot.i18n.localize('error', ctx=ctx)+"I don't have permission to read that channel!"))
return
msgs = []
async for msg in channel.history(limit=10, oldest_first=True):
msgs.append(msg.clean_content)
samples = await gen_sample('\n<|endoftext|>\n'.join(msgs))
if not samples:
await ctx.send(embed=error_gen(ctx, 'Sample output failed, please try again.'))
return
messages = clamp(messages, 1, min(5, len(samples)))
if messages == 1:
for msg in line_split('{}: {}'.format(ctx.author.mention, samples[0])):
await ctx.send(msg, allowed_mentions=ping_user(ctx))
return
for msg_c in range(0, messages):
for msg in line_split('**[{}]** {}'.format(msg_c + 1, samples[msg_c])):
await ctx.send(msg)
def setup(bot):
bot.add_cog(GPT2(bot))
import logging
import re
from discord.ext import tasks
import datetime
from cogs.utils.utils import *
from cogs.utils.bot import Semicolon
import random
from numpy import math
LOCK = asyncio.Lock()
newline_split = re.compile(r'\n?<\|endoftext\|>\n')
async def gen_sample(conditional_input: str = '', complete: bool = None):
"""Hacky function to generate samples from the gpt-2 instance"""
async with LOCK:
path = open('gpt2.txt', 'r').read().split('\n')[0].strip()
# path = os.path.join(os.path.expanduser('~'), 'GitHub', 'gpt-2')
cmd = "bash htstem_gen.sh"
if conditional_input:
cmd = "bash htstem_condition.sh"
inputpath = os.path.join(path, 'input.txt')
if not complete:
conditional_input += '\n<|endoftext|>\n'
with open(inputpath, 'w') as f:
f.write(conditional_input)
process = await asyncio.create_subprocess_shell(cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=path)
stdout, stderr = await process.communicate()
if process.returncode != 0:
logging.exception("Samples did not properly generate!")
return
samples = stdout.decode().split("=" * 40 + " SAMPLE 1 " + "=" * 40+'\n')[1]
return newline_split.split(samples)
async def is_stem(ctx: commands.Context) -> bool:
"""Returns bool of if current guild is STEM or not"""
return guild_check(ctx, 282219466589208576)
class GPT2(commands.Cog):
def __init__(self, bot: Semicolon):
self.bot = bot
self.random_prompt.start()
@tasks.loop(hours=5)
async def random_prompt(self):
dtnow = datetime.datetime.utcnow().replace(microsecond=0, second=0, minute=0)
post_timer_hours = 6
wait_hour = post_timer_hours * math.ceil(dtnow.hour / post_timer_hours)
wait_until = dtnow + datetime.timedelta(hours=wait_hour-dtnow.hour)
# TODO: sometimes this gives results that immediately post, should probably look into, adding this hack for now
if wait_until < datetime.datetime.utcnow():
return
logging.info(f'Sample sleeping until {wait_until}')
await discord.utils.sleep_until(wait_until)
logging.info('Sending sample')
samples = await gen_sample()
botespam = self.bot.get_channel(282500390891683841)
async with botespam.typing():
await botespam.send(samples[0])
@random_prompt.before_loop
async def before_random_prompt(self):
await self.bot.wait_until_ready()
def cog_unload(self):
self.random_prompt.cancel()
@commands.command()
@commands.is_owner()
async def get_training_data(self, ctx: commands.Context):
"""Gets training data for a GPT-2 model :)"""
async with ctx.typing():
datapath = data_path('{}.txt'.format(ctx.guild.id))
botself = ctx.guild.me
sadreplace = re.compile(r'no(ah|elle)kiq', re.IGNORECASE)
for chan in ctx.guild.text_channels:
if chan.category_id == 360588732333817857: # avoid htstem mod channels
continue
perms = chan.permissions_for(botself)
if not (perms.read_messages and perms.read_message_history):
continue
async for msg in chan.history(limit=None, oldest_first=True):
if random.random() > 0.2:
continue
msgcontent = msg.clean_content # or use .system_content if you want <@id>'s and @e's
if not msgcontent: # if msg content is blank
continue
with open(datapath, 'a') as f:
f.write('{}\n<|endoftext|>\n'.format(sadreplace.sub("lexikiq", msgcontent)))
await ctx.send('done lol')
@commands.group(aliases=['gpt2'])
@commands.check(is_stem)
async def prompt(self, ctx: commands.Context):
"""Communicate with a GPT-2 117M instance trained on HTwins STEM.
Specify "enable" as the first argument to have the AI finish your message instead of responding to it.
GPT-2 117M is trained on various text around the internet and may provide random results.
It was tuned overnight using a fifth of the messages in HTwins STEM (~95k) public channels.
(Only a fifth was used to avoid out of memory errors. I need more RAM...)
It went through 76k training steps and reached a loss of ~0.08, where it was stopped to avoid overfitting."""
if ctx.invoked_subcommand is None:
subcmds = [x.name for x in ctx.command.commands]
await ctx.send(embed=error_gen(ctx, 'Please use one of the subcommands: `{}`'.format('`, `'.join(subcmds))))
@prompt.command(usage='[optional: messages to generate] <input sentence(s)>')
@commands.check(is_stem)
async def respond(self, ctx: commands.Context, messages: typing.Optional[int] = 1, *, input_text: str):
"""Responds to your message"""
if LOCK.locked():
await ctx.send(embed=error_gen(ctx, 'Please wait for existing commands to finish.'), delete_after=7)
return
async with ctx.typing():
samples = await gen_sample(input_text)
messages = clamp(messages, 1, min(5, len(samples)))
if not samples:
await ctx.send(embed=error_gen(ctx, 'Sample output failed, please try again.'))
return
if messages == 1:
for msg in line_split('{}: {}'.format(ctx.author.mention, samples[0])):
await ctx.send(msg, allowed_mentions=ping_user(ctx))
return
for msg_c in range(0, messages):
for msg in line_split('**[{}]** {}'.format(msg_c+1, samples[msg_c])):
await ctx.send(msg)
@prompt.command(usage='<input sentence(s)>')
@commands.check(is_stem)
async def complete(self, ctx: commands.Context, *, input_text: str):
"""'Auto-completes' your input text. May not complete anything."""
if LOCK.locked():
await ctx.send(embed=error_gen(ctx, 'Please wait for existing commands to finish.'), delete_after=7)
return
async with ctx.typing():
samples = await gen_sample(input_text, True)
if not samples:
await ctx.send(embed=error_gen(ctx, 'Sample output failed, please try again.'))
return
for msg in line_split('{}: {}{}'.format(ctx.author.mention, input_text, samples[0])):
await ctx.send(msg, allowed_mentions=ping_user(ctx))
@prompt.command()
@commands.check(is_stem)
async def channel(self, ctx: commands.Context, channel: discord.TextChannel, messages: typing.Optional[int] = 1):
"""Responds to the last 10 messages of a specified channel.
[messages] refers to the number of messages that should be generated in response."""
if LOCK.locked():
await ctx.send(embed=error_gen(ctx, 'Please wait for existing commands to finish.'), delete_after=7)
return
async with ctx.typing():
g_perms = ctx.guild.default_role.permissions.read_messages
c_perms = channel.overwrites_for(ctx.guild.default_role).read_messages
if channel.category_id == 360588732333817857 or not (g_perms and (c_perms or c_perms is None)):
await ctx.send(embed=error_gen(ctx, ctx.bot.i18n.localize('error', ctx=ctx)+'Nice try, pal.'))
return
botself = ctx.guild.me
perms = channel.permissions_for(botself)
if not (perms.read_messages and perms.read_message_history):
await ctx.send(embed=error_gen(ctx, ctx.bot.i18n.localize('error', ctx=ctx)+"I don't have permission to read that channel!"))
return
msgs = []
async for msg in channel.history(limit=10, oldest_first=True):
msgs.append(msg.clean_content)
samples = await gen_sample('\n<|endoftext|>\n'.join(msgs))
if not samples:
await ctx.send(embed=error_gen(ctx, 'Sample output failed, please try again.'))
return
messages = clamp(messages, 1, min(5, len(samples)))
if messages == 1:
for msg in line_split('{}: {}'.format(ctx.author.mention, samples[0])):
await ctx.send(msg, allowed_mentions=ping_user(ctx))
return
for msg_c in range(0, messages):
for msg in line_split('**[{}]** {}'.format(msg_c + 1, samples[msg_c])):
await ctx.send(msg)
def setup(bot):
bot.add_cog(GPT2(bot))
import asyncpg
from discord.ext import tasks
import datetime
from cogs.utils.bot import Semicolon
from cogs.utils.utils import *
import traceback
from colour import Color
default_loop_interval = 2.0
time_output = "%m/%d/%Y %H:%M:%S"
def time_format(seconds, localizer: Localizer, raw_output=False):
<