Implement OCR action. (20009cc9) · Commits · FUC / cogBot

cog/actions.py

+70 −20

Original line number	Diff line number	Diff line
		@@ -5,6 +5,7 @@ All actions have async execute methods.
		"""
		import asyncio
		import concurrent
		import concurrent.futures as cfut
		import datetime
		import functools
		import logging
		@@ -1178,18 +1179,28 @@ class Near(Action):
		await self.bot.send_message(self.msg.channel, msg)


		class OCR(Action):
		"""
		Management interface for OCR operations.
		"""
		async def execute(self):
		reply = None

		if self.args.subcmd == "preps":
		reply = cogdb.query.ocr_prep_report(self.session)
		elif self.args.subcmd == "refresh": # pragma: no cover
		await monitor_ocr_sheet(self.bot, delay_minutes=0, repeat=False)

		if reply:
		await self.bot.send_message(self.msg.channel, reply)


		class Pin(Action):
		"""
		Create an objetives pin.
		"""
		# TODO: Incomplete, expect bot to manage pin entirely. Left undocumented.
		async def execute(self):
		# TODO: Remove on shipping feature
		# ocr_scanner = get_scanner('hudson_ocr')
		# await ocr_scanner.update_cells()
		# with cogdb.session_scope(cogdb.Session) as session:
		# ocr_scanner.parse_sheet(session)

		systems = cogdb.query.fort_get_targets(self.session)
		systems.reverse()
		systems += cogdb.query.fort_get_next_targets(self.session, count=5)
		@@ -1837,11 +1848,8 @@ async def monitor_carrier_events(client, *, next_summary, last_timestamp=None, d
		msgs = cog.util.generative_split(tracks, str, header=header)

		# Only send messages if generated and channel set
		chan_id = cog.util.get_config("carrier_channel", default=None)
		if msgs and chan_id and msgs != [header]:
		chan = client.get_channel(chan_id)
		for msg in msgs:
		await client.send_message(chan, msg)
		if msgs != [header]:
		await report_to_leadership(client, msgs)

		asyncio.ensure_future(
		monitor_carrier_events(
		@@ -1851,28 +1859,70 @@ async def monitor_carrier_events(client, *, next_summary, last_timestamp=None, d
		)


		async def monitor_ocr_sheet(client, *, last_timestamp=None, delay=30):
		async def monitor_ocr_sheet(client, *, last_timestamp=None, delay_minutes=30, repeat=True):
		"""
		Simple async task that just checks for changes to the OCR sheet.
		This task will schedule itself infinitely on a delay.

		Args:
		client: The bot client itself.

		Kwargs:
		last_timestamp: The last timestamp the bot woke up at.
		delay_minutes: The minutes between polling sheet for new change.
		repeat: If true, will schedule itself infinitely.
		"""
		start = datetime.datetime.utcnow()
		if not last_timestamp:
		last_timestamp = start

		timedelta_to_wait = (last_timestamp + datetime.timedelta(minutes=delay)) - start
		timedelta_to_wait = (last_timestamp + datetime.timedelta(minutes=delay_minutes)) - start
		if timedelta_to_wait.seconds > 0:
		await asyncio.sleep(delay)
		await asyncio.sleep(timedelta_to_wait.seconds)

		# Update database by triggering manual refresh
		ocr_scanner = get_scanner('hudson_ocr')
		await ocr_scanner.update_cells()
		with cfut.ProcessPoolExecutor(max_workers=1) as pool:
		await client.loop.run_in_executor(
		pool, ocr_scanner.scheduler_run,
		)

		# Data refreshed, analyse and update
		with cogdb.session_scope(cogdb.Session) as session:
		updates = cogdb.query.ocr_update_fort_status(session)
		if updates:
		await get_scanner('hudson_cattle').send_batch(updates)

		prep_report = cogdb.query.ocr_prep_report(session)

		# TODO: Update database first.
		# TODO: Analyse information here.
		# Only send messages if generated and channel set
		await report_to_leadership(client, [prep_report])

		# A onetime flag to trigger for testing
		if repeat:
		asyncio.ensure_future(
		monitor_carrier_events(
		client, last_timestamp=last_timestamp, delay=delay
		monitor_ocr_sheet(
		client, last_timestamp=last_timestamp, delay_minutes=delay_minutes
		)
		)


		async def report_to_leadership(client, msgs):
		"""
		Send messages to the channel configured to receive reports.

		Args:
		client: The bot client.
		msgs: A list of messages, each should be under discord char limit.
		"""
		chan_id = cog.util.get_config("carrier_channel", default=None)
		if msgs and chan_id:
		chan = client.get_channel(chan_id)
		for msg in msgs:
		await client.send_message(chan, msg)


		SCANNERS = {}
		SCOUT_RND = { # TODO: Extract to data config or tables.
		1: [

cog/bot.py

+1 −0

Original line number	Diff line number	Diff line
		@@ -199,6 +199,7 @@ class CogBot(discord.Client):
		presence_task(self),
		simple_heartbeat(),
		cog.actions.monitor_carrier_events(self, next_summary=next_summary, delay=60),
		cog.actions.monitor_ocr_sheet(self),
		cogdb.monitor_pools(),
		))

cog/parse.py

+12 −0

Original line number	Diff line number	Diff line
		@@ -391,6 +391,18 @@ def subs_near(subs, prefix):
		subcmd.add_argument('-m', '--medium', action='store_true', default=False, help='The include mpad only stations.')


		@register_parser
		def subs_ocr(subs, prefix):
		""" Subcommand parsing for ocr """
		sub = subs.add_parser(prefix + 'ocr', description='Manage the OCR system and query things.')
		sub.set_defaults(cmd='OCR')
		subcmds = sub.add_subparsers(title='subcommands',
		description='OCR Subcommands', dest='subcmd')

		subcmds.add_parser('preps', help='Show all preps.')
		subcmds.add_parser('refresh', help='Immediately refresh from OCR sheet.')


		@register_parser
		def subs_pin(subs, prefix):
		""" Subcommand parsing for pin """

cogdb/query.py

+44 −0

Original line number	Diff line number	Diff line
		@@ -18,6 +18,7 @@ from cogdb.schema import (DiscordUser, FortSystem, FortPrep, FortDrop, FortUser,
		TrackSystem, TrackSystemCached, TrackByID, OCRTracker, OCRTrigger,
		OCRPrep, Global)
		from cogdb.eddb import HUDSON_CONTROLS, WINTERS_CONTROLS
		from cogdb.scanners import FortScanner

		DEFER_MISSING = get_config("limits", "defer_missing", default=750)
		MAX_DROP = get_config("limits", "max_drop", default=1000)
		@@ -1130,3 +1131,46 @@ def get_current_global(session):
		session.flush()

		return globe


		def ocr_update_fort_status(session):
		"""
		Iterate every fort in the system and update fort_status, um_status and triggers if needed.
		For any system that is updated generate an update_system_dict to be sent in batch.

		Returns: A list of FortScanner.update_system_dicts that will update the sheet with OCR changes.
		"""
		cell_updates = []

		for sys in session.query(FortSystem):
		if not sys.ocr_tracker:
		continue

		changed = False
		if sys.ocr_tracker.fort > sys.fort_status:
		sys.fort_status = sys.ocr_tracker.fort
		changed = True
		if sys.ocr_tracker.um > sys.um_status:
		sys.um_status = sys.ocr_tracker.um
		changed = True
		if changed:
		cell_updates += FortScanner.update_system_dict(sys.sheet_col, sys.fort_status, sys.um_status)

		return cell_updates


		def ocr_prep_report(session):
		"""
		Generate a small report on the preps currently tracked.

		Returns: Report on current consolidation and prep merits. (String)
		"""
		globe = get_current_global(session)
		msg = """__Hudson Preps Report__

		Current Consolidation: {}%
		""".format(globe.consolidation)
		for prep in session.query(OCRPrep).order_by(OCRPrep.merits.desc()).all():
		msg += "\n" + str(prep)

		return msg

cogdb/scanners.py

+50 −15

Original line number	Diff line number	Diff line
		@@ -751,6 +751,10 @@ class CarrierScanner(FortScanner):
		return found


		# TODO: Correct possible corruption of system names from OCR
		# Use some string similarity algorithm vs known controls.
		# Examples: 16 CYGN = 16 CYGNI
		# At present bot will ignore updates where system name is corrupted.
		class OCRScanner(FortScanner):
		"""
		Scanner for the Hudson OCR sheet.
		@@ -771,14 +775,16 @@ class OCRScanner(FortScanner):
		return super().__repr__().replace('FortScanner', 'OCRScanner')

		def parse_sheet(self, session=None):
		sys_map = self.generate_system_map()

		# Update consolidation vote
		globe = cogdb.query.get_current_global(session)
		globe.consolidation = int(self.cells_row_major[self.prep_consolidation_row][self.prep_col])

		# Date in format: 2021-08-22 20:33:07
		sheet_date = datetime.datetime.strptime(self.cells_row_major[0][2], "%Y-%m-%d %H:%M:%S")
		ocr_trackers = self.ocr_trackers(sheet_date)
		ocr_preps = self.ocr_preps(sheet_date)
		ocr_trackers = self.ocr_trackers(sheet_date, sys_map)
		ocr_preps = self.ocr_preps(sheet_date, sys_map)

		# flush to db
		cogdb.query.update_ocr_live(session, ocr_trackers)
		@@ -787,10 +793,38 @@ class OCRScanner(FortScanner):
		# TODO: Enable weekly limit when fully tested.
		# oldest_trigger = cogdb.query.get_oldest_ocr_trigger(session)
		# if self.should_update_trigger(oldest_trigger, sheet_date):
		ocr_triggers = self.ocr_triggers(sheet_date)
		ocr_triggers = self.ocr_triggers(sheet_date, sys_map)
		cogdb.query.update_ocr_trigger(session, ocr_triggers)
		session.commit()

		def generate_system_map(self):
		"""
		Use this map to correct systm names and ensure system name is not corrupted in ocr sheet.
		Looks up candidates against the EDDB database, maps the CAPS -> Normal system names.
		Any system names corrupted won't be in the map and will generate errors when looked up.

		Dictionary Format:
		{
		"16 CYGNI": "16 Cygni",
		"ADEO": "Adeo",
		...
		}

		Returns: A dictionary mapping system names from ALL CAPS to normal eddb name.
		"""
		systems_in_sheets = [x.upper() for x in cogdb.eddb.HUDSON_CONTROLS] + \
		[x for x in self.cells_col_major[self.prep_col][2:7] if x]

		# Generate a map for system name correction
		with cogdb.session_scope(cogdb.EDDBSession) as eddb_session:
		eddb_systems = eddb_session.query(cogdb.eddb.System).\
		filter(cogdb.eddb.System.name.in_(systems_in_sheets)).\
		all()
		mapping_eddb = {x.name.lower(): x.name for x in eddb_systems}

		return {x: mapping_eddb[x.lower()] for x in systems_in_sheets
		if x.lower() in mapping_eddb}

		def should_update_trigger(self, oldest_trigger, sheet_date):
		"""
		Triggers are updated exactly ONCE post tick.
		@@ -811,7 +845,7 @@ class OCRScanner(FortScanner):

		return trigger_stale or just_past_tick

		def ocr_trackers(self, sheet_date):
		def ocr_trackers(self, sheet_date, sys_map):
		"""
		Parse and return all OCR Tracking information for current forts.

		@@ -823,39 +857,40 @@ class OCRScanner(FortScanner):
		trackers = {}
		for row in self.cells_row_major[self.start_row:]:
		try:
		system = row[0].lower().capitalize()
		system = sys_map[row[0]]
		trackers[system] = {
		'system': system,
		'fort': int(row[1]),
		'um': int(row[2]),
		'updated_at': sheet_date,
		}
		except ValueError:
		logging.getLogger(__name__).error("Failed to parse row: %s", str(row))
		except (KeyError, ValueError):
		logging.getLogger(__name__).info("Failed to parse row: %s", str(row))

		return trackers

		def ocr_preps(self, sheet_date):
		def ocr_preps(self, sheet_date, sys_map):
		"""
		Parse and return all OCR Preps listed.
		"""
		preps = {}
		for row in self.cells_row_major[self.start_row:]:
		try:
		system = row[self.prep_col].lower().capitalize()
		system = row[self.prep_col]
		if not system:
		break
		system = sys_map[system]
		preps[system] = {
		'system': system,
		'merits': int(row[self.prep_col + 1]),
		'updated_at': sheet_date,
		}
		except ValueError:
		logging.getLogger(__name__).error("Failed to parse row: %s", str(row))
		except (KeyError, ValueError):
		logging.getLogger(__name__).info("Failed to parse row: %s", str(row))

		return preps

		def ocr_triggers(self, sheet_date):
		def ocr_triggers(self, sheet_date, sys_map):
		"""
		Parse and return all OCR Triggers listed for the cycle.

		@@ -867,7 +902,7 @@ class OCRScanner(FortScanner):
		triggers = {}
		for row in self.cells_row_major[self.start_row:]:
		try:
		system = row[self.trigger_col].lower().capitalize()
		system = sys_map[row[self.trigger_col]]
		triggers[system] = {
		'system': system,
		'last_upkeep': int(row[self.trigger_col + 1]),
		@@ -876,8 +911,8 @@ class OCRScanner(FortScanner):
		'um_trigger': int(row[self.trigger_col + 4]),
		'updated_at': sheet_date,
		}
		except ValueError:
		logging.getLogger(__name__).error("Failed to parse row: %s", str(row))
		except (KeyError, ValueError):
		logging.getLogger(__name__).info("Failed to parse row: %s", str(row))

		return triggers