Determine held SpySystems to update based on held_updated_at. (088c0884) · Commits · FUC / cogBot

cog/actions.py

+6 −12

Original line number	Diff line number	Diff line
		@@ -1718,15 +1718,12 @@ class Scrape(Action):
		Returns: A message to return to invoker.
		"""
		power = cogdb.eddb.get_power_by_name(eddb_session, self.args.name)
		await self.msg.channel.send(f"Will scrape all controls for: {power.text}. Ok? Y/N")
		await self.msg.channel.send(f"Will scrape all controls for {power.text} with held merits updated older than {self.args.hours} hours. Ok? Y/N")
		response = await self.bot.wait_for('message', check=lambda msg: msg.author == self.msg.author and msg.channel == self.msg.channel)
		if not response.content.lower().startswith("y"):
		return 'Cancelling power scrape.'

		if self.args.start:
		self.args.start = " ".join(self.args.start)
		await spy.schedule_power_scrape(eddb_session, power.text,
		callback=self.msg.channel.send, start=self.args.start)
		await spy.schedule_power_scrape(eddb_session, power.text, callback=self.msg.channel.send, hours_old=self.args.hours)
		return f'Scheduled scrape for {power.text}.'

		async def execute(self):
		@@ -2533,7 +2530,7 @@ async def push_spy_to_sheets(): # pragma: no cover \| tested elsewhere
		await scanner.send_batch(payloads, input_opt='USER_ENTERED')


		async def monitor_powerplay_api(client, *, repeat=True, delay=1800, last_scrape=None):
		async def monitor_powerplay_api(client, *, repeat=True, delay=1800):
		"""Poll the powerplay page for info every delay seconds.

		N.B. This depends on multiple scanners being operable. Start this task ONLY when they are ready.
		@@ -2542,14 +2539,11 @@ async def monitor_powerplay_api(client, *, repeat=True, delay=1800, last_scrape=
		client: The discord.py client.
		repeat: If True schedule self at end of execution to run again.
		delay: The delay in seconds between checks.
		last_scrape: A datetime.datetime UTC native object. If None don't run scrape.
		"""
		await asyncio.sleep(delay)
		if repeat and last_scrape:
		last_scrape = await spy.schedule_held(last_scrape)
		asyncio.ensure_future(
		monitor_powerplay_api(client, repeat=repeat, delay=delay, last_scrape=last_scrape)
		)
		if repeat:
		await spy.schedule_federal_held()
		asyncio.ensure_future(monitor_powerplay_api(client, repeat=repeat, delay=delay))

		log = logging.getLogger(__name__)
		try:

cog/bot.py

+1 −2

Original line number	Diff line number	Diff line
		@@ -208,8 +208,7 @@ class CogBot(discord.Client):
		cog.actions.monitor_snipe_merits(self),
		cog.actions.monitor_spy_site(self, repeat=True, delay=900),
		cog.actions.monitor_powerplay_api(self, repeat=False, delay=75), # Runs scrape right after launch once
		cog.actions.monitor_powerplay_api(self, repeat=True, delay=1800,
		last_scrape=datetime.datetime.utcnow()), # Every 30 mins scrape
		cog.actions.monitor_powerplay_api(self, repeat=True, delay=1800), # Every 30 mins scrape
		cogdb.eddb.monitor_eddb_caches(),
		cogdb.monitor_pools(),
		))

cog/parse.py

+4 −2

Original line number	Diff line number	Diff line
		@@ -607,8 +607,10 @@ def subs_scrape(subs, prefix):
		Trigger an update of the sheets based on basic information query.
		{prefix}scrape held FW
		Get information for all controls of power and update sheets.
		{prefix}scrape held ZH --start Kaura
		Update systems with held merits older than 7 hours.
		{prefix}scrape held ZH --hours 3
		Get information for all controls of power and update sheets.
		Update systems with held merits older than 3 hours.
		{prefix}scrape bgs Abi, Rana
		Scrape all bgs information listed systems and ...
		if a system is a control, scrape all exploited systems as well.
		@@ -622,7 +624,7 @@ def subs_scrape(subs, prefix):
		subcmd = subcmds.add_parser('bgs', help='Scrape the bgs .')
		subcmd.add_argument('systems', nargs='+', help='The systems to query for information.')
		subcmd = subcmds.add_parser('held', help='Query the held information for controls of a pwoer.')
		subcmd.add_argument('--start', nargs='+', help='The name of the power to track.')
		subcmd.add_argument('--hours', type=int, default=7, help='Update all systems older than this cutoff.')
		subcmd.add_argument('name', nargs='+', help='The name of the power to track.')
		subcmd = subcmds.add_parser('power', help='Refresh the information and push to sheets.')

cogdb/spy_squirrel.py

+70 −58

Original line number	Diff line number	Diff line
		@@ -351,7 +351,7 @@ class SpySystem(ReprMixin, TimestampMixin, Base):
		updated_at = sqla.Column(sqla.Integer, default=time.time)
		held_merits = sqla.Column(sqla.Integer, default=0)
		stolen_forts = sqla.Column(sqla.Integer, default=0)
		held_updated_at = sqla.Column(sqla.Integer, default=time.time)
		held_updated_at = sqla.Column(sqla.Integer, default=10) # Intentionally old, force held if queried

		# Relationships
		system = sqla.orm.relationship(
		@@ -858,16 +858,13 @@ def response_json_update_system_info(eddb_session, info):
		ship_map_traffic = ship_type_to_id_map(traffic_text=True)
		log = logging.getLogger(__name__)

		now_time = time.time()
		for sys_name, sys_info in info.items():
		# Handle held merits
		if 'power' in sys_info:
		try:
		system = eddb_session.query(SpySystem).\
		filter(SpySystem.system_name == sys_name).\
		one()
		system.held_merits = sys_info['power']['held_merits']
		system.stolen_forts = sys_info['power']['stolen_forts']
		log.info("Updating held merits in %s", sys_name)
		log.info("Updating SpySystem %s", sys_name)
		except sqla.orm.exc.NoResultFound:
		eddb_system = eddb_session.query(System).\
		filter(System.name == sys_name).\
		@@ -879,12 +876,18 @@ def response_json_update_system_info(eddb_session, info):
		'power_state_id': eddb_system.power_state_id,
		'held_merits': sys_info['power']['held_merits'],
		'stolen_forts': sys_info['power']['stolen_forts'],
		'held_updated_at': time.time(),
		}
		system = SpySystem(**kwargs)
		eddb_session.add(system)
		log.warning("Adding SpySystem for held merits: %s", sys_name)

		# Always update held time based on response
		system.held_updated_at = now_time
		if 'power' in sys_info:
		system.held_merits = sys_info['power']['held_merits']
		system.stolen_forts = sys_info['power']['stolen_forts']
		log.info("Updating held merits in %s", sys_name)

		if 'top5_power' in sys_info:
		log.warning("Parsing top 5 bounties for: %s", sys_name)
		for b_info in sys_info['top5_power'].values():
		@@ -1061,32 +1064,26 @@ def get_vote_of_power(eddb_session, power='%hudson'):
		return vote_amount


		async def schedule_held(last_scrape): # pragma: no cover, would ping API point needlessly
		"""Schedule a scrape of federal powers if gap is sufficient since last.

		Args:
		last_scrape: A datetime.datetime object, should be UTC native.
		async def schedule_federal_held(): # pragma: no cover, would ping API point needlessly
		"""Schedule a scrape of federal powers if there are SpySystems that need held updated.

		Returns: The datetime.datetime UTC native object of last time run.
		"""
		now = datetime.datetime.utcnow()
		log = logging.getLogger(__name__)
		log.warning("Scheduling federal scrap: %s", now)
		log.warning("Checking held merits for federal systesm: %s", now)

		if (now - last_scrape) >= EIGHT_HOURS:
		for power_name in ('Felicia Winters', 'Zachary Hudson'):
		log.warning("Scheduling federal start: %s, %s", power_name, datetime.datetime.utcnow())
		log.warning("Checking held merits for: %s, %s", power_name, datetime.datetime.utcnow())
		with cogdb.session_scope(cogdb.EDDBSession) as eddb_session:
		try:
		await schedule_power_scrape(eddb_session, power_name)
		except cog.exc.InvalidCommandArgs:
		pass
		await asyncio.sleep(random.randint(1500, 2250)) # Randomly delay between 25 and 37.5 mins

		return last_scrape
		await asyncio.sleep(random.randint(HELD_DELAY) random.randint(1, 3)) # Randomly delay between


		async def schedule_power_scrape(eddb_session, power_name, *, callback=None, start=None): # pragma: no cover, would ping API point needlessly
		async def schedule_power_scrape(eddb_session, power_name, *, callback=None, hours_old=7): # pragma: no cover, would ping API point needlessly
		"""Schedule a scrape of controls of a given power for detailed information.

		This function will prevent multiple concurrent scrapes at same time.
		@@ -1104,21 +1101,12 @@ async def schedule_power_scrape(eddb_session, power_name, *, callback=None, star
		HELD_RUNNING.format(power_name=power_name, date=HELD_POWERS[power_name]['start_date'])
		)

		control_names = cogdb.eddb.get_controls_of_power(eddb_session, power=power_name)
		systems, _ = cogdb.eddb.get_all_systems_named(eddb_session, control_names)

		# If start present, start posting systems AFTER indicated start system
		if start:
		start = start.lower()
		keep_systems, new_systems = False, []
		for system in systems:
		if keep_systems:
		new_systems += [system]
		if system.name.lower() == start:
		keep_systems = True
		if not new_systems:
		raise cog.exc.InvalidCommandArgs("Please check start system name, could not match.")
		systems = new_systems
		systems = get_controls_outdated_held(eddb_session, power=power_name, hours_old=hours_old)
		sys_names = ", ".join([x.name for x in systems])
		if callback:
		msg = f"Will update the following systems:\n\n{sys_names}"
		await callback(msg)
		logging.getLogger(__name__).info(msg)

		HELD_POWERS[power_name] = {
		'start_date': datetime.datetime.utcnow(),
		@@ -1173,6 +1161,30 @@ async def post_systems(systems, callback=None): # pragma: no cover, would ping
		return influence_ids


		def get_controls_outdated_held(eddb_session, *, power='%hudson', hours_old=7):
		"""
		Get all control Systems of a power mentioned where the held_updated_at date
		is at least hours_old.

		Args:
		eddb_session: A session onto the EDDB db.
		power: The loose like match of the power, i.e. "%hudson".
		hours_old: Update any systems that have held_data this many hours old. Default: >= 7
		"""
		cutoff = time.time() - (hours_old * 60 * 60)

		return eddb_session.query(cogdb.eddb.System).\
		join(SpySystem, cogdb.eddb.System.name == SpySystem.system_name).\
		join(cogdb.eddb.PowerState, cogdb.eddb.System.power_state_id == cogdb.eddb.PowerState.id).\
		join(cogdb.eddb.Power, cogdb.eddb.System.power_id == cogdb.eddb.Power.id).\
		filter(
		cogdb.eddb.Power.text.ilike(power),
		cogdb.eddb.PowerState.text == "Control",
		SpySystem.held_updated_at < cutoff).\
		order_by(System.name).\
		all()


		def preload_spy_tables(eddb_session):
		"""
		Preload the spy tables with constant values.

tests/cogdb/test_spy_squirrel.py

+9 −0

Original line number	Diff line number	Diff line
		@@ -3,6 +3,7 @@ Tests for cogdb.spy
		"""
		import os
		import pathlib
		import time

		import pytest
		import sqlalchemy as sqla
		@@ -73,6 +74,7 @@ def spy_test_bed(eddb_session):
		spy.SpySystem(
		id=1,
		ed_system_id=10477373803,
		system_name="Rana",
		power_id=9,
		power_state_id=16,
		income=122,
		@@ -83,10 +85,12 @@ def spy_test_bed(eddb_session):
		um=40000,
		um_trigger=33998,
		updated_at=FIXED_TIMESTAMP,
		held_updated_at=(time.time() - 36000),
		),
		spy.SpySystem(
		id=2,
		ed_system_id=11665533904241,
		system_name="Nanomam",
		power_id=9,
		power_state_id=64,
		income=0,
		@@ -95,6 +99,7 @@ def spy_test_bed(eddb_session):
		fort_trigger=4872,
		um_trigger=7198,
		updated_at=FIXED_TIMESTAMP,
		held_updated_at=time.time(),
		),
		spy.SpyTraffic(
		id=1,
		@@ -635,3 +640,7 @@ def test_preload_spy_tables(empty_spy, eddb_session):
		def test_get_vote_of_power(empty_spy, eddb_session, spy_test_bed):
		assert 75 == spy.get_vote_of_power(eddb_session)
		assert 0 == spy.get_vote_of_power(eddb_session, power='winters')


		def test_get_controls_outdated_held(empty_spy, db_cleanup, spy_test_bed, eddb_session):
		assert "Rana" in [x.name for x in spy.get_controls_outdated_held(eddb_session, power='%hudson')]