Commit 414f73d2 authored by Jeremy Pallats's avatar Jeremy Pallats 💬
Browse files

Make Use of POST Hooks 

- OCR will now update only on POST notification.
- Move SCANNERS into more suitable module.
parent 53eecd8a

cog/actions.py

+4 −65
Original line number Diff line number Diff line
@@ -33,6 +33,7 @@ import cog.inara 
import cog.tbl

import cog.util

from cogdb.schema import FortUser, UMUser, EUMSheet

from cogdb.scanners import get_scanner





async def bot_shutdown(bot):  # pragma: no cover
@@ -408,6 +409,7 @@ class Admin(Action): 
            InternalException - No parseable numeric component found in tab.

            RemoteError - The sheet/tab combination could not be resolved. Tab needs creating.

        """

        scanners = cogdb.scanners.SCANNERS

        # Zero trackers for new ocr data

        cogdb.query.post_cycle_db_cleanup(self.session)

        self.bot.deny_commands = True
@@ -420,13 +422,13 @@ class Admin(Action): 
                scanner_configs[name]['page'] = new_page



                try:

                    await SCANNERS[name].asheet.change_worksheet(new_page)

                    await scanners[name].asheet.change_worksheet(new_page)

                except gspread.exceptions.WorksheetNotFound as exc:

                    msg = f"Missing **{new_page}** worksheet on {name}. Please fix and rerun cycle. No change made."

                    raise cog.exc.InvalidCommandArgs(msg) from exc



                self.bot.sched.schedule(name, delay=1)

                lines += [[await SCANNERS[name].asheet.title(), new_page]]

                lines += [[await scanners[name].asheet.title(), new_page]]



            await cog.util.CONF.aupdate("scanners", value=scanner_configs)
@@ -2136,28 +2138,6 @@ def filter_top_dusers(guild, dusers, exclude_roles, limit=5): 
    return top_recruits, top_members





def init_scanner(name):

    """

    Initialize a scanner based on configuration.

    """

    print("Intializing scanner -> ", name)

    logging.getLogger(__name__).info("Initializing the %s scanner.", name)

    sheet = cog.util.CONF.scanners.get(name)

    cls = getattr(cogdb.scanners, sheet["cls"])

    scanner = cls(sheet)

    SCANNERS[name] = scanner





def get_scanner(name):

    """

    Store scanners in this module for shared use.

    """

    try:

        return SCANNERS[name]

    except KeyError as exc:

        raise cog.exc.InvalidCommandArgs("The scanners are not ready. Please try again in 15 seconds.") from exc





async def monitor_carrier_events(client, *, next_summary, last_timestamp=None, delay=60):  # pragma: no cover

    """

    Simple async task that just checks for new events every delay.
@@ -2204,46 +2184,6 @@ async def monitor_carrier_events(client, *, next_summary, last_timestamp=None, d 
    )





async def monitor_ocr_sheet(client, *, delay=300, repeat=True):  # pragma: no cover

    """

    Simple async task that just checks for changes to the OCR sheet.

    This task will schedule itself infinitely on a delay.



    Args:

        client: The bot client itself.



    Kwargs:

        delay: The seconds between checking the sheet. Default 30 minutes.

        repeat: If true, will schedule itself infinitely.

    """

    if delay >= 1:

        await asyncio.sleep(delay)



    # Update database by triggering manual refresh

    ocr_scanner = get_scanner('hudson_ocr')

    await ocr_scanner.update_cells()

    with cfut.ProcessPoolExecutor(max_workers=1) as pool:

        await client.loop.run_in_executor(

            pool, ocr_scanner.scheduler_run,

        )



    # Data refreshed, analyse and update

    with cogdb.session_scope(cogdb.Session) as session:

        cell_updates = cogdb.query.ocr_update_fort_status(session)

        if cell_updates:

            await get_scanner('hudson_cattle').send_batch(cell_updates)

            logging.getLogger(__name__).info("Sent update to sheet.")

            logging.getLogger(__name__).info(str(cell_updates))



    # A onetime flag to trigger for testing

    if repeat:

        asyncio.ensure_future(

            monitor_ocr_sheet(

                client, delay=delay, repeat=repeat

            )

        )





async def monitor_snipe_merits(client, *, repeat=True):  # pragma: no cover

    """

    Schedule self to check snipe merits at the following times.
@@ -2317,7 +2257,6 @@ async def report_to_leadership(client, msgs): # pragma: no cover 
            await client.send_message(chan, msg)





SCANNERS = {}

SCOUT_RND = {  # TODO: Extract to data config or tables.

    1: [

        "Epsilon Scorpii",

cog/bot.py

+2 −3
Original line number Diff line number Diff line
@@ -175,10 +175,10 @@ class CogBot(discord.Client): 
        self.emoji.update(self.guilds)



        # This block is effectively a one time setup.

        if not cog.actions.SCANNERS:

        if not cogdb.scanners.SCANNERS:

            async def scanner_startup_task():

                scanners = await cogdb.scanners.init_scanners()

                cog.actions.SCANNERS = scanners

                cogdb.scanners.SCANNERS = scanners



                self.sched.register('hudson_cattle', scanners['hudson_cattle'],

                                    ('Drop', 'Fort', 'User'))
@@ -205,7 +205,6 @@ class CogBot(discord.Client): 
                simple_heartbeat(),

                cog.util.CONF.monitor(),

                cog.actions.monitor_carrier_events(self, next_summary=next_summary, delay=60),

                cog.actions.monitor_ocr_sheet(self),

                cog.actions.monitor_snipe_merits(self),

                cogdb.eddb.monitor_eddb_caches(),

                cogdb.monitor_pools(),

cog/scheduler.py

+7 −2
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ import aiozmq 
import aiozmq.rpc



import cog.util

import cogdb.scanners



ADDR = 'tcp://127.0.0.1:{}'.format(cog.util.CONF.ports.zmq)

POOL = cfut.ProcessPoolExecutor(max_workers=os.cpu_count())
@@ -139,6 +140,10 @@ class Scheduler(aiozmq.rpc.AttrHandler): 
        self.count = (self.count + 1) % 1000

        logging.getLogger(__name__).info(

            'POST %d received: %s %s', self.count, scanner, timestamp)

        if scanner == 'hudson_ocr':

            print("OCR")

            asyncio.ensure_future(cogdb.scanners.handle_ocr_sheet_update(cog.util.BOT))

        else:

            self.schedule(scanner)

            print('SCHEDULED: ', scanner, timestamp)

cogdb/scanners.py

+38 −0
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ All sheet scanners are stored here for now 
Sheet scanners make heavy use of cog.sheets.AsyncGSheet

"""

import asyncio

import concurrent.futures as cfut

import datetime

import logging

import re
@@ -20,6 +21,7 @@ from cogdb.schema import (FortSystem, FortPrep, FortDrop, FortUser, 




SNIPE_FIRST_ID = 10001

SCANNERS = {}





class FortScanner():
@@ -1057,3 +1059,39 @@ async def init_scanners(): 
    await asyncio.gather(*init_coros)



    return scanners





async def handle_ocr_sheet_update(client):  # pragma: no cover

    """

    This task is to be run only when the OCR sheet has been updated.

    Update the OCR information in the db and then take any actions

    required with changes.



    Args:

        client: The bot client itself.

    """

    # Update database by triggering manual refresh

    ocr_scanner = get_scanner('hudson_ocr')

    await ocr_scanner.update_cells()

    with cfut.ProcessPoolExecutor(max_workers=1) as pool:

        await client.loop.run_in_executor(

            pool, ocr_scanner.scheduler_run,

        )



    # Data refreshed, analyse and update

    with cogdb.session_scope(cogdb.Session) as session:

        cell_updates = cogdb.query.ocr_update_fort_status(session)

        if cell_updates:

            await get_scanner('hudson_cattle').send_batch(cell_updates)

            logging.getLogger(__name__).info("Sent update to sheet.")

            logging.getLogger(__name__).info(str(cell_updates))





def get_scanner(name):

    """

    Store scanners in this module for shared use.

    """

    try:

        return SCANNERS[name]

    except KeyError as exc:

        raise cog.exc.InvalidCommandArgs("The scanners are not ready. Please try again in 15 seconds.") from exc

tests/cog/test_actions.py

+20 −20
Original line number Diff line number Diff line
@@ -34,7 +34,7 @@ pytestmark = pytest.mark.usefixtures("patch_scanners") 
@pytest.fixture

def patch_scanners():

    """ Patch the scanners. """

    old_scanners = cog.actions.SCANNERS

    old_scanners = cogdb.scanners.SCANNERS

    scanner = aiomock.Mock(user_row=5)



    async def send_batch_(payloads, *args, input_opt=''):
@@ -58,7 +58,7 @@ def patch_scanners(): 
    scanner.get_batch = get_batch_

    scanner.find_dupe = find_dupe_

    scanner.update_cells = update_cells_

    cog.actions.SCANNERS = {

    cogdb.scanners.SCANNERS = {

        'hudson_carriers': scanner,

        'hudson_cattle': scanner,

        'hudson_kos': scanner,
@@ -69,7 +69,7 @@ def patch_scanners(): 


    yield scanner



    cog.actions.SCANNERS = old_scanners

    cogdb.scanners.SCANNERS = old_scanners





def action_map(fake_message, fake_bot):
@@ -264,7 +264,7 @@ async def test_cmd_admin_removeum_fail(f_admins, f_bot, db_cleanup): 
@pytest.mark.asyncio

async def test_cmd_admin_removeum(f_bot, f_dusers, f_admins, f_um_testbed,

                                  f_umformula_values, patch_scanners, db_cleanup):

    fake_um = cog.actions.SCANNERS['hudson_undermine']

    fake_um = cogdb.scanners.SCANNERS['hudson_undermine']

    fake_um._values = [f_umformula_values[4:]]



    msg = fake_msg_gears("!admin removeum Pequen")
@@ -301,7 +301,7 @@ async def test_cmd_admin_addum_fail(f_admins, f_bot, db_cleanup): 


@pytest.mark.asyncio

async def test_cmd_admin_addum(f_admins, f_bot, f_asheet_umscanner, patch_scanners, db_cleanup):

    fake_um = cog.actions.SCANNERS['hudson_undermine']

    fake_um = cogdb.scanners.SCANNERS['hudson_undermine']

    fake_cols = await f_asheet_umscanner.whole_sheet()

    fake_cols = fake_cols[:13]

    fake_cols = [[columns[i] for columns in fake_cols] for i in range(17)]
@@ -555,7 +555,7 @@ async def test_cmd_fort_set(session, f_bot, f_dusers, f_fort_testbed): 
    expect = [

        {'range': 'H6:H7', 'values': [[7000], [222]]},

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio
@@ -680,7 +680,7 @@ async def test_cmd_drop_simple(session, f_bot, f_dusers, f_fort_testbed): 
        {'range': 'H6:H7', 'values': [[6000], [0]]},

        {'range': 'H15:H15', 'values': [[978]]},

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio
@@ -702,7 +702,7 @@ async def test_cmd_drop_negative(session, f_bot, f_dusers, f_fort_testbed): 
        {'range': 'H6:H7', 'values': [[5322], [0]]},

        {'range': 'H15:H15', 'values': [[300]]},

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio
@@ -728,7 +728,7 @@ async def test_cmd_drop_newuser(session, f_bot, f_dusers, f_fort_testbed): 
        {'range': 'H6:H7', 'values': [[5922], [0]]},

        {'range': 'H18:H18', 'values': [[500]]},

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio
@@ -750,7 +750,7 @@ async def test_cmd_drop_set(session, f_bot, f_dusers, f_fort_testbed): 
        {'range': 'H6:H7', 'values': [[6500], [0]]},

        {'range': 'H15:H15', 'values': [[978]]},

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio
@@ -779,7 +779,7 @@ Bonus for highest contribution: 
        {'range': 'K6:K7', 'values': [[7400], [0]]},

        {'range': 'K15:K15', 'values': [[400]]},

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio
@@ -807,7 +807,7 @@ Power |```""" 
    expect = [

        {'range': 'K18:L18', 'values': [[2000, 0]]},

    ]

    assert cog.actions.SCANNERS['hudson_undermine'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_undermine'].payloads == expect





@pytest.mark.asyncio
@@ -837,7 +837,7 @@ Power |```""" 
        {'range': 'A20:B20', 'values': [['test cry', 'test name']]},

        {'range': 'K20:L20', 'values': [[1000, 0]]},

    ]

    assert cog.actions.SCANNERS['hudson_undermine'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_undermine'].payloads == expect





@pytest.mark.asyncio
@@ -868,7 +868,7 @@ Burr | 0 | 8000```""" 
        {'range': 'F18:G18', 'values': [[0, 1950]]},

        {'range': 'H18:I18', 'values': [[0, 8000]]},

    ]

    assert cog.actions.SCANNERS['hudson_undermine'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_undermine'].payloads == expect





@pytest.mark.asyncio
@@ -898,7 +898,7 @@ Power |``` 
    expect = [

        {'range': 'K18:L18', 'values': [[10000, 0]]},

    ]

    assert cog.actions.SCANNERS['hudson_undermine'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_undermine'].payloads == expect





@pytest.mark.asyncio
@@ -921,7 +921,7 @@ async def test_cmd_hold_died(session, f_bot, f_dusers, f_um_testbed): 
        {'range': 'F18:G18', 'values': [[0, 1550]]},

        {'range': 'H18:I18', 'values': [[0, 5800]]},

    ]

    assert cog.actions.SCANNERS['hudson_undermine'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_undermine'].payloads == expect





@pytest.mark.asyncio
@@ -956,7 +956,7 @@ Burr | 2200 | 5800```""" 
    expect = [

        {'range': 'F18:G18', 'values': [[0, 1950]]}

    ]

    assert cog.actions.SCANNERS['hudson_undermine'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_undermine'].payloads == expect





@pytest.mark.asyncio
@@ -1426,7 +1426,7 @@ Power |```""" 
    expect = [

        {'range': 'F10:F13', 'values': [[12000], [0.4], ['Hold Merits'], [600]]}

    ]

    assert cog.actions.SCANNERS['hudson_undermine'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_undermine'].payloads == expect





@pytest.mark.asyncio
@@ -1531,7 +1531,7 @@ Burr | 2200 | 5800```""" 
        {'range': 'A15:B15', 'values': [['User1 are forting late!', 'NotUser1']]},

        {'range': 'A18:B18', 'values': [['We go pew pew!', 'NotUser1']]}

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio
@@ -1572,7 +1572,7 @@ Burr | 2200 | 5800```""" 
        {'range': 'A15:B15', 'values': [['A new cry', 'User1']]},

        {'range': 'A18:B18', 'values': [['A new cry', 'User1']]}

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio