Commit 29efcc42 authored by Jeremy Pallats's avatar Jeremy Pallats 💬
Browse files

Make Use of POST Hooks 

- OCR will now update only on POST notification.
- Move SCANNERS into more suitable module.
parent 53eecd8a

cog/actions.py

+4 −53
Original line number Diff line number Diff line
@@ -33,6 +33,7 @@ import cog.inara 
import cog.tbl

import cog.util

from cogdb.schema import FortUser, UMUser, EUMSheet

from cogdb.scanners import get_scanner





async def bot_shutdown(bot):  # pragma: no cover
@@ -408,6 +409,7 @@ class Admin(Action): 
            InternalException - No parseable numeric component found in tab.

            RemoteError - The sheet/tab combination could not be resolved. Tab needs creating.

        """

        scanners = cogdb.scanners.SCANNERS

        # Zero trackers for new ocr data

        cogdb.query.post_cycle_db_cleanup(self.session)

        self.bot.deny_commands = True
@@ -420,13 +422,13 @@ class Admin(Action): 
                scanner_configs[name]['page'] = new_page



                try:

                    await SCANNERS[name].asheet.change_worksheet(new_page)

                    await scanners[name].asheet.change_worksheet(new_page)

                except gspread.exceptions.WorksheetNotFound as exc:

                    msg = f"Missing **{new_page}** worksheet on {name}. Please fix and rerun cycle. No change made."

                    raise cog.exc.InvalidCommandArgs(msg) from exc



                self.bot.sched.schedule(name, delay=1)

                lines += [[await SCANNERS[name].asheet.title(), new_page]]

                lines += [[await scanners[name].asheet.title(), new_page]]



            await cog.util.CONF.aupdate("scanners", value=scanner_configs)
@@ -2148,16 +2150,6 @@ def init_scanner(name): 
    SCANNERS[name] = scanner





def get_scanner(name):

    """

    Store scanners in this module for shared use.

    """

    try:

        return SCANNERS[name]

    except KeyError as exc:

        raise cog.exc.InvalidCommandArgs("The scanners are not ready. Please try again in 15 seconds.") from exc





async def monitor_carrier_events(client, *, next_summary, last_timestamp=None, delay=60):  # pragma: no cover

    """

    Simple async task that just checks for new events every delay.
@@ -2204,46 +2196,6 @@ async def monitor_carrier_events(client, *, next_summary, last_timestamp=None, d 
    )





async def monitor_ocr_sheet(client, *, delay=300, repeat=True):  # pragma: no cover

    """

    Simple async task that just checks for changes to the OCR sheet.

    This task will schedule itself infinitely on a delay.



    Args:

        client: The bot client itself.



    Kwargs:

        delay: The seconds between checking the sheet. Default 30 minutes.

        repeat: If true, will schedule itself infinitely.

    """

    if delay >= 1:

        await asyncio.sleep(delay)



    # Update database by triggering manual refresh

    ocr_scanner = get_scanner('hudson_ocr')

    await ocr_scanner.update_cells()

    with cfut.ProcessPoolExecutor(max_workers=1) as pool:

        await client.loop.run_in_executor(

            pool, ocr_scanner.scheduler_run,

        )



    # Data refreshed, analyse and update

    with cogdb.session_scope(cogdb.Session) as session:

        cell_updates = cogdb.query.ocr_update_fort_status(session)

        if cell_updates:

            await get_scanner('hudson_cattle').send_batch(cell_updates)

            logging.getLogger(__name__).info("Sent update to sheet.")

            logging.getLogger(__name__).info(str(cell_updates))



    # A onetime flag to trigger for testing

    if repeat:

        asyncio.ensure_future(

            monitor_ocr_sheet(

                client, delay=delay, repeat=repeat

            )

        )





async def monitor_snipe_merits(client, *, repeat=True):  # pragma: no cover

    """

    Schedule self to check snipe merits at the following times.
@@ -2317,7 +2269,6 @@ async def report_to_leadership(client, msgs): # pragma: no cover 
            await client.send_message(chan, msg)





SCANNERS = {}

SCOUT_RND = {  # TODO: Extract to data config or tables.

    1: [

        "Epsilon Scorpii",

cog/bot.py

+2 −3
Original line number Diff line number Diff line
@@ -175,10 +175,10 @@ class CogBot(discord.Client): 
        self.emoji.update(self.guilds)



        # This block is effectively a one time setup.

        if not cog.actions.SCANNERS:

        if not cogdb.scanners.SCANNERS:

            async def scanner_startup_task():

                scanners = await cogdb.scanners.init_scanners()

                cog.actions.SCANNERS = scanners

                cogdb.scanners.SCANNERS = scanners



                self.sched.register('hudson_cattle', scanners['hudson_cattle'],

                                    ('Drop', 'Fort', 'User'))
@@ -205,7 +205,6 @@ class CogBot(discord.Client): 
                simple_heartbeat(),

                cog.util.CONF.monitor(),

                cog.actions.monitor_carrier_events(self, next_summary=next_summary, delay=60),

                cog.actions.monitor_ocr_sheet(self),

                cog.actions.monitor_snipe_merits(self),

                cogdb.eddb.monitor_eddb_caches(),

                cogdb.monitor_pools(),

cog/scheduler.py

+7 −2
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ import aiozmq 
import aiozmq.rpc



import cog.util

import cogdb.scanners



ADDR = 'tcp://127.0.0.1:{}'.format(cog.util.CONF.ports.zmq)

POOL = cfut.ProcessPoolExecutor(max_workers=os.cpu_count())
@@ -139,6 +140,10 @@ class Scheduler(aiozmq.rpc.AttrHandler): 
        self.count = (self.count + 1) % 1000

        logging.getLogger(__name__).info(

            'POST %d received: %s %s', self.count, scanner, timestamp)

        if scanner == 'hudson_ocr':

            print("OCR")

            asyncio.ensure_future(cogdb.scanners.handle_ocr_sheet_update(cog.util.BOT))

        else:

            self.schedule(scanner)

            print('SCHEDULED: ', scanner, timestamp)

cogdb/scanners.py

+40 −0
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ All sheet scanners are stored here for now 
Sheet scanners make heavy use of cog.sheets.AsyncGSheet

"""

import asyncio

import concurrent.futures as cfut

import datetime

import logging

import re
@@ -20,6 +21,7 @@ from cogdb.schema import (FortSystem, FortPrep, FortDrop, FortUser, 




SNIPE_FIRST_ID = 10001

SCANNERS = {}





class FortScanner():
@@ -1057,3 +1059,41 @@ async def init_scanners(): 
    await asyncio.gather(*init_coros)



    return scanners





async def handle_ocr_sheet_update(client):  # pragma: no cover

    """

    This task is to be run only when the OCR sheet has been updated.



    Args:

        client: The bot client itself.



    Kwargs:

        delay: The seconds between checking the sheet. Default 30 minutes.

        repeat: If true, will schedule itself infinitely.

    """

    # Update database by triggering manual refresh

    ocr_scanner = get_scanner('hudson_ocr')

    await ocr_scanner.update_cells()

    with cfut.ProcessPoolExecutor(max_workers=1) as pool:

        await client.loop.run_in_executor(

            pool, ocr_scanner.scheduler_run,

        )



    # Data refreshed, analyse and update

    with cogdb.session_scope(cogdb.Session) as session:

        cell_updates = cogdb.query.ocr_update_fort_status(session)

        if cell_updates:

            await get_scanner('hudson_cattle').send_batch(cell_updates)

            logging.getLogger(__name__).info("Sent update to sheet.")

            logging.getLogger(__name__).info(str(cell_updates))





def get_scanner(name):

    """

    Store scanners in this module for shared use.

    """

    try:

        return SCANNERS[name]

    except KeyError as exc:

        raise cog.exc.InvalidCommandArgs("The scanners are not ready. Please try again in 15 seconds.") from exc

tests/cog/test_actions.py

+20 −20
Original line number Diff line number Diff line
@@ -34,7 +34,7 @@ pytestmark = pytest.mark.usefixtures("patch_scanners") 
@pytest.fixture

def patch_scanners():

    """ Patch the scanners. """

    old_scanners = cog.actions.SCANNERS

    old_scanners = cogdb.scanners.SCANNERS

    scanner = aiomock.Mock(user_row=5)



    async def send_batch_(payloads, *args, input_opt=''):
@@ -58,7 +58,7 @@ def patch_scanners(): 
    scanner.get_batch = get_batch_

    scanner.find_dupe = find_dupe_

    scanner.update_cells = update_cells_

    cog.actions.SCANNERS = {

    cogdb.scanners.SCANNERS = {

        'hudson_carriers': scanner,

        'hudson_cattle': scanner,

        'hudson_kos': scanner,
@@ -69,7 +69,7 @@ def patch_scanners(): 


    yield scanner



    cog.actions.SCANNERS = old_scanners

    cogdb.scanners.SCANNERS = old_scanners





def action_map(fake_message, fake_bot):
@@ -264,7 +264,7 @@ async def test_cmd_admin_removeum_fail(f_admins, f_bot, db_cleanup): 
@pytest.mark.asyncio

async def test_cmd_admin_removeum(f_bot, f_dusers, f_admins, f_um_testbed,

                                  f_umformula_values, patch_scanners, db_cleanup):

    fake_um = cog.actions.SCANNERS['hudson_undermine']

    fake_um = cogdb.scanners.SCANNERS['hudson_undermine']

    fake_um._values = [f_umformula_values[4:]]



    msg = fake_msg_gears("!admin removeum Pequen")
@@ -301,7 +301,7 @@ async def test_cmd_admin_addum_fail(f_admins, f_bot, db_cleanup): 


@pytest.mark.asyncio

async def test_cmd_admin_addum(f_admins, f_bot, f_asheet_umscanner, patch_scanners, db_cleanup):

    fake_um = cog.actions.SCANNERS['hudson_undermine']

    fake_um = cogdb.scanners.SCANNERS['hudson_undermine']

    fake_cols = await f_asheet_umscanner.whole_sheet()

    fake_cols = fake_cols[:13]

    fake_cols = [[columns[i] for columns in fake_cols] for i in range(17)]
@@ -555,7 +555,7 @@ async def test_cmd_fort_set(session, f_bot, f_dusers, f_fort_testbed): 
    expect = [

        {'range': 'H6:H7', 'values': [[7000], [222]]},

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio
@@ -680,7 +680,7 @@ async def test_cmd_drop_simple(session, f_bot, f_dusers, f_fort_testbed): 
        {'range': 'H6:H7', 'values': [[6000], [0]]},

        {'range': 'H15:H15', 'values': [[978]]},

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio
@@ -702,7 +702,7 @@ async def test_cmd_drop_negative(session, f_bot, f_dusers, f_fort_testbed): 
        {'range': 'H6:H7', 'values': [[5322], [0]]},

        {'range': 'H15:H15', 'values': [[300]]},

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio
@@ -728,7 +728,7 @@ async def test_cmd_drop_newuser(session, f_bot, f_dusers, f_fort_testbed): 
        {'range': 'H6:H7', 'values': [[5922], [0]]},

        {'range': 'H18:H18', 'values': [[500]]},

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio
@@ -750,7 +750,7 @@ async def test_cmd_drop_set(session, f_bot, f_dusers, f_fort_testbed): 
        {'range': 'H6:H7', 'values': [[6500], [0]]},

        {'range': 'H15:H15', 'values': [[978]]},

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio
@@ -779,7 +779,7 @@ Bonus for highest contribution: 
        {'range': 'K6:K7', 'values': [[7400], [0]]},

        {'range': 'K15:K15', 'values': [[400]]},

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio
@@ -807,7 +807,7 @@ Power |```""" 
    expect = [

        {'range': 'K18:L18', 'values': [[2000, 0]]},

    ]

    assert cog.actions.SCANNERS['hudson_undermine'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_undermine'].payloads == expect





@pytest.mark.asyncio
@@ -837,7 +837,7 @@ Power |```""" 
        {'range': 'A20:B20', 'values': [['test cry', 'test name']]},

        {'range': 'K20:L20', 'values': [[1000, 0]]},

    ]

    assert cog.actions.SCANNERS['hudson_undermine'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_undermine'].payloads == expect





@pytest.mark.asyncio
@@ -868,7 +868,7 @@ Burr | 0 | 8000```""" 
        {'range': 'F18:G18', 'values': [[0, 1950]]},

        {'range': 'H18:I18', 'values': [[0, 8000]]},

    ]

    assert cog.actions.SCANNERS['hudson_undermine'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_undermine'].payloads == expect





@pytest.mark.asyncio
@@ -898,7 +898,7 @@ Power |``` 
    expect = [

        {'range': 'K18:L18', 'values': [[10000, 0]]},

    ]

    assert cog.actions.SCANNERS['hudson_undermine'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_undermine'].payloads == expect





@pytest.mark.asyncio
@@ -921,7 +921,7 @@ async def test_cmd_hold_died(session, f_bot, f_dusers, f_um_testbed): 
        {'range': 'F18:G18', 'values': [[0, 1550]]},

        {'range': 'H18:I18', 'values': [[0, 5800]]},

    ]

    assert cog.actions.SCANNERS['hudson_undermine'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_undermine'].payloads == expect





@pytest.mark.asyncio
@@ -956,7 +956,7 @@ Burr | 2200 | 5800```""" 
    expect = [

        {'range': 'F18:G18', 'values': [[0, 1950]]}

    ]

    assert cog.actions.SCANNERS['hudson_undermine'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_undermine'].payloads == expect





@pytest.mark.asyncio
@@ -1426,7 +1426,7 @@ Power |```""" 
    expect = [

        {'range': 'F10:F13', 'values': [[12000], [0.4], ['Hold Merits'], [600]]}

    ]

    assert cog.actions.SCANNERS['hudson_undermine'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_undermine'].payloads == expect





@pytest.mark.asyncio
@@ -1531,7 +1531,7 @@ Burr | 2200 | 5800```""" 
        {'range': 'A15:B15', 'values': [['User1 are forting late!', 'NotUser1']]},

        {'range': 'A18:B18', 'values': [['We go pew pew!', 'NotUser1']]}

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio
@@ -1572,7 +1572,7 @@ Burr | 2200 | 5800```""" 
        {'range': 'A15:B15', 'values': [['A new cry', 'User1']]},

        {'range': 'A18:B18', 'values': [['A new cry', 'User1']]}

    ]

    assert cog.actions.SCANNERS['hudson_cattle'].payloads == expect

    assert cogdb.scanners.SCANNERS['hudson_cattle'].payloads == expect





@pytest.mark.asyncio