Commit 04e1fa15 authored by Jeremy Pallats's avatar Jeremy Pallats 💬
Browse files

Implement validation for OCR into model. 

- Add tests to cover new OCR parts of schema including validation.
- New ValidationFail exception for validating model.
parent 9917528b

cog/exc.py

+6 −0
Original line number Diff line number Diff line
@@ -108,6 +108,12 @@ class InternalException(CogException): 
        super().__init__(msg, lvl)





class ValidationFail(InternalException):

    """ Raise when a validation function on db has failed. """

    def __init__(self, msg=''):

        super().__init__(msg)





class ColOverflow(InternalException):

    """ Raise when a column has reached end, increment next column.  """

    def __init__(self):

cogdb/query.py

+18 −16
Original line number Diff line number Diff line
@@ -978,11 +978,12 @@ def update_ocr_live(session, trackers_dict, sheet_date=None): 


        data = copy_tracker_dict[system.system]



        # TODO: Move to obj.update(), validate internally.

        system.fort = data.get('fort', system.fort)

        system.um = data.get('um', system.um)

        system.updated_at = data.get('updated_at', sheet_date)

        data['updated_at'] = data.get('updated_at', sheet_date)

        try:

            system.update(**data)

            updated += [system.system]

        except cog.exc.ValidationFail:

            pass



        del copy_tracker_dict[system.system]
@@ -1029,13 +1030,12 @@ def update_ocr_trigger(session, trigger_dict, sheet_date=None): 
    for system in ocr_systems:

        data = copy_trigger_dict[system.system]



        # TODO: Move to obj.update(), validate internally.

        system.fort_trigger = data.get("fort_trigger", system.fort_trigger)

        system.um_trigger = data.get("um_trigger", system.um_trigger)

        system.base_income = data.get("base_income", system.base_income)

        system.last_upkeep = data.get("last_upkeep", system.last_upkeep)

        system.updated_at = data.get('updated_at', sheet_date)

        data['updated_at'] = data.get('updated_at', sheet_date)

        try:

            system.update(**data)

            updated += [system.system]

        except cog.exc.ValidationFail:

            pass



        del copy_trigger_dict[system.system]
@@ -1083,10 +1083,12 @@ def update_ocr_prep(session, prep_dict, sheet_date=None): 


        data = copy_prep_dict[system.system]



        # TODO: Move to obj.update(), validate internally.

        system.merits = data.get("merits", system.merits)

        system.updated_at = data.get("updated_at", system.updated_at)

        data['updated_at'] = data.get('updated_at', sheet_date)

        try:

            system.update(**data)

            updated += [system.system]

        except cog.exc.ValidationFail:

            pass



        del copy_prep_dict[system.system]

cogdb/schema.py

+170 −2
Original line number Diff line number Diff line
@@ -22,6 +22,8 @@ LEN_NAME = 100 
LEN_REASON = 400

LEN_SHEET_COL = 5

LEN_CARRIER = 7

SENSIBLE_OCR_MERITS = 150000

SENSIBLE_OCR_INCOME = 250

EVENT_CARRIER = """

CREATE EVENT IF NOT EXISTS clean_carriers

ON SCHEDULE
@@ -1023,6 +1025,46 @@ class OCRTracker(Base): 
    def __hash__(self):

        return hash("{}".format(self.system))



    def update(self, **kwargs):

        """

        Update the object with expected kwargs.



        kwargs:

            fort: The new fort merit value.

            um: The new um merit value.

            updated_at: The new date time to set for this update. (Required)



        Raises:

            ValidationFail - The kwargs did not contain updated_at or it was not suitable.

        """

        if 'updated_at' not in kwargs:

            raise cog.exc.ValidationFail("Expected key 'updated_at' is missing.")



        self.updated_at = kwargs['updated_at']

        for key in ['fort', 'um']:

            try:

                setattr(self, key, kwargs[key])

            except (KeyError, cog.exc.ValidationFail):

                pass



    @sqla_orm.validates('fort', 'um')

    def validate_merits(self, key, value):

        try:

            if value < 0 or value > SENSIBLE_OCR_MERITS or value < getattr(self, key):

                raise cog.exc.ValidationFail("Bounds check failed for: {} with value {}".format(key, value))

        except TypeError:

            pass



        return value



    @sqla_orm.validates('updated_at')

    def validate_updated_at(self, key, value):

        if not value or not isinstance(value, datetime.datetime) or (

                self.updated_at and value < self.updated_at):

            raise cog.exc.ValidationFail("Date invalid or was older than current value.")



        return value





class OCRTrigger(Base):

    """
@@ -1041,15 +1083,16 @@ class OCRTrigger(Base): 
    updated_at = sqla.Column(sqla.DateTime(timezone=False), default=datetime.datetime.utcnow())  # All dates UTC



    def __repr__(self):

        keys = ['id', 'system', 'fort_trigger', 'um_trigger', 'updated_at']

        keys = ['id', 'system', 'fort_trigger', 'um_trigger', 'base_income', 'last_upkeep', 'updated_at']

        kwargs = ['{}={!r}'.format(key, getattr(self, key)) for key in keys]



        return "{}({})".format(self.__class__.__name__, ', '.join(kwargs))



    def __str__(self):

        """ A pretty one line to give all information. """

        return "{system}: {fort_trigger}:{um_trigger}  Last Update at {date}".format(

        return "{system}: {fort_trigger}:{um_trigger} with income of {base_income} and upkeep {last_upkeep}  Last Update at {date}".format(

            um_trigger=self.um_trigger, fort_trigger=self.fort_trigger,

            base_income=self.base_income, last_upkeep=self.last_upkeep,

            system=self.system, date=self.updated_at)



    def __eq__(self, other):
@@ -1058,6 +1101,57 @@ class OCRTrigger(Base): 
    def __hash__(self):

        return hash("{}".format(self.system))



    def update(self, **kwargs):

        """

        Update the object with expected kwargs.



        kwargs:

            fort_trigger: The new fort trigger.

            um_trigger: The new um trigger.

            base_income: The new base income of the system.

            last_upkeep: The last cycle upkeep of this system.

            updated_at: The new date time to set for this update. (Required)



        Raises:

            ValidationFail - The kwargs did not contain updated_at or it was not suitable.

        """

        if 'updated_at' not in kwargs:

            raise cog.exc.ValidationFail("Expected key 'updated_at' is missing.")



        self.updated_at = kwargs['updated_at']

        for key in ['fort_trigger', 'um_trigger', 'base_income', 'last_upkeep']:

            try:

                setattr(self, key, kwargs[key])

            except (KeyError, cog.exc.ValidationFail):

                pass



    @sqla_orm.validates('fort_trigger', 'um_trigger')

    def validate_triggers(self, key, value):

        try:

            if value < 0 or value > SENSIBLE_OCR_MERITS:

                raise cog.exc.ValidationFail("Bounds check failed for: {} with value {}".format(key, value))

        except TypeError:

            pass



        return value



    @sqla_orm.validates('base_income', 'last_upkeep')

    def validate_incomes(self, key, value):

        try:

            if value < 0 or value > SENSIBLE_OCR_INCOME:

                raise cog.exc.ValidationFail("Bounds check failed for: {} with value {}".format(key, value))

        except TypeError:

            pass



        return value



    @sqla_orm.validates('updated_at')

    def validate_updated_at(self, key, value):

        if not value or not isinstance(value, datetime.datetime) or (self.updated_at and value < self.updated_at):

            raise cog.exc.ValidationFail("Date invalid or was older than current value.")



        return value





class OCRPrep(Base):

    """
@@ -1089,6 +1183,43 @@ class OCRPrep(Base): 
    def __hash__(self):

        return hash("{}".format(self.system))



    def update(self, **kwargs):

        """

        Update the object with expected kwargs.



        kwargs:

            merits: The current merits for the system.

            updated_at: The new date time to set for this update. (Required)



        Raises:

            ValidationFail - The kwargs did not contain updated_at or it was not suitable.

        """

        if 'updated_at' not in kwargs:

            raise cog.exc.ValidationFail("Expected key 'updated_at' is missing.")



        self.updated_at = kwargs['updated_at']

        try:

            self.merits = kwargs['merits']

        except (KeyError, cog.exc.ValidationFail):

            pass



    @sqla_orm.validates('merits')

    def validate_triggers(self, key, value):

        try:

            if value < 0 or value > SENSIBLE_OCR_MERITS:

                raise cog.exc.ValidationFail("Bounds check failed for: {} with value {}".format(key, value))

        except TypeError:

            pass



        return value



    @sqla_orm.validates('updated_at')

    def validate_updated_at(self, key, value):

        if not value or not isinstance(value, datetime.datetime) or (self.updated_at and value < self.updated_at):

            raise cog.exc.ValidationFail("Date invalid or was older than current value.")



        return value





class Global(Base):

    """
@@ -1118,6 +1249,43 @@ class Global(Base): 
    def __hash__(self):

        return hash("{}".format(self.id))



    def update(self, **kwargs):

        """

        Update the object with expected kwargs.



        kwargs:

            consolidation: The consolidation % of the curent vote.

            updated_at: The new date time to set for this update. (Required)



        Raises:

            ValidationFail - The kwargs did not contain updated_at or it was not suitable.

        """

        if 'updated_at' not in kwargs:

            raise cog.exc.ValidationFail("Expected key 'updated_at' is missing.")



        self.updated_at = kwargs['updated_at']

        try:

            self.consolidation = kwargs['consolidation']

        except (KeyError, cog.exc.ValidationFail):

            pass



    @sqla_orm.validates('consolidation')

    def validate_triggers(self, key, value):

        try:

            if value < 0 or value > 100:

                raise cog.exc.ValidationFail("Bounds check failed for: {} with value {}".format(key, value))

        except TypeError:

            pass



        return value



    @sqla_orm.validates('updated_at')

    def validate_updated_at(self, key, value):

        if not value or not isinstance(value, datetime.datetime) or (self.updated_at and value < self.updated_at):

            raise cog.exc.ValidationFail("Date invalid or was older than current value.")



        return value





def kwargs_um_system(cells, sheet_col):

    """

tests/cogdb/test_query.py

+8 −7
Original line number Diff line number Diff line
@@ -11,7 +11,7 @@ import cogdb 
from cogdb.schema import (DiscordUser, FortSystem, FortUser, FortOrder,

                          UMUser, UMHold, AdminPerm, ChannelPerm, RolePerm,

                          KOS, TrackSystem, TrackSystemCached, TrackByID,

                          OCRTracker, OCRTrigger, OCRPrep)

                          OCRTracker, OCRTrigger, OCRPrep, Global)

import cogdb.query



from tests.data import SYSTEMS, USERS
@@ -648,7 +648,7 @@ def test_users_with_um_merits(session, f_dusers, f_fort_testbed, f_um_testbed): 




def test_update_ocr_live(session, f_ocr_testbed):

    session.add(OCRTracker(system='Sol', fort=7777, um=9999, updated_at=datetime.datetime(2021, 8, 23, 0, 33, 20)))

    session.add(OCRTracker(system='Sol', fort=3333, um=2222, updated_at=datetime.datetime(2021, 8, 23, 0, 33, 20)))

    session.commit()

    test_data = {

        'Adeo': {
@@ -659,8 +659,8 @@ def test_update_ocr_live(session, f_ocr_testbed): 
        },

        'Sol': {

            'system': 'Sol',

            'fort': 3333,

            'um': 400,

            'fort': 7777,

            'um': 8888,

            'updated_at': datetime.datetime(2021, 8, 23, 0, 33, 20),

        },

    }
@@ -668,7 +668,7 @@ def test_update_ocr_live(session, f_ocr_testbed): 


    tracks = session.query(OCRTracker).order_by(OCRTracker.system.asc()).all()

    assert tracks[0].um == 2332

    assert tracks[1].fort == 3333

    assert tracks[1].fort == 7777





def test_update_ocr_trigger(session, f_ocr_testbed):
@@ -722,7 +722,7 @@ def test_update_ocr_prep(session, f_ocr_testbed): 
    assert tracks[1].merits == 333





def test_get_oldest_trigger(session):

def test_get_oldest_trigger_empty(session):

    oldest = cogdb.query.get_oldest_ocr_trigger(session)

    assert oldest is None
@@ -754,7 +754,8 @@ def test_get_oldest_trigger(session, f_ocr_testbed): 
    assert oldest.system == "Adeo"





def test_get_current_global(session):

def test_get_current_global(session, f_global_testbed):

    session.query(Global).delete()

    current = cogdb.query.get_current_global(session)

    assert current.consolidation == 0

    current.consolidation = 77

tests/cogdb/test_scanners.py

+4 −4
Original line number Diff line number Diff line
@@ -482,7 +482,7 @@ async def test_carrierscanner_parse_sheet(session, f_track_testbed): 




@pytest.mark.asyncio

async def test_ocrscanner_parse_sheet(session, f_asheet_ocrscanner, f_ocr_testbed):

async def test_ocrscanner_parse_sheet(session, f_asheet_ocrscanner, f_ocr_testbed, f_global_testbed):

    o_scanner = OCRScanner(f_asheet_ocrscanner)

    await o_scanner.update_cells()

    o_scanner.parse_sheet(session)
@@ -491,7 +491,7 @@ async def test_ocrscanner_parse_sheet(session, f_asheet_ocrscanner, f_ocr_testbe 
        assert new_session.query(OCRTracker).filter(OCRTracker.system == 'Adeo').one().fort == 3576

        assert new_session.query(OCRPrep).filter(OCRPrep.system == 'Bolg').one().merits == 8592

        assert new_session.query(OCRTrigger).filter(OCRTrigger.system == 'Adeo').one().fort_trigger == 3576

        assert new_session.query(Global).one().consolidation == 76

        assert new_session.query(Global).all()[-1].consolidation == 76





@pytest.mark.asyncio
@@ -505,12 +505,12 @@ async def test_ocrscanner_should_update_triggers_none(session, f_asheet_ocrscann 
@pytest.mark.asyncio

async def test_ocrscanner_should_update_triggers_stale(session, f_asheet_ocrscanner, f_ocr_testbed):

    trigger = f_ocr_testbed[0][0]

    trigger.updated_at = trigger.updated_at - datetime.timedelta(days=14)

    old_trigger = OCRTrigger(system='Adeo', updated_at=(trigger.updated_at - datetime.timedelta(days=14)))



    o_scanner = OCRScanner(f_asheet_ocrscanner)

    await o_scanner.update_cells()

    sheet_date = datetime.datetime.strptime(o_scanner.cells_row_major[0][2], "%Y-%m-%d %H:%M:%S")

    assert o_scanner.should_update_trigger(trigger, sheet_date)

    assert o_scanner.should_update_trigger(old_trigger, sheet_date)





@pytest.mark.asyncio