Commit fa291361 authored by Erik Kalkoken's avatar Erik Kalkoken
Browse files

Add option to load dogmas with types

parent 258fdacd
Pipeline #208905382 passed with stages
in 7 minutes and 5 seconds
......@@ -7,6 +7,31 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
## [Unreleased] - yyyy-mm-dd
## [0.6.0] - 2020-10-27
### Added
- Can now also get dogmas when loading types via `eveuniverse_load_types` management command
- Added option to load dogmas when updating/creating an EveType
- New function `eveuniverse.core.esitools.is_esi_online()` for querying the current status of the Eve servers
- Added info logging to load tasks
- Added info logging for `eveuniverse.tools.testdata.create_testdata()`
### Changed
- BREAKING CHANGE: Changed interface of the test tool: `eveuniverse.tools.testdata.create_testdata()`.
- Inline objects can now also be loaded async when `wait_for_children` is set to `False`
- `all_models()` is now member of `eveuniverse.models.EveUniverseBaseModel` and also returns Inline models
It requires a list of specifications instead of a dict. Also, you now need to provide the name of the model with `ModelSpec` instead of in the dict as before.
- Reduced duration for loading testdata with `eveuniverse.tools.testdata.load_testdata_from_dict()`
- Added inline models to docs
- Added core functions to docs
### Fixed
- Name field of EveDogmaEffect was too small
- Testdata creation now also supports inline models, e.g. Dogmas
## [0.5.0] - 2020-10-23
### Added
......
......@@ -8,6 +8,29 @@ This chapter contains the developer reference documentation of the public API fo
.. _api-eve-models:
Base classes
============
.. autoclass:: eveuniverse.models.EveUniverseBaseModel
:members:
.. autoclass:: eveuniverse.models.EveUniverseEntityModel
:members:
Core functions
==============
esitools
----------------
.. automodule:: eveuniverse.core.esitools
:members:
eveimageserver
----------------
.. automodule:: eveuniverse.core.eveimageserver
:members:
Eve Models
==========
......@@ -49,6 +72,11 @@ EveDogmaEffect
.. _api-models-eve-entity:
EveDogmaEffectModifier
----------------------
.. autoclass:: eveuniverse.models.EveDogmaEffectModifier
:members:
Eve Entity
--------------
......@@ -123,11 +151,26 @@ EveStation
.. autoclass:: eveuniverse.models.EveStation
:members:
EveStationService
-----------------
.. autoclass:: eveuniverse.models.EveStationService
:members:
EveType
---------
.. autoclass:: eveuniverse.models.EveType
:members:
EveTypeDogmaAttribute
---------------------
.. autoclass:: eveuniverse.models.EveTypeDogmaAttribute
:members:
EveTypeDogmaEffect
------------------
.. autoclass:: eveuniverse.models.EveTypeDogmaEffect
:members:
EveUnit
---------
.. autoclass:: eveuniverse.models.EveUnit
......
......@@ -253,18 +253,17 @@ from . import test_data_filename
class CreateEveUniverseTestData(TestCase):
def test_create_testdata(self):
testdata_spec = {
"EveFaction": ModelSpec(ids=[500001], include_children=False),
"EveType": ModelSpec(
testdata_spec = [
ModelSpec("EveFaction", ids=[500001]),
ModelSpec(
"EveType",
ids=[603, 621, 638, 2488, 2977, 3756, 11379, 16238, 34562, 37483],
include_children=False,
),
"EveSolarSystem": ModelSpec(
ids=[30001161, 30004976, 30004984, 30045349, 31000005],
include_children=False,
ModelSpec(
"EveSolarSystem", ids=[30001161, 30004976, 30004984, 30045349, 31000005],
),
"EveRegion": ModelSpec(ids=[10000038], include_children=True,),
}
ModelSpec("EveRegion", ids=[10000038], include_children=True),
]
create_testdata(testdata_spec, test_data_filename())
```
......
......@@ -117,12 +117,40 @@ Note that all settings are optional and the app will use the documented default
The following management commands are available:
- **eveuniverse_load_data**: This command will load a complete set of data form ESI and store it locally. Useful to optimize performance or when you want to provide the user with drop-down lists. Available sets:
- **map**: All regions, constellations and solar systems
- **ships**: All ship types
- **structures**: All structures types
- **eveuniverse_purge_all**: This command will purge ALL data of your models.
- **eveuniverse_load_type**: This command can load a specific set of types. This is a helper command meant to be called from other apps only.
### eveuniverse_load_data
This command will load a complete set of data form ESI and store it locally. Useful to optimize performance or when you want to provide the user with drop-down lists. Available sets:
- **map**: All regions, constellations and solar systems
- **ships**: All ship types
- **structures**: All structures types
### eveuniverse_purge_all
This command will purge ALL data of your models
### eveuniverse_load_types
```text
Loads large sets of types as specified from ESI into the local database. This
is a helper command meant to be called from other apps only.
positional arguments:
app_name Name of app this data is loaded for
optional arguments:
-h, --help show this help message and exit
--category_id CATEGORY_ID
Eve category ID to be loaded excl. dogma
--category_id_with_dogma CATEGORY_ID_WITH_DOGMA
Eve category ID to be loaded incl. dogma
--group_id GROUP_ID Eve group ID to be loaded excl. dogma
--group_id_with_dogma GROUP_ID_WITH_DOGMA
Eve group ID to be loaded incl. dogma
--type_id TYPE_ID Eve type ID to be loaded excl. dogma
--type_id_with_dogma TYPE_ID_WITH_DOGMA
Eve type ID to be loaded incl. dogma
```
## Database tools
......
default_app_config = "eveuniverse.apps.EveuniverseConfig"
__version__ = "0.5.0"
__version__ = "0.6.0"
__title__ = "Eve Universe"
from bravado.exception import HTTPError
from ..providers import esi
def is_esi_online() -> bool:
"""Checks if the Eve servers are online. Returns True if there are, else False"""
try:
status = esi.client.Status.get_status().results()
if status.get("vip"):
return False
except HTTPError:
return False
return True
......@@ -22,22 +22,56 @@ class Command(BaseCommand):
"--category_id",
action="append",
type=int,
help="Eve category ID to be loaded",
help="Eve category ID to be loaded excl. dogma",
)
parser.add_argument(
"--group_id", action="append", type=int, help="Eve group ID to be loaded"
"--category_id_with_dogma",
action="append",
type=int,
help="Eve category ID to be loaded incl. dogma",
)
parser.add_argument(
"--type_id", action="append", type=int, help="Eve type ID to be loaded"
"--group_id",
action="append",
type=int,
help="Eve group ID to be loaded excl. dogma",
)
parser.add_argument(
"--group_id_with_dogma",
action="append",
type=int,
help="Eve group ID to be loaded incl. dogma",
)
parser.add_argument(
"--type_id",
action="append",
type=int,
help="Eve type ID to be loaded excl. dogma",
)
parser.add_argument(
"--type_id_with_dogma",
action="append",
type=int,
help="Eve type ID to be loaded incl. dogma",
)
def handle(self, *args, **options):
app_name = options["app_name"]
category_ids = options["category_id"]
category_ids_with_dogma = options["category_id_with_dogma"]
group_ids = options["group_id"]
group_ids_with_dogma = options["group_id_with_dogma"]
type_ids = options["type_id"]
type_ids_with_dogma = options["type_id_with_dogma"]
if not category_ids and not group_ids and not type_ids:
if (
not category_ids
and not category_ids_with_dogma
and not group_ids
and not group_ids_with_dogma
and not type_ids
and not type_ids_with_dogma
):
self.stdout.write(self.style.WARNING("No IDs specified. Nothing to do."))
return
......@@ -59,9 +93,17 @@ class Command(BaseCommand):
)
user_input = get_input("Are you sure you want to proceed? (Y/n)?")
if user_input == "Y":
load_eve_types.delay(
category_ids=category_ids, group_ids=group_ids, type_ids=type_ids
)
if category_ids or group_ids or type_ids:
load_eve_types.delay(
category_ids=category_ids, group_ids=group_ids, type_ids=type_ids
)
if category_ids_with_dogma or group_ids_with_dogma or type_ids_with_dogma:
load_eve_types.delay(
category_ids=category_ids_with_dogma,
group_ids=group_ids_with_dogma,
type_ids=type_ids_with_dogma,
force_loading_dogma=True,
)
self.stdout.write(self.style.SUCCESS("Data loading has been started!"))
else:
self.stdout.write(self.style.WARNING("Aborted"))
......@@ -4,7 +4,7 @@ from django.core.management.base import BaseCommand
from django.db import transaction
from ... import __title__
from ...models import EveUniverseEntityModel
from ...models import EveUniverseBaseModel
from . import get_input
from ...utils import LoggerAddTag
......@@ -21,7 +21,7 @@ class Command(BaseCommand):
def _purge_all_data(self):
"""updates all SDE models from ESI and provides progress output"""
with transaction.atomic():
for MyModel in EveUniverseEntityModel.all_models():
for MyModel in EveUniverseBaseModel.all_models():
self.stdout.write(
"Deleting {:,} objects from {}".format(
MyModel.objects.count(),
......
......@@ -49,16 +49,16 @@ class EveUniverseBaseModelManager(models.Manager):
try:
value = ParentClass.objects.get(id=esi_value)
except ParentClass.DoesNotExist:
if mapping.create_related and hasattr(
ParentClass.objects, "update_or_create_esi"
):
value, _ = ParentClass.objects.update_or_create_esi(
id=esi_value,
include_children=False,
wait_for_children=True,
)
else:
value = None
value = None
if mapping.create_related:
try:
value, _ = ParentClass.objects.update_or_create_esi(
id=esi_value,
include_children=False,
wait_for_children=True,
)
except AttributeError:
pass
else:
if mapping.is_charfield and esi_value is None:
......@@ -111,6 +111,7 @@ class EveUniverseEntityModelManager(EveUniverseBaseModelManager):
id: int,
include_children: bool = False,
wait_for_children: bool = True,
enabled_sections: Iterable[str] = None,
) -> Tuple[models.Model, bool]:
"""updates or creates an Eve universe object by fetching it from ESI (blocking).
Will always get/create parent objects
......@@ -119,11 +120,13 @@ class EveUniverseEntityModelManager(EveUniverseBaseModelManager):
id: Eve Online ID of object
include_children: if child objects should be updated/created as well (if any)
wait_for_children: when true child objects will be updated/created blocking (if any), else async
enabled_sections: Sections to load regardless of current settings, e.g. `EveUniverseEntityModel.LOAD_DOGMAS` will always load dogmas for EveTypes
Returns:
A tuple consisting of the requested object and a created flag
"""
id = int(id)
enabled_sections = set(enabled_sections) if enabled_sections else set()
add_prefix = make_logger_prefix("%s(id=%s)" % (self.model.__name__, id))
try:
eve_data_obj = self._transform_esi_response_for_list_endpoints(
......@@ -132,14 +135,15 @@ class EveUniverseEntityModelManager(EveUniverseBaseModelManager):
if eve_data_obj:
defaults = self._defaults_from_esi_obj(eve_data_obj)
obj, created = self.update_or_create(id=id, defaults=defaults)
inline_objects = self.model._inline_objects()
inline_objects = self.model._inline_objects(enabled_sections)
if inline_objects:
self._update_or_create_inline_objects(
parent_eve_data_obj=eve_data_obj,
parent_obj=obj,
inline_objects=inline_objects,
wait_for_children=wait_for_children,
)
if eve_data_obj and include_children:
if include_children:
self._update_or_create_children(
parent_eve_data_obj=eve_data_obj,
include_children=include_children,
......@@ -199,10 +203,15 @@ class EveUniverseEntityModelManager(EveUniverseBaseModelManager):
parent_eve_data_obj: dict,
parent_obj: models.Model,
inline_objects: dict,
wait_for_children: bool,
) -> None:
"""updates_or_creates eve objects that are returns "inline" from ESI
for the parent eve objects as defined for this parent model (if any)
"""
from .tasks import (
update_or_create_inline_object as task_update_or_create_inline_object,
)
if not parent_eve_data_obj or not parent_obj:
raise ValueError(
"%s: Tried to create inline object from empty parent object"
......@@ -237,28 +246,31 @@ class EveUniverseEntityModelManager(EveUniverseBaseModelManager):
)
)
parent2_model_name = ParentClass2.__name__ if ParentClass2 else None
other_pk_info = {
"name": other_pk[0],
"esi_name": other_pk[1].esi_name,
"is_fk": other_pk[1].is_fk,
}
for eve_data_obj in parent_eve_data_obj[inline_field]:
args = {parent_fk: parent_obj}
esi_value = eve_data_obj[other_pk[1].esi_name]
if other_pk[1].is_fk:
try:
value = ParentClass2.objects.get(id=esi_value)
except ParentClass2.DoesNotExist:
if hasattr(ParentClass2.objects, "update_or_create_esi"):
(
value,
_,
) = ParentClass2.objects.get_or_create_esi(id=esi_value)
else:
value = None
if wait_for_children:
parent_obj._update_or_create_inline_object(
parent_fk=parent_fk,
eve_data_obj=eve_data_obj,
other_pk_info=other_pk_info,
parent2_model_name=parent2_model_name,
inline_model_name=model_name,
)
else:
value = esi_value
args[other_pk[0]] = value
args["defaults"] = InlineModel.objects._defaults_from_esi_obj(
eve_data_obj,
)
InlineModel.objects.update_or_create(**args)
task_update_or_create_inline_object(
parent_model_name=self.model.__name__,
parent_object_pk=parent_obj.pk,
parent_fk=parent_fk,
eve_data_obj=eve_data_obj,
other_pk_info=other_pk_info,
parent2_model_name=parent2_model_name,
inline_model_name=model_name,
)
def _update_or_create_children(
self,
......@@ -458,6 +470,7 @@ class EveStargateManager(EveUniverseEntityModelManager):
*,
include_children: bool = False,
wait_for_children: bool = True,
enabled_sections: Iterable[str] = None,
) -> Tuple[models.Model, bool]:
"""updates or creates an EveStargate object by fetching it from ESI (blocking).
Will always get/create parent objects
......@@ -497,6 +510,7 @@ class EveStationManager(EveUniverseEntityModelManager):
parent_eve_data_obj: dict,
parent_obj: models.Model,
inline_objects: dict,
wait_for_children: bool,
) -> None:
"""updates_or_creates station service objects for EveStations"""
from .models import EveStationService
......@@ -571,6 +585,7 @@ class EveEntityManager(EveUniverseEntityModelManager):
id: int,
include_children: bool = False,
wait_for_children: bool = True,
enabled_sections: Iterable[str] = None,
) -> Tuple[models.Model, bool]:
"""updates or creates an EveEntity object by fetching it from ESI (blocking).
......
# Generated by Django 3.1.2 on 2020-10-26 22:42
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("eveuniverse", "0003_evemarketprice"),
]
operations = [
migrations.AlterField(
model_name="evedogmaeffect",
name="name",
field=models.CharField(
db_index=True, default="", help_text="Eve Online name", max_length=400
),
),
]
......@@ -3,7 +3,7 @@ import inspect
import logging
import math
import sys
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict, List, Optional, Tuple, Set
from bravado.exception import HTTPNotFound
......@@ -59,7 +59,7 @@ EsiMapping = namedtuple(
class EveUniverseBaseModel(models.Model):
"""Base properties and features"""
"""Base class for all Eve Universe Models"""
objects = EveUniverseBaseModelManager()
......@@ -99,6 +99,42 @@ class EveUniverseBaseModel(models.Model):
return f"{self.__class__.__name__}({', '.join(fields_2)})"
@classmethod
def all_models(cls) -> List[Dict[models.Model, int]]:
"""returns a list of all Eve Universe model classes sorted by load order"""
mappings = list()
for _, ModelClass in inspect.getmembers(sys.modules[__name__], inspect.isclass):
if issubclass(
ModelClass, (EveUniverseEntityModel, EveUniverseInlineModel)
) and ModelClass not in (
cls,
EveUniverseEntityModel,
EveUniverseInlineModel,
):
mappings.append(
{
"model": ModelClass,
"load_order": ModelClass._eve_universe_meta_attr(
"load_order", is_mandatory=True
),
}
)
return [y["model"] for y in sorted(mappings, key=lambda x: x["load_order"])]
@classmethod
def get_model_class(cls, model_name: str) -> models.Model:
"""returns the model class for the given name"""
classes = {
x[0]: x[1]
for x in inspect.getmembers(sys.modules[__name__], inspect.isclass)
if issubclass(x[1], (EveUniverseBaseModel, EveUniverseInlineModel))
}
try:
return classes[model_name]
except KeyError:
raise ValueError("Unknown model_name: %s" % model_name)
@classmethod
def _esi_mapping(cls) -> dict:
field_mappings = cls._eve_universe_meta_attr("field_mappings")
......@@ -167,27 +203,30 @@ class EveUniverseBaseModel(models.Model):
cls, attr_name: str, is_mandatory: bool = False
) -> Optional[Any]:
"""returns value of an attribute from EveUniverseMeta or None"""
if not hasattr(cls, "EveUniverseMeta"):
raise ValueError("EveUniverseMeta not defined for class %s" % cls.__name__)
if hasattr(cls.EveUniverseMeta, attr_name):
try:
value = getattr(cls.EveUniverseMeta, attr_name)
else:
except AttributeError:
value = None
if is_mandatory:
raise ValueError(
"Mandatory attribute EveUniverseMeta.%s not defined "
"for class %s" % (attr_name, cls.__name__)
)
return value
class EveUniverseEntityModel(EveUniverseBaseModel):
"""Eve Universe Entity model
"""Base class for Eve Universe Entity models
Entity models are normal Eve entities that have a dedicated ESI endpoint
"""
# sections
LOAD_DOGMAS = "dogmas"
# TODO: Implement other sections
# icons
DEFAULT_ICON_SIZE = 64
id = models.PositiveIntegerField(primary_key=True, help_text="Eve Online ID")
......@@ -211,6 +250,39 @@ class EveUniverseEntityModel(EveUniverseBaseModel):
def __str__(self) -> str:
return self.name
def _update_or_create_inline_object(
self,
parent_fk: str,
eve_data_obj: dict,
other_pk_info: dict,
parent2_model_name: str,
inline_model_name: str,
):
"""Updates or creates a single inline object.
Will automatically create additional parent objects as needed
"""
InlineModel = self.get_model_class(inline_model_name)
args = {parent_fk: self}
esi_value = eve_data_obj.get(other_pk_info["esi_name"])
if other_pk_info["is_fk"]:
ParentClass2 = self.get_model_class(parent2_model_name)
try:
value = ParentClass2.objects.get(id=esi_value)
except ParentClass2.DoesNotExist:
try:
value, _ = ParentClass2.objects.get_or_create_esi(id=esi_value)
except AttributeError:
value = None
else: