Commit 36585806 authored by Elger Jonker's avatar Elger Jonker

remove rating column


Former-commit-id: 1cc0bb62
parent efa78d16
......@@ -20,8 +20,8 @@ class OrganizationRatingAdmin(ImportExportModelAdmin, admin.ModelAdmin):
id=format(obj.organization_id))
list_display = ('organization', 'high', 'medium', 'low', 'when', 'inspect_organization')
search_fields = (['organization__name', 'rating', 'high', 'medium', 'low', 'when', 'calculation'])
list_filter = ['organization', 'organization__country', 'organization__type__name', 'rating', 'when'][::-1]
search_fields = (['organization__name', 'when'])
list_filter = ['organization', 'organization__country', 'organization__type__name', 'when'][::-1]
fields = ('organization', 'total_urls', 'total_endpoints',
'high',
'medium',
......@@ -74,8 +74,8 @@ class UrlRatingAdmin(ImportExportModelAdmin, admin.ModelAdmin):
id=format(obj.url_id))
list_display = ('url', 'high', 'medium', 'low', 'total_endpoints', 'when', 'inspect_url')
search_fields = (['url__organization__name', 'rating', 'high', 'medium', 'low', 'when', 'calculation'])
list_filter = ['url', 'rating', 'when', 'url__organization__type', 'url__organization__country'][::-1]
search_fields = (['url__organization__name', 'when'])
list_filter = ['url', 'when', 'url__organization__type', 'url__organization__country'][::-1]
fields = ('url', 'total_endpoints',
'total_issues',
......@@ -268,7 +268,7 @@ class VulnerabilityStatisticAdmin(ImportExportModelAdmin, admin.ModelAdmin):
@admin.register(MapDataCache)
class MapDataCacheAdmin(ImportExportModelAdmin, admin.ModelAdmin):
list_display = ('pk', 'country', 'organization_type', 'filters', 'when', 'length')
list_display = ('pk', 'country', 'organization_type', 'filters', 'when')
list_filter = ['country', 'organization_type', 'filters', 'when'][::-1]
search_fields = (['country', 'organization_type', 'filters'])
......@@ -276,4 +276,5 @@ class MapDataCacheAdmin(ImportExportModelAdmin, admin.ModelAdmin):
@staticmethod
def length(obj):
# retrieving this causes a massive slowdown on getting the dataset
return len(str(obj.dataset))
# Generated by Django 2.1.3 on 2018-12-13 11:16
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('map', '0030_configuration_is_reported'),
]
operations = [
migrations.RemoveField(
model_name='organizationrating',
name='rating',
),
migrations.RemoveField(
model_name='urlrating',
name='rating',
),
]
......@@ -31,11 +31,6 @@ class OrganizationRating(models.Model):
scanner to explain why something is the way it is (over time).
"""
organization = models.ForeignKey(Organization, on_delete=models.CASCADE)
rating = models.IntegerField(
help_text="Amount of points scored by the organization based on a sum of all URL ratings at"
" this moment. Rating -1 is used as a default first rating, which are displayed "
"in gray on the map. All next ratings are between 0 (perfect) and 2147483647."
)
total_issues = models.IntegerField(help_text="The summed number of all vulnerabilities and failures.", default=0)
high = models.IntegerField(help_text="The number of high risk vulnerabilities and failures.", default=0)
......@@ -122,12 +117,6 @@ class UrlRating(models.Model):
Aggregrates the results of many scanners to determine a rating for a URL.
"""
url = models.ForeignKey(Url, on_delete=models.CASCADE)
rating = models.IntegerField(
help_text="Amount of points scored after rating the URL. Ratings are usually positive, yet "
"this is not a positive integerfield because we might use -1 as an 'unknown' "
"default value for when there are no ratings at all. Ratings can go from 0 "
"up to 2147483647."
)
# cumulative issues on both the url and all underlying endpoints.
total_issues = models.IntegerField(help_text="The summed number of all vulnerabilities and failures.", default=0)
......
......@@ -987,7 +987,6 @@ def save_url_rating(url: Url, date: datetime, high: int, medium: int, low: int,
u.when = datetime(year=date.year, month=date.month, day=date.day,
hour=23, minute=59, second=59, microsecond=999999, tzinfo=pytz.utc)
u.rating = 0
u.calculation = calculation
u.total_endpoints = total_endpoints
......@@ -1113,7 +1112,6 @@ def rate_organization_on_moment(organization: Organization, when: datetime = Non
if OrganizationRating.objects.all().filter(organization=organization, when=when).exists():
log.info("Rating already exists for %s on %s. Not overwriting." % (organization, when))
total_rating = 0
total_high, total_medium, total_low = 0, 0, 0
# Done: closing off urls, after no relevant endpoints, but still resolvable. Done.
......@@ -1135,7 +1133,6 @@ def rate_organization_on_moment(organization: Organization, when: datetime = Non
url_issues_low, endpoint_issues_high, endpoint_issues_medium, endpoint_issues_low, = 0, 0, 0, 0
for urlrating in all_url_ratings:
total_rating += urlrating.rating
total_high += urlrating.high
total_medium += urlrating.medium
total_low += urlrating.low
......@@ -1180,7 +1177,6 @@ def rate_organization_on_moment(organization: Organization, when: datetime = Non
calculation = {
"organization": {
"name": organization.name,
"rating": total_rating,
"high": total_high,
"medium": total_medium,
"low": total_low,
......@@ -1211,7 +1207,6 @@ def rate_organization_on_moment(organization: Organization, when: datetime = Non
log.debug("The calculation for %s on %s has changed, so we're saving this rating." % (organization, when))
organizationrating = OrganizationRating()
organizationrating.organization = organization
organizationrating.rating = total_rating
organizationrating.high = total_high
organizationrating.medium = total_medium
organizationrating.low = total_low
......@@ -1255,7 +1250,6 @@ def get_latest_urlratings_fast(urls: List[Url], when):
sql = '''SELECT
id,
rating,
high,
medium,
low,
......@@ -1326,12 +1320,10 @@ def default_organization_rating(organizations: List[Organization]):
r = OrganizationRating()
r.when = when
r.rating = -1
r.organization = organization
r.calculation = {
"organization": {
"name": organization.name,
"rating": "-1",
"high": "0",
"medium": "0",
"low": "0",
......@@ -1427,7 +1419,6 @@ def calculate_vulnerability_statistics(days: int = 366):
sql = """
SELECT
map_organizationrating.rating,
organization.name,
organizations_organizationtype.name,
coordinate_stack.area,
......@@ -1577,10 +1568,13 @@ def calculate_map_data_today():
@app.task(queue='storage')
def calculate_map_data(days: int = 366):
from django.db import OperationalError
log.info("calculate_map_data")
map_configurations = Configuration.objects.all().filter(
is_displayed=True).order_by('display_order').values('country', 'organization_type__name', 'organization_type')
is_displayed=True
).order_by('display_order').values('country', 'organization_type__name', 'organization_type')
for map_configuration in map_configurations:
for days_back in list(reversed(range(0, days))):
......@@ -1598,10 +1592,12 @@ def calculate_map_data(days: int = 366):
map_configuration['country'], map_configuration['organization_type__name'],
days_back, when, scan_type
))
data = get_map_data(map_configuration['country'], map_configuration['organization_type__name'],
days_back, scan_type)
from django.db import OperationalError
data = get_map_data(
map_configuration['country'],
map_configuration['organization_type__name'],
days_back,
scan_type
)
try:
cached = MapDataCache()
......
......@@ -412,8 +412,7 @@ def organization_report(request, country: str = "NL", organization_type="municip
country=get_country(country),
type=get_organization_type(organization_type))
ratings = organization.filter(organizationrating__when__lte=when)
values = ratings.values('organizationrating__rating',
'organizationrating__calculation',
values = ratings.values('organizationrating__calculation',
'organizationrating__when',
'name',
'pk',
......@@ -435,7 +434,6 @@ def organization_report(request, country: str = "NL", organization_type="municip
"slug": slugify(values['name']),
"id": values['pk'],
"twitter_handle": values['twitter_handle'],
"rating": values['organizationrating__rating'],
"when": values['organizationrating__when'].isoformat(),
# fixing json being presented and escaped as a string, this makes it a lot slowr
......@@ -488,7 +486,7 @@ def terrible_urls(request, country: str = "NL", organization_type="municipality"
sql = '''
SELECT
rating,
low,
organization.name,
organizations_organizationtype.name,
organization.id,
......@@ -573,7 +571,7 @@ def top_fail(request, country: str = "NL", organization_type="municipality", wee
sql = '''
SELECT
rating,
low,
organization.name,
organizations_organizationtype.name,
organization.id,
......@@ -662,7 +660,7 @@ def top_win(request, country: str = "NL", organization_type="municipality", week
cursor = connection.cursor()
sql = '''
SELECT
rating,
low,
organization.name,
organizations_organizationtype.name,
organization.id,
......@@ -787,8 +785,8 @@ def stats(request, country: str = "NL", organization_type="municipality", weeks_
# do not create stats over empty organizations. That would count empty organizations.
# you can't really filter them out above? todo: Figure that out at a next release.
if rating.rating == -1:
continue
# if rating.rating == -1:
# continue
measurement["total_organizations"] += 1
......@@ -1554,7 +1552,7 @@ def get_map_data(country: str = "NL", organization_type: str = "municipality", d
# a bit slower it seems, but still well within acceptable levels.
sql = """
SELECT
map_organizationrating.rating,
map_organizationrating.low,
organization.name,
organizations_organizationtype.name,
coordinate_stack.area,
......
......@@ -158,7 +158,7 @@ class CoordinateAdminInline(CompactInline):
class OrganizationRatingAdminInline(CompactInline):
model = OrganizationRating
extra = 0
readonly_fields = ('organization', 'rating', 'high', 'medium', 'low', 'when', 'calculation')
readonly_fields = ('organization', 'high', 'medium', 'low', 'when', 'calculation')
can_delete = False
ordering = ["-when"]
......@@ -166,7 +166,7 @@ class OrganizationRatingAdminInline(CompactInline):
class UrlRatingAdminInline(CompactInline):
model = UrlRating
extra = 0
readonly_fields = ('url', 'rating', 'high', 'medium', 'low', 'when', 'calculation')
readonly_fields = ('url', 'high', 'medium', 'low', 'when', 'calculation')
can_delete = False
ordering = ["-when"]
......@@ -694,7 +694,6 @@ class DatasetAdmin(ImportExportModelAdmin, admin.ModelAdmin):
form = DatasetForm
save_as = True
save_on_top = True
preserve_filters = True
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment