Cleanup unused imports and fix other pyflakes issues.

parent ad6d3ad8
Pipeline #13509137 failed with stage
in 7 minutes and 40 seconds
......@@ -5,9 +5,8 @@ import logging
import time
from collections import Counter
import celery.exceptions
import kombu.exceptions
from celery.result import AsyncResult, GroupResult, ResultSet
from celery.result import ResultSet
from django.conf import settings
from django.core.management.base import BaseCommand
......@@ -58,7 +57,6 @@ class TaskCommand(BaseCommand):
def _add_arguments(self, parser):
"""Method to allow subclasses to add command specific arguments."""
pass
def add_arguments(self, parser):
"""Add common argument for Celery tasks."""
......
from __future__ import absolute_import, unicode_literals
import os
import sys
from django.core.management.base import BaseCommand
......
from __future__ import absolute_import, unicode_literals
import os
import sys
import warnings
from django.core.management import call_command
......
......@@ -3,7 +3,6 @@ import logging
from datetime import datetime
import pytz
from dateutil.parser import parse
from dateutil.relativedelta import relativedelta # history
from django.core.exceptions import ObjectDoesNotExist
......@@ -282,9 +281,6 @@ def timeline(url):
def rate_timeline(timeline, url):
logger.info("Rebuilding ratings for for %s" % url)
# will be filled with dates and
url_rating_jsons = {}
previous_ratings = {}
previous_endpoints = []
for moment in timeline:
......@@ -615,7 +611,6 @@ def show_timeline_console(timeline, url):
print("")
print(url.url)
for moment in timeline:
scores = []
# prepare endpoints to contain ratings
# for ep in timeline[moment]['endpoints']:
# ep["ratings"] = {}
......@@ -807,7 +802,6 @@ def rate_organization(organization, when=""):
all_url_ratings.append(urlrating)
except UrlRating.DoesNotExist:
logger.warning("Url has no rating at this moment: %s %s" % (url, when))
pass
# sort all_url_ratings on rating desc.
# https://stackoverflow.com/questions/403421/
......@@ -1297,7 +1291,6 @@ def get_report_from_scanner_tls_qualys(endpoint, when):
except TlsQualysScan.DoesNotExist:
# can happen that a rating simply isn't there yet. Perfectly possible.
logger.debug("No tls qualys scan on endpoint %s." % endpoint)
pass
return 0, ""
......
......@@ -3,7 +3,7 @@ import logging
from failmap_admin.celery import app
from failmap_admin.map.determineratings import (default_ratings, rate_organizations_efficient,
rerate_existing_urls)
from failmap_admin.map.models import OrganizationRating, UrlRating
from failmap_admin.map.models import OrganizationRating
log = logging.getLogger(__name__)
......
from django.test import TestCase
# Create your tests here.
......@@ -94,10 +94,10 @@ class SmartAddUrl(DashboardModule):
# the first/only object.
# are domains unique? no. Might cause issues.
domainandtld = xtrct.domain + '.' + xtrct.suffix
# pyflakes domainandtld = xtrct.domain + '.' + xtrct.suffix
completedomain = xtrct.subdomain + '.' + xtrct.domain + '.' + xtrct.suffix
o = Url.objects.get(url=domainandtld)
# pyflakes o = Url.objects.get(url=domainandtld)
newurl = Url(url=completedomain)
newurl.save()
# newurl.organization = o.organization.all()
......
import ipaddress
import logging
import tldextract
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization
from failmap_admin.scanners.models import Endpoint, TlsQualysScan, Url
logger = logging.getLogger(__package__)
......
import ipaddress
import logging
import tldextract
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.map.models import OrganizationRating, UrlRating
......
......@@ -4,9 +4,7 @@ from datetime import datetime
import pytz
from django.core.management.commands.dumpdata import Command as DumpDataCommand
from django.core.serializers import serialize
from django.db import connection
from failmap_admin import settings
from failmap_admin.organizations.models import Coordinate, Organization, OrganizationType, Url
from failmap_admin.scanners.models import Endpoint, EndpointGenericScan, TlsQualysScan
......
import logging
import warnings
from datetime import datetime
import pytz
from django.core.management.commands.loaddata import Command as LoadDataCommand
from django.db import connection
from failmap_admin import settings
......
import ipaddress
import logging
import tldextract
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization
from failmap_admin.scanners.models import Endpoint, TlsQualysScan, Url
logger = logging.getLogger(__package__)
......
import ipaddress
import logging
import tldextract
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization
from failmap_admin.scanners.models import Endpoint, EndpointGenericScan, TlsQualysScan, Url
logger = logging.getLogger(__package__)
......
from django.contrib import admin
from jet.admin import CompactInline
from failmap_admin.map.determineratings import OrganizationRating, UrlRating, rate_url
from failmap_admin.map.determineratings import rate_url
from failmap_admin.scanners.scanner_tls_qualys import ScannerTlsQualys
from .models import (Endpoint, EndpointGenericScan, EndpointGenericScanScratchpad, Screenshot,
State, TlsQualysScan, TlsQualysScratchpad, Url)
State, TlsQualysScan, TlsQualysScratchpad)
class TlsQualysScanAdminInline(CompactInline):
......
import ipaddress
import logging
import tldextract
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization
from failmap_admin.scanners.models import Endpoint, TlsQualysScan, Url
from failmap_admin.scanners.models import Endpoint, Url
logger = logging.getLogger(__package__)
......@@ -53,7 +50,7 @@ opendata.arnhem.nl 2 december 2016 15:01 4 december 2016 15:50 1
# https://docs.djangoproject.com/en/1.11/howto/custom-management-commands/
def handle(self, *args, **options):
endpoints = Endpoint.objects.all()
# pyflakes endpoints = Endpoint.objects.all()
# Command.urls_with_ridiculous_number_of_endpoints()
Command.resequence_endpoint_deaths()
......
import logging
from datetime import datetime
import pytz
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.map.determineratings import (rate_organization_efficient,
rerate_url_with_timeline, show_timeline_console,
timeline)
from failmap_admin.map.models import OrganizationRating
from failmap_admin.organizations.models import Organization, Url
from failmap_admin.scanners.models import Endpoint
# from failmap_admin.scanners.scanner_security_headers import scan_all_urls_celery, scan_headers
from failmap_admin.scanners.state_manager import StateManager
from failmap_admin.scanners.scanner_security_headers import scan_all_urls_celery, scan_headers
logger = logging.getLogger(__package__)
......@@ -128,12 +123,12 @@ def develop_determineratings():
# for url in urls:
# DetermineRatings.get_url_score_modular(url)
when = datetime(2016, 12, 31, 0, 0, tzinfo=pytz.utc)
# pyflakes when = datetime(2016, 12, 31, 0, 0, tzinfo=pytz.utc)
# when = datetime.now(pytz.utc)
organization = Organization.objects.filter(name="Arnhem").get()
clear_organization_and_urls(organization)
rate_organization_urls_efficient(organization, create_history=True)
# pyflakes clear_organization_and_urls(organization)
# pyflakes rate_organization_urls_efficient(organization, create_history=True)
# ratings are always different since we now also save last scan date.
# only creates things for near midnight. Should check if today, and then save for now.
rate_organization_efficient(organization, create_history=True)
......
......@@ -5,7 +5,6 @@ from time import sleep
import pytz
from django.core.management.base import BaseCommand
from failmap_admin.map.determineratings import rerate_url_with_timeline
from failmap_admin.organizations.models import Url
from failmap_admin.scanners.scanner_dns import brute_known_subdomains, certificate_transparency
from failmap_admin.scanners.scanner_http import scan_url_list_standard_ports
......
import ipaddress
import logging
import tldextract
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization
from failmap_admin.scanners.models import Endpoint, TlsQualysScan, Url
from failmap_admin.scanners.models import Endpoint
logger = logging.getLogger(__package__)
......
import ipaddress
import logging
import tldextract
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization
from failmap_admin.scanners.models import Endpoint, TlsQualysScan, Url
from failmap_admin.scanners.models import Endpoint, Url
logger = logging.getLogger(__package__)
......
import ipaddress
import logging
import tldextract
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization
from failmap_admin.scanners.models import Endpoint, TlsQualysScan, TlsQualysScratchpad, Url
from failmap_admin.scanners.models import TlsQualysScan, TlsQualysScratchpad, Url
logger = logging.getLogger(__package__)
......
import ipaddress
import logging
import tldextract
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization
from failmap_admin.scanners.models import Endpoint, TlsQualysScan, TlsQualysScratchpad, Url
from failmap_admin.scanners.models import TlsQualysScratchpad
logger = logging.getLogger(__package__)
......
from django.core.management.base import BaseCommand
from failmap_admin.map.determineratings import DetermineRatings
from failmap_admin.organizations.models import Organization
from failmap_admin.scanners.models import Url
# Only the latest ratings...
class Command(BaseCommand):
......
import datetime
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
......
......@@ -30,6 +30,5 @@ class Command(BaseCommand):
screenshot.url = Url.objects.all().filter(url=url).first()
except ObjectDoesNotExist:
print("No URL exists for screenshot: %s, saving without one." % screenshot)
pass
screenshot.save()
import datetime
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
......
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from failmap_admin.map.determineratings import DetermineRatings
from failmap_admin.organizations.models import Organization
from failmap_admin.scanners.models import Url
from failmap_admin.scanners.scanner_tls_qualys import ScannerTlsQualys, TlsQualysScratchpad
from failmap_admin.scanners.scanner_tls_qualys import ScannerTlsQualys
class Command(BaseCommand):
......
import ipaddress
import logging
import tldextract
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization
from failmap_admin.scanners.models import Endpoint, TlsQualysScan, Url
from failmap_admin.scanners.models import Endpoint, Url
logger = logging.getLogger(__package__)
......
import ipaddress
import logging
import tldextract
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization
from failmap_admin.scanners.models import Endpoint, EndpointGenericScan, TlsQualysScan, Url
from failmap_admin.scanners.models import EndpointGenericScan
logger = logging.getLogger(__package__)
......
import logging
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization, Url
from failmap_admin.scanners.scanner_plain_http import scan_all_urls
logger = logging.getLogger(__package__)
......
import json
import logging
from failmap_admin.app.management.commands._private import TaskCommand
......
......@@ -34,7 +34,7 @@ class Command(BaseCommand):
url = input("Type the url, without protocol:")
url = Url.objects.all().filter(url=url).first()
s = ScannerTlsQualys()
s = scanner_tls_qualys.ScannerTlsQualys()
s.scan([url.url])
rerate_url_with_timeline(url=url)
......
import logging
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from failmap_admin.organizations.models import Organization, Url
from failmap_admin.scanners.scanner_tls import test_real
logger = logging.getLogger(__package__)
......
......@@ -4,9 +4,7 @@ from time import sleep
import pytz
from django.core.management.base import BaseCommand
from django.db.models import Q
from failmap_admin.organizations.models import Url
from failmap_admin.scanners.models import Endpoint
from failmap_admin.scanners.scanner_screenshot import screenshot_endpoint
......@@ -60,4 +58,3 @@ class Command(BaseCommand):
screenshot_endpoint(endpoint)
except TimeoutError:
logger.warning('Took too long to make screenshot of: %s' % endpoint)
pass
......@@ -2,8 +2,6 @@
# Generated by Django 1.10.5 on 2017-02-27 16:17
from __future__ import unicode_literals
from datetime import date, datetime, timedelta
from django.db import migrations, models
......
......@@ -5,9 +5,6 @@ from __future__ import unicode_literals
import django.db.models.deletion
from django.db import migrations, models
from failmap_admin.organizations.models import Url
from failmap_admin.scanners.models import Endpoint
"""
+------+------------------------------------+----------------------------+----------------------------+------+----------+---------+----------------+---------------+
......
......@@ -21,7 +21,7 @@ import subprocess
import untangle
from django.conf import settings
from failmap_admin.organizations.models import Organization, Url
from failmap_admin.organizations.models import Url
from failmap_admin.scanners.scanner_http import resolves
logger = logging.getLogger(__package__)
......
......@@ -229,7 +229,6 @@ def get_ips(url):
except Exception as ex:
# when not known: [Errno 8] nodename nor servname provided, or not known
logger.debug("Get IPv4 error: %s" % ex)
pass
try:
x = socket.getaddrinfo(url, None, socket.AF_INET6)
......@@ -237,7 +236,6 @@ def get_ips(url):
except Exception as ex:
# when not known: [Errno 8nodename nor servname provided, or not known
logger.debug("Get IPv6 error: %s" % ex)
pass
logger.debug("%s has IPv4 address: %s" % (url, ip4))
logger.debug("%s has IPv6 address: %s" % (url, ip6))
......
......@@ -6,12 +6,6 @@ future.
"""
import logging
import subprocess
from datetime import datetime
import pytz
import untangle
from django.core.exceptions import ObjectDoesNotExist
from failmap_admin.organizations.models import Url
from failmap_admin.scanners.endpoint_scan_manager import EndpointScanManager
......
......@@ -18,14 +18,11 @@ import os
import platform
import re
from datetime import datetime
from time import sleep
import pytz
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from PIL import Image
from failmap_admin.organizations.models import Url
from failmap_admin.scanners.models import Endpoint, Screenshot
from failmap_admin.scanners.timeout import timeout
......
......@@ -3,16 +3,12 @@ Check if the https site uses HSTS to tell the browser the site should only be re
(useful until browsers do https by default, instead of by choice)
"""
import logging
import random
import time
from datetime import datetime
from typing import List
import celery
import pytz
import requests
from celery import Celery, Task, group
from celery.task import task
from celery import group
from requests import ConnectionError, ConnectTimeout, HTTPError, ReadTimeout, Timeout
from failmap_admin.celery import ParentFailed, app
......
......@@ -9,7 +9,6 @@ import untangle
from django.conf import settings
from celery_test import app
from failmap_admin.scanners.endpoint_scan_manager import EndpointScanManager
from failmap_admin.scanners.models import Endpoint
from failmap_admin.scanners.timeout import timeout
......@@ -550,6 +549,6 @@ def cert_chain_is_complete(url, port):
:param port:
:return:
"""
tool = settings.TOOLS['TLS']['cert_chain_resolver'][platform.system()]
# pyflakes tool = settings.TOOLS['TLS']['cert_chain_resolver'][platform.system()]
raise NotImplemented
......@@ -3,7 +3,6 @@
import ipaddress
import json
import logging
import sys
from datetime import date, datetime, timedelta
from random import randint
from time import sleep
......@@ -13,10 +12,9 @@ import pytz
import requests
from django.core.exceptions import ObjectDoesNotExist
from failmap_admin.map.determineratings import (rate_organization_efficient, rate_organizations,
rate_url, rerate_url_with_timeline)
from failmap_admin.map.determineratings import rate_organization_efficient, rerate_url_with_timeline
from failmap_admin.organizations.models import Url
from failmap_admin.scanners.models import Endpoint, TlsQualysScan, TlsQualysScratchpad, Url
from failmap_admin.scanners.models import Endpoint, TlsQualysScan, TlsQualysScratchpad
from failmap_admin.scanners.state_manager import StateManager
from ..celery import app
......
"""Import modules containing tasks that need to be auto-discovered by Django Celery."""
from . import scanner_security_headers, scanner_tls_qualys
# explicitly declare the imported modules as this modules 'content', prevents pyflakes issues
__all__ = [scanner_tls_qualys, scanner_security_headers]
"""Integration tests of scanner commands."""
import json
import os
import pytest
from django.core.management import call_command
......@@ -41,7 +40,7 @@ def test_security_headers_notfound(responses, db, faalonië):
call_command('scan-security-headers', '-v3', '-o', NON_EXISTING_ORGANIZATION)
def test_security_headers(responses, db, faalonië):
def test_security_headers_failure(responses, db, faalonië):
"""Test with failing endpoint."""
responses.add(responses.GET, 'https://' + faalonië['url'].url + ':443/', status=500)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment