Commit 0847522c authored by Thorsten Simons's avatar Thorsten Simons

1.5.1 - replaced *shutil.unpack_archive* with *zipfile.Zipfile extractall*, as...

1.5.1 - replaced *shutil.unpack_archive* with *zipfile.Zipfile extractall*, as *unpack_archive* seems to have issues with zip- file members > 2 GB.
parent bc3ece23
Release History
===============
**1.5.1 2019-03-19**
* replaced *shutil.unpack_archive* with *zipfile.Zipfile.extractall*, as *unpack_archive* seems to
have issues with zip- file members > 2 GB.
**1.5.0 2019-01-24**
* added a table for MAPI-related logs to the database, as well as queries
specially tailored for MAPI
**1.4.5 2019-01-23**
* added a query that list users accessing HCP
......
......@@ -23,7 +23,6 @@
import sys
from time import time
from os.path import exists, join, dirname
from traceback import print_exc
try:
from setproctitle import setproctitle
except ImportError:
......@@ -99,7 +98,7 @@ def main():
node = infile.split('/')[-6]
print('\treading node {} - {} '.format(node,infile), end='')
with open(infile, 'r') as inhdl:
_cnt = db.loaddb(inhdl, node)
_cnt = db.loaddb(inhdl, node, infile.split('/')[-1].startswith('mapi'))
print('- {:,} records'.format(_cnt))
_logrecs += _cnt
print('loading database with {:,} records took {:.3f} seconds'
......
......@@ -115,6 +115,19 @@ class DB():
" latency INT)")
self.con.commit()
self.con.execute("CREATE TABLE mapirecs (node TEXT,\n"
" clientip TEXT,\n"
" user TEXT,\n"
" timestamp FLOAT,\n"
" timestampstr TEXT,\n"
" request TEXT,\n"
" path TEXT,\n"
" httpcode INT,\n"
" size INT,\n"
" namespace TEXT,\n"
" latency INT)")
self.con.commit()
self.con.execute("INSERT INTO admin (magic, creationdate, start, end)"
" VALUES ('hcprequestanalytics', ?,"
" 99999999999.9, 0.0)",
......@@ -123,11 +136,13 @@ class DB():
def loaddb(self, inhdl, node):
def loaddb(self, inhdl, node, mapi=False):
"""
Load the database with the records from a single file
:param inhdl: a filehandle for an open accesslog file
:param node: the node being worked on
:param mapi: True if the logfile worked on is a mapi_request logfile
:return: the no. of read records
"""
......@@ -183,13 +198,14 @@ class DB():
continue
count += 1
cur.execute('INSERT INTO logrecs'
cur.execute('INSERT INTO {}'
' (node, clientip, user, timestamp, timestampstr,'
' request, path, httpcode, size, namespace, '
' latency) VALUES'
' (:node, :clientip, :user, :timestamp,'
' :timestampstr, :request, :path, :httpcode,'
' :size, :namespace, :latency)',
' :size, :namespace, :latency)'
.format('logrecs' if not mapi else 'mapirecs'),
_r)
self.con.commit()
......@@ -345,6 +361,7 @@ def runquery(db, qtitle, query):
con.create_function("getNamespace", 2, getNamespace)
con.create_function("getTenant", 1, getTenant)
con.create_function("getProtocol", 1, getProtocol)
con.create_function("getMapiPath", 1, getMapiPath)
cur = con.cursor()
cur.execute(query)
......@@ -466,3 +483,12 @@ def getProtocol(field):
except IndexError:
return 'native REST'
def getMapiPath(field):
"""
Remove the paramaters from a log records path field.
:param field: the mentioned log record's namespace field
:return: the path w/o parameters
"""
return field.split('?')[0]
......@@ -20,10 +20,13 @@
from tempfile import TemporaryDirectory
from shutil import unpack_archive
from zipfile import ZipFile
from os import listdir, walk
from os.path import join
from ipaddress import ip_address
from pprint import pprint
class Handler(object):
"""
......@@ -53,7 +56,8 @@ class Handler(object):
# 179.tar.bz2
# manifest.csv
print('un-packing {}'.format(self.logpkg))
unpack_archive(self.logpkg, self.tmpdir.name, format='zip')
_z = ZipFile(self.logpkg)
_z.extractall(path=self.tmpdir.name)
# find the folder where the file was uncompressed to
archdir = ''
......@@ -97,7 +101,8 @@ class Handler(object):
for root, dirs, files in walk(self.tmpdir.name):
for f in files:
if f.startswith('http_gateway_request.log'):
if (f.startswith('http_gateway_request.log') or
f.startswith('mapi_gateway_request.log')):
flist.append(join(root, f))
return flist
......
......@@ -360,3 +360,19 @@ query : SELECT getTenant(namespace) as Tenant, user,
request, httpcode
Freeze pane : C5
[mapi_endp_req_http]
comment : MAPI request: endpoints, request, http code
query : SELECT getMapiPath(path) as endpoint, request, httpcode,
count(*) as count,
min(latency), avg(latency), max(latency)
FROM mapirecs GROUP BY getMapiPath(path), request, node, httpcode
Freeze pane : A5
[mapi_user_req_http]
comment : MAPI requests by user
query : SELECT user, getMapiPath(path) as endpoint, request, node, httpcode,
count(*) as count,
min(latency), avg(latency), max(latency)
FROM mapirecs GROUP BY user, getMapiPath(path), request, node, httpcode
Freeze pane : A5
......@@ -200,7 +200,7 @@ class Xlsx(Csv):
def close(self, start='', end=''):
"""
Create a Content sheez and close the workbook.
Create a Content sheet and close the workbook.
:param start: the timestamp of the first rec in the DB
:param end: the timestamp of the last rec in the DB
......
......@@ -46,6 +46,7 @@ setup(
version=str(Gvars.s_version),
description=Gvars.Description,
long_description=long_description,
long_description_content_type = 'text/markdown',
# The project's main homepage.
url='https://gitlab.com/simont3/hcprequestanalytics',
......
......@@ -27,8 +27,8 @@ class Gvars:
"""
# version control
s_version = "1.4.5"
s_builddate = '2019-01-23'
s_version = "1.5.1"
s_builddate = '2019-03-19'
s_build = "{}/Sm".format(s_builddate)
s_minPython = "3.5"
s_description = "hcprequestanalytics"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment