Commit d8342b76 authored by Thorsten Simons's avatar Thorsten Simons

1.3.1 - added timestamp of first and last record to xlsx file; added SQL...

1.3.1 - added timestamp of first and last record to xlsx file; added SQL function ``tp(size, latency)`` to calculate the throughput; adopted queries to use ``tp()``
parent 3d677b2c
Release History
===============
**1.3.1 2017-10-05**
* added timestamp of first and last record to xlsx file
* added SQL function ``tp(size, latency)`` to calculate the throughput
* adopted queries to use ``tp()``
**1.3.0 2017-10-03**
* some more xlsx luxury
......
......@@ -90,7 +90,10 @@ class DB():
Setup a new database.
"""
self.con.execute("CREATE TABLE admin (magic TEXT, creationdate TEXT)")
self.con.execute("CREATE TABLE admin (magic TEXT,"
" creationdate TEXT,"
" start FLOAT,"
" end FLOAT)")
self.con.commit()
self.con.execute("CREATE TABLE logrecs (node TEXT,\n"
......@@ -106,8 +109,9 @@ class DB():
" latency INT)")
self.con.commit()
self.con.execute("INSERT INTO admin (magic, creationdate)"
" VALUES ('hcprequestanalytics', ?)",
self.con.execute("INSERT INTO admin (magic, creationdate, start, end)"
" VALUES ('hcprequestanalytics', ?,"
" 99999999999.9, 0.0)",
(asctime(),))
self.con.commit()
......@@ -127,7 +131,8 @@ class DB():
# 10.49.253.207 - - [12/Sep/2017:08:03:39 +0200] "OPTIONS /fcfs_data/Symantec_EV/ HTTP/1.1" 403 0 0
cur = self.con.cursor()
# cur.execute('BEGIN IMMEDIATE TRANSACTION')
cur.execute("SELECT * from admin")
_admin = row2dict(cur.fetchone())
count = 0
for l in inhdl.readlines():
......@@ -138,10 +143,11 @@ class DB():
if len(rec) == 11:
rec.insert(10, 'Default.Default')
_ts = rec[3][1:] + rec[4][:-1]
_tsnum = mktime(strptime(_ts,'%d/%b/%Y:%H:%M:%S%z'))
_r = {'node': node,
'clientip': rec[0],
'user': rec[2],
'timestamp': mktime(strptime(_ts,'%d/%b/%Y:%H:%M:%S%z')),
'timestamp': _tsnum,
'timestampstr': _ts,
'request': rec[5][1:],
'path': rec[6],
......@@ -150,6 +156,8 @@ class DB():
'namespace': rec[10],
'latency': int(rec[11])
}
_admin['start'] = _tsnum if _tsnum < _admin['start'] else _admin['start']
_admin['end'] = _tsnum if _tsnum > _admin['end'] else _admin['end']
except IndexError as e:
print('IndexError on {} \n\t- {}'.format(rec, e))
continue
......@@ -165,8 +173,19 @@ class DB():
_r)
self.con.commit()
cur.execute('UPDATE admin SET start = :start, end = :end', _admin)
self.con.commit()
return count
def gettimestamps(self):
"""
Get the timestamps of the first and last entry in the database.
"""
cur = self.con.execute("SELECT start, end from admin")
_admin = cur.fetchone()
return _admin['start'], _admin['end']
def loadqueries(self, aq=None):
"""
Load the built-in (and evetually, additional queries)
......@@ -253,8 +272,8 @@ class DB():
_csv.closesheet(self.queries.c.get(mps[fu], 'freeze pane',
fallback=''))
_csv.close()
_start, _end = self.gettimestamps()
_csv.close(start=_start, end=_end)
def close(self):
"""
......@@ -290,6 +309,7 @@ def runquery(db, qtitle, query):
con = sqlite3.connect(db)
con.row_factory = sqlite3.Row
con.create_aggregate("percentile", 2, PercentileFunc)
con.create_function("tp", 2, tpfunc)
cur = con.cursor()
cur.execute(query)
......@@ -323,3 +343,16 @@ class PercentileFunc():
self.list.sort()
return self.list[
int(round((len(self.list) - 1) * self.percent / 100.0))]
def tpfunc(size, duration):
"""
Calculate the throughput from size and time.
:param size: object size
:param duration: the time the transfer took
:return: the throughput in Bytes/sec
"""
duration = duration if duration > 0.0001 else 0.0001
ret = size / duration * 1000
# print('size={} / latency={} = {} KB/sec'.format(size, duration, ret))
return ret
......@@ -167,7 +167,7 @@ freeze pane : E5
[500_largest]
comment : The records with the 500 largest requests
query : SELECT request, httpcode, node, latency, size,
size/(latency/1000)/1024 as 'KB/sec', clientip, user,
tp(size,latency) as 'Bytes/sec', clientip, user,
timestamp, timestampstr, path, namespace
FROM (SELECT * FROM logrecs ORDER BY size DESC LIMIT 500)
ORDER BY request, httpcode, node
......@@ -215,29 +215,29 @@ query : SELECT request, count(*),
freeze pane : C5
[500_highest_throughput]
comment : The 500 records with the highest throughput (KB/sec) for objects >= 1 Byte
comment : The 500 records with the highest throughput (Bytes/sec) for objects >= 1 Byte
query : SELECT * from
(select request, node, clientip, httpcode,
size/(latency/1000)/1024 as 'KB/sec', size,
tp(size, latency) as 'Bytes/sec', size,
latency from logrecs where size >= 1)
order by 'KB/sec' desc limit 500;
order by 'Bytes/sec' desc limit 500;
freeze pane : E5
[percentile_throughput_kb]
comment : No. of records per request, with percentiles on throughput (KB/sec) for objects >= 10MB
query : SELECT request,
count(*), min(size), avg(size), max(size),
percentile(size/(latency/1000)/1024, 10) as 'pctl-10 (KB/sec)',
percentile(size/(latency/1000)/1024, 20) as 'pctl-20 (KB/sec)',
percentile(size/(latency/1000)/1024, 30) as 'pctl-30 (KB/sec)',
percentile(size/(latency/1000)/1024, 40) as 'pctl-40 (KB/sec)',
percentile(size/(latency/1000)/1024, 50) as 'pctl-50 (KB/sec)',
percentile(size/(latency/1000)/1024, 60) as 'pctl-60 (KB/sec)',
percentile(size/(latency/1000)/1024, 70) as 'pctl-70 (KB/sec)',
percentile(size/(latency/1000)/1024, 80) as 'pctl-80 (KB/sec)',
percentile(size/(latency/1000)/1024, 90) as 'pctl-90 (KB/sec)',
percentile(size/(latency/1000)/1024, 95) as 'pctl-95 (KB/sec)',
percentile(size/(latency/1000)/1024, 99) as 'pctl-99 (KB/sec)',
percentile(size/(latency/1000)/1024, 99.9) as 'pctl-99.9 (KB/sec)'
percentile(tp(size, latency)), 10) as 'pctl-10 (KB/sec)',
percentile(tp(size, latency))/1024, 20) as 'pctl-20 (B/sec)',
percentile(tp(size, latency))/1024, 30) as 'pctl-30 (B/sec)',
percentile(tp(size, latency))/1024, 40) as 'pctl-40 (B/sec)',
percentile(tp(size, latency))/1024, 50) as 'pctl-50 (B/sec)',
percentile(tp(size, latency))/1024, 60) as 'pctl-60 (B/sec)',
percentile(tp(size, latency))/1024, 70) as 'pctl-70 (B/sec)',
percentile(tp(size, latency))/1024, 80) as 'pctl-80 (B/sec)',
percentile(tp(size, latency))/1024, 90) as 'pctl-90 (B/sec)',
percentile(tp(size, latency))/1024, 95) as 'pctl-95 (B/sec)',
percentile(tp(size, latency))/1024, 99) as 'pctl-99 (B/sec)',
percentile(tp(size, latency))/1024, 99.9) as 'pctl-99.9 (B/sec)'
FROM logrecs where size >= 10048576 and latency > 500 GROUP BY request
freeze pane : C5
......@@ -22,8 +22,8 @@
import sys
import csv
import time
import xlsxwriter
from time import asctime, localtime, strftime
class Csv(object):
......@@ -69,7 +69,7 @@ class Csv(object):
"""
self.hdl.close()
def close(self):
def close(self, **kwargs):
"""
Close the object.
"""
......@@ -174,9 +174,12 @@ class Xlsx(Csv):
self.ws.freeze_panes(fp)
return
def close(self):
def close(self, start='', end=''):
"""
Create a Content sheez and close the workbook.
:param start: the timestamp of the first rec in the DB
:param end: the timestamp of the last rec in the DB
"""
row = 3
col = 1
......@@ -219,9 +222,17 @@ class Xlsx(Csv):
# footer
self.contentws.merge_range(row+2, 1, row+2, 3,
'created {}'.format(time.asctime()),
'created {} from log records starting at {}, ending at {}'
.format(asctime(),
strftime('%a %b %d %H:%M:%S %Y',
localtime(start)),
strftime('%a %b %d %H:%M:%S %Y',
localtime(end))),
footer)
# Thu Oct 5 20:48:16 2017
# '%a %b %d %H:%M:%S %Y'
# make this the visible sheet on workbook open
self.contentws.set_first_sheet()
self.contentws.activate()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment