Commit 211e376f authored by Thorsten Simons's avatar Thorsten Simons

1.5.4 - added the option to analyze S-node logs from a **hcphealth** database,...

1.5.4 - added the option to analyze S-node logs from a **hcphealth** database, added *MQE* related queries, matured the database functions to withstand incorrect values in numerical fields
parent 7ef2541e
Release History
===============
**1.5.4 2019-06-05**
* added the option to analyze S-node logs from a **hcphealth** database
* added *MQE* related queries
* matured the database functions to withstand incorrect values in numerical fields
**1.5.3 2019-05-21**
* added the *clientip_node* query
......
......@@ -85,6 +85,7 @@ csv files can be requested as well.
-a ADDITIONALQUERIES a file containg addition queries (see documentation)
-d DB the database file
-p PREFIX prefix for the output files
-s analyze requests recorded by snodes
-c create CSV files instead of a XLSX file
--procs PROCESSES no. of subprocesses to run, defaults to no. of CPUs
......
......@@ -60,6 +60,10 @@ def parseargs():
default=False, action='store_true',
help='create CSV files instead of a XLSX file')
analyzep.add_argument('-s', dest='snode', required=False,
default=False, action='store_true',
help='analyze requests recorded by snodes')
analyzep.add_argument('--procs', dest='processes', required=False,
default=None, type=int,
help='no. of subprocesses to run, defaults to no. '
......
......@@ -33,10 +33,11 @@ from hcpreq.db import DB
from hcpreq.logs import Handler
def opendb(db, addqueries):
def opendb(snode, db, addqueries):
"""
Open and check the database, load the queries.
:param snode: True if working on snode logs
:param db: the database file's name
:param addqueries: the name of a file w/ additional queries
:return: a database object
......@@ -44,7 +45,7 @@ def opendb(db, addqueries):
db = DB(db)
db.opendb()
db.checkdb()
db.loadqueries(aq=addqueries)
db.loadqueries(snode, aq=addqueries)
return db
......@@ -111,7 +112,7 @@ def main():
elif opts.cmd == 'analyze':
if setproctitle:
setproctitle('hcprequestanalytics is analyzing {}'.format(opts.db))
db = opendb(opts.db, opts.additionalqueries)
db = opendb(opts.snode, opts.db, opts.additionalqueries)
try:
_st = time()
db.mpanalyze(opts.prefix, queries=opts.queries, csvtype=opts.csv,
......
......@@ -223,16 +223,24 @@ class DB():
_admin = cur.fetchone()
return _admin['start'], _admin['end']
def loadqueries(self, aq=None):
def loadqueries(self, snode=False, aq=None):
"""
Load the built-in (and evetually, additional queries)
:param aq: the name of a file containing additional queries
:param snode: True if working on snode logs
:param aq: the name of a file containing additional queries
"""
# make sure we find our standard queries even when frozen
if getattr(sys, 'frozen', False):
_stdq = join(sys._MEIPASS, 'hcpreq/queries')
if not snode:
_stdq = join(sys._MEIPASS, 'hcpreq/queries')
else:
_stdq = join(sys._MEIPASS, 'hcpreq/s_queries')
else:
_stdq = join(dirname(__file__), 'queries')
if not snode:
_stdq = join(dirname(__file__), 'queries')
else:
_stdq = join(dirname(__file__), 's_queries')
if aq:
_files = [_stdq, aq]
......@@ -402,7 +410,8 @@ class PercentileFunc():
self.percent = None
def step(self, value, percent):
if value is None:
# if value is None:
if type(value) not in [int, float]:
return
if self.percent is None:
self.percent = percent
......@@ -424,7 +433,13 @@ def tpfunc(size, duration):
:param duration: the time the transfer took
:return: the throughput in Bytes/sec
"""
duration = duration if duration > 0.0001 else 0.0001
if type(duration) not in [int, float]:
duration = 0.0001
else:
duration = duration if duration > 0.0001 else 0.0001
if type(size) != int:
size = 0
ret = size / duration * 1000
# print('size={} / latency={} = {} KB/sec'.format(size, duration, ret))
......
......@@ -26,6 +26,18 @@ comment : No. of records, overall
query : SELECT count(*) as count FROM logrecs
freeze pane : A5
[count_mqequeries]
comment : No. of records to the MQE, overall
query : SELECT count(*) as count
FROM logrecs WHERE path GLOB '/query*';
freeze pane : A5
[clientip_mqequeries]
comment : No. of records to the MQE, per client IP address
query : SELECT clientip, count(*) as count
FROM logrecs where path glob '/query*' GROUP BY clientip;
freeze pane : A5
[clientip]
comment : No. of records per client IP address
query : SELECT clientip, count(*) as count,
......
# The MIT License (MIT)
#
# Copyright (c) 2017-2019 Thorsten Simons (sw@snomis.eu)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
[count]
comment : No. of records, overall
query : SELECT count(*) as count
FROM logrecs_s
WHERE length(node) > 3
freeze pane : A5
[clientip]
comment : No. of records per client IP address
query : SELECT clientip, count(*) as count,
min(size), avg(size), max(size),
min(latency), avg(latency),
max(latency)
FROM logrecs_s
WHERE length(node) > 3
GROUP BY clientip
freeze pane : C5
[clientip_httpcode]
comment : No. of records per http code per client IP address
query : SELECT clientip, httpcode, count(*) as count,
min(size), avg(size), max(size),
min(latency), avg(latency),
max(latency)
FROM logrecs_s
WHERE length(node) > 3
GROUP BY clientip, httpcode
freeze pane : D5
[node]
comment : No. of records per node
query : SELECT node, count(*) as count,
min(size), avg(size), max(size),
min(latency), avg(latency),
max(latency)
FROM logrecs_s
WHERE length(node) > 3
GROUP BY node
freeze pane : C5
[500_largest_req_httpcode_node]
comment : The records with the 500 largest requests by req, httpcode, node
query : SELECT request, httpcode, node, latency, size,
tp(size,latency) as Bytes_sec, clientip,
timestamp, timestampstr, path, namespace
FROM (SELECT * FROM logrecs_s
WHERE length(node) > 3
ORDER BY size DESC LIMIT 500)
ORDER BY request, httpcode, node
freeze pane : D5
[500_worst_latency]
comment : The records with the 500 worst latencies
query : SELECT request, httpcode, latency, size, tp(size,latency) as Bytes_sec,
clientip, timestamp, timestampstr, path, namespace
FROM (SELECT * FROM logrecs_s
WHERE length(node) > 3 and size > 0
ORDER BY latency DESC LIMIT 500)
ORDER BY request, httpcode
freeze pane : C5
[percentile_req]
comment : No. of records per request analysis, including percentiles for size and latency
query : SELECT request, count(*) as count,
min(size), avg(size), max(size),
percentile(size, 10) as 'pctl-10 (size)',
percentile(size, 20) as 'pctl-20 (size)',
percentile(size, 30) as 'pctl-30 (size)',
percentile(size, 40) as 'pctl-40 (size)',
percentile(size, 50) as 'pctl-50 (size)',
percentile(size, 60) as 'pctl-60 (size)',
percentile(size, 70) as 'pctl-70 (size)',
percentile(size, 80) as 'pctl-80 (size)',
percentile(size, 90) as 'pctl-90 (size)',
percentile(size, 95) as 'pctl-95 (size)',
percentile(size, 99) as 'pctl-99 (size)',
percentile(size, 99.9) as 'pctl-99.9 (size)',
min(latency), avg(latency),
max(latency),
percentile(latency, 10) as 'pctl-10 (latency)',
percentile(latency, 20) as 'pctl-20 (latency)',
percentile(latency, 30) as 'pctl-30 (latency)',
percentile(latency, 40) as 'pctl-40 (latency)',
percentile(latency, 50) as 'pctl-50 (latency)',
percentile(latency, 60) as 'pctl-60 (latency)',
percentile(latency, 70) as 'pctl-70 (latency)',
percentile(latency, 80) as 'pctl-80 (latency)',
percentile(latency, 90) as 'pctl-90 (latency)',
percentile(latency, 95) as 'pctl-95 (latency)',
percentile(latency, 99) as 'pctl-99 (latency)',
percentile(latency, 99.9) as 'pctl-99.9 (latency)'
FROM logrecs_s
WHERE length(node) > 3
GROUP BY request
freeze pane : C5
[500_highest_throughput]
comment : The 500 records with the highest throughput (Bytes/sec)
query : SELECT request, httpcode, clientip, tp(size, latency) as Bytes_sec,
latency, size, timestamp, timestampstr, path, namespace
FROM logrecs_s
WHERE length(node) > 3 and size > 0 and latency > 0
ORDER BY Bytes_sec DESC
LIMIT 500;
freeze pane : D5
......@@ -17,7 +17,8 @@ block_cipher = None
a = Analysis(['hcprequestanalytics.py'],
pathex=_pathext,
binaries=[],
datas=[('hcpreq/queries', './hcpreq')],
datas=[('hcpreq/queries', './hcpreq'),
('hcpreq/s_queries', './hcpreq')],
hiddenimports=[],
hookspath=[],
runtime_hooks=[],
......
......@@ -27,8 +27,8 @@ class Gvars:
"""
# version control
s_version = "1.5.3"
s_builddate = '2019-05-21'
s_version = "1.5.4"
s_builddate = '2019-06-05'
s_build = "{}/Sm".format(s_builddate)
s_minPython = "3.5"
s_description = "hcprequestanalytics"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment