Skip to content

GitLab

Commit 3f43437d authored by Heinz N. Gies's avatar Heinz N. Gies
Browse files

HPTS support

parent 8bff6ec7
Pipeline #18459145 passed with stage
in 1 minute and 16 seconds
Hide whitespace changes
Inline Side-by-side
Showing with 281 additions and 242 deletions
apps/dalmatiner_frontend/priv/static/bucket.html
View file @ 3f43437d
<!doctype html>
<html class="no-js" lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>DalmatinerDB | Buckets</title>
<link rel="stylesheet" href="/css/foundation.css" />
<script src="/js/vendor/modernizr.js"></script>
<script src="/js/vendor/jquery.js"></script>
<script src="/js/vendor/jquery.cookie.js"></script>
<script src="/js/vendor/fastclick.js"></script>
<script src="/js/foundation.min.js"></script>
<script src="/js/msgpack.base.js"></script>
<script src="/js/msgpack.codec.js"></script>
<script src="/js/msgpack.js"></script>
<script src="/js/bucket.js"></script>
</head>
<body>
<nav class="top-bar" data-topbar>
<ul class="title-area">
<li class="name">
<h1><a href="#">Dalmatiner DB</a></h1>
</li>
<!-- Remove the class "menu-icon" to get rid of menu icon. Take out "Menu" to just have icon alone -->
<li class="toggle-topbar menu-icon"><a href="#"><span>Menu</span></a></li>
</ul>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>DalmatinerDB | Buckets</title>
<link rel="stylesheet" href="/css/foundation.css" />
<script src="/js/vendor/modernizr.js"></script>
<script src="/js/vendor/jquery.js"></script>
<script src="/js/vendor/jquery.cookie.js"></script>
<script src="/js/vendor/fastclick.js"></script>
<script src="/js/foundation.min.js"></script>
<script src="/js/msgpack.base.js"></script>
<script src="/js/msgpack.codec.js"></script>
<script src="/js/msgpack.js"></script>
<script src="/js/bucket.js"></script>
</head>
<body>
<nav class="top-bar" data-topbar>
<ul class="title-area">
<li class="name">
<h1><a href="#">Dalmatiner DB</a></h1>
</li>
<!-- Remove the class "menu-icon" to get rid of menu icon. Take out "Menu" to just have icon alone -->
<li class="toggle-topbar menu-icon"><a href="#"><span>Menu</span></a></li>
</ul>
<section class="top-bar-section">
<!-- Left Nav Section -->
<ul class="left tabs" data-options="deep_linking:true" data-tab>
<li><a href="/">Query</a></li>
<li><a href="/events">Events</a></li>
<li><a href="/buckets">Buckets</a></li>
<li><a href="/functions">Functions</a></li>
</ul>
</section>
</nav>
<div class="row">
<div class="large-12 columns">
<table>
<thead>
<tr>
<th>Bucket</th>
<th>Resolution</th>
<th>PPF</th>
<th>Grace</th>
<th>TTL</th>
</tr>
</thead>
<tbody id="buckets">
</tbody>
</table>
</div>
</div>
<section class="top-bar-section">
<!-- Left Nav Section -->
<ul class="left tabs" data-options="deep_linking:true" data-tab>
<li><a href="/">Query</a></li>
<li><a href="/events">Events</a></li>
<li><a href="/buckets">Buckets</a></li>
<li><a href="/functions">Functions</a></li>
</ul>
</section>
</nav>
<div class="row">
<div class="large-12 columns">
<table>
<thead>
<tr>
<th>Bucket</th>
<th>Resolution</th>
<th>PPF</th>
<th>Grace</th>
<th>TTL</th>
<th>HPTS</th>
</tr>
</thead>
<tbody id="buckets">
</tbody>
</table>
</div>
</div>
</body>
</html>
apps/dalmatiner_frontend/priv/static/js/bucket.js
View file @ 3f43437d
$(function() {
msgpack.download("", {header: {accept:"application/x-msgpack"}}, function(d) {
console.log(d)
console.log("buckets:", d)
d.forEach(function(e) {
$("#buckets").append("<tr>"+
"<td><a href='/buckets/"+ e.name + "'>" + e.name +"</a></td>"+
"<td>" + e.resolution + "ms</td>"+
"<td>" + e.ppf + "</td>"+
"<td>" + e.grace + "ns</td>"+
"<td>" + e.ttl + "</td>"+
"<td>" + e.resolution + "ms</td>" +
"<td>" + e.ppf + "</td>" +
"<td>" + e.grace + "ns</td>" +
"<td>" + e.ttl + "</td>" +
"<td>" + e.hpts + "</td>" +
"</tr>")
})
})
......
apps/dalmatiner_frontend/priv/static/js/query.js
View file @ 3f43437d
......@@ -5,166 +5,178 @@ $("#timewrap").hide();
// If browser supports history pushStete, avoid reloading page on query change
if (typeof(history) == "object" && typeof(history.pushState) == "function") {
$("#queryfrom").on("submit", function(e) {
var query = $("#query").val();
history.pushState({query: query}, document.title, "/?query=" + encodeURIComponent(query));
e.preventDefault();
q();
});
$(window).on("popstate", function(e) {
var m = window.location.search.match(/(\?|&)query=(.*)(&|$)/);
var query = m ? decodeURIComponent(m[2]) : "";
if (query) {
$("#query").val(query);
q();
} else {
window.location.reload();
}
});
$("#queryfrom").on("submit", function(e) {
var query = $("#query").val();
history.pushState({query: query}, document.title, "/?query=" + encodeURIComponent(query));
e.preventDefault();
q();
});
$(window).on("popstate", function(e) {
var m = window.location.search.match(/(\?|&)query=(.*)(&|$)/);
var query = m ? decodeURIComponent(m[2]) : "";
if (query) {
$("#query").val(query);
q();
} else {
window.location.reload();
}
});
}
var QueryString = function () {
// This function is anonymous, is executed immediately and
// the return value is assigned to QueryString!
var query_string = {};
var query = window.location.search.substring(1);
var vars = query.split("&");
for (var i=0;i<vars.length;i++) {
var pair = vars[i].split("=");
// If first entry with this name
if (typeof query_string[pair[0]] === "undefined") {
query_string[pair[0]] = pair[1];
// If second entry with this name
} else if (typeof query_string[pair[0]] === "string") {
var arr = [ query_string[pair[0]], pair[1] ];
query_string[pair[0]] = arr;
// If third or later entry with this name
} else {
query_string[pair[0]].push(pair[1]);
}
}
return query_string;
}();
// This function is anonymous, is executed immediately and
// the return value is assigned to QueryString!
var query_string = {};
var query = window.location.search.substring(1);
var vars = query.split("&");
for (var i=0;i<vars.length;i++) {
var pair = vars[i].split("=");
// If first entry with this name
if (typeof query_string[pair[0]] === "undefined") {
query_string[pair[0]] = pair[1];
// If second entry with this name
} else if (typeof query_string[pair[0]] === "string") {
var arr = [ query_string[pair[0]], pair[1] ];
query_string[pair[0]] = arr;
// If third or later entry with this name
} else {
query_string[pair[0]].push(pair[1]);
}
}
return query_string;
}();
if (QueryString.metric && QueryString.bucket) {
var metric = decodeURIComponent(QueryString.metric);
var bucket = decodeURIComponent(QueryString.bucket);
if (! bucket.match(/^'.*'$/)) {
bucket = "'" + bucket + "'";
}
$("#query").val("SELECT " + metric + " BUCKET " + bucket + " LAST 60s");
q();
var metric = decodeURIComponent(QueryString.metric);
var bucket = decodeURIComponent(QueryString.bucket);
if (! bucket.match(/^'.*'$/)) {
bucket = "'" + bucket + "'";
}
$("#query").val("SELECT " + metric + " BUCKET " + bucket + " LAST 60s");
q();
} else if (QueryString.query) {
var query = decodeURIComponent(QueryString.query).replace(/\+/g, ' ');
$("#query").val(query);
q();
var query = decodeURIComponent(QueryString.query).replace(/\+/g, ' ');
$("#query").val(query);
q();
}
function render_metrics(start, metrics, markers) {
var data,
legend = [];
$("#results").text("");
if (metrics.length > 0) {
$("#results").append($("<h3></h3>").text("Metrics")).append($("<hr/>"))
data = metrics.map(function(s) {
var resolution = s.resolution,
values = s.values,
points = new Array(values.length);
legend.push(s.name);
for (var i = 0; i < values.length; i++) {
points[i] = {
"date": new Date((start + (i * resolution)) / 1000) ,
"value": values[i]
};
}
return points;
});
var data,
legend = [];
$("#results").text("");
if (metrics.length > 0) {
$("#results").append($("<h3></h3>").text("Metrics")).append($("<hr/>"));
data = metrics.map(function(s) {
var resolution = s.resolution,
values = s.values;
legend.push(s.name);
if (s.hpts) {
var points = new Array();
j = 0;
for (var i = 0; i < values.length; i++) {
if (values[i].date > 0) {
points[j] = values[i];
points[j].date = new Date(values[i].date / (1000 * 1000));
j++;
}
}
return points;
} else {
var points = new Array(values.length);
for (var i = 0; i < values.length; i++) {
points[i] = {
"date": new Date((start + (i * resolution)) / 1000) ,
"value": values[i]
};
}
return points;
}
});
MG.data_graphic({
data: data,
height: 400,
markers: markers,
full_width: true,
target: '#results',
legend: legend,
max_data_size: 11,
legend_target: '#legend',
missing_is_hidden: true,
aggregate_rollover: true
});
}
MG.data_graphic({
data: data,
height: 400,
markers: markers,
full_width: true,
target: '#results',
legend: legend,
max_data_size: 11,
legend_target: '#legend',
missing_is_hidden: true,
aggregate_rollover: true
});
}
}
function render_event(event) {
var div = $("<div class='events'></div>"),
hdr = $("<h5></h5>").text(event.name),
tbl = $("<table></table>"),
th = $("<thead><tr><th>Date</th><th>Event</th></tr></thead>"),
tb = $("<tbody></tbody>");
event.values.map(function (e) {
var tr = $("<tr></tr>"),
date = new Date(e.timestamp/1000/1000);
tr.append($("<td></td>").text(date))
.append($("<td></td>").append($("<pre></pre>").text(JSON.stringify(e.event, null, 2))));
tb.append(tr)
});
tbl.append(th)
tbl.append(tb)
div.append(hdr) .append(tbl)
$("#events").append(div)
var div = $("<div class='events'></div>"),
hdr = $("<h5></h5>").text(event.name),
tbl = $("<table></table>"),
th = $("<thead><tr><th>Date</th><th>Event</th></tr></thead>"),
tb = $("<tbody></tbody>");
event.values.map(function (e) {
var tr = $("<tr></tr>"),
date = new Date(e.timestamp/1000/1000);
tr.append($("<td></td>").text(date))
.append($("<td></td>").append($("<pre></pre>").text(JSON.stringify(e.event, null, 2))));
tb.append(tr);
});
tbl.append(th);
tbl.append(tb);
div.append(hdr) .append(tbl);
$("#events").append(div);
}
function render_events(start, events) {
$("#events").text("");
if (events.length > 0) {
$("#events").append($("<h3></h3>").text("Events")).append($("<hr/>"))
events.map(render_event)
}
$("#events").text("");
if (events.length > 0) {
$("#events").append($("<h3></h3>").text("Events")).append($("<hr/>"));
events.map(render_event);
}
}
function render_graph(graph) {
$("#graph").text("")
if (graph) {
$("#graph")
.append($("<h3></h3>").text("Query graph"))
.append($("<pre/>")
.append(Viz(graph, {scale: 0.1})))
}
$("#graph").text("");
if (graph) {
$("#graph")
.append($("<h3></h3>").text("Query graph"))
.append($("<pre/>")
.append(Viz(graph, {scale: 0.1})));
}
}
function q() {
var query = $("#query").val();
var base = "?"
$("#events").text("").append($("<hr/>"))
if ($("#debug").is(':checked')) {
base = "?graph&q=";
} else {
base = "?q=";
}
msgpack.download(base + query, {header: {accept:"application/x-msgpack"}}, function(res) {
$("#permalink").attr("href", "/?query=" + encodeURIComponent(query));
$("#permalink").show();
$("#time").text((res.query_time / 1000) + "ms");
$("#timewrap").show();
var start = res.start * 1000,
markers = [];
var query = $("#query").val();
var base = "?";
$("#events").text("").append($("<hr/>"));
if ($("#debug").is(':checked')) {
base = "?graph&q=";
} else {
base = "?q=";
}
msgpack.download(base + query, {header: {accept:"application/x-msgpack"}}, function(res) {
console.log("results:", res);
$("#permalink").attr("href", "/?query=" + encodeURIComponent(query));
$("#permalink").show();
$("#time").text((res.query_time / 1000) + "ms");
$("#timewrap").show();
gres = res
metrics = res.results.filter(function(e) {
return e.type == "metrics"
});
console.log(metrics[0])
console.log("Fetched " + metrics[0].values.length * metrics.length + " elements in " +
res.query_time / 1000 + "ms");
events = res.results.filter(function(e) {
return e.type == "events"
});
render_metrics(start, metrics, markers);
render_events(start, events);
render_graph(res.graph)
var start = res.start * 1000,
markers = [];
});
gres = res;
metrics = res.results.filter(function(e) {
return e.type == "metrics";
});
console.log(metrics[0]);
console.log("Fetched " + metrics[0].values.length * metrics.length + " elements in " +
res.query_time / 1000 + "ms");
events = res.results.filter(function(e) {
return e.type == "events";
});
render_metrics(start, metrics, markers);
render_events(start, events);
render_graph(res.graph);
});
}
apps/dalmatiner_frontend/src/dalmatiner_idx_handler.erl
View file @ 3f43437d
......@@ -53,17 +53,34 @@ handle(Req, State) ->
end.
encode_reply(Start, T, R2) ->
R3 = [#{name => Name,
resolution => Resolution,
values => mmath_bin:to_list(Data),
metadata => maps:map(fun to_null/2, Mdata),
type => <<"metrics">>}
R3 = [#{
name => Name,
resolution => Resolution,
values => mmath_bin:to_list(Data),
metadata => maps:map(fun to_null/2, Mdata),
type => <<"metrics">>,
hpts => false
}
|| #{name := Name,
data := Data,
type := metrics,
metadata := Mdata,
resolution := Resolution} <- R2],
R4 = [#{name => Name,
R4 = [#{
name => Name,
resolution => Resolution,
values => [#{date => D, value => V}
|| {D, V} <- mmath_hpts:to_list(Data)],
metadata => maps:map(fun to_null/2, Mdata),
type => <<"metrics">>,
hpts => true
}
|| #{name := Name,
data := Data,
type := hpts,
metadata := Mdata,
resolution := Resolution} <- R2],
R5 = [#{name => Name,
metadata => maps:map(fun to_null/2, Mdata),
values => [#{timestamp => Ts, event => E}
|| {Ts, E} <- Data],
......@@ -74,7 +91,7 @@ encode_reply(Start, T, R2) ->
type := events} <- R2],
D = #{start => Start,
query_time => T,
results => R3 ++ R4},
results => R3 ++ R4 ++ R5},
case R2 of
[#{type := graph,
value := Graph} | _] ->
......
rebar.config
View file @ 3f43437d
......@@ -16,7 +16,7 @@
{jsone, "~>1.2.3"},
fifo_utils,
{msgpack, "0.3.5"},
{dqe, ".*", {git, "https://gitlab.com/Project-FiFo/DalmatinerDB/dqe.git", {branch, "master"}}},
{dqe, ".*", {git, "https://gitlab.com/Project-FiFo/DalmatinerDB/dqe.git", {branch, "hpts"}}},
{otters, "~>0.2.0"},
eper,
recon
......
rebar.lock
View file @ 3f43437d
......@@ -2,36 +2,48 @@
[{<<"cf">>,{pkg,<<"cf">>,<<"0.2.2">>},1},
{<<"cowboy">>,{pkg,<<"cowboy">>,<<"1.0.4">>},0},
{<<"cowlib">>,{pkg,<<"cowlib">>,<<"1.0.2">>},1},
{<<"ddb_client">>,{pkg,<<"ddb_client">>,<<"0.5.7">>},2},
{<<"ddb_connection">>,{pkg,<<"ddb_connection">>,<<"0.4.3">>},1},
{<<"ddb_client">>,
{git,"https://gitlab.com/Project-FiFo/DalmatinerDB/ddb_client.git",
{ref,"ee1f4131767ed3cbf93e47f50200667369cf7c3d"}},
2},
{<<"ddb_connection">>,
{git,"https://gitlab.com/Project-FiFo/DalmatinerDB/ddb_connection.git",
{ref,"d296650d4f527c7c1d59543d932be169072d8f4e"}},
1},
{<<"dflow">>,{pkg,<<"dflow">>,<<"0.3.0">>},1},
{<<"dproto">>,{pkg,<<"dproto">>,<<"0.5.2">>},1},
{<<"dproto">>,
{git,"https://gitlab.com/Project-FiFo/DalmatinerDB/dproto.git",
{ref,"83996cf9a1173acc6e67e738f5e1357af5cbdb60"}},
2},
{<<"dqe">>,
{git,"https://gitlab.com/Project-FiFo/DalmatinerDB/dqe.git",
{ref,"6d8f70fa79c6ef438cfa73fc630b86281f612053"}},
{ref,"d07f74322c065edbc13b937eada2f8ea469abc65"}},
0},
{<<"dqe_fun">>,{pkg,<<"dqe_fun">>,<<"0.2.1">>},1},
{<<"dqe_idx">>,{pkg,<<"dqe_idx">>,<<"0.4.3">>},1},
{<<"dqe_idx_ddb">>,{pkg,<<"dqe_idx_ddb">>,<<"0.5.1">>},1},
{<<"dqe_idx_ddb">>,
{git,"https://gitlab.com/Project-FiFo/DalmatinerDB/dqe_idx_ddb.git",
{ref,"6b3f931ec073ee2d95af6560f94777f775407d91"}},
1},
{<<"dqe_idx_pg">>,{pkg,<<"dqe_idx_pg">>,<<"0.5.7">>},1},
{<<"dynamic_compile">>,{pkg,<<"dynamic_compile">>,<<"1.0.0">>},1},
{<<"eper">>,{pkg,<<"eper">>,<<"0.94.0">>},0},
{<<"epgsql">>,{pkg,<<"epgsql">>,<<"3.3.0">>},3},
{<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"1.0.0">>},0},
{<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"1.0.5">>},0},
{<<"fifo_lager">>,{pkg,<<"fifo_lager">>,<<"0.1.8">>},0},
{<<"fifo_utils">>,{pkg,<<"fifo_utils">>,<<"0.1.51">>},0},
{<<"fifo_utils">>,{pkg,<<"fifo_utils">>,<<"0.1.52">>},0},
{<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},1},
{<<"hdr_histogram">>,{pkg,<<"hdr_histogram">>,<<"0.3.2">>},1},
{<<"ibrowse">>,{pkg,<<"ibrowse">>,<<"4.4.0">>},1},
{<<"jsone">>,{pkg,<<"jsone">>,<<"1.2.6">>},0},
{<<"jsx">>,{pkg,<<"jsx">>,<<"2.8.2">>},2},
{<<"jsxd">>,{pkg,<<"jsxd">>,<<"0.2.4">>},2},
{<<"lager">>,{pkg,<<"lager">>,<<"3.5.1">>},0},
{<<"jsx">>,{pkg,<<"jsx">>,<<"2.8.3">>},2},
{<<"jsxd">>,{pkg,<<"jsxd">>,<<"0.2.4">>},3},
{<<"lager">>,{pkg,<<"lager">>,<<"3.5.2">>},0},
{<<"lager_graylog">>,{pkg,<<"lager_graylog">>,<<"0.1.3">>},1},
{<<"lager_logstash_backend">>,
{pkg,<<"lager_logstash_backend">>,<<"0.1.3">>},
1},
{<<"mmath">>,{pkg,<<"mmath">>,<<"0.2.19">>},1},
{<<"mmath">>,{pkg,<<"mmath">>,<<"0.2.24">>},1},
{<<"msgpack">>,{pkg,<<"msgpack">>,<<"0.3.5">>},0},
{<<"otters">>,{pkg,<<"otters">>,<<"0.2.10">>},0},
{<<"pgapp">>,{pkg,<<"pgapp">>,<<"0.0.2">>},2},
......@@ -41,50 +53,46 @@
<