Commit 9421395c authored by Tom Powell's avatar Tom Powell

Stats and API

parent 0883529c
Pipeline #35179281 passed with stages
in 2 minutes and 20 seconds
......@@ -102,10 +102,10 @@ class TestWebhooks(UsesModels):
class TestWeb(UsesModels):
def test_home(self):
build1 = Build(build_id=1)
runner = Runner(runner_name="foobar", runner_id="foobar")
build2 = Build(build_id=2, build_status="success", build_device="mako", build_version="cm-14.1", build_type="userdebug", build_date=datetime.datetime.now(), build_runner=runner)
def test_get(self):
build1 = Build(build_id=1, build_date=datetime.datetime.strptime("2018-01-01", "%Y-%m-%d"), build_status="pending", build_version="cm-14.1")
runner = Runner(runner_name="foobar", runner_id="foobar", runner_sponsor="Me", runner_sponsor_url="You")
build2 = Build(build_id=2, build_status="success", build_device="mako", build_version="cm-14.1", build_type="userdebug", build_date=datetime.datetime.strptime("2018-01-01", "%Y-%m-%d"), build_runner=runner)
db.session.add(build1)
db.session.add(runner)
db.session.add(build2)
......@@ -122,5 +122,17 @@ class TestWeb(UsesModels):
response = self.client.get("/?status=success&device=mako")
assert response.status_code == 200
response = self.client.get("/api/v1/runners")
assert response.status_code == 200
response = self.client.get("/api/v1/builds")
assert response.status_code == 200
response = self.client.get("/api/v1/stats")
assert response.status_code == 200
response = self.client.get("/stats")
assert response.status_code == 200
if __name__ == "__main__":
unittest.main()
......@@ -2,7 +2,7 @@ import datetime
import os
import requests
from flask import Flask, render_template, request, abort
from flask import Flask, render_template, request, abort, jsonify
from flask_bootstrap import Bootstrap
from flask_caching import Cache
from flask_migrate import Migrate
......@@ -10,6 +10,8 @@ from flask_nav import Nav
from flask_nav.elements import Navbar, Text, View
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy import orm, func
from ui import gitlab, config, models
app = Flask(__name__)
......@@ -22,8 +24,9 @@ nav = Nav(app)
nav.register_element('top', Navbar(
"LineageOS Builds",
View('Builds', '.index'),
View('Runners', '.runners')
View('Builds', '.web_index'),
View('Runners', '.web_runners'),
View('Stats', '.web_stats')
))
headers = {'Private-Token': os.environ.get('GITLAB_TOKEN', '')}
......@@ -44,31 +47,127 @@ def parse_args():
args['build_date'] = datetime.datetime.strptime(request.args.get('date'), '%Y-%m-%d').date()
return args
def stats():
runner_build_times = models.Build.query.join(models.Build.build_runner).with_entities(
models.Runner.runner_name,
models.Build.build_version,
func.avg(models.Build.build_duration),
func.max(models.Build.build_duration),
func.min(models.Build.build_duration),
func.sum(models.Build.build_duration)
).group_by(models.Build.build_version, models.Runner.runner_name).all()
all_build_times = models.Build.query.with_entities(
models.Build.build_version,
func.avg(models.Build.build_duration),
func.max(models.Build.build_duration),
func.min(models.Build.build_duration),
func.sum(models.Build.build_duration)
).group_by(models.Build.build_version).all()
runner_build_status = models.Build.query.join(models.Build.build_runner).with_entities(
models.Runner.runner_name,
models.Build.build_status,
func.count(models.Build.build_status)
).group_by(models.Runner.runner_name, models.Build.build_status).all()
stats = {
'builds': {
'all': {}
},
'times': {
'all': {}
}
}
for build_time in all_build_times:
stats['times']['all'][build_time[0]] = {
'avg': build_time[1] if build_time[1] else 0,
'max': build_time[2] if build_time[2] else 0,
'min': build_time[3] if build_time[3] else 0,
'sum': build_time[4] if build_time[4] else 0,
}
for build_time in runner_build_times:
stats['times'].setdefault(build_time[0], {})[build_time[1]] = {
'avg': build_time[2] if build_time[2] else 0,
'max': build_time[3] if build_time[3] else 0,
'min': build_time[4] if build_time[4] else 0,
'sum': build_time[5] if build_time[5] else 0,
}
for build_status in runner_build_status:
stats['builds']['all'].setdefault(build_status[1], 0)
stats['builds']['all'][build_status[1]] += build_status[2]
stats['builds'].setdefault(build_status[0], {})[build_status[1]] = build_status[2]
return stats
@app.route('/')
def index():
def web_index():
try:
args = parse_args()
except ValueError:
return "Invalid Date", 400
builds = models.Build.query.filter_by(**args).order_by(models.Build.build_date.desc(), models.Build.build_id).paginate(per_page=20)
builds = models.Build.paginate(args)
return render_template('builds.html', builds=builds)
@app.route('/runners/<string:runner>')
def runner(runner):
def web_runner(runner):
try:
args = parse_args()
except ValueError:
return "Invalid Date", 400
runner = models.Runner.query.filter_by(runner_name=runner).first()
runner = models.Runner.get({'runner_name': runner}).first()
args['build_runner'] = runner
builds = models.Build.query.filter_by(**args).order_by(models.Build.build_date.desc(), models.Build.build_id).paginate(per_page=20)
builds = models.Build.paginate(args)
return render_template('runner.html', runner=runner, builds=builds)
@app.route('/stats')
def web_stats():
return render_template('stats.html', stats=stats())
@app.route("/runners/")
def runners():
runners = models.Runner.query.order_by(models.Runner.runner_sponsor, models.Runner.runner_name).all()
def web_runners():
runners = models.Runner.get().all()
return render_template('runners.html', runners=runners)
@app.route('/api/v1/builds')
def api_builds():
try:
args = parse_args()
except ValueError:
return jsonify({'error': 'Invalid Date'}), 400
builds = models.Build.paginate(args).items
if not builds:
abort(404)
return jsonify([x.as_dict() for x in builds])
@app.route('/api/v1/runners')
def api_runners():
runners = models.Runner.get().all()
if not runners:
abort(404)
return jsonify([x.as_dict() for x in runners])
@app.route('/api/v1/runners/<string:runner>')
def api_runner(runner):
try:
args = parse_args()
except ValueError:
return jsonify({"Invalid Date"}), 400
runner = models.Runner.get({"runner_name": runner}).first()
if not runner:
abort(404)
return jsonify(runner.as_dict())
@app.route('/api/v1/stats')
def api_stats():
return jsonify(stats())
@app.route('/stats')
@app.route("/webhook", methods=('POST',))
def process_webhook():
gitlab.webhooks.process(request)
......
......@@ -2,6 +2,7 @@ import os
SQLALCHEMY_DATABASE_URI = os.environ.get("SQLALCHEMY_DATABASE_URI", 'sqlite:////tmp/ui.db')
SQLALCHEMY_TRACK_MODIFICATIONS = False
SQLALCHEMY_ECHO = 'FLASK_DEBUG' in os.environ
CACHE_TYPE = 'simple'
GITLAB_WEBHOOK_TOKEN = os.environ.get("GITLAB_WEBHOOK_TOKEN", "secret")
......
import datetime
from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy()
......@@ -16,11 +17,29 @@ class Build(db.Model):
def __repr__(self):
return f"{self.build_id} {self.build_device} {self.build_version} {self.build_type} {self.build_runner_id}"
def as_dict(self):
return {
'id': self.build_id,
'status': self.build_status,
'device': self.build_device,
'version': self.build_version,
'type': self.build_type,
'date': self.build_date.strftime('%Y-%m-%d'),
'duration': self.build_duration,
'runner': self.build_runner.runner_name if self.build_runner else None
}
@classmethod
def get_or_create_by_id(cls, build_id):
build = cls.query.filter_by(build_id=build_id).first()
return build if build else cls(build_id=build_id)
@classmethod
def paginate(cls, args=None):
if not args:
args = {}
return cls.query.filter_by(**args).order_by(cls.build_date.desc(), cls.build_id).paginate(per_page=min([args.get("per_page", 20), 200]))
class Runner(db.Model):
runner_id = db.Column(db.String, primary_key=True, autoincrement=False)
runner_name = db.Column(db.String)
......@@ -28,6 +47,14 @@ class Runner(db.Model):
runner_sponsor = db.Column(db.String)
runner_sponsor_url = db.Column(db.String)
def as_dict(self):
return {
'id': self.runner_id,
'name': self.runner_name,
'sponsor': self.runner_sponsor,
'sponsor_url': self.runner_sponsor_url
}
@classmethod
def get_or_create_by_id(cls, runner_id):
runner = cls.query.filter_by(runner_id=runner_id).first()
......@@ -36,3 +63,9 @@ class Runner(db.Model):
db.session.add(runner)
db.session.commit()
return runner
@classmethod
def get(cls, args=None):
if not args:
args = {}
return cls.query.filter_by(**args).order_by(cls.runner_sponsor, cls.runner_name)
{%- extends "base.html" %}
{% import "bootstrap/utils.html" as utils %}
{% from "bootstrap/pagination.html" import render_pagination %}
{% block content %}
{% set statuses = stats['builds']['all'].keys() | sort %}
<div class="container">
Status
<table class="table table-striped">
<tr>
<th>Runner</th>
{% for status in statuses %}
<th>{{status | title}}</th>
{% endfor %}
</tr>
{% for runner in stats['builds'].keys() | sort %}
<tr>
<td>{% if runner != 'all' %}<a href="/runners/{{runner}}">{{runner}}</a>{% else %}{{runner}}{% endif %}</td>
{% for status in statuses %}
<td>{{stats['builds'].get(runner).get(status, 0)}}</td>
{% endfor %}
</tr>
{% endfor %}
</table>
{% for version in stats['times']['all'] | sort(reverse=True) %}
Times for {{version}}
<table class="table table-striped">
<tr>
<th>Runner</th>
<th>Average</th>
<th>Minimum</th>
<th>Maximum</th>
<th>Total</th>
</tr>
{% for runner in stats['builds'].keys() | sort %}
<tr>
<td>{% if runner != 'all' %}<a href="/runners/{runner}">{{runner}}</a>{% else %}{{runner}}{% endif %}</td>
<td>{{(stats['times'][runner][version]['avg'] / 60) | round(2)}} minutes</td>
<td>{{(stats['times'][runner][version]['min'] / 60) | round(2)}} minutes</td>
<td>{{(stats['times'][runner][version]['max'] / 60) | round(2)}} minutes</td>
<td>{{(stats['times'][runner][version]['sum'] / 60 / 60 / 24) | round(2)}} days </td>
</tr>
{% endfor %}
</table>
{% endfor %}
</div>
{% endblock %}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment