Commit f548b692 authored by David Spencer's avatar David Spencer

More refactoring.

parent 6ab09f80
......@@ -2,6 +2,16 @@
Periodically create a set of web pages reporting on SlackBuilds.org
## External requirements
python3
python-requests
python-chardet
idna
python-certifi
Jinja2
MarkupSafe
## Running
* Clone this repository
......
......@@ -114,15 +114,16 @@ logging.basicConfig( filename=os.path.join(logsdir,"sbodash_{:s}.log".format(upd
format="%(asctime)s %(funcName)s %(message)s",
level=getattr(logging,loglevel) )
logging.debug("{:s} = {:s}".format("updateref",updateref.strftime("%Y-%m-%d %T")))
logging.debug("{:s} = {:s}".format("dashdir",dashdir))
logging.debug("{:s} = {:s}".format("templatesdir",templatesdir))
logging.debug("{:s} = {:s}".format("datadir",datadir))
logging.debug("{:s} = {:s}".format("sbdir",sbdir))
logging.debug("{:s} = {:s}".format("sitedir",sitedir))
logging.debug("{:s} = {:s}".format("siteurl",siteurl))
logging.debug("{:s} = {:d}".format("keepstats",keepstats))
logging.debug("{:s} = {:s}".format("logsdir",logsdir))
logging.debug("{:s} = {:s}".format("loglevel",loglevel))
logging.debug("configuration:")
logging.debug(" {:s} = {:s}".format("updateref",updateref.strftime("%Y-%m-%d %T")))
logging.debug(" {:s} = {:s}".format("dashdir",dashdir))
logging.debug(" {:s} = {:s}".format("templatesdir",templatesdir))
logging.debug(" {:s} = {:s}".format("datadir",datadir))
logging.debug(" {:s} = {:s}".format("sbdir",sbdir))
logging.debug(" {:s} = {:s}".format("sitedir",sitedir))
logging.debug(" {:s} = {:s}".format("siteurl",siteurl))
logging.debug(" {:s} = {:d}".format("keepstats",keepstats))
logging.debug(" {:s} = {:s}".format("logsdir",logsdir))
logging.debug(" {:s} = {:s}".format("loglevel",loglevel))
#-----------------------------------------------------------------------
This diff is collapsed.
#!/usr/bin/python3
"""
SBo Dashboard
update_problems
......@@ -6,15 +5,6 @@ Create a report of SBo problems
David Spencer 2018-08-05
See LICENCE for copyright information
Environment variables: see globals.py
Requires: python3, and the following (built with python3 support)
python-requests
python-chardet
idna
python-certifi
"""
import sys
......@@ -29,20 +19,14 @@ import requests
import globals
import utils
import stats
import slackbuilds
#-----------------------------------------------------------------------
# Handy functions
def find_catnam(prgnam):
matches=glob.glob(globals.sbdir+"/*/"+prgnam)
if len(matches) == 0:
return "Unknown"
elif len(matches) == 1:
return matches[0].split("/")[-2]
else:
return "Ambiguous"
def catnamprgnam(p):
"""
Function for sorting the 'problems' dict by keys 'catnam' and 'prgnam'
"""
return p["catnam"]+"/"+p["prgnam"]
#-----------------------------------------------------------------------
......@@ -66,8 +50,8 @@ def repologyproblems(repo):
for p in problist:
prgnam=p["name"]
catnam=find_catnam(prgnam)
maintainer=p["maintainer"]
catnam=slackbuilds.SBinfo[prgnam]["catnam"]
maintainer=slackbuilds.SBinfo[prgnam]["maintainer"]
# All Repology problems currently start with "Homepage link" and
# a URL, but we'll classify any that don't as "other". To make the
......@@ -100,8 +84,10 @@ def update_repology_problems():
logging.info("started")
global Problems
Problems = repologyproblems('slackbuilds')
stats.addstats("probcount",len(Problems))
logging.info("finished")
probcount=len(Problems)
stats.setStats("problems.repology.count",probcount)
stats.addStats("problems.count",probcount)
logging.info("finished -- {:d} problems".format(probcount))
#-----------------------------------------------------------------------
......
#!/usr/bin/python3 -B
#-----------------------------------------------------------------------
# SBo Dashboard
# sbodash.py
# Get config, define common variables and functions
# sbodash
# Top level command
# No arguments, see 'sbodash.ini' for configuration
#
# David Spencer 2018-08-13
# See LICENCE for copyright information
# See README.md for external requirements
#-----------------------------------------------------------------------
import os
import pickle
import logging
import globals
......@@ -16,27 +19,26 @@ import stats
logging.info("Run started")
from support import update_slackbuilds, render_index, render_support
from slackbuilds import update_slackbuilds, read_slackbuilds_info
update_slackbuilds()
render_index()
render_support()
read_slackbuilds_info()
from maintainers import update_maintainers, render_maintainers
update_maintainers()
render_maintainers()
from problems import update_repology_problems, update_download_problems,
update_download_problems, update_security_problems, render_problems
from problems import update_repology_problems, update_download_problems, update_security_problems, update_build_problems, render_problems
update_repology_problems()
update_download_problems()
update_security_problems()
update_build_problems()
render_problems()
stats.SaveStatsHistory()
from support import render_index, render_support
render_index()
render_support()
stats.saveStatsHistory()
logging.info("Run finished")
......
"""
SBo Dashboard
slackbuilds.py
Functions and dict for accessing a local SlackBuilds.org clone
David Spencer 2018-08-28
See LICENCE for copyright information
"""
import os
import re
import glob
import subprocess
import logging
import globals
import stats
#-----------------------------------------------------------------------
def update_slackbuilds():
"""
Update the local slackbuilds git repo from remote
"""
logging.info("started")
#### needs error handling
if os.path.isfile(os.path.join(globals.sbdir,"ChangeLog.txt")):
subprocess.run(
"git checkout master ; git pull --ff-only origin",
shell=True,
cwd=globals.sbdir,
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
else:
subprocess.run(
"git clone git://git.slackbuilds.org/slackbuilds.git {:s}".format(globals.sbdir),
shell=True,
cwd=globals.dashdir,
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
logging.info("finished")
#-----------------------------------------------------------------------
# Create a dict from the SlackBuilds info files, for easy access
SBinfo={}
def read_slackbuilds_info():
"""
Read data from SlackBuilds info files into the SBinfo dict
"""
logging.debug("reading .info files")
# These regexes are used >7000 times, so we might as well compile them.
re_maint=re.compile(r"MAINTAINER=\".*\"")
re_email=re.compile(r"EMAIL=\".*\"")
infocount=0
for infofile in glob.glob(globals.sbdir+"/*/*/*.info"):
infocount = infocount+1
prgnam=infofile.split("/")[-2]
catnam=infofile.split("/")[-3]
try:
with open(infofile, "r", encoding="utf-8") as openinfo:
info = openinfo.read()
except:
with open(infofile, "r", encoding="latin-1") as openinfo:
info = openinfo.read()
maintainerarray=re_maint.findall(info)
emailarray=re_email.findall(info)
if len(maintainerarray) == 1 and len(emailarray) == 1:
maintainer=maintainerarray[0][12:-1]
email=emailarray[0][7:-1]
else:
# We're not here to validate the .info files.
# If there's not exactly one MAINTAINER and EMAIL, bollocks to it.
logging.warn("{:s} has {:d} maintainers and {:d} emails".format(infofile,len(maintainerarray),len(emailarray)))
continue
SBinfo.update({ prgnam: { "catnam": catnam, "maintainer": maintainer, "email": email} })
if infocount != len(SBinfo):
logging.warn("{:d} duplicate prgnams in slackbuilds repo".format(infocount-len(SBinfo)))
stats.setStats("slackbuilds.count",infocount)
logging.debug("finished reading {:d} .info files".format(infocount))
#-----------------------------------------------------------------------
......@@ -34,7 +34,14 @@ except:
#-----------------------------------------------------------------------
# Handy functions
def addstats(key,n=1):
def setStats(key,n=0):
"""
Set Stats[key] to 'n'
"""
Stats[key] = n
return
def addStats(key,n=1):
"""
Add 'n' to Stats[key], or set it if key doesn't exist.
"""
......@@ -44,7 +51,7 @@ def addstats(key,n=1):
Stats[key] = n
return
def SaveStatsHistory():
def saveStatsHistory():
"""
Expire old stats history and insert new stats.
"""
......
......@@ -5,21 +5,16 @@ Create the site index and support files, update the slackbuilds
David Spencer 2018-08-05
See LICENCE for copyright information
Requires: python3, and the following (all built with python3 support)
Jinja2
MarkupSafe
"""
import os
import datetime
import jinja2
import utils
import subprocess
import logging
import globals
import utils
#-----------------------------------------------------------------------
......@@ -48,28 +43,3 @@ def render_support():
logging.info("finished")
#-----------------------------------------------------------------------
def update_slackbuilds():
logging.info("started")
#### needs error handling
if os.path.isfile(os.path.join(globals.sbdir,"ChangeLog.txt")):
subprocess.run(
"git checkout master ; git pull --ff-only origin",
shell=True,
cwd=globals.sbdir,
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
else:
subprocess.run(
"git clone git://git.slackbuilds.org/slackbuilds.git {:s}".format(globals.sbdir),
shell=True,
cwd=globals.dashdir,
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
logging.info("finished")
#-----------------------------------------------------------------------
......@@ -9,14 +9,14 @@
<p>Updated: {{ page_datetime }}</p>
<p>SlackBuilds
<br>&nbsp;&nbsp;&nbsp;total: {{ "{:d}".format(Stats["sbcount"]) }}
<br>&nbsp;&nbsp;&nbsp;newest: {{ "{:d} ({:.1f}%)".format(Stats["new"],Stats["new"]*100/Stats["sbcount"]) }}
<br>&nbsp;&nbsp;&nbsp;out of date: {{ "{:d} ({:.1f}%)".format(Stats["old"],Stats["old"]*100/Stats["sbcount"]) }}
<br>&nbsp;&nbsp;&nbsp;total: {{ "{:d}".format(Stats["slackbuilds.count"]) }}
<br>&nbsp;&nbsp;&nbsp;newest: {{ "{:d} ({:.1f}%)".format(Stats["slackbuilds.uptodate"],Stats["slackbuilds.uptodate"]*100/Stats["slackbuilds.count"]) }}
<br>&nbsp;&nbsp;&nbsp;out of date: {{ "{:d} ({:.1f}%)".format(Stats["slackbuilds.outofdate"],Stats["slackbuilds.outofdate"]*100/Stats["slackbuilds.count"]) }}
</p>
<p>Maintainers
<br>&nbsp;&nbsp;&nbsp;total: {{ Stats["maint"] }}
<br>&nbsp;&nbsp;&nbsp;active: {{ "{:d} ({:.1f}%)".format(Stats["active"],Stats["active"]*100/Stats["maint"]) }} since 14.2
<br>&nbsp;&nbsp;&nbsp;recent: {{ "{:d} ({:.1f}%)".format(Stats["recent"],Stats["recent"]*100/Stats["maint"]) }} last 180 days
<br>&nbsp;&nbsp;&nbsp;total: {{ Stats["maintainers.count"] }}
<br>&nbsp;&nbsp;&nbsp;active: {{ "{:d} ({:.1f}%)".format(Stats["maintainers.active"],Stats["maintainers.active"]*100/Stats["maintainers.count"]) }} since 14.2
<br>&nbsp;&nbsp;&nbsp;recent: {{ "{:d} ({:.1f}%)".format(Stats["maintainers.recent"],Stats["maintainers.recent"]*100/Stats["maintainers.count"]) }} last 180 days
</p>
<table class="sortable">
......
......@@ -8,7 +8,7 @@
<p>Updated: {{ page_datetime }}</p>
<p>Problems: {{ "{:d}".format(Stats["probcount"]) }}</p>
<p>Problems: {{ "{:d}".format(Stats["problems.count"]) }}</p>
<table class="sortable">
......
......@@ -108,5 +108,4 @@ def save(filename,obj):
with open(data_path,"wb") as data_file:
pickle.dump(obj, file=data_file)
#-----------------------------------------------------------------------
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment