Commit e8a4bece authored by David Spencer's avatar David Spencer

work in progress

parent 3547c8f9
"""
SBo Dashboard
browse.py
Render browseable maintainer and slackbuild detail pages
David Spencer 2018-08-27
See LICENCE for copyright information
"""
import logging
import globals
import stats
import utils
from slackbuilds import SBinfo
from maintainers import Maintainerinfo
from problems import Problems
#-----------------------------------------------------------------------
def render_maintainer_detail():
logging.info("started")
for mntnam in SBinfo.Maintainerinfo.items():
MaintainerDetail = [] #### temporary
utils.renderer( page_subdir="maintainers",
page_name=email, #### needs to be Unix-safe
page_title=mntnam,
template="maintainer_detail",
TemplateData=MaintainerDetail )
logging.info("finished")
#-----------------------------------------------------------------------
def render_slackbuild_detail():
logging.info("started")
for prgnam, prginfo in SBinfo.items():
catnam=prginfo["catnam"]
SlackbuildDetail = { "prgnam": prgnam,
"catnam": catnam,
"version": prginfo["version"],
"homepage": prginfo["homepage"],
"maintainer": prginfo["maintainer"],
"email": prginfo["email"]
}
utils.renderer( page_subdir="slackbuilds/"+catnam,
page_name=prgnam,
page_title=catnam+"/"+prgnam,
template="slackbuild_detail",
TemplateData=SlackbuildDetail )
logging.info("finished")
#-----------------------------------------------------------------------
"""
SBo Dashboard
gitfuncs.py
Functions for accessing the local SlackBuilds.org clone
David Spencer 2018
See LICENCE for copyright information
"""
import os
import subprocess
import globals
#-----------------------------------------------------------------------
def getnewest(catnam,prgnam):
glog = subprocess.run(
"git log --pretty=\"format:%ai %h %s\" -n 1 .",
shell=True,
cwd=os.path.join(globals.sbdir,catnam,prgnam),
check=True,
stdout=subprocess.PIPE
)
return(glog.stdout.decode("utf-8"))
#-----------------------------------------------------------------------
def clone_sbo():
subprocess.run(
"git clone git://git.slackbuilds.org/slackbuilds.git {:s}".format(globals.sbdir),
shell=True,
cwd=globals.dashdir,
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
#-----------------------------------------------------------------------
def pull_sbo():
subprocess.run(
"git checkout master ; git pull --ff-only origin",
shell=True,
cwd=globals.sbdir,
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
#-----------------------------------------------------------------------
def megalog(SEP):
"""
This produces log lines of the form
dateSEPcommitSEPnameSEPemail
where
%ad date = YYYY-MM-DD (we don't care about the time)
%h commit = abbreviated SHA1
%aN author = author name, contains spaces
%aE email = author email, can contain spaces :(
(see man git-log)
"""
gitlog = subprocess.run(
"git log --format=\"%ad{0:s}%h{0:s}%aN{0:s}%aE\" --date=\"format:%Y-%m-%d\"".format(SEP),
shell=True, # Unsafe? phooey.
cwd=globals.sbdir,
check=True,
stdout=subprocess.PIPE
)
return gitlog.stdout
......@@ -3,7 +3,7 @@ SBo Dashboard
globals.py
Initialise global variables
David Spencer 2018-08-27
David Spencer 2018
See LICENCE for copyright information
"""
......@@ -13,75 +13,54 @@ import datetime
import configparser
import logging
#-----------------------------------------------------------------------
def configulator(inidict, varname, defaultvalue=""):
"""
Get config values from environment and from a configparser ini dict
* environment overrides the ini dict
* default value is substituted if unset *or empty*
"""
value = os.environ.get( varname.upper(), inidict.get(varname,defaultvalue) )
if value == "":
value = defaultvalue
return value
#-----------------------------------------------------------------------
# Create a timestamp for this run
updateref = datetime.datetime.now(datetime.timezone.utc)
#-----------------------------------------------------------------------
# Create configuration variables
# Create the configuration variables
# Get config values from ini file
config = configparser.ConfigParser()
config.read("sbodash.ini")
ini_locations = config["Locations"]
dashdir = ini_locations.get("dashdir","")
templatesdir = ini_locations.get("templatesdir","")
datadir = ini_locations.get("datadir","")
sbdir = ini_locations.get("sbdir","")
sitedir = ini_locations.get("sitedir","")
siteurl = ini_locations.get("siteurl","")
ini_locations = config["Locations"]
dashdir = configulator( ini_locations, "dashdir", os.getcwd() )
templdir = configulator( ini_locations, "templdir", "templates" )
datadir = configulator( ini_locations, "datadir", "data" )
sbdir = configulator( ini_locations, "sbdir", "data/slackbuilds" )
sitedir = configulator( ini_locations, "sitedir", "site" )
siteurl = configulator( ini_locations, "siteurl", "http://localhost/sbodash" )
ini_statistics = config["Statistics"]
keepstats = int(ini_statistics.get("keepstats",""))
ini_logging = config["Logging"]
logsdir = ini_logging.get("logsdir","")
loglevel = ini_logging.get("loglevel","").upper()
# Environment variable overrides
if "DASHDIR" in os.environ:
dashdir = os.getenv("DASHDIR")
if "TEMPLATESDIR" in os.environ:
templatesdir = os.getenv("TEMPLATESDIR")
if "DATADIR" in os.environ:
datadir = os.getenv("DATADIR")
if "SBDIR" in os.environ:
sbdir = os.getenv("SBDIR")
if "SITEDIR" in os.environ:
sitedir = os.getenv("SITEDIR")
if "SITEURL" in os.environ:
siteurl = os.getenv("SITEURL")
if "KEEPSTATS" in os.environ:
keepstats = int(os.getenv("KEEPSTATS"))
if "LOGSDIR" in os.environ:
logsdir = os.getenv("LOGSDIR")
if "LOGLEVEL" in os.environ:
loglevel = os.getenv("LOGLEVEL").upper()
# Provide defaults
if dashdir == "":
dashdir = os.getcwd()
if templatesdir == "":
templatesdir = "templates"
if datadir == "":
datadir = "data"
if sbdir == "":
sbdir = "data/slackbuilds"
if sitedir == "":
sitedir = "site"
if siteurl == "":
siteurl = "http://localhost/sbo-dash"
if keepstats == "":
keepstats = 7
if logsdir == "":
logsdir = "logs"
if loglevel == "":
loglevel = "INFO"
keepstats = int( configulator( ini_statistics, "keepstats", "366" ) )
ini_logging = config["Logging"]
logsdir = configulator( ini_logging, "logsdir", "logs" )
loglevel = configulator( ini_logging, "loglevel", "INFO" ).upper()
ini_updates = config["Updates"]
upd_sbo = int( configulator( ini_updates, "upd_sbo", "7" ) )
upd_repgy = int( configulator( ini_updates, "upd_repgy", "7" ) )
# Fix up paths
if not os.path.isabs(dashdir):
os.path.join(os.getcwd(),dashdir)
if not os.path.isabs(templatesdir):
templatesdir = os.path.join(dashdir,templatesdir)
if not os.path.isabs(templdir):
templdir = os.path.join(dashdir,templdir)
if not os.path.isabs(datadir):
datadir = os.path.join(dashdir,datadir)
if not os.path.isabs(sbdir):
......@@ -94,12 +73,12 @@ if not os.path.isabs(logsdir):
# These directories must exist
if not os.path.isdir(dashdir):
sys.exit("Not a directory: {:s}".format(dashdir))
if not os.path.isdir(templatesdir):
sys.exit("Not a directory: {:s}".format(templatesdir))
if not os.path.isdir(templdir):
sys.exit("Not a directory: {:s}".format(templdir))
# These directories will be created if they do not exist
os.makedirs(datadir,exist_ok=True)
os.makedirs(sbdir,exist_ok=True)
os.makedirs(sbdir, exist_ok=True)
os.makedirs(sitedir,exist_ok=True)
os.makedirs(logsdir,exist_ok=True)
......@@ -116,14 +95,14 @@ logging.basicConfig( filename=os.path.join(logsdir,"sbodash_{:s}.log".format(upd
logging.debug("configuration:")
logging.debug(" {:s} = {:s}".format("updateref",updateref.strftime("%Y-%m-%d %T")))
logging.debug(" {:s} = {:s}".format("dashdir",dashdir))
logging.debug(" {:s} = {:s}".format("templatesdir",templatesdir))
logging.debug(" {:s} = {:s}".format("datadir",datadir))
logging.debug(" {:s} = {:s}".format("sbdir",sbdir))
logging.debug(" {:s} = {:s}".format("sitedir",sitedir))
logging.debug(" {:s} = {:s}".format("siteurl",siteurl))
logging.debug(" {:s} = {:s}".format("dashdir", dashdir))
logging.debug(" {:s} = {:s}".format("templdir", templdir))
logging.debug(" {:s} = {:s}".format("datadir", datadir))
logging.debug(" {:s} = {:s}".format("sbdir", sbdir))
logging.debug(" {:s} = {:s}".format("sitedir", sitedir))
logging.debug(" {:s} = {:s}".format("siteurl", siteurl))
logging.debug(" {:s} = {:d}".format("keepstats",keepstats))
logging.debug(" {:s} = {:s}".format("logsdir",logsdir))
logging.debug(" {:s} = {:s}".format("loglevel",loglevel))
logging.debug(" {:s} = {:s}".format("logsdir", logsdir))
logging.debug(" {:s} = {:s}".format("loglevel", loglevel))
#-----------------------------------------------------------------------
This diff is collapsed.
"""
SBo Dashboard
update_problems
Create a report of SBo problems
problems.py
Functions and data structures relating to problems
David Spencer 2018-08-05
David Spencer 2018
See LICENCE for copyright information
"""
......@@ -21,7 +21,7 @@ import utils
import stats
import slackbuilds
Problems={}
Problems = []
#-----------------------------------------------------------------------
......@@ -33,39 +33,33 @@ def catnamprgnam(p):
#-----------------------------------------------------------------------
def repologyproblems(repo):
# Get a list of problems from the Repology API for the specified repo
# Return a list of dicts:
# { "catnam":"SBo category", "prgnam":"SBo prgnam", "maintainer":"email",
# "problem":"problem type", "description":"description text",
# "source":"repology" "since":"datestring" }
repologyAPI="https://repology.org/api/v1/repository/{:s}/problems".format(repo)
# the API spec doesn't say that the 'problems' result set is is chunked,
# so let's assume that it isn't.
problist=requests.get(repologyAPI).json()
if len(problist) == 0:
return None
def process_repology_problems(rplist,data_updated):
"""
Get a list of problems from Repology for the specified repo.
Returns a list of dicts:
{ "catnam":"SBo category", "prgnam":"SBo prgnam", "maintainer":"email",
"problem":"problem type", "description":"description text",
"source":"repology" "since":"datestring" }
"""
problems=[]
for p in problist:
problems = []
for p in rplist:
prgnam=p["name"]
catnam=slackbuilds.SBinfo[prgnam]["catnam"]
maintainer=slackbuilds.SBinfo[prgnam]["maintainer"]
email=slackbuilds.SBinfo[prgnam]["email"]
prgnam = p["name"]
catnam = slackbuilds.SBinfo[prgnam]["catnam"]
maintainer = slackbuilds.SBinfo[prgnam]["maintainer"]
email = slackbuilds.SBinfo[prgnam]["email"]
# All Repology problems currently start with "Homepage link" and
# a URL, but we'll classify any that don't as "other". To make the
# included URL clickable we need to remove the surrounding quotes.
hstr="Homepage link \""
hstr = "Homepage link \""
if p["problem"].startswith(hstr):
problem="homepage"
description=p["problem"][len(hstr):].replace("\" "," ",1)
problem = "homepage"
description = p["problem"][len(hstr):].replace("\" "," ",1)
else:
problem="other"
description=p["problem"]
problem = "other"
description = p["problem"]
problems.append({ "prgnam": prgnam,
"catnam": catnam,
......@@ -74,38 +68,76 @@ def repologyproblems(repo):
"problem": problem,
"description": description,
"source": "repology",
"since": globals.updateref.strftime("%Y-%m-%d")
"since": data_updated.strftime("%Y-%m-%d")
})
return sorted(problems,key=catnamprgnam)
#-----------------------------------------------------------------------
def update_repology_problems():
def get_repology_problems():
"""
Retrieve problems from Repology
Retrieve repology problems from remote server.
"""
repologyAPI = "https://repology.org/api/v1/repository/{:s}/problems".format(repo)
# The API spec doesn't say whether the 'problems' result set is is chunked.
# In practice it seems to be truncated at 500 problems, so let's not go there.
repology_problems = requests.get(repologyAPI).json()
#-----------------------------------------------------------------------
def load_repology_problems():
"""
Load repology problems into 'Problems'
"""
logging.info("started")
global Problems
Problems = repologyproblems('slackbuilds')
probcount=len(Problems)
logging.info("started")
repo = "slackbuilds"
datafile = "repology_problems.p"
updatefile = "repology_problems_updated.p"
prev_repology_updated = utils.load(updatefile)
if prev_repology_updated is None or prev_repology_updated + datetime.timedelta(days=globals.upd_repgy) < globals.updateref:
logging.info("update needed; retrieving from Repology")
logging.debug("retrieved {:d} problems from repology".format(len(repology_problems)))
utils.save(repology_problems,datafile)
data_updated = globals.updateref
utils.save(data_updated,updatefile)
else:
logging.debug("update not needed")
repology_problems=utils.load(datafile)
if repology_problems is None:
logging.error("Error: failed to load old repology problems")
return
logging.debug("loaded {:d} problems from storage".format(len(repology_problems)))
data_updated = prev_repology_updated
if len(repology_problems) == 0:
return
Problems = process_repology_problems(repology_problems,data_updated)
probcount = len(Problems)
stats.setStats("problems.repology.count",probcount)
stats.addStats("problems.count",probcount)
logging.info("finished -- {:d} problems".format(probcount))
#-----------------------------------------------------------------------
def update_download_problems():
def load_download_problems():
pass
#-----------------------------------------------------------------------
def update_security_problems():
def load_security_problems():
pass
#-----------------------------------------------------------------------
def update_build_problems():
def load_build_problems():
pass
#-----------------------------------------------------------------------
......@@ -117,7 +149,7 @@ def render_problems():
logging.info("started")
utils.renderer( page_subdir="reports",
page_name="problems.html",
page_title="SBo Problems",
page_title="SBo Problems Report",
TemplateData=Problems )
logging.info("finished")
......
......@@ -5,7 +5,7 @@
# Top level command
# No arguments, see 'sbodash.ini' for configuration
#
# David Spencer 2018-08-13
# David Spencer 2018
# See LICENCE for copyright information
# See README.md for external requirements
#-----------------------------------------------------------------------
......@@ -17,33 +17,30 @@ import stats
#-----------------------------------------------------------------------
logging.info("Run started")
logging.info("RUN STARTED")
from slackbuilds import update_slackbuilds, read_slackbuilds_info
update_slackbuilds()
read_slackbuilds_info()
from slackbuilds import load_slackbuilds, render_slackbuild_detail
load_slackbuilds()
render_slackbuild_detail()
from maintainers import update_maintainers, render_maintainers
update_maintainers()
from maintainers import load_maintainers, render_maintainers, render_maintainer_detail
load_maintainers()
render_maintainers()
render_maintainer_detail()
from problems import update_repology_problems, update_download_problems, update_security_problems, update_build_problems, render_problems
update_repology_problems()
update_download_problems()
update_security_problems()
update_build_problems()
from problems import load_repology_problems, load_download_problems, load_security_problems, load_build_problems, render_problems
load_repology_problems()
load_download_problems()
load_security_problems()
load_build_problems()
render_problems()
from details import render_maintainer_detail, render_slackbuild_detail
render_maintainer_detail()
render_slackbuild_detail()
from support import render_support, render_indexes
render_support()
render_indexes()
stats.saveStatsHistory()
logging.info("Run finished")
logging.info("RUN FINISHED")
#-----------------------------------------------------------------------
......@@ -24,7 +24,7 @@
dashdir =
; Jinja2 templates are here
templatesdir = templates
templdir = templates
; Storage for working data (will be created if necessary)
datadir = data
......@@ -50,9 +50,19 @@ loglevel = INFO
;-----------------------------------------------------------------------
[Updates]
; minimum interval for pulling the SlackBuilds.org git repo (days)
upd_sbo = 7
; minimum interval for fetching maintainers and problems from repology (days)
upd_repgy = 7
;-----------------------------------------------------------------------
[Statistics]
; Statistics retention period (days)
keepstats = 7
keepstats = 366
;-----------------------------------------------------------------------
......@@ -3,7 +3,7 @@ SBo Dashboard
slackbuilds.py
Functions and dict for accessing a local SlackBuilds.org clone
David Spencer 2018-08-28
David Spencer 2018
See LICENCE for copyright information
"""
......@@ -12,63 +12,40 @@ import re
import glob
import subprocess
import logging
import datetime
import globals
import stats
import utils
import gitfuncs
SBinfo={}
SBinfo = {}
SBindex = []
#-----------------------------------------------------------------------
def update_slackbuilds():
def process_slackbuilds(sbinfofile,sbindexfile,updatefile):
"""
Update the local slackbuilds git repo from remote
"""
logging.info("started")
#### needs error handling
if os.path.isfile(os.path.join(globals.sbdir,"ChangeLog.txt")):
subprocess.run(
"git checkout master ; git pull --ff-only origin",
shell=True,
cwd=globals.sbdir,
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
else:
subprocess.run(
"git clone git://git.slackbuilds.org/slackbuilds.git {:s}".format(globals.sbdir),
shell=True,
cwd=globals.dashdir,
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
logging.info("finished")
#-----------------------------------------------------------------------
def read_slackbuilds_info():
"""
Read data from SlackBuilds info files into the SBinfo dict
Read data from SlackBuilds info files and git into SBinfo and SBindex
and save them into the persistent data files
"""
logging.debug("reading .info files")
# These regexes are used >7000 times, so we might as well compile them.
re_version=re.compile(r"VERSION=\".*\"")
re_homepage=re.compile(r"HOMEPAGE=\".*\"")
re_maint=re.compile(r"MAINTAINER=\".*\"")
re_email=re.compile(r"EMAIL=\".*\"")
re_version = re.compile(r"VERSION=\".*\"")
re_homepage = re.compile(r"HOMEPAGE=\".*\"")
re_maint = re.compile(r"MAINTAINER=\".*\"")
re_email = re.compile(r"EMAIL=\".*\"")
infocount=0
infocount = 0
catnamlist = []
for infofile in glob.glob(globals.sbdir+"/*/*/*.info"):
infocount = infocount+1
for infofile in sorted(glob.glob(globals.sbdir+"/*/*/*.info")):
infocount += 1
logging.debug(infofile)
prgnam=infofile.split("/")[-2]
catnam=infofile.split("/")[-3]
catnam = infofile.split("/")[-3]
prgnam = infofile.split("/")[-2]
try:
with open(infofile, "r", encoding="utf-8") as openinfo:
......@@ -77,32 +54,105 @@ def read_slackbuilds_info():
with open(infofile, "r", encoding="latin-1") as openinfo:
info = openinfo.read()
versionarray=re_version.findall(info)
homepagearray=re_homepage.findall(info)
maintainerarray=re_maint.findall(info)
emailarray=re_email.findall(info)
versionarray = re_version.findall(info)
homepagearray = re_homepage.findall(info)
maintainerarray = re_maint.findall(info)
emailarray = re_email.findall(info)
if len(versionarray) == 1 and len(homepagearray) == 1 and len(maintainerarray) == 1 and len(emailarray) == 1:
version=versionarray[0][9:-1]
homepage=homepagearray[0][10:-1]
maintainer=maintainerarray[0][12:-1]
email=emailarray[0][7:-1]
version = versionarray[0][9:-1]
homepage = homepagearray[0][10:-1]
maintainer = maintainerarray[0][12:-1]
email = emailarray[0][7:-1]
else:
# We're not here to validate the .info files.
# If there's not exactly one MAINTAINER and EMAIL, bollocks to it.
logging.warn("{:s} has duplicated entries".format(infofile))
logging.warn("{:s} has missing or broken or duplicated entries".format(infofile))
continue
gitnewest = gitfuncs.getnewest(catnam,prgnam)
SBinfo.update({ prgnam: { "catnam": catnam,
"version": version,
"homepage": homepage,
"maintainer": maintainer,
"email": email
"email": email,
"gitnewest": gitnewest
} })
if catnam not in catnamlist:
catnamlist.append(catnam)
SBindex.append({"catnam":catnam, "prgnamlist":[prgnam]})
else:
SBindex[-1]["prgnamlist"].append(prgnam)
if infocount != len(SBinfo):
logging.warn("{:d} duplicate prgnams in slackbuilds repo".format(infocount-len(SBinfo)))
utils.save(SBinfo,sbinfofile)
utils.save(SBindex,sbindexfile)
utils.save(globals.updateref,updatefile)
stats.setStats("slackbuilds.count",infocount)
logging.debug("finished -- {:d} .info files".format(infocount))
#-----------------------------------------------------------------------
def load_slackbuilds():
"""
Update the local slackbuilds git repo from remote, and load the data
"""
logging.info("started")
sbinfofile = "slackbuilds_SBinfo.p"
sbindexfile = "slackbuilds_SBindex.p"
updatefile = "slackbuilds_updated.p"
if not os.path.isfile(os.path.join(globals.sbdir,"ChangeLog.txt")):
logging.info("cloning from SBo repo")
gitfuncs.clone_sbo()
process_slackbuilds(sbinfofile,sbindexfile,updatefile)
else:
prev_updated_sbo = utils.load(updatefile)
if prev_updated_sbo is None or prev_updated_sbo + datetime.timedelta(days=globals.upd_sbo) < globals.updateref:
logging.info("update needed; pulling from SBo repo")
gitfuncs.pull_sbo()
process_slackbuilds(sbinfofile,sbindexfile,updatefile)
else:
SBinfo=utils.load(sbinfofile)
SBindex=utils.load(sbindexfile)
logging.debug("update not needed")
logging.info("finished")
#-----------------------------------------------------------------------