...
 
Commits (2)
......@@ -5,7 +5,7 @@ Periodically create a set of web pages reporting on SlackBuilds.org
## Running
* Clone this repository
* Edit sbo-dashboard.conf
* Run './runall'
* Check sbodash.ini and edit it as required
* Run ./sbodash
* Make your webserver serve the 'site' subdirectory
* Re-run './runall' periodically (e.g. once a week)
* Re-run ./sbodash periodically (e.g. once a week)
"""
SBo Dashboard
globals.py
Initialise global variables
David Spencer 2018-08-27
See LICENCE for copyright information
"""
import os
import sys
import datetime
import configparser
import logging
#-----------------------------------------------------------------------
# Create a timestamp for this run
updateref = datetime.datetime.now(datetime.timezone.utc)
#-----------------------------------------------------------------------
# Create configuration variables
# Get config values from ini file
config = configparser.ConfigParser()
config.read("sbodash.ini")
ini_locations = config["Locations"]
dashdir = ini_locations.get("dashdir","")
templatesdir = ini_locations.get("templatesdir","")
datadir = ini_locations.get("datadir","")
sbdir = ini_locations.get("sbdir","")
sitedir = ini_locations.get("sitedir","")
siteurl = ini_locations.get("siteurl","")
ini_statistics = config["Statistics"]
keepstats = int(ini_statistics.get("keepstats",""))
ini_logging = config["Logging"]
logsdir = ini_logging.get("logsdir","")
loglevel = ini_logging.get("loglevel","").upper()
# Environment variable overrides
if "DASHDIR" in os.environ:
dashdir = os.getenv("DASHDIR")
if "TEMPLATESDIR" in os.environ:
templatesdir = os.getenv("TEMPLATESDIR")
if "DATADIR" in os.environ:
datadir = os.getenv("DATADIR")
if "SBDIR" in os.environ:
sbdir = os.getenv("SBDIR")
if "SITEDIR" in os.environ:
sitedir = os.getenv("SITEDIR")
if "SITEURL" in os.environ:
siteurl = os.getenv("SITEURL")
if "KEEPSTATS" in os.environ:
keepstats = int(os.getenv("KEEPSTATS"))
if "LOGSDIR" in os.environ:
logsdir = os.getenv("LOGSDIR")
if "LOGLEVEL" in os.environ:
loglevel = os.getenv("LOGLEVEL").upper()
# Provide defaults
if dashdir == "":
dashdir = os.getcwd()
if templatesdir == "":
templatesdir = "templates"
if datadir == "":
datadir = "data"
if sbdir == "":
sbdir = "data/slackbuilds"
if sitedir == "":
sitedir = "site"
if siteurl == "":
siteurl = "http://localhost/sbo-dash"
if keepstats == "":
keepstats = 7
if logsdir == "":
logsdir = "logs"
if loglevel == "":
loglevel = "INFO"
# Fix up paths
if not os.path.isabs(dashdir):
os.path.join(os.getcwd(),dashdir)
if not os.path.isabs(templatesdir):
templatesdir = os.path.join(dashdir,templatesdir)
if not os.path.isabs(datadir):
datadir = os.path.join(dashdir,datadir)
if not os.path.isabs(sbdir):
sbdir = os.path.join(dashdir,sbdir)
if not os.path.isabs(sitedir):
sitedir = os.path.join(dashdir,sitedir)
if not os.path.isabs(logsdir):
logsdir = os.path.join(dashdir,logsdir)
# These directories must exist
if not os.path.isdir(dashdir):
sys.exit("Not a directory: {:s}".format(dashdir))
if not os.path.isdir(templatesdir):
sys.exit("Not a directory: {:s}".format(templatesdir))
# These directories will be created if they do not exist
os.makedirs(datadir,exist_ok=True)
os.makedirs(sbdir,exist_ok=True)
os.makedirs(sitedir,exist_ok=True)
os.makedirs(logsdir,exist_ok=True)
# The loglevel must be a valid loglevel
if loglevel not in ( "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL" ):
sys.exit("Not a valid log level: {:s}".format(loglevel))
#-----------------------------------------------------------------------
# Set up logging, and log some debug info
logging.basicConfig( filename=os.path.join(logsdir,"sbodash_{:s}.log".format(updateref.strftime("%Y-%m-%d"))),
format="%(asctime)s %(funcName)s %(message)s",
level=getattr(logging,loglevel) )
logging.debug("{:s} = {:s}".format("updateref",updateref.strftime("%Y-%m-%d %T")))
logging.debug("{:s} = {:s}".format("dashdir",dashdir))
logging.debug("{:s} = {:s}".format("templatesdir",templatesdir))
logging.debug("{:s} = {:s}".format("datadir",datadir))
logging.debug("{:s} = {:s}".format("sbdir",sbdir))
logging.debug("{:s} = {:s}".format("sitedir",sitedir))
logging.debug("{:s} = {:s}".format("siteurl",siteurl))
logging.debug("{:s} = {:d}".format("keepstats",keepstats))
logging.debug("{:s} = {:s}".format("logsdir",logsdir))
logging.debug("{:s} = {:s}".format("loglevel",loglevel))
#-----------------------------------------------------------------------
This diff is collapsed.
......@@ -7,7 +7,7 @@ Create a report of SBo problems
David Spencer 2018-08-05
See LICENCE for copyright information
Environment variables: see sbodash.py
Environment variables: see globals.py
Requires: python3, and the following (built with python3 support)
python-requests
......@@ -21,30 +21,20 @@ import sys
import os
import glob
import subprocess
import logging
import json
import datetime
import pickle
import requests
import sbodash
#-----------------------------------------------------------------------
OldProblems={}
OldStats={}
data_path=sbodash.datadir+"/problems.data"
try:
with open(data_path,"rb") as data_file:
(OldProblems, OldStats) = pickle.load(data_file)
except FileNotFoundError:
pass
import globals
import utils
import stats
#-----------------------------------------------------------------------
# Handy functions
def find_catnam(prgnam):
matches=glob.glob(sbodash.sbdir+"/*/"+prgnam)
matches=glob.glob(globals.sbdir+"/*/"+prgnam)
if len(matches) == 0:
return "Unknown"
elif len(matches) == 1:
......@@ -52,16 +42,17 @@ def find_catnam(prgnam):
else:
return "Ambiguous"
def catnamprgnam(p):
return p["catnam"]+"/"+p["prgnam"]
#-----------------------------------------------------------------------
def repologyproblems(repo):
# Get a list of problems from the Repology API for the specified repo
# Return a list of dicts:
# { "catnam":"SBo category", "prgnam":"SBo prgnam", "maintainer":"email",
# "problem":"problem type", "description":"description text" }
# "problem":"problem type", "description":"description text",
# "source":"repology" "since":"datestring" }
repologyAPI="https://repology.org/api/v1/repository/{:s}/problems".format(repo)
......@@ -93,25 +84,51 @@ def repologyproblems(repo):
"catnam": catnam,
"maintainer": maintainer,
"problem": problem,
"description": description
"description": description,
"source": "repology",
"since": globals.updateref.strftime("%Y-%m-%d")
})
return sorted(problems,key=catnamprgnam)
#-----------------------------------------------------------------------
def update_repology_problems():
"""
Retrieve problems from Repology
"""
logging.info("started")
global Problems
Problems = repologyproblems('slackbuilds')
stats.addstats("probcount",len(Problems))
logging.info("finished")
#-----------------------------------------------------------------------
Problems = repologyproblems('slackbuilds')
sbodash.addstats("problems",len(Problems))
def update_download_problems():
pass
#-----------------------------------------------------------------------
try:
os.makedirs(os.path.dirname(data_path))
except FileExistsError:
def update_security_problems():
pass
with open(data_path,"wb") as data_file:
pickle.dump( (Problems,sbodash.Stats), file=data_file )
#-----------------------------------------------------------------------
# Finished!
def update_build_problems():
pass
#-----------------------------------------------------------------------
def render_problems():
"""
Render the problems report page
"""
logging.info("started")
utils.renderer( page_name="problems",
page_subdir="reports",
page_title="SBo Problems",
Table=Problems )
logging.info("finished")
#-----------------------------------------------------------------------
#!/usr/bin/python3
"""
SBo Dashboard
render_index
Create the site index
David Spencer 2018-08-05
See LICENCE for copyright information
Requires: python3, and the following (all built with python3 support)
Jinja2
MarkupSafe
"""
import os
import datetime
import jinja2
import sbodash
page_datetime = datetime.datetime.fromtimestamp(sbodash.updated,datetime.timezone.utc)
page_path = os.path.join(sbodash.sitedir,"index.html")
#-----------------------------------------------------------------------
jenv = jinja2.Environment(loader=jinja2.FileSystemLoader(sbodash.templatesdir),autoescape=None)
mytemplate = jenv.get_template(os.path.basename(__file__).replace("render_","",1)+'.jinja2')
try:
os.makedirs(os.path.dirname(page_path))
except FileExistsError:
pass
with open(page_path,"w",encoding="utf-8") as page_file:
print(
mytemplate.render(
siteurl=sbodash.siteurl,
page_title="SBo Dashboard",
page_datetime=page_datetime.strftime("%Y-%m-%d %T %Z"),
),
file=page_file
)
#-----------------------------------------------------------------------
#!/usr/bin/python3
"""
SBo Dashboard
render_maintainers
Create a report of SBo maintainers and their activity status
David Spencer 2018-05-28
See LICENCE for copyright information
Requires: python3, and the following (built with python3 support)
Jinja2
MarkupSafe
Don't run this too often, because it hammers the Repology API for
maybe twenty minutes. Repology results may be empty or incomplete due
to redaction of email addresses. For production use, once per public
update would be normal.
"""
import sys
import os
import datetime
import pickle
import jinja2
import sbodash
report_name=os.path.basename(__file__).replace("render_","",1)
report_title="SBo {:s}".format(report_name.capitalize())
report_path=sbodash.sitedir+"/reports/{:s}.html".format(report_name)
report_datetime=datetime.datetime.now(datetime.timezone.utc)
#-----------------------------------------------------------------------
data_path=sbodash.datadir+"/maintainers.data"
with open(data_path,"rb") as data_file:
(Maintainerinfo, Stats) = pickle.load(data_file)
#-----------------------------------------------------------------------
jenv = jinja2.Environment(loader=jinja2.FileSystemLoader(sbodash.templatesdir),autoescape=None)
mytemplate = jenv.get_template(os.path.basename(__file__).replace("render_","",1)+'.jinja2')
try:
os.makedirs(os.path.dirname(report_path))
except FileExistsError:
pass
with open(report_path,"w",encoding="utf-8") as report_file:
print(
mytemplate.render(
siteurl=sbodash.siteurl,
page_title=report_title,
page_datetime=report_datetime.strftime("%Y-%m-%d %T %Z"),
Maintainerinfo=Maintainerinfo,
Stats=Stats
),
file=report_file
)
#-----------------------------------------------------------------------
# Finished!
#!/usr/bin/python3
"""
SBo Dashboard
render_problems
Create a report of SBo problems
David Spencer 2018-08-05
See LICENCE for copyright information
Requires: python3, and the following (built with python3 support)
Jinja2
MarkupSafe
"""
import sys
import os
import datetime
import pickle
import jinja2
import sbodash
report_name=os.path.basename(__file__).replace("render_","",1)
report_title="SBo {:s}".format(report_name.capitalize())
report_path=sbodash.sitedir+"/reports/{:s}.html".format(report_name)
report_datetime=datetime.datetime.now(datetime.timezone.utc)
#-----------------------------------------------------------------------
data_path=sbodash.datadir+"/problems.data"
with open(data_path,"rb") as data_file:
(Problems, Stats) = pickle.load(data_file)
#-----------------------------------------------------------------------
jenv = jinja2.Environment(loader=jinja2.FileSystemLoader(sbodash.templatesdir),autoescape=None)
mytemplate = jenv.get_template(os.path.basename(__file__).replace("render_","",1)+'.jinja2')
try:
os.makedirs(os.path.dirname(report_path))
except FileExistsError:
pass
with open(report_path,"w",encoding="utf-8") as report_file:
print(
mytemplate.render(
siteurl=sbodash.siteurl,
page_title=report_title,
page_datetime=report_datetime.strftime("%Y-%m-%d %T %Z"),
Problems=Problems,
Stats=Stats
),
file=report_file
)
#-----------------------------------------------------------------------
# Finished!
#!/bin/sh
#-----------------------------------------------------------------------
# SBo Dashboard
# render_support -- sync the 'support' directory and files onto the site
# David Spencer 2018-08-11
# See LICENCE for copyright information
#-----------------------------------------------------------------------
set -eu
rsync -a --delete "$DASHDIR"/support/ "$SITEDIR"/support/
exit 0
#-----------------------------------------------------------------------
#!/bin/sh
#-----------------------------------------------------------------------
# SBo Dashboard
# runall -- create/refresh the entire SBo Dashboard site
# David Spencer 2018-08-11
# See LICENCE for copyright information
#-----------------------------------------------------------------------
set -eu
export UPDATED=${UPDATED:-$(date "+%s")}
export PYTHONDONTWRITEBYTECODE="kthxbye"
CONFIG=${CONFIG:-$(pwd)/sbo-dashboard.conf}
. "$CONFIG" || { echo "Invalid config: " >&2; exit 1; }
mkdir -p "$DATADIR" "$SBDIR" "$LOGSDIR" "$SITEDIR"
LOGFILE="$LOGSDIR"/runall_$(date -d "@${UPDATED}" "+%F").log
echo "$(date "+%F %T") $(basename "$0") started timestamp=${UPDATED}" >>"$LOGFILE"
for run in \
update_slackbuilds \
update_maintainers \
update_problems \
render_index \
render_support \
render_maintainers \
render_problems \
; do
if [ -x "${run}" ]; then
echo "$(date "+%F %T") ${run} started" >>"$LOGFILE"
./"${run}" >>"$LOGFILE" 2>&1
stat=$?
if [ "$stat" = 0 ]; then
echo "$(date "+%F %T") ${run} finished ok" >>"$LOGFILE"
else
echo "$(date "+%F %T") ${run} failed status=${stat}" >>"$LOGFILE"
fi
else
echo "$(date "+%F %T") ${run} skipped" >>"$LOGFILE"
fi
done
echo "$(date "+%F %T") $(basename "$0") finished" >>"$LOGFILE"
exit 0
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# SBo Dashboard
# sbo-dashboard.conf -- config file
#-----------------------------------------------------------------------
export DASHDIR=$(pwd)
# Storage for working data
export DATADIR="$DASHDIR"/data
# Storage for a SlackBuilds.org git clone
export SBDIR="$DATADIR"/slackbuilds
# Logs are here
export LOGSDIR="$DASHDIR"/logs
# Jinja2 templates are here
export TEMPLATESDIR="$DASHDIR"/templates
# The rendered site is here
export SITEDIR="$DASHDIR"/site
# The rendered site is served at this URL
export SITEURL="http://localhost/sbo-dash"
# Statistics retention period (days)
export KEEPSTATS="7"
#-----------------------------------------------------------------------
#!/usr/bin/python3 -B
# SBo Dashboard
# sbodash.py
# Get config, define common variables and functions
# David Spencer 2018-08-13
# See LICENCE for copyright information
import os
import pickle
import logging
import globals
import stats
#-----------------------------------------------------------------------
logging.info("Run started")
from support import update_slackbuilds, render_index, render_support
update_slackbuilds()
render_index()
render_support()
from maintainers import update_maintainers, render_maintainers
update_maintainers()
render_maintainers()
from problems import update_repology_problems, update_download_problems,
update_download_problems, update_security_problems, render_problems
update_repology_problems()
update_download_problems()
update_security_problems()
update_build_problems()
render_problems()
stats.SaveStatsHistory()
logging.info("Run finished")
#-----------------------------------------------------------------------
;-----------------------------------------------------------------------
; SBo Dashboard
; sbo-dashboard.ini -- config file
;
; Why is this a stinking ini file?
; 1. Because this config format has Python standard library support
; 2. Because this config format is anecdotally idiot-proof
;
; Why do ini files stink?
; 1. Because they are unnecessarily complex
; 2. Because they are empirically not idiot-proof; for example,
; quotes surrounding values are significant, so don't do that
;
; You can override these definitions with environment variables,
; for example
; export SBDIR=/store/slackbuilds
;
;-----------------------------------------------------------------------
[Locations]
; path to the directory containing sbo-dashboard
; (if empty, the current working directory will be assumed)
dashdir =
; Jinja2 templates are here
templatesdir = templates
; Storage for working data (will be created if necessary)
datadir = data
; Storage for a SlackBuilds.org git clone (will be created if necessary)
sbdir = data/slackbuilds
; The rendered site is here (will be created if necessary)
sitedir = site
; The rendered site is served at this URL
siteurl = http://localhost/sbodash
;-----------------------------------------------------------------------
[Logging]
; Logs are here (will be created if necessary)
logsdir = logs
; Standard Python log level: DEBUG INFO WARNING ERROR CRITICAL
loglevel = INFO
;-----------------------------------------------------------------------
[Statistics]
; Statistics retention period (days)
keepstats = 7
;-----------------------------------------------------------------------
# SBo Dashboard
# sbodash.py
# Get config from environment, define common variables and functions
# David Spencer 2018-08-13
# See LICENCE for copyright information
import os
#-----------------------------------------------------------------------
try:
dashdir = os.getenv("DASHDIR")
datadir = os.getenv("DATADIR")
sbdir = os.getenv("SBDIR")
logsdir = os.getenv("LOGSDIR")
templatesdir = os.getenv("TEMPLATESDIR")
sitedir = os.getenv("SITEDIR")
siteurl = os.getenv("SITEURL")
except:
print("Environment variables not set.", file=sys.stderr)
#-----------------------------------------------------------------------
def unspamtrap(addy):
# Remove typical "anti-spam" obfuscations from email addresses.
fixedaddy = addy
fixedaddy=fixedaddy.replace(".nospam","")
fixedaddy=fixedaddy.replace("(removeNOandSPAM)","")
fixedaddy=fixedaddy.replace("SPAM","")
fixedaddy=fixedaddy.replace(" at ","@")
fixedaddy=fixedaddy.replace("<at>","@")
fixedaddy=fixedaddy.replace("_at_","@")
fixedaddy=fixedaddy.replace("-at-","@")
fixedaddy=fixedaddy.replace("{at}","@")
fixedaddy=fixedaddy.replace("~at~","@")
fixedaddy=fixedaddy.replace("(at)","@")
fixedaddy=fixedaddy.replace("[at]","@")
fixedaddy=fixedaddy.replace("[AT]","@")
fixedaddy=fixedaddy.replace("AT","@")
fixedaddy=fixedaddy.replace("[@]","@")
fixedaddy=fixedaddy.replace("{@}","@")
fixedaddy=fixedaddy.replace("_@_","@")
fixedaddy=fixedaddy.replace(" dot ",".")
fixedaddy=fixedaddy.replace("<dot>",".")
fixedaddy=fixedaddy.replace("[dot]",".")
fixedaddy=fixedaddy.replace("[DOT]",".")
fixedaddy=fixedaddy.replace("DOT",".")
fixedaddy=fixedaddy.replace("[.]",".")
fixedaddy=fixedaddy.replace("{.}",".")
fixedaddy=fixedaddy.replace("{dot}",".")
fixedaddy=fixedaddy.replace("(dot)",".")
fixedaddy=fixedaddy.replace("~dot~",".")
fixedaddy=fixedaddy.replace(" ","")
fixedaddy=fixedaddy.lower()
return(fixedaddy)
#-----------------------------------------------------------------------
Stats={}
def addstats(key,n=1):
"""
Add 'n' to Stats[key], or set it if key doesn't exist.
"""
if key in Stats:
Stats[key] += n
else:
Stats[key] = n
#-----------------------------------------------------------------------
"""
SBo Dashboard
stats.py
Statistics: global variables, persistent storage and utility functions
David Spencer 2018-08-27
See LICENCE for copyright information
"""
import os
import datetime
import pickle
import logging
import globals
#-----------------------------------------------------------------------
Stats={}
StatsHistory={}
#-----------------------------------------------------------------------
# Initialise StatsHistory from storage
stats_path=os.path.join(globals.datadir,"StatsHistory.p")
try:
with open(stats_path,"rb") as stats_file:
StatsHistory = pickle.load(stats_file)
except:
# no storage => this is the first ever run, so don't error
pass
#-----------------------------------------------------------------------
# Handy functions
def addstats(key,n=1):
"""
Add 'n' to Stats[key], or set it if key doesn't exist.
"""
if key in Stats:
Stats[key] += n
else:
Stats[key] = n
return
def SaveStatsHistory():
"""
Expire old stats history and insert new stats.
"""
expdate = globals.updateref - datetime.timedelta(days=globals.keepstats)
for olddate in StatsHistory:
if olddate < expdate:
del StatsHistory[olddate]
StatsHistory.update({globals.updateref: Stats})
with open(stats_path,"wb") as stats_file:
pickle.dump(StatsHistory, file=stats_file)
return
"""
SBo Dashboard
support.py
Create the site index and support files, update the slackbuilds
David Spencer 2018-08-05
See LICENCE for copyright information
Requires: python3, and the following (all built with python3 support)
Jinja2
MarkupSafe
"""
import os
import datetime
import jinja2
import utils
import subprocess
import logging
import globals
#-----------------------------------------------------------------------
def render_index():
"""
Render the site index page.
"""
logging.info("started")
utils.renderer(page_name="index",page_title="SBo Dashboard Index")
logging.info("finished")
#-----------------------------------------------------------------------
def render_support():
"""
Copy the support files into the site's support directory.
"""
logging.info("started")
subprocess.run(
"rsync -a --delete support/ {:s}/support/".format(globals.sitedir),
shell=True,
cwd=globals.dashdir,
check=True,
stdout=subprocess.DEVNULL
)
logging.info("finished")
#-----------------------------------------------------------------------
def update_slackbuilds():
logging.info("started")
#### needs error handling
if os.path.isfile(os.path.join(globals.sbdir,"ChangeLog.txt")):
subprocess.run(
"git checkout master ; git pull --ff-only origin",
shell=True,
cwd=globals.sbdir,
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
else:
subprocess.run(
"git clone git://git.slackbuilds.org/slackbuilds.git {:s}".format(globals.sbdir),
shell=True,
cwd=globals.dashdir,
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
logging.info("finished")
#-----------------------------------------------------------------------
......@@ -34,7 +34,7 @@
</thead>
<tbody>
{% for mname, mdeets in Maintainerinfo|dictsort(true) %}
{% for mname, mdeets in Table|dictsort(true) %}
<tr id="m{{ mdeets['mnum'] }}" class="{{ 'm-'+mdeets['status'] }}">
<td class="m">{{ mdeets["tdmaintainer"] }}</td>
<td class="l">{{ mdeets["tdlatest"] }}</td>
......@@ -49,7 +49,7 @@
</table>
{% for mname, mdeets in Maintainerinfo.items() %}
{% for mname, mdeets in Table.items() %}
<div id="sb{{ mdeets['mnum'] }}" class="modal">
<div>
<a href="#close" title="Close" class="closebutton">&otimes;</a>
......
......@@ -8,7 +8,7 @@
<p>Updated: {{ page_datetime }}</p>
<p>Problems: {{ "{:d}".format(Stats["problems"]) }}</p>
<p>Problems: {{ "{:d}".format(Stats["probcount"]) }}</p>
<table class="sortable">
......@@ -17,20 +17,24 @@
<tr>
<th class="catnam"><a href="#!">catnam</a></th>
<th class="prgnam"><a href="#!">prgnam</a></th>
<th class="source"><a href="#!">source</a></th>
<th class="problem"><a href="#!">problem</a></th>
<th class="description"><a href="#!">description</a></th>
<th class="since"><a href="#!">since</a></th>
<th class="maintainer"><a href="#!">maintainer</a></th>
</tr>
</thead>
<tbody>
{% for p in Problems %}
<tr id="{{ p['catnam']+'/'+p['prgnam'] }}">
<td class="catnam">{{ p['catnam'] }}</td>
<td class="prgnam"><a href="{{ siteurl }}/slackbuilds/{{ p['catnam']+"/"+p['prgnam'] }}" target="_blank">{{ p['prgnam'] }}</a></td>
<td class="problem">{{ p['problem'] }}</td>
<td class="description">{{ p['description']|urlize(40,target='_blank') }}</td>
<td class="maintainer"><a href="{{ siteurl }}/maintainers/{{ p['maintainer'] }}" target="_blank">{{ p['maintainer'] }}</a></td>
{% for problem in Table %}
<tr id="{{ problem['catnam']+'/'+problem['prgnam'] }}">
<td class="catnam">{{ problem['catnam'] }}</td>
<td class="prgnam"><a href="{{ siteurl }}/slackbuilds/{{ problem['catnam']+"/"+problem['prgnam'] }}" target="_blank">{{ problem['prgnam'] }}</a></td>
<td class="source">{{ problem['source'] }}</td>
<td class="problem">{{ problem['problem'] }}</td>
<td class="description">{{ problem['description']|urlize(40,target='_blank') }}</td>
<td class="since">{{ problem['since'] }}</td>
<td class="maintainer"><a href="{{ siteurl }}/maintainers/{{ problem['maintainer'] }}" target="_blank">{{ problem['maintainer'] }}</a></td>
</tr>
{% endfor %}
</tbody>
......
This diff is collapsed.
#!/bin/sh
#-----------------------------------------------------------------------
# SBo Dashboard
# update_slackbuilds -- create or refresh the SlackBuilds.org repo
# David Spencer 2018-08-12
# See LICENCE for copyright information
#
# Usage: update_slackbuilds
# Environment variables:
# SBDIR -- path to SBo git repo -- if unset, just bomb out
#
# Note: execution order is significant -- this must be run before other
# updates that depend on it (i.e., most of them)
#-----------------------------------------------------------------------
set -eu
mkdir -p "${SBDIR}"
if [ -f "${SBDIR}"/ChangeLog.txt ]; then
cd "${SBDIR}"
git fetch --all || true
git checkout master
git merge --ff-only origin/master || true
cd - >/dev/null
else
git clone git://git.slackbuilds.org/slackbuilds.git "${SBDIR}"
fi
exit 0
#-----------------------------------------------------------------------
"""
SBo Dashboard
utils.py
Utility functions
David Spencer 2018-08-27
See LICENCE for copyright information
"""
import os
import logging
import pickle
import jinja2
import globals
import stats
#-----------------------------------------------------------------------
def unspamtrap(address):
# Remove typical "anti-spam" obfuscations from an email address.
fixedaddress = address
fixedaddress=fixedaddress.replace(".nospam","")
fixedaddress=fixedaddress.replace("(removeNOandSPAM)","")
fixedaddress=fixedaddress.replace("SPAM","")
fixedaddress=fixedaddress.replace(" at ","@")
fixedaddress=fixedaddress.replace("<at>","@")
fixedaddress=fixedaddress.replace("_at_","@")
fixedaddress=fixedaddress.replace("-at-","@")
fixedaddress=fixedaddress.replace("{at}","@")
fixedaddress=fixedaddress.replace("~at~","@")
fixedaddress=fixedaddress.replace("(at)","@")
fixedaddress=fixedaddress.replace("[at]","@")
fixedaddress=fixedaddress.replace("[AT]","@")
fixedaddress=fixedaddress.replace("AT","@")
fixedaddress=fixedaddress.replace("[@]","@")
fixedaddress=fixedaddress.replace("{@}","@")
fixedaddress=fixedaddress.replace("_@_","@")
fixedaddress=fixedaddress.replace(" dot ",".")
fixedaddress=fixedaddress.replace("<dot>",".")
fixedaddress=fixedaddress.replace("[dot]",".")
fixedaddress=fixedaddress.replace("[DOT]",".")
fixedaddress=fixedaddress.replace("DOT",".")
fixedaddress=fixedaddress.replace("[.]",".")
fixedaddress=fixedaddress.replace("{.}",".")
fixedaddress=fixedaddress.replace("{dot}",".")
fixedaddress=fixedaddress.replace("(dot)",".")
fixedaddress=fixedaddress.replace("~dot~",".")
fixedaddress=fixedaddress.replace(" ","")
fixedaddress=fixedaddress.lower()
return(fixedaddress)
#-----------------------------------------------------------------------
def renderer( page_name,
page_subdir="",
page_title="",
Table=[] ):
"""
Render a page
"""
if page_title == "":
page_title = page_name.capitalize()
page_path=os.path.join(globals.sitedir,page_subdir,page_name+".html")
os.makedirs(os.path.dirname(page_path),exist_ok=True)
jenv = jinja2.Environment(loader=jinja2.FileSystemLoader(globals.templatesdir),autoescape=None)
mytemplate = jenv.get_template(page_name+".jinja2")
with open(page_path,"w",encoding="utf-8") as page_file:
print(
mytemplate.render(
siteurl=globals.siteurl,
page_title=page_title,
page_datetime=globals.updateref.strftime("%Y-%m-%d %T %Z"),
Table=Table,
Stats=stats.Stats
),
file=page_file
)
#-----------------------------------------------------------------------
# Persistent data storage for modules
def load(obj,filename):
"""
Persistent storage -- load obj from filename.
If the file isn't found, we don't set obj -- the caller should have
initialised an empty object in whatever way it wants
"""
data_path=os.path.join(globals.datadir,filename)
try:
with open(data_path,"rb") as data_file:
obj = pickle.load(data_file)
except FileNotFoundError:
pass
def save(filename,obj):
"""
Persistent storage -- save obj into filename.
Hey, Python weenies! My save is just like my load!
This is what symmetry looks like, Guido!
"""
data_path=os.path.join(globals.datadir,filename)
with open(data_path,"wb") as data_file:
pickle.dump(obj, file=data_file)
#-----------------------------------------------------------------------