Commit 4b1c5dfb authored by David Spencer's avatar David Spencer

Code tidyups

parent e8a4bece
...@@ -15,11 +15,14 @@ import globals ...@@ -15,11 +15,14 @@ import globals
#----------------------------------------------------------------------- #-----------------------------------------------------------------------
def getnewest(catnam,prgnam): def getnewest(catnam,prgnam):
"""
Return git log information for the newest commit in catnam/prgnam
(this is painfully slow)
"""
glog = subprocess.run( glog = subprocess.run(
"git log --pretty=\"format:%ai %h %s\" -n 1 .", "git log --pretty=\"format:%ai %h %s\" -n 1 {:s}".format(catnam+"/"+prgnam),
shell=True, shell=True,
cwd=os.path.join(globals.sbdir,catnam,prgnam), cwd=globals.sbdir,
check=True, check=True,
stdout=subprocess.PIPE stdout=subprocess.PIPE
) )
...@@ -29,6 +32,9 @@ def getnewest(catnam,prgnam): ...@@ -29,6 +32,9 @@ def getnewest(catnam,prgnam):
#----------------------------------------------------------------------- #-----------------------------------------------------------------------
def clone_sbo(): def clone_sbo():
"""
Initial clone from SlackBuilds.org
"""
subprocess.run( subprocess.run(
"git clone git://git.slackbuilds.org/slackbuilds.git {:s}".format(globals.sbdir), "git clone git://git.slackbuilds.org/slackbuilds.git {:s}".format(globals.sbdir),
shell=True, shell=True,
...@@ -41,6 +47,9 @@ def clone_sbo(): ...@@ -41,6 +47,9 @@ def clone_sbo():
#----------------------------------------------------------------------- #-----------------------------------------------------------------------
def pull_sbo(): def pull_sbo():
"""
Make sure we're on 'master' and do a fast forward pull from origin
"""
subprocess.run( subprocess.run(
"git checkout master ; git pull --ff-only origin", "git checkout master ; git pull --ff-only origin",
shell=True, shell=True,
......
...@@ -34,7 +34,7 @@ def get_repology_by_maintainer(email,sess): ...@@ -34,7 +34,7 @@ def get_repology_by_maintainer(email,sess):
""" """
repologyAPI = "https://repology.org/api/v1/metapackages/" repologyAPI = "https://repology.org/api/v1/metapackages/"
filters = {"maintainer":email} filters = {"maintainer": email}
metapkglist = collections.OrderedDict() metapkglist = collections.OrderedDict()
metapkgchunk = sess.get(repologyAPI,params=filters).json() metapkgchunk = sess.get(repologyAPI,params=filters).json()
...@@ -72,7 +72,11 @@ def get_repology_all_maintainers(sess): ...@@ -72,7 +72,11 @@ def get_repology_all_maintainers(sess):
#----------------------------------------------------------------------- #-----------------------------------------------------------------------
def setlatest(mntnam,cdate,commit): def setlatest(mntnam,cdate,commit):
# Record commit details into Maintainerinfo[mntnam] """
Record commit details into Maintainerinfo[mntnam]
"""
global Maintainerinfo, Maintainerindex
if "latest" not in Maintainerinfo[mntnam].keys() or Maintainerinfo[mntnam]["latest"]["date"] < cdate: if "latest" not in Maintainerinfo[mntnam].keys() or Maintainerinfo[mntnam]["latest"]["date"] < cdate:
Maintainerinfo[mntnam]["latest"] = {"date":cdate, "commit":commit} Maintainerinfo[mntnam]["latest"] = {"date":cdate, "commit":commit}
...@@ -82,6 +86,7 @@ def process_maintainers(): ...@@ -82,6 +86,7 @@ def process_maintainers():
""" """
Read retrieved info from Repology and populate Maintainerinfo Read retrieved info from Repology and populate Maintainerinfo
""" """
global Maintainerinfo, Maintainerindex
logging.info("started") logging.info("started")
# (1) Correlate all the names and all the emails of each # (1) Correlate all the names and all the emails of each
...@@ -238,6 +243,7 @@ def load_maintainers(): ...@@ -238,6 +243,7 @@ def load_maintainers():
""" """
Load maintainers from repology into 'Maintainerinfo' Load maintainers from repology into 'Maintainerinfo'
""" """
global Maintainerinfo, Maintainerindex
logging.info("started") logging.info("started")
repo = "slackbuilds" repo = "slackbuilds"
...@@ -275,16 +281,23 @@ def render_maintainers(): ...@@ -275,16 +281,23 @@ def render_maintainers():
""" """
Render the maintainers report page Render the maintainers report page
""" """
global Maintainerinfo, Maintainerindex
logging.info("started") logging.info("started")
utils.renderer( page_subdir="reports", utils.renderer( page_subdir="reports",
page_name="maintainers.html", page_name="maintainers.html",
page_title="SBo Maintainers Report", page_title="SBo Maintainers Report",
TemplateData=Maintainerinfo ) TemplateData=Maintainerinfo )
logging.info("finished") logging.info("finished")
#----------------------------------------------------------------------- #-----------------------------------------------------------------------
def render_maintainer_detail(): def render_maintainer_detail():
"""
Render all the maintainer detail pages
"""
global Maintainerinfo, Maintainerindex
logging.info("started") logging.info("started")
for mntnam in SBinfo.Maintainerinfo.items(): for mntnam in SBinfo.Maintainerinfo.items():
......
...@@ -77,9 +77,10 @@ def process_repology_problems(rplist,data_updated): ...@@ -77,9 +77,10 @@ def process_repology_problems(rplist,data_updated):
def get_repology_problems(): def get_repology_problems():
""" """
Retrieve repology problems from remote server. Retrieve repology problems from remote server
""" """
repologyAPI = "https://repology.org/api/v1/repository/{:s}/problems".format(repo) repologyAPI = "https://repology.org/api/v1/repository/{:s}/problems".format(repo)
# The API spec doesn't say whether the 'problems' result set is is chunked. # The API spec doesn't say whether the 'problems' result set is is chunked.
# In practice it seems to be truncated at 500 problems, so let's not go there. # In practice it seems to be truncated at 500 problems, so let's not go there.
repology_problems = requests.get(repologyAPI).json() repology_problems = requests.get(repologyAPI).json()
...@@ -146,11 +147,14 @@ def render_problems(): ...@@ -146,11 +147,14 @@ def render_problems():
""" """
Render the problems report page Render the problems report page
""" """
global Problems
logging.info("started") logging.info("started")
utils.renderer( page_subdir="reports", utils.renderer( page_subdir="reports",
page_name="problems.html", page_name="problems.html",
page_title="SBo Problems Report", page_title="SBo Problems Report",
TemplateData=Problems ) TemplateData=Problems )
logging.info("finished") logging.info("finished")
#----------------------------------------------------------------------- #-----------------------------------------------------------------------
...@@ -29,6 +29,7 @@ def process_slackbuilds(sbinfofile,sbindexfile,updatefile): ...@@ -29,6 +29,7 @@ def process_slackbuilds(sbinfofile,sbindexfile,updatefile):
Read data from SlackBuilds info files and git into SBinfo and SBindex Read data from SlackBuilds info files and git into SBinfo and SBindex
and save them into the persistent data files and save them into the persistent data files
""" """
global SBinfo, SBindex
logging.debug("reading .info files") logging.debug("reading .info files")
# These regexes are used >7000 times, so we might as well compile them. # These regexes are used >7000 times, so we might as well compile them.
...@@ -101,6 +102,7 @@ def load_slackbuilds(): ...@@ -101,6 +102,7 @@ def load_slackbuilds():
""" """
Update the local slackbuilds git repo from remote, and load the data Update the local slackbuilds git repo from remote, and load the data
""" """
global SBinfo, SBindex
logging.info("started") logging.info("started")
sbinfofile = "slackbuilds_SBinfo.p" sbinfofile = "slackbuilds_SBinfo.p"
...@@ -127,6 +129,10 @@ def load_slackbuilds(): ...@@ -127,6 +129,10 @@ def load_slackbuilds():
#----------------------------------------------------------------------- #-----------------------------------------------------------------------
def render_slackbuild_detail(): def render_slackbuild_detail():
"""
Render all the slackbuild detail pages
"""
global SBinfo, SBindex
logging.info("started") logging.info("started")
for prgnam, prginfo in SBinfo.items(): for prgnam, prginfo in SBinfo.items():
......
...@@ -34,15 +34,17 @@ except: ...@@ -34,15 +34,17 @@ except:
def setStats(key,n=0): def setStats(key,n=0):
""" """
Set Stats[key] to 'n', default 0. Set Stats[key] to 'n', default 0
""" """
global Stats, StatsHistory
Stats[key] = n Stats[key] = n
return return
def addStats(key,n=1): def addStats(key,n=1):
""" """
Add 'n' to Stats[key], or set it if key doesn't exist. Add 'n' to Stats[key], or set it if key doesn't exist
""" """
global Stats, StatsHistory
if key in Stats: if key in Stats:
Stats[key] += n Stats[key] += n
else: else:
...@@ -51,8 +53,9 @@ def addStats(key,n=1): ...@@ -51,8 +53,9 @@ def addStats(key,n=1):
def getStats(key): def getStats(key):
""" """
Get Stats[key], returning 0 when key doesn't exist. Get Stats[key], returning 0 when key doesn't exist
""" """
global Stats, StatsHistory
if key in Stats: if key in Stats:
return Stats[key] return Stats[key]
else: else:
...@@ -60,8 +63,9 @@ def getStats(key): ...@@ -60,8 +63,9 @@ def getStats(key):
def saveStatsHistory(): def saveStatsHistory():
""" """
Expire old stats history and insert new stats. Expire old stats history and insert new stats
""" """
global Stats, StatsHistory
expdate = globals.updateref - datetime.timedelta(days=globals.keepstats) expdate = globals.updateref - datetime.timedelta(days=globals.keepstats)
for olddate in StatsHistory: for olddate in StatsHistory:
if olddate < expdate: if olddate < expdate:
......
...@@ -15,27 +15,27 @@ import logging ...@@ -15,27 +15,27 @@ import logging
import globals import globals
import utils import utils
from maintainers import Maintainerindex import maintainers
from slackbuilds import SBindex import slackbuilds
#----------------------------------------------------------------------- #-----------------------------------------------------------------------
def render_indexes(): def render_indexes():
""" """
Render the index pages and the homepage. Render the index pages and the homepage
""" """
logging.info("started") logging.info("started")
utils.renderer( page_subdir="maintainers", utils.renderer( page_subdir="maintainers",
page_title="SBo Maintainers Index", page_title="SBo Maintainers Index",
template="maintainers_index", template="maintainers_index",
TemplateData=Maintainerindex TemplateData=maintainers.Maintainerindex
) )
utils.renderer( page_subdir="slackbuilds", utils.renderer( page_subdir="slackbuilds",
page_title="SBo SlackBuilds Index", page_title="SBo SlackBuilds Index",
template="slackbuilds_index", template="slackbuilds_index",
TemplateData=SBindex TemplateData=slackbuilds.SBindex
) )
# build logs -- for now, just create an empty directory # build logs -- for now, just create an empty directory
...@@ -51,7 +51,7 @@ def render_indexes(): ...@@ -51,7 +51,7 @@ def render_indexes():
def render_support(): def render_support():
""" """
Copy the support files into the site's support directory. Copy the support files into the site's support directory
""" """
logging.info("started") logging.info("started")
subprocess.run( subprocess.run(
......
...@@ -18,7 +18,9 @@ import stats ...@@ -18,7 +18,9 @@ import stats
#----------------------------------------------------------------------- #-----------------------------------------------------------------------
def unspamtrap(address): def unspamtrap(address):
# Remove typical "anti-spam" obfuscations from an email address. """
Remove typical "anti-spam" obfuscations from an email address
"""
fixedaddress = address fixedaddress = address
fixedaddress = fixedaddress.replace(".nospam","") fixedaddress = fixedaddress.replace(".nospam","")
...@@ -112,7 +114,7 @@ def load(filename): ...@@ -112,7 +114,7 @@ def load(filename):
def save(obj,filename): def save(obj,filename):
""" """
Persistent storage -- save obj into filename. Persistent storage -- save obj into filename
""" """
data_path = os.path.join(globals.datadir,filename) data_path = os.path.join(globals.datadir,filename)
with open(data_path,"wb") as data_file: with open(data_path,"wb") as data_file:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment