slackbuilds.py 5.6 KB
Newer Older
David Spencer's avatar
David Spencer committed
1 2 3 4 5
"""
SBo Dashboard
slackbuilds.py
Functions and dict for accessing a local SlackBuilds.org clone

David Spencer's avatar
David Spencer committed
6
David Spencer 2018
David Spencer's avatar
David Spencer committed
7 8 9 10 11 12 13 14
See LICENCE for copyright information
"""

import os
import re
import glob
import subprocess
import logging
David Spencer's avatar
David Spencer committed
15
import datetime
David Spencer's avatar
David Spencer committed
16 17 18

import globals
import stats
David Spencer's avatar
David Spencer committed
19 20
import utils
import gitfuncs
David Spencer's avatar
David Spencer committed
21

David Spencer's avatar
David Spencer committed
22 23
SBinfo = {}
SBindex = []
David Spencer's avatar
David Spencer committed
24

David Spencer's avatar
David Spencer committed
25 26
#-----------------------------------------------------------------------

David Spencer's avatar
David Spencer committed
27
def process_slackbuilds(sbinfofile,sbindexfile,updatefile):
David Spencer's avatar
David Spencer committed
28
    """
David Spencer's avatar
David Spencer committed
29 30
    Read data from SlackBuilds info files and git into SBinfo and SBindex
    and save them into the persistent data files
David Spencer's avatar
David Spencer committed
31
    """
David Spencer's avatar
David Spencer committed
32
    global SBinfo, SBindex
David Spencer's avatar
David Spencer committed
33 34 35
    logging.debug("reading .info files")

    # These regexes are used >7000 times, so we might as well compile them.
David Spencer's avatar
David Spencer committed
36 37 38 39
    re_version = re.compile(r"VERSION=\".*\"")
    re_homepage = re.compile(r"HOMEPAGE=\".*\"")
    re_maint = re.compile(r"MAINTAINER=\".*\"")
    re_email = re.compile(r"EMAIL=\".*\"")
David Spencer's avatar
David Spencer committed
40

David Spencer's avatar
David Spencer committed
41 42
    infocount = 0
    catnamlist = []
David Spencer's avatar
David Spencer committed
43

David Spencer's avatar
David Spencer committed
44 45 46
    for infofile in sorted(glob.glob(globals.sbdir+"/*/*/*.info")):
        infocount += 1
        logging.debug(infofile)
David Spencer's avatar
David Spencer committed
47

David Spencer's avatar
David Spencer committed
48 49
        catnam = infofile.split("/")[-3]
        prgnam = infofile.split("/")[-2]
David Spencer's avatar
David Spencer committed
50 51 52 53 54 55 56 57

        try:
            with open(infofile, "r", encoding="utf-8") as openinfo:
                info = openinfo.read()
        except:
            with open(infofile, "r", encoding="latin-1") as openinfo:
                info = openinfo.read()

David Spencer's avatar
David Spencer committed
58 59 60 61
        versionarray = re_version.findall(info)
        homepagearray = re_homepage.findall(info)
        maintainerarray = re_maint.findall(info)
        emailarray = re_email.findall(info)
David Spencer's avatar
David Spencer committed
62
        if len(versionarray) == 1 and len(homepagearray) == 1 and len(maintainerarray) == 1 and len(emailarray) == 1:
David Spencer's avatar
David Spencer committed
63 64 65 66
            version = versionarray[0][9:-1]
            homepage = homepagearray[0][10:-1]
            maintainer = maintainerarray[0][12:-1]
            email = emailarray[0][7:-1]
David Spencer's avatar
David Spencer committed
67 68 69
        else:
            # We're not here to validate the .info files.
            # If there's not exactly one MAINTAINER and EMAIL, bollocks to it.
David Spencer's avatar
David Spencer committed
70
            logging.warn("{:s} has missing or broken or duplicated entries".format(infofile))
David Spencer's avatar
David Spencer committed
71 72
            continue

David Spencer's avatar
David Spencer committed
73 74
        gitnewest = gitfuncs.getnewest(catnam,prgnam)

David Spencer's avatar
David Spencer committed
75 76 77 78
        SBinfo.update({ prgnam: { "catnam": catnam,
                                  "version": version,
                                  "homepage": homepage,
                                  "maintainer": maintainer,
David Spencer's avatar
David Spencer committed
79 80
                                  "email": email,
                                  "gitnewest": gitnewest
David Spencer's avatar
David Spencer committed
81
                                  } })
David Spencer's avatar
David Spencer committed
82

David Spencer's avatar
David Spencer committed
83 84 85 86 87 88
        if catnam not in catnamlist:
            catnamlist.append(catnam)
            SBindex.append({"catnam":catnam, "prgnamlist":[prgnam]})
        else:
            SBindex[-1]["prgnamlist"].append(prgnam)

David Spencer's avatar
David Spencer committed
89 90 91
    if infocount != len(SBinfo):
        logging.warn("{:d} duplicate prgnams in slackbuilds repo".format(infocount-len(SBinfo)))

David Spencer's avatar
David Spencer committed
92 93 94 95
    utils.save(SBinfo,sbinfofile)
    utils.save(SBindex,sbindexfile)
    utils.save(globals.updateref,updatefile)

David Spencer's avatar
David Spencer committed
96
    stats.setStats("slackbuilds.count",infocount)
David Spencer's avatar
David Spencer committed
97
    logging.debug("finished -- {:d} .info files".format(infocount))
David Spencer's avatar
David Spencer committed
98 99

#-----------------------------------------------------------------------
David Spencer's avatar
David Spencer committed
100 101 102 103 104

def load_slackbuilds():
    """
    Update the local slackbuilds git repo from remote, and load the data
    """
David Spencer's avatar
David Spencer committed
105
    global SBinfo, SBindex
David Spencer's avatar
David Spencer committed
106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131
    logging.info("started")

    sbinfofile  = "slackbuilds_SBinfo.p"
    sbindexfile = "slackbuilds_SBindex.p"
    updatefile  = "slackbuilds_updated.p"

    if not os.path.isfile(os.path.join(globals.sbdir,"ChangeLog.txt")):
        logging.info("cloning from SBo repo")
        gitfuncs.clone_sbo()
        process_slackbuilds(sbinfofile,sbindexfile,updatefile)
    else:
        prev_updated_sbo = utils.load(updatefile)
        if prev_updated_sbo is None or prev_updated_sbo + datetime.timedelta(days=globals.upd_sbo) < globals.updateref:
            logging.info("update needed; pulling from SBo repo")
            gitfuncs.pull_sbo()
            process_slackbuilds(sbinfofile,sbindexfile,updatefile)
        else:
            SBinfo=utils.load(sbinfofile)
            SBindex=utils.load(sbindexfile)
            logging.debug("update not needed")

    logging.info("finished")

#-----------------------------------------------------------------------

def render_slackbuild_detail():
David Spencer's avatar
David Spencer committed
132 133 134 135
    """
    Render all the slackbuild detail pages
    """
    global SBinfo, SBindex
David Spencer's avatar
David Spencer committed
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
    logging.info("started")

    for prgnam, prginfo in SBinfo.items():
        catnam = prginfo["catnam"]
        logging.debug(catnam+"/"+prgnam)
        gitnewest = prginfo["gitnewest"]
        commitinfo = gitnewest.split(" ")
        commitdate = " ".join(commitinfo[0:3])
        commithash = commitinfo[3]
        commitsubj = " ".join(commitinfo[4:])
        SlackbuildDetail = { "prgnam": prgnam,
                             "catnam": catnam,
                             "version": prginfo["version"],
                             "homepage": prginfo["homepage"],
                             "maintainer": prginfo["maintainer"],
                             "email": prginfo["email"],
                             "commitdate": commitdate,
                             "commithash": commithash,
                             "commitsubj": commitsubj
                             }
        utils.renderer( page_subdir="slackbuilds/"+catnam,
                        page_name=prgnam,
                        page_title=catnam+"/"+prgnam,
                        template="slackbuild_detail",
                        TemplateData=SlackbuildDetail )

    logging.info("finished")

#-----------------------------------------------------------------------