Commit 2fb48855 authored by Lucas Moura's avatar Lucas Moura

Merge tag 'upstream/0.7'

Upstream version 0.7

# gpg: Signature made Sun 07 Aug 2016 06:56:41 PM BRT using RSA key ID 2DA5B48C
# gpg: Good signature from "Lucas Albuquerque Medeiros de Moura <lucas.moura128@gmail.com>"
parents e472b9ca f1518355
......@@ -135,11 +135,12 @@ class PkgMatchDecider(xapian.MatchDecider):
xapian.MatchDecider.__init__(self)
self.pkgs_list = pkgs_list
def __call__(self, doc):
def __call__(self, xapian_document):
"""
True if the package is not already installed and is not a lib or a doc.
True if the package is not already installed and is not a lib or
a xapian_document.
"""
pkg = doc.get_data()
pkg = xapian_document.get_data()
is_new = pkg not in self.pkgs_list
is_new = is_new and ':' not in pkg
......@@ -180,6 +181,45 @@ class PkgExpandDecider(xapian.ExpandDecider):
return is_new_pkg
class PkgReverseDependeciesDecider(xapian.MatchDecider):
"""
Extend xapian.MatchDecider to consider only packages on valid list
"""
def __init__(self, reverse_dependencies, user_installed_pkgs):
"""
Set initial parameters.
"""
xapian.MatchDecider.__init__(self)
self.reverse_dependencies = reverse_dependencies
self.pkg_init_decider = PkgInitDecider()
self.pkg_match_decider = PkgMatchDecider(user_installed_pkgs)
self.cache = apt.Cache()
def __call__(self, xapian_document):
"""
True if the package is on pkg_list
"""
pkg = xapian_document.get_data()
if pkg not in self.reverse_dependencies:
return False
if pkg not in self.cache:
return False
if self.cache[pkg].section == 'doc':
return False
decider = self.pkg_init_decider
pkg_candidate = self.cache[pkg].candidate
if not decider.is_program_dependencies_installed(pkg_candidate):
return False
return self.pkg_match_decider(xapian_document)
class TagExpandDecider(xapian.ExpandDecider):
"""
......
......@@ -105,9 +105,15 @@ class Initialize:
try:
if os.path.exists(self.config.base_dir):
shutil.rmtree(self.config.base_dir)
if os.path.exists(self.config.user_data_dir):
shutil.rmtree(self.config.user_data_dir)
if os.path.exists(self.config.axi_desktopapps):
shutil.rmtree(self.config.axi_desktopapps)
if os.path.exists(self.config.filters_dir):
shutil.rmtree(self.config.filters_dir)
os.makedirs(self.config.base_dir)
os.makedirs(self.config.filters_dir)
except OSError:
raise
......
......@@ -13,11 +13,15 @@ class AppRecommender:
self.recommender = Recommender()
self.config = Config()
def make_recommendation(self, print_recommendation=True):
def make_recommendation(self, reference_pkgs=None,
print_recommendation=True):
begin_time = datetime.datetime.now()
logging.info("Computation started at %s" % begin_time)
user = LocalSystem()
if not reference_pkgs:
reference_pkgs = []
user = LocalSystem(reference_pkgs)
recommendation_size = Config().num_recommendations
user_recommendation = (self.recommender.get_recommendation(
user, recommendation_size))
......
......@@ -17,7 +17,6 @@ PERMISSION_DENIED = 3
def parse_options(args, config):
if args['strategy']:
config.strategy = args['strategy']
if args['debug']:
......@@ -32,10 +31,10 @@ def parse_options(args, config):
config.num_recommendations = args['num_recommendations']
def run_apprecommender():
def run_apprecommender(reference_pkgs):
try:
app_recommender = AppRecommender()
app_recommender.make_recommendation()
app_recommender.make_recommendation(reference_pkgs)
return SUCCESS
except xapian.DatabaseOpeningError:
return ERROR_INIT
......@@ -72,7 +71,8 @@ def run(args):
else:
config = Config()
parse_options(args, config)
return run_apprecommender()
reference_pkgs = args['packages']
return run_apprecommender(reference_pkgs)
def main():
......
......@@ -51,4 +51,9 @@ def get_parser():
help='classify recommendations and help apprecommender to improve',
action='store_true')
parser.add_argument(
'-p', '--packages',
help="Add reference package for strategy 'cbpkg'",
type=str, nargs='+', default=[])
return parser
......@@ -164,6 +164,8 @@ class Recommender:
self.strategy = strategy.ContentBased("half", profile_size)
elif strategy_str == "cbtm":
self.strategy = strategy.ContentBased("time", profile_size)
elif strategy_str == "cbpkg":
self.strategy = strategy.PackageReference("mix", profile_size)
elif strategy_str == "mlbva":
self.strategy = strategy.MachineLearningBVA("mlbva_mix",
profile_size)
......
......@@ -26,6 +26,7 @@ import operator
import pickle
import re
import recommender
import subprocess
import xapian
import numpy as np
......@@ -33,7 +34,8 @@ import numpy as np
from abc import ABCMeta, abstractmethod
from apprecommender.config import Config
from apprecommender.decider import PkgMatchDecider
from apprecommender.decider import (PkgMatchDecider,
PkgReverseDependeciesDecider)
from apprecommender.ml.bag_of_words import BagOfWords
from apprecommender.ml.bayes_matrix import BayesMatrix
from apprecommender.ml.data import MachineLearningData
......@@ -66,16 +68,20 @@ class ContentBased(RecommendationStrategy):
self.profile_size = profile_size
def get_sugestion_from_profile(self, rec, user, profile,
recommendation_size, because=True):
recommendation_size, because=True,
pkg_decider=None):
query = xapian.Query(xapian.Query.OP_OR, profile)
enquire = xapian.Enquire(rec.items_repository)
enquire.set_weighting_scheme(rec.weight)
enquire.set_query(query)
user_profile = None
if pkg_decider is None:
pkg_decider = PkgMatchDecider(user.installed_pkgs)
# Retrieve matching packages
try:
mset = enquire.get_mset(0, recommendation_size, None,
PkgMatchDecider(user.installed_pkgs))
pkg_decider)
except xapian.DatabaseError as error:
logging.critical("Content-based strategy: " + error.get_msg())
......@@ -105,6 +111,56 @@ class ContentBased(RecommendationStrategy):
return result
class PackageReference(ContentBased):
def __init__(self, content, profile_size):
ContentBased.__init__(self, content, profile_size)
self.content = content
self.description = 'Package-reference'
self.profile_size = profile_size
self.cache = apt.Cache()
self.pkgs_regex = re.compile(r'^\s+(?:\|)?(.+)$', re.MULTILINE)
def get_reverse_dependencies_pkgs(self, reference_pkgs):
reverse_dependencies_pkgs = []
for pkg in reference_pkgs:
command = 'apt-cache rdepends {}'.format(pkg)
rdepends = subprocess.check_output(command,
stderr=subprocess.STDOUT,
shell=True)
rdepends_pkgs = self.pkgs_regex.findall(rdepends)
reverse_dependencies_pkgs += rdepends_pkgs
return reverse_dependencies_pkgs
def content_profile_for_reference_pkgs(self, reference_pkgs):
content_profile = reference_pkgs[:]
for pkg in reference_pkgs:
content_profile += pkg.split('-')
content_profile = list(set(content_profile))
return content_profile
def run(self, rec, user, rec_size):
reference_pkgs = user.reference_pkgs
reverse_dependencies_pkgs = self.get_reverse_dependencies_pkgs(
reference_pkgs)
pkg_decider = PkgReverseDependeciesDecider(reverse_dependencies_pkgs,
user.installed_pkgs)
profile = user.content_profile(rec.items_repository, self.content,
self.profile_size, rec.valid_tags)
profile += self.content_profile_for_reference_pkgs(reference_pkgs)
rec.items_repository = xapian.Database(Config().axi)
result = self.get_sugestion_from_profile(
rec, user, profile, rec_size, pkg_decider=pkg_decider)
return result
class MachineLearning(ContentBased):
__metaclass__ = ABCMeta
......
......@@ -25,7 +25,7 @@ from apprecommender.recommender import RecommendationResult, Recommender
from apprecommender.user import User
from apprecommender.config import Config
from apprecommender.strategy import (ContentBased, MachineLearningBVA,
MachineLearningBOW)
MachineLearningBOW, PackageReference)
class RecommendationResultTests(unittest.TestCase):
......@@ -82,6 +82,9 @@ class RecommenderTests(unittest.TestCase):
self.rec.set_strategy("mlbow_eset")
self.assertIsInstance(self.rec.strategy, MachineLearningBOW)
self.assertEqual(self.rec.strategy.content, "mlbow_mix_eset")
self.rec.set_strategy("cbpkg")
self.assertIsInstance(self.rec.strategy, PackageReference)
self.assertEqual(self.rec.strategy.content, "mix")
def test_get_recommendation(self):
user = User({"inkscape": 1, "gimp": 1, "eog": 1, "vim": 1})
......
......@@ -73,7 +73,8 @@ class User:
Define a user of a recommender.
"""
def __init__(self, item_score, user_id=0, arch=0, demo_profiles_set=0):
def __init__(self, item_score, user_id=0, arch=0, demo_profiles_set=0,
reference_pkgs=None):
"""
Set initial user attributes. pkg_profile gets the whole set of items,
a random user_id is set if none was provided and the demographic
......@@ -84,6 +85,8 @@ class User:
self.installed_pkgs = data.get_user_installed_pkgs()
self.arch = arch
self.reference_pkgs = reference_pkgs if reference_pkgs else []
if user_id:
self.user_id = user_id
else:
......@@ -335,7 +338,7 @@ class LocalSystem(User):
system as the set of selected itens.
"""
def __init__(self):
def __init__(self, reference_pkgs=None):
"""
Set initial parameters.
"""
......@@ -344,7 +347,7 @@ class LocalSystem(User):
self.user_id = "local-" + str(datetime.datetime.now())
User.__init__(self, item_score)
User.__init__(self, item_score, reference_pkgs=reference_pkgs)
def get_system_pkgs(self):
system_pkgs = []
......
......@@ -7,11 +7,11 @@ import sys
sys.path.insert(0, "{0}/../".format(os.path.dirname(__file__)))
from apprecommender.ml.pkg_time import PkgTime
from apprecommender.config import Config
def main():
path = os.path.expanduser(
'~/.app-recommender/user_data/pkgs_classifications.txt')
path = Config().user_data_dir + 'pkgs_classifications.txt'
if not os.path.exists(path):
print 'Could not find file pkgs_classification'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment