Commit f1518355 authored by Lucas Moura's avatar Lucas Moura

Imported Upstream version 0.7

parent 97e221e8
Pipeline #3883938 failed with stage
in 52 seconds
...@@ -135,11 +135,12 @@ class PkgMatchDecider(xapian.MatchDecider): ...@@ -135,11 +135,12 @@ class PkgMatchDecider(xapian.MatchDecider):
xapian.MatchDecider.__init__(self) xapian.MatchDecider.__init__(self)
self.pkgs_list = pkgs_list self.pkgs_list = pkgs_list
def __call__(self, doc): def __call__(self, xapian_document):
""" """
True if the package is not already installed and is not a lib or a doc. True if the package is not already installed and is not a lib or
a xapian_document.
""" """
pkg = doc.get_data() pkg = xapian_document.get_data()
is_new = pkg not in self.pkgs_list is_new = pkg not in self.pkgs_list
is_new = is_new and ':' not in pkg is_new = is_new and ':' not in pkg
...@@ -180,6 +181,45 @@ class PkgExpandDecider(xapian.ExpandDecider): ...@@ -180,6 +181,45 @@ class PkgExpandDecider(xapian.ExpandDecider):
return is_new_pkg return is_new_pkg
class PkgReverseDependeciesDecider(xapian.MatchDecider):
"""
Extend xapian.MatchDecider to consider only packages on valid list
"""
def __init__(self, reverse_dependencies, user_installed_pkgs):
"""
Set initial parameters.
"""
xapian.MatchDecider.__init__(self)
self.reverse_dependencies = reverse_dependencies
self.pkg_init_decider = PkgInitDecider()
self.pkg_match_decider = PkgMatchDecider(user_installed_pkgs)
self.cache = apt.Cache()
def __call__(self, xapian_document):
"""
True if the package is on pkg_list
"""
pkg = xapian_document.get_data()
if pkg not in self.reverse_dependencies:
return False
if pkg not in self.cache:
return False
if self.cache[pkg].section == 'doc':
return False
decider = self.pkg_init_decider
pkg_candidate = self.cache[pkg].candidate
if not decider.is_program_dependencies_installed(pkg_candidate):
return False
return self.pkg_match_decider(xapian_document)
class TagExpandDecider(xapian.ExpandDecider): class TagExpandDecider(xapian.ExpandDecider):
""" """
......
...@@ -105,9 +105,15 @@ class Initialize: ...@@ -105,9 +105,15 @@ class Initialize:
try: try:
if os.path.exists(self.config.base_dir): if os.path.exists(self.config.base_dir):
shutil.rmtree(self.config.base_dir) if os.path.exists(self.config.user_data_dir):
shutil.rmtree(self.config.user_data_dir)
if os.path.exists(self.config.axi_desktopapps):
shutil.rmtree(self.config.axi_desktopapps)
if os.path.exists(self.config.filters_dir):
shutil.rmtree(self.config.filters_dir)
os.makedirs(self.config.base_dir)
os.makedirs(self.config.filters_dir) os.makedirs(self.config.filters_dir)
except OSError: except OSError:
raise raise
......
...@@ -13,11 +13,15 @@ class AppRecommender: ...@@ -13,11 +13,15 @@ class AppRecommender:
self.recommender = Recommender() self.recommender = Recommender()
self.config = Config() self.config = Config()
def make_recommendation(self, print_recommendation=True): def make_recommendation(self, reference_pkgs=None,
print_recommendation=True):
begin_time = datetime.datetime.now() begin_time = datetime.datetime.now()
logging.info("Computation started at %s" % begin_time) logging.info("Computation started at %s" % begin_time)
user = LocalSystem() if not reference_pkgs:
reference_pkgs = []
user = LocalSystem(reference_pkgs)
recommendation_size = Config().num_recommendations recommendation_size = Config().num_recommendations
user_recommendation = (self.recommender.get_recommendation( user_recommendation = (self.recommender.get_recommendation(
user, recommendation_size)) user, recommendation_size))
......
...@@ -17,7 +17,6 @@ PERMISSION_DENIED = 3 ...@@ -17,7 +17,6 @@ PERMISSION_DENIED = 3
def parse_options(args, config): def parse_options(args, config):
if args['strategy']: if args['strategy']:
config.strategy = args['strategy'] config.strategy = args['strategy']
if args['debug']: if args['debug']:
...@@ -32,10 +31,10 @@ def parse_options(args, config): ...@@ -32,10 +31,10 @@ def parse_options(args, config):
config.num_recommendations = args['num_recommendations'] config.num_recommendations = args['num_recommendations']
def run_apprecommender(): def run_apprecommender(reference_pkgs):
try: try:
app_recommender = AppRecommender() app_recommender = AppRecommender()
app_recommender.make_recommendation() app_recommender.make_recommendation(reference_pkgs)
return SUCCESS return SUCCESS
except xapian.DatabaseOpeningError: except xapian.DatabaseOpeningError:
return ERROR_INIT return ERROR_INIT
...@@ -72,7 +71,8 @@ def run(args): ...@@ -72,7 +71,8 @@ def run(args):
else: else:
config = Config() config = Config()
parse_options(args, config) parse_options(args, config)
return run_apprecommender() reference_pkgs = args['packages']
return run_apprecommender(reference_pkgs)
def main(): def main():
......
...@@ -51,4 +51,9 @@ def get_parser(): ...@@ -51,4 +51,9 @@ def get_parser():
help='classify recommendations and help apprecommender to improve', help='classify recommendations and help apprecommender to improve',
action='store_true') action='store_true')
parser.add_argument(
'-p', '--packages',
help="Add reference package for strategy 'cbpkg'",
type=str, nargs='+', default=[])
return parser return parser
...@@ -164,6 +164,8 @@ class Recommender: ...@@ -164,6 +164,8 @@ class Recommender:
self.strategy = strategy.ContentBased("half", profile_size) self.strategy = strategy.ContentBased("half", profile_size)
elif strategy_str == "cbtm": elif strategy_str == "cbtm":
self.strategy = strategy.ContentBased("time", profile_size) self.strategy = strategy.ContentBased("time", profile_size)
elif strategy_str == "cbpkg":
self.strategy = strategy.PackageReference("mix", profile_size)
elif strategy_str == "mlbva": elif strategy_str == "mlbva":
self.strategy = strategy.MachineLearningBVA("mlbva_mix", self.strategy = strategy.MachineLearningBVA("mlbva_mix",
profile_size) profile_size)
......
...@@ -26,6 +26,7 @@ import operator ...@@ -26,6 +26,7 @@ import operator
import pickle import pickle
import re import re
import recommender import recommender
import subprocess
import xapian import xapian
import numpy as np import numpy as np
...@@ -33,7 +34,8 @@ import numpy as np ...@@ -33,7 +34,8 @@ import numpy as np
from abc import ABCMeta, abstractmethod from abc import ABCMeta, abstractmethod
from apprecommender.config import Config from apprecommender.config import Config
from apprecommender.decider import PkgMatchDecider from apprecommender.decider import (PkgMatchDecider,
PkgReverseDependeciesDecider)
from apprecommender.ml.bag_of_words import BagOfWords from apprecommender.ml.bag_of_words import BagOfWords
from apprecommender.ml.bayes_matrix import BayesMatrix from apprecommender.ml.bayes_matrix import BayesMatrix
from apprecommender.ml.data import MachineLearningData from apprecommender.ml.data import MachineLearningData
...@@ -66,16 +68,20 @@ class ContentBased(RecommendationStrategy): ...@@ -66,16 +68,20 @@ class ContentBased(RecommendationStrategy):
self.profile_size = profile_size self.profile_size = profile_size
def get_sugestion_from_profile(self, rec, user, profile, def get_sugestion_from_profile(self, rec, user, profile,
recommendation_size, because=True): recommendation_size, because=True,
pkg_decider=None):
query = xapian.Query(xapian.Query.OP_OR, profile) query = xapian.Query(xapian.Query.OP_OR, profile)
enquire = xapian.Enquire(rec.items_repository) enquire = xapian.Enquire(rec.items_repository)
enquire.set_weighting_scheme(rec.weight) enquire.set_weighting_scheme(rec.weight)
enquire.set_query(query) enquire.set_query(query)
user_profile = None user_profile = None
if pkg_decider is None:
pkg_decider = PkgMatchDecider(user.installed_pkgs)
# Retrieve matching packages # Retrieve matching packages
try: try:
mset = enquire.get_mset(0, recommendation_size, None, mset = enquire.get_mset(0, recommendation_size, None,
PkgMatchDecider(user.installed_pkgs)) pkg_decider)
except xapian.DatabaseError as error: except xapian.DatabaseError as error:
logging.critical("Content-based strategy: " + error.get_msg()) logging.critical("Content-based strategy: " + error.get_msg())
...@@ -105,6 +111,56 @@ class ContentBased(RecommendationStrategy): ...@@ -105,6 +111,56 @@ class ContentBased(RecommendationStrategy):
return result return result
class PackageReference(ContentBased):
def __init__(self, content, profile_size):
ContentBased.__init__(self, content, profile_size)
self.content = content
self.description = 'Package-reference'
self.profile_size = profile_size
self.cache = apt.Cache()
self.pkgs_regex = re.compile(r'^\s+(?:\|)?(.+)$', re.MULTILINE)
def get_reverse_dependencies_pkgs(self, reference_pkgs):
reverse_dependencies_pkgs = []
for pkg in reference_pkgs:
command = 'apt-cache rdepends {}'.format(pkg)
rdepends = subprocess.check_output(command,
stderr=subprocess.STDOUT,
shell=True)
rdepends_pkgs = self.pkgs_regex.findall(rdepends)
reverse_dependencies_pkgs += rdepends_pkgs
return reverse_dependencies_pkgs
def content_profile_for_reference_pkgs(self, reference_pkgs):
content_profile = reference_pkgs[:]
for pkg in reference_pkgs:
content_profile += pkg.split('-')
content_profile = list(set(content_profile))
return content_profile
def run(self, rec, user, rec_size):
reference_pkgs = user.reference_pkgs
reverse_dependencies_pkgs = self.get_reverse_dependencies_pkgs(
reference_pkgs)
pkg_decider = PkgReverseDependeciesDecider(reverse_dependencies_pkgs,
user.installed_pkgs)
profile = user.content_profile(rec.items_repository, self.content,
self.profile_size, rec.valid_tags)
profile += self.content_profile_for_reference_pkgs(reference_pkgs)
rec.items_repository = xapian.Database(Config().axi)
result = self.get_sugestion_from_profile(
rec, user, profile, rec_size, pkg_decider=pkg_decider)
return result
class MachineLearning(ContentBased): class MachineLearning(ContentBased):
__metaclass__ = ABCMeta __metaclass__ = ABCMeta
......
...@@ -25,7 +25,7 @@ from apprecommender.recommender import RecommendationResult, Recommender ...@@ -25,7 +25,7 @@ from apprecommender.recommender import RecommendationResult, Recommender
from apprecommender.user import User from apprecommender.user import User
from apprecommender.config import Config from apprecommender.config import Config
from apprecommender.strategy import (ContentBased, MachineLearningBVA, from apprecommender.strategy import (ContentBased, MachineLearningBVA,
MachineLearningBOW) MachineLearningBOW, PackageReference)
class RecommendationResultTests(unittest.TestCase): class RecommendationResultTests(unittest.TestCase):
...@@ -82,6 +82,9 @@ class RecommenderTests(unittest.TestCase): ...@@ -82,6 +82,9 @@ class RecommenderTests(unittest.TestCase):
self.rec.set_strategy("mlbow_eset") self.rec.set_strategy("mlbow_eset")
self.assertIsInstance(self.rec.strategy, MachineLearningBOW) self.assertIsInstance(self.rec.strategy, MachineLearningBOW)
self.assertEqual(self.rec.strategy.content, "mlbow_mix_eset") self.assertEqual(self.rec.strategy.content, "mlbow_mix_eset")
self.rec.set_strategy("cbpkg")
self.assertIsInstance(self.rec.strategy, PackageReference)
self.assertEqual(self.rec.strategy.content, "mix")
def test_get_recommendation(self): def test_get_recommendation(self):
user = User({"inkscape": 1, "gimp": 1, "eog": 1, "vim": 1}) user = User({"inkscape": 1, "gimp": 1, "eog": 1, "vim": 1})
......
...@@ -73,7 +73,8 @@ class User: ...@@ -73,7 +73,8 @@ class User:
Define a user of a recommender. Define a user of a recommender.
""" """
def __init__(self, item_score, user_id=0, arch=0, demo_profiles_set=0): def __init__(self, item_score, user_id=0, arch=0, demo_profiles_set=0,
reference_pkgs=None):
""" """
Set initial user attributes. pkg_profile gets the whole set of items, Set initial user attributes. pkg_profile gets the whole set of items,
a random user_id is set if none was provided and the demographic a random user_id is set if none was provided and the demographic
...@@ -84,6 +85,8 @@ class User: ...@@ -84,6 +85,8 @@ class User:
self.installed_pkgs = data.get_user_installed_pkgs() self.installed_pkgs = data.get_user_installed_pkgs()
self.arch = arch self.arch = arch
self.reference_pkgs = reference_pkgs if reference_pkgs else []
if user_id: if user_id:
self.user_id = user_id self.user_id = user_id
else: else:
...@@ -335,7 +338,7 @@ class LocalSystem(User): ...@@ -335,7 +338,7 @@ class LocalSystem(User):
system as the set of selected itens. system as the set of selected itens.
""" """
def __init__(self): def __init__(self, reference_pkgs=None):
""" """
Set initial parameters. Set initial parameters.
""" """
...@@ -344,7 +347,7 @@ class LocalSystem(User): ...@@ -344,7 +347,7 @@ class LocalSystem(User):
self.user_id = "local-" + str(datetime.datetime.now()) self.user_id = "local-" + str(datetime.datetime.now())
User.__init__(self, item_score) User.__init__(self, item_score, reference_pkgs=reference_pkgs)
def get_system_pkgs(self): def get_system_pkgs(self):
system_pkgs = [] system_pkgs = []
......
...@@ -7,11 +7,11 @@ import sys ...@@ -7,11 +7,11 @@ import sys
sys.path.insert(0, "{0}/../".format(os.path.dirname(__file__))) sys.path.insert(0, "{0}/../".format(os.path.dirname(__file__)))
from apprecommender.ml.pkg_time import PkgTime from apprecommender.ml.pkg_time import PkgTime
from apprecommender.config import Config
def main(): def main():
path = os.path.expanduser( path = Config().user_data_dir + 'pkgs_classifications.txt'
'~/.app-recommender/user_data/pkgs_classifications.txt')
if not os.path.exists(path): if not os.path.exists(path):
print 'Could not find file pkgs_classification' print 'Could not find file pkgs_classification'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment