Commit f4ba211d authored by Lucas Moura's avatar Lucas Moura

Imported Upstream version 0.7.2

parent 4eb13cfe
Pipeline #3936775 failed with stage
in 47 seconds
......@@ -13,5 +13,6 @@ udd.sql.gz
apprecommender.egg-info/
apprecommender/tests/test_data/.sample_axi/
apprecommender/tests/test_data/.sample_axi/
apprecommender/tests/.apt_run/
apprecommender/web/submissions/
.travis.yml.container
DPkg::Post-Invoke {"apprec-apt --post-invoke";};
DPkg::Pre-Install-Pkgs {"apprec-apt --pre-install-pkgs";};
#!/usr/bin/env python
import argparse
import os
import shutil
import re
from apprecommender.config import Config
from apprecommender.main.app_recommender import AppRecommender
from apprecommender.user import LocalSystem
class AptRun:
def __init__(self):
apt_folder = os.path.join(Config().base_dir, "apt_run/")
self.set_folder(apt_folder)
def set_folder(self, folder_path):
folder_path = os.path.expanduser(folder_path)
folder_path = os.path.abspath(folder_path)
if folder_path[-1] != '/':
folder_path += '/'
self.apt_folder = folder_path
self.installed_pkgs_file = self.apt_folder + "installed_pkgs.txt"
def enable(self):
if not self.is_enable():
os.makedirs(self.apt_folder)
return True
return False
def disable(self):
if self.is_enable():
shutil.rmtree(self.apt_folder)
return True
return False
def is_enable(self):
return os.path.exists(self.apt_folder)
def get_user_pkgs(self):
user = LocalSystem()
user_pkgs = user.pkg_profile
return user_pkgs
def pre_install_pkgs(self):
if self.is_enable():
user_pkgs = self.get_user_pkgs()
with open(self.installed_pkgs_file, 'w') as text:
text.write("\n".join(user_pkgs))
def post_invoke(self):
if not self.is_enable():
return []
with open(self.installed_pkgs_file, 'r') as text:
pre_installed_pkgs = set([line.strip() for line in text])
pkgs = self.get_user_pkgs()
pos_installed_pkgs = set([pkg.strip() for pkg in pkgs])
installed_pkgs = list(pos_installed_pkgs - pre_installed_pkgs)
return installed_pkgs
def get_recommendation_pkgs(self, installed_pkgs):
app_recommender = AppRecommender()
app_recommender.recommender.set_strategy('cbpkg')
rec = app_recommender.make_recommendation(
reference_pkgs=installed_pkgs, print_recommendation=False)
pkgs_regex = re.compile(r'^\d:\s([^\s]+)', re.MULTILINE)
pkgs = pkgs_regex.findall(rec.__str__())
return pkgs
def make_recommendations(self, installed_pkgs):
if len(installed_pkgs) > 0:
pkgs = self.get_recommendation_pkgs(installed_pkgs)
if len(pkgs) > 0:
print '\nApprecommeder: The following packages are' \
' interesting'
for pkg in pkgs:
print ' - {}'.format(pkg)
def get_args():
aptrun_description = 'Integration of AppRecommender with apt'
parser = argparse.ArgumentParser(description=aptrun_description)
parser.add_argument(
'--pre-install-pkgs',
help='Indentify installed packages before install the new packages',
action='store_true')
parser.add_argument(
'--post-invoke',
help='Indentify the installed packages and makes recommendation',
action='store_true')
args = vars(parser.parse_args())
return args
def main():
args = get_args()
apt_run = AptRun()
if args['pre_install_pkgs']:
apt_run.pre_install_pkgs()
elif args['post_invoke']:
installed_pkgs = apt_run.post_invoke()
apt_run.make_recommendations(installed_pkgs)
if __name__ == '__main__':
main()
......@@ -9,6 +9,8 @@ from apprecommender.strategy import (MachineLearning, MachineLearningBVA,
MachineLearningBOW,
MachineLearningTrainError)
from apprecommender.main import collect_user_data
from apprecommender.main import show_classifications
from apprecommender.main.apt_run import AptRun
from apprecommender.main.options import get_parser
SUCCESS = 0
......@@ -47,31 +49,77 @@ def run_apprecommender(reference_pkgs):
return PERMISSION_DENIED
def run_initialize():
print "Initializing AppRecommender"
initialize = Initialize()
try:
initialize.prepare_data()
except OSError:
return PERMISSION_DENIED
return SUCCESS
def run_train():
print "Training machine learning"
try:
MachineLearning.train(MachineLearningBVA)
MachineLearning.train(MachineLearningBOW)
except IOError:
return PERMISSION_DENIED
except MachineLearningTrainError:
return ERROR_INIT_TRAIN
return SUCCESS
def run(args):
if args['init']:
print "Initializing AppRecommender"
initialize = Initialize()
if args['update']:
init_result = run_initialize()
try:
initialize.prepare_data()
except OSError:
return PERMISSION_DENIED
if init_result != SUCCESS:
return init_result
return SUCCESS
elif args['train']:
print "Training machine learning"
train_result = run_train()
try:
MachineLearning.train(MachineLearningBVA)
MachineLearning.train(MachineLearningBOW)
except IOError:
return PERMISSION_DENIED
except MachineLearningTrainError:
return ERROR_INIT_TRAIN
if train_result != SUCCESS:
return train_result
return SUCCESS
elif args['init']:
return run_initialize()
elif args['train']:
return run_train()
elif args['contribute']:
collect_user_data.main()
elif args['show_classifications']:
show_classifications.main()
elif args['enable_apt']:
try:
apt_run = AptRun()
if apt_run.enable():
print 'AppRecommender now makes recommendations when you ' \
' install new packages with apt'
else:
print 'This is already enabled'
return SUCCESS
except OSError:
return PERMISSION_DENIED
elif args['disable_apt']:
try:
apt_run = AptRun()
if apt_run.disable():
print 'AppRecommender now dont makes recommendations when' \
'you install new packages with apt'
else:
print 'This is already disabled'
return SUCCESS
except OSError:
return PERMISSION_DENIED
else:
config = Config()
parse_options(args, config)
......
......@@ -56,4 +56,24 @@ def get_parser():
help="Add reference package for strategy 'cbpkg'",
type=str, nargs='+', default=[])
parser.add_argument(
'--show-classifications',
help='Show the user classifications for machine learning algorithms',
action='store_true')
parser.add_argument(
'-e', '--enable-apt',
help='Enable recommendations when install a package with apt',
action='store_true')
parser.add_argument(
'-r', '--disable-apt',
help='Disable recommendations when install a package with apt',
action='store_true')
parser.add_argument(
'--update',
help='Run both init and train commands',
action='store_true')
return parser
......@@ -2,9 +2,6 @@
import os
import pickle
import sys
sys.path.insert(0, "{0}/../".format(os.path.dirname(__file__)))
from apprecommender.ml.pkg_time import PkgTime
from apprecommender.config import Config
......@@ -44,7 +41,3 @@ def main():
print pkg_text.format(pkg, pkgs_times[pkg][1].strip())
print '\nNum pkgs: {}'.format(len(pkg_classification))
if __name__ == '__main__':
main()
......@@ -21,16 +21,17 @@ __license__ = """
import apt
import heapq
import inspect
import logging
import operator
import os
import strategy
import xapian
from collections import namedtuple
from fuzzywuzzy import fuzz
from operator import attrgetter
import apprecommender.strategy
from apprecommender.config import Config
......@@ -144,48 +145,38 @@ class Recommender:
self.set_strategy(self.cfg.strategy)
def get_all_strategies(self):
cls_members = inspect.getmembers(apprecommender.strategy,
inspect.isclass)
valid_strategies = {}
for name, obj in cls_members:
class_strategies = getattr(obj, 'get_valid_strategies', None)
if class_strategies:
valid_strategies = dict(class_strategies(), **valid_strategies)
return valid_strategies
def set_strategy(self, strategy_str, n=0):
"""
Set the recommendation strategy.
"""
valid_strategies = self.get_all_strategies()
profile_size = n if n else self.cfg.profile_size
self.items_repository = self.axi_desktopapps
self.valid_pkgs = self.valid_desktopapps
logging.info("Setting recommender strategy to \'%s\'" % strategy_str)
if strategy_str == "cb":
self.strategy = strategy.ContentBased("mix", profile_size)
elif strategy_str == "cbt":
self.strategy = strategy.ContentBased("tag", profile_size)
elif strategy_str == "cbd":
self.strategy = strategy.ContentBased("desc", profile_size)
elif strategy_str == "cbh":
self.strategy = strategy.ContentBased("half", profile_size)
elif strategy_str == "cbtm":
self.strategy = strategy.ContentBased("time", profile_size)
elif strategy_str == "cbpkg":
self.strategy = strategy.PackageReference("mix", profile_size)
elif strategy_str == "mlbva":
self.strategy = strategy.MachineLearningBVA("mlbva_mix",
profile_size)
elif strategy_str == "mlbow":
self.strategy = strategy.MachineLearningBOW("mlbow_mix",
profile_size)
elif strategy_str == "cb_eset":
self.strategy = strategy.ContentBased("mix_eset", profile_size)
elif strategy_str == "cbt_eset":
self.strategy = strategy.ContentBased("tag_eset", profile_size)
elif strategy_str == "cbd_eset":
self.strategy = strategy.ContentBased("desc_eset", profile_size)
elif strategy_str == "cbh_eset":
self.strategy = strategy.ContentBased("half_eset", profile_size)
elif strategy_str == "mlbva_eset":
self.strategy = strategy.MachineLearningBVA("mlbva_mix_eset",
profile_size)
elif strategy_str == "mlbow_eset":
self.strategy = strategy.MachineLearningBOW("mlbow_mix_eset",
profile_size)
if strategy_str in valid_strategies:
strategy_type = valid_strategies[strategy_str]
class_name = strategy_type.class_name
content_type = strategy_type.content_type
self.strategy = eval(class_name)(
content_type, profile_size)
else:
logging.info("Strategy not defined.")
self.strategy = None
......
......@@ -21,6 +21,7 @@ __license__ = """
"""
import apt
import collections
import logging
import operator
import pickle
......@@ -39,12 +40,16 @@ from apprecommender.decider import (PkgMatchDecider,
from apprecommender.ml.bag_of_words import BagOfWords
from apprecommender.ml.bayes_matrix import BayesMatrix
from apprecommender.ml.data import MachineLearningData
from apprecommender.utils import get_class_and_module_name
XAPIAN_DATABASE_PATH = Config().axi_desktopapps
USER_DATA_DIR = Config().user_data_dir
PKGS_CLASSIFICATIONS_INDICES = (USER_DATA_DIR +
'pkgs_classifications_indices.txt')
StrategyType = collections.namedtuple('StrategyType',
'class_name content_type')
class RecommendationStrategy(object):
......@@ -62,6 +67,21 @@ class ContentBased(RecommendationStrategy):
Content-based recommendation strategy based on Apt-xapian-index.
"""
@classmethod
def get_valid_strategies(cls):
class_name = get_class_and_module_name(cls)
valid_strategies = {'cb': StrategyType(class_name, 'mix'),
'cbt': StrategyType(class_name, 'tag'),
'cbd': StrategyType(class_name, 'desc'),
'cbh': StrategyType(class_name, 'half'),
'cbtm': StrategyType(class_name, 'time'),
'cb_eset': StrategyType(class_name, 'mix_eset'),
'cbt_eset': StrategyType(class_name, 'tag_eset'),
'cbd_eset': StrategyType(class_name, 'desc_eset'),
'cbh_eset': StrategyType(class_name, 'half_eset')}
return valid_strategies
def __init__(self, content, profile_size):
self.description = "Content-based"
self.content = content
......@@ -113,6 +133,13 @@ class ContentBased(RecommendationStrategy):
class PackageReference(ContentBased):
@classmethod
def get_valid_strategies(cls):
class_name = get_class_and_module_name(cls)
valid_strategies = {'cbpkg': StrategyType(class_name, 'mix')}
return valid_strategies
def __init__(self, content, profile_size):
ContentBased.__init__(self, content, profile_size)
self.content = content
......@@ -293,7 +320,7 @@ class MachineLearning(ContentBased):
try:
MachineLearning.PKGS_CLASSIFICATIONS = ml_data.create_data(
labels)
if len(MachineLearning.PKGS_CLASSIFICATIONS) > 0:
if len(MachineLearning.PKGS_CLASSIFICATIONS) >= 10:
cls.run_train(MachineLearning.PKGS_CLASSIFICATIONS)
else:
raise MachineLearningTrainError()
......@@ -345,6 +372,29 @@ class MachineLearning(ContentBased):
class MachineLearningBVA(MachineLearning):
@classmethod
def get_valid_strategies(cls):
class_name = get_class_and_module_name(cls)
valid_strategies = {'mlbva': StrategyType(class_name, 'mlbva_mix'),
'mlbva_eset': StrategyType(class_name,
'mlbva_mix_eset')}
return valid_strategies
@classmethod
def run_train(cls, pkgs_classifications):
all_matrix = (np.matrix(pkgs_classifications.values()))
data_matrix = all_matrix[0:, 0:-1]
classifications = all_matrix[0:, -1]
order_of_classifications = ['NU', 'U', 'RU']
bayes_matrix = BayesMatrix()
bayes_matrix.training(data_matrix, classifications,
order_of_classifications)
BayesMatrix.save(bayes_matrix,
MachineLearningData.MACHINE_LEARNING_TRAINING)
def __init__(self, content, profile_size, suggestion_size=200):
super(MachineLearningBVA, self).__init__(
content, profile_size, suggestion_size)
......@@ -380,22 +430,17 @@ class MachineLearningBVA(MachineLearning):
return attribute_vector
@classmethod
def run_train(cls, pkgs_classifications):
all_matrix = (np.matrix(pkgs_classifications.values()))
data_matrix = all_matrix[0:, 0:-1]
classifications = all_matrix[0:, -1]
order_of_classifications = ['NU', 'U', 'RU']
bayes_matrix = BayesMatrix()
bayes_matrix.training(data_matrix, classifications,
order_of_classifications)
BayesMatrix.save(bayes_matrix,
MachineLearningData.MACHINE_LEARNING_TRAINING)
class MachineLearningBOW(MachineLearning):
@classmethod
def get_valid_strategies(cls):
class_name = get_class_and_module_name(cls)
valid_strategies = {'mlbow': StrategyType(class_name, 'mlbow_mix'),
'mlbow_eset': StrategyType(class_name,
'mlbow_mix_eset')}
class MachineLearningBOW(MachineLearning):
return valid_strategies
def __init__(self, content, profile_size, suggestion_size=200):
super(MachineLearningBOW, self).__init__(
......
#!/usr/bin/env python
import os
import shutil
import unittest
from mock import patch
from apprecommender.main.apt_run import AptRun
from apprecommender.user import LocalSystem
class AptRunTests(unittest.TestCase):
TEST_FOLDER = 'apprecommender/tests/.apt_run'
def setUp(self):
if os.path.exists(AptRunTests.TEST_FOLDER):
shutil.rmtree(AptRunTests.TEST_FOLDER)
def test_enable_apt_run(self):
apt_run = AptRun()
apt_run.set_folder(AptRunTests.TEST_FOLDER)
apt_run.enable()
result = apt_run.is_enable()
self.assertTrue(result)
def test_dont_enable_apt_run_if_already_enabled(self):
apt_run = AptRun()
apt_run.set_folder(AptRunTests.TEST_FOLDER)
apt_run.enable()
result = apt_run.enable()
self.assertFalse(result)
def test_disable_apt_run(self):
apt_run = AptRun()
apt_run.set_folder(AptRunTests.TEST_FOLDER)
os.makedirs(AptRunTests.TEST_FOLDER)
apt_run.disable()
result = apt_run.is_enable()
self.assertFalse(result)
def test_dont_disable_apt_run_if_already_disabled(self):
apt_run = AptRun()
apt_run.set_folder(AptRunTests.TEST_FOLDER)
result = apt_run.disable()
self.assertFalse(result)
def test_pre_install_pkgs(self):
apt_run = AptRun()
apt_run.set_folder(AptRunTests.TEST_FOLDER)
apt_run.enable()
apt_run.pre_install_pkgs()
user = LocalSystem()
assert_pkgs = user.pkg_profile
pkgs = []
with open(apt_run.installed_pkgs_file, 'r') as text:
pkgs = [line.strip() for line in text]
self.assertListEqual(assert_pkgs, pkgs)
@patch('apprecommender.main.apt_run.AptRun.get_user_pkgs')
def test_post_invoke(self, mock_user_pkgs):
mock_user_pkgs.return_value = ['vim', 'gedit']
apt_run = AptRun()
apt_run.set_folder(AptRunTests.TEST_FOLDER)
apt_run.enable()
with open(apt_run.installed_pkgs_file, 'w') as text:
text.write('gedit')
installed_pkgs = apt_run.post_invoke()
self.assertEqual(['vim'], installed_pkgs)
......@@ -424,4 +424,6 @@ class LocalSystem(User):
pkgs = self.__remove_lib_packages(pkgs)
pkgs = self.__remove_apt_packages(pkgs)
return set([pkg for pkg in pkgs if pkg not in system_pkgs])
pkgs -= set(system_pkgs)
return pkgs
......@@ -19,3 +19,10 @@ def print_progress_bar(number, n_numbers, message='Progress',
sys.stdout.flush()
if number == n_numbers:
print '\n'
def get_class_and_module_name(cls):
class_name = cls.__name__
module_name = cls.__module__
return module_name + '.' + class_name
......@@ -15,6 +15,7 @@ setup(
entry_points={
'console_scripts': [
'apprec = apprecommender.main.cli:main',
'apprec-apt = apprecommender.main.apt_run:main',
]
},
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment