Commit 943fbb27 authored by Markus Shepherd's avatar Markus Shepherd

Merge branch 'upgrade' into 'master'

Upgrade dependencies

See merge request !74
parents f0da9451 dfd35d40
......@@ -17,17 +17,14 @@ gunicorn = "*"
ludoj-recommender = {editable = true, git = "https://gitlab.com/mshepherd/ludoj-recommender.git"}
markdown = "*"
python-dateutil = "*"
turicreate = "*"
whitenoise = {extras = ["brotli"], version = "*"}
[dev-packages]
# ipython = "*"
ludoj-scraper = {editable = true, git = "https://gitlab.com/mshepherd/ludoj-scraper.git"}
lxml = "*"
# mypy = "*"
# pylint = "*"
pynt = "*"
pynt-contrib = "*"
# pytest = "*"
python-dotenv = "*"
rcssmin = "*"
requests = "*"
......
This diff is collapsed.
1-6-1
\ No newline at end of file
1-6-2
\ No newline at end of file
......@@ -303,9 +303,12 @@ def link(
os.path.join(SCRAPED_DATA_DIR, 'scraped', 'wikidata_GameItem.jl'),
),
id_prefixes=('bgg', 'bga', 'spielen', 'luding', 'wikidata'),
training_file=os.path.join(BASE_DIR, 'cluster', 'training.json'),
manual_labelling=False,
threshold=None,
recall_weight=.5,
output=os.path.join(SCRAPED_DATA_DIR, 'links.json'),
pretty_print=True,
):
''' link items '''
from ludoj_scraper.cluster import link_games
......@@ -315,9 +318,43 @@ def link(
gazetteer=gazetteer,
paths=paths,
id_prefixes=id_prefixes,
training_file=training_file if manual_labelling else None,
manual_labelling=manual_labelling,
threshold=parse_float(threshold),
recall_weight=parse_float(recall_weight),
output=output,
pretty_print=pretty_print,
)
@task()
def labellinks(
gazetteer=os.path.join(BASE_DIR, 'cluster', 'gazetteer.pickle'),
paths=(
os.path.join(SCRAPED_DATA_DIR, 'scraped', 'bgg_GameItem.jl'),
os.path.join(SCRAPED_DATA_DIR, 'scraped', 'bga_GameItem.jl'),
os.path.join(SCRAPED_DATA_DIR, 'scraped', 'spielen_GameItem.jl'),
os.path.join(SCRAPED_DATA_DIR, 'scraped', 'luding_GameItem.jl'),
os.path.join(SCRAPED_DATA_DIR, 'scraped', 'wikidata_GameItem.jl'),
),
id_prefixes=('bgg', 'bga', 'spielen', 'luding', 'wikidata'),
training_file=os.path.join(BASE_DIR, 'cluster', 'training.json'),
threshold=None,
recall_weight=.5,
output=os.path.join(SCRAPED_DATA_DIR, 'links.json'),
pretty_print=True,
):
''' label new training examples and link items '''
link(
gazetteer=gazetteer,
paths=paths,
id_prefixes=id_prefixes,
training_file=training_file,
manual_labelling=True,
threshold=threshold,
recall_weight=recall_weight,
output=output,
pretty_print=pretty_print,
)
......
No preview for this file type
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment