Commit f6e8a8ce authored by Biagio's avatar Biagio

Merge branch 'body_parser'

parents f78e6302 c4c7e768
......@@ -9,6 +9,12 @@ from keld.body_parser import parse as parse_doc
from keld.doc_scraper import download_everything
from tqdm import tqdm
from keld.commons import TEST_DOCS_DIR
from keld.commons.logging import setup_logger
here = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
tqdm.monitor_interval = 0
logger = setup_logger(f"keld", os.path.join(here, f"keld.log"))
def parse(filepath, output_dir=None):
......@@ -61,7 +67,14 @@ def parse_all(batch_no=None):
if not docname.endswith((".doc", ".DOC")):
continue
docpath = os.path.join(TEST_DOCS_DIR, docname)
parse(docpath)
try:
parse(docpath)
except KeyboardInterrupt:
raise
except Exception as e:
# raise
logger.exception(f"Fatal error for {docname}")
tqdm.write(f">>>> Fatal error for {docname}: {e}")
pbar.set_description(docname)
pbar.close()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment