Commit dba0b0a5 authored by Matěj Cepl's avatar Matěj Cepl

Add optional parameter to yamlish.load ignore_wrong_characters

When set to True, then yamlish doesn't crash on bad UTF8 characters
but rather skips them.

Fixes #2
parent 7297401f
......@@ -12,7 +12,7 @@ def read(fname):
setup(
name='yamlish',
version="0.14",
version="0.15",
description='Python implementation of YAMLish',
author='Matěj Cepl',
author_email='mcepl@redhat.com',
......
......@@ -9,6 +9,7 @@ import textwrap
INPUT = 1
OUTPUT = 2
#logging.basicConfig(level=logging.DEBUG)
def _generate_test_name(source):
"""
......@@ -18,7 +19,7 @@ def _generate_test_name(source):
return "test_%s" % out
def _create_input_test(test_src, tested_function):
def _create_input_test(test_src, tested_function, options=None):
"""
Decorate tested function to be used as a method for TestCase.
"""
......@@ -31,10 +32,11 @@ def _create_input_test(test_src, tested_function):
got = ""
if 'error' in test_src:
self.assertRaises(test_src['error'], tested_function,
test_src['in'])
test_src['in'], options)
else:
want = test_src['out']
got = tested_function(test_src['in'])
got = tested_function(test_src['in'], options)
logging.debug('got = type %s', type(got))
logging.debug("test_src['out'] = %s", unicode(test_src['out']))
self.assertEqual(got, want, """Result matches
expected = %s
......@@ -45,7 +47,7 @@ def _create_input_test(test_src, tested_function):
return do_test_expected
def _create_output_test(test_src, tested_function):
def _create_output_test(test_src, tested_function, options=None):
"""
Decorate tested function to be used as a method for TestCase.
"""
......@@ -58,7 +60,7 @@ def _create_output_test(test_src, tested_function):
# We currently don't throw any exceptions in Writer, so this
# this is always false
if 'error' in test_src:
self.assertRaises(test_src['error'], yamlish.dumps, test_src['in'])
self.assertRaises(test_src['error'], yamlish.dumps, test_src['in'], options)
else:
logging.debug("out:\n%s", textwrap.dedent(test_src['out']))
want = yaml.load(textwrap.dedent(test_src['out']))
......@@ -74,7 +76,8 @@ def _create_output_test(test_src, tested_function):
return do_test_expected
def generate_testsuite(test_data, test_case_shell, test_fce, direction=INPUT):
def generate_testsuite(test_data, test_case_shell, test_fce, direction=INPUT,
options=None):
"""
Generate tests from the test data, class to build upon and function
to use for testing.
......@@ -85,8 +88,8 @@ def generate_testsuite(test_data, test_case_shell, test_fce, direction=INPUT):
continue
name = _generate_test_name(in_test['name'])
if direction == INPUT:
test_method = _create_input_test(in_test, test_fce)
test_method = _create_input_test(in_test, test_fce, options=options)
elif direction == OUTPUT:
test_method = _create_output_test(in_test, test_fce)
test_method = _create_output_test(in_test, test_fce, options=options)
test_method.__name__ = str('test_%s' % name)
setattr(test_case_shell, test_method.__name__, test_method)
......@@ -6,16 +6,18 @@ import unittest
test_data_list = [{
"name": 'Non UTF8 test',
"in": ['--- \xbd\xd0\xe1 \xd1\xeb\xdb\xde \xdc\xdd\xde\xd3\xde' +
'\xdd\xd0 \xe7\xd5\xdb\xdd\xd5;\n', '...', ],
"out": "Нас было много на челне;",
"in": [b"--- macro `BR\xc3\xc2\xa0fbi' not defined",
'...', ],
"out": "macro `BR\xa0fbi' not defined",
}]
""
class TestReader(unittest.TestCase): # IGNORE:C0111
pass
test.generate_testsuite(test_data_list, TestReader, yamlish.load)
test.generate_testsuite(test_data_list, TestReader, yamlish.load,
options={'ignore_wrong_characters': True})
if __name__ == "__main__":
unittest.main()
......@@ -117,8 +117,9 @@ class NullHandler(logging.Handler):
def emit(self, record):
pass
log = logging.getLogger("bayeux")
log = logging.getLogger("yamlish")
log.addHandler(NullHandler())
#log.setLevel(logging.DEBUG)
__docformat__ = 'reStructuredText'
__version__ = "0.10"
......@@ -167,7 +168,7 @@ yaml.add_representer(str, str_representer_compact_multiline,
yaml.add_representer(unicode, str_representer_compact_multiline,
Dumper=_YamlishDumper)
def load(source):
def load(source, ignore_wrong_characters=False):
"""
Return object loaded from a YAML document in source.
......@@ -176,16 +177,26 @@ def load(source):
many others).
"""
out = None
log.debug("inobj:\n%s", source)
log.debug("inobj: (%s)\n%s", type(source), source)
log.debug('before ignore_wrong_characters = %s', ignore_wrong_characters)
if isinstance(source, (str, unicode)):
out = yaml.load(source, Loader=_YamlishLoader)
log.debug("out (string) = %s", out)
elif hasattr(source, "__iter__"):
inobj = ""
for line in source:
try:
inobj += line + '\n'
out = load(inobj)
except UnicodeDecodeError:
log.debug('in ignore_wrong_characters = %s', ignore_wrong_characters)
if ignore_wrong_characters:
inobj += line.decode('utf8', errors='ignore') + '\n'
else:
raise
log.debug('restarting load with inobj as string')
out = load(inobj, ignore_wrong_characters)
log.debug("out (iter) = %s", out)
log.debug("out (iter) = type %s", type(out))
return out
def dump(source, destination):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment