Commit dba0b0a5 authored by Matěj Cepl's avatar Matěj Cepl

Add optional parameter to yamlish.load ignore_wrong_characters

When set to True, then yamlish doesn't crash on bad UTF8 characters
but rather skips them.

Fixes #2
parent 7297401f
...@@ -12,7 +12,7 @@ def read(fname): ...@@ -12,7 +12,7 @@ def read(fname):
setup( setup(
name='yamlish', name='yamlish',
version="0.14", version="0.15",
description='Python implementation of YAMLish', description='Python implementation of YAMLish',
author='Matěj Cepl', author='Matěj Cepl',
author_email='mcepl@redhat.com', author_email='mcepl@redhat.com',
......
...@@ -9,6 +9,7 @@ import textwrap ...@@ -9,6 +9,7 @@ import textwrap
INPUT = 1 INPUT = 1
OUTPUT = 2 OUTPUT = 2
#logging.basicConfig(level=logging.DEBUG)
def _generate_test_name(source): def _generate_test_name(source):
""" """
...@@ -18,7 +19,7 @@ def _generate_test_name(source): ...@@ -18,7 +19,7 @@ def _generate_test_name(source):
return "test_%s" % out return "test_%s" % out
def _create_input_test(test_src, tested_function): def _create_input_test(test_src, tested_function, options=None):
""" """
Decorate tested function to be used as a method for TestCase. Decorate tested function to be used as a method for TestCase.
""" """
...@@ -31,10 +32,11 @@ def _create_input_test(test_src, tested_function): ...@@ -31,10 +32,11 @@ def _create_input_test(test_src, tested_function):
got = "" got = ""
if 'error' in test_src: if 'error' in test_src:
self.assertRaises(test_src['error'], tested_function, self.assertRaises(test_src['error'], tested_function,
test_src['in']) test_src['in'], options)
else: else:
want = test_src['out'] want = test_src['out']
got = tested_function(test_src['in']) got = tested_function(test_src['in'], options)
logging.debug('got = type %s', type(got))
logging.debug("test_src['out'] = %s", unicode(test_src['out'])) logging.debug("test_src['out'] = %s", unicode(test_src['out']))
self.assertEqual(got, want, """Result matches self.assertEqual(got, want, """Result matches
expected = %s expected = %s
...@@ -45,7 +47,7 @@ def _create_input_test(test_src, tested_function): ...@@ -45,7 +47,7 @@ def _create_input_test(test_src, tested_function):
return do_test_expected return do_test_expected
def _create_output_test(test_src, tested_function): def _create_output_test(test_src, tested_function, options=None):
""" """
Decorate tested function to be used as a method for TestCase. Decorate tested function to be used as a method for TestCase.
""" """
...@@ -58,7 +60,7 @@ def _create_output_test(test_src, tested_function): ...@@ -58,7 +60,7 @@ def _create_output_test(test_src, tested_function):
# We currently don't throw any exceptions in Writer, so this # We currently don't throw any exceptions in Writer, so this
# this is always false # this is always false
if 'error' in test_src: if 'error' in test_src:
self.assertRaises(test_src['error'], yamlish.dumps, test_src['in']) self.assertRaises(test_src['error'], yamlish.dumps, test_src['in'], options)
else: else:
logging.debug("out:\n%s", textwrap.dedent(test_src['out'])) logging.debug("out:\n%s", textwrap.dedent(test_src['out']))
want = yaml.load(textwrap.dedent(test_src['out'])) want = yaml.load(textwrap.dedent(test_src['out']))
...@@ -74,7 +76,8 @@ def _create_output_test(test_src, tested_function): ...@@ -74,7 +76,8 @@ def _create_output_test(test_src, tested_function):
return do_test_expected return do_test_expected
def generate_testsuite(test_data, test_case_shell, test_fce, direction=INPUT): def generate_testsuite(test_data, test_case_shell, test_fce, direction=INPUT,
options=None):
""" """
Generate tests from the test data, class to build upon and function Generate tests from the test data, class to build upon and function
to use for testing. to use for testing.
...@@ -85,8 +88,8 @@ def generate_testsuite(test_data, test_case_shell, test_fce, direction=INPUT): ...@@ -85,8 +88,8 @@ def generate_testsuite(test_data, test_case_shell, test_fce, direction=INPUT):
continue continue
name = _generate_test_name(in_test['name']) name = _generate_test_name(in_test['name'])
if direction == INPUT: if direction == INPUT:
test_method = _create_input_test(in_test, test_fce) test_method = _create_input_test(in_test, test_fce, options=options)
elif direction == OUTPUT: elif direction == OUTPUT:
test_method = _create_output_test(in_test, test_fce) test_method = _create_output_test(in_test, test_fce, options=options)
test_method.__name__ = str('test_%s' % name) test_method.__name__ = str('test_%s' % name)
setattr(test_case_shell, test_method.__name__, test_method) setattr(test_case_shell, test_method.__name__, test_method)
...@@ -6,16 +6,18 @@ import unittest ...@@ -6,16 +6,18 @@ import unittest
test_data_list = [{ test_data_list = [{
"name": 'Non UTF8 test', "name": 'Non UTF8 test',
"in": ['--- \xbd\xd0\xe1 \xd1\xeb\xdb\xde \xdc\xdd\xde\xd3\xde' + "in": [b"--- macro `BR\xc3\xc2\xa0fbi' not defined",
'\xdd\xd0 \xe7\xd5\xdb\xdd\xd5;\n', '...', ], '...', ],
"out": "Нас было много на челне;", "out": "macro `BR\xa0fbi' not defined",
}] }]
""
class TestReader(unittest.TestCase): # IGNORE:C0111 class TestReader(unittest.TestCase): # IGNORE:C0111
pass pass
test.generate_testsuite(test_data_list, TestReader, yamlish.load) test.generate_testsuite(test_data_list, TestReader, yamlish.load,
options={'ignore_wrong_characters': True})
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
...@@ -117,8 +117,9 @@ class NullHandler(logging.Handler): ...@@ -117,8 +117,9 @@ class NullHandler(logging.Handler):
def emit(self, record): def emit(self, record):
pass pass
log = logging.getLogger("bayeux") log = logging.getLogger("yamlish")
log.addHandler(NullHandler()) log.addHandler(NullHandler())
#log.setLevel(logging.DEBUG)
__docformat__ = 'reStructuredText' __docformat__ = 'reStructuredText'
__version__ = "0.10" __version__ = "0.10"
...@@ -167,7 +168,7 @@ yaml.add_representer(str, str_representer_compact_multiline, ...@@ -167,7 +168,7 @@ yaml.add_representer(str, str_representer_compact_multiline,
yaml.add_representer(unicode, str_representer_compact_multiline, yaml.add_representer(unicode, str_representer_compact_multiline,
Dumper=_YamlishDumper) Dumper=_YamlishDumper)
def load(source): def load(source, ignore_wrong_characters=False):
""" """
Return object loaded from a YAML document in source. Return object loaded from a YAML document in source.
...@@ -176,16 +177,26 @@ def load(source): ...@@ -176,16 +177,26 @@ def load(source):
many others). many others).
""" """
out = None out = None
log.debug("inobj:\n%s", source) log.debug("inobj: (%s)\n%s", type(source), source)
log.debug('before ignore_wrong_characters = %s', ignore_wrong_characters)
if isinstance(source, (str, unicode)): if isinstance(source, (str, unicode)):
out = yaml.load(source, Loader=_YamlishLoader) out = yaml.load(source, Loader=_YamlishLoader)
log.debug("out (string) = %s", out) log.debug("out (string) = %s", out)
elif hasattr(source, "__iter__"): elif hasattr(source, "__iter__"):
inobj = "" inobj = ""
for line in source: for line in source:
inobj += line + '\n' try:
out = load(inobj) inobj += line + '\n'
except UnicodeDecodeError:
log.debug('in ignore_wrong_characters = %s', ignore_wrong_characters)
if ignore_wrong_characters:
inobj += line.decode('utf8', errors='ignore') + '\n'
else:
raise
log.debug('restarting load with inobj as string')
out = load(inobj, ignore_wrong_characters)
log.debug("out (iter) = %s", out) log.debug("out (iter) = %s", out)
log.debug("out (iter) = type %s", type(out))
return out return out
def dump(source, destination): def dump(source, destination):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment