Commit e69feaf4 authored by Almar Klein's avatar Almar Klein

Python implementations implicitly convert Numpy scalars to native float/int

parent 585ad21b
Pipeline #32101720 passed with stages
in 5 minutes and 35 seconds
......@@ -293,7 +293,7 @@ function encode_type_id(f, c, extension_id) {
BsdfSerializer.prototype.encode_object = function (f, value, extension_id) {
var iext, ext;
// We prefer to fail on undefined, instead of silently converting to null like JSON
// if (typeof value == 'undefined') { encode_type_id(f, 'v', extension_id); }
if (typeof value == 'undefined') { throw new TypeError("BSDF cannot encode undefined, use null instead."); }
......
......@@ -26,7 +26,7 @@ function test_basics_floats() {
function test_fail_good() {
// We want to catch use of undefined nicely
var caught = '';
try {
......@@ -35,7 +35,7 @@ function test_fail_good() {
caught = e.message;
}
assert(caught.includes('cannot encode undefined'), caught); // String.includes() not for IE
// Dito, but nested
var caught = '';
try {
......@@ -195,11 +195,11 @@ function test_integer_encoding(){
function test_extensions1() {
function MyUndefined() {
}
myundefined = new MyUndefined();
var data1 = [3, 4, myundefined];
var serializer;
......
......@@ -43,7 +43,7 @@ logger = logging.getLogger(__name__)
# introduced. An implementation must display a warning when the file
# being read has a higher minor version. The patch version is increased
# for subsequent releases of the implementation.
VERSION = 2, 2, 0
VERSION = 2, 2, 1
__version__ = '.'.join(str(i) for i in VERSION)
......@@ -275,6 +275,22 @@ class BsdfSerializer(object):
raise ValueError('Can only have one stream per file.')
streams.append(value)
value._activate(f, self._encode, self._decode) # noqa
elif getattr(value, "shape", None) == () and str(
getattr(value, "dtype", "")
).startswith(("uint", "int", "float")):
# Implicit conversion of numpy scalars
if 'int' in str(value.dtype):
value = int(value)
if -32768 <= value <= 32767:
f.write(x(b'h', ext_id) + spack('h', value))
else:
f.write(x(b'i', ext_id) + spack('<q', value))
else:
value = float(value)
if self._float64:
f.write(x(b'd', ext_id) + spack('<d', value))
else:
f.write(x(b'f', ext_id) + spack('<f', value))
else:
if ext_id is not None:
raise ValueError(
......
......@@ -73,6 +73,10 @@ def fail(msg):
sys.exit(m)
def _print(*args, **kwargs):
print(*args, **kwargs) # noqa
def cmd_help(command=None):
""" Show the help text.
......@@ -87,7 +91,7 @@ def cmd_help(command=None):
func = globals().get('cmd_' + command, None)
if func is None:
fail('Cannot give help for invalid command: %r' % command)
print(func.__doc__.strip())
_print(func.__doc__.strip())
return
# Otherwise, show generic help
......@@ -112,7 +116,7 @@ def cmd_help(command=None):
lines.append("Run 'bsdf help command' "
"or 'bsdf command --help' to learn more.")
print('\n'.join(lines))
_print('\n'.join(lines))
def cmd_version():
......@@ -122,7 +126,7 @@ def cmd_version():
"""
module_name = bsdf.__name__ + '.py'
version_string = '.'.join([str(i) for i in bsdf.VERSION])
print('%s version %s' % (module_name, version_string))
_print('%s version %s' % (module_name, version_string))
def cmd_info(filename):
......@@ -161,7 +165,7 @@ def cmd_info(filename):
# Print info
lines = [' ' + line for line in lines]
lines.insert(0, 'BSDF info for: ' + os.path.abspath(filename))
print('\n'.join(lines))
_print('\n'.join(lines))
def cmd_create(filename, code):
......@@ -222,7 +226,7 @@ def cmd_convert(filename1, filename2):
pass
raise
print('Wrote', filename2)
_print('Wrote', filename2)
def cmd_view(filename, depth=1e9, info=False):
......@@ -250,7 +254,7 @@ def cmd_view(filename, depth=1e9, info=False):
if info:
cmd_info(filename)
print('')
_print('')
with open(filename, 'rb') as f:
......@@ -268,7 +272,7 @@ def cmd_view(filename, depth=1e9, info=False):
if minor_version > bsdf.VERSION[1]: # minor should be < ours
t = ('Warning: reading file with higher minor version (%s) '
'than the implementation (%s).')
print(t % (bsdf.__version__, file_version))
_print(t % (bsdf.__version__, file_version))
_view_decode(f, 0, int(depth))
......@@ -297,12 +301,12 @@ def _view_decode(f, depth, maxdepth, noindent=False):
def printval(s):
if depth <= maxdepth:
indent = '' if noindent else ' ' * depth
print(indent + s + ext_id)
_print(indent + s + ext_id)
def printraw(s, **kwargs):
if depth <= maxdepth:
indent = ' ' * depth
print(indent + s, **kwargs)
_print(indent + s, **kwargs)
if c == b'v':
printval('null')
......
......@@ -20,7 +20,7 @@ for line in open(fname, 'rb').read().decode('utf-8').splitlines():
doc = doc.strip(' \t\n\r"')
elif doc and doc.startswith('"""'):
doc += line + '\n'
assert version, 'could not find version'
assert doc, 'could not find docs'
......
......@@ -17,7 +17,7 @@ def call(*cmd):
def lint(ctx):
""" Run style tests with flake8. """
# Print nice messages when all is well; flake8 does not celebrate.
ret_code = subprocess.call(['flake8', 'bsdf.py', 'bsdf_cli.py'], cwd=this_dir)
ret_code = subprocess.call(['flake8', 'bsdf.py', 'bsdf_cli.py', '--ignore=S1'], cwd=this_dir)
if ret_code == 0:
print('No style errors found')
sys.exit(ret_code)
......
......@@ -23,14 +23,14 @@ class StringIO:
self._parts = []
self.closed = False
self.isatty = False
def write(self, msg):
self._parts.append(msg)
return len(msg)
def getvalue(self):
return ''.join(self._parts)
def flush(self):
pass
......@@ -67,14 +67,14 @@ def test_help():
assert r1 == r2 == r3
assert 'Binary Structured Data Format' in r1
assert 'available commands' in r1.lower()
# Test help on a command
r1, e1 = run_local('help', 'version')
r2, e2 = run_local('version', '--help')
assert e1 == e2 == ''
assert r1 == r2
assert 'usage: bsdf version' in r1.lower()
# But command needs to exist
r3, e3 = run_local('help', 'fooo')
assert not r3
......@@ -82,22 +82,22 @@ def test_help():
def test_wrong_commands():
# No command
r, e = run_local()
assert not r
assert 'no command given' in e.lower()
# Nonexisting command
r, e = run_local('fooo')
assert not r
assert 'fooo' in e and 'invalid command' in e.lower()
# Wrong number of args
r, e = run_local('version', 'foo')
assert not r
assert 'positional arguments' in e.lower() or 'no arguments' in e.lower()
# Wrong kwargs
r, e = run_local('version', '--foo')
assert not r
......@@ -109,10 +109,10 @@ def test_version():
r2, e2 = run_local('--version')
r3, e3 = run_local('-v')
r4 = run_remote('version')
assert e1 == e2 == e3 == ''
assert r1 == r2 == r3 == r4
assert r1 == r2 == r3 == r4
assert r1.startswith('bsdf.py version ')
assert r1.strip().split(' ')[-1] == bsdf.__version__
......@@ -120,19 +120,19 @@ def test_version():
def test_info():
data1 = [3, 4, 5]
bsdf.save(tempfilename, data1)
r, e = run_local('info', tempfilename)
r = r.replace(' ', ' ').replace(' ', ' ').replace(' ', ' ')
assert not e
assert 'file_size' in r
assert 'valid: true' in r
assert 'version:' in r
# Not a file
r, e = run_local('info', '~/foo_does_not_exist.bsdf')
assert not r
assert 'invalid file' in e
# Not a bsdf file
with open(tempfilename, 'wb') as f:
pass
......@@ -148,7 +148,7 @@ def test_create():
r, e = run_local('create', tempfilename, '[3,4,5]*10')
data = bsdf.load(tempfilename)
assert data == [3, 4, 5] * 10
# Fail
r, e = run_local('create', tempfilename, '1/0')
assert not r
......@@ -158,25 +158,25 @@ def test_create():
def test_convert():
tempfilename1 = tempfilename
tempfilename2 = tempfilename + '.json'
data1 = [4, 5, 6]
bsdf.save(tempfilename, data1)
# Convert to json
r, e = run_local('convert', tempfilename1, tempfilename2)
assert not e
assert tempfilename2 in r
# Convert back
r, e = run_local('convert', tempfilename2, tempfilename1)
assert not e
assert tempfilename1 in r
# Check
assert open(tempfilename2, 'rb').read().decode().strip() == '[4, 5, 6]'
data2 = bsdf.load(tempfilename)
assert data1 == data2
# Fail, unknown extension
r, e = run_local('convert', tempfilename1+'.png', tempfilename1)
assert not r
......@@ -185,10 +185,10 @@ def test_convert():
r, e = run_local('convert', tempfilename1, tempfilename1+'.png')
assert not r
assert 'unknown' in e.lower() and 'extension' in e.lower() and 'save' in e
if sys.version_info < (3, ):
return
# Cannot convert bytes and nan
bsdf.save(tempfilename, [bsdf.Blob(b'xx'), float('nan')])
r, e = run_local('convert', tempfilename1, tempfilename2)
......@@ -198,11 +198,11 @@ def test_convert():
def test_view():
# Create file
data1 = [1, 2, 3, [4, 4, 4, 4, 4], 8, 9]
bsdf.save(tempfilename, data1)
# Test content, plain
r, e = run_local('view', tempfilename)
assert not e
......@@ -210,7 +210,7 @@ def test_view():
assert r.count('4') == 5
assert '5' in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Test content, plus info
r, e = run_local('view', tempfilename, '--info')
assert not e
......@@ -219,40 +219,40 @@ def test_view():
assert r.count('4') >= 5 # can also occur in meta info
assert '5' in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Test content, max depth 1
r, e = run_local('view', tempfilename, '--depth=1')
assert not e
assert r.count('4') == 0 # collapsed
assert '5' in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Test content, max depth 0
r, e = run_local('view', tempfilename, '--depth=0')
assert not e
assert r.count('\n') == 1 # all collapsed
assert '5' not in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Fail - not a file
r, e = run_local('view', '~/foo_does_not_exist.bsdf')
assert not r
assert 'invalid file' in e
# Fail - not a bsdf file
with open(tempfilename, 'wb') as f:
pass
r, e = run_local('view', tempfilename)
assert not r
assert 'does not look like a BSDF file' in e
# Fail - major version is off
bb = bsdf.encode(data1)
with open(tempfilename, 'wb') as f:
f.write(bb[:4] + b'\x00' + bb[5:])
r, e = run_local('view', tempfilename)
assert r == '' and 'different major version' in e
# Warn - minor version is lower than file
bb = bsdf.encode(data1)
with open(tempfilename, 'wb') as f:
......@@ -261,7 +261,7 @@ def test_view():
assert not e
assert 'higher minor version' in r
assert r.count('4') >= 5
# Test string truncation
too_long = u'x' * 200
just_right = u'x' + '_' * 38 + 'x'
......@@ -271,7 +271,7 @@ def test_view():
assert not e
assert too_long not in r and ('x'* 39 + u'\u2026') in r.replace('\\u2026', u'\u2026')
assert just_right in r
# Test float32 for cov
data1 = [3.14159, 42.0]
bsdf.save(tempfilename, data1, float64=False)
......@@ -295,18 +295,18 @@ def test_view():
def test_view_random():
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '..', '..', '_tools')))
import datagen
excludes = {}
for e in os.getenv('BSDF_TEST_EXCLUDES', '').split(','):
excludes[e.strip()] = True
types = list(datagen.ALL_TYPES)
if 'ndarray' in excludes:
types.remove('ndarray')
# Process a few random dicts
for iter in range(8):
random.seed(time.time())
......@@ -316,7 +316,7 @@ def test_view_random():
assert not e
assert r
# not much more to assert, except that it does not crash
# Process a few random lists
for iter in range(8):
random.seed(time.time())
......
......@@ -46,7 +46,7 @@ def test_liststreaming1():
def test_liststreaming_write_fails():
# Only write mode
f = io.BytesIO()
ls = bsdf.ListStream('r')
......@@ -56,21 +56,21 @@ def test_liststreaming_write_fails():
ls.append(3)
with raises(IOError):
ls.close()
# Dont append (or close) before use
ls = bsdf.ListStream()
with raises(IOError):
ls.append(3)
with raises(IOError):
ls.close()
# Only use once!
f = io.BytesIO()
ls = bsdf.ListStream()
bsdf.save(f, ls)
with raises(IOError):
bsdf.save(f, ls)
# Do not write when closed!
f = io.BytesIO()
ls = bsdf.ListStream()
......@@ -80,7 +80,7 @@ def test_liststreaming_write_fails():
ls.append(3)
with raises(IOError):
ls.close()
# Only ListStream
class MyStream(bsdf.BaseStream):
pass
......@@ -105,38 +105,38 @@ def test_liststreaming_write_fails():
def test_streaming_closing123():
""" Writing a streamed list, closing the stream. """
for iter in range(4):
f = io.BytesIO()
thelist = bsdf.ListStream()
bsdf.save(f, thelist)
thelist.append('hi')
# closing here will write the length of the stream, marking the stream as closed
if iter in (1, 2):
thelist.close()
for i in range(3):
thelist.append(i)
if iter == 2:
thelist.close()
if iter == 3:
thelist.close(True)
assert thelist.count == 4
assert thelist.index == thelist.count
bb = f.getvalue()
b1 = bsdf.decode(bb)
b2 = bsdf.decode(bb, load_streaming=True)
if iter in (0, 1, 2):
assert isinstance(b2, bsdf.ListStream) and not isinstance(b2, list)
else:
assert not isinstance(b2, bsdf.ListStream) and isinstance(b2, list)
if iter == 0:
# Not closed
assert b1 == ['hi', 0, 1, 2]
......@@ -182,13 +182,13 @@ def test_liststreaming_reading1():
assert x.next() == 2
with raises(StopIteration):
x.next()
# Support iteration
b = bsdf.decode(bb, load_streaming=True)
x = b[-1]
xx = [i for i in x]
assert xx == ['hi', 0, 1, 2]
# Cannot read after file is closed
f = io.BytesIO(bb)
b = bsdf.load(f, load_streaming=True)
......@@ -196,14 +196,14 @@ def test_liststreaming_reading1():
f.close()
with raises(IOError):
x.next()
# Cannot read when in read-mode
ls = bsdf.ListStream()
with raises(IOError):
ls.next()
with raises(IOError):
list(ls)
# mmm, this would never happen, I guess, but need associated file!
ls = bsdf.ListStream('r')
with raises(IOError):
......@@ -211,23 +211,23 @@ def test_liststreaming_reading1():
def test_liststream_modding():
# Create a file
ls = bsdf.ListStream()
with open(tempfilename, 'wb') as f:
bsdf.save(f, ls)
ls.append('foo')
ls.append('bar')
assert bsdf.load(tempfilename) == ['foo', 'bar']
# Append items hard-core more
with open(tempfilename, 'ab') as f:
f.write(b'v')
f.write(b'h*\x00') # ord('*') == 42
assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42]
# Append items using the BSDF API
with open(tempfilename, 'r+b') as f:
ls = bsdf.load(f, load_streaming=True)
......@@ -236,14 +236,14 @@ def test_liststream_modding():
ls.append(4)
ls.append(5)
ls.close() # also close here
assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42, 4, 5]
# Try adding more items
with open(tempfilename, 'ab') as f:
f.write(b'v')
f.write(b'h*\x00') # ord('*') == 42
# But no effect
assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42, 4, 5]
......@@ -251,23 +251,23 @@ def test_liststream_modding():
## Blobs
def test_blob_writing1():
# Use blob to specify bytes
blob = bsdf.Blob(b'xxxx')
bb1 = bsdf.encode([2, 3, blob, 5])
# Again, with extra size
blob = bsdf.Blob(b'xxxx', extra_size=100)
bb2 = bsdf.encode([2, 3, blob, 5])
# Again, with checksum
blob = bsdf.Blob(b'xxxx', use_checksum=True)
bb3 = bsdf.encode([2, 3, blob, 5])
assert len(bb2) == len(bb1) + 100
assert len(bb3) == len(bb1) + 16
assert bsdf.decode(bb1) == [2, 3, b'xxxx', 5]
assert bsdf.decode(bb2) == [2, 3, b'xxxx', 5]
assert bsdf.decode(bb3) == [2, 3, b'xxxx', 5]
......@@ -286,16 +286,16 @@ def test_blob_writing1():
def test_blob_reading1():
blob = bsdf.Blob(b'xxxx')
bb1 = bsdf.encode([2, 3, blob, 5])
res1 = bsdf.decode(bb1)
assert isinstance(res1[2], bytes)
res1 = bsdf.decode(bb1, lazy_blob=True)
assert not isinstance(res1[2], bytes) and isinstance(res1[2], bsdf.Blob)
res1[2].get_bytes() == b'xxxx'
......@@ -303,22 +303,22 @@ def test_blob_reading2():
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', extra_size=2))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
# Always seek first
blob.seek(0)
assert blob.read(2) == b'xx'
assert blob.tell() == 2
assert blob.read(2) == b'yy'
# We can overwrite, but changes an internal file that we cannot use :P
blob.write(b'aa')
blob.seek(0)
assert blob.read(2) == b'xx'
blob.seek(-2) # relative to allocated size
assert blob.tell() == 6
# can just about do this, due to extra size
blob.seek(8)
# But this is too far
......@@ -341,14 +341,14 @@ def test_blob_reading3(): # compression
blob = bsdf.load(f, lazy_blob=True)
#
blob.get_bytes() == b'xxyyzz'
# BZ2
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', compression=2))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
#
blob.get_bytes() == b'xxyyzz'
# But we cannot read or write
blob.seek(0)
with raises(IOError):
......@@ -358,11 +358,11 @@ def test_blob_reading3(): # compression
def test_blob_modding1(): # plain
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', extra_size=2))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
blob.seek(4)
blob.write(b'aa')
blob.update_checksum()
......@@ -370,11 +370,11 @@ def test_blob_modding1(): # plain
def test_blob_modding2(): # with checksum
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', extra_size=2, use_checksum=True))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
blob.seek(4)
blob.write(b'aa')
blob.update_checksum()
......@@ -383,26 +383,26 @@ def test_blob_modding2(): # with checksum
def test_blob_modding3(): # actual files
bsdf.save(tempfilename, bsdf.Blob(b'xxyyzz', extra_size=2))
# Can read, but not modify in rb mode
with open(tempfilename, 'rb') as f:
blob = bsdf.load(f, lazy_blob=True)
blob.seek(0)
assert blob.read(2) == b'xx'
blob.seek(4)
with raises(IOError):
blob.write(