Commit 83248235 authored by Dan Gass's avatar Dan Gass
Browse files

Merge branch 'dmgass/improve_pack' into 'alpha_6'

Improve pack and dump support.

See merge request dangass/plum!33
parent 47f62e21
......@@ -6,6 +6,7 @@ import os
import timeit
from argparse import ArgumentParser
from importlib import import_module
from fnmatch import fnmatch
from glob import glob
from texttable import Texttable
......@@ -18,22 +19,28 @@ except ImportError:
import time_baseline
def measure(modname, clsname, methname, nloops):
setup = f'from {modname} import {clsname}; timer = {clsname}()'
code = f'timer.{methname}()'
return timeit.timeit(code, setup=setup, number=nloops) * 1000000000 / nloops
def measure(method, nloops, repeat):
return min(timeit.repeat(method, number=nloops, repeat=repeat)) * 1000000000 / nloops
def main(sysargs=None):
parser = ArgumentParser(description='Compare memory pack/unpack package performance.')
parser.add_argument('--nloops', type=int, default=10000)
parser.add_argument('--nloops', type=int, default=10)
parser.add_argument('--repeat', type=int, default=1000)
parser.add_argument('--filter', type=str, default='*')
parser.add_argument('--modules', type=str, default=None)
args = parser.parse_args(sysargs)
clsnames = sorted(c for c in dir(time_baseline) if c.endswith('Timer'))
scripts = sorted(glob('time_*.py'))
if xnd is None:
scripts = [s for s in scripts if 'xnd' not in s]
modnames = [os.path.splitext(filename)[0] for filename in scripts]
if args.modules:
modnames = ['time_baseline'] + sorted(args.modules.split(','))
else:
scripts = sorted(glob('time_*.py'))
if xnd is None:
scripts = [s for s in scripts if 'xnd' not in s]
modnames = [os.path.splitext(filename)[0] for filename in scripts]
modules = [import_module(modname) for modname in modnames]
rows = [[''] + [modname.replace('time_', '') for modname in modnames[1:]]]
......@@ -42,6 +49,9 @@ def main(sysargs=None):
cls = getattr(time_baseline, clsname)
for methname in sorted(m for m in dir(cls) if m.startswith('time_')):
methdesc = methname.replace('time_', '')
desc = clsdesc + ' ' + methdesc.replace('_', ' ')
if not fnmatch(desc, args.filter):
continue
times = []
for modname, module in zip(modnames, modules):
try:
......@@ -50,11 +60,11 @@ def main(sysargs=None):
times.append(None)
else:
try:
getattr(modcls, methname)
method = getattr(modcls(), methname)
except AttributeError:
times.append(None)
else:
times.append(measure(modname, clsname, methname, args.nloops))
times.append(measure(method, args.nloops, args.repeat))
baseline = times.pop(0)
times = [t - baseline if t is not None else None for t in times]
......@@ -66,20 +76,17 @@ def main(sysargs=None):
for t in times:
if t is None:
percentages.append('')
#elif t is minval:
# percentages.append(' (fastest)')
else:
percentages.append(f' ({t / minval:.1f}X)')
times = ['' if t is None else f'{int(t)}ns' for t in times]
rows.append([clsdesc + ' ' + methdesc.replace('_', ' ')] +
[t + p for t, p in zip(times, percentages)])
table = Texttable()
table.set_cols_dtype(['t'] + (['a'] * (len(modnames) - 1)))
table.set_cols_align(["l"] + (['r'] * (len(modnames) - 1)))
table.add_rows(rows)
print(table.draw())
rows.append([desc] + [t + p for t, p in zip(times, percentages)])
table = Texttable()
table.set_cols_dtype(['t'] + (['a'] * (len(modnames) - 1)))
table.set_cols_align(["l"] + (['r'] * (len(modnames) - 1)))
table.add_rows(rows)
print(table.draw())
if __name__ == '__main__':
......
import timeit
class Statement(object):
nloops = 10
repeat = 1000
def __init__(self, desc, code):
self.desc = desc
self.code = code
@property
def time(self):
times = timeit.repeat(self.code, repeat=self.repeat, number=self.nloops)
return min(times) * 1000000000.0 / self.nloops
def compare(desc, statements):
print((" " + desc + " ").center(80, '='))
times = [statement.time for statement in statements]
base_time = min(times)
xfactors = [t / base_time for t in times]
for time, statement, xfactor in sorted(zip(times, statements, xfactors)):
print(f'{xfactor:.1f}X {time:.1f}ns {statement.desc:s}')
class MetaStruct1(type):
def __call__(cls, *args, **kwargs):
self = dict.__new__(cls, *args, **kwargs)
# dict.__init__(self, *args, **kwargs)
self['m0'] = 0
class Struct1(dict, metaclass=MetaStruct1):
pass
class MetaStruct2(type):
pass
class Struct2(dict, metaclass=MetaStruct2):
def __init__(self, *args, **kwargs):
self['m0'] = 0
class StructClsAttr:
A1_2 = 99, [1, 2, 3, 4, 99]
A1 = 99
A2 = [1, 2, 3, 4, 99]
@classmethod
def access_individually(cls):
return cls.A1 in cls.A2
@classmethod
def access_together(cls):
A1, A2 = cls.A1_2
return A1 in A2
class StructCopy1(dict):
__internals__ = (
{'m0': 0, 'm1': 1, 'm2': ...},
('m0', 'm1', 'm2'),
)
def __init__(self, m0=..., m1=..., m2=...):
defaults, names = self.__internals__
args = locals()
for name, default in defaults.items():
item = args[name]
if item is not None:
self[name] = item
else:
if default is ...:
raise TypeError('missing argument')
class StructCopy2(dict):
__internals__ = (
(0, 1, ...),
('m0', 'm1', 'm2'),
)
def __init__(self, m0=..., m1=..., m2=...):
defaults, names = self.__internals__
self['m0'] = m0 if m0 is not ... else defaults[0]
self['m1'] = m1 if m1 is not ... else defaults[1]
if m2 is not ...:
self['m2'] = m2
else:
raise TypeError(...)
class StructCopy3(dict):
__internals__ = (
{'m0': 0, 'm1': 1, 'm2': ...},
('m0', 'm1', 'm2'),
)
def __init__(self, *args, **kwargs):
defaults, names = self.__internals__
self.update(defaults)
if args:
if len(args) == 1:
self.update(args[0])
if kwargs:
self.update(kwargs)
if len(self) != len(names):
raise TypeError('extra args')
for name, value in self.items():
if value is ...:
raise TypeError('missing arg')
class StructSimple1(dict):
__internals__ = (
{'m0': 0, 'm1': 1, 'm2': ...},
('m0', 'm1', 'm2'),
)
def __init__(self, m0, m1, m2):
self['m0'] = m0
self['m1'] = m1
self['m2'] = m2
class StructSimple2(dict):
__internals__ = (
{'m0': 0, 'm1': 1, 'm2': ...},
('m0', 'm1', 'm2'),
)
def __init__(self, m0, m1, m2):
defaults, names = self.__internals__
values = locals()
for name in names:
self[name] = values[name]
class StructSimple3(dict):
__internals__ = (
{'m0': 0, 'm1': 1, 'm2': ...},
('m0', 'm1', 'm2'),
)
def __init__(self, **kwargs):
defaults, names = self.__internals__
if len(kwargs) != len(names):
raise TypeError('extra args')
for name in names:
self[name] = kwargs[name]
class MyList(list):
def __init__(self, m0, m1, m2):
self.extend((m0, m1, m2))
class MyTuple(tuple):
def __new__(cls, m0, m1, m2):
super().__new__(cls, (m0, m1, m2))
class ViaAttr:
A = 1
B = 2
C = 3
D = 4
E = 5
A_E = 1, 2, 3, 4, 5
@classmethod
def unpack5(cls):
return cls.A + cls.B + cls.C + cls.D + cls.E
@classmethod
def unpack1(cls):
a, b, c, d, e = cls.A_E
return a + b + c + d + e
class ViaSlotMeta(type):
def __new__(mcs, name, bases, namespace):
namespace['__slots__'] = ('A', 'B', 'C', 'D', 'E', 'A_E')
cls = type.__new__(mcs, name, bases, namespace)
cls.A = 1
cls.B = 2
cls.C = 3
cls.D = 4
cls.E = 5
cls.A_E = (1, 2, 3, 4, 5)
return cls
class ViaSlot(metaclass=ViaSlotMeta):
@classmethod
def unpack5(cls):
return cls.A + cls.B + cls.C + cls.D + cls.E
@classmethod
def unpack1(cls):
a, b, c, d, e = cls.A_E
return a + b + c + d + e
if __name__ == '__main__':
compare('meta __call__ vs. __init__', [
Statement('call', lambda : Struct1(m0=0)),
Statement('init', lambda : Struct2(m0=0)),
])
compare('class attribute access vs locals', [
Statement('as clsattr', StructClsAttr.access_individually),
Statement('as locals', StructClsAttr.access_together),
])
compare('dict init dedicated args vs. generic', [
Statement('dedicated - locals', lambda : StructCopy1(m0=0, m2=2)),
Statement('dedicated - statements', lambda : StructCopy2(m0=0, m2=2)),
Statement('generic', lambda : StructCopy3(m0=0, m2=2)),
Statement('dedicated+', lambda : StructCopy2(**{"m0": 0, "m2": 2})),
Statement('generic+', lambda : StructCopy3(**{"m0": 0, "m2": 2})),
Statement('simple dedicated', lambda : StructSimple1(m0=0, m1=1, m2=2)),
Statement('simple locals', lambda : StructSimple2(m0=0, m1=1, m2=2)),
Statement('simple generic', lambda : StructSimple3(m0=0, m1=1, m2=2)),
])
compare('list vs. tuple', [
Statement('list', lambda: MyList(1, 2, 3)),
Statement('tuple', lambda : MyTuple(1, 2, 3)),
])
compare('via attr vs slot', [
Statement('attr1', ViaAttr.unpack1),
Statement('attr5', ViaAttr.unpack5),
Statement('slot1', ViaSlot.unpack1),
Statement('slot5', ViaSlot.unpack5),
])
......@@ -37,3 +37,54 @@ class UInt32Timer(UInt32Data):
def time_pack_ne(self):
pass
class StructureData:
BE_BYTES, LE_BYTES, NE_BYTES = make_bln_bytes(b'\x01\x02\x03\x04', b'\x05\x06', b'\x07')
VALUE_M0 = 0x01020304
VALUE_M1 = 0x0506
VALUE_M2 = 0x07
class DictTimer(StructureData):
def time_unpack_be(self):
pass
def time_unpack_le(self):
pass
def time_unpack_ne(self):
pass
def time_pack_be(self):
pass
def time_pack_le(self):
pass
def time_pack_ne(self):
pass
class ListTimer(StructureData):
def time_unpack_be(self):
pass
def time_unpack_le(self):
pass
def time_unpack_ne(self):
pass
def time_pack_be(self):
pass
def time_pack_le(self):
pass
def time_pack_ne(self):
pass
......@@ -4,10 +4,25 @@
from cffi import FFI
from time_baseline import UInt32Data
from time_baseline import StructureData, UInt32Data
ffi = FFI()
ffi.cdef("""
typedef struct {
unsigned int m0;
unsigned short int m1;
unsigned char m2;
} mystruct;
typedef struct {
unsigned int f1: 4;
unsigned int f2: 4;
unsigned int f3: 8;
} bitstruct;
""")
class UInt32Timer(UInt32Data):
......@@ -15,4 +30,13 @@ class UInt32Timer(UInt32Data):
assert ffi.cast("unsigned int *", ffi.from_buffer(self.NE_BYTES))[0] == self.VALUE
def time_pack_ne(self):
assert bytes(ffi.buffer(ffi.new('unsigned int *', self.VALUE))) == self.NE_BYTES
bytes(ffi.buffer(ffi.new('unsigned int *', self.VALUE)))
class DictTimer(StructureData):
def time_unpack_ne(self):
assert ffi.cast("mystruct *", ffi.from_buffer(self.NE_BYTES))[0].m0 == self.VALUE_M0
def time_pack_ne(self):
bytes(ffi.buffer(ffi.new('mystruct *', {'m0': self.VALUE_M0, 'm1': self.VALUE_M1, 'm2': self.VALUE_M2})))
......@@ -3,8 +3,11 @@
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
from construct import Int32ul, Int32ub, Int32un
from construct import Int16ul, Int16ub, Int16un
from construct import Int8ul, Int8ub, Int8un
from construct import Sequence, Struct
from time_baseline import UInt32Data
from time_baseline import UInt32Data, StructureData
class UInt32Timer(UInt32Data):
......@@ -19,10 +22,62 @@ class UInt32Timer(UInt32Data):
assert Int32un.parse(self.NE_BYTES) == self.VALUE
def time_pack_be(self):
assert Int32ub.build(self.VALUE) == self.BE_BYTES
Int32ub.build(self.VALUE)
def time_pack_le(self):
assert Int32ul.build(self.VALUE) == self.LE_BYTES
Int32ul.build(self.VALUE)
def time_pack_ne(self):
assert Int32un.build(self.VALUE) == self.NE_BYTES
Int32un.build(self.VALUE)
seq_be = Sequence(Int32ub, Int16ub, Int8ub)
seq_le = Sequence(Int32ul, Int16ul, Int8ul)
seq_ne = Sequence(Int32un, Int16un, Int8un)
class ListTimer(StructureData):
def time_unpack_be(self):
assert seq_be.parse(self.BE_BYTES)[0] == self.VALUE_M0
def time_unpack_le(self):
assert seq_le.parse(self.LE_BYTES)[0] == self.VALUE_M0
def time_unpack_ne(self):
assert seq_ne.parse(self.NE_BYTES)[0] == self.VALUE_M0
def time_pack_be(self):
seq_be.build([self.VALUE_M0, self.VALUE_M1, self.VALUE_M2])
def time_pack_le(self):
seq_le.build([self.VALUE_M0, self.VALUE_M1, self.VALUE_M2])
def time_pack_ne(self):
seq_ne.build([self.VALUE_M0, self.VALUE_M1, self.VALUE_M2])
structure_be = Struct('m0' / Int32ub, 'm1' / Int16ub, 'm2' / Int8ub)
structure_le = Struct('m0' / Int32ul, 'm1' / Int16ul, 'm2' / Int8ul)
structure_ne = Struct('m0' / Int32un, 'm1' / Int16un, 'm2' / Int8un)
class DictTimer(StructureData):
def time_unpack_be(self):
assert structure_be.parse(self.BE_BYTES)['m0'] == self.VALUE_M0
def time_unpack_le(self):
assert structure_le.parse(self.LE_BYTES)['m0'] == self.VALUE_M0
def time_unpack_ne(self):
assert structure_ne.parse(self.NE_BYTES)['m0'] == self.VALUE_M0
def time_pack_be(self):
structure_be.build({'m0': self.VALUE_M0, 'm1': self.VALUE_M1, 'm2': self.VALUE_M2})
def time_pack_le(self):
structure_le.build({'m0': self.VALUE_M0, 'm1': self.VALUE_M1, 'm2': self.VALUE_M2})
def time_pack_ne(self):
structure_ne.build({'m0': self.VALUE_M0, 'm1': self.VALUE_M1, 'm2': self.VALUE_M2})
......@@ -3,11 +3,13 @@
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
from plum import pack, unpack
from plum.int.big import UInt32 as UInt32be
from plum.int.little import UInt32 as UInt32le
from plum.int.native import UInt32 as UInt32ne
from plum.int.big import UInt8 as UInt8be, UInt16 as UInt16be, UInt32 as UInt32be
from plum.int.little import UInt8 as UInt8le, UInt16 as UInt16le, UInt32 as UInt32le
from plum.int.native import UInt8 as UInt8ne, UInt16 as UInt16ne, UInt32 as UInt32ne
from plum.structure import Structure
from plum.structure import Structure
from time_baseline import UInt32Data
from time_baseline import UInt32Data, StructureData
class UInt32Timer(UInt32Data):
......@@ -29,3 +31,82 @@ class UInt32Timer(UInt32Data):
def time_pack_ne(self):
assert pack(UInt32ne, self.VALUE) == self.NE_BYTES
class StructBE(Structure):
m0: UInt32be
m1: UInt16be
m2: UInt8be
class StructLE(Structure):
m0: UInt32le
m1: UInt16le
m2: UInt8le
class StructNE(Structure):
m0: UInt32ne
m1: UInt16ne