Commit 826350bb authored by Micaël Bergeron's avatar Micaël Bergeron

fixing the serializers

parent e7822fb7
......@@ -4,19 +4,21 @@ import sys
from enum import Enum
from elt.cli import ActionEnum, OptionEnum, parser_logging
from elt.utils import setup_logging
from elt.schema.serializers.singer import SingerSerializer
from elt.schema.serializers.meltano import MeltanoSerializer
from elt.schema.serializers import serializer_for
def action_convert(args):
schema = load(, sys.stdin)
dump(sys.stdout, schema)
input = serializer_for(args.source, args.schema)
schema = input.load(sys.stdin).schema
output = serializer_for(args.destination, schema)
output.dump(sys.stdout)
class SchemaType(OptionEnum):
MELTANO = "meltano"
SINGER = "singer"
KETTLE = "kettle"
# SINGER = "singer"
class Actions(ActionEnum):
......@@ -28,11 +30,16 @@ def parse():
parser_logging(parser)
parser.add_argument('-S', '--schema',
required=True,
help="Schema name")
parser.add_argument('-s',
dest="source",
type=SchemaType,
choices=list(SchemaType),
required=True,
default=SchemaType.MELTANO,
help="Specifies input schema type.")
parser.add_argument('-d',
......
from .base import Serializer
from .kettle import KettleSerializer
from .meltano import MeltanoSerializer
serializer_map = {
'meltano': MeltanoSerializer,
'kettle': KettleSerializer,
}
def serializer_for(name: str, *args) -> Serializer:
return serializer_map[name](*args)
......@@ -17,10 +17,10 @@ class Serializer:
raise NotImplementedError
def load(self, reader: TextIO) -> 'Serializer':
return loads(reader.read())
return self.loads(reader.read())
def dumps(self) -> str:
raise NotImplementedError
def dump(self, writer: TextIO):
writer.write(self.dumps(self.schema))
writer.write(self.dumps())
......@@ -7,9 +7,10 @@ from .base import Serializer
data_type_map = {
'String': DBType.String,
'Number': DBType.Long,
'Number': DBType.Double,
'Boolean': DBType.Boolean,
'Date': DBType.Date
'Date': DBType.Date,
'Integer': DBType.Long,
}
......@@ -45,7 +46,7 @@ class KettleSerializer(Serializer):
dt_type = data_type_map[raw_type]
# date time can have a timezone or not, it depends on the format
# either a `date` or a `timestamp`: it depends on the format
if dt_type == DBType.Date:
dt_type = dt_type if raw_format == "yyyy-MM-dd" else DBType.Timestamp
......
......@@ -47,7 +47,7 @@ entity01:
long: bigint
text: text
entity01_long_mapping_key: long
""
"""
schema = MeltanoSerializer('yaml').loads(yaml_schema).schema
assert(len(schema.columns.values()) == 3)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment