Commit 750c00b5 authored by Alex Zamai's avatar Alex Zamai Committed by Jacob Schatz

Stripe Extraction

parent 40c3d3bf
import psycopg2
import getpass
import os
import datetime
import logging
import typing
from datetime import timedelta, datetime, date, time
from enum import Enum
from argparse import ArgumentParser
......@@ -41,6 +41,7 @@ class Password:
class LogLevel:
LOG_LEVEL_STRINGS = ['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG']
@staticmethod
def parse(value):
log_level_int = getattr(logging, value, logging.INFO)
# check the logging log_level_choices have not changed from our expected values
......@@ -49,34 +50,28 @@ class LogLevel:
class DateWindow:
def parse_date(value):
if value is None:
return datetime.date.today()
return datetime.datetime.strptime(value, "%Y-%m-%d")
def __init__(self, args, formatter=datetime.datetime.isoformat):
def __init__(self, args, formatter=datetime.isoformat):
self.formatter = formatter
if args.days is not None:
today = datetime.date.today()
days = lambda d: datetime.timedelta(days=d)
if args.days:
today = date.today()
# Tomorrow at 00:00:00 UTC
self.end = datetime.datetime.combine(today + days(1),
datetime.time())
self.end = datetime.combine(today + timedelta(days=1), time())
# N days ago at 00:00:00 UTC
self.start = datetime.datetime.combine(today - days(args.days),
datetime.time())
self.start = datetime.combine(today - timedelta(days=args.days), time())
else:
self.start = DateWindow.parse_date(args.start)
self.end = DateWindow.parse_date(args.end)
@staticmethod
def parse_date(value):
if value is None:
return date.today()
return datetime.strptime(value, "%Y-%m-%d")
def range(self):
return (self.start, self.end)
def formatted_range(self):
def formatted_range(self) -> typing.Iterable:
return map(self.formatter, self.range())
......
......@@ -24,7 +24,6 @@ def engine_uri(**db_config):
SystemModel = declarative_base(metadata=MetaData(schema='meltano'))
Model = declarative_base()
session_factory = sessionmaker()
Session = scoped_session(session_factory)
......
......@@ -44,13 +44,16 @@ Column = namedtuple('Column', [
class Schema:
@staticmethod
def mapping_key_name(column: Column):
return "{}_{}_mapping_key".format(column.table_name,
column.column_name)
@staticmethod
def table_key(column: Column):
return column.table_name
@staticmethod
def column_key(column: Column):
return (column.table_name, column.column_name)
......
import re
import sys
import psycopg2
import logging
import re
from requests.auth import HTTPBasicAuth
from functools import reduce
from elt.db import DB
......@@ -19,6 +20,7 @@ def compose(*fs):
"""
return reduce(lambda f, g: lambda x: f(g(x)), fs, lambda x: x)
# from http://www.dolphmathews.com/2012/09/slugify-string-in-python.html
def slugify(s):
"""
......@@ -58,13 +60,19 @@ def slugify(s):
return s
def setup_db(args=None):
if args is None:
DB.setup()
else:
DB.setup(**vars(args))
def setup_logging(args):
logging.basicConfig(stream=sys.stdout,
format="[%(levelname)s][%(asctime)s] %(message)s",
level=int(args.log_level))
def get_basic_auth(user, token):
return HTTPBasicAuth(user, token)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment