move the tests in the correct folder

parent 062d39ca
......@@ -4,7 +4,7 @@ verify_ssl = true
name = "pypi"
[packages]
"-e src/" = "*" # installs dep spesified in the `install_requires` part of the setup.py
"b7287bc" = {path = "./src", editable = true}
[dev-packages]
"flake8" = "*"
......@@ -12,4 +12,4 @@ name = "pypi"
pytest = "*"
[requires]
python_version = "3.6"
python_version = "3.6"
\ No newline at end of file
version: '3.1'
services:
db:
container_name: meltano-dw
build: docker/warehouse
image: meltano/dw:devenv
environment:
POSTGRES_DB: $PG_DATABASE
POSTGRES_USER: $PG_USERNAME
POSTGRES_PASSWORD: $PG_PASSWORD
ports:
- 5432:5432
elt:
container_name: meltano-elt
build: docker/elt
image: registry.gitlab.com/meltano/meltano-elt/extract:devenv
command: tail -f /dev/null # keep running
# command: pipenv shell
volumes:
- .:/meltano
depends_on:
- db
FROM registry.gitlab.com/meltano/meltano-load-postgresql
ADD ../../meltano-extract-fastly /app/extract/fastly
RUN pip install /app/extract/fastly
CMD set -o pipefail && meltano extract fastly | meltano load postgresql
\ No newline at end of file
FROM registry.gitlab.com/meltano/meltano-elt/extract:latest
ENV SHELL=/bin/bash
WORKDIR /bizops
RUN pip3 install pipenv
RUN pipenv --site-packages
RUN pipenv install --dev --skip-lock
\ No newline at end of file
FROM postgres:alpine
COPY init-user.sh /docker-entrypoint/initdb.d/init-user.sh
#!/bin/bash
set -e
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
CREATE DATABASE ${POSTGRES_DB};
GRANT ALL PRIVILEGES ON DATABASE ${POSTGRES_DB} TO ${POSTGRES_USER};
EOSQL
import logging
from .main import root
def main():
logging.basicConfig(level=logging.DEBUG)
root()
......@@ -3,6 +3,7 @@ import importlib
import sys
import logging
import functools
import os
from meltano.common.service import MeltanoService
from meltano.common.manifest_reader import ManifestReader
......@@ -12,6 +13,7 @@ from meltano.common.db import DB
from meltano.common.utils import setup_db, pop_all
from meltano.schema import schema_apply
from meltano.stream import MeltanoStream
from meltano.cli.params import MANIFEST_TYPE
service = MeltanoService()
......@@ -44,11 +46,6 @@ def build_loader(name):
return loader
def build_manifest(source, manifest_path):
with open(manifest_path, 'r') as file:
return ManifestReader(source).load(file)
def register_manifest(manifest):
for id in service.register_manifest(manifest):
logging.debug("Registered entity {}".format(id))
......@@ -83,7 +80,6 @@ def db_options(func):
@click.password_option(envvar='PG_PASSWORD')
@functools.wraps(func)
def wrapper(*args, **kwargs):
import pdb; pdb.set_trace()
config = pop_all(("schema", "host", "port", "database", "table_name", "user", "password"), kwargs)
DB.setup(**config)
return func(*args, **kwargs)
......@@ -117,11 +113,9 @@ def discover(extractor, source_name):
@root.command()
@click.argument('extractor')
@click.argument('manifest')
@click.argument('manifest', type=MANIFEST_TYPE)
def extract(extractor, manifest):
source, _ = manifest.split(".")
register_manifest(build_manifest(source, manifest))
register_manifest(manifest)
extractor = build_extractor(extractor)
logging.info("Extracting data...")
......@@ -131,11 +125,9 @@ def extract(extractor, manifest):
@root.command()
@click.argument('loader')
@click.argument('manifest')
@click.argument('manifest', type=MANIFEST_TYPE)
def load(loader, manifest):
source, _ = manifest.split(".")
register_manifest(build_manifest(source, manifest))
register_manifest(manifest)
loader = build_loader(loader)
logging.info("Waiting for data...")
......@@ -145,15 +137,7 @@ def load(loader, manifest):
@root.command()
@db_options
@click.argument('manifest')
@click.argument('manifest', type=MANIFEST_TYPE)
def apply_schema(manifest):
source, _ = manifest.split(".")
manifest = build_manifest(source, manifest)
with DB.open() as db:
schema_apply(db, manifest.as_schema())
def main():
logging.basicConfig(level=logging.DEBUG)
root()
import click
import os
from meltano.common.manifest_reader import ManifestReader
class ManifestParamType(click.ParamType):
name = 'manifest'
def convert(self, value, param, ctx):
source_name, _ = os.path.basename(value).split(".")
reader = ManifestReader(source_name)
try:
with open(value, 'r') as file:
return reader.load(file)
except IOError:
self.fail('cannot open %s.' % value, param, ctx)
MANIFEST_TYPE = ManifestParamType()
......@@ -21,7 +21,6 @@ class ManifestWriter:
*entities
])
import pdb; pdb.set_trace()
yaml.dump(raw_manifest, self.file,
Dumper=yamlordereddictloader.SafeDumper,
default_flow_style=False)
......
......@@ -83,7 +83,6 @@ class FastlyExtractor(MeltanoExtractor):
"""
async with self.create_session() as session:
billing = await self.req(session, "billing/v2/year/2018/month/06")
# import pdb; pdb.set_trace()
yield df_to_entity("Billing", billing)
def discover_entities(self):
......
......@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
setup(
name='meltano',
version='0.2.0-dev',
version='0.1.0-dev',
description='Meltano framework.',
author='Meltano Team & Contributors',
author_email='meltano@gitlab.com',
......
......@@ -7,8 +7,6 @@ def test_config():
.loads(SAMPLE_KETTLE_CONFIG_EXTRA) \
.schema
import pdb; pdb.set_trace()
assert(("User", "Id") in schema.columns)
assert(("User", "LastName") in schema.columns)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment