Commit 70e6dac1 authored by Jacob Schatz's avatar Jacob Schatz

Merge branch 'monorepo' into 'master'

Monorepo

See merge request meltano/meltano!1
parents 8c805cb5 6529daab
*/__pycache__
*.log
*.pyc
*.DS_Store
*.idea
*.csv
*.egg-info/
tmp_log.txt
.env
.vscode/
build/
dist/
.pytest_cache/
......@@ -66,30 +66,37 @@ publish:
- dev
when: manual
dev_build_meltano_cli:
.docker_build: &docker_build
image: docker:latest
stage: build
variables:
DOCKERFILE: .
DOCKER_DRIVER: overlay2
services:
- docker:dind
script:
- export DOCKER_IMAGE_NAME=${IMAGE_NAME:-$CI_PROJECT_SLUG}
- export DOCKER_IMAGE_TAG=${IMAGE_TAG:-$CI_COMMIT_REF_NAME}
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $CI_REGISTRY_IMAGE/meltano-cli:$CI_COMMIT_REF_NAME .
- docker push $CI_REGISTRY_IMAGE/meltano-cli:$CI_COMMIT_REF_NAME
- docker build -t $CI_REGISTRY_IMAGE/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG -f $DOCKERFILE .
- docker push $CI_REGISTRY_IMAGE/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG
dev_build_meltano:
<<: *docker_build
image: docker:latest
variables:
DOCKER_DRIVER: overlay2
IMAGE_NAME: meltano
DOCKERFILE: ./docker/Dockerfile
except:
- master
publish_build_meltano_cli:
image: docker:latest
stage: build
build_meltano:
<<: *docker_build
variables:
DOCKER_DRIVER: overlay2
services:
- docker:dind
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $CI_REGISTRY_IMAGE/meltano-cli:latest .
- docker push $CI_REGISTRY_IMAGE/meltano-cli:latest
IMAGE_NAME: meltano
IMAGE_TAG: latest
DOCKERFILE: ./docker/Dockerfile
only:
- master
This diff is collapsed.
version: '3.1'
services:
db:
container_name: meltano-dw
build: docker/warehouse
image: meltano/dw:devenv
environment:
POSTGRES_DB: $PG_DATABASE
POSTGRES_USER: $PG_USERNAME
POSTGRES_PASSWORD: $PG_PASSWORD
ports:
- 5432:5432
elt:
container_name: meltano-elt
build: docker/elt
image: registry.gitlab.com/meltano/meltano-elt/extract:devenv
command: tail -f /dev/null # keep running
# command: pipenv shell
volumes:
- .:/meltano
depends_on:
- db
......@@ -2,7 +2,7 @@ FROM python:3.6.6
RUN set -ex && pip3 install pipenv --upgrade
ADD . /app/
ADD src/meltano /app/
WORKDIR /app
RUN pip install .
\ No newline at end of file
FROM registry.gitlab.com/meltano/meltano-elt/extract:latest
ENV SHELL=/bin/bash
WORKDIR /bizops
RUN pip3 install pipenv
RUN pipenv --site-packages
RUN pipenv install --dev --skip-lock
\ No newline at end of file
FROM postgres:alpine
COPY init-user.sh /docker-entrypoint/initdb.d/init-user.sh
#!/bin/bash
set -e
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
CREATE DATABASE ${POSTGRES_DB};
GRANT ALL PRIVILEGES ON DATABASE ${POSTGRES_DB} TO ${POSTGRES_USER};
EOSQL
This diff is collapsed.
{
"presets": [
["env", {
"modules": false,
"targets": {
"browsers": ["> 1%", "last 2 versions", "not ie <= 8"]
}
}],
"stage-2"
],
"plugins": ["transform-vue-jsx", "transform-runtime"],
"env": {
"test": {
"presets": ["env", "stage-2"],
"plugins": ["transform-vue-jsx", "transform-es2015-modules-commonjs", "dynamic-import-node"]
}
}
}
root = true
[*]
charset = utf-8
indent_style = space
indent_size = 2
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
/build/
/config/
/dist/
/*.js
/test/unit/coverage/
// https://eslint.org/docs/user-guide/configuring
module.exports = {
root: true,
parserOptions: {
parser: 'babel-eslint'
},
env: {
browser: true,
},
// https://github.com/vuejs/eslint-plugin-vue#priority-a-essential-error-prevention
// consider switching to `plugin:vue/strongly-recommended` or `plugin:vue/recommended` for stricter rules.
extends: ['plugin:vue/essential', 'airbnb-base'],
// required to lint *.vue files
plugins: [
'vue'
],
// check if imports actually resolve
settings: {
'import/resolver': {
webpack: {
config: 'build/webpack.base.conf.js'
}
}
},
// add your custom rules here
rules: {
// don't require .vue extension when importing
'import/extensions': ['error', 'always', {
js: 'never',
vue: 'never'
}],
// disallow reassignment of function parameters
// disallow parameter object manipulation except for specific exclusions
'no-param-reassign': ['error', {
props: true,
ignorePropertyModificationsFor: [
'state', // for vuex state
'acc', // for reduce accumulators
'e' // for e.returnvalue
]
}],
// allow optionalDependencies
'import/no-extraneous-dependencies': ['error', {
optionalDependencies: ['test/unit/index.js']
}],
// allow debugger during development
'no-debugger': process.env.NODE_ENV === 'production' ? 'error' : 'off'
}
}
.DS_Store
node_modules/
dist/
npm-debug.log*
yarn-debug.log*
yarn-error.log*
/test/unit/coverage/
/test/e2e/reports/
selenium-debug.log
api/parser
# Editor directories and files
.idea
.vscode
*.suo
*.ntvs*
*.njsproj
*.sln
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
api/tmp/output.json
// https://github.com/michael-ciniawsky/postcss-load-config
module.exports = {
"plugins": {
"postcss-import": {},
"postcss-url": {},
// to edit target browsers: use "browserslist" field in package.json
"autoprefixer": {}
}
}
# melt
> meltano visualization with lookml files
## Build Setup
``` bash
# install dependencies
npm install
# serve with hot reload at localhost:8080
npm run dev
# build for production with minification
npm run build
# build for production and view the bundle analyzer report
npm run build --report
# run unit tests
npm run unit
# run e2e tests
npm run e2e
# run all tests
npm test
```
For a detailed explanation on how things work, check out the [guide](http://vuejs-templates.github.io/webpack/) and [docs for vue-loader](http://vuejs.github.io/vue-loader).
# Running the API
## First Time
Open the python shell. `python`
```
cd api
python
```
From the shell:
```
>>> from app import db
>>> db.create_all()
```
```
pipenv install
```
## Run API For Real
```
cd api
flask run
```
\ No newline at end of file
[[source]]
name = "pypi"
verify_ssl = true
url = "https://pypi.org/simple"
[packages]
sqlalchemy = "*"
flask = "*"
"psycopg2" = "*"
"psycopg2-binary" = "*"
flask-cors = "*"
gitpython = "*"
markdown = "*"
flask-sqlalchemy = "*"
pypika = "*"
[dev-packages]
[requires]
python_version = "3.7"
This diff is collapsed.
import os
import logging
import datetime
from logging.handlers import RotatingFileHandler
from flask import Flask, request
from flask import jsonify
from flask_sqlalchemy import SQLAlchemy
from flask_cors import CORS
from external_connector import ExternalConnector
app = Flask(__name__)
app.config.from_object('config')
if os.environ['FLASK_ENV'] == 'development':
CORS(app)
db = SQLAlchemy(app)
connector = ExternalConnector()
logger = logging.getLogger('melt_logger')
handler = RotatingFileHandler(app.config['LOG_PATH'], maxBytes=2000, backupCount=10)
logger.addHandler(handler)
now = str(datetime.datetime.utcnow().strftime('%b %d %Y %I:%M:%S:%f'))
logger.warning('Melt started at: {}'.format(now))
@app.before_request
def before_request():
logger.info('[{}] request: {}'.format(request.remote_addr, now))
from controllers import projects
from controllers import repos
from controllers import settings
from controllers import sql
app.register_blueprint(projects.bp)
app.register_blueprint(repos.bp)
app.register_blueprint(settings.bp)
app.register_blueprint(sql.bp)
@app.route("/")
def hello():
return jsonify({"hello": 1})
@app.route("/drop_it_like_its_hot")
def reset_db():
try:
db.drop_all()
except sqlalchemy.exc.OperationalError as err:
logging.error("Failed drop database.")
db.create_all()
return jsonify({"dropped_it":"like_its_hot"})
\ No newline at end of file
import os
def get_env_variable(name):
try:
return os.environ[name]
except KeyError:
message = "Expected environment variable '{}' not set.".format(name)
raise Exception(message)
# the values of those depend on your setup
POSTGRES_URL = get_env_variable("POSTGRES_URL")
POSTGRES_USER = get_env_variable("POSTGRES_USER")
POSTGRES_PASSWORD = get_env_variable("POSTGRES_PASSWORD")
POSTGRES_DB = get_env_variable("POSTGRES_DB")
LOG_PATH = get_env_variable('LOG_PATH')
ENV = 'development'
BASE_DIR = os.path.abspath(os.path.dirname(__file__))
SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{user}:{pw}@{url}/{db}'.format(user=POSTGRES_USER,pw=POSTGRES_PASSWORD,url=POSTGRES_URL,db=POSTGRES_DB)
SQLALCHEMY_TRACK_MODIFICATIONS = False
# SQLALCHEMY_ECHO=True
THREADS_PER_PAGE = 2
SECRET_KEY = "damnitjanice"
\ No newline at end of file
from flask import (
Blueprint, jsonify, request
)
from models.projects import Project
from models.projects import Settings
from app import db
bp = Blueprint('projects', __name__, url_prefix='/projects')
@bp.route('/', methods=['GET'])
def index():
p = Project.query.first()
return jsonify({'name': p.name, 'git_url': p.git_url}) if p else jsonify({'name': '', 'git_url': ''})
@bp.route('/new', methods=['POST'])
def add():
incoming = request.get_json()
name = incoming.get('name')
git_url = incoming.get('git_url')
settings = Settings()
project = Project(name=name, git_url=git_url)
project.settings = settings
db.session.add(settings)
db.session.add(project)
db.session.commit()
return jsonify({'name': name, 'git_url': git_url})
\ No newline at end of file
This diff is collapsed.
from app import db
from flask import (
Blueprint, jsonify, request, config, current_app
)
from models.projects import Settings
bp = Blueprint('settings', __name__, url_prefix='/settings')
@bp.route('/', methods=['GET'])
def index():
settings = Settings.query.first()
return jsonify(settings.serializable())
@bp.route('/new', methods=['POST'])
def new():
settings = request.get_json()
current_settings = Settings.query.first()
current_settings.settings = settings
db.session.add(current_settings)
db.session.commit()
return jsonify(settings)
@bp.route('/connections/<name>/test')
def test(name):
current_settings = Settings.query.first().settings
connections = current_settings['connections']
try:
found_connection = next(connection for connection in connections if connection['name'] == name)
except Exception as e:
found_connection = {}
return jsonify(found_connection)
\ No newline at end of file
import json
import sqlalchemy
import psycopg2
import re
from collections import OrderedDict
from decimal import Decimal
from datetime import date, datetime
from pypika import Query, Table, Field
from app import db
from flask import (
Blueprint, jsonify, request
)
from .utils import SqlHelper
from models.projects import Project, Settings
from models.data import (
Model, Explore, View, Dimension, Measure, Join
)
connections = {}
def default(obj):
if isinstance(obj, Decimal):
return str(obj)
elif isinstance(obj, (datetime, date)):
return obj.isoformat()
raise TypeError("Object of type '%s' is not JSON serialize-able" % type(obj).__name__)
def update_connections():
current_connections = Settings.query.first().settings['connections']
for connection in current_connections:
connection_name = connection['name']
if connection_name not in connections:
this_connection = {}
if connection['dialect'] == 'postgresql':
connection_url = 'postgresql+psycopg2://{user}:{pw}@{host}:{port}/{db}'.format(user=connection['username'],pw=connection['password'],host=connection['host'],port=connection['port'], db=connection['database'])
this_connection['connection_url'] = connection_url
this_connection['engine'] = sqlalchemy.create_engine(this_connection['connection_url'])
connections[connection_name] = this_connection
bp = Blueprint('sql', __name__, url_prefix='/sql')
@bp.route('/', methods=['GET'])
def index():
return jsonify({'result': True})
@bp.route('/get/<model_name>/<explore_name>', methods=['POST'])
def get_sql(model_name, explore_name):
update_connections()
sqlHelper = SqlHelper()
model = Model.query.filter(Model.name == model_name).first()
explore = Explore.query.filter(Explore.name == explore_name).first()
incoming_json = request.get_json()
view_name = incoming_json['view']
limit = incoming_json['limit']
view = View.query.filter(View.name == view_name).first()
incoming_dimensions = incoming_json['dimensions']
incoming_joins = incoming_json['joins']
incoming_dimension_groups = incoming_json['dimension_groups']
incoming_measures = incoming_json['measures']
incoming_filters = incoming_json['filters']
# get all timeframes
timeframes = [t['timeframes'] for t in incoming_dimension_groups]
# flatten list of timeframes
timeframes = [y for x in timeframes for y in x]
group_by = sqlHelper.group_by(incoming_joins, incoming_dimensions, timeframes)
filter_by = sqlHelper.filter_by(incoming_filters, explore_name)
to_run = incoming_json['run']
base_table = view.settings['sql_table_name']
dimensions = filter(lambda x: x.name in incoming_dimensions, view.dimensions)
set_dimensions = dimensions
dimensions = map(lambda x: x.settings['sql'].replace("${TABLE}", explore_name), dimensions)
dimensions = ',\n\t '.join(map(lambda x: '{}'.format(x), dimensions))
dimension_groups = sqlHelper.dimension_groups(view.name, explore_name, incoming_dimension_groups)
measures = filter(lambda x: x.name in incoming_measures, view.measures)
set_measures = list(measures)
measures = filter(lambda x: x.name in incoming_measures, view.measures)
measures = ',\n\t '.join([sqlHelper.get_func(x.name, x.settings['type'], explore_name, x.settings['sql']) for x in measures])
join_dimensions = sqlHelper.join_dimensions(incoming_joins)
join_measures = sqlHelper.join_measures(incoming_joins)
if join_dimensions:
dimensions = join_dimensions
if join_measures:
measures = join_measures