Commit 6d2eba29 authored by Micaël Bergeron's avatar Micaël Bergeron

Add 'src/melt/' from commit 'f9a47087'

git-subtree-dir: src/melt
git-subtree-mainline: 3d44a6de
git-subtree-split: f9a47087
parents 3d44a6de f9a47087
{
"presets": [
["env", {
"modules": false,
"targets": {
"browsers": ["> 1%", "last 2 versions", "not ie <= 8"]
}
}],
"stage-2"
],
"plugins": ["transform-vue-jsx", "transform-runtime"],
"env": {
"test": {
"presets": ["env", "stage-2"],
"plugins": ["transform-vue-jsx", "transform-es2015-modules-commonjs", "dynamic-import-node"]
}
}
}
root = true
[*]
charset = utf-8
indent_style = space
indent_size = 2
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
/build/
/config/
/dist/
/*.js
/test/unit/coverage/
// https://eslint.org/docs/user-guide/configuring
module.exports = {
root: true,
parserOptions: {
parser: 'babel-eslint'
},
env: {
browser: true,
},
// https://github.com/vuejs/eslint-plugin-vue#priority-a-essential-error-prevention
// consider switching to `plugin:vue/strongly-recommended` or `plugin:vue/recommended` for stricter rules.
extends: ['plugin:vue/essential', 'airbnb-base'],
// required to lint *.vue files
plugins: [
'vue'
],
// check if imports actually resolve
settings: {
'import/resolver': {
webpack: {
config: 'build/webpack.base.conf.js'
}
}
},
// add your custom rules here
rules: {
// don't require .vue extension when importing
'import/extensions': ['error', 'always', {
js: 'never',
vue: 'never'
}],
// disallow reassignment of function parameters
// disallow parameter object manipulation except for specific exclusions
'no-param-reassign': ['error', {
props: true,
ignorePropertyModificationsFor: [
'state', // for vuex state
'acc', // for reduce accumulators
'e' // for e.returnvalue
]
}],
// allow optionalDependencies
'import/no-extraneous-dependencies': ['error', {
optionalDependencies: ['test/unit/index.js']
}],
// allow debugger during development
'no-debugger': process.env.NODE_ENV === 'production' ? 'error' : 'off'
}
}
.DS_Store
node_modules/
dist/
npm-debug.log*
yarn-debug.log*
yarn-error.log*
/test/unit/coverage/
/test/e2e/reports/
selenium-debug.log
api/parser
# Editor directories and files
.idea
.vscode
*.suo
*.ntvs*
*.njsproj
*.sln
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
api/tmp/output.json
// https://github.com/michael-ciniawsky/postcss-load-config
module.exports = {
"plugins": {
"postcss-import": {},
"postcss-url": {},
// to edit target browsers: use "browserslist" field in package.json
"autoprefixer": {}
}
}
# melt
> meltano visualization with lookml files
## Build Setup
``` bash
# install dependencies
npm install
# serve with hot reload at localhost:8080
npm run dev
# build for production with minification
npm run build
# build for production and view the bundle analyzer report
npm run build --report
# run unit tests
npm run unit
# run e2e tests
npm run e2e
# run all tests
npm test
```
For a detailed explanation on how things work, check out the [guide](http://vuejs-templates.github.io/webpack/) and [docs for vue-loader](http://vuejs.github.io/vue-loader).
# Running the API
## First Time
Open the python shell. `python`
```
cd api
python
```
From the shell:
```
>>> from app import db
>>> db.create_all()
```
```
pipenv install
```
## Run API For Real
```
cd api
flask run
```
\ No newline at end of file
[[source]]
name = "pypi"
verify_ssl = true
url = "https://pypi.org/simple"
[packages]
sqlalchemy = "*"
flask = "*"
"psycopg2" = "*"
"psycopg2-binary" = "*"
flask-cors = "*"
gitpython = "*"
markdown = "*"
flask-sqlalchemy = "*"
pypika = "*"
[dev-packages]
[requires]
python_version = "3.7"
This diff is collapsed.
import os
import logging
import datetime
from logging.handlers import RotatingFileHandler
from flask import Flask, request
from flask import jsonify
from flask_sqlalchemy import SQLAlchemy
from flask_cors import CORS
from external_connector import ExternalConnector
app = Flask(__name__)
app.config.from_object('config')
if os.environ['FLASK_ENV'] == 'development':
CORS(app)
db = SQLAlchemy(app)
connector = ExternalConnector()
logger = logging.getLogger('melt_logger')
handler = RotatingFileHandler(app.config['LOG_PATH'], maxBytes=2000, backupCount=10)
logger.addHandler(handler)
now = str(datetime.datetime.utcnow().strftime('%b %d %Y %I:%M:%S:%f'))
logger.warning('Melt started at: {}'.format(now))
@app.before_request
def before_request():
logger.info('[{}] request: {}'.format(request.remote_addr, now))
from controllers import projects
from controllers import repos
from controllers import settings
from controllers import sql
app.register_blueprint(projects.bp)
app.register_blueprint(repos.bp)
app.register_blueprint(settings.bp)
app.register_blueprint(sql.bp)
@app.route("/")
def hello():
return jsonify({"hello": 1})
@app.route("/drop_it_like_its_hot")
def reset_db():
try:
db.drop_all()
except sqlalchemy.exc.OperationalError as err:
logging.error("Failed drop database.")
db.create_all()
return jsonify({"dropped_it":"like_its_hot"})
\ No newline at end of file
import os
def get_env_variable(name):
try:
return os.environ[name]
except KeyError:
message = "Expected environment variable '{}' not set.".format(name)
raise Exception(message)
# the values of those depend on your setup
POSTGRES_URL = get_env_variable("POSTGRES_URL")
POSTGRES_USER = get_env_variable("POSTGRES_USER")
POSTGRES_PASSWORD = get_env_variable("POSTGRES_PASSWORD")
POSTGRES_DB = get_env_variable("POSTGRES_DB")
LOG_PATH = get_env_variable('LOG_PATH')
ENV = 'development'
BASE_DIR = os.path.abspath(os.path.dirname(__file__))
SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{user}:{pw}@{url}/{db}'.format(user=POSTGRES_USER,pw=POSTGRES_PASSWORD,url=POSTGRES_URL,db=POSTGRES_DB)
SQLALCHEMY_TRACK_MODIFICATIONS = False
# SQLALCHEMY_ECHO=True
THREADS_PER_PAGE = 2
SECRET_KEY = "damnitjanice"
\ No newline at end of file
from flask import (
Blueprint, jsonify, request
)
from models.projects import Project
from models.projects import Settings
from app import db
bp = Blueprint('projects', __name__, url_prefix='/projects')
@bp.route('/', methods=['GET'])
def index():
p = Project.query.first()
return jsonify({'name': p.name, 'git_url': p.git_url}) if p else jsonify({'name': '', 'git_url': ''})
@bp.route('/new', methods=['POST'])
def add():
incoming = request.get_json()
name = incoming.get('name')
git_url = incoming.get('git_url')
settings = Settings()
project = Project(name=name, git_url=git_url)
project.settings = settings
db.session.add(settings)
db.session.add(project)
db.session.commit()
return jsonify({'name': name, 'git_url': git_url})
\ No newline at end of file
This diff is collapsed.
from app import db
from flask import (
Blueprint, jsonify, request, config, current_app
)
from models.projects import Settings
bp = Blueprint('settings', __name__, url_prefix='/settings')
@bp.route('/', methods=['GET'])
def index():
settings = Settings.query.first()
return jsonify(settings.serializable())
@bp.route('/new', methods=['POST'])
def new():
settings = request.get_json()
current_settings = Settings.query.first()
current_settings.settings = settings
db.session.add(current_settings)
db.session.commit()
return jsonify(settings)
@bp.route('/connections/<name>/test')
def test(name):
current_settings = Settings.query.first().settings
connections = current_settings['connections']
try:
found_connection = next(connection for connection in connections if connection['name'] == name)
except Exception as e:
found_connection = {}
return jsonify(found_connection)
\ No newline at end of file
import json
import sqlalchemy
import psycopg2
import re
from collections import OrderedDict
from decimal import Decimal
from datetime import date, datetime
from pypika import Query, Table, Field
from app import db
from flask import (
Blueprint, jsonify, request
)
from .utils import SqlHelper
from models.projects import Project, Settings
from models.data import (
Model, Explore, View, Dimension, Measure, Join
)
connections = {}
def default(obj):
if isinstance(obj, Decimal):
return str(obj)
elif isinstance(obj, (datetime, date)):
return obj.isoformat()
raise TypeError("Object of type '%s' is not JSON serialize-able" % type(obj).__name__)
def update_connections():
current_connections = Settings.query.first().settings['connections']
for connection in current_connections:
connection_name = connection['name']
if connection_name not in connections:
this_connection = {}
if connection['dialect'] == 'postgresql':
connection_url = 'postgresql+psycopg2://{user}:{pw}@{host}:{port}/{db}'.format(user=connection['username'],pw=connection['password'],host=connection['host'],port=connection['port'], db=connection['database'])
this_connection['connection_url'] = connection_url
this_connection['engine'] = sqlalchemy.create_engine(this_connection['connection_url'])
connections[connection_name] = this_connection
bp = Blueprint('sql', __name__, url_prefix='/sql')
@bp.route('/', methods=['GET'])
def index():
return jsonify({'result': True})
@bp.route('/get/<model_name>/<explore_name>', methods=['POST'])
def get_sql(model_name, explore_name):
update_connections()
sqlHelper = SqlHelper()
model = Model.query.filter(Model.name == model_name).first()
explore = Explore.query.filter(Explore.name == explore_name).first()
incoming_json = request.get_json()
view_name = incoming_json['view']
limit = incoming_json['limit']
view = View.query.filter(View.name == view_name).first()
incoming_dimensions = incoming_json['dimensions']
incoming_joins = incoming_json['joins']
incoming_dimension_groups = incoming_json['dimension_groups']
incoming_measures = incoming_json['measures']
incoming_filters = incoming_json['filters']
# get all timeframes
timeframes = [t['timeframes'] for t in incoming_dimension_groups]
# flatten list of timeframes
timeframes = [y for x in timeframes for y in x]
group_by = sqlHelper.group_by(incoming_joins, incoming_dimensions, timeframes)
filter_by = sqlHelper.filter_by(incoming_filters, explore_name)
to_run = incoming_json['run']
base_table = view.settings['sql_table_name']
dimensions = filter(lambda x: x.name in incoming_dimensions, view.dimensions)
set_dimensions = dimensions
dimensions = map(lambda x: x.settings['sql'].replace("${TABLE}", explore_name), dimensions)
dimensions = ',\n\t '.join(map(lambda x: '{}'.format(x), dimensions))
dimension_groups = sqlHelper.dimension_groups(view.name, explore_name, incoming_dimension_groups)
measures = filter(lambda x: x.name in incoming_measures, view.measures)
set_measures = list(measures)
measures = filter(lambda x: x.name in incoming_measures, view.measures)
measures = ',\n\t '.join([sqlHelper.get_func(x.name, x.settings['type'], explore_name, x.settings['sql']) for x in measures])
join_dimensions = sqlHelper.join_dimensions(incoming_joins)
join_measures = sqlHelper.join_measures(incoming_joins)
if join_dimensions:
dimensions = join_dimensions
if join_measures:
measures = join_measures
join_sql = sqlHelper.joins_by(incoming_joins, view)
to_join = []
if dimensions:
to_join.append(dimensions)
if dimension_groups:
to_join.append(dimension_groups)
if measures:
to_join.append(measures)
set_dimensions = ([d.settings for d in set_dimensions])
set_measures = ([m.settings for m in set_measures])
measures_as_dict = {}
for settings in set_measures:
new_key = re.sub(r'\$\{[A-Za-z]+\}', explore_name, settings['sql']).rstrip()
measures_as_dict[new_key] = settings
incoming_order = incoming_json['order']
incoming_order_desc = incoming_json['desc']
order_by = 'ORDER BY {}'.format(incoming_order) if incoming_order else ''
if incoming_order_desc:
order_by = '{} DESC'.format(order_by)
base_sql = 'SELECT\n\t{}\nFROM {} AS {} \n{} {} \n{} \n{} \nLIMIT {};'.format(',\n '.join(to_join), base_table, explore_name, filter_by, join_sql, group_by, order_by, limit);
if to_run:
db_to_connect = model.settings['connection']
if not db_to_connect in connections:
return jsonify({'error': True, 'code': 'Missing connection details to {}. Create a connection to {} in the settings.'.format(db_to_connect, db_to_connect)}), 422
engine = connections[model.settings['connection']]['engine']
try:
results = engine.execute(base_sql)
except sqlalchemy.exc.DBAPIError as e:
return jsonify({'error': True, 'code': e.code, 'orig': e.orig.diag.message_primary, 'statement': e.statement}), 422
results = [OrderedDict(row) for row in results]
base_dict = {'sql': base_sql, 'results': results, 'error': False}
if not len(results):
base_dict['empty'] = True
else:
base_dict['empty'] = False
base_dict['keys'] = list(results[0].keys())
base_dict['measures'] = measures_as_dict
return json.dumps(base_dict, default=default)
else:
return json.dumps({'sql': base_sql}, default=default)
@bp.route('/distinct/<model_name>/<explore_name>', methods=['POST'])
def get_distinct_field_name(model_name, explore_name):
update_connections()
incoming_json = request.get_json()
field_name = incoming_json['field'].replace('${TABLE}', explore_name)
model = Model.query.filter(Model.name == model_name).first()
explore = Explore.query.filter(Explore.name == explore_name).first()
base_table = explore.view.settings['sql_table_name']
base_sql = 'SELECT DISTINCT {} FROM {} AS {} ORDER BY {}'.format(field_name, base_table, explore_name, field_name)
engine = connections[model.settings['connection']]['engine']
results = engine.execute(base_sql)
results = [dict(row) for row in results]
return json.dumps({'sql': base_sql, 'results': results, 'keys': list(results[0].keys())}, default=default)
\ No newline at end of file
import re
from sqlalchemy import String, cast
from models.data import View, Dimension, DimensionGroup, Measure, Join
from pypika import Query, Table, Field
class SqlHelper():
def table(self, name, alias):
(schema, name) = name.split('.')
return Table(name, schema=schema, alias=alias)
def field(self, name, table):
table = self.table(table)
return Field(name, table)
def dimension(self, d, table):
d.settings['sql'].replace("${TABLE}", table)
def fields(self, fields, table):
return [self.field(f, table) for f in fields]
def dimension_groups(self, view_name, explore_name, dimension_groups):
base_sqls = []
for dimension_group in dimension_groups:
dimension_group_queried = DimensionGroup.query\
.join(View, DimensionGroup.view_id == View.id)\
.filter(View.name == view_name)\
.filter(DimensionGroup.name == dimension_group['name'])\
.first()
name = dimension_group_queried.table_column_name
for timeframe in dimension_group['timeframes']:
if timeframe == 'date*':
base_sqls.append('DATE({}) AS "{}_date"'\
.format(name, name))
elif timeframe == 'month*':
base_sqls.append('TO_CHAR(DATE_TRUNC(\'month\', {} ), \'YYYY-MM\') AS "{}_month"'\
.format(name, name))
elif timeframe == 'week*':
base_sqls.append('TO_CHAR(DATE_TRUNC(\'week\', {} ), \'YYYY-MM-DD\') AS "{}_week"'\
.format(name, name))
elif timeframe == 'year*':
base_sqls.append('EXTRACT(YEAR FROM {} )::integer AS "{}_year"'\
.format(name, name))
return ',\n'.join(base_sqls)
def get_func(self, name, t, table, sql):
func = t.lower()
print(func)
if func == 'sum':
return self.sum(table, sql)
elif func == 'count':
return self.count(table, sql)
elif func == 'number':
return self.number(name, table, sql)
def number(self, name, table, sql):
replaced_sql = sql.replace('${TABLE}', table);
return '{} AS "{}.{}"'.format(replaced_sql, table, name)
def sum(self, table, sql):
table_name = sql.replace('${TABLE}', table)
return 'COALESCE(SUM({}), 0) AS "{}"'.format(table_name, sql.replace('${TABLE}', table))
def count(self, table, sql):
primary_key = Dimension.query\
.join(View, Dimension.view_id == View.id)\
.filter(cast(Dimension.settings["primary_key"], String) == 'true')\
.filter(View.name == table)\
.first()
table_name = primary_key.settings['sql'].replace('${TABLE}', primary_key.view.name)
return 'COUNT({}) AS "{}.count"'\
.format(table_name, primary_key.view.name)
def group_by(self, joins, dimensions, timeframes):
length = 0
if joins or dimensions or timeframes:
if joins:
for join in joins:
if 'dimensions' in join:
length = length + len(join['dimensions'])
elif dimensions: