Commit 854db744 authored by Jacob Schatz's avatar Jacob Schatz

Merge branch 'pypika-replace' into 'master'

Pypika replace

See merge request meltano/melt!2
parents f9a47087 753e53eb
from enum import Enum
from .substitution import Substitution
from pypika import Field, functions as fn
class AggregateType(Enum):
unknown = 'UNKNOWN'
count = 'count'
sum = 'sum'
class Aggregate():
def __init__(self, measure, table):
sql = measure.settings['sql']
self.substitution = Substitution(sql, table)
self.measure = measure
self.table = table
self.sql = self.substitution.sql
self.aggregateType = AggregateType.unknown
def getAggregateType(self):
type_ = self.measure.settings['type']
if type_ == AggregateType.sum.value:
self.aggregateType = AggregateType.sum
elif type_ == AggregateType.count.value:
self.aggregateType = AggregateType.count
self.aggregateType = AggregateType.unknown
raise Exception('Aggregate Type {} not implemented yet'.format(type_))
def setAggregateSQLSum(self):
self.sql = fn.Coalesce(fn.Sum(self.sql), 0, alias=self.substitution.alias)
def setAggregateSQLCount(self):
self.sql = fn.Coalesce(fn.Count(self.sql), 0, alias=self.substitution.alias)
\ No newline at end of file
......@@ -55,69 +55,13 @@ def get_sql(model_name, explore_name):
model = Model.query.filter( == model_name).first()
explore = Explore.query.filter( == explore_name).first()
incoming_json = request.get_json()
view_name = incoming_json['view']
limit = incoming_json['limit']
view = View.query.filter( == view_name).first()
incoming_dimensions = incoming_json['dimensions']
incoming_joins = incoming_json['joins']
incoming_dimension_groups = incoming_json['dimension_groups']
incoming_measures = incoming_json['measures']
incoming_filters = incoming_json['filters']
# get all timeframes
timeframes = [t['timeframes'] for t in incoming_dimension_groups]
# flatten list of timeframes
timeframes = [y for x in timeframes for y in x]
group_by = sqlHelper.group_by(incoming_joins, incoming_dimensions, timeframes)
filter_by = sqlHelper.filter_by(incoming_filters, explore_name)
to_run = incoming_json['run']
base_table = view.settings['sql_table_name']
dimensions = filter(lambda x: in incoming_dimensions, view.dimensions)
set_dimensions = dimensions
dimensions = map(lambda x: x.settings['sql'].replace("${TABLE}", explore_name), dimensions)
dimensions = ',\n\t '.join(map(lambda x: '{}'.format(x), dimensions))
dimension_groups = sqlHelper.dimension_groups(, explore_name, incoming_dimension_groups)
measures = filter(lambda x: in incoming_measures, view.measures)
set_measures = list(measures)
measures = filter(lambda x: in incoming_measures, view.measures)
measures = ',\n\t '.join([sqlHelper.get_func(, x.settings['type'], explore_name, x.settings['sql']) for x in measures])
join_dimensions = sqlHelper.join_dimensions(incoming_joins)
join_measures = sqlHelper.join_measures(incoming_joins)
if join_dimensions:
dimensions = join_dimensions
if join_measures:
measures = join_measures
join_sql = sqlHelper.joins_by(incoming_joins, view)
to_join = []
if dimensions:
if dimension_groups:
if measures:
set_dimensions = ([d.settings for d in set_dimensions])
set_measures = ([m.settings for m in set_measures])
measures_as_dict = {}
for settings in set_measures:
new_key = re.sub(r'\$\{[A-Za-z]+\}', explore_name, settings['sql']).rstrip()
measures_as_dict[new_key] = settings
outgoing_sql = sqlHelper.get_sql(explore, incoming_json)
incoming_order = incoming_json['order']
incoming_order_desc = incoming_json['desc']
order_by = 'ORDER BY {}'.format(incoming_order) if incoming_order else ''
if incoming_order_desc:
order_by = '{} DESC'.format(order_by)
base_sql = 'SELECT\n\t{}\nFROM {} AS {} \n{} {} \n{} \n{} \nLIMIT {};'.format(',\n '.join(to_join), base_table, explore_name, filter_by, join_sql, group_by, order_by, limit);
if to_run:
db_to_connect = model.settings['connection']
if not db_to_connect in connections:
......@@ -140,7 +84,7 @@ def get_sql(model_name, explore_name):
base_dict['measures'] = measures_as_dict
return json.dumps(base_dict, default=default)
return json.dumps({'sql': base_sql}, default=default)
return json.dumps({'sql': outgoing_sql}, default=default)
@bp.route('/distinct/<model_name>/<explore_name>', methods=['POST'])
def get_distinct_field_name(model_name, explore_name):
import re
from collections import namedtuple
from enum import Enum
from pypika import Field, Case
class SubstitutionType(Enum):
unknown = 'UNKNOWN'
table = 'TABLE'
dimension = 'DIMENSION'
view_dimension = 'VIEW_DIMENSION'
view_sql_table_name = 'VIEW_SQL_TABLE_NAME'
class Substitution():
def __init__(self, _input, table, dimension=None):
self.input = _input
self.alias = None
self.sql = None
self.table = table
if not dimension:
self.type = 'string'
self.type = dimension.settings['type']
self.outer_pattern = r'(\$\{[\w\.]*\})'
self.inner_pattern = r'\$\{([\w\.]*)\}'
self.substitutionType = SubstitutionType.unknown
self.placeholders = self.placeholder_match()
def getSubstitutionType(self):
# trying guess the substitutionType in a cheap way
if '.' in self.input and '${TABLE}' not in self.input:
if 'SQL_TABLE_NAME' in self.input:
self.substitutionType = SubstitutionType.view_sql_table_name
self.substitutionType = SubstitutionType.view_dimension
elif '${TABLE}' in self.input:
self.substitutionType = SubstitutionType.table
elif ' ' not in self.input:
self.substitutionType = SubstitutionType.dimension
self.substitutionType = SubstitutionType.unknown
def placeholder_match(self):
outer_results = re.findall(self.outer_pattern, self.input);
inner_results = re.findall(self.inner_pattern, self.input);
Results = namedtuple('Results', 'inner outer')
return Results(inner=inner_results, outer=outer_results)
def setSql(self):
if self.substitutionType is SubstitutionType.table:
raise Exception('Substitution Type {} not implemented yet'.format(self.substitutionType.value))
def setSqlTableType(self):
self.sql = self.input.replace(self.placeholders.outer[0], self.table._table_name)
(table, field) = self.sql.split('.')
self.alias = self.sql
if self.type == 'yesno':
field = Field(field, table=self.table)
self.sql = Case(alias=self.sql)\
.when(field, 'yes')\
self.sql = Field(field, table=self.table, alias=self.sql)
This diff is collapsed.
......@@ -17,6 +17,9 @@
<a class="navbar-item" href="/">
<a class="navbar-item" href="/extract">
<div class="navbar-item has-dropdown is-hoverable">
<a class="navbar-link" href="#">
......@@ -447,6 +447,10 @@ export default {
<style lang="scss" scoped>
code {
white-space: pre;
word-wrap: break-word;
.panel-block {
position: relative;
&.indented {
......@@ -522,9 +526,6 @@ export default {
padding: 1.5rem;
padding-left: 0;
code {
white-space: pre;
.chart-buttons {
margin-top: -34px;
margin-left: 70px;
......@@ -7,11 +7,36 @@
<ul class="menu-list">
<li><a>Deploy Keys</a></li>
<div class="column section">
<section class="section">
<h2 class="title">Connections</h2>
<h2 class="title">Deploy Keys</h2>
<!-- <h2 class="title">Connections</h2> -->
<div class="columns is-multiline is-mobile">
<div class="column is-half">
<div class="card">
<header class="card-header">
<p class="card-header-title">
GitLab Deploy Key
<div class="card-content">
<div class="content">
<span class="is-pulled-right">GL Deploy</span>
<span class="is-pulled-right">**************</span>
<p v-if="!hasConnections">No Database Connections</p>
<div class="columns is-multiline is-mobile">
<div class="column is-half"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment