Commit 6617e643 authored by Matteo's avatar Matteo 🎸
Browse files

Add calculation logic

parent de6fb252
def calc_spark_configuration(*args, **kwargs):
pass
def calc_executor_cores(available_cores):
"""Calculate optimal number of cores per executor
:param available_cores:
:return:
"""
executor_cores_max = 5
if available_cores >= executor_cores_max:
executor_cores = min(executor_cores_max, available_cores // 2)
else:
executor_cores = max(1, available_cores // 2)
remainder_cores = available_cores % executor_cores
while remainder_cores > 1 and executor_cores > 2:
executor_cores -= 1
remainder_cores = available_cores % executor_cores
return executor_cores
def calc_spark_configuration(form):
"""Calculate an optimal Spark configuration based on the hardware
:param form: a WTF form object, with expected payload as:
- `form.cluster_nodes.data`
- `form.node_cpus.data`
- `form.node_ram.data`
- `form.check_for_cool.data`
:return: A dictionary containing the calculated settings for Spark
"""
n_nodes = form.cluster_nodes.data
n_cores_node = form.node_cpus.data
node_memory = form.node_ram.data
# One core per node is reserved for Yarn/Hadoop daemons
available_cores = n_cores_node - 1
# TODO: This should require a better optimization. 5 is a placeholder
executor_cores = calc_executor_cores(available_cores)
# Total CPUs available for Spark in the cluster
available_cores_total = available_cores * n_nodes
executors_total = available_cores_total / executor_cores
node_executors = executors_total / n_nodes
executor_memory = node_memory / node_executors
# 7% is the default heap overhead value in Spark
heap_overhead_percentage = 0.07
executor_memory = (1 - heap_overhead_percentage) * executor_memory
sparkconf = dict(
executor_instances=int(executors_total),
executor_cores=int(executor_cores),
executor_memory=int(executor_memory),
)
return sparkconf
from flask_wtf import FlaskForm
from wtforms import IntegerField, FloatField, BooleanField, SubmitField
from wtforms.validators import DataRequired
from wtforms.validators import DataRequired, NumberRange
class SparkConfForm(FlaskForm):
cluster_nodes = IntegerField("Number of Nodes", validators=[DataRequired()])
node_cpus = IntegerField("Number of CPUs per Node", validators=[DataRequired()])
node_ram = FloatField("Available RAM per Node (GiB)", validators=[DataRequired()])
cluster_nodes = IntegerField("Number of Nodes", validators=[DataRequired(), NumberRange(1, 2048)])
node_cpus = IntegerField("Number of CPUs per Node", validators=[DataRequired(), NumberRange(1, 1024)])
node_ram = FloatField("Available RAM per Node (GiB)", validators=[DataRequired(), NumberRange(1, 4096)])
check_for_cool = BooleanField("Check this to make the magic happen")
submit = SubmitField("Calculate Configuration")
......@@ -4,40 +4,17 @@ from app.forms import SparkConfForm
from app.configurator import calc_spark_configuration
# @app.route("/")
# @app.route("/index")
# def index():
# return render_template(
# "index.html", title="Spark Configurator", main_header="Spark Configurator"
# )
@app.route("/", methods=["GET", "POST"])
@app.route("/index", methods=["GET", "POST"])
def calculate():
def index():
form = SparkConfForm()
result = None
if form.validate_on_submit():
flash(
"Configuration data successfully sent: {}".format(
(
form.cluster_nodes.data,
form.node_cpus.data,
form.node_ram.data,
form.check_for_cool.data,
)
)
)
# result = calc_spark_configuration(form)
return redirect(url_for("calculate"))
# return render_template(
# "result_output.html",
# form=form,
# result=result,
# )
result = calc_spark_configuration(form)
return render_template(
"form.html",
# title="Spark Configurator",
form=form,
result=result,
)
......
......@@ -2,5 +2,5 @@
{% block app_content %}
<h1>Page Not Found</h1>
<p><a href="{{ url_for('calculate') }}">Back</a></p>
<p><a href="{{ url_for('index') }}">Back</a></p>
{% endblock %}
......@@ -19,11 +19,11 @@
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="{{ url_for('calculate') }}">Spark Configurator</a>
<a class="navbar-brand" href="{{ url_for('index') }}">Home</a>
</div>
<div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
<ul class="nav navbar-nav">
<!-- <li><a href="{{ url_for('calculate') }}">Home</a></li>-->
<!-- <li><a href="{{ url_for('index') }}">Home</a></li>-->
<li><a href="{{ url_for('about') }}">About</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
......
......@@ -4,3 +4,23 @@
{% block form_content %}
{{ wtf.quick_form(form) }}
{% endblock %}
{% block calc_result %}
{% if form.validate_on_submit() %}
<h3>Input data</h3>
<li>Number of nodes: {{ form.cluster_nodes.data }}</li>
<li>CPUs per node: {{ form.node_cpus.data }}</li>
<li>RAM per node: {{ form.node_ram.data }}</li>
<li>Magic requested: {{ form.check_for_cool.data }}</li>
{% endif %}
{% if result %}
<h3>Recommended Configuration</h3>
<div class="well well-lg"><code>
spark.executor.instances: "{{ result.executor_instances }}"
<br>
spark.executor.cores: "{{ result.executor_cores }}"
<br>
spark.executor.memory: "{{ result.executor_memory }}g"
</code></div>
{% endif %}
{% endblock %}
{% extends "form.html" %}
{% block calc_result %}
<li>{{ result.executor_instances }}</li>
<li>{{ result.executor_cores }}</li>
<li>{{ result.executor_memory }}</li>
{% endblock %}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment