Commit 5e91c102 authored by Frank Schoeneman's avatar Frank Schoeneman

Remove old files and directories.

parent 323aed95
This diff is collapsed.
This diff is collapsed.
#!/bin/bash
####### PLANEX SPECIFIC - DO NOT EDIT THIS SECTION
#SBATCH --clusters=mae
#SBATCH --partition=planex
#SBATCH --account=pi-jzola
#SBATCH --exclusive
#SBATCH --mem=64000
####### CUSTOMIZE THIS SECTION FOR YOUR JOB
####### NOTE: --ntasks-per-node SHOULD BE SET TO INCLUDE ALL CORES IN A NODE
####### YOU CAN CONTROL CORE-TO-EXECUTOR RATIO VIA SPARK_ARGS
#SBATCH --job-name="spark"
#SBATCH --nodes=5
#SBATCH --ntasks-per-node=20
#SBATCH --output=%j.stdout
#SBATCH --error=%j.stderr
#SBATCH --time=24:00:00
# IF SET TO 1 SPARK MASTER RUNS ON A SEPARATE NODE
exclude_master=1
# IF SET TO 1 SCRATCH AND TMP WILL BE RM -RF (RECOMMENDED)
nodes_clean=1
# MAKE SURE THAT SPARK_LOG_DIR, SPARK_LOCAL_DIRS
# AND SPARK_WORKER_DIR
# ARE SET IN YOUR BASHRC, FOR EXAMPLE:
# export SPARK_LOG_DIR=/scratch/
# export SPARK_LOCAL_DIRS=/scratch/
# export SPARK_WORKER_DIR=/scratch/
# ADD EXTRA MODULES HERE IF NEEDED
# YOU MAY WANT TO CHANGE SPARK VERSION
module load java/1.8.0_45
module load hadoop/2.6.0
#module load spark/2.2.0
SPARK_HOME=/projects/academic/jzola/fvschoen/spark-2.2.1
module load mkl
source $MKL/bin/mklvars.sh intel64
export LD_LIBRARY_PATH=`(pwd)`:$LD_LIBRARY_PATH
module list
echo $LD_LIBRARY_PATH
# SET YOUR COMMAND AND ARGUMENTS
#PROG="rrpknn_list.py"
PROG="brute_kNN.py"
DATA_PATH=/projects/academic/jzola/fvschoen/Isomap-Spark/PySpark/data/test_50K_1000d.tsv
ARGS="-p 100 -k 10 -n 50000 -d 1000 -f "$DATA_PATH" -o testout"
# SET EXTRA OPTIONS TO spark-submit
# EXAMPLE OPTIONS:
# --num-executors
# --executor-cores
# --executor-memory
# --driver-cores
# --driver-memory
# --py-files
SPARK_ARGS="--conf spark.driver.maxResultSize=20g --conf \"spark.locality.wait=0.5\" --driver-memory 40G --driver-cores 20 --executor-memory 55G --num-executors 4 --executor-cores 20"
####### DO NOT EDIT BELOW
SPARK_PATH=$SPARK_HOME
# GET LIST OF NODES
NODES=(`srun hostname | sort | uniq`)
NUM_NODES=${#NODES[@]}
LAST=$((NUM_NODES - 1))
# FIRST NODE IS MASTER
ssh ${NODES[0]} "cd $SPARK_PATH; ./sbin/start-master.sh"
MASTER="spark://${NODES[0]}:7077"
WHO=`whoami`
echo -e "you can use this:\n ssh $WHO@rush.ccr.buffalo.edu -L 4040:${NODES[0]}:4040 -N\nto enable local dasboard"
TEMP_OUT_DIR=$SLURM_SUBMIT_DIR/$SLURM_JOB_ID-spark
# save history log to spark dir
ARGS=$ARGS" -e "$TEMP_OUT_DIR
# ALL NODES ARE WORKERS
mkdir -p $TEMP_OUT_DIR
for i in `seq $exclude_master $LAST`; do
ssh ${NODES[$i]} "cd $SPARK_PATH; nohup ./bin/spark-class org.apache.spark.deploy.worker.Worker $MASTER &> $TEMP_OUT_DIR/nohup-${NODES[$i]}.$i.out" &
done
# SUBMIT JOB
$SPARK_PATH/bin/spark-submit --master $MASTER $SPARK_ARGS $PROG $ARGS
# CLEAN SPARK JOB
ssh ${NODES[0]} "cd $SPARK_PATH; ./sbin/stop-master.sh"
for i in `seq 0 $LAST`; do
ssh ${NODES[$i]} "killall java"
done
if [ $nodes_clean -eq 1 ]; then
for i in `seq 0 $LAST`; do
ssh ${NODES[$i]} "find /scratch ! -path /scratch/$SLURM_JOB_ID -user $(whoami) -delete; find /tmp ! -path /tmp/$SLURM_JOB_ID -user $(whoami) -delete"
done
fi
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment